From 119b5fcc1338461fd25afa52909a6d7dc0a43eba Mon Sep 17 00:00:00 2001 From: zhujiaxin Date: Thu, 9 Nov 2023 20:27:13 +0800 Subject: [PATCH] kernel.patch Signed-off-by: zhujiaxin --- .../bindings/devfreq/event/rockchip-dfi.txt | 18 +- .../bindings/display/bridge/analogix_dp.txt | 6 + .../bindings/iio/adc/rockchip-saradc.yaml | 1 + .../bindings/iommu/rockchip,iommu.txt | 4 + .../mtd/rockchip,nand-controller.yaml | 161 + .../bindings/nvmem/rockchip-efuse.yaml | 3 + .../bindings/nvmem/rockchip-otp.txt | 2 + .../bindings/phy/phy-rockchip-typec.txt | 35 + .../bindings/power/rockchip-io-domain.txt | 26 +- .../bindings/soc/rockchip/power_domain.txt | 3 + .../devicetree/bindings/sound/rt5651.txt | 2 + .../devicetree/bindings/spi/spi-rockchip.yaml | 1 + .../bindings/thermal/rockchip-thermal.txt | 1 + arch/arm/boot/dts/rk3036.dtsi | 13 +- arch/arm/boot/dts/rk3066a-rayeager.dts | 14 + arch/arm/boot/dts/rk3066a.dtsi | 8 +- arch/arm/boot/dts/rk3188.dtsi | 8 +- arch/arm/boot/dts/rk322x.dtsi | 8 +- arch/arm/boot/dts/rk3288.dtsi | 8 +- arch/arm/boot/dts/rv1108.dtsi | 16 +- arch/arm/kernel/psci_smp.c | 3 +- arch/arm/kernel/reboot.c | 7 +- arch/arm64/Kconfig.platforms | 3 - arch/arm64/Makefile | 13 + arch/arm64/boot/dts/rockchip/Makefile | 40 + .../boot/dts/rockchip/px30-ad-d6-anx6345.dts | 759 + .../px30-ad-r35-mb-rk618-dual-lvds.dts | 147 + .../px30-ad-r35-mb-rk618-hdmi-lvds.dts | 241 + .../rockchip/px30-ad-r35-mb-rk618-hdmi.dts | 105 + .../rockchip/px30-ad-r35-mb-rk618-lvds.dts | 146 + .../boot/dts/rockchip/px30-ad-r35-mb.dtsi | 823 + .../arm64/boot/dts/rockchip/px30-android.dtsi | 131 + .../dts/rockchip/px30-ddr4p416dd6-timing.dtsi | 216 + .../rockchip/px30-dram-default-timing.dtsi | 294 + .../dts/rockchip/px30-evb-ddr3-lvds-v10.dts | 689 + .../dts/rockchip/px30-evb-ddr3-v10-avb.dts | 109 + .../dts/rockchip/px30-evb-ddr3-v10-linux.dts | 939 + .../px30-evb-ddr3-v10-robot-linux.dts | 627 + .../px30-evb-ddr3-v10-robot-no-gpu-linux.dts | 627 + .../boot/dts/rockchip/px30-evb-ddr3-v10.dts | 127 + .../boot/dts/rockchip/px30-evb-ddr3-v10.dtsi | 815 + .../dts/rockchip/px30-evb-ddr3-v11-avb.dts | 276 + .../dts/rockchip/px30-evb-ddr3-v11-linux.dts | 296 + .../boot/dts/rockchip/px30-evb-ddr3-v11.dts | 293 + .../boot/dts/rockchip/px30-evb-ddr4-v10.dts | 853 + .../dts/rockchip/px30-evb-ext-rk618-avb.dts | 11 + .../boot/dts/rockchip/px30-evb-ext-rk618.dts | 33 + .../boot/dts/rockchip/px30-evb-ext-rk618.dtsi | 202 + .../boot/dts/rockchip/px30-robot-no-gpu.dtsi | 57 + arch/arm64/boot/dts/rockchip/px30-robot.dtsi | 93 + .../dts/rockchip/px30-z7-a0-rk618-dsi.dts | 875 + arch/arm64/boot/dts/rockchip/px30.dtsi | 35 +- .../rockchip/rk1808-dram-default-timing.dtsi | 302 + .../boot/dts/rockchip/rk1808-evb-v10.dts | 305 + .../dts/rockchip/rk1808-evb-x4-second.dts | 272 + .../arm64/boot/dts/rockchip/rk1808-evb-x4.dts | 271 + arch/arm64/boot/dts/rockchip/rk1808-evb.dtsi | 717 + arch/arm64/boot/dts/rockchip/rk1808-fpga.dts | 58 + arch/arm64/boot/dts/rockchip/rk1808.dtsi | 3040 + arch/arm64/boot/dts/rockchip/rk1808k.dtsi | 51 + .../boot/dts/rockchip/rk3308-ai-va-v10.dts | 681 + .../boot/dts/rockchip/rk3308-evb-amic-v10.dts | 55 + .../boot/dts/rockchip/rk3308-evb-amic-v11.dts | 56 + .../dts/rockchip/rk3308-evb-dmic-i2s-v10.dts | 101 + .../dts/rockchip/rk3308-evb-dmic-i2s-v11.dts | 77 + .../dts/rockchip/rk3308-evb-dmic-pdm-v10.dts | 88 + .../dts/rockchip/rk3308-evb-dmic-pdm-v11.dts | 92 + .../boot/dts/rockchip/rk3308-evb-ext-v10.dtsi | 235 + .../boot/dts/rockchip/rk3308-evb-v10.dtsi | 780 + .../boot/dts/rockchip/rk3308-evb-v11.dtsi | 835 + arch/arm64/boot/dts/rockchip/rk3308-fpga.dts | 51 + .../arm64/boot/dts/rockchip/rk3308-roc-cc.dts | 4 +- .../rk3308-voice-module-board-v10.dts | 19 + arch/arm64/boot/dts/rockchip/rk3308.dtsi | 40 +- .../dts/rockchip/rk3308b-evb-amic-v10.dts | 62 + .../dts/rockchip/rk3308b-evb-ext-v10.dtsi | 124 + .../boot/dts/rockchip/rk3308b-evb-v10.dtsi | 784 + arch/arm64/boot/dts/rockchip/rk3308k.dtsi | 46 + .../dts/rockchip/rk3326-863-cif-sensor.dtsi | 54 + .../dts/rockchip/rk3326-863-lp3-v10-avb.dts | 112 + .../rockchip/rk3326-863-lp3-v10-rkisp1.dts | 103 + .../boot/dts/rockchip/rk3326-863-lp3-v10.dts | 42 + .../boot/dts/rockchip/rk3326-863-lp3-v10.dtsi | 833 + .../boot/dts/rockchip/rk3326-86v-v10.dts | 840 + .../dts/rockchip/rk3326-evb-ai-va-v10.dts | 1308 + .../rk3326-evb-ai-va-v11-i2s-dmic.dts | 1330 + .../dts/rockchip/rk3326-evb-ai-va-v11.dts | 1317 + .../dts/rockchip/rk3326-evb-ai-va-v12.dts | 1317 + .../dts/rockchip/rk3326-evb-lp3-v10-avb.dts | 91 + .../dts/rockchip/rk3326-evb-lp3-v10-linux.dts | 1024 + .../rk3326-evb-lp3-v10-robot-linux.dts | 748 + .../rk3326-evb-lp3-v10-robot-no-gpu-linux.dts | 728 + .../boot/dts/rockchip/rk3326-evb-lp3-v10.dts | 37 + .../boot/dts/rockchip/rk3326-evb-lp3-v10.dtsi | 887 + .../dts/rockchip/rk3326-evb-lp3-v11-avb.dts | 351 + .../boot/dts/rockchip/rk3326-evb-lp3-v11.dts | 297 + .../arm64/boot/dts/rockchip/rk3326-linux.dtsi | 120 + .../boot/dts/rockchip/rk3328-android.dtsi | 88 + .../dts/rockchip/rk3328-box-liantong-avb.dts | 13 + .../boot/dts/rockchip/rk3328-box-liantong.dts | 34 + .../dts/rockchip/rk3328-box-liantong.dtsi | 673 + .../rockchip/rk3328-box-plus-dram-timing.dtsi | 221 + .../rockchip/rk3328-dram-2layer-timing.dtsi | 257 + .../rockchip/rk3328-dram-default-timing.dtsi | 311 + .../dts/rockchip/rk3328-evb-android-avb.dts | 13 + .../boot/dts/rockchip/rk3328-evb-android.dts | 34 + .../boot/dts/rockchip/rk3328-evb-android.dtsi | 715 + arch/arm64/boot/dts/rockchip/rk3328-evb.dts | 22 +- .../boot/dts/rockchip/rk3328-nanopi-r2s.dts | 4 +- .../arm64/boot/dts/rockchip/rk3328-roc-cc.dts | 4 +- .../rockchip/rk3328-rock64-android-avb.dts | 32 + .../dts/rockchip/rk3328-rock64-android.dts | 34 + .../dts/rockchip/rk3328-rock64-android.dtsi | 612 + .../arm64/boot/dts/rockchip/rk3328-rock64.dts | 14 +- arch/arm64/boot/dts/rockchip/rk3328.dtsi | 38 +- .../boot/dts/rockchip/rk3368-808-evb.dts | 189 + arch/arm64/boot/dts/rockchip/rk3368-808.dtsi | 982 + .../boot/dts/rockchip/rk3368-android.dtsi | 357 + .../boot/dts/rockchip/rk3368-cif-sensor.dtsi | 35 + .../rockchip/rk3368-dram-default-timing.dtsi | 66 + .../arm64/boot/dts/rockchip/rk3368-p9-avb.dts | 35 + arch/arm64/boot/dts/rockchip/rk3368-p9.dts | 34 + arch/arm64/boot/dts/rockchip/rk3368-p9.dtsi | 841 + .../dts/rockchip/rk3368-px5-evb-android.dts | 993 + .../boot/dts/rockchip/rk3368-r88-dcdc.dts | 676 + .../boot/dts/rockchip/rk3368-sheep-lvds.dts | 662 + arch/arm64/boot/dts/rockchip/rk3368-sheep.dts | 746 + .../dts/rockchip/rk3368-sziauto-rk618.dts | 808 + .../arm64/boot/dts/rockchip/rk3368-tablet.dts | 1070 + .../boot/dts/rockchip/rk3368-xikp-avb.dts | 139 + arch/arm64/boot/dts/rockchip/rk3368-xikp.dts | 33 + arch/arm64/boot/dts/rockchip/rk3368-xikp.dtsi | 893 + arch/arm64/boot/dts/rockchip/rk3368.dtsi | 24 +- .../dts/rockchip/rk3368a-817-tablet-bnd.dts | 1074 + .../boot/dts/rockchip/rk3368a-817-tablet.dts | 1333 + .../boot/dts/rockchip/rk3399-android.dtsi | 339 + .../boot/dts/rockchip/rk3399-box-rev1.dts | 134 + .../boot/dts/rockchip/rk3399-box-rev2.dts | 158 + arch/arm64/boot/dts/rockchip/rk3399-box.dtsi | 891 + .../rockchip/rk3399-dram-default-timing.dtsi | 80 + .../boot/dts/rockchip/rk3399-early-opp.dtsi | 127 + .../boot/dts/rockchip/rk3399-evb-cros.dtsi | 181 + .../rk3399-evb-ind-lpddr4-android-avb.dts | 389 + .../rk3399-evb-ind-lpddr4-android.dts | 157 + .../rockchip/rk3399-evb-ind-lpddr4-linux.dts | 321 + .../rk3399-evb-ind-lpddr4-v13-android-avb.dts | 425 + .../boot/dts/rockchip/rk3399-evb-ind.dtsi | 1430 + .../dts/rockchip/rk3399-evb-rev1-android.dts | 145 + .../dts/rockchip/rk3399-evb-rev1-cros.dts | 51 + .../boot/dts/rockchip/rk3399-evb-rev1.dtsi | 352 + .../dts/rockchip/rk3399-evb-rev2-android.dts | 156 + .../dts/rockchip/rk3399-evb-rev2-cros.dts | 51 + .../boot/dts/rockchip/rk3399-evb-rev2.dtsi | 366 + .../rockchip/rk3399-evb-rev3-android-edp.dts | 126 + .../rockchip/rk3399-evb-rev3-android-lp4.dts | 233 + .../rk3399-evb-rev3-android-mipi-edp.dts | 300 + .../dts/rockchip/rk3399-evb-rev3-android.dts | 176 + .../dts/rockchip/rk3399-evb-rev3-cros.dts | 51 + .../boot/dts/rockchip/rk3399-evb-rev3.dtsi | 353 + arch/arm64/boot/dts/rockchip/rk3399-evb.dtsi | 646 + .../rockchip/rk3399-excavator-sapphire.dtsi | 324 + .../dts/rockchip/rk3399-firefly-android.dts | 1102 + .../dts/rockchip/rk3399-firefly-linux.dts | 1074 + arch/arm64/boot/dts/rockchip/rk3399-fpga.dts | 64 + .../boot/dts/rockchip/rk3399-gru-gru.dts | 165 + .../boot/dts/rockchip/rk3399-gru-kevin-r0.dts | 118 + .../boot/dts/rockchip/rk3399-gru-kevin-r1.dts | 85 + .../boot/dts/rockchip/rk3399-gru-scarlet.dtsi | 2 +- arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi | 4 +- .../arm64/boot/dts/rockchip/rk3399-linux.dtsi | 306 + .../dts/rockchip/rk3399-mid-818-android.dts | 1121 + arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi | 304 +- .../boot/dts/rockchip/rk3399-roc-pc.dtsi | 3 - .../boot/dts/rockchip/rk3399-rock960-ab.dts | 1088 + .../boot/dts/rockchip/rk3399-rock960.dts | 53 +- .../rk3399-sapphire-excavator-box.dts | 181 + .../rk3399-sapphire-excavator-edp-avb.dts | 128 + .../rk3399-sapphire-excavator-edp.dts | 82 + .../rk3399-sapphire-excavator-edp.dtsi | 488 + ...ire-excavator-linux-for-rk1808-cascade.dts | 487 + .../rk3399-sapphire-excavator-linux.dts | 452 + .../rk3399-sapphire-excavator-lp4-linux.dts | 497 + .../boot/dts/rockchip/rk3399-sapphire.dtsi | 2 + .../dts/rockchip/rk3399-sched-energy.dtsi | 121 + .../boot/dts/rockchip/rk3399-tve1030g-avb.dts | 170 + .../boot/dts/rockchip/rk3399-tve1030g.dts | 42 + .../boot/dts/rockchip/rk3399-tve1030g.dtsi | 1039 + .../boot/dts/rockchip/rk3399-tve1205g.dts | 1179 + .../dts/rockchip/rk3399-videostrong-linux.dts | 293 + .../boot/dts/rockchip/rk3399-vop-clk-set.dtsi | 181 + arch/arm64/boot/dts/rockchip/rk3399.dtsi | 401 +- arch/arm64/boot/dts/rockchip/rk3399k-opp.dtsi | 24 + .../rockchip/rk3399pro-evb-lp4-v11-linux.dts | 1293 + .../dts/rockchip/rk3399pro-evb-v10-linux.dts | 1203 + .../boot/dts/rockchip/rk3399pro-evb-v10.dts | 1061 + .../dts/rockchip/rk3399pro-evb-v11-linux.dts | 1223 + .../boot/dts/rockchip/rk3399pro-evb-v11.dts | 1045 + .../dts/rockchip/rk3399pro-evb-v14-linux.dts | 247 + .../dts/rockchip/rk3399pro-npu-evb-v10.dts | 140 + .../boot/dts/rockchip/rk3399pro-npu.dtsi | 826 + .../boot/dts/rockchip/rk3566-box-demo-v10.dts | 15 + .../dts/rockchip/rk3566-box-demo-v10.dtsi | 528 + arch/arm64/boot/dts/rockchip/rk3566-box.dtsi | 436 + arch/arm64/boot/dts/rockchip/rk3566-eink.dtsi | 100 + .../dts/rockchip/rk3566-evb-mipitest-v10.dts | 7 + .../dts/rockchip/rk3566-evb-mipitest-v10.dtsi | 507 + arch/arm64/boot/dts/rockchip/rk3566-evb.dtsi | 7 + .../rockchip/rk3566-evb1-ddr4-v10-linux.dts | 13 + .../rockchip/rk3566-evb1-ddr4-v10-lvds.dts | 99 + .../dts/rockchip/rk3566-evb1-ddr4-v10.dts | 7 + .../dts/rockchip/rk3566-evb1-ddr4-v10.dtsi | 489 + .../rockchip/rk3566-evb2-lp4x-v10-eink.dts | 345 + .../rk3566-evb2-lp4x-v10-i2s-mic-array.dts | 102 + .../rockchip/rk3566-evb2-lp4x-v10-linux.dts | 8 + .../rk3566-evb2-lp4x-v10-pdm-mic-array.dts | 111 + .../dts/rockchip/rk3566-evb2-lp4x-v10.dts | 8 + .../dts/rockchip/rk3566-evb2-lp4x-v10.dtsi | 599 + .../rockchip/rk3566-evb3-ddr3-v10-linux.dts | 8 + .../dts/rockchip/rk3566-evb3-ddr3-v10.dts | 8 + .../dts/rockchip/rk3566-evb3-ddr3-v10.dtsi | 499 + .../dts/rockchip/rk3566-evb5-lp4x-v10.dts | 7 + .../dts/rockchip/rk3566-evb5-lp4x-v10.dtsi | 317 + .../dts/rockchip/rk3566-rk817-eink-w103.dts | 1115 + .../dts/rockchip/rk3566-rk817-eink-w6.dts | 968 + .../boot/dts/rockchip/rk3566-rk817-eink.dts | 957 + .../dts/rockchip/rk3566-rk817-tablet-k108.dts | 1307 + .../rockchip/rk3566-rk817-tablet-rkg11.dts | 1180 + .../dts/rockchip/rk3566-rk817-tablet-v10.dts | 1208 + .../boot/dts/rockchip/rk3566-rk817-tablet.dts | 1213 + arch/arm64/boot/dts/rockchip/rk3566.dtsi | 53 + .../boot/dts/rockchip/rk3568-android.dtsi | 74 + .../rockchip/rk3568-dram-default-timing.dtsi | 81 + arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi | 1807 + .../rk3568-evb1-ddr4-v10-linux-spi-nor.dts | 18 + .../rockchip/rk3568-evb1-ddr4-v10-linux.dts | 8 + .../dts/rockchip/rk3568-evb1-ddr4-v10.dts | 8 + .../dts/rockchip/rk3568-evb1-ddr4-v10.dtsi | 490 + .../rk3568-evb2-lp4x-v10-bt1120-to-hdmi.dts | 79 + .../dts/rockchip/rk3568-evb2-lp4x-v10.dts | 8 + .../dts/rockchip/rk3568-evb2-lp4x-v10.dtsi | 493 + .../boot/dts/rockchip/rk3568-evb4-lp3-v10.dts | 12 + .../dts/rockchip/rk3568-evb5-ddr4-v10.dts | 8 + .../dts/rockchip/rk3568-evb5-ddr4-v10.dtsi | 539 + .../rockchip/rk3568-evb6-ddr3-v10-linux.dts | 8 + ...568-evb6-ddr3-v10-rk628-bt1120-to-hdmi.dts | 127 + .../rk3568-evb6-ddr3-v10-rk628-rgb2hdmi.dts | 96 + ...3568-evb6-ddr3-v10-rk630-bt656-to-cvbs.dts | 70 + .../dts/rockchip/rk3568-evb6-ddr3-v10.dts | 8 + .../dts/rockchip/rk3568-evb6-ddr3-v10.dtsi | 490 + .../dts/rockchip/rk3568-evb7-ddr4-v10.dts | 12 + .../rockchip/rk3568-iotest-ddr3-v10-linux.dts | 69 + .../dts/rockchip/rk3568-iotest-ddr3-v10.dts | 47 + .../arm64/boot/dts/rockchip/rk3568-linux.dtsi | 60 + .../rk3568-nvr-demo-v10-linux-spi-nand.dts | 31 + .../rockchip/rk3568-nvr-demo-v10-linux.dts | 27 + .../boot/dts/rockchip/rk3568-nvr-demo-v10.dts | 27 + .../dts/rockchip/rk3568-nvr-demo-v10.dtsi | 442 + .../rk3568-nvr-demo-v12-linux-spi-nand.dts | 20 + .../rockchip/rk3568-nvr-demo-v12-linux.dts | 16 + .../dts/rockchip/rk3568-nvr-demo-v12.dtsi | 39 + .../boot/dts/rockchip/rk3568-nvr-linux.dtsi | 55 + arch/arm64/boot/dts/rockchip/rk3568-nvr.dtsi | 510 + .../boot/dts/rockchip/rk3568-pinctrl.dtsi | 3119 + .../dts/rockchip/rk3568-toybrick-base.dts | 14 + .../dts/rockchip/rk3568-toybrick-edp.dtsi | 141 + .../rk3568-toybrick-mipi-tx0-beiqicloud.dtsi | 395 + .../rockchip/rk3568-toybrick-mipi-tx0.dtsi | 91 + .../rockchip/rk3568-toybrick-mipi-tx1.dtsi | 97 + .../rockchip/rk3568-toybrick-x0-android.dts | 13 + .../rk3568-toybrick-x0-linux-factory.dts | 155 + .../dts/rockchip/rk3568-toybrick-x0-linux.dts | 14 + .../boot/dts/rockchip/rk3568-toybrick-x0.dtsi | 868 + .../rockchip/rk3568-toybrick-x10-linux.dts | 13 + .../dts/rockchip/rk3568-toybrick-x10.dtsi | 816 + .../boot/dts/rockchip/rk3568-toybrick.dtsi | 1891 + arch/arm64/boot/dts/rockchip/rk3568.dtsi | 3449 + .../boot/dts/rockchip/rockchip-pinconf.dtsi | 382 + arch/arm64/include/asm/system_info.h | 14 + arch/arm64/kernel/cpuinfo.c | 6 + arch/arm64/kernel/process.c | 2 + drivers/Kconfig | 3 + drivers/Makefile | 2 + drivers/block/nbd.c | 6 - drivers/clk/Kconfig | 7 + drivers/clk/clk.c | 2 +- drivers/clk/rockchip/Kconfig | 60 +- drivers/clk/rockchip/Makefile | 5 + drivers/clk/rockchip/clk-cpu.c | 92 +- drivers/clk/rockchip/clk-dclk-divider.c | 168 + drivers/clk/rockchip/clk-ddr.c | 171 +- drivers/clk/rockchip/clk-half-divider.c | 35 +- drivers/clk/rockchip/clk-pll.c | 779 +- drivers/clk/rockchip/clk-pvtm.c | 310 + drivers/clk/rockchip/clk-px30.c | 232 +- drivers/clk/rockchip/clk-rk1808.c | 1249 + drivers/clk/rockchip/clk-rk3036.c | 120 +- drivers/clk/rockchip/clk-rk3128.c | 160 +- drivers/clk/rockchip/clk-rk3188.c | 164 +- drivers/clk/rockchip/clk-rk3228.c | 246 +- drivers/clk/rockchip/clk-rk3288.c | 183 +- drivers/clk/rockchip/clk-rk3308.c | 186 +- drivers/clk/rockchip/clk-rk3328.c | 248 +- drivers/clk/rockchip/clk-rk3368.c | 189 +- drivers/clk/rockchip/clk-rk3399.c | 589 +- drivers/clk/rockchip/clk-rk3568.c | 1757 + drivers/clk/rockchip/clk-rv1108.c | 131 +- drivers/clk/rockchip/clk-rv1126.c | 1586 + drivers/clk/rockchip/clk.c | 200 +- drivers/clk/rockchip/clk.h | 358 +- drivers/clk/rockchip/regmap/Kconfig | 16 + drivers/clk/rockchip/regmap/Makefile | 13 + .../rockchip/regmap/clk-regmap-composite.c | 400 + .../clk/rockchip/regmap/clk-regmap-divider.c | 104 + .../regmap/clk-regmap-fractional-divider.c | 157 + drivers/clk/rockchip/regmap/clk-regmap-gate.c | 82 + drivers/clk/rockchip/regmap/clk-regmap-mux.c | 79 + drivers/clk/rockchip/regmap/clk-regmap-pll.c | 363 + drivers/clk/rockchip/regmap/clk-regmap.h | 311 + drivers/clk/rockchip/regmap/clk-rk618.c | 408 + drivers/clk/rockchip/regmap/clk-rk628.c | 569 + drivers/clocksource/Kconfig | 4 +- drivers/clocksource/timer-rockchip.c | 33 + drivers/cpufreq/Kconfig.arm | 10 + drivers/cpufreq/Makefile | 1 + drivers/cpufreq/cpufreq-dt-platdev.c | 37 +- drivers/cpufreq/cpufreq-dt.c | 168 +- drivers/cpufreq/cpufreq.c | 7 +- drivers/cpufreq/cpufreq_userspace.c | 6 +- drivers/cpufreq/rockchip-cpufreq.c | 638 + drivers/cpufreq/rockchip-cpufreq.h | 24 + drivers/cpuidle/driver.c | 1 + drivers/cpuidle/governor.c | 2 + drivers/devfreq/Kconfig | 13 +- drivers/devfreq/Makefile | 3 +- drivers/devfreq/devfreq.c | 35 + drivers/devfreq/event/Kconfig | 7 + drivers/devfreq/event/Makefile | 1 + drivers/devfreq/event/rockchip-dfi.c | 563 +- drivers/devfreq/event/rockchip-nocp.c | 166 + drivers/devfreq/rk3399_dmc.c | 518 - drivers/devfreq/rockchip_bus.c | 500 + drivers/devfreq/rockchip_dmc.c | 3283 + drivers/devfreq/rockchip_dmc_common.c | 168 + drivers/devfreq/rockchip_dmc_dbg.c | 1061 + drivers/devfreq/rockchip_dmc_timing.h | 1307 + drivers/dma-buf/Kconfig | 1 - drivers/dma-buf/dma-buf.c | 144 +- drivers/dma-buf/dma-fence.c | 70 +- drivers/dma-buf/dma-heap.c | 223 +- drivers/dma-buf/heaps/Kconfig | 16 +- drivers/dma-buf/heaps/Makefile | 3 +- drivers/dma-buf/heaps/cma_heap.c | 336 +- drivers/dma-buf/heaps/deferred-free-helper.c | 138 + drivers/dma-buf/heaps/deferred-free-helper.h | 55 + drivers/dma-buf/heaps/heap-helpers.c | 271 - drivers/dma-buf/heaps/heap-helpers.h | 53 - drivers/dma-buf/heaps/page_pool.c | 247 + drivers/dma-buf/heaps/page_pool.h | 55 + drivers/dma-buf/heaps/system_heap.c | 575 +- drivers/dma-buf/sw_sync.c | 12 + drivers/dma-buf/sync_debug.c | 2 + drivers/dma-buf/sync_debug.h | 7 + drivers/firmware/Kconfig | 9 +- drivers/firmware/Makefile | 1 + drivers/firmware/rockchip_sip.c | 578 + drivers/gpio/Kconfig | 8 + drivers/gpio/Makefile | 1 + drivers/gpio/gpio-rockchip.c | 746 + drivers/gpio/gpiolib-of.c | 11 + drivers/gpio/gpiolib-of.h | 5 + drivers/gpu/Makefile | 2 +- drivers/gpu/arm/Kbuild | 31 + drivers/gpu/arm/Kconfig | 48 + drivers/gpu/arm/bifrost/Kbuild | 229 + drivers/gpu/arm/bifrost/Kconfig | 308 + drivers/gpu/arm/bifrost/Makefile | 38 + drivers/gpu/arm/bifrost/Makefile.kbase | 23 + drivers/gpu/arm/bifrost/Mconfig | 277 + drivers/gpu/arm/bifrost/arbiter/Kbuild | 24 + .../arm/bifrost/arbiter/mali_kbase_arbif.c | 175 + .../arm/bifrost/arbiter/mali_kbase_arbif.h | 133 + .../bifrost/arbiter/mali_kbase_arbiter_defs.h | 95 + .../arbiter/mali_kbase_arbiter_interface.h | 181 + .../bifrost/arbiter/mali_kbase_arbiter_pm.c | 676 + .../bifrost/arbiter/mali_kbase_arbiter_pm.h | 159 + drivers/gpu/arm/bifrost/backend/gpu/Kbuild | 65 + .../backend/gpu/mali_kbase_backend_config.h | 31 + .../gpu/mali_kbase_cache_policy_backend.c | 34 + .../gpu/mali_kbase_cache_policy_backend.h | 39 + .../gpu/mali_kbase_clk_rate_trace_mgr.c | 287 + .../gpu/mali_kbase_clk_rate_trace_mgr.h | 155 + .../gpu/mali_kbase_debug_job_fault_backend.c | 164 + .../bifrost/backend/gpu/mali_kbase_devfreq.c | 847 + .../bifrost/backend/gpu/mali_kbase_devfreq.h | 47 + .../backend/gpu/mali_kbase_gpuprops_backend.c | 158 + .../backend/gpu/mali_kbase_instr_backend.c | 520 + .../backend/gpu/mali_kbase_instr_defs.h | 63 + .../backend/gpu/mali_kbase_instr_internal.h | 44 + .../backend/gpu/mali_kbase_irq_internal.h | 44 + .../backend/gpu/mali_kbase_irq_linux.c | 504 + .../bifrost/backend/gpu/mali_kbase_jm_as.c | 245 + .../bifrost/backend/gpu/mali_kbase_jm_defs.h | 113 + .../bifrost/backend/gpu/mali_kbase_jm_hw.c | 1462 + .../backend/gpu/mali_kbase_jm_internal.h | 181 + .../bifrost/backend/gpu/mali_kbase_jm_rb.c | 1659 + .../bifrost/backend/gpu/mali_kbase_jm_rb.h | 83 + .../backend/gpu/mali_kbase_js_backend.c | 365 + .../backend/gpu/mali_kbase_js_internal.h | 74 + .../backend/gpu/mali_kbase_l2_mmu_config.c | 133 + .../backend/gpu/mali_kbase_l2_mmu_config.h | 55 + .../backend/gpu/mali_kbase_pm_always_on.c | 67 + .../backend/gpu/mali_kbase_pm_always_on.h | 81 + .../backend/gpu/mali_kbase_pm_backend.c | 788 + .../bifrost/backend/gpu/mali_kbase_pm_ca.c | 114 + .../bifrost/backend/gpu/mali_kbase_pm_ca.h | 89 + .../backend/gpu/mali_kbase_pm_ca_devfreq.h | 60 + .../backend/gpu/mali_kbase_pm_coarse_demand.c | 66 + .../backend/gpu/mali_kbase_pm_coarse_demand.h | 69 + .../bifrost/backend/gpu/mali_kbase_pm_defs.h | 560 + .../backend/gpu/mali_kbase_pm_driver.c | 2545 + .../backend/gpu/mali_kbase_pm_internal.h | 739 + .../backend/gpu/mali_kbase_pm_l2_states.h | 38 + .../backend/gpu/mali_kbase_pm_mcu_states.h | 39 + .../backend/gpu/mali_kbase_pm_metrics.c | 324 + .../backend/gpu/mali_kbase_pm_policy.c | 268 + .../backend/gpu/mali_kbase_pm_policy.h | 106 + .../backend/gpu/mali_kbase_pm_shader_states.h | 45 + .../arm/bifrost/backend/gpu/mali_kbase_time.c | 81 + drivers/gpu/arm/bifrost/build.bp | 186 + .../context/backend/mali_kbase_context_csf.c | 177 + .../context/backend/mali_kbase_context_jm.c | 230 + .../arm/bifrost/context/mali_kbase_context.c | 339 + .../arm/bifrost/context/mali_kbase_context.h | 157 + .../context/mali_kbase_context_internal.h | 60 + drivers/gpu/arm/bifrost/csf/Kbuild | 40 + .../arm/bifrost/csf/mali_base_csf_kernel.h | 598 + .../csf/mali_gpu_csf_control_registers.h | 33 + .../arm/bifrost/csf/mali_gpu_csf_registers.h | 1252 + drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c | 2547 + drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h | 444 + .../bifrost/csf/mali_kbase_csf_csg_debugfs.c | 460 + .../bifrost/csf/mali_kbase_csf_csg_debugfs.h | 48 + .../gpu/arm/bifrost/csf/mali_kbase_csf_defs.h | 883 + .../arm/bifrost/csf/mali_kbase_csf_firmware.c | 1993 + .../arm/bifrost/csf/mali_kbase_csf_firmware.h | 663 + .../bifrost/csf/mali_kbase_csf_firmware_cfg.c | 306 + .../bifrost/csf/mali_kbase_csf_firmware_cfg.h | 72 + .../csf/mali_kbase_csf_firmware_no_mali.c | 1012 + .../csf/mali_kbase_csf_heap_context_alloc.c | 196 + .../csf/mali_kbase_csf_heap_context_alloc.h | 76 + .../arm/bifrost/csf/mali_kbase_csf_ioctl.h | 379 + .../gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c | 1737 + .../gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h | 305 + .../bifrost/csf/mali_kbase_csf_kcpu_debugfs.c | 199 + .../bifrost/csf/mali_kbase_csf_kcpu_debugfs.h | 38 + .../csf/mali_kbase_csf_protected_memory.c | 120 + .../csf/mali_kbase_csf_protected_memory.h | 72 + .../bifrost/csf/mali_kbase_csf_reset_gpu.c | 355 + .../bifrost/csf/mali_kbase_csf_scheduler.c | 4135 ++ .../bifrost/csf/mali_kbase_csf_scheduler.h | 408 + .../bifrost/csf/mali_kbase_csf_tiler_heap.c | 584 + .../bifrost/csf/mali_kbase_csf_tiler_heap.h | 113 + .../csf/mali_kbase_csf_tiler_heap_debugfs.c | 107 + .../csf/mali_kbase_csf_tiler_heap_debugfs.h | 38 + .../csf/mali_kbase_csf_tiler_heap_def.h | 112 + .../arm/bifrost/csf/mali_kbase_csf_timeout.c | 169 + .../arm/bifrost/csf/mali_kbase_csf_timeout.h | 69 + .../bifrost/csf/mali_kbase_csf_tl_reader.c | 555 + .../bifrost/csf/mali_kbase_csf_tl_reader.h | 181 + .../bifrost/csf/mali_kbase_csf_trace_buffer.c | 623 + .../bifrost/csf/mali_kbase_csf_trace_buffer.h | 177 + .../mali_kbase_debug_ktrace_codes_csf.h | 116 + .../mali_kbase_debug_ktrace_codes_jm.h | 173 + .../backend/mali_kbase_debug_ktrace_csf.c | 143 + .../backend/mali_kbase_debug_ktrace_csf.h | 148 + .../mali_kbase_debug_ktrace_defs_csf.h | 85 + .../backend/mali_kbase_debug_ktrace_defs_jm.h | 102 + .../backend/mali_kbase_debug_ktrace_jm.c | 115 + .../backend/mali_kbase_debug_ktrace_jm.h | 309 + .../mali_kbase_debug_linux_ktrace_csf.h | 147 + .../mali_kbase_debug_linux_ktrace_jm.h | 165 + .../bifrost/debug/mali_kbase_debug_ktrace.c | 356 + .../bifrost/debug/mali_kbase_debug_ktrace.h | 226 + .../debug/mali_kbase_debug_ktrace_codes.h | 165 + .../debug/mali_kbase_debug_ktrace_defs.h | 183 + .../debug/mali_kbase_debug_ktrace_internal.h | 89 + .../debug/mali_kbase_debug_linux_ktrace.h | 111 + .../device/backend/mali_kbase_device_csf.c | 274 + .../device/backend/mali_kbase_device_hw_csf.c | 161 + .../device/backend/mali_kbase_device_hw_jm.c | 100 + .../device/backend/mali_kbase_device_jm.c | 264 + .../arm/bifrost/device/mali_kbase_device.c | 411 + .../arm/bifrost/device/mali_kbase_device.h | 177 + .../arm/bifrost/device/mali_kbase_device_hw.c | 184 + .../device/mali_kbase_device_internal.h | 78 + .../gpu/backend/mali_kbase_gpu_fault_csf.c | 105 + .../gpu/backend/mali_kbase_gpu_fault_jm.c | 177 + .../gpu/backend/mali_kbase_gpu_regmap_csf.h | 297 + .../gpu/backend/mali_kbase_gpu_regmap_jm.h | 288 + drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.c | 41 + drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.h | 31 + .../bifrost/gpu/mali_kbase_gpu_coherency.h | 31 + .../arm/bifrost/gpu/mali_kbase_gpu_fault.h | 48 + .../gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h | 119 + .../arm/bifrost/gpu/mali_kbase_gpu_regmap.h | 428 + drivers/gpu/arm/bifrost/ipa/Kbuild | 28 + drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c | 672 + drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h | 254 + .../arm/bifrost/ipa/mali_kbase_ipa_debugfs.c | 322 + .../arm/bifrost/ipa/mali_kbase_ipa_debugfs.h | 68 + .../arm/bifrost/ipa/mali_kbase_ipa_simple.c | 356 + .../arm/bifrost/ipa/mali_kbase_ipa_simple.h | 45 + .../ipa/mali_kbase_ipa_vinstr_common.c | 349 + .../ipa/mali_kbase_ipa_vinstr_common.h | 217 + .../bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c | 490 + .../gpu/arm/bifrost/jm/mali_base_jm_kernel.h | 1079 + .../gpu/arm/bifrost/jm/mali_kbase_jm_defs.h | 844 + .../gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h | 216 + drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h | 892 + .../gpu/arm/bifrost/jm/mali_kbase_js_defs.h | 409 + .../arm/bifrost/mali_base_hwconfig_features.h | 515 + .../arm/bifrost/mali_base_hwconfig_issues.h | 684 + drivers/gpu/arm/bifrost/mali_base_kernel.h | 807 + drivers/gpu/arm/bifrost/mali_base_mem_priv.h | 57 + drivers/gpu/arm/bifrost/mali_kbase.h | 614 + .../arm/bifrost/mali_kbase_as_fault_debugfs.c | 113 + .../arm/bifrost/mali_kbase_as_fault_debugfs.h | 50 + drivers/gpu/arm/bifrost/mali_kbase_bits.h | 41 + .../gpu/arm/bifrost/mali_kbase_cache_policy.c | 67 + .../gpu/arm/bifrost/mali_kbase_cache_policy.h | 50 + drivers/gpu/arm/bifrost/mali_kbase_caps.h | 65 + drivers/gpu/arm/bifrost/mali_kbase_ccswe.c | 105 + drivers/gpu/arm/bifrost/mali_kbase_ccswe.h | 97 + drivers/gpu/arm/bifrost/mali_kbase_config.c | 48 + drivers/gpu/arm/bifrost/mali_kbase_config.h | 393 + .../arm/bifrost/mali_kbase_config_defaults.h | 213 + .../gpu/arm/bifrost/mali_kbase_core_linux.c | 5001 ++ .../arm/bifrost/mali_kbase_cs_experimental.h | 51 + .../gpu/arm/bifrost/mali_kbase_ctx_sched.c | 355 + .../gpu/arm/bifrost/mali_kbase_ctx_sched.h | 209 + drivers/gpu/arm/bifrost/mali_kbase_debug.c | 44 + drivers/gpu/arm/bifrost/mali_kbase_debug.h | 169 + .../arm/bifrost/mali_kbase_debug_job_fault.c | 566 + .../arm/bifrost/mali_kbase_debug_job_fault.h | 116 + .../arm/bifrost/mali_kbase_debug_mem_view.c | 313 + .../arm/bifrost/mali_kbase_debug_mem_view.h | 40 + .../arm/bifrost/mali_kbase_debugfs_helper.c | 183 + .../arm/bifrost/mali_kbase_debugfs_helper.h | 141 + drivers/gpu/arm/bifrost/mali_kbase_defs.h | 1807 + .../arm/bifrost/mali_kbase_disjoint_events.c | 81 + .../gpu/arm/bifrost/mali_kbase_dma_fence.c | 473 + .../gpu/arm/bifrost/mali_kbase_dma_fence.h | 144 + .../gpu/arm/bifrost/mali_kbase_dummy_job_wa.c | 442 + .../gpu/arm/bifrost/mali_kbase_dummy_job_wa.h | 74 + drivers/gpu/arm/bifrost/mali_kbase_event.c | 274 + drivers/gpu/arm/bifrost/mali_kbase_fence.c | 154 + drivers/gpu/arm/bifrost/mali_kbase_fence.h | 284 + .../gpu/arm/bifrost/mali_kbase_fence_defs.h | 64 + .../gpu/arm/bifrost/mali_kbase_fence_ops.c | 84 + drivers/gpu/arm/bifrost/mali_kbase_gator.h | 53 + .../bifrost/mali_kbase_gpu_memory_debugfs.c | 104 + .../bifrost/mali_kbase_gpu_memory_debugfs.h | 54 + drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c | 649 + drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h | 135 + .../arm/bifrost/mali_kbase_gpuprops_types.h | 98 + drivers/gpu/arm/bifrost/mali_kbase_gwt.c | 273 + drivers/gpu/arm/bifrost/mali_kbase_gwt.h | 55 + drivers/gpu/arm/bifrost/mali_kbase_hw.c | 437 + drivers/gpu/arm/bifrost/mali_kbase_hw.h | 70 + .../arm/bifrost/mali_kbase_hwaccess_backend.h | 45 + .../arm/bifrost/mali_kbase_hwaccess_defs.h | 51 + .../bifrost/mali_kbase_hwaccess_gpuprops.h | 87 + .../arm/bifrost/mali_kbase_hwaccess_instr.h | 151 + .../gpu/arm/bifrost/mali_kbase_hwaccess_jm.h | 304 + .../gpu/arm/bifrost/mali_kbase_hwaccess_pm.h | 229 + .../arm/bifrost/mali_kbase_hwaccess_time.h | 56 + drivers/gpu/arm/bifrost/mali_kbase_hwcnt.c | 794 + .../bifrost/mali_kbase_hwcnt_accumulator.h | 146 + .../arm/bifrost/mali_kbase_hwcnt_backend.h | 220 + .../arm/bifrost/mali_kbase_hwcnt_backend_jm.c | 736 + .../arm/bifrost/mali_kbase_hwcnt_backend_jm.h | 61 + .../arm/bifrost/mali_kbase_hwcnt_context.h | 119 + .../gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c | 571 + .../gpu/arm/bifrost/mali_kbase_hwcnt_gpu.h | 217 + .../gpu/arm/bifrost/mali_kbase_hwcnt_legacy.c | 152 + .../gpu/arm/bifrost/mali_kbase_hwcnt_legacy.h | 94 + .../gpu/arm/bifrost/mali_kbase_hwcnt_reader.h | 106 + .../gpu/arm/bifrost/mali_kbase_hwcnt_types.c | 604 + .../gpu/arm/bifrost/mali_kbase_hwcnt_types.h | 1142 + .../bifrost/mali_kbase_hwcnt_virtualizer.c | 790 + .../bifrost/mali_kbase_hwcnt_virtualizer.h | 145 + drivers/gpu/arm/bifrost/mali_kbase_ioctl.h | 838 + drivers/gpu/arm/bifrost/mali_kbase_jd.c | 1819 + .../gpu/arm/bifrost/mali_kbase_jd_debugfs.c | 250 + .../gpu/arm/bifrost/mali_kbase_jd_debugfs.h | 45 + drivers/gpu/arm/bifrost/mali_kbase_jm.c | 155 + drivers/gpu/arm/bifrost/mali_kbase_jm.h | 119 + drivers/gpu/arm/bifrost/mali_kbase_js.c | 3760 ++ drivers/gpu/arm/bifrost/mali_kbase_js.h | 40 + .../gpu/arm/bifrost/mali_kbase_js_ctx_attr.c | 283 + .../gpu/arm/bifrost/mali_kbase_js_ctx_attr.h | 155 + .../gpu/arm/bifrost/mali_kbase_kinstr_jm.c | 895 + .../gpu/arm/bifrost/mali_kbase_kinstr_jm.h | 283 + .../arm/bifrost/mali_kbase_kinstr_jm_reader.h | 70 + drivers/gpu/arm/bifrost/mali_kbase_linux.h | 48 + drivers/gpu/arm/bifrost/mali_kbase_mem.c | 4734 ++ drivers/gpu/arm/bifrost/mali_kbase_mem.h | 1962 + .../gpu/arm/bifrost/mali_kbase_mem_linux.c | 3425 + .../gpu/arm/bifrost/mali_kbase_mem_linux.h | 478 + .../gpu/arm/bifrost/mali_kbase_mem_lowlevel.h | 166 + drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c | 856 + .../arm/bifrost/mali_kbase_mem_pool_debugfs.c | 191 + .../arm/bifrost/mali_kbase_mem_pool_debugfs.h | 123 + .../arm/bifrost/mali_kbase_mem_pool_group.c | 115 + .../arm/bifrost/mali_kbase_mem_pool_group.h | 92 + .../bifrost/mali_kbase_mem_profile_debugfs.c | 134 + .../bifrost/mali_kbase_mem_profile_debugfs.h | 64 + .../mali_kbase_mem_profile_debugfs_buf_size.h | 39 + .../arm/bifrost/mali_kbase_mipe_gen_header.h | 219 + .../gpu/arm/bifrost/mali_kbase_mipe_proto.h | 127 + .../gpu/arm/bifrost/mali_kbase_native_mgm.c | 153 + .../gpu/arm/bifrost/mali_kbase_native_mgm.h | 39 + .../arm/bifrost/mali_kbase_platform_fake.c | 124 + drivers/gpu/arm/bifrost/mali_kbase_pm.c | 292 + drivers/gpu/arm/bifrost/mali_kbase_pm.h | 251 + .../bifrost/mali_kbase_regs_history_debugfs.c | 245 + .../bifrost/mali_kbase_regs_history_debugfs.h | 85 + .../gpu/arm/bifrost/mali_kbase_reset_gpu.h | 129 + drivers/gpu/arm/bifrost/mali_kbase_smc.c | 91 + drivers/gpu/arm/bifrost/mali_kbase_smc.h | 72 + drivers/gpu/arm/bifrost/mali_kbase_softjobs.c | 1805 + drivers/gpu/arm/bifrost/mali_kbase_strings.c | 28 + drivers/gpu/arm/bifrost/mali_kbase_strings.h | 24 + drivers/gpu/arm/bifrost/mali_kbase_sync.h | 231 + .../gpu/arm/bifrost/mali_kbase_sync_android.c | 542 + .../gpu/arm/bifrost/mali_kbase_sync_common.c | 51 + .../gpu/arm/bifrost/mali_kbase_sync_file.c | 372 + .../arm/bifrost/mali_kbase_trace_gpu_mem.c | 227 + .../arm/bifrost/mali_kbase_trace_gpu_mem.h | 103 + drivers/gpu/arm/bifrost/mali_kbase_utility.h | 55 + drivers/gpu/arm/bifrost/mali_kbase_vinstr.c | 1083 + drivers/gpu/arm/bifrost/mali_kbase_vinstr.h | 91 + drivers/gpu/arm/bifrost/mali_linux_trace.h | 552 + drivers/gpu/arm/bifrost/mali_malisw.h | 109 + .../bifrost/mali_power_gpu_frequency_trace.c | 27 + .../bifrost/mali_power_gpu_frequency_trace.h | 69 + drivers/gpu/arm/bifrost/mali_uk.h | 84 + .../bifrost/mmu/backend/mali_kbase_mmu_csf.c | 532 + .../bifrost/mmu/backend/mali_kbase_mmu_jm.c | 440 + drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c | 2275 + drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h | 156 + .../gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h | 107 + .../bifrost/mmu/mali_kbase_mmu_hw_direct.c | 274 + .../arm/bifrost/mmu/mali_kbase_mmu_internal.h | 73 + .../bifrost/mmu/mali_kbase_mmu_mode_aarch64.c | 200 + .../bifrost/mmu/mali_kbase_mmu_mode_lpae.c | 215 + drivers/gpu/arm/bifrost/platform/Kconfig | 30 + .../arm/bifrost/platform/devicetree/Kbuild | 25 + .../devicetree/mali_kbase_clk_rate_trace.c | 68 + .../devicetree/mali_kbase_config_devicetree.c | 41 + .../devicetree/mali_kbase_config_platform.h | 49 + .../devicetree/mali_kbase_runtime_pm.c | 185 + drivers/gpu/arm/bifrost/platform/rk/Kbuild | 17 + .../gpu/arm/bifrost/platform/rk/custom_log.h | 192 + .../platform/rk/mali_kbase_config_platform.h | 88 + .../platform/rk/mali_kbase_config_rk.c | 459 + .../arm/bifrost/platform/rk/mali_kbase_rk.h | 62 + .../gpu/arm/bifrost/platform/vexpress/Kbuild | 24 + .../vexpress/mali_kbase_config_platform.h | 39 + .../vexpress/mali_kbase_config_vexpress.c | 69 + .../bifrost/platform/vexpress_1xv7_a57/Kbuild | 24 + .../mali_kbase_config_platform.h | 39 + .../mali_kbase_config_vexpress.c | 65 + .../platform/vexpress_6xvirtex7_10mhz/Kbuild | 25 + .../mali_kbase_config_platform.h | 39 + .../mali_kbase_config_vexpress.c | 67 + .../gpu/arm/bifrost/protected_mode_switcher.h | 69 + drivers/gpu/arm/bifrost/tests/Kbuild | 24 + drivers/gpu/arm/bifrost/tests/Kconfig | 24 + drivers/gpu/arm/bifrost/tests/Mconfig | 38 + .../bifrost/tests/include/kutf/kutf_helpers.h | 85 + .../tests/include/kutf/kutf_helpers_user.h | 179 + .../arm/bifrost/tests/include/kutf/kutf_mem.h | 73 + .../tests/include/kutf/kutf_resultset.h | 181 + .../bifrost/tests/include/kutf/kutf_suite.h | 569 + .../bifrost/tests/include/kutf/kutf_utils.h | 60 + drivers/gpu/arm/bifrost/tests/kutf/Kbuild | 26 + drivers/gpu/arm/bifrost/tests/kutf/Kconfig | 28 + drivers/gpu/arm/bifrost/tests/kutf/Makefile | 35 + drivers/gpu/arm/bifrost/tests/kutf/build.bp | 36 + .../gpu/arm/bifrost/tests/kutf/kutf_helpers.c | 131 + .../bifrost/tests/kutf/kutf_helpers_user.c | 468 + drivers/gpu/arm/bifrost/tests/kutf/kutf_mem.c | 108 + .../arm/bifrost/tests/kutf/kutf_resultset.c | 164 + .../gpu/arm/bifrost/tests/kutf/kutf_suite.c | 1224 + .../gpu/arm/bifrost/tests/kutf/kutf_utils.c | 76 + .../mali_kutf_clk_rate_trace/kernel/Kbuild | 26 + .../mali_kutf_clk_rate_trace/kernel/Kconfig | 30 + .../mali_kutf_clk_rate_trace/kernel/Makefile | 57 + .../mali_kutf_clk_rate_trace/kernel/build.bp | 34 + .../kernel/mali_kutf_clk_rate_trace_test.c | 890 + .../mali_kutf_clk_rate_trace_test.h | 148 + .../bifrost/tests/mali_kutf_irq_test/Kbuild | 26 + .../bifrost/tests/mali_kutf_irq_test/Kconfig | 29 + .../bifrost/tests/mali_kutf_irq_test/Makefile | 51 + .../bifrost/tests/mali_kutf_irq_test/build.bp | 35 + .../mali_kutf_irq_test_main.c | 278 + .../arm/bifrost/thirdparty/mali_kbase_mmap.c | 368 + .../tl/backend/mali_kbase_timeline_csf.c | 172 + .../tl/backend/mali_kbase_timeline_jm.c | 97 + .../gpu/arm/bifrost/tl/mali_kbase_timeline.c | 308 + .../gpu/arm/bifrost/tl/mali_kbase_timeline.h | 121 + .../arm/bifrost/tl/mali_kbase_timeline_io.c | 362 + .../arm/bifrost/tl/mali_kbase_timeline_priv.h | 73 + .../arm/bifrost/tl/mali_kbase_tl_serialize.h | 125 + .../gpu/arm/bifrost/tl/mali_kbase_tlstream.c | 306 + .../gpu/arm/bifrost/tl/mali_kbase_tlstream.h | 169 + .../arm/bifrost/tl/mali_kbase_tracepoints.c | 2974 + .../arm/bifrost/tl/mali_kbase_tracepoints.h | 2926 + drivers/gpu/arm/bifrost_for_linux/Kbuild | 172 + drivers/gpu/arm/bifrost_for_linux/Kconfig | 196 + drivers/gpu/arm/bifrost_for_linux/Makefile | 42 + .../gpu/arm/bifrost_for_linux/Makefile.kbase | 17 + .../arm/bifrost_for_linux/backend/gpu/Kbuild | 60 + .../backend/gpu/mali_kbase_backend_config.h | 29 + .../gpu/mali_kbase_cache_policy_backend.c | 29 + .../gpu/mali_kbase_cache_policy_backend.h | 34 + .../gpu/mali_kbase_debug_job_fault_backend.c | 157 + .../backend/gpu/mali_kbase_devfreq.c | 495 + .../backend/gpu/mali_kbase_devfreq.h | 25 + .../backend/gpu/mali_kbase_device_hw.c | 255 + .../backend/gpu/mali_kbase_device_internal.h | 67 + .../backend/gpu/mali_kbase_gpu.c | 123 + .../backend/gpu/mali_kbase_gpuprops_backend.c | 110 + .../backend/gpu/mali_kbase_instr_backend.c | 492 + .../backend/gpu/mali_kbase_instr_defs.h | 58 + .../backend/gpu/mali_kbase_instr_internal.h | 45 + .../backend/gpu/mali_kbase_irq_internal.h | 39 + .../backend/gpu/mali_kbase_irq_linux.c | 469 + .../backend/gpu/mali_kbase_jm_as.c | 235 + .../backend/gpu/mali_kbase_jm_defs.h | 123 + .../backend/gpu/mali_kbase_jm_hw.c | 1512 + .../backend/gpu/mali_kbase_jm_internal.h | 164 + .../backend/gpu/mali_kbase_jm_rb.c | 1947 + .../backend/gpu/mali_kbase_jm_rb.h | 76 + .../backend/gpu/mali_kbase_js_affinity.c | 303 + .../backend/gpu/mali_kbase_js_affinity.h | 129 + .../backend/gpu/mali_kbase_js_backend.c | 356 + .../backend/gpu/mali_kbase_js_internal.h | 69 + .../backend/gpu/mali_kbase_mmu_hw_direct.c | 401 + .../backend/gpu/mali_kbase_mmu_hw_direct.h | 42 + .../backend/gpu/mali_kbase_pm_always_on.c | 63 + .../backend/gpu/mali_kbase_pm_always_on.h | 77 + .../backend/gpu/mali_kbase_pm_backend.c | 478 + .../backend/gpu/mali_kbase_pm_ca.c | 182 + .../backend/gpu/mali_kbase_pm_ca.h | 92 + .../backend/gpu/mali_kbase_pm_ca_devfreq.c | 129 + .../backend/gpu/mali_kbase_pm_ca_devfreq.h | 55 + .../backend/gpu/mali_kbase_pm_ca_fixed.c | 65 + .../backend/gpu/mali_kbase_pm_ca_fixed.h | 40 + .../backend/gpu/mali_kbase_pm_coarse_demand.c | 70 + .../backend/gpu/mali_kbase_pm_coarse_demand.h | 64 + .../backend/gpu/mali_kbase_pm_defs.h | 519 + .../backend/gpu/mali_kbase_pm_demand.c | 73 + .../backend/gpu/mali_kbase_pm_demand.h | 64 + .../backend/gpu/mali_kbase_pm_driver.c | 1672 + .../backend/gpu/mali_kbase_pm_internal.h | 548 + .../backend/gpu/mali_kbase_pm_metrics.c | 401 + .../backend/gpu/mali_kbase_pm_policy.c | 973 + .../backend/gpu/mali_kbase_pm_policy.h | 227 + .../backend/gpu/mali_kbase_time.c | 103 + .../backend/gpu/mali_kbase_time.h | 52 + .../gpu/arm/bifrost_for_linux/docs/Doxyfile | 126 + .../docs/policy_operation_diagram.dot | 112 + .../docs/policy_overview.dot | 63 + drivers/gpu/arm/bifrost_for_linux/ipa/Kbuild | 27 + .../bifrost_for_linux/ipa/mali_kbase_ipa.c | 580 + .../bifrost_for_linux/ipa/mali_kbase_ipa.h | 165 + .../ipa/mali_kbase_ipa_debugfs.c | 219 + .../ipa/mali_kbase_ipa_debugfs.h | 49 + .../ipa/mali_kbase_ipa_simple.c | 327 + .../ipa/mali_kbase_ipa_simple.h | 40 + .../ipa/mali_kbase_ipa_vinstr_common.c | 217 + .../ipa/mali_kbase_ipa_vinstr_common.h | 161 + .../ipa/mali_kbase_ipa_vinstr_g71.c | 136 + .../mali_base_hwconfig_features.h | 282 + .../mali_base_hwconfig_issues.h | 1126 + .../arm/bifrost_for_linux/mali_base_kernel.h | 1822 + .../bifrost_for_linux/mali_base_mem_priv.h | 52 + .../mali_base_vendor_specific_func.h | 24 + .../gpu/arm/bifrost_for_linux/mali_kbase.h | 616 + .../mali_kbase_10969_workaround.c | 210 + .../mali_kbase_10969_workaround.h | 23 + .../mali_kbase_as_fault_debugfs.c | 102 + .../mali_kbase_as_fault_debugfs.h | 45 + .../mali_kbase_cache_policy.c | 54 + .../mali_kbase_cache_policy.h | 45 + .../arm/bifrost_for_linux/mali_kbase_config.c | 51 + .../arm/bifrost_for_linux/mali_kbase_config.h | 343 + .../mali_kbase_config_defaults.h | 226 + .../bifrost_for_linux/mali_kbase_context.c | 362 + .../bifrost_for_linux/mali_kbase_context.h | 90 + .../bifrost_for_linux/mali_kbase_core_linux.c | 4971 ++ .../bifrost_for_linux/mali_kbase_ctx_sched.c | 203 + .../bifrost_for_linux/mali_kbase_ctx_sched.h | 131 + .../arm/bifrost_for_linux/mali_kbase_debug.c | 39 + .../arm/bifrost_for_linux/mali_kbase_debug.h | 164 + .../mali_kbase_debug_job_fault.c | 499 + .../mali_kbase_debug_job_fault.h | 96 + .../mali_kbase_debug_mem_view.c | 306 + .../mali_kbase_debug_mem_view.h | 25 + .../arm/bifrost_for_linux/mali_kbase_defs.h | 1641 + .../arm/bifrost_for_linux/mali_kbase_device.c | 674 + .../mali_kbase_disjoint_events.c | 76 + .../bifrost_for_linux/mali_kbase_dma_fence.c | 449 + .../bifrost_for_linux/mali_kbase_dma_fence.h | 131 + .../arm/bifrost_for_linux/mali_kbase_event.c | 259 + .../arm/bifrost_for_linux/mali_kbase_fence.c | 196 + .../arm/bifrost_for_linux/mali_kbase_fence.h | 270 + .../bifrost_for_linux/mali_kbase_fence_defs.h | 51 + .../arm/bifrost_for_linux/mali_kbase_gator.h | 45 + .../bifrost_for_linux/mali_kbase_gator_api.c | 334 + .../bifrost_for_linux/mali_kbase_gator_api.h | 219 + .../mali_kbase_gator_hwcnt_names.h | 2167 + .../mali_kbase_gator_hwcnt_names_thex.h | 291 + .../mali_kbase_gator_hwcnt_names_tmix.h | 291 + .../mali_kbase_gator_hwcnt_names_tsix.h | 291 + .../arm/bifrost_for_linux/mali_kbase_gpu_id.h | 118 + .../mali_kbase_gpu_memory_debugfs.c | 97 + .../mali_kbase_gpu_memory_debugfs.h | 37 + .../bifrost_for_linux/mali_kbase_gpuprops.c | 514 + .../bifrost_for_linux/mali_kbase_gpuprops.h | 84 + .../mali_kbase_gpuprops_types.h | 92 + .../gpu/arm/bifrost_for_linux/mali_kbase_hw.c | 446 + .../gpu/arm/bifrost_for_linux/mali_kbase_hw.h | 65 + .../mali_kbase_hwaccess_backend.h | 54 + .../mali_kbase_hwaccess_defs.h | 36 + .../mali_kbase_hwaccess_gpuprops.h | 47 + .../mali_kbase_hwaccess_instr.h | 116 + .../mali_kbase_hwaccess_jm.h | 381 + .../mali_kbase_hwaccess_pm.h | 209 + .../mali_kbase_hwaccess_time.h | 53 + .../mali_kbase_hwcnt_reader.h | 66 + .../arm/bifrost_for_linux/mali_kbase_ioctl.h | 658 + .../gpu/arm/bifrost_for_linux/mali_kbase_jd.c | 1847 + .../bifrost_for_linux/mali_kbase_jd_debugfs.c | 235 + .../bifrost_for_linux/mali_kbase_jd_debugfs.h | 40 + .../gpu/arm/bifrost_for_linux/mali_kbase_jm.c | 131 + .../gpu/arm/bifrost_for_linux/mali_kbase_jm.h | 110 + .../gpu/arm/bifrost_for_linux/mali_kbase_js.c | 2798 + .../gpu/arm/bifrost_for_linux/mali_kbase_js.h | 925 + .../mali_kbase_js_ctx_attr.c | 301 + .../mali_kbase_js_ctx_attr.h | 158 + .../bifrost_for_linux/mali_kbase_js_defs.h | 386 + .../arm/bifrost_for_linux/mali_kbase_linux.h | 43 + .../arm/bifrost_for_linux/mali_kbase_mem.c | 2875 + .../arm/bifrost_for_linux/mali_kbase_mem.h | 1142 + .../bifrost_for_linux/mali_kbase_mem_linux.c | 2678 + .../bifrost_for_linux/mali_kbase_mem_linux.h | 240 + .../mali_kbase_mem_lowlevel.h | 89 + .../bifrost_for_linux/mali_kbase_mem_pool.c | 651 + .../mali_kbase_mem_pool_debugfs.c | 88 + .../mali_kbase_mem_pool_debugfs.h | 40 + .../mali_kbase_mem_profile_debugfs.c | 121 + .../mali_kbase_mem_profile_debugfs.h | 59 + .../mali_kbase_mem_profile_debugfs_buf_size.h | 33 + .../arm/bifrost_for_linux/mali_kbase_mmu.c | 2138 + .../arm/bifrost_for_linux/mali_kbase_mmu_hw.h | 123 + .../mali_kbase_mmu_mode_aarch64.c | 214 + .../mali_kbase_mmu_mode_lpae.c | 199 + .../mali_kbase_platform_fake.c | 119 + .../gpu/arm/bifrost_for_linux/mali_kbase_pm.c | 205 + .../gpu/arm/bifrost_for_linux/mali_kbase_pm.h | 171 + .../mali_kbase_profiling_gator_api.h | 40 + .../mali_kbase_regs_history_debugfs.c | 130 + .../mali_kbase_regs_history_debugfs.h | 50 + .../arm/bifrost_for_linux/mali_kbase_replay.c | 1166 + .../arm/bifrost_for_linux/mali_kbase_smc.c | 74 + .../arm/bifrost_for_linux/mali_kbase_smc.h | 67 + .../bifrost_for_linux/mali_kbase_softjobs.c | 1513 + .../bifrost_for_linux/mali_kbase_strings.c | 23 + .../bifrost_for_linux/mali_kbase_strings.h | 19 + .../arm/bifrost_for_linux/mali_kbase_sync.h | 203 + .../mali_kbase_sync_android.c | 537 + .../mali_kbase_sync_common.c | 43 + .../bifrost_for_linux/mali_kbase_sync_file.c | 348 + .../bifrost_for_linux/mali_kbase_tlstream.c | 2569 + .../bifrost_for_linux/mali_kbase_tlstream.h | 623 + .../bifrost_for_linux/mali_kbase_trace_defs.h | 264 + .../mali_kbase_trace_timeline.c | 236 + .../mali_kbase_trace_timeline.h | 363 + .../mali_kbase_trace_timeline_defs.h | 140 + .../arm/bifrost_for_linux/mali_kbase_uku.h | 532 + .../bifrost_for_linux/mali_kbase_utility.c | 33 + .../bifrost_for_linux/mali_kbase_utility.h | 37 + .../arm/bifrost_for_linux/mali_kbase_vinstr.c | 2072 + .../arm/bifrost_for_linux/mali_kbase_vinstr.h | 155 + .../mali_linux_kbase_trace.h | 201 + .../arm/bifrost_for_linux/mali_linux_trace.h | 189 + .../gpu/arm/bifrost_for_linux/mali_malisw.h | 131 + .../bifrost_for_linux/mali_midg_coherency.h | 26 + .../arm/bifrost_for_linux/mali_midg_regmap.h | 611 + .../gpu/arm/bifrost_for_linux/mali_timeline.h | 396 + drivers/gpu/arm/bifrost_for_linux/mali_uk.h | 141 + .../arm/bifrost_for_linux/platform/Kconfig | 24 + .../platform/devicetree/Kbuild | 18 + .../devicetree/mali_kbase_config_devicetree.c | 40 + .../devicetree/mali_kbase_config_platform.h | 80 + .../devicetree/mali_kbase_runtime_pm.c | 121 + .../platform/mali_kbase_platform_common.h | 26 + .../arm/bifrost_for_linux/platform/rk/Kbuild | 17 + .../platform/rk/custom_log.h | 192 + .../platform/rk/mali_kbase_config_platform.h | 88 + .../platform/rk/mali_kbase_config_rk.c | 459 + .../platform/rk/mali_kbase_rk.h | 62 + .../platform/vexpress/Kbuild | 19 + .../vexpress/mali_kbase_config_platform.h | 75 + .../vexpress/mali_kbase_config_vexpress.c | 85 + .../vexpress/mali_kbase_cpu_vexpress.c | 279 + .../vexpress/mali_kbase_cpu_vexpress.h | 38 + .../platform/vexpress_1xv7_a57/Kbuild | 18 + .../mali_kbase_config_platform.h | 73 + .../mali_kbase_config_vexpress.c | 79 + .../platform/vexpress_6xvirtex7_10mhz/Kbuild | 19 + .../mali_kbase_config_platform.h | 75 + .../mali_kbase_config_vexpress.c | 83 + .../mali_kbase_cpu_vexpress.c | 71 + .../mali_kbase_cpu_vexpress.h | 28 + .../platform_dummy/mali_ukk_os.h | 53 + .../protected_mode_switcher.h | 64 + drivers/gpu/arm/bifrost_for_linux/sconscript | 72 + .../gpu/arm/bifrost_for_linux/tests/Kbuild | 17 + .../gpu/arm/bifrost_for_linux/tests/Kconfig | 17 + .../tests/include/kutf/kutf_helpers.h | 216 + .../tests/include/kutf/kutf_helpers_user.h | 179 + .../tests/include/kutf/kutf_mem.h | 68 + .../tests/include/kutf/kutf_resultset.h | 121 + .../tests/include/kutf/kutf_suite.h | 568 + .../tests/include/kutf/kutf_utils.h | 55 + .../arm/bifrost_for_linux/tests/kutf/Kbuild | 20 + .../arm/bifrost_for_linux/tests/kutf/Kconfig | 22 + .../arm/bifrost_for_linux/tests/kutf/Makefile | 29 + .../tests/kutf/kutf_helpers.c | 768 + .../tests/kutf/kutf_helpers_user.c | 460 + .../bifrost_for_linux/tests/kutf/kutf_mem.c | 102 + .../tests/kutf/kutf_resultset.c | 95 + .../bifrost_for_linux/tests/kutf/kutf_suite.c | 1398 + .../bifrost_for_linux/tests/kutf/kutf_utils.c | 71 + .../bifrost_for_linux/tests/kutf/sconscript | 21 + .../tests/mali_kutf_irq_test/Kbuild | 20 + .../tests/mali_kutf_irq_test/Kconfig | 23 + .../tests/mali_kutf_irq_test/Makefile | 47 + .../mali_kutf_irq_test_main.c | 269 + .../tests/mali_kutf_irq_test/sconscript | 30 + .../arm/bifrost_for_linux/tests/sconscript | 38 + drivers/gpu/arm/mali400/.gitignore | 1 + drivers/gpu/arm/mali400/Kbuild | 2 + drivers/gpu/arm/mali400/mali/.gitignore | 1 + drivers/gpu/arm/mali400/mali/Kbuild | 254 + drivers/gpu/arm/mali400/mali/Kconfig | 118 + drivers/gpu/arm/mali400/mali/Makefile | 206 + .../arm/mali400/mali/common/mali_broadcast.c | 142 + .../arm/mali400/mali/common/mali_broadcast.h | 57 + .../mali400/mali/common/mali_control_timer.c | 139 + .../mali400/mali/common/mali_control_timer.h | 30 + .../gpu/arm/mali400/mali/common/mali_dlbu.c | 213 + .../gpu/arm/mali400/mali/common/mali_dlbu.h | 45 + .../mali400/mali/common/mali_dvfs_policy.c | 308 + .../mali400/mali/common/mali_dvfs_policy.h | 34 + .../arm/mali400/mali/common/mali_executor.c | 2707 + .../arm/mali400/mali/common/mali_executor.h | 102 + drivers/gpu/arm/mali400/mali/common/mali_gp.c | 357 + drivers/gpu/arm/mali400/mali/common/mali_gp.h | 127 + .../gpu/arm/mali400/mali/common/mali_gp_job.c | 306 + .../gpu/arm/mali400/mali/common/mali_gp_job.h | 324 + .../gpu/arm/mali400/mali/common/mali_group.c | 1875 + .../gpu/arm/mali400/mali/common/mali_group.h | 460 + .../arm/mali400/mali/common/mali_hw_core.c | 47 + .../arm/mali400/mali/common/mali_hw_core.h | 111 + .../mali400/mali/common/mali_kernel_common.h | 181 + .../mali400/mali/common/mali_kernel_core.c | 1349 + .../mali400/mali/common/mali_kernel_core.h | 57 + .../mali/common/mali_kernel_utilization.c | 440 + .../mali/common/mali_kernel_utilization.h | 72 + .../mali400/mali/common/mali_kernel_vsync.c | 45 + .../arm/mali400/mali/common/mali_l2_cache.c | 534 + .../arm/mali400/mali/common/mali_l2_cache.h | 124 + .../mali400/mali/common/mali_mem_validation.c | 68 + .../mali400/mali/common/mali_mem_validation.h | 19 + .../gpu/arm/mali400/mali/common/mali_mmu.c | 433 + .../gpu/arm/mali400/mali/common/mali_mmu.h | 124 + .../mali/common/mali_mmu_page_directory.c | 495 + .../mali/common/mali_mmu_page_directory.h | 110 + .../gpu/arm/mali400/mali/common/mali_osk.h | 1389 + .../arm/mali400/mali/common/mali_osk_bitops.h | 162 + .../arm/mali400/mali/common/mali_osk_list.h | 273 + .../arm/mali400/mali/common/mali_osk_mali.h | 152 + .../mali400/mali/common/mali_osk_profiling.h | 146 + .../arm/mali400/mali/common/mali_osk_types.h | 471 + drivers/gpu/arm/mali400/mali/common/mali_pm.c | 1362 + drivers/gpu/arm/mali400/mali/common/mali_pm.h | 91 + .../arm/mali400/mali/common/mali_pm_domain.c | 209 + .../arm/mali400/mali/common/mali_pm_domain.h | 104 + .../arm/mali400/mali/common/mali_pm_metrics.c | 255 + .../arm/mali400/mali/common/mali_pm_metrics.h | 74 + .../gpu/arm/mali400/mali/common/mali_pmu.c | 270 + .../gpu/arm/mali400/mali/common/mali_pmu.h | 123 + drivers/gpu/arm/mali400/mali/common/mali_pp.c | 502 + drivers/gpu/arm/mali400/mali/common/mali_pp.h | 138 + .../gpu/arm/mali400/mali/common/mali_pp_job.c | 316 + .../gpu/arm/mali400/mali/common/mali_pp_job.h | 594 + .../arm/mali400/mali/common/mali_scheduler.c | 1548 + .../arm/mali400/mali/common/mali_scheduler.h | 131 + .../mali/common/mali_scheduler_types.h | 29 + .../arm/mali400/mali/common/mali_session.c | 155 + .../arm/mali400/mali/common/mali_session.h | 136 + .../arm/mali400/mali/common/mali_soft_job.c | 438 + .../arm/mali400/mali/common/mali_soft_job.h | 190 + .../mali/common/mali_spinlock_reentrant.c | 77 + .../mali/common/mali_spinlock_reentrant.h | 70 + .../arm/mali400/mali/common/mali_timeline.c | 1964 + .../arm/mali400/mali/common/mali_timeline.h | 587 + .../mali/common/mali_timeline_fence_wait.c | 218 + .../mali/common/mali_timeline_fence_wait.h | 67 + .../mali/common/mali_timeline_sync_fence.c | 179 + .../mali/common/mali_timeline_sync_fence.h | 51 + .../gpu/arm/mali400/mali/common/mali_ukk.h | 551 + .../mali/common/mali_user_settings_db.c | 147 + .../mali/common/mali_user_settings_db.h | 39 + .../mali/include/linux/mali/mali_utgard.h | 526 + .../include/linux/mali/mali_utgard_ioctl.h | 97 + .../linux/mali/mali_utgard_profiling_events.h | 190 + .../mali/mali_utgard_profiling_gator_api.h | 305 + .../include/linux/mali/mali_utgard_uk_types.h | 1108 + .../linux/license/gpl/mali_kernel_license.h | 30 + .../gpu/arm/mali400/mali/linux/mali_devfreq.c | 354 + .../gpu/arm/mali400/mali/linux/mali_devfreq.h | 17 + .../mali/linux/mali_device_pause_resume.c | 36 + .../arm/mali400/mali/linux/mali_dma_fence.c | 439 + .../arm/mali400/mali/linux/mali_dma_fence.h | 124 + .../mali400/mali/linux/mali_internal_sync.c | 783 + .../mali400/mali/linux/mali_internal_sync.h | 191 + .../mali400/mali/linux/mali_kernel_linux.c | 1154 + .../mali400/mali/linux/mali_kernel_linux.h | 36 + .../mali400/mali/linux/mali_kernel_sysfs.c | 1410 + .../mali400/mali/linux/mali_kernel_sysfs.h | 29 + .../arm/mali400/mali/linux/mali_linux_trace.h | 161 + .../gpu/arm/mali400/mali/linux/mali_memory.c | 531 + .../gpu/arm/mali400/mali/linux/mali_memory.h | 143 + .../mali/linux/mali_memory_block_alloc.c | 362 + .../mali/linux/mali_memory_block_alloc.h | 58 + .../arm/mali400/mali/linux/mali_memory_cow.c | 776 + .../arm/mali400/mali/linux/mali_memory_cow.h | 48 + .../mali/linux/mali_memory_defer_bind.c | 262 + .../mali/linux/mali_memory_defer_bind.h | 64 + .../mali400/mali/linux/mali_memory_dma_buf.c | 369 + .../mali400/mali/linux/mali_memory_dma_buf.h | 53 + .../mali400/mali/linux/mali_memory_external.c | 89 + .../mali400/mali/linux/mali_memory_external.h | 29 + .../mali400/mali/linux/mali_memory_manager.c | 993 + .../mali400/mali/linux/mali_memory_manager.h | 51 + .../mali400/mali/linux/mali_memory_os_alloc.c | 810 + .../mali400/mali/linux/mali_memory_os_alloc.h | 54 + .../mali400/mali/linux/mali_memory_secure.c | 170 + .../mali400/mali/linux/mali_memory_secure.h | 30 + .../mali/linux/mali_memory_swap_alloc.c | 943 + .../mali/linux/mali_memory_swap_alloc.h | 121 + .../mali400/mali/linux/mali_memory_types.h | 219 + .../arm/mali400/mali/linux/mali_memory_ump.c | 154 + .../arm/mali400/mali/linux/mali_memory_ump.h | 29 + .../arm/mali400/mali/linux/mali_memory_util.c | 158 + .../arm/mali400/mali/linux/mali_memory_util.h | 20 + .../mali400/mali/linux/mali_memory_virtual.c | 127 + .../mali400/mali/linux/mali_memory_virtual.h | 35 + .../arm/mali400/mali/linux/mali_osk_atomics.c | 59 + .../arm/mali400/mali/linux/mali_osk_bitmap.c | 152 + .../gpu/arm/mali400/mali/linux/mali_osk_irq.c | 200 + .../arm/mali400/mali/linux/mali_osk_locks.c | 287 + .../arm/mali400/mali/linux/mali_osk_locks.h | 326 + .../mali/linux/mali_osk_low_level_mem.c | 146 + .../arm/mali400/mali/linux/mali_osk_mali.c | 505 + .../arm/mali400/mali/linux/mali_osk_math.c | 27 + .../arm/mali400/mali/linux/mali_osk_memory.c | 61 + .../arm/mali400/mali/linux/mali_osk_misc.c | 81 + .../mali/linux/mali_osk_notification.c | 182 + .../gpu/arm/mali400/mali/linux/mali_osk_pm.c | 83 + .../mali400/mali/linux/mali_osk_profiling.c | 1282 + .../mali400/mali/linux/mali_osk_specific.h | 74 + .../arm/mali400/mali/linux/mali_osk_time.c | 59 + .../arm/mali400/mali/linux/mali_osk_timers.c | 76 + .../mali400/mali/linux/mali_osk_wait_queue.c | 78 + .../gpu/arm/mali400/mali/linux/mali_osk_wq.c | 240 + .../mali/linux/mali_pmu_power_up_down.c | 23 + .../mali/linux/mali_profiling_events.h | 17 + .../mali/linux/mali_profiling_gator_api.h | 17 + .../mali/linux/mali_profiling_internal.c | 275 + .../mali/linux/mali_profiling_internal.h | 35 + .../gpu/arm/mali400/mali/linux/mali_sync.c | 665 + .../gpu/arm/mali400/mali/linux/mali_sync.h | 169 + .../arm/mali400/mali/linux/mali_uk_types.h | 17 + .../arm/mali400/mali/linux/mali_ukk_core.c | 171 + .../gpu/arm/mali400/mali/linux/mali_ukk_gp.c | 91 + .../gpu/arm/mali400/mali/linux/mali_ukk_mem.c | 333 + .../gpu/arm/mali400/mali/linux/mali_ukk_pp.c | 105 + .../mali400/mali/linux/mali_ukk_profiling.c | 183 + .../mali400/mali/linux/mali_ukk_soft_job.c | 90 + .../mali400/mali/linux/mali_ukk_timeline.c | 88 + .../arm/mali400/mali/linux/mali_ukk_vsync.c | 39 + .../mali400/mali/linux/mali_ukk_wrappers.h | 82 + .../gpu/arm/mali400/mali/platform/arm/arm.c | 629 + .../mali/platform/arm/arm_core_scaling.c | 122 + .../mali/platform/arm/arm_core_scaling.h | 44 + .../arm/mali400/mali/platform/arm/juno_opp.c | 127 + .../arm/mali400/mali/platform/rk/custom_log.h | 209 + drivers/gpu/arm/mali400/mali/platform/rk/rk.c | 676 + .../gpu/arm/mali400/mali/platform/rk/rk_ext.h | 37 + drivers/gpu/arm/mali400/mali/readme.txt | 28 + .../gpu/arm/mali400/mali/regs/mali_200_regs.h | 131 + .../gpu/arm/mali400/mali/regs/mali_gp_regs.h | 172 + .../mali/timestamp-arm11-cc/mali_timestamp.c | 13 + .../mali/timestamp-arm11-cc/mali_timestamp.h | 48 + .../mali/timestamp-default/mali_timestamp.c | 13 + .../mali/timestamp-default/mali_timestamp.h | 26 + drivers/gpu/arm/mali400/rk_ver_info.txt | 11 + drivers/gpu/arm/mali400/ump/Kbuild | 92 + drivers/gpu/arm/mali400/ump/Kconfig | 17 + drivers/gpu/arm/mali400/ump/Makefile | 67 + drivers/gpu/arm/mali400/ump/Makefile.common | 20 + .../gpu/arm/mali400/ump/arch-default/config.h | 24 + .../arm/mali400/ump/arch-pb-virtex5/config.h | 18 + drivers/gpu/arm/mali400/ump/arch/config.h | 24 + .../arm/mali400/ump/common/ump_kernel_api.c | 455 + .../mali400/ump/common/ump_kernel_common.c | 358 + .../mali400/ump/common/ump_kernel_common.h | 125 + .../common/ump_kernel_descriptor_mapping.c | 155 + .../common/ump_kernel_descriptor_mapping.h | 89 + .../ump/common/ump_kernel_memory_backend.h | 48 + .../mali400/ump/common/ump_kernel_ref_drv.c | 181 + .../arm/mali400/ump/common/ump_kernel_types.h | 58 + drivers/gpu/arm/mali400/ump/common/ump_osk.h | 48 + .../gpu/arm/mali400/ump/common/ump_uk_types.h | 202 + drivers/gpu/arm/mali400/ump/common/ump_ukk.h | 60 + .../linux/license/gpl/ump_kernel_license.h | 30 + drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h | 54 + .../arm/mali400/ump/linux/ump_kernel_linux.c | 449 + .../arm/mali400/ump/linux/ump_kernel_linux.h | 18 + .../ump_kernel_memory_backend_dedicated.c | 271 + .../ump_kernel_memory_backend_dedicated.h | 23 + .../ump/linux/ump_kernel_memory_backend_os.c | 235 + .../ump/linux/ump_kernel_memory_backend_os.h | 23 + .../ump/linux/ump_kernel_random_mapping.c | 222 + .../ump/linux/ump_kernel_random_mapping.h | 84 + .../mali400/ump/linux/ump_memory_backend.c | 65 + .../arm/mali400/ump/linux/ump_osk_atomics.c | 27 + .../mali400/ump/linux/ump_osk_low_level_mem.c | 314 + .../gpu/arm/mali400/ump/linux/ump_osk_misc.c | 36 + .../mali400/ump/linux/ump_ukk_ref_wrappers.c | 230 + .../mali400/ump/linux/ump_ukk_ref_wrappers.h | 36 + .../arm/mali400/ump/linux/ump_ukk_wrappers.c | 280 + .../arm/mali400/ump/linux/ump_ukk_wrappers.h | 46 + drivers/gpu/arm/mali400/ump/readme.txt | 28 + drivers/gpu/arm/mali400/umplock/Makefile | 69 + .../gpu/arm/mali400/umplock/umplock_driver.c | 618 + .../gpu/arm/mali400/umplock/umplock_ioctl.h | 66 + drivers/gpu/arm/midgard/Kbuild | 221 + drivers/gpu/arm/midgard/Kconfig | 248 + drivers/gpu/arm/midgard/Makefile | 42 + drivers/gpu/arm/midgard/Makefile.kbase | 17 + drivers/gpu/arm/midgard/backend/gpu/Kbuild | 60 + .../backend/gpu/mali_kbase_backend_config.h | 29 + .../gpu/mali_kbase_cache_policy_backend.c | 29 + .../gpu/mali_kbase_cache_policy_backend.h | 34 + .../gpu/mali_kbase_debug_job_fault_backend.c | 157 + .../midgard/backend/gpu/mali_kbase_devfreq.c | 458 + .../midgard/backend/gpu/mali_kbase_devfreq.h | 24 + .../backend/gpu/mali_kbase_device_hw.c | 255 + .../backend/gpu/mali_kbase_device_internal.h | 67 + .../arm/midgard/backend/gpu/mali_kbase_gpu.c | 123 + .../backend/gpu/mali_kbase_gpuprops_backend.c | 110 + .../backend/gpu/mali_kbase_instr_backend.c | 492 + .../backend/gpu/mali_kbase_instr_defs.h | 58 + .../backend/gpu/mali_kbase_instr_internal.h | 45 + .../backend/gpu/mali_kbase_irq_internal.h | 39 + .../backend/gpu/mali_kbase_irq_linux.c | 469 + .../midgard/backend/gpu/mali_kbase_jm_as.c | 237 + .../midgard/backend/gpu/mali_kbase_jm_defs.h | 123 + .../midgard/backend/gpu/mali_kbase_jm_hw.c | 1518 + .../backend/gpu/mali_kbase_jm_internal.h | 164 + .../midgard/backend/gpu/mali_kbase_jm_rb.c | 1952 + .../midgard/backend/gpu/mali_kbase_jm_rb.h | 76 + .../backend/gpu/mali_kbase_js_affinity.c | 303 + .../backend/gpu/mali_kbase_js_affinity.h | 129 + .../backend/gpu/mali_kbase_js_backend.c | 356 + .../backend/gpu/mali_kbase_js_internal.h | 69 + .../backend/gpu/mali_kbase_mmu_hw_direct.c | 407 + .../backend/gpu/mali_kbase_mmu_hw_direct.h | 42 + .../backend/gpu/mali_kbase_pm_always_on.c | 63 + .../backend/gpu/mali_kbase_pm_always_on.h | 77 + .../backend/gpu/mali_kbase_pm_backend.c | 482 + .../midgard/backend/gpu/mali_kbase_pm_ca.c | 182 + .../midgard/backend/gpu/mali_kbase_pm_ca.h | 92 + .../backend/gpu/mali_kbase_pm_ca_devfreq.c | 129 + .../backend/gpu/mali_kbase_pm_ca_devfreq.h | 55 + .../backend/gpu/mali_kbase_pm_ca_fixed.c | 65 + .../backend/gpu/mali_kbase_pm_ca_fixed.h | 40 + .../backend/gpu/mali_kbase_pm_coarse_demand.c | 70 + .../backend/gpu/mali_kbase_pm_coarse_demand.h | 64 + .../midgard/backend/gpu/mali_kbase_pm_defs.h | 519 + .../backend/gpu/mali_kbase_pm_demand.c | 73 + .../backend/gpu/mali_kbase_pm_demand.h | 64 + .../backend/gpu/mali_kbase_pm_driver.c | 1713 + .../backend/gpu/mali_kbase_pm_internal.h | 548 + .../backend/gpu/mali_kbase_pm_metrics.c | 401 + .../backend/gpu/mali_kbase_pm_policy.c | 973 + .../backend/gpu/mali_kbase_pm_policy.h | 227 + .../arm/midgard/backend/gpu/mali_kbase_time.c | 103 + .../arm/midgard/backend/gpu/mali_kbase_time.h | 52 + drivers/gpu/arm/midgard/docs/Doxyfile | 126 + .../midgard/docs/policy_operation_diagram.dot | 112 + .../gpu/arm/midgard/docs/policy_overview.dot | 63 + drivers/gpu/arm/midgard/ipa/Kbuild | 24 + drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c | 585 + drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h | 148 + .../arm/midgard/ipa/mali_kbase_ipa_debugfs.c | 219 + .../arm/midgard/ipa/mali_kbase_ipa_debugfs.h | 49 + .../arm/midgard/ipa/mali_kbase_ipa_simple.c | 222 + .../arm/midgard/mali_base_hwconfig_features.h | 311 + .../arm/midgard/mali_base_hwconfig_issues.h | 1098 + drivers/gpu/arm/midgard/mali_base_kernel.h | 1858 + drivers/gpu/arm/midgard/mali_base_mem_priv.h | 52 + .../midgard/mali_base_vendor_specific_func.h | 24 + drivers/gpu/arm/midgard/mali_kbase.h | 612 + .../arm/midgard/mali_kbase_10969_workaround.c | 209 + .../arm/midgard/mali_kbase_10969_workaround.h | 23 + .../arm/midgard/mali_kbase_as_fault_debugfs.c | 102 + .../arm/midgard/mali_kbase_as_fault_debugfs.h | 45 + .../gpu/arm/midgard/mali_kbase_cache_policy.c | 64 + .../gpu/arm/midgard/mali_kbase_cache_policy.h | 45 + drivers/gpu/arm/midgard/mali_kbase_config.c | 51 + drivers/gpu/arm/midgard/mali_kbase_config.h | 345 + .../arm/midgard/mali_kbase_config_defaults.h | 227 + drivers/gpu/arm/midgard/mali_kbase_context.c | 342 + drivers/gpu/arm/midgard/mali_kbase_context.h | 90 + .../gpu/arm/midgard/mali_kbase_core_linux.c | 4990 ++ .../gpu/arm/midgard/mali_kbase_ctx_sched.c | 208 + .../gpu/arm/midgard/mali_kbase_ctx_sched.h | 134 + drivers/gpu/arm/midgard/mali_kbase_debug.c | 39 + drivers/gpu/arm/midgard/mali_kbase_debug.h | 164 + .../arm/midgard/mali_kbase_debug_job_fault.c | 499 + .../arm/midgard/mali_kbase_debug_job_fault.h | 96 + .../arm/midgard/mali_kbase_debug_mem_view.c | 306 + .../arm/midgard/mali_kbase_debug_mem_view.h | 25 + drivers/gpu/arm/midgard/mali_kbase_defs.h | 1602 + drivers/gpu/arm/midgard/mali_kbase_device.c | 674 + .../arm/midgard/mali_kbase_disjoint_events.c | 76 + .../gpu/arm/midgard/mali_kbase_dma_fence.c | 449 + .../gpu/arm/midgard/mali_kbase_dma_fence.h | 131 + drivers/gpu/arm/midgard/mali_kbase_event.c | 259 + drivers/gpu/arm/midgard/mali_kbase_fence.c | 200 + drivers/gpu/arm/midgard/mali_kbase_fence.h | 275 + .../gpu/arm/midgard/mali_kbase_fence_defs.h | 51 + drivers/gpu/arm/midgard/mali_kbase_gator.h | 45 + .../gpu/arm/midgard/mali_kbase_gator_api.c | 334 + .../gpu/arm/midgard/mali_kbase_gator_api.h | 219 + .../midgard/mali_kbase_gator_hwcnt_names.h | 2170 + .../mali_kbase_gator_hwcnt_names_thex.h | 291 + .../mali_kbase_gator_hwcnt_names_tmix.h | 291 + .../mali_kbase_gator_hwcnt_names_tsix.h | 291 + drivers/gpu/arm/midgard/mali_kbase_gpu_id.h | 123 + .../midgard/mali_kbase_gpu_memory_debugfs.c | 97 + .../midgard/mali_kbase_gpu_memory_debugfs.h | 37 + drivers/gpu/arm/midgard/mali_kbase_gpuprops.c | 510 + drivers/gpu/arm/midgard/mali_kbase_gpuprops.h | 84 + .../arm/midgard/mali_kbase_gpuprops_types.h | 92 + drivers/gpu/arm/midgard/mali_kbase_hw.c | 453 + drivers/gpu/arm/midgard/mali_kbase_hw.h | 65 + .../arm/midgard/mali_kbase_hwaccess_backend.h | 54 + .../arm/midgard/mali_kbase_hwaccess_defs.h | 36 + .../midgard/mali_kbase_hwaccess_gpuprops.h | 47 + .../arm/midgard/mali_kbase_hwaccess_instr.h | 116 + .../gpu/arm/midgard/mali_kbase_hwaccess_jm.h | 381 + .../gpu/arm/midgard/mali_kbase_hwaccess_pm.h | 209 + .../arm/midgard/mali_kbase_hwaccess_time.h | 53 + .../gpu/arm/midgard/mali_kbase_hwcnt_reader.h | 66 + drivers/gpu/arm/midgard/mali_kbase_ioctl.h | 656 + drivers/gpu/arm/midgard/mali_kbase_jd.c | 1903 + .../gpu/arm/midgard/mali_kbase_jd_debugfs.c | 233 + .../gpu/arm/midgard/mali_kbase_jd_debugfs.h | 39 + drivers/gpu/arm/midgard/mali_kbase_jm.c | 131 + drivers/gpu/arm/midgard/mali_kbase_jm.h | 110 + drivers/gpu/arm/midgard/mali_kbase_js.c | 2834 + drivers/gpu/arm/midgard/mali_kbase_js.h | 925 + .../gpu/arm/midgard/mali_kbase_js_ctx_attr.c | 301 + .../gpu/arm/midgard/mali_kbase_js_ctx_attr.h | 158 + drivers/gpu/arm/midgard/mali_kbase_js_defs.h | 386 + drivers/gpu/arm/midgard/mali_kbase_linux.h | 43 + drivers/gpu/arm/midgard/mali_kbase_mem.c | 2657 + drivers/gpu/arm/midgard/mali_kbase_mem.h | 1068 + .../gpu/arm/midgard/mali_kbase_mem_linux.c | 2578 + .../gpu/arm/midgard/mali_kbase_mem_linux.h | 231 + .../gpu/arm/midgard/mali_kbase_mem_lowlevel.h | 45 + drivers/gpu/arm/midgard/mali_kbase_mem_pool.c | 569 + .../arm/midgard/mali_kbase_mem_pool_debugfs.c | 81 + .../arm/midgard/mali_kbase_mem_pool_debugfs.h | 36 + .../midgard/mali_kbase_mem_profile_debugfs.c | 121 + .../midgard/mali_kbase_mem_profile_debugfs.h | 59 + .../mali_kbase_mem_profile_debugfs_buf_size.h | 33 + drivers/gpu/arm/midgard/mali_kbase_mmu.c | 2088 + drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h | 123 + drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h | 47 + .../arm/midgard/mali_kbase_mmu_mode_aarch64.c | 200 + .../arm/midgard/mali_kbase_mmu_mode_lpae.c | 198 + .../arm/midgard/mali_kbase_platform_fake.c | 124 + drivers/gpu/arm/midgard/mali_kbase_pm.c | 205 + drivers/gpu/arm/midgard/mali_kbase_pm.h | 171 + .../midgard/mali_kbase_profiling_gator_api.h | 40 + .../midgard/mali_kbase_regs_history_debugfs.c | 130 + .../midgard/mali_kbase_regs_history_debugfs.h | 50 + drivers/gpu/arm/midgard/mali_kbase_replay.c | 1166 + drivers/gpu/arm/midgard/mali_kbase_smc.c | 86 + drivers/gpu/arm/midgard/mali_kbase_smc.h | 67 + drivers/gpu/arm/midgard/mali_kbase_softjobs.c | 1549 + drivers/gpu/arm/midgard/mali_kbase_strings.c | 23 + drivers/gpu/arm/midgard/mali_kbase_strings.h | 19 + drivers/gpu/arm/midgard/mali_kbase_sync.h | 203 + .../gpu/arm/midgard/mali_kbase_sync_android.c | 537 + .../gpu/arm/midgard/mali_kbase_sync_common.c | 43 + .../gpu/arm/midgard/mali_kbase_sync_file.c | 359 + drivers/gpu/arm/midgard/mali_kbase_tlstream.c | 2572 + drivers/gpu/arm/midgard/mali_kbase_tlstream.h | 623 + .../gpu/arm/midgard/mali_kbase_trace_defs.h | 264 + .../arm/midgard/mali_kbase_trace_timeline.c | 236 + .../arm/midgard/mali_kbase_trace_timeline.h | 363 + .../midgard/mali_kbase_trace_timeline_defs.h | 140 + drivers/gpu/arm/midgard/mali_kbase_uku.h | 545 + drivers/gpu/arm/midgard/mali_kbase_utility.c | 33 + drivers/gpu/arm/midgard/mali_kbase_utility.h | 37 + drivers/gpu/arm/midgard/mali_kbase_vinstr.c | 2070 + drivers/gpu/arm/midgard/mali_kbase_vinstr.h | 155 + .../gpu/arm/midgard/mali_linux_kbase_trace.h | 201 + drivers/gpu/arm/midgard/mali_linux_trace.h | 189 + drivers/gpu/arm/midgard/mali_malisw.h | 131 + drivers/gpu/arm/midgard/mali_midg_coherency.h | 26 + drivers/gpu/arm/midgard/mali_midg_regmap.h | 611 + drivers/gpu/arm/midgard/mali_timeline.h | 396 + drivers/gpu/arm/midgard/mali_uk.h | 141 + drivers/gpu/arm/midgard/platform/Kconfig | 24 + .../arm/midgard/platform/devicetree/Kbuild | 18 + .../devicetree/mali_kbase_config_devicetree.c | 31 + .../devicetree/mali_kbase_config_platform.h | 73 + .../devicetree/mali_kbase_runtime_pm.c | 100 + .../platform/mali_kbase_platform_common.h | 28 + .../platform/mali_kbase_platform_fake.h | 38 + drivers/gpu/arm/midgard/platform/rk/Kbuild | 17 + .../gpu/arm/midgard/platform/rk/custom_log.h | 209 + .../platform/rk/mali_kbase_config_platform.h | 88 + .../platform/rk/mali_kbase_config_rk.c | 492 + .../arm/midgard/platform/rk/mali_kbase_rk.h | 62 + .../gpu/arm/midgard/platform/vexpress/Kbuild | 18 + .../vexpress/mali_kbase_config_platform.h | 75 + .../vexpress/mali_kbase_config_vexpress.c | 85 + .../vexpress/mali_kbase_cpu_vexpress.c | 279 + .../vexpress/mali_kbase_cpu_vexpress.h | 38 + .../midgard/platform/vexpress_1xv7_a57/Kbuild | 16 + .../mali_kbase_config_platform.h | 73 + .../mali_kbase_config_vexpress.c | 79 + .../platform/vexpress_6xvirtex7_10mhz/Kbuild | 18 + .../mali_kbase_config_platform.h | 75 + .../mali_kbase_config_vexpress.c | 83 + .../mali_kbase_cpu_vexpress.c | 71 + .../mali_kbase_cpu_vexpress.h | 28 + .../arm/midgard/platform_dummy/mali_ukk_os.h | 53 + .../gpu/arm/midgard/protected_mode_switcher.h | 64 + drivers/gpu/arm/midgard/rename.h | 422 + drivers/gpu/arm/midgard/sconscript | 92 + drivers/gpu/arm/midgard/tests/Kbuild | 17 + drivers/gpu/arm/midgard/tests/Kconfig | 17 + .../arm/midgard/tests/include/kutf/kutf_mem.h | 65 + .../tests/include/kutf/kutf_resultset.h | 121 + .../midgard/tests/include/kutf/kutf_suite.h | 508 + .../midgard/tests/include/kutf/kutf_utils.h | 55 + drivers/gpu/arm/midgard/tests/kutf/Kbuild | 20 + drivers/gpu/arm/midgard/tests/kutf/Kconfig | 22 + drivers/gpu/arm/midgard/tests/kutf/Makefile | 29 + drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c | 94 + .../arm/midgard/tests/kutf/kutf_resultset.c | 95 + .../gpu/arm/midgard/tests/kutf/kutf_suite.c | 1041 + .../gpu/arm/midgard/tests/kutf/kutf_utils.c | 71 + drivers/gpu/arm/midgard/tests/kutf/sconscript | 21 + .../midgard/tests/mali_kutf_irq_test/Kbuild | 20 + .../midgard/tests/mali_kutf_irq_test/Kconfig | 23 + .../midgard/tests/mali_kutf_irq_test/Makefile | 51 + .../mali_kutf_irq_test_main.c | 257 + .../tests/mali_kutf_irq_test/sconscript | 30 + drivers/gpu/arm/midgard/tests/sconscript | 37 + drivers/gpu/arm/sconscript | 25 + drivers/gpu/drm/Kconfig | 4 + .../drm/bridge/analogix/analogix_dp_core.c | 742 +- .../drm/bridge/analogix/analogix_dp_core.h | 46 +- .../gpu/drm/bridge/analogix/analogix_dp_reg.c | 959 +- .../gpu/drm/bridge/analogix/analogix_dp_reg.h | 97 +- drivers/gpu/drm/bridge/synopsys/Makefile | 5 +- drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c | 3 + .../gpu/drm/bridge/synopsys/dw-hdmi-hdcp.c | 748 + .../gpu/drm/bridge/synopsys/dw-hdmi-hdcp.h | 54 + .../drm/bridge/synopsys/dw-hdmi-qp-audio.h | 29 + .../bridge/synopsys/dw-hdmi-qp-i2s-audio.c | 262 + drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c | 2455 + drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h | 824 + drivers/gpu/drm/bridge/synopsys/dw-hdmi.c | 1744 +- drivers/gpu/drm/bridge/synopsys/dw-hdmi.h | 55 + drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c | 111 +- drivers/gpu/drm/drm_atomic_helper.c | 12 +- drivers/gpu/drm/drm_atomic_state_helper.c | 7 + drivers/gpu/drm/drm_atomic_uapi.c | 14 + drivers/gpu/drm/drm_auth.c | 42 +- drivers/gpu/drm/drm_color_mgmt.c | 41 +- drivers/gpu/drm/drm_debugfs.c | 3 +- drivers/gpu/drm/drm_edid.c | 122 +- drivers/gpu/drm/drm_file.c | 69 +- drivers/gpu/drm/drm_fourcc.c | 10 + drivers/gpu/drm/drm_ioctl.c | 8 +- drivers/gpu/drm/drm_lease.c | 81 +- drivers/gpu/drm/drm_mipi_dsi.c | 1 + drivers/gpu/drm/drm_mode_config.c | 16 + drivers/gpu/drm/drm_modes.c | 2 + drivers/gpu/drm/drm_prime.c | 23 + drivers/gpu/drm/drm_sync_helper.c | 314 + drivers/gpu/drm/drm_vblank.c | 9 +- drivers/gpu/drm/i915/gt/intel_engine_cs.c | 3 +- drivers/gpu/drm/i915/gt/uc/intel_guc.c | 18 + drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c | 131 +- drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h | 80 +- drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h | 5 - drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c | 27 +- drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h | 2 - drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h | 6 +- drivers/gpu/drm/nouveau/nouveau_drm.c | 42 +- drivers/gpu/drm/nouveau/nouveau_drv.h | 5 - drivers/gpu/drm/panel/Kconfig | 1 - drivers/gpu/drm/panel/panel-ilitek-ili9881c.c | 12 +- drivers/gpu/drm/panel/panel-simple.c | 493 +- drivers/gpu/drm/panel/panel-simple.h | 19 + drivers/gpu/drm/rockchip/Kconfig | 34 +- drivers/gpu/drm/rockchip/Makefile | 12 +- .../gpu/drm/rockchip/analogix_dp-rockchip.c | 426 +- drivers/gpu/drm/rockchip/cdn-dp-core.c | 109 +- drivers/gpu/drm/rockchip/cdn-dp-core.h | 5 +- .../gpu/drm/rockchip/cdn-dp-link-training.c | 439 + drivers/gpu/drm/rockchip/dw-dp.c | 2519 + .../gpu/drm/rockchip/dw-mipi-dsi-rockchip.c | 596 +- .../gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c | 1591 + drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c | 3007 +- drivers/gpu/drm/rockchip/ebc-dev/Kconfig | 14 + drivers/gpu/drm/rockchip/ebc-dev/Makefile | 9 + .../drm/rockchip/ebc-dev/bufmanage/Makefile | 3 + .../drm/rockchip/ebc-dev/bufmanage/buf_list.c | 171 + .../drm/rockchip/ebc-dev/bufmanage/buf_list.h | 95 + .../rockchip/ebc-dev/bufmanage/buf_manage.c | 327 + .../rockchip/ebc-dev/bufmanage/buf_manage.h | 49 + drivers/gpu/drm/rockchip/ebc-dev/ebc_dev.h | 114 + drivers/gpu/drm/rockchip/ebc-dev/ebc_dev_v8.S | 55302 ++++++++++++++++ drivers/gpu/drm/rockchip/ebc-dev/ebc_panel.h | 50 + drivers/gpu/drm/rockchip/ebc-dev/ebc_public.c | 36 + .../gpu/drm/rockchip/ebc-dev/epdlut/Makefile | 5 + .../gpu/drm/rockchip/ebc-dev/epdlut/epd_lut.c | 72 + .../gpu/drm/rockchip/ebc-dev/epdlut/epd_lut.h | 66 + .../rockchip/ebc-dev/epdlut/pvi_waveform_v8.S | 18656 ++++++ .../rockchip/ebc-dev/epdlut/rkf_waveform_v8.S | 19357 ++++++ .../gpu/drm/rockchip/ebc-dev/pmic/Makefile | 4 + .../gpu/drm/rockchip/ebc-dev/pmic/ebc_pmic.c | 70 + .../gpu/drm/rockchip/ebc-dev/pmic/ebc_pmic.h | 60 + .../gpu/drm/rockchip/ebc-dev/pmic/tps65185.c | 672 + .../gpu/drm/rockchip/ebc-dev/tcon/Makefile | 3 + .../gpu/drm/rockchip/ebc-dev/tcon/ebc_tcon.c | 416 + .../gpu/drm/rockchip/ebc-dev/tcon/ebc_tcon.h | 119 + .../gpu/drm/rockchip/ebc-dev/tcon/eink_tcon.c | 301 + drivers/gpu/drm/rockchip/inno_hdmi.c | 2 +- drivers/gpu/drm/rockchip/rk3066_hdmi.c | 2 +- drivers/gpu/drm/rockchip/rk618/Kconfig | 9 + drivers/gpu/drm/rockchip/rk618/Makefile | 12 + drivers/gpu/drm/rockchip/rk618/rk618_dither.c | 50 + drivers/gpu/drm/rockchip/rk618/rk618_dither.h | 17 + drivers/gpu/drm/rockchip/rk618/rk618_dsi.c | 1201 + drivers/gpu/drm/rockchip/rk618/rk618_hdmi.c | 1605 + drivers/gpu/drm/rockchip/rk618/rk618_lvds.c | 310 + drivers/gpu/drm/rockchip/rk618/rk618_rgb.c | 303 + drivers/gpu/drm/rockchip/rk618/rk618_scaler.c | 424 + drivers/gpu/drm/rockchip/rk618/rk618_vif.c | 260 + drivers/gpu/drm/rockchip/rk628/Kconfig | 8 + drivers/gpu/drm/rockchip/rk628/Makefile | 14 + .../gpu/drm/rockchip/rk628/rk628_combrxphy.c | 995 + .../gpu/drm/rockchip/rk628/rk628_combtxphy.c | 515 + .../gpu/drm/rockchip/rk628/rk628_combtxphy.h | 13 + drivers/gpu/drm/rockchip/rk628/rk628_dsi.c | 1395 + drivers/gpu/drm/rockchip/rk628/rk628_gvi.c | 668 + drivers/gpu/drm/rockchip/rk628/rk628_hdmi.c | 1615 + drivers/gpu/drm/rockchip/rk628/rk628_hdmirx.c | 982 + drivers/gpu/drm/rockchip/rk628/rk628_lvds.c | 325 + .../drm/rockchip/rk628/rk628_post_process.c | 488 + drivers/gpu/drm/rockchip/rk628/rk628_rgb.c | 387 + .../gpu/drm/rockchip/rockchip-mipi-csi-tx.c | 1403 + .../gpu/drm/rockchip/rockchip-mipi-csi-tx.h | 289 + .../gpu/drm/rockchip/rockchip_drm_backlight.c | 514 + .../gpu/drm/rockchip/rockchip_drm_backlight.h | 64 + .../gpu/drm/rockchip/rockchip_drm_debugfs.c | 217 + .../gpu/drm/rockchip/rockchip_drm_debugfs.h | 72 + drivers/gpu/drm/rockchip/rockchip_drm_drv.c | 1399 +- drivers/gpu/drm/rockchip/rockchip_drm_drv.h | 387 +- drivers/gpu/drm/rockchip/rockchip_drm_fb.c | 150 +- drivers/gpu/drm/rockchip/rockchip_drm_fb.h | 16 + drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c | 18 +- drivers/gpu/drm/rockchip/rockchip_drm_gem.c | 637 +- drivers/gpu/drm/rockchip/rockchip_drm_gem.h | 47 +- drivers/gpu/drm/rockchip/rockchip_drm_logo.c | 1071 + drivers/gpu/drm/rockchip/rockchip_drm_logo.h | 42 + drivers/gpu/drm/rockchip/rockchip_drm_tve.c | 615 + drivers/gpu/drm/rockchip/rockchip_drm_tve.h | 165 + drivers/gpu/drm/rockchip/rockchip_drm_vconn.c | 537 + drivers/gpu/drm/rockchip/rockchip_drm_vop.c | 4645 +- drivers/gpu/drm/rockchip/rockchip_drm_vop.h | 1142 +- drivers/gpu/drm/rockchip/rockchip_drm_vop2.c | 8670 +++ drivers/gpu/drm/rockchip/rockchip_drm_vvop.c | 371 + drivers/gpu/drm/rockchip/rockchip_lvds.c | 971 +- drivers/gpu/drm/rockchip/rockchip_rgb.c | 487 +- drivers/gpu/drm/rockchip/rockchip_rgb.h | 6 +- drivers/gpu/drm/rockchip/rockchip_vop2_clk.c | 337 + drivers/gpu/drm/rockchip/rockchip_vop2_reg.c | 2796 + drivers/gpu/drm/rockchip/rockchip_vop_reg.c | 2416 +- drivers/gpu/drm/rockchip/rockchip_vop_reg.h | 816 +- drivers/i2c/busses/i2c-rk3x.c | 269 +- drivers/i2c/i2c-core-base.c | 51 +- drivers/iio/adc/Kconfig | 7 + drivers/iio/adc/rockchip_saradc.c | 143 +- drivers/input/Kconfig | 4 + drivers/input/Makefile | 2 + drivers/input/keyboard/Kconfig | 3 + drivers/input/keyboard/hil_kbd.c | 1 - drivers/input/keyboard/rk_keys.c | 555 + drivers/input/remotectl/Kconfig | 19 + drivers/input/remotectl/Makefile | 8 + .../input/remotectl/rockchip_pwm_remotectl.c | 828 + .../input/remotectl/rockchip_pwm_remotectl.h | 160 + drivers/input/sensors/Kconfig | 23 + drivers/input/sensors/Makefile | 15 + drivers/input/sensors/accel/Kconfig | 149 + drivers/input/sensors/accel/Makefile | 22 + drivers/input/sensors/accel/bma2xx.c | 2268 + drivers/input/sensors/accel/da223_core.c | 1907 + drivers/input/sensors/accel/da223_core.h | 303 + drivers/input/sensors/accel/da223_cust.c | 898 + drivers/input/sensors/accel/da223_cust.h | 44 + drivers/input/sensors/accel/dmard10.c | 454 + drivers/input/sensors/accel/icm2060x_acc.c | 260 + drivers/input/sensors/accel/kxtik.c | 359 + drivers/input/sensors/accel/kxtj9.c | 332 + drivers/input/sensors/accel/lis3dh.c | 303 + drivers/input/sensors/accel/lsm303d.c | 362 + drivers/input/sensors/accel/lsm330_acc.c | 278 + drivers/input/sensors/accel/mc3230.c | 1353 + drivers/input/sensors/accel/mma7660.c | 258 + drivers/input/sensors/accel/mma8452.c | 424 + drivers/input/sensors/accel/mpu6500_acc.c | 300 + drivers/input/sensors/accel/mpu6880_acc.c | 296 + drivers/input/sensors/accel/mxc622x.c | 284 + drivers/input/sensors/accel/mxc6655xa.c | 262 + drivers/input/sensors/accel/sc7660.c | 1655 + drivers/input/sensors/accel/sc7a20.c | 1768 + drivers/input/sensors/accel/sc7a30.c | 1215 + drivers/input/sensors/accel/stk8baxx.c | 957 + drivers/input/sensors/angle/Kconfig | 27 + drivers/input/sensors/angle/Makefile | 3 + drivers/input/sensors/angle/angle_kxtik.c | 398 + drivers/input/sensors/angle/angle_lis3dh.c | 342 + drivers/input/sensors/compass/Kconfig | 58 + drivers/input/sensors/compass/Makefile | 8 + drivers/input/sensors/compass/ak09911.c | 667 + drivers/input/sensors/compass/ak09918.c | 692 + drivers/input/sensors/compass/ak8963.c | 714 + drivers/input/sensors/compass/ak8975.c | 650 + drivers/input/sensors/gyro/Kconfig | 42 + drivers/input/sensors/gyro/Makefile | 11 + drivers/input/sensors/gyro/ewtsa.c | 461 + drivers/input/sensors/gyro/icm2060x_gyro.c | 216 + drivers/input/sensors/gyro/l3g20d.c | 257 + drivers/input/sensors/gyro/l3g4200d.c | 257 + drivers/input/sensors/gyro/lsm330_gyro.c | 262 + drivers/input/sensors/gyro/mpu6500_gyro.c | 216 + drivers/input/sensors/gyro/mpu6880_gyro.c | 216 + drivers/input/sensors/hall/Kconfig | 16 + drivers/input/sensors/hall/Makefile | 3 + drivers/input/sensors/hall/mh248.c | 165 + drivers/input/sensors/hall/och165t_hall.c | 159 + drivers/input/sensors/lsensor/Kconfig | 50 + drivers/input/sensors/lsensor/Makefile | 13 + drivers/input/sensors/lsensor/cm3217.c | 237 + drivers/input/sensors/lsensor/cm3218.c | 415 + drivers/input/sensors/lsensor/cm3232.c | 245 + drivers/input/sensors/lsensor/isl29023.c | 273 + drivers/input/sensors/lsensor/ls_al3006.c | 301 + drivers/input/sensors/lsensor/ls_ap321xx.c | 415 + drivers/input/sensors/lsensor/ls_em3071x.c | 283 + drivers/input/sensors/lsensor/ls_stk3171.c | 321 + drivers/input/sensors/lsensor/ls_stk3410.c | 374 + drivers/input/sensors/lsensor/ls_us5152.c | 440 + drivers/input/sensors/lsensor/ls_us5152.h | 235 + drivers/input/sensors/pressure/Kconfig | 15 + drivers/input/sensors/pressure/Makefile | 7 + drivers/input/sensors/pressure/pr_ms5607.c | 297 + drivers/input/sensors/psensor/Kconfig | 32 + drivers/input/sensors/psensor/Makefile | 8 + drivers/input/sensors/psensor/ps_al3006.c | 261 + drivers/input/sensors/psensor/ps_ap321xx.c | 325 + drivers/input/sensors/psensor/ps_em3071x.c | 280 + drivers/input/sensors/psensor/ps_stk3171.c | 268 + drivers/input/sensors/psensor/ps_stk3410.c | 377 + drivers/input/sensors/sensor-dev.c | 2042 + drivers/input/sensors/sensor-i2c.c | 245 + drivers/input/sensors/temperature/Kconfig | 15 + drivers/input/sensors/temperature/Makefile | 7 + .../input/sensors/temperature/tmp_ms5607.c | 314 + drivers/input/touchscreen/Kconfig | 18 + drivers/input/touchscreen/Makefile | 4 + drivers/input/touchscreen/cy8c_touchpad.c | 296 + drivers/input/touchscreen/cyttsp5/Kconfig | 205 + drivers/input/touchscreen/cyttsp5/Makefile | 23 + .../input/touchscreen/cyttsp5/cyttsp5_btn.c | 369 + .../input/touchscreen/cyttsp5/cyttsp5_core.c | 6292 ++ .../input/touchscreen/cyttsp5/cyttsp5_core.h | 182 + .../input/touchscreen/cyttsp5/cyttsp5_debug.c | 393 + .../cyttsp5/cyttsp5_device_access.c | 5219 ++ .../touchscreen/cyttsp5/cyttsp5_devtree.c | 785 + .../input/touchscreen/cyttsp5/cyttsp5_i2c.c | 248 + .../touchscreen/cyttsp5/cyttsp5_loader.c | 1586 + .../touchscreen/cyttsp5/cyttsp5_mt_common.c | 793 + .../input/touchscreen/cyttsp5/cyttsp5_mta.c | 85 + .../input/touchscreen/cyttsp5/cyttsp5_mtb.c | 93 + .../touchscreen/cyttsp5/cyttsp5_params.h | 1548 + .../cyttsp5/cyttsp5_params_pid00.h | 1548 + .../cyttsp5/cyttsp5_params_pid01.h | 1548 + .../touchscreen/cyttsp5/cyttsp5_platform.c | 289 + .../touchscreen/cyttsp5/cyttsp5_platform.h | 60 + .../touchscreen/cyttsp5/cyttsp5_proximity.c | 555 + .../input/touchscreen/cyttsp5/cyttsp5_regs.h | 1202 + .../input/touchscreen/cyttsp5/cyttsp5_spi.c | 255 + .../cyttsp5/cyttsp5_test_device_access_api.c | 442 + .../touchscreen/focaltech_touch/Makefile | 18 + .../focaltech_touch/focaltech_common.h | 159 + .../focaltech_touch/focaltech_config.h | 244 + .../focaltech_touch/focaltech_core.c | 1783 + .../focaltech_touch/focaltech_core.h | 254 + .../focaltech_touch/focaltech_esdcheck.c | 552 + .../focaltech_touch/focaltech_ex_fun.c | 1130 + .../focaltech_touch/focaltech_ex_mode.c | 382 + .../focaltech_touch/focaltech_flash.c | 1971 + .../focaltech_touch/focaltech_flash.h | 182 + .../focaltech_touch/focaltech_flash/Makefile | 8 + .../focaltech_upgrade_ft8201.c | 1033 + .../focaltech_touch/focaltech_gesture.c | 597 + .../focaltech_touch/focaltech_i2c.c | 240 + .../focaltech_point_report_check.c | 135 + .../focaltech_touch/focaltech_test/Makefile | 11 + .../focaltech_test/focaltech_test.c | 1948 + .../focaltech_test/focaltech_test.h | 589 + .../focaltech_test/focaltech_test_ini.c | 1220 + .../focaltech_test/focaltech_test_ini.h | 144 + .../focaltech_test/supported_ic/Makefile | 6 + .../supported_ic/focaltech_test_ft8201.c | 1034 + .../include/firmware/fw_sample.h | 0 .../FT8006M_Pramboot_V1.6_20180426_le.h | 299 + .../focaltech_touch_ft5436/Makefile | 16 + .../focaltech_touch_ft5436/focaltech_common.h | 166 + .../focaltech_touch_ft5436/focaltech_config.h | 241 + .../focaltech_touch_ft5436/focaltech_core.c | 1747 + .../focaltech_touch_ft5436/focaltech_core.h | 263 + .../focaltech_esdcheck.c | 464 + .../focaltech_touch_ft5436/focaltech_ex_fun.c | 1181 + .../focaltech_ex_mode.c | 307 + .../focaltech_touch_ft5436/focaltech_flash.c | 2011 + .../focaltech_touch_ft5436/focaltech_flash.h | 205 + .../focaltech_flash/Makefile | 7 + .../focaltech_upgrade_ft5436.c | 154 + .../focaltech_gesture.c | 460 + .../focaltech_touch_ft5436/focaltech_i2c.c | 193 + .../focaltech_point_report_check.c | 135 + .../include/firmware/fw_sample.h | 3223 + drivers/input/touchscreen/gsl3673.c | 1302 + drivers/input/touchscreen/gsl3673.h | 5091 ++ drivers/input/touchscreen/gsl3673_800x1280.c | 1247 + drivers/input/touchscreen/gsl3673_800x1280.h | 5187 ++ drivers/input/touchscreen/gsl3676/Makefile | 3 + drivers/input/touchscreen/gsl3676/gsl3676.c | 1261 + .../touchscreen/gsl3676/rk3368_th863c_10.h | 5331 ++ drivers/input/touchscreen/gsl_point_id.c | 3100 + drivers/input/touchscreen/gslx680.c | 1905 + drivers/input/touchscreen/gslx680.h | 10368 +++ drivers/input/touchscreen/gslx6801.c | 1680 + drivers/input/touchscreen/gslx6801.h | 4955 ++ .../input/touchscreen/gslx680_d708/Makefile | 9 + .../gslx680_d708/rockchip_gsl3670.h | 5047 ++ .../gslx680_d708/rockchip_gslX680.c | 1393 + .../gslx680_d708/rockchip_gslX680_88v.h | 5248 ++ drivers/input/touchscreen/gslx680_firefly.c | 1130 + drivers/input/touchscreen/gslx680_firefly.h | 5120 ++ drivers/input/touchscreen/gslx680_pad.c | 1198 + drivers/input/touchscreen/gslx680_pad.h | 10232 +++ drivers/input/touchscreen/gslx680a.c | 1320 + .../gt1x/GT5688_Config_20170713_1080_1920.cfg | 1 + drivers/input/touchscreen/gt1x/Makefile | 8 + drivers/input/touchscreen/gt1x/gt1x.c | 809 + drivers/input/touchscreen/gt1x/gt1x.h | 64 + drivers/input/touchscreen/gt1x/gt1x_cfg.h | 27 + drivers/input/touchscreen/gt1x/gt1x_extents.c | 928 + .../input/touchscreen/gt1x/gt1x_firmware.h | 548 + drivers/input/touchscreen/gt1x/gt1x_generic.c | 2445 + drivers/input/touchscreen/gt1x/gt1x_generic.h | 595 + drivers/input/touchscreen/gt1x/gt1x_tools.c | 433 + drivers/input/touchscreen/gt1x/gt1x_update.c | 1456 + ...T9110P(2020)V71_Config_20201028_170326.cfg | 19 + .../GT9110P_Config_20160217_1526_2048_97.cfg | 1 + .../gt9xx/GT9271_Config_20170526.cfg | 1 + ...-1398V1-1060-GT911_Config_20201204_V66.cfg | 1 + drivers/input/touchscreen/gt9xx/Makefile | 6 + ...GT9271_1060_Config_20140821_1341110X42.cfg | 1 + ...WGJ10162_GT9271_Config_20140820_182456.cfg | 1 + ...0187_GT910_Config_20140623_104014_0X41.cfg | 1 + ...187_GT9271_Config_20140623_104014_0X41.cfg | 1 + ...006B_GT911_Config_20140625_085816_0X43.cfg | 1 + ...06B_GT9271_Config_20140625_085816_0X41.cfg | 1 + drivers/input/touchscreen/gt9xx/goodix_tool.c | 627 + drivers/input/touchscreen/gt9xx/gt9xx.c | 3236 + drivers/input/touchscreen/gt9xx/gt9xx.h | 397 + drivers/input/touchscreen/gt9xx/gt9xx_cfg.h | 65 + .../input/touchscreen/gt9xx/gt9xx_firmware.h | 2338 + .../input/touchscreen/gt9xx/gt9xx_update.c | 3637 + .../hyn_cst2xx/CST21680SE_S126_D863_7.h | 514 + .../hyn_cst2xx/CST21680SE_S126_D89.h | 514 + drivers/input/touchscreen/hyn_cst2xx/Makefile | 2 + .../input/touchscreen/hyn_cst2xx/hyn_cst2xx.c | 1970 + drivers/input/touchscreen/tp_gslx680_board.h | 4908 ++ drivers/input/touchscreen/tp_suspend.h | 109 + .../LX20JS06_A1_CT363_V03_5198_121015.dat | 4096 ++ drivers/input/touchscreen/vtl_ts/Makefile | 9 + drivers/input/touchscreen/vtl_ts/apk.c | 232 + drivers/input/touchscreen/vtl_ts/apk.h | 7 + drivers/input/touchscreen/vtl_ts/chip.c | 860 + drivers/input/touchscreen/vtl_ts/chip.h | 13 + ...07&f317&f316_CT363S_01_V10_F7E9_140212.dat | 2048 + .../lx--js77_97_CT365_V01_E7DA_130419.dat | 2048 + drivers/input/touchscreen/vtl_ts/tp_fw.h | 2051 + drivers/input/touchscreen/vtl_ts/vtl_ts.c | 728 + drivers/input/touchscreen/vtl_ts/vtl_ts.h | 193 + .../wgj36js07_A1_CT363_V01_0187_140306.dat | 2048 + drivers/input/touchscreen/wacom_w9013.c | 510 + drivers/iommu/Kconfig | 3 +- drivers/iommu/dma-iommu-ops.c | 473 + drivers/iommu/dma-iommu.c | 50 + drivers/iommu/iommu.c | 155 +- drivers/iommu/rockchip-iommu.c | 690 +- drivers/irqchip/Kconfig | 5 +- drivers/irqchip/irq-gic-v3-its.c | 60 +- drivers/irqchip/irq-gic-v3.c | 26 + drivers/irqchip/irq-gic-v4.c | 19 + drivers/irqchip/irq-meson-gpio.c | 89 +- drivers/mailbox/rockchip-mailbox.c | 135 +- drivers/media/i2c/Kconfig | 11 + drivers/media/i2c/Makefile | 1 + drivers/media/i2c/gc8034.c | 2827 + drivers/media/i2c/rk628_csi.c | 2696 + drivers/media/i2c/rk628_csi.h | 344 + drivers/media/platform/Kconfig | 3 + drivers/media/platform/Makefile | 3 + drivers/media/platform/rockchip-tsp/Kconfig | 12 + drivers/media/platform/rockchip-tsp/Makefile | 6 + .../platform/rockchip-tsp/rockchip_tsp.c | 1182 + .../platform/rockchip-tsp/rockchip_tsp.h | 230 + drivers/media/platform/rockchip/cif/Kconfig | 40 + drivers/media/platform/rockchip/cif/Makefile | 11 + drivers/media/platform/rockchip/cif/capture.c | 7915 +++ .../media/platform/rockchip/cif/cif-luma.c | 520 + .../media/platform/rockchip/cif/cif-luma.h | 83 + .../media/platform/rockchip/cif/cif-scale.c | 1202 + drivers/media/platform/rockchip/cif/common.c | 187 + drivers/media/platform/rockchip/cif/common.h | 27 + drivers/media/platform/rockchip/cif/dev.c | 1928 + drivers/media/platform/rockchip/cif/dev.h | 786 + drivers/media/platform/rockchip/cif/hw.c | 1186 + drivers/media/platform/rockchip/cif/hw.h | 130 + .../media/platform/rockchip/cif/mipi-csi2.c | 940 + .../media/platform/rockchip/cif/mipi-csi2.h | 159 + drivers/media/platform/rockchip/cif/procfs.c | 406 + drivers/media/platform/rockchip/cif/procfs.h | 25 + drivers/media/platform/rockchip/cif/regs.h | 979 + .../media/platform/rockchip/cif/subdev-itf.c | 635 + .../media/platform/rockchip/cif/subdev-itf.h | 69 + drivers/media/platform/rockchip/cif/version.h | 74 + drivers/media/platform/rockchip/isp/Kconfig | 39 + drivers/media/platform/rockchip/isp/Makefile | 42 + drivers/media/platform/rockchip/isp/bridge.c | 629 + drivers/media/platform/rockchip/isp/bridge.h | 105 + .../media/platform/rockchip/isp/bridge_v20.c | 1094 + .../media/platform/rockchip/isp/bridge_v30.c | 236 + drivers/media/platform/rockchip/isp/capture.c | 1717 + drivers/media/platform/rockchip/isp/capture.h | 295 + .../media/platform/rockchip/isp/capture_v1x.c | 957 + .../media/platform/rockchip/isp/capture_v1x.h | 17 + .../media/platform/rockchip/isp/capture_v20.c | 1969 + .../media/platform/rockchip/isp/capture_v21.c | 1699 + .../media/platform/rockchip/isp/capture_v2x.h | 48 + .../media/platform/rockchip/isp/capture_v30.c | 1495 + .../media/platform/rockchip/isp/capture_v3x.h | 23 + drivers/media/platform/rockchip/isp/common.c | 369 + drivers/media/platform/rockchip/isp/common.h | 212 + drivers/media/platform/rockchip/isp/csi.c | 637 + drivers/media/platform/rockchip/isp/csi.h | 83 + drivers/media/platform/rockchip/isp/dev.c | 913 + drivers/media/platform/rockchip/isp/dev.h | 238 + drivers/media/platform/rockchip/isp/dmarx.c | 1183 + drivers/media/platform/rockchip/isp/dmarx.h | 67 + drivers/media/platform/rockchip/isp/hw.c | 1019 + drivers/media/platform/rockchip/isp/hw.h | 89 + .../platform/rockchip/isp/isp_external.h | 44 + .../media/platform/rockchip/isp/isp_ispp.h | 138 + .../platform/rockchip/isp/isp_mipi_luma.c | 524 + .../platform/rockchip/isp/isp_mipi_luma.h | 79 + .../media/platform/rockchip/isp/isp_params.c | 459 + .../media/platform/rockchip/isp/isp_params.h | 147 + .../platform/rockchip/isp/isp_params_v1x.c | 2352 + .../platform/rockchip/isp/isp_params_v1x.h | 87 + .../platform/rockchip/isp/isp_params_v21.c | 4195 ++ .../platform/rockchip/isp/isp_params_v21.h | 175 + .../platform/rockchip/isp/isp_params_v2x.c | 4466 ++ .../platform/rockchip/isp/isp_params_v2x.h | 359 + .../platform/rockchip/isp/isp_params_v3x.c | 4688 ++ .../platform/rockchip/isp/isp_params_v3x.h | 210 + .../media/platform/rockchip/isp/isp_stats.c | 373 + .../media/platform/rockchip/isp/isp_stats.h | 89 + .../platform/rockchip/isp/isp_stats_v1x.c | 420 + .../platform/rockchip/isp/isp_stats_v1x.h | 42 + .../platform/rockchip/isp/isp_stats_v21.c | 1166 + .../platform/rockchip/isp/isp_stats_v21.h | 52 + .../platform/rockchip/isp/isp_stats_v2x.c | 1526 + .../platform/rockchip/isp/isp_stats_v2x.h | 70 + .../platform/rockchip/isp/isp_stats_v3x.c | 1309 + .../platform/rockchip/isp/isp_stats_v3x.h | 52 + drivers/media/platform/rockchip/isp/procfs.c | 805 + drivers/media/platform/rockchip/isp/procfs.h | 21 + drivers/media/platform/rockchip/isp/regs.c | 328 + drivers/media/platform/rockchip/isp/regs.h | 2031 + .../media/platform/rockchip/isp/regs_v2x.h | 2717 + .../media/platform/rockchip/isp/regs_v3x.h | 1822 + drivers/media/platform/rockchip/isp/rkisp.c | 3578 + drivers/media/platform/rockchip/isp/rkisp.h | 197 + .../platform/rockchip/isp/rkisp_tb_helper.c | 266 + .../platform/rockchip/isp/rkisp_tb_helper.h | 33 + drivers/media/platform/rockchip/isp/version.h | 224 + .../platform/rockchip/isp/videobuf2-rdma-sg.c | 608 + drivers/media/platform/rockchip/isp1/Kconfig | 12 + drivers/media/platform/rockchip/isp1/Makefile | 10 + .../media/platform/rockchip/isp1/capture.c | 2313 + .../media/platform/rockchip/isp1/capture.h | 219 + drivers/media/platform/rockchip/isp1/common.h | 133 + drivers/media/platform/rockchip/isp1/dev.c | 1200 + drivers/media/platform/rockchip/isp1/dev.h | 173 + drivers/media/platform/rockchip/isp1/dmarx.c | 687 + drivers/media/platform/rockchip/isp1/dmarx.h | 29 + .../media/platform/rockchip/isp1/isp_params.c | 2655 + .../media/platform/rockchip/isp1/isp_params.h | 151 + .../media/platform/rockchip/isp1/isp_stats.c | 684 + .../media/platform/rockchip/isp1/isp_stats.h | 112 + drivers/media/platform/rockchip/isp1/regs.c | 244 + drivers/media/platform/rockchip/isp1/regs.h | 1994 + drivers/media/platform/rockchip/isp1/rkisp1.c | 1873 + drivers/media/platform/rockchip/isp1/rkisp1.h | 163 + .../media/platform/rockchip/isp1/version.h | 61 + drivers/media/platform/rockchip/ispp/Kconfig | 31 + drivers/media/platform/rockchip/ispp/Makefile | 23 + drivers/media/platform/rockchip/ispp/common.c | 595 + drivers/media/platform/rockchip/ispp/common.h | 145 + drivers/media/platform/rockchip/ispp/dev.c | 385 + drivers/media/platform/rockchip/ispp/dev.h | 60 + drivers/media/platform/rockchip/ispp/fec.c | 480 + drivers/media/platform/rockchip/ispp/fec.h | 27 + drivers/media/platform/rockchip/ispp/hw.c | 510 + drivers/media/platform/rockchip/ispp/hw.h | 67 + drivers/media/platform/rockchip/ispp/ispp.c | 590 + drivers/media/platform/rockchip/ispp/ispp.h | 50 + drivers/media/platform/rockchip/ispp/params.c | 427 + drivers/media/platform/rockchip/ispp/params.h | 69 + .../media/platform/rockchip/ispp/params_v10.c | 739 + .../media/platform/rockchip/ispp/params_v20.c | 209 + drivers/media/platform/rockchip/ispp/procfs.c | 143 + drivers/media/platform/rockchip/ispp/procfs.h | 21 + drivers/media/platform/rockchip/ispp/regs.h | 523 + drivers/media/platform/rockchip/ispp/stats.c | 388 + drivers/media/platform/rockchip/ispp/stats.h | 53 + drivers/media/platform/rockchip/ispp/stream.c | 2016 + drivers/media/platform/rockchip/ispp/stream.h | 280 + .../media/platform/rockchip/ispp/stream_v10.c | 1733 + .../media/platform/rockchip/ispp/stream_v20.c | 463 + .../media/platform/rockchip/ispp/version.h | 138 + drivers/media/usb/uvc/uvc_driver.c | 7 +- drivers/media/v4l2-core/v4l2-async.c | 54 + drivers/mfd/rk808.c | 928 +- drivers/mmc/core/block.c | 3 + drivers/mmc/core/block.h | 1 + drivers/mmc/core/core.h | 2 + drivers/mmc/core/host.c | 45 + drivers/mmc/core/sdio.c | 424 +- drivers/mmc/host/Makefile | 1 + drivers/mmc/host/dw_mmc-rockchip.c | 95 +- drivers/mmc/host/dw_mmc.h | 2 + drivers/mmc/host/rk_sdmmc.h | 387 + drivers/mmc/host/rk_sdmmc_ops.c | 194 + drivers/mmc/host/rk_sdmmc_ops.h | 11 + drivers/mmc/host/sdhci-of-dwcmshc.c | 260 +- drivers/mtd/nand/raw/Kconfig | 12 + drivers/mtd/nand/raw/Makefile | 1 + .../mtd/nand/raw/rockchip-nand-controller.c | 1495 + .../aquantia/atlantic/hw_atl/hw_atl_utils.c | 10 - drivers/net/ethernet/stmicro/stmmac/Makefile | 3 +- .../ethernet/stmicro/stmmac/dwmac-rk-tool.c | 1613 + .../ethernet/stmicro/stmmac/dwmac-rk-tool.h | 25 + .../net/ethernet/stmicro/stmmac/dwmac-rk.c | 932 +- drivers/net/wireless/Kconfig | 1 + drivers/net/wireless/Makefile | 1 + drivers/net/wireless/marvell/mwifiex/usb.c | 3 +- drivers/net/wireless/rockchip_wlan/Kconfig | 39 + drivers/net/wireless/rockchip_wlan/Makefile | 3 + .../net/wireless/rockchip_wlan/rkwifi/Kconfig | 33 + .../wireless/rockchip_wlan/rkwifi/Makefile | 10 + .../rockchip_wlan/rkwifi/bcmdhd_wifi6/Kconfig | 59 + .../rkwifi/bcmdhd_wifi6/Makefile | 258 + .../rkwifi/bcmdhd_wifi6/aiutils.c | 2099 + .../rkwifi/bcmdhd_wifi6/bcm_app_utils.c | 1033 + .../rkwifi/bcmdhd_wifi6/bcmbloom.c | 243 + .../rkwifi/bcmdhd_wifi6/bcmevent.c | 396 + .../rkwifi/bcmdhd_wifi6/bcmsdh.c | 893 + .../rkwifi/bcmdhd_wifi6/bcmsdh_linux.c | 524 + .../rkwifi/bcmdhd_wifi6/bcmsdh_sdmmc.c | 1801 + .../rkwifi/bcmdhd_wifi6/bcmsdh_sdmmc_linux.c | 381 + .../rkwifi/bcmdhd_wifi6/bcmsdspi_linux.c | 438 + .../rkwifi/bcmdhd_wifi6/bcmspibrcm.c | 1800 + .../rkwifi/bcmdhd_wifi6/bcmstdlib_s.c | 314 + .../rkwifi/bcmdhd_wifi6/bcmutils.c | 4379 ++ .../rkwifi/bcmdhd_wifi6/bcmwifi_channels.c | 1494 + .../rkwifi/bcmdhd_wifi6/bcmwifi_channels.h | 816 + .../rkwifi/bcmdhd_wifi6/bcmwifi_rates.h | 832 + .../rkwifi/bcmdhd_wifi6/bcmwifi_rspec.h | 213 + .../rkwifi/bcmdhd_wifi6/bcmxtlv.c | 623 + .../rockchip_wlan/rkwifi/bcmdhd_wifi6/dbus.c | 2903 + .../rkwifi/bcmdhd_wifi6/dbus_usb.c | 1173 + .../rkwifi/bcmdhd_wifi6/dbus_usb_linux.c | 3404 + .../rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd.h | 3433 + .../rkwifi/bcmdhd_wifi6/dhd_bus.h | 357 + .../rkwifi/bcmdhd_wifi6/dhd_buzzz.h | 38 + .../rkwifi/bcmdhd_wifi6/dhd_ccode.c | 274 + .../rkwifi/bcmdhd_wifi6/dhd_cdc.c | 981 + .../rkwifi/bcmdhd_wifi6/dhd_cfg80211.c | 332 + .../rkwifi/bcmdhd_wifi6/dhd_cfg80211.h | 55 + .../rkwifi/bcmdhd_wifi6/dhd_common.c | 7496 +++ .../rkwifi/bcmdhd_wifi6/dhd_config.c | 4730 ++ .../rkwifi/bcmdhd_wifi6/dhd_config.h | 384 + .../rkwifi/bcmdhd_wifi6/dhd_csi.c | 220 + .../rkwifi/bcmdhd_wifi6/dhd_csi.h | 77 + .../rkwifi/bcmdhd_wifi6/dhd_custom_gpio.c | 279 + .../rkwifi/bcmdhd_wifi6/dhd_custom_hikey.c | 294 + .../bcmdhd_wifi6/dhd_custom_memprealloc.c | 562 + .../rkwifi/bcmdhd_wifi6/dhd_dbg.h | 367 + .../rkwifi/bcmdhd_wifi6/dhd_dbg_ring.c | 426 + .../rkwifi/bcmdhd_wifi6/dhd_dbg_ring.h | 141 + .../rkwifi/bcmdhd_wifi6/dhd_debug.c | 2411 + .../rkwifi/bcmdhd_wifi6/dhd_debug.h | 853 + .../rkwifi/bcmdhd_wifi6/dhd_debug_linux.c | 516 + .../rkwifi/bcmdhd_wifi6/dhd_flowring.c | 1238 + .../rkwifi/bcmdhd_wifi6/dhd_flowring.h | 289 + .../rkwifi/bcmdhd_wifi6/dhd_gpio.c | 350 + .../rkwifi/bcmdhd_wifi6/dhd_ip.c | 1429 + .../rkwifi/bcmdhd_wifi6/dhd_ip.h | 97 + .../rkwifi/bcmdhd_wifi6/dhd_linux.c | 22453 +++++++ .../rkwifi/bcmdhd_wifi6/dhd_linux.h | 448 + .../rkwifi/bcmdhd_wifi6/dhd_linux_exportfs.c | 1420 + .../rkwifi/bcmdhd_wifi6/dhd_linux_lb.c | 1324 + .../rkwifi/bcmdhd_wifi6/dhd_linux_pktdump.c | 1472 + .../rkwifi/bcmdhd_wifi6/dhd_linux_pktdump.h | 125 + .../rkwifi/bcmdhd_wifi6/dhd_linux_platdev.c | 1041 + .../rkwifi/bcmdhd_wifi6/dhd_linux_priv.h | 432 + .../rkwifi/bcmdhd_wifi6/dhd_linux_sched.c | 52 + .../rkwifi/bcmdhd_wifi6/dhd_linux_wq.c | 397 + .../rkwifi/bcmdhd_wifi6/dhd_linux_wq.h | 93 + .../rkwifi/bcmdhd_wifi6/dhd_mschdbg.c | 789 + .../rkwifi/bcmdhd_wifi6/dhd_mschdbg.h | 40 + .../rkwifi/bcmdhd_wifi6/dhd_msgbuf.c | 11162 ++++ .../rkwifi/bcmdhd_wifi6/dhd_pcie.c | 11618 ++++ .../rkwifi/bcmdhd_wifi6/dhd_pcie.h | 596 + .../rkwifi/bcmdhd_wifi6/dhd_pcie_linux.c | 2738 + .../rkwifi/bcmdhd_wifi6/dhd_pno.c | 4458 ++ .../rkwifi/bcmdhd_wifi6/dhd_pno.h | 590 + .../rkwifi/bcmdhd_wifi6/dhd_proto.h | 243 + .../rkwifi/bcmdhd_wifi6/dhd_rtt.c | 4103 ++ .../rkwifi/bcmdhd_wifi6/dhd_rtt.h | 537 + .../rkwifi/bcmdhd_wifi6/dhd_sdio.c | 10489 +++ .../rkwifi/bcmdhd_wifi6/dhd_static_buf.c | 586 + .../rkwifi/bcmdhd_wifi6/dhd_wlfc.c | 4616 ++ .../rkwifi/bcmdhd_wifi6/dhd_wlfc.h | 796 + .../rkwifi/bcmdhd_wifi6/dngl_stats.h | 387 + .../rkwifi/bcmdhd_wifi6/dngl_wlhdr.h | 44 + .../rockchip_wlan/rkwifi/bcmdhd_wifi6/frag.c | 113 + .../rockchip_wlan/rkwifi/bcmdhd_wifi6/frag.h | 39 + .../rkwifi/bcmdhd_wifi6/hdf_public_ap6275s.h | 125 + .../rkwifi/bcmdhd_wifi6/hnd_pktpool.c | 1428 + .../rkwifi/bcmdhd_wifi6/hnd_pktq.c | 1429 + .../rkwifi/bcmdhd_wifi6/hndlhl.c | 538 + .../rkwifi/bcmdhd_wifi6/hndmem.c | 430 + .../rkwifi/bcmdhd_wifi6/hndpmu.c | 789 + .../rkwifi/bcmdhd_wifi6/include/802.11.h | 5364 ++ .../rkwifi/bcmdhd_wifi6/include/802.11e.h | 140 + .../rkwifi/bcmdhd_wifi6/include/802.11s.h | 335 + .../rkwifi/bcmdhd_wifi6/include/802.1d.h | 54 + .../rkwifi/bcmdhd_wifi6/include/802.3.h | 56 + .../rkwifi/bcmdhd_wifi6/include/aidmp.h | 430 + .../rkwifi/bcmdhd_wifi6/include/bcm_cfg.h | 33 + .../bcmdhd_wifi6/include/bcm_mpool_pub.h | 351 + .../rkwifi/bcmdhd_wifi6/include/bcm_ring.h | 614 + .../rkwifi/bcmdhd_wifi6/include/bcmarp.h | 91 + .../rkwifi/bcmdhd_wifi6/include/bcmbloom.h | 80 + .../rkwifi/bcmdhd_wifi6/include/bcmcdc.h | 122 + .../rkwifi/bcmdhd_wifi6/include/bcmdefs.h | 616 + .../rkwifi/bcmdhd_wifi6/include/bcmdevs.h | 945 + .../rkwifi/bcmdhd_wifi6/include/bcmdhcp.h | 93 + .../rkwifi/bcmdhd_wifi6/include/bcmendian.h | 417 + .../rkwifi/bcmdhd_wifi6/include/bcmeth.h | 116 + .../rkwifi/bcmdhd_wifi6/include/bcmevent.h | 1297 + .../rkwifi/bcmdhd_wifi6/include/bcmicmp.h | 90 + .../rkwifi/bcmdhd_wifi6/include/bcmiov.h | 357 + .../rkwifi/bcmdhd_wifi6/include/bcmip.h | 251 + .../rkwifi/bcmdhd_wifi6/include/bcmipv6.h | 162 + .../rkwifi/bcmdhd_wifi6/include/bcmmsgbuf.h | 1395 + .../rkwifi/bcmdhd_wifi6/include/bcmnvram.h | 329 + .../rkwifi/bcmdhd_wifi6/include/bcmpcie.h | 537 + .../rkwifi/bcmdhd_wifi6/include/bcmpcispi.h | 182 + .../rkwifi/bcmdhd_wifi6/include/bcmperf.h | 40 + .../rkwifi/bcmdhd_wifi6/include/bcmsdbus.h | 184 + .../rkwifi/bcmdhd_wifi6/include/bcmsdh.h | 279 + .../bcmdhd_wifi6/include/bcmsdh_sdmmc.h | 130 + .../rkwifi/bcmdhd_wifi6/include/bcmsdpcm.h | 310 + .../rkwifi/bcmdhd_wifi6/include/bcmsdspi.h | 139 + .../rkwifi/bcmdhd_wifi6/include/bcmsdstd.h | 282 + .../rkwifi/bcmdhd_wifi6/include/bcmspi.h | 44 + .../rkwifi/bcmdhd_wifi6/include/bcmspibrcm.h | 168 + .../rkwifi/bcmdhd_wifi6/include/bcmsrom_fmt.h | 1014 + .../rkwifi/bcmdhd_wifi6/include/bcmsrom_tbl.h | 1459 + .../rkwifi/bcmdhd_wifi6/include/bcmstdlib_s.h | 52 + .../rkwifi/bcmdhd_wifi6/include/bcmtcp.h | 93 + .../rkwifi/bcmdhd_wifi6/include/bcmtlv.h | 343 + .../rkwifi/bcmdhd_wifi6/include/bcmudp.h | 61 + .../rkwifi/bcmdhd_wifi6/include/bcmutils.h | 1342 + .../bcmdhd_wifi6/include/brcm_nl80211.h | 73 + .../rkwifi/bcmdhd_wifi6/include/dbus.h | 599 + .../rkwifi/bcmdhd_wifi6/include/dhd_daemon.h | 67 + .../rkwifi/bcmdhd_wifi6/include/dhdioctl.h | 278 + .../rkwifi/bcmdhd_wifi6/include/dnglevent.h | 142 + .../rkwifi/bcmdhd_wifi6/include/eapol.h | 283 + .../rkwifi/bcmdhd_wifi6/include/epivers.h | 52 + .../rkwifi/bcmdhd_wifi6/include/etd.h | 626 + .../rkwifi/bcmdhd_wifi6/include/ethernet.h | 225 + .../rkwifi/bcmdhd_wifi6/include/event_log.h | 423 + .../bcmdhd_wifi6/include/event_log_payload.h | 1017 + .../bcmdhd_wifi6/include/event_log_set.h | 124 + .../bcmdhd_wifi6/include/event_log_tag.h | 479 + .../rkwifi/bcmdhd_wifi6/include/event_trace.h | 124 + .../rkwifi/bcmdhd_wifi6/include/fils.h | 295 + .../rkwifi/bcmdhd_wifi6/include/hnd_armtrap.h | 90 + .../rkwifi/bcmdhd_wifi6/include/hnd_cons.h | 87 + .../rkwifi/bcmdhd_wifi6/include/hnd_debug.h | 169 + .../rkwifi/bcmdhd_wifi6/include/hnd_pktpool.h | 244 + .../rkwifi/bcmdhd_wifi6/include/hnd_pktq.h | 326 + .../rkwifi/bcmdhd_wifi6/include/hnd_trap.h | 40 + .../rkwifi/bcmdhd_wifi6/include/hndchipc.h | 54 + .../rkwifi/bcmdhd_wifi6/include/hndlhl.h | 62 + .../rkwifi/bcmdhd_wifi6/include/hndmem.h | 81 + .../rkwifi/bcmdhd_wifi6/include/hndoobr.h | 76 + .../rkwifi/bcmdhd_wifi6/include/hndpmu.h | 80 + .../rkwifi/bcmdhd_wifi6/include/hndsoc.h | 351 + .../rkwifi/bcmdhd_wifi6/include/linux_osl.h | 636 + .../rkwifi/bcmdhd_wifi6/include/linux_pkt.h | 236 + .../rkwifi/bcmdhd_wifi6/include/linuxver.h | 900 + .../rkwifi/bcmdhd_wifi6/include/lpflags.h | 46 + .../rkwifi/bcmdhd_wifi6/include/mbo.h | 286 + .../rkwifi/bcmdhd_wifi6/include/miniopt.h | 80 + .../rkwifi/bcmdhd_wifi6/include/msf.h | 67 + .../rkwifi/bcmdhd_wifi6/include/msgtrace.h | 63 + .../rkwifi/bcmdhd_wifi6/include/nan.h | 1559 + .../rkwifi/bcmdhd_wifi6/include/osl.h | 378 + .../rkwifi/bcmdhd_wifi6/include/osl_decl.h | 38 + .../rkwifi/bcmdhd_wifi6/include/osl_ext.h | 766 + .../rkwifi/bcmdhd_wifi6/include/p2p.h | 702 + .../bcmdhd_wifi6/include/packed_section_end.h | 60 + .../include/packed_section_start.h | 105 + .../rkwifi/bcmdhd_wifi6/include/pcicfg.h | 402 + .../rkwifi/bcmdhd_wifi6/include/pcie_core.h | 1193 + .../rkwifi/bcmdhd_wifi6/include/rte_ioctl.h | 104 + .../rkwifi/bcmdhd_wifi6/include/sbchipc.h | 4619 ++ .../rkwifi/bcmdhd_wifi6/include/sbconfig.h | 286 + .../rkwifi/bcmdhd_wifi6/include/sbgci.h | 274 + .../rkwifi/bcmdhd_wifi6/include/sbhndarm.h | 142 + .../rkwifi/bcmdhd_wifi6/include/sbhnddma.h | 450 + .../rkwifi/bcmdhd_wifi6/include/sbpcmcia.h | 138 + .../rkwifi/bcmdhd_wifi6/include/sbsdio.h | 189 + .../rkwifi/bcmdhd_wifi6/include/sbsdpcmdev.h | 312 + .../rkwifi/bcmdhd_wifi6/include/sbsocram.h | 205 + .../rkwifi/bcmdhd_wifi6/include/sbsysmem.h | 181 + .../rkwifi/bcmdhd_wifi6/include/sdio.h | 626 + .../rkwifi/bcmdhd_wifi6/include/sdioh.h | 451 + .../rkwifi/bcmdhd_wifi6/include/sdiovar.h | 125 + .../rkwifi/bcmdhd_wifi6/include/sdspi.h | 79 + .../rkwifi/bcmdhd_wifi6/include/siutils.h | 810 + .../rkwifi/bcmdhd_wifi6/include/spid.h | 169 + .../rkwifi/bcmdhd_wifi6/include/trxhdr.h | 96 + .../rkwifi/bcmdhd_wifi6/include/typedefs.h | 368 + .../rkwifi/bcmdhd_wifi6/include/usbrdl.h | 135 + .../rkwifi/bcmdhd_wifi6/include/vlan.h | 98 + .../rkwifi/bcmdhd_wifi6/include/wlfc_proto.h | 414 + .../rkwifi/bcmdhd_wifi6/include/wlioctl.h | 20752 ++++++ .../bcmdhd_wifi6/include/wlioctl_defs.h | 2466 + .../bcmdhd_wifi6/include/wlioctl_utils.h | 62 + .../rkwifi/bcmdhd_wifi6/include/wpa.h | 312 + .../rkwifi/bcmdhd_wifi6/include/wps.h | 386 + .../rkwifi/bcmdhd_wifi6/linux_osl.c | 2124 + .../rkwifi/bcmdhd_wifi6/linux_osl_priv.h | 185 + .../rkwifi/bcmdhd_wifi6/linux_pkt.c | 694 + .../rkwifi/bcmdhd_wifi6/otpdefs.h | 122 + .../rkwifi/bcmdhd_wifi6/pcie_core.c | 159 + .../rkwifi/bcmdhd_wifi6/sbutils.c | 1094 + .../rkwifi/bcmdhd_wifi6/siutils.c | 3801 ++ .../rkwifi/bcmdhd_wifi6/siutils_priv.h | 356 + .../rkwifi/bcmdhd_wifi6/wl_android.c | 6644 ++ .../rkwifi/bcmdhd_wifi6/wl_android.h | 420 + .../rkwifi/bcmdhd_wifi6/wl_android_ext.c | 7974 +++ .../rkwifi/bcmdhd_wifi6/wl_cfg80211.c | 25039 +++++++ .../rkwifi/bcmdhd_wifi6/wl_cfg80211.h | 2451 + .../rkwifi/bcmdhd_wifi6/wl_cfg_btcoex.c | 569 + .../rkwifi/bcmdhd_wifi6/wl_cfgnan.c | 8182 +++ .../rkwifi/bcmdhd_wifi6/wl_cfgnan.h | 899 + .../rkwifi/bcmdhd_wifi6/wl_cfgp2p.c | 2792 + .../rkwifi/bcmdhd_wifi6/wl_cfgp2p.h | 458 + .../rkwifi/bcmdhd_wifi6/wl_cfgscan.c | 3348 + .../rkwifi/bcmdhd_wifi6/wl_cfgscan.h | 101 + .../rkwifi/bcmdhd_wifi6/wl_cfgvendor.c | 8751 +++ .../rkwifi/bcmdhd_wifi6/wl_cfgvendor.h | 721 + .../rkwifi/bcmdhd_wifi6/wl_dbg.h | 381 + .../rkwifi/bcmdhd_wifi6/wl_escan.c | 1692 + .../rkwifi/bcmdhd_wifi6/wl_escan.h | 78 + .../rkwifi/bcmdhd_wifi6/wl_event.c | 506 + .../rkwifi/bcmdhd_wifi6/wl_ext_genl.c | 569 + .../rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_iw.c | 4342 ++ .../rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_iw.h | 178 + .../rkwifi/bcmdhd_wifi6/wl_linux_mon.c | 474 + .../rkwifi/bcmdhd_wifi6/wl_roam.c | 375 + .../rkwifi/bcmdhd_wifi6/wldev_common.c | 535 + .../rkwifi/bcmdhd_wifi6/wldev_common.h | 135 + .../rockchip_wlan/rkwifi/rk_wifi_config.c | 30 + drivers/nvmem/core.c | 4 + drivers/nvmem/rockchip-efuse.c | 348 +- drivers/nvmem/rockchip-otp.c | 22 +- drivers/of/Kconfig | 7 + drivers/opp/debugfs.c | 44 + drivers/opp/of.c | 2 +- drivers/pci/controller/Makefile | 7 +- drivers/pci/controller/dwc/Kconfig | 9 + drivers/pci/controller/dwc/Makefile | 1 + drivers/pci/controller/dwc/pcie-designware.h | 1 + drivers/pci/controller/dwc/pcie-dw-rockchip.c | 1789 + drivers/pci/controller/pcie-rockchip.c | 5 + drivers/pci/controller/rockchip-pcie-dma.c | 763 + drivers/pci/controller/rockchip-pcie-dma.h | 199 + drivers/phy/rockchip/Kconfig | 100 + drivers/phy/rockchip/Makefile | 15 +- .../rockchip/phy-rockchip-csi2-dphy-common.h | 114 + .../phy/rockchip/phy-rockchip-csi2-dphy-hw.c | 1078 + drivers/phy/rockchip/phy-rockchip-csi2-dphy.c | 658 + .../phy/rockchip/phy-rockchip-inno-combphy.c | 1024 + .../phy/rockchip/phy-rockchip-inno-dsidphy.c | 434 +- .../phy/rockchip/phy-rockchip-inno-hdmi-phy.c | 1335 + .../rockchip/phy-rockchip-inno-mipi-dphy.c | 853 + drivers/phy/rockchip/phy-rockchip-inno-usb2.c | 2597 +- drivers/phy/rockchip/phy-rockchip-inno-usb3.c | 1132 + .../phy-rockchip-inno-video-combo-phy.c | 874 + .../rockchip/phy-rockchip-inno-video-phy.c | 245 + drivers/phy/rockchip/phy-rockchip-mipi-rx.c | 1800 + .../rockchip/phy-rockchip-naneng-combphy.c | 850 + .../phy/rockchip/phy-rockchip-naneng-edp.c | 400 + .../phy/rockchip/phy-rockchip-naneng-usb2.c | 1929 + drivers/phy/rockchip/phy-rockchip-pcie.c | 12 +- .../phy/rockchip/phy-rockchip-samsung-dcphy.c | 1973 + .../phy-rockchip-samsung-hdptx-hdmi.c | 1944 + .../phy/rockchip/phy-rockchip-samsung-hdptx.c | 1188 + .../phy/rockchip/phy-rockchip-snps-pcie3.c | 277 + drivers/phy/rockchip/phy-rockchip-typec.c | 733 +- drivers/phy/rockchip/phy-rockchip-usb.c | 628 +- drivers/phy/rockchip/phy-rockchip-usbdp.c | 1458 + drivers/pinctrl/Kconfig | 7 +- drivers/pinctrl/pinctrl-rk628.c | 1768 + drivers/pinctrl/pinctrl-rk805.c | 350 +- drivers/pinctrl/pinctrl-rockchip.c | 1753 +- drivers/pinctrl/pinctrl-rockchip.h | 277 + drivers/power/ec_battery.c | 466 + drivers/power/reset/gpio-poweroff.c | 1 - drivers/power/reset/reboot-mode.h | 7 + drivers/power/supply/Kconfig | 46 +- drivers/power/supply/Makefile | 7 + drivers/power/supply/ab8500_btemp.c | 1 - drivers/power/supply/ab8500_charger.c | 19 +- drivers/power/supply/ab8500_fg.c | 1 - drivers/power/supply/axp288_fuel_gauge.c | 4 +- drivers/power/supply/bq25700_charger.c | 2105 + drivers/power/supply/charger-manager.c | 1 - drivers/power/supply/cw2015_battery.c | 4 +- drivers/power/supply/max17042_battery.c | 10 +- drivers/power/supply/power_supply_core.c | 78 + drivers/power/supply/power_supply_sysfs.c | 1 + drivers/power/supply/rk816_battery.c | 5172 ++ drivers/power/supply/rk816_battery.h | 286 + drivers/power/supply/rk817_battery.c | 3418 + drivers/power/supply/rk817_charger.c | 1798 + drivers/power/supply/rk818_battery.c | 3622 + drivers/power/supply/rk818_battery.h | 168 + drivers/power/supply/rk818_charger.c | 1889 + drivers/power/supply/rt5033_battery.c | 7 - drivers/power/supply/sc2731_charger.c | 1 - drivers/power/supply/sc27xx_fuel_gauge.c | 1 - drivers/power/supply/smb347-charger.c | 10 - drivers/power/sy6982c_charger.c | 401 + drivers/power/universal_charger.c | 419 + drivers/pwm/Kconfig | 6 + drivers/pwm/pwm-rockchip.c | 109 +- drivers/pwm/sysfs.c | 43 + drivers/regulator/Kconfig | 6 + drivers/regulator/Makefile | 4 + drivers/regulator/core.c | 345 + drivers/regulator/fan53555.c | 399 +- drivers/regulator/lp8752.c | 316 + drivers/regulator/mp8865-regulator.c | 248 + drivers/regulator/of_regulator.c | 8 +- drivers/regulator/rk808-regulator.c | 534 +- drivers/regulator/scmi-regulator.c | 421 + drivers/regulator/syr82x.c | 520 + drivers/regulator/tps549b22-regulator.c | 437 + drivers/regulator/xz3216.c | 404 + drivers/rk_nand/Kconfig | 10 + drivers/rk_nand/Makefile | 9 + drivers/rk_nand/rk_ftl_api.h | 41 + drivers/rk_nand/rk_ftl_arm_v7.S | 30165 +++++++++ drivers/rk_nand/rk_ftl_arm_v7_thumb.S | 30192 +++++++++ drivers/rk_nand/rk_ftl_arm_v8.S | 27968 ++++++++ drivers/rk_nand/rk_ftlv5_arm32.S | 27612 ++++++++ drivers/rk_nand/rk_ftlv5_arm64.S | 25632 +++++++ drivers/rk_nand/rk_nand_base.c | 456 + drivers/rk_nand/rk_nand_base.h | 46 + drivers/rk_nand/rk_nand_blk.c | 711 + drivers/rk_nand/rk_nand_blk.h | 67 + drivers/rk_nand/rk_zftl_arm32.S | 37095 +++++++++++ drivers/rk_nand/rk_zftl_arm64.S | 35207 ++++++++++ drivers/rkflash/Kconfig | 80 + drivers/rkflash/Makefile | 14 + drivers/rkflash/flash.c | 505 + drivers/rkflash/flash.h | 132 + drivers/rkflash/flash_com.h | 77 + drivers/rkflash/nand_boot.c | 83 + drivers/rkflash/nandc.c | 467 + drivers/rkflash/nandc.h | 346 + drivers/rkflash/rk_sftl.h | 48 + drivers/rkflash/rk_sftl_arm_v7.S | 15787 +++++ drivers/rkflash/rk_sftl_arm_v7_thumb.S | 15586 +++++ drivers/rkflash/rk_sftl_arm_v8.S | 15768 +++++ drivers/rkflash/rkflash_api.h | 58 + drivers/rkflash/rkflash_blk.c | 789 + drivers/rkflash/rkflash_blk.h | 121 + drivers/rkflash/rkflash_debug.c | 116 + drivers/rkflash/rkflash_debug.h | 42 + drivers/rkflash/rknandc_base.c | 192 + drivers/rkflash/rksfc_base.c | 267 + drivers/rkflash/sfc.c | 255 + drivers/rkflash/sfc.h | 224 + drivers/rkflash/sfc_nand.c | 1051 + drivers/rkflash/sfc_nand.h | 133 + drivers/rkflash/sfc_nand_boot.c | 89 + drivers/rkflash/sfc_nand_mtd.c | 423 + drivers/rkflash/sfc_nand_mtd.h | 82 + drivers/rkflash/sfc_nand_mtd_bbt.c | 441 + drivers/rkflash/sfc_nor.c | 783 + drivers/rkflash/sfc_nor.h | 176 + drivers/rkflash/sfc_nor_boot.c | 180 + drivers/rkflash/sfc_nor_mtd.c | 266 + drivers/rkflash/typedef.h | 30 + drivers/soc/rockchip/Kconfig | 125 +- drivers/soc/rockchip/Kconfig.cpu | 62 + drivers/soc/rockchip/Makefile | 14 + drivers/soc/rockchip/flash_vendor_storage.c | 363 + drivers/soc/rockchip/flash_vendor_storage.h | 16 + drivers/soc/rockchip/grf.c | 65 + drivers/soc/rockchip/io-domain.c | 95 +- drivers/soc/rockchip/mtd_vendor_storage.c | 462 + drivers/soc/rockchip/pm_domains.c | 1000 +- drivers/soc/rockchip/rk_camera_sensor_info.h | 421 + drivers/soc/rockchip/rk_fiq_debugger.c | 1051 + drivers/soc/rockchip/rk_vendor_storage.c | 52 + drivers/soc/rockchip/rockchip-cpuinfo.c | 243 + drivers/soc/rockchip/rockchip_debug.c | 423 + drivers/soc/rockchip/rockchip_debug.h | 14 + drivers/soc/rockchip/rockchip_decompress.c | 333 + drivers/soc/rockchip/rockchip_ipa.c | 232 + drivers/soc/rockchip/rockchip_opp_select.c | 1211 + drivers/soc/rockchip/rockchip_pm_config.c | 303 + drivers/soc/rockchip/rockchip_pvtm.c | 985 + drivers/soc/rockchip/rockchip_ramdisk.c | 285 + .../soc/rockchip/rockchip_system_monitor.c | 1698 + .../rockchip/rockchip_thunderboot_crypto.c | 440 + .../soc/rockchip/rockchip_thunderboot_mmc.c | 148 + .../soc/rockchip/rockchip_thunderboot_sfc.c | 141 + drivers/soc/rockchip/sdmmc_vendor_storage.c | 635 + drivers/spi/spi-rockchip.c | 173 +- drivers/spi/spidev.c | 1 + drivers/staging/android/Kconfig | 2 + drivers/staging/android/Makefile | 1 + drivers/staging/android/fiq_debugger/Kconfig | 72 + drivers/staging/android/fiq_debugger/Makefile | 2 + .../android/fiq_debugger/fiq_debugger.c | 1672 + .../android/fiq_debugger/fiq_debugger.h | 73 + .../android/fiq_debugger/fiq_debugger_arm.c | 273 + .../android/fiq_debugger/fiq_debugger_arm64.c | 198 + .../android/fiq_debugger/fiq_debugger_priv.h | 39 + .../fiq_debugger/fiq_debugger_ringbuf.h | 94 + .../android/fiq_debugger/fiq_watchdog.c | 56 + .../android/fiq_debugger/fiq_watchdog.h | 20 + drivers/staging/android/ion/Kconfig | 15 +- drivers/staging/android/ion/Makefile | 6 +- drivers/staging/android/ion/heaps/Kconfig | 15 + drivers/staging/android/ion/heaps/Makefile | 5 + .../android/ion/{ => heaps}/ion_cma_heap.c | 65 +- .../android/ion/{ => heaps}/ion_page_pool.c | 19 +- .../staging/android/ion/heaps/ion_page_pool.h | 67 + .../android/ion/{ => heaps}/ion_system_heap.c | 229 +- drivers/staging/android/ion/ion.c | 592 +- drivers/staging/android/ion/ion.h | 302 - drivers/staging/android/ion/ion_buffer.c | 278 + drivers/staging/android/ion/ion_dma_buf.c | 359 + drivers/staging/android/ion/ion_heap.c | 323 +- drivers/staging/android/ion/ion_private.h | 58 + .../staging/android/ion/ion_protected_heap.c | 526 + drivers/staging/android/ion/ion_trace.h | 55 + drivers/staging/android/ion/rockchip/Makefile | 3 + .../android/ion/rockchip/rockchip_ion.c | 224 + .../ion/rockchip/rockchip_ion_snapshot.c | 145 + drivers/staging/blackbox/Kconfig | 11 + drivers/staging/blackbox/Makefile | 2 + drivers/staging/blackbox/blackbox_storage.c | 2 +- drivers/staging/blackbox/rockchip/Makefile | 4 + .../blackbox/rockchip/system_adapter.c | 312 + .../rockchip/system_adapter_by_memory.c | 313 + drivers/thermal/rockchip_thermal.c | 584 +- drivers/thermal/thermal_core.c | 1 + drivers/thermal/thermal_core.h | 2 - drivers/tty/serial/8250/8250.h | 15 + drivers/tty/serial/8250/8250_core.c | 10 +- drivers/tty/serial/8250/8250_dma.c | 205 +- drivers/tty/serial/8250/8250_dw.c | 92 +- drivers/tty/serial/8250/8250_dwlib.c | 12 + drivers/tty/serial/8250/8250_port.c | 103 +- drivers/tty/vt/keyboard.c | 1 + drivers/usb/core/hub.c | 6 +- drivers/usb/core/quirks.c | 8 + drivers/usb/dwc2/core.h | 4 + drivers/usb/dwc2/gadget.c | 13 + drivers/usb/dwc2/hcd.c | 59 +- drivers/usb/dwc2/hcd_intr.c | 9 +- drivers/usb/dwc2/hcd_queue.c | 8 +- drivers/usb/dwc2/params.c | 10 + drivers/usb/dwc2/platform.c | 119 +- drivers/usb/dwc3/Kconfig | 10 + drivers/usb/dwc3/Makefile | 1 + drivers/usb/dwc3/core.c | 221 +- drivers/usb/dwc3/core.h | 88 +- drivers/usb/dwc3/debug.h | 8 +- drivers/usb/dwc3/debugfs.c | 16 +- drivers/usb/dwc3/drd.c | 44 +- drivers/usb/dwc3/dwc3-imx8mp.c | 364 + drivers/usb/dwc3/dwc3-keystone.c | 9 +- drivers/usb/dwc3/dwc3-pci.c | 69 +- drivers/usb/dwc3/dwc3-qcom.c | 14 + drivers/usb/dwc3/ep0.c | 12 +- drivers/usb/dwc3/gadget.c | 563 +- drivers/usb/dwc3/gadget.h | 6 +- drivers/usb/dwc3/trace.c | 7 + drivers/usb/gadget/Kconfig | 39 + drivers/usb/gadget/composite.c | 104 +- drivers/usb/gadget/configfs.c | 294 +- drivers/usb/gadget/epautoconf.c | 24 + drivers/usb/gadget/function/Makefile | 6 + drivers/usb/gadget/function/f_accessory.c | 1556 + drivers/usb/gadget/function/f_audio_source.c | 1071 + drivers/usb/gadget/function/f_fs.c | 18 +- drivers/usb/gadget/function/f_mass_storage.c | 10 + drivers/usb/gadget/function/f_midi.c | 66 + drivers/usb/gadget/function/f_printer.c | 5 + drivers/usb/gadget/function/f_rndis.c | 26 +- drivers/usb/gadget/function/f_uac1.c | 0 drivers/usb/gadget/function/f_uac1_legacy.c | 6 + drivers/usb/gadget/function/f_uac2.c | 395 +- drivers/usb/gadget/function/f_uvc.c | 448 +- drivers/usb/gadget/function/u_audio.c | 368 +- drivers/usb/gadget/function/u_audio.h | 12 + drivers/usb/gadget/function/u_ether.c | 33 +- drivers/usb/gadget/function/u_ether.h | 12 + .../usb/gadget/function/u_ether_configfs.h | 15 +- drivers/usb/gadget/function/u_serial.c | 8 +- drivers/usb/gadget/function/u_uac.h | 194 + drivers/usb/gadget/function/u_uac1.h | 0 drivers/usb/gadget/function/u_uac1_legacy.c | 4 + drivers/usb/gadget/function/u_uac2.h | 4 + drivers/usb/gadget/function/u_uvc.h | 9 +- drivers/usb/gadget/function/uvc.h | 9 +- drivers/usb/gadget/function/uvc_configfs.c | 358 +- drivers/usb/gadget/function/uvc_queue.c | 8 + drivers/usb/gadget/function/uvc_v4l2.c | 24 +- drivers/usb/gadget/function/uvc_video.c | 40 +- drivers/usb/gadget/legacy/audio.c | 0 drivers/usb/gadget/legacy/webcam.c | 2 + drivers/usb/gadget/udc/core.c | 113 +- drivers/usb/gadget/udc/snps_udc_core.c | 30 +- drivers/usb/gadget/udc/trace.c | 3 + drivers/usb/gadget/udc/udc-xilinx.c | 2 +- drivers/usb/host/Makefile | 6 +- drivers/usb/host/ehci-platform.c | 49 + drivers/usb/host/ohci-platform.c | 4 +- drivers/usb/host/xhci-hub.c | 7 + drivers/usb/host/xhci-mem.c | 174 +- drivers/usb/host/xhci-mtk-sch.c | 244 +- drivers/usb/host/xhci-mtk.c | 19 +- drivers/usb/host/xhci-mtk.h | 40 +- drivers/usb/host/xhci-pci.c | 4 +- drivers/usb/host/xhci-plat.c | 43 +- drivers/usb/host/xhci-plat.h | 8 + drivers/usb/host/xhci-ring.c | 905 +- drivers/usb/host/xhci-trace.c | 3 + drivers/usb/host/xhci.c | 228 +- drivers/usb/host/xhci.h | 90 +- drivers/usb/storage/scsiglue.c | 4 + drivers/usb/storage/unusual_devs.h | 6 + drivers/usb/storage/unusual_uas.h | 18 + drivers/usb/typec/altmodes/Kconfig | 1 + drivers/usb/typec/altmodes/displayport.c | 68 +- drivers/usb/typec/class.c | 450 +- drivers/usb/typec/mux/intel_pmc_mux.c | 17 +- drivers/usb/typec/tcpm/Kconfig | 7 + drivers/usb/typec/tcpm/Makefile | 1 + drivers/usb/typec/tcpm/fusb302.c | 29 +- drivers/usb/typec/tcpm/tcpci.c | 176 +- drivers/usb/typec/tcpm/tcpci.h | 47 +- drivers/usb/typec/tcpm/tcpci_husb311.c | 298 + drivers/usb/typec/tcpm/tcpci_maxim.c | 70 +- drivers/usb/typec/tcpm/tcpm.c | 1308 +- drivers/usb/typec/tcpm/wcove.c | 3 +- drivers/usb/typec/ucsi/Kconfig | 1 + drivers/usb/typec/ucsi/displayport.c | 32 +- drivers/usb/typec/ucsi/ucsi.c | 56 +- drivers/usb/typec/ucsi/ucsi.h | 4 + drivers/video/Kconfig | 5 + drivers/video/Makefile | 1 + drivers/video/backlight/pwm_bl.c | 2 + drivers/video/rockchip/Kconfig | 5 + drivers/video/rockchip/Makefile | 5 + drivers/video/rockchip/iep/Kconfig | 10 + drivers/video/rockchip/iep/Makefile | 4 + .../video/rockchip/iep/hw_iep_config_addr.h | 99 + drivers/video/rockchip/iep/hw_iep_reg.c | 1530 + drivers/video/rockchip/iep/hw_iep_reg.h | 525 + drivers/video/rockchip/iep/iep.h | 276 + drivers/video/rockchip/iep/iep_drv.c | 1320 + drivers/video/rockchip/iep/iep_drv.h | 159 + drivers/video/rockchip/iep/iep_iommu_drm.c | 465 + drivers/video/rockchip/iep/iep_iommu_ops.c | 244 + drivers/video/rockchip/iep/iep_iommu_ops.h | 121 + drivers/video/rockchip/mpp/Kconfig | 68 + drivers/video/rockchip/mpp/Makefile | 28 + .../video/rockchip/mpp/hack/mpp_hack_px30.c | 230 + .../video/rockchip/mpp/hack/mpp_hack_px30.h | 27 + .../mpp/hack/mpp_rkvdec2_hack_rk3568.c | 732 + .../mpp/hack/mpp_rkvdec2_link_hack_rk3568.c | 213 + drivers/video/rockchip/mpp/mpp_common.c | 2173 + drivers/video/rockchip/mpp/mpp_common.h | 784 + drivers/video/rockchip/mpp/mpp_debug.h | 127 + drivers/video/rockchip/mpp/mpp_iep2.c | 1019 + drivers/video/rockchip/mpp/mpp_iommu.c | 493 + drivers/video/rockchip/mpp/mpp_iommu.h | 140 + drivers/video/rockchip/mpp/mpp_jpgdec.c | 648 + drivers/video/rockchip/mpp/mpp_rkvdec.c | 1949 + drivers/video/rockchip/mpp/mpp_rkvdec2.c | 1277 + drivers/video/rockchip/mpp/mpp_rkvdec2.h | 220 + drivers/video/rockchip/mpp/mpp_rkvdec2_link.c | 1956 + drivers/video/rockchip/mpp/mpp_rkvdec2_link.h | 169 + drivers/video/rockchip/mpp/mpp_rkvenc.c | 1512 + drivers/video/rockchip/mpp/mpp_rkvenc2.c | 1712 + drivers/video/rockchip/mpp/mpp_service.c | 436 + drivers/video/rockchip/mpp/mpp_vdpu1.c | 876 + drivers/video/rockchip/mpp/mpp_vdpu2.c | 770 + drivers/video/rockchip/mpp/mpp_vepu1.c | 797 + drivers/video/rockchip/mpp/mpp_vepu2.c | 1165 + .../video/rockchip/mpp/rockchip_iep2_regs.h | 174 + drivers/video/rockchip/rga/Kconfig | 10 + drivers/video/rockchip/rga/Makefile | 4 + drivers/video/rockchip/rga/RGA_API.c | 201 + drivers/video/rockchip/rga/RGA_API.h | 40 + drivers/video/rockchip/rga/rga.h | 508 + drivers/video/rockchip/rga/rga_drv.c | 2550 + drivers/video/rockchip/rga/rga_mmu_info.c | 1315 + drivers/video/rockchip/rga/rga_mmu_info.h | 24 + drivers/video/rockchip/rga/rga_reg_info.c | 1587 + drivers/video/rockchip/rga/rga_reg_info.h | 467 + drivers/video/rockchip/rga/rga_rop.h | 56 + drivers/video/rockchip/rga/rga_type.h | 49 + drivers/video/rockchip/rga2/Kconfig | 30 + drivers/video/rockchip/rga2/Makefile | 5 + drivers/video/rockchip/rga2/RGA2_API.c | 23 + drivers/video/rockchip/rga2/RGA2_API.h | 59 + drivers/video/rockchip/rga2/rga2.h | 792 + drivers/video/rockchip/rga2/rga2_debugger.c | 396 + drivers/video/rockchip/rga2/rga2_debugger.h | 120 + drivers/video/rockchip/rga2/rga2_drv.c | 2264 + drivers/video/rockchip/rga2/rga2_mmu_info.c | 1831 + drivers/video/rockchip/rga2/rga2_mmu_info.h | 35 + drivers/video/rockchip/rga2/rga2_reg_info.c | 1689 + drivers/video/rockchip/rga2/rga2_reg_info.h | 331 + drivers/video/rockchip/rga2/rga2_rop.h | 56 + drivers/video/rockchip/rga2/rga2_type.h | 49 + fs/ubifs/recovery.c | 41 +- fs/xfs/xfs_inode.c | 14 +- include/drm/bridge/analogix_dp.h | 34 +- include/drm/bridge/dw_hdmi.h | 106 +- include/drm/bridge/dw_mipi_dsi.h | 1 + include/drm/drm_auth.h | 1 - include/drm/drm_connector.h | 54 + include/drm/drm_crtc.h | 11 +- include/drm/drm_drv.h | 10 + include/drm/drm_edid.h | 32 + include/drm/drm_file.h | 21 +- include/drm/drm_mipi_dsi.h | 12 + include/drm/drm_mode_config.h | 13 + include/drm/drm_mode_object.h | 2 +- include/drm/drm_prime.h | 1 + include/drm/drm_sync_helper.h | 156 + include/dt-bindings/clock/px30-cru.h | 2 + include/dt-bindings/clock/rk1808-cru.h | 473 + include/dt-bindings/clock/rk3036-cru.h | 4 + include/dt-bindings/clock/rk3128-cru.h | 1 + include/dt-bindings/clock/rk3188-cru-common.h | 7 +- include/dt-bindings/clock/rk3288-cru.h | 4 + include/dt-bindings/clock/rk3368-cru.h | 1 + include/dt-bindings/clock/rk3399-cru.h | 21 +- include/dt-bindings/clock/rk3568-cru.h | 932 + include/dt-bindings/clock/rk618-cru.h | 38 + include/dt-bindings/clock/rk628-cgu.h | 50 + include/dt-bindings/clock/rockchip,rk3036.h | 155 + include/dt-bindings/clock/rockchip,rk312x.h | 167 + include/dt-bindings/clock/rockchip,rk3188.h | 13 + include/dt-bindings/clock/rockchip,rk3228.h | 167 + include/dt-bindings/clock/rockchip,rk3288.h | 220 + include/dt-bindings/clock/rockchip,rk3368.h | 263 + include/dt-bindings/clock/rockchip-ddr.h | 63 + include/dt-bindings/clock/rockchip.h | 101 + include/dt-bindings/clock/rv1126-cru.h | 633 + include/dt-bindings/display/drm_mipi_dsi.h | 53 + .../dt-bindings/display/media-bus-format.h | 152 + include/dt-bindings/display/mipi_dsi.h | 106 + include/dt-bindings/display/rockchip_vop.h | 21 + include/dt-bindings/dram/rockchip,rk322x.h | 90 + include/dt-bindings/dram/rockchip,rk3368.h | 80 + include/dt-bindings/input/rk-input.h | 814 + include/dt-bindings/memory/px30-dram.h | 132 + include/dt-bindings/memory/rk1808-dram.h | 180 + include/dt-bindings/memory/rk3128-dram.h | 95 + include/dt-bindings/memory/rk3288-dram.h | 127 + include/dt-bindings/memory/rk3328-dram.h | 159 + include/dt-bindings/memory/rk3368-dram.h | 109 + include/dt-bindings/memory/rk3399-dram.h | 107 + include/dt-bindings/memory/rk3568-dram.h | 111 + include/dt-bindings/memory/rockchip-dram.h | 94 + include/dt-bindings/memory/rv1126-dram.h | 161 + include/dt-bindings/phy/phy-snps-pcie3.h | 21 + include/dt-bindings/pinctrl/rockchip-rk3036.h | 267 + include/dt-bindings/pinctrl/rockchip-rk312x.h | 384 + include/dt-bindings/pinctrl/rockchip-rk3188.h | 457 + include/dt-bindings/pinctrl/rockchip-rk3288.h | 666 + include/dt-bindings/power/rk1808-power.h | 20 + include/dt-bindings/power/rk3568-power.h | 32 + include/dt-bindings/power/rk3588-power.h | 69 + include/dt-bindings/power/rv1126-power.h | 34 + include/dt-bindings/reset/rk628-rgu.h | 43 + include/dt-bindings/sensor-dev.h | 18 + include/dt-bindings/soc/rockchip,boot-mode.h | 10 +- .../dt-bindings/soc/rockchip-system-status.h | 48 + include/dt-bindings/suspend/rockchip-px30.h | 53 + include/dt-bindings/suspend/rockchip-rk1808.h | 46 + include/dt-bindings/suspend/rockchip-rk322x.h | 57 + include/dt-bindings/suspend/rockchip-rk3288.h | 59 + include/dt-bindings/suspend/rockchip-rk3308.h | 104 + include/dt-bindings/suspend/rockchip-rk3328.h | 19 + include/dt-bindings/suspend/rockchip-rk3368.h | 56 + include/dt-bindings/suspend/rockchip-rk3399.h | 61 + include/dt-bindings/suspend/rockchip-rk3568.h | 57 + include/dt-bindings/suspend/rockchip-rv1126.h | 46 + include/linux/android_kabi.h | 113 + include/linux/bpf.h | 3 +- include/linux/clk-provider.h | 9 + include/linux/clk/rockchip.h | 18 + include/linux/cpuset.h | 7 +- include/linux/dma-buf-cache.h | 32 + include/linux/dma-buf.h | 116 +- include/linux/dma-fence.h | 3 + include/linux/dma-heap.h | 62 +- include/linux/dma-iommu.h | 16 + include/linux/dma-map-ops.h | 14 + include/linux/dma-mapping.h | 17 + include/linux/dmaengine.h | 5 + include/linux/energy_model.h | 12 +- include/linux/extcon.h | 1 + include/linux/freezer.h | 6 + include/linux/initramfs.h | 40 + include/linux/iommu.h | 47 +- include/linux/iova.h | 1 + include/linux/irqchip/arm-gic-v3.h | 2 + include/linux/irqchip/arm-gic-v4.h | 4 + include/linux/memblock.h | 24 +- include/linux/memory_group_manager.h | 198 + include/linux/mfd/rk808.h | 453 +- include/linux/mm.h | 2 + include/linux/mmc/host.h | 4 + include/linux/mmu_context.h | 8 + include/linux/phy/pcie.h | 12 + include/linux/phy/phy-rockchip-typec.h | 34 + include/linux/power/bq25700-charge.h | 20 + include/linux/power/cw2015_battery.h | 119 + include/linux/power/rk_usbbc.h | 44 + include/linux/power_supply.h | 23 + include/linux/pwm.h | 3 + include/linux/reboot.h | 20 + include/linux/regulator/driver.h | 3 + include/linux/regulator/fan53555.h | 2 + include/linux/rfkill-bt.h | 73 + include/linux/rfkill-wlan.h | 62 + include/linux/rk_keys.h | 19 + include/linux/rockchip-iovmm.h | 91 + include/linux/rockchip/cpu.h | 186 + include/linux/rockchip/grf.h | 700 + include/linux/rockchip/psci.h | 76 + include/linux/rockchip/rockchip_sip.h | 350 + include/linux/rockchip_ion.h | 41 + include/linux/sched.h | 2 +- include/linux/sched/signal.h | 1 + include/linux/sched/xacct.h | 9 + include/linux/sensor-dev.h | 350 + include/linux/soc/rockchip/pvtm.h | 17 + include/linux/soc/rockchip/rk_fiq_debugger.h | 22 + .../linux/soc/rockchip/rk_vendor_storage.h | 58 + .../linux/soc/rockchip/rockchip_decompress.h | 22 + .../rockchip/rockchip_thunderboot_crypto.h | 9 + include/linux/stmmac.h | 1 + include/linux/thermal.h | 5 + include/linux/usb/audio-v2.h | 0 include/linux/usb/audio.h | 0 include/linux/usb/composite.h | 3 + include/linux/usb/f_accessory.h | 23 + include/linux/usb/gadget.h | 20 + include/linux/usb/pd.h | 7 + include/linux/usb/pd_ext_sdb.h | 4 - include/linux/usb/pd_vdo.h | 314 +- include/linux/usb/quirks.h | 3 + include/linux/usb/tcpm.h | 56 +- include/linux/usb/typec.h | 39 + include/linux/usb/typec_altmode.h | 10 + include/linux/usb/typec_tbt.h | 6 +- include/linux/usb/usbnet.h | 2 - include/linux/wakelock.h | 76 + include/linux/wakeup_reason.h | 37 + include/media/v4l2-async.h | 15 + include/soc/rockchip/android-version.h | 17 + include/soc/rockchip/pm_domains.h | 53 + include/soc/rockchip/rk3368-mailbox.h | 22 + include/soc/rockchip/rkfb_dmc.h | 11 + include/soc/rockchip/rockchip-system-status.h | 53 + include/soc/rockchip/rockchip_dmc.h | 121 + include/soc/rockchip/rockchip_iommu.h | 24 + include/soc/rockchip/rockchip_ipa.h | 40 + include/soc/rockchip/rockchip_opp_select.h | 152 + include/soc/rockchip/rockchip_sip.h | 8 +- .../soc/rockchip/rockchip_system_monitor.h | 205 + include/soc/rockchip/scpi.h | 147 + include/trace/events/cpuhp.h | 22 + include/trace/events/sched.h | 7 +- include/trace/events/thermal_ipa_power.h | 47 + include/uapi/drm/drm_fourcc.h | 2 + include/uapi/drm/rockchip_drm.h | 134 + include/uapi/linux/fec-config.h | 33 + include/uapi/linux/media-bus-format.h | 6 +- include/uapi/linux/rk-camera-module.h | 560 + include/uapi/linux/rk-led-flash.h | 9 + include/uapi/linux/rk-pcie-dma.h | 61 + include/uapi/linux/rk-preisp.h | 118 + include/uapi/linux/rk-video-format.h | 35 + include/uapi/linux/rk803.h | 16 + include/uapi/linux/rk_vcm_head.h | 115 + include/uapi/linux/rkcif-config.h | 57 + include/uapi/linux/rkisp1-config.h | 872 + include/uapi/linux/rkisp2-config.h | 1853 + include/uapi/linux/rkisp21-config.h | 804 + include/uapi/linux/rkisp3-config.h | 1072 + include/uapi/linux/rkispp-config.h | 370 + include/uapi/linux/serial_reg.h | 1 + include/uapi/linux/usb/f_accessory.h | 146 + include/uapi/linux/usb/g_uvc.h | 4 +- include/uapi/linux/usb/video.h | 58 + include/uapi/linux/videodev2.h | 1 + init/Kconfig | 6 + init/initramfs.c | 23 + init/main.c | 8 + ipc/msg.c | 2 +- ipc/sem.c | 6 +- ipc/shm.c | 2 +- ipc/util.c | 16 +- kernel/bpf/syscall.c | 57 +- kernel/bpf/verifier.c | 17 +- kernel/cgroup/cgroup-v1.c | 3 +- kernel/cgroup/cpuset.c | 150 +- kernel/cgroup/legacy_freezer.c | 1 + kernel/cpu.c | 6 +- kernel/irq/generic-chip.c | 1 + kernel/power/Makefile | 1 + kernel/power/energy_model.c | 24 +- kernel/power/hibernate.c | 2 +- kernel/power/power.h | 2 +- kernel/power/process.c | 13 +- kernel/power/snapshot.c | 14 +- kernel/power/suspend.c | 19 +- kernel/power/wakeup_reason.c | 438 + kernel/reboot.c | 21 + kernel/sched/core.c | 65 +- kernel/sched/fair.c | 16 +- kernel/sched/loadavg.c | 1 + kernel/sched/pelt.c | 38 +- kernel/sched/rt.c | 40 +- kernel/sched/sched-pelt.h | 21 +- kernel/sched/sched.h | 2 + kernel/sched/topology.c | 3 + kernel/sched/wait.c | 8 +- make-boot.sh | 31 + make-ohos.sh | 142 + mm/hugetlb.c | 23 +- mm/memblock.c | 51 + mm/page_alloc.c | 23 +- net/bluetooth/l2cap_sock.c | 10 +- net/bluetooth/sco.c | 18 +- net/rfkill/Kconfig | 9 + net/rfkill/Makefile | 2 + net/rfkill/rfkill-bt.c | 827 + net/rfkill/rfkill-wlan.c | 979 + scripts/Makefile | 1 + scripts/Makefile.lib | 5 +- scripts/clang-wrapper.py | 102 + scripts/dtc/checks.c | 18 + scripts/mkimg | 235 + scripts/repack-bootimg | 153 + scripts/resource_tool.c | 1575 + security/Kconfig.hardening | 3 +- sound/drivers/aloop.c | 26 +- sound/soc/codecs/Kconfig | 63 +- sound/soc/codecs/Makefile | 26 + sound/soc/codecs/bt-sco.c | 8 +- sound/soc/codecs/dummy-codec.c | 115 + sound/soc/codecs/es7202.c | 1170 + sound/soc/codecs/es7202.h | 95 + sound/soc/codecs/es7210.c | 1754 + sound/soc/codecs/es7210.h | 76 + sound/soc/codecs/es7243e.c | 2671 + sound/soc/codecs/es7243e_usr_cfg.h | 274 + sound/soc/codecs/es8311.c | 756 + sound/soc/codecs/es8311.h | 69 + sound/soc/codecs/es8323.c | 998 + sound/soc/codecs/es8323.h | 151 + sound/soc/codecs/es8328.c | 18 + sound/soc/codecs/es8396.c | 3207 + sound/soc/codecs/es8396.h | 354 + sound/soc/codecs/rk1000_codec.c | 580 + sound/soc/codecs/rk1000_codec.h | 320 + sound/soc/codecs/rk312x_codec.c | 2526 + sound/soc/codecs/rk312x_codec.h | 566 + sound/soc/codecs/rk3228_codec.c | 543 + sound/soc/codecs/rk3228_codec.h | 218 + sound/soc/codecs/rk3308_codec.c | 5128 ++ sound/soc/codecs/rk3308_codec.h | 1069 + sound/soc/codecs/rk3308_codec_provider.h | 28 + sound/soc/codecs/rk817_codec.c | 1364 + sound/soc/codecs/rk817_codec.h | 215 + sound/soc/codecs/rk_codec_digital.c | 1109 + sound/soc/codecs/rk_codec_digital.h | 319 + sound/soc/codecs/rt5640.c | 4 +- sound/soc/codecs/rt5651.c | 85 +- sound/soc/codecs/rt5651.h | 3 + sound/soc/codecs/wm8974.c | 38 + sound/soc/rockchip/Kconfig | 39 + sound/soc/rockchip/Makefile | 19 +- sound/soc/rockchip/rockchip_audio_pwm.c | 372 + sound/soc/rockchip/rockchip_audio_pwm.h | 48 + sound/soc/rockchip/rockchip_cdndp.c | 166 + sound/soc/rockchip/rockchip_da7219.c | 235 + sound/soc/rockchip/rockchip_hdmi.c | 279 + sound/soc/rockchip/rockchip_hdmi_analog.c | 222 + sound/soc/rockchip/rockchip_hdmi_dp.c | 193 + sound/soc/rockchip/rockchip_i2s.c | 226 +- sound/soc/rockchip/rockchip_i2s.h | 10 +- sound/soc/rockchip/rockchip_i2s_tdm.c | 1952 + sound/soc/rockchip/rockchip_i2s_tdm.h | 393 + sound/soc/rockchip/rockchip_multi_dais.c | 541 + sound/soc/rockchip/rockchip_multi_dais.h | 33 + sound/soc/rockchip/rockchip_multi_dais_pcm.c | 704 + sound/soc/rockchip/rockchip_multicodecs.c | 809 + sound/soc/rockchip/rockchip_pdm.c | 136 +- sound/soc/rockchip/rockchip_pdm.h | 6 + sound/soc/rockchip/rockchip_rt5651.c | 245 + sound/soc/rockchip/rockchip_spdif.c | 45 +- sound/soc/rockchip/rockchip_spdifrx.c | 415 + sound/soc/rockchip/rockchip_spdifrx.h | 113 + sound/soc/rockchip/rockchip_vad.c | 1316 + sound/soc/rockchip/rockchip_vad.h | 231 + sound/soc/rockchip/vad_preprocess.h | 29 + sound/soc/rockchip/vad_preprocess_arm.S | 348 + sound/soc/rockchip/vad_preprocess_arm64.S | 273 + sound/soc/rockchip/vad_preprocess_thumb.S | 360 + sound/usb/quirks.c | 6 + .../multibuff/device_app/aio_multibuff.c | 68 +- .../simple/device_app/aio_simple.c | 68 +- 2840 files changed, 1645582 insertions(+), 12930 deletions(-) create mode 100644 Documentation/devicetree/bindings/mtd/rockchip,nand-controller.yaml create mode 100755 arch/arm64/boot/dts/rockchip/px30-ad-d6-anx6345.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-dual-lvds.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi-lvds.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-lvds.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-ad-r35-mb.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/px30-android.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/px30-ddr4p416dd6-timing.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/px30-dram-default-timing.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr3-lvds-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-no-gpu-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ddr4-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dts create mode 100755 arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/px30-robot-no-gpu.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/px30-robot.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/px30-z7-a0-rk618-dsi.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk1808-dram-default-timing.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk1808-evb-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk1808-evb-x4-second.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk1808-evb-x4.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk1808-evb.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk1808-fpga.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk1808.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk1808k.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-ai-va-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v11.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v11.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v11.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-evb-ext-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-evb-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-evb-v11.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-fpga.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3308-voice-module-board-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3308b-evb-amic-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3308b-evb-ext-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3308b-evb-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3308k.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-863-cif-sensor.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-rkisp1.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-86v-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11-i2s-dmic.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v12.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-no-gpu-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3326-linux.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-android.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-box-liantong-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-box-plus-dram-timing.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-dram-2layer-timing.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-dram-default-timing.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-evb-android-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-evb-android.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-evb-android.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-rock64-android-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-808-evb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-808.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-android.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-cif-sensor.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-dram-default-timing.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-p9-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-p9.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-p9.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-px5-evb-android.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-r88-dcdc.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-sheep-lvds.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-sheep.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-sziauto-rk618.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-tablet.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-xikp-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-xikp.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368-xikp.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3368a-817-tablet-bnd.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3368a-817-tablet.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-android.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-box-rev1.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-box-rev2.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-box.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-dram-default-timing.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-early-opp.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-cros.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-v13-android-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-ind.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-android.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-cros.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev1.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-android.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-cros.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev2.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-edp.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-lp4.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-mipi-edp.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-cros.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb-rev3.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-evb.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-excavator-sapphire.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-firefly-android.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-firefly-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-fpga.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-gru-gru.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r0.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r1.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-linux.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-mid-818-android.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-rock960-ab.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-box.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux-for-rk1808-cascade.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-lp4-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-sched-energy.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-tve1030g-avb.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-tve1205g.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-videostrong-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399-vop-clk-set.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399k-opp.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3399pro-evb-lp4-v11-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399pro-evb-v14-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399pro-npu-evb-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3399pro-npu.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-box.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-eink.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-lvds.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-eink.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-i2s-mic-array.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-pdm-mic-array.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w103.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w6.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-rk817-eink.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-k108.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-rkg11.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3566.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-android.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-dram-default-timing.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux-spi-nor.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10-bt1120-to-hdmi.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb4-lp3-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-bt1120-to-hdmi.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-rgb2hdmi.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk630-bt656-to-cvbs.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-evb7-ddr4-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-linux.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux-spi-nand.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux-spi-nand.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-nvr-linux.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-nvr.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-base.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-edp.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0-beiqicloud.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx1.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-android.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux-factory.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10-linux.dts create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568-toybrick.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rk3568.dtsi create mode 100755 arch/arm64/boot/dts/rockchip/rockchip-pinconf.dtsi create mode 100755 arch/arm64/include/asm/system_info.h create mode 100755 drivers/clk/rockchip/clk-dclk-divider.c create mode 100755 drivers/clk/rockchip/clk-pvtm.c create mode 100755 drivers/clk/rockchip/clk-rk1808.c create mode 100755 drivers/clk/rockchip/clk-rk3568.c create mode 100755 drivers/clk/rockchip/clk-rv1126.c create mode 100755 drivers/clk/rockchip/regmap/Kconfig create mode 100755 drivers/clk/rockchip/regmap/Makefile create mode 100755 drivers/clk/rockchip/regmap/clk-regmap-composite.c create mode 100755 drivers/clk/rockchip/regmap/clk-regmap-divider.c create mode 100755 drivers/clk/rockchip/regmap/clk-regmap-fractional-divider.c create mode 100755 drivers/clk/rockchip/regmap/clk-regmap-gate.c create mode 100755 drivers/clk/rockchip/regmap/clk-regmap-mux.c create mode 100755 drivers/clk/rockchip/regmap/clk-regmap-pll.c create mode 100755 drivers/clk/rockchip/regmap/clk-regmap.h create mode 100755 drivers/clk/rockchip/regmap/clk-rk618.c create mode 100755 drivers/clk/rockchip/regmap/clk-rk628.c create mode 100755 drivers/cpufreq/rockchip-cpufreq.c create mode 100755 drivers/cpufreq/rockchip-cpufreq.h create mode 100755 drivers/devfreq/event/rockchip-nocp.c delete mode 100644 drivers/devfreq/rk3399_dmc.c create mode 100755 drivers/devfreq/rockchip_bus.c create mode 100755 drivers/devfreq/rockchip_dmc.c create mode 100755 drivers/devfreq/rockchip_dmc_common.c create mode 100755 drivers/devfreq/rockchip_dmc_dbg.c create mode 100755 drivers/devfreq/rockchip_dmc_timing.h create mode 100755 drivers/dma-buf/heaps/deferred-free-helper.c create mode 100755 drivers/dma-buf/heaps/deferred-free-helper.h delete mode 100644 drivers/dma-buf/heaps/heap-helpers.c delete mode 100644 drivers/dma-buf/heaps/heap-helpers.h create mode 100755 drivers/dma-buf/heaps/page_pool.c create mode 100755 drivers/dma-buf/heaps/page_pool.h create mode 100755 drivers/firmware/rockchip_sip.c create mode 100755 drivers/gpio/gpio-rockchip.c create mode 100755 drivers/gpu/arm/Kbuild create mode 100755 drivers/gpu/arm/Kconfig create mode 100755 drivers/gpu/arm/bifrost/Kbuild create mode 100755 drivers/gpu/arm/bifrost/Kconfig create mode 100755 drivers/gpu/arm/bifrost/Makefile create mode 100755 drivers/gpu/arm/bifrost/Makefile.kbase create mode 100755 drivers/gpu/arm/bifrost/Mconfig create mode 100755 drivers/gpu/arm/bifrost/arbiter/Kbuild create mode 100755 drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c create mode 100755 drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h create mode 100755 drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_defs.h create mode 100755 drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_interface.h create mode 100755 drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c create mode 100755 drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/Kbuild create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_defs.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_internal.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_internal.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_internal.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca_devfreq.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_l2_states.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_mcu_states.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h create mode 100755 drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c create mode 100755 drivers/gpu/arm/bifrost/build.bp create mode 100755 drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c create mode 100755 drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c create mode 100755 drivers/gpu/arm/bifrost/context/mali_kbase_context.c create mode 100755 drivers/gpu/arm/bifrost/context/mali_kbase_context.h create mode 100755 drivers/gpu/arm/bifrost/context/mali_kbase_context_internal.h create mode 100755 drivers/gpu/arm/bifrost/csf/Kbuild create mode 100755 drivers/gpu/arm/bifrost/csf/mali_base_csf_kernel.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_gpu_csf_control_registers.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_gpu_csf_registers.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_def.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.h create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c create mode 100755 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h create mode 100755 drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h create mode 100755 drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_jm.h create mode 100755 drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c create mode 100755 drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.h create mode 100755 drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_csf.h create mode 100755 drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h create mode 100755 drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c create mode 100755 drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h create mode 100755 drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h create mode 100755 drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h create mode 100755 drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c create mode 100755 drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.h create mode 100755 drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h create mode 100755 drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_defs.h create mode 100755 drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_internal.h create mode 100755 drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h create mode 100755 drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c create mode 100755 drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c create mode 100755 drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c create mode 100755 drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c create mode 100755 drivers/gpu/arm/bifrost/device/mali_kbase_device.c create mode 100755 drivers/gpu/arm/bifrost/device/mali_kbase_device.h create mode 100755 drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c create mode 100755 drivers/gpu/arm/bifrost/device/mali_kbase_device_internal.h create mode 100755 drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c create mode 100755 drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_jm.c create mode 100755 drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_csf.h create mode 100755 drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_jm.h create mode 100755 drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.c create mode 100755 drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.h create mode 100755 drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h create mode 100755 drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_fault.h create mode 100755 drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h create mode 100755 drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_regmap.h create mode 100755 drivers/gpu/arm/bifrost/ipa/Kbuild create mode 100755 drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c create mode 100755 drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h create mode 100755 drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c create mode 100755 drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.h create mode 100755 drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c create mode 100755 drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.h create mode 100755 drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c create mode 100755 drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h create mode 100755 drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c create mode 100755 drivers/gpu/arm/bifrost/jm/mali_base_jm_kernel.h create mode 100755 drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h create mode 100755 drivers/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h create mode 100755 drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h create mode 100755 drivers/gpu/arm/bifrost/jm/mali_kbase_js_defs.h create mode 100755 drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h create mode 100755 drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h create mode 100755 drivers/gpu/arm/bifrost/mali_base_kernel.h create mode 100755 drivers/gpu/arm/bifrost/mali_base_mem_priv.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_bits.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_caps.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_ccswe.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_ccswe.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_config.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_config.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_core_linux.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_debug.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_debug.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_defs.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_dma_fence.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_dma_fence.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_event.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_fence.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_fence.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_fence_defs.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_gator.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_gpuprops_types.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_gwt.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_gwt.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hw.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hw.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwaccess_backend.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwaccess_gpuprops.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwaccess_instr.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_accumulator.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_context.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_reader.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_ioctl.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_jd.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_jm.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_jm.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_js.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_js.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm_reader.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_linux.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mipe_gen_header.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_mipe_proto.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_native_mgm.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_platform_fake.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_pm.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_pm.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_smc.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_smc.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_softjobs.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_strings.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_strings.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_sync.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_sync_android.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_sync_common.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_sync_file.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_utility.h create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_vinstr.c create mode 100755 drivers/gpu/arm/bifrost/mali_kbase_vinstr.h create mode 100755 drivers/gpu/arm/bifrost/mali_linux_trace.h create mode 100755 drivers/gpu/arm/bifrost/mali_malisw.h create mode 100755 drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.c create mode 100755 drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.h create mode 100755 drivers/gpu/arm/bifrost/mali_uk.h create mode 100755 drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c create mode 100755 drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c create mode 100755 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c create mode 100755 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h create mode 100755 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h create mode 100755 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c create mode 100755 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h create mode 100755 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c create mode 100755 drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_lpae.c create mode 100755 drivers/gpu/arm/bifrost/platform/Kconfig create mode 100755 drivers/gpu/arm/bifrost/platform/devicetree/Kbuild create mode 100755 drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_clk_rate_trace.c create mode 100755 drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_devicetree.c create mode 100755 drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c create mode 100755 drivers/gpu/arm/bifrost/platform/rk/Kbuild create mode 100755 drivers/gpu/arm/bifrost/platform/rk/custom_log.h create mode 100755 drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c create mode 100755 drivers/gpu/arm/bifrost/platform/rk/mali_kbase_rk.h create mode 100755 drivers/gpu/arm/bifrost/platform/vexpress/Kbuild create mode 100755 drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_vexpress.c create mode 100755 drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/Kbuild create mode 100755 drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c create mode 100755 drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/Kbuild create mode 100755 drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c create mode 100755 drivers/gpu/arm/bifrost/protected_mode_switcher.h create mode 100755 drivers/gpu/arm/bifrost/tests/Kbuild create mode 100755 drivers/gpu/arm/bifrost/tests/Kconfig create mode 100755 drivers/gpu/arm/bifrost/tests/Mconfig create mode 100755 drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers.h create mode 100755 drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers_user.h create mode 100755 drivers/gpu/arm/bifrost/tests/include/kutf/kutf_mem.h create mode 100755 drivers/gpu/arm/bifrost/tests/include/kutf/kutf_resultset.h create mode 100755 drivers/gpu/arm/bifrost/tests/include/kutf/kutf_suite.h create mode 100755 drivers/gpu/arm/bifrost/tests/include/kutf/kutf_utils.h create mode 100755 drivers/gpu/arm/bifrost/tests/kutf/Kbuild create mode 100755 drivers/gpu/arm/bifrost/tests/kutf/Kconfig create mode 100755 drivers/gpu/arm/bifrost/tests/kutf/Makefile create mode 100755 drivers/gpu/arm/bifrost/tests/kutf/build.bp create mode 100755 drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers.c create mode 100755 drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c create mode 100755 drivers/gpu/arm/bifrost/tests/kutf/kutf_mem.c create mode 100755 drivers/gpu/arm/bifrost/tests/kutf/kutf_resultset.c create mode 100755 drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c create mode 100755 drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kbuild create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kconfig create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Makefile create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/build.bp create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kbuild create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kconfig create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/build.bp create mode 100755 drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c create mode 100755 drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c create mode 100755 drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c create mode 100755 drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c create mode 100755 drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c create mode 100755 drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.h create mode 100755 drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c create mode 100755 drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_priv.h create mode 100755 drivers/gpu/arm/bifrost/tl/mali_kbase_tl_serialize.h create mode 100755 drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c create mode 100755 drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.h create mode 100755 drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c create mode 100755 drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/Kconfig create mode 100755 drivers/gpu/arm/bifrost_for_linux/Makefile create mode 100755 drivers/gpu/arm/bifrost_for_linux/Makefile.kbase create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_backend_config.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_debug_job_fault_backend.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_hw.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_internal.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpu.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpuprops_backend.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_backend.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_defs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_internal.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_internal.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_linux.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_as.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_defs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_hw.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_internal.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_backend.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_internal.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_backend.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_defs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_driver.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_internal.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_metrics.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/docs/Doxyfile create mode 100755 drivers/gpu/arm/bifrost_for_linux/docs/policy_operation_diagram.dot create mode 100755 drivers/gpu/arm/bifrost_for_linux/docs/policy_overview.dot create mode 100755 drivers/gpu/arm/bifrost_for_linux/ipa/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_g71.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_features.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_issues.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_base_kernel.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_base_mem_priv.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_base_vendor_specific_func.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_config_defaults.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_core_linux.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_defs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_device.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_disjoint_events.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_event.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence_defs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_thex.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tmix.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tsix.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_id.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops_types.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_backend.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_defs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_gpuprops.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_instr.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_jm.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_pm.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_time.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwcnt_reader.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_ioctl.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_defs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_linux.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_lowlevel.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs_buf_size.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_hw.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_aarch64.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_lpae.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_platform_fake.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_profiling_gator_api.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_replay.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_softjobs.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_android.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_common.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_file.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_defs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline_defs.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_uku.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_linux_kbase_trace.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_linux_trace.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_malisw.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_midg_coherency.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_midg_regmap.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_timeline.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/mali_uk.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/Kconfig create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/devicetree/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_devicetree.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_runtime_pm.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/mali_kbase_platform_common.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/rk/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/rk/custom_log.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_rk.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_rk.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_vexpress.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/platform_dummy/mali_ukk_os.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/protected_mode_switcher.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/sconscript create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/Kconfig create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers_user.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_mem.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_resultset.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_suite.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_utils.h create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kconfig create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/kutf/Makefile create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers_user.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_mem.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_resultset.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_suite.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_utils.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/kutf/sconscript create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kbuild create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kconfig create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Makefile create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/sconscript create mode 100755 drivers/gpu/arm/bifrost_for_linux/tests/sconscript create mode 100755 drivers/gpu/arm/mali400/.gitignore create mode 100755 drivers/gpu/arm/mali400/Kbuild create mode 100755 drivers/gpu/arm/mali400/mali/.gitignore create mode 100755 drivers/gpu/arm/mali400/mali/Kbuild create mode 100755 drivers/gpu/arm/mali400/mali/Kconfig create mode 100755 drivers/gpu/arm/mali400/mali/Makefile create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_broadcast.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_broadcast.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_control_timer.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_control_timer.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_dlbu.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_dlbu.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_executor.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_executor.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_gp.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_gp.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_gp_job.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_gp_job.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_group.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_group.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_hw_core.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_hw_core.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_mmu.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_mmu.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_osk.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_osk_list.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_osk_types.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pm.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pm.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pmu.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pmu.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pp.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pp.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pp_job.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_pp_job.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_scheduler.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_scheduler.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_session.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_session.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_soft_job.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_soft_job.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_timeline.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_timeline.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_ukk.h create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c create mode 100755 drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h create mode 100755 drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h create mode 100755 drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h create mode 100755 drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h create mode 100755 drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h create mode 100755 drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_devfreq.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_devfreq.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_sync.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_sync.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c create mode 100755 drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h create mode 100755 drivers/gpu/arm/mali400/mali/platform/arm/arm.c create mode 100755 drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c create mode 100755 drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h create mode 100755 drivers/gpu/arm/mali400/mali/platform/arm/juno_opp.c create mode 100755 drivers/gpu/arm/mali400/mali/platform/rk/custom_log.h create mode 100755 drivers/gpu/arm/mali400/mali/platform/rk/rk.c create mode 100755 drivers/gpu/arm/mali400/mali/platform/rk/rk_ext.h create mode 100755 drivers/gpu/arm/mali400/mali/readme.txt create mode 100755 drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h create mode 100755 drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h create mode 100755 drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c create mode 100755 drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h create mode 100755 drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c create mode 100755 drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h create mode 100755 drivers/gpu/arm/mali400/rk_ver_info.txt create mode 100755 drivers/gpu/arm/mali400/ump/Kbuild create mode 100755 drivers/gpu/arm/mali400/ump/Kconfig create mode 100755 drivers/gpu/arm/mali400/ump/Makefile create mode 100755 drivers/gpu/arm/mali400/ump/Makefile.common create mode 100755 drivers/gpu/arm/mali400/ump/arch-default/config.h create mode 100755 drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h create mode 100755 drivers/gpu/arm/mali400/ump/arch/config.h create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_osk.h create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_uk_types.h create mode 100755 drivers/gpu/arm/mali400/ump/common/ump_ukk.h create mode 100755 drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c create mode 100755 drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h create mode 100755 drivers/gpu/arm/mali400/ump/readme.txt create mode 100755 drivers/gpu/arm/mali400/umplock/Makefile create mode 100755 drivers/gpu/arm/mali400/umplock/umplock_driver.c create mode 100755 drivers/gpu/arm/mali400/umplock/umplock_ioctl.h create mode 100755 drivers/gpu/arm/midgard/Kbuild create mode 100755 drivers/gpu/arm/midgard/Kconfig create mode 100755 drivers/gpu/arm/midgard/Makefile create mode 100755 drivers/gpu/arm/midgard/Makefile.kbase create mode 100755 drivers/gpu/arm/midgard/backend/gpu/Kbuild create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpu.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c create mode 100755 drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.h create mode 100755 drivers/gpu/arm/midgard/docs/Doxyfile create mode 100755 drivers/gpu/arm/midgard/docs/policy_operation_diagram.dot create mode 100755 drivers/gpu/arm/midgard/docs/policy_overview.dot create mode 100755 drivers/gpu/arm/midgard/ipa/Kbuild create mode 100755 drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c create mode 100755 drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h create mode 100755 drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c create mode 100755 drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h create mode 100755 drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c create mode 100755 drivers/gpu/arm/midgard/mali_base_hwconfig_features.h create mode 100755 drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h create mode 100755 drivers/gpu/arm/midgard/mali_base_kernel.h create mode 100755 drivers/gpu/arm/midgard/mali_base_mem_priv.h create mode 100755 drivers/gpu/arm/midgard/mali_base_vendor_specific_func.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_10969_workaround.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_10969_workaround.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_cache_policy.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_cache_policy.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_config.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_config.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_config_defaults.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_context.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_context.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_core_linux.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_debug.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_debug.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_defs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_device.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_dma_fence.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_dma_fence.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_event.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_fence.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_fence.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_fence_defs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gator.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gator_api.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gator_api.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_thex.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tmix.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tsix.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gpu_id.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gpuprops.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gpuprops.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_hw.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_hw.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_ioctl.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_jd.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_jm.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_jm.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_js.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_js.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_js_defs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_linux.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem_linux.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem_linux.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem_pool.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mmu.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mmu_mode_aarch64.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_mmu_mode_lpae.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_platform_fake.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_pm.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_pm.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_profiling_gator_api.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_replay.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_smc.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_smc.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_softjobs.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_strings.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_strings.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_sync.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_sync_android.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_sync_common.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_sync_file.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_tlstream.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_tlstream.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_trace_defs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_trace_timeline.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_trace_timeline.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_trace_timeline_defs.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_uku.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_utility.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_utility.h create mode 100755 drivers/gpu/arm/midgard/mali_kbase_vinstr.c create mode 100755 drivers/gpu/arm/midgard/mali_kbase_vinstr.h create mode 100755 drivers/gpu/arm/midgard/mali_linux_kbase_trace.h create mode 100755 drivers/gpu/arm/midgard/mali_linux_trace.h create mode 100755 drivers/gpu/arm/midgard/mali_malisw.h create mode 100755 drivers/gpu/arm/midgard/mali_midg_coherency.h create mode 100755 drivers/gpu/arm/midgard/mali_midg_regmap.h create mode 100755 drivers/gpu/arm/midgard/mali_timeline.h create mode 100755 drivers/gpu/arm/midgard/mali_uk.h create mode 100755 drivers/gpu/arm/midgard/platform/Kconfig create mode 100755 drivers/gpu/arm/midgard/platform/devicetree/Kbuild create mode 100755 drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c create mode 100755 drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c create mode 100755 drivers/gpu/arm/midgard/platform/mali_kbase_platform_common.h create mode 100755 drivers/gpu/arm/midgard/platform/mali_kbase_platform_fake.h create mode 100755 drivers/gpu/arm/midgard/platform/rk/Kbuild create mode 100755 drivers/gpu/arm/midgard/platform/rk/custom_log.h create mode 100755 drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c create mode 100755 drivers/gpu/arm/midgard/platform/rk/mali_kbase_rk.h create mode 100755 drivers/gpu/arm/midgard/platform/vexpress/Kbuild create mode 100755 drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c create mode 100755 drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.c create mode 100755 drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.h create mode 100755 drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild create mode 100755 drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c create mode 100755 drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild create mode 100755 drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h create mode 100755 drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c create mode 100755 drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c create mode 100755 drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.h create mode 100755 drivers/gpu/arm/midgard/platform_dummy/mali_ukk_os.h create mode 100755 drivers/gpu/arm/midgard/protected_mode_switcher.h create mode 100755 drivers/gpu/arm/midgard/rename.h create mode 100755 drivers/gpu/arm/midgard/sconscript create mode 100755 drivers/gpu/arm/midgard/tests/Kbuild create mode 100755 drivers/gpu/arm/midgard/tests/Kconfig create mode 100755 drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h create mode 100755 drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h create mode 100755 drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h create mode 100755 drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h create mode 100755 drivers/gpu/arm/midgard/tests/kutf/Kbuild create mode 100755 drivers/gpu/arm/midgard/tests/kutf/Kconfig create mode 100755 drivers/gpu/arm/midgard/tests/kutf/Makefile create mode 100755 drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c create mode 100755 drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c create mode 100755 drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c create mode 100755 drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c create mode 100755 drivers/gpu/arm/midgard/tests/kutf/sconscript create mode 100755 drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild create mode 100755 drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig create mode 100755 drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile create mode 100755 drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c create mode 100755 drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/sconscript create mode 100755 drivers/gpu/arm/midgard/tests/sconscript create mode 100755 drivers/gpu/arm/sconscript create mode 100755 drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.c create mode 100755 drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.h create mode 100755 drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-audio.h create mode 100755 drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-i2s-audio.c create mode 100755 drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c create mode 100755 drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h create mode 100755 drivers/gpu/drm/drm_sync_helper.c create mode 100755 drivers/gpu/drm/panel/panel-simple.h create mode 100755 drivers/gpu/drm/rockchip/cdn-dp-link-training.c create mode 100755 drivers/gpu/drm/rockchip/dw-dp.c create mode 100755 drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/Kconfig create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/Makefile create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/bufmanage/Makefile create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/bufmanage/buf_list.c create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/bufmanage/buf_list.h create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/bufmanage/buf_manage.c create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/bufmanage/buf_manage.h create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/ebc_dev.h create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/ebc_dev_v8.S create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/ebc_panel.h create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/ebc_public.c create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/epdlut/Makefile create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/epdlut/epd_lut.c create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/epdlut/epd_lut.h create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/epdlut/pvi_waveform_v8.S create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/epdlut/rkf_waveform_v8.S create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/pmic/Makefile create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/pmic/ebc_pmic.c create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/pmic/ebc_pmic.h create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/pmic/tps65185.c create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/tcon/Makefile create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/tcon/ebc_tcon.c create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/tcon/ebc_tcon.h create mode 100755 drivers/gpu/drm/rockchip/ebc-dev/tcon/eink_tcon.c create mode 100755 drivers/gpu/drm/rockchip/rk618/Kconfig create mode 100755 drivers/gpu/drm/rockchip/rk618/Makefile create mode 100755 drivers/gpu/drm/rockchip/rk618/rk618_dither.c create mode 100755 drivers/gpu/drm/rockchip/rk618/rk618_dither.h create mode 100755 drivers/gpu/drm/rockchip/rk618/rk618_dsi.c create mode 100755 drivers/gpu/drm/rockchip/rk618/rk618_hdmi.c create mode 100755 drivers/gpu/drm/rockchip/rk618/rk618_lvds.c create mode 100755 drivers/gpu/drm/rockchip/rk618/rk618_rgb.c create mode 100755 drivers/gpu/drm/rockchip/rk618/rk618_scaler.c create mode 100755 drivers/gpu/drm/rockchip/rk618/rk618_vif.c create mode 100755 drivers/gpu/drm/rockchip/rk628/Kconfig create mode 100755 drivers/gpu/drm/rockchip/rk628/Makefile create mode 100755 drivers/gpu/drm/rockchip/rk628/rk628_combrxphy.c create mode 100755 drivers/gpu/drm/rockchip/rk628/rk628_combtxphy.c create mode 100755 drivers/gpu/drm/rockchip/rk628/rk628_combtxphy.h create mode 100755 drivers/gpu/drm/rockchip/rk628/rk628_dsi.c create mode 100755 drivers/gpu/drm/rockchip/rk628/rk628_gvi.c create mode 100755 drivers/gpu/drm/rockchip/rk628/rk628_hdmi.c create mode 100755 drivers/gpu/drm/rockchip/rk628/rk628_hdmirx.c create mode 100755 drivers/gpu/drm/rockchip/rk628/rk628_lvds.c create mode 100755 drivers/gpu/drm/rockchip/rk628/rk628_post_process.c create mode 100755 drivers/gpu/drm/rockchip/rk628/rk628_rgb.c create mode 100755 drivers/gpu/drm/rockchip/rockchip-mipi-csi-tx.c create mode 100755 drivers/gpu/drm/rockchip/rockchip-mipi-csi-tx.h create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_backlight.c create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_backlight.h create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_debugfs.c create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_debugfs.h create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_logo.c create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_logo.h create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_tve.c create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_tve.h create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_vconn.c create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_vop2.c create mode 100755 drivers/gpu/drm/rockchip/rockchip_drm_vvop.c create mode 100755 drivers/gpu/drm/rockchip/rockchip_vop2_clk.c create mode 100755 drivers/gpu/drm/rockchip/rockchip_vop2_reg.c create mode 100755 drivers/input/keyboard/rk_keys.c create mode 100755 drivers/input/remotectl/Kconfig create mode 100755 drivers/input/remotectl/Makefile create mode 100755 drivers/input/remotectl/rockchip_pwm_remotectl.c create mode 100755 drivers/input/remotectl/rockchip_pwm_remotectl.h create mode 100755 drivers/input/sensors/Kconfig create mode 100755 drivers/input/sensors/Makefile create mode 100755 drivers/input/sensors/accel/Kconfig create mode 100755 drivers/input/sensors/accel/Makefile create mode 100755 drivers/input/sensors/accel/bma2xx.c create mode 100755 drivers/input/sensors/accel/da223_core.c create mode 100755 drivers/input/sensors/accel/da223_core.h create mode 100755 drivers/input/sensors/accel/da223_cust.c create mode 100755 drivers/input/sensors/accel/da223_cust.h create mode 100755 drivers/input/sensors/accel/dmard10.c create mode 100755 drivers/input/sensors/accel/icm2060x_acc.c create mode 100755 drivers/input/sensors/accel/kxtik.c create mode 100755 drivers/input/sensors/accel/kxtj9.c create mode 100755 drivers/input/sensors/accel/lis3dh.c create mode 100755 drivers/input/sensors/accel/lsm303d.c create mode 100755 drivers/input/sensors/accel/lsm330_acc.c create mode 100755 drivers/input/sensors/accel/mc3230.c create mode 100755 drivers/input/sensors/accel/mma7660.c create mode 100755 drivers/input/sensors/accel/mma8452.c create mode 100755 drivers/input/sensors/accel/mpu6500_acc.c create mode 100755 drivers/input/sensors/accel/mpu6880_acc.c create mode 100755 drivers/input/sensors/accel/mxc622x.c create mode 100755 drivers/input/sensors/accel/mxc6655xa.c create mode 100755 drivers/input/sensors/accel/sc7660.c create mode 100755 drivers/input/sensors/accel/sc7a20.c create mode 100755 drivers/input/sensors/accel/sc7a30.c create mode 100755 drivers/input/sensors/accel/stk8baxx.c create mode 100755 drivers/input/sensors/angle/Kconfig create mode 100755 drivers/input/sensors/angle/Makefile create mode 100755 drivers/input/sensors/angle/angle_kxtik.c create mode 100755 drivers/input/sensors/angle/angle_lis3dh.c create mode 100755 drivers/input/sensors/compass/Kconfig create mode 100755 drivers/input/sensors/compass/Makefile create mode 100755 drivers/input/sensors/compass/ak09911.c create mode 100755 drivers/input/sensors/compass/ak09918.c create mode 100755 drivers/input/sensors/compass/ak8963.c create mode 100755 drivers/input/sensors/compass/ak8975.c create mode 100755 drivers/input/sensors/gyro/Kconfig create mode 100755 drivers/input/sensors/gyro/Makefile create mode 100755 drivers/input/sensors/gyro/ewtsa.c create mode 100755 drivers/input/sensors/gyro/icm2060x_gyro.c create mode 100755 drivers/input/sensors/gyro/l3g20d.c create mode 100755 drivers/input/sensors/gyro/l3g4200d.c create mode 100755 drivers/input/sensors/gyro/lsm330_gyro.c create mode 100755 drivers/input/sensors/gyro/mpu6500_gyro.c create mode 100755 drivers/input/sensors/gyro/mpu6880_gyro.c create mode 100755 drivers/input/sensors/hall/Kconfig create mode 100755 drivers/input/sensors/hall/Makefile create mode 100755 drivers/input/sensors/hall/mh248.c create mode 100755 drivers/input/sensors/hall/och165t_hall.c create mode 100755 drivers/input/sensors/lsensor/Kconfig create mode 100755 drivers/input/sensors/lsensor/Makefile create mode 100755 drivers/input/sensors/lsensor/cm3217.c create mode 100755 drivers/input/sensors/lsensor/cm3218.c create mode 100755 drivers/input/sensors/lsensor/cm3232.c create mode 100755 drivers/input/sensors/lsensor/isl29023.c create mode 100755 drivers/input/sensors/lsensor/ls_al3006.c create mode 100755 drivers/input/sensors/lsensor/ls_ap321xx.c create mode 100755 drivers/input/sensors/lsensor/ls_em3071x.c create mode 100755 drivers/input/sensors/lsensor/ls_stk3171.c create mode 100755 drivers/input/sensors/lsensor/ls_stk3410.c create mode 100755 drivers/input/sensors/lsensor/ls_us5152.c create mode 100755 drivers/input/sensors/lsensor/ls_us5152.h create mode 100755 drivers/input/sensors/pressure/Kconfig create mode 100755 drivers/input/sensors/pressure/Makefile create mode 100755 drivers/input/sensors/pressure/pr_ms5607.c create mode 100755 drivers/input/sensors/psensor/Kconfig create mode 100755 drivers/input/sensors/psensor/Makefile create mode 100755 drivers/input/sensors/psensor/ps_al3006.c create mode 100755 drivers/input/sensors/psensor/ps_ap321xx.c create mode 100755 drivers/input/sensors/psensor/ps_em3071x.c create mode 100755 drivers/input/sensors/psensor/ps_stk3171.c create mode 100755 drivers/input/sensors/psensor/ps_stk3410.c create mode 100755 drivers/input/sensors/sensor-dev.c create mode 100755 drivers/input/sensors/sensor-i2c.c create mode 100755 drivers/input/sensors/temperature/Kconfig create mode 100755 drivers/input/sensors/temperature/Makefile create mode 100755 drivers/input/sensors/temperature/tmp_ms5607.c create mode 100755 drivers/input/touchscreen/cy8c_touchpad.c create mode 100755 drivers/input/touchscreen/cyttsp5/Kconfig create mode 100755 drivers/input/touchscreen/cyttsp5/Makefile create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_btn.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_core.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_core.h create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_debug.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_device_access.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_devtree.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_i2c.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_loader.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_mt_common.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_mta.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_mtb.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_params.h create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_params_pid00.h create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_params_pid01.h create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_platform.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_platform.h create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_proximity.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_regs.h create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_spi.c create mode 100755 drivers/input/touchscreen/cyttsp5/cyttsp5_test_device_access_api.c create mode 100755 drivers/input/touchscreen/focaltech_touch/Makefile create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_common.h create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_config.h create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_core.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_core.h create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_esdcheck.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_ex_fun.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_ex_mode.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_flash.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_flash.h create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_flash/Makefile create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_flash/focaltech_upgrade_ft8201.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_gesture.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_i2c.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_point_report_check.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_test/Makefile create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_test/focaltech_test.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_test/focaltech_test.h create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_test/focaltech_test_ini.c create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_test/focaltech_test_ini.h create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_test/supported_ic/Makefile create mode 100755 drivers/input/touchscreen/focaltech_touch/focaltech_test/supported_ic/focaltech_test_ft8201.c create mode 100755 drivers/input/touchscreen/focaltech_touch/include/firmware/fw_sample.h create mode 100755 drivers/input/touchscreen/focaltech_touch/include/pramboot/FT8006M_Pramboot_V1.6_20180426_le.h create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/Makefile create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_common.h create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_config.h create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_core.c create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_core.h create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_esdcheck.c create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_ex_fun.c create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_ex_mode.c create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_flash.c create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_flash.h create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_flash/Makefile create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_flash/focaltech_upgrade_ft5436.c create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_gesture.c create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_i2c.c create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/focaltech_point_report_check.c create mode 100755 drivers/input/touchscreen/focaltech_touch_ft5436/include/firmware/fw_sample.h create mode 100755 drivers/input/touchscreen/gsl3673.c create mode 100755 drivers/input/touchscreen/gsl3673.h create mode 100755 drivers/input/touchscreen/gsl3673_800x1280.c create mode 100755 drivers/input/touchscreen/gsl3673_800x1280.h create mode 100755 drivers/input/touchscreen/gsl3676/Makefile create mode 100755 drivers/input/touchscreen/gsl3676/gsl3676.c create mode 100755 drivers/input/touchscreen/gsl3676/rk3368_th863c_10.h create mode 100755 drivers/input/touchscreen/gsl_point_id.c create mode 100755 drivers/input/touchscreen/gslx680.c create mode 100755 drivers/input/touchscreen/gslx680.h create mode 100755 drivers/input/touchscreen/gslx6801.c create mode 100755 drivers/input/touchscreen/gslx6801.h create mode 100755 drivers/input/touchscreen/gslx680_d708/Makefile create mode 100755 drivers/input/touchscreen/gslx680_d708/rockchip_gsl3670.h create mode 100755 drivers/input/touchscreen/gslx680_d708/rockchip_gslX680.c create mode 100755 drivers/input/touchscreen/gslx680_d708/rockchip_gslX680_88v.h create mode 100755 drivers/input/touchscreen/gslx680_firefly.c create mode 100755 drivers/input/touchscreen/gslx680_firefly.h create mode 100755 drivers/input/touchscreen/gslx680_pad.c create mode 100755 drivers/input/touchscreen/gslx680_pad.h create mode 100755 drivers/input/touchscreen/gslx680a.c create mode 100755 drivers/input/touchscreen/gt1x/GT5688_Config_20170713_1080_1920.cfg create mode 100755 drivers/input/touchscreen/gt1x/Makefile create mode 100755 drivers/input/touchscreen/gt1x/gt1x.c create mode 100755 drivers/input/touchscreen/gt1x/gt1x.h create mode 100755 drivers/input/touchscreen/gt1x/gt1x_cfg.h create mode 100755 drivers/input/touchscreen/gt1x/gt1x_extents.c create mode 100755 drivers/input/touchscreen/gt1x/gt1x_firmware.h create mode 100755 drivers/input/touchscreen/gt1x/gt1x_generic.c create mode 100755 drivers/input/touchscreen/gt1x/gt1x_generic.h create mode 100755 drivers/input/touchscreen/gt1x/gt1x_tools.c create mode 100755 drivers/input/touchscreen/gt1x/gt1x_update.c create mode 100755 drivers/input/touchscreen/gt9xx/GT9110P(2020)V71_Config_20201028_170326.cfg create mode 100755 drivers/input/touchscreen/gt9xx/GT9110P_Config_20160217_1526_2048_97.cfg create mode 100755 drivers/input/touchscreen/gt9xx/GT9271_Config_20170526.cfg create mode 100755 drivers/input/touchscreen/gt9xx/HLS-0102-1398V1-1060-GT911_Config_20201204_V66.cfg create mode 100755 drivers/input/touchscreen/gt9xx/Makefile create mode 100755 drivers/input/touchscreen/gt9xx/WGJ10162B_GT9271_1060_Config_20140821_1341110X42.cfg create mode 100755 drivers/input/touchscreen/gt9xx/WGJ10162_GT9271_Config_20140820_182456.cfg create mode 100755 drivers/input/touchscreen/gt9xx/WGJ10187_GT910_Config_20140623_104014_0X41.cfg create mode 100755 drivers/input/touchscreen/gt9xx/WGJ10187_GT9271_Config_20140623_104014_0X41.cfg create mode 100755 drivers/input/touchscreen/gt9xx/WGJ89006B_GT911_Config_20140625_085816_0X43.cfg create mode 100755 drivers/input/touchscreen/gt9xx/WGJ89006B_GT9271_Config_20140625_085816_0X41.cfg create mode 100755 drivers/input/touchscreen/gt9xx/goodix_tool.c create mode 100755 drivers/input/touchscreen/gt9xx/gt9xx.c create mode 100755 drivers/input/touchscreen/gt9xx/gt9xx.h create mode 100755 drivers/input/touchscreen/gt9xx/gt9xx_cfg.h create mode 100755 drivers/input/touchscreen/gt9xx/gt9xx_firmware.h create mode 100755 drivers/input/touchscreen/gt9xx/gt9xx_update.c create mode 100755 drivers/input/touchscreen/hyn_cst2xx/CST21680SE_S126_D863_7.h create mode 100755 drivers/input/touchscreen/hyn_cst2xx/CST21680SE_S126_D89.h create mode 100755 drivers/input/touchscreen/hyn_cst2xx/Makefile create mode 100755 drivers/input/touchscreen/hyn_cst2xx/hyn_cst2xx.c create mode 100755 drivers/input/touchscreen/tp_gslx680_board.h create mode 100755 drivers/input/touchscreen/tp_suspend.h create mode 100755 drivers/input/touchscreen/vtl_ts/LX20JS06_A1_CT363_V03_5198_121015.dat create mode 100755 drivers/input/touchscreen/vtl_ts/Makefile create mode 100755 drivers/input/touchscreen/vtl_ts/apk.c create mode 100755 drivers/input/touchscreen/vtl_ts/apk.h create mode 100755 drivers/input/touchscreen/vtl_ts/chip.c create mode 100755 drivers/input/touchscreen/vtl_ts/chip.h create mode 100755 drivers/input/touchscreen/vtl_ts/f307&f317&f316_CT363S_01_V10_F7E9_140212.dat create mode 100755 drivers/input/touchscreen/vtl_ts/lx--js77_97_CT365_V01_E7DA_130419.dat create mode 100755 drivers/input/touchscreen/vtl_ts/tp_fw.h create mode 100755 drivers/input/touchscreen/vtl_ts/vtl_ts.c create mode 100755 drivers/input/touchscreen/vtl_ts/vtl_ts.h create mode 100755 drivers/input/touchscreen/vtl_ts/wgj36js07_A1_CT363_V01_0187_140306.dat create mode 100755 drivers/input/touchscreen/wacom_w9013.c create mode 100755 drivers/iommu/dma-iommu-ops.c create mode 100755 drivers/media/i2c/gc8034.c create mode 100755 drivers/media/i2c/rk628_csi.c create mode 100755 drivers/media/i2c/rk628_csi.h create mode 100755 drivers/media/platform/rockchip-tsp/Kconfig create mode 100755 drivers/media/platform/rockchip-tsp/Makefile create mode 100755 drivers/media/platform/rockchip-tsp/rockchip_tsp.c create mode 100755 drivers/media/platform/rockchip-tsp/rockchip_tsp.h create mode 100755 drivers/media/platform/rockchip/cif/Kconfig create mode 100755 drivers/media/platform/rockchip/cif/Makefile create mode 100755 drivers/media/platform/rockchip/cif/capture.c create mode 100755 drivers/media/platform/rockchip/cif/cif-luma.c create mode 100755 drivers/media/platform/rockchip/cif/cif-luma.h create mode 100755 drivers/media/platform/rockchip/cif/cif-scale.c create mode 100755 drivers/media/platform/rockchip/cif/common.c create mode 100755 drivers/media/platform/rockchip/cif/common.h create mode 100755 drivers/media/platform/rockchip/cif/dev.c create mode 100755 drivers/media/platform/rockchip/cif/dev.h create mode 100755 drivers/media/platform/rockchip/cif/hw.c create mode 100755 drivers/media/platform/rockchip/cif/hw.h create mode 100755 drivers/media/platform/rockchip/cif/mipi-csi2.c create mode 100755 drivers/media/platform/rockchip/cif/mipi-csi2.h create mode 100755 drivers/media/platform/rockchip/cif/procfs.c create mode 100755 drivers/media/platform/rockchip/cif/procfs.h create mode 100755 drivers/media/platform/rockchip/cif/regs.h create mode 100755 drivers/media/platform/rockchip/cif/subdev-itf.c create mode 100755 drivers/media/platform/rockchip/cif/subdev-itf.h create mode 100755 drivers/media/platform/rockchip/cif/version.h create mode 100755 drivers/media/platform/rockchip/isp/Kconfig create mode 100755 drivers/media/platform/rockchip/isp/Makefile create mode 100755 drivers/media/platform/rockchip/isp/bridge.c create mode 100755 drivers/media/platform/rockchip/isp/bridge.h create mode 100755 drivers/media/platform/rockchip/isp/bridge_v20.c create mode 100755 drivers/media/platform/rockchip/isp/bridge_v30.c create mode 100755 drivers/media/platform/rockchip/isp/capture.c create mode 100755 drivers/media/platform/rockchip/isp/capture.h create mode 100755 drivers/media/platform/rockchip/isp/capture_v1x.c create mode 100755 drivers/media/platform/rockchip/isp/capture_v1x.h create mode 100755 drivers/media/platform/rockchip/isp/capture_v20.c create mode 100755 drivers/media/platform/rockchip/isp/capture_v21.c create mode 100755 drivers/media/platform/rockchip/isp/capture_v2x.h create mode 100755 drivers/media/platform/rockchip/isp/capture_v30.c create mode 100755 drivers/media/platform/rockchip/isp/capture_v3x.h create mode 100755 drivers/media/platform/rockchip/isp/common.c create mode 100755 drivers/media/platform/rockchip/isp/common.h create mode 100755 drivers/media/platform/rockchip/isp/csi.c create mode 100755 drivers/media/platform/rockchip/isp/csi.h create mode 100755 drivers/media/platform/rockchip/isp/dev.c create mode 100755 drivers/media/platform/rockchip/isp/dev.h create mode 100755 drivers/media/platform/rockchip/isp/dmarx.c create mode 100755 drivers/media/platform/rockchip/isp/dmarx.h create mode 100755 drivers/media/platform/rockchip/isp/hw.c create mode 100755 drivers/media/platform/rockchip/isp/hw.h create mode 100755 drivers/media/platform/rockchip/isp/isp_external.h create mode 100755 drivers/media/platform/rockchip/isp/isp_ispp.h create mode 100755 drivers/media/platform/rockchip/isp/isp_mipi_luma.c create mode 100755 drivers/media/platform/rockchip/isp/isp_mipi_luma.h create mode 100755 drivers/media/platform/rockchip/isp/isp_params.c create mode 100755 drivers/media/platform/rockchip/isp/isp_params.h create mode 100755 drivers/media/platform/rockchip/isp/isp_params_v1x.c create mode 100755 drivers/media/platform/rockchip/isp/isp_params_v1x.h create mode 100755 drivers/media/platform/rockchip/isp/isp_params_v21.c create mode 100755 drivers/media/platform/rockchip/isp/isp_params_v21.h create mode 100755 drivers/media/platform/rockchip/isp/isp_params_v2x.c create mode 100755 drivers/media/platform/rockchip/isp/isp_params_v2x.h create mode 100755 drivers/media/platform/rockchip/isp/isp_params_v3x.c create mode 100755 drivers/media/platform/rockchip/isp/isp_params_v3x.h create mode 100755 drivers/media/platform/rockchip/isp/isp_stats.c create mode 100755 drivers/media/platform/rockchip/isp/isp_stats.h create mode 100755 drivers/media/platform/rockchip/isp/isp_stats_v1x.c create mode 100755 drivers/media/platform/rockchip/isp/isp_stats_v1x.h create mode 100755 drivers/media/platform/rockchip/isp/isp_stats_v21.c create mode 100755 drivers/media/platform/rockchip/isp/isp_stats_v21.h create mode 100755 drivers/media/platform/rockchip/isp/isp_stats_v2x.c create mode 100755 drivers/media/platform/rockchip/isp/isp_stats_v2x.h create mode 100755 drivers/media/platform/rockchip/isp/isp_stats_v3x.c create mode 100755 drivers/media/platform/rockchip/isp/isp_stats_v3x.h create mode 100755 drivers/media/platform/rockchip/isp/procfs.c create mode 100755 drivers/media/platform/rockchip/isp/procfs.h create mode 100755 drivers/media/platform/rockchip/isp/regs.c create mode 100755 drivers/media/platform/rockchip/isp/regs.h create mode 100755 drivers/media/platform/rockchip/isp/regs_v2x.h create mode 100755 drivers/media/platform/rockchip/isp/regs_v3x.h create mode 100755 drivers/media/platform/rockchip/isp/rkisp.c create mode 100755 drivers/media/platform/rockchip/isp/rkisp.h create mode 100755 drivers/media/platform/rockchip/isp/rkisp_tb_helper.c create mode 100755 drivers/media/platform/rockchip/isp/rkisp_tb_helper.h create mode 100755 drivers/media/platform/rockchip/isp/version.h create mode 100755 drivers/media/platform/rockchip/isp/videobuf2-rdma-sg.c create mode 100755 drivers/media/platform/rockchip/isp1/Kconfig create mode 100755 drivers/media/platform/rockchip/isp1/Makefile create mode 100755 drivers/media/platform/rockchip/isp1/capture.c create mode 100755 drivers/media/platform/rockchip/isp1/capture.h create mode 100755 drivers/media/platform/rockchip/isp1/common.h create mode 100755 drivers/media/platform/rockchip/isp1/dev.c create mode 100755 drivers/media/platform/rockchip/isp1/dev.h create mode 100755 drivers/media/platform/rockchip/isp1/dmarx.c create mode 100755 drivers/media/platform/rockchip/isp1/dmarx.h create mode 100755 drivers/media/platform/rockchip/isp1/isp_params.c create mode 100755 drivers/media/platform/rockchip/isp1/isp_params.h create mode 100755 drivers/media/platform/rockchip/isp1/isp_stats.c create mode 100755 drivers/media/platform/rockchip/isp1/isp_stats.h create mode 100755 drivers/media/platform/rockchip/isp1/regs.c create mode 100755 drivers/media/platform/rockchip/isp1/regs.h create mode 100755 drivers/media/platform/rockchip/isp1/rkisp1.c create mode 100755 drivers/media/platform/rockchip/isp1/rkisp1.h create mode 100755 drivers/media/platform/rockchip/isp1/version.h create mode 100755 drivers/media/platform/rockchip/ispp/Kconfig create mode 100755 drivers/media/platform/rockchip/ispp/Makefile create mode 100755 drivers/media/platform/rockchip/ispp/common.c create mode 100755 drivers/media/platform/rockchip/ispp/common.h create mode 100755 drivers/media/platform/rockchip/ispp/dev.c create mode 100755 drivers/media/platform/rockchip/ispp/dev.h create mode 100755 drivers/media/platform/rockchip/ispp/fec.c create mode 100755 drivers/media/platform/rockchip/ispp/fec.h create mode 100755 drivers/media/platform/rockchip/ispp/hw.c create mode 100755 drivers/media/platform/rockchip/ispp/hw.h create mode 100755 drivers/media/platform/rockchip/ispp/ispp.c create mode 100755 drivers/media/platform/rockchip/ispp/ispp.h create mode 100755 drivers/media/platform/rockchip/ispp/params.c create mode 100755 drivers/media/platform/rockchip/ispp/params.h create mode 100755 drivers/media/platform/rockchip/ispp/params_v10.c create mode 100755 drivers/media/platform/rockchip/ispp/params_v20.c create mode 100755 drivers/media/platform/rockchip/ispp/procfs.c create mode 100755 drivers/media/platform/rockchip/ispp/procfs.h create mode 100755 drivers/media/platform/rockchip/ispp/regs.h create mode 100755 drivers/media/platform/rockchip/ispp/stats.c create mode 100755 drivers/media/platform/rockchip/ispp/stats.h create mode 100755 drivers/media/platform/rockchip/ispp/stream.c create mode 100755 drivers/media/platform/rockchip/ispp/stream.h create mode 100755 drivers/media/platform/rockchip/ispp/stream_v10.c create mode 100755 drivers/media/platform/rockchip/ispp/stream_v20.c create mode 100755 drivers/media/platform/rockchip/ispp/version.h create mode 100755 drivers/mmc/host/rk_sdmmc.h create mode 100755 drivers/mmc/host/rk_sdmmc_ops.c create mode 100755 drivers/mmc/host/rk_sdmmc_ops.h create mode 100644 drivers/mtd/nand/raw/rockchip-nand-controller.c create mode 100755 drivers/net/ethernet/stmicro/stmmac/dwmac-rk-tool.c create mode 100755 drivers/net/ethernet/stmicro/stmmac/dwmac-rk-tool.h create mode 100755 drivers/net/wireless/rockchip_wlan/Kconfig create mode 100755 drivers/net/wireless/rockchip_wlan/Makefile create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/Kconfig create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/Makefile create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/Kconfig create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/Makefile create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/aiutils.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcm_app_utils.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmbloom.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmevent.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmsdh.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmsdh_linux.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmsdh_sdmmc.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmsdh_sdmmc_linux.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmsdspi_linux.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmspibrcm.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmstdlib_s.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmutils.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmwifi_channels.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmwifi_channels.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmwifi_rates.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmwifi_rspec.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/bcmxtlv.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dbus.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dbus_usb.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dbus_usb_linux.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_bus.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_buzzz.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_ccode.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_cdc.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_cfg80211.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_cfg80211.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_common.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_config.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_config.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_csi.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_csi.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_custom_gpio.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_custom_hikey.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_custom_memprealloc.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_dbg.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_dbg_ring.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_dbg_ring.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_debug.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_debug.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_debug_linux.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_flowring.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_flowring.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_gpio.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_ip.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_ip.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux_exportfs.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux_lb.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux_pktdump.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux_pktdump.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux_platdev.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux_priv.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux_sched.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux_wq.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_linux_wq.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_mschdbg.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_mschdbg.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_msgbuf.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_pcie.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_pcie.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_pcie_linux.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_pno.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_pno.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_proto.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_rtt.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_rtt.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_sdio.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_static_buf.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_wlfc.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dhd_wlfc.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dngl_stats.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/dngl_wlhdr.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/frag.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/frag.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/hdf_public_ap6275s.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/hnd_pktpool.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/hnd_pktq.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/hndlhl.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/hndmem.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/hndpmu.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/802.11.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/802.11e.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/802.11s.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/802.1d.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/802.3.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/aidmp.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcm_cfg.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcm_mpool_pub.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcm_ring.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmarp.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmbloom.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmcdc.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmdefs.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmdevs.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmdhcp.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmendian.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmeth.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmevent.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmicmp.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmiov.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmip.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmipv6.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmmsgbuf.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmnvram.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmpcie.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmpcispi.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmperf.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmsdbus.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmsdh.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmsdh_sdmmc.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmsdpcm.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmsdspi.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmsdstd.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmspi.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmspibrcm.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmsrom_fmt.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmsrom_tbl.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmstdlib_s.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmtcp.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmtlv.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmudp.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/bcmutils.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/brcm_nl80211.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/dbus.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/dhd_daemon.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/dhdioctl.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/dnglevent.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/eapol.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/epivers.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/etd.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/ethernet.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/event_log.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/event_log_payload.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/event_log_set.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/event_log_tag.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/event_trace.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/fils.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hnd_armtrap.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hnd_cons.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hnd_debug.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hnd_pktpool.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hnd_pktq.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hnd_trap.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hndchipc.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hndlhl.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hndmem.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hndoobr.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hndpmu.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/hndsoc.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/linux_osl.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/linux_pkt.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/linuxver.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/lpflags.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/mbo.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/miniopt.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/msf.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/msgtrace.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/nan.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/osl.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/osl_decl.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/osl_ext.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/p2p.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/packed_section_end.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/packed_section_start.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/pcicfg.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/pcie_core.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/rte_ioctl.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sbchipc.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sbconfig.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sbgci.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sbhndarm.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sbhnddma.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sbpcmcia.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sbsdio.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sbsdpcmdev.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sbsocram.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sbsysmem.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sdio.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sdioh.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sdiovar.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/sdspi.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/siutils.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/spid.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/trxhdr.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/typedefs.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/usbrdl.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/vlan.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/wlfc_proto.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/wlioctl.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/wlioctl_defs.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/wlioctl_utils.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/wpa.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/include/wps.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/linux_osl.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/linux_osl_priv.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/linux_pkt.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/otpdefs.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/pcie_core.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/sbutils.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/siutils.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/siutils_priv.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_android.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_android.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_android_ext.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfg80211.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfg80211.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfg_btcoex.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfgnan.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfgnan.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfgp2p.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfgp2p.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfgscan.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfgscan.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfgvendor.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_cfgvendor.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_dbg.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_escan.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_escan.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_event.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_ext_genl.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_iw.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_iw.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_linux_mon.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wl_roam.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wldev_common.c create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/bcmdhd_wifi6/wldev_common.h create mode 100755 drivers/net/wireless/rockchip_wlan/rkwifi/rk_wifi_config.c create mode 100755 drivers/pci/controller/dwc/pcie-dw-rockchip.c create mode 100755 drivers/pci/controller/rockchip-pcie-dma.c create mode 100755 drivers/pci/controller/rockchip-pcie-dma.h create mode 100755 drivers/phy/rockchip/phy-rockchip-csi2-dphy-common.h create mode 100755 drivers/phy/rockchip/phy-rockchip-csi2-dphy-hw.c create mode 100755 drivers/phy/rockchip/phy-rockchip-csi2-dphy.c create mode 100755 drivers/phy/rockchip/phy-rockchip-inno-combphy.c create mode 100755 drivers/phy/rockchip/phy-rockchip-inno-hdmi-phy.c create mode 100755 drivers/phy/rockchip/phy-rockchip-inno-mipi-dphy.c create mode 100755 drivers/phy/rockchip/phy-rockchip-inno-usb3.c create mode 100755 drivers/phy/rockchip/phy-rockchip-inno-video-combo-phy.c create mode 100755 drivers/phy/rockchip/phy-rockchip-inno-video-phy.c create mode 100755 drivers/phy/rockchip/phy-rockchip-mipi-rx.c create mode 100755 drivers/phy/rockchip/phy-rockchip-naneng-combphy.c create mode 100755 drivers/phy/rockchip/phy-rockchip-naneng-edp.c create mode 100755 drivers/phy/rockchip/phy-rockchip-naneng-usb2.c create mode 100755 drivers/phy/rockchip/phy-rockchip-samsung-dcphy.c create mode 100755 drivers/phy/rockchip/phy-rockchip-samsung-hdptx-hdmi.c create mode 100755 drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c create mode 100755 drivers/phy/rockchip/phy-rockchip-snps-pcie3.c create mode 100755 drivers/phy/rockchip/phy-rockchip-usbdp.c create mode 100755 drivers/pinctrl/pinctrl-rk628.c create mode 100755 drivers/pinctrl/pinctrl-rockchip.h create mode 100755 drivers/power/ec_battery.c create mode 100755 drivers/power/reset/reboot-mode.h create mode 100755 drivers/power/supply/bq25700_charger.c create mode 100755 drivers/power/supply/rk816_battery.c create mode 100755 drivers/power/supply/rk816_battery.h create mode 100755 drivers/power/supply/rk817_battery.c create mode 100755 drivers/power/supply/rk817_charger.c create mode 100755 drivers/power/supply/rk818_battery.c create mode 100755 drivers/power/supply/rk818_battery.h create mode 100755 drivers/power/supply/rk818_charger.c create mode 100755 drivers/power/sy6982c_charger.c create mode 100755 drivers/power/universal_charger.c create mode 100755 drivers/regulator/lp8752.c create mode 100755 drivers/regulator/mp8865-regulator.c create mode 100755 drivers/regulator/scmi-regulator.c create mode 100755 drivers/regulator/syr82x.c create mode 100755 drivers/regulator/tps549b22-regulator.c create mode 100755 drivers/regulator/xz3216.c create mode 100755 drivers/rk_nand/Kconfig create mode 100755 drivers/rk_nand/Makefile create mode 100755 drivers/rk_nand/rk_ftl_api.h create mode 100755 drivers/rk_nand/rk_ftl_arm_v7.S create mode 100755 drivers/rk_nand/rk_ftl_arm_v7_thumb.S create mode 100755 drivers/rk_nand/rk_ftl_arm_v8.S create mode 100755 drivers/rk_nand/rk_ftlv5_arm32.S create mode 100755 drivers/rk_nand/rk_ftlv5_arm64.S create mode 100755 drivers/rk_nand/rk_nand_base.c create mode 100755 drivers/rk_nand/rk_nand_base.h create mode 100755 drivers/rk_nand/rk_nand_blk.c create mode 100755 drivers/rk_nand/rk_nand_blk.h create mode 100755 drivers/rk_nand/rk_zftl_arm32.S create mode 100755 drivers/rk_nand/rk_zftl_arm64.S create mode 100755 drivers/rkflash/Kconfig create mode 100755 drivers/rkflash/Makefile create mode 100755 drivers/rkflash/flash.c create mode 100755 drivers/rkflash/flash.h create mode 100755 drivers/rkflash/flash_com.h create mode 100755 drivers/rkflash/nand_boot.c create mode 100755 drivers/rkflash/nandc.c create mode 100755 drivers/rkflash/nandc.h create mode 100755 drivers/rkflash/rk_sftl.h create mode 100755 drivers/rkflash/rk_sftl_arm_v7.S create mode 100755 drivers/rkflash/rk_sftl_arm_v7_thumb.S create mode 100755 drivers/rkflash/rk_sftl_arm_v8.S create mode 100755 drivers/rkflash/rkflash_api.h create mode 100755 drivers/rkflash/rkflash_blk.c create mode 100755 drivers/rkflash/rkflash_blk.h create mode 100755 drivers/rkflash/rkflash_debug.c create mode 100755 drivers/rkflash/rkflash_debug.h create mode 100755 drivers/rkflash/rknandc_base.c create mode 100755 drivers/rkflash/rksfc_base.c create mode 100755 drivers/rkflash/sfc.c create mode 100755 drivers/rkflash/sfc.h create mode 100755 drivers/rkflash/sfc_nand.c create mode 100755 drivers/rkflash/sfc_nand.h create mode 100755 drivers/rkflash/sfc_nand_boot.c create mode 100755 drivers/rkflash/sfc_nand_mtd.c create mode 100755 drivers/rkflash/sfc_nand_mtd.h create mode 100755 drivers/rkflash/sfc_nand_mtd_bbt.c create mode 100755 drivers/rkflash/sfc_nor.c create mode 100755 drivers/rkflash/sfc_nor.h create mode 100755 drivers/rkflash/sfc_nor_boot.c create mode 100755 drivers/rkflash/sfc_nor_mtd.c create mode 100755 drivers/rkflash/typedef.h create mode 100755 drivers/soc/rockchip/Kconfig.cpu create mode 100755 drivers/soc/rockchip/flash_vendor_storage.c create mode 100755 drivers/soc/rockchip/flash_vendor_storage.h create mode 100755 drivers/soc/rockchip/mtd_vendor_storage.c create mode 100755 drivers/soc/rockchip/rk_camera_sensor_info.h create mode 100755 drivers/soc/rockchip/rk_fiq_debugger.c create mode 100755 drivers/soc/rockchip/rk_vendor_storage.c create mode 100755 drivers/soc/rockchip/rockchip-cpuinfo.c create mode 100755 drivers/soc/rockchip/rockchip_debug.c create mode 100755 drivers/soc/rockchip/rockchip_debug.h create mode 100755 drivers/soc/rockchip/rockchip_decompress.c create mode 100755 drivers/soc/rockchip/rockchip_ipa.c create mode 100755 drivers/soc/rockchip/rockchip_opp_select.c create mode 100755 drivers/soc/rockchip/rockchip_pm_config.c create mode 100755 drivers/soc/rockchip/rockchip_pvtm.c create mode 100755 drivers/soc/rockchip/rockchip_ramdisk.c create mode 100755 drivers/soc/rockchip/rockchip_system_monitor.c create mode 100755 drivers/soc/rockchip/rockchip_thunderboot_crypto.c create mode 100755 drivers/soc/rockchip/rockchip_thunderboot_mmc.c create mode 100755 drivers/soc/rockchip/rockchip_thunderboot_sfc.c create mode 100755 drivers/soc/rockchip/sdmmc_vendor_storage.c create mode 100755 drivers/staging/android/fiq_debugger/Kconfig create mode 100755 drivers/staging/android/fiq_debugger/Makefile create mode 100755 drivers/staging/android/fiq_debugger/fiq_debugger.c create mode 100755 drivers/staging/android/fiq_debugger/fiq_debugger.h create mode 100755 drivers/staging/android/fiq_debugger/fiq_debugger_arm.c create mode 100755 drivers/staging/android/fiq_debugger/fiq_debugger_arm64.c create mode 100755 drivers/staging/android/fiq_debugger/fiq_debugger_priv.h create mode 100755 drivers/staging/android/fiq_debugger/fiq_debugger_ringbuf.h create mode 100755 drivers/staging/android/fiq_debugger/fiq_watchdog.c create mode 100755 drivers/staging/android/fiq_debugger/fiq_watchdog.h create mode 100755 drivers/staging/android/ion/heaps/Kconfig create mode 100755 drivers/staging/android/ion/heaps/Makefile rename drivers/staging/android/ion/{ => heaps}/ion_cma_heap.c (72%) mode change 100644 => 100755 rename drivers/staging/android/ion/{ => heaps}/ion_page_pool.c (87%) mode change 100644 => 100755 create mode 100755 drivers/staging/android/ion/heaps/ion_page_pool.h rename drivers/staging/android/ion/{ => heaps}/ion_system_heap.c (65%) mode change 100644 => 100755 delete mode 100644 drivers/staging/android/ion/ion.h create mode 100755 drivers/staging/android/ion/ion_buffer.c create mode 100755 drivers/staging/android/ion/ion_dma_buf.c create mode 100755 drivers/staging/android/ion/ion_private.h create mode 100755 drivers/staging/android/ion/ion_protected_heap.c create mode 100755 drivers/staging/android/ion/ion_trace.h create mode 100755 drivers/staging/android/ion/rockchip/Makefile create mode 100755 drivers/staging/android/ion/rockchip/rockchip_ion.c create mode 100755 drivers/staging/android/ion/rockchip/rockchip_ion_snapshot.c create mode 100644 drivers/staging/blackbox/rockchip/Makefile create mode 100644 drivers/staging/blackbox/rockchip/system_adapter.c create mode 100644 drivers/staging/blackbox/rockchip/system_adapter_by_memory.c mode change 100644 => 100755 drivers/usb/core/hub.c mode change 100644 => 100755 drivers/usb/dwc2/core.h mode change 100644 => 100755 drivers/usb/dwc2/gadget.c mode change 100644 => 100755 drivers/usb/dwc2/hcd.c mode change 100644 => 100755 drivers/usb/dwc2/hcd_intr.c mode change 100644 => 100755 drivers/usb/dwc2/hcd_queue.c mode change 100644 => 100755 drivers/usb/dwc2/params.c mode change 100644 => 100755 drivers/usb/dwc2/platform.c mode change 100644 => 100755 drivers/usb/dwc3/Kconfig mode change 100644 => 100755 drivers/usb/dwc3/Makefile mode change 100644 => 100755 drivers/usb/dwc3/core.c mode change 100644 => 100755 drivers/usb/dwc3/core.h mode change 100644 => 100755 drivers/usb/dwc3/debug.h mode change 100644 => 100755 drivers/usb/dwc3/debugfs.c mode change 100644 => 100755 drivers/usb/dwc3/drd.c create mode 100755 drivers/usb/dwc3/dwc3-imx8mp.c mode change 100644 => 100755 drivers/usb/dwc3/dwc3-keystone.c mode change 100644 => 100755 drivers/usb/dwc3/dwc3-pci.c mode change 100644 => 100755 drivers/usb/dwc3/dwc3-qcom.c mode change 100644 => 100755 drivers/usb/dwc3/ep0.c mode change 100644 => 100755 drivers/usb/dwc3/gadget.c mode change 100644 => 100755 drivers/usb/dwc3/gadget.h mode change 100644 => 100755 drivers/usb/dwc3/trace.c mode change 100644 => 100755 drivers/usb/gadget/configfs.c mode change 100644 => 100755 drivers/usb/gadget/function/Makefile create mode 100755 drivers/usb/gadget/function/f_accessory.c create mode 100755 drivers/usb/gadget/function/f_audio_source.c mode change 100644 => 100755 drivers/usb/gadget/function/f_fs.c mode change 100644 => 100755 drivers/usb/gadget/function/f_mass_storage.c mode change 100644 => 100755 drivers/usb/gadget/function/f_midi.c mode change 100644 => 100755 drivers/usb/gadget/function/f_printer.c mode change 100644 => 100755 drivers/usb/gadget/function/f_uac1.c mode change 100644 => 100755 drivers/usb/gadget/function/f_uac2.c mode change 100644 => 100755 drivers/usb/gadget/function/u_audio.c mode change 100644 => 100755 drivers/usb/gadget/function/u_audio.h mode change 100644 => 100755 drivers/usb/gadget/function/u_ether.c mode change 100644 => 100755 drivers/usb/gadget/function/u_ether.h mode change 100644 => 100755 drivers/usb/gadget/function/u_ether_configfs.h mode change 100644 => 100755 drivers/usb/gadget/function/u_serial.c create mode 100755 drivers/usb/gadget/function/u_uac.h mode change 100644 => 100755 drivers/usb/gadget/function/u_uac1.h mode change 100644 => 100755 drivers/usb/gadget/function/u_uac2.h mode change 100644 => 100755 drivers/usb/gadget/legacy/audio.c mode change 100644 => 100755 drivers/usb/gadget/udc/core.c mode change 100644 => 100755 drivers/usb/gadget/udc/snps_udc_core.c mode change 100644 => 100755 drivers/usb/gadget/udc/trace.c mode change 100644 => 100755 drivers/usb/gadget/udc/udc-xilinx.c mode change 100644 => 100755 drivers/usb/host/Makefile mode change 100644 => 100755 drivers/usb/host/ehci-platform.c mode change 100644 => 100755 drivers/usb/host/ohci-platform.c mode change 100644 => 100755 drivers/usb/host/xhci-hub.c mode change 100644 => 100755 drivers/usb/host/xhci-mem.c mode change 100644 => 100755 drivers/usb/host/xhci-mtk-sch.c mode change 100644 => 100755 drivers/usb/host/xhci-mtk.c mode change 100644 => 100755 drivers/usb/host/xhci-mtk.h mode change 100644 => 100755 drivers/usb/host/xhci-plat.c mode change 100644 => 100755 drivers/usb/host/xhci-plat.h mode change 100644 => 100755 drivers/usb/host/xhci-ring.c mode change 100644 => 100755 drivers/usb/host/xhci-trace.c mode change 100644 => 100755 drivers/usb/host/xhci.c mode change 100644 => 100755 drivers/usb/host/xhci.h mode change 100644 => 100755 drivers/usb/typec/altmodes/Kconfig mode change 100644 => 100755 drivers/usb/typec/altmodes/displayport.c mode change 100644 => 100755 drivers/usb/typec/class.c mode change 100644 => 100755 drivers/usb/typec/mux/intel_pmc_mux.c mode change 100644 => 100755 drivers/usb/typec/tcpm/Kconfig mode change 100644 => 100755 drivers/usb/typec/tcpm/Makefile mode change 100644 => 100755 drivers/usb/typec/tcpm/fusb302.c mode change 100644 => 100755 drivers/usb/typec/tcpm/tcpci.c mode change 100644 => 100755 drivers/usb/typec/tcpm/tcpci.h create mode 100755 drivers/usb/typec/tcpm/tcpci_husb311.c mode change 100644 => 100755 drivers/usb/typec/tcpm/tcpci_maxim.c mode change 100644 => 100755 drivers/usb/typec/tcpm/tcpm.c mode change 100644 => 100755 drivers/usb/typec/tcpm/wcove.c mode change 100644 => 100755 drivers/usb/typec/ucsi/Kconfig mode change 100644 => 100755 drivers/usb/typec/ucsi/displayport.c mode change 100644 => 100755 drivers/usb/typec/ucsi/ucsi.c mode change 100644 => 100755 drivers/usb/typec/ucsi/ucsi.h create mode 100755 drivers/video/rockchip/Kconfig create mode 100755 drivers/video/rockchip/Makefile create mode 100755 drivers/video/rockchip/iep/Kconfig create mode 100755 drivers/video/rockchip/iep/Makefile create mode 100755 drivers/video/rockchip/iep/hw_iep_config_addr.h create mode 100755 drivers/video/rockchip/iep/hw_iep_reg.c create mode 100755 drivers/video/rockchip/iep/hw_iep_reg.h create mode 100755 drivers/video/rockchip/iep/iep.h create mode 100755 drivers/video/rockchip/iep/iep_drv.c create mode 100755 drivers/video/rockchip/iep/iep_drv.h create mode 100755 drivers/video/rockchip/iep/iep_iommu_drm.c create mode 100755 drivers/video/rockchip/iep/iep_iommu_ops.c create mode 100755 drivers/video/rockchip/iep/iep_iommu_ops.h create mode 100755 drivers/video/rockchip/mpp/Kconfig create mode 100755 drivers/video/rockchip/mpp/Makefile create mode 100755 drivers/video/rockchip/mpp/hack/mpp_hack_px30.c create mode 100755 drivers/video/rockchip/mpp/hack/mpp_hack_px30.h create mode 100755 drivers/video/rockchip/mpp/hack/mpp_rkvdec2_hack_rk3568.c create mode 100755 drivers/video/rockchip/mpp/hack/mpp_rkvdec2_link_hack_rk3568.c create mode 100755 drivers/video/rockchip/mpp/mpp_common.c create mode 100755 drivers/video/rockchip/mpp/mpp_common.h create mode 100755 drivers/video/rockchip/mpp/mpp_debug.h create mode 100755 drivers/video/rockchip/mpp/mpp_iep2.c create mode 100755 drivers/video/rockchip/mpp/mpp_iommu.c create mode 100755 drivers/video/rockchip/mpp/mpp_iommu.h create mode 100755 drivers/video/rockchip/mpp/mpp_jpgdec.c create mode 100755 drivers/video/rockchip/mpp/mpp_rkvdec.c create mode 100755 drivers/video/rockchip/mpp/mpp_rkvdec2.c create mode 100755 drivers/video/rockchip/mpp/mpp_rkvdec2.h create mode 100755 drivers/video/rockchip/mpp/mpp_rkvdec2_link.c create mode 100755 drivers/video/rockchip/mpp/mpp_rkvdec2_link.h create mode 100755 drivers/video/rockchip/mpp/mpp_rkvenc.c create mode 100755 drivers/video/rockchip/mpp/mpp_rkvenc2.c create mode 100755 drivers/video/rockchip/mpp/mpp_service.c create mode 100755 drivers/video/rockchip/mpp/mpp_vdpu1.c create mode 100755 drivers/video/rockchip/mpp/mpp_vdpu2.c create mode 100755 drivers/video/rockchip/mpp/mpp_vepu1.c create mode 100755 drivers/video/rockchip/mpp/mpp_vepu2.c create mode 100755 drivers/video/rockchip/mpp/rockchip_iep2_regs.h create mode 100755 drivers/video/rockchip/rga/Kconfig create mode 100755 drivers/video/rockchip/rga/Makefile create mode 100755 drivers/video/rockchip/rga/RGA_API.c create mode 100755 drivers/video/rockchip/rga/RGA_API.h create mode 100755 drivers/video/rockchip/rga/rga.h create mode 100755 drivers/video/rockchip/rga/rga_drv.c create mode 100755 drivers/video/rockchip/rga/rga_mmu_info.c create mode 100755 drivers/video/rockchip/rga/rga_mmu_info.h create mode 100755 drivers/video/rockchip/rga/rga_reg_info.c create mode 100755 drivers/video/rockchip/rga/rga_reg_info.h create mode 100755 drivers/video/rockchip/rga/rga_rop.h create mode 100755 drivers/video/rockchip/rga/rga_type.h create mode 100755 drivers/video/rockchip/rga2/Kconfig create mode 100755 drivers/video/rockchip/rga2/Makefile create mode 100755 drivers/video/rockchip/rga2/RGA2_API.c create mode 100755 drivers/video/rockchip/rga2/RGA2_API.h create mode 100755 drivers/video/rockchip/rga2/rga2.h create mode 100755 drivers/video/rockchip/rga2/rga2_debugger.c create mode 100755 drivers/video/rockchip/rga2/rga2_debugger.h create mode 100755 drivers/video/rockchip/rga2/rga2_drv.c create mode 100755 drivers/video/rockchip/rga2/rga2_mmu_info.c create mode 100755 drivers/video/rockchip/rga2/rga2_mmu_info.h create mode 100755 drivers/video/rockchip/rga2/rga2_reg_info.c create mode 100755 drivers/video/rockchip/rga2/rga2_reg_info.h create mode 100755 drivers/video/rockchip/rga2/rga2_rop.h create mode 100755 drivers/video/rockchip/rga2/rga2_type.h create mode 100755 include/drm/drm_sync_helper.h create mode 100755 include/dt-bindings/clock/rk1808-cru.h create mode 100755 include/dt-bindings/clock/rk3568-cru.h create mode 100755 include/dt-bindings/clock/rk618-cru.h create mode 100755 include/dt-bindings/clock/rk628-cgu.h create mode 100755 include/dt-bindings/clock/rockchip,rk3036.h create mode 100755 include/dt-bindings/clock/rockchip,rk312x.h create mode 100755 include/dt-bindings/clock/rockchip,rk3188.h create mode 100755 include/dt-bindings/clock/rockchip,rk3228.h create mode 100755 include/dt-bindings/clock/rockchip,rk3288.h create mode 100755 include/dt-bindings/clock/rockchip,rk3368.h create mode 100755 include/dt-bindings/clock/rockchip-ddr.h create mode 100755 include/dt-bindings/clock/rockchip.h create mode 100755 include/dt-bindings/clock/rv1126-cru.h create mode 100755 include/dt-bindings/display/drm_mipi_dsi.h create mode 100755 include/dt-bindings/display/media-bus-format.h create mode 100755 include/dt-bindings/display/mipi_dsi.h create mode 100644 include/dt-bindings/display/rockchip_vop.h create mode 100755 include/dt-bindings/dram/rockchip,rk322x.h create mode 100755 include/dt-bindings/dram/rockchip,rk3368.h create mode 100755 include/dt-bindings/input/rk-input.h create mode 100755 include/dt-bindings/memory/px30-dram.h create mode 100755 include/dt-bindings/memory/rk1808-dram.h create mode 100755 include/dt-bindings/memory/rk3128-dram.h create mode 100755 include/dt-bindings/memory/rk3288-dram.h create mode 100755 include/dt-bindings/memory/rk3328-dram.h create mode 100755 include/dt-bindings/memory/rk3368-dram.h create mode 100755 include/dt-bindings/memory/rk3399-dram.h create mode 100755 include/dt-bindings/memory/rk3568-dram.h create mode 100755 include/dt-bindings/memory/rockchip-dram.h create mode 100755 include/dt-bindings/memory/rv1126-dram.h create mode 100755 include/dt-bindings/phy/phy-snps-pcie3.h create mode 100755 include/dt-bindings/pinctrl/rockchip-rk3036.h create mode 100755 include/dt-bindings/pinctrl/rockchip-rk312x.h create mode 100755 include/dt-bindings/pinctrl/rockchip-rk3188.h create mode 100755 include/dt-bindings/pinctrl/rockchip-rk3288.h create mode 100755 include/dt-bindings/power/rk1808-power.h create mode 100755 include/dt-bindings/power/rk3568-power.h create mode 100755 include/dt-bindings/power/rk3588-power.h create mode 100755 include/dt-bindings/power/rv1126-power.h create mode 100755 include/dt-bindings/reset/rk628-rgu.h create mode 100755 include/dt-bindings/sensor-dev.h create mode 100755 include/dt-bindings/soc/rockchip-system-status.h create mode 100755 include/dt-bindings/suspend/rockchip-px30.h create mode 100755 include/dt-bindings/suspend/rockchip-rk1808.h create mode 100755 include/dt-bindings/suspend/rockchip-rk322x.h create mode 100755 include/dt-bindings/suspend/rockchip-rk3288.h create mode 100755 include/dt-bindings/suspend/rockchip-rk3308.h create mode 100755 include/dt-bindings/suspend/rockchip-rk3328.h create mode 100755 include/dt-bindings/suspend/rockchip-rk3368.h create mode 100755 include/dt-bindings/suspend/rockchip-rk3399.h create mode 100755 include/dt-bindings/suspend/rockchip-rk3568.h create mode 100755 include/dt-bindings/suspend/rockchip-rv1126.h create mode 100755 include/linux/android_kabi.h create mode 100755 include/linux/clk/rockchip.h create mode 100755 include/linux/dma-buf-cache.h create mode 100755 include/linux/initramfs.h create mode 100755 include/linux/memory_group_manager.h create mode 100755 include/linux/phy/pcie.h create mode 100755 include/linux/phy/phy-rockchip-typec.h create mode 100755 include/linux/power/bq25700-charge.h create mode 100755 include/linux/power/cw2015_battery.h create mode 100755 include/linux/power/rk_usbbc.h create mode 100755 include/linux/rfkill-bt.h create mode 100755 include/linux/rfkill-wlan.h create mode 100755 include/linux/rk_keys.h create mode 100755 include/linux/rockchip-iovmm.h create mode 100755 include/linux/rockchip/cpu.h create mode 100755 include/linux/rockchip/grf.h create mode 100755 include/linux/rockchip/psci.h create mode 100755 include/linux/rockchip/rockchip_sip.h create mode 100755 include/linux/rockchip_ion.h create mode 100755 include/linux/sensor-dev.h create mode 100755 include/linux/soc/rockchip/pvtm.h create mode 100755 include/linux/soc/rockchip/rk_fiq_debugger.h create mode 100755 include/linux/soc/rockchip/rk_vendor_storage.h create mode 100755 include/linux/soc/rockchip/rockchip_decompress.h create mode 100755 include/linux/soc/rockchip/rockchip_thunderboot_crypto.h mode change 100644 => 100755 include/linux/usb/audio-v2.h mode change 100644 => 100755 include/linux/usb/audio.h create mode 100755 include/linux/usb/f_accessory.h mode change 100644 => 100755 include/linux/usb/gadget.h mode change 100644 => 100755 include/linux/usb/typec.h create mode 100755 include/linux/wakelock.h create mode 100755 include/linux/wakeup_reason.h create mode 100755 include/soc/rockchip/android-version.h create mode 100755 include/soc/rockchip/pm_domains.h create mode 100755 include/soc/rockchip/rk3368-mailbox.h create mode 100755 include/soc/rockchip/rkfb_dmc.h create mode 100755 include/soc/rockchip/rockchip-system-status.h create mode 100755 include/soc/rockchip/rockchip_dmc.h create mode 100755 include/soc/rockchip/rockchip_iommu.h create mode 100755 include/soc/rockchip/rockchip_ipa.h create mode 100755 include/soc/rockchip/rockchip_opp_select.h create mode 100755 include/soc/rockchip/rockchip_system_monitor.h create mode 100755 include/soc/rockchip/scpi.h create mode 100755 include/trace/events/thermal_ipa_power.h create mode 100755 include/uapi/drm/rockchip_drm.h create mode 100755 include/uapi/linux/fec-config.h create mode 100755 include/uapi/linux/rk-camera-module.h create mode 100755 include/uapi/linux/rk-led-flash.h create mode 100755 include/uapi/linux/rk-pcie-dma.h create mode 100755 include/uapi/linux/rk-preisp.h create mode 100755 include/uapi/linux/rk-video-format.h create mode 100755 include/uapi/linux/rk803.h create mode 100755 include/uapi/linux/rk_vcm_head.h create mode 100755 include/uapi/linux/rkcif-config.h create mode 100755 include/uapi/linux/rkisp1-config.h create mode 100755 include/uapi/linux/rkisp2-config.h create mode 100755 include/uapi/linux/rkisp21-config.h create mode 100755 include/uapi/linux/rkisp3-config.h create mode 100755 include/uapi/linux/rkispp-config.h create mode 100755 include/uapi/linux/usb/f_accessory.h create mode 100755 kernel/power/wakeup_reason.c create mode 100755 make-boot.sh create mode 100755 make-ohos.sh create mode 100755 net/rfkill/rfkill-bt.c create mode 100755 net/rfkill/rfkill-wlan.c create mode 100755 scripts/clang-wrapper.py create mode 100755 scripts/mkimg create mode 100755 scripts/repack-bootimg create mode 100755 scripts/resource_tool.c create mode 100755 sound/soc/codecs/dummy-codec.c create mode 100755 sound/soc/codecs/es7202.c create mode 100755 sound/soc/codecs/es7202.h create mode 100755 sound/soc/codecs/es7210.c create mode 100755 sound/soc/codecs/es7210.h create mode 100755 sound/soc/codecs/es7243e.c create mode 100755 sound/soc/codecs/es7243e_usr_cfg.h create mode 100755 sound/soc/codecs/es8311.c create mode 100755 sound/soc/codecs/es8311.h create mode 100755 sound/soc/codecs/es8323.c create mode 100755 sound/soc/codecs/es8323.h create mode 100755 sound/soc/codecs/es8396.c create mode 100755 sound/soc/codecs/es8396.h create mode 100755 sound/soc/codecs/rk1000_codec.c create mode 100755 sound/soc/codecs/rk1000_codec.h create mode 100755 sound/soc/codecs/rk312x_codec.c create mode 100755 sound/soc/codecs/rk312x_codec.h create mode 100755 sound/soc/codecs/rk3228_codec.c create mode 100755 sound/soc/codecs/rk3228_codec.h create mode 100755 sound/soc/codecs/rk3308_codec.c create mode 100755 sound/soc/codecs/rk3308_codec.h create mode 100755 sound/soc/codecs/rk3308_codec_provider.h create mode 100755 sound/soc/codecs/rk817_codec.c create mode 100755 sound/soc/codecs/rk817_codec.h create mode 100755 sound/soc/codecs/rk_codec_digital.c create mode 100755 sound/soc/codecs/rk_codec_digital.h create mode 100644 sound/soc/rockchip/rockchip_audio_pwm.c create mode 100644 sound/soc/rockchip/rockchip_audio_pwm.h create mode 100644 sound/soc/rockchip/rockchip_cdndp.c create mode 100644 sound/soc/rockchip/rockchip_da7219.c create mode 100644 sound/soc/rockchip/rockchip_hdmi.c create mode 100644 sound/soc/rockchip/rockchip_hdmi_analog.c create mode 100644 sound/soc/rockchip/rockchip_hdmi_dp.c create mode 100644 sound/soc/rockchip/rockchip_i2s_tdm.c create mode 100644 sound/soc/rockchip/rockchip_i2s_tdm.h create mode 100644 sound/soc/rockchip/rockchip_multi_dais.c create mode 100644 sound/soc/rockchip/rockchip_multi_dais.h create mode 100644 sound/soc/rockchip/rockchip_multi_dais_pcm.c create mode 100644 sound/soc/rockchip/rockchip_multicodecs.c create mode 100644 sound/soc/rockchip/rockchip_rt5651.c create mode 100644 sound/soc/rockchip/rockchip_spdifrx.c create mode 100644 sound/soc/rockchip/rockchip_spdifrx.h create mode 100644 sound/soc/rockchip/rockchip_vad.c create mode 100644 sound/soc/rockchip/rockchip_vad.h create mode 100644 sound/soc/rockchip/vad_preprocess.h create mode 100644 sound/soc/rockchip/vad_preprocess_arm.S create mode 100644 sound/soc/rockchip/vad_preprocess_arm64.S create mode 100644 sound/soc/rockchip/vad_preprocess_thumb.S diff --git a/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt b/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt index 148191b0fc15..eee2a7f7cb92 100644 --- a/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt +++ b/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt @@ -1,8 +1,22 @@ -* Rockchip rk3399 DFI device +* Rockchip DFI device Required properties: -- compatible: Must be "rockchip,rk3399-dfi". +- compatible: Should be one of the following. + - "rockchip,px30-dfi" - for PX30 SoCs. + - "rockchip,rk1808-dfi" - for RK1808 SoCs. + - "rockchip,rk3128-dfi" - for RK3128 SoCs. + - "rockchip,rk3288-dfi" - for RK3288 SoCs. + - "rockchip,rk3328-dfi" - for RK3328 SoCs. + - "rockchip,rk3368-dfi" - for RK3368 SoCs. + - "rockchip,rk3399-dfi" - for RK3399 SoCs. + - "rockchip,rk3568-dfi" - for RK3568 SoCs. + - "rockchip,rv1126-dfi" - for RV1126 SoCs. + +Required properties for RK3368: +- rockchip,grf: phandle to the syscon managing the "general register files" + +Required properties for RK3399: - reg: physical base address of each DFI and length of memory mapped region - rockchip,pmu: phandle to the syscon managing the "pmu general register files" - clocks: phandles for clock specified in "clock-names" property diff --git a/Documentation/devicetree/bindings/display/bridge/analogix_dp.txt b/Documentation/devicetree/bindings/display/bridge/analogix_dp.txt index 027d76c27a41..a548f404033c 100644 --- a/Documentation/devicetree/bindings/display/bridge/analogix_dp.txt +++ b/Documentation/devicetree/bindings/display/bridge/analogix_dp.txt @@ -21,17 +21,23 @@ Required properties for dp-controller: from general PHY binding: Should be "dp". Optional properties for dp-controller: + -analogix,video-bist-enable: + Enable video bist pattern for DP_TX debugging. -force-hpd: Indicate driver need force hpd when hpd detect failed, this is used for some eDP screen which don't have hpd signal. -hpd-gpios: Hotplug detect GPIO. Indicates which GPIO should be used for hotplug detection + -panel-self-test: + Enable optional LCD Panel Self Test. -port@[X]: SoC specific port nodes with endpoint definitions as defined in Documentation/devicetree/bindings/media/video-interfaces.txt, please refer to the SoC specific binding document: * Documentation/devicetree/bindings/display/exynos/exynos_dp.txt * Documentation/devicetree/bindings/display/rockchip/analogix_dp-rockchip.txt + -support-psr: + Enable Source's PSR capability. [1]: Documentation/devicetree/bindings/media/video-interfaces.txt ------------------------------------------------------------------------------- diff --git a/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml b/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml index 1bb76197787b..de8e78d56dea 100644 --- a/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml @@ -18,6 +18,7 @@ properties: - items: - enum: - rockchip,px30-saradc + - rockchip,rk1808-saradc - rockchip,rk3308-saradc - rockchip,rk3328-saradc - rockchip,rv1108-saradc diff --git a/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt b/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt index 6ecefea1c6f9..0b906fb630e2 100644 --- a/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt +++ b/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt @@ -24,6 +24,10 @@ Optional properties: - rockchip,disable-mmu-reset : Don't use the mmu reset operation. Some mmu instances may produce unexpected results when the reset operation is used. +- rk_iommu,disable_reset_quirk : Same with above, for compatible with previous code + +- rockchip,skip-mmu-read : Some iommu instances are not able to be read, skip + reading operation to make iommu work as normal Example: diff --git a/Documentation/devicetree/bindings/mtd/rockchip,nand-controller.yaml b/Documentation/devicetree/bindings/mtd/rockchip,nand-controller.yaml new file mode 100644 index 000000000000..0922536b1811 --- /dev/null +++ b/Documentation/devicetree/bindings/mtd/rockchip,nand-controller.yaml @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mtd/rockchip,nand-controller.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Rockchip SoCs NAND FLASH Controller (NFC) + +allOf: + - $ref: "nand-controller.yaml#" + +maintainers: + - Heiko Stuebner + +properties: + compatible: + oneOf: + - const: rockchip,px30-nfc + - const: rockchip,rk2928-nfc + - const: rockchip,rv1108-nfc + - items: + - const: rockchip,rk3036-nfc + - const: rockchip,rk2928-nfc + - items: + - const: rockchip,rk3308-nfc + - const: rockchip,rv1108-nfc + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + minItems: 1 + items: + - description: Bus Clock + - description: Module Clock + + clock-names: + minItems: 1 + items: + - const: ahb + - const: nfc + + assigned-clocks: + maxItems: 1 + + assigned-clock-rates: + maxItems: 1 + + power-domains: + maxItems: 1 + +patternProperties: + "^nand@[0-7]$": + type: object + properties: + reg: + minimum: 0 + maximum: 7 + + nand-ecc-mode: + const: hw + + nand-ecc-step-size: + const: 1024 + + nand-ecc-strength: + enum: [16, 24, 40, 60, 70] + description: | + The ECC configurations that can be supported are as follows. + NFC v600 ECC 16, 24, 40, 60 + RK2928, RK3066, RK3188 + + NFC v622 ECC 16, 24, 40, 60 + RK3036, RK3128 + + NFC v800 ECC 16 + RK3308, RV1108 + + NFC v900 ECC 16, 40, 60, 70 + RK3326, PX30 + + nand-bus-width: + const: 8 + + rockchip,boot-blks: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 2 + default: 16 + description: + The NFC driver need this information to select ECC + algorithms supported by the boot ROM. + Only used in combination with 'nand-is-boot-medium'. + + rockchip,boot-ecc-strength: + enum: [16, 24, 40, 60, 70] + allOf: + - $ref: /schemas/types.yaml#/definitions/uint32 + description: | + If specified it indicates that a different BCH/ECC setting is + supported by the boot ROM. + NFC v600 ECC 16, 24 + RK2928, RK3066, RK3188 + + NFC v622 ECC 16, 24, 40, 60 + RK3036, RK3128 + + NFC v800 ECC 16 + RK3308, RV1108 + + NFC v900 ECC 16, 70 + RK3326, PX30 + + Only used in combination with 'nand-is-boot-medium'. + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + +unevaluatedProperties: false + +examples: + - | + #include + #include + nfc: nand-controller@ff4b0000 { + compatible = "rockchip,rk3308-nfc", + "rockchip,rv1108-nfc"; + reg = <0xff4b0000 0x4000>; + interrupts = ; + clocks = <&cru HCLK_NANDC>, <&cru SCLK_NANDC>; + clock-names = "ahb", "nfc"; + assigned-clocks = <&clks SCLK_NANDC>; + assigned-clock-rates = <150000000>; + + pinctrl-0 = <&flash_ale &flash_bus8 &flash_cle &flash_csn0 + &flash_rdn &flash_rdy &flash_wrn>; + pinctrl-names = "default"; + + #address-cells = <1>; + #size-cells = <0>; + + nand@0 { + reg = <0>; + label = "rk-nand"; + nand-bus-width = <8>; + nand-ecc-mode = "hw"; + nand-ecc-step-size = <1024>; + nand-ecc-strength = <16>; + nand-is-boot-medium; + rockchip,boot-blks = <8>; + rockchip,boot-ecc-strength = <16>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml b/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml index 104dd508565e..af1b9c31862a 100644 --- a/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml +++ b/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml @@ -15,10 +15,13 @@ allOf: properties: compatible: enum: + - rockchip,rk1808-efuse - rockchip,rk3066a-efuse + - rockchip,rk3128-efuse - rockchip,rk3188-efuse - rockchip,rk3228-efuse - rockchip,rk3288-efuse + - rockchip,rk3288-secure-efuse - rockchip,rk3328-efuse - rockchip,rk3368-efuse - rockchip,rk3399-efuse diff --git a/Documentation/devicetree/bindings/nvmem/rockchip-otp.txt b/Documentation/devicetree/bindings/nvmem/rockchip-otp.txt index 40f649f7c2e5..6e61345829e7 100644 --- a/Documentation/devicetree/bindings/nvmem/rockchip-otp.txt +++ b/Documentation/devicetree/bindings/nvmem/rockchip-otp.txt @@ -7,6 +7,8 @@ Required properties: - reg: Should contain the registers location and size - clocks: Must contain an entry for each entry in clock-names. - clock-names: Should be "otp", "apb_pclk" and "phy". + +Optional properties: - resets: Must contain an entry for each entry in reset-names. See ../../reset/reset.txt for details. - reset-names: Should be "phy". diff --git a/Documentation/devicetree/bindings/phy/phy-rockchip-typec.txt b/Documentation/devicetree/bindings/phy/phy-rockchip-typec.txt index 960da7fcaa9e..ac82f7b4dd58 100644 --- a/Documentation/devicetree/bindings/phy/phy-rockchip-typec.txt +++ b/Documentation/devicetree/bindings/phy/phy-rockchip-typec.txt @@ -17,6 +17,11 @@ Required properties: Optional properties: - extcon : extcon specifier for the Power Delivery + - rockchip,phy-config : A list of voltage swing(mV) and pre-emphasis + (dB) pairs. They are 3 blocks of 4 entries and + correspond to s0p0 ~ s0p3, s1p0 ~ s1p3, + s2p0 ~ s2p3, s3p0 ~ s2p3 swing and pre-emphasis + values. Required nodes : a sub-node is required for each port the phy provides. The sub-node name is used to identify dp or usb3 port, @@ -50,6 +55,21 @@ Example: <&cru SRST_P_UPHY0_TCPHY>; reset-names = "uphy", "uphy-pipe", "uphy-tcphy"; + rockchip,phy-config = <0x2a 0x00>, + <0x1f 0x15>, + <0x14 0x22>, + <0x02 0x2b>, + + <0x21 0x00>, + <0x12 0x15>, + <0x02 0x22>, + <0 0>, + + <0x15 0x00>, + <0x00 0x15>, + <0 0>, + <0 0>; + tcphy0_dp: dp-port { #phy-cells = <0>; }; @@ -74,6 +94,21 @@ Example: <&cru SRST_P_UPHY1_TCPHY>; reset-names = "uphy", "uphy-pipe", "uphy-tcphy"; + rockchip,phy-config = <0x2a 0x00>, + <0x1f 0x15>, + <0x14 0x22>, + <0x02 0x2b>, + + <0x21 0x00>, + <0x12 0x15>, + <0x02 0x22>, + <0 0>, + + <0x15 0x00>, + <0x00 0x15>, + <0 0>, + <0 0>; + tcphy1_dp: dp-port { #phy-cells = <0>; }; diff --git a/Documentation/devicetree/bindings/power/rockchip-io-domain.txt b/Documentation/devicetree/bindings/power/rockchip-io-domain.txt index e66fd4eab71c..09824bca248e 100644 --- a/Documentation/devicetree/bindings/power/rockchip-io-domain.txt +++ b/Documentation/devicetree/bindings/power/rockchip-io-domain.txt @@ -41,8 +41,10 @@ Required properties: - "rockchip,rk3368-pmu-io-voltage-domain" for rk3368 pmu-domains - "rockchip,rk3399-io-voltage-domain" for rk3399 - "rockchip,rk3399-pmu-io-voltage-domain" for rk3399 pmu-domains + - "rockchip,rk3568-pmu-io-voltage-domain" for rk3568 pmu-domains - "rockchip,rv1108-io-voltage-domain" for rv1108 - "rockchip,rv1108-pmu-io-voltage-domain" for rv1108 pmu-domains + - "rockchip,rv1126-pmu-io-voltage-domain" for rv1126 pmu-domains Deprecated properties: - rockchip,grf: phandle to the syscon managing the "general register files" @@ -112,11 +114,33 @@ Possible supplies for rk3399: - bt656-supply: The supply connected to APIO2_VDD. - audio-supply: The supply connected to APIO5_VDD. - sdmmc-supply: The supply connected to SDMMC0_VDD. -- gpio1830 The supply connected to APIO4_VDD. +- gpio1830-supply: The supply connected to APIO4_VDD. Possible supplies for rk3399 pmu-domains: - pmu1830-supply:The supply connected to PMUIO2_VDD. +Possible supplies for rk3568 pmu-domains: +- vccio1-supply:The supply connected to VCCIO1. +- vccio2-supply:The supply connected to VCCIO2, can be reserved since ignored by driver. +- vccio3-supply:The supply connected to VCCIO3. +- vccio4-supply:The supply connected to VCCIO4. +- vccio5-supply:The supply connected to VCCIO5. +- vccio6-supply:The supply connected to VCCIO6. +- vccio7-supply:The supply connected to VCCIO7. +- pmuio1-supply:The supply connected to PMUIO1, 3.3v only can be reserved +- pmuio2-supply:The supply connected to PMUIO2. + +Possible supplies for rv1126 pmu-domains: +- vccio1-supply:The supply connected to VCCIO1_VDD. +- vccio2-supply:The supply connected to VCCIO2_VDD. +- vccio3-supply:The supply connected to VCCIO3_VDD. +- vccio4-supply:The supply connected to VCCIO4_VDD. +- vccio5-supply:The supply connected to VCCIO5_VDD. +- vccio6-supply:The supply connected to VCCIO6_VDD. +- vccio7-supply:The supply connected to VCCIO7_VDD. +- pmuio1-supply:The supply connected to PMUIO1_VDD. +- pmuio2-supply:The supply connected to PMUIO2_VDD. + Example: io-domains { diff --git a/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt b/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt index 8304eceb62e4..e1920b15c86f 100644 --- a/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt +++ b/Documentation/devicetree/bindings/soc/rockchip/power_domain.txt @@ -16,6 +16,7 @@ Required properties for power domain controller: "rockchip,rk3366-power-controller" - for RK3366 SoCs. "rockchip,rk3368-power-controller" - for RK3368 SoCs. "rockchip,rk3399-power-controller" - for RK3399 SoCs. + "rockchip,rk3568-power-controller" - for RK3568 SoCs. - #power-domain-cells: Number of cells in a power-domain specifier. Should be 1 for multiple PM domains. - #address-cells: Should be 1. @@ -34,6 +35,7 @@ Required properties for power domain sub nodes: "include/dt-bindings/power/rk3366-power.h" - for RK3366 type power domain. "include/dt-bindings/power/rk3368-power.h" - for RK3368 type power domain. "include/dt-bindings/power/rk3399-power.h" - for RK3399 type power domain. + "include/dt-bindings/power/rk3568-power.h" - for RK3568 type power domain. - clocks (optional): phandles to clocks which need to be enabled while power domain switches state. - pm_qos (optional): phandles to qos blocks which need to be saved and restored @@ -114,6 +116,7 @@ The index should use macros in: "include/dt-bindings/power/rk3366-power.h" - for rk3366 type power domain. "include/dt-bindings/power/rk3368-power.h" - for rk3368 type power domain. "include/dt-bindings/power/rk3399-power.h" - for rk3399 type power domain. + "include/dt-bindings/power/rk3568-power.h" - for rk3568 type power domain. Example of the node using power domain: diff --git a/Documentation/devicetree/bindings/sound/rt5651.txt b/Documentation/devicetree/bindings/sound/rt5651.txt index 56e736a1cba9..e7cd338d2b59 100644 --- a/Documentation/devicetree/bindings/sound/rt5651.txt +++ b/Documentation/devicetree/bindings/sound/rt5651.txt @@ -38,6 +38,8 @@ Optional properties: 2: Scale current by 1.0 3: Scale current by 1.5 +- spk-con-gpio: speaker amplifier enable/disable control + Pins on the device (for linking into audio routes) for RT5651: * DMIC L1 diff --git a/Documentation/devicetree/bindings/spi/spi-rockchip.yaml b/Documentation/devicetree/bindings/spi/spi-rockchip.yaml index 1e6cf29e6388..7f987e79337c 100644 --- a/Documentation/devicetree/bindings/spi/spi-rockchip.yaml +++ b/Documentation/devicetree/bindings/spi/spi-rockchip.yaml @@ -33,6 +33,7 @@ properties: - rockchip,rk3328-spi - rockchip,rk3368-spi - rockchip,rk3399-spi + - rockchip,rv1126-spi - const: rockchip,rk3066-spi reg: diff --git a/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt b/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt index 7f94669e9ebe..346e466c2006 100644 --- a/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt +++ b/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt @@ -9,6 +9,7 @@ Required properties: "rockchip,rk3328-tsadc": found on RK3328 SoCs "rockchip,rk3368-tsadc": found on RK3368 SoCs "rockchip,rk3399-tsadc": found on RK3399 SoCs + "rockchip,rk3568-tsadc": found on RK3568 SoCs - reg : physical base address of the controller and length of memory mapped region. - interrupts : The interrupt number to the cpu. The interrupt specifier format diff --git a/arch/arm/boot/dts/rk3036.dtsi b/arch/arm/boot/dts/rk3036.dtsi index 093567022386..dc4c26a8a196 100644 --- a/arch/arm/boot/dts/rk3036.dtsi +++ b/arch/arm/boot/dts/rk3036.dtsi @@ -85,6 +85,11 @@ display-subsystem { ports = <&vop_out>; }; + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + }; + timer { compatible = "arm,armv7-timer"; arm,cpu-registers-not-fw-configured; @@ -360,7 +365,7 @@ pwm0: pwm@20050000 { #pwm-cells = <3>; clocks = <&cru PCLK_PWM>; clock-names = "pwm"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_pin>; status = "disabled"; }; @@ -371,7 +376,7 @@ pwm1: pwm@20050010 { #pwm-cells = <3>; clocks = <&cru PCLK_PWM>; clock-names = "pwm"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_pin>; status = "disabled"; }; @@ -382,7 +387,7 @@ pwm2: pwm@20050020 { #pwm-cells = <3>; clocks = <&cru PCLK_PWM>; clock-names = "pwm"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm2_pin>; status = "disabled"; }; @@ -393,7 +398,7 @@ pwm3: pwm@20050030 { #pwm-cells = <2>; clocks = <&cru PCLK_PWM>; clock-names = "pwm"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm3_pin>; status = "disabled"; }; diff --git a/arch/arm/boot/dts/rk3066a-rayeager.dts b/arch/arm/boot/dts/rk3066a-rayeager.dts index 309518403d86..1a9891f802f8 100644 --- a/arch/arm/boot/dts/rk3066a-rayeager.dts +++ b/arch/arm/boot/dts/rk3066a-rayeager.dts @@ -23,6 +23,20 @@ ir: ir-receiver { pinctrl-0 = <&ir_int>; }; + vdd_logic: vdd-logic { + compatible = "pwm-regulator"; + + pwms = <&pwm3 0 25000>; + pwm-dutycycle-range = <100 0>; + + regulator-name = "vdd_logic"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1400000>; + status = "okay"; + }; + keys: gpio-keys { compatible = "gpio-keys"; diff --git a/arch/arm/boot/dts/rk3066a.dtsi b/arch/arm/boot/dts/rk3066a.dtsi index bbc3bff50856..49b8652da097 100644 --- a/arch/arm/boot/dts/rk3066a.dtsi +++ b/arch/arm/boot/dts/rk3066a.dtsi @@ -800,22 +800,22 @@ power-domain@RK3066_PD_GPU { }; &pwm0 { - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_out>; }; &pwm1 { - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_out>; }; &pwm2 { - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm2_out>; }; &pwm3 { - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm3_out>; }; diff --git a/arch/arm/boot/dts/rk3188.dtsi b/arch/arm/boot/dts/rk3188.dtsi index ddf23748ead4..e3fe43499c8a 100644 --- a/arch/arm/boot/dts/rk3188.dtsi +++ b/arch/arm/boot/dts/rk3188.dtsi @@ -738,22 +738,22 @@ power-domain@RK3188_PD_GPU { }; &pwm0 { - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_out>; }; &pwm1 { - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_out>; }; &pwm2 { - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm2_out>; }; &pwm3 { - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm3_out>; }; diff --git a/arch/arm/boot/dts/rk322x.dtsi b/arch/arm/boot/dts/rk322x.dtsi index 2f17bf35d7a6..aae403bfaff1 100644 --- a/arch/arm/boot/dts/rk322x.dtsi +++ b/arch/arm/boot/dts/rk322x.dtsi @@ -398,7 +398,7 @@ pwm0: pwm@110b0000 { #pwm-cells = <3>; clocks = <&cru PCLK_PWM>; clock-names = "pwm"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_pin>; status = "disabled"; }; @@ -409,7 +409,7 @@ pwm1: pwm@110b0010 { #pwm-cells = <3>; clocks = <&cru PCLK_PWM>; clock-names = "pwm"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_pin>; status = "disabled"; }; @@ -420,7 +420,7 @@ pwm2: pwm@110b0020 { #pwm-cells = <3>; clocks = <&cru PCLK_PWM>; clock-names = "pwm"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm2_pin>; status = "disabled"; }; @@ -431,7 +431,7 @@ pwm3: pwm@110b0030 { #pwm-cells = <2>; clocks = <&cru PCLK_PWM>; clock-names = "pwm"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm3_pin>; status = "disabled"; }; diff --git a/arch/arm/boot/dts/rk3288.dtsi b/arch/arm/boot/dts/rk3288.dtsi index aab28161b9ae..1706453385fe 100644 --- a/arch/arm/boot/dts/rk3288.dtsi +++ b/arch/arm/boot/dts/rk3288.dtsi @@ -695,7 +695,7 @@ pwm0: pwm@ff680000 { compatible = "rockchip,rk3288-pwm"; reg = <0x0 0xff680000 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_pin>; clocks = <&cru PCLK_RKPWM>; clock-names = "pwm"; @@ -706,7 +706,7 @@ pwm1: pwm@ff680010 { compatible = "rockchip,rk3288-pwm"; reg = <0x0 0xff680010 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_pin>; clocks = <&cru PCLK_RKPWM>; clock-names = "pwm"; @@ -717,7 +717,7 @@ pwm2: pwm@ff680020 { compatible = "rockchip,rk3288-pwm"; reg = <0x0 0xff680020 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm2_pin>; clocks = <&cru PCLK_RKPWM>; clock-names = "pwm"; @@ -728,7 +728,7 @@ pwm3: pwm@ff680030 { compatible = "rockchip,rk3288-pwm"; reg = <0x0 0xff680030 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm3_pin>; clocks = <&cru PCLK_RKPWM>; clock-names = "pwm"; diff --git a/arch/arm/boot/dts/rv1108.dtsi b/arch/arm/boot/dts/rv1108.dtsi index a1a08cb9364e..b079c0008032 100644 --- a/arch/arm/boot/dts/rv1108.dtsi +++ b/arch/arm/boot/dts/rv1108.dtsi @@ -217,7 +217,7 @@ pwm4: pwm@10280000 { interrupts = ; clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm4_pin>; #pwm-cells = <3>; status = "disabled"; @@ -229,7 +229,7 @@ pwm5: pwm@10280010 { interrupts = ; clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm5_pin>; #pwm-cells = <3>; status = "disabled"; @@ -241,7 +241,7 @@ pwm6: pwm@10280020 { interrupts = ; clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm6_pin>; #pwm-cells = <3>; status = "disabled"; @@ -253,7 +253,7 @@ pwm7: pwm@10280030 { interrupts = ; clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm7_pin>; #pwm-cells = <3>; status = "disabled"; @@ -392,7 +392,7 @@ pwm0: pwm@20040000 { interrupts = ; clocks = <&cru SCLK_PWM0_PMU>, <&cru PCLK_PWM0_PMU>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_pin>; #pwm-cells = <3>; status = "disabled"; @@ -404,7 +404,7 @@ pwm1: pwm@20040010 { interrupts = ; clocks = <&cru SCLK_PWM0_PMU>, <&cru PCLK_PWM0_PMU>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_pin>; #pwm-cells = <3>; status = "disabled"; @@ -416,7 +416,7 @@ pwm2: pwm@20040020 { interrupts = ; clocks = <&cru SCLK_PWM0_PMU>, <&cru PCLK_PWM0_PMU>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm2_pin>; #pwm-cells = <3>; status = "disabled"; @@ -428,7 +428,7 @@ pwm3: pwm@20040030 { interrupts = ; clocks = <&cru SCLK_PWM0_PMU>, <&cru PCLK_PWM0_PMU>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm3_pin>; #pwm-cells = <3>; status = "disabled"; diff --git a/arch/arm/kernel/psci_smp.c b/arch/arm/kernel/psci_smp.c index d4392e177484..059db0b1c1de 100644 --- a/arch/arm/kernel/psci_smp.c +++ b/arch/arm/kernel/psci_smp.c @@ -107,11 +107,12 @@ static int psci_cpu_kill(unsigned int cpu) #endif -bool __init psci_smp_available(void) +bool psci_smp_available(void) { /* is cpu_on available at least? */ return (psci_ops.cpu_on != NULL); } +EXPORT_SYMBOL(psci_smp_available); const struct smp_operations psci_smp_ops __initconst = { .smp_boot_secondary = psci_boot_secondary, diff --git a/arch/arm/kernel/reboot.c b/arch/arm/kernel/reboot.c index 0ce388f15422..6f874647b63b 100644 --- a/arch/arm/kernel/reboot.c +++ b/arch/arm/kernel/reboot.c @@ -18,7 +18,6 @@ typedef void (*phys_reset_t)(unsigned long, bool); /* * Function pointers to optional machine specific functions */ -void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); void (*pm_power_off)(void); EXPORT_SYMBOL(pm_power_off); @@ -138,10 +137,8 @@ void machine_restart(char *cmd) local_irq_disable(); smp_send_stop(); - if (arm_pm_restart) - arm_pm_restart(reboot_mode, cmd); - else - do_kernel_restart(cmd); + do_kernel_pre_restart(cmd); + do_kernel_restart(cmd); /* Give a grace period for failure to restart of 1s */ mdelay(1000); diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms index 889e78f40a25..4a783983410a 100644 --- a/arch/arm64/Kconfig.platforms +++ b/arch/arm64/Kconfig.platforms @@ -224,11 +224,8 @@ config ARCH_RENESAS config ARCH_ROCKCHIP bool "Rockchip Platforms" select ARCH_HAS_RESET_CONTROLLER - select GPIOLIB select PINCTRL - select PINCTRL_ROCKCHIP select PM - select ROCKCHIP_TIMER help This enables support for the ARMv8 based Rockchip chipsets, like the RK3368. diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 485b7dbd4f9e..7b56d7f760a8 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -198,3 +198,16 @@ define archhelp echo ' (distribution) /sbin/installkernel or' echo ' install to $$(INSTALL_PATH) and run lilo' endef + +MAKE_MODULES ?= y + +%.img: +ifeq ("$(CONFIG_MODULES)$(MAKE_MODULES)$(srctree)","yy$(objtree)") + $(Q)$(MAKE) rockchip/$*.dtb Image.lz4 modules +else + $(Q)$(MAKE) rockchip/$*.dtb Image.lz4 +endif + $(Q)$(srctree)/scripts/mkimg --dtb $*.dtb + +CLEAN_DIRS += out +CLEAN_FILES += boot.img kernel.img resource.img zboot.img diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile index 26661c7b736b..a2386c005926 100644 --- a/arch/arm64/boot/dts/rockchip/Makefile +++ b/arch/arm64/boot/dts/rockchip/Makefile @@ -44,3 +44,43 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rockpro64.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire-excavator.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399pro-rock-pi-n10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-box-demo-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb-mipitest-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb1-ddr4-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb1-ddr4-v10-linux.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb1-ddr4-v10-lvds.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb2-lp4x-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb2-lp4x-v10-eink.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb2-lp4x-v10-i2s-mic-array.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb2-lp4x-v10-linux.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb2-lp4x-v10-pdm-mic-array.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb3-ddr3-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb3-ddr3-v10-linux.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-evb5-lp4x-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-eink.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-eink-w6.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-eink-w103.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-tablet.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-tablet-k108.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-tablet-rkg11.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-rk817-tablet-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb1-ddr4-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb1-ddr4-v10-linux.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb1-ddr4-v10-linux-spi-nor.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb2-lp4x-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb2-lp4x-v10-bt1120-to-hdmi.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb4-lp3-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb5-ddr4-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb6-ddr3-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb6-ddr3-v10-linux.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb6-ddr3-v10-rk628-bt1120-to-hdmi.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb6-ddr3-v10-rk628-rgb2hdmi.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb6-ddr3-v10-rk630-bt656-to-cvbs.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb7-ddr4-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-iotest-ddr3-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-iotest-ddr3-v10-linux.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nvr-demo-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nvr-demo-v10-linux.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nvr-demo-v10-linux-spi-nand.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nvr-demo-v12-linux.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-nvr-demo-v12-linux-spi-nand.dtb diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-d6-anx6345.dts b/arch/arm64/boot/dts/rockchip/px30-ad-d6-anx6345.dts new file mode 100755 index 000000000000..2f258286286a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-ad-d6-anx6345.dts @@ -0,0 +1,759 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "px30.dtsi" +#include "px30-android.dtsi" + +/ { + model = "Rockchip PX30 AD D6 board"; + compatible = "rockchip,px30-ad-d6", "rockchip,px30"; + + dvdd12_anx: dvdd12-anx { + compatible = "regulator-fixed"; + regulator-name = "dvdd12-anx"; + regulator-boot-on; + gpio = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + enable-active-high; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + enable-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + auto-freq-en = <0>; + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_soc"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc1v0_soc"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v0_pmu"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_sd"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-name = "vcc2v8_dvp"; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vdd1v5_dvp"; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc5v0_host: SWITCH_REG1 { + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&io_domains { + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc_3v0>; + vccio4-supply = <&vcc3v0_pmu>; + vccio5-supply = <&vcc_3v0>; + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pmu_io_domains { + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; + status = "okay"; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + rockchip,sleep-debug-en = <1>; + status = "okay"; +}; + +&saradc { + vref-supply = <&vcc1v8_soc>; + status = "okay"; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "init", "default"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&display_subsystem { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&i2c1 { + status = "okay"; + + anx6345@38 { + compatible = "analogix,anx6345"; + reg = <0x38>; + reset-gpios = <&gpio1 RK_PB0 GPIO_ACTIVE_LOW>; + panel-supply = <&vcc3v3_lcd>; + dvdd25-supply = <&vcc3v3_lcd>; + dvdd12-supply = <&dvdd12_anx>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + anx6345_in_rgb: endpoint { + remote-endpoint = <&rgb_out_anx6345>; + }; + }; + }; + }; +}; + +&rgb { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&lcdc_rgb_pins>; + pinctrl-1 = <&lcdc_sleep_pins>; + status = "okay"; + + ports { + port@1 { + reg = <1>; + + rgb_out_anx6345: endpoint { + remote-endpoint = <&anx6345_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vopb { + status = "okay"; +}; + +&rgb_in_vopl { + status = "disabled"; +}; + +&route_rgb { + connect = <&vopb_out_rgb>; + status = "okay"; +}; + +&pinctrl { + lcdc { + lcdc_rgb_pins: lcdc-rgb-pins { + rockchip,pins = + <3 RK_PA0 1 &pcfg_pull_none_8ma>, /* LCDC_DCLK */ + <3 RK_PA1 1 &pcfg_pull_none_8ma>, /* LCDC_HSYNC */ + <3 RK_PA2 1 &pcfg_pull_none_8ma>, /* LCDC_VSYNC */ + <3 RK_PA3 1 &pcfg_pull_none_8ma>, /* LCDC_DEN */ + <3 RK_PD3 1 &pcfg_pull_none_8ma>, /* LCDC_D23 */ + <3 RK_PD2 1 &pcfg_pull_none_8ma>, /* LCDC_D22 */ + <3 RK_PD1 1 &pcfg_pull_none_8ma>, /* LCDC_D21 */ + <3 RK_PD0 1 &pcfg_pull_none_8ma>, /* LCDC_D20 */ + <3 RK_PC7 1 &pcfg_pull_none_8ma>, /* LCDC_D19 */ + <3 RK_PC6 1 &pcfg_pull_none_8ma>, /* LCDC_D18 */ + <3 RK_PC5 1 &pcfg_pull_none_8ma>, /* LCDC_D17 */ + <3 RK_PC4 1 &pcfg_pull_none_8ma>, /* LCDC_D16 */ + <3 RK_PC3 1 &pcfg_pull_none_8ma>, /* LCDC_D15 */ + <3 RK_PC2 1 &pcfg_pull_none_8ma>, /* LCDC_D14 */ + <3 RK_PC1 1 &pcfg_pull_none_8ma>, /* LCDC_D13 */ + <3 RK_PC0 1 &pcfg_pull_none_8ma>, /* LCDC_D12 */ + <3 RK_PB7 1 &pcfg_pull_none_8ma>, /* LCDC_D11 */ + <3 RK_PB6 1 &pcfg_pull_none_8ma>, /* LCDC_D10 */ + <3 RK_PB5 1 &pcfg_pull_none_8ma>, /* LCDC_D9 */ + <3 RK_PB4 1 &pcfg_pull_none_8ma>, /* LCDC_D8 */ + <3 RK_PB3 1 &pcfg_pull_none_8ma>, /* LCDC_D7 */ + <3 RK_PB2 1 &pcfg_pull_none_8ma>, /* LCDC_D6 */ + <3 RK_PB1 1 &pcfg_pull_none_8ma>, /* LCDC_D5 */ + <3 RK_PB0 1 &pcfg_pull_none_8ma>, /* LCDC_D4 */ + <3 RK_PA7 1 &pcfg_pull_none_8ma>, /* LCDC_D3 */ + <3 RK_PA6 1 &pcfg_pull_none_8ma>, /* LCDC_D2 */ + <3 RK_PA5 1 &pcfg_pull_none_8ma>, /* LCDC_D1 */ + <3 RK_PA4 1 &pcfg_pull_none_8ma>; /* LCDC_D0 */ + }; + + lcdc_sleep_pins: lcdc-sleep-pins { + rockchip,pins = + <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_DCLK */ + <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_HSYNC */ + <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_VSYNC */ + <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_DEN */ + <3 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D23 */ + <3 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D22 */ + <3 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D21 */ + <3 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D20 */ + <3 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D19 */ + <3 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D18 */ + <3 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D17 */ + <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D16 */ + <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D15 */ + <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D14 */ + <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D13 */ + <3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D12 */ + <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D11 */ + <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D10 */ + <3 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D9 */ + <3 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D8 */ + <3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D7 */ + <3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D6 */ + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D5 */ + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D4 */ + <3 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D3 */ + <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D2 */ + <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D1 */ + <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; /* LCDC_D0 */ + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + + fstab { + compatible = "android,fstab"; + + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-dual-lvds.dts b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-dual-lvds.dts new file mode 100755 index 000000000000..09fc265a8d7a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-dual-lvds.dts @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include "px30-ad-r35-mb.dtsi" + +/ { + panel { + compatible = "lg,lm215wf3", "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc3v3_lcd>; + enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <120>; + enable-delay-ms = <120>; + disable-delay-ms = <120>; + unprepare-delay-ms = <120>; + bus-format = ; + width-mm = <476>; + height-mm = <268>; + + display-timings { + native-mode = <&timing1>; + + timing1: timing1 { + clock-frequency = <144000000>; + hactive = <1920>; + vactive = <1080>; + hback-porch = <96>; + hfront-porch = <96>; + vback-porch = <8>; + vfront-porch = <8>; + hsync-len = <64>; + vsync-len = <4>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in_lvds: endpoint { + remote-endpoint = <&lvds_out_panel>; + }; + }; + }; +}; + +&dmc { + auto-freq-en = <0>; +}; + +&i2c0 { + status = "okay"; + + rk618@50 { + compatible = "rockchip,rk618"; + reg = <0x50>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "clkin"; + assigned-clocks = <&cru SCLK_I2S1_OUT>; + assigned-clock-rates = <11289600>; + reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; + status = "okay"; + + clock: cru { + compatible = "rockchip,rk618-cru"; + clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; + clock-names = "clkin", "lcdc0_dclkp"; + assigned-clocks = <&clock SCALER_PLLIN_CLK>, + <&clock VIF_PLLIN_CLK>, + <&clock SCALER_CLK>, + <&clock VIF0_PRE_CLK>, + <&clock CODEC_CLK>, + <&clock DITHER_CLK>; + assigned-clock-parents = <&cru SCLK_I2S1_OUT>, + <&clock LCDC0_CLK>, + <&clock SCALER_PLL_CLK>, + <&clock VIF_PLL_CLK>, + <&cru SCLK_I2S1_OUT>, + <&clock VIF0_CLK>; + #clock-cells = <1>; + status = "okay"; + }; + + lvds { + compatible = "rockchip,rk618-lvds"; + clocks = <&clock LVDS_CLK>; + clock-names = "lvds"; + dual-channel; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + lvds_in_rgb: endpoint { + remote-endpoint = <&rgb_out_lvds>; + }; + }; + + port@1 { + reg = <1>; + + lvds_out_panel: endpoint { + remote-endpoint = <&panel_in_lvds>; + }; + }; + }; + }; + }; +}; + +&rgb { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + rgb_out_lvds: endpoint { + remote-endpoint = <&lvds_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vopb { + status = "disabled"; +}; + +&rgb_in_vopl { + status = "okay"; +}; + +&route_rgb { + connect = <&vopl_out_rgb>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi-lvds.dts b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi-lvds.dts new file mode 100755 index 000000000000..d4f202189250 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi-lvds.dts @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include "px30-ad-r35-mb.dtsi" + +/ { + panel { + compatible = "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc3v3_lcd>; + enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <120>; + enable-delay-ms = <120>; + disable-delay-ms = <120>; + unprepare-delay-ms = <120>; + bus-format = ; + width-mm = <231>; + height-mm = <154>; + + display-timings { + native-mode = <&timing1>; + + timing1: timing1 { + clock-frequency = <72000000>; + hactive = <1280>; + vactive = <800>; + hback-porch = <60>; + hfront-porch = <60>; + vback-porch = <16>; + vfront-porch = <16>; + hsync-len = <40>; + vsync-len = <6>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in_lvds: endpoint { + remote-endpoint = <&lvds_out_panel>; + }; + }; + }; +}; + +&dmc { + auto-freq-en = <0>; +}; + +&i2c0 { + status = "okay"; + + rk618@50 { + compatible = "rockchip,rk618"; + reg = <0x50>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "clkin"; + assigned-clocks = <&cru SCLK_I2S1_OUT>; + assigned-clock-rates = <11289600>; + reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; + status = "okay"; + + clock: cru { + compatible = "rockchip,rk618-cru"; + clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; + clock-names = "clkin", "lcdc0_dclkp"; + assigned-clocks = <&clock SCALER_PLLIN_CLK>, + <&clock VIF_PLLIN_CLK>, + <&clock SCALER_CLK>, + <&clock VIF0_PRE_CLK>, + <&clock CODEC_CLK>, + <&clock DITHER_CLK>; + assigned-clock-parents = <&cru SCLK_I2S1_OUT>, + <&clock LCDC0_CLK>, + <&clock SCALER_PLL_CLK>, + <&clock VIF_PLL_CLK>, + <&cru SCLK_I2S1_OUT>, + <&clock VIF0_CLK>; + #clock-cells = <1>; + status = "okay"; + }; + + hdmi { + compatible = "rockchip,rk618-hdmi"; + clocks = <&clock HDMI_CLK>; + clock-names = "hdmi"; + assigned-clocks = <&clock HDMI_CLK>; + assigned-clock-parents = <&clock VIF0_CLK>; + interrupt-parent = <&gpio2>; + interrupts = <12 IRQ_TYPE_LEVEL_HIGH>; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hdmi_in_vif: endpoint { + remote-endpoint = <&vif_out_hdmi>; + }; + }; + + port@1 { + reg = <1>; + + hdmi_out_scaler: endpoint { + remote-endpoint = <&scaler_in_hdmi>; + }; + }; + }; + }; + + lvds { + compatible = "rockchip,rk618-lvds"; + clocks = <&clock LVDS_CLK>; + clock-names = "lvds"; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + lvds_in_scaler: endpoint { + remote-endpoint = <&scaler_out_lvds>; + }; + }; + + port@1 { + reg = <1>; + + lvds_out_panel: endpoint { + remote-endpoint = <&panel_in_lvds>; + }; + }; + }; + }; + + scaler { + compatible = "rockchip,rk618-scaler"; + clocks = <&clock SCALER_CLK>, <&clock VIF0_CLK>, + <&clock DITHER_CLK>; + clock-names = "scaler", "vif", "dither"; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + scaler_in_hdmi: endpoint { + remote-endpoint = <&hdmi_out_scaler>; + }; + }; + + port@1 { + reg = <1>; + + scaler_out_lvds: endpoint { + remote-endpoint = <&lvds_in_scaler>; + }; + }; + }; + }; + + vif { + compatible = "rockchip,rk618-vif"; + clocks = <&clock VIF0_CLK>, <&clock VIF0_PRE_CLK>; + clock-names = "vif", "vif_pre"; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + vif_in_rgb: endpoint { + remote-endpoint = <&rgb_out_vif>; + }; + }; + + port@1 { + reg = <1>; + + vif_out_hdmi: endpoint { + remote-endpoint = <&hdmi_in_vif>; + }; + }; + }; + }; + }; +}; + +&vopl { + assigned-clocks = <&cru PLL_NPLL>; + assigned-clock-rates = <1188000000>; +}; + +&rgb { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + rgb_out_vif: endpoint { + remote-endpoint = <&vif_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vopb { + status = "disabled"; +}; + +&rgb_in_vopl { + status = "okay"; +}; + +&route_rgb { + connect = <&vopl_out_rgb>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi.dts b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi.dts new file mode 100755 index 000000000000..0ea056682e57 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-hdmi.dts @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include "px30-ad-r35-mb.dtsi" + +&dmc { + auto-freq-en = <0>; +}; + +&i2c0 { + status = "okay"; + + rk618@50 { + compatible = "rockchip,rk618"; + reg = <0x50>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "clkin"; + assigned-clocks = <&cru SCLK_I2S1_OUT>; + assigned-clock-rates = <11289600>; + reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; + status = "okay"; + + clock: cru { + compatible = "rockchip,rk618-cru"; + clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; + clock-names = "clkin", "lcdc0_dclkp"; + assigned-clocks = <&clock SCALER_PLLIN_CLK>, + <&clock VIF_PLLIN_CLK>, + <&clock SCALER_CLK>, + <&clock VIF0_PRE_CLK>, + <&clock CODEC_CLK>, + <&clock DITHER_CLK>; + assigned-clock-parents = <&cru SCLK_I2S1_OUT>, + <&clock LCDC0_CLK>, + <&clock SCALER_PLL_CLK>, + <&clock VIF_PLL_CLK>, + <&cru SCLK_I2S1_OUT>, + <&clock VIF0_CLK>; + #clock-cells = <1>; + status = "okay"; + }; + + hdmi { + compatible = "rockchip,rk618-hdmi"; + clocks = <&clock HDMI_CLK>; + clock-names = "hdmi"; + assigned-clocks = <&clock HDMI_CLK>; + assigned-clock-parents = <&clock VIF0_CLK>; + interrupt-parent = <&gpio2>; + interrupts = <12 IRQ_TYPE_LEVEL_HIGH>; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hdmi_in_rgb: endpoint { + remote-endpoint = <&rgb_out_hdmi>; + }; + }; + }; + }; + }; +}; + +&vopl { + assigned-clocks = <&cru PLL_NPLL>; + assigned-clock-rates = <1188000000>; +}; + +&rgb { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + rgb_out_hdmi: endpoint { + remote-endpoint = <&hdmi_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vopb { + status = "disabled"; +}; + +&rgb_in_vopl { + status = "okay"; +}; + +&route_rgb { + connect = <&vopl_out_rgb>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-lvds.dts b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-lvds.dts new file mode 100755 index 000000000000..8b54a9a18fcc --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb-rk618-lvds.dts @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include "px30-ad-r35-mb.dtsi" + +/ { + panel { + compatible = "chunghwa,claa101wh31-cw", "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc3v3_lcd>; + enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <120>; + enable-delay-ms = <120>; + disable-delay-ms = <120>; + unprepare-delay-ms = <120>; + bus-format = ; + width-mm = <231>; + height-mm = <154>; + + display-timings { + native-mode = <&timing1>; + + timing1: timing1 { + clock-frequency = <72000000>; + hactive = <1280>; + vactive = <800>; + hback-porch = <60>; + hfront-porch = <60>; + vback-porch = <16>; + vfront-porch = <16>; + hsync-len = <40>; + vsync-len = <6>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in_lvds: endpoint { + remote-endpoint = <&lvds_out_panel>; + }; + }; + }; +}; + +&dmc { + auto-freq-en = <0>; +}; + +&i2c0 { + status = "okay"; + + rk618@50 { + compatible = "rockchip,rk618"; + reg = <0x50>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "clkin"; + assigned-clocks = <&cru SCLK_I2S1_OUT>; + assigned-clock-rates = <12000000>; + reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; + status = "okay"; + + clock: cru { + compatible = "rockchip,rk618-cru"; + clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; + clock-names = "clkin", "lcdc0_dclkp"; + assigned-clocks = <&clock SCALER_PLLIN_CLK>, + <&clock VIF_PLLIN_CLK>, + <&clock SCALER_CLK>, + <&clock VIF0_PRE_CLK>, + <&clock CODEC_CLK>, + <&clock DITHER_CLK>; + assigned-clock-parents = <&cru SCLK_I2S1_OUT>, + <&clock LCDC0_CLK>, + <&clock SCALER_PLL_CLK>, + <&clock VIF_PLL_CLK>, + <&cru SCLK_I2S1_OUT>, + <&clock VIF0_CLK>; + #clock-cells = <1>; + status = "okay"; + }; + + lvds { + compatible = "rockchip,rk618-lvds"; + clocks = <&clock LVDS_CLK>; + clock-names = "lvds"; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + lvds_in_rgb: endpoint { + remote-endpoint = <&rgb_out_lvds>; + }; + }; + + port@1 { + reg = <1>; + + lvds_out_panel: endpoint { + remote-endpoint = <&panel_in_lvds>; + }; + }; + }; + }; + }; +}; + +&rgb { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + rgb_out_lvds: endpoint { + remote-endpoint = <&lvds_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vopb { + status = "disabled"; +}; + +&rgb_in_vopl { + status = "okay"; +}; + +&route_rgb { + connect = <&vopl_out_rgb>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb.dtsi b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb.dtsi new file mode 100755 index 000000000000..bbcd18959dc4 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-ad-r35-mb.dtsi @@ -0,0 +1,823 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include "px30.dtsi" +#include "px30-android.dtsi" + +/ { + model = "Rockchip PX30 AD R35 MB board"; + compatible = "rockchip,px30-ad-r35-mb", "rockchip,px30"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1270000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <602000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <952000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <290000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + enable-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&display_subsystem { + status = "okay"; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x1>; + regulator-name = "vdd_logic"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x1>; + regulator-name = "vdd_arm"; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x1>; + + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x1>; + regulator-name = "vcc_3v0"; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_soc"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc1v0_soc"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v0_pmu"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_sd"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-name = "vcc2v8_dvp"; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vdd1v5_dvp"; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc3v3_lcd: SWITCH_REG1 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + + vcc5v0_host: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_host"; + }; + }; + }; +}; + +&io_domains { + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc_3v0>; + vccio4-supply = <&vcc3v0_pmu>; + vccio5-supply = <&vcc_3v0>; + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pmu_io_domains { + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; + status = "okay"; +}; + +&pwm1 { + status = "okay"; +}; + +&saradc { + vref-supply = <&vcc1v8_soc>; + status = "okay"; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "init", "default"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "pvo,p101nwwbp-01g", "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + prepare-delay-ms = <20>; + reset-delay-ms = <20>; + init-delay-ms = <20>; + enable-delay-ms = <20>; + disable-delay-ms = <20>; + unprepare-delay-ms = <20>; + + width-mm = <135>; + height-mm = <216>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 15 00 02 E0 00 + 15 00 02 E1 93 + 15 00 02 E2 65 + 15 00 02 E3 F8 + 15 00 02 80 03 + 15 00 02 E0 04 + 15 00 02 2B 2B + 15 00 02 2D 03 + 15 00 02 2E 44 + 15 00 02 E0 01 + 15 00 02 00 00 + 15 00 02 01 6D + 15 00 02 0C 74 + 15 00 02 17 00 + 15 00 02 18 A7 + 15 00 02 19 01 + 15 00 02 1A 00 + 15 00 02 1B A7 + 15 00 02 1C 01 + 15 00 02 1F 6A + 15 00 02 20 23 + 15 00 02 21 23 + 15 00 02 22 0E + 15 00 02 35 28 + 15 00 02 37 59 + 15 00 02 38 05 + 15 00 02 39 04 + 15 00 02 3A 08 + 15 00 02 3B 08 + 15 00 02 3C 7C + 15 00 02 3D FF + 15 00 02 3E FF + 15 00 02 3F FF + 15 00 02 40 06 + 15 00 02 41 A0 + 15 00 02 43 08 + 15 00 02 44 0B + 15 00 02 45 88 + 15 00 02 4B 04 + 15 00 02 55 01 + 15 00 02 56 01 + 15 00 02 57 8D + 15 00 02 58 0A + 15 00 02 59 0A + 15 00 02 5A 28 + 15 00 02 5B 1E + 15 00 02 5C 16 + 15 00 02 5D 76 + 15 00 02 5E 58 + 15 00 02 5F 46 + 15 00 02 60 39 + 15 00 02 61 37 + 15 00 02 62 2A + 15 00 02 63 2F + 15 00 02 64 18 + 15 00 02 65 39 + 15 00 02 66 38 + 15 00 02 67 3A + 15 00 02 68 5A + 15 00 02 69 46 + 15 00 02 6A 4C + 15 00 02 6B 3F + 15 00 02 6C 3D + 15 00 02 6D 2F + 15 00 02 6E 1E + 15 00 02 6F 00 + 15 00 02 70 76 + 15 00 02 71 58 + 15 00 02 72 46 + 15 00 02 73 39 + 15 00 02 74 33 + 15 00 02 75 22 + 15 00 02 76 27 + 15 00 02 77 14 + 15 00 02 78 29 + 15 00 02 79 2A + 15 00 02 7A 28 + 15 00 02 7B 46 + 15 00 02 7C 38 + 15 00 02 7D 3E + 15 00 02 7E 31 + 15 00 02 7F 29 + 15 00 02 80 1B + 15 00 02 81 0A + 15 00 02 82 00 + 15 00 02 E0 02 + 15 00 02 00 44 + 15 00 02 01 44 + 15 00 02 02 45 + 15 00 02 03 45 + 15 00 02 04 46 + 15 00 02 05 46 + 15 00 02 06 47 + 15 00 02 07 47 + 15 00 02 08 1D + 15 00 02 09 1D + 15 00 02 0A 1D + 15 00 02 0B 1D + 15 00 02 0C 1D + 15 00 02 0D 1D + 15 00 02 0E 1D + 15 00 02 0F 57 + 15 00 02 10 57 + 15 00 02 11 58 + 15 00 02 12 58 + 15 00 02 13 40 + 15 00 02 14 55 + 15 00 02 15 55 + 15 00 02 16 44 + 15 00 02 17 44 + 15 00 02 18 45 + 15 00 02 19 45 + 15 00 02 1A 46 + 15 00 02 1B 46 + 15 00 02 1C 47 + 15 00 02 1D 47 + 15 00 02 1E 1D + 15 00 02 1F 1D + 15 00 02 20 1D + 15 00 02 21 1D + 15 00 02 22 1D + 15 00 02 23 1D + 15 00 02 24 1D + 15 00 02 25 57 + 15 00 02 26 57 + 15 00 02 27 58 + 15 00 02 28 58 + 15 00 02 29 40 + 15 00 02 2A 55 + 15 00 02 2B 55 + 15 00 02 58 40 + 15 00 02 59 00 + 15 00 02 5A 00 + 15 00 02 5B 00 + 15 00 02 5C 0A + 15 00 02 5D 10 + 15 00 02 5E 01 + 15 00 02 5F 02 + 15 00 02 60 00 + 15 00 02 61 01 + 15 00 02 62 02 + 15 00 02 63 0B + 15 00 02 64 6A + 15 00 02 65 00 + 15 00 02 66 00 + 15 00 02 67 31 + 15 00 02 68 0B + 15 00 02 69 1E + 15 00 02 6A 6A + 15 00 02 6B 04 + 15 00 02 6C 00 + 15 00 02 6D 04 + 15 00 02 6E 00 + 15 00 02 6F 88 + 15 00 02 70 00 + 15 00 02 71 00 + 15 00 02 72 06 + 15 00 02 73 7B + 15 00 02 74 00 + 15 00 02 75 F8 + 15 00 02 76 00 + 15 00 02 77 0D + 15 00 02 78 14 + 15 00 02 79 00 + 15 00 02 7A 00 + 15 00 02 7B 00 + 15 00 02 7C 00 + 15 00 02 7D 03 + 15 00 02 7E 7B + 15 00 02 E0 00 + 15 00 02 E6 02 + 15 00 02 E7 06 + 15 80 01 11 + 15 16 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <68500000>; + hactive = <800>; + hfront-porch = <16>; + hsync-len = <16>; + hback-porch = <48>; + vactive = <1280>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <4>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + + fstab { + compatible = "android,fstab"; + + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-android.dtsi b/arch/arm64/boot/dts/rockchip/px30-android.dtsi new file mode 100755 index 000000000000..aecb6344f01f --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-android.dtsi @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/ { + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff160000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init kpti=0"; + }; + + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <0>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "okay"; + }; + + firmware { + firmware_android: android {}; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + drm_logo: drm-logo@00000000 { + compatible = "rockchip,drm-logo"; + reg = <0x0 0x0 0x0 0x0>; + }; + + ramoops: ramoops@110000 { + compatible = "ramoops"; + reg = <0x0 0x110000 0x0 0xf0000>; + record-size = <0x20000>; + console-size = <0x80000>; + ftrace-size = <0x00000>; + pmsg-size = <0x50000>; + }; + }; +}; + +&cpu0_opp_table { + rockchip,avs = <1>; +}; + +&display_subsystem { + status = "disabled"; + ports = <&vopb_out>, <&vopl_out>; + logo-memory-region = <&drm_logo>; + + route { + route_lvds: route-lvds { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopb_out_lvds>; + }; + + route_dsi: route-dsi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopb_out_dsi>; + }; + + route_rgb: route-rgb { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopb_out_rgb>; + }; + }; +}; + +&dsi { + panel@0 { + reg = <0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&rng { + status = "okay"; +}; + +&video_phy { + status = "okay"; +}; + +&vopb { + support-multi-area; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-ddr4p416dd6-timing.dtsi b/arch/arm64/boot/dts/rockchip/px30-ddr4p416dd6-timing.dtsi new file mode 100755 index 000000000000..fde5895db94a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-ddr4p416dd6-timing.dtsi @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd. + +&ddr_timing { + /* CA de-skew, one step is 47.8ps, range 0-15 */ + ddr3a1_ddr4a9_de-skew = <0>; + ddr3a0_ddr4a10_de-skew = <5>; + ddr3a3_ddr4a6_de-skew = <2>; + ddr3a2_ddr4a4_de-skew = <4>; + ddr3a5_ddr4a8_de-skew = <2>; + ddr3a4_ddr4a5_de-skew = <1>; + ddr3a7_ddr4a11_de-skew = <2>; + ddr3a6_ddr4a7_de-skew = <0>; + ddr3a9_ddr4a0_de-skew = <4>; + ddr3a8_ddr4a13_de-skew = <0>; + ddr3a11_ddr4a3_de-skew = <4>; + ddr3a10_ddr4cs0_de-skew = <5>; + ddr3a13_ddr4a2_de-skew = <2>; + ddr3a12_ddr4ba1_de-skew = <2>; + ddr3a15_ddr4odt0_de-skew = <7>; + ddr3a14_ddr4a1_de-skew = <4>; + ddr3ba1_ddr4a15_de-skew = <3>; + ddr3ba0_ddr4bg0_de-skew = <4>; + ddr3ras_ddr4cke_de-skew = <6>; + ddr3ba2_ddr4ba0_de-skew = <3>; + ddr3we_ddr4bg1_de-skew = <3>; + ddr3cas_ddr4a12_de-skew = <6>; + ddr3ckn_ddr4ckn_de-skew = <8>; + ddr3ckp_ddr4ckp_de-skew = <8>; + ddr3cke_ddr4a16_de-skew = <4>; + ddr3odt0_ddr4a14_de-skew = <4>; + ddr3cs0_ddr4act_de-skew = <7>; + ddr3reset_ddr4reset_de-skew = <3>; + ddr3cs1_ddr4cs1_de-skew = <5>; + ddr3odt1_ddr4odt1_de-skew = <6>; + + /* DATA de-skew + * RX one step is 25.1ps, range 0-15 + * TX one step is 47.8ps, range 0-15 + */ + cs0_dm0_rx_de-skew = <7>; + cs0_dm0_tx_de-skew = <9>; + cs0_dq0_rx_de-skew = <10>; + cs0_dq0_tx_de-skew = <10>; + cs0_dq1_rx_de-skew = <11>; + cs0_dq1_tx_de-skew = <10>; + cs0_dq2_rx_de-skew = <8>; + cs0_dq2_tx_de-skew = <10>; + cs0_dq3_rx_de-skew = <9>; + cs0_dq3_tx_de-skew = <9>; + cs0_dq4_rx_de-skew = <10>; + cs0_dq4_tx_de-skew = <10>; + cs0_dq5_rx_de-skew = <10>; + cs0_dq5_tx_de-skew = <10>; + cs0_dq6_rx_de-skew = <10>; + cs0_dq6_tx_de-skew = <10>; + cs0_dq7_rx_de-skew = <10>; + cs0_dq7_tx_de-skew = <10>; + cs0_dqs0_rx_de-skew = <5>; + cs0_dqs0p_tx_de-skew = <11>; + cs0_dqs0n_tx_de-skew = <11>; + + cs0_dm1_rx_de-skew = <7>; + cs0_dm1_tx_de-skew = <9>; + cs0_dq8_rx_de-skew = <8>; + cs0_dq8_tx_de-skew = <8>; + cs0_dq9_rx_de-skew = <10>; + cs0_dq9_tx_de-skew = <9>; + cs0_dq10_rx_de-skew = <8>; + cs0_dq10_tx_de-skew = <8>; + cs0_dq11_rx_de-skew = <9>; + cs0_dq11_tx_de-skew = <10>; + cs0_dq12_rx_de-skew = <9>; + cs0_dq12_tx_de-skew = <8>; + cs0_dq13_rx_de-skew = <8>; + cs0_dq13_tx_de-skew = <9>; + cs0_dq14_rx_de-skew = <9>; + cs0_dq14_tx_de-skew = <8>; + cs0_dq15_rx_de-skew = <9>; + cs0_dq15_tx_de-skew = <9>; + cs0_dqs1_rx_de-skew = <5>; + cs0_dqs1p_tx_de-skew = <11>; + cs0_dqs1n_tx_de-skew = <11>; + + cs0_dm2_rx_de-skew = <7>; + cs0_dm2_tx_de-skew = <10>; + cs0_dq16_rx_de-skew = <10>; + cs0_dq16_tx_de-skew = <9>; + cs0_dq17_rx_de-skew = <10>; + cs0_dq17_tx_de-skew = <9>; + cs0_dq18_rx_de-skew = <9>; + cs0_dq18_tx_de-skew = <8>; + cs0_dq19_rx_de-skew = <10>; + cs0_dq19_tx_de-skew = <9>; + cs0_dq20_rx_de-skew = <10>; + cs0_dq20_tx_de-skew = <10>; + cs0_dq21_rx_de-skew = <10>; + cs0_dq21_tx_de-skew = <9>; + cs0_dq22_rx_de-skew = <9>; + cs0_dq22_tx_de-skew = <8>; + cs0_dq23_rx_de-skew = <10>; + cs0_dq23_tx_de-skew = <9>; + cs0_dqs2_rx_de-skew = <5>; + cs0_dqs2p_tx_de-skew = <10>; + cs0_dqs2n_tx_de-skew = <10>; + + cs0_dm3_rx_de-skew = <7>; + cs0_dm3_tx_de-skew = <7>; + cs0_dq24_rx_de-skew = <9>; + cs0_dq24_tx_de-skew = <8>; + cs0_dq25_rx_de-skew = <9>; + cs0_dq25_tx_de-skew = <8>; + cs0_dq26_rx_de-skew = <9>; + cs0_dq26_tx_de-skew = <8>; + cs0_dq27_rx_de-skew = <9>; + cs0_dq27_tx_de-skew = <9>; + cs0_dq28_rx_de-skew = <9>; + cs0_dq28_tx_de-skew = <9>; + cs0_dq29_rx_de-skew = <9>; + cs0_dq29_tx_de-skew = <9>; + cs0_dq30_rx_de-skew = <10>; + cs0_dq30_tx_de-skew = <9>; + cs0_dq31_rx_de-skew = <10>; + cs0_dq31_tx_de-skew = <9>; + cs0_dqs3_rx_de-skew = <6>; + cs0_dqs3p_tx_de-skew = <10>; + cs0_dqs3n_tx_de-skew = <10>; + + cs1_dm0_rx_de-skew = <7>; + cs1_dm0_tx_de-skew = <9>; + cs1_dq0_rx_de-skew = <10>; + cs1_dq0_tx_de-skew = <10>; + cs1_dq1_rx_de-skew = <11>; + cs1_dq1_tx_de-skew = <10>; + cs1_dq2_rx_de-skew = <8>; + cs1_dq2_tx_de-skew = <10>; + cs1_dq3_rx_de-skew = <9>; + cs1_dq3_tx_de-skew = <9>; + cs1_dq4_rx_de-skew = <10>; + cs1_dq4_tx_de-skew = <10>; + cs1_dq5_rx_de-skew = <10>; + cs1_dq5_tx_de-skew = <10>; + cs1_dq6_rx_de-skew = <10>; + cs1_dq6_tx_de-skew = <10>; + cs1_dq7_rx_de-skew = <10>; + cs1_dq7_tx_de-skew = <10>; + cs1_dqs0_rx_de-skew = <5>; + cs1_dqs0p_tx_de-skew = <11>; + cs1_dqs0n_tx_de-skew = <11>; + + cs1_dm1_rx_de-skew = <7>; + cs1_dm1_tx_de-skew = <9>; + cs1_dq8_rx_de-skew = <8>; + cs1_dq8_tx_de-skew = <8>; + cs1_dq9_rx_de-skew = <10>; + cs1_dq9_tx_de-skew = <9>; + cs1_dq10_rx_de-skew = <8>; + cs1_dq10_tx_de-skew = <8>; + cs1_dq11_rx_de-skew = <9>; + cs1_dq11_tx_de-skew = <10>; + cs1_dq12_rx_de-skew = <9>; + cs1_dq12_tx_de-skew = <8>; + cs1_dq13_rx_de-skew = <8>; + cs1_dq13_tx_de-skew = <9>; + cs1_dq14_rx_de-skew = <9>; + cs1_dq14_tx_de-skew = <8>; + cs1_dq15_rx_de-skew = <9>; + cs1_dq15_tx_de-skew = <9>; + cs1_dqs1_rx_de-skew = <5>; + cs1_dqs1p_tx_de-skew = <11>; + cs1_dqs1n_tx_de-skew = <11>; + + cs1_dm2_rx_de-skew = <7>; + cs1_dm2_tx_de-skew = <10>; + cs1_dq16_rx_de-skew = <10>; + cs1_dq16_tx_de-skew = <9>; + cs1_dq17_rx_de-skew = <10>; + cs1_dq17_tx_de-skew = <9>; + cs1_dq18_rx_de-skew = <9>; + cs1_dq18_tx_de-skew = <8>; + cs1_dq19_rx_de-skew = <10>; + cs1_dq19_tx_de-skew = <9>; + cs1_dq20_rx_de-skew = <10>; + cs1_dq20_tx_de-skew = <10>; + cs1_dq21_rx_de-skew = <10>; + cs1_dq21_tx_de-skew = <9>; + cs1_dq22_rx_de-skew = <9>; + cs1_dq22_tx_de-skew = <8>; + cs1_dq23_rx_de-skew = <10>; + cs1_dq23_tx_de-skew = <9>; + cs1_dqs2_rx_de-skew = <5>; + cs1_dqs2p_tx_de-skew = <10>; + cs1_dqs2n_tx_de-skew = <10>; + + cs1_dm3_rx_de-skew = <7>; + cs1_dm3_tx_de-skew = <7>; + cs1_dq24_rx_de-skew = <9>; + cs1_dq24_tx_de-skew = <8>; + cs1_dq25_rx_de-skew = <9>; + cs1_dq25_tx_de-skew = <8>; + cs1_dq26_rx_de-skew = <9>; + cs1_dq26_tx_de-skew = <8>; + cs1_dq27_rx_de-skew = <9>; + cs1_dq27_tx_de-skew = <9>; + cs1_dq28_rx_de-skew = <9>; + cs1_dq28_tx_de-skew = <9>; + cs1_dq29_rx_de-skew = <9>; + cs1_dq29_tx_de-skew = <9>; + cs1_dq30_rx_de-skew = <10>; + cs1_dq30_tx_de-skew = <9>; + cs1_dq31_rx_de-skew = <10>; + cs1_dq31_tx_de-skew = <9>; + cs1_dqs3_rx_de-skew = <6>; + cs1_dqs3p_tx_de-skew = <10>; + cs1_dqs3n_tx_de-skew = <10>; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/px30-dram-default-timing.dtsi new file mode 100755 index 000000000000..c75c5ef4ef2a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-dram-default-timing.dtsi @@ -0,0 +1,294 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +#include +#include + +/ { + ddr_timing: ddr_timing { + compatible = "rockchip,ddr-timing"; + ddr2_speed_bin = ; + ddr3_speed_bin = ; + ddr4_speed_bin = ; + pd_idle = <13>; + sr_idle = <93>; + sr_mc_gate_idle = <0>; + srpd_lite_idle = <0>; + standby_idle = <0>; + + auto_pd_dis_freq = <1066>; + auto_sr_dis_freq = <800>; + ddr2_dll_dis_freq = <300>; + ddr3_dll_dis_freq = <300>; + ddr4_dll_dis_freq = <625>; + phy_dll_dis_freq = <400>; + + ddr2_odt_dis_freq = <100>; + phy_ddr2_odt_dis_freq = <100>; + ddr2_drv = ; + ddr2_odt = ; + phy_ddr2_ca_drv = ; + phy_ddr2_ck_drv = ; + phy_ddr2_dq_drv = ; + phy_ddr2_odt = ; + + ddr3_odt_dis_freq = <400>; + phy_ddr3_odt_dis_freq = <400>; + ddr3_drv = ; + ddr3_odt = ; + phy_ddr3_ca_drv = ; + phy_ddr3_ck_drv = ; + phy_ddr3_dq_drv = ; + phy_ddr3_odt = ; + + phy_lpddr2_odt_dis_freq = <666>; + lpddr2_drv = ; + phy_lpddr2_ca_drv = ; + phy_lpddr2_ck_drv = ; + phy_lpddr2_dq_drv = ; + phy_lpddr2_odt = ; + + lpddr3_odt_dis_freq = <400>; + phy_lpddr3_odt_dis_freq = <400>; + lpddr3_drv = ; + lpddr3_odt = ; + phy_lpddr3_ca_drv = ; + phy_lpddr3_ck_drv = ; + phy_lpddr3_dq_drv = ; + phy_lpddr3_odt = ; + + lpddr4_odt_dis_freq = <800>; + phy_lpddr4_odt_dis_freq = <800>; + lpddr4_drv = ; + lpddr4_dq_odt = ; + lpddr4_ca_odt = ; + phy_lpddr4_ca_drv = ; + phy_lpddr4_ck_cs_drv = ; + phy_lpddr4_dq_drv = ; + phy_lpddr4_odt = ; + + ddr4_odt_dis_freq = <666>; + phy_ddr4_odt_dis_freq = <666>; + ddr4_drv = ; + ddr4_odt = ; + phy_ddr4_ca_drv = ; + phy_ddr4_ck_drv = ; + phy_ddr4_dq_drv = ; + phy_ddr4_odt = ; + + /* CA de-skew, one step is 47.8ps, range 0-15 */ + ddr3a1_ddr4a9_de-skew = <6>; + ddr3a0_ddr4a10_de-skew = <7>; + ddr3a3_ddr4a6_de-skew = <7>; + ddr3a2_ddr4a4_de-skew = <7>; + ddr3a5_ddr4a8_de-skew = <7>; + ddr3a4_ddr4a5_de-skew = <7>; + ddr3a7_ddr4a11_de-skew = <7>; + ddr3a6_ddr4a7_de-skew = <6>; + ddr3a9_ddr4a0_de-skew = <7>; + ddr3a8_ddr4a13_de-skew = <7>; + ddr3a11_ddr4a3_de-skew = <7>; + ddr3a10_ddr4cs0_de-skew = <7>; + ddr3a13_ddr4a2_de-skew = <7>; + ddr3a12_ddr4ba1_de-skew = <7>; + ddr3a15_ddr4odt0_de-skew = <7>; + ddr3a14_ddr4a1_de-skew = <7>; + ddr3ba1_ddr4a15_de-skew = <7>; + ddr3ba0_ddr4bg0_de-skew = <7>; + ddr3ras_ddr4cke_de-skew = <7>; + ddr3ba2_ddr4ba0_de-skew = <7>; + ddr3we_ddr4bg1_de-skew = <7>; + ddr3cas_ddr4a12_de-skew = <7>; + ddr3ckn_ddr4ckn_de-skew = <7>; + ddr3ckp_ddr4ckp_de-skew = <7>; + ddr3cke_ddr4a16_de-skew = <7>; + ddr3odt0_ddr4a14_de-skew = <7>; + ddr3cs0_ddr4act_de-skew = <6>; + ddr3reset_ddr4reset_de-skew = <7>; + ddr3cs1_ddr4cs1_de-skew = <6>; + ddr3odt1_ddr4odt1_de-skew = <7>; + + /* DATA de-skew + * RX one step is 25.1ps, range 0-15 + * TX one step is 47.8ps, range 0-15 + */ + cs0_dm0_rx_de-skew = <7>; + cs0_dm0_tx_de-skew = <7>; + cs0_dq0_rx_de-skew = <8>; + cs0_dq0_tx_de-skew = <8>; + cs0_dq1_rx_de-skew = <9>; + cs0_dq1_tx_de-skew = <8>; + cs0_dq2_rx_de-skew = <8>; + cs0_dq2_tx_de-skew = <8>; + cs0_dq3_rx_de-skew = <8>; + cs0_dq3_tx_de-skew = <8>; + cs0_dq4_rx_de-skew = <9>; + cs0_dq4_tx_de-skew = <8>; + cs0_dq5_rx_de-skew = <9>; + cs0_dq5_tx_de-skew = <8>; + cs0_dq6_rx_de-skew = <9>; + cs0_dq6_tx_de-skew = <8>; + cs0_dq7_rx_de-skew = <8>; + cs0_dq7_tx_de-skew = <8>; + cs0_dqs0_rx_de-skew = <6>; + cs0_dqs0p_tx_de-skew = <9>; + cs0_dqs0n_tx_de-skew = <9>; + + cs0_dm1_rx_de-skew = <7>; + cs0_dm1_tx_de-skew = <6>; + cs0_dq8_rx_de-skew = <8>; + cs0_dq8_tx_de-skew = <7>; + cs0_dq9_rx_de-skew = <9>; + cs0_dq9_tx_de-skew = <7>; + cs0_dq10_rx_de-skew = <8>; + cs0_dq10_tx_de-skew = <8>; + cs0_dq11_rx_de-skew = <8>; + cs0_dq11_tx_de-skew = <7>; + cs0_dq12_rx_de-skew = <8>; + cs0_dq12_tx_de-skew = <8>; + cs0_dq13_rx_de-skew = <9>; + cs0_dq13_tx_de-skew = <7>; + cs0_dq14_rx_de-skew = <9>; + cs0_dq14_tx_de-skew = <8>; + cs0_dq15_rx_de-skew = <9>; + cs0_dq15_tx_de-skew = <7>; + cs0_dqs1_rx_de-skew = <7>; + cs0_dqs1p_tx_de-skew = <9>; + cs0_dqs1n_tx_de-skew = <9>; + + cs0_dm2_rx_de-skew = <7>; + cs0_dm2_tx_de-skew = <7>; + cs0_dq16_rx_de-skew = <9>; + cs0_dq16_tx_de-skew = <9>; + cs0_dq17_rx_de-skew = <7>; + cs0_dq17_tx_de-skew = <9>; + cs0_dq18_rx_de-skew = <7>; + cs0_dq18_tx_de-skew = <8>; + cs0_dq19_rx_de-skew = <7>; + cs0_dq19_tx_de-skew = <9>; + cs0_dq20_rx_de-skew = <9>; + cs0_dq20_tx_de-skew = <9>; + cs0_dq21_rx_de-skew = <9>; + cs0_dq21_tx_de-skew = <9>; + cs0_dq22_rx_de-skew = <8>; + cs0_dq22_tx_de-skew = <9>; + cs0_dq23_rx_de-skew = <8>; + cs0_dq23_tx_de-skew = <9>; + cs0_dqs2_rx_de-skew = <6>; + cs0_dqs2p_tx_de-skew = <9>; + cs0_dqs2n_tx_de-skew = <9>; + + cs0_dm3_rx_de-skew = <7>; + cs0_dm3_tx_de-skew = <7>; + cs0_dq24_rx_de-skew = <8>; + cs0_dq24_tx_de-skew = <8>; + cs0_dq25_rx_de-skew = <9>; + cs0_dq25_tx_de-skew = <9>; + cs0_dq26_rx_de-skew = <9>; + cs0_dq26_tx_de-skew = <8>; + cs0_dq27_rx_de-skew = <9>; + cs0_dq27_tx_de-skew = <8>; + cs0_dq28_rx_de-skew = <9>; + cs0_dq28_tx_de-skew = <9>; + cs0_dq29_rx_de-skew = <9>; + cs0_dq29_tx_de-skew = <9>; + cs0_dq30_rx_de-skew = <8>; + cs0_dq30_tx_de-skew = <8>; + cs0_dq31_rx_de-skew = <8>; + cs0_dq31_tx_de-skew = <8>; + cs0_dqs3_rx_de-skew = <7>; + cs0_dqs3p_tx_de-skew = <9>; + cs0_dqs3n_tx_de-skew = <9>; + + cs1_dm0_rx_de-skew = <7>; + cs1_dm0_tx_de-skew = <7>; + cs1_dq0_rx_de-skew = <8>; + cs1_dq0_tx_de-skew = <8>; + cs1_dq1_rx_de-skew = <9>; + cs1_dq1_tx_de-skew = <8>; + cs1_dq2_rx_de-skew = <8>; + cs1_dq2_tx_de-skew = <8>; + cs1_dq3_rx_de-skew = <8>; + cs1_dq3_tx_de-skew = <8>; + cs1_dq4_rx_de-skew = <8>; + cs1_dq4_tx_de-skew = <8>; + cs1_dq5_rx_de-skew = <9>; + cs1_dq5_tx_de-skew = <8>; + cs1_dq6_rx_de-skew = <9>; + cs1_dq6_tx_de-skew = <8>; + cs1_dq7_rx_de-skew = <8>; + cs1_dq7_tx_de-skew = <8>; + cs1_dqs0_rx_de-skew = <6>; + cs1_dqs0p_tx_de-skew = <9>; + cs1_dqs0n_tx_de-skew = <9>; + + cs1_dm1_rx_de-skew = <7>; + cs1_dm1_tx_de-skew = <7>; + cs1_dq8_rx_de-skew = <8>; + cs1_dq8_tx_de-skew = <8>; + cs1_dq9_rx_de-skew = <8>; + cs1_dq9_tx_de-skew = <7>; + cs1_dq10_rx_de-skew = <7>; + cs1_dq10_tx_de-skew = <8>; + cs1_dq11_rx_de-skew = <8>; + cs1_dq11_tx_de-skew = <8>; + cs1_dq12_rx_de-skew = <8>; + cs1_dq12_tx_de-skew = <7>; + cs1_dq13_rx_de-skew = <8>; + cs1_dq13_tx_de-skew = <8>; + cs1_dq14_rx_de-skew = <8>; + cs1_dq14_tx_de-skew = <8>; + cs1_dq15_rx_de-skew = <8>; + cs1_dq15_tx_de-skew = <7>; + cs1_dqs1_rx_de-skew = <7>; + cs1_dqs1p_tx_de-skew = <9>; + cs1_dqs1n_tx_de-skew = <9>; + + cs1_dm2_rx_de-skew = <7>; + cs1_dm2_tx_de-skew = <8>; + cs1_dq16_rx_de-skew = <8>; + cs1_dq16_tx_de-skew = <9>; + cs1_dq17_rx_de-skew = <8>; + cs1_dq17_tx_de-skew = <9>; + cs1_dq18_rx_de-skew = <7>; + cs1_dq18_tx_de-skew = <8>; + cs1_dq19_rx_de-skew = <8>; + cs1_dq19_tx_de-skew = <9>; + cs1_dq20_rx_de-skew = <9>; + cs1_dq20_tx_de-skew = <9>; + cs1_dq21_rx_de-skew = <9>; + cs1_dq21_tx_de-skew = <9>; + cs1_dq22_rx_de-skew = <8>; + cs1_dq22_tx_de-skew = <9>; + cs1_dq23_rx_de-skew = <8>; + cs1_dq23_tx_de-skew = <9>; + cs1_dqs2_rx_de-skew = <6>; + cs1_dqs2p_tx_de-skew = <9>; + cs1_dqs2n_tx_de-skew = <9>; + + cs1_dm3_rx_de-skew = <7>; + cs1_dm3_tx_de-skew = <7>; + cs1_dq24_rx_de-skew = <8>; + cs1_dq24_tx_de-skew = <9>; + cs1_dq25_rx_de-skew = <9>; + cs1_dq25_tx_de-skew = <9>; + cs1_dq26_rx_de-skew = <9>; + cs1_dq26_tx_de-skew = <8>; + cs1_dq27_rx_de-skew = <8>; + cs1_dq27_tx_de-skew = <8>; + cs1_dq28_rx_de-skew = <9>; + cs1_dq28_tx_de-skew = <9>; + cs1_dq29_rx_de-skew = <9>; + cs1_dq29_tx_de-skew = <9>; + cs1_dq30_rx_de-skew = <9>; + cs1_dq30_tx_de-skew = <8>; + cs1_dq31_rx_de-skew = <8>; + cs1_dq31_tx_de-skew = <8>; + cs1_dqs3_rx_de-skew = <7>; + cs1_dqs3p_tx_de-skew = <9>; + cs1_dqs3n_tx_de-skew = <9>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-lvds-v10.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-lvds-v10.dts new file mode 100755 index 000000000000..eb60e71b77f3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-lvds-v10.dts @@ -0,0 +1,689 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include +#include +#include +#include +#include "px30.dtsi" +#include "px30-android.dtsi" + +/ { + model = "Rockchip PX30 evb ddr3 lvds board"; + compatible = "rockchip,px30-evb-ddr3-lvds-v10", "rockchip,px30"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1270000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <602000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <952000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <290000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + panel { + compatible = "samsung,lsl070nl01", "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc3v3_lcd>; + enable-delay-ms = <20>; + prepare-delay-ms = <20>; + unprepare-delay-ms = <20>; + disable-delay-ms = <20>; + bus-format = ; + + width-mm = <217>; + height-mm = <136>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <49500000>; + hactive = <1024>; + vactive = <600>; + hback-porch = <90>; + hfront-porch = <90>; + vback-porch = <10>; + vfront-porch = <10>; + hsync-len = <90>; + vsync-len = <10>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in_lvds: endpoint { + remote-endpoint = <&lvds_out_panel>; + }; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + /*clocks = <&rk809 1>;*/ + /*clock-names = "ext_clock";*/ + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + /*clocks = <&rk809 1>;*/ + /*clock-names = "ext_clock";*/ + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&display_subsystem { + status = "okay"; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_phy>; + clock_in_out = "output"; + snps,reset-gpio = <&gpio2 13 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 50000 50000>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x1>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x1>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x1>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x1>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc3v3_lcd: SWITCH_REG1 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + + vcc5v0_host: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_host"; + }; + }; + }; +}; + +&i2c1 { + status = "okay"; + + sensor@f { + status = "okay"; + compatible = "ak8963"; + reg = <0x0f>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <1>; + reprobe_en = <1>; + }; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; + }; + + sensor@4c { + status = "okay"; + compatible = "gs_mma7660"; + reg = <0x4c>; + type = ; + irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <2>; + reprobe_en = <1>; + }; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc_3v0>; + vccio4-supply = <&vcc3v0_pmu>; + vccio5-supply = <&vcc_3v0>; +}; + +&lvds { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + lvds_out_panel: endpoint { + remote-endpoint = <&panel_in_lvds>; + }; + }; + }; +}; + +&lvds_in_vopb { + status = "okay"; +}; + +&lvds_in_vopl { + status = "disabled"; +}; + +&route_lvds { + connect = <&vopb_out_lvds>; + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-avb.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-avb.dts new file mode 100755 index 000000000000..fca982485b16 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-avb.dts @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017-2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "px30-evb-ddr3-v10.dtsi" + +/ { + model = "Rockchip PX30 evb ddr3 board"; + compatible = "rockchip,px30-evb-ddr3-v10-avb", "rockchip,px30"; +}; + +&chosen { + bootargs_ext = "androidboot.boot_devices=ff390000.dwmmc,ff3b0000.nandc"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <20>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-linux.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-linux.dts new file mode 100755 index 000000000000..ff24bdd46aea --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-linux.dts @@ -0,0 +1,939 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "px30.dtsi" +#include "rk3326-linux.dtsi" + +/ { + model = "Rockchip linux PX30 evb ddr3 board"; + compatible = "rockchip,px30-evb-ddr3-v10-linux", "rockchip,px30"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <0>; + rockchip,android-charge-on = <1>; + rockchip,uboot-low-power-voltage = <3500>; + rockchip,screen-on-voltage = <3600>; + status = "okay"; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio2 RK_PB0 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + /*clocks = <&rk809 1>;*/ + /*clock-names = "ext_clock";*/ + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PB0 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&display_subsystem { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <20>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_phy>; + clock_in_out = "output"; + snps,reset-gpio = <&gpio2 13 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 50000 50000>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc5v0_host: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + + sensor@f { + status = "okay"; + compatible = "ak8963"; + reg = <0x0f>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <1>; + reprobe_en = <1>; + }; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + power-supply = <&vcc3v3_lcd>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; + }; + + sensor@4c { + status = "okay"; + compatible = "gs_mma7660"; + reg = <0x4c>; + type = ; + irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <2>; + reprobe_en = <1>; + }; +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vdd1v5_dvp>; + + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0>; + port { + ucam_out: endpoint { + remote-endpoint = <&mipi_in_ucam>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc_3v0>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc_3v0>; + vccio4-supply = <&vcc3v0_pmu>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rkisp1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ + diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-linux.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-linux.dts new file mode 100755 index 000000000000..317b53b2a1eb --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-linux.dts @@ -0,0 +1,627 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include "px30-robot.dtsi" + +/ { + model = "Rockchip linux PX30 evb ddr3 board"; + compatible = "rockchip,px30-evb-ddr3-v10-linux", "rockchip,px30"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + /*clocks = <&rk809 1>;*/ + /*clock-names = "ext_clock";*/ + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc5v0_host: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + pinctrl-0 = <&i2c2_xfer>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vdd1v5_dvp>; + + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0>; + port { + ucam_out: endpoint { + remote-endpoint = <&mipi_in_ucam>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc_3v0>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc_3v0>; + vccio4-supply = <&vcc3v0_pmu>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&rk_rga { + status = "okay"; +}; + +&rkisp1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-no-gpu-linux.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-no-gpu-linux.dts new file mode 100755 index 000000000000..81f44c36686c --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10-robot-no-gpu-linux.dts @@ -0,0 +1,627 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include "px30-robot-no-gpu.dtsi" + +/ { + model = "Rockchip linux PX30 evb ddr3 board"; + compatible = "rockchip,px30-evb-ddr3-v10-linux", "rockchip,px30"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + /*clocks = <&rk809 1>;*/ + /*clock-names = "ext_clock";*/ + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc5v0_host: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + pinctrl-0 = <&i2c2_xfer>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vdd1v5_dvp>; + + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0>; + port { + ucam_out: endpoint { + remote-endpoint = <&mipi_in_ucam>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc_3v0>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc_3v0>; + vccio4-supply = <&vcc3v0_pmu>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&rk_rga { + status = "okay"; +}; + +&rkisp1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dts new file mode 100755 index 000000000000..b814812edbe9 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dts @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include "px30-evb-ddr3-v10.dtsi" + +/ { + model = "Rockchip PX30 evb ddr3 board"; + compatible = "rockchip,px30-evb-ddr3-v10", "rockchip,px30"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <20>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dtsi b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dtsi new file mode 100755 index 000000000000..3d8e81009bef --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v10.dtsi @@ -0,0 +1,815 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017-2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "px30.dtsi" +#include "px30-android.dtsi" + +/ { + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <0>; + rockchip,android-charge-on = <1>; + rockchip,uboot-low-power-voltage = <3500>; + rockchip,screen-on-voltage = <3600>; + status = "okay"; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + /*clocks = <&rk809 1>;*/ + /*clock-names = "ext_clock";*/ + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PB0 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&display_subsystem { + status = "okay"; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_phy>; + clock_in_out = "output"; + snps,reset-gpio = <&gpio2 13 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 50000 50000>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc5v0_host: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG1 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + + sensor@f { + status = "okay"; + compatible = "ak8963"; + reg = <0x0f>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <1>; + reprobe_en = <1>; + }; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + power-supply = <&vcc3v3_lcd>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; + }; + + sensor@4c { + status = "okay"; + compatible = "gs_mma7660"; + reg = <0x4c>; + type = ; + irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <8>; + reprobe_en = <1>; + }; +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ucam_out: endpoint { + remote-endpoint = <&mipi_in_ucam>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc_3v0>; + vccio4-supply = <&vcc3v0_pmu>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rkisp1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-avb.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-avb.dts new file mode 100755 index 000000000000..f2659a7f524e --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-avb.dts @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017-2020 Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "px30-evb-ddr3-v10.dtsi" + +/ { + model = "Rockchip PX30 evb ddr3 board"; + compatible = "rockchip,px30-evb-ddr3-v11-avb", "rockchip,px30"; +}; + +&chosen { + bootargs_ext = "androidboot.boot_devices=ff390000.dwmmc,ff3b0000.nandc"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + prepare-delay-ms = <0>; + reset-delay-ms = <0>; + init-delay-ms = <80>; + enable-delay-ms = <0>; + disable-delay-ms = <10>; + unprepare-delay-ms = <60>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 39 00 04 ff 98 81 03 + 15 00 02 01 00 + 15 00 02 02 00 + 15 00 02 03 53 + 15 00 02 04 53 + 15 00 02 05 13 + 15 00 02 06 04 + 15 00 02 07 02 + 15 00 02 08 02 + 15 00 02 09 00 + 15 00 02 0a 00 + 15 00 02 0b 00 + 15 00 02 0c 00 + 15 00 02 0d 00 + 15 00 02 0e 00 + 15 00 02 0f 00 + + 15 00 02 10 00 + 15 00 02 11 00 + 15 00 02 12 00 + 15 00 02 13 00 + 15 00 02 14 00 + 15 00 02 15 08 + 15 00 02 16 10 + 15 00 02 17 00 + 15 00 02 18 08 + 15 00 02 19 00 + 15 00 02 1a 00 + 15 00 02 1b 00 + 15 00 02 1c 00 + 15 00 02 1d 00 + 15 00 02 1e c0 + 15 00 02 1f 80 + + 15 00 02 20 02 + 15 00 02 21 09 + 15 00 02 22 00 + 15 00 02 23 00 + 15 00 02 24 00 + 15 00 02 25 00 + 15 00 02 26 00 + 15 00 02 27 00 + 15 00 02 28 55 + 15 00 02 29 03 + 15 00 02 2a 00 + 15 00 02 2b 00 + 15 00 02 2c 00 + 15 00 02 2d 00 + 15 00 02 2e 00 + 15 00 02 2f 00 + + 15 00 02 30 00 + 15 00 02 31 00 + 15 00 02 32 00 + 15 00 02 33 00 + 15 00 02 34 04 + 15 00 02 35 05 + 15 00 02 36 05 + 15 00 02 37 00 + 15 00 02 38 3c + 15 00 02 39 35 + 15 00 02 3a 00 + 15 00 02 3b 40 + 15 00 02 3c 00 + 15 00 02 3d 00 + 15 00 02 3e 00 + 15 00 02 3f 00 + + 15 00 02 40 00 + 15 00 02 41 88 + 15 00 02 42 00 + 15 00 02 43 00 + 15 00 02 44 1f + + 15 00 02 50 01 + 15 00 02 51 23 + 15 00 02 52 45 + 15 00 02 53 67 + 15 00 02 54 89 + 15 00 02 55 ab + 15 00 02 56 01 + 15 00 02 57 23 + 15 00 02 58 45 + 15 00 02 59 67 + 15 00 02 5a 89 + 15 00 02 5b ab + 15 00 02 5c cd + 15 00 02 5d ef + 15 00 02 5e 03 + 15 00 02 5f 14 + + 15 00 02 60 15 + 15 00 02 61 0c + 15 00 02 62 0d + 15 00 02 63 0e + 15 00 02 64 0f + 15 00 02 65 10 + 15 00 02 66 11 + 15 00 02 67 08 + 15 00 02 68 02 + 15 00 02 69 0a + 15 00 02 6a 02 + 15 00 02 6b 02 + 15 00 02 6c 02 + 15 00 02 6d 02 + 15 00 02 6e 02 + 15 00 02 6f 02 + + 15 00 02 70 02 + 15 00 02 71 02 + 15 00 02 72 06 + 15 00 02 73 02 + 15 00 02 74 02 + 15 00 02 75 14 + 15 00 02 76 15 + 15 00 02 77 0f + 15 00 02 78 0e + 15 00 02 79 0d + 15 00 02 7a 0c + 15 00 02 7b 11 + 15 00 02 7c 10 + 15 00 02 7d 06 + 15 00 02 7e 02 + 15 00 02 7f 0a + + 15 00 02 80 02 + 15 00 02 81 02 + 15 00 02 82 02 + 15 00 02 83 02 + 15 00 02 84 02 + 15 00 02 85 02 + 15 00 02 86 02 + 15 00 02 87 02 + 15 00 02 88 08 + 15 00 02 89 02 + 15 00 02 8a 02 + + 39 00 04 ff 98 81 04 + 15 00 02 00 80 + 15 00 02 70 00 + 15 00 02 71 00 + 15 00 02 66 fe + 15 00 02 82 15 + 15 00 02 84 15 + 15 00 02 85 15 + 15 00 02 3a 24 + 15 00 02 32 ac + 15 00 02 8c 80 + 15 00 02 3c f5 + 15 00 02 88 33 + + 39 00 04 ff 98 81 01 + 15 00 02 22 0a + 15 00 02 31 00 + 15 00 02 53 78 + 15 00 02 50 5b + 15 00 02 51 5b + 15 00 02 60 20 + 15 00 02 61 00 + 15 00 02 62 0d + 15 00 02 63 00 + + 15 00 02 a0 00 + 15 00 02 a1 10 + 15 00 02 a2 1c + 15 00 02 a3 13 + 15 00 02 a4 15 + 15 00 02 a5 26 + 15 00 02 a6 1a + 15 00 02 a7 1d + 15 00 02 a8 67 + 15 00 02 a9 1c + 15 00 02 aa 29 + 15 00 02 ab 5b + 15 00 02 ac 26 + 15 00 02 ad 28 + 15 00 02 ae 5c + 15 00 02 af 30 + 15 00 02 b0 31 + 15 00 02 b1 2e + 15 00 02 b2 32 + 15 00 02 b3 00 + + 15 00 02 c0 00 + 15 00 02 c1 10 + 15 00 02 c2 1c + 15 00 02 c3 13 + 15 00 02 c4 15 + 15 00 02 c5 26 + 15 00 02 c6 1a + 15 00 02 c7 1d + 15 00 02 c8 67 + 15 00 02 c9 1c + 15 00 02 ca 29 + 15 00 02 cb 5b + 15 00 02 cc 26 + 15 00 02 cd 28 + 15 00 02 ce 5c + 15 00 02 cf 30 + 15 00 02 d0 31 + 15 00 02 d1 2e + 15 00 02 d2 32 + 15 00 02 d3 00 + 39 00 04 ff 98 81 00 + 05 00 01 11 + 05 01 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing1>; + + timing1: timing1 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; + +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-linux.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-linux.dts new file mode 100755 index 000000000000..54a4c19d11b3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11-linux.dts @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "px30-evb-ddr3-v10-linux.dts" + +/ { + model = "Rockchip linux PX30 evb ddr3 board"; + compatible = "rockchip,px30-evb-ddr3-v11-linux", "rockchip,px30"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + prepare-delay-ms = <0>; + reset-delay-ms = <0>; + init-delay-ms = <80>; + enable-delay-ms = <0>; + disable-delay-ms = <10>; + unprepare-delay-ms = <60>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 39 00 04 ff 98 81 03 + 15 00 02 01 00 + 15 00 02 02 00 + 15 00 02 03 53 + 15 00 02 04 53 + 15 00 02 05 13 + 15 00 02 06 04 + 15 00 02 07 02 + 15 00 02 08 02 + 15 00 02 09 00 + 15 00 02 0a 00 + 15 00 02 0b 00 + 15 00 02 0c 00 + 15 00 02 0d 00 + 15 00 02 0e 00 + 15 00 02 0f 00 + + 15 00 02 10 00 + 15 00 02 11 00 + 15 00 02 12 00 + 15 00 02 13 00 + 15 00 02 14 00 + 15 00 02 15 08 + 15 00 02 16 10 + 15 00 02 17 00 + 15 00 02 18 08 + 15 00 02 19 00 + 15 00 02 1a 00 + 15 00 02 1b 00 + 15 00 02 1c 00 + 15 00 02 1d 00 + 15 00 02 1e c0 + 15 00 02 1f 80 + + 15 00 02 20 02 + 15 00 02 21 09 + 15 00 02 22 00 + 15 00 02 23 00 + 15 00 02 24 00 + 15 00 02 25 00 + 15 00 02 26 00 + 15 00 02 27 00 + 15 00 02 28 55 + 15 00 02 29 03 + 15 00 02 2a 00 + 15 00 02 2b 00 + 15 00 02 2c 00 + 15 00 02 2d 00 + 15 00 02 2e 00 + 15 00 02 2f 00 + + 15 00 02 30 00 + 15 00 02 31 00 + 15 00 02 32 00 + 15 00 02 33 00 + 15 00 02 34 04 + 15 00 02 35 05 + 15 00 02 36 05 + 15 00 02 37 00 + 15 00 02 38 3c + 15 00 02 39 35 + 15 00 02 3a 00 + 15 00 02 3b 40 + 15 00 02 3c 00 + 15 00 02 3d 00 + 15 00 02 3e 00 + 15 00 02 3f 00 + + 15 00 02 40 00 + 15 00 02 41 88 + 15 00 02 42 00 + 15 00 02 43 00 + 15 00 02 44 1f + + 15 00 02 50 01 + 15 00 02 51 23 + 15 00 02 52 45 + 15 00 02 53 67 + 15 00 02 54 89 + 15 00 02 55 ab + 15 00 02 56 01 + 15 00 02 57 23 + 15 00 02 58 45 + 15 00 02 59 67 + 15 00 02 5a 89 + 15 00 02 5b ab + 15 00 02 5c cd + 15 00 02 5d ef + 15 00 02 5e 03 + 15 00 02 5f 14 + + 15 00 02 60 15 + 15 00 02 61 0c + 15 00 02 62 0d + 15 00 02 63 0e + 15 00 02 64 0f + 15 00 02 65 10 + 15 00 02 66 11 + 15 00 02 67 08 + 15 00 02 68 02 + 15 00 02 69 0a + 15 00 02 6a 02 + 15 00 02 6b 02 + 15 00 02 6c 02 + 15 00 02 6d 02 + 15 00 02 6e 02 + 15 00 02 6f 02 + + 15 00 02 70 02 + 15 00 02 71 02 + 15 00 02 72 06 + 15 00 02 73 02 + 15 00 02 74 02 + 15 00 02 75 14 + 15 00 02 76 15 + 15 00 02 77 0f + 15 00 02 78 0e + 15 00 02 79 0d + 15 00 02 7a 0c + 15 00 02 7b 11 + 15 00 02 7c 10 + 15 00 02 7d 06 + 15 00 02 7e 02 + 15 00 02 7f 0a + + 15 00 02 80 02 + 15 00 02 81 02 + 15 00 02 82 02 + 15 00 02 83 02 + 15 00 02 84 02 + 15 00 02 85 02 + 15 00 02 86 02 + 15 00 02 87 02 + 15 00 02 88 08 + 15 00 02 89 02 + 15 00 02 8a 02 + + 39 00 04 ff 98 81 04 + 15 00 02 00 80 + 15 00 02 70 00 + 15 00 02 71 00 + 15 00 02 66 fe + 15 00 02 82 15 + 15 00 02 84 15 + 15 00 02 85 15 + 15 00 02 3a 24 + 15 00 02 32 ac + 15 00 02 8c 80 + 15 00 02 3c f5 + 15 00 02 88 33 + + 39 00 04 ff 98 81 01 + 15 00 02 22 0a + 15 00 02 31 00 + 15 00 02 53 78 + 15 00 02 50 5b + 15 00 02 51 5b + 15 00 02 60 20 + 15 00 02 61 00 + 15 00 02 62 0d + 15 00 02 63 00 + + 15 00 02 a0 00 + 15 00 02 a1 10 + 15 00 02 a2 1c + 15 00 02 a3 13 + 15 00 02 a4 15 + 15 00 02 a5 26 + 15 00 02 a6 1a + 15 00 02 a7 1d + 15 00 02 a8 67 + 15 00 02 a9 1c + 15 00 02 aa 29 + 15 00 02 ab 5b + 15 00 02 ac 26 + 15 00 02 ad 28 + 15 00 02 ae 5c + 15 00 02 af 30 + 15 00 02 b0 31 + 15 00 02 b1 2e + 15 00 02 b2 32 + 15 00 02 b3 00 + + 15 00 02 c0 00 + 15 00 02 c1 10 + 15 00 02 c2 1c + 15 00 02 c3 13 + 15 00 02 c4 15 + 15 00 02 c5 26 + 15 00 02 c6 1a + 15 00 02 c7 1d + 15 00 02 c8 67 + 15 00 02 c9 1c + 15 00 02 ca 29 + 15 00 02 cb 5b + 15 00 02 cc 26 + 15 00 02 cd 28 + 15 00 02 ce 5c + 15 00 02 cf 30 + 15 00 02 d0 31 + 15 00 02 d1 2e + 15 00 02 d2 32 + 15 00 02 d3 00 + 39 00 04 ff 98 81 00 + 05 00 01 11 + 05 01 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing1>; + + timing1: timing1 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11.dts new file mode 100755 index 000000000000..2e59a09d3c5c --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr3-v11.dts @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "px30-evb-ddr3-v10.dtsi" + +/ { + model = "Rockchip PX30 evb ddr3 board"; + compatible = "rockchip,px30-evb-ddr3-v11", "rockchip,px30"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + prepare-delay-ms = <0>; + reset-delay-ms = <0>; + init-delay-ms = <80>; + enable-delay-ms = <0>; + disable-delay-ms = <10>; + unprepare-delay-ms = <60>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 39 00 04 ff 98 81 03 + 15 00 02 01 00 + 15 00 02 02 00 + 15 00 02 03 53 + 15 00 02 04 53 + 15 00 02 05 13 + 15 00 02 06 04 + 15 00 02 07 02 + 15 00 02 08 02 + 15 00 02 09 00 + 15 00 02 0a 00 + 15 00 02 0b 00 + 15 00 02 0c 00 + 15 00 02 0d 00 + 15 00 02 0e 00 + 15 00 02 0f 00 + + 15 00 02 10 00 + 15 00 02 11 00 + 15 00 02 12 00 + 15 00 02 13 00 + 15 00 02 14 00 + 15 00 02 15 08 + 15 00 02 16 10 + 15 00 02 17 00 + 15 00 02 18 08 + 15 00 02 19 00 + 15 00 02 1a 00 + 15 00 02 1b 00 + 15 00 02 1c 00 + 15 00 02 1d 00 + 15 00 02 1e c0 + 15 00 02 1f 80 + + 15 00 02 20 02 + 15 00 02 21 09 + 15 00 02 22 00 + 15 00 02 23 00 + 15 00 02 24 00 + 15 00 02 25 00 + 15 00 02 26 00 + 15 00 02 27 00 + 15 00 02 28 55 + 15 00 02 29 03 + 15 00 02 2a 00 + 15 00 02 2b 00 + 15 00 02 2c 00 + 15 00 02 2d 00 + 15 00 02 2e 00 + 15 00 02 2f 00 + + 15 00 02 30 00 + 15 00 02 31 00 + 15 00 02 32 00 + 15 00 02 33 00 + 15 00 02 34 04 + 15 00 02 35 05 + 15 00 02 36 05 + 15 00 02 37 00 + 15 00 02 38 3c + 15 00 02 39 35 + 15 00 02 3a 00 + 15 00 02 3b 40 + 15 00 02 3c 00 + 15 00 02 3d 00 + 15 00 02 3e 00 + 15 00 02 3f 00 + + 15 00 02 40 00 + 15 00 02 41 88 + 15 00 02 42 00 + 15 00 02 43 00 + 15 00 02 44 1f + + 15 00 02 50 01 + 15 00 02 51 23 + 15 00 02 52 45 + 15 00 02 53 67 + 15 00 02 54 89 + 15 00 02 55 ab + 15 00 02 56 01 + 15 00 02 57 23 + 15 00 02 58 45 + 15 00 02 59 67 + 15 00 02 5a 89 + 15 00 02 5b ab + 15 00 02 5c cd + 15 00 02 5d ef + 15 00 02 5e 03 + 15 00 02 5f 14 + + 15 00 02 60 15 + 15 00 02 61 0c + 15 00 02 62 0d + 15 00 02 63 0e + 15 00 02 64 0f + 15 00 02 65 10 + 15 00 02 66 11 + 15 00 02 67 08 + 15 00 02 68 02 + 15 00 02 69 0a + 15 00 02 6a 02 + 15 00 02 6b 02 + 15 00 02 6c 02 + 15 00 02 6d 02 + 15 00 02 6e 02 + 15 00 02 6f 02 + + 15 00 02 70 02 + 15 00 02 71 02 + 15 00 02 72 06 + 15 00 02 73 02 + 15 00 02 74 02 + 15 00 02 75 14 + 15 00 02 76 15 + 15 00 02 77 0f + 15 00 02 78 0e + 15 00 02 79 0d + 15 00 02 7a 0c + 15 00 02 7b 11 + 15 00 02 7c 10 + 15 00 02 7d 06 + 15 00 02 7e 02 + 15 00 02 7f 0a + + 15 00 02 80 02 + 15 00 02 81 02 + 15 00 02 82 02 + 15 00 02 83 02 + 15 00 02 84 02 + 15 00 02 85 02 + 15 00 02 86 02 + 15 00 02 87 02 + 15 00 02 88 08 + 15 00 02 89 02 + 15 00 02 8a 02 + + 39 00 04 ff 98 81 04 + 15 00 02 00 80 + 15 00 02 70 00 + 15 00 02 71 00 + 15 00 02 66 fe + 15 00 02 82 15 + 15 00 02 84 15 + 15 00 02 85 15 + 15 00 02 3a 24 + 15 00 02 32 ac + 15 00 02 8c 80 + 15 00 02 3c f5 + 15 00 02 88 33 + + 39 00 04 ff 98 81 01 + 15 00 02 22 0a + 15 00 02 31 00 + 15 00 02 53 78 + 15 00 02 50 5b + 15 00 02 51 5b + 15 00 02 60 20 + 15 00 02 61 00 + 15 00 02 62 0d + 15 00 02 63 00 + + 15 00 02 a0 00 + 15 00 02 a1 10 + 15 00 02 a2 1c + 15 00 02 a3 13 + 15 00 02 a4 15 + 15 00 02 a5 26 + 15 00 02 a6 1a + 15 00 02 a7 1d + 15 00 02 a8 67 + 15 00 02 a9 1c + 15 00 02 aa 29 + 15 00 02 ab 5b + 15 00 02 ac 26 + 15 00 02 ad 28 + 15 00 02 ae 5c + 15 00 02 af 30 + 15 00 02 b0 31 + 15 00 02 b1 2e + 15 00 02 b2 32 + 15 00 02 b3 00 + + 15 00 02 c0 00 + 15 00 02 c1 10 + 15 00 02 c2 1c + 15 00 02 c3 13 + 15 00 02 c4 15 + 15 00 02 c5 26 + 15 00 02 c6 1a + 15 00 02 c7 1d + 15 00 02 c8 67 + 15 00 02 c9 1c + 15 00 02 ca 29 + 15 00 02 cb 5b + 15 00 02 cc 26 + 15 00 02 cd 28 + 15 00 02 ce 5c + 15 00 02 cf 30 + 15 00 02 d0 31 + 15 00 02 d1 2e + 15 00 02 d2 32 + 15 00 02 d3 00 + 39 00 04 ff 98 81 00 + 05 00 01 11 + 05 01 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing1>; + + timing1: timing1 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; + +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ddr4-v10.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ddr4-v10.dts new file mode 100755 index 000000000000..7bfe640d32de --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ddr4-v10.dts @@ -0,0 +1,853 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "px30.dtsi" +#include "px30-android.dtsi" +#include "px30-ddr4p416dd6-timing.dtsi" +/ { + model = "Rockchip PX30 evb ddr4 board"; + compatible = "rockchip,px30-evb-ddr4-v10", "rockchip,px30"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio2 RK_PB0 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + /*clocks = <&rk809 1>;*/ + /*clock-names = "ext_clock";*/ + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + /*clocks = <&rk809 1>;*/ + /*clock-names = "ext_clock";*/ + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&display_subsystem { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <20>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&chosen { + bootargs_ext = "androidboot.boot_devices=ff390000.dwmmc,ff3b0000.nandc"; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_phy>; + assigned-clocks = <&cru SCLK_GMAC>; + assigned-clock-parents = <&gmac_clkin>; + clock_in_out = "input"; + pinctrl-names = "default"; + pinctrl-0 = <&rmii_pins &mac_refclk>; + snps,reset-gpio = <&gpio2 13 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 50000 50000>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + regulator-initial-mode = <0x1>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <2500000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc5v0_host: SWITCH_REG1 { + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + + sensor@f { + status = "okay"; + compatible = "ak8963"; + reg = <0x0f>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <1>; + reprobe_en = <1>; + }; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + power-supply = <&vcc3v3_lcd>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; + }; + + sensor@4c { + status = "okay"; + compatible = "gs_mma7660"; + reg = <0x4c>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <6>; + reprobe_en = <1>; + }; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc_3v0>; + vccio4-supply = <&vcc3v0_pmu>; + vccio5-supply = <&vcc_3v0>; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&nandc0 { + status = "okay"; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618-avb.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618-avb.dts new file mode 100755 index 000000000000..ea44da5066af --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618-avb.dts @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include "px30-evb-ext-rk618.dtsi" + +/ { + model = "Rockchip PX30 EVB EXT RK618 board"; + compatible = "rockchip,px30-evb-ext-rk618-avb", "rockchip,px30"; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dts b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dts new file mode 100755 index 000000000000..69f44b4d115c --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dts @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "px30-evb-ext-rk618.dtsi" + +/ { + model = "Rockchip PX30 EVB EXT RK618 board"; + compatible = "rockchip,px30-evb-ext-rk618", "rockchip,px30"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dtsi b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dtsi new file mode 100755 index 000000000000..0eac63181407 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-evb-ext-rk618.dtsi @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include "px30-evb-ddr3-v10.dtsi" + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <20>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; +}; + +&dmc { + auto-freq-en = <0>; +}; + +&vcc3v0_pmu { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-state-mem { + regulator-suspend-microvolt = <3300000>; + }; +}; + +&i2c1 { + + rk618@50 { + compatible = "rockchip,rk618"; + reg = <0x50>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "clkin"; + assigned-clocks = <&cru SCLK_I2S1_OUT>; + assigned-clock-rates = <11289600>; + reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; + status = "okay"; + + clock: cru { + compatible = "rockchip,rk618-cru"; + clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; + clock-names = "clkin", "lcdc0_dclkp"; + assigned-clocks = <&clock SCALER_PLLIN_CLK>, + <&clock VIF_PLLIN_CLK>, + <&clock SCALER_CLK>, + <&clock VIF0_PRE_CLK>, + <&clock CODEC_CLK>, + <&clock DITHER_CLK>; + assigned-clock-parents = <&cru SCLK_I2S1_OUT>, + <&clock LCDC0_CLK>, + <&clock SCALER_PLL_CLK>, + <&clock VIF_PLL_CLK>, + <&cru SCLK_I2S1_OUT>, + <&clock VIF0_CLK>; + #clock-cells = <1>; + status = "okay"; + }; + + hdmi { + compatible = "rockchip,rk618-hdmi"; + clocks = <&clock HDMI_CLK>; + clock-names = "hdmi"; + assigned-clocks = <&clock HDMI_CLK>; + assigned-clock-parents = <&clock VIF0_CLK>; + interrupt-parent = <&gpio2>; + interrupts = <12 IRQ_TYPE_LEVEL_HIGH>; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hdmi_in_rgb: endpoint { + remote-endpoint = <&rgb_out_hdmi>; + }; + }; + }; + }; + }; +}; + +&rgb { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + rgb_out_hdmi: endpoint { + remote-endpoint = <&hdmi_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vopb { + status = "disabled"; +}; + +&rgb_in_vopl { + status = "okay"; +}; + +&route_rgb { + connect = <&vopl_out_rgb>; + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-robot-no-gpu.dtsi b/arch/arm64/boot/dts/rockchip/px30-robot-no-gpu.dtsi new file mode 100755 index 000000000000..e3f4274b2309 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-robot-no-gpu.dtsi @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ +#include "px30-robot.dtsi" + +/ { + compatible = "rockchip,linux", "rockchip,px30-robot-no-gpu"; + + /* Remove gpu thermal and gpu cooling map */ + /delete-node/ thermal-zones; + + thermal_zones: thermal-zones { + soc_thermal: soc-thermal { + polling-delay-passive = <20>; + polling-delay = <1000>; + sustainable-power = <252>; + + thermal-sensors = <&tsadc 0>; + trips { + threshold: trip-point-0 { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + target: trip-point-1 { + temperature = <90000>; + hysteresis = <2000>; + type = "passive"; + }; + soc_crit: soc-crit { + temperature = <115000>; + hysteresis = <2000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&target>; + cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + + gpu_thermal: gpu-thermal { + polling-delay-passive = <100>; /* milliseconds */ + polling-delay = <1000>; /* milliseconds */ + + thermal-sensors = <&tsadc 1>; + }; + }; +}; + +&gpu { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-robot.dtsi b/arch/arm64/boot/dts/rockchip/px30-robot.dtsi new file mode 100755 index 000000000000..f48a753e6453 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-robot.dtsi @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include "px30.dtsi" + +/ { + compatible = "rockchip,linux", "rockchip,px30-robot"; + + chosen { + bootargs = "console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootwait"; + }; + + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <0>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "okay"; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + ramoops: ramoops@8000000 { + compatible = "ramoops"; + reg = <0x0 0x8000000 0x0 0xa0000>; + record-size = <0x20000>; + console-size = <0x80000>; + ftrace-size = <0x00000>; + pmsg-size = <0x00000>; + }; + }; +}; + +&cpu0_opp_table { + /delete-node/ opp-1248000000; + /delete-node/ opp-1296000000; + /delete-node/ opp-1416000000; + /delete-node/ opp-1512000000; +}; + +&dmc_opp_table { + /delete-node/ opp-666000000; + /delete-node/ opp-768000000; +}; + +&i2s1_2ch { + rockchip,playback-only; +}; + +&rng { + status = "okay"; +}; + +&soc_thermal { + trips { + threshold: trip-point-0 { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + target: trip-point-1 { + temperature = <90000>; + hysteresis = <2000>; + type = "passive"; + }; + soc_crit: soc-crit { + temperature = <115000>; + hysteresis = <2000>; + type = "critical"; + }; + }; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&uart2 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30-z7-a0-rk618-dsi.dts b/arch/arm64/boot/dts/rockchip/px30-z7-a0-rk618-dsi.dts new file mode 100755 index 000000000000..1657151c988f --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/px30-z7-a0-rk618-dsi.dts @@ -0,0 +1,875 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include +#include "px30.dtsi" +#include "px30-android.dtsi" + +/ { + model = "Rockchip PX30 Z7 A0 board"; + compatible = "rockchip,px30-z7-a0", "rockchip,px30"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; +}; + +&display_subsystem { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + reset-gpios = <&gpio2 RK_PA1 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <20>; + reset-delay-ms = <20>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <20>; + unprepare-delay-ms = <20>; + + width-mm = <95>; + height-mm = <151>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 15 00 02 b0 00 + 15 00 02 d6 01 + 39 00 06 b3 14 08 00 22 00 + 15 00 02 b4 0c + 15 00 02 de 00 + 39 00 03 b6 3a d3 + 15 00 02 51 e0 + 15 00 02 53 04 + 15 00 02 3a 77 + 15 00 02 35 01 + 39 00 05 2a 00 00 04 af + 39 00 05 2b 00 00 07 7f + 05 96 01 29 + 05 14 01 11 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <156000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <60>; + hfront-porch = <80>; + vback-porch = <4>; + vfront-porch = <4>; + hsync-len = <10>; + vsync-len = <1>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + auto-freq-en = <0>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_soc"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc1v0_soc"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v0_pmu"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_sd"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-name = "vcc2v8_dvp"; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vdd1v5_dvp"; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc5v0_host: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + }; + }; + + rk618@50 { + compatible = "rockchip,rk618"; + reg = <0x50>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "clkin"; + assigned-clocks = <&cru SCLK_I2S1_OUT>; + assigned-clock-rates = <12000000>; + reset-gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; + status = "okay"; + + clock: cru { + compatible = "rockchip,rk618-cru"; + clocks = <&cru SCLK_I2S1_OUT>, <&cru DCLK_VOPL>; + clock-names = "clkin", "lcdc0_dclkp"; + assigned-clocks = <&clock SCALER_PLLIN_CLK>, + <&clock VIF_PLLIN_CLK>, + <&clock SCALER_CLK>, + <&clock VIF0_PRE_CLK>, + <&clock CODEC_CLK>, + <&clock DITHER_CLK>; + assigned-clock-parents = <&cru SCLK_I2S1_OUT>, + <&clock LCDC0_CLK>, + <&clock SCALER_PLL_CLK>, + <&clock VIF_PLL_CLK>, + <&cru SCLK_I2S1_OUT>, + <&clock VIF0_CLK>; + #clock-cells = <1>; + status = "okay"; + }; + + dsi { + compatible = "rockchip,rk618-dsi"; + clocks = <&clock MIPI_CLK>; + clock-names = "dsi"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + dsi_in_rgb: endpoint { + remote-endpoint = <&rgb_out_dsi>; + }; + }; + }; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd>; + backlight = <&backlight>; + reset-gpios = <&gpio2 RK_PA0 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <20>; + reset-delay-ms = <20>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <20>; + unprepare-delay-ms = <20>; + + width-mm = <95>; + height-mm = <151>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | + MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | + MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 15 00 02 b0 00 + 15 00 02 d6 01 + 39 00 06 b3 14 08 00 22 00 + 15 00 02 b4 0c + 15 00 02 DE 00 + 39 00 03 b6 3a d3 + 15 00 02 51 E0 + 15 00 02 53 04 + 15 00 02 3a 77 + 15 00 02 35 01 + 39 00 05 2A 00 00 04 AF + 39 00 05 2B 00 00 07 7F + 05 96 01 29 + 05 14 01 11 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing1>; + + timing1: timing1 { + clock-frequency = <156000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <60>; + hfront-porch = <80>; + vback-porch = <4>; + vfront-porch = <4>; + hsync-len = <10>; + vsync-len = <1>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; + }; + }; +}; + +&io_domains { + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc_3v0>; + vccio4-supply = <&vcc3v0_pmu>; + vccio5-supply = <&vcc_3v0>; + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm0 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + rockchip,sleep-debug-en = <1>; + status = "okay"; +}; + +&saradc { + vref-supply = <&vcc1v8_soc>; + status = "okay"; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "init", "default"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&rgb { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + rgb_out_dsi: endpoint { + remote-endpoint = <&dsi_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vopl { + status = "okay"; +}; + +&rgb_in_vopb { + status = "disabled"; +}; + +&route_rgb { + connect = <&vopl_out_rgb>; + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + + fstab { + compatible = "android,fstab"; + + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/px30.dtsi b/arch/arm64/boot/dts/rockchip/px30.dtsi index 0d6761074b11..237c0d9af325 100644 --- a/arch/arm64/boot/dts/rockchip/px30.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30.dtsi @@ -25,6 +25,9 @@ aliases { i2c1 = &i2c1; i2c2 = &i2c2; i2c3 = &i2c3; + mmc0 = &sdmmc; + mmc1 = &sdio; + mmc2 = &emmc; serial0 = &uart0; serial1 = &uart1; serial2 = &uart2; @@ -244,20 +247,20 @@ power: power-controller { #size-cells = <0>; /* These power domains are grouped by VD_LOGIC */ - power-domain@PX30_PD_USB { + pd_usb@PX30_PD_USB { reg = ; clocks = <&cru HCLK_HOST>, <&cru HCLK_OTG>, <&cru SCLK_OTG_ADP>; pm_qos = <&qos_usb_host>, <&qos_usb_otg>; }; - power-domain@PX30_PD_SDCARD { + pd_sdcard@PX30_PD_SDCARD { reg = ; clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>; pm_qos = <&qos_sdmmc>; }; - power-domain@PX30_PD_GMAC { + pd_gmac@PX30_PD_GMAC { reg = ; clocks = <&cru ACLK_GMAC>, <&cru PCLK_GMAC>, @@ -265,7 +268,7 @@ power-domain@PX30_PD_GMAC { <&cru SCLK_GMAC_RX_TX>; pm_qos = <&qos_gmac>; }; - power-domain@PX30_PD_MMC_NAND { + pd_mmc_nand@PX30_PD_MMC_NAND { reg = ; clocks = <&cru HCLK_NANDC>, <&cru HCLK_EMMC>, @@ -278,14 +281,14 @@ power-domain@PX30_PD_MMC_NAND { pm_qos = <&qos_emmc>, <&qos_nand>, <&qos_sdio>, <&qos_sfc>; }; - power-domain@PX30_PD_VPU { + pd_vpu@PX30_PD_VPU { reg = ; clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>, <&cru SCLK_CORE_VPU>; pm_qos = <&qos_vpu>, <&qos_vpu_r128>; }; - power-domain@PX30_PD_VO { + pd_vo@PX30_PD_VO { reg = ; clocks = <&cru ACLK_RGA>, <&cru ACLK_VOPB>, @@ -301,7 +304,7 @@ power-domain@PX30_PD_VO { pm_qos = <&qos_rga_rd>, <&qos_rga_wr>, <&qos_vop_m0>, <&qos_vop_m1>; }; - power-domain@PX30_PD_VI { + pd_vi@PX30_PD_VI { reg = ; clocks = <&cru ACLK_CIF>, <&cru ACLK_ISP>, @@ -312,7 +315,7 @@ power-domain@PX30_PD_VI { <&qos_isp_wr>, <&qos_isp_m1>, <&qos_vip>; }; - power-domain@PX30_PD_GPU { + pd_gpu@PX30_PD_GPU { reg = ; clocks = <&cru SCLK_GPU>; pm_qos = <&qos_gpu>; @@ -612,7 +615,7 @@ pwm0: pwm@ff200000 { reg = <0x0 0xff200000 0x0 0x10>; clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_pin>; #pwm-cells = <3>; status = "disabled"; @@ -623,7 +626,7 @@ pwm1: pwm@ff200010 { reg = <0x0 0xff200010 0x0 0x10>; clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_pin>; #pwm-cells = <3>; status = "disabled"; @@ -634,7 +637,7 @@ pwm2: pwm@ff200020 { reg = <0x0 0xff200020 0x0 0x10>; clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm2_pin>; #pwm-cells = <3>; status = "disabled"; @@ -645,7 +648,7 @@ pwm3: pwm@ff200030 { reg = <0x0 0xff200030 0x0 0x10>; clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm3_pin>; #pwm-cells = <3>; status = "disabled"; @@ -656,7 +659,7 @@ pwm4: pwm@ff208000 { reg = <0x0 0xff208000 0x0 0x10>; clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm4_pin>; #pwm-cells = <3>; status = "disabled"; @@ -667,7 +670,7 @@ pwm5: pwm@ff208010 { reg = <0x0 0xff208010 0x0 0x10>; clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm5_pin>; #pwm-cells = <3>; status = "disabled"; @@ -678,7 +681,7 @@ pwm6: pwm@ff208020 { reg = <0x0 0xff208020 0x0 0x10>; clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm6_pin>; #pwm-cells = <3>; status = "disabled"; @@ -689,7 +692,7 @@ pwm7: pwm@ff208030 { reg = <0x0 0xff208030 0x0 0x10>; clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm7_pin>; #pwm-cells = <3>; status = "disabled"; diff --git a/arch/arm64/boot/dts/rockchip/rk1808-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk1808-dram-default-timing.dtsi new file mode 100755 index 000000000000..0fa79e2f05c0 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk1808-dram-default-timing.dtsi @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include +#include + +/ { + ddr_timing: ddr_timing { + compatible = "rockchip,ddr-timing"; + ddr2_speed_bin = ; + ddr3_speed_bin = ; + ddr4_speed_bin = ; + pd_idle = <0>; + sr_idle = <0>; + sr_mc_gate_idle = <0>; + srpd_lite_idle = <0>; + standby_idle = <0>; + + auto_pd_dis_freq = <1066>; + auto_sr_dis_freq = <800>; + ddr2_dll_dis_freq = <300>; + ddr3_dll_dis_freq = <300>; + ddr4_dll_dis_freq = <625>; + phy_dll_dis_freq = <400>; + + ddr2_odt_dis_freq = <100>; + phy_ddr2_odt_dis_freq = <100>; + ddr2_drv = ; + ddr2_odt = ; + phy_ddr2_ca_drv = ; + phy_ddr2_ck_drv = ; + phy_ddr2_dq_drv = ; + phy_ddr2_odt = ; + + ddr3_odt_dis_freq = <400>; + phy_ddr3_odt_dis_freq = <400>; + ddr3_drv = ; + ddr3_odt = ; + phy_ddr3_ca_drv = ; + phy_ddr3_ck_drv = ; + phy_ddr3_dq_drv = ; + phy_ddr3_odt = ; + + phy_lpddr2_odt_dis_freq = <666>; + lpddr2_drv = ; + phy_lpddr2_ca_drv = ; + phy_lpddr2_ck_drv = ; + phy_lpddr2_dq_drv = ; + phy_lpddr2_odt = ; + + lpddr3_odt_dis_freq = <400>; + phy_lpddr3_odt_dis_freq = <400>; + lpddr3_drv = ; + lpddr3_odt = ; + phy_lpddr3_ca_drv = ; + phy_lpddr3_ck_drv = ; + phy_lpddr3_dq_drv = ; + phy_lpddr3_odt = ; + + lpddr4_odt_dis_freq = <800>; + phy_lpddr4_odt_dis_freq = <800>; + lpddr4_drv = ; + lpddr4_dq_odt = ; + lpddr4_ca_odt = ; + phy_lpddr4_ca_drv = ; + phy_lpddr4_ck_cs_drv = ; + phy_lpddr4_dq_drv = ; + phy_lpddr4_odt = ; + + ddr4_odt_dis_freq = <666>; + phy_ddr4_odt_dis_freq = <666>; + ddr4_drv = ; + ddr4_odt = ; + phy_ddr4_ca_drv = ; + phy_ddr4_ck_drv = ; + phy_ddr4_dq_drv = ; + phy_ddr4_odt = ; + + /* + * CA de-skew, one step is 15ps, range 0-31 + * DDR3 CA define is different from others(DDR4/LPDDR2/LPDDR3). + */ + a0_ddr3a9_de-skew = <7>; + a1_ddr3a14_de-skew = <7>; + a2_ddr3a13_de-skew = <7>; + a3_ddr3a11_de-skew = <7>; + a4_ddr3a2_de-skew = <7>; + a5_ddr3a4_de-skew = <7>; + a6_ddr3a3_de-skew = <7>; + a7_ddr3a6_de-skew = <7>; + a8_ddr3a5_de-skew = <7>; + a9_ddr3a1_de-skew = <7>; + a10_ddr3a0_de-skew = <7>; + a11_ddr3a7_de-skew = <7>; + a12_ddr3casb_de-skew = <7>; + a13_ddr3a8_de-skew = <7>; + a14_ddr3odt0_de-skew = <7>; + a15_ddr3ba1_de-skew = <7>; + a16_ddr3rasb_de-skew = <7>; + a17_ddr3null_de-skew = <7>; + ba0_ddr3ba2_de-skew = <7>; + ba1_ddr3a12_de-skew = <7>; + bg0_ddr3ba0_de-skew = <7>; + bg1_ddr3web_de-skew = <7>; + cke_ddr3cke_de-skew = <7>; + ck_ddr3ck_de-skew = <7>; + ckb_ddr3ckb_de-skew = <7>; + csb0_ddr3a10_de-skew = <7>; + odt0_ddr3a15_de-skew = <7>; + resetn_ddr3resetn_de-skew = <7>; + actn_ddr3csb0_de-skew = <7>; + csb1_ddr3csb1_de-skew = <7>; + odt1_ddr3odt1_de-skew = <7>; + + /* DATA de-skew, one step is 15ps, range 0-31 */ + /* cs0_skew_a */ + cs0_dm0_rx_de-skew = <7>; + cs0_dm0_tx_de-skew = <7>; + cs0_dq0_rx_de-skew = <7>; + cs0_dq0_tx_de-skew = <7>; + cs0_dq1_rx_de-skew = <7>; + cs0_dq1_tx_de-skew = <7>; + cs0_dq2_rx_de-skew = <7>; + cs0_dq2_tx_de-skew = <7>; + cs0_dq3_rx_de-skew = <7>; + cs0_dq3_tx_de-skew = <7>; + cs0_dq4_rx_de-skew = <7>; + cs0_dq4_tx_de-skew = <7>; + cs0_dq5_rx_de-skew = <7>; + cs0_dq5_tx_de-skew = <7>; + cs0_dq6_rx_de-skew = <7>; + cs0_dq6_tx_de-skew = <7>; + cs0_dq7_rx_de-skew = <7>; + cs0_dq7_tx_de-skew = <7>; + cs0_dqs0p_rx_de-skew = <14>; + cs0_dqs0p_tx_de-skew = <9>; + cs0_dqs0n_tx_de-skew = <9>; + cs0_dm1_rx_de-skew = <7>; + cs0_dm1_tx_de-skew = <7>; + cs0_dq8_rx_de-skew = <7>; + cs0_dq8_tx_de-skew = <7>; + cs0_dq9_rx_de-skew = <7>; + cs0_dq9_tx_de-skew = <7>; + cs0_dq10_rx_de-skew = <7>; + cs0_dq10_tx_de-skew = <7>; + cs0_dq11_rx_de-skew = <7>; + cs0_dq11_tx_de-skew = <7>; + cs0_dq12_rx_de-skew = <7>; + cs0_dq12_tx_de-skew = <7>; + cs0_dq13_rx_de-skew = <7>; + cs0_dq13_tx_de-skew = <7>; + cs0_dq14_rx_de-skew = <7>; + cs0_dq14_tx_de-skew = <7>; + cs0_dq15_rx_de-skew = <7>; + cs0_dq15_tx_de-skew = <7>; + cs0_dqs1p_rx_de-skew = <14>; + cs0_dqs1p_tx_de-skew = <9>; + cs0_dqs1n_tx_de-skew = <9>; + cs0_dqs0n_rx_de-skew = <14>; + cs0_dqs1n_rx_de-skew = <14>; + + /* cs0_skew_b */ + cs0_dm2_rx_de-skew = <7>; + cs0_dm2_tx_de-skew = <7>; + cs0_dq16_rx_de-skew = <7>; + cs0_dq16_tx_de-skew = <7>; + cs0_dq17_rx_de-skew = <7>; + cs0_dq17_tx_de-skew = <7>; + cs0_dq18_rx_de-skew = <7>; + cs0_dq18_tx_de-skew = <7>; + cs0_dq19_rx_de-skew = <7>; + cs0_dq19_tx_de-skew = <7>; + cs0_dq20_rx_de-skew = <7>; + cs0_dq20_tx_de-skew = <7>; + cs0_dq21_rx_de-skew = <7>; + cs0_dq21_tx_de-skew = <7>; + cs0_dq22_rx_de-skew = <7>; + cs0_dq22_tx_de-skew = <7>; + cs0_dq23_rx_de-skew = <7>; + cs0_dq23_tx_de-skew = <7>; + cs0_dqs2p_rx_de-skew = <14>; + cs0_dqs2p_tx_de-skew = <9>; + cs0_dqs2n_tx_de-skew = <9>; + cs0_dm3_rx_de-skew = <7>; + cs0_dm3_tx_de-skew = <7>; + cs0_dq24_rx_de-skew = <7>; + cs0_dq24_tx_de-skew = <7>; + cs0_dq25_rx_de-skew = <7>; + cs0_dq25_tx_de-skew = <7>; + cs0_dq26_rx_de-skew = <7>; + cs0_dq26_tx_de-skew = <7>; + cs0_dq27_rx_de-skew = <7>; + cs0_dq27_tx_de-skew = <7>; + cs0_dq28_rx_de-skew = <7>; + cs0_dq28_tx_de-skew = <7>; + cs0_dq29_rx_de-skew = <7>; + cs0_dq29_tx_de-skew = <7>; + cs0_dq30_rx_de-skew = <7>; + cs0_dq30_tx_de-skew = <7>; + cs0_dq31_rx_de-skew = <7>; + cs0_dq31_tx_de-skew = <7>; + cs0_dqs3p_rx_de-skew = <14>; + cs0_dqs3p_tx_de-skew = <9>; + cs0_dqs3n_tx_de-skew = <9>; + cs0_dqs2n_rx_de-skew = <14>; + cs0_dqs3n_rx_de-skew = <14>; + + /* cs1_skew_a */ + cs1_dm0_rx_de-skew = <7>; + cs1_dm0_tx_de-skew = <7>; + cs1_dq0_rx_de-skew = <7>; + cs1_dq0_tx_de-skew = <7>; + cs1_dq1_rx_de-skew = <7>; + cs1_dq1_tx_de-skew = <7>; + cs1_dq2_rx_de-skew = <7>; + cs1_dq2_tx_de-skew = <7>; + cs1_dq3_rx_de-skew = <7>; + cs1_dq3_tx_de-skew = <7>; + cs1_dq4_rx_de-skew = <7>; + cs1_dq4_tx_de-skew = <7>; + cs1_dq5_rx_de-skew = <7>; + cs1_dq5_tx_de-skew = <7>; + cs1_dq6_rx_de-skew = <7>; + cs1_dq6_tx_de-skew = <7>; + cs1_dq7_rx_de-skew = <7>; + cs1_dq7_tx_de-skew = <7>; + cs1_dqs0p_rx_de-skew = <14>; + cs1_dqs0p_tx_de-skew = <9>; + cs1_dqs0n_tx_de-skew = <9>; + cs1_dm1_rx_de-skew = <7>; + cs1_dm1_tx_de-skew = <7>; + cs1_dq8_rx_de-skew = <7>; + cs1_dq8_tx_de-skew = <7>; + cs1_dq9_rx_de-skew = <7>; + cs1_dq9_tx_de-skew = <7>; + cs1_dq10_rx_de-skew = <7>; + cs1_dq10_tx_de-skew = <7>; + cs1_dq11_rx_de-skew = <7>; + cs1_dq11_tx_de-skew = <7>; + cs1_dq12_rx_de-skew = <7>; + cs1_dq12_tx_de-skew = <7>; + cs1_dq13_rx_de-skew = <7>; + cs1_dq13_tx_de-skew = <7>; + cs1_dq14_rx_de-skew = <7>; + cs1_dq14_tx_de-skew = <7>; + cs1_dq15_rx_de-skew = <7>; + cs1_dq15_tx_de-skew = <7>; + cs1_dqs1p_rx_de-skew = <14>; + cs1_dqs1p_tx_de-skew = <9>; + cs1_dqs1n_tx_de-skew = <9>; + cs1_dqs0n_rx_de-skew = <14>; + cs1_dqs1n_rx_de-skew = <14>; + + /* cs1_skew_b */ + cs1_dm2_rx_de-skew = <7>; + cs1_dm2_tx_de-skew = <7>; + cs1_dq16_rx_de-skew = <7>; + cs1_dq16_tx_de-skew = <7>; + cs1_dq17_rx_de-skew = <7>; + cs1_dq17_tx_de-skew = <7>; + cs1_dq18_rx_de-skew = <7>; + cs1_dq18_tx_de-skew = <7>; + cs1_dq19_rx_de-skew = <7>; + cs1_dq19_tx_de-skew = <7>; + cs1_dq20_rx_de-skew = <7>; + cs1_dq20_tx_de-skew = <7>; + cs1_dq21_rx_de-skew = <7>; + cs1_dq21_tx_de-skew = <7>; + cs1_dq22_rx_de-skew = <7>; + cs1_dq22_tx_de-skew = <7>; + cs1_dq23_rx_de-skew = <7>; + cs1_dq23_tx_de-skew = <7>; + cs1_dqs2p_rx_de-skew = <14>; + cs1_dqs2p_tx_de-skew = <9>; + cs1_dqs2n_tx_de-skew = <9>; + cs1_dm3_rx_de-skew = <7>; + cs1_dm3_tx_de-skew = <7>; + cs1_dq24_rx_de-skew = <7>; + cs1_dq24_tx_de-skew = <7>; + cs1_dq25_rx_de-skew = <7>; + cs1_dq25_tx_de-skew = <7>; + cs1_dq26_rx_de-skew = <7>; + cs1_dq26_tx_de-skew = <7>; + cs1_dq27_rx_de-skew = <7>; + cs1_dq27_tx_de-skew = <7>; + cs1_dq28_rx_de-skew = <7>; + cs1_dq28_tx_de-skew = <7>; + cs1_dq29_rx_de-skew = <7>; + cs1_dq29_tx_de-skew = <7>; + cs1_dq30_rx_de-skew = <7>; + cs1_dq30_tx_de-skew = <7>; + cs1_dq31_rx_de-skew = <7>; + cs1_dq31_tx_de-skew = <7>; + cs1_dqs3p_rx_de-skew = <14>; + cs1_dqs3p_tx_de-skew = <9>; + cs1_dqs3n_tx_de-skew = <9>; + cs1_dqs2n_rx_de-skew = <14>; + cs1_dqs3n_rx_de-skew = <14>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk1808-evb-v10.dts b/arch/arm64/boot/dts/rockchip/rk1808-evb-v10.dts new file mode 100755 index 000000000000..a09824816188 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk1808-evb-v10.dts @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include "rk1808-evb.dtsi" + +/ { + model = "Rockchip RK1808 EVB V10 Board"; + compatible = "rockchip,rk1808-evb-v10", "rockchip,rk1808"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff550000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootfstype=ext4 rootwait kpti=0 snd_aloop.index=7"; + }; + + vad-sound { + status = "okay"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk1808-vad"; + rockchip,cpu = <&i2s0>; + rockchip,codec = <&vad>; + }; +}; + +&adc_key { + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <18000>; + }; +}; + +&display_subsystem { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; + power-supply = <&vcc5v0_sys>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <20>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&i2c3 { + status = "okay"; + + clock-frequency = <100000>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vdd1v5_dvp>; + dvdd-supply = <&vcc1v8_dvp>; + pwdn-gpios = <&gpio0 23 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0>; + port { + ucam_out: endpoint { + remote-endpoint = <&mipi_in_ucam>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s0 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&i2s1 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy { + status = "okay"; +}; + +&mipi_dphy_rx { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&rk_rga { + status = "okay"; +}; + +&rk809_sound { + status = "okay"; +}; + +&rkisp1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rng { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&route_dsi { + status = "disabled"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <0>; /* tshut polarity 0:LOW 1:HIGH */ + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&vad { + status = "okay"; + rockchip,audio-src = <&i2s0>; + rockchip,buffer-time-ms = <200>; + rockchip,det-channel = <0>; + rockchip,mode = <1>; + #sound-dai-cells = <0>; +}; + +&vop_lite { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&vpu_service { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk1808-evb-x4-second.dts b/arch/arm64/boot/dts/rockchip/rk1808-evb-x4-second.dts new file mode 100755 index 000000000000..413d4f6fa29d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk1808-evb-x4-second.dts @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include "rk1808-evb.dtsi" + +/ { + model = "Rockchip RK1808 EVB X4 Board"; + compatible = "rockchip,rk1808-evb-x4", "rockchip,rk1808"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff550000 console=ttyFIQ0 dump_initrd init=/init kpti=0"; + }; +}; + +&adc_key { + power-key { + linux,code = ; + label = "power key"; + press-threshold-microvolt = <18000>; + }; +}; + +/delete-node/ &backlight; +/delete-node/ &vcc1v8_dvp; +/delete-node/ &vdd1v5_dvp; +/delete-node/ &vcc2v8_dvp; + +&cif { + status = "okay"; + + port { + cif_in: endpoint@0 { + remote-endpoint = <&dphy_rx_out>; + data-lanes = <1 2 3 4>; + }; + }; +}; + +&cif_mmu { + status = "okay"; +}; + +&cru { + assigned-clocks = + <&cru PLL_GPLL>, <&cru PLL_CPLL>, + <&cru PLL_PPLL>, <&cru ARMCLK>, + <&cru MSCLK_PERI>, <&cru LSCLK_PERI>, + <&cru HSCLK_BUS_PRE>, <&cru MSCLK_BUS_PRE>, + <&cru LSCLK_BUS_PRE>, <&cru DCLK_VOPRAW>; + assigned-clock-rates = + <1188000000>, <1000000000>, + <100000000>, <816000000>, + <200000000>, <100000000>, + <300000000>, <200000000>, + <100000000>, <80000000>; +}; + +&csi_tx { + status = "okay"; + csi-tx-bypass-mode = <1>; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET | + MIPI_DSI_CLOCK_NON_CONTINUOUS)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing_1280x3_720>; + + timing_1280x3_720: timing-1280x3-720 { + clock-frequency = <80000000>; + hactive = <3840>; + vactive = <720>; + hfront-porch = <1200>; + hsync-len = <500>; + hback-porch = <30>; + vfront-porch = <40>; + vsync-len = <20>; + vback-porch = <40>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + timing_4k: timing-4k { + clock-frequency = <250000000>; + hactive = <3840>; + vactive = <2160>; + hfront-porch = <1500>; + hsync-len = <500>; + hback-porch = <30>; + vfront-porch = <40>; + vsync-len = <20>; + vback-porch = <40>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + timing_4096: timing-4096 { + clock-frequency = <100000000>; + hactive = <4096>; + vactive = <2048>; + hfront-porch = <1500>; + hsync-len = <500>; + hback-porch = <30>; + vfront-porch = <40>; + vsync-len = <20>; + vback-porch = <40>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + timing_1920x3_1080: timing-1920x3-1080 { + clock-frequency = <250000000>; + hactive = <5760>; + vactive = <1080>; + hfront-porch = <1500>; + hsync-len = <70>; + hback-porch = <30>; + vfront-porch = <40>; + vsync-len = <20>; + vback-porch = <40>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; +}; + +&display_subsystem { + status = "okay"; +}; + +&emmc { + status = "disabled"; +}; + +&gmac { + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + + vcamera@30 { + compatible = "rockchip,virtual-camera"; + reg = <0x30>; + width = <3840>; + height = <720>; + bus-format = ; + + port { + vcamera_out: endpoint { + remote-endpoint = <&dphy_rx_in>; + link-frequencies = /bits/ 64 <320000000>; + }; + }; + }; +}; + +&i2c1 { + status = "disabled"; +}; + +&i2c4 { + status = "disabled"; +}; + +&mipi_dphy { + status = "okay"; +}; + +&mipi_dphy_rx { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&vcamera_out>; + data-lanes = <1 2 3 4>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_in>; + }; + }; + }; +}; + +&rk809_codec { + status = "disabled"; +}; + +&rk_rga { + status = "okay"; +}; + +&route_csi { + status = "disabled"; +}; + +&sdmmc { + status = "disabled"; +}; + +&sdio { + status = "disabled"; +}; + +&sfc { + status = "okay"; +}; + +&uart4 { + status = "disabled"; +}; + +&wireless_bluetooth { + status = "disabled"; +}; + +&wireless_wlan { + status = "disabled"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <0>; /* tshut polarity 0:LOW 1:HIGH */ + pinctrl-names = "init", "default"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&vop_raw { + status = "okay"; +}; + +&vopr_mmu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk1808-evb-x4.dts b/arch/arm64/boot/dts/rockchip/rk1808-evb-x4.dts new file mode 100755 index 000000000000..17993d1ff44a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk1808-evb-x4.dts @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include "rk1808-evb.dtsi" + +/ { + model = "Rockchip RK1808 EVB X4 Board"; + compatible = "rockchip,rk1808-evb-x4", "rockchip,rk1808"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff550000 console=ttyFIQ0 dump_initrd init=/init kpti=0"; + }; +}; + +&adc_key { + power-key { + linux,code = ; + label = "power key"; + press-threshold-microvolt = <18000>; + }; +}; + +/delete-node/ &backlight; +/delete-node/ &vcc1v8_dvp; +/delete-node/ &vdd1v5_dvp; +/delete-node/ &vcc2v8_dvp; + +&cif { + status = "okay"; + + port { + cif_in: endpoint@0 { + remote-endpoint = <&dphy_rx_out>; + data-lanes = <1 2 3 4>; + }; + }; +}; + +&cif_mmu { + status = "okay"; +}; + +&cru { + assigned-clocks = + <&cru PLL_GPLL>, <&cru PLL_CPLL>, + <&cru PLL_PPLL>, <&cru ARMCLK>, + <&cru MSCLK_PERI>, <&cru LSCLK_PERI>, + <&cru HSCLK_BUS_PRE>, <&cru MSCLK_BUS_PRE>, + <&cru LSCLK_BUS_PRE>, <&cru DCLK_VOPRAW>; + assigned-clock-rates = + <1188000000>, <1000000000>, + <100000000>, <816000000>, + <200000000>, <100000000>, + <300000000>, <200000000>, + <100000000>, <80000000>; +}; + +&csi_tx { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET | + MIPI_DSI_CLOCK_NON_CONTINUOUS)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing_1280x3_720>; + + timing_1280x3_720: timing-1280x3-720 { + clock-frequency = <80000000>; + hactive = <3840>; + vactive = <720>; + hfront-porch = <1200>; + hsync-len = <500>; + hback-porch = <30>; + vfront-porch = <40>; + vsync-len = <20>; + vback-porch = <40>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + timing_4k: timing-4k { + clock-frequency = <250000000>; + hactive = <3840>; + vactive = <2160>; + hfront-porch = <1500>; + hsync-len = <500>; + hback-porch = <30>; + vfront-porch = <40>; + vsync-len = <20>; + vback-porch = <40>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + timing_4096: timing-4096 { + clock-frequency = <190000000>; + hactive = <4096>; + vactive = <2048>; + hfront-porch = <1500>; + hsync-len = <500>; + hback-porch = <30>; + vfront-porch = <40>; + vsync-len = <20>; + vback-porch = <40>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + timing_1920x3_1080: timing-1920x3-1080 { + clock-frequency = <250000000>; + hactive = <5760>; + vactive = <1080>; + hfront-porch = <1500>; + hsync-len = <70>; + hback-porch = <30>; + vfront-porch = <40>; + vsync-len = <20>; + vback-porch = <40>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; +}; + +&display_subsystem { + status = "okay"; +}; + +&emmc { + status = "disabled"; +}; + +&gmac { + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + + vcamera@30 { + compatible = "rockchip,virtual-camera"; + reg = <0x30>; + width = <1280>; + height = <720>; + bus-format = ; + + port { + vcamera_out: endpoint { + remote-endpoint = <&dphy_rx_in>; + link-frequencies = /bits/ 64 <320000000>; + }; + }; + }; +}; + +&i2c1 { + status = "disabled"; +}; + +&i2c4 { + status = "disabled"; +}; + +&mipi_dphy { + status = "okay"; +}; + +&mipi_dphy_rx { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&vcamera_out>; + data-lanes = <1 2 3 4>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_in>; + }; + }; + }; +}; + +&rk809_codec { + status = "disabled"; +}; + +&rk_rga { + status = "okay"; +}; + +&route_csi { + status = "disabled"; +}; + +&sdmmc { + status = "disabled"; +}; + +&sdio { + status = "disabled"; +}; + +&sfc { + status = "okay"; +}; + +&uart4 { + status = "disabled"; +}; + +&wireless_bluetooth { + status = "disabled"; +}; + +&wireless_wlan { + status = "disabled"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <0>; /* tshut polarity 0:LOW 1:HIGH */ + pinctrl-names = "init", "default"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&vop_raw { + status = "okay"; +}; + +&vopr_mmu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk1808-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk1808-evb.dtsi new file mode 100755 index 000000000000..3b9c88660701 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk1808-evb.dtsi @@ -0,0 +1,717 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd. + +#include +#include +#include +#include +#include +#include "rk1808.dtsi" + +/ { + model = "Rockchip RK1808 EVB"; + compatible = "rockchip,rk1808-evb", "rockchip,rk1808"; + + adc_key: adc-keys { + compatible = "adc-keys"; + autorepeat; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + display_subsystem: display-subsystem { + compatible = "rockchip,display-subsystem"; + ports = <&vop_lite_out>, <&vop_raw_out>; + logo-memory-region = <&drm_logo>; + status = "disabled"; + + route { + route_csi: route-csi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vop_raw_out_csi>; + }; + + route_dsi: route-dsi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vop_lite_out_dsi>; + }; + + route_rgb: route-rgb { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vop_lite_out_rgb>; + }; + }; + }; + + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <0>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + status = "okay"; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + drm_logo: drm-logo@00000000 { + compatible = "rockchip,drm-logo"; + reg = <0x0 0x0 0x0 0x0>; + }; + + ramoops: ramoops@110000 { + compatible = "ramoops"; + reg = <0x0 0x110000 0x0 0xf0000>; + record-size = <0x30000>; + console-size = <0xc0000>; + ftrace-size = <0x00000>; + pmsg-size = <0x00000>; + }; + }; + + rk809_sound: rk809-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio4 RK_PC0 GPIO_ACTIVE_LOW>; + }; + + vcc_otg_vbus: otg-vbus-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&otg_vbus_drv>; + regulator-name = "vcc_otg_vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + }; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + wireless_bluetooth: wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio4 RK_PB7 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart4_rts>; + pinctrl-1 = <&uart4_rts_gpio>; + BT,power_gpio = <&gpio4 RK_PC3 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless_wlan: wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_wake_host>; + wifi_chip_type = "ap6212"; + WIFI,host_wake_irq = <&gpio4 RK_PC1 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu1 { + cpu-supply = <&vdd_cpu>; +}; + +&combphy { + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + max-frequency = <200000000>; + mmc-hs200-1_8v; + no-sdio; + no-sd; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio0 10 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + assigned-clocks = <&cru SCLK_GMAC>; + assigned-clock-parents = <&gmac_clkin>; + tx_delay = <0x50>; + rx_delay = <0x3a>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <400000>; + + vdd_npu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel_gpio>; + vsel-gpios = <&gpio0 5 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_npu"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <950000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <0>; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_null>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <0>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc_buck5>; + vcc6-supply = <&vcc_buck5>; + vcc7-supply = <&vcc5v0_sys>; + vcc8-supply = <&vcc_3v3>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + rtc { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_log: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <950000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_log"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <800000>; + }; + }; + + vdd_cpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <950000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_cpu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vdda_0v8: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <800000>; + regulator-name = "vdda_0v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <800000>; + }; + }; + + vcc_1v8: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd_0v8: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <800000>; + + regulator-name = "vdd_0v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <800000>; + }; + }; + + vcca_1v8: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcca_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + + }; + }; + + vcc1v8_dvp: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG6 { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vccio_sd: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc3v3_sd: LDO_REG9 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc3v3_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_buck5: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2200000>; + regulator-max-microvolt = <2200000>; + regulator-name = "vcc_buck5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <2200000>; + }; + }; + + vcc5v0_host: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_host"; + }; + + vccio_3v3: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vccio_3v3"; + }; + + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_2CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +&i2c4 { + status = "okay"; + + sensor@d { + status = "okay"; + compatible = "ak8963"; + reg = <0x0d>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <1>; + reprobe_en = <1>; + }; + + sensor@4c { + status = "okay"; + compatible = "gs_mma7660"; + reg = <0x4c>; + type = ; + irq-gpio = <&gpio0 RK_PC6 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <2>; + reprobe_en = <1>; + }; +}; + +&npu { + npu-supply = <&vdd_npu>; + status = "okay"; +}; + +&pcie0 { + reset-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + /* Disable usbdrd_dwc3 and usbdrd3 if using pcie0 */ + status = "disabled"; +}; + +&power { + pd_npu-supply = <&vdd_npu>; +}; + +&pwm1 { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdio { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sd; + no-mmc; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <300>; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vmmc-supply = <&vcc3v3_sd>; + vqmmc-supply = <&vccio_sd>; + status = "okay"; +}; + +&uart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_xfer &uart4_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; +}; + +&u2phy_host { + status = "okay"; +}; + +&u2phy_otg { + status = "okay"; + vbus-supply = <&vcc_otg_vbus>; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usbdrd3 { + status = "okay"; + extcon = <&u2phy>; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + vsel_gpio: vsel-gpio { + rockchip,pins = + <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <4 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb2 { + otg_vbus_drv: otg-vbus-drv { + rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart4_rts_gpio: uart4-rts-gpio { + rockchip,pins = <4 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_wake_host: wifi-wake-host { + rockchip,pins = + <4 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk1808-fpga.dts b/arch/arm64/boot/dts/rockchip/rk1808-fpga.dts new file mode 100755 index 000000000000..d021918cacd1 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk1808-fpga.dts @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + +/dts-v1/; +#include "rk1808.dtsi" + +/ { + model = "Rockchip rk1808 fpga board"; + compatible = "rockchip,fpga", "rockchip,rk1808"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff550000 console=ttyFIQ0 root=/dev/mmcblk1p8 rootfstype=ext4 rootwait clk_ignore_unused"; + }; + + memory@200000 { + device_type = "memory"; + reg = <0x0 0x00200000 0x0 0x0FE00000>; + }; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + rockchip,irq-mode-enable = <1>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + status = "okay"; + }; +}; + +&emmc { + max-frequency = <400000>; + clocks = <&xin24m>, <&xin24m>, <&xin24m>, <&xin24m>; + clock-names = "biu", "ciu", "ciu-drv", "ciu-sample"; + mmc-hs200-1_8v; + no-sdio; + no-sd; + num-slots = <1>; + status = "okay"; +}; + +&npu { + status = "okay"; +}; + +&sdmmc { + max-frequency = <400000>; + clocks = <&xin24m>, <&xin24m>, <&xin24m>, <&xin24m>; + clock-names = "biu", "ciu", "ciu-drv", "ciu-sample"; + no-sdio; + no-mmc; + status = "okay"; +}; + +/* If fiq_debugger set okay, need to define uart2 and to be disabled */ +&uart2 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk1808.dtsi b/arch/arm64/boot/dts/rockchip/rk1808.dtsi new file mode 100755 index 000000000000..3469abac4f34 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk1808.dtsi @@ -0,0 +1,3040 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "rk1808-dram-default-timing.dtsi" + +/ { + compatible = "rockchip,rk1808"; + + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + serial4 = &uart4; + serial5 = &uart5; + serial6 = &uart6; + serial7 = &uart7; + spi0 = &spi0; + spi1 = &spi1; + spi2 = &spi2; + }; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a35", "arm,armv8"; + reg = <0x0 0x0>; + enable-method = "psci"; + clocks = <&cru ARMCLK>; + operating-points-v2 = <&cpu0_opp_table>; + dynamic-power-coefficient = <74>; + #cooling-cells = <2>; + cpu-idle-states = <&CPU_SLEEP>; + power-model { + compatible = "simple-power-model"; + ref-leakage = <31>; + static-coefficient = <100000>; + ts = <597400 241050 (-2450) 70>; + thermal-zone = "soc-thermal"; + }; + }; + + cpu1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a35", "arm,armv8"; + reg = <0x0 0x1>; + enable-method = "psci"; + clocks = <&cru ARMCLK>; + operating-points-v2 = <&cpu0_opp_table>; + dynamic-power-coefficient = <74>; + cpu-idle-states = <&CPU_SLEEP>; + }; + + idle-states { + entry-method = "psci"; + + CPU_SLEEP: cpu-sleep { + compatible = "arm,idle-state"; + local-timer-stop; + arm,psci-suspend-param = <0x0010000>; + entry-latency-us = <120>; + exit-latency-us = <250>; + min-residency-us = <900>; + }; + + CLUSTER_SLEEP: cluster-sleep { + compatible = "arm,idle-state"; + local-timer-stop; + arm,psci-suspend-param = <0x1010000>; + entry-latency-us = <400>; + exit-latency-us = <500>; + min-residency-us = <2000>; + }; + }; + }; + + cpu0_opp_table: cpu0-opp-table { + compatible = "operating-points-v2"; + opp-shared; + + rockchip,temp-hysteresis = <5000>; + rockchip,low-temp = <0>; + rockchip,low-temp-min-volt = <800000>; + rockchip,low-temp-adjust-volt = < + /* MHz MHz uV */ + 0 1608 50000 + >; + + rockchip,max-volt = <950000>; + rockchip,evb-irdrop = <25000>; + nvmem-cells = <&cpu_leakage>; + nvmem-cell-names = "leakage"; + + rockchip,pvtm-voltage-sel = < + 0 69000 0 + 69001 74000 1 + 74001 99999 2 + >; + rockchip,pvtm-freq = <408000>; + rockchip,pvtm-volt = <800000>; + rockchip,pvtm-ch = <0 0>; + rockchip,pvtm-sample-time = <1000>; + rockchip,pvtm-number = <10>; + rockchip,pvtm-error = <1000>; + rockchip,pvtm-ref-temp = <25>; + rockchip,pvtm-temp-prop = <(-20) (-26)>; + rockchip,thermal-zone = "soc-thermal"; + + opp-408000000 { + opp-hz = /bits/ 64 <408000000>; + opp-microvolt = <750000 750000 950000>; + clock-latency-ns = <40000>; + opp-suspend; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <750000 750000 950000>; + clock-latency-ns = <40000>; + }; + opp-816000000 { + opp-hz = /bits/ 64 <816000000>; + opp-microvolt = <750000 750000 950000>; + clock-latency-ns = <40000>; + }; + opp-1008000000 { + opp-hz = /bits/ 64 <1008000000>; + opp-microvolt = <750000 750000 950000>; + clock-latency-ns = <40000>; + }; + opp-1200000000 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <800000 800000 950000>; + opp-microvolt-L0 = <800000 800000 950000>; + opp-microvolt-L1 = <750000 750000 950000>; + opp-microvolt-L2 = <750000 750000 950000>; + clock-latency-ns = <40000>; + }; + opp-1296000000 { + opp-hz = /bits/ 64 <1296000000>; + opp-microvolt = <825000 825000 950000>; + opp-microvolt-L0 = <825000 825000 950000>; + opp-microvolt-L1 = <775000 775000 950000>; + opp-microvolt-L2 = <750000 750000 950000>; + clock-latency-ns = <40000>; + }; + opp-1416000000 { + opp-hz = /bits/ 64 <1416000000>; + opp-microvolt = <850000 850000 950000>; + opp-microvolt-L0 = <850000 850000 950000>; + opp-microvolt-L1 = <800000 800000 950000>; + opp-microvolt-L2 = <775000 775000 950000>; + clock-latency-ns = <40000>; + }; + opp-1512000000 { + opp-hz = /bits/ 64 <1512000000>; + opp-microvolt = <875000 875000 950000>; + opp-microvolt-L0 = <875000 875000 950000>; + opp-microvolt-L1 = <825000 825000 950000>; + opp-microvolt-L2 = <800000 800000 950000>; + clock-latency-ns = <40000>; + }; + opp-1608000000 { + opp-hz = /bits/ 64 <1608000000>; + opp-microvolt = <900000 900000 950000>; + opp-microvolt-L0 = <900000 900000 950000>; + opp-microvolt-L1 = <850000 850000 950000>; + opp-microvolt-L2 = <825000 825000 950000>; + clock-latency-ns = <40000>; + }; + }; + + arm-pmu { + compatible = "arm,cortex-a53-pmu"; + interrupts = , + ; + interrupt-affinity = <&cpu0>, <&cpu1>; + }; + + cpuinfo { + compatible = "rockchip,cpuinfo"; + nvmem-cells = <&efuse_id>, <&efuse_cpu_version>; + nvmem-cell-names = "id", "cpu-version"; + }; + + bus_soc: bus-soc { + compatible = "rockchip,rk1808-bus"; + rockchip,busfreq-policy = "smc"; + soc-bus0 { + bus-id = <0>; + cfg-val = <0x1e0>; + enable-msk = <0x407f>; + status = "okay"; + }; + soc-bus1 { + bus-id = <1>; + cfg-val = <0x12c0>; + enable-msk = <0x41ff>; + status = "okay"; + }; + soc-bus2 { + bus-id = <2>; + cfg-val = <0x12c0>; + enable-msk = <0x4005>; + status = "okay"; + }; + soc-bus3 { + bus-id = <3>; + cfg-val = <0x12c0>; + enable-msk = <0x4001>; + status = "okay"; + }; + soc-bus4 { + bus-id = <4>; + cfg-val = <0x12c0>; + enable-msk = <0x4001>; + status = "disabled"; + }; + }; + + firmware { + optee: optee { + compatible = "linaro,optee-tz"; + method = "smc"; + status = "disabled"; + }; + }; + + gmac_clkin: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "gmac_clkin"; + #clock-cells = <0>; + }; + + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + }; + + rockchip_suspend: rockchip-suspend { + compatible = "rockchip,pm-rk1808"; + status = "disabled"; + rockchip,sleep-debug-en = <0>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF + | RKPM_SLP_PMU_PMUALIVE_32K + | RKPM_SLP_PMU_DIS_OSC + | RKPM_SLP_PMIC_LP + | RKPM_SLP_32K_EXT + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + ) + >; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , + , + , + ; + arm,no-tick-in-suspend; + }; + + xin24m: xin24m { + compatible = "fixed-clock"; + clock-frequency = <24000000>; + clock-output-names = "xin24m"; + #clock-cells = <0>; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&clkin_32k>; + }; + + pcie0: pcie@fc400000 { + compatible = "rockchip,rk1808-pcie", "snps,dw-pcie"; + #address-cells = <3>; + #size-cells = <2>; + bus-range = <0x0 0x1f>; + clocks = <&cru HSCLK_PCIE>, <&cru LSCLK_PCIE>, + <&cru ACLK_PCIE>, <&cru PCLK_PCIE>, + <&cru SCLK_PCIE_AUX>; + clock-names = "hsclk", "lsclk", + "aclk", "pclk", + "sclk-aux"; + interrupts = , + , + , + ; + interrupt-names = "sys", "legacy", "msg", "err"; + linux,pci-domain = <0>; + num-ib-windows = <6>; + num-ob-windows = <2>; + msi-map = <0x0 &its 0x0 0x1000>; + num-lanes = <2>; + phys = <&combphy PHY_TYPE_PCIE>; + phy-names = "pcie-phy"; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_clkreq>; + power-domains = <&power RK1808_PD_PCIE>; + ranges = <0x00000800 0x0 0xf8000000 0x0 0xf8000000 0x0 0x800000 + 0x83000000 0x0 0xf8800000 0x0 0xf8800000 0x0 0x3700000 + 0x81000000 0x0 0xfbf00000 0x0 0xfbf00000 0x0 0x100000>; + reg = <0x0 0xfc000000 0x0 0x400000>, + <0x0 0xfc400000 0x0 0x10000>; + reg-names = "pcie-dbi", "pcie-apb"; + resets = <&cru SRST_PCIE_NIU_H>, <&cru SRST_PCIE_NIU_L>, + <&cru SRST_PCIEGRF_P>, <&cru SRST_PCIECTL_P>, + <&cru SRST_PCIECTL_POWERUP>, <&cru SRST_PCIECTL_MST_A>, + <&cru SRST_PCIECTL_SLV_A>, <&cru SRST_PCIECTL_DBI_A>, + <&cru SRST_PCIECTL_BUTTON>, <&cru SRST_PCIECTL_PE>, + <&cru SRST_PCIECTL_CORE>, <&cru SRST_PCIECTL_NSTICKY>, + <&cru SRST_PCIECTL_STICKY>, <&cru SRST_PCIECTL_PWR>, + <&cru SRST_PCIE_NIU_A>, <&cru SRST_PCIE_NIU_P>; + reset-names = "niu-h", "niu-l", "grf-p", "ctl-p", + "ctl-powerup", "ctl-mst-a", "ctl-slv-a", + "ctl-dbi-a", "ctl-button", "ctl-pe", + "ctl-core", "ctl-nsticky", "ctl-sticky", + "ctl-pwr", "ctl-niu-a", "ctl-niu-p"; + rockchip,usbpciegrf = <&usb_pcie_grf>; + rockchip,pmugrf = <&pmugrf>; + status = "disabled"; + }; + + usbdrd3: usb { + compatible = "rockchip,rk1808-dwc3", "rockchip,rk3399-dwc3"; + clocks = <&cru SCLK_USB3_OTG0_REF>, <&cru ACLK_USB3OTG>, + <&cru SCLK_USB3_OTG0_SUSPEND>; + clock-names = "ref_clk", "bus_clk", + "suspend_clk"; + assigned-clocks = <&cru SCLK_USB3_OTG0_SUSPEND>; + assigned-clock-rates = <24000000>; + power-domains = <&power RK1808_PD_PCIE>; + resets = <&cru SRST_USB3_OTG_A>; + reset-names = "usb3-otg"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + status = "disabled"; + + usbdrd_dwc3: dwc3@fd000000 { + compatible = "snps,dwc3"; + reg = <0x0 0xfd000000 0x0 0x200000>; + interrupts = ; + dr_mode = "otg"; + phys = <&u2phy_otg>, <&combphy PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + phy_type = "utmi_wide"; + snps,dis_enblslpm_quirk; + snps,dis-u1u2-quirk; + snps,dis-u2-freeclk-exists-quirk; + snps,dis_u2_susphy_quirk; + snps,dis_u3_susphy_quirk; + snps,dis-del-phy-power-chg-quirk; + snps,tx-ipgap-linecheck-dis-quirk; + snps,xhci-trb-ent-quirk; + status = "disabled"; + }; + }; + + grf: syscon@fe000000 { + compatible = "rockchip,rk1808-grf", "syscon", "simple-mfd"; + reg = <0x0 0xfe000000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + npu_pvtm: npu-pvtm { + compatible = "rockchip,rk1808-npu-pvtm"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + pvtm@2 { + reg = <2>; + clocks = <&cru SCLK_PVTM_NPU>; + clock-names = "clk"; + }; + }; + + rgb: rgb { + compatible = "rockchip,rk1808-rgb"; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + rgb_in_vop_lite: endpoint { + remote-endpoint = <&vop_lite_out_rgb>; + }; + }; + }; + }; + }; + + usb2phy_grf: syscon@fe010000 { + compatible = "rockchip,rk1808-usb2phy-grf", "syscon", + "simple-mfd"; + reg = <0x0 0xfe010000 0x0 0x8000>; + #address-cells = <1>; + #size-cells = <1>; + + u2phy: usb2-phy@100 { + compatible = "rockchip,rk1808-usb2phy"; + reg = <0x100 0x10>; + clocks = <&cru SCLK_USBPHY_REF>; + clock-names = "phyclk"; + #clock-cells = <0>; + assigned-clocks = <&cru USB480M>; + assigned-clock-parents = <&u2phy>; + clock-output-names = "usb480m_phy"; + status = "disabled"; + + u2phy_host: host-port { + #phy-cells = <0>; + interrupts = ; + interrupt-names = "linestate"; + status = "disabled"; + }; + + u2phy_otg: otg-port { + #phy-cells = <0>; + interrupts = , + , + ; + interrupt-names = "otg-bvalid", "otg-id", + "linestate"; + status = "disabled"; + }; + }; + }; + + combphy_grf: syscon@fe018000 { + compatible = "rockchip,usb3phy-grf", "syscon"; + reg = <0x0 0xfe018000 0x0 0x8000>; + }; + + pmugrf: syscon@fe020000 { + compatible = "rockchip,rk1808-pmugrf", "syscon", "simple-mfd"; + reg = <0x0 0xfe020000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + pmu_pvtm: pmu-pvtm { + compatible = "rockchip,rk1808-pmu-pvtm"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + pvtm@1 { + reg = <1>; + clocks = <&cru SCLK_PVTM_PMU>; + clock-names = "clk"; + }; + }; + + reboot-mode { + compatible = "syscon-reboot-mode"; + offset = <0x200>; + mode-bootloader = ; + mode-charge = ; + mode-fastboot = ; + mode-loader = ; + mode-normal = ; + mode-recovery = ; + mode-ums = ; + }; + }; + + usb_pcie_grf: syscon@fe040000 { + compatible = "rockchip,usb-pcie-grf", "syscon"; + reg = <0x0 0xfe040000 0x0 0x1000>; + }; + + coregrf: syscon@fe050000 { + compatible = "rockchip,rk1808-coregrf", "syscon", "simple-mfd"; + reg = <0x0 0xfe050000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + pvtm: pvtm { + compatible = "rockchip,rk1808-pvtm"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + pvtm@0 { + reg = <0>; + clocks = <&cru SCLK_PVTM_CORE>; + clock-names = "clk"; + }; + }; + }; + + qos_npu: qos@fe850000 { + compatible = "syscon"; + reg = <0x0 0xfe850000 0x0 0x20>; + }; + + qos_pcie: qos@fe880000 { + compatible = "syscon"; + reg = <0x0 0xfe880000 0x0 0x20>; + status = "disabled"; + }; + + qos_usb2: qos@fe890000 { + compatible = "syscon"; + reg = <0x0 0xfe890000 0x0 0x20>; + status = "disabled"; + }; + + qos_usb3: qos@fe890080 { + compatible = "syscon"; + reg = <0x0 0xfe890080 0x0 0x20>; + status = "disabled"; + }; + + qos_isp: qos@fe8a0000 { + compatible = "syscon"; + reg = <0x0 0xfe8a0000 0x0 0x20>; + }; + + qos_rga_rd: qos@fe8a0080 { + compatible = "syscon"; + reg = <0x0 0xfe8a0080 0x0 0x20>; + }; + + qos_rga_wr: qos@fe8a0100 { + compatible = "syscon"; + reg = <0x0 0xfe8a0100 0x0 0x20>; + }; + + qos_cif: qos@fe8a0180 { + compatible = "syscon"; + reg = <0x0 0xfe8a0180 0x0 0x20>; + }; + + qos_vop_raw: qos@fe8b0000 { + compatible = "syscon"; + reg = <0x0 0xfe8b0000 0x0 0x20>; + }; + + qos_vop_lite: qos@fe8b0080 { + compatible = "syscon"; + reg = <0x0 0xfe8b0080 0x0 0x20>; + }; + + qos_vpu: qos@fe8c0000 { + compatible = "syscon"; + reg = <0x0 0xfe8c0000 0x0 0x20>; + }; + + sram: sram@fec00000 { + compatible = "mmio-sram"; + reg = <0x0 0xfec00000 0x0 0x200000>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x0 0xfec00000 0x200000>; + /* reserved for ddr dvfs and system suspend/resume */ + ddr-sram@0 { + reg = <0x0 0x8000>; + }; + /* reserved for vad audio buffer */ + vad_sram: vad-sram@1c0000 { + reg = <0x1c0000 0x40000>; + }; + }; + + hwlock: hwspinlock@ff040000 { + compatible = "rockchip,hwspinlock"; + reg = <0 0xff040000 0 0x10000>; + #hwlock-cells = <1>; + }; + + gic: interrupt-controller@ff100000 { + compatible = "arm,gic-v3"; + #interrupt-cells = <3>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + interrupt-controller; + + reg = <0x0 0xff100000 0 0x10000>, /* GICD */ + <0x0 0xff140000 0 0xc0000>, /* GICR */ + <0x0 0xff300000 0 0x10000>, /* GICC */ + <0x0 0xff310000 0 0x10000>, /* GICH */ + <0x0 0xff320000 0 0x10000>; /* GICV */ + interrupts = ; + its: interrupt-controller@ff120000 { + compatible = "arm,gic-v3-its"; + msi-controller; + reg = <0x0 0xff120000 0x0 0x20000>; + }; + }; + + efuse: efuse@ff260000 { + compatible = "rockchip,rk1808-efuse"; + reg = <0x0 0xff3b0000 0x0 0x50>; + #address-cells = <1>; + #size-cells = <1>; + clocks = <&cru SCLK_EFUSE_NS>, <&cru PCLK_EFUSE>; + clock-names = "sclk_efuse", "pclk_efuse"; + assigned-clocks = <&cru SCLK_EFUSE_NS>; + assigned-clock-rates = <24000000>; + rockchip,efuse-size = <0x20>; + + /* Data cells */ + efuse_id: id@7 { + reg = <0x07 0x10>; + }; + cpu_leakage: cpu-leakage@17 { + reg = <0x17 0x1>; + }; + logic_leakage: logic-leakage@18 { + reg = <0x18 0x1>; + }; + npu_leakage: npu-leakage@19 { + reg = <0x19 0x1>; + }; + efuse_cpu_version: cpu-version@1c { + reg = <0x1c 0x1>; + bits = <3 3>; + }; + }; + + cru: clock-controller@ff350000 { + compatible = "rockchip,rk1808-cru"; + reg = <0x0 0xff350000 0x0 0x5000>; + rockchip,grf = <&grf>; + rockchip,pmugrf = <&pmugrf>; + #clock-cells = <1>; + #reset-cells = <1>; + + assigned-clocks = + <&cru SCLK_32K_IOE>, + <&cru PLL_GPLL>, <&cru PLL_CPLL>, + <&cru PLL_PPLL>, <&cru ARMCLK>, + <&cru MSCLK_PERI>, <&cru LSCLK_PERI>, + <&cru HSCLK_BUS_PRE>, <&cru MSCLK_BUS_PRE>, + <&cru LSCLK_BUS_PRE>; + assigned-clock-parents = <&xin32k>; + assigned-clock-rates = + <32768>, + <1188000000>, <1000000000>, + <100000000>, <816000000>, + <200000000>, <100000000>, + <300000000>, <200000000>, + <100000000>; + }; + + mipi_dphy_rx: mipi-dphy-rx@ff360000 { + compatible = "rockchip,rk1808-mipi-dphy-rx"; + reg = <0x0 0xff360000 0x0 0x4000>; + clocks = <&cru PCLK_MIPICSIPHY>; + clock-names = "pclk"; + power-domains = <&power RK1808_PD_VIO>; + rockchip,grf = <&grf>; + status = "disabled"; + }; + + mipi_dphy: mipi-dphy@ff370000 { + compatible = "rockchip,rk1808-mipi-dphy"; + reg = <0x0 0xff370000 0x0 0x500>; + clocks = <&cru SCLK_MIPIDSIPHY_REF>, <&cru PCLK_MIPIDSIPHY>; + clock-names = "ref", "pclk"; + clock-output-names = "mipi_dphy_pll"; + #clock-cells = <0>; + resets = <&cru SRST_MIPIDSIPHY_P>; + reset-names = "apb"; + #phy-cells = <0>; + rockchip,grf = <&grf>; + status = "disabled"; + }; + + combphy: phy@ff380000 { + compatible = "rockchip,rk1808-combphy"; + reg = <0x0 0xff380000 0x0 0x10000>; + #phy-cells = <1>; + clocks = <&cru SCLK_PCIEPHY_REF>; + clock-names = "refclk"; + assigned-clocks = <&cru SCLK_PCIEPHY_REF>; + assigned-clock-rates = <25000000>; + resets = <&cru SRST_USB3_OTG_A>, <&cru SRST_PCIEPHY_POR>, + <&cru SRST_PCIEPHY_P>, <&cru SRST_PCIEPHY_PIPE>, + <&cru SRST_USB3PHY_GRF_P>; + reset-names = "otg-rst", "combphy-por", + "combphy-apb", "combphy-pipe", + "usb3phy_grf_p"; + rockchip,combphygrf = <&combphy_grf>; + rockchip,usbpciegrf = <&usb_pcie_grf>; + status = "disabled"; + }; + + thermal_zones: thermal-zones { + soc_thermal: soc-thermal { + polling-delay-passive = <20>; /* milliseconds */ + polling-delay = <1000>; /* milliseconds */ + sustainable-power = <977>; /* milliwatts */ + + thermal-sensors = <&tsadc 0>; + + trips { + threshold: trip-point-0 { + /* millicelsius */ + temperature = <75000>; + /* millicelsius */ + hysteresis = <2000>; + type = "passive"; + }; + target: trip-point-1 { + /* millicelsius */ + temperature = <85000>; + /* millicelsius */ + hysteresis = <2000>; + type = "passive"; + }; + soc_crit: soc-crit { + /* millicelsius */ + temperature = <115000>; + /* millicelsius */ + hysteresis = <2000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&target>; + cooling-device = + <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <4096>; + }; + map1 { + trip = <&target>; + cooling-device = + <&npu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + }; + }; + }; + + tsadc: tsadc@ff3a0000 { + compatible = "rockchip,rk1808-tsadc"; + reg = <0x0 0xff3a0000 0x0 0x100>; + interrupts = ; + rockchip,grf = <&grf>; + clocks = <&cru SCLK_TSADC>, <&cru PCLK_TSADC>; + clock-names = "tsadc", "apb_pclk"; + assigned-clocks = <&cru SCLK_TSADC>; + assigned-clock-rates = <650000>; + resets = <&cru SRST_TSADC>; + reset-names = "tsadc-apb"; + #thermal-sensor-cells = <1>; + rockchip,hw-tshut-temp = <120000>; + status = "disabled"; + }; + + saradc: saradc@ff3c0000 { + compatible = "rockchip,rk1808-saradc", "rockchip,rk3399-saradc"; + reg = <0x0 0xff3c0000 0x0 0x100>; + interrupts = ; + #io-channel-cells = <1>; + clocks = <&cru SCLK_SARADC>, <&cru PCLK_SARADC>; + clock-names = "saradc", "apb_pclk"; + resets = <&cru SRST_SARADC_P>; + reset-names = "saradc-apb"; + status = "disabled"; + }; + + pwm0: pwm@ff3d0000 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff3d0000 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm1: pwm@ff3d0010 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff3d0010 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm2: pwm@ff3d0020 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff3d0020 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm3: pwm@ff3d0030 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff3d0030 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm3_pin>; + clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm4: pwm@ff3d8000 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff3d8000 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm4_pin>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm5: pwm@ff3d8010 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff3d8010 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm5_pin>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm6: pwm@ff3d8020 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff3d8020 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm6_pin>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm7: pwm@ff3d8030 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff3d8030 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm7_pin>; + clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pmu: power-management@ff3e0000 { + compatible = "rockchip,rk1808-pmu", "syscon", "simple-mfd"; + reg = <0x0 0xff3e0000 0x0 0x1000>; + + power: power-controller { + compatible = "rockchip,rk1808-power-controller"; + #power-domain-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + /* These power domains are grouped by VD_NPU */ + pd_npu@RK1808_VD_NPU { + reg = ; + clocks = <&cru SCLK_NPU>, + <&cru ACLK_NPU>, + <&cru HCLK_NPU>; + pm_qos = <&qos_npu>; + }; + + /* These power domains are grouped by VD_LOGIC */ + pd_pcie@RK1808_PD_PCIE { + reg = ; + clocks = <&cru HSCLK_PCIE>, + <&cru LSCLK_PCIE>, + <&cru ACLK_PCIE>, + <&cru ACLK_PCIE_MST>, + <&cru ACLK_PCIE_SLV>, + <&cru PCLK_PCIE>, + <&cru SCLK_PCIE_AUX>, + <&cru SCLK_PCIE_AUX>, + <&cru ACLK_USB3OTG>, + <&cru HCLK_HOST>, + <&cru HCLK_HOST_ARB>, + <&cru SCLK_USB3_OTG0_REF>, + <&cru SCLK_USB3_OTG0_SUSPEND>; + pm_qos = <&qos_pcie>, + <&qos_usb2>, + <&qos_usb3>; + }; + pd_vpu@RK1808_PD_VPU { + reg = ; + clocks = <&cru ACLK_VPU>, + <&cru HCLK_VPU>; + pm_qos = <&qos_vpu>; + }; + pd_vio@RK1808_PD_VIO { + reg = ; + clocks = <&cru HSCLK_VIO>, + <&cru LSCLK_VIO>, + <&cru ACLK_VOPRAW>, + <&cru HCLK_VOPRAW>, + <&cru ACLK_VOPLITE>, + <&cru HCLK_VOPLITE>, + <&cru PCLK_DSI_TX>, + <&cru PCLK_CSI_TX>, + <&cru ACLK_RGA>, + <&cru HCLK_RGA>, + <&cru ACLK_ISP>, + <&cru HCLK_ISP>, + <&cru ACLK_CIF>, + <&cru HCLK_CIF>, + <&cru PCLK_CSI2HOST>, + <&cru DCLK_VOPRAW>, + <&cru DCLK_VOPLITE>; + pm_qos = <&qos_rga_rd>, <&qos_rga_wr>, + <&qos_isp>, <&qos_cif>, + <&qos_vop_raw>, <&qos_vop_lite>; + }; + }; + }; + + i2c0: i2c@ff410000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xff410000 0x0 0x1000>; + clocks = <&cru SCLK_PMU_I2C0>, <&cru PCLK_I2C0_PMU>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + dmac: dmac@ff4e0000 { + compatible = "arm,pl330", "arm,primecell"; + reg = <0x0 0xff4e0000 0x0 0x4000>; + interrupts = ; + clocks = <&cru ACLK_DMAC>; + clock-names = "apb_pclk"; + #dma-cells = <1>; + arm,pl330-periph-burst; + }; + + uart0: serial@ff430000 { + compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff430000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART0_PMU>, <&cru PCLK_UART0_PMU>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac 0>, <&dmac 1>; + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts &uart0_rts>; + status = "disabled"; + }; + + i2c1: i2c@ff500000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xff500000 0x0 0x1000>; + clocks = <&cru SCLK_I2C1>, <&cru PCLK_I2C1>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c2: i2c@ff504000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xff504000 0x0 0x1000>; + clocks = <&cru SCLK_I2C2>, <&cru PCLK_I2C2>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2m0_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c3: i2c@ff508000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xff508000 0x0 0x1000>; + clocks = <&cru SCLK_I2C3>, <&cru PCLK_I2C3>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c4: i2c@ff50c000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xff50c000 0x0 0x1000>; + clocks = <&cru SCLK_I2C4>, <&cru PCLK_I2C4>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c4_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c5: i2c@ff510000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xff510000 0x0 0x1000>; + clocks = <&cru SCLK_I2C5>, <&cru PCLK_I2C5>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c5_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi0: spi@ff520000 { + compatible = "rockchip,rk1808-spi", "rockchip,rk3066-spi"; + reg = <0x0 0xff520000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&cru SCLK_SPI0>, <&cru PCLK_SPI0>; + clock-names = "spiclk", "apb_pclk"; + dmas = <&dmac 10>, <&dmac 11>; + pinctrl-names = "default", "high_speed"; + pinctrl-0 = <&spi0_clk &spi0_csn &spi0_miso &spi0_mosi>; + pinctrl-1 = <&spi0_clk_hs &spi0_csn &spi0_miso_hs &spi0_mosi_hs>; + status = "disabled"; + }; + + spi1: spi@ff530000 { + compatible = "rockchip,rk1808-spi", "rockchip,rk3066-spi"; + reg = <0x0 0xff530000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&cru SCLK_SPI1>, <&cru PCLK_SPI1>; + clock-names = "spiclk", "apb_pclk"; + dmas = <&dmac 12>, <&dmac 13>; + pinctrl-names = "default", "high_speed"; + pinctrl-0 = <&spi1_clk &spi1_csn0 &spi1_csn1 &spi1_miso &spi1_mosi>; + pinctrl-1 = <&spi1_clk_hs &spi1_csn0 &spi1_csn1 &spi1_miso_hs &spi1_mosi_hs>; + status = "disabled"; + }; + + uart1: serial@ff540000 { + compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff540000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART1>, <&cru PCLK_UART1>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac 2>, <&dmac 3>; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1_cts &uart1_rts>; + status = "disabled"; + }; + + uart2: serial@ff550000 { + compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff550000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART2>, <&cru PCLK_UART2>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac 4>, <&dmac 5>; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "disabled"; + }; + + uart3: serial@ff560000 { + compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff560000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART3>, <&cru PCLK_UART3>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac 6>, <&dmac 7>; + pinctrl-names = "default"; + pinctrl-0 = <&uart3m0_xfer &uart3_ctsm0 &uart3_rtsm0>; + status = "disabled"; + }; + + uart4: serial@ff570000 { + compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff570000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART4>, <&cru PCLK_UART4>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac 8>, <&dmac 9>; + pinctrl-names = "default"; + pinctrl-0 = <&uart4_xfer &uart4_cts &uart4_rts>; + status = "disabled"; + }; + + spi2: spi@ff580000 { + compatible = "rockchip,rk1808-spi", "rockchip,rk3066-spi"; + reg = <0x0 0xff580000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&cru SCLK_SPI2>, <&cru PCLK_SPI2>; + clock-names = "spiclk", "apb_pclk"; + dmas = <&dmac 14>, <&dmac 15>; + pinctrl-names = "default", "high_speed"; + pinctrl-0 = <&spi2m0_clk &spi2m0_csn &spi2m0_miso &spi2m0_mosi>; + pinctrl-1 = <&spi2m0_clk_hs &spi2m0_csn &spi2m0_miso_hs &spi2m0_mosi_hs>; + status = "disabled"; + }; + + uart5: serial@ff5a0000 { + compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff5a0000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART5>, <&cru PCLK_UART5>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac 25>, <&dmac 26>; + pinctrl-names = "default"; + pinctrl-0 = <&uart5_xfer>; + status = "disabled"; + }; + + uart6: serial@ff5b0000 { + compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff5b0000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART6>, <&cru PCLK_UART6>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac 27>, <&dmac 28>; + pinctrl-names = "default"; + pinctrl-0 = <&uart6_xfer>; + status = "disabled"; + }; + + uart7: serial@ff5c0000 { + compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff5c0000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART7>, <&cru PCLK_UART7>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac 29>, <&dmac 30>; + pinctrl-names = "default"; + pinctrl-0 = <&uart7_xfer>; + status = "disabled"; + }; + + pwm8: pwm@ff5d0000 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff5d0000 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm8_pin>; + clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm9: pwm@fff5d0010 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff5d0010 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm9_pin>; + clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm10: pwm@ff5d0020 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff5d0020 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm10_pin>; + clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm11: pwm@ff5d0030 { + compatible = "rockchip,rk1808-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xff5d0030 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm11_pin>; + clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + rng: rng@ff630000 { + compatible = "rockchip,cryptov2-rng"; + reg = <0x0 0xff630000 0x0 0x4000>; + clocks = <&cru SCLK_CRYPTO>, <&cru SCLK_CRYPTO_APK>, + <&cru ACLK_CRYPTO>, <&cru HCLK_CRYPTO>; + clock-names = "clk_crypto", "clk_crypto_apk", + "aclk_crypto", "hclk_crypto"; + resets = <&cru SRST_CRYPTO_CORE>; + reset-names = "reset"; + status = "disabled"; + }; + + dcf: dcf@ff640000 { + compatible = "syscon"; + reg = <0x0 0xff640000 0x0 0x1000>; + }; + + rktimer: rktimer@ff700000 { + compatible = "rockchip,rk3288-timer"; + reg = <0x0 0xff700000 0x0 0x1000>; + interrupts = ; + clocks = <&cru PCLK_TIMER>, <&cru SCLK_TIMER0>; + clock-names = "pclk", "timer"; + }; + + wdt: watchdog@ff720000 { + compatible = "snps,dw-wdt"; + reg = <0x0 0xff720000 0x0 0x100>; + clocks = <&cru PCLK_WDT>; + interrupts = ; + status = "okay"; + }; + + i2s0: i2s@ff7e0000 { + compatible = "rockchip,rk1808-i2s-tdm"; + reg = <0x0 0xff7e0000 0x0 0x1000>; + interrupts = ; + clocks = <&cru SCLK_I2S0_8CH_TX>, <&cru SCLK_I2S0_8CH_RX>, <&cru HCLK_I2S0_8CH>; + clock-names = "mclk_tx", "mclk_rx", "hclk"; + dmas = <&dmac 16>, <&dmac 17>; + dma-names = "tx", "rx"; + resets = <&cru SRST_I2S0_TX>, <&cru SRST_I2S0_RX>; + reset-names = "tx-m", "rx-m"; + rockchip,cru = <&cru>; + rockchip,grf = <&grf>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s0_8ch_sclktx + &i2s0_8ch_sclkrx + &i2s0_8ch_lrcktx + &i2s0_8ch_lrckrx + &i2s0_8ch_sdi0 + &i2s0_8ch_sdi1 + &i2s0_8ch_sdi2 + &i2s0_8ch_sdi3 + &i2s0_8ch_sdo0 + &i2s0_8ch_sdo1 + &i2s0_8ch_sdo2 + &i2s0_8ch_sdo3 + &i2s0_8ch_mclk>; + status = "disabled"; + }; + + i2s1: i2s@ff7f0000 { + compatible = "rockchip,rk1808-i2s", "rockchip,rk3066-i2s"; + reg = <0x0 0xff7f0000 0x0 0x1000>; + interrupts = ; + clocks = <&cru SCLK_I2S1_2CH>, <&cru HCLK_I2S1_2CH>; + clock-names = "i2s_clk", "i2s_hclk"; + dmas = <&dmac 18>, <&dmac 19>; + dma-names = "tx", "rx"; + resets = <&cru SRST_I2S1>, <&cru SRST_I2S1_H>; + reset-names = "reset-m", "reset-h"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_sclk + &i2s1_2ch_lrck + &i2s1_2ch_sdi + &i2s1_2ch_sdo>; + status = "disabled"; + }; + + pdm: pdm@ff800000 { + compatible = "rockchip,rk1808-pdm", "rockchip,pdm"; + reg = <0x0 0xff800000 0x0 0x1000>; + clocks = <&cru SCLK_PDM>, <&cru HCLK_PDM>; + clock-names = "pdm_clk", "pdm_hclk"; + dmas = <&dmac 24>; + dma-names = "rx"; + resets = <&cru SRST_PDM>; + reset-names = "pdm-m"; + pinctrl-names = "default"; + pinctrl-0 = <&pdm_clk + &pdm_clk1 + &pdm_sdi0 + &pdm_sdi1 + &pdm_sdi2 + &pdm_sdi3>; + status = "disabled"; + }; + + vad: vad@ff810000 { + compatible = "rockchip,rk1808-vad"; + reg = <0x0 0xff810000 0x0 0x10000>; + reg-names = "vad"; + clocks = <&cru HCLK_VAD>; + clock-names = "hclk"; + interrupts = ; + rockchip,audio-sram = <&vad_sram>; + rockchip,audio-src = <0>; + rockchip,det-channel = <0>; + rockchip,mode = <1>; + status = "disabled"; + }; + + dfi: dfi@ff9c0000 { + reg = <0x00 0xff9c0000 0x00 0x400>; + compatible = "rockchip,rk1808-dfi"; + rockchip,pmugrf = <&pmugrf>; + status = "disabled"; + }; + + dmc: dmc { + compatible = "rockchip,rk1808-dmc"; + dcf_reg = <&dcf>; + interrupts = ; + interrupt-names = "complete_irq"; + devfreq-events = <&dfi>; + clocks = <&cru SCLK_DDRCLK>; + clock-names = "dmc_clk"; + operating-points-v2 = <&dmc_opp_table>; + ddr_timing = <&ddr_timing>; + upthreshold = <40>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 924000 + SYS_STATUS_REBOOT 450000 + SYS_STATUS_SUSPEND 328000 + SYS_STATUS_VIDEO_1080P 924000 + SYS_STATUS_BOOST 924000 + SYS_STATUS_ISP 924000 + SYS_STATUS_PERFORMANCE 924000 + >; + auto-min-freq = <328000>; + auto-freq-en = <0>; + #cooling-cells = <2>; + status = "disabled"; + }; + + dmc_opp_table: dmc-opp-table { + compatible = "operating-points-v2"; + + rockchip,max-volt = <950000>; + rockchip,evb-irdrop = <12500>; + nvmem-cells = <&logic_leakage>; + nvmem-cell-names = "leakage"; + rockchip,temp-hysteresis = <5000>; + rockchip,low-temp = <0>; + rockchip,low-temp-min-volt = <800000>; + + opp-192000000 { + opp-hz = /bits/ 64 <192000000>; + opp-microvolt = <800000>; + }; + opp-324000000 { + opp-hz = /bits/ 64 <324000000>; + opp-microvolt = <800000>; + }; + opp-450000000 { + opp-hz = /bits/ 64 <450000000>; + opp-microvolt = <800000>; + }; + opp-528000000 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <800000>; + }; + opp-664000000 { + opp-hz = /bits/ 64 <664000000>; + opp-microvolt = <800000>; + }; + opp-784000000 { + opp-hz = /bits/ 64 <784000000>; + opp-microvolt = <800000>; + }; + opp-924000000 { + opp-hz = /bits/ 64 <924000000>; + opp-microvolt = <800000>; + }; + /* 1066M is only for ddr4 */ + opp-1066000000 { + opp-hz = /bits/ 64 <1066000000>; + opp-microvolt = <800000>; + status = "disabled"; + }; + }; + + rk_rga: rk_rga@ffaf0000 { + compatible = "rockchip,rga2"; + dev_mode = <0>; + reg = <0x0 0xffaf0000 0x0 0x1000>; + interrupts = ; + clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru SCLK_RGA>; + clock-names = "aclk_rga", "hclk_rga", "clk_rga"; + power-domains = <&power RK1808_PD_VIO>; + status = "disabled"; + }; + + cif: cif@ffae0000 { + compatible = "rockchip,rk1808-cif"; + reg = <0x0 0xffae0000 0x0 0x200>; + reg-names = "cif_regs"; + interrupts = ; + interrupt-names = "cif-intr"; + clocks = <&cru ACLK_CIF>, <&cru DCLK_CIF>, + <&cru HCLK_CIF>, <&cru SCLK_CIF_OUT>; + clock-names = "aclk_cif", "dclk_cif", + "hclk_cif", "sclk_cif_out"; + resets = <&cru SRST_CIF_A>, <&cru SRST_CIF_H>, + <&cru SRST_CIF_I>, <&cru SRST_CIF_D>, + <&cru SRST_CIF_PCLKIN>; + reset-names = "rst_cif_a", "rst_cif_h", + "rst_cif_i", "rst_cif_d", + "rst_cif_pclkin"; + power-domains = <&power RK1808_PD_VIO>; + iommus = <&cif_mmu>; + status = "disabled"; + }; + + cif_mmu: iommu@ffae0800 { + compatible = "rockchip,iommu"; + reg = <0x0 0xffae0800 0x0 0x100>; + interrupts = ; + interrupt-names = "cif_mmu"; + clocks = <&cru ACLK_CIF>, <&cru HCLK_CIF>; + clock-names = "aclk", "iface"; + power-domains = <&power RK1808_PD_VIO>; + #iommu-cells = <0>; + status = "disabled"; + }; + + vop_lite: vop@ffb00000 { + compatible = "rockchip,rk1808-vop-lit"; + reg = <0x0 0xffb00000 0x0 0x200>; + reg-names = "regs"; + interrupts = ; + clocks = <&cru ACLK_VOPLITE>, <&cru DCLK_VOPLITE>, + <&cru HCLK_VOPLITE>; + clock-names = "aclk_vop", "dclk_vop", "hclk_vop"; + power-domains = <&power RK1808_PD_VIO>; + iommus = <&vopl_mmu>; + status = "disabled"; + + vop_lite_out: port { + #address-cells = <1>; + #size-cells = <0>; + + vop_lite_out_dsi: endpoint@0 { + reg = <0>; + remote-endpoint = <&dsi_in_vop_lite>; + }; + + vop_lite_out_rgb: endpoint@1 { + reg = <1>; + remote-endpoint = <&rgb_in_vop_lite>; + }; + }; + }; + + vopl_mmu: iommu@ffb00f00 { + compatible = "rockchip,iommu"; + reg = <0x0 0xffb00f00 0x0 0x100>; + interrupts = ; + interrupt-names = "vopl_mmu"; + clocks = <&cru ACLK_VOPLITE>, <&cru HCLK_VOPLITE>; + clock-names = "aclk", "iface"; + power-domains = <&power RK1808_PD_VIO>; + #iommu-cells = <0>; + status = "disabled"; + }; + + mipi_csi2: mipi-csi2@ffb10000 { + compatible = "rockchip,rk1808-mipi-csi2"; + reg = <0x0 0xffb10000 0x0 0x100>; + reg-names = "csihost_regs"; + interrupts = , + ; + interrupt-names = "csi-intr1", "csi-intr2"; + clocks = <&cru PCLK_CSI2HOST>; + clock-names = "pclk_csi2host"; + status = "disabled"; + }; + + csi_tx: csi@ffb20000 { + compatible = "rockchip,rk1808-mipi-csi"; + reg = <0x0 0xffb20000 0x0 0x500>; + reg-names = "csi_regs"; + interrupts = ; + clocks = <&cru PCLK_CSI_TX>, <&mipi_dphy>; + clock-names = "pclk", "hs_clk"; + resets = <&cru SRST_CSITX_P>, + <&cru SRST_CSITX_TXBYTEHS>, + <&cru SRST_CSITX_TXESC>, + <&cru SRST_CSITX_CAM>, + <&cru SRST_CSITX_I>; + reset-names = "tx_apb", "tx_bytehs", "tx_esc", "tx_cam", "tx_i"; + phys = <&mipi_dphy>; + phy-names = "mipi_dphy"; + power-domains = <&power RK1808_PD_VIO>; + rockchip,grf = <&grf>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + ports { + + port { + csi_in_vop_raw: endpoint { + remote-endpoint = <&vop_raw_out_csi>; + }; + }; + }; + }; + + dsi: dsi@ffb30000 { + compatible = "rockchip,rk1808-mipi-dsi"; + reg = <0x0 0xffb30000 0x0 0x500>; + interrupts = ; + clocks = <&cru PCLK_DSI_TX>, <&mipi_dphy>; + clock-names = "pclk", "hs_clk"; + resets = <&cru SRST_MIPIDSI_HOST_P>; + reset-names = "apb"; + phys = <&mipi_dphy>; + phy-names = "mipi_dphy"; + power-domains = <&power RK1808_PD_VIO>; + rockchip,grf = <&grf>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + ports { + port { + dsi_in_vop_lite: endpoint { + remote-endpoint = <&vop_lite_out_dsi>; + }; + }; + }; + }; + + vop_raw: vop@ffb40000 { + compatible = "rockchip,rk1808-vop-raw"; + reg = <0x0 0xffb40000 0x0 0x500>; + reg-names = "regs"; + interrupts = ; + clocks = <&cru ACLK_VOPRAW>, <&cru DCLK_VOPRAW>, + <&cru HCLK_VOPRAW>; + clock-names = "aclk_vop", "dclk_vop", "hclk_vop"; + power-domains = <&power RK1808_PD_VIO>; + iommus = <&vopr_mmu>; + status = "disabled"; + + vop_raw_out: port { + #address-cells = <1>; + #size-cells = <0>; + + vop_raw_out_csi: endpoint@0 { + reg = <0>; + remote-endpoint = <&csi_in_vop_raw>; + }; + }; + }; + + vopr_mmu: iommu@ffb40f00 { + compatible = "rockchip,iommu"; + reg = <0x0 0xffb40f00 0x0 0x100>; + interrupts = ; + interrupt-names = "vopr_mmu"; + clocks = <&cru ACLK_VOPRAW>, <&cru HCLK_VOPRAW>; + clock-names = "aclk", "iface"; + power-domains = <&power RK1808_PD_VIO>; + #iommu-cells = <0>; + status = "disabled"; + }; + + rkisp1: rkisp1@ffb50000 { + compatible = "rockchip,rk1808-rkisp1"; + reg = <0x0 0xffb50000 0x0 0x8000>; + interrupts = , + , + ; + interrupt-names = "isp_irq", "mi_irq", "mipi_irq"; + clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>, + <&cru SCLK_ISP>, <&cru DCLK_CIF>; + clock-names = "aclk_isp", "hclk_isp", + "clk_isp", "pclk_isp"; + power-domains = <&power RK1808_PD_VIO>; + iommus = <&isp_mmu>; + rockchip,grf = <&grf>; + status = "disabled"; + }; + + isp_mmu: iommu@ffb58000 { + compatible = "rockchip,iommu"; + reg = <0x0 0xffb58000 0x0 0x100>; + interrupts = ; + interrupt-names = "isp_mmu"; + clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>, + <&cru SCLK_ISP>; + clock-names = "aclk", "iface", "sclk"; + power-domains = <&power RK1808_PD_VIO>; + rk_iommu,disable_reset_quirk; + #iommu-cells = <0>; + status = "disabled"; + }; + + vpu_service: vpu_service@ffb80000 { + compatible = "rockchip,vpu_service"; + reg = <0x0 0xffb80000 0x0 0x800>; + interrupts = , + ; + interrupt-names = "irq_enc", "irq_dec"; + clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; + clock-names = "aclk_vcodec", "hclk_vcodec"; + power-domains = <&power RK1808_PD_VPU>; + resets = <&cru SRST_VPU_A>, <&cru SRST_VPU_H>; + reset-names = "video_a", "video_h"; + iommus = <&vpu_mmu>; + iommu_enabled = <1>; + allocator = <1>; /* 0 means ion, 1 means drm */ + status = "disabled"; + }; + + vpu_mmu: iommu@ffb80800 { + compatible = "rockchip,iommu"; + reg = <0x0 0xffb80800 0x0 0x100>; + interrupts = ; + interrupt-names = "vpu_mmu"; + clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; + clock-names = "aclk", "iface"; + power-domains = <&power RK1808_PD_VPU>; + #iommu-cells = <0>; + status = "disabled"; + }; + + sdio: dwmmc@ffc60000 { + compatible = "rockchip,rk1808-dw-mshc", "rockchip,rk3288-dw-mshc"; + reg = <0x0 0xffc60000 0x0 0x4000>; + clocks = <&cru HCLK_SDIO>, <&cru SCLK_SDIO>, + <&cru SCLK_SDIO_DRV>, <&cru SCLK_SDIO_SAMPLE>; + clock-names = "biu", "ciu", "ciu-drv", "ciu-sample"; + max-frequency = <150000000>; + fifo-depth = <0x100>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_clk &sdmmc1_cmd &sdmmc1_bus4>; + status = "disabled"; + }; + + npu: npu@ffbc0000 { + compatible = "rockchip,npu"; + reg = <0x0 0xffbc0000 0x0 0x1000>; + clocks = <&cru SCLK_NPU>, <&cru ACLK_NPU>, <&cru HCLK_NPU>; + clock-names = "sclk_npu", "aclk_npu", "hclk_npu"; + assigned-clocks = <&cru SCLK_NPU>; + assigned-clock-rates = <800000000>; + interrupts = ; + power-domains = <&power RK1808_VD_NPU>; + operating-points-v2 = <&npu_opp_table>; + #cooling-cells = <2>; + status = "disabled"; + + npu_power_model: power-model { + compatible = "simple-power-model"; + ref-leakage = <31>; + static-coefficient = <100000>; + dynamic-coefficient = <3080>; + ts = <88610 303120 (-5000) 100>; + thermal-zone = "soc-thermal"; + }; + }; + + npu_opp_table: npu-opp-table { + compatible = "operating-points-v2"; + + rockchip,thermal-zone = "soc-thermal"; + rockchip,temp-hysteresis = <5000>; + rockchip,low-temp = <0>; + rockchip,low-temp-min-volt = <800000>; + rockchip,low-temp-adjust-volt = < + /* MHz MHz uV */ + 0 792 50000 + >; + + rockchip,max-volt = <880000>; + rockchip,evb-irdrop = <37500>; + nvmem-cells = <&npu_leakage>; + nvmem-cell-names = "leakage"; + + rockchip,pvtm-voltage-sel = < + 0 69000 0 + 69001 74000 1 + 74001 99999 2 + >; + rockchip,pvtm-ch = <0 0>; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <750000 750000 880000>; + }; + opp-297000000 { + opp-hz = /bits/ 64 <297000000>; + opp-microvolt = <750000 750000 880000>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <750000 750000 880000>; + }; + opp-594000000 { + opp-hz = /bits/ 64 <594000000>; + opp-microvolt = <750000 750000 880000>; + }; + opp-792000000 { + opp-hz = /bits/ 64 <792000000>; + opp-microvolt = <850000 850000 880000>; + opp-microvolt-L0 = <850000 850000 880000>; + opp-microvolt-L1 = <825000 825000 880000>; + opp-microvolt-L2 = <800000 800000 880000>; + }; + }; + + sfc: sfc@ffc50000 { + compatible = "rockchip,sfc"; + reg = <0x0 0xffc50000 0x0 0x4000>; + interrupts = ; + clocks = <&cru SCLK_SFC>, <&cru HCLK_SFC>; + clock-names = "clk_sfc", "hclk_sfc"; + assigned-clocks = <&cru SCLK_SFC>; + assigned-clock-rates = <100000000>; + status = "disabled"; + }; + + sdmmc: dwmmc@ffcf0000 { + compatible = "rockchip,rk1808-dw-mshc", "rockchip,rk3288-dw-mshc"; + reg = <0x0 0xffcf0000 0x0 0x4000>; + clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>, + <&cru SCLK_SDMMC_DRV>, <&cru SCLK_SDMMC_SAMPLE>; + clock-names = "biu", "ciu", "ciu-drv", "ciu-sample"; + max-frequency = <150000000>; + fifo-depth = <0x100>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_bus4 &sdmmc0_detn>; + status = "disabled"; + }; + + emmc: dwmmc@ffd00000 { + compatible = "rockchip,rk1808-dw-mshc", "rockchip,rk3288-dw-mshc"; + reg = <0x0 0xffd00000 0x0 0x4000>; + clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>, + <&cru SCLK_EMMC_DRV>, <&cru SCLK_EMMC_SAMPLE>; + clock-names = "biu", "ciu", "ciu-drv", "ciu-sample"; + max-frequency = <150000000>; + fifo-depth = <0x100>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; + status = "disabled"; + }; + + usb_host0_ehci: usb@ffd80000 { + compatible = "generic-ehci"; + reg = <0x0 0xffd80000 0x0 0x10000>; + interrupts = ; + clocks = <&cru HCLK_HOST>, <&cru HCLK_HOST_ARB>, + <&u2phy>; + clock-names = "usbhost", "arbiter", "utmi"; + phys = <&u2phy_host>; + phy-names = "usb"; + status = "disabled"; + power-domains = <&power RK1808_PD_PCIE>; + }; + + usb_host0_ohci: usb@ffd90000 { + compatible = "generic-ohci"; + reg = <0x0 0xffd90000 0x0 0x10000>; + interrupts = ; + clocks = <&cru HCLK_HOST>, <&cru HCLK_HOST_ARB>, + <&u2phy>; + clock-names = "usbhost", "arbiter", "utmi"; + phys = <&u2phy_host>; + phy-names = "usb"; + status = "disabled"; + power-domains = <&power RK1808_PD_PCIE>; + }; + + gmac: ethernet@ffdd0000 { + compatible = "rockchip,rk1808-gmac"; + reg = <0x0 0xffdd0000 0x0 0x10000>; + rockchip,grf = <&grf>; + interrupts = ; + interrupt-names = "macirq"; + clocks = <&cru SCLK_GMAC>, <&cru SCLK_GMAC_RX_TX>, + <&cru SCLK_GMAC_RX_TX>, <&cru SCLK_GMAC_REF>, + <&cru SCLK_GMAC_REFOUT>, <&cru ACLK_GMAC>, + <&cru PCLK_GMAC>, <&cru SCLK_GMAC_RGMII_SPEED>; + clock-names = "stmmaceth", "mac_clk_rx", + "mac_clk_tx", "clk_mac_ref", + "clk_mac_refout", "aclk_mac", + "pclk_mac", "clk_mac_speed"; + phy-mode = "rgmii"; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + resets = <&cru SRST_GAMC_A>; + reset-names = "stmmaceth"; + /* power-domains = <&power RK1808_PD_GMAC>; */ + status = "disabled"; + }; + + rockchip_system_monitor: rockchip-system-monitor { + compatible = "rockchip,system-monitor"; + + rockchip,thermal-zone = "soc-thermal"; + rockchip,polling-delay = <200>; /* milliseconds */ + }; + + pinctrl: pinctrl { + compatible = "rockchip,rk1808-pinctrl"; + rockchip,grf = <&grf>; + rockchip,pmu = <&pmugrf>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + gpio0: gpio0@ff4c0000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff4c0000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO0_PMU>, <&cru DBCLK_PMU_GPIO0>; + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio1: gpio1@ff690000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff690000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO1>, <&cru DBCLK_GPIO1>; + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio2: gpio2@ff6a0000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff6a0000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO2>, <&cru DBCLK_GPIO2>; + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio3: gpio3@ff6b0000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff6b0000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO3>, <&cru DBCLK_GPIO3>; + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio4: gpio4@ff6c0000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff6c0000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO4>, <&cru DBCLK_GPIO4>; + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + pcfg_pull_up: pcfg-pull-up { + bias-pull-up; + }; + + pcfg_pull_down: pcfg-pull-down { + bias-pull-down; + }; + + pcfg_pull_none: pcfg-pull-none { + bias-disable; + }; + + pcfg_pull_none_2ma: pcfg-pull-none-2ma { + bias-disable; + drive-strength = <2>; + }; + + pcfg_pull_up_2ma: pcfg-pull-up-2ma { + bias-pull-up; + drive-strength = <2>; + }; + + pcfg_pull_up_4ma: pcfg-pull-up-4ma { + bias-pull-up; + drive-strength = <4>; + }; + + pcfg_pull_none_4ma: pcfg-pull-none-4ma { + bias-disable; + drive-strength = <4>; + }; + + pcfg_pull_down_4ma: pcfg-pull-down-4ma { + bias-pull-down; + drive-strength = <4>; + }; + + pcfg_pull_none_8ma: pcfg-pull-none-8ma { + bias-disable; + drive-strength = <8>; + }; + + pcfg_pull_up_8ma: pcfg-pull-up-8ma { + bias-pull-up; + drive-strength = <8>; + }; + + pcfg_pull_none_12ma: pcfg-pull-none-12ma { + bias-disable; + drive-strength = <12>; + }; + + pcfg_pull_up_12ma: pcfg-pull-up-12ma { + bias-pull-up; + drive-strength = <12>; + }; + + pcfg_pull_none_smt: pcfg-pull-none-smt { + bias-disable; + input-schmitt-enable; + }; + + pcfg_pull_none_2ma_smt: pcfg-pull-none-2ma-smt { + bias-disable; + drive-strength = <2>; + input-schmitt-enable; + }; + + pcfg_output_high: pcfg-output-high { + output-high; + }; + + pcfg_output_low: pcfg-output-low { + output-low; + }; + + pcfg_input_high: pcfg-input-high { + bias-pull-up; + input-enable; + }; + + pcfg_input: pcfg-input { + input-enable; + }; + + pcfg_input_smt: pcfg-input-smt { + input-enable; + input-schmitt-enable; + }; + + cif-m0 { + cif_clkout_m0: cif-clkout-m0 { + rockchip,pins = + <2 RK_PB7 1 &pcfg_pull_none>; + }; + + cif_d12d15_m0: cif-d12d15-m0 { + rockchip,pins = + <2 RK_PA0 1 &pcfg_pull_none>,/* cif_d12 */ + <2 RK_PA1 1 &pcfg_pull_none>,/* cif_d13 */ + <2 RK_PA2 1 &pcfg_pull_none>,/* cif_d14 */ + <2 RK_PA3 1 &pcfg_pull_none>;/* cif_d15 */ + }; + + cif_d10d11_m0: cif-d10d11-m0 { + rockchip,pins = + <2 RK_PC2 1 &pcfg_pull_none>,/* cif_d10 */ + <2 RK_PC3 1 &pcfg_pull_none>;/* cif_d11 */ + }; + + cif_d2d9_m0: cif-d2d9-m0 { + rockchip,pins = + <2 RK_PA4 1 &pcfg_pull_none>,/* cif_d2 */ + <2 RK_PA5 1 &pcfg_pull_none>,/* cif_d3 */ + <2 RK_PA6 1 &pcfg_pull_none>,/* cif_d4 */ + <2 RK_PA7 1 &pcfg_pull_none>,/* cif_d5 */ + <2 RK_PB0 1 &pcfg_pull_none>,/* cif_d6 */ + <2 RK_PB1 1 &pcfg_pull_none>,/* cif_d7 */ + <2 RK_PB2 1 &pcfg_pull_none>,/* cif_d8 */ + <2 RK_PB3 1 &pcfg_pull_none>,/* cif_d9 */ + <2 RK_PB4 1 &pcfg_pull_none>,/* cif_vsync */ + <2 RK_PB5 1 &pcfg_pull_none>,/* cif_href */ + <2 RK_PB6 1 &pcfg_pull_none>;/* cif_clkin */ + }; + + cif_d0d1_m0: cif-d0d1-m0 { + rockchip,pins = + <2 RK_PC0 1 &pcfg_pull_none>,/* cif_d0 */ + <2 RK_PC1 1 &pcfg_pull_none>;/* cif_d1 */ + }; + }; + + emmc { + emmc_clk: emmc-clk { + rockchip,pins = + /* emmc_clkout */ + <1 RK_PB1 1 &pcfg_pull_up_4ma>; + }; + + emmc_rstnout: emmc-rstnout { + rockchip,pins = + /* emmc_rstn */ + <1 RK_PB3 1 &pcfg_pull_none>; + }; + + emmc_bus8: emmc-bus8 { + rockchip,pins = + /* emmc_d0 */ + <1 RK_PA0 1 &pcfg_pull_up_4ma>, + /* emmc_d1 */ + <1 RK_PA1 1 &pcfg_pull_up_4ma>, + /* emmc_d2 */ + <1 RK_PA2 1 &pcfg_pull_up_4ma>, + /* emmc_d3 */ + <1 RK_PA3 1 &pcfg_pull_up_4ma>, + /* emmc_d4 */ + <1 RK_PA4 1 &pcfg_pull_up_4ma>, + /* emmc_d5 */ + <1 RK_PA5 1 &pcfg_pull_up_4ma>, + /* emmc_d6 */ + <1 RK_PA6 1 &pcfg_pull_up_4ma>, + /* emmc_d7 */ + <1 RK_PA7 1 &pcfg_pull_up_4ma>; + }; + + emmc_pwren: emmc-pwren { + rockchip,pins = + <1 RK_PB0 1 &pcfg_pull_none>; + }; + + emmc_cmd: emmc-cmd { + rockchip,pins = + <1 RK_PB2 1 &pcfg_pull_up_4ma>; + }; + }; + + gmac { + rgmii_pins: rgmii-pins { + rockchip,pins = + /* rgmii_txen */ + <2 RK_PA1 2 &pcfg_pull_none_4ma>, + /* rgmii_txd1 */ + <2 RK_PA2 2 &pcfg_pull_none_4ma>, + /* rgmii_txd0 */ + <2 RK_PA3 2 &pcfg_pull_none_4ma>, + /* rgmii_rxd0 */ + <2 RK_PA4 2 &pcfg_pull_none>, + /* rgmii_rxd1 */ + <2 RK_PA5 2 &pcfg_pull_none>, + /* rgmii_rxdv */ + <2 RK_PA7 2 &pcfg_pull_none>, + /* rgmii_mdio */ + <2 RK_PB0 2 &pcfg_pull_none_2ma>, + /* rgmii_mdc */ + <2 RK_PB2 2 &pcfg_pull_none_2ma>, + /* rgmii_txd3 */ + <2 RK_PB3 2 &pcfg_pull_none_4ma>, + /* rgmii_txd2 */ + <2 RK_PB4 2 &pcfg_pull_none_4ma>, + /* rgmii_rxd2 */ + <2 RK_PB5 2 &pcfg_pull_none>, + /* rgmii_rxd3 */ + <2 RK_PB6 2 &pcfg_pull_none>, + /* rgmii_clk */ + <2 RK_PB7 2 &pcfg_pull_none>, + /* rgmii_txclk */ + <2 RK_PC1 2 &pcfg_pull_none_4ma>, + /* rgmii_rxclk */ + <2 RK_PC2 2 &pcfg_pull_none>; + }; + + rmii_pins: rmii-pins { + rockchip,pins = + /* rmii_txen */ + <2 RK_PA1 2 &pcfg_pull_none_4ma>, + /* rmii_txd1 */ + <2 RK_PA2 2 &pcfg_pull_none_4ma>, + /* rmii_txd0 */ + <2 RK_PA3 2 &pcfg_pull_none_4ma>, + /* rmii_rxd0 */ + <2 RK_PA4 2 &pcfg_pull_none>, + /* rmii_rxd1 */ + <2 RK_PA5 2 &pcfg_pull_none>, + /* rmii_rxer */ + <2 RK_PA6 2 &pcfg_pull_none>, + /* rmii_rxdv */ + <2 RK_PA7 2 &pcfg_pull_none>, + /* rmii_mdio */ + <2 RK_PB0 2 &pcfg_pull_none_2ma>, + /* rmii_mdc */ + <2 RK_PB2 2 &pcfg_pull_none_2ma>, + /* rmii_clk */ + <2 RK_PB7 2 &pcfg_pull_none>; + }; + }; + + i2c0 { + i2c0_xfer: i2c0-xfer { + rockchip,pins = + /* i2c0_sda */ + <0 RK_PB1 1 &pcfg_pull_none_2ma_smt>, + /* i2c0_scl */ + <0 RK_PB0 1 &pcfg_pull_none_2ma_smt>; + }; + }; + + i2c1 { + i2c1_xfer: i2c1-xfer { + rockchip,pins = + /* i2c1_sda */ + <0 RK_PC1 1 &pcfg_pull_none_2ma_smt>, + /* i2c1_scl */ + <0 RK_PC0 1 &pcfg_pull_none_2ma_smt>; + }; + }; + + i2c2m0 { + i2c2m0_xfer: i2c2m0-xfer { + rockchip,pins = + /* i2c2m0_sda */ + <3 RK_PB4 2 &pcfg_pull_none_2ma_smt>, + /* i2c2m0_scl */ + <3 RK_PB3 2 &pcfg_pull_none_2ma_smt>; + }; + }; + + i2c2m1 { + i2c2m1_xfer: i2c2m1-xfer { + rockchip,pins = + /* i2c2m1_sda */ + <1 RK_PB5 2 &pcfg_pull_none_2ma_smt>, + /* i2c2m1_scl */ + <1 RK_PB4 2 &pcfg_pull_none_2ma_smt>; + }; + }; + + i2c3 { + i2c3_xfer: i2c3-xfer { + rockchip,pins = + /* i2c3_sda */ + <2 RK_PD1 1 &pcfg_pull_none_2ma_smt>, + /* i2c3_scl */ + <2 RK_PD0 1 &pcfg_pull_none_2ma_smt>; + }; + }; + + i2c4 { + i2c4_xfer: i2c4-xfer { + rockchip,pins = + /* i2c4_sda */ + <3 RK_PC3 3 &pcfg_pull_none_2ma_smt>, + /* i2c4_scl */ + <3 RK_PC2 3 &pcfg_pull_none_2ma_smt>; + }; + }; + + i2c5 { + i2c5_xfer: i2c5-xfer { + rockchip,pins = + /* i2c5_sda */ + <4 RK_PC2 1 &pcfg_pull_none_2ma_smt>, + /* i2c5_scl */ + <4 RK_PC1 1 &pcfg_pull_none_2ma_smt>; + }; + }; + + i2s1 { + i2s1_2ch_lrck: i2s1-2ch-lrck { + rockchip,pins = + <3 RK_PA0 1 &pcfg_pull_none_2ma>; + }; + i2s1_2ch_sclk: i2s1-2ch-sclk { + rockchip,pins = + <3 RK_PA1 1 &pcfg_pull_none_2ma>; + }; + i2s1_2ch_mclk: i2s1-2ch-mclk { + rockchip,pins = + <3 RK_PA2 1 &pcfg_pull_none_2ma>; + }; + i2s1_2ch_sdo: i2s1-2ch-sdo { + rockchip,pins = + <3 RK_PA3 1 &pcfg_pull_none_2ma>; + }; + i2s1_2ch_sdi: i2s1-2ch-sdi { + rockchip,pins = + <3 RK_PA4 1 &pcfg_pull_none_2ma>; + }; + }; + + i2s0 { + i2s0_8ch_sdi3: i2s0-8ch-sdi3 { + rockchip,pins = + <3 RK_PA5 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_sdi2: i2s0-8ch-sdi2 { + rockchip,pins = + <3 RK_PA6 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_sdi1: i2s0-8ch-sdi1 { + rockchip,pins = + <3 RK_PA7 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_sclkrx: i2s0-8ch-sclkrx { + rockchip,pins = + <3 RK_PB0 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_lrckrx: i2s0-8ch-lrckrx { + rockchip,pins = + <3 RK_PB1 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_sdo3: i2s0-8ch-sdo3 { + rockchip,pins = + <3 RK_PB2 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_sdo2: i2s0-8ch-sdo2 { + rockchip,pins = + <3 RK_PB3 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_sdo1: i2s0-8ch-sdo1 { + rockchip,pins = + <3 RK_PB4 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_mclk: i2s0-8ch-mclk { + rockchip,pins = + <3 RK_PB5 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_lrcktx: i2s0-8ch-lrcktx { + rockchip,pins = + <3 RK_PB6 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_sclktx: i2s0-8ch-sclktx { + rockchip,pins = + <3 RK_PB7 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_sdo0: i2s0-8ch-sdo0 { + rockchip,pins = + <3 RK_PC0 1 &pcfg_pull_none_2ma>; + }; + i2s0_8ch_sdi0: i2s0-8ch-sdi0 { + rockchip,pins = + <3 RK_PC1 1 &pcfg_pull_none_2ma>; + }; + }; + + lcdc { + lcdc_rgb_dclk_pin: lcdc-rgb-dclk-pin { + rockchip,pins = + /* lcdc_clkm0 */ + <2 RK_PC6 3 &pcfg_pull_none>; + }; + + lcdc_rgb_den_pin: lcdc-rgb-den-pin { + rockchip,pins = + /* lcdc_denm0 */ + <2 RK_PC7 3 &pcfg_pull_none>; + }; + + lcdc_rgb_m0_hsync_pin: lcdc-rgb-m0-hsync-pin { + rockchip,pins = + /* lcdc_hsyncm0 */ + <2 RK_PB2 3 &pcfg_pull_none>; + }; + + lcdc_rgb_m0_vsync_pin: lcdc-rgb-m0-vsync-pin { + rockchip,pins = + /* lcdc_vsyncm0 */ + <2 RK_PB3 3 &pcfg_pull_none>; + }; + + lcdc_rgb_m1_hsync_pin: lcdc-rgb-m1-hsync-pin { + rockchip,pins = + /* lcdc_hsyncm1 */ + <3 RK_PB2 3 &pcfg_pull_none>; + }; + + lcdc_rgb_m1_vsync_pin: lcdc-rgb-m1-vsync-pin { + rockchip,pins = + /* lcdc_vsyncm1 */ + <3 RK_PB3 3 &pcfg_pull_none>; + }; + + lcdc_rgb666_data_pins: lcdc-rgb666-data-pins { + rockchip,pins = + /* lcdc_d0m0 */ + <2 RK_PA2 3 &pcfg_pull_none>, + /* lcdc_d1m0 */ + <2 RK_PA3 3 &pcfg_pull_none>, + /* lcdc_d2m0 */ + <2 RK_PC2 3 &pcfg_pull_none>, + /* lcdc_d3m0 */ + <2 RK_PC3 3 &pcfg_pull_none>, + /* lcdc_d4m0 */ + <2 RK_PC4 3 &pcfg_pull_none>, + /* lcdc_d5m0 */ + <2 RK_PC5 3 &pcfg_pull_none>, + /* lcdc_d6m0 */ + <2 RK_PA0 3 &pcfg_pull_none>, + /* lcdc_d7m0 */ + <2 RK_PA1 3 &pcfg_pull_none>, + /* lcdc_d8 */ + <3 RK_PC2 1 &pcfg_pull_none>, + /* lcdc_d9 */ + <3 RK_PC3 1 &pcfg_pull_none>, + /* lcdc_d10 */ + <3 RK_PC4 1 &pcfg_pull_none>, + /* lcdc_d11 */ + <3 RK_PC5 1 &pcfg_pull_none>, + /* lcdc_d12 */ + <3 RK_PC6 1 &pcfg_pull_none>, + /* lcdc_d13 */ + <3 RK_PC7 1 &pcfg_pull_none>, + /* lcdc_d14 */ + <3 RK_PD0 1 &pcfg_pull_none>, + /* lcdc_d15 */ + <3 RK_PD1 1 &pcfg_pull_none>, + /* lcdc_d16 */ + <3 RK_PD2 1 &pcfg_pull_none>, + /* lcdc_d17 */ + <3 RK_PD3 1 &pcfg_pull_none>; + }; + + lcdc_rgb565_data_pins: lcdc-rgb565-data-pins { + rockchip,pins = + /* lcdc_d0m0 */ + <2 RK_PA2 3 &pcfg_pull_none>, + /* lcdc_d1m0 */ + <2 RK_PA3 3 &pcfg_pull_none>, + /* lcdc_d2m0 */ + <2 RK_PC2 3 &pcfg_pull_none>, + /* lcdc_d3m0 */ + <2 RK_PC3 3 &pcfg_pull_none>, + /* lcdc_d4m0 */ + <2 RK_PC4 3 &pcfg_pull_none>, + /* lcdc_d5m0 */ + <2 RK_PC5 3 &pcfg_pull_none>, + /* lcdc_d6m0 */ + <2 RK_PA0 3 &pcfg_pull_none>, + /* lcdc_d7m0 */ + <2 RK_PA1 3 &pcfg_pull_none>, + /* lcdc_d8 */ + <3 RK_PC2 1 &pcfg_pull_none>, + /* lcdc_d9 */ + <3 RK_PC3 1 &pcfg_pull_none>, + /* lcdc_d10 */ + <3 RK_PC4 1 &pcfg_pull_none>, + /* lcdc_d11 */ + <3 RK_PC5 1 &pcfg_pull_none>, + /* lcdc_d12 */ + <3 RK_PC6 1 &pcfg_pull_none>, + /* lcdc_d13 */ + <3 RK_PC7 1 &pcfg_pull_none>, + /* lcdc_d14 */ + <3 RK_PD0 1 &pcfg_pull_none>, + /* lcdc_d15 */ + <3 RK_PD1 1 &pcfg_pull_none>; + }; + }; + + pciusb { + pciusb_pins: pciusb-pins { + rockchip,pins = + /* pciusb_debug0 */ + <4 RK_PB4 3 &pcfg_pull_none>, + /* pciusb_debug1 */ + <4 RK_PB5 3 &pcfg_pull_none>, + /* pciusb_debug2 */ + <4 RK_PB6 3 &pcfg_pull_none>, + /* pciusb_debug3 */ + <4 RK_PB7 3 &pcfg_pull_none>, + /* pciusb_debug4 */ + <4 RK_PC0 3 &pcfg_pull_none>, + /* pciusb_debug5 */ + <4 RK_PC1 3 &pcfg_pull_none>, + /* pciusb_debug6 */ + <4 RK_PC2 3 &pcfg_pull_none>, + /* pciusb_debug7 */ + <4 RK_PC3 3 &pcfg_pull_none>; + }; + + pcie_clkreq: pcie-clkreq { + rockchip,pins = + /* pcie_clkreqn_m1 */ + <0 RK_PC6 1 &pcfg_pull_none >; + }; + }; + + pdm { + pdm_clk: pdm-clk { + rockchip,pins = + /* pdm_clk0 */ + <3 RK_PB0 2 &pcfg_pull_none_2ma>; + }; + + pdm_sdi3: pdm-sdi3 { + rockchip,pins = + <3 RK_PA5 2 &pcfg_pull_none_2ma>; + }; + + pdm_sdi2: pdm-sdi2 { + rockchip,pins = + <3 RK_PA6 2 &pcfg_pull_none_2ma>; + }; + + pdm_sdi1: pdm-sdi1 { + rockchip,pins = + <3 RK_PA7 2 &pcfg_pull_none_2ma>; + }; + + pdm_clk1: pdm-clk1 { + rockchip,pins = + <3 RK_PB1 2 &pcfg_pull_none_2ma>; + }; + + pdm_sdi0: pdm-sdi0 { + rockchip,pins = + <3 RK_PC1 2 &pcfg_pull_none_2ma>; + }; + }; + + pwm0 { + pwm0_pin: pwm0-pin { + rockchip,pins = + <0 RK_PB7 1 &pcfg_pull_none_2ma>; + }; + }; + + pwm1 { + pwm1_pin: pwm1-pin { + rockchip,pins = + <0 RK_PC3 1 &pcfg_pull_none_2ma>; + }; + }; + + pwm2 { + pwm2_pin: pwm2-pin { + rockchip,pins = + <0 RK_PC5 1 &pcfg_pull_none_2ma>; + }; + }; + + pwm3 { + pwm3_pin: pwm3-pin { + rockchip,pins = + <0 RK_PC4 1 &pcfg_pull_none_2ma>; + }; + }; + + pwm4 { + pwm4_pin: pwm4-pin { + rockchip,pins = + <1 RK_PB6 2 &pcfg_pull_none_2ma>; + }; + }; + + pwm5 { + pwm5_pin: pwm5-pin { + rockchip,pins = + <1 RK_PB7 2 &pcfg_pull_none_2ma>; + }; + }; + pwm6 { + pwm6_pin: pwm6-pin { + rockchip,pins = + <3 RK_PA1 2 &pcfg_pull_none_2ma>; + }; + }; + + pwm7 { + pwm7_pin: pwm7-pin { + rockchip,pins = + <3 RK_PA2 2 &pcfg_pull_none_2ma>; + }; + }; + + pwm8 { + pwm8_pin: pwm8-pin { + rockchip,pins = + <3 RK_PD0 2 &pcfg_pull_none_2ma>; + }; + }; + + pwm9 { + pwm9_pin: pwm9-pin { + rockchip,pins = + <3 RK_PD1 2 &pcfg_pull_none_2ma>; + }; + }; + + pwm10 { + pwm10_pin: pwm10-pin { + rockchip,pins = + <3 RK_PD2 2 &pcfg_pull_none_2ma>; + }; + }; + + pwm11 { + pwm11_pin: pwm11-pin { + rockchip,pins = + <3 RK_PD3 2 &pcfg_pull_none_2ma>; + }; + }; + + sdmmc0 { + sdmmc0_bus4: sdmmc0-bus4 { + rockchip,pins = + /* sdmmc0_d0 */ + <4 RK_PA2 1 &pcfg_pull_up_8ma>, + /* sdmmc0_d1 */ + <4 RK_PA3 1 &pcfg_pull_up_8ma>, + /* sdmmc0_d2 */ + <4 RK_PA4 1 &pcfg_pull_up_8ma>, + /* sdmmc0_d3 */ + <4 RK_PA5 1 &pcfg_pull_up_8ma>; + }; + + sdmmc0_cmd: sdmmc0-cmd { + rockchip,pins = + <4 RK_PA0 1 &pcfg_pull_up_8ma>; + }; + + sdmmc0_clk: sdmmc0-clk { + rockchip,pins = + <4 RK_PA1 1 &pcfg_pull_up_8ma>; + }; + + sdmmc0_detn: sdmmc0-detn { + rockchip,pins = + <0 RK_PA3 1 &pcfg_pull_none>; + }; + }; + + sdmmc1 { + sdmmc1_bus4: sdmmc1-bus4 { + rockchip,pins = + /* sdmmc1_d0 */ + <4 RK_PB0 1 &pcfg_pull_up_4ma>, + /* sdmmc1_d1 */ + <4 RK_PB1 1 &pcfg_pull_up_4ma>, + /* sdmmc1_d2 */ + <4 RK_PB2 1 &pcfg_pull_up_4ma>, + /* sdmmc1_d3 */ + <4 RK_PB3 1 &pcfg_pull_up_4ma>; + }; + + sdmmc1_cmd: sdmmc1-cmd { + rockchip,pins = + <4 RK_PA6 1 &pcfg_pull_up_4ma>; + }; + + sdmmc1_clk: sdmmc1-clk { + rockchip,pins = + <4 RK_PA7 1 &pcfg_pull_up_4ma>; + }; + }; + + spi0 { + spi0_mosi: spi0-mosi { + rockchip,pins = + <1 RK_PB4 1 &pcfg_pull_up_2ma>; + }; + + spi0_miso: spi0-miso { + rockchip,pins = + <1 RK_PB5 1 &pcfg_pull_up_2ma>; + }; + + spi0_csn: spi0-csn { + rockchip,pins = + <1 RK_PB6 1 &pcfg_pull_up_2ma>; + }; + + spi0_clk: spi0-clk { + rockchip,pins = + <1 RK_PB7 1 &pcfg_pull_up_2ma>; + }; + + spi0_mosi_hs: spi0-mosi-hs { + rockchip,pins = + <1 RK_PB4 1 &pcfg_pull_up_2ma>; + }; + + spi0_miso_hs: spi0-miso-hs { + rockchip,pins = + <1 RK_PB5 1 &pcfg_pull_up_2ma>; + }; + + spi0_csn_hs: spi0-csn-hs { + rockchip,pins = + <1 RK_PB6 1 &pcfg_pull_up_2ma>; + }; + + spi0_clk_hs: spi0-clk-hs { + rockchip,pins = + <1 RK_PB7 1 &pcfg_pull_up_2ma>; + }; + }; + + spi1m0 { + spi1_clk: spi1-clk { + rockchip,pins = + <4 RK_PB4 2 &pcfg_pull_up_2ma>; + }; + + spi1_mosi: spi1-mosi { + rockchip,pins = + <4 RK_PB5 2 &pcfg_pull_up_2ma>; + }; + + spi1_csn0: spi1-csn0 { + rockchip,pins = + <4 RK_PB6 2 &pcfg_pull_up_2ma>; + }; + + spi1_miso: spi1-miso { + rockchip,pins = + <4 RK_PB7 2 &pcfg_pull_up_2ma>; + }; + + spi1_csn1: spi1-csn1 { + rockchip,pins = + <4 RK_PC0 2 &pcfg_pull_up_2ma>; + }; + + spi1_clk_hs: spi1-clk-hs { + rockchip,pins = + <4 RK_PB4 2 &pcfg_pull_up_2ma>; + }; + + spi1_mosi_hs: spi1-mosi-hs { + rockchip,pins = + <4 RK_PB5 2 &pcfg_pull_up_2ma>; + }; + + spi1_csn0_hs: spi1-csn0-hs { + rockchip,pins = + <4 RK_PB6 2 &pcfg_pull_up_2ma>; + }; + + spi1_miso_hs: spi1-miso-hs { + rockchip,pins = + <4 RK_PB7 2 &pcfg_pull_up_2ma>; + }; + + spi1_csn1_hs: spi1-csn1-hs { + rockchip,pins = + <4 RK_PC0 2 &pcfg_pull_up_2ma>; + }; + }; + + spi1m1 { + spi1m1_clk: spi1m1-clk { + rockchip,pins = + <3 RK_PC7 3 &pcfg_pull_up_2ma>; + }; + + spi1m1_mosi: spi1m1-mosi { + rockchip,pins = + <3 RK_PD0 3 &pcfg_pull_up_2ma>; + }; + + spi1m1_csn0: spi1m1-csn0 { + rockchip,pins = + <3 RK_PD1 3 &pcfg_pull_up_2ma>; + }; + + spi1m1_miso: spi1m1-miso { + rockchip,pins = + <3 RK_PD2 3 &pcfg_pull_up_2ma>; + }; + + spi1m1_csn1: spi1m1-csn1 { + rockchip,pins = + <3 RK_PD3 3 &pcfg_pull_up_2ma>; + }; + + spi1m1_clk_hs: spi1m1-clk-hs { + rockchip,pins = + <3 RK_PC7 3 &pcfg_pull_up_2ma>; + }; + + spi1m1_mosi_hs: spi1m1-mosi-hs { + rockchip,pins = + <3 RK_PD0 3 &pcfg_pull_up_2ma>; + }; + + spi1m1_csn0_hs: spi1m1-csn0-hs { + rockchip,pins = + <3 RK_PD1 3 &pcfg_pull_up_2ma>; + }; + + spi1m1_miso_hs: spi1m1-miso-hs { + rockchip,pins = + <3 RK_PD2 3 &pcfg_pull_up_2ma>; + }; + + spi1m1_csn1_hs: spi1m1-csn1-hs { + rockchip,pins = + <3 RK_PD3 3 &pcfg_pull_up_2ma>; + }; + }; + + spi2m0 { + spi2m0_miso: spi2m0-miso { + rockchip,pins = + <1 RK_PA6 2 &pcfg_pull_up_2ma>; + }; + + spi2m0_clk: spi2m0-clk { + rockchip,pins = + <1 RK_PA7 2 &pcfg_pull_up_2ma>; + }; + + spi2m0_mosi: spi2m0-mosi { + rockchip,pins = + <1 RK_PB0 2 &pcfg_pull_up_2ma>; + }; + + spi2m0_csn: spi2m0-csn { + rockchip,pins = + <1 RK_PB1 2 &pcfg_pull_up_2ma>; + }; + + spi2m0_miso_hs: spi2m0-miso-hs { + rockchip,pins = + <1 RK_PA6 2 &pcfg_pull_up_2ma>; + }; + + spi2m0_clk_hs: spi2m0-clk-hs { + rockchip,pins = + <1 RK_PA7 2 &pcfg_pull_up_2ma>; + }; + + spi2m0_mosi_hs: spi2m0-mosi-hs { + rockchip,pins = + <1 RK_PB0 2 &pcfg_pull_up_2ma>; + }; + + spi2m0_csn_hs: spi2m0-csn-hs { + rockchip,pins = + <1 RK_PB1 2 &pcfg_pull_up_2ma>; + }; + }; + + spi2m1 { + spi2m1_miso: spi2m1-miso { + rockchip,pins = + <2 RK_PA4 3 &pcfg_pull_up_2ma>; + }; + + spi2m1_clk: spi2m1-clk { + rockchip,pins = + <2 RK_PA5 3 &pcfg_pull_up_2ma>; + }; + + spi2m1_mosi: spi2m1-mosi { + rockchip,pins = + <2 RK_PA6 3 &pcfg_pull_up_2ma>; + }; + + spi2m1_csn: spi2m1-csn { + rockchip,pins = + <2 RK_PA7 3 &pcfg_pull_up_2ma>; + }; + + spi2m1_miso_hs: spi2m1-miso-hs { + rockchip,pins = + <2 RK_PA4 3 &pcfg_pull_up_2ma>; + }; + + spi2m1_clk_hs: spi2m1-clk-hs { + rockchip,pins = + <2 RK_PA5 3 &pcfg_pull_up_2ma>; + }; + + spi2m1_mosi_hs: spi2m1-mosi-hs { + rockchip,pins = + <2 RK_PA6 3 &pcfg_pull_up_2ma>; + }; + + spi2m1_csn_hs: spi2m1-csn-hs { + rockchip,pins = + <2 RK_PA7 3 &pcfg_pull_up_2ma>; + }; + }; + + uart0 { + uart0_xfer: uart0-xfer { + rockchip,pins = + /* uart0_rx */ + <0 RK_PB3 1 &pcfg_pull_up_2ma>, + /* uart0_tx */ + <0 RK_PB2 1 &pcfg_pull_up_2ma>; + }; + + uart0_cts: uart0-cts { + rockchip,pins = + <0 RK_PB4 1 &pcfg_pull_none>; + }; + + uart0_rts: uart0-rts { + rockchip,pins = + <0 RK_PB5 1 &pcfg_pull_none>; + }; + }; + + uart1 { + uart1m0_xfer: uart1m0-xfer { + rockchip,pins = + /* uart1_rxm0 */ + <4 RK_PB0 2 &pcfg_pull_up_2ma>, + /* uart1_txm0 */ + <4 RK_PB1 2 &pcfg_pull_up_2ma>; + }; + + uart1m1_xfer: uart1m1-xfer { + rockchip,pins = + /* uart1_rxm1 */ + <1 RK_PB4 3 &pcfg_pull_up_2ma>, + /* uart1_txm1 */ + <1 RK_PB5 3 &pcfg_pull_up_2ma>; + }; + + uart1_cts: uart1-cts { + rockchip,pins = + <4 RK_PB2 2 &pcfg_pull_none>; + }; + + uart1_rts: uart1-rts { + rockchip,pins = + <4 RK_PB3 2 &pcfg_pull_none>; + }; + }; + + uart2 { + uart2m0_xfer: uart2m0-xfer { + rockchip,pins = + /* uart2_rxm0 */ + <4 RK_PA3 2 &pcfg_pull_up_2ma>, + /* uart2_txm0 */ + <4 RK_PA2 2 &pcfg_pull_up_2ma>; + }; + + uart2m1_xfer: uart2m1-xfer { + rockchip,pins = + /* uart2_rxm1 */ + <2 RK_PD1 2 &pcfg_pull_up_2ma>, + /* uart2_txm1 */ + <2 RK_PD0 2 &pcfg_pull_up_2ma>; + }; + + uart2m2_xfer: uart2m2-xfer { + rockchip,pins = + /* uart2_rxm2 */ + <3 RK_PA4 2 &pcfg_pull_up_2ma>, + /* uart2_txm2 */ + <3 RK_PA3 2 &pcfg_pull_up_2ma>; + }; + }; + + uart3 { + uart3m0_xfer: uart3m0-xfer { + rockchip,pins = + /* uart3_rxm0 */ + <0 RK_PC4 2 &pcfg_pull_up_2ma>, + /* uart3_txm0 */ + <0 RK_PC3 2 &pcfg_pull_up_2ma>; + }; + + uart3_ctsm0: uart3-ctsm0 { + rockchip,pins = + <0 RK_PC6 2 &pcfg_pull_none>; + }; + + uart3_rtsm0: uart3-rtsm0 { + rockchip,pins = + <0 RK_PC7 2 &pcfg_pull_none>; + }; + }; + + uart4 { + uart4_xfer: uart4-xfer { + rockchip,pins = + /* uart4_rx */ + <4 RK_PB4 1 &pcfg_pull_up_2ma>, + /* uart4_tx */ + <4 RK_PB5 1 &pcfg_pull_up_2ma>; + }; + + uart4_cts: uart4-cts { + rockchip,pins = + <4 RK_PB6 1 &pcfg_pull_none>; + }; + + uart4_rts: uart4-rts { + rockchip,pins = + <4 RK_PB7 1 &pcfg_pull_none>; + }; + }; + + uart5 { + uart5_xfer: uart5-xfer { + rockchip,pins = + /* uart5_rx */ + <3 RK_PC3 2 &pcfg_pull_up_2ma>, + /* uart5_tx */ + <3 RK_PC2 2 &pcfg_pull_up_2ma>; + }; + }; + + uart6 { + uart6_xfer: uart6-xfer { + rockchip,pins = + /* uart6_rx */ + <3 RK_PC5 2 &pcfg_pull_up_2ma>, + /* uart6_tx */ + <3 RK_PC4 2 &pcfg_pull_up_2ma>; + }; + }; + + uart7 { + uart7_xfer: uart7-xfer { + rockchip,pins = + /* uart7_rx */ + <3 RK_PC7 2 &pcfg_pull_up_2ma>, + /* uart7_tx */ + <3 RK_PC6 2 &pcfg_pull_up_2ma>; + }; + }; + + tsadc { + tsadc_otp_gpio: tsadc-otp-gpio { + rockchip,pins = + <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + tsadc_otp_out: tsadc-otp-out { + rockchip,pins = + <0 RK_PA6 2 &pcfg_pull_none>; + }; + }; + + xin32k { + clkin_32k: clkin-32k { + rockchip,pins = + <0 RK_PC2 1 &pcfg_pull_none>; + }; + + clkout_32k: clkout-32k { + rockchip,pins = + <0 RK_PC2 1 &pcfg_pull_none>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk1808k.dtsi b/arch/arm64/boot/dts/rockchip/rk1808k.dtsi new file mode 100755 index 000000000000..78bd92e4c2c6 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk1808k.dtsi @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +&cpu0_opp_table { + rockchip,high-temp = <85000>; + rockchip,high-temp-max-freq = <1008000>; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_log>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 924000 + SYS_STATUS_REBOOT 924000 + >; +}; + +&dmc_opp_table { + rockchip,high-temp = <85000>; + rockchip,high-temp-max-freq = <664000>; + rockchip,thermal-zone = "soc-thermal"; +}; + +&thermal_zones { + soc-thermal { + sustainable-power = <1224>; + k_pu = <27>; + k_po = <55>; + k_i = <0>; + + trips { + trip-point-0 { + temperature = <85000>; + }; + trip-point-1 { + temperature = <100000>; + }; + }; + /delete-node/ cooling-maps; + cooling-maps { + map0 { + trip = <&target>; + cooling-device = + <&npu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-ai-va-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308-ai-va-v10.dts new file mode 100755 index 000000000000..c77d2e963266 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-ai-va-v10.dts @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; + +#include +#include "rk3308.dtsi" + +/ { + model = "Rockchip RK3308 voice assistant v10 board"; + compatible = "rockchip,rk3308-ai-va-v10", "rockchip,rk3308"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff0c0000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootfstype=squashfs rootwait snd_aloop.index=7 snd_aloop.use_raw_jiffies=1"; + }; + + adc-keys0 { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + func-key { + linux,code = ; + label = "function"; + press-threshold-microvolt = <17000>; + }; + }; + + adc-keys1 { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + play-key { + linux,code = ; + label = "play"; + press-threshold-microvolt = <625000>; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + + pinctrl-names = "default"; + pinctrl-0 = <&mic_mute>; + + mute { + gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "GPIO Mic Mute"; + debounce-interval = <100>; + }; + }; + + rotary { + compatible = "rotary-encoder"; + pinctrl-names = "default"; + pinctrl-0 = <&rotary_gpio>; + gpios = <&gpio2 RK_PB3 GPIO_ACTIVE_LOW>, + <&gpio2 RK_PB4 GPIO_ACTIVE_LOW>; + linux,axis = <0>; /* REL_X */ + rotary-encoder,relative-axis; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; + }; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "i2s_8ch_0"; + + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&i2s_8ch_0>; + }; + + codec { + sound-dai = <&dummy_codec>; + }; + }; + + simple-audio-card,dai-link@1 { + format = "i2s"; + cpu { + sound-dai = <&i2s_8ch_0>; + }; + + codec { + sound-dai = <&tas5711>; + }; + }; + }; + + dummy_codec: dummy-codec { + compatible = "rockchip,dummy-codec"; + #sound-dai-cells = <0>; + }; + + vdd_log: vdd_core: vdd-core { + compatible = "pwm-regulator"; + pwms = <&pwm0 0 5000 1>; + regulator-name = "vdd_core"; + regulator-min-microvolt = <827000>; + regulator-max-microvolt = <1340000>; + regulator-init-microvolt = <1015000>; + regulator-always-on; + regulator-boot-on; + regulator-settling-time-up-us = <250>; + status = "okay"; + }; + + vdd_1v0: vdd-1v0 { + compatible = "regulator-fixed"; + regulator-name = "vdd_1v0"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + }; + + vccio_sdio: vcc_1v8: vcc-1v8 { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc_io>; + }; + + vcc_1v8_codec: vcc-1v8-codec { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8_codec"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc_io>; + }; + + vcc_ddr: vcc-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + }; + + vcc_3v3_codec: vcc_io: vcc-io { + compatible = "regulator-fixed"; + regulator-name = "vcc_io"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vccio_flash: vccio-flash { + compatible = "regulator-fixed"; + regulator-name = "vccio_flash"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart4_rts>; + pinctrl-1 = <&uart4_rts_gpio>; + BT,power_gpio = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6255"; + WIFI,host_wake_irq = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cpu0 { + cpu-supply = <&vdd_core>; +}; + +&dmc { + center-supply = <&vdd_core>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&fiq_debugger { + status = "okay"; +}; + +&io_domains { + status = "okay"; + + vccio0-supply = <&vcc_io>; + vccio1-supply = <&vcc_io>; + vccio2-supply = <&vcc_1v8>; + vccio3-supply = <&vccio_flash>; + vccio4-supply = <&vccio_sdio>; + vccio5-supply = <&vcc_io>; +}; + +&i2c1 { + clock-frequency = <400000>; + status = "okay"; + + is31fl3236: led-controller@3c { + compatible = "issi,is31fl3236"; + reg = <0x3c>; + #address-cells = <1>; + #size-cells = <0>; + reset-gpios = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; + status = "okay"; + + led1: led@1 { + label = "led1"; + reg = <1>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led2: led@2 { + label = "led2"; + reg = <2>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led3: led@3 { + label = "led3"; + reg = <3>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led4: led@4 { + label = "led4"; + reg = <4>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led5: led@5 { + label = "led5"; + reg = <5>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led6: led@6 { + label = "led6"; + reg = <6>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led7: led@7 { + label = "led7"; + reg = <7>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led8: led@8 { + label = "led8"; + reg = <8>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led9: led@9 { + label = "led9"; + reg = <9>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led10: led@10 { + label = "led10"; + reg = <10>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led11: led@11 { + label = "led11"; + reg = <11>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led12: led@12 { + label = "led12"; + reg = <12>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led13: led@13 { + label = "led13"; + reg = <13>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led14: led@14 { + label = "led14"; + reg = <14>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led15: led@15 { + label = "led15"; + reg = <15>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led16: led@16 { + label = "led16"; + reg = <16>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led17: led@17 { + label = "led17"; + reg = <17>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led18: led@18 { + label = "led18"; + reg = <18>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led19: led@19 { + label = "led19"; + reg = <19>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led20: led@20 { + label = "led20"; + reg = <20>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led21: led@21 { + label = "led21"; + reg = <21>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led22: led@22 { + label = "led22"; + reg = <22>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led23: led@23 { + label = "led23"; + reg = <23>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led124: led@24 { + label = "led24"; + reg = <24>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led25: led@25 { + label = "led25"; + reg = <25>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led26: led@26 { + label = "led26"; + reg = <26>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led27: led@27 { + label = "led27"; + reg = <27>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led28: led@28 { + label = "led28"; + reg = <28>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led29: led@29 { + label = "led29"; + reg = <29>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led30: led@30 { + label = "led30"; + reg = <30>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led31: led@31 { + label = "led31"; + reg = <31>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led32: led@32 { + label = "led32"; + reg = <32>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led33: led@33 { + label = "led33"; + reg = <33>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led34: led@34 { + label = "led34"; + reg = <34>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led35: led@35 { + label = "led35"; + reg = <35>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led36: led@36 { + label = "led36"; + reg = <36>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + }; + + tas5711: tas5711@1b { + #sound-dai-cells = <0>; + compatible = "ti,tas5711"; + reg = <0x1b>; + clocks = <&cru SCLK_I2S0_8CH_TX_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_0_mclk>; + pdn-gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; + reset-gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_LOW>; + }; +}; + +&i2s_8ch_0 { + status = "okay"; + assigned-clocks = <&cru SCLK_I2S0_8CH_RX>; + assigned-clock-parents = <&cru SCLK_I2S0_8CH_TX_MUX>; + rockchip,clk-trcm = <1>; + #sound-dai-cells = <0>; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&rtc_32k>; + + buttons { + mic_mute: mic-mute { + rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + rotary { + rotary_gpio: rotary-gpio { + rockchip,pins = + <2 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pwm0 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin_pull_down>; +}; + +&rockchip_suspend { + rockchip,pwm-regulator-config = < + (0 + | RKPM_PWM_REGULATOR + ) + >; + + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <0>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&uart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_xfer &uart4_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v10.dts new file mode 100755 index 000000000000..557daa856132 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v10.dts @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; + +#include "rk3308-evb-v10.dtsi" + +/ { + model = "Rockchip RK3308 evb analog mic board"; + compatible = "rockchip,rk3308-evb-amic-v10", "rockchip,rk3308"; + + vad_acodec_sound: vad-acodec-sound { + status = "okay"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-vad"; + rockchip,codec-hp-det; + rockchip,mclk-fs = <256>; + rockchip,cpu = <&i2s_8ch_2>; + rockchip,codec = <&acodec>, <&vad>; + }; +}; + +&acodec { + rockchip,micbias1; + rockchip,micbias2; + rockchip,en-always-grps = <0 1 2>; +}; + +&acodec_sound { + status = "disabled"; +}; + +&bluetooth_sound { + status = "okay"; +}; + +&i2s_2ch_0 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&is31fl3236 { + reg = <0x3f>; +}; + +&vad { + status = "okay"; + rockchip,audio-src = <&i2s_8ch_2>; + rockchip,buffer-time-ms = <200>; + rockchip,mode = <1>; + #sound-dai-cells = <0>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v11.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v11.dts new file mode 100755 index 000000000000..90a29a903545 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-amic-v11.dts @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; + +#include "rk3308-evb-v11.dtsi" + +/ { + model = "Rockchip RK3308 evb analog mic v11 board"; + compatible = "rockchip,rk3308-evb-amic-v11", "rockchip,rk3308"; + + vad_acodec_sound: vad-acodec-sound { + status = "okay"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-vad"; + rockchip,codec-hp-det; + rockchip,mclk-fs = <256>; + rockchip,cpu = <&i2s_8ch_2>; + rockchip,codec = <&acodec>, <&vad>; + }; +}; + +&acodec { + rockchip,micbias1; + rockchip,micbias2; + rockchip,en-always-grps = <1 2 3>; + rockchip,adc-grps-route = <1 2 3 0>; +}; + +&acodec_sound { + status = "disabled"; +}; + +&bluetooth_sound { + status = "okay"; +}; + +&i2s_2ch_0 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&is31fl3236 { + reg = <0x3f>; +}; + +&vad { + status = "okay"; + rockchip,audio-src = <&i2s_8ch_2>; + rockchip,det-channel = <0>; + rockchip,buffer-time-ms = <200>; + rockchip,mode = <1>; + #sound-dai-cells = <0>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v10.dts new file mode 100755 index 000000000000..88c1e9c6c8ac --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v10.dts @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; + +#include "rk3308-evb-v10.dtsi" + +/ { + model = "Rockchip RK3308 evb digital-i2s mic board"; + compatible = "rockchip,rk3308-evb-dmic-i2s-v10", "rockchip,rk3308"; + + i2s_16ch_dais: i2s-16ch-dais { + status = "disabled"; + compatible = "rockchip,rk3308-multi-dais", "rockchip,multi-dais"; + dais = <&i2s_8ch_0>, <&i2s_8ch_1>; + capture,channel-mapping = <8 8>; + playback,channel-mapping = <0 0>; + bitclock-master = <1 0>; + frame-master = <1 0>; + rockchip,grf = <&grf>; + }; + + i2s_8ch_0_2_dais: i2s-8ch-0-2-dais { + status = "okay"; + compatible = "rockchip,rk3308-multi-dais", "rockchip,multi-dais"; + dais = <&i2s_8ch_0>, <&i2s_8ch_2>; + capture,channel-mapping = <6 2>; + playback,channel-mapping = <0 2>; + }; + + i2s-dmic-array { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,i2s-dmic-array"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch_0>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + vad-sound { + status = "okay"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-vad"; + rockchip,cpu = <&i2s_8ch_0_2_dais>; + rockchip,codec = <&acodec>, <&vad>; + }; +}; + +&acodec_sound { + status = "disabled"; +}; + +&bluetooth_sound { + status = "okay"; +}; + +&i2s_2ch_0 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&i2s_8ch_0 { + status = "okay"; + rockchip,no-dmaengine; + #sound-dai-cells = <0>; +}; + +&i2s_8ch_1 { + status = "disabled"; + #sound-dai-cells = <0>; + rockchip,no-dmaengine; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_1_m0_sdo0 + &i2s_8ch_1_m0_sdo1_sdi3 + &i2s_8ch_1_m0_sdo2_sdi2 + &i2s_8ch_1_m0_sdo3_sdi1 + &i2s_8ch_1_m0_sdi0>; +}; + +&i2s_8ch_2 { + status = "okay"; + rockchip,no-dmaengine; + #sound-dai-cells = <0>; +}; + +&vad { + status = "okay"; + rockchip,audio-src = <&i2s_8ch_0>; + rockchip,buffer-time-ms = <200>; + rockchip,det-channel = <0>; + rockchip,mode = <1>; + #sound-dai-cells = <0>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v11.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v11.dts new file mode 100755 index 000000000000..364f74a40d9d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-i2s-v11.dts @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; + +#include "rk3308-evb-v11.dtsi" + +/ { + model = "Rockchip RK3308 evb digital-i2s mic v11 board"; + compatible = "rockchip,rk3308-evb-dmic-i2s-v11", "rockchip,rk3308"; + + i2s_8ch_0_2_dais: i2s-8ch-0-2-dais { + status = "okay"; + compatible = "rockchip,rk3308-multi-dais", "rockchip,multi-dais"; + dais = <&i2s_8ch_0>, <&i2s_8ch_2>; + capture,channel-mapping = <6 2>; + playback,channel-mapping = <0 2>; + }; + + i2s-dmic-array { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,i2s-dmic-array"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch_0>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + vad-sound { + status = "okay"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-vad"; + rockchip,cpu = <&i2s_8ch_0_2_dais>; + rockchip,codec = <&acodec>, <&vad>; + }; +}; + +&acodec_sound { + status = "disabled"; +}; + +&bluetooth_sound { + status = "okay"; +}; + +&i2s_2ch_0 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&i2s_8ch_0 { + status = "okay"; + rockchip,no-dmaengine; + #sound-dai-cells = <0>; +}; + +&i2s_8ch_2 { + status = "okay"; + rockchip,no-dmaengine; + #sound-dai-cells = <0>; +}; + +&vad { + status = "okay"; + rockchip,audio-src = <&i2s_8ch_0>; + rockchip,buffer-time-ms = <200>; + rockchip,det-channel = <0>; + rockchip,mode = <1>; + #sound-dai-cells = <0>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v10.dts new file mode 100755 index 000000000000..e2891ce1092c --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v10.dts @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; + +#include "rk3308-evb-v10.dtsi" + +/ { + model = "Rockchip RK3308 evb digital-pdm mic board"; + compatible = "rockchip,rk3308-evb-dmic-pdm-v10", "rockchip,rk3308"; + + pdm_i2s_dais: pdm-i2s-dais { + status = "okay"; + compatible = "rockchip,rk3308-multi-dais", "rockchip,multi-dais"; + dais = <&pdm_8ch>, <&i2s_8ch_2>; + capture,channel-mapping = <6 2>; + playback,channel-mapping = <0 2>; + }; + + pdm-mic-array { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,pdm-mic-array"; + simple-audio-card,cpu { + sound-dai = <&pdm_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + vad-sound { + status = "okay"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-vad"; + rockchip,cpu = <&pdm_i2s_dais>; + rockchip,codec = <&acodec>, <&vad>; + }; +}; + +&acodec_sound { + status = "disabled"; +}; + +&bluetooth_sound { + status = "okay"; +}; + +&i2s_2ch_0 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&pdm_8ch { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,no-dmaengine; + pinctrl-names = "default"; + pinctrl-0 = <&pdm_m2_clk + &pdm_m2_clkm + &pdm_m2_sdi0 + &pdm_m2_sdi1 + &pdm_m2_sdi2 + &pdm_m2_sdi3>; +}; + +&vad { + status = "okay"; + rockchip,audio-src = <&pdm_8ch>; + rockchip,buffer-time-ms = <200>; + rockchip,det-channel = <2>; + rockchip,mode = <1>; + #sound-dai-cells = <0>; +}; + +&pdm_i2s_dais { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&i2s_8ch_2 { + status = "okay"; + rockchip,no-dmaengine; + #sound-dai-cells = <0>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v11.dts b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v11.dts new file mode 100755 index 000000000000..c4a7178f1dc8 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-dmic-pdm-v11.dts @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; + +#include "rk3308-evb-v11.dtsi" + +/ { + model = "Rockchip RK3308 evb digital-pdm mic v11 board"; + compatible = "rockchip,rk3308-evb-dmic-pdm-v11", "rockchip,rk3308"; + + pdm_i2s_dais: pdm-i2s-dais { + status = "okay"; + compatible = "rockchip,rk3308-multi-dais", "rockchip,multi-dais"; + dais = <&pdm_8ch>, <&i2s_8ch_2>; + capture,channel-mapping = <6 2>; + playback,channel-mapping = <0 2>; + bitclock-inversion = <1 0>; + }; + + pdm-mic-array { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,pdm-mic-array"; + simple-audio-card,cpu { + sound-dai = <&pdm_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + vad-sound { + status = "okay"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-vad"; + rockchip,cpu = <&pdm_i2s_dais>; + rockchip,codec = <&acodec>, <&vad>; + }; +}; + +&rk_timer_rtc { + status = "okay"; +}; + +&acodec_sound { + status = "disabled"; +}; + +&bluetooth_sound { + status = "okay"; +}; + +&i2s_2ch_0 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&pdm_8ch { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,no-dmaengine; + pinctrl-names = "default"; + pinctrl-0 = <&pdm_m2_clk + &pdm_m2_clkm + &pdm_m2_sdi0 + &pdm_m2_sdi1 + &pdm_m2_sdi2 + &pdm_m2_sdi3>; +}; + +&vad { + status = "okay"; + rockchip,audio-src = <&pdm_8ch>; + rockchip,det-channel = <0>; + rockchip,mode = <1>; + rockchip,buffer-time-ms = <200>; + #sound-dai-cells = <0>; +}; + +&i2s_8ch_2 { + status = "okay"; + rockchip,no-dmaengine; + #sound-dai-cells = <0>; +}; + +&pdm_i2s_dais { + status = "okay"; + #sound-dai-cells = <0>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-ext-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3308-evb-ext-v10.dtsi new file mode 100755 index 000000000000..1c0e66384c16 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-ext-v10.dtsi @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/ { + backlight: backlight { + status = "okay"; + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + panel: panel { + compatible = "simple-panel"; + bus-format = ; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>; + enable-delay-ms = <20>; + reset-gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_LOW>; + reset-delay-ms = <10>; + prepare-delay-ms = <20>; + unprepare-delay-ms = <20>; + disable-delay-ms = <20>; + /* spi-sdo-gpios = <&gpio3 RK_PD2 GPIO_ACTIVE_HIGH>; */ + spi-sdi-gpios = <&gpio1 RK_PC7 GPIO_ACTIVE_HIGH>; + spi-scl-gpios = <&gpio1 RK_PD0 GPIO_ACTIVE_HIGH>; + spi-cs-gpios = <&gpio1 RK_PD1 GPIO_ACTIVE_HIGH>; + width-mm = <217>; + height-mm = <136>; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&spi_init_cmd>; + rockchip,cmd-type = "spi"; + + /* type:0 is cmd, 1 is data */ + panel-init-sequence = [ + /* type delay num val1 val2 val3 */ + 00 00 01 e0 + 01 00 01 00 + 01 00 01 07 + 01 00 01 0f + 01 00 01 0d + 01 00 01 1b + 01 00 01 0a + 01 00 01 3c + 01 00 01 78 + 01 00 01 4a + 01 00 01 07 + 01 00 01 0e + 01 00 01 09 + 01 00 01 1b + 01 00 01 1e + 01 00 01 0f + 00 00 01 e1 + 01 00 01 00 + 01 00 01 22 + 01 00 01 24 + 01 00 01 06 + 01 00 01 12 + 01 00 01 07 + 01 00 01 36 + 01 00 01 47 + 01 00 01 47 + 01 00 01 06 + 01 00 01 0a + 01 00 01 07 + 01 00 01 30 + 01 00 01 37 + 01 00 01 0f + + 00 00 01 c0 + 01 00 01 10 + 01 00 01 10 + + 00 00 01 c1 + 01 00 01 41 + + 00 00 01 c5 + 01 00 01 00 + 01 00 01 22 + 01 00 01 80 + + 00 00 01 36 + 01 00 01 48 + + 00 00 01 3a /* interface mode control */ + 01 00 01 66 + + 00 00 01 b0 /* interface mode control */ + 01 00 01 00 + + 00 00 01 b1 /* frame rate 70hz */ + 01 00 01 b0 + 01 00 01 11 + 00 00 01 b4 + 01 00 01 02 + 00 00 01 B6 /* RGB/MCU Interface Control */ + 01 00 01 32 /* 02 mcu, 32 rgb */ + 01 00 01 02 + + 00 00 01 b7 + 01 00 01 c6 + + 00 00 01 be + 01 00 01 00 + 01 00 01 04 + + 00 00 01 e9 + 01 00 01 00 + + 00 00 01 f7 + 01 00 01 a9 + 01 00 01 51 + 01 00 01 2c + 01 00 01 82 + + 00 78 01 11 + 00 00 01 29 + ]; + + panel-exit-sequence = [ + /* type delay num val1 val2 val3 */ + 00 0a 01 28 + 00 78 01 10 + ]; + + display-timings { + native-mode = <&kd050fwfba002_timing>; + + kd050fwfba002_timing: timing0 { + clock-frequency = <12000000>; + hactive = <320>; + vactive = <480>; + hback-porch = <10>; + hfront-porch = <5>; + vback-porch = <10>; + vfront-porch = <5>; + hsync-len = <10>; + vsync-len = <10>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in_rgb: endpoint { + remote-endpoint = <&rgb_out_panel>; + }; + }; + }; +}; + +&display_subsystem { + status = "okay"; +}; + +&pinctrl { + spi_panel { + spi_init_cmd: spi-init-cmd { + rockchip,pins = + /* spi sdi */ + <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>, + /* spi scl */ + <1 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>, + /* spi cs */ + <1 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pwm1 { + status = "okay"; +}; + +&rgb { + status = "okay"; + + ports { + rgb_out: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + rgb_out_panel: endpoint@0 { + reg = <0>; + remote-endpoint = <&panel_in_rgb>; + }; + }; + }; +}; + +&route_rgb { + status = "okay"; +}; + +&vop { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3308-evb-v10.dtsi new file mode 100755 index 000000000000..3d052d0da477 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-v10.dtsi @@ -0,0 +1,780 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +#include +#include "rk3308.dtsi" + +/ { + model = "Rockchip RK3308 EVB"; + compatible = "rockchip,rk3308-evb", "rockchip,rk3308"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff0c0000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootfstype=squashfs rootwait snd_aloop.index=7 snd_aloop.use_raw_jiffies=1"; + }; + + adc-keys0 { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + func-key { + linux,code = ; + label = "function"; + press-threshold-microvolt = <18000>; + }; + }; + + adc-keys1 { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "micmute"; + press-threshold-microvolt = <1130000>; + }; + + home-key { + linux,code = ; + label = "mode"; + press-threshold-microvolt = <901000>; + }; + + menu-key { + linux,code = ; + label = "play"; + press-threshold-microvolt = <624000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <18000>; + }; + }; + + dummy_codec: dummy-codec { + compatible = "rockchip,dummy-codec"; + #sound-dai-cells = <0>; + }; + + gpio-keys { + compatible = "gpio-keys"; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = <&pwr_key>; + + power { + gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "GPIO Key Power"; + wakeup-source; + debounce-interval = <100>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; + }; + + acodec_sound: acodec-sound { + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-acodec"; + rockchip,codec-hp-det; + rockchip,mclk-fs = <256>; + rockchip,cpu = <&i2s_8ch_2>; + rockchip,codec = <&acodec>; + }; + + bluetooth_sound: bluetooth-sound { + status = "disabled"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-pcm"; + rockchip,mclk-fs = <128>; + rockchip,cpu = <&i2s_2ch_0>; + rockchip,codec = <&dummy_codec>; + rockchip,format = "dsp_b"; + rockchip,bitclock-inversion = <0>; + rockchip,wait-card-locked = <0>; + }; + + spdif_rx_sound: spdif-rx-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,spdif-rx-sound"; + simple-audio-card,cpu { + sound-dai = <&spdif_rx>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + spdif_tx_sound: spdif-tx-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,spdif-tx-sound"; + simple-audio-card,cpu { + sound-dai = <&spdif_tx>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + vdd_log: vdd_core: vdd-core { + compatible = "pwm-regulator"; + pwms = <&pwm0 0 5000 1>; + regulator-name = "vdd_core"; + regulator-min-microvolt = <827000>; + regulator-max-microvolt = <1340000>; + regulator-init-microvolt = <1015000>; + regulator-early-min-microvolt = <1015000>; + regulator-always-on; + regulator-boot-on; + regulator-settling-time-up-us = <250>; + status = "okay"; + }; + + vdd_1v0: vdd-1v0 { + compatible = "regulator-fixed"; + regulator-name = "vdd_1v0"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + }; + + vccio_sdio: vcc_1v8: vcc-1v8 { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc_io>; + }; + + vcc_sd: vcc-sd { + compatible = "regulator-fixed"; + gpio = <&gpio4 RK_PD6 GPIO_ACTIVE_LOW>; + regulator-name = "vcc_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vcc_1v8_codec: vcc-1v8-codec { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8_codec"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc_io>; + }; + + vcc_ddr: vcc-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + }; + + vcc_3v3_codec: vcc_io: vcc-io { + compatible = "regulator-fixed"; + regulator-name = "vcc_io"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vccio_flash: vccio-flash { + compatible = "regulator-fixed"; + regulator-name = "vccio_flash"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vbus_host: vbus-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb_drv>; + regulator-name = "vbus_host"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart4_rts>; + pinctrl-1 = <&uart4_rts_gpio>; + BT,power_gpio = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6255"; + WIFI,host_wake_irq = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&acodec { + status = "okay"; + + rockchip,no-deep-low-power; + rockchip,loopback-grp = <3>; + hp-ctl-gpios = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + spk-ctl-gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; +}; + +&cpu0 { + cpu-supply = <&vdd_core>; +}; + +&dmc { + center-supply = <&vdd_core>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&fiq_debugger { + status = "okay"; +}; + +&mac { + phy-supply = <&vcc_phy>; + assigned-clocks = <&cru SCLK_MAC>; + assigned-clock-parents = <&mac_clkin>; + clock_in_out = "input"; + pinctrl-names = "default"; + pinctrl-0 = <&rmii_pins &mac_refclk>; + snps,reset-gpio = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 50000 50000>; + status = "disable"; +}; + +&io_domains { + status = "okay"; + + vccio0-supply = <&vcc_io>; + vccio1-supply = <&vcc_io>; + vccio2-supply = <&vcc_1v8>; + vccio3-supply = <&vccio_flash>; + vccio4-supply = <&vccio_sdio>; + vccio5-supply = <&vcc_io>; +}; + +&i2c1 { + clock-frequency = <400000>; + status = "okay"; + + is31fl3236: led-controller@3c { + compatible = "issi,is31fl3236"; + reg = <0x3c>; + #address-cells = <1>; + #size-cells = <0>; + reset-gpios = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; + status = "okay"; + + led1: led@1 { + label = "led1"; + reg = <1>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led2: led@2 { + label = "led2"; + reg = <2>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led3: led@3 { + label = "led3"; + reg = <3>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led4: led@4 { + label = "led4"; + reg = <4>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led5: led@5 { + label = "led5"; + reg = <5>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led6: led@6 { + label = "led6"; + reg = <6>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led7: led@7 { + label = "led7"; + reg = <7>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led8: led@8 { + label = "led8"; + reg = <8>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led9: led@9 { + label = "led9"; + reg = <9>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led10: led@10 { + label = "led10"; + reg = <10>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led11: led@11 { + label = "led11"; + reg = <11>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led12: led@12 { + label = "led12"; + reg = <12>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led13: led@13 { + label = "led13"; + reg = <13>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led14: led@14 { + label = "led14"; + reg = <14>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led15: led@15 { + label = "led15"; + reg = <15>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led16: led@16 { + label = "led16"; + reg = <16>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led17: led@17 { + label = "led17"; + reg = <17>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led18: led@18 { + label = "led18"; + reg = <18>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led19: led@19 { + label = "led19"; + reg = <19>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led20: led@20 { + label = "led20"; + reg = <20>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led21: led@21 { + label = "led21"; + reg = <21>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led22: led@22 { + label = "led22"; + reg = <22>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led23: led@23 { + label = "led23"; + reg = <23>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led124: led@24 { + label = "led24"; + reg = <24>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led25: led@25 { + label = "led25"; + reg = <25>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led26: led@26 { + label = "led26"; + reg = <26>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led27: led@27 { + label = "led27"; + reg = <27>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led28: led@28 { + label = "led28"; + reg = <28>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led29: led@29 { + label = "led29"; + reg = <29>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led30: led@30 { + label = "led30"; + reg = <30>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led31: led@31 { + label = "led31"; + reg = <31>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led32: led@32 { + label = "led32"; + reg = <32>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led33: led@33 { + label = "led33"; + reg = <33>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led34: led@34 { + label = "led34"; + reg = <34>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led35: led@35 { + label = "led35"; + reg = <35>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led36: led@36 { + label = "led36"; + reg = <36>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + }; +}; + +&i2s_8ch_2 { + status = "okay"; +}; + +&nandc { + status = "okay"; +}; + +&rockchip_suspend { + rockchip,pwm-regulator-config = < + (0 + | RKPM_PWM_REGULATOR + ) + >; + + status = "okay"; +}; + +&rng { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <300>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sfc { + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <0>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&rtc_32k>; + + buttons { + pwr_key: pwr-key { + rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb { + usb_drv: usb-drv { + rockchip,pins = + <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pwm0 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin_pull_down>; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + phy-supply = <&vbus_host>; + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&uart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_xfer &uart4_cts>; + status = "okay"; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci{ + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-evb-v11.dtsi b/arch/arm64/boot/dts/rockchip/rk3308-evb-v11.dtsi new file mode 100755 index 000000000000..7b692a810333 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-evb-v11.dtsi @@ -0,0 +1,835 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include +#include "rk3308.dtsi" + +/ { + model = "Rockchip RK3308 EVB V11"; + compatible = "rockchip,rk3308-evb-v11", "rockchip,rk3308"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff0c0000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootfstype=squashfs rootwait snd_aloop.index=7 snd_aloop.use_raw_jiffies=1"; + }; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "micmute"; + press-threshold-microvolt = <1130000>; + }; + + home-key { + linux,code = ; + label = "mode"; + press-threshold-microvolt = <901000>; + }; + + menu-key { + linux,code = ; + label = "play"; + press-threshold-microvolt = <624000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <18000>; + }; + }; + + dummy_codec: dummy-codec { + compatible = "rockchip,dummy-codec"; + #sound-dai-cells = <0>; + }; + + gpio-keys { + compatible = "gpio-keys"; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = <&pwr_key>; + + power { + gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "GPIO Key Power"; + wakeup-source; + debounce-interval = <100>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; + }; + + acodec_sound: acodec-sound { + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-acodec"; + rockchip,codec-hp-det; + rockchip,mclk-fs = <256>; + rockchip,cpu = <&i2s_8ch_2>; + rockchip,codec = <&acodec>; + }; + + bluetooth_sound: bluetooth-sound { + status = "disabled"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-pcm"; + rockchip,mclk-fs = <128>; + rockchip,cpu = <&i2s_2ch_0>; + rockchip,codec = <&dummy_codec>; + rockchip,format = "dsp_b"; + rockchip,bitclock-inversion = <0>; + rockchip,wait-card-locked = <0>; + }; + + spdif_rx_sound: spdif-rx-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,spdif-rx-sound"; + simple-audio-card,cpu { + sound-dai = <&spdif_rx>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + spdif_tx_sound: spdif-tx-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,spdif-tx-sound"; + simple-audio-card,cpu { + sound-dai = <&spdif_tx>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + tas5731_sound: tas5731-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,tas5731"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch_1>; + }; + simple-audio-card,codec { + sound-dai = <&tas5731>; + }; + }; + + vdd_core: vdd-core { + compatible = "pwm-regulator"; + pwms = <&pwm0 0 5000 1>; + regulator-name = "vdd_core"; + regulator-min-microvolt = <827000>; + regulator-max-microvolt = <1340000>; + regulator-init-microvolt = <1015000>; + regulator-always-on; + regulator-boot-on; + regulator-settling-time-up-us = <250>; + status = "okay"; + }; + + vdd_log: vdd-log { + compatible = "regulator-fixed"; + regulator-name = "vdd_log"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + }; + + vdd_1v0: vdd-1v0 { + compatible = "regulator-fixed"; + regulator-name = "vdd_1v0"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + }; + + vccio_sdio: vcc_1v8: vcc-1v8 { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc_io>; + }; + + vccio_sd: vccio-sd { + compatible = "regulator-gpio"; + regulator-name = "vccio_sd"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; + states = <1800000 0x0 + 3300000 0x1>; + }; + + vcc_sd: vcc-sd { + compatible = "regulator-fixed"; + gpio = <&gpio4 RK_PD6 GPIO_ACTIVE_LOW>; + regulator-name = "vcc_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vcc_1v8_codec: vcc-1v8-codec { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8_codec"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc_io>; + }; + + vcc_ddr: vcc-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + }; + + vcc_3v3_codec: vcc_io: vcc-io { + compatible = "regulator-fixed"; + regulator-name = "vcc_io"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vccio_flash: vccio-flash { + compatible = "regulator-fixed"; + regulator-name = "vccio_flash"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vbus_host: vbus-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb_drv>; + regulator-name = "vbus_host"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart4_rts>; + pinctrl-1 = <&uart4_rts_gpio>; + BT,power_gpio = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_wake_host>; + wifi_chip_type = "ap6255"; + WIFI,host_wake_irq = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>; + status = "okay"; + }; +}; + +&acodec { + status = "okay"; + + rockchip,no-deep-low-power; + rockchip,loopback-grp = <0>; + hp-ctl-gpios = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + spk-ctl-gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; +}; + +&cpu0 { + cpu-supply = <&vdd_core>; +}; + +&cpu0_opp_table { + opp-1200000000 { + status = "okay"; + }; + opp-1296000000 { + status = "okay"; + }; +}; + +&dmc { + center-supply = <&vdd_log>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "disabled"; +}; + +&fiq_debugger { + status = "okay"; +}; + +&mac { + phy-supply = <&vcc_phy>; + assigned-clocks = <&cru SCLK_MAC>; + assigned-clock-parents = <&mac_clkin>; + clock_in_out = "input"; + pinctrl-names = "default"; + pinctrl-0 = <&rmii_pins &mac_refclk>; + snps,reset-gpio = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 50000 50000>; + status = "disable"; +}; + +&io_domains { + status = "okay"; + + vccio0-supply = <&vcc_io>; + vccio1-supply = <&vcc_io>; + vccio2-supply = <&vcc_1v8>; + vccio3-supply = <&vccio_flash>; + vccio4-supply = <&vccio_sdio>; + vccio5-supply = <&vccio_sd>; +}; + +&i2c1 { + clock-frequency = <400000>; + status = "okay"; + + tas5731: tas5731@1a { + #sound-dai-cells = <0>; + compatible = "ti,tas5731"; + reg = <0x1a>; + clocks = <&cru SCLK_I2S1_8CH_TX_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_1_m0_mclk>; + pdn-gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; + reset-gpios = <&gpio1 RK_PA1 GPIO_ACTIVE_LOW>; + }; + + is31fl3236: led-controller@3c { + compatible = "issi,is31fl3236"; + reg = <0x3c>; + #address-cells = <1>; + #size-cells = <0>; + reset-gpios = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; + status = "okay"; + + led1: led@1 { + label = "led1"; + reg = <1>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led2: led@2 { + label = "led2"; + reg = <2>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led3: led@3 { + label = "led3"; + reg = <3>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led4: led@4 { + label = "led4"; + reg = <4>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led5: led@5 { + label = "led5"; + reg = <5>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led6: led@6 { + label = "led6"; + reg = <6>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led7: led@7 { + label = "led7"; + reg = <7>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led8: led@8 { + label = "led8"; + reg = <8>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led9: led@9 { + label = "led9"; + reg = <9>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led10: led@10 { + label = "led10"; + reg = <10>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led11: led@11 { + label = "led11"; + reg = <11>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led12: led@12 { + label = "led12"; + reg = <12>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led13: led@13 { + label = "led13"; + reg = <13>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led14: led@14 { + label = "led14"; + reg = <14>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led15: led@15 { + label = "led15"; + reg = <15>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led16: led@16 { + label = "led16"; + reg = <16>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led17: led@17 { + label = "led17"; + reg = <17>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led18: led@18 { + label = "led18"; + reg = <18>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led19: led@19 { + label = "led19"; + reg = <19>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led20: led@20 { + label = "led20"; + reg = <20>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led21: led@21 { + label = "led21"; + reg = <21>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led22: led@22 { + label = "led22"; + reg = <22>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led23: led@23 { + label = "led23"; + reg = <23>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led124: led@24 { + label = "led24"; + reg = <24>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led25: led@25 { + label = "led25"; + reg = <25>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led26: led@26 { + label = "led26"; + reg = <26>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led27: led@27 { + label = "led27"; + reg = <27>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led28: led@28 { + label = "led28"; + reg = <28>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led29: led@29 { + label = "led29"; + reg = <29>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led30: led@30 { + label = "led30"; + reg = <30>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led31: led@31 { + label = "led31"; + reg = <31>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led32: led@32 { + label = "led32"; + reg = <32>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led33: led@33 { + label = "led33"; + reg = <33>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led34: led@34 { + label = "led34"; + reg = <34>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led35: led@35 { + label = "led35"; + reg = <35>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led36: led@36 { + label = "led36"; + reg = <36>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + }; +}; + +&i2s_8ch_1 { + status = "disabled"; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_1_m0_sclktx + &i2s_8ch_1_m0_lrcktx + &i2s_8ch_1_m0_sdo0 + &i2s_8ch_1_m0_mclk>; +}; + +&i2s_8ch_2 { + status = "okay"; +}; + +&nandc { + status = "okay"; +}; + +&rockchip_suspend { + rockchip,pwm-regulator-config = < + (0 + | RKPM_PWM_REGULATOR + ) + >; + + status = "okay"; +}; + +&rng { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <300>; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vmmc-supply = <&vcc_sd>; + vqmmc-supply = <&vccio_sd>; + status = "disabled"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <0>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&rtc_32k>; + + buttons { + pwr_key: pwr-key { + rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb { + usb_drv: usb-drv { + rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_wake_host: wifi-wake-host { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pwm0 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin_pull_down>; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + phy-supply = <&vbus_host>; + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&uart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_xfer &uart4_cts>; + status = "okay"; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci{ + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-fpga.dts b/arch/arm64/boot/dts/rockchip/rk3308-fpga.dts new file mode 100755 index 000000000000..225aca323f4d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-fpga.dts @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; + +#include "rk3308.dtsi" + +/ { + model = "Rockchip RK3308 FPGA Platform"; + compatible = "rockchip,rk3308-fpga", "rockchip,rk3308"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff0b0000 console=ttyFIQ0 init=/init initrd=0x9000000,0x18bfc0"; + }; + + memory@200000 { + device_type = "memory"; + reg = <0x0 0x00200000 0x0 0x0FE00000>; + }; +}; + +&fiq_debugger { + rockchip,serial-id = <1>; + rockchip,irq-mode-enable = <1>; + status = "ok"; +}; + +&cpu1 { + /delete-property/enable-method; +}; + +&cpu2 { + /delete-property/enable-method; +}; + +&cpu3 { + /delete-property/enable-method; +}; + +&emmc { + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + non-removable; + num-slots = <1>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts b/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts index 213c0759c4b8..b89eaf1075ab 100644 --- a/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts +++ b/arch/arm64/boot/dts/rockchip/rk3308-roc-cc.dts @@ -78,8 +78,8 @@ vcc_sdmmc: vcc-sdmmc { regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3300000>; gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_HIGH>; - states = <1800000 0x0>, - <3300000 0x1>; + states = <1800000 0x0 + 3300000 0x1>; vin-supply = <&vcc5v0_sys>; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3308-voice-module-board-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308-voice-module-board-v10.dts new file mode 100755 index 000000000000..2586d9905072 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308-voice-module-board-v10.dts @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; + +#include "arm/rk3308-voice-module-board-v10-aarch32.dts" + +/ { + model = "Rockchip RK3308 Voice Module Board V10"; + compatible = "rockchip,rk3308-voice-module-board-v10", "rockchip,rk3308"; +}; + +&ramoops { + reg = <0x0 0x110000 0x0 0xf0000>; + record-size = <0x30000>; + console-size = <0xc0000>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308.dtsi b/arch/arm64/boot/dts/rockchip/rk3308.dtsi index 2560b98771ca..cc86c0880cec 100644 --- a/arch/arm64/boot/dts/rockchip/rk3308.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3308.dtsi @@ -10,6 +10,7 @@ #include #include #include +#include #include / { @@ -147,6 +148,21 @@ psci { method = "smc"; }; + rockchip_suspend: rockchip-suspend { + compatible = "rockchip,pm-rk3308"; + status = "disabled"; + rockchip,sleep-mode-config = < + (0 + | RKPM_PMU_HW_PLLS_PD + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO0_WAKEUP_EN + ) + >; + }; + timer { compatible = "arm,armv8-timer"; interrupts = , @@ -366,7 +382,7 @@ pwm8: pwm@ff160000 { reg = <0x0 0xff160000 0x0 0x10>; clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm8_pin>; #pwm-cells = <3>; status = "disabled"; @@ -377,7 +393,7 @@ pwm9: pwm@ff160010 { reg = <0x0 0xff160010 0x0 0x10>; clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm9_pin>; #pwm-cells = <3>; status = "disabled"; @@ -388,7 +404,7 @@ pwm10: pwm@ff160020 { reg = <0x0 0xff160020 0x0 0x10>; clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm10_pin>; #pwm-cells = <3>; status = "disabled"; @@ -399,7 +415,7 @@ pwm11: pwm@ff160030 { reg = <0x0 0xff160030 0x0 0x10>; clocks = <&cru SCLK_PWM2>, <&cru PCLK_PWM2>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm11_pin>; #pwm-cells = <3>; status = "disabled"; @@ -410,7 +426,7 @@ pwm4: pwm@ff170000 { reg = <0x0 0xff170000 0x0 0x10>; clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm4_pin>; #pwm-cells = <3>; status = "disabled"; @@ -421,7 +437,7 @@ pwm5: pwm@ff170010 { reg = <0x0 0xff170010 0x0 0x10>; clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm5_pin>; #pwm-cells = <3>; status = "disabled"; @@ -432,7 +448,7 @@ pwm6: pwm@ff170020 { reg = <0x0 0xff170020 0x0 0x10>; clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm6_pin>; #pwm-cells = <3>; status = "disabled"; @@ -443,7 +459,7 @@ pwm7: pwm@ff170030 { reg = <0x0 0xff170030 0x0 0x10>; clocks = <&cru SCLK_PWM1>, <&cru PCLK_PWM1>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm7_pin>; #pwm-cells = <3>; status = "disabled"; @@ -454,7 +470,7 @@ pwm0: pwm@ff180000 { reg = <0x0 0xff180000 0x0 0x10>; clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_pin>; #pwm-cells = <3>; status = "disabled"; @@ -465,7 +481,7 @@ pwm1: pwm@ff180010 { reg = <0x0 0xff180010 0x0 0x10>; clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_pin>; #pwm-cells = <3>; status = "disabled"; @@ -476,7 +492,7 @@ pwm2: pwm@ff180020 { reg = <0x0 0xff180020 0x0 0x10>; clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm2_pin>; #pwm-cells = <3>; status = "disabled"; @@ -487,7 +503,7 @@ pwm3: pwm@ff180030 { reg = <0x0 0xff180030 0x0 0x10>; clocks = <&cru SCLK_PWM0>, <&cru PCLK_PWM0>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm3_pin>; #pwm-cells = <3>; status = "disabled"; diff --git a/arch/arm64/boot/dts/rockchip/rk3308b-evb-amic-v10.dts b/arch/arm64/boot/dts/rockchip/rk3308b-evb-amic-v10.dts new file mode 100755 index 000000000000..d5e2d9ebd4aa --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308b-evb-amic-v10.dts @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; + +#include "rk3308b-evb-v10.dtsi" + +/ { + model = "Rockchip RK3308b evb analog mic v10 board"; + compatible = "rockchip,rk3308b-evb-amic-v10", "rockchip,rk3308"; + + vad_acodec_sound: vad-acodec-sound { + status = "okay"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-vad"; + rockchip,codec-hp-det; + rockchip,mclk-fs = <256>; + rockchip,cpu = <&i2s_8ch_2>; + rockchip,codec = <&acodec>, <&vad>; + }; +}; + +&acodec { + rockchip,micbias1; + rockchip,micbias2; + rockchip,en-always-grps = <1 2 3>; + rockchip,adc-grps-route = <1 2 3 0>; +}; + +&acodec_sound { + status = "disabled"; +}; + +&bluetooth_sound { + status = "okay"; +}; + +&i2s_8ch_0 { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_0_sclktx + &i2s_8ch_0_lrcktx + &i2s_8ch_0_sdi0 + &i2s_8ch_0_sdo2>; +}; + +&is31fl3236 { + reg = <0x3f>; +}; + +&vad { + status = "okay"; + rockchip,audio-src = <&i2s_8ch_2>; + rockchip,det-channel = <0>; + rockchip,buffer-time-ms = <200>; + rockchip,mode = <1>; + #sound-dai-cells = <0>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308b-evb-ext-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3308b-evb-ext-v10.dtsi new file mode 100755 index 000000000000..e2414b67f0d1 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308b-evb-ext-v10.dtsi @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + */ + +/ { + backlight: backlight { + status = "okay"; + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + panel: panel { + compatible = "simple-panel"; + bus-format = ; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>; + enable-delay-ms = <20>; + reset-gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_LOW>; + reset-delay-ms = <10>; + prepare-delay-ms = <20>; + unprepare-delay-ms = <20>; + disable-delay-ms = <20>; + width-mm = <95>; + height-mm = <54>; + status = "okay"; + + display-timings { + native-mode = <&stt0430_enl2c_timing>; + + stt0430_enl2c_timing: timing0 { + clock-frequency = <12000000>; + hactive = <480>; + vactive = <272>; + hback-porch = <60>; + hfront-porch = <20>; + vback-porch = <28>; + vfront-porch = <20>; + hsync-len = <20>; + vsync-len = <20>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in_rgb: endpoint { + remote-endpoint = <&rgb_out_panel>; + }; + }; + }; +}; + +&display_subsystem { + status = "okay"; +}; + +&pwm1 { + status = "okay"; +}; + +&rgb { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&lcdc_ctl &lcdc_rgb888_m1>; + + ports { + rgb_out: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + rgb_out_panel: endpoint@0 { + reg = <0>; + remote-endpoint = <&panel_in_rgb>; + }; + }; + }; +}; + +&route_rgb { + status = "okay"; +}; + +&vop { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308b-evb-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3308b-evb-v10.dtsi new file mode 100755 index 000000000000..902ae3544235 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308b-evb-v10.dtsi @@ -0,0 +1,784 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include +#include "rk3308.dtsi" + +/ { + model = "Rockchip RK3308B EVB V10"; + compatible = "rockchip,rk3308b-evb-v10", "rockchip,rk3308"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff0e0000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rootfstype=squashfs rootwait snd_aloop.index=7 snd_aloop.use_raw_jiffies=1"; + }; + + acodec_sound: acodec-sound { + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-acodec"; + rockchip,codec-hp-det; + rockchip,mclk-fs = <256>; + rockchip,cpu = <&i2s_8ch_2>; + rockchip,codec = <&acodec>; + }; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "micmute"; + press-threshold-microvolt = <1130000>; + }; + + home-key { + linux,code = ; + label = "mode"; + press-threshold-microvolt = <901000>; + }; + + menu-key { + linux,code = ; + label = "play"; + press-threshold-microvolt = <624000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <18000>; + }; + }; + + bluetooth_sound: bluetooth-sound { + status = "disabled"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3308-pcm"; + rockchip,mclk-fs = <128>; + rockchip,cpu = <&i2s_8ch_0>; + rockchip,codec = <&dummy_codec>; + rockchip,format = "dsp_b"; + rockchip,bitclock-inversion = <0>; + rockchip,wait-card-locked = <0>; + }; + + dummy_codec: dummy-codec { + compatible = "rockchip,dummy-codec"; + #sound-dai-cells = <0>; + }; + + gpio-keys { + compatible = "gpio-keys"; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = <&pwr_key>; + + power { + gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "GPIO Key Power"; + wakeup-source; + debounce-interval = <100>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_LOW>; + }; + + spdif_rx_sound: spdif-rx-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,spdif-rx-sound"; + simple-audio-card,cpu { + sound-dai = <&spdif_rx>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + spdif_tx_sound: spdif-tx-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,spdif-tx-sound"; + simple-audio-card,cpu { + sound-dai = <&spdif_tx>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + vdd_core: vdd-core { + compatible = "pwm-regulator"; + pwms = <&pwm0 0 5000 1>; + regulator-name = "vdd_core"; + regulator-min-microvolt = <827000>; + regulator-max-microvolt = <1340000>; + regulator-init-microvolt = <1015000>; + regulator-always-on; + regulator-boot-on; + regulator-settling-time-up-us = <250>; + status = "okay"; + }; + + vdd_log: vdd-log { + compatible = "regulator-fixed"; + regulator-name = "vdd_log"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + }; + + vdd_1v0: vdd-1v0 { + compatible = "regulator-fixed"; + regulator-name = "vdd_1v0"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + }; + + vccio_sdio: vcc_1v8: vcc-1v8 { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc_io>; + }; + + vcc_1v8_codec: vcc-1v8-codec { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8_codec"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc_io>; + }; + + vcc_ddr: vcc-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + }; + + vcc_3v3_codec: vcc_io: vcc-io { + compatible = "regulator-fixed"; + regulator-name = "vcc_io"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vccio_flash: vccio-flash { + compatible = "regulator-fixed"; + regulator-name = "vccio_flash"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vbus_host: vbus-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb_drv>; + regulator-name = "vbus_host"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio2 RK_PA3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_rts_gpio>; + BT,power_gpio = <&gpio2 RK_PA6 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio2 RK_PB0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_wake_host>; + wifi_chip_type = "ap6255"; + WIFI,host_wake_irq = <&gpio2 RK_PB4 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&acodec { + status = "okay"; + + rockchip,no-deep-low-power; + hp-ctl-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_HIGH>; + spk-ctl-gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; +}; + +&cpu0 { + cpu-supply = <&vdd_core>; +}; + +&cpu0_opp_table { + opp-1200000000 { + status = "okay"; + }; + opp-1296000000 { + status = "okay"; + }; +}; + +&dmc { + center-supply = <&vdd_log>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "disabled"; +}; + +&fiq_debugger { + rockchip,serial-id = <4>; + status = "okay"; +}; + +&io_domains { + status = "okay"; + + vccio0-supply = <&vcc_io>; + vccio1-supply = <&vcc_io>; + vccio2-supply = <&vccio_sdio>; + vccio3-supply = <&vccio_flash>; + vccio4-supply = <&vcc_io>; + vccio5-supply = <&vccio_sdio>; +}; + +&i2c1 { + clock-frequency = <400000>; + status = "okay"; + + is31fl3236: led-controller@3c { + compatible = "issi,is31fl3236"; + reg = <0x3c>; + #address-cells = <1>; + #size-cells = <0>; + reset-gpios = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; + status = "okay"; + + led1: led@1 { + label = "led1"; + reg = <1>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led2: led@2 { + label = "led2"; + reg = <2>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led3: led@3 { + label = "led3"; + reg = <3>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led4: led@4 { + label = "led4"; + reg = <4>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led5: led@5 { + label = "led5"; + reg = <5>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led6: led@6 { + label = "led6"; + reg = <6>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led7: led@7 { + label = "led7"; + reg = <7>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led8: led@8 { + label = "led8"; + reg = <8>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led9: led@9 { + label = "led9"; + reg = <9>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led10: led@10 { + label = "led10"; + reg = <10>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led11: led@11 { + label = "led11"; + reg = <11>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led12: led@12 { + label = "led12"; + reg = <12>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led13: led@13 { + label = "led13"; + reg = <13>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led14: led@14 { + label = "led14"; + reg = <14>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led15: led@15 { + label = "led15"; + reg = <15>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led16: led@16 { + label = "led16"; + reg = <16>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led17: led@17 { + label = "led17"; + reg = <17>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led18: led@18 { + label = "led18"; + reg = <18>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led19: led@19 { + label = "led19"; + reg = <19>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led20: led@20 { + label = "led20"; + reg = <20>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led21: led@21 { + label = "led21"; + reg = <21>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led22: led@22 { + label = "led22"; + reg = <22>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led23: led@23 { + label = "led23"; + reg = <23>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led124: led@24 { + label = "led24"; + reg = <24>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led25: led@25 { + label = "led25"; + reg = <25>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led26: led@26 { + label = "led26"; + reg = <26>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led27: led@27 { + label = "led27"; + reg = <27>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led28: led@28 { + label = "led28"; + reg = <28>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led29: led@29 { + label = "led29"; + reg = <29>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led30: led@30 { + label = "led30"; + reg = <30>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led31: led@31 { + label = "led31"; + reg = <31>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led32: led@32 { + label = "led32"; + reg = <32>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led33: led@33 { + label = "led33"; + reg = <33>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led34: led@34 { + label = "led34"; + reg = <34>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led35: led@35 { + label = "led35"; + reg = <35>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led36: led@36 { + label = "led36"; + reg = <36>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + }; +}; + +&i2s_8ch_1 { + status = "disabled"; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_1_m0_sclktx + &i2s_8ch_1_m0_lrcktx + &i2s_8ch_1_m0_sdo0 + &i2s_8ch_1_m0_mclk>; +}; + +&i2s_8ch_2 { + status = "okay"; +}; + +&mac { + phy-supply = <&vcc_phy>; + assigned-clocks = <&cru SCLK_MAC>; + assigned-clock-parents = <&mac_clkin>; + clock_in_out = "input"; + pinctrl-names = "default"; + pinctrl-0 = <&rmiim1_pins &macm1_refclk>; + snps,reset-gpio = <&gpio4 RK_PC0 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 50000 50000>; + status = "disable"; +}; + +&nandc { + status = "okay"; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&rtc_32k>; + + buttons { + pwr_key: pwr-key { + rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb { + usb_drv: usb-drv { + rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_wake_host: wifi-wake-host { + rockchip,pins = <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pwm0 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin_pull_down>; +}; + +&rng { + status = "okay"; +}; + +&rockchip_suspend { + rockchip,pwm-regulator-config = < + (0 + | RKPM_PWM_REGULATOR + ) + >; + + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdmmc { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sfc { + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <0>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + pinctrl-names = "init", "default", "sleep"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_gpio>; + pinctrl-2 = <&tsadc_otp_gpio>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + phy-supply = <&vbus_host>; + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci{ + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3308k.dtsi b/arch/arm64/boot/dts/rockchip/rk3308k.dtsi new file mode 100755 index 000000000000..ffd53fd158c5 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3308k.dtsi @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include +#include "rk3308.dtsi" + +/ { + uboot-wide-temperature { + status = "okay"; + compatible = "rockchip,uboot-wide-temperature"; + }; +}; + +&cpu0_opp_table { + rockchip,high-temp = <55000>; + rockchip,high-temp-max-volt = <1125000>; +}; + +&rockchip_suspend { + rockchip,sleep-mode-config = < + (0 + | RKPM_PMU_HW_PLLS_PD + | RKPM_PWM_VOLTAGE_DEFAULT + ) + >; +}; + +&thermal_zones { + soc-thermal { + sustainable-power = <422>; + k_pu = <6>; + k_po = <1024>; + k_i = <0>; + + trips { + trip-point-0 { + temperature = <55000>; + }; + trip-point-1 { + temperature = <90000>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-863-cif-sensor.dtsi b/arch/arm64/boot/dts/rockchip/rk3326-863-cif-sensor.dtsi new file mode 100755 index 000000000000..c01f4d014a1b --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-863-cif-sensor.dtsi @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +#include "../../../../../drivers/soc/rockchip/rk_camera_sensor_info.h" +/{ + cif_sensor: cif_sensor { + compatible = "rockchip,sensor"; + status = "disabled"; + + gc2145_b { + is_front = <0>; + powerdown-gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_HIGH>; + pwdn_active = ; + pwr_active = ; + rockchip,power_pmu_name1 = "vcc2v8_dvp"; + rockchip,power_pmu_voltage1 = <2800000>; + rockchip,power_pmu_name2 = "vcc1v8_dvp"; + rockchip,power_pmu_voltage2 = <1800000>; + mir = <0>; + flash_attach = <0>; + resolution = ; + powerup_sequence = ; + orientation = <90>; + i2c_add = ; + i2c_chl = <2>; + cif_chl = <0>; + mclk_rate = <24>; + }; + + gc0312_f { + is_front = <1>; + powerdown-gpios = <&gpio2 RK_PB6 GPIO_ACTIVE_HIGH>; + pwdn_active = ; + pwr_active = ; + rockchip,power_pmu_name1 = "vcc2v8_dvp"; + rockchip,power_pmu_voltage1 = <2800000>; + rockchip,power_pmu_name2 = "vcc1v8_dvp"; + rockchip,power_pmu_voltage2 = <1800000>; + mir = <0>; + flash_attach = <0>; + resolution = ; + powerup_sequence = ; + orientation = <270>; + i2c_add = ; + i2c_chl = <2>; + cif_chl = <0>; + mclk_rate = <24>; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-avb.dts b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-avb.dts new file mode 100755 index 000000000000..174d05e37ccb --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-avb.dts @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3326-863-lp3-v10.dtsi" + +/ { + model = "Rockchip rk3326 863 avb board"; + compatible = "rockchip,rk3326-863-lp3-v10-avb", "rockchip,rk3326"; +}; + +&firmware_android { + compatible = "android,firmware"; + boot_devices = "ff390000.dwmmc,ff3b0000.nandc"; + vbmeta { + compatible = "android,vbmeta"; + parts = "vbmeta,boot,system,vendor,dtbo"; + }; + fstab { + compatible = "android,fstab"; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,avb"; + }; + }; +}; + +&i2c2 { + status = "okay"; + + gc0312@21 { + status = "okay"; + compatible = "galaxycore,gc0312"; + reg = <0x21>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vcc1v8_dvp>; + + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + + port { + gc0312_out: endpoint { + remote-endpoint = <&dvp_in_fcam>; + }; + }; + }; + + gc2145@3c { + status = "okay"; + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vcc1v8_dvp>; + + pwdn-gpios = <&gpio2 13 GPIO_ACTIVE_HIGH>; + + port { + gc2145_out: endpoint { + remote-endpoint = <&dvp_in_bcam>; + }; + }; + }; +}; + +&isp_mmu { + status = "okay"; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0 &dvp_d10d11_m0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dvp_in_fcam: endpoint@0 { + reg = <0>; + remote-endpoint = <&gc0312_out>; + }; + + dvp_in_bcam: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc2145_out>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-rkisp1.dts b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-rkisp1.dts new file mode 100755 index 000000000000..d80dad694c1d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10-rkisp1.dts @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3326-863-lp3-v10.dtsi" + +/ { + model = "Rockchip rk3326 863 rkisp1 board"; + compatible = "rockchip,rk3326-863-lp3-v10-rkisp1", "rockchip,rk3326"; +}; + +&chosen { + bootargs_ext = "androidboot.boot_devices=ff390000.dwmmc,ff3b0000.nandc"; +}; + +&i2c2 { + status = "okay"; + + gc0312: gc0312@21 { + status = "okay"; + compatible = "galaxycore,gc0312"; + reg = <0x21>; + //pinctrl-names = "default"; + //pinctrl-0 = <&cif_clkout_m0>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vcc1v8_dvp>; + + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc0312_out: endpoint { + remote-endpoint = <&dvp_in_fcam>; + }; + }; + }; + + gc2145: gc2145@3c { + status = "okay"; + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + //pinctrl-names = "default"; + //pinctrl-0 = <&cif_clkout_m0>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vcc1v8_dvp>; + + pwdn-gpios = <&gpio2 13 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_out: endpoint { + remote-endpoint = <&dvp_in_bcam>; + }; + }; + }; +}; + +&isp_mmu { + status = "okay"; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&dvp_d0d1_m0 &dvp_d2d9_m0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dvp_in_fcam: endpoint@0 { + reg = <0>; + remote-endpoint = <&gc0312_out>; + }; + + dvp_in_bcam: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc2145_out>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dts new file mode 100755 index 000000000000..7a399b3356ac --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dts @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include "rk3326-863-lp3-v10.dtsi" + +/ { + model = "Rockchip rk3326 863 board"; + compatible = "rockchip,rk3326-863-lp3-v10", "rockchip,rk3326"; +}; + +&cif { + status = "okay"; +}; + +&cif_sensor { + status = "okay"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dtsi new file mode 100755 index 000000000000..6e6e3be2f104 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-863-lp3-v10.dtsi @@ -0,0 +1,833 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "rk3326.dtsi" +#include "rk3326-863-cif-sensor.dtsi" +#include "px30-android.dtsi" + +/ { + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 10 10 11 11 12 12 13 + 13 14 14 15 15 16 16 17 + 17 18 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3500>; + rockchip,screen-on-voltage = <3600>; + status = "okay"; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip-rk817-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&cru SCLK_WIFI_PMU>; + clock-names = "clk_wifi_pmu"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "rtl8723cs"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + WIFI,vbat_gpio = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + + +&display_subsystem { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "aoly,sl008pa21y1285-b00", "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>; + reset-gpios = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <20>; + reset-delay-ms = <20>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <20>; + unprepare-delay-ms = <20>; + + width-mm = <108>; + height-mm = <172>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 78 01 11 + 05 14 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <66000000>; + hactive = <800>; + vactive = <1280>; + hfront-porch = <2>; + hsync-len = <18>; + hback-porch = <18>; + vfront-porch = <4>; + vsync-len = <4>; + vback-porch = <16>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&cpu0_opp_table { + /* + * max IR-drop values on different freq condition for this board! + */ + rockchip,board-irdrop = < + /*MHz MHz uV */ + 0 815 37500 + 816 1119 50000 + 1200 1512 75000 + >; +}; + +&dmc_opp_table { + /* + * max IR-drop values on different freq condition for this board! + */ + rockchip,board-irdrop = < + /*MHz MHz uV */ + 451 800 75000 + >; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vcc_3v0>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_ts_gpio1: rk817_ts_gpio1 { + pins = "gpio_ts"; + function = "pin_fun1"; + /* output-low; */ + /* input-enable; */ + }; + + rk817_gt_gpio2: rk817_gt_gpio2 { + pins = "gpio_gt"; + function = "pin_fun1"; + }; + + rk817_pin_ts: rk817_pin_ts { + pins = "gpio_ts"; + function = "pin_fun0"; + }; + + rk817_pin_gt: rk817_pin_gt { + pins = "gpio_gt"; + function = "pin_fun0"; + }; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3500 3548 3592 3636 3687 3740 3780 + 3806 3827 3846 3864 3889 3929 3964 + 3993 4015 4030 4041 4056 4076 4148>; + design_capacity = <4000>; + design_qmax = <4200>; + bat_res = <100>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3500>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4200>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&u2phy>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + + ts@40 { + status = "okay"; + compatible = "GSL,GSL3673_800X1280"; + reg = <0x40>; + irq_gpio_number = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + }; + + sensor@19 { + status = "okay"; + compatible = "gs_lis3dh"; + reg = <0x19>; + type = ; + irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <7>; + reprobe_en = <1>; + }; +}; + +&i2c2 { + status = "okay"; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc_3v0>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc2v8_dvp>; + vccio4-supply = <&vcc_3v0>; + vccio5-supply = <&vcc_3v0>; +}; + +&nandc0 { + status = "okay"; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + rockchip,low-power-mode; + status = "okay"; + }; + + u2phy_otg: otg-port { + rockchip,low-power-mode; + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&vip_mmu { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-86v-v10.dts b/arch/arm64/boot/dts/rockchip/rk3326-86v-v10.dts new file mode 100755 index 000000000000..86d048c0d731 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-86v-v10.dts @@ -0,0 +1,840 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "rk3326.dtsi" +#include "rk3326-863-cif-sensor.dtsi" +#include "px30-android.dtsi" + +/ { + model = "Rockchip rk3326 86v board"; + compatible = "rockchip,rk3326-86v-v10", "rockchip,rk3326"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <617000>; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 10 10 11 11 12 12 13 + 13 14 14 15 15 16 16 17 + 17 18 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <0>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3500>; + rockchip,screen-on-voltage = <3600>; + status = "okay"; + }; + + panel { + compatible ="simple-panel"; + enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_LOW>; + reset-gpios = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + bus-format = ; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <51200000>; + hactive = <1024>; + vactive = <600>; + hback-porch = <100>; + hfront-porch = <120>; + vback-porch = <10>; + vfront-porch = <15>; + hsync-len = <100>; + vsync-len = <10>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in_rgb: endpoint { + remote-endpoint = <&rgb_out_panel>; + }; + }; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&cru SCLK_WIFI_PMU>; + clock-names = "clk_wifi_pmu"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; +}; + +&cif { + status = "okay"; +}; + +&cif_sensor { + status = "okay"; +}; + +&display_subsystem { + status = "okay"; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&cpu0_opp_table { + /* + * max IR-drop values on different freq condition for this board! + */ + rockchip,board-irdrop = < + /*MHz MHz uV */ + 0 815 75000 + 816 1119 75000 + 1200 1512 75000 + >; +}; + +&dmc_opp_table { + /* + * max IR-drop values on different freq condition for this board! + */ + rockchip,board-irdrop = < + /*MHz MHz uV */ + 451 800 75000 + >; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "disabled"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + /*mmc-hs200-1_8v;*/ + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <10 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vcc_3v0>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_ts_gpio1: rk817_ts_gpio1 { + pins = "gpio_ts"; + function = "pin_fun1"; + /* output-low; */ + /* input-enable; */ + }; + + rk817_gt_gpio2: rk817_gt_gpio2 { + pins = "gpio_gt"; + function = "pin_fun1"; + }; + + rk817_pin_ts: rk817_pin_ts { + pins = "gpio_ts"; + function = "pin_fun0"; + }; + + rk817_pin_gt: rk817_pin_gt { + pins = "gpio_gt"; + function = "pin_fun0"; + }; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3500 3548 3592 3636 3687 3740 3780 + 3806 3827 3846 3864 3889 3929 3964 + 3993 4015 4030 4041 4056 4076 4148>; + design_capacity = <4000>; + design_qmax = <4200>; + bat_res = <100>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3500>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4200>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&u2phy>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + + ts@40 { + compatible = "gslX680-d708"; + reg = <0x40>; + touch-gpio = <&gpio0 RK_PB3 IRQ_TYPE_LEVEL_LOW>; + wake-gpio = <&gpio0 RK_PC1 IRQ_TYPE_LEVEL_LOW>; + screen_max_x = <1024>; + screen_max_y = <600>; + revert_x = <1>; + status = "okay"; + }; + + sensor@1d { + status = "okay"; + compatible = "gs_lsm303d"; + reg = <0x1d>; + type = ; + irq-gpio = <&gpio0 RK_PA1 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <5>; + reprobe_en = <1>; + }; +}; + +&i2c2 { + status = "okay"; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc_3v0>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc2v8_dvp>; + vccio4-supply = <&vcc_3v0>; + vccio5-supply = <&vcc_3v0>; +}; + +&rgb { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + rgb_out_panel: endpoint { + remote-endpoint = <&panel_in_rgb>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcdc { + lcdc_m1_rgb_pins: lcdc-m1-rgb-pins { + rockchip,pins = + <3 RK_PA0 1 &pcfg_pull_none>, /* LCDC_DCLK */ + <3 RK_PA4 1 &pcfg_pull_none_8ma>, /* LCDC_D0 */ + <3 RK_PA6 1 &pcfg_pull_none_8ma>, /* LCDC_D2 */ + <3 RK_PB2 1 &pcfg_pull_none_8ma>, /* LCDC_D6 */ + <3 RK_PB3 1 &pcfg_pull_none_8ma>, /* LCDC_D7 */ + <3 RK_PB5 1 &pcfg_pull_none_8ma>, /* LCDC_D9 */ + <3 RK_PC0 1 &pcfg_pull_none_8ma>, /* LCDC_D12 */ + <3 RK_PC1 1 &pcfg_pull_none_8ma>, /* LCDC_D13 */ + <3 RK_PC2 1 &pcfg_pull_none_8ma>, /* LCDC_D14 */ + <3 RK_PC3 1 &pcfg_pull_none_8ma>, /* LCDC_D15 */ + <3 RK_PC4 1 &pcfg_pull_none_8ma>, /* LCDC_D16 */ + <3 RK_PC5 1 &pcfg_pull_none_8ma>; /* LCDC_D17 */ + }; + + lcdc_m1_sleep_pins: lcdc-m1-sleep-pins { + rockchip,pins = + <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_DCLK */ + <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D0 */ + <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D2 */ + <3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D6 */ + <3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D7 */ + <3 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D9 */ + <3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D12 */ + <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D13 */ + <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D14 */ + <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D15 */ + <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>, /* LCDC_D16 */ + <3 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; /* LCDC_D17 */ + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&route_rgb { + connect = <&vopb_out_rgb>; + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "disabled"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + rockchip,low-power-mode; + status = "okay"; + }; + + u2phy_otg: otg-port { + rockchip,low-power-mode; + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&vip_mmu { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v10.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v10.dts new file mode 100755 index 000000000000..02308ebae99e --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v10.dts @@ -0,0 +1,1308 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "rk3326.dtsi" +#include "rk3326-863-cif-sensor.dtsi" +#include "px30-android.dtsi" + +/ { + model = "Rockchip rk3326 ai voice assistant evb board"; + compatible = "rockchip,rk3326-evb-ai-va-v10", "rockchip,rk3326"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + mute-key { + linux,code = ; + label = "mute"; + press-threshold-microvolt = <1119000>; + }; + + mode-key { + linux,code = ; + label = "mode"; + press-threshold-microvolt = <892000>; + }; + + media-key { + linux,code = ; + label = "media"; + press-threshold-microvolt = <616000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <15000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_2ch>; + }; + codec { + sound-dai = <&rk809_codec 0>; + }; + }; + simple-audio-card,dai-link@1 { + format = "pdm"; + cpu { + sound-dai = <&pdm>; + }; + codec { + sound-dai = <&rk809_codec 1>; + }; + }; + }; + + bt-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "dsp_a"; + simple-audio-card,bitclock-inversion = <1>; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,bt"; + simple-audio-card,cpu { + sound-dai = <&i2s2_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&bt_sco>; + }; + }; + + bt_sco: bt-sco { + compatible = "delta,dfbmcs320"; + #sound-dai-cells = <0>; + status = "okay"; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PB3 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + test-power { + status = "okay"; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PA2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&display_subsystem { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <40>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <66000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <280>; + i2c-scl-falling-time-ns = <16>; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + + vcc5v0_host: SWITCH_REG1 { + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <1>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + pdmdata-out-enable; + use-ext-amplifier; + adc-for-loopback; + spk-ctl-gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_HIGH>; + hp-volume = <20>; + spk-volume = <20>; + }; + }; +}; + +&i2c1 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <275>; + i2c-scl-falling-time-ns = <16>; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; + }; + + is31fl3236: led-controller@3c { + compatible = "issi,is31fl3236"; + reg = <0x3c>; + #address-cells = <1>; + #size-cells = <0>; + reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; + status = "okay"; + + led1: led@1 { + label = "led1"; + reg = <1>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led2: led@2 { + label = "led2"; + reg = <2>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led3: led@3 { + label = "led3"; + reg = <3>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led4: led@4 { + label = "led4"; + reg = <4>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led5: led@5 { + label = "led5"; + reg = <5>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led6: led@6 { + label = "led6"; + reg = <6>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led7: led@7 { + label = "led7"; + reg = <7>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led8: led@8 { + label = "led8"; + reg = <8>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led9: led@9 { + label = "led9"; + reg = <9>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led10: led@10 { + label = "led10"; + reg = <10>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led11: led@11 { + label = "led11"; + reg = <11>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led12: led@12 { + label = "led12"; + reg = <12>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led13: led@13 { + label = "led13"; + reg = <13>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led14: led@14 { + label = "led14"; + reg = <14>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led15: led@15 { + label = "led15"; + reg = <15>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led16: led@16 { + label = "led16"; + reg = <16>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led17: led@17 { + label = "led17"; + reg = <17>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led18: led@18 { + label = "led18"; + reg = <18>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led19: led@19 { + label = "led19"; + reg = <19>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led20: led@20 { + label = "led20"; + reg = <20>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led21: led@21 { + label = "led21"; + reg = <21>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led22: led@22 { + label = "led22"; + reg = <22>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led23: led@23 { + label = "led23"; + reg = <23>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led124: led@24 { + label = "led24"; + reg = <24>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led25: led@25 { + label = "led25"; + reg = <25>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led26: led@26 { + label = "led26"; + reg = <26>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led27: led@27 { + label = "led27"; + reg = <27>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led28: led@28 { + label = "led28"; + reg = <28>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led29: led@29 { + label = "led29"; + reg = <29>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led30: led@30 { + label = "led30"; + reg = <30>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led31: led@31 { + label = "led31"; + reg = <31>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led32: led@32 { + label = "led32"; + reg = <32>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led33: led@33 { + label = "led33"; + reg = <33>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led34: led@34 { + label = "led34"; + reg = <34>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led35: led@35 { + label = "led35"; + reg = <35>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led36: led@36 { + label = "led36"; + reg = <36>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + }; + + ls_stk3410: light@48 { + compatible = "ls_stk3410"; + status = "okay"; + reg = <0x48>; + type = ; + irq_enable = <0>; + als_threshold_high = <100>; + als_threshold_low = <10>; + als_ctrl_gain = <2>; /* 0:x1 1:x4 2:x16 3:x64 */ + poll_delay_ms = <100>; + }; + + ps_stk3410: proximity@48 { + compatible = "ps_stk3410"; + status = "okay"; + reg = <0x48>; + type = ; + //pinctrl-names = "default"; + //pinctrl-0 = <&gpio2_c3>; + //irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; + //irq_enable = <1>; + ps_threshold_high = <0x200>; + ps_threshold_low = <0x100>; + ps_ctrl_gain = <3>; /* 0:x1 1:x4 2:x16 3:x64 */ + ps_led_current = <3>; /* 0:12.5mA 1:25mA 2:50mA 3:100mA */ + poll_delay_ms = <100>; + }; + +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + //pinctrl-names = "default"; + //pinctrl-0 = <&cif_clkout_m0>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; + pinctrl-0 = <&i2s1_2ch_sclk + &i2s1_2ch_lrck + &i2s1_2ch_sdo>; +}; + +&i2s2_2ch { + status = "okay"; + rockchip,bclk-fs = <64>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc1v8_dvp>; + vccio4-supply = <&vcc1v8_soc>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pdm { + status = "okay"; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pdm_clk0m1 + &pdm_clk1 + &pdm_sdi0m1 + &pdm_sdi1 + &pdm_sdi2 + &pdm_sdi3>; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx_out>; + }; + + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11-i2s-dmic.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11-i2s-dmic.dts new file mode 100755 index 000000000000..0af03d1ec2a8 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11-i2s-dmic.dts @@ -0,0 +1,1330 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "rk3326.dtsi" +#include "rk3326-863-cif-sensor.dtsi" +#include "px30-android.dtsi" + +/ { + model = "Rockchip rk3326 ai voice assistant evb v11 i2s-dmic board"; + compatible = "rockchip,rk3326-evb-ai-va-v11-i2s-dmic", "rockchip,rk3326"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + mute-key { + linux,code = ; + label = "mute"; + press-threshold-microvolt = <1119000>; + }; + + mode-key { + linux,code = ; + label = "mode"; + press-threshold-microvolt = <892000>; + }; + + media-key { + linux,code = ; + label = "media"; + press-threshold-microvolt = <616000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <15000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + multi_dais: multi-dais { + status = "okay"; + compatible = "rockchip,multi-dais"; + dais = <&pdm>, <&i2s0_8ch>; + capture,channel-mapping = <2 6>; + playback,channel-mapping = <0 0>; + #sound-dai-cells = <0>; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_2ch>; + }; + codec { + sound-dai = <&rk809_codec 0>; + }; + }; + simple-audio-card,dai-link@1 { + format = "i2s"; + cpu { + sound-dai = <&multi_dais>; + }; + codec { + sound-dai = <&rk809_codec 1>; + }; + }; + }; + + bt-sound { + compatible = "simple-audio-card"; + status = "disabled"; + simple-audio-card,format = "dsp_a"; + simple-audio-card,bitclock-inversion = <1>; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,bt"; + simple-audio-card,cpu { + sound-dai = <&i2s2_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&bt_sco>; + }; + }; + + bt_sco: bt-sco { + compatible = "delta,dfbmcs320"; + #sound-dai-cells = <0>; + status = "okay"; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + test-power { + status = "okay"; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6255"; + WIFI,host_wake_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PA7 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&display_subsystem { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <40>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <66000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <280>; + i2c-scl-falling-time-ns = <16>; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <10 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc5v0_host: SWITCH_REG1 { + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <1>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + pdmdata-out-enable; + use-ext-amplifier; + adc-for-loopback; + spk-ctl-gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_HIGH>; + hp-volume = <20>; + spk-volume = <20>; + }; + }; +}; + +&i2c1 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <275>; + i2c-scl-falling-time-ns = <16>; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PB3 IRQ_TYPE_LEVEL_LOW>; + }; + + is31fl3236: led-controller@3c { + compatible = "issi,is31fl3236"; + reg = <0x3c>; + #address-cells = <1>; + #size-cells = <0>; + reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; + status = "okay"; + + led1: led@1 { + label = "led1"; + reg = <1>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led2: led@2 { + label = "led2"; + reg = <2>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led3: led@3 { + label = "led3"; + reg = <3>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led4: led@4 { + label = "led4"; + reg = <4>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led5: led@5 { + label = "led5"; + reg = <5>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led6: led@6 { + label = "led6"; + reg = <6>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led7: led@7 { + label = "led7"; + reg = <7>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led8: led@8 { + label = "led8"; + reg = <8>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led9: led@9 { + label = "led9"; + reg = <9>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led10: led@10 { + label = "led10"; + reg = <10>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led11: led@11 { + label = "led11"; + reg = <11>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led12: led@12 { + label = "led12"; + reg = <12>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led13: led@13 { + label = "led13"; + reg = <13>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led14: led@14 { + label = "led14"; + reg = <14>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led15: led@15 { + label = "led15"; + reg = <15>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led16: led@16 { + label = "led16"; + reg = <16>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led17: led@17 { + label = "led17"; + reg = <17>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led18: led@18 { + label = "led18"; + reg = <18>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led19: led@19 { + label = "led19"; + reg = <19>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led20: led@20 { + label = "led20"; + reg = <20>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led21: led@21 { + label = "led21"; + reg = <21>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led22: led@22 { + label = "led22"; + reg = <22>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led23: led@23 { + label = "led23"; + reg = <23>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led124: led@24 { + label = "led24"; + reg = <24>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led25: led@25 { + label = "led25"; + reg = <25>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led26: led@26 { + label = "led26"; + reg = <26>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led27: led@27 { + label = "led27"; + reg = <27>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led28: led@28 { + label = "led28"; + reg = <28>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led29: led@29 { + label = "led29"; + reg = <29>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led30: led@30 { + label = "led30"; + reg = <30>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led31: led@31 { + label = "led31"; + reg = <31>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led32: led@32 { + label = "led32"; + reg = <32>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led33: led@33 { + label = "led33"; + reg = <33>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led34: led@34 { + label = "led34"; + reg = <34>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led35: led@35 { + label = "led35"; + reg = <35>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led36: led@36 { + label = "led36"; + reg = <36>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + }; + + ls_stk3410: light@48 { + compatible = "ls_stk3410"; + status = "okay"; + reg = <0x48>; + type = ; + irq_enable = <0>; + als_threshold_high = <100>; + als_threshold_low = <10>; + als_ctrl_gain = <2>; /* 0:x1 1:x4 2:x16 3:x64 */ + poll_delay_ms = <100>; + }; + + ps_stk3410: proximity@48 { + compatible = "ps_stk3410"; + status = "okay"; + reg = <0x48>; + type = ; + //pinctrl-names = "default"; + //pinctrl-0 = <&gpio2_c3>; + //irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; + //irq_enable = <1>; + ps_threshold_high = <0x200>; + ps_threshold_low = <0x100>; + ps_ctrl_gain = <3>; /* 0:x1 1:x4 2:x16 3:x64 */ + ps_led_current = <3>; /* 0:12.5mA 1:25mA 2:50mA 3:100mA */ + poll_delay_ms = <100>; + }; + +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + //pinctrl-names = "default"; + //pinctrl-0 = <&cif_clkout_m0>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s0_8ch { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,no-dmaengine; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; + pinctrl-0 = <&i2s1_2ch_sclk + &i2s1_2ch_lrck + &i2s1_2ch_sdo>; +}; + +&i2s2_2ch { + status = "okay"; + rockchip,bclk-fs = <64>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc1v8_dvp>; + vccio4-supply = <&vcc1v8_soc>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pdm { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,no-dmaengine; + pinctrl-names = "default"; + pinctrl-0 = <&pdm_clk0m1 + &pdm_sdi0m1>; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&tp_int>; + + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + touchscreen-int { + tp_int: tp-int { + rockchip,pins = <0 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + vbus-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx_out>; + }; + + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11.dts new file mode 100755 index 000000000000..fa8b7cff619f --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v11.dts @@ -0,0 +1,1317 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "rk3326.dtsi" +#include "rk3326-863-cif-sensor.dtsi" +#include "px30-android.dtsi" + +/ { + model = "Rockchip rk3326 ai voice assistant evb v11 board"; + compatible = "rockchip,rk3326-evb-ai-va-v11", "rockchip,rk3326"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + mute-key { + linux,code = ; + label = "mute"; + press-threshold-microvolt = <1119000>; + }; + + mode-key { + linux,code = ; + label = "mode"; + press-threshold-microvolt = <892000>; + }; + + media-key { + linux,code = ; + label = "media"; + press-threshold-microvolt = <616000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <15000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_2ch>; + }; + codec { + sound-dai = <&rk809_codec 0>; + }; + }; + simple-audio-card,dai-link@1 { + format = "pdm"; + cpu { + sound-dai = <&pdm>; + }; + codec { + sound-dai = <&rk809_codec 1>; + }; + }; + }; + + bt-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "dsp_a"; + simple-audio-card,bitclock-inversion = <1>; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,bt"; + simple-audio-card,cpu { + sound-dai = <&i2s2_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&bt_sco>; + }; + }; + + bt_sco: bt-sco { + compatible = "delta,dfbmcs320"; + #sound-dai-cells = <0>; + status = "okay"; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + test-power { + status = "okay"; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6255"; + WIFI,host_wake_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PA7 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&display_subsystem { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <40>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <66000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <280>; + i2c-scl-falling-time-ns = <16>; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <10 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc5v0_host: SWITCH_REG1 { + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <1>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + pdmdata-out-enable; + use-ext-amplifier; + adc-for-loopback; + spk-ctl-gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_HIGH>; + hp-volume = <20>; + spk-volume = <20>; + }; + }; +}; + +&i2c1 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <275>; + i2c-scl-falling-time-ns = <16>; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PB3 IRQ_TYPE_LEVEL_LOW>; + }; + + is31fl3236: led-controller@3c { + compatible = "issi,is31fl3236"; + reg = <0x3c>; + #address-cells = <1>; + #size-cells = <0>; + reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; + status = "okay"; + + led1: led@1 { + label = "led1"; + reg = <1>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led2: led@2 { + label = "led2"; + reg = <2>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led3: led@3 { + label = "led3"; + reg = <3>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led4: led@4 { + label = "led4"; + reg = <4>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led5: led@5 { + label = "led5"; + reg = <5>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led6: led@6 { + label = "led6"; + reg = <6>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led7: led@7 { + label = "led7"; + reg = <7>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led8: led@8 { + label = "led8"; + reg = <8>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led9: led@9 { + label = "led9"; + reg = <9>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led10: led@10 { + label = "led10"; + reg = <10>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led11: led@11 { + label = "led11"; + reg = <11>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led12: led@12 { + label = "led12"; + reg = <12>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led13: led@13 { + label = "led13"; + reg = <13>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led14: led@14 { + label = "led14"; + reg = <14>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led15: led@15 { + label = "led15"; + reg = <15>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led16: led@16 { + label = "led16"; + reg = <16>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led17: led@17 { + label = "led17"; + reg = <17>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led18: led@18 { + label = "led18"; + reg = <18>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led19: led@19 { + label = "led19"; + reg = <19>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led20: led@20 { + label = "led20"; + reg = <20>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led21: led@21 { + label = "led21"; + reg = <21>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led22: led@22 { + label = "led22"; + reg = <22>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led23: led@23 { + label = "led23"; + reg = <23>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led124: led@24 { + label = "led24"; + reg = <24>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led25: led@25 { + label = "led25"; + reg = <25>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led26: led@26 { + label = "led26"; + reg = <26>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led27: led@27 { + label = "led27"; + reg = <27>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led28: led@28 { + label = "led28"; + reg = <28>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led29: led@29 { + label = "led29"; + reg = <29>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led30: led@30 { + label = "led30"; + reg = <30>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led31: led@31 { + label = "led31"; + reg = <31>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led32: led@32 { + label = "led32"; + reg = <32>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led33: led@33 { + label = "led33"; + reg = <33>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led34: led@34 { + label = "led34"; + reg = <34>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led35: led@35 { + label = "led35"; + reg = <35>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led36: led@36 { + label = "led36"; + reg = <36>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + }; + + ls_stk3410: light@48 { + compatible = "ls_stk3410"; + status = "okay"; + reg = <0x48>; + type = ; + irq_enable = <0>; + als_threshold_high = <100>; + als_threshold_low = <10>; + als_ctrl_gain = <2>; /* 0:x1 1:x4 2:x16 3:x64 */ + poll_delay_ms = <100>; + }; + + ps_stk3410: proximity@48 { + compatible = "ps_stk3410"; + status = "okay"; + reg = <0x48>; + type = ; + //pinctrl-names = "default"; + //pinctrl-0 = <&gpio2_c3>; + //irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; + //irq_enable = <1>; + ps_threshold_high = <0x200>; + ps_threshold_low = <0x100>; + ps_ctrl_gain = <3>; /* 0:x1 1:x4 2:x16 3:x64 */ + ps_led_current = <3>; /* 0:12.5mA 1:25mA 2:50mA 3:100mA */ + poll_delay_ms = <100>; + }; + +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + //pinctrl-names = "default"; + //pinctrl-0 = <&cif_clkout_m0>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; + pinctrl-0 = <&i2s1_2ch_sclk + &i2s1_2ch_lrck + &i2s1_2ch_sdo>; +}; + +&i2s2_2ch { + status = "okay"; + rockchip,bclk-fs = <64>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc1v8_dvp>; + vccio4-supply = <&vcc1v8_soc>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pdm { + status = "okay"; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pdm_clk0m1 + &pdm_clk1 + &pdm_sdi0m1 + &pdm_sdi1 + &pdm_sdi2 + &pdm_sdi3>; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&tp_int>; + + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + touchscreen-int { + tp_int: tp-int { + rockchip,pins = <0 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + vbus-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx_out>; + }; + + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v12.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v12.dts new file mode 100755 index 000000000000..9669993499e0 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-ai-va-v12.dts @@ -0,0 +1,1317 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "rk3326.dtsi" +#include "rk3326-863-cif-sensor.dtsi" +#include "px30-android.dtsi" + +/ { + model = "Rockchip rk3326 ai voice assistant evb v12 board"; + compatible = "rockchip,rk3326-evb-ai-va-v12", "rockchip,rk3326"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + mute-key { + linux,code = ; + label = "mute"; + press-threshold-microvolt = <1119000>; + }; + + mode-key { + linux,code = ; + label = "mode"; + press-threshold-microvolt = <892000>; + }; + + media-key { + linux,code = ; + label = "media"; + press-threshold-microvolt = <616000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <15000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_2ch>; + }; + codec { + sound-dai = <&rk809_codec 0>; + }; + }; + simple-audio-card,dai-link@1 { + format = "pdm"; + cpu { + sound-dai = <&pdm>; + }; + codec { + sound-dai = <&rk809_codec 1>; + }; + }; + }; + + bt-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "dsp_a"; + simple-audio-card,bitclock-inversion = <1>; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,bt"; + simple-audio-card,cpu { + sound-dai = <&i2s2_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&bt_sco>; + }; + }; + + bt_sco: bt-sco { + compatible = "delta,dfbmcs320"; + #sound-dai-cells = <0>; + status = "okay"; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + test-power { + status = "okay"; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6256"; + WIFI,host_wake_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PA7 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&display_subsystem { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <40>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <66000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <280>; + i2c-scl-falling-time-ns = <16>; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <10 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc3v3_sys: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc5v0_host: SWITCH_REG1 { + regulator-name = "vcc5v0_host"; + }; + + vcc3v3_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc3v3_lcd"; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <1>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + pdmdata-out-enable; + use-ext-amplifier; + adc-for-loopback; + spk-ctl-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + hp-volume = <20>; + spk-volume = <20>; + }; + }; +}; + +&i2c1 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <275>; + i2c-scl-falling-time-ns = <16>; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PB3 IRQ_TYPE_LEVEL_LOW>; + }; + + is31fl3236: led-controller@3c { + compatible = "issi,is31fl3236"; + reg = <0x3c>; + #address-cells = <1>; + #size-cells = <0>; + reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; + status = "okay"; + + led1: led@1 { + label = "led1"; + reg = <1>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led2: led@2 { + label = "led2"; + reg = <2>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <0>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led3: led@3 { + label = "led3"; + reg = <3>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led4: led@4 { + label = "led4"; + reg = <4>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led5: led@5 { + label = "led5"; + reg = <5>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led6: led@6 { + label = "led6"; + reg = <6>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led7: led@7 { + label = "led7"; + reg = <7>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led8: led@8 { + label = "led8"; + reg = <8>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <200>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led9: led@9 { + label = "led9"; + reg = <9>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led10: led@10 { + label = "led10"; + reg = <10>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led11: led@11 { + label = "led11"; + reg = <11>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <300>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led12: led@12 { + label = "led12"; + reg = <12>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led13: led@13 { + label = "led13"; + reg = <13>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led14: led@14 { + label = "led14"; + reg = <14>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <400>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led15: led@15 { + label = "led15"; + reg = <15>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led16: led@16 { + label = "led16"; + reg = <16>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led17: led@17 { + label = "led17"; + reg = <17>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <500>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led18: led@18 { + label = "led18"; + reg = <18>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led19: led@19 { + label = "led19"; + reg = <19>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led20: led@20 { + label = "led20"; + reg = <20>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <600>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led21: led@21 { + label = "led21"; + reg = <21>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led22: led@22 { + label = "led22"; + reg = <22>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led23: led@23 { + label = "led23"; + reg = <23>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <700>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led124: led@24 { + label = "led24"; + reg = <24>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led25: led@25 { + label = "led25"; + reg = <25>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led26: led@26 { + label = "led26"; + reg = <26>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <800>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led27: led@27 { + label = "led27"; + reg = <27>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led28: led@28 { + label = "led28"; + reg = <28>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led29: led@29 { + label = "led29"; + reg = <29>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <900>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led30: led@30 { + label = "led30"; + reg = <30>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led31: led@31 { + label = "led31"; + reg = <31>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led32: led@32 { + label = "led32"; + reg = <32>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1000>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led33: led@33 { + label = "led33"; + reg = <33>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + + led34: led@34 { + label = "led34"; + reg = <34>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led35: led@35 { + label = "led35"; + reg = <35>; + led-max-microamp = <10000>; + linux,default-trigger = "timer"; + linux,default-trigger-delay-ms = <1100>; + linux,blink-delay-on-ms = <100>; + linux,blink-delay-off-ms = <1200>; + }; + + led36: led@36 { + label = "led36"; + reg = <36>; + led-max-microamp = <10000>; + linux,default-trigger = "default-on"; + }; + }; + + ls_stk3410: light@48 { + compatible = "ls_stk3410"; + status = "okay"; + reg = <0x48>; + type = ; + irq_enable = <0>; + als_threshold_high = <100>; + als_threshold_low = <10>; + als_ctrl_gain = <2>; /* 0:x1 1:x4 2:x16 3:x64 */ + poll_delay_ms = <100>; + }; + + ps_stk3410: proximity@48 { + compatible = "ps_stk3410"; + status = "okay"; + reg = <0x48>; + type = ; + //pinctrl-names = "default"; + //pinctrl-0 = <&gpio2_c3>; + //irq-gpio = <&gpio0 RK_PB7 IRQ_TYPE_LEVEL_LOW>; + //irq_enable = <1>; + ps_threshold_high = <0x200>; + ps_threshold_low = <0x100>; + ps_ctrl_gain = <3>; /* 0:x1 1:x4 2:x16 3:x64 */ + ps_led_current = <3>; /* 0:12.5mA 1:25mA 2:50mA 3:100mA */ + poll_delay_ms = <100>; + }; + +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + //pinctrl-names = "default"; + //pinctrl-0 = <&cif_clkout_m0>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; + pinctrl-0 = <&i2s1_2ch_sclk + &i2s1_2ch_lrck + &i2s1_2ch_sdo>; +}; + +&i2s2_2ch { + status = "okay"; + rockchip,bclk-fs = <64>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc1v8_dvp>; + vccio4-supply = <&vcc1v8_soc>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pdm { + status = "okay"; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pdm_clk0m1 + &pdm_clk1 + &pdm_sdi0m1 + &pdm_sdi1 + &pdm_sdi2 + &pdm_sdi3>; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&tp_int>; + + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + touchscreen-int { + tp_int: tp-int { + rockchip,pins = <0 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + vbus-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx_out>; + }; + + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-avb.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-avb.dts new file mode 100755 index 000000000000..4c12a79f814b --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-avb.dts @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3326-evb-lp3-v10.dtsi" + +/ { + model = "Rockchip rk3326 evb board"; + compatible = "rockchip,rk3326-evb-lp3-v10-avb", "rockchip,rk3326"; +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + //pinctrl-names = "default"; + //pinctrl-0 = <&cif_clkout_m0>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx_out>; + }; + + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-linux.dts new file mode 100755 index 000000000000..67c131c9618a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-linux.dts @@ -0,0 +1,1024 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "rk3326.dtsi" +#include "rk3326-linux.dtsi" + +/ { + model = "Rockchip rk3326 evb lpddr3 v10 board for linux"; + compatible = "rockchip,rk3326-evb-lp3-v10-linux", "rockchip,rk3326"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff160000 console=ttyFIQ0 rw root=PARTUUID=614e0000-0000 rootfstype=ext4 rootwait"; + }; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "MIC_IN", "Microphone Jack", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + rk_headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + /*clocks = <&rk817 1>;*/ + /*clock-names = "ext_clock";*/ + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + vcc18_lcd_n: vcc18-lcd-n { + compatible = "regulator-fixed"; + regulator-name = "vcc18_lcd_n"; + regulator-boot-on; + gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cif_new { + status = "okay"; + + port { + cif_in: endpoint { + remote-endpoint = <&gc2155_out>; + vsync-active = <0>; + hsync-active = <1>; + }; + }; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&display_subsystem { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + power-supply = <&vcc18_lcd_n>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <20>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <66000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <280>; + i2c-scl-falling-time-ns = <16>; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vcc_3v0>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_ts_gpio1: rk817_ts_gpio1 { + pins = "gpio_ts"; + function = "pin_fun1"; + /* output-low; */ + /* input-enable; */ + }; + + rk817_gt_gpio2: rk817_gt_gpio2 { + pins = "gpio_gt"; + function = "pin_fun1"; + }; + + rk817_pin_ts: rk817_pin_ts { + pins = "gpio_ts"; + function = "pin_fun0"; + }; + + rk817_pin_gt: rk817_pin_gt { + pins = "gpio_gt"; + function = "pin_fun0"; + }; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3500 3625 3685 3697 3718 3735 3748 + 3760 3774 3788 3802 3816 3834 3853 + 3877 3908 3946 3975 4018 4071 4106>; + design_capacity = <2500>; + design_qmax = <2750>; + bat_res = <100>; + sleep_enter_current = <300>; + sleep_exit_current = <300>; + sleep_filter_current = <100>; + power_off_thresd = <3500>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <1>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4200>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&u2phy>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <275>; + i2c-scl-falling-time-ns = <16>; + + sensor@f { + status = "okay"; + compatible = "ak8963"; + reg = <0x0f>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <1>; + reprobe_en = <1>; + }; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + power-supply = <&vcc18_lcd_n>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; + }; + + sensor@4c { + status = "okay"; + compatible = "gs_mma7660"; + reg = <0x4c>; + type = ; + irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <1>; + reprobe_en = <1>; + }; +}; + +&i2c2 { + status = "okay"; + clock-frequency = <400000>; + + /* 24M mclk is shared for multiple cameras */ + pinctrl-0 = <&i2c2_xfer &cif_clkout_m0>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + gc2155: gc2155@3c { + compatible = "gc,gc2155"; + reg = <0x3c>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_pin_m0>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vcc1v8_dvp>; + + /* hw changed the pwdn to gpio2_b5 */ + pwdn-gpios = <&gpio2 13 GPIO_ACTIVE_HIGH>; + + port { + gc2155_out: endpoint { + remote-endpoint = <&cif_in>; + }; + }; + }; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vdd1v5_dvp>; + + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + + port { + ucam_out: endpoint { + remote-endpoint = <&mipi_in_ucam>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc1v8_dvp>; + vccio4-supply = <&vcc_3v0>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&rkisp1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&vip_mmu { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&pinctrl { + cif-pin-m0 { + cif_pin_m0: cif-pin-m0 { + rockchip,pins = + <2 RK_PA0 1 &pcfg_pull_none>,/* cif_data2 */ + <2 RK_PA1 1 &pcfg_pull_none>,/* cif_data3 */ + <2 RK_PA2 1 &pcfg_pull_none>,/* cif_data4 */ + <2 RK_PA3 1 &pcfg_pull_none>,/* cif_data5 */ + <2 RK_PA4 1 &pcfg_pull_none>,/* cif_data6 */ + <2 RK_PA5 1 &pcfg_pull_none>,/* cif_data7 */ + <2 RK_PA6 1 &pcfg_pull_none>,/* cif_data8 */ + <2 RK_PA7 1 &pcfg_pull_none>,/* cif_data9 */ + <2 RK_PB0 1 &pcfg_pull_none>,/* cif_sync */ + <2 RK_PB1 1 &pcfg_pull_none>,/* cif_href */ + <2 RK_PB2 1 &pcfg_pull_none>;/* cif_clkin */ + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-linux.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-linux.dts new file mode 100755 index 000000000000..6966806150f5 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-linux.dts @@ -0,0 +1,748 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include "px30-robot.dtsi" + +/ { + model = "Rockchip rk3326 evb lpddr3 v10 board for robot linux"; + compatible = "rockchip,rk3326-evb-lp3-v10-robot-linux", "rockchip,rk3326"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + /*clocks = <&rk817 1>;*/ + /*clock-names = "ext_clock";*/ + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cif_new { + status = "okay"; + + port { + cif_in: endpoint { + remote-endpoint = <&gc2155_out>; + vsync-active = <0>; + hsync-active = <1>; + }; + }; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <280>; + i2c-scl-falling-time-ns = <16>; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vcc_3v0>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_ts_gpio1: rk817_ts_gpio1 { + pins = "gpio_ts"; + function = "pin_fun1"; + /* output-low; */ + /* input-enable; */ + }; + + rk817_gt_gpio2: rk817_gt_gpio2 { + pins = "gpio_gt"; + function = "pin_fun1"; + }; + + rk817_pin_ts: rk817_pin_ts { + pins = "gpio_ts"; + function = "pin_fun0"; + }; + + rk817_pin_gt: rk817_pin_gt { + pins = "gpio_gt"; + function = "pin_fun0"; + }; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3500 3625 3685 3697 3718 3735 3748 + 3760 3774 3788 3802 3816 3834 3853 + 3877 3908 3946 3975 4018 4071 4106>; + design_capacity = <2500>; + design_qmax = <2750>; + bat_res = <100>; + sleep_enter_current = <300>; + sleep_exit_current = <300>; + sleep_filter_current = <100>; + power_off_thresd = <3500>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <1>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4200>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&u2phy>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c2 { + status = "okay"; + clock-frequency = <400000>; + + /* 24M mclk is shared for multiple cameras */ + pinctrl-0 = <&i2c2_xfer &cif_clkout_m0>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + gc2155: gc2155@3c { + compatible = "gc,gc2155"; + reg = <0x3c>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_pin_m0>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vcc1v8_dvp>; + + /* hw changed the pwdn to gpio2_b5 */ + pwdn-gpios = <&gpio2 13 GPIO_ACTIVE_HIGH>; + + port { + gc2155_out: endpoint { + remote-endpoint = <&cif_in>; + }; + }; + }; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vcc1v8_dvp>; + + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + + port { + ucam_out: endpoint { + remote-endpoint = <&mipi_in_ucam>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc1v8_dvp>; + vccio4-supply = <&vcc_3v0>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&rkisp1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&rk_rga { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "init", "default"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&vip_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&pinctrl { + cif-pin-m0 { + cif_pin_m0: cif-pin-m0 { + rockchip,pins = + <2 RK_PA0 1 &pcfg_pull_none>,/* cif_data2 */ + <2 RK_PA1 1 &pcfg_pull_none>,/* cif_data3 */ + <2 RK_PA2 1 &pcfg_pull_none>,/* cif_data4 */ + <2 RK_PA3 1 &pcfg_pull_none>,/* cif_data5 */ + <2 RK_PA4 1 &pcfg_pull_none>,/* cif_data6 */ + <2 RK_PA5 1 &pcfg_pull_none>,/* cif_data7 */ + <2 RK_PA6 1 &pcfg_pull_none>,/* cif_data8 */ + <2 RK_PA7 1 &pcfg_pull_none>,/* cif_data9 */ + <2 RK_PB0 1 &pcfg_pull_none>,/* cif_sync */ + <2 RK_PB1 1 &pcfg_pull_none>,/* cif_href */ + <2 RK_PB2 1 &pcfg_pull_none>;/* cif_clkin */ + }; + + cif_pin_m1: cif-pin-m1 { + rockchip,pins = + <3 RK_PA3 3 &pcfg_pull_none>,/* cif_data2 */ + <3 RK_PA5 3 &pcfg_pull_none>,/* cif_data3 */ + <3 RK_PA7 3 &pcfg_pull_none>,/* cif_data4 */ + <3 RK_PB0 3 &pcfg_pull_none>,/* cif_data5 */ + <3 RK_PB1 3 &pcfg_pull_none>,/* cif_data6 */ + <3 RK_PB4 3 &pcfg_pull_none>,/* cif_data7 */ + <3 RK_PB6 3 &pcfg_pull_none>,/* cif_data8 */ + <3 RK_PB7 3 &pcfg_pull_none>,/* cif_data9 */ + <3 RK_PD1 3 &pcfg_pull_none>,/* cif_sync */ + <3 RK_PD2 3 &pcfg_pull_none>,/* cif_href */ + <3 RK_PD3 3 &pcfg_pull_none>;/* cif_clkin */ + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-no-gpu-linux.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-no-gpu-linux.dts new file mode 100755 index 000000000000..b3b9efec92e4 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10-robot-no-gpu-linux.dts @@ -0,0 +1,728 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include "px30-robot-no-gpu.dtsi" + +/ { + model = "Rockchip rk3326 evb lpddr3 v10 board for robot linux"; + compatible = "rockchip,rk3326-evb-lp3-v10-robot-linux", "rockchip,rk3326"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + /*clocks = <&rk817 1>;*/ + /*clock-names = "ext_clock";*/ + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cif_new { + status = "okay"; + + port { + cif_in: endpoint { + remote-endpoint = <&gc2155_out>; + vsync-active = <0>; + hsync-active = <1>; + }; + }; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <280>; + i2c-scl-falling-time-ns = <16>; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vcc_3v0>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_ts_gpio1: rk817_ts_gpio1 { + pins = "gpio_ts"; + function = "pin_fun1"; + /* output-low; */ + /* input-enable; */ + }; + + rk817_gt_gpio2: rk817_gt_gpio2 { + pins = "gpio_gt"; + function = "pin_fun1"; + }; + + rk817_pin_ts: rk817_pin_ts { + pins = "gpio_ts"; + function = "pin_fun0"; + }; + + rk817_pin_gt: rk817_pin_gt { + pins = "gpio_gt"; + function = "pin_fun0"; + }; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3500 3625 3685 3697 3718 3735 3748 + 3760 3774 3788 3802 3816 3834 3853 + 3877 3908 3946 3975 4018 4071 4106>; + design_capacity = <2500>; + design_qmax = <2750>; + bat_res = <100>; + sleep_enter_current = <300>; + sleep_exit_current = <300>; + sleep_filter_current = <100>; + power_off_thresd = <3500>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <1>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4200>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&u2phy>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c2 { + status = "okay"; + clock-frequency = <400000>; + + /* 24M mclk is shared for multiple cameras */ + pinctrl-0 = <&i2c2_xfer &cif_clkout_m0>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + gc2155: gc2155@3c { + compatible = "gc,gc2155"; + reg = <0x3c>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_pin_m0>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vcc1v8_dvp>; + + /* hw changed the pwdn to gpio2_b5 */ + pwdn-gpios = <&gpio2 13 GPIO_ACTIVE_HIGH>; + + port { + gc2155_out: endpoint { + remote-endpoint = <&cif_in>; + }; + }; + }; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vcc1v8_dvp>; + + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + + port { + ucam_out: endpoint { + remote-endpoint = <&mipi_in_ucam>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc1v8_dvp>; + vccio4-supply = <&vcc_3v0>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&rkisp1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&rk_rga { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "init", "default"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&vip_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&pinctrl { + cif-pin-m0 { + cif_pin_m0: cif-pin-m0 { + rockchip,pins = + <2 RK_PA0 1 &pcfg_pull_none>,/* cif_data2 */ + <2 RK_PA1 1 &pcfg_pull_none>,/* cif_data3 */ + <2 RK_PA2 1 &pcfg_pull_none>,/* cif_data4 */ + <2 RK_PA3 1 &pcfg_pull_none>,/* cif_data5 */ + <2 RK_PA4 1 &pcfg_pull_none>,/* cif_data6 */ + <2 RK_PA5 1 &pcfg_pull_none>,/* cif_data7 */ + <2 RK_PA6 1 &pcfg_pull_none>,/* cif_data8 */ + <2 RK_PA7 1 &pcfg_pull_none>,/* cif_data9 */ + <2 RK_PB0 1 &pcfg_pull_none>,/* cif_sync */ + <2 RK_PB1 1 &pcfg_pull_none>,/* cif_href */ + <2 RK_PB2 1 &pcfg_pull_none>;/* cif_clkin */ + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dts new file mode 100755 index 000000000000..58bbfdafb489 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3326-evb-lp3-v10.dtsi" + +/ { + model = "Rockchip rk3326 evb board"; + compatible = "rockchip,rk3326-evb-lp3-v10", "rockchip,rk3326"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; + +&rk_isp { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dtsi new file mode 100755 index 000000000000..940c05df8304 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v10.dtsi @@ -0,0 +1,887 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include +#include +#include "rk3326.dtsi" +#include "rk3326-863-cif-sensor.dtsi" +#include "px30-android.dtsi" + +/ { + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + /*clocks = <&rk817 1>;*/ + /*clock-names = "ext_clock";*/ + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "AP6210"; + WIFI,host_wake_irq = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio1 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart1_rts>; + pinctrl-1 = <&uart1_rts_gpio>; + BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + vcc18_lcd_n: vcc18-lcd-n { + compatible = "regulator-fixed"; + regulator-name = "vcc18_lcd_n"; + regulator-boot-on; + gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + enable-active-high; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; +}; + +&bus_apll { + bus-supply = <&vdd_logic>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&display_subsystem { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + power-supply = <&vcc18_lcd_n>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <20>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 05 fa 01 11 + 39 00 04 b9 f1 12 83 + 39 00 1c ba 33 81 05 f9 0e 0e 00 00 00 + 00 00 00 00 00 44 25 00 91 0a + 00 00 02 4f 01 00 00 37 + 15 00 02 b8 25 + 39 00 04 bf 02 11 00 + 39 00 0b b3 0c 10 0a 50 03 ff 00 00 00 + 00 + 39 00 0a c0 73 73 50 50 00 00 08 70 00 + 15 00 02 bc 46 + 15 00 02 cc 0b + 15 00 02 b4 80 + 39 00 04 b2 c8 12 30 + 39 00 0f e3 07 07 0b 0b 03 0b 00 00 00 + 00 ff 00 c0 10 + 39 00 0d c1 53 00 1e 1e 77 e1 cc dd 67 + 77 33 33 + 39 00 07 c6 00 00 ff ff 01 ff + 39 00 03 b5 09 09 + 39 00 03 b6 87 95 + 39 00 40 e9 c2 10 05 05 10 05 a0 12 31 + 23 3f 81 0a a0 37 18 00 80 01 + 00 00 00 00 80 01 00 00 00 48 + f8 86 42 08 88 88 80 88 88 88 + 58 f8 87 53 18 88 88 81 88 88 + 88 00 00 00 01 00 00 00 00 00 + 00 00 00 00 + 39 00 3e ea 00 1a 00 00 00 00 02 00 00 + 00 00 00 1f 88 81 35 78 88 88 + 85 88 88 88 0f 88 80 24 68 88 + 88 84 88 88 88 23 10 00 00 1c + 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 30 05 a0 00 00 + 00 00 + 39 00 23 e0 00 06 08 2a 31 3f 38 36 07 + 0c 0d 11 13 12 13 11 18 00 06 + 08 2a 31 3f 38 36 07 0c 0d 11 + 13 12 13 11 18 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <66000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&route_dsi { + connect = <&vopb_out_dsi>; + status = "okay"; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_logic>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <280>; + i2c-scl-falling-time-ns = <16>; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <7 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_rst>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vcc_3v0>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_ts_gpio1: rk817_ts_gpio1 { + pins = "gpio_ts"; + function = "pin_fun1"; + /* output-low; */ + /* input-enable; */ + }; + + rk817_gt_gpio2: rk817_gt_gpio2 { + pins = "gpio_gt"; + function = "pin_fun1"; + }; + + rk817_pin_ts: rk817_pin_ts { + pins = "gpio_ts"; + function = "pin_fun0"; + }; + + rk817_pin_gt: rk817_pin_gt { + pins = "gpio_gt"; + function = "pin_fun0"; + }; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_arm"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v0_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc3v0_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3500 3625 3685 3697 3718 3735 3748 + 3760 3774 3788 3802 3816 3834 3853 + 3877 3908 3946 3975 4018 4071 4106>; + design_capacity = <2500>; + design_qmax = <2750>; + bat_res = <100>; + sleep_enter_current = <300>; + sleep_exit_current = <300>; + sleep_filter_current = <100>; + power_off_thresd = <3500>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <1>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4200>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&u2phy>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S1_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1_2ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <275>; + i2c-scl-falling-time-ns = <16>; + + sensor@f { + status = "okay"; + compatible = "ak8963"; + reg = <0x0f>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <1>; + reprobe_en = <1>; + }; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + power-supply = <&vcc18_lcd_n>; + goodix,rst-gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PA5 IRQ_TYPE_LEVEL_LOW>; + }; + + sensor@4c { + status = "okay"; + compatible = "gs_mma7660"; + reg = <0x4c>; + type = ; + irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <1>; + reprobe_en = <1>; + }; +}; + +&i2c2 { + status = "okay"; +}; + +&i2s1_2ch { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc1v8_soc>; + vccio2-supply = <&vccio_sd>; + vccio3-supply = <&vcc1v8_dvp>; + vccio4-supply = <&vcc_3v0>; + vccio5-supply = <&vcc_3v0>; +}; + +&isp_mmu { + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA4 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA4 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmuio1-supply = <&vcc3v0_pmu>; + pmuio2-supply = <&vcc3v0_pmu>; +}; + +&pwm1 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc1v8_soc>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-sdio; + no-mmc; + card-detect-delay = <800>; + ignore-pm-notify; + /*cd-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; [> CD GPIO <]*/ + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + vqmmc-supply = <&vccio_sd>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + no-sd; + no-mmc; + ignore-pm-notify; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + status = "okay"; + }; +}; + +&usb20_otg { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11-avb.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11-avb.dts new file mode 100755 index 000000000000..2f0c3fc3a38d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11-avb.dts @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; +#include "rk3326-evb-lp3-v10.dtsi" + +/ { + model = "Rockchip rk3326 evb board"; + compatible = "rockchip,rk3326-evb-lp3-v11-avb", "rockchip,rk3326"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc18_lcd_n>; + backlight = <&backlight>; + prepare-delay-ms = <0>; + reset-delay-ms = <0>; + init-delay-ms = <80>; + enable-delay-ms = <0>; + disable-delay-ms = <10>; + unprepare-delay-ms = <60>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 39 00 04 ff 98 81 03 + 15 00 02 01 00 + 15 00 02 02 00 + 15 00 02 03 53 + 15 00 02 04 53 + 15 00 02 05 13 + 15 00 02 06 04 + 15 00 02 07 02 + 15 00 02 08 02 + 15 00 02 09 00 + 15 00 02 0a 00 + 15 00 02 0b 00 + 15 00 02 0c 00 + 15 00 02 0d 00 + 15 00 02 0e 00 + 15 00 02 0f 00 + + 15 00 02 10 00 + 15 00 02 11 00 + 15 00 02 12 00 + 15 00 02 13 00 + 15 00 02 14 00 + 15 00 02 15 08 + 15 00 02 16 10 + 15 00 02 17 00 + 15 00 02 18 08 + 15 00 02 19 00 + 15 00 02 1a 00 + 15 00 02 1b 00 + 15 00 02 1c 00 + 15 00 02 1d 00 + 15 00 02 1e c0 + 15 00 02 1f 80 + + 15 00 02 20 02 + 15 00 02 21 09 + 15 00 02 22 00 + 15 00 02 23 00 + 15 00 02 24 00 + 15 00 02 25 00 + 15 00 02 26 00 + 15 00 02 27 00 + 15 00 02 28 55 + 15 00 02 29 03 + 15 00 02 2a 00 + 15 00 02 2b 00 + 15 00 02 2c 00 + 15 00 02 2d 00 + 15 00 02 2e 00 + 15 00 02 2f 00 + + 15 00 02 30 00 + 15 00 02 31 00 + 15 00 02 32 00 + 15 00 02 33 00 + 15 00 02 34 04 + 15 00 02 35 05 + 15 00 02 36 05 + 15 00 02 37 00 + 15 00 02 38 3c + 15 00 02 39 35 + 15 00 02 3a 00 + 15 00 02 3b 40 + 15 00 02 3c 00 + 15 00 02 3d 00 + 15 00 02 3e 00 + 15 00 02 3f 00 + + 15 00 02 40 00 + 15 00 02 41 88 + 15 00 02 42 00 + 15 00 02 43 00 + 15 00 02 44 1f + + 15 00 02 50 01 + 15 00 02 51 23 + 15 00 02 52 45 + 15 00 02 53 67 + 15 00 02 54 89 + 15 00 02 55 ab + 15 00 02 56 01 + 15 00 02 57 23 + 15 00 02 58 45 + 15 00 02 59 67 + 15 00 02 5a 89 + 15 00 02 5b ab + 15 00 02 5c cd + 15 00 02 5d ef + 15 00 02 5e 03 + 15 00 02 5f 14 + + 15 00 02 60 15 + 15 00 02 61 0c + 15 00 02 62 0d + 15 00 02 63 0e + 15 00 02 64 0f + 15 00 02 65 10 + 15 00 02 66 11 + 15 00 02 67 08 + 15 00 02 68 02 + 15 00 02 69 0a + 15 00 02 6a 02 + 15 00 02 6b 02 + 15 00 02 6c 02 + 15 00 02 6d 02 + 15 00 02 6e 02 + 15 00 02 6f 02 + + 15 00 02 70 02 + 15 00 02 71 02 + 15 00 02 72 06 + 15 00 02 73 02 + 15 00 02 74 02 + 15 00 02 75 14 + 15 00 02 76 15 + 15 00 02 77 0f + 15 00 02 78 0e + 15 00 02 79 0d + 15 00 02 7a 0c + 15 00 02 7b 11 + 15 00 02 7c 10 + 15 00 02 7d 06 + 15 00 02 7e 02 + 15 00 02 7f 0a + + 15 00 02 80 02 + 15 00 02 81 02 + 15 00 02 82 02 + 15 00 02 83 02 + 15 00 02 84 02 + 15 00 02 85 02 + 15 00 02 86 02 + 15 00 02 87 02 + 15 00 02 88 08 + 15 00 02 89 02 + 15 00 02 8a 02 + + 39 00 04 ff 98 81 04 + 15 00 02 00 80 + 15 00 02 70 00 + 15 00 02 71 00 + 15 00 02 66 fe + 15 00 02 82 15 + 15 00 02 84 15 + 15 00 02 85 15 + 15 00 02 3a 24 + 15 00 02 32 ac + 15 00 02 8c 80 + 15 00 02 3c f5 + 15 00 02 88 33 + + 39 00 04 ff 98 81 01 + 15 00 02 22 0a + 15 00 02 31 00 + 15 00 02 53 78 + 15 00 02 50 5b + 15 00 02 51 5b + 15 00 02 60 20 + 15 00 02 61 00 + 15 00 02 62 0d + 15 00 02 63 00 + + 15 00 02 a0 00 + 15 00 02 a1 10 + 15 00 02 a2 1c + 15 00 02 a3 13 + 15 00 02 a4 15 + 15 00 02 a5 26 + 15 00 02 a6 1a + 15 00 02 a7 1d + 15 00 02 a8 67 + 15 00 02 a9 1c + 15 00 02 aa 29 + 15 00 02 ab 5b + 15 00 02 ac 26 + 15 00 02 ad 28 + 15 00 02 ae 5c + 15 00 02 af 30 + 15 00 02 b0 31 + 15 00 02 b1 2e + 15 00 02 b2 32 + 15 00 02 b3 00 + + 15 00 02 c0 00 + 15 00 02 c1 10 + 15 00 02 c2 1c + 15 00 02 c3 13 + 15 00 02 c4 15 + 15 00 02 c5 26 + 15 00 02 c6 1a + 15 00 02 c7 1d + 15 00 02 c8 67 + 15 00 02 c9 1c + 15 00 02 ca 29 + 15 00 02 cb 5b + 15 00 02 cc 26 + 15 00 02 cd 28 + 15 00 02 ce 5c + 15 00 02 cf 30 + 15 00 02 d0 31 + 15 00 02 d1 2e + 15 00 02 d2 32 + 15 00 02 d3 00 + 39 00 04 ff 98 81 00 + 05 00 01 11 + 05 01 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing1>; + + timing1: timing1 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; + +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <100000>; + + /* These are relatively safe rise/fall times; TODO: measure */ + i2c-scl-falling-time-ns = <50>; + i2c-scl-rising-time-ns = <300>; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /*reset-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>;*/ + pwdn-gpios = <&gpio2 14 GPIO_ACTIVE_HIGH>; + //pinctrl-names = "default"; + //pinctrl-0 = <&cif_clkout_m0>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout_m0 &dvp_d0d1_m0 &dvp_d2d9_m0>; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx_out>; + }; + + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11.dts b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11.dts new file mode 100755 index 000000000000..139efd9d1332 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-evb-lp3-v11.dts @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; +#include "rk3326-evb-lp3-v10.dtsi" + +/ { + model = "Rockchip rk3326 evb board"; + compatible = "rockchip,rk3326-evb-lp3-v11", "rockchip,rk3326"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc18_lcd_n>; + backlight = <&backlight>; + prepare-delay-ms = <0>; + reset-delay-ms = <0>; + init-delay-ms = <80>; + enable-delay-ms = <0>; + disable-delay-ms = <10>; + unprepare-delay-ms = <60>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 39 00 04 ff 98 81 03 + 15 00 02 01 00 + 15 00 02 02 00 + 15 00 02 03 53 + 15 00 02 04 53 + 15 00 02 05 13 + 15 00 02 06 04 + 15 00 02 07 02 + 15 00 02 08 02 + 15 00 02 09 00 + 15 00 02 0a 00 + 15 00 02 0b 00 + 15 00 02 0c 00 + 15 00 02 0d 00 + 15 00 02 0e 00 + 15 00 02 0f 00 + + 15 00 02 10 00 + 15 00 02 11 00 + 15 00 02 12 00 + 15 00 02 13 00 + 15 00 02 14 00 + 15 00 02 15 08 + 15 00 02 16 10 + 15 00 02 17 00 + 15 00 02 18 08 + 15 00 02 19 00 + 15 00 02 1a 00 + 15 00 02 1b 00 + 15 00 02 1c 00 + 15 00 02 1d 00 + 15 00 02 1e c0 + 15 00 02 1f 80 + + 15 00 02 20 02 + 15 00 02 21 09 + 15 00 02 22 00 + 15 00 02 23 00 + 15 00 02 24 00 + 15 00 02 25 00 + 15 00 02 26 00 + 15 00 02 27 00 + 15 00 02 28 55 + 15 00 02 29 03 + 15 00 02 2a 00 + 15 00 02 2b 00 + 15 00 02 2c 00 + 15 00 02 2d 00 + 15 00 02 2e 00 + 15 00 02 2f 00 + + 15 00 02 30 00 + 15 00 02 31 00 + 15 00 02 32 00 + 15 00 02 33 00 + 15 00 02 34 04 + 15 00 02 35 05 + 15 00 02 36 05 + 15 00 02 37 00 + 15 00 02 38 3c + 15 00 02 39 35 + 15 00 02 3a 00 + 15 00 02 3b 40 + 15 00 02 3c 00 + 15 00 02 3d 00 + 15 00 02 3e 00 + 15 00 02 3f 00 + + 15 00 02 40 00 + 15 00 02 41 88 + 15 00 02 42 00 + 15 00 02 43 00 + 15 00 02 44 1f + + 15 00 02 50 01 + 15 00 02 51 23 + 15 00 02 52 45 + 15 00 02 53 67 + 15 00 02 54 89 + 15 00 02 55 ab + 15 00 02 56 01 + 15 00 02 57 23 + 15 00 02 58 45 + 15 00 02 59 67 + 15 00 02 5a 89 + 15 00 02 5b ab + 15 00 02 5c cd + 15 00 02 5d ef + 15 00 02 5e 03 + 15 00 02 5f 14 + + 15 00 02 60 15 + 15 00 02 61 0c + 15 00 02 62 0d + 15 00 02 63 0e + 15 00 02 64 0f + 15 00 02 65 10 + 15 00 02 66 11 + 15 00 02 67 08 + 15 00 02 68 02 + 15 00 02 69 0a + 15 00 02 6a 02 + 15 00 02 6b 02 + 15 00 02 6c 02 + 15 00 02 6d 02 + 15 00 02 6e 02 + 15 00 02 6f 02 + + 15 00 02 70 02 + 15 00 02 71 02 + 15 00 02 72 06 + 15 00 02 73 02 + 15 00 02 74 02 + 15 00 02 75 14 + 15 00 02 76 15 + 15 00 02 77 0f + 15 00 02 78 0e + 15 00 02 79 0d + 15 00 02 7a 0c + 15 00 02 7b 11 + 15 00 02 7c 10 + 15 00 02 7d 06 + 15 00 02 7e 02 + 15 00 02 7f 0a + + 15 00 02 80 02 + 15 00 02 81 02 + 15 00 02 82 02 + 15 00 02 83 02 + 15 00 02 84 02 + 15 00 02 85 02 + 15 00 02 86 02 + 15 00 02 87 02 + 15 00 02 88 08 + 15 00 02 89 02 + 15 00 02 8a 02 + + 39 00 04 ff 98 81 04 + 15 00 02 00 80 + 15 00 02 70 00 + 15 00 02 71 00 + 15 00 02 66 fe + 15 00 02 82 15 + 15 00 02 84 15 + 15 00 02 85 15 + 15 00 02 3a 24 + 15 00 02 32 ac + 15 00 02 8c 80 + 15 00 02 3c f5 + 15 00 02 88 33 + + 39 00 04 ff 98 81 01 + 15 00 02 22 0a + 15 00 02 31 00 + 15 00 02 53 78 + 15 00 02 50 5b + 15 00 02 51 5b + 15 00 02 60 20 + 15 00 02 61 00 + 15 00 02 62 0d + 15 00 02 63 00 + + 15 00 02 a0 00 + 15 00 02 a1 10 + 15 00 02 a2 1c + 15 00 02 a3 13 + 15 00 02 a4 15 + 15 00 02 a5 26 + 15 00 02 a6 1a + 15 00 02 a7 1d + 15 00 02 a8 67 + 15 00 02 a9 1c + 15 00 02 aa 29 + 15 00 02 ab 5b + 15 00 02 ac 26 + 15 00 02 ad 28 + 15 00 02 ae 5c + 15 00 02 af 30 + 15 00 02 b0 31 + 15 00 02 b1 2e + 15 00 02 b2 32 + 15 00 02 b3 00 + + 15 00 02 c0 00 + 15 00 02 c1 10 + 15 00 02 c2 1c + 15 00 02 c3 13 + 15 00 02 c4 15 + 15 00 02 c5 26 + 15 00 02 c6 1a + 15 00 02 c7 1d + 15 00 02 c8 67 + 15 00 02 c9 1c + 15 00 02 ca 29 + 15 00 02 cb 5b + 15 00 02 cc 26 + 15 00 02 cd 28 + 15 00 02 ce 5c + 15 00 02 cf 30 + 15 00 02 d0 31 + 15 00 02 d1 2e + 15 00 02 d2 32 + 15 00 02 d3 00 + 39 00 04 ff 98 81 00 + 05 00 01 11 + 05 01 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing1>; + + timing1: timing1 { + clock-frequency = <64000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <22>; + vsync-len = <4>; + vback-porch = <11>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; + +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; +}; + +&rk_isp { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-linux.dtsi b/arch/arm64/boot/dts/rockchip/rk3326-linux.dtsi new file mode 100755 index 000000000000..a386cd291cf3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3326-linux.dtsi @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/ { + compatible = "rockchip,linux", "rockchip,rk3326"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff160000 console=ttyFIQ0 rw root=PARTUUID=614e0000-0000 rootwait"; + }; + + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <0>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "okay"; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + drm_logo: drm-logo@00000000 { + compatible = "rockchip,drm-logo"; + reg = <0x0 0x0 0x0 0x0>; + }; + }; +}; + +&cpu0_opp_table { + rockchip,avs = <1>; +}; + +&display_subsystem { + status = "disabled"; + ports = <&vopb_out>, <&vopl_out>; + logo-memory-region = <&drm_logo>; + + route { + route_lvds: route-lvds { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopb_out_lvds>; + }; + + route_dsi: route-dsi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopb_out_dsi>; + }; + + route_rgb: route-rgb { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopb_out_rgb>; + }; + }; +}; + +&rng { + status = "okay"; +}; + +&video_phy { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-android.dtsi new file mode 100755 index 000000000000..809b6501b78f --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-android.dtsi @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/ { + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff130000 kpti=0 coherent_pool=1m"; + }; + + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,signal-irq = <159>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <0>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + status = "okay"; + }; + + firmware { + firmware_android: android {}; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + drm_logo: drm-logo@00000000 { + compatible = "rockchip,drm-logo"; + reg = <0x0 0x0 0x0 0x0>; + }; + + ramoops: ramoops@110000 { + compatible = "ramoops"; + reg = <0x0 0x110000 0x0 0xf0000>; + record-size = <0x20000>; + console-size = <0x80000>; + ftrace-size = <0x00000>; + pmsg-size = <0x50000>; + }; + + secure_memory: secure-memory@20000000 { + compatible = "rockchip,secure-memory"; + reg = <0x0 0x20000000 0x0 0x0>; + }; + + /* global autoconfigured region for contiguous allocations */ + linux,cma { + compatible = "shared-dma-pool"; + reusable; + size = <0x0 0x2000000>; + linux,cma-default; + }; + }; +}; + +&display_subsystem { + logo-memory-region = <&drm_logo>; + status = "okay"; + secure-memory-region = <&secure_memory>; + route { + route_hdmi: route-hdmi { + status = "okay"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "fullscreen"; + charge_logo,mode = "fullscreen"; + connect = <&vop_out_hdmi>; + }; + route_tve: route-tve { + status = "okay"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "fullscreen"; + charge_logo,mode = "fullscreen"; + connect = <&vop_out_tve>; + }; + }; +}; + +&rng { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-box-liantong-avb.dts b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong-avb.dts new file mode 100755 index 000000000000..ee04d8988fbb --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong-avb.dts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + */ + +/dts-v1/; +#include "rk3328-box-liantong.dtsi" + +/ { + model = "Rockchip RK3328 box liantong avb"; + compatible = "rockchip,rk3328-box-liantong-avb", "rockchip,rk3328"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dts b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dts new file mode 100755 index 000000000000..dcff87208d39 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dts @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include "rk3328-box-liantong.dtsi" + +/ { + model = "Rockchip RK3328 box liantong"; + compatible = "rockchip,rk3328-box-liantong", "rockchip,rk3328"; +}; + +&firmware_android{ + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/platform/ff520000.dwmmc/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/platform/ff520000.dwmmc/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dtsi new file mode 100755 index 000000000000..ad7ef6be49fa --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-box-liantong.dtsi @@ -0,0 +1,673 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + */ + +#include "rk3328.dtsi" +#include "rk3328-android.dtsi" +#include "rk3328-box-plus-dram-timing.dtsi" +#include + +/ { + gmac_clkin: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "gmac_clkin"; + #clock-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio1 18 GPIO_ACTIVE_LOW>; + }; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip-rk3328"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&codec>; + }; + }; + + hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip-hdmi"; + simple-audio-card,cpu { + sound-dai = <&i2s0>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + regulators { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <0>; + + vccio_1v8_reg: regulator@0 { + compatible = "regulator-fixed"; + regulator-name = "vccio_1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + + vccio_3v3_reg: regulator@1 { + compatible = "regulator-fixed"; + regulator-name = "vccio_3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + }; + + rtc-fake { + compatible = "rtc-fake"; + status = "okay"; + }; + + spdif-sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip-spdif"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + vcc_host_vbus: host-vbus-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host_vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + }; + + vcc_otg_vbus: otg-vbus-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PA2 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&otg_vbus_drv>; + regulator-name = "vcc_otg_vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc_sd: sdmmc-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 30 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0m1_gpio>; + regulator-name = "vcc_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vccio_3v3_reg>; + }; + + vdd_arm: vdd-center { + compatible = "pwm-regulator"; + rockchip,pwm_id = <0>; + rockchip,pwm_voltage = <1250000>; + pwms = <&pwm0 0 5000 1>; + regulator-name = "vcc_arm"; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1400000>; + regulator-settling-time-up-us = <250>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_logic: vdd-log { + compatible = "pwm-regulator"; + rockchip,pwm_id = <1>; + rockchip,pwm_voltage = <1100000>; + pwms = <&pwm1 0 5000 1>; + regulator-name = "vcc_log"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <1300000>; + regulator-settling-time-up-us = <250>; + regulator-always-on; + regulator-boot-on; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio1 10 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_gpios>; + BT,power_gpio = <&gpio1 24 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio1 26 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "rtl8822bs"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio1 19 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&avsd { + status = "okay"; +}; + +&codec { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 786000 + SYS_STATUS_REBOOT 786000 + SYS_STATUS_SUSPEND 786000 + SYS_STATUS_VIDEO_1080P 786000 + SYS_STATUS_VIDEO_4K 786000 + SYS_STATUS_VIDEO_4K_10B 786000 + SYS_STATUS_PERFORMANCE 786000 + SYS_STATUS_BOOST 786000 + >; +}; + +&dmc_opp_table { + opp-800000000 { + status = "disabled"; + }; + + opp-850000000 { + status = "disabled"; + }; + + opp-933000000 { + status = "disabled"; + }; + + opp-1066000000 { + status = "disabled"; + }; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; + status = "okay"; +}; + +&gmac2io { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio1 RK_PC2 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_MAC2IO>, <&cru SCLK_MAC2IO_EXT>; + assigned-clock-parents = <&gmac_clkin>, <&gmac_clkin>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmiim1_pins>; + tx_delay = <0x26>; + rx_delay = <0x11>; + status = "disabled"; +}; + +&gmac2phy { + phy-supply = <&vcc_phy>; + clock_in_out = "output"; + assigned-clocks = <&cru SCLK_MAC2PHY_SRC>; + assigned-clock-rate = <50000000>; + assigned-clocks = <&cru SCLK_MAC2PHY>; + assigned-clock-parents = <&cru SCLK_MAC2PHY_SRC>; + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_logic>; +}; + +&hdmi { + #sound-dai-cells = <0>; + ddc-i2c-scl-high-time-ns = <9625>; + ddc-i2c-scl-low-time-ns = <10000>; + status = "okay"; +}; + +&hdmiphy { + rockchip,phy-table = + <165000000 0x07 0x0a 0x0a 0x0a 0x00 0x00 0x08 + 0x08 0x08 0x00 0xac 0xcc 0xcc 0xcc>, + <340000000 0x0b 0x0d 0x0d 0x0d 0x07 0x15 0x08 + 0x08 0x08 0x3f 0xac 0xcc 0xcd 0xdd>, + <594000000 0x10 0x1a 0x1a 0x1a 0x07 0x15 0x08 + 0x08 0x08 0x00 0xac 0xcc 0xcc 0xcc>; + status = "okay"; +}; + +&secure_memory { + /* + * enable like this: + * reg = <0x0 0x20000000 0x0 0x10000000>; + */ + reg = <0x0 0x20000000 0x0 0x0>; +}; + +&i2s0 { + #sound-dai-cells = <0>; + rockchip,bclk-fs = <128>; + status = "okay"; +}; + +&i2s1 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vccio_3v3_reg>; + vccio2-supply = <&vccio_1v8_reg>; + vccio3-supply = <&vccio_3v3_reg>; + vccio4-supply = <&vccio_1v8_reg>; + vccio5-supply = <&vccio_3v3_reg>; + vccio6-supply = <&vccio_3v3_reg>; + pmuio-supply = <&vccio_3v3_reg>; +}; + +&mpp_srv { + status = "okay"; +}; + +&pinctrl { + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + otg_vbus_drv: otg-vbus-drv { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <1 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pwm0 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm0_pin_pull_up>; +}; + +&pwm1 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin_pull_up>; +}; + +&pwm3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pwmir_pin>; + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <1>; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_PLAYPAUSE>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xa4 KEY_SETUP>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +&rga { + status = "okay"; +}; + +&rkvdec { + status = "okay"; + vcodec-supply = <&vdd_logic>; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,virtual-poweroff = <1>; + rockchip,sleep-mode-config = < + (0 + |RKPM_SLP_CTR_VOL_PWM0 + |RKPM_SLP_CTR_VOL_PWM1 + ) + >; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + cap-sdio-irq; + disable-wp; + keep-power-in-suspend; + max-frequency = <125000000>; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + no-sd; + no-mmc; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc_ext { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + keep-power-in-suspend; + max-frequency = <150000000>; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0ext_clk &sdmmc0ext_cmd &sdmmc0ext_dectn &sdmmc0ext_bus4>; + no-sdio; + no-mmc; + status = "okay"; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + max-frequency = <150000000>; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_dectn &sdmmc0_bus4>; + no-sdio; + no-mmc; + status = "okay"; + vmmc-supply = <&vcc_sd>; +}; + +&spdif { + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&spdifm0_tx>; + status = "okay"; +}; + +&threshold { + temperature = <90000>; /* millicelsius */ +}; + +&target { + temperature = <105000>; /* millicelsius */ +}; + +&soc_crit { + temperature = <115000>; /* millicelsius */ +}; + +&tsadc { + rockchip,hw-tshut-temp = <120000>; + status = "okay"; +}; + +&tve { + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + vbus-supply = <&vcc_otg_vbus>; + status = "okay"; + }; +}; + +&u3phy { + vbus-supply = <&vcc_host_vbus>; + status = "okay"; +}; + +&u3phy_utmi { + status = "okay"; +}; + +&u3phy_pipe { + status = "okay"; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usbdrd3 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vepu22 { + status = "okay"; +}; + +&vepu22_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-box-plus-dram-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-box-plus-dram-timing.dtsi new file mode 100755 index 000000000000..0ea270539a23 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-box-plus-dram-timing.dtsi @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + */ +#include +#include + +&ddr_timing { + /* CA de-skew, one step is 47.8ps, range 0-15 */ + ddr3a1_ddr4a9_de-skew = <0>; + ddr3a0_ddr4a10_de-skew = <0>; + ddr3a3_ddr4a6_de-skew = <1>; + ddr3a2_ddr4a4_de-skew = <1>; + ddr3a5_ddr4a8_de-skew = <0>; + ddr3a4_ddr4a5_de-skew = <2>; + ddr3a7_ddr4a11_de-skew = <0>; + ddr3a6_ddr4a7_de-skew = <2>; + ddr3a9_ddr4a0_de-skew = <1>; + ddr3a8_ddr4a13_de-skew = <0>; + ddr3a11_ddr4a3_de-skew = <2>; + ddr3a10_ddr4cs0_de-skew = <0>; + ddr3a13_ddr4a2_de-skew = <1>; + ddr3a12_ddr4ba1_de-skew = <0>; + ddr3a15_ddr4odt0_de-skew = <0>; + ddr3a14_ddr4a1_de-skew = <1>; + ddr3ba1_ddr4a15_de-skew = <0>; + ddr3ba0_ddr4bg0_de-skew = <0>; + ddr3ras_ddr4cke_de-skew = <0>; + ddr3ba2_ddr4ba0_de-skew = <1>; + ddr3we_ddr4bg1_de-skew = <1>; + ddr3cas_ddr4a12_de-skew = <0>; + ddr3ckn_ddr4ckn_de-skew = <5>; + ddr3ckp_ddr4ckp_de-skew = <5>; + ddr3cke_ddr4a16_de-skew = <1>; + ddr3odt0_ddr4a14_de-skew = <0>; + ddr3cs0_ddr4act_de-skew = <1>; + ddr3reset_ddr4reset_de-skew = <0>; + ddr3cs1_ddr4cs1_de-skew = <0>; + ddr3odt1_ddr4odt1_de-skew = <0>; + + /* DATA de-skew + * RX one step is 25.1ps, range 0-15 + * TX one step is 47.8ps, range 0-15 + */ + cs0_dm0_rx_de-skew = <7>; + cs0_dm0_tx_de-skew = <8>; + cs0_dq0_rx_de-skew = <7>; + cs0_dq0_tx_de-skew = <8>; + cs0_dq1_rx_de-skew = <7>; + cs0_dq1_tx_de-skew = <8>; + cs0_dq2_rx_de-skew = <7>; + cs0_dq2_tx_de-skew = <8>; + cs0_dq3_rx_de-skew = <7>; + cs0_dq3_tx_de-skew = <8>; + cs0_dq4_rx_de-skew = <7>; + cs0_dq4_tx_de-skew = <8>; + cs0_dq5_rx_de-skew = <7>; + cs0_dq5_tx_de-skew = <8>; + cs0_dq6_rx_de-skew = <7>; + cs0_dq6_tx_de-skew = <8>; + cs0_dq7_rx_de-skew = <7>; + cs0_dq7_tx_de-skew = <8>; + cs0_dqs0_rx_de-skew = <6>; + cs0_dqs0p_tx_de-skew = <9>; + cs0_dqs0n_tx_de-skew = <9>; + + cs0_dm1_rx_de-skew = <7>; + cs0_dm1_tx_de-skew = <7>; + cs0_dq8_rx_de-skew = <7>; + cs0_dq8_tx_de-skew = <8>; + cs0_dq9_rx_de-skew = <7>; + cs0_dq9_tx_de-skew = <7>; + cs0_dq10_rx_de-skew = <7>; + cs0_dq10_tx_de-skew = <8>; + cs0_dq11_rx_de-skew = <7>; + cs0_dq11_tx_de-skew = <7>; + cs0_dq12_rx_de-skew = <7>; + cs0_dq12_tx_de-skew = <8>; + cs0_dq13_rx_de-skew = <7>; + cs0_dq13_tx_de-skew = <7>; + cs0_dq14_rx_de-skew = <7>; + cs0_dq14_tx_de-skew = <8>; + cs0_dq15_rx_de-skew = <7>; + cs0_dq15_tx_de-skew = <7>; + cs0_dqs1_rx_de-skew = <7>; + cs0_dqs1p_tx_de-skew = <9>; + cs0_dqs1n_tx_de-skew = <9>; + + cs0_dm2_rx_de-skew = <7>; + cs0_dm2_tx_de-skew = <8>; + cs0_dq16_rx_de-skew = <7>; + cs0_dq16_tx_de-skew = <8>; + cs0_dq17_rx_de-skew = <7>; + cs0_dq17_tx_de-skew = <8>; + cs0_dq18_rx_de-skew = <7>; + cs0_dq18_tx_de-skew = <8>; + cs0_dq19_rx_de-skew = <7>; + cs0_dq19_tx_de-skew = <8>; + cs0_dq20_rx_de-skew = <7>; + cs0_dq20_tx_de-skew = <8>; + cs0_dq21_rx_de-skew = <7>; + cs0_dq21_tx_de-skew = <8>; + cs0_dq22_rx_de-skew = <7>; + cs0_dq22_tx_de-skew = <8>; + cs0_dq23_rx_de-skew = <7>; + cs0_dq23_tx_de-skew = <8>; + cs0_dqs2_rx_de-skew = <6>; + cs0_dqs2p_tx_de-skew = <9>; + cs0_dqs2n_tx_de-skew = <9>; + + cs0_dm3_rx_de-skew = <7>; + cs0_dm3_tx_de-skew = <7>; + cs0_dq24_rx_de-skew = <7>; + cs0_dq24_tx_de-skew = <8>; + cs0_dq25_rx_de-skew = <7>; + cs0_dq25_tx_de-skew = <7>; + cs0_dq26_rx_de-skew = <7>; + cs0_dq26_tx_de-skew = <7>; + cs0_dq27_rx_de-skew = <7>; + cs0_dq27_tx_de-skew = <7>; + cs0_dq28_rx_de-skew = <7>; + cs0_dq28_tx_de-skew = <7>; + cs0_dq29_rx_de-skew = <7>; + cs0_dq29_tx_de-skew = <7>; + cs0_dq30_rx_de-skew = <7>; + cs0_dq30_tx_de-skew = <7>; + cs0_dq31_rx_de-skew = <7>; + cs0_dq31_tx_de-skew = <7>; + cs0_dqs3_rx_de-skew = <7>; + cs0_dqs3p_tx_de-skew = <9>; + cs0_dqs3n_tx_de-skew = <9>; + + cs1_dm0_rx_de-skew = <7>; + cs1_dm0_tx_de-skew = <8>; + cs1_dq0_rx_de-skew = <7>; + cs1_dq0_tx_de-skew = <8>; + cs1_dq1_rx_de-skew = <7>; + cs1_dq1_tx_de-skew = <8>; + cs1_dq2_rx_de-skew = <7>; + cs1_dq2_tx_de-skew = <8>; + cs1_dq3_rx_de-skew = <7>; + cs1_dq3_tx_de-skew = <8>; + cs1_dq4_rx_de-skew = <7>; + cs1_dq4_tx_de-skew = <8>; + cs1_dq5_rx_de-skew = <7>; + cs1_dq5_tx_de-skew = <8>; + cs1_dq6_rx_de-skew = <7>; + cs1_dq6_tx_de-skew = <8>; + cs1_dq7_rx_de-skew = <7>; + cs1_dq7_tx_de-skew = <8>; + cs1_dqs0_rx_de-skew = <6>; + cs1_dqs0p_tx_de-skew = <9>; + cs1_dqs0n_tx_de-skew = <9>; + + cs1_dm1_rx_de-skew = <7>; + cs1_dm1_tx_de-skew = <7>; + cs1_dq8_rx_de-skew = <7>; + cs1_dq8_tx_de-skew = <8>; + cs1_dq9_rx_de-skew = <7>; + cs1_dq9_tx_de-skew = <7>; + cs1_dq10_rx_de-skew = <7>; + cs1_dq10_tx_de-skew = <8>; + cs1_dq11_rx_de-skew = <7>; + cs1_dq11_tx_de-skew = <7>; + cs1_dq12_rx_de-skew = <7>; + cs1_dq12_tx_de-skew = <8>; + cs1_dq13_rx_de-skew = <7>; + cs1_dq13_tx_de-skew = <7>; + cs1_dq14_rx_de-skew = <7>; + cs1_dq14_tx_de-skew = <8>; + cs1_dq15_rx_de-skew = <7>; + cs1_dq15_tx_de-skew = <7>; + cs1_dqs1_rx_de-skew = <7>; + cs1_dqs1p_tx_de-skew = <9>; + cs1_dqs1n_tx_de-skew = <9>; + + cs1_dm2_rx_de-skew = <7>; + cs1_dm2_tx_de-skew = <8>; + cs1_dq16_rx_de-skew = <7>; + cs1_dq16_tx_de-skew = <8>; + cs1_dq17_rx_de-skew = <7>; + cs1_dq17_tx_de-skew = <8>; + cs1_dq18_rx_de-skew = <7>; + cs1_dq18_tx_de-skew = <8>; + cs1_dq19_rx_de-skew = <7>; + cs1_dq19_tx_de-skew = <8>; + cs1_dq20_rx_de-skew = <7>; + cs1_dq20_tx_de-skew = <8>; + cs1_dq21_rx_de-skew = <7>; + cs1_dq21_tx_de-skew = <8>; + cs1_dq22_rx_de-skew = <7>; + cs1_dq22_tx_de-skew = <8>; + cs1_dq23_rx_de-skew = <7>; + cs1_dq23_tx_de-skew = <8>; + cs1_dqs2_rx_de-skew = <6>; + cs1_dqs2p_tx_de-skew = <9>; + cs1_dqs2n_tx_de-skew = <9>; + + cs1_dm3_rx_de-skew = <7>; + cs1_dm3_tx_de-skew = <7>; + cs1_dq24_rx_de-skew = <7>; + cs1_dq24_tx_de-skew = <8>; + cs1_dq25_rx_de-skew = <7>; + cs1_dq25_tx_de-skew = <7>; + cs1_dq26_rx_de-skew = <7>; + cs1_dq26_tx_de-skew = <7>; + cs1_dq27_rx_de-skew = <7>; + cs1_dq27_tx_de-skew = <7>; + cs1_dq28_rx_de-skew = <7>; + cs1_dq28_tx_de-skew = <7>; + cs1_dq29_rx_de-skew = <7>; + cs1_dq29_tx_de-skew = <7>; + cs1_dq30_rx_de-skew = <7>; + cs1_dq30_tx_de-skew = <7>; + cs1_dq31_rx_de-skew = <7>; + cs1_dq31_tx_de-skew = <7>; + cs1_dqs3_rx_de-skew = <7>; + cs1_dqs3p_tx_de-skew = <9>; + cs1_dqs3n_tx_de-skew = <9>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-dram-2layer-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-dram-2layer-timing.dtsi new file mode 100755 index 000000000000..940024920b5d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-dram-2layer-timing.dtsi @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include + +&ddr_timing { + /* CA de-skew, one step is 47.8ps, range 0-15 */ + ddr3a1_ddr4a9_de-skew = <2>; + ddr3a0_ddr4a10_de-skew = <4>; + ddr3a3_ddr4a6_de-skew = <6>; + ddr3a2_ddr4a4_de-skew = <5>; + ddr3a5_ddr4a8_de-skew = <7>; + ddr3a4_ddr4a5_de-skew = <7>; + ddr3a7_ddr4a11_de-skew = <7>; + ddr3a6_ddr4a7_de-skew = <7>; + ddr3a9_ddr4a0_de-skew = <7>; + ddr3a8_ddr4a13_de-skew = <4>; + ddr3a11_ddr4a3_de-skew = <4>; + ddr3a10_ddr4cs0_de-skew = <4>; + ddr3a13_ddr4a2_de-skew = <7>; + ddr3a12_ddr4ba1_de-skew = <5>; + ddr3a15_ddr4odt0_de-skew = <7>; + ddr3a14_ddr4a1_de-skew = <6>; + ddr3ba1_ddr4a15_de-skew = <3>; + ddr3ba0_ddr4bg0_de-skew = <9>; + ddr3ras_ddr4cke_de-skew = <6>; + ddr3ba2_ddr4ba0_de-skew = <8>; + ddr3we_ddr4bg1_de-skew = <4>; + ddr3cas_ddr4a12_de-skew = <4>; + ddr3ckn_ddr4ckn_de-skew = <14>; + ddr3ckp_ddr4ckp_de-skew = <14>; + ddr3cke_ddr4a16_de-skew = <5>; + ddr3odt0_ddr4a14_de-skew = <9>; + ddr3cs0_ddr4act_de-skew = <9>; + ddr3reset_ddr4reset_de-skew = <10>; + ddr3cs1_ddr4cs1_de-skew = <7>; + ddr3odt1_ddr4odt1_de-skew = <7>; + + /* DATA de-skew + * RX one step is 25.1ps, range 0-15 + * TX one step is 47.8ps, range 0-15 + */ + cs0_dm0_rx_de-skew = <15>; + cs0_dm0_tx_de-skew = <14>; + cs0_dq0_rx_de-skew = <13>; + cs0_dq0_tx_de-skew = <13>; + cs0_dq1_rx_de-skew = <14>; + cs0_dq1_tx_de-skew = <14>; + cs0_dq2_rx_de-skew = <13>; + cs0_dq2_tx_de-skew = <13>; + cs0_dq3_rx_de-skew = <15>; + cs0_dq3_tx_de-skew = <14>; + cs0_dq4_rx_de-skew = <15>; + cs0_dq4_tx_de-skew = <14>; + cs0_dq5_rx_de-skew = <15>; + cs0_dq5_tx_de-skew = <14>; + cs0_dq6_rx_de-skew = <15>; + cs0_dq6_tx_de-skew = <14>; + cs0_dq7_rx_de-skew = <15>; + cs0_dq7_tx_de-skew = <14>; + cs0_dqs0_rx_de-skew = <13>; + cs0_dqs0p_tx_de-skew = <15>; + cs0_dqs0n_tx_de-skew = <15>; + + cs0_dm1_rx_de-skew = <11>; + cs0_dm1_tx_de-skew = <11>; + cs0_dq8_rx_de-skew = <12>; + cs0_dq8_tx_de-skew = <13>; + cs0_dq9_rx_de-skew = <13>; + cs0_dq9_tx_de-skew = <12>; + cs0_dq10_rx_de-skew = <12>; + cs0_dq10_tx_de-skew = <13>; + cs0_dq11_rx_de-skew = <15>; + cs0_dq11_tx_de-skew = <13>; + cs0_dq12_rx_de-skew = <9>; + cs0_dq12_tx_de-skew = <11>; + cs0_dq13_rx_de-skew = <12>; + cs0_dq13_tx_de-skew = <12>; + cs0_dq14_rx_de-skew = <9>; + cs0_dq14_tx_de-skew = <11>; + cs0_dq15_rx_de-skew = <13>; + cs0_dq15_tx_de-skew = <12>; + cs0_dqs1_rx_de-skew = <14>; + cs0_dqs1p_tx_de-skew = <14>; + cs0_dqs1n_tx_de-skew = <14>; + + cs0_dm2_rx_de-skew = <10>; + cs0_dm2_tx_de-skew = <12>; + cs0_dq16_rx_de-skew = <11>; + cs0_dq16_tx_de-skew = <12>; + cs0_dq17_rx_de-skew = <11>; + cs0_dq17_tx_de-skew = <12>; + cs0_dq18_rx_de-skew = <11>; + cs0_dq18_tx_de-skew = <12>; + cs0_dq19_rx_de-skew = <13>; + cs0_dq19_tx_de-skew = <13>; + cs0_dq20_rx_de-skew = <12>; + cs0_dq20_tx_de-skew = <13>; + cs0_dq21_rx_de-skew = <11>; + cs0_dq21_tx_de-skew = <12>; + cs0_dq22_rx_de-skew = <12>; + cs0_dq22_tx_de-skew = <12>; + cs0_dq23_rx_de-skew = <12>; + cs0_dq23_tx_de-skew = <12>; + cs0_dqs2_rx_de-skew = <11>; + cs0_dqs2p_tx_de-skew = <14>; + cs0_dqs2n_tx_de-skew = <14>; + + cs0_dm3_rx_de-skew = <10>; + cs0_dm3_tx_de-skew = <11>; + cs0_dq24_rx_de-skew = <9>; + cs0_dq24_tx_de-skew = <12>; + cs0_dq25_rx_de-skew = <10>; + cs0_dq25_tx_de-skew = <12>; + cs0_dq26_rx_de-skew = <13>; + cs0_dq26_tx_de-skew = <13>; + cs0_dq27_rx_de-skew = <14>; + cs0_dq27_tx_de-skew = <13>; + cs0_dq28_rx_de-skew = <8>; + cs0_dq28_tx_de-skew = <10>; + cs0_dq29_rx_de-skew = <10>; + cs0_dq29_tx_de-skew = <12>; + cs0_dq30_rx_de-skew = <14>; + cs0_dq30_tx_de-skew = <13>; + cs0_dq31_rx_de-skew = <15>; + cs0_dq31_tx_de-skew = <14>; + cs0_dqs3_rx_de-skew = <12>; + cs0_dqs3p_tx_de-skew = <15>; + cs0_dqs3n_tx_de-skew = <15>; + + cs1_dm0_rx_de-skew = <11>; + cs1_dm0_tx_de-skew = <10>; + cs1_dq0_rx_de-skew = <9>; + cs1_dq0_tx_de-skew = <9>; + cs1_dq1_rx_de-skew = <10>; + cs1_dq1_tx_de-skew = <10>; + cs1_dq2_rx_de-skew = <9>; + cs1_dq2_tx_de-skew = <9>; + cs1_dq3_rx_de-skew = <11>; + cs1_dq3_tx_de-skew = <10>; + cs1_dq4_rx_de-skew = <11>; + cs1_dq4_tx_de-skew = <10>; + cs1_dq5_rx_de-skew = <11>; + cs1_dq5_tx_de-skew = <10>; + cs1_dq6_rx_de-skew = <11>; + cs1_dq6_tx_de-skew = <10>; + cs1_dq7_rx_de-skew = <11>; + cs1_dq7_tx_de-skew = <10>; + cs1_dqs0_rx_de-skew = <9>; + cs1_dqs0p_tx_de-skew = <10>; + cs1_dqs0n_tx_de-skew = <10>; + + cs1_dm1_rx_de-skew = <7>; + cs1_dm1_tx_de-skew = <7>; + cs1_dq8_rx_de-skew = <8>; + cs1_dq8_tx_de-skew = <9>; + cs1_dq9_rx_de-skew = <9>; + cs1_dq9_tx_de-skew = <8>; + cs1_dq10_rx_de-skew = <8>; + cs1_dq10_tx_de-skew = <9>; + cs1_dq11_rx_de-skew = <11>; + cs1_dq11_tx_de-skew = <9>; + cs1_dq12_rx_de-skew = <5>; + cs1_dq12_tx_de-skew = <7>; + cs1_dq13_rx_de-skew = <8>; + cs1_dq13_tx_de-skew = <8>; + cs1_dq14_rx_de-skew = <5>; + cs1_dq14_tx_de-skew = <7>; + cs1_dq15_rx_de-skew = <9>; + cs1_dq15_tx_de-skew = <8>; + cs1_dqs1_rx_de-skew = <9>; + cs1_dqs1p_tx_de-skew = <10>; + cs1_dqs1n_tx_de-skew = <10>; + + cs1_dm2_rx_de-skew = <6>; + cs1_dm2_tx_de-skew = <8>; + cs1_dq16_rx_de-skew = <7>; + cs1_dq16_tx_de-skew = <8>; + cs1_dq17_rx_de-skew = <7>; + cs1_dq17_tx_de-skew = <8>; + cs1_dq18_rx_de-skew = <7>; + cs1_dq18_tx_de-skew = <8>; + cs1_dq19_rx_de-skew = <9>; + cs1_dq19_tx_de-skew = <9>; + cs1_dq20_rx_de-skew = <8>; + cs1_dq20_tx_de-skew = <9>; + cs1_dq21_rx_de-skew = <7>; + cs1_dq21_tx_de-skew = <8>; + cs1_dq22_rx_de-skew = <8>; + cs1_dq22_tx_de-skew = <8>; + cs1_dq23_rx_de-skew = <8>; + cs1_dq23_tx_de-skew = <8>; + cs1_dqs2_rx_de-skew = <7>; + cs1_dqs2p_tx_de-skew = <9>; + cs1_dqs2n_tx_de-skew = <9>; + + cs1_dm3_rx_de-skew = <4>; + cs1_dm3_tx_de-skew = <5>; + cs1_dq24_rx_de-skew = <3>; + cs1_dq24_tx_de-skew = <6>; + cs1_dq25_rx_de-skew = <4>; + cs1_dq25_tx_de-skew = <6>; + cs1_dq26_rx_de-skew = <7>; + cs1_dq26_tx_de-skew = <7>; + cs1_dq27_rx_de-skew = <8>; + cs1_dq27_tx_de-skew = <7>; + cs1_dq28_rx_de-skew = <2>; + cs1_dq28_tx_de-skew = <4>; + cs1_dq29_rx_de-skew = <4>; + cs1_dq29_tx_de-skew = <6>; + cs1_dq30_rx_de-skew = <8>; + cs1_dq30_tx_de-skew = <7>; + cs1_dq31_rx_de-skew = <9>; + cs1_dq31_tx_de-skew = <8>; + cs1_dqs3_rx_de-skew = <6>; + cs1_dqs3p_tx_de-skew = <8>; + cs1_dqs3n_tx_de-skew = <8>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-dram-default-timing.dtsi new file mode 100755 index 000000000000..a3f5ff4bdc47 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-dram-default-timing.dtsi @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include + +/ { + ddr_timing: ddr_timing { + compatible = "rockchip,ddr-timing"; + ddr3_speed_bin = ; + ddr4_speed_bin = ; + pd_idle = <0>; + sr_idle = <0>; + sr_mc_gate_idle = <0>; + srpd_lite_idle = <0>; + standby_idle = <0>; + + auto_pd_dis_freq = <1066>; + auto_sr_dis_freq = <800>; + ddr3_dll_dis_freq = <300>; + ddr4_dll_dis_freq = <625>; + phy_dll_dis_freq = <400>; + + ddr3_odt_dis_freq = <100>; + phy_ddr3_odt_dis_freq = <100>; + ddr3_drv = ; + ddr3_odt = ; + phy_ddr3_ca_drv = ; + phy_ddr3_ck_drv = ; + phy_ddr3_dq_drv = ; + phy_ddr3_odt = ; + + lpddr3_odt_dis_freq = <666>; + phy_lpddr3_odt_dis_freq = <666>; + lpddr3_drv = ; + lpddr3_odt = ; + phy_lpddr3_ca_drv = ; + phy_lpddr3_ck_drv = ; + phy_lpddr3_dq_drv = ; + phy_lpddr3_odt = ; + + lpddr4_odt_dis_freq = <800>; + phy_lpddr4_odt_dis_freq = <800>; + lpddr4_drv = ; + lpddr4_dq_odt = ; + lpddr4_ca_odt = ; + phy_lpddr4_ca_drv = ; + phy_lpddr4_ck_cs_drv = ; + phy_lpddr4_dq_drv = ; + phy_lpddr4_odt = ; + + ddr4_odt_dis_freq = <666>; + phy_ddr4_odt_dis_freq = <666>; + ddr4_drv = ; + ddr4_odt = ; + phy_ddr4_ca_drv = ; + phy_ddr4_ck_drv = ; + phy_ddr4_dq_drv = ; + phy_ddr4_odt = ; + + /* CA de-skew, one step is 47.8ps, range 0-15 */ + ddr3a1_ddr4a9_de-skew = <7>; + ddr3a0_ddr4a10_de-skew = <7>; + ddr3a3_ddr4a6_de-skew = <8>; + ddr3a2_ddr4a4_de-skew = <8>; + ddr3a5_ddr4a8_de-skew = <7>; + ddr3a4_ddr4a5_de-skew = <9>; + ddr3a7_ddr4a11_de-skew = <7>; + ddr3a6_ddr4a7_de-skew = <9>; + ddr3a9_ddr4a0_de-skew = <8>; + ddr3a8_ddr4a13_de-skew = <7>; + ddr3a11_ddr4a3_de-skew = <9>; + ddr3a10_ddr4cs0_de-skew = <7>; + ddr3a13_ddr4a2_de-skew = <8>; + ddr3a12_ddr4ba1_de-skew = <7>; + ddr3a15_ddr4odt0_de-skew = <7>; + ddr3a14_ddr4a1_de-skew = <8>; + ddr3ba1_ddr4a15_de-skew = <7>; + ddr3ba0_ddr4bg0_de-skew = <7>; + ddr3ras_ddr4cke_de-skew = <7>; + ddr3ba2_ddr4ba0_de-skew = <8>; + ddr3we_ddr4bg1_de-skew = <8>; + ddr3cas_ddr4a12_de-skew = <7>; + ddr3ckn_ddr4ckn_de-skew = <8>; + ddr3ckp_ddr4ckp_de-skew = <8>; + ddr3cke_ddr4a16_de-skew = <8>; + ddr3odt0_ddr4a14_de-skew = <7>; + ddr3cs0_ddr4act_de-skew = <8>; + ddr3reset_ddr4reset_de-skew = <7>; + ddr3cs1_ddr4cs1_de-skew = <7>; + ddr3odt1_ddr4odt1_de-skew = <7>; + + /* DATA de-skew + * RX one step is 25.1ps, range 0-15 + * TX one step is 47.8ps, range 0-15 + */ + cs0_dm0_rx_de-skew = <7>; + cs0_dm0_tx_de-skew = <8>; + cs0_dq0_rx_de-skew = <7>; + cs0_dq0_tx_de-skew = <8>; + cs0_dq1_rx_de-skew = <7>; + cs0_dq1_tx_de-skew = <8>; + cs0_dq2_rx_de-skew = <7>; + cs0_dq2_tx_de-skew = <8>; + cs0_dq3_rx_de-skew = <7>; + cs0_dq3_tx_de-skew = <8>; + cs0_dq4_rx_de-skew = <7>; + cs0_dq4_tx_de-skew = <8>; + cs0_dq5_rx_de-skew = <7>; + cs0_dq5_tx_de-skew = <8>; + cs0_dq6_rx_de-skew = <7>; + cs0_dq6_tx_de-skew = <8>; + cs0_dq7_rx_de-skew = <7>; + cs0_dq7_tx_de-skew = <8>; + cs0_dqs0_rx_de-skew = <6>; + cs0_dqs0p_tx_de-skew = <9>; + cs0_dqs0n_tx_de-skew = <9>; + + cs0_dm1_rx_de-skew = <7>; + cs0_dm1_tx_de-skew = <7>; + cs0_dq8_rx_de-skew = <7>; + cs0_dq8_tx_de-skew = <8>; + cs0_dq9_rx_de-skew = <7>; + cs0_dq9_tx_de-skew = <7>; + cs0_dq10_rx_de-skew = <7>; + cs0_dq10_tx_de-skew = <8>; + cs0_dq11_rx_de-skew = <7>; + cs0_dq11_tx_de-skew = <7>; + cs0_dq12_rx_de-skew = <7>; + cs0_dq12_tx_de-skew = <8>; + cs0_dq13_rx_de-skew = <7>; + cs0_dq13_tx_de-skew = <7>; + cs0_dq14_rx_de-skew = <7>; + cs0_dq14_tx_de-skew = <8>; + cs0_dq15_rx_de-skew = <7>; + cs0_dq15_tx_de-skew = <7>; + cs0_dqs1_rx_de-skew = <7>; + cs0_dqs1p_tx_de-skew = <9>; + cs0_dqs1n_tx_de-skew = <9>; + + cs0_dm2_rx_de-skew = <7>; + cs0_dm2_tx_de-skew = <8>; + cs0_dq16_rx_de-skew = <7>; + cs0_dq16_tx_de-skew = <8>; + cs0_dq17_rx_de-skew = <7>; + cs0_dq17_tx_de-skew = <8>; + cs0_dq18_rx_de-skew = <7>; + cs0_dq18_tx_de-skew = <8>; + cs0_dq19_rx_de-skew = <7>; + cs0_dq19_tx_de-skew = <8>; + cs0_dq20_rx_de-skew = <7>; + cs0_dq20_tx_de-skew = <8>; + cs0_dq21_rx_de-skew = <7>; + cs0_dq21_tx_de-skew = <8>; + cs0_dq22_rx_de-skew = <7>; + cs0_dq22_tx_de-skew = <8>; + cs0_dq23_rx_de-skew = <7>; + cs0_dq23_tx_de-skew = <8>; + cs0_dqs2_rx_de-skew = <6>; + cs0_dqs2p_tx_de-skew = <9>; + cs0_dqs2n_tx_de-skew = <9>; + + cs0_dm3_rx_de-skew = <7>; + cs0_dm3_tx_de-skew = <7>; + cs0_dq24_rx_de-skew = <7>; + cs0_dq24_tx_de-skew = <8>; + cs0_dq25_rx_de-skew = <7>; + cs0_dq25_tx_de-skew = <7>; + cs0_dq26_rx_de-skew = <7>; + cs0_dq26_tx_de-skew = <7>; + cs0_dq27_rx_de-skew = <7>; + cs0_dq27_tx_de-skew = <7>; + cs0_dq28_rx_de-skew = <7>; + cs0_dq28_tx_de-skew = <7>; + cs0_dq29_rx_de-skew = <7>; + cs0_dq29_tx_de-skew = <7>; + cs0_dq30_rx_de-skew = <7>; + cs0_dq30_tx_de-skew = <7>; + cs0_dq31_rx_de-skew = <7>; + cs0_dq31_tx_de-skew = <7>; + cs0_dqs3_rx_de-skew = <7>; + cs0_dqs3p_tx_de-skew = <9>; + cs0_dqs3n_tx_de-skew = <9>; + + cs1_dm0_rx_de-skew = <7>; + cs1_dm0_tx_de-skew = <8>; + cs1_dq0_rx_de-skew = <7>; + cs1_dq0_tx_de-skew = <8>; + cs1_dq1_rx_de-skew = <7>; + cs1_dq1_tx_de-skew = <8>; + cs1_dq2_rx_de-skew = <7>; + cs1_dq2_tx_de-skew = <8>; + cs1_dq3_rx_de-skew = <7>; + cs1_dq3_tx_de-skew = <8>; + cs1_dq4_rx_de-skew = <7>; + cs1_dq4_tx_de-skew = <8>; + cs1_dq5_rx_de-skew = <7>; + cs1_dq5_tx_de-skew = <8>; + cs1_dq6_rx_de-skew = <7>; + cs1_dq6_tx_de-skew = <8>; + cs1_dq7_rx_de-skew = <7>; + cs1_dq7_tx_de-skew = <8>; + cs1_dqs0_rx_de-skew = <6>; + cs1_dqs0p_tx_de-skew = <9>; + cs1_dqs0n_tx_de-skew = <9>; + + cs1_dm1_rx_de-skew = <7>; + cs1_dm1_tx_de-skew = <7>; + cs1_dq8_rx_de-skew = <7>; + cs1_dq8_tx_de-skew = <8>; + cs1_dq9_rx_de-skew = <7>; + cs1_dq9_tx_de-skew = <7>; + cs1_dq10_rx_de-skew = <7>; + cs1_dq10_tx_de-skew = <8>; + cs1_dq11_rx_de-skew = <7>; + cs1_dq11_tx_de-skew = <7>; + cs1_dq12_rx_de-skew = <7>; + cs1_dq12_tx_de-skew = <8>; + cs1_dq13_rx_de-skew = <7>; + cs1_dq13_tx_de-skew = <7>; + cs1_dq14_rx_de-skew = <7>; + cs1_dq14_tx_de-skew = <8>; + cs1_dq15_rx_de-skew = <7>; + cs1_dq15_tx_de-skew = <7>; + cs1_dqs1_rx_de-skew = <7>; + cs1_dqs1p_tx_de-skew = <9>; + cs1_dqs1n_tx_de-skew = <9>; + + cs1_dm2_rx_de-skew = <7>; + cs1_dm2_tx_de-skew = <8>; + cs1_dq16_rx_de-skew = <7>; + cs1_dq16_tx_de-skew = <8>; + cs1_dq17_rx_de-skew = <7>; + cs1_dq17_tx_de-skew = <8>; + cs1_dq18_rx_de-skew = <7>; + cs1_dq18_tx_de-skew = <8>; + cs1_dq19_rx_de-skew = <7>; + cs1_dq19_tx_de-skew = <8>; + cs1_dq20_rx_de-skew = <7>; + cs1_dq20_tx_de-skew = <8>; + cs1_dq21_rx_de-skew = <7>; + cs1_dq21_tx_de-skew = <8>; + cs1_dq22_rx_de-skew = <7>; + cs1_dq22_tx_de-skew = <8>; + cs1_dq23_rx_de-skew = <7>; + cs1_dq23_tx_de-skew = <8>; + cs1_dqs2_rx_de-skew = <6>; + cs1_dqs2p_tx_de-skew = <9>; + cs1_dqs2n_tx_de-skew = <9>; + + cs1_dm3_rx_de-skew = <7>; + cs1_dm3_tx_de-skew = <7>; + cs1_dq24_rx_de-skew = <7>; + cs1_dq24_tx_de-skew = <8>; + cs1_dq25_rx_de-skew = <7>; + cs1_dq25_tx_de-skew = <7>; + cs1_dq26_rx_de-skew = <7>; + cs1_dq26_tx_de-skew = <7>; + cs1_dq27_rx_de-skew = <7>; + cs1_dq27_tx_de-skew = <7>; + cs1_dq28_rx_de-skew = <7>; + cs1_dq28_tx_de-skew = <7>; + cs1_dq29_rx_de-skew = <7>; + cs1_dq29_tx_de-skew = <7>; + cs1_dq30_rx_de-skew = <7>; + cs1_dq30_tx_de-skew = <7>; + cs1_dq31_rx_de-skew = <7>; + cs1_dq31_tx_de-skew = <7>; + cs1_dqs3_rx_de-skew = <7>; + cs1_dqs3p_tx_de-skew = <9>; + cs1_dqs3n_tx_de-skew = <9>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-evb-android-avb.dts b/arch/arm64/boot/dts/rockchip/rk3328-evb-android-avb.dts new file mode 100755 index 000000000000..0d72e05e2d59 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-evb-android-avb.dts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + */ + +/dts-v1/; +#include "rk3328-evb-android.dtsi" + +/ { + model = "Rockchip RK3328 EVB avb"; + compatible = "rockchip,rk3328-evb-avb", "rockchip,rk3328"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dts b/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dts new file mode 100755 index 000000000000..6d50444a5a1d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dts @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include "rk3328-evb-android.dtsi" + +/ { + model = "Rockchip RK3328 EVB"; + compatible = "rockchip,rk3328-evb", "rockchip,rk3328"; +}; + +&firmware_android{ + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/platform/ff520000.dwmmc/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/platform/ff520000.dwmmc/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dtsi new file mode 100755 index 000000000000..e050047420a6 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-evb-android.dtsi @@ -0,0 +1,715 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + */ + +#include "rk3328.dtsi" +#include "rk3328-android.dtsi" +#include + +/ { + gmac_clkin: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "gmac_clkin"; + #clock-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio1 18 GPIO_ACTIVE_LOW>; + }; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip-rk3328"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&codec>; + }; + }; + + hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip-hdmi"; + simple-audio-card,cpu { + sound-dai = <&i2s0>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + spdif-sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip-spdif"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + vcc_host_vbus: host-vbus-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host_vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + }; + + vcc_otg_vbus: otg-vbus-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PD3 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&otg_vbus_drv>; + regulator-name = "vcc_otg_vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc_sd: sdmmc-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 30 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0m1_gpio>; + regulator-name = "vcc_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_io>; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk805 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio1 10 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_gpios>; + BT,power_gpio = <&gpio1 21 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio1 26 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio1 19 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&avsd { + status = "okay"; +}; + +&codec { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; + rockchip,default-sample-phase = <90>; + status = "okay"; +}; + +&gmac2io { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio1 RK_PC2 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_MAC2IO>, <&cru SCLK_MAC2IO_EXT>; + assigned-clock-parents = <&gmac_clkin>, <&gmac_clkin>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmiim1_pins>; + tx_delay = <0x26>; + rx_delay = <0x11>; + status = "disabled"; +}; + +&gmac2phy { + phy-supply = <&vcc_phy>; + clock_in_out = "output"; + assigned-clocks = <&cru SCLK_MAC2PHY_SRC>; + assigned-clock-rate = <50000000>; + assigned-clocks = <&cru SCLK_MAC2PHY>; + assigned-clock-parents = <&cru SCLK_MAC2PHY_SRC>; + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_logic>; +}; + +&hdmi { + #sound-dai-cells = <0>; + ddc-i2c-scl-high-time-ns = <9625>; + ddc-i2c-scl-low-time-ns = <10000>; + status = "okay"; +}; + +&hdmiphy { + rockchip,phy-table = + <165000000 0x07 0x0a 0x0a 0x0a 0x00 0x00 0x08 + 0x08 0x08 0x00 0xac 0xcc 0xcc 0xcc>, + <340000000 0x0b 0x0d 0x0d 0x0d 0x07 0x15 0x08 + 0x08 0x08 0x3f 0xac 0xcc 0xcd 0xdd>, + <594000000 0x10 0x1a 0x1a 0x1a 0x07 0x15 0x08 + 0x08 0x08 0x00 0xac 0xcc 0xcc 0xcc>; + status = "okay"; +}; + +&i2c1 { + status = "okay"; + + rk805: rk805@18 { + compatible = "rockchip,rk805"; + status = "okay"; + reg = <0x18>; + interrupt-parent = <&gpio2>; + interrupts = <6 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + wakeup-source; + gpio-controller; + #gpio-cells = <2>; + #clock-cells = <1>; + clock-output-names = "rk805-clkout1", "rk805-clkout2"; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc5-supply = <&vcc_io>; + vcc6-supply = <&vcc_io>; + + rtc { + status = "okay"; + }; + + pwrkey { + status = "disabled"; + }; + + gpio { + status = "okay"; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-name = "vdd_logic"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1450000>; + regulator-initial-mode = <0x1>; + regulator-ramp-delay = <12500>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-mode = <0x2>; + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-name = "vdd_arm"; + regulator-init-microvolt = <1225000>; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1450000>; + regulator-initial-mode = <0x1>; + regulator-ramp-delay = <12500>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-mode = <0x2>; + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-mode = <0x2>; + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-name = "vcc_io"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-mode = <0x2>; + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vdd_18: LDO_REG1 { + regulator-name = "vdd_18"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_18emmc: LDO_REG2 { + regulator-name = "vcc_18emmc"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd_11: LDO_REG3 { + regulator-name = "vdd_11"; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1100000>; + }; + }; + }; + }; +}; + +&secure_memory { + /* + * enable like this: + * reg = <0x0 0x20000000 0x0 0x10000000>; + */ + reg = <0x0 0x20000000 0x0 0x0>; +}; + +&i2s0 { + #sound-dai-cells = <0>; + rockchip,bclk-fs = <128>; + status = "okay"; +}; + +&i2s1 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc_io>; + vccio2-supply = <&vcc_18emmc>; + vccio3-supply = <&vcc_io>; + vccio4-supply = <&vdd_18>; + vccio5-supply = <&vcc_io>; + vccio6-supply = <&vcc_io>; + pmuio-supply = <&vcc_io>; +}; + +&mpp_srv { + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <2 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; /* gpio2_a6 */ + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + otg_vbus_drv: otg-vbus-drv { + rockchip,pins = + <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <1 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pwm3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pwmir_pin>; + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <1>; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_PLAYPAUSE>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xa4 KEY_SETUP>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +&rga { + status = "okay"; +}; + +&rkvdec { + status = "okay"; + vcodec-supply = <&vdd_logic>; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,virtual-poweroff = <1>; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&sdio { + bus-width = <4>; + cap-sd-highspeed; + cap-sdio-irq; + disable-wp; + keep-power-in-suspend; + max-frequency = <125000000>; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + no-sd; + no-mmc; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + max-frequency = <150000000>; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_dectn &sdmmc0_bus4>; + no-sdio; + no-mmc; + status = "okay"; + vmmc-supply = <&vcc_sd>; +}; + +&spdif { + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&spdifm2_tx>; + status = "okay"; +}; + +&threshold { + temperature = <90000>; /* millicelsius */ +}; + +&target { + temperature = <105000>; /* millicelsius */ +}; + +&soc_crit { + temperature = <115000>; /* millicelsius */ +}; + +&tsadc { + rockchip,hw-tshut-temp = <120000>; + status = "okay"; +}; + +&tve { + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + status = "okay"; + }; + + u2phy_otg: otg-port { + vbus-supply = <&vcc_otg_vbus>; + status = "okay"; + }; +}; + +&u3phy { + vbus-supply = <&vcc_host_vbus>; + status = "okay"; +}; + +&u3phy_utmi { + status = "okay"; +}; + +&u3phy_pipe { + status = "okay"; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usbdrd3 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vepu22 { + status = "okay"; +}; + +&vepu22_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-evb.dts b/arch/arm64/boot/dts/rockchip/rk3328-evb.dts index a48767931af6..82d2f01a8be6 100644 --- a/arch/arm64/boot/dts/rockchip/rk3328-evb.dts +++ b/arch/arm64/boot/dts/rockchip/rk3328-evb.dts @@ -37,6 +37,18 @@ sdio_pwrseq: sdio-pwrseq { reset-gpios = <&gpio1 18 GPIO_ACTIVE_LOW>; }; + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,signal-irq = <159>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <0>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + status = "okay"; + }; + vcc_sd: sdmmc-regulator { compatible = "regulator-fixed"; gpio = <&gpio0 30 GPIO_ACTIVE_LOW>; @@ -64,6 +76,14 @@ vcc_phy: vcc-phy-regulator { regulator-always-on; regulator-boot-on; }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + }; &cpu0 { @@ -109,7 +129,7 @@ rk805: pmic@18 { interrupt-parent = <&gpio2>; interrupts = <6 IRQ_TYPE_LEVEL_LOW>; #clock-cells = <1>; - clock-output-names = "xin32k", "rk805-clkout2"; + clock-output-names = "rk805-clkout1", "rk805-clkout2"; gpio-controller; #gpio-cells = <2>; pinctrl-names = "default"; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s.dts b/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s.dts index 83a0bdbe00d6..1eecad724f04 100644 --- a/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s.dts +++ b/arch/arm64/boot/dts/rockchip/rk3328-nanopi-r2s.dts @@ -71,8 +71,8 @@ vcc_io_sdio: sdmmcio-regulator { regulator-settling-time-us = <5000>; regulator-type = "voltage"; startup-delay-us = <2000>; - states = <1800000 0x1>, - <3300000 0x0>; + states = <1800000 0x1 + 3300000 0x0>; vin-supply = <&vcc_io_33>; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-roc-cc.dts b/arch/arm64/boot/dts/rockchip/rk3328-roc-cc.dts index 22ab5e1d7319..a0c4d321cc0b 100644 --- a/arch/arm64/boot/dts/rockchip/rk3328-roc-cc.dts +++ b/arch/arm64/boot/dts/rockchip/rk3328-roc-cc.dts @@ -45,8 +45,8 @@ vcc_sd: sdmmc-regulator { vcc_sdio: sdmmcio-regulator { compatible = "regulator-gpio"; gpios = <&grf_gpio 0 GPIO_ACTIVE_HIGH>; - states = <1800000 0x1>, - <3300000 0x0>; + states = <1800000 0x1 + 3300000 0x0>; regulator-name = "vcc_sdio"; regulator-type = "voltage"; regulator-min-microvolt = <1800000>; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-rock64-android-avb.dts b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android-avb.dts new file mode 100755 index 000000000000..adc1dd7cd4da --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android-avb.dts @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + */ + +/dts-v1/; +#include "rk3328-rock64-android.dtsi" + +/ { + model = "Pine64 Rock64 avb"; + compatible = "pine64,rock64-android-avb", "rockchip,rk3328"; +}; + +&firmware_android{ + compatible = "android,firmware"; + boot_devices = "ff520000.dwmmc"; + vbmeta { + compatible = "android,vbmeta"; + parts = "vbmeta,boot,system,vendor,dtbo"; + }; + fstab { + compatible = "android,fstab"; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,avb"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dts b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dts new file mode 100755 index 000000000000..66e83916c728 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dts @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include "rk3328-rock64-android.dtsi" + +/ { + model = "Pine64 Rock64"; + compatible = "pine64,rock64-android", "rockchip,rk3328"; +}; + +&firmware_android{ + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/platform/ff520000.dwmmc/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/platform/ff520000.dwmmc/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dtsi new file mode 100755 index 000000000000..3dad4f4fafcb --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3328-rock64-android.dtsi @@ -0,0 +1,612 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + */ + +#include "rk3328.dtsi" +#include "rk3328-android.dtsi" +#include + +/ { + gmac_clkin: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "gmac_clkin"; + #clock-cells = <0>; + }; + + vcc_sd: sdmmc-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 30 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0m1_gpio>; + regulator-name = "vcc_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_io>; + }; + + vcc_host_5v: vcc-host-5v-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb30_host_drv>; + regulator-name = "vcc_host_5v"; + regulator-always-on; + vin-supply = <&vcc_sys>; + }; + + vcc_host1_5v: vcc_otg_5v: vcc-host1-5v-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 27 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb20_host_drv>; + regulator-name = "vcc_host1_5v"; + regulator-always-on; + vin-supply = <&vcc_sys>; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip-rk3328"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&codec>; + }; + }; + + hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip-hdmi"; + simple-audio-card,cpu { + sound-dai = <&i2s0>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; +}; + +&avsd { + status = "okay"; +}; + +&codec { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_arm>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; + status = "okay"; +}; + +&gmac2io { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio1 RK_PC2 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_MAC2IO>, <&cru SCLK_MAC2IO_EXT>; + assigned-clock-parents = <&gmac_clkin>, <&gmac_clkin>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmiim1_pins>; + tx_delay = <0x26>; + rx_delay = <0x11>; + status = "okay"; +}; + +&gmac2phy { + phy-supply = <&vcc_phy>; + clock_in_out = "output"; + assigned-clocks = <&cru SCLK_MAC2PHY_SRC>; + assigned-clock-rate = <50000000>; + assigned-clocks = <&cru SCLK_MAC2PHY>; + assigned-clock-parents = <&cru SCLK_MAC2PHY_SRC>; + status = "disabled"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_logic>; +}; + +&hdmi { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&hdmiphy { + status = "okay"; +}; + +&i2c1 { + status = "okay"; + + rk805: rk805@18 { + compatible = "rockchip,rk805"; + status = "okay"; + reg = <0x18>; + interrupt-parent = <&gpio2>; + interrupts = <6 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + wakeup-source; + gpio-controller; + #gpio-cells = <2>; + #clock-cells = <1>; + clock-output-names = "rk805-clkout1", "rk805-clkout2"; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc5-supply = <&vcc_io>; + vcc6-supply = <&vcc_io>; + + rtc { + status = "okay"; + }; + + pwrkey { + status = "disabled"; + }; + + gpio { + status = "okay"; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-name = "vdd_logic"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1450000>; + regulator-initial-mode = <0x1>; + regulator-ramp-delay = <12500>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-mode = <0x2>; + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vdd_arm: DCDC_REG2 { + regulator-name = "vdd_arm"; + regulator-init-microvolt = <1225000>; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1450000>; + regulator-initial-mode = <0x1>; + regulator-ramp-delay = <12500>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-mode = <0x2>; + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-mode = <0x2>; + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-name = "vcc_io"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-mode = <0x2>; + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vdd_18: LDO_REG1 { + regulator-name = "vdd_18"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_18emmc: LDO_REG2 { + regulator-name = "vcc_18emmc"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd_11: LDO_REG3 { + regulator-name = "vdd_11"; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1100000>; + }; + }; + }; + }; +}; + +&i2s0 { + #sound-dai-cells = <0>; + rockchip,bclk-fs = <128>; + status = "okay"; +}; + +&i2s1 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&io_domains { + status = "okay"; + + vccio1-supply = <&vcc_io>; + vccio2-supply = <&vcc_18emmc>; + vccio3-supply = <&vcc_io>; + vccio4-supply = <&vdd_18>; + vccio5-supply = <&vcc_io>; + vccio6-supply = <&vcc_io>; + pmuio-supply = <&vcc_io>; +}; + +&mpp_srv { + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <2 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb2 { + usb20_host_drv: usb20-host-drv { + rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb3 { + usb30_host_drv: usb30-host-drv { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pwm3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pwmir_pin>; + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <1>; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_PLAYPAUSE>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xa4 KEY_SETUP>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +&rga { + status = "okay"; +}; + +&rkvdec { + status = "okay"; + vcodec-supply = <&vdd_logic>; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,virtual-poweroff = <1>; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + max-frequency = <150000000>; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_dectn &sdmmc0_bus4>; + no-sdio; + no-mmc; + status = "okay"; + vmmc-supply = <&vcc_sd>; +}; + +&spi0 { + status = "okay"; + + flash@0 { + compatible = "gigadevice,gd25q128", "jedec,spi-nor"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + m25p,fast-read; + /* The max SCLK of the flash 104/80 MHZ */ + spi-max-frequency = <50000000>; + }; +}; + +&threshold { + temperature = <90000>; /* millicelsius */ +}; + +&target { + temperature = <105000>; /* millicelsius */ +}; + +&soc_crit { + temperature = <115000>; /* millicelsius */ +}; + +&tsadc { + rockchip,hw-tshut-temp = <120000>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + +}; + +&u2phy_host { + phy-supply = <&vcc_host1_5v>; + status = "okay"; +}; + +&u2phy_otg { + phy-supply = <&vcc_otg_5v>; + status = "okay"; +}; + +&u3phy { + phy-supply = <&vcc_host_5v>; + status = "okay"; +}; + +&u3phy_utmi { + status = "okay"; +}; + +&u3phy_pipe { + status = "okay"; +}; + +&usb20_otg { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usbdrd3 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vepu22 { + status = "okay"; +}; + +&vepu22_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts b/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts index 95ab6928cfd4..4c33c21eee8d 100644 --- a/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts +++ b/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts @@ -21,6 +21,13 @@ gmac_clkin: external-gmac-clock { #clock-cells = <0>; }; + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + vcc_sd: sdmmc-regulator { compatible = "regulator-fixed"; gpio = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; @@ -176,7 +183,7 @@ rk805: pmic@18 { interrupt-parent = <&gpio2>; interrupts = <6 IRQ_TYPE_LEVEL_LOW>; #clock-cells = <1>; - clock-output-names = "xin32k", "rk805-clkout2"; + clock-output-names = "rk805-clkout1", "rk805-clkout2"; gpio-controller; #gpio-cells = <2>; pinctrl-names = "default"; @@ -384,11 +391,6 @@ &usb20_otg { status = "okay"; }; -&usbdrd3 { - dr_mode = "host"; - status = "okay"; -}; - &usb_host0_ehci { status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3328.dtsi b/arch/arm64/boot/dts/rockchip/rk3328.dtsi index 72112fe05a5c..a3fcf36f8ecc 100644 --- a/arch/arm64/boot/dts/rockchip/rk3328.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3328.dtsi @@ -318,13 +318,13 @@ power: power-controller { #address-cells = <1>; #size-cells = <0>; - power-domain@RK3328_PD_HEVC { + pd_hevc@RK3328_PD_HEVC { reg = ; }; - power-domain@RK3328_PD_VIDEO { + pd_video@RK3328_PD_VIDEO { reg = ; }; - power-domain@RK3328_PD_VPU { + pd_vpu@RK3328_PD_VPU { reg = ; clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; }; @@ -456,7 +456,7 @@ wdt: watchdog@ff1a0000 { compatible = "snps,dw-wdt"; reg = <0x0 0xff1a0000 0x0 0x100>; interrupts = ; - clocks = <&cru PCLK_WDT>; + clocks = <&cru PCLK_BUS_PRE>; }; pwm0: pwm@ff1b0000 { @@ -464,7 +464,7 @@ pwm0: pwm@ff1b0000 { reg = <0x0 0xff1b0000 0x0 0x10>; clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_pin>; #pwm-cells = <3>; status = "disabled"; @@ -475,7 +475,7 @@ pwm1: pwm@ff1b0010 { reg = <0x0 0xff1b0010 0x0 0x10>; clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_pin>; #pwm-cells = <3>; status = "disabled"; @@ -486,7 +486,7 @@ pwm2: pwm@ff1b0020 { reg = <0x0 0xff1b0020 0x0 0x10>; clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm2_pin>; #pwm-cells = <3>; status = "disabled"; @@ -498,7 +498,7 @@ pwm3: pwm@ff1b0030 { interrupts = ; clocks = <&cru SCLK_PWM>, <&cru PCLK_PWM>; clock-names = "pwm", "pclk"; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwmir_pin>; #pwm-cells = <3>; status = "disabled"; @@ -552,10 +552,9 @@ tsadc: tsadc@ff250000 { assigned-clock-rates = <50000>; clocks = <&cru SCLK_TSADC>, <&cru PCLK_TSADC>; clock-names = "tsadc", "apb_pclk"; - pinctrl-names = "init", "default", "sleep"; + pinctrl-names = "gpio", "otpout"; pinctrl-0 = <&otp_pin>; pinctrl-1 = <&otp_out>; - pinctrl-2 = <&otp_pin>; resets = <&cru SRST_TSADC>; reset-names = "tsadc-apb"; rockchip,grf = <&grf>; @@ -984,25 +983,6 @@ usb_host0_ohci: usb@ff5d0000 { status = "disabled"; }; - usbdrd3: usb@ff600000 { - compatible = "rockchip,rk3328-dwc3", "snps,dwc3"; - reg = <0x0 0xff600000 0x0 0x100000>; - interrupts = ; - clocks = <&cru SCLK_USB3OTG_REF>, <&cru SCLK_USB3OTG_SUSPEND>, - <&cru ACLK_USB3OTG>; - clock-names = "ref_clk", "suspend_clk", - "bus_clk"; - dr_mode = "otg"; - phy_type = "utmi_wide"; - snps,dis-del-phy-power-chg-quirk; - snps,dis_enblslpm_quirk; - snps,dis-tx-ipgap-linecheck-quirk; - snps,dis-u2-freeclk-exists-quirk; - snps,dis_u2_susphy_quirk; - snps,dis_u3_susphy_quirk; - status = "disabled"; - }; - gic: interrupt-controller@ff811000 { compatible = "arm,gic-400"; #interrupt-cells = <3>; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-808-evb.dts b/arch/arm64/boot/dts/rockchip/rk3368-808-evb.dts new file mode 100755 index 000000000000..7e803a8ed30c --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-808-evb.dts @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3368-808.dtsi" + +/ { + model = "Rockchip rk3368 808 evb board"; + compatible = "rockchip,rk3368-808-evb", "rockchip,rk3368"; +}; + +&chosen { + bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 androidboot.selinux=permissive init=/init kpti=0"; +}; + +&fiq_debugger { + status = "okay"; +}; + +&cif { + status = "disabled"; +}; + +&cif_clkout { + /* cif_clkout */ + rockchip,pins = <1 RK_PB3 1 &pcfg_pull_none_4ma>; +}; + +&dmc { + vop-dclk-mode = <1>; + status = "okay"; +}; + +&isp_dvp_d2d9 { + rockchip,pins = + /* cif_data4 ... cif_data9 */ + <1 RK_PA2 1 &pcfg_pull_down>, + <1 RK_PA3 1 &pcfg_pull_down>, + <1 RK_PA4 1 &pcfg_pull_down>, + <1 RK_PA5 1 &pcfg_pull_down>, + <1 RK_PA6 1 &pcfg_pull_down>, + <1 RK_PA7 1 &pcfg_pull_down>, + /* cif_sync, cif_href */ + <1 RK_PB0 1 &pcfg_pull_down>, + <1 RK_PB1 1 &pcfg_pull_down>, + /* cif_clkin */ + <1 RK_PB2 1 &pcfg_pull_down>; +}; + +&isp_dvp_d10d11 { + rockchip,pins = + /* cif_data10, cif_data11 */ + <1 RK_PB6 1 &pcfg_pull_down>, + <1 RK_PB7 1 &pcfg_pull_down>; +}; + +&i2c3 { + status = "okay"; + + gc2145: gc2145@3c { + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + + pinctrl-names = "default"; + pinctrl-0 = <&isp_dvp_d2d9 &isp_dvp_d10d11 &cif_clkout>; + power-gpio = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_out: endpoint { + remote-endpoint = <&isp_dvp_in>; + }; + }; + }; + + ov5695: ov5695@36 { + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + pwdn-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; + +}; + +&isp { + status = "disabled"; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&pinctrl { + pcfg_pull_none_4ma: pcfg-pull-none-4ma { + bias-disable; + drive-strength = <4>; + }; +}; + +&rkisp1 { + status = "okay"; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_dvp_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc2145_out>; + }; + + isp_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx_out>; + }; + + }; +}; + +/* + * In sleep mode, should be close vcca_33 and vcc_lan, + * but due to small defects in hardware settings and + * the system sleeps and wakes up, 4g module can not disconnect the network, + * so system sleep mode cannot be turned off vcca_33 and vcc_lan. + * This configuration will result in increased power consumption, + * please configure according to the actual needs of the project. + */ +&vcca_33 { + regulator-state-mem { + regulator-on-in-suspend; + }; +}; + +&vcc_lan { + regulator-state-mem { + regulator-on-in-suspend; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-808.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-808.dtsi new file mode 100755 index 000000000000..fdea0c73932e --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-808.dtsi @@ -0,0 +1,982 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" + +/ { + rt5640-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rt5640-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&i2s_8ch>; + }; + codec { + sound-dai = <&rt5640>; + }; + }; + + simple-audio-card,dai-link@1 { + format = "i2s"; + cpu { + sound-dai = <&i2s_8ch>; + }; + codec { + sound-dai = <&hdmi>; + }; + }; + }; + + rk_headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio2 RK_PC3 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + ext_gmac: gmac-clk { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "ext_gmac"; + #clock-cells = <0>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PB2 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; + + vcc_lcd: vcc-lcd-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio3 RK_PD5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd_pwr>; + regulator-name = "vcc_lcd"; + enable-active-high; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_otg_vbus: otg-vbus-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&otg_vbus_drv>; + regulator-name = "vcc_otg_vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm3 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + panel { + compatible = "samsung,lsl070nl01", "simple-panel"; + backlight = <&backlight>; + enable-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_HIGH>; + enable-delay-ms = <120>; + prepare-delay-ms = <2>; + unprepare-delay-ms = <20>; + disable-delay-ms = <50>; + width-mm = <68>; + height-mm = <121>; + rockchip,data-mapping = "vesa"; + rockchip,data-width = <24>; + rockchip,output = "lvds"; + status = "disabled"; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <160000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <60>; + hfront-porch = <80>; + vback-porch = <25>; + vfront-porch = <35>; + hsync-len = <1>; + vsync-len = <1>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3500>; + rockchip,screen-on-voltage = <3600>; + status = "okay"; + }; + + gpio_keys: gpio-keys { + compatible = "gpio-keys"; + autorepeat; + + power { + debounce-interval = <100>; + gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; + label = "GPIO Key Power"; + linux,code = ; + wakeup-source; + }; + }; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1024000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1000>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <170000>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6255"; + sdio_vref = <1800>; //1800mv or 3300mv + WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_rts_gpio>; + + /* BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; */ + BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; + + status = "okay"; + }; + + rk_modem: rk-modem { + compatible="4g-modem-platdata"; + pinctrl-names = "default"; + pinctrl-0 = <<e_vbat <e_power_en <e_reset>; + 4G,vbat-gpio = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>; + 4G,power-gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_LOW>; + 4G,reset-gpio = <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b3 { + cpu-supply = <&vdd_cpu>; +}; + +&gpu { + logic-supply = <&vdd_log>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_log>; + devfreq-events = <&dfi>; + upthreshold = <60>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 600000 + SYS_STATUS_REBOOT 600000 + SYS_STATUS_SUSPEND 240000 + SYS_STATUS_VIDEO_1080P 396000 + SYS_STATUS_VIDEO_4K 600000 + SYS_STATUS_PERFORMANCE 600000 + SYS_STATUS_BOOST 396000 + SYS_STATUS_DUALVIEW 600000 + SYS_STATUS_ISP 528000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 582 240000 + 583 99999 396000 + >; + auto-min-freq = <240000>; + auto-freq-en = <0>; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF + | RKPM_SLP_PMU_PLLS_PWRDN + | RKPM_SLP_PMU_PMUALIVE_32K + | RKPM_SLP_SFT_PLLS_DEEP + | RKPM_SLP_PMU_DIS_OSC + | RKPM_SLP_SFT_PD_NBSCUS + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_USB_WKUP_EN + | RKPM_CLUSTER_L_WKUP_EN + ) + >; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&sdmmc { + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "disabled"; +}; + +&sdio0 { + clock-frequency = <100000000>; + clock-freq-min-max = <200000 100000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk808: pmic@1b { + status = "okay"; + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio0>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int>, <&pmic_sleep>; + rockchip,system-power-controller; + wakeup-source; + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc_io>; + vcc9-supply = <&vcc_sys>; + vcc10-supply = <&vcc_sys>; + vcc11-supply = <&vcc_sys>; + vcc12-supply = <&vcc_io>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + #clock-cells = <1>; + + regulators { + vdd_cpu: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vdd_cpu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_log: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vdd_log"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_io"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc18_flash: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc18_flash"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcca_33: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcca_33"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_10: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd_10"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcca_18: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca_18"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vdd10_lcd: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd10_lcd"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_18: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_18"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc18_lcd: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc18_lcd"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sd: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_lan: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_lan"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; + +}; + +&i2c1 { + status = "okay"; + + rt5640: rt5640@1c { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "realtek,rt5640"; + reg = <0x1c>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + realtek,in1-differential; + /* spk-con-gpio = <&gpio3 9 GPIO_ACTIVE_HIGH>; */ + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + }; + + mpu6500_acc: mpu_acc@68 { + status = "okay"; + compatible = "mpu6500_acc"; + pinctrl-names = "default"; + pinctrl-0 = <&mpu6500_irq_gpio>; + reg = <0x68>; + irq-gpio = <&gpio2 17 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + power-off-in-suspend = <1>; + layout = <5>; + + }; + + mpu6500_gyro: mpu_gyro@68 { + status = "okay"; + compatible = "mpu6500_gyro"; + reg = <0x68>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + power-off-in-suspend = <1>; + layout = <5>; + }; + + ak8963_compass: ak8963_compass@d { + status = "okay"; + compatible = "ak8963"; + pinctrl-names = "default"; + pinctrl-0 = <&ak8963_irq_gpio>; + reg = <0x0d>; + type = ; + irq-gpio = <&gpio2 18 IRQ_TYPE_EDGE_RISING>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <7>; + }; +}; + +&i2c2 { + status = "okay"; + + gslx680@40 { + compatible = "gslX6801"; + reg = <0x40>; + screen_max_x = <1920>; + screen_max_y = <1200>; + power-supply = <&vcc_lcd>; + touch-gpio = <&gpio0 12 IRQ_TYPE_LEVEL_LOW>; + reset-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + +}; + +&i2c3 { + status = "okay"; +}; + +&i2s_8ch { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_2ch_bus>; +}; + +&io_domains { + status = "okay"; + + audio-supply = <&vcca_18>; + dvp-supply = <&vcc_18>; + flash0-supply = <&vcc18_flash>; + gpio30-supply = <&vcc_io>; + gpio1830-supply = <&vcc_io>; + sdcard-supply = <&vccio_sd>; + wifi-supply = <&vcc_io>; +}; + +&pmu_io_domains { + status = "okay"; + + pmu-supply = <&vcc_io>; + vop-supply = <&vcca_33>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm3 { + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&uart2 { + status = "disabled"; +}; + +&saradc { + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_otg: otg-port { + status = "okay"; + vbus-supply = <&vcc_otg_vbus>; + }; + + u2phy_host: host-port { + status = "okay"; + }; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&edp { + status = "disabled"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "sitronix,st7703", "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + power-supply = <&vcc_lcd>; + prepare-delay-ms = <2>; + reset-delay-ms = <1>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <20>; + + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&st7703_timing>; + + st7703_timing: timing0 { + clock-frequency = <160000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <60>; + hfront-porch = <80>; + vback-porch = <25>; + vfront-porch = <35>; + hsync-len = <1>; + vsync-len = <1>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&video_phy { + status = "okay"; +}; + +&route_dsi { + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&vdd_cpu>; + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 11 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_MAC>; + assigned-clock-parents = <&ext_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&hdmi { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&route_hdmi { + status = "okay"; +}; + +&pinctrl { + camera { + camera_pwr: camera-pwr { + rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + lcd { + lcd_pwr: lcd-pwr { + rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + i2s { + i2s_2ch_bus: i2s-2ch-bus { + rockchip,pins = <2 RK_PB4 1 &pcfg_pull_none>, + <2 RK_PB5 1 &pcfg_pull_none>, + <2 RK_PB6 1 &pcfg_pull_none>, + <2 RK_PB7 1 &pcfg_pull_none>, + <2 RK_PC0 1 &pcfg_pull_none>; + }; + }; + + pmic { + pmic_sleep: pmic-sleep { + rockchip,pins = <0 RK_PA0 2 &pcfg_pull_none>; + }; + + pmic_int: pmic-int { + rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + mpu6500 { + mpu6500_irq_gpio: mpu6500-irq-gpio { + rockchip,pins = <2 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + ak8963 { + ak8963_irq_gpio: ak8963_irq_gpio { + rockchip,pins = <2 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + dc_det { + dc_irq_gpio: dc-irq-gpio { + rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb { + otg_vbus_drv: otg-vbus-drv { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_rts_gpio: uart0-rts-gpio { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + rk-modem { + lte_vbat: lte-vbat { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lte_power_en: lte-power-en { + rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lte_reset: lte-reset { + rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3368-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-android.dtsi new file mode 100755 index 000000000000..a3e0bbaaf7c5 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-android.dtsi @@ -0,0 +1,357 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/ { + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff690000 firmware_class.path=/system/vendor/firmware"; + }; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + rockchip,irq-mode-enable = <0>; /* If enable uart uses irq instead of fiq */ + rockchip,baudrate = <115200>; /* Only 115200 and 1500000 */ + pinctrl-names = "default"; + pinctrl-0 = <&uart2_xfer>; + interrupts = ; /* signal irq */ + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + drm_logo: drm-logo@00000000 { + compatible = "rockchip,drm-logo"; + reg = <0x0 0x0 0x0 0x0>; + }; + + ramoops: ramoops@110000 { + compatible = "ramoops"; + reg = <0x0 0x110000 0x0 0xf0000>; + record-size = <0x20000>; + console-size = <0x80000>; + ftrace-size = <0x00000>; + pmsg-size = <0x50000>; + }; + + /* global autoconfigured region for contiguous allocations */ + linux,cma { + compatible = "shared-dma-pool"; + reusable; + size = <0x0 0x2000000>; + linux,cma-default; + }; + }; + + ion { + compatible = "rockchip,ion"; + #address-cells = <1>; + #size-cells = <0>; + + cma-heap { + reg = <0x00000000 0x2800000>; + }; + + system-heap { + }; + }; + + firmware { + firmware_android: android {}; + }; + + rga@ff920000 { + compatible = "rockchip,rga2"; + dev_mode = <1>; + reg = <0x0 0xff920000 0x0 0x1000>; + interrupts = ; + clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru SCLK_RGA>; + clock-names = "aclk_rga", "hclk_rga", "clk_rga"; + status = "okay"; + }; +}; + +&cluster1_opp { + rockchip,avs = <1>; +}; + +&display_subsystem { + status = "okay"; + + logo-memory-region = <&drm_logo>; + route { + route_dsi: route-dsi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vop_out_dsi>; + }; + + route_edp: route-edp { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vop_out_edp>; + }; + + route_hdmi: route-hdmi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vop_out_hdmi>; + }; + + route_lvds: route-lvds { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vop_out_lvds>; + }; + + route_rgb: route-rgb { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vop_out_rgb>; + }; + + }; +}; + +&dsi { + panel@0 { + reg = <0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&hevc { + status = "okay"; +}; + +&hevc_mmu { + status = "okay"; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&mailbox { + status = "okay"; +}; + +&mailbox_scpi { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&vop { + support-multi-area; + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; + +&isp { + status = "okay"; +}; + +&isp_mmu { + status = "okay"; +}; + +&cif { + status = "okay"; +}; + +&rng { + status = "okay"; +}; + +&vip_mmu { + status = "okay"; +}; + +&video_phy { + status = "okay"; +}; + +&usb_otg { + status = "okay"; +}; + +&pinctrl { + isp { + cif_clkout: cif-clkout { + rockchip,pins = <1 RK_PB3 1 &pcfg_pull_none>;//cif_clkout + }; + + isp_dvp_d2d9: isp-dvp-d2d9 { + rockchip,pins = + <1 RK_PA0 1 &pcfg_pull_none>,//cif_data2 + <1 RK_PA1 1 &pcfg_pull_none>,//cif_data3 + <1 RK_PA2 1 &pcfg_pull_none>,//cif_data4 + <1 RK_PA3 1 &pcfg_pull_none>,//cif_data5 + <1 RK_PA4 1 &pcfg_pull_none>,//cif_data6 + <1 RK_PA5 1 &pcfg_pull_none>,//cif_data7 + <1 RK_PA6 1 &pcfg_pull_none>,//cif_data8 + <1 RK_PA7 1 &pcfg_pull_none>,//cif_data9 + <1 RK_PB0 1 &pcfg_pull_none>,//cif_sync + <1 RK_PB1 1 &pcfg_pull_none>,//cif_href + <1 RK_PB2 1 &pcfg_pull_none>,//cif_clkin + <1 RK_PB3 1 &pcfg_pull_none>;//cif_clkout + }; + + isp_dvp_d0d1: isp-dvp-d0d1 { + rockchip,pins = + <1 RK_PB4 1 &pcfg_pull_none>,//cif_data0 + <1 RK_PB5 1 &pcfg_pull_none>;//cif_data1 + }; + + isp_dvp_d10d11:isp_d10d11 { + rockchip,pins = + <1 RK_PB6 1 &pcfg_pull_none>,//cif_data10 + <1 RK_PB7 1 &pcfg_pull_none>;//cif_data11 + }; + + isp_dvp_d0d7: isp-dvp-d0d7 { + rockchip,pins = + <1 RK_PB4 1 &pcfg_pull_none>,//cif_data0 + <1 RK_PB5 1 &pcfg_pull_none>,//cif_data1 + <1 RK_PA0 1 &pcfg_pull_none>,//cif_data2 + <1 RK_PA1 1 &pcfg_pull_none>,//cif_data3 + <1 RK_PA2 1 &pcfg_pull_none>,//cif_data4 + <1 RK_PA3 1 &pcfg_pull_none>,//cif_data5 + <1 RK_PA4 1 &pcfg_pull_none>,//cif_data6 + <1 RK_PA5 1 &pcfg_pull_none>;//cif_data7 + }; + + isp_dvp_d4d11: isp-dvp-d4d11 { + rockchip,pins = + <1 RK_PA2 1 &pcfg_pull_none>,//cif_data4 + <1 RK_PA3 1 &pcfg_pull_none>,//cif_data5 + <1 RK_PA4 1 &pcfg_pull_none>,//cif_data6 + <1 RK_PA5 1 &pcfg_pull_none>,//cif_data7 + <1 RK_PA6 1 &pcfg_pull_none>,//cif_data8 + <1 RK_PA7 1 &pcfg_pull_none>,//cif_data9 + <1 RK_PB6 1 &pcfg_pull_none>,//cif_data10 + <1 RK_PC1 1 &pcfg_pull_none>;//cif_data11 + }; + + isp_shutter: isp-shutter { + rockchip,pins = + <3 RK_PC3 2 &pcfg_pull_none>, //SHUTTEREN + <3 RK_PC6 2 &pcfg_pull_none>;//SHUTTERTRIG + }; + + isp_flash_trigger: isp-flash-trigger { + rockchip,pins = <3 RK_PC4 2 &pcfg_pull_none>; //ISP_FLASHTRIGOU + }; + + isp_prelight: isp-prelight { + rockchip,pins = <3 RK_PC5 2 &pcfg_pull_none>;//ISP_PRELIGHTTRIG + }; + + isp_flash_trigger_as_gpio: isp_flash_trigger_as_gpio { + rockchip,pins = <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>;//ISP_FLASHTRIGOU + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-cif-sensor.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-cif-sensor.dtsi new file mode 100755 index 000000000000..7d2aac22c020 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-cif-sensor.dtsi @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +#include "../../../../../drivers/soc/rockchip/rk_camera_sensor_info.h" + +/{ + cif_sensor: cif_sensor { + compatible = "rockchip,sensor"; + status = "okay"; + + tp2825 { + status = "okay"; + is_front = <0>; + powerdown-gpios = <&gpio1 12 GPIO_ACTIVE_HIGH>; + irq-gpios = <&gpio1 13 IRQ_TYPE_EDGE_FALLING>; + pwdn_active = ; + mir = <0>; + flash_attach = <1>; + flash_active = <1>; + resolution = ; + powerup_sequence = ; + orientation = <0>; + i2c_add = ; + i2c_chl = <3>; + cif_chl = <0>; + ad_chl = <0>; // 0 ~ 4; + mclk_rate = <24>; + rockchip,camera-module-defrect0 = <960 480 0 4 960 472>; + rockchip,camera-module-interface0 = "cvbs_ntsc"; + rockchip,camera-module-defrect1 = <960 576 0 4 960 568>; + rockchip,camera-module-interface1 = "cvbs_pal"; + rockchip,camera-module-defrect2 = <1280 720 8 20 1280 720>; + rockchip,camera-module-interface2 = "bt601_8_pp"; + rockchip,camera-module-channel = <4 0>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-dram-default-timing.dtsi new file mode 100755 index 000000000000..10bfbebda989 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-dram-default-timing.dtsi @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include + +/ { + ddr_timing: ddr_timing { + compatible = "rockchip,ddr-timing"; + dram_spd_bin = ; + sr_idle = <1>; + pd_idle = <0x20>; + dram_dll_disb_freq = <300>; + phy_dll_disb_freq = <400>; + dram_odt_disb_freq = <333>; + phy_odt_disb_freq = <333>; + ddr3_drv = ; + ddr3_odt = ; + lpddr3_drv = ; + lpddr3_odt = ; + lpddr2_drv = ; /* lpddr2 not supported odt */ + phy_clk_drv = ; + phy_cmd_drv = ; + phy_dqs_drv = ; + phy_odt = ; + ddr_2t = ; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-p9-avb.dts b/arch/arm64/boot/dts/rockchip/rk3368-p9-avb.dts new file mode 100755 index 000000000000..6a971781032a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-p9-avb.dts @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3368-p9.dtsi" + +/ { + model = "Rockchip rk3368 p9 avb board"; + compatible = "rockchip,p9-avb", "rockchip,rk3368"; +}; + +&firmware_android { + compatible = "android,firmware"; + boot_devices = "ff0f0000.dwmmc,ff400000.nandc"; + vbmeta { + compatible = "android,vbmeta"; + parts = "vbmeta,dtbo"; + }; + fstab { + compatible = "android,fstab"; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,avb"; + }; + }; +}; + +&chosen { + bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 androidboot.selinux=permissive init=/init kpti=0"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-p9.dts b/arch/arm64/boot/dts/rockchip/rk3368-p9.dts new file mode 100755 index 000000000000..29658be08696 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-p9.dts @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3368-p9.dtsi" + +/ { + model = "Rockchip rk3368 p9 board"; + compatible = "rockchip,p9", "rockchip,rk3368"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3368-p9.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-p9.dtsi new file mode 100755 index 000000000000..7eeb9db4f13b --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-p9.dtsi @@ -0,0 +1,841 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include +#include +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" + +/ { + es8316-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk-es8316-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&es8316>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk818 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + enable-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3500>; + rockchip,screen-on-voltage = <3600>; + status = "okay"; + }; + + rk_key: rockchip-key { + compatible = "rockchip,key"; + status = "okay"; + + io-channels = <&saradc 1>; + + vol-up-key { + linux,code = <115>; + label = "volume up"; + rockchip,adc_value = <1>; + }; + + vol-down-key { + linux,code = <114>; + label = "volume down"; + rockchip,adc_value = <170>; + }; + + power-key { + gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; + linux,code = <116>; + label = "power"; + gpio-key,wakeup; + }; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + /* wifi_chip_type - wifi chip define + * ap6210, ap6330, ap6335 + * rtl8188eu, rtl8723bs, rtl8723bu + * esp8089 + */ + wifi_chip_type = "ap6210"; + sdio_vref = <1800>; //1800mv or 3300mv + WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk818 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_rts_gpio>; + + //BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; + BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; + + status = "okay"; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_host: vcc-host { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; +}; + +&cpu_l0 { + cpu-supply = <&syr827>; +}; + +&cpu_l1 { + cpu-supply = <&syr827>; +}; + +&cpu_l2 { + cpu-supply = <&syr827>; +}; + +&cpu_l3 { + cpu-supply = <&syr827>; +}; + +&cpu_b0 { + cpu-supply = <&syr827>; +}; + +&cpu_b1 { + cpu-supply = <&syr827>; +}; + +&cpu_b2 { + cpu-supply = <&syr827>; +}; + +&cpu_b3 { + cpu-supply = <&syr827>; +}; + +&gpu { + logic-supply = <&vdd_logic>; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF + | RKPM_SLP_PMU_PLLS_PWRDN + | RKPM_SLP_PMU_PMUALIVE_32K + | RKPM_SLP_SFT_PLLS_DEEP + | RKPM_SLP_PMU_DIS_OSC + | RKPM_SLP_SFT_PD_NBSCUS + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_USB_WKUP_EN + | RKPM_CLUSTER_L_WKUP_EN + ) + >; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; + status = "okay"; +}; + +&nandc0 { + status = "disabled"; +}; + +&sdmmc { + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "disabled"; +}; + +&sdio0 { + clock-frequency = <50000000>; + clock-freq-min-max = <200000 50000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + syr827: syr827@40 { + compatible = "silergy,syr827"; + status = "okay"; + reg = <0x40>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_arm"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + pinctrl-0 = <&vsel_gpio>; + vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + rk818: pmic@1c { + compatible = "rockchip,rk818"; + status = "okay"; + reg = <0x1c>; + clock-output-names = "rk818-clkout1", "wifibt_32kin"; + interrupt-parent = <&gpio0>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc_sys>; + vcc9-supply = <&vcc_io>; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-name = "vdd_logic"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1450000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-name = "vdd_gpu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1250000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_io"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca_codec: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcca_codec"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_tp: LDO_REG2 { + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_10: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd_10"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc18_lcd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc18_lcd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_pmu: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vccio_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd10_lcd: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd10_lcd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_18: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_18"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_wl: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vccio_wl"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: SWITCH_REG { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + boost_otg: DCDC_BOOST { + regulator-name = "boost_otg"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + + hdmi_switch: HDMI_SWITCH { + regulator-always-on; + regulator-boot-on; + regulator-name = "hdmi_switch"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + battery { + compatible = "rk818-battery"; + pinctrl-names = "default"; + pinctrl-0 = <&dc_irq_gpio>; + ocv_table = < + 3400 3650 3693 3707 3731 3749 3760 + 3770 3782 3796 3812 3829 3852 3882 + 3915 3951 3981 4047 4086 4132 4182>; + design_capacity = <8650>; + design_qmax = <8800>; + bat_res = <85>; + max_input_current = <2000>; + max_chrg_current = <1800>; + max_chrg_voltage = <4200>; + sleep_enter_current = <600>; + sleep_exit_current = <600>; + power_off_thresd = <3400>; + zero_algorithm_vol = <3850>; + fb_temperature = <115>; + sample_res = <20>; + max_soc_offset = <60>; + energy_mode = <0>; + monitor_sec = <5>; + virtual_power = <0>; + power_dc2otg = <1>; + support_usb_adp = <1>; + support_dc_adp = <1>; + dc_det_gpio = <&gpio0 17 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&i2c1 { + status = "okay"; + + es8316: es8316@10 { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "everest,es8316"; + reg = <0x10>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + spk-con-gpio = <&gpio0 27 GPIO_ACTIVE_HIGH>; + hp-det-gpio = <&gpio0 23 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + }; +}; + +&i2c2 { + status = "okay"; + + gt9xx: gt9xx@14 { + compatible = "goodix,gt9xx"; + reg = <0x14>; + touch-gpio = <&gpio0 12 IRQ_TYPE_LEVEL_LOW>; + reset-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; + max-x = <1920>; + max-y = <1200>; + tp-size = <89>; + status = "okay"; + tp-supply = <&vcc_tp>; + }; +}; + +&i2c3 { + status = "okay"; +}; + +&i2c4 { + status = "okay"; + + mpu6500_acc: mpu_acc@68 { + compatible = "mpu6500_acc"; + reg = <0x68>; + irq-gpio = <&gpio3 14 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + layout = <7>; + }; + + mpu6500_gyro: mpu_gyro@68 { + compatible = "mpu6500_gyro"; + reg = <0x68>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + layout = <7>; + }; + + mpu6500@68 { + status = "disabled"; + compatible = "invensense,mpu6500"; + pinctrl-names = "default"; + pinctrl-0 = <&mpu6500_irq_gpio>; + reg = <0x68>; + irq-gpio = <&gpio3 14 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <1 0 0 0 1 0 0 0 1>; + orientation-x= <1>; + orientation-y= <0>; + orientation-z= <1>; + support-hw-poweroff = <1>; + mpu-debug = <1>; + }; +}; + +&i2s_8ch { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + dvp-supply = <&vcc_18>; + audio-supply = <&vcc_io>; + gpio30-supply = <&vcc_io>; + gpio1830-supply = <&vcc_io>; + sdcard-supply = <&vccio_sd>; + wifi-supply = <&vccio_wl>; +}; + +&pmu_io_domains { + status = "okay"; + + pmu-supply = <&vccio_pmu>; + vop-supply = <&vccio_pmu>; +}; + +&pwm0 { + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&saradc { + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + phy-supply = <&vcc_host>; + status = "okay"; + }; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <120>; + enable-delay-ms = <200>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <145000000>; + hactive = <1920>; + vactive = <1200>; + hback-porch = <16>; + hfront-porch = <24>; + vback-porch = <10>; + vfront-porch = <16>; + hsync-len = <10>; + vsync-len = <3>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&route_dsi { + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&syr827>; + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel_gpio: vsel-gpio { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + mpu6500 { + mpu6500_irq_gpio: mpu6500-irq-gpio { + rockchip,pins = <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + dc_det { + dc_irq_gpio: dc-irq-gpio { + rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_rts_gpio: uart0-rts-gpio { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3368-px5-evb-android.dts b/arch/arm64/boot/dts/rockchip/rk3368-px5-evb-android.dts new file mode 100755 index 000000000000..1d50d3fa549f --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-px5-evb-android.dts @@ -0,0 +1,993 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" +#include "rk3368-cif-sensor.dtsi" +#include +#include + +/ { + model = "Rockchip PX5 EVB V11"; + compatible = "rockchip,px5-evb", "rockchip,px5", "rockchip,rk3368"; + + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff1c0000 firmware_class.path=/system/vendor/firmware"; + }; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <4>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <0>; + /* Only 115200 and 1500000 */ + rockchip,baudrate = <115200>; + pinctrl-names = "default"; + pinctrl-0 = <&uart4_xfer>; + interrupts = ; + }; + + firmware { + android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/platform/ff0f0000.dwmmc/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/platform/ff0f0000.dwmmc/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait"; + }; + }; + }; + }; + + xin32k: xin32k { + status = "okay"; + compatible = "pwm-clock"; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + pwms = <&pwm1 0 30518 0>; /* 1 / 30518 ns = 32.7675 KHz */ + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_LOW>; /* GPIO3_A5 */ + }; + + es8396-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,es8396-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Microphone Jack", + "Line", "Microphone Headset", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "MIC", "Microphone Jack", + "DMIC", "Microphone Headset", + "Headphone Jack", "LOUTP", + "Headphone Jack", "ROUTN"; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&es8396>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + enable-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + }; + + rk_key: rockchip-key { + compatible = "rockchip,key"; + status = "okay"; + + io-channels = <&saradc 1>; + + vol-up-key { + linux,code = <115>; + label = "volume up"; + rockchip,adc_value = <1>; + }; + + vol-down-key { + linux,code = <114>; + label = "volume down"; + rockchip,adc_value = <170>; + }; + + power-key { + gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_LOW>; + linux,code = <116>; + label = "power"; + gpio-key,wakeup; + }; + + menu-key { + linux,code = <59>; + label = "menu"; + rockchip,adc_value = <355>; + }; + + home-key { + linux,code = <102>; + label = "home"; + rockchip,adc_value = <746>; + }; + + back-key { + linux,code = <158>; + label = "back"; + rockchip,adc_value = <560>; + }; + + camera-key { + linux,code = <212>; + label = "camera"; + rockchip,adc_value = <450>; + }; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + + wifi_chip_type = "rtl8723ds"; + sdio_vref = <1800>; /*1800mv or 3300mv*/ + WIFI,host_wake_irq = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; + WIFI,vbat_gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + + keep_bt_power_on; + BT,power_gpio = <&gpio3 4 GPIO_ACTIVE_HIGH>; /* GPIO3_A4 */ + BT,reset_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; /* GPIO3_A2 */ + BT,wake_gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>; /* GPIO3_A7 */ + BT,wake_host_irq = <&gpio3 3 GPIO_ACTIVE_HIGH>; /* GPIO3_A3 */ + + status = "okay"; + }; + + gpio_det: gpio-det { + compatible = "gpio-detection"; + status = "okay"; + + pinctrl-0 = <&gpio3_b1 &gpio3_b2>; + pinctrl-names = "default"; + + car-reverse { + car-reverse-gpios = <&gpio3 10 GPIO_ACTIVE_LOW>; + linux,debounce-ms = <5>; + label = "car-reverse"; + gpio,wakeup; + }; + + car-acc { + car-acc-gpios = <&gpio3 9 GPIO_ACTIVE_LOW>; + linux,debounce-ms = <5>; + label = "car-acc"; + gpio,wakeup; + }; + }; + + vcc_sys: vcc-sys-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_host: vcc-host { + compatible = "regulator-fixed"; + enable-active-low; + gpio = <&gpio0 RK_PA3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + }; + + vcc18_lcd_n: vcc18-lcd-n { + compatible = "regulator-fixed"; + regulator-name = "vcc18_lcd_n"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + gpio = <&gpio2 RK_PD3 GPIO_ACTIVE_HIGH>; + enable-active-high; + startup-delay-us = <70000>; + vin-supply = <&vcc_18>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + panel { + compatible = "samsung,lsl070nl01", "simple-panel"; + power-supply = <&vcc33_lcd>; + backlight = <&backlight>; + prepare-delay-ms = <120>; + unprepare-delay-ms = <120>; + bus-format = ; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <48000000>; + hactive = <1024>; + vactive = <600>; + hback-porch = <90>; + hfront-porch = <90>; + vback-porch = <10>; + vfront-porch = <10>; + hsync-len = <90>; + vsync-len = <10>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in_lvds: endpoint { + remote-endpoint = <&lvds_out_panel>; + }; + }; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&cif_sensor { + status = "okay"; +}; + +&cluster0_opp { + rockchip,threshold-freq = <408000>; + rockchip,freq-limit; +}; + +&cluster1_opp { + rockchip,threshold-freq = <1416000>; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b3 { + cpu-supply = <&vdd_cpu>; +}; + +&gpu { + logic-supply = <&vdd_log>; +}; + +&isp { + pinctrl-names = + "isp_mipi_fl_prefl", "isp_flash_as_gpio", + "isp_flash_as_trigger_out"; + pinctrl-0 = <&isp_prelight>; + pinctrl-1 = <&isp_flash_trigger_as_gpio>; + pinctrl-2 = <&isp_flash_trigger>; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF + | RKPM_SLP_PMU_PLLS_PWRDN + | RKPM_SLP_PMU_PMUALIVE_32K + | RKPM_SLP_SFT_PLLS_DEEP + | RKPM_SLP_PMU_DIS_OSC + | RKPM_SLP_SFT_PD_NBSCUS + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_USB_WKUP_EN + | RKPM_CLUSTER_L_WKUP_EN + ) + >; +}; + +&emmc { + status = "okay"; + bus-width = <8>; + clock-frequency = <150000000>; + keep-power-in-suspend; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; + vmmc-supply = <&vcc_io>; + vqmmc-supply = <&vcc18_flash>; +}; + +&sdmmc { + status = "okay"; + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + vmmc-supply = <&vcc_sd>; + vqmmc-supply = <&vccio_sd>; +}; + +&sdio0 { + status = "okay"; + clock-frequency = <50000000>; + clock-freq-min-max = <200000 50000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; +}; + +&i2c0 { + status = "okay"; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio0>; + interrupts = <5 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc_io>; + vcc9-supply = <&vcc_sys>; + vcc10-supply = <&vcc_sys>; + vcc11-supply = <&vcc_sys>; + vcc12-supply = <&vcc_io>; + + regulators { + vdd_cpu: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vdd_cpu"; + regulator-ramp-delay = <6000>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vdd_log: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vdd_log"; + regulator-ramp-delay = <6000>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_io"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc18_flash: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc18_flash"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcca_33: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcca_33"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vdd_10: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd_10"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + avdd_33: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "avdd_33"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vdd10_lcd: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd10_lcd"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_18: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_18"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc18_lcd: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc18_lcd"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sd: SWITCH_REG1 { + regulator-boot-on; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc33_lcd: SWITCH_REG2 { + regulator-boot-on; + regulator-name = "vcc33_lcd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&i2c1 { + status = "okay"; + + bma2x2: bma250@18 { + compatible = "bma2xx_acc"; + status = "okay"; + reg = <0x18>; + type = ; + pinctrl-names = "default"; + pinctrl-0 = <&gpio2_c1>; + irq-gpio = <&gpio2 17 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <1>; + poll_delay_ms = <200>; + layout = <6>; + reprobe_en = <1>;/* this sensor need to be probe again */ + }; + + ls_stk3410: light@48 { + compatible = "ls_stk3410"; + status = "okay"; + reg = <0x48>; + type = ; + irq_enable = <0>; + als_threshold_high = <100>; + als_threshold_low = <10>; + als_ctrl_gain = <2>; /* 0:x1 1:x4 2:x16 3:x64 */ + poll_delay_ms = <100>; + }; + + ps_stk3410: proximity@48 { + compatible = "ps_stk3410"; + status = "okay"; + reg = <0x48>; + type = ; + pinctrl-names = "default"; + pinctrl-0 = <&gpio2_c3>; + irq-gpio = <&gpio2 19 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <1>; + ps_threshold_high = <0x200>; + ps_threshold_low = <0x100>; + ps_ctrl_gain = <3>; /* 0:x1 1:x4 2:x16 3:x64 */ + ps_led_current = <3>; /* 0:12.5mA 1:25mA 2:50mA 3:100mA */ + poll_delay_ms = <100>; + }; +}; + +&i2c2 { + status = "okay"; + clock-frequency = <200000>; + + gsl1680: touchscreen@40 { + compatible = "silead,gsl1680"; + reg = <0x40>; + interrupt-parent = <&gpio3>; + interrupts = <28 IRQ_TYPE_EDGE_FALLING>; + power-gpios = <&gpio3 RK_PB7 GPIO_ACTIVE_HIGH>; + touchscreen-size-x = <800>; + touchscreen-size-y = <1280>; + silead,max-fingers = <5>; + }; + + gt9xx: gt9xx@14 { + compatible = "goodix,gt9xx"; + reg = <0x14>; + touch-gpio = <&gpio3 RK_PD4 IRQ_TYPE_LEVEL_HIGH>; + reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + max-x = <1024>; + max-y = <600>; + tp-size = <910>; + tp-supply = <&vcc_io>; + status = "okay"; + }; +}; + +&i2c3 { + status = "okay"; + clock-frequency = <200000>; + + fm1288: fm1288@60{ + compatible = "fm1288"; + reg = <0x60>; + pwd-gpios = <&gpio0 RK_PD1 GPIO_ACTIVE_HIGH>; + bypass-gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + es8396: es8396@10 { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "es8396"; + reg = <0x10>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + spk-con-gpio = <&gpio2 RK_PC7 GPIO_ACTIVE_HIGH>; + lineout-con-gpio = <&gpio0 RK_PD7 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + }; +}; + +&i2c4 { + status = "okay"; +}; + +&i2s_8ch { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_bus>; +}; + +&io_domains { + status = "okay"; + dvp-supply = <&vcc_18>; + audio-supply = <&vcc_io>; + gpio30-supply = <&vcc_io>; + gpio1830-supply = <&vcc_io>; + sdcard-supply = <&vccio_sd>; + wifi-supply = <&vcc_io>; +}; + +&pmu_io_domains { + status = "okay"; + pmu-supply = <&vcc_io>; + vop-supply = <&vcca_33>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm1 { + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + phy-supply = <&vcc_host>; + status = "okay"; + }; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + vop-dclk-mode = <1>; + center-supply = <&vdd_log>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 600000 + SYS_STATUS_REBOOT 600000 + SYS_STATUS_SUSPEND 192000 + SYS_STATUS_VIDEO_1080P 600000 + SYS_STATUS_VIDEO_4K 600000 + SYS_STATUS_PERFORMANCE 600000 + SYS_STATUS_BOOST 600000 + SYS_STATUS_DUALVIEW 600000 + SYS_STATUS_ISP 600000 + >; + status = "okay"; +}; + +&lvds { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + lvds_out_panel: endpoint { + remote-endpoint = <&panel_in_lvds>; + }; + }; + }; +}; + +&route_lvds { + status = "okay"; +}; + +&mailbox { + status = "okay"; +}; + +&mailbox_scpi { + status = "okay"; +}; + +&saradc { + status = "okay"; +}; + +&thermal_zones { + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&vdd_cpu>; + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_xfer &uart1_cts>; + status = "okay"; +}; + +&uart2 { + status = "disabled"; +}; + +&uart4 { + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int: pmic-int { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + dc_det { + dc_irq_gpio: dc-irq-gpio { + rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_rts_gpio: uart0-rts-gpio { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + gpio0_gpio { + gpio0_c7: gpio0-c7 { + rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_down>; + }; + gpio0_a3: gpio0-a3 { + rockchip,pins = <0 RK_PA3 3 &pcfg_pull_none>; + }; + gpio0_c2: gpio0-c2 { + rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + gpio0_c3: gpio0-c3 { + rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + gpio0_c1: gpio0-c1 { + rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + gpio2_c1: gpio2-c1 { + rockchip,pins = <2 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + gpio2_c3: gpio2-c3 { + rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + gpio3_b1: gpio3-b1 { + rockchip,pins = <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + gpio3_b2: gpio3-b2 { + rockchip,pins = <3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + i2s { + i2s_8ch_bus: i2s-8ch-bus { + rockchip,pins = <2 RK_PB4 1 &pcfg_pull_none>, + <2 RK_PB5 1 &pcfg_pull_none>, + <2 RK_PB6 1 &pcfg_pull_none>, + <2 RK_PB7 1 &pcfg_pull_none>, + <2 RK_PC0 1 &pcfg_pull_none>, + <2 RK_PC2 1 &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-r88-dcdc.dts b/arch/arm64/boot/dts/rockchip/rk3368-r88-dcdc.dts new file mode 100755 index 000000000000..4a755f1207c1 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-r88-dcdc.dts @@ -0,0 +1,676 @@ +/* + * Copyright (c) 2015 Heiko Stuebner + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" +#include + +/ { + model = "Rockchip R88"; + compatible = "rockchip,r88", "rockchip,rk3368"; + + hdmi_sound: hdmi-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,hdmi"; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + keys: gpio-keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pwr_key>; + + button@0 { + gpio-key,wakeup = <1>; + gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; + label = "GPIO Power"; + linux,code = <116>; + }; + }; + + leds: gpio-leds { + compatible = "gpio-leds"; + + work { + gpios = <&gpio3 29 GPIO_ACTIVE_HIGH>; + label = "r88:green:led"; + pinctrl-names = "default"; + pinctrl-0 = <&led_ctl>; + }; + }; + + ir: ir-receiver { + compatible = "gpio-ir-receiver"; + gpios = <&gpio3 30 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&ir_int>; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_host: vcc-host { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + regulators { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <0>; + + vccio_1v8_reg: regulator@0 { + compatible = "regulator-fixed"; + regulator-name = "vccio_1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + + vccio_3v3_reg: regulator@1 { + compatible = "regulator-fixed"; + regulator-name = "vccio_3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + }; + + vdd_gpu: vdd-arm-regulator { + compatible = "pwm-regulator"; + rockchip,pwm_id = <1>; + rockchip,pwm_voltage = <1100000>; + pwms = <&pwm1 0 25000 1>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1400000>; + regulator-always-on; + regulator-boot-on; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "rtl8189es"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + //clocks = <&rk808 1>; + //clock-names = "ext_clock"; + /* wifi-bt-power-toggle; */ + uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_gpios>; + /* BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; */ + BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&i2s_8ch { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&emmc { + status = "okay"; + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; +}; + +&sdmmc { + status = "disabled"; + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rmii"; + clock_in_out = "output"; + snps,reset-gpio = <&gpio3 12 0>; + snps,reset-active-low; + snps,reset-delays-us = <0 50000 50000>; + //assigned-clocks = <&cru SCLK_RMII_SRC>; + //assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rmii_pins>; + tx_delay = <0x30>; + rx_delay = <0x10>; + status = "ok"; +}; + +&pwm1 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm1_pin_pull_down>; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <100000>; + + vdd_cpu: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + status = "okay"; + vin-supply = <&vcc_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-names = "default"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + +/* xz3215: xz3215@40 { + compatible = "xz3216"; + reg = <0x40>; + status = "disabled"; + pinctrl-names = "default"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; + regulators { + #address-cells = <1>; + #size-cells = <0>; + vdd_cpu: regulator@0 { + reg = <0>; + regulator-compatible = "xz_dcdc1"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <603000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <1000>; + regulator-always-on; + regulator-boot-on; + fcs,suspend-voltage-selector = <1>; + //regulator-initial-mode = <0x1>; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +*/ + + hym8563: hym8563@51 { + compatible = "haoyu,hym8563"; + reg = <0x51>; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + /* rtc_int is not connected */ + }; +}; + +&io_domains { + status = "ok"; + + dvp-supply = <&vccio_1v8_reg>; /* DVPIO_VDD */ + /*flash0-supply = <&vcc18_flash>;*/ /* FLASH0_VDD (emmc) */ + sdcard-supply = <&vccio_3v3_reg>; /* SDMMC0_VDD (sdmmc) */ + + audio-supply = <&vccio_3v3_reg>; /* APIO3_VDD */ + gpio30-supply = <&vccio_3v3_reg>; /* APIO1_VDD */ + gpio1830-supply = <&vccio_3v3_reg>; /* APIO4_VDD (gpujtag) */ + wifi-supply = <&vccio_3v3_reg>; /* APIO2_VDD (sdio0) */ +}; + +&sdio0 { + bus-width = <4>; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + max-frequency = <100000000>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + no-sd; + no-mmc; + status = "okay"; +}; + +&pinctrl { + pcfg_pull_none_drv_8ma: pcfg-pull-none-drv-8ma { + bias-disable; + drive-strength = <8>; + }; + + pcfg_pull_up_drv_8ma: pcfg-pull-up-drv-8ma { + bias-pull-up; + drive-strength = <8>; + }; + + pmic { + pmic_int: pmic-int { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + vsel1_gpio:vsel1_gpio{ + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + emmc { + emmc_bus8: emmc-bus8 { + rockchip,pins = <1 RK_PC2 2 &pcfg_pull_up_drv_8ma>, + <1 RK_PC3 2 &pcfg_pull_up_drv_8ma>, + <1 RK_PC4 2 &pcfg_pull_up_drv_8ma>, + <1 RK_PC5 2 &pcfg_pull_up_drv_8ma>, + <1 RK_PC6 2 &pcfg_pull_up_drv_8ma>, + <1 RK_PC7 2 &pcfg_pull_up_drv_8ma>, + <1 RK_PD0 2 &pcfg_pull_up_drv_8ma>, + <1 RK_PD1 2 &pcfg_pull_up_drv_8ma>; + }; + + emmc-clk { + rockchip,pins = <2 RK_PA4 2 &pcfg_pull_none_drv_8ma>; + }; + + emmc-cmd { + rockchip,pins = <1 RK_PD2 2 &pcfg_pull_up_drv_8ma>; + }; + + emmc_reset: emmc-reset { + rockchip,pins = <2 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + ir { + ir_int: ir-int { + rockchip,pins = <3 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + keys { + pwr_key: pwr-key { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + leds { + stby_pwren: stby-pwren { + rockchip,pins = <0 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + led_ctl: led-ctl { + rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + + pmu-supply = <&vccio_3v3_reg>; + vop-supply = <&vccio_3v3_reg>; +}; + +&saradc { + vref-supply = <&vccio_1v8_reg>; + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_otg { + dr_mode = "device"; + status = "okay"; +}; + +&u2phy { + status = "okay"; +}; + +&u2phy_host { + status = "okay"; +}; + +&wdt { + status = "okay"; +}; + +&route_hdmi { + status = "okay"; +}; + +&hdmi { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&mailbox { + status = "okay"; +}; + +&mailbox_scpi { + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&vdd_cpu>; + status = "okay"; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b3 { + cpu-supply = <&vdd_cpu>; +}; + +&gpu { + logic-supply = <&vdd_gpu>; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF + | RKPM_SLP_PMU_PLLS_PWRDN + | RKPM_SLP_PMU_PMUALIVE_32K + | RKPM_SLP_SFT_PLLS_DEEP + | RKPM_SLP_PMU_DIS_OSC + | RKPM_SLP_SFT_PD_NBSCUS + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_USB_WKUP_EN + | RKPM_CLUSTER_L_WKUP_EN + ) + >; +}; + +&pwm3 { + status = "okay"; + + interrupts = ; + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_VOLUMEDOWN>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-sheep-lvds.dts b/arch/arm64/boot/dts/rockchip/rk3368-sheep-lvds.dts new file mode 100755 index 000000000000..b7b11f895cab --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-sheep-lvds.dts @@ -0,0 +1,662 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" + +/ { + model = "Rockchip Sheep board"; + compatible = "rockchip,sheep", "rockchip,rk3368"; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rt5640-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rt5640>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 PWM_POLARITY_INVERTED>; + brightness-levels = < + 135 135 136 136 137 137 138 138 + 139 139 140 140 141 141 142 142 + 143 143 143 144 144 145 145 146 + 146 147 147 148 148 149 149 150 + 150 151 151 151 152 152 153 153 + 154 154 155 155 156 156 157 157 + 158 158 159 159 159 160 160 161 + 161 162 162 163 163 164 164 165 + 165 166 166 167 167 167 168 168 + 169 169 170 170 171 171 172 172 + 173 173 174 174 175 175 175 176 + 176 177 177 178 178 179 179 180 + 180 181 181 182 182 183 183 183 + 184 184 185 185 186 186 187 187 + 188 188 189 189 190 190 191 191 + 191 192 192 193 193 194 194 195 + 195 196 196 197 197 198 198 199 + 199 199 200 200 201 201 202 202 + 203 203 204 204 205 205 206 206 + 207 207 207 208 208 209 209 210 + 210 211 211 212 212 213 213 214 + 214 215 215 215 216 216 217 217 + 218 218 219 219 220 220 221 221 + 222 222 223 223 223 224 224 225 + 225 226 226 227 227 228 228 229 + 229 230 230 231 231 231 232 232 + 233 233 234 234 235 235 236 236 + 237 237 238 238 239 239 239 240 + 240 241 241 242 242 243 243 244 + 244 245 245 246 246 247 247 247 + 248 248 249 249 250 250 251 251 + 252 252 253 253 254 254 255 255>; + default-brightness-level = <200>; + enable-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + }; + + rk_key: rockchip-key { + compatible = "rockchip,key"; + status = "okay"; + + io-channels = <&saradc 1>; + + vol-up-key { + linux,code = <115>; + label = "volume up"; + rockchip,adc_value = <1>; + }; + + vol-down-key { + linux,code = <114>; + label = "volume down"; + rockchip,adc_value = <170>; + }; + + power-key { + gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; + linux,code = <116>; + label = "power"; + gpio-key,wakeup; + }; + + menu-key { + linux,code = <59>; + label = "menu"; + rockchip,adc_value = <355>; + }; + + home-key { + linux,code = <102>; + label = "home"; + rockchip,adc_value = <746>; + }; + + back-key { + linux,code = <158>; + label = "back"; + rockchip,adc_value = <560>; + }; + + camera-key { + linux,code = <212>; + label = "camera"; + rockchip,adc_value = <450>; + }; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_host: vcc-host { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + + panel { + compatible = "simple-panel"; + backlight = <&backlight>; + enable-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; + bus-format = ; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <54000000>; + hactive = <1024>; + vactive = <600>; + hback-porch = <134>; + hfront-porch = <134>; + vback-porch = <10>; + vfront-porch = <10>; + hsync-len = <134>; + vsync-len = <10>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in_lvds: endpoint { + remote-endpoint = <&lvds_out_panel>; + }; + }; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&emmc { + status = "okay"; + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; +}; + +&sdmmc { + status = "okay"; + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; +}; + +&i2c0 { + status = "okay"; + + syr827: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + status = "okay"; + + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_arm"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + rk818: pmic@1c { + compatible = "rockchip,rk818"; + reg = <0x1c>; + status = "okay"; + + clock-output-names = "rk818-clkout1", "wifibt_32kin"; + interrupt-parent = <&gpio0>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc_sys>; + vcc9-supply = <&vcc_io>; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-name = "vdd_logic"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1450000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-name = "vdd_gpu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1250000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_io"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca_codec: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcca_codec"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_tp: LDO_REG2 { + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_10: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd_10"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc18_lcd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc18_lcd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_pmu: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vccio_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd10_lcd: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd10_lcd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_18: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_18"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_wl: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_wl"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vccio_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: SWITCH_REG { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + boost_otg: DCDC_BOOST { + regulator-name = "boost_otg"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + }; + }; +}; + +&cpu_l0 { + cpu-supply = <&syr827>; +}; + +&cpu_l1 { + cpu-supply = <&syr827>; +}; + +&cpu_l2 { + cpu-supply = <&syr827>; +}; + +&cpu_l3 { + cpu-supply = <&syr827>; +}; + +&cpu_b0 { + cpu-supply = <&syr827>; +}; + +&cpu_b1 { + cpu-supply = <&syr827>; +}; + +&cpu_b2 { + cpu-supply = <&syr827>; +}; + +&cpu_b3 { + cpu-supply = <&syr827>; +}; + +&gpu { + logic-supply = <&vdd_logic>; +}; + +&i2c1 { + status = "okay"; + + rt5640: rt5640@1c { + compatible = "realtek,rt5640"; + reg = <0x1c>; + #sound-dai-cells = <0>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + realtek,in1-differential; + status = "okay"; + }; +}; + +&i2c2 { + status = "okay"; + + gt9xx: gt9xx@14 { + compatible = "goodix,gt9xx"; + reg = <0x14>; + touch-gpio = <&gpio0 12 IRQ_TYPE_LEVEL_LOW>; + reset-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; + max-x = <1200>; + max-y = <1900>; + tp-size = <911>; + tp-supply = <&vcc_tp>; + status = "okay"; + }; +}; + +&i2s_8ch { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + dvp-supply = <&vcc_18>; + audio-supply = <&vcc_io>; + gpio30-supply = <&vcc_io>; + gpio1830-supply = <&vcc_io>; + sdcard-supply = <&vccio_sd>; + wifi-supply = <&vccio_wl>; +}; + +&pmu_io_domains { + status = "okay"; + pmu-supply = <&vcc_io>; + vop-supply = <&vcc_io>; +}; + +&pwm0 { + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + phy-supply = <&vcc_host>; + status = "okay"; + }; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&mailbox { + status = "okay"; +}; + +&mailbox_scpi { + status = "okay"; +}; + +&lvds { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + lvds_out_panel: endpoint { + remote-endpoint = <&panel_in_lvds>; + }; + }; + }; + +}; + +&route_lvds { + status = "okay"; +}; + +&saradc { + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&syr827>; + status = "okay"; +}; + +&hdmi { + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-sheep.dts b/arch/arm64/boot/dts/rockchip/rk3368-sheep.dts new file mode 100755 index 000000000000..b9b6b24ff37b --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-sheep.dts @@ -0,0 +1,746 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" + +/ { + model = "Rockchip Sheep board"; + compatible = "rockchip,sheep", "rockchip,rk3368"; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rt5640-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rt5640>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 PWM_POLARITY_INVERTED>; + brightness-levels = < + 135 135 136 136 137 137 138 138 + 139 139 140 140 141 141 142 142 + 143 143 143 144 144 145 145 146 + 146 147 147 148 148 149 149 150 + 150 151 151 151 152 152 153 153 + 154 154 155 155 156 156 157 157 + 158 158 159 159 159 160 160 161 + 161 162 162 163 163 164 164 165 + 165 166 166 167 167 167 168 168 + 169 169 170 170 171 171 172 172 + 173 173 174 174 175 175 175 176 + 176 177 177 178 178 179 179 180 + 180 181 181 182 182 183 183 183 + 184 184 185 185 186 186 187 187 + 188 188 189 189 190 190 191 191 + 191 192 192 193 193 194 194 195 + 195 196 196 197 197 198 198 199 + 199 199 200 200 201 201 202 202 + 203 203 204 204 205 205 206 206 + 207 207 207 208 208 209 209 210 + 210 211 211 212 212 213 213 214 + 214 215 215 215 216 216 217 217 + 218 218 219 219 220 220 221 221 + 222 222 223 223 223 224 224 225 + 225 226 226 227 227 228 228 229 + 229 230 230 231 231 231 232 232 + 233 233 234 234 235 235 236 236 + 237 237 238 238 239 239 239 240 + 240 241 241 242 242 243 243 244 + 244 245 245 246 246 247 247 247 + 248 248 249 249 250 250 251 251 + 252 252 253 253 254 254 255 255>; + default-brightness-level = <200>; + enable-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + }; + + rk_key: rockchip-key { + compatible = "rockchip,key"; + status = "okay"; + + io-channels = <&saradc 1>; + + vol-up-key { + linux,code = <115>; + label = "volume up"; + rockchip,adc_value = <1>; + }; + + vol-down-key { + linux,code = <114>; + label = "volume down"; + rockchip,adc_value = <170>; + }; + + power-key { + gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; + linux,code = <116>; + label = "power"; + gpio-key,wakeup; + }; + + menu-key { + linux,code = <59>; + label = "menu"; + rockchip,adc_value = <355>; + }; + + home-key { + linux,code = <102>; + label = "home"; + rockchip,adc_value = <746>; + }; + + back-key { + linux,code = <158>; + label = "back"; + rockchip,adc_value = <560>; + }; + + camera-key { + linux,code = <212>; + label = "camera"; + rockchip,adc_value = <450>; + }; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_host: vcc-host { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + }; + + vcc_otg_vbus: otg-vbus-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PD1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&otg_vbus_drv>; + regulator-name = "vcc_otg_vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&emmc { + status = "okay"; + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; +}; + +&sdmmc { + status = "okay"; + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; +}; + +&i2c0 { + status = "okay"; + + syr827: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + status = "okay"; + + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_arm"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + pinctrl-0 = <&vsel_gpio>; + vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + rk818: pmic@1c { + compatible = "rockchip,rk818"; + reg = <0x1c>; + status = "okay"; + + clock-output-names = "rk818-clkout1", "wifibt_32kin"; + interrupt-parent = <&gpio0>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc_sys>; + vcc9-supply = <&vcc_io>; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-name = "vdd_logic"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1450000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-name = "vdd_gpu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1250000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_io"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca_codec: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcca_codec"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_tp: LDO_REG2 { + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_10: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd_10"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc18_lcd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc18_lcd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_pmu: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vccio_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd10_lcd: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd10_lcd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_18: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_18"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_wl: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_wl"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vccio_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: SWITCH_REG { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + boost_otg: DCDC_BOOST { + regulator-name = "boost_otg"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + }; + + battery { + compatible = "rk818-battery"; + pinctrl-names = "default"; + pinctrl-0 = <&dc_irq_gpio>; + ocv_table = < + 3400 3650 3693 3707 3731 3749 3760 + 3770 3782 3796 3812 3829 3852 3882 + 3915 3951 3981 4047 4086 4132 4182>; + design_capacity = <8650>; + design_qmax = <8800>; + bat_res = <85>; + max_input_current = <2000>; + max_chrg_current = <1800>; + max_chrg_voltage = <4200>; + sleep_enter_current = <600>; + sleep_exit_current = <600>; + power_off_thresd = <3400>; + zero_algorithm_vol = <3850>; + fb_temperature = <115>; + sample_res = <10>; + max_soc_offset = <60>; + energy_mode = <0>; + monitor_sec = <5>; + virtual_power = <0>; + power_dc2otg = <1>; + support_usb_adp = <1>; + support_dc_adp = <1>; + dc_det_gpio = <&gpio0 17 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&cpu_l0 { + cpu-supply = <&syr827>; +}; + +&cpu_l1 { + cpu-supply = <&syr827>; +}; + +&cpu_l2 { + cpu-supply = <&syr827>; +}; + +&cpu_l3 { + cpu-supply = <&syr827>; +}; + +&cpu_b0 { + cpu-supply = <&syr827>; +}; + +&cpu_b1 { + cpu-supply = <&syr827>; +}; + +&cpu_b2 { + cpu-supply = <&syr827>; +}; + +&cpu_b3 { + cpu-supply = <&syr827>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&gpu { + logic-supply = <&vdd_logic>; +}; + +&rockchip_suspend { + status = "okay"; +}; + +&i2c1 { + status = "okay"; + + rt5640: rt5640@1c { + compatible = "realtek,rt5640"; + reg = <0x1c>; + #sound-dai-cells = <0>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + realtek,in1-differential; + status = "okay"; + }; +}; + +&i2c2 { + status = "okay"; + + gt9xx: gt9xx@14 { + compatible = "goodix,gt9xx"; + reg = <0x14>; + touch-gpio = <&gpio0 12 IRQ_TYPE_LEVEL_LOW>; + reset-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; + max-x = <1200>; + max-y = <1900>; + tp-size = <911>; + tp-supply = <&vcc_tp>; + status = "okay"; + }; +}; + +&i2s_8ch { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + dvp-supply = <&vcc_18>; + audio-supply = <&vcc_io>; + gpio30-supply = <&vcc_io>; + gpio1830-supply = <&vcc_io>; + sdcard-supply = <&vccio_sd>; + wifi-supply = <&vccio_wl>; +}; + +&pmu_io_domains { + status = "okay"; + + pmu-supply = <&vcc_io>; + vop-supply = <&vcc_io>; +}; + +&pwm0 { + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + phy-supply = <&vcc_host>; + status = "okay"; + }; + + u2phy_otg: otg-port { + vbus-supply = <&vcc_otg_vbus>; + status = "okay"; + }; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&mailbox { + status = "okay"; +}; + +&mailbox_scpi { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <120>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <150000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <80>; + hfront-porch = <81>; + vback-porch = <21>; + vfront-porch = <21>; + hsync-len = <10>; + vsync-len = <3>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&route_dsi { + status = "okay"; +}; + +&saradc { + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&syr827>; + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel_gpio: vsel-gpio { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + dc_det { + dc_irq_gpio: dc-irq-gpio { + rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + otg_vbus_drv: otg-bus-drv { + rockchip,pins = <0 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-sziauto-rk618.dts b/arch/arm64/boot/dts/rockchip/rk3368-sziauto-rk618.dts new file mode 100755 index 000000000000..eff7c2fed990 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-sziauto-rk618.dts @@ -0,0 +1,808 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" +#include +#include + +/ { + model = "Rockchip rk3368 Sziauto board"; + compatible = "rockchip,sziauto", "rockchip,rk3368"; + + panel { + compatible = "simple-panel"; + backlight = <&backlight>; + enable-gpios = <&gpio3 13 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + disable-delay-ms = <20>; + unprepare-delay-ms = <20>; + bus-format = ; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <136000000>; + hactive = <1920>; + vactive = <1080>; + hback-porch = <60>; + hfront-porch = <60>; + hsync-len = <40>; + vback-porch = <4>; + vfront-porch = <4>; + vsync-len = <3>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + port { + panel_in_lvds: endpoint { + remote-endpoint = <&lvds_out_panel>; + }; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk818 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 50000 0>; + brightness-levels = < + 32 32 34 34 36 36 38 38 40 40 + 42 42 44 44 46 46 48 48 50 50 + 52 52 54 54 56 56 58 58 60 60 + 62 62 64 64 66 66 68 68 70 70 + 72 72 74 74 76 76 78 78 80 80 + 82 82 84 84 86 86 88 88 90 90 + 92 92 94 94 96 96 98 98 100 100 + 102 102 104 104 106 106 108 108 110 110 + 112 112 114 114 116 116 118 118 120 120 + 122 122 124 124 126 126 128 128 130 130 + 132 132 134 134 136 136 138 138 140 140 + 142 142 144 144 146 146 148 148 150 150 + 152 152 154 154 156 156 158 158 160 160 + 162 162 164 164 166 166 168 168 170 170 + 172 172 174 174 176 176 178 178 180 180 + 182 182 184 184 186 186 188 188 190 190 + 192 192 194 194 196 196 198 198 200 200 + 202 202 204 204 206 206 208 208 210 210 + 212 212 214 214 216 216 218 218 220 220 + 222 222 224 224 225 225 226 226 227 227 + 228 228 229 229 230 230 231 231 232 232 + 233 233 234 234 235 235 236 236 237 237 + 238 238 239 239 240 240 241 241 242 242 + 243 243 244 244 245 245 246 246 247 247 + 248 248 249 249 250 250 251 251 252 252 + 253 253 254 254 255 255>; + default-brightness-level = <120>; + enable-gpios = <&gpio3 28 GPIO_ACTIVE_HIGH>; + }; + + rockchip-key { + compatible = "rockchip,key"; + io-channels = <&saradc 1>; + status = "okay"; + + vol-up-key { + linux,code = <115>; + label = "volume up"; + rockchip,adc_value = <1>; + }; + + vol-down-key { + linux,code = <114>; + label = "volume down"; + rockchip,adc_value = <170>; + }; + + power-key { + gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; + linux,code = <116>; + label = "power"; + gpio-key,wakeup; + }; + + menu-key { + linux,code = <59>; + label = "menu"; + rockchip,adc_value = <355>; + }; + + home-key { + linux,code = <102>; + label = "home"; + rockchip,adc_value = <746>; + }; + + back-key { + linux,code = <158>; + label = "back"; + rockchip,adc_value = <560>; + }; + + camera-key { + linux,code = <212>; + label = "camera"; + rockchip,adc_value = <450>; + }; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_host: vcc-host { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&i2c5 { + status = "okay"; + + rk618@50 { + compatible = "rockchip,rk618"; + reg = <0x50>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "clkin"; + assigned-clocks = <&cru SCLK_I2S_8CH_OUT>; + assigned-clock-rates = <11289600>; + reset-gpios = <&gpio3 14 GPIO_ACTIVE_LOW>; + status = "okay"; + + clock: cru { + compatible = "rockchip,rk618-cru"; + clocks = <&cru SCLK_I2S_8CH_OUT>, <&cru DCLK_VOP>; + clock-names = "clkin", "lcdc0_dclkp"; + assigned-clocks = <&clock SCALER_PLLIN_CLK>, <&clock VIF_PLLIN_CLK>, + <&clock SCALER_CLK>, <&clock VIF0_PRE_CLK>, + <&clock CODEC_CLK>, <&clock DITHER_CLK>; + assigned-clock-parents = <&cru SCLK_I2S_8CH_OUT>, <&clock LCDC0_CLK>, + <&clock SCALER_PLL_CLK>, <&clock VIF_PLL_CLK>, + <&cru SCLK_I2S_8CH_OUT>, <&clock VIF0_CLK>; + #clock-cells = <1>; + status = "okay"; + }; + + hdmi { + compatible = "rockchip,rk618-hdmi"; + clocks = <&clock HDMI_CLK>; + clock-names = "hdmi"; + assigned-clocks = <&clock HDMI_CLK>; + assigned-clock-parents = <&clock VIF0_CLK>; + interrupt-parent = <&gpio3>; + interrupts = <23 IRQ_TYPE_LEVEL_HIGH>; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hdmi_in_vif: endpoint { + remote-endpoint = <&vif_out_hdmi>; + }; + }; + + port@1 { + reg = <1>; + + hdmi_out_scaler: endpoint { + remote-endpoint = <&scaler_in_hdmi>; + }; + }; + }; + }; + + lvds { + compatible = "rockchip,rk618-lvds"; + clocks = <&clock LVDS_CLK>; + clock-names = "lvds"; + dual-channel; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + lvds_in_scaler: endpoint { + remote-endpoint = <&scaler_out_lvds>; + }; + }; + + port@1 { + reg = <1>; + + lvds_out_panel: endpoint { + remote-endpoint = <&panel_in_lvds>; + }; + }; + }; + }; + + scaler { + compatible = "rockchip,rk618-scaler"; + clocks = <&clock SCALER_CLK>, <&clock VIF0_CLK>, + <&clock DITHER_CLK>; + clock-names = "scaler", "vif", "dither"; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + scaler_in_hdmi: endpoint { + remote-endpoint = <&hdmi_out_scaler>; + }; + }; + + port@1 { + reg = <1>; + + scaler_out_lvds: endpoint { + remote-endpoint = <&lvds_in_scaler>; + }; + }; + }; + }; + + vif { + compatible = "rockchip,rk618-vif"; + clocks = <&clock VIF0_CLK>, <&clock VIF0_PRE_CLK>; + clock-names = "vif", "vif_pre"; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + vif_in_rgb: endpoint { + remote-endpoint = <&rgb_out_vif>; + }; + }; + + port@1 { + reg = <1>; + + vif_out_hdmi: endpoint { + remote-endpoint = <&hdmi_in_vif>; + }; + }; + }; + }; + }; +}; + +&rgb { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + rgb_out_vif: endpoint { + remote-endpoint = <&vif_in_rgb>; + }; + }; + }; +}; + +&route_rgb { + status = "okay"; +}; + +&emmc { + status = "okay"; + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; +}; + +&sdmmc { + status = "disabled"; + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; +}; + +&sdio0 { + clock-frequency = <50000000>; + clock-freq-min-max = <200000 50000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + syr827: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + fcs,suspend-voltage-selector = <1>; + + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_arm"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + rk818: pmic@1c { + compatible = "rockchip,rk818"; + reg = <0x1c>; + clock-output-names = "rk818-clkout1", "wifibt_32kin"; + interrupt-parent = <&gpio0>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc_sys>; + vcc9-supply = <&vcc_io>; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-name = "vdd_logic"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1450000>; + regulator-ramp-delay = <6001>; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-name = "vdd_gpu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1250000>; + regulator-ramp-delay = <6001>; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_io"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca_codec: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcca_codec"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_tp: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_tp"; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_10: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd_10"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc18_lcd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc18_lcd"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_pmu: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vccio_pmu"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd10_lcd: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd10_lcd"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_18: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_18"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_wl: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_wl"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vccio_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: SWITCH_REG { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_sd"; + + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + boost_otg: DCDC_BOOST { + regulator-name = "boost_otg"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + }; + }; +}; + +&cpu_l0 { + cpu-supply = <&syr827>; +}; + +&cpu_l1 { + cpu-supply = <&syr827>; +}; + +&cpu_l2 { + cpu-supply = <&syr827>; +}; + +&cpu_l3 { + cpu-supply = <&syr827>; +}; + +&cpu_b0 { + cpu-supply = <&syr827>; +}; + +&cpu_b1 { + cpu-supply = <&syr827>; +}; + +&cpu_b2 { + cpu-supply = <&syr827>; +}; + +&cpu_b3 { + cpu-supply = <&syr827>; +}; + +&gpu { + logic-supply = <&vdd_logic>; +}; + +&io_domains { + dvp-supply = <&vcc_18>; + audio-supply = <&vcc_io>; + gpio30-supply = <&vcc_io>; + gpio1830-supply = <&vcc_io>; + sdcard-supply = <&vccio_sd>; + wifi-supply = <&vccio_wl>; + status = "okay"; +}; + +&pmu_io_domains { + pmu-supply = <&vcc_io>; + vop-supply = <&vcc_io>; + status = "okay"; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm1 { + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + phy-supply = <&vcc_host>; + status = "okay"; + }; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&mailbox { + status = "okay"; +}; + +&mailbox_scpi { + status = "okay"; +}; + +&saradc { + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&syr827>; + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-tablet.dts b/arch/arm64/boot/dts/rockchip/rk3368-tablet.dts new file mode 100755 index 000000000000..d7dc6d67e242 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-tablet.dts @@ -0,0 +1,1070 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" +/ { + model = "Rockchip rk3368 tablet board"; + compatible = "rockchip,tablet", "rockchip,rk3368"; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1024000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1000>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <170000>; + }; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PD0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 1>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 16 16 16 16 16 16 16 + 17 17 17 17 17 17 17 17 + 18 18 18 18 18 18 18 18 + 19 19 19 19 19 19 19 19 + 20 20 20 20 20 20 20 20 + 21 21 21 21 21 21 21 21 + 22 22 22 22 22 22 22 22 + 23 23 23 23 23 23 23 23 + 24 24 24 24 24 24 24 24 + 25 25 25 25 25 25 25 25 + 26 26 26 26 26 26 26 26 + 27 27 27 27 27 27 27 27 + 28 28 28 28 28 28 28 28 + 27 27 27 27 27 27 27 27 + 30 30 30 30 30 30 30 30 + 31 31 31 31 31 31 31 31 + 32 32 32 32 32 32 32 32 + 33 33 33 33 33 33 33 33 + 34 34 34 34 34 34 34 34 + 35 35 35 35 35 35 35 35 + 36 36 36 36 36 36 36 36 + 37 37 37 37 37 37 37 37 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38>; + default-brightness-level = <200>; + enable-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3500>; + rockchip,screen-on-voltage = <3600>; + status = "okay"; + }; + + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init kpti=0"; + }; + + es8316-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,es8316-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch>; + system-clock-frequency = <11289600>; + }; + simple-audio-card,codec { + sound-dai = <&es8316>; + system-clock-frequency = <11289600>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk818 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + gpio_keys: gpio-keys { + compatible = "gpio-keys"; + autorepeat; + + power { + debounce-interval = <100>; + gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; + label = "GPIO Key Power"; + linux,code = ; + wakeup-source; + }; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + /* wifi_chip_type - wifi chip define + * ap6210, ap6330, ap6335 + * rtl8188eu, rtl8723bs, rtl8723bu + * esp8089 + */ + wifi_chip_type = "ap6255"; + WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk818 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_rts_gpio>; + + //BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; + BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; + + status = "okay"; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_host: vcc-host { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + +}; + +&cif_clkout { + /* cif_clkout */ + rockchip,pins = <1 RK_PB3 1 &pcfg_pull_none_4ma>; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b3 { + cpu-supply = <&vdd_cpu>; +}; + +&gpu { + logic-supply = <&vdd_logic>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_logic>; + devfreq-events = <&dfi>; + upthreshold = <60>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 600000 + SYS_STATUS_REBOOT 600000 + SYS_STATUS_SUSPEND 240000 + SYS_STATUS_VIDEO_1080P 396000 + SYS_STATUS_VIDEO_4K 600000 + SYS_STATUS_PERFORMANCE 600000 + SYS_STATUS_BOOST 396000 + SYS_STATUS_DUALVIEW 600000 + SYS_STATUS_ISP 528000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 582 240000 + 583 99999 396000 + >; + auto-min-freq = <240000>; + auto-freq-en = <1>; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; + back-gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio3 RK_PC4 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <8>; + enable-delay-ms = <3>; + reset-delay-ms = <50>; + init-delay-ms = <20>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + width-mm = <153>; + height-mm = <85>; + panel-init-sequence = [ + 05 1e 01 01 + 15 00 02 80 47 + 15 00 02 81 40 + 15 00 02 82 04 + 15 00 02 83 77 + 15 00 02 84 0f + 15 00 02 85 70 + 15 78 02 86 70 + ]; + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <49500000>; + hactive = <1024>; + vactive = <600>; + hback-porch = <120>; + hfront-porch = <80>; + vback-porch = <14>; + vfront-porch = <14>; + hsync-len = <40>; + vsync-len = <4>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@10 { + compatible = "tcs,tcs452x"; + reg = <0x10>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel_gpio>; + vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk818: pmic@1c { + compatible = "rockchip,rk818"; + status = "okay"; + reg = <0x1c>; + clock-output-names = "rk818-clkout1", "wifibt_32kin"; + interrupt-parent = <&gpio0>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + extcon = <&u2phy>; + #clock-cells = <1>; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc_sys>; + vcc9-supply = <&vcc_io>; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-name = "vdd_logic"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1450000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-name = "vdd_gpu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1250000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_io"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca_codec: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcca_codec"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_tp: LDO_REG2 { + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_10: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd_10"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc18_lcd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc18_lcd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_pmu: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vccio_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd10_lcd: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd10_lcd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_18: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_18"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_wl: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vccio_wl"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: SWITCH_REG { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + boost_otg: DCDC_BOOST { + regulator-name = "boost_otg"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk818-battery"; + pinctrl-names = "default"; + pinctrl-0 = <&dc_irq_gpio>; + ocv_table = < + 3400 3652 3680 3707 3730 3747 3764 + 3772 3781 3792 3807 3828 3861 3899 + 3929 3958 3987 4038 4079 4127 4186>; + design_capacity = <7536>; + design_qmax = <8290>; + bat_res = <100>; + max_input_current = <1750>; + max_chrg_current = <2000>; + max_chrg_voltage = <4200>; + sleep_enter_current = <600>; + sleep_exit_current = <600>; + power_off_thresd = <3400>; + zero_algorithm_vol = <3850>; + fb_temperature = <115>; + sample_res = <20>; + max_soc_offset = <60>; + energy_mode = <0>; + monitor_sec = <5>; + virtual_power = <0>; + power_dc2otg = <0>; + support_usb_adp = <1>; + support_dc_adp = <1>; + dc_det_gpio = <&gpio0 17 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&i2c1 { + status = "okay"; + + es8316: es8316@10 { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "everest,es8316"; + reg = <0x10>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + spk-con-gpio = <&gpio0 27 GPIO_ACTIVE_HIGH>; + hp-det-gpio = <&gpio0 23 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + extcon = <&rk_headset>; + }; +}; + +&i2c2 { + status = "okay"; + + ts@5a { + compatible = "cst2xxse"; + reg = <0x5a>; + irq-gpio = <&gpio0 RK_PB4 IRQ_TYPE_LEVEL_LOW>; + //touch-gpio = <&gpio1 GPIO_B0 IRQ_TYPE_LEVEL_LOW>; /* TP_INT == GPIO1_B0 */ + //reset-gpio = <&gpio0 GPIO_D1 GPIO_ACTIVE_LOW>; /* TP_RST == GPIO0_D1 */ + //power-gpio = <&gpio0 GPIO_C5 GPIO_ACTIVE_LOW>; + //max-x = <800>; + //max-y = <480>; + status = "okay"; + }; + +}; + +&i2c3 { + status = "okay"; + + gc0312: gc0312@21 { + status = "okay"; + compatible = "galaxycore,gc0312"; + reg = <0x21>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + pwdn-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc0312_out: endpoint { + remote-endpoint = <&dvp_in_fcam>; + }; + }; + }; + + gc2145: gc2145@3c { + status = "okay"; + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_out: endpoint { + remote-endpoint = <&dvp_in_bcam>; + }; + }; + }; + + ov8858: ov8858@36 { + status = "disabled"; + compatible = "ovti,ov8858"; + reg = <0x36>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan-9569A2"; + power-gpio = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + + port { + ov8858_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c4 { + status = "okay"; + + mpu6500@68 { + status = "disabled"; + compatible = "invensense,mpu6500"; + pinctrl-names = "default"; + pinctrl-0 = <&mpu6500_irq_gpio>; + reg = <0x68>; + irq-gpio = <&gpio3 14 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <1 0 0 0 1 0 0 0 1>; + orientation-x= <1>; + orientation-y= <0>; + orientation-z= <1>; + support-hw-poweroff = <1>; + mpu-debug = <1>; + }; + + sensor@4c { + status = "okay"; + compatible = "gs_mc3230"; + reg = <0x4c>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <9>; + reprobe_en = <1>; + irq-gpio = <&gpio3 RK_PB6 IRQ_TYPE_LEVEL_LOW>; + }; + + sensor@18 { + status = "okay"; + compatible = "gs_sc7a30"; + reg = <0x18>; + type = ; + irq-gpio = <&gpio3 RK_PB6 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <6>; + reprobe_en = <1>; + }; + + sensor@10 { + status = "okay"; + compatible = "light_cm3218"; + pinctrl-names = "default"; + pinctrl-0 = <&cm3218_irq_gpio>; + reg = <0x10>; + type = ; + irq-gpio = <&gpio3 15 IRQ_TYPE_EDGE_FALLING>; + irq_enable = <1>; + poll_delay_ms = <30>; + }; +}; + +&i2s_8ch { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + dvp-supply = <&vcc_18>; + audio-supply = <&vcc_io>; + gpio30-supply = <&vcc_io>; + gpio1830-supply = <&vcc_io>; + sdcard-supply = <&vccio_sd>; + wifi-supply = <&vccio_wl>; +}; + +&isp_dvp_d2d9 { + rockchip,pins = + /* cif_data4 ... cif_data9 */ + <1 RK_PA2 1 &pcfg_pull_down>, + <1 RK_PA3 1 &pcfg_pull_down>, + <1 RK_PA4 1 &pcfg_pull_down>, + <1 RK_PA5 1 &pcfg_pull_down>, + <1 RK_PA6 1 &pcfg_pull_down>, + <1 RK_PA7 1 &pcfg_pull_down>, + /* cif_sync, cif_href */ + <1 RK_PB0 1 &pcfg_pull_down>, + <1 RK_PB1 1 &pcfg_pull_down>, + /* cif_clkin */ + <1 RK_PB2 1 &pcfg_pull_down>; +}; + +&isp_dvp_d10d11 { + rockchip,pins = + /* cif_data10, cif_data11 */ + <1 RK_PB6 1 &pcfg_pull_down>, + <1 RK_PB7 1 &pcfg_pull_down>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov8858_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + + pmu-supply = <&vccio_pmu>; + vop-supply = <&vccio_pmu>; +}; + +&pwm0 { + status = "okay"; +}; + +&pinctrl { + camera { + camera_pwr: camera-pwr { + rockchip,pins = <0 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel_gpio: vsel-gpio { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + mpu6500 { + mpu6500_irq_gpio: mpu6500-irq-gpio { + rockchip,pins = <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + cm3218 { + cm3218_irq_gpio: cm3218-irq-gpio { + rockchip,pins = <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + dc_det { + dc_irq_gpio: dc-irq-gpio { + rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pcfg_pull_none_4ma: pcfg-pull-none-4ma { + bias-disable; + drive-strength = <4>; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_rts_gpio: uart0-rts-gpio { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&isp_dvp_d2d9 &isp_dvp_d10d11 &cif_clkout>; + port { + #address-cells = <1>; + #size-cells = <0>; + + dvp_in_fcam: endpoint@0 { + reg = <0>; + remote-endpoint = <&gc0312_out>; + }; + + dvp_in_bcam: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc2145_out>; + }; + + isp_mipi_in: endpoint@2 { + reg = <2>; + remote-endpoint = <&dphy_rx_out>; + }; + }; +}; + +&route_dsi { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF + | RKPM_SLP_PMU_PLLS_PWRDN + | RKPM_SLP_PMU_PMUALIVE_32K + | RKPM_SLP_SFT_PLLS_DEEP + | RKPM_SLP_PMU_DIS_OSC + | RKPM_SLP_SFT_PD_NBSCUS + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_USB_WKUP_EN + | RKPM_CLUSTER_L_WKUP_EN + ) + >; +}; + +&saradc { + status = "okay"; +}; + +&sdmmc { + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "disabled"; +}; + +&sdio0 { + max-frequency = <100000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + rockchip,default-sample-phase = <90>; + sd-uhs-sdr104; + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&vdd_cpu>; + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_otg: otg-port { + status = "okay"; + }; + + u2phy_host: host-port { + phy-supply = <&vcc_host>; + status = "okay"; + }; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3368-xikp-avb.dts b/arch/arm64/boot/dts/rockchip/rk3368-xikp-avb.dts new file mode 100755 index 000000000000..b455e32c3e29 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-xikp-avb.dts @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3368-xikp.dtsi" + +/ { + model = "Rockchip rk3368 xkp avb board"; + compatible = "rockchip,xkp-avb", "rockchip,rk3368"; +}; + +&firmware_android { + compatible = "android,firmware"; + boot_devices = "ff0f0000.dwmmc,ff400000.nandc"; + vbmeta { + compatible = "android,vbmeta"; + parts = "vbmeta,dtbo"; + }; + fstab { + compatible = "android,fstab"; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,avb"; + }; + }; +}; + +&chosen { + bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 androidboot.selinux=permissive init=/init kpti=0"; +}; + +&i2c3 { + status = "okay"; + + gc2145: gc2145@3c { + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + + power-gpio = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_out: endpoint { + remote-endpoint = <&isp_dvp_in>; + }; + }; + }; + + ov8858: ov8858@36 { + compatible = "ovti,ov8858"; + reg = <0x36>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan-9569A2"; + power-gpio = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + port { + ov8858_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; + +}; + +&isp { + status = "disabled"; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov8858_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&isp_dvp_d2d9 &isp_dvp_d10d11 &cif_clkout>; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_dvp_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc2145_out>; + }; + + isp_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx_out>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-xikp.dts b/arch/arm64/boot/dts/rockchip/rk3368-xikp.dts new file mode 100755 index 000000000000..bc320ffdc3b7 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-xikp.dts @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3368-xikp.dtsi" + +/ { + model = "Rockchip rk3368 xkp board"; + compatible = "rockchip,xkp", "rockchip,rk3368"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; \ No newline at end of file diff --git a/arch/arm64/boot/dts/rockchip/rk3368-xikp.dtsi b/arch/arm64/boot/dts/rockchip/rk3368-xikp.dtsi new file mode 100755 index 000000000000..aa73096aff9d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-xikp.dtsi @@ -0,0 +1,893 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include +#include +#include +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" + +/ { + es8316-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk-es8316-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&es8316>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk818 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3500>; + rockchip,screen-on-voltage = <3600>; + status = "okay"; + }; + + gpio_keys: gpio-keys { + compatible = "gpio-keys"; + autorepeat; + + power { + debounce-interval = <100>; + gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; + label = "GPIO Key Power"; + linux,code = ; + wakeup-source; + }; + }; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1024000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1000>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <170000>; + }; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + /* wifi_chip_type - wifi chip define + * ap6210, ap6330, ap6335 + * rtl8188eu, rtl8723bs, rtl8723bu + * esp8089 + */ + wifi_chip_type = "ap6255"; + sdio_vref = <1800>; //1800mv or 3300mv + WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk818 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_rts_gpio>; + + //BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; + BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; + + status = "okay"; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_host: vcc-host { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; +}; + +&cpu_l0 { + cpu-supply = <&syr827>; +}; + +&cpu_l1 { + cpu-supply = <&syr827>; +}; + +&cpu_l2 { + cpu-supply = <&syr827>; +}; + +&cpu_l3 { + cpu-supply = <&syr827>; +}; + +&cpu_b0 { + cpu-supply = <&syr827>; +}; + +&cpu_b1 { + cpu-supply = <&syr827>; +}; + +&cpu_b2 { + cpu-supply = <&syr827>; +}; + +&cpu_b3 { + cpu-supply = <&syr827>; +}; + +&gpu { + logic-supply = <&vdd_logic>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_logic>; + devfreq-events = <&dfi>; + upthreshold = <60>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 600000 + SYS_STATUS_REBOOT 600000 + SYS_STATUS_SUSPEND 240000 + SYS_STATUS_VIDEO_1080P 396000 + SYS_STATUS_VIDEO_4K 600000 + SYS_STATUS_PERFORMANCE 600000 + SYS_STATUS_BOOST 396000 + SYS_STATUS_DUALVIEW 600000 + SYS_STATUS_ISP 528000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 582 240000 + 583 99999 396000 + >; + auto-min-freq = <240000>; + auto-freq-en = <1>; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF + | RKPM_SLP_PMU_PLLS_PWRDN + | RKPM_SLP_PMU_PMUALIVE_32K + | RKPM_SLP_SFT_PLLS_DEEP + | RKPM_SLP_PMU_DIS_OSC + | RKPM_SLP_SFT_PD_NBSCUS + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_USB_WKUP_EN + | RKPM_CLUSTER_L_WKUP_EN + ) + >; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&sdmmc { + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "disabled"; +}; + +&sdio0 { + clock-frequency = <100000000>; + clock-freq-min-max = <200000 100000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + syr827: syr827@40 { + compatible = "silergy,syr827"; + status = "okay"; + reg = <0x40>; + + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_arm"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + pinctrl-0 = <&vsel_gpio>; + vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + rk818: pmic@1c { + compatible = "rockchip,rk818"; + status = "okay"; + reg = <0x1c>; + clock-output-names = "rk818-clkout1", "wifibt_32kin"; + interrupt-parent = <&gpio0>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + extcon = <&u2phy>; + #clock-cells = <1>; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc_sys>; + vcc9-supply = <&vcc_io>; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-name = "vdd_logic"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1450000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-name = "vdd_gpu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1250000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_io"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca_codec: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcca_codec"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_tp: LDO_REG2 { + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_10: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd_10"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc18_lcd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc18_lcd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_pmu: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vccio_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd10_lcd: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vdd10_lcd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc_18: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_18"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_wl: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vccio_wl"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vccio_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: SWITCH_REG { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + boost_otg: DCDC_BOOST { + regulator-name = "boost_otg"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk818-battery"; + pinctrl-names = "default"; + pinctrl-0 = <&dc_irq_gpio>; + ocv_table = < + 3400 3652 3680 3707 3730 3747 3764 + 3772 3781 3792 3807 3828 3861 3899 + 3929 3958 3987 4038 4079 4127 4186>; + design_capacity = <7536>; + design_qmax = <8290>; + bat_res = <100>; + max_input_current = <1750>; + max_chrg_current = <2000>; + max_chrg_voltage = <4200>; + sleep_enter_current = <600>; + sleep_exit_current = <600>; + power_off_thresd = <3400>; + zero_algorithm_vol = <3850>; + fb_temperature = <115>; + sample_res = <20>; + max_soc_offset = <60>; + energy_mode = <0>; + monitor_sec = <5>; + virtual_power = <0>; + power_dc2otg = <0>; + support_usb_adp = <1>; + support_dc_adp = <1>; + dc_det_gpio = <&gpio0 17 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&i2c1 { + status = "okay"; + + es8316: es8316@10 { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "everest,es8316"; + reg = <0x10>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + spk-con-gpio = <&gpio3 9 GPIO_ACTIVE_HIGH>; + hp-det-gpio = <&gpio0 23 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + }; +}; + +&i2c2 { + status = "okay"; + + gt9xx: gt9xx@14 { + compatible = "goodix,gt9xx"; + reg = <0x14>; + touch-gpio = <&gpio0 12 IRQ_TYPE_LEVEL_HIGH>; + reset-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; + max-x = <1920>; + max-y = <1200>; + tp-size = <89>; + configfile-num = <1>; + status = "okay"; + tp-supply = <&vcc_tp>; + }; +}; + +&i2c3 { + status = "okay"; +}; + +&i2c4 { + status = "okay"; + + mpu6500@68 { + status = "disabled"; + compatible = "invensense,mpu6500"; + pinctrl-names = "default"; + pinctrl-0 = <&mpu6500_irq_gpio>; + reg = <0x68>; + irq-gpio = <&gpio3 14 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <1 0 0 0 1 0 0 0 1>; + orientation-x= <1>; + orientation-y= <0>; + orientation-z= <1>; + support-hw-poweroff = <1>; + mpu-debug = <1>; + }; + + sensor@4c { + status = "okay"; + compatible = "gs_mc3230"; + reg = <0x4c>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <4>; + reprobe_en = <1>; + }; + + sensor@19 { + status = "okay"; + compatible = "gs_lis3dh"; + reg = <0x19>; + type = ; + irq-gpio = <&gpio3 14 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <6>; + reprobe_en = <1>; + }; + + sensor@10 { + status = "okay"; + compatible = "light_cm3218"; + pinctrl-names = "default"; + pinctrl-0 = <&cm3218_irq_gpio>; + reg = <0x10>; + type = ; + irq-gpio = <&gpio3 15 IRQ_TYPE_EDGE_FALLING>; + irq_enable = <1>; + poll_delay_ms = <30>; + }; +}; + +&i2s_8ch { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + dvp-supply = <&vcc_18>; + audio-supply = <&vcc_io>; + gpio30-supply = <&vcc_io>; + gpio1830-supply = <&vcc_io>; + sdcard-supply = <&vccio_sd>; + wifi-supply = <&vccio_wl>; +}; + +&pmu_io_domains { + status = "okay"; + + pmu-supply = <&vccio_pmu>; + vop-supply = <&vccio_pmu>; +}; + +&pwm0 { + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&saradc { + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_host: host-port { + phy-supply = <&vcc_host>; + status = "okay"; + }; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <120>; + enable-delay-ms = <200>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <159000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <60>; + hfront-porch = <80>; + vback-porch = <25>; + vfront-porch = <35>; + hsync-len = <1>; + vsync-len = <1>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&route_dsi { + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&syr827>; + status = "okay"; +}; + +&pinctrl { + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel_gpio: vsel-gpio { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + mpu6500 { + mpu6500_irq_gpio: mpu6500-irq-gpio { + rockchip,pins = <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + cm3218 { + cm3218_irq_gpio: cm3218-irq-gpio { + rockchip,pins = <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + dc_det { + dc_irq_gpio: dc-irq-gpio { + rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_rts_gpio: uart0-rts-gpio { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3368.dtsi b/arch/arm64/boot/dts/rockchip/rk3368.dtsi index 3746f23dc3df..e962c7962789 100644 --- a/arch/arm64/boot/dts/rockchip/rk3368.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3368.dtsi @@ -562,7 +562,7 @@ pwm0: pwm@ff680000 { compatible = "rockchip,rk3368-pwm", "rockchip,rk3288-pwm"; reg = <0x0 0xff680000 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_pin>; clocks = <&cru PCLK_PWM1>; clock-names = "pwm"; @@ -573,7 +573,7 @@ pwm1: pwm@ff680010 { compatible = "rockchip,rk3368-pwm", "rockchip,rk3288-pwm"; reg = <0x0 0xff680010 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_pin>; clocks = <&cru PCLK_PWM1>; clock-names = "pwm"; @@ -593,7 +593,7 @@ pwm3: pwm@ff680030 { compatible = "rockchip,rk3368-pwm", "rockchip,rk3288-pwm"; reg = <0x0 0xff680030 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm3_pin>; clocks = <&cru PCLK_PWM1>; clock-names = "pwm"; @@ -1009,17 +1009,33 @@ pwm0 { pwm0_pin: pwm0-pin { rockchip,pins = <3 RK_PB0 2 &pcfg_pull_none>; }; + + pwm0_pin_pull_down: pwm0-pin-pull-down { + rockchip,pins = <3 RK_PB0 2 &pcfg_pull_down>; + }; + + vop_pwm_pin: vop-pwm { + rockchip,pins = <3 RK_PB0 3 &pcfg_pull_none>; + }; }; pwm1 { pwm1_pin: pwm1-pin { rockchip,pins = <0 RK_PB0 2 &pcfg_pull_none>; }; + + pwm1_pin_pull_down: pwm1-pin-pull-down { + rockchip,pins = <0 RK_PB0 2 &pcfg_pull_down>; + }; }; pwm3 { pwm3_pin: pwm3-pin { - rockchip,pins = <3 RK_PD5 3 &pcfg_pull_none>; + rockchip,pins = <3 RK_PD6 3 &pcfg_pull_none>; + }; + + pwm3_pin_pull_down: pwm3-pin-pull-down { + rockchip,pins = <3 RK_PD6 3 &pcfg_pull_down>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet-bnd.dts b/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet-bnd.dts new file mode 100755 index 000000000000..0a489bf8fc84 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet-bnd.dts @@ -0,0 +1,1074 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; +#include +#include +#include +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" +/ { + model = "Rockchip rk3368a tablet rk817 board"; + compatible = "rockchip,tablet", "rockchip,rk3368a", "rockchip,rk3368"; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1024000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1000>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <170000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 1>; + brightness-levels = < + 10 10 10 11 12 13 14 15 + 16 16 16 16 16 16 16 16 + 17 17 17 17 17 17 17 17 + 18 18 18 18 18 18 18 18 + 19 19 19 19 19 19 19 19 + 20 20 20 20 20 20 20 20 + 21 21 21 21 21 21 21 21 + 22 22 22 22 22 22 22 22 + 23 23 23 23 23 23 23 23 + 24 24 24 24 24 24 24 24 + 25 25 25 25 25 25 25 25 + 26 26 26 26 26 26 26 26 + 27 27 27 27 27 27 27 27 + 28 28 28 28 28 28 28 28 + 27 27 27 27 27 27 27 27 + 30 30 30 30 30 30 30 30 + 31 31 31 31 31 31 31 31 + 32 32 32 32 32 32 32 32 + 33 33 33 33 33 33 33 33 + 34 34 34 34 34 34 34 34 + 35 35 35 35 35 35 35 35 + 36 36 36 36 36 36 36 36 + 37 37 37 37 37 37 37 37 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38 + 38 38 38 38 38 38 38 38>; + default-brightness-level = <200>; + enable-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3500>; + rockchip,screen-on-voltage = <3600>; + status = "okay"; + }; + + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init kpti=0"; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip-rk817-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + /* wifi_chip_type - wifi chip define + * ap6210, ap6330, ap6335 + * rtl8188eu, rtl8723bs, rtl8723bu + * esp8089 + */ + wifi_chip_type = "ap6255"; + WIFI,vbat_gpio = <&gpio0 2 GPIO_ACTIVE_HIGH>; + WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_rts_gpio>; + + //BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; + BT,reset_gpio = <&gpio3 5 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; + + status = "okay"; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_host: vcc-host { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + +}; + +&cif_clkout { + /* cif_clkout */ + rockchip,pins = <1 RK_PB3 1 &pcfg_pull_none_4ma>; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b3 { + cpu-supply = <&vdd_cpu>; +}; + +&gpu { + logic-supply = <&vdd_logic>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_logic>; + devfreq-events = <&dfi>; + upthreshold = <60>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 600000 + SYS_STATUS_REBOOT 600000 + SYS_STATUS_SUSPEND 240000 + SYS_STATUS_VIDEO_1080P 396000 + SYS_STATUS_VIDEO_4K 600000 + SYS_STATUS_PERFORMANCE 600000 + SYS_STATUS_BOOST 396000 + SYS_STATUS_DUALVIEW 600000 + SYS_STATUS_ISP 528000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 582 240000 + 583 99999 396000 + >; + vop-dclk-mode = <1>; + auto-min-freq = <240000>; + auto-freq-en = <0>; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; + /* back-gpios = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; */ + reset-gpios = <&gpio3 RK_PB1 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <8>; + enable-delay-ms = <3>; + reset-delay-ms = <50>; + init-delay-ms = <20>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + width-mm = <153>; + height-mm = <85>; + panel-init-sequence = [ + 05 1e 01 01 + 15 00 02 80 47 + 15 00 02 81 40 + 15 00 02 82 04 + 15 00 02 83 77 + 15 00 02 84 0f + 15 00 02 85 70 + 15 78 02 86 70 + ]; + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <49500000>; + hactive = <1024>; + vactive = <600>; + hback-porch = <120>; + hfront-porch = <80>; + vback-porch = <14>; + vfront-porch = <14>; + hsync-len = <40>; + vsync-len = <4>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@10 { + compatible = "tcs,tcs452x"; + reg = <0x10>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel_gpio>; + vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vcc_io>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_ts_gpio1: rk817_ts_gpio1 { + pins = "gpio_ts"; + function = "pin_fun1"; + /* output-low; */ + /* input-enable; */ + }; + + rk817_gt_gpio2: rk817_gt_gpio2 { + pins = "gpio_gt"; + function = "pin_fun1"; + }; + + rk817_pin_ts: rk817_pin_ts { + pins = "gpio_ts"; + function = "pin_fun0"; + }; + + rk817_pin_gt: rk817_pin_gt { + pins = "gpio_gt"; + function = "pin_fun0"; + }; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_3v3: DCDC_REG2 { + regulator-name = "vcc_3v3"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_io"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v3_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-name = "boost"; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3500 3548 3592 3636 3687 3740 3780 + 3806 3827 3846 3864 3889 3929 3964 + 3993 4015 4030 4041 4056 4076 4148>; + design_capacity = <4000>; + design_qmax = <4200>; + bat_res = <100>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3500>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4200>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&u2phy>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + status = "okay"; + }; + }; +}; + +&i2c2 { + status = "okay"; + + ts@5a { + compatible = "cst2xxse"; + reg = <0x5a>; + irq-gpio = <&gpio0 RK_PB4 IRQ_TYPE_LEVEL_LOW>; + //touch-gpio = <&gpio1 GPIO_B0 IRQ_TYPE_LEVEL_LOW>; /* TP_INT == GPIO1_B0 */ + //reset-gpio = <&gpio0 GPIO_D1 GPIO_ACTIVE_LOW>; /* TP_RST == GPIO0_D1 */ + //power-gpio = <&gpio0 GPIO_C5 GPIO_ACTIVE_LOW>; + //max-x = <800>; + //max-y = <480>; + status = "okay"; + }; +}; + +&i2c3 { + status = "okay"; + + gc0312: gc0312@21 { + status = "okay"; + compatible = "galaxycore,gc0312"; + reg = <0x21>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vdd1v5_dvp>; + + pwdn-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc0312_out: endpoint { + remote-endpoint = <&dvp_in_fcam>; + }; + }; + }; + + gc2145: gc2145@3c { + status = "okay"; + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vdd1v5_dvp>; + + pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_out: endpoint { + remote-endpoint = <&dvp_in_bcam>; + }; + }; + }; + + ov8858: ov8858@36 { + status = "disabled"; + compatible = "ovti,ov8858"; + reg = <0x36>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vdd1v5_dvp>; + + pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan-9569A2"; + port { + ov8858_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c4 { + status = "okay"; + + sc7a30: sc7a30@18 { + status = "okay"; + compatible = "gs_sc7a30"; + reg = <0x18>; + type = ; + pinctrl-names = "default"; + pinctrl-0 = <&sc7a30_irq_gpio>; + irq-gpio = <&gpio3 RK_PB6 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <6>; + reprobe_en = <1>; + }; + +}; + +&i2s_8ch { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + dvp-supply = <&vcc1v8_dvp>; + audio-supply = <&vcc_io>; + gpio30-supply = <&vcc_io>; + gpio1830-supply = <&vcc_io>; + sdcard-supply = <&vccio_sd>; + wifi-supply = <&vcc_3v3>; +}; + +&isp_dvp_d2d9 { + rockchip,pins = + /* cif_data4 ... cif_data9 */ + <1 RK_PA2 1 &pcfg_pull_down>, + <1 RK_PA3 1 &pcfg_pull_down>, + <1 RK_PA4 1 &pcfg_pull_down>, + <1 RK_PA5 1 &pcfg_pull_down>, + <1 RK_PA6 1 &pcfg_pull_down>, + <1 RK_PA7 1 &pcfg_pull_down>, + /* cif_sync, cif_href */ + <1 RK_PB0 1 &pcfg_pull_down>, + <1 RK_PB1 1 &pcfg_pull_down>, + /* cif_clkin */ + <1 RK_PB2 1 &pcfg_pull_down>; +}; + +&isp_dvp_d10d11 { + rockchip,pins = + /* cif_data10, cif_data11 */ + <1 RK_PB6 1 &pcfg_pull_down>, + <1 RK_PB7 1 &pcfg_pull_down>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov8858_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + + pmu-supply = <&vcc3v3_pmu>; + vop-supply = <&vcc3v3_pmu>; +}; + +&pwm0 { + status = "okay"; +}; + +&pinctrl { + + headphone { + hp_det: hp-det { + rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA0 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA0 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA0 2 &pcfg_pull_none>; + }; + + vsel_gpio: vsel-gpio { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sc7a30 { + sc7a30_irq_gpio: sc7a30_irq_gpio { + rockchip,pins = <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + dc_det { + dc_irq_gpio: dc-irq-gpio { + rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pcfg_pull_none_4ma: pcfg-pull-none-4ma { + bias-disable; + drive-strength = <4>; + }; + + pcfg_pull_none_smt: pcfg-pull-none-smt { + bias-disable; + input-schmitt-enable; + }; + + pcfg_output_high: pcfg-output-high { + output-high; + }; + + pcfg_output_low: pcfg-output-low { + output-low; + }; + + pcfg_input_high: pcfg-input-high { + bias-pull-up; + input-enable; + }; + + pcfg_input: pcfg-input { + input-enable; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_rts_gpio: uart0-rts-gpio { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&isp_dvp_d2d9 &isp_dvp_d10d11 &cif_clkout>; + port { + #address-cells = <1>; + #size-cells = <0>; + + dvp_in_fcam: endpoint@0 { + reg = <0>; + remote-endpoint = <&gc0312_out>; + }; + + dvp_in_bcam: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc2145_out>; + }; + + isp_mipi_in: endpoint@2 { + reg = <2>; + remote-endpoint = <&dphy_rx_out>; + }; + }; +}; + +&route_dsi { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF + | RKPM_SLP_PMU_PLLS_PWRDN + | RKPM_SLP_PMU_PMUALIVE_32K + | RKPM_SLP_SFT_PLLS_DEEP + | RKPM_SLP_PMU_DIS_OSC + | RKPM_SLP_SFT_PD_NBSCUS + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_USB_WKUP_EN + | RKPM_CLUSTER_L_WKUP_EN + ) + >; +}; + +&saradc { + status = "okay"; +}; + +&sdmmc { + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + vmmc-supply = <&vcc_sd>; + status = "disabled"; +}; + +&sdio0 { + max-frequency = <50000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + rockchip,default-sample-phase = <90>; + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&vdd_cpu>; + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_otg: otg-port { + status = "okay"; + }; + + u2phy_host: host-port { + phy-supply = <&vcc_host>; + status = "okay"; + }; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet.dts b/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet.dts new file mode 100755 index 000000000000..31f25e44504d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368a-817-tablet.dts @@ -0,0 +1,1333 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; +#include +#include +#include +#include "rk3368.dtsi" +#include "rk3368-android.dtsi" +/ { + model = "Rockchip rk3368a tablet rk817 board"; + compatible = "rockchip,tablet", "rockchip,rk3368a", "rockchip,rk3368"; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1024000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1000>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <170000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 1>; + brightness-levels = < + 30 30 30 31 31 31 32 32 + 32 33 33 33 34 34 34 35 + 35 35 36 36 36 37 37 37 + 38 38 38 39 39 39 40 40 + 40 41 41 41 42 42 42 43 + 43 43 44 44 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 232 232 233 233 233 234 234 + 234 235 235 235 236 236 236 237 + 237 238 238 239 239 240 240 240>; + default-brightness-level = <200>; + enable-gpios = <&gpio3 28 GPIO_ACTIVE_HIGH>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3400>; + rockchip,screen-on-voltage = <3400>; + status = "okay"; + }; + + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff690000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init kpti=0"; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip-rk817-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>; /* GPIO3_A4 */ + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + /* wifi_chip_type - wifi chip define + * ap6210, ap6330, ap6335 + * rtl8188eu, rtl8723bs, rtl8723bu + * esp8089 + */ + wifi_chip_type = "rtl8723bs"; + WIFI,vbat_gpio = <&gpio0 2 GPIO_ACTIVE_HIGH>; + WIFI,host_wake_irq = <&gpio3 6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio2 27 GPIO_ACTIVE_LOW>; + pinctrl-names = "default","rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_rts_gpio>; + + //BT,power_gpio = <&gpio3 3 GPIO_ACTIVE_HIGH>; + BT,reset_gpio = <&gpio3 13 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 7 GPIO_ACTIVE_HIGH>; + + status = "okay"; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_host: vcc-host { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + +}; + +&cif_clkout { + /* cif_clkout */ + rockchip,pins = <1 RK_PB3 1 &pcfg_pull_none_4ma>; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu_b3 { + cpu-supply = <&vdd_cpu>; +}; + +&gpu { + logic-supply = <&vdd_logic>; +}; + +&dsi { + status = "okay"; + //rockchip,lane-rate = <500>; + + panel:panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <20>; + reset-delay-ms = <20>; + init-delay-ms = <20>; + enable-delay-ms = <120>; + disable-delay-ms = <20>; + unprepare-delay-ms = <20>; + + width-mm = <135>; + height-mm = <216>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 39 00 04 FF 98 81 03 + 15 00 02 01 00 + 15 00 02 02 00 + 15 00 02 03 53 + 15 00 02 04 00 + 15 00 02 05 00 + 15 00 02 06 08 + 15 00 02 07 00 + 15 00 02 08 00 + 15 00 02 09 00 + 15 00 02 0A 00 + 15 00 02 0B 00 + 15 00 02 0C 00 + 15 00 02 0D 00 + 15 00 02 0E 00 + 15 00 02 0F 26 + 15 00 02 10 26 + 15 00 02 11 00 + 15 00 02 12 00 + 15 00 02 13 00 + 15 00 02 14 00 + 15 00 02 15 00 + 15 00 02 16 00 + 15 00 02 17 00 + 15 00 02 18 00 + 15 00 02 19 00 + 15 00 02 1A 00 + 15 00 02 1B 00 + 15 00 02 1C 00 + 15 00 02 1D 00 + 15 00 02 1E 40 + 15 00 02 1F C0 + 15 00 02 20 06 + 15 00 02 21 01 + 15 00 02 22 07 + 15 00 02 23 00 + 15 00 02 24 8A + 15 00 02 25 8A + 15 00 02 26 00 + 15 00 02 27 00 + 15 00 02 28 33 + 15 00 02 29 33 + 15 00 02 2A 00 + 15 00 02 2B 00 + 15 00 02 2C 08 + 15 00 02 2D 08 + 15 00 02 2E 0B + 15 00 02 2F 0B + 15 00 02 30 00 + 15 00 02 31 00 + 15 00 02 32 42 + 15 00 02 33 00 + 15 00 02 34 00 + 15 00 02 35 0A + 15 00 02 36 00 + 15 00 02 37 08 + 15 00 02 38 3C + 15 00 02 39 00 + 15 00 02 3A 00 + 15 00 02 3B 00 + 15 00 02 3C 00 + 15 00 02 3D 00 + 15 00 02 3E 00 + 15 00 02 3F 00 + 15 00 02 40 00 + 15 00 02 41 00 + 15 00 02 42 00 + 15 00 02 43 08 + 15 00 02 44 00 + 15 00 02 50 01 + 15 00 02 51 23 + 15 00 02 52 45 + 15 00 02 53 67 + 15 00 02 54 89 + 15 00 02 55 AB + 15 00 02 56 01 + 15 00 02 57 23 + 15 00 02 58 45 + 15 00 02 59 67 + 15 00 02 5A 89 + 15 00 02 5B AB + 15 00 02 5C CD + 15 00 02 5D EF + 15 00 02 5E 00 + 15 00 02 5F 01 + 15 00 02 60 01 + 15 00 02 61 06 + 15 00 02 62 06 + 15 00 02 63 06 + 15 00 02 64 06 + 15 00 02 65 00 + 15 00 02 66 00 + 15 00 02 67 17 + 15 00 02 68 02 + 15 00 02 69 16 + 15 00 02 6A 16 + 15 00 02 6B 02 + 15 00 02 6C 0D + 15 00 02 6D 0D + 15 00 02 6E 0C + 15 00 02 6F 0C + 15 00 02 70 0F + 15 00 02 71 0F + 15 00 02 72 0E + 15 00 02 73 0E + 15 00 02 74 02 + 15 00 02 75 01 + 15 00 02 76 01 + 15 00 02 77 06 + 15 00 02 78 06 + 15 00 02 79 06 + 15 00 02 7A 06 + 15 00 02 7B 00 + 15 00 02 7C 00 + 15 00 02 7D 17 + 15 00 02 7E 02 + 15 00 02 7F 16 + 15 00 02 80 16 + 15 00 02 81 02 + 15 00 02 82 0D + 15 00 02 83 0D + 15 00 02 84 0C + 15 00 02 85 0C + 15 00 02 86 0F + 15 00 02 87 0F + 15 00 02 88 0E + 15 00 02 89 0E + 15 00 02 8A 02 + 39 00 04 FF 98 81 04 + 15 00 02 6E 2B + 15 00 02 6F 35 + 15 00 02 3A A4 + 15 00 02 8D 1A + 15 00 02 87 BA + 15 00 02 B2 D1 + 15 00 02 88 0B + 15 00 02 38 01 + 15 00 02 39 00 + 15 00 02 B5 07 + 15 00 02 31 75 + 15 00 02 3B 98 + 39 00 04 FF 98 81 01 + 15 00 02 22 0A + 15 00 02 31 00 + 15 00 02 53 40 + 15 00 02 55 40 + 15 00 02 50 95 + 15 00 02 51 90 + 15 00 02 60 22 + 15 00 02 62 20 + 15 00 02 A0 00 + 15 00 02 A1 1B + 15 00 02 A2 2A + 15 00 02 A3 14 + 15 00 02 A4 17 + 15 00 02 A5 2B + 15 00 02 A6 1F + 15 00 02 A7 20 + 15 00 02 A8 93 + 15 00 02 A9 1E + 15 00 02 AA 2A + 15 00 02 AB 7E + 15 00 02 AC 1B + 15 00 02 AD 19 + 15 00 02 AE 4C + 15 00 02 AF 22 + 15 00 02 B0 28 + 15 00 02 B1 4B + 15 00 02 B2 59 + 15 00 02 B3 23 + 15 00 02 C0 00 + 15 00 02 C1 1B + 15 00 02 C2 2A + 15 00 02 C3 14 + 15 00 02 C4 17 + 15 00 02 C5 2B + 15 00 02 C6 1F + 15 00 02 C7 20 + 15 00 02 C8 93 + 15 00 02 C9 1E + 15 00 02 CA 2A + 15 00 02 CB 7E + 15 00 02 CC 1B + 15 00 02 CD 19 + 15 00 02 CE 4C + 15 00 02 CF 22 + 15 00 02 D0 28 + 15 00 02 D1 4B + 15 00 02 D2 59 + 15 00 02 D3 23 + //39 00 04 FF 98 81 04 + //05 00 02 2D 80 + //05 00 02 2F 31 + 39 00 04 FF 98 81 00 + 05 78 01 11 + 05 14 01 29 + 15 00 02 35 00 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <78000000>; + hactive = <800>; + vactive = <1280>; + hfront-porch = <60>;//70 //16 + hsync-len = <30>; //20 //5 + hback-porch = <60>; //59 + vfront-porch = <20>; //16 //8 + vsync-len = <8>; //5 + vback-porch = <16>; //22 //3 + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_logic>; + devfreq-events = <&dfi>; + upthreshold = <60>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 600000 + SYS_STATUS_REBOOT 600000 + SYS_STATUS_SUSPEND 240000 + SYS_STATUS_VIDEO_1080P 396000 + SYS_STATUS_VIDEO_4K 600000 + SYS_STATUS_PERFORMANCE 600000 + SYS_STATUS_BOOST 396000 + SYS_STATUS_DUALVIEW 600000 + SYS_STATUS_ISP 528000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 582 240000 + 583 99999 396000 + >; + vop-dclk-mode = <1>; + auto-min-freq = <240000>; + auto-freq-en = <0>; +}; + +&emmc { + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sdio; + no-sd; + disable-wp; + non-removable; + num-slots = <1>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: syr827@40 { + compatible = "silergy,syr827"; + status = "okay"; + reg = <0x40>; + + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + pinctrl-0 = <&vsel_gpio>; + vsel-gpios = <&gpio0 0 GPIO_ACTIVE_HIGH>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <1>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vcc_io>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_ts_gpio1: rk817_ts_gpio1 { + pins = "gpio_ts"; + function = "pin_fun1"; + /* output-low; */ + /* input-enable; */ + }; + + rk817_gt_gpio2: rk817_gt_gpio2 { + pins = "gpio_gt"; + function = "pin_fun1"; + }; + + rk817_pin_ts: rk817_pin_ts { + pins = "gpio_ts"; + function = "pin_fun0"; + }; + + rk817_pin_gt: rk817_pin_gt { + pins = "gpio_gt"; + function = "pin_fun0"; + }; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <950000>; + }; + }; + + vcc_3v3: DCDC_REG2 { + regulator-name = "vcc_3v3"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_io: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_io"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_1v0: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-name = "vcc_1v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc1v8_soc: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v0_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + + regulator-name = "vcc1v0_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1000000>; + }; + }; + + vcc3v3_pmu: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_sd: LDO_REG6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + + }; + }; + + vcc2v8_dvp: LDO_REG7 { + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <2800000>; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd1v5_dvp: LDO_REG9 { + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-name = "boost"; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3500 3548 3592 3636 3687 3740 3780 + 3806 3827 3846 3864 3889 3929 3964 + 3993 4015 4030 4041 4056 4076 4148>; + design_capacity = <5000>; + design_qmax = <5500>; + bat_res = <100>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3400>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4200>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&u2phy>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + out-l2spk-r2hp; + spk-ctl-gpios = <&gpio0 RK_PD3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + }; +}; + +&i2c2 { + status = "okay"; + + ts@40 { + status = "okay"; + compatible = "GSL,GSL_THZY"; + reg = <0x40>; + irq_gpio_number = <&gpio0 RK_PB4 IRQ_TYPE_LEVEL_HIGH>; + rst_gpio_number = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c3 { + status = "okay"; + + gc032a: gc032a@21 { + status = "okay"; + compatible = "galaxycore,gc032a"; + reg = <0x21>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vdd1v5_dvp>; + + pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc0312_out: endpoint { + remote-endpoint = <&dvp_in_fcam>; + }; + }; + }; + + gc2145: gc2145@3c { + status = "okay"; + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vdd1v5_dvp>; + + pwdn-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio3 RK_PC4 GPIO_ACTIVE_LOW>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_mipi_out: endpoint { + remote-endpoint = <&mipi_in_bcam>; + data-lanes = <1>; + }; + }; + }; + + ov8858: ov8858@36 { + status = "disabled"; + compatible = "ovti,ov8858"; + reg = <0x36>; + clocks = <&cru SCLK_VIP_OUT>; + clock-names = "xvclk"; + + avdd-supply = <&vcc2v8_dvp>; + dovdd-supply = <&vcc1v8_dvp>; + dvdd-supply = <&vdd1v5_dvp>; + + pwdn-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan-9569A2"; + port { + ov8858_out: endpoint { + remote-endpoint = <&mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c4 { + status = "okay"; + + sensor@26 { + compatible = "gs_da223"; + reg = <0x26>; + type = ; + irq_enable = <0>; + poll_delay_ms = <10>; + layout = <3>; + }; + + sensor@19 { + compatible = "gs_sc7a20"; + reg = <0x19>; + type = ; + irq_enable = <0>; + poll_delay_ms = <10>; + layout = <1>; + }; + + mpu6500@68 { + status = "disabled"; + compatible = "invensense,mpu6500"; + pinctrl-names = "default"; + pinctrl-0 = <&mpu6500_irq_gpio>; + reg = <0x68>; + irq-gpio = <&gpio3 14 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <1 0 0 0 1 0 0 0 1>; + orientation-x= <1>; + orientation-y= <0>; + orientation-z= <1>; + support-hw-poweroff = <1>; + mpu-debug = <1>; + }; + + sensor@4c { + status = "disabled"; + compatible = "gs_mc3230"; + reg = <0x4c>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <9>; + reprobe_en = <1>; + irq-gpio = <&gpio3 RK_PB6 IRQ_TYPE_LEVEL_LOW>; + }; + + sensor@18 { + status = "gs_mc3230"; + compatible = "gs_sc7a30"; + reg = <0x18>; + type = ; + irq-gpio = <&gpio3 RK_PB6 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <6>; + reprobe_en = <1>; + }; + + sensor@10 { + status = "gs_mc3230"; + compatible = "light_cm3218"; + pinctrl-names = "default"; + pinctrl-0 = <&cm3218_irq_gpio>; + reg = <0x10>; + type = ; + irq-gpio = <&gpio3 15 IRQ_TYPE_EDGE_FALLING>; + irq_enable = <1>; + poll_delay_ms = <30>; + }; +}; + +&i2s_8ch { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + + dvp-supply = <&vcc1v8_dvp>; + audio-supply = <&vcc_io>; + gpio30-supply = <&vcc_io>; + gpio1830-supply = <&vcc_io>; + sdcard-supply = <&vccio_sd>; + wifi-supply = <&vcc_3v3>; +}; + +&isp_dvp_d2d9 { + rockchip,pins = + /* cif_data4 ... cif_data9 */ + <1 RK_PA2 1 &pcfg_pull_down>, + <1 RK_PA3 1 &pcfg_pull_down>, + <1 RK_PA4 1 &pcfg_pull_down>, + <1 RK_PA5 1 &pcfg_pull_down>, + <1 RK_PA6 1 &pcfg_pull_down>, + <1 RK_PA7 1 &pcfg_pull_down>, + /* cif_sync, cif_href */ + <1 RK_PB0 1 &pcfg_pull_down>, + <1 RK_PB1 1 &pcfg_pull_down>, + /* cif_clkin */ + <1 RK_PB2 1 &pcfg_pull_down>; +}; + +&isp_dvp_d10d11 { + rockchip,pins = + /* cif_data10, cif_data11 */ + <1 RK_PB6 1 &pcfg_pull_down>, + <1 RK_PB7 1 &pcfg_pull_down>; +}; + +&isp_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + mipi_in_bcam: endpoint@0 { + reg = <0>; + remote-endpoint = <&gc2145_mipi_out>; + data-lanes = <1>; + }; + + mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov8858_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp_mipi_in>; + }; + }; + }; +}; + +&nandc0 { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + + pmu-supply = <&vcc3v3_pmu>; + vop-supply = <&vcc3v3_pmu>; +}; + +&pwm0 { + status = "okay"; +}; + +&pinctrl { + + headphone { + hp_det: hp-det { + rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA0 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA0 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA0 2 &pcfg_pull_none>; + }; + + vsel_gpio: vsel-gpio { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + mpu6500 { + mpu6500_irq_gpio: mpu6500-irq-gpio { + rockchip,pins = <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + cm3218 { + cm3218_irq_gpio: cm3218-irq-gpio { + rockchip,pins = <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + dc_det { + dc_irq_gpio: dc-irq-gpio { + rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pcfg_pull_none_4ma: pcfg-pull-none-4ma { + bias-disable; + drive-strength = <4>; + }; + + pcfg_pull_none_smt: pcfg-pull-none-smt { + bias-disable; + input-schmitt-enable; + }; + + pcfg_output_high: pcfg-output-high { + output-high; + }; + + pcfg_output_low: pcfg-output-low { + output-low; + }; + + pcfg_input_high: pcfg-input-high { + bias-pull-up; + input-enable; + }; + + pcfg_input: pcfg-input { + input-enable; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_rts_gpio: uart0-rts-gpio { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&rkisp1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&isp_dvp_d2d9 &isp_dvp_d10d11 &cif_clkout>; + port { + #address-cells = <1>; + #size-cells = <0>; + + dvp_in_fcam: endpoint@0 { + reg = <0>; + remote-endpoint = <&gc0312_out>; + }; + + isp_mipi_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&dphy_rx_out>; + }; + }; +}; + +&route_dsi { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + Rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF + | RKPM_SLP_PMU_PLLS_PWRDN + | RKPM_SLP_PMU_PMUALIVE_32K + | RKPM_SLP_SFT_PLLS_DEEP + | RKPM_SLP_PMU_DIS_OSC + | RKPM_SLP_SFT_PD_NBSCUS + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_USB_WKUP_EN + | RKPM_CLUSTER_L_WKUP_EN + ) + >; +}; + +&saradc { + status = "okay"; +}; + +&sdmmc { + clock-frequency = <37500000>; + clock-freq-min-max = <400000 37500000>; + no-sdio; + no-mmc; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <200>; + disable-wp; + num-slots = <1>; + vmmc-supply = <&vcc_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "disabled"; +}; + +&sdio0 { + max-frequency = <50000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + rockchip,default-sample-phase = <90>; + status = "okay"; +}; + +&tsadc { + tsadc-supply = <&vdd_cpu>; + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&u2phy { + status = "okay"; + + u2phy_otg: otg-port { + status = "okay"; + }; + + u2phy_host: host-port { + phy-supply = <&vcc_host>; + status = "okay"; + }; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-android.dtsi new file mode 100755 index 000000000000..ffaa92b7a89e --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-android.dtsi @@ -0,0 +1,339 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include "rk3399-vop-clk-set.dtsi" +#include + +/ { + compatible = "rockchip,android", "rockchip,rk3399"; + + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff1a0000 coherent_pool=1m"; + }; + + cpuinfo { + compatible = "rockchip,cpuinfo"; + nvmem-cells = <&cpu_id>; + nvmem-cell-names = "id"; + }; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + rockchip,irq-mode-enable = <1>; /* If enable uart uses irq instead of fiq */ + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + pinctrl-names = "default"; + pinctrl-0 = <&uart2c_xfer>; + interrupts = ; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + drm_logo: drm-logo@00000000 { + compatible = "rockchip,drm-logo"; + reg = <0x0 0x0 0x0 0x0>; + }; + + ramoops: ramoops@110000 { + compatible = "ramoops"; + reg = <0x0 0x110000 0x0 0xf0000>; + record-size = <0x20000>; + console-size = <0x80000>; + ftrace-size = <0x00000>; + pmsg-size = <0x50000>; + }; + + secure_memory: secure-memory@20000000 { + compatible = "rockchip,secure-memory"; + reg = <0x0 0x20000000 0x0 0x10000000>; + status = "disabled"; + }; + + stb_devinfo: stb-devinfo@00000000 { + compatible = "rockchip,stb-devinfo"; + reg = <0x0 0x0 0x0 0x0>; + }; + }; + + gpio_keys: gpio-keys { + compatible = "gpio-keys"; + autorepeat; + pinctrl-names = "default"; + pinctrl-0 = <&pwrbtn>; + + power { + debounce-interval = <100>; + gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; + label = "GPIO Key Power"; + linux,code = ; + wakeup-source; + }; + }; + + rga: rga@ff680000 { + compatible = "rockchip,rga2"; + dev_mode = <1>; + reg = <0x0 0xff680000 0x0 0x1000>; + interrupts = ; + clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru SCLK_RGA_CORE>; + clock-names = "aclk_rga", "hclk_rga", "clk_rga"; + power-domains = <&power RK3399_PD_RGA>; + status = "okay"; + }; + + hdmi_dp_sound: hdmi-dp-sound { + status = "disabled"; + compatible = "rockchip,rk3399-hdmi-dp"; + rockchip,cpu = <&i2s2>; + rockchip,codec = <&hdmi>, <&cdn_dp>; + }; + + firmware { + firmware_android: android {}; + }; +}; + +&uart2 { + status = "disabled"; +}; + +&vopb { + support-multi-area; + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + support-multi-area; + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&hdmi { + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + ddc-i2c-scl-high-time-ns = <9625>; + ddc-i2c-scl-low-time-ns = <10000>; + status = "okay"; +}; + +&display_subsystem { + status = "okay"; + + ports = <&vopb_out>, <&vopl_out>; + logo-memory-region = <&drm_logo>; + secure-memory-region = <&secure_memory>; + route { + route_hdmi: route-hdmi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopb_out_hdmi>; + }; + + route_dsi: route-dsi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopb_out_dsi>; + }; + + route_dsi1: route-dsi1 { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopl_out_dsi1>; + }; + + route_edp: route-edp { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopb_out_edp>; + }; + }; +}; + +&dsi { + panel@0 { + reg = <0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&i2s2 { + #sound-dai-cells = <0>; + rockchip,bclk-fs = <128>; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + dr_mode = "otg"; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&pvtm { + status = "okay"; +}; + +&rng { + status = "okay"; +}; + +&pinctrl { + isp { + cif_clkout: cif-clkout { + rockchip,pins = + /*cif_clkout*/ + <2 RK_PB3 3 &pcfg_pull_none>; + }; + + isp_dvp_d0d7: isp-dvp-d0d7 { + rockchip,pins = + /*cif_data0*/ + <2 RK_PA0 3 &pcfg_pull_none>, + /*cif_data1*/ + <2 RK_PA1 3 &pcfg_pull_none>, + /*cif_data2*/ + <2 RK_PA2 3 &pcfg_pull_none>, + /*cif_data3*/ + <2 RK_PA3 3 &pcfg_pull_none>, + /*cif_data4*/ + <2 RK_PA4 3 &pcfg_pull_none>, + /*cif_data5*/ + <2 RK_PA5 3 &pcfg_pull_none>, + /*cif_data6*/ + <2 RK_PA6 3 &pcfg_pull_none>, + /*cif_data7*/ + <2 RK_PA7 3 &pcfg_pull_none>, + /*cif_sync*/ + <2 RK_PB0 3 &pcfg_pull_none>, + /*cif_href*/ + <2 RK_PB1 3 &pcfg_pull_none>, + /*cif_clkin*/ + <2 RK_PB2 3 &pcfg_pull_none>; + }; + }; + + buttons { + pwrbtn: pwrbtn { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-box-rev1.dts b/arch/arm64/boot/dts/rockchip/rk3399-box-rev1.dts new file mode 100755 index 000000000000..c4572da2ec87 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-box-rev1.dts @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3399-box.dtsi" + +/ { + model = "Rockchip RK3399 Board rev1 (BOX)"; + compatible = "rockchip-box-rev1","rockchip,rk3399-box"; +}; + +&pinctrl { + sdio0 { + sdio0_bus1: sdio0-bus1 { + rockchip,pins = + <2 RK_PC4 1 &pcfg_pull_up_20ma>; + }; + + sdio0_bus4: sdio0-bus4 { + rockchip,pins = + <2 RK_PC4 1 &pcfg_pull_up_20ma>, + <2 RK_PC5 1 &pcfg_pull_up_20ma>, + <2 RK_PC6 1 &pcfg_pull_up_20ma>, + <2 RK_PC7 1 &pcfg_pull_up_20ma>; + }; + + sdio0_cmd: sdio0-cmd { + rockchip,pins = + <2 RK_PD0 1 &pcfg_pull_up_20ma>; + }; + + sdio0_clk: sdio0-clk { + rockchip,pins = + <2 RK_PD1 1 &pcfg_pull_none_20ma>; + }; + }; + + sdmmc { + sdmmc_bus1: sdmmc-bus1 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_8ma>; + }; + + sdmmc_bus4: sdmmc-bus4 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_8ma>, + <4 RK_PB1 1 &pcfg_pull_up_8ma>, + <4 RK_PB2 1 &pcfg_pull_up_8ma>, + <4 RK_PB3 1 &pcfg_pull_up_8ma>; + }; + + sdmmc_clk: sdmmc-clk { + rockchip,pins = + <4 RK_PB4 1 &pcfg_pull_none_18ma>; + }; + + sdmmc_cmd: sdmmc-cmd { + rockchip,pins = + <4 RK_PB5 1 &pcfg_pull_up_8ma>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&i2c4 { + status = "okay"; + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; +}; + +&hdmi_in_vopl { + status = "disabled"; +}; + +&dp_in_vopb { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-box-rev2.dts b/arch/arm64/boot/dts/rockchip/rk3399-box-rev2.dts new file mode 100755 index 000000000000..1c9b884736b1 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-box-rev2.dts @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3399-box.dtsi" + +/ { + model = "Rockchip RK3399 Board rev2 (BOX)"; + compatible = "rockchip-box-rev2","rockchip,rk3399-box"; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&cpt_gpio>; + + sdio0 { + sdio0_bus1: sdio0-bus1 { + rockchip,pins = + <2 RK_PC4 1 &pcfg_pull_up_20ma>; + }; + + sdio0_bus4: sdio0-bus4 { + rockchip,pins = + <2 RK_PC4 1 &pcfg_pull_up_20ma>, + <2 RK_PC5 1 &pcfg_pull_up_20ma>, + <2 RK_PC6 1 &pcfg_pull_up_20ma>, + <2 RK_PC7 1 &pcfg_pull_up_20ma>; + }; + + sdio0_cmd: sdio0-cmd { + rockchip,pins = + <2 RK_PD0 1 &pcfg_pull_up_20ma>; + }; + + sdio0_clk: sdio0-clk { + rockchip,pins = + <2 RK_PD1 1 &pcfg_pull_none_20ma>; + }; + }; + + sdmmc { + sdmmc_bus1: sdmmc-bus1 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_8ma>; + }; + + sdmmc_bus4: sdmmc-bus4 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_8ma>, + <4 RK_PB1 1 &pcfg_pull_up_8ma>, + <4 RK_PB2 1 &pcfg_pull_up_8ma>, + <4 RK_PB3 1 &pcfg_pull_up_8ma>; + }; + + sdmmc_clk: sdmmc-clk { + rockchip,pins = + <4 RK_PB4 1 &pcfg_pull_none_18ma>; + }; + + sdmmc_cmd: sdmmc-cmd { + rockchip,pins = + <4 RK_PB5 1 &pcfg_pull_up_8ma>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + compat { + cpt_gpio: cpt-gpio { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_output_low>; + }; + }; +}; + +&i2c4 { + status = "okay"; + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; +}; + +&hdmi_in_vopl { + status = "disabled"; +}; + +&dp_in_vopb { + status = "disabled"; +}; + +&route_hdmi { + status = "okay"; +}; + +&hdmi { + status = "okay"; + rockchip,phy-table = + <74250000 0x8009 0x0004 0x0272>, + <165000000 0x802b 0x0004 0x0209>, + <297000000 0x8039 0x0005 0x028d>, + <594000000 0x8039 0x0000 0x019d>, + <000000000 0x0000 0x0000 0x0000>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-box.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-box.dtsi new file mode 100755 index 000000000000..704e0df4e122 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-box.dtsi @@ -0,0 +1,891 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include "rk3399.dtsi" +#include "rk3399-android.dtsi" +#include "rk3399-opp.dtsi" + +/ { + compatible = "rockchip,rk3399-box","rockchip,rk3399"; + + vcc1v8_s0: vcc1v8-s0 { + compatible = "regulator-fixed"; + regulator-name = "vcc1v8_s0"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + vin-supply = <&vcc_sys>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vdd_log: vdd-log { + compatible = "pwm-regulator"; + pwms = <&pwm2 0 25000 1>; + regulator-name = "vdd_log"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1400000>; + regulator-always-on; + regulator-boot-on; + + /* for rockchip boot on */ + rockchip,pwm_id= <2>; + rockchip,pwm_voltage = <900000>; + + vin-supply = <&vcc_sys>; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + spdif-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "okay"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + /* wifi-bt-power-toggle; */ + uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_gpios>; + /* BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; */ + BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + test-power { + status = "okay"; + }; +}; + +&hdmi_dp_sound { + status = "okay"; +}; + +&hdmi { + pinctrl-names = "default"; + pinctrl-0 = <&hdmi_i2c_xfer>, <&hdmi_cec>; +}; + +&sdmmc { + clock-frequency = <100000000>; + clock-freq-min-max = <100000 100000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + num-slots = <1>; + //sd-uhs-sdr104; + vqmmc-supply = <&vcc_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + card-detect-delay = <800>; + status = "okay"; +}; + +&sdio0 { + clock-frequency = <100000000>; + clock-freq-min-max = <200000 100000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&emmc_phy { + status = "okay"; +}; + +&sdhci { + bus-width = <8>; + mmc-hs400-1_8v; + no-sdio; + no-sd; + non-removable; + mmc-hs400-enhanced-strobe; + status = "okay"; +}; + +&i2s0 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <8>; + #sound-dai-cells = <0>; +}; + +&i2s2 { + #sound-dai-cells = <0>; +}; + +&spdif { + pinctrl-0 = <&spdif_bus_1>; + status = "okay"; + #sound-dai-cells = <0>; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <168>; + i2c-scl-falling-time-ns = <4>; + clock-frequency = <400000>; + + vdd_cpu_b: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_sys>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: syr828@41 { + compatible = "silergy,syr828"; + reg = <0x41>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_sys>; + regulator-initial-mode = <1>; /* 1:force PWM 2:auto */ + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc_sys>; + vcc10-supply = <&vcc_sys>; + vcc11-supply = <&vcc_sys>; + vcc12-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc_1v8>; + + regulators { + vdd_center: DCDC_REG1 { + regulator-name = "vdd_center"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-name = "vdd_cpu_l"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-name = "vcc_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-name = "vcc_1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc1v8_dvp: LDO_REG1 { + regulator-name = "vcc1v8_dvp"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcca1v8_hdmi: LDO_REG2 { + regulator-name = "vcca1v8_hdmi"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcca_1v8: LDO_REG3 { + regulator-name = "vcca_1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sd: LDO_REG4 { + regulator-name = "vcc_sd"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v0_sd: LDO_REG5 { + regulator-name = "vcc3v0_sd"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-name = "vcc_1v5"; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcca0v9_hdmi: LDO_REG7 { + regulator-name = "vcca0v9_hdmi"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vcc_3v0: LDO_REG8 { + regulator-name = "vcc_3v0"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v3_s3: SWITCH_REG1 { + regulator-name = "vcc3v3_s3"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_s0: SWITCH_REG2 { + regulator-name = "vcc3v3_s0"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + }; + }; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&threshold { + temperature = <85000>; +}; + +&target { + temperature = <100000>; +}; + +&soc_crit { + temperature = <115000>; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-mode = <1>; + /* tshut polarity 0:LOW 1:HIGH */ + rockchip,hw-tshut-polarity = <1>; + rockchip,hw-tshut-temp = <120000>; + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + dr_mode = "otg"; + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_1 { + dr_mode = "host"; + status = "okay"; +}; + +&pwm2 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin_pull_down>; +}; + +&pwm3 { + status = "okay"; + + interrupts = ; + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <1>; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_VOLUMEDOWN>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&rgmii_pins>; + pinctrl-1 = <&rgmii_sleep_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&saradc { + status = "okay"; +}; + +&i2s2 { + status = "okay"; +}; + +&io_domains { + status = "okay"; + + bt656-supply = <&vcc1v8_s0>; /* bt656_gpio2ab_ms */ + audio-supply = <&vcc1v8_s0>; /* audio_gpio3d4a_ms */ + sdmmc-supply = <&vcc_sd>; /* sdmmc_gpio4b_ms */ + gpio1830-supply = <&vcc_3v0>; /* gpio1833_gpio4cd_ms */ +}; + +&pinctrl { + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + gmac { + rgmii_sleep_pins: rgmii-sleep-pins { + rockchip,pins = + <3 RK_PB7 RK_FUNC_GPIO &pcfg_output_low>; + }; + }; +}; + +&pvtm { + status = "okay"; +}; + +&pmu_pvtm { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <0>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_PWM_WKUP_EN + ) + >; + rockchip,pwm-regulator-config = < + (0 + | PWM2_REGULATOR_EN + ) + >; + rockchip,power-ctrl = + <&gpio1 17 GPIO_ACTIVE_HIGH>, + <&gpio1 14 GPIO_ACTIVE_HIGH>; +}; + +&vopb { + assigned-clocks = <&cru DCLK_VOP0_DIV>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; + +&vopl { + assigned-clocks = <&cru DCLK_VOP1_DIV>; + assigned-clock-parents = <&cru PLL_CPLL>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-dram-default-timing.dtsi new file mode 100755 index 000000000000..981777b82329 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-dram-default-timing.dtsi @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include + +/ { + ddr_timing: ddr_timing { + compatible = "rockchip,ddr-timing"; + ddr3_speed_bin = <21>; + pd_idle = <0>; + sr_idle = <0>; + sr_mc_gate_idle = <0>; + srpd_lite_idle = <0>; + standby_idle = <0>; + auto_lp_dis_freq = <666>; + ddr3_dll_dis_freq = <300>; + phy_dll_dis_freq = <260>; + + ddr3_odt_dis_freq = <666>; + ddr3_drv = ; + ddr3_odt = ; + phy_ddr3_ca_drv = ; + phy_ddr3_dq_drv = ; + phy_ddr3_odt = ; + + lpddr3_odt_dis_freq = <666>; + lpddr3_drv = ; + lpddr3_odt = ; + phy_lpddr3_ca_drv = ; + phy_lpddr3_dq_drv = ; + phy_lpddr3_odt = ; + + lpddr4_odt_dis_freq = <800>; + lpddr4_drv = ; + lpddr4_dq_odt = ; + lpddr4_ca_odt = ; + phy_lpddr4_ca_drv = ; + phy_lpddr4_ck_cs_drv = ; + phy_lpddr4_dq_drv = ; + phy_lpddr4_odt = ; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-early-opp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-early-opp.dtsi new file mode 100755 index 000000000000..2a675ec7c342 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-early-opp.dtsi @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * NOTE: this file exists for the sake of early (pre-ES2) silicon. ES2 silicon + * will have different power characteristics. + */ + +/ { + /delete-node/ opp-table0; + /delete-node/ opp-table1; + /delete-node/ opp-table2; + + cluster0_opp: opp-table0 { + compatible = "operating-points-v2"; + opp-shared; + + opp-408000000 { + opp-hz = /bits/ 64 <408000000>; + opp-microvolt = <900000 900000 1200000>; + clock-latency-ns = <40000>; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <900000 900000 1200000>; + }; + opp-816000000 { + opp-hz = /bits/ 64 <816000000>; + opp-microvolt = <900000 900000 1200000>; + }; + opp-1008000000 { + opp-hz = /bits/ 64 <1008000000>; + opp-microvolt = <900000 900000 1200000>; + }; + opp-1200000000 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <950000 950000 1200000>; + }; + opp-1416000000 { + opp-hz = /bits/ 64 <1416000000>; + opp-microvolt = <1050000 1050000 1200000>; + }; + }; + + cluster1_opp: opp-table1 { + compatible = "operating-points-v2"; + opp-shared; + + opp-408000000 { + opp-hz = /bits/ 64 <408000000>; + opp-microvolt = <900000 900000 1200000>; + clock-latency-ns = <40000>; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <900000 900000 1200000>; + }; + opp-816000000 { + opp-hz = /bits/ 64 <816000000>; + opp-microvolt = <900000 900000 1200000>; + }; + opp-1008000000 { + opp-hz = /bits/ 64 <1008000000>; + opp-microvolt = <950000 950000 1200000>; + }; + opp-1200000000 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <1000000 1000000 1200000>; + }; + }; + + gpu_opp_table: opp-table2 { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <900000>; + }; + opp-297000000 { + opp-hz = /bits/ 64 <297000000>; + opp-microvolt = <900000>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <900000>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-cros.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb-cros.dtsi new file mode 100755 index 000000000000..4cac4981b267 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-cros.dtsi @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/ { + model = "Rockchip RK3399 Evaluation Board (Chrome OS)"; + compatible = "google,rk3399evb", "rockchip,rk3399-evb", "rockchip,rk3399"; + + edp_panel: edp-panel { + compatible = "lg,lp097qx1-spa1", "panel-simple"; + backlight = <&backlight>; + power-supply = <&vcc3v3_s0>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + + ports { + panel_in_edp: endpoint { + remote-endpoint = <&edp_out_panel>; + }; + }; + }; + + hdmi_codec: hdmi-codec { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "HDMI-CODEC"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + sound { + compatible = "rockchip,cdndp-sound"; + rockchip,cpu = <&i2s2>; + rockchip,codec = <&cdn_dp>; + }; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>, <&fusb1>; + + ports { + /* Don't use vopl for dp, save it for edp */ + dp_in: port { + /delete-node/ endpoint@1; + }; + }; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible ="boe,tv080wum-nl0"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&edp { + status = "disabled"; + + ports { + edp_out: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + edp_out_panel: endpoint@0 { + reg = <0>; + remote-endpoint = <&panel_in_edp>; + }; + }; + }; +}; + +&hdmi { + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + status = "disabled"; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; + + /* Don't use vopl for dp, save it for edp */ + vopl_out: port { + /delete-node/ endpoint@3; + }; +}; + +&vopl_mmu { + status = "okay"; +}; + +&display_subsystem { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android-avb.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android-avb.dts new file mode 100755 index 000000000000..343a9fc389e1 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android-avb.dts @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; + +#include "rk3399-evb-ind.dtsi" +#include "rk3399-android.dtsi" + +/ { + model = "Rockchip RK3399 EVB IND LPDDR4 Board edp (Android)"; + compatible = "rockchip,android", "rockchip,rk3399-evb-ind-lpddr4-android", "rockchip,rk3399"; + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff1a0000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init initrd=0x62000001,0x00800000 coherent_pool=1m"; + }; + + iram: sram@ff8d0000 { + compatible = "mmio-sram"; + reg = <0x0 0xff8d0000 0x0 0x20000>; + }; + + vcc_lcd: vcc-lcd { + compatible = "regulator-fixed"; + regulator-name = "vcc_lcd"; + startup-delay-us = <20000>; + enable-active-high; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc_lcd>; + reset-gpios = <&gpio1 23 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <20>; + reset-delay-ms = <20>; + enable-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + test-power { + status = "okay"; + }; +}; + +&backlight { + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; +}; + +&dmac_bus { + iram = <&iram>; + rockchip,force-iram; +}; + +&dp_sound { + status = "disabled"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopl { + status = "disabled"; +}; + +&i2c1 { + status = "okay"; + + sgm3784: sgm3784@30 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "sgmicro,gsm3784"; + reg = <0x30>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + //enable-gpio = <&gpio2 RK_PB4 GPIO_ACTIVE_HIGH>; + //strobe-gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; + status = "okay"; + sgm3784_led0: led@0 { + reg = <0x0>; + led-max-microamp = <299200>; + flash-max-microamp = <1122000>; + flash-max-timeout-us = <1600000>; + }; + + sgm3784_led1: led@1 { + reg = <0x1>; + led-max-microamp = <299200>; + flash-max-microamp = <1122000>; + flash-max-timeout-us = <1600000>; + }; + }; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + gc2145: gc2145@3c{ + status = "okay"; + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + power-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio2 12 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_out: endpoint { + remote-endpoint = <&dvp_in_fcam>; + }; + }; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "okay"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + /* reset-gpios = <>; */ + reset-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-CT0116"; + rockchip,camera-module-lens-name = "Largan-50013A1"; + lens-focus = <&vm149c>; + flash-leds = <&sgm3784_led0 &sgm3784_led1>; + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + //remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2>; + }; + }; + }; + + ov4689: ov4689@36 { + compatible = "ovti,ov4689"; + status = "disabled"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + /* reset-gpios = <>; */ + reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "JSD3425-C1"; + rockchip,camera-module-lens-name = "JSD3425-C1"; + port { + ucam_out1: endpoint { + //remote-endpoint = <&mipi_in_ucam0>; + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&hdmi_sound { + status = "okay"; +}; + +&route_edp { + status = "okay"; +}; + +&route_hdmi { + status = "okay"; + connect = <&vopl_out_hdmi>; +}; + +&i2s1 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&rk809_sound { + status = "okay"; +}; + +&hdmi_in_vopb { + status = "disabled"; +}; + +&rkisp1_0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rkisp1_1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout &isp_dvp_d0d7>; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + dvp_in_fcam: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc2145_out>; + }; + }; +}; + +/* + * if enable dp_sound, should disable spdif_sound and spdif_out + */ +&spdif_out { + status = "disabled"; +}; + +&spdif_sound { + status = "disabled"; +}; + +&i2s0 { + #sound-dai-cells = <0>; + status = "disabled"; +}; + +&tc358749x_sound { + status = "disabled"; +}; + +&pinctrl { + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android.dts new file mode 100755 index 000000000000..72aa97affa99 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-android.dts @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; + +#include "rk3399-evb-ind.dtsi" +#include "rk3399-android.dtsi" + +/ { + model = "Rockchip RK3399 EVB IND LPDDR4 Board edp (Android)"; + compatible = "rockchip,android", "rockchip,rk3399-evb-ind-lpddr4-android", "rockchip,rk3399"; + + iram: sram@ff8d0000 { + compatible = "mmio-sram"; + reg = <0x0 0xff8d0000 0x0 0x20000>; + }; + + vcc_lcd: vcc-lcd { + compatible = "regulator-fixed"; + regulator-name = "vcc_lcd"; + startup-delay-us = <20000>; + enable-active-high; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc_lcd>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio1 23 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + test-power { + status = "okay"; + }; +}; + +&backlight { + enable-gpios = <&gpio1 18 GPIO_ACTIVE_HIGH>; +}; + +&dmac_bus { + iram = <&iram>; + rockchip,force-iram; +}; + +&dp_sound { + status = "disabled"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopl { + status = "disabled"; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&hdmi_sound { + status = "okay"; +}; + +&route_edp { + status = "okay"; +}; + +&i2s1 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&rk809_sound { + status = "okay"; +}; + +&hdmi_in_vopb { + status = "disabled"; +}; + +/* + * if enable dp_sound, should disable spdif_sound and spdif_out + */ +&spdif_out { + status = "disabled"; +}; + +&spdif_sound { + status = "disabled"; +}; + +&i2s0 { + #sound-dai-cells = <0>; + status = "disabled"; +}; + +&tc358749x_sound { + status = "disabled"; +}; + +&pinctrl { + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-linux.dts new file mode 100755 index 000000000000..5b6b7131fdd1 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-linux.dts @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019-2020 Fuzhou Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; + +#include "rk3399-evb-ind.dtsi" +#include "rk3399-linux.dtsi" + +/ { + model = "Rockchip RK3399 EVB IND LPDDR4 Board edp (Linux)"; + compatible = "rockchip,linux", "rockchip,rk3399-evb-ind-lpddr4-linux", "rockchip,rk3399"; + + iram: sram@ff8d0000 { + compatible = "mmio-sram"; + reg = <0x0 0xff8d0000 0x0 0x20000>; + }; + + vcc_lcd: vcc-lcd { + compatible = "regulator-fixed"; + regulator-name = "vcc_lcd"; + startup-delay-us = <20000>; + enable-active-high; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc_lcd>; + reset-gpios = <&gpio1 23 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + test-power { + status = "okay"; + }; +}; + +&backlight { + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; +}; + +&display_subsystem { + status = "okay"; +}; + +&dmac_bus { + iram = <&iram>; + rockchip,force-iram; +}; + +&dp_sound { + status = "disabled"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopb { + status = "disabled"; +}; + +&hdmi { + pinctrl-names = "default"; + pinctrl-0 = <&hdmi_i2c_xfer>, <&hdmi_cec>; + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + status = "okay"; +}; + +&hdmi_in_vopl { + status = "disabled"; +}; + +&i2c1 { + status = "okay"; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "okay"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 RK_PD0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-CT0116"; + rockchip,camera-module-lens-name = "Largan-50013A1"; + + lens-focus = <&vm149c>; + + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s0 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&i2s1 { + #sound-dai-cells = <0>; + status = "disabled"; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&hdmi_sound { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + /* Unlinked camera */ + //remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&rk809_sound { + status = "okay"; +}; + +&rkisp1_0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rkisp1_1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + }; +}; + +&route_edp { + status = "okay"; +}; + +/* + * if enable dp_sound, should disable spdif_sound and spdif_out + */ +&spdif_out { + status = "disabled"; +}; + +&spdif_sound { + status = "disabled"; +}; + +&tc358749x_sound { + status = "disabled"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&pinctrl { + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-v13-android-avb.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-v13-android-avb.dts new file mode 100755 index 000000000000..8439d91dd125 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind-lpddr4-v13-android-avb.dts @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; + +#include "rk3399-evb-ind.dtsi" +#include "rk3399-android.dtsi" + +/ { + model = "Rockchip RK3399 EVB IND LPDDR4 Board edp (Android)"; + compatible = "rockchip,android", "rockchip,rk3399-evb-ind-v13-lpddr4-android", "rockchip,rk3399"; + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff1a0000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init initrd=0x62000001,0x00800000 coherent_pool=1m"; + }; + + iram: sram@ff8d0000 { + compatible = "mmio-sram"; + reg = <0x0 0xff8d0000 0x0 0x20000>; + }; + + hub_reset: hub_reset { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio4 18 GPIO_ACTIVE_HIGH>; + regulator-name = "hub_reset"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + vcc_lcd: vcc-lcd { + compatible = "regulator-fixed"; + regulator-name = "vcc_lcd"; + startup-delay-us = <20000>; + enable-active-high; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc_lcd>; + reset-gpios = <&gpio1 23 GPIO_ACTIVE_LOW>; + prepare-delay-ms = <20>; + reset-delay-ms = <20>; + enable-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + test-power { + status = "okay"; + }; +}; + +&backlight { + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; +}; + +&dmac_bus { + iram = <&iram>; + rockchip,force-iram; +}; + +&dp_sound { + status = "disabled"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopl { + status = "disabled"; +}; + +&i2c1 { + status = "okay"; + + sgm3784: sgm3784@30 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "sgmicro,gsm3784"; + reg = <0x30>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + //enable-gpio = <&gpio2 RK_PB4 GPIO_ACTIVE_HIGH>; + //strobe-gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; + status = "okay"; + sgm3784_led0: led@0 { + reg = <0x0>; + led-max-microamp = <299200>; + flash-max-microamp = <1122000>; + flash-max-timeout-us = <1600000>; + }; + + sgm3784_led1: led@1 { + reg = <0x1>; + led-max-microamp = <299200>; + flash-max-microamp = <1122000>; + flash-max-timeout-us = <1600000>; + }; + }; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + gc2145: gc2145@3c{ + status = "okay"; + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + power-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio2 12 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_out: endpoint { + remote-endpoint = <&dvp_in_fcam>; + }; + }; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "okay"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + /* reset-gpios = <>; */ + reset-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-CT0116"; + rockchip,camera-module-lens-name = "Largan-50013A1"; + lens-focus = <&vm149c>; + flash-leds = <&sgm3784_led0 &sgm3784_led1>; + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + //remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2>; + }; + }; + }; + + ov4689: ov4689@36 { + compatible = "ovti,ov4689"; + status = "disabled"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + /* reset-gpios = <>; */ + reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "JSD3425-C1"; + rockchip,camera-module-lens-name = "JSD3425-C1"; + port { + ucam_out1: endpoint { + //remote-endpoint = <&mipi_in_ucam0>; + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&hdmi_sound { + status = "okay"; +}; + +&route_edp { + status = "okay"; +}; + +&route_hdmi { + status = "okay"; + connect = <&vopl_out_hdmi>; +}; + +&i2s1 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&rk809_sound { + status = "okay"; +}; + +&hdmi_in_vopb { + status = "disabled"; +}; + +&rkisp1_0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rkisp1_1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout &isp_dvp_d0d7>; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + dvp_in_fcam: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc2145_out>; + }; + }; +}; + + +&vcca_0v9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vcca_0v9"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; +}; + +&vcc0v9_soc { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-name = "vcc0v9_soc"; + regulator-state-mem { + regulator-off-in-suspend; + }; +}; + +/* + * if enable dp_sound, should disable spdif_sound and spdif_out + */ +&spdif_out { + status = "disabled"; +}; + +&spdif_sound { + status = "disabled"; +}; + +&i2s0 { + #sound-dai-cells = <0>; + status = "disabled"; +}; + +&tc358749x_sound { + status = "disabled"; +}; + +&pinctrl { + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-ind.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind.dtsi new file mode 100755 index 000000000000..dc821a2d45c4 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-ind.dtsi @@ -0,0 +1,1430 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include "dt-bindings/pwm/pwm.h" +#include "rk3399.dtsi" +#include "rk3399-opp.dtsi" +#include +#include "rk3399-vop-clk-set.dtsi" +#include + +/ { + compatible = "rockchip,rk3399-evb-ind", "rockchip,rk3399"; + + adc_keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1000>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <170000>; + }; + }; + + backlight: backlight { + status = "okay"; + compatible = "pwm-backlight"; + pwms = <&pwm2 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + dw_hdmi_audio: dw-hdmi-audio { + status = "disabled"; + compatible = "rockchip,dw-hdmi-audio"; + #sound-dai-cells = <0>; + }; + + rk809_sound: rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + dp_sound: dp-sound { + status = "disabled"; + compatible = "rockchip,cdndp-sound"; + rockchip,cpu = <&spdif>; + rockchip,codec = <&cdn_dp 1>; + }; + + hdmi_sound: hdmi-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,hdmi"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + spdif_sound: spdif-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "okay"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>, <&bt_reset_gpio>, <&bt_wake_gpio>, <&bt_irq_gpio>; + pinctrl-1 = <&uart0_gpios>; + //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ + BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ + BT,wake_gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ + BT,wake_host_irq = <&gpio2 27 GPIO_ACTIVE_HIGH>; /* GPIO2_D3 */ + status = "okay"; + }; + + rk_modem: rk-modem { + compatible="4g-modem-platdata"; + pinctrl-names = "default"; + pinctrl-0 = <<e_vbat <e_power_en <e_reset>; + 4G,vbat-gpio = <&gpio4 RK_PD0 GPIO_ACTIVE_HIGH>; + 4G,power-gpio = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; + 4G,reset-gpio = <&gpio4 RK_PD4 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + + tc358749x_sound:tc358749x-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rk,hdmiin-tc358749x-codec"; + simple-audio-card,bitclock-master = <&sound0_master>; + simple-audio-card,frame-master = <&sound0_master>; + simple-audio-card,cpu { + sound-dai = <&i2s0>; + }; + sound0_master: simple-audio-card,codec { + sound-dai = <&tc358749x>; + }; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 8 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vcc5v0_usbnet: vcc5v0-usbnet { + compatible = "regulator-fixed"; + enable-active-high; + /*disabled r8152 usb net default*/ + //regulator-always-on; + //regulator-boot-on; + gpio = <&gpio4 18 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usbnet_pwr_drv>; + regulator-name = "vcc5v0_usbnet"; + startup-delay-us = <20000>; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + status = "okay"; + }; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio4 26 GPIO_ACTIVE_HIGH>; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vdd_log: vdd-log { + compatible = "regulator-fixed"; + regulator-name = "vdd_log"; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <950000>; + regulator-always-on; + regulator-boot-on; + }; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&dp_in_vopb { + status = "disabled"; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; + upthreshold = <40>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 856000 + SYS_STATUS_REBOOT 856000 + SYS_STATUS_SUSPEND 328000 + SYS_STATUS_VIDEO_1080P 666000 + SYS_STATUS_VIDEO_4K 856000 + SYS_STATUS_VIDEO_4K_10B 856000 + SYS_STATUS_PERFORMANCE 856000 + SYS_STATUS_BOOST 856000 + SYS_STATUS_DUALVIEW 856000 + SYS_STATUS_ISP 856000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 762 416000 + 763 3012 666000 + 3013 99999 856000 + >; + + vop-pn-msch-readlatency = < + 0 0x20 + 4 0x20 + >; + + auto-min-freq = <328000>; + auto-freq-en = <0>; +}; + +&dmc_opp_table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-328000000 { + opp-hz = /bits/ 64 <328000000>; + opp-microvolt = <900000>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-416000000 { + opp-hz = /bits/ 64 <416000000>; + opp-microvolt = <900000>; + }; + opp-528000000 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-666000000 { + opp-hz = /bits/ 64 <666000000>; + opp-microvolt = <900000>; + }; + opp-800000000 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-856000000 { + opp-hz = /bits/ 64 <856000000>; + opp-microvolt = <900000>; + }; + opp-928000000 { + opp-hz = /bits/ 64 <928000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; +}; + +&emmc_phy { + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 150000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + tx_delay = <0x22>; + rx_delay = <0x08>; + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&hdmi { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <168>; + i2c-scl-falling-time-ns = <4>; + clock-frequency = <400000>; + + vdd_cpu_b: tcs452x@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: tcs452x@10 { + compatible = "tcs,tcs452x"; + reg = <0x10>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int_l>; + pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>,<&rk809_slppin_null>; + rockchip,system-power-controller; + #clock-cells = <1>; + pmic-reset-func = <0>; + wakeup-source; + clock-output-names = "xin32k", "rk808-clkout2"; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc_buck5>; + vcc6-supply = <&vcc_buck5>; + vcc7-supply = <&vcc5v0_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + rtc { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk809_slppin_null: rk809_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk809_slppin_slp: rk809_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk809_slppin_pwrdn: rk809_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk809_slppin_rst: rk809_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_center: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_center"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_sys: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_buck5: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2200000>; + regulator-max-microvolt = <2200000>; + regulator-name = "vcc_buck5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <2200000>; + }; + }; + + vcca_0v9: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vcca_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc0v9_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-name = "vcc0v9_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vcca_1v8: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcca_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd1v5_dvp: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc5v0_usb: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_usb"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vccio_3v3: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vccio_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + /* + battery { + compatible = "rk817,battery"; + ocv_table = <7000 7250 7370 7384 7436 7470 7496 + 7520 7548 7576 7604 7632 7668 7706 + 7754 7816 7892 7950 8036 8142 8212>; + design_capacity = <2500>; + design_qmax = <2750>; + bat_res = <100>; + sleep_enter_current = <300>; + sleep_exit_current = <300>; + sleep_filter_current = <100>; + power_off_thresd = <7000>; + zero_algorithm_vol = <7700>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + bat_res_up = <140>; + bat_res_down = <20>; + }; + */ + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + assigned-clocks = <&cru SCLK_I2SOUT_SRC>; + assigned-clock-parents = <&cru SCLK_I2S1_8CH>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <300>; + i2c-scl-falling-time-ns = <15>; + clock-frequency = <400000>; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio1 1 GPIO_ACTIVE_HIGH>; + }; + + tc358749x: tc358749x@0f { + #sound-dai-cells = <0>; + compatible = "toshiba,tc358749x"; + reg = <0x0f>; + power-gpios = <&gpio0 13 GPIO_ACTIVE_HIGH>; + stanby-gpios = <&gpio3 25 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 2 GPIO_ACTIVE_HIGH>; + int-gpios = <&gpio4 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmiin_gpios>; + status = "okay"; + }; +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <475>; + i2c-scl-falling-time-ns = <26>; + + mpu6500@68 { + status = "disabled"; + compatible = "invensense,mpu6500"; + reg = <0x68>; + irq-gpio = <&gpio1 22 IRQ_TYPE_EDGE_RISING>; + pinctrl-names = "default"; + pinctrl-0 = <&mpu6500_irq_gpio>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <0 1 0 1 0 0 0 0 1>; + orientation-x= <0>; + orientation-y= <0>; + orientation-z= <0>; + mpu-debug = <1>; + }; + + mpu6500_acc: mpu_acc@68 { + compatible = "mpu6500_acc"; + reg = <0x68>; + irq-gpio = <&gpio1 22 IRQ_TYPE_EDGE_RISING>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + layout = <8>; + }; + + mpu6500_gyro: mpu_gyro@68 { + compatible = "mpu6500_gyro"; + reg = <0x68>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + layout = <8>; + }; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; + vbus-5v-gpios = <&gpio1 18 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + sensor@0d { + status = "okay"; + compatible = "ak8963"; + pinctrl-names = "default"; + pinctrl-0 = <&ak8963_irq_gpio>; + reg = <0x0d>; + type = ; + irq-gpio = <&gpio1 0 IRQ_TYPE_EDGE_RISING>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <6>; + }; + + bq25700: bq25700@6b {//6a + compatible = "ti,bq25703"; + reg = <0x6b>; + extcon = <&fusb0>; + + interrupt-parent = <&gpio0>; + interrupts = <5 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&charger_ok>; + ti,charge-current = <2500000>; + ti,max-input-voltage = <20000000>; + ti,max-input-current = <6000000>; + ti,max-charge-voltage = <8750000>; + ti,input-current = <500000>; + ti,input-current-sdp = <500000>; + ti,input-current-dcp = <2000000>; + ti,input-current-cdp = <2000000>; + ti,minimum-sys-voltage = <7400000>; + ti,otg-voltage = <5000000>; + ti,otg-current = <500000>; + pd-charge-only = <0>; + status = "disabled"; + }; +}; + +&i2c6 { + cw2015@62 { + status = "disabled"; + compatible = "cw201x"; + reg = <0x62>; + bat_config_info = <0x15 0x42 0x60 0x59 0x52 0x58 0x4D 0x48 + 0x48 0x44 0x44 0x46 0x49 0x48 0x32 0x24 + 0x20 0x17 0x13 0x0F 0x19 0x3E 0x51 0x45 + 0x08 0x76 0x0B 0x85 0x0E 0x1C 0x2E 0x3E + 0x4D 0x52 0x52 0x57 0x3D 0x1B 0x6A 0x2D + 0x25 0x43 0x52 0x87 0x8F 0x91 0x94 0x52 + 0x82 0x8C 0x92 0x96 0xFF 0x7B 0xBB 0xCB + 0x2F 0x7D 0x72 0xA5 0xB5 0xC1 0x46 0xAE>; + monitor_sec = <5>; + virtual_power = <0>; + }; +}; + +&i2s0 { + status = "disabled"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <8>; + #sound-dai-cells = <0>; +}; + +&i2s1 { + #sound-dai-cells = <0>; + status = "disabled"; +}; + +&i2s2 { + #sound-dai-cells = <0>; + dmas = <&dmac_bus 4>; + dma-names = "tx"; + status = "disabled"; +}; + +&io_domains { + status = "okay"; + + bt656-supply = <&vcc_3v0>; /* bt656_gpio2ab_ms */ + audio-supply = <&vcca_1v8>; /* audio_gpio3d4a_ms */ + sdmmc-supply = <&vccio_sd>; /* sdmmc_gpio4b_ms */ + gpio1830-supply = <&vcc_3v0>; /* gpio1833_gpio4cd_ms */ +}; + +&pmu_io_domains { + pmu1830-supply = <&vcc_1v8>; + status = "okay"; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + ep-gpios = <&gpio2 RK_PA4 GPIO_ACTIVE_HIGH>; + num-lanes = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_clkreqn_cpm>; + status = "okay"; +}; + +&pwm0 { + status = "disabled"; +}; + +&pwm2 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin_pull_down>; +}; + +&pwm3 { + status = "okay"; + + interrupts = ; + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <4>; + pinctrl-names = "default"; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_VOLUMEDOWN>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_PWM_WKUP_EN + ) + >; + rockchip,pwm-regulator-config = < + (0 + | PWM2_REGULATOR_EN + ) + >; + rockchip,power-ctrl = + <&gpio1 17 GPIO_ACTIVE_HIGH>, + <&gpio1 14 GPIO_ACTIVE_HIGH>; +}; + +&saradc { + status = "okay"; +}; + +&spdif { + status = "okay"; + pinctrl-0 = <&spdif_bus>; + i2c-scl-rising-time-ns = <450>; + i2c-scl-falling-time-ns = <15>; + #sound-dai-cells = <0>; +}; + +&sdio0 { + clock-frequency = <150000000>; + clock-freq-min-max = <200000 150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc { + clock-frequency = <150000000>; + clock-freq-min-max = <100000 150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + num-slots = <1>; + sd-uhs-sdr104; + vmmc-supply = <&vcc_sd>; + vqmmc-supply = <&vccio_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "okay"; +}; + +&sdhci { + bus-width = <8>; + mmc-hs400-1_8v; + no-sdio; + no-sd; + non-removable; + keep-power-in-suspend; + mmc-hs400-enhanced-strobe; + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-mode = <1>; + /* tshut polarity 0:LOW 1:HIGH */ + rockchip,hw-tshut-polarity = <1>; + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&uart2 { + status = "disabled"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + status = "okay"; + }; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd_dwc3_1 { + status = "okay"; + dr_mode = "host"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&vopb { + assigned-clocks = <&cru DCLK_VOP0_DIV>; + assigned-clock-parents = <&cru PLL_CPLL>; +}; + +&vopl { + assigned-clocks = <&cru DCLK_VOP1_DIV>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; + +&pinctrl { + + ak8963 { + ak8963_irq_gpio: ak8963-irq-gpio { + rockchip,pins = <1 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + charger { + charger_ok: charge-ok { + rockchip,pins = + <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + i2s0 { + i2s0_8ch_bus: i2s0-8ch-bus { + rockchip,pins = + <3 RK_PD0 1 &pcfg_pull_none>, + <3 RK_PD2 1 &pcfg_pull_none>, + <3 RK_PD3 1 &pcfg_pull_none>, + <3 RK_PD4 1 &pcfg_pull_none>, + <3 RK_PD5 1 &pcfg_pull_none>, + <3 RK_PD6 1 &pcfg_pull_none>, + <3 RK_PD7 1 &pcfg_pull_none>; + }; + + i2s_8ch_mclk: i2s-8ch-mclk { + rockchip,pins = <4 RK_PA0 1 &pcfg_pull_none>; + }; + }; + + i2s1 { + i2s1_2ch_bus: i2s1-2ch-bus { + rockchip,pins = + <4 RK_PA3 1 &pcfg_pull_none>, + <4 RK_PA5 1 &pcfg_pull_none>, + <4 RK_PA6 1 &pcfg_pull_none>, + <4 RK_PA7 1 &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = + <0 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + hdmiin { + hdmiin_gpios: hdmiin_gpios { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>, + <3 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>, + <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>, + <4 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>, + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + mpu6500 { + mpu6500_irq_gpio: mpu6500-irq-gpio { + rockchip,pins = <1 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + soc_slppin_gpio: soc-slppin-gpio { + rockchip,pins = + <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc-slppin-slp { + rockchip,pins = + <1 RK_PA5 1 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_reset_gpio: bt-reset-gpio { + rockchip,pins = + <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_wake_gpio: bt-wake-gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_irq_gpio: bt-irq-gpio { + rockchip,pins = + <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + rk-modem { + lte_vbat: lte-vbat { + rockchip,pins = <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lte_power_en: lte-power-en { + rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lte_reset: lte-reset { + rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + vcc_sd { + vcc_sd_h: vcc-sd-h { + rockchip,pins = + <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usbnet { + usbnet_pwr_drv: usbnet-pwr-drv { + rockchip,pins = + <4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-android.dts new file mode 100755 index 000000000000..bb7ddac77f23 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-android.dts @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "rk3399-evb-rev1.dtsi" +#include "rk3399-android.dtsi" + +/ { + model = "Rockchip RK3399 Evaluation Board v1 (Android)"; + compatible = "rockchip,android", "rockchip,rk3399-evb-rev1", "rockchip,rk3399"; +}; + +&vdd_log { + rockchip,pwm_id= <2>; + rockchip,pwm_voltage = <900000>; +}; + +&vdd_center { + rockchip,pwm_id= <3>; + rockchip,pwm_voltage = <900000>; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible ="simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <160000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <21>; + hfront-porch = <120>; + vback-porch = <18>; + vfront-porch = <21>; + hsync-len = <20>; + vsync-len = <3>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&route_dsi { + status = "okay"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-cros.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-cros.dts new file mode 100755 index 000000000000..bf27556295ec --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1-cros.dts @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "rk3399-evb-rev1.dtsi" +#include "rk3399-evb-cros.dtsi" + +/ { + model = "Rockchip RK3399 Evaluation Board v1 (Chrome OS)"; + compatible = "google,rk3399evb-rev1", "rockchip,rk3399-evb-rev1", "rockchip,rk3399"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1.dtsi new file mode 100755 index 000000000000..f35d6ee56072 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev1.dtsi @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "rk3399-evb.dtsi" +#include "rk3399-early-opp.dtsi" + +/ { + compatible = "rockchip,rk3399-evb-rev1", "rockchip,rk3399"; + + vdd_log: vdd-log { + compatible = "pwm-regulator"; + rockchip,pwm_id = <2>; + rockchip,pwm_voltage = <900000>; + pwms = <&pwm2 0 25000 1>; + regulator-name = "vdd_log"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1400000>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_center: vdd-center { + compatible = "pwm-regulator"; + rockchip,pwm_id = <3>; + rockchip,pwm_voltage = <900000>; + pwms = <&pwm3 0 25000 1>; + regulator-name = "vdd_center"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1400000>; + regulator-always-on; + regulator-boot-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; +}; + +&cpu_l0 { + dynamic-power-coefficient = <121>; +}; + +&cpu_b0 { + dynamic-power-coefficient = <1068>; +}; + +&soc_thermal { + sustainable-power = <1600>; /* milliwatts */ + + cooling-maps { + map0 { + trip = <&target>; + cooling-device = + <&cpu_l0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <10240>; + }; + map1 { + trip = <&target>; + cooling-device = + <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + map2 { + trip = <&target>; + cooling-device = + <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <10240>; + }; + }; +}; + +&gpu_power_model { + dynamic-power = <1780>; +}; + +&i2c0 { + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + mp8865: mp8865@68 { + compatible = "mps,mp8865"; + reg = <0x68>; + regulators { + vdd_gpu: mp8865_dcdc1 { + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <8000>; + regulator-always-on; + regulator-boot-on; + }; + }; + }; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l &pmic_dvs2>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + vcc10-supply = <&vcc3v3_sys>; + vcc11-supply = <&vcc3v3_sys>; + vcc12-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc1v8_pmu>; + + regulators { + vdd_cpu_b: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_cpu_b"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc1v8_dvp: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc3v0_tp: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc3v0_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcca3v0_codec: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcca3v0_codec"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcca1v8_codec: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_codec"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_3v0: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v3_s3: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s3"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_s0: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s0"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + }; + }; +}; + +&i2c4 { + fusb1: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb1_int>; + vbus-5v-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + }; +}; + +&pwm2 { + status = "okay"; +}; + +&pwm3 { + status = "okay"; +}; + +&u2phy0_otg { + rockchip,utmi-avalid; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-android.dts new file mode 100755 index 000000000000..b15fb8a9d088 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-android.dts @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "rk3399-evb-rev2.dtsi" +#include "rk3399-android.dtsi" + +/ { + model = "Rockchip RK3399 Evaluation Board v2 (Android)"; + compatible = "rockchip,android", "rockchip,rk3399-evb-rev2", "rockchip,rk3399"; +}; + +&vdd_center { + rockchip,pwm_id= <3>; + rockchip,pwm_voltage = <900000>; +}; + +&i2s2 { + status = "okay"; +}; + +&spdif { + status = "okay"; +}; + +&spdif_out { + status = "okay"; +}; + +&spdif_sound { + status = "okay"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible ="simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <160000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <21>; + hfront-porch = <120>; + vback-porch = <18>; + vfront-porch = <21>; + hsync-len = <20>; + vsync-len = <3>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&route_dsi { + status = "okay"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-cros.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-cros.dts new file mode 100755 index 000000000000..2daf9f1235f3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2-cros.dts @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "rk3399-evb-rev2.dtsi" +#include "rk3399-evb-cros.dtsi" + +/ { + model = "Rockchip RK3399 Evaluation Board v2 (Chrome OS)"; + compatible = "google,rk3399evb-rev2", "rockchip,rk3399-evb-rev2", "rockchip,rk3399"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2.dtsi new file mode 100755 index 000000000000..4b23c1fcadd8 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev2.dtsi @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "rk3399-evb.dtsi" +#include "rk3399-early-opp.dtsi" + +/ { + compatible = "rockchip,rk3399-evb-rev2", "rockchip,rk3399"; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + vdd_center: vdd-center { + compatible = "pwm-regulator"; + rockchip,pwm_id = <3>; + rockchip,pwm_voltage = <900000>; + pwms = <&pwm3 0 25000 1>; + regulator-name = "vdd_center"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1400000>; + regulator-always-on; + regulator-boot-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; +}; + +&cpu_l0 { + dynamic-power-coefficient = <121>; +}; + +&cpu_b0 { + dynamic-power-coefficient = <1068>; +}; + +&soc_thermal { + sustainable-power = <1600>; /* milliwatts */ + + cooling-maps { + map0 { + trip = <&target>; + cooling-device = + <&cpu_l0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <10240>; + }; + map1 { + trip = <&target>; + cooling-device = + <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + map2 { + trip = <&target>; + cooling-device = + <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <10240>; + }; + }; +}; + +&gpu_power_model { + dynamic-power = <1780>; +}; + +&i2c0 { + fusb1: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb1_int>; + vbus-5v-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + vdd_cpu_b: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + lp8752: lp8752@60 { + compatible = "ti,lp8752"; + reg = <0x60>; + vin0-supply = <&vcc5v0_sys>; + regulators { + vdd_gpu: lp8752_buck0 { + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <735000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <6000>; + regulator-always-on; + regulator-boot-on; + }; + }; + }; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l &pmic_dvs2>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + vcc10-supply = <&vcc3v3_sys>; + vcc11-supply = <&vcc3v3_sys>; + vcc12-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc1v8_pmu>; + + regulators { + vdd_log: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_log"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc1v8_dvp: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v0_tp: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc3v0_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcca3v0_codec: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcca3v0_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcca1v8_codec: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v3_s3: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_s0: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&pwm3 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm3a_pin_pull_down>; +}; + +&u2phy0_otg { + rockchip,utmi-avalid; +}; + +&i2c6 { + status = "okay"; + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-edp.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-edp.dts new file mode 100755 index 000000000000..7f730f5633dd --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-edp.dts @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "rk3399-evb-rev3.dtsi" +#include "rk3399-android.dtsi" + +/ { + model = "Rockchip RK3399 Evaluation Board v3 edp (Android)"; + compatible = "rockchip,android", "rockchip,rk3399-evb-rev3", "rockchip,rk3399"; + + edp_panel: edp-panel { + compatible = "lg,lp079qx1-sp0v", "panel-simple"; + backlight = <&backlight>; + power-supply = <&vcc3v3_s0>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + + ports { + panel_in_edp: endpoint { + remote-endpoint = <&edp_out_panel>; + }; + }; + }; +}; + +&i2s2 { + status = "okay"; +}; + +>9xx { + status = "disabled"; +}; + +&i2c4 { + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; + }; +}; + +&edp { + force-hpd; + status = "okay"; + + ports { + edp_out: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + edp_out_panel: endpoint@0 { + reg = <0>; + remote-endpoint = <&panel_in_edp>; + }; + }; + }; +}; + +&route_edp { + status = "okay"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-lp4.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-lp4.dts new file mode 100755 index 000000000000..85f4356a0030 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-lp4.dts @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; + +#include "rk3399-evb-rev3.dtsi" +#include "rk3399-android.dtsi" + +/ { + model = "Rockchip RK3399 Evaluation Board v3 (Android) LPDDR4"; + compatible = "rockchip,android", "rockchip,rk3399-evb-rev3-android-lp4", "rockchip,rk3399"; + + /* first 64k(0xff8c0000~0xff8d0000) for ddr and suspend */ + iram: sram@ff8d0000 { + compatible = "mmio-sram"; + reg = <0x0 0xff8d0000 0x0 0x20000>; /* 128k */ + }; +}; + +&dmac_bus { + iram = <&iram>; + rockchip,force-iram; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible ="simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <160000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <21>; + hfront-porch = <120>; + vback-porch = <18>; + vfront-porch = <21>; + hsync-len = <20>; + vsync-len = <3>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + dsp_lut: dsp-lut { + gamma-lut = < + 0x00000000 0x00010101 0x00020202 0x00030303 0x00040404 0x00050505 0x00060606 0x00070707 + 0x00080808 0x00090909 0x000a0a0a 0x000b0b0b 0x000c0c0c 0x000d0d0d 0x000e0e0e 0x000f0f0f + 0x00101010 0x00111111 0x00121212 0x00131313 0x00141414 0x00151515 0x00161616 0x00171717 + 0x00181818 0x00191919 0x001a1a1a 0x001b1b1b 0x001c1c1c 0x001d1d1d 0x001e1e1e 0x001f1f1f + 0x00202020 0x00212121 0x00222222 0x00232323 0x00242424 0x00252525 0x00262626 0x00272727 + 0x00282828 0x00292929 0x002a2a2a 0x002b2b2b 0x002c2c2c 0x002d2d2d 0x002e2e2e 0x002f2f2f + 0x00303030 0x00313131 0x00323232 0x00333333 0x00343434 0x00353535 0x00363636 0x00373737 + 0x00383838 0x00393939 0x003a3a3a 0x003b3b3b 0x003c3c3c 0x003d3d3d 0x003e3e3e 0x003f3f3f + 0x00404040 0x00414141 0x00424242 0x00434343 0x00444444 0x00454545 0x00464646 0x00474747 + 0x00484848 0x00494949 0x004a4a4a 0x004b4b4b 0x004c4c4c 0x004d4d4d 0x004e4e4e 0x004f4f4f + 0x00505050 0x00515151 0x00525252 0x00535353 0x00545454 0x00555555 0x00565656 0x00575757 + 0x00585858 0x00595959 0x005a5a5a 0x005b5b5b 0x005c5c5c 0x005d5d5d 0x005e5e5e 0x005f5f5f + 0x00606060 0x00616161 0x00626262 0x00636363 0x00646464 0x00656565 0x00666666 0x00676767 + 0x00686868 0x00696969 0x006a6a6a 0x006b6b6b 0x006c6c6c 0x006d6d6d 0x006e6e6e 0x006f6f6f + 0x00707070 0x00717171 0x00727272 0x00737373 0x00747474 0x00757575 0x00767676 0x00777777 + 0x00787878 0x00797979 0x007a7a7a 0x007b7b7b 0x007c7c7c 0x007d7d7d 0x007e7e7e 0x007f7f7f + 0x00808080 0x00818181 0x00828282 0x00838383 0x00848484 0x00858585 0x00868686 0x00878787 + 0x00888888 0x00898989 0x008a8a8a 0x008b8b8b 0x008c8c8c 0x008d8d8d 0x008e8e8e 0x008f8f8f + 0x00909090 0x00919191 0x00929292 0x00939393 0x00949494 0x00959595 0x00969696 0x00979797 + 0x00989898 0x00999999 0x009a9a9a 0x009b9b9b 0x009c9c9c 0x009d9d9d 0x009e9e9e 0x009f9f9f + 0x00a0a0a0 0x00a1a1a1 0x00a2a2a2 0x00a3a3a3 0x00a4a4a4 0x00a5a5a5 0x00a6a6a6 0x00a7a7a7 + 0x00a8a8a8 0x00a9a9a9 0x00aaaaaa 0x00ababab 0x00acacac 0x00adadad 0x00aeaeae 0x00afafaf + 0x00b0b0b0 0x00b1b1b1 0x00b2b2b2 0x00b3b3b3 0x00b4b4b4 0x00b5b5b5 0x00b6b6b6 0x00b7b7b7 + 0x00b8b8b8 0x00b9b9b9 0x00bababa 0x00bbbbbb 0x00bcbcbc 0x00bdbdbd 0x00bebebe 0x00bfbfbf + 0x00c0c0c0 0x00c1c1c1 0x00c2c2c2 0x00c3c3c3 0x00c4c4c4 0x00c5c5c5 0x00c6c6c6 0x00c7c7c7 + 0x00c8c8c8 0x00c9c9c9 0x00cacaca 0x00cbcbcb 0x00cccccc 0x00cdcdcd 0x00cecece 0x00cfcfcf + 0x00d0d0d0 0x00d1d1d1 0x00d2d2d2 0x00d3d3d3 0x00d4d4d4 0x00d5d5d5 0x00d6d6d6 0x00d7d7d7 + 0x00d8d8d8 0x00d9d9d9 0x00dadada 0x00dbdbdb 0x00dcdcdc 0x00dddddd 0x00dedede 0x00dfdfdf + 0x00e0e0e0 0x00e1e1e1 0x00e2e2e2 0x00e3e3e3 0x00e4e4e4 0x00e5e5e5 0x00e6e6e6 0x00e7e7e7 + 0x00e8e8e8 0x00e9e9e9 0x00eaeaea 0x00ebebeb 0x00ececec 0x00ededed 0x00eeeeee 0x00efefef + 0x00f0f0f0 0x00f1f1f1 0x00f2f2f2 0x00f3f3f3 0x00f4f4f4 0x00f5f5f5 0x00f6f6f6 0x00f7f7f7 + 0x00f8f8f8 0x00f9f9f9 0x00fafafa 0x00fbfbfb 0x00fcfcfc 0x00fdfdfd 0x00fefefe 0x00ffffff>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&cdn_dp { + extcon = <&fusb0>, <&fusb1>; + status = "okay"; +}; + +&route_dsi { + status = "okay"; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 856000 + SYS_STATUS_REBOOT 416000 + SYS_STATUS_SUSPEND 416000 + SYS_STATUS_VIDEO_1080P 416000 + SYS_STATUS_VIDEO_4K 856000 + SYS_STATUS_VIDEO_4K_10B 856000 + SYS_STATUS_PERFORMANCE 856000 + SYS_STATUS_BOOST 856000 + SYS_STATUS_DUALVIEW 856000 + SYS_STATUS_ISP 856000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 577 416000 + 578 99999 856000 + >; + auto-min-freq = <416000>; + auto-freq-en = <1>; +}; + +&dmc_opp_table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-416000000 { + opp-hz = /bits/ 64 <416000000>; + opp-microvolt = <900000>; + }; + opp-528000000 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-800000000 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-856000000 { + opp-hz = /bits/ 64 <856000000>; + opp-microvolt = <900000>; + }; + opp-928000000 { + opp-hz = /bits/ 64 <928000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-1056000000 { + opp-hz = /bits/ 64 <1056000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-mipi-edp.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-mipi-edp.dts new file mode 100755 index 000000000000..fffd92f00ff4 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android-mipi-edp.dts @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "rk3399-evb-rev3.dtsi" +#include "rk3399-android.dtsi" + +/ { + model = "Rockchip RK3399 Evaluation Board v3 (Android)"; + compatible = "rockchip,android", "rockchip,rk3399-evb-rev3", "rockchip,rk3399"; + + edp_panel: edp-panel { + compatible = "lg,lp079qx1-sp0v", "panel-simple"; + backlight = <&backlight1>; + power-supply = <&vcc3v3_s0>; + + ports { + panel_in_edp: endpoint { + remote-endpoint = <&edp_out_panel>; + }; + }; + + disp_timings: display-timings { + native-mode = <&F402>; + + F402: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + }; + + backlight1: backlight1 { + compatible = "pwm-backlight"; + pwms = <&pwm1 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; +}; + +&pwm1 { + status = "okay"; +}; + +&backlight1 { + status = "okay"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&dsi_in_vopb { + status = "okay"; +}; + +&edp_in_vopl { + status = "okay"; +}; + +&edp_in_vopb { + status = "disabled"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible ="simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <160000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <21>; + hfront-porch = <120>; + vback-porch = <18>; + vfront-porch = <21>; + hsync-len = <20>; + vsync-len = <3>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + dsp_lut: dsp-lut { + gamma-lut = < + 0x00000000 0x00010101 0x00020202 0x00030303 0x00040404 0x00050505 0x00060606 0x00070707 + 0x00080808 0x00090909 0x000a0a0a 0x000b0b0b 0x000c0c0c 0x000d0d0d 0x000e0e0e 0x000f0f0f + 0x00101010 0x00111111 0x00121212 0x00131313 0x00141414 0x00151515 0x00161616 0x00171717 + 0x00181818 0x00191919 0x001a1a1a 0x001b1b1b 0x001c1c1c 0x001d1d1d 0x001e1e1e 0x001f1f1f + 0x00202020 0x00212121 0x00222222 0x00232323 0x00242424 0x00252525 0x00262626 0x00272727 + 0x00282828 0x00292929 0x002a2a2a 0x002b2b2b 0x002c2c2c 0x002d2d2d 0x002e2e2e 0x002f2f2f + 0x00303030 0x00313131 0x00323232 0x00333333 0x00343434 0x00353535 0x00363636 0x00373737 + 0x00383838 0x00393939 0x003a3a3a 0x003b3b3b 0x003c3c3c 0x003d3d3d 0x003e3e3e 0x003f3f3f + 0x00404040 0x00414141 0x00424242 0x00434343 0x00444444 0x00454545 0x00464646 0x00474747 + 0x00484848 0x00494949 0x004a4a4a 0x004b4b4b 0x004c4c4c 0x004d4d4d 0x004e4e4e 0x004f4f4f + 0x00505050 0x00515151 0x00525252 0x00535353 0x00545454 0x00555555 0x00565656 0x00575757 + 0x00585858 0x00595959 0x005a5a5a 0x005b5b5b 0x005c5c5c 0x005d5d5d 0x005e5e5e 0x005f5f5f + 0x00606060 0x00616161 0x00626262 0x00636363 0x00646464 0x00656565 0x00666666 0x00676767 + 0x00686868 0x00696969 0x006a6a6a 0x006b6b6b 0x006c6c6c 0x006d6d6d 0x006e6e6e 0x006f6f6f + 0x00707070 0x00717171 0x00727272 0x00737373 0x00747474 0x00757575 0x00767676 0x00777777 + 0x00787878 0x00797979 0x007a7a7a 0x007b7b7b 0x007c7c7c 0x007d7d7d 0x007e7e7e 0x007f7f7f + 0x00808080 0x00818181 0x00828282 0x00838383 0x00848484 0x00858585 0x00868686 0x00878787 + 0x00888888 0x00898989 0x008a8a8a 0x008b8b8b 0x008c8c8c 0x008d8d8d 0x008e8e8e 0x008f8f8f + 0x00909090 0x00919191 0x00929292 0x00939393 0x00949494 0x00959595 0x00969696 0x00979797 + 0x00989898 0x00999999 0x009a9a9a 0x009b9b9b 0x009c9c9c 0x009d9d9d 0x009e9e9e 0x009f9f9f + 0x00a0a0a0 0x00a1a1a1 0x00a2a2a2 0x00a3a3a3 0x00a4a4a4 0x00a5a5a5 0x00a6a6a6 0x00a7a7a7 + 0x00a8a8a8 0x00a9a9a9 0x00aaaaaa 0x00ababab 0x00acacac 0x00adadad 0x00aeaeae 0x00afafaf + 0x00b0b0b0 0x00b1b1b1 0x00b2b2b2 0x00b3b3b3 0x00b4b4b4 0x00b5b5b5 0x00b6b6b6 0x00b7b7b7 + 0x00b8b8b8 0x00b9b9b9 0x00bababa 0x00bbbbbb 0x00bcbcbc 0x00bdbdbd 0x00bebebe 0x00bfbfbf + 0x00c0c0c0 0x00c1c1c1 0x00c2c2c2 0x00c3c3c3 0x00c4c4c4 0x00c5c5c5 0x00c6c6c6 0x00c7c7c7 + 0x00c8c8c8 0x00c9c9c9 0x00cacaca 0x00cbcbcb 0x00cccccc 0x00cdcdcd 0x00cecece 0x00cfcfcf + 0x00d0d0d0 0x00d1d1d1 0x00d2d2d2 0x00d3d3d3 0x00d4d4d4 0x00d5d5d5 0x00d6d6d6 0x00d7d7d7 + 0x00d8d8d8 0x00d9d9d9 0x00dadada 0x00dbdbdb 0x00dcdcdc 0x00dddddd 0x00dedede 0x00dfdfdf + 0x00e0e0e0 0x00e1e1e1 0x00e2e2e2 0x00e3e3e3 0x00e4e4e4 0x00e5e5e5 0x00e6e6e6 0x00e7e7e7 + 0x00e8e8e8 0x00e9e9e9 0x00eaeaea 0x00ebebeb 0x00ececec 0x00ededed 0x00eeeeee 0x00efefef + 0x00f0f0f0 0x00f1f1f1 0x00f2f2f2 0x00f3f3f3 0x00f4f4f4 0x00f5f5f5 0x00f6f6f6 0x00f7f7f7 + 0x00f8f8f8 0x00f9f9f9 0x00fafafa 0x00fbfbfb 0x00fcfcfc 0x00fdfdfd 0x00fefefe 0x00ffffff>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&cdn_dp { + extcon = <&fusb0>, <&fusb1>; + status = "okay"; +}; + +&route_dsi { + status = "okay"; + connect = <&vopb_out_dsi>; +}; + +&edp { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&edp_hpd>; + + ports { + edp_out: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + edp_out_panel: endpoint@0 { + reg = <0>; + remote-endpoint = <&panel_in_edp>; + }; + }; + }; +}; + +&route_edp { + status = "okay"; + connect = <&vopl_out_edp>; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +>9xx { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android.dts new file mode 100755 index 000000000000..9ba1b0381fed --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-android.dts @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "rk3399-evb-rev3.dtsi" +#include "rk3399-android.dtsi" + +/ { + model = "Rockchip RK3399 Evaluation Board v3 (Android)"; + compatible = "rockchip,android", "rockchip,rk3399-evb-rev3", "rockchip,rk3399"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible ="simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <160000000>; + hactive = <1200>; + vactive = <1920>; + hback-porch = <21>; + hfront-porch = <120>; + vback-porch = <18>; + vfront-porch = <21>; + hsync-len = <20>; + vsync-len = <3>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + dsp_lut: dsp-lut { + gamma-lut = < + 0x00000000 0x00010101 0x00020202 0x00030303 0x00040404 0x00050505 0x00060606 0x00070707 + 0x00080808 0x00090909 0x000a0a0a 0x000b0b0b 0x000c0c0c 0x000d0d0d 0x000e0e0e 0x000f0f0f + 0x00101010 0x00111111 0x00121212 0x00131313 0x00141414 0x00151515 0x00161616 0x00171717 + 0x00181818 0x00191919 0x001a1a1a 0x001b1b1b 0x001c1c1c 0x001d1d1d 0x001e1e1e 0x001f1f1f + 0x00202020 0x00212121 0x00222222 0x00232323 0x00242424 0x00252525 0x00262626 0x00272727 + 0x00282828 0x00292929 0x002a2a2a 0x002b2b2b 0x002c2c2c 0x002d2d2d 0x002e2e2e 0x002f2f2f + 0x00303030 0x00313131 0x00323232 0x00333333 0x00343434 0x00353535 0x00363636 0x00373737 + 0x00383838 0x00393939 0x003a3a3a 0x003b3b3b 0x003c3c3c 0x003d3d3d 0x003e3e3e 0x003f3f3f + 0x00404040 0x00414141 0x00424242 0x00434343 0x00444444 0x00454545 0x00464646 0x00474747 + 0x00484848 0x00494949 0x004a4a4a 0x004b4b4b 0x004c4c4c 0x004d4d4d 0x004e4e4e 0x004f4f4f + 0x00505050 0x00515151 0x00525252 0x00535353 0x00545454 0x00555555 0x00565656 0x00575757 + 0x00585858 0x00595959 0x005a5a5a 0x005b5b5b 0x005c5c5c 0x005d5d5d 0x005e5e5e 0x005f5f5f + 0x00606060 0x00616161 0x00626262 0x00636363 0x00646464 0x00656565 0x00666666 0x00676767 + 0x00686868 0x00696969 0x006a6a6a 0x006b6b6b 0x006c6c6c 0x006d6d6d 0x006e6e6e 0x006f6f6f + 0x00707070 0x00717171 0x00727272 0x00737373 0x00747474 0x00757575 0x00767676 0x00777777 + 0x00787878 0x00797979 0x007a7a7a 0x007b7b7b 0x007c7c7c 0x007d7d7d 0x007e7e7e 0x007f7f7f + 0x00808080 0x00818181 0x00828282 0x00838383 0x00848484 0x00858585 0x00868686 0x00878787 + 0x00888888 0x00898989 0x008a8a8a 0x008b8b8b 0x008c8c8c 0x008d8d8d 0x008e8e8e 0x008f8f8f + 0x00909090 0x00919191 0x00929292 0x00939393 0x00949494 0x00959595 0x00969696 0x00979797 + 0x00989898 0x00999999 0x009a9a9a 0x009b9b9b 0x009c9c9c 0x009d9d9d 0x009e9e9e 0x009f9f9f + 0x00a0a0a0 0x00a1a1a1 0x00a2a2a2 0x00a3a3a3 0x00a4a4a4 0x00a5a5a5 0x00a6a6a6 0x00a7a7a7 + 0x00a8a8a8 0x00a9a9a9 0x00aaaaaa 0x00ababab 0x00acacac 0x00adadad 0x00aeaeae 0x00afafaf + 0x00b0b0b0 0x00b1b1b1 0x00b2b2b2 0x00b3b3b3 0x00b4b4b4 0x00b5b5b5 0x00b6b6b6 0x00b7b7b7 + 0x00b8b8b8 0x00b9b9b9 0x00bababa 0x00bbbbbb 0x00bcbcbc 0x00bdbdbd 0x00bebebe 0x00bfbfbf + 0x00c0c0c0 0x00c1c1c1 0x00c2c2c2 0x00c3c3c3 0x00c4c4c4 0x00c5c5c5 0x00c6c6c6 0x00c7c7c7 + 0x00c8c8c8 0x00c9c9c9 0x00cacaca 0x00cbcbcb 0x00cccccc 0x00cdcdcd 0x00cecece 0x00cfcfcf + 0x00d0d0d0 0x00d1d1d1 0x00d2d2d2 0x00d3d3d3 0x00d4d4d4 0x00d5d5d5 0x00d6d6d6 0x00d7d7d7 + 0x00d8d8d8 0x00d9d9d9 0x00dadada 0x00dbdbdb 0x00dcdcdc 0x00dddddd 0x00dedede 0x00dfdfdf + 0x00e0e0e0 0x00e1e1e1 0x00e2e2e2 0x00e3e3e3 0x00e4e4e4 0x00e5e5e5 0x00e6e6e6 0x00e7e7e7 + 0x00e8e8e8 0x00e9e9e9 0x00eaeaea 0x00ebebeb 0x00ececec 0x00ededed 0x00eeeeee 0x00efefef + 0x00f0f0f0 0x00f1f1f1 0x00f2f2f2 0x00f3f3f3 0x00f4f4f4 0x00f5f5f5 0x00f6f6f6 0x00f7f7f7 + 0x00f8f8f8 0x00f9f9f9 0x00fafafa 0x00fbfbfb 0x00fcfcfc 0x00fdfdfd 0x00fefefe 0x00ffffff>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&cdn_dp { + extcon = <&fusb0>, <&fusb1>; + status = "okay"; +}; + +&route_dsi { + status = "okay"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-cros.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-cros.dts new file mode 100755 index 000000000000..e1bb5f129680 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3-cros.dts @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "rk3399-evb-rev3.dtsi" +#include "rk3399-evb-cros.dtsi" + +/ { + model = "Rockchip RK3399 Evaluation Board v3 (Chrome OS)"; + compatible = "google,rk3399evb-rev3", "rockchip,rk3399-evb-rev3", "rockchip,rk3399"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3.dtsi new file mode 100755 index 000000000000..e34379ded3a5 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb-rev3.dtsi @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "rk3399-evb.dtsi" +#include + +/ { + compatible = "rockchip,rk3399-evb-rev3", "rockchip,rk3399"; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + vdd_center: vdd-center { + compatible = "pwm-regulator"; + rockchip,pwm_id = <2>; + rockchip,pwm_voltage = <900000>; + pwms = <&pwm2 0 25000 1>; + regulator-name = "vdd_center"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1400000>; + regulator-always-on; + regulator-boot-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + + rockchip_suspend: rockchip-suspend { + compatible = "rockchip,pm-rk3399"; + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + ) + >; + rockchip,pwm-regulator-config = < + (0 + | PWM2_REGULATOR_EN + ) + >; + rockchip,power-ctrl = + <&gpio1 17 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c0 { + fusb1: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb1_int>; + vbus-5v-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + vdd_cpu_b: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: syr828@41 { + compatible = "silergy,syr828"; + reg = <0x41>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l &pmic_dvs2>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + vcc10-supply = <&vcc3v3_sys>; + vcc11-supply = <&vcc3v3_sys>; + vcc12-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc1v8_pmu>; + + regulators { + vdd_log: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_log"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc1v8_dvp: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v0_tp: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc3v0_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcca3v0_codec: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcca3v0_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcca1v8_codec: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v3_s3: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s3"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_s0: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&es8316 { + reg = <0x11>; +}; + +&i2c6 { + status = "okay"; + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&pwm2 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin_pull_down>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-evb.dtsi new file mode 100755 index 000000000000..aa8ea436ae9b --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-evb.dtsi @@ -0,0 +1,646 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include "rk3399.dtsi" +#include "rk3399-opp.dtsi" + +/ { + compatible = "rockchip,rk3399-evb", "rockchip,rk3399"; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <297500>; + }; + + menu-key { + label = "menu"; + linux,code = ; + press-threshold-microvolt = <1305500>; + }; + + home-key { + label = "home"; + linux,code = ; + press-threshold-microvolt = <621250>; + }; + + back-key { + label = "back"; + linux,code = ; + press-threshold-microvolt = <980000>; + }; + + camera-key { + label = "camera"; + linux,code = ; + press-threshold-microvolt = <787500>; + }; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + es8316-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,es8316-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s0>; + }; + simple-audio-card,codec { + sound-dai = <&es8316>; + }; + }; + + hdmi_sound: hdmi-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,hdmi"; + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + simple-audio-card,codec { + sound-dai = <&dw_hdmi_audio>; + }; + }; + + dw_hdmi_audio: dw-hdmi-audio { + status = "disabled"; + compatible = "rockchip,dw-hdmi-audio"; + #sound-dai-cells = <0>; + }; + + spdif_sound: spdif-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "disabled"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_gpios>; + //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ + BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ + BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ + BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ + status = "okay"; + }; + + test-power { + status = "okay"; + }; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&sdmmc { + clock-frequency = <150000000>; + clock-freq-min-max = <400000 150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + num-slots = <1>; + //sd-uhs-sdr104; + vqmmc-supply = <&vcc_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "okay"; +}; + +&sdio0 { + clock-frequency = <150000000>; + clock-freq-min-max = <200000 150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&emmc_phy { + status = "okay"; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; + upthreshold = <40>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 800000 + SYS_STATUS_REBOOT 528000 + SYS_STATUS_SUSPEND 200000 + SYS_STATUS_VIDEO_1080P 200000 + SYS_STATUS_VIDEO_4K 600000 + SYS_STATUS_VIDEO_4K_10B 800000 + SYS_STATUS_PERFORMANCE 800000 + SYS_STATUS_BOOST 400000 + SYS_STATUS_DUALVIEW 600000 + SYS_STATUS_ISP 600000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 577 200000 + 578 1701 300000 + 1702 99999 400000 + >; + auto-min-freq = <200000>; +}; + +&sdhci { + bus-width = <8>; + mmc-hs400-1_8v; + no-sdio; + no-sd; + non-removable; + keep-power-in-suspend; + mmc-hs400-enhanced-strobe; + status = "okay"; +}; + +&i2s0 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <8>; + #sound-dai-cells = <0>; +}; + +&i2s2 { + #sound-dai-cells = <0>; +}; + +&spdif { + #sound-dai-cells = <0>; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <450>; + i2c-scl-falling-time-ns = <15>; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <300>; + i2c-scl-falling-time-ns = <15>; + + es8316: es8316@10 { + #sound-dai-cells = <0>; + compatible = "everest,es8316"; + reg = <0x10>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + spk-con-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; + hp-det-gpio = <&gpio4 28 GPIO_ACTIVE_LOW>; + }; +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <600>; + i2c-scl-falling-time-ns = <20>; + + gt9xx: gt9xx@14 { + compatible = "goodix,gt9xx"; + reg = <0x14>; + touch-gpio = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; + reset-gpio = <&gpio4 22 GPIO_ACTIVE_HIGH>; + max-x = <1200>; + max-y = <1900>; + tp-size = <911>; + tp-supply = <&vcc3v0_tp>; + }; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; + }; +}; + +&io_domains { + status = "okay"; + + bt656-supply = <&vcc1v8_dvp>; + audio-supply = <&vcca1v8_codec>; + sdmmc-supply = <&vcc_sd>; + gpio1830-supply = <&vcc_3v0>; +}; + +&pcie_phy { + status = "disabled"; +}; + +&pcie0 { + ep-gpios = <&gpio3 13 GPIO_ACTIVE_HIGH>; + num-lanes = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_clkreqn_cpm>; + status = "disabled"; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + extcon = <&fusb1>; + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + extcon = <&fusb1>; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_1 { + status = "okay"; + extcon = <&fusb1>; +}; + +&pwm0 { + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&pinctrl { + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + pmic_dvs2: pmic-dvs2 { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + fusb1_int: fusb1-int { + rockchip,pins = <1 RK_PD0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pvtm { + status = "okay"; +}; + +&pmu_pvtm { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc1v8_pmu>; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-excavator-sapphire.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-excavator-sapphire.dtsi new file mode 100755 index 000000000000..adbda6ba4e0d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-excavator-sapphire.dtsi @@ -0,0 +1,324 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "rk3399-sapphire.dtsi" +#include +/ { + compatible = "rockchip,rk3399-sapphire-excavator", "rockchip,rk3399"; + + rt5651_sound: rt5651-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "realtek,rt5651-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s0>; + }; + simple-audio-card,codec { + sound-dai = <&rt5651>; + }; + }; + + spdif-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "okay"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>, <&bt_reset_gpio>, <&bt_wake_gpio>, <&bt_irq_gpio>; + pinctrl-1 = <&uart0_gpios>; + //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ + BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ + BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ + BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ + status = "okay"; + }; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; + upthreshold = <40>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 800000 + SYS_STATUS_REBOOT 528000 + SYS_STATUS_SUSPEND 200000 + SYS_STATUS_VIDEO_1080P 200000 + SYS_STATUS_VIDEO_4K 600000 + SYS_STATUS_VIDEO_4K_10B 800000 + SYS_STATUS_PERFORMANCE 800000 + SYS_STATUS_BOOST 600000 + SYS_STATUS_DUALVIEW 600000 + SYS_STATUS_ISP 600000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 762 200000 + 763 1893 400000 + 1894 3012 528000 + 3013 99999 800000 + >; + auto-freq-en = <1>; + auto-min-freq = <200000>; +}; + +&spdif { + status = "okay"; + pinctrl-0 = <&spdif_bus>; + i2c-scl-rising-time-ns = <450>; + i2c-scl-falling-time-ns = <15>; + #sound-dai-cells = <0>; +}; + +&i2s0 { + status = "okay"; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <8>; + #sound-dai-cells = <0>; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <300>; + i2c-scl-falling-time-ns = <15>; + + rt5651: rt5651@1a { + #sound-dai-cells = <0>; + compatible = "rockchip,rt5651"; + reg = <0x1a>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + spk-con-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; + hp-det-gpio = <&gpio4 28 GPIO_ACTIVE_LOW>; + }; +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <600>; + i2c-scl-falling-time-ns = <20>; + + mpu6500@68 { + status = "disabled"; + compatible = "invensense,mpu6500"; + reg = <0x68>; + irq-gpio = <&gpio1 22 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <0 1 0 1 0 0 0 0 1>; + orientation-x= <1>; + orientation-y= <0>; + orientation-z= <0>; + mpu-debug = <1>; + }; + + mpu6500_acc: mpu_acc@68 { + compatible = "mpu6500_acc"; + reg = <0x68>; + irq-gpio = <&gpio1 22 IRQ_TYPE_EDGE_RISING>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + layout = <2>; + }; + + mpu6500_gyro: mpu_gyro@68 { + compatible = "mpu6500_gyro"; + reg = <0x68>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + layout = <2>; + }; +}; + +&pcie0 { + ep-gpios = <&gpio2 4 GPIO_ACTIVE_HIGH>; + num-lanes = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_clkreqn_cpm>; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&saradc { + status = "okay"; +}; + +&sdio0 { + max-frequency = <100000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_PWM_WKUP_EN + ) + >; + rockchip,pwm-regulator-config = < + (0 + | PWM2_REGULATOR_EN + ) + >; + rockchip,power-ctrl = + <&gpio1 17 GPIO_ACTIVE_HIGH>, + <&gpio1 14 GPIO_ACTIVE_HIGH>; +}; + +&pinctrl { + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + bt_reset_gpio: bt-reset-gpio { + rockchip,pins = + <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + bt_wake_gpio: bt-wake-gpio { + rockchip,pins = + <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + bt_irq_gpio: bt-irq-gpio { + rockchip,pins = + <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-firefly-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-firefly-android.dts new file mode 100755 index 000000000000..c372ade536f2 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-firefly-android.dts @@ -0,0 +1,1102 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "dt-bindings/pwm/pwm.h" +#include "rk3399.dtsi" +#include "rk3399-opp.dtsi" +#include +#include +#include "rk3399-vop-clk-set.dtsi" + +/ { + model = "Rockchip RK3399 Firefly Board (Android)"; + compatible = "rockchip,rk3399-firefly-android", "rockchip,rk3399"; + + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff1a0000 coherent_pool=1m"; + }; + + cpuinfo { + compatible = "rockchip,cpuinfo"; + nvmem-cells = <&cpu_id>; + nvmem-cell-names = "id"; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + dw_hdmi_audio: dw-hdmi-audio { + status = "okay"; + compatible = "rockchip,dw-hdmi-audio"; + #sound-dai-cells = <0>; + }; + + edp_panel: edp-panel { + compatible = "sharp,lcd-f402", "panel-simple"; + status = "okay"; + + backlight = <&backlight>; + power-supply = <&vcc_lcd>; + enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd_panel_reset>; + + ports { + panel_in_edp: endpoint { + remote-endpoint = <&edp_out_panel>; + }; + }; + }; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <0>; + /* Only 115200 and 1500000 */ + rockchip,baudrate = <1500000>; + pinctrl-names = "default"; + pinctrl-0 = <&uart2c_xfer>; + interrupts = ; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + drm_logo: drm-logo@00000000 { + compatible = "rockchip,drm-logo"; + reg = <0x0 0x0 0x0 0x0>; + }; + + stb_devinfo: stb-devinfo@00000000 { + compatible = "rockchip,stb-devinfo"; + reg = <0x0 0x0 0x0 0x0>; + }; + + ramoops: ramoops@110000 { + compatible = "ramoops"; + reg = <0x0 0x110000 0x0 0xf0000>; + record-size = <0x20000>; + console-size = <0x80000>; + ftrace-size = <0x00000>; + pmsg-size = <0x50000>; + }; + }; + + rockchip-key { + compatible = "rockchip,key"; + status = "okay"; + + io-channels = <&saradc 1>; + power-key { + gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; + linux,code = <116>; + label = "power"; + gpio-key,wakeup; + }; + }; + + rt5640-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rt5640-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&rt5640>; + }; + }; + + hdmi_sound: hdmi-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,hdmi"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + simple-audio-card,codec { + sound-dai = <&dw_hdmi_audio>; + }; + }; + + hdmi_codec: hdmi-codec { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "HDMI-CODEC"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + rga: rga@ff680000 { + compatible = "rockchip,rga2"; + dev_mode = <1>; + reg = <0x0 0xff680000 0x0 0x1000>; + interrupts = ; + clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru SCLK_RGA_CORE>; + clock-names = "aclk_rga", "hclk_rga", "clk_rga"; + power-domains = <&power RK3399_PD_RGA>; + dma-coherent; + status = "okay"; + }; + + spdif-sound { + compatible = "simple-audio-card"; + status = "okay"; + + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + compatible = "linux,spdif-dit"; + status = "okay"; + + #sound-dai-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ + }; + + vcc3v3_pcie: vcc3v3-pcie-regulator { + compatible = "regulator-fixed"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + gpio = <&gpio1 17 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_drv>; + regulator-name = "vcc3v3_pcie"; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vdd_log: vdd-log { + compatible = "pwm-regulator"; + pwms = <&pwm2 0 25000 1>; + regulator-name = "vdd_log"; + regulator-min-microvolt = <800000>; + /* + * the firefly hardware using 3.0 v as APIO2_VDD + * voltage, but the pwm divider resistance is designed + * based on hardware which the APIO2_VDD is 1.8v, so we + * need to change the regulator-max-microvolt from 1.4v + * to 1.0v, so the pwm can output 0.9v voltage. + */ + regulator-max-microvolt = <1000000>; + regulator-always-on; + regulator-boot-on; + + /* for rockchip boot on */ + rockchip,pwm_id= <2>; + rockchip,pwm_voltage = <900000>; + }; + + vccadc_ref: vccadc-ref { + compatible = "regulator-fixed"; + regulator-name = "vcc1v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + vcc_lcd: vcc-lcd-regulator { + compatible = "regulator-fixed"; + regulator-always-on; + regulator-boot-on; + enable-active-high; + gpio = <&gpio1 1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd_en>; + regulator-name = "vcc_lcd"; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_gpios>; + //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ + BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ + BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ + BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ + status = "okay"; + }; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&display_subsystem { + status = "okay"; + + ports = <&vopb_out>, <&vopl_out>; + logo-memory-region = <&drm_logo>; + + route { + route_hdmi: route-hdmi { + status = "okay"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "fullscreen"; + charge_logo,mode = "center"; + connect = <&vopl_out_hdmi>; + }; + + route_edp: route-edp { + status = "okay"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "fullscreen"; + charge_logo,mode = "center"; + connect = <&vopb_out_edp>; + }; + }; +}; + +&edp { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&edp_hpd>; + + ports { + edp_out: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + edp_out_panel: endpoint@0 { + reg = <0>; + remote-endpoint = <&panel_in_edp>; + }; + }; + }; +}; + +&emmc_phy { + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&hdmi { + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <168>; + i2c-scl-falling-time-ns = <4>; + clock-frequency = <400000>; + + vdd_cpu_b: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + vsel-gpios = <&gpio1 18 GPIO_ACTIVE_HIGH>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: syr828@41 { + compatible = "silergy,syr828"; + reg = <0x41>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l &pmic_dvs2>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + vcc10-supply = <&vcc3v3_sys>; + vcc11-supply = <&vcc3v3_sys>; + vcc12-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc1v8_pmu>; + + regulators { + vdd_center: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_center"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc1v8_dvp: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v0_tp: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc3v0_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcca3v0_codec: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcca3v0_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcca1v8_codec: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v3_s3: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_s0: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <300>; + i2c-scl-falling-time-ns = <15>; + + rt5640: rt5640@1c { + #sound-dai-cells = <0>; + compatible = "realtek,rt5640"; + reg = <0x1c>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + realtek,in1-differential; + pinctrl-names = "default"; + pinctrl-0 = <&rt5640_hpcon &i2s_8ch_mclk>; + hp-con-gpio = <&gpio4 21 GPIO_ACTIVE_HIGH>; + //hp-det-gpio = <&gpio4 28 GPIO_ACTIVE_LOW>; + io-channels = <&saradc 4>; + hp-det-adc-value = <500>; + status = "okay"; + }; +}; + +&i2c3 { + status = "okay"; + i2c-scl-rising-time-ns = <450>; + i2c-scl-falling-time-ns = <15>; +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <600>; + i2c-scl-falling-time-ns = <20>; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; + vbus-5v-gpios = <&gpio2 0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + gsl3680: gsl3680@41 { + compatible = "gslX680-pad"; + reg = <0x41>; + screen_max_x = <1536>; + screen_max_y = <2048>; + touch-gpio = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; + reset-gpio = <&gpio0 12 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + mpu6050: mpu@68 { + compatible = "invensense,mpu6050"; + reg = <0x68>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <0 1 0 1 0 0 0 0 1>; + orientation-x= <1>; + orientation-y= <1>; + orientation-z= <1>; + irq-gpio = <&gpio1 4 IRQ_TYPE_LEVEL_LOW>; + mpu-debug = <1>; + status = "okay"; + }; +}; + +&i2s0 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <8>; + #sound-dai-cells = <0>; +}; + +&i2s1 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <2>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&io_domains { + status = "okay"; + + bt656-supply = <&vcc1v8_dvp>; /* bt656_gpio2ab_ms */ + audio-supply = <&vcca1v8_codec>; /* audio_gpio3d4a_ms */ + sdmmc-supply = <&vcc_sd>; /* sdmmc_gpio4b_ms */ + gpio1830-supply = <&vcc_3v0>; /* gpio1833_gpio4cd_ms */ +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + ep-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; + num-lanes = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_clkreqn_cpm>; + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_3v0>; +}; + +&pinctrl { + + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <4 RK_PD5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + lcd_en: lcd-en { + rockchip,pins = <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pcie { + pcie_drv: pcie-drv { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + pcie_3g_drv: pcie-3g-drv { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + rt5640 { + rt5640_hpcon: rt5640-hpcon { + rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + pmic_dvs2: pmic-dvs2 { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm2 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin_pull_down>; +}; + +&rockchip_suspend { + rockchip,power-ctrl = + <&gpio1 18 GPIO_ACTIVE_LOW>, + <&gpio1 14 GPIO_ACTIVE_HIGH>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vccadc_ref>; +}; + +&sdhci { + bus-width = <8>; + keep-power-in-suspend; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + non-removable; + status = "okay"; + no-sdio; + no-sd; +}; + +&sdmmc { + max-frequency = <150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + num-slots = <1>; + vqmmc-supply = <&vcc_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "okay"; +}; + +&sdio0 { + max-frequency = <50000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&spdif { + status = "okay"; + pinctrl-0 = <&spdif_bus_1>; + i2c-scl-rising-time-ns = <450>; + i2c-scl-falling-time-ns = <15>; + #sound-dai-cells = <0>; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-mode = <1>; + /* tshut polarity 0:LOW 1:HIGH */ + rockchip,hw-tshut-polarity = <1>; + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd_dwc3_1 { + status = "okay"; + dr_mode = "host"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-firefly-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399-firefly-linux.dts new file mode 100755 index 000000000000..5a023389a033 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-firefly-linux.dts @@ -0,0 +1,1074 @@ +/* + * Copyright (c) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "dt-bindings/pwm/pwm.h" +#include "rk3399.dtsi" +#include "rk3399-opp.dtsi" +#include "rk3399-linux.dtsi" +#include + +/ { + model = "Rockchip RK3399 Firefly Board (Linux Opensource)"; + compatible = "rockchip,rk3399-firefly-linux", "rockchip,rk3399"; + + backlight: backlight { + status = "disabled"; + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + dw_hdmi_audio: dw-hdmi-audio { + status = "disabled"; + compatible = "rockchip,dw-hdmi-audio"; + #sound-dai-cells = <0>; + }; + + edp_panel: edp-panel { + status = "disabled"; + compatible = "sharp,lcd-f402", "panel-simple"; + backlight = <&backlight>; + power-supply = <&vcc_lcd>; + enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd_panel_reset>; + + ports { + panel_in_edp: endpoint { + remote-endpoint = <&edp_out_panel>; + }; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = <&pwrbtn>; + + button@0 { + gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "GPIO Key Power"; + linux,input-type = <1>; + gpio-key,wakeup = <1>; + debounce-interval = <100>; + }; + }; + + rt5640-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rt5640-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&rt5640>; + }; + }; + + hdmi_sound: hdmi-sound { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,hdmi"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + simple-audio-card,codec { + sound-dai = <&dw_hdmi_audio>; + }; + }; + + hdmi_codec: hdmi-codec { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "HDMI-CODEC"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + spdif-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "okay"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ + }; + + vcc3v3_pcie: vcc3v3-pcie-regulator { + compatible = "regulator-fixed"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + gpio = <&gpio1 17 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_drv>; + regulator-name = "vcc3v3_pcie"; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vdd_log: vdd-log { + compatible = "pwm-regulator"; + pwms = <&pwm2 0 25000 1>; + regulator-name = "vdd_log"; + regulator-min-microvolt = <800000>; + /* + * the firefly hardware using 3.0 v as APIO2_VDD + * voltage, but the pwm divider resistance is designed + * based on hardware which the APIO2_VDD is 1.8v, so we + * need to change the regulator-max-microvolt from 1.4v + * to 1.0v, so the pwm can output 0.9v voltage. + */ + regulator-max-microvolt = <1000000>; + regulator-always-on; + regulator-boot-on; + + /* for rockchip boot on */ + rockchip,pwm_id= <2>; + rockchip,pwm_voltage = <900000>; + }; + + vccadc_ref: vccadc-ref { + compatible = "regulator-fixed"; + regulator-name = "vcc1v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + vcc_lcd: vcc-lcd-regulator { + compatible = "regulator-fixed"; + regulator-always-on; + regulator-boot-on; + enable-active-high; + gpio = <&gpio1 1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd_en>; + regulator-name = "vcc_lcd"; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_gpios>; + //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ + BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ + BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ + BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ + status = "okay"; + }; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&display_subsystem { + status = "okay"; +}; + +&edp { + status = "disabled"; + + ports { + edp_out: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + edp_out_panel: endpoint@0 { + reg = <0>; + remote-endpoint = <&panel_in_edp>; + }; + }; + }; +}; + +&emmc_phy { + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&hdmi { + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <168>; + i2c-scl-falling-time-ns = <4>; + clock-frequency = <400000>; + + vdd_cpu_b: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + vsel-gpios = <&gpio1 18 GPIO_ACTIVE_HIGH>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: syr828@41 { + compatible = "silergy,syr828"; + reg = <0x41>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l &pmic_dvs2>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + vcc10-supply = <&vcc3v3_sys>; + vcc11-supply = <&vcc3v3_sys>; + vcc12-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc1v8_pmu>; + + regulators { + vdd_center: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_center"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc1v8_dvp: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v0_tp: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc3v0_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcca3v0_codec: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcca3v0_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcca1v8_codec: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v3_s3: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_s0: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <300>; + i2c-scl-falling-time-ns = <15>; + + rt5640: rt5640@1c { + #sound-dai-cells = <0>; + compatible = "realtek,rt5640"; + reg = <0x1c>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + realtek,in1-differential; + pinctrl-names = "default"; + pinctrl-0 = <&rt5640_hpcon &i2s_8ch_mclk>; + hp-con-gpio = <&gpio4 21 GPIO_ACTIVE_HIGH>; + //hp-det-gpio = <&gpio4 28 GPIO_ACTIVE_LOW>; + io-channels = <&saradc 4>; + hp-det-adc-value = <500>; + }; + + camera0: ov13850@10 { + status = "okay"; + compatible = "omnivision,ov13850-v4l2-i2c-subdev"; + reg = < 0x10 >; + device_type = "v4l2-i2c-subdev"; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "clk_cif_out"; + + pinctrl-names = "rockchip,camera_default", "rockchip,camera_sleep"; + pinctrl-0 = <&cam0_default_pins>; + pinctrl-1 = <&cam0_sleep_pins>; + + rockchip,pd-gpio = <&gpio2 12 GPIO_ACTIVE_LOW>; + rockchip,pwr-gpio = <&gpio1 23 GPIO_ACTIVE_HIGH>; + rockchip,pwr-2nd-gpio = <&gpio1 22 GPIO_ACTIVE_HIGH>; + rockchip,rst-gpio = <&gpio0 8 GPIO_ACTIVE_LOW>; + + rockchip,camera-module-mclk-name = "clk_cif_out"; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "cmk-cb0695-fv1"; + rockchip,camera-module-len-name = "lg9569a2"; + rockchip,camera-module-fov-h = "66.0"; + rockchip,camera-module-fov-v = "50.1"; + rockchip,camera-module-orientation = <0>; + rockchip,camera-module-iq-flip = <0>; + rockchip,camera-module-iq-mirror = <0>; + rockchip,camera-module-flip = <1>; + rockchip,camera-module-mirror = <0>; + + rockchip,camera-module-defrect0 = <2112 1568 0 0 2112 1568>; + rockchip,camera-module-defrect1 = <4224 3136 0 0 4224 3136>; + rockchip,camera-module-defrect3 = <3264 2448 0 0 3264 2448>; + rockchip,camera-module-flash-support = <0>; + rockchip,camera-module-mipi-dphy-index = <0>; + }; +}; + +&i2c3 { + status = "okay"; + i2c-scl-rising-time-ns = <450>; + i2c-scl-falling-time-ns = <15>; +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <600>; + i2c-scl-falling-time-ns = <20>; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; + vbus-5v-gpios = <&gpio2 0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + gsl3680: gsl3680@41 { + status = "disabled"; + compatible = "gslX680-pad"; + reg = <0x41>; + screen_max_x = <1536>; + screen_max_y = <2048>; + touch-gpio = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; + reset-gpio = <&gpio0 12 GPIO_ACTIVE_HIGH>; + }; + + mpu6050: mpu@68 { + status = "disabled"; + compatible = "invensense,mpu6050"; + reg = <0x68>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <0 1 0 1 0 0 0 0 1>; + orientation-x= <1>; + orientation-y= <1>; + orientation-z= <1>; + irq-gpio = <&gpio1 4 IRQ_TYPE_LEVEL_LOW>; + mpu-debug = <1>; + }; +}; + +&i2s0 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <8>; + #sound-dai-cells = <0>; +}; + +&i2s1 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <2>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&io_domains { + status = "okay"; + + bt656-supply = <&vcc1v8_dvp>; /* bt656_gpio2ab_ms */ + audio-supply = <&vcca1v8_codec>; /* audio_gpio3d4a_ms */ + sdmmc-supply = <&vcc_sd>; /* sdmmc_gpio4b_ms */ + gpio1830-supply = <&vcc_3v0>; /* gpio1833_gpio4cd_ms */ +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + ep-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; + num-lanes = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_clkreqn_cpm>; + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_3v0>; +}; + +&pinctrl { + buttons { + pwrbtn: pwrbtn { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <4 RK_PD5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + lcd_en: lcd-en { + rockchip,pins = <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pcie { + pcie_drv: pcie-drv { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + pcie_3g_drv: pcie-3g-drv { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + rt5640 { + rt5640_hpcon: rt5640-hpcon { + rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + pmic_dvs2: pmic-dvs2 { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm2 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin_pull_down>; +}; + +&rockchip_suspend { + rockchip,power-ctrl = + <&gpio1 18 GPIO_ACTIVE_LOW>, + <&gpio1 14 GPIO_ACTIVE_HIGH>; +}; + +&route_edp { + status = "disabled"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vccadc_ref>; +}; + +&sdhci { + bus-width = <8>; + keep-power-in-suspend; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + non-removable; + status = "okay"; + no-sdio; + no-sd; +}; + +&sdmmc { + max-frequency = <150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + num-slots = <1>; + vqmmc-supply = <&vcc_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "okay"; +}; + +&sdio0 { + max-frequency = <50000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&spdif { + status = "okay"; + pinctrl-0 = <&spdif_bus_1>; + i2c-scl-rising-time-ns = <450>; + i2c-scl-falling-time-ns = <15>; + #sound-dai-cells = <0>; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-mode = <1>; + /* tshut polarity 0:LOW 1:HIGH */ + rockchip,hw-tshut-polarity = <1>; + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd_dwc3_1 { + status = "okay"; + dr_mode = "host"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&cif_isp0 { + rockchip,camera-modules-attached = <&camera0>; + status = "okay"; +}; + +&isp0_mmu { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-fpga.dts b/arch/arm64/boot/dts/rockchip/rk3399-fpga.dts new file mode 100755 index 000000000000..1192dfa42940 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-fpga.dts @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3399.dtsi" + +/ { + model = "Rockchip RK3399 FPGA Board"; + compatible = "rockchip,fpga", "rockchip,rk3399"; + + chosen { + bootargs = "init=/init console=uart,mmio32,0xff1a0000"; + }; + + memory@00000000 { + device_type = "memory"; + reg = <0x0 0x00000000 0x0 0x20000000>; + }; +}; + +&uart2 { + status = "okay"; + clocks = <&xin24m>, <&xin24m>; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-gru.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-gru.dts new file mode 100755 index 000000000000..e8d771e2b90e --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-gru.dts @@ -0,0 +1,165 @@ +/* + * Google Gru-Gru Rev 0+ board device tree source + * + * Copyright 2016 Google, Inc + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3399-gru.dtsi" + +/ { + model = "Google Gru"; + compatible = "google,gru-rev15", "google,gru-rev14", + "google,gru-rev13", "google,gru-rev12", + "google,gru-rev11", "google,gru-rev10", + "google,gru-rev9", "google,gru-rev8", + "google,gru-rev7", "google,gru-rev6", + "google,gru-rev5", "google,gru-rev4", + "google,gru-rev3", "google,gru-rev2", + "google,gru-rev1", "google,gru-rev0", + "google,gru", "rockchip,rk3399"; + + // TODO: Model: + // - pp1200_mipi_cam + // - pp1800_mipi_cam + // - pp2800_mipi_cam + + /* pp1800 children */ + + pp1800_fp: pp1800-fp { + compatible = "regulator-fixed"; + regulator-name = "pp1800_fp"; + pinctrl-names = "default"; + pinctrl-0 = <&fp_en>; + + enable-active-high; + gpio = <&gpio1 1 GPIO_ACTIVE_HIGH>; + + regulator-always-on; // For bringup??? + regulator-boot-on; // For bringup??? + + vin-supply = <&pp1800>; + }; + + /* pp3300 children */ + + pp3300_fp: pp3300-fp { + compatible = "regulator-fixed"; + regulator-name = "pp3300_fp"; + /* NOTE: fp_en pinctrl in pp1800_fp */ + + enable-active-high; + gpio = <&gpio1 1 GPIO_ACTIVE_HIGH>; + + regulator-always-on; // For bringup??? + regulator-boot-on; // For bringup??? + + vin-supply = <&pp3300>; + }; +}; + +ap_i2c_dvs: &i2c0 { + status = "okay"; + + // TODO: bus speed + // ...with no speed, it should just use 100kHz + // TODO: rise / fall times? + + /* LP8556 */ + backlight@2c { + compatible = "ti,lp8556"; + reg = <0x2c>; + + // TODO: Where do we specify AP_BL_EN??? + + bl-name = "lcd-bl"; + dev-ctrl = /bits/ 8 <0x85>; // TODO: It depends on the device. + init-brt = /bits/ 8 <0x10>; // TODO: What should it be? + + power-supply = <&pp3300_disp>; + }; +}; + +ap_i2c_cam: &i2c2 { + status = "okay"; + + // TODO: bus speed + // ...with no speed, it should just use 100kHz + // TODO: rise / fall times? + + // TODO: I belive this is for the MIPI camera. +}; + +ap_i2c_nfc: &i2c7 { + status = "okay"; + + // TODO: bus speed + // ...with no speed, it should just use 100kHz + // TODO: rise / fall times? + + // TODO: Add the proper NFC reference... +}; + +&spi4 { + status = "okay"; + + // TODO: more properly. Hacky spidev for now??? + fingerprint@0 { + compatible = "spidev"; + spi-max-frequency = <10000000>; + reg = <0>; + }; +}; + +/* PINCTRL: always below everything else */ + +&pinctrl { + discrete-regulators { + fp_en: fp-en { + rockchip,pins = <1 RK_PA1 RK_FUNC_GPIO + &pcfg_pull_none>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r0.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r0.dts new file mode 100755 index 000000000000..7a1c36e981d5 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r0.dts @@ -0,0 +1,118 @@ +/* + * Google Gru-Kevin Rev 0 board device tree source + * + * Copyright 2016 Google, Inc + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3399-gru.dtsi" + +/ { + model = "Google Kevin Rev 0"; + compatible = "google,kevin-rev0", + "google,kevin", "google,gru", "rockchip,rk3399"; +}; + +&ap_i2c_tp { + trackpad@4a { + compatible = "atmel,maxtouch"; + reg = <0x4a>; + pinctrl-names = "default"; + pinctrl-0 = <&trackpad_int_l>; + interrupt-parent = <&gpio1>; + interrupts = <4 IRQ_TYPE_LEVEL_LOW>; + linux,gpio-keymap = ; + }; +}; + +/* GPIO overrides for -r0; in same order as parent */ + +&pp3000 { + gpio = <&gpio1 12 GPIO_ACTIVE_HIGH>; +}; + +&pp1800_audio { + gpio = <&gpio0 1 GPIO_ACTIVE_HIGH>; +}; + +&pp1800_pcie { + gpio = <&gpio0 8 GPIO_ACTIVE_HIGH>; +}; + +&sdmmc { + cd-gpios = <&gpio4 26 GPIO_ACTIVE_LOW>; +}; + +&cros_ec { + interrupts = <2 IRQ_TYPE_LEVEL_LOW>; +}; + +/* Pinctrl overrides for -r0; in same order as parent */ + +&ec_ap_int_l { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; +}; + +&pp1500_en { + rockchip,pins = <1 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>; +}; + +&pp1800_audio_en { + rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO + &pcfg_pull_none>; +}; + +&pp3000_en { + rockchip,pins = <1 RK_PB4 RK_FUNC_GPIO + &pcfg_pull_none>; +}; + +&wlan_module_pd_l { + rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO + &pcfg_pull_none>; +}; + +&sdmmc_cd_gpio { + rockchip,pins = <4 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r1.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r1.dts new file mode 100755 index 000000000000..44b04e1606fc --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin-r1.dts @@ -0,0 +1,85 @@ +/* + * Google Gru-Kevin Rev 1+ board device tree source + * + * Copyright 2016 Google, Inc + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3399-gru.dtsi" + +/ { + model = "Google Kevin"; + compatible = "google,kevin-rev15", "google,kevin-rev14", + "google,kevin-rev13", "google,kevin-rev12", + "google,kevin-rev11", "google,kevin-rev10", + "google,kevin-rev9", "google,kevin-rev8", + "google,kevin-rev7", "google,kevin-rev6", + "google,kevin-rev5", "google,kevin-rev4", + "google,kevin-rev3", "google,kevin-rev2", + "google,kevin-rev1", "google,kevin", + "google,gru", "rockchip,rk3399"; +}; + +&ap_i2c_tp { + trackpad@4a { + compatible = "atmel,maxtouch"; + reg = <0x4a>; + pinctrl-names = "default"; + pinctrl-0 = <&trackpad_int_l>; + interrupt-parent = <&gpio1>; + interrupts = <4 IRQ_TYPE_LEVEL_LOW>; + linux,gpio-keymap = ; + }; +}; + +/* PINCTRL: always below everything else */ + +&pinctrl { + pen-eject { + pen_eject_l: pen-eject-l { + rockchip,pins = <0 RK_PB5 RK_FUNC_GPIO + &pcfg_pull_up>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi index e9ecffc409c0..60cd1c18cd4e 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi @@ -245,7 +245,7 @@ &ppvar_gpu_pwm { }; &ppvar_sd_card_io { - states = <1800000 0x0>, <3300000 0x1>; + states = <1800000 0x0 3300000 0x1>; regulator-max-microvolt = <3300000>; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi index fb0a13cad6c9..1ce91e36aa49 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi @@ -247,8 +247,8 @@ ppvar_sd_card_io: ppvar-sd-card-io { enable-active-high; enable-gpio = <&gpio2 2 GPIO_ACTIVE_HIGH>; gpios = <&gpio2 28 GPIO_ACTIVE_HIGH>; - states = <1800000 0x1>, - <3000000 0x0>; + states = <1800000 0x1 + 3000000 0x0>; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3000000>; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-linux.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-linux.dtsi new file mode 100755 index 000000000000..0549701d615e --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-linux.dtsi @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include "rk3399-vop-clk-set.dtsi" + +/ { + compatible = "rockchip,linux", "rockchip,rk3399"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff1a0000 console=ttyFIQ0 rw root=PARTUUID=614e0000-0000 rootfstype=ext4 rootwait coherent_pool=1m"; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + drm_logo: drm-logo@00000000 { + compatible = "rockchip,drm-logo"; + reg = <0x0 0x0 0x0 0x0>; + }; + + ramoops_mem: region@110000 { + reg = <0x0 0x110000 0x0 0xf0000>; + reg-names = "ramoops_mem"; + }; + }; + + ramoops: ramoops { + compatible = "ramoops"; + record-size = <0x0 0x40000>; + console-size = <0x0 0x80000>; + ftrace-size = <0x0 0x00000>; + pmsg-size = <0x0 0x00000>; + memory-region = <&ramoops_mem>; + }; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,signal-irq = <182>; + rockchip,wake-irq = <0>; + rockchip,irq-mode-enable = <1>; /* If enable uart uses irq instead of fiq */ + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + pinctrl-names = "default"; + pinctrl-0 = <&uart2c_xfer>; + }; + + cif_isp0: cif_isp@ff910000 { + compatible = "rockchip,rk3399-cif-isp"; + rockchip,grf = <&grf>; + reg = <0x0 0xff910000 0x0 0x4000>, <0x0 0xff968000 0x0 0x8000>; + reg-names = "register", "dsihost-register"; + clocks = + <&cru ACLK_ISP0_NOC>, <&cru ACLK_ISP0_WRAPPER>, + <&cru HCLK_ISP0_NOC>, <&cru HCLK_ISP0_WRAPPER>, + <&cru SCLK_ISP0>, <&cru SCLK_DPHY_RX0_CFG>, + <&cru SCLK_CIF_OUT>, <&cru SCLK_CIF_OUT>, + <&cru SCLK_MIPIDPHY_REF>; + clock-names = + "aclk_isp0_noc", "aclk_isp0_wrapper", + "hclk_isp0_noc", "hclk_isp0_wrapper", + "clk_isp0", "pclk_dphyrx", + "clk_cif_out", "clk_cif_pll", + "pclk_dphy_ref"; + interrupts = ; + interrupt-names = "cif_isp10_irq"; + power-domains = <&power RK3399_PD_ISP0>; + rockchip,isp,iommu-enable = <1>; + iommus = <&isp0_mmu>; + status = "disabled"; + }; + + cif_isp1: cif_isp@ff920000 { + compatible = "rockchip,rk3399-cif-isp"; + rockchip,grf = <&grf>; + reg = <0x0 0xff920000 0x0 0x4000>, <0x0 0xff968000 0x0 0x8000>; + reg-names = "register", "dsihost-register"; + clocks = + <&cru ACLK_ISP1_NOC>, <&cru ACLK_ISP1_WRAPPER>, + <&cru HCLK_ISP1_NOC>, <&cru HCLK_ISP1_WRAPPER>, + <&cru SCLK_ISP1>, <&cru PCLK_ISP1_WRAPPER>, + <&cru SCLK_DPHY_TX1RX1_CFG>, + <&cru PCLK_MIPI_DSI1>, <&cru SCLK_MIPIDPHY_CFG>, + <&cru SCLK_CIF_OUT>, <&cru SCLK_CIF_OUT>, + <&cru SCLK_MIPIDPHY_REF>; + clock-names = + "aclk_isp1_noc", "aclk_isp1_wrapper", + "hclk_isp1_noc", "hclk_isp1_wrapper", + "clk_isp1", "pclkin_isp1", + "pclk_dphytxrx", + "pclk_mipi_dsi","mipi_dphy_cfg", + "clk_cif_out", "clk_cif_pll", + "pclk_dphy_ref"; + interrupts = ; + interrupt-names = "cif_isp10_irq"; + power-domains = <&power RK3399_PD_ISP1>; + rockchip,isp,iommu-enable = <1>; + iommus = <&isp1_mmu>; + status = "disabled"; + }; + + rga: rga@ff680000 { + compatible = "rockchip,rga2"; + dev_mode = <1>; + reg = <0x0 0xff680000 0x0 0x1000>; + interrupts = ; + clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru SCLK_RGA_CORE>; + clock-names = "aclk_rga", "hclk_rga", "clk_rga"; + power-domains = <&power RK3399_PD_RGA>; + status = "okay"; + }; +}; + +&display_subsystem { + status = "disabled"; + + ports = <&vopb_out>, <&vopl_out>; + logo-memory-region = <&drm_logo>; + + route { + route_hdmi: route-hdmi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopb_out_hdmi>; + }; + + route_dsi: route-dsi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopl_out_dsi>; + }; + + route_edp: route-edp { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vopl_out_edp>; + }; + }; +}; + +&edp { + /delete-property/pinctrl-names; + /delete-property/pinctrl-0; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&pvtm { + status = "okay"; +}; + +&rkvdec { + status = "okay"; + /* 0 means ion, 1 means drm */ + //allocator = <0>; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vpu_mmu { + status = "okay"; +}; + +&uart2 { + status = "disabled"; +}; + +&pinctrl { + isp { + cif_clkout: cif-clkout { + rockchip,pins = + /* cif_clkout */ + <2 RK_PB3 3 &pcfg_pull_none>; + }; + + isp_dvp_d0d7: isp-dvp-d0d7 { + rockchip,pins = + <4 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>, + /* cif_clkout */ + <2 RK_PB3 3 &pcfg_pull_none>, + /* cif_data0 */ + <2 RK_PA0 3 &pcfg_pull_none>, + /* cif_data1 */ + <2 RK_PA1 3 &pcfg_pull_none>, + /* cif_data2 */ + <2 RK_PA2 3 &pcfg_pull_none>, + /* cif_data3 */ + <2 RK_PA3 3 &pcfg_pull_none>, + /* cif_data4 */ + <2 RK_PA4 3 &pcfg_pull_none>, + /* cif_data5 */ + <2 RK_PA5 3 &pcfg_pull_none>, + /* cif_data6 */ + <2 RK_PA6 3 &pcfg_pull_none>, + /* cif_data7 */ + <2 RK_PA7 3 &pcfg_pull_none>, + /* cif_sync */ + <2 RK_PB0 3 &pcfg_pull_none>, + /* cif_href */ + <2 RK_PB1 3 &pcfg_pull_none>, + /* cif_clkin */ + <2 RK_PB2 3 &pcfg_pull_none>; + }; + + isp_shutter: isp-shutter { + rockchip,pins = + /* SHUTTEREN */ + <1 RK_PA1 1 &pcfg_pull_none>, + /* SHUTTERTRIG */ + <1 RK_PA0 1 &pcfg_pull_none>; + }; + + isp_flash_trigger: isp-flash-trigger { + /* ISP_FLASHTRIGOU */ + rockchip,pins = <1 RK_PA3 1 &pcfg_pull_none>; + }; + + isp_flash_trigger_as_gpio: isp-flash-trigger-as-gpio { + /* ISP_FLASHTRIGOU */ + rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + cam_pins { + cam0_default_pins: cam0-default-pins { + rockchip,pins = + <4 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PB3 3 &pcfg_pull_none>; + }; + cam0_sleep_pins: cam0-sleep-pins { + rockchip,pins = + <4 RK_PD3 3 &pcfg_pull_none>, + <2 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-mid-818-android.dts b/arch/arm64/boot/dts/rockchip/rk3399-mid-818-android.dts new file mode 100755 index 000000000000..8610539ef284 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-mid-818-android.dts @@ -0,0 +1,1121 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include +#include +#include +#include +#include "rk3399.dtsi" +#include "rk3399-android.dtsi" +#include "rk3399-opp.dtsi" +#include "rk3399-vop-clk-set.dtsi" + +/ { + compatible = "rockchip,rk3399-mid", "rockchip,rk3399"; + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff1a0000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init initrd=0x62000001,0x00800000 coherent_pool=1m"; + }; + + adc_keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1000>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <170000>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio4 RK_PD4 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <6700>; + rockchip,screen-on-voltage = <6800>; + status = "okay"; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk818 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ + }; + + hall_sensor: hall-mh248 { + compatible = "hall-mh248"; + pinctrl-names = "default"; + pinctrl-0 = <&mh248_irq_gpio>; + irq-gpio = <&gpio1 2 IRQ_TYPE_EDGE_BOTH>; + hall-active = <1>; + status = "okay"; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3900000>; + regulator-max-microvolt = <3900000>; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vdd_log: vdd-log { + compatible = "pwm-regulator"; + pwms = <&pwm2 0 25000 1>; + rockchip,pwm_id= <2>; + rockchip,pwm_voltage = <900000>; + regulator-name = "vdd_log"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-boot-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + + edp_panel: edp-panel { + compatible = "lg,lp079qx1-sp0v", "panel-simple"; + bus-format = ; + bpc = <6>; + backlight = <&backlight>; + power-supply = <&vcc3v3_s0>; + enable-gpios = <&gpio3 8 GPIO_ACTIVE_HIGH>; + ports { + panel_in_edp: endpoint { + remote-endpoint = <&edp_out_panel>; + }; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 255 200 199 198 197 197 196 195 194 193 193 192 + 191 190 189 189 188 187 186 185 185 184 183 182 + 181 181 180 179 178 177 177 176 175 174 173 173 + 172 171 170 169 169 168 167 166 165 165 164 163 + 162 161 161 160 159 158 157 157 156 155 154 153 + 153 152 151 150 149 149 148 147 146 145 145 144 + 143 142 141 141 140 139 138 137 137 136 135 134 + 133 133 132 131 130 129 129 128 127 126 125 125 + 124 123 122 121 121 120 119 118 117 117 116 115 + 114 113 113 112 111 110 109 109 108 107 106 105 + 105 104 103 102 101 101 100 99 98 97 97 96 + 95 94 93 93 92 91 90 89 89 88 87 86 + 85 85 84 83 82 81 81 80 79 78 77 77 + 76 75 74 73 73 72 71 70 69 69 68 67 + 66 65 65 64 63 62 61 61 60 59 58 57 + 57 56 55 54 53 53 52 51 50 49 49 48 + 47 46 45 45 44 43 42 41 41 40 39 38 + 37 37 36 35 34 33 33 32 31 30 29 29 + 28 27 26 25 25 24 23 22 21 21 20 19 + 18 17 17 16 15 14 13 13 12 11 10 9 + 9 8 7 6 5 5 4 3 2 1 1 0 + 0 0 0 0>; + default-brightness-level = <200>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + es8316-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,es8316-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s0>; + }; + simple-audio-card,codec { + sound-dai = <&es8316>; + }; + }; + + spdif-sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,spdif"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk818 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk818 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>, <&bt_reset_gpio>, <&bt_wake_gpio>, <&bt_irq_gpio>; + pinctrl-1 = <&uart0_gpios>; + //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ + BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ + BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ + BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ + status = "okay"; + }; + + vibrator { + compatible = "rk-vibrator-gpio"; + vibrator-gpio = <&gpio4 30 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; + upthreshold = <40>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 800000 + SYS_STATUS_REBOOT 528000 + SYS_STATUS_SUSPEND 200000 + SYS_STATUS_VIDEO_1080P 200000 + SYS_STATUS_VIDEO_4K 600000 + SYS_STATUS_VIDEO_4K_10B 800000 + SYS_STATUS_PERFORMANCE 800000 + SYS_STATUS_BOOST 600000 + SYS_STATUS_DUALVIEW 600000 + SYS_STATUS_ISP 600000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 762 200000 + 763 1893 400000 + 1894 3012 528000 + 3013 99999 800000 + >; + + auto-min-freq = <200000>; +}; + +&sdmmc { + clock-frequency = <50000000>; + clock-freq-min-max = <400000 150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + num-slots = <1>; + //sd-uhs-sdr104; + vqmmc-supply = <&vcc_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "okay"; +}; + +&sdio0 { + clock-frequency = <150000000>; + clock-freq-min-max = <200000 150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&emmc_phy { + status = "okay"; +}; + +&sdhci { + bus-width = <8>; + mmc-hs400-1_8v; + no-sdio; + no-sd; + non-removable; + keep-power-in-suspend; + mmc-hs400-enhanced-strobe; + status = "okay"; +}; + +&i2s0 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <8>; + #sound-dai-cells = <0>; +}; + +&i2s2 { + #sound-dai-cells = <0>; +}; + +&spdif { + status = "disabled"; + #sound-dai-cells = <0>; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <180>; + i2c-scl-falling-time-ns = <30>; + clock-frequency = <400000>; + + vdd_cpu_b: syr837@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + vin-supply = <&vcc_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: syr828@41 { + compatible = "silergy,syr828"; + status = "okay"; + reg = <0x41>; + vin-supply = <&vcc_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <735000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk818: pmic@1c { + compatible = "rockchip,rk818"; + status = "okay"; + reg = <0x1c>; + clock-output-names = "rk818-clkout1", "wifibt_32kin"; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + rk818,support_dc_chg = <1>;/*1: dc chg; 0:usb chg*/ + wakeup-source; + extcon = <&fusb0>; + #clock-cells = <1>; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc_sys>; + vcc9-supply = <&vcc3v3_sys>; + + regulators { + vdd_cpu_l: DCDC_REG1 { + regulator-name = "vdd_cpu_l"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_center: DCDC_REG2 { + regulator-name = "vdd_center"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-name = "vcc_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcca3v0_codec: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcca3v0_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v0_tp: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc3v0_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca1v8_codec: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_power_on: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_power_on"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_3v0: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcc1v8_dvp: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_s3: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_s3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v3_s0: SWITCH_REG { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s0"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + boost_otg: DCDC_BOOST { + regulator-name = "boost_otg"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + }; + }; + + battery { + compatible = "rk818-battery"; + ocv_table = <3400 3675 3689 3716 3740 3756 3768 3780 + 3793 3807 3827 3853 3896 3937 3974 4007 4066 + 4110 4161 4217 4308>; + design_capacity = <7916>; + design_qmax = <8708>; + bat_res = <65>; + max_input_current = <3000>; + max_chrg_current = <3000>; + max_chrg_voltage = <4350>; + sleep_enter_current = <300>; + sleep_exit_current = <300>; + power_off_thresd = <3400>; + zero_algorithm_vol = <3950>; + fb_temperature = <105>; + sample_res = <20>; + max_soc_offset = <60>; + energy_mode = <0>; + monitor_sec = <5>; + virtual_power = <0>; + power_dc2otg = <0>; + }; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <140>; + i2c-scl-falling-time-ns = <30>; + + es8316: es8316@10 { + #sound-dai-cells = <0>; + compatible = "everest,es8316"; + reg = <0x11>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + spk-con-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + clock-frequency = <400000>; + + lsm330_accel@1e { + status = "okay"; + compatible = "lsm330_acc"; + pinctrl-names = "default"; + pinctrl-0 = <&lsm330a_irq_gpio>; + reg = <0x1e>; + irq-gpio = <&gpio2 27 IRQ_TYPE_EDGE_RISING>; + type = ; + irq_enable = <1>; + poll_delay_ms = <30>; + power-off-in-suspend = <1>; + layout = <4>; + }; + + lsm330_gyro@6a { + status = "okay"; + compatible = "lsm330_gyro"; + pinctrl-names = "default"; + pinctrl-0 = <&lsm330g_irq_gpio>; + reg = <0x6a>; + irq-gpio = <&gpio1 20 IRQ_TYPE_EDGE_RISING>; + type = ; + irq_enable = <0>; + power-off-in-suspend = <1>; + poll_delay_ms = <30>; + }; + + mpu6500@68 { + status = "disabled"; + compatible = "invensense,mpu6500"; + pinctrl-names = "default"; + pinctrl-0 = <&mpu6500_irq_gpio>; + reg = <0x68>; + irq-gpio = <&gpio2 27 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <1 0 0 0 1 0 0 0 1>; + orientation-x= <1>; + orientation-y= <1>; + orientation-z= <0>; + support-hw-poweroff = <1>; + mpu-debug = <1>; + }; + + sensor@0d { + status = "okay"; + compatible = "ak8963"; + pinctrl-names = "default"; + pinctrl-0 = <&ak8963_irq_gpio>; + reg = <0x0d>; + type = ; + irq-gpio = <&gpio2 28 IRQ_TYPE_EDGE_RISING>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <3>; + }; + + sensor@10 { + status = "okay"; + compatible = "capella,light_cm3218"; + pinctrl-names = "default"; + pinctrl-0 = <&cm3218_irq_gpio>; + reg = <0x10>; + type = ; + irq-gpio = <&gpio4 24 IRQ_TYPE_EDGE_FALLING>; + irq_enable = <1>; + poll_delay_ms = <30>; + }; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + int-n-gpios = <&gpio1 1 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&i2c5 { + status = "okay"; + i2c-scl-rising-time-ns = <150>; + i2c-scl-falling-time-ns = <30>; + clock-frequency = <400000>; + + gt9xx: gt9xx@14 { + compatible = "goodix,gt9xx"; + reg = <0x14>; + touch-gpio = <&gpio3 12 IRQ_TYPE_LEVEL_LOW>; + reset-gpio = <&gpio3 13 GPIO_ACTIVE_HIGH>; + max-x = <1536>; + max-y = <2048>; + tp-size = <970>; + tp-supply = <&vcc3v0_tp>; + }; +}; + +&io_domains { + status = "okay"; + + bt656-supply = <&vcc1v8_dvp>; + audio-supply = <&vcca1v8_codec>; + sdmmc-supply = <&vcc_sd>; + gpio1830-supply = <&vcc_3v0>; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&spi1 { + status = "disabled"; + max-freq = <50000000>; + mpu6500@0 { + status = "disabled"; + compatible = "inv-spi,mpu6500"; + pinctrl-names = "default"; + pinctrl-0 = <&mpu6500_irq_gpio>; + irq-gpio = <&gpio2 27 IRQ_TYPE_EDGE_RISING>; + reg = <0>; + spi-max-frequency = <1000000>; + spi-cpha; + spi-cpol; + mpu-int_config = <0x00>; + mpu-level_shifter = <0>; + mpu-orientation = <1 0 0 0 1 0 0 0 1>; + orientation-x= <1>; + orientation-y= <0>; + orientation-z= <1>; + support-hw-poweroff = <1>; + mpu-debug = <1>; + }; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm2 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin_pull_down>; +}; + +&saradc { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_OSC_DIS + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + ) + >; + rockchip,pwm-regulator-config = < + (0 + | PWM2_REGULATOR_EN + ) + >; + rockchip,power-ctrl = + <&gpio1 17 GPIO_ACTIVE_HIGH>, + <&gpio1 14 GPIO_ACTIVE_HIGH>; +}; + +&pinctrl { + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_reset_gpio: bt-reset-gpio { + rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_wake_gpio: bt-wake-gpio { + rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_irq_gpio: bt-irq-gpio { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + pmic_dvs2: pmic-dvs2 { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + hallsensor { + mh248_irq_gpio: mh248-irq-gpio { + rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lsm330_a { + lsm330a_irq_gpio: lsm330a-irq-gpio { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + lsm330_g { + lsm330g_irq_gpio: lsm330g-irq-gpio { + rockchip,pins = <1 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + mpu6500 { + mpu6500_irq_gpio: mpu6500-irq-gpio { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + ak8963 { + ak8963_irq_gpio: ak8963-irq-gpio { + rockchip,pins = <2 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + cm3218 { + cm3218_irq_gpio: cm3218-irq-gpio { + rockchip,pins = <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&edp { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&edp_hpd>; + + ports { + edp_out: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + edp_out_panel: endpoint@0 { + reg = <0>; + remote-endpoint = <&panel_in_edp>; + }; + }; + }; +}; + +&edp_in_vopl { + status = "disabled"; +}; + +&hdmi { + status = "disabled"; +}; + +&hdmi_in_vopb { + status = "disabled"; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&dp_in_vopb { + status = "disabled"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&route_edp { + status = "okay"; + logo,mode = "center"; +}; + +&vopb { + assigned-clocks = <&cru DCLK_VOP0_DIV>; + assigned-clock-parents = <&cru PLL_CPLL>; +}; + +&vopl { + assigned-clocks = <&cru DCLK_VOP1_DIV>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi index d6f1095abb04..f9f2cc8abec7 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-opp.dtsi @@ -3,35 +3,99 @@ * Copyright (c) 2016-2017 Fuzhou Rockchip Electronics Co., Ltd */ +#include "rk3399-sched-energy.dtsi" + / { cluster0_opp: opp-table0 { compatible = "operating-points-v2"; opp-shared; - opp00 { + rockchip,temp-hysteresis = <5000>; + rockchip,low-temp = <0>; + rockchip,low-temp-min-volt = <900000>; + + nvmem-cells = <&cpul_leakage>, <&specification_serial_number>, + <&customer_demand>; + nvmem-cell-names = "cpu_leakage", + "specification_serial_number", + "customer_demand"; + clocks = <&cru PLL_APLLL>; + rockchip,avs-scale = <20>; + rockchip,bin-scaling-sel = < + 0 30 + 1 34 + >; + + rockchip,pvtm-voltage-sel = < + 0 143500 0 + 143501 148500 1 + 148501 152000 2 + 152001 999999 3 + >; + rockchip,pvtm-freq = <408000>; + rockchip,pvtm-volt = <1000000>; + rockchip,pvtm-ch = <0 0>; + rockchip,pvtm-sample-time = <1000>; + rockchip,pvtm-number = <10>; + rockchip,pvtm-error = <1000>; + rockchip,pvtm-ref-temp = <41>; + rockchip,pvtm-temp-prop = <115 66>; + rockchip,pvtm-thermal-zone = "soc-thermal"; + + opp-408000000 { opp-hz = /bits/ 64 <408000000>; - opp-microvolt = <800000>; + opp-microvolt = <825000 825000 1250000>; + opp-microvolt-L0 = <825000 825000 1250000>; + opp-microvolt-L1 = <825000 825000 1250000>; + opp-microvolt-L2 = <825000 825000 1250000>; + opp-microvolt-L3 = <825000 825000 1250000>; clock-latency-ns = <40000>; }; - opp01 { + opp-600000000 { opp-hz = /bits/ 64 <600000000>; - opp-microvolt = <800000>; + opp-microvolt = <825000 825000 1250000>; + opp-microvolt-L0 = <825000 825000 1250000>; + opp-microvolt-L1 = <825000 825000 1250000>; + opp-microvolt-L2 = <825000 825000 1250000>; + opp-microvolt-L3 = <825000 825000 1250000>; + clock-latency-ns = <40000>; }; - opp02 { + opp-816000000 { opp-hz = /bits/ 64 <816000000>; - opp-microvolt = <850000>; + opp-microvolt = <850000 850000 1250000>; + opp-microvolt-L0 = <850000 850000 1250000>; + opp-microvolt-L1 = <825000 825000 1250000>; + opp-microvolt-L2 = <825000 825000 1250000>; + opp-microvolt-L3 = <825000 825000 1250000>; + clock-latency-ns = <40000>; + opp-suspend; }; - opp03 { + opp-1008000000 { opp-hz = /bits/ 64 <1008000000>; - opp-microvolt = <925000>; + opp-microvolt = <925000 925000 1250000>; + opp-microvolt-L0 = <925000 925000 1250000>; + opp-microvolt-L1 = <900000 900000 1250000>; + opp-microvolt-L2 = <875000 875000 1250000>; + opp-microvolt-L3 = <850000 850000 1250000>; + clock-latency-ns = <40000>; }; - opp04 { + opp-1200000000 { opp-hz = /bits/ 64 <1200000000>; - opp-microvolt = <1000000>; + opp-microvolt = <1000000 1000000 1250000>; + opp-microvolt-L0 = <1000000 1000000 1250000>; + opp-microvolt-L1 = <975000 975000 1250000>; + opp-microvolt-L2 = <950000 950000 1250000>; + opp-microvolt-L3 = <925000 925000 1250000>; + clock-latency-ns = <40000>; }; - opp05 { + opp-1416000000 { opp-hz = /bits/ 64 <1416000000>; - opp-microvolt = <1125000>; + opp-microvolt = <1125000 1125000 1250000>; + opp-microvolt-L0 = <1125000 1125000 1250000>; + opp-microvolt-L1 = <1100000 1100000 1250000>; + opp-microvolt-L2 = <1075000 1075000 1200000>; + opp-microvolt-L3 = <1050000 1050000 1250000>; + clock-latency-ns = <40000>; }; }; @@ -39,93 +103,253 @@ cluster1_opp: opp-table1 { compatible = "operating-points-v2"; opp-shared; - opp00 { + rockchip,temp-hysteresis = <5000>; + rockchip,low-temp = <0>; + rockchip,low-temp-min-volt = <900000>; + + nvmem-cells = <&cpub_leakage>, <&specification_serial_number>, + <&customer_demand>; + nvmem-cell-names = "cpu_leakage", + "specification_serial_number", + "customer_demand"; + clocks = <&cru PLL_APLLB>; + rockchip,avs-scale = <8>; + rockchip,bin-scaling-sel = < + 0 8 + 1 17 + >; + + rockchip,pvtm-voltage-sel = < + 0 149000 0 + 149001 155000 1 + 155001 159000 2 + 159001 161000 3 + 161001 999999 4 + >; + rockchip,pvtm-freq = <408000>; + rockchip,pvtm-volt = <1000000>; + rockchip,pvtm-ch = <1 0>; + rockchip,pvtm-sample-time = <1000>; + rockchip,pvtm-number = <10>; + rockchip,pvtm-error = <1000>; + rockchip,pvtm-ref-temp = <41>; + rockchip,pvtm-temp-prop = <71 35>; + rockchip,pvtm-thermal-zone = "soc-thermal"; + + opp-408000000 { opp-hz = /bits/ 64 <408000000>; - opp-microvolt = <800000>; + opp-microvolt = <825000 825000 1250000>; + opp-microvolt-L0 = <825000 825000 1250000>; + opp-microvolt-L1 = <825000 825000 1250000>; + opp-microvolt-L2 = <825000 825000 1250000>; + opp-microvolt-L3 = <825000 825000 1250000>; + opp-microvolt-L4 = <825000 825000 1250000>; clock-latency-ns = <40000>; }; - opp01 { + opp-600000000 { opp-hz = /bits/ 64 <600000000>; - opp-microvolt = <800000>; + opp-microvolt = <825000 825000 1250000>; + opp-microvolt-L0 = <825000 825000 1250000>; + opp-microvolt-L1 = <825000 825000 1250000>; + opp-microvolt-L2 = <825000 825000 1250000>; + opp-microvolt-L3 = <825000 825000 1250000>; + opp-microvolt-L4 = <825000 825000 1250000>; + clock-latency-ns = <40000>; }; - opp02 { + opp-816000000 { opp-hz = /bits/ 64 <816000000>; - opp-microvolt = <825000>; + opp-microvolt = <825000 825000 1250000>; + opp-microvolt-L0 = <825000 825000 1250000>; + opp-microvolt-L1 = <825000 825000 1250000>; + opp-microvolt-L2 = <825000 825000 1250000>; + opp-microvolt-L3 = <825000 825000 1250000>; + opp-microvolt-L4 = <825000 825000 1250000>; + clock-latency-ns = <40000>; + opp-suspend; }; - opp03 { + opp-1008000000 { opp-hz = /bits/ 64 <1008000000>; - opp-microvolt = <875000>; + opp-microvolt = <875000 875000 1250000>; + opp-microvolt-L0 = <875000 875000 1250000>; + opp-microvolt-L1 = <850000 850000 1250000>; + opp-microvolt-L2 = <850000 850000 1250000>; + opp-microvolt-L3 = <850000 850000 1250000>; + opp-microvolt-L4 = <850000 850000 1250000>; + clock-latency-ns = <40000>; }; - opp04 { + opp-1200000000 { opp-hz = /bits/ 64 <1200000000>; - opp-microvolt = <950000>; + opp-microvolt = <950000 950000 1250000>; + opp-microvolt-L0 = <950000 950000 1250000>; + opp-microvolt-L1 = <925000 925000 1250000>; + opp-microvolt-L2 = <900000 900000 1250000>; + opp-microvolt-L3 = <875000 875000 1250000>; + opp-microvolt-L4 = <875000 875000 1250000>; + clock-latency-ns = <40000>; }; - opp05 { + opp-1416000000 { opp-hz = /bits/ 64 <1416000000>; - opp-microvolt = <1025000>; + opp-microvolt = <1025000 1025000 1250000>; + opp-microvolt-L0 = <1025000 1025000 1250000>; + opp-microvolt-L1 = <1000000 1000000 1250000>; + opp-microvolt-L2 = <1000000 1000000 1250000>; + opp-microvolt-L3 = <975000 975000 1250000>; + opp-microvolt-L4 = <975000 975000 1250000>; + clock-latency-ns = <40000>; }; - opp06 { + opp-1608000000 { opp-hz = /bits/ 64 <1608000000>; - opp-microvolt = <1100000>; + opp-microvolt = <1100000 1100000 1250000>; + opp-microvolt-L0 = <1100000 1100000 1250000>; + opp-microvolt-L1 = <1075000 1075000 1250000>; + opp-microvolt-L2 = <1050000 1050000 1250000>; + opp-microvolt-L3 = <1025000 1025000 1250000>; + opp-microvolt-L4 = <1025000 1025000 1250000>; + clock-latency-ns = <40000>; }; - opp07 { + opp-1800000000 { opp-hz = /bits/ 64 <1800000000>; - opp-microvolt = <1200000>; + opp-microvolt = <1200000 1200000 1250000>; + opp-microvolt-L0 = <1200000 1200000 1250000>; + opp-microvolt-L1 = <1175000 1175000 1250000>; + opp-microvolt-L2 = <1150000 1150000 1250000>; + opp-microvolt-L3 = <1125000 1125000 1250000>; + opp-microvolt-L4 = <1100000 1100000 1250000>; + clock-latency-ns = <40000>; }; }; gpu_opp_table: opp-table2 { compatible = "operating-points-v2"; - opp00 { + rockchip,thermal-zone = "soc-thermal"; + rockchip,temp-hysteresis = <5000>; + rockchip,low-temp = <0>; + rockchip,low-temp-min-volt = <900000>; + + nvmem-cells = <&gpu_leakage>; + nvmem-cell-names = "gpu_leakage"; + + rockchip,pvtm-voltage-sel = < + 0 121000 0 + 121001 125500 1 + 125501 128500 2 + 128501 999999 3 + >; + rockchip,pvtm-freq = <200000>; + rockchip,pvtm-volt = <900000>; + rockchip,pvtm-ch = <3 0>; + rockchip,pvtm-sample-time = <1000>; + rockchip,pvtm-number = <10>; + rockchip,pvtm-error = <1000>; + rockchip,pvtm-ref-temp = <41>; + rockchip,pvtm-temp-prop = <46 12>; + rockchip,pvtm-thermal-zone = "gpu-thermal"; + + opp-200000000 { opp-hz = /bits/ 64 <200000000>; - opp-microvolt = <800000>; + opp-microvolt = <825000>; + opp-microvolt-L0 = <825000>; + opp-microvolt-L1 = <825000>; + opp-microvolt-L2 = <825000>; + opp-microvolt-L3 = <825000>; }; - opp01 { - opp-hz = /bits/ 64 <297000000>; - opp-microvolt = <800000>; + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <825000>; + opp-microvolt-L0 = <825000>; + opp-microvolt-L1 = <825000>; + opp-microvolt-L2 = <825000>; + opp-microvolt-L3 = <825000>; }; - opp02 { + opp-400000000 { opp-hz = /bits/ 64 <400000000>; opp-microvolt = <825000>; + opp-microvolt-L0 = <825000>; + opp-microvolt-L1 = <825000>; + opp-microvolt-L2 = <825000>; + opp-microvolt-L3 = <825000>; }; - opp03 { - opp-hz = /bits/ 64 <500000000>; - opp-microvolt = <875000>; - }; - opp04 { + opp-600000000 { opp-hz = /bits/ 64 <600000000>; opp-microvolt = <925000>; + opp-microvolt-L0 = <925000>; + opp-microvolt-L1 = <925000>; + opp-microvolt-L2 = <900000>; + opp-microvolt-L3 = <900000>; }; - opp05 { + opp-800000000 { opp-hz = /bits/ 64 <800000000>; opp-microvolt = <1100000>; + opp-microvolt-L0 = <1100000>; + opp-microvolt-L1 = <1075000>; + opp-microvolt-L2 = <1050000>; + opp-microvolt-L3 = <1025000>; + }; + }; + + dmc_opp_table: opp-table3 { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <900000>; + }; + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <900000>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <900000>; + }; + opp-528000000 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <900000>; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <900000>; + }; + opp-800000000 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <900000>; }; }; }; &cpu_l0 { operating-points-v2 = <&cluster0_opp>; + sched-energy-costs = <&RK3399_CPU_COST_0 &RK3399_CLUSTER_COST_0>; }; &cpu_l1 { operating-points-v2 = <&cluster0_opp>; + sched-energy-costs = <&RK3399_CPU_COST_0 &RK3399_CLUSTER_COST_0>; }; &cpu_l2 { operating-points-v2 = <&cluster0_opp>; + sched-energy-costs = <&RK3399_CPU_COST_0 &RK3399_CLUSTER_COST_0>; }; &cpu_l3 { operating-points-v2 = <&cluster0_opp>; + sched-energy-costs = <&RK3399_CPU_COST_0 &RK3399_CLUSTER_COST_0>; }; &cpu_b0 { operating-points-v2 = <&cluster1_opp>; + sched-energy-costs = <&RK3399_CPU_COST_1 &RK3399_CLUSTER_COST_1>; }; &cpu_b1 { operating-points-v2 = <&cluster1_opp>; + sched-energy-costs = <&RK3399_CPU_COST_1 &RK3399_CLUSTER_COST_1>; +}; + +&dmc { + operating-points-v2 = <&dmc_opp_table>; }; &gpu { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi index 35b7ab3bf10c..20309076dbac 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi @@ -384,7 +384,6 @@ regulator-state-mem { vcc_sdio: LDO_REG4 { regulator-name = "vcc_sdio"; - regulator-always-on; regulator-boot-on; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3000000>; @@ -489,8 +488,6 @@ vdd_gpu: regulator@41 { regulator-min-microvolt = <712500>; regulator-max-microvolt = <1500000>; regulator-ramp-delay = <1000>; - regulator-always-on; - regulator-boot-on; vin-supply = <&vcc3v3_sys>; regulator-state-mem { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock960-ab.dts b/arch/arm64/boot/dts/rockchip/rk3399-rock960-ab.dts new file mode 100755 index 000000000000..86e45dc0074d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-rock960-ab.dts @@ -0,0 +1,1088 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; + +#include +#include +#include "rk3399.dtsi" +#include "rk3399-linux.dtsi" +#include "rk3399-opp.dtsi" + +/ { + + model = "ROCK960 - 96boards based on Rockchip RK3399"; + compatible = "rockchip,rock960","rockchip,rk3399"; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,signal-irq = <182>; + rockchip,wake-irq = <0>; + rockchip,irq-mode-enable = <1>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + pinctrl-names = "default"; + pinctrl-0 = <&uart2c_xfer>; + }; + + vcc1v8_s0: vcc1v8-s0 { + compatible = "regulator-fixed"; + regulator-name = "vcc1v8_s0"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + vin-supply = <&vcc_sys>; + }; + + vcc3v3_pcie: vcc3v3-pcie-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio3 11 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_drv>; + regulator-boot-on; + regulator-always-on; + regulator-name = "vcc3v3_pcie"; + vin-supply = <&vcc3v3_sys>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vdd_log: vdd-log { + compatible = "pwm-regulator"; + pwms = <&pwm2 0 25000 1>; + regulator-name = "vdd_log"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1400000>; + regulator-always-on; + regulator-boot-on; + + /* for rockchip boot on */ + rockchip,pwm_id= <2>; + rockchip,pwm_voltage = <900000>; + + vin-supply = <&vcc_sys>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + hdmi_codec: hdmi-codec { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "HDMI-CODEC"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + spdif-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "okay"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + /* wifi-bt-power-toggle; */ + uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_gpios>; + /* BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; */ + BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 27 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + test-power { + status = "okay"; + }; +}; + +&hdmi { + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + status = "okay"; +}; + +&sdmmc { + clock-frequency = <100000000>; + clock-freq-min-max = <100000 100000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + num-slots = <1>; + //sd-uhs-sdr104; + vqmmc-supply = <&vcc_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + card-detect-delay = <800>; + status = "okay"; +}; + +&sdio0 { + clock-frequency = <100000000>; + clock-freq-min-max = <200000 100000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&emmc_phy { + status = "okay"; +}; + +&sdhci { + bus-width = <8>; + mmc-hs400-1_8v; + no-sdio; + no-sd; + non-removable; + mmc-hs400-enhanced-strobe; + status = "okay"; +}; + +&i2s0 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <8>; + #sound-dai-cells = <0>; +}; + +&i2s2 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&spdif { + pinctrl-0 = <&spdif_bus_1>; + status = "okay"; + #sound-dai-cells = <0>; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <168>; + i2c-scl-falling-time-ns = <4>; + clock-frequency = <400000>; + + vdd_cpu_b: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_sys>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: syr828@41 { + compatible = "silergy,syr828"; + reg = <0x41>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_sys>; + regulator-initial-mode = <1>; /* 1:force PWM 2:auto */ + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "xin32k", "rk808-clkout2"; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc_sys>; + vcc10-supply = <&vcc_sys>; + vcc11-supply = <&vcc_sys>; + vcc12-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc_1v8>; + + regulators { + vdd_center: DCDC_REG1 { + regulator-name = "vdd_center"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-name = "vdd_cpu_l"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-name = "vcc_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-name = "vcc_1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc1v8_dvp: LDO_REG1 { + regulator-name = "vcc1v8_dvp"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcca1v8_hdmi: LDO_REG2 { + regulator-name = "vcca1v8_hdmi"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcca_1v8: LDO_REG3 { + regulator-name = "vcca_1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sd: LDO_REG4 { + regulator-name = "vcc_sd"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v0_sd: LDO_REG5 { + regulator-name = "vcc3v0_sd"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-name = "vcc_1v5"; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcca0v9_hdmi: LDO_REG7 { + regulator-name = "vcca0v9_hdmi"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vcc_3v0: LDO_REG8 { + regulator-name = "vcc_3v0"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v3_s3: SWITCH_REG1 { + regulator-name = "vcc3v3_s3"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_s0: SWITCH_REG2 { + regulator-name = "vcc3v3_s0"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + }; + }; +}; + +&i2c1 { + status = "okay"; +}; + +&i2c6 { + status = "okay"; +}; + +&i2c4 { + status = "okay"; + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + vbus-5v-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&i2c2 { + status = "okay"; + camera0: camera-module@10 { + status = "disabled"; + compatible = "omnivision,ov13850-v4l2-i2c-subdev"; + reg = < 0x10 >; + device_type = "v4l2-i2c-subdev"; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "clk_cif_out"; + pinctrl-names = "rockchip,camera_default", + "rockchip,camera_sleep"; + pinctrl-0 = <&cam0_default_pins>; + pinctrl-1 = <&cam0_sleep_pins>; + //rockchip,pd-gpio = <&gpio4 4 GPIO_ACTIVE_LOW>; + rockchip,pwr-gpio = <&gpio4 4 GPIO_ACTIVE_HIGH>; + rockchip,rst-gpio = <&gpio3 29 GPIO_ACTIVE_LOW>; + rockchip,camera-module-mclk-name = "clk_cif_out"; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "cmk-cb0695-fv1"; + rockchip,camera-module-len-name = "lg9569a2"; + rockchip,camera-module-fov-h = "66.0"; + rockchip,camera-module-fov-v = "50.1"; + rockchip,camera-module-orientation = <0>; + rockchip,camera-module-iq-flip = <0>; + rockchip,camera-module-iq-mirror = <0>; + rockchip,camera-module-flip = <1>; + rockchip,camera-module-mirror = <0>; + + rockchip,camera-module-defrect0 = <2112 1568 0 0 2112 1568>; + rockchip,camera-module-defrect1 = <4224 3136 0 0 4224 3136>; + rockchip,camera-module-defrect3 = <3264 2448 0 0 3264 2448>; + rockchip,camera-module-flash-support = <1>; + rockchip,camera-module-mipi-dphy-index = <0>; + }; + + camera1: camera-module@36 { + status = "disabled"; + compatible = "omnivision,ov4690-v4l2-i2c-subdev"; + reg = <0x36>; + device_type = "v4l2-i2c-subdev"; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "clk_cif_out"; + pinctrl-names = "rockchip,camera_default", + "rockchip,camera_sleep"; + pinctrl-0 = <&cam0_default_pins>; + pinctrl-1 = <&cam0_sleep_pins>; + rockchip,pd-gpio = <&gpio3 4 GPIO_ACTIVE_LOW>; + //rockchip,pwr-gpio = <&gpio3 13 0>; + rockchip,rst-gpio = <&gpio2 10 GPIO_ACTIVE_LOW>; + rockchip,camera-module-mclk-name = "clk_cif_out"; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "LA6111PA"; + rockchip,camera-module-len-name = "YM6011P"; + rockchip,camera-module-fov-h = "116"; + rockchip,camera-module-fov-v = "61"; + rockchip,camera-module-orientation = <0>; + rockchip,camera-module-iq-flip = <0>; + rockchip,camera-module-iq-mirror = <0>; + rockchip,camera-module-flip = <0>; + rockchip,camera-module-mirror = <1>; + + rockchip,camera-module-defrect0 = <2688 1520 0 0 2688 1520>; + rockchip,camera-module-flash-support = <0>; + rockchip,camera-module-mipi-dphy-index = <0>; + }; + +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&threshold { + temperature = <85000>; +}; + +&target { + temperature = <100000>; +}; + +&soc_crit { + temperature = <105000>; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-mode = <1>; + /* tshut polarity 0:LOW 1:HIGH */ + rockchip,hw-tshut-polarity = <1>; + rockchip,hw-tshut-temp = <110000>; + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + dmas = <&dmac_peri 0>, <&dmac_peri 1>; + dma-names = "tx", "rx"; + status = "okay"; +}; + +&uart3 { + compatible = "rockchip,rk3399-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff1b0000 0x0 0x100>; + clocks = <&cru SCLK_UART3>, <&cru PCLK_UART3>; + clock-names = "baudclk", "apb_pclk"; + interrupts = ; + dmas = <&dmac_peri 6>, <&dmac_peri 7>; + dma-names = "tx", "rx"; + reg-shift = <2>; + reg-io-width = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&uart3_xfer &uart3_cts &uart3_rts>; + status = "okay"; +}; + +&uart4 { + status = "okay"; + dmas = <&dmac_peri 8>, <&dmac_peri 9>; + dma-names = "tx", "rx"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + dr_mode = "otg"; + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_1 { + dr_mode = "host"; + status = "okay"; +}; + +&pwm2 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin_pull_down>; +}; + +&pwm3 { + status = "okay"; + + interrupts = ; + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <1>; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_VOLUMEDOWN>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 15 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&rgmii_pins>; + pinctrl-1 = <&rgmii_sleep_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "disabled"; +}; + +&saradc { + status = "okay"; +}; + +&io_domains { + status = "okay"; + + bt656-supply = <&vcc1v8_s0>; /* bt656_gpio2ab_ms */ + audio-supply = <&vcc1v8_s0>; /* audio_gpio3d4a_ms */ + sdmmc-supply = <&vcc_sd>; /* sdmmc_gpio4b_ms */ + gpio1830-supply = <&vcc_3v0>; /* gpio1833_gpio4cd_ms */ +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + ep-gpios = <&gpio3 9 GPIO_ACTIVE_HIGH>; + num-lanes = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_clkreqn_cpm>; + status = "okay"; +}; + +&pinctrl { + + sdio0 { + sdio0_bus1: sdio0-bus1 { + rockchip,pins = + <2 RK_PC4 1 &pcfg_pull_up_20ma>; + }; + + sdio0_bus4: sdio0-bus4 { + rockchip,pins = + <2 RK_PC4 1 &pcfg_pull_up_20ma>, + <2 RK_PC5 1 &pcfg_pull_up_20ma>, + <2 RK_PC6 1 &pcfg_pull_up_20ma>, + <2 RK_PC7 1 &pcfg_pull_up_20ma>; + }; + + sdio0_cmd: sdio0-cmd { + rockchip,pins = + <2 RK_PD0 1 &pcfg_pull_up_20ma>; + }; + + sdio0_clk: sdio0-clk { + rockchip,pins = + <2 RK_PD1 1 &pcfg_pull_none_20ma>; + }; + }; + + sdmmc { + sdmmc_bus1: sdmmc-bus1 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_8ma>; + }; + + sdmmc_bus4: sdmmc-bus4 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_8ma>, + <4 RK_PB1 1 &pcfg_pull_up_8ma>, + <4 RK_PB2 1 &pcfg_pull_up_8ma>, + <4 RK_PB3 1 &pcfg_pull_up_8ma>; + }; + + sdmmc_clk: sdmmc-clk { + rockchip,pins = + <4 RK_PB4 1 &pcfg_pull_none_18ma>; + }; + + sdmmc_cmd: sdmmc-cmd { + rockchip,pins = + <4 RK_PB5 1 &pcfg_pull_up_8ma>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pcie { + pcie_drv: pcie-drv { + rockchip,pins = + <3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + gmac { + rgmii_sleep_pins: rgmii-sleep-pins { + rockchip,pins = + <3 RK_PB7 RK_FUNC_GPIO &pcfg_output_low>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pvtm { + status = "okay"; +}; + +&pmu_pvtm { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <0>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_PWM_WKUP_EN + ) + >; + rockchip,pwm-regulator-config = < + (0 + | PWM2_REGULATOR_EN + ) + >; + rockchip,power-ctrl = + <&gpio1 17 GPIO_ACTIVE_HIGH>, + <&gpio1 14 GPIO_ACTIVE_HIGH>; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&cif_isp0 { + rockchip,camera-modules-attached = <&camera0>; + status = "okay"; +}; + +&isp0_mmu { + status = "okay"; +}; + +&cif_isp1 { + rockchip,camera-modules-attached = <&camera1>; + status = "disabled"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&vpu { + status = "okay"; + /* 0 means ion, 1 means drm */ + //allocator = <0>; +}; + +&rkvdec { + status = "okay"; + /* 0 means ion, 1 means drm */ + //allocator = <0>; +}; + +&display_subsystem { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock960.dts b/arch/arm64/boot/dts/rockchip/rk3399-rock960.dts index c88295782e7b..1a23e8f3cdf6 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-rock960.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-rock960.dts @@ -63,6 +63,20 @@ bt_active_led: led-6 { }; +&cpu_alert0 { + temperature = <65000>; +}; + +&cpu_thermal { + sustainable-power = <1550>; + + cooling-maps { + map0 { + trip = <&cpu_alert1>; + }; + }; +}; + &pcie0 { ep-gpios = <&gpio2 RK_PA2 GPIO_ACTIVE_HIGH>; }; @@ -125,45 +139,6 @@ &spi4 { status = "okay"; }; -&thermal_zones { - cpu_thermal: cpu { - polling-delay-passive = <100>; - polling-delay = <1000>; - thermal-sensors = <&tsadc 0>; - sustainable-power = <1550>; - - trips { - cpu_alert0: cpu_alert0 { - temperature = <65000>; - hysteresis = <2000>; - type = "passive"; - }; - - cpu_alert1: cpu_alert1 { - temperature = <75000>; - hysteresis = <2000>; - type = "passive"; - }; - - cpu_crit: cpu_crit { - temperature = <95000>; - hysteresis = <2000>; - type = "critical"; - }; - }; - - cooling-maps { - map0 { - - trip = <&cpu_alert1>; - cooling-device = - <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_b1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; - }; -}; - &usbdrd_dwc3_0 { dr_mode = "otg"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-box.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-box.dts new file mode 100755 index 000000000000..3106512b87cd --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-box.dts @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include +#include "rk3399-excavator-sapphire.dtsi" +#include "rk3399-android.dtsi" + +/ { + compatible = "rockchip,rk3399-excavator-box", "rockchip,rk3399"; + + test-power { + status = "okay"; + }; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&hdmi_dp_sound { + status = "okay"; +}; + +&pwm3 { + status = "okay"; + interrupts = ; + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_VOLUMEDOWN>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp-avb.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp-avb.dts new file mode 100755 index 000000000000..3e9ef7b0773b --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp-avb.dts @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3399-sapphire-excavator-edp.dtsi" + +/ { + model = "Rockchip RK3399 Excavator Board edp avb (Android)"; + compatible = "rockchip,android", "rockchip,rk3399-excavator-edp-avb", "rockchip,rk3399"; + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xff1a0000 console=ttyFIQ0 androidboot.baseband=N/A androidboot.veritymode=enforcing androidboot.hardware=rk30board androidboot.console=ttyFIQ0 init=/init initrd=0x62000001,0x00800000 coherent_pool=1m"; + }; + + ext_cam_clk: external-camera-clock { + compatible = "fixed-clock"; + clock-frequency = <27000000>; + clock-output-names = "CLK_CAMERA_27MHZ"; + #clock-cells = <0>; + }; +}; + +&i2c1 { + status = "okay"; + + /delete-node/ tc358749x@0f; + + tc35874x: tc35874x@0f { + status = "disabled"; + reg = <0x0f>; + compatible = "toshiba,tc358749"; + clocks = <&ext_cam_clk>; + clock-names = "refclk"; + reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>; + /* interrupt-parent = <&gpio2>; */ + /* interrupts = <12 IRQ_TYPE_LEVEL_HIGH>; */ + pinctrl-names = "default"; + pinctrl-0 = <&tc35874x_gpios>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TC358749XBG"; + rockchip,camera-module-lens-name = "NC"; + + port { + hdmiin_out0: endpoint { + remote-endpoint = <&hdmi_to_mipi_in>; + data-lanes = <1 2 3 4>; + clock-noncontinuous; + link-frequencies = + /bits/ 64 <297000000>; + }; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + + hdmi_to_mipi_in: endpoint@2 { + reg = <2>; + remote-endpoint = <&hdmiin_out0>; + data-lanes = <1 2 3 4>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&mipi_dphy_tx1rx1 { + status = "disabled"; +}; + +&pinctrl { + hdmiin { + tc35874x_gpios: tc35874x_gpios { + rockchip,pins = + /* PWREN_3.3 */ + <2 RK_PA5 RK_FUNC_GPIO &pcfg_output_high>, + /* PWREN_1.2 */ + <2 RK_PA6 RK_FUNC_GPIO &pcfg_output_high>, + /* HDMIIN_RST */ + <2 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, + /* HDMIIN_STBY */ + <2 RK_PB0 RK_FUNC_GPIO &pcfg_output_high>, + /* MIPI_RST */ + <2 RK_PB1 RK_FUNC_GPIO &pcfg_output_high>, + /* CSI_CTL */ + <2 RK_PB2 RK_FUNC_GPIO &pcfg_output_low>, + /* HDMIIN_INT */ + <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&rkisp1_0 { + status = "okay"; +}; + +&rkisp1_1 { + status = "disabled"; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dts new file mode 100755 index 000000000000..c2f8673198d5 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dts @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3399-sapphire-excavator-edp.dtsi" + +/ { + model = "Rockchip RK3399 Excavator Board edp (Android)"; + compatible = "rockchip,android", "rockchip,rk3399-excavator-edp", "rockchip,rk3399"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&dp_sound { + status = "disabled"; +}; + +&hdmi_dp_sound { + status = "disabled"; +}; + +&hdmi_sound { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dtsi new file mode 100755 index 000000000000..8156e5f7c795 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-edp.dtsi @@ -0,0 +1,488 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "rk3399-excavator-sapphire.dtsi" +#include "rk3399-android.dtsi" +#include "rk3399-vop-clk-set.dtsi" + +/ { + vcc_lcd: vcc-lcd { + compatible = "regulator-fixed"; + regulator-name = "vcc_lcd"; + gpio = <&gpio4 30 GPIO_ACTIVE_HIGH>; + startup-delay-us = <20000>; + enable-active-high; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc_lcd>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + test-power { + status = "okay"; + }; + + hdmiin_sound: hdmiin-sound { + compatible = "rockchip,rockchip-rt5651-sound"; + rockchip,cpu = <&i2s0>; + rockchip,codec = <&rt5651 &rt5651>; + status = "okay"; + }; +}; + +&backlight { + status = "okay"; + enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopl { + status = "disabled"; +}; + +&hdmi_in_vopb { + status = "disabled"; +}; + +&rt5651 { + status = "okay"; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&hdmi_dp_sound { + status = "okay"; +}; + +&hdmiin_sound { + status = "disabled"; +}; + +&dp_in_vopb { + status = "disabled"; +}; + +&i2s2 { + status = "okay"; +}; + +&i2c1 { + status = "okay"; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; + }; + + sgm3784: sgm3784@30 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "sgmicro,gsm3784"; + reg = <0x30>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + enable-gpio = <&gpio2 RK_PB4 GPIO_ACTIVE_HIGH>; + strobe-gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; + status = "okay"; + sgm3784_led0: led@0 { + reg = <0x0>; + led-max-microamp = <299200>; + flash-max-microamp = <1122000>; + flash-max-timeout-us = <1600000>; + }; + + sgm3784_led1: led@1 { + reg = <0x1>; + led-max-microamp = <299200>; + flash-max-microamp = <1122000>; + flash-max-timeout-us = <1600000>; + }; + }; + + tc358749x: tc358749x@0f { + compatible = "toshiba,tc358749x"; + reg = <0x0f>; + power-gpios = <&gpio2 6 GPIO_ACTIVE_HIGH>; + power18-gpios = <&gpio2 9 GPIO_ACTIVE_HIGH>; + power33-gpios = <&gpio2 5 GPIO_ACTIVE_HIGH>; + csi-ctl-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; + stanby-gpios = <&gpio2 8 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio2 7 GPIO_ACTIVE_HIGH>; + int-gpios = <&gpio2 12 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmiin_gpios>; + status = "disabled"; + }; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + gc2145: gc2145@3c{ + status = "okay"; + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout>; + + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + pwdn-gpios = <&gpio2 28 GPIO_ACTIVE_HIGH>; //ok + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_out: endpoint { + remote-endpoint = <&dvp_in_fcam>; + }; + }; + }; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "okay"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + /* reset-gpios = <>; */ + reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; // conflict with csi-ctl-gpios + pwdn-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-CT0116"; + rockchip,camera-module-lens-name = "Largan-50013A1"; + lens-focus = <&vm149c>; + flash-leds = <&sgm3784_led0 &sgm3784_led1>; + + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + //remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2>; + }; + }; + }; + + ov4689: ov4689@36 { + compatible = "ovti,ov4689"; + status = "disabled"; + reg = <0x36>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + /* reset-gpios = <>; */ + pwdn-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; // conflict with backlight + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "JSD3425-C1"; + rockchip,camera-module-lens-name = "JSD3425-C1"; + port { + ucam_out1: endpoint { + //remote-endpoint = <&mipi_in_ucam0>; + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c6 { + cw2015@62 { + status = "disabled"; + compatible = "cw201x"; + reg = <0x62>; + bat_config_info = <0x15 0x42 0x60 0x59 0x52 0x58 0x4D 0x48 + 0x48 0x44 0x44 0x46 0x49 0x48 0x32 0x24 + 0x20 0x17 0x13 0x0F 0x19 0x3E 0x51 0x45 + 0x08 0x76 0x0B 0x85 0x0E 0x1C 0x2E 0x3E + 0x4D 0x52 0x52 0x57 0x3D 0x1B 0x6A 0x2D + 0x25 0x43 0x52 0x87 0x8F 0x91 0x94 0x52 + 0x82 0x8C 0x92 0x96 0xFF 0x7B 0xBB 0xCB + 0x2F 0x7D 0x72 0xA5 0xB5 0xC1 0x46 0xAE>; + monitor_sec = <5>; + virtual_power = <0>; + }; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&mipi_dphy_rx0 { + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&mipi_dphy_tx1rx1 { + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&vopb { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP0_DIV>; + assigned-clock-parents = <&cru PLL_CPLL>; +}; + +&vopl { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP1_DIV>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + status = "okay"; +}; + +&rkisp1_0 { + status = "disabled"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rkisp1_1 { + status = "disabled"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + dvp_in_fcam: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc2145_out>; + }; + }; +}; + +&route_edp { + status = "okay"; +}; + +&route_hdmi { + status = "okay"; + connect = <&vopl_out_hdmi>; +}; + +&rt5651_sound { + status = "okay"; +}; + +&pinctrl { + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + hdmiin { + hdmiin_gpios: hdmiin_gpios { + rockchip,pins = + <2 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux-for-rk1808-cascade.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux-for-rk1808-cascade.dts new file mode 100755 index 000000000000..e1e482938481 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux-for-rk1808-cascade.dts @@ -0,0 +1,487 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; + +#include "rk3399-excavator-sapphire.dtsi" +#include "rk3399-linux.dtsi" +#include + +/ { + model = "Rockchip RK3399 Excavator Board (Linux Opensource)"; + compatible = "rockchip,rk3399-excavator-linux", "rockchip,rk3399"; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,signal-irq = <182>; + rockchip,wake-irq = <0>; + rockchip,irq-mode-enable = <1>; /* If enable uart uses irq instead of fiq */ + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + pinctrl-names = "default"; + pinctrl-0 = <&uart2c_xfer>; + }; + + edp_panel: edp-panel { + compatible = "lg,lp079qx1-sp0v", "panel-simple"; + backlight = <&backlight>; + power-supply = <&vcc3v3_s0>; + enable-gpios = <&gpio4 30 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd_panel_reset>; + + ports { + panel_in_edp: endpoint { + remote-endpoint = <&edp_out_panel>; + }; + }; + }; + + hdmi_sound: hdmi-sound { + status = "okay"; + }; + + gpio-keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = <&pwrbtn>; + + button@0 { + gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "GPIO Key Power"; + linux,input-type = <1>; + gpio-key,wakeup = <1>; + debounce-interval = <100>; + }; + }; + + vccadc_ref: vccadc-ref { + compatible = "regulator-fixed"; + regulator-name = "vcc1v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + ext_cam_clk: external-camera-clock { + compatible = "fixed-clock"; + clock-frequency = <27000000>; + clock-output-names = "CLK_CAMERA_27MHZ"; + #clock-cells = <0>; + }; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + button-up { + label = "Volume Up"; + linux,code = ; + press-threshold-microvolt = <100000>; + }; + + button-down { + label = "Volume Down"; + linux,code = ; + press-threshold-microvolt = <300000>; + }; + + back { + label = "Back"; + linux,code = ; + press-threshold-microvolt = <985000>; + }; + + menu { + label = "Menu"; + linux,code = ; + press-threshold-microvolt = <1314000>; + }; + }; +}; + +&rkisp1_0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&isp0_mmu { + status = "okay"; +}; + +&rkisp1_1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + }; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + /* Unlinked camera */ + //remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&isp1_mmu { + status = "okay"; +}; + +&saradc { + vref-supply = <&vccadc_ref>; +}; + +&backlight { + status = "okay"; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; +}; + +&cdn_dp { + status = "disabled"; +}; + +&display_subsystem { + status = "okay"; +}; + +&dsi_in_vopl { + status = "okay"; +}; + +&dsi_in_vopb { + status = "disabled"; +}; + +&dsi { + status = "okay"; + + panel@0 { + compatible ="simple-panel-dsi"; + reg = <0>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST)>; + dsi,format = ; + dsi,lanes = <4>; + + display-timings { + native-mode = <&timing0_720p_30hz>; + + timing0_720p_30hz: timing0-720p-30hz { + clock-frequency = <96000000>; + hactive = <1280>; + vactive = <720>; + hback-porch = <200>; + hfront-porch = <1000>; + vback-porch = <100>; + vfront-porch = <200>; + hsync-len = <200>; + vsync-len = <200>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + + timing1_1080p_30hz: timing0-1080p-30hz { + clock-frequency = <76000000>; + hactive = <1920>; + vactive = <1080>; + hback-porch = <100>; + hfront-porch = <200>; + vback-porch = <10>; + vfront-porch = <10>; + hsync-len = <20>; + vsync-len = <20>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + + timing2_1080p_87hz: timing1-1080p-87hz { + clock-frequency = <220000000>; + hactive = <1920>; + vactive = <1080>; + hback-porch = <200>; + hfront-porch = <120>; + vback-porch = <20>; + vfront-porch = <2>; + hsync-len = <20>; + vsync-len = <10>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&edp_in_vopl { + status = "disabled"; +}; + +&edp_in_vopb { + status = "okay"; +}; + +&route_edp { + status = "okay"; +}; + +&edp { + force-hpd; + status = "okay"; + + ports { + edp_out: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + edp_out_panel: endpoint@0 { + reg = <0>; + remote-endpoint = <&panel_in_edp>; + }; + }; + }; +}; + +&hdmi { + /* remove the hdmi_cec, reused by edp_hpd */ + pinctrl-0 = <&hdmi_i2c_xfer>; + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + status = "okay"; +}; + +&i2c1 { + status = "okay"; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; + }; + + tc358749x: tc358749x@f { + compatible = "toshiba,tc358749"; + reg = <0xf>; + clocks = <&ext_cam_clk>; + clock-names = "refclk"; + reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>; + interrupt-parent = <&gpio2>; + interrupts = <12 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmiin_gpios>; + status = "disabled"; + port { + hdmiin_out0: endpoint { + /* Unlinked mipi dphy rx0 */ + //remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + clock-noncontinuous; + link-frequencies = + /bits/ 64 <297000000>; + }; + }; + }; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "okay"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + lens-focus = <&vm149c>; + + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c4 { + status = "okay"; +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + status = "okay"; +}; + +&pinctrl { + buttons { + pwrbtn: pwrbtn { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + hdmiin { + hdmiin_gpios: hdmiin-gpios { + rockchip,pins = + <2 RK_PA5 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PA6 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PB0 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PB1 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PB2 RK_FUNC_GPIO &pcfg_output_low>, + <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux.dts new file mode 100755 index 000000000000..9f370a7bff1c --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-linux.dts @@ -0,0 +1,452 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "rk3399-excavator-sapphire.dtsi" +#include "rk3399-linux.dtsi" +#include + +/ { + model = "Rockchip RK3399 Excavator Board (Linux Opensource)"; + compatible = "rockchip,rk3399-excavator-linux", "rockchip,rk3399"; + + vcc_lcd: vcc-lcd { + compatible = "regulator-fixed"; + regulator-name = "vcc_lcd"; + gpio = <&gpio4 30 GPIO_ACTIVE_HIGH>; + startup-delay-us = <20000>; + enable-active-high; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc_lcd>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + hdmi_sound: hdmi-sound { + status = "okay"; + }; + + gpio-keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = <&pwrbtn>; + + button@0 { + gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "GPIO Key Power"; + linux,input-type = <1>; + gpio-key,wakeup = <1>; + debounce-interval = <100>; + }; + }; + + vccadc_ref: vccadc-ref { + compatible = "regulator-fixed"; + regulator-name = "vcc1v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + ext_cam_clk: external-camera-clock { + compatible = "fixed-clock"; + clock-frequency = <27000000>; + clock-output-names = "CLK_CAMERA_27MHZ"; + #clock-cells = <0>; + }; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + button-up { + label = "Volume Up"; + linux,code = ; + press-threshold-microvolt = <100000>; + }; + + button-down { + label = "Volume Down"; + linux,code = ; + press-threshold-microvolt = <300000>; + }; + + back { + label = "Back"; + linux,code = ; + press-threshold-microvolt = <985000>; + }; + + menu { + label = "Menu"; + linux,code = ; + press-threshold-microvolt = <1314000>; + }; + }; +}; + +&rkisp1_0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&isp0_mmu { + status = "okay"; +}; + +&rkisp1_1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + }; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + /* Unlinked camera */ + //remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&isp1_mmu { + status = "okay"; +}; + +&saradc { + vref-supply = <&vccadc_ref>; +}; + +&backlight { + status = "okay"; + enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&display_subsystem { + status = "okay"; +}; + +&route_edp { + status = "okay"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopb { + status = "disabled"; +}; + +&hdmi { + pinctrl-names = "default"; + pinctrl-0 = <&hdmi_i2c_xfer>, <&hdmi_cec>; + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + status = "okay"; +}; + +&hdmi_in_vopl { + status = "disabled"; +}; + +&i2c1 { + status = "okay"; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; + }; + + tc358749x: tc358749x@f { + compatible = "toshiba,tc358749"; + reg = <0xf>; + clocks = <&ext_cam_clk>; + clock-names = "refclk"; + reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>; + interrupt-parent = <&gpio2>; + interrupts = <12 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmiin_gpios>; + status = "disabled"; + port { + hdmiin_out0: endpoint { + /* Unlinked mipi dphy rx0 */ + //remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + clock-noncontinuous; + link-frequencies = + /bits/ 64 <297000000>; + }; + }; + }; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "okay"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + + lens-focus = <&vm149c>; + + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c4 { + status = "okay"; +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + status = "okay"; +}; + +&vopb { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP0_DIV>; + assigned-clock-parents = <&cru PLL_CPLL>; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP1_DIV>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; + +&vopl_mmu { + status = "okay"; +}; + +&pinctrl { + buttons { + pwrbtn: pwrbtn { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + hdmiin { + hdmiin_gpios: hdmiin-gpios { + rockchip,pins = + <2 RK_PA5 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PA6 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PB0 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PB1 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PB2 RK_FUNC_GPIO &pcfg_output_low>, + <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-lp4-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-lp4-linux.dts new file mode 100755 index 000000000000..da471a636103 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire-excavator-lp4-linux.dts @@ -0,0 +1,497 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2020 Fuzhou Rockchip Electronics Co., Ltd. + +/dts-v1/; + +#include "rk3399-excavator-sapphire.dtsi" +#include "rk3399-linux.dtsi" +#include + +/ { + model = "Rockchip RK3399 Excavator Board (Linux Opensource)"; + compatible = "rockchip,rk3399-excavator-linux", "rockchip,rk3399"; + + vcc_lcd: vcc-lcd { + compatible = "regulator-fixed"; + regulator-name = "vcc_lcd"; + gpio = <&gpio4 30 GPIO_ACTIVE_HIGH>; + startup-delay-us = <20000>; + enable-active-high; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc_lcd>; + enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + hdmi_sound: hdmi-sound { + status = "okay"; + }; + + gpio-keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = <&pwrbtn>; + + button@0 { + gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "GPIO Key Power"; + linux,input-type = <1>; + gpio-key,wakeup = <1>; + debounce-interval = <100>; + }; + }; + + vccadc_ref: vccadc-ref { + compatible = "regulator-fixed"; + regulator-name = "vcc1v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + ext_cam_clk: external-camera-clock { + compatible = "fixed-clock"; + clock-frequency = <27000000>; + clock-output-names = "CLK_CAMERA_27MHZ"; + #clock-cells = <0>; + }; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + button-up { + label = "Volume Up"; + linux,code = ; + press-threshold-microvolt = <100000>; + }; + + button-down { + label = "Volume Down"; + linux,code = ; + press-threshold-microvolt = <300000>; + }; + + back { + label = "Back"; + linux,code = ; + press-threshold-microvolt = <985000>; + }; + + menu { + label = "Menu"; + linux,code = ; + press-threshold-microvolt = <1314000>; + }; + }; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; + upthreshold = <40>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 856000 + SYS_STATUS_REBOOT 856000 + SYS_STATUS_SUSPEND 328000 + SYS_STATUS_VIDEO_1080P 666000 + SYS_STATUS_VIDEO_4K 856000 + SYS_STATUS_VIDEO_4K_10B 856000 + SYS_STATUS_PERFORMANCE 856000 + SYS_STATUS_BOOST 856000 + SYS_STATUS_DUALVIEW 856000 + SYS_STATUS_ISP 856000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 762 416000 + 763 3012 666000 + 3013 99999 856000 + >; + + vop-pn-msch-readlatency = < + 0 0x20 + 4 0x20 + >; + + auto-min-freq = <328000>; + auto-freq-en = <0>; +}; + +&dmc_opp_table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-328000000 { + opp-hz = /bits/ 64 <328000000>; + opp-microvolt = <900000>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-416000000 { + opp-hz = /bits/ 64 <416000000>; + opp-microvolt = <900000>; + }; + opp-528000000 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-666000000 { + opp-hz = /bits/ 64 <666000000>; + opp-microvolt = <900000>; + }; + opp-800000000 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-856000000 { + opp-hz = /bits/ 64 <856000000>; + opp-microvolt = <900000>; + }; + opp-928000000 { + opp-hz = /bits/ 64 <928000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; +}; + +&rkisp1_0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&isp0_mmu { + status = "okay"; +}; + +&rkisp1_1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + }; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + /* Unlinked camera */ + //remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&isp1_mmu { + status = "okay"; +}; + +&saradc { + vref-supply = <&vccadc_ref>; +}; + +&backlight { + status = "okay"; + enable-gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&display_subsystem { + status = "okay"; +}; + +&route_edp { + status = "okay"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopb { + status = "disabled"; +}; + +&hdmi { + pinctrl-names = "default"; + pinctrl-0 = <&hdmi_i2c_xfer>, <&hdmi_cec>; + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + status = "okay"; +}; + +&hdmi_in_vopl { + status = "disabled"; +}; + +&i2c1 { + status = "okay"; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 22 GPIO_ACTIVE_HIGH>; + }; + + tc358749x: tc358749x@f { + compatible = "toshiba,tc358749"; + reg = <0xf>; + clocks = <&ext_cam_clk>; + clock-names = "refclk"; + reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>; + interrupt-parent = <&gpio2>; + interrupts = <12 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmiin_gpios>; + status = "disabled"; + port { + hdmiin_out0: endpoint { + /* Unlinked mipi dphy rx0 */ + //remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + clock-noncontinuous; + link-frequencies = + /bits/ 64 <297000000>; + }; + }; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "okay"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c4 { + status = "okay"; +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + status = "okay"; +}; + +&vopb { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP0_DIV>; + assigned-clock-parents = <&cru PLL_CPLL>; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP1_DIV>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; + +&vopl_mmu { + status = "okay"; +}; + +&pinctrl { + buttons { + pwrbtn: pwrbtn { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + hdmiin { + hdmiin_gpios: hdmiin-gpios { + rockchip,pins = + <2 RK_PA5 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PA6 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PB0 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PB1 RK_FUNC_GPIO &pcfg_output_high>, + <2 RK_PB2 RK_FUNC_GPIO &pcfg_output_low>, + <2 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi index 701a567d7638..3d9e27750139 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi @@ -515,6 +515,8 @@ &pwm0 { &pwm2 { status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin_pull_down>; }; &saradc { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sched-energy.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-sched-energy.dtsi new file mode 100755 index 000000000000..373a776b9207 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-sched-energy.dtsi @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/ { + energy-costs { + RK3399_CPU_COST_0: rk3399-core-cost0 { + busy-cost-data = < + 108 46 /* 408M */ + 159 67 /* 600M */ + 216 90 /* 816M */ + 267 120 /* 1008M */ + 318 153 /* 1200M */ + 375 198 /* 1416M */ + 401 222 /* 1512M */ + >; + idle-cost-data = < + 6 + 6 + 0 + 0 + >; + }; + + RK3399_CPU_COST_1: rk3399-core-cost1 { + busy-cost-data = < + 210 129 /* 408MHz */ + 308 184 /* 600MHz */ + 419 246 /* 816MHz */ + 518 335 /* 1008MHz */ + 617 428 /* 1200MHz */ + 728 573 /* 1416MHz */ + 827 724 /* 1608MHz */ + 925 900 /* 1800MHz */ + 1024 1108 /* 1992MHz */ + >; + idle-cost-data = < + 15 + 15 + 0 + 0 + >; + }; + + RK3399_CLUSTER_COST_0: rk3399-cluster-cost0 { + busy-cost-data = < + 108 46 /* 408M */ + 159 67 /* 600M */ + 216 90 /* 816M */ + 267 120 /* 1008M */ + 318 153 /* 1200M */ + 375 198 /* 1416M */ + 401 222 /* 1512M */ + >; + idle-cost-data = < + 56 + 56 + 56 + 56 + >; + }; + + RK3399_CLUSTER_COST_1: rk3399-cluster-cost1 { + busy-cost-data = < + 210 129 /* 408MHz */ + 308 184 /* 600MHz */ + 419 246 /* 816MHz */ + 518 335 /* 1008MHz */ + 617 428 /* 1200MHz */ + 728 573 /* 1416MHz */ + 827 724 /* 1608MHz */ + 925 900 /* 1800MHz */ + 1024 1108 /* 1992MHz */ + >; + idle-cost-data = < + 65 + 65 + 65 + 65 + >; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-tve1030g-avb.dts b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g-avb.dts new file mode 100755 index 000000000000..eeca9cf54373 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g-avb.dts @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include "rk3399-tve1030g.dtsi" + +/ { + compatible = "rockchip,rk3399-tve1030g-avb", "rockchip,rk3399"; +}; + +&i2c1 { + status = "okay"; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "disabled"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + /* reset-gpios = <>; */ + reset-gpios = <&gpio2 10 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-CT0116"; + rockchip,camera-module-lens-name = "Largan-50013A1"; + lens-focus = <&vm149c>; + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + //remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2>; + }; + }; + }; + + gc2355: gc2355@3c { + status = "okay"; + compatible = "galaxycore,gc2355"; + reg = <0x3c>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* avdd-supply = <>; */ + /* dvdd-supply = <>; */ + /* dovdd-supply = <>; */ + /* reset-gpios = <>; */ + pwdn-gpios = <&gpio2 12 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CMK-CW2392"; + rockchip,camera-module-lens-name = "M206A-201"; + port { + ucam_out1: endpoint { + //remote-endpoint = <&mipi_in_ucam0>; + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1>; + }; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out1>; + data-lanes = <1>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&rkisp1_0 { + status = "disabled"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rkisp1_1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + }; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dts b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dts new file mode 100755 index 000000000000..28f81ee8f7f5 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dts @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include "rk3399-tve1030g.dtsi" + +/ { + compatible = "rockchip,rk3399-tve1030g", "rockchip,rk3399"; + +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&isp0 { + status = "okay"; +}; + +&isp1 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dtsi new file mode 100755 index 000000000000..fb700431e356 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-tve1030g.dtsi @@ -0,0 +1,1039 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; +#include +#include +#include "rk3399.dtsi" +#include "rk3399-android.dtsi" +#include "rk3399-opp.dtsi" +#include "rk3399-vop-clk-set.dtsi" +#include + +/ { + adc_keys { + compatible = "adc-keys"; + io-channels = <&saradc 1>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1000>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <170000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + }; + + es8316-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,es8316-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s0>; + system-clock-frequency = <11289600>; + }; + simple-audio-card,codec { + sound-dai = <&es8316>; + system-clock-frequency = <11289600>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio4 RK_PD4 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <6700>; + rockchip,screen-on-voltage = <6800>; + status = "okay"; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3900000>; + regulator-max-microvolt = <3900000>; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio4 RK_PD1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc5v0_host"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + enable-active-high; + }; + + vdd_log: vdd-log { + compatible = "pwm-regulator"; + pwms = <&pwm2 0 25000 1>; + rockchip,pwm_id= <2>; + rockchip,pwm_voltage = <900000>; + regulator-name = "vdd_log"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-boot-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6255"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>, <&bt_reset_gpio>, <&bt_wake_gpio>, <&bt_irq_gpio>; + pinctrl-1 = <&uart0_gpios>; + BT,reset_gpio = <&gpio0 RK_PB1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; + upthreshold = <20>; + downdifferential = <10>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 856000 + SYS_STATUS_REBOOT 856000 + SYS_STATUS_SUSPEND 416000 + SYS_STATUS_VIDEO_1080P 416000 + SYS_STATUS_VIDEO_4K 666000 + SYS_STATUS_VIDEO_4K_10B 856000 + SYS_STATUS_PERFORMANCE 856000 + SYS_STATUS_BOOST 856000 + SYS_STATUS_DUALVIEW 856000 + SYS_STATUS_ISP 856000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 762 328000 + 763 3012 666000 + 3013 99999 856000 + >; + + auto-min-freq = <328000>; + auto-freq-en = <1>; +}; + +&dmc_opp_table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-328000000 { + opp-hz = /bits/ 64 <328000000>; + opp-microvolt = <900000>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-416000000 { + opp-hz = /bits/ 64 <416000000>; + opp-microvolt = <900000>; + }; + opp-528000000 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-666000000 { + opp-hz = /bits/ 64 <666000000>; + opp-microvolt = <900000>; + }; + opp-800000000 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-856000000 { + opp-hz = /bits/ 64 <856000000>; + opp-microvolt = <900000>; + }; + opp-928000000 { + opp-hz = /bits/ 64 <928000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; +}; +&dp_in_vopb { + status = "disabled"; +}; + +&dsi { + status = "okay"; + rockchip,lane-rate = <1000>; + dsi_panel: panel@0 { + status = "okay"; + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + reset-gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd_rst_gpio>; + reset-delay-ms = <60>; + enable-delay-ms = <60>; + prepare-delay-ms = <60>; + unprepare-delay-ms = <60>; + disable-delay-ms = <60>; + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + panel-init-sequence = [ + 15 05 02 8F A5 + 15 14 02 01 00 + 15 05 02 8F A5 + 15 00 02 83 AA + 15 00 02 84 11 + 15 00 02 A9 4B + 15 00 02 83 00 + 15 00 02 84 00 + 15 00 02 8F 00 + ]; + + disp_timings: display-timings { + native-mode = <&timing0>; + timing0: timing0 { + clock-frequency = <150000000>; + hactive = <1200>; + hfront-porch = <80>; + hback-porch = <60>; + hsync-len = <1>; + vactive = <1920>; + vfront-porch = <35>; + vback-porch = <25>; + vsync-len = <1>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&emmc_phy { + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&hdmi { + status = "okay"; +}; + +&hdmi_dp_sound { + status = "okay"; +}; + +&hdmi_in_vopb { + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <180>; + i2c-scl-falling-time-ns = <30>; + clock-frequency = <400000>; + + vdd_cpu_b: syr837@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + vin-supply = <&vcc_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: syr828@41 { + compatible = "silergy,syr828"; + status = "okay"; + reg = <0x41>; + vin-supply = <&vcc_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <735000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + vcc10-supply = <&vcc3v3_sys>; + vcc11-supply = <&vcc3v3_sys>; + vcc12-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc1v8_pmu>; + + regulators { + + vdd_center: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_center"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + vcc_1v8: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + vcc1v8_dvp: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + vcc3v0_tp: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc3v0_tp"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + vcc1v8_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + vcc_sd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + vcca3v0_codec: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcca3v0_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + vcca1v8_codec: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + vcc_3v0: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + vcc3v3_s3: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + vcc3v3_s0: SWITCH_REG2 { + regulator-boot-on; + regulator-always-on; + regulator-name = "vcc3v3_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <140>; + i2c-scl-falling-time-ns = <30>; + + es8316: es8316@11 { + #sound-dai-cells = <0>; + compatible = "everest,es8316"; + reg = <0x11>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + spk-con-gpio = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + extcon = <&rk_headset>; + }; +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + clock-frequency = <100000>; + + bq25700: bq25700@6b { + compatible = "ti,bq25703"; + reg = <0x6b>; + extcon = <&fusb0>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&charger_ok>; + ti,charge-current = <1500000>; + ti,max-charge-voltage = <8704000>; + ti,max-input-voltage = <20000000>; + ti,max-input-current = <6000000>; + ti,input-current-sdp = <500000>; + ti,input-current-dcp = <2000000>; + ti,input-current-cdp = <2000000>; + ti,input-current-dc = <2000000>; + ti,minimum-sys-voltage = <6700000>; + ti,otg-voltage = <5000000>; + ti,otg-current = <500000>; + ti,input-current = <500000>; + pd-charge-only = <0>; + typec0-enable-gpios = <&gpio1 RK_PA3 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + + cw2015: cw2015@62 { + status = "okay"; + compatible = "cw201x"; + reg = <0x62>; + bat_config_info = <0x15 0xA8 0x5D 0x5D 0x59 0x55 0x57 0x50 + 0x4B 0x4F 0x55 0x53 0x43 0x37 0x2F 0x28 + 0x21 0x18 0x15 0x17 0x27 0x43 0x57 0x4F + 0x13 0x5E 0x0A 0xE1 0x19 0x31 0x3C 0x46 + 0x4C 0x52 0x50 0x54 0x44 0x1E 0x7E 0x4C + 0x1C 0x4A 0x52 0x87 0x8F 0x91 0x94 0x52 + 0x82 0x8C 0x92 0x96 0x00 0xAD 0xFB 0xCB + 0x2F 0x7D 0x72 0xA5 0xB5 0xC1 0x1C 0x09>; + monitor_sec = <2>; + virtual_power = <0>; + divider_res1 = <200>; + divider_res2 = <200>; + }; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + vbus-5v-gpios = <&gpio1 RK_PD0 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; + discharge-gpios = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + charge-dev = <&bq25700>; + support-uboot-charge = <1>; + port-num = <0>; + status = "okay"; + }; + + kxtj: kxtj2@0e { + status = "okay"; + compatible = "gs_kxtj9"; + pinctrl-names = "default"; + pinctrl-0 = <&kxtj2_irq_gpio>; + reg = <0x0e>; + irq-gpio = <&gpio1 RK_PC6 IRQ_TYPE_EDGE_RISING>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + power-off-in-suspend = <1>; + layout = <5>; + }; +}; + +&i2c5 { + status = "okay"; + i2c-scl-rising-time-ns = <150>; + i2c-scl-falling-time-ns = <30>; + clock-frequency = <100000>; + + gslx680: gslx680@40 { + compatible = "gslX680_tve"; + reg = <0x40>; + pinctrl-names = "default"; + pinctrl-0 = <&tp_irq_gpio>; + touch-gpio = <&gpio3 RK_PB0 IRQ_TYPE_EDGE_RISING>; + reset-gpio = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; + max-x = <1200>; + max-y = <1920>; + tp-size = <80>; + tp-supply = <&vcc3v0_tp>; + status = "okay"; + }; +}; + +&i2s0 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <8>; + #sound-dai-cells = <0>; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&io_domains { + status = "okay"; + bt656-supply = <&vcc1v8_dvp>; + audio-supply = <&vcca1v8_codec>; + sdmmc-supply = <&vcc_sd>; + gpio1830-supply = <&vcc_3v0>; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&pinctrl { + + charger { + charger_ok: charge-ok { + rockchip,pins = + <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + kxtj2 { + kxtj2_irq_gpio: kxtj2-irq-gpio { + rockchip,pins = <1 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + lcd_rst { + lcd_rst_gpio: lcd-rst-gpio { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + tp_irq { + tp_irq_gpio: tp-irq-gpio { + rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_reset_gpio: bt-reset-gpio { + rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_wake_gpio: bt-wake-gpio { + rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_irq_gpio: bt-irq-gpio { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm2 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin_pull_down>; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_OSC_DIS + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + ) + >; + rockchip,pwm-regulator-config = < + (0 + | PWM2_REGULATOR_EN + ) + >; + rockchip,power-ctrl = + <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, + <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; +}; + +&route_dsi { + status = "okay"; + logo,mode = "center"; +}; + +&saradc { + status = "okay"; +}; + +&sdhci { + bus-width = <8>; + mmc-hs400-1_8v; + no-sdio; + no-sd; + non-removable; + keep-power-in-suspend; + mmc-hs400-enhanced-strobe; + status = "okay"; +}; + +&sdio0 { + clock-frequency = <100000000>; + clock-freq-min-max = <200000 100000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc { + clock-frequency = <50000000>; + clock-freq-min-max = <400000 150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + num-slots = <1>; + //sd-uhs-sdr104; + vqmmc-supply = <&vcc_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "okay"; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&uart2 { + status = "disabled"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&vopb { + assigned-clocks = <&cru DCLK_VOP0_DIV>; + assigned-clock-parents = <&cru PLL_CPLL>; +}; + +&vopl { + assigned-clocks = <&cru DCLK_VOP1_DIV>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-tve1205g.dts b/arch/arm64/boot/dts/rockchip/rk3399-tve1205g.dts new file mode 100755 index 000000000000..ac9d28db5b76 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-tve1205g.dts @@ -0,0 +1,1179 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include +#include "rk3399.dtsi" +#include "rk3399-android.dtsi" +#include "rk3399-opp.dtsi" +#include +#include + +/ { + compatible = "rockchip,rk3399-mid", "rockchip,rk3399"; + + edp_panel: edp-panel { + compatible = "auo,b125han03"; + backlight = <&backlight>; + power-supply = <&vcc3v3_s0>; + enable-gpios = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; + bus-format = ; + bpc = <6>; + prepare-delay-ms = <50>; + ports { + panel_in_edp: endpoint { + remote-endpoint = <&edp_out_panel>; + }; + }; + }; + + usb_cam_gpio: usb-cam-gpio { + compatible = "usb-cam-gpio"; + pinctrl-names = "default"; + pinctrl-0 = <&usb_cam_on_gpio>; + hd-cam-gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; + ir-cam-gpios = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + vcc_sys: vcc-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3900000>; + regulator-max-microvolt = <3900000>; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vdd_log: vdd-log { + compatible = "pwm-regulator"; + pwms = <&pwm2 0 25000 1>; + rockchip,pwm_id= <2>; + rockchip,pwm_voltage = <900000>; + regulator-name = "vdd_log"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-always-on; + regulator-boot-on; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 1 51 52 52 53 53 54 + 54 55 55 56 56 57 57 58 + 58 59 59 60 61 61 62 63 + 63 64 65 65 66 67 67 68 + 69 69 70 71 71 72 73 73 + 74 75 75 76 77 77 78 79 + 79 80 80 81 81 82 83 83 + 84 85 86 86 87 88 89 89 + 90 91 92 92 93 94 95 95 + 96 97 98 98 99 100 101 101 + 102 103 104 104 105 106 107 107 + 108 109 110 110 111 112 113 113 + 114 115 116 116 117 118 119 119 + 120 121 122 122 123 124 125 125 + 126 127 128 128 129 130 131 131 + 132 133 134 134 135 136 137 137 + 138 139 140 140 141 142 143 143 + 144 145 146 146 147 148 149 149 + 150 151 152 152 153 154 155 155 + 156 157 158 158 159 160 161 161 + 162 163 164 164 165 166 167 167 + 168 169 170 170 171 172 173 173 + 174 175 176 176 177 178 179 179 + 180 181 182 182 183 184 185 185 + 186 187 188 188 189 190 191 191 + 216 217 218 218 219 220 221 221 + 222 223 224 224 225 226 227 227 + 228 229 230 230 231 232 233 233 + 234 235 236 236 237 238 239 239 + 240 241 242 242 243 244 245 245 + 246 247 248 248 249 250 251 251 + 252 253 254 254 255 255 255 255>; + default-brightness-level = <200>; + enable-gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + cx2072x-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,cx2072x-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Microphone Jack", + "Line", "Microphone Headset", + "Headphone", "Headphone Jack", + "Speaker", "Speaker External"; + simple-audio-card,routing = + "PORTC", "Microphone Jack", + "PortD Mic Bias", "Microphone Headset", + "Headphone Jack", "PORTA", + "Speaker External", "PORTG"; + simple-audio-card,cpu { + sound-dai = <&i2s0>; + }; + simple-audio-card,codec { + sound-dai = <&cx2072x>; + }; + }; + + sound { + compatible = "rockchip,cdndp-sound"; + rockchip,cpu = <&i2s2>; + rockchip,codec = <&cdn_dp>; + status = "okay"; + }; + + bt-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "dsp_b"; + simple-audio-card,bitclock-inversion = <1>; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,bt"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&bt_sco>; + }; + }; + + bt_sco: bt-sco { + compatible = "delta,dfbmcs320"; + #sound-dai-cells = <0>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ + }; + + leds: gpio-leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 =<&leds_gpio>; + + led@1 { + gpios = <&gpio1 RK_PC6 GPIO_ACTIVE_HIGH>; + label = "battery_led_amber"; + retain-state-suspended; + }; + + led@2 { + gpios = <&gpio1 RK_PD0 GPIO_ACTIVE_HIGH>; + label = "battery_led_white"; + retain-state-suspended; + }; + + led@3 { + gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_HIGH>; + label = "call_answer_led"; + }; + + led@4 { + gpios = <&gpio3 RK_PB1 GPIO_ACTIVE_HIGH>; + label = "call_decline_led"; + }; + + led@5 { + gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_HIGH>; + label = "rec_mute_led"; + }; + + led@6 { + gpios = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; + label = "play_mute_led"; + }; + + led@7 { + gpios = <&gpio4 RK_PD2 GPIO_ACTIVE_HIGH>; + label = "wl_led"; + }; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_gpios>; + //BT,power_gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; /* GPIOx_xx */ + BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ + BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ + BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ + status = "okay"; + }; + + uboot-charge { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-exit-charge-level = <2>; + rockchip,uboot-low-power-level = <1>; + rockchip,uboot-charge-brightness = <0>; + max-input-voltage = <20000>; + max-input-current = <6000>; + }; + + vibrator { + compatible = "rk-vibrator-gpio"; + vibrator-gpio = <&gpio4 30 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio4 28 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + hall_sensor: hall-mh248 { + compatible = "hall-mh248"; + pinctrl-names = "default"; + pinctrl-0 = <&mh248_irq_gpio>; + irq-gpio = <&gpio0 RK_PA1 IRQ_TYPE_EDGE_BOTH>; + hall-active = <1>; + status = "okay"; + }; +}; + +&rk_key { + compatible = "rockchip,key"; + status = "okay"; + + io-channels = <&saradc 1>; + + vol-up-key { + linux,code = <114>; + label = "volume up"; + rockchip,adc_value = <1>; + }; + + vol-down-key { + linux,code = <115>; + label = "volume down"; + rockchip,adc_value = <170>; + }; + + power-key { + gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; + linux,code = <116>; + label = "power"; + gpio-key,wakeup; + }; + + menu-key { + linux,code = <59>; + label = "menu"; + rockchip,adc_value = <746>; + }; + + home-key { + linux,code = <102>; + label = "home"; + rockchip,adc_value = <355>; + }; + + back-key { + linux,code = <158>; + label = "back"; + rockchip,adc_value = <560>; + }; + + camera-key { + linux,code = <212>; + label = "camera"; + rockchip,adc_value = <450>; + }; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&edp { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&edp_hpd>; + + ports { + edp_out: port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + edp_out_panel: endpoint@0 { + reg = <0>; + remote-endpoint = <&panel_in_edp>; + }; + }; + }; +}; + +&edp_in_vopl { + status = "disabled"; +}; + +&emmc_phy { + status = "okay"; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&hdmi { + status = "disabled"; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>, <&fusb1>; +}; + +&dp_in_vopb { + status = "disabled"; +}; + +&i2s0 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <8>; + rockchip,capture-channels = <8>; + rockchip,bclk-fs = <32>; + #sound-dai-cells = <0>; +}; + +&i2s1 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <2>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&i2s2 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <180>; + i2c-scl-falling-time-ns = <30>; + clock-frequency = <400000>; + + vdd_cpu_b: syr837@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + vin-supply = <&vcc_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: syr828@41 { + compatible = "silergy,syr828"; + status = "okay"; + reg = <0x41>; + vin-supply = <&vcc_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <735000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + vcc10-supply = <&vcc3v3_sys>; + vcc11-supply = <&vcc3v3_sys>; + vcc12-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc1v8_pmu>; + + regulators { + vdd_center: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_center"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc1v8_dvp: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v0_tp: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc3v0_tp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sd: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcca3v0_codec: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcca3v0_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcca1v8_codec: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v3_s3: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_s0: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <140>; + i2c-scl-falling-time-ns = <30>; + + cx2072x:cx2072x@33 { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "cnxt,cx20723"; + reg = <0x33>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + spk-con-gpio = <&gpio0 11 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c3 { + status="okay"; + + hidkey@68 { + clock-frequency = <100000>; + compatible = "hid-over-i2c"; + interrupt-parent = <&gpio0>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&hidkey_irq_gpio>; + reg = <0x68>; + hid-descr-addr = <0x0001>; + hid-support-wakeup; + }; + + ec_battery@76 { + compatible = "rockchip,ec-battery"; + reg = <0x76>; + virtual_power = <0>; + monitor_sec = <5>; + ec-notify-gpios = <&gpio1 RK_PC4 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c2 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + clock-frequency = <400000>; + + touchpad: touchpad@2c { + compatible = "hid-over-i2c"; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&touchpad_irq_gpio>; + reg = <0x2c>; + hid-descr-addr = <0x002c>; + }; +}; + +&i2c4 { + status = "okay"; + clock-frequency = <100000>; + bq25700: bq25700@09 {//6a + compatible = "ti,bq25700"; + reg = <0x09>; + extcon = <&fusb0>, <&fusb1>; + + interrupt-parent = <&gpio1>; + interrupts = <23 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&charger_ok>; + ti,charge-current = <2500000>; + ti,max-input-voltage = <20000000>; + ti,max-input-current = <6000000>; + ti,max-charge-voltage = <8750000>; + ti,input-current = <500000>; + ti,input-current-sdp = <500000>; + ti,input-current-dcp = <2000000>; + ti,input-current-cdp = <2000000>; + ti,minimum-sys-voltage = <7400000>; + ti,otg-voltage = <5000000>; + ti,otg-current = <500000>; + pd-charge-only = <1>; + typec0-enable-gpios = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; + typec1-enable-gpios = <&gpio1 RK_PA4 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c6 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + clock-frequency = <400000>; + + fusb1: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb1_int>; + vbus-5v-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + discharge-gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + charge-dev = <&bq25700>; + support-uboot-charge = <1>; + port-num = <1>; + status = "okay"; + }; +}; + +&i2c7 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + clock-frequency = <400000>; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + vbus-5v-gpios = <&gpio4 RK_PD5 GPIO_ACTIVE_HIGH>; + int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; + discharge-gpios = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + charge-dev = <&bq25700>; + support-uboot-charge = <1>; + port-num = <0>; + status = "okay"; + }; +}; + +&io_domains { + status = "okay"; + bt656-supply = <&vcc_3v0>; + audio-supply = <&vcca1v8_codec>; + sdmmc-supply = <&vcc_sd>; + gpio1830-supply = <&vcc_3v0>; +}; + +&dsi { + status = "disabled"; +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + ep-gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>; + num-lanes = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_clkreqn_cpm>; + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm2 { + status = "okay"; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2_pin_pull_down>; +}; + +&route_edp { + status = "okay"; + logo,mode = "center"; +}; + +&saradc { + status = "okay"; +}; + +&sdmmc { + clock-frequency = <50000000>; + clock-freq-min-max = <400000 150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + num-slots = <1>; + //sd-uhs-sdr104; + vqmmc-supply = <&vcc_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + status = "okay"; +}; + +&sdio0 { + clock-frequency = <150000000>; + clock-freq-min-max = <200000 150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdhci { + bus-width = <8>; + mmc-hs400-1_8v; + no-sdio; + no-sd; + non-removable; + keep-power-in-suspend; + mmc-hs400-enhanced-strobe; + status = "okay"; +}; + +&spdif { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&spi1 { + status = "disabled"; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + extcon = <&fusb1>; + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + extcon = <&fusb1>; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_host>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_1 { + status = "okay"; + extcon = <&fusb1>; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_PWM_WKUP_EN + ) + >; + rockchip,pwm-regulator-config = < + (0 + | PWM2_REGULATOR_EN + ) + >; + rockchip,power-ctrl = + <&gpio1 17 GPIO_ACTIVE_HIGH>, + <&gpio1 14 GPIO_ACTIVE_HIGH>; +}; + +&pinctrl { + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + hallsensor { + mh248_irq_gpio: mh248-irq-gpio { + rockchip,pins = <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + hidkey { + hidkey_irq_gpio: hidkey-irq-gpio { + rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + touchpad { + touchpad_irq_gpio: touchpad-irq-gpio { + rockchip,pins = <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + charger { + charger_ok: charge-ok { + rockchip,pins = + <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + gpio-leds { + leds_gpio: leds-gpio { + rockchip,pins = + <1 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>, + <1 RK_PD0 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>, + <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>, + <2 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>, + <4 RK_PD2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb2 { + host_vbus_drv: host-vbus-drv { + rockchip,pins = + <4 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb_camera { + usb_cam_on_gpio: usb-cam-on-gpio { + rockchip,pins = + <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + fusb1_int: fusb1-int { + rockchip,pins = + <1 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-videostrong-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399-videostrong-linux.dts new file mode 100755 index 000000000000..10ba2048483d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-videostrong-linux.dts @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * + * SPDX-License-Identifier: (GPL-2.0+ OR MIT) + */ + +/dts-v1/; + +#include "rk3399-sapphire.dtsi" +#include "rk3399-linux.dtsi" +#include + +/ { + model = "Rockchip RK3399 Videostrong Board (Linux Opensource)"; + compatible = "rockchip,rk3399-videostrong-linux", "rockchip,rk3399"; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,signal-irq = <182>; + rockchip,wake-irq = <0>; + rockchip,irq-mode-enable = <1>; /* If enable uart uses irq instead of fiq */ + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + pinctrl-names = "default"; + pinctrl-0 = <&uart2c_xfer>; + }; + + gpio-keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + autorepeat; + + pinctrl-names = "default"; + pinctrl-0 = <&pwrbtn>; + + button@0 { + gpios = <&gpio0 5 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "GPIO Key Power"; + linux,input-type = <1>; + gpio-key,wakeup = <1>; + debounce-interval = <100>; + }; + }; + + hdmi_sound: hdmi-sound { + status = "okay"; + }; + + rt5640-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rt5640-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&rt5640>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; /* GPIO0_B2 */ + }; + + spdif-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "okay"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + vccadc_ref: vccadc-ref { + compatible = "regulator-fixed"; + regulator-name = "vcc1v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 19 GPIO_ACTIVE_LOW>; /* GPIO2_C3 */ + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>; + pinctrl-1 = <&uart0_gpios>; + BT,reset_gpio = <&gpio0 9 GPIO_ACTIVE_HIGH>; /* GPIO0_B1 */ + BT,wake_gpio = <&gpio2 26 GPIO_ACTIVE_HIGH>; /* GPIO2_D2 */ + BT,wake_host_irq = <&gpio0 4 GPIO_ACTIVE_HIGH>; /* GPIO0_A4 */ + status = "okay"; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6354"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 3 GPIO_ACTIVE_HIGH>; /* GPIO0_a3 */ + status = "okay"; + }; +}; + +&dfi { + status = "okay"; +}; + +&display_subsystem { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; + upthreshold = <40>; + downdifferential = <20>; + system-status-freq = < + /* system status freq(KHz) */ + SYS_STATUS_NORMAL 800000 + SYS_STATUS_REBOOT 528000 + SYS_STATUS_SUSPEND 200000 + SYS_STATUS_VIDEO_1080P 200000 + SYS_STATUS_VIDEO_4K 600000 + SYS_STATUS_VIDEO_4K_10B 800000 + SYS_STATUS_PERFORMANCE 800000 + SYS_STATUS_BOOST 400000 + SYS_STATUS_DUALVIEW 600000 + SYS_STATUS_ISP 600000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 577 200000 + 578 1701 300000 + 1702 99999 400000 + >; + auto-min-freq = <200000>; +}; + +&hdmi { + /* remove the hdmi_cec, reused by edp_hpd */ + pinctrl-0 = <&hdmi_i2c_xfer>; + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + status = "okay"; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <300>; + i2c-scl-falling-time-ns = <15>; + + rt5640: rt5640@1c { + #sound-dai-cells = <0>; + compatible = "realtek,rt5640"; + reg = <0x1c>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + realtek,in1-differential; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + }; +}; + +&i2s1 { + status = "okay"; + rockchip,i2s-broken-burst-len; + rockchip,playback-channels = <2>; + rockchip,capture-channels = <2>; + #sound-dai-cells = <0>; +}; + +&rkvdec { + status = "okay"; + /* 0 means ion, 1 means drm */ + //allocator = <0>; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + | RKPM_PWM_WKUP_EN + ) + >; + rockchip,pwm-regulator-config = < + (0 + | PWM2_REGULATOR_EN + ) + >; + rockchip,power-ctrl = + <&gpio1 17 GPIO_ACTIVE_HIGH>, + <&gpio1 14 GPIO_ACTIVE_HIGH>; +}; + +&spdif { + status = "okay"; + pinctrl-0 = <&spdif_bus>; + i2c-scl-rising-time-ns = <450>; + i2c-scl-falling-time-ns = <15>; + #sound-dai-cells = <0>; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&pinctrl { + buttons { + pwrbtn: pwrbtn { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-vop-clk-set.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-vop-clk-set.dtsi new file mode 100755 index 000000000000..5ed8dac6cf7d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-vop-clk-set.dtsi @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * This define is for support double show any dclk frequency. + * dclk_vop will have a exclusive pll as parent. + * set dclk_vop will change the pll rate as well. + */ + +#ifdef RK3399_TWO_PLL_FOR_VOP + +&sdhci { + assigned-clocks = <&cru SCLK_EMMC>; + assigned-clock-parents = <&cru PLL_GPLL>; + assigned-clock-rates = <200000000>; +}; + +&uart0 { + assigned-clocks = <&cru SCLK_UART0_SRC>; + assigned-clock-parents = <&cru PLL_GPLL>; +}; + +&uart1 { + assigned-clocks = <&cru SCLK_UART_SRC>; + assigned-clock-parents = <&cru PLL_GPLL>; +}; + +&uart2 { + assigned-clocks = <&cru SCLK_UART_SRC>; + assigned-clock-parents = <&cru PLL_GPLL>; +}; + +&uart3 { + assigned-clocks = <&cru SCLK_UART_SRC>; + assigned-clock-parents = <&cru PLL_GPLL>; +}; + +&uart4 { + assigned-clocks = <&pmucru SCLK_UART4_SRC>; + assigned-clock-parents = <&pmucru PLL_PPLL>; +}; + +&spdif { + assigned-clocks = <&cru SCLK_SPDIF_DIV>; + assigned-clock-parents = <&cru PLL_GPLL>; +}; + +&i2s0{ + assigned-clocks = <&cru SCLK_I2S0_DIV>; + assigned-clock-parents = <&cru PLL_GPLL>; +}; + +&i2s1 { + assigned-clocks = <&cru SCLK_I2S1_DIV>; + assigned-clock-parents = <&cru PLL_GPLL>; +}; + +&i2s2 { + assigned-clocks = <&cru SCLK_I2S2_DIV>; + assigned-clock-parents = <&cru PLL_GPLL>; +}; + +&cru { + assigned-clocks = + <&cru ACLK_PERIHP>, <&cru ACLK_PERILP0>, + <&cru HCLK_PERILP1>, <&cru SCLK_SDMMC>, + <&cru ACLK_EMMC>, <&cru ACLK_CENTER>, + <&cru HCLK_SD>, <&cru SCLK_VDU_CA>, + <&cru SCLK_VDU_CORE>, <&cru ACLK_USB3>, + <&cru FCLK_CM0S>, <&cru ACLK_CCI>, + <&cru PCLK_ALIVE>, <&cru ACLK_GMAC>, + <&cru SCLK_CS>, <&cru SCLK_CCI_TRACE>, + <&cru ARMCLKL>, <&cru ARMCLKB>, + <&cru PLL_NPLL>, <&cru ACLK_GPU>, + <&cru PLL_GPLL>, <&cru ACLK_PERIHP>, + <&cru HCLK_PERIHP>, <&cru PCLK_PERIHP>, + <&cru ACLK_PERILP0>, <&cru HCLK_PERILP0>, + <&cru PCLK_PERILP0>, <&cru HCLK_PERILP1>, + <&cru PCLK_PERILP1>, <&cru SCLK_I2C1>, + <&cru SCLK_I2C2>, <&cru SCLK_I2C3>, + <&cru SCLK_I2C5>, <&cru SCLK_I2C6>, + <&cru SCLK_I2C7>, <&cru SCLK_SPI0>, + <&cru SCLK_SPI1>, <&cru SCLK_SPI2>, + <&cru SCLK_SPI4>, <&cru SCLK_SPI5>, + <&cru ACLK_GIC>, <&cru ACLK_ISP0>, + <&cru ACLK_ISP1>, <&cru SCLK_VOP0_PWM>, + <&cru SCLK_VOP1_PWM>, <&cru PCLK_EDP>, + <&cru ACLK_HDCP>, <&cru ACLK_VIO>, + <&cru HCLK_SD>, <&cru SCLK_CRYPTO0>, + <&cru SCLK_CRYPTO1>, <&cru SCLK_EMMC>, + <&cru ACLK_EMMC>, <&cru ACLK_CENTER>, + <&cru ACLK_IEP>, <&cru ACLK_RGA>, + <&cru SCLK_RGA_CORE>, <&cru ACLK_VDU>, + <&cru ACLK_VCODEC>, <&cru PCLK_DDR>, + <&cru ACLK_GMAC>, <&cru SCLK_VDU_CA>, + <&cru SCLK_VDU_CORE>, <&cru ACLK_USB3>, + <&cru FCLK_CM0S>, <&cru ACLK_CCI>, + <&cru PCLK_ALIVE>, <&cru SCLK_CS>, + <&cru SCLK_CCI_TRACE>, <&cru ACLK_VOP0>, + <&cru HCLK_VOP0>, <&cru ACLK_VOP1>, + <&cru HCLK_VOP1>; + assigned-clock-rates = + <75000000>, <50000000>, + <50000000>, <50000000>, + <50000000>, <100000000>, + <50000000>, <150000000>, + <150000000>, <150000000>, + <50000000>, <150000000>, + <50000000>, <100000000>, + <75000000>, <75000000>, + <816000000>, <816000000>, + <600000000>, <200000000>, + <800000000>, <150000000>, + <75000000>, <37500000>, + <300000000>, <100000000>, + <50000000>, <100000000>, + <50000000>, <100000000>, + <100000000>, <100000000>, + <100000000>, <100000000>, + <100000000>, <50000000>, + <50000000>, <50000000>, + <50000000>, <50000000>, + <200000000>, <400000000>, + <400000000>, <100000000>, + <100000000>, <100000000>, + <400000000>, <400000000>, + <200000000>, <100000000>, + <200000000>, <200000000>, + <100000000>, <400000000>, + <400000000>, <400000000>, + <400000000>, <300000000>, + <400000000>, <200000000>, + <400000000>, <300000000>, + <300000000>, <300000000>, + <300000000>, <600000000>,/* aclk_cci */ + <100000000>, <150000000>, + <150000000>, <400000000>, + <100000000>, <400000000>, + <100000000>; +}; +#endif + diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi index c5f3d4f8f4d2..7f4a812f4337 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi @@ -9,8 +9,13 @@ #include #include #include +#include +#include +#include #include +#include "rk3399-dram-default-timing.dtsi" + / { compatible = "rockchip,rk3399"; @@ -164,7 +169,7 @@ CLUSTER_SLEEP: cluster-sleep { }; }; - display-subsystem { + display_subsystem: display-subsystem { compatible = "rockchip,display-subsystem"; ports = <&vopl_out>, <&vopb_out>; }; @@ -200,6 +205,20 @@ xin24m: xin24m { #clock-cells = <0>; }; + dummy_cpll: dummy_cpll { + compatible = "fixed-clock"; + clock-frequency = <0>; + clock-output-names = "dummy_cpll"; + #clock-cells = <0>; + }; + + dummy_vpll: dummy_vpll { + compatible = "fixed-clock"; + clock-frequency = <0>; + clock-output-names = "dummy_vpll"; + #clock-cells = <0>; + }; + amba: bus { compatible = "simple-bus"; #address-cells = <2>; @@ -346,6 +365,7 @@ sdhci: sdhci@fe330000 { phy-names = "phy_arasan"; power-domains = <&power RK3399_PD_EMMC>; disable-cqe-dcmd; + disable-cqe; status = "disabled"; }; @@ -764,76 +784,58 @@ spi5: spi@ff200000 { }; thermal_zones: thermal-zones { - cpu_thermal: cpu { - polling-delay-passive = <100>; + soc_thermal: cpu_thermal: cpu-thermal { + polling-delay-passive = <20>; polling-delay = <1000>; + sustainable-power = <1000>; /* milliwatts */ thermal-sensors = <&tsadc 0>; trips { - cpu_alert0: cpu_alert0 { + threshold: cpu_alert0: cpu_alert0 { temperature = <70000>; hysteresis = <2000>; type = "passive"; }; - cpu_alert1: cpu_alert1 { - temperature = <75000>; + target: cpu_alert1: cpu_alert1 { + temperature = <85000>; hysteresis = <2000>; type = "passive"; }; - cpu_crit: cpu_crit { - temperature = <95000>; - hysteresis = <2000>; + soc_crit: cpu_crit: cpu_crit { + temperature = <115000>; /* millicelsius */ + hysteresis = <2000>; /* millicelsius */ type = "critical"; }; }; cooling-maps { map0 { - trip = <&cpu_alert0>; + trip = <&target>; cooling-device = - <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_b1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&cpu_l0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <4096>; }; map1 { - trip = <&cpu_alert1>; + trip = <&target>; + cooling-device = + <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + map2 { + trip = <&target>; cooling-device = - <&cpu_l0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_l1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_l2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_l3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu_b1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <4096>; }; }; }; - gpu_thermal: gpu { + gpu_thermal: gpu-thermal { polling-delay-passive = <100>; polling-delay = <1000>; thermal-sensors = <&tsadc 1>; - - trips { - gpu_alert0: gpu_alert0 { - temperature = <75000>; - hysteresis = <2000>; - type = "passive"; - }; - gpu_crit: gpu_crit { - temperature = <95000>; - hysteresis = <2000>; - type = "critical"; - }; - }; - - cooling-maps { - map0 { - trip = <&gpu_alert0>; - cooling-device = - <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; }; }; @@ -849,10 +851,9 @@ tsadc: tsadc@ff260000 { reset-names = "tsadc-apb"; rockchip,grf = <&grf>; rockchip,hw-tshut-temp = <95000>; - pinctrl-names = "init", "default", "sleep"; + pinctrl-names = "gpio", "otpout"; pinctrl-0 = <&otp_pin>; pinctrl-1 = <&otp_out>; - pinctrl-2 = <&otp_pin>; #thermal-sensor-cells = <1>; status = "disabled"; }; @@ -1000,26 +1001,26 @@ power: power-controller { #size-cells = <0>; /* These power domains are grouped by VD_CENTER */ - power-domain@RK3399_PD_IEP { + pd_iep@RK3399_PD_IEP { reg = ; clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>; pm_qos = <&qos_iep>; }; - power-domain@RK3399_PD_RGA { + pd_rga@RK3399_PD_RGA { reg = ; clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>; pm_qos = <&qos_rga_r>, <&qos_rga_w>; }; - power-domain@RK3399_PD_VCODEC { + pd_vcodec@RK3399_PD_VCODEC { reg = ; clocks = <&cru ACLK_VCODEC>, <&cru HCLK_VCODEC>; pm_qos = <&qos_video_m0>; }; - power-domain@RK3399_PD_VDU { + pd_vdu@RK3399_PD_VDU { reg = ; clocks = <&cru ACLK_VDU>, <&cru HCLK_VDU>; @@ -1028,94 +1029,94 @@ power-domain@RK3399_PD_VDU { }; /* These power domains are grouped by VD_GPU */ - power-domain@RK3399_PD_GPU { + pd_gpu@RK3399_PD_GPU { reg = ; clocks = <&cru ACLK_GPU>; pm_qos = <&qos_gpu>; }; /* These power domains are grouped by VD_LOGIC */ - power-domain@RK3399_PD_EDP { + pd_edp@RK3399_PD_EDP { reg = ; clocks = <&cru PCLK_EDP_CTRL>; }; - power-domain@RK3399_PD_EMMC { + pd_emmc@RK3399_PD_EMMC { reg = ; clocks = <&cru ACLK_EMMC>; pm_qos = <&qos_emmc>; }; - power-domain@RK3399_PD_GMAC { + pd_gmac@RK3399_PD_GMAC { reg = ; clocks = <&cru ACLK_GMAC>, <&cru PCLK_GMAC>; pm_qos = <&qos_gmac>; }; - power-domain@RK3399_PD_SD { + pd_sd@RK3399_PD_SD { reg = ; clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>; pm_qos = <&qos_sd>; }; - power-domain@RK3399_PD_SDIOAUDIO { + pd_sdioaudio@RK3399_PD_SDIOAUDIO { reg = ; clocks = <&cru HCLK_SDIO>; pm_qos = <&qos_sdioaudio>; }; - power-domain@RK3399_PD_TCPD0 { + pd_tcpc0@RK3399_PD_TCPD0 { reg = ; clocks = <&cru SCLK_UPHY0_TCPDCORE>, <&cru SCLK_UPHY0_TCPDPHY_REF>; }; - power-domain@RK3399_PD_TCPD1 { + pd_tcpc1@RK3399_PD_TCPD1 { reg = ; clocks = <&cru SCLK_UPHY1_TCPDCORE>, <&cru SCLK_UPHY1_TCPDPHY_REF>; }; - power-domain@RK3399_PD_USB3 { + pd_usb3@RK3399_PD_USB3 { reg = ; clocks = <&cru ACLK_USB3>; pm_qos = <&qos_usb_otg0>, <&qos_usb_otg1>; }; - power-domain@RK3399_PD_VIO { + pd_vio@RK3399_PD_VIO { reg = ; #address-cells = <1>; #size-cells = <0>; - power-domain@RK3399_PD_HDCP { + pd_hdcp@RK3399_PD_HDCP { reg = ; clocks = <&cru ACLK_HDCP>, <&cru HCLK_HDCP>, <&cru PCLK_HDCP>; pm_qos = <&qos_hdcp>; }; - power-domain@RK3399_PD_ISP0 { + pd_isp0@RK3399_PD_ISP0 { reg = ; clocks = <&cru ACLK_ISP0>, <&cru HCLK_ISP0>; pm_qos = <&qos_isp0_m0>, <&qos_isp0_m1>; }; - power-domain@RK3399_PD_ISP1 { + pd_isp1@RK3399_PD_ISP1 { reg = ; clocks = <&cru ACLK_ISP1>, <&cru HCLK_ISP1>; pm_qos = <&qos_isp1_m0>, <&qos_isp1_m1>; }; - power-domain@RK3399_PD_VO { + pd_vo@RK3399_PD_VO { reg = ; #address-cells = <1>; #size-cells = <0>; - power-domain@RK3399_PD_VOPB { + pd_vopb@RK3399_PD_VOPB { reg = ; clocks = <&cru ACLK_VOP0>, <&cru HCLK_VOP0>; pm_qos = <&qos_vop_big_r>, <&qos_vop_big_w>; }; - power-domain@RK3399_PD_VOPL { + pd_vopl@RK3399_PD_VOPL { reg = ; clocks = <&cru ACLK_VOP1>, <&cru HCLK_VOP1>; @@ -1134,6 +1135,33 @@ pmu_io_domains: io-domains { compatible = "rockchip,rk3399-pmu-io-voltage-domain"; status = "disabled"; }; + + reboot_mode: reboot-mode { + compatible = "syscon-reboot-mode"; + offset = <0x300>; + mode-charge = ; + mode-fastboot = ; + mode-loader = ; + mode-normal = ; + mode-panic = ; + mode-recovery = ; + mode-ums = ; + }; + + pmu_pvtm: pmu-pvtm { + compatible = "rockchip,rk3399-pmu-pvtm"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + pvtm@4 { + reg = <4>; + clocks = <&pmucru SCLK_PVTM_PMU>; + clock-names = "clk"; + resets = <&pmucru SRST_PVTM>; + reset-names = "rst"; + }; + }; }; spi3: spi@ff350000 { @@ -1211,7 +1239,7 @@ pwm0: pwm@ff420000 { compatible = "rockchip,rk3399-pwm", "rockchip,rk3288-pwm"; reg = <0x0 0xff420000 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm0_pin>; clocks = <&pmucru PCLK_RKPWM_PMU>; clock-names = "pwm"; @@ -1222,7 +1250,7 @@ pwm1: pwm@ff420010 { compatible = "rockchip,rk3399-pwm", "rockchip,rk3288-pwm"; reg = <0x0 0xff420010 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm1_pin>; clocks = <&pmucru PCLK_RKPWM_PMU>; clock-names = "pwm"; @@ -1233,7 +1261,7 @@ pwm2: pwm@ff420020 { compatible = "rockchip,rk3399-pwm", "rockchip,rk3288-pwm"; reg = <0x0 0xff420020 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm2_pin>; clocks = <&pmucru PCLK_RKPWM_PMU>; clock-names = "pwm"; @@ -1244,13 +1272,32 @@ pwm3: pwm@ff420030 { compatible = "rockchip,rk3399-pwm", "rockchip,rk3288-pwm"; reg = <0x0 0xff420030 0x0 0x10>; #pwm-cells = <3>; - pinctrl-names = "default"; + pinctrl-names = "active"; pinctrl-0 = <&pwm3a_pin>; clocks = <&pmucru PCLK_RKPWM_PMU>; clock-names = "pwm"; status = "disabled"; }; + dfi: dfi@ff630000 { + reg = <0x00 0xff630000 0x00 0x4000>; + compatible = "rockchip,rk3399-dfi"; + rockchip,pmu = <&pmugrf>; + clocks = <&cru PCLK_DDR_MON>; + clock-names = "pclk_ddr_mon"; + status = "disabled"; + }; + + dmc: dmc { + compatible = "rockchip,rk3399-dmc"; + devfreq-events = <&dfi>; + interrupts = ; + clocks = <&cru SCLK_DDRC>; + clock-names = "dmc_clk"; + ddr_timing = <&ddr_timing>; + status = "disabled"; + }; + vpu: video-codec@ff650000 { compatible = "rockchip,rk3399-vpu"; reg = <0x0 0xff650000 0x0 0x800>; @@ -1296,6 +1343,20 @@ vdec_mmu: iommu@ff660480 { #iommu-cells = <0>; }; + iep: iep@ff670000 { + compatible = "rockchip,iep"; + iommu_enabled = <1>; + iommus = <&iep_mmu>; + reg = <0x0 0xff670000 0x0 0x800>; + interrupts = ; + clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>; + clock-names = "aclk_iep", "hclk_iep"; + power-domains = <&power RK3399_PD_IEP>; + allocator = <1>; + version = <2>; + status = "disabled"; + }; + iep_mmu: iommu@ff670800 { compatible = "rockchip,iommu"; reg = <0x0 0xff670800 0x0 0x40>; @@ -1327,6 +1388,10 @@ efuse0: efuse@ff690000 { clock-names = "pclk_efuse"; /* Data cells */ + specification_serial_number: specification-serial-number@6 { + reg = <0x06 0x1>; + bits = <0 5>; + }; cpu_id: cpu-id@7 { reg = <0x07 0x10>; }; @@ -1348,6 +1413,10 @@ logic_leakage: logic-leakage@1b { wafer_info: wafer-info@1c { reg = <0x1c 0x1>; }; + customer_demand: customer-demand@22 { + reg = <0x22 0x1>; + bits = <4 4>; + }; }; pmucru: pmu-clock-controller@ff750000 { @@ -1485,6 +1554,42 @@ pcie_phy: pcie-phy { reset-names = "phy"; status = "disabled"; }; + + pvtm: pvtm { + compatible = "rockchip,rk3399-pvtm"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + pvtm@0 { + reg = <0>; + clocks = <&cru SCLK_PVTM_CORE_L>; + clock-names = "clk"; + resets = <&cru SRST_PVTM_CORE_L>; + reset-names = "rst"; + }; + pvtm@1 { + reg = <1>; + clocks = <&cru SCLK_PVTM_CORE_B>; + clock-names = "clk"; + resets = <&cru SRST_PVTM_CORE_B>; + reset-names = "rst"; + }; + pvtm@2 { + reg = <2>; + clocks = <&cru SCLK_PVTM_DDR>; + clock-names = "clk"; + resets = <&cru SRST_PVTM_DDR>; + reset-names = "rst"; + }; + pvtm@3 { + reg = <3>; + clocks = <&cru SCLK_PVTM_GPU>; + clock-names = "clk"; + resets = <&cru SRST_PVTM_GPU>; + reset-names = "rst"; + }; + }; }; tcphy0: phy@ff7c0000 { @@ -1611,6 +1716,16 @@ i2s2: i2s@ff8a0000 { status = "disabled"; }; + rng: rng@ff8b8000 { + compatible = "rockchip,cryptov1-rng"; + reg = <0x0 0xff8b8000 0x0 0x1000>; + clocks = <&cru SCLK_CRYPTO1>, <&cru HCLK_S_CRYPTO1>; + clock-names = "clk_crypto", "hclk_crypto"; + assigned-clocks = <&cru SCLK_CRYPTO1>, <&cru HCLK_S_CRYPTO1>; + assigned-clock-rates = <150000000>, <100000000>; + status = "disabled"; + }; + vopl: vop@ff8f0000 { compatible = "rockchip,rk3399-vop-lit"; reg = <0x0 0xff8f0000 0x0 0x3efc>; @@ -1656,6 +1771,17 @@ vopl_out_dp: endpoint@4 { }; }; + vop1_pwm: voppwm@ff8f01a0 { + compatible = "rockchip,vop-pwm"; + reg = <0x0 0xff8f01a0 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&vop1_pwm_pin>; + clocks = <&cru SCLK_VOP1_PWM>; + clock-names = "pwm"; + status = "disabled"; + }; + vopl_mmu: iommu@ff8f3f00 { compatible = "rockchip,iommu"; reg = <0x0 0xff8f3f00 0x0 0x100>; @@ -1713,6 +1839,17 @@ vopb_out_dp: endpoint@4 { }; }; + vop0_pwm: voppwm@ff9001a0 { + compatible = "rockchip,vop-pwm"; + reg = <0x0 0xff9001a0 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&vop0_pwm_pin>; + clocks = <&cru SCLK_VOP0_PWM>; + clock-names = "pwm"; + status = "disabled"; + }; + vopb_mmu: iommu@ff903f00 { compatible = "rockchip,iommu"; reg = <0x0 0xff903f00 0x0 0x100>; @@ -1915,7 +2052,95 @@ gpu: gpu@ff9a0000 { clocks = <&cru ACLK_GPU>; #cooling-cells = <2>; power-domains = <&power RK3399_PD_GPU>; + power-off-delay-ms = <200>; + upthreshold = <40>; + downdifferential = <10>; status = "disabled"; + + gpu_power_model: power_model { + compatible = "arm,mali-simple-power-model"; + static-coefficient = <411000>; + dynamic-coefficient = <733>; + ts = <32000 4700 (-80) 2>; + thermal-zone = "gpu-thermal"; + }; + }; + + nocp_cci_msch0: nocp-cci-msch0@ffa86000 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa86000 0x0 0x400>; + }; + + nocp_gpu_msch0: nocp-gpu-msch0@ffa86400 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa86400 0x0 0x400>; + }; + + nocp_hp_msch0: nocp-hp-msch0@ffa86800 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa86800 0x0 0x400>; + }; + + nocp_lp_msch0: nocp-lp-msch0@ffa86c00 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa86c00 0x0 0x400>; + }; + + nocp_video_msch0: nocp-video-msch0@ffa87000 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa87000 0x0 0x400>; + }; + + nocp_vio0_msch0: nocp-vio0-msch0@ffa87400 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa87400 0x0 0x400>; + }; + + nocp_vio1_msch0: nocp-vio1-msch0@ffa87800 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa87800 0x0 0x400>; + }; + + nocp_cci_msch1: nocp-cci-msch1@ffa8e000 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa8e000 0x0 0x400>; + }; + + nocp_gpu_msch1: nocp-gpu-msch1@ffa8e400 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa8e400 0x0 0x400>; + }; + + nocp_hp_msch1: nocp-hp-msch1@ffa8e800 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa8e800 0x0 0x400>; + }; + + nocp_lp_msch1: nocp-lp-msch1@ffa8ec00 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa8ec00 0x0 0x400>; + }; + + nocp_video_msch1: nocp-video-msch1@ffa8f000 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa8f000 0x0 0x400>; + }; + + nocp_vio0_msch1: nocp-vio0-msch1@ffa8f400 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa8f400 0x0 0x400>; + }; + + nocp_vio1_msch1: nocp-vio1-msch1@ffa8f800 { + compatible = "rockchip,rk3399-nocp"; + reg = <0x0 0xffa8f800 0x0 0x400>; + }; + + rockchip_system_monitor: rockchip-system-monitor { + compatible = "rockchip,system-monitor"; + + rockchip,thermal-zone = "soc-thermal"; + rockchip,polling-delay = <200>; /* milliseconds */ }; pinctrl: pinctrl { @@ -2179,6 +2404,13 @@ i2c3_xfer: i2c3-xfer { <4 RK_PC1 1 &pcfg_pull_none>, <4 RK_PC0 1 &pcfg_pull_none>; }; + + i2c3_gpio: i2c3_gpio { + rockchip,pins = + <4 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>, + <4 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; i2c4 { @@ -2342,7 +2574,7 @@ sdmmc_wp: sdmmc-wp { }; }; - suspend { + sleep { ap_pwroff: ap-pwroff { rockchip,pins = <1 RK_PA5 1 &pcfg_pull_none>; }; @@ -2644,6 +2876,11 @@ pwm3a_pin: pwm3a-pin { rockchip,pins = <0 RK_PA6 1 &pcfg_pull_none>; }; + + pwm3a_pin_pull_down: pwm3a-pin-pull-down { + rockchip,pins = + <0 RK_PA6 1 &pcfg_pull_down>; + }; }; pwm3b { @@ -2651,6 +2888,11 @@ pwm3b_pin: pwm3b-pin { rockchip,pins = <1 RK_PB6 1 &pcfg_pull_none>; }; + + pwm3b_pin_pull_down: pwm3b-pin-pull-down { + rockchip,pins = + <1 RK_PB6 1 &pcfg_pull_down>; + }; }; hdmi { @@ -2679,4 +2921,27 @@ pcie_clkreqnb_cpm: pci-clkreqnb-cpm { }; }; + + rockchip_suspend: rockchip-suspend { + compatible = "rockchip,pm-rk3399"; + status = "disabled"; + rockchip,sleep-debug-en = <0>; + rockchip,virtual-poweroff = <0>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_OSC_DIS + | RKPM_SLP_CENTER_PD + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + ) + >; + }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399k-opp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399k-opp.dtsi new file mode 100755 index 000000000000..59f200e0b9cb --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399k-opp.dtsi @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +&cluster0_opp { + rockchip,high-temp = <70000>; + rockchip,high-temp-max-volt = <1125000>; + opp-1512000000 { + opp-hz = /bits/ 64 <1512000000>; + opp-microvolt = <1150000 1150000 1250000>; + clock-latency-ns = <40000>; + }; +}; + +&cluster1_opp { + rockchip,high-temp = <70000>; + rockchip,high-temp-max-volt = <1200000>; + opp-2016000000 { + opp-hz = /bits/ 64 <2016000000>; + opp-microvolt = <1250000 1250000 1250000>; + clock-latency-ns = <40000>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-lp4-v11-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-lp4-v11-linux.dts new file mode 100755 index 000000000000..f975f47b76d6 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-lp4-v11-linux.dts @@ -0,0 +1,1293 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2020 Fuzhou Rockchip Electronics Co., Ltd + +/dts-v1/; +#include +#include +#include +#include +#include +#include +#include "rk3399pro.dtsi" +#include "rk3399-linux.dtsi" +#include "rk3399-opp.dtsi" +#include "rk3399-vop-clk-set.dtsi" + +/ { + compatible = "rockchip,rk3399pro-evb-v11-linux", "rockchip,rk3399pro"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + enable-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + rockchip,irq-mode-enable = <0>; /* If enable uart uses irq instead of fiq */ + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + pinctrl-names = "default"; + pinctrl-0 = <&uart2c_xfer>; + interrupts = ; + }; + + hdmi_sound: hdmi-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,hdmi"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + enable-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + reset-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + rk809_sound: rk809-sound { + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk809-codec"; + rockchip,codec-hp-det; + rockchip,mclk-fs = <256>; + rockchip,cpu = <&i2s1>; + rockchip,codec = <&rk809_codec>; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 3>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + }; + + usbacm_video_control: usbacm-video-control { + compatible = "rockchip,usbacm-video-control"; + status = "disabled"; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6398s"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>, <&bt_irq_gpio>; + pinctrl-1 = <&uart0_gpios>; + BT,reset_gpio = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&display_subsystem { + status = "okay"; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; + upthreshold = <40>; + downdifferential = <20>; + system-status-freq = < + /*system status freq(KHz)*/ + SYS_STATUS_NORMAL 856000 + SYS_STATUS_REBOOT 856000 + SYS_STATUS_SUSPEND 328000 + SYS_STATUS_VIDEO_1080P 666000 + SYS_STATUS_VIDEO_4K 856000 + SYS_STATUS_VIDEO_4K_10B 856000 + SYS_STATUS_PERFORMANCE 856000 + SYS_STATUS_BOOST 856000 + SYS_STATUS_DUALVIEW 856000 + SYS_STATUS_ISP 856000 + >; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 762 416000 + 763 3012 666000 + 3013 99999 856000 + >; + + vop-pn-msch-readlatency = < + 0 0x20 + 4 0x20 + >; + + auto-min-freq = <328000>; + auto-freq-en = <0>; +}; + +&dmc_opp_table { + compatible = "operating-points-v2"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-328000000 { + opp-hz = /bits/ 64 <328000000>; + opp-microvolt = <900000>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-416000000 { + opp-hz = /bits/ 64 <416000000>; + opp-microvolt = <900000>; + }; + opp-528000000 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-666000000 { + opp-hz = /bits/ 64 <666000000>; + opp-microvolt = <900000>; + }; + opp-800000000 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; + opp-856000000 { + opp-hz = /bits/ 64 <856000000>; + opp-microvolt = <900000>; + }; + opp-928000000 { + opp-hz = /bits/ 64 <928000000>; + opp-microvolt = <900000>; + status = "disabled"; + }; +}; + +&dp_in_vopb { + status = "disabled"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopb { + status = "disabled"; +}; + +&emmc_phy { + status = "okay"; +}; + +&fiq_debugger { + pinctrl-0 = <&uart2a_xfer>; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&hdmi { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,phy-table = + <74250000 0x8009 0x0004 0x0272>, + <165000000 0x802b 0x0004 0x0209>, + <297000000 0x8039 0x0005 0x028d>, + <594000000 0x8039 0x0000 0x00f6>, + <000000000 0x0000 0x0000 0x0000>; +}; + +&hdmi_in_vopl { + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <180>; + i2c-scl-falling-time-ns = <30>; + clock-frequency = <400000>; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int_l>; + pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk809_slppin_null>; + rockchip,system-power-controller; + pmic-reset-func = <0>; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc_buck5>; + vcc6-supply = <&vcc_buck5>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + rtc { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk809_slppin_null: rk809_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk809_slppin_slp: rk809_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk809_slppin_pwrdn: rk809_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk809_slppin_rst: rk809_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_center: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_center"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_sys: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_buck5: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2200000>; + regulator-max-microvolt = <2200000>; + regulator-name = "vcc_buck5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <2200000>; + }; + }; + + vcca_0v9: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vcca_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc0v9_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-name = "vcc0v9_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vcca_1v8: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcca_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd1v5_dvp: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc5v0_usb: SWITCH_REG1 { + regulator-name = "vcc5v0_usb"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vccio_3v3: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vccio_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; + + vdd_cpu_b: tcs452x@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: tcs452x@10 { + compatible = "tcs,tcs452x"; + reg = <0x10>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <735000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + bq25700: bq25700@6b { + compatible = "ti,bq25703"; + reg = <0x6b>; + extcon = <&fusb0>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&charger_ok_int>; + ti,charge-current = <1500000>; + ti,max-charge-voltage = <8704000>; + ti,max-input-voltage = <20000000>; + ti,max-input-current = <6000000>; + ti,input-current-sdp = <500000>; + ti,input-current-dcp = <2000000>; + ti,input-current-cdp = <2000000>; + ti,input-current-dc = <2000000>; + ti,minimum-sys-voltage = <6700000>; + ti,otg-voltage = <5000000>; + ti,otg-current = <500000>; + ti,input-current = <500000>; + pd-charge-only = <0>; + status = "disabled"; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <140>; + i2c-scl-falling-time-ns = <30>; + + mpu6500@68 { + status = "okay"; + compatible = "invensense,mpu6500"; + reg = <0x68>; + irq-gpio = <&gpio3 RK_PD2 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <0 1 0 1 0 0 0 0 1>; + orientation-x= <1>; + orientation-y= <0>; + orientation-z= <0>; + mpu-debug = <1>; + }; + + sensor@d { + status = "okay"; + compatible = "ak8963"; + reg = <0x0d>; + type = ; + irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_EDGE_RISING>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <3>; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "okay"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; + + imx327: imx327@1a { + compatible = "sony,imx327"; + status = "okay"; + reg = <0x1a>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ucam_out2: endpoint { + remote-endpoint = <&mipi_in_ucam2>; + data-lanes = <1 2>; + }; + }; + }; + +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio4 RK_PC3 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c8 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + clock-frequency = <100000>; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; + vbus-5v-gpios = <&gpio0 RK_PA1 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + +}; + +&i2s1 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&io_domains { + status = "okay"; + bt656-supply = <&vcca_1v8>; + audio-supply = <&vcca_1v8>; + sdmmc-supply = <&vccio_sd>; + gpio1830-supply = <&vcc_3v0>; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + /* Unlinked camera */ + //remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + mipi_in_ucam2: endpoint@2 { + reg = <2>; + remote-endpoint = <&ucam_out2>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm2 { + status = "okay"; +}; + +&rkisp1_0 { + status = "okay"; + assigned-clocks = <&cru PLL_NPLL>, <&cru SCLK_CIF_OUT_SRC>, <&cru SCLK_CIF_OUT>; + assigned-clock-rates = <594000000>, <594000000>, <37125000>; + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rkisp1_1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + }; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_OSC_DIS + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = ; + rockchip,pwm-regulator-config = ; + rockchip,power-ctrl = + <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, + <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; +}; + +&route_edp { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdmmc { + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; +}; + +&spi1 { + status = "okay"; + max-freq = <48000000>; /* spi internal clk, don't modify */ + spi_dev@0 { + compatible = "rockchip,spidev"; + reg = <0>; + spi-max-frequency = <12000000>; + spi-lsb-first; + }; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd_dwc3_1 { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&npu_ref_clk>; + + bq2570 { + charger_ok_int: charger-ok-int { + rockchip,pins = + <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd_rst { + lcd_rst_gpio: lcd-rst-gpio { + rockchip,pins = + <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + npu_clk { + npu_ref_clk: npu-ref-clk { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + soc_slppin_gpio: soc-slppin-gpio { + rockchip,pins = + <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc-slppin-slp { + rockchip,pins = + <1 RK_PA5 1 &pcfg_pull_down>; + }; + + soc_slppin_rst: soc-slppin-rst { + rockchip,pins = + <1 RK_PA5 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdmmc { + sdmmc_bus1: sdmmc-bus1 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_10ma>; + }; + + sdmmc_bus4: sdmmc-bus4 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_10ma>, + <4 RK_PB1 1 &pcfg_pull_up_10ma>, + <4 RK_PB2 1 &pcfg_pull_up_10ma>, + <4 RK_PB3 1 &pcfg_pull_up_10ma>; + }; + + sdmmc_clk: sdmmc-clk { + rockchip,pins = + <4 RK_PB4 1 &pcfg_pull_none_10ma>; + }; + + sdmmc_cmd: sdmmc-cmd { + rockchip,pins = + <4 RK_PB5 1 &pcfg_pull_up_10ma>; + }; + }; + + tp_irq { + tp_irq_gpio: tp-irq-gpio { + rockchip,pins = + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + wireless-bluetooth { + bt_irq_gpio: bt-irq-gpio { + rockchip,pins = + <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10-linux.dts new file mode 100755 index 000000000000..08ac2a477852 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10-linux.dts @@ -0,0 +1,1203 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + +/dts-v1/; +#include +#include +#include +#include +#include +#include +#include "rk3399pro.dtsi" +#include "rk3399-linux.dtsi" +#include "rk3399-opp.dtsi" +#include "rk3399-vop-clk-set.dtsi" + +/ { + compatible = "rockchip,rk3399pro-evb-v10-linux", "rockchip,rk3399pro"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + enable-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + rockchip,irq-mode-enable = <0>; /* If enable uart uses irq instead of fiq */ + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + pinctrl-names = "default"; + pinctrl-0 = <&uart2c_xfer>; + interrupts = ; + }; + + hdmi_sound: hdmi-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,hdmi"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + enable-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + reset-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + rk809_sound: rk809-sound { + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk809-codec"; + rockchip,codec-hp-det; + rockchip,mclk-fs = <256>; + rockchip,cpu = <&i2s1>; + rockchip,codec = <&rk809_codec>; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 3>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6398s"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>, <&bt_irq_gpio>; + pinctrl-1 = <&uart0_gpios>; + BT,reset_gpio = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&display_subsystem { + status = "okay"; +}; + +&dp_in_vopb { + status = "disabled"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopb { + status = "disabled"; +}; + +&emmc_phy { + status = "okay"; +}; + +&fiq_debugger { + pinctrl-0 = <&uart2a_xfer>; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&hdmi { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,phy-table = + <74250000 0x8009 0x0004 0x0272>, + <165000000 0x802b 0x0004 0x0209>, + <297000000 0x8039 0x0005 0x028d>, + <594000000 0x8039 0x0000 0x00f6>, + <000000000 0x0000 0x0000 0x0000>; +}; + +&hdmi_in_vopl { + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <180>; + i2c-scl-falling-time-ns = <30>; + clock-frequency = <400000>; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int_l>; + pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk809_slppin_rst>; + rockchip,system-power-controller; + pmic-reset-func = <0>; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc_buck5>; + vcc6-supply = <&vcc_buck5>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + rtc { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk809_slppin_null: rk809_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk809_slppin_slp: rk809_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk809_slppin_pwrdn: rk809_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk809_slppin_rst: rk809_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_log: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_log"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_sys: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_buck5: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2200000>; + regulator-max-microvolt = <2200000>; + regulator-name = "vcc_buck5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <2200000>; + }; + }; + + vcca_0v9: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vcca_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc0v9_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-name = "vcc0v9_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vcca_1v8: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcca_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd1v5_dvp: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc5v0_usb: SWITCH_REG1 { + regulator-name = "vcc5v0_usb"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vccio_3v3: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vccio_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; + + vdd_cpu_b: syr837@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: syr828@41 { + compatible = "silergy,syr828"; + status = "okay"; + reg = <0x41>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <735000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + bq25700: bq25700@6b { + compatible = "ti,bq25703"; + reg = <0x6b>; + extcon = <&fusb0>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&charger_ok_int>; + ti,charge-current = <1500000>; + ti,max-charge-voltage = <8704000>; + ti,max-input-voltage = <20000000>; + ti,max-input-current = <6000000>; + ti,input-current-sdp = <500000>; + ti,input-current-dcp = <2000000>; + ti,input-current-cdp = <2000000>; + ti,input-current-dc = <2000000>; + ti,minimum-sys-voltage = <6700000>; + ti,otg-voltage = <5000000>; + ti,otg-current = <500000>; + ti,input-current = <500000>; + pd-charge-only = <0>; + status = "disabled"; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <140>; + i2c-scl-falling-time-ns = <30>; + + mpu6500@68 { + status = "okay"; + compatible = "invensense,mpu6500"; + reg = <0x68>; + irq-gpio = <&gpio3 RK_PD2 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <0 1 0 1 0 0 0 0 1>; + orientation-x= <1>; + orientation-y= <0>; + orientation-z= <0>; + mpu-debug = <1>; + }; + + sensor@d { + status = "okay"; + compatible = "ak8963"; + reg = <0x0d>; + type = ; + irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_EDGE_RISING>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <3>; + }; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "disabled"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + lens-focus = <&vm149c>; + + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; + + imx327: imx327@1a { + compatible = "sony,imx327"; + status = "okay"; + reg = <0x1a>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ucam_out2: endpoint { + remote-endpoint = <&mipi_in_ucam2>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio4 RK_PC3 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c8 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + clock-frequency = <100000>; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + +}; + +&i2s1 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&io_domains { + status = "okay"; + bt656-supply = <&vcca_1v8>; + audio-supply = <&vcca_1v8>; + sdmmc-supply = <&vccio_sd>; + gpio1830-supply = <&vcc_3v0>; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + /* Unlinked camera */ + //remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + + mipi_in_ucam2: endpoint@2 { + reg = <2>; + remote-endpoint = <&ucam_out2>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm2 { + status = "okay"; +}; + +&rkisp1_0 { + status = "okay"; + assigned-clocks = <&cru PLL_NPLL>, <&cru SCLK_CIF_OUT_SRC>, <&cru SCLK_CIF_OUT>; + assigned-clock-rates = <594000000>, <594000000>, <37125000>; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rkisp1_1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + }; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_OSC_DIS + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = ; + rockchip,pwm-regulator-config = ; + rockchip,power-ctrl = + <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, + <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; +}; + +&route_edp { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdmmc { + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; +}; + +&spi1 { + status = "okay"; + max-freq = <48000000>; /* spi internal clk, don't modify */ + spi_dev@0 { + compatible = "rockchip,spidev"; + reg = <0>; + spi-max-frequency = <12000000>; + spi-lsb-first; + }; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd_dwc3_1 { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&npu_ref_clk>; + + bq2570 { + charger_ok_int: charger-ok-int { + rockchip,pins = + <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd_rst { + lcd_rst_gpio: lcd-rst-gpio { + rockchip,pins = + <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + npu_clk { + npu_ref_clk: npu-ref-clk { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + soc_slppin_gpio: soc-slppin-gpio { + rockchip,pins = + <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc-slppin-slp { + rockchip,pins = + <1 RK_PA5 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc-slppin-rst { + rockchip,pins = + <1 RK_PA5 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdmmc { + sdmmc_bus1: sdmmc-bus1 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_10ma>; + }; + + sdmmc_bus4: sdmmc-bus4 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_10ma>, + <4 RK_PB1 1 &pcfg_pull_up_10ma>, + <4 RK_PB2 1 &pcfg_pull_up_10ma>, + <4 RK_PB3 1 &pcfg_pull_up_10ma>; + }; + + sdmmc_clk: sdmmc-clk { + rockchip,pins = + <4 RK_PB4 1 &pcfg_pull_none_10ma>; + }; + + sdmmc_cmd: sdmmc-cmd { + rockchip,pins = + <4 RK_PB5 1 &pcfg_pull_up_10ma>; + }; + }; + + tp_irq { + tp_irq_gpio: tp-irq-gpio { + rockchip,pins = + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + wireless-bluetooth { + bt_irq_gpio: bt-irq-gpio { + rockchip,pins = + <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10.dts new file mode 100755 index 000000000000..398f962114cf --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v10.dts @@ -0,0 +1,1061 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + +/dts-v1/; +#include +#include +#include +#include +#include +#include +#include "rk3399pro.dtsi" +#include "rk3399-android.dtsi" +#include "rk3399-opp.dtsi" +#include "rk3399-vop-clk-set.dtsi" + +/ { + compatible = "rockchip,rk3399pro-evb-v10", "rockchip,rk3399pro"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + enable-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + hdmi_sound: hdmi-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,hdmi"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + enable-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + reset-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 3>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6398s"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>, <&bt_irq_gpio>; + pinctrl-1 = <&uart0_gpios>; + BT,reset_gpio = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&dp_in_vopb { + status = "disabled"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopl { + status = "disabled"; +}; + +&emmc_phy { + status = "okay"; +}; + +&fiq_debugger { + pinctrl-0 = <&uart2a_xfer>; +}; + +&firmware_android { + compatible = "android,firmware"; + fstab { + compatible = "android,fstab"; + system { + compatible = "android,system"; + dev = "/dev/block/by-name/system"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + vendor { + compatible = "android,vendor"; + dev = "/dev/block/by-name/vendor"; + type = "ext4"; + mnt_flags = "ro,barrier=1,inode_readahead_blks=8"; + fsmgr_flags = "wait,verify"; + }; + }; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&hdmi { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,phy-table = + <74250000 0x8009 0x0004 0x0272>, + <165000000 0x802b 0x0004 0x0209>, + <297000000 0x8039 0x0005 0x028d>, + <594000000 0x8039 0x0000 0x00f6>, + <000000000 0x0000 0x0000 0x0000>; +}; + +&hdmi_dp_sound { + status = "okay"; +}; + +&hdmi_in_vopb { + status = "disabled"; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <180>; + i2c-scl-falling-time-ns = <30>; + clock-frequency = <400000>; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int_l>; + pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>,<&rk809_slppin_rst>; + rockchip,system-power-controller; + pmic-reset-func = <0>; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc_buck5>; + vcc6-supply = <&vcc_buck5>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + rtc { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk809_slppin_null: rk809_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk809_slppin_slp: rk809_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk809_slppin_pwrdn: rk809_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk809_slppin_rst: rk809_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_log: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_log"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_sys: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_buck5: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2200000>; + regulator-max-microvolt = <2200000>; + regulator-name = "vcc_buck5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <2200000>; + }; + }; + + vcca_0v9: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vcca_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc0v9_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-name = "vcc0v9_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vcca_1v8: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcca_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd1v5_dvp: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc5v0_usb: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_usb"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vccio_3v3: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vccio_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; + + vdd_cpu_b: syr837@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: syr828@41 { + compatible = "silergy,syr828"; + status = "okay"; + reg = <0x41>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <735000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + bq25700: bq25700@6b { + compatible = "ti,bq25703"; + reg = <0x6b>; + extcon = <&fusb0>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&charger_ok_int>; + ti,charge-current = <1500000>; + ti,max-charge-voltage = <8704000>; + ti,max-input-voltage = <20000000>; + ti,max-input-current = <6000000>; + ti,input-current-sdp = <500000>; + ti,input-current-dcp = <2000000>; + ti,input-current-cdp = <2000000>; + ti,input-current-dc = <2000000>; + ti,minimum-sys-voltage = <6700000>; + ti,otg-voltage = <5000000>; + ti,otg-current = <500000>; + ti,input-current = <500000>; + pd-charge-only = <0>; + status = "disabled"; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <140>; + i2c-scl-falling-time-ns = <30>; + + mpu6500@68 { + status = "okay"; + compatible = "invensense,mpu6500"; + reg = <0x68>; + irq-gpio = <&gpio3 RK_PD2 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <0 1 0 1 0 0 0 0 1>; + orientation-x= <0>; + orientation-y= <0>; + orientation-z= <1>; + mpu-debug = <1>; + }; + + sensor@d { + status = "okay"; + compatible = "ak8963"; + reg = <0x0d>; + type = ; + irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_EDGE_RISING>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <3>; + }; +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio4 RK_PC3 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c8 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + clock-frequency = <100000>; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + +}; + +&i2s1 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + bt656-supply = <&vcca_1v8>; + audio-supply = <&vcca_1v8>; + sdmmc-supply = <&vccio_sd>; + gpio1830-supply = <&vcc_3v0>; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&pcie_phy { + status = "disabled"; +}; + +&pcie0 { + status = "disabled"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm2 { + status = "okay"; +}; + +&rk_key { + status = "disabled"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_OSC_DIS + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = ; + rockchip,pwm-regulator-config = ; + rockchip,power-ctrl = + <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, + <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; +}; + +&route_edp { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdmmc { + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; +}; + +&spi1 { + status = "okay"; + max-freq = <48000000>; /* spi internal clk, don't modify */ + spi_dev@0 { + compatible = "rockchip,spidev"; + reg = <0>; + spi-max-frequency = <12000000>; + spi-lsb-first; + }; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd_dwc3_1 { + status = "okay"; +}; + +&vopb { + assigned-clocks = <&cru DCLK_VOP0_DIV>; + assigned-clock-parents = <&cru PLL_CPLL>; +}; + +&vopl { + assigned-clocks = <&cru DCLK_VOP1_DIV>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&npu_ref_clk>; + + bq2570 { + charger_ok_int: charger-ok-int { + rockchip,pins = + <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd_rst { + lcd_rst_gpio: lcd-rst-gpio { + rockchip,pins = + <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + npu_clk { + npu_ref_clk: npu-ref-clk { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + soc_slppin_gpio: soc-slppin-gpio { + rockchip,pins = + <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc-slppin-slp { + rockchip,pins = + <1 RK_PA5 1 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdmmc { + sdmmc_bus1: sdmmc-bus1 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_10ma>; + }; + + sdmmc_bus4: sdmmc-bus4 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_10ma>, + <4 RK_PB1 1 &pcfg_pull_up_10ma>, + <4 RK_PB2 1 &pcfg_pull_up_10ma>, + <4 RK_PB3 1 &pcfg_pull_up_10ma>; + }; + + sdmmc_clk: sdmmc-clk { + rockchip,pins = + <4 RK_PB4 1 &pcfg_pull_none_10ma>; + }; + + sdmmc_cmd: sdmmc-cmd { + rockchip,pins = + <4 RK_PB5 1 &pcfg_pull_up_10ma>; + }; + }; + + tp_irq { + tp_irq_gpio: tp-irq-gpio { + rockchip,pins = + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + bt_irq_gpio: bt-irq-gpio { + rockchip,pins = + <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11-linux.dts new file mode 100755 index 000000000000..c3faaa1ba4d3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11-linux.dts @@ -0,0 +1,1223 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + +/dts-v1/; +#include +#include +#include +#include +#include +#include +#include "rk3399pro.dtsi" +#include "rk3399-linux.dtsi" +#include "rk3399-opp.dtsi" +#include "rk3399-vop-clk-set.dtsi" + +/ { + compatible = "rockchip,rk3399pro-evb-v11-linux", "rockchip,rk3399pro"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + enable-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + fiq_debugger: fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + rockchip,irq-mode-enable = <0>; /* If enable uart uses irq instead of fiq */ + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + pinctrl-names = "default"; + pinctrl-0 = <&uart2c_xfer>; + interrupts = ; + }; + + hdmi_sound: hdmi-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,hdmi"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + enable-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + reset-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + rk809_sound: rk809-sound { + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk809-codec"; + rockchip,codec-hp-det; + rockchip,mclk-fs = <256>; + rockchip,cpu = <&i2s1>; + rockchip,codec = <&rk809_codec>; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 3>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + }; + + usbacm_video_control: usbacm-video-control { + compatible = "rockchip,usbacm-video-control"; + status = "disabled"; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6398s"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>, <&bt_irq_gpio>; + pinctrl-1 = <&uart0_gpios>; + BT,reset_gpio = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&display_subsystem { + status = "okay"; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; +}; + +&dp_in_vopb { + status = "disabled"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopb { + status = "disabled"; +}; + +&emmc_phy { + status = "okay"; +}; + +&fiq_debugger { + pinctrl-0 = <&uart2a_xfer>; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&hdmi { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,phy-table = + <74250000 0x8009 0x0004 0x0272>, + <165000000 0x802b 0x0004 0x0209>, + <297000000 0x8039 0x0005 0x028d>, + <594000000 0x8039 0x0000 0x00f6>, + <000000000 0x0000 0x0000 0x0000>; +}; + +&hdmi_in_vopl { + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <180>; + i2c-scl-falling-time-ns = <30>; + clock-frequency = <400000>; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int_l>; + pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk809_slppin_null>; + rockchip,system-power-controller; + pmic-reset-func = <0>; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc_buck5>; + vcc6-supply = <&vcc_buck5>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + rtc { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk809_slppin_null: rk809_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk809_slppin_slp: rk809_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk809_slppin_pwrdn: rk809_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk809_slppin_rst: rk809_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_center: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_center"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_sys: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_buck5: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2200000>; + regulator-max-microvolt = <2200000>; + regulator-name = "vcc_buck5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <2200000>; + }; + }; + + vcca_0v9: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vcca_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc0v9_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-name = "vcc0v9_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vcca_1v8: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcca_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd1v5_dvp: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc5v0_usb: SWITCH_REG1 { + regulator-name = "vcc5v0_usb"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vccio_3v3: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vccio_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; + + vdd_cpu_b: tcs452x@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: tcs452x@10 { + compatible = "tcs,tcs452x"; + reg = <0x10>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <735000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <1000>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + bq25700: bq25700@6b { + compatible = "ti,bq25703"; + reg = <0x6b>; + extcon = <&fusb0>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&charger_ok_int>; + ti,charge-current = <1500000>; + ti,max-charge-voltage = <8704000>; + ti,max-input-voltage = <20000000>; + ti,max-input-current = <6000000>; + ti,input-current-sdp = <500000>; + ti,input-current-dcp = <2000000>; + ti,input-current-cdp = <2000000>; + ti,input-current-dc = <2000000>; + ti,minimum-sys-voltage = <6700000>; + ti,otg-voltage = <5000000>; + ti,otg-current = <500000>; + ti,input-current = <500000>; + pd-charge-only = <0>; + status = "disabled"; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <140>; + i2c-scl-falling-time-ns = <30>; + + mpu6500@68 { + status = "okay"; + compatible = "invensense,mpu6500"; + reg = <0x68>; + irq-gpio = <&gpio3 RK_PD2 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <0 1 0 1 0 0 0 0 1>; + orientation-x= <1>; + orientation-y= <0>; + orientation-z= <0>; + mpu-debug = <1>; + }; + + sensor@d { + status = "okay"; + compatible = "ak8963"; + reg = <0x0d>; + type = ; + irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_EDGE_RISING>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <3>; + }; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "okay"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + + lens-focus = <&vm149c>; + + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; + + imx327: imx327@1a { + compatible = "sony,imx327"; + status = "okay"; + reg = <0x1a>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ucam_out2: endpoint { + remote-endpoint = <&mipi_in_ucam2>; + data-lanes = <1 2>; + }; + }; + }; + +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio4 RK_PC3 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c8 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + clock-frequency = <100000>; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; + vbus-5v-gpios = <&gpio0 RK_PA1 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + +}; + +&i2s1 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&io_domains { + status = "okay"; + bt656-supply = <&vcca_1v8>; + audio-supply = <&vcca_1v8>; + sdmmc-supply = <&vccio_sd>; + gpio1830-supply = <&vcc_3v0>; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + /* Unlinked camera */ + //remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + + mipi_in_ucam2: endpoint@2 { + reg = <2>; + remote-endpoint = <&ucam_out2>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&pcie_phy { + status = "okay"; +}; + +&pcie0 { + status = "okay"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm2 { + status = "okay"; +}; + +&rkisp1_0 { + status = "okay"; + assigned-clocks = <&cru PLL_NPLL>, <&cru SCLK_CIF_OUT_SRC>, <&cru SCLK_CIF_OUT>; + assigned-clock-rates = <594000000>, <594000000>, <37125000>; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&rkisp1_1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp1_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_tx1rx1_out>; + }; + }; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_OSC_DIS + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = ; + rockchip,pwm-regulator-config = ; + rockchip,power-ctrl = + <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, + <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; +}; + +&route_edp { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdmmc { + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; +}; + +&spi1 { + status = "okay"; + max-freq = <48000000>; /* spi internal clk, don't modify */ + spi_dev@0 { + compatible = "rockchip,spidev"; + reg = <0>; + spi-max-frequency = <12000000>; + spi-lsb-first; + }; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd_dwc3_1 { + status = "okay"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&npu_ref_clk>; + + bq2570 { + charger_ok_int: charger-ok-int { + rockchip,pins = + <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + camera { + cam_pwren_high: cam-pwren-high { + rockchip,pins = + <4 RK_PC5 RK_FUNC_GPIO &pcfg_output_high>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd_rst { + lcd_rst_gpio: lcd-rst-gpio { + rockchip,pins = + <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + npu_clk { + npu_ref_clk: npu-ref-clk { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + soc_slppin_gpio: soc-slppin-gpio { + rockchip,pins = + <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc-slppin-slp { + rockchip,pins = + <1 RK_PA5 1 &pcfg_pull_down>; + }; + + soc_slppin_rst: soc-slppin-rst { + rockchip,pins = + <1 RK_PA5 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdmmc { + sdmmc_bus1: sdmmc-bus1 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_10ma>; + }; + + sdmmc_bus4: sdmmc-bus4 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_10ma>, + <4 RK_PB1 1 &pcfg_pull_up_10ma>, + <4 RK_PB2 1 &pcfg_pull_up_10ma>, + <4 RK_PB3 1 &pcfg_pull_up_10ma>; + }; + + sdmmc_clk: sdmmc-clk { + rockchip,pins = + <4 RK_PB4 1 &pcfg_pull_none_10ma>; + }; + + sdmmc_cmd: sdmmc-cmd { + rockchip,pins = + <4 RK_PB5 1 &pcfg_pull_up_10ma>; + }; + }; + + tp_irq { + tp_irq_gpio: tp-irq-gpio { + rockchip,pins = + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + wireless-bluetooth { + bt_irq_gpio: bt-irq-gpio { + rockchip,pins = + <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ +/* DON'T PUT ANYTHING BELOW HERE. PUT IT ABOVE PINCTRL */ diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11.dts new file mode 100755 index 000000000000..7d118a783353 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v11.dts @@ -0,0 +1,1045 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + +/dts-v1/; +#include +#include +#include +#include +#include +#include +#include "rk3399pro.dtsi" +#include "rk3399-android.dtsi" +#include "rk3399-opp.dtsi" +#include "rk3399-vop-clk-set.dtsi" + +/ { + model = "Rockchip RK3399pro evb v11 board"; + compatible = "rockchip,rk3399pro-evb-v11", "rockchip,rk3399pro"; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 2>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + esc-key { + linux,code = ; + label = "esc"; + press-threshold-microvolt = <1310000>; + }; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <987000>; + }; + + home-key { + linux,code = ; + label = "home"; + press-threshold-microvolt = <624000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <300000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <17000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + enable-gpios = <&gpio1 RK_PB5 GPIO_ACTIVE_HIGH>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + hdmi_sound: hdmi-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,hdmi"; + + simple-audio-card,cpu { + sound-dai = <&i2s2>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + panel: panel { + compatible = "simple-panel"; + backlight = <&backlight>; + enable-gpios = <&gpio4 RK_PD6 GPIO_ACTIVE_HIGH>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + reset-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + + rk809-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 3>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + }; + + vcc_phy: vcc-phy-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_phy"; + regulator-always-on; + regulator-boot-on; + }; + + vcc5v0_sys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6398s"; + sdio_vref = <1800>; + WIFI,host_wake_irq = <&gpio0 RK_PA3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + uart_rts_gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart0_rts>, <&bt_irq_gpio>; + pinctrl-1 = <&uart0_gpios>; + BT,reset_gpio = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cdn_dp { + status = "okay"; + extcon = <&fusb0>; + phys = <&tcphy0_dp>; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&dmc { + status = "okay"; + center-supply = <&vdd_center>; +}; + +&dp_in_vopb { + status = "disabled"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; +}; + +&edp_in_vopl { + status = "disabled"; +}; + +&emmc_phy { + status = "okay"; +}; + +&fiq_debugger { + pinctrl-0 = <&uart2a_xfer>; +}; + +&gmac { + phy-supply = <&vcc_phy>; + phy-mode = "rgmii"; + clock_in_out = "input"; + snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&gpu { + status = "okay"; + mali-supply = <&vdd_gpu>; +}; + +&hdmi { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,phy-table = + <74250000 0x8009 0x0004 0x0272>, + <165000000 0x802b 0x0004 0x0209>, + <297000000 0x8039 0x0005 0x028d>, + <594000000 0x8039 0x0000 0x00f6>, + <000000000 0x0000 0x0000 0x0000>; +}; + +&hdmi_dp_sound { + status = "okay"; +}; + +&hdmi_in_vopb { + status = "disabled"; +}; + +&i2s2 { + #sound-dai-cells = <0>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + i2c-scl-rising-time-ns = <180>; + i2c-scl-falling-time-ns = <30>; + clock-frequency = <400000>; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int_l>; + pinctrl-1 = <&soc_slppin_slp>, <&rk809_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk809_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>,<&rk809_slppin_null>; + rockchip,system-power-controller; + pmic-reset-func = <0>; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc_buck5>; + vcc6-supply = <&vcc_buck5>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + pwrkey { + status = "okay"; + }; + + rtc { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk809_slppin_null: rk809_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk809_slppin_slp: rk809_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk809_slppin_pwrdn: rk809_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk809_slppin_rst: rk809_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_center: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_center"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_sys: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_buck5: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2200000>; + regulator-max-microvolt = <2200000>; + regulator-name = "vcc_buck5"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <2200000>; + }; + }; + + vcca_0v9: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vcca_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc0v9_soc: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + + regulator-name = "vcc0v9_soc"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vcca_1v8: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + regulator-name = "vcca_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd1v5_dvp: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vdd1v5_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + + regulator-name = "vcc_1v5"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + + regulator-name = "vcc_3v0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_sd: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + regulator-name = "vcc_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc5v0_usb: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_usb"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vccio_3v3: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vccio_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s_8ch_mclk>; + hp-volume = <20>; + spk-volume = <3>; + status = "okay"; + }; + }; + + vdd_cpu_b: tcs452x@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: tcs452x@10 { + compatible = "tcs,tcs452x"; + reg = <0x10>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <735000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + bq25700: bq25700@6b { + compatible = "ti,bq25703"; + reg = <0x6b>; + extcon = <&fusb0>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&charger_ok_int>; + ti,charge-current = <1500000>; + ti,max-charge-voltage = <8704000>; + ti,max-input-voltage = <20000000>; + ti,max-input-current = <6000000>; + ti,input-current-sdp = <500000>; + ti,input-current-dcp = <2000000>; + ti,input-current-cdp = <2000000>; + ti,input-current-dc = <2000000>; + ti,minimum-sys-voltage = <6700000>; + ti,otg-voltage = <5000000>; + ti,otg-current = <500000>; + ti,input-current = <500000>; + pd-charge-only = <0>; + status = "disabled"; + }; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <140>; + i2c-scl-falling-time-ns = <30>; + + mpu6500@68 { + status = "okay"; + compatible = "invensense,mpu6500"; + reg = <0x68>; + irq-gpio = <&gpio3 RK_PD2 IRQ_TYPE_EDGE_RISING>; + mpu-int_config = <0x10>; + mpu-level_shifter = <0>; + mpu-orientation = <0 1 0 1 0 0 0 0 1>; + orientation-x= <0>; + orientation-y= <0>; + orientation-z= <1>; + mpu-debug = <1>; + }; + + sensor@d { + status = "okay"; + compatible = "ak8963"; + reg = <0x0d>; + type = ; + irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_EDGE_RISING>; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <3>; + }; +}; + +&i2c4 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + + gsl3673: gsl3673@40 { + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio4 RK_PC3 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c8 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + clock-frequency = <100000>; + + fusb0: fusb30x@22 { + compatible = "fairchild,fusb302"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + int-n-gpios = <&gpio1 RK_PA2 GPIO_ACTIVE_HIGH>; + vbus-5v-gpios = <&gpio0 RK_PA1 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + +}; + +&i2s1 { + status = "okay"; + #sound-dai-cells = <0>; +}; + +&io_domains { + status = "okay"; + bt656-supply = <&vcca_1v8>; + audio-supply = <&vcca_1v8>; + sdmmc-supply = <&vccio_sd>; + gpio1830-supply = <&vcc_3v0>; +}; + +&isp0_mmu { + status = "okay"; +}; + +&isp1_mmu { + status = "okay"; +}; + +&pcie_phy { + status = "disabled"; +}; + +&pcie0 { + status = "disabled"; +}; + +&pmu_io_domains { + status = "okay"; + pmu1830-supply = <&vcc_1v8>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm2 { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMPD + | RKPM_SLP_PERILPPD + | RKPM_SLP_DDR_RET + | RKPM_SLP_PLLPD + | RKPM_SLP_CENTER_PD + | RKPM_SLP_OSC_DIS + | RKPM_SLP_AP_PWROFF + ) + >; + rockchip,wakeup-config = ; + rockchip,pwm-regulator-config = ; + rockchip,power-ctrl = + <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>, + <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; +}; + +&route_edp { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdmmc { + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; +}; + +&spi1 { + status = "okay"; + max-freq = <48000000>; /* spi internal clk, don't modify */ + spi_dev@0 { + compatible = "rockchip,spidev"; + reg = <0>; + spi-max-frequency = <12000000>; + spi-lsb-first; + }; +}; + +&tcphy0 { + extcon = <&fusb0>; + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + extcon = <&fusb0>; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_usb>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts>; + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + extcon = <&fusb0>; +}; + +&usbdrd_dwc3_1 { + status = "okay"; +}; + +&vopb { + assigned-clocks = <&cru DCLK_VOP0_DIV>; + assigned-clock-parents = <&cru PLL_CPLL>; +}; + +&vopl { + assigned-clocks = <&cru DCLK_VOP1_DIV>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; + +&pinctrl { + pinctrl-names = "default"; + pinctrl-0 = <&npu_ref_clk>; + + bq2570 { + charger_ok_int: charger-ok-int { + rockchip,pins = + <1 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + fusb30x { + fusb0_int: fusb0-int { + rockchip,pins = + <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd_rst { + lcd_rst_gpio: lcd-rst-gpio { + rockchip,pins = + <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + npu_clk { + npu_ref_clk: npu-ref-clk { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <1 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + vsel1_gpio: vsel1-gpio { + rockchip,pins = + <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + vsel2_gpio: vsel2-gpio { + rockchip,pins = + <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + soc_slppin_gpio: soc-slppin-gpio { + rockchip,pins = + <1 RK_PA5 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc-slppin-slp { + rockchip,pins = + <1 RK_PA5 1 &pcfg_pull_down>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdmmc { + sdmmc_bus1: sdmmc-bus1 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_10ma>; + }; + + sdmmc_bus4: sdmmc-bus4 { + rockchip,pins = + <4 RK_PB0 1 &pcfg_pull_up_10ma>, + <4 RK_PB1 1 &pcfg_pull_up_10ma>, + <4 RK_PB2 1 &pcfg_pull_up_10ma>, + <4 RK_PB3 1 &pcfg_pull_up_10ma>; + }; + + sdmmc_clk: sdmmc-clk { + rockchip,pins = + <4 RK_PB4 1 &pcfg_pull_none_10ma>; + }; + + sdmmc_cmd: sdmmc-cmd { + rockchip,pins = + <4 RK_PB5 1 &pcfg_pull_up_10ma>; + }; + }; + + tp_irq { + tp_irq_gpio: tp-irq-gpio { + rockchip,pins = + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + wireless-bluetooth { + uart0_gpios: uart0-gpios { + rockchip,pins = + <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_irq_gpio: bt-irq-gpio { + rockchip,pins = + <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v14-linux.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v14-linux.dts new file mode 100755 index 000000000000..4f8546eb80bf --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399pro-evb-v14-linux.dts @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + * + */ + +/dts-v1/; +#include "rk3399pro-evb-v11-linux.dts" + +/ { + model = "Rockchip RK3399pro evb v14 board for linux"; + compatible = "rockchip,rk3399pro-evb-v14-linux", "rockchip,rk3399pro"; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + + dma_trans: dma_trans@3c000000 { + //no-map; + reg = <0x0 0x3c000000 0x0 0x04000000>; + }; + }; +}; + +/delete-node/ &imx327; +/delete-node/ &ov13850; +/delete-node/ &vm149c; +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + pinctrl-0 = <&i2c1_xfer>, <&cam_pwren_high>; + + jaguar1: jaguar1@30 { + compatible = "jaguar1-v4l2"; + status = "okay"; + reg = <0x30>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* + * pd-gpios = <&gpio4 21 GPIO_ACTIVE_HIGH>; // conflict with csi-ctl-gpios + * rst-gpios = <&gpio0 12 GPIO_ACTIVE_HIGH>; + */ + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "jaguar1"; + rockchip,camera-module-lens-name = "jaguar1"; + port { + cam_out: endpoint { + remote-endpoint = <&usbacm_video_control_in>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&i2c4 { + status = "okay"; + + i2c-scl-rising-time-ns = <345>; + i2c-scl-falling-time-ns = <11>; + + vm149c: vm149c@0c { + compatible = "silicon touch,vm149c"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + ov13850: ov13850@10 { + compatible = "ovti,ov13850"; + status = "okay"; + reg = <0x10>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio4 RK_PD2 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 RK_PD1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-CT0116"; + rockchip,camera-module-lens-name = "Largan-50013A1"; + lens-focus = <&vm149c>; + + port { + ucam_out1: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2>; + }; + }; + }; + + imx327: imx327@1a { + compatible = "sony,imx327"; + status = "okay"; + reg = <0x1a>; + clocks = <&cru SCLK_CIF_OUT>; + clock-names = "xvclk"; + /* conflict with csi-ctl-gpios */ + reset-gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clkout>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ucam_out2: endpoint { + remote-endpoint = <&mipi_in_ucam2>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&mipi_dphy_rx0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&usbacm_video_control_out>; + data-lanes = <1 2 3 4>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_rx0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_mipi_in>; + }; + }; + }; +}; + +&mipi_dphy_tx1rx1 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam1: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out1>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy_tx1rx1_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp1_mipi_in>; + }; + }; + }; +}; + +&pcie0 { + /delete-property/ ep-gpios; + num-lanes = <4>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_clkreqn_cpm>; + max-link-speed = <1>; + memory-region = <&dma_trans>; + busno = <0>; + rockchip,dma_trx_enabled = <1>; + rockchip,deferred = <1>; + status = "okay"; +}; + +&rkisp1_0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_mipi_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy_rx0_out>; + }; + }; +}; + +&usbacm_video_control { + status = "okay"; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "usbacm_video_control"; + rockchip,camera-module-lens-name = "usbacm_video_control"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + usbacm_video_control_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&cam_out>; + data-lanes = <1 2 3 4>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + usbacm_video_control_out: endpoint@1 { + reg = <1>; + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-npu-evb-v10.dts b/arch/arm64/boot/dts/rockchip/rk3399pro-npu-evb-v10.dts new file mode 100755 index 000000000000..96a3fb56cfe5 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399pro-npu-evb-v10.dts @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +/dts-v1/; +#include +#include +#include "rk3399pro-npu.dtsi" + +/ { + model = "Rockchip RK3399pro-npu EVB V10 Board"; + compatible = "rockchip,rk3399pro-npu-evb-v10", "rockchip,rk3399pro-npu"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xff550000 console=ttyFIQ0 init=/init kpti=0"; + }; + + keys: gpio-keys { + compatible = "gpio-keys"; + pinctrl-names = "default"; + pinctrl-0 = <&pwr_key>; + + power { + gpios = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; + label = "GPIO Power"; + linux,code = <116>; + wakeup-source; + }; + }; + + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <0>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "okay"; + }; + + vdd_cpu: vdd-cpu { + compatible = "regulator-fixed"; + regulator-name = "vdd_cpu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <850000>; + regulator-max-microvolt = <850000>; + }; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu1 { + cpu-supply = <&vdd_cpu>; +}; + +&i2c1 { + status = "okay"; + + vdd_npu: tcs452x@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel_gpio>; + vsel-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_npu"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <800000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-initial-state = <3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; +}; + +&npu { + npu-supply = <&vdd_npu>; + status = "okay"; +}; + +&combphy { + status = "okay"; +}; + +&u2phy { + status = "okay"; +}; + +&u2phy_otg { + status = "okay"; +}; + +&usbdrd3 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */ + pinctrl-names = "init", "default"; + pinctrl-0 = <&tsadc_otp_gpio>; + pinctrl-1 = <&tsadc_otp_out>; + status = "okay"; +}; + +&pinctrl { + vsel_gpio: vsel-gpio { + rockchip,pins = + <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + pwr_key: pwr-key { + rockchip,pins = + <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-npu.dtsi b/arch/arm64/boot/dts/rockchip/rk3399pro-npu.dtsi new file mode 100755 index 000000000000..3f176b3bc94f --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399pro-npu.dtsi @@ -0,0 +1,826 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd. + +#include +#include +#include +#include +#include +#include +#include + +/ { + compatible = "rockchip,rk3399pro-npu"; + + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + serial2 = &uart2; + }; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a35", "arm,armv8"; + reg = <0x0 0x0>; + enable-method = "psci"; + clocks = <&cru ARMCLK>; + operating-points-v2 = <&cpu0_opp_table>; + dynamic-power-coefficient = <74>; + #cooling-cells = <2>; + power-model { + compatible = "simple-power-model"; + ref-leakage = <31>; + static-coefficient = <100000>; + ts = <597400 241050 (-2450) 70>; + thermal-zone = "soc-thermal"; + }; + }; + + cpu1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a35", "arm,armv8"; + reg = <0x0 0x1>; + enable-method = "psci"; + clocks = <&cru ARMCLK>; + operating-points-v2 = <&cpu0_opp_table>; + dynamic-power-coefficient = <74>; + }; + }; + + cpu0_opp_table: cpu0-opp-table { + compatible = "operating-points-v2"; + opp-shared; + + opp-408000000 { + opp-hz = /bits/ 64 <408000000>; + opp-microvolt = <750000 750000 950000>; + clock-latency-ns = <40000>; + opp-suspend; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <750000 750000 950000>; + clock-latency-ns = <40000>; + }; + opp-816000000 { + opp-hz = /bits/ 64 <816000000>; + opp-microvolt = <750000 750000 950000>; + clock-latency-ns = <40000>; + }; + opp-1008000000 { + opp-hz = /bits/ 64 <1008000000>; + opp-microvolt = <750000 750000 950000>; + clock-latency-ns = <40000>; + }; + opp-1200000000 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <750000 750000 950000>; + clock-latency-ns = <40000>; + }; + }; + + arm-pmu { + compatible = "arm,cortex-a53-pmu"; + interrupts = , + ; + interrupt-affinity = <&cpu0>, <&cpu1>; + }; + + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , + , + , + ; + arm,no-tick-in-suspend; + }; + + xin24m: xin24m { + compatible = "fixed-clock"; + clock-frequency = <24000000>; + clock-output-names = "xin24m"; + #clock-cells = <0>; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&clkin_32k>; + }; + + usbdrd3: usb { + compatible = "rockchip,rk1808-dwc3", "rockchip,rk3399-dwc3"; + clocks = <&cru SCLK_USB3_OTG0_REF>, <&cru ACLK_USB3OTG>, + <&cru SCLK_USB3_OTG0_SUSPEND>; + clock-names = "ref_clk", "bus_clk", + "suspend_clk"; + assigned-clocks = <&cru SCLK_USB3_OTG0_SUSPEND>; + assigned-clock-rates = <24000000>; + power-domains = <&power RK1808_PD_PCIE>; + resets = <&cru SRST_USB3_OTG_A>; + reset-names = "usb3-otg"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + status = "disabled"; + + usbdrd_dwc3: dwc3@fd000000 { + compatible = "snps,dwc3"; + reg = <0x0 0xfd000000 0x0 0x200000>; + interrupts = ; + dr_mode = "peripheral"; + phys = <&u2phy_otg>, <&combphy PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + phy_type = "utmi_wide"; + snps,dis_enblslpm_quirk; + snps,dis-u1u2-quirk; + snps,dis-u2-freeclk-exists-quirk; + snps,dis_u2_susphy_quirk; + snps,dis_u3_susphy_quirk; + snps,dis-del-phy-power-chg-quirk; + snps,tx-ipgap-linecheck-dis-quirk; + snps,xhci-trb-ent-quirk; + status = "disabled"; + }; + }; + + grf: syscon@fe000000 { + compatible = "rockchip,rk1808-grf", "syscon", "simple-mfd"; + reg = <0x0 0xfe000000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + npu_pvtm: npu-pvtm { + compatible = "rockchip,rk1808-npu-pvtm"; + clocks = <&cru SCLK_PVTM_NPU>; + clock-names = "npu"; + status = "okay"; + }; + }; + + usb2phy_grf: syscon@fe010000 { + compatible = "rockchip,rk1808-usb2phy-grf", "syscon", + "simple-mfd"; + reg = <0x0 0xfe010000 0x0 0x8000>; + #address-cells = <1>; + #size-cells = <1>; + + u2phy: usb2-phy@100 { + compatible = "rockchip,rk1808-usb2phy"; + reg = <0x100 0x10>; + clocks = <&cru SCLK_USBPHY_REF>; + clock-names = "phyclk"; + #clock-cells = <0>; + assigned-clocks = <&cru USB480M>; + assigned-clock-parents = <&u2phy>; + clock-output-names = "usb480m_phy"; + status = "disabled"; + + u2phy_host: host-port { + #phy-cells = <0>; + interrupts = ; + interrupt-names = "linestate"; + status = "disabled"; + }; + + u2phy_otg: otg-port { + #phy-cells = <0>; + interrupts = , + , + ; + interrupt-names = "otg-bvalid", "otg-id", + "linestate"; + status = "disabled"; + }; + }; + }; + + combphy_grf: syscon@fe018000 { + compatible = "rockchip,usb3phy-grf", "syscon"; + reg = <0x0 0xfe018000 0x0 0x8000>; + }; + + pmugrf: syscon@fe020000 { + compatible = "rockchip,rk1808-pmugrf", "syscon", "simple-mfd"; + reg = <0x0 0xfe020000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + pmu_pvtm: pmu-pvtm { + compatible = "rockchip,rk1808-pmu-pvtm"; + clocks = <&cru SCLK_PVTM_PMU>; + clock-names = "pmu"; + status = "okay"; + }; + }; + + usb_pcie_grf: syscon@fe040000 { + compatible = "rockchip,usb-pcie-grf", "syscon"; + reg = <0x0 0xfe040000 0x0 0x1000>; + }; + + coregrf: syscon@fe050000 { + compatible = "rockchip,rk1808-coregrf", "syscon", "simple-mfd"; + reg = <0x0 0xfe050000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + pvtm: pvtm { + compatible = "rockchip,rk1808-pvtm"; + clocks = <&cru SCLK_PVTM_CORE>; + clock-names = "core"; + status = "okay"; + }; + }; + + qos_npu: qos@fe850000 { + compatible = "syscon"; + reg = <0x0 0xfe850000 0x0 0x20>; + }; + + qos_pcie: qos@fe880000 { + compatible = "syscon"; + reg = <0x0 0xfe880000 0x0 0x20>; + status = "disabled"; + }; + + qos_usb2: qos@fe890000 { + compatible = "syscon"; + reg = <0x0 0xfe890000 0x0 0x20>; + status = "disabled"; + }; + + qos_usb3: qos@fe890080 { + compatible = "syscon"; + reg = <0x0 0xfe890080 0x0 0x20>; + status = "disabled"; + }; + + qos_isp: qos@fe8a0000 { + compatible = "syscon"; + reg = <0x0 0xfe8a0000 0x0 0x20>; + }; + + qos_rga_rd: qos@fe8a0080 { + compatible = "syscon"; + reg = <0x0 0xfe8a0080 0x0 0x20>; + }; + + qos_rga_wr: qos@fe8a0100 { + compatible = "syscon"; + reg = <0x0 0xfe8a0100 0x0 0x20>; + }; + + qos_cif: qos@fe8a0180 { + compatible = "syscon"; + reg = <0x0 0xfe8a0180 0x0 0x20>; + }; + + qos_vop_raw: qos@fe8b0000 { + compatible = "syscon"; + reg = <0x0 0xfe8b0000 0x0 0x20>; + }; + + qos_vop_lite: qos@fe8b0080 { + compatible = "syscon"; + reg = <0x0 0xfe8b0080 0x0 0x20>; + }; + + qos_vpu: qos@fe8c0000 { + compatible = "syscon"; + reg = <0x0 0xfe8c0000 0x0 0x20>; + }; + + gic: interrupt-controller@ff100000 { + compatible = "arm,gic-v3"; + #interrupt-cells = <3>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + interrupt-controller; + + reg = <0x0 0xff100000 0 0x10000>, /* GICD */ + <0x0 0xff140000 0 0xc0000>, /* GICR */ + <0x0 0xff300000 0 0x10000>, /* GICC */ + <0x0 0xff310000 0 0x10000>, /* GICH */ + <0x0 0xff320000 0 0x10000>; /* GICV */ + interrupts = ; + its: interrupt-controller@ff120000 { + compatible = "arm,gic-v3-its"; + msi-controller; + reg = <0x0 0xff120000 0x0 0x20000>; + }; + }; + + cru: clock-controller@ff350000 { + compatible = "rockchip,rk1808-cru"; + reg = <0x0 0xff350000 0x0 0x5000>; + rockchip,grf = <&grf>; + #clock-cells = <1>; + #reset-cells = <1>; + + assigned-clocks = + <&cru PLL_GPLL>, <&cru PLL_CPLL>, + <&cru PLL_PPLL>, <&cru ARMCLK>, + <&cru MSCLK_PERI>, <&cru LSCLK_PERI>, + <&cru HSCLK_BUS_PRE>, <&cru MSCLK_BUS_PRE>, + <&cru LSCLK_BUS_PRE>; + assigned-clock-rates = + <1188000000>, <1000000000>, + <100000000>, <1200000000>, + <200000000>, <100000000>, + <300000000>, <200000000>, + <100000000>; + }; + + combphy: phy@ff380000 { + compatible = "rockchip,rk1808-combphy"; + reg = <0x0 0xff380000 0x0 0x10000>; + #phy-cells = <1>; + clocks = <&cru SCLK_PCIEPHY_REF>; + clock-names = "refclk"; + assigned-clocks = <&cru SCLK_PCIEPHY_REF>; + assigned-clock-rates = <25000000>; + resets = <&cru SRST_USB3_OTG_A>, <&cru SRST_PCIEPHY_POR>, + <&cru SRST_PCIEPHY_P>, <&cru SRST_PCIEPHY_PIPE>; + reset-names = "otg-rst", "combphy-por", + "combphy-apb", "combphy-pipe"; + rockchip,combphygrf = <&combphy_grf>; + rockchip,usbpciegrf = <&usb_pcie_grf>; + status = "disabled"; + }; + + thermal_zones: thermal-zones { + soc_thermal: soc-thermal { + polling-delay-passive = <20>; /* milliseconds */ + polling-delay = <1000>; /* milliseconds */ + sustainable-power = <977>; /* milliwatts */ + + thermal-sensors = <&tsadc 0>; + + trips { + threshold: trip-point-0 { + /* millicelsius */ + temperature = <75000>; + /* millicelsius */ + hysteresis = <2000>; + type = "passive"; + }; + target: trip-point-1 { + /* millicelsius */ + temperature = <85000>; + /* millicelsius */ + hysteresis = <2000>; + type = "passive"; + }; + soc_crit: soc-crit { + /* millicelsius */ + temperature = <115000>; + /* millicelsius */ + hysteresis = <2000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&target>; + cooling-device = + <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <4096>; + }; + map1 { + trip = <&target>; + cooling-device = + <&npu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + }; + }; + }; + + tsadc: tsadc@ff3a0000 { + compatible = "rockchip,rk1808-tsadc"; + reg = <0x0 0xff3a0000 0x0 0x100>; + interrupts = ; + rockchip,grf = <&grf>; + clocks = <&cru SCLK_TSADC>, <&cru PCLK_TSADC>; + clock-names = "tsadc", "apb_pclk"; + assigned-clocks = <&cru SCLK_TSADC>; + assigned-clock-rates = <650000>; + resets = <&cru SRST_TSADC>; + reset-names = "tsadc-apb"; + #thermal-sensor-cells = <1>; + rockchip,hw-tshut-temp = <120000>; + status = "disabled"; + }; + + pmu: power-management@ff3e0000 { + compatible = "rockchip,rk1808-pmu", "syscon", "simple-mfd"; + reg = <0x0 0xff3e0000 0x0 0x1000>; + + power: power-controller { + compatible = "rockchip,rk1808-power-controller"; + #power-domain-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + /* These power domains are grouped by VD_NPU */ + pd_npu@RK1808_VD_NPU { + reg = ; + clocks = <&cru SCLK_NPU>, + <&cru ACLK_NPU>, + <&cru HCLK_NPU>; + pm_qos = <&qos_npu>; + }; + + /* These power domains are grouped by VD_LOGIC */ + pd_pcie@RK1808_PD_PCIE { + reg = ; + clocks = <&cru HSCLK_PCIE>, + <&cru LSCLK_PCIE>, + <&cru ACLK_PCIE>, + <&cru ACLK_PCIE_MST>, + <&cru ACLK_PCIE_SLV>, + <&cru PCLK_PCIE>, + <&cru SCLK_PCIE_AUX>, + <&cru SCLK_PCIE_AUX>, + <&cru ACLK_USB3OTG>, + <&cru HCLK_HOST>, + <&cru HCLK_HOST_ARB>, + <&cru SCLK_USB3_OTG0_REF>, + <&cru SCLK_USB3_OTG0_SUSPEND>; + pm_qos = <&qos_pcie>, + <&qos_usb2>, + <&qos_usb3>; + }; + pd_vpu@RK1808_PD_VPU { + reg = ; + clocks = <&cru ACLK_VPU>, + <&cru HCLK_VPU>; + pm_qos = <&qos_vpu>; + }; + pd_vio@RK1808_PD_VIO { + reg = ; + clocks = <&cru HSCLK_VIO>, + <&cru LSCLK_VIO>, + <&cru ACLK_VOPRAW>, + <&cru HCLK_VOPRAW>, + <&cru ACLK_VOPLITE>, + <&cru HCLK_VOPLITE>, + <&cru PCLK_DSI_TX>, + <&cru PCLK_CSI_TX>, + <&cru ACLK_RGA>, + <&cru HCLK_RGA>, + <&cru ACLK_ISP>, + <&cru HCLK_ISP>, + <&cru ACLK_CIF>, + <&cru HCLK_CIF>, + <&cru PCLK_CSI2HOST>, + <&cru DCLK_VOPRAW>, + <&cru DCLK_VOPLITE>; + pm_qos = <&qos_rga_rd>, <&qos_rga_wr>, + <&qos_isp>, <&qos_cif>, + <&qos_vop_raw>, <&qos_vop_lite>; + }; + }; + }; + + i2c0: i2c@ff410000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xff410000 0x0 0x1000>; + clocks = <&cru SCLK_PMU_I2C0>, <&cru PCLK_I2C0_PMU>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + dmac: dmac@ff4e0000 { + compatible = "arm,pl330", "arm,primecell"; + reg = <0x0 0xff4e0000 0x0 0x4000>; + interrupts = ; + clocks = <&cru ACLK_DMAC>; + clock-names = "apb_pclk"; + #dma-cells = <1>; + arm,pl330-periph-burst; + }; + + i2c1: i2c@ff500000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xff500000 0x0 0x1000>; + clocks = <&cru SCLK_I2C1>, <&cru PCLK_I2C1>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + uart2: serial@ff550000 { + compatible = "rockchip,rk1808-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff550000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART2>, <&cru PCLK_UART2>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac 4>, <&dmac 5>; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "disabled"; + }; + + rktimer: rktimer@ff700000 { + compatible = "rockchip,rk3288-timer"; + reg = <0x0 0xff700000 0x0 0x1000>; + interrupts = ; + clocks = <&cru PCLK_TIMER>, <&cru SCLK_TIMER0>; + clock-names = "pclk", "timer"; + }; + + npu: npu@ffbc0000 { + compatible = "rockchip,npu"; + reg = <0x0 0xffbc0000 0x0 0x1000>; + clocks = <&cru SCLK_NPU>, <&cru ACLK_NPU>, <&cru HCLK_NPU>; + clock-names = "sclk_npu", "aclk_npu", "hclk_npu"; + assigned-clocks = <&cru SCLK_NPU>; + assigned-clock-rates = <800000000>; + interrupts = ; + power-domains = <&power RK1808_VD_NPU>; + operating-points-v2 = <&npu_opp_table>; + #cooling-cells = <2>; + status = "disabled"; + + npu_power_model: power-model { + compatible = "simple-power-model"; + ref-leakage = <31>; + static-coefficient = <100000>; + dynamic-coefficient = <3080>; + ts = <88610 303120 (-5000) 100>; + thermal-zone = "soc-thermal"; + }; + }; + + npu_opp_table: npu-opp-table { + compatible = "operating-points-v2"; + + rockchip,max-volt = <880000>; + rockchip,evb-irdrop = <37500>; + + rockchip,pvtm-voltage-sel = < + 0 69000 0 + 69001 74000 1 + 74001 99999 2 + >; + rockchip,pvtm-freq = <200000>; + rockchip,pvtm-volt = <800000>; + rockchip,pvtm-ch = <0 0>; + rockchip,pvtm-sample-time = <1000>; + rockchip,pvtm-number = <10>; + rockchip,pvtm-error = <1000>; + rockchip,pvtm-ref-temp = <25>; + rockchip,pvtm-temp-prop = <(-20) (-26)>; + rockchip,thermal-zone = "soc-thermal"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <750000 750000 880000>; + }; + opp-297000000 { + opp-hz = /bits/ 64 <297000000>; + opp-microvolt = <750000 750000 880000>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <750000 750000 880000>; + }; + opp-594000000 { + opp-hz = /bits/ 64 <594000000>; + opp-microvolt = <750000 750000 880000>; + }; + opp-792000000 { + opp-hz = /bits/ 64 <792000000>; + opp-microvolt = <850000 850000 880000>; + opp-microvolt-L0 = <850000 850000 880000>; + opp-microvolt-L1 = <825000 825000 880000>; + opp-microvolt-L2 = <800000 800000 880000>; + }; + }; + + pinctrl: pinctrl { + compatible = "rockchip,rk1808-pinctrl"; + rockchip,grf = <&grf>; + rockchip,pmu = <&pmugrf>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + gpio0: gpio0@ff4c0000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff4c0000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO0_PMU>, <&cru DBCLK_PMU_GPIO0>; + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio1: gpio1@ff690000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff690000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO1>, <&cru DBCLK_GPIO1>; + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio2: gpio2@ff6a0000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff6a0000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO2>, <&cru DBCLK_GPIO2>; + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio3: gpio3@ff6b0000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff6b0000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO3>, <&cru DBCLK_GPIO3>; + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio4: gpio4@ff6c0000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff6c0000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO4>, <&cru DBCLK_GPIO4>; + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + pcfg_pull_down: pcfg-pull-down { + bias-pull-down; + }; + + pcfg_pull_none: pcfg-pull-none { + bias-disable; + }; + + pcfg_pull_up_2ma: pcfg-pull-up-2ma { + bias-pull-up; + drive-strength = <2>; + }; + + pcfg_pull_none_smt: pcfg-pull-none-smt { + bias-disable; + input-schmitt-enable; + }; + + pcfg_pull_none_2ma_smt: pcfg-pull-none-2ma-smt { + bias-disable; + drive-strength = <2>; + input-schmitt-enable; + }; + + pcfg_output_high: pcfg-output-high { + output-high; + }; + + pcfg_input_smt: pcfg-input-smt { + input-enable; + input-schmitt-enable; + }; + + i2c0 { + i2c0_xfer: i2c0-xfer { + rockchip,pins = + /* i2c0_sda */ + <0 RK_PB1 1 &pcfg_pull_none_2ma_smt>, + /* i2c0_scl */ + <0 RK_PB0 1 &pcfg_pull_none_2ma_smt>; + }; + }; + + i2c1 { + i2c1_xfer: i2c1-xfer { + rockchip,pins = + /* i2c1_sda */ + <0 RK_PC1 1 &pcfg_pull_none_2ma_smt>, + /* i2c1_scl */ + <0 RK_PC0 1 &pcfg_pull_none_2ma_smt>; + }; + }; + + pciusb { + pciusb_pins: pciusb-pins { + rockchip,pins = + /* pciusb_debug0 */ + <4 RK_PB4 3 &pcfg_pull_none>, + /* pciusb_debug1 */ + <4 RK_PB5 3 &pcfg_pull_none>, + /* pciusb_debug2 */ + <4 RK_PB6 3 &pcfg_pull_none>, + /* pciusb_debug3 */ + <4 RK_PB7 3 &pcfg_pull_none>, + /* pciusb_debug4 */ + <4 RK_PC0 3 &pcfg_pull_none>, + /* pciusb_debug5 */ + <4 RK_PC1 3 &pcfg_pull_none>, + /* pciusb_debug6 */ + <4 RK_PC2 3 &pcfg_pull_none>, + /* pciusb_debug7 */ + <4 RK_PC3 3 &pcfg_pull_none>; + }; + }; + + uart2 { + uart2m0_xfer: uart2m0-xfer { + rockchip,pins = + /* uart2_rxm0 */ + <4 RK_PA3 2 &pcfg_pull_up_2ma>, + /* uart2_txm0 */ + <4 RK_PA2 2 &pcfg_pull_up_2ma>; + }; + + uart2m1_xfer: uart2m1-xfer { + rockchip,pins = + /* uart2_rxm1 */ + <2 RK_PD1 2 &pcfg_pull_up_2ma>, + /* uart2_txm1 */ + <2 RK_PD0 2 &pcfg_pull_up_2ma>; + }; + + uart2m2_xfer: uart2m2-xfer { + rockchip,pins = + /* uart2_rxm2 */ + <3 RK_PA4 2 &pcfg_pull_up_2ma>, + /* uart2_txm2 */ + <3 RK_PA3 2 &pcfg_pull_up_2ma>; + }; + }; + + tsadc { + tsadc_otp_gpio: tsadc-otp-gpio { + rockchip,pins = + <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + tsadc_otp_out: tsadc-otp-out { + rockchip,pins = + <0 RK_PA6 2 &pcfg_pull_none>; + }; + }; + + xin32k { + clkin_32k: clkin-32k { + rockchip,pins = + <0 RK_PC2 1 &pcfg_input_smt>; + }; + + clkout_32k: clkout-32k { + rockchip,pins = + <0 RK_PC2 1 &pcfg_output_high>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dts new file mode 100755 index 000000000000..1bd285bfc695 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dts @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3566-box-demo-v10.dtsi" +#include "rk3568-android.dtsi" + +/ { + model = "Rockchip RK3566 BOX DEMO V10 ANDROID Board"; + compatible = "rockchip,rk3566-box-demo-v10", "rockchip,rk3566"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dtsi new file mode 100755 index 000000000000..41ac0af11188 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-box-demo-v10.dtsi @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3566-box.dtsi" + +/ { + model = "Rockchip RK3566 BOX DEMO V10 Board"; + compatible = "rockchip,rk3568-box-demo-v10", "rockchip,rk3566"; + + gpio-leds { + compatible = "gpio-leds"; + + ir-led { + gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + work-led { + gpios = <&gpio0 RK_PC3 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "timer"; + }; + }; + + vcc2v5_sys: vcc2v5-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc2v5-sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + vin-supply = <&vcc3v3_sys>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&pmucru CLK_RTC_32K>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h &wifi_32k>; + reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + }; + + vcc3v3_sd: vcc3v3-sd-regulator { + compatible = "regulator-gpio"; + regulator-name = "vcc3v3_sd"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x1 + 3300000 0x0>; + }; + + vccio_sd: vccio-sd-regulator { + compatible = "regulator-gpio"; + regulator-name = "vccio_sd"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <1800000 0x0 + 3300000 0x1>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_host_en>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vcc5v0_otg: vcc5v0-otg-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_otg_en>; + regulator-name = "vcc5v0_otg"; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; + + wireless_wlan: wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6398s"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless_bluetooth: wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&pmucru CLK_RTC_32K>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&combphy1_usq { + assigned-clocks = <&pmucru CLK_PCIEPHY1_REF>; + assigned-clock-rates = <100000000>; + status = "okay"; +}; + +&combphy2_psq { + status = "okay"; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc4c33_out>; + data-lanes = <1 2>; + }; + + mipi_in_ucam1: endpoint@2 { + reg = <2>; + remote-endpoint = <&gc8034_out>; + data-lanes = <1 2 3 4>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy_out: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&gmac1 { + phy-mode = "rgmii"; + clock_in_out = "input"; + + snps,reset-gpio = <&gpio4 RK_PC2 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&gmac1_clkin>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m1_miim + &gmac1m1_tx_bus2 + &gmac1m1_rx_bus2 + &gmac1m1_rgmii_clk + &gmac1m1_rgmii_bus + &gmac1m1_clkinout>; + + tx_delay = <0x4f>; + rx_delay = <0x2d>; + + phy-handle = <&rgmii_phy1>; + status = "okay"; +}; + +&i2c2{ + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2m1_xfer>; + + gc8034: gc8034@37 { + status = "okay"; + compatible = "galaxycore,gc8034"; + reg = <0x37>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + /*pwren-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>;*/ + reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "RK-CMK-8M-2-v1"; + rockchip,camera-module-lens-name = "CK8401"; + port { + gc8034_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; + + gc4c33: gc4c33@29 { + status = "okay"; + compatible = "galaxycore,gc4c33"; + reg = <0x29>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + /*pwren-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>;*/ + reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "PCORW0009A"; + rockchip,camera-module-lens-name = "40IRC-4M"; + port { + gc4c33_out: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&mdio1 { + rgmii_phy1: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&pwm15 { + compatible = "rockchip,remotectl-pwm"; + pinctrl-names = "default"; + pinctrl-0 = <&pwm15m1_pins>; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <0>; + status = "okay"; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_PLAYPAUSE>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xa4 KEY_SETUP>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +/* Need to be modified according to the actual hardware */ +&pmu_io_domains { + status = "okay"; + pmuio2-supply = <&vcc_3v3>; + vccio1-supply = <&vcc_3v3>; + vccio3-supply = <&vcc_3v3>; + vccio4-supply = <&vcc_3v3>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc_3v3>; + vccio7-supply = <&vcc_3v3>; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy_out>; + }; + }; +}; + +&sata2 { + status = "okay"; +}; + +&sdmmc0 { + max-frequency = <50000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + //sd-uhs-sdr104; + vmmc-supply = <&vcc3v3_sd>; + vqmmc-supply = <&vccio_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; + status = "okay"; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + non-removable; + mmc-pwrseq = <&sdio_pwrseq>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&u2phy0_host { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&u2phy0_otg { + vbus-supply = <&vcc5v0_otg>; + status = "okay"; +}; + +&u2phy1_host { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&u2phy1_otg { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&usb2phy1 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&pinctrl { + cam { + camera_pwr: camera-pwr { + rockchip,pins = + /* camera power en */ + <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + wifi_32k: wifi-32k { + rockchip,pins = <2 RK_PC6 1 &pcfg_pull_none>; + }; + }; + + usb { + vcc5v0_host_en: vcc5v0-host-en { + rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + vcc5v0_otg_en: vcc5v0-otg-en { + rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-box.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-box.dtsi new file mode 100755 index 000000000000..f0feae6730e3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-box.dtsi @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3566.dtsi" +#include +#include +#include +#include +#include + +/ { + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <297500>; + }; + }; + + dc_12v: dc-12v { + compatible = "regulator-fixed"; + regulator-name = "dc_12v"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + }; + + hdmi_sound: hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "hdmi-sound"; + status = "okay"; + + simple-audio-card,cpu { + sound-dai = <&i2s0_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + rknpu_reserved: rknpu { + compatible = "shared-dma-pool"; + inactive; + reusable; + size = <0x0 0x20000000>; + alignment = <0x0 0x1000>; + status = "disabled"; + }; + }; + + spdif-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "okay"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&dc_12v>; + }; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&dc_12v>; + }; + + vcc_1v8: vcc_1v8 { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc5v0_sys>; + }; + + vcc_3v3: vcc_3v3{ + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc5v0_sys>; + }; + + vdd_fixed: vdd-fixed { + compatible = "regulator-fixed"; + regulator-name = "vdd_fixed"; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <950000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + }; + + vdd_cpu: vdd-cpu { + compatible = "pwm-regulator"; + pwms = <&pwm0 0 5000 1>; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1200000>; + regulator-init-microvolt = <950000>; + regulator-always-on; + regulator-boot-on; + regulator-settling-time-up-us = <250>; + pwm-supply = <&vcc5v0_sys>; + status = "okay"; + }; + + vdd_logic: vdd-logic { + compatible = "pwm-regulator"; + pwms = <&pwm1 0 5000 1>; + regulator-name = "vdd_logic"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1100000>; + regulator-init-microvolt = <950000>; + regulator-always-on; + regulator-boot-on; + regulator-settling-time-up-us = <250>; + pwm-supply = <&vcc5v0_sys>; + status = "okay"; + }; +}; + +&bus_npu { + bus-supply = <&vdd_logic>; + pvtm-supply = <&vdd_cpu>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + auto-freq-en = <0>; + center-supply = <&vdd_fixed>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_fixed>; + status = "okay"; +}; + +&gpu_opp_table { + /delete-node/ opp-800000000; +}; + +&hdmi { + status = "okay"; + rockchip,phy-table = + <92812500 0x8009 0x0000 0x0270>, + <165000000 0x800b 0x0000 0x026d>, + <185625000 0x800b 0x0000 0x01ed>, + <297000000 0x800b 0x0000 0x01ad>, + <594000000 0x8029 0x0000 0x0088>, + <000000000 0x0000 0x0000 0x0000>; +}; + +&hdmi_in_vp0 { + status = "okay"; +}; + +&hdmi_in_vp1 { + status = "disabled"; +}; + +&i2s0_8ch { + status = "okay"; +}; + +&i2s1_8ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&pwm0 { + status = "okay"; + pinctrl-names = "active"; +}; + +&pwm1 { + status = "okay"; + pinctrl-names = "active"; +}; + +&rk_rga { + status = "okay"; +}; + +&rknpu { + memory-region = <&rknpu_reserved>; + rknpu-supply = <&vdd_fixed>; + status = "okay"; +}; + +&rknpu_mmu { + status = "disabled"; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvenc { + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_CENTER_OFF + | RKPM_SLP_HW_PLLS_OFF + | RKPM_SLP_PMUALIVE_32K + | RKPM_SLP_PMIC_LP + | RKPM_SLP_32K_PVTM + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_PWM0_WKUP_EN + | RKPM_CPU0_WKUP_EN + ) + >; +}; + +&route_hdmi { + status = "okay"; + connect = <&vp0_out_hdmi>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + status = "okay"; +}; + +&sfc { + status = "okay"; +}; + +&spdif_8ch { + status = "okay"; +}; + +&tsadc { + status = "okay"; +}; + +&u2phy0_host { + status = "okay"; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&u2phy1_host { + status = "disabled"; +}; + +&u2phy1_otg { + status = "disabled"; +}; + +&usb2phy1 { + status = "disabled"; +}; + +&usb_host0_ehci { + status = "disabled"; +}; + +&usb_host0_ohci { + status = "disabled"; +}; + +&usb_host1_ehci { + status = "disabled"; +}; + +&usb_host1_ohci { + status = "disabled"; +}; + +&usbdrd_dwc3 { + dr_mode = "otg"; + phys = <&u2phy0_otg>; + maximum-speed = "high-speed"; + extcon = <&usb2phy0>; + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&usbhost_dwc3 { + status = "okay"; +}; + +&usbhost30 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP1>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; + +&vop_mmu { + status = "okay"; +}; + + diff --git a/arch/arm64/boot/dts/rockchip/rk3566-eink.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-eink.dtsi new file mode 100755 index 000000000000..09791eb16abf --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-eink.dtsi @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +/ { + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + waveform_reserved: waveform@10800000 { + reg = <0x0 0x10800000 0x0 0x100000>; + }; + + display_reserved: framebuffer@10900000 { + reg = <0x0 0x10900000 0x0 0x2000000>; + }; + }; + + ebc_dev: ebc-dev { + compatible = "rockchip,ebc-dev"; + ebc_tcon = <&ebc>; + eink_tcon = <&eink>; + memory-region = <&display_reserved>; + waveform-region = <&waveform_reserved>; + status = "okay"; + }; +}; + +&cpu0_opp_table { + opp-216000000 { + opp-hz = /bits/ 64 <216000000>; + opp-microvolt = <825000 825000 1150000>; + clock-latency-ns = <40000>; + }; + opp-312000000 { + opp-hz = /bits/ 64 <312000000>; + opp-microvolt = <825000 825000 1150000>; + clock-latency-ns = <40000>; + }; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + auto-freq-en = <0>; + status = "okay"; +}; + +&dmc_opp_table { + opp-324000000 { + opp-hz = /bits/ 64 <324000000>; + opp-microvolt = <875000>; + }; + opp-528000000 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <875000>; + }; +}; + +&ebc { + status = "okay"; +}; + +&eink { + status = "okay"; +}; + +&gpu_opp_table { + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + opp-microvolt = <825000>; + }; + opp-150000000 { + opp-hz = /bits/ 64 <150000000>; + opp-microvolt = <825000>; + }; +}; + +&rockchip_suspend { + status = "okay"; + + rockchip,sleep-debug-en = <0>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF_LOGOFF + | RKPM_SLP_CENTER_OFF + | RKPM_SLP_HW_PLLS_OFF + | RKPM_SLP_PMUALIVE_32K + | RKPM_SLP_OSC_DIS + | RKPM_SLP_PMIC_LP + | RKPM_SLP_32K_PVTM + ) + >; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dts new file mode 100755 index 000000000000..3f215884a070 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +#include "rk3566-evb-mipitest-v10.dtsi" +#include "rk3568-android.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dtsi new file mode 100755 index 000000000000..227bc4e39d99 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb-mipitest-v10.dtsi @@ -0,0 +1,507 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; + +#include +#include +#include "rk3566.dtsi" +#include "rk3566-evb.dtsi" + +/ { + model = "Rockchip RK3566 EVB MIPITEST V10 Board"; + compatible = "rockchip,rk3566-evb-mipitest-v10", "rockchip,rk3566"; + + pcie20_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie20_3v3"; + regulator-min-microvolt = <0100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <0100000 0x0 + 3300000 0x1>; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + vcc3v3_vga: vcc3v3-vga { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_vga"; + regulator-always-on; + regulator-boot-on; + gpio = <&gpio2 RK_PB1 GPIO_ACTIVE_HIGH>; + enable-active-high; + vin-supply = <&vcc3v3_sys>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; +}; + +&audiopwmout_diff { + status = "disabled"; +}; + +&combphy1_usq { + status = "okay"; +}; + +&combphy2_psq { + status = "okay"; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy1 { + status = "okay"; + + /* + * dphy1 only used for split mode, + * can be used concurrently with dphy2 + * full mode and split mode are mutually exclusive + */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dphy1_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy1_out: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&csi2_dphy2 { + status = "okay"; + + /* + * dphy2 only used for split mode, + * can be used concurrently with dphy1 + * full mode and split mode are mutually exclusive + */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dphy2_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov02k10_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy2_out: endpoint@1 { + reg = <1>; + remote-endpoint = <&mipi_csi2_input>; + }; + }; + }; +}; + +&dig_acodec { + status = "disabled"; + rockchip,pwm-output-mode; + pinctrl-names = "default"; + pinctrl-0 = <&audiopwm_loutp + &audiopwm_loutn + &audiopwm_routp + &audiopwm_routn + >; +}; + +/* + * video_phy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; + reset-gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd0_rst_gpio>; +}; + +/* + * video_phy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; + reset-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd1_rst_gpio>; +}; + +&edp { + hpd-gpios = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&edp_phy { + status = "okay"; +}; + +&edp_in_vp0 { + status = "okay"; +}; + +&edp_in_vp1 { + status = "disabled"; +}; + +/* + * power-supply should switche to vcc3v3_lcd1_n + * when mipi panel is connected to dsi1. + */ +>1x { + status = "disabled"; + power-supply = <&vcc3v3_lcd0_n>; +}; + +&hdmi { + status = "disabled"; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2m1_xfer>; + + /* split mode: lane0/1 */ + ov5695: ov5695@36 { + status = "okay"; + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clkout0>; + reset-gpios = <&gpio4 RK_PC0 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio2 RK_PC5 GPIO_ACTIVE_HIGH>; + /*power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;*/ + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&dphy1_in>; + data-lanes = <1 2>; + }; + }; + }; + + ov02k10: ov02k10@36 { + status = "okay"; + compatible = "ovti,ov02k10"; + reg = <0x36>; + clocks = <&cru CLK_CAM1_OUT>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clkout1>; + reset-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_HIGH>; + power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov02k10_out: endpoint { + remote-endpoint = <&dphy2_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s3_2ch { + status = "disabled"; +}; + +&mipi_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&dphy2_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +&pcie2x1 { + reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie20_3v3>; + status = "disabled"; +}; + +&pdm { + status = "disabled"; + pinctrl-names = "default"; + pinctrl-0 = <&pdmm1_clk1 + &pdmm1_sdi1 + &pdmm1_sdi2 + &pdmm1_sdi3>; +}; + +&pdmics { + status = "disabled"; +}; + +&pdm_mic_array { + status = "disabled"; +}; + +&rkcif { + status = "okay"; +}; + +&rkcif_mipi_lvds { + status = "okay"; + + port { + cif_mipi_in: endpoint { + remote-endpoint = <&mipi_csi2_output>; + data-lanes = <1 2>; + }; + }; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy1_out>; + }; + }; +}; + +&route_dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&sdmmc2 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m1_xfer &uart1m1_ctsn>; +}; + +&u2phy1_host { + status = "disabled"; +}; + +&u2phy1_otg { + status = "disabled"; +}; + +&usb2phy1 { + status = "disabled"; +}; + +&usb_host1_ohci { + status = "disabled"; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&wireless_bluetooth { + uart_rts_gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m1_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio4 RK_PA5 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; + status = "disabled"; +}; + +&wireless_wlan { + status = "disabled"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; +}; + +&pinctrl { + cam { + camera_pwr: camera-pwr { + rockchip,pins = + /* camera power en */ + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + lcd0 { + lcd0_rst_gpio: lcd0-rst-gpio { + rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + lcd1 { + lcd1_rst_gpio: lcd1-rst-gpio { + rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb.dtsi new file mode 100755 index 000000000000..d1aa123cd347 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb.dtsi @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-linux.dts new file mode 100755 index 000000000000..a22cc8cec6af --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-linux.dts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3566-evb1-ddr4-v10.dtsi" +#include "rk3568-linux.dtsi" + +/ { + model = "Rockchip RK3566 EVB1 DDR4 V10 Linux Board"; + compatible = "rockchip,rk3566-evb1-ddr4-v10-linux", "rockchip,rk3566"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-lvds.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-lvds.dts new file mode 100755 index 000000000000..98a384041b8b --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10-lvds.dts @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +#include +#include "rk3566-evb1-ddr4-v10.dtsi" +#include "rk3568-android.dtsi" + +/ { + panel { + compatible = "simple-panel"; + backlight = <&backlight>; + power-supply = <&vcc3v3_lcd1_n>; + enable-delay-ms = <20>; + prepare-delay-ms = <20>; + unprepare-delay-ms = <20>; + disable-delay-ms = <20>; + bus-format = ; + width-mm = <217>; + height-mm = <136>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <68000000>; + hactive = <800>; + vactive = <1280>; + hback-porch = <30>; + hfront-porch = <30>; + vback-porch = <4>; + vfront-porch = <2>; + hsync-len = <4>; + vsync-len = <2>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dual-lvds-even-pixels; + panel_in_lvds: endpoint { + remote-endpoint = <&lvds_out_panel>; + }; + }; + }; + }; +}; + +&dsi0 { + status = "disabled"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "disabled"; +}; + +&video_phy0 { + status = "okay"; +}; + +&lvds { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + lvds_out_panel: endpoint { + remote-endpoint = <&panel_in_lvds>; + }; + }; + }; +}; + +&lvds_in_vp1 { + status = "okay"; +}; + +&lvds_in_vp2 { + status = "disabled"; +}; + +&route_lvds { + status = "okay"; + connect = <&vp1_out_lvds>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dts new file mode 100755 index 000000000000..f602ed98d7ce --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +#include "rk3566-evb1-ddr4-v10.dtsi" +#include "rk3568-android.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dtsi new file mode 100755 index 000000000000..385933be9f45 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb1-ddr4-v10.dtsi @@ -0,0 +1,489 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; + +#include +#include +#include "rk3566.dtsi" +#include "rk3566-evb.dtsi" + +/ { + model = "Rockchip RK3566 EVB1 DDR4 V10 Board"; + compatible = "rockchip,rk3566-evb1-ddr4-v10", "rockchip,rk3566"; + + pcie20_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie20_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + vcc3v3_vga: vcc3v3-vga { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_vga"; + regulator-always-on; + regulator-boot-on; + gpio = <&gpio2 RK_PB1 GPIO_ACTIVE_HIGH>; + enable-active-high; + vin-supply = <&vcc3v3_sys>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; +}; + +&audiopwmout_diff { + status = "disabled"; +}; + +&combphy1_usq { + status = "okay"; +}; + +&combphy2_psq { + status = "okay"; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy1 { + status = "okay"; + + /* + * dphy1 only used for split mode, + * can be used concurrently with dphy2 + * full mode and split mode are mutually exclusive + */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dphy1_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy1_out: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&csi2_dphy2 { + status = "okay"; + + /* + * dphy2 only used for split mode, + * can be used concurrently with dphy1 + * full mode and split mode are mutually exclusive + */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dphy2_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov02k10_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy2_out: endpoint@1 { + reg = <1>; + remote-endpoint = <&mipi_csi2_input>; + }; + }; + }; +}; + +&dig_acodec { + status = "disabled"; + rockchip,pwm-output-mode; + pinctrl-names = "default"; + pinctrl-0 = <&audiopwm_loutp + &audiopwm_loutn + &audiopwm_routp + &audiopwm_routn + >; +}; + +/* + * video_phy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; + reset-gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd0_rst_gpio>; +}; + +/* + * video_phy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; + reset-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd1_rst_gpio>; +}; + +&edp { + hpd-gpios = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&edp_phy { + status = "okay"; +}; + +&edp_in_vp0 { + status = "okay"; +}; + +&edp_in_vp1 { + status = "disabled"; +}; + +/* + * power-supply should switche to vcc3v3_lcd1_n + * when mipi panel is connected to dsi1. + */ +>1x { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2m1_xfer>; + + /* split mode: lane0/1 */ + ov5695: ov5695@36 { + status = "okay"; + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clkout0>; + reset-gpios = <&gpio4 RK_PC0 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio2 RK_PC5 GPIO_ACTIVE_HIGH>; + /*power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;*/ + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&dphy1_in>; + data-lanes = <1 2>; + }; + }; + }; + + ov02k10: ov02k10@36 { + status = "okay"; + compatible = "ovti,ov02k10"; + reg = <0x36>; + clocks = <&cru CLK_CAM1_OUT>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clkout1>; + reset-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_HIGH>; + power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov02k10_out: endpoint { + remote-endpoint = <&dphy2_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2s3_2ch { + status = "disabled"; +}; + +&mipi_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&dphy2_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +&pcie2x1 { + reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie20_3v3>; + status = "okay"; +}; + +&pdm { + status = "disabled"; + pinctrl-names = "default"; + pinctrl-0 = <&pdmm1_clk1 + &pdmm1_sdi1 + &pdmm1_sdi2 + &pdmm1_sdi3>; +}; + +&pdmics { + status = "disabled"; +}; + +&pdm_mic_array { + status = "disabled"; +}; + +&rkcif { + status = "okay"; +}; + +&rkcif_mipi_lvds { + status = "okay"; + + port { + cif_mipi_in: endpoint { + remote-endpoint = <&mipi_csi2_output>; + data-lanes = <1 2>; + }; + }; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy1_out>; + }; + }; +}; + +&route_dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&sdmmc2 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m1_xfer &uart1m1_ctsn>; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&wireless_bluetooth { + uart_rts_gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m1_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio4 RK_PA5 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&wireless_wlan { + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; +}; + +&work_led { + gpios = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; +}; + +&pinctrl { + cam { + camera_pwr: camera-pwr { + rockchip,pins = + /* camera power en */ + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + lcd0 { + lcd0_rst_gpio: lcd0-rst-gpio { + rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + lcd1 { + lcd1_rst_gpio: lcd1-rst-gpio { + rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-eink.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-eink.dts new file mode 100755 index 000000000000..83546eef6529 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-eink.dts @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3566-evb2-lp4x-v10.dtsi" +#include "rk3568-android.dtsi" +#include "rk3566-eink.dtsi" + +/ { + model = "Rockchip RK3566 EVB2 LP4X V10 Eink Board"; + compatible = "rockchip,rk3566-evb2-lp4x-v10-eink", "rockchip,rk3566"; +}; + +&backlight { + status = "disabled"; +}; + +&backlight1 { + status = "disabled"; +}; + +&dsi0 { + status = "disabled"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_panel { + status = "disabled"; +}; + +&dsi1_panel { + status = "disabled"; +}; + +&ebc { + /* clock rate 1000M/n, (n=1~32) */ + assigned-clocks = <&cru CPLL_333M>, <&cru DCLK_EBC>; + //assigned-clock-rates = <340000000>, <340000000>; + assigned-clock-rates = <250000000>, <250000000>; + //assigned-clock-rates = <100000000>, <100000000>; + status = "okay"; +}; + +&ebc_dev { + pmic = <&tps65185>; + status = "okay"; +#if 0 + /* ED097TC2U1 */ + panel,width = <1200>; + panel,height = <825>; + panel,vir_width = <1200>; + panel,vir_height = <825>; + panel,sdck = <25000000>; + panel,lsl = <4>; + panel,lbl = <4>; + panel,ldl = <300>; + panel,lel = <36>; + panel,gdck-sta = <18>; + panel,lgonl = <265>; + panel,fsl = <2>; + panel,fbl = <4>; + panel,fdl = <825>; + panel,fel = <24>; + panel,mirror = <0>; + panel,panel_16bit = <0>; + panel,panel_color = <0>; + panel,width-mm = <203>; + panel,height-mm = <140>; +#endif +#if 1 + /* ES103TC1 */ + panel,width = <1872>; + panel,height = <1404>; + panel,vir_width = <1872>; + panel,vir_height = <1404>; + panel,sdck = <33300000>; + panel,lsl = <18>; + panel,lbl = <17>; + panel,ldl = <234>; + panel,lel = <7>; + panel,gdck-sta = <34>; + panel,lgonl = <192>; + panel,fsl = <1>; + panel,fbl = <4>; + panel,fdl = <1404>; + panel,fel = <12>; + panel,mirror = <0>; + panel,panel_16bit = <1>; + panel,panel_color = <0>; + panel,width-mm = <157>; + panel,height-mm = <210>; +#endif +#if 0 + /* ES133TC1 */ + panel,width = <2200>; + panel,height = <1650>; + panel,vir_width = <2208>; + panel,vir_height = <1650>; + panel,sdck = <37500000>; + panel,lsl = <4>; + panel,lbl = <8>; + panel,ldl = <275>; + panel,lel = <14>; + panel,gdck-sta = <34>; + panel,lgonl = <217>; + panel,fsl = <1>; + panel,fbl = <4>; + panel,fdl = <1650>; + panel,fel = <6>; + panel,mirror = <0>; + panel,panel_16bit = <1>; + panel,panel_color = <0>; + panel,width-mm = <157>; + panel,height-mm = <210>; +#endif +#if 0 + panel,width = <2232>; + panel,height = <1680>; + panel,vir_width = <2240>; + panel,vir_height = <1680>; + panel,sdck = <33300000>; + panel,lsl = <4>; + panel,lbl = <8>; + panel,ldl = <279>; + panel,lel = <14>; + panel,gdck-sta = <34>; + panel,lgonl = <217>; + panel,fsl = <1>; + panel,fbl = <4>; + panel,fdl = <1680>; + panel,fel = <6>; + panel,mirror = <0>; + panel,panel_16bit = <1>; + panel,panel_color = <0>; + panel,width-mm = <157>; + panel,height-mm = <210>; +#endif +}; + +&gmac1 { + status = "disabled"; +}; + +>1x { + status = "disabled"; +}; + +&hdmi { + status = "disabled"; +}; + +&hdmi_in_vp0 { + status = "disabled"; +}; + +&hdmi_sound{ + status = "disabled"; +}; + +&i2c1 { + status = "okay"; + + tsc@24 { + status = "okay"; + compatible = "cy,cyttsp5_i2c_adapter"; + reg = <0x24>; + cy,adapter_id = "cyttsp5_i2c_adapter"; + //cytp-supply = <&vcc_sd>; + cy,core { + cy,name = "cyttsp5_core"; + pinctrl-names = "default"; + pinctrl-0 = <&tsc_gpio>; + cy,irq_gpio = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + cy,rst_gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + cy,hid_desc_register = <1>; + /* CY_CORE_FLAG_RESTORE_PARAMETERS */ + cy,flags = <6>; + /* CY_CORE_EWG_NONE */ + cy,easy_wakeup_gesture = <0>; + cy,btn_keys = <172 /* KEY_HOMEPAGE */ + /* previously was KEY_HOME, new Android versions use KEY_HOMEPAGE */ + 139 /* KEY_MENU */ + 158 /* KEY_BACK */ + 217 /* KEY_SEARCH */ + 114 /* KEY_VOLUMEDOWN */ + 115 /* KEY_VOLUMEUP */ + 212 /* KEY_CAMERA */ + 116>; /* KEY_POWER */ + cy,btn_keys-tag = <0>; + cy,mt { + cy,name = "cyttsp5_mt"; + cy,inp_dev_name = "cyttsp5_mt"; + cy,flags = <0>; + cy,abs = + /* ABS_MT_POSITION_X, CY_ABS_MIN_X, CY_ABS_MAX_X, 0, 0 */ + <0x35 0 1872 0 0 + /* ABS_MT_POSITION_Y, CY_ABS_MIN_Y, CY_ABS_MAX_Y, 0, 0 */ + 0x36 0 1404 0 0 + /* ABS_MT_PRESSURE, CY_ABS_MIN_P, CY_ABS_MAX_P, 0, 0 */ + 0x3a 0 255 0 0 + /* CY_IGNORE_VALUE, CY_ABS_MIN_W, CY_ABS_MAX_W, 0, 0 */ + 0xffff 0 255 0 0 + /* ABS_MT_TRACKING_ID, CY_ABS_MIN_T, CY_ABS_MAX_T, 0, 0 */ + 0x39 0 15 0 0 + /* ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0 */ + 0x30 0 255 0 0 + /* ABS_MT_TOUCH_MINOR, 0, 255, 0, 0 */ + 0x31 0 255 0 0 + /* ABS_MT_ORIENTATION, -127, 127, 0, 0 */ + 0x34 0xffffff81 127 0 0 + /* ABS_MT_TOOL_TYPE, 0, MT_TOOL_MAX, 0, 0 */ + 0x37 0 1 0 0 + /* ABS_DISTANCE, 0, 255, 0, 0 */ + 0x19 0 255 0 0>; + + cy,vkeys_x = <1872>; + cy,vkeys_y = <1404>; + + cy,revert_x = <0>; + cy,revert_y = <0>; + cy,xy_exchange = <0>; + + cy,virtual_keys = /* KeyCode CenterX CenterY Width Height */ + /* KEY_BACK */ + <158 1360 90 160 180 + /* KEY_MENU */ + 139 1360 270 160 180 + /* KEY_HOMEPAGE */ + 172 1360 450 160 180 + /* KEY SEARCH */ + 217 1360 630 160 180>; + }; + + cy,btn { + cy,name = "cyttsp5_btn"; + cy,inp_dev_name = "cyttsp5_btn"; + }; + + cy,proximity { + cy,name = "cyttsp5_proximity"; + cy,inp_dev_name = "cyttsp5_proximity"; + cy,abs = + /* ABS_DISTANCE, CY_PROXIMITY_MIN_VAL, CY_PROXIMITY_MAX_VAL, 0, 0 */ + <0x19 0 1 0 0>; + }; + }; + }; + + tps65185: tps65185@68 { + status = "okay"; + compatible = "ti,tps65185"; + reg = <0x68>; + pinctrl-names = "default"; + pinctrl-0 = <&tps65185_gpio>; + int-gpios = <&gpio3 RK_PA7 GPIO_ACTIVE_HIGH>; + wakeup-gpios = <&gpio3 RK_PC3 GPIO_ACTIVE_HIGH>; + vcomctl-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; + powerup-gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c2 { + status = "disabled"; +}; + +&i2c3 { + status = "okay"; + + wacom: wacom@9 { + compatible = "wacom,w9013"; + reg = <0x09>; + pinctrl-names = "default"; + pinctrl-0 = <&wacom_gpio>; + gpio_detect = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; + gpio_intr = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + gpio_rst = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; + revert_x = <0>; + revert_y = <0>; + xy_exchange = <0>; + }; +}; + +&video_phy0 { + status = "disabled"; +}; + +&mxc6655xa { + status = "disabled"; +}; + +&pinctrl { + tps_pmic { + tps65185_gpio: tps65185-gpio { + rockchip,pins = + <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, + <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + tsc { + tsc_gpio: tsc-gpio { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + wacom { + wacom_gpio: wacom-gpio { + rockchip,pins = + <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>, + <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcc3v3_pmu>; + vccio1-supply = <&vccio_acodec>; + vccio3-supply = <&vccio_sd>; + vccio4-supply = <&vcc_1v8>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc_3v3>; + vccio7-supply = <&vcc_3v3>; +}; + +&vcc_camera { + status = "disabled"; +}; + +&wireless_bluetooth { + status = "disabled"; +}; + +&wireless_wlan { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-i2s-mic-array.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-i2s-mic-array.dts new file mode 100755 index 000000000000..5ba1318bcee1 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-i2s-mic-array.dts @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3566-evb2-lp4x-v10.dtsi" +#include "rk3568-android.dtsi" + +/ { + model = "Rockchip RK3566 EVB2 LP4X V10 Board I2S Mic Array"; + compatible = "rockchip,rk3566-evb2-lp4x-v10", "rockchip,rk3566"; + + rk809_sound_micarray: rk809-sound-micarray { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_8ch>; + }; + codec { + sound-dai = <&rk809_codec 0>; + }; + }; + simple-audio-card,dai-link@1 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_8ch>; + }; + codec { + sound-dai = <&es7243e>; + }; + }; + }; +}; + +&i2c3 { + status = "okay"; + + es7243e: es7243e@10 { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "ES7243E_MicArray_0"; + reg = <0x10>; + }; + + es7243e_11: es7243e@11 { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "ES7243E_MicArray_1"; + reg = <0x11>; + }; + + es7243e_12: es7243e@12 { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "ES7243E_MicArray_2"; + reg = <0x12>; + }; +}; + +&i2s1_8ch { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,clk-trcm = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sclkrx + &i2s1m0_lrckrx + &i2s1m0_sdo0 + &i2s1m0_sdi0 + &i2s1m0_sdi1 + &i2s1m0_sdi2 + &i2s1m0_sdi3>; +}; + +&rk809_codec { + #sound-dai-cells = <1>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + pdmdata-out-enable; + adc-for-loopback; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + status = "okay"; +}; + +&rk809_sound { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-linux.dts new file mode 100755 index 000000000000..957a99b168bb --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-linux.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3566-evb2-lp4x-v10.dtsi" +#include "rk3568-linux.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-pdm-mic-array.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-pdm-mic-array.dts new file mode 100755 index 000000000000..55e9679194cb --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10-pdm-mic-array.dts @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3566-evb2-lp4x-v10.dtsi" +#include "rk3568-android.dtsi" + +/ { + model = "Rockchip RK3566 EVB2 LP4X V10 Board PDM Mic Array"; + compatible = "rockchip,rk3566-evb2-lp4x-v10", "rockchip,rk3566"; + + rk809_sound_micarray: rk809-sound-micarray { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_8ch>; + }; + codec { + sound-dai = <&rk809_codec 0>; + }; + }; + simple-audio-card,dai-link@1 { + format = "pdm"; + cpu { + sound-dai = <&pdm>; + }; + codec { + sound-dai = <&rk809_codec 1>; + }; + }; + simple-audio-card,dai-link@2 { + format = "pdm"; + cpu { + sound-dai = <&pdm>; + }; + codec { + sound-dai = <&es7202>; + }; + }; + }; +}; + +&i2c3 { + status = "okay"; + + es7202: es7202@30 { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "ES7202_PDM_ADC_1"; + reg = <0x30>; + }; + + es7202_31: es7202@31 { + status = "okay"; + #sound-dai-cells = <0>; + compatible = "ES7202_PDM_ADC_2"; + reg = <0x31>; + }; +}; + +&i2s1_8ch { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdo0>; +}; + +&pdm { + status = "okay"; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pdmm0_clk + &pdmm0_clk1 + &pdmm0_sdi0 + &pdmm0_sdi1 + &pdmm0_sdi2 + &pdmm0_sdi3>; +}; + +&rk809_codec { + #sound-dai-cells = <1>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + pdmdata-out-enable; + adc-for-loopback; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + status = "okay"; +}; + +&rk809_sound { + status = "disabled"; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dts new file mode 100755 index 000000000000..3b36bdba12b6 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3566-evb2-lp4x-v10.dtsi" +#include "rk3568-android.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dtsi new file mode 100755 index 000000000000..4a6bec70f33e --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb2-lp4x-v10.dtsi @@ -0,0 +1,599 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include +#include "rk3566.dtsi" +#include "rk3566-evb.dtsi" + +/ { + model = "Rockchip RK3566 EVB2 LP4X V10 Board"; + compatible = "rockchip,rk3566-evb2-lp4x-v10", "rockchip,rk3566"; + + pcie20_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie20_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; +}; + +&combphy1_usq { + status = "okay"; +}; + +&combphy2_psq { + status = "okay"; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "disabled"; + /* + * dphy0 only used for full mode, + * full mode and split mode are mutually exclusive + */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dphy0_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc8034_out>; + data-lanes = <1 2 3 4>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy0_out: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&csi2_dphy1 { + status = "okay"; + + /* + * dphy1 only used for split mode, + * can be used concurrently with dphy2 + * full mode and split mode are mutually exclusive + */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dphy1_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy1_out: endpoint@1 { + reg = <1>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&csi2_dphy2 { + status = "okay"; + + /* + * dphy2 only used for split mode, + * can be used concurrently with dphy1 + * full mode and split mode are mutually exclusive + */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dphy2_in: endpoint@1 { + reg = <1>; + remote-endpoint = <&gc5025_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + dphy2_out: endpoint@1 { + reg = <1>; + remote-endpoint = <&mipi_csi2_input>; + }; + }; + }; +}; + +/* + * video_phy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; + reset-gpios = <&gpio3 RK_PA3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd0_rst_gpio>; +}; + +/* + * video_phy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; + reset-gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd1_rst_gpio>; +}; + +&gmac1 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m1_miim + &gmac1m1_tx_bus2 + &gmac1m1_rx_bus2 + &gmac1m1_rgmii_clk + &gmac1m1_rgmii_bus>; + + tx_delay = <0x4f>; + rx_delay = <0x25>; + + phy-handle = <&rgmii_phy0>; + status = "okay"; +}; + +&i2c2 { + status = "okay"; + pinctrl-0 = <&i2c2m1_xfer>; + + /* split mode: lane0/1 */ + ov5695: ov5695@36 { + status = "okay"; + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; + /*power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;*/ + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&dphy1_in>; + data-lanes = <1 2>; + }; + }; + }; + + /* split mode: lane:2/3 */ + gc5025: gc5025@37 { + status = "okay"; + compatible = "galaxycore,gc5025"; + reg = <0x37>; + clocks = <&pmucru CLK_WIFI>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&refclk_pins>; + reset-gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_LOW>; + power-domains = <&power RK3568_PD_VI>; + /*power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;*/ + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + gc5025_out: endpoint { + remote-endpoint = <&dphy2_in>; + data-lanes = <1 2>; + }; + }; + }; + + /* full mode: lane0-3 */ + gc8034: gc8034@37 { + compatible = "galaxycore,gc8034"; + status = "okay"; + reg = <0x37>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "RK-CMK-8M-2-v1"; + rockchip,camera-module-lens-name = "CK8401"; + port { + gc8034_out: endpoint { + remote-endpoint = <&dphy0_in>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&i2c4 { + /* i2c4 sda conflict with camera pwdn */ + status = "disabled"; + + /* + * gc2145 needs to be disabled, + * when gmac1 is enabled; + * pinctrl conflicts; + */ + gc2145: gc2145@3c { + status = "disabled"; + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + /* conflict with gmac1m1_rgmii_pins & cif_clk*/ + pinctrl-0 = <&cif_clk &cif_dvp_clk &cif_dvp_bus16>; + + /*avdd-supply = <&vcc2v8_dvp>;*/ + /*dovdd-supply = <&vcc1v8_dvp>;*/ + /*dvdd-supply = <&vcc1v8_dvp>;*/ + + /*reset-gpios = <&gpio3 RK_PB5 GPIO_ACTIVE_LOW>;*/ + pwdn-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_out: endpoint { + remote-endpoint = <&dvp_in_bcam>; + }; + }; + }; +}; + +&mdio1 { + rgmii_phy0: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + + + +/* + * power-supply should switche to vcc3v3_lcd1_n + * when mipi panel is connected to dsi1. + */ +>1x { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&mipi_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&dphy2_out>; + data-lanes = <1 2>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi_in>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +&pcie2x1 { + reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie20_3v3>; + status = "okay"; +}; + +&pinctrl { + cam { + camera_pwr: camera-pwr { + rockchip,pins = + /* camera power en */ + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + lcd0 { + lcd0_rst_gpio: lcd0-rst-gpio { + rockchip,pins = <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + lcd1 { + lcd1_rst_gpio: lcd1-rst-gpio { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&rkcif { + status = "okay"; +}; + +&rkcif_dvp { + status = "disabled"; + + port { + /* Parallel bus endpoint */ + dvp_in_bcam: endpoint { + remote-endpoint = <&gc2145_out>; + bus-width = <8>; + vsync-active = <0>; + hsync-active = <1>; + }; + }; +}; + +&rkcif_mipi_lvds { + status = "okay"; + + port { + cif_mipi_in: endpoint { + remote-endpoint = <&mipi_csi2_output>; + data-lanes = <1 2>; + }; + }; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&dphy1_out>; + }; + }; +}; + +&route_dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&sdmmc2 { + status = "disabled"; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdio_pwrseq { + reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; +}; + +&spdif_8ch { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&spdifm1_tx>; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; +&wireless_wlan { + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; +}; + +&work_led { + gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; +}; + +&wireless_bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10-linux.dts new file mode 100755 index 000000000000..e292b0dd1846 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10-linux.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3566-evb3-ddr3-v10.dtsi" +#include "rk3568-linux.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dts new file mode 100755 index 000000000000..09f5260fb375 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3566-evb3-ddr3-v10.dtsi" +#include "rk3568-android.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dtsi new file mode 100755 index 000000000000..f936cc186cfb --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb3-ddr3-v10.dtsi @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include "rk3566.dtsi" +#include "rk3566-evb.dtsi" + +/ { + model = "Rockchip RK3566 EVB3 DDR3 V10 Board"; + compatible = "rockchip,rk3566-evb3-DDR3-v10", "rockchip,rk3566"; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio4 RK_PB3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + vcc3v3_vga: vcc3v3-vga { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_vga"; + regulator-always-on; + regulator-boot-on; + gpio = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; + enable-active-high; + vin-supply = <&vcc3v3_sys>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; +}; + +&combphy1_usq { + status = "okay"; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + mipi_in_ucam1: endpoint@2 { + reg = <2>; + remote-endpoint = <&gc8034_out>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +/* + * video_phy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; + reset-gpios = <&gpio1 RK_PA5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd0_rst_gpio>; +}; + +/* + * video_phy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; + reset-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd1_rst_gpio>; +}; + +&edp { + hpd-gpios = <&gpio1 RK_PA7 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&edp_phy { + status = "okay"; +}; + +&edp_in_vp0 { + status = "okay"; +}; + +&edp_in_vp1 { + status = "disabled"; +}; + +&gmac1 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m0_miim + &gmac1m0_tx_bus2_level3 + &gmac1m0_rx_bus2 + &gmac1m0_rgmii_clk_level2 + &gmac1m0_rgmii_bus_level3>; + + tx_delay = <0x41>; + rx_delay = <0x2e>; + + phy-handle = <&rgmii_phy1>; + status = "okay"; +}; + +>1x { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&i2c2 { + status = "okay"; + pinctrl-0 = <&i2c2m1_xfer>; + + gc2145: gc2145@3c { + status = "okay"; + compatible = "galaxycore,gc2145"; + reg = <0x3c>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk &cif_dvp_clk &cif_dvp_bus16>; + + /*avdd-supply = <&vcc2v8_dvp>;*/ + /*dovdd-supply = <&vcc1v8_dvp>;*/ + /*dvdd-supply = <&vcc1v8_dvp>;*/ + + power-gpios = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; + /*reset-gpios = <&gpio3 RK_PB5 GPIO_ACTIVE_LOW>;*/ + pwdn-gpios = <&gpio3 RK_PD2 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "CameraKing"; + rockchip,camera-module-lens-name = "Largan"; + port { + gc2145_out: endpoint { + remote-endpoint = <&dvp_in_bcam>; + }; + }; + }; + + ov5695: ov5695@36 { + status = "okay"; + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clkout0>; + reset-gpios = <&gpio3 RK_PD0 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio3 RK_PC6 GPIO_ACTIVE_HIGH>; + /*power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;*/ + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; + + gc8034: gc8034@37 { + compatible = "galaxycore,gc8034"; + status = "okay"; + reg = <0x37>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clkout0>; + reset-gpios = <&gpio3 RK_PD0 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio3 RK_PC6 GPIO_ACTIVE_LOW>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "RK-CMK-8M-2-v1"; + rockchip,camera-module-lens-name = "CK8401"; + port { + gc8034_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; + +}; + +&i2s1_8ch { + status = "disabled"; +}; + +&i2s3_2ch { + status = "okay"; + pinctrl-names = "default"; + rockchip,clk-trcm = <1>; + pinctrl-0 = <&i2s3m1_sclk + &i2s3m1_lrck + &i2s3m1_sdi + &i2s3m1_sdo>; +}; + +&mdio1 { + rgmii_phy1: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +&pdm { + status = "disabled"; + pinctrl-names = "default"; + pinctrl-0 = <&pdmm1_clk1 + &pdmm1_sdi1 + &pdmm1_sdi2 + &pdmm1_sdi3>; +}; + +&pdmics { + status = "disabled"; +}; + +&pdm_mic_array { + status = "disabled"; +}; + +&pinctrl { + cam { + camera_pwr: camera-pwr { + rockchip,pins = + /* camera power en */ + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <4 RK_PB3 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + lcd0 { + lcd0_rst_gpio: lcd-rst-gpio { + rockchip,pins = <1 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + lcd1 { + lcd1_rst_gpio: lcd1-rst-gpio { + rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&rkcif { + status = "okay"; +}; + +&rkcif_dvp { + status = "okay"; + + port { + /* Parallel bus endpoint */ + dvp_in_bcam: endpoint { + remote-endpoint = <&gc2145_out>; + bus-width = <8>; + vsync-active = <0>; + hsync-active = <1>; + }; + }; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy_out>; + }; + }; +}; + +&rk809_codec { + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru I2S3_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S3_MCLKOUT>, <&cru I2S3_MCLK_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S3_MCLKOUT_TX>, <&cru I2S3_MCLKOUT>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s3m1_mclk>; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + status = "okay"; +}; + +&rk809_sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,cpu { + sound-dai = <&i2s3_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; +}; + +&route_dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&sata1 { + status = "okay"; +}; + +&sdio_pwrseq { + reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc2 { + status = "disabled"; +}; + +&spdif_8ch { + status = "disabled"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&wireless_bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&wireless_wlan { + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; +}; + +&work_led { + gpios = <&gpio0 RK_PD3 GPIO_ACTIVE_HIGH>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dts new file mode 100755 index 000000000000..600fc3c39586 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +#include "rk3566-evb5-lp4x-v10.dtsi" +#include "rk3568-android.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dtsi new file mode 100755 index 000000000000..83586659530d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-evb5-lp4x-v10.dtsi @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; + +#include +#include +#include "rk3566.dtsi" +#include "rk3566-evb.dtsi" + +/ { + model = "Rockchip RK3566 EVB5 LP4X V10 Board"; + compatible = "rockchip,rk3566-evb5-lp4x-v10", "rockchip,rk3566"; + + pcie20_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie20_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + vcc3v3_vga: vcc3v3-vga { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_vga"; + regulator-always-on; + regulator-boot-on; + gpio = <&gpio2 RK_PB1 GPIO_ACTIVE_HIGH>; + enable-active-high; + vin-supply = <&vcc3v3_sys>; + }; +}; + +&audiopwmout_diff { + status = "disabled"; +}; + +&combphy1_usq { + status = "okay"; +}; + +&combphy2_psq { + status = "disabled"; +}; + +&dig_acodec { + status = "disabled"; + rockchip,pwm-output-mode; + pinctrl-names = "default"; + pinctrl-0 = <&audiopwm_loutp + &audiopwm_loutn + &audiopwm_routp + &audiopwm_routn + >; +}; + +/* + * video_phy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; + reset-gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd0_rst_gpio>; +}; + +/* + * video_phy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; + reset-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd1_rst_gpio>; +}; + +&edp { + hpd-gpios = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; + status = "disabled"; +}; + +&edp_phy { + status = "disabled"; +}; + +&edp_in_vp0 { + status = "disabled"; +}; + +&edp_in_vp1 { + status = "disabled"; +}; + +&gmac1 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio4 RK_PC2 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m0_miim + &gmac1m0_tx_bus2_level3 + &gmac1m0_rx_bus2 + &gmac1m0_rgmii_clk_level2 + &gmac1m0_rgmii_bus_level3>; + + tx_delay = <0x41>; + rx_delay = <0x2e>; + + phy-handle = <&rgmii_phy1>; + status = "disabled"; +}; + +/* + * power-supply should switche to vcc3v3_lcd1_n + * when mipi panel is connected to dsi1. + */ +>1x { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&i2c5 { + status = "disabled"; +}; + +&i2s3_2ch { + status = "disabled"; +}; + +&mdio1 { + rgmii_phy1: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +&pcie2x1 { + reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie20_3v3>; + status = "disabled"; +}; + +&pdm { + status = "disabled"; + pinctrl-names = "default"; + pinctrl-0 = <&pdmm1_clk1 + &pdmm1_sdi1 + &pdmm1_sdi2 + &pdmm1_sdi3>; +}; + +&pdmics { + status = "disabled"; +}; + +&pdm_mic_array { + status = "disabled"; +}; + +&route_dsi0 { + status = "okay"; +}; + +&sdmmc2 { + status = "disabled"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m1_xfer &uart1m1_ctsn>; +}; + +&uart3 { + status = "disabled"; +}; + +&uart4 { + status = "disabled"; +}; + +&uart7 { + status = "disabled"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "disabled"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "disabled"; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&wireless_bluetooth { + uart_rts_gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m1_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio4 RK_PA5 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; + status = "disabled"; +}; + +&wireless_wlan { + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + status = "disabled"; +}; + +&work_led { + status = "disabled"; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + lcd0 { + lcd0_rst_gpio: lcd0-rst-gpio { + rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + lcd1 { + lcd1_rst_gpio: lcd1-rst-gpio { + rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w103.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w103.dts new file mode 100755 index 000000000000..3f559fca27f8 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w103.dts @@ -0,0 +1,1115 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include +#include +#include +#include "rk3566.dtsi" +#include "rk3568-android.dtsi" +#include "rk3566-eink.dtsi" + +/ { + model = "Rockchip RK3566 RK817 EINK LP4X Board"; + compatible = "rockchip,rk3566-rk817-eink", "rockchip,rk3566"; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3350>; + rockchip,screen-on-voltage = <3400>; + rockchip,auto-wakeup-interval = <60>; + status = "okay"; + }; + + adc_keys: adc-keys { + status = "okay"; + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <9000>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <235000>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + }; + + gpio_keys: gpio-keys { + status = "disabled"; + compatible = "gpio-keys"; + autorepeat; + + BACK { + label = "GPIO Key Home"; + debounce-interval = <10>; + interrupt-parent = <&gpio0>; + interrupts = <13 IRQ_TYPE_LEVEL_LOW>; + linux,input-type = ; + linux,code = ; + }; + }; + + hdmi_sound: hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip,hdmi"; + status = "disabled"; + + simple-audio-card,cpu { + sound-dai = <&i2s0_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + leds: gpio-leds { + compatible = "gpio-leds"; + + led@2 { + gpios = <&gpio3 RK_PC5 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "battery-charging"; + label = "battery_charging"; + retain-state-suspended; + }; + }; + + hall_sensor: hall-mh248 { + compatible = "hall-mh248"; + irq-gpio = <&gpio0 RK_PC7 IRQ_TYPE_EDGE_BOTH>; + hall-active = <1>; + status = "okay"; + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_tp: vcc-tp-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PD6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc_tp_en>; + regulator-name = "vcc_tp"; + regulator-boot-on; + startup-delay-us = <10000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_LOW>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6255"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_vbat &wifi_host_wake_irq>; + WIFI,vbat_gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; + WIFI,host_wake_irq = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>, + <&bt_reset_gpio>, + <&bt_wake_gpio>, + <&bt_irq_gpio>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PC3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&ebc { + /* clock rate 1000M/n, (n=1~32) */ + assigned-clocks = <&cru CPLL_333M>, <&cru DCLK_EBC>; + //assigned-clock-rates = <340000000>, <340000000>; + assigned-clock-rates = <250000000>, <250000000>; + status = "okay"; +}; + +&ebc_dev { + pmic = <&tps65185>; + status = "okay"; +#if 0 + /* ED097TC2U1 */ + panel,width = <1200>; + panel,height = <825>; + panel,vir_width = <1200>; + panel,vir_height = <825>; + panel,sdck = <25000000>; + panel,lsl = <4>; + panel,lbl = <4>; + panel,ldl = <300>; + panel,lel = <36>; + panel,gdck-sta = <18>; + panel,lgonl = <265>; + panel,fsl = <2>; + panel,fbl = <4>; + panel,fdl = <825>; + panel,fel = <24>; + panel,mirror = <0>; + panel,panel_16bit = <0>; + panel,panel_color = <0>; + panel,width-mm = <203>; + panel,height-mm = <140>; +#endif +#if 1 + /* ES103TC1 */ + panel,width = <1872>; + panel,height = <1404>; + panel,vir_width = <1872>; + panel,vir_height = <1404>; + panel,sdck = <33300000>; + panel,lsl = <18>; + panel,lbl = <17>; + panel,ldl = <234>; + panel,lel = <7>; + panel,gdck-sta = <34>; + panel,lgonl = <192>; + panel,fsl = <1>; + panel,fbl = <4>; + panel,fdl = <1404>; + panel,fel = <12>; + panel,mirror = <0>; + panel,panel_16bit = <1>; + panel,panel_color = <0>; + panel,width-mm = <157>; + panel,height-mm = <210>; +#endif +#if 0 + /* ES133TC1 */ + panel,width = <2200>; + panel,height = <1650>; + panel,vir_width = <2208>; + panel,vir_height = <1650>; + panel,sdck = <37500000>; + panel,lsl = <4>; + panel,lbl = <8>; + panel,ldl = <275>; + panel,lel = <14>; + panel,gdck-sta = <34>; + panel,lgonl = <217>; + panel,fsl = <1>; + panel,fbl = <4>; + panel,fdl = <1650>; + panel,fel = <6>; + panel,mirror = <0>; + panel,panel_16bit = <1>; + panel,panel_color = <0>; + panel,width-mm = <157>; + panel,height-mm = <210>; +#endif +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@0 { + reg = <0>; + /*remote-endpoint = <&ov5648_out>;*/ + data-lanes = <1 2>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + status = "disabled"; +}; + +&hdmi_in_vp0 { + status = "disabled"; +}; + +&hdmi_in_vp1 { + status = "disabled"; +}; + +&hdmi_sound { + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vccsys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <0>; + regulator-initial-mode = <0x2>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1100000>; + regulator-changeable-in-suspend; + }; + }; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default"; +// pinctrl-names = "default", "pmic-sleep", +// "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; +// pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; +// pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; +// pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <0>; + /* not save the PMIC_POWER_EN register in uboot */ + not-save-power-en = <1>; + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vccsys>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + regulator-changeable-in-suspend; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_gpu"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vcca1v8_pmu: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + regulator-changeable-in-suspend; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + regulator-changeable-in-suspend; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_acodec"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + regulator-changeable-in-suspend; + }; + }; + + vcc_1v8: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + sleep_sta_ctl: LDO_REG9 { + regulator-name = "sleep_sta_ctl"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + regulator-changeable-in-suspend; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3400 3513 3578 3687 3734 3752 3763 + 3766 3771 3784 3804 3836 3885 3925 + 3962 4005 4063 4114 4169 4227 4303>; + design_capacity = <4150>; + design_qmax = <4565>; + bat_res = <100>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3450>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + low_power_sleep = <1>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4300>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&usb2phy0>; + gate_function_disable = <1>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + hp-volume = <20>; + spk-volume = <3>; + out-l2spk-r2hp; + spk-ctl-gpios = <&gpio1 RK_PB1 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + + wacom: wacom@9 { + compatible = "wacom,w9013"; + reg = <0x09>; + pwr-supply = <&vcc_tp>; + pinctrl-names = "default"; + pinctrl-0 = <&wacom_gpio>; + gpio_detect = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + gpio_intr = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + gpio_rst = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + revert_x = <0>; + revert_y = <0>; + xy_exchange = <0>; + }; +}; + +&i2c3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m1_xfer>; + + tps65185: tps65185@68 { + status = "okay"; + compatible = "ti,tps65185"; + reg = <0x68>; + pinctrl-names = "default"; + pinctrl-0 = <&tps65185_gpio>; + wakeup-gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_HIGH>; + vcomctl-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; + int-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; + powerup-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_HIGH>; + poweren-gpios = <&gpio3 RK_PB1 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c4 { + //camera +}; + +&i2c5 { + status = "okay"; + + sensor@4c { + status = "okay"; + compatible = "gs_mma7660"; + reg = <0x4c>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <6>; + reprobe_en = <1>; + }; + + tsc@24 { + status = "okay"; + compatible = "cy,cyttsp5_i2c_adapter"; + reg = <0x24>; + cy,adapter_id = "cyttsp5_i2c_adapter"; + pinctrl-names = "default"; + pinctrl-0 = <&tsc_gpio>; + cytp-supply = <&vcc_tp>; + cy,core { + cy,name = "cyttsp5_core"; + cy,irq_gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + cy,rst_gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + cy,hid_desc_register = <1>; + /* CY_CORE_FLAG_RESTORE_PARAMETERS */ + cy,flags = <6>; + /* CY_CORE_EWG_NONE */ + cy,easy_wakeup_gesture = <0>; + cy,btn_keys = <172 /* KEY_HOMEPAGE */ + /* previously was KEY_HOME, new Android versions use KEY_HOMEPAGE */ + 139 /* KEY_MENU */ + 158 /* KEY_BACK */ + 217 /* KEY_SEARCH */ + 114 /* KEY_VOLUMEDOWN */ + 115 /* KEY_VOLUMEUP */ + 212 /* KEY_CAMERA */ + 116>; /* KEY_POWER */ + cy,btn_keys-tag = <0>; + cy,mt { + cy,name = "cyttsp5_mt"; + cy,inp_dev_name = "cyttsp5_mt"; + cy,flags = <0xA8>; + cy,abs = + /* ABS_MT_POSITION_X, CY_ABS_MIN_X, CY_ABS_MAX_X, 0, 0 */ + <0x35 0 1404 0 0 + /* ABS_MT_POSITION_Y, CY_ABS_MIN_Y, CY_ABS_MAX_Y, 0, 0 */ + 0x36 0 1872 0 0 + /* ABS_MT_PRESSURE, CY_ABS_MIN_P, CY_ABS_MAX_P, 0, 0 */ + 0x3a 0 255 0 0 + /* CY_IGNORE_VALUE, CY_ABS_MIN_W, CY_ABS_MAX_W, 0, 0 */ + 0xffff 0 255 0 0 + /* ABS_MT_TRACKING_ID, CY_ABS_MIN_T, CY_ABS_MAX_T, 0, 0 */ + 0x39 0 15 0 0 + /* ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0 */ + 0x30 0 255 0 0 + /* ABS_MT_TOUCH_MINOR, 0, 255, 0, 0 */ + 0x31 0 255 0 0 + /* ABS_MT_ORIENTATION, -127, 127, 0, 0 */ + 0x34 0xffffff81 127 0 0 + /* ABS_MT_TOOL_TYPE, 0, MT_TOOL_MAX, 0, 0 */ + 0x37 0 1 0 0 + /* ABS_DISTANCE, 0, 255, 0, 0 */ + 0x19 0 255 0 0>; + + cy,vkeys_x = <1404>; + cy,vkeys_y = <1872>; + cy,revert_x = <0>; + cy,revert_y = <0>; + cy,xy_exchange = <0>; + + cy,virtual_keys = + /* KeyCode CenterX CenterY Width Height */ + /* KEY_BACK */ + <158 1360 90 160 180 + /* KEY_MENU */ + 139 1360 270 160 180 + /* KEY_HOMEPAGE */ + 172 1360 450 160 180 + /* KEY SEARCH */ + 217 1360 630 160 180>; + }; + + cy,btn { + cy,name = "cyttsp5_btn"; + cy,inp_dev_name = "cyttsp5_btn"; + }; + + cy,proximity { + cy,name = "cyttsp5_proximity"; + cy,inp_dev_name = "cyttsp5_proximity"; + cy,abs = + /* ABS_DISTANCE, CY_PROXIMITY_MIN_VAL, CY_PROXIMITY_MAX_VAL, 0, 0 */ + <0x19 0 1 0 0>; + }; + }; + }; +}; + +&i2s0_8ch { + status = "disabled"; +}; + +&i2s1_8ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&video_phy0 { + status = "disabled"; +}; + +&mpp_srv { + status = "okay"; +}; + +&nandc0 { + status = "disabled"; +}; + +&pinctrl { + wacom { + wacom_gpio: wacom-gpio { + rockchip,pins = + <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + tsc { + tsc_gpio: tsc-gpio { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_down>, //touch q gpio + <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + tps_pmic { + tps65185_gpio: tps65185-gpio { + rockchip,pins = + <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA2 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_vbat: wifi-vbat { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_reset_gpio: bt-reset-gpio { + rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_wake_gpio: bt-wake-gpio { + rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_irq_gpio: bt-irq-gpio { + rockchip,pins = <0 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + vcc-tp { + vcc_tp_en: vcc-tp-en { + rockchip,pins = <0 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcc3v3_pmu>; + vccio1-supply = <&vccio_acodec>; + vccio3-supply = <&vcc_1v8>; + vccio4-supply = <&vcca1v8_pmu>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc_3v3>; + vccio7-supply = <&vcc_1v8>; +}; + +&pwm0 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy0_out>; + }; + }; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvenc { + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + + rockchip,regulator-off-in-mem-lite = + <&vdd_cpu>, <&vdd_logic>, <&vdd_gpu>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, + <&vccio_acodec>, <&vccio_sd>, <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>, + <&sleep_sta_ctl>; + rockchip,regulator-on-in-mem-lite = + <&vcc_ddr>, <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>; + + rockchip,regulator-off-in-mem = + <&vdd_cpu>, <&vdd_logic>, <&vdd_gpu>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, + <&vccio_acodec>, <&vccio_sd>, <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>, + <&sleep_sta_ctl>; + rockchip,regulator-on-in-mem = + <&vcc_ddr>, <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>; + + rockchip,regulator-off-in-mem-ultra = + <&vdd_logic>, <&vdd_gpu>, <&vcc_ddr>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, + <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>, <&vccio_acodec>, <&vccio_sd>, + <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>; + rockchip,regulator-on-in-mem-ultra = <&vdd_cpu>, <&sleep_sta_ctl>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + keep-power-in-suspend; + max-frequency = <200000000>; + status = "okay"; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + rockchip,default-sample-phase = <90>; + status = "okay"; +}; + +&tsadc { + status = "okay"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w6.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w6.dts new file mode 100755 index 000000000000..2e6d3160aa16 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink-w6.dts @@ -0,0 +1,968 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include +#include +#include +#include "rk3566.dtsi" +#include "rk3568-android.dtsi" +#include "rk3566-eink.dtsi" + +/ { + model = "Rockchip RK3566 RK817 EINK W6 LP4X Board"; + compatible = "rockchip,rk3566-rk817-eink-W6", "rockchip,rk3566"; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3350>; + rockchip,screen-on-voltage = <3400>; + rockchip,auto-wakeup-interval = <60>; + status = "okay"; + }; + + adc_keys: adc-keys { + status = "disabled"; + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + }; + + hdmi_sound: hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip,hdmi"; + status = "disabled"; + + simple-audio-card,cpu { + sound-dai = <&i2s0_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + leds: gpio-leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 =<&leds_gpio>; + + led@1 { + gpios = <&gpio3 RK_PC4 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "battery-full"; + label = "battery_full"; + retain-state-suspended; + }; + + led@2 { + gpios = <&gpio3 RK_PC5 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "battery-charging"; + label = "battery_charging"; + retain-state-suspended; + }; + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio4 RK_PC3 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_rst>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; + + vcc_tp: vcc-tp-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc_tp_en>; + regulator-name = "vcc_tp"; + }; + + dummy_codec: dummy-codec { + compatible = "rockchip,dummy-codec"; + #sound-dai-cells = <0>; + }; + + mic_sound: mic-sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,rk-mic-sound"; + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&dummy_codec>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_LOW>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6255"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_vbat &wifi_host_wake_irq>; + WIFI,vbat_gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; + WIFI,host_wake_irq = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PC3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&ebc { + /* clock rate 1000M/n, (n=1~32) */ + assigned-clocks = <&cru CPLL_333M>, <&cru DCLK_EBC>; + assigned-clock-rates = <85000000>, <85000000>; + status = "okay"; +}; + +&ebc_dev { + pmic = <&tps65185>; + status = "okay"; + + /* ED060XCD */ + panel,width = <1024>; + panel,height = <758>; + panel,vir_width = <1024>; + panel,vir_height = <758>; + panel,sdck = <20000000>; + panel,lsl = <6>; + panel,lbl = <6>; + panel,ldl = <256>; + panel,lel = <38>; + panel,gdck-sta = <4>; + panel,lgonl = <262>; + panel,fsl = <2>; + panel,fbl = <4>; + panel,fdl = <758>; + panel,fel = <5>; + panel,mirror = <0>; + panel,panel_16bit = <0>; + panel,panel_color = <0>; + panel,width-mm = <90>; + panel,height-mm = <122>; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@0 { + reg = <0>; + remote-endpoint = <&ov5648_out>; + data-lanes = <1 2>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + status = "disabled"; +}; + +&hdmi_in_vp0 { + status = "disabled"; +}; + +&hdmi_in_vp1 { + status = "disabled"; +}; + +&hdmi_sound { + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vccsys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <0>; + regulator-boot-on; + regulator-always-on; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1100000>; + regulator-changeable-in-suspend; + }; + }; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; +// pinctrl-names = "default", "pmic-sleep", +// "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; +// pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; +// pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; +// pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <0>; + /* not save the PMIC_POWER_EN register in uboot */ + not-save-power-en = <1>; + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vccsys>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + regulator-changeable-in-suspend; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_gpu"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vcca1v8_pmu: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + regulator-changeable-in-suspend; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + regulator-changeable-in-suspend; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_acodec"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + regulator-changeable-in-suspend; + }; + }; + + vcc_1v8: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + sleep_sta_ctl: LDO_REG9 { + regulator-name = "sleep_sta_ctl"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + regulator-changeable-in-suspend; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-changeable-in-suspend; + }; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3400 3513 3578 3687 3734 3752 3763 + 3766 3771 3784 3804 3836 3885 3925 + 3962 4005 4063 4114 4169 4227 4303>; + design_capacity = <2250>; + design_qmax = <2750>; + bat_res = <100>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3450>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + low_power_sleep = <1>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4300>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&usb2phy0>; + gate_function_disable = <1>; + }; + }; +}; + +&i2c1 { + status = "okay"; + + ov5648: ov5648@36 { + status = "okay"; + compatible = "ovti,ov5648"; + reg = <0x36>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + /* avdd-supply = <&vcc2v8_dvp>; */ + dovdd-supply = <&vcc1v8_dvp>; + /* dvdd-supply = <&vcc1v8_dvp>; */ + + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clkout0>; + //reset pin control by hardware,used this pin switch to mipi input + //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera + //reset-gpios = <&gpio4 RK_PC3 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "HS5885-BNSM1018-V01"; + rockchip,camera-module-lens-name = "default"; + port { + ov5648_out: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m1_xfer>; + + tps65185: tps65185@68 { + compatible = "ti,tps65185"; + reg = <0x68>; + pinctrl-names = "default"; + pinctrl-0 = <&tps65185_gpio>; + int-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; + wakeup-gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_HIGH>; + vcomctl-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; + powerup-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_HIGH>; + poweren-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + }; + +}; + +&i2c5 { + status = "okay"; + + sensor@4c { + status = "okay"; + compatible = "gs_mma7660"; + reg = <0x4c>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <6>; + reprobe_en = <1>; + }; + + gt9xx: gt9xx@14 { + compatible = "goodix,gt9xx"; + reg = <0x14>; + pinctrl-names = "default"; + pinctrl-0 = <&tp_gpio>; + touch-gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + reset-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + touchscreen-size-x = <1024>; + touchscreen-size-y = <758>; + max-x = <1024>; + max-y = <758>; + tp-size = <9111>; + tp-supply = <&vcc_tp>; + wakeup-source; + touchscreen-key-map = <158>; //KEY_HOMEPAGE=172,KEY_BACK=158,KEY_MENU=139 + goodix,driver-send-cfg = <0>; + goodix,cfg-group0 =[ + 42 00 03 00 04 0A 45 03 22 1F 28 0F 64 3C 03 0F 00 00 00 00 11 00 + 08 00 00 00 00 8B 29 0E 71 6F B2 04 00 00 00 39 02 10 00 21 00 00 + 00 03 64 32 00 00 00 3C 78 94 D5 02 07 00 00 04 C8 40 00 B1 4A 00 + 9E 55 00 8E 61 00 7F 70 00 7F 70 00 00 00 F0 90 3C FF FF 07 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 1C 1A 18 16 14 12 10 0E 0C 0A 08 06 04 02 FF FF FF FF FF FF + FF FF FF FF FF FF FF FF FF FF 00 02 04 06 08 0A 0C 0F 10 12 13 16 + 18 1C 1D 1E 1F 20 21 22 FF FF FF FF FF FF FF FF FF FF FF FF FF FF + FF FF FF FF FF FF FF FF F6 01 + ]; + }; + + ft5436: focaltech@38 { + status = "okay"; + compatible = "focaltech,ft5436"; + reg = <0x38>; + pinctrl-names = "default"; + pinctrl-0 = <&tp_gpio>; + vdd-supply = <&vcc_tp>; + focaltech,reset-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + focaltech,irq-gpio = <&gpio0 RK_PA6 IRQ_TYPE_EDGE_FALLING>; + focaltech,max-touch-number = <5>; + focaltech,display-coords = <0 0 1024 758>; + focaltech,have-key = <1>; + focaltech,key-number = <1>; + + focaltech,key-x-coords = <300>; + focaltech,key-y-coords = <1200>; + wakeup-source; + }; +}; + +&i2s0_8ch { + status = "disabled"; +}; + +&i2s1_8ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&video_phy0 { + status = "disabled"; +}; + +&mpp_srv { + status = "okay"; +}; + +&nandc0 { + status = "disabled"; +}; + +&pinctrl { + cam { + cam_clkout0: cam-clkout0 { + rockchip,pins = + /* cam_clkout0 */ + <4 RK_PA7 1 &pcfg_pull_none>; + }; + + camera_rst: camera-rst { + rockchip,pins = + <4 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + leds { + leds_gpio: leds-gpio { + rockchip,pins = + <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>, + <3 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + tps_pmic { + tps65185_gpio: tps65185-gpio { + rockchip,pins = + <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PC1 RK_FUNC_GPIO &pcfg_output_high>; + }; + }; + + tp { + tp_gpio: tp-gpio { + rockchip,pins = + <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA2 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + vcc-tp { + vcc_tp_en: vcc-tp-en { + rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_vbat: wifi-vbat { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcca1v8_pmu>; + vccio1-supply = <&vccio_acodec>; + vccio3-supply = <&vcc_3v3>; + vccio4-supply = <&vcca1v8_pmu>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc_3v3>; + vccio7-supply = <&vcc1v8_dvp>; +}; + +&pwm4 { + status = "disabled"; +}; + +&rk_rga { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy0_out>; + }; + }; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvenc { + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; + + rockchip,regulator-off-in-mem-lite = + <&vdd_cpu>, <&vdd_logic>, <&vdd_gpu>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, + <&vccio_acodec>, <&vccio_sd>, <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>, + <&sleep_sta_ctl>; + rockchip,regulator-on-in-mem-lite = + <&vcc_ddr>, <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>; + + rockchip,regulator-off-in-mem = + <&vdd_cpu>, <&vdd_logic>, <&vdd_gpu>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, + <&vccio_acodec>, <&vccio_sd>, <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>, + <&sleep_sta_ctl>; + rockchip,regulator-on-in-mem = + <&vcc_ddr>, <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>; + + rockchip,regulator-off-in-mem-ultra = + <&vdd_logic>, <&vdd_gpu>, <&vcc_ddr>, <&vcc_3v3>, <&vdda_0v9>, <&vcc_1v8>, + <&vdda0v9_pmu>, <&vcca1v8_pmu>, <&vcc3v3_pmu>, <&vccio_acodec>, <&vccio_sd>, + <&vcc1v8_dvp>, <&dcdc_boost>, <&otg_switch>; + rockchip,regulator-on-in-mem-ultra = <&vdd_cpu>, <&sleep_sta_ctl>; +}; + +&saradc { + status = "disabled"; + vref-supply = <&vcc_1v8>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + keep-power-in-suspend; + max-frequency = <200000000>; + status = "okay"; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + rockchip,default-sample-phase = <90>; + status = "okay"; +}; + +&tsadc { + status = "okay"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink.dts new file mode 100755 index 000000000000..50412b44bb71 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-eink.dts @@ -0,0 +1,957 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include +#include +#include +#include "rk3566.dtsi" +#include "rk3568-android.dtsi" +#include "rk3566-eink.dtsi" + +/ { + model = "Rockchip RK3566 RK817 EINK LP4X Board"; + compatible = "rockchip,rk3566-rk817-eink", "rockchip,rk3566"; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3350>; + rockchip,screen-on-voltage = <3400>; + rockchip,auto-wakeup-interval = <60>; + status = "okay"; + }; + + adc_keys: adc-keys { + status = "disabled"; + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <297500>; + }; + }; + + hdmi_sound: hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip,hdmi"; + status = "disabled"; + + simple-audio-card,cpu { + sound-dai = <&i2s0_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + status = "disabled"; + compatible = "mmc-pwrseq-simple"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + }; + + vcc_sd: vcc-sd { + compatible = "regulator-gpio"; + enable-active-low; + enable-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc_sd_h>; + regulator-name = "vcc_sd"; + states = <3300000 0x0 + 3300000 0x1>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6255"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "disabled"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "disabled"; + }; +}; + +&ebc { + /* clock rate 1000M/n, (n=1~32) */ + assigned-clocks = <&cru CPLL_333M>, <&cru DCLK_EBC>; + assigned-clock-rates = <250000000>, <250000000>; + status = "okay"; +}; + +&ebc_dev { + pmic = <&tps65185>; + status = "okay"; +#if 0 + /* ED097TC2U1 */ + panel,width = <1200>; + panel,height = <825>; + panel,vir_width = <1200>; + panel,vir_height = <825>; + panel,sdck = <25000000>; + panel,lsl = <4>; + panel,lbl = <4>; + panel,ldl = <300>; + panel,lel = <36>; + panel,gdck-sta = <18>; + panel,lgonl = <265>; + panel,fsl = <2>; + panel,fbl = <4>; + panel,fdl = <825>; + panel,fel = <24>; + panel,mirror = <0>; + panel,panel_16bit = <0>; + panel,panel_color = <0>; + panel,width-mm = <203>; + panel,height-mm = <140>; +#else + /* ES103TC1 */ + panel,width = <1872>; + panel,height = <1404>; + panel,vir_width = <1872>; + panel,vir_height = <1404>; + panel,sdck = <33300000>; + panel,lsl = <18>; + panel,lbl = <17>; + panel,ldl = <234>; + panel,lel = <7>; + panel,gdck-sta = <34>; + panel,lgonl = <192>; + panel,fsl = <1>; + panel,fbl = <4>; + panel,fdl = <1404>; + panel,fel = <12>; + panel,mirror = <0>; + panel,panel_16bit = <1>; + panel,panel_color = <0>; + panel,width-mm = <157>; + panel,height-mm = <210>; +#endif +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + status = "disabled"; +}; + +&hdmi_in_vp0 { + status = "disabled"; +}; + +&hdmi_in_vp1 { + status = "disabled"; +}; + +&hdmi_sound { + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vccsys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <0>; + /* not save the PMIC_POWER_EN register in uboot */ + not-save-power-en = <1>; + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vccsys>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_gpu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca1v8_pmu: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_acodec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_1v8: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc2v8_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3400 3513 3578 3687 3734 3752 3763 + 3766 3771 3784 3804 3836 3885 3925 + 3962 4005 4063 4114 4169 4227 4303>; + design_capacity = <5000>; + design_qmax = <5500>; + bat_res = <100>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3450>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4300>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&usb2phy0>; + gate_function_disable = <1>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + hp-volume = <20>; + spk-volume = <3>; + out-l2spk-r2hp; + spk-ctl-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + tps65185: tps65185@68 { + status = "okay"; + compatible = "ti,tps65185"; + reg = <0x68>; + pinctrl-names = "default"; + pinctrl-0 = <&tps65185_gpio>; + int-gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_HIGH>; + wakeup-gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_HIGH>; + vcomctl-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_HIGH>; + powerup-gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c2 { + status = "okay"; + wacom: wacom@9 { + compatible = "wacom,w9013"; + reg = <0x09>; + pinctrl-names = "default"; + pinctrl-0 = <&wacom_gpio>; + gpio_detect = <&gpio3 RK_PA3 GPIO_ACTIVE_HIGH>; + gpio_intr = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; + gpio_rst = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; + revert_x = <0>; + revert_y = <0>; + xy_exchange = <0>; + }; +}; + +&i2c3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m1_xfer>; + + ts@40 { + compatible = "gslX680-pad"; + reg = <0x40>; + touch-gpio = <&gpio3 RK_PB0 IRQ_TYPE_LEVEL_HIGH>; + reset-gpio = <&gpio3 RK_PB1 IRQ_TYPE_LEVEL_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&tp_gpio>; + screen_max_x = <1200>; + screen_max_y = <1920>; + revert_x = <0>; + revert_y = <1>; + revert_xy = <0>; + chip_id = <1>; + status = "disabled"; + }; + + tsc@24 { + status = "okay"; + compatible = "cy,cyttsp5_i2c_adapter"; + reg = <0x24>; + cy,adapter_id = "cyttsp5_i2c_adapter"; + //cytp-supply = <&vcc_sd>; + cy,core { + cy,name = "cyttsp5_core"; + pinctrl-names = "default"; + pinctrl-0 = <&tsc_gpio>; + cy,irq_gpio = <&gpio3 RK_PB1 GPIO_ACTIVE_HIGH>; + cy,rst_gpio = <&gpio3 RK_PB2 GPIO_ACTIVE_HIGH>; + cy,1v8_gpio = <&gpio3 RK_PC4 GPIO_ACTIVE_HIGH>; + cy,2v8_gpio = <&gpio3 RK_PC5 GPIO_ACTIVE_HIGH>; + cy,hid_desc_register = <1>; + /* CY_CORE_FLAG_RESTORE_PARAMETERS */ + cy,flags = <6>; + /* CY_CORE_EWG_NONE */ + cy,easy_wakeup_gesture = <0>; + cy,btn_keys = <172 /* KEY_HOMEPAGE */ + /* previously was KEY_HOME, new Android versions use KEY_HOMEPAGE */ + 139 /* KEY_MENU */ + 158 /* KEY_BACK */ + 217 /* KEY_SEARCH */ + 114 /* KEY_VOLUMEDOWN */ + 115 /* KEY_VOLUMEUP */ + 212 /* KEY_CAMERA */ + 116>; /* KEY_POWER */ + cy,btn_keys-tag = <0>; + cy,mt { + cy,name = "cyttsp5_mt"; + cy,inp_dev_name = "cyttsp5_mt"; + cy,flags = <0>; + cy,abs = + /* ABS_MT_POSITION_X, CY_ABS_MIN_X, CY_ABS_MAX_X, 0, 0 */ + <0x35 0 1872 0 0 + /* ABS_MT_POSITION_Y, CY_ABS_MIN_Y, CY_ABS_MAX_Y, 0, 0 */ + 0x36 0 1404 0 0 + /* ABS_MT_PRESSURE, CY_ABS_MIN_P, CY_ABS_MAX_P, 0, 0 */ + 0x3a 0 255 0 0 + /* CY_IGNORE_VALUE, CY_ABS_MIN_W, CY_ABS_MAX_W, 0, 0 */ + 0xffff 0 255 0 0 + /* ABS_MT_TRACKING_ID, CY_ABS_MIN_T, CY_ABS_MAX_T, 0, 0 */ + 0x39 0 15 0 0 + /* ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0 */ + 0x30 0 255 0 0 + /* ABS_MT_TOUCH_MINOR, 0, 255, 0, 0 */ + 0x31 0 255 0 0 + /* ABS_MT_ORIENTATION, -127, 127, 0, 0 */ + 0x34 0xffffff81 127 0 0 + /* ABS_MT_TOOL_TYPE, 0, MT_TOOL_MAX, 0, 0 */ + 0x37 0 1 0 0 + /* ABS_DISTANCE, 0, 255, 0, 0 */ + 0x19 0 255 0 0>; + + cy,vkeys_x = <1872>; + cy,vkeys_y = <1404>; + cy,revert_x = <0>; + cy,revert_y = <1>; + cy,xy_exchange = <0>; + + cy,virtual_keys = + /* KeyCode CenterX CenterY Width Height */ + /* KEY_BACK */ + <158 1360 90 160 180 + /* KEY_MENU */ + 139 1360 270 160 180 + /* KEY_HOMEPAGE */ + 172 1360 450 160 180 + /* KEY SEARCH */ + 217 1360 630 160 180>; + }; + + cy,btn { + cy,name = "cyttsp5_btn"; + cy,inp_dev_name = "cyttsp5_btn"; + }; + + cy,proximity { + cy,name = "cyttsp5_proximity"; + cy,inp_dev_name = "cyttsp5_proximity"; + cy,abs = + /* ABS_DISTANCE, CY_PROXIMITY_MIN_VAL, CY_PROXIMITY_MAX_VAL, 0, 0 */ + <0x19 0 1 0 0>; + }; + }; + }; +}; + +&i2c5 { + status = "disabled"; + + kxtj: kxtj3@e { + status = "disabled"; + compatible = "gs_kxtj9"; + pinctrl-names = "default"; + pinctrl-0 = <&kxtj3_irq_gpio>; + reg = <0x0e>; + irq-gpio = <&gpio4 RK_PC6 IRQ_TYPE_EDGE_RISING>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + power-off-in-suspend = <1>; + layout = <5>; + }; +}; + +&i2s0_8ch { + status = "disabled"; +}; + +&i2s1_8ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&video_phy0 { + status = "disabled"; +}; + +&mpp_srv { + status = "okay"; +}; + +&nandc0 { + status = "disabled"; +}; + +&pinctrl { + wacom { + wacom_gpio: wacom-gpio { + rockchip,pins = + <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + tsc { + tsc_gpio: tsc-gpio { + rockchip,pins = + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + tps_pmic { + tps65185_gpio: tps65185-gpio { + rockchip,pins = + <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + tp { + tp_gpio: tp-gpio { + rockchip,pins = + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sensor { + kxtj3_irq_gpio: kxtj3-irq-gpio { + rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA2 2 &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + vcc_sd { + vcc_sd_h: vcc-sd-h { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcc3v3_pmu>; + vccio1-supply = <&vccio_acodec>; + vccio3-supply = <&vccio_sd>; + vccio4-supply = <&vcc_3v3>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc_3v3>; + vccio7-supply = <&vcc_3v3>; +}; + +&pwm4 { + status = "disabled"; +}; + +&rk_rga { + status = "okay"; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvenc { + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&saradc { + status = "disabled"; + vref-supply = <&vcc_1v8>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + keep-power-in-suspend; + max-frequency = <200000000>; + status = "okay"; +}; + +&sdmmc0 { + max-frequency = <50000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + sd-uhs-sdr104; + vmmc-supply = <&vcc_sd>; + vqmmc-supply = <&vccio_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; + status = "okay"; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + status = "disabled"; +}; + +&tsadc { + status = "okay"; +}; + +&uart1 { + status = "disabled"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-k108.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-k108.dts new file mode 100755 index 000000000000..f6f4534aad50 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-k108.dts @@ -0,0 +1,1307 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include +#include +#include +#include "rk3566.dtsi" +#include "rk3568-android.dtsi" + +/ { + model = "Rockchip RK3566 RK817 TABLET K108 LP4X Board"; + compatible = "rockchip,rk3566-rk817-tablet-k108", "rockchip,rk3566"; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <297500>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm4 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3350>; + rockchip,screen-on-voltage = <3400>; + status = "okay"; + }; + + flash_rgb13h: flash-rgb13h { + status = "okay"; + compatible = "led,rgb13h"; + label = "gpio-flash"; + pinctrl-names = "default"; + pinctrl-0 = <&flash_led_gpios>; + led-max-microamp = <20000>; + flash-max-microamp = <20000>; + flash-max-timeout-us = <1000000>; + enable-gpio = <&gpio4 RK_PB7 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + hall_sensor: hall-mh248 { + compatible = "hall-mh248"; + irq-gpio = <&gpio3 RK_PA6 IRQ_TYPE_EDGE_BOTH>; + hall-active = <1>; + status = "okay"; + }; + + hdmi_sound: hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip,hdmi"; + status = "okay"; + + simple-audio-card,cpu { + sound-dai = <&i2s0_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_rst>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio4 RK_PC4 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + }; + + vcc_sd: vcc-sd { + compatible = "regulator-gpio"; + enable-active-low; + enable-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc_sd_h>; + regulator-name = "vcc_sd"; + states = <3300000 0x0 + 3300000 0x1>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio3 RK_PA3 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_host_en>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "rtl8723cs"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; + WIFI,vbat_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&combphy1_usq { + rockchip,dis-u3otg1-port; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@0 { + reg = <0>; + remote-endpoint = <&gc2385_out>; + data-lanes = <1>; + }; + mipi_in_ucam1: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov8858_out>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&dsi0 { + status = "okay"; + rockchip,lane-rate = <1000>; + panel@0 { + compatible = "aoly,sl008pa21y1285-b00", "simple-panel-dsi"; + reg = <0>; + + backlight = <&backlight>; + power-supply=<&vcc_3v3>; + enable-gpios = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + stbyb-gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>; + + pinctrl-names = "default"; + pinctrl-0 = <&lcd_enable_gpio>, <&lcd_rst_gpio>, <&lcd_stanby_gpio>; + + prepare-delay-ms = <120>; + reset-delay-ms = <120>; + init-delay-ms = <120>; + stbyb-delay-ms = <120>; + enable-delay-ms = <120>; + disable-delay-ms = <120>; + unprepare-delay-ms = <120>; + + width-mm = <229>; + height-mm = <143>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 29 00 04 FF 98 81 03 + 23 00 02 01 00 + 23 00 02 02 00 + 23 00 02 03 73 + 23 00 02 04 00 + 23 00 02 05 00 + 23 00 02 06 08 + 23 00 02 07 00 + 23 00 02 08 00 + 23 00 02 09 00 + 23 00 02 0A 01 + 23 00 02 0B 01 + 23 00 02 0C 00 + 23 00 02 0D 01 + 23 00 02 0E 01 + 23 00 02 0F 00 + 23 00 02 10 00 + 23 00 02 11 00 + 23 00 02 12 00 + 23 00 02 13 1F + 23 00 02 14 1F + 23 00 02 15 00 + 23 00 02 16 00 + 23 00 02 17 00 + 23 00 02 18 00 + 23 00 02 19 00 + 23 00 02 1A 00 + 23 00 02 1B 00 + 23 00 02 1C 00 + 23 00 02 1D 00 + 23 00 02 1E 40 + 23 00 02 1F C0 + 23 00 02 20 06 + 23 00 02 21 01 + 23 00 02 22 06 + 23 00 02 23 01 + 23 00 02 24 88 + 23 00 02 25 88 + 23 00 02 26 00 + 23 00 02 27 00 + 23 00 02 28 3B + 23 00 02 29 03 + 23 00 02 2A 00 + 23 00 02 2B 00 + 23 00 02 2C 00 + 23 00 02 2D 00 + 23 00 02 2E 00 + 23 00 02 2F 00 + 23 00 02 30 00 + 23 00 02 31 00 + 23 00 02 32 00 + 23 00 02 33 00 + 23 00 02 34 00 + 23 00 02 35 00 + 23 00 02 36 00 + 23 00 02 37 00 + 23 00 02 38 00 + 23 00 02 39 00 + 23 00 02 3A 00 + 23 00 02 3B 00 + 23 00 02 3C 00 + 23 00 02 3D 00 + 23 00 02 3E 00 + 23 00 02 3F 00 + 23 00 02 40 00 + 23 00 02 41 00 + 23 00 02 42 00 + 23 00 02 43 00 + 23 00 02 44 00 + 23 00 02 50 01 + 23 00 02 51 23 + 23 00 02 52 45 + 23 00 02 53 67 + 23 00 02 54 89 + 23 00 02 55 AB + 23 00 02 56 01 + 23 00 02 57 23 + 23 00 02 58 45 + 23 00 02 59 67 + 23 00 02 5A 89 + 23 00 02 5B AB + 23 00 02 5C CD + 23 00 02 5D EF + 23 00 02 5E 00 + 23 00 02 5F 01 + 23 00 02 60 01 + 23 00 02 61 06 + 23 00 02 62 06 + 23 00 02 63 07 + 23 00 02 64 07 + 23 00 02 65 00 + 23 00 02 66 00 + 23 00 02 67 02 + 23 00 02 68 02 + 23 00 02 69 05 + 23 00 02 6A 05 + 23 00 02 6B 02 + 23 00 02 6C 0D + 23 00 02 6D 0D + 23 00 02 6E 0C + 23 00 02 6F 0C + 23 00 02 70 0F + 23 00 02 71 0F + 23 00 02 72 0E + 23 00 02 73 0E + 23 00 02 74 02 + 23 00 02 75 01 + 23 00 02 76 01 + 23 00 02 77 06 + 23 00 02 78 06 + 23 00 02 79 07 + 23 00 02 7A 07 + 23 00 02 7B 00 + 23 00 02 7C 00 + 23 00 02 7D 02 + 23 00 02 7E 02 + 23 00 02 7F 05 + 23 00 02 80 05 + 23 00 02 81 02 + 23 00 02 82 0D + 23 00 02 83 0D + 23 00 02 84 0C + 23 00 02 85 0C + 23 00 02 86 0F + 23 00 02 87 0F + 23 00 02 88 0E + 23 00 02 89 0E + 23 00 02 8A 02 + 29 00 04 FF 98 81 04 + 23 00 02 6C 15 + 23 00 02 6E 2A + 23 00 02 6F 33 + 23 00 02 8D 1B + 23 00 02 87 BA + 23 00 02 3A 24 + 23 00 02 26 76 + 23 00 02 B2 D1 + 29 00 04 FF 98 81 01 + 23 00 02 22 0A + 23 00 02 31 00 + 23 00 02 43 66 + 23 00 02 53 40 + 23 00 02 50 87 + 23 00 02 51 82 + 23 00 02 60 15 + 23 00 02 61 01 + 23 00 02 62 0C + 23 00 02 63 00 + 29 00 04 FF 98 81 00 + 23 00 02 35 00 + + 05 78 01 11 + 05 dc 01 29 + //05 00 01 35 + ]; + + panel-exit-sequence = [ + 05 dc 01 28 + 05 78 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <160000000>; + hactive = <1200>; + vactive = <1920>; + + hsync-len = <1>;//19 + hback-porch = <60>;//40 + hfront-porch = <80>;//123 + + vsync-len = <1>; + vback-porch = <25>; + vfront-porch = <35>; + + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <1>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; + +}; + +&dsi0_in_vp0 { + status = "okay"; +}; + +&dsi0_in_vp1 { + status = "disabled"; +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + status = "okay"; +}; + +&hdmi_in_vp0 { + status = "okay"; +}; + +&hdmi_in_vp1 { + status = "disabled"; +}; + +&hdmi_sound { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vccsys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <0>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vccsys>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_gpu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca1v8_pmu: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_acodec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v8: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc2v8_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3400 3513 3578 3687 3734 3752 3763 + 3766 3771 3784 3804 3836 3885 3925 + 3962 4005 4063 4114 4169 4227 4303>; + design_capacity = <5000>; + design_qmax = <5500>; + bat_res = <100>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3450>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4300>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&usb2phy0>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + hp-volume = <20>; + spk-volume = <3>; + out-l2spk-r2hp; + spk-ctl-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + }; +}; + +&i2c2 { + status = "okay"; + pinctrl-0 = <&i2c2m1_xfer>; + + dw9714: dw9714@c { + compatible = "dongwoon,dw9714"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,vcm-start-current = <10>; + rockchip,vcm-rated-current = <85>; + rockchip,vcm-step-mode = <5>; + rockchip,camera-module-facing = "back"; + }; + + gc2385: gc2385@37 { + compatible = "galaxycore,gc2385"; + status = "okay"; + reg = <0x37>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clk>; + + //reset pin control by hardware,used this pin switch to mipi input + //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera + reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 10 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "HS5885-BNSM1018-V01"; + rockchip,camera-module-lens-name = "default"; + port { + gc2385_out: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1>; + }; + }; + }; + + ov8858: ov8858@36 { + status = "okay"; + compatible = "ovti,ov8858"; + reg = <0x36>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "rockchip,camera_default", "rockchip,camera_sleep"; + pinctrl-0 = <&cam_clkout0>; + pinctrl-1 = <&cam_sleep>; + //reset pin control by hardware,used this pin switch to mipi input + //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera + reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 11 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "HS5885-BNSM1018-V01"; + rockchip,camera-module-lens-name = "default"; + flash-leds = <&flash_rgb13h>; + lens-focus = <&dw9714>; + port { + ov8858_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&i2c3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m1_xfer>; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <138>; + i2c-scl-falling-time-ns = <4>; + + gt9xx: gt9xx@14 { + compatible = "goodix,gt9xx"; + reg = <0x14>; + pinctrl-names = "default"; + pinctrl-0 = <>9xx_gpio>; + touch-gpio = <&gpio3 RK_PB0 IRQ_TYPE_LEVEL_HIGH>; + reset-gpio = <&gpio3 RK_PB1 GPIO_ACTIVE_HIGH>; + max-x = <1200>; + max-y = <1920>; + tp-size = <9110>; + tp-supply = <&vcc_3v3>; + }; +}; + +&i2c5 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <144>; + i2c-scl-falling-time-ns = <4>; + + sensor@4c { + compatible = "gs_mc3230"; + reg = <0x4c>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <9>; + reprobe_en = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sensor_gpio>; + irq-gpio = <&gpio3 RK_PA2 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +&i2s0_8ch { + status = "okay"; +}; + +&i2s1_8ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&video_phy0 { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pinctrl { + cam { + cam_clkout0: cam-clkout0 { + rockchip,pins = + /* cam_clkout0 */ + <4 RK_PA7 1 &pcfg_pull_none>; + }; + + cam_sleep: cam-sleep { + rockchip,pins = + /* cam_sleep */ + <4 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + camera_rst: camera-rst { + rockchip,pins = + /* front camera reset */ + <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, + /* back camra reset */ + <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + flash_led_gpios: flash-led { + rockchip,pins = + /* flash led enable */ + <4 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + gt9xx { + gt9xx_gpio: gt9xx-gpio { + rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + lcd { + lcd_rst_gpio: lcd-rst-gpio { + rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lcd_enable_gpio: lcd-enable-gpio { + rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lcd_stanby_gpio: lcd-stanby-gpio { + rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA2 2 &pcfg_pull_none>; + }; + }; + + sensor { + sensor_gpio: sensor-gpio { + rockchip,pins = <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + vcc_sd { + vcc_sd_h: vcc-sd-h { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdmmc0 { + sdmmc0_det_gpio: sdmmc0-det-gpio { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb { + vcc5v0_host_en: vcc5v0-host-en { + rockchip,pins = <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcc3v3_pmu>; + vccio1-supply = <&vccio_acodec>; + vccio3-supply = <&vccio_sd>; + vccio4-supply = <&vcc3v3_pmu>; + vccio5-supply = <&vcc_1v8>; + vccio6-supply = <&vcc1v8_dvp>; + vccio7-supply = <&vcc_3v3>; +}; + +&pwm4 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy0_out>; + }; + }; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvenc { + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&route_dsi0 { + status = "okay"; +}; + +&route_hdmi { + status = "okay"; + connect = <&vp0_out_hdmi>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + max-frequency = <200000000>; + status = "okay"; +}; + +&sdmmc0 { + max-frequency = <150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + sd-uhs-sdr104; + vmmc-supply = <&vcc_sd>; + vqmmc-supply = <&vccio_sd>; + cd-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det_gpio>; + status = "okay"; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + rockchip,default-sample-phase = <90>; + status = "okay"; +}; + +&tsadc { + status = "okay"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&u2phy0_host { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&usbhost30 { + status = "okay"; +}; + +&usbhost_dwc3 { + phys = <&u2phy0_host>; + phy-names = "usb2-phy"; + maximum-speed = "high-speed"; + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-rkg11.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-rkg11.dts new file mode 100755 index 000000000000..8102da0270fa --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-rkg11.dts @@ -0,0 +1,1180 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include +#include +#include +#include "rk3566.dtsi" +#include "rk3568-android.dtsi" + +/ { + model = "Rockchip RK3566 RK817 TABLET RKG11 LP4 Board"; + compatible = "rockchip,rk3566-rk817-tablet-rkg11", "rockchip,rk3566"; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <297500>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm4 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3350>; + rockchip,screen-on-voltage = <3400>; + status = "okay"; + }; + + es7210_sound: es7210-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,es7210"; + + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&es7210>; + }; + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc3v3_lcd0_n: vcc3v3-lcd0-n { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_lcd0_n"; + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,cpu { + sound-dai = <&i2s3_2ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio0 RK_PC4 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>, + <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "rtl8821cs"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,vbat_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + hall_sensor: hall-mh248 { + compatible = "hall-mh248"; + pinctrl-names = "default"; + pinctrl-0 = <&mh248_irq_gpio>; + irq-gpio = <&gpio0 RK_PC6 IRQ_TYPE_EDGE_BOTH>; + hall-active = <1>; + status = "okay"; + }; + + vibrator { + compatible = "rk-vibrator-gpio"; + vibrator-gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_LOW>; + status = "okay"; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + rknpu_reserved: rknpu { + compatible = "shared-dma-pool"; + inactive; + reusable; + size = <0x0 0x20000000>; + alignment = <0x0 0x1000>; + }; + }; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@0 { + reg = <0>; + remote-endpoint = <&gc5035_out>; + data-lanes = <1 2>; + }; + mipi_in_ucam1: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov8858_out>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&dsi0 { + status = "okay"; + rockchip,dual-channel = <&dsi1>; + panel@0 { + status = "okay"; + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + power-supply = <&vcc3v3_lcd0_n>; + //vsp-supply = <&outp>; + //vsn-supply = <&outn>; + //enable-gpios = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&lcd_enable_gpio>, <&lcd_rst_gpio>, <&lcd_panel_vsp>, <&lcd_panel_vsn>; + + prepare-delay-ms = <60>; + unprepare-delay-ms = <60>; + enable-delay-ms = <60>; + disable-delay-ms = <60>; + init-delay-ms = <60>; + reset-delay-ms = <60>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <8>; + panel-init-sequence = [ + 05 20 01 11 + 05 96 01 29 + ]; + + panel-exit-sequence = [ + 05 05 01 28 + 05 78 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + timing0: timing0 { + clock-frequency = <255000000>; + hactive = <1600>; + vactive = <2176>; + hsync-len = <14>; //20, 50 + hback-porch = <25>; //50, 56 + hfront-porch = <25>;//50, 30 + vsync-len = <8>; + vback-porch = <73>; + vfront-porch = <250>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <1>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; + +}; + +&dsi1 { + status = "okay"; +}; + +&dsi0_in_vp0 { + status = "okay"; +}; + +&dsi0_in_vp1 { + status = "disabled"; +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vccsys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <0>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vccsys>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_gpu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca1v8_pmu: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_acodec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_1v8: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc2v8_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3510 3679 3691 3714 3738 3759 3776 + 3795 3811 3834 3852 3881 3942 3976 + 4012 4075 4114 4177 4232 4277 4351>; + design_capacity = <7916>; + design_qmax = <8708>; + bat_res = <110>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3450>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4300>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&usb2phy0>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru I2S3_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S3_MCLKOUT>, <&cru I2S3_MCLK_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S3_MCLKOUT_TX>, <&cru I2S3_MCLKOUT>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s3m1_mclk>; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + use-ext-amplifier; + //out-l2spk-r2hp; + spk-ctl-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + + dio5632@3e { + compatible = "DIO5632"; + reg = <0x3e>; + status = "disabled"; + + outp: outp@3e { + regulator-name = "LCD_VSP"; + vin-supply = <&vccsys>; + enable-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; + }; + + outn: outn@3e { + regulator-name = "LCD_VSN"; + vin-supply = <&vccsys>; + enable-gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + }; + }; + + es7210: es7210@43 { + #sound-dai-cells = <0>; + compatible = "ES7210_MicArray_0"; + reg = <0x43>; + clocks = <&cru I2S1_MCLKOUT_RX>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT_RX>; + assigned-clock-parents = <&cru CLK_I2S1_8CH_RX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + }; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2m1_xfer>; + + gc5035: gc5035@37 { + compatible = "galaxycore,gc5035"; + status = "okay"; + reg = <0x37>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + + //reset pin control by hardware,used this pin switch to mipi input + //0->FRONT camera, 1->REAR camera + reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 10 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "XHG-RKX11F-V5"; + rockchip,camera-module-lens-name = "HR232H65"; + port { + gc5035_out: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; + + ov8858: ov8858@36 { + status = "okay"; + compatible = "ovti,ov8858"; + reg = <0x36>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "rockchip,camera_default", "rockchip,camera_sleep"; + pinctrl-0 = <&cam_clkout0>; + pinctrl-1 = <&cam_sleep>; + //reset pin control by hardware,used this pin switch to mipi input + //0->FRONT camera, 1->REAR camera + reset-gpios = <&gpio4 17 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 11 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "XHG-RKX11B-V10"; + rockchip,camera-module-lens-name = "default"; + port { + ov8858_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&i2c3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m1_xfer>; + + focaltech: focaltech@38 { + status = "okay"; + compatible = "focaltech,fts"; + reg = <0x38>; + power-supply = <&vcc3v3_lcd0_n>; + pinctrl-names = "default"; + pinctrl-0 = <&tp_gpio>; + focaltech,irq-gpio = <&gpio3 RK_PB0 IRQ_TYPE_LEVEL_LOW>; + focaltech,reset-gpio = <&gpio3 RK_PB1 GPIO_ACTIVE_HIGH>; + focaltech,have-key = <0>; + focaltech,key-number = <3>; + focaltech,keys = <256 1068 64 64 128 1068 64 64 192 1068 64 64>; + focaltech,key-x-coord = <1600>; + focaltech,key-y-coord = <2176>; + focaltech,max-touch-number = <5>; + }; +}; + +&i2c5 { + status = "okay"; + + sensor@18 { + compatible = "gs_sc7a20"; + reg = <0x18>; + type = ; + irq_enable = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&sensor_gpio>; + irq-gpio = <&gpio3 RK_PA2 IRQ_TYPE_LEVEL_LOW>; + poll_delay_ms = <10>; + layout = <7>; + status = "disabled"; + }; + + ls_em3071x@24 { + compatible = "ls_em3071x"; + reg = <0x24>; + type = ; + irq_enable = <0>; + poll_delay_ms = <100>; + status = "okay"; + }; + + ps_em3071x@24 { + compatible = "ps_em3071x"; + reg = <0x24>; + type = ; + pinctrl-names = "default"; + pinctrl-0 = <&em3071x_irq_gpio>; + irq-gpio = <&gpio3 RK_PA6 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <1>; + ps_threshold_high = <25>; + ps_threshold_low = <15>; + poll_delay_ms = <100>; + status = "okay"; + }; + + icm20607_acc@68 { + compatible = "icm2060x_acc"; + reg = <0x68>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + layout = <1>; + status = "okay"; + }; + + icm20607_gyro@68 { + compatible = "icm2060x_gyro"; + reg = <0x68>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + layout = <1>; + status = "okay"; + }; + + ak09918_compass: ak09918_compass@c { + compatible = "ak09918"; + reg = <0x0c>; + type = ; + irq_enable = <0>; + poll_delay_ms = <30>; + layout = <1>; + status = "okay"; + }; +}; + +&i2s1_8ch { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,clk-trcm = <2>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclkrx + &i2s1m0_lrckrx + &i2s1m0_sdi0 + &i2s1m0_sdi1 + &i2s1m0_sdi2 + &i2s1m0_sdi3>; +}; + +&i2s3_2ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s3m1_sclk + &i2s3m1_lrck + &i2s3m1_sdi + &i2s3m1_sdo>; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pinctrl { + cam { + cam_clkout0: cam-clkout0 { + rockchip,pins = + /* cam_clkout0 */ + <4 RK_PA7 1 &pcfg_pull_none>; + }; + + cam_sleep: cam-sleep { + rockchip,pins = + /* cam_sleep */ + <4 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + camera_rst: camera-rst { + rockchip,pins = + /* front camera reset */ + <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, + /* back camra reset */ + <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + flash_led_gpios: flash-led { + rockchip,pins = + /* flash led enable */ + <4 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + tp { + tp_gpio: tp-gpio { + rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd { + lcd_rst_gpio: lcd-rst-gpio { + rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lcd_enable_gpio: lcd-enable-gpio { + rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lcd_panel_vsp: lcd-panel-vsp { + rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + lcd_panel_vsn: lcd-panel-vsn { + rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA2 2 &pcfg_pull_none>; + }; + }; + + sensor { + sensor_gpio:sensor-gpio { + rockchip,pins = <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + em3071x_irq_gpio: em3071x-irq-gpio { + rockchip,pins = <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + mh248_irq_gpio: mh248-irq-gpio { + rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>, + <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcc3v3_pmu>; + vccio1-supply = <&vccio_acodec>; + vccio3-supply = <&vccio_sd>; + vccio4-supply = <&vcca1v8_pmu>; + vccio5-supply = <&vcc_1v8>; + vccio6-supply = <&vcc1v8_dvp>; + vccio7-supply = <&vccio_acodec>; +}; + +&pwm4 { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy0_out>; + }; + }; +}; + +&rknpu { + memory-region = <&rknpu_reserved>; + rknpu-supply = <&vdd_gpu>; + status = "okay"; +}; + +&rknpu_mmu { + status = "disabled"; +}; + +&rk_rga { + status = "okay"; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvenc { + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&route_dsi0 { + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + max-frequency = <200000000>; + status = "okay"; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + rockchip,default-sample-phase = <90>; + status = "okay"; +}; + +&tsadc { + status = "okay"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-v10.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-v10.dts new file mode 100755 index 000000000000..1d4a0484473a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet-v10.dts @@ -0,0 +1,1208 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include +#include +#include +#include "rk3566.dtsi" +#include "rk3568-android.dtsi" + +/ { + model = "Rockchip RK3566 RK817 TABLET LP4X Board"; + compatible = "rockchip,rk3566-rk817-tablet", "rockchip,rk3566"; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <297500>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm4 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 50 51 52 53 54 + 55 55 56 57 58 59 60 61 + 62 63 64 64 65 65 66 67 + 68 69 70 71 71 72 73 74 + 75 76 77 78 79 79 80 81 + 82 83 84 85 86 86 87 88 + 89 90 91 92 93 94 94 95 + 96 97 98 99 100 101 101 102 + 103 104 105 106 107 107 108 109 + 110 111 112 113 114 115 115 116 + 117 118 119 120 121 122 123 123 + 124 125 126 127 128 129 130 130 + 131 132 133 134 135 136 136 137 + 138 139 140 141 142 143 143 144 + 145 146 147 147 148 149 150 151 + 152 153 154 155 156 156 157 158 + 159 157 158 159 160 161 162 162 + 163 164 165 166 167 168 169 169 + 170 171 172 173 174 175 175 176 + 177 178 179 180 181 182 182 183 + 184 185 186 187 188 189 190 190 + 191 192 193 194 195 196 197 197 + 198 199 200 201 202 203 204 204 + 205 206 207 208 209 209 210 211 + 212 213 213 214 214 215 215 216 + 216 217 217 218 218 219 219 220 + >; + default-brightness-level = <200>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3350>; + rockchip,screen-on-voltage = <3400>; + status = "okay"; + }; + + flash_rgb13h: flash-rgb13h { + status = "okay"; + compatible = "led,rgb13h"; + label = "gpio-flash"; + pinctrl-names = "default"; + pinctrl-0 = <&flash_led_gpios>; + led-max-microamp = <20000>; + flash-max-microamp = <20000>; + flash-max-timeout-us = <1000000>; + enable-gpio = <&gpio4 6 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + hdmi_sound: hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip,hdmi"; + status = "okay"; + + simple-audio-card,cpu { + sound-dai = <&i2s0_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_rst>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio4 RK_PC4 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + }; + + vcc_sd: vcc-sd { + compatible = "regulator-gpio"; + enable-active-low; + enable-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc_sd_h>; + regulator-name = "vcc_sd"; + states = <3300000 0x0 + 3300000 0x1>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6255"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@0 { + reg = <0>; + remote-endpoint = <&gc2385_out>; + data-lanes = <1>; + }; + mipi_in_ucam1: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov8858_out>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&dsi0 { + status = "okay"; + rockchip,lane-rate = <1000>; + panel@0 { + compatible = "aoly,sl008pa21y1285-b00", "simple-panel-dsi"; + reg = <0>; + + backlight = <&backlight>; + //power-supply=<&vcc_3v3>; + enable-gpios = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + stbyb-gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>; + + pinctrl-names = "default"; + pinctrl-0 = <&lcd_enable_gpio>, <&lcd_rst_gpio>, <&lcd_stanby_gpio>; + + prepare-delay-ms = <120>; + reset-delay-ms = <120>; + init-delay-ms = <120>; + stbyb-delay-ms = <120>; + enable-delay-ms = <120>; + disable-delay-ms = <120>; + unprepare-delay-ms = <120>; + + width-mm = <229>; + height-mm = <143>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 23 00 02 B0 01 + 23 00 02 C3 0F + 23 00 02 C4 00 + 23 00 02 C5 00 + 23 00 02 C6 00 + 23 00 02 C7 00 + 23 00 02 C8 0D + 23 00 02 C9 12 + 23 00 02 CA 11 + 23 00 02 CD 1D + 23 00 02 CE 1B + 23 00 02 CF 0B + 23 00 02 D0 09 + 23 00 02 D1 07 + 23 00 02 D2 05 + 23 00 02 D3 01 + 23 00 02 D7 10 + 23 00 02 D8 00 + 23 00 02 D9 00 + 23 00 02 DA 00 + 23 00 02 DB 00 + 23 00 02 DC 0E + 23 00 02 DD 12 + 23 00 02 DE 11 + 23 00 02 E1 1E + 23 00 02 E2 1C + 23 00 02 E3 0C + 23 00 02 E4 0A + 23 00 02 E5 08 + 23 00 02 E6 06 + 23 00 02 E7 02 + 23 00 02 B0 03 + 23 00 02 BE 03 + 23 00 02 CC 44 + 23 00 02 C8 07 + 23 00 02 C9 05 + 23 00 02 CA 42 + 23 00 02 CD 3E + 23 00 02 CF 60 + 23 00 02 D2 04 + 23 00 02 D3 04 + 23 00 02 D4 01 + 23 00 02 D5 00 + 23 00 02 D6 03 + 23 00 02 D7 04 + 23 00 02 D9 01 + 23 00 02 DB 01 + 23 00 02 E4 F0 + 23 00 02 E5 0A + 23 00 02 B0 00 + 23 00 02 BA 8F// NEW ADD + 23 00 02 BD 63 + 23 00 02 C2 08 + 23 00 02 C4 10 + 23 00 02 B0 02 + 23 00 02 C0 00 + 23 00 02 C1 0A + 23 00 02 C2 20 + 23 00 02 C3 24 + 23 00 02 C4 23 + 23 00 02 C5 29 + 23 00 02 C6 23 + 23 00 02 C7 1C + 23 00 02 C8 19 + 23 00 02 C9 17 + 23 00 02 CA 17 + 23 00 02 CB 18 + 23 00 02 CC 1A + 23 00 02 CD 1E + 23 00 02 CE 20 + 23 00 02 CF 23 + 23 00 02 D0 07 + 23 00 02 D1 00 + 23 00 02 D2 00 + 23 00 02 D3 0A + 23 00 02 D4 13 + 23 00 02 D5 1C + 23 00 02 D6 1A + 23 00 02 D7 13 + 23 00 02 D8 17 + 23 00 02 D9 1C + 23 00 02 DA 19 + 23 00 02 DB 17 + 23 00 02 DC 17 + 23 00 02 DD 18 + 23 00 02 DE 1A + 23 00 02 DF 1E + 23 00 02 E0 20 + 23 00 02 E1 23 + 23 00 02 E2 07 + + 05 78 01 11 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 dc 01 28 + 05 78 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <160000000>; + hactive = <1200>; + vactive = <1920>; + + hsync-len = <1>;//19 + hback-porch = <60>;//40 + hfront-porch = <80>;//123 + + vsync-len = <1>; + vback-porch = <25>; + vfront-porch = <35>; + + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <1>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; + +}; + +&dsi0_in_vp0 { + status = "okay"; +}; + +&dsi0_in_vp1 { + status = "disabled"; +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + status = "okay"; +}; + +&hdmi_in_vp0 { + status = "okay"; +}; + +&hdmi_in_vp1 { + status = "disabled"; +}; + +&hdmi_sound { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vccsys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <0>; + /* not save the PMIC_POWER_EN register in uboot */ + not-save-power-en = <1>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vccsys>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_gpu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca1v8_pmu: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_acodec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v8: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc2v8_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3400 3513 3578 3687 3734 3752 3763 + 3766 3771 3784 3804 3836 3885 3925 + 3962 4005 4063 4114 4169 4227 4303>; + design_capacity = <5000>; + design_qmax = <5500>; + bat_res = <100>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3350>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4300>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&usb2phy0>; + gate_function_disable = <1>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + hp-volume = <20>; + spk-volume = <3>; + out-l2spk-r2hp; + spk-ctl-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + }; +}; + +&i2c2 { + status = "okay"; + pinctrl-0 = <&i2c2m1_xfer>; + + dw9714: dw9714@c { + compatible = "dongwoon,dw9714"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,vcm-start-current = <10>; + rockchip,vcm-rated-current = <85>; + rockchip,vcm-step-mode = <5>; + rockchip,camera-module-facing = "back"; + }; + + gc2385: gc2385@37 { + compatible = "galaxycore,gc2385"; + status = "okay"; + reg = <0x37>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clk>; + + //reset pin control by hardware,used this pin switch to mipi input + //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera + reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 10 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "HS5885-BNSM1018-V01"; + rockchip,camera-module-lens-name = "default"; + port { + gc2385_out: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1>; + }; + }; + }; + + ov8858: ov8858@36 { + status = "okay"; + compatible = "ovti,ov8858"; + reg = <0x36>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "rockchip,camera_default", "rockchip,camera_sleep"; + pinctrl-0 = <&cam_clkout0>; + pinctrl-1 = <&cam_sleep>; + //reset pin control by hardware,used this pin switch to mipi input + //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera + reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 11 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "HS5885-BNSM1018-V01"; + rockchip,camera-module-lens-name = "default"; + flash-leds = <&flash_rgb13h>; + lens-focus = <&dw9714>; + port { + ov8858_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&i2c3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m1_xfer>; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <138>; + i2c-scl-falling-time-ns = <4>; + + ts@40 { + compatible = "gslX680-pad"; + reg = <0x40>; + touch-gpio = <&gpio3 RK_PB0 IRQ_TYPE_LEVEL_HIGH>; + reset-gpio = <&gpio3 RK_PB1 IRQ_TYPE_LEVEL_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&tp_gpio>; + screen_max_x = <1200>; + screen_max_y = <1920>; + revert_x = <0>; + revert_y = <1>; + revert_xy = <0>; + chip_id = <1>; + status = "okay"; + }; +}; + +&i2c5 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <144>; + i2c-scl-falling-time-ns = <4>; + + sensor@18 { + compatible = "gs_sc7a20"; + reg = <0x18>; + type = ; + irq_enable = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&sensor_gpio>; + irq-gpio = <&gpio3 RK_PA2 IRQ_TYPE_EDGE_RISING>; + poll_delay_ms = <10>; + layout = <1>; + }; +}; + +&i2s0_8ch { + status = "okay"; +}; + +&i2s1_8ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&video_phy0 { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pinctrl { + cam { + cam_clkout0: cam-clkout0 { + rockchip,pins = + /* cam_clkout0 */ + <4 RK_PA7 1 &pcfg_pull_none>; + }; + + cam_sleep: cam-sleep { + rockchip,pins = + /* cam_sleep */ + <4 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + camera_rst: camera-rst { + rockchip,pins = + /* front camera reset */ + <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, + /* back camra reset */ + <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + flash_led_gpios: flash-led { + rockchip,pins = + /* flash led enable */ + <4 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + tp { + tp_gpio: tp-gpio { + rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + lcd { + lcd_rst_gpio: lcd-rst-gpio { + rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lcd_enable_gpio: lcd-enable-gpio { + rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lcd_stanby_gpio: lcd-stanby-gpio { + rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA2 2 &pcfg_pull_none>; + }; + }; + + sensor { + sensor_gpio: sensor-gpio { + rockchip,pins = <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + vcc_sd { + vcc_sd_h: vcc-sd-h { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcc3v3_pmu>; + vccio1-supply = <&vccio_acodec>; + vccio3-supply = <&vccio_sd>; + vccio4-supply = <&vcca1v8_pmu>; + vccio5-supply = <&vcc_1v8>; + vccio6-supply = <&vcc1v8_dvp>; + vccio7-supply = <&vcc_3v3>; +}; + +&pwm4 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy0_out>; + }; + }; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvenc { + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&route_dsi0 { + status = "okay"; +}; + +&route_hdmi { + status = "okay"; + connect = <&vp0_out_hdmi>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + max-frequency = <200000000>; + status = "okay"; +}; + +&sdmmc0 { + max-frequency = <150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + sd-uhs-sdr104; + vmmc-supply = <&vcc_sd>; + vqmmc-supply = <&vccio_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; + status = "okay"; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + rockchip,default-sample-phase = <90>; + status = "okay"; +}; + +&tsadc { + status = "okay"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet.dts b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet.dts new file mode 100755 index 000000000000..40022eb6c646 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-rk817-tablet.dts @@ -0,0 +1,1213 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include +#include +#include +#include "rk3566.dtsi" +#include "rk3568-android.dtsi" + +/ { + model = "Rockchip RK3566 RK817 TABLET LP4X Board"; + compatible = "rockchip,rk3566-rk817-tablet", "rockchip,rk3566"; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <297500>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm4 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 50 51 52 53 54 + 55 55 56 57 58 59 60 61 + 62 63 64 64 65 65 66 67 + 68 69 70 71 71 72 73 74 + 75 76 77 78 79 79 80 81 + 82 83 84 85 86 86 87 88 + 89 90 91 92 93 94 94 95 + 96 97 98 99 100 101 101 102 + 103 104 105 106 107 107 108 109 + 110 111 112 113 114 115 115 116 + 117 118 119 120 121 122 123 123 + 124 125 126 127 128 129 130 130 + 131 132 133 134 135 136 136 137 + 138 139 140 141 142 143 143 144 + 145 146 147 147 148 149 150 151 + 152 153 154 155 156 156 157 158 + 159 157 158 159 160 161 162 162 + 163 164 165 166 167 168 169 169 + 170 171 172 173 174 175 175 176 + 177 178 179 180 181 182 182 183 + 184 185 186 187 188 189 190 190 + 191 192 193 194 195 196 197 197 + 198 199 200 201 202 203 204 204 + 205 206 207 208 209 209 210 211 + 212 213 213 214 214 215 215 216 + 216 217 217 218 218 219 219 220 + >; + default-brightness-level = <200>; + }; + + charge-animation { + compatible = "rockchip,uboot-charge"; + rockchip,uboot-charge-on = <1>; + rockchip,android-charge-on = <0>; + rockchip,uboot-low-power-voltage = <3350>; + rockchip,screen-on-voltage = <3400>; + status = "okay"; + }; + + flash_rgb13h: flash-rgb13h { + status = "okay"; + compatible = "led,rgb13h"; + label = "gpio-flash"; + pinctrl-names = "default"; + pinctrl-0 = <&flash_led_gpios>; + led-max-microamp = <20000>; + flash-max-microamp = <20000>; + flash-max-timeout-us = <1000000>; + enable-gpio = <&gpio4 6 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + hdmi_sound: hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip,hdmi"; + status = "okay"; + + simple-audio-card,cpu { + sound-dai = <&i2s0_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + vccsys: vccsys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v8_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio4 RK_PB1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_rst>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; + + rk817-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk817-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk817_codec>; + }; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio4 RK_PC4 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 2>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_LOW>; + }; + + vcc_sd: vcc-sd { + compatible = "regulator-gpio"; + enable-active-low; + enable-gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc_sd_h>; + regulator-name = "vcc_sd"; + states = <3300000 0x0 + 3300000 0x1>; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6255"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&bus_npu { + bus-supply = <&vdd_logic>; + pvtm-supply = <&vdd_cpu>; + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@0 { + reg = <0>; + remote-endpoint = <&gc2385_out>; + data-lanes = <1>; + }; + mipi_in_ucam1: endpoint@1 { + reg = <1>; + remote-endpoint = <&ov8858_out>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&dsi0 { + status = "okay"; + rockchip,lane-rate = <1000>; + panel@0 { + compatible = "aoly,sl008pa21y1285-b00", "simple-panel-dsi"; + reg = <0>; + + backlight = <&backlight>; + //power-supply=<&vcc_3v3>; + enable-gpios = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_LOW>; + + pinctrl-names = "default"; + pinctrl-0 = <&lcd_enable_gpio>, <&lcd_rst_gpio>; + + prepare-delay-ms = <120>; + reset-delay-ms = <120>; + init-delay-ms = <120>; + stbyb-delay-ms = <120>; + enable-delay-ms = <120>; + disable-delay-ms = <120>; + unprepare-delay-ms = <120>; + + width-mm = <229>; + height-mm = <143>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + + panel-init-sequence = [ + 23 00 02 B0 01 + 23 00 02 C3 0F + 23 00 02 C4 00 + 23 00 02 C5 00 + 23 00 02 C6 00 + 23 00 02 C7 00 + 23 00 02 C8 0D + 23 00 02 C9 12 + 23 00 02 CA 11 + 23 00 02 CD 1D + 23 00 02 CE 1B + 23 00 02 CF 0B + 23 00 02 D0 09 + 23 00 02 D1 07 + 23 00 02 D2 05 + 23 00 02 D3 01 + 23 00 02 D7 10 + 23 00 02 D8 00 + 23 00 02 D9 00 + 23 00 02 DA 00 + 23 00 02 DB 00 + 23 00 02 DC 0E + 23 00 02 DD 12 + 23 00 02 DE 11 + 23 00 02 E1 1E + 23 00 02 E2 1C + 23 00 02 E3 0C + 23 00 02 E4 0A + 23 00 02 E5 08 + 23 00 02 E6 06 + 23 00 02 E7 02 + 23 00 02 B0 03 + 23 00 02 BE 03 + 23 00 02 CC 44 + 23 00 02 C8 07 + 23 00 02 C9 05 + 23 00 02 CA 42 + 23 00 02 CD 3E + 23 00 02 CF 60 + 23 00 02 D2 04 + 23 00 02 D3 04 + 23 00 02 D4 01 + 23 00 02 D5 00 + 23 00 02 D6 03 + 23 00 02 D7 04 + 23 00 02 D9 01 + 23 00 02 DB 01 + 23 00 02 E4 F0 + 23 00 02 E5 0A + 23 00 02 B0 00 + 23 00 02 BA 8F// NEW ADD + 23 00 02 BD 63 + 23 00 02 C2 08 + 23 00 02 C4 10 + 23 00 02 B0 02 + 23 00 02 C0 00 + 23 00 02 C1 0A + 23 00 02 C2 20 + 23 00 02 C3 24 + 23 00 02 C4 23 + 23 00 02 C5 29 + 23 00 02 C6 23 + 23 00 02 C7 1C + 23 00 02 C8 19 + 23 00 02 C9 17 + 23 00 02 CA 17 + 23 00 02 CB 18 + 23 00 02 CC 1A + 23 00 02 CD 1E + 23 00 02 CE 20 + 23 00 02 CF 23 + 23 00 02 D0 07 + 23 00 02 D1 00 + 23 00 02 D2 00 + 23 00 02 D3 0A + 23 00 02 D4 13 + 23 00 02 D5 1C + 23 00 02 D6 1A + 23 00 02 D7 13 + 23 00 02 D8 17 + 23 00 02 D9 1C + 23 00 02 DA 19 + 23 00 02 DB 17 + 23 00 02 DC 17 + 23 00 02 DD 18 + 23 00 02 DE 1A + 23 00 02 DF 1E + 23 00 02 E0 20 + 23 00 02 E1 23 + 23 00 02 E2 07 + + 05 78 01 11 + 05 32 01 29 + ]; + + panel-exit-sequence = [ + 05 dc 01 28 + 05 78 01 10 + ]; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <160000000>; + hactive = <1200>; + vactive = <1920>; + + hsync-len = <1>;//19 + hback-porch = <60>;//40 + hfront-porch = <80>;//123 + + vsync-len = <1>; + vback-porch = <25>; + vfront-porch = <35>; + + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <1>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; + +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + status = "okay"; +}; + +&hdmi_in_vp0 { + status = "okay"; +}; + +&hdmi_in_vp1 { + status = "disabled"; +}; + +&hdmi_sound { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vccsys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <0>; + /* not save the PMIC_POWER_EN register in uboot */ + not-save-power-en = <1>; + + vcc1-supply = <&vccsys>; + vcc2-supply = <&vccsys>; + vcc3-supply = <&vccsys>; + vcc4-supply = <&vccsys>; + vcc5-supply = <&vccsys>; + vcc6-supply = <&vccsys>; + vcc7-supply = <&vccsys>; + vcc8-supply = <&vccsys>; + vcc9-supply = <&dcdc_boost>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_gpu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca1v8_pmu: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_acodec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc_1v8: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc2v8_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-name = "boost"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + battery { + compatible = "rk817,battery"; + ocv_table = <3400 3513 3578 3687 3734 3752 3763 + 3766 3771 3784 3804 3836 3885 3925 + 3962 4005 4063 4114 4169 4227 4303>; + design_capacity = <5000>; + design_qmax = <5500>; + bat_res = <100>; + sleep_enter_current = <150>; + sleep_exit_current = <180>; + sleep_filter_current = <100>; + power_off_thresd = <3350>; + zero_algorithm_vol = <3850>; + max_soc_offset = <60>; + monitor_sec = <5>; + sample_res = <10>; + virtual_power = <0>; + }; + + charger { + compatible = "rk817,charger"; + min_input_voltage = <4500>; + max_input_current = <1500>; + max_chrg_current = <2000>; + max_chrg_voltage = <4300>; + chrg_term_mode = <0>; + chrg_finish_cur = <300>; + virtual_power = <0>; + dc_det_adc = <0>; + extcon = <&usb2phy0>; + }; + + rk817_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + hp-volume = <20>; + spk-volume = <3>; + out-l2spk-r2hp; + spk-ctl-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + }; +}; + +&i2c2 { + status = "okay"; + pinctrl-0 = <&i2c2m1_xfer>; + + dw9714: dw9714@c { + compatible = "dongwoon,dw9714"; + status = "okay"; + reg = <0x0c>; + rockchip,camera-module-index = <0>; + rockchip,vcm-start-current = <10>; + rockchip,vcm-rated-current = <85>; + rockchip,vcm-step-mode = <5>; + rockchip,camera-module-facing = "back"; + }; + + gc2385: gc2385@37 { + compatible = "galaxycore,gc2385"; + status = "okay"; + reg = <0x37>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "rockchip,camera_default"; + pinctrl-0 = <&cif_clk>; + + //reset pin control by hardware,used this pin switch to mipi input + //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera + reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 10 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "front"; + rockchip,camera-module-name = "HS5885-BNSM1018-V01"; + rockchip,camera-module-lens-name = "default"; + port { + gc2385_out: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1>; + }; + }; + }; + + ov8858: ov8858@36 { + status = "okay"; + compatible = "ovti,ov8858"; + reg = <0x36>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "rockchip,camera_default", "rockchip,camera_sleep"; + pinctrl-0 = <&cam_clkout0>; + pinctrl-1 = <&cam_sleep>; + //reset pin control by hardware,used this pin switch to mipi input + //1->2LANE(LANE 0&1) FRONT camera, 0->4LANE REAR camera + reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 11 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "HS5885-BNSM1018-V01"; + rockchip,camera-module-lens-name = "default"; + flash-leds = <&flash_rgb13h>; + lens-focus = <&dw9714>; + port { + ov8858_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&i2c3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m1_xfer>; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <138>; + i2c-scl-falling-time-ns = <4>; + + ts@40 { + compatible = "gslX680-pad"; + reg = <0x40>; + touch-gpio = <&gpio3 RK_PB0 IRQ_TYPE_LEVEL_HIGH>; + reset-gpio = <&gpio3 RK_PB1 IRQ_TYPE_LEVEL_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&tp_gpio>; + screen_max_x = <1200>; + screen_max_y = <1920>; + revert_x = <0>; + revert_y = <1>; + revert_xy = <0>; + chip_id = <1>; + status = "okay"; + }; +}; + +&i2c5 { + status = "okay"; + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <144>; + i2c-scl-falling-time-ns = <4>; + + sensor@18 { + compatible = "gs_sc7a20"; + reg = <0x18>; + type = ; + irq_enable = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&sensor_gpio>; + irq-gpio = <&gpio3 RK_PA2 IRQ_TYPE_EDGE_RISING>; + poll_delay_ms = <10>; + layout = <1>; + }; +}; + +&i2s0_8ch { + status = "okay"; +}; + +&i2s1_8ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&video_phy0 { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&nandc0 { + status = "okay"; +}; + +&pinctrl { + cam { + cam_clkout0: cam-clkout0 { + rockchip,pins = + /* cam_clkout0 */ + <4 RK_PA7 1 &pcfg_pull_none>; + }; + + cam_sleep: cam-sleep { + rockchip,pins = + /* cam_sleep */ + <4 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + camera_rst: camera-rst { + rockchip,pins = + /* front camera reset */ + <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, + /* back camra reset */ + <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + flash_led_gpios: flash-led { + rockchip,pins = + /* flash led enable */ + <4 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + tp { + tp_gpio: tp-gpio { + rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + lcd { + lcd_rst_gpio: lcd-rst-gpio { + rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + lcd_enable_gpio: lcd-enable-gpio { + rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA2 2 &pcfg_pull_none>; + }; + }; + + sensor { + sensor_gpio: sensor-gpio { + rockchip,pins = <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + vcc_sd { + vcc_sd_h: vcc-sd-h { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcc_3v3>; + vccio1-supply = <&vccio_acodec>; + vccio3-supply = <&vccio_sd>; + vccio4-supply = <&vcc_3v3>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc_3v3>; + vccio7-supply = <&vcc_3v3>; +}; + +&pwm4 { + status = "okay"; +}; + +&rk_rga { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy0_out>; + }; + }; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvenc { + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&route_dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&route_hdmi { + status = "okay"; + connect = <&vp0_out_hdmi>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + max-frequency = <200000000>; + status = "okay"; +}; + +&sdmmc0 { + max-frequency = <150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + sd-uhs-sdr104; + vmmc-supply = <&vcc_sd>; + vqmmc-supply = <&vccio_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; + status = "okay"; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + rockchip,default-sample-phase = <90>; + status = "okay"; +}; + +&tsadc { + status = "okay"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usbdrd_dwc3 { + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>; + assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566.dtsi b/arch/arm64/boot/dts/rockchip/rk3566.dtsi new file mode 100755 index 000000000000..01b6499b678d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566.dtsi @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +#include "rk3568.dtsi" + +/ { + aliases { + /delete-property/ ethernet0; + }; +}; + +&cpu0_opp_table { + /delete-node/ opp-1992000000; +}; + +&power { + pd_pipe@RK3568_PD_PIPE { + reg = ; + clocks = <&cru PCLK_PIPE>; + pm_qos = <&qos_pcie2x1>, + <&qos_sata1>, + <&qos_sata2>, + <&qos_usb3_0>, + <&qos_usb3_1>; + }; +}; + +&rkisp { + rockchip,iq-feature = /bits/ 64 <0x3FBF7FE67FF>; +}; + +&usbdrd_dwc3 { + phys = <&u2phy0_otg>; + phy-names = "usb2-phy"; + extcon = <&usb2phy0>; + maximum-speed = "high-speed"; + snps,dis_u2_susphy_quirk; +}; + +/delete-node/ &combphy0_us; +/delete-node/ &gmac0_clkin; +/delete-node/ &gmac0_xpcsclk; +/delete-node/ &gmac0; +/delete-node/ &pcie30_phy_grf; +/delete-node/ &pcie30phy; +/delete-node/ &pcie3x1; +/delete-node/ &pcie3x2; +/delete-node/ &qos_pcie3x1; +/delete-node/ &qos_pcie3x2; +/delete-node/ &qos_sata0; +/delete-node/ &sata0; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-android.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-android.dtsi new file mode 100755 index 000000000000..01cd37c04e1a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-android.dtsi @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/ { + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0"; + }; + + aliases { + mmc0 = &sdmmc0; + mmc1 = &sdmmc1; + mmc2 = &sdhci; + mmc3 = &sdmmc2; + }; + + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <1>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "okay"; + }; + + debug: debug@fd904000 { + compatible = "rockchip,debug"; + reg = <0x0 0xfd904000 0x0 0x1000>, + <0x0 0xfd905000 0x0 0x1000>, + <0x0 0xfd906000 0x0 0x1000>, + <0x0 0xfd907000 0x0 0x1000>; + }; + + cspmu: cspmu@fd90c000 { + compatible = "rockchip,cspmu"; + reg = <0x0 0xfd90c000 0x0 0x1000>, + <0x0 0xfd90d000 0x0 0x1000>, + <0x0 0xfd90e000 0x0 0x1000>, + <0x0 0xfd90f000 0x0 0x1000>; + }; +}; + +&reserved_memory { + linux,cma { + compatible = "shared-dma-pool"; + inactive; + reusable; + reg = <0x0 0x10000000 0x0 0x00800000>; + linux,cma-default; + }; + + ramoops: ramoops@110000 { + compatible = "ramoops"; + reg = <0x0 0x110000 0x0 0xf0000>; + record-size = <0x20000>; + console-size = <0x80000>; + ftrace-size = <0x00000>; + pmsg-size = <0x50000>; + }; +}; + +&rng { + status = "okay"; +}; + +&vop { + support-multi-area; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-dram-default-timing.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-dram-default-timing.dtsi new file mode 100755 index 000000000000..43f978809c02 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-dram-default-timing.dtsi @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include +#include + +/ { + ddr_timing: ddr_timing { + compatible = "rockchip,ddr-timing"; + ddr2_speed_bin = ; + ddr3_speed_bin = ; + ddr4_speed_bin = ; + pd_idle = <13>; + sr_idle = <93>; + sr_mc_gate_idle = <0>; + srpd_lite_idle = <0>; + standby_idle = <0>; + + auto_pd_dis_freq = <1066>; + auto_sr_dis_freq = <800>; + ddr2_dll_dis_freq = <300>; + ddr3_dll_dis_freq = <300>; + ddr4_dll_dis_freq = <625>; + phy_dll_dis_freq = <400>; + + ddr2_odt_dis_freq = <100>; + phy_ddr2_odt_dis_freq = <100>; + ddr2_drv = ; + ddr2_odt = ; + phy_ddr2_ca_drv = ; + phy_ddr2_ck_drv = ; + phy_ddr2_dq_drv = ; + phy_ddr2_odt = ; + + ddr3_odt_dis_freq = <333>; + phy_ddr3_odt_dis_freq = <333>; + ddr3_drv = ; + ddr3_odt = ; + phy_ddr3_ca_drv = ; + phy_ddr3_ck_drv = ; + phy_ddr3_dq_drv = ; + phy_ddr3_odt = ; + + phy_lpddr2_odt_dis_freq = <333>; + lpddr2_drv = ; + phy_lpddr2_ca_drv = ; + phy_lpddr2_ck_drv = ; + phy_lpddr2_dq_drv = ; + phy_lpddr2_odt = ; + + lpddr3_odt_dis_freq = <333>; + phy_lpddr3_odt_dis_freq = <333>; + lpddr3_drv = ; + lpddr3_odt = ; + phy_lpddr3_ca_drv = ; + phy_lpddr3_ck_drv = ; + phy_lpddr3_dq_drv = ; + phy_lpddr3_odt = ; + + lpddr4_odt_dis_freq = <333>; + phy_lpddr4_odt_dis_freq = <333>; + lpddr4_drv = ; + lpddr4_dq_odt = ; + lpddr4_ca_odt = ; + phy_lpddr4_ca_drv = ; + phy_lpddr4_ck_cs_drv = ; + phy_lpddr4_dq_drv = ; + phy_lpddr4_odt = ; + + ddr4_odt_dis_freq = <625>; + phy_ddr4_odt_dis_freq = <625>; + ddr4_drv = ; + ddr4_odt = ; + phy_ddr4_ca_drv = ; + phy_ddr4_ck_drv = ; + phy_ddr4_dq_drv = ; + phy_ddr4_odt = ; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi new file mode 100755 index 000000000000..2ca4a18314d2 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi @@ -0,0 +1,1807 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include +#include +#include +#include +#include +#include + +/ { + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <297500>; + }; + + menu-key { + label = "menu"; + linux,code = ; + press-threshold-microvolt = <980000>; + }; + + back-key { + label = "back"; + linux,code = ; + press-threshold-microvolt = <1305500>; + }; + }; + + audiopwmout_diff: audiopwmout-diff { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,audiopwmout-diff"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,bitclock-master = <&master>; + simple-audio-card,frame-master = <&master>; + simple-audio-card,cpu { + sound-dai = <&i2s3_2ch>; + }; + master: simple-audio-card,codec { + sound-dai = <&dig_acodec>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm4 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + }; + + backlight1: backlight1 { + compatible = "pwm-backlight"; + pwms = <&pwm5 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + }; + + dc_12v: dc-12v { + compatible = "regulator-fixed"; + regulator-name = "dc_12v"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + }; + + hdmi_sound: hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip,hdmi"; + status = "disabled"; + + simple-audio-card,cpu { + sound-dai = <&i2s0_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + leds: leds { + compatible = "gpio-leds"; + work_led: work { + gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + }; + }; + + pdmics: dummy-codec { + status = "disabled"; + compatible = "rockchip,dummy-codec"; + #sound-dai-cells = <0>; + }; + + pdm_mic_array: pdm-mic-array { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,pdm-mic-array"; + simple-audio-card,cpu { + sound-dai = <&pdm>; + }; + simple-audio-card,codec { + sound-dai = <&pdmics>; + }; + }; + + rk809_sound: rk809-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + spdif-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "okay"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + vad_sound: vad-sound { + status = "disabled"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3568-vad"; + rockchip,cpu = <&i2s1_8ch>; + rockchip,codec = <&rk809_codec>, <&vad>; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&dc_12v>; + }; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&dc_12v>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_host_en>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vcc5v0_otg: vcc5v0-otg-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_otg_en>; + regulator-name = "vcc5v0_otg"; + }; + + vcc3v3_lcd0_n: vcc3v3-lcd0-n { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_lcd0_n"; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_lcd1_n: vcc3v3-lcd1-n { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_lcd1_n"; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; + }; + + wireless_wlan: wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6398s"; + status = "okay"; + }; + + wireless_bluetooth: wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart8m0_rtsn>; + pinctrl-1 = <&uart8_gpios>; + BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + test-power { + status = "okay"; + }; +}; + +&bus_npu { + bus-supply = <&vdd_logic>; + pvtm-supply = <&vdd_cpu>; + status = "okay"; +}; + +&can0 { + assigned-clocks = <&cru CLK_CAN0>; + assigned-clock-rates = <150000000>; + pinctrl-names = "default"; + pinctrl-0 = <&can0m1_pins>; + status = "disabled"; +}; + +&can1 { + assigned-clocks = <&cru CLK_CAN1>; + assigned-clock-rates = <150000000>; + pinctrl-names = "default"; + pinctrl-0 = <&can1m1_pins>; + status = "disabled"; +}; + +&can2 { + assigned-clocks = <&cru CLK_CAN2>; + assigned-clock-rates = <150000000>; + pinctrl-names = "default"; + pinctrl-0 = <&can2m1_pins>; + status = "disabled"; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&dsi0 { + status = "disabled"; + //rockchip,lane-rate = <1000>; + dsi0_panel: panel@0 { + status = "okay"; + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + reset-delay-ms = <60>; + enable-delay-ms = <60>; + prepare-delay-ms = <60>; + unprepare-delay-ms = <60>; + disable-delay-ms = <60>; + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + panel-init-sequence = [ + 23 00 02 FE 21 + 23 00 02 04 00 + 23 00 02 00 64 + 23 00 02 2A 00 + 23 00 02 26 64 + 23 00 02 54 00 + 23 00 02 50 64 + 23 00 02 7B 00 + 23 00 02 77 64 + 23 00 02 A2 00 + 23 00 02 9D 64 + 23 00 02 C9 00 + 23 00 02 C5 64 + 23 00 02 01 71 + 23 00 02 27 71 + 23 00 02 51 71 + 23 00 02 78 71 + 23 00 02 9E 71 + 23 00 02 C6 71 + 23 00 02 02 89 + 23 00 02 28 89 + 23 00 02 52 89 + 23 00 02 79 89 + 23 00 02 9F 89 + 23 00 02 C7 89 + 23 00 02 03 9E + 23 00 02 29 9E + 23 00 02 53 9E + 23 00 02 7A 9E + 23 00 02 A0 9E + 23 00 02 C8 9E + 23 00 02 09 00 + 23 00 02 05 B0 + 23 00 02 31 00 + 23 00 02 2B B0 + 23 00 02 5A 00 + 23 00 02 55 B0 + 23 00 02 80 00 + 23 00 02 7C B0 + 23 00 02 A7 00 + 23 00 02 A3 B0 + 23 00 02 CE 00 + 23 00 02 CA B0 + 23 00 02 06 C0 + 23 00 02 2D C0 + 23 00 02 56 C0 + 23 00 02 7D C0 + 23 00 02 A4 C0 + 23 00 02 CB C0 + 23 00 02 07 CF + 23 00 02 2F CF + 23 00 02 58 CF + 23 00 02 7E CF + 23 00 02 A5 CF + 23 00 02 CC CF + 23 00 02 08 DD + 23 00 02 30 DD + 23 00 02 59 DD + 23 00 02 7F DD + 23 00 02 A6 DD + 23 00 02 CD DD + 23 00 02 0E 15 + 23 00 02 0A E9 + 23 00 02 36 15 + 23 00 02 32 E9 + 23 00 02 5F 15 + 23 00 02 5B E9 + 23 00 02 85 15 + 23 00 02 81 E9 + 23 00 02 AD 15 + 23 00 02 A9 E9 + 23 00 02 D3 15 + 23 00 02 CF E9 + 23 00 02 0B 14 + 23 00 02 33 14 + 23 00 02 5C 14 + 23 00 02 82 14 + 23 00 02 AA 14 + 23 00 02 D0 14 + 23 00 02 0C 36 + 23 00 02 34 36 + 23 00 02 5D 36 + 23 00 02 83 36 + 23 00 02 AB 36 + 23 00 02 D1 36 + 23 00 02 0D 6B + 23 00 02 35 6B + 23 00 02 5E 6B + 23 00 02 84 6B + 23 00 02 AC 6B + 23 00 02 D2 6B + 23 00 02 13 5A + 23 00 02 0F 94 + 23 00 02 3B 5A + 23 00 02 37 94 + 23 00 02 64 5A + 23 00 02 60 94 + 23 00 02 8A 5A + 23 00 02 86 94 + 23 00 02 B2 5A + 23 00 02 AE 94 + 23 00 02 D8 5A + 23 00 02 D4 94 + 23 00 02 10 D1 + 23 00 02 38 D1 + 23 00 02 61 D1 + 23 00 02 87 D1 + 23 00 02 AF D1 + 23 00 02 D5 D1 + 23 00 02 11 04 + 23 00 02 39 04 + 23 00 02 62 04 + 23 00 02 88 04 + 23 00 02 B0 04 + 23 00 02 D6 04 + 23 00 02 12 05 + 23 00 02 3A 05 + 23 00 02 63 05 + 23 00 02 89 05 + 23 00 02 B1 05 + 23 00 02 D7 05 + 23 00 02 18 AA + 23 00 02 14 36 + 23 00 02 42 AA + 23 00 02 3D 36 + 23 00 02 69 AA + 23 00 02 65 36 + 23 00 02 8F AA + 23 00 02 8B 36 + 23 00 02 B7 AA + 23 00 02 B3 36 + 23 00 02 DD AA + 23 00 02 D9 36 + 23 00 02 15 74 + 23 00 02 3F 74 + 23 00 02 66 74 + 23 00 02 8C 74 + 23 00 02 B4 74 + 23 00 02 DA 74 + 23 00 02 16 9F + 23 00 02 40 9F + 23 00 02 67 9F + 23 00 02 8D 9F + 23 00 02 B5 9F + 23 00 02 DB 9F + 23 00 02 17 DC + 23 00 02 41 DC + 23 00 02 68 DC + 23 00 02 8E DC + 23 00 02 B6 DC + 23 00 02 DC DC + 23 00 02 1D FF + 23 00 02 19 03 + 23 00 02 47 FF + 23 00 02 43 03 + 23 00 02 6E FF + 23 00 02 6A 03 + 23 00 02 94 FF + 23 00 02 90 03 + 23 00 02 BC FF + 23 00 02 B8 03 + 23 00 02 E2 FF + 23 00 02 DE 03 + 23 00 02 1A 35 + 23 00 02 44 35 + 23 00 02 6B 35 + 23 00 02 91 35 + 23 00 02 B9 35 + 23 00 02 DF 35 + 23 00 02 1B 45 + 23 00 02 45 45 + 23 00 02 6C 45 + 23 00 02 92 45 + 23 00 02 BA 45 + 23 00 02 E0 45 + 23 00 02 1C 55 + 23 00 02 46 55 + 23 00 02 6D 55 + 23 00 02 93 55 + 23 00 02 BB 55 + 23 00 02 E1 55 + 23 00 02 22 FF + 23 00 02 1E 68 + 23 00 02 4C FF + 23 00 02 48 68 + 23 00 02 73 FF + 23 00 02 6F 68 + 23 00 02 99 FF + 23 00 02 95 68 + 23 00 02 C1 FF + 23 00 02 BD 68 + 23 00 02 E7 FF + 23 00 02 E3 68 + 23 00 02 1F 7E + 23 00 02 49 7E + 23 00 02 70 7E + 23 00 02 96 7E + 23 00 02 BE 7E + 23 00 02 E4 7E + 23 00 02 20 97 + 23 00 02 4A 97 + 23 00 02 71 97 + 23 00 02 97 97 + 23 00 02 BF 97 + 23 00 02 E5 97 + 23 00 02 21 B5 + 23 00 02 4B B5 + 23 00 02 72 B5 + 23 00 02 98 B5 + 23 00 02 C0 B5 + 23 00 02 E6 B5 + 23 00 02 25 F0 + 23 00 02 23 E8 + 23 00 02 4F F0 + 23 00 02 4D E8 + 23 00 02 76 F0 + 23 00 02 74 E8 + 23 00 02 9C F0 + 23 00 02 9A E8 + 23 00 02 C4 F0 + 23 00 02 C2 E8 + 23 00 02 EA F0 + 23 00 02 E8 E8 + 23 00 02 24 FF + 23 00 02 4E FF + 23 00 02 75 FF + 23 00 02 9B FF + 23 00 02 C3 FF + 23 00 02 E9 FF + 23 00 02 FE 3D + 23 00 02 00 04 + 23 00 02 FE 23 + 23 00 02 08 82 + 23 00 02 0A 00 + 23 00 02 0B 00 + 23 00 02 0C 01 + 23 00 02 16 00 + 23 00 02 18 02 + 23 00 02 1B 04 + 23 00 02 19 04 + 23 00 02 1C 81 + 23 00 02 1F 00 + 23 00 02 20 03 + 23 00 02 23 04 + 23 00 02 21 01 + 23 00 02 54 63 + 23 00 02 55 54 + 23 00 02 6E 45 + 23 00 02 6D 36 + 23 00 02 FE 3D + 23 00 02 55 78 + 23 00 02 FE 20 + 23 00 02 26 30 + 23 00 02 FE 3D + 23 00 02 20 71 + 23 00 02 50 8F + 23 00 02 51 8F + 23 00 02 FE 00 + 23 00 02 35 00 + 05 78 01 11 + 05 1E 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + disp_timings0: display-timings { + native-mode = <&dsi0_timing0>; + dsi0_timing0: timing0 { + clock-frequency = <132000000>; + hactive = <1080>; + vactive = <1920>; + hfront-porch = <15>; + hsync-len = <2>; + hback-porch = <30>; + vfront-porch = <15>; + vsync-len = <2>; + vback-porch = <15>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <1>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; + +}; + +&dsi1 { + status = "disabled"; + //rockchip,lane-rate = <1000>; + dsi1_panel: panel@0 { + status = "okay"; + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight1>; + reset-delay-ms = <60>; + enable-delay-ms = <60>; + prepare-delay-ms = <60>; + unprepare-delay-ms = <60>; + disable-delay-ms = <60>; + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + panel-init-sequence = [ + 23 00 02 FE 21 + 23 00 02 04 00 + 23 00 02 00 64 + 23 00 02 2A 00 + 23 00 02 26 64 + 23 00 02 54 00 + 23 00 02 50 64 + 23 00 02 7B 00 + 23 00 02 77 64 + 23 00 02 A2 00 + 23 00 02 9D 64 + 23 00 02 C9 00 + 23 00 02 C5 64 + 23 00 02 01 71 + 23 00 02 27 71 + 23 00 02 51 71 + 23 00 02 78 71 + 23 00 02 9E 71 + 23 00 02 C6 71 + 23 00 02 02 89 + 23 00 02 28 89 + 23 00 02 52 89 + 23 00 02 79 89 + 23 00 02 9F 89 + 23 00 02 C7 89 + 23 00 02 03 9E + 23 00 02 29 9E + 23 00 02 53 9E + 23 00 02 7A 9E + 23 00 02 A0 9E + 23 00 02 C8 9E + 23 00 02 09 00 + 23 00 02 05 B0 + 23 00 02 31 00 + 23 00 02 2B B0 + 23 00 02 5A 00 + 23 00 02 55 B0 + 23 00 02 80 00 + 23 00 02 7C B0 + 23 00 02 A7 00 + 23 00 02 A3 B0 + 23 00 02 CE 00 + 23 00 02 CA B0 + 23 00 02 06 C0 + 23 00 02 2D C0 + 23 00 02 56 C0 + 23 00 02 7D C0 + 23 00 02 A4 C0 + 23 00 02 CB C0 + 23 00 02 07 CF + 23 00 02 2F CF + 23 00 02 58 CF + 23 00 02 7E CF + 23 00 02 A5 CF + 23 00 02 CC CF + 23 00 02 08 DD + 23 00 02 30 DD + 23 00 02 59 DD + 23 00 02 7F DD + 23 00 02 A6 DD + 23 00 02 CD DD + 23 00 02 0E 15 + 23 00 02 0A E9 + 23 00 02 36 15 + 23 00 02 32 E9 + 23 00 02 5F 15 + 23 00 02 5B E9 + 23 00 02 85 15 + 23 00 02 81 E9 + 23 00 02 AD 15 + 23 00 02 A9 E9 + 23 00 02 D3 15 + 23 00 02 CF E9 + 23 00 02 0B 14 + 23 00 02 33 14 + 23 00 02 5C 14 + 23 00 02 82 14 + 23 00 02 AA 14 + 23 00 02 D0 14 + 23 00 02 0C 36 + 23 00 02 34 36 + 23 00 02 5D 36 + 23 00 02 83 36 + 23 00 02 AB 36 + 23 00 02 D1 36 + 23 00 02 0D 6B + 23 00 02 35 6B + 23 00 02 5E 6B + 23 00 02 84 6B + 23 00 02 AC 6B + 23 00 02 D2 6B + 23 00 02 13 5A + 23 00 02 0F 94 + 23 00 02 3B 5A + 23 00 02 37 94 + 23 00 02 64 5A + 23 00 02 60 94 + 23 00 02 8A 5A + 23 00 02 86 94 + 23 00 02 B2 5A + 23 00 02 AE 94 + 23 00 02 D8 5A + 23 00 02 D4 94 + 23 00 02 10 D1 + 23 00 02 38 D1 + 23 00 02 61 D1 + 23 00 02 87 D1 + 23 00 02 AF D1 + 23 00 02 D5 D1 + 23 00 02 11 04 + 23 00 02 39 04 + 23 00 02 62 04 + 23 00 02 88 04 + 23 00 02 B0 04 + 23 00 02 D6 04 + 23 00 02 12 05 + 23 00 02 3A 05 + 23 00 02 63 05 + 23 00 02 89 05 + 23 00 02 B1 05 + 23 00 02 D7 05 + 23 00 02 18 AA + 23 00 02 14 36 + 23 00 02 42 AA + 23 00 02 3D 36 + 23 00 02 69 AA + 23 00 02 65 36 + 23 00 02 8F AA + 23 00 02 8B 36 + 23 00 02 B7 AA + 23 00 02 B3 36 + 23 00 02 DD AA + 23 00 02 D9 36 + 23 00 02 15 74 + 23 00 02 3F 74 + 23 00 02 66 74 + 23 00 02 8C 74 + 23 00 02 B4 74 + 23 00 02 DA 74 + 23 00 02 16 9F + 23 00 02 40 9F + 23 00 02 67 9F + 23 00 02 8D 9F + 23 00 02 B5 9F + 23 00 02 DB 9F + 23 00 02 17 DC + 23 00 02 41 DC + 23 00 02 68 DC + 23 00 02 8E DC + 23 00 02 B6 DC + 23 00 02 DC DC + 23 00 02 1D FF + 23 00 02 19 03 + 23 00 02 47 FF + 23 00 02 43 03 + 23 00 02 6E FF + 23 00 02 6A 03 + 23 00 02 94 FF + 23 00 02 90 03 + 23 00 02 BC FF + 23 00 02 B8 03 + 23 00 02 E2 FF + 23 00 02 DE 03 + 23 00 02 1A 35 + 23 00 02 44 35 + 23 00 02 6B 35 + 23 00 02 91 35 + 23 00 02 B9 35 + 23 00 02 DF 35 + 23 00 02 1B 45 + 23 00 02 45 45 + 23 00 02 6C 45 + 23 00 02 92 45 + 23 00 02 BA 45 + 23 00 02 E0 45 + 23 00 02 1C 55 + 23 00 02 46 55 + 23 00 02 6D 55 + 23 00 02 93 55 + 23 00 02 BB 55 + 23 00 02 E1 55 + 23 00 02 22 FF + 23 00 02 1E 68 + 23 00 02 4C FF + 23 00 02 48 68 + 23 00 02 73 FF + 23 00 02 6F 68 + 23 00 02 99 FF + 23 00 02 95 68 + 23 00 02 C1 FF + 23 00 02 BD 68 + 23 00 02 E7 FF + 23 00 02 E3 68 + 23 00 02 1F 7E + 23 00 02 49 7E + 23 00 02 70 7E + 23 00 02 96 7E + 23 00 02 BE 7E + 23 00 02 E4 7E + 23 00 02 20 97 + 23 00 02 4A 97 + 23 00 02 71 97 + 23 00 02 97 97 + 23 00 02 BF 97 + 23 00 02 E5 97 + 23 00 02 21 B5 + 23 00 02 4B B5 + 23 00 02 72 B5 + 23 00 02 98 B5 + 23 00 02 C0 B5 + 23 00 02 E6 B5 + 23 00 02 25 F0 + 23 00 02 23 E8 + 23 00 02 4F F0 + 23 00 02 4D E8 + 23 00 02 76 F0 + 23 00 02 74 E8 + 23 00 02 9C F0 + 23 00 02 9A E8 + 23 00 02 C4 F0 + 23 00 02 C2 E8 + 23 00 02 EA F0 + 23 00 02 E8 E8 + 23 00 02 24 FF + 23 00 02 4E FF + 23 00 02 75 FF + 23 00 02 9B FF + 23 00 02 C3 FF + 23 00 02 E9 FF + 23 00 02 FE 3D + 23 00 02 00 04 + 23 00 02 FE 23 + 23 00 02 08 82 + 23 00 02 0A 00 + 23 00 02 0B 00 + 23 00 02 0C 01 + 23 00 02 16 00 + 23 00 02 18 02 + 23 00 02 1B 04 + 23 00 02 19 04 + 23 00 02 1C 81 + 23 00 02 1F 00 + 23 00 02 20 03 + 23 00 02 23 04 + 23 00 02 21 01 + 23 00 02 54 63 + 23 00 02 55 54 + 23 00 02 6E 45 + 23 00 02 6D 36 + 23 00 02 FE 3D + 23 00 02 55 78 + 23 00 02 FE 20 + 23 00 02 26 30 + 23 00 02 FE 3D + 23 00 02 20 71 + 23 00 02 50 8F + 23 00 02 51 8F + 23 00 02 FE 00 + 23 00 02 35 00 + 05 78 01 11 + 05 1E 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + disp_timings1: display-timings { + native-mode = <&dsi1_timing0>; + dsi1_timing0: timing0 { + clock-frequency = <132000000>; + hactive = <1080>; + vactive = <1920>; + hfront-porch = <15>; + hsync-len = <2>; + hback-porch = <30>; + vfront-porch = <15>; + vsync-len = <2>; + vback-porch = <15>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <1>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi1: endpoint { + remote-endpoint = <&dsi1_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi1_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi1>; + }; + }; + }; + +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + status = "okay"; + rockchip,phy-table = + <92812500 0x8009 0x0000 0x0270>, + <165000000 0x800b 0x0000 0x026d>, + <185625000 0x800b 0x0000 0x01ed>, + <297000000 0x800b 0x0000 0x01ad>, + <594000000 0x8029 0x0000 0x0088>, + <000000000 0x0000 0x0000 0x0000>; +}; + +&hdmi_in_vp0 { + status = "okay"; +}; + +&hdmi_in_vp1 { + status = "disabled"; +}; + +&hdmi_sound { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; + + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <0>; + /* not save the PMIC_POWER_EN register in uboot */ + not-save-power-en = <1>; + + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_gpu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vdd_npu: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_npu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_image: LDO_REG1 { + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_image"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_acodec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca_1v8: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca1v8_pmu: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcca1v8_image: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_image"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v3: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_sd: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + pinctrl-names = "default"; + pinctrl-0 = <&touch_gpio>; + goodix,rst-gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +&i2c5 { + status = "okay"; + + mxc6655xa: mxc6655xa@15 { + status = "okay"; + compatible = "gs_mxc6655xa"; + pinctrl-names = "default"; + pinctrl-0 = <&mxc6655xa_irq_gpio>; + reg = <0x15>; + irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + power-off-in-suspend = <1>; + layout = <1>; + }; +}; + +&i2s0_8ch { + status = "okay"; +}; + +&i2s1_8ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&nandc0 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + nand@0 { + reg = <0>; + nand-bus-width = <8>; + nand-ecc-mode = "hw"; + nand-ecc-strength = <16>; + nand-ecc-step-size = <1024>; + }; +}; + +&pinctrl { + + mxc6655xa { + mxc6655xa_irq_gpio: mxc6655xa_irq_gpio { + rockchip,pins = <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_none>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA2 2 &pcfg_pull_none>; + }; + }; + + touch { + touch_gpio: touch-gpio { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb { + vcc5v0_host_en: vcc5v0-host-en { + rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + vcc5v0_otg_en: vcc5v0-otg-en { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart8_gpios: uart8-gpios { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + + /* + * There are 10 independent IO domains in RK3566/RK3568, including PMUIO[0:2] and VCCIO[1:7]. + * 1/ PMUIO0 and PMUIO1 are fixed-level power domains which cannot be configured; + * 2/ PMUIO2 and VCCIO1,VCCIO[3:7] domains require that their hardware power supply voltages + * must be consistent with the software configuration correspondingly + * a/ When the hardware IO level is connected to 1.8V, the software voltage configuration + * should also be configured to 1.8V accordingly; + * b/ When the hardware IO level is connected to 3.3V, the software voltage configuration + * should also be configured to 3.3V accordingly; + * 3/ VCCIO2 voltage control selection (0xFDC20140) + * BIT[0]: 0x0: from GPIO_0A7 (default) + * BIT[0]: 0x1: from GRF + * Default is determined by Pin FLASH_VOL_SEL/GPIO0_A7: + * L:VCCIO2 must supply 3.3V + * H:VCCIO2 must supply 1.8V + */ +&pmu_io_domains { + status = "okay"; + pmuio2-supply = <&vcc3v3_pmu>; + vccio1-supply = <&vccio_acodec>; + vccio3-supply = <&vccio_sd>; + vccio4-supply = <&vcc_3v3>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc_3v3>; + vccio7-supply = <&vcc_3v3>; +}; + +&pwm4 { + status = "okay"; +}; + +&pwm5 { + status = "okay"; +}; + +&pwm7 { + status = "okay"; + + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pwm7_pins>; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_VOLUMEDOWN>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +&rk_rga { + status = "okay"; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvenc { + venc-supply = <&vdd_logic>; + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&rknpu { + rknpu-supply = <&vdd_npu>; + status = "okay"; +}; + +&rknpu_mmu { + status = "okay"; +}; + +&route_hdmi { + status = "okay"; + connect = <&vp0_out_hdmi>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcca_1v8>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + max-frequency = <200000000>; + status = "okay"; +}; + +&sdmmc0 { + max-frequency = <150000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + sd-uhs-sdr104; + vmmc-supply = <&vcc3v3_sd>; + vqmmc-supply = <&vccio_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; + status = "okay"; +}; + +&sfc { + status = "okay"; +}; + +&spdif_8ch { + status = "okay"; +}; + +&tsadc { + status = "okay"; +}; + +&u2phy0_host { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&u2phy0_otg { + vbus-supply = <&vcc5v0_otg>; + status = "okay"; +}; + +&u2phy1_host { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&u2phy1_otg { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usb2phy1 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd_dwc3 { + dr_mode = "otg"; + extcon = <&usb2phy0>; + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&usbhost_dwc3 { + status = "okay"; +}; + +&usbhost30 { + status = "okay"; +}; + +&vad { + rockchip,audio-src = <&i2s1_8ch>; + rockchip,buffer-time-ms = <128>; + rockchip,det-channel = <0>; + rockchip,mode = <0>; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>; + assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux-spi-nor.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux-spi-nor.dts new file mode 100755 index 000000000000..259b483af0f4 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux-spi-nor.dts @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb1-ddr4-v10.dtsi" +#include "rk3568-linux.dtsi" + +/ { + model = "Rockchip RK3568 EVB1 DDR4 V10 Linux SPI NOR Board"; + compatible = "rockchip,rk3568-evb1-ddr4-v10-linux-spi-nor", "rockchip,rk3568"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 root=/dev/mtdblock3 rootfstype=squashfs rootwait"; + }; + +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux.dts new file mode 100755 index 000000000000..55c255998473 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10-linux.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb1-ddr4-v10.dtsi" +#include "rk3568-linux.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dts new file mode 100755 index 000000000000..00c28f39269b --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb1-ddr4-v10.dtsi" +#include "rk3568-android.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dtsi new file mode 100755 index 000000000000..64881d20183f --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb1-ddr4-v10.dtsi @@ -0,0 +1,490 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include "rk3568.dtsi" +#include "rk3568-evb.dtsi" + +/ { + model = "Rockchip RK3568 EVB1 DDR4 V10 Board"; + compatible = "rockchip,rk3568-evb1-ddr4-v10", "rockchip,rk3568"; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + vcc2v5_sys: vcc2v5-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc2v5-sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + vin-supply = <&vcc3v3_sys>; + }; + + vcc3v3_vga: vcc3v3-vga { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_vga"; + regulator-always-on; + regulator-boot-on; + gpio = <&gpio0 RK_PD5 GPIO_ACTIVE_HIGH>; + enable-active-high; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_avdd0v9: pcie30-avdd0v9 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd0v9"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_avdd1v8: pcie30-avdd1v8 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie30_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + vcc3v3_bu: vcc3v3-bu { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_bu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc5v0_sys>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; +}; + +&combphy0_us { + status = "okay"; +}; + +&combphy1_usq { + status = "okay"; +}; + +&combphy2_psq { + status = "okay"; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2 3 4>; + }; + mipi_in_ucam1: endpoint@2 { + reg = <2>; + remote-endpoint = <&gc8034_out>; + data-lanes = <1 2 3 4>; + }; + mipi_in_ucam2: endpoint@3 { + reg = <3>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +/* + * video_phy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; +}; + +/* + * video_phy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; +}; + +&edp { + hpd-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&edp_phy { + status = "okay"; +}; + +&edp_in_vp0 { + status = "okay"; +}; + +&edp_in_vp1 { + status = "disabled"; +}; + +&gmac0 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0>; + assigned-clock-parents = <&cru SCLK_GMAC0_RGMII_SPEED>, <&cru CLK_MAC0_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_miim + &gmac0_tx_bus2 + &gmac0_rx_bus2 + &gmac0_rgmii_clk + &gmac0_rgmii_bus>; + + tx_delay = <0x3c>; + rx_delay = <0x2f>; + + phy-handle = <&rgmii_phy0>; + status = "okay"; +}; + +&gmac1 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio2 RK_PD1 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m1_miim + &gmac1m1_tx_bus2 + &gmac1m1_rx_bus2 + &gmac1m1_rgmii_clk + &gmac1m1_rgmii_bus>; + + tx_delay = <0x4f>; + rx_delay = <0x26>; + + phy-handle = <&rgmii_phy1>; + status = "okay"; +}; + +/* + * power-supply should switche to vcc3v3_lcd1_n + * when mipi panel is connected to dsi1. + */ +>1x { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&i2c4 { + status = "okay"; + gc8034: gc8034@37 { + compatible = "galaxycore,gc8034"; + status = "okay"; + reg = <0x37>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; + rockchip,grf = <&grf>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "RK-CMK-8M-2-v1"; + rockchip,camera-module-lens-name = "CK8401"; + port { + gc8034_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; + os04a10: os04a10@36 { + compatible = "ovti,os04a10"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT1607-FV1"; + rockchip,camera-module-lens-name = "M12-40IRC-4MP-F16"; + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + }; + }; + }; + ov5695: ov5695@36 { + status = "okay"; + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in_ucam2>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&mdio0 { + rgmii_phy0: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&mdio1 { + rgmii_phy1: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +&pcie30phy { + status = "okay"; +}; + +&pcie3x2 { + reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie30_3v3>; + status = "okay"; +}; + +&pinctrl { + cam { + camera_pwr: camera-pwr { + rockchip,pins = + /* camera power en */ + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart8_gpios: uart8-gpios { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy_out>; + }; + }; +}; + +&route_dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&sata2 { + status = "okay"; +}; + +&sdmmc2 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&spdif_8ch { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&spdifm1_tx>; +}; + +&uart8 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn>; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&wireless_wlan { + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; +}; + +&wireless_bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart8m0_rtsn>; + pinctrl-1 = <&uart8_gpios>; + BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10-bt1120-to-hdmi.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10-bt1120-to-hdmi.dts new file mode 100755 index 000000000000..7a5a7feb52a7 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10-bt1120-to-hdmi.dts @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb2-lp4x-v10.dtsi" +#include "rk3568-android.dtsi" + +&dsi0 { + status = "disabled"; +}; + +&hdmi { + status = "disabled"; +}; + +&i2c3 { + clock-frequency = <400000>; + status = "okay"; + + sii9022: sii9022@39 { + compatible = "sil,sii9022"; + reg = <0x39>; + pinctrl-names = "default"; + pinctrl-0 = <&sii902x_hdmi_int>; + interrupt-parent = <&gpio3>; + interrupts = ; + reset-gpio = <&gpio3 RK_PC4 GPIO_ACTIVE_LOW>; + enable-gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + bus-format = <1>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sii9022_in_rgb: endpoint { + remote-endpoint = <&rgb_out_sii9022>; + }; + }; + }; + }; +}; + +&video_phy0 { + status = "disabled"; +}; + +&rgb { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&bt1120_pins>; + + ports { + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + rgb_out_sii9022: endpoint@0 { + reg = <0>; + remote-endpoint = <&sii9022_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vp2 { + status = "okay"; +}; + +&vcc3v3_lcd1_n { + status = "disabled"; + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dts new file mode 100755 index 000000000000..23f5f036c4b2 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb2-lp4x-v10.dtsi" +#include "rk3568-android.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dtsi new file mode 100755 index 000000000000..c2d0b697a9bd --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb2-lp4x-v10.dtsi @@ -0,0 +1,493 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include "rk3568.dtsi" +#include "rk3568-evb.dtsi" + +/ { + model = "Rockchip RK3568 EVB2 LP4X V10 Board"; + compatible = "rockchip,rk3568-evb2-lp4x-v10", "rockchip,rk3568"; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio4 RK_PC1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + vcc2v5_sys: vcc2v5-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc2v5-sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie20_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie20_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + pcie30_avdd0v9: pcie30-avdd0v9 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd0v9"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_avdd1v8: pcie30-avdd1v8 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie30_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + qsgmii_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "qsgmii_3v3"; + regulator-min-microvolt = <0100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <0100000 0x0 + 3300000 0x1>; + }; + + vcc3v3_bu: vcc3v3-bu { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_bu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc5v0_sys>; + }; +}; + +&combphy0_us { + status = "okay"; +}; + +&combphy1_usq { + status = "okay"; +}; + +&combphy2_psq { + status = "okay"; +}; + +/* + * video_phy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; +}; + +/* + * video_phy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; +}; + +&gmac0 { + phy-supply = <&qsgmii_3v3>; + phy-mode = "qsgmii"; + rockchip,xpcs = <&xpcs>; + + assigned-clocks = <&cru SCLK_GMAC0_RX_TX>; + assigned-clock-parents = <&gmac0_xpcsclk>; + + power-domains = <&power RK3568_PD_PIPE>; + phys = <&combphy1_usq PHY_TYPE_QSGMII>; + phy-handle = <&qsgmii_phy0>; + + status = "okay"; +}; + +&gmac1 { + phy-supply = <&qsgmii_3v3>; + phy-mode = "qsgmii"; + + snps,reset-gpio = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>; + assigned-clock-parents = <&gmac1_xpcsclk>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m1_miim>; + + power-domains = <&power RK3568_PD_PIPE>; + phy-handle = <&qsgmii_phy1>; + + status = "okay"; +}; + +/* + * power-supply should switche to vcc3v3_lcd1_n + * when mipi panel is connected to dsi1. + */ +>1x { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&i2c4 { + status = "okay"; + + gs_mxc6655xa: gs_mxc6655xa@15 { + status = "okay"; + compatible = "gs_mxc6655xa"; + pinctrl-names = "default"; + pinctrl-0 = <&mxc6655xa_irq_gpio>; + reg = <0x15>; + irq-gpio = <&gpio3 RK_PD7 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + power-off-in-suspend = <1>; + layout = <1>; + }; +}; + +&i2c5 { + status = "disabled"; +}; + +&mdio1 { + qsgmii_phy0: phy@0 { + compatible = "ethernet-phy-id001c.c942", "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; + qsgmii_phy1: phy@1 { + compatible = "ethernet-phy-id001c.c942", "ethernet-phy-ieee802.3-c22"; + reg = <0x1>; + }; + qsgmii_phy2: phy@2 { + compatible = "ethernet-phy-id001c.c942", "ethernet-phy-ieee802.3-c22"; + reg = <0x2>; + }; + qsgmii_phy3: phy@3 { + compatible = "ethernet-phy-id001c.c942", "ethernet-phy-ieee802.3-c22"; + reg = <0x3>; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +&pcie2x1 { + reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie20_3v3>; + status = "okay"; +}; + +&pcie30phy { + status = "okay"; +}; + +&pcie3x2 { + reset-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie30_3v3>; + status = "okay"; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <4 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + mxc6655xa { + mxc6655xa_irq_gpio: mxc6655xa_irq_gpio { + rockchip,pins = <3 RK_PD7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sii902x { + sii902x_hdmi_int: sii902x-hdmi-int { + rockchip,pins = <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + wifi_32k: wifi-32k { + rockchip,pins = <2 RK_PC6 1 &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + vccio6-supply = <&vcc_3v3>; +}; + +&pwm3 { + status = "okay"; + + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pwm3_pins>; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_VOLUMEDOWN>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +&pwm7 { + status = "disabled"; +}; + +&route_dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&sdio_pwrseq { + clocks = <&pmucru CLK_RTC_32K>; + pinctrl-0 = <&wifi_enable_h &wifi_32k>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc2 { + status = "disabled"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&wireless_wlan { + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; +}; + +&wireless_bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&pmucru CLK_RTC_32K>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&xpcs { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb4-lp3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb4-lp3-v10.dts new file mode 100755 index 000000000000..5884da41f121 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb4-lp3-v10.dts @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb1-ddr4-v10.dtsi" +#include "rk3568-android.dtsi" +/{ + model = "Rockchip RK3568 EVB4 LP3 V10 Board"; + compatible = "rockchip,rk3568-evb4-lp3-v10", "rockchip,rk3568"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dts new file mode 100755 index 000000000000..e9eb333079a2 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb5-ddr4-v10.dtsi" +#include "rk3568-android.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dtsi new file mode 100755 index 000000000000..e8a00fc85483 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb5-ddr4-v10.dtsi @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include "rk3568.dtsi" +#include "rk3568-evb.dtsi" + +/ { + model = "Rockchip RK3568 EVB5 DDR4 V10 Board"; + compatible = "rockchip,rk3568-evb5-ddr4-v10", "rockchip,rk3568"; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + vcc2v5_sys: vcc2v5-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc2v5-sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_avdd0v9: pcie30-avdd0v9 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd0v9"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_avdd1v8: pcie30-avdd1v8 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie30_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + vcc3v3_bu: vcc3v3-bu { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_bu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc5v0_sys>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; +}; + +&combphy0_us { + status = "okay"; +}; + +&combphy1_usq { + rockchip,sgmii-mac-sel = <0>; + status = "okay"; +}; + +&combphy2_psq { + status = "okay"; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2 3 4>; + }; + mipi_in_ucam1: endpoint@2 { + reg = <2>; + remote-endpoint = <&gc8034_out>; + data-lanes = <1 2 3 4>; + }; + mipi_in_ucam2: endpoint@3 { + reg = <3>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +/* + * video_phy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; +}; + +/* + * video_phy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; +}; + +&edp { + hpd-gpios = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&edp_phy { + status = "okay"; +}; + +&edp_in_vp0 { + status = "okay"; +}; + +&edp_in_vp1 { + status = "disabled"; +}; + +&gmac0 { + phy-mode = "sgmii"; + + rockchip,pipegrf = <&pipegrf>; + rockchip,xpcs = <&xpcs>; + + snps,reset-gpio = <&gpio2 RK_PC2 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC0_RX_TX>; + assigned-clock-parents = <&gmac0_xpcsclk>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_miim>; + + power-domains = <&power RK3568_PD_PIPE>; + phys = <&combphy1_usq PHY_TYPE_SGMII>; + phy-handle = <&sgmii_phy>; + + status = "okay"; +}; + +&gmac1 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m1_miim + &gmac1m1_tx_bus2_level3 + &gmac1m1_rx_bus2 + &gmac1m1_rgmii_clk_level2 + &gmac1m1_rgmii_bus_level3>; + + tx_delay = <0x46>; + rx_delay = <0x2f>; + + phy-handle = <&rgmii_phy1>; + status = "okay"; +}; + +/* + * power-supply should switche to vcc3v3_lcd1_n + * when mipi panel is connected to dsi1. + */ +>1x { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&i2c3 { + clock-frequency = <400000>; + status = "okay"; + + sii9022: sii9022@39 { + compatible = "sil,sii9022"; + reg = <0x39>; + pinctrl-names = "default"; + pinctrl-0 = <&sii902x_hdmi_int>; + interrupt-parent = <&gpio4>; + interrupts = ; + reset-gpio = <&gpio3 RK_PC4 GPIO_ACTIVE_LOW>; + enable-gpio = <&gpio0 RK_PD5 GPIO_ACTIVE_HIGH>; + bus-format = <0>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + sii9022_in_rgb: endpoint { + remote-endpoint = <&rgb_out_sii9022>; + }; + }; + }; + }; +}; + +&i2c4 { + status = "okay"; + gc8034: gc8034@37 { + compatible = "galaxycore,gc8034"; + status = "okay"; + reg = <0x37>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; + rockchip,grf = <&grf>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "RK-CMK-8M-2-v1"; + rockchip,camera-module-lens-name = "CK8401"; + port { + gc8034_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; + os04a10: os04a10@36 { + compatible = "ovti,os04a10"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT1607-FV1"; + rockchip,camera-module-lens-name = "M12-40IRC-4MP-F16"; + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + }; + }; + }; + ov5695: ov5695@36 { + status = "okay"; + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in_ucam2>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c5 { + status = "disabled"; +}; + +&mdio0 { + sgmii_phy: phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x1>; + }; +}; + +&mdio1 { + rgmii_phy1: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +&pcie30phy { + status = "okay"; +}; + +&pcie3x2 { + reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie30_3v3>; + status = "okay"; +}; + +&pinctrl { + cam { + camera_pwr: camera-pwr { + rockchip,pins = + /* camera power en */ + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sii902x { + sii902x_hdmi_int: sii902x-hdmi-int { + rockchip,pins = <4 RK_PD2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart8_gpios: uart8-gpios { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&rgb { + status = "okay"; + + ports { + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + rgb_out_sii9022: endpoint@0 { + reg = <0>; + remote-endpoint = <&sii9022_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vp2 { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy_out>; + }; + }; +}; + +&route_dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&sata2 { + status = "okay"; +}; + +&sdmmc2 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&spdif_8ch { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&spdifm1_tx>; +}; + +&uart8 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn>; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&wireless_wlan { + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; +}; + +&wireless_bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart8m0_rtsn>; + pinctrl-1 = <&uart8_gpios>; + BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-linux.dts new file mode 100755 index 000000000000..a11975fc7d53 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-linux.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb6-ddr3-v10.dtsi" +#include "rk3568-linux.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-bt1120-to-hdmi.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-bt1120-to-hdmi.dts new file mode 100755 index 000000000000..0c5ea5eb32ba --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-bt1120-to-hdmi.dts @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +#include "rk3568-evb6-ddr3-v10.dtsi" +#include "rk3568-android.dtsi" + +&dsi0 { + status = "disabled"; +}; + +&i2c3 { + clock-frequency = <400000>; + status = "okay"; + + rk628: rk628@50 { + reg = <0x50>; + interrupt-parent = <&gpio0>; + interrupts = ; + enable-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>; + status = "okay"; + }; +}; + +&video_phy0 { + status = "disabled"; +}; + +#include + +&rk628_hdmi { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hdmi_in_post_process: endpoint { + remote-endpoint = <&post_process_out_hdmi>; + }; + }; + }; +}; + +&rk628_post_process { + pinctrl-names = "default"; + pinctrl-0 = <&vop_pins>; + status = "okay"; + + mode-sync-pol = <0>; + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + post_process_in_bt1120: endpoint { + remote-endpoint = <&bt1120_out_post_process>; + }; + }; + + port@1 { + reg = <1>; + + post_process_out_hdmi: endpoint { + remote-endpoint = <&hdmi_in_post_process>; + }; + }; + }; +}; + +&rk628_bt1120_rx { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + bt1120_in_rgb: endpoint { + remote-endpoint = <&rgb_out_bt1120>; + }; + }; + + port@1 { + reg = <1>; + + bt1120_out_post_process: endpoint { + remote-endpoint = <&post_process_in_bt1120>; + }; + }; + }; +}; + +&rgb { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&bt1120_pins>; + + ports { + port@1 { + reg = <1>; + + rgb_out_bt1120: endpoint { + remote-endpoint = <&bt1120_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vp2 { + status = "okay"; +}; + +&vcc3v3_lcd1_n { + status = "disabled"; + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-rgb2hdmi.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-rgb2hdmi.dts new file mode 100755 index 000000000000..597fa0b0b089 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk628-rgb2hdmi.dts @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +#include "rk3568-evb6-ddr3-v10.dtsi" +#include "rk3568-android.dtsi" + +&dsi0 { + status = "disabled"; +}; + +&i2c3 { + clock-frequency = <400000>; + status = "okay"; + + rk628: rk628@50 { + reg = <0x50>; + interrupt-parent = <&gpio0>; + interrupts = ; + enable-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>; + status = "okay"; + }; +}; + +#include + +&rk628_hdmi { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hdmi_in_post_process: endpoint { + remote-endpoint = <&post_process_out_hdmi>; + }; + }; + }; +}; + +&rk628_post_process { + pinctrl-names = "default"; + pinctrl-0 = <&vop_pins>; + status = "okay"; + + mode-sync-pol = <0>; + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + post_process_in_rgb: endpoint { + remote-endpoint = <&rgb_out_post_process>; + }; + }; + + port@1 { + reg = <1>; + + post_process_out_hdmi: endpoint { + remote-endpoint = <&hdmi_in_post_process>; + }; + }; + }; +}; + +&rgb { + status = "okay"; + + ports { + port@1 { + reg = <1>; + + rgb_out_post_process: endpoint { + remote-endpoint = <&post_process_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vp2 { + status = "okay"; +}; + +&vcc3v3_lcd1_n { + status = "disabled"; + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk630-bt656-to-cvbs.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk630-bt656-to-cvbs.dts new file mode 100755 index 000000000000..24c4debec4f3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10-rk630-bt656-to-cvbs.dts @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +#include +#include "rk3568-evb6-ddr3-v10.dtsi" +#include "rk3568-android.dtsi" + +&dsi0 { + status = "disabled"; +}; + +&hdmi { + status = "disabled"; +}; + +&i2c3 { + status = "okay"; + clock-frequency = <100000>; + + rk630: rk630@50 { + compatible = "rockchip,rk630"; + reg = <0x50>; + reset-gpios = <&gpio2 RK_PC7 GPIO_ACTIVE_LOW>; + status = "okay"; + + rk630_tve: rk630-tve { + compatible = "rockchip,rk630-tve"; + status = "okay"; + + ports { + port { + rk630_tve_in_rgb: endpoint { + remote-endpoint = <&rgb_out_rk630_tve>; + }; + }; + }; + }; + }; +}; + +&rgb { + pinctrl-names = "default"; + pinctrl-0 = <&bt656m0_pins>; /* bt656m0_pins or bt656m1_pins */ + status = "okay"; + + ports { + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + rgb_out_rk630_tve: endpoint@0 { + reg = <0>; + remote-endpoint = <&rk630_tve_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vp2 { + status = "okay"; +}; + +&vcc3v3_lcd1_n { + status = "disabled"; + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dts new file mode 100755 index 000000000000..6e34fa48abc4 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb6-ddr3-v10.dtsi" +#include "rk3568-android.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dtsi new file mode 100755 index 000000000000..2f3b5d74e1cf --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb6-ddr3-v10.dtsi @@ -0,0 +1,490 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3568.dtsi" +#include "rk3568-evb.dtsi" + +/ { + model = "Rockchip RK3568 EVB6 DDR3 V10 Board"; + compatible = "rockchip,rk3568-evb6-ddr3-v10", "rockchip,rk3568"; + + pcie20_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie20_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + pcie30_avdd0v9: pcie30-avdd0v9 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd0v9"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_avdd1v8: pcie30-avdd1v8 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie30_3v3"; + regulator-min-microvolt = <0100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <0100000 0x0 + 3300000 0x1>; + }; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; +}; + +&combphy0_us { + status = "okay"; +}; + +&combphy1_usq { + rockchip,dis-u3otg1-port; + status = "okay"; +}; + +&combphy2_psq { + status = "okay"; +}; + +/* + * video_phy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; +}; + +/* + * video_phy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; +}; + +/* + * power-supply should switche to vcc3v3_lcd1_n + * when mipi panel is connected to dsi1. + */ +>1x { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2m1_xfer>; + + mxc6655xa: mxc6655xa@15 { + status = "okay"; + compatible = "gs_mxc6655xa"; + pinctrl-names = "default"; + pinctrl-0 = <&mxc6655xa_irq_gpio>; + reg = <0x15>; + irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + power-off-in-suspend = <1>; + layout = <4>; + }; +}; + +&i2c4 { + status = "okay"; + os04a10: os04a10@36 { + compatible = "ovti,os04a10"; + reg = <0x36>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clkout0>; + reset-gpios = <&gpio2 RK_PD5 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; + /* power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; */ + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT1607-FV1"; + /* rockchip,camera-module-lens-name = "M12-4IR-4MP-F16"; */ + rockchip,camera-module-lens-name = "M12-40IRC-4MP-F16"; + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + }; + }; + }; + gc8034: gc8034@37 { + compatible = "galaxycore,gc8034"; + reg = <0x37>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clkout0>; + reset-gpios = <&gpio2 RK_PD5 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_LOW>; + rockchip,grf = <&grf>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "RK-CMK-8M-2-v1"; + rockchip,camera-module-lens-name = "CK8401"; + port { + gc8034_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; + ov5695: ov5695@36 { + status = "okay"; + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clkout0>; + reset-gpios = <&gpio2 RK_PD5 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in_ucam2>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&i2c5 { + status = "disabled"; + + /delete-node/ mxc6655xa@15; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2 3 4>; + }; + mipi_in_ucam1: endpoint@2 { + reg = <2>; + remote-endpoint = <&gc8034_out>; + data-lanes = <1 2 3 4>; + }; + mipi_in_ucam2: endpoint@3 { + reg = <3>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy_out>; + }; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +&pcie30phy { + status = "okay"; +}; + +&pcie2x1 { + reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie20_3v3>; + status = "okay"; +}; + +&pcie3x1 { + rockchip,bifurcation; + reset-gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie30_3v3>; + status = "okay"; +}; + +&pcie3x2 { + rockchip,bifurcation; + reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie30_3v3>; + status = "okay"; +}; + +&pinctrl { + cam { + camera_pwr: camera-pwr { + rockchip,pins = + /* camera power en */ + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + wifi_32k: wifi-32k { + rockchip,pins = <2 RK_PC6 1 &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <2 RK_PB2 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart1_gpios: uart1-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&route_dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&sdmmc1 { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc2 { + status = "disabled"; +}; + +&sdio_pwrseq { + clocks = <&pmucru CLK_RTC_32K>; + pinctrl-0 = <&wifi_enable_h &wifi_32k>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; +}; + +&spdif_8ch { + status = "disabled"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&usbhost_dwc3 { + phys = <&u2phy0_host>; + phy-names = "usb2-phy"; + maximum-speed = "high-speed"; + status = "okay"; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&wireless_wlan { + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio2 RK_PB2 GPIO_ACTIVE_HIGH>; +}; + +&wireless_bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&pmucru CLK_RTC_32K>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart1m0_rtsn>; + pinctrl-1 = <&uart1_gpios>; + BT,reset_gpio = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&gmac1 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio2 RK_PD1 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m0_miim + &gmac1m0_tx_bus2_level3 + &gmac1m0_rx_bus2 + &gmac1m0_rgmii_clk_level2 + &gmac1m0_rgmii_bus_level3>; + + tx_delay = <0x46>; + rx_delay = <0x2f>; + + phy-handle = <&rgmii_phy1>; + status = "okay"; +}; + +&mdio1 { + rgmii_phy1: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb7-ddr4-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb7-ddr4-v10.dts new file mode 100755 index 000000000000..ecb96832c292 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb7-ddr4-v10.dts @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568-evb1-ddr4-v10.dtsi" +#include "rk3568-android.dtsi" +/{ + model = "Rockchip RK3568 EVB7 DDR4 V10 Board"; + compatible = "rockchip,rk3568-evb7-ddr4-v10", "rockchip,rk3568"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10-linux.dts new file mode 100755 index 000000000000..d832a40d37e9 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10-linux.dts @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3568.dtsi" +#include + +/ { + model = "Rockchip RK3568 IOTEST DDR3 V10 Board"; + compatible = "rockchip,rk3568-iotest-ddr3-v10", "rockchip,rk3568"; + + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rw rootwait"; + }; + + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <1>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "okay"; + }; + + test-power { + status = "okay"; + }; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + max-frequency = <200000000>; + status = "okay"; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usbdrd_dwc3 { + dr_mode = "otg"; + phys = <&u2phy0_otg>; + phy-names = "usb2-phy"; + extcon = <&usb2phy0>; + maximum-speed = "high-speed"; + snps,dis_u2_susphy_quirk; + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +/delete-node/ &display_subsystem; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10.dts new file mode 100755 index 000000000000..489967f1318f --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-iotest-ddr3-v10.dts @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3568.dtsi" +#include "rk3568-evb.dtsi" +#include "rk3568-android.dtsi" + +/ { + model = "Rockchip RK3568 IOTEST DDR3 V10 Board"; + compatible = "rockchip,rk3568-iotest-ddr3-v10", "rockchip,rk3568"; +}; + +&usb_host0_ehci { + status = "disabled"; +}; + +&usb_host0_ohci { + status = "disabled"; +}; + +&usb_host1_ehci { + status = "disabled"; +}; + +&usb_host1_ohci { + status = "disabled"; +}; + +&usbdrd_dwc3 { + phys = <&u2phy0_otg>; + phy-names = "usb2-phy"; + maximum-speed = "high-speed"; + snps,dis_u2_susphy_quirk; + status = "okay"; +}; + +&usbhost_dwc3 { + phys = <&u2phy1_otg>; + phy-names = "usb2-phy"; + maximum-speed = "high-speed"; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-linux.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-linux.dtsi new file mode 100755 index 000000000000..838606964e15 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-linux.dtsi @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/ { + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 ohos.boot.eng_mode=on root=PARTUUID=614e0000-0000 hardware=rk3568 default_boot_device=fe310000.sdhci rw rootwait ohos.required_mount.system=/dev/block/platform/fe310000.sdhci/by-name/system@/usr@ext4@ro,barrier=1@wait,required ohos.required_mount.vendor=/dev/block/platform/fe310000.sdhci/by-name/vendor@/vendor@ext4@ro,barrier=1@wait,required ohos.required_mount.misc=/dev/block/platform/fe310000.sdhci/by-name/misc@none@none@none@wait,required ohos.required_mount.bootctrl=/dev/block/platform/fe310000.sdhci/by-name/bootctrl@none@none@none@wait,required"; + }; + + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <1>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "okay"; + }; + + debug: debug@fd904000 { + compatible = "rockchip,debug"; + reg = <0x0 0xfd904000 0x0 0x1000>, + <0x0 0xfd905000 0x0 0x1000>, + <0x0 0xfd906000 0x0 0x1000>, + <0x0 0xfd907000 0x0 0x1000>; + }; + + cspmu: cspmu@fd90c000 { + compatible = "rockchip,cspmu"; + reg = <0x0 0xfd90c000 0x0 0x1000>, + <0x0 0xfd90d000 0x0 0x1000>, + <0x0 0xfd90e000 0x0 0x1000>, + <0x0 0xfd90f000 0x0 0x1000>; + }; +}; + +&reserved_memory { + ramoops: ramoops@110000 { + compatible = "ramoops"; + reg = <0x0 0x110000 0x0 0xf0000>; + record-size = <0x20000>; + console-size = <0x80000>; + ftrace-size = <0x00000>; + pmsg-size = <0x00000>; + blackbox-size = <0x50000>; + }; +}; + +&rng { + status = "okay"; +}; + +&vop { + disable-afbc-win; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux-spi-nand.dts b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux-spi-nand.dts new file mode 100755 index 000000000000..50bd024528df --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux-spi-nand.dts @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; + +#include "rk3568-nvr-demo-v10.dtsi" +#include "rk3568-nvr-linux.dtsi" + +/ { + model = "Rockchip RK3568 NVR DEMO DDR4 V10 Linux SPI NAND Board"; + compatible = "rockchip,rk3568-nvr-demo-ddr4-v10-linux-spi-nand", "rockchip,rk3568"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 ubi.mtd=3 root=ubi0:rootfs rootfstype=ubifs"; + }; + +}; + +&pcie30phy { + status = "disabled"; +}; + +&pcie3x1 { + status = "disabled"; +}; + +&pcie3x2 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux.dts new file mode 100755 index 000000000000..3317db6ee671 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10-linux.dts @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3568-nvr-demo-v10.dtsi" +#include "rk3568-nvr-linux.dtsi" + +/ { + model = "Rockchip RK3568 NVR DEMO DDR4 V10 Linux Board"; + compatible = "rockchip,rk3568-nvr-demo-ddr4-v10-linux", "rockchip,rk3568"; +}; + +&pcie30phy { + status = "disabled"; +}; + +&pcie3x1 { + status = "disabled"; +}; + +&pcie3x2 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dts new file mode 100755 index 000000000000..053d20259e93 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dts @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3568-nvr-demo-v10.dtsi" +#include "rk3568-android.dtsi" + +/ { + model = "Rockchip RK3568 NVR DEMO DDR4 V10 ANDROID Board"; + compatible = "rockchip,rk3568-nvr-demo-ddr4-v10", "rockchip,rk3568"; +}; + +&pcie30phy { + status = "disabled"; +}; + +&pcie3x1 { + status = "disabled"; +}; + +&pcie3x2 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dtsi new file mode 100755 index 000000000000..40a2096bc58b --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v10.dtsi @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3568-nvr.dtsi" +#include + +/ { + model = "Rockchip RK3568 NVR DEMO V10 Board"; + compatible = "rockchip,rk3568-nvr-demo-v10", "rockchip,rk3568"; + + gpio-leds { + compatible = "gpio-leds"; + + hdd-led { + gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + net-led { + gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + work-led { + gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "timer"; + }; + }; + + i2s1_sound: i2s1-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,i2s1-sound"; + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&es8311>; + }; + }; + + vcc2v5_sys: vcc2v5-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc2v5-sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie30_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + pcie30_avdd0v9: pcie30-avdd0v9 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd0v9"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_avdd1v8: pcie30-avdd1v8 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc3v3_sys>; + }; + + vcc3v3_bu: vcc3v3-bu { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_bu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc5v0_sys>; + }; +}; + +&combphy1_usq { + pinctrl-names = "default"; + pinctrl-0 = <&sata_pm_reset>; + rockchip,dis-u3otg1-port; + status = "okay"; +}; + +&combphy2_psq{ + status = "okay"; +}; + +&gmac0 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio1 RK_PB0 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0>; + assigned-clock-parents = <&cru SCLK_GMAC0_RGMII_SPEED>, <&cru CLK_MAC0_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_miim + &gmac0_tx_bus2 + &gmac0_rx_bus2 + &gmac0_rgmii_clk + &gmac0_rgmii_bus>; + + tx_delay = <0x43>; + rx_delay = <0x33>; + + phy-handle = <&rgmii_phy0>; + status = "okay"; +}; + +&gmac1 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio1 RK_PB1 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m1_miim + &gmac1m1_tx_bus2 + &gmac1m1_rx_bus2 + &gmac1m1_rgmii_clk + &gmac1m1_rgmii_bus>; + + tx_delay = <0x4f>; + rx_delay = <0x2d>; + + phy-handle = <&rgmii_phy1>; + status = "okay"; +}; + +&i2c1 { + status = "okay"; + hym8563: hym8563@51 { + compatible = "haoyu,hym8563"; + reg = <0x51>; + + pinctrl-names = "default"; + pinctrl-0 = <&rtc_int>; + + interrupt-parent = <&gpio0>; + interrupts = ; + }; +}; + +&i2c3 { + status = "okay"; + clock-frequency = <400000>; + + es8311: es8311@18 { + compatible = "everest,es8311"; + reg = <0x18>; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + adc-pga-gain = <6>; /* 18dB */ + adc-volume = <0xbf>; /* 0dB */ + dac-volume = <0xbf>; /* 0dB */ + aec-mode = "dac left, adc right"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + spk-ctl-gpios = <&gpio2 RK_PA5 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + }; + + rk618@50 { + compatible = "rockchip,rk618"; + reg = <0x50>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s3m1_mclk &rk618_int>; + clocks = <&cru I2S3_MCLKOUT>; + clock-names = "clkin"; + assigned-clocks =<&cru I2S3_MCLKOUT>, <&cru I2S3_MCLK_IOE>; + assigned-clock-parents = <&cru I2S3_MCLKOUT_TX>, <&cru I2S3_MCLKOUT>; + assigned-clock-rates = <11289600>; + reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_LOW>; + status = "okay"; + + clock: cru { + compatible = "rockchip,rk618-cru"; + clocks = <&cru I2S3_MCLKOUT>, <&cru DCLK_VOP2>; + clock-names = "clkin", "lcdc0_dclkp"; + assigned-clocks = <&clock SCALER_PLLIN_CLK>, + <&clock VIF_PLLIN_CLK>, + <&clock SCALER_CLK>, + <&clock VIF0_PRE_CLK>, + <&clock CODEC_CLK>, + <&clock DITHER_CLK>; + assigned-clock-parents = <&cru I2S3_MCLKOUT_TX>, + <&clock LCDC0_CLK>, + <&clock SCALER_PLL_CLK>, + <&clock VIF_PLL_CLK>, + <&cru I2S3_MCLKOUT>, + <&clock VIF0_CLK>; + #clock-cells = <1>; + status = "okay"; + }; + + hdmi { + compatible = "rockchip,rk618-hdmi"; + clocks = <&clock HDMI_CLK>; + clock-names = "hdmi"; + assigned-clocks = <&clock HDMI_CLK>; + assigned-clock-parents = <&clock VIF0_CLK>; + interrupt-parent = <&gpio0>; + interrupts = <8 IRQ_TYPE_LEVEL_HIGH>; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + hdmi_in_rgb: endpoint { + remote-endpoint = <&rgb_out_hdmi>; + }; + }; + }; + }; + }; +}; + +&mdio0 { + rgmii_phy0: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&mdio1 { + rgmii_phy1: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&pcie30phy { + status = "okay"; +}; + +&pcie3x1 { + reset-gpios = <&gpio0 RK_PC3 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie30_3v3>; + status = "okay"; +}; + +&pcie3x2 { + reset-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie30_3v3>; + status = "okay"; +}; + +&pwm15 { + compatible = "rockchip,remotectl-pwm"; + pinctrl-names = "default"; + pinctrl-0 = <&pwm15m1_pins>; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <0>; + status = "okay"; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_PLAYPAUSE>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xa4 KEY_SETUP>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +&rgb { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&lcdc_ctl>; + ports { + port@1 { + reg = <1>; + + rgb_out_hdmi: endpoint { + remote-endpoint = <&hdmi_in_rgb>; + }; + }; + }; +}; + +&rgb_in_vp2 { + status = "okay"; +}; + +&sata1 { + status = "okay"; +}; + +&sata2 { + status = "okay"; +}; + +&pinctrl { + rk618 { + rk618_reset: rk618-reeset { + rockchip,pins = <1 RK_PB2 RK_FUNC_GPIO &pcfg_output_high>; + }; + rk618_int: rk618-int { + rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + rtc { + rtc_int: rtc-int { + rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sata { + sata_pm_reset: sata-pm-reset { + rockchip,pins = <4 RK_PD2 RK_FUNC_GPIO &pcfg_output_high>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux-spi-nand.dts b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux-spi-nand.dts new file mode 100755 index 000000000000..de8c1a1e734f --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux-spi-nand.dts @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; + +#include "rk3568-nvr-demo-v12.dtsi" +#include "rk3568-nvr-linux.dtsi" + +/ { + model = "Rockchip RK3568 NVR DEMO DDR4 V12 Linux SPI NAND Board"; + compatible = "rockchip,rk3568-nvr-demo-ddr4-v12-linux-spi-nand", "rockchip,rk3568"; + + chosen { + bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 ubi.mtd=3 root=ubi0:rootfs rootfstype=ubifs"; + }; + +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux.dts new file mode 100755 index 000000000000..b605c3d57d26 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12-linux.dts @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3568-nvr-demo-v12.dtsi" +#include "rk3568-nvr-linux.dtsi" + +/ { + model = "Rockchip RK3568 NVR DEMO DDR4 V12 Linux Board"; + compatible = "rockchip,rk3568-nvr-demo-ddr4-v12-linux", "rockchip,rk3568"; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12.dtsi new file mode 100755 index 000000000000..6bb50368a73b --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-demo-v12.dtsi @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3568-nvr-demo-v10.dtsi" + +/ { + model = "Rockchip RK3568 NVR DEMO V12 Board"; + compatible = "rockchip,rk3568-nvr-demo-v12", "rockchip,rk3568"; +}; + +&gmac0 { + tx_delay = <0x35>; + rx_delay = <0x2d>; +}; + +&gmac1 { + tx_delay = <0x43>; + rx_delay = <0x27>; +}; + +&gpu { + mali-supply = <&vdd_npu>; + status = "okay"; +}; + +&vdd_logic { + regulator-min-microvolt = <810000>; + regulator-max-microvolt = <1000000>; +}; + +&vdd_npu { + regulator-min-microvolt = <810000>; + regulator-max-microvolt = <1100000>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr-linux.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-nvr-linux.dtsi new file mode 100755 index 000000000000..f9908b61cb41 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr-linux.dtsi @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Rockchip Electronics Co., Ltd. + * + */ + +/ { + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0xfe660000 console=ttyFIQ0 root=PARTUUID=614e0000-0000 rw rootwait"; + }; + + fiq-debugger { + compatible = "rockchip,fiq-debugger"; + rockchip,serial-id = <2>; + rockchip,wake-irq = <0>; + /* If enable uart uses irq instead of fiq */ + rockchip,irq-mode-enable = <1>; + rockchip,baudrate = <1500000>; /* Only 115200 and 1500000 */ + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "okay"; + }; + + debug: debug@fd904000 { + compatible = "rockchip,debug"; + reg = <0x0 0xfd904000 0x0 0x1000>, + <0x0 0xfd905000 0x0 0x1000>, + <0x0 0xfd906000 0x0 0x1000>, + <0x0 0xfd907000 0x0 0x1000>; + }; + + cspmu: cspmu@fd90c000 { + compatible = "rockchip,cspmu"; + reg = <0x0 0xfd90c000 0x0 0x1000>, + <0x0 0xfd90d000 0x0 0x1000>, + <0x0 0xfd90e000 0x0 0x1000>, + <0x0 0xfd90f000 0x0 0x1000>; + }; +}; + +&reserved_memory { + ramoops: ramoops@110000 { + compatible = "ramoops"; + reg = <0x0 0x110000 0x0 0xf0000>; + record-size = <0x20000>; + console-size = <0x80000>; + ftrace-size = <0x00000>; + pmsg-size = <0x50000>; + }; +}; + +&rng { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nvr.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-nvr.dtsi new file mode 100755 index 000000000000..e1ec294b2c82 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-nvr.dtsi @@ -0,0 +1,510 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include "rk3568.dtsi" +#include +#include +#include +#include +#include + +/ { + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <297500>; + }; + }; + + dc_12v: dc-12v { + compatible = "regulator-fixed"; + regulator-name = "dc_12v"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + }; + + hdmi_sound: hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "hdmi-sound"; + status = "okay"; + + simple-audio-card,cpu { + sound-dai = <&i2s0_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + rknpu_reserved: rknpu { + compatible = "shared-dma-pool"; + inactive; + reusable; + size = <0x0 0x20000000>; + alignment = <0x0 0x1000>; + }; + }; + + spdif-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "okay"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&dc_12v>; + }; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&dc_12v>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_host_en>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vcc_1v8: vcc_1v8 { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc5v0_sys>; + }; + + vcc_3v3: vcc_3v3{ + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc5v0_sys>; + }; + + vdd_fixed: vdd-fixed { + compatible = "regulator-fixed"; + regulator-name = "vdd_fixed"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + }; + + vdd_logic: vdd-logic { + compatible = "pwm-regulator"; + pwms = <&pwm1 0 5000 1>; + regulator-name = "vdd_logic"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1200000>; + regulator-init-microvolt = <900000>; + regulator-always-on; + regulator-boot-on; + regulator-settling-time-up-us = <250>; + pwm-supply = <&vcc5v0_sys>; + status = "okay"; + }; + + vdd_npu: vdd-npu { + compatible = "pwm-regulator"; + pwms = <&pwm2 0 5000 1>; + regulator-name = "vdd_npu"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1200000>; + regulator-init-microvolt = <950000>; + regulator-always-on; + regulator-boot-on; + regulator-settling-time-up-us = <250>; + pwm-supply = <&vcc5v0_sys>; + status = "okay"; + }; +}; + +&bus_npu { + bus-supply = <&vdd_logic>; + pvtm-supply = <&vdd_cpu>; + status = "okay"; +}; + +&combphy0_us { + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu0_opp_table { + /delete-node/ opp-1992000000; +}; + +&CPU_SLEEP { + status = "disabled"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "okay"; +}; + +&edp { + hpd-gpios = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&edp_in_vp0 { + status = "disabled"; +}; + +&edp_in_vp1 { + status = "okay"; +}; + +&edp_phy { + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_fixed>; + status = "okay"; +}; + +&gpu_opp_table { + /delete-node/ opp-800000000; +}; + +&hdmi { + status = "okay"; +}; + +&hdmi_in_vp0 { + status = "okay"; +}; + +&hdmi_in_vp1 { + status = "disabled"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; +}; + +&i2s0_8ch { + status = "okay"; +}; + +&i2s1_8ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +/* Need to be modified according to the actual hardware */ +&pmu_io_domains { + status = "okay"; + pmuio2-supply = <&vcc_3v3>; + vccio1-supply = <&vcc_3v3>; + vccio3-supply = <&vcc_3v3>; + vccio4-supply = <&vcc_3v3>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc_3v3>; + vccio7-supply = <&vcc_3v3>; +}; + +&pwm1 { + status = "okay"; + pinctrl-names = "active"; +}; + +&pwm2 { + status = "okay"; + pinctrl-names = "active"; +}; + +&rk_rga { + status = "okay"; +}; + +&rknpu { + memory-region = <&rknpu_reserved>; + rknpu-supply = <&vdd_npu>; + status = "okay"; +}; + +&rknpu_mmu { + status = "disabled"; +}; + +&rkvdec { + rockchip,disable-auto-freq; + assigned-clock-rates = <396000000>, <396000000>, <396000000>, <600000000>; + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvdec_sram { + reg = <0x0 0x10000>; +}; + +&rkvenc { + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&route_hdmi { + status = "okay"; + connect = <&vp0_out_hdmi>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + status = "okay"; +}; + +&sfc { + status = "okay"; +}; + +&spdif_8ch { + status = "okay"; +}; + +&sram { + reg = <0x0 0xfdcc0000 0x0 0x10000>; + ranges = <0x0 0x0 0xfdcc0000 0x10000>; +}; + +&tsadc { + status = "okay"; +}; + +&u2phy0_host { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&u2phy0_otg { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&u2phy1_host { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&u2phy1_otg { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usb2phy1 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd_dwc3 { + dr_mode = "otg"; + extcon = <&usb2phy0>; + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&usbhost_dwc3 { + phys = <&u2phy0_host>; + phy-names = "usb2-phy"; + maximum-speed = "high-speed"; + status = "okay"; +}; + +&usbhost30 { + status = "okay"; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP1>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; + +&vop_mmu { + status = "okay"; +}; + +&pinctrl { + usb { + vcc5v0_host_en: vcc5v0-host-en { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi new file mode 100755 index 000000000000..30d8cedfbdc7 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-pinctrl.dtsi @@ -0,0 +1,3119 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +#include +#include "rockchip-pinconf.dtsi" + +/* + * This file is auto generated by pin2dts tool, please keep these code + * by adding changes at end of this file. + */ +&pinctrl { + acodec { + /omit-if-no-ref/ + acodec_pins: acodec-pins { + rockchip,pins = + /* acodec_adc_sync */ + <1 RK_PB1 5 &pcfg_pull_none>, + /* acodec_adcclk */ + <1 RK_PA1 5 &pcfg_pull_none>, + /* acodec_adcdata */ + <1 RK_PA0 5 &pcfg_pull_none>, + /* acodec_dac_datal */ + <1 RK_PA7 5 &pcfg_pull_none>, + /* acodec_dac_datar */ + <1 RK_PB0 5 &pcfg_pull_none>, + /* acodec_dacclk */ + <1 RK_PA3 5 &pcfg_pull_none>, + /* acodec_dacsync */ + <1 RK_PA5 5 &pcfg_pull_none>; + }; + }; + + audiopwm { + /omit-if-no-ref/ + audiopwm_lout: audiopwm-lout { + rockchip,pins = + /* audiopwm_lout */ + <1 RK_PA0 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + audiopwm_loutn: audiopwm-loutn { + rockchip,pins = + /* audiopwm_loutn */ + <1 RK_PA1 6 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + audiopwm_loutp: audiopwm-loutp { + rockchip,pins = + /* audiopwm_loutp */ + <1 RK_PA0 6 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + audiopwm_rout: audiopwm-rout { + rockchip,pins = + /* audiopwm_rout */ + <1 RK_PA1 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + audiopwm_routn: audiopwm-routn { + rockchip,pins = + /* audiopwm_routn */ + <1 RK_PA7 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + audiopwm_routp: audiopwm-routp { + rockchip,pins = + /* audiopwm_routp */ + <1 RK_PA6 4 &pcfg_pull_none>; + }; + }; + + bt656 { + /omit-if-no-ref/ + bt656m0_pins: bt656m0-pins { + rockchip,pins = + /* bt656_clkm0 */ + <3 RK_PA0 2 &pcfg_pull_none>, + /* bt656_d0m0 */ + <2 RK_PD0 2 &pcfg_pull_none>, + /* bt656_d1m0 */ + <2 RK_PD1 2 &pcfg_pull_none>, + /* bt656_d2m0 */ + <2 RK_PD2 2 &pcfg_pull_none>, + /* bt656_d3m0 */ + <2 RK_PD3 2 &pcfg_pull_none>, + /* bt656_d4m0 */ + <2 RK_PD4 2 &pcfg_pull_none>, + /* bt656_d5m0 */ + <2 RK_PD5 2 &pcfg_pull_none>, + /* bt656_d6m0 */ + <2 RK_PD6 2 &pcfg_pull_none>, + /* bt656_d7m0 */ + <2 RK_PD7 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + bt656m1_pins: bt656m1-pins { + rockchip,pins = + /* bt656_clkm1 */ + <4 RK_PB4 5 &pcfg_pull_none>, + /* bt656_d0m1 */ + <3 RK_PC6 5 &pcfg_pull_none>, + /* bt656_d1m1 */ + <3 RK_PC7 5 &pcfg_pull_none>, + /* bt656_d2m1 */ + <3 RK_PD0 5 &pcfg_pull_none>, + /* bt656_d3m1 */ + <3 RK_PD1 5 &pcfg_pull_none>, + /* bt656_d4m1 */ + <3 RK_PD2 5 &pcfg_pull_none>, + /* bt656_d5m1 */ + <3 RK_PD3 5 &pcfg_pull_none>, + /* bt656_d6m1 */ + <3 RK_PD4 5 &pcfg_pull_none>, + /* bt656_d7m1 */ + <3 RK_PD5 5 &pcfg_pull_none>; + }; + }; + + bt1120 { + /omit-if-no-ref/ + bt1120_pins: bt1120-pins { + rockchip,pins = + /* bt1120_clk */ + <3 RK_PA6 2 &pcfg_pull_none>, + /* bt1120_d0 */ + <3 RK_PA1 2 &pcfg_pull_none>, + /* bt1120_d1 */ + <3 RK_PA2 2 &pcfg_pull_none>, + /* bt1120_d2 */ + <3 RK_PA3 2 &pcfg_pull_none>, + /* bt1120_d3 */ + <3 RK_PA4 2 &pcfg_pull_none>, + /* bt1120_d4 */ + <3 RK_PA5 2 &pcfg_pull_none>, + /* bt1120_d5 */ + <3 RK_PA7 2 &pcfg_pull_none>, + /* bt1120_d6 */ + <3 RK_PB0 2 &pcfg_pull_none>, + /* bt1120_d7 */ + <3 RK_PB1 2 &pcfg_pull_none>, + /* bt1120_d8 */ + <3 RK_PB2 2 &pcfg_pull_none>, + /* bt1120_d9 */ + <3 RK_PB3 2 &pcfg_pull_none>, + /* bt1120_d10 */ + <3 RK_PB4 2 &pcfg_pull_none>, + /* bt1120_d11 */ + <3 RK_PB5 2 &pcfg_pull_none>, + /* bt1120_d12 */ + <3 RK_PB6 2 &pcfg_pull_none>, + /* bt1120_d13 */ + <3 RK_PC1 2 &pcfg_pull_none>, + /* bt1120_d14 */ + <3 RK_PC2 2 &pcfg_pull_none>, + /* bt1120_d15 */ + <3 RK_PC3 2 &pcfg_pull_none>; + }; + }; + + cam { + /omit-if-no-ref/ + cam_clkout0: cam-clkout0 { + rockchip,pins = + /* cam_clkout0 */ + <4 RK_PA7 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + cam_clkout1: cam-clkout1 { + rockchip,pins = + /* cam_clkout1 */ + <4 RK_PB0 1 &pcfg_pull_none>; + }; + }; + + can0 { + /omit-if-no-ref/ + can0m0_pins: can0m0-pins { + rockchip,pins = + /* can0_rxm0 */ + <0 RK_PB4 2 &pcfg_pull_none>, + /* can0_txm0 */ + <0 RK_PB3 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + can0m1_pins: can0m1-pins { + rockchip,pins = + /* can0_rxm1 */ + <2 RK_PA2 4 &pcfg_pull_none>, + /* can0_txm1 */ + <2 RK_PA1 4 &pcfg_pull_none>; + }; + }; + + can1 { + /omit-if-no-ref/ + can1m0_pins: can1m0-pins { + rockchip,pins = + /* can1_rxm0 */ + <1 RK_PA0 3 &pcfg_pull_none>, + /* can1_txm0 */ + <1 RK_PA1 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + can1m1_pins: can1m1-pins { + rockchip,pins = + /* can1_rxm1 */ + <4 RK_PC2 3 &pcfg_pull_none>, + /* can1_txm1 */ + <4 RK_PC3 3 &pcfg_pull_none>; + }; + }; + + can2 { + /omit-if-no-ref/ + can2m0_pins: can2m0-pins { + rockchip,pins = + /* can2_rxm0 */ + <4 RK_PB4 3 &pcfg_pull_none>, + /* can2_txm0 */ + <4 RK_PB5 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + can2m1_pins: can2m1-pins { + rockchip,pins = + /* can2_rxm1 */ + <2 RK_PB1 4 &pcfg_pull_none>, + /* can2_txm1 */ + <2 RK_PB2 4 &pcfg_pull_none>; + }; + }; + + cif { + /omit-if-no-ref/ + cif_clk: cif-clk { + rockchip,pins = + /* cif_clkout */ + <4 RK_PC0 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + cif_dvp_clk: cif-dvp-clk { + rockchip,pins = + /* cif_clkin */ + <4 RK_PC1 1 &pcfg_pull_none>, + /* cif_href */ + <4 RK_PB6 1 &pcfg_pull_none>, + /* cif_vsync */ + <4 RK_PB7 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + cif_dvp_bus16: cif-dvp-bus16 { + rockchip,pins = + /* cif_d8 */ + <3 RK_PD6 1 &pcfg_pull_none>, + /* cif_d9 */ + <3 RK_PD7 1 &pcfg_pull_none>, + /* cif_d10 */ + <4 RK_PA0 1 &pcfg_pull_none>, + /* cif_d11 */ + <4 RK_PA1 1 &pcfg_pull_none>, + /* cif_d12 */ + <4 RK_PA2 1 &pcfg_pull_none>, + /* cif_d13 */ + <4 RK_PA3 1 &pcfg_pull_none>, + /* cif_d14 */ + <4 RK_PA4 1 &pcfg_pull_none>, + /* cif_d15 */ + <4 RK_PA5 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + cif_dvp_bus8: cif-dvp-bus8 { + rockchip,pins = + /* cif_d0 */ + <3 RK_PC6 1 &pcfg_pull_none>, + /* cif_d1 */ + <3 RK_PC7 1 &pcfg_pull_none>, + /* cif_d2 */ + <3 RK_PD0 1 &pcfg_pull_none>, + /* cif_d3 */ + <3 RK_PD1 1 &pcfg_pull_none>, + /* cif_d4 */ + <3 RK_PD2 1 &pcfg_pull_none>, + /* cif_d5 */ + <3 RK_PD3 1 &pcfg_pull_none>, + /* cif_d6 */ + <3 RK_PD4 1 &pcfg_pull_none>, + /* cif_d7 */ + <3 RK_PD5 1 &pcfg_pull_none>; + }; + }; + + clk32k { + /omit-if-no-ref/ + clk32k_in: clk32k-in { + rockchip,pins = + /* clk32k_in */ + <0 RK_PB0 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + clk32k_out0: clk32k-out0 { + rockchip,pins = + /* clk32k_out0 */ + <0 RK_PB0 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + clk32k_out1: clk32k-out1 { + rockchip,pins = + /* clk32k_out1 */ + <2 RK_PC6 1 &pcfg_pull_none>; + }; + }; + + cpu { + /omit-if-no-ref/ + cpu_pins: cpu-pins { + rockchip,pins = + /* cpu_avs */ + <0 RK_PB7 2 &pcfg_pull_none>; + }; + }; + + ebc { + /omit-if-no-ref/ + ebc_extern: ebc-extern { + rockchip,pins = + /* ebc_sdce1 */ + <4 RK_PA7 2 &pcfg_pull_none>, + /* ebc_sdce2 */ + <4 RK_PB0 2 &pcfg_pull_none>, + /* ebc_sdce3 */ + <4 RK_PB1 2 &pcfg_pull_none>, + /* ebc_sdshr */ + <4 RK_PB5 2 &pcfg_pull_none>, + /* ebc_vcom */ + <4 RK_PB2 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + ebc_pins: ebc-pins { + rockchip,pins = + /* ebc_gdclk */ + <4 RK_PC0 2 &pcfg_pull_none>, + /* ebc_gdoe */ + <4 RK_PB3 2 &pcfg_pull_none>, + /* ebc_gdsp */ + <4 RK_PB4 2 &pcfg_pull_none>, + /* ebc_sdce0 */ + <4 RK_PA6 2 &pcfg_pull_none>, + /* ebc_sdclk */ + <4 RK_PC1 2 &pcfg_pull_none>, + /* ebc_sddo0 */ + <3 RK_PC6 2 &pcfg_pull_none>, + /* ebc_sddo1 */ + <3 RK_PC7 2 &pcfg_pull_none>, + /* ebc_sddo2 */ + <3 RK_PD0 2 &pcfg_pull_none>, + /* ebc_sddo3 */ + <3 RK_PD1 2 &pcfg_pull_none>, + /* ebc_sddo4 */ + <3 RK_PD2 2 &pcfg_pull_none>, + /* ebc_sddo5 */ + <3 RK_PD3 2 &pcfg_pull_none>, + /* ebc_sddo6 */ + <3 RK_PD4 2 &pcfg_pull_none>, + /* ebc_sddo7 */ + <3 RK_PD5 2 &pcfg_pull_none>, + /* ebc_sddo8 */ + <3 RK_PD6 2 &pcfg_pull_none>, + /* ebc_sddo9 */ + <3 RK_PD7 2 &pcfg_pull_none>, + /* ebc_sddo10 */ + <4 RK_PA0 2 &pcfg_pull_none>, + /* ebc_sddo11 */ + <4 RK_PA1 2 &pcfg_pull_none>, + /* ebc_sddo12 */ + <4 RK_PA2 2 &pcfg_pull_none>, + /* ebc_sddo13 */ + <4 RK_PA3 2 &pcfg_pull_none>, + /* ebc_sddo14 */ + <4 RK_PA4 2 &pcfg_pull_none>, + /* ebc_sddo15 */ + <4 RK_PA5 2 &pcfg_pull_none>, + /* ebc_sdle */ + <4 RK_PB6 2 &pcfg_pull_none>, + /* ebc_sdoe */ + <4 RK_PB7 2 &pcfg_pull_none>; + }; + }; + + edpdp { + /omit-if-no-ref/ + edpdpm0_pins: edpdpm0-pins { + rockchip,pins = + /* edpdp_hpdinm0 */ + <4 RK_PC4 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + edpdpm1_pins: edpdpm1-pins { + rockchip,pins = + /* edpdp_hpdinm1 */ + <0 RK_PC2 2 &pcfg_pull_none>; + }; + }; + + emmc { + /omit-if-no-ref/ + emmc_rstnout: emmc-rstnout { + rockchip,pins = + /* emmc_rstn */ + <1 RK_PC7 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + emmc_bus8: emmc-bus8 { + rockchip,pins = + /* emmc_d0 */ + <1 RK_PB4 1 &pcfg_pull_up_drv_level_2>, + /* emmc_d1 */ + <1 RK_PB5 1 &pcfg_pull_up_drv_level_2>, + /* emmc_d2 */ + <1 RK_PB6 1 &pcfg_pull_up_drv_level_2>, + /* emmc_d3 */ + <1 RK_PB7 1 &pcfg_pull_up_drv_level_2>, + /* emmc_d4 */ + <1 RK_PC0 1 &pcfg_pull_up_drv_level_2>, + /* emmc_d5 */ + <1 RK_PC1 1 &pcfg_pull_up_drv_level_2>, + /* emmc_d6 */ + <1 RK_PC2 1 &pcfg_pull_up_drv_level_2>, + /* emmc_d7 */ + <1 RK_PC3 1 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + emmc_clk: emmc-clk { + rockchip,pins = + /* emmc_clkout */ + <1 RK_PC5 1 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + emmc_cmd: emmc-cmd { + rockchip,pins = + /* emmc_cmd */ + <1 RK_PC4 1 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + emmc_datastrobe: emmc-datastrobe { + rockchip,pins = + /* emmc_datastrobe */ + <1 RK_PC6 1 &pcfg_pull_none>; + }; + }; + + eth0 { + /omit-if-no-ref/ + eth0_pins: eth0-pins { + rockchip,pins = + /* eth0_refclko25m */ + <2 RK_PC1 2 &pcfg_pull_none>; + }; + }; + + eth1 { + /omit-if-no-ref/ + eth1m0_pins: eth1m0-pins { + rockchip,pins = + /* eth1_refclko25mm0 */ + <3 RK_PB0 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + eth1m1_pins: eth1m1-pins { + rockchip,pins = + /* eth1_refclko25mm1 */ + <4 RK_PB3 3 &pcfg_pull_none>; + }; + }; + + flash { + /omit-if-no-ref/ + flash_pins: flash-pins { + rockchip,pins = + /* flash_ale */ + <1 RK_PD0 2 &pcfg_pull_none>, + /* flash_cle */ + <1 RK_PC6 3 &pcfg_pull_none>, + /* flash_cs0n */ + <1 RK_PD3 2 &pcfg_pull_none>, + /* flash_cs1n */ + <1 RK_PD4 2 &pcfg_pull_none>, + /* flash_d0 */ + <1 RK_PB4 2 &pcfg_pull_none>, + /* flash_d1 */ + <1 RK_PB5 2 &pcfg_pull_none>, + /* flash_d2 */ + <1 RK_PB6 2 &pcfg_pull_none>, + /* flash_d3 */ + <1 RK_PB7 2 &pcfg_pull_none>, + /* flash_d4 */ + <1 RK_PC0 2 &pcfg_pull_none>, + /* flash_d5 */ + <1 RK_PC1 2 &pcfg_pull_none>, + /* flash_d6 */ + <1 RK_PC2 2 &pcfg_pull_none>, + /* flash_d7 */ + <1 RK_PC3 2 &pcfg_pull_none>, + /* flash_dqs */ + <1 RK_PC5 2 &pcfg_pull_none>, + /* flash_rdn */ + <1 RK_PD2 2 &pcfg_pull_none>, + /* flash_rdy */ + <1 RK_PD1 2 &pcfg_pull_none>, + /* flash_volsel */ + <0 RK_PA7 1 &pcfg_pull_none>, + /* flash_wpn */ + <1 RK_PC7 3 &pcfg_pull_none>, + /* flash_wrn */ + <1 RK_PC4 2 &pcfg_pull_none>; + }; + }; + + fspi { + /omit-if-no-ref/ + fspi_pins: fspi-pins { + rockchip,pins = + /* fspi_clk */ + <1 RK_PD0 1 &pcfg_pull_none>, + /* fspi_cs0n */ + <1 RK_PD3 1 &pcfg_pull_none>, + /* fspi_d0 */ + <1 RK_PD1 1 &pcfg_pull_none>, + /* fspi_d1 */ + <1 RK_PD2 1 &pcfg_pull_none>, + /* fspi_d2 */ + <1 RK_PC7 2 &pcfg_pull_none>, + /* fspi_d3 */ + <1 RK_PD4 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + fspi_cs1: fspi-cs1 { + rockchip,pins = + /* fspi_cs1n */ + <1 RK_PC6 2 &pcfg_pull_up>; + }; + }; + + gmac0 { + /omit-if-no-ref/ + gmac0_miim: gmac0-miim { + rockchip,pins = + /* gmac0_mdc */ + <2 RK_PC3 2 &pcfg_pull_none>, + /* gmac0_mdio */ + <2 RK_PC4 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac0_clkinout: gmac0-clkinout { + rockchip,pins = + /* gmac0_mclkinout */ + <2 RK_PC2 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac0_rx_er: gmac0-rx-er { + rockchip,pins = + /* gmac0_rxer */ + <2 RK_PC5 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac0_rx_bus2: gmac0-rx-bus2 { + rockchip,pins = + /* gmac0_rxd0 */ + <2 RK_PB6 1 &pcfg_pull_none>, + /* gmac0_rxd1 */ + <2 RK_PB7 2 &pcfg_pull_none>, + /* gmac0_rxdvcrs */ + <2 RK_PC0 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac0_tx_bus2: gmac0-tx-bus2 { + rockchip,pins = + /* gmac0_txd0 */ + <2 RK_PB3 1 &pcfg_pull_none_drv_level_2>, + /* gmac0_txd1 */ + <2 RK_PB4 1 &pcfg_pull_none_drv_level_2>, + /* gmac0_txen */ + <2 RK_PB5 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac0_rgmii_clk: gmac0-rgmii-clk { + rockchip,pins = + /* gmac0_rxclk */ + <2 RK_PA5 2 &pcfg_pull_none>, + /* gmac0_txclk */ + <2 RK_PB0 2 &pcfg_pull_none_drv_level_1>; + }; + + /omit-if-no-ref/ + gmac0_rgmii_bus: gmac0-rgmii-bus { + rockchip,pins = + /* gmac0_rxd2 */ + <2 RK_PA3 2 &pcfg_pull_none>, + /* gmac0_rxd3 */ + <2 RK_PA4 2 &pcfg_pull_none>, + /* gmac0_txd2 */ + <2 RK_PA6 2 &pcfg_pull_none_drv_level_2>, + /* gmac0_txd3 */ + <2 RK_PA7 2 &pcfg_pull_none_drv_level_2>; + }; + }; + + gmac1 { + /omit-if-no-ref/ + gmac1m0_miim: gmac1m0-miim { + rockchip,pins = + /* gmac1_mdcm0 */ + <3 RK_PC4 3 &pcfg_pull_none>, + /* gmac1_mdiom0 */ + <3 RK_PC5 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m0_clkinout: gmac1m0-clkinout { + rockchip,pins = + /* gmac1_mclkinoutm0 */ + <3 RK_PC0 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m0_rx_er: gmac1m0-rx-er { + rockchip,pins = + /* gmac1_rxerm0 */ + <3 RK_PB4 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m0_rx_bus2: gmac1m0-rx-bus2 { + rockchip,pins = + /* gmac1_rxd0m0 */ + <3 RK_PB1 3 &pcfg_pull_none>, + /* gmac1_rxd1m0 */ + <3 RK_PB2 3 &pcfg_pull_none>, + /* gmac1_rxdvcrsm0 */ + <3 RK_PB3 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m0_tx_bus2: gmac1m0-tx-bus2 { + rockchip,pins = + /* gmac1_txd0m0 */ + <3 RK_PB5 3 &pcfg_pull_none_drv_level_2>, + /* gmac1_txd1m0 */ + <3 RK_PB6 3 &pcfg_pull_none_drv_level_2>, + /* gmac1_txenm0 */ + <3 RK_PB7 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m0_rgmii_clk: gmac1m0-rgmii-clk { + rockchip,pins = + /* gmac1_rxclkm0 */ + <3 RK_PA7 3 &pcfg_pull_none>, + /* gmac1_txclkm0 */ + <3 RK_PA6 3 &pcfg_pull_none_drv_level_1>; + }; + + /omit-if-no-ref/ + gmac1m0_rgmii_bus: gmac1m0-rgmii-bus { + rockchip,pins = + /* gmac1_rxd2m0 */ + <3 RK_PA4 3 &pcfg_pull_none>, + /* gmac1_rxd3m0 */ + <3 RK_PA5 3 &pcfg_pull_none>, + /* gmac1_txd2m0 */ + <3 RK_PA2 3 &pcfg_pull_none_drv_level_2>, + /* gmac1_txd3m0 */ + <3 RK_PA3 3 &pcfg_pull_none_drv_level_2>; + }; + + /omit-if-no-ref/ + gmac1m1_miim: gmac1m1-miim { + rockchip,pins = + /* gmac1_mdcm1 */ + <4 RK_PB6 3 &pcfg_pull_none>, + /* gmac1_mdiom1 */ + <4 RK_PB7 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m1_clkinout: gmac1m1-clkinout { + rockchip,pins = + /* gmac1_mclkinoutm1 */ + <4 RK_PC1 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m1_rx_er: gmac1m1-rx-er { + rockchip,pins = + /* gmac1_rxerm1 */ + <4 RK_PB2 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m1_rx_bus2: gmac1m1-rx-bus2 { + rockchip,pins = + /* gmac1_rxd0m1 */ + <4 RK_PA7 3 &pcfg_pull_none>, + /* gmac1_rxd1m1 */ + <4 RK_PB0 3 &pcfg_pull_none>, + /* gmac1_rxdvcrsm1 */ + <4 RK_PB1 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m1_tx_bus2: gmac1m1-tx-bus2 { + rockchip,pins = + /* gmac1_txd0m1 */ + <4 RK_PA4 3 &pcfg_pull_none_drv_level_2>, + /* gmac1_txd1m1 */ + <4 RK_PA5 3 &pcfg_pull_none_drv_level_2>, + /* gmac1_txenm1 */ + <4 RK_PA6 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m1_rgmii_clk: gmac1m1-rgmii-clk { + rockchip,pins = + /* gmac1_rxclkm1 */ + <4 RK_PA3 3 &pcfg_pull_none>, + /* gmac1_txclkm1 */ + <4 RK_PA0 3 &pcfg_pull_none_drv_level_1>; + }; + + /omit-if-no-ref/ + gmac1m1_rgmii_bus: gmac1m1-rgmii-bus { + rockchip,pins = + /* gmac1_rxd2m1 */ + <4 RK_PA1 3 &pcfg_pull_none>, + /* gmac1_rxd3m1 */ + <4 RK_PA2 3 &pcfg_pull_none>, + /* gmac1_txd2m1 */ + <3 RK_PD6 3 &pcfg_pull_none_drv_level_2>, + /* gmac1_txd3m1 */ + <3 RK_PD7 3 &pcfg_pull_none_drv_level_2>; + }; + }; + + gpu { + /omit-if-no-ref/ + gpu_pins: gpu-pins { + rockchip,pins = + /* gpu_avs */ + <0 RK_PC0 2 &pcfg_pull_none>, + /* gpu_pwren */ + <0 RK_PA6 4 &pcfg_pull_none>; + }; + }; + + hdmitx { + /omit-if-no-ref/ + hdmitxm0_cec: hdmitxm0-cec { + rockchip,pins = + /* hdmitxm0_cec */ + <4 RK_PD1 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + hdmitxm1_cec: hdmitxm1-cec { + rockchip,pins = + /* hdmitxm1_cec */ + <0 RK_PC7 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + hdmitx_scl: hdmitx-scl { + rockchip,pins = + /* hdmitx_scl */ + <4 RK_PC7 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + hdmitx_sda: hdmitx-sda { + rockchip,pins = + /* hdmitx_sda */ + <4 RK_PD0 1 &pcfg_pull_none>; + }; + }; + + i2c0 { + /omit-if-no-ref/ + i2c0_xfer: i2c0-xfer { + rockchip,pins = + /* i2c0_scl */ + <0 RK_PB1 1 &pcfg_pull_none_smt>, + /* i2c0_sda */ + <0 RK_PB2 1 &pcfg_pull_none_smt>; + }; + }; + + i2c1 { + /omit-if-no-ref/ + i2c1_xfer: i2c1-xfer { + rockchip,pins = + /* i2c1_scl */ + <0 RK_PB3 1 &pcfg_pull_none_smt>, + /* i2c1_sda */ + <0 RK_PB4 1 &pcfg_pull_none_smt>; + }; + }; + + i2c2 { + /omit-if-no-ref/ + i2c2m0_xfer: i2c2m0-xfer { + rockchip,pins = + /* i2c2_sclm0 */ + <0 RK_PB5 1 &pcfg_pull_none_smt>, + /* i2c2_sdam0 */ + <0 RK_PB6 1 &pcfg_pull_none_smt>; + }; + + /omit-if-no-ref/ + i2c2m1_xfer: i2c2m1-xfer { + rockchip,pins = + /* i2c2_sclm1 */ + <4 RK_PB5 1 &pcfg_pull_none_smt>, + /* i2c2_sdam1 */ + <4 RK_PB4 1 &pcfg_pull_none_smt>; + }; + }; + + i2c3 { + /omit-if-no-ref/ + i2c3m0_xfer: i2c3m0-xfer { + rockchip,pins = + /* i2c3_sclm0 */ + <1 RK_PA1 1 &pcfg_pull_none_smt>, + /* i2c3_sdam0 */ + <1 RK_PA0 1 &pcfg_pull_none_smt>; + }; + + /omit-if-no-ref/ + i2c3m1_xfer: i2c3m1-xfer { + rockchip,pins = + /* i2c3_sclm1 */ + <3 RK_PB5 4 &pcfg_pull_none_smt>, + /* i2c3_sdam1 */ + <3 RK_PB6 4 &pcfg_pull_none_smt>; + }; + }; + + i2c4 { + /omit-if-no-ref/ + i2c4m0_xfer: i2c4m0-xfer { + rockchip,pins = + /* i2c4_sclm0 */ + <4 RK_PB3 1 &pcfg_pull_none_smt>, + /* i2c4_sdam0 */ + <4 RK_PB2 1 &pcfg_pull_none_smt>; + }; + + /omit-if-no-ref/ + i2c4m1_xfer: i2c4m1-xfer { + rockchip,pins = + /* i2c4_sclm1 */ + <2 RK_PB2 2 &pcfg_pull_none_smt>, + /* i2c4_sdam1 */ + <2 RK_PB1 2 &pcfg_pull_none_smt>; + }; + }; + + i2c5 { + /omit-if-no-ref/ + i2c5m0_xfer: i2c5m0-xfer { + rockchip,pins = + /* i2c5_sclm0 */ + <3 RK_PB3 4 &pcfg_pull_none_smt>, + /* i2c5_sdam0 */ + <3 RK_PB4 4 &pcfg_pull_none_smt>; + }; + + /omit-if-no-ref/ + i2c5m1_xfer: i2c5m1-xfer { + rockchip,pins = + /* i2c5_sclm1 */ + <4 RK_PC7 2 &pcfg_pull_none_smt>, + /* i2c5_sdam1 */ + <4 RK_PD0 2 &pcfg_pull_none_smt>; + }; + }; + + i2s1 { + /omit-if-no-ref/ + i2s1m0_lrckrx: i2s1m0-lrckrx { + rockchip,pins = + /* i2s1m0_lrckrx */ + <1 RK_PA6 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_lrcktx: i2s1m0-lrcktx { + rockchip,pins = + /* i2s1m0_lrcktx */ + <1 RK_PA5 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_mclk: i2s1m0-mclk { + rockchip,pins = + /* i2s1m0_mclk */ + <1 RK_PA2 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_sclkrx: i2s1m0-sclkrx { + rockchip,pins = + /* i2s1m0_sclkrx */ + <1 RK_PA4 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_sclktx: i2s1m0-sclktx { + rockchip,pins = + /* i2s1m0_sclktx */ + <1 RK_PA3 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_sdi0: i2s1m0-sdi0 { + rockchip,pins = + /* i2s1m0_sdi0 */ + <1 RK_PB3 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_sdi1: i2s1m0-sdi1 { + rockchip,pins = + /* i2s1m0_sdi1 */ + <1 RK_PB2 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_sdi2: i2s1m0-sdi2 { + rockchip,pins = + /* i2s1m0_sdi2 */ + <1 RK_PB1 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_sdi3: i2s1m0-sdi3 { + rockchip,pins = + /* i2s1m0_sdi3 */ + <1 RK_PB0 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_sdo0: i2s1m0-sdo0 { + rockchip,pins = + /* i2s1m0_sdo0 */ + <1 RK_PA7 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_sdo1: i2s1m0-sdo1 { + rockchip,pins = + /* i2s1m0_sdo1 */ + <1 RK_PB0 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_sdo2: i2s1m0-sdo2 { + rockchip,pins = + /* i2s1m0_sdo2 */ + <1 RK_PB1 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m0_sdo3: i2s1m0-sdo3 { + rockchip,pins = + /* i2s1m0_sdo3 */ + <1 RK_PB2 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_lrckrx: i2s1m1-lrckrx { + rockchip,pins = + /* i2s1m1_lrckrx */ + <4 RK_PA7 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_lrcktx: i2s1m1-lrcktx { + rockchip,pins = + /* i2s1m1_lrcktx */ + <3 RK_PD0 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_mclk: i2s1m1-mclk { + rockchip,pins = + /* i2s1m1_mclk */ + <3 RK_PC6 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_sclkrx: i2s1m1-sclkrx { + rockchip,pins = + /* i2s1m1_sclkrx */ + <4 RK_PA6 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_sclktx: i2s1m1-sclktx { + rockchip,pins = + /* i2s1m1_sclktx */ + <3 RK_PC7 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_sdi0: i2s1m1-sdi0 { + rockchip,pins = + /* i2s1m1_sdi0 */ + <3 RK_PD2 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_sdi1: i2s1m1-sdi1 { + rockchip,pins = + /* i2s1m1_sdi1 */ + <3 RK_PD3 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_sdi2: i2s1m1-sdi2 { + rockchip,pins = + /* i2s1m1_sdi2 */ + <3 RK_PD4 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_sdi3: i2s1m1-sdi3 { + rockchip,pins = + /* i2s1m1_sdi3 */ + <3 RK_PD5 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_sdo0: i2s1m1-sdo0 { + rockchip,pins = + /* i2s1m1_sdo0 */ + <3 RK_PD1 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_sdo1: i2s1m1-sdo1 { + rockchip,pins = + /* i2s1m1_sdo1 */ + <4 RK_PB0 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_sdo2: i2s1m1-sdo2 { + rockchip,pins = + /* i2s1m1_sdo2 */ + <4 RK_PB1 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m1_sdo3: i2s1m1-sdo3 { + rockchip,pins = + /* i2s1m1_sdo3 */ + <4 RK_PB5 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_lrckrx: i2s1m2-lrckrx { + rockchip,pins = + /* i2s1m2_lrckrx */ + <3 RK_PC5 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_lrcktx: i2s1m2-lrcktx { + rockchip,pins = + /* i2s1m2_lrcktx */ + <2 RK_PD2 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_mclk: i2s1m2-mclk { + rockchip,pins = + /* i2s1m2_mclk */ + <2 RK_PD0 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_sclkrx: i2s1m2-sclkrx { + rockchip,pins = + /* i2s1m2_sclkrx */ + <3 RK_PC3 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_sclktx: i2s1m2-sclktx { + rockchip,pins = + /* i2s1m2_sclktx */ + <2 RK_PD1 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_sdi0: i2s1m2-sdi0 { + rockchip,pins = + /* i2s1m2_sdi0 */ + <2 RK_PD3 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_sdi1: i2s1m2-sdi1 { + rockchip,pins = + /* i2s1m2_sdi1 */ + <2 RK_PD4 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_sdi2: i2s1m2-sdi2 { + rockchip,pins = + /* i2s1m2_sdi2 */ + <2 RK_PD5 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_sdi3: i2s1m2-sdi3 { + rockchip,pins = + /* i2s1m2_sdi3 */ + <2 RK_PD6 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_sdo0: i2s1m2-sdo0 { + rockchip,pins = + /* i2s1m2_sdo0 */ + <2 RK_PD7 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_sdo1: i2s1m2-sdo1 { + rockchip,pins = + /* i2s1m2_sdo1 */ + <3 RK_PA0 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_sdo2: i2s1m2-sdo2 { + rockchip,pins = + /* i2s1m2_sdo2 */ + <3 RK_PC1 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s1m2_sdo3: i2s1m2-sdo3 { + rockchip,pins = + /* i2s1m2_sdo3 */ + <3 RK_PC2 5 &pcfg_pull_none>; + }; + }; + + i2s2 { + /omit-if-no-ref/ + i2s2m0_lrckrx: i2s2m0-lrckrx { + rockchip,pins = + /* i2s2m0_lrckrx */ + <2 RK_PC0 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m0_lrcktx: i2s2m0-lrcktx { + rockchip,pins = + /* i2s2m0_lrcktx */ + <2 RK_PC3 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m0_mclk: i2s2m0-mclk { + rockchip,pins = + /* i2s2m0_mclk */ + <2 RK_PC1 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m0_sclkrx: i2s2m0-sclkrx { + rockchip,pins = + /* i2s2m0_sclkrx */ + <2 RK_PB7 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m0_sclktx: i2s2m0-sclktx { + rockchip,pins = + /* i2s2m0_sclktx */ + <2 RK_PC2 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m0_sdi: i2s2m0-sdi { + rockchip,pins = + /* i2s2m0_sdi */ + <2 RK_PC5 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m0_sdo: i2s2m0-sdo { + rockchip,pins = + /* i2s2m0_sdo */ + <2 RK_PC4 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m1_lrckrx: i2s2m1-lrckrx { + rockchip,pins = + /* i2s2m1_lrckrx */ + <4 RK_PA5 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m1_lrcktx: i2s2m1-lrcktx { + rockchip,pins = + /* i2s2m1_lrcktx */ + <4 RK_PA4 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m1_mclk: i2s2m1-mclk { + rockchip,pins = + /* i2s2m1_mclk */ + <4 RK_PB6 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m1_sclkrx: i2s2m1-sclkrx { + rockchip,pins = + /* i2s2m1_sclkrx */ + <4 RK_PC1 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m1_sclktx: i2s2m1-sclktx { + rockchip,pins = + /* i2s2m1_sclktx */ + <4 RK_PB7 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m1_sdi: i2s2m1-sdi { + rockchip,pins = + /* i2s2m1_sdi */ + <4 RK_PB2 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s2m1_sdo: i2s2m1-sdo { + rockchip,pins = + /* i2s2m1_sdo */ + <4 RK_PB3 5 &pcfg_pull_none>; + }; + }; + + i2s3 { + /omit-if-no-ref/ + i2s3m0_lrck: i2s3m0-lrck { + rockchip,pins = + /* i2s3m0_lrck */ + <3 RK_PA4 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s3m0_mclk: i2s3m0-mclk { + rockchip,pins = + /* i2s3m0_mclk */ + <3 RK_PA2 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s3m0_sclk: i2s3m0-sclk { + rockchip,pins = + /* i2s3m0_sclk */ + <3 RK_PA3 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s3m0_sdi: i2s3m0-sdi { + rockchip,pins = + /* i2s3m0_sdi */ + <3 RK_PA6 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s3m0_sdo: i2s3m0-sdo { + rockchip,pins = + /* i2s3m0_sdo */ + <3 RK_PA5 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s3m1_lrck: i2s3m1-lrck { + rockchip,pins = + /* i2s3m1_lrck */ + <4 RK_PC4 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s3m1_mclk: i2s3m1-mclk { + rockchip,pins = + /* i2s3m1_mclk */ + <4 RK_PC2 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s3m1_sclk: i2s3m1-sclk { + rockchip,pins = + /* i2s3m1_sclk */ + <4 RK_PC3 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s3m1_sdi: i2s3m1-sdi { + rockchip,pins = + /* i2s3m1_sdi */ + <4 RK_PC6 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + i2s3m1_sdo: i2s3m1-sdo { + rockchip,pins = + /* i2s3m1_sdo */ + <4 RK_PC5 5 &pcfg_pull_none>; + }; + }; + + isp { + /omit-if-no-ref/ + isp_pins: isp-pins { + rockchip,pins = + /* isp_flashtrigin */ + <4 RK_PB4 4 &pcfg_pull_none>, + /* isp_flashtrigout */ + <4 RK_PA6 1 &pcfg_pull_none>, + /* isp_prelighttrig */ + <4 RK_PB1 1 &pcfg_pull_none>; + }; + }; + + jtag { + /omit-if-no-ref/ + jtag_pins: jtag-pins { + rockchip,pins = + /* jtag_tck */ + <1 RK_PD7 2 &pcfg_pull_none>, + /* jtag_tms */ + <2 RK_PA0 2 &pcfg_pull_none>; + }; + }; + + lcdc { + /omit-if-no-ref/ + lcdc_ctl: lcdc-ctl { + rockchip,pins = + /* lcdc_clk */ + <3 RK_PA0 1 &pcfg_pull_none>, + /* lcdc_d0 */ + <2 RK_PD0 1 &pcfg_pull_none>, + /* lcdc_d1 */ + <2 RK_PD1 1 &pcfg_pull_none>, + /* lcdc_d2 */ + <2 RK_PD2 1 &pcfg_pull_none>, + /* lcdc_d3 */ + <2 RK_PD3 1 &pcfg_pull_none>, + /* lcdc_d4 */ + <2 RK_PD4 1 &pcfg_pull_none>, + /* lcdc_d5 */ + <2 RK_PD5 1 &pcfg_pull_none>, + /* lcdc_d6 */ + <2 RK_PD6 1 &pcfg_pull_none>, + /* lcdc_d7 */ + <2 RK_PD7 1 &pcfg_pull_none>, + /* lcdc_d8 */ + <3 RK_PA1 1 &pcfg_pull_none>, + /* lcdc_d9 */ + <3 RK_PA2 1 &pcfg_pull_none>, + /* lcdc_d10 */ + <3 RK_PA3 1 &pcfg_pull_none>, + /* lcdc_d11 */ + <3 RK_PA4 1 &pcfg_pull_none>, + /* lcdc_d12 */ + <3 RK_PA5 1 &pcfg_pull_none>, + /* lcdc_d13 */ + <3 RK_PA6 1 &pcfg_pull_none>, + /* lcdc_d14 */ + <3 RK_PA7 1 &pcfg_pull_none>, + /* lcdc_d15 */ + <3 RK_PB0 1 &pcfg_pull_none>, + /* lcdc_d16 */ + <3 RK_PB1 1 &pcfg_pull_none>, + /* lcdc_d17 */ + <3 RK_PB2 1 &pcfg_pull_none>, + /* lcdc_d18 */ + <3 RK_PB3 1 &pcfg_pull_none>, + /* lcdc_d19 */ + <3 RK_PB4 1 &pcfg_pull_none>, + /* lcdc_d20 */ + <3 RK_PB5 1 &pcfg_pull_none>, + /* lcdc_d21 */ + <3 RK_PB6 1 &pcfg_pull_none>, + /* lcdc_d22 */ + <3 RK_PB7 1 &pcfg_pull_none>, + /* lcdc_d23 */ + <3 RK_PC0 1 &pcfg_pull_none>, + /* lcdc_den */ + <3 RK_PC3 1 &pcfg_pull_none>, + /* lcdc_hsync */ + <3 RK_PC1 1 &pcfg_pull_none>, + /* lcdc_vsync */ + <3 RK_PC2 1 &pcfg_pull_none>; + }; + }; + + mcu { + /omit-if-no-ref/ + mcu_pins: mcu-pins { + rockchip,pins = + /* mcu_jtagtck */ + <0 RK_PB4 4 &pcfg_pull_none>, + /* mcu_jtagtdi */ + <0 RK_PC1 4 &pcfg_pull_none>, + /* mcu_jtagtdo */ + <0 RK_PB3 4 &pcfg_pull_none>, + /* mcu_jtagtms */ + <0 RK_PC2 4 &pcfg_pull_none>, + /* mcu_jtagtrstn */ + <0 RK_PC3 4 &pcfg_pull_none>; + }; + }; + + npu { + /omit-if-no-ref/ + npu_pins: npu-pins { + rockchip,pins = + /* npu_avs */ + <0 RK_PC1 2 &pcfg_pull_none>; + }; + }; + + pcie20 { + /omit-if-no-ref/ + pcie20m0_pins: pcie20m0-pins { + rockchip,pins = + /* pcie20_clkreqnm0 */ + <0 RK_PA5 3 &pcfg_pull_none>, + /* pcie20_perstnm0 */ + <0 RK_PB6 3 &pcfg_pull_none>, + /* pcie20_wakenm0 */ + <0 RK_PB5 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pcie20m1_pins: pcie20m1-pins { + rockchip,pins = + /* pcie20_clkreqnm1 */ + <2 RK_PD0 4 &pcfg_pull_none>, + /* pcie20_perstnm1 */ + <3 RK_PC1 4 &pcfg_pull_none>, + /* pcie20_wakenm1 */ + <2 RK_PD1 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pcie20m2_pins: pcie20m2-pins { + rockchip,pins = + /* pcie20_clkreqnm2 */ + <1 RK_PB0 4 &pcfg_pull_none>, + /* pcie20_perstnm2 */ + <1 RK_PB2 4 &pcfg_pull_none>, + /* pcie20_wakenm2 */ + <1 RK_PB1 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pcie20_buttonrstn: pcie20-buttonrstn { + rockchip,pins = + /* pcie20_buttonrstn */ + <0 RK_PB4 3 &pcfg_pull_none>; + }; + }; + + pcie30x1 { + /omit-if-no-ref/ + pcie30x1m0_pins: pcie30x1m0-pins { + rockchip,pins = + /* pcie30x1_clkreqnm0 */ + <0 RK_PA4 3 &pcfg_pull_none>, + /* pcie30x1_perstnm0 */ + <0 RK_PC3 3 &pcfg_pull_none>, + /* pcie30x1_wakenm0 */ + <0 RK_PC2 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pcie30x1m1_pins: pcie30x1m1-pins { + rockchip,pins = + /* pcie30x1_clkreqnm1 */ + <2 RK_PD2 4 &pcfg_pull_none>, + /* pcie30x1_perstnm1 */ + <3 RK_PA1 4 &pcfg_pull_none>, + /* pcie30x1_wakenm1 */ + <2 RK_PD3 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pcie30x1m2_pins: pcie30x1m2-pins { + rockchip,pins = + /* pcie30x1_clkreqnm2 */ + <1 RK_PA5 4 &pcfg_pull_none>, + /* pcie30x1_perstnm2 */ + <1 RK_PA2 4 &pcfg_pull_none>, + /* pcie30x1_wakenm2 */ + <1 RK_PA3 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pcie30x1_buttonrstn: pcie30x1-buttonrstn { + rockchip,pins = + /* pcie30x1_buttonrstn */ + <0 RK_PB3 3 &pcfg_pull_none>; + }; + }; + + pcie30x2 { + /omit-if-no-ref/ + pcie30x2m0_pins: pcie30x2m0-pins { + rockchip,pins = + /* pcie30x2_clkreqnm0 */ + <0 RK_PA6 2 &pcfg_pull_none>, + /* pcie30x2_perstnm0 */ + <0 RK_PC6 3 &pcfg_pull_none>, + /* pcie30x2_wakenm0 */ + <0 RK_PC5 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pcie30x2m1_pins: pcie30x2m1-pins { + rockchip,pins = + /* pcie30x2_clkreqnm1 */ + <2 RK_PD4 4 &pcfg_pull_none>, + /* pcie30x2_perstnm1 */ + <2 RK_PD6 4 &pcfg_pull_none>, + /* pcie30x2_wakenm1 */ + <2 RK_PD5 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pcie30x2m2_pins: pcie30x2m2-pins { + rockchip,pins = + /* pcie30x2_clkreqnm2 */ + <4 RK_PC2 4 &pcfg_pull_none>, + /* pcie30x2_perstnm2 */ + <4 RK_PC4 4 &pcfg_pull_none>, + /* pcie30x2_wakenm2 */ + <4 RK_PC3 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pcie30x2_buttonrstn: pcie30x2-buttonrstn { + rockchip,pins = + /* pcie30x2_buttonrstn */ + <0 RK_PB0 3 &pcfg_pull_none>; + }; + }; + + pdm { + /omit-if-no-ref/ + pdmm0_clk: pdmm0-clk { + rockchip,pins = + /* pdm_clk0m0 */ + <1 RK_PA6 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm0_clk1: pdmm0-clk1 { + rockchip,pins = + /* pdmm0_clk1 */ + <1 RK_PA4 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm0_sdi0: pdmm0-sdi0 { + rockchip,pins = + /* pdmm0_sdi0 */ + <1 RK_PB3 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm0_sdi1: pdmm0-sdi1 { + rockchip,pins = + /* pdmm0_sdi1 */ + <1 RK_PB2 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm0_sdi2: pdmm0-sdi2 { + rockchip,pins = + /* pdmm0_sdi2 */ + <1 RK_PB1 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm0_sdi3: pdmm0-sdi3 { + rockchip,pins = + /* pdmm0_sdi3 */ + <1 RK_PB0 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm1_clk: pdmm1-clk { + rockchip,pins = + /* pdm_clk0m1 */ + <3 RK_PD6 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm1_clk1: pdmm1-clk1 { + rockchip,pins = + /* pdmm1_clk1 */ + <4 RK_PA0 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm1_sdi0: pdmm1-sdi0 { + rockchip,pins = + /* pdmm1_sdi0 */ + <3 RK_PD7 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm1_sdi1: pdmm1-sdi1 { + rockchip,pins = + /* pdmm1_sdi1 */ + <4 RK_PA1 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm1_sdi2: pdmm1-sdi2 { + rockchip,pins = + /* pdmm1_sdi2 */ + <4 RK_PA2 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm1_sdi3: pdmm1-sdi3 { + rockchip,pins = + /* pdmm1_sdi3 */ + <4 RK_PA3 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm2_clk1: pdmm2-clk1 { + rockchip,pins = + /* pdmm2_clk1 */ + <3 RK_PC4 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm2_sdi0: pdmm2-sdi0 { + rockchip,pins = + /* pdmm2_sdi0 */ + <3 RK_PB3 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm2_sdi1: pdmm2-sdi1 { + rockchip,pins = + /* pdmm2_sdi1 */ + <3 RK_PB4 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm2_sdi2: pdmm2-sdi2 { + rockchip,pins = + /* pdmm2_sdi2 */ + <3 RK_PB7 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pdmm2_sdi3: pdmm2-sdi3 { + rockchip,pins = + /* pdmm2_sdi3 */ + <3 RK_PC0 5 &pcfg_pull_none>; + }; + }; + + pmic { + /omit-if-no-ref/ + pmic_pins: pmic-pins { + rockchip,pins = + /* pmic_sleep */ + <0 RK_PA2 1 &pcfg_pull_none>; + }; + }; + + pmu { + /omit-if-no-ref/ + pmu_pins: pmu-pins { + rockchip,pins = + /* pmu_debug0 */ + <0 RK_PA5 4 &pcfg_pull_none>, + /* pmu_debug1 */ + <0 RK_PA6 3 &pcfg_pull_none>, + /* pmu_debug2 */ + <0 RK_PC4 4 &pcfg_pull_none>, + /* pmu_debug3 */ + <0 RK_PC5 4 &pcfg_pull_none>, + /* pmu_debug4 */ + <0 RK_PC6 4 &pcfg_pull_none>, + /* pmu_debug5 */ + <0 RK_PC7 4 &pcfg_pull_none>; + }; + }; + + pwm0 { + /omit-if-no-ref/ + pwm0m0_pins: pwm0m0-pins { + rockchip,pins = + /* pwm0_m0 */ + <0 RK_PB7 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm0m1_pins: pwm0m1-pins { + rockchip,pins = + /* pwm0_m1 */ + <0 RK_PC7 2 &pcfg_pull_none>; + }; + }; + + pwm1 { + /omit-if-no-ref/ + pwm1m0_pins: pwm1m0-pins { + rockchip,pins = + /* pwm1_m0 */ + <0 RK_PC0 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm1m1_pins: pwm1m1-pins { + rockchip,pins = + /* pwm1_m1 */ + <0 RK_PB5 4 &pcfg_pull_none>; + }; + }; + + pwm2 { + /omit-if-no-ref/ + pwm2m0_pins: pwm2m0-pins { + rockchip,pins = + /* pwm2_m0 */ + <0 RK_PC1 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm2m1_pins: pwm2m1-pins { + rockchip,pins = + /* pwm2_m1 */ + <0 RK_PB6 4 &pcfg_pull_none>; + }; + }; + + pwm3 { + /omit-if-no-ref/ + pwm3_pins: pwm3-pins { + rockchip,pins = + /* pwm3_ir */ + <0 RK_PC2 1 &pcfg_pull_none>; + }; + }; + + pwm4 { + /omit-if-no-ref/ + pwm4_pins: pwm4-pins { + rockchip,pins = + /* pwm4 */ + <0 RK_PC3 1 &pcfg_pull_none>; + }; + }; + + pwm5 { + /omit-if-no-ref/ + pwm5_pins: pwm5-pins { + rockchip,pins = + /* pwm5 */ + <0 RK_PC4 1 &pcfg_pull_none>; + }; + }; + + pwm6 { + /omit-if-no-ref/ + pwm6_pins: pwm6-pins { + rockchip,pins = + /* pwm6 */ + <0 RK_PC5 1 &pcfg_pull_none>; + }; + }; + + pwm7 { + /omit-if-no-ref/ + pwm7_pins: pwm7-pins { + rockchip,pins = + /* pwm7_ir */ + <0 RK_PC6 1 &pcfg_pull_none>; + }; + }; + + pwm8 { + /omit-if-no-ref/ + pwm8m0_pins: pwm8m0-pins { + rockchip,pins = + /* pwm8_m0 */ + <3 RK_PB1 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm8m1_pins: pwm8m1-pins { + rockchip,pins = + /* pwm8_m1 */ + <1 RK_PD5 4 &pcfg_pull_none>; + }; + }; + + pwm9 { + /omit-if-no-ref/ + pwm9m0_pins: pwm9m0-pins { + rockchip,pins = + /* pwm9_m0 */ + <3 RK_PB2 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm9m1_pins: pwm9m1-pins { + rockchip,pins = + /* pwm9_m1 */ + <1 RK_PD6 4 &pcfg_pull_none>; + }; + }; + + pwm10 { + /omit-if-no-ref/ + pwm10m0_pins: pwm10m0-pins { + rockchip,pins = + /* pwm10_m0 */ + <3 RK_PB5 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm10m1_pins: pwm10m1-pins { + rockchip,pins = + /* pwm10_m1 */ + <2 RK_PA1 2 &pcfg_pull_none>; + }; + }; + + pwm11 { + /omit-if-no-ref/ + pwm11m0_pins: pwm11m0-pins { + rockchip,pins = + /* pwm11_irm0 */ + <3 RK_PB6 5 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm11m1_pins: pwm11m1-pins { + rockchip,pins = + /* pwm11_irm1 */ + <4 RK_PC0 3 &pcfg_pull_none>; + }; + }; + + pwm12 { + /omit-if-no-ref/ + pwm12m0_pins: pwm12m0-pins { + rockchip,pins = + /* pwm12_m0 */ + <3 RK_PB7 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm12m1_pins: pwm12m1-pins { + rockchip,pins = + /* pwm12_m1 */ + <4 RK_PC5 1 &pcfg_pull_none>; + }; + }; + + pwm13 { + /omit-if-no-ref/ + pwm13m0_pins: pwm13m0-pins { + rockchip,pins = + /* pwm13_m0 */ + <3 RK_PC0 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm13m1_pins: pwm13m1-pins { + rockchip,pins = + /* pwm13_m1 */ + <4 RK_PC6 1 &pcfg_pull_none>; + }; + }; + + pwm14 { + /omit-if-no-ref/ + pwm14m0_pins: pwm14m0-pins { + rockchip,pins = + /* pwm14_m0 */ + <3 RK_PC4 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm14m1_pins: pwm14m1-pins { + rockchip,pins = + /* pwm14_m1 */ + <4 RK_PC2 1 &pcfg_pull_none>; + }; + }; + + pwm15 { + /omit-if-no-ref/ + pwm15m0_pins: pwm15m0-pins { + rockchip,pins = + /* pwm15_irm0 */ + <3 RK_PC5 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + pwm15m1_pins: pwm15m1-pins { + rockchip,pins = + /* pwm15_irm1 */ + <4 RK_PC3 1 &pcfg_pull_none>; + }; + }; + + refclk { + /omit-if-no-ref/ + refclk_pins: refclk-pins { + rockchip,pins = + /* refclk_ou */ + <0 RK_PA0 1 &pcfg_pull_none>; + }; + }; + + sata { + /omit-if-no-ref/ + sata_pins: sata-pins { + rockchip,pins = + /* sata_cpdet */ + <0 RK_PA4 2 &pcfg_pull_none>, + /* sata_cppod */ + <0 RK_PA6 1 &pcfg_pull_none>, + /* sata_mpswitch */ + <0 RK_PA5 2 &pcfg_pull_none>; + }; + }; + + sata0 { + /omit-if-no-ref/ + sata0_pins: sata0-pins { + rockchip,pins = + /* sata0_actled */ + <4 RK_PC6 3 &pcfg_pull_none>; + }; + }; + + sata1 { + /omit-if-no-ref/ + sata1_pins: sata1-pins { + rockchip,pins = + /* sata1_actled */ + <4 RK_PC5 3 &pcfg_pull_none>; + }; + }; + + sata2 { + /omit-if-no-ref/ + sata2_pins: sata2-pins { + rockchip,pins = + /* sata2_actled */ + <4 RK_PC4 3 &pcfg_pull_none>; + }; + }; + + scr { + /omit-if-no-ref/ + scr_pins: scr-pins { + rockchip,pins = + /* scr_clk */ + <1 RK_PA2 3 &pcfg_pull_none>, + /* scr_det */ + <1 RK_PA7 3 &pcfg_pull_up>, + /* scr_io */ + <1 RK_PA3 3 &pcfg_pull_up>, + /* scr_rst */ + <1 RK_PA5 3 &pcfg_pull_none>; + }; + }; + + sdmmc0 { + /omit-if-no-ref/ + sdmmc0_bus4: sdmmc0-bus4 { + rockchip,pins = + /* sdmmc0_d0 */ + <1 RK_PD5 1 &pcfg_pull_up_drv_level_2>, + /* sdmmc0_d1 */ + <1 RK_PD6 1 &pcfg_pull_up_drv_level_2>, + /* sdmmc0_d2 */ + <1 RK_PD7 1 &pcfg_pull_up_drv_level_2>, + /* sdmmc0_d3 */ + <2 RK_PA0 1 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc0_clk: sdmmc0-clk { + rockchip,pins = + /* sdmmc0_clk */ + <2 RK_PA2 1 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc0_cmd: sdmmc0-cmd { + rockchip,pins = + /* sdmmc0_cmd */ + <2 RK_PA1 1 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc0_det: sdmmc0-det { + rockchip,pins = + /* sdmmc0_det */ + <0 RK_PA4 1 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + sdmmc0_pwren: sdmmc0-pwren { + rockchip,pins = + /* sdmmc0_pwren */ + <0 RK_PA5 1 &pcfg_pull_none>; + }; + }; + + sdmmc1 { + /omit-if-no-ref/ + sdmmc1_bus4: sdmmc1-bus4 { + rockchip,pins = + /* sdmmc1_d0 */ + <2 RK_PA3 1 &pcfg_pull_up_drv_level_2>, + /* sdmmc1_d1 */ + <2 RK_PA4 1 &pcfg_pull_up_drv_level_2>, + /* sdmmc1_d2 */ + <2 RK_PA5 1 &pcfg_pull_up_drv_level_2>, + /* sdmmc1_d3 */ + <2 RK_PA6 1 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc1_clk: sdmmc1-clk { + rockchip,pins = + /* sdmmc1_clk */ + <2 RK_PB0 1 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc1_cmd: sdmmc1-cmd { + rockchip,pins = + /* sdmmc1_cmd */ + <2 RK_PA7 1 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc1_det: sdmmc1-det { + rockchip,pins = + /* sdmmc1_det */ + <2 RK_PB2 1 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + sdmmc1_pwren: sdmmc1-pwren { + rockchip,pins = + /* sdmmc1_pwren */ + <2 RK_PB1 1 &pcfg_pull_none>; + }; + }; + + sdmmc2 { + /omit-if-no-ref/ + sdmmc2m0_bus4: sdmmc2m0-bus4 { + rockchip,pins = + /* sdmmc2_d0m0 */ + <3 RK_PC6 3 &pcfg_pull_up_drv_level_2>, + /* sdmmc2_d1m0 */ + <3 RK_PC7 3 &pcfg_pull_up_drv_level_2>, + /* sdmmc2_d2m0 */ + <3 RK_PD0 3 &pcfg_pull_up_drv_level_2>, + /* sdmmc2_d3m0 */ + <3 RK_PD1 3 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc2m0_clk: sdmmc2m0-clk { + rockchip,pins = + /* sdmmc2_clkm0 */ + <3 RK_PD3 3 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc2m0_cmd: sdmmc2m0-cmd { + rockchip,pins = + /* sdmmc2_cmdm0 */ + <3 RK_PD2 3 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc2m0_det: sdmmc2m0-det { + rockchip,pins = + /* sdmmc2_detm0 */ + <3 RK_PD4 3 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + sdmmc2m0_pwren: sdmmc2m0-pwren { + rockchip,pins = + /* sdmmc2m0_pwren */ + <3 RK_PD5 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + sdmmc2m1_bus4: sdmmc2m1-bus4 { + rockchip,pins = + /* sdmmc2_d0m1 */ + <3 RK_PA1 5 &pcfg_pull_up_drv_level_2>, + /* sdmmc2_d1m1 */ + <3 RK_PA2 5 &pcfg_pull_up_drv_level_2>, + /* sdmmc2_d2m1 */ + <3 RK_PA3 5 &pcfg_pull_up_drv_level_2>, + /* sdmmc2_d3m1 */ + <3 RK_PA4 5 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc2m1_clk: sdmmc2m1-clk { + rockchip,pins = + /* sdmmc2_clkm1 */ + <3 RK_PA6 5 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc2m1_cmd: sdmmc2m1-cmd { + rockchip,pins = + /* sdmmc2_cmdm1 */ + <3 RK_PA5 5 &pcfg_pull_up_drv_level_2>; + }; + + /omit-if-no-ref/ + sdmmc2m1_det: sdmmc2m1-det { + rockchip,pins = + /* sdmmc2_detm1 */ + <3 RK_PA7 4 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + sdmmc2m1_pwren: sdmmc2m1-pwren { + rockchip,pins = + /* sdmmc2m1_pwren */ + <3 RK_PB0 4 &pcfg_pull_none>; + }; + }; + + spdif { + /omit-if-no-ref/ + spdifm0_tx: spdifm0-tx { + rockchip,pins = + /* spdifm0_tx */ + <1 RK_PA4 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spdifm1_tx: spdifm1-tx { + rockchip,pins = + /* spdifm1_tx */ + <3 RK_PC5 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spdifm2_tx: spdifm2-tx { + rockchip,pins = + /* spdifm2_tx */ + <4 RK_PC4 2 &pcfg_pull_none>; + }; + }; + + spi0 { + /omit-if-no-ref/ + spi0m0_pins: spi0m0-pins { + rockchip,pins = + /* spi0_clkm0 */ + <0 RK_PB5 2 &pcfg_pull_none>, + /* spi0_misom0 */ + <0 RK_PC5 2 &pcfg_pull_none>, + /* spi0_mosim0 */ + <0 RK_PB6 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi0m0_cs0: spi0m0-cs0 { + rockchip,pins = + /* spi0_cs0m0 */ + <0 RK_PC6 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi0m0_cs1: spi0m0-cs1 { + rockchip,pins = + /* spi0_cs1m0 */ + <0 RK_PC4 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi0m1_pins: spi0m1-pins { + rockchip,pins = + /* spi0_clkm1 */ + <2 RK_PD3 3 &pcfg_pull_none>, + /* spi0_misom1 */ + <2 RK_PD0 3 &pcfg_pull_none>, + /* spi0_mosim1 */ + <2 RK_PD1 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi0m1_cs0: spi0m1-cs0 { + rockchip,pins = + /* spi0_cs0m1 */ + <2 RK_PD2 3 &pcfg_pull_none>; + }; + }; + + spi1 { + /omit-if-no-ref/ + spi1m0_pins: spi1m0-pins { + rockchip,pins = + /* spi1_clkm0 */ + <2 RK_PB5 3 &pcfg_pull_none>, + /* spi1_misom0 */ + <2 RK_PB6 3 &pcfg_pull_none>, + /* spi1_mosim0 */ + <2 RK_PB7 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi1m0_cs0: spi1m0-cs0 { + rockchip,pins = + /* spi1_cs0m0 */ + <2 RK_PC0 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi1m0_cs1: spi1m0-cs1 { + rockchip,pins = + /* spi1_cs1m0 */ + <2 RK_PC6 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi1m1_pins: spi1m1-pins { + rockchip,pins = + /* spi1_clkm1 */ + <3 RK_PC3 3 &pcfg_pull_none>, + /* spi1_misom1 */ + <3 RK_PC2 3 &pcfg_pull_none>, + /* spi1_mosim1 */ + <3 RK_PC1 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi1m1_cs0: spi1m1-cs0 { + rockchip,pins = + /* spi1_cs0m1 */ + <3 RK_PA1 3 &pcfg_pull_none>; + }; + }; + + spi2 { + /omit-if-no-ref/ + spi2m0_pins: spi2m0-pins { + rockchip,pins = + /* spi2_clkm0 */ + <2 RK_PC1 4 &pcfg_pull_none>, + /* spi2_misom0 */ + <2 RK_PC2 4 &pcfg_pull_none>, + /* spi2_mosim0 */ + <2 RK_PC3 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi2m0_cs0: spi2m0-cs0 { + rockchip,pins = + /* spi2_cs0m0 */ + <2 RK_PC4 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi2m0_cs1: spi2m0-cs1 { + rockchip,pins = + /* spi2_cs1m0 */ + <2 RK_PC5 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi2m1_pins: spi2m1-pins { + rockchip,pins = + /* spi2_clkm1 */ + <3 RK_PA0 3 &pcfg_pull_none>, + /* spi2_misom1 */ + <2 RK_PD7 3 &pcfg_pull_none>, + /* spi2_mosim1 */ + <2 RK_PD6 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi2m1_cs0: spi2m1-cs0 { + rockchip,pins = + /* spi2_cs0m1 */ + <2 RK_PD5 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi2m1_cs1: spi2m1-cs1 { + rockchip,pins = + /* spi2_cs1m1 */ + <2 RK_PD4 3 &pcfg_pull_none>; + }; + }; + + spi3 { + /omit-if-no-ref/ + spi3m0_pins: spi3m0-pins { + rockchip,pins = + /* spi3_clkm0 */ + <4 RK_PB3 4 &pcfg_pull_none>, + /* spi3_misom0 */ + <4 RK_PB0 4 &pcfg_pull_none>, + /* spi3_mosim0 */ + <4 RK_PB2 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi3m0_cs0: spi3m0-cs0 { + rockchip,pins = + /* spi3_cs0m0 */ + <4 RK_PA6 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi3m0_cs1: spi3m0-cs1 { + rockchip,pins = + /* spi3_cs1m0 */ + <4 RK_PA7 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi3m1_pins: spi3m1-pins { + rockchip,pins = + /* spi3_clkm1 */ + <4 RK_PC2 2 &pcfg_pull_none>, + /* spi3_misom1 */ + <4 RK_PC5 2 &pcfg_pull_none>, + /* spi3_mosim1 */ + <4 RK_PC3 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi3m1_cs0: spi3m1-cs0 { + rockchip,pins = + /* spi3_cs0m1 */ + <4 RK_PC6 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + spi3m1_cs1: spi3m1-cs1 { + rockchip,pins = + /* spi3_cs1m1 */ + <4 RK_PD1 2 &pcfg_pull_none>; + }; + }; + + tsadc { + /omit-if-no-ref/ + tsadcm0_shut: tsadcm0-shut { + rockchip,pins = + /* tsadcm0_shut */ + <0 RK_PA1 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + tsadcm1_shut: tsadcm1-shut { + rockchip,pins = + /* tsadcm1_shut */ + <0 RK_PA2 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + tsadc_shutorg: tsadc-shutorg { + rockchip,pins = + /* tsadc_shutorg */ + <0 RK_PA1 2 &pcfg_pull_none>; + }; + }; + + uart0 { + /omit-if-no-ref/ + uart0_xfer: uart0-xfer { + rockchip,pins = + /* uart0_rx */ + <0 RK_PC0 3 &pcfg_pull_up>, + /* uart0_tx */ + <0 RK_PC1 3 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart0_ctsn: uart0-ctsn { + rockchip,pins = + /* uart0_ctsn */ + <0 RK_PC7 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart0_rtsn: uart0-rtsn { + rockchip,pins = + /* uart0_rtsn */ + <0 RK_PC4 3 &pcfg_pull_none>; + }; + }; + + uart1 { + /omit-if-no-ref/ + uart1m0_xfer: uart1m0-xfer { + rockchip,pins = + /* uart1_rxm0 */ + <2 RK_PB3 2 &pcfg_pull_up>, + /* uart1_txm0 */ + <2 RK_PB4 2 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart1m0_ctsn: uart1m0-ctsn { + rockchip,pins = + /* uart1m0_ctsn */ + <2 RK_PB6 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart1m0_rtsn: uart1m0-rtsn { + rockchip,pins = + /* uart1m0_rtsn */ + <2 RK_PB5 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart1m1_xfer: uart1m1-xfer { + rockchip,pins = + /* uart1_rxm1 */ + <3 RK_PD7 4 &pcfg_pull_up>, + /* uart1_txm1 */ + <3 RK_PD6 4 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart1m1_ctsn: uart1m1-ctsn { + rockchip,pins = + /* uart1m1_ctsn */ + <4 RK_PC1 4 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart1m1_rtsn: uart1m1-rtsn { + rockchip,pins = + /* uart1m1_rtsn */ + <4 RK_PB6 4 &pcfg_pull_none>; + }; + }; + + uart2 { + /omit-if-no-ref/ + uart2m0_xfer: uart2m0-xfer { + rockchip,pins = + /* uart2_rxm0 */ + <0 RK_PD0 1 &pcfg_pull_up>, + /* uart2_txm0 */ + <0 RK_PD1 1 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart2m1_xfer: uart2m1-xfer { + rockchip,pins = + /* uart2_rxm1 */ + <1 RK_PD6 2 &pcfg_pull_up>, + /* uart2_txm1 */ + <1 RK_PD5 2 &pcfg_pull_up>; + }; + }; + + uart3 { + /omit-if-no-ref/ + uart3m0_xfer: uart3m0-xfer { + rockchip,pins = + /* uart3_rxm0 */ + <1 RK_PA0 2 &pcfg_pull_up>, + /* uart3_txm0 */ + <1 RK_PA1 2 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart3m0_ctsn: uart3m0-ctsn { + rockchip,pins = + /* uart3m0_ctsn */ + <1 RK_PA3 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart3m0_rtsn: uart3m0-rtsn { + rockchip,pins = + /* uart3m0_rtsn */ + <1 RK_PA2 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart3m1_xfer: uart3m1-xfer { + rockchip,pins = + /* uart3_rxm1 */ + <3 RK_PC0 4 &pcfg_pull_up>, + /* uart3_txm1 */ + <3 RK_PB7 4 &pcfg_pull_up>; + }; + }; + + uart4 { + /omit-if-no-ref/ + uart4m0_xfer: uart4m0-xfer { + rockchip,pins = + /* uart4_rxm0 */ + <1 RK_PA4 2 &pcfg_pull_up>, + /* uart4_txm0 */ + <1 RK_PA6 2 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart4m0_ctsn: uart4m0-ctsn { + rockchip,pins = + /* uart4m0_ctsn */ + <1 RK_PA7 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart4m0_rtsn: uart4m0-rtsn { + rockchip,pins = + /* uart4m0_rtsn */ + <1 RK_PA5 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart4m1_xfer: uart4m1-xfer { + rockchip,pins = + /* uart4_rxm1 */ + <3 RK_PB1 4 &pcfg_pull_up>, + /* uart4_txm1 */ + <3 RK_PB2 4 &pcfg_pull_up>; + }; + }; + + uart5 { + /omit-if-no-ref/ + uart5m0_xfer: uart5m0-xfer { + rockchip,pins = + /* uart5_rxm0 */ + <2 RK_PA1 3 &pcfg_pull_up>, + /* uart5_txm0 */ + <2 RK_PA2 3 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart5m0_ctsn: uart5m0-ctsn { + rockchip,pins = + /* uart5m0_ctsn */ + <1 RK_PD7 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart5m0_rtsn: uart5m0-rtsn { + rockchip,pins = + /* uart5m0_rtsn */ + <2 RK_PA0 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart5m1_xfer: uart5m1-xfer { + rockchip,pins = + /* uart5_rxm1 */ + <3 RK_PC3 4 &pcfg_pull_up>, + /* uart5_txm1 */ + <3 RK_PC2 4 &pcfg_pull_up>; + }; + }; + + uart6 { + /omit-if-no-ref/ + uart6m0_xfer: uart6m0-xfer { + rockchip,pins = + /* uart6_rxm0 */ + <2 RK_PA3 3 &pcfg_pull_up>, + /* uart6_txm0 */ + <2 RK_PA4 3 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart6m0_ctsn: uart6m0-ctsn { + rockchip,pins = + /* uart6m0_ctsn */ + <2 RK_PC0 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart6m0_rtsn: uart6m0-rtsn { + rockchip,pins = + /* uart6m0_rtsn */ + <2 RK_PB7 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart6m1_xfer: uart6m1-xfer { + rockchip,pins = + /* uart6_rxm1 */ + <1 RK_PD6 3 &pcfg_pull_up>, + /* uart6_txm1 */ + <1 RK_PD5 3 &pcfg_pull_up>; + }; + }; + + uart7 { + /omit-if-no-ref/ + uart7m0_xfer: uart7m0-xfer { + rockchip,pins = + /* uart7_rxm0 */ + <2 RK_PA5 3 &pcfg_pull_up>, + /* uart7_txm0 */ + <2 RK_PA6 3 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart7m0_ctsn: uart7m0-ctsn { + rockchip,pins = + /* uart7m0_ctsn */ + <2 RK_PC2 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart7m0_rtsn: uart7m0-rtsn { + rockchip,pins = + /* uart7m0_rtsn */ + <2 RK_PC1 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart7m1_xfer: uart7m1-xfer { + rockchip,pins = + /* uart7_rxm1 */ + <3 RK_PC5 4 &pcfg_pull_up>, + /* uart7_txm1 */ + <3 RK_PC4 4 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart7m2_xfer: uart7m2-xfer { + rockchip,pins = + /* uart7_rxm2 */ + <4 RK_PA3 4 &pcfg_pull_up>, + /* uart7_txm2 */ + <4 RK_PA2 4 &pcfg_pull_up>; + }; + }; + + uart8 { + /omit-if-no-ref/ + uart8m0_xfer: uart8m0-xfer { + rockchip,pins = + /* uart8_rxm0 */ + <2 RK_PC6 2 &pcfg_pull_up>, + /* uart8_txm0 */ + <2 RK_PC5 3 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart8m0_ctsn: uart8m0-ctsn { + rockchip,pins = + /* uart8m0_ctsn */ + <2 RK_PB2 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart8m0_rtsn: uart8m0-rtsn { + rockchip,pins = + /* uart8m0_rtsn */ + <2 RK_PB1 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart8m1_xfer: uart8m1-xfer { + rockchip,pins = + /* uart8_rxm1 */ + <3 RK_PA0 4 &pcfg_pull_up>, + /* uart8_txm1 */ + <2 RK_PD7 4 &pcfg_pull_up>; + }; + }; + + uart9 { + /omit-if-no-ref/ + uart9m0_xfer: uart9m0-xfer { + rockchip,pins = + /* uart9_rxm0 */ + <2 RK_PA7 3 &pcfg_pull_up>, + /* uart9_txm0 */ + <2 RK_PB0 3 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart9m0_ctsn: uart9m0-ctsn { + rockchip,pins = + /* uart9m0_ctsn */ + <2 RK_PC4 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart9m0_rtsn: uart9m0-rtsn { + rockchip,pins = + /* uart9m0_rtsn */ + <2 RK_PC3 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + uart9m1_xfer: uart9m1-xfer { + rockchip,pins = + /* uart9_rxm1 */ + <4 RK_PC6 4 &pcfg_pull_up>, + /* uart9_txm1 */ + <4 RK_PC5 4 &pcfg_pull_up>; + }; + + /omit-if-no-ref/ + uart9m2_xfer: uart9m2-xfer { + rockchip,pins = + /* uart9_rxm2 */ + <4 RK_PA5 4 &pcfg_pull_up>, + /* uart9_txm2 */ + <4 RK_PA4 4 &pcfg_pull_up>; + }; + }; + + vop { + /omit-if-no-ref/ + vopm0_pins: vopm0-pins { + rockchip,pins = + /* vop_pwmm0 */ + <0 RK_PC3 2 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + vopm1_pins: vopm1-pins { + rockchip,pins = + /* vop_pwmm1 */ + <3 RK_PC4 2 &pcfg_pull_none>; + }; + }; +}; + +/* + * This part is edited handly. + */ +&pinctrl { + spi0-hs { + /omit-if-no-ref/ + spi0m0_pins_hs: spi0m0-pins { + rockchip,pins = + /* spi0_clkm0 */ + <0 RK_PB5 2 &pcfg_pull_up_drv_level_1>, + /* spi0_misom0 */ + <0 RK_PC5 2 &pcfg_pull_up_drv_level_1>, + /* spi0_mosim0 */ + <0 RK_PB6 2 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi0m0_cs0_hs: spi0m0-cs0 { + rockchip,pins = + /* spi0_cs0m0 */ + <0 RK_PC6 2 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi0m0_cs1_hs: spi0m0-cs1 { + rockchip,pins = + /* spi0_cs1m0 */ + <0 RK_PC4 2 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi0m1_pins_hs: spi0m1-pins { + rockchip,pins = + /* spi0_clkm1 */ + <2 RK_PD3 3 &pcfg_pull_up_drv_level_1>, + /* spi0_misom1 */ + <2 RK_PD0 3 &pcfg_pull_up_drv_level_1>, + /* spi0_mosim1 */ + <2 RK_PD1 3 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi0m1_cs0_hs: spi0m1-cs0 { + rockchip,pins = + /* spi0_cs0m1 */ + <2 RK_PD2 3 &pcfg_pull_up_drv_level_1>; + }; + }; + + spi1-hs { + /omit-if-no-ref/ + spi1m0_pins_hs: spi1m0-pins { + rockchip,pins = + /* spi1_clkm0 */ + <2 RK_PB5 3 &pcfg_pull_up_drv_level_1>, + /* spi1_misom0 */ + <2 RK_PB6 3 &pcfg_pull_up_drv_level_1>, + /* spi1_mosim0 */ + <2 RK_PB7 4 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi1m0_cs0_hs: spi1m0-cs0 { + rockchip,pins = + /* spi1_cs0m0 */ + <2 RK_PC0 4 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi1m0_cs1_hs: spi1m0-cs1 { + rockchip,pins = + /* spi1_cs1m0 */ + <2 RK_PC6 3 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi1m1_pins_hs: spi1m1-pins { + rockchip,pins = + /* spi1_clkm1 */ + <3 RK_PC3 3 &pcfg_pull_up_drv_level_1>, + /* spi1_misom1 */ + <3 RK_PC2 3 &pcfg_pull_up_drv_level_1>, + /* spi1_mosim1 */ + <3 RK_PC1 3 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi1m1_cs0_hs: spi1m1-cs0 { + rockchip,pins = + /* spi1_cs0m1 */ + <3 RK_PA1 3 &pcfg_pull_up_drv_level_1>; + }; + }; + + spi2-hs { + /omit-if-no-ref/ + spi2m0_pins_hs: spi2m0-pins { + rockchip,pins = + /* spi2_clkm0 */ + <2 RK_PC1 4 &pcfg_pull_up_drv_level_1>, + /* spi2_misom0 */ + <2 RK_PC2 4 &pcfg_pull_up_drv_level_1>, + /* spi2_mosim0 */ + <2 RK_PC3 4 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi2m0_cs0_hs: spi2m0-cs0 { + rockchip,pins = + /* spi2_cs0m0 */ + <2 RK_PC4 4 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi2m0_cs1_hs: spi2m0-cs1 { + rockchip,pins = + /* spi2_cs1m0 */ + <2 RK_PC5 4 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi2m1_pins_hs: spi2m1-pins { + rockchip,pins = + /* spi2_clkm1 */ + <3 RK_PA0 3 &pcfg_pull_up_drv_level_1>, + /* spi2_misom1 */ + <2 RK_PD7 3 &pcfg_pull_up_drv_level_1>, + /* spi2_mosim1 */ + <2 RK_PD6 3 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi2m1_cs0_hs: spi2m1-cs0 { + rockchip,pins = + /* spi2_cs0m1 */ + <2 RK_PD5 3 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi2m1_cs1_hs: spi2m1-cs1 { + rockchip,pins = + /* spi2_cs1m1 */ + <2 RK_PD4 3 &pcfg_pull_up_drv_level_1>; + }; + }; + + spi3-hs { + /omit-if-no-ref/ + spi3m0_pins_hs: spi3m0-pins { + rockchip,pins = + /* spi3_clkm0 */ + <4 RK_PB3 4 &pcfg_pull_up_drv_level_1>, + /* spi3_misom0 */ + <4 RK_PB0 4 &pcfg_pull_up_drv_level_1>, + /* spi3_mosim0 */ + <4 RK_PB2 4 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi3m0_cs0_hs: spi3m0-cs0 { + rockchip,pins = + /* spi3_cs0m0 */ + <4 RK_PA6 4 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi3m0_cs1_hs: spi3m0-cs1 { + rockchip,pins = + /* spi3_cs1m0 */ + <4 RK_PA7 4 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi3m1_pins_hs: spi3m1-pins { + rockchip,pins = + /* spi3_clkm1 */ + <4 RK_PC2 2 &pcfg_pull_up_drv_level_1>, + /* spi3_misom1 */ + <4 RK_PC5 2 &pcfg_pull_up_drv_level_1>, + /* spi3_mosim1 */ + <4 RK_PC3 2 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi3m1_cs0_hs: spi3m1-cs0 { + rockchip,pins = + /* spi3_cs0m1 */ + <4 RK_PC6 2 &pcfg_pull_up_drv_level_1>; + }; + + /omit-if-no-ref/ + spi3m1_cs1_hs: spi3m1-cs1 { + rockchip,pins = + /* spi3_cs1m1 */ + <4 RK_PD1 2 &pcfg_pull_up_drv_level_1>; + }; + }; + + gmac-txd-level3 { + /omit-if-no-ref/ + gmac0_tx_bus2_level3: gmac0-tx-bus2-level3 { + rockchip,pins = + /* gmac0_txd0 */ + <2 RK_PB3 1 &pcfg_pull_none_drv_level_3>, + /* gmac0_txd1 */ + <2 RK_PB4 1 &pcfg_pull_none_drv_level_3>, + /* gmac0_txen */ + <2 RK_PB5 1 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac0_rgmii_bus_level3: gmac0-rgmii-bus-level3 { + rockchip,pins = + /* gmac0_rxd2 */ + <2 RK_PA3 2 &pcfg_pull_none>, + /* gmac0_rxd3 */ + <2 RK_PA4 2 &pcfg_pull_none>, + /* gmac0_txd2 */ + <2 RK_PA6 2 &pcfg_pull_none_drv_level_3>, + /* gmac0_txd3 */ + <2 RK_PA7 2 &pcfg_pull_none_drv_level_3>; + }; + + /omit-if-no-ref/ + gmac1m0_tx_bus2_level3: gmac1m0-tx-bus2-level3 { + rockchip,pins = + /* gmac1_txd0m0 */ + <3 RK_PB5 3 &pcfg_pull_none_drv_level_3>, + /* gmac1_txd1m0 */ + <3 RK_PB6 3 &pcfg_pull_none_drv_level_3>, + /* gmac1_txenm0 */ + <3 RK_PB7 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m0_rgmii_bus_level3: gmac1m0-rgmii-bus-level3 { + rockchip,pins = + /* gmac1_rxd2m0 */ + <3 RK_PA4 3 &pcfg_pull_none>, + /* gmac1_rxd3m0 */ + <3 RK_PA5 3 &pcfg_pull_none>, + /* gmac1_txd2m0 */ + <3 RK_PA2 3 &pcfg_pull_none_drv_level_3>, + /* gmac1_txd3m0 */ + <3 RK_PA3 3 &pcfg_pull_none_drv_level_3>; + }; + + /omit-if-no-ref/ + gmac1m1_tx_bus2_level3: gmac1m1-tx-bus2-level3 { + rockchip,pins = + /* gmac1_txd0m1 */ + <4 RK_PA4 3 &pcfg_pull_none_drv_level_3>, + /* gmac1_txd1m1 */ + <4 RK_PA5 3 &pcfg_pull_none_drv_level_3>, + /* gmac1_txenm1 */ + <4 RK_PA6 3 &pcfg_pull_none>; + }; + + /omit-if-no-ref/ + gmac1m1_rgmii_bus_level3: gmac1m1-rgmii-bus-level3 { + rockchip,pins = + /* gmac1_rxd2m1 */ + <4 RK_PA1 3 &pcfg_pull_none>, + /* gmac1_rxd3m1 */ + <4 RK_PA2 3 &pcfg_pull_none>, + /* gmac1_txd2m1 */ + <3 RK_PD6 3 &pcfg_pull_none_drv_level_3>, + /* gmac1_txd3m1 */ + <3 RK_PD7 3 &pcfg_pull_none_drv_level_3>; + }; + }; + + gmac-txc-level2 { + /omit-if-no-ref/ + gmac0_rgmii_clk_level2: gmac0-rgmii-clk-level2 { + rockchip,pins = + /* gmac0_rxclk */ + <2 RK_PA5 2 &pcfg_pull_none>, + /* gmac0_txclk */ + <2 RK_PB0 2 &pcfg_pull_none_drv_level_2>; + }; + + /omit-if-no-ref/ + gmac1m0_rgmii_clk_level2: gmac1m0-rgmii-clk-level2 { + rockchip,pins = + /* gmac1_rxclkm0 */ + <3 RK_PA7 3 &pcfg_pull_none>, + /* gmac1_txclkm0 */ + <3 RK_PA6 3 &pcfg_pull_none_drv_level_2>; + }; + + /omit-if-no-ref/ + gmac1m1_rgmii_clk_level2: gmac1m1-rgmii-clk-level2 { + rockchip,pins = + /* gmac1_rxclkm1 */ + <4 RK_PA3 3 &pcfg_pull_none>, + /* gmac1_txclkm1 */ + <4 RK_PA0 3 &pcfg_pull_none_drv_level_2>; + }; + }; + + gpio-func { + /omit-if-no-ref/ + tsadc_gpio_func: tsadc-gpio-func { + rockchip,pins = + <0 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-base.dts b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-base.dts new file mode 100755 index 000000000000..0ef442123d05 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-base.dts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ +/dts-v1/; +#include "rk3568.dtsi" +#include "rk3568-linux.dtsi" +#include "rk3568-toybrick-x0.dtsi" +//#include "rk3568-toybrick-isp-camera-board.dtsi" +/ { + compatible = "rockchip,rk3568-toybrick-dev-linux-x0","rockchip,rk3568"; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-edp.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-edp.dtsi new file mode 100755 index 000000000000..459fc3b6d6a0 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-edp.dtsi @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/ { + compatible = "rockchip,rk3568-toybrick-edp", "rockchip,rk3568"; + + edp_panel: edp_panel { + compatible = "simple-panel"; + backlight = <&backlight>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + reset-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; + +}; + + +&backlight { + status = "okay"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; + +}; + +&route_edp { + status = "okay"; +}; + +&edp_phy { + status = "okay"; +}; + +&edp_in_vp0 { + status = "disabled"; +}; + +&edp_in_vp1 { + status = "okay"; +}; + +&edp_panel { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <600>; + i2c-scl-falling-time-ns = <20>; + gsl3673: gsl3673@40 { + status = "okay"; + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + }; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0-beiqicloud.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0-beiqicloud.dtsi new file mode 100755 index 000000000000..25889a206778 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0-beiqicloud.dtsi @@ -0,0 +1,395 @@ +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/ { + compatible = "rockchip,rk3568-toybrick-mipi-tx0", "rockchip,rk3568"; +}; + +/* + * mipi_dphy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; + + dsi0_panel: panel@0 { + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + // power-supply = <&lcd_pwr>; + prepare-delay-ms = <2>; + reset-delay-ms = <100>; + init-delay-ms = <20>; + pinctrl-names = "default"; + pinctrl-0 = <&mipi_power_en>; + enable-gpios = <&gpio3 RK_PB5 GPIO_ACTIVE_HIGH>; + enable-delay-ms = <120>; + disable-delay-ms = <50>; + unprepare-delay-ms = <20>; + width-mm = <68>; + height-mm = <121>; + + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + pnel-type = <5>; + + panel-init-sequence = [ + 05 64 01 11 + 39 00 04 FF 98 81 03 + 15 00 02 01 00 + 15 00 02 02 00 + 15 00 02 03 72 + 15 00 02 04 00 + 15 00 02 05 00 + 15 00 02 06 09 + 15 00 02 07 00 + 15 00 02 08 00 + 15 00 02 09 01 + 15 00 02 0A 00 + 15 00 02 0B 00 + 15 00 02 0C 01 + 15 00 02 0D 00 + 15 00 02 0E 00 + 15 00 02 0F 00 + 15 00 02 10 00 + 15 00 02 11 00 + 15 00 02 12 00 + 15 00 02 13 00 + 15 00 02 14 00 + 15 00 02 15 00 + 15 00 02 16 00 + 15 00 02 17 00 + 15 00 02 18 00 + 15 00 02 19 00 + 15 00 02 1A 00 + 15 00 02 1B 00 + 15 00 02 1C 00 + 15 00 02 1D 00 + 15 00 02 1E 40 + 15 00 02 1F 80 + 15 00 02 20 05 + 15 00 02 21 02 + 15 00 02 22 00 + 15 00 02 23 00 + 15 00 02 24 00 + 15 00 02 25 00 + 15 00 02 26 00 + 15 00 02 27 00 + 15 00 02 28 33 + 15 00 02 29 02 + 15 00 02 2A 00 + 15 00 02 2B 00 + 15 00 02 2C 00 + 15 00 02 2D 00 + 15 00 02 2E 00 + 15 00 02 2F 00 + 15 00 02 30 00 + 15 00 02 31 00 + 15 00 02 32 00 + 15 00 02 33 00 + 15 00 02 34 04 + 15 00 02 35 00 + 15 00 02 36 00 + 15 00 02 37 00 + 15 00 02 38 3C + 15 00 02 39 00 + 15 00 02 3A 40 + 15 00 02 3B 40 + 15 00 02 3C 00 + 15 00 02 3D 00 + 15 00 02 3E 00 + 15 00 02 3F 00 + 15 00 02 40 00 + 15 00 02 41 00 + 15 00 02 42 00 + 15 00 02 43 00 + 15 00 02 44 00 + 15 00 02 50 01 + 15 00 02 51 23 + 15 00 02 52 45 + 15 00 02 53 67 + 15 00 02 54 89 + 15 00 02 55 AB + 15 00 02 56 01 + 15 00 02 57 23 + 15 00 02 58 45 + 15 00 02 59 67 + 15 00 02 5A 89 + 15 00 02 5B AB + 15 00 02 5C CD + 15 00 02 5D EF + 15 00 02 5E 11 + 15 00 02 5F 01 + 15 00 02 60 00 + 15 00 02 61 15 + 15 00 02 62 14 + 15 00 02 63 0E + 15 00 02 64 0F + 15 00 02 65 0C + 15 00 02 66 0D + 15 00 02 67 06 + 15 00 02 68 02 + 15 00 02 69 02 + 15 00 02 6A 02 + 15 00 02 6B 02 + 15 00 02 6C 02 + 15 00 02 6D 02 + 15 00 02 6E 07 + 15 00 02 6F 02 + 15 00 02 70 02 + 15 00 02 71 02 + 15 00 02 72 02 + 15 00 02 73 02 + 15 00 02 74 02 + 15 00 02 75 01 + 15 00 02 76 00 + 15 00 02 77 14 + 15 00 02 78 15 + 15 00 02 79 0E + 15 00 02 7A 0F + 15 00 02 7B 0C + 15 00 02 7C 0D + 15 00 02 7D 06 + 15 00 02 7E 02 + 15 00 02 7F 02 + 15 00 02 80 02 + 15 00 02 81 02 + 15 00 02 82 02 + 15 00 02 83 02 + 15 00 02 84 07 + 15 00 02 85 02 + 15 00 02 86 02 + 15 00 02 87 02 + 15 00 02 88 02 + 15 00 02 89 02 + 15 00 02 8A 02 + 39 00 04 FF 98 81 04 + 15 00 02 6C 15 + 15 00 02 6E 2A + 15 00 02 6F 33 + 15 00 02 3A 94 + 15 00 02 8D 1A + 15 00 02 87 BA + 15 00 02 26 76 + 15 00 02 B2 D1 + 15 00 02 B5 06 + 39 00 04 FF 98 81 01 + 15 00 02 22 0A + 15 00 02 31 00 + 15 00 02 40 13 + 15 00 02 53 84 + 15 00 02 55 8F + 15 00 02 50 AE + 15 00 02 51 AE + 15 00 02 60 28 + 15 00 02 A0 0F + 15 00 02 A1 1B + 15 00 02 A2 28 + 15 00 02 A3 12 + 15 00 02 A4 15 + 15 00 02 A5 28 + 15 00 02 A6 1B + 15 00 02 A7 1E + 15 00 02 A8 79 + 15 00 02 A9 1B + 15 00 02 AA 27 + 15 00 02 AB 69 + 15 00 02 AC 19 + 15 00 02 AD 18 + 15 00 02 AE 4C + 15 00 02 AF 21 + 15 00 02 B0 28 + 15 00 02 B1 52 + 15 00 02 B2 65 + 15 00 02 C0 04 + 15 00 02 C1 1B + 15 00 02 C2 27 + 15 00 02 C3 13 + 15 00 02 C4 15 + 15 00 02 C5 28 + 15 00 02 C6 1C + 15 00 02 C7 1E + 15 00 02 C8 79 + 15 00 02 C9 1A + 15 00 02 CA 27 + 15 00 02 CB 69 + 15 00 02 CC 1A + 15 00 02 CD 18 + 15 00 02 CE 4C + 15 00 02 CF 21 + 15 00 02 D0 27 + 15 00 02 D1 52 + 15 00 02 D2 65 + 15 00 02 D3 3F + 39 00 04 FF 98 81 00 + + 05 32 01 29 + 15 00 02 35 00 + ]; + + panel-exit-sequence = [ + 05 32 01 28 + 05 C8 01 10 + ]; + + disp_timings:display-timings { + native-mode = <&dsi0_timing0>; + + dsi0_timing0: timing0 { + clock-frequency = <75000000>; + hactive = <720>; + vactive = <1280>; + hfront-porch = <40>; + hsync-len = <10>; + hback-porch = <40>; + vfront-porch = <10>; + vsync-len = <36>; + vback-porch = <15>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; + +}; + +&i2c1 { + status = "okay"; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x5d>; + pinctrl-names = "default"; + power-supply = <&vcc3v3_lcd0_n>; + pinctrl-0 = <&touch_gpio>; + // goodix,enable-gpio = <&gpio3 RK_PB5 GPIO_ACTIVE_HIGH>; + goodix,rst-gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + }; + + gt9xx: gt9xx@5d { + compatible = "goodix,gt9xx"; + status = "okay"; + reg = <0x5d>; + reset-gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + touch-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + max-x = <7200>; + max-y = <1280>; + tp-size = <911>; + pinctrl-names = "default"; + pinctrl-0 = <&touch_gpio>; + power-supply = <&vcc3v3_lcd0_n>; + }; +}; + +&dsi1 { + status = "disabled"; +}; + +&edp { + status = "disabled"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&backlight { + status = "okay"; +}; + +&video_phy0 { + status = "okay"; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&route_dsi0 { + status = "okay"; +}; + +&pinctrl { + mipi_pwren { + mipi_power_en: mipi_power_en { + rockchip,pins = <3 RK_PB5 RK_FUNC_GPIO &pcfg_output_high>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0.dtsi new file mode 100755 index 000000000000..bd32a4246df3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx0.dtsi @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/ { + compatible = "rockchip,rk3568-toybrick-mipi-tx0", "rockchip,rk3568"; +}; + +/* + * mipi_dphy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; +}; + +&dsi1 { + status = "disabled"; +}; + +&edp { + status = "disabled"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&backlight { + status = "okay"; +}; + +>1x { + status = "okay"; + power-supply = <&vcc3v3_lcd0_n>; +}; + +&video_phy0 { + status = "okay"; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx1.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx1.dtsi new file mode 100755 index 000000000000..70fc8ffc3e41 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-mipi-tx1.dtsi @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/ { + compatible = "rockchip,rk3568-toybrick-mipi-tx1", "rockchip,rk3568"; +}; + +&dsi0 { + status = "disabled"; +}; + +/* + * mipi_dphy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "okay"; +}; + +&edp { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; + +}; + +&dsi1_in_vp1 { + status = "okay"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; +}; + +&backlight1 { + status = "okay"; +}; + +&video_phy1 { + status = "okay"; +}; + +>1x { + status = "okay"; + power-supply = <&vcc3v3_lcd0_n>; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-android.dts b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-android.dts new file mode 100755 index 000000000000..ccfd369d0cd3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-android.dts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ +/dts-v1/; +#include "rk3568.dtsi" +#include "rk3568-android.dtsi" +#include "rk3568-toybrick-x0.dtsi" +/ { + compatible = "rockchip,rk3568-toybrick-dev-android-x0","rockchip,rk3568"; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux-factory.dts b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux-factory.dts new file mode 100755 index 000000000000..3107e3a9937a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux-factory.dts @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ +/dts-v1/; +#include "rk3568.dtsi" +#include "rk3568-linux.dtsi" +#include "rk3568-toybrick-x0.dtsi" +/delete-node/ &board_id; +/ { + compatible = "rockchip,rk3568-toybrick-core-linux-factory-edp-mipi1-x0", "rockchip,rk3568"; + + edp_panel: edp_panel { + compatible = "simple-panel"; + backlight = <&backlight>; + prepare-delay-ms = <20>; + enable-delay-ms = <20>; + reset-delay-ms = <20>; + + display-timings { + native-mode = <&timing0>; + + timing0: timing0 { + clock-frequency = <200000000>; + hactive = <1536>; + vactive = <2048>; + hfront-porch = <12>; + hsync-len = <16>; + hback-porch = <48>; + vfront-porch = <8>; + vsync-len = <4>; + vback-porch = <8>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + panel_in: endpoint { + remote-endpoint = <&edp_out>; + }; + }; + }; +}; + +&backlight { + status = "okay"; +}; + +&edp { + status = "okay"; + force-hpd; + + ports { + port@1 { + reg = <1>; + + edp_out: endpoint { + remote-endpoint = <&panel_in>; + }; + }; + }; + +}; + +&edp_phy { + status = "okay"; +}; + +&edp_in_vp0 { + status = "disabled"; +}; + +&edp_in_vp1 { + status = "okay"; +}; + +&edp_panel { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&i2c1 { + status = "okay"; + i2c-scl-rising-time-ns = <600>; + i2c-scl-falling-time-ns = <20>; + gsl3673: gsl3673@40 { + status = "okay"; + compatible = "GSL,GSL3673"; + reg = <0x40>; + screen_max_x = <1536>; + screen_max_y = <2048>; + irq_gpio_number = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + rst_gpio_number = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + }; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&hdmi { + status = "disabled"; +}; + +&dsi0 { + status = "disabled"; +}; + +/* + * mipi_dphy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "okay"; +}; + +&dsi1_in_vp0 { + status = "okay"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; +}; + +&backlight1 { + status = "okay"; +}; + +&video_phy1 { + status = "okay"; +}; + +>1x { + status = "okay"; + power-supply = <&vcc3v3_lcd0_n>; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux.dts new file mode 100755 index 000000000000..0fb7b0a65fe5 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0-linux.dts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ +/dts-v1/; +#include "rk3568.dtsi" +#include "rk3568-linux.dtsi" +#include "rk3568-toybrick-x0.dtsi" +#include "rk3568-toybrick-mipi-tx0-beiqicloud.dtsi" +/delete-node/ &board_id; +/ { + compatible = "rockchip,rk3568-toybrick-dev-linux-x0","rockchip,rk3568"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0.dtsi new file mode 100755 index 000000000000..b6ec191b0107 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x0.dtsi @@ -0,0 +1,868 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ +#include +#include +#include "rk3568-toybrick.dtsi" +/delete-node/ &adc_keys; + +/ { + compatible = "rockchip,rk3568-toybrick", "rockchip,rk3568"; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + gpio_leds: gpio-leds { + compatible = "gpio-leds"; + led@1 { + gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>; + label = "blue"; // Blue LED + retain-state-suspended; + }; + + led@2 { + gpios = <&gpio4 RK_PC3 GPIO_ACTIVE_HIGH>; + label = "red"; // Red LED + retain-state-suspended; + }; + + led@3 { + gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; + label = "green"; // Green LED + retain-state-suspended; + }; + }; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <1250000>; + }; + + mute-key { + linux,code = ; + label = "mute"; + press-threshold-microvolt = <850000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <400000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <20000>; + }; + }; + + rt5672-sound { + compatible = "rockchip-rt5670"; + status = "disabled"; + dais { + dai0 { + audio-codec = <&rt5670>; + audio-controller = <&i2s1_8ch>; + format = "i2s"; + }; + dai1 { + audio-codec = <&rt5670>; + audio-controller = <&i2s1_8ch>; + format = "i2s"; + }; + dai2 { + audio-codec = <&es7210>; + audio-controller = <&i2s1_8ch>; + format = "i2s"; + }; + }; + }; + + vcc2v5_sys: vcc2v5-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc2v5-sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie20_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie20_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + pcie30_avdd0v9: pcie30-avdd0v9 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd0v9"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_avdd1v8: pcie30-avdd1v8 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie30_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + vcc3v3_bu: vcc3v3-bu { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_bu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc5v0_sys>; + }; + + rk809_sound_micarray: rk809-sound-micarray { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_8ch>; + }; + codec { + sound-dai = <&rk809_codec 0>; + }; + }; + simple-audio-card,dai-link@1 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_8ch>; + }; + codec { + sound-dai = <&es7210>; + }; + }; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; +}; + + + +&i2s1_8ch { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_sclkrx + &i2s1m0_lrcktx + &i2s1m0_sclkrx + &i2s1m0_lrckrx + &i2s1m0_sdo0 + &i2s1m0_sdi0 + &i2s1m0_sdi1 + &i2s1m0_sdi2 + &i2s1m0_sdi3>; +}; + + +#if 0 + +&rk809_codec { + #sound-dai-cells = <1>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + pdmdata-out-enable; + adc-for-loopback; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + status = "okay"; +}; +#endif +&rk809_sound { + status = "okay"; +}; + +&combphy0_us { + status = "okay"; +}; + +&combphy1_usq { + status = "okay"; +}; + +&combphy2_psq { + status = "okay"; +}; + +/* + * mipi_dphy0 needs to be enabled + * when dsi0 is enabled + */ +&dsi0 { + status = "okay"; +}; + +&dsi0_in_vp0 { + status = "disabled"; +}; + +&dsi0_in_vp1 { + status = "okay"; +}; + +&dsi0_panel { + power-supply = <&vcc3v3_lcd0_n>; +}; + +/* + * mipi_dphy1 needs to be enabled + * when dsi1 is enabled + */ +&dsi1 { + status = "disabled"; +}; + +&dsi1_in_vp0 { + status = "disabled"; +}; + +&dsi1_in_vp1 { + status = "disabled"; +}; + +&dsi1_panel { + power-supply = <&vcc3v3_lcd1_n>; +}; + +&gmac0 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0>; + assigned-clock-parents = <&cru SCLK_GMAC0_RGMII_SPEED>, <&cru CLK_MAC0_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_miim + &gmac0_tx_bus2 + &gmac0_rx_bus2 + &gmac0_rgmii_clk + &gmac0_rgmii_bus>; + + tx_delay = <0x2d>; + rx_delay = <0x13>; + + phy-handle = <&rgmii_phy0>; + status = "okay"; +}; + +&gmac1 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio2 RK_PD1 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m1_miim + &gmac1m1_tx_bus2 + &gmac1m1_rx_bus2 + &gmac1m1_rgmii_clk + &gmac1m1_rgmii_bus>; + + tx_delay = <0x37>; + rx_delay = <0x0f>; + + phy-handle = <&rgmii_phy1>; + status = "okay"; +}; + +/* + * power-supply should switche to vcc3v3_lcd1_n + * when mipi panel is connected to dsi1. + */ +>1x { + power-supply = <&vcc3v3_lcd0_n>; +}; + +&i2c3 { + status = "okay"; + rt5670: rt5670@1c { + status = "okay"; + #sound-dai-cell = <0>; + compatible = "realtek,rt5670"; + reg = <0x1c>; + }; + + es7210: es7210@40 { + #sound-dai-cells = <0>; + compatible = "MicArray_0"; + reg = <0x40>; + clocks = <&cru I2S1_MCLKOUT_RX>;//csqerr + clock-names = "mclk"; + }; + + es7210_1: es7210@42 { + compatible = "MicArray_1"; + reg = <0x42>; + }; +}; + +&i2c5 { + status = "okay"; + + gs_mxc6655xa: gs_mxc6655xa@15 { + status = "okay"; + compatible = "gs_mxc6655xa"; + pinctrl-names = "default"; + pinctrl-0 = <&mxc6655xa_irq_gpio>; + reg = <0x15>; + irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + power-off-in-suspend = <1>; + layout = <1>; + }; + + mxc6655xa: mxc6655xa@15 { + status = "disabled"; + compatible = "gs_mxc6655xa"; + pinctrl-names = "default"; + pinctrl-0 = <&mxc6655xa_irq_gpio>; + reg = <0x15>; + irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + power-off-in-suspend = <1>; + layout = <1>; + }; + + hym8563: hym8563@51 { + compatible = "haoyu,hym8563"; + reg = <0x51>; + pinctrl-names = "default"; + pinctrl-0 = <&rtc_int>; + + interrupt-parent = <&gpio0>; + interrupts = ; + }; +}; + +&mdio0 { + rgmii_phy0: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&mdio1 { + rgmii_phy1: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&i2c4 { + status = "okay"; + + gc8034: gc8034@37 { + compatible = "galaxycore,gc8034"; + reg = <0x37>; + clocks = <&cru CLK_CIF_OUT>;//CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; + rockchip,grf = <&grf>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "RK-CMK-8M-2-v1"; + rockchip,camera-module-lens-name = "CK8401"; + port { + gc8034_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; + + ov9750_1: ov9750_1@36 { + compatible = "ovti,ov9750"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + power-domains = <&power RK3568_PD_VI>; + reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT0854-FV1"; + rockchip,camera-module-lens-name = "CHT-842B-MD"; + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; +#if 0 + os04a10: os04a10@36 { + compatible = "ovti,os04a10"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; + /* power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; */ + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT1607-FV1"; + /* rockchip,camera-module-lens-name = "M12-4IR-4MP-F16"; */ + rockchip,camera-module-lens-name = "M12-40IRC-4MP-F16"; + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + }; + }; + }; +#endif + ov5695: ov5695@36 { + status = "okay"; + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in_ucam2>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + mipi_in_ucam1: endpoint@2 { + reg = <2>; + remote-endpoint = <&gc8034_out>; + data-lanes = <1 2 3 4>; + }; + mipi_in_ucam2: endpoint@3 { + reg = <3>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy_out>; + }; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +/* +&pcie2x1 { + reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie20_3v3>; + status = "okay"; +}; +*/ + +&pcie30phy { + status = "okay"; +}; + +&pcie3x2 { + reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie30_3v3>; + status = "okay"; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + leds_gpio: leds-gpio { + rockchip,pins = + <4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>, + <4 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>, + <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + mxc6655xa { + mxc6655xa_irq_gpio: mxc6655xa_irq_gpio { + rockchip,pins = <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +/* + sii902x { + sii902x_hdmi_int: sii902x-hdmi-int { + rockchip,pins = <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +*/ + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; + }; +/* + wifi_32k: wifi-32k {//csqerr + rockchip,pins = <2 RK_PC6 1 &pcfg_pull_none>; + }; +*/ + + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart8_gpios: uart8-gpios { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + rtc { + rtc_int: rtc-int { + rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + i2s1 { + /omit-if-no-ref/ + i2s1m0_lrckrx: i2s1m0-lrckrx { + rockchip,pins = + /* i2s1m0_lrckrx */ + <1 RK_PA6 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_lrcktx: i2s1m0-lrcktx { + rockchip,pins = + /* i2s1m0_lrcktx */ + <1 RK_PA5 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_mclk: i2s1m0-mclk { + rockchip,pins = + /* i2s1m0_mclk */ + <1 RK_PA2 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sclkrx: i2s1m0-sclkrx { + rockchip,pins = + /* i2s1m0_sclkrx */ + <1 RK_PA4 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sclktx: i2s1m0-sclktx { + rockchip,pins = + /* i2s1m0_sclktx */ + <1 RK_PA3 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sdi0: i2s1m0-sdi0 { + rockchip,pins = + /* i2s1m0_sdi0 */ + <1 RK_PB3 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sdi1: i2s1m0-sdi1 { + rockchip,pins = + /* i2s1m0_sdi1 */ + <1 RK_PB2 2 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sdi2: i2s1m0-sdi2 { + rockchip,pins = + /* i2s1m0_sdi2 */ + <1 RK_PB1 2 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sdi3: i2s1m0-sdi3 { + rockchip,pins = + /* i2s1m0_sdi3 */ + <1 RK_PB0 2 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sdo0: i2s1m0-sdo0 { + rockchip,pins = + /* i2s1m0_sdo0 */ + <1 RK_PA7 1 &pcfg_pull_up_drv_level_4>; + }; + }; + + cam { + camera_pwr: camera-pwr { + rockchip,pins = + /* camera power en */ + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pwm7 { + status = "okay"; +}; + +&route_dsi0 { + status = "okay"; + connect = <&vp1_out_dsi0>; +}; + +&sdio_pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; + post-power-on-delay-ms = <20>; + status = "okay"; +}; + +&sdmmc1 { + status = "disabled"; +}; + +&sdmmc2 { + max-frequency = <150000000>; + supports-sdio; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&uart1 { + status = "disabled"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&uart3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart3m1_xfer>; +}; + +&uart4 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart4m1_xfer>; +}; + +&uart8 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn>; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&wireless_wlan { + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; +}; + +&wireless_bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart8m0_rtsn>; + pinctrl-1 = <&uart8_gpios>; + BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&sata2 { + status = "okay"; +}; + +&edp { + status = "disabled"; +}; + + +&dsi0 { + status = "disabled"; +}; + +&dsi1 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10-linux.dts b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10-linux.dts new file mode 100755 index 000000000000..8dd494ce3a1f --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10-linux.dts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ +/dts-v1/; +#include "rk3568.dtsi" +#include "rk3568-linux.dtsi" +#include "rk3568-toybrick-x10.dtsi" +/delete-node/ &board_id; +/ { + compatible = "rockchip,rk3568-toybrick-dev-linux-x0","rockchip,rk3568"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10.dtsi new file mode 100755 index 000000000000..7c1919d7d258 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick-x10.dtsi @@ -0,0 +1,816 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ +#include +#include +#include "rk3568-toybrick.dtsi" +/delete-node/ &adc_keys; + +/ { + compatible = "rockchip,rk3568-toybrick", "rockchip,rk3568"; + + rk_headset: rk-headset { + compatible = "rockchip_headset"; + headset_gpio = <&gpio3 RK_PC3 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&hp_det>; + io-channels = <&saradc 1>; + }; + + gpio_leds: gpio-leds { + compatible = "gpio-leds"; + led@1 { + gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>; + label = "blue"; // Blue LED + retain-state-suspended; + }; + + led@2 { + gpios = <&gpio4 RK_PC3 GPIO_ACTIVE_HIGH>; + label = "red"; // Red LED + retain-state-suspended; + }; + + led@3 { + gpios = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; + label = "green"; // Green LED + retain-state-suspended; + }; + }; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + poll-interval = <100>; + keyup-threshold-microvolt = <1800000>; + + menu-key { + linux,code = ; + label = "menu"; + press-threshold-microvolt = <1250000>; + }; + + mute-key { + linux,code = ; + label = "mute"; + press-threshold-microvolt = <850000>; + }; + + vol-down-key { + linux,code = ; + label = "volume down"; + press-threshold-microvolt = <400000>; + }; + + vol-up-key { + linux,code = ; + label = "volume up"; + press-threshold-microvolt = <20000>; + }; + }; + + rt5672-sound { + compatible = "rockchip-rt5670"; + status = "disabled"; + dais { + dai0 { + audio-codec = <&rt5670>; + audio-controller = <&i2s1_8ch>; + format = "i2s"; + }; + dai1 { + audio-codec = <&rt5670>; + audio-controller = <&i2s1_8ch>; + format = "i2s"; + }; + dai2 { + audio-codec = <&es7210>; + audio-controller = <&i2s1_8ch>; + format = "i2s"; + }; + }; + }; + + vcc2v5_sys: vcc2v5-ddr { + compatible = "regulator-fixed"; + regulator-name = "vcc2v5-sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie20_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie20_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + pcie30_avdd0v9: pcie30-avdd0v9 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd0v9"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_avdd1v8: pcie30-avdd1v8 { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_3v3: gpio-regulator { + compatible = "regulator-gpio"; + regulator-name = "pcie30_3v3"; + regulator-min-microvolt = <100000>; + regulator-max-microvolt = <3300000>; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + gpios-states = <0x1>; + states = <100000 0x0 + 3300000 0x1>; + }; + + vcc3v3_bu: vcc3v3-bu { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_bu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc5v0_sys>; + }; + + rk809_sound_micarray: rk809-sound-micarray { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_8ch>; + }; + codec { + sound-dai = <&rk809_codec 0>; + }; + }; + simple-audio-card,dai-link@1 { + format = "i2s"; + cpu { + sound-dai = <&i2s1_8ch>; + }; + codec { + sound-dai = <&es7210>; + }; + }; + }; + + vcc_camera: vcc-camera-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&camera_pwr>; + regulator-name = "vcc_camera"; + enable-active-high; + regulator-always-on; + regulator-boot-on; + }; + + vcc_4g: vcc-4g-regulator { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc_4g_drv>; + regulator-name = "vcc_4g"; + enable-active-low; + regulator-always-on; + regulator-boot-on; + /*regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>;*/ + }; +}; + + + +&i2s1_8ch { + status = "okay"; + #sound-dai-cells = <0>; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_sclkrx + &i2s1m0_lrcktx + &i2s1m0_sclkrx + &i2s1m0_lrckrx + &i2s1m0_sdo0 + &i2s1m0_sdi0 + &i2s1m0_sdi1 + &i2s1m0_sdi2 + &i2s1m0_sdi3>; +}; + + +#if 0 + +&rk809_codec { + #sound-dai-cells = <1>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + pdmdata-out-enable; + adc-for-loopback; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + status = "okay"; +}; +#endif +&rk809_sound { + status = "disabled"; +}; + +&combphy0_us { + status = "okay"; +}; + +&combphy1_usq { + status = "okay"; +}; + +&combphy2_psq { + status = "okay"; +}; + +&gmac0 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0>; + assigned-clock-parents = <&cru SCLK_GMAC0_RGMII_SPEED>, <&cru CLK_MAC0_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_miim + &gmac0_tx_bus2 + &gmac0_rx_bus2 + &gmac0_rgmii_clk + &gmac0_rgmii_bus>; + + tx_delay = <0x37>; + rx_delay = <0x2e>; + + phy-handle = <&rgmii_phy0>; + status = "okay"; +}; + +&gmac1 { + phy-mode = "rgmii"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio2 RK_PD1 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>; + assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>, <&cru CLK_MAC1_2TOP>; + assigned-clock-rates = <0>, <125000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gmac1m1_miim + &gmac1m1_tx_bus2 + &gmac1m1_rx_bus2 + &gmac1m1_rgmii_clk + &gmac1m1_rgmii_bus>; + + tx_delay = <0x47>; + rx_delay = <0x28>; + + phy-handle = <&rgmii_phy1>; + status = "okay"; +}; + +&i2c3 { + status = "okay"; + rt5670: rt5670@1c { + status = "okay"; + #sound-dai-cell = <0>; + compatible = "realtek,rt5670"; + reg = <0x1c>; + }; + + es7210: es7210@40 { + #sound-dai-cells = <0>; + compatible = "MicArray_0"; + reg = <0x40>; + clocks = <&cru I2S1_MCLKOUT_RX>;//csqerr + clock-names = "mclk"; + }; + + es7210_1: es7210@42 { + compatible = "MicArray_1"; + reg = <0x42>; + }; +}; + +&i2c5 { + status = "okay"; + + gs_mxc6655xa: gs_mxc6655xa@15 { + status = "okay"; + compatible = "gs_mxc6655xa"; + pinctrl-names = "default"; + pinctrl-0 = <&mxc6655xa_irq_gpio>; + reg = <0x15>; + irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + power-off-in-suspend = <1>; + layout = <1>; + }; + + mxc6655xa: mxc6655xa@15 { + status = "disabled"; + compatible = "gs_mxc6655xa"; + pinctrl-names = "default"; + pinctrl-0 = <&mxc6655xa_irq_gpio>; + reg = <0x15>; + irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + power-off-in-suspend = <1>; + layout = <1>; + }; + + hym8563: hym8563@51 { + compatible = "haoyu,hym8563"; + reg = <0x51>; + pinctrl-names = "default"; + pinctrl-0 = <&rtc_int>; + + interrupt-parent = <&gpio0>; + interrupts = ; + }; +}; + +&mdio0 { + rgmii_phy0: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&mdio1 { + rgmii_phy1: phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x0>; + }; +}; + +&i2c4 { + status = "okay"; + + gc8034: gc8034@37 { + compatible = "galaxycore,gc8034"; + reg = <0x37>; + clocks = <&cru CLK_CIF_OUT>;//CLK_CAM0_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; + rockchip,grf = <&grf>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "RK-CMK-8M-2-v1"; + rockchip,camera-module-lens-name = "CK8401"; + port { + gc8034_out: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; + + ov9750_1: ov9750_1@36 { + compatible = "ovti,ov9750"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + power-domains = <&power RK3568_PD_VI>; + reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_HIGH>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT0854-FV1"; + rockchip,camera-module-lens-name = "CHT-842B-MD"; + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2>; + }; + }; + }; +#if 0 + os04a10: os04a10@36 { + compatible = "ovti,os04a10"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; + /* power-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; */ + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT1607-FV1"; + /* rockchip,camera-module-lens-name = "M12-4IR-4MP-F16"; */ + rockchip,camera-module-lens-name = "M12-40IRC-4MP-F16"; + port { + ucam_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + }; + }; + }; +#endif + ov5695: ov5695@36 { + status = "okay"; + compatible = "ovti,ov5695"; + reg = <0x36>; + clocks = <&cru CLK_CIF_OUT>; + clock-names = "xvclk"; + power-domains = <&power RK3568_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cif_clk>; + reset-gpios = <&gpio0 RK_PD6 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_LOW>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "TongJu"; + rockchip,camera-module-lens-name = "CHT842-MD"; + port { + ov5695_out: endpoint { + remote-endpoint = <&mipi_in_ucam2>; + data-lanes = <1 2>; + }; + }; + }; +}; + +&csi2_dphy_hw { + status = "okay"; +}; + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&ucam_out0>; + data-lanes = <1 2>; + }; + mipi_in_ucam1: endpoint@2 { + reg = <2>; + remote-endpoint = <&gc8034_out>; + data-lanes = <1 2 3 4>; + }; + mipi_in_ucam2: endpoint@3 { + reg = <3>; + remote-endpoint = <&ov5695_out>; + data-lanes = <1 2>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&isp0_in>; + }; + }; + }; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp0_in: endpoint@0 { + reg = <0>; + remote-endpoint = <&csidphy_out>; + }; + }; +}; + +&video_phy0 { + status = "okay"; +}; + +&video_phy1 { + status = "disabled"; +}; + +/* +&pcie2x1 { + reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie20_3v3>; + status = "okay"; +}; +*/ + +&pcie30phy { + status = "okay"; +}; + +&pcie3x2 { + reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&pcie30_3v3>; + status = "okay"; +}; + +&pinctrl { + headphone { + hp_det: hp-det { + rockchip,pins = <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + leds_gpio: leds-gpio { + rockchip,pins = + <4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>, + <4 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>, + <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + mxc6655xa { + mxc6655xa_irq_gpio: mxc6655xa_irq_gpio { + rockchip,pins = <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +/* + sii902x { + sii902x_hdmi_int: sii902x-hdmi-int { + rockchip,pins = <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +*/ + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; + }; +/* + wifi_32k: wifi-32k {//csqerr + rockchip,pins = <2 RK_PC6 1 &pcfg_pull_none>; + }; +*/ + + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-bluetooth { + uart8_gpios: uart8-gpios { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + rtc { + rtc_int: rtc-int { + rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + i2s1 { + /omit-if-no-ref/ + i2s1m0_lrckrx: i2s1m0-lrckrx { + rockchip,pins = + /* i2s1m0_lrckrx */ + <1 RK_PA6 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_lrcktx: i2s1m0-lrcktx { + rockchip,pins = + /* i2s1m0_lrcktx */ + <1 RK_PA5 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_mclk: i2s1m0-mclk { + rockchip,pins = + /* i2s1m0_mclk */ + <1 RK_PA2 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sclkrx: i2s1m0-sclkrx { + rockchip,pins = + /* i2s1m0_sclkrx */ + <1 RK_PA4 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sclktx: i2s1m0-sclktx { + rockchip,pins = + /* i2s1m0_sclktx */ + <1 RK_PA3 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sdi0: i2s1m0-sdi0 { + rockchip,pins = + /* i2s1m0_sdi0 */ + <1 RK_PB3 1 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sdi1: i2s1m0-sdi1 { + rockchip,pins = + /* i2s1m0_sdi1 */ + <1 RK_PB2 2 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sdi2: i2s1m0-sdi2 { + rockchip,pins = + /* i2s1m0_sdi2 */ + <1 RK_PB1 2 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sdi3: i2s1m0-sdi3 { + rockchip,pins = + /* i2s1m0_sdi3 */ + <1 RK_PB0 2 &pcfg_pull_up_drv_level_4>; + }; + /omit-if-no-ref/ + i2s1m0_sdo0: i2s1m0-sdo0 { + rockchip,pins = + /* i2s1m0_sdo0 */ + <1 RK_PA7 1 &pcfg_pull_up_drv_level_4>; + }; + }; + + cam { + camera_pwr: camera-pwr { + rockchip,pins = + /* camera power en */ + <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + 4g { + vcc_4g_drv: vcc-4g-drv { + rockchip,pins = + <0 RK_PC0 RK_FUNC_GPIO &pcfg_output_low>, + <3 RK_PC5 RK_FUNC_GPIO &pcfg_output_low>, + <3 RK_PB2 RK_FUNC_GPIO &pcfg_output_high>; + }; + }; +}; + +&pwm7 { + status = "okay"; +}; + +&sdio_pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; + post-power-on-delay-ms = <20>; + status = "okay"; +}; + +&sdmmc1 { + status = "disabled"; +}; + +&sdmmc2 { + max-frequency = <150000000>; + supports-sdio; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; + sd-uhs-sdr104; + status = "okay"; +}; + +&uart1 { + status = "disabled"; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer &uart1m0_ctsn>; +}; + +&uart8 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn>; +}; + +&vcc3v3_lcd0_n { + gpio = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&vcc3v3_lcd1_n { + gpio = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + enable-active-high; +}; + +&wireless_wlan { + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>; +}; + +&wireless_bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart8m0_rtsn>; + pinctrl-1 = <&uart8_gpios>; + BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&sata2 { + status = "okay"; +}; + +&leds { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-toybrick.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-toybrick.dtsi new file mode 100755 index 000000000000..8be2d36f38c0 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568-toybrick.dtsi @@ -0,0 +1,1891 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + */ + +#include +#include +#include +#include +#include +#include + +/ { + /* + * extliux conf: extlinux.conf.${FLAG}.${BOARD_ID} + * dtb file: toybrick.dtb.${FLAG}.${BOARD_ID} + */ + board_id: board-id { + compatible = "board-id"; + io-channels = <&saradc 4>; + /* + * ID: adc-value/adc-io + * ------------------------- + * 0: adc-io is low level + * 1: 0 ~ 100 + * 2: 100 ~ 199 + * 3: 200 ~ 299 + * 4: 300 ~ 399 + * 5: 400 ~ 499 + * 6: 500 ~ 599 + * 7: 600 ~ 699 + * 8: 700 ~ 799 + * 9: 800 ~ 899 + * 10: 900 ~ 1024 + */ + adc-io = <29>; // GPIO0_D5 + thresholds = <100 200 300 400 500 600 700 800 900>; + }; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <100>; + + vol-up-key { + label = "volume up"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + + vol-down-key { + label = "volume down"; + linux,code = ; + press-threshold-microvolt = <297500>; + }; + + menu-key { + label = "menu"; + linux,code = ; + press-threshold-microvolt = <980000>; + }; + + back-key { + label = "back"; + linux,code = ; + press-threshold-microvolt = <1305500>; + }; + }; + + audiopwmout_diff: audiopwmout-diff { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,audiopwmout-diff"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,bitclock-master = <&master>; + simple-audio-card,frame-master = <&master>; + simple-audio-card,cpu { + sound-dai = <&i2s3_2ch>; + }; + master: simple-audio-card,codec { + sound-dai = <&dig_acodec>; + }; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm4 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + }; + + backlight1: backlight1 { + compatible = "pwm-backlight"; + pwms = <&pwm5 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + }; + + dc_12v: dc-12v { + compatible = "regulator-fixed"; + regulator-name = "dc_12v"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + }; + + hdmi_sound: hdmi-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <128>; + simple-audio-card,name = "rockchip,hdmi"; + status = "disabled"; + + simple-audio-card,cpu { + sound-dai = <&i2s0_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&hdmi>; + }; + }; + + leds: leds { + compatible = "gpio-leds"; + work_led: work { + gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + }; + }; + + pdmics: dummy-codec { + status = "disabled"; + compatible = "rockchip,dummy-codec"; + #sound-dai-cells = <0>; + }; + + pdm_mic_array: pdm-mic-array { + status = "disabled"; + compatible = "simple-audio-card"; + simple-audio-card,name = "rockchip,pdm-mic-array"; + simple-audio-card,cpu { + sound-dai = <&pdm>; + }; + simple-audio-card,codec { + sound-dai = <&pdmics>; + }; + }; + + rk809_sound: rk809-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rockchip,rk809-codec"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&rk809_codec>; + }; + }; + + spdif-sound { + status = "okay"; + compatible = "simple-audio-card"; + simple-audio-card,name = "ROCKCHIP,SPDIF"; + simple-audio-card,cpu { + sound-dai = <&spdif_8ch>; + }; + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + status = "okay"; + compatible = "linux,spdif-dit"; + #sound-dai-cells = <0>; + }; + + vad_sound: vad-sound { + status = "disabled"; + compatible = "rockchip,multicodecs-card"; + rockchip,card-name = "rockchip,rk3568-vad"; + rockchip,cpu = <&i2s1_8ch>; + rockchip,codec = <&rk809_codec>, <&vad>; + }; + + bt-sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "dsp_b"; + simple-audio-card,bitclock-inversion = <1>; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,name = "rockchip,bt"; + #simple-audio-card,bitclock-master = <&sound2_master>; + #simple-audio-card,frame-master = <&sound2_master>; + simple-audio-card,cpu { + sound-dai = <&i2s3_2ch>; + }; + sound2_master:simple-audio-card,codec { + #sound-dai-cells = <0>; + sound-dai = <&bt_sco>; + }; + }; + + bt_sco: bt-sco { + compatible = "delta,dfbmcs320"; + #sound-dai-cells = <0>; + status = "okay"; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&dc_12v>; + }; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&dc_12v>; + }; + + vcc5v0_host: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_host_en>; + regulator-name = "vcc5v0_host"; + regulator-always-on; + }; + + vcc5v0_otg: vcc5v0-otg-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_otg_en>; + regulator-name = "vcc5v0_otg"; + }; + + vcc3v3_lcd0_n: vcc3v3-lcd0-n { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_lcd0_n"; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_lcd1_n: vcc3v3-lcd1-n { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_lcd1_n"; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>; + }; + + wireless_wlan: wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + wifi_chip_type = "ap6398s"; + status = "okay"; + }; + + wireless_bluetooth: wireless-bluetooth { + compatible = "bluetooth-platdata"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + //wifi-bt-power-toggle; + uart_rts_gpios = <&gpio2 RK_PB1 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart8m0_rtsn>; + pinctrl-1 = <&uart8_gpios>; + BT,reset_gpio = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + test-power { + status = "okay"; + }; +}; + +&i2s3_2ch { + status = "okay"; +}; + +&bus_npu { + bus-supply = <&vdd_logic>; + pvtm-supply = <&vdd_cpu>; + status = "okay"; +}; +&can0 { + assigned-clocks = <&cru CLK_CAN0>; + assigned-clock-rates = <150000000>; + pinctrl-names = "default"; + pinctrl-0 = <&can0m1_pins>; + status = "disabled"; +}; + +&can1 { + assigned-clocks = <&cru CLK_CAN1>; + assigned-clock-rates = <150000000>; + pinctrl-names = "default"; + pinctrl-0 = <&can1m1_pins>; + status = "disabled"; +}; + +&can2 { + assigned-clocks = <&cru CLK_CAN2>; + assigned-clock-rates = <150000000>; + pinctrl-names = "default"; + pinctrl-0 = <&can2m1_pins>; + status = "disabled"; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&dfi { + status = "okay"; +}; + +&dmc { + center-supply = <&vdd_logic>; + status = "okay"; +}; + +&dsi0 { + status = "disabled"; + //rockchip,lane-rate = <1000>; + dsi0_panel: panel@0 { + status = "okay"; + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight>; + reset-delay-ms = <60>; + enable-delay-ms = <60>; + prepare-delay-ms = <60>; + unprepare-delay-ms = <60>; + disable-delay-ms = <60>; + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + panel-init-sequence = [ + 23 00 02 FE 21 + 23 00 02 04 00 + 23 00 02 00 64 + 23 00 02 2A 00 + 23 00 02 26 64 + 23 00 02 54 00 + 23 00 02 50 64 + 23 00 02 7B 00 + 23 00 02 77 64 + 23 00 02 A2 00 + 23 00 02 9D 64 + 23 00 02 C9 00 + 23 00 02 C5 64 + 23 00 02 01 71 + 23 00 02 27 71 + 23 00 02 51 71 + 23 00 02 78 71 + 23 00 02 9E 71 + 23 00 02 C6 71 + 23 00 02 02 89 + 23 00 02 28 89 + 23 00 02 52 89 + 23 00 02 79 89 + 23 00 02 9F 89 + 23 00 02 C7 89 + 23 00 02 03 9E + 23 00 02 29 9E + 23 00 02 53 9E + 23 00 02 7A 9E + 23 00 02 A0 9E + 23 00 02 C8 9E + 23 00 02 09 00 + 23 00 02 05 B0 + 23 00 02 31 00 + 23 00 02 2B B0 + 23 00 02 5A 00 + 23 00 02 55 B0 + 23 00 02 80 00 + 23 00 02 7C B0 + 23 00 02 A7 00 + 23 00 02 A3 B0 + 23 00 02 CE 00 + 23 00 02 CA B0 + 23 00 02 06 C0 + 23 00 02 2D C0 + 23 00 02 56 C0 + 23 00 02 7D C0 + 23 00 02 A4 C0 + 23 00 02 CB C0 + 23 00 02 07 CF + 23 00 02 2F CF + 23 00 02 58 CF + 23 00 02 7E CF + 23 00 02 A5 CF + 23 00 02 CC CF + 23 00 02 08 DD + 23 00 02 30 DD + 23 00 02 59 DD + 23 00 02 7F DD + 23 00 02 A6 DD + 23 00 02 CD DD + 23 00 02 0E 15 + 23 00 02 0A E9 + 23 00 02 36 15 + 23 00 02 32 E9 + 23 00 02 5F 15 + 23 00 02 5B E9 + 23 00 02 85 15 + 23 00 02 81 E9 + 23 00 02 AD 15 + 23 00 02 A9 E9 + 23 00 02 D3 15 + 23 00 02 CF E9 + 23 00 02 0B 14 + 23 00 02 33 14 + 23 00 02 5C 14 + 23 00 02 82 14 + 23 00 02 AA 14 + 23 00 02 D0 14 + 23 00 02 0C 36 + 23 00 02 34 36 + 23 00 02 5D 36 + 23 00 02 83 36 + 23 00 02 AB 36 + 23 00 02 D1 36 + 23 00 02 0D 6B + 23 00 02 35 6B + 23 00 02 5E 6B + 23 00 02 84 6B + 23 00 02 AC 6B + 23 00 02 D2 6B + 23 00 02 13 5A + 23 00 02 0F 94 + 23 00 02 3B 5A + 23 00 02 37 94 + 23 00 02 64 5A + 23 00 02 60 94 + 23 00 02 8A 5A + 23 00 02 86 94 + 23 00 02 B2 5A + 23 00 02 AE 94 + 23 00 02 D8 5A + 23 00 02 D4 94 + 23 00 02 10 D1 + 23 00 02 38 D1 + 23 00 02 61 D1 + 23 00 02 87 D1 + 23 00 02 AF D1 + 23 00 02 D5 D1 + 23 00 02 11 04 + 23 00 02 39 04 + 23 00 02 62 04 + 23 00 02 88 04 + 23 00 02 B0 04 + 23 00 02 D6 04 + 23 00 02 12 05 + 23 00 02 3A 05 + 23 00 02 63 05 + 23 00 02 89 05 + 23 00 02 B1 05 + 23 00 02 D7 05 + 23 00 02 18 AA + 23 00 02 14 36 + 23 00 02 42 AA + 23 00 02 3D 36 + 23 00 02 69 AA + 23 00 02 65 36 + 23 00 02 8F AA + 23 00 02 8B 36 + 23 00 02 B7 AA + 23 00 02 B3 36 + 23 00 02 DD AA + 23 00 02 D9 36 + 23 00 02 15 74 + 23 00 02 3F 74 + 23 00 02 66 74 + 23 00 02 8C 74 + 23 00 02 B4 74 + 23 00 02 DA 74 + 23 00 02 16 9F + 23 00 02 40 9F + 23 00 02 67 9F + 23 00 02 8D 9F + 23 00 02 B5 9F + 23 00 02 DB 9F + 23 00 02 17 DC + 23 00 02 41 DC + 23 00 02 68 DC + 23 00 02 8E DC + 23 00 02 B6 DC + 23 00 02 DC DC + 23 00 02 1D FF + 23 00 02 19 03 + 23 00 02 47 FF + 23 00 02 43 03 + 23 00 02 6E FF + 23 00 02 6A 03 + 23 00 02 94 FF + 23 00 02 90 03 + 23 00 02 BC FF + 23 00 02 B8 03 + 23 00 02 E2 FF + 23 00 02 DE 03 + 23 00 02 1A 35 + 23 00 02 44 35 + 23 00 02 6B 35 + 23 00 02 91 35 + 23 00 02 B9 35 + 23 00 02 DF 35 + 23 00 02 1B 45 + 23 00 02 45 45 + 23 00 02 6C 45 + 23 00 02 92 45 + 23 00 02 BA 45 + 23 00 02 E0 45 + 23 00 02 1C 55 + 23 00 02 46 55 + 23 00 02 6D 55 + 23 00 02 93 55 + 23 00 02 BB 55 + 23 00 02 E1 55 + 23 00 02 22 FF + 23 00 02 1E 68 + 23 00 02 4C FF + 23 00 02 48 68 + 23 00 02 73 FF + 23 00 02 6F 68 + 23 00 02 99 FF + 23 00 02 95 68 + 23 00 02 C1 FF + 23 00 02 BD 68 + 23 00 02 E7 FF + 23 00 02 E3 68 + 23 00 02 1F 7E + 23 00 02 49 7E + 23 00 02 70 7E + 23 00 02 96 7E + 23 00 02 BE 7E + 23 00 02 E4 7E + 23 00 02 20 97 + 23 00 02 4A 97 + 23 00 02 71 97 + 23 00 02 97 97 + 23 00 02 BF 97 + 23 00 02 E5 97 + 23 00 02 21 B5 + 23 00 02 4B B5 + 23 00 02 72 B5 + 23 00 02 98 B5 + 23 00 02 C0 B5 + 23 00 02 E6 B5 + 23 00 02 25 F0 + 23 00 02 23 E8 + 23 00 02 4F F0 + 23 00 02 4D E8 + 23 00 02 76 F0 + 23 00 02 74 E8 + 23 00 02 9C F0 + 23 00 02 9A E8 + 23 00 02 C4 F0 + 23 00 02 C2 E8 + 23 00 02 EA F0 + 23 00 02 E8 E8 + 23 00 02 24 FF + 23 00 02 4E FF + 23 00 02 75 FF + 23 00 02 9B FF + 23 00 02 C3 FF + 23 00 02 E9 FF + 23 00 02 FE 3D + 23 00 02 00 04 + 23 00 02 FE 23 + 23 00 02 08 82 + 23 00 02 0A 00 + 23 00 02 0B 00 + 23 00 02 0C 01 + 23 00 02 16 00 + 23 00 02 18 02 + 23 00 02 1B 04 + 23 00 02 19 04 + 23 00 02 1C 81 + 23 00 02 1F 00 + 23 00 02 20 03 + 23 00 02 23 04 + 23 00 02 21 01 + 23 00 02 54 63 + 23 00 02 55 54 + 23 00 02 6E 45 + 23 00 02 6D 36 + 23 00 02 FE 3D + 23 00 02 55 78 + 23 00 02 FE 20 + 23 00 02 26 30 + 23 00 02 FE 3D + 23 00 02 20 71 + 23 00 02 50 8F + 23 00 02 51 8F + 23 00 02 FE 00 + 23 00 02 35 00 + 05 78 01 11 + 05 1E 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + disp_timings0: display-timings { + native-mode = <&dsi0_timing0>; + dsi0_timing0: timing0 { + clock-frequency = <132000000>; + hactive = <1080>; + vactive = <1920>; + hfront-porch = <15>; + hsync-len = <2>; + hback-porch = <30>; + vfront-porch = <15>; + vsync-len = <2>; + vback-porch = <15>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <1>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; + +}; + +&dsi1 { + status = "disabled"; + //rockchip,lane-rate = <1000>; + dsi1_panel: panel@0 { + status = "okay"; + compatible = "simple-panel-dsi"; + reg = <0>; + backlight = <&backlight1>; + reset-delay-ms = <60>; + enable-delay-ms = <60>; + prepare-delay-ms = <60>; + unprepare-delay-ms = <60>; + disable-delay-ms = <60>; + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + panel-init-sequence = [ + 23 00 02 FE 21 + 23 00 02 04 00 + 23 00 02 00 64 + 23 00 02 2A 00 + 23 00 02 26 64 + 23 00 02 54 00 + 23 00 02 50 64 + 23 00 02 7B 00 + 23 00 02 77 64 + 23 00 02 A2 00 + 23 00 02 9D 64 + 23 00 02 C9 00 + 23 00 02 C5 64 + 23 00 02 01 71 + 23 00 02 27 71 + 23 00 02 51 71 + 23 00 02 78 71 + 23 00 02 9E 71 + 23 00 02 C6 71 + 23 00 02 02 89 + 23 00 02 28 89 + 23 00 02 52 89 + 23 00 02 79 89 + 23 00 02 9F 89 + 23 00 02 C7 89 + 23 00 02 03 9E + 23 00 02 29 9E + 23 00 02 53 9E + 23 00 02 7A 9E + 23 00 02 A0 9E + 23 00 02 C8 9E + 23 00 02 09 00 + 23 00 02 05 B0 + 23 00 02 31 00 + 23 00 02 2B B0 + 23 00 02 5A 00 + 23 00 02 55 B0 + 23 00 02 80 00 + 23 00 02 7C B0 + 23 00 02 A7 00 + 23 00 02 A3 B0 + 23 00 02 CE 00 + 23 00 02 CA B0 + 23 00 02 06 C0 + 23 00 02 2D C0 + 23 00 02 56 C0 + 23 00 02 7D C0 + 23 00 02 A4 C0 + 23 00 02 CB C0 + 23 00 02 07 CF + 23 00 02 2F CF + 23 00 02 58 CF + 23 00 02 7E CF + 23 00 02 A5 CF + 23 00 02 CC CF + 23 00 02 08 DD + 23 00 02 30 DD + 23 00 02 59 DD + 23 00 02 7F DD + 23 00 02 A6 DD + 23 00 02 CD DD + 23 00 02 0E 15 + 23 00 02 0A E9 + 23 00 02 36 15 + 23 00 02 32 E9 + 23 00 02 5F 15 + 23 00 02 5B E9 + 23 00 02 85 15 + 23 00 02 81 E9 + 23 00 02 AD 15 + 23 00 02 A9 E9 + 23 00 02 D3 15 + 23 00 02 CF E9 + 23 00 02 0B 14 + 23 00 02 33 14 + 23 00 02 5C 14 + 23 00 02 82 14 + 23 00 02 AA 14 + 23 00 02 D0 14 + 23 00 02 0C 36 + 23 00 02 34 36 + 23 00 02 5D 36 + 23 00 02 83 36 + 23 00 02 AB 36 + 23 00 02 D1 36 + 23 00 02 0D 6B + 23 00 02 35 6B + 23 00 02 5E 6B + 23 00 02 84 6B + 23 00 02 AC 6B + 23 00 02 D2 6B + 23 00 02 13 5A + 23 00 02 0F 94 + 23 00 02 3B 5A + 23 00 02 37 94 + 23 00 02 64 5A + 23 00 02 60 94 + 23 00 02 8A 5A + 23 00 02 86 94 + 23 00 02 B2 5A + 23 00 02 AE 94 + 23 00 02 D8 5A + 23 00 02 D4 94 + 23 00 02 10 D1 + 23 00 02 38 D1 + 23 00 02 61 D1 + 23 00 02 87 D1 + 23 00 02 AF D1 + 23 00 02 D5 D1 + 23 00 02 11 04 + 23 00 02 39 04 + 23 00 02 62 04 + 23 00 02 88 04 + 23 00 02 B0 04 + 23 00 02 D6 04 + 23 00 02 12 05 + 23 00 02 3A 05 + 23 00 02 63 05 + 23 00 02 89 05 + 23 00 02 B1 05 + 23 00 02 D7 05 + 23 00 02 18 AA + 23 00 02 14 36 + 23 00 02 42 AA + 23 00 02 3D 36 + 23 00 02 69 AA + 23 00 02 65 36 + 23 00 02 8F AA + 23 00 02 8B 36 + 23 00 02 B7 AA + 23 00 02 B3 36 + 23 00 02 DD AA + 23 00 02 D9 36 + 23 00 02 15 74 + 23 00 02 3F 74 + 23 00 02 66 74 + 23 00 02 8C 74 + 23 00 02 B4 74 + 23 00 02 DA 74 + 23 00 02 16 9F + 23 00 02 40 9F + 23 00 02 67 9F + 23 00 02 8D 9F + 23 00 02 B5 9F + 23 00 02 DB 9F + 23 00 02 17 DC + 23 00 02 41 DC + 23 00 02 68 DC + 23 00 02 8E DC + 23 00 02 B6 DC + 23 00 02 DC DC + 23 00 02 1D FF + 23 00 02 19 03 + 23 00 02 47 FF + 23 00 02 43 03 + 23 00 02 6E FF + 23 00 02 6A 03 + 23 00 02 94 FF + 23 00 02 90 03 + 23 00 02 BC FF + 23 00 02 B8 03 + 23 00 02 E2 FF + 23 00 02 DE 03 + 23 00 02 1A 35 + 23 00 02 44 35 + 23 00 02 6B 35 + 23 00 02 91 35 + 23 00 02 B9 35 + 23 00 02 DF 35 + 23 00 02 1B 45 + 23 00 02 45 45 + 23 00 02 6C 45 + 23 00 02 92 45 + 23 00 02 BA 45 + 23 00 02 E0 45 + 23 00 02 1C 55 + 23 00 02 46 55 + 23 00 02 6D 55 + 23 00 02 93 55 + 23 00 02 BB 55 + 23 00 02 E1 55 + 23 00 02 22 FF + 23 00 02 1E 68 + 23 00 02 4C FF + 23 00 02 48 68 + 23 00 02 73 FF + 23 00 02 6F 68 + 23 00 02 99 FF + 23 00 02 95 68 + 23 00 02 C1 FF + 23 00 02 BD 68 + 23 00 02 E7 FF + 23 00 02 E3 68 + 23 00 02 1F 7E + 23 00 02 49 7E + 23 00 02 70 7E + 23 00 02 96 7E + 23 00 02 BE 7E + 23 00 02 E4 7E + 23 00 02 20 97 + 23 00 02 4A 97 + 23 00 02 71 97 + 23 00 02 97 97 + 23 00 02 BF 97 + 23 00 02 E5 97 + 23 00 02 21 B5 + 23 00 02 4B B5 + 23 00 02 72 B5 + 23 00 02 98 B5 + 23 00 02 C0 B5 + 23 00 02 E6 B5 + 23 00 02 25 F0 + 23 00 02 23 E8 + 23 00 02 4F F0 + 23 00 02 4D E8 + 23 00 02 76 F0 + 23 00 02 74 E8 + 23 00 02 9C F0 + 23 00 02 9A E8 + 23 00 02 C4 F0 + 23 00 02 C2 E8 + 23 00 02 EA F0 + 23 00 02 E8 E8 + 23 00 02 24 FF + 23 00 02 4E FF + 23 00 02 75 FF + 23 00 02 9B FF + 23 00 02 C3 FF + 23 00 02 E9 FF + 23 00 02 FE 3D + 23 00 02 00 04 + 23 00 02 FE 23 + 23 00 02 08 82 + 23 00 02 0A 00 + 23 00 02 0B 00 + 23 00 02 0C 01 + 23 00 02 16 00 + 23 00 02 18 02 + 23 00 02 1B 04 + 23 00 02 19 04 + 23 00 02 1C 81 + 23 00 02 1F 00 + 23 00 02 20 03 + 23 00 02 23 04 + 23 00 02 21 01 + 23 00 02 54 63 + 23 00 02 55 54 + 23 00 02 6E 45 + 23 00 02 6D 36 + 23 00 02 FE 3D + 23 00 02 55 78 + 23 00 02 FE 20 + 23 00 02 26 30 + 23 00 02 FE 3D + 23 00 02 20 71 + 23 00 02 50 8F + 23 00 02 51 8F + 23 00 02 FE 00 + 23 00 02 35 00 + 05 78 01 11 + 05 1E 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + disp_timings1: display-timings { + native-mode = <&dsi1_timing0>; + dsi1_timing0: timing0 { + clock-frequency = <132000000>; + hactive = <1080>; + vactive = <1920>; + hfront-porch = <15>; + hsync-len = <2>; + hback-porch = <30>; + vfront-porch = <15>; + vsync-len = <2>; + vback-porch = <15>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <1>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi1: endpoint { + remote-endpoint = <&dsi1_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi1_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi1>; + }; + }; + }; + +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + status = "okay"; + rockchip,phy-table = + <92812500 0x8009 0x0000 0x0270>, + <165000000 0x800b 0x0000 0x026d>, + <185625000 0x800b 0x0000 0x01ed>, + <297000000 0x800b 0x0000 0x01ad>, + <594000000 0x8029 0x0000 0x0088>, + <000000000 0x0000 0x0000 0x0000>; +}; + +&hdmi_in_vp0 { + status = "okay"; +}; + +&hdmi_in_vp1 { + status = "disabled"; +}; + +&hdmi_sound { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: tcs4525@1c { + compatible = "tcs,tcs452x"; + reg = <0x1c>; + vin-supply = <&vcc5v0_sys>; + regulator-compatible = "fan53555-reg"; + regulator-name = "vdd_cpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <2300>; + fcs,suspend-voltage-selector = <1>; + regulator-boot-on; + regulator-always-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default", "pmic-sleep", + "pmic-power-off", "pmic-reset"; + pinctrl-0 = <&pmic_int>; + pinctrl-1 = <&soc_slppin_slp>, <&rk817_slppin_slp>; + pinctrl-2 = <&soc_slppin_gpio>, <&rk817_slppin_pwrdn>; + pinctrl-3 = <&soc_slppin_gpio>, <&rk817_slppin_rst>; + + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + //fb-inner-reg-idxs = <2>; + /* 1: rst regs (default in codes), 0: rst the pmic */ + pmic-reset-func = <0>; + /* not save the PMIC_POWER_EN register in uboot */ + not-save-power-en = <1>; + + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc5-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk8xx: pinctrl_rk8xx { + gpio-controller; + #gpio-cells = <2>; + + rk817_slppin_null: rk817_slppin_null { + pins = "gpio_slp"; + function = "pin_fun0"; + }; + + rk817_slppin_slp: rk817_slppin_slp { + pins = "gpio_slp"; + function = "pin_fun1"; + }; + + rk817_slppin_pwrdn: rk817_slppin_pwrdn { + pins = "gpio_slp"; + function = "pin_fun2"; + }; + + rk817_slppin_rst: rk817_slppin_rst { + pins = "gpio_slp"; + function = "pin_fun3"; + }; + }; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_gpu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vdd_npu: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_npu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_image: LDO_REG1 { + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_image"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vccio_acodec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca_1v8: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca1v8_pmu: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcca1v8_image: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_image"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v3: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_sd: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc3v3_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + + rk809_codec: codec { + #sound-dai-cells = <0>; + compatible = "rockchip,rk809-codec", "rockchip,rk817-codec"; + clocks = <&cru I2S1_MCLKOUT>; + clock-names = "mclk"; + assigned-clocks = <&cru I2S1_MCLKOUT>, <&cru I2S1_MCLK_TX_IOE>; + assigned-clock-rates = <12288000>; + assigned-clock-parents = <&cru I2S1_MCLKOUT_TX>, <&cru I2S1_MCLKOUT_TX>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>; + hp-volume = <20>; + spk-volume = <3>; + mic-in-differential; + status = "okay"; + }; + }; +}; + +&i2c1 { + status = "okay"; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + status = "disabled"; + reg = <0x14>; + pinctrl-names = "default"; + pinctrl-0 = <&touch_gpio>; + goodix,rst-gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + }; + + gt9xx: gt9xx@5d { + compatible = "goodix,gt9xx"; + status = "okay"; + reg = <0x5d>; + reset-gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + touch-gpio = <&gpio0 RK_PB5 IRQ_TYPE_LEVEL_LOW>; + max-x = <7200>; + max-y = <1280>; + tp-size = <911>; + pinctrl-names = "default"; + pinctrl-0 = <&touch_gpio>; + power-supply = <&vcc3v3_lcd0_n>; + }; +}; + +&i2c5 { + status = "okay"; + + mxc6655xa: mxc6655xa@15 { + status = "okay"; + compatible = "gs_mxc6655xa"; + pinctrl-names = "default"; + pinctrl-0 = <&mxc6655xa_irq_gpio>; + reg = <0x15>; + irq-gpio = <&gpio3 RK_PC1 IRQ_TYPE_LEVEL_LOW>; + irq_enable = <0>; + poll_delay_ms = <30>; + type = ; + power-off-in-suspend = <1>; + layout = <1>; + }; +}; + +&i2s0_8ch { + status = "okay"; +}; + +&i2s1_8ch { + status = "okay"; + rockchip,clk-trcm = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&jpegd { + status = "okay"; +}; + +&jpegd_mmu { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&nandc0 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + nand@0 { + reg = <0>; + nand-bus-width = <8>; + nand-ecc-mode = "hw"; + nand-ecc-strength = <16>; + nand-ecc-step-size = <1024>; + }; +}; + +&pinctrl { + + mxc6655xa { + mxc6655xa_irq_gpio: mxc6655xa_irq_gpio { + rockchip,pins = <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int: pmic_int { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + soc_slppin_gpio: soc_slppin_gpio { + rockchip,pins = + <0 RK_PA2 RK_FUNC_GPIO &pcfg_output_low_pull_down>; + }; + + soc_slppin_slp: soc_slppin_slp { + rockchip,pins = + <0 RK_PA2 1 &pcfg_pull_up>; + }; + + soc_slppin_rst: soc_slppin_rst { + rockchip,pins = + <0 RK_PA2 2 &pcfg_pull_none>; + }; + }; + + touch { + touch_gpio: touch-gpio { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb { + vcc5v0_host_en: vcc5v0-host-en { + rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + vcc5v0_otg_en: vcc5v0-otg-en { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart8_gpios: uart8-gpios { + rockchip,pins = <2 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +/** + * Model: TB-RK3568X + * ----------------------------------------------------------- + * There are 10 independent IO domains in RK3566/RK3568, including PMUIO[0:2] and VCCIO[1:7]. + * 1/ PMUIO0 and PMUIO1 are fixed-level power domains which cannot be configured; + * 2/ PMUIO2 and VCCIO1,VCCIO[3:7] domains require that their hardware power supply voltages + * must be consistent with the software configuration correspondingly + * a/ When the hardware IO level is connected to 1.8V, the software voltage configuration + * should also be configured to 1.8V accordingly; + * b/ When the hardware IO level is connected to 3.3V, the software voltage configuration + * should also be configured to 3.3V accordingly; + * 3/ VCCIO2 voltage control selection (0xFDC20140) + * BIT[0]: 0x0: from GPIO_0A7 (default) + * BIT[0]: 0x1: from GRF + * Default is determined by Pin FLASH_VOL_SEL/GPIO0_A7: + * L:VCCIO2 must supply 3.3V + * H:VCCIO2 must supply 1.8V + * | supply | domain | net | source | voltage | + * ----------------------------------------------------------- + * | pmuio1-supply | PMUIO1 | vcc3v3_pmu | LDO6 | 3.3V | + * | pmuio2-supply | PMUIO2 | vcc3v3_pmu | LDO6 | 3.3V | + * | vccio1-supply | VCCIO1 | vccio_acodec | LDO4 | 1.8V | + * | vccio2-supply | VCCIO2 | vccio_flash | vcc_1v8 | 1.8V | + * | vccio3-supply | VCCIO3 | vccio_sd | LDO5 | 3.3V | + * | vccio4-supply | VCCIO4 | vcc_1v8 | DCDC5 | 1.8V | + * | vccio5-supply | VCCIO5 | vcc_3v3 | SWITCH1 | 3.3V | + * | vccio6-supply | VCCIO6 | vcc_1v8 | DCDC5 | 1.8V | + * | vccio7-supply | VCCIO7 | vcc_3v3 | SWITCH1 | 3.3V | + * ----------------------------------------------------------- + */ +&pmu_io_domains { + status = "okay"; + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcc3v3_pmu>; + vccio1-supply = <&vccio_acodec>; + // vccio2-supply = <&vccio_flash>; + vccio3-supply = <&vccio_sd>; + vccio4-supply = <&vcc_1v8>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc_1v8>; + vccio7-supply = <&vcc_3v3>; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm4 { + status = "okay"; +}; + +&pwm5 { + status = "okay"; +}; + +&pwm7 { + status = "okay"; + + compatible = "rockchip,remotectl-pwm"; + remote_pwm_id = <3>; + handle_cpu_id = <1>; + remote_support_psci = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pwm7_pins>; + + ir_key1 { + rockchip,usercode = <0x4040>; + rockchip,key_table = + <0xf2 KEY_REPLY>, + <0xba KEY_BACK>, + <0xf4 KEY_UP>, + <0xf1 KEY_DOWN>, + <0xef KEY_LEFT>, + <0xee KEY_RIGHT>, + <0xbd KEY_HOME>, + <0xea KEY_VOLUMEUP>, + <0xe3 KEY_VOLUMEDOWN>, + <0xe2 KEY_SEARCH>, + <0xb2 KEY_POWER>, + <0xbc KEY_MUTE>, + <0xec KEY_MENU>, + <0xbf 0x190>, + <0xe0 0x191>, + <0xe1 0x192>, + <0xe9 183>, + <0xe6 248>, + <0xe8 185>, + <0xe7 186>, + <0xf0 388>, + <0xbe 0x175>; + }; + + ir_key2 { + rockchip,usercode = <0xff00>; + rockchip,key_table = + <0xf9 KEY_HOME>, + <0xbf KEY_BACK>, + <0xfb KEY_MENU>, + <0xaa KEY_REPLY>, + <0xb9 KEY_UP>, + <0xe9 KEY_DOWN>, + <0xb8 KEY_LEFT>, + <0xea KEY_RIGHT>, + <0xeb KEY_VOLUMEDOWN>, + <0xef KEY_VOLUMEUP>, + <0xf7 KEY_MUTE>, + <0xe7 KEY_POWER>, + <0xfc KEY_POWER>, + <0xa9 KEY_VOLUMEDOWN>, + <0xa8 KEY_VOLUMEDOWN>, + <0xe0 KEY_VOLUMEDOWN>, + <0xa5 KEY_VOLUMEDOWN>, + <0xab 183>, + <0xb7 388>, + <0xe8 388>, + <0xf8 184>, + <0xaf 185>, + <0xed KEY_VOLUMEDOWN>, + <0xee 186>, + <0xb3 KEY_VOLUMEDOWN>, + <0xf1 KEY_VOLUMEDOWN>, + <0xf2 KEY_VOLUMEDOWN>, + <0xf3 KEY_SEARCH>, + <0xb4 KEY_VOLUMEDOWN>, + <0xbe KEY_SEARCH>; + }; + + ir_key3 { + rockchip,usercode = <0x1dcc>; + rockchip,key_table = + <0xee KEY_REPLY>, + <0xf0 KEY_BACK>, + <0xf8 KEY_UP>, + <0xbb KEY_DOWN>, + <0xef KEY_LEFT>, + <0xed KEY_RIGHT>, + <0xfc KEY_HOME>, + <0xf1 KEY_VOLUMEUP>, + <0xfd KEY_VOLUMEDOWN>, + <0xb7 KEY_SEARCH>, + <0xff KEY_POWER>, + <0xf3 KEY_MUTE>, + <0xbf KEY_MENU>, + <0xf9 0x191>, + <0xf5 0x192>, + <0xb3 388>, + <0xbe KEY_1>, + <0xba KEY_2>, + <0xb2 KEY_3>, + <0xbd KEY_4>, + <0xf9 KEY_5>, + <0xb1 KEY_6>, + <0xfc KEY_7>, + <0xf8 KEY_8>, + <0xb0 KEY_9>, + <0xb6 KEY_0>, + <0xb5 KEY_BACKSPACE>; + }; +}; + +&rk_rga { + status = "okay"; +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&rkvenc { + venc-supply = <&vdd_logic>; + status = "okay"; +}; + +&rkvenc_mmu { + status = "okay"; +}; + +&rknpu { + rknpu-supply = <&vdd_npu>; + status = "okay"; +}; + +&rknpu_mmu { + status = "okay"; +}; + +&route_hdmi { + status = "okay"; + connect = <&vp0_out_hdmi>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcca_1v8>; +}; + +&sdhci { + bus-width = <8>; + supports-emmc; + non-removable; + max-frequency = <200000000>; + status = "okay"; +}; + +&sdmmc0 { + max-frequency = <150000000>; + supports-sd; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + sd-uhs-sdr104; + vmmc-supply = <&vcc3v3_sd>; + vqmmc-supply = <&vccio_sd>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; + status = "okay"; +}; + +&sfc { + status = "okay"; +}; + +&spdif_8ch { + status = "okay"; +}; + +&tsadc { + status = "okay"; +}; + +&u2phy0_host { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&u2phy0_otg { + vbus-supply = <&vcc5v0_otg>; + status = "okay"; +}; + +&u2phy1_host { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&u2phy1_otg { + phy-supply = <&vcc5v0_host>; + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usb2phy1 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd_dwc3 { + dr_mode = "otg"; + extcon = <&usb2phy0>; + status = "okay"; +}; + +&usbdrd30 { + status = "okay"; +}; + +&usbhost_dwc3 { + status = "okay"; +}; + +&usbhost30 { + status = "okay"; +}; + +&vad { + rockchip,audio-src = <&i2s1_8ch>; + rockchip,buffer-time-ms = <128>; + rockchip,det-channel = <0>; + rockchip,mode = <0>; +}; + +&vdpu { + status = "okay"; +}; + +&vdpu_mmu { + status = "okay"; +}; + +&vepu { + status = "okay"; +}; + +&vepu_mmu { + status = "okay"; +}; + +&vop { + status = "okay"; + assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>; + assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>; +}; + +&vop_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568.dtsi b/arch/arm64/boot/dts/rockchip/rk3568.dtsi new file mode 100755 index 000000000000..372617fee467 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3568.dtsi @@ -0,0 +1,3449 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "rk3568-dram-default-timing.dtsi" + +/ { + compatible = "rockchip,rk3568"; + + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + csi2dphy0 = &csi2_dphy0; + csi2dphy1 = &csi2_dphy1; + csi2dphy2 = &csi2_dphy2; + dsi0 = &dsi0; + dsi1 = &dsi1; + ethernet0 = &gmac0; + ethernet1 = &gmac1; + gpio0 = &gpio0; + gpio1 = &gpio1; + gpio2 = &gpio2; + gpio3 = &gpio3; + gpio4 = &gpio4; + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + mmc0 = &sdhci; + mmc1 = &sdmmc0; + mmc2 = &sdmmc1; + mmc3 = &sdmmc2; + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + serial4 = &uart4; + serial5 = &uart5; + serial6 = &uart6; + serial7 = &uart7; + serial8 = &uart8; + serial9 = &uart9; + spi0 = &spi0; + spi1 = &spi1; + spi2 = &spi2; + spi3 = &spi3; + }; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a55"; + reg = <0x0 0x0>; + enable-method = "psci"; + clocks = <&scmi_clk 0>; + operating-points-v2 = <&cpu0_opp_table>; + cpu-idle-states = <&CPU_SLEEP>; + #cooling-cells = <2>; + dynamic-power-coefficient = <187>; + }; + + cpu1: cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a55"; + reg = <0x0 0x100>; + enable-method = "psci"; + clocks = <&scmi_clk 0>; + operating-points-v2 = <&cpu0_opp_table>; + cpu-idle-states = <&CPU_SLEEP>; + }; + + cpu2: cpu@200 { + device_type = "cpu"; + compatible = "arm,cortex-a55"; + reg = <0x0 0x200>; + enable-method = "psci"; + clocks = <&scmi_clk 0>; + operating-points-v2 = <&cpu0_opp_table>; + cpu-idle-states = <&CPU_SLEEP>; + }; + + cpu3: cpu@300 { + device_type = "cpu"; + compatible = "arm,cortex-a55"; + reg = <0x0 0x300>; + enable-method = "psci"; + clocks = <&scmi_clk 0>; + operating-points-v2 = <&cpu0_opp_table>; + cpu-idle-states = <&CPU_SLEEP>; + }; + + idle-states { + entry-method = "psci"; + CPU_SLEEP: cpu-sleep { + compatible = "arm,idle-state"; + local-timer-stop; + arm,psci-suspend-param = <0x0010000>; + entry-latency-us = <100>; + exit-latency-us = <120>; + min-residency-us = <1000>; + }; + }; + }; + + cpu0_opp_table: cpu0-opp-table { + compatible = "operating-points-v2"; + opp-shared; + + mbist-vmin = <825000 900000 950000>; + nvmem-cells = <&cpu_leakage>, <&core_pvtm>, <&mbist_vmin>; + nvmem-cell-names = "leakage", "pvtm", "mbist-vmin"; + rockchip,pvtm-voltage-sel = < + 0 82000 0 + 82001 93000 1 + 93001 100000 2 + >; + rockchip,pvtm-freq = <408000>; + rockchip,pvtm-volt = <900000>; + rockchip,pvtm-ch = <0 5>; + rockchip,pvtm-sample-time = <1000>; + rockchip,pvtm-number = <10>; + rockchip,pvtm-error = <1000>; + rockchip,pvtm-ref-temp = <40>; + rockchip,pvtm-temp-prop = <26 26>; + rockchip,thermal-zone = "soc-thermal"; + rockchip,temp-hysteresis = <5000>; + rockchip,low-temp = <0>; + rockchip,low-temp-adjust-volt = < + /* MHz MHz uV */ + 0 1608 75000 + >; + + opp-408000000 { + opp-hz = /bits/ 64 <408000000>; + opp-microvolt = <825000 825000 1150000>; + clock-latency-ns = <40000>; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <825000 825000 1150000>; + clock-latency-ns = <40000>; + }; + opp-816000000 { + opp-hz = /bits/ 64 <816000000>; + opp-microvolt = <825000 825000 1150000>; + clock-latency-ns = <40000>; + opp-suspend; + }; + opp-1104000000 { + opp-hz = /bits/ 64 <1104000000>; + opp-microvolt = <825000 825000 1150000>; + clock-latency-ns = <40000>; + }; + opp-1416000000 { + opp-hz = /bits/ 64 <1416000000>; + opp-microvolt = <925000 925000 1150000>; + clock-latency-ns = <40000>; + }; + opp-1608000000 { + opp-hz = /bits/ 64 <1608000000>; + opp-microvolt = <1000000 1000000 1150000>; + clock-latency-ns = <40000>; + }; + opp-1800000000 { + opp-hz = /bits/ 64 <1800000000>; + opp-microvolt = <1050000 1050000 1150000>; + clock-latency-ns = <40000>; + }; + opp-1992000000 { + opp-hz = /bits/ 64 <1992000000>; + opp-microvolt = <1150000 1150000 1150000>; + clock-latency-ns = <40000>; + }; + }; + + arm-pmu { + compatible = "arm,cortex-a55-pmu", "arm,armv8-pmuv3"; + interrupts = , + , + , + ; + interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>; + }; + + cpuinfo { + compatible = "rockchip,cpuinfo"; + nvmem-cells = <&otp_id>, <&otp_cpu_version>, <&cpu_code>; + nvmem-cell-names = "id", "cpu-version", "cpu-code"; + }; + + display_subsystem: display-subsystem { + compatible = "rockchip,display-subsystem"; + memory-region = <&drm_logo>, <&drm_cubic_lut>; + memory-region-names = "drm-logo", "drm-cubic-lut"; + ports = <&vop_out>; + devfreq = <&dmc>; + + route { + route_dsi0: route-dsi0 { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vp0_out_dsi0>; + }; + route_dsi1: route-dsi1 { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vp0_out_dsi1>; + }; + route_edp: route-edp { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vp0_out_edp>; + }; + route_hdmi: route-hdmi { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vp1_out_hdmi>; + }; + route_lvds: route-lvds { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vp1_out_lvds>; + }; + route_rgb: route-rgb { + status = "disabled"; + logo,uboot = "logo.bmp"; + logo,kernel = "logo_kernel.bmp"; + logo,mode = "center"; + charge_logo,mode = "center"; + connect = <&vp2_out_rgb>; + }; + }; + }; + + firmware { + optee: optee { + compatible = "linaro,optee-tz"; + method = "smc"; + }; + + scmi: scmi { + compatible = "arm,scmi-smc"; + shmem = <&scmi_shmem>; + arm,smc-id = <0x82000010>; + #address-cells = <1>; + #size-cells = <0>; + + scmi_clk: protocol@14 { + reg = <0x14>; + #clock-cells = <1>; + + rockchip,clk-init = <1416000000>; + }; + }; + + sdei: sdei { + compatible = "arm,sdei-1.0"; + method = "smc"; + }; + }; + + mpp_srv: mpp-srv { + compatible = "rockchip,mpp-service"; + rockchip,taskqueue-count = <6>; + rockchip,resetgroup-count = <6>; + status = "disabled"; + }; + + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + }; + + reserved_memory: reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + drm_logo: drm-logo@00000000 { + compatible = "rockchip,drm-logo"; + reg = <0x0 0x0 0x0 0x0>; + }; + + drm_cubic_lut: drm-cubic-lut@00000000 { + compatible = "rockchip,drm-cubic-lut"; + reg = <0x0 0x0 0x0 0x0>; + }; + }; + + rockchip_suspend: rockchip-suspend { + compatible = "rockchip,pm-rk3568"; + status = "disabled"; + rockchip,sleep-debug-en = <1>; + rockchip,sleep-mode-config = < + (0 + | RKPM_SLP_ARMOFF_LOGOFF + | RKPM_SLP_CENTER_OFF + | RKPM_SLP_HW_PLLS_OFF + | RKPM_SLP_PMUALIVE_32K + | RKPM_SLP_OSC_DIS + | RKPM_SLP_PMIC_LP + | RKPM_SLP_32K_PVTM + ) + >; + rockchip,wakeup-config = < + (0 + | RKPM_GPIO_WKUP_EN + ) + >; + }; + + rockchip_system_monitor: rockchip-system-monitor { + compatible = "rockchip,system-monitor"; + + rockchip,thermal-zone = "soc-thermal"; + }; + + thermal_zones: thermal-zones { + soc_thermal: soc-thermal { + polling-delay-passive = <20>; /* milliseconds */ + polling-delay = <1000>; /* milliseconds */ + sustainable-power = <905>; /* milliwatts */ + + thermal-sensors = <&tsadc 0>; + trips { + threshold: trip-point-0 { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + target: trip-point-1 { + temperature = <85000>; + hysteresis = <2000>; + type = "passive"; + }; + soc_crit: soc-crit { + /* millicelsius */ + temperature = <115000>; + /* millicelsius */ + hysteresis = <2000>; + type = "critical"; + }; + }; + cooling-maps { + map0 { + trip = <&target>; + cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + map1 { + trip = <&target>; + cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + }; + }; + + gpu_thermal: gpu-thermal { + polling-delay-passive = <20>; /* milliseconds */ + polling-delay = <1000>; /* milliseconds */ + + thermal-sensors = <&tsadc 1>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , + , + , + ; + arm,no-tick-in-suspend; + }; + + gmac0_clkin: external-gmac0-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "gmac0_clkin"; + #clock-cells = <0>; + }; + + gmac1_clkin: external-gmac1-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "gmac1_clkin"; + #clock-cells = <0>; + }; + + gmac0_xpcsclk: xpcs-gmac0-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clk_gmac0_xpcs_mii"; + #clock-cells = <0>; + }; + + gmac1_xpcsclk: xpcs-gmac1-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clk_gmac1_xpcs_mii"; + #clock-cells = <0>; + }; + + i2s1_mclkin_rx: i2s1-mclkin-rx { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <12288000>; + clock-output-names = "i2s1_mclkin_rx"; + }; + + i2s1_mclkin_tx: i2s1-mclkin-tx { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <12288000>; + clock-output-names = "i2s1_mclkin_tx"; + }; + + i2s2_mclkin: i2s2-mclkin { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <12288000>; + clock-output-names = "i2s2_mclkin"; + }; + + i2s3_mclkin: i2s3-mclkin { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <12288000>; + clock-output-names = "i2s3_mclkin"; + }; + + mpll: mpll { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <800000000>; + clock-output-names = "mpll"; + }; + + xin24m: xin24m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "xin24m"; + }; + + xin32k: xin32k { + compatible = "fixed-clock"; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + #clock-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&clk32k_out0>; + }; + + scmi_shmem: scmi-shmem@10f000 { + compatible = "arm,scmi-shmem"; + reg = <0x0 0x0010f000 0x0 0x100>; + }; + + sata0: sata@fc000000 { + compatible = "snps,dwc-ahci"; + reg = <0 0xfc000000 0 0x1000>; + clocks = <&cru ACLK_SATA0>, <&cru CLK_SATA0_PMALIVE>, + <&cru CLK_SATA0_RXOOB>; + clock-names = "sata", "pmalive", "rxoob"; + interrupts = ; + interrupt-names = "hostc"; + phys = <&combphy0_us PHY_TYPE_SATA>; + phy-names = "sata-phy"; + ports-implemented = <0x1>; + power-domains = <&power RK3568_PD_PIPE>; + status = "disabled"; + }; + + sata1: sata@fc400000 { + compatible = "snps,dwc-ahci"; + reg = <0 0xfc400000 0 0x1000>; + clocks = <&cru ACLK_SATA1>, <&cru CLK_SATA1_PMALIVE>, + <&cru CLK_SATA1_RXOOB>; + clock-names = "sata", "pmalive", "rxoob"; + interrupts = ; + interrupt-names = "hostc"; + phys = <&combphy1_usq PHY_TYPE_SATA>; + phy-names = "sata-phy"; + ports-implemented = <0x1>; + power-domains = <&power RK3568_PD_PIPE>; + status = "disabled"; + }; + + sata2: sata@fc800000 { + compatible = "snps,dwc-ahci"; + reg = <0 0xfc800000 0 0x1000>; + clocks = <&cru ACLK_SATA2>, <&cru CLK_SATA2_PMALIVE>, + <&cru CLK_SATA2_RXOOB>; + clock-names = "sata", "pmalive", "rxoob"; + interrupts = ; + interrupt-names = "hostc"; + phys = <&combphy2_psq PHY_TYPE_SATA>; + phy-names = "sata-phy"; + ports-implemented = <0x1>; + power-domains = <&power RK3568_PD_PIPE>; + status = "disabled"; + }; + + usbdrd30: usbdrd { + compatible = "rockchip,rk3568-dwc3", "rockchip,rk3399-dwc3"; + clocks = <&cru CLK_USB3OTG0_REF>, <&cru CLK_USB3OTG0_SUSPEND>, + <&cru ACLK_USB3OTG0>, <&cru PCLK_PIPE>; + clock-names = "ref_clk", "suspend_clk", + "bus_clk", "pipe_clk"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + status = "disabled"; + + usbdrd_dwc3: dwc3@fcc00000 { + compatible = "snps,dwc3"; + reg = <0x0 0xfcc00000 0x0 0x400000>; + interrupts = ; + dr_mode = "otg"; + phys = <&u2phy0_otg>, <&combphy0_us PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + phy_type = "utmi_wide"; + power-domains = <&power RK3568_PD_PIPE>; + resets = <&cru SRST_USB3OTG0>; + reset-names = "usb3-otg"; + snps,dis_enblslpm_quirk; + snps,dis-u2-freeclk-exists-quirk; + snps,dis-del-phy-power-chg-quirk; + snps,dis-tx-ipgap-linecheck-quirk; + snps,xhci-trb-ent-quirk; + status = "disabled"; + }; + }; + + usbhost30: usbhost { + compatible = "rockchip,rk3568-dwc3", "rockchip,rk3399-dwc3"; + clocks = <&cru CLK_USB3OTG1_REF>, <&cru CLK_USB3OTG1_SUSPEND>, + <&cru ACLK_USB3OTG1>, <&cru PCLK_PIPE>; + clock-names = "ref_clk", "suspend_clk", + "bus_clk", "pipe_clk"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + status = "disabled"; + + usbhost_dwc3: dwc3@fd000000 { + compatible = "snps,dwc3"; + reg = <0x0 0xfd000000 0x0 0x400000>; + interrupts = ; + dr_mode = "host"; + phys = <&u2phy0_host>, <&combphy1_usq PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + phy_type = "utmi_wide"; + power-domains = <&power RK3568_PD_PIPE>; + resets = <&cru SRST_USB3OTG1>; + reset-names = "usb3-host"; + snps,dis_enblslpm_quirk; + snps,dis-u2-freeclk-exists-quirk; + snps,dis-del-phy-power-chg-quirk; + snps,dis-tx-ipgap-linecheck-quirk; + snps,xhci-trb-ent-quirk; + status = "disabled"; + }; + }; + + gic: interrupt-controller@fd400000 { + compatible = "arm,gic-v3"; + #interrupt-cells = <3>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + interrupt-controller; + + reg = <0x0 0xfd400000 0 0x10000>, /* GICD */ + <0x0 0xfd460000 0 0xc0000>; /* GICR */ + interrupts = ; + its: interrupt-controller@fd440000 { + compatible = "arm,gic-v3-its"; + msi-controller; + #msi-cells = <1>; + reg = <0x0 0xfd440000 0x0 0x20000>; + }; + }; + + usb_host0_ehci: usb@fd800000 { + compatible = "generic-ehci"; + reg = <0x0 0xfd800000 0x0 0x40000>; + interrupts = ; + clocks = <&cru HCLK_USB2HOST0>, <&cru HCLK_USB2HOST0_ARB>, + <&cru PCLK_USB>, <&usb2phy1>; + clock-names = "usbhost", "arbiter", "pclk", "utmi"; + phys = <&u2phy1_otg>; + phy-names = "usb2-phy"; + status = "disabled"; + }; + + usb_host0_ohci: usb@fd840000 { + compatible = "generic-ohci"; + reg = <0x0 0xfd840000 0x0 0x40000>; + interrupts = ; + clocks = <&cru HCLK_USB2HOST0>, <&cru HCLK_USB2HOST0_ARB>, + <&cru PCLK_USB>, <&usb2phy1>; + clock-names = "usbhost", "arbiter", "pclk", "utmi"; + phys = <&u2phy1_otg>; + phy-names = "usb2-phy"; + status = "disabled"; + }; + + usb_host1_ehci: usb@fd880000 { + compatible = "generic-ehci"; + reg = <0x0 0xfd880000 0x0 0x40000>; + interrupts = ; + clocks = <&cru HCLK_USB2HOST1>, <&cru HCLK_USB2HOST1_ARB>, + <&cru PCLK_USB>, <&usb2phy1>; + clock-names = "usbhost", "arbiter", "pclk", "utmi"; + phys = <&u2phy1_host>; + phy-names = "usb2-phy"; + status = "disabled"; + }; + + usb_host1_ohci: usb@fd8c0000 { + compatible = "generic-ohci"; + reg = <0x0 0xfd8c0000 0x0 0x40000>; + interrupts = ; + clocks = <&cru HCLK_USB2HOST1>, <&cru HCLK_USB2HOST1_ARB>, + <&cru PCLK_USB>, <&usb2phy1>; + clock-names = "usbhost", "arbiter", "pclk", "utmi"; + phys = <&u2phy1_host>; + phy-names = "usb2-phy"; + status = "disabled"; + }; + + xpcs: syscon@fda00000 { + compatible = "rockchip,rk3568-xpcs", "syscon"; + reg = <0x0 0xfda00000 0x0 0x200000>; + status = "disabled"; + }; + + pmugrf: syscon@fdc20000 { + compatible = "rockchip,rk3568-pmugrf", "syscon", "simple-mfd"; + reg = <0x0 0xfdc20000 0x0 0x10000>; + + pmu_io_domains: io-domains { + compatible = "rockchip,rk3568-pmu-io-voltage-domain"; + status = "disabled"; + }; + + reboot_mode: reboot-mode { + compatible = "syscon-reboot-mode"; + offset = <0x200>; + mode-bootloader = ; + mode-charge = ; + mode-fastboot = ; + mode-loader = ; + mode-normal = ; + mode-recovery = ; + mode-ums = ; + mode-panic = ; + mode-watchdog = ; + }; + }; + + pipegrf: syscon@fdc50000 { + compatible = "rockchip,rk3568-pipegrf", "syscon"; + reg = <0x0 0xfdc50000 0x0 0x1000>; + }; + + grf: syscon@fdc60000 { + compatible = "rockchip,rk3568-grf", "syscon", "simple-mfd"; + reg = <0x0 0xfdc60000 0x0 0x10000>; + + io_domains: io-domains { + compatible = "rockchip,rk3568-io-voltage-domain"; + status = "disabled"; + }; + + lvds: lvds { + compatible = "rockchip,rk3568-lvds"; + phys = <&video_phy0>; + phy-names = "phy"; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + lvds_in_vp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&vp1_out_lvds>; + status = "disabled"; + }; + + lvds_in_vp2: endpoint@2 { + reg = <2>; + remote-endpoint = <&vp2_out_lvds>; + status = "disabled"; + }; + }; + }; + }; + + rgb: rgb { + compatible = "rockchip,rk3568-rgb"; + pinctrl-names = "default"; + pinctrl-0 = <&lcdc_ctl>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + rgb_in_vp2: endpoint@2 { + reg = <2>; + remote-endpoint = <&vp2_out_rgb>; + status = "disabled"; + }; + }; + }; + }; + + }; + + pipe_phy_grf0: syscon@fdc70000 { + compatible = "rockchip,pipe-phy-grf", "syscon"; + reg = <0x0 0xfdc70000 0x0 0x1000>; + }; + + pipe_phy_grf1: syscon@fdc80000 { + compatible = "rockchip,pipe-phy-grf", "syscon"; + reg = <0x0 0xfdc80000 0x0 0x1000>; + }; + + pipe_phy_grf2: syscon@fdc90000 { + compatible = "rockchip,pipe-phy-grf", "syscon"; + reg = <0x0 0xfdc90000 0x0 0x1000>; + }; + + usb2phy0_grf: syscon@fdca0000 { + compatible = "rockchip,rk3568-usb2phy-grf", "syscon"; + reg = <0x0 0xfdca0000 0x0 0x8000>; + }; + + usb2phy1_grf: syscon@fdca8000 { + compatible = "rockchip,rk3568-usb2phy-grf", "syscon"; + reg = <0x0 0xfdca8000 0x0 0x8000>; + }; + + edp_phy: edp-phy@fdcb0000 { + compatible = "rockchip,rk3568-edp-phy"; + reg = <0x0 0xfdcb0000 0x0 0x8000>; + clocks = <&pmucru XIN_OSC0_EDPPHY_G>, <&cru PCLK_EDPPHY_GRF>; + clock-names = "refclk", "pclk"; + resets = <&cru SRST_P_EDPPHY_GRF>; + reset-names = "apb"; + #phy-cells = <0>; + status = "disabled"; + }; + + pcie30_phy_grf: syscon@fdcb8000 { + compatible = "rockchip,pcie30-phy-grf", "syscon"; + reg = <0x0 0xfdcb8000 0x0 0x10000>; + }; + + sram: sram@fdcc0000 { + compatible = "mmio-sram"; + reg = <0x0 0xfdcc0000 0x0 0xb000>; + + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x0 0xfdcc0000 0xb000>; + + /* start address and size should be 4k algin */ + rkvdec_sram: rkvdec-sram@0 { + reg = <0x0 0xb000>; + }; + }; + + pmucru: clock-controller@fdd00000 { + compatible = "rockchip,rk3568-pmucru"; + reg = <0x0 0xfdd00000 0x0 0x1000>; + rockchip,grf = <&grf>; + rockchip,pmugrf = <&pmugrf>; + #clock-cells = <1>; + #reset-cells = <1>; + + assigned-clocks = <&pmucru SCLK_32K_IOE>; + assigned-clock-parents = <&pmucru CLK_RTC_32K>; + }; + + cru: clock-controller@fdd20000 { + compatible = "rockchip,rk3568-cru"; + reg = <0x0 0xfdd20000 0x0 0x1000>; + rockchip,grf = <&grf>; + #clock-cells = <1>; + #reset-cells = <1>; + + assigned-clocks = + <&pmucru CLK_RTC_32K>, <&cru ACLK_RKVDEC_PRE>, + <&cru CLK_RKVDEC_CORE>, <&pmucru PLL_PPLL>, + <&pmucru PCLK_PMU>, <&cru PLL_CPLL>, + <&cru CPLL_500M>, <&cru CPLL_333M>, + <&cru CPLL_250M>, <&cru CPLL_125M>, + <&cru CPLL_100M>, <&cru CPLL_62P5M>, + <&cru CPLL_50M>, <&cru CPLL_25M>, + <&cru PLL_GPLL>, + <&cru ACLK_BUS>, <&cru PCLK_BUS>, + <&cru ACLK_TOP_HIGH>, <&cru ACLK_TOP_LOW>, + <&cru HCLK_TOP>, <&cru PCLK_TOP>, + <&cru ACLK_PERIMID>, <&cru HCLK_PERIMID>, + <&cru PLL_NPLL>, <&cru ACLK_PIPE>, + <&cru PCLK_PIPE>, <&cru CLK_I2S0_8CH_TX_SRC>, + <&cru CLK_I2S0_8CH_RX_SRC>, <&cru CLK_I2S1_8CH_TX_SRC>, + <&cru CLK_I2S1_8CH_RX_SRC>, <&cru CLK_I2S2_2CH_SRC>, + <&cru CLK_I2S2_2CH_SRC>, <&cru CLK_I2S3_2CH_RX_SRC>, + <&cru CLK_I2S3_2CH_TX_SRC>, <&cru MCLK_SPDIF_8CH_SRC>, + <&cru ACLK_VOP>; + assigned-clock-rates = + <32768>, <300000000>, + <300000000>, <200000000>, + <100000000>, <1000000000>, + <500000000>, <333000000>, + <250000000>, <125000000>, + <100000000>, <62500000>, + <50000000>, <25000000>, + <1188000000>, + <150000000>, <100000000>, + <500000000>, <400000000>, + <150000000>, <100000000>, + <300000000>, <150000000>, + <1200000000>, <400000000>, + <100000000>, <1188000000>, + <1188000000>, <1188000000>, + <1188000000>, <1188000000>, + <1188000000>, <1188000000>, + <1188000000>, <1188000000>, + <500000000>; + assigned-clock-parents = + <&pmucru CLK_RTC32K_FRAC>, <&cru PLL_GPLL>, + <&cru PLL_GPLL>; + }; + + i2c0: i2c@fdd40000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xfdd40000 0x0 0x1000>; + clocks = <&pmucru CLK_I2C0>, <&pmucru PCLK_I2C0>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + uart0: serial@fdd50000 { + compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; + reg = <0x0 0xfdd50000 0x0 0x100>; + interrupts = ; + clocks = <&pmucru SCLK_UART0>, <&pmucru PCLK_UART0>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac0 0>, <&dmac0 1>; + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer>; + status = "disabled"; + }; + + pwm0: pwm@fdd70000 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfdd70000 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm0m0_pins>; + clocks = <&pmucru CLK_PWM0>, <&pmucru PCLK_PWM0>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm1: pwm@fdd70010 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfdd70010 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm1m0_pins>; + clocks = <&pmucru CLK_PWM0>, <&pmucru PCLK_PWM0>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm2: pwm@fdd70020 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfdd70020 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm2m0_pins>; + clocks = <&pmucru CLK_PWM0>, <&pmucru PCLK_PWM0>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm3: pwm@fdd70030 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfdd70030 0x0 0x10>; + interrupts = , + ; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm3_pins>; + clocks = <&pmucru CLK_PWM0>, <&pmucru PCLK_PWM0>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pmu: power-management@fdd90000 { + compatible = "rockchip,rk3568-pmu", "syscon", "simple-mfd"; + reg = <0x0 0xfdd90000 0x0 0x1000>; + + power: power-controller { + compatible = "rockchip,rk3568-power-controller"; + #power-domain-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + /* These power domains are grouped by VD_NPU */ + /*pd_npu@RK3568_PD_NPU { + reg = ; + clocks = <&cru ACLK_NPU_PRE>, + <&cru HCLK_NPU_PRE>, + <&cru PCLK_NPU_PRE>; + pm_qos = <&qos_npu>; + };*/ + /* These power domains are grouped by VD_GPU */ + pd_gpu@RK3568_PD_GPU { + reg = ; + clocks = <&cru ACLK_GPU_PRE>, + <&cru PCLK_GPU_PRE>; + pm_qos = <&qos_gpu>; + }; + /* These power domains are grouped by VD_LOGIC */ + pd_vi@RK3568_PD_VI { + reg = ; + clocks = <&cru HCLK_VI>, + <&cru PCLK_VI>; + pm_qos = <&qos_isp>, + <&qos_vicap0>, + <&qos_vicap1>; + }; + pd_vo@RK3568_PD_VO { + reg = ; + clocks = <&cru HCLK_VO>, + <&cru PCLK_VO>, + <&cru ACLK_VOP_PRE>; + pm_qos = <&qos_hdcp>, + <&qos_vop_m0>, + <&qos_vop_m1>; + }; + pd_rga@RK3568_PD_RGA { + reg = ; + clocks = <&cru HCLK_RGA_PRE>, + <&cru PCLK_RGA_PRE>; + pm_qos = <&qos_ebc>, + <&qos_iep>, + <&qos_jpeg_dec>, + <&qos_jpeg_enc>, + <&qos_rga_rd>, + <&qos_rga_wr>; + }; + pd_vpu@RK3568_PD_VPU { + reg = ; + clocks = <&cru HCLK_VPU_PRE>; + pm_qos = <&qos_vpu>; + }; + pd_rkvdec@RK3568_PD_RKVDEC { + clocks = <&cru HCLK_RKVDEC_PRE>; + reg = ; + pm_qos = <&qos_rkvdec>; + }; + pd_rkvenc@RK3568_PD_RKVENC { + reg = ; + clocks = <&cru HCLK_RKVENC_PRE>; + pm_qos = <&qos_rkvenc_rd_m0>, + <&qos_rkvenc_rd_m1>, + <&qos_rkvenc_wr_m0>; + }; + pd_pipe@RK3568_PD_PIPE { + reg = ; + clocks = <&cru PCLK_PIPE>; + pm_qos = <&qos_pcie2x1>, + <&qos_pcie3x1>, + <&qos_pcie3x2>, + <&qos_sata0>, + <&qos_sata1>, + <&qos_sata2>, + <&qos_usb3_0>, + <&qos_usb3_1>; + }; + }; + }; + + pvtm@fde00000 { + compatible = "rockchip,rk3568-core-pvtm"; + reg = <0x0 0xfde00000 0x0 0x100>; + #address-cells = <1>; + #size-cells = <0>; + pvtm@0 { + reg = <0>; + clocks = <&cru CLK_CORE_PVTM>, <&cru PCLK_CORE_PVTM>; + clock-names = "clk", "pclk"; + resets = <&cru SRST_CORE_PVTM>, <&cru SRST_P_CORE_PVTM>; + reset-names = "rts", "rst-p"; + thermal-zone = "soc-thermal"; + }; + }; + + rknpu: npu@fde40000 { + compatible = "rockchip,rk3568-rknpu", "rockchip,rknpu"; + reg = <0x0 0xfde40000 0x0 0x10000>; + interrupts = ; + clocks = <&scmi_clk 2>, <&cru CLK_NPU>, <&cru ACLK_NPU>, <&cru HCLK_NPU>; + clock-names = "scmi_clk", "clk", "aclk", "hclk"; + assigned-clocks = <&cru CLK_NPU>; + assigned-clock-rates = <600000000>; + resets = <&cru SRST_A_NPU>, <&cru SRST_H_NPU>; + reset-names = "srst_a", "srst_h"; + power-domains = <&power RK3568_PD_NPU>; + operating-points-v2 = <&npu_opp_table>; + iommus = <&rknpu_mmu>; + status = "disabled"; + }; + + npu_opp_table: npu-opp-table { + compatible = "operating-points-v2"; + + mbist-vmin = <825000 900000 950000>; + nvmem-cells = <&npu_leakage>, <&core_pvtm>, <&mbist_vmin>; + nvmem-cell-names = "leakage", "pvtm", "mbist-vmin"; + rockchip,temp-hysteresis = <5000>; + rockchip,low-temp = <0>; + rockchip,low-temp-adjust-volt = < + /* MHz MHz uV */ + 0 700 50000 + >; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <825000 825000 1000000>; + }; + opp-300000000 { + opp-hz = /bits/ 64 <297000000>; + opp-microvolt = <825000 825000 1000000>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <825000 825000 1000000>; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <825000 825000 1000000>; + }; + opp-700000000 { + opp-hz = /bits/ 64 <700000000>; + opp-microvolt = <850000 850000 1000000>; + }; + opp-800000000 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <875000 875000 1000000>; + }; + opp-900000000 { + opp-hz = /bits/ 64 <900000000>; + opp-microvolt = <925000 925000 1000000>; + }; + opp-1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <1000000 1000000 1000000>; + status = "disabled"; + }; + }; + + bus_npu: bus-npu { + compatible = "rockchip,rk3568-bus"; + rockchip,busfreq-policy = "clkfreq"; + clocks = <&scmi_clk 2>; + clock-names = "bus"; + operating-points-v2 = <&bus_npu_opp_table>; + status = "disabled"; + }; + + bus_npu_opp_table: bus-npu-opp-table { + compatible = "operating-points-v2"; + opp-shared; + + nvmem-cells = <&core_pvtm>; + nvmem-cell-names = "pvtm"; + rockchip,pvtm-voltage-sel = < + 0 82000 0 + 82001 93000 1 + 93001 100000 2 + >; + rockchip,pvtm-ch = <0 5>; + + opp-1000000000 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <950000>; + opp-microvolt-L0 = <950000>; + opp-microvolt-L1 = <925000>; + opp-microvolt-L2 = <0>; + }; + opp-900000000 { + opp-hz = /bits/ 64 <900000000>; + opp-microvolt = <0>; + }; + }; + + rknpu_mmu: iommu@fde4b000 { + compatible = "rockchip,iommu-v2"; + reg = <0x0 0xfde4b000 0x0 0x40>; + interrupts = ; + interrupt-names = "rknpu_mmu"; + clocks = <&cru ACLK_NPU>, <&cru HCLK_NPU>; + clock-names = "aclk", "iface"; + power-domains = <&power RK3568_PD_NPU>; + #iommu-cells = <0>; + status = "disabled"; + }; + + gpu: gpu@fde60000 { + compatible = "arm,mali-bifrost"; + reg = <0x0 0xfde60000 0x0 0x4000>; + + interrupts = , + , + ; + interrupt-names = "GPU", "MMU", "JOB"; + + upthreshold = <40>; + downdifferential = <10>; + + clocks = <&scmi_clk 1>, <&cru CLK_GPU>; + clock-names = "clk_mali", "clk_gpu"; + power-domains = <&power RK3568_PD_GPU>; + #cooling-cells = <2>; + operating-points-v2 = <&gpu_opp_table>; + + status = "disabled"; + gpu_power_model: power-model { + compatible = "simple-power-model"; + leakage-range= <5 15>; + ls = <(-24002) 22823 0>; + static-coefficient = <100000>; + dynamic-coefficient = <953>; + ts = <(-108890) 63610 (-1355) 20>; + thermal-zone = "gpu-thermal"; + }; + }; + + gpu_opp_table: opp-table2 { + compatible = "operating-points-v2"; + + mbist-vmin = <825000 900000 950000>; + nvmem-cells = <&gpu_leakage>, <&core_pvtm>, <&mbist_vmin>; + nvmem-cell-names = "leakage", "pvtm", "mbist-vmin"; + + opp-200000000 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <825000>; + }; + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <825000>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <825000>; + }; + opp-600000000 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <825000>; + }; + opp-700000000 { + opp-hz = /bits/ 64 <700000000>; + opp-microvolt = <900000>; + }; + opp-800000000 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <950000>; + }; + }; + + pvtm@fde80000 { + compatible = "rockchip,rk3568-gpu-pvtm"; + reg = <0x0 0xfde80000 0x0 0x100>; + #address-cells = <1>; + #size-cells = <0>; + pvtm@1 { + reg = <1>; + clocks = <&cru CLK_GPU_PVTM>, <&cru PCLK_GPU_PVTM>; + clock-names = "clk", "pclk"; + resets = <&cru SRST_GPU_PVTM>, <&cru SRST_P_GPU_PVTM>; + reset-names = "rts", "rst-p"; + thermal-zone = "gpu-thermal"; + }; + }; + + pvtm@fde90000 { + compatible = "rockchip,rk3568-npu-pvtm"; + reg = <0x0 0xfde90000 0x0 0x100>; + #address-cells = <1>; + #size-cells = <0>; + pvtm@2 { + reg = <2>; + clocks = <&cru CLK_NPU_PVTM>, <&cru PCLK_NPU_PVTM>, + <&cru HCLK_NPU_PRE>; + clock-names = "clk", "pclk", "hclk"; + resets = <&cru SRST_NPU_PVTM>, <&cru SRST_P_NPU_PVTM>; + reset-names = "rts", "rst-p"; + thermal-zone = "soc-thermal"; + }; + }; + + vdpu: vdpu@fdea0400 { + compatible = "rockchip,vpu-decoder-v2"; + reg = <0x0 0xfdea0400 0x0 0x400>; + interrupts = ; + interrupt-names = "irq_dec"; + clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; + clock-names = "aclk_vcodec", "hclk_vcodec"; + resets = <&cru SRST_A_VPU>, <&cru SRST_H_VPU>; + reset-names = "video_a", "video_h"; + iommus = <&vdpu_mmu>; + power-domains = <&power RK3568_PD_VPU>; + rockchip,srv = <&mpp_srv>; + rockchip,taskqueue-node = <0>; + rockchip,resetgroup-node = <0>; + status = "disabled"; + }; + + vdpu_mmu: iommu@fdea0800 { + compatible = "rockchip,iommu-v2"; + reg = <0x0 0xfdea0800 0x0 0x40>; + interrupts = ; + interrupt-names = "vdpu_mmu"; + clock-names = "aclk", "iface"; + clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; + power-domains = <&power RK3568_PD_VPU>; + #iommu-cells = <0>; + status = "disabled"; + }; + + rk_rga: rk_rga@fdeb0000 { + compatible = "rockchip,rga2"; + reg = <0x0 0xfdeb0000 0x0 0x1000>; + interrupts = ; + clocks = <&cru ACLK_RGA>, <&cru HCLK_RGA>, <&cru CLK_RGA_CORE>; + clock-names = "aclk_rga", "hclk_rga", "clk_rga"; + power-domains = <&power RK3568_PD_RGA>; + status = "disabled"; + }; + + ebc: ebc@fdec0000 { + compatible = "rockchip,rk3568-ebc-tcon"; + reg = <0x0 0xfdec0000 0x0 0x5000>; + interrupts = ; + clocks = <&cru HCLK_EBC>, <&cru DCLK_EBC>; + clock-names = "hclk", "dclk"; + power-domains = <&power RK3568_PD_RGA>; + rockchip,grf = <&grf>; + pinctrl-names = "default"; + pinctrl-0 = <&ebc_pins>; + status = "disabled"; + }; + + jpegd: jpegd@fded0000 { + compatible = "rockchip,rkv-jpeg-decoder-v1"; + reg = <0x0 0xfded0000 0x0 0x400>; + interrupts = ; + clocks = <&cru ACLK_JDEC>, <&cru HCLK_JDEC>; + clock-names = "aclk_vcodec", "hclk_vcodec"; + rockchip,disable-auto-freq; + resets = <&cru SRST_A_JDEC>, <&cru SRST_H_JDEC>; + reset-names = "video_a", "video_h"; + iommus = <&jpegd_mmu>; + rockchip,srv = <&mpp_srv>; + rockchip,taskqueue-node = <1>; + rockchip,resetgroup-node = <1>; + power-domains = <&power RK3568_PD_RGA>; + status = "disabled"; + }; + + jpegd_mmu: iommu@fded0480 { + compatible = "rockchip,iommu-v2"; + reg = <0x0 0xfded0480 0x0 0x40>; + interrupts = ; + interrupt-names = "jpegd_mmu"; + clock-names = "aclk", "iface"; + clocks = <&cru ACLK_JDEC>, <&cru HCLK_JDEC>; + power-domains = <&power RK3568_PD_RGA>; + #iommu-cells = <0>; + status = "disabled"; + }; + + vepu: vepu@fdee0000 { + compatible = "rockchip,vpu-encoder-v2"; + reg = <0x0 0xfdee0000 0x0 0x400>; + interrupts = ; + clocks = <&cru ACLK_JENC>, <&cru HCLK_JENC>; + clock-names = "aclk_vcodec", "hclk_vcodec"; + rockchip,disable-auto-freq; + resets = <&cru SRST_A_JENC>, <&cru SRST_H_JENC>; + reset-names = "video_a", "video_h"; + iommus = <&vepu_mmu>; + rockchip,srv = <&mpp_srv>; + rockchip,taskqueue-node = <2>; + rockchip,resetgroup-node = <2>; + power-domains = <&power RK3568_PD_RGA>; + status = "disabled"; + }; + + vepu_mmu: iommu@fdee0800 { + compatible = "rockchip,iommu-v2"; + reg = <0x0 0xfdee0800 0x0 0x40>; + interrupts = ; + interrupt-names = "vepu_mmu"; + clock-names = "aclk", "iface"; + clocks = <&cru ACLK_JENC>, <&cru HCLK_JENC>; + power-domains = <&power RK3568_PD_RGA>; + #iommu-cells = <0>; + status = "disabled"; + }; + + iep: iep@fdef0000 { + compatible = "rockchip,iep-v2"; + reg = <0x0 0xfdef0000 0x0 0x500>; + interrupts = ; + clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>, <&cru CLK_IEP_CORE>; + clock-names = "aclk", "hclk", "sclk"; + resets = <&cru SRST_A_IEP>, <&cru SRST_H_IEP>, + <&cru SRST_IEP_CORE>; + reset-names = "rst_a", "rst_h", "rst_s"; + power-domains = <&power RK3568_PD_RGA>; + rockchip,srv = <&mpp_srv>; + rockchip,taskqueue-node = <5>; + rockchip,resetgroup-node = <5>; + iommus = <&iep_mmu>; + status = "disabled"; + }; + + iep_mmu: iommu@fdef0800 { + compatible = "rockchip,iommu-v2"; + reg = <0x0 0xfdef0800 0x0 0x100>; + interrupts = ; + interrupt-names = "iep_mmu"; + clocks = <&cru ACLK_IEP>, <&cru HCLK_IEP>; + clock-names = "aclk", "iface"; + #iommu-cells = <0>; + power-domains = <&power RK3568_PD_RGA>; + //rockchip,disable-device-link-resume; + status = "disabled"; + }; + + eink: eink@fdf00000 { + compatible = "rockchip,rk3568-eink-tcon"; + reg = <0x0 0xfdf00000 0x0 0x74>; + interrupts = ; + clocks = <&cru PCLK_EINK>, <&cru HCLK_EINK>; + clock-names = "pclk", "hclk"; + status = "disabled"; + }; + + rkvenc: rkvenc@fdf40000 { + compatible = "rockchip,rkv-encoder-v1"; + reg = <0x0 0xfdf40000 0x0 0x400>; + interrupts = ; + interrupt-names = "irq_enc"; + clocks = <&cru ACLK_RKVENC>, <&cru HCLK_RKVENC>, + <&cru CLK_RKVENC_CORE>; + clock-names = "aclk_vcodec", "hclk_vcodec", "clk_core"; + rockchip,normal-rates = <297000000>, <0>, <297000000>; + resets = <&cru SRST_A_RKVENC>, <&cru SRST_H_RKVENC>, + <&cru SRST_RKVENC_CORE>; + reset-names = "video_a", "video_h", "video_core"; + assigned-clocks = <&cru ACLK_RKVENC>, <&cru CLK_RKVENC_CORE>; + assigned-clock-rates = <297000000>, <297000000>; + iommus = <&rkvenc_mmu>; + node-name = "rkvenc"; + rockchip,srv = <&mpp_srv>; + rockchip,taskqueue-node = <3>; + rockchip,resetgroup-node = <3>; + power-domains = <&power RK3568_PD_RKVENC>; + operating-points-v2 = <&rkvenc_opp_table>; + status = "disabled"; + }; + + rkvenc_opp_table: rkvenc-opp-table { + compatible = "operating-points-v2"; + + nvmem-cells = <&core_pvtm>; + nvmem-cell-names = "pvtm"; + rockchip,pvtm-voltage-sel = < + 0 82000 0 + 82001 93000 1 + 93001 100000 2 + >; + rockchip,pvtm-ch = <0 5>; + + opp-297000000 { + opp-hz = /bits/ 64 <297000000>; + opp-microvolt = <0>; + }; + opp-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <950000>; + opp-microvolt-L0 = <950000>; + opp-microvolt-L1 = <925000>; + opp-microvolt-L2 = <0>; + }; + }; + + rkvenc_mmu: iommu@fdf40f00 { + compatible = "rockchip,iommu-v2"; + reg = <0x0 0xfdf40f00 0x0 0x40>, <0x0 0xfdf40f40 0x0 0x40>; + interrupts = , + ; + interrupt-names = "rkvenc_mmu0", "rkvenc_mmu1"; + clocks = <&cru ACLK_RKVENC>, <&cru HCLK_RKVENC>; + clock-names = "aclk", "iface"; + rockchip,disable-mmu-reset; + rockchip,enable-cmd-retry; + #iommu-cells = <0>; + power-domains = <&power RK3568_PD_RKVENC>; + status = "disabled"; + }; + + rkvdec: rkvdec@fdf80200 { + compatible = "rockchip,rkv-decoder-rk3568", "rockchip,rkv-decoder-v2"; + reg = <0x0 0xfdf80200 0x0 0x400>; + interrupts = ; + interrupt-names = "irq_dec"; + clocks = <&cru ACLK_RKVDEC>, <&cru HCLK_RKVDEC>, + <&cru CLK_RKVDEC_CA>, <&cru CLK_RKVDEC_CORE>, + <&cru CLK_RKVDEC_HEVC_CA>; + clock-names = "aclk_vcodec", "hclk_vcodec","clk_cabac", + "clk_core", "clk_hevc_cabac"; + rockchip,normal-rates = <297000000>, <0>, <297000000>, + <297000000>, <600000000>; + rockchip,advanced-rates = <396000000>, <0>, <396000000>, + <396000000>, <600000000>; + rockchip,default-max-load = <2088960>; + resets = <&cru SRST_A_RKVDEC>, <&cru SRST_H_RKVDEC>, + <&cru SRST_RKVDEC_CA>, <&cru SRST_RKVDEC_CORE>, + <&cru SRST_RKVDEC_HEVC_CA>; + assigned-clocks = <&cru ACLK_RKVDEC>, <&cru CLK_RKVDEC_CA>, + <&cru CLK_RKVDEC_CORE>, <&cru CLK_RKVDEC_HEVC_CA>; + assigned-clock-rates = <297000000>, <297000000>, <297000000>, <297000000>; + reset-names = "video_a", "video_h", "video_cabac", + "video_core", "video_hevc_cabac"; + power-domains = <&power RK3568_PD_RKVDEC>; + iommus = <&rkvdec_mmu>; + rockchip,srv = <&mpp_srv>; + rockchip,taskqueue-node = <4>; + rockchip,resetgroup-node = <4>; + rockchip,sram = <&rkvdec_sram>; + /* rcb_iova: start and size */ + rockchip,rcb-iova = <0x10000000 65536>; + rockchip,rcb-min-width = <512>; + status = "disabled"; + }; + + rkvdec_mmu: iommu@fdf80800 { + compatible = "rockchip,iommu-v2"; + reg = <0x0 0xfdf80800 0x0 0x40>, <0x0 0xfdf80840 0x0 0x40>; + interrupts = ; + interrupt-names = "rkvdec_mmu"; + clocks = <&cru ACLK_RKVDEC>, <&cru HCLK_RKVDEC>; + clock-names = "aclk", "iface"; + power-domains = <&power RK3568_PD_RKVDEC>; + #iommu-cells = <0>; + status = "disabled"; + }; + + mipi_csi2: mipi-csi2@fdfb0000 { + compatible = "rockchip,rk3568-mipi-csi2"; + reg = <0x0 0xfdfb0000 0x0 0x10000>; + reg-names = "csihost_regs"; + interrupts = , + ; + interrupt-names = "csi-intr1", "csi-intr2"; + clocks = <&cru PCLK_CSI2HOST1>; + clock-names = "pclk_csi2host"; + resets = <&cru SRST_P_CSI2HOST1>; + reset-names = "srst_csihost_p"; + status = "disabled"; + }; + + rkcif: rkcif@fdfe0000 { + compatible = "rockchip,rk3568-cif"; + reg = <0x0 0xfdfe0000 0x0 0x8000>; + reg-names = "cif_regs"; + interrupts = ; + interrupt-names = "cif-intr"; + + clocks = <&cru ACLK_VICAP>, <&cru HCLK_VICAP>, + <&cru DCLK_VICAP>, <&cru ICLK_VICAP_G>; + clock-names = "aclk_cif", "hclk_cif", + "dclk_cif", "iclk_cif_g"; + resets = <&cru SRST_A_VICAP>, <&cru SRST_H_VICAP>, + <&cru SRST_D_VICAP>, <&cru SRST_P_VICAP>, + <&cru SRST_I_VICAP>; + reset-names = "rst_cif_a", "rst_cif_h", + "rst_cif_d", "rst_cif_p", + "rst_cif_i"; + assigned-clocks = <&cru DCLK_VICAP>; + assigned-clock-rates = <300000000>; + power-domains = <&power RK3568_PD_VI>; + rockchip,grf = <&grf>; + iommus = <&rkcif_mmu>; + status = "disabled"; + }; + + rkcif_mmu: iommu@fdfe0800 { + compatible = "rockchip,iommu-v2"; + reg = <0x0 0xfdfe0800 0x0 0x100>; + interrupts = ; + interrupt-names = "cif_mmu"; + clocks = <&cru ACLK_VICAP>, <&cru HCLK_VICAP>; + clock-names = "aclk", "iface"; + power-domains = <&power RK3568_PD_VI>; + rockchip,disable-mmu-reset; + #iommu-cells = <0>; + status = "disabled"; + }; + + rkcif_dvp: rkcif_dvp { + compatible = "rockchip,rkcif-dvp"; + rockchip,hw = <&rkcif>; + status = "disabled"; + }; + + rkcif_dvp_sditf: rkcif_dvp_sditf { + compatible = "rockchip,rkcif-sditf"; + rockchip,cif = <&rkcif_dvp>; + status = "disabled"; + }; + + rkcif_mipi_lvds: rkcif_mipi_lvds { + compatible = "rockchip,rkcif-mipi-lvds"; + rockchip,hw = <&rkcif>; + status = "disabled"; + }; + + rkcif_mipi_lvds_sditf: rkcif_mipi_lvds_sditf { + compatible = "rockchip,rkcif-sditf"; + rockchip,cif = <&rkcif_mipi_lvds>; + status = "disabled"; + }; + + rkisp: rkisp@fdff0000 { + compatible = "rockchip,rk3568-rkisp"; + reg = <0x0 0xfdff0000 0x0 0x10000>; + interrupts = , + , + ; + interrupt-names = "mipi_irq", "mi_irq", "isp_irq"; + clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>, <&cru CLK_ISP>; + clock-names = "aclk_isp", "hclk_isp", "clk_isp"; + resets = <&cru SRST_ISP>, <&cru SRST_H_ISP>; + reset-names = "isp", "isp-h"; + rockchip,grf = <&grf>; + power-domains = <&power RK3568_PD_VI>; + iommus = <&rkisp_mmu>; + rockchip,iq-feature = /bits/ 64 <0x3FBFFFE67FF>; + status = "disabled"; + }; + + rkisp_mmu: iommu@fdff1a00 { + compatible = "rockchip,iommu-v2"; + reg = <0x0 0xfdff1a00 0x0 0x100>; + interrupts = ; + interrupt-names = "isp_mmu"; + clocks = <&cru ACLK_ISP>, <&cru HCLK_ISP>; + clock-names = "aclk", "iface"; + power-domains = <&power RK3568_PD_VI>; + #iommu-cells = <0>; + rockchip,disable-mmu-reset; + status = "disabled"; + }; + + rkisp_vir0: rkisp-vir0 { + compatible = "rockchip,rkisp-vir"; + rockchip,hw = <&rkisp>; + status = "disabled"; + }; + + rkisp_vir1: rkisp-vir1 { + compatible = "rockchip,rkisp-vir"; + rockchip,hw = <&rkisp>; + status = "disabled"; + }; + + gmac1: ethernet@fe010000 { + compatible = "rockchip,rk3568-gmac", "snps,dwmac-4.20a"; + reg = <0x0 0xfe010000 0x0 0x10000>; + interrupts = , + ; + interrupt-names = "macirq", "eth_wake_irq"; + rockchip,grf = <&grf>; + clocks = <&cru SCLK_GMAC1>, <&cru SCLK_GMAC1_RX_TX>, + <&cru SCLK_GMAC1_RX_TX>, <&cru CLK_MAC1_REFOUT>, + <&cru ACLK_GMAC1>, <&cru PCLK_GMAC1>, + <&cru SCLK_GMAC1_RX_TX>, <&cru CLK_GMAC1_PTP_REF>, + <&cru PCLK_XPCS>; + clock-names = "stmmaceth", "mac_clk_rx", + "mac_clk_tx", "clk_mac_refout", + "aclk_mac", "pclk_mac", + "clk_mac_speed", "ptp_ref", + "pclk_xpcs"; + resets = <&cru SRST_A_GMAC1>; + reset-names = "stmmaceth"; + + snps,mixed-burst; + snps,tso; + + snps,axi-config = <&gmac1_stmmac_axi_setup>; + snps,mtl-rx-config = <&gmac1_mtl_rx_setup>; + snps,mtl-tx-config = <&gmac1_mtl_tx_setup>; + status = "disabled"; + + mdio1: mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <0x1>; + #size-cells = <0x0>; + }; + + gmac1_stmmac_axi_setup: stmmac-axi-config { + snps,wr_osr_lmt = <4>; + snps,rd_osr_lmt = <8>; + snps,blen = <0 0 0 0 16 8 4>; + }; + + gmac1_mtl_rx_setup: rx-queues-config { + snps,rx-queues-to-use = <1>; + queue0 {}; + }; + + gmac1_mtl_tx_setup: tx-queues-config { + snps,tx-queues-to-use = <1>; + queue0 {}; + }; + }; + + vop: vop@fe040000 { + compatible = "rockchip,rk3568-vop"; + reg = <0x0 0xfe040000 0x0 0x3000>, <0x0 0xfe044000 0x0 0x1000>; + reg-names = "regs", "gamma_lut"; + rockchip,grf = <&grf>; + interrupts = ; + clocks = <&cru ACLK_VOP>, <&cru HCLK_VOP>, <&cru DCLK_VOP0>, <&cru DCLK_VOP1>, <&cru DCLK_VOP2>; + clock-names = "aclk_vop", "hclk_vop", "dclk_vp0", "dclk_vp1", "dclk_vp2"; + iommus = <&vop_mmu>; + power-domains = <&power RK3568_PD_VO>; + status = "disabled"; + + vop_out: ports { + #address-cells = <1>; + #size-cells = <0>; + + vp0: port@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + vp0_out_dsi0: endpoint@0 { + reg = <0>; + remote-endpoint = <&dsi0_in_vp0>; + }; + + vp0_out_dsi1: endpoint@1 { + reg = <1>; + remote-endpoint = <&dsi1_in_vp0>; + }; + + vp0_out_edp: endpoint@2 { + reg = <2>; + remote-endpoint = <&edp_in_vp0>; + }; + + vp0_out_hdmi: endpoint@3 { + reg = <3>; + remote-endpoint = <&hdmi_in_vp0>; + }; + }; + + vp1: port@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + + vp1_out_dsi0: endpoint@0 { + reg = <0>; + remote-endpoint = <&dsi0_in_vp1>; + }; + + vp1_out_dsi1: endpoint@1 { + reg = <1>; + remote-endpoint = <&dsi1_in_vp1>; + }; + + vp1_out_edp: endpoint@2 { + reg = <2>; + remote-endpoint = <&edp_in_vp1>; + }; + + vp1_out_hdmi: endpoint@3 { + reg = <3>; + remote-endpoint = <&hdmi_in_vp1>; + }; + + vp1_out_lvds: endpoint@4 { + reg = <4>; + remote-endpoint = <&lvds_in_vp1>; + }; + }; + + vp2: port@2 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <2>; + + vp2_out_lvds: endpoint@0 { + reg = <0>; + remote-endpoint = <&lvds_in_vp2>; + }; + + vp2_out_rgb: endpoint@1 { + reg = <1>; + remote-endpoint = <&rgb_in_vp2>; + }; + }; + }; + }; + + vop_mmu: iommu@fe043e00 { + compatible = "rockchip,iommu-v2"; + reg = <0x0 0xfe043e00 0x0 0x100>, <0x0 0xfe043f00 0x0 0x100>; + interrupts = ; + interrupt-names = "vop_mmu"; + clocks = <&cru ACLK_VOP>, <&cru HCLK_VOP>; + clock-names = "aclk", "iface"; + #iommu-cells = <0>; + rockchip,disable-device-link-resume; + status = "disabled"; + }; + + dsi0: dsi@fe060000 { + compatible = "rockchip,rk3568-mipi-dsi"; + reg = <0x0 0xfe060000 0x0 0x10000>; + interrupts = ; + clocks = <&cru PCLK_DSITX_0>, <&cru HCLK_VO>; + clock-names = "pclk", "hclk"; + resets = <&cru SRST_P_DSITX_0>; + reset-names = "apb"; + phys = <&video_phy0>; + phy-names = "dphy"; + power-domains = <&power RK3568_PD_VO>; + rockchip,grf = <&grf>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + dsi0_in: port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dsi0_in_vp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&vp0_out_dsi0>; + status = "disabled"; + }; + + dsi0_in_vp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&vp1_out_dsi0>; + status = "disabled"; + }; + }; + }; + }; + + dsi1: dsi@fe070000 { + compatible = "rockchip,rk3568-mipi-dsi"; + reg = <0x0 0xfe070000 0x0 0x10000>; + interrupts = ; + clocks = <&cru PCLK_DSITX_1>, <&cru HCLK_VO>; + clock-names = "pclk", "hclk"; + resets = <&cru SRST_P_DSITX_1>; + reset-names = "apb"; + phys = <&video_phy1>; + phy-names = "dphy"; + power-domains = <&power RK3568_PD_VO>; + rockchip,grf = <&grf>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + dsi1_in: port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + dsi1_in_vp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&vp0_out_dsi1>; + status = "disabled"; + }; + + dsi1_in_vp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&vp1_out_dsi1>; + status = "disabled"; + }; + }; + }; + }; + + hdmi: hdmi@fe0a0000 { + compatible = "rockchip,rk3568-dw-hdmi"; + reg = <0x0 0xfe0a0000 0x0 0x20000>; + interrupts = ; + clocks = <&cru PCLK_HDMI_HOST>, + <&cru CLK_HDMI_SFR>, + <&cru CLK_HDMI_CEC>, + <&pmucru PLL_HPLL>, + <&cru HCLK_VOP>; + clock-names = "iahb", "isfr", "cec", "ref", "hclk"; + power-domains = <&power RK3568_PD_VO>; + reg-io-width = <4>; + rockchip,grf = <&grf>; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmitx_scl &hdmitx_sda &hdmitxm0_cec>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + hdmi_in: port { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + hdmi_in_vp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&vp0_out_hdmi>; + status = "disabled"; + }; + hdmi_in_vp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&vp1_out_hdmi>; + status = "disabled"; + }; + }; + }; + }; + + edp: edp@fe0c0000 { + compatible = "rockchip,rk3568-edp"; + reg = <0x0 0xfe0c0000 0x0 0x10000>; + interrupts = ; + clocks = <&pmucru XIN_OSC0_EDPPHY_G>, <&cru PCLK_EDP_CTRL>, + <&cru CLK_EDP_200M>, <&cru HCLK_VO>; + clock-names = "dp", "pclk", "spdif", "hclk"; + resets = <&cru SRST_EDP_24M>, <&cru SRST_P_EDP_CTRL>; + reset-names = "dp", "apb"; + phys = <&edp_phy>; + phy-names = "dp"; + power-domains = <&power RK3568_PD_VO>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + edp_in: port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + edp_in_vp0: endpoint@0 { + reg = <0>; + remote-endpoint = <&vp0_out_edp>; + status = "disabled"; + }; + + edp_in_vp1: endpoint@1 { + reg = <1>; + remote-endpoint = <&vp1_out_edp>; + status = "disabled"; + }; + }; + }; + }; + + qos_gpu: qos@fe128000 { + compatible = "syscon"; + reg = <0x0 0xfe128000 0x0 0x20>; + }; + + qos_rkvenc_rd_m0: qos@fe138080 { + compatible = "syscon"; + reg = <0x0 0xfe138080 0x0 0x20>; + }; + + qos_rkvenc_rd_m1: qos@fe138100 { + compatible = "syscon"; + reg = <0x0 0xfe138100 0x0 0x20>; + }; + + qos_rkvenc_wr_m0: qos@fe138180 { + compatible = "syscon"; + reg = <0x0 0xfe138180 0x0 0x20>; + }; + + qos_isp: qos@fe148000 { + compatible = "syscon"; + reg = <0x0 0xfe148000 0x0 0x20>; + }; + + qos_vicap0: qos@fe148080 { + compatible = "syscon"; + reg = <0x0 0xfe148080 0x0 0x20>; + }; + + qos_vicap1: qos@fe148100 { + compatible = "syscon"; + reg = <0x0 0xfe148100 0x0 0x20>; + }; + + qos_vpu: qos@fe150000 { + compatible = "syscon"; + reg = <0x0 0xfe150000 0x0 0x20>; + }; + + qos_ebc: qos@fe158000 { + compatible = "syscon"; + reg = <0x0 0xfe158000 0x0 0x20>; + }; + + qos_iep: qos@fe158100 { + compatible = "syscon"; + reg = <0x0 0xfe158100 0x0 0x20>; + }; + + qos_jpeg_dec: qos@fe158180 { + compatible = "syscon"; + reg = <0x0 0xfe158180 0x0 0x20>; + }; + + qos_jpeg_enc: qos@fe158200 { + compatible = "syscon"; + reg = <0x0 0xfe158200 0x0 0x20>; + }; + + qos_rga_rd: qos@fe158280 { + compatible = "syscon"; + reg = <0x0 0xfe158280 0x0 0x20>; + }; + + qos_rga_wr: qos@fe158300 { + compatible = "syscon"; + reg = <0x0 0xfe158300 0x0 0x20>; + }; + + qos_npu: qos@fe180000 { + compatible = "syscon"; + reg = <0x0 0xfe180000 0x0 0x20>; + }; + + qos_pcie2x1: qos@fe190000 { + compatible = "syscon"; + reg = <0x0 0xfe190000 0x0 0x20>; + }; + + qos_pcie3x1: qos@fe190080 { + compatible = "syscon"; + reg = <0x0 0xfe190080 0x0 0x20>; + }; + + qos_pcie3x2: qos@fe190100 { + compatible = "syscon"; + reg = <0x0 0xfe190100 0x0 0x20>; + }; + + qos_sata0: qos@fe190200 { + compatible = "syscon"; + reg = <0x0 0xfe190200 0x0 0x20>; + }; + + qos_sata1: qos@fe190280 { + compatible = "syscon"; + reg = <0x0 0xfe190280 0x0 0x20>; + }; + + qos_sata2: qos@fe190300 { + compatible = "syscon"; + reg = <0x0 0xfe190300 0x0 0x20>; + }; + + qos_usb3_0: qos@fe190380 { + compatible = "syscon"; + reg = <0x0 0xfe190380 0x0 0x20>; + }; + + qos_usb3_1: qos@fe190400 { + compatible = "syscon"; + reg = <0x0 0xfe190400 0x0 0x20>; + }; + + qos_rkvdec: qos@fe198000 { + compatible = "syscon"; + reg = <0x0 0xfe198000 0x0 0x20>; + }; + + qos_hdcp: qos@fe1a8000 { + compatible = "syscon"; + reg = <0x0 0xfe1a8000 0x0 0x20>; + }; + + qos_vop_m0: qos@fe1a8080 { + compatible = "syscon"; + reg = <0x0 0xfe1a8080 0x0 0x20>; + }; + + qos_vop_m1: qos@fe1a8100 { + compatible = "syscon"; + reg = <0x0 0xfe1a8100 0x0 0x20>; + }; + + sdmmc2: dwmmc@fe000000 { + compatible = "rockchip,rk3568-dw-mshc", + "rockchip,rk3288-dw-mshc"; + reg = <0x0 0xfe000000 0x0 0x4000>; + interrupts = ; + max-frequency = <150000000>; + clocks = <&cru HCLK_SDMMC2>, <&cru CLK_SDMMC2>, + <&cru SCLK_SDMMC2_DRV>, <&cru SCLK_SDMMC2_SAMPLE>; + clock-names = "biu", "ciu", "ciu-drive", "ciu-sample"; + fifo-depth = <0x100>; + resets = <&cru SRST_SDMMC2>; + reset-names = "reset"; + status = "disabled"; + }; + + dfi: dfi@fe230000 { + reg = <0x00 0xfe230000 0x00 0x400>; + compatible = "rockchip,rk3568-dfi"; + rockchip,pmugrf = <&pmugrf>; + status = "disabled"; + }; + + dmc: dmc { + compatible = "rockchip,rk3568-dmc"; + interrupts = ; + interrupt-names = "complete"; + devfreq-events = <&dfi>; + clocks = <&scmi_clk 3>; + clock-names = "dmc_clk"; + operating-points-v2 = <&dmc_opp_table>; + ddr_timing = <&ddr_timing>; + vop-bw-dmc-freq = < + /* min_bw(MB/s) max_bw(MB/s) freq(KHz) */ + 0 505 324000 + 506 99999 528000 + >; + upthreshold = <40>; + downdifferential = <20>; + system-status-level = < + /*system status freq level*/ + SYS_STATUS_NORMAL DMC_FREQ_LEVEL_MID_HIGH + SYS_STATUS_REBOOT DMC_FREQ_LEVEL_HIGH + SYS_STATUS_SUSPEND DMC_FREQ_LEVEL_LOW + SYS_STATUS_VIDEO_4K DMC_FREQ_LEVEL_MID_HIGH + SYS_STATUS_VIDEO_4K_10B DMC_FREQ_LEVEL_MID_HIGH + SYS_STATUS_BOOST DMC_FREQ_LEVEL_HIGH + SYS_STATUS_ISP DMC_FREQ_LEVEL_HIGH + SYS_STATUS_PERFORMANCE DMC_FREQ_LEVEL_HIGH + SYS_STATUS_DUALVIEW DMC_FREQ_LEVEL_HIGH + >; + auto-min-freq = <324000>; + auto-freq-en = <1>; + #cooling-cells = <2>; + status = "disabled"; + }; + + dmc_opp_table: dmc-opp-table { + compatible = "operating-points-v2"; + + mbist-vmin = <825000 900000 950000>; + nvmem-cells = <&log_leakage>, <&core_pvtm>, <&mbist_vmin>; + nvmem-cell-names = "leakage", "pvtm", "mbist-vmin"; + rockchip,temp-hysteresis = <5000>; + rockchip,low-temp = <0>; + rockchip,low-temp-adjust-volt = < + /* MHz MHz uV */ + 0 1560 25000 + >; + rockchip,leakage-voltage-sel = < + 1 80 0 + 81 254 1 + >; + + opp-1560000000 { + opp-hz = /bits/ 64 <1560000000>; + opp-microvolt = <900000>; + opp-microvolt-L0 = <900000>; + opp-microvolt-L1 = <850000>; + }; + }; + + pcie2x1: pcie@fe260000 { + compatible = "rockchip,rk3568-pcie", "snps,dw-pcie"; + #address-cells = <3>; + #size-cells = <2>; + bus-range = <0x0 0xf>; + clocks = <&cru ACLK_PCIE20_MST>, <&cru ACLK_PCIE20_SLV>, + <&cru ACLK_PCIE20_DBI>, <&cru PCLK_PCIE20>, + <&cru CLK_PCIE20_AUX_NDFT>; + clock-names = "aclk_mst", "aclk_slv", + "aclk_dbi", "pclk", "aux"; + device_type = "pci"; + interrupts = , + , + , + , + ; + interrupt-names = "sys", "pmc", "msg", "legacy", "err"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie2x1_intc 0>, + <0 0 0 2 &pcie2x1_intc 1>, + <0 0 0 3 &pcie2x1_intc 2>, + <0 0 0 4 &pcie2x1_intc 3>; + linux,pci-domain = <0>; + num-ib-windows = <6>; + num-ob-windows = <2>; + max-link-speed = <2>; + msi-map = <0x0 &its 0x0 0x1000>; + num-lanes = <1>; + phys = <&combphy2_psq PHY_TYPE_PCIE>; + phy-names = "pcie-phy"; + power-domains = <&power RK3568_PD_PIPE>; + ranges = <0x00000800 0x0 0x00000000 0x3 0x00000000 0x0 0x800000 + 0x81000000 0x0 0x00800000 0x3 0x00800000 0x0 0x100000 + 0x83000000 0x0 0x00900000 0x3 0x00900000 0x0 0x3f700000>; + reg = <0x3 0xc0000000 0x0 0x400000>, + <0x0 0xfe260000 0x0 0x10000>; + reg-names = "pcie-dbi", "pcie-apb"; + resets = <&cru SRST_PCIE20_POWERUP>; + reset-names = "pipe"; + status = "disabled"; + + pcie2x1_intc: legacy-interrupt-controller { + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-parent = <&gic>; + interrupts = ; + }; + }; + + pcie3x1: pcie@fe270000 { + compatible = "rockchip,rk3568-pcie", "snps,dw-pcie"; + #address-cells = <3>; + #size-cells = <2>; + bus-range = <0x10 0x1f>; + clocks = <&cru ACLK_PCIE30X1_MST>, <&cru ACLK_PCIE30X1_SLV>, + <&cru ACLK_PCIE30X1_DBI>, <&cru PCLK_PCIE30X1>, + <&cru CLK_PCIE30X1_AUX_NDFT>; + clock-names = "aclk_mst", "aclk_slv", + "aclk_dbi", "pclk", "aux"; + device_type = "pci"; + interrupts = , + , + , + , + ; + interrupt-names = "sys", "pmc", "msg", "legacy", "err"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie3x1_intc 0>, + <0 0 0 2 &pcie3x1_intc 1>, + <0 0 0 3 &pcie3x1_intc 2>, + <0 0 0 4 &pcie3x1_intc 3>; + linux,pci-domain = <1>; + num-ib-windows = <6>; + num-ob-windows = <2>; + max-link-speed = <3>; + msi-map = <0x1000 &its 0x1000 0x1000>; + num-lanes = <1>; + phys = <&pcie30phy>; + phy-names = "pcie-phy"; + power-domains = <&power RK3568_PD_PIPE>; + ranges = <0x00000800 0x0 0x40000000 0x3 0x40000000 0x0 0x800000 + 0x81000000 0x0 0x40800000 0x3 0x40800000 0x0 0x100000 + 0x83000000 0x0 0x40900000 0x3 0x40900000 0x0 0x3f700000>; + reg = <0x3 0xc0400000 0x0 0x400000>, + <0x0 0xfe270000 0x0 0x10000>; + reg-names = "pcie-dbi", "pcie-apb"; + resets = <&cru SRST_PCIE30X1_POWERUP>; + reset-names = "pipe"; + /* rockchip,bifurcation; lane1 when using 1+1 */ + status = "disabled"; + + pcie3x1_intc: legacy-interrupt-controller { + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-parent = <&gic>; + interrupts = ; + }; + }; + + pcie3x2: pcie@fe280000 { + compatible = "rockchip,rk3568-pcie", "snps,dw-pcie"; + #address-cells = <3>; + #size-cells = <2>; + bus-range = <0x20 0x2f>; + clocks = <&cru ACLK_PCIE30X2_MST>, <&cru ACLK_PCIE30X2_SLV>, + <&cru ACLK_PCIE30X2_DBI>, <&cru PCLK_PCIE30X2>, + <&cru CLK_PCIE30X2_AUX_NDFT>; + clock-names = "aclk_mst", "aclk_slv", + "aclk_dbi", "pclk", "aux"; + device_type = "pci"; + interrupts = , + , + , + , + ; + interrupt-names = "sys", "pmc", "msg", "legacy", "err"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie3x2_intc 0>, + <0 0 0 2 &pcie3x2_intc 1>, + <0 0 0 3 &pcie3x2_intc 2>, + <0 0 0 4 &pcie3x2_intc 3>; + linux,pci-domain = <2>; + num-ib-windows = <6>; + num-ob-windows = <2>; + max-link-speed = <3>; + msi-map = <0x2000 &its 0x2000 0x1000>; + num-lanes = <2>; + phys = <&pcie30phy>; + phy-names = "pcie-phy"; + power-domains = <&power RK3568_PD_PIPE>; + ranges = <0x00000800 0x0 0x80000000 0x3 0x80000000 0x0 0x800000 + 0x81000000 0x0 0x80800000 0x3 0x80800000 0x0 0x100000 + 0x83000000 0x0 0x80900000 0x3 0x80900000 0x0 0x3f700000>; + reg = <0x3 0xc0800000 0x0 0x400000>, + <0x0 0xfe280000 0x0 0x10000>; + reg-names = "pcie-dbi", "pcie-apb"; + resets = <&cru SRST_PCIE30X2_POWERUP>; + reset-names = "pipe"; + /* rockchip,bifurcation; lane0 when using 1+1 */ + status = "disabled"; + + pcie3x2_intc: legacy-interrupt-controller { + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-parent = <&gic>; + interrupts = ; + }; + }; + + gmac0: ethernet@fe2a0000 { + compatible = "rockchip,rk3568-gmac", "snps,dwmac-4.20a"; + reg = <0x0 0xfe2a0000 0x0 0x10000>; + interrupts = , + ; + interrupt-names = "macirq", "eth_wake_irq"; + rockchip,grf = <&grf>; + clocks = <&cru SCLK_GMAC0>, <&cru SCLK_GMAC0_RX_TX>, + <&cru SCLK_GMAC0_RX_TX>, <&cru CLK_MAC0_REFOUT>, + <&cru ACLK_GMAC0>, <&cru PCLK_GMAC0>, + <&cru SCLK_GMAC0_RX_TX>, <&cru CLK_GMAC0_PTP_REF>, + <&cru PCLK_XPCS>; + clock-names = "stmmaceth", "mac_clk_rx", + "mac_clk_tx", "clk_mac_refout", + "aclk_mac", "pclk_mac", + "clk_mac_speed", "ptp_ref", + "pclk_xpcs"; + resets = <&cru SRST_A_GMAC0>; + reset-names = "stmmaceth"; + + snps,mixed-burst; + snps,tso; + + snps,axi-config = <&gmac0_stmmac_axi_setup>; + snps,mtl-rx-config = <&gmac0_mtl_rx_setup>; + snps,mtl-tx-config = <&gmac0_mtl_tx_setup>; + status = "disabled"; + + mdio0: mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <0x1>; + #size-cells = <0x0>; + }; + + gmac0_stmmac_axi_setup: stmmac-axi-config { + snps,wr_osr_lmt = <4>; + snps,rd_osr_lmt = <8>; + snps,blen = <0 0 0 0 16 8 4>; + }; + + gmac0_mtl_rx_setup: rx-queues-config { + snps,rx-queues-to-use = <1>; + queue0 {}; + }; + + gmac0_mtl_tx_setup: tx-queues-config { + snps,tx-queues-to-use = <1>; + queue0 {}; + }; + }; + + sdmmc0: dwmmc@fe2b0000 { + compatible = "rockchip,rk3568-dw-mshc", + "rockchip,rk3288-dw-mshc"; + reg = <0x0 0xfe2b0000 0x0 0x4000>; + interrupts = ; + max-frequency = <150000000>; + clocks = <&cru HCLK_SDMMC0>, <&cru CLK_SDMMC0>, + <&cru SCLK_SDMMC0_DRV>, <&cru SCLK_SDMMC0_SAMPLE>; + clock-names = "biu", "ciu", "ciu-drive", "ciu-sample"; + fifo-depth = <0x100>; + resets = <&cru SRST_SDMMC0>; + reset-names = "reset"; + status = "disabled"; + }; + + sdmmc1: dwmmc@fe2c0000 { + compatible = "rockchip,rk3568-dw-mshc", + "rockchip,rk3288-dw-mshc"; + reg = <0x0 0xfe2c0000 0x0 0x4000>; + interrupts = ; + max-frequency = <150000000>; + clocks = <&cru HCLK_SDMMC1>, <&cru CLK_SDMMC1>, + <&cru SCLK_SDMMC1_DRV>, <&cru SCLK_SDMMC1_SAMPLE>; + clock-names = "biu", "ciu", "ciu-drive", "ciu-sample"; + fifo-depth = <0x100>; + resets = <&cru SRST_SDMMC1>; + reset-names = "reset"; + status = "disabled"; + }; + + sfc: sfc@fe300000 { + compatible = "rockchip,sfc"; + reg = <0x0 0xfe300000 0x0 0x4000>; + interrupts = ; + clocks = <&cru SCLK_SFC>, <&cru HCLK_SFC>; + clock-names = "clk_sfc", "hclk_sfc"; + assigned-clocks = <&cru SCLK_SFC>; + assigned-clock-rates = <100000000>; + status = "disabled"; + }; + + sdhci: sdhci@fe310000 { + compatible = "rockchip,dwcmshc-sdhci", "snps,dwcmshc-sdhci"; + reg = <0x0 0xfe310000 0x0 0x10000>; + interrupts = ; + assigned-clocks = <&cru BCLK_EMMC>, <&cru TCLK_EMMC>; + assigned-clock-rates = <200000000>, <24000000>; + clocks = <&cru CCLK_EMMC>, <&cru HCLK_EMMC>, + <&cru ACLK_EMMC>, <&cru BCLK_EMMC>, + <&cru TCLK_EMMC>; + clock-names = "core", "bus", "axi", "block", "timer"; + status = "disabled"; + }; + + nandc0: nandc@fe330000 { + compatible = "rockchip,rk-nandc-v9"; + reg = <0x0 0xfe330000 0x0 0x4000>; + interrupts = ; + nandc_id = <0>; + clocks = <&cru NCLK_NANDC>, <&cru HCLK_NANDC>; + clock-names = "clk_nandc", "hclk_nandc"; + status = "disabled"; + }; + + crypto: crypto@fe380000 { + compatible = "rockchip,rk3568-crypto"; + reg = <0x0 0xfe380000 0x0 0x4000>; + interrupts = ; + clocks = <&cru ACLK_CRYPTO_NS>, <&cru HCLK_CRYPTO_NS>, + <&cru CLK_CRYPTO_NS_CORE>, <&cru CLK_CRYPTO_NS_PKA>; + clock-names = "aclk", "hclk", "sclk", "apb_pclk"; + assigned-clocks = <&cru CLK_CRYPTO_NS_CORE>; + assigned-clock-rates = <200000000>; + resets = <&cru SRST_CRYPTO_NS_CORE>; + reset-names = "crypto-rst"; + status = "disabled"; + }; + + rng: rng@fe388000 { + compatible = "rockchip,cryptov2-rng"; + reg = <0x0 0xfe388000 0x0 0x2000>; + clocks = <&cru CLK_TRNG_NS>, <&cru HCLK_TRNG_NS>; + clock-names = "clk_trng", "hclk_trng"; + resets = <&cru SRST_TRNG_NS>; + reset-names = "reset"; + status = "disabled"; + }; + + otp: otp@fe38c000 { + compatible = "rockchip,rk3568-otp"; + reg = <0x0 0xfe38c000 0x0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + clocks = <&cru CLK_OTPC_NS_USR>, <&cru CLK_OTPC_NS_SBPI>, + <&cru PCLK_OTPC_NS>, <&cru PCLK_OTPPHY>; + clock-names = "usr", "sbpi", "apb", "phy"; + resets = <&cru SRST_OTPPHY>; + reset-names = "otp_phy"; + + /* Data cells */ + cpu_code: cpu-code@2 { + reg = <0x02 0x2>; + }; + otp_cpu_version: cpu-version@8 { + reg = <0x08 0x1>; + bits = <3 3>; + }; + mbist_vmin: mbist-vmin@9 { + reg = <0x09 0x1>; + bits = <0 4>; + }; + otp_id: id@a { + reg = <0x0a 0x10>; + }; + cpu_leakage: cpu-leakage@1a { + reg = <0x1a 0x1>; + }; + log_leakage: log-leakage@1b { + reg = <0x1b 0x1>; + }; + npu_leakage: npu-leakage@1c { + reg = <0x1c 0x1>; + }; + gpu_leakage: gpu-leakage@1d { + reg = <0x1d 0x1>; + }; + core_pvtm:core-pvtm@2a { + reg = <0x2a 0x2>; + }; + }; + + i2s0_8ch: i2s@fe400000 { + compatible = "rockchip,rk3568-i2s-tdm"; + reg = <0x0 0xfe400000 0x0 0x1000>; + interrupts = ; + clocks = <&cru MCLK_I2S0_8CH_TX>, <&cru MCLK_I2S0_8CH_RX>, <&cru HCLK_I2S0_8CH>; + clock-names = "mclk_tx", "mclk_rx", "hclk"; + dmas = <&dmac1 0>; + dma-names = "tx"; + resets = <&cru SRST_M_I2S0_8CH_TX>, <&cru SRST_M_I2S0_8CH_RX>; + reset-names = "tx-m", "rx-m"; + rockchip,cru = <&cru>; + rockchip,grf = <&grf>; + rockchip,playback-only; + #sound-dai-cells = <0>; + status = "disabled"; + }; + + i2s1_8ch: i2s@fe410000 { + compatible = "rockchip,rk3568-i2s-tdm"; + reg = <0x0 0xfe410000 0x0 0x1000>; + interrupts = ; + clocks = <&cru MCLK_I2S1_8CH_TX>, <&cru MCLK_I2S1_8CH_RX>, <&cru HCLK_I2S1_8CH>; + clock-names = "mclk_tx", "mclk_rx", "hclk"; + dmas = <&dmac1 2>, <&dmac1 3>; + dma-names = "tx", "rx"; + resets = <&cru SRST_M_I2S1_8CH_TX>, <&cru SRST_M_I2S1_8CH_RX>; + reset-names = "tx-m", "rx-m"; + rockchip,cru = <&cru>; + rockchip,grf = <&grf>; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_sclkrx + &i2s1m0_lrcktx + &i2s1m0_lrckrx + &i2s1m0_sdi0 + &i2s1m0_sdi1 + &i2s1m0_sdi2 + &i2s1m0_sdi3 + &i2s1m0_sdo0 + &i2s1m0_sdo1 + &i2s1m0_sdo2 + &i2s1m0_sdo3>; + status = "disabled"; + }; + + i2s2_2ch: i2s@fe420000 { + compatible = "rockchip,rk3568-i2s-tdm"; + reg = <0x0 0xfe420000 0x0 0x1000>; + interrupts = ; + clocks = <&cru MCLK_I2S2_2CH>, <&cru MCLK_I2S2_2CH>, <&cru HCLK_I2S2_2CH>; + clock-names = "mclk_tx", "mclk_rx", "hclk"; + dmas = <&dmac1 4>, <&dmac1 5>; + dma-names = "tx", "rx"; + rockchip,cru = <&cru>; + rockchip,grf = <&grf>; + rockchip,clk-trcm = <1>; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s2m0_sclktx + &i2s2m0_lrcktx + &i2s2m0_sdi + &i2s2m0_sdo>; + status = "disabled"; + }; + + i2s3_2ch: i2s@fe430000 { + compatible = "rockchip,rk3568-i2s-tdm"; + reg = <0x0 0xfe430000 0x0 0x1000>; + interrupts = ; + clocks = <&cru MCLK_I2S3_2CH_TX>, <&cru MCLK_I2S3_2CH_RX>, <&cru HCLK_I2S3_2CH>; + clock-names = "mclk_tx", "mclk_rx", "hclk"; + dmas = <&dmac1 6>, <&dmac1 7>; + dma-names = "tx", "rx"; + resets = <&cru SRST_M_I2S3_2CH_TX>, <&cru SRST_M_I2S3_2CH_RX>; + reset-names = "tx-m", "rx-m"; + rockchip,cru = <&cru>; + rockchip,grf = <&grf>; + rockchip,clk-trcm = <1>; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s3m0_sclk + &i2s3m0_lrck + &i2s3m0_sdi + &i2s3m0_sdo>; + status = "disabled"; + }; + + pdm: pdm@fe440000 { + compatible = "rockchip,rk3568-pdm", "rockchip,pdm"; + reg = <0x0 0xfe440000 0x0 0x1000>; + clocks = <&cru MCLK_PDM>, <&cru HCLK_PDM>; + clock-names = "pdm_clk", "pdm_hclk"; + dmas = <&dmac1 9>; + dma-names = "rx"; + pinctrl-names = "default"; + pinctrl-0 = <&pdmm0_clk + &pdmm0_clk1 + &pdmm0_sdi0 + &pdmm0_sdi1 + &pdmm0_sdi2 + &pdmm0_sdi3>; + #sound-dai-cells = <0>; + status = "disabled"; + }; + + vad: vad@fe450000 { + compatible = "rockchip,rk3568-vad"; + reg = <0x0 0xfe450000 0x0 0x10000>; + reg-names = "vad"; + clocks = <&cru HCLK_VAD>; + clock-names = "hclk"; + interrupts = ; + rockchip,audio-src = <0>; + rockchip,det-channel = <0>; + rockchip,mode = <0>; + #sound-dai-cells = <0>; + status = "disabled"; + }; + + spdif_8ch: spdif@fe460000 { + compatible = "rockchip,rk3568-spdif"; + reg = <0x0 0xfe460000 0x0 0x1000>; + interrupts = ; + dmas = <&dmac1 1>; + dma-names = "tx"; + clock-names = "mclk", "hclk"; + clocks = <&cru MCLK_SPDIF_8CH>, <&cru HCLK_SPDIF_8CH>; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&spdifm0_tx>; + status = "disabled"; + }; + + audpwm: audpwm@fe470000 { + compatible = "rockchip,rk3568-audio-pwm", "rockchip,audio-pwm-v1"; + reg = <0x0 0xfe470000 0x0 0x1000>; + clocks = <&cru SCLK_AUDPWM>, <&cru HCLK_AUDPWM>; + clock-names = "clk", "hclk"; + dmas = <&dmac1 8>; + dma-names = "tx"; + #sound-dai-cells = <0>; + rockchip,sample-width-bits = <11>; + rockchip,interpolat-points = <1>; + status = "disabled"; + }; + + dig_acodec: codec-digital@fe478000 { + compatible = "rockchip,rk3568-codec-digital", "rockchip,codec-digital-v1"; + reg = <0x0 0xfe478000 0x0 0x1000>; + clocks = <&cru CLK_ACDCDIG_ADC>, <&cru CLK_ACDCDIG_DAC>, + <&cru CLK_ACDCDIG_I2C>, <&cru HCLK_ACDCDIG>; + clock-names = "adc", "dac", "i2c", "pclk"; + pinctrl-names = "default"; + pinctrl-0 = <&acodec_pins>; + resets = <&cru SRST_ACDCDIG>; + reset-names = "reset" ; + rockchip,grf = <&grf>; + #sound-dai-cells = <0>; + status = "disabled"; + }; + + dmac0: dmac@fe530000 { + compatible = "arm,pl330", "arm,primecell"; + reg = <0x0 0xfe530000 0x0 0x4000>; + interrupts = , + ; + clocks = <&cru ACLK_BUS>; + clock-names = "apb_pclk"; + #dma-cells = <1>; + arm,pl330-periph-burst; + }; + + dmac1: dmac@fe550000 { + compatible = "arm,pl330", "arm,primecell"; + reg = <0x0 0xfe550000 0x0 0x4000>; + interrupts = , + ; + clocks = <&cru ACLK_BUS>; + clock-names = "apb_pclk"; + #dma-cells = <1>; + arm,pl330-periph-burst; + }; + + scr: rkscr@fe560000 { + compatible = "rockchip-scr"; + reg = <0x0 0xfe560000 0x0 0x10000>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&scr_pins>; + clocks = <&cru PCLK_SCR>; + clock-names = "g_pclk_sim_card"; + status = "disabled"; + }; + + can0: can@fe570000 { + compatible = "rockchip,canfd-1.0"; + reg = <0x0 0xfe570000 0x0 0x1000>; + interrupts = ; + clocks = <&cru CLK_CAN0>, <&cru PCLK_CAN0>; + clock-names = "baudclk", "apb_pclk"; + resets = <&cru SRST_CAN0>, <&cru SRST_P_CAN0>; + reset-names = "can", "can-apb"; + tx-fifo-depth = <1>; + rx-fifo-depth = <6>; + status = "disabled"; + }; + + can1: can@fe580000 { + compatible = "rockchip,canfd-1.0"; + reg = <0x0 0xfe580000 0x0 0x1000>; + interrupts = ; + clocks = <&cru CLK_CAN1>, <&cru PCLK_CAN1>; + clock-names = "baudclk", "apb_pclk"; + resets = <&cru SRST_CAN1>, <&cru SRST_P_CAN1>; + reset-names = "can", "can-apb"; + tx-fifo-depth = <1>; + rx-fifo-depth = <6>; + status = "disabled"; + }; + + can2: can@fe590000 { + compatible = "rockchip,canfd-1.0"; + reg = <0x0 0xfe590000 0x0 0x1000>; + interrupts = ; + clocks = <&cru CLK_CAN2>, <&cru PCLK_CAN2>; + clock-names = "baudclk", "apb_pclk"; + resets = <&cru SRST_CAN2>, <&cru SRST_P_CAN2>; + reset-names = "can", "can-apb"; + tx-fifo-depth = <1>; + rx-fifo-depth = <6>; + status = "disabled"; + }; + + i2c1: i2c@fe5a0000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xfe5a0000 0x0 0x1000>; + clocks = <&cru CLK_I2C1>, <&cru PCLK_I2C1>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c2: i2c@fe5b0000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xfe5b0000 0x0 0x1000>; + clocks = <&cru CLK_I2C2>, <&cru PCLK_I2C2>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2m0_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c3: i2c@fe5c0000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xfe5c0000 0x0 0x1000>; + clocks = <&cru CLK_I2C3>, <&cru PCLK_I2C3>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m0_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c4: i2c@fe5d0000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xfe5d0000 0x0 0x1000>; + clocks = <&cru CLK_I2C4>, <&cru PCLK_I2C4>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c4m0_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c5: i2c@fe5e0000 { + compatible = "rockchip,rk3399-i2c"; + reg = <0x0 0xfe5e0000 0x0 0x1000>; + clocks = <&cru CLK_I2C5>, <&cru PCLK_I2C5>; + clock-names = "i2c", "pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c5m0_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + rktimer: timer@fe5f0000 { + compatible = "rockchip,rk3568-timer", "rockchip,rk3288-timer"; + reg = <0x0 0xfe5f0000 0x0 0x1000>; + interrupts = ; + clocks = <&cru PCLK_TIMER>, <&cru CLK_TIMER0>; + clock-names = "pclk", "timer"; + }; + + wdt: watchdog@fe600000 { + compatible = "snps,dw-wdt"; + reg = <0x0 0xfe600000 0x0 0x100>; + clocks = <&cru TCLK_WDT_NS>, <&cru PCLK_WDT_NS>; + clock-names = "tclk", "pclk"; + interrupts = ; + status = "okay"; + }; + + spi0: spi@fe610000 { + compatible = "rockchip,rk3066-spi"; + reg = <0x0 0xfe610000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&cru CLK_SPI0>, <&cru PCLK_SPI0>; + clock-names = "spiclk", "apb_pclk"; + dmas = <&dmac0 20>, <&dmac0 21>; + dma-names = "tx", "rx"; + pinctrl-names = "default", "high_speed"; + pinctrl-0 = <&spi0m0_cs0 &spi0m0_cs1 &spi0m0_pins>; + pinctrl-1 = <&spi0m0_cs0 &spi0m0_cs1 &spi0m0_pins_hs>; + status = "disabled"; + }; + + spi1: spi@fe620000 { + compatible = "rockchip,rk3066-spi"; + reg = <0x0 0xfe620000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&cru CLK_SPI1>, <&cru PCLK_SPI1>; + clock-names = "spiclk", "apb_pclk"; + dmas = <&dmac0 22>, <&dmac0 23>; + dma-names = "tx", "rx"; + pinctrl-names = "default", "high_speed"; + pinctrl-0 = <&spi1m0_cs0 &spi1m0_cs1 &spi1m0_pins>; + pinctrl-1 = <&spi1m0_cs0 &spi1m0_cs1 &spi1m0_pins_hs>; + status = "disabled"; + }; + + spi2: spi@fe630000 { + compatible = "rockchip,rk3066-spi"; + reg = <0x0 0xfe630000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&cru CLK_SPI2>, <&cru PCLK_SPI2>; + clock-names = "spiclk", "apb_pclk"; + dmas = <&dmac0 24>, <&dmac0 25>; + dma-names = "tx", "rx"; + pinctrl-names = "default", "high_speed"; + pinctrl-0 = <&spi2m0_cs0 &spi2m0_cs1 &spi2m0_pins>; + pinctrl-1 = <&spi2m0_cs0 &spi2m0_cs1 &spi2m0_pins_hs>; + status = "disabled"; + }; + + spi3: spi@fe640000 { + compatible = "rockchip,rk3066-spi"; + reg = <0x0 0xfe640000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&cru CLK_SPI3>, <&cru PCLK_SPI3>; + clock-names = "spiclk", "apb_pclk"; + dmas = <&dmac0 26>, <&dmac0 27>; + dma-names = "tx", "rx"; + pinctrl-names = "default", "high_speed"; + pinctrl-0 = <&spi3m0_cs0 &spi3m0_cs1 &spi3m0_pins>; + pinctrl-1 = <&spi3m0_cs0 &spi3m0_cs1 &spi3m0_pins_hs>; + status = "disabled"; + }; + + uart1: serial@fe650000 { + compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; + reg = <0x0 0xfe650000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART1>, <&cru PCLK_UART1>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac0 2>, <&dmac0 3>; + pinctrl-names = "default"; + pinctrl-0 = <&uart1m0_xfer>; + status = "disabled"; + }; + + uart2: serial@fe660000 { + compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; + reg = <0x0 0xfe660000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART2>, <&cru PCLK_UART2>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac0 4>, <&dmac0 5>; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer>; + status = "disabled"; + }; + + uart3: serial@fe670000 { + compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; + reg = <0x0 0xfe670000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART3>, <&cru PCLK_UART3>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac0 6>, <&dmac0 7>; + pinctrl-names = "default"; + pinctrl-0 = <&uart3m0_xfer>; + status = "disabled"; + }; + + uart4: serial@fe680000 { + compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; + reg = <0x0 0xfe680000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART4>, <&cru PCLK_UART4>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac0 8>, <&dmac0 9>; + pinctrl-names = "default"; + pinctrl-0 = <&uart4m0_xfer>; + status = "disabled"; + }; + + uart5: serial@fe690000 { + compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; + reg = <0x0 0xfe690000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART5>, <&cru PCLK_UART5>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac0 10>, <&dmac0 11>; + pinctrl-names = "default"; + pinctrl-0 = <&uart5m0_xfer>; + status = "disabled"; + }; + + uart6: serial@fe6a0000 { + compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; + reg = <0x0 0xfe6a0000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART6>, <&cru PCLK_UART6>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac0 12>, <&dmac0 13>; + pinctrl-names = "default"; + pinctrl-0 = <&uart6m0_xfer>; + status = "disabled"; + }; + + uart7: serial@fe6b0000 { + compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; + reg = <0x0 0xfe6b0000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART7>, <&cru PCLK_UART7>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac0 14>, <&dmac0 15>; + pinctrl-names = "default"; + pinctrl-0 = <&uart7m0_xfer>; + status = "disabled"; + }; + + uart8: serial@fe6c0000 { + compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; + reg = <0x0 0xfe6c0000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART8>, <&cru PCLK_UART8>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac0 16>, <&dmac0 17>; + pinctrl-names = "default"; + pinctrl-0 = <&uart8m0_xfer>; + status = "disabled"; + }; + + uart9: serial@fe6d0000 { + compatible = "rockchip,rk3568-uart", "snps,dw-apb-uart"; + reg = <0x0 0xfe6d0000 0x0 0x100>; + interrupts = ; + clocks = <&cru SCLK_UART9>, <&cru PCLK_UART9>; + clock-names = "baudclk", "apb_pclk"; + reg-shift = <2>; + reg-io-width = <4>; + dmas = <&dmac0 18>, <&dmac0 19>; + pinctrl-names = "default"; + pinctrl-0 = <&uart9m0_xfer>; + status = "disabled"; + }; + + pwm4: pwm@fe6e0000 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe6e0000 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm4_pins>; + clocks = <&cru CLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm5: pwm@fe6e0010 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe6e0010 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm5_pins>; + clocks = <&cru CLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm6: pwm@fe6e0020 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe6e0020 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm6_pins>; + clocks = <&cru CLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm7: pwm@fe6e0030 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe6e0030 0x0 0x10>; + interrupts = , + ; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm7_pins>; + clocks = <&cru CLK_PWM1>, <&cru PCLK_PWM1>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm8: pwm@fe6f0000 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe6f0000 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm8m0_pins>; + clocks = <&cru CLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm9: pwm@fe6f0010 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe6f0010 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm9m0_pins>; + clocks = <&cru CLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm10: pwm@fe6f0020 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe6f0020 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm10m0_pins>; + clocks = <&cru CLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm11: pwm@fe6f0030 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe6f0030 0x0 0x10>; + interrupts = , + ; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm11m0_pins>; + clocks = <&cru CLK_PWM2>, <&cru PCLK_PWM2>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm12: pwm@fe700000 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe700000 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm12m0_pins>; + clocks = <&cru CLK_PWM3>, <&cru PCLK_PWM3>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm13: pwm@fe700010 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe700010 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm13m0_pins>; + clocks = <&cru CLK_PWM3>, <&cru PCLK_PWM3>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm14: pwm@fe700020 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe700020 0x0 0x10>; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm14m0_pins>; + clocks = <&cru CLK_PWM3>, <&cru PCLK_PWM3>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + pwm15: pwm@fe700030 { + compatible = "rockchip,rk3568-pwm", "rockchip,rk3328-pwm"; + reg = <0x0 0xfe700030 0x0 0x10>; + interrupts = , + ; + #pwm-cells = <3>; + pinctrl-names = "active"; + pinctrl-0 = <&pwm15m0_pins>; + clocks = <&cru CLK_PWM3>, <&cru PCLK_PWM3>; + clock-names = "pwm", "pclk"; + status = "disabled"; + }; + + tsadc: tsadc@fe710000 { + compatible = "rockchip,rk3568-tsadc"; + reg = <0x0 0xfe710000 0x0 0x100>; + interrupts = ; + rockchip,grf = <&grf>; + clocks = <&cru CLK_TSADC>, <&cru PCLK_TSADC>; + clock-names = "tsadc", "apb_pclk"; + assigned-clocks = <&cru CLK_TSADC_TSEN>, <&cru CLK_TSADC>; + assigned-clock-rates = <17000000>, <700000>; + resets = <&cru SRST_TSADC>, <&cru SRST_P_TSADC>, + <&cru SRST_TSADCPHY>; + reset-names = "tsadc", "tsadc-apb", "tsadc-phy"; + #thermal-sensor-cells = <1>; + rockchip,hw-tshut-temp = <120000>; + rockchip,hw-tshut-mode = <0>; /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-polarity = <0>; /* tshut polarity 0:LOW 1:HIGH */ + pinctrl-names = "gpio", "otpout"; + pinctrl-0 = <&tsadc_gpio_func>; + pinctrl-1 = <&tsadc_shutorg>; + status = "disabled"; + }; + + saradc: saradc@fe720000 { + compatible = "rockchip,rk3568-saradc", "rockchip,rk3399-saradc"; + reg = <0x0 0xfe720000 0x0 0x100>; + interrupts = ; + #io-channel-cells = <1>; + clocks = <&cru CLK_SARADC>, <&cru PCLK_SARADC>; + clock-names = "saradc", "apb_pclk"; + resets = <&cru SRST_P_SARADC>; + reset-names = "saradc-apb"; + status = "disabled"; + }; + + mailbox: mailbox@fe780000 { + compatible = "rockchip,rk3568-mailbox", + "rockchip,rk3368-mailbox"; + reg = <0x0 0xfe780000 0x0 0x1000>; + interrupts = , + , + , + ; + clocks = <&cru PCLK_MAILBOX>; + clock-names = "pclk_mailbox"; + #mbox-cells = <1>; + status = "disabled"; + }; + + combphy0_us: phy@fe820000 { + compatible = "rockchip,rk3568-naneng-combphy"; + reg = <0x0 0xfe820000 0x0 0x100>; + #phy-cells = <1>; + clocks = <&pmucru CLK_PCIEPHY0_REF>, <&cru PCLK_PIPEPHY0>, + <&cru PCLK_PIPE>; + clock-names = "refclk", "apbclk", "pipe_clk"; + assigned-clocks = <&pmucru CLK_PCIEPHY0_REF>; + assigned-clock-rates = <100000000>; + resets = <&cru SRST_P_PIPEPHY0>, <&cru SRST_PIPEPHY0>; + reset-names = "combphy-apb", "combphy"; + rockchip,pipe-grf = <&pipegrf>; + rockchip,pipe-phy-grf = <&pipe_phy_grf0>; + status = "disabled"; + }; + + combphy1_usq: phy@fe830000 { + compatible = "rockchip,rk3568-naneng-combphy"; + reg = <0x0 0xfe830000 0x0 0x100>; + #phy-cells = <1>; + clocks = <&pmucru CLK_PCIEPHY1_REF>, <&cru PCLK_PIPEPHY1>, + <&cru PCLK_PIPE>; + clock-names = "refclk", "apbclk", "pipe_clk"; + assigned-clocks = <&pmucru CLK_PCIEPHY1_REF>; + assigned-clock-rates = <100000000>; + resets = <&cru SRST_P_PIPEPHY1>, <&cru SRST_PIPEPHY1>; + reset-names = "combphy-apb", "combphy"; + rockchip,pipe-grf = <&pipegrf>; + rockchip,pipe-phy-grf = <&pipe_phy_grf1>; + status = "disabled"; + }; + + combphy2_psq: phy@fe840000 { + compatible = "rockchip,rk3568-naneng-combphy"; + reg = <0x0 0xfe840000 0x0 0x100>; + #phy-cells = <1>; + clocks = <&pmucru CLK_PCIEPHY2_REF>, <&cru PCLK_PIPEPHY2>, + <&cru PCLK_PIPE>; + clock-names = "refclk", "apbclk", "pipe_clk"; + assigned-clocks = <&pmucru CLK_PCIEPHY2_REF>; + assigned-clock-rates = <100000000>; + resets = <&cru SRST_P_PIPEPHY2>, <&cru SRST_PIPEPHY2>; + reset-names = "combphy-apb", "combphy"; + rockchip,pipe-grf = <&pipegrf>; + rockchip,pipe-phy-grf = <&pipe_phy_grf2>; + status = "disabled"; + }; + + video_phy0: phy@fe850000 { + compatible = "rockchip,rk3568-dsi-dphy", "rockchip,rk3568-video-phy"; + reg = <0x0 0xfe850000 0x0 0x10000>, + <0x0 0xfe060000 0x0 0x10000>; + reg-names = "phy", "host"; + clocks = <&pmucru CLK_MIPIDSIPHY0_REF>, + <&cru PCLK_MIPIDSIPHY0>, <&cru PCLK_DSITX_0>; + clock-names = "ref", "pclk", "pclk_host"; + #clock-cells = <0>; + resets = <&cru SRST_P_MIPIDSIPHY0>; + reset-names = "apb"; + power-domains = <&power RK3568_PD_VO>; + #phy-cells = <0>; + status = "disabled"; + }; + + video_phy1: phy@fe860000 { + compatible = "rockchip,rk3568-dsi-dphy", "rockchip,rk3568-video-phy"; + reg = <0x0 0xfe860000 0x0 0x10000>, + <0x0 0xfe070000 0x0 0x10000>; + reg-names = "phy", "host"; + clocks = <&pmucru CLK_MIPIDSIPHY1_REF>, + <&cru PCLK_MIPIDSIPHY1>, <&cru PCLK_DSITX_1>; + clock-names = "ref", "pclk", "pclk_host"; + #clock-cells = <0>; + resets = <&cru SRST_P_MIPIDSIPHY1>; + reset-names = "apb"; + power-domains = <&power RK3568_PD_VO>; + #phy-cells = <0>; + status = "disabled"; + }; + + csi2_dphy_hw: csi2-dphy-hw@fe870000 { + compatible = "rockchip,rk3568-csi2-dphy-hw"; + reg = <0x0 0xfe870000 0x0 0x1000>; + clocks = <&cru PCLK_MIPICSIPHY>; + clock-names = "pclk"; + rockchip,grf = <&grf>; + status = "disabled"; + }; + + /* + * csi2_dphy0: used for csi2 dphy full mode, + is mutually exclusive with + csi2_dphy1 and csi2_dphy2 + * csi2_dphy1: used for csi2 dphy split mode, + physical lanes use lane0 and lane1, + can be used with csi2_dphy2 parallel + * csi2_dphy2: used for csi2 dphy split mode, + physical lanes use lane2 and lane3, + can be used with csi2_dphy1 parallel + */ + csi2_dphy0: csi2-dphy0 { + compatible = "rockchip,rk3568-csi2-dphy"; + rockchip,hw = <&csi2_dphy_hw>; + status = "disabled"; + }; + + csi2_dphy1: csi2-dphy1 { + compatible = "rockchip,rk3568-csi2-dphy"; + rockchip,hw = <&csi2_dphy_hw>; + status = "disabled"; + }; + + csi2_dphy2: csi2-dphy2 { + compatible = "rockchip,rk3568-csi2-dphy"; + rockchip,hw = <&csi2_dphy_hw>; + status = "disabled"; + }; + + usb2phy0: usb2-phy@fe8a0000 { + compatible = "rockchip,rk3568-usb2phy"; + reg = <0x0 0xfe8a0000 0x0 0x10000>; + interrupts = ; + clocks = <&pmucru CLK_USBPHY0_REF>; + clock-names = "phyclk"; + #clock-cells = <0>; + assigned-clocks = <&cru USB480M>; + assigned-clock-parents = <&usb2phy0>; + clock-output-names = "usb480m_phy"; + rockchip,usbgrf = <&usb2phy0_grf>; + status = "disabled"; + + u2phy0_host: host-port { + #phy-cells = <0>; + status = "disabled"; + }; + + u2phy0_otg: otg-port { + #phy-cells = <0>; + status = "disabled"; + }; + }; + + usb2phy1: usb2-phy@fe8b0000 { + compatible = "rockchip,rk3568-usb2phy"; + reg = <0x0 0xfe8b0000 0x0 0x10000>; + interrupts = ; + clocks = <&pmucru CLK_USBPHY1_REF>; + clock-names = "phyclk"; + #clock-cells = <0>; + rockchip,usbgrf = <&usb2phy1_grf>; + status = "disabled"; + + u2phy1_host: host-port { + #phy-cells = <0>; + status = "disabled"; + }; + + u2phy1_otg: otg-port { + #phy-cells = <0>; + status = "disabled"; + }; + }; + + pcie30phy: phy@fe8c0000 { + compatible = "rockchip,rk3568-pcie3-phy"; + reg = <0x0 0xfe8c0000 0x0 0x20000>; + #phy-cells = <0>; + clocks = <&pmucru CLK_PCIE30PHY_REF_M>, <&pmucru CLK_PCIE30PHY_REF_N>, + <&cru PCLK_PCIE30PHY>; + clock-names = "refclk_m", "refclk_n", "pclk"; + resets = <&cru SRST_PCIE30PHY>; + reset-names = "phy"; + rockchip,phy-grf = <&pcie30_phy_grf>; + status = "disabled"; + }; + + pinctrl: pinctrl { + compatible = "rockchip,rk3568-pinctrl"; + rockchip,grf = <&grf>; + rockchip,pmu = <&pmugrf>; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + gpio0: gpio0@fdd60000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xfdd60000 0x0 0x100>; + interrupts = ; + clocks = <&pmucru PCLK_GPIO0>, <&pmucru DBCLK_GPIO0>; + + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio1: gpio1@fe740000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xfe740000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO1>, <&cru DBCLK_GPIO1>; + + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio2: gpio2@fe750000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xfe750000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO2>, <&cru DBCLK_GPIO2>; + + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio3: gpio3@fe760000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xfe760000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO3>, <&cru DBCLK_GPIO3>; + + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio4: gpio4@fe770000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xfe770000 0x0 0x100>; + interrupts = ; + clocks = <&cru PCLK_GPIO4>, <&cru DBCLK_GPIO4>; + + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; +}; + +#include "rk3568-pinctrl.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rockchip-pinconf.dtsi b/arch/arm64/boot/dts/rockchip/rockchip-pinconf.dtsi new file mode 100755 index 000000000000..fc0145333257 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rockchip-pinconf.dtsi @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020~2021 Rockchip Electronics Co., Ltd. + */ + +&pinctrl { + + /omit-if-no-ref/ + pcfg_pull_up: pcfg-pull-up { + bias-pull-up; + }; + + /omit-if-no-ref/ + pcfg_pull_down: pcfg-pull-down { + bias-pull-down; + }; + + /omit-if-no-ref/ + pcfg_pull_none: pcfg-pull-none { + bias-disable; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_0: pcfg-pull-none-drv-level-0 { + bias-disable; + drive-strength = <0>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_1: pcfg-pull-none-drv-level-1 { + bias-disable; + drive-strength = <1>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_2: pcfg-pull-none-drv-level-2 { + bias-disable; + drive-strength = <2>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_3: pcfg-pull-none-drv-level-3 { + bias-disable; + drive-strength = <3>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_4: pcfg-pull-none-drv-level-4 { + bias-disable; + drive-strength = <4>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_5: pcfg-pull-none-drv-level-5 { + bias-disable; + drive-strength = <5>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_6: pcfg-pull-none-drv-level-6 { + bias-disable; + drive-strength = <6>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_7: pcfg-pull-none-drv-level-7 { + bias-disable; + drive-strength = <7>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_8: pcfg-pull-none-drv-level-8 { + bias-disable; + drive-strength = <8>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_9: pcfg-pull-none-drv-level-9 { + bias-disable; + drive-strength = <9>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_10: pcfg-pull-none-drv-level-10 { + bias-disable; + drive-strength = <10>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_11: pcfg-pull-none-drv-level-11 { + bias-disable; + drive-strength = <11>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_12: pcfg-pull-none-drv-level-12 { + bias-disable; + drive-strength = <12>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_13: pcfg-pull-none-drv-level-13 { + bias-disable; + drive-strength = <13>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_14: pcfg-pull-none-drv-level-14 { + bias-disable; + drive-strength = <14>; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_15: pcfg-pull-none-drv-level-15 { + bias-disable; + drive-strength = <15>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_0: pcfg-pull-up-drv-level-0 { + bias-pull-up; + drive-strength = <0>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_1: pcfg-pull-up-drv-level-1 { + bias-pull-up; + drive-strength = <1>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_2: pcfg-pull-up-drv-level-2 { + bias-pull-up; + drive-strength = <2>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_3: pcfg-pull-up-drv-level-3 { + bias-pull-up; + drive-strength = <3>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_4: pcfg-pull-up-drv-level-4 { + bias-pull-up; + drive-strength = <4>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_5: pcfg-pull-up-drv-level-5 { + bias-pull-up; + drive-strength = <5>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_6: pcfg-pull-up-drv-level-6 { + bias-pull-up; + drive-strength = <6>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_7: pcfg-pull-up-drv-level-7 { + bias-pull-up; + drive-strength = <7>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_8: pcfg-pull-up-drv-level-8 { + bias-pull-up; + drive-strength = <8>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_9: pcfg-pull-up-drv-level-9 { + bias-pull-up; + drive-strength = <9>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_10: pcfg-pull-up-drv-level-10 { + bias-pull-up; + drive-strength = <10>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_11: pcfg-pull-up-drv-level-11 { + bias-pull-up; + drive-strength = <11>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_12: pcfg-pull-up-drv-level-12 { + bias-pull-up; + drive-strength = <12>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_13: pcfg-pull-up-drv-level-13 { + bias-pull-up; + drive-strength = <13>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_14: pcfg-pull-up-drv-level-14 { + bias-pull-up; + drive-strength = <14>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_drv_level_15: pcfg-pull-up-drv-level-15 { + bias-pull-up; + drive-strength = <15>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_0: pcfg-pull-down-drv-level-0 { + bias-pull-down; + drive-strength = <0>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_1: pcfg-pull-down-drv-level-1 { + bias-pull-down; + drive-strength = <1>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_2: pcfg-pull-down-drv-level-2 { + bias-pull-down; + drive-strength = <2>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_3: pcfg-pull-down-drv-level-3 { + bias-pull-down; + drive-strength = <3>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_4: pcfg-pull-down-drv-level-4 { + bias-pull-down; + drive-strength = <4>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_5: pcfg-pull-down-drv-level-5 { + bias-pull-down; + drive-strength = <5>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_6: pcfg-pull-down-drv-level-6 { + bias-pull-down; + drive-strength = <6>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_7: pcfg-pull-down-drv-level-7 { + bias-pull-down; + drive-strength = <7>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_8: pcfg-pull-down-drv-level-8 { + bias-pull-down; + drive-strength = <8>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_9: pcfg-pull-down-drv-level-9 { + bias-pull-down; + drive-strength = <9>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_10: pcfg-pull-down-drv-level-10 { + bias-pull-down; + drive-strength = <10>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_11: pcfg-pull-down-drv-level-11 { + bias-pull-down; + drive-strength = <11>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_12: pcfg-pull-down-drv-level-12 { + bias-pull-down; + drive-strength = <12>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_13: pcfg-pull-down-drv-level-13 { + bias-pull-down; + drive-strength = <13>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_14: pcfg-pull-down-drv-level-14 { + bias-pull-down; + drive-strength = <14>; + }; + + /omit-if-no-ref/ + pcfg_pull_down_drv_level_15: pcfg-pull-down-drv-level-15 { + bias-pull-down; + drive-strength = <15>; + }; + + /omit-if-no-ref/ + pcfg_pull_up_smt: pcfg-pull-up-smt { + bias-pull-up; + input-schmitt-enable; + }; + + /omit-if-no-ref/ + pcfg_pull_down_smt: pcfg-pull-down-smt { + bias-pull-down; + input-schmitt-enable; + }; + + /omit-if-no-ref/ + pcfg_pull_none_smt: pcfg-pull-none-smt { + bias-disable; + input-schmitt-enable; + }; + + /omit-if-no-ref/ + pcfg_pull_none_drv_level_0_smt: pcfg-pull-none-drv-level-0-smt { + bias-disable; + drive-strength = <0>; + input-schmitt-enable; + }; + + /omit-if-no-ref/ + pcfg_output_high: pcfg-output-high { + output-high; + }; + + /omit-if-no-ref/ + pcfg_output_high_pull_up: pcfg-output-high-pull-up { + output-high; + bias-pull-up; + }; + + /omit-if-no-ref/ + pcfg_output_high_pull_down: pcfg-output-high-pull-down { + output-high; + bias-pull-down; + }; + + /omit-if-no-ref/ + pcfg_output_high_pull_none: pcfg-output-high-pull-none { + output-high; + bias-disable; + }; + + /omit-if-no-ref/ + pcfg_output_low: pcfg-output-low { + output-low; + }; + + /omit-if-no-ref/ + pcfg_output_low_pull_up: pcfg-output-low-pull-up { + output-low; + bias-pull-up; + }; + + /omit-if-no-ref/ + pcfg_output_low_pull_down: pcfg-output-low-pull-down { + output-low; + bias-pull-down; + }; + + /omit-if-no-ref/ + pcfg_output_low_pull_none: pcfg-output-low-pull-none { + output-low; + bias-disable; + }; +}; + diff --git a/arch/arm64/include/asm/system_info.h b/arch/arm64/include/asm/system_info.h new file mode 100755 index 000000000000..a82fe791b2f6 --- /dev/null +++ b/arch/arm64/include/asm/system_info.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_ARM_SYSTEM_INFO_H +#define __ASM_ARM_SYSTEM_INFO_H + +#ifndef __ASSEMBLY__ + +/* information about the system we're running on */ +extern unsigned int system_rev; +extern unsigned int system_serial_low; +extern unsigned int system_serial_high; + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_ARM_SYSTEM_INFO_H */ diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 4c0e72781f31..7221da6cf9ef 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -25,6 +25,12 @@ #include #include +unsigned int system_serial_low; +EXPORT_SYMBOL(system_serial_low); + +unsigned int system_serial_high; +EXPORT_SYMBOL(system_serial_high); + /* * In case the boot CPU is hotpluggable, we record its initial state and * current state separately. Certain system registers may contain different diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 3696dbcbfa80..ead9c33ebe91 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -191,6 +191,8 @@ void machine_restart(char *cmd) local_irq_disable(); smp_send_stop(); + do_kernel_pre_restart(cmd); + /* * UpdateCapsule() depends on the system being reset via * ResetSystem(). diff --git a/drivers/Kconfig b/drivers/Kconfig index 216c52363bd2..8fe7763236c3 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -246,4 +246,7 @@ source "drivers/hck/Kconfig" source "drivers/auth_ctl/Kconfig" +source "drivers/rkflash/Kconfig" + +source "drivers/rk_nand/Kconfig" endmenu diff --git a/drivers/Makefile b/drivers/Makefile index d0ff4fdb0dfb..888700ee86f5 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -196,3 +196,5 @@ obj-$(CONFIG_MOST) += most/ obj-$(CONFIG_ACCESS_TOKENID) += accesstokenid/ obj-$(CONFIG_VENDOR_HOOKS) += hooks/ obj-$(CONFIG_HCK_VENDOR_HOOKS) += hck/ +obj-$(CONFIG_RK_FLASH) += rkflash/ +obj-$(CONFIG_RK_NAND) += rk_nand/ diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index ecdf16bb130e..3dd80678499c 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -2400,12 +2400,6 @@ static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info) } dev_list = nla_nest_start_noflag(reply, NBD_ATTR_DEVICE_LIST); - if (!dev_list) { - nlmsg_free(reply); - ret = -EMSGSIZE; - goto out; - } - if (index == -1) { ret = idr_for_each(&nbd_index_idr, &status_cb, reply); if (ret) { diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index c715d4681a0b..42bb63d80971 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -38,6 +38,13 @@ menuconfig COMMON_CLK if COMMON_CLK +config COMMON_CLK_PROCFS + bool "Common Clock PROCFS interface" + depends on COMMON_CLK && PROC_FS && ARCH_ROCKCHIP + default n + help + Turns on the PROCFS interface for clock. + config COMMON_CLK_WM831X tristate "Clock driver for WM831x/2x PMICs" depends on MFD_WM831X diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 3575afe16a57..d2f0c049b837 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -1324,7 +1324,7 @@ static int __init clk_disable_unused(void) return 0; } -late_initcall_sync(clk_disable_unused); +//late_initcall_sync(clk_disable_unused); static int clk_core_determine_round_nolock(struct clk_core *core, struct clk_rate_request *req) diff --git a/drivers/clk/rockchip/Kconfig b/drivers/clk/rockchip/Kconfig index 47cd6c5de837..02c5df791017 100644 --- a/drivers/clk/rockchip/Kconfig +++ b/drivers/clk/rockchip/Kconfig @@ -2,7 +2,7 @@ # common clock support for ROCKCHIP SoC family. config COMMON_CLK_ROCKCHIP - bool "Rockchip clock controller common support" + tristate "Rockchip clock controller common support" depends on ARCH_ROCKCHIP default ARCH_ROCKCHIP help @@ -10,69 +10,105 @@ config COMMON_CLK_ROCKCHIP if COMMON_CLK_ROCKCHIP config CLK_PX30 - bool "Rockchip PX30 clock controller support" + tristate "Rockchip PX30 clock controller support" + depends on ARM64 || COMPILE_TEST default y help Build the driver for PX30 Clock Driver. config CLK_RV110X - bool "Rockchip RV110x clock controller support" + tristate "Rockchip RV110x clock controller support" + depends on ARM || COMPILE_TEST default y help Build the driver for RV110x Clock Driver. +config CLK_RV1126 + tristate "Rockchip RV1126 clock controller support" + depends on ARM || COMPILE_TEST + default y + help + Build the driver for RV1126 Clock Driver. + +config CLK_RK1808 + tristate "Rockchip RK1808 clock controller support" + depends on ARM64 || COMPILE_TEST + default y + help + Build the driver for RK1808 Clock Driver. + config CLK_RK3036 - bool "Rockchip RK3036 clock controller support" + tristate "Rockchip RK3036 clock controller support" + depends on ARM || COMPILE_TEST default y help Build the driver for RK3036 Clock Driver. config CLK_RK312X - bool "Rockchip RK312x clock controller support" + tristate "Rockchip RK312x clock controller support" + depends on ARM || COMPILE_TEST default y help Build the driver for RK312x Clock Driver. config CLK_RK3188 - bool "Rockchip RK3188 clock controller support" + tristate "Rockchip RK3188 clock controller support" + depends on ARM || COMPILE_TEST default y help Build the driver for RK3188 Clock Driver. config CLK_RK322X - bool "Rockchip RK322x clock controller support" + tristate "Rockchip RK322x clock controller support" + depends on ARM || COMPILE_TEST default y help Build the driver for RK322x Clock Driver. config CLK_RK3288 - bool "Rockchip RK3288 clock controller support" - depends on ARM + tristate "Rockchip RK3288 clock controller support" + depends on ARM || COMPILE_TEST default y help Build the driver for RK3288 Clock Driver. config CLK_RK3308 - bool "Rockchip RK3308 clock controller support" + tristate "Rockchip RK3308 clock controller support" + depends on ARM64 || COMPILE_TEST default y help Build the driver for RK3308 Clock Driver. config CLK_RK3328 - bool "Rockchip RK3328 clock controller support" + tristate "Rockchip RK3328 clock controller support" + depends on ARM64 || COMPILE_TEST default y help Build the driver for RK3328 Clock Driver. config CLK_RK3368 - bool "Rockchip RK3368 clock controller support" + tristate "Rockchip RK3368 clock controller support" + depends on ARM64 || COMPILE_TEST default y help Build the driver for RK3368 Clock Driver. config CLK_RK3399 tristate "Rockchip RK3399 clock controller support" + depends on ARM64 || COMPILE_TEST default y help Build the driver for RK3399 Clock Driver. + +config CLK_RK3568 + tristate "Rockchip RK3568 clock controller support" + depends on ARM64 || COMPILE_TEST + default y + help + Build the driver for RK3568 Clock Driver. + +config ROCKCHIP_CLK_COMPENSATION + bool "Rockchip Clk Compensation" + help + Say y here to enable clk compensation(+/- 1000 ppm). endif diff --git a/drivers/clk/rockchip/Makefile b/drivers/clk/rockchip/Makefile index a99e4d9bbae1..a4c718bf1126 100644 --- a/drivers/clk/rockchip/Makefile +++ b/drivers/clk/rockchip/Makefile @@ -13,10 +13,14 @@ clk-rockchip-y += clk-inverter.o clk-rockchip-y += clk-mmc-phase.o clk-rockchip-y += clk-muxgrf.o clk-rockchip-y += clk-ddr.o +clk-rockchip-y += clk-dclk-divider.o +clk-rockchip-y += clk-pvtm.o clk-rockchip-$(CONFIG_RESET_CONTROLLER) += softrst.o obj-$(CONFIG_CLK_PX30) += clk-px30.o obj-$(CONFIG_CLK_RV110X) += clk-rv1108.o +obj-$(CONFIG_CLK_RV1126) += clk-rv1126.o +obj-$(CONFIG_CLK_RK1808) += clk-rk1808.o obj-$(CONFIG_CLK_RK3036) += clk-rk3036.o obj-$(CONFIG_CLK_RK312X) += clk-rk3128.o obj-$(CONFIG_CLK_RK3188) += clk-rk3188.o @@ -26,3 +30,4 @@ obj-$(CONFIG_CLK_RK3308) += clk-rk3308.o obj-$(CONFIG_CLK_RK3328) += clk-rk3328.o obj-$(CONFIG_CLK_RK3368) += clk-rk3368.o obj-$(CONFIG_CLK_RK3399) += clk-rk3399.o +obj-$(CONFIG_CLK_RK3568) += clk-rk3568.o diff --git a/drivers/clk/rockchip/clk-cpu.c b/drivers/clk/rockchip/clk-cpu.c index 0dc478a19451..55416812bed2 100644 --- a/drivers/clk/rockchip/clk-cpu.c +++ b/drivers/clk/rockchip/clk-cpu.c @@ -51,6 +51,7 @@ */ struct rockchip_cpuclk { struct clk_hw hw; + struct clk_hw *pll_hw; struct clk_mux cpu_mux; const struct clk_ops *cpu_mux_ops; @@ -88,10 +89,10 @@ static unsigned long rockchip_cpuclk_recalc_rate(struct clk_hw *hw, { struct rockchip_cpuclk *cpuclk = to_rockchip_cpuclk_hw(hw); const struct rockchip_cpuclk_reg_data *reg_data = cpuclk->reg_data; - u32 clksel0 = readl_relaxed(cpuclk->reg_base + reg_data->core_reg); + u32 clksel0 = readl_relaxed(cpuclk->reg_base + reg_data->core_reg[0]); - clksel0 >>= reg_data->div_core_shift; - clksel0 &= reg_data->div_core_mask; + clksel0 >>= reg_data->div_core_shift[0]; + clksel0 &= reg_data->div_core_mask[0]; return parent_rate / (clksel0 + 1); } @@ -124,6 +125,7 @@ static int rockchip_cpuclk_pre_rate_change(struct rockchip_cpuclk *cpuclk, const struct rockchip_cpuclk_rate_table *rate; unsigned long alt_prate, alt_div; unsigned long flags; + int i = 0; /* check validity of the new rate */ rate = rockchip_get_cpuclk_settings(cpuclk, ndata->new_rate); @@ -133,6 +135,8 @@ static int rockchip_cpuclk_pre_rate_change(struct rockchip_cpuclk *cpuclk, return -EINVAL; } + rockchip_boost_enable_recovery_sw_low(cpuclk->pll_hw); + alt_prate = clk_get_rate(cpuclk->alt_parent); spin_lock_irqsave(cpuclk->lock, flags); @@ -146,10 +150,10 @@ static int rockchip_cpuclk_pre_rate_change(struct rockchip_cpuclk *cpuclk, if (alt_prate > ndata->old_rate) { /* calculate dividers */ alt_div = DIV_ROUND_UP(alt_prate, ndata->old_rate) - 1; - if (alt_div > reg_data->div_core_mask) { + if (alt_div > reg_data->div_core_mask[0]) { pr_warn("%s: limiting alt-divider %lu to %d\n", - __func__, alt_div, reg_data->div_core_mask); - alt_div = reg_data->div_core_mask; + __func__, alt_div, reg_data->div_core_mask[0]); + alt_div = reg_data->div_core_mask[0]; } /* @@ -162,20 +166,21 @@ static int rockchip_cpuclk_pre_rate_change(struct rockchip_cpuclk *cpuclk, pr_debug("%s: setting div %lu as alt-rate %lu > old-rate %lu\n", __func__, alt_div, alt_prate, ndata->old_rate); - writel(HIWORD_UPDATE(alt_div, reg_data->div_core_mask, - reg_data->div_core_shift) | - HIWORD_UPDATE(reg_data->mux_core_alt, - reg_data->mux_core_mask, - reg_data->mux_core_shift), - cpuclk->reg_base + reg_data->core_reg); - } else { - /* select alternate parent */ - writel(HIWORD_UPDATE(reg_data->mux_core_alt, - reg_data->mux_core_mask, - reg_data->mux_core_shift), - cpuclk->reg_base + reg_data->core_reg); + for (i = 0; i < reg_data->num_cores; i++) { + writel(HIWORD_UPDATE(alt_div, reg_data->div_core_mask[i], + reg_data->div_core_shift[i]), + cpuclk->reg_base + reg_data->core_reg[i]); + } } + rockchip_boost_add_core_div(cpuclk->pll_hw, alt_prate); + + /* select alternate parent */ + writel(HIWORD_UPDATE(reg_data->mux_core_alt, + reg_data->mux_core_mask, + reg_data->mux_core_shift), + cpuclk->reg_base + reg_data->core_reg[0]); + spin_unlock_irqrestore(cpuclk->lock, flags); return 0; } @@ -186,6 +191,7 @@ static int rockchip_cpuclk_post_rate_change(struct rockchip_cpuclk *cpuclk, const struct rockchip_cpuclk_reg_data *reg_data = cpuclk->reg_data; const struct rockchip_cpuclk_rate_table *rate; unsigned long flags; + int i = 0; rate = rockchip_get_cpuclk_settings(cpuclk, ndata->new_rate); if (!rate) { @@ -206,16 +212,23 @@ static int rockchip_cpuclk_post_rate_change(struct rockchip_cpuclk *cpuclk, * primary parent by the extra dividers that were needed for the alt. */ - writel(HIWORD_UPDATE(0, reg_data->div_core_mask, - reg_data->div_core_shift) | - HIWORD_UPDATE(reg_data->mux_core_main, - reg_data->mux_core_mask, - reg_data->mux_core_shift), - cpuclk->reg_base + reg_data->core_reg); + writel(HIWORD_UPDATE(reg_data->mux_core_main, + reg_data->mux_core_mask, + reg_data->mux_core_shift), + cpuclk->reg_base + reg_data->core_reg[0]); + + /* remove dividers */ + for (i = 0; i < reg_data->num_cores; i++) { + writel(HIWORD_UPDATE(0, reg_data->div_core_mask[i], + reg_data->div_core_shift[i]), + cpuclk->reg_base + reg_data->core_reg[i]); + } if (ndata->old_rate > ndata->new_rate) rockchip_cpuclk_set_dividers(cpuclk, rate); + rockchip_boost_disable_recovery_sw(cpuclk->pll_hw); + spin_unlock_irqrestore(cpuclk->lock, flags); return 0; } @@ -244,14 +257,16 @@ static int rockchip_cpuclk_notifier_cb(struct notifier_block *nb, } struct clk *rockchip_clk_register_cpuclk(const char *name, - const char *const *parent_names, u8 num_parents, + u8 num_parents, + struct clk *parent, struct clk *alt_parent, const struct rockchip_cpuclk_reg_data *reg_data, const struct rockchip_cpuclk_rate_table *rates, int nrates, void __iomem *reg_base, spinlock_t *lock) { struct rockchip_cpuclk *cpuclk; struct clk_init_data init; - struct clk *clk, *cclk; + struct clk *clk, *cclk, *pll_clk; + const char *parent_name; int ret; if (num_parents < 2) { @@ -259,12 +274,18 @@ struct clk *rockchip_clk_register_cpuclk(const char *name, return ERR_PTR(-EINVAL); } + if (IS_ERR(parent) || IS_ERR(alt_parent)) { + pr_err("%s: invalid parent clock(s)\n", __func__); + return ERR_PTR(-EINVAL); + } + cpuclk = kzalloc(sizeof(*cpuclk), GFP_KERNEL); if (!cpuclk) return ERR_PTR(-ENOMEM); + parent_name = clk_hw_get_name(__clk_get_hw(parent)); init.name = name; - init.parent_names = &parent_names[reg_data->mux_core_main]; + init.parent_names = &parent_name; init.num_parents = 1; init.ops = &rockchip_cpuclk_ops; @@ -281,8 +302,19 @@ struct clk *rockchip_clk_register_cpuclk(const char *name, cpuclk->reg_data = reg_data; cpuclk->clk_nb.notifier_call = rockchip_cpuclk_notifier_cb; cpuclk->hw.init = &init; + if (reg_data->pll_name) { + pll_clk = clk_get_parent(parent); + if (!pll_clk) { + pr_err("%s: could not lookup pll clock: (%s)\n", + __func__, reg_data->pll_name); + ret = -EINVAL; + goto free_cpuclk; + } + cpuclk->pll_hw = __clk_get_hw(pll_clk); + rockchip_boost_init(cpuclk->pll_hw); + } - cpuclk->alt_parent = __clk_lookup(parent_names[reg_data->mux_core_alt]); + cpuclk->alt_parent = alt_parent; if (!cpuclk->alt_parent) { pr_err("%s: could not lookup alternate parent: (%d)\n", __func__, reg_data->mux_core_alt); @@ -297,11 +329,11 @@ struct clk *rockchip_clk_register_cpuclk(const char *name, goto free_cpuclk; } - clk = __clk_lookup(parent_names[reg_data->mux_core_main]); + clk = parent; if (!clk) { pr_err("%s: could not lookup parent clock: (%d) %s\n", __func__, reg_data->mux_core_main, - parent_names[reg_data->mux_core_main]); + parent_name); ret = -EINVAL; goto free_alt_parent; } diff --git a/drivers/clk/rockchip/clk-dclk-divider.c b/drivers/clk/rockchip/clk-dclk-divider.c new file mode 100755 index 000000000000..77c35b42207e --- /dev/null +++ b/drivers/clk/rockchip/clk-dclk-divider.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include +#include +#include +#include +#include +#include "clk.h" + +#define div_mask(width) ((1 << (width)) - 1) + +static unsigned long clk_dclk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_divider *divider = to_clk_divider(hw); + unsigned int val; + + val = readl(divider->reg) >> divider->shift; + val &= div_mask(divider->width); + + return DIV_ROUND_UP_ULL(((u64)parent_rate), val + 1); +} + +static long clk_dclk_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *prate) +{ + struct clk_divider *divider = to_clk_divider(hw); + int div, maxdiv = div_mask(divider->width) + 1; + + div = DIV_ROUND_UP_ULL(divider->max_prate, rate); + if (div % 2) + div = __rounddown_pow_of_two(div); + div = div > maxdiv ? maxdiv : div; + *prate = div * rate; + return rate; +} + +static int clk_dclk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_divider *divider = to_clk_divider(hw); + unsigned int value; + unsigned long flags = 0; + u32 val; + + value = divider_get_val(rate, parent_rate, divider->table, + divider->width, divider->flags); + + if (divider->lock) + spin_lock_irqsave(divider->lock, flags); + else + __acquire(divider->lock); + + if (divider->flags & CLK_DIVIDER_HIWORD_MASK) { + val = div_mask(divider->width) << (divider->shift + 16); + } else { + val = readl(divider->reg); + val &= ~(div_mask(divider->width) << divider->shift); + } + val |= value << divider->shift; + writel(val, divider->reg); + + if (divider->lock) + spin_unlock_irqrestore(divider->lock, flags); + else + __release(divider->lock); + + return 0; +} + +const struct clk_ops clk_dclk_divider_ops = { + .recalc_rate = clk_dclk_recalc_rate, + .round_rate = clk_dclk_round_rate, + .set_rate = clk_dclk_set_rate, +}; +EXPORT_SYMBOL_GPL(clk_dclk_divider_ops); + +/** + * Register a clock branch. + * Most clock branches have a form like + * + * src1 --|--\ + * |M |--[GATE]-[DIV]- + * src2 --|--/ + * + * sometimes without one of those components. + */ +struct clk *rockchip_clk_register_dclk_branch(const char *name, + const char *const *parent_names, + u8 num_parents, + void __iomem *base, + int muxdiv_offset, u8 mux_shift, + u8 mux_width, u8 mux_flags, + int div_offset, u8 div_shift, + u8 div_width, u8 div_flags, + struct clk_div_table *div_table, + int gate_offset, + u8 gate_shift, u8 gate_flags, + unsigned long flags, + unsigned long max_prate, + spinlock_t *lock) +{ + struct clk *clk; + struct clk_mux *mux = NULL; + struct clk_gate *gate = NULL; + struct clk_divider *div = NULL; + const struct clk_ops *mux_ops = NULL, *div_ops = NULL, + *gate_ops = NULL; + + if (num_parents > 1) { + mux = kzalloc(sizeof(*mux), GFP_KERNEL); + if (!mux) + return ERR_PTR(-ENOMEM); + + mux->reg = base + muxdiv_offset; + mux->shift = mux_shift; + mux->mask = BIT(mux_width) - 1; + mux->flags = mux_flags; + mux->lock = lock; + mux_ops = (mux_flags & CLK_MUX_READ_ONLY) ? &clk_mux_ro_ops + : &clk_mux_ops; + } + + if (gate_offset >= 0) { + gate = kzalloc(sizeof(*gate), GFP_KERNEL); + if (!gate) + goto err_gate; + + gate->flags = gate_flags; + gate->reg = base + gate_offset; + gate->bit_idx = gate_shift; + gate->lock = lock; + gate_ops = &clk_gate_ops; + } + + if (div_width > 0) { + div = kzalloc(sizeof(*div), GFP_KERNEL); + if (!div) + goto err_div; + + div->flags = div_flags; + if (div_offset) + div->reg = base + div_offset; + else + div->reg = base + muxdiv_offset; + div->shift = div_shift; + div->width = div_width; + div->lock = lock; + div->max_prate = max_prate; + div_ops = &clk_dclk_divider_ops; + } + + clk = clk_register_composite(NULL, name, parent_names, num_parents, + mux ? &mux->hw : NULL, mux_ops, + div ? &div->hw : NULL, div_ops, + gate ? &gate->hw : NULL, gate_ops, + flags); + + return clk; +err_div: + kfree(gate); +err_gate: + kfree(mux); + return ERR_PTR(-ENOMEM); +} diff --git a/drivers/clk/rockchip/clk-ddr.c b/drivers/clk/rockchip/clk-ddr.c index 86718c54e56b..3c8bcbee2048 100644 --- a/drivers/clk/rockchip/clk-ddr.c +++ b/drivers/clk/rockchip/clk-ddr.c @@ -8,10 +8,20 @@ #include #include #include +#include +#include #include #include +#include +#include +#ifdef CONFIG_ARM +#include +#endif + #include "clk.h" +#define MHZ (1000000) + struct rockchip_ddrclk { struct clk_hw hw; void __iomem *reg_base; @@ -21,25 +31,47 @@ struct rockchip_ddrclk { int div_shift; int div_width; int ddr_flag; - spinlock_t *lock; }; #define to_rockchip_ddrclk_hw(hw) container_of(hw, struct rockchip_ddrclk, hw) +struct share_params_ddrclk { + u32 hz; + u32 lcdc_type; +}; + +struct rockchip_ddrclk_data { + void __iomem *params; + int (*dmcfreq_wait_complete)(void); +}; + +static struct rockchip_ddrclk_data ddr_data = {NULL, NULL}; + +void rockchip_set_ddrclk_params(void __iomem *params) +{ + ddr_data.params = params; +} +EXPORT_SYMBOL(rockchip_set_ddrclk_params); + +void rockchip_set_ddrclk_dmcfreq_wait_complete(int (*func)(void)) +{ + ddr_data.dmcfreq_wait_complete = func; +} +EXPORT_SYMBOL(rockchip_set_ddrclk_dmcfreq_wait_complete); + static int rockchip_ddrclk_sip_set_rate(struct clk_hw *hw, unsigned long drate, unsigned long prate) { - struct rockchip_ddrclk *ddrclk = to_rockchip_ddrclk_hw(hw); - unsigned long flags; struct arm_smccc_res res; - spin_lock_irqsave(ddrclk->lock, flags); arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, drate, 0, ROCKCHIP_SIP_CONFIG_DRAM_SET_RATE, 0, 0, 0, 0, &res); - spin_unlock_irqrestore(ddrclk->lock, flags); - return res.a0; + if (res.a0) + return 0; + else + return -EPERM; } static unsigned long @@ -87,18 +119,134 @@ static const struct clk_ops rockchip_ddrclk_sip_ops = { .get_parent = rockchip_ddrclk_get_parent, }; +static u32 ddr_clk_cached; + +static int rockchip_ddrclk_scpi_set_rate(struct clk_hw *hw, unsigned long drate, + unsigned long prate) +{ + u32 ret; + u32 lcdc_type = 0; + struct share_params_ddrclk *p; + + p = (struct share_params_ddrclk *)ddr_data.params; + if (p) + lcdc_type = p->lcdc_type; + + ret = scpi_ddr_set_clk_rate(drate / MHZ, lcdc_type); + if (ret) { + ddr_clk_cached = ret; + ret = 0; + } else { + ddr_clk_cached = 0; + ret = -1; + } + + return ret; +} + +static unsigned long rockchip_ddrclk_scpi_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + if (ddr_clk_cached) + return (MHZ * ddr_clk_cached); + else + return (MHZ * scpi_ddr_get_clk_rate()); +} + +static long rockchip_ddrclk_scpi_round_rate(struct clk_hw *hw, + unsigned long rate, + unsigned long *prate) +{ + rate = rate / MHZ; + rate = (rate / 12) * 12; + + return (rate * MHZ); +} + +static const struct clk_ops rockchip_ddrclk_scpi_ops = { + .recalc_rate = rockchip_ddrclk_scpi_recalc_rate, + .set_rate = rockchip_ddrclk_scpi_set_rate, + .round_rate = rockchip_ddrclk_scpi_round_rate, + .get_parent = rockchip_ddrclk_get_parent, +}; + +static int rockchip_ddrclk_sip_set_rate_v2(struct clk_hw *hw, + unsigned long drate, + unsigned long prate) +{ + struct share_params_ddrclk *p; + struct arm_smccc_res res; + + p = (struct share_params_ddrclk *)ddr_data.params; + if (p) + p->hz = drate; + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_SET_RATE); + + if ((int)res.a1 == SIP_RET_SET_RATE_TIMEOUT) { + if (ddr_data.dmcfreq_wait_complete) + ddr_data.dmcfreq_wait_complete(); + } + + return res.a0; +} + +static unsigned long rockchip_ddrclk_sip_recalc_rate_v2 + (struct clk_hw *hw, unsigned long parent_rate) +{ + struct arm_smccc_res res; + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_GET_RATE); + if (!res.a0) + return res.a1; + else + return 0; +} + +static long rockchip_ddrclk_sip_round_rate_v2(struct clk_hw *hw, + unsigned long rate, + unsigned long *prate) +{ + struct share_params_ddrclk *p; + struct arm_smccc_res res; + + p = (struct share_params_ddrclk *)ddr_data.params; + if (p) + p->hz = rate; + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_ROUND_RATE); + if (!res.a0) + return res.a1; + else + return 0; +} + +static const struct clk_ops rockchip_ddrclk_sip_ops_v2 = { + .recalc_rate = rockchip_ddrclk_sip_recalc_rate_v2, + .set_rate = rockchip_ddrclk_sip_set_rate_v2, + .round_rate = rockchip_ddrclk_sip_round_rate_v2, + .get_parent = rockchip_ddrclk_get_parent, +}; + struct clk *rockchip_clk_register_ddrclk(const char *name, int flags, const char *const *parent_names, u8 num_parents, int mux_offset, int mux_shift, int mux_width, int div_shift, int div_width, - int ddr_flag, void __iomem *reg_base, - spinlock_t *lock) + int ddr_flag, void __iomem *reg_base) { struct rockchip_ddrclk *ddrclk; struct clk_init_data init; struct clk *clk; +#ifdef CONFIG_ARM + if (!psci_smp_available()) + return NULL; +#endif + ddrclk = kzalloc(sizeof(*ddrclk), GFP_KERNEL); if (!ddrclk) return ERR_PTR(-ENOMEM); @@ -114,6 +262,12 @@ struct clk *rockchip_clk_register_ddrclk(const char *name, int flags, case ROCKCHIP_DDRCLK_SIP: init.ops = &rockchip_ddrclk_sip_ops; break; + case ROCKCHIP_DDRCLK_SCPI: + init.ops = &rockchip_ddrclk_scpi_ops; + break; + case ROCKCHIP_DDRCLK_SIP_V2: + init.ops = &rockchip_ddrclk_sip_ops_v2; + break; default: pr_err("%s: unsupported ddrclk type %d\n", __func__, ddr_flag); kfree(ddrclk); @@ -121,7 +275,6 @@ struct clk *rockchip_clk_register_ddrclk(const char *name, int flags, } ddrclk->reg_base = reg_base; - ddrclk->lock = lock; ddrclk->hw.init = &init; ddrclk->mux_offset = mux_offset; ddrclk->mux_shift = mux_shift; diff --git a/drivers/clk/rockchip/clk-half-divider.c b/drivers/clk/rockchip/clk-half-divider.c index ccd5c270c213..b978af08d84f 100644 --- a/drivers/clk/rockchip/clk-half-divider.c +++ b/drivers/clk/rockchip/clk-half-divider.c @@ -14,9 +14,9 @@ static bool _is_best_half_div(unsigned long rate, unsigned long now, unsigned long best, unsigned long flags) { if (flags & CLK_DIVIDER_ROUND_CLOSEST) - return abs(rate - now) < abs(rate - best); + return abs(rate - now) <= abs(rate - best); - return now <= rate && now > best; + return now <= rate && now >= best; } static unsigned long clk_half_divider_recalc_rate(struct clk_hw *hw, @@ -38,7 +38,7 @@ static int clk_half_divider_bestdiv(struct clk_hw *hw, unsigned long rate, { unsigned int i, bestdiv = 0; unsigned long parent_rate, best = 0, now, maxdiv; - unsigned long parent_rate_saved = *best_parent_rate; + bool is_bestdiv = false; if (!rate) rate = 1; @@ -51,7 +51,7 @@ static int clk_half_divider_bestdiv(struct clk_hw *hw, unsigned long rate, if (bestdiv < 3) bestdiv = 0; else - bestdiv = (bestdiv - 3) / 2; + bestdiv = DIV_ROUND_UP(bestdiv - 3, 2); bestdiv = bestdiv > maxdiv ? maxdiv : bestdiv; return bestdiv; } @@ -63,28 +63,20 @@ static int clk_half_divider_bestdiv(struct clk_hw *hw, unsigned long rate, maxdiv = min(ULONG_MAX / rate, maxdiv); for (i = 0; i <= maxdiv; i++) { - if (((u64)rate * (i * 2 + 3)) == ((u64)parent_rate_saved * 2)) { - /* - * It's the most ideal case if the requested rate can be - * divided from parent clock without needing to change - * parent rate, so return the divider immediately. - */ - *best_parent_rate = parent_rate_saved; - return i; - } parent_rate = clk_hw_round_rate(clk_hw_get_parent(hw), ((u64)rate * (i * 2 + 3)) / 2); now = DIV_ROUND_UP_ULL(((u64)parent_rate * 2), (i * 2 + 3)); if (_is_best_half_div(rate, now, best, flags)) { + is_bestdiv = true; bestdiv = i; best = now; *best_parent_rate = parent_rate; } } - if (!bestdiv) { + if (!is_bestdiv) { bestdiv = div_mask(width); *best_parent_rate = clk_hw_round_rate(clk_hw_get_parent(hw), 1); } @@ -114,7 +106,7 @@ static int clk_half_divider_set_rate(struct clk_hw *hw, unsigned long rate, u32 val; value = DIV_ROUND_UP_ULL(((u64)parent_rate * 2), rate); - value = (value - 3) / 2; + value = DIV_ROUND_UP(value - 3, 2); value = min_t(unsigned int, value, div_mask(divider->width)); if (divider->lock) @@ -160,10 +152,10 @@ struct clk *rockchip_clk_register_halfdiv(const char *name, u8 num_parents, void __iomem *base, int muxdiv_offset, u8 mux_shift, u8 mux_width, u8 mux_flags, - u8 div_shift, u8 div_width, - u8 div_flags, int gate_offset, - u8 gate_shift, u8 gate_flags, - unsigned long flags, + int div_offset, u8 div_shift, + u8 div_width, u8 div_flags, + int gate_offset, u8 gate_shift, + u8 gate_flags, unsigned long flags, spinlock_t *lock) { struct clk_hw *hw = ERR_PTR(-ENOMEM); @@ -205,7 +197,10 @@ struct clk *rockchip_clk_register_halfdiv(const char *name, goto err_div; div->flags = div_flags; - div->reg = base + muxdiv_offset; + if (div_offset) + div->reg = base + div_offset; + else + div->reg = base + muxdiv_offset; div->shift = div_shift; div->width = div_width; div->lock = lock; diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c index d0bd513ff3c3..c7e28d95f8a0 100644 --- a/drivers/clk/rockchip/clk-pll.c +++ b/drivers/clk/rockchip/clk-pll.c @@ -15,6 +15,9 @@ #include #include #include +#include +#include +#include #include "clk.h" #define PLL_MODE_MASK 0x3 @@ -38,15 +41,291 @@ struct rockchip_clk_pll { u8 flags; const struct rockchip_pll_rate_table *rate_table; unsigned int rate_count; + int sel; + unsigned long scaling; spinlock_t *lock; struct rockchip_clk_provider *ctx; + + bool boost_enabled; + u32 boost_backup_pll_usage; + unsigned long boost_backup_pll_rate; + unsigned long boost_low_rate; + unsigned long boost_high_rate; + struct regmap *boost; +#ifdef CONFIG_DEBUG_FS + struct hlist_node debug_node; +#endif }; #define to_rockchip_clk_pll(_hw) container_of(_hw, struct rockchip_clk_pll, hw) #define to_rockchip_clk_pll_nb(nb) \ container_of(nb, struct rockchip_clk_pll, clk_nb) +static void rockchip_boost_disable_low(struct rockchip_clk_pll *pll); + +#define MHZ (1000UL * 1000UL) +#define KHZ (1000UL) + +/* CLK_PLL_TYPE_RK3066_AUTO type ops */ +#define PLL_FREF_MIN (269 * KHZ) +#define PLL_FREF_MAX (2200 * MHZ) + +#define PLL_FVCO_MIN (440 * MHZ) +#define PLL_FVCO_MAX (2200 * MHZ) + +#define PLL_FOUT_MIN (27500 * KHZ) +#define PLL_FOUT_MAX (2200 * MHZ) + +#define PLL_NF_MAX (4096) +#define PLL_NR_MAX (64) +#define PLL_NO_MAX (16) + +/* CLK_PLL_TYPE_RK3036/3366/3399_AUTO type ops */ +#define MIN_FOUTVCO_FREQ (800 * MHZ) +#define MAX_FOUTVCO_FREQ (2000 * MHZ) + +static struct rockchip_pll_rate_table auto_table; +#ifdef CONFIG_DEBUG_FS +static HLIST_HEAD(clk_boost_list); +static DEFINE_MUTEX(clk_boost_lock); +#endif + +int rockchip_pll_clk_adaptive_scaling(struct clk *clk, int sel) +{ + struct clk *parent = clk_get_parent(clk); + struct rockchip_clk_pll *pll; + + if (IS_ERR_OR_NULL(parent)) + return -EINVAL; + + pll = to_rockchip_clk_pll(__clk_get_hw(parent)); + if (!pll) + return -EINVAL; + + pll->sel = sel; + + return 0; +} +EXPORT_SYMBOL(rockchip_pll_clk_adaptive_scaling); + +int rockchip_pll_clk_rate_to_scale(struct clk *clk, unsigned long rate) +{ + const struct rockchip_pll_rate_table *rate_table; + struct clk *parent = clk_get_parent(clk); + struct rockchip_clk_pll *pll; + unsigned int i; + + if (IS_ERR_OR_NULL(parent)) + return -EINVAL; + + pll = to_rockchip_clk_pll(__clk_get_hw(parent)); + if (!pll) + return -EINVAL; + + rate_table = pll->rate_table; + for (i = 0; i < pll->rate_count; i++) { + if (rate >= rate_table[i].rate) + return i; + } + + return -EINVAL; +} +EXPORT_SYMBOL(rockchip_pll_clk_rate_to_scale); + +int rockchip_pll_clk_scale_to_rate(struct clk *clk, unsigned int scale) +{ + const struct rockchip_pll_rate_table *rate_table; + struct clk *parent = clk_get_parent(clk); + struct rockchip_clk_pll *pll; + unsigned int i; + + if (IS_ERR_OR_NULL(parent)) + return -EINVAL; + + pll = to_rockchip_clk_pll(__clk_get_hw(parent)); + if (!pll) + return -EINVAL; + + rate_table = pll->rate_table; + for (i = 0; i < pll->rate_count; i++) { + if (i == scale) + return rate_table[i].rate; + } + + return -EINVAL; +} +EXPORT_SYMBOL(rockchip_pll_clk_scale_to_rate); + +static struct rockchip_pll_rate_table *rk_pll_rate_table_get(void) +{ + return &auto_table; +} + +static int rockchip_pll_clk_set_postdiv(unsigned long fout_hz, + u32 *postdiv1, + u32 *postdiv2, + u32 *foutvco) +{ + unsigned long freq; + + if (fout_hz < MIN_FOUTVCO_FREQ) { + for (*postdiv1 = 1; *postdiv1 <= 7; (*postdiv1)++) { + for (*postdiv2 = 1; *postdiv2 <= 7; (*postdiv2)++) { + freq = fout_hz * (*postdiv1) * (*postdiv2); + if (freq >= MIN_FOUTVCO_FREQ && + freq <= MAX_FOUTVCO_FREQ) { + *foutvco = freq; + return 0; + } + } + } + pr_err("CANNOT FIND postdiv1/2 to make fout in range from 800M to 2000M,fout = %lu\n", + fout_hz); + } else { + *postdiv1 = 1; + *postdiv2 = 1; + } + return 0; +} + +static struct rockchip_pll_rate_table * +rockchip_pll_clk_set_by_auto(struct rockchip_clk_pll *pll, + unsigned long fin_hz, + unsigned long fout_hz) +{ + struct rockchip_pll_rate_table *rate_table = rk_pll_rate_table_get(); + /* FIXME set postdiv1/2 always 1*/ + u32 foutvco = fout_hz; + u64 fin_64, frac_64; + u32 f_frac, postdiv1, postdiv2; + unsigned long clk_gcd = 0; + + if (fin_hz == 0 || fout_hz == 0 || fout_hz == fin_hz) + return NULL; + + rockchip_pll_clk_set_postdiv(fout_hz, &postdiv1, &postdiv2, &foutvco); + rate_table->postdiv1 = postdiv1; + rate_table->postdiv2 = postdiv2; + rate_table->dsmpd = 1; + + if (fin_hz / MHZ * MHZ == fin_hz && fout_hz / MHZ * MHZ == fout_hz) { + fin_hz /= MHZ; + foutvco /= MHZ; + clk_gcd = gcd(fin_hz, foutvco); + rate_table->refdiv = fin_hz / clk_gcd; + rate_table->fbdiv = foutvco / clk_gcd; + + rate_table->frac = 0; + + pr_debug("fin = %lu, fout = %lu, clk_gcd = %lu, refdiv = %u, fbdiv = %u, postdiv1 = %u, postdiv2 = %u, frac = %u\n", + fin_hz, fout_hz, clk_gcd, rate_table->refdiv, + rate_table->fbdiv, rate_table->postdiv1, + rate_table->postdiv2, rate_table->frac); + } else { + pr_debug("frac div running, fin_hz = %lu, fout_hz = %lu, fin_INT_mhz = %lu, fout_INT_mhz = %lu\n", + fin_hz, fout_hz, + fin_hz / MHZ * MHZ, + fout_hz / MHZ * MHZ); + pr_debug("frac get postdiv1 = %u, postdiv2 = %u, foutvco = %u\n", + rate_table->postdiv1, rate_table->postdiv2, foutvco); + clk_gcd = gcd(fin_hz / MHZ, foutvco / MHZ); + rate_table->refdiv = fin_hz / MHZ / clk_gcd; + rate_table->fbdiv = foutvco / MHZ / clk_gcd; + pr_debug("frac get refdiv = %u, fbdiv = %u\n", + rate_table->refdiv, rate_table->fbdiv); + + rate_table->frac = 0; + + f_frac = (foutvco % MHZ); + fin_64 = fin_hz; + do_div(fin_64, (u64)rate_table->refdiv); + frac_64 = (u64)f_frac << 24; + do_div(frac_64, fin_64); + rate_table->frac = (u32)frac_64; + if (rate_table->frac > 0) + rate_table->dsmpd = 0; + pr_debug("frac = %x\n", rate_table->frac); + } + return rate_table; +} + +static struct rockchip_pll_rate_table * +rockchip_rk3066_pll_clk_set_by_auto(struct rockchip_clk_pll *pll, + unsigned long fin_hz, + unsigned long fout_hz) +{ + struct rockchip_pll_rate_table *rate_table = rk_pll_rate_table_get(); + u32 nr, nf, no, nonr; + u32 nr_out, nf_out, no_out; + u32 n; + u32 numerator, denominator; + u64 fref, fvco, fout; + unsigned long clk_gcd = 0; + + nr_out = PLL_NR_MAX + 1; + no_out = 0; + nf_out = 0; + + if (fin_hz == 0 || fout_hz == 0 || fout_hz == fin_hz) + return NULL; + + clk_gcd = gcd(fin_hz, fout_hz); + + numerator = fout_hz / clk_gcd; + denominator = fin_hz / clk_gcd; + + for (n = 1;; n++) { + nf = numerator * n; + nonr = denominator * n; + if (nf > PLL_NF_MAX || nonr > (PLL_NO_MAX * PLL_NR_MAX)) + break; + + for (no = 1; no <= PLL_NO_MAX; no++) { + if (!(no == 1 || !(no % 2))) + continue; + + if (nonr % no) + continue; + nr = nonr / no; + + if (nr > PLL_NR_MAX) + continue; + + fref = fin_hz / nr; + if (fref < PLL_FREF_MIN || fref > PLL_FREF_MAX) + continue; + + fvco = fref * nf; + if (fvco < PLL_FVCO_MIN || fvco > PLL_FVCO_MAX) + continue; + + fout = fvco / no; + if (fout < PLL_FOUT_MIN || fout > PLL_FOUT_MAX) + continue; + + /* select the best from all available PLL settings */ + if ((no > no_out) || + ((no == no_out) && (nr < nr_out))) { + nr_out = nr; + nf_out = nf; + no_out = no; + } + } + } + + /* output the best PLL setting */ + if ((nr_out <= PLL_NR_MAX) && (no_out > 0)) { + rate_table->nr = nr_out; + rate_table->nf = nf_out; + rate_table->no = no_out; + } else { + return NULL; + } + + return rate_table; +} + static const struct rockchip_pll_rate_table *rockchip_get_pll_settings( struct rockchip_clk_pll *pll, unsigned long rate) { @@ -54,28 +333,27 @@ static const struct rockchip_pll_rate_table *rockchip_get_pll_settings( int i; for (i = 0; i < pll->rate_count; i++) { - if (rate == rate_table[i].rate) + if (rate == rate_table[i].rate) { + if (i < pll->sel) { + pll->scaling = rate; + return &rate_table[pll->sel]; + } + pll->scaling = 0; return &rate_table[i]; + } } + pll->scaling = 0; - return NULL; + if (pll->type == pll_rk3066) + return rockchip_rk3066_pll_clk_set_by_auto(pll, 24 * MHZ, rate); + else + return rockchip_pll_clk_set_by_auto(pll, 24 * MHZ, rate); } static long rockchip_pll_round_rate(struct clk_hw *hw, unsigned long drate, unsigned long *prate) { - struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); - const struct rockchip_pll_rate_table *rate_table = pll->rate_table; - int i; - - /* Assumming rate_table is in descending order */ - for (i = 0; i < pll->rate_count; i++) { - if (drate >= rate_table[i].rate) - return rate_table[i].rate; - } - - /* return minimum supported value */ - return rate_table[i - 1].rate; + return drate; } /* @@ -136,6 +414,30 @@ static int rockchip_rk3036_pll_wait_lock(struct rockchip_clk_pll *pll) return ret; } +static unsigned long +rockchip_rk3036_pll_con_to_rate(struct rockchip_clk_pll *pll, + u32 con0, u32 con1) +{ + unsigned int fbdiv, postdiv1, refdiv, postdiv2; + u64 rate64 = 24000000; + + fbdiv = ((con0 >> RK3036_PLLCON0_FBDIV_SHIFT) & + RK3036_PLLCON0_FBDIV_MASK); + postdiv1 = ((con0 >> RK3036_PLLCON0_POSTDIV1_SHIFT) & + RK3036_PLLCON0_POSTDIV1_MASK); + refdiv = ((con1 >> RK3036_PLLCON1_REFDIV_SHIFT) & + RK3036_PLLCON1_REFDIV_MASK); + postdiv2 = ((con1 >> RK3036_PLLCON1_POSTDIV2_SHIFT) & + RK3036_PLLCON1_POSTDIV2_MASK); + + rate64 *= fbdiv; + do_div(rate64, refdiv); + do_div(rate64, postdiv1); + do_div(rate64, postdiv2); + + return (unsigned long)rate64; +} + static void rockchip_rk3036_pll_get_params(struct rockchip_clk_pll *pll, struct rockchip_pll_rate_table *rate) { @@ -165,7 +467,10 @@ static unsigned long rockchip_rk3036_pll_recalc_rate(struct clk_hw *hw, { struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); struct rockchip_pll_rate_table cur; - u64 rate64 = prate; + u64 rate64 = prate, frac_rate64 = prate; + + if (pll->sel && pll->scaling) + return pll->scaling; rockchip_rk3036_pll_get_params(pll, &cur); @@ -174,7 +479,7 @@ static unsigned long rockchip_rk3036_pll_recalc_rate(struct clk_hw *hw, if (cur.dsmpd == 0) { /* fractional mode */ - u64 frac_rate64 = prate * cur.frac; + frac_rate64 *= cur.frac; do_div(frac_rate64, cur.refdiv); rate64 += frac_rate64 >> 24; @@ -231,6 +536,8 @@ static int rockchip_rk3036_pll_set_params(struct rockchip_clk_pll *pll, pllcon |= rate->frac << RK3036_PLLCON2_FRAC_SHIFT; writel_relaxed(pllcon, pll->reg_base + RK3036_PLLCON(2)); + rockchip_boost_disable_low(pll); + /* wait for the pll to lock */ ret = rockchip_rk3036_pll_wait_lock(pll); if (ret) { @@ -412,6 +719,9 @@ static unsigned long rockchip_rk3066_pll_recalc_rate(struct clk_hw *hw, return prate; } + if (pll->sel && pll->scaling) + return pll->scaling; + rockchip_rk3066_pll_get_params(pll, &cur); rate64 *= cur.nf; @@ -485,9 +795,18 @@ static int rockchip_rk3066_pll_set_rate(struct clk_hw *hw, unsigned long drate, { struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); const struct rockchip_pll_rate_table *rate; + unsigned long old_rate = rockchip_rk3066_pll_recalc_rate(hw, prate); + struct regmap *grf = pll->ctx->grf; + int ret; - pr_debug("%s: changing %s to %lu with a parent rate of %lu\n", - __func__, clk_hw_get_name(hw), drate, prate); + if (IS_ERR(grf)) { + pr_debug("%s: grf regmap not available, aborting rate change\n", + __func__); + return PTR_ERR(grf); + } + + pr_debug("%s: changing %s from %lu to %lu with a parent rate of %lu\n", + __func__, clk_hw_get_name(hw), old_rate, drate, prate); /* Get required rate settings from table */ rate = rockchip_get_pll_settings(pll, drate); @@ -497,7 +816,11 @@ static int rockchip_rk3066_pll_set_rate(struct clk_hw *hw, unsigned long drate, return -EINVAL; } - return rockchip_rk3066_pll_set_params(pll, rate); + ret = rockchip_rk3066_pll_set_params(pll, rate); + if (ret) + pll->scaling = 0; + + return ret; } static int rockchip_rk3066_pll_enable(struct clk_hw *hw) @@ -649,6 +972,9 @@ static unsigned long rockchip_rk3399_pll_recalc_rate(struct clk_hw *hw, struct rockchip_pll_rate_table cur; u64 rate64 = prate; + if (pll->sel && pll->scaling) + return pll->scaling; + rockchip_rk3399_pll_get_params(pll, &cur); rate64 *= cur.fbdiv; @@ -692,6 +1018,11 @@ static int rockchip_rk3399_pll_set_params(struct rockchip_clk_pll *pll, rate_change_remuxed = 1; } + /* set pll power down */ + writel(HIWORD_UPDATE(RK3399_PLLCON3_PWRDOWN, + RK3399_PLLCON3_PWRDOWN, 0), + pll->reg_base + RK3399_PLLCON(3)); + /* update pll values */ writel_relaxed(HIWORD_UPDATE(rate->fbdiv, RK3399_PLLCON0_FBDIV_MASK, RK3399_PLLCON0_FBDIV_SHIFT), @@ -715,6 +1046,11 @@ static int rockchip_rk3399_pll_set_params(struct rockchip_clk_pll *pll, RK3399_PLLCON3_DSMPD_SHIFT), pll->reg_base + RK3399_PLLCON(3)); + /* set pll power up */ + writel(HIWORD_UPDATE(0, + RK3399_PLLCON3_PWRDOWN, 0), + pll->reg_base + RK3399_PLLCON(3)); + /* wait for the pll to lock */ ret = rockchip_rk3399_pll_wait_lock(pll); if (ret) { @@ -734,9 +1070,11 @@ static int rockchip_rk3399_pll_set_rate(struct clk_hw *hw, unsigned long drate, { struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); const struct rockchip_pll_rate_table *rate; + unsigned long old_rate = rockchip_rk3399_pll_recalc_rate(hw, prate); + int ret; - pr_debug("%s: changing %s to %lu with a parent rate of %lu\n", - __func__, __clk_get_name(hw->clk), drate, prate); + pr_debug("%s: changing %s from %lu to %lu with a parent rate of %lu\n", + __func__, __clk_get_name(hw->clk), old_rate, drate, prate); /* Get required rate settings from table */ rate = rockchip_get_pll_settings(pll, drate); @@ -746,7 +1084,11 @@ static int rockchip_rk3399_pll_set_rate(struct clk_hw *hw, unsigned long drate, return -EINVAL; } - return rockchip_rk3399_pll_set_params(pll, rate); + ret = rockchip_rk3399_pll_set_params(pll, rate); + if (ret) + pll->scaling = 0; + + return ret; } static int rockchip_rk3399_pll_enable(struct clk_hw *hw) @@ -842,6 +1184,80 @@ static const struct clk_ops rockchip_rk3399_pll_clk_ops = { .init = rockchip_rk3399_pll_init, }; +#ifdef CONFIG_ROCKCHIP_CLK_COMPENSATION +int rockchip_pll_clk_compensation(struct clk *clk, int ppm) +{ + struct clk *parent = clk_get_parent(clk); + struct rockchip_clk_pll *pll; + static u32 frac, fbdiv; + bool negative; + u32 pllcon, pllcon0, pllcon2, fbdiv_mask, frac_mask, frac_shift; + u64 fracdiv, m, n; + + if ((ppm > 1000) || (ppm < -1000)) + return -EINVAL; + + if (IS_ERR_OR_NULL(parent)) + return -EINVAL; + + pll = to_rockchip_clk_pll(__clk_get_hw(parent)); + if (!pll) + return -EINVAL; + + switch (pll->type) { + case pll_rk3036: + case pll_rk3328: + pllcon0 = RK3036_PLLCON(0); + pllcon2 = RK3036_PLLCON(2); + fbdiv_mask = RK3036_PLLCON0_FBDIV_MASK; + frac_mask = RK3036_PLLCON2_FRAC_MASK; + frac_shift = RK3036_PLLCON2_FRAC_SHIFT; + break; + case pll_rk3066: + return -EINVAL; + case pll_rk3399: + pllcon0 = RK3399_PLLCON(0); + pllcon2 = RK3399_PLLCON(2); + fbdiv_mask = RK3399_PLLCON0_FBDIV_MASK; + frac_mask = RK3399_PLLCON2_FRAC_MASK; + frac_shift = RK3399_PLLCON2_FRAC_SHIFT; + break; + default: + return -EINVAL; + } + + negative = !!(ppm & BIT(31)); + ppm = negative ? ~ppm + 1 : ppm; + + if (!frac) { + frac = readl_relaxed(pll->reg_base + pllcon2) & frac_mask; + fbdiv = readl_relaxed(pll->reg_base + pllcon0) & fbdiv_mask; + } + + /* + * delta frac frac ppm + * -------------- = (fbdiv + ----------) * --------- + * 1 << 24 1 << 24 1000000 + * + */ + m = div64_u64((uint64_t)frac * ppm, 1000000); + n = div64_u64((uint64_t)ppm << 24, 1000000) * fbdiv; + + fracdiv = negative ? frac - (m + n) : frac + (m + n); + + if (!frac || fracdiv > frac_mask) + return -EINVAL; + + pllcon = readl_relaxed(pll->reg_base + pllcon2); + pllcon &= ~(frac_mask << frac_shift); + pllcon |= fracdiv << frac_shift; + writel_relaxed(pllcon, pll->reg_base + pllcon2); + + return 0; +} +EXPORT_SYMBOL(rockchip_pll_clk_compensation); +#endif + /* * Common registering of pll clocks */ @@ -914,8 +1330,12 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx, /* now create the actual pll */ init.name = pll_name; +#ifndef CONFIG_ROCKCHIP_LOW_PERFORMANCE /* keep all plls untouched for now */ init.flags = flags | CLK_IGNORE_UNUSED; +#else + init.flags = flags; +#endif init.parent_names = &parent_names[0]; init.num_parents = 1; @@ -940,7 +1360,7 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx, switch (pll_type) { case pll_rk3036: case pll_rk3328: - if (!pll->rate_table) + if (!pll->rate_table || IS_ERR(ctx->grf)) init.ops = &rockchip_rk3036_pll_clk_norate_ops; else init.ops = &rockchip_rk3036_pll_clk_ops; @@ -988,3 +1408,316 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx, kfree(pll); return mux_clk; } + +static unsigned long rockchip_pll_con_to_rate(struct rockchip_clk_pll *pll, + u32 con0, u32 con1) +{ + switch (pll->type) { + case pll_rk3036: + case pll_rk3328: + return rockchip_rk3036_pll_con_to_rate(pll, con0, con1); + case pll_rk3066: + break; + case pll_rk3399: + break; + default: + pr_warn("%s: Unknown pll type\n", __func__); + } + + return 0; +} + +void rockchip_boost_init(struct clk_hw *hw) +{ + struct rockchip_clk_pll *pll; + struct device_node *np; + u32 value, con0, con1; + + if (!hw) + return; + pll = to_rockchip_clk_pll(hw); + np = of_parse_phandle(pll->ctx->cru_node, "rockchip,boost", 0); + if (!np) { + pr_debug("%s: failed to get boost np\n", __func__); + return; + } + pll->boost = syscon_node_to_regmap(np); + if (IS_ERR(pll->boost)) { + pr_debug("%s: failed to get boost regmap\n", __func__); + return; + } + + if (!of_property_read_u32(np, "rockchip,boost-low-con0", &con0) && + !of_property_read_u32(np, "rockchip,boost-low-con1", &con1)) { + pr_debug("boost-low-con=0x%x 0x%x\n", con0, con1); + regmap_write(pll->boost, BOOST_PLL_L_CON(0), + HIWORD_UPDATE(con0, BOOST_PLL_CON_MASK, 0)); + regmap_write(pll->boost, BOOST_PLL_L_CON(1), + HIWORD_UPDATE(con1, BOOST_PLL_CON_MASK, 0)); + pll->boost_low_rate = rockchip_pll_con_to_rate(pll, con0, + con1); + pr_debug("boost-low-rate=%lu\n", pll->boost_low_rate); + } + if (!of_property_read_u32(np, "rockchip,boost-high-con0", &con0) && + !of_property_read_u32(np, "rockchip,boost-high-con1", &con1)) { + pr_debug("boost-high-con=0x%x 0x%x\n", con0, con1); + regmap_write(pll->boost, BOOST_PLL_H_CON(0), + HIWORD_UPDATE(con0, BOOST_PLL_CON_MASK, 0)); + regmap_write(pll->boost, BOOST_PLL_H_CON(1), + HIWORD_UPDATE(con1, BOOST_PLL_CON_MASK, 0)); + pll->boost_high_rate = rockchip_pll_con_to_rate(pll, con0, + con1); + pr_debug("boost-high-rate=%lu\n", pll->boost_high_rate); + } + if (!of_property_read_u32(np, "rockchip,boost-backup-pll", &value)) { + pr_debug("boost-backup-pll=0x%x\n", value); + regmap_write(pll->boost, BOOST_CLK_CON, + HIWORD_UPDATE(value, BOOST_BACKUP_PLL_MASK, + BOOST_BACKUP_PLL_SHIFT)); + } + if (!of_property_read_u32(np, "rockchip,boost-backup-pll-usage", + &pll->boost_backup_pll_usage)) { + pr_debug("boost-backup-pll-usage=0x%x\n", + pll->boost_backup_pll_usage); + regmap_write(pll->boost, BOOST_CLK_CON, + HIWORD_UPDATE(pll->boost_backup_pll_usage, + BOOST_BACKUP_PLL_USAGE_MASK, + BOOST_BACKUP_PLL_USAGE_SHIFT)); + } + if (!of_property_read_u32(np, "rockchip,boost-switch-threshold", + &value)) { + pr_debug("boost-switch-threshold=0x%x\n", value); + regmap_write(pll->boost, BOOST_SWITCH_THRESHOLD, value); + } + if (!of_property_read_u32(np, "rockchip,boost-statis-threshold", + &value)) { + pr_debug("boost-statis-threshold=0x%x\n", value); + regmap_write(pll->boost, BOOST_STATIS_THRESHOLD, value); + } + if (!of_property_read_u32(np, "rockchip,boost-statis-enable", + &value)) { + pr_debug("boost-statis-enable=0x%x\n", value); + regmap_write(pll->boost, BOOST_BOOST_CON, + HIWORD_UPDATE(value, BOOST_STATIS_ENABLE_MASK, + BOOST_STATIS_ENABLE_SHIFT)); + } + if (!of_property_read_u32(np, "rockchip,boost-enable", &value)) { + pr_debug("boost-enable=0x%x\n", value); + regmap_write(pll->boost, BOOST_BOOST_CON, + HIWORD_UPDATE(value, BOOST_ENABLE_MASK, + BOOST_ENABLE_SHIFT)); + if (value) + pll->boost_enabled = true; + } +#ifdef CONFIG_DEBUG_FS + if (pll->boost_enabled) { + mutex_lock(&clk_boost_lock); + hlist_add_head(&pll->debug_node, &clk_boost_list); + mutex_unlock(&clk_boost_lock); + } +#endif +} + +void rockchip_boost_enable_recovery_sw_low(struct clk_hw *hw) +{ + struct rockchip_clk_pll *pll; + unsigned int val; + + if (!hw) + return; + pll = to_rockchip_clk_pll(hw); + if (!pll->boost_enabled) + return; + + regmap_write(pll->boost, BOOST_BOOST_CON, + HIWORD_UPDATE(1, BOOST_RECOVERY_MASK, + BOOST_RECOVERY_SHIFT)); + do { + regmap_read(pll->boost, BOOST_FSM_STATUS, &val); + } while (!(val & BOOST_BUSY_STATE)); + + regmap_write(pll->boost, BOOST_BOOST_CON, + HIWORD_UPDATE(1, BOOST_SW_CTRL_MASK, + BOOST_SW_CTRL_SHIFT) | + HIWORD_UPDATE(1, BOOST_LOW_FREQ_EN_MASK, + BOOST_LOW_FREQ_EN_SHIFT)); +} + +static void rockchip_boost_disable_low(struct rockchip_clk_pll *pll) +{ + if (!pll->boost_enabled) + return; + + regmap_write(pll->boost, BOOST_BOOST_CON, + HIWORD_UPDATE(0, BOOST_LOW_FREQ_EN_MASK, + BOOST_LOW_FREQ_EN_SHIFT)); +} + +void rockchip_boost_disable_recovery_sw(struct clk_hw *hw) +{ + struct rockchip_clk_pll *pll; + + if (!hw) + return; + pll = to_rockchip_clk_pll(hw); + if (!pll->boost_enabled) + return; + + regmap_write(pll->boost, BOOST_BOOST_CON, + HIWORD_UPDATE(0, BOOST_RECOVERY_MASK, + BOOST_RECOVERY_SHIFT)); + regmap_write(pll->boost, BOOST_BOOST_CON, + HIWORD_UPDATE(0, BOOST_SW_CTRL_MASK, + BOOST_SW_CTRL_SHIFT)); +} + +void rockchip_boost_add_core_div(struct clk_hw *hw, unsigned long prate) +{ + struct rockchip_clk_pll *pll; + unsigned int div; + + if (!hw) + return; + pll = to_rockchip_clk_pll(hw); + if (!pll->boost_enabled || pll->boost_backup_pll_rate == prate) + return; + + /* todo */ + if (pll->boost_backup_pll_usage == BOOST_BACKUP_PLL_USAGE_TARGET) + return; + /* + * cpu clock rate should be less than or equal to + * low rate when change pll rate in boost module + */ + if (pll->boost_low_rate && prate > pll->boost_low_rate) { + div = DIV_ROUND_UP(prate, pll->boost_low_rate) - 1; + regmap_write(pll->boost, BOOST_CLK_CON, + HIWORD_UPDATE(div, BOOST_CORE_DIV_MASK, + BOOST_CORE_DIV_SHIFT)); + pll->boost_backup_pll_rate = prate; + } +} + +#ifdef CONFIG_DEBUG_FS +#include + +#ifndef MODULE +static int boost_summary_show(struct seq_file *s, void *data) +{ + struct rockchip_clk_pll *pll = (struct rockchip_clk_pll *)s->private; + u32 boost_count = 0; + u32 freq_cnt0 = 0, freq_cnt1 = 0; + u64 freq_cnt = 0, high_freq_time = 0; + u32 short_count = 0, short_threshold = 0; + u32 interval_time = 0; + + seq_puts(s, " device boost_count high_freq_count high_freq_time short_count short_threshold interval_count\n"); + seq_puts(s, "------------------------------------------------------------------------------------------------------\n"); + seq_printf(s, " %s\n", clk_hw_get_name(&pll->hw)); + + regmap_read(pll->boost, BOOST_SWITCH_CNT, &boost_count); + + regmap_read(pll->boost, BOOST_HIGH_PERF_CNT0, &freq_cnt0); + regmap_read(pll->boost, BOOST_HIGH_PERF_CNT1, &freq_cnt1); + freq_cnt = ((u64)freq_cnt1 << 32) + (u64)freq_cnt0; + high_freq_time = freq_cnt; + do_div(high_freq_time, 24); + + regmap_read(pll->boost, BOOST_SHORT_SWITCH_CNT, &short_count); + regmap_read(pll->boost, BOOST_STATIS_THRESHOLD, &short_threshold); + regmap_read(pll->boost, BOOST_SWITCH_THRESHOLD, &interval_time); + + seq_printf(s, "%22u %17llu %15llu %12u %16u %15u\n", + boost_count, freq_cnt, high_freq_time, short_count, + short_threshold, interval_time); + + return 0; +} + +static int boost_summary_open(struct inode *inode, struct file *file) +{ + return single_open(file, boost_summary_show, inode->i_private); +} + +static const struct file_operations boost_summary_fops = { + .open = boost_summary_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int boost_config_show(struct seq_file *s, void *data) +{ + struct rockchip_clk_pll *pll = (struct rockchip_clk_pll *)s->private; + + seq_printf(s, "boost_enabled: %d\n", pll->boost_enabled); + seq_printf(s, "boost_low_rate: %lu\n", pll->boost_low_rate); + seq_printf(s, "boost_high_rate: %lu\n", pll->boost_high_rate); + + return 0; +} + +static int boost_config_open(struct inode *inode, struct file *file) +{ + return single_open(file, boost_config_show, inode->i_private); +} + +static const struct file_operations boost_config_fops = { + .open = boost_config_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int boost_debug_create_one(struct rockchip_clk_pll *pll, + struct dentry *rootdir) +{ + struct dentry *pdentry, *d; + + pdentry = debugfs_lookup(clk_hw_get_name(&pll->hw), rootdir); + if (!pdentry) { + pr_err("%s: failed to lookup %s dentry\n", __func__, + clk_hw_get_name(&pll->hw)); + return -ENOMEM; + } + + d = debugfs_create_file("boost_summary", 0444, pdentry, + pll, &boost_summary_fops); + if (!d) { + pr_err("%s: failed to create boost_summary file\n", __func__); + return -ENOMEM; + } + + d = debugfs_create_file("boost_config", 0444, pdentry, + pll, &boost_config_fops); + if (!d) { + pr_err("%s: failed to create boost config file\n", __func__); + return -ENOMEM; + } + + return 0; +} + +static int __init boost_debug_init(void) +{ + struct rockchip_clk_pll *pll; + struct dentry *rootdir; + + rootdir = debugfs_lookup("clk", NULL); + if (!rootdir) { + pr_err("%s: failed to lookup clk dentry\n", __func__); + return -ENOMEM; + } + + mutex_lock(&clk_boost_lock); + + hlist_for_each_entry(pll, &clk_boost_list, debug_node) + boost_debug_create_one(pll, rootdir); + + mutex_unlock(&clk_boost_lock); + + return 0; +} +late_initcall(boost_debug_init); +#endif /* MODULE */ +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/clk/rockchip/clk-pvtm.c b/drivers/clk/rockchip/clk-pvtm.c new file mode 100755 index 000000000000..ad02b6a571d8 --- /dev/null +++ b/drivers/clk/rockchip/clk-pvtm.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CLK_SEL_EXTERNAL_32K 0 +#define CLK_SEL_INTERNAL_PVTM 1 + +#define wr_msk_bit(v, off, msk) ((v) << (off) | (msk << (16 + (off)))) + +struct rockchip_clock_pvtm; + +struct rockchip_clock_pvtm_info { + u32 con; + u32 sta; + u32 sel_con; + u32 sel_shift; + u32 sel_value; + u32 sel_mask; + u32 div_shift; + u32 div_mask; + + u32 (*get_value)(struct rockchip_clock_pvtm *pvtm, + unsigned int time_us); + int (*init_freq)(struct rockchip_clock_pvtm *pvtm); + int (*sel_enable)(struct rockchip_clock_pvtm *pvtm); +}; + +struct rockchip_clock_pvtm { + const struct rockchip_clock_pvtm_info *info; + struct regmap *grf; + struct clk *pvtm_clk; + struct clk *clk; + unsigned long rate; +}; + +static unsigned long xin32k_pvtm_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + return 32768; +} + +static const struct clk_ops xin32k_pvtm = { + .recalc_rate = xin32k_pvtm_recalc_rate, +}; + +static void rockchip_clock_pvtm_delay(unsigned int delay) +{ + unsigned int ms = delay / 1000; + unsigned int us = delay % 1000; + + if (ms > 0) { + if (ms < 20) + us += ms * 1000; + else + msleep(ms); + } + + if (us >= 10) + usleep_range(us, us + 100); + else + udelay(us); +} + +static int rockchip_clock_sel_internal_pvtm(struct rockchip_clock_pvtm *pvtm) +{ + int ret = 0; + + ret = regmap_write(pvtm->grf, pvtm->info->sel_con, + wr_msk_bit(pvtm->info->sel_value, + pvtm->info->sel_shift, + pvtm->info->sel_mask)); + if (ret != 0) + pr_err("%s: fail to write register\n", __func__); + + return ret; +} + +/* get pmu pvtm value */ +static u32 rockchip_clock_pvtm_get_value(struct rockchip_clock_pvtm *pvtm, + u32 time_us) +{ + const struct rockchip_clock_pvtm_info *info = pvtm->info; + u32 val = 0, sta = 0; + u32 clk_cnt, check_cnt; + + /* 24m clk ,24cnt=1us */ + clk_cnt = time_us * 24; + + regmap_write(pvtm->grf, info->con + 0x4, clk_cnt); + regmap_write(pvtm->grf, info->con, wr_msk_bit(3, 0, 0x3)); + + rockchip_clock_pvtm_delay(time_us); + + check_cnt = 100; + while (check_cnt--) { + regmap_read(pvtm->grf, info->sta, &sta); + if (sta & 0x1) + break; + udelay(4); + } + + if (check_cnt) { + regmap_read(pvtm->grf, info->sta + 0x4, &val); + } else { + pr_err("%s: wait pvtm_done timeout!\n", __func__); + val = 0; + } + + regmap_write(pvtm->grf, info->con, wr_msk_bit(0, 0, 0x3)); + + return val; +} + +static int rockchip_clock_pvtm_init_freq(struct rockchip_clock_pvtm *pvtm) +{ + u32 pvtm_cnt = 0; + u32 div, time_us; + int ret = 0; + + time_us = 1000; + pvtm_cnt = pvtm->info->get_value(pvtm, time_us); + pr_debug("get pvtm_cnt = %d\n", pvtm_cnt); + + /* set pvtm_div to get rate */ + div = DIV_ROUND_UP(1000 * pvtm_cnt, pvtm->rate); + if (div > pvtm->info->div_mask) { + pr_err("pvtm_div out of bounary! set max instead\n"); + div = pvtm->info->div_mask; + } + + pr_debug("set div %d, rate %luKHZ\n", div, pvtm->rate); + ret = regmap_write(pvtm->grf, pvtm->info->con, + wr_msk_bit(div, pvtm->info->div_shift, + pvtm->info->div_mask)); + if (ret != 0) + goto out; + + /* pmu pvtm oscilator enable */ + ret = regmap_write(pvtm->grf, pvtm->info->con, + wr_msk_bit(1, 1, 0x1)); + if (ret != 0) + goto out; + + ret = pvtm->info->sel_enable(pvtm); +out: + if (ret != 0) + pr_err("%s: fail to write register\n", __func__); + + return ret; +} + +static int clock_pvtm_regitstor(struct device *dev, + struct rockchip_clock_pvtm *pvtm) +{ + struct clk_init_data init = {}; + struct clk_hw *clk_hw; + + /* Init the xin32k_pvtm */ + pvtm->info->init_freq(pvtm); + + init.parent_names = NULL; + init.num_parents = 0; + init.name = "xin32k_pvtm"; + init.ops = &xin32k_pvtm; + + clk_hw = devm_kzalloc(dev, sizeof(*clk_hw), GFP_KERNEL); + if (!clk_hw) + return -ENOMEM; + clk_hw->init = &init; + + /* optional override of the clockname */ + of_property_read_string_index(dev->of_node, "clock-output-names", + 0, &init.name); + pvtm->clk = devm_clk_register(dev, clk_hw); + if (IS_ERR(pvtm->clk)) + return PTR_ERR(pvtm->clk); + + return of_clk_add_provider(dev->of_node, of_clk_src_simple_get, + pvtm->clk); +} + +static const struct rockchip_clock_pvtm_info rk3368_pvtm_data = { + .con = 0x180, + .sta = 0x190, + .sel_con = 0x100, + .sel_shift = 6, + .sel_value = CLK_SEL_INTERNAL_PVTM, + .sel_mask = 0x1, + .div_shift = 2, + .div_mask = 0x3f, + + .sel_enable = rockchip_clock_sel_internal_pvtm, + .get_value = rockchip_clock_pvtm_get_value, + .init_freq = rockchip_clock_pvtm_init_freq, +}; + +static const struct of_device_id rockchip_clock_pvtm_match[] = { + { + .compatible = "rockchip,rk3368-pvtm-clock", + .data = (void *)&rk3368_pvtm_data, + }, + {} +}; +MODULE_DEVICE_TABLE(of, rockchip_clock_pvtm_match); + +static int rockchip_clock_pvtm_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + struct rockchip_clock_pvtm *pvtm; + int error; + u32 rate; + + pvtm = devm_kzalloc(dev, sizeof(*pvtm), GFP_KERNEL); + if (!pvtm) + return -ENOMEM; + + match = of_match_node(rockchip_clock_pvtm_match, np); + if (!match) + return -ENXIO; + + pvtm->info = (const struct rockchip_clock_pvtm_info *)match->data; + if (!pvtm->info) + return -EINVAL; + + if (!dev->parent || !dev->parent->of_node) + return -EINVAL; + + pvtm->grf = syscon_node_to_regmap(dev->parent->of_node); + if (IS_ERR(pvtm->grf)) + return PTR_ERR(pvtm->grf); + + if (!of_property_read_u32(np, "pvtm-rate", &rate)) + pvtm->rate = rate; + else + pvtm->rate = 32768; + + pvtm->pvtm_clk = devm_clk_get(&pdev->dev, "pvtm_pmu_clk"); + if (IS_ERR(pvtm->pvtm_clk)) { + error = PTR_ERR(pvtm->pvtm_clk); + if (error != -EPROBE_DEFER) + dev_err(&pdev->dev, + "failed to get pvtm core clock: %d\n", + error); + goto out_probe; + } + + error = clk_prepare_enable(pvtm->pvtm_clk); + if (error) { + dev_err(&pdev->dev, "failed to enable the clock: %d\n", + error); + goto out_probe; + } + + platform_set_drvdata(pdev, pvtm); + + error = clock_pvtm_regitstor(&pdev->dev, pvtm); + if (error) { + dev_err(&pdev->dev, "failed to registor clock: %d\n", + error); + goto out_clk_put; + } + + return error; + +out_clk_put: + clk_disable_unprepare(pvtm->pvtm_clk); +out_probe: + return error; +} + +static int rockchip_clock_pvtm_remove(struct platform_device *pdev) +{ + struct rockchip_clock_pvtm *pvtm = platform_get_drvdata(pdev); + struct device_node *np = pdev->dev.of_node; + + of_clk_del_provider(np); + clk_disable_unprepare(pvtm->pvtm_clk); + + return 0; +} + +static struct platform_driver rockchip_clock_pvtm_driver = { + .driver = { + .name = "rockchip-clcok-pvtm", + .of_match_table = rockchip_clock_pvtm_match, + }, + .probe = rockchip_clock_pvtm_probe, + .remove = rockchip_clock_pvtm_remove, +}; + +module_platform_driver(rockchip_clock_pvtm_driver); + +MODULE_DESCRIPTION("Rockchip Clock Pvtm Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/rockchip/clk-px30.c b/drivers/clk/rockchip/clk-px30.c index 6fb9c98b7d24..89ecbd4ed0c9 100644 --- a/drivers/clk/rockchip/clk-px30.c +++ b/drivers/clk/rockchip/clk-px30.c @@ -6,13 +6,16 @@ #include #include +#include #include #include +#include #include #include #include "clk.h" #define PX30_GRF_SOC_STATUS0 0x480 +#define PX30_FRAC_MAX_PRATE 600000000 enum px30_plls { apll, dpll, cpll, npll, apll_b_h, apll_b_l, @@ -124,29 +127,31 @@ static struct rockchip_cpuclk_rate_table px30_cpuclk_rates[] __initdata = { }; static const struct rockchip_cpuclk_reg_data px30_cpuclk_data = { - .core_reg = PX30_CLKSEL_CON(0), - .div_core_shift = 0, - .div_core_mask = 0xf, + .core_reg[0] = PX30_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0xf, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 0, .mux_core_shift = 7, .mux_core_mask = 0x1, + .pll_name = "pll_apll", }; PNAME(mux_pll_p) = { "xin24m"}; PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc32k_pmu" }; -PNAME(mux_armclk_p) = { "apll_core", "gpll_core" }; PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr" }; PNAME(mux_ddrstdby_p) = { "clk_ddrphy1x", "clk_stdby_2wrap" }; -PNAME(mux_4plls_p) = { "gpll", "dummy_cpll", "usb480m", "npll" }; +PNAME(mux_gpll_dmycpll_usb480m_npll_p) = { "gpll", "dummy_cpll", "usb480m", "npll" }; +PNAME(mux_gpll_dmycpll_usb480m_dmynpll_p) = { "gpll", "dummy_cpll", "usb480m", "dummy_npll" }; PNAME(mux_cpll_npll_p) = { "cpll", "npll" }; PNAME(mux_npll_cpll_p) = { "npll", "cpll" }; PNAME(mux_gpll_cpll_p) = { "gpll", "dummy_cpll" }; -PNAME(mux_gpll_npll_p) = { "gpll", "npll" }; +PNAME(mux_gpll_npll_p) = { "gpll", "dummy_npll" }; PNAME(mux_gpll_xin24m_p) = { "gpll", "xin24m"}; -PNAME(mux_gpll_cpll_npll_p) = { "gpll", "dummy_cpll", "npll" }; -PNAME(mux_gpll_cpll_npll_xin24m_p) = { "gpll", "dummy_cpll", "npll", "xin24m" }; -PNAME(mux_gpll_xin24m_npll_p) = { "gpll", "xin24m", "npll"}; +PNAME(mux_gpll_cpll_npll_p) = { "gpll", "dummy_cpll", "dummy_npll" }; +PNAME(mux_gpll_cpll_npll_xin24m_p) = { "gpll", "dummy_cpll", "dummy_npll", "xin24m" }; +PNAME(mux_gpll_xin24m_npll_p) = { "gpll", "xin24m", "dummy_npll"}; PNAME(mux_pdm_p) = { "clk_pdm_src", "clk_pdm_frac" }; PNAME(mux_i2s0_tx_p) = { "clk_i2s0_tx_src", "clk_i2s0_tx_frac", "mclk_i2s0_tx_in", "xin12m"}; PNAME(mux_i2s0_rx_p) = { "clk_i2s0_rx_src", "clk_i2s0_rx_frac", "mclk_i2s0_rx_in", "xin12m"}; @@ -158,13 +163,13 @@ PNAME(mux_i2s1_out_p) = { "clk_i2s1", "xin12m"}; PNAME(mux_i2s2_out_p) = { "clk_i2s2", "xin12m"}; PNAME(mux_i2s0_tx_rx_p) = { "clk_i2s0_tx_mux", "clk_i2s0_rx_mux"}; PNAME(mux_i2s0_rx_tx_p) = { "clk_i2s0_rx_mux", "clk_i2s0_tx_mux"}; -PNAME(mux_uart_src_p) = { "gpll", "xin24m", "usb480m", "npll" }; +PNAME(mux_uart_src_p) = { "gpll", "xin24m", "usb480m", "dummy_npll" }; PNAME(mux_uart1_p) = { "clk_uart1_src", "clk_uart1_np5", "clk_uart1_frac" }; PNAME(mux_uart2_p) = { "clk_uart2_src", "clk_uart2_np5", "clk_uart2_frac" }; PNAME(mux_uart3_p) = { "clk_uart3_src", "clk_uart3_np5", "clk_uart3_frac" }; PNAME(mux_uart4_p) = { "clk_uart4_src", "clk_uart4_np5", "clk_uart4_frac" }; PNAME(mux_uart5_p) = { "clk_uart5_src", "clk_uart5_np5", "clk_uart5_frac" }; -PNAME(mux_cif_out_p) = { "xin24m", "dummy_cpll", "npll", "usb480m" }; +PNAME(mux_cif_out_p) = { "xin24m", "dummy_cpll", "dummy_npll", "usb480m" }; PNAME(mux_dclk_vopb_p) = { "dclk_vopb_src", "dclk_vopb_frac", "xin24m" }; PNAME(mux_dclk_vopl_p) = { "dclk_vopl_src", "dclk_vopl_frac", "xin24m" }; PNAME(mux_nandc_p) = { "clk_nandc_div", "clk_nandc_div50" }; @@ -191,7 +196,7 @@ static struct rockchip_pll_clock px30_pll_clks[] __initdata = { 0, PX30_PLL_CON(16), PX30_MODE_CON, 2, 2, 0, px30_pll_rates), [npll] = PLL(pll_rk3328, PLL_NPLL, "npll", mux_pll_p, - 0, PX30_PLL_CON(24), + CLK_IS_CRITICAL, PX30_PLL_CON(24), PX30_MODE_CON, 6, 4, 0, px30_pll_rates), }; @@ -209,11 +214,11 @@ static struct rockchip_clk_branch px30_pdm_fracmux __initdata = PX30_CLKSEL_CON(26), 15, 1, MFLAGS); static struct rockchip_clk_branch px30_i2s0_tx_fracmux __initdata = - MUX(0, "clk_i2s0_tx_mux", mux_i2s0_tx_p, CLK_SET_RATE_PARENT, + MUX(SCLK_I2S0_TX_MUX, "clk_i2s0_tx_mux", mux_i2s0_tx_p, CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(28), 10, 2, MFLAGS); static struct rockchip_clk_branch px30_i2s0_rx_fracmux __initdata = - MUX(0, "clk_i2s0_rx_mux", mux_i2s0_rx_p, CLK_SET_RATE_PARENT, + MUX(SCLK_I2S0_RX_MUX, "clk_i2s0_rx_mux", mux_i2s0_rx_p, CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(58), 10, 2, MFLAGS); static struct rockchip_clk_branch px30_i2s1_fracmux __initdata = @@ -301,22 +306,12 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { PX30_CLKGATE_CON(17), 4, GFLAGS), /* PD_GPU */ - COMPOSITE_NODIV(0, "clk_gpu_src", mux_4plls_p, 0, - PX30_CLKSEL_CON(1), 6, 2, MFLAGS, - PX30_CLKGATE_CON(0), 8, GFLAGS), - COMPOSITE_NOMUX(0, "clk_gpu_div", "clk_gpu_src", 0, - PX30_CLKSEL_CON(1), 0, 4, DFLAGS, - PX30_CLKGATE_CON(0), 12, GFLAGS), - COMPOSITE_NOMUX_HALFDIV(0, "clk_gpu_np5", "clk_gpu_src", 0, - PX30_CLKSEL_CON(1), 8, 4, DFLAGS, - PX30_CLKGATE_CON(0), 9, GFLAGS), - COMPOSITE_NODIV(SCLK_GPU, "clk_gpu", mux_gpu_p, CLK_SET_RATE_PARENT, - PX30_CLKSEL_CON(1), 15, 1, MFLAGS, + GATE(SCLK_GPU, "clk_gpu", "clk_gpu_src", 0, PX30_CLKGATE_CON(0), 10, GFLAGS), COMPOSITE_NOMUX(0, "aclk_gpu", "clk_gpu", CLK_IGNORE_UNUSED, PX30_CLKSEL_CON(1), 13, 2, DFLAGS, PX30_CLKGATE_CON(17), 10, GFLAGS), - GATE(0, "aclk_gpu_niu", "aclk_gpu", CLK_IGNORE_UNUSED, + GATE(0, "aclk_gpu_niu", "aclk_gpu", CLK_IS_CRITICAL, PX30_CLKGATE_CON(0), 11, GFLAGS), GATE(0, "aclk_gpu_prf", "aclk_gpu", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(17), 8, GFLAGS), @@ -424,16 +419,16 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "dclk_vopb_frac", "dclk_vopb_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(6), 0, PX30_CLKGATE_CON(2), 3, GFLAGS, - &px30_dclk_vopb_fracmux), + &px30_dclk_vopb_fracmux, 0), GATE(DCLK_VOPB, "dclk_vopb", "dclk_vopb_mux", CLK_SET_RATE_PARENT, PX30_CLKGATE_CON(2), 4, GFLAGS), - COMPOSITE(0, "dclk_vopl_src", mux_npll_cpll_p, 0, + COMPOSITE(0, "dclk_vopl_src", mux_npll_cpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, PX30_CLKSEL_CON(8), 11, 1, MFLAGS, 0, 8, DFLAGS, PX30_CLKGATE_CON(2), 6, GFLAGS), COMPOSITE_FRACMUX(0, "dclk_vopl_frac", "dclk_vopl_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(9), 0, PX30_CLKGATE_CON(2), 7, GFLAGS, - &px30_dclk_vopl_fracmux), + &px30_dclk_vopl_fracmux, 0), GATE(DCLK_VOPL, "dclk_vopl", "dclk_vopl_mux", CLK_SET_RATE_PARENT, PX30_CLKGATE_CON(2), 8, GFLAGS), @@ -452,13 +447,13 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { * Clock-Architecture Diagram 7 */ - COMPOSITE_NODIV(ACLK_PERI_SRC, "aclk_peri_src", mux_gpll_cpll_p, 0, + COMPOSITE_NODIV(ACLK_PERI_SRC, "aclk_peri_src", mux_gpll_cpll_p, CLK_IS_CRITICAL, PX30_CLKSEL_CON(14), 15, 1, MFLAGS, PX30_CLKGATE_CON(5), 7, GFLAGS), - COMPOSITE_NOMUX(ACLK_PERI_PRE, "aclk_peri_pre", "aclk_peri_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(ACLK_PERI_PRE, "aclk_peri_pre", "aclk_peri_src", CLK_IS_CRITICAL, PX30_CLKSEL_CON(14), 0, 5, DFLAGS, PX30_CLKGATE_CON(5), 8, GFLAGS), - DIV(HCLK_PERI_PRE, "hclk_peri_pre", "aclk_peri_src", CLK_IGNORE_UNUSED, + DIV(HCLK_PERI_PRE, "hclk_peri_pre", "aclk_peri_src", CLK_IS_CRITICAL, PX30_CLKSEL_CON(14), 8, 5, DFLAGS), /* PD_MMC_NAND */ @@ -535,7 +530,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { PX30_CLKGATE_CON(6), 15, GFLAGS), /* PD_USB */ - GATE(HCLK_USB, "hclk_usb", "hclk_peri_pre", 0, + GATE(HCLK_USB, "hclk_usb", "hclk_peri_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(7), 2, GFLAGS), GATE(SCLK_OTG_ADP, "clk_otg_adp", "clk_rtc32k_pmu", 0, PX30_CLKGATE_CON(7), 3, GFLAGS), @@ -570,19 +565,19 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { */ /* PD_BUS */ - COMPOSITE_NODIV(ACLK_BUS_SRC, "aclk_bus_src", mux_gpll_cpll_p, CLK_IGNORE_UNUSED, + COMPOSITE_NODIV(ACLK_BUS_SRC, "aclk_bus_src", mux_gpll_cpll_p, CLK_IS_CRITICAL, PX30_CLKSEL_CON(23), 15, 1, MFLAGS, PX30_CLKGATE_CON(8), 6, GFLAGS), - COMPOSITE_NOMUX(HCLK_BUS_PRE, "hclk_bus_pre", "aclk_bus_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(HCLK_BUS_PRE, "hclk_bus_pre", "aclk_bus_src", CLK_IS_CRITICAL, PX30_CLKSEL_CON(24), 0, 5, DFLAGS, PX30_CLKGATE_CON(8), 8, GFLAGS), - COMPOSITE_NOMUX(ACLK_BUS_PRE, "aclk_bus_pre", "aclk_bus_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(ACLK_BUS_PRE, "aclk_bus_pre", "aclk_bus_src", CLK_IS_CRITICAL, PX30_CLKSEL_CON(23), 8, 5, DFLAGS, PX30_CLKGATE_CON(8), 7, GFLAGS), - COMPOSITE_NOMUX(PCLK_BUS_PRE, "pclk_bus_pre", "aclk_bus_pre", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(PCLK_BUS_PRE, "pclk_bus_pre", "aclk_bus_pre", CLK_IS_CRITICAL, PX30_CLKSEL_CON(24), 8, 2, DFLAGS, PX30_CLKGATE_CON(8), 9, GFLAGS), - GATE(0, "pclk_top_pre", "pclk_bus_pre", CLK_IGNORE_UNUSED, + GATE(0, "pclk_top_pre", "pclk_bus_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(8), 10, GFLAGS), COMPOSITE(0, "clk_pdm_src", mux_gpll_xin24m_npll_p, 0, @@ -591,7 +586,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_pdm_frac", "clk_pdm_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(27), 0, PX30_CLKGATE_CON(9), 10, GFLAGS, - &px30_pdm_fracmux), + &px30_pdm_fracmux, PX30_FRAC_MAX_PRATE), GATE(SCLK_PDM, "clk_pdm", "clk_pdm_mux", CLK_SET_RATE_PARENT, PX30_CLKGATE_CON(9), 11, GFLAGS), @@ -601,11 +596,11 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s0_tx_frac", "clk_i2s0_tx_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(29), 0, PX30_CLKGATE_CON(9), 13, GFLAGS, - &px30_i2s0_tx_fracmux), + &px30_i2s0_tx_fracmux, PX30_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S0_TX, "clk_i2s0_tx", mux_i2s0_tx_rx_p, CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(28), 12, 1, MFLAGS, PX30_CLKGATE_CON(9), 14, GFLAGS), - COMPOSITE_NODIV(0, "clk_i2s0_tx_out_pre", mux_i2s0_tx_out_p, 0, + COMPOSITE_NODIV(0, "clk_i2s0_tx_out_pre", mux_i2s0_tx_out_p, CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(28), 14, 2, MFLAGS, PX30_CLKGATE_CON(9), 15, GFLAGS), GATE(SCLK_I2S0_TX_OUT, "clk_i2s0_tx_out", "clk_i2s0_tx_out_pre", CLK_SET_RATE_PARENT, @@ -617,7 +612,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s0_rx_frac", "clk_i2s0_rx_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(59), 0, PX30_CLKGATE_CON(17), 1, GFLAGS, - &px30_i2s0_rx_fracmux), + &px30_i2s0_rx_fracmux, PX30_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S0_RX, "clk_i2s0_rx", mux_i2s0_rx_tx_p, CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(58), 12, 1, MFLAGS, PX30_CLKGATE_CON(17), 2, GFLAGS), @@ -633,10 +628,10 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s1_frac", "clk_i2s1_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(31), 0, PX30_CLKGATE_CON(10), 1, GFLAGS, - &px30_i2s1_fracmux), + &px30_i2s1_fracmux, PX30_FRAC_MAX_PRATE), GATE(SCLK_I2S1, "clk_i2s1", "clk_i2s1_mux", CLK_SET_RATE_PARENT, PX30_CLKGATE_CON(10), 2, GFLAGS), - COMPOSITE_NODIV(0, "clk_i2s1_out_pre", mux_i2s1_out_p, 0, + COMPOSITE_NODIV(0, "clk_i2s1_out_pre", mux_i2s1_out_p, CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(30), 15, 1, MFLAGS, PX30_CLKGATE_CON(10), 3, GFLAGS), GATE(SCLK_I2S1_OUT, "clk_i2s1_out", "clk_i2s1_out_pre", CLK_SET_RATE_PARENT, @@ -648,7 +643,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s2_frac", "clk_i2s2_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(33), 0, PX30_CLKGATE_CON(10), 5, GFLAGS, - &px30_i2s2_fracmux), + &px30_i2s2_fracmux, PX30_FRAC_MAX_PRATE), GATE(SCLK_I2S2, "clk_i2s2", "clk_i2s2_mux", CLK_SET_RATE_PARENT, PX30_CLKGATE_CON(10), 6, GFLAGS), COMPOSITE_NODIV(0, "clk_i2s2_out_pre", mux_i2s2_out_p, 0, @@ -666,7 +661,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(36), 0, PX30_CLKGATE_CON(10), 14, GFLAGS, - &px30_uart1_fracmux), + &px30_uart1_fracmux, PX30_FRAC_MAX_PRATE), GATE(SCLK_UART1, "clk_uart1", "clk_uart1_mux", CLK_SET_RATE_PARENT, PX30_CLKGATE_CON(10), 15, GFLAGS), @@ -679,8 +674,8 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(39), 0, PX30_CLKGATE_CON(11), 2, GFLAGS, - &px30_uart2_fracmux), - GATE(SCLK_UART2, "clk_uart2", "clk_uart2_mux", CLK_SET_RATE_PARENT, + &px30_uart2_fracmux, PX30_FRAC_MAX_PRATE), + GATE(SCLK_UART2, "clk_uart2", "clk_uart2_mux", CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, PX30_CLKGATE_CON(11), 3, GFLAGS), COMPOSITE(0, "clk_uart3_src", mux_uart_src_p, 0, @@ -692,7 +687,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(42), 0, PX30_CLKGATE_CON(11), 6, GFLAGS, - &px30_uart3_fracmux), + &px30_uart3_fracmux, PX30_FRAC_MAX_PRATE), GATE(SCLK_UART3, "clk_uart3", "clk_uart3_mux", CLK_SET_RATE_PARENT, PX30_CLKGATE_CON(11), 7, GFLAGS), @@ -705,7 +700,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(45), 0, PX30_CLKGATE_CON(11), 10, GFLAGS, - &px30_uart4_fracmux), + &px30_uart4_fracmux, PX30_FRAC_MAX_PRATE), GATE(SCLK_UART4, "clk_uart4", "clk_uart4_mux", CLK_SET_RATE_PARENT, PX30_CLKGATE_CON(11), 11, GFLAGS), @@ -718,7 +713,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_uart5_frac", "clk_uart5_src", CLK_SET_RATE_PARENT, PX30_CLKSEL_CON(48), 0, PX30_CLKGATE_CON(11), 14, GFLAGS, - &px30_uart5_fracmux), + &px30_uart5_fracmux, PX30_FRAC_MAX_PRATE), GATE(SCLK_UART5, "clk_uart5", "clk_uart5_mux", CLK_SET_RATE_PARENT, PX30_CLKGATE_CON(11), 15, GFLAGS), @@ -799,29 +794,29 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { GATE(0, "pclk_ddrphy", "pclk_top_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(16), 3, GFLAGS), GATE(PCLK_MIPIDSIPHY, "pclk_mipidsiphy", "pclk_top_pre", 0, PX30_CLKGATE_CON(16), 4, GFLAGS), GATE(PCLK_MIPICSIPHY, "pclk_mipicsiphy", "pclk_top_pre", 0, PX30_CLKGATE_CON(16), 5, GFLAGS), - GATE(PCLK_USB_GRF, "pclk_usb_grf", "pclk_top_pre", 0, PX30_CLKGATE_CON(16), 6, GFLAGS), + GATE(PCLK_USB_GRF, "pclk_usb_grf", "pclk_top_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(16), 6, GFLAGS), GATE(0, "pclk_cpu_hoost", "pclk_top_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(16), 7, GFLAGS), /* PD_VI */ - GATE(0, "aclk_vi_niu", "aclk_vi_pre", 0, PX30_CLKGATE_CON(4), 15, GFLAGS), + GATE(0, "aclk_vi_niu", "aclk_vi_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(4), 15, GFLAGS), GATE(ACLK_CIF, "aclk_cif", "aclk_vi_pre", 0, PX30_CLKGATE_CON(5), 1, GFLAGS), GATE(ACLK_ISP, "aclk_isp", "aclk_vi_pre", 0, PX30_CLKGATE_CON(5), 3, GFLAGS), - GATE(0, "hclk_vi_niu", "hclk_vi_pre", 0, PX30_CLKGATE_CON(5), 0, GFLAGS), + GATE(0, "hclk_vi_niu", "hclk_vi_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(5), 0, GFLAGS), GATE(HCLK_CIF, "hclk_cif", "hclk_vi_pre", 0, PX30_CLKGATE_CON(5), 2, GFLAGS), GATE(HCLK_ISP, "hclk_isp", "hclk_vi_pre", 0, PX30_CLKGATE_CON(5), 4, GFLAGS), /* PD_VO */ - GATE(0, "aclk_vo_niu", "aclk_vo_pre", 0, PX30_CLKGATE_CON(3), 0, GFLAGS), + GATE(0, "aclk_vo_niu", "aclk_vo_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(3), 0, GFLAGS), GATE(ACLK_VOPB, "aclk_vopb", "aclk_vo_pre", 0, PX30_CLKGATE_CON(3), 3, GFLAGS), GATE(ACLK_RGA, "aclk_rga", "aclk_vo_pre", 0, PX30_CLKGATE_CON(3), 7, GFLAGS), GATE(ACLK_VOPL, "aclk_vopl", "aclk_vo_pre", 0, PX30_CLKGATE_CON(3), 5, GFLAGS), - GATE(0, "hclk_vo_niu", "hclk_vo_pre", 0, PX30_CLKGATE_CON(3), 1, GFLAGS), + GATE(0, "hclk_vo_niu", "hclk_vo_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(3), 1, GFLAGS), GATE(HCLK_VOPB, "hclk_vopb", "hclk_vo_pre", 0, PX30_CLKGATE_CON(3), 4, GFLAGS), GATE(HCLK_RGA, "hclk_rga", "hclk_vo_pre", 0, PX30_CLKGATE_CON(3), 8, GFLAGS), GATE(HCLK_VOPL, "hclk_vopl", "hclk_vo_pre", 0, PX30_CLKGATE_CON(3), 6, GFLAGS), - GATE(0, "pclk_vo_niu", "pclk_vo_pre", 0, PX30_CLKGATE_CON(3), 2, GFLAGS), + GATE(0, "pclk_vo_niu", "pclk_vo_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(3), 2, GFLAGS), GATE(PCLK_MIPI_DSI, "pclk_mipi_dsi", "pclk_vo_pre", 0, PX30_CLKGATE_CON(3), 9, GFLAGS), /* PD_BUS */ @@ -843,7 +838,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { GATE(0, "pclk_bus_niu", "pclk_bus_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(13), 10, GFLAGS), GATE(PCLK_DCF, "pclk_dcf", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 0, GFLAGS), GATE(PCLK_UART1, "pclk_uart1", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 5, GFLAGS), - GATE(PCLK_UART2, "pclk_uart2", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 6, GFLAGS), + GATE(PCLK_UART2, "pclk_uart2", "pclk_bus_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(14), 6, GFLAGS), GATE(PCLK_UART3, "pclk_uart3", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 7, GFLAGS), GATE(PCLK_UART4, "pclk_uart4", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 8, GFLAGS), GATE(PCLK_UART5, "pclk_uart5", "pclk_bus_pre", 0, PX30_CLKGATE_CON(14), 9, GFLAGS), @@ -884,7 +879,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_sdmmc_pre", 0, PX30_CLKGATE_CON(7), 1, GFLAGS), /* PD_PERI */ - GATE(0, "aclk_peri_niu", "aclk_peri_pre", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(5), 9, GFLAGS), + GATE(0, "aclk_peri_niu", "aclk_peri_pre", CLK_IS_CRITICAL, PX30_CLKGATE_CON(5), 9, GFLAGS), /* PD_MMC_NAND */ GATE(HCLK_NANDC, "hclk_nandc", "hclk_mmc_nand", 0, PX30_CLKGATE_CON(5), 15, GFLAGS), @@ -894,7 +889,7 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { GATE(HCLK_SFC, "hclk_sfc", "hclk_mmc_nand", 0, PX30_CLKGATE_CON(6), 11, GFLAGS), /* PD_USB */ - GATE(0, "hclk_usb_niu", "hclk_usb", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(7), 4, GFLAGS), + GATE(0, "hclk_usb_niu", "hclk_usb", CLK_IS_CRITICAL, PX30_CLKGATE_CON(7), 4, GFLAGS), GATE(HCLK_OTG, "hclk_otg", "hclk_usb", 0, PX30_CLKGATE_CON(7), 5, GFLAGS), GATE(HCLK_HOST, "hclk_host", "hclk_usb", 0, PX30_CLKGATE_CON(7), 6, GFLAGS), GATE(HCLK_HOST_ARB, "hclk_host_arb", "hclk_usb", CLK_IGNORE_UNUSED, PX30_CLKGATE_CON(7), 8, GFLAGS), @@ -910,6 +905,18 @@ static struct rockchip_clk_branch px30_clk_branches[] __initdata = { PX30_CLKGATE_CON(8), 3, GFLAGS), }; +static struct rockchip_clk_branch px30_gpu_src_clk[] __initdata = { + COMPOSITE(0, "clk_gpu_src", mux_gpll_dmycpll_usb480m_dmynpll_p, 0, + PX30_CLKSEL_CON(1), 6, 2, MFLAGS, 0, 4, DFLAGS, + PX30_CLKGATE_CON(0), 8, GFLAGS), +}; + +static struct rockchip_clk_branch rk3326_gpu_src_clk[] __initdata = { + COMPOSITE(0, "clk_gpu_src", mux_gpll_dmycpll_usb480m_npll_p, 0, + PX30_CLKSEL_CON(1), 6, 2, MFLAGS, 0, 4, DFLAGS, + PX30_CLKGATE_CON(0), 8, GFLAGS), +}; + static struct rockchip_clk_branch px30_clk_pmu_branches[] __initdata = { /* * Clock-Architecture Diagram 2 @@ -918,7 +925,7 @@ static struct rockchip_clk_branch px30_clk_pmu_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_rtc32k_frac", "xin24m", CLK_IGNORE_UNUSED, PX30_PMU_CLKSEL_CON(1), 0, PX30_PMU_CLKGATE_CON(0), 13, GFLAGS, - &px30_rtc32k_pmu_fracmux), + &px30_rtc32k_pmu_fracmux, 0), COMPOSITE_NOMUX(XIN24M_DIV, "xin24m_div", "xin24m", CLK_IGNORE_UNUSED, PX30_PMU_CLKSEL_CON(0), 8, 5, DFLAGS, @@ -940,14 +947,14 @@ static struct rockchip_clk_branch px30_clk_pmu_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_pmu_src", CLK_SET_RATE_PARENT, PX30_PMU_CLKSEL_CON(5), 0, PX30_PMU_CLKGATE_CON(1), 2, GFLAGS, - &px30_uart0_pmu_fracmux), + &px30_uart0_pmu_fracmux, PX30_FRAC_MAX_PRATE), GATE(SCLK_UART0_PMU, "clk_uart0_pmu", "clk_uart0_pmu_mux", CLK_SET_RATE_PARENT, PX30_PMU_CLKGATE_CON(1), 3, GFLAGS), GATE(SCLK_PVTM_PMU, "clk_pvtm_pmu", "xin24m", 0, PX30_PMU_CLKGATE_CON(1), 4, GFLAGS), - COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "gpll", 0, + COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "gpll", CLK_IS_CRITICAL, PX30_PMU_CLKSEL_CON(0), 0, 5, DFLAGS, PX30_PMU_CLKGATE_CON(0), 0, GFLAGS), @@ -976,28 +983,7 @@ static struct rockchip_clk_branch px30_clk_pmu_branches[] __initdata = { GATE(0, "pclk_cru_pmu", "pclk_pmu_pre", CLK_IGNORE_UNUSED, PX30_PMU_CLKGATE_CON(0), 8, GFLAGS), }; -static const char *const px30_cru_critical_clocks[] __initconst = { - "aclk_bus_pre", - "pclk_bus_pre", - "hclk_bus_pre", - "aclk_peri_pre", - "hclk_peri_pre", - "aclk_gpu_niu", - "pclk_top_pre", - "pclk_pmu_pre", - "hclk_usb_niu", - "pclk_vo_niu", - "aclk_vo_niu", - "hclk_vo_niu", - "aclk_vi_niu", - "hclk_vi_niu", - "pll_npll", - "usb480m", - "clk_uart2", - "pclk_uart2", - "pclk_usb_grf", -}; - +static struct rockchip_clk_provider *cru_ctx; static void __init px30_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; @@ -1021,14 +1007,12 @@ static void __init px30_clk_init(struct device_node *np) PX30_GRF_SOC_STATUS0); rockchip_clk_register_branches(ctx, px30_clk_branches, ARRAY_SIZE(px30_clk_branches)); - - rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", - mux_armclk_p, ARRAY_SIZE(mux_armclk_p), - &px30_cpuclk_data, px30_cpuclk_rates, - ARRAY_SIZE(px30_cpuclk_rates)); - - rockchip_clk_protect_critical(px30_cru_critical_clocks, - ARRAY_SIZE(px30_cru_critical_clocks)); + if (of_machine_is_compatible("rockchip,px30")) + rockchip_clk_register_branches(ctx, px30_gpu_src_clk, + ARRAY_SIZE(px30_gpu_src_clk)); + else + rockchip_clk_register_branches(ctx, rk3326_gpu_src_clk, + ARRAY_SIZE(rk3326_gpu_src_clk)); rockchip_register_softrst(np, 12, reg_base + PX30_SOFTRST_CON(0), ROCKCHIP_SOFTRST_HIWORD_MASK); @@ -1036,6 +1020,8 @@ static void __init px30_clk_init(struct device_node *np) rockchip_register_restart_notifier(ctx, PX30_GLB_SRST_FST, NULL); rockchip_clk_of_add_provider(np, ctx); + + cru_ctx = ctx; } CLK_OF_DECLARE(px30_cru, "rockchip,px30-cru", px30_clk_init); @@ -1043,6 +1029,7 @@ static void __init px30_pmu_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; + struct clk **pmucru_clks, **cru_clks; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -1055,13 +1042,72 @@ static void __init px30_pmu_clk_init(struct device_node *np) pr_err("%s: rockchip pmu clk init failed\n", __func__); return; } + pmucru_clks = ctx->clk_data.clks; + cru_clks = cru_ctx->clk_data.clks; rockchip_clk_register_plls(ctx, px30_pmu_pll_clks, ARRAY_SIZE(px30_pmu_pll_clks), PX30_GRF_SOC_STATUS0); + rockchip_clk_register_armclk(cru_ctx, ARMCLK, "armclk", + 2, cru_clks[PLL_APLL], pmucru_clks[PLL_GPLL], + &px30_cpuclk_data, px30_cpuclk_rates, + ARRAY_SIZE(px30_cpuclk_rates)); + rockchip_clk_register_branches(ctx, px30_clk_pmu_branches, ARRAY_SIZE(px30_clk_pmu_branches)); rockchip_clk_of_add_provider(np, ctx); } CLK_OF_DECLARE(px30_cru_pmu, "rockchip,px30-pmucru", px30_pmu_clk_init); + +struct clk_px30_inits { + void (*inits)(struct device_node *np); +}; + +static const struct clk_px30_inits clk_px30_init = { + .inits = px30_clk_init, +}; + +static const struct clk_px30_inits clk_px30_pmu_init = { + .inits = px30_pmu_clk_init, +}; + +static const struct of_device_id clk_px30_match_table[] = { + { + .compatible = "rockchip,px30-cru", + .data = &clk_px30_init, + }, { + .compatible = "rockchip,px30-pmucru", + .data = &clk_px30_pmu_init, + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_px30_match_table); + +static int __init clk_px30_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + const struct clk_px30_inits *init_data; + + match = of_match_device(clk_px30_match_table, &pdev->dev); + if (!match || !match->data) + return -EINVAL; + + init_data = match->data; + if (init_data->inits) + init_data->inits(np); + + return 0; +} + +static struct platform_driver clk_px30_driver = { + .driver = { + .name = "clk-px30", + .of_match_table = clk_px30_match_table, + }, +}; +builtin_platform_driver_probe(clk_px30_driver, clk_px30_probe); + +MODULE_DESCRIPTION("Rockchip PX30 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rk1808.c b/drivers/clk/rockchip/clk-rk1808.c new file mode 100755 index 000000000000..cb9483623f13 --- /dev/null +++ b/drivers/clk/rockchip/clk-rk1808.c @@ -0,0 +1,1249 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Fuzhou Rockchip Electronics Co., Ltd + * Author: Elaine Zhang + */ +#include +#include +#include +#include +#include +#include +#include +#include "clk.h" + +#define RK1808_GRF_SOC_STATUS0 0x480 +#define RK1808_PMUGRF_SOC_CON0 0x100 +#define RK1808_UART_FRAC_MAX_PRATE 800000000 +#define RK1808_PDM_FRAC_MAX_PRATE 300000000 +#define RK1808_I2S_FRAC_MAX_PRATE 600000000 +#define RK1808_VOP_RAW_FRAC_MAX_PRATE 300000000 +#define RK1808_VOP_LITE_FRAC_MAX_PRATE 400000000 + +enum rk1808_plls { + apll, dpll, cpll, gpll, npll, ppll, +}; + +static struct rockchip_pll_rate_table rk1808_pll_rates[] = { + /* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */ + RK3036_PLL_RATE(1608000000, 1, 67, 1, 1, 1, 0), + RK3036_PLL_RATE(1584000000, 1, 66, 1, 1, 1, 0), + RK3036_PLL_RATE(1560000000, 1, 65, 1, 1, 1, 0), + RK3036_PLL_RATE(1536000000, 1, 64, 1, 1, 1, 0), + RK3036_PLL_RATE(1512000000, 1, 63, 1, 1, 1, 0), + RK3036_PLL_RATE(1488000000, 1, 62, 1, 1, 1, 0), + RK3036_PLL_RATE(1464000000, 1, 61, 1, 1, 1, 0), + RK3036_PLL_RATE(1440000000, 1, 60, 1, 1, 1, 0), + RK3036_PLL_RATE(1416000000, 1, 59, 1, 1, 1, 0), + RK3036_PLL_RATE(1392000000, 1, 58, 1, 1, 1, 0), + RK3036_PLL_RATE(1368000000, 1, 57, 1, 1, 1, 0), + RK3036_PLL_RATE(1344000000, 1, 56, 1, 1, 1, 0), + RK3036_PLL_RATE(1320000000, 1, 55, 1, 1, 1, 0), + RK3036_PLL_RATE(1296000000, 1, 54, 1, 1, 1, 0), + RK3036_PLL_RATE(1272000000, 1, 53, 1, 1, 1, 0), + RK3036_PLL_RATE(1248000000, 1, 52, 1, 1, 1, 0), + RK3036_PLL_RATE(1200000000, 1, 50, 1, 1, 1, 0), + RK3036_PLL_RATE(1188000000, 1, 99, 2, 1, 1, 0), + RK3036_PLL_RATE(1104000000, 1, 46, 1, 1, 1, 0), + RK3036_PLL_RATE(1100000000, 2, 275, 3, 1, 1, 0), + RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0), + RK3036_PLL_RATE(1000000000, 1, 125, 3, 1, 1, 0), + RK3036_PLL_RATE(984000000, 1, 82, 2, 1, 1, 0), + RK3036_PLL_RATE(960000000, 1, 80, 2, 1, 1, 0), + RK3036_PLL_RATE(936000000, 1, 78, 2, 1, 1, 0), + RK3036_PLL_RATE(912000000, 1, 76, 2, 1, 1, 0), + RK3036_PLL_RATE(900000000, 1, 75, 2, 1, 1, 0), + RK3036_PLL_RATE(888000000, 1, 74, 2, 1, 1, 0), + RK3036_PLL_RATE(864000000, 1, 72, 2, 1, 1, 0), + RK3036_PLL_RATE(840000000, 1, 70, 2, 1, 1, 0), + RK3036_PLL_RATE(816000000, 1, 68, 2, 1, 1, 0), + RK3036_PLL_RATE(800000000, 1, 100, 3, 1, 1, 0), + RK3036_PLL_RATE(700000000, 1, 175, 2, 1, 1, 0), + RK3036_PLL_RATE(696000000, 1, 58, 2, 1, 1, 0), + RK3036_PLL_RATE(624000000, 1, 52, 2, 1, 1, 0), + RK3036_PLL_RATE(600000000, 1, 75, 3, 1, 1, 0), + RK3036_PLL_RATE(594000000, 1, 99, 4, 1, 1, 0), + RK3036_PLL_RATE(504000000, 1, 63, 3, 1, 1, 0), + RK3036_PLL_RATE(500000000, 1, 125, 6, 1, 1, 0), + RK3036_PLL_RATE(416000000, 1, 52, 3, 1, 1, 0), + RK3036_PLL_RATE(408000000, 1, 68, 2, 2, 1, 0), + RK3036_PLL_RATE(312000000, 1, 52, 2, 2, 1, 0), + RK3036_PLL_RATE(216000000, 1, 72, 4, 2, 1, 0), + RK3036_PLL_RATE(200000000, 1, 200, 6, 4, 1, 0), + RK3036_PLL_RATE(100000000, 1, 150, 6, 6, 1, 0), + RK3036_PLL_RATE(96000000, 1, 64, 4, 4, 1, 0), + { /* sentinel */ }, +}; + +#define RK1808_DIV_ACLKM_MASK 0x7 +#define RK1808_DIV_ACLKM_SHIFT 12 +#define RK1808_DIV_PCLK_DBG_MASK 0xf +#define RK1808_DIV_PCLK_DBG_SHIFT 8 + +#define RK1808_CLKSEL0(_aclk_core, _pclk_dbg) \ +{ \ + .reg = RK1808_CLKSEL_CON(0), \ + .val = HIWORD_UPDATE(_aclk_core, RK1808_DIV_ACLKM_MASK, \ + RK1808_DIV_ACLKM_SHIFT) | \ + HIWORD_UPDATE(_pclk_dbg, RK1808_DIV_PCLK_DBG_MASK, \ + RK1808_DIV_PCLK_DBG_SHIFT), \ +} + +#define RK1808_CPUCLK_RATE(_prate, _aclk_core, _pclk_dbg) \ +{ \ + .prate = _prate, \ + .divs = { \ + RK1808_CLKSEL0(_aclk_core, _pclk_dbg), \ + }, \ +} + +static struct rockchip_cpuclk_rate_table rk1808_cpuclk_rates[] __initdata = { + RK1808_CPUCLK_RATE(1608000000, 1, 7), + RK1808_CPUCLK_RATE(1512000000, 1, 7), + RK1808_CPUCLK_RATE(1488000000, 1, 5), + RK1808_CPUCLK_RATE(1416000000, 1, 5), + RK1808_CPUCLK_RATE(1392000000, 1, 5), + RK1808_CPUCLK_RATE(1296000000, 1, 5), + RK1808_CPUCLK_RATE(1200000000, 1, 5), + RK1808_CPUCLK_RATE(1104000000, 1, 5), + RK1808_CPUCLK_RATE(1008000000, 1, 5), + RK1808_CPUCLK_RATE(912000000, 1, 5), + RK1808_CPUCLK_RATE(816000000, 1, 3), + RK1808_CPUCLK_RATE(696000000, 1, 3), + RK1808_CPUCLK_RATE(600000000, 1, 3), + RK1808_CPUCLK_RATE(408000000, 1, 1), + RK1808_CPUCLK_RATE(312000000, 1, 1), + RK1808_CPUCLK_RATE(216000000, 1, 1), + RK1808_CPUCLK_RATE(96000000, 1, 1), +}; + +static const struct rockchip_cpuclk_reg_data rk1808_cpuclk_data = { + .core_reg[0] = RK1808_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0xf, + .num_cores = 1, + .mux_core_alt = 2, + .mux_core_main = 0, + .mux_core_shift = 6, + .mux_core_mask = 0x3, +}; + +PNAME(mux_pll_p) = { "xin24m", "xin32k"}; +PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "xin32k" }; +PNAME(mux_gpll_cpll_p) = { "gpll", "cpll" }; +PNAME(mux_gpll_cpll_apll_p) = { "gpll", "cpll", "apll" }; +PNAME(mux_npu_p) = { "clk_npu_div", "clk_npu_np5" }; +PNAME(mux_ddr_p) = { "dpll_ddr", "gpll_ddr" }; +PNAME(mux_cpll_gpll_npll_p) = { "cpll", "gpll", "npll" }; +PNAME(mux_gpll_cpll_npll_p) = { "gpll", "cpll", "npll" }; +PNAME(mux_dclk_vopraw_p) = { "dclk_vopraw_src", "dclk_vopraw_frac", "xin24m" }; +PNAME(mux_dclk_voplite_p) = { "dclk_voplite_src", "dclk_voplite_frac", "xin24m" }; +PNAME(mux_24m_npll_gpll_usb480m_p) = { "xin24m", "npll", "gpll", "usb480m" }; +PNAME(mux_usb3_otg0_suspend_p) = { "xin32k", "xin24m" }; +PNAME(mux_pcie_aux_p) = { "xin24m", "clk_pcie_src" }; +PNAME(mux_gpll_cpll_npll_24m_p) = { "gpll", "cpll", "npll", "xin24m" }; +PNAME(mux_sdio_p) = { "clk_sdio_div", "clk_sdio_div50" }; +PNAME(mux_sdmmc_p) = { "clk_sdmmc_div", "clk_sdmmc_div50" }; +PNAME(mux_emmc_p) = { "clk_emmc_div", "clk_emmc_div50" }; +PNAME(mux_cpll_npll_ppll_p) = { "cpll", "npll", "ppll" }; +PNAME(mux_gmac_p) = { "clk_gmac_src", "gmac_clkin" }; +PNAME(mux_gmac_rgmii_speed_p) = { "clk_gmac_tx_src", "clk_gmac_tx_src", "clk_gmac_tx_div50", "clk_gmac_tx_div5" }; +PNAME(mux_gmac_rmii_speed_p) = { "clk_gmac_rx_div20", "clk_gmac_rx_div2" }; +PNAME(mux_gmac_rx_tx_p) = { "clk_gmac_rgmii_speed", "clk_gmac_rmii_speed" }; +PNAME(mux_gpll_usb480m_cpll_npll_p) = { "gpll", "usb480m", "cpll", "npll" }; +PNAME(mux_uart1_p) = { "clk_uart1_src", "clk_uart1_np5", "clk_uart1_frac", "xin24m" }; +PNAME(mux_uart2_p) = { "clk_uart2_src", "clk_uart2_np5", "clk_uart2_frac", "xin24m" }; +PNAME(mux_uart3_p) = { "clk_uart3_src", "clk_uart3_np5", "clk_uart3_frac", "xin24m" }; +PNAME(mux_uart4_p) = { "clk_uart4_src", "clk_uart4_np5", "clk_uart4_frac", "xin24m" }; +PNAME(mux_uart5_p) = { "clk_uart5_src", "clk_uart5_np5", "clk_uart5_frac", "xin24m" }; +PNAME(mux_uart6_p) = { "clk_uart6_src", "clk_uart6_np5", "clk_uart6_frac", "xin24m" }; +PNAME(mux_uart7_p) = { "clk_uart7_src", "clk_uart7_np5", "clk_uart7_frac", "xin24m" }; +PNAME(mux_gpll_xin24m_p) = { "gpll", "xin24m" }; +PNAME(mux_gpll_cpll_xin24m_p) = { "gpll", "cpll", "xin24m" }; +PNAME(mux_gpll_xin24m_cpll_npll_p) = { "gpll", "xin24m", "cpll", "npll" }; +PNAME(mux_pdm_p) = { "clk_pdm_src", "clk_pdm_frac" }; +PNAME(mux_i2s0_8ch_tx_p) = { "clk_i2s0_8ch_tx_src", "clk_i2s0_8ch_tx_frac", "mclk_i2s0_8ch_in", "xin12m" }; +PNAME(mux_i2s0_8ch_tx_rx_p) = { "clk_i2s0_8ch_tx_mux", "clk_i2s0_8ch_rx_mux"}; +PNAME(mux_i2s0_8ch_tx_out_p) = { "clk_i2s0_8ch_tx", "xin12m", "clk_i2s0_8ch_rx" }; +PNAME(mux_i2s0_8ch_rx_p) = { "clk_i2s0_8ch_rx_src", "clk_i2s0_8ch_rx_frac", "mclk_i2s0_8ch_in", "xin12m" }; +PNAME(mux_i2s0_8ch_rx_tx_p) = { "clk_i2s0_8ch_rx_mux", "clk_i2s0_8ch_tx_mux"}; +PNAME(mux_i2s0_8ch_rx_out_p) = { "clk_i2s0_8ch_rx", "xin12m", "clk_i2s0_8ch_tx" }; +PNAME(mux_i2s1_2ch_p) = { "clk_i2s1_2ch_src", "clk_i2s1_2ch_frac", "mclk_i2s1_2ch_in", "xin12m" }; +PNAME(mux_i2s1_2ch_out_p) = { "clk_i2s1_2ch", "xin12m" }; +PNAME(mux_rtc32k_pmu_p) = { "xin32k", "pmu_pvtm_32k", "clk_rtc32k_frac" }; +PNAME(mux_wifi_pmu_p) = { "xin24m", "clk_wifi_pmu_src" }; +PNAME(mux_gpll_usb480m_cpll_ppll_p) = { "gpll", "usb480m", "cpll", "ppll" }; +PNAME(mux_uart0_pmu_p) = { "clk_uart0_pmu_src", "clk_uart0_np5", "clk_uart0_frac", "xin24m" }; +PNAME(mux_usbphy_ref_p) = { "xin24m", "clk_ref24m_pmu" }; +PNAME(mux_mipidsiphy_ref_p) = { "xin24m", "clk_ref24m_pmu" }; +PNAME(mux_pciephy_ref_p) = { "xin24m", "clk_pciephy_src" }; +PNAME(mux_ppll_xin24m_p) = { "ppll", "xin24m" }; +PNAME(mux_xin24m_32k_p) = { "xin24m", "xin32k" }; +PNAME(mux_clk_32k_ioe_p) = { "clk_rtc32k_pmu", "xin32k" }; + +static struct rockchip_pll_clock rk1808_pll_clks[] __initdata = { + [apll] = PLL(pll_rk3036, PLL_APLL, "apll", mux_pll_p, + 0, RK1808_PLL_CON(0), + RK1808_MODE_CON, 0, 0, 0, rk1808_pll_rates), + [dpll] = PLL(pll_rk3036, PLL_DPLL, "dpll", mux_pll_p, + 0, RK1808_PLL_CON(8), + RK1808_MODE_CON, 2, 1, 0, NULL), + [cpll] = PLL(pll_rk3036, PLL_CPLL, "cpll", mux_pll_p, + 0, RK1808_PLL_CON(16), + RK1808_MODE_CON, 4, 2, 0, rk1808_pll_rates), + [gpll] = PLL(pll_rk3036, PLL_GPLL, "gpll", mux_pll_p, + 0, RK1808_PLL_CON(24), + RK1808_MODE_CON, 6, 3, 0, rk1808_pll_rates), + [npll] = PLL(pll_rk3036, PLL_NPLL, "npll", mux_pll_p, + 0, RK1808_PLL_CON(32), + RK1808_MODE_CON, 8, 5, 0, rk1808_pll_rates), + [ppll] = PLL(pll_rk3036, PLL_PPLL, "ppll", mux_pll_p, + 0, RK1808_PMU_PLL_CON(0), + RK1808_PMU_MODE_CON, 0, 4, 0, rk1808_pll_rates), +}; + +#define MFLAGS CLK_MUX_HIWORD_MASK +#define DFLAGS CLK_DIVIDER_HIWORD_MASK +#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE) + +static struct rockchip_clk_branch rk1808_uart1_fracmux __initdata = + MUX(0, "clk_uart1_mux", mux_uart1_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(39), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_uart2_fracmux __initdata = + MUX(0, "clk_uart2_mux", mux_uart2_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(42), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_uart3_fracmux __initdata = + MUX(0, "clk_uart3_mux", mux_uart3_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(45), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_uart4_fracmux __initdata = + MUX(0, "clk_uart4_mux", mux_uart4_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(48), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_uart5_fracmux __initdata = + MUX(0, "clk_uart5_mux", mux_uart5_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(51), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_uart6_fracmux __initdata = + MUX(0, "clk_uart6_mux", mux_uart6_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(54), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_uart7_fracmux __initdata = + MUX(0, "clk_uart7_mux", mux_uart7_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(57), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_dclk_vopraw_fracmux __initdata = + MUX(0, "dclk_vopraw_mux", mux_dclk_vopraw_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(5), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_dclk_voplite_fracmux __initdata = + MUX(0, "dclk_voplite_mux", mux_dclk_voplite_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(7), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_pdm_fracmux __initdata = + MUX(0, "clk_pdm_mux", mux_pdm_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(30), 15, 1, MFLAGS); + +static struct rockchip_clk_branch rk1808_i2s0_8ch_tx_fracmux __initdata = + MUX(SCLK_I2S0_8CH_TX_MUX, "clk_i2s0_8ch_tx_mux", mux_i2s0_8ch_tx_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(32), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_i2s0_8ch_rx_fracmux __initdata = + MUX(SCLK_I2S0_8CH_RX_MUX, "clk_i2s0_8ch_rx_mux", mux_i2s0_8ch_rx_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(34), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_i2s1_2ch_fracmux __initdata = + MUX(0, "clk_i2s1_2ch_mux", mux_i2s1_2ch_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(36), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_rtc32k_pmu_fracmux __initdata = + MUX(SCLK_RTC32K_PMU, "clk_rtc32k_pmu", mux_rtc32k_pmu_p, CLK_SET_RATE_PARENT, + RK1808_PMU_CLKSEL_CON(0), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_uart0_pmu_fracmux __initdata = + MUX(0, "clk_uart0_pmu_mux", mux_uart0_pmu_p, CLK_SET_RATE_PARENT, + RK1808_PMU_CLKSEL_CON(4), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rk1808_clk_branches[] __initdata = { + /* + * Clock-Architecture Diagram 1 + */ + + MUX(USB480M, "usb480m", mux_usb480m_p, CLK_SET_RATE_PARENT, + RK1808_MODE_CON, 10, 2, MFLAGS), + FACTOR(0, "xin12m", "xin24m", 0, 1, 2), + + /* + * Clock-Architecture Diagram 2 + */ + + GATE(0, "apll_core", "apll", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(0), 0, GFLAGS), + GATE(0, "cpll_core", "cpll", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(0), 0, GFLAGS), + GATE(0, "gpll_core", "gpll", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(0), 0, GFLAGS), + COMPOSITE_NOMUX(0, "pclk_core_dbg", "armclk", CLK_IGNORE_UNUSED, + RK1808_CLKSEL_CON(0), 8, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK1808_CLKGATE_CON(0), 3, GFLAGS), + COMPOSITE_NOMUX(0, "aclk_core", "armclk", CLK_IGNORE_UNUSED, + RK1808_CLKSEL_CON(0), 12, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK1808_CLKGATE_CON(0), 2, GFLAGS), + + GATE(0, "clk_jtag", "jtag_clkin", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(0), 4, GFLAGS), + + GATE(SCLK_PVTM_CORE, "clk_pvtm_core", "xin24m", 0, + RK1808_CLKGATE_CON(0), 5, GFLAGS), + + COMPOSITE_NOMUX(MSCLK_CORE_NIU, "msclk_core_niu", "gpll", CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(18), 0, 5, DFLAGS, + RK1808_CLKGATE_CON(0), 1, GFLAGS), + + /* + * Clock-Architecture Diagram 3 + */ + + COMPOSITE(ACLK_GIC_PRE, "aclk_gic_pre", mux_gpll_cpll_p, CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(15), 11, 1, MFLAGS, 12, 4, DFLAGS, + RK1808_CLKGATE_CON(1), 0, GFLAGS), + GATE(0, "aclk_gic_niu", "aclk_gic_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(1), 1, GFLAGS), + GATE(ACLK_GIC, "aclk_gic", "aclk_gic_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(1), 2, GFLAGS), + GATE(0, "aclk_core2gic", "aclk_gic_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(1), 3, GFLAGS), + GATE(0, "aclk_gic2core", "aclk_gic_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(1), 4, GFLAGS), + GATE(0, "aclk_spinlock", "aclk_gic_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(1), 4, GFLAGS), + + COMPOSITE(0, "aclk_vpu_pre", mux_gpll_cpll_p, 0, + RK1808_CLKSEL_CON(16), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK1808_CLKGATE_CON(8), 8, GFLAGS), + COMPOSITE_NOMUX(0, "hclk_vpu_pre", "aclk_vpu_pre", 0, + RK1808_CLKSEL_CON(16), 8, 4, DFLAGS, + RK1808_CLKGATE_CON(8), 9, GFLAGS), + GATE(ACLK_VPU, "aclk_vpu", "aclk_vpu_pre", 0, + RK1808_CLKGATE_CON(8), 12, GFLAGS), + GATE(0, "aclk_vpu_niu", "aclk_vpu_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(8), 10, GFLAGS), + GATE(HCLK_VPU, "hclk_vpu", "hclk_vpu_pre", 0, + RK1808_CLKGATE_CON(8), 13, GFLAGS), + GATE(0, "hclk_vpu_niu", "hclk_vpu_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(8), 11, GFLAGS), + + /* + * Clock-Architecture Diagram 4 + */ + COMPOSITE_NOGATE(0, "clk_npu_div", mux_gpll_cpll_p, CLK_OPS_PARENT_ENABLE, + RK1808_CLKSEL_CON(1), 8, 2, MFLAGS, 0, 4, DFLAGS), + COMPOSITE_NOGATE_HALFDIV(0, "clk_npu_np5", mux_gpll_cpll_p, CLK_OPS_PARENT_ENABLE, + RK1808_CLKSEL_CON(1), 10, 2, MFLAGS, 4, 4, DFLAGS), + MUX(0, "clk_npu_pre", mux_npu_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(1), 15, 1, MFLAGS), + FACTOR(0, "clk_npu_scan", "clk_npu_pre", 0, 1, 2), + GATE(SCLK_NPU, "clk_npu", "clk_npu_pre", 0, + RK1808_CLKGATE_CON(1), 10, GFLAGS), + + COMPOSITE(0, "aclk_npu_pre", mux_gpll_cpll_p, 0, + RK1808_CLKSEL_CON(2), 14, 1, MFLAGS, 0, 4, DFLAGS, + RK1808_CLKGATE_CON(1), 8, GFLAGS), + COMPOSITE(0, "hclk_npu_pre", mux_gpll_cpll_p, 0, + RK1808_CLKSEL_CON(2), 15, 1, MFLAGS, 8, 4, DFLAGS, + RK1808_CLKGATE_CON(1), 9, GFLAGS), + GATE(ACLK_NPU, "aclk_npu", "aclk_npu_pre", 0, + RK1808_CLKGATE_CON(1), 11, GFLAGS), + GATE(0, "aclk_npu_niu", "aclk_npu_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(1), 13, GFLAGS), + COMPOSITE_NOMUX(0, "aclk_npu2mem", "aclk_npu_pre", CLK_IGNORE_UNUSED, + RK1808_CLKSEL_CON(2), 4, 4, DFLAGS, + RK1808_CLKGATE_CON(1), 15, GFLAGS), + GATE(HCLK_NPU, "hclk_npu", "hclk_npu_pre", 0, + RK1808_CLKGATE_CON(1), 12, GFLAGS), + GATE(0, "hclk_npu_niu", "hclk_npu_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(1), 14, GFLAGS), + + GATE(SCLK_PVTM_NPU, "clk_pvtm_npu", "xin24m", 0, + RK1808_CLKGATE_CON(0), 15, GFLAGS), + + COMPOSITE(ACLK_IMEM_PRE, "aclk_imem_pre", mux_gpll_cpll_p, CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(17), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK1808_CLKGATE_CON(7), 0, GFLAGS), + GATE(ACLK_IMEM0, "aclk_imem0", "aclk_imem_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(7), 6, GFLAGS), + GATE(0, "aclk_imem0_niu", "aclk_imem_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(7), 10, GFLAGS), + GATE(ACLK_IMEM1, "aclk_imem1", "aclk_imem_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(7), 7, GFLAGS), + GATE(0, "aclk_imem1_niu", "aclk_imem_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(7), 11, GFLAGS), + GATE(ACLK_IMEM2, "aclk_imem2", "aclk_imem_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(7), 8, GFLAGS), + GATE(0, "aclk_imem2_niu", "aclk_imem_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(7), 12, GFLAGS), + GATE(ACLK_IMEM3, "aclk_imem3", "aclk_imem_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(7), 9, GFLAGS), + GATE(0, "aclk_imem3_niu", "aclk_imem_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(7), 13, GFLAGS), + + COMPOSITE(HSCLK_IMEM, "hsclk_imem", mux_gpll_cpll_p, CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(17), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK1808_CLKGATE_CON(7), 5, GFLAGS), + + /* + * Clock-Architecture Diagram 5 + */ + GATE(0, "clk_ddr_mon_timer", "xin24m", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 0, GFLAGS), + + GATE(0, "clk_ddr_mon", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 11, GFLAGS), + GATE(0, "aclk_split", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 15, GFLAGS), + GATE(0, "clk_ddr_msch", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 8, GFLAGS), + GATE(0, "clk_ddrdfi_ctl", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 3, GFLAGS), + GATE(0, "clk_stdby", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 13, GFLAGS), + GATE(0, "aclk_ddrc", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 5, GFLAGS), + GATE(0, "clk_core_ddrc", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 6, GFLAGS), + + GATE(0, "dpll_ddr", "dpll", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(8), 5, GFLAGS), + GATE(0, "gpll_ddr", "gpll", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(8), 6, GFLAGS), + + COMPOSITE_NOGATE(SCLK_DDRCLK, "sclk_ddrc", mux_ddr_p, CLK_IGNORE_UNUSED, + RK1808_CLKSEL_CON(3), 7, 1, MFLAGS, 0, 5, DFLAGS), + FACTOR(0, "clk_ddrphy1x_out", "sclk_ddrc", CLK_IGNORE_UNUSED, 1, 1), + + COMPOSITE_NOMUX(PCLK_DDR, "pclk_ddr", "gpll", CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(3), 8, 5, DFLAGS, + RK1808_CLKGATE_CON(2), 1, GFLAGS), + GATE(PCLK_DDRMON, "pclk_ddrmon", "pclk_ddr", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 10, GFLAGS), + GATE(PCLK_DDRC, "pclk_ddrc", "pclk_ddr", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 7, GFLAGS), + GATE(PCLK_MSCH, "pclk_msch", "pclk_ddr", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 9, GFLAGS), + GATE(PCLK_STDBY, "pclk_stdby", "pclk_ddr", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 12, GFLAGS), + GATE(0, "pclk_ddr_grf", "pclk_ddr", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(2), 14, GFLAGS), + GATE(0, "pclk_ddrdfi_ctl", "pclk_ddr", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(2), 2, GFLAGS), + + /* + * Clock-Architecture Diagram 6 + */ + + COMPOSITE(HSCLK_VIO, "hsclk_vio", mux_gpll_cpll_p, 0, + RK1808_CLKSEL_CON(4), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK1808_CLKGATE_CON(3), 0, GFLAGS), + COMPOSITE_NOMUX(LSCLK_VIO, "lsclk_vio", "hsclk_vio", 0, + RK1808_CLKSEL_CON(4), 8, 4, DFLAGS, + RK1808_CLKGATE_CON(3), 12, GFLAGS), + GATE(0, "hsclk_vio_niu", "hsclk_vio", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(4), 0, GFLAGS), + GATE(0, "lsclk_vio_niu", "lsclk_vio", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(4), 1, GFLAGS), + GATE(ACLK_VOPRAW, "aclk_vopraw", "hsclk_vio", 0, + RK1808_CLKGATE_CON(4), 2, GFLAGS), + GATE(HCLK_VOPRAW, "hclk_vopraw", "lsclk_vio", 0, + RK1808_CLKGATE_CON(4), 3, GFLAGS), + GATE(ACLK_VOPLITE, "aclk_voplite", "hsclk_vio", 0, + RK1808_CLKGATE_CON(4), 4, GFLAGS), + GATE(HCLK_VOPLITE, "hclk_voplite", "lsclk_vio", 0, + RK1808_CLKGATE_CON(4), 5, GFLAGS), + GATE(PCLK_DSI_TX, "pclk_dsi_tx", "lsclk_vio", 0, + RK1808_CLKGATE_CON(4), 6, GFLAGS), + GATE(PCLK_CSI_TX, "pclk_csi_tx", "lsclk_vio", 0, + RK1808_CLKGATE_CON(4), 7, GFLAGS), + GATE(ACLK_RGA, "aclk_rga", "hsclk_vio", 0, + RK1808_CLKGATE_CON(4), 8, GFLAGS), + GATE(HCLK_RGA, "hclk_rga", "lsclk_vio", 0, + RK1808_CLKGATE_CON(4), 9, GFLAGS), + GATE(ACLK_ISP, "aclk_isp", "hsclk_vio", 0, + RK1808_CLKGATE_CON(4), 13, GFLAGS), + GATE(HCLK_ISP, "hclk_isp", "lsclk_vio", 0, + RK1808_CLKGATE_CON(4), 14, GFLAGS), + GATE(ACLK_CIF, "aclk_cif", "hsclk_vio", 0, + RK1808_CLKGATE_CON(4), 10, GFLAGS), + GATE(HCLK_CIF, "hclk_cif", "lsclk_vio", 0, + RK1808_CLKGATE_CON(4), 11, GFLAGS), + GATE(PCLK_CSI2HOST, "pclk_csi2host", "lsclk_vio", 0, + RK1808_CLKGATE_CON(4), 12, GFLAGS), + + COMPOSITE(0, "dclk_vopraw_src", mux_cpll_gpll_npll_p, 0, + RK1808_CLKSEL_CON(5), 10, 2, MFLAGS, 0, 8, DFLAGS, + RK1808_CLKGATE_CON(3), 1, GFLAGS), + COMPOSITE_FRACMUX(0, "dclk_vopraw_frac", "dclk_vopraw_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(6), 0, + RK1808_CLKGATE_CON(3), 2, GFLAGS, + &rk1808_dclk_vopraw_fracmux, RK1808_VOP_RAW_FRAC_MAX_PRATE), + GATE(DCLK_VOPRAW, "dclk_vopraw", "dclk_vopraw_mux", 0, + RK1808_CLKGATE_CON(3), 3, GFLAGS), + + COMPOSITE(0, "dclk_voplite_src", mux_cpll_gpll_npll_p, 0, + RK1808_CLKSEL_CON(7), 10, 2, MFLAGS, 0, 8, DFLAGS, + RK1808_CLKGATE_CON(3), 4, GFLAGS), + COMPOSITE_FRACMUX(0, "dclk_voplite_frac", "dclk_voplite_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(8), 0, + RK1808_CLKGATE_CON(3), 5, GFLAGS, + &rk1808_dclk_voplite_fracmux, RK1808_VOP_LITE_FRAC_MAX_PRATE), + GATE(DCLK_VOPLITE, "dclk_voplite", "dclk_voplite_mux", 0, + RK1808_CLKGATE_CON(3), 6, GFLAGS), + + COMPOSITE_NOMUX(SCLK_TXESC, "clk_txesc", "gpll", 0, + RK1808_CLKSEL_CON(9), 0, 12, DFLAGS, + RK1808_CLKGATE_CON(3), 7, GFLAGS), + + COMPOSITE(SCLK_RGA, "clk_rga", mux_gpll_cpll_npll_p, 0, + RK1808_CLKSEL_CON(10), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK1808_CLKGATE_CON(3), 8, GFLAGS), + + COMPOSITE(SCLK_ISP, "clk_isp", mux_gpll_cpll_npll_p, 0, + RK1808_CLKSEL_CON(10), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK1808_CLKGATE_CON(3), 10, GFLAGS), + + COMPOSITE(DCLK_CIF, "dclk_cif", mux_cpll_gpll_npll_p, 0, + RK1808_CLKSEL_CON(11), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK1808_CLKGATE_CON(3), 11, GFLAGS), + + COMPOSITE(SCLK_CIF_OUT, "clk_cif_out", mux_24m_npll_gpll_usb480m_p, 0, + RK1808_CLKSEL_CON(11), 6, 2, MFLAGS, 0, 6, DFLAGS, + RK1808_CLKGATE_CON(3), 9, GFLAGS), + + /* + * Clock-Architecture Diagram 7 + */ + + /* PD_PCIE */ + COMPOSITE_NODIV(0, "clk_pcie_src", mux_gpll_cpll_p, 0, + RK1808_CLKSEL_CON(12), 15, 1, MFLAGS, + RK1808_CLKGATE_CON(5), 0, GFLAGS), + DIV(HSCLK_PCIE, "hsclk_pcie", "clk_pcie_src", 0, + RK1808_CLKSEL_CON(12), 0, 5, DFLAGS), + DIV(LSCLK_PCIE, "lsclk_pcie", "clk_pcie_src", 0, + RK1808_CLKSEL_CON(12), 8, 5, DFLAGS), + GATE(0, "hsclk_pcie_niu", "hsclk_pcie", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(6), 0, GFLAGS), + GATE(0, "lsclk_pcie_niu", "lsclk_pcie", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(6), 1, GFLAGS), + GATE(0, "pclk_pcie_grf", "lsclk_pcie", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(6), 5, GFLAGS), + GATE(ACLK_USB3OTG, "aclk_usb3otg", "hsclk_pcie", 0, + RK1808_CLKGATE_CON(6), 6, GFLAGS), + GATE(HCLK_HOST, "hclk_host", "lsclk_pcie", 0, + RK1808_CLKGATE_CON(6), 7, GFLAGS), + GATE(HCLK_HOST_ARB, "hclk_host_arb", "lsclk_pcie", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(6), 8, GFLAGS), + + COMPOSITE(ACLK_PCIE, "aclk_pcie", mux_gpll_cpll_p, 0, + RK1808_CLKSEL_CON(15), 8, 1, MFLAGS, 0, 4, DFLAGS, + RK1808_CLKGATE_CON(5), 5, GFLAGS), + DIV(0, "pclk_pcie_pre", "aclk_pcie", 0, + RK1808_CLKSEL_CON(15), 4, 4, DFLAGS), + GATE(0, "aclk_pcie_niu", "aclk_pcie", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(6), 10, GFLAGS), + GATE(ACLK_PCIE_MST, "aclk_pcie_mst", "aclk_pcie", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(6), 2, GFLAGS), + GATE(ACLK_PCIE_SLV, "aclk_pcie_slv", "aclk_pcie", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(6), 3, GFLAGS), + GATE(0, "pclk_pcie_niu", "pclk_pcie_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(6), 11, GFLAGS), + GATE(0, "pclk_pcie_dbi", "pclk_pcie_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(6), 4, GFLAGS), + GATE(PCLK_PCIE, "pclk_pcie", "pclk_pcie_pre", 0, + RK1808_CLKGATE_CON(6), 9, GFLAGS), + + COMPOSITE(0, "clk_pcie_aux_src", mux_cpll_gpll_npll_p, 0, + RK1808_CLKSEL_CON(14), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(5), 3, GFLAGS), + COMPOSITE_NODIV(SCLK_PCIE_AUX, "clk_pcie_aux", mux_pcie_aux_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(14), 12, 1, MFLAGS, + RK1808_CLKGATE_CON(5), 4, GFLAGS), + + GATE(SCLK_USB3_OTG0_REF, "clk_usb3_otg0_ref", "xin24m", 0, + RK1808_CLKGATE_CON(5), 1, GFLAGS), + + COMPOSITE(SCLK_USB3_OTG0_SUSPEND, "clk_usb3_otg0_suspend", mux_usb3_otg0_suspend_p, 0, + RK1808_CLKSEL_CON(13), 12, 1, MFLAGS, 0, 10, DFLAGS, + RK1808_CLKGATE_CON(5), 2, GFLAGS), + + /* + * Clock-Architecture Diagram 8 + */ + + /* PD_PHP */ + + COMPOSITE_NODIV(0, "clk_peri_src", mux_gpll_cpll_p, CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(19), 15, 1, MFLAGS, + RK1808_CLKGATE_CON(8), 0, GFLAGS), + COMPOSITE_NOMUX(MSCLK_PERI, "msclk_peri", "clk_peri_src", CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(19), 0, 5, DFLAGS, + RK1808_CLKGATE_CON(8), 1, GFLAGS), + COMPOSITE_NOMUX(LSCLK_PERI, "lsclk_peri", "clk_peri_src", CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(19), 8, 5, DFLAGS, + RK1808_CLKGATE_CON(8), 2, GFLAGS), + GATE(0, "msclk_peri_niu", "msclk_peri", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(8), 3, GFLAGS), + GATE(0, "lsclk_peri_niu", "lsclk_peri", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(8), 4, GFLAGS), + + /* PD_MMC */ + + GATE(0, "hclk_mmc_sfc", "msclk_peri", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(9), 0, GFLAGS), + GATE(0, "hclk_mmc_sfc_niu", "hclk_mmc_sfc", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(9), 11, GFLAGS), + GATE(HCLK_EMMC, "hclk_emmc", "hclk_mmc_sfc", 0, + RK1808_CLKGATE_CON(9), 12, GFLAGS), + GATE(HCLK_SFC, "hclk_sfc", "hclk_mmc_sfc", 0, + RK1808_CLKGATE_CON(9), 13, GFLAGS), + + COMPOSITE(SCLK_SDIO_DIV, "clk_sdio_div", mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, + RK1808_CLKSEL_CON(22), 14, 2, MFLAGS, 0, 8, DFLAGS, + RK1808_CLKGATE_CON(9), 1, GFLAGS), + COMPOSITE_DIV_OFFSET(SCLK_SDIO_DIV50, "clk_sdio_div50", + mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, + RK1808_CLKSEL_CON(22), 14, 2, MFLAGS, + RK1808_CLKSEL_CON(23), 0, 8, DFLAGS, + RK1808_CLKGATE_CON(9), 2, GFLAGS), + COMPOSITE_NODIV(SCLK_SDIO, "clk_sdio", mux_sdio_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK1808_CLKSEL_CON(23), 15, 1, MFLAGS, + RK1808_CLKGATE_CON(9), 3, GFLAGS), + + MMC(SCLK_SDIO_DRV, "sdio_drv", "clk_sdio", RK1808_SDIO_CON0, 1), + MMC(SCLK_SDIO_SAMPLE, "sdio_sample", "clk_sdio", RK1808_SDIO_CON1, 1), + + COMPOSITE(SCLK_EMMC_DIV, "clk_emmc_div", + mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, + RK1808_CLKSEL_CON(24), 14, 2, MFLAGS, 0, 8, DFLAGS, + RK1808_CLKGATE_CON(9), 4, GFLAGS), + COMPOSITE_DIV_OFFSET(SCLK_EMMC_DIV50, "clk_emmc_div50", mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, + RK1808_CLKSEL_CON(24), 14, 2, MFLAGS, + RK1808_CLKSEL_CON(25), 0, 8, DFLAGS, + RK1808_CLKGATE_CON(9), 5, GFLAGS), + COMPOSITE_NODIV(SCLK_EMMC, "clk_emmc", mux_emmc_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK1808_CLKSEL_CON(25), 15, 1, MFLAGS, + RK1808_CLKGATE_CON(9), 6, GFLAGS), + MMC(SCLK_EMMC_DRV, "emmc_drv", "clk_emmc", RK1808_EMMC_CON0, 1), + MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "clk_emmc", RK1808_EMMC_CON1, 1), + + COMPOSITE(SCLK_SDMMC_DIV, "clk_sdmmc_div", mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, + RK1808_CLKSEL_CON(20), 14, 2, MFLAGS, 0, 8, DFLAGS, + RK1808_CLKGATE_CON(9), 7, GFLAGS), + COMPOSITE_DIV_OFFSET(SCLK_SDMMC_DIV50, "clk_sdmmc_div50", + mux_gpll_cpll_npll_24m_p, CLK_IGNORE_UNUSED, + RK1808_CLKSEL_CON(20), 14, 2, MFLAGS, + RK1808_CLKSEL_CON(21), 0, 8, DFLAGS, + RK1808_CLKGATE_CON(9), 8, GFLAGS), + COMPOSITE_NODIV(SCLK_SDMMC, "clk_sdmmc", mux_sdmmc_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK1808_CLKSEL_CON(21), 15, 1, MFLAGS, + RK1808_CLKGATE_CON(9), 9, GFLAGS), + MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "clk_sdmmc", RK1808_SDMMC_CON0, 1), + MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "clk_sdmmc", RK1808_SDMMC_CON1, 1), + + COMPOSITE(SCLK_SFC, "clk_sfc", mux_gpll_cpll_p, 0, + RK1808_CLKSEL_CON(26), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(9), 10, GFLAGS), + + /* PD_MAC */ + + GATE(0, "pclk_sd_gmac", "lsclk_peri", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(10), 2, GFLAGS), + GATE(0, "aclk_sd_gmac", "msclk_peri", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(10), 0, GFLAGS), + GATE(0, "hclk_sd_gmac", "msclk_peri", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(10), 1, GFLAGS), + GATE(0, "pclk_gmac_niu", "pclk_sd_gmac", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(10), 10, GFLAGS), + GATE(PCLK_GMAC, "pclk_gmac", "pclk_sd_gmac", 0, + RK1808_CLKGATE_CON(10), 12, GFLAGS), + GATE(0, "aclk_gmac_niu", "aclk_sd_gmac", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(10), 8, GFLAGS), + GATE(ACLK_GMAC, "aclk_gmac", "aclk_sd_gmac", 0, + RK1808_CLKGATE_CON(10), 11, GFLAGS), + GATE(0, "hclk_gmac_niu", "hclk_sd_gmac", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(10), 9, GFLAGS), + GATE(HCLK_SDIO, "hclk_sdio", "hclk_sd_gmac", 0, + RK1808_CLKGATE_CON(10), 13, GFLAGS), + GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_sd_gmac", 0, + RK1808_CLKGATE_CON(10), 14, GFLAGS), + + COMPOSITE(SCLK_GMAC_OUT, "clk_gmac_out", mux_cpll_npll_ppll_p, 0, + RK1808_CLKSEL_CON(18), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK1808_CLKGATE_CON(10), 15, GFLAGS), + + COMPOSITE(SCLK_GMAC_SRC, "clk_gmac_src", mux_cpll_npll_ppll_p, 0, + RK1808_CLKSEL_CON(26), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK1808_CLKGATE_CON(10), 3, GFLAGS), + MUX(SCLK_GMAC, "clk_gmac", mux_gmac_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK1808_CLKSEL_CON(27), 0, 1, MFLAGS), + GATE(SCLK_GMAC_REF, "clk_gmac_ref", "clk_gmac", 0, + RK1808_CLKGATE_CON(10), 4, GFLAGS), + GATE(0, "clk_gmac_tx_src", "clk_gmac", 0, + RK1808_CLKGATE_CON(10), 7, GFLAGS), + GATE(0, "clk_gmac_rx_src", "clk_gmac", 0, + RK1808_CLKGATE_CON(10), 6, GFLAGS), + GATE(SCLK_GMAC_REFOUT, "clk_gmac_refout", "clk_gmac", 0, + RK1808_CLKGATE_CON(10), 5, GFLAGS), + FACTOR(0, "clk_gmac_tx_div5", "clk_gmac_tx_src", 0, 1, 5), + FACTOR(0, "clk_gmac_tx_div50", "clk_gmac_tx_src", 0, 1, 50), + FACTOR(0, "clk_gmac_rx_div2", "clk_gmac_rx_src", 0, 1, 2), + FACTOR(0, "clk_gmac_rx_div20", "clk_gmac_rx_src", 0, 1, 20), + MUX(SCLK_GMAC_RGMII_SPEED, "clk_gmac_rgmii_speed", mux_gmac_rgmii_speed_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(27), 2, 2, MFLAGS), + MUX(SCLK_GMAC_RMII_SPEED, "clk_gmac_rmii_speed", mux_gmac_rmii_speed_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(27), 1, 1, MFLAGS), + MUX(SCLK_GMAC_RX_TX, "clk_gmac_rx_tx", mux_gmac_rx_tx_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(27), 4, 1, MFLAGS), + + /* + * Clock-Architecture Diagram 9 + */ + + /* PD_BUS */ + + COMPOSITE_NODIV(0, "clk_bus_src", mux_gpll_cpll_p, CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(27), 15, 1, MFLAGS, + RK1808_CLKGATE_CON(11), 0, GFLAGS), + COMPOSITE_NOMUX(HSCLK_BUS_PRE, "hsclk_bus_pre", "clk_bus_src", CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(27), 8, 5, DFLAGS, + RK1808_CLKGATE_CON(11), 1, GFLAGS), + COMPOSITE_NOMUX(MSCLK_BUS_PRE, "msclk_bus_pre", "clk_bus_src", CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(28), 0, 5, DFLAGS, + RK1808_CLKGATE_CON(11), 2, GFLAGS), + COMPOSITE_NOMUX(LSCLK_BUS_PRE, "lsclk_bus_pre", "clk_bus_src", CLK_IS_CRITICAL, + RK1808_CLKSEL_CON(28), 8, 5, DFLAGS, + RK1808_CLKGATE_CON(11), 3, GFLAGS), + GATE(0, "hsclk_bus_niu", "hsclk_bus_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(15), 0, GFLAGS), + GATE(0, "msclk_bus_niu", "msclk_bus_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(15), 1, GFLAGS), + GATE(0, "msclk_sub", "msclk_bus_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(15), 2, GFLAGS), + GATE(ACLK_DMAC, "aclk_dmac", "msclk_bus_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(14), 15, GFLAGS), + GATE(HCLK_ROM, "hclk_rom", "msclk_bus_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(15), 4, GFLAGS), + GATE(ACLK_CRYPTO, "aclk_crypto", "msclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 5, GFLAGS), + GATE(HCLK_CRYPTO, "hclk_crypto", "msclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 6, GFLAGS), + GATE(ACLK_DCF, "aclk_dcf", "msclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 7, GFLAGS), + GATE(0, "lsclk_bus_niu", "lsclk_bus_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(15), 3, GFLAGS), + GATE(PCLK_DCF, "pclk_dcf", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 8, GFLAGS), + GATE(PCLK_UART1, "pclk_uart1", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 9, GFLAGS), + GATE(PCLK_UART2, "pclk_uart2", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 10, GFLAGS), + GATE(PCLK_UART3, "pclk_uart3", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 11, GFLAGS), + GATE(PCLK_UART4, "pclk_uart4", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 12, GFLAGS), + GATE(PCLK_UART5, "pclk_uart5", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 13, GFLAGS), + GATE(PCLK_UART6, "pclk_uart6", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 14, GFLAGS), + GATE(PCLK_UART7, "pclk_uart7", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(15), 15, GFLAGS), + GATE(PCLK_I2C1, "pclk_i2c1", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 0, GFLAGS), + GATE(PCLK_I2C2, "pclk_i2c2", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 1, GFLAGS), + GATE(PCLK_I2C3, "pclk_i2c3", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 2, GFLAGS), + GATE(PCLK_I2C4, "pclk_i2c4", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(17), 4, GFLAGS), + GATE(PCLK_I2C5, "pclk_i2c5", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(17), 5, GFLAGS), + GATE(PCLK_SPI0, "pclk_spi0", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 3, GFLAGS), + GATE(PCLK_SPI1, "pclk_spi1", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 4, GFLAGS), + GATE(PCLK_SPI2, "pclk_spi2", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 5, GFLAGS), + GATE(PCLK_TSADC, "pclk_tsadc", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 9, GFLAGS), + GATE(PCLK_SARADC, "pclk_saradc", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 10, GFLAGS), + GATE(PCLK_EFUSE, "pclk_efuse", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 11, GFLAGS), + GATE(PCLK_GPIO1, "pclk_gpio1", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 12, GFLAGS), + GATE(PCLK_GPIO2, "pclk_gpio2", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 13, GFLAGS), + GATE(PCLK_GPIO3, "pclk_gpio3", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 14, GFLAGS), + GATE(PCLK_GPIO4, "pclk_gpio4", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 15, GFLAGS), + GATE(PCLK_PWM0, "pclk_pwm0", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 6, GFLAGS), + GATE(PCLK_PWM1, "pclk_pwm1", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 7, GFLAGS), + GATE(PCLK_PWM2, "pclk_pwm2", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(16), 8, GFLAGS), + GATE(PCLK_TIMER, "pclk_timer", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(17), 0, GFLAGS), + GATE(PCLK_WDT, "pclk_wdt", "lsclk_bus_pre", 0, + RK1808_CLKGATE_CON(17), 1, GFLAGS), + GATE(0, "pclk_grf", "lsclk_bus_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(17), 2, GFLAGS), + GATE(0, "pclk_sgrf", "lsclk_bus_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(17), 3, GFLAGS), + GATE(0, "hclk_audio_pre", "msclk_bus_pre", 0, + RK1808_CLKGATE_CON(17), 8, GFLAGS), + GATE(0, "pclk_top_pre", "lsclk_bus_pre", CLK_IS_CRITICAL, + RK1808_CLKGATE_CON(11), 4, GFLAGS), + + COMPOSITE(SCLK_CRYPTO, "clk_crypto", mux_gpll_cpll_p, 0, + RK1808_CLKSEL_CON(29), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK1808_CLKGATE_CON(11), 5, GFLAGS), + COMPOSITE(SCLK_CRYPTO_APK, "clk_crypto_apk", mux_gpll_cpll_p, 0, + RK1808_CLKSEL_CON(29), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK1808_CLKGATE_CON(11), 6, GFLAGS), + + COMPOSITE(0, "clk_uart1_src", mux_gpll_usb480m_cpll_npll_p, 0, + RK1808_CLKSEL_CON(38), 14, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(11), 8, GFLAGS), + COMPOSITE_NOMUX_HALFDIV(0, "clk_uart1_np5", "clk_uart1_src", 0, + RK1808_CLKSEL_CON(39), 0, 7, DFLAGS, + RK1808_CLKGATE_CON(11), 9, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(40), 0, + RK1808_CLKGATE_CON(11), 10, GFLAGS, + &rk1808_uart1_fracmux, RK1808_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART1, "clk_uart1", "clk_uart1_mux", 0, + RK1808_CLKGATE_CON(11), 11, GFLAGS), + + COMPOSITE(0, "clk_uart2_src", mux_gpll_usb480m_cpll_npll_p, 0, + RK1808_CLKSEL_CON(41), 14, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(11), 12, GFLAGS), + COMPOSITE_NOMUX_HALFDIV(0, "clk_uart2_np5", "clk_uart2_src", 0, + RK1808_CLKSEL_CON(42), 0, 7, DFLAGS, + RK1808_CLKGATE_CON(11), 13, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(43), 0, + RK1808_CLKGATE_CON(11), 14, GFLAGS, + &rk1808_uart2_fracmux, RK1808_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART2, "clk_uart2", "clk_uart2_mux", 0, + RK1808_CLKGATE_CON(11), 15, GFLAGS), + + COMPOSITE(0, "clk_uart3_src", mux_gpll_usb480m_cpll_npll_p, 0, + RK1808_CLKSEL_CON(44), 14, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(12), 0, GFLAGS), + COMPOSITE_NOMUX_HALFDIV(0, "clk_uart3_np5", "clk_uart3_src", 0, + RK1808_CLKSEL_CON(45), 0, 7, DFLAGS, + RK1808_CLKGATE_CON(12), 1, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(46), 0, + RK1808_CLKGATE_CON(12), 2, GFLAGS, + &rk1808_uart3_fracmux, RK1808_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART3, "clk_uart3", "clk_uart3_mux", 0, + RK1808_CLKGATE_CON(12), 3, GFLAGS), + + COMPOSITE(0, "clk_uart4_src", mux_gpll_usb480m_cpll_npll_p, 0, + RK1808_CLKSEL_CON(47), 14, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(12), 4, GFLAGS), + COMPOSITE_NOMUX_HALFDIV(0, "clk_uart4_np5", "clk_uart4_src", 0, + RK1808_CLKSEL_CON(48), 0, 7, DFLAGS, + RK1808_CLKGATE_CON(12), 5, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(49), 0, + RK1808_CLKGATE_CON(12), 6, GFLAGS, + &rk1808_uart4_fracmux, RK1808_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART4, "clk_uart4", "clk_uart4_mux", 0, + RK1808_CLKGATE_CON(12), 7, GFLAGS), + + COMPOSITE(0, "clk_uart5_src", mux_gpll_usb480m_cpll_npll_p, 0, + RK1808_CLKSEL_CON(50), 14, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(12), 8, GFLAGS), + COMPOSITE_NOMUX_HALFDIV(0, "clk_uart5_np5", "clk_uart5_src", 0, + RK1808_CLKSEL_CON(51), 0, 7, DFLAGS, + RK1808_CLKGATE_CON(12), 9, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart5_frac", "clk_uart5_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(52), 0, + RK1808_CLKGATE_CON(12), 10, GFLAGS, + &rk1808_uart5_fracmux, RK1808_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART5, "clk_uart5", "clk_uart5_mux", 0, + RK1808_CLKGATE_CON(12), 11, GFLAGS), + + COMPOSITE(0, "clk_uart6_src", mux_gpll_usb480m_cpll_npll_p, 0, + RK1808_CLKSEL_CON(53), 14, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(12), 12, GFLAGS), + COMPOSITE_NOMUX_HALFDIV(0, "clk_uart6_np5", "clk_uart6_src", 0, + RK1808_CLKSEL_CON(54), 0, 7, DFLAGS, + RK1808_CLKGATE_CON(12), 13, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart6_frac", "clk_uart6_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(55), 0, + RK1808_CLKGATE_CON(12), 14, GFLAGS, + &rk1808_uart6_fracmux, RK1808_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART6, "clk_uart6", "clk_uart6_mux", 0, + RK1808_CLKGATE_CON(12), 15, GFLAGS), + + COMPOSITE(0, "clk_uart7_src", mux_gpll_usb480m_cpll_npll_p, 0, + RK1808_CLKSEL_CON(56), 14, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 0, GFLAGS), + COMPOSITE_NOMUX_HALFDIV(0, "clk_uart7_np5", "clk_uart7_src", 0, + RK1808_CLKSEL_CON(57), 0, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 1, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart7_frac", "clk_uart7_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(58), 0, + RK1808_CLKGATE_CON(13), 2, GFLAGS, + &rk1808_uart7_fracmux, RK1808_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART7, "clk_uart7", "clk_uart7_mux", 0, + RK1808_CLKGATE_CON(13), 3, GFLAGS), + + COMPOSITE(SCLK_I2C1, "clk_i2c1", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(59), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 4, GFLAGS), + COMPOSITE(SCLK_I2C2, "clk_i2c2", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(59), 15, 1, MFLAGS, 8, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 5, GFLAGS), + COMPOSITE(SCLK_I2C3, "clk_i2c3", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(60), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 6, GFLAGS), + COMPOSITE(SCLK_I2C4, "clk_i2c4", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(71), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(14), 6, GFLAGS), + COMPOSITE(SCLK_I2C5, "clk_i2c5", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(71), 15, 1, MFLAGS, 8, 7, DFLAGS, + RK1808_CLKGATE_CON(14), 7, GFLAGS), + + COMPOSITE(SCLK_SPI0, "clk_spi0", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(60), 15, 1, MFLAGS, 8, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 7, GFLAGS), + COMPOSITE(SCLK_SPI1, "clk_spi1", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(61), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 8, GFLAGS), + COMPOSITE(SCLK_SPI2, "clk_spi2", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(61), 15, 1, MFLAGS, 8, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 9, GFLAGS), + + COMPOSITE_NOMUX(SCLK_TSADC, "clk_tsadc", "xin24m", 0, + RK1808_CLKSEL_CON(62), 0, 11, DFLAGS, + RK1808_CLKGATE_CON(13), 13, GFLAGS), + COMPOSITE_NOMUX(SCLK_SARADC, "clk_saradc", "xin24m", 0, + RK1808_CLKSEL_CON(63), 0, 11, DFLAGS, + RK1808_CLKGATE_CON(13), 14, GFLAGS), + + COMPOSITE(SCLK_EFUSE_S, "clk_efuse_s", mux_gpll_cpll_xin24m_p, 0, + RK1808_CLKSEL_CON(64), 6, 2, MFLAGS, 0, 6, DFLAGS, + RK1808_CLKGATE_CON(14), 0, GFLAGS), + COMPOSITE(SCLK_EFUSE_NS, "clk_efuse_ns", mux_gpll_cpll_xin24m_p, 0, + RK1808_CLKSEL_CON(64), 14, 2, MFLAGS, 8, 6, DFLAGS, + RK1808_CLKGATE_CON(14), 1, GFLAGS), + + COMPOSITE(DBCLK_GPIO1, "dbclk_gpio1", mux_xin24m_32k_p, 0, + RK1808_CLKSEL_CON(65), 15, 1, MFLAGS, 0, 11, DFLAGS, + RK1808_CLKGATE_CON(14), 2, GFLAGS), + COMPOSITE(DBCLK_GPIO2, "dbclk_gpio2", mux_xin24m_32k_p, 0, + RK1808_CLKSEL_CON(66), 15, 1, MFLAGS, 0, 11, DFLAGS, + RK1808_CLKGATE_CON(14), 3, GFLAGS), + COMPOSITE(DBCLK_GPIO3, "dbclk_gpio3", mux_xin24m_32k_p, 0, + RK1808_CLKSEL_CON(67), 15, 1, MFLAGS, 0, 11, DFLAGS, + RK1808_CLKGATE_CON(14), 4, GFLAGS), + COMPOSITE(DBCLK_GPIO4, "dbclk_gpio4", mux_xin24m_32k_p, 0, + RK1808_CLKSEL_CON(68), 15, 1, MFLAGS, 0, 11, DFLAGS, + RK1808_CLKGATE_CON(14), 5, GFLAGS), + + COMPOSITE(SCLK_PWM0, "clk_pwm0", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(69), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 10, GFLAGS), + COMPOSITE(SCLK_PWM1, "clk_pwm1", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(69), 15, 1, MFLAGS, 8, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 11, GFLAGS), + COMPOSITE(SCLK_PWM2, "clk_pwm2", mux_gpll_xin24m_p, 0, + RK1808_CLKSEL_CON(70), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(13), 12, GFLAGS), + + GATE(SCLK_TIMER0, "sclk_timer0", "xin24m", 0, + RK1808_CLKGATE_CON(14), 8, GFLAGS), + GATE(SCLK_TIMER1, "sclk_timer1", "xin24m", 0, + RK1808_CLKGATE_CON(14), 9, GFLAGS), + GATE(SCLK_TIMER2, "sclk_timer2", "xin24m", 0, + RK1808_CLKGATE_CON(14), 10, GFLAGS), + GATE(SCLK_TIMER3, "sclk_timer3", "xin24m", 0, + RK1808_CLKGATE_CON(14), 11, GFLAGS), + GATE(SCLK_TIMER4, "sclk_timer4", "xin24m", 0, + RK1808_CLKGATE_CON(14), 12, GFLAGS), + GATE(SCLK_TIMER5, "sclk_timer5", "xin24m", 0, + RK1808_CLKGATE_CON(14), 13, GFLAGS), + + /* + * Clock-Architecture Diagram 10 + */ + + /* PD_AUDIO */ + + GATE(0, "hclk_audio_niu", "hclk_audio_pre", CLK_IGNORE_UNUSED, + RK1808_CLKGATE_CON(18), 11, GFLAGS), + GATE(HCLK_VAD, "hclk_vad", "hclk_audio_pre", 0, + RK1808_CLKGATE_CON(18), 12, GFLAGS), + GATE(HCLK_PDM, "hclk_pdm", "hclk_audio_pre", 0, + RK1808_CLKGATE_CON(18), 13, GFLAGS), + GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_audio_pre", 0, + RK1808_CLKGATE_CON(18), 14, GFLAGS), + GATE(HCLK_I2S1_2CH, "hclk_i2s1_2ch", "hclk_audio_pre", 0, + RK1808_CLKGATE_CON(18), 15, GFLAGS), + + COMPOSITE(0, "clk_pdm_src", mux_gpll_xin24m_cpll_npll_p, 0, + RK1808_CLKSEL_CON(30), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(17), 9, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_pdm_frac", "clk_pdm_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(31), 0, + RK1808_CLKGATE_CON(17), 10, GFLAGS, + &rk1808_pdm_fracmux, RK1808_PDM_FRAC_MAX_PRATE), + GATE(SCLK_PDM, "clk_pdm", "clk_pdm_mux", 0, + RK1808_CLKGATE_CON(17), 11, GFLAGS), + + COMPOSITE(SCLK_I2S0_8CH_TX_SRC, "clk_i2s0_8ch_tx_src", mux_gpll_cpll_npll_p, 0, + RK1808_CLKSEL_CON(32), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(17), 12, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_i2s0_8ch_tx_frac", "clk_i2s0_8ch_tx_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(33), 0, + RK1808_CLKGATE_CON(17), 13, GFLAGS, + &rk1808_i2s0_8ch_tx_fracmux, RK1808_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S0_8CH_TX, "clk_i2s0_8ch_tx", mux_i2s0_8ch_tx_rx_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(32), 12, 1, MFLAGS, + RK1808_CLKGATE_CON(17), 14, GFLAGS), + COMPOSITE_NODIV(SCLK_I2S0_8CH_TX_OUT, "clk_i2s0_8ch_tx_out", mux_i2s0_8ch_tx_out_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(32), 14, 2, MFLAGS, + RK1808_CLKGATE_CON(17), 15, GFLAGS), + + COMPOSITE(SCLK_I2S0_8CH_RX_SRC, "clk_i2s0_8ch_rx_src", mux_gpll_cpll_npll_p, 0, + RK1808_CLKSEL_CON(34), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(18), 0, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_i2s0_8ch_rx_frac", "clk_i2s0_8ch_rx_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(35), 0, + RK1808_CLKGATE_CON(18), 1, GFLAGS, + &rk1808_i2s0_8ch_rx_fracmux, RK1808_I2S_FRAC_MAX_PRATE), + COMPOSITE_NODIV(SCLK_I2S0_8CH_RX, "clk_i2s0_8ch_rx", mux_i2s0_8ch_rx_tx_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(34), 12, 1, MFLAGS, + RK1808_CLKGATE_CON(18), 2, GFLAGS), + COMPOSITE_NODIV(SCLK_I2S0_8CH_RX_OUT, "clk_i2s0_8ch_rx_out", mux_i2s0_8ch_rx_out_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(34), 14, 2, MFLAGS, + RK1808_CLKGATE_CON(18), 3, GFLAGS), + + COMPOSITE(SCLK_I2S1_2CH_SRC, "clk_i2s1_2ch_src", mux_gpll_cpll_npll_p, 0, + RK1808_CLKSEL_CON(36), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_CLKGATE_CON(18), 4, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_i2s1_2ch_frac", "clk_i2s1_2ch_src", CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(37), 0, + RK1808_CLKGATE_CON(18), 5, GFLAGS, + &rk1808_i2s1_2ch_fracmux, RK1808_I2S_FRAC_MAX_PRATE), + GATE(SCLK_I2S1_2CH, "clk_i2s1_2ch", "clk_i2s1_2ch_mux", 0, + RK1808_CLKGATE_CON(18), 6, GFLAGS), + COMPOSITE_NODIV(SCLK_I2S1_2CH_OUT, "clk_i2s1_2ch_out", mux_i2s1_2ch_out_p, CLK_SET_RATE_PARENT, + RK1808_CLKSEL_CON(36), 15, 1, MFLAGS, + RK1808_CLKGATE_CON(18), 7, GFLAGS), + + /* + * Clock-Architecture Diagram 10 + */ + + /* PD_BUS */ + + GATE(0, "pclk_top_niu", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 0, GFLAGS), + GATE(0, "pclk_top_cru", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 1, GFLAGS), + GATE(0, "pclk_ddrphy", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 2, GFLAGS), + GATE(PCLK_MIPIDSIPHY, "pclk_mipidsiphy", "pclk_top_pre", 0, RK1808_CLKGATE_CON(19), 3, GFLAGS), + GATE(PCLK_MIPICSIPHY, "pclk_mipicsiphy", "pclk_top_pre", 0, RK1808_CLKGATE_CON(19), 4, GFLAGS), + + GATE(PCLK_USB3PHY_PIPE, "pclk_usb3phy_pipe", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 6, GFLAGS), + GATE(0, "pclk_usb3_grf", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 7, GFLAGS), + GATE(0, "pclk_usb_grf", "pclk_top_pre", CLK_IGNORE_UNUSED, RK1808_CLKGATE_CON(19), 8, GFLAGS), + + /* + * Clock-Architecture Diagram 11 + */ + + /* PD_PMU */ + + COMPOSITE_FRACMUX(SCLK_RTC32K_FRAC, "clk_rtc32k_frac", "xin24m", CLK_IGNORE_UNUSED, + RK1808_PMU_CLKSEL_CON(1), 0, + RK1808_PMU_CLKGATE_CON(0), 13, GFLAGS, + &rk1808_rtc32k_pmu_fracmux, 0), + + COMPOSITE_NOMUX(XIN24M_DIV, "xin24m_div", "xin24m", CLK_IGNORE_UNUSED, + RK1808_PMU_CLKSEL_CON(0), 8, 5, DFLAGS, + RK1808_PMU_CLKGATE_CON(0), 12, GFLAGS), + + COMPOSITE_NOMUX(0, "clk_wifi_pmu_src", "ppll", 0, + RK1808_PMU_CLKSEL_CON(2), 8, 6, DFLAGS, + RK1808_PMU_CLKGATE_CON(0), 14, GFLAGS), + COMPOSITE_NODIV(SCLK_WIFI_PMU, "clk_wifi_pmu", mux_wifi_pmu_p, CLK_SET_RATE_PARENT, + RK1808_PMU_CLKSEL_CON(2), 15, 1, MFLAGS, + RK1808_PMU_CLKGATE_CON(0), 15, GFLAGS), + + COMPOSITE(0, "clk_uart0_pmu_src", mux_gpll_usb480m_cpll_ppll_p, 0, + RK1808_PMU_CLKSEL_CON(3), 14, 2, MFLAGS, 0, 7, DFLAGS, + RK1808_PMU_CLKGATE_CON(1), 0, GFLAGS), + COMPOSITE_NOMUX_HALFDIV(0, "clk_uart0_np5", "clk_uart0_pmu_src", 0, + RK1808_PMU_CLKSEL_CON(4), 0, 7, DFLAGS, + RK1808_PMU_CLKGATE_CON(1), 1, GFLAGS), + COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_pmu_src", CLK_SET_RATE_PARENT, + RK1808_PMU_CLKSEL_CON(5), 0, + RK1808_PMU_CLKGATE_CON(1), 2, GFLAGS, + &rk1808_uart0_pmu_fracmux, RK1808_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART0_PMU, "clk_uart0_pmu", "clk_uart0_pmu_mux", CLK_SET_RATE_PARENT, + RK1808_PMU_CLKGATE_CON(1), 3, GFLAGS), + + GATE(SCLK_PVTM_PMU, "clk_pvtm_pmu", "xin24m", 0, + RK1808_PMU_CLKGATE_CON(1), 4, GFLAGS), + + COMPOSITE(SCLK_PMU_I2C0, "clk_pmu_i2c0", mux_ppll_xin24m_p, 0, + RK1808_PMU_CLKSEL_CON(7), 15, 1, MFLAGS, 8, 7, DFLAGS, + RK1808_PMU_CLKGATE_CON(1), 5, GFLAGS), + + COMPOSITE(DBCLK_PMU_GPIO0, "dbclk_gpio0", mux_xin24m_32k_p, 0, + RK1808_PMU_CLKSEL_CON(6), 15, 1, MFLAGS, 0, 11, DFLAGS, + RK1808_PMU_CLKGATE_CON(1), 6, GFLAGS), + + COMPOSITE_NOMUX(SCLK_REF24M_PMU, "clk_ref24m_pmu", "ppll", 0, + RK1808_PMU_CLKSEL_CON(2), 0, 6, DFLAGS, + RK1808_PMU_CLKGATE_CON(1), 8, GFLAGS), + COMPOSITE_NODIV(SCLK_USBPHY_REF, "clk_usbphy_ref", mux_usbphy_ref_p, CLK_SET_RATE_PARENT, + RK1808_PMU_CLKSEL_CON(2), 6, 1, MFLAGS, + RK1808_PMU_CLKGATE_CON(1), 9, GFLAGS), + COMPOSITE_NODIV(SCLK_MIPIDSIPHY_REF, "clk_mipidsiphy_ref", mux_mipidsiphy_ref_p, CLK_SET_RATE_PARENT, + RK1808_PMU_CLKSEL_CON(2), 7, 1, MFLAGS, + RK1808_PMU_CLKGATE_CON(1), 10, GFLAGS), + + FACTOR(0, "clk_ppll_ph0", "ppll", 0, 1, 2), + COMPOSITE_NOMUX(0, "clk_pciephy_src", "clk_ppll_ph0", 0, + RK1808_PMU_CLKSEL_CON(7), 0, 2, DFLAGS, + RK1808_PMU_CLKGATE_CON(1), 11, GFLAGS), + COMPOSITE_NODIV(SCLK_PCIEPHY_REF, "clk_pciephy_ref", mux_pciephy_ref_p, CLK_SET_RATE_PARENT, + RK1808_PMU_CLKSEL_CON(7), 4, 1, MFLAGS, + RK1808_PMU_CLKGATE_CON(1), 12, GFLAGS), + + COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "ppll", CLK_IS_CRITICAL, + RK1808_PMU_CLKSEL_CON(0), 0, 5, DFLAGS, + RK1808_PMU_CLKGATE_CON(0), 0, GFLAGS), + + GATE(0, "pclk_pmu_niu", "pclk_pmu_pre", CLK_IS_CRITICAL, RK1808_PMU_CLKGATE_CON(0), 1, GFLAGS), + GATE(0, "pclk_pmu_sgrf", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK1808_PMU_CLKGATE_CON(0), 2, GFLAGS), + GATE(0, "pclk_pmu_grf", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK1808_PMU_CLKGATE_CON(0), 3, GFLAGS), + GATE(0, "pclk_pmu", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK1808_PMU_CLKGATE_CON(0), 4, GFLAGS), + GATE(0, "pclk_pmu_mem", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK1808_PMU_CLKGATE_CON(0), 5, GFLAGS), + GATE(PCLK_GPIO0_PMU, "pclk_gpio0_pmu", "pclk_pmu_pre", 0, RK1808_PMU_CLKGATE_CON(0), 6, GFLAGS), + GATE(PCLK_UART0_PMU, "pclk_uart0_pmu", "pclk_pmu_pre", 0, RK1808_PMU_CLKGATE_CON(0), 7, GFLAGS), + GATE(0, "pclk_cru_pmu", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK1808_PMU_CLKGATE_CON(0), 8, GFLAGS), + GATE(PCLK_I2C0_PMU, "pclk_i2c0_pmu", "pclk_pmu_pre", 0, RK1808_PMU_CLKGATE_CON(0), 9, GFLAGS), + + MUXPMUGRF(SCLK_32K_IOE, "clk_32k_ioe", mux_clk_32k_ioe_p, 0, + RK1808_PMUGRF_SOC_CON0, 0, 1, MFLAGS) +}; + +static void __iomem *rk1808_cru_base; + +void rk1808_dump_cru(void) +{ + if (rk1808_cru_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk1808_cru_base, + 0x500, false); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk1808_cru_base + 0x4000, + 0x100, false); + } +} +EXPORT_SYMBOL_GPL(rk1808_dump_cru); + +static int rk1808_clk_panic(struct notifier_block *this, + unsigned long ev, void *ptr) +{ + rk1808_dump_cru(); + return NOTIFY_DONE; +} + +static struct notifier_block rk1808_clk_panic_block = { + .notifier_call = rk1808_clk_panic, +}; + +static void __init rk1808_clk_init(struct device_node *np) +{ + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; + struct clk **clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + pr_err("%s: could not map cru region\n", __func__); + return; + } + + rk1808_cru_base = reg_base; + + ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); + if (IS_ERR(ctx)) { + pr_err("%s: rockchip clk init failed\n", __func__); + iounmap(reg_base); + return; + } + clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk1808_pll_clks, + ARRAY_SIZE(rk1808_pll_clks), + RK1808_GRF_SOC_STATUS0); + rockchip_clk_register_branches(ctx, rk1808_clk_branches, + ARRAY_SIZE(rk1808_clk_branches)); + + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", + 3, clks[PLL_APLL], clks[PLL_GPLL], + &rk1808_cpuclk_data, rk1808_cpuclk_rates, + ARRAY_SIZE(rk1808_cpuclk_rates)); + + rockchip_register_softrst(np, 16, reg_base + RK1808_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); + + rockchip_register_restart_notifier(ctx, RK1808_GLB_SRST_FST, NULL); + + rockchip_clk_of_add_provider(np, ctx); + + atomic_notifier_chain_register(&panic_notifier_list, + &rk1808_clk_panic_block); +} + +CLK_OF_DECLARE(rk1808_cru, "rockchip,rk1808-cru", rk1808_clk_init); + +static int __init clk_rk1808_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + + rk1808_clk_init(np); + + return 0; +} + +static const struct of_device_id clk_rk1808_match_table[] = { + { + .compatible = "rockchip,rk1808-cru", + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk1808_match_table); + +static struct platform_driver clk_rk1808_driver = { + .driver = { + .name = "clk-rk1808", + .of_match_table = clk_rk1808_match_table, + }, +}; +builtin_platform_driver_probe(clk_rk1808_driver, clk_rk1808_probe); + +MODULE_DESCRIPTION("Rockchip RK1808 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rk3036.c b/drivers/clk/rockchip/clk-rk3036.c index 6a46f85ad837..02770ff67e14 100644 --- a/drivers/clk/rockchip/clk-rk3036.c +++ b/drivers/clk/rockchip/clk-rk3036.c @@ -9,13 +9,18 @@ #include #include +#include #include #include +#include #include #include #include "clk.h" #define RK3036_GRF_SOC_STATUS0 0x14c +#define RK3036_UART_FRAC_MAX_PRATE 600000000 +#define RK3036_I2S_FRAC_MAX_PRATE 600000000 +#define RK3036_SPDIF_FRAC_MAX_PRATE 600000000 enum rk3036_plls { apll, dpll, gpll, @@ -96,15 +101,19 @@ static struct rockchip_pll_rate_table rk3036_pll_rates[] = { } static struct rockchip_cpuclk_rate_table rk3036_cpuclk_rates[] __initdata = { + RK3036_CPUCLK_RATE(1200000000, 4), + RK3036_CPUCLK_RATE(1008000000, 4), RK3036_CPUCLK_RATE(816000000, 4), RK3036_CPUCLK_RATE(600000000, 4), + RK3036_CPUCLK_RATE(408000000, 4), RK3036_CPUCLK_RATE(312000000, 4), }; static const struct rockchip_cpuclk_reg_data rk3036_cpuclk_data = { - .core_reg = RK2928_CLKSEL_CON(0), - .div_core_shift = 0, - .div_core_mask = 0x1f, + .core_reg[0] = RK2928_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 0, .mux_core_shift = 7, @@ -113,13 +122,13 @@ static const struct rockchip_cpuclk_reg_data rk3036_cpuclk_data = { PNAME(mux_pll_p) = { "xin24m", "xin24m" }; -PNAME(mux_armclk_p) = { "apll", "gpll_armclk" }; PNAME(mux_busclk_p) = { "apll", "dpll_cpu", "gpll_cpu" }; PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr" }; PNAME(mux_pll_src_3plls_p) = { "apll", "dpll", "gpll" }; PNAME(mux_timer_p) = { "xin24m", "pclk_peri_src" }; PNAME(mux_pll_src_apll_dpll_gpll_usb480m_p) = { "apll", "dpll", "gpll", "usb480m" }; +PNAME(mux_pll_src_dmyapll_dpll_gpll_xin24_p) = { "dummy_apll", "dpll", "gpll", "xin24m" }; PNAME(mux_mmc_src_p) = { "apll", "dpll", "gpll", "xin24m" }; PNAME(mux_i2s_pre_p) = { "i2s_src", "i2s_frac", "ext_i2s", "xin12m" }; @@ -157,7 +166,7 @@ static struct rockchip_clk_branch rk3036_uart2_fracmux __initdata = RK2928_CLKSEL_CON(15), 8, 2, MFLAGS); static struct rockchip_clk_branch rk3036_i2s_fracmux __initdata = - MUX(0, "i2s_pre", mux_i2s_pre_p, CLK_SET_RATE_PARENT, + MUX(SCLK_I2S_PRE, "i2s_pre", mux_i2s_pre_p, CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(3), 8, 2, MFLAGS); static struct rockchip_clk_branch rk3036_spdif_fracmux __initdata = @@ -193,32 +202,32 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { RK2928_CLKSEL_CON(1), 4, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, RK2928_CLKGATE_CON(0), 7, GFLAGS), - GATE(0, "dpll_cpu", "dpll", 0, RK2928_CLKGATE_CON(10), 8, GFLAGS), - GATE(0, "gpll_cpu", "gpll", 0, RK2928_CLKGATE_CON(0), 1, GFLAGS), - COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_busclk_p, 0, + GATE(0, "dpll_cpu", "dpll", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 8, GFLAGS), + GATE(0, "gpll_cpu", "gpll", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(0), 1, GFLAGS), + COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_busclk_p, CLK_IS_CRITICAL, RK2928_CLKSEL_CON(0), 14, 2, MFLAGS, 8, 5, DFLAGS), - GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", CLK_IGNORE_UNUSED, + GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(0), 3, GFLAGS), - COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(1), 12, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, RK2928_CLKGATE_CON(0), 5, GFLAGS), - COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(1), 8, 2, DFLAGS | CLK_DIVIDER_READ_ONLY, RK2928_CLKGATE_CON(0), 4, GFLAGS), - COMPOSITE(0, "aclk_peri_src", mux_pll_src_3plls_p, 0, + COMPOSITE(0, "aclk_peri_src", mux_pll_src_3plls_p, CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 14, 2, MFLAGS, 0, 5, DFLAGS, RK2928_CLKGATE_CON(2), 0, GFLAGS), - GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", 0, + GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 1, GFLAGS), - DIV(0, "pclk_peri_src", "aclk_peri_src", CLK_IGNORE_UNUSED, + DIV(0, "pclk_peri_src", "aclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO), - GATE(PCLK_PERI, "pclk_peri", "pclk_peri_src", 0, + GATE(PCLK_PERI, "pclk_peri", "pclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 3, GFLAGS), - DIV(0, "hclk_peri_src", "aclk_peri_src", CLK_IGNORE_UNUSED, + DIV(0, "hclk_peri_src", "aclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO), - GATE(HCLK_PERI, "hclk_peri", "hclk_peri_src", 0, + GATE(HCLK_PERI, "hclk_peri", "hclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 2, GFLAGS), COMPOSITE_NODIV(SCLK_TIMER0, "sclk_timer0", mux_timer_p, CLK_IGNORE_UNUSED, @@ -248,15 +257,15 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(17), 0, RK2928_CLKGATE_CON(1), 9, GFLAGS, - &rk3036_uart0_fracmux), + &rk3036_uart0_fracmux, RK3036_UART_FRAC_MAX_PRATE), COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(18), 0, RK2928_CLKGATE_CON(1), 11, GFLAGS, - &rk3036_uart1_fracmux), + &rk3036_uart1_fracmux, RK3036_UART_FRAC_MAX_PRATE), COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(19), 0, RK2928_CLKGATE_CON(1), 13, GFLAGS, - &rk3036_uart2_fracmux), + &rk3036_uart2_fracmux, RK3036_UART_FRAC_MAX_PRATE), COMPOSITE(0, "aclk_vcodec", mux_pll_src_3plls_p, 0, RK2928_CLKSEL_CON(32), 14, 2, MFLAGS, 8, 5, DFLAGS, @@ -264,7 +273,7 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { FACTOR_GATE(HCLK_VCODEC, "hclk_vcodec", "aclk_vcodec", 0, 1, 4, RK2928_CLKGATE_CON(3), 12, GFLAGS), - COMPOSITE(0, "aclk_hvec", mux_pll_src_3plls_p, 0, + COMPOSITE(ACLK_HEVC, "aclk_hevc", mux_pll_src_3plls_p, 0, RK2928_CLKSEL_CON(20), 0, 2, MFLAGS, 2, 5, DFLAGS, RK2928_CLKGATE_CON(10), 6, GFLAGS), @@ -306,10 +315,10 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { COMPOSITE(0, "i2s_src", mux_pll_src_3plls_p, 0, RK2928_CLKSEL_CON(3), 14, 2, MFLAGS, 0, 7, DFLAGS, RK2928_CLKGATE_CON(0), 9, GFLAGS), - COMPOSITE_FRACMUX(0, "i2s_frac", "i2s_src", CLK_SET_RATE_PARENT, + COMPOSITE_FRACMUX(SCLK_I2S_FRAC, "i2s_frac", "i2s_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(7), 0, RK2928_CLKGATE_CON(0), 10, GFLAGS, - &rk3036_i2s_fracmux), + &rk3036_i2s_fracmux, RK3036_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S_OUT, "i2s_clkout", mux_i2s_clkout_p, 0, RK2928_CLKSEL_CON(3), 12, 1, MFLAGS, RK2928_CLKGATE_CON(0), 13, GFLAGS), @@ -322,7 +331,7 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "spdif_frac", "spdif_src", 0, RK2928_CLKSEL_CON(9), 0, RK2928_CLKGATE_CON(2), 12, GFLAGS, - &rk3036_spdif_fracmux), + &rk3036_spdif_fracmux, RK3036_SPDIF_FRAC_MAX_PRATE), GATE(SCLK_OTGPHY0, "sclk_otgphy0", "xin12m", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(1), 5, GFLAGS), @@ -339,7 +348,7 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { RK2928_CLKSEL_CON(16), 8, 2, MFLAGS, 10, 5, DFLAGS, RK2928_CLKGATE_CON(10), 4, GFLAGS), - COMPOSITE(SCLK_SFC, "sclk_sfc", mux_pll_src_apll_dpll_gpll_usb480m_p, 0, + COMPOSITE(SCLK_SFC, "sclk_sfc", mux_pll_src_dmyapll_dpll_gpll_xin24_p, 0, RK2928_CLKSEL_CON(16), 0, 2, MFLAGS, 2, 5, DFLAGS, RK2928_CLKGATE_CON(10), 5, GFLAGS), @@ -369,7 +378,7 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { /* pclk_cpu gates */ GATE(PCLK_GRF, "pclk_grf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 4, GFLAGS), - GATE(PCLK_DDRUPCTL, "pclk_ddrupctl", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 7, GFLAGS), + GATE(PCLK_DDRUPCTL, "pclk_ddrupctl", "pclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(5), 7, GFLAGS), GATE(PCLK_ACODEC, "pclk_acodec", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 14, GFLAGS), GATE(PCLK_HDMI, "pclk_hdmi", "pclk_cpu", 0, RK2928_CLKGATE_CON(3), 8, GFLAGS), @@ -402,7 +411,7 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { GATE(HCLK_OTG0, "hclk_otg0", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 13, GFLAGS), GATE(HCLK_OTG1, "hclk_otg1", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(7), 3, GFLAGS), GATE(HCLK_I2S, "hclk_i2s", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 2, GFLAGS), - GATE(0, "hclk_sfc", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 14, GFLAGS), + GATE(HCLK_SFC, "hclk_sfc", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 14, GFLAGS), GATE(HCLK_MAC, "hclk_mac", "hclk_peri", 0, RK2928_CLKGATE_CON(3), 5, GFLAGS), /* pclk_peri gates */ @@ -423,19 +432,24 @@ static struct rockchip_clk_branch rk3036_clk_branches[] __initdata = { GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 11, GFLAGS), }; -static const char *const rk3036_critical_clocks[] __initconst = { - "aclk_cpu", - "aclk_peri", - "hclk_peri", - "pclk_peri", - "pclk_ddrupctl", -}; +static void __iomem *rk3036_cru_base; + +static void rk3036_dump_cru(void) +{ + if (rk3036_cru_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk3036_cru_base, + 0x1f8, false); + } +} static void __init rk3036_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; struct clk *clk; + struct clk **clks; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -456,6 +470,7 @@ static void __init rk3036_clk_init(struct device_node *np) iounmap(reg_base); return; } + clks = ctx->clk_data.clks; clk = clk_register_fixed_factor(NULL, "usb480m", "xin24m", 0, 20, 1); if (IS_ERR(clk)) @@ -467,11 +482,9 @@ static void __init rk3036_clk_init(struct device_node *np) RK3036_GRF_SOC_STATUS0); rockchip_clk_register_branches(ctx, rk3036_clk_branches, ARRAY_SIZE(rk3036_clk_branches)); - rockchip_clk_protect_critical(rk3036_critical_clocks, - ARRAY_SIZE(rk3036_critical_clocks)); rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", - mux_armclk_p, ARRAY_SIZE(mux_armclk_p), + 2, clks[PLL_APLL], clks[PLL_GPLL], &rk3036_cpuclk_data, rk3036_cpuclk_rates, ARRAY_SIZE(rk3036_cpuclk_rates)); @@ -481,5 +494,38 @@ static void __init rk3036_clk_init(struct device_node *np) rockchip_register_restart_notifier(ctx, RK2928_GLB_SRST_FST, NULL); rockchip_clk_of_add_provider(np, ctx); + + if (!rk_dump_cru) { + rk3036_cru_base = reg_base; + rk_dump_cru = rk3036_dump_cru; + } } CLK_OF_DECLARE(rk3036_cru, "rockchip,rk3036-cru", rk3036_clk_init); + +static int __init clk_rk3036_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + + rk3036_clk_init(np); + + return 0; +} + +static const struct of_device_id clk_rk3036_match_table[] = { + { + .compatible = "rockchip,rk3036-cru", + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk3036_match_table); + +static struct platform_driver clk_rk3036_driver = { + .driver = { + .name = "clk-rk3036", + .of_match_table = clk_rk3036_match_table, + }, +}; +builtin_platform_driver_probe(clk_rk3036_driver, clk_rk3036_probe); + +MODULE_DESCRIPTION("Rockchip RK3036 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rk3128.c b/drivers/clk/rockchip/clk-rk3128.c index 4b1122e98e16..6f6f44ac7257 100644 --- a/drivers/clk/rockchip/clk-rk3128.c +++ b/drivers/clk/rockchip/clk-rk3128.c @@ -6,13 +6,19 @@ #include #include +#include #include #include +#include +#include #include #include #include "clk.h" #define RK3128_GRF_SOC_STATUS0 0x14c +#define RK3128_UART_FRAC_MAX_PRATE 600000000 +#define RK3128_I2S_FRAC_MAX_PRATE 600000000 +#define RK3128_SPDIF_FRAC_MAX_PRATE 600000000 enum rk3128_plls { apll, dpll, cpll, gpll, @@ -117,9 +123,10 @@ static struct rockchip_cpuclk_rate_table rk3128_cpuclk_rates[] __initdata = { }; static const struct rockchip_cpuclk_reg_data rk3128_cpuclk_data = { - .core_reg = RK2928_CLKSEL_CON(0), - .div_core_shift = 0, - .div_core_mask = 0x1f, + .core_reg[0] = RK2928_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 0, .mux_core_shift = 7, @@ -129,7 +136,6 @@ static const struct rockchip_cpuclk_reg_data rk3128_cpuclk_data = { PNAME(mux_pll_p) = { "clk_24m", "xin24m" }; PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_div2_ddr" }; -PNAME(mux_armclk_p) = { "apll_core", "gpll_div2_core" }; PNAME(mux_usb480m_p) = { "usb480m_phy", "xin24m" }; PNAME(mux_aclk_cpu_src_p) = { "cpll", "gpll", "gpll_div2", "gpll_div3" }; @@ -231,15 +237,15 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { RK2928_MISC_CON, 15, 1, MFLAGS), /* PD_CPU */ - COMPOSITE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, 0, + COMPOSITE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, CLK_IS_CRITICAL, RK2928_CLKSEL_CON(0), 13, 2, MFLAGS, 8, 5, DFLAGS, RK2928_CLKGATE_CON(0), 1, GFLAGS), - GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", 0, + GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(0), 3, GFLAGS), - COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", 0, + COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(1), 8, 2, DFLAGS, RK2928_CLKGATE_CON(0), 4, GFLAGS), - COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_src", 0, + COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(1), 12, 2, DFLAGS, RK2928_CLKGATE_CON(0), 5, GFLAGS), COMPOSITE_NOMUX(SCLK_CRYPTO, "clk_crypto", "aclk_cpu_src", 0, @@ -263,34 +269,33 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { RK2928_CLKGATE_CON(3), 10, GFLAGS), /* PD_VIO */ - COMPOSITE(ACLK_VIO0, "aclk_vio0", mux_pll_src_5plls_p, 0, + COMPOSITE(ACLK_VIO0, "aclk_vio0", mux_pll_src_5plls_p, CLK_IS_CRITICAL, RK2928_CLKSEL_CON(31), 5, 3, MFLAGS, 0, 5, DFLAGS, RK2928_CLKGATE_CON(3), 0, GFLAGS), COMPOSITE(ACLK_VIO1, "aclk_vio1", mux_pll_src_5plls_p, 0, RK2928_CLKSEL_CON(31), 13, 3, MFLAGS, 8, 5, DFLAGS, RK2928_CLKGATE_CON(1), 4, GFLAGS), - COMPOSITE(HCLK_VIO, "hclk_vio", mux_pll_src_4plls_p, 0, - RK2928_CLKSEL_CON(30), 14, 2, MFLAGS, 8, 5, DFLAGS, + FACTOR_GATE(HCLK_VIO, "hclk_vio", "aclk_vio0", CLK_IS_CRITICAL, 1, 4, RK2928_CLKGATE_CON(0), 11, GFLAGS), /* PD_PERI */ - GATE(0, "gpll_peri", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_peri", "gpll", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 0, GFLAGS), - GATE(0, "cpll_peri", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_peri", "cpll", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 0, GFLAGS), - GATE(0, "gpll_div2_peri", "gpll_div2", CLK_IGNORE_UNUSED, + GATE(0, "gpll_div2_peri", "gpll_div2", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 0, GFLAGS), - GATE(0, "gpll_div3_peri", "gpll_div3", CLK_IGNORE_UNUSED, + GATE(0, "gpll_div3_peri", "gpll_div3", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 0, GFLAGS), - COMPOSITE_NOGATE(0, "aclk_peri_src", mux_aclk_peri_src_p, 0, + COMPOSITE_NOGATE(0, "aclk_peri_src", mux_aclk_peri_src_p, CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 14, 2, MFLAGS, 0, 5, DFLAGS), - COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", 0, + COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, RK2928_CLKGATE_CON(2), 3, GFLAGS), - COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", 0, + COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, RK2928_CLKGATE_CON(2), 2, GFLAGS), - GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", 0, + GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 1, GFLAGS), GATE(SCLK_TIMER0, "sclk_timer0", "xin24m", 0, @@ -303,7 +308,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { RK2928_CLKGATE_CON(10), 6, GFLAGS), GATE(SCLK_TIMER4, "sclk_timer4", "xin24m", 0, RK2928_CLKGATE_CON(10), 7, GFLAGS), - GATE(SCLK_TIMER5, "sclk_timer5", "xin24m", 0, + GATE(SCLK_TIMER5, "sclk_timer5", "xin24m", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 8, GFLAGS), GATE(SCLK_PVTM_CORE, "clk_pvtm_core", "xin24m", 0, @@ -312,7 +317,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { RK2928_CLKGATE_CON(10), 1, GFLAGS), GATE(SCLK_PVTM_FUNC, "clk_pvtm_func", "xin24m", 0, RK2928_CLKGATE_CON(10), 2, GFLAGS), - GATE(SCLK_MIPI_24M, "clk_mipi_24m", "xin24m", CLK_IGNORE_UNUSED, + GATE(SCLK_MIPI_24M, "clk_mipi_24m", "xin24m", 0, RK2928_CLKGATE_CON(2), 15, GFLAGS), COMPOSITE(SCLK_SDMMC, "sclk_sdmmc0", mux_mmc_src_p, 0, @@ -359,7 +364,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(8), 0, RK2928_CLKGATE_CON(4), 5, GFLAGS, - &rk3128_i2s0_fracmux), + &rk3128_i2s0_fracmux, RK3128_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S0, "sclk_i2s0", "i2s0_pre", CLK_SET_RATE_PARENT, RK2928_CLKGATE_CON(4), 6, GFLAGS), @@ -369,7 +374,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(7), 0, RK2928_CLKGATE_CON(0), 10, GFLAGS, - &rk3128_i2s1_fracmux), + &rk3128_i2s1_fracmux, RK3128_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT, RK2928_CLKGATE_CON(0), 14, GFLAGS), COMPOSITE_NODIV(SCLK_I2S_OUT, "i2s_out", mux_i2s_out_p, 0, @@ -382,7 +387,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "spdif_frac", "sclk_spdif_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(20), 0, RK2928_CLKGATE_CON(2), 12, GFLAGS, - &rk3128_spdif_fracmux), + &rk3128_spdif_fracmux, RK3128_SPDIF_FRAC_MAX_PRATE), GATE(0, "jtag", "ext_jtag", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(1), 3, GFLAGS), @@ -419,15 +424,15 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(17), 0, RK2928_CLKGATE_CON(1), 9, GFLAGS, - &rk3128_uart0_fracmux), + &rk3128_uart0_fracmux, RK3128_UART_FRAC_MAX_PRATE), COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(18), 0, RK2928_CLKGATE_CON(1), 11, GFLAGS, - &rk3128_uart1_fracmux), + &rk3128_uart1_fracmux, RK3128_UART_FRAC_MAX_PRATE), COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(19), 0, RK2928_CLKGATE_CON(1), 13, GFLAGS, - &rk3128_uart2_fracmux), + &rk3128_uart2_fracmux, RK3128_UART_FRAC_MAX_PRATE), COMPOSITE(SCLK_MAC_SRC, "sclk_gmac_src", mux_pll_src_3plls_p, 0, RK2928_CLKSEL_CON(5), 6, 2, MFLAGS, 0, 5, DFLAGS, @@ -451,7 +456,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { RK2928_CLKSEL_CON(2), 14, 2, MFLAGS, 8, 5, DFLAGS, RK2928_CLKGATE_CON(10), 15, GFLAGS), - COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "cpll", 0, + COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "cpll", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(29), 8, 6, DFLAGS, RK2928_CLKGATE_CON(1), 0, GFLAGS), @@ -473,7 +478,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { GATE(HCLK_RGA, "hclk_rga", "hclk_vio", 0, RK2928_CLKGATE_CON(6), 10, GFLAGS), GATE(HCLK_LCDC0, "hclk_lcdc0", "hclk_vio", 0, RK2928_CLKGATE_CON(6), 1, GFLAGS), GATE(HCLK_IEP, "hclk_iep", "hclk_vio", 0, RK2928_CLKGATE_CON(9), 7, GFLAGS), - GATE(0, "hclk_vio_niu", "hclk_vio", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(6), 12, GFLAGS), + GATE(0, "hclk_vio_niu", "hclk_vio", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(6), 12, GFLAGS), GATE(HCLK_CIF, "hclk_cif", "hclk_vio", 0, RK2928_CLKGATE_CON(6), 4, GFLAGS), GATE(HCLK_EBC, "hclk_ebc", "hclk_vio", 0, RK2928_CLKGATE_CON(9), 9, GFLAGS), @@ -499,6 +504,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { GATE(0, "hclk_emmc_peri", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 6, GFLAGS), GATE(HCLK_NANDC, "hclk_nandc", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 9, GFLAGS), GATE(HCLK_USBHOST, "hclk_usbhost", "hclk_peri", 0, RK2928_CLKGATE_CON(10), 14, GFLAGS), + GATE(HCLK_SFC, "hclk_sfc", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 1, GFLAGS), GATE(PCLK_SIM_CARD, "pclk_sim_card", "pclk_peri", 0, RK2928_CLKGATE_CON(9), 12, GFLAGS), GATE(PCLK_GMAC, "pclk_gmac", "pclk_peri", 0, RK2928_CLKGATE_CON(10), 11, GFLAGS), @@ -533,8 +539,8 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { GATE(0, "pclk_grf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 4, GFLAGS), GATE(0, "pclk_mipiphy", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 0, GFLAGS), - GATE(0, "pclk_pmu", "pclk_pmu_pre", 0, RK2928_CLKGATE_CON(9), 2, GFLAGS), - GATE(0, "pclk_pmu_niu", "pclk_pmu_pre", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(9), 3, GFLAGS), + GATE(0, "pclk_pmu", "pclk_pmu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(9), 2, GFLAGS), + GATE(0, "pclk_pmu_niu", "pclk_pmu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(9), 3, GFLAGS), /* PD_MMC */ MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "sclk_sdmmc", RK3228_SDMMC_CON0, 1), @@ -562,21 +568,30 @@ static struct rockchip_clk_branch rk3128_clk_branches[] __initdata = { GATE(PCLK_HDMI, "pclk_hdmi", "pclk_cpu", 0, RK2928_CLKGATE_CON(3), 8, GFLAGS), }; -static const char *const rk3128_critical_clocks[] __initconst = { - "aclk_cpu", - "hclk_cpu", - "pclk_cpu", - "aclk_peri", - "hclk_peri", - "pclk_peri", - "pclk_pmu", - "sclk_timer5", -}; +static void __iomem *rk312x_reg_base; + +void rkclk_cpuclk_div_setting(int div) +{ + if (cpu_is_rk312x()) + writel_relaxed((0x001f0000 | (div - 1)), + rk312x_reg_base + RK2928_CLKSEL_CON(0)); +} + +static void rk3128_dump_cru(void) +{ + if (rk312x_reg_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk312x_reg_base, + 0x1f8, false); + } +} static struct rockchip_clk_provider *__init rk3128_common_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; + struct clk **clks; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -584,12 +599,14 @@ static struct rockchip_clk_provider *__init rk3128_common_clk_init(struct device return ERR_PTR(-ENOMEM); } + rk312x_reg_base = reg_base; ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); if (IS_ERR(ctx)) { pr_err("%s: rockchip clk init failed\n", __func__); iounmap(reg_base); return ERR_PTR(-ENOMEM); } + clks = ctx->clk_data.clks; rockchip_clk_register_plls(ctx, rk3128_pll_clks, ARRAY_SIZE(rk3128_pll_clks), @@ -598,7 +615,7 @@ static struct rockchip_clk_provider *__init rk3128_common_clk_init(struct device ARRAY_SIZE(common_clk_branches)); rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", - mux_armclk_p, ARRAY_SIZE(mux_armclk_p), + 2, clks[PLL_APLL], clks[PLL_GPLL_DIV2], &rk3128_cpuclk_data, rk3128_cpuclk_rates, ARRAY_SIZE(rk3128_cpuclk_rates)); @@ -607,6 +624,9 @@ static struct rockchip_clk_provider *__init rk3128_common_clk_init(struct device rockchip_register_restart_notifier(ctx, RK2928_GLB_SRST_FST, NULL); + if (!rk_dump_cru) + rk_dump_cru = rk3128_dump_cru; + return ctx; } @@ -620,8 +640,6 @@ static void __init rk3126_clk_init(struct device_node *np) rockchip_clk_register_branches(ctx, rk3126_clk_branches, ARRAY_SIZE(rk3126_clk_branches)); - rockchip_clk_protect_critical(rk3128_critical_clocks, - ARRAY_SIZE(rk3128_critical_clocks)); rockchip_clk_of_add_provider(np, ctx); } @@ -638,10 +656,60 @@ static void __init rk3128_clk_init(struct device_node *np) rockchip_clk_register_branches(ctx, rk3128_clk_branches, ARRAY_SIZE(rk3128_clk_branches)); - rockchip_clk_protect_critical(rk3128_critical_clocks, - ARRAY_SIZE(rk3128_critical_clocks)); rockchip_clk_of_add_provider(np, ctx); } CLK_OF_DECLARE(rk3128_cru, "rockchip,rk3128-cru", rk3128_clk_init); + +struct clk_rk3128_inits { + void (*inits)(struct device_node *np); +}; + +static const struct clk_rk3128_inits clk_rk3126_init = { + .inits = rk3126_clk_init, +}; + +static const struct clk_rk3128_inits clk_rk3128_init = { + .inits = rk3128_clk_init, +}; + +static const struct of_device_id clk_rk3128_match_table[] = { + { + .compatible = "rockchip,rk3126-cru", + .data = &clk_rk3126_init, + }, { + .compatible = "rockchip,rk3128-cru", + .data = &clk_rk3128_init, + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk3128_match_table); + +static int __init clk_rk3128_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + const struct clk_rk3128_inits *init_data; + + match = of_match_device(clk_rk3128_match_table, &pdev->dev); + if (!match || !match->data) + return -EINVAL; + + init_data = match->data; + if (init_data->inits) + init_data->inits(np); + + return 0; +} + +static struct platform_driver clk_rk3128_driver = { + .driver = { + .name = "clk-rk3128", + .of_match_table = clk_rk3128_match_table, + }, +}; +builtin_platform_driver_probe(clk_rk3128_driver, clk_rk3128_probe); + +MODULE_DESCRIPTION("Rockchip RK3128 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rk3188.c b/drivers/clk/rockchip/clk-rk3188.c index 730020fcc7fe..052669bf8978 100644 --- a/drivers/clk/rockchip/clk-rk3188.c +++ b/drivers/clk/rockchip/clk-rk3188.c @@ -5,15 +5,21 @@ */ #include +#include #include #include #include #include +#include #include #include "clk.h" #define RK3066_GRF_SOC_STATUS 0x15c #define RK3188_GRF_SOC_STATUS 0xac +#define RK3188_UART_FRAC_MAX_PRATE 600000000 +#define RK3188_I2S_FRAC_MAX_PRATE 600000000 +#define RK3188_SPDIF_FRAC_MAX_PRATE 600000000 +#define RK3188_HSADC_FRAC_MAX_PRATE 300000000 enum rk3188_plls { apll, cpll, dpll, gpll, @@ -145,9 +151,10 @@ static struct rockchip_cpuclk_rate_table rk3066_cpuclk_rates[] __initdata = { }; static const struct rockchip_cpuclk_reg_data rk3066_cpuclk_data = { - .core_reg = RK2928_CLKSEL_CON(0), - .div_core_shift = 0, - .div_core_mask = 0x1f, + .core_reg[0] = RK2928_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 0, .mux_core_shift = 8, @@ -184,9 +191,10 @@ static struct rockchip_cpuclk_rate_table rk3188_cpuclk_rates[] __initdata = { }; static const struct rockchip_cpuclk_reg_data rk3188_cpuclk_data = { - .core_reg = RK2928_CLKSEL_CON(0), - .div_core_shift = 9, - .div_core_mask = 0x1f, + .core_reg[0] = RK2928_CLKSEL_CON(0), + .div_core_shift[0] = 9, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 0, .mux_core_shift = 8, @@ -194,7 +202,6 @@ static const struct rockchip_cpuclk_reg_data rk3188_cpuclk_data = { }; PNAME(mux_pll_p) = { "xin24m", "xin32k" }; -PNAME(mux_armclk_p) = { "apll", "gpll_armclk" }; PNAME(mux_ddrphy_p) = { "dpll", "gpll_ddr" }; PNAME(mux_pll_src_gpll_cpll_p) = { "gpll", "cpll" }; PNAME(mux_pll_src_cpll_gpll_p) = { "cpll", "gpll" }; @@ -299,14 +306,14 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { RK2928_CLKSEL_CON(26), 8, 1, MFLAGS, 0, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, RK2928_CLKGATE_CON(0), 2, GFLAGS), - GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_pre", 0, + GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(0), 3, GFLAGS), GATE(0, "atclk_cpu", "pclk_cpu_pre", 0, RK2928_CLKGATE_CON(0), 6, GFLAGS), - GATE(PCLK_CPU, "pclk_cpu", "pclk_cpu_pre", 0, + GATE(PCLK_CPU, "pclk_cpu", "pclk_cpu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(0), 5, GFLAGS), - GATE(HCLK_CPU, "hclk_cpu", "hclk_cpu_pre", CLK_IGNORE_UNUSED, + GATE(HCLK_CPU, "hclk_cpu", "hclk_cpu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(0), 4, GFLAGS), COMPOSITE(0, "aclk_lcdc0_pre", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED, @@ -316,12 +323,12 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { RK2928_CLKSEL_CON(31), 15, 1, MFLAGS, 8, 5, DFLAGS, RK2928_CLKGATE_CON(1), 4, GFLAGS), - GATE(ACLK_PERI, "aclk_peri", "aclk_peri_pre", 0, + GATE(ACLK_PERI, "aclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 1, GFLAGS), - COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_pre", 0, + COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, RK2928_CLKGATE_CON(2), 2, GFLAGS), - COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_pre", 0, + COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, RK2928_CLKGATE_CON(2), 3, GFLAGS), @@ -354,7 +361,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { RK2928_CLKGATE_CON(2), 5, GFLAGS), MUX(SCLK_MAC, "sclk_macref", mux_sclk_macref_p, CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(21), 4, 1, MFLAGS), - GATE(0, "sclk_mac_lbtest", "sclk_macref", 0, + GATE(0, "sclk_mac_lbtest", "sclk_macref", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 12, GFLAGS), COMPOSITE(0, "hsadc_src", mux_pll_src_gpll_cpll_p, 0, @@ -363,7 +370,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "hsadc_frac", "hsadc_src", 0, RK2928_CLKSEL_CON(23), 0, RK2928_CLKGATE_CON(2), 7, GFLAGS, - &common_hsadc_out_fracmux), + &common_hsadc_out_fracmux, RK3188_HSADC_FRAC_MAX_PRATE), INVERTER(SCLK_HSADC, "sclk_hsadc", "sclk_hsadc_out", RK2928_CLKSEL_CON(22), 7, IFLAGS), @@ -377,7 +384,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "spdif_frac", "spdif_pre", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(9), 0, RK2928_CLKGATE_CON(0), 14, GFLAGS, - &common_spdif_fracmux), + &common_spdif_fracmux, RK3188_SPDIF_FRAC_MAX_PRATE), /* * Clock-Architecture Diagram 4 @@ -411,28 +418,28 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_pre", 0, RK2928_CLKSEL_CON(17), 0, RK2928_CLKGATE_CON(1), 9, GFLAGS, - &common_uart0_fracmux), + &common_uart0_fracmux, RK3188_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "uart1_pre", "uart_src", 0, RK2928_CLKSEL_CON(14), 0, 7, DFLAGS, RK2928_CLKGATE_CON(1), 10, GFLAGS), COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_pre", 0, RK2928_CLKSEL_CON(18), 0, RK2928_CLKGATE_CON(1), 11, GFLAGS, - &common_uart1_fracmux), + &common_uart1_fracmux, RK3188_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "uart2_pre", "uart_src", 0, RK2928_CLKSEL_CON(15), 0, 7, DFLAGS, RK2928_CLKGATE_CON(1), 12, GFLAGS), COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_pre", 0, RK2928_CLKSEL_CON(19), 0, RK2928_CLKGATE_CON(1), 13, GFLAGS, - &common_uart2_fracmux), + &common_uart2_fracmux, RK3188_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "uart3_pre", "uart_src", 0, RK2928_CLKSEL_CON(16), 0, 7, DFLAGS, RK2928_CLKGATE_CON(1), 14, GFLAGS), COMPOSITE_FRACMUX(0, "uart3_frac", "uart3_pre", 0, RK2928_CLKSEL_CON(20), 0, RK2928_CLKGATE_CON(1), 15, GFLAGS, - &common_uart3_fracmux), + &common_uart3_fracmux, RK3188_UART_FRAC_MAX_PRATE), GATE(SCLK_JTAG, "jtag", "ext_jtag", 0, RK2928_CLKGATE_CON(1), 3, GFLAGS), @@ -449,11 +456,11 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = { /* hclk_cpu gates */ GATE(HCLK_ROM, "hclk_rom", "hclk_cpu", 0, RK2928_CLKGATE_CON(5), 6, GFLAGS), - GATE(HCLK_I2S0, "hclk_i2s0", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 2, GFLAGS), + GATE(HCLK_I2S0_2CH, "hclk_i2s0_2ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 2, GFLAGS), GATE(HCLK_SPDIF, "hclk_spdif", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 1, GFLAGS), - GATE(0, "hclk_cpubus", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 8, GFLAGS), + GATE(0, "hclk_cpubus", "hclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(4), 8, GFLAGS), /* hclk_ahb2apb is part of a clk branch */ - GATE(0, "hclk_vio_bus", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 12, GFLAGS), + GATE(0, "hclk_vio_bus", "hclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(6), 12, GFLAGS), GATE(HCLK_LCDC0, "hclk_lcdc0", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 1, GFLAGS), GATE(HCLK_LCDC1, "hclk_lcdc1", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 2, GFLAGS), GATE(HCLK_CIF0, "hclk_cif0", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 4, GFLAGS), @@ -571,7 +578,7 @@ static struct rockchip_clk_branch rk3066a_clk_branches[] __initdata = { GATE(CORE_L2C, "core_l2c", "aclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(9), 4, GFLAGS), - COMPOSITE(0, "aclk_peri_pre", mux_pll_src_gpll_cpll_p, 0, + COMPOSITE(0, "aclk_peri_pre", mux_pll_src_gpll_cpll_p, CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 15, 1, MFLAGS, 0, 5, DFLAGS, RK2928_CLKGATE_CON(2), 0, GFLAGS), @@ -618,24 +625,24 @@ static struct rockchip_clk_branch rk3066a_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_pre", 0, RK2928_CLKSEL_CON(6), 0, RK2928_CLKGATE_CON(0), 8, GFLAGS, - &rk3066a_i2s0_fracmux), + &rk3066a_i2s0_fracmux, RK3188_I2S_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "i2s1_pre", "i2s_src", 0, RK2928_CLKSEL_CON(3), 0, 7, DFLAGS, RK2928_CLKGATE_CON(0), 9, GFLAGS), COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_pre", 0, RK2928_CLKSEL_CON(7), 0, RK2928_CLKGATE_CON(0), 10, GFLAGS, - &rk3066a_i2s1_fracmux), + &rk3066a_i2s1_fracmux, RK3188_I2S_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "i2s2_pre", "i2s_src", 0, RK2928_CLKSEL_CON(4), 0, 7, DFLAGS, RK2928_CLKGATE_CON(0), 11, GFLAGS), COMPOSITE_FRACMUX(0, "i2s2_frac", "i2s2_pre", 0, RK2928_CLKSEL_CON(8), 0, RK2928_CLKGATE_CON(0), 12, GFLAGS, - &rk3066a_i2s2_fracmux), + &rk3066a_i2s2_fracmux, RK3188_I2S_FRAC_MAX_PRATE), - GATE(HCLK_I2S1, "hclk_i2s1", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 3, GFLAGS), - GATE(HCLK_I2S2, "hclk_i2s2", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 4, GFLAGS), + GATE(HCLK_I2S1_2CH, "hclk_i2s1_2ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 3, GFLAGS), + GATE(HCLK_I2S_8CH, "hclk_i2s_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 4, GFLAGS), GATE(HCLK_CIF1, "hclk_cif1", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 6, GFLAGS), GATE(HCLK_HDMI, "hclk_hdmi", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 14, GFLAGS), @@ -676,7 +683,7 @@ static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = { div_rk3188_aclk_core_t, RK2928_CLKGATE_CON(0), 7, GFLAGS), /* do not source aclk_cpu_pre from the apll, to keep complexity down */ - COMPOSITE_NOGATE(0, "aclk_cpu_pre", mux_aclk_cpu_p, CLK_SET_RATE_NO_REPARENT, + COMPOSITE_NOGATE(ACLK_CPU_PRE, "aclk_cpu_pre", mux_aclk_cpu_p, CLK_SET_RATE_NO_REPARENT, RK2928_CLKSEL_CON(0), 5, 1, MFLAGS, 0, 5, DFLAGS), DIV(0, "pclk_cpu_pre", "aclk_cpu_pre", 0, RK2928_CLKSEL_CON(1), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO), @@ -689,7 +696,7 @@ static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = { GATE(CORE_L2C, "core_l2c", "armclk", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(9), 4, GFLAGS), - COMPOSITE(0, "aclk_peri_pre", mux_pll_src_cpll_gpll_p, 0, + COMPOSITE(0, "aclk_peri_pre", mux_pll_src_cpll_gpll_p, CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 15, 1, MFLAGS, 0, 5, DFLAGS, RK2928_CLKGATE_CON(2), 0, GFLAGS), @@ -726,7 +733,7 @@ static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_pre", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(7), 0, RK2928_CLKGATE_CON(0), 10, GFLAGS, - &rk3188_i2s0_fracmux), + &rk3188_i2s0_fracmux, RK3188_I2S_FRAC_MAX_PRATE), GATE(0, "hclk_imem0", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 14, GFLAGS), GATE(0, "hclk_imem1", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 15, GFLAGS), @@ -743,17 +750,6 @@ static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = { GATE(ACLK_GPS, "aclk_gps", "aclk_peri", 0, RK2928_CLKGATE_CON(8), 13, GFLAGS), }; -static const char *const rk3188_critical_clocks[] __initconst = { - "aclk_cpu", - "aclk_peri", - "hclk_peri", - "pclk_cpu", - "pclk_peri", - "hclk_cpubus", - "hclk_vio_bus", - "sclk_mac_lbtest", -}; - static struct rockchip_clk_provider *__init rk3188_common_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; @@ -786,10 +782,12 @@ static struct rockchip_clk_provider *__init rk3188_common_clk_init(struct device static void __init rk3066a_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; + struct clk **clks; ctx = rk3188_common_clk_init(np); if (IS_ERR(ctx)) return; + clks = ctx->clk_data.clks; rockchip_clk_register_plls(ctx, rk3066_pll_clks, ARRAY_SIZE(rk3066_pll_clks), @@ -797,11 +795,9 @@ static void __init rk3066a_clk_init(struct device_node *np) rockchip_clk_register_branches(ctx, rk3066a_clk_branches, ARRAY_SIZE(rk3066a_clk_branches)); rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", - mux_armclk_p, ARRAY_SIZE(mux_armclk_p), + 2, clks[PLL_APLL], clks[PLL_GPLL], &rk3066_cpuclk_data, rk3066_cpuclk_rates, ARRAY_SIZE(rk3066_cpuclk_rates)); - rockchip_clk_protect_critical(rk3188_critical_clocks, - ARRAY_SIZE(rk3188_critical_clocks)); rockchip_clk_of_add_provider(np, ctx); } CLK_OF_DECLARE(rk3066a_cru, "rockchip,rk3066a-cru", rk3066a_clk_init); @@ -809,13 +805,14 @@ CLK_OF_DECLARE(rk3066a_cru, "rockchip,rk3066a-cru", rk3066a_clk_init); static void __init rk3188a_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; - struct clk *clk1, *clk2; + struct clk **clks; unsigned long rate; int ret; ctx = rk3188_common_clk_init(np); if (IS_ERR(ctx)) return; + clks = ctx->clk_data.clks; rockchip_clk_register_plls(ctx, rk3188_pll_clks, ARRAY_SIZE(rk3188_pll_clks), @@ -823,29 +820,25 @@ static void __init rk3188a_clk_init(struct device_node *np) rockchip_clk_register_branches(ctx, rk3188_clk_branches, ARRAY_SIZE(rk3188_clk_branches)); rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", - mux_armclk_p, ARRAY_SIZE(mux_armclk_p), + 2, clks[PLL_APLL], clks[PLL_GPLL], &rk3188_cpuclk_data, rk3188_cpuclk_rates, ARRAY_SIZE(rk3188_cpuclk_rates)); /* reparent aclk_cpu_pre from apll */ - clk1 = __clk_lookup("aclk_cpu_pre"); - clk2 = __clk_lookup("gpll"); - if (clk1 && clk2) { - rate = clk_get_rate(clk1); + if (clks[ACLK_CPU_PRE] && clks[PLL_GPLL]) { + rate = clk_get_rate(clks[ACLK_CPU_PRE]); - ret = clk_set_parent(clk1, clk2); + ret = clk_set_parent(clks[ACLK_CPU_PRE], clks[PLL_GPLL]); if (ret < 0) pr_warn("%s: could not reparent aclk_cpu_pre to gpll\n", __func__); - clk_set_rate(clk1, rate); + clk_set_rate(clks[ACLK_CPU_PRE], rate); } else { pr_warn("%s: missing clocks to reparent aclk_cpu_pre to gpll\n", __func__); } - rockchip_clk_protect_critical(rk3188_critical_clocks, - ARRAY_SIZE(rk3188_critical_clocks)); rockchip_clk_of_add_provider(np, ctx); } CLK_OF_DECLARE(rk3188a_cru, "rockchip,rk3188a-cru", rk3188a_clk_init); @@ -871,3 +864,62 @@ static void __init rk3188_clk_init(struct device_node *np) rk3188a_clk_init(np); } CLK_OF_DECLARE(rk3188_cru, "rockchip,rk3188-cru", rk3188_clk_init); + +struct clk_rk3188_inits { + void (*inits)(struct device_node *np); +}; + +static const struct clk_rk3188_inits clk_rk3066a_init = { + .inits = rk3066a_clk_init, +}; + +static const struct clk_rk3188_inits clk_rk3188a_init = { + .inits = rk3188a_clk_init, +}; + +static const struct clk_rk3188_inits clk_rk3188_init = { + .inits = rk3188_clk_init, +}; + +static const struct of_device_id clk_rk3188_match_table[] = { + { + .compatible = "rockchip,rk3066a-cru", + .data = &clk_rk3066a_init, + }, { + .compatible = "rockchip,rk3188a-cru", + .data = &clk_rk3188a_init, + }, { + .compatible = "rockchip,rk3188-cru", + .data = &rk3188_clk_init, + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk3188_match_table); + +static int __init clk_rk3188_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + const struct clk_rk3188_inits *init_data; + + match = of_match_device(clk_rk3188_match_table, &pdev->dev); + if (!match || !match->data) + return -EINVAL; + + init_data = match->data; + if (init_data->inits) + init_data->inits(np); + + return 0; +} + +static struct platform_driver clk_rk3188_driver = { + .driver = { + .name = "clk-rk3188", + .of_match_table = clk_rk3188_match_table, + }, +}; +builtin_platform_driver_probe(clk_rk3188_driver, clk_rk3188_probe); + +MODULE_DESCRIPTION("Rockchip RK3188 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rk3228.c b/drivers/clk/rockchip/clk-rk3228.c index 47d6482dda9d..01ff90c8a59f 100644 --- a/drivers/clk/rockchip/clk-rk3228.c +++ b/drivers/clk/rockchip/clk-rk3228.c @@ -7,14 +7,20 @@ #include #include +#include #include #include +#include #include #include #include "clk.h" #define RK3228_GRF_SOC_STATUS0 0x480 +#define RK3228_UART_FRAC_MAX_PRATE 600000000 +#define RK3228_SPDIF_FRAC_MAX_PRATE 600000000 +#define RK3228_I2S_FRAC_MAX_PRATE 600000000 + enum rk3228_plls { apll, dpll, cpll, gpll, }; @@ -78,22 +84,22 @@ static struct rockchip_pll_rate_table rk3228_pll_rates[] = { #define RK3228_DIV_PCLK_MASK 0x7 #define RK3228_DIV_PCLK_SHIFT 12 -#define RK3228_CLKSEL1(_core_aclk_div, _core_peri_div) \ - { \ - .reg = RK2928_CLKSEL_CON(1), \ - .val = HIWORD_UPDATE(_core_peri_div, RK3228_DIV_PERI_MASK, \ - RK3228_DIV_PERI_SHIFT) | \ - HIWORD_UPDATE(_core_aclk_div, RK3228_DIV_ACLK_MASK, \ - RK3228_DIV_ACLK_SHIFT), \ +#define RK3228_CLKSEL1(_core_aclk_div, _core_peri_div) \ +{ \ + .reg = RK2928_CLKSEL_CON(1), \ + .val = HIWORD_UPDATE(_core_peri_div, RK3228_DIV_PERI_MASK, \ + RK3228_DIV_PERI_SHIFT) | \ + HIWORD_UPDATE(_core_aclk_div, RK3228_DIV_ACLK_MASK, \ + RK3228_DIV_ACLK_SHIFT), \ } -#define RK3228_CPUCLK_RATE(_prate, _core_aclk_div, _core_peri_div) \ - { \ - .prate = _prate, \ - .divs = { \ - RK3228_CLKSEL1(_core_aclk_div, _core_peri_div), \ - }, \ - } +#define RK3228_CPUCLK_RATE(_prate, _core_aclk_div, _core_peri_div) \ +{ \ + .prate = _prate, \ + .divs = { \ + RK3228_CLKSEL1(_core_aclk_div, _core_peri_div), \ + }, \ +} static struct rockchip_cpuclk_rate_table rk3228_cpuclk_rates[] __initdata = { RK3228_CPUCLK_RATE(1800000000, 1, 7), @@ -119,9 +125,10 @@ static struct rockchip_cpuclk_rate_table rk3228_cpuclk_rates[] __initdata = { }; static const struct rockchip_cpuclk_reg_data rk3228_cpuclk_data = { - .core_reg = RK2928_CLKSEL_CON(0), - .div_core_shift = 0, - .div_core_mask = 0x1f, + .core_reg[0] = RK2928_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 0, .mux_core_shift = 6, @@ -131,7 +138,6 @@ static const struct rockchip_cpuclk_reg_data rk3228_cpuclk_data = { PNAME(mux_pll_p) = { "clk_24m", "xin24m" }; PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr", "apll_ddr" }; -PNAME(mux_armclk_p) = { "apll_core", "gpll_core", "dpll_core" }; PNAME(mux_usb480m_phy_p) = { "usb480m_phy0", "usb480m_phy1" }; PNAME(mux_usb480m_p) = { "usb480m_phy", "xin24m" }; PNAME(mux_hdmiphy_p) = { "hdmiphy_phy", "xin24m" }; @@ -231,12 +237,12 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { RK2928_CLKGATE_CON(7), 0, GFLAGS), /* PD_CORE */ - GATE(0, "dpll_core", "dpll", CLK_IGNORE_UNUSED, - RK2928_CLKGATE_CON(0), 6, GFLAGS), GATE(0, "apll_core", "apll", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(0), 6, GFLAGS), GATE(0, "gpll_core", "gpll", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(0), 6, GFLAGS), + GATE(0, "dpll_core", "dpll", CLK_IGNORE_UNUSED, + RK2928_CLKGATE_CON(0), 6, GFLAGS), COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IGNORE_UNUSED, RK2928_CLKSEL_CON(1), 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, RK2928_CLKGATE_CON(4), 1, GFLAGS), @@ -253,27 +259,27 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { RK2928_MISC_CON, 15, 1, MFLAGS), /* PD_BUS */ - GATE(0, "hdmiphy_aclk_cpu", "hdmiphy", CLK_IGNORE_UNUSED, + GATE(0, "hdmiphy_aclk_cpu", "hdmiphy", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(0), 1, GFLAGS), - GATE(0, "gpll_aclk_cpu", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_aclk_cpu", "gpll", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(0), 1, GFLAGS), - GATE(0, "cpll_aclk_cpu", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_aclk_cpu", "cpll", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(0), 1, GFLAGS), - COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, 0, + COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, CLK_IS_CRITICAL, RK2928_CLKSEL_CON(0), 13, 2, MFLAGS, 8, 5, DFLAGS), - GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", 0, + GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(6), 0, GFLAGS), - COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", 0, + COMPOSITE_NOMUX(HCLK_CPU, "hclk_cpu", "aclk_cpu_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(1), 8, 2, DFLAGS, RK2928_CLKGATE_CON(6), 1, GFLAGS), - COMPOSITE_NOMUX(0, "pclk_bus_src", "aclk_cpu_src", 0, + COMPOSITE_NOMUX(0, "pclk_bus_src", "aclk_cpu_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(1), 12, 3, DFLAGS, RK2928_CLKGATE_CON(6), 2, GFLAGS), - GATE(PCLK_CPU, "pclk_cpu", "pclk_bus_src", 0, + GATE(PCLK_CPU, "pclk_cpu", "pclk_bus_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(6), 3, GFLAGS), - GATE(0, "pclk_phy_pre", "pclk_bus_src", 0, + GATE(0, "pclk_phy_pre", "pclk_bus_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(6), 4, GFLAGS), - GATE(0, "pclk_ddr_pre", "pclk_bus_src", 0, + GATE(0, "pclk_ddr_pre", "pclk_bus_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(6), 13, GFLAGS), /* PD_VIDEO */ @@ -308,9 +314,9 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { RK2928_CLKSEL_CON(31), 13, 2, MFLAGS, 8, 5, DFLAGS, RK2928_CLKGATE_CON(1), 4, GFLAGS), - MUX(0, "sclk_rga_src", mux_pll_src_4plls_p, 0, + MUX(0, "sclk_rga_src", mux_pll_src_4plls_p, CLK_IS_CRITICAL, RK2928_CLKSEL_CON(33), 13, 2, MFLAGS), - COMPOSITE_NOMUX(ACLK_RGA_PRE, "aclk_rga_pre", "sclk_rga_src", 0, + COMPOSITE_NOMUX(ACLK_RGA_PRE, "aclk_rga_pre", "sclk_rga_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(33), 8, 5, DFLAGS, RK2928_CLKGATE_CON(1), 2, GFLAGS), COMPOSITE(SCLK_RGA, "sclk_rga", mux_sclk_rga_p, 0, @@ -333,21 +339,21 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { RK2928_CLKGATE_CON(3), 8, GFLAGS), /* PD_PERI */ - GATE(0, "cpll_peri", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_peri", "cpll", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 0, GFLAGS), - GATE(0, "gpll_peri", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_peri", "gpll", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 0, GFLAGS), - GATE(0, "hdmiphy_peri", "hdmiphy", CLK_IGNORE_UNUSED, + GATE(0, "hdmiphy_peri", "hdmiphy", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(2), 0, GFLAGS), - COMPOSITE_NOGATE(0, "aclk_peri_src", mux_aclk_peri_src_p, 0, + COMPOSITE_NOGATE(0, "aclk_peri_src", mux_aclk_peri_src_p, CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 10, 2, MFLAGS, 0, 5, DFLAGS), - COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", 0, + COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 12, 3, DFLAGS, RK2928_CLKGATE_CON(5), 2, GFLAGS), - COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", 0, + COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKSEL_CON(10), 8, 2, DFLAGS, RK2928_CLKGATE_CON(5), 1, GFLAGS), - GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", 0, + GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(5), 0, GFLAGS), GATE(SCLK_TIMER0, "sclk_timer0", "xin24m", 0, @@ -419,7 +425,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(8), 0, RK2928_CLKGATE_CON(0), 4, GFLAGS, - &rk3228_i2s0_fracmux), + &rk3228_i2s0_fracmux, RK3228_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S0, "sclk_i2s0", "i2s0_pre", CLK_SET_RATE_PARENT, RK2928_CLKGATE_CON(0), 5, GFLAGS), @@ -429,7 +435,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(7), 0, RK2928_CLKGATE_CON(0), 11, GFLAGS, - &rk3228_i2s1_fracmux), + &rk3228_i2s1_fracmux, RK3228_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT, RK2928_CLKGATE_CON(0), 14, GFLAGS), COMPOSITE_NODIV(SCLK_I2S_OUT, "i2s_out", mux_i2s_out_p, 0, @@ -442,7 +448,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s2_frac", "i2s2_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(30), 0, RK2928_CLKGATE_CON(0), 8, GFLAGS, - &rk3228_i2s2_fracmux), + &rk3228_i2s2_fracmux, RK3228_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S2, "sclk_i2s2", "i2s2_pre", CLK_SET_RATE_PARENT, RK2928_CLKGATE_CON(0), 9, GFLAGS), @@ -452,7 +458,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "spdif_frac", "sclk_spdif_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(20), 0, RK2928_CLKGATE_CON(2), 12, GFLAGS, - &rk3228_spdif_fracmux), + &rk3228_spdif_fracmux, RK3228_SPDIF_FRAC_MAX_PRATE), GATE(0, "jtag", "ext_jtag", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(1), 3, GFLAGS), @@ -487,15 +493,15 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(17), 0, RK2928_CLKGATE_CON(1), 9, GFLAGS, - &rk3228_uart0_fracmux), + &rk3228_uart0_fracmux, RK3228_UART_FRAC_MAX_PRATE), COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(18), 0, RK2928_CLKGATE_CON(1), 11, GFLAGS, - &rk3228_uart1_fracmux), + &rk3228_uart1_fracmux, RK3228_UART_FRAC_MAX_PRATE), COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(19), 0, RK2928_CLKGATE_CON(1), 13, GFLAGS, - &rk3228_uart2_fracmux), + &rk3228_uart2_fracmux, RK3228_UART_FRAC_MAX_PRATE), COMPOSITE(SCLK_NANDC, "sclk_nandc", mux_pll_src_2plls_p, 0, RK2928_CLKSEL_CON(2), 14, 1, MFLAGS, 8, 5, DFLAGS, @@ -529,22 +535,22 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { /* PD_VOP */ GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, RK2928_CLKGATE_CON(13), 0, GFLAGS), - GATE(0, "aclk_rga_noc", "aclk_rga_pre", 0, RK2928_CLKGATE_CON(13), 11, GFLAGS), + GATE(0, "aclk_rga_noc", "aclk_rga_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 11, GFLAGS), GATE(ACLK_IEP, "aclk_iep", "aclk_iep_pre", 0, RK2928_CLKGATE_CON(13), 2, GFLAGS), - GATE(0, "aclk_iep_noc", "aclk_iep_pre", 0, RK2928_CLKGATE_CON(13), 9, GFLAGS), + GATE(0, "aclk_iep_noc", "aclk_iep_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 9, GFLAGS), GATE(ACLK_VOP, "aclk_vop", "aclk_vop_pre", 0, RK2928_CLKGATE_CON(13), 5, GFLAGS), - GATE(0, "aclk_vop_noc", "aclk_vop_pre", 0, RK2928_CLKGATE_CON(13), 12, GFLAGS), + GATE(0, "aclk_vop_noc", "aclk_vop_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 12, GFLAGS), GATE(ACLK_HDCP, "aclk_hdcp", "aclk_hdcp_pre", 0, RK2928_CLKGATE_CON(14), 10, GFLAGS), - GATE(0, "aclk_hdcp_noc", "aclk_hdcp_pre", 0, RK2928_CLKGATE_CON(13), 10, GFLAGS), + GATE(0, "aclk_hdcp_noc", "aclk_hdcp_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 10, GFLAGS), GATE(HCLK_RGA, "hclk_rga", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 1, GFLAGS), GATE(HCLK_IEP, "hclk_iep", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 3, GFLAGS), GATE(HCLK_VOP, "hclk_vop", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 6, GFLAGS), - GATE(0, "hclk_vio_ahb_arbi", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 7, GFLAGS), - GATE(0, "hclk_vio_noc", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 8, GFLAGS), - GATE(0, "hclk_vop_noc", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(13), 13, GFLAGS), + GATE(0, "hclk_vio_ahb_arbi", "hclk_vio_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 7, GFLAGS), + GATE(0, "hclk_vio_noc", "hclk_vio_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 8, GFLAGS), + GATE(0, "hclk_vop_noc", "hclk_vio_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(13), 13, GFLAGS), GATE(HCLK_VIO_H2P, "hclk_vio_h2p", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 7, GFLAGS), GATE(HCLK_HDCP_MMU, "hclk_hdcp_mmu", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 12, GFLAGS), GATE(PCLK_HDMI_CTRL, "pclk_hdmi_ctrl", "hclk_vio_pre", 0, RK2928_CLKGATE_CON(14), 6, GFLAGS), @@ -560,29 +566,29 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { GATE(HCLK_EMMC, "hclk_emmc", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 2, GFLAGS), GATE(HCLK_NANDC, "hclk_nandc", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 3, GFLAGS), GATE(HCLK_HOST0, "hclk_host0", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 6, GFLAGS), - GATE(0, "hclk_host0_arb", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 7, GFLAGS), + GATE(0, "hclk_host0_arb", "hclk_peri", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(11), 7, GFLAGS), GATE(HCLK_HOST1, "hclk_host1", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 8, GFLAGS), - GATE(0, "hclk_host1_arb", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 9, GFLAGS), + GATE(0, "hclk_host1_arb", "hclk_peri", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(11), 9, GFLAGS), GATE(HCLK_HOST2, "hclk_host2", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 10, GFLAGS), GATE(HCLK_OTG, "hclk_otg", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 12, GFLAGS), - GATE(0, "hclk_otg_pmu", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 13, GFLAGS), - GATE(0, "hclk_host2_arb", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 14, GFLAGS), + GATE(0, "hclk_otg_pmu", "hclk_peri", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(11), 13, GFLAGS), + GATE(0, "hclk_host2_arb", "hclk_peri", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(11), 14, GFLAGS), GATE(0, "hclk_peri_noc", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(12), 1, GFLAGS), GATE(PCLK_GMAC, "pclk_gmac", "pclk_peri", 0, RK2928_CLKGATE_CON(11), 5, GFLAGS), - GATE(0, "pclk_peri_noc", "pclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(12), 2, GFLAGS), + GATE(0, "pclk_peri_noc", "pclk_peri", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(12), 2, GFLAGS), /* PD_GPU */ GATE(ACLK_GPU, "aclk_gpu", "aclk_gpu_pre", 0, RK2928_CLKGATE_CON(7), 14, GFLAGS), - GATE(0, "aclk_gpu_noc", "aclk_gpu_pre", 0, RK2928_CLKGATE_CON(7), 15, GFLAGS), + GATE(0, "aclk_gpu_noc", "aclk_gpu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(7), 15, GFLAGS), /* PD_BUS */ - GATE(0, "sclk_initmem_mbist", "aclk_cpu", 0, RK2928_CLKGATE_CON(8), 1, GFLAGS), - GATE(0, "aclk_initmem", "aclk_cpu", 0, RK2928_CLKGATE_CON(8), 0, GFLAGS), + GATE(0, "sclk_initmem_mbist", "aclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(8), 1, GFLAGS), + GATE(0, "aclk_initmem", "aclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(8), 0, GFLAGS), GATE(ACLK_DMAC, "aclk_dmac_bus", "aclk_cpu", 0, RK2928_CLKGATE_CON(8), 2, GFLAGS), GATE(0, "aclk_bus_noc", "aclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 1, GFLAGS), - GATE(0, "hclk_rom", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 3, GFLAGS), + GATE(0, "hclk_rom", "hclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(8), 3, GFLAGS), GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 7, GFLAGS), GATE(HCLK_I2S1_8CH, "hclk_i2s1_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 8, GFLAGS), GATE(HCLK_I2S2_2CH, "hclk_i2s2_2ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 9, GFLAGS), @@ -591,9 +597,9 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { GATE(HCLK_M_CRYPTO, "hclk_crypto_mst", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 11, GFLAGS), GATE(HCLK_S_CRYPTO, "hclk_crypto_slv", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 12, GFLAGS), - GATE(0, "pclk_ddrupctl", "pclk_ddr_pre", 0, RK2928_CLKGATE_CON(8), 4, GFLAGS), - GATE(0, "pclk_ddrmon", "pclk_ddr_pre", 0, RK2928_CLKGATE_CON(8), 6, GFLAGS), - GATE(0, "pclk_msch_noc", "pclk_ddr_pre", 0, RK2928_CLKGATE_CON(10), 2, GFLAGS), + GATE(0, "pclk_ddrupctl", "pclk_ddr_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(8), 4, GFLAGS), + GATE(0, "pclk_ddrmon", "pclk_ddr_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(8), 6, GFLAGS), + GATE(0, "pclk_msch_noc", "pclk_ddr_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 2, GFLAGS), GATE(PCLK_EFUSE_1024, "pclk_efuse_1024", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 13, GFLAGS), GATE(PCLK_EFUSE_256, "pclk_efuse_256", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 14, GFLAGS), @@ -602,7 +608,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { GATE(PCLK_I2C2, "pclk_i2c2", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 1, GFLAGS), GATE(PCLK_I2C3, "pclk_i2c3", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 2, GFLAGS), GATE(PCLK_TIMER, "pclk_timer0", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 4, GFLAGS), - GATE(0, "pclk_stimer", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 5, GFLAGS), + GATE(0, "pclk_stimer", "pclk_cpu", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(9), 5, GFLAGS), GATE(PCLK_SPI0, "pclk_spi0", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 6, GFLAGS), GATE(PCLK_PWM, "pclk_rk_pwm", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 7, GFLAGS), GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 8, GFLAGS), @@ -616,73 +622,51 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { GATE(PCLK_GRF, "pclk_grf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 0, GFLAGS), GATE(0, "pclk_cru", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 1, GFLAGS), GATE(0, "pclk_sgrf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 2, GFLAGS), - GATE(0, "pclk_sim", "pclk_cpu", 0, RK2928_CLKGATE_CON(10), 3, GFLAGS), + GATE(0, "pclk_sim", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 3, GFLAGS), - GATE(0, "pclk_ddrphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 3, GFLAGS), - GATE(0, "pclk_acodecphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 5, GFLAGS), + GATE(0, "pclk_ddrphy", "pclk_phy_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 3, GFLAGS), + GATE(0, "pclk_acodecphy", "pclk_phy_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 5, GFLAGS), GATE(PCLK_HDMI_PHY, "pclk_hdmiphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 7, GFLAGS), - GATE(0, "pclk_vdacphy", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 8, GFLAGS), - GATE(0, "pclk_phy_noc", "pclk_phy_pre", 0, RK2928_CLKGATE_CON(10), 9, GFLAGS), + GATE(0, "pclk_vdacphy", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 8, GFLAGS), + GATE(0, "pclk_phy_noc", "pclk_phy_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(10), 9, GFLAGS), GATE(ACLK_VPU, "aclk_vpu", "aclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 0, GFLAGS), - GATE(0, "aclk_vpu_noc", "aclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 4, GFLAGS), + GATE(0, "aclk_vpu_noc", "aclk_vpu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(15), 4, GFLAGS), GATE(ACLK_RKVDEC, "aclk_rkvdec", "aclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 2, GFLAGS), - GATE(0, "aclk_rkvdec_noc", "aclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 6, GFLAGS), + GATE(0, "aclk_rkvdec_noc", "aclk_rkvdec_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(15), 6, GFLAGS), GATE(HCLK_VPU, "hclk_vpu", "hclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 1, GFLAGS), - GATE(0, "hclk_vpu_noc", "hclk_vpu_pre", 0, RK2928_CLKGATE_CON(15), 5, GFLAGS), + GATE(0, "hclk_vpu_noc", "hclk_vpu_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(15), 5, GFLAGS), GATE(HCLK_RKVDEC, "hclk_rkvdec", "hclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 3, GFLAGS), - GATE(0, "hclk_rkvdec_noc", "hclk_rkvdec_pre", 0, RK2928_CLKGATE_CON(15), 7, GFLAGS), + GATE(0, "hclk_rkvdec_noc", "hclk_rkvdec_pre", CLK_IS_CRITICAL, RK2928_CLKGATE_CON(15), 7, GFLAGS), /* PD_MMC */ MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "sclk_sdmmc", RK3228_SDMMC_CON0, 1), - MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "sclk_sdmmc", RK3228_SDMMC_CON1, 0), + MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "sclk_sdmmc", RK3228_SDMMC_CON1, 1), MMC(SCLK_SDIO_DRV, "sdio_drv", "sclk_sdio", RK3228_SDIO_CON0, 1), - MMC(SCLK_SDIO_SAMPLE, "sdio_sample", "sclk_sdio", RK3228_SDIO_CON1, 0), + MMC(SCLK_SDIO_SAMPLE, "sdio_sample", "sclk_sdio", RK3228_SDIO_CON1, 1), MMC(SCLK_EMMC_DRV, "emmc_drv", "sclk_emmc", RK3228_EMMC_CON0, 1), - MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "sclk_emmc", RK3228_EMMC_CON1, 0), + MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "sclk_emmc", RK3228_EMMC_CON1, 1), }; -static const char *const rk3228_critical_clocks[] __initconst = { - "aclk_cpu", - "pclk_cpu", - "hclk_cpu", - "aclk_peri", - "hclk_peri", - "pclk_peri", - "aclk_rga_noc", - "aclk_iep_noc", - "aclk_vop_noc", - "aclk_hdcp_noc", - "hclk_vio_ahb_arbi", - "hclk_vio_noc", - "hclk_vop_noc", - "hclk_host0_arb", - "hclk_host1_arb", - "hclk_host2_arb", - "hclk_otg_pmu", - "aclk_gpu_noc", - "sclk_initmem_mbist", - "aclk_initmem", - "hclk_rom", - "pclk_ddrupctl", - "pclk_ddrmon", - "pclk_msch_noc", - "pclk_stimer", - "pclk_ddrphy", - "pclk_acodecphy", - "pclk_phy_noc", - "aclk_vpu_noc", - "aclk_rkvdec_noc", - "hclk_vpu_noc", - "hclk_rkvdec_noc", -}; +static void __iomem *rk3228_cru_base; + +static void rk3228_dump_cru(void) +{ + if (rk3228_cru_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk3228_cru_base, + 0x1f8, false); + } +} static void __init rk3228_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; + struct clk **clks; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -696,17 +680,16 @@ static void __init rk3228_clk_init(struct device_node *np) iounmap(reg_base); return; } + clks = ctx->clk_data.clks; rockchip_clk_register_plls(ctx, rk3228_pll_clks, ARRAY_SIZE(rk3228_pll_clks), RK3228_GRF_SOC_STATUS0); rockchip_clk_register_branches(ctx, rk3228_clk_branches, ARRAY_SIZE(rk3228_clk_branches)); - rockchip_clk_protect_critical(rk3228_critical_clocks, - ARRAY_SIZE(rk3228_critical_clocks)); rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", - mux_armclk_p, ARRAY_SIZE(mux_armclk_p), + 3, clks[PLL_APLL], clks[PLL_GPLL], &rk3228_cpuclk_data, rk3228_cpuclk_rates, ARRAY_SIZE(rk3228_cpuclk_rates)); @@ -716,5 +699,38 @@ static void __init rk3228_clk_init(struct device_node *np) rockchip_register_restart_notifier(ctx, RK3228_GLB_SRST_FST, NULL); rockchip_clk_of_add_provider(np, ctx); + + if (!rk_dump_cru) { + rk3228_cru_base = reg_base; + rk_dump_cru = rk3228_dump_cru; + } } CLK_OF_DECLARE(rk3228_cru, "rockchip,rk3228-cru", rk3228_clk_init); + +static int __init clk_rk3228_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + + rk3228_clk_init(np); + + return 0; +} + +static const struct of_device_id clk_rk3228_match_table[] = { + { + .compatible = "rockchip,rk3228-cru", + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk3228_match_table); + +static struct platform_driver clk_rk3228_driver = { + .driver = { + .name = "clk-rk3228", + .of_match_table = clk_rk3228_match_table, + }, +}; +builtin_platform_driver_probe(clk_rk3228_driver, clk_rk3228_probe); + +MODULE_DESCRIPTION("Rockchip RK3228 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rk3288.c b/drivers/clk/rockchip/clk-rk3288.c index 93c794695c46..3e054ddf1931 100644 --- a/drivers/clk/rockchip/clk-rk3288.c +++ b/drivers/clk/rockchip/clk-rk3288.c @@ -6,14 +6,20 @@ #include #include +#include #include #include +#include #include #include #include "clk.h" +#include #define RK3288_GRF_SOC_CON(x) (0x244 + x * 4) #define RK3288_GRF_SOC_STATUS1 0x284 +#define RK3288_UART_FRAC_MAX_PRATE 600000000 +#define RK3288_I2S_FRAC_MAX_PRATE 600000000 +#define RK3288_SPDIF_FRAC_MAX_PRATE 600000000 enum rk3288_variant { RK3288_CRU, @@ -66,32 +72,32 @@ static struct rockchip_pll_rate_table rk3288_pll_rates[] = { RK3066_PLL_RATE(1248000000, 1, 52, 1), RK3066_PLL_RATE(1224000000, 1, 51, 1), RK3066_PLL_RATE(1200000000, 1, 50, 1), - RK3066_PLL_RATE(1188000000, 2, 99, 1), + RK3066_PLL_RATE(1188000000, 1, 99, 2), RK3066_PLL_RATE(1176000000, 1, 49, 1), RK3066_PLL_RATE(1128000000, 1, 47, 1), RK3066_PLL_RATE(1104000000, 1, 46, 1), RK3066_PLL_RATE(1008000000, 1, 84, 2), RK3066_PLL_RATE( 912000000, 1, 76, 2), - RK3066_PLL_RATE( 891000000, 8, 594, 2), + RK3066_PLL_RATE( 891000000, 2, 297, 4), RK3066_PLL_RATE( 888000000, 1, 74, 2), RK3066_PLL_RATE( 816000000, 1, 68, 2), - RK3066_PLL_RATE( 798000000, 2, 133, 2), + RK3066_PLL_RATE( 798000000, 1, 133, 4), RK3066_PLL_RATE( 792000000, 1, 66, 2), RK3066_PLL_RATE( 768000000, 1, 64, 2), - RK3066_PLL_RATE( 742500000, 8, 495, 2), + RK3066_PLL_RATE( 742500000, 4, 495, 4), RK3066_PLL_RATE( 696000000, 1, 58, 2), RK3066_PLL_RATE_NB(621000000, 1, 207, 8, 1), RK3066_PLL_RATE( 600000000, 1, 50, 2), - RK3066_PLL_RATE_NB(594000000, 1, 198, 8, 1), + RK3066_PLL_RATE_NB(594000000, 1, 99, 4, 1), RK3066_PLL_RATE( 552000000, 1, 46, 2), RK3066_PLL_RATE( 504000000, 1, 84, 4), - RK3066_PLL_RATE( 500000000, 3, 125, 2), + RK3066_PLL_RATE( 500000000, 1, 125, 6), RK3066_PLL_RATE( 456000000, 1, 76, 4), RK3066_PLL_RATE( 428000000, 1, 107, 6), RK3066_PLL_RATE( 408000000, 1, 68, 4), - RK3066_PLL_RATE( 400000000, 3, 100, 2), + RK3066_PLL_RATE( 400000000, 1, 100, 6), RK3066_PLL_RATE_NB( 394000000, 1, 197, 12, 1), - RK3066_PLL_RATE( 384000000, 2, 128, 4), + RK3066_PLL_RATE( 384000000, 1, 64, 4), RK3066_PLL_RATE( 360000000, 1, 60, 4), RK3066_PLL_RATE_NB( 356000000, 1, 178, 12, 1), RK3066_PLL_RATE_NB( 324000000, 1, 189, 14, 1), @@ -100,6 +106,7 @@ static struct rockchip_pll_rate_table rk3288_pll_rates[] = { RK3066_PLL_RATE_NB( 303000000, 1, 202, 16, 1), RK3066_PLL_RATE( 300000000, 1, 75, 6), RK3066_PLL_RATE_NB( 297750000, 2, 397, 16, 1), + RK3066_PLL_RATE( 297000000, 1, 99, 8), RK3066_PLL_RATE_NB( 293250000, 2, 391, 16, 1), RK3066_PLL_RATE_NB( 292500000, 1, 195, 16, 1), RK3066_PLL_RATE( 273600000, 1, 114, 10), @@ -117,6 +124,7 @@ static struct rockchip_pll_rate_table rk3288_pll_rates[] = { RK3066_PLL_RATE( 195428571, 1, 114, 14), RK3066_PLL_RATE( 160000000, 1, 80, 12), RK3066_PLL_RATE( 157500000, 1, 105, 16), + RK3066_PLL_RATE( 148500000, 1, 99, 16), RK3066_PLL_RATE( 126000000, 1, 84, 16), { /* sentinel */ }, }; @@ -179,9 +187,10 @@ static struct rockchip_cpuclk_rate_table rk3288_cpuclk_rates[] __initdata = { }; static const struct rockchip_cpuclk_reg_data rk3288_cpuclk_data = { - .core_reg = RK3288_CLKSEL_CON(0), - .div_core_shift = 8, - .div_core_mask = 0x1f, + .core_reg[0] = RK3288_CLKSEL_CON(0), + .div_core_shift[0] = 8, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 0, .mux_core_shift = 15, @@ -189,7 +198,6 @@ static const struct rockchip_cpuclk_reg_data rk3288_cpuclk_data = { }; PNAME(mux_pll_p) = { "xin24m", "xin32k" }; -PNAME(mux_armclk_p) = { "apll_core", "gpll_core" }; PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr" }; PNAME(mux_aclk_cpu_src_p) = { "cpll_aclk_cpu", "gpll_aclk_cpu" }; @@ -330,20 +338,20 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { RK3288_CLKSEL_CON(26), 2, 1, MFLAGS, 0, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO), - GATE(0, "gpll_aclk_cpu", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_aclk_cpu", "gpll", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(0), 10, GFLAGS), - GATE(0, "cpll_aclk_cpu", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_aclk_cpu", "cpll", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(0), 11, GFLAGS), - COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, CLK_IGNORE_UNUSED, + COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, CLK_IS_CRITICAL, RK3288_CLKSEL_CON(1), 15, 1, MFLAGS, 3, 5, DFLAGS), DIV(0, "aclk_cpu_pre", "aclk_cpu_src", CLK_SET_RATE_PARENT, RK3288_CLKSEL_CON(1), 0, 3, DFLAGS), - GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_pre", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(0), 3, GFLAGS), - COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_pre", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(PCLK_CPU, "pclk_cpu", "aclk_cpu_pre", CLK_IS_CRITICAL, RK3288_CLKSEL_CON(1), 12, 3, DFLAGS, RK3288_CLKGATE_CON(0), 5, GFLAGS), - COMPOSITE_NOMUX_DIVTBL(HCLK_CPU, "hclk_cpu", "aclk_cpu_pre", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX_DIVTBL(HCLK_CPU, "hclk_cpu", "aclk_cpu_pre", CLK_IS_CRITICAL, RK3288_CLKSEL_CON(1), 8, 2, DFLAGS, div_hclk_cpu_t, RK3288_CLKGATE_CON(0), 4, GFLAGS), GATE(0, "c2c_host", "aclk_cpu_src", 0, @@ -362,7 +370,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s_frac", "i2s_src", CLK_SET_RATE_PARENT, RK3288_CLKSEL_CON(8), 0, RK3288_CLKGATE_CON(4), 2, GFLAGS, - &rk3288_i2s_fracmux), + &rk3288_i2s_fracmux, RK3288_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S0_OUT, "i2s0_clkout", mux_i2s_clkout_p, 0, RK3288_CLKSEL_CON(4), 12, 1, MFLAGS, RK3288_CLKGATE_CON(4), 0, GFLAGS), @@ -377,7 +385,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "spdif_frac", "spdif_src", CLK_SET_RATE_PARENT, RK3288_CLKSEL_CON(9), 0, RK3288_CLKGATE_CON(4), 5, GFLAGS, - &rk3288_spdif_fracmux), + &rk3288_spdif_fracmux, RK3288_SPDIF_FRAC_MAX_PRATE), GATE(SCLK_SPDIF, "sclk_spdif", "spdif_mux", CLK_SET_RATE_PARENT, RK3288_CLKGATE_CON(4), 6, GFLAGS), COMPOSITE_NOMUX(0, "spdif_8ch_pre", "spdif_src", CLK_SET_RATE_PARENT, @@ -386,7 +394,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "spdif_8ch_frac", "spdif_8ch_pre", CLK_SET_RATE_PARENT, RK3288_CLKSEL_CON(41), 0, RK3288_CLKGATE_CON(4), 8, GFLAGS, - &rk3288_spdif_8ch_fracmux), + &rk3288_spdif_8ch_fracmux, RK3288_SPDIF_FRAC_MAX_PRATE), GATE(SCLK_SPDIF8CH, "sclk_spdif_8ch", "spdif_8ch_mux", CLK_SET_RATE_PARENT, RK3288_CLKGATE_CON(4), 9, GFLAGS), @@ -486,9 +494,9 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { COMPOSITE_NOGATE(SCLK_VIP_OUT, "sclk_vip_out", mux_vip_out_p, 0, RK3288_CLKSEL_CON(26), 15, 1, MFLAGS, 9, 5, DFLAGS), - DIV(0, "pclk_pd_alive", "gpll", 0, + DIV(PCLK_PD_ALIVE, "pclk_pd_alive", "gpll", CLK_IS_CRITICAL, RK3288_CLKSEL_CON(33), 8, 5, DFLAGS), - COMPOSITE_NOMUX(0, "pclk_pd_pmu", "gpll", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(PCLK_PD_PMU, "pclk_pd_pmu", "gpll", CLK_IS_CRITICAL, RK3288_CLKSEL_CON(33), 0, 5, DFLAGS, RK3288_CLKGATE_CON(5), 8, GFLAGS), @@ -496,16 +504,16 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { RK3288_CLKSEL_CON(34), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3288_CLKGATE_CON(5), 7, GFLAGS), - COMPOSITE(0, "aclk_peri_src", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED, + COMPOSITE(0, "aclk_peri_src", mux_pll_src_cpll_gpll_p, CLK_IS_CRITICAL, RK3288_CLKSEL_CON(10), 15, 1, MFLAGS, 0, 5, DFLAGS, RK3288_CLKGATE_CON(2), 0, GFLAGS), - COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", 0, + COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK3288_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, RK3288_CLKGATE_CON(2), 3, GFLAGS), - COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK3288_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, RK3288_CLKGATE_CON(2), 2, GFLAGS), - GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IGNORE_UNUSED, + GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(2), 1, GFLAGS), /* @@ -587,7 +595,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, RK3288_CLKSEL_CON(17), 0, RK3288_CLKGATE_CON(1), 9, GFLAGS, - &rk3288_uart0_fracmux), + &rk3288_uart0_fracmux, RK3288_UART_FRAC_MAX_PRATE), MUX(0, "uart_src", mux_pll_src_cpll_gpll_p, 0, RK3288_CLKSEL_CON(13), 15, 1, MFLAGS), COMPOSITE_NOMUX(0, "uart1_src", "uart_src", 0, @@ -596,28 +604,28 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, RK3288_CLKSEL_CON(18), 0, RK3288_CLKGATE_CON(1), 11, GFLAGS, - &rk3288_uart1_fracmux), + &rk3288_uart1_fracmux, RK3288_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "uart2_src", "uart_src", 0, RK3288_CLKSEL_CON(15), 0, 7, DFLAGS, RK3288_CLKGATE_CON(1), 12, GFLAGS), COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, RK3288_CLKSEL_CON(19), 0, RK3288_CLKGATE_CON(1), 13, GFLAGS, - &rk3288_uart2_fracmux), + &rk3288_uart2_fracmux, RK3288_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "uart3_src", "uart_src", 0, RK3288_CLKSEL_CON(16), 0, 7, DFLAGS, RK3288_CLKGATE_CON(1), 14, GFLAGS), COMPOSITE_FRACMUX(0, "uart3_frac", "uart3_src", CLK_SET_RATE_PARENT, RK3288_CLKSEL_CON(20), 0, RK3288_CLKGATE_CON(1), 15, GFLAGS, - &rk3288_uart3_fracmux), + &rk3288_uart3_fracmux, RK3288_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "uart4_src", "uart_src", 0, RK3288_CLKSEL_CON(3), 0, 7, DFLAGS, RK3288_CLKGATE_CON(2), 12, GFLAGS), COMPOSITE_FRACMUX(0, "uart4_frac", "uart4_src", CLK_SET_RATE_PARENT, RK3288_CLKSEL_CON(7), 0, RK3288_CLKGATE_CON(2), 13, GFLAGS, - &rk3288_uart4_fracmux), + &rk3288_uart4_fracmux, RK3288_UART_FRAC_MAX_PRATE), COMPOSITE(0, "mac_pll_src", mux_pll_src_npll_cpll_gpll_p, 0, RK3288_CLKSEL_CON(21), 0, 2, MFLAGS, 8, 5, DFLAGS, @@ -665,7 +673,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { GATE(0, "sclk_intmem0", "aclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(10), 5, GFLAGS), GATE(0, "sclk_intmem1", "aclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(10), 6, GFLAGS), GATE(0, "sclk_intmem2", "aclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(10), 7, GFLAGS), - GATE(ACLK_DMAC1, "aclk_dmac1", "aclk_cpu", 0, RK3288_CLKGATE_CON(10), 12, GFLAGS), + GATE(ACLK_DMAC1, "aclk_dmac1", "aclk_cpu", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(10), 12, GFLAGS), GATE(0, "aclk_strc_sys", "aclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(10), 13, GFLAGS), GATE(0, "aclk_intmem", "aclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(10), 4, GFLAGS), GATE(ACLK_CRYPTO, "aclk_crypto", "aclk_cpu", 0, RK3288_CLKGATE_CON(11), 6, GFLAGS), @@ -691,7 +699,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { GATE(PCLK_TZPC, "pclk_tzpc", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 3, GFLAGS), GATE(PCLK_UART2, "pclk_uart2", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 9, GFLAGS), GATE(PCLK_EFUSE256, "pclk_efuse_256", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 10, GFLAGS), - GATE(PCLK_RKPWM, "pclk_rkpwm", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 11, GFLAGS), + GATE(PCLK_RKPWM, "pclk_rkpwm", "pclk_cpu", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(11), 11, GFLAGS), /* ddrctrl [DDR Controller PHY clock] gates */ GATE(0, "nclk_ddrupctl0", "ddrphy", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(11), 4, GFLAGS), @@ -727,7 +735,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { GATE(HCLK_SDIO1, "hclk_sdio1", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 5, GFLAGS), GATE(HCLK_EMMC, "hclk_emmc", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 6, GFLAGS), GATE(HCLK_HSADC, "hclk_hsadc", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 7, GFLAGS), - GATE(0, "pmu_hclk_otg0", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 5, GFLAGS), + GATE(0, "pmu_hclk_otg0", "hclk_peri", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(7), 5, GFLAGS), /* pclk_peri gates */ GATE(0, "pclk_peri_matrix", "pclk_peri", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(6), 1, GFLAGS), @@ -767,7 +775,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { GATE(PCLK_GPIO5, "pclk_gpio5", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 5, GFLAGS), GATE(PCLK_GPIO6, "pclk_gpio6", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 6, GFLAGS), GATE(PCLK_GRF, "pclk_grf", "pclk_pd_alive", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(14), 11, GFLAGS), - GATE(0, "pclk_alive_niu", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 12, GFLAGS), + GATE(0, "pclk_alive_niu", "pclk_pd_alive", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(14), 12, GFLAGS), /* Watchdog pclk is controlled by RK3288_SGRF_SOC_CON0[1]. */ SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_pd_alive"), @@ -775,7 +783,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { /* pclk_pd_pmu gates */ GATE(PCLK_PMU, "pclk_pmu", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(17), 0, GFLAGS), GATE(0, "pclk_intmem1", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(17), 1, GFLAGS), - GATE(0, "pclk_pmu_niu", "pclk_pd_pmu", 0, RK3288_CLKGATE_CON(17), 2, GFLAGS), + GATE(0, "pclk_pmu_niu", "pclk_pd_pmu", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(17), 2, GFLAGS), GATE(PCLK_SGRF, "pclk_sgrf", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(17), 3, GFLAGS), GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_pd_pmu", 0, RK3288_CLKGATE_CON(17), 4, GFLAGS), @@ -784,7 +792,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { GATE(HCLK_VOP0, "hclk_vop0", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 6, GFLAGS), GATE(HCLK_VOP1, "hclk_vop1", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 8, GFLAGS), GATE(HCLK_VIO_AHB_ARBI, "hclk_vio_ahb_arbi", "hclk_vio", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(15), 9, GFLAGS), - GATE(HCLK_VIO_NIU, "hclk_vio_niu", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 10, GFLAGS), + GATE(HCLK_VIO_NIU, "hclk_vio_niu", "hclk_vio", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(15), 10, GFLAGS), GATE(HCLK_VIP, "hclk_vip", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 15, GFLAGS), GATE(HCLK_IEP, "hclk_iep", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 3, GFLAGS), GATE(HCLK_ISP, "hclk_isp", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 1, GFLAGS), @@ -800,17 +808,17 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { /* aclk_vio0 gates */ GATE(ACLK_VOP0, "aclk_vop0", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 5, GFLAGS), GATE(ACLK_IEP, "aclk_iep", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 2, GFLAGS), - GATE(ACLK_VIO0_NIU, "aclk_vio0_niu", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 11, GFLAGS), + GATE(ACLK_VIO0_NIU, "aclk_vio0_niu", "aclk_vio0", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(15), 11, GFLAGS), GATE(ACLK_VIP, "aclk_vip", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 14, GFLAGS), /* aclk_vio1 gates */ GATE(ACLK_VOP1, "aclk_vop1", "aclk_vio1", 0, RK3288_CLKGATE_CON(15), 7, GFLAGS), GATE(ACLK_ISP, "aclk_isp", "aclk_vio1", 0, RK3288_CLKGATE_CON(16), 2, GFLAGS), - GATE(ACLK_VIO1_NIU, "aclk_vio1_niu", "aclk_vio1", 0, RK3288_CLKGATE_CON(15), 12, GFLAGS), + GATE(ACLK_VIO1_NIU, "aclk_vio1_niu", "aclk_vio1", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(15), 12, GFLAGS), /* aclk_rga_pre gates */ GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, RK3288_CLKGATE_CON(15), 0, GFLAGS), - GATE(ACLK_RGA_NIU, "aclk_rga_niu", "aclk_rga_pre", 0, RK3288_CLKGATE_CON(15), 13, GFLAGS), + GATE(ACLK_RGA_NIU, "aclk_rga_niu", "aclk_rga_pre", CLK_IS_CRITICAL, RK3288_CLKGATE_CON(15), 13, GFLAGS), /* * Other ungrouped clocks. @@ -832,23 +840,6 @@ static struct rockchip_clk_branch rk3288_hclkvio_branch[] __initdata = { RK3288_CLKSEL_CON(28), 8, 5, DFLAGS), }; -static const char *const rk3288_critical_clocks[] __initconst = { - "aclk_cpu", - "aclk_peri", - "aclk_peri_niu", - "aclk_vio0_niu", - "aclk_vio1_niu", - "aclk_rga_niu", - "hclk_peri", - "hclk_vio_niu", - "pclk_alive_niu", - "pclk_pd_pmu", - "pclk_pmu_niu", - "pmu_hclk_otg0", - /* pwm-regulators on some boards, so handoff-critical later */ - "pclk_rkpwm", -}; - static void __iomem *rk3288_cru_base; /* @@ -927,10 +918,21 @@ static struct syscore_ops rk3288_clk_syscore_ops = { .resume = rk3288_clk_resume, }; +static void rk3288_dump_cru(void) +{ + if (rk3288_cru_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk3288_cru_base, + 0x21c, false); + } +} + static void __init rk3288_common_init(struct device_node *np, enum rk3288_variant soc) { struct rockchip_clk_provider *ctx; + struct clk **clks; rk3288_cru_base = of_iomap(np, 0); if (!rk3288_cru_base) { @@ -944,6 +946,7 @@ static void __init rk3288_common_init(struct device_node *np, iounmap(rk3288_cru_base); return; } + clks = ctx->clk_data.clks; rockchip_clk_register_plls(ctx, rk3288_pll_clks, ARRAY_SIZE(rk3288_pll_clks), @@ -958,11 +961,8 @@ static void __init rk3288_common_init(struct device_node *np, rockchip_clk_register_branches(ctx, rk3288_hclkvio_branch, ARRAY_SIZE(rk3288_hclkvio_branch)); - rockchip_clk_protect_critical(rk3288_critical_clocks, - ARRAY_SIZE(rk3288_critical_clocks)); - rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", - mux_armclk_p, ARRAY_SIZE(mux_armclk_p), + 2, clks[PLL_APLL], clks[PLL_GPLL], &rk3288_cpuclk_data, rk3288_cpuclk_rates, ARRAY_SIZE(rk3288_cpuclk_rates)); @@ -972,9 +972,14 @@ static void __init rk3288_common_init(struct device_node *np, rockchip_register_restart_notifier(ctx, RK3288_GLB_SRST_FST, rk3288_clk_shutdown); - register_syscore_ops(&rk3288_clk_syscore_ops); + + if (!psci_smp_available()) + register_syscore_ops(&rk3288_clk_syscore_ops); rockchip_clk_of_add_provider(np, ctx); + + if (!rk_dump_cru) + rk_dump_cru = rk3288_dump_cru; } static void __init rk3288_clk_init(struct device_node *np) @@ -988,3 +993,55 @@ static void __init rk3288w_clk_init(struct device_node *np) rk3288_common_init(np, RK3288W_CRU); } CLK_OF_DECLARE(rk3288w_cru, "rockchip,rk3288w-cru", rk3288w_clk_init); + +struct clk_rk3288_inits { + void (*inits)(struct device_node *np); +}; + +static const struct clk_rk3288_inits clk_rk3288_init = { + .inits = rk3288_clk_init, +}; + +static const struct clk_rk3288_inits clk_rk3288w_init = { + .inits = rk3288w_clk_init, +}; + +static const struct of_device_id clk_rk3288_match_table[] = { + { + .compatible = "rockchip,rk3288-cru", + .data = &clk_rk3288_init, + }, { + .compatible = "rockchip,rk3288w-cru", + .data = &clk_rk3288w_init, + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk3288_match_table); + +static int __init clk_rk3288_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + const struct clk_rk3288_inits *init_data; + + match = of_match_device(clk_rk3288_match_table, &pdev->dev); + if (!match || !match->data) + return -EINVAL; + + init_data = match->data; + if (init_data->inits) + init_data->inits(np); + + return 0; +} + +static struct platform_driver clk_rk3288_driver = { + .driver = { + .name = "clk-rk3288", + .of_match_table = clk_rk3288_match_table, + }, +}; +builtin_platform_driver_probe(clk_rk3288_driver, clk_rk3288_probe); + +MODULE_DESCRIPTION("Rockchip RK3288 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rk3308.c b/drivers/clk/rockchip/clk-rk3308.c index 5bf15f2a44b7..539d37ae89c9 100644 --- a/drivers/clk/rockchip/clk-rk3308.c +++ b/drivers/clk/rockchip/clk-rk3308.c @@ -8,11 +8,20 @@ #include #include #include +#include +#include +#include #include #include #include "clk.h" #define RK3308_GRF_SOC_STATUS0 0x380 +#define RK3308_VOP_FRAC_MAX_PRATE 270000000 +#define RK3308B_VOP_FRAC_MAX_PRATE 800000000 +#define RK3308_UART_FRAC_MAX_PRATE 800000000 +#define RK3308_PDM_FRAC_MAX_PRATE 800000000 +#define RK3308_SPDIF_FRAC_MAX_PRATE 800000000 +#define RK3308_I2S_FRAC_MAX_PRATE 800000000 enum rk3308_plls { apll, dpll, vpll0, vpll1, @@ -109,9 +118,10 @@ static struct rockchip_cpuclk_rate_table rk3308_cpuclk_rates[] __initdata = { }; static const struct rockchip_cpuclk_reg_data rk3308_cpuclk_data = { - .core_reg = RK3308_CLKSEL_CON(0), - .div_core_shift = 0, - .div_core_mask = 0xf, + .core_reg[0] = RK3308_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0xf, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 0, .mux_core_shift = 6, @@ -120,7 +130,6 @@ static const struct rockchip_cpuclk_reg_data rk3308_cpuclk_data = { PNAME(mux_pll_p) = { "xin24m" }; PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc32k" }; -PNAME(mux_armclk_p) = { "apll_core", "vpll0_core", "vpll1_core" }; PNAME(mux_dpll_vpll0_p) = { "dpll", "vpll0" }; PNAME(mux_dpll_vpll0_xin24m_p) = { "dpll", "vpll0", "xin24m" }; PNAME(mux_dpll_vpll0_vpll1_p) = { "dpll", "vpll0", "vpll1" }; @@ -174,6 +183,8 @@ PNAME(mux_spdif_tx_src_p) = { "clk_spdif_tx_div", "clk_spdif_tx_div50" }; PNAME(mux_spdif_tx_p) = { "clk_spdif_tx_src", "clk_spdif_tx_frac", "mclk_i2s0_2ch_in" }; PNAME(mux_spdif_rx_src_p) = { "clk_spdif_rx_div", "clk_spdif_rx_div50" }; PNAME(mux_spdif_rx_p) = { "clk_spdif_rx_src", "clk_spdif_rx_frac" }; +PNAME(mux_uart_src_p) = { "xin24m", "usb480m", "dpll", "vpll0", "vpll1" }; +static u32 uart_src_mux_idx[] = { 4, 3, 0, 1, 2 }; static struct rockchip_pll_clock rk3308_pll_clks[] __initdata = { [apll] = PLL(pll_rk3328, PLL_APLL, "apll", mux_pll_p, @@ -311,68 +322,68 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { * Clock-Architecture Diagram 3 */ - COMPOSITE_NODIV(ACLK_BUS_SRC, "clk_bus_src", mux_dpll_vpll0_vpll1_p, CLK_IGNORE_UNUSED, + COMPOSITE_NODIV(ACLK_BUS_SRC, "clk_bus_src", mux_dpll_vpll0_vpll1_p, CLK_IS_CRITICAL, RK3308_CLKSEL_CON(5), 6, 2, MFLAGS, RK3308_CLKGATE_CON(1), 0, GFLAGS), - COMPOSITE_NOMUX(PCLK_BUS, "pclk_bus", "clk_bus_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(PCLK_BUS, "pclk_bus", "clk_bus_src", CLK_IS_CRITICAL, RK3308_CLKSEL_CON(6), 8, 5, DFLAGS, RK3308_CLKGATE_CON(1), 3, GFLAGS), GATE(PCLK_DDR, "pclk_ddr", "pclk_bus", CLK_IGNORE_UNUSED, RK3308_CLKGATE_CON(4), 15, GFLAGS), - COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus", "clk_bus_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus", "clk_bus_src", CLK_IS_CRITICAL, RK3308_CLKSEL_CON(6), 0, 5, DFLAGS, RK3308_CLKGATE_CON(1), 2, GFLAGS), - COMPOSITE_NOMUX(ACLK_BUS, "aclk_bus", "clk_bus_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(ACLK_BUS, "aclk_bus", "clk_bus_src", CLK_IS_CRITICAL, RK3308_CLKSEL_CON(5), 0, 5, DFLAGS, RK3308_CLKGATE_CON(1), 1, GFLAGS), - COMPOSITE(0, "clk_uart0_src", mux_dpll_vpll0_vpll1_usb480m_xin24m_p, 0, - RK3308_CLKSEL_CON(10), 13, 3, MFLAGS, 0, 5, DFLAGS, + COMPOSITE_MUXTBL(0, "clk_uart0_src", mux_uart_src_p, 0, + RK3308_CLKSEL_CON(10), 13, 3, MFLAGS, uart_src_mux_idx, 0, 5, DFLAGS, RK3308_CLKGATE_CON(1), 9, GFLAGS), COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(12), 0, RK3308_CLKGATE_CON(1), 11, GFLAGS, - &rk3308_uart0_fracmux), + &rk3308_uart0_fracmux, RK3308_UART_FRAC_MAX_PRATE), GATE(SCLK_UART0, "clk_uart0", "clk_uart0_mux", 0, RK3308_CLKGATE_CON(1), 12, GFLAGS), - COMPOSITE(0, "clk_uart1_src", mux_dpll_vpll0_vpll1_usb480m_xin24m_p, 0, - RK3308_CLKSEL_CON(13), 13, 3, MFLAGS, 0, 5, DFLAGS, + COMPOSITE_MUXTBL(0, "clk_uart1_src", mux_uart_src_p, 0, + RK3308_CLKSEL_CON(13), 13, 3, MFLAGS, uart_src_mux_idx, 0, 5, DFLAGS, RK3308_CLKGATE_CON(1), 13, GFLAGS), COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(15), 0, RK3308_CLKGATE_CON(1), 15, GFLAGS, - &rk3308_uart1_fracmux), + &rk3308_uart1_fracmux, RK3308_UART_FRAC_MAX_PRATE), GATE(SCLK_UART1, "clk_uart1", "clk_uart1_mux", 0, RK3308_CLKGATE_CON(2), 0, GFLAGS), - COMPOSITE(0, "clk_uart2_src", mux_dpll_vpll0_vpll1_usb480m_xin24m_p, 0, - RK3308_CLKSEL_CON(16), 13, 3, MFLAGS, 0, 5, DFLAGS, + COMPOSITE_MUXTBL(0, "clk_uart2_src", mux_uart_src_p, 0, + RK3308_CLKSEL_CON(16), 13, 3, MFLAGS, uart_src_mux_idx, 0, 5, DFLAGS, RK3308_CLKGATE_CON(2), 1, GFLAGS), COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(18), 0, RK3308_CLKGATE_CON(2), 3, GFLAGS, - &rk3308_uart2_fracmux), + &rk3308_uart2_fracmux, RK3308_UART_FRAC_MAX_PRATE), GATE(SCLK_UART2, "clk_uart2", "clk_uart2_mux", CLK_SET_RATE_PARENT, RK3308_CLKGATE_CON(2), 4, GFLAGS), - COMPOSITE(0, "clk_uart3_src", mux_dpll_vpll0_vpll1_usb480m_xin24m_p, 0, - RK3308_CLKSEL_CON(19), 13, 3, MFLAGS, 0, 5, DFLAGS, + COMPOSITE_MUXTBL(0, "clk_uart3_src", mux_uart_src_p, 0, + RK3308_CLKSEL_CON(19), 13, 3, MFLAGS, uart_src_mux_idx, 0, 5, DFLAGS, RK3308_CLKGATE_CON(2), 5, GFLAGS), COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(21), 0, RK3308_CLKGATE_CON(2), 7, GFLAGS, - &rk3308_uart3_fracmux), + &rk3308_uart3_fracmux, RK3308_UART_FRAC_MAX_PRATE), GATE(SCLK_UART3, "clk_uart3", "clk_uart3_mux", 0, RK3308_CLKGATE_CON(2), 8, GFLAGS), - COMPOSITE(0, "clk_uart4_src", mux_dpll_vpll0_vpll1_usb480m_xin24m_p, 0, - RK3308_CLKSEL_CON(22), 13, 3, MFLAGS, 0, 5, DFLAGS, + COMPOSITE_MUXTBL(0, "clk_uart4_src", mux_uart_src_p, 0, + RK3308_CLKSEL_CON(22), 13, 3, MFLAGS, uart_src_mux_idx, 0, 5, DFLAGS, RK3308_CLKGATE_CON(2), 9, GFLAGS), COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(24), 0, RK3308_CLKGATE_CON(2), 11, GFLAGS, - &rk3308_uart4_fracmux), + &rk3308_uart4_fracmux, RK3308_UART_FRAC_MAX_PRATE), GATE(SCLK_UART4, "clk_uart4", "clk_uart4_mux", 0, RK3308_CLKGATE_CON(2), 12, GFLAGS), @@ -449,10 +460,6 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE(0, "dclk_vop_src", mux_dpll_vpll0_vpll1_p, 0, RK3308_CLKSEL_CON(8), 10, 2, MFLAGS, 0, 8, DFLAGS, RK3308_CLKGATE_CON(1), 6, GFLAGS), - COMPOSITE_FRACMUX(0, "dclk_vop_frac", "dclk_vop_src", CLK_SET_RATE_PARENT, - RK3308_CLKSEL_CON(9), 0, - RK3308_CLKGATE_CON(1), 7, GFLAGS, - &rk3308_dclk_vop_fracmux), GATE(DCLK_VOP, "dclk_vop", "dclk_vop_mux", 0, RK3308_CLKGATE_CON(1), 8, GFLAGS), @@ -460,16 +467,16 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { * Clock-Architecture Diagram 4 */ - COMPOSITE_NODIV(ACLK_PERI_SRC, "clk_peri_src", mux_dpll_vpll0_vpll1_p, CLK_IGNORE_UNUSED, + COMPOSITE_NODIV(ACLK_PERI_SRC, "clk_peri_src", mux_dpll_vpll0_vpll1_p, CLK_IS_CRITICAL, RK3308_CLKSEL_CON(36), 6, 2, MFLAGS, RK3308_CLKGATE_CON(8), 0, GFLAGS), - COMPOSITE_NOMUX(ACLK_PERI, "aclk_peri", "clk_peri_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(ACLK_PERI, "aclk_peri", "clk_peri_src", CLK_IS_CRITICAL, RK3308_CLKSEL_CON(36), 0, 5, DFLAGS, RK3308_CLKGATE_CON(8), 1, GFLAGS), - COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "clk_peri_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "clk_peri_src", CLK_IS_CRITICAL, RK3308_CLKSEL_CON(37), 0, 5, DFLAGS, RK3308_CLKGATE_CON(8), 2, GFLAGS), - COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "clk_peri_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "clk_peri_src", CLK_IS_CRITICAL, RK3308_CLKSEL_CON(37), 8, 5, DFLAGS, RK3308_CLKGATE_CON(8), 3, GFLAGS), @@ -560,10 +567,10 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { GATE(0, "clk_ddr_msch_peribus", "clk_ddrphy1x_out", CLK_IGNORE_UNUSED, RK3308_CLKGATE_CON(4), 13, GFLAGS), - COMPOSITE(SCLK_DDRCLK, "clk_ddrphy4x_src", mux_dpll_vpll0_vpll1_p, CLK_IGNORE_UNUSED, + COMPOSITE(SCLK_DDRCLK, "clk_ddrphy4x_src", mux_dpll_vpll0_vpll1_p, CLK_IS_CRITICAL, RK3308_CLKSEL_CON(1), 6, 2, MFLAGS, 0, 3, DFLAGS, RK3308_CLKGATE_CON(0), 10, GFLAGS), - GATE(0, "clk_ddrphy4x", "clk_ddrphy4x_src", CLK_IGNORE_UNUSED, + GATE(0, "clk_ddrphy4x", "clk_ddrphy4x_src", CLK_IS_CRITICAL, RK3308_CLKGATE_CON(0), 11, GFLAGS), FACTOR_GATE(0, "clk_ddr_stdby_div4", "clk_ddrphy4x", CLK_IGNORE_UNUSED, 1, 4, RK3308_CLKGATE_CON(0), 13, GFLAGS), @@ -583,7 +590,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_rtc32k_frac", "xin24m", CLK_IGNORE_UNUSED, RK3308_CLKSEL_CON(3), 0, RK3308_CLKGATE_CON(4), 3, GFLAGS, - &rk3308_rtc32k_fracmux), + &rk3308_rtc32k_fracmux, 0), MUX(0, "clk_rtc32k_div_src", mux_vpll0_vpll1_p, 0, RK3308_CLKSEL_CON(2), 10, 1, MFLAGS), COMPOSITE_NOMUX(0, "clk_rtc32k_div", "clk_rtc32k_div_src", CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, @@ -617,13 +624,13 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { * Clock-Architecture Diagram 7 */ - COMPOSITE_NODIV(0, "clk_audio_src", mux_vpll0_vpll1_xin24m_p, 0, + COMPOSITE_NODIV(0, "clk_audio_src", mux_vpll0_vpll1_xin24m_p, CLK_IS_CRITICAL, RK3308_CLKSEL_CON(45), 6, 2, MFLAGS, RK3308_CLKGATE_CON(10), 0, GFLAGS), - COMPOSITE_NOMUX(HCLK_AUDIO, "hclk_audio", "clk_audio_src", 0, + COMPOSITE_NOMUX(HCLK_AUDIO, "hclk_audio", "clk_audio_src", CLK_IS_CRITICAL, RK3308_CLKSEL_CON(45), 0, 5, DFLAGS, RK3308_CLKGATE_CON(10), 1, GFLAGS), - COMPOSITE_NOMUX(PCLK_AUDIO, "pclk_audio", "clk_audio_src", 0, + COMPOSITE_NOMUX(PCLK_AUDIO, "pclk_audio", "clk_audio_src", CLK_IS_CRITICAL, RK3308_CLKSEL_CON(45), 8, 5, DFLAGS, RK3308_CLKGATE_CON(10), 2, GFLAGS), @@ -633,7 +640,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_pdm_frac", "clk_pdm_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(47), 0, RK3308_CLKGATE_CON(10), 4, GFLAGS, - &rk3308_pdm_fracmux), + &rk3308_pdm_fracmux, RK3308_PDM_FRAC_MAX_PRATE), GATE(SCLK_PDM, "clk_pdm", "clk_pdm_mux", 0, RK3308_CLKGATE_CON(10), 5, GFLAGS), @@ -643,7 +650,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s0_8ch_tx_frac", "clk_i2s0_8ch_tx_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(53), 0, RK3308_CLKGATE_CON(10), 13, GFLAGS, - &rk3308_i2s0_8ch_tx_fracmux), + &rk3308_i2s0_8ch_tx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S0_8CH_TX, "clk_i2s0_8ch_tx", mux_i2s0_8ch_tx_rx_p, CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(52), 12, 1, MFLAGS, RK3308_CLKGATE_CON(10), 14, GFLAGS), @@ -657,7 +664,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s0_8ch_rx_frac", "clk_i2s0_8ch_rx_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(55), 0, RK3308_CLKGATE_CON(11), 1, GFLAGS, - &rk3308_i2s0_8ch_rx_fracmux), + &rk3308_i2s0_8ch_rx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S0_8CH_RX, "clk_i2s0_8ch_rx", mux_i2s0_8ch_rx_tx_p, CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(54), 12, 1, MFLAGS, RK3308_CLKGATE_CON(11), 2, GFLAGS), @@ -670,7 +677,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s1_8ch_tx_frac", "clk_i2s1_8ch_tx_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(57), 0, RK3308_CLKGATE_CON(11), 5, GFLAGS, - &rk3308_i2s1_8ch_tx_fracmux), + &rk3308_i2s1_8ch_tx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S1_8CH_TX, "clk_i2s1_8ch_tx", mux_i2s1_8ch_tx_rx_p, CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(56), 12, 1, MFLAGS, RK3308_CLKGATE_CON(11), 6, GFLAGS), @@ -684,7 +691,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s1_8ch_rx_frac", "clk_i2s1_8ch_rx_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(59), 0, RK3308_CLKGATE_CON(11), 9, GFLAGS, - &rk3308_i2s1_8ch_rx_fracmux), + &rk3308_i2s1_8ch_rx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S1_8CH_RX, "clk_i2s1_8ch_rx", mux_i2s1_8ch_rx_tx_p, CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(58), 12, 1, MFLAGS, RK3308_CLKGATE_CON(11), 10, GFLAGS), @@ -697,7 +704,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s2_8ch_tx_frac", "clk_i2s2_8ch_tx_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(61), 0, RK3308_CLKGATE_CON(11), 13, GFLAGS, - &rk3308_i2s2_8ch_tx_fracmux), + &rk3308_i2s2_8ch_tx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S2_8CH_TX, "clk_i2s2_8ch_tx", mux_i2s2_8ch_tx_rx_p, CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(60), 12, 1, MFLAGS, RK3308_CLKGATE_CON(11), 14, GFLAGS), @@ -711,7 +718,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s2_8ch_rx_frac", "clk_i2s2_8ch_rx_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(63), 0, RK3308_CLKGATE_CON(12), 1, GFLAGS, - &rk3308_i2s2_8ch_rx_fracmux), + &rk3308_i2s2_8ch_rx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S2_8CH_RX, "clk_i2s2_8ch_rx", mux_i2s2_8ch_rx_tx_p, CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(62), 12, 1, MFLAGS, RK3308_CLKGATE_CON(12), 2, GFLAGS), @@ -724,7 +731,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s3_8ch_tx_frac", "clk_i2s3_8ch_tx_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(65), 0, RK3308_CLKGATE_CON(12), 5, GFLAGS, - &rk3308_i2s3_8ch_tx_fracmux), + &rk3308_i2s3_8ch_tx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S3_8CH_TX, "clk_i2s3_8ch_tx", mux_i2s3_8ch_tx_rx_p, CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(64), 12, 1, MFLAGS, RK3308_CLKGATE_CON(12), 6, GFLAGS), @@ -738,7 +745,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s3_8ch_rx_frac", "clk_i2s3_8ch_rx_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(67), 0, RK3308_CLKGATE_CON(12), 9, GFLAGS, - &rk3308_i2s3_8ch_rx_fracmux), + &rk3308_i2s3_8ch_rx_fracmux, RK3308_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S3_8CH_RX, "clk_i2s3_8ch_rx", mux_i2s3_8ch_rx_tx_p, CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(66), 12, 1, MFLAGS, RK3308_CLKGATE_CON(12), 10, GFLAGS), @@ -751,7 +758,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s0_2ch_frac", "clk_i2s0_2ch_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(69), 0, RK3308_CLKGATE_CON(12), 13, GFLAGS, - &rk3308_i2s0_2ch_fracmux), + &rk3308_i2s0_2ch_fracmux, RK3308_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S0_2CH, "clk_i2s0_2ch", "clk_i2s0_2ch_mux", 0, RK3308_CLKGATE_CON(12), 14, GFLAGS), COMPOSITE_NODIV(SCLK_I2S0_2CH_OUT, "clk_i2s0_2ch_out", mux_i2s0_2ch_out_p, CLK_SET_RATE_PARENT, @@ -764,7 +771,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s1_2ch_frac", "clk_i2s1_2ch_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(71), 0, RK3308_CLKGATE_CON(13), 1, GFLAGS, - &rk3308_i2s1_2ch_fracmux), + &rk3308_i2s1_2ch_fracmux, RK3308_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S1_2CH, "clk_i2s1_2ch", "clk_i2s1_2ch_mux", 0, RK3308_CLKGATE_CON(13), 2, GFLAGS), COMPOSITE_NODIV(SCLK_I2S1_2CH_OUT, "clk_i2s1_2ch_out", mux_i2s1_2ch_out_p, CLK_SET_RATE_PARENT, @@ -782,7 +789,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_spdif_tx_frac", "clk_spdif_tx_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(49), 0, RK3308_CLKGATE_CON(10), 7, GFLAGS, - &rk3308_spdif_tx_fracmux), + &rk3308_spdif_tx_fracmux, RK3308_SPDIF_FRAC_MAX_PRATE), GATE(SCLK_SPDIF_TX, "clk_spdif_tx", "clk_spdif_tx_mux", 0, RK3308_CLKGATE_CON(10), 8, GFLAGS), @@ -797,7 +804,7 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_spdif_rx_frac", "clk_spdif_rx_src", CLK_SET_RATE_PARENT, RK3308_CLKSEL_CON(51), 0, RK3308_CLKGATE_CON(10), 10, GFLAGS, - &rk3308_spdif_rx_fracmux), + &rk3308_spdif_rx_fracmux, RK3308_SPDIF_FRAC_MAX_PRATE), GATE(SCLK_SPDIF_RX, "clk_spdif_rx", "clk_spdif_rx_mux", 0, RK3308_CLKGATE_CON(10), 11, GFLAGS), @@ -900,22 +907,37 @@ static struct rockchip_clk_branch rk3308_clk_branches[] __initdata = { GATE(PCLK_OWIRE, "pclk_owire", "pclk_bus", CLK_IGNORE_UNUSED, RK3308_CLKGATE_CON(7), 15, GFLAGS), }; -static const char *const rk3308_critical_clocks[] __initconst = { - "aclk_bus", - "hclk_bus", - "pclk_bus", - "aclk_peri", - "hclk_peri", - "pclk_peri", - "hclk_audio", - "pclk_audio", - "sclk_ddrc", +static struct rockchip_clk_branch rk3308_dclk_vop_frac[] __initdata = { + COMPOSITE_FRACMUX(0, "dclk_vop_frac", "dclk_vop_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(9), 0, + RK3308_CLKGATE_CON(1), 7, GFLAGS, + &rk3308_dclk_vop_fracmux, RK3308_VOP_FRAC_MAX_PRATE), +}; + +static struct rockchip_clk_branch rk3308b_dclk_vop_frac[] __initdata = { + COMPOSITE_FRACMUX(0, "dclk_vop_frac", "dclk_vop_src", CLK_SET_RATE_PARENT, + RK3308_CLKSEL_CON(9), 0, + RK3308_CLKGATE_CON(1), 7, GFLAGS, + &rk3308_dclk_vop_fracmux, RK3308B_VOP_FRAC_MAX_PRATE), }; +static void __iomem *rk3308_cru_base; + +void rk3308_dump_cru(void) +{ + if (rk3308_cru_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk3308_cru_base, + 0x500, false); + } +} + static void __init rk3308_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; + struct clk **clks; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -929,17 +951,22 @@ static void __init rk3308_clk_init(struct device_node *np) iounmap(reg_base); return; } + clks = ctx->clk_data.clks; rockchip_clk_register_plls(ctx, rk3308_pll_clks, ARRAY_SIZE(rk3308_pll_clks), RK3308_GRF_SOC_STATUS0); rockchip_clk_register_branches(ctx, rk3308_clk_branches, ARRAY_SIZE(rk3308_clk_branches)); - rockchip_clk_protect_critical(rk3308_critical_clocks, - ARRAY_SIZE(rk3308_critical_clocks)); + if (soc_is_rk3308b()) + rockchip_clk_register_branches(ctx, rk3308b_dclk_vop_frac, + ARRAY_SIZE(rk3308b_dclk_vop_frac)); + else + rockchip_clk_register_branches(ctx, rk3308_dclk_vop_frac, + ARRAY_SIZE(rk3308_dclk_vop_frac)); rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", - mux_armclk_p, ARRAY_SIZE(mux_armclk_p), + 3, clks[PLL_APLL], clks[PLL_VPLL0], &rk3308_cpuclk_data, rk3308_cpuclk_rates, ARRAY_SIZE(rk3308_cpuclk_rates)); @@ -949,6 +976,39 @@ static void __init rk3308_clk_init(struct device_node *np) rockchip_register_restart_notifier(ctx, RK3308_GLB_SRST_FST, NULL); rockchip_clk_of_add_provider(np, ctx); + + if (!rk_dump_cru) { + rk3308_cru_base = reg_base; + rk_dump_cru = rk3308_dump_cru; + } } CLK_OF_DECLARE(rk3308_cru, "rockchip,rk3308-cru", rk3308_clk_init); + +static int __init clk_rk3308_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + + rk3308_clk_init(np); + + return 0; +} + +static const struct of_device_id clk_rk3308_match_table[] = { + { + .compatible = "rockchip,rk3308-cru", + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk3308_match_table); + +static struct platform_driver clk_rk3308_driver = { + .driver = { + .name = "clk-rk3308", + .of_match_table = clk_rk3308_match_table, + }, +}; +builtin_platform_driver_probe(clk_rk3308_driver, clk_rk3308_probe); + +MODULE_DESCRIPTION("Rockchip RK3308 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rk3328.c b/drivers/clk/rockchip/clk-rk3328.c index 2429b7c2a8b3..b8064dd74bd6 100644 --- a/drivers/clk/rockchip/clk-rk3328.c +++ b/drivers/clk/rockchip/clk-rk3328.c @@ -6,8 +6,10 @@ #include #include +#include #include #include +#include #include #include #include "clk.h" @@ -16,6 +18,9 @@ #define RK3328_GRF_SOC_STATUS0 0x480 #define RK3328_GRF_MAC_CON1 0x904 #define RK3328_GRF_MAC_CON2 0x908 +#define RK3328_I2S_FRAC_MAX_PRATE 600000000 +#define RK3328_UART_FRAC_MAX_PRATE 600000000 +#define RK3328_SPDIF_FRAC_MAX_PRATE 600000000 enum rk3328_plls { apll, dpll, cpll, gpll, npll, @@ -130,9 +135,10 @@ static struct rockchip_cpuclk_rate_table rk3328_cpuclk_rates[] __initdata = { }; static const struct rockchip_cpuclk_reg_data rk3328_cpuclk_data = { - .core_reg = RK3328_CLKSEL_CON(0), - .div_core_shift = 0, - .div_core_mask = 0x1f, + .core_reg[0] = RK3328_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 3, .mux_core_shift = 6, @@ -290,18 +296,18 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { RK3328_CLKGATE_CON(0), 1, GFLAGS), GATE(0, "npll_core", "npll", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(0), 12, GFLAGS), - COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IS_CRITICAL, RK3328_CLKSEL_CON(1), 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, RK3328_CLKGATE_CON(7), 0, GFLAGS), - COMPOSITE_NOMUX(0, "aclk_core", "armclk", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(0, "aclk_core", "armclk", CLK_IS_CRITICAL, RK3328_CLKSEL_CON(1), 4, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, RK3328_CLKGATE_CON(7), 1, GFLAGS), - GATE(0, "aclk_core_niu", "aclk_core", 0, + GATE(0, "aclk_core_niu", "aclk_core", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(13), 0, GFLAGS), - GATE(0, "aclk_gic400", "aclk_core", CLK_IGNORE_UNUSED, + GATE(0, "aclk_gic400", "aclk_core", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(13), 1, GFLAGS), - GATE(0, "clk_jtag", "jtag_clkin", CLK_IGNORE_UNUSED, + GATE(0, "clk_jtag", "jtag_clkin", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(7), 2, GFLAGS), /* PD_GPU */ @@ -310,34 +316,34 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { RK3328_CLKGATE_CON(6), 6, GFLAGS), GATE(ACLK_GPU, "aclk_gpu", "aclk_gpu_pre", CLK_SET_RATE_PARENT, RK3328_CLKGATE_CON(14), 0, GFLAGS), - GATE(0, "aclk_gpu_niu", "aclk_gpu_pre", 0, + GATE(0, "aclk_gpu_niu", "aclk_gpu_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(14), 1, GFLAGS), /* PD_DDR */ - COMPOSITE(0, "clk_ddr", mux_ddrphy_p, CLK_IGNORE_UNUSED, + COMPOSITE(0, "clk_ddr", mux_ddrphy_p, CLK_IS_CRITICAL, RK3328_CLKSEL_CON(3), 8, 2, MFLAGS, 0, 3, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, RK3328_CLKGATE_CON(0), 4, GFLAGS), - GATE(0, "clk_ddrmsch", "clk_ddr", CLK_IGNORE_UNUSED, + GATE(0, "clk_ddrmsch", "clk_ddr", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(18), 6, GFLAGS), - GATE(0, "clk_ddrupctl", "clk_ddr", CLK_IGNORE_UNUSED, + GATE(0, "clk_ddrupctl", "clk_ddr", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(18), 5, GFLAGS), GATE(0, "aclk_ddrupctl", "clk_ddr", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(18), 4, GFLAGS), GATE(0, "clk_ddrmon", "xin24m", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(0), 6, GFLAGS), - COMPOSITE(PCLK_DDR, "pclk_ddr", mux_2plls_hdmiphy_p, 0, + COMPOSITE(PCLK_DDR, "pclk_ddr", mux_2plls_hdmiphy_p, CLK_IS_CRITICAL, RK3328_CLKSEL_CON(4), 13, 2, MFLAGS, 8, 3, DFLAGS, RK3328_CLKGATE_CON(7), 4, GFLAGS), - GATE(0, "pclk_ddrupctl", "pclk_ddr", CLK_IGNORE_UNUSED, + GATE(0, "pclk_ddrupctl", "pclk_ddr", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(18), 1, GFLAGS), - GATE(0, "pclk_ddr_msch", "pclk_ddr", CLK_IGNORE_UNUSED, + GATE(0, "pclk_ddr_msch", "pclk_ddr", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(18), 2, GFLAGS), - GATE(0, "pclk_ddr_mon", "pclk_ddr", CLK_IGNORE_UNUSED, + GATE(0, "pclk_ddr_mon", "pclk_ddr", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(18), 3, GFLAGS), GATE(0, "pclk_ddrstdby", "pclk_ddr", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(18), 7, GFLAGS), - GATE(0, "pclk_ddr_grf", "pclk_ddr", CLK_IGNORE_UNUSED, + GATE(0, "pclk_ddr_grf", "pclk_ddr", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(18), 9, GFLAGS), /* @@ -345,18 +351,18 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { */ /* PD_BUS */ - COMPOSITE(ACLK_BUS_PRE, "aclk_bus_pre", mux_2plls_hdmiphy_p, 0, + COMPOSITE(ACLK_BUS_PRE, "aclk_bus_pre", mux_2plls_hdmiphy_p, CLK_IS_CRITICAL, RK3328_CLKSEL_CON(0), 13, 2, MFLAGS, 8, 5, DFLAGS, RK3328_CLKGATE_CON(8), 0, GFLAGS), - COMPOSITE_NOMUX(HCLK_BUS_PRE, "hclk_bus_pre", "aclk_bus_pre", 0, + COMPOSITE_NOMUX(HCLK_BUS_PRE, "hclk_bus_pre", "aclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKSEL_CON(1), 8, 2, DFLAGS, RK3328_CLKGATE_CON(8), 1, GFLAGS), - COMPOSITE_NOMUX(PCLK_BUS_PRE, "pclk_bus_pre", "aclk_bus_pre", 0, + COMPOSITE_NOMUX(PCLK_BUS_PRE, "pclk_bus_pre", "aclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKSEL_CON(1), 12, 3, DFLAGS, RK3328_CLKGATE_CON(8), 2, GFLAGS), - GATE(0, "pclk_bus", "pclk_bus_pre", 0, + GATE(0, "pclk_bus", "pclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(8), 3, GFLAGS), - GATE(0, "pclk_phy_pre", "pclk_bus_pre", 0, + GATE(0, "pclk_phy_pre", "pclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(8), 4, GFLAGS), COMPOSITE(SCLK_TSP, "clk_tsp", mux_2plls_p, 0, @@ -372,7 +378,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s0_frac", "clk_i2s0_div", CLK_SET_RATE_PARENT, RK3328_CLKSEL_CON(7), 0, RK3328_CLKGATE_CON(1), 2, GFLAGS, - &rk3328_i2s0_fracmux), + &rk3328_i2s0_fracmux, RK3328_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S0, "clk_i2s0", "i2s0_pre", CLK_SET_RATE_PARENT, RK3328_CLKGATE_CON(1), 3, GFLAGS), @@ -382,7 +388,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s1_frac", "clk_i2s1_div", CLK_SET_RATE_PARENT, RK3328_CLKSEL_CON(9), 0, RK3328_CLKGATE_CON(1), 5, GFLAGS, - &rk3328_i2s1_fracmux), + &rk3328_i2s1_fracmux, RK3328_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S1, "clk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT, RK3328_CLKGATE_CON(1), 6, GFLAGS), COMPOSITE_NODIV(SCLK_I2S1_OUT, "i2s1_out", mux_i2s1out_p, 0, @@ -395,7 +401,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_i2s2_frac", "clk_i2s2_div", CLK_SET_RATE_PARENT, RK3328_CLKSEL_CON(11), 0, RK3328_CLKGATE_CON(1), 9, GFLAGS, - &rk3328_i2s2_fracmux), + &rk3328_i2s2_fracmux, RK3328_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S2, "clk_i2s2", "i2s2_pre", CLK_SET_RATE_PARENT, RK3328_CLKGATE_CON(1), 10, GFLAGS), COMPOSITE_NODIV(SCLK_I2S2_OUT, "i2s2_out", mux_i2s2out_p, 0, @@ -408,7 +414,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_spdif_frac", "clk_spdif_div", CLK_SET_RATE_PARENT, RK3328_CLKSEL_CON(13), 0, RK3328_CLKGATE_CON(1), 13, GFLAGS, - &rk3328_spdif_fracmux), + &rk3328_spdif_fracmux, RK3328_SPDIF_FRAC_MAX_PRATE), /* PD_UART */ COMPOSITE(0, "clk_uart0_div", mux_2plls_u480m_p, 0, @@ -423,15 +429,15 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_div", CLK_SET_RATE_PARENT, RK3328_CLKSEL_CON(15), 0, RK3328_CLKGATE_CON(1), 15, GFLAGS, - &rk3328_uart0_fracmux), + &rk3328_uart0_fracmux, RK3328_UART_FRAC_MAX_PRATE), COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_div", CLK_SET_RATE_PARENT, RK3328_CLKSEL_CON(17), 0, RK3328_CLKGATE_CON(2), 1, GFLAGS, - &rk3328_uart1_fracmux), + &rk3328_uart1_fracmux, RK3328_UART_FRAC_MAX_PRATE), COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_div", CLK_SET_RATE_PARENT, RK3328_CLKSEL_CON(19), 0, RK3328_CLKGATE_CON(2), 3, GFLAGS, - &rk3328_uart2_fracmux), + &rk3328_uart2_fracmux, RK3328_UART_FRAC_MAX_PRATE), /* * Clock-Architecture Diagram 4 @@ -505,9 +511,9 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { RK3328_CLKGATE_CON(24), 0, GFLAGS), GATE(HCLK_RKVDEC, "hclk_rkvdec", "hclk_rkvdec_pre", CLK_SET_RATE_PARENT, RK3328_CLKGATE_CON(24), 1, GFLAGS), - GATE(0, "aclk_rkvdec_niu", "aclk_rkvdec_pre", 0, + GATE(0, "aclk_rkvdec_niu", "aclk_rkvdec_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(24), 2, GFLAGS), - GATE(0, "hclk_rkvdec_niu", "hclk_rkvdec_pre", 0, + GATE(0, "hclk_rkvdec_niu", "hclk_rkvdec_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(24), 3, GFLAGS), COMPOSITE(SCLK_VDEC_CABAC, "sclk_vdec_cabac", mux_4plls_p, 0, @@ -527,35 +533,36 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { RK3328_CLKGATE_CON(23), 0, GFLAGS), GATE(HCLK_VPU, "hclk_vpu", "hclk_vpu_pre", CLK_SET_RATE_PARENT, RK3328_CLKGATE_CON(23), 1, GFLAGS), - GATE(0, "aclk_vpu_niu", "aclk_vpu_pre", 0, + GATE(0, "aclk_vpu_niu", "aclk_vpu_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(23), 2, GFLAGS), - GATE(0, "hclk_vpu_niu", "hclk_vpu_pre", 0, + GATE(0, "hclk_vpu_niu", "hclk_vpu_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(23), 3, GFLAGS), COMPOSITE(ACLK_RKVENC, "aclk_rkvenc", mux_4plls_p, 0, RK3328_CLKSEL_CON(51), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3328_CLKGATE_CON(6), 3, GFLAGS), - FACTOR_GATE(HCLK_RKVENC, "hclk_rkvenc", "aclk_rkvenc", 0, 1, 4, + + COMPOSITE(SCLK_VENC_CORE, "sclk_venc_core", mux_4plls_p, 0, + RK3328_CLKSEL_CON(51), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK3328_CLKGATE_CON(6), 4, GFLAGS), + FACTOR_GATE(0, "hclk_venc", "sclk_venc_core", 0, 1, 4, RK3328_CLKGATE_CON(11), 4, GFLAGS), - GATE(0, "aclk_rkvenc_niu", "aclk_rkvenc", 0, + + GATE(0, "aclk_rkvenc_niu", "sclk_venc_core", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(25), 0, GFLAGS), - GATE(0, "hclk_rkvenc_niu", "hclk_rkvenc", 0, + GATE(0, "hclk_rkvenc_niu", "hclk_venc", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(25), 1, GFLAGS), - GATE(ACLK_H265, "aclk_h265", "aclk_rkvenc", 0, + GATE(ACLK_H265, "aclk_h265", "sclk_venc_core", 0, RK3328_CLKGATE_CON(25), 2, GFLAGS), - GATE(PCLK_H265, "pclk_h265", "hclk_rkvenc", 0, + GATE(PCLK_H265, "pclk_h265", "hclk_venc", 0, RK3328_CLKGATE_CON(25), 3, GFLAGS), - GATE(ACLK_H264, "aclk_h264", "aclk_rkvenc", 0, + GATE(ACLK_H264, "aclk_h264", "sclk_venc_core", 0, RK3328_CLKGATE_CON(25), 4, GFLAGS), - GATE(HCLK_H264, "hclk_h264", "hclk_rkvenc", 0, + GATE(HCLK_H264, "hclk_h264", "hclk_venc", 0, RK3328_CLKGATE_CON(25), 5, GFLAGS), - GATE(ACLK_AXISRAM, "aclk_axisram", "aclk_rkvenc", CLK_IGNORE_UNUSED, + GATE(ACLK_AXISRAM, "aclk_axisram", "sclk_venc_core", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(25), 6, GFLAGS), - COMPOSITE(SCLK_VENC_CORE, "sclk_venc_core", mux_4plls_p, 0, - RK3328_CLKSEL_CON(51), 14, 2, MFLAGS, 8, 5, DFLAGS, - RK3328_CLKGATE_CON(6), 4, GFLAGS), - COMPOSITE(SCLK_VENC_DSP, "sclk_venc_dsp", mux_4plls_p, 0, RK3328_CLKSEL_CON(52), 14, 2, MFLAGS, 8, 5, DFLAGS, RK3328_CLKGATE_CON(6), 7, GFLAGS), @@ -602,21 +609,21 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { */ /* PD_PERI */ - GATE(0, "gpll_peri", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_peri", "gpll", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(4), 0, GFLAGS), - GATE(0, "cpll_peri", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_peri", "cpll", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(4), 1, GFLAGS), - GATE(0, "hdmiphy_peri", "hdmiphy", CLK_IGNORE_UNUSED, + GATE(0, "hdmiphy_peri", "hdmiphy", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(4), 2, GFLAGS), - COMPOSITE_NOGATE(ACLK_PERI_PRE, "aclk_peri_pre", mux_aclk_peri_pre_p, 0, + COMPOSITE_NOGATE(ACLK_PERI_PRE, "aclk_peri_pre", mux_aclk_peri_pre_p, CLK_IS_CRITICAL, RK3328_CLKSEL_CON(28), 6, 2, MFLAGS, 0, 5, DFLAGS), - COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_pre", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL, RK3328_CLKSEL_CON(29), 0, 2, DFLAGS, RK3328_CLKGATE_CON(10), 2, GFLAGS), - COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_pre", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL, RK3328_CLKSEL_CON(29), 4, 3, DFLAGS, RK3328_CLKGATE_CON(10), 1, GFLAGS), - GATE(ACLK_PERI, "aclk_peri", "aclk_peri_pre", CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, + GATE(ACLK_PERI, "aclk_peri", "aclk_peri_pre", CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, RK3328_CLKGATE_CON(10), 0, GFLAGS), COMPOSITE(SCLK_SDMMC, "clk_sdmmc", mux_2plls_24m_u480m_p, 0, @@ -701,30 +708,30 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { /* PD_VOP */ GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, RK3328_CLKGATE_CON(21), 10, GFLAGS), - GATE(0, "aclk_rga_niu", "aclk_rga_pre", 0, RK3328_CLKGATE_CON(22), 3, GFLAGS), + GATE(0, "aclk_rga_niu", "aclk_rga_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(22), 3, GFLAGS), GATE(ACLK_VOP, "aclk_vop", "aclk_vop_pre", 0, RK3328_CLKGATE_CON(21), 2, GFLAGS), - GATE(0, "aclk_vop_niu", "aclk_vop_pre", 0, RK3328_CLKGATE_CON(21), 4, GFLAGS), + GATE(0, "aclk_vop_niu", "aclk_vop_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(21), 4, GFLAGS), GATE(ACLK_IEP, "aclk_iep", "aclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 6, GFLAGS), GATE(ACLK_CIF, "aclk_cif", "aclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 8, GFLAGS), GATE(ACLK_HDCP, "aclk_hdcp", "aclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 15, GFLAGS), - GATE(0, "aclk_vio_niu", "aclk_vio_pre", 0, RK3328_CLKGATE_CON(22), 2, GFLAGS), + GATE(0, "aclk_vio_niu", "aclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(22), 2, GFLAGS), GATE(HCLK_VOP, "hclk_vop", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 3, GFLAGS), - GATE(0, "hclk_vop_niu", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 5, GFLAGS), + GATE(0, "hclk_vop_niu", "hclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(21), 5, GFLAGS), GATE(HCLK_IEP, "hclk_iep", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 7, GFLAGS), GATE(HCLK_CIF, "hclk_cif", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 9, GFLAGS), GATE(HCLK_RGA, "hclk_rga", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 11, GFLAGS), - GATE(0, "hclk_ahb1tom", "hclk_vio_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(21), 12, GFLAGS), - GATE(0, "pclk_vio_h2p", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 13, GFLAGS), - GATE(0, "hclk_vio_h2p", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(21), 14, GFLAGS), + GATE(0, "hclk_ahb1tom", "hclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(21), 12, GFLAGS), + GATE(0, "pclk_vio_h2p", "hclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(21), 13, GFLAGS), + GATE(0, "hclk_vio_h2p", "hclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(21), 14, GFLAGS), GATE(HCLK_HDCP, "hclk_hdcp", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(22), 0, GFLAGS), - GATE(0, "hclk_vio_niu", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(22), 1, GFLAGS), + GATE(0, "hclk_vio_niu", "hclk_vio_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(22), 1, GFLAGS), GATE(PCLK_HDMI, "pclk_hdmi", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(22), 4, GFLAGS), GATE(PCLK_HDCP, "pclk_hdcp", "hclk_vio_pre", 0, RK3328_CLKGATE_CON(22), 5, GFLAGS), /* PD_PERI */ - GATE(0, "aclk_peri_noc", "aclk_peri", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(19), 11, GFLAGS), + GATE(0, "aclk_peri_noc", "aclk_peri", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(19), 11, GFLAGS), GATE(ACLK_USB3OTG, "aclk_usb3otg", "aclk_peri", 0, RK3328_CLKGATE_CON(19), 14, GFLAGS), GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 0, GFLAGS), @@ -734,26 +741,26 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { GATE(HCLK_HOST0, "hclk_host0", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 6, GFLAGS), GATE(HCLK_HOST0_ARB, "hclk_host0_arb", "hclk_peri", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(19), 7, GFLAGS), GATE(HCLK_OTG, "hclk_otg", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 8, GFLAGS), - GATE(HCLK_OTG_PMU, "hclk_otg_pmu", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 9, GFLAGS), - GATE(0, "hclk_peri_niu", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 12, GFLAGS), - GATE(0, "pclk_peri_niu", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 13, GFLAGS), + GATE(HCLK_OTG_PMU, "hclk_otg_pmu", "hclk_peri", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(19), 9, GFLAGS), + GATE(0, "hclk_peri_niu", "hclk_peri", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(19), 12, GFLAGS), + GATE(0, "pclk_peri_niu", "hclk_peri", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(19), 13, GFLAGS), /* PD_GMAC */ GATE(ACLK_MAC2PHY, "aclk_mac2phy", "aclk_gmac", 0, RK3328_CLKGATE_CON(26), 0, GFLAGS), GATE(ACLK_MAC2IO, "aclk_mac2io", "aclk_gmac", 0, RK3328_CLKGATE_CON(26), 2, GFLAGS), - GATE(0, "aclk_gmac_niu", "aclk_gmac", 0, RK3328_CLKGATE_CON(26), 4, GFLAGS), + GATE(0, "aclk_gmac_niu", "aclk_gmac", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(26), 4, GFLAGS), GATE(PCLK_MAC2PHY, "pclk_mac2phy", "pclk_gmac", 0, RK3328_CLKGATE_CON(26), 1, GFLAGS), GATE(PCLK_MAC2IO, "pclk_mac2io", "pclk_gmac", 0, RK3328_CLKGATE_CON(26), 3, GFLAGS), - GATE(0, "pclk_gmac_niu", "pclk_gmac", 0, RK3328_CLKGATE_CON(26), 5, GFLAGS), + GATE(0, "pclk_gmac_niu", "pclk_gmac", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(26), 5, GFLAGS), /* PD_BUS */ - GATE(0, "aclk_bus_niu", "aclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 12, GFLAGS), + GATE(0, "aclk_bus_niu", "aclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 12, GFLAGS), GATE(ACLK_DCF, "aclk_dcf", "aclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 11, GFLAGS), GATE(ACLK_TSP, "aclk_tsp", "aclk_bus_pre", 0, RK3328_CLKGATE_CON(17), 12, GFLAGS), - GATE(0, "aclk_intmem", "aclk_bus_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(15), 0, GFLAGS), + GATE(0, "aclk_intmem", "aclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 0, GFLAGS), GATE(ACLK_DMAC, "aclk_dmac_bus", "aclk_bus_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(15), 1, GFLAGS), - GATE(0, "hclk_rom", "hclk_bus_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(15), 2, GFLAGS), + GATE(0, "hclk_rom", "hclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 2, GFLAGS), GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 3, GFLAGS), GATE(HCLK_I2S1_8CH, "hclk_i2s1_8ch", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 4, GFLAGS), GATE(HCLK_I2S2_2CH, "hclk_i2s2_2ch", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 5, GFLAGS), @@ -761,17 +768,17 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { GATE(HCLK_TSP, "hclk_tsp", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(17), 11, GFLAGS), GATE(HCLK_CRYPTO_MST, "hclk_crypto_mst", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 7, GFLAGS), GATE(HCLK_CRYPTO_SLV, "hclk_crypto_slv", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 8, GFLAGS), - GATE(0, "hclk_bus_niu", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(15), 13, GFLAGS), + GATE(0, "hclk_bus_niu", "hclk_bus_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 13, GFLAGS), GATE(HCLK_PDM, "hclk_pdm", "hclk_bus_pre", 0, RK3328_CLKGATE_CON(28), 0, GFLAGS), - GATE(0, "pclk_bus_niu", "pclk_bus", 0, RK3328_CLKGATE_CON(15), 14, GFLAGS), + GATE(0, "pclk_bus_niu", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 14, GFLAGS), GATE(0, "pclk_efuse", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(15), 9, GFLAGS), GATE(0, "pclk_otp", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(28), 4, GFLAGS), GATE(PCLK_I2C0, "pclk_i2c0", "pclk_bus", 0, RK3328_CLKGATE_CON(15), 10, GFLAGS), GATE(PCLK_I2C1, "pclk_i2c1", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 0, GFLAGS), GATE(PCLK_I2C2, "pclk_i2c2", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 1, GFLAGS), GATE(PCLK_I2C3, "pclk_i2c3", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 2, GFLAGS), - GATE(PCLK_TIMER, "pclk_timer0", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 3, GFLAGS), + GATE(PCLK_TIMER, "pclk_timer0", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(16), 3, GFLAGS), GATE(0, "pclk_stimer", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 4, GFLAGS), GATE(PCLK_SPI, "pclk_spi", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 5, GFLAGS), GATE(PCLK_PWM, "pclk_rk_pwm", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 6, GFLAGS), @@ -784,12 +791,12 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { GATE(PCLK_UART2, "pclk_uart2", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 13, GFLAGS), GATE(PCLK_TSADC, "pclk_tsadc", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 14, GFLAGS), GATE(PCLK_DCF, "pclk_dcf", "pclk_bus", 0, RK3328_CLKGATE_CON(16), 15, GFLAGS), - GATE(PCLK_GRF, "pclk_grf", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 0, GFLAGS), - GATE(0, "pclk_cru", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 4, GFLAGS), - GATE(0, "pclk_sgrf", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 6, GFLAGS), + GATE(PCLK_GRF, "pclk_grf", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(17), 0, GFLAGS), + GATE(0, "pclk_cru", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(17), 4, GFLAGS), + GATE(0, "pclk_sgrf", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(17), 6, GFLAGS), GATE(0, "pclk_sim", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 10, GFLAGS), GATE(PCLK_SARADC, "pclk_saradc", "pclk_bus", 0, RK3328_CLKGATE_CON(17), 15, GFLAGS), - GATE(0, "pclk_pmu", "pclk_bus", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(28), 3, GFLAGS), + GATE(0, "pclk_pmu", "pclk_bus", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(28), 3, GFLAGS), /* Watchdog pclk is controlled from the secure GRF */ SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_bus"), @@ -798,11 +805,11 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { GATE(PCLK_USB3PHY_PIPE, "pclk_usb3phy_pipe", "pclk_phy_pre", 0, RK3328_CLKGATE_CON(28), 2, GFLAGS), GATE(PCLK_USB3_GRF, "pclk_usb3_grf", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 2, GFLAGS), GATE(PCLK_USB2_GRF, "pclk_usb2_grf", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 14, GFLAGS), - GATE(0, "pclk_ddrphy", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 13, GFLAGS), + GATE(0, "pclk_ddrphy", "pclk_phy_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(17), 13, GFLAGS), GATE(PCLK_ACODECPHY, "pclk_acodecphy", "pclk_phy_pre", 0, RK3328_CLKGATE_CON(17), 5, GFLAGS), GATE(PCLK_HDMIPHY, "pclk_hdmiphy", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 7, GFLAGS), GATE(0, "pclk_vdacphy", "pclk_phy_pre", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(17), 8, GFLAGS), - GATE(0, "pclk_phy_niu", "pclk_phy_pre", 0, RK3328_CLKGATE_CON(15), 15, GFLAGS), + GATE(0, "pclk_phy_niu", "pclk_phy_pre", CLK_IS_CRITICAL, RK3328_CLKGATE_CON(15), 15, GFLAGS), /* PD_MMC */ MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "clk_sdmmc", @@ -826,61 +833,11 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = { RK3328_SDMMC_EXT_CON1, 1), }; -static const char *const rk3328_critical_clocks[] __initconst = { - "aclk_bus", - "aclk_bus_niu", - "pclk_bus", - "pclk_bus_niu", - "hclk_bus", - "hclk_bus_niu", - "aclk_peri", - "hclk_peri", - "hclk_peri_niu", - "pclk_peri", - "pclk_peri_niu", - "pclk_dbg", - "aclk_core_niu", - "aclk_gic400", - "aclk_intmem", - "hclk_rom", - "pclk_grf", - "pclk_cru", - "pclk_sgrf", - "pclk_timer0", - "clk_timer0", - "pclk_ddr_msch", - "pclk_ddr_mon", - "pclk_ddr_grf", - "clk_ddrupctl", - "clk_ddrmsch", - "hclk_ahb1tom", - "clk_jtag", - "pclk_ddrphy", - "pclk_pmu", - "hclk_otg_pmu", - "aclk_rga_niu", - "pclk_vio_h2p", - "hclk_vio_h2p", - "aclk_vio_niu", - "hclk_vio_niu", - "aclk_vop_niu", - "hclk_vop_niu", - "aclk_gpu_niu", - "aclk_rkvdec_niu", - "hclk_rkvdec_niu", - "aclk_vpu_niu", - "hclk_vpu_niu", - "aclk_rkvenc_niu", - "hclk_rkvenc_niu", - "aclk_gmac_niu", - "pclk_gmac_niu", - "pclk_phy_niu", -}; - static void __init rk3328_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; + struct clk **clks; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -894,17 +851,16 @@ static void __init rk3328_clk_init(struct device_node *np) iounmap(reg_base); return; } + clks = ctx->clk_data.clks; rockchip_clk_register_plls(ctx, rk3328_pll_clks, ARRAY_SIZE(rk3328_pll_clks), RK3328_GRF_SOC_STATUS0); rockchip_clk_register_branches(ctx, rk3328_clk_branches, ARRAY_SIZE(rk3328_clk_branches)); - rockchip_clk_protect_critical(rk3328_critical_clocks, - ARRAY_SIZE(rk3328_critical_clocks)); rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", - mux_armclk_p, ARRAY_SIZE(mux_armclk_p), + 4, clks[PLL_APLL], clks[PLL_GPLL], &rk3328_cpuclk_data, rk3328_cpuclk_rates, ARRAY_SIZE(rk3328_cpuclk_rates)); @@ -916,3 +872,31 @@ static void __init rk3328_clk_init(struct device_node *np) rockchip_clk_of_add_provider(np, ctx); } CLK_OF_DECLARE(rk3328_cru, "rockchip,rk3328-cru", rk3328_clk_init); + +static int __init clk_rk3328_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + + rk3328_clk_init(np); + + return 0; +} + +static const struct of_device_id clk_rk3328_match_table[] = { + { + .compatible = "rockchip,rk3328-cru", + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk3328_match_table); + +static struct platform_driver clk_rk3328_driver = { + .driver = { + .name = "clk-rk3328", + .of_match_table = clk_rk3328_match_table, + }, +}; +builtin_platform_driver_probe(clk_rk3328_driver, clk_rk3328_probe); + +MODULE_DESCRIPTION("Rockchip RK3328 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rk3368.c b/drivers/clk/rockchip/clk-rk3368.c index 55443349439b..4b4513a26740 100644 --- a/drivers/clk/rockchip/clk-rk3368.c +++ b/drivers/clk/rockchip/clk-rk3368.c @@ -5,13 +5,19 @@ #include #include +#include #include #include +#include #include #include #include "clk.h" #define RK3368_GRF_SOC_STATUS0 0x480 +#define RK3368_I2S_FRAC_MAX_PRATE 600000000 +#define RK3368_UART_FRAC_MAX_PRATE 600000000 +#define RK3368_SPDIF_FRAC_MAX_PRATE 600000000 +#define RK3368_DCLK_PARENT_MAX_PRATE 600000000 enum rk3368_plls { apllb, aplll, dpll, cpll, gpll, npll, @@ -87,23 +93,40 @@ static struct rockchip_pll_rate_table rk3368_pll_rates[] = { { /* sentinel */ }, }; +static struct rockchip_pll_rate_table rk3368_npll_rates[] = { + RK3066_PLL_RATE_NB(594000000, 1, 99, 4, 32), + RK3066_PLL_RATE_NB(585000000, 6, 585, 4, 32), + RK3066_PLL_RATE_NB(432000000, 3, 216, 4, 32), + RK3066_PLL_RATE_NB(426000000, 3, 213, 4, 32), + RK3066_PLL_RATE_NB(400000000, 1, 100, 6, 32), + RK3066_PLL_RATE_NB(342000000, 3, 171, 4, 32), + RK3066_PLL_RATE_NB(297000000, 2, 198, 8, 16), + RK3066_PLL_RATE_NB(270000000, 1, 135, 12, 32), + RK3066_PLL_RATE_NB(260000000, 1, 130, 12, 32), + RK3066_PLL_RATE_NB(148500000, 1, 99, 16, 32), + RK3066_PLL_RATE_NB(146250000, 6, 585, 16, 32), + RK3066_PLL_RATE_NB(108000000, 1, 54, 12, 32), + RK3066_PLL_RATE_NB(106500000, 4, 213, 12, 32), + RK3066_PLL_RATE_NB(85500000, 4, 171, 12, 32), + RK3066_PLL_RATE_NB(74250000, 4, 198, 16, 32), +}; + PNAME(mux_pll_p) = { "xin24m", "xin32k" }; -PNAME(mux_armclkb_p) = { "apllb_core", "gpllb_core" }; -PNAME(mux_armclkl_p) = { "aplll_core", "gplll_core" }; PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr" }; PNAME(mux_cs_src_p) = { "apllb_cs", "aplll_cs", "gpll_cs"}; PNAME(mux_aclk_bus_src_p) = { "cpll_aclk_bus", "gpll_aclk_bus" }; PNAME(mux_pll_src_cpll_gpll_p) = { "cpll", "gpll" }; -PNAME(mux_pll_src_cpll_gpll_npll_p) = { "cpll", "gpll", "npll" }; -PNAME(mux_pll_src_npll_cpll_gpll_p) = { "npll", "cpll", "gpll" }; +PNAME(mux_pll_src_cpll_gpll_npll_p) = { "cpll", "gpll", "dummy_npll" }; +PNAME(mux_pll_src_dmycpll_dmygpll_npll_p) = { "dummy_cpll", "dummy_gpll", "npll" }; +PNAME(mux_pll_src_npll_cpll_gpll_p) = { "dummy_npll", "cpll", "gpll" }; PNAME(mux_pll_src_cpll_gpll_usb_p) = { "cpll", "gpll", "usbphy_480m" }; PNAME(mux_pll_src_cpll_gpll_usb_usb_p) = { "cpll", "gpll", "usbphy_480m", "usbphy_480m" }; PNAME(mux_pll_src_cpll_gpll_usb_npll_p) = { "cpll", "gpll", "usbphy_480m", - "npll" }; -PNAME(mux_pll_src_cpll_gpll_npll_npll_p) = { "cpll", "gpll", "npll", "npll" }; -PNAME(mux_pll_src_cpll_gpll_npll_usb_p) = { "cpll", "gpll", "npll", + "dummy_npll" }; +PNAME(mux_pll_src_cpll_gpll_npll_npll_p) = { "cpll", "gpll", "dummy_npll", "dummy_npll" }; +PNAME(mux_pll_src_cpll_gpll_npll_usb_p) = { "cpll", "gpll", "dummy_npll", "usbphy_480m" }; PNAME(mux_i2s_8ch_pre_p) = { "i2s_8ch_src", "i2s_8ch_frac", @@ -138,7 +161,7 @@ static struct rockchip_pll_clock rk3368_pll_clks[] __initdata = { [gpll] = PLL(pll_rk3066, PLL_GPLL, "gpll", mux_pll_p, 0, RK3368_PLL_CON(16), RK3368_PLL_CON(19), 8, 4, ROCKCHIP_PLL_SYNC_RATE, rk3368_pll_rates), [npll] = PLL(pll_rk3066, PLL_NPLL, "npll", mux_pll_p, 0, RK3368_PLL_CON(20), - RK3368_PLL_CON(23), 8, 5, ROCKCHIP_PLL_SYNC_RATE, rk3368_pll_rates), + RK3368_PLL_CON(23), 8, 5, 0, rk3368_npll_rates), }; static struct clk_div_table div_ddrphy_t[] = { @@ -154,9 +177,10 @@ static struct clk_div_table div_ddrphy_t[] = { #define IFLAGS ROCKCHIP_INVERTER_HIWORD_MASK static const struct rockchip_cpuclk_reg_data rk3368_cpuclkb_data = { - .core_reg = RK3368_CLKSEL_CON(0), - .div_core_shift = 0, - .div_core_mask = 0x1f, + .core_reg[0] = RK3368_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 0, .mux_core_shift = 7, @@ -164,11 +188,12 @@ static const struct rockchip_cpuclk_reg_data rk3368_cpuclkb_data = { }; static const struct rockchip_cpuclk_reg_data rk3368_cpuclkl_data = { - .core_reg = RK3368_CLKSEL_CON(2), - .div_core_shift = 0, + .core_reg[0] = RK3368_CLKSEL_CON(2), + .div_core_shift[0] = 0, .mux_core_alt = 1, + .num_cores = 1, .mux_core_main = 0, - .div_core_mask = 0x1f, + .div_core_mask[0] = 0x1f, .mux_core_shift = 7, .mux_core_mask = 0x1, }; @@ -315,8 +340,8 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { RK3368_CLKSEL_CON(4), 8, 5, DFLAGS, RK3368_CLKGATE_CON(0), 13, GFLAGS), - COMPOSITE(0, "aclk_cci_pre", mux_pll_src_cpll_gpll_usb_npll_p, CLK_IGNORE_UNUSED, - RK3368_CLKSEL_CON(5), 6, 2, MFLAGS, 0, 7, DFLAGS, + COMPOSITE(ACLK_CCI_PRE, "aclk_cci_pre", mux_pll_src_cpll_gpll_usb_npll_p, CLK_IGNORE_UNUSED, + RK3368_CLKSEL_CON(5), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3368_CLKGATE_CON(0), 12, GFLAGS), GATE(SCLK_PVTM_CORE, "sclk_pvtm_core", "xin24m", 0, RK3368_CLKGATE_CON(7), 10, GFLAGS), @@ -332,19 +357,19 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { GATE(0, "sclk_ddr4x", "ddrphy_src", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(6), 15, GFLAGS), - GATE(0, "gpll_aclk_bus", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_aclk_bus", "gpll", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(1), 10, GFLAGS), - GATE(0, "cpll_aclk_bus", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_aclk_bus", "cpll", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(1), 11, GFLAGS), - COMPOSITE_NOGATE(0, "aclk_bus_src", mux_aclk_bus_src_p, CLK_IGNORE_UNUSED, + COMPOSITE_NOGATE(0, "aclk_bus_src", mux_aclk_bus_src_p, CLK_IS_CRITICAL, RK3368_CLKSEL_CON(8), 7, 1, MFLAGS, 0, 5, DFLAGS), - GATE(ACLK_BUS, "aclk_bus", "aclk_bus_src", CLK_IGNORE_UNUSED, + GATE(ACLK_BUS, "aclk_bus", "aclk_bus_src", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(1), 0, GFLAGS), - COMPOSITE_NOMUX(PCLK_BUS, "pclk_bus", "aclk_bus_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(PCLK_BUS, "pclk_bus", "aclk_bus_src", CLK_IS_CRITICAL, RK3368_CLKSEL_CON(8), 12, 3, DFLAGS, RK3368_CLKGATE_CON(1), 2, GFLAGS), - COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus", "aclk_bus_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus", "aclk_bus_src", CLK_IS_CRITICAL, RK3368_CLKSEL_CON(8), 8, 2, DFLAGS, RK3368_CLKGATE_CON(1), 1, GFLAGS), COMPOSITE_NOMUX(0, "sclk_crypto", "aclk_bus_src", 0, @@ -358,7 +383,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { * stclk_mcu is listed as child of fclk_mcu_src in diagram 5, * but stclk_mcu has an additional own divider in diagram 2 */ - COMPOSITE_NOMUX(0, "stclk_mcu", "fclk_mcu_src", 0, + COMPOSITE_NOMUX(0, "stclk_mcu", "fclk_mcu_src", CLK_IGNORE_UNUSED, RK3368_CLKSEL_CON(12), 8, 3, DFLAGS, RK3368_CLKGATE_CON(13), 13, GFLAGS), @@ -368,7 +393,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s_8ch_frac", "i2s_8ch_src", CLK_SET_RATE_PARENT, RK3368_CLKSEL_CON(28), 0, RK3368_CLKGATE_CON(6), 2, GFLAGS, - &rk3368_i2s_8ch_fracmux), + &rk3368_i2s_8ch_fracmux, RK3368_I2S_FRAC_MAX_PRATE), COMPOSITE_NODIV(SCLK_I2S_8CH_OUT, "i2s_8ch_clkout", mux_i2s_8ch_clkout_p, 0, RK3368_CLKSEL_CON(27), 15, 1, MFLAGS, RK3368_CLKGATE_CON(6), 0, GFLAGS), @@ -380,7 +405,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "spdif_8ch_frac", "spdif_8ch_src", CLK_SET_RATE_PARENT, RK3368_CLKSEL_CON(32), 0, RK3368_CLKGATE_CON(6), 5, GFLAGS, - &rk3368_spdif_8ch_fracmux), + &rk3368_spdif_8ch_fracmux, RK3368_SPDIF_FRAC_MAX_PRATE), GATE(SCLK_SPDIF_8CH, "sclk_spdif_8ch", "spdif_8ch_pre", CLK_SET_RATE_PARENT, RK3368_CLKGATE_CON(6), 6, GFLAGS), COMPOSITE(0, "i2s_2ch_src", mux_pll_src_cpll_gpll_p, 0, @@ -389,7 +414,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s_2ch_frac", "i2s_2ch_src", CLK_SET_RATE_PARENT, RK3368_CLKSEL_CON(54), 0, RK3368_CLKGATE_CON(5), 14, GFLAGS, - &rk3368_i2s_2ch_fracmux), + &rk3368_i2s_2ch_fracmux, RK3368_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S_2CH, "sclk_i2s_2ch", "i2s_2ch_pre", CLK_SET_RATE_PARENT, RK3368_CLKGATE_CON(5), 15, GFLAGS), @@ -445,9 +470,9 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { RK3368_CLKSEL_CON(18), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3368_CLKGATE_CON(4), 4, GFLAGS), - COMPOSITE(DCLK_VOP, "dclk_vop", mux_pll_src_cpll_gpll_npll_p, 0, + COMPOSITE_DCLK(DCLK_VOP, "dclk_vop", mux_pll_src_dmycpll_dmygpll_npll_p, CLK_SET_RATE_PARENT, RK3368_CLKSEL_CON(20), 8, 2, MFLAGS, 0, 8, DFLAGS, - RK3368_CLKGATE_CON(4), 1, GFLAGS), + RK3368_CLKGATE_CON(4), 1, GFLAGS, RK3368_DCLK_PARENT_MAX_PRATE), GATE(SCLK_VOP0_PWM, "sclk_vop0_pwm", "xin24m", 0, RK3368_CLKGATE_CON(4), 2, GFLAGS), @@ -488,12 +513,12 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { RK3368_CLKSEL_CON(55), 6, 2, MFLAGS, 0, 6, DFLAGS, RK3368_CLKGATE_CON(5), 5, GFLAGS), - DIV(0, "pclk_pd_alive", "gpll", 0, + DIV(0, "pclk_pd_alive", "gpll", CLK_IS_CRITICAL, RK3368_CLKSEL_CON(10), 8, 5, DFLAGS), /* sclk_timer has a gate in the sgrf */ - COMPOSITE_NOMUX(0, "pclk_pd_pmu", "gpll", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(0, "pclk_pd_pmu", "gpll", CLK_IS_CRITICAL, RK3368_CLKSEL_CON(10), 0, 5, DFLAGS, RK3368_CLKGATE_CON(7), 9, GFLAGS), GATE(SCLK_PVTM_PMU, "sclk_pvtm_pmu", "xin24m", 0, @@ -512,16 +537,16 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { GATE(SCLK_PVTM_GPU, "sclk_pvtm_gpu", "xin24m", 0, RK3368_CLKGATE_CON(7), 11, GFLAGS), - COMPOSITE(0, "aclk_peri_src", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED, + COMPOSITE(0, "aclk_peri_src", mux_pll_src_cpll_gpll_p, CLK_IS_CRITICAL, RK3368_CLKSEL_CON(9), 7, 1, MFLAGS, 0, 5, DFLAGS, RK3368_CLKGATE_CON(3), 0, GFLAGS), - COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", 0, + COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK3368_CLKSEL_CON(9), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, RK3368_CLKGATE_CON(3), 3, GFLAGS), - COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK3368_CLKSEL_CON(9), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, RK3368_CLKGATE_CON(3), 2, GFLAGS), - GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IGNORE_UNUSED, + GATE(ACLK_PERI, "aclk_peri", "aclk_peri_src", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(3), 1, GFLAGS), GATE(0, "sclk_mipidsi_24m", "xin24m", 0, RK3368_CLKGATE_CON(4), 14, GFLAGS), @@ -590,7 +615,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, RK3368_CLKSEL_CON(34), 0, RK3368_CLKGATE_CON(2), 1, GFLAGS, - &rk3368_uart0_fracmux), + &rk3368_uart0_fracmux, RK3368_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "uart1_src", "uart_src", 0, RK3368_CLKSEL_CON(35), 0, 7, DFLAGS, @@ -598,7 +623,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, RK3368_CLKSEL_CON(36), 0, RK3368_CLKGATE_CON(2), 3, GFLAGS, - &rk3368_uart1_fracmux), + &rk3368_uart1_fracmux, RK3368_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "uart3_src", "uart_src", 0, RK3368_CLKSEL_CON(39), 0, 7, DFLAGS, @@ -606,7 +631,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart3_frac", "uart3_src", CLK_SET_RATE_PARENT, RK3368_CLKSEL_CON(40), 0, RK3368_CLKGATE_CON(2), 7, GFLAGS, - &rk3368_uart3_fracmux), + &rk3368_uart3_fracmux, RK3368_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "uart4_src", "uart_src", 0, RK3368_CLKSEL_CON(41), 0, 7, DFLAGS, @@ -614,7 +639,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart4_frac", "uart4_src", CLK_SET_RATE_PARENT, RK3368_CLKSEL_CON(42), 0, RK3368_CLKGATE_CON(2), 9, GFLAGS, - &rk3368_uart4_fracmux), + &rk3368_uart4_fracmux, RK3368_UART_FRAC_MAX_PRATE), COMPOSITE(0, "mac_pll_src", mux_pll_src_npll_cpll_gpll_p, 0, RK3368_CLKSEL_CON(43), 6, 2, MFLAGS, 0, 5, DFLAGS, @@ -668,7 +693,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { /* aclk_bus gates */ GATE(0, "aclk_strc_sys", "aclk_bus", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(12), 12, GFLAGS), - GATE(ACLK_DMAC_BUS, "aclk_dmac_bus", "aclk_bus", 0, RK3368_CLKGATE_CON(12), 11, GFLAGS), + GATE(ACLK_DMAC_BUS, "aclk_dmac_bus", "aclk_bus", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(12), 11, GFLAGS), GATE(0, "sclk_intmem1", "aclk_bus", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(12), 6, GFLAGS), GATE(0, "sclk_intmem0", "aclk_bus", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(12), 5, GFLAGS), GATE(0, "aclk_intmem", "aclk_bus", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(12), 4, GFLAGS), @@ -680,9 +705,9 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { /* clk_hsadc_tsp is part of diagram2 */ /* fclk_mcu_src gates */ - GATE(0, "hclk_noc_mcu", "fclk_mcu_src", 0, RK3368_CLKGATE_CON(13), 14, GFLAGS), - GATE(0, "fclk_mcu", "fclk_mcu_src", 0, RK3368_CLKGATE_CON(13), 12, GFLAGS), - GATE(0, "hclk_mcu", "fclk_mcu_src", 0, RK3368_CLKGATE_CON(13), 11, GFLAGS), + GATE(0, "hclk_noc_mcu", "fclk_mcu_src", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(13), 14, GFLAGS), + GATE(0, "fclk_mcu", "fclk_mcu_src", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(13), 12, GFLAGS), + GATE(0, "hclk_mcu", "fclk_mcu_src", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(13), 11, GFLAGS), /* hclk_cpu gates */ GATE(HCLK_SPDIF, "hclk_spdif", "hclk_bus", 0, RK3368_CLKGATE_CON(12), 10, GFLAGS), @@ -694,14 +719,14 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { GATE(MCLK_CRYPTO, "mclk_crypto", "hclk_bus", 0, RK3368_CLKGATE_CON(13), 3, GFLAGS), /* pclk_cpu gates */ - GATE(PCLK_DDRPHY, "pclk_ddrphy", "pclk_bus", 0, RK3368_CLKGATE_CON(12), 14, GFLAGS), - GATE(PCLK_DDRUPCTL, "pclk_ddrupctl", "pclk_bus", 0, RK3368_CLKGATE_CON(12), 13, GFLAGS), + GATE(PCLK_DDRPHY, "pclk_ddrphy", "pclk_bus", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(12), 14, GFLAGS), + GATE(PCLK_DDRUPCTL, "pclk_ddrupctl", "pclk_bus", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(12), 13, GFLAGS), GATE(PCLK_I2C1, "pclk_i2c1", "pclk_bus", 0, RK3368_CLKGATE_CON(12), 3, GFLAGS), GATE(PCLK_I2C0, "pclk_i2c0", "pclk_bus", 0, RK3368_CLKGATE_CON(12), 2, GFLAGS), GATE(PCLK_MAILBOX, "pclk_mailbox", "pclk_bus", 0, RK3368_CLKGATE_CON(12), 1, GFLAGS), GATE(PCLK_PWM0, "pclk_pwm0", "pclk_bus", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(12), 0, GFLAGS), GATE(PCLK_SIM, "pclk_sim", "pclk_bus", 0, RK3368_CLKGATE_CON(13), 8, GFLAGS), - GATE(PCLK_PWM1, "pclk_pwm1", "pclk_bus", 0, RK3368_CLKGATE_CON(13), 6, GFLAGS), + GATE(PCLK_PWM1, "pclk_pwm1", "pclk_bus", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(13), 6, GFLAGS), GATE(PCLK_UART2, "pclk_uart2", "pclk_bus", 0, RK3368_CLKGATE_CON(13), 5, GFLAGS), GATE(PCLK_EFUSE256, "pclk_efuse_256", "pclk_bus", 0, RK3368_CLKGATE_CON(13), 1, GFLAGS), GATE(0, "pclk_efuse_1024", "pclk_bus", 0, RK3368_CLKGATE_CON(13), 0, GFLAGS), @@ -778,7 +803,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { GATE(HCLK_HSIC, "hclk_hsic", "hclk_peri", 0, RK3368_CLKGATE_CON(20), 5, GFLAGS), GATE(HCLK_HOST1, "hclk_host1", "hclk_peri", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(20), 4, GFLAGS), GATE(HCLK_HOST0, "hclk_host0", "hclk_peri", 0, RK3368_CLKGATE_CON(20), 3, GFLAGS), - GATE(0, "pmu_hclk_otg0", "hclk_peri", 0, RK3368_CLKGATE_CON(20), 2, GFLAGS), + GATE(0, "pmu_hclk_otg0", "hclk_peri", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(20), 2, GFLAGS), GATE(HCLK_OTG0, "hclk_otg0", "hclk_peri", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(20), 1, GFLAGS), GATE(HCLK_HSADC, "hclk_hsadc", "hclk_peri", 0, RK3368_CLKGATE_CON(21), 3, GFLAGS), GATE(HCLK_EMMC, "hclk_emmc", "hclk_peri", 0, RK3368_CLKGATE_CON(21), 2, GFLAGS), @@ -805,8 +830,8 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { /* pclk_pd_alive gates */ GATE(PCLK_TIMER1, "pclk_timer1", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 13, GFLAGS), GATE(PCLK_TIMER0, "pclk_timer0", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 12, GFLAGS), - GATE(0, "pclk_alive_niu", "pclk_pd_alive", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(22), 9, GFLAGS), - GATE(PCLK_GRF, "pclk_grf", "pclk_pd_alive", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(22), 8, GFLAGS), + GATE(0, "pclk_alive_niu", "pclk_pd_alive", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(22), 9, GFLAGS), + GATE(PCLK_GRF, "pclk_grf", "pclk_pd_alive", CLK_IS_CRITICAL, RK3368_CLKGATE_CON(22), 8, GFLAGS), GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 3, GFLAGS), GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 2, GFLAGS), GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_pd_alive", 0, RK3368_CLKGATE_CON(22), 1, GFLAGS), @@ -844,27 +869,23 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { GATE(SCLK_TIMER00, "sclk_timer00", "xin24m", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(24), 0, GFLAGS), }; -static const char *const rk3368_critical_clocks[] __initconst = { - "aclk_bus", - "aclk_peri", - /* - * pwm1 supplies vdd_logic on a lot of boards, is currently unhandled - * but needs to stay enabled there (including its parents) at all times. - */ - "pclk_pwm1", - "pclk_pd_pmu", - "pclk_pd_alive", - "pclk_peri", - "hclk_peri", - "pclk_ddrphy", - "pclk_ddrupctl", - "pmu_hclk_otg0", -}; +static void __iomem *rk3368_cru_base; + +static void rk3368_dump_cru(void) +{ + if (rk3368_cru_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk3368_cru_base, + 0x41c, false); + } +} static void __init rk3368_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; + struct clk **clks; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -878,22 +899,21 @@ static void __init rk3368_clk_init(struct device_node *np) iounmap(reg_base); return; } + clks = ctx->clk_data.clks; rockchip_clk_register_plls(ctx, rk3368_pll_clks, ARRAY_SIZE(rk3368_pll_clks), RK3368_GRF_SOC_STATUS0); rockchip_clk_register_branches(ctx, rk3368_clk_branches, ARRAY_SIZE(rk3368_clk_branches)); - rockchip_clk_protect_critical(rk3368_critical_clocks, - ARRAY_SIZE(rk3368_critical_clocks)); rockchip_clk_register_armclk(ctx, ARMCLKB, "armclkb", - mux_armclkb_p, ARRAY_SIZE(mux_armclkb_p), + 2, clks[PLL_APLLB], clks[PLL_GPLL], &rk3368_cpuclkb_data, rk3368_cpuclkb_rates, ARRAY_SIZE(rk3368_cpuclkb_rates)); rockchip_clk_register_armclk(ctx, ARMCLKL, "armclkl", - mux_armclkl_p, ARRAY_SIZE(mux_armclkl_p), + 2, clks[PLL_APLLL], clks[PLL_GPLL], &rk3368_cpuclkl_data, rk3368_cpuclkl_rates, ARRAY_SIZE(rk3368_cpuclkl_rates)); @@ -903,5 +923,38 @@ static void __init rk3368_clk_init(struct device_node *np) rockchip_register_restart_notifier(ctx, RK3368_GLB_SRST_FST, NULL); rockchip_clk_of_add_provider(np, ctx); + + if (!rk_dump_cru) { + rk3368_cru_base = reg_base; + rk_dump_cru = rk3368_dump_cru; + } } CLK_OF_DECLARE(rk3368_cru, "rockchip,rk3368-cru", rk3368_clk_init); + +static int __init clk_rk3368_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + + rk3368_clk_init(np); + + return 0; +} + +static const struct of_device_id clk_rk3368_match_table[] = { + { + .compatible = "rockchip,rk3368-cru", + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk3368_match_table); + +static struct platform_driver clk_rk3368_driver = { + .driver = { + .name = "clk-rk3368", + .of_match_table = clk_rk3368_match_table, + }, +}; +builtin_platform_driver_probe(clk_rk3368_driver, clk_rk3368_probe); + +MODULE_DESCRIPTION("Rockchip RK3368 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rk3399.c b/drivers/clk/rockchip/clk-rk3399.c index 7df2f1e00347..d5071884b3d5 100644 --- a/drivers/clk/rockchip/clk-rk3399.c +++ b/drivers/clk/rockchip/clk-rk3399.c @@ -15,6 +15,12 @@ #include #include "clk.h" +#define RK3399_I2S_FRAC_MAX_PRATE 800000000 +#define RK3399_UART_FRAC_MAX_PRATE 800000000 +#define RK3399_SPDIF_FRAC_MAX_PRATE 600000000 +#define RK3399_VOP_FRAC_MAX_PRATE 600000000 +#define RK3399_WIFI_FRAC_MAX_PRATE 600000000 + enum rk3399_plls { lpll, bpll, dpll, cpll, gpll, npll, vpll, }; @@ -105,25 +111,95 @@ static struct rockchip_pll_rate_table rk3399_pll_rates[] = { { /* sentinel */ }, }; +static struct rockchip_pll_rate_table rk3399_vpll_rates[] = { + /* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */ + RK3036_PLL_RATE( 594000000, 1, 123, 5, 1, 0, 12582912), /* vco = 2970000000 */ + RK3036_PLL_RATE( 593406593, 1, 123, 5, 1, 0, 10508804), /* vco = 2967032965 */ + RK3036_PLL_RATE( 297000000, 1, 123, 5, 2, 0, 12582912), /* vco = 2970000000 */ + RK3036_PLL_RATE( 296703297, 1, 123, 5, 2, 0, 10508807), /* vco = 2967032970 */ + RK3036_PLL_RATE( 148500000, 1, 129, 7, 3, 0, 15728640), /* vco = 3118500000 */ + RK3036_PLL_RATE( 148351648, 1, 123, 5, 4, 0, 10508800), /* vco = 2967032960 */ + RK3036_PLL_RATE( 106500000, 1, 124, 7, 4, 0, 4194304), /* vco = 2982000000 */ + RK3036_PLL_RATE( 74250000, 1, 129, 7, 6, 0, 15728640), /* vco = 3118500000 */ + RK3036_PLL_RATE( 74175824, 1, 129, 7, 6, 0, 13550823), /* vco = 3115384608 */ + RK3036_PLL_RATE( 65000000, 1, 113, 7, 6, 0, 12582912), /* vco = 2730000000 */ + RK3036_PLL_RATE( 59340659, 1, 121, 7, 7, 0, 2581098), /* vco = 2907692291 */ + RK3036_PLL_RATE( 54000000, 1, 110, 7, 7, 0, 4194304), /* vco = 2646000000 */ + RK3036_PLL_RATE( 27000000, 1, 55, 7, 7, 0, 2097152), /* vco = 1323000000 */ + RK3036_PLL_RATE( 26973027, 1, 55, 7, 7, 0, 1173232), /* vco = 1321678323 */ + { /* sentinel */ }, +}; + /* CRU parents */ PNAME(mux_pll_p) = { "xin24m", "xin32k" }; -PNAME(mux_armclkl_p) = { "clk_core_l_lpll_src", - "clk_core_l_bpll_src", - "clk_core_l_dpll_src", - "clk_core_l_gpll_src" }; -PNAME(mux_armclkb_p) = { "clk_core_b_lpll_src", - "clk_core_b_bpll_src", - "clk_core_b_dpll_src", - "clk_core_b_gpll_src" }; PNAME(mux_ddrclk_p) = { "clk_ddrc_lpll_src", "clk_ddrc_bpll_src", "clk_ddrc_dpll_src", "clk_ddrc_gpll_src" }; + +PNAME(mux_pll_src_vpll_cpll_gpll_p) = { "vpll", "cpll", "gpll" }; +PNAME(mux_pll_src_dmyvpll_cpll_gpll_p) = { "dummy_vpll", "cpll", "gpll" }; + +#ifdef RK3399_TWO_PLL_FOR_VOP +PNAME(mux_aclk_cci_p) = { "dummy_cpll", + "gpll_aclk_cci_src", + "npll_aclk_cci_src", + "dummy_vpll" }; +PNAME(mux_cci_trace_p) = { "dummy_cpll", + "gpll_cci_trace" }; +PNAME(mux_cs_p) = { "dummy_cpll", "gpll_cs", + "npll_cs"}; +PNAME(mux_aclk_perihp_p) = { "dummy_cpll", + "gpll_aclk_perihp_src" }; + +PNAME(mux_pll_src_cpll_gpll_p) = { "dummy_cpll", "gpll" }; +PNAME(mux_pll_src_cpll_gpll_npll_p) = { "dummy_cpll", "gpll", "npll" }; +PNAME(mux_pll_src_cpll_gpll_ppll_p) = { "dummy_cpll", "gpll", "ppll" }; +PNAME(mux_pll_src_cpll_gpll_upll_p) = { "dummy_cpll", "gpll", "upll" }; +PNAME(mux_pll_src_npll_cpll_gpll_p) = { "npll", "dummy_cpll", "gpll" }; +PNAME(mux_pll_src_cpll_gpll_npll_ppll_p) = { "dummy_cpll", "gpll", "npll", + "ppll" }; +PNAME(mux_pll_src_cpll_gpll_npll_24m_p) = { "dummy_cpll", "gpll", "npll", + "xin24m" }; +PNAME(mux_pll_src_cpll_gpll_npll_usbphy480m_p) = { "dummy_cpll", "gpll", "npll", + "clk_usbphy_480m" }; +PNAME(mux_pll_src_ppll_cpll_gpll_npll_p) = { "ppll", "dummy_cpll", "gpll", + "npll", "upll" }; +PNAME(mux_pll_src_cpll_gpll_npll_upll_24m_p) = { "dummy_cpll", "gpll", "npll", + "upll", "xin24m" }; +PNAME(mux_pll_src_cpll_gpll_npll_ppll_upll_24m_p) = { "dummy_cpll", "gpll", "npll", + "ppll", "upll", "xin24m" }; +/* + * We hope to be able to HDMI/DP can obtain better signal quality, + * therefore, we move VOP pwm and aclk clocks to other PLLs, let + * HDMI/DP phyclock can monopolize VPLL. + */ +PNAME(mux_pll_src_dmyvpll_cpll_gpll_npll_p) = { "dummy_vpll", "dummy_cpll", "gpll", + "npll" }; +PNAME(mux_pll_src_dmyvpll_cpll_gpll_gpll_p) = { "dummy_vpll", "dummy_cpll", "gpll", + "gpll" }; +PNAME(mux_pll_src_24m_32k_cpll_gpll_p) = { "xin24m", "xin32k", + "dummy_cpll", "gpll" }; + +PNAME(mux_aclk_emmc_p) = { "dummy_cpll", + "gpll_aclk_emmc_src" }; + +PNAME(mux_aclk_perilp0_p) = { "dummy_cpll", + "gpll_aclk_perilp0_src" }; + +PNAME(mux_fclk_cm0s_p) = { "dummy_cpll", + "gpll_fclk_cm0s_src" }; + +PNAME(mux_hclk_perilp1_p) = { "dummy_cpll", + "gpll_hclk_perilp1_src" }; +PNAME(mux_aclk_gmac_p) = { "dummy_cpll", + "gpll_aclk_gmac_src" }; +#else PNAME(mux_aclk_cci_p) = { "cpll_aclk_cci_src", "gpll_aclk_cci_src", "npll_aclk_cci_src", - "vpll_aclk_cci_src" }; + "dummy_vpll" }; PNAME(mux_cci_trace_p) = { "cpll_cci_trace", "gpll_cci_trace" }; PNAME(mux_cs_p) = { "cpll_cs", "gpll_cs", @@ -148,26 +224,17 @@ PNAME(mux_pll_src_cpll_gpll_npll_upll_24m_p) = { "cpll", "gpll", "npll", "upll", "xin24m" }; PNAME(mux_pll_src_cpll_gpll_npll_ppll_upll_24m_p) = { "cpll", "gpll", "npll", "ppll", "upll", "xin24m" }; - -PNAME(mux_pll_src_vpll_cpll_gpll_p) = { "vpll", "cpll", "gpll" }; -PNAME(mux_pll_src_vpll_cpll_gpll_npll_p) = { "vpll", "cpll", "gpll", +/* + * We hope to be able to HDMI/DP can obtain better signal quality, + * therefore, we move VOP pwm and aclk clocks to other PLLs, let + * HDMI/DP phyclock can monopolize VPLL. + */ +PNAME(mux_pll_src_dmyvpll_cpll_gpll_npll_p) = { "dummy_vpll", "cpll", "gpll", "npll" }; -PNAME(mux_pll_src_vpll_cpll_gpll_24m_p) = { "vpll", "cpll", "gpll", - "xin24m" }; - -PNAME(mux_dclk_vop0_p) = { "dclk_vop0_div", - "dclk_vop0_frac" }; -PNAME(mux_dclk_vop1_p) = { "dclk_vop1_div", - "dclk_vop1_frac" }; - -PNAME(mux_clk_cif_p) = { "clk_cifout_src", "xin24m" }; - -PNAME(mux_pll_src_24m_usbphy480m_p) = { "xin24m", "clk_usbphy_480m" }; -PNAME(mux_pll_src_24m_pciephy_p) = { "xin24m", "clk_pciephy_ref100m" }; +PNAME(mux_pll_src_dmyvpll_cpll_gpll_gpll_p) = { "dummy_vpll", "cpll", "gpll", + "gpll" }; PNAME(mux_pll_src_24m_32k_cpll_gpll_p) = { "xin24m", "xin32k", "cpll", "gpll" }; -PNAME(mux_pciecore_cru_phy_p) = { "clk_pcie_core_cru", - "clk_pcie_core_phy" }; PNAME(mux_aclk_emmc_p) = { "cpll_aclk_emmc_src", "gpll_aclk_emmc_src" }; @@ -180,14 +247,26 @@ PNAME(mux_fclk_cm0s_p) = { "cpll_fclk_cm0s_src", PNAME(mux_hclk_perilp1_p) = { "cpll_hclk_perilp1_src", "gpll_hclk_perilp1_src" }; +PNAME(mux_aclk_gmac_p) = { "cpll_aclk_gmac_src", + "gpll_aclk_gmac_src" }; +#endif + +PNAME(mux_dclk_vop0_p) = { "dclk_vop0_div", + "dummy_dclk_vop0_frac" }; +PNAME(mux_dclk_vop1_p) = { "dclk_vop1_div", + "dummy_dclk_vop1_frac" }; + +PNAME(mux_clk_cif_p) = { "clk_cifout_src", "xin24m" }; +PNAME(mux_pll_src_24m_usbphy480m_p) = { "xin24m", "clk_usbphy_480m" }; +PNAME(mux_pll_src_24m_pciephy_p) = { "xin24m", "clk_pciephy_ref100m" }; +PNAME(mux_pciecore_cru_phy_p) = { "clk_pcie_core_cru", + "clk_pcie_core_phy" }; PNAME(mux_clk_testout1_p) = { "clk_testout1_pll_src", "xin24m" }; PNAME(mux_clk_testout2_p) = { "clk_testout2_pll_src", "xin24m" }; PNAME(mux_usbphy_480m_p) = { "clk_usbphy0_480m_src", "clk_usbphy1_480m_src" }; -PNAME(mux_aclk_gmac_p) = { "cpll_aclk_gmac_src", - "gpll_aclk_gmac_src" }; PNAME(mux_rmii_p) = { "clk_gmac", "clkin_gmac" }; PNAME(mux_spdif_p) = { "clk_spdif_div", "clk_spdif_frac", "clkin_i2s", "xin12m" }; @@ -201,20 +280,22 @@ PNAME(mux_i2sch_p) = { "clk_i2s0", "clk_i2s1", "clk_i2s2" }; PNAME(mux_i2sout_p) = { "clk_i2sout_src", "xin12m" }; -PNAME(mux_uart0_p) = { "clk_uart0_div", "clk_uart0_frac", "xin24m" }; -PNAME(mux_uart1_p) = { "clk_uart1_div", "clk_uart1_frac", "xin24m" }; -PNAME(mux_uart2_p) = { "clk_uart2_div", "clk_uart2_frac", "xin24m" }; -PNAME(mux_uart3_p) = { "clk_uart3_div", "clk_uart3_frac", "xin24m" }; +PNAME(mux_uart0_p) = { "xin24m", "clk_uart0_div", "clk_uart0_frac" }; +PNAME(mux_uart1_p) = { "xin24m", "clk_uart1_div", "clk_uart1_frac" }; +PNAME(mux_uart2_p) = { "xin24m", "clk_uart2_div", "clk_uart2_frac" }; +PNAME(mux_uart3_p) = { "xin24m", "clk_uart3_div", "clk_uart3_frac" }; /* PMU CRU parents */ PNAME(mux_ppll_24m_p) = { "ppll", "xin24m" }; PNAME(mux_24m_ppll_p) = { "xin24m", "ppll" }; PNAME(mux_fclk_cm0s_pmu_ppll_p) = { "fclk_cm0s_pmu_ppll_src", "xin24m" }; PNAME(mux_wifi_pmu_p) = { "clk_wifi_div", "clk_wifi_frac" }; -PNAME(mux_uart4_pmu_p) = { "clk_uart4_div", "clk_uart4_frac", - "xin24m" }; +PNAME(mux_uart4_pmu_p) = { "xin24m", "clk_uart4_div", + "clk_uart4_frac" }; PNAME(mux_clk_testout2_2io_p) = { "clk_testout2", "clk_32k_suspend_pmu" }; +static u32 uart_mux_idx[] = { 2, 0, 1 }; + static struct rockchip_pll_clock rk3399_pll_clks[] __initdata = { [lpll] = PLL(pll_rk3399, PLL_APLLL, "lpll", mux_pll_p, 0, RK3399_PLL_CON(0), RK3399_PLL_CON(3), 8, 31, 0, rk3399_pll_rates), @@ -222,18 +303,23 @@ static struct rockchip_pll_clock rk3399_pll_clks[] __initdata = { RK3399_PLL_CON(11), 8, 31, 0, rk3399_pll_rates), [dpll] = PLL(pll_rk3399, PLL_DPLL, "dpll", mux_pll_p, 0, RK3399_PLL_CON(16), RK3399_PLL_CON(19), 8, 31, 0, NULL), +#ifdef RK3399_TWO_PLL_FOR_VOP + [cpll] = PLL(pll_rk3399, PLL_CPLL, "cpll", mux_pll_p, 0, RK3399_PLL_CON(24), + RK3399_PLL_CON(27), 8, 31, 0, rk3399_pll_rates), +#else [cpll] = PLL(pll_rk3399, PLL_CPLL, "cpll", mux_pll_p, 0, RK3399_PLL_CON(24), RK3399_PLL_CON(27), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk3399_pll_rates), +#endif [gpll] = PLL(pll_rk3399, PLL_GPLL, "gpll", mux_pll_p, 0, RK3399_PLL_CON(32), - RK3399_PLL_CON(35), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk3399_pll_rates), + RK3399_PLL_CON(35), 8, 31, 0, rk3399_pll_rates), [npll] = PLL(pll_rk3399, PLL_NPLL, "npll", mux_pll_p, 0, RK3399_PLL_CON(40), RK3399_PLL_CON(43), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk3399_pll_rates), [vpll] = PLL(pll_rk3399, PLL_VPLL, "vpll", mux_pll_p, 0, RK3399_PLL_CON(48), - RK3399_PLL_CON(51), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk3399_pll_rates), + RK3399_PLL_CON(51), 8, 31, 0, rk3399_vpll_rates), }; static struct rockchip_pll_clock rk3399_pmu_pll_clks[] __initdata = { - [ppll] = PLL(pll_rk3399, PLL_PPLL, "ppll", mux_pll_p, 0, RK3399_PMU_PLL_CON(0), + [ppll] = PLL(pll_rk3399, PLL_PPLL, "ppll", mux_pll_p, CLK_IS_CRITICAL, RK3399_PMU_PLL_CON(0), RK3399_PMU_PLL_CON(3), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk3399_pll_rates), }; @@ -259,24 +345,24 @@ static struct rockchip_clk_branch rk3399_i2s2_fracmux __initdata = RK3399_CLKSEL_CON(30), 8, 2, MFLAGS); static struct rockchip_clk_branch rk3399_uart0_fracmux __initdata = - MUX(SCLK_UART0, "clk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT, - RK3399_CLKSEL_CON(33), 8, 2, MFLAGS); + MUXTBL(SCLK_UART0, "clk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(33), 8, 2, MFLAGS, uart_mux_idx); static struct rockchip_clk_branch rk3399_uart1_fracmux __initdata = - MUX(SCLK_UART1, "clk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT, - RK3399_CLKSEL_CON(34), 8, 2, MFLAGS); + MUXTBL(SCLK_UART1, "clk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(34), 8, 2, MFLAGS, uart_mux_idx); static struct rockchip_clk_branch rk3399_uart2_fracmux __initdata = - MUX(SCLK_UART2, "clk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT, - RK3399_CLKSEL_CON(35), 8, 2, MFLAGS); + MUXTBL(SCLK_UART2, "clk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(35), 8, 2, MFLAGS, uart_mux_idx); static struct rockchip_clk_branch rk3399_uart3_fracmux __initdata = - MUX(SCLK_UART3, "clk_uart3", mux_uart3_p, CLK_SET_RATE_PARENT, - RK3399_CLKSEL_CON(36), 8, 2, MFLAGS); + MUXTBL(SCLK_UART3, "clk_uart3", mux_uart3_p, CLK_SET_RATE_PARENT, + RK3399_CLKSEL_CON(36), 8, 2, MFLAGS, uart_mux_idx); static struct rockchip_clk_branch rk3399_uart4_pmu_fracmux __initdata = - MUX(SCLK_UART4_PMU, "clk_uart4_pmu", mux_uart4_pmu_p, CLK_SET_RATE_PARENT, - RK3399_PMU_CLKSEL_CON(5), 8, 2, MFLAGS); + MUXTBL(SCLK_UART4_PMU, "clk_uart4_pmu", mux_uart4_pmu_p, CLK_SET_RATE_PARENT, + RK3399_PMU_CLKSEL_CON(5), 8, 2, MFLAGS, uart_mux_idx); static struct rockchip_clk_branch rk3399_dclk_vop0_fracmux __initdata = MUX(DCLK_VOP0, "dclk_vop0", mux_dclk_vop0_p, CLK_SET_RATE_PARENT, @@ -291,9 +377,10 @@ static struct rockchip_clk_branch rk3399_pmuclk_wifi_fracmux __initdata = RK3399_PMU_CLKSEL_CON(1), 14, 1, MFLAGS); static const struct rockchip_cpuclk_reg_data rk3399_cpuclkl_data = { - .core_reg = RK3399_CLKSEL_CON(0), - .div_core_shift = 0, - .div_core_mask = 0x1f, + .core_reg[0] = RK3399_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 3, .mux_core_main = 0, .mux_core_shift = 6, @@ -301,9 +388,10 @@ static const struct rockchip_cpuclk_reg_data rk3399_cpuclkl_data = { }; static const struct rockchip_cpuclk_reg_data rk3399_cpuclkb_data = { - .core_reg = RK3399_CLKSEL_CON(2), - .div_core_shift = 0, - .div_core_mask = 0x1f, + .core_reg[0] = RK3399_CLKSEL_CON(2), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 3, .mux_core_main = 1, .mux_core_shift = 6, @@ -406,9 +494,9 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { GATE(SCLK_USB2PHY1_REF, "clk_usb2phy1_ref", "xin24m", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(6), 6, GFLAGS), - GATE(0, "clk_usbphy0_480m_src", "clk_usbphy0_480m", 0, + GATE(SCLK_USBPHY0_480M_SRC, "clk_usbphy0_480m_src", "clk_usbphy0_480m", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(13), 12, GFLAGS), - GATE(0, "clk_usbphy1_480m_src", "clk_usbphy1_480m", 0, + GATE(SCLK_USBPHY1_480M_SRC, "clk_usbphy1_480m_src", "clk_usbphy1_480m", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(13), 12, GFLAGS), MUX(0, "clk_usbphy_480m", mux_usbphy_480m_p, 0, RK3399_CLKSEL_CON(14), 6, 1, MFLAGS), @@ -423,7 +511,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { COMPOSITE(ACLK_USB3, "aclk_usb3", mux_pll_src_cpll_gpll_npll_p, 0, RK3399_CLKSEL_CON(39), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3399_CLKGATE_CON(12), 0, GFLAGS), - GATE(ACLK_USB3_NOC, "aclk_usb3_noc", "aclk_usb3", CLK_IGNORE_UNUSED, + GATE(ACLK_USB3_NOC, "aclk_usb3_noc", "aclk_usb3", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(30), 0, GFLAGS), GATE(ACLK_USB3OTG0, "aclk_usb3otg0", "aclk_usb3", 0, RK3399_CLKGATE_CON(30), 1, GFLAGS), @@ -549,7 +637,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { GATE(ACLK_GMAC, "aclk_gmac", "aclk_gmac_pre", 0, RK3399_CLKGATE_CON(32), 0, GFLAGS), - GATE(ACLK_GMAC_NOC, "aclk_gmac_noc", "aclk_gmac_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_GMAC_NOC, "aclk_gmac_noc", "aclk_gmac_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(32), 1, GFLAGS), GATE(ACLK_PERF_GMAC, "aclk_perf_gmac", "aclk_gmac_pre", 0, RK3399_CLKGATE_CON(32), 4, GFLAGS), @@ -559,7 +647,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(6), 11, GFLAGS), GATE(PCLK_GMAC, "pclk_gmac", "pclk_gmac_pre", 0, RK3399_CLKGATE_CON(32), 2, GFLAGS), - GATE(PCLK_GMAC_NOC, "pclk_gmac_noc", "pclk_gmac_pre", CLK_IGNORE_UNUSED, + GATE(PCLK_GMAC_NOC, "pclk_gmac_noc", "pclk_gmac_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(32), 3, GFLAGS), COMPOSITE(SCLK_MAC, "clk_gmac", mux_pll_src_cpll_gpll_npll_p, 0, @@ -578,13 +666,13 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(5), 9, GFLAGS), /* spdif */ - COMPOSITE(0, "clk_spdif_div", mux_pll_src_cpll_gpll_p, 0, + COMPOSITE(SCLK_SPDIF_DIV, "clk_spdif_div", mux_pll_src_cpll_gpll_p, 0, RK3399_CLKSEL_CON(32), 7, 1, MFLAGS, 0, 7, DFLAGS, RK3399_CLKGATE_CON(8), 13, GFLAGS), - COMPOSITE_FRACMUX(0, "clk_spdif_frac", "clk_spdif_div", 0, + COMPOSITE_FRACMUX(0, "clk_spdif_frac", "clk_spdif_div", CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(99), 0, RK3399_CLKGATE_CON(8), 14, GFLAGS, - &rk3399_spdif_fracmux), + &rk3399_spdif_fracmux, RK3399_SPDIF_FRAC_MAX_PRATE), GATE(SCLK_SPDIF_8CH, "clk_spdif", "clk_spdif_mux", CLK_SET_RATE_PARENT, RK3399_CLKGATE_CON(8), 15, GFLAGS), @@ -592,84 +680,84 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKSEL_CON(32), 15, 1, MFLAGS, 8, 5, DFLAGS, RK3399_CLKGATE_CON(10), 6, GFLAGS), /* i2s */ - COMPOSITE(0, "clk_i2s0_div", mux_pll_src_cpll_gpll_p, 0, + COMPOSITE(SCLK_I2S0_DIV, "clk_i2s0_div", mux_pll_src_cpll_gpll_p, 0, RK3399_CLKSEL_CON(28), 7, 1, MFLAGS, 0, 7, DFLAGS, RK3399_CLKGATE_CON(8), 3, GFLAGS), - COMPOSITE_FRACMUX(0, "clk_i2s0_frac", "clk_i2s0_div", 0, + COMPOSITE_FRACMUX(0, "clk_i2s0_frac", "clk_i2s0_div", CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(96), 0, RK3399_CLKGATE_CON(8), 4, GFLAGS, - &rk3399_i2s0_fracmux), + &rk3399_i2s0_fracmux, RK3399_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S0_8CH, "clk_i2s0", "clk_i2s0_mux", CLK_SET_RATE_PARENT, RK3399_CLKGATE_CON(8), 5, GFLAGS), - COMPOSITE(0, "clk_i2s1_div", mux_pll_src_cpll_gpll_p, 0, + COMPOSITE(SCLK_I2S1_DIV, "clk_i2s1_div", mux_pll_src_cpll_gpll_p, 0, RK3399_CLKSEL_CON(29), 7, 1, MFLAGS, 0, 7, DFLAGS, RK3399_CLKGATE_CON(8), 6, GFLAGS), - COMPOSITE_FRACMUX(0, "clk_i2s1_frac", "clk_i2s1_div", 0, + COMPOSITE_FRACMUX(0, "clk_i2s1_frac", "clk_i2s1_div", CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(97), 0, RK3399_CLKGATE_CON(8), 7, GFLAGS, - &rk3399_i2s1_fracmux), + &rk3399_i2s1_fracmux, RK3399_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S1_8CH, "clk_i2s1", "clk_i2s1_mux", CLK_SET_RATE_PARENT, RK3399_CLKGATE_CON(8), 8, GFLAGS), - COMPOSITE(0, "clk_i2s2_div", mux_pll_src_cpll_gpll_p, 0, + COMPOSITE(SCLK_I2S2_DIV, "clk_i2s2_div", mux_pll_src_cpll_gpll_p, 0, RK3399_CLKSEL_CON(30), 7, 1, MFLAGS, 0, 7, DFLAGS, RK3399_CLKGATE_CON(8), 9, GFLAGS), - COMPOSITE_FRACMUX(0, "clk_i2s2_frac", "clk_i2s2_div", 0, + COMPOSITE_FRACMUX(0, "clk_i2s2_frac", "clk_i2s2_div", CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(98), 0, RK3399_CLKGATE_CON(8), 10, GFLAGS, - &rk3399_i2s2_fracmux), + &rk3399_i2s2_fracmux, RK3399_I2S_FRAC_MAX_PRATE), GATE(SCLK_I2S2_8CH, "clk_i2s2", "clk_i2s2_mux", CLK_SET_RATE_PARENT, RK3399_CLKGATE_CON(8), 11, GFLAGS), - MUX(0, "clk_i2sout_src", mux_i2sch_p, CLK_SET_RATE_PARENT, + MUX(SCLK_I2SOUT_SRC, "clk_i2sout_src", mux_i2sch_p, CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(31), 0, 2, MFLAGS), COMPOSITE_NODIV(SCLK_I2S_8CH_OUT, "clk_i2sout", mux_i2sout_p, CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(31), 2, 1, MFLAGS, RK3399_CLKGATE_CON(8), 12, GFLAGS), /* uart */ - MUX(0, "clk_uart0_src", mux_pll_src_cpll_gpll_upll_p, 0, + MUX(SCLK_UART0_SRC, "clk_uart0_src", mux_pll_src_cpll_gpll_upll_p, 0, RK3399_CLKSEL_CON(33), 12, 2, MFLAGS), COMPOSITE_NOMUX(0, "clk_uart0_div", "clk_uart0_src", 0, RK3399_CLKSEL_CON(33), 0, 7, DFLAGS, RK3399_CLKGATE_CON(9), 0, GFLAGS), - COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_div", 0, + COMPOSITE_FRACMUX(0, "clk_uart0_frac", "clk_uart0_div", CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(100), 0, RK3399_CLKGATE_CON(9), 1, GFLAGS, - &rk3399_uart0_fracmux), + &rk3399_uart0_fracmux, RK3399_UART_FRAC_MAX_PRATE), - MUX(0, "clk_uart_src", mux_pll_src_cpll_gpll_p, 0, + MUX(SCLK_UART_SRC, "clk_uart_src", mux_pll_src_cpll_gpll_p, 0, RK3399_CLKSEL_CON(33), 15, 1, MFLAGS), COMPOSITE_NOMUX(0, "clk_uart1_div", "clk_uart_src", 0, RK3399_CLKSEL_CON(34), 0, 7, DFLAGS, RK3399_CLKGATE_CON(9), 2, GFLAGS), - COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_div", 0, + COMPOSITE_FRACMUX(0, "clk_uart1_frac", "clk_uart1_div", CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(101), 0, RK3399_CLKGATE_CON(9), 3, GFLAGS, - &rk3399_uart1_fracmux), + &rk3399_uart1_fracmux, RK3399_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "clk_uart2_div", "clk_uart_src", 0, RK3399_CLKSEL_CON(35), 0, 7, DFLAGS, RK3399_CLKGATE_CON(9), 4, GFLAGS), - COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_div", 0, + COMPOSITE_FRACMUX(0, "clk_uart2_frac", "clk_uart2_div", CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(102), 0, RK3399_CLKGATE_CON(9), 5, GFLAGS, - &rk3399_uart2_fracmux), + &rk3399_uart2_fracmux, RK3399_UART_FRAC_MAX_PRATE), COMPOSITE_NOMUX(0, "clk_uart3_div", "clk_uart_src", 0, RK3399_CLKSEL_CON(36), 0, 7, DFLAGS, RK3399_CLKGATE_CON(9), 6, GFLAGS), - COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_div", 0, + COMPOSITE_FRACMUX(0, "clk_uart3_frac", "clk_uart3_div", CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(103), 0, RK3399_CLKGATE_CON(9), 7, GFLAGS, - &rk3399_uart3_fracmux), + &rk3399_uart3_fracmux, RK3399_UART_FRAC_MAX_PRATE), - COMPOSITE(PCLK_DDR, "pclk_ddr", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED, + COMPOSITE(PCLK_DDR, "pclk_ddr", mux_pll_src_cpll_gpll_p, CLK_IS_CRITICAL, RK3399_CLKSEL_CON(6), 15, 1, MFLAGS, 8, 5, DFLAGS, RK3399_CLKGATE_CON(3), 4, GFLAGS), - GATE(PCLK_CENTER_MAIN_NOC, "pclk_center_main_noc", "pclk_ddr", CLK_IGNORE_UNUSED, + GATE(PCLK_CENTER_MAIN_NOC, "pclk_center_main_noc", "pclk_ddr", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(18), 10, GFLAGS), GATE(PCLK_DDR_MON, "pclk_ddr_mon", "pclk_ddr", 0, RK3399_CLKGATE_CON(18), 12, GFLAGS), @@ -686,30 +774,30 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(3), 6, GFLAGS), /* cci */ - GATE(0, "cpll_aclk_cci_src", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_aclk_cci_src", "cpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(2), 0, GFLAGS), - GATE(0, "gpll_aclk_cci_src", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_aclk_cci_src", "gpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(2), 1, GFLAGS), - GATE(0, "npll_aclk_cci_src", "npll", CLK_IGNORE_UNUSED, + GATE(0, "npll_aclk_cci_src", "npll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(2), 2, GFLAGS), - GATE(0, "vpll_aclk_cci_src", "vpll", CLK_IGNORE_UNUSED, + GATE(0, "vpll_aclk_cci_src", "vpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(2), 3, GFLAGS), - COMPOSITE(0, "aclk_cci_pre", mux_aclk_cci_p, CLK_IGNORE_UNUSED, + COMPOSITE(0, "aclk_cci_pre", mux_aclk_cci_p, CLK_IS_CRITICAL, RK3399_CLKSEL_CON(5), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3399_CLKGATE_CON(2), 4, GFLAGS), - GATE(ACLK_ADB400M_PD_CORE_L, "aclk_adb400m_pd_core_l", "aclk_cci_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_ADB400M_PD_CORE_L, "aclk_adb400m_pd_core_l", "aclk_cci_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(15), 0, GFLAGS), - GATE(ACLK_ADB400M_PD_CORE_B, "aclk_adb400m_pd_core_b", "aclk_cci_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_ADB400M_PD_CORE_B, "aclk_adb400m_pd_core_b", "aclk_cci_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(15), 1, GFLAGS), - GATE(ACLK_CCI, "aclk_cci", "aclk_cci_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_CCI, "aclk_cci", "aclk_cci_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(15), 2, GFLAGS), - GATE(ACLK_CCI_NOC0, "aclk_cci_noc0", "aclk_cci_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_CCI_NOC0, "aclk_cci_noc0", "aclk_cci_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(15), 3, GFLAGS), - GATE(ACLK_CCI_NOC1, "aclk_cci_noc1", "aclk_cci_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_CCI_NOC1, "aclk_cci_noc1", "aclk_cci_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(15), 4, GFLAGS), - GATE(ACLK_CCI_GRF, "aclk_cci_grf", "aclk_cci_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_CCI_GRF, "aclk_cci_grf", "aclk_cci_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(15), 7, GFLAGS), GATE(0, "cpll_cci_trace", "cpll", CLK_IGNORE_UNUSED, @@ -717,20 +805,20 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { GATE(0, "gpll_cci_trace", "gpll", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(2), 6, GFLAGS), COMPOSITE(SCLK_CCI_TRACE, "clk_cci_trace", mux_cci_trace_p, CLK_IGNORE_UNUSED, - RK3399_CLKSEL_CON(5), 15, 2, MFLAGS, 8, 5, DFLAGS, + RK3399_CLKSEL_CON(5), 15, 1, MFLAGS, 8, 5, DFLAGS, RK3399_CLKGATE_CON(2), 7, GFLAGS), - GATE(0, "cpll_cs", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_cs", "cpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(2), 8, GFLAGS), - GATE(0, "gpll_cs", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_cs", "gpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(2), 9, GFLAGS), - GATE(0, "npll_cs", "npll", CLK_IGNORE_UNUSED, + GATE(0, "npll_cs", "npll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(2), 10, GFLAGS), - COMPOSITE_NOGATE(0, "clk_cs", mux_cs_p, CLK_IGNORE_UNUSED, + COMPOSITE_NOGATE(SCLK_CS, "clk_cs", mux_cs_p, CLK_IS_CRITICAL, RK3399_CLKSEL_CON(4), 6, 2, MFLAGS, 0, 5, DFLAGS), GATE(0, "clk_dbg_cxcs", "clk_cs", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(15), 5, GFLAGS), - GATE(0, "clk_dbg_noc", "clk_cs", CLK_IGNORE_UNUSED, + GATE(0, "clk_dbg_noc", "clk_cs", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(15), 6, GFLAGS), /* vcodec */ @@ -742,12 +830,12 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(4), 1, GFLAGS), GATE(HCLK_VCODEC, "hclk_vcodec", "hclk_vcodec_pre", 0, RK3399_CLKGATE_CON(17), 2, GFLAGS), - GATE(0, "hclk_vcodec_noc", "hclk_vcodec_pre", CLK_IGNORE_UNUSED, + GATE(0, "hclk_vcodec_noc", "hclk_vcodec_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(17), 3, GFLAGS), GATE(ACLK_VCODEC, "aclk_vcodec", "aclk_vcodec_pre", 0, RK3399_CLKGATE_CON(17), 0, GFLAGS), - GATE(0, "aclk_vcodec_noc", "aclk_vcodec_pre", CLK_IGNORE_UNUSED, + GATE(0, "aclk_vcodec_noc", "aclk_vcodec_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(17), 1, GFLAGS), /* vdu */ @@ -766,12 +854,12 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(4), 3, GFLAGS), GATE(HCLK_VDU, "hclk_vdu", "hclk_vdu_pre", 0, RK3399_CLKGATE_CON(17), 10, GFLAGS), - GATE(HCLK_VDU_NOC, "hclk_vdu_noc", "hclk_vdu_pre", CLK_IGNORE_UNUSED, + GATE(HCLK_VDU_NOC, "hclk_vdu_noc", "hclk_vdu_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(17), 11, GFLAGS), GATE(ACLK_VDU, "aclk_vdu", "aclk_vdu_pre", 0, RK3399_CLKGATE_CON(17), 8, GFLAGS), - GATE(ACLK_VDU_NOC, "aclk_vdu_noc", "aclk_vdu_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_VDU_NOC, "aclk_vdu_noc", "aclk_vdu_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(17), 9, GFLAGS), /* iep */ @@ -783,12 +871,12 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(4), 7, GFLAGS), GATE(HCLK_IEP, "hclk_iep", "hclk_iep_pre", 0, RK3399_CLKGATE_CON(16), 2, GFLAGS), - GATE(HCLK_IEP_NOC, "hclk_iep_noc", "hclk_iep_pre", CLK_IGNORE_UNUSED, + GATE(HCLK_IEP_NOC, "hclk_iep_noc", "hclk_iep_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(16), 3, GFLAGS), GATE(ACLK_IEP, "aclk_iep", "aclk_iep_pre", 0, RK3399_CLKGATE_CON(16), 0, GFLAGS), - GATE(ACLK_IEP_NOC, "aclk_iep_noc", "aclk_iep_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_IEP_NOC, "aclk_iep_noc", "aclk_iep_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(16), 1, GFLAGS), /* rga */ @@ -804,21 +892,21 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(4), 9, GFLAGS), GATE(HCLK_RGA, "hclk_rga", "hclk_rga_pre", 0, RK3399_CLKGATE_CON(16), 10, GFLAGS), - GATE(HCLK_RGA_NOC, "hclk_rga_noc", "hclk_rga_pre", CLK_IGNORE_UNUSED, + GATE(HCLK_RGA_NOC, "hclk_rga_noc", "hclk_rga_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(16), 11, GFLAGS), GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, RK3399_CLKGATE_CON(16), 8, GFLAGS), - GATE(ACLK_RGA_NOC, "aclk_rga_noc", "aclk_rga_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_RGA_NOC, "aclk_rga_noc", "aclk_rga_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(16), 9, GFLAGS), /* center */ - COMPOSITE(0, "aclk_center", mux_pll_src_cpll_gpll_npll_p, CLK_IGNORE_UNUSED, + COMPOSITE(ACLK_CENTER, "aclk_center", mux_pll_src_cpll_gpll_npll_p, CLK_IS_CRITICAL, RK3399_CLKSEL_CON(12), 14, 2, MFLAGS, 8, 5, DFLAGS, RK3399_CLKGATE_CON(3), 7, GFLAGS), - GATE(ACLK_CENTER_MAIN_NOC, "aclk_center_main_noc", "aclk_center", CLK_IGNORE_UNUSED, + GATE(ACLK_CENTER_MAIN_NOC, "aclk_center_main_noc", "aclk_center", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(19), 0, GFLAGS), - GATE(ACLK_CENTER_PERI_NOC, "aclk_center_peri_noc", "aclk_center", CLK_IGNORE_UNUSED, + GATE(ACLK_CENTER_PERI_NOC, "aclk_center_peri_noc", "aclk_center", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(19), 1, GFLAGS), /* gpu */ @@ -835,25 +923,25 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(13), 1, GFLAGS), /* perihp */ - GATE(0, "cpll_aclk_perihp_src", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_aclk_perihp_src", "cpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(5), 1, GFLAGS), - GATE(0, "gpll_aclk_perihp_src", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_aclk_perihp_src", "gpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(5), 0, GFLAGS), - COMPOSITE(ACLK_PERIHP, "aclk_perihp", mux_aclk_perihp_p, CLK_IGNORE_UNUSED, + COMPOSITE(ACLK_PERIHP, "aclk_perihp", mux_aclk_perihp_p, CLK_IS_CRITICAL, RK3399_CLKSEL_CON(14), 7, 1, MFLAGS, 0, 5, DFLAGS, RK3399_CLKGATE_CON(5), 2, GFLAGS), - COMPOSITE_NOMUX(HCLK_PERIHP, "hclk_perihp", "aclk_perihp", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(HCLK_PERIHP, "hclk_perihp", "aclk_perihp", CLK_IS_CRITICAL, RK3399_CLKSEL_CON(14), 8, 2, DFLAGS, RK3399_CLKGATE_CON(5), 3, GFLAGS), - COMPOSITE_NOMUX(PCLK_PERIHP, "pclk_perihp", "aclk_perihp", CLK_IGNORE_UNUSED, - RK3399_CLKSEL_CON(14), 12, 2, DFLAGS, + COMPOSITE_NOMUX(PCLK_PERIHP, "pclk_perihp", "aclk_perihp", CLK_IS_CRITICAL, + RK3399_CLKSEL_CON(14), 12, 3, DFLAGS, RK3399_CLKGATE_CON(5), 4, GFLAGS), GATE(ACLK_PERF_PCIE, "aclk_perf_pcie", "aclk_perihp", 0, RK3399_CLKGATE_CON(20), 2, GFLAGS), GATE(ACLK_PCIE, "aclk_pcie", "aclk_perihp", 0, RK3399_CLKGATE_CON(20), 10, GFLAGS), - GATE(0, "aclk_perihp_noc", "aclk_perihp", CLK_IGNORE_UNUSED, + GATE(0, "aclk_perihp_noc", "aclk_perihp", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(20), 12, GFLAGS), GATE(HCLK_HOST0, "hclk_host0", "hclk_perihp", 0, @@ -866,16 +954,16 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(20), 8, GFLAGS), GATE(HCLK_HSIC, "hclk_hsic", "hclk_perihp", 0, RK3399_CLKGATE_CON(20), 9, GFLAGS), - GATE(0, "hclk_perihp_noc", "hclk_perihp", CLK_IGNORE_UNUSED, + GATE(0, "hclk_perihp_noc", "hclk_perihp", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(20), 13, GFLAGS), GATE(0, "hclk_ahb1tom", "hclk_perihp", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(20), 15, GFLAGS), - GATE(PCLK_PERIHP_GRF, "pclk_perihp_grf", "pclk_perihp", CLK_IGNORE_UNUSED, + GATE(PCLK_PERIHP_GRF, "pclk_perihp_grf", "pclk_perihp", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(20), 4, GFLAGS), GATE(PCLK_PCIE, "pclk_pcie", "pclk_perihp", 0, RK3399_CLKGATE_CON(20), 11, GFLAGS), - GATE(0, "pclk_perihp_noc", "pclk_perihp", CLK_IGNORE_UNUSED, + GATE(0, "pclk_perihp_noc", "pclk_perihp", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(20), 14, GFLAGS), GATE(PCLK_HSICPHY, "pclk_hsicphy", "pclk_perihp", 0, RK3399_CLKGATE_CON(31), 8, GFLAGS), @@ -886,7 +974,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(12), 13, GFLAGS), GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_sd", 0, RK3399_CLKGATE_CON(33), 8, GFLAGS), - GATE(0, "hclk_sdmmc_noc", "hclk_sd", CLK_IGNORE_UNUSED, + GATE(0, "hclk_sdmmc_noc", "hclk_sd", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(33), 9, GFLAGS), COMPOSITE(SCLK_SDIO, "clk_sdio", mux_pll_src_cpll_gpll_npll_ppll_upll_24m_p, 0, @@ -933,23 +1021,23 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKSEL_CON(21), 7, 1, MFLAGS, 0, 5, DFLAGS), GATE(ACLK_EMMC_CORE, "aclk_emmccore", "aclk_emmc", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(32), 8, GFLAGS), - GATE(ACLK_EMMC_NOC, "aclk_emmc_noc", "aclk_emmc", CLK_IGNORE_UNUSED, + GATE(ACLK_EMMC_NOC, "aclk_emmc_noc", "aclk_emmc", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(32), 9, GFLAGS), GATE(ACLK_EMMC_GRF, "aclk_emmcgrf", "aclk_emmc", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(32), 10, GFLAGS), /* perilp0 */ - GATE(0, "cpll_aclk_perilp0_src", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_aclk_perilp0_src", "cpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(7), 1, GFLAGS), - GATE(0, "gpll_aclk_perilp0_src", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_aclk_perilp0_src", "gpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(7), 0, GFLAGS), - COMPOSITE(ACLK_PERILP0, "aclk_perilp0", mux_aclk_perilp0_p, CLK_IGNORE_UNUSED, + COMPOSITE(ACLK_PERILP0, "aclk_perilp0", mux_aclk_perilp0_p, CLK_IS_CRITICAL, RK3399_CLKSEL_CON(23), 7, 1, MFLAGS, 0, 5, DFLAGS, RK3399_CLKGATE_CON(7), 2, GFLAGS), - COMPOSITE_NOMUX(HCLK_PERILP0, "hclk_perilp0", "aclk_perilp0", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(HCLK_PERILP0, "hclk_perilp0", "aclk_perilp0", CLK_IS_CRITICAL, RK3399_CLKSEL_CON(23), 8, 2, DFLAGS, RK3399_CLKGATE_CON(7), 3, GFLAGS), - COMPOSITE_NOMUX(PCLK_PERILP0, "pclk_perilp0", "aclk_perilp0", 0, + COMPOSITE_NOMUX(PCLK_PERILP0, "pclk_perilp0", "aclk_perilp0", CLK_IS_CRITICAL, RK3399_CLKSEL_CON(23), 12, 3, DFLAGS, RK3399_CLKGATE_CON(7), 4, GFLAGS), @@ -964,8 +1052,8 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { GATE(SCLK_INTMEM5, "clk_intmem5", "aclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(23), 7, GFLAGS), GATE(ACLK_DCF, "aclk_dcf", "aclk_perilp0", 0, RK3399_CLKGATE_CON(23), 8, GFLAGS), GATE(ACLK_DMAC0_PERILP, "aclk_dmac0_perilp", "aclk_perilp0", 0, RK3399_CLKGATE_CON(25), 5, GFLAGS), - GATE(ACLK_DMAC1_PERILP, "aclk_dmac1_perilp", "aclk_perilp0", 0, RK3399_CLKGATE_CON(25), 6, GFLAGS), - GATE(ACLK_PERILP0_NOC, "aclk_perilp0_noc", "aclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 7, GFLAGS), + GATE(ACLK_DMAC1_PERILP, "aclk_dmac1_perilp", "aclk_perilp0", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 6, GFLAGS), + GATE(ACLK_PERILP0_NOC, "aclk_perilp0_noc", "aclk_perilp0", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 7, GFLAGS), /* hclk_perilp0 gates */ GATE(HCLK_ROM, "hclk_rom", "hclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(24), 4, GFLAGS), @@ -973,7 +1061,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { GATE(HCLK_S_CRYPTO0, "hclk_s_crypto0", "hclk_perilp0", 0, RK3399_CLKGATE_CON(24), 6, GFLAGS), GATE(HCLK_M_CRYPTO1, "hclk_m_crypto1", "hclk_perilp0", 0, RK3399_CLKGATE_CON(24), 14, GFLAGS), GATE(HCLK_S_CRYPTO1, "hclk_s_crypto1", "hclk_perilp0", 0, RK3399_CLKGATE_CON(24), 15, GFLAGS), - GATE(HCLK_PERILP0_NOC, "hclk_perilp0_noc", "hclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 8, GFLAGS), + GATE(HCLK_PERILP0_NOC, "hclk_perilp0_noc", "hclk_perilp0", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 8, GFLAGS), /* pclk_perilp0 gates */ GATE(PCLK_DCF, "pclk_dcf", "pclk_perilp0", 0, RK3399_CLKGATE_CON(23), 9, GFLAGS), @@ -1001,29 +1089,29 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { GATE(HCLK_M0_PERILP, "hclk_m0_perilp", "fclk_cm0s", 0, RK3399_CLKGATE_CON(24), 9, GFLAGS), GATE(DCLK_M0_PERILP, "dclk_m0_perilp", "fclk_cm0s", 0, RK3399_CLKGATE_CON(24), 10, GFLAGS), GATE(SCLK_M0_PERILP_DEC, "clk_m0_perilp_dec", "fclk_cm0s", 0, RK3399_CLKGATE_CON(24), 11, GFLAGS), - GATE(HCLK_M0_PERILP_NOC, "hclk_m0_perilp_noc", "fclk_cm0s", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 11, GFLAGS), + GATE(HCLK_M0_PERILP_NOC, "hclk_m0_perilp_noc", "fclk_cm0s", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 11, GFLAGS), /* perilp1 */ - GATE(0, "cpll_hclk_perilp1_src", "cpll", CLK_IGNORE_UNUSED, + GATE(0, "cpll_hclk_perilp1_src", "cpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(8), 1, GFLAGS), - GATE(0, "gpll_hclk_perilp1_src", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "gpll_hclk_perilp1_src", "gpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(8), 0, GFLAGS), - COMPOSITE_NOGATE(HCLK_PERILP1, "hclk_perilp1", mux_hclk_perilp1_p, CLK_IGNORE_UNUSED, + COMPOSITE_NOGATE(HCLK_PERILP1, "hclk_perilp1", mux_hclk_perilp1_p, CLK_IS_CRITICAL, RK3399_CLKSEL_CON(25), 7, 1, MFLAGS, 0, 5, DFLAGS), - COMPOSITE_NOMUX(PCLK_PERILP1, "pclk_perilp1", "hclk_perilp1", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(PCLK_PERILP1, "pclk_perilp1", "hclk_perilp1", CLK_IS_CRITICAL, RK3399_CLKSEL_CON(25), 8, 3, DFLAGS, RK3399_CLKGATE_CON(8), 2, GFLAGS), /* hclk_perilp1 gates */ - GATE(0, "hclk_perilp1_noc", "hclk_perilp1", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 9, GFLAGS), - GATE(0, "hclk_sdio_noc", "hclk_perilp1", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 12, GFLAGS), + GATE(0, "hclk_perilp1_noc", "hclk_perilp1", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 9, GFLAGS), + GATE(0, "hclk_sdio_noc", "hclk_perilp1", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 12, GFLAGS), GATE(HCLK_I2S0_8CH, "hclk_i2s0", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 0, GFLAGS), GATE(HCLK_I2S1_8CH, "hclk_i2s1", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 1, GFLAGS), GATE(HCLK_I2S2_8CH, "hclk_i2s2", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 2, GFLAGS), GATE(HCLK_SPDIF, "hclk_spdif", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 3, GFLAGS), GATE(HCLK_SDIO, "hclk_sdio", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 4, GFLAGS), GATE(PCLK_SPI5, "pclk_spi5", "hclk_perilp1", 0, RK3399_CLKGATE_CON(34), 5, GFLAGS), - GATE(0, "hclk_sdioaudio_noc", "hclk_perilp1", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(34), 6, GFLAGS), + GATE(0, "hclk_sdioaudio_noc", "hclk_perilp1", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(34), 6, GFLAGS), /* pclk_perilp1 gates */ GATE(PCLK_UART0, "pclk_uart0", "pclk_perilp1", 0, RK3399_CLKGATE_CON(22), 0, GFLAGS), @@ -1046,7 +1134,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { GATE(PCLK_SPI2, "pclk_spi2", "pclk_perilp1", 0, RK3399_CLKGATE_CON(23), 12, GFLAGS), GATE(PCLK_SPI4, "pclk_spi4", "pclk_perilp1", 0, RK3399_CLKGATE_CON(23), 13, GFLAGS), GATE(PCLK_PERIHP_GRF, "pclk_perilp_sgrf", "pclk_perilp1", 0, RK3399_CLKGATE_CON(24), 13, GFLAGS), - GATE(0, "pclk_perilp1_noc", "pclk_perilp1", 0, RK3399_CLKGATE_CON(25), 10, GFLAGS), + GATE(0, "pclk_perilp1_noc", "pclk_perilp1", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(25), 10, GFLAGS), /* saradc */ COMPOSITE_NOMUX(SCLK_SARADC, "clk_saradc", "xin24m", 0, @@ -1075,24 +1163,23 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { COMPOSITE(ACLK_VIO, "aclk_vio", mux_pll_src_cpll_gpll_ppll_p, CLK_IGNORE_UNUSED, RK3399_CLKSEL_CON(42), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3399_CLKGATE_CON(11), 0, GFLAGS), - COMPOSITE_NOMUX(PCLK_VIO, "pclk_vio", "aclk_vio", 0, + COMPOSITE_NOMUX(PCLK_VIO, "pclk_vio", "aclk_vio", CLK_IS_CRITICAL, RK3399_CLKSEL_CON(43), 0, 5, DFLAGS, RK3399_CLKGATE_CON(11), 1, GFLAGS), - GATE(ACLK_VIO_NOC, "aclk_vio_noc", "aclk_vio", CLK_IGNORE_UNUSED, + GATE(ACLK_VIO_NOC, "aclk_vio_noc", "aclk_vio", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(29), 0, GFLAGS), GATE(PCLK_MIPI_DSI0, "pclk_mipi_dsi0", "pclk_vio", 0, RK3399_CLKGATE_CON(29), 1, GFLAGS), GATE(PCLK_MIPI_DSI1, "pclk_mipi_dsi1", "pclk_vio", 0, RK3399_CLKGATE_CON(29), 2, GFLAGS), - GATE(PCLK_VIO_GRF, "pclk_vio_grf", "pclk_vio", CLK_IGNORE_UNUSED, + GATE(PCLK_VIO_GRF, "pclk_vio_grf", "pclk_vio", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(29), 12, GFLAGS), /* hdcp */ - COMPOSITE(ACLK_HDCP, "aclk_hdcp", mux_pll_src_cpll_gpll_ppll_p, 0, - RK3399_CLKSEL_CON(42), 14, 2, MFLAGS, 8, 5, DFLAGS, - RK3399_CLKGATE_CON(11), 12, GFLAGS), + COMPOSITE_NOGATE(ACLK_HDCP, "aclk_hdcp", mux_pll_src_cpll_gpll_ppll_p, 0, + RK3399_CLKSEL_CON(42), 14, 2, MFLAGS, 8, 5, DFLAGS), COMPOSITE_NOMUX(HCLK_HDCP, "hclk_hdcp", "aclk_hdcp", 0, RK3399_CLKSEL_CON(43), 5, 5, DFLAGS, RK3399_CLKGATE_CON(11), 3, GFLAGS), @@ -1100,17 +1187,17 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKSEL_CON(43), 10, 5, DFLAGS, RK3399_CLKGATE_CON(11), 10, GFLAGS), - GATE(ACLK_HDCP_NOC, "aclk_hdcp_noc", "aclk_hdcp", CLK_IGNORE_UNUSED, + GATE(ACLK_HDCP_NOC, "aclk_hdcp_noc", "aclk_hdcp", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(29), 4, GFLAGS), GATE(ACLK_HDCP22, "aclk_hdcp22", "aclk_hdcp", 0, RK3399_CLKGATE_CON(29), 10, GFLAGS), - GATE(HCLK_HDCP_NOC, "hclk_hdcp_noc", "hclk_hdcp", CLK_IGNORE_UNUSED, + GATE(HCLK_HDCP_NOC, "hclk_hdcp_noc", "hclk_hdcp", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(29), 5, GFLAGS), GATE(HCLK_HDCP22, "hclk_hdcp22", "hclk_hdcp", 0, RK3399_CLKGATE_CON(29), 9, GFLAGS), - GATE(PCLK_HDCP_NOC, "pclk_hdcp_noc", "pclk_hdcp", CLK_IGNORE_UNUSED, + GATE(PCLK_HDCP_NOC, "pclk_hdcp_noc", "pclk_hdcp", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(29), 3, GFLAGS), GATE(PCLK_HDMI_CTRL, "pclk_hdmi_ctrl", "pclk_hdcp", 0, RK3399_CLKGATE_CON(29), 6, GFLAGS), @@ -1129,7 +1216,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { COMPOSITE(PCLK_EDP, "pclk_edp", mux_pll_src_cpll_gpll_p, 0, RK3399_CLKSEL_CON(44), 15, 1, MFLAGS, 8, 6, DFLAGS, RK3399_CLKGATE_CON(11), 11, GFLAGS), - GATE(PCLK_EDP_NOC, "pclk_edp_noc", "pclk_edp", CLK_IGNORE_UNUSED, + GATE(PCLK_EDP_NOC, "pclk_edp_noc", "pclk_edp", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(32), 12, GFLAGS), GATE(PCLK_EDP_CTRL, "pclk_edp_ctrl", "pclk_edp", 0, RK3399_CLKGATE_CON(32), 13, GFLAGS), @@ -1143,7 +1230,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(11), 7, GFLAGS), /* vop0 */ - COMPOSITE(ACLK_VOP0_PRE, "aclk_vop0_pre", mux_pll_src_vpll_cpll_gpll_npll_p, 0, + COMPOSITE(ACLK_VOP0_PRE, "aclk_vop0_pre", mux_pll_src_dmyvpll_cpll_gpll_npll_p, 0, RK3399_CLKSEL_CON(47), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3399_CLKGATE_CON(10), 8, GFLAGS), COMPOSITE_NOMUX(0, "hclk_vop0_pre", "aclk_vop0_pre", 0, @@ -1152,28 +1239,35 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { GATE(ACLK_VOP0, "aclk_vop0", "aclk_vop0_pre", 0, RK3399_CLKGATE_CON(28), 3, GFLAGS), - GATE(ACLK_VOP0_NOC, "aclk_vop0_noc", "aclk_vop0_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_VOP0_NOC, "aclk_vop0_noc", "aclk_vop0_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(28), 1, GFLAGS), GATE(HCLK_VOP0, "hclk_vop0", "hclk_vop0_pre", 0, RK3399_CLKGATE_CON(28), 2, GFLAGS), - GATE(HCLK_VOP0_NOC, "hclk_vop0_noc", "hclk_vop0_pre", CLK_IGNORE_UNUSED, + GATE(HCLK_VOP0_NOC, "hclk_vop0_noc", "hclk_vop0_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(28), 0, GFLAGS), - COMPOSITE(DCLK_VOP0_DIV, "dclk_vop0_div", mux_pll_src_vpll_cpll_gpll_p, 0, +#ifdef RK3399_TWO_PLL_FOR_VOP + COMPOSITE(DCLK_VOP0_DIV, "dclk_vop0_div", mux_pll_src_vpll_cpll_gpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK3399_CLKSEL_CON(49), 8, 2, MFLAGS, 0, 8, DFLAGS, + RK3399_CLKGATE_CON(10), 12, GFLAGS), +#else + COMPOSITE(DCLK_VOP0_DIV, "dclk_vop0_div", mux_pll_src_vpll_cpll_gpll_p, CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(49), 8, 2, MFLAGS, 0, 8, DFLAGS, RK3399_CLKGATE_CON(10), 12, GFLAGS), +#endif - COMPOSITE_FRACMUX_NOGATE(DCLK_VOP0_FRAC, "dclk_vop0_frac", "dclk_vop0_div", 0, + /* The VOP0 is main screen, it is able to re-set parent rate. */ + COMPOSITE_FRACMUX_NOGATE(0, "dclk_vop0_frac", "dclk_vop0_div", CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(106), 0, - &rk3399_dclk_vop0_fracmux), + &rk3399_dclk_vop0_fracmux, RK3399_VOP_FRAC_MAX_PRATE), - COMPOSITE(SCLK_VOP0_PWM, "clk_vop0_pwm", mux_pll_src_vpll_cpll_gpll_24m_p, 0, + COMPOSITE(SCLK_VOP0_PWM, "clk_vop0_pwm", mux_pll_src_dmyvpll_cpll_gpll_gpll_p, 0, RK3399_CLKSEL_CON(51), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3399_CLKGATE_CON(10), 14, GFLAGS), /* vop1 */ - COMPOSITE(ACLK_VOP1_PRE, "aclk_vop1_pre", mux_pll_src_vpll_cpll_gpll_npll_p, 0, + COMPOSITE(ACLK_VOP1_PRE, "aclk_vop1_pre", mux_pll_src_dmyvpll_cpll_gpll_npll_p, 0, RK3399_CLKSEL_CON(48), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3399_CLKGATE_CON(10), 10, GFLAGS), COMPOSITE_NOMUX(0, "hclk_vop1_pre", "aclk_vop1_pre", 0, @@ -1182,23 +1276,30 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { GATE(ACLK_VOP1, "aclk_vop1", "aclk_vop1_pre", 0, RK3399_CLKGATE_CON(28), 7, GFLAGS), - GATE(ACLK_VOP1_NOC, "aclk_vop1_noc", "aclk_vop1_pre", CLK_IGNORE_UNUSED, + GATE(ACLK_VOP1_NOC, "aclk_vop1_noc", "aclk_vop1_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(28), 5, GFLAGS), GATE(HCLK_VOP1, "hclk_vop1", "hclk_vop1_pre", 0, RK3399_CLKGATE_CON(28), 6, GFLAGS), - GATE(HCLK_VOP1_NOC, "hclk_vop1_noc", "hclk_vop1_pre", CLK_IGNORE_UNUSED, + GATE(HCLK_VOP1_NOC, "hclk_vop1_noc", "hclk_vop1_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(28), 4, GFLAGS), - COMPOSITE(DCLK_VOP1_DIV, "dclk_vop1_div", mux_pll_src_vpll_cpll_gpll_p, 0, + /* The VOP1 is sub screen, it is note able to re-set parent rate. */ +#ifdef RK3399_TWO_PLL_FOR_VOP + COMPOSITE(DCLK_VOP1_DIV, "dclk_vop1_div", mux_pll_src_vpll_cpll_gpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, RK3399_CLKSEL_CON(50), 8, 2, MFLAGS, 0, 8, DFLAGS, RK3399_CLKGATE_CON(10), 13, GFLAGS), +#else + COMPOSITE(DCLK_VOP1_DIV, "dclk_vop1_div", mux_pll_src_dmyvpll_cpll_gpll_p, 0, + RK3399_CLKSEL_CON(50), 8, 2, MFLAGS, 0, 8, DFLAGS, + RK3399_CLKGATE_CON(10), 13, GFLAGS), +#endif COMPOSITE_FRACMUX_NOGATE(DCLK_VOP1_FRAC, "dclk_vop1_frac", "dclk_vop1_div", 0, RK3399_CLKSEL_CON(107), 0, - &rk3399_dclk_vop1_fracmux), + &rk3399_dclk_vop1_fracmux, RK3399_VOP_FRAC_MAX_PRATE), - COMPOSITE(SCLK_VOP1_PWM, "clk_vop1_pwm", mux_pll_src_vpll_cpll_gpll_24m_p, CLK_IGNORE_UNUSED, + COMPOSITE(SCLK_VOP1_PWM, "clk_vop1_pwm", mux_pll_src_dmyvpll_cpll_gpll_gpll_p, 0, RK3399_CLKSEL_CON(52), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3399_CLKGATE_CON(10), 15, GFLAGS), @@ -1210,14 +1311,12 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKSEL_CON(53), 8, 5, DFLAGS, RK3399_CLKGATE_CON(12), 9, GFLAGS), - GATE(ACLK_ISP0_NOC, "aclk_isp0_noc", "aclk_isp0", CLK_IGNORE_UNUSED, + GATE(ACLK_ISP0_NOC, "aclk_isp0_noc", "aclk_isp0", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(27), 1, GFLAGS), GATE(ACLK_ISP0_WRAPPER, "aclk_isp0_wrapper", "aclk_isp0", 0, RK3399_CLKGATE_CON(27), 5, GFLAGS), - GATE(HCLK_ISP1_WRAPPER, "hclk_isp1_wrapper", "aclk_isp0", 0, - RK3399_CLKGATE_CON(27), 7, GFLAGS), - GATE(HCLK_ISP0_NOC, "hclk_isp0_noc", "hclk_isp0", CLK_IGNORE_UNUSED, + GATE(HCLK_ISP0_NOC, "hclk_isp0_noc", "hclk_isp0", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(27), 0, GFLAGS), GATE(HCLK_ISP0_WRAPPER, "hclk_isp0_wrapper", "hclk_isp0", 0, RK3399_CLKGATE_CON(27), 4, GFLAGS), @@ -1233,13 +1332,15 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKSEL_CON(54), 8, 5, DFLAGS, RK3399_CLKGATE_CON(12), 11, GFLAGS), - GATE(ACLK_ISP1_NOC, "aclk_isp1_noc", "aclk_isp1", CLK_IGNORE_UNUSED, + GATE(ACLK_ISP1_NOC, "aclk_isp1_noc", "aclk_isp1", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(27), 3, GFLAGS), + GATE(ACLK_ISP1_WRAPPER, "aclk_isp1_wrapper", "aclk_isp1", 0, + RK3399_CLKGATE_CON(27), 8, GFLAGS), - GATE(HCLK_ISP1_NOC, "hclk_isp1_noc", "hclk_isp1", CLK_IGNORE_UNUSED, + GATE(HCLK_ISP1_NOC, "hclk_isp1_noc", "hclk_isp1", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(27), 2, GFLAGS), - GATE(ACLK_ISP1_WRAPPER, "aclk_isp1_wrapper", "hclk_isp1", 0, - RK3399_CLKGATE_CON(27), 8, GFLAGS), + GATE(HCLK_ISP1_WRAPPER, "hclk_isp1_wrapper", "hclk_isp1", 0, + RK3399_CLKGATE_CON(27), 7, GFLAGS), COMPOSITE(SCLK_ISP1, "clk_isp1", mux_pll_src_cpll_gpll_npll_p, 0, RK3399_CLKSEL_CON(55), 14, 2, MFLAGS, 8, 5, DFLAGS, @@ -1257,7 +1358,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(27), 6, GFLAGS), /* cif */ - COMPOSITE_NODIV(0, "clk_cifout_src", mux_pll_src_cpll_gpll_npll_p, 0, + COMPOSITE_NODIV(SCLK_CIF_OUT_SRC, "clk_cifout_src", mux_pll_src_cpll_gpll_npll_p, 0, RK3399_CLKSEL_CON(56), 6, 2, MFLAGS, RK3399_CLKGATE_CON(10), 7, GFLAGS), @@ -1265,12 +1366,12 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKSEL_CON(56), 5, 1, MFLAGS, 0, 5, DFLAGS), /* gic */ - COMPOSITE(ACLK_GIC_PRE, "aclk_gic_pre", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED, + COMPOSITE(ACLK_GIC_PRE, "aclk_gic_pre", mux_pll_src_cpll_gpll_p, CLK_IS_CRITICAL, RK3399_CLKSEL_CON(56), 15, 1, MFLAGS, 8, 5, DFLAGS, RK3399_CLKGATE_CON(12), 12, GFLAGS), - GATE(ACLK_GIC, "aclk_gic", "aclk_gic_pre", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(33), 0, GFLAGS), - GATE(ACLK_GIC_NOC, "aclk_gic_noc", "aclk_gic_pre", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(33), 1, GFLAGS), + GATE(ACLK_GIC, "aclk_gic", "aclk_gic_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(33), 0, GFLAGS), + GATE(ACLK_GIC_NOC, "aclk_gic_noc", "aclk_gic_pre", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(33), 1, GFLAGS), GATE(ACLK_GIC_ADB400_CORE_L_2_GIC, "aclk_gic_adb400_core_l_2_gic", "aclk_gic_pre", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(33), 2, GFLAGS), GATE(ACLK_GIC_ADB400_CORE_B_2_GIC, "aclk_gic_adb400_core_b_2_gic", "aclk_gic_pre", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(33), 3, GFLAGS), GATE(ACLK_GIC_ADB400_GIC_2_CORE_L, "aclk_gic_adb400_gic_2_core_l", "aclk_gic_pre", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(33), 4, GFLAGS), @@ -1301,19 +1402,19 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { SGRF_GATE(PCLK_WDT, "pclk_wdt", "pclk_alive"), GATE(SCLK_MIPIDPHY_REF, "clk_mipidphy_ref", "xin24m", 0, RK3399_CLKGATE_CON(11), 14, GFLAGS), - GATE(SCLK_DPHY_PLL, "clk_dphy_pll", "clk_mipidphy_ref", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(21), 0, GFLAGS), + GATE(SCLK_DPHY_PLL, "clk_dphy_pll", "clk_mipidphy_ref", 0, RK3399_CLKGATE_CON(21), 0, GFLAGS), GATE(SCLK_MIPIDPHY_CFG, "clk_mipidphy_cfg", "xin24m", 0, RK3399_CLKGATE_CON(11), 15, GFLAGS), - GATE(SCLK_DPHY_TX0_CFG, "clk_dphy_tx0_cfg", "clk_mipidphy_cfg", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(21), 1, GFLAGS), - GATE(SCLK_DPHY_TX1RX1_CFG, "clk_dphy_tx1rx1_cfg", "clk_mipidphy_cfg", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(21), 2, GFLAGS), - GATE(SCLK_DPHY_RX0_CFG, "clk_dphy_rx0_cfg", "clk_mipidphy_cfg", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(21), 3, GFLAGS), + GATE(SCLK_DPHY_TX0_CFG, "clk_dphy_tx0_cfg", "clk_mipidphy_cfg", 0, RK3399_CLKGATE_CON(21), 1, GFLAGS), + GATE(SCLK_DPHY_TX1RX1_CFG, "clk_dphy_tx1rx1_cfg", "clk_mipidphy_cfg", 0, RK3399_CLKGATE_CON(21), 2, GFLAGS), + GATE(SCLK_DPHY_RX0_CFG, "clk_dphy_rx0_cfg", "clk_mipidphy_cfg", 0, RK3399_CLKGATE_CON(21), 3, GFLAGS), /* testout */ MUX(0, "clk_test_pre", mux_pll_src_cpll_gpll_p, CLK_SET_RATE_PARENT, RK3399_CLKSEL_CON(58), 7, 1, MFLAGS), COMPOSITE_FRAC(0, "clk_test_frac", "clk_test_pre", 0, RK3399_CLKSEL_CON(105), 0, - RK3399_CLKGATE_CON(13), 9, GFLAGS), + RK3399_CLKGATE_CON(13), 9, GFLAGS, 0), DIV(0, "clk_test_24m", "xin24m", 0, RK3399_CLKSEL_CON(57), 6, 10, DFLAGS), @@ -1385,13 +1486,13 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(13), 11, GFLAGS), /* ddrc */ - GATE(0, "clk_ddrc_lpll_src", "lpll", 0, RK3399_CLKGATE_CON(3), + GATE(0, "clk_ddrc_lpll_src", "lpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(3), 0, GFLAGS), - GATE(0, "clk_ddrc_bpll_src", "bpll", 0, RK3399_CLKGATE_CON(3), + GATE(0, "clk_ddrc_bpll_src", "bpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(3), 1, GFLAGS), - GATE(0, "clk_ddrc_dpll_src", "dpll", 0, RK3399_CLKGATE_CON(3), + GATE(0, "clk_ddrc_dpll_src", "dpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(3), 2, GFLAGS), - GATE(0, "clk_ddrc_gpll_src", "gpll", 0, RK3399_CLKGATE_CON(3), + GATE(0, "clk_ddrc_gpll_src", "gpll", CLK_IS_CRITICAL, RK3399_CLKGATE_CON(3), 3, GFLAGS), COMPOSITE_DDRCLK(SCLK_DDRC, "sclk_ddrc", mux_ddrclk_p, 0, RK3399_CLKSEL_CON(6), 4, 2, 0, 0, ROCKCHIP_DDRCLK_SIP), @@ -1402,10 +1503,10 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = { * PMU CRU Clock-Architecture */ - GATE(0, "fclk_cm0s_pmu_ppll_src", "ppll", 0, + GATE(0, "fclk_cm0s_pmu_ppll_src", "ppll", CLK_IS_CRITICAL, RK3399_PMU_CLKGATE_CON(0), 1, GFLAGS), - COMPOSITE_NOGATE(FCLK_CM0S_SRC_PMU, "fclk_cm0s_src_pmu", mux_fclk_cm0s_pmu_ppll_p, 0, + COMPOSITE_NOGATE(FCLK_CM0S_SRC_PMU, "fclk_cm0s_src_pmu", mux_fclk_cm0s_pmu_ppll_p, CLK_IS_CRITICAL, RK3399_PMU_CLKSEL_CON(0), 15, 1, MFLAGS, 8, 5, DFLAGS), COMPOSITE(SCLK_SPI3_PMU, "clk_spi3_pmu", mux_24m_ppll_p, 0, @@ -1416,9 +1517,9 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = { RK3399_PMU_CLKSEL_CON(1), 13, 1, MFLAGS, 8, 5, DFLAGS, RK3399_PMU_CLKGATE_CON(0), 8, GFLAGS), - COMPOSITE_FRACMUX_NOGATE(0, "clk_wifi_frac", "clk_wifi_div", 0, + COMPOSITE_FRACMUX_NOGATE(0, "clk_wifi_frac", "clk_wifi_div", CLK_SET_RATE_PARENT, RK3399_PMU_CLKSEL_CON(7), 0, - &rk3399_pmuclk_wifi_fracmux), + &rk3399_pmuclk_wifi_fracmux, RK3399_WIFI_FRAC_MAX_PRATE), MUX(0, "clk_timer_src_pmu", mux_pll_p, CLK_IGNORE_UNUSED, RK3399_PMU_CLKSEL_CON(1), 15, 1, MFLAGS), @@ -1440,23 +1541,26 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = { MUX(0, "clk_testout_2io", mux_clk_testout2_2io_p, CLK_IGNORE_UNUSED, RK3399_PMU_CLKSEL_CON(4), 15, 1, MFLAGS), - COMPOSITE(0, "clk_uart4_div", mux_24m_ppll_p, 0, - RK3399_PMU_CLKSEL_CON(5), 10, 1, MFLAGS, 0, 7, DFLAGS, + MUX(SCLK_UART4_SRC, "clk_uart4_src", mux_24m_ppll_p, CLK_SET_RATE_NO_REPARENT, + RK3399_PMU_CLKSEL_CON(5), 10, 1, MFLAGS), + + COMPOSITE_NOMUX(0, "clk_uart4_div", "clk_uart4_src", CLK_SET_RATE_PARENT, + RK3399_PMU_CLKSEL_CON(5), 0, 7, DFLAGS, RK3399_PMU_CLKGATE_CON(0), 5, GFLAGS), - COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_div", 0, + COMPOSITE_FRACMUX(0, "clk_uart4_frac", "clk_uart4_div", CLK_SET_RATE_PARENT, RK3399_PMU_CLKSEL_CON(6), 0, RK3399_PMU_CLKGATE_CON(0), 6, GFLAGS, - &rk3399_uart4_pmu_fracmux), + &rk3399_uart4_pmu_fracmux, RK3399_UART_FRAC_MAX_PRATE), - DIV(PCLK_SRC_PMU, "pclk_pmu_src", "ppll", CLK_IGNORE_UNUSED, + DIV(PCLK_SRC_PMU, "pclk_pmu_src", "ppll", CLK_IS_CRITICAL, RK3399_PMU_CLKSEL_CON(0), 0, 5, DFLAGS), /* pmu clock gates */ GATE(SCLK_TIMER12_PMU, "clk_timer0_pmu", "clk_timer_src_pmu", 0, RK3399_PMU_CLKGATE_CON(0), 3, GFLAGS), GATE(SCLK_TIMER13_PMU, "clk_timer1_pmu", "clk_timer_src_pmu", 0, RK3399_PMU_CLKGATE_CON(0), 4, GFLAGS), - GATE(SCLK_PVTM_PMU, "clk_pvtm_pmu", "xin24m", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(0), 7, GFLAGS), + GATE(SCLK_PVTM_PMU, "clk_pvtm_pmu", "xin24m", 0, RK3399_PMU_CLKGATE_CON(0), 7, GFLAGS), GATE(PCLK_PMU, "pclk_pmu", "pclk_pmu_src", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(1), 0, GFLAGS), GATE(PCLK_PMUGRF_PMU, "pclk_pmugrf_pmu", "pclk_pmu_src", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(1), 1, GFLAGS), @@ -1464,69 +1568,60 @@ static struct rockchip_clk_branch rk3399_clk_pmu_branches[] __initdata = { GATE(PCLK_GPIO0_PMU, "pclk_gpio0_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 3, GFLAGS), GATE(PCLK_GPIO1_PMU, "pclk_gpio1_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 4, GFLAGS), GATE(PCLK_SGRF_PMU, "pclk_sgrf_pmu", "pclk_pmu_src", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(1), 5, GFLAGS), - GATE(PCLK_NOC_PMU, "pclk_noc_pmu", "pclk_pmu_src", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(1), 6, GFLAGS), + GATE(PCLK_NOC_PMU, "pclk_noc_pmu", "pclk_pmu_src", CLK_IS_CRITICAL, RK3399_PMU_CLKGATE_CON(1), 6, GFLAGS), GATE(PCLK_I2C0_PMU, "pclk_i2c0_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 7, GFLAGS), GATE(PCLK_I2C4_PMU, "pclk_i2c4_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 8, GFLAGS), GATE(PCLK_I2C8_PMU, "pclk_i2c8_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 9, GFLAGS), - GATE(PCLK_RKPWM_PMU, "pclk_rkpwm_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 10, GFLAGS), + GATE(PCLK_RKPWM_PMU, "pclk_rkpwm_pmu", "pclk_pmu_src", CLK_IS_CRITICAL, RK3399_PMU_CLKGATE_CON(1), 10, GFLAGS), GATE(PCLK_SPI3_PMU, "pclk_spi3_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 11, GFLAGS), GATE(PCLK_TIMER_PMU, "pclk_timer_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 12, GFLAGS), GATE(PCLK_MAILBOX_PMU, "pclk_mailbox_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 13, GFLAGS), GATE(PCLK_UART4_PMU, "pclk_uart4_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 14, GFLAGS), GATE(PCLK_WDT_M0_PMU, "pclk_wdt_m0_pmu", "pclk_pmu_src", 0, RK3399_PMU_CLKGATE_CON(1), 15, GFLAGS), - GATE(FCLK_CM0S_PMU, "fclk_cm0s_pmu", "fclk_cm0s_src_pmu", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(2), 0, GFLAGS), - GATE(SCLK_CM0S_PMU, "sclk_cm0s_pmu", "fclk_cm0s_src_pmu", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(2), 1, GFLAGS), - GATE(HCLK_CM0S_PMU, "hclk_cm0s_pmu", "fclk_cm0s_src_pmu", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(2), 2, GFLAGS), - GATE(DCLK_CM0S_PMU, "dclk_cm0s_pmu", "fclk_cm0s_src_pmu", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(2), 3, GFLAGS), - GATE(HCLK_NOC_PMU, "hclk_noc_pmu", "fclk_cm0s_src_pmu", CLK_IGNORE_UNUSED, RK3399_PMU_CLKGATE_CON(2), 5, GFLAGS), + GATE(FCLK_CM0S_PMU, "fclk_cm0s_pmu", "fclk_cm0s_src_pmu", 0, RK3399_PMU_CLKGATE_CON(2), 0, GFLAGS), + GATE(SCLK_CM0S_PMU, "sclk_cm0s_pmu", "fclk_cm0s_src_pmu", 0, RK3399_PMU_CLKGATE_CON(2), 1, GFLAGS), + GATE(HCLK_CM0S_PMU, "hclk_cm0s_pmu", "fclk_cm0s_src_pmu", 0, RK3399_PMU_CLKGATE_CON(2), 2, GFLAGS), + GATE(DCLK_CM0S_PMU, "dclk_cm0s_pmu", "fclk_cm0s_src_pmu", 0, RK3399_PMU_CLKGATE_CON(2), 3, GFLAGS), + GATE(HCLK_NOC_PMU, "hclk_noc_pmu", "fclk_cm0s_src_pmu", CLK_IS_CRITICAL, RK3399_PMU_CLKGATE_CON(2), 5, GFLAGS), }; -static const char *const rk3399_cru_critical_clocks[] __initconst = { - "aclk_cci_pre", - "aclk_gic", - "aclk_gic_noc", - "aclk_hdcp_noc", - "hclk_hdcp_noc", - "pclk_hdcp_noc", - "pclk_perilp0", - "pclk_perilp0", - "hclk_perilp0", - "hclk_perilp0_noc", - "pclk_perilp1", - "pclk_perilp1_noc", - "pclk_perihp", - "pclk_perihp_noc", - "hclk_perihp", - "aclk_perihp", - "aclk_perihp_noc", - "aclk_perilp0", - "aclk_perilp0_noc", - "hclk_perilp1", - "hclk_perilp1_noc", - "aclk_dmac0_perilp", - "aclk_emmc_noc", - "gpll_hclk_perilp1_src", - "gpll_aclk_perilp0_src", - "gpll_aclk_perihp_src", - "aclk_vio_noc", +static void __iomem *rk3399_cru_base; +static void __iomem *rk3399_pmucru_base; - /* ddrc */ - "sclk_ddrc" -}; +void rk3399_dump_cru(void) +{ + if (rk3399_cru_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk3399_cru_base, + 0x594, false); + } + if (rk3399_pmucru_base) { + pr_warn("PMU CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk3399_pmucru_base, + 0x134, false); + } +} +EXPORT_SYMBOL_GPL(rk3399_dump_cru); + +static int rk3399_clk_panic(struct notifier_block *this, + unsigned long ev, void *ptr) +{ + rk3399_dump_cru(); + return NOTIFY_DONE; +} -static const char *const rk3399_pmucru_critical_clocks[] __initconst = { - "ppll", - "pclk_pmu_src", - "fclk_cm0s_src_pmu", - "clk_timer_src_pmu", - "pclk_rkpwm_pmu", +static struct notifier_block rk3399_clk_panic_block = { + .notifier_call = rk3399_clk_panic, }; static void __init rk3399_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; + struct clk **clks; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -1534,12 +1629,15 @@ static void __init rk3399_clk_init(struct device_node *np) return; } + rk3399_cru_base = reg_base; + ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); if (IS_ERR(ctx)) { pr_err("%s: rockchip clk init failed\n", __func__); iounmap(reg_base); return; } + clks = ctx->clk_data.clks; rockchip_clk_register_plls(ctx, rk3399_pll_clks, ARRAY_SIZE(rk3399_pll_clks), -1); @@ -1547,16 +1645,13 @@ static void __init rk3399_clk_init(struct device_node *np) rockchip_clk_register_branches(ctx, rk3399_clk_branches, ARRAY_SIZE(rk3399_clk_branches)); - rockchip_clk_protect_critical(rk3399_cru_critical_clocks, - ARRAY_SIZE(rk3399_cru_critical_clocks)); - rockchip_clk_register_armclk(ctx, ARMCLKL, "armclkl", - mux_armclkl_p, ARRAY_SIZE(mux_armclkl_p), + 4, clks[PLL_APLLL], clks[PLL_GPLL], &rk3399_cpuclkl_data, rk3399_cpuclkl_rates, ARRAY_SIZE(rk3399_cpuclkl_rates)); rockchip_clk_register_armclk(ctx, ARMCLKB, "armclkb", - mux_armclkb_p, ARRAY_SIZE(mux_armclkb_p), + 4, clks[PLL_APLLB], clks[PLL_GPLL], &rk3399_cpuclkb_data, rk3399_cpuclkb_rates, ARRAY_SIZE(rk3399_cpuclkb_rates)); @@ -1580,6 +1675,8 @@ static void __init rk3399_pmu_clk_init(struct device_node *np) return; } + rk3399_pmucru_base = reg_base; + ctx = rockchip_clk_init(np, reg_base, CLKPMU_NR_CLKS); if (IS_ERR(ctx)) { pr_err("%s: rockchip pmu clk init failed\n", __func__); @@ -1593,13 +1690,13 @@ static void __init rk3399_pmu_clk_init(struct device_node *np) rockchip_clk_register_branches(ctx, rk3399_clk_pmu_branches, ARRAY_SIZE(rk3399_clk_pmu_branches)); - rockchip_clk_protect_critical(rk3399_pmucru_critical_clocks, - ARRAY_SIZE(rk3399_pmucru_critical_clocks)); - rockchip_register_softrst(np, 2, reg_base + RK3399_PMU_SOFTRST_CON(0), ROCKCHIP_SOFTRST_HIWORD_MASK); rockchip_clk_of_add_provider(np, ctx); + + atomic_notifier_chain_register(&panic_notifier_list, + &rk3399_clk_panic_block); } CLK_OF_DECLARE(rk3399_cru_pmu, "rockchip,rk3399-pmucru", rk3399_pmu_clk_init); diff --git a/drivers/clk/rockchip/clk-rk3568.c b/drivers/clk/rockchip/clk-rk3568.c new file mode 100755 index 000000000000..44ce9cc542ce --- /dev/null +++ b/drivers/clk/rockchip/clk-rk3568.c @@ -0,0 +1,1757 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 Rockchip Electronics Co. Ltd. + * Author: Elaine Zhang + */ + +#include +#include +#include +#include +#include +#include +#include +#include "clk.h" + +#define RK3568_GRF_SOC_CON1 0x504 +#define RK3568_GRF_SOC_CON2 0x508 +#define RK3568_GRF_SOC_STATUS0 0x580 +#define RK3568_PMU_GRF_SOC_CON0 0x100 + +#define RK3568_FRAC_MAX_PRATE 1000000000 +#define RK3568_SPDIF_FRAC_MAX_PRATE 600000000 +#define RK3568_UART_FRAC_MAX_PRATE 600000000 +#define RK3568_DCLK_PARENT_MAX_PRATE 600000000 + +enum rk3568_pmu_plls { + ppll, hpll, +}; + +enum rk3568_plls { + apll, dpll, gpll, cpll, npll, vpll, +}; + +static struct rockchip_pll_rate_table rk3568_pll_rates[] = { + /* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */ + RK3036_PLL_RATE(2208000000, 1, 92, 1, 1, 1, 0), + RK3036_PLL_RATE(2184000000, 1, 91, 1, 1, 1, 0), + RK3036_PLL_RATE(2160000000, 1, 90, 1, 1, 1, 0), + RK3036_PLL_RATE(2088000000, 1, 87, 1, 1, 1, 0), + RK3036_PLL_RATE(2064000000, 1, 86, 1, 1, 1, 0), + RK3036_PLL_RATE(2040000000, 1, 85, 1, 1, 1, 0), + RK3036_PLL_RATE(2016000000, 1, 84, 1, 1, 1, 0), + RK3036_PLL_RATE(1992000000, 1, 83, 1, 1, 1, 0), + RK3036_PLL_RATE(1920000000, 1, 80, 1, 1, 1, 0), + RK3036_PLL_RATE(1896000000, 1, 79, 1, 1, 1, 0), + RK3036_PLL_RATE(1800000000, 1, 75, 1, 1, 1, 0), + RK3036_PLL_RATE(1704000000, 1, 71, 1, 1, 1, 0), + RK3036_PLL_RATE(1608000000, 1, 67, 1, 1, 1, 0), + RK3036_PLL_RATE(1600000000, 3, 200, 1, 1, 1, 0), + RK3036_PLL_RATE(1584000000, 1, 132, 2, 1, 1, 0), + RK3036_PLL_RATE(1560000000, 1, 130, 2, 1, 1, 0), + RK3036_PLL_RATE(1536000000, 1, 128, 2, 1, 1, 0), + RK3036_PLL_RATE(1512000000, 1, 126, 2, 1, 1, 0), + RK3036_PLL_RATE(1488000000, 1, 124, 2, 1, 1, 0), + RK3036_PLL_RATE(1464000000, 1, 122, 2, 1, 1, 0), + RK3036_PLL_RATE(1440000000, 1, 120, 2, 1, 1, 0), + RK3036_PLL_RATE(1416000000, 1, 118, 2, 1, 1, 0), + RK3036_PLL_RATE(1400000000, 3, 350, 2, 1, 1, 0), + RK3036_PLL_RATE(1392000000, 1, 116, 2, 1, 1, 0), + RK3036_PLL_RATE(1368000000, 1, 114, 2, 1, 1, 0), + RK3036_PLL_RATE(1344000000, 1, 112, 2, 1, 1, 0), + RK3036_PLL_RATE(1320000000, 1, 110, 2, 1, 1, 0), + RK3036_PLL_RATE(1296000000, 1, 108, 2, 1, 1, 0), + RK3036_PLL_RATE(1272000000, 1, 106, 2, 1, 1, 0), + RK3036_PLL_RATE(1248000000, 1, 104, 2, 1, 1, 0), + RK3036_PLL_RATE(1200000000, 1, 100, 2, 1, 1, 0), + RK3036_PLL_RATE(1188000000, 1, 99, 2, 1, 1, 0), + RK3036_PLL_RATE(1104000000, 1, 92, 2, 1, 1, 0), + RK3036_PLL_RATE(1100000000, 3, 275, 2, 1, 1, 0), + RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0), + RK3036_PLL_RATE(1000000000, 3, 250, 2, 1, 1, 0), + RK3036_PLL_RATE(912000000, 1, 76, 2, 1, 1, 0), + RK3036_PLL_RATE(816000000, 1, 68, 2, 1, 1, 0), + RK3036_PLL_RATE(800000000, 3, 200, 2, 1, 1, 0), + RK3036_PLL_RATE(700000000, 3, 350, 4, 1, 1, 0), + RK3036_PLL_RATE(696000000, 1, 116, 4, 1, 1, 0), + RK3036_PLL_RATE(600000000, 1, 100, 4, 1, 1, 0), + RK3036_PLL_RATE(594000000, 1, 99, 4, 1, 1, 0), + RK3036_PLL_RATE(500000000, 1, 125, 6, 1, 1, 0), + RK3036_PLL_RATE(408000000, 1, 68, 2, 2, 1, 0), + RK3036_PLL_RATE(312000000, 1, 78, 6, 1, 1, 0), + RK3036_PLL_RATE(216000000, 1, 72, 4, 2, 1, 0), + RK3036_PLL_RATE(200000000, 1, 100, 3, 4, 1, 0), + RK3036_PLL_RATE(148500000, 1, 99, 4, 4, 1, 0), + RK3036_PLL_RATE(100000000, 1, 150, 6, 6, 1, 0), + RK3036_PLL_RATE(96000000, 1, 96, 6, 4, 1, 0), + RK3036_PLL_RATE(74250000, 2, 99, 4, 4, 1, 0), + { /* sentinel */ }, +}; + +#define RK3568_DIV_ATCLK_CORE_MASK 0x1f +#define RK3568_DIV_ATCLK_CORE_SHIFT 0 +#define RK3568_DIV_GICCLK_CORE_MASK 0x1f +#define RK3568_DIV_GICCLK_CORE_SHIFT 8 +#define RK3568_DIV_PCLK_CORE_MASK 0x1f +#define RK3568_DIV_PCLK_CORE_SHIFT 0 +#define RK3568_DIV_PERIPHCLK_CORE_MASK 0x1f +#define RK3568_DIV_PERIPHCLK_CORE_SHIFT 8 +#define RK3568_DIV_ACLK_CORE_MASK 0x1f +#define RK3568_DIV_ACLK_CORE_SHIFT 8 + +#define RK3568_DIV_SCLK_CORE_MASK 0xf +#define RK3568_DIV_SCLK_CORE_SHIFT 0 +#define RK3568_MUX_SCLK_CORE_MASK 0x3 +#define RK3568_MUX_SCLK_CORE_SHIFT 8 +#define RK3568_MUX_SCLK_CORE_NPLL_MASK 0x1 +#define RK3568_MUX_SCLK_CORE_NPLL_SHIFT 15 +#define RK3568_MUX_CLK_CORE_APLL_MASK 0x1 +#define RK3568_MUX_CLK_CORE_APLL_SHIFT 7 +#define RK3568_MUX_CLK_PVTPLL_MASK 0x1 +#define RK3568_MUX_CLK_PVTPLL_SHIFT 15 + +#define RK3568_CLKSEL1(_sclk_core) \ +{ \ + .reg = RK3568_CLKSEL_CON(2), \ + .val = HIWORD_UPDATE(_sclk_core, RK3568_MUX_SCLK_CORE_NPLL_MASK, \ + RK3568_MUX_SCLK_CORE_NPLL_SHIFT) | \ + HIWORD_UPDATE(_sclk_core, RK3568_MUX_SCLK_CORE_MASK, \ + RK3568_MUX_SCLK_CORE_SHIFT) | \ + HIWORD_UPDATE(1, RK3568_DIV_SCLK_CORE_MASK, \ + RK3568_DIV_SCLK_CORE_SHIFT), \ +} + +#define RK3568_CLKSEL2(_aclk_core) \ +{ \ + .reg = RK3568_CLKSEL_CON(5), \ + .val = HIWORD_UPDATE(_aclk_core, RK3568_DIV_ACLK_CORE_MASK, \ + RK3568_DIV_ACLK_CORE_SHIFT), \ +} + +#define RK3568_CLKSEL3(_atclk_core, _gic_core) \ +{ \ + .reg = RK3568_CLKSEL_CON(3), \ + .val = HIWORD_UPDATE(_atclk_core, RK3568_DIV_ATCLK_CORE_MASK, \ + RK3568_DIV_ATCLK_CORE_SHIFT) | \ + HIWORD_UPDATE(_gic_core, RK3568_DIV_GICCLK_CORE_MASK, \ + RK3568_DIV_GICCLK_CORE_SHIFT), \ +} + +#define RK3568_CLKSEL4(_pclk_core, _periph_core) \ +{ \ + .reg = RK3568_CLKSEL_CON(4), \ + .val = HIWORD_UPDATE(_pclk_core, RK3568_DIV_PCLK_CORE_MASK, \ + RK3568_DIV_PCLK_CORE_SHIFT) | \ + HIWORD_UPDATE(_periph_core, RK3568_DIV_PERIPHCLK_CORE_MASK, \ + RK3568_DIV_PERIPHCLK_CORE_SHIFT), \ +} + +#define RK3568_CPUCLK_RATE(_prate, _sclk, _acore, _atcore, _gicclk, _pclk, _periph) \ +{ \ + .prate = _prate##U, \ + .divs = { \ + RK3568_CLKSEL1(_sclk), \ + RK3568_CLKSEL2(_acore), \ + RK3568_CLKSEL3(_atcore, _gicclk), \ + RK3568_CLKSEL4(_pclk, _periph), \ + }, \ +} + +static struct rockchip_cpuclk_rate_table rk3568_cpuclk_rates[] __initdata = { + RK3568_CPUCLK_RATE(1800000000, 0, 1, 7, 7, 7, 7), + RK3568_CPUCLK_RATE(1704000000, 0, 1, 7, 7, 7, 7), + RK3568_CPUCLK_RATE(1608000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1584000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1560000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1536000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1512000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1488000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1464000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1440000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1416000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1392000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1368000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1344000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1320000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1296000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1272000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1248000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1224000000, 0, 1, 5, 5, 5, 5), + RK3568_CPUCLK_RATE(1200000000, 0, 1, 3, 3, 3, 3), + RK3568_CPUCLK_RATE(1104000000, 0, 1, 3, 3, 3, 3), + RK3568_CPUCLK_RATE(1008000000, 0, 1, 3, 3, 3, 3), + RK3568_CPUCLK_RATE(912000000, 0, 1, 3, 3, 3, 3), + RK3568_CPUCLK_RATE(816000000, 0, 1, 3, 3, 3, 3), + RK3568_CPUCLK_RATE(696000000, 0, 1, 3, 3, 3, 3), + RK3568_CPUCLK_RATE(600000000, 0, 1, 3, 3, 3, 3), + RK3568_CPUCLK_RATE(408000000, 0, 1, 3, 3, 3, 3), + RK3568_CPUCLK_RATE(312000000, 0, 1, 3, 3, 3, 3), + RK3568_CPUCLK_RATE(216000000, 0, 1, 3, 3, 3, 3), + RK3568_CPUCLK_RATE(96000000, 0, 1, 3, 3, 3, 3), +}; + +static const struct rockchip_cpuclk_reg_data rk3568_cpuclk_data = { + .core_reg[0] = RK3568_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .core_reg[1] = RK3568_CLKSEL_CON(0), + .div_core_shift[1] = 8, + .div_core_mask[1] = 0x1f, + .core_reg[2] = RK3568_CLKSEL_CON(1), + .div_core_shift[2] = 0, + .div_core_mask[2] = 0x1f, + .core_reg[3] = RK3568_CLKSEL_CON(1), + .div_core_shift[3] = 8, + .div_core_mask[3] = 0x1f, + .num_cores = 4, + .mux_core_alt = 1, + .mux_core_main = 0, + .mux_core_shift = 6, + .mux_core_mask = 0x1, +}; + +PNAME(mux_pll_p) = { "xin24m" }; +PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc_32k" }; +PNAME(clk_i2s0_8ch_tx_p) = { "clk_i2s0_8ch_tx_src", "clk_i2s0_8ch_tx_frac", "i2s0_mclkin", "xin_osc0_half" }; +PNAME(clk_i2s0_8ch_rx_p) = { "clk_i2s0_8ch_rx_src", "clk_i2s0_8ch_rx_frac", "i2s0_mclkin", "xin_osc0_half" }; +PNAME(clk_i2s1_8ch_tx_p) = { "clk_i2s1_8ch_tx_src", "clk_i2s1_8ch_tx_frac", "i2s1_mclkin", "xin_osc0_half" }; +PNAME(clk_i2s1_8ch_rx_p) = { "clk_i2s1_8ch_rx_src", "clk_i2s1_8ch_rx_frac", "i2s1_mclkin", "xin_osc0_half" }; +PNAME(clk_i2s2_2ch_p) = { "clk_i2s2_2ch_src", "clk_i2s2_2ch_frac", "i2s2_mclkin", "xin_osc0_half "}; +PNAME(clk_i2s3_2ch_tx_p) = { "clk_i2s3_2ch_tx_src", "clk_i2s3_2ch_tx_frac", "i2s3_mclkin", "xin_osc0_half" }; +PNAME(clk_i2s3_2ch_rx_p) = { "clk_i2s3_2ch_rx_src", "clk_i2s3_2ch_rx_frac", "i2s3_mclkin", "xin_osc0_half" }; +PNAME(mclk_spdif_8ch_p) = { "mclk_spdif_8ch_src", "mclk_spdif_8ch_frac" }; +PNAME(sclk_audpwm_p) = { "sclk_audpwm_src", "sclk_audpwm_frac" }; +PNAME(sclk_uart1_p) = { "clk_uart1_src", "clk_uart1_frac", "xin24m" }; +PNAME(sclk_uart2_p) = { "clk_uart2_src", "clk_uart2_frac", "xin24m" }; +PNAME(sclk_uart3_p) = { "clk_uart3_src", "clk_uart3_frac", "xin24m" }; +PNAME(sclk_uart4_p) = { "clk_uart4_src", "clk_uart4_frac", "xin24m" }; +PNAME(sclk_uart5_p) = { "clk_uart5_src", "clk_uart5_frac", "xin24m" }; +PNAME(sclk_uart6_p) = { "clk_uart6_src", "clk_uart6_frac", "xin24m" }; +PNAME(sclk_uart7_p) = { "clk_uart7_src", "clk_uart7_frac", "xin24m" }; +PNAME(sclk_uart8_p) = { "clk_uart8_src", "clk_uart8_frac", "xin24m" }; +PNAME(sclk_uart9_p) = { "clk_uart9_src", "clk_uart9_frac", "xin24m" }; +PNAME(sclk_uart0_p) = { "sclk_uart0_div", "sclk_uart0_frac", "xin24m" }; +PNAME(clk_rtc32k_pmu_p) = { "clk_32k_pvtm", "xin32k", "clk_rtc32k_frac" }; +PNAME(mpll_gpll_cpll_npll_p) = { "mpll", "gpll", "cpll", "npll" }; +PNAME(gpll_cpll_npll_p) = { "gpll", "cpll", "npll" }; +PNAME(npll_gpll_p) = { "npll", "gpll" }; +PNAME(cpll_gpll_p) = { "cpll", "gpll" }; +PNAME(gpll_cpll_p) = { "gpll", "cpll" }; +PNAME(gpll_cpll_npll_vpll_p) = { "gpll", "cpll", "npll", "vpll" }; +PNAME(apll_gpll_npll_p) = { "apll", "gpll", "npll" }; +PNAME(sclk_core_pre_p) = { "sclk_core_src", "npll" }; +PNAME(gpll150_gpll100_gpll75_xin24m_p) = { "gpll_150m", "gpll_100m", "gpll_75m", "xin24m" }; +PNAME(clk_gpu_pre_mux_p) = { "clk_gpu_src", "gpu_pvtpll_out" }; +PNAME(clk_npu_pre_ndft_p) = { "clk_npu_src", "clk_npu_np5"}; +PNAME(clk_npu_p) = { "clk_npu_pre_ndft", "npu_pvtpll_out" }; +PNAME(dpll_gpll_cpll_p) = { "dpll", "gpll", "cpll" }; +PNAME(clk_ddr1x_p) = { "clk_ddrphy1x_src", "dpll" }; +PNAME(gpll200_gpll150_gpll100_xin24m_p) = { "gpll_200m", "gpll_150m", "gpll_100m", "xin24m" }; +PNAME(gpll100_gpll75_gpll50_p) = { "gpll_100m", "gpll_75m", "cpll_50m" }; +PNAME(i2s0_mclkout_tx_p) = { "mclk_i2s0_8ch_tx", "xin_osc0_half" }; +PNAME(i2s0_mclkout_rx_p) = { "mclk_i2s0_8ch_rx", "xin_osc0_half" }; +PNAME(i2s1_mclkout_tx_p) = { "mclk_i2s1_8ch_tx", "xin_osc0_half" }; +PNAME(i2s1_mclkout_rx_p) = { "mclk_i2s1_8ch_rx", "xin_osc0_half" }; +PNAME(i2s2_mclkout_p) = { "mclk_i2s2_2ch", "xin_osc0_half" }; +PNAME(i2s3_mclkout_tx_p) = { "mclk_i2s3_2ch_tx", "xin_osc0_half" }; +PNAME(i2s3_mclkout_rx_p) = { "mclk_i2s3_2ch_rx", "xin_osc0_half" }; +PNAME(mclk_pdm_p) = { "gpll_300m", "cpll_250m", "gpll_200m", "gpll_100m" }; +PNAME(clk_i2c_p) = { "gpll_200m", "gpll_100m", "xin24m", "cpll_100m" }; +PNAME(gpll200_gpll150_gpll100_p) = { "gpll_200m", "gpll_150m", "gpll_100m" }; +PNAME(gpll300_gpll200_gpll100_p) = { "gpll_300m", "gpll_200m", "gpll_100m" }; +PNAME(clk_nandc_p) = { "gpll_200m", "gpll_150m", "cpll_100m", "xin24m" }; +PNAME(sclk_sfc_p) = { "xin24m", "cpll_50m", "gpll_75m", "gpll_100m", "cpll_125m", "gpll_150m" }; +PNAME(gpll200_gpll150_cpll125_p) = { "gpll_200m", "gpll_150m", "cpll_125m" }; +PNAME(cclk_emmc_p) = { "xin24m", "gpll_200m", "gpll_150m", "cpll_100m", "cpll_50m", "clk_osc0_div_375k" }; +PNAME(aclk_pipe_p) = { "gpll_400m", "gpll_300m", "gpll_200m", "xin24m" }; +PNAME(gpll200_cpll125_p) = { "gpll_200m", "cpll_125m" }; +PNAME(gpll300_gpll200_gpll100_xin24m_p) = { "gpll_300m", "gpll_200m", "gpll_100m", "xin24m" }; +PNAME(clk_sdmmc_p) = { "xin24m", "gpll_400m", "gpll_300m", "cpll_100m", "cpll_50m", "clk_osc0_div_750k" }; +PNAME(cpll125_cpll50_cpll25_xin24m_p) = { "cpll_125m", "cpll_50m", "cpll_25m", "xin24m" }; +PNAME(clk_gmac_ptp_p) = { "cpll_62p5", "gpll_100m", "cpll_50m", "xin24m" }; +PNAME(cpll333_gpll300_gpll200_p) = { "cpll_333m", "gpll_300m", "gpll_200m" }; +PNAME(cpll_gpll_hpll_p) = { "cpll", "gpll", "hpll" }; +PNAME(gpll_usb480m_xin24m_p) = { "gpll", "usb480m", "xin24m", "xin24m" }; +PNAME(gpll300_cpll250_gpll100_xin24m_p) = { "gpll_300m", "cpll_250m", "gpll_100m", "xin24m" }; +PNAME(cpll_gpll_hpll_vpll_p) = { "cpll", "gpll", "hpll", "vpll" }; +PNAME(hpll_vpll_gpll_cpll_p) = { "hpll", "vpll", "gpll", "cpll" }; +PNAME(gpll400_cpll333_gpll200_p) = { "gpll_400m", "cpll_333m", "gpll_200m" }; +PNAME(gpll100_gpll75_cpll50_xin24m_p) = { "gpll_100m", "gpll_75m", "cpll_50m", "xin24m" }; +PNAME(xin24m_gpll100_cpll100_p) = { "xin24m", "gpll_100m", "cpll_100m" }; +PNAME(gpll_cpll_usb480m_p) = { "gpll", "cpll", "usb480m" }; +PNAME(gpll100_xin24m_cpll100_p) = { "gpll_100m", "xin24m", "cpll_100m" }; +PNAME(gpll200_xin24m_cpll100_p) = { "gpll_200m", "xin24m", "cpll_100m" }; +PNAME(xin24m_32k_p) = { "xin24m", "clk_rtc_32k" }; +PNAME(cpll500_gpll400_gpll300_xin24m_p) = { "cpll_500m", "gpll_400m", "gpll_300m", "xin24m" }; +PNAME(gpll400_gpll300_gpll200_xin24m_p) = { "gpll_400m", "gpll_300m", "gpll_200m", "xin24m" }; +PNAME(xin24m_cpll100_p) = { "xin24m", "cpll_100m" }; +PNAME(ppll_usb480m_cpll_gpll_p) = { "ppll", "usb480m", "cpll", "gpll"}; +PNAME(clk_usbphy0_ref_p) = { "clk_ref24m", "xin_osc0_usbphy0_g" }; +PNAME(clk_usbphy1_ref_p) = { "clk_ref24m", "xin_osc0_usbphy1_g" }; +PNAME(clk_mipidsiphy0_ref_p) = { "clk_ref24m", "xin_osc0_mipidsiphy0_g" }; +PNAME(clk_mipidsiphy1_ref_p) = { "clk_ref24m", "xin_osc0_mipidsiphy1_g" }; +PNAME(clk_wifi_p) = { "clk_wifi_osc0", "clk_wifi_div" }; +PNAME(clk_pciephy0_ref_p) = { "clk_pciephy0_osc0", "clk_pciephy0_div" }; +PNAME(clk_pciephy1_ref_p) = { "clk_pciephy1_osc0", "clk_pciephy1_div" }; +PNAME(clk_pciephy2_ref_p) = { "clk_pciephy2_osc0", "clk_pciephy2_div" }; +PNAME(mux_gmac0_p) = { "clk_mac0_2top", "gmac0_clkin" }; +PNAME(mux_gmac0_rgmii_speed_p) = { "clk_gmac0", "clk_gmac0", "clk_gmac0_tx_div50", "clk_gmac0_tx_div5" }; +PNAME(mux_gmac0_rmii_speed_p) = { "clk_gmac0_rx_div20", "clk_gmac0_rx_div2" }; +PNAME(mux_gmac0_rx_tx_p) = { "clk_gmac0_rgmii_speed", "clk_gmac0_rmii_speed", "clk_gmac0_xpcs_mii" }; +PNAME(mux_gmac1_p) = { "clk_mac1_2top", "gmac1_clkin" }; +PNAME(mux_gmac1_rgmii_speed_p) = { "clk_gmac1", "clk_gmac1", "clk_gmac1_tx_div50", "clk_gmac1_tx_div5" }; +PNAME(mux_gmac1_rmii_speed_p) = { "clk_gmac1_rx_div20", "clk_gmac1_rx_div2" }; +PNAME(mux_gmac1_rx_tx_p) = { "clk_gmac1_rgmii_speed", "clk_gmac1_rmii_speed", "clk_gmac1_xpcs_mii" }; +PNAME(clk_hdmi_ref_p) = { "hpll", "hpll_ph0" }; +PNAME(clk_pdpmu_p) = { "ppll", "gpll" }; +PNAME(clk_mac_2top_p) = { "cpll_125m", "cpll_50m", "cpll_25m", "ppll" }; +PNAME(clk_pwm0_p) = { "xin24m", "clk_pdpmu" }; +PNAME(aclk_rkvdec_pre_p) = { "gpll", "cpll" }; +PNAME(clk_rkvdec_core_p) = { "gpll", "cpll", "dummy_npll", "dummy_vpll" }; +PNAME(clk_32k_ioe_p) = { "clk_rtc_32k", "xin32k" }; +PNAME(i2s1_mclkout_p) = { "i2s1_mclkout_rx", "i2s1_mclkout_tx" }; +PNAME(i2s3_mclkout_p) = { "i2s3_mclkout_rx", "i2s3_mclkout_tx" }; +PNAME(i2s1_mclk_rx_ioe_p) = { "i2s1_mclkin_rx", "i2s1_mclkout_rx" }; +PNAME(i2s1_mclk_tx_ioe_p) = { "i2s1_mclkin_tx", "i2s1_mclkout_tx" }; +PNAME(i2s2_mclk_ioe_p) = { "i2s2_mclkin", "i2s2_mclkout" }; +PNAME(i2s3_mclk_ioe_p) = { "i2s3_mclkin", "i2s3_mclkout" }; + +static struct rockchip_pll_clock rk3568_pmu_pll_clks[] __initdata = { + [ppll] = PLL(pll_rk3328, PLL_PPLL, "ppll", mux_pll_p, + 0, RK3568_PMU_PLL_CON(0), + RK3568_PMU_MODE_CON0, 0, 4, 0, rk3568_pll_rates), + [hpll] = PLL(pll_rk3328, PLL_HPLL, "hpll", mux_pll_p, + 0, RK3568_PMU_PLL_CON(16), + RK3568_PMU_MODE_CON0, 2, 7, 0, rk3568_pll_rates), +}; + +static struct rockchip_pll_clock rk3568_pll_clks[] __initdata = { + [apll] = PLL(pll_rk3328, PLL_APLL, "apll", mux_pll_p, + 0, RK3568_PLL_CON(0), + RK3568_MODE_CON0, 0, 0, 0, rk3568_pll_rates), + [dpll] = PLL(pll_rk3328, PLL_DPLL, "dpll", mux_pll_p, + 0, RK3568_PLL_CON(8), + RK3568_MODE_CON0, 2, 1, 0, NULL), + [cpll] = PLL(pll_rk3328, PLL_CPLL, "cpll", mux_pll_p, + 0, RK3568_PLL_CON(24), + RK3568_MODE_CON0, 4, 2, 0, rk3568_pll_rates), + [gpll] = PLL(pll_rk3328, PLL_GPLL, "gpll", mux_pll_p, + 0, RK3568_PLL_CON(16), + RK3568_MODE_CON0, 6, 3, 0, rk3568_pll_rates), + [npll] = PLL(pll_rk3328, PLL_NPLL, "npll", mux_pll_p, + CLK_IS_CRITICAL, RK3568_PLL_CON(32), + RK3568_MODE_CON0, 10, 5, 0, rk3568_pll_rates), + [vpll] = PLL(pll_rk3328, PLL_VPLL, "vpll", mux_pll_p, + 0, RK3568_PLL_CON(40), + RK3568_MODE_CON0, 12, 6, 0, rk3568_pll_rates), +}; + +#define MFLAGS CLK_MUX_HIWORD_MASK +#define DFLAGS CLK_DIVIDER_HIWORD_MASK +#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE) + +static struct rockchip_clk_branch rk3568_i2s0_8ch_tx_fracmux __initdata = + MUX(CLK_I2S0_8CH_TX, "clk_i2s0_8ch_tx", clk_i2s0_8ch_tx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(11), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_i2s0_8ch_rx_fracmux __initdata = + MUX(CLK_I2S0_8CH_RX, "clk_i2s0_8ch_rx", clk_i2s0_8ch_rx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(13), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_i2s1_8ch_tx_fracmux __initdata = + MUX(CLK_I2S1_8CH_TX, "clk_i2s1_8ch_tx", clk_i2s1_8ch_tx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(15), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_i2s1_8ch_rx_fracmux __initdata = + MUX(CLK_I2S1_8CH_RX, "clk_i2s1_8ch_rx", clk_i2s1_8ch_rx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(17), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_i2s2_2ch_fracmux __initdata = + MUX(CLK_I2S2_2CH, "clk_i2s2_2ch", clk_i2s2_2ch_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(19), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_i2s3_2ch_tx_fracmux __initdata = + MUX(CLK_I2S3_2CH_TX, "clk_i2s3_2ch_tx", clk_i2s3_2ch_tx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(21), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_i2s3_2ch_rx_fracmux __initdata = + MUX(CLK_I2S3_2CH_RX, "clk_i2s3_2ch_rx", clk_i2s3_2ch_rx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(83), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_spdif_8ch_fracmux __initdata = + MUX(MCLK_SPDIF_8CH, "mclk_spdif_8ch", mclk_spdif_8ch_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(23), 15, 1, MFLAGS); + +static struct rockchip_clk_branch rk3568_audpwm_fracmux __initdata = + MUX(SCLK_AUDPWM, "sclk_audpwm", sclk_audpwm_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(25), 15, 1, MFLAGS); + +static struct rockchip_clk_branch rk3568_uart1_fracmux __initdata = + MUX(0, "sclk_uart1_mux", sclk_uart1_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(52), 12, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_uart2_fracmux __initdata = + MUX(0, "sclk_uart2_mux", sclk_uart2_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(54), 12, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_uart3_fracmux __initdata = + MUX(0, "sclk_uart3_mux", sclk_uart3_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(56), 12, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_uart4_fracmux __initdata = + MUX(0, "sclk_uart4_mux", sclk_uart4_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(58), 12, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_uart5_fracmux __initdata = + MUX(0, "sclk_uart5_mux", sclk_uart5_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(60), 12, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_uart6_fracmux __initdata = + MUX(0, "sclk_uart6_mux", sclk_uart6_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(62), 12, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_uart7_fracmux __initdata = + MUX(0, "sclk_uart7_mux", sclk_uart7_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(64), 12, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_uart8_fracmux __initdata = + MUX(0, "sclk_uart8_mux", sclk_uart8_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(66), 12, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_uart9_fracmux __initdata = + MUX(0, "sclk_uart9_mux", sclk_uart9_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(68), 12, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_uart0_fracmux __initdata = + MUX(0, "sclk_uart0_mux", sclk_uart0_p, CLK_SET_RATE_PARENT, + RK3568_PMU_CLKSEL_CON(4), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_rtc32k_pmu_fracmux __initdata = + MUX(CLK_RTC_32K, "clk_rtc_32k", clk_rtc32k_pmu_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK3568_PMU_CLKSEL_CON(0), 6, 2, MFLAGS); + +static struct rockchip_clk_branch rk3568_clk_npu_np5 __initdata = + COMPOSITE_HALFDIV(CLK_NPU_NP5, "clk_npu_np5", npll_gpll_p, 0, + RK3568_CLKSEL_CON(7), 7, 1, MFLAGS, 4, 2, DFLAGS, + RK3568_CLKGATE_CON(3), 1, GFLAGS); + +static struct rockchip_clk_branch rk3568_clk_branches[] __initdata = { + /* + * Clock-Architecture Diagram 1 + */ + /* SRC_CLK */ + COMPOSITE_NOMUX(0, "gpll_400m", "gpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(75), 0, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 0, GFLAGS), + COMPOSITE_NOMUX(0, "gpll_300m", "gpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(75), 8, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 1, GFLAGS), + COMPOSITE_NOMUX(0, "gpll_200m", "gpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(76), 0, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 2, GFLAGS), + COMPOSITE_NOMUX(0, "gpll_150m", "gpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(76), 8, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 3, GFLAGS), + COMPOSITE_NOMUX(0, "gpll_100m", "gpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(77), 0, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 4, GFLAGS), + COMPOSITE_NOMUX(0, "gpll_75m", "gpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(77), 8, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 5, GFLAGS), + COMPOSITE_NOMUX(0, "gpll_20m", "gpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(78), 0, 6, DFLAGS, + RK3568_CLKGATE_CON(35), 6, GFLAGS), + COMPOSITE_NOMUX(CPLL_500M, "cpll_500m", "cpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(78), 8, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 7, GFLAGS), + COMPOSITE_NOMUX(CPLL_333M, "cpll_333m", "cpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(79), 0, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 8, GFLAGS), + COMPOSITE_NOMUX(CPLL_250M, "cpll_250m", "cpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(79), 8, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 9, GFLAGS), + COMPOSITE_NOMUX(CPLL_125M, "cpll_125m", "cpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(80), 0, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 10, GFLAGS), + COMPOSITE_NOMUX(CPLL_100M, "cpll_100m", "cpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(82), 0, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 11, GFLAGS), + COMPOSITE_NOMUX(CPLL_62P5M, "cpll_62p5", "cpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(80), 8, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 12, GFLAGS), + COMPOSITE_NOMUX(CPLL_50M, "cpll_50m", "cpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(81), 0, 5, DFLAGS, + RK3568_CLKGATE_CON(35), 13, GFLAGS), + COMPOSITE_NOMUX(CPLL_25M, "cpll_25m", "cpll", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(81), 8, 6, DFLAGS, + RK3568_CLKGATE_CON(35), 14, GFLAGS), + COMPOSITE_NOMUX(0, "clk_osc0_div_750k", "xin24m", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(82), 8, 6, DFLAGS, + RK3568_CLKGATE_CON(35), 15, GFLAGS), + FACTOR(0, "clk_osc0_div_375k", "clk_osc0_div_750k", 0, 1, 2), + FACTOR(0, "xin_osc0_half", "xin24m", 0, 1, 2), + MUX(USB480M, "usb480m", mux_usb480m_p, CLK_SET_RATE_PARENT, + RK3568_MODE_CON0, 14, 2, MFLAGS), + + /* PD_CORE */ + COMPOSITE(0, "sclk_core_src", apll_gpll_npll_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(2), 8, 2, MFLAGS, 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3568_CLKGATE_CON(0), 5, GFLAGS), + COMPOSITE_NODIV(0, "sclk_core", sclk_core_pre_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(2), 15, 1, MFLAGS, + RK3568_CLKGATE_CON(0), 7, GFLAGS), + + COMPOSITE_NOMUX(0, "atclk_core", "armclk", CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(3), 0, 5, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3568_CLKGATE_CON(0), 8, GFLAGS), + COMPOSITE_NOMUX(0, "gicclk_core", "armclk", CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(3), 8, 5, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3568_CLKGATE_CON(0), 9, GFLAGS), + COMPOSITE_NOMUX(0, "pclk_core_pre", "armclk", CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(4), 0, 5, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3568_CLKGATE_CON(0), 10, GFLAGS), + COMPOSITE_NOMUX(0, "periphclk_core_pre", "armclk", CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(4), 8, 5, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3568_CLKGATE_CON(0), 11, GFLAGS), + COMPOSITE_NOMUX(0, "tsclk_core", "periphclk_core_pre", CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(5), 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3568_CLKGATE_CON(0), 14, GFLAGS), + COMPOSITE_NOMUX(0, "cntclk_core", "periphclk_core_pre", CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(5), 4, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3568_CLKGATE_CON(0), 15, GFLAGS), + COMPOSITE_NOMUX(0, "aclk_core", "sclk_core", CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(5), 8, 5, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3568_CLKGATE_CON(1), 0, GFLAGS), + + COMPOSITE_NODIV(ACLK_CORE_NIU2BUS, "aclk_core_niu2bus", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(5), 14, 2, MFLAGS, + RK3568_CLKGATE_CON(1), 2, GFLAGS), + + GATE(CLK_CORE_PVTM, "clk_core_pvtm", "xin24m", 0, + RK3568_CLKGATE_CON(1), 10, GFLAGS), + GATE(CLK_CORE_PVTM_CORE, "clk_core_pvtm_core", "armclk", 0, + RK3568_CLKGATE_CON(1), 11, GFLAGS), + GATE(CLK_CORE_PVTPLL, "clk_core_pvtpll", "armclk", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(1), 12, GFLAGS), + GATE(PCLK_CORE_PVTM, "pclk_core_pvtm", "pclk_core_pre", 0, + RK3568_CLKGATE_CON(1), 9, GFLAGS), + + /* PD_GPU */ + COMPOSITE(CLK_GPU_SRC, "clk_gpu_src", mpll_gpll_cpll_npll_p, 0, + RK3568_CLKSEL_CON(6), 6, 2, MFLAGS | CLK_MUX_READ_ONLY, 0, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, + RK3568_CLKGATE_CON(2), 0, GFLAGS), + MUX(CLK_GPU_PRE_MUX, "clk_gpu_pre_mux", clk_gpu_pre_mux_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(6), 11, 1, MFLAGS | CLK_MUX_READ_ONLY), + DIV(ACLK_GPU_PRE, "aclk_gpu_pre", "clk_gpu_pre_mux", 0, + RK3568_CLKSEL_CON(6), 8, 2, DFLAGS), + DIV(PCLK_GPU_PRE, "pclk_gpu_pre", "clk_gpu_pre_mux", 0, + RK3568_CLKSEL_CON(6), 12, 4, DFLAGS), + GATE(CLK_GPU, "clk_gpu", "clk_gpu_pre_mux", 0, + RK3568_CLKGATE_CON(2), 3, GFLAGS), + + GATE(PCLK_GPU_PVTM, "pclk_gpu_pvtm", "pclk_gpu_pre", 0, + RK3568_CLKGATE_CON(2), 6, GFLAGS), + GATE(CLK_GPU_PVTM, "clk_gpu_pvtm", "xin24m", 0, + RK3568_CLKGATE_CON(2), 7, GFLAGS), + GATE(CLK_GPU_PVTM_CORE, "clk_gpu_pvtm_core", "clk_gpu_src", 0, + RK3568_CLKGATE_CON(2), 8, GFLAGS), + GATE(CLK_GPU_PVTPLL, "clk_gpu_pvtpll", "clk_gpu_src", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(2), 9, GFLAGS), + + /* PD_NPU */ + COMPOSITE_BROTHER(CLK_NPU_SRC, "clk_npu_src", npll_gpll_p, 0, + RK3568_CLKSEL_CON(7), 6, 1, MFLAGS, 0, 4, DFLAGS, + RK3568_CLKGATE_CON(3), 0, GFLAGS, + &rk3568_clk_npu_np5), + MUX(CLK_NPU_PRE_NDFT, "clk_npu_pre_ndft", clk_npu_pre_ndft_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + RK3568_CLKSEL_CON(7), 8, 1, MFLAGS), + MUX(CLK_NPU, "clk_npu", clk_npu_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(7), 15, 1, MFLAGS), + COMPOSITE_NOMUX(HCLK_NPU_PRE, "hclk_npu_pre", "clk_npu", 0, + RK3568_CLKSEL_CON(8), 0, 4, DFLAGS, + RK3568_CLKGATE_CON(3), 2, GFLAGS), + COMPOSITE_NOMUX(PCLK_NPU_PRE, "pclk_npu_pre", "clk_npu", 0, + RK3568_CLKSEL_CON(8), 4, 4, DFLAGS, + RK3568_CLKGATE_CON(3), 3, GFLAGS), + GATE(ACLK_NPU_PRE, "aclk_npu_pre", "clk_npu", 0, + RK3568_CLKGATE_CON(3), 4, GFLAGS), + GATE(ACLK_NPU, "aclk_npu", "aclk_npu_pre", 0, + RK3568_CLKGATE_CON(3), 7, GFLAGS), + GATE(HCLK_NPU, "hclk_npu", "hclk_npu_pre", 0, + RK3568_CLKGATE_CON(3), 8, GFLAGS), + + GATE(PCLK_NPU_PVTM, "pclk_npu_pvtm", "pclk_npu_pre", 0, + RK3568_CLKGATE_CON(3), 9, GFLAGS), + GATE(CLK_NPU_PVTM, "clk_npu_pvtm", "xin24m", 0, + RK3568_CLKGATE_CON(3), 10, GFLAGS), + GATE(CLK_NPU_PVTM_CORE, "clk_npu_pvtm_core", "clk_npu_pre_ndft", 0, + RK3568_CLKGATE_CON(3), 11, GFLAGS), + GATE(CLK_NPU_PVTPLL, "clk_npu_pvtpll", "clk_npu_pre_ndft", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(3), 12, GFLAGS), + + /* PD_DDR */ + COMPOSITE(CLK_DDRPHY1X_SRC, "clk_ddrphy1x_src", dpll_gpll_cpll_p, CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(9), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(4), 0, GFLAGS), + MUXGRF(CLK_DDR1X, "clk_ddr1x", clk_ddr1x_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(9), 15, 1, MFLAGS), + + COMPOSITE_NOMUX(CLK_MSCH, "clk_msch", "clk_ddr1x", CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(10), 0, 2, DFLAGS, + RK3568_CLKGATE_CON(4), 2, GFLAGS), + GATE(CLK24_DDRMON, "clk24_ddrmon", "xin24m", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(4), 15, GFLAGS), + + /* PD_GIC_AUDIO */ + COMPOSITE_NODIV(ACLK_GIC_AUDIO, "aclk_gic_audio", gpll200_gpll150_gpll100_xin24m_p, CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(10), 8, 2, MFLAGS, + RK3568_CLKGATE_CON(5), 0, GFLAGS), + COMPOSITE_NODIV(HCLK_GIC_AUDIO, "hclk_gic_audio", gpll150_gpll100_gpll75_xin24m_p, CLK_IGNORE_UNUSED, + RK3568_CLKSEL_CON(10), 10, 2, MFLAGS, + RK3568_CLKGATE_CON(5), 1, GFLAGS), + GATE(HCLK_SDMMC_BUFFER, "hclk_sdmmc_buffer", "hclk_gic_audio", 0, + RK3568_CLKGATE_CON(5), 8, GFLAGS), + COMPOSITE_NODIV(DCLK_SDMMC_BUFFER, "dclk_sdmmc_buffer", gpll100_gpll75_gpll50_p, 0, + RK3568_CLKSEL_CON(10), 12, 2, MFLAGS, + RK3568_CLKGATE_CON(5), 9, GFLAGS), + GATE(ACLK_GIC600, "aclk_gic600", "aclk_gic_audio", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(5), 4, GFLAGS), + GATE(ACLK_SPINLOCK, "aclk_spinlock", "aclk_gic_audio", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(5), 7, GFLAGS), + GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_gic_audio", 0, + RK3568_CLKGATE_CON(5), 10, GFLAGS), + GATE(HCLK_I2S1_8CH, "hclk_i2s1_8ch", "hclk_gic_audio", 0, + RK3568_CLKGATE_CON(5), 11, GFLAGS), + GATE(HCLK_I2S2_2CH, "hclk_i2s2_2ch", "hclk_gic_audio", 0, + RK3568_CLKGATE_CON(5), 12, GFLAGS), + GATE(HCLK_I2S3_2CH, "hclk_i2s3_2ch", "hclk_gic_audio", 0, + RK3568_CLKGATE_CON(5), 13, GFLAGS), + + COMPOSITE(CLK_I2S0_8CH_TX_SRC, "clk_i2s0_8ch_tx_src", gpll_cpll_npll_p, 0, + RK3568_CLKSEL_CON(11), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(6), 0, GFLAGS), + COMPOSITE_FRACMUX(CLK_I2S0_8CH_TX_FRAC, "clk_i2s0_8ch_tx_frac", "clk_i2s0_8ch_tx_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(12), 0, + RK3568_CLKGATE_CON(6), 1, GFLAGS, + &rk3568_i2s0_8ch_tx_fracmux, RK3568_FRAC_MAX_PRATE), + GATE(MCLK_I2S0_8CH_TX, "mclk_i2s0_8ch_tx", "clk_i2s0_8ch_tx", 0, + RK3568_CLKGATE_CON(6), 2, GFLAGS), + COMPOSITE_NODIV(I2S0_MCLKOUT_TX, "i2s0_mclkout_tx", i2s0_mclkout_tx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(11), 15, 1, MFLAGS, + RK3568_CLKGATE_CON(6), 3, GFLAGS), + + COMPOSITE(CLK_I2S0_8CH_RX_SRC, "clk_i2s0_8ch_rx_src", gpll_cpll_npll_p, 0, + RK3568_CLKSEL_CON(13), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(6), 4, GFLAGS), + COMPOSITE_FRACMUX(CLK_I2S0_8CH_RX_FRAC, "clk_i2s0_8ch_rx_frac", "clk_i2s0_8ch_rx_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(14), 0, + RK3568_CLKGATE_CON(6), 5, GFLAGS, + &rk3568_i2s0_8ch_rx_fracmux, RK3568_FRAC_MAX_PRATE), + GATE(MCLK_I2S0_8CH_RX, "mclk_i2s0_8ch_rx", "clk_i2s0_8ch_rx", 0, + RK3568_CLKGATE_CON(6), 6, GFLAGS), + COMPOSITE_NODIV(I2S0_MCLKOUT_RX, "i2s0_mclkout_rx", i2s0_mclkout_rx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(13), 15, 1, MFLAGS, + RK3568_CLKGATE_CON(6), 7, GFLAGS), + + COMPOSITE(CLK_I2S1_8CH_TX_SRC, "clk_i2s1_8ch_tx_src", gpll_cpll_npll_p, 0, + RK3568_CLKSEL_CON(15), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(6), 8, GFLAGS), + COMPOSITE_FRACMUX(CLK_I2S1_8CH_TX_FRAC, "clk_i2s1_8ch_tx_frac", "clk_i2s1_8ch_tx_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(16), 0, + RK3568_CLKGATE_CON(6), 9, GFLAGS, + &rk3568_i2s1_8ch_tx_fracmux, RK3568_FRAC_MAX_PRATE), + GATE(MCLK_I2S1_8CH_TX, "mclk_i2s1_8ch_tx", "clk_i2s1_8ch_tx", 0, + RK3568_CLKGATE_CON(6), 10, GFLAGS), + COMPOSITE_NODIV(I2S1_MCLKOUT_TX, "i2s1_mclkout_tx", i2s1_mclkout_tx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(15), 15, 1, MFLAGS, + RK3568_CLKGATE_CON(6), 11, GFLAGS), + + COMPOSITE(CLK_I2S1_8CH_RX_SRC, "clk_i2s1_8ch_rx_src", gpll_cpll_npll_p, 0, + RK3568_CLKSEL_CON(17), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(6), 12, GFLAGS), + COMPOSITE_FRACMUX(CLK_I2S1_8CH_RX_FRAC, "clk_i2s1_8ch_rx_frac", "clk_i2s1_8ch_rx_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(18), 0, + RK3568_CLKGATE_CON(6), 13, GFLAGS, + &rk3568_i2s1_8ch_rx_fracmux, RK3568_FRAC_MAX_PRATE), + GATE(MCLK_I2S1_8CH_RX, "mclk_i2s1_8ch_rx", "clk_i2s1_8ch_rx", 0, + RK3568_CLKGATE_CON(6), 14, GFLAGS), + COMPOSITE_NODIV(I2S1_MCLKOUT_RX, "i2s1_mclkout_rx", i2s1_mclkout_rx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(17), 15, 1, MFLAGS, + RK3568_CLKGATE_CON(6), 15, GFLAGS), + + COMPOSITE(CLK_I2S2_2CH_SRC, "clk_i2s2_2ch_src", gpll_cpll_npll_p, 0, + RK3568_CLKSEL_CON(19), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(7), 0, GFLAGS), + COMPOSITE_FRACMUX(CLK_I2S2_2CH_FRAC, "clk_i2s2_2ch_frac", "clk_i2s2_2ch_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(20), 0, + RK3568_CLKGATE_CON(7), 1, GFLAGS, + &rk3568_i2s2_2ch_fracmux, RK3568_FRAC_MAX_PRATE), + GATE(MCLK_I2S2_2CH, "mclk_i2s2_2ch", "clk_i2s2_2ch", 0, + RK3568_CLKGATE_CON(7), 2, GFLAGS), + COMPOSITE_NODIV(I2S2_MCLKOUT, "i2s2_mclkout", i2s2_mclkout_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(19), 15, 1, MFLAGS, + RK3568_CLKGATE_CON(7), 3, GFLAGS), + + COMPOSITE(CLK_I2S3_2CH_TX_SRC, "clk_i2s3_2ch_tx_src", gpll_cpll_npll_p, 0, + RK3568_CLKSEL_CON(21), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(7), 4, GFLAGS), + COMPOSITE_FRACMUX(CLK_I2S3_2CH_TX_FRAC, "clk_i2s3_2ch_tx_frac", "clk_i2s3_2ch_tx_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(22), 0, + RK3568_CLKGATE_CON(7), 5, GFLAGS, + &rk3568_i2s3_2ch_tx_fracmux, RK3568_FRAC_MAX_PRATE), + GATE(MCLK_I2S3_2CH_TX, "mclk_i2s3_2ch_tx", "clk_i2s3_2ch_tx", 0, + RK3568_CLKGATE_CON(7), 6, GFLAGS), + COMPOSITE_NODIV(I2S3_MCLKOUT_TX, "i2s3_mclkout_tx", i2s3_mclkout_tx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(21), 15, 1, MFLAGS, + RK3568_CLKGATE_CON(7), 7, GFLAGS), + + COMPOSITE(CLK_I2S3_2CH_RX_SRC, "clk_i2s3_2ch_rx_src", gpll_cpll_npll_p, 0, + RK3568_CLKSEL_CON(83), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(7), 8, GFLAGS), + COMPOSITE_FRACMUX(CLK_I2S3_2CH_RX_FRAC, "clk_i2s3_2ch_rx_frac", "clk_i2s3_2ch_rx_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(84), 0, + RK3568_CLKGATE_CON(7), 9, GFLAGS, + &rk3568_i2s3_2ch_rx_fracmux, RK3568_FRAC_MAX_PRATE), + GATE(MCLK_I2S3_2CH_RX, "mclk_i2s3_2ch_rx", "clk_i2s3_2ch_rx", 0, + RK3568_CLKGATE_CON(7), 10, GFLAGS), + COMPOSITE_NODIV(I2S3_MCLKOUT_RX, "i2s3_mclkout_rx", i2s3_mclkout_rx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(83), 15, 1, MFLAGS, + RK3568_CLKGATE_CON(7), 11, GFLAGS), + + MUXGRF(I2S1_MCLKOUT, "i2s1_mclkout", i2s1_mclkout_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK3568_GRF_SOC_CON1, 5, 1, MFLAGS), + MUXGRF(I2S3_MCLKOUT, "i2s3_mclkout", i2s3_mclkout_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK3568_GRF_SOC_CON2, 15, 1, MFLAGS), + MUXGRF(I2S1_MCLK_RX_IOE, "i2s1_mclk_rx_ioe", i2s1_mclk_rx_ioe_p, 0, + RK3568_GRF_SOC_CON2, 0, 1, MFLAGS), + MUXGRF(I2S1_MCLK_TX_IOE, "i2s1_mclk_tx_ioe", i2s1_mclk_tx_ioe_p, 0, + RK3568_GRF_SOC_CON2, 1, 1, MFLAGS), + MUXGRF(I2S2_MCLK_IOE, "i2s2_mclk_ioe", i2s2_mclk_ioe_p, 0, + RK3568_GRF_SOC_CON2, 2, 1, MFLAGS), + MUXGRF(I2S3_MCLK_IOE, "i2s3_mclk_ioe", i2s3_mclk_ioe_p, 0, + RK3568_GRF_SOC_CON2, 3, 1, MFLAGS), + + GATE(HCLK_PDM, "hclk_pdm", "hclk_gic_audio", 0, + RK3568_CLKGATE_CON(5), 14, GFLAGS), + COMPOSITE_NODIV(MCLK_PDM, "mclk_pdm", mclk_pdm_p, 0, + RK3568_CLKSEL_CON(23), 8, 2, MFLAGS, + RK3568_CLKGATE_CON(5), 15, GFLAGS), + GATE(HCLK_VAD, "hclk_vad", "hclk_gic_audio", 0, + RK3568_CLKGATE_CON(7), 12, GFLAGS), + GATE(HCLK_SPDIF_8CH, "hclk_spdif_8ch", "hclk_gic_audio", 0, + RK3568_CLKGATE_CON(7), 13, GFLAGS), + + COMPOSITE(MCLK_SPDIF_8CH_SRC, "mclk_spdif_8ch_src", cpll_gpll_p, 0, + RK3568_CLKSEL_CON(23), 14, 1, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(7), 14, GFLAGS), + COMPOSITE_FRACMUX(MCLK_SPDIF_8CH_FRAC, "mclk_spdif_8ch_frac", "mclk_spdif_8ch_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(24), 0, + RK3568_CLKGATE_CON(7), 15, GFLAGS, + &rk3568_spdif_8ch_fracmux, RK3568_SPDIF_FRAC_MAX_PRATE), + + GATE(HCLK_AUDPWM, "hclk_audpwm", "hclk_gic_audio", 0, + RK3568_CLKGATE_CON(8), 0, GFLAGS), + COMPOSITE(SCLK_AUDPWM_SRC, "sclk_audpwm_src", gpll_cpll_p, 0, + RK3568_CLKSEL_CON(25), 14, 1, MFLAGS, 0, 6, DFLAGS, + RK3568_CLKGATE_CON(8), 1, GFLAGS), + COMPOSITE_FRACMUX(SCLK_AUDPWM_FRAC, "sclk_audpwm_frac", "sclk_audpwm_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(26), 0, + RK3568_CLKGATE_CON(8), 2, GFLAGS, + &rk3568_audpwm_fracmux, RK3568_FRAC_MAX_PRATE), + + GATE(HCLK_ACDCDIG, "hclk_acdcdig", "hclk_gic_audio", 0, + RK3568_CLKGATE_CON(8), 3, GFLAGS), + COMPOSITE_NODIV(CLK_ACDCDIG_I2C, "clk_acdcdig_i2c", clk_i2c_p, 0, + RK3568_CLKSEL_CON(23), 10, 2, MFLAGS, + RK3568_CLKGATE_CON(8), 4, GFLAGS), + GATE(CLK_ACDCDIG_DAC, "clk_acdcdig_dac", "mclk_i2s3_2ch_tx", 0, + RK3568_CLKGATE_CON(8), 5, GFLAGS), + GATE(CLK_ACDCDIG_ADC, "clk_acdcdig_adc", "mclk_i2s3_2ch_rx", 0, + RK3568_CLKGATE_CON(8), 6, GFLAGS), + + /* PD_SECURE_FLASH */ + COMPOSITE_NODIV(ACLK_SECURE_FLASH, "aclk_secure_flash", gpll200_gpll150_gpll100_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(27), 0, 2, MFLAGS, + RK3568_CLKGATE_CON(8), 7, GFLAGS), + COMPOSITE_NODIV(HCLK_SECURE_FLASH, "hclk_secure_flash", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(27), 2, 2, MFLAGS, + RK3568_CLKGATE_CON(8), 8, GFLAGS), + GATE(ACLK_CRYPTO_NS, "aclk_crypto_ns", "aclk_secure_flash", 0, + RK3568_CLKGATE_CON(8), 11, GFLAGS), + GATE(HCLK_CRYPTO_NS, "hclk_crypto_ns", "hclk_secure_flash", 0, + RK3568_CLKGATE_CON(8), 12, GFLAGS), + COMPOSITE_NODIV(CLK_CRYPTO_NS_CORE, "clk_crypto_ns_core", gpll200_gpll150_gpll100_p, 0, + RK3568_CLKSEL_CON(27), 4, 2, MFLAGS, + RK3568_CLKGATE_CON(8), 13, GFLAGS), + COMPOSITE_NODIV(CLK_CRYPTO_NS_PKA, "clk_crypto_ns_pka", gpll300_gpll200_gpll100_p, 0, + RK3568_CLKSEL_CON(27), 6, 2, MFLAGS, + RK3568_CLKGATE_CON(8), 14, GFLAGS), + GATE(CLK_CRYPTO_NS_RNG, "clk_crypto_ns_rng", "hclk_secure_flash", 0, + RK3568_CLKGATE_CON(8), 15, GFLAGS), + GATE(HCLK_TRNG_NS, "hclk_trng_ns", "hclk_secure_flash", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(9), 10, GFLAGS), + GATE(CLK_TRNG_NS, "clk_trng_ns", "hclk_secure_flash", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(9), 11, GFLAGS), + GATE(PCLK_OTPC_NS, "pclk_otpc_ns", "hclk_secure_flash", 0, + RK3568_CLKGATE_CON(26), 9, GFLAGS), + GATE(CLK_OTPC_NS_SBPI, "clk_otpc_ns_sbpi", "xin24m", 0, + RK3568_CLKGATE_CON(26), 10, GFLAGS), + GATE(CLK_OTPC_NS_USR, "clk_otpc_ns_usr", "xin_osc0_half", 0, + RK3568_CLKGATE_CON(26), 11, GFLAGS), + GATE(HCLK_NANDC, "hclk_nandc", "hclk_secure_flash", 0, + RK3568_CLKGATE_CON(9), 0, GFLAGS), + COMPOSITE_NODIV(NCLK_NANDC, "nclk_nandc", clk_nandc_p, 0, + RK3568_CLKSEL_CON(28), 0, 2, MFLAGS, + RK3568_CLKGATE_CON(9), 1, GFLAGS), + GATE(HCLK_SFC, "hclk_sfc", "hclk_secure_flash", 0, + RK3568_CLKGATE_CON(9), 2, GFLAGS), + GATE(HCLK_SFC_XIP, "hclk_sfc_xip", "hclk_secure_flash", 0, + RK3568_CLKGATE_CON(9), 3, GFLAGS), + COMPOSITE_NODIV(SCLK_SFC, "sclk_sfc", sclk_sfc_p, 0, + RK3568_CLKSEL_CON(28), 4, 3, MFLAGS, + RK3568_CLKGATE_CON(9), 4, GFLAGS), + GATE(ACLK_EMMC, "aclk_emmc", "aclk_secure_flash", 0, + RK3568_CLKGATE_CON(9), 5, GFLAGS), + GATE(HCLK_EMMC, "hclk_emmc", "hclk_secure_flash", 0, + RK3568_CLKGATE_CON(9), 6, GFLAGS), + COMPOSITE_NODIV(BCLK_EMMC, "bclk_emmc", gpll200_gpll150_cpll125_p, 0, + RK3568_CLKSEL_CON(28), 8, 2, MFLAGS, + RK3568_CLKGATE_CON(9), 7, GFLAGS), + COMPOSITE_NODIV(CCLK_EMMC, "cclk_emmc", cclk_emmc_p, 0, + RK3568_CLKSEL_CON(28), 12, 3, MFLAGS, + RK3568_CLKGATE_CON(9), 8, GFLAGS), + GATE(TCLK_EMMC, "tclk_emmc", "xin24m", 0, + RK3568_CLKGATE_CON(9), 9, GFLAGS), + MMC(SCLK_EMMC_DRV, "emmc_drv", "cclk_emmc", RK3568_EMMC_CON0, 1), + MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "cclk_emmc", RK3568_EMMC_CON1, 1), + + /* PD_PIPE */ + COMPOSITE_NODIV(ACLK_PIPE, "aclk_pipe", aclk_pipe_p, 0, + RK3568_CLKSEL_CON(29), 0, 2, MFLAGS, + RK3568_CLKGATE_CON(10), 0, GFLAGS), + COMPOSITE_NOMUX(PCLK_PIPE, "pclk_pipe", "aclk_pipe", 0, + RK3568_CLKSEL_CON(29), 4, 4, DFLAGS, + RK3568_CLKGATE_CON(10), 1, GFLAGS), + GATE(ACLK_PCIE20_MST, "aclk_pcie20_mst", "aclk_pipe", 0, + RK3568_CLKGATE_CON(12), 0, GFLAGS), + GATE(ACLK_PCIE20_SLV, "aclk_pcie20_slv", "aclk_pipe", 0, + RK3568_CLKGATE_CON(12), 1, GFLAGS), + GATE(ACLK_PCIE20_DBI, "aclk_pcie20_dbi", "aclk_pipe", 0, + RK3568_CLKGATE_CON(12), 2, GFLAGS), + GATE(PCLK_PCIE20, "pclk_pcie20", "pclk_pipe", 0, + RK3568_CLKGATE_CON(12), 3, GFLAGS), + GATE(CLK_PCIE20_AUX_NDFT, "clk_pcie20_aux_ndft", "xin24m", 0, + RK3568_CLKGATE_CON(12), 4, GFLAGS), + GATE(ACLK_PCIE30X1_MST, "aclk_pcie30x1_mst", "aclk_pipe", 0, + RK3568_CLKGATE_CON(12), 8, GFLAGS), + GATE(ACLK_PCIE30X1_SLV, "aclk_pcie30x1_slv", "aclk_pipe", 0, + RK3568_CLKGATE_CON(12), 9, GFLAGS), + GATE(ACLK_PCIE30X1_DBI, "aclk_pcie30x1_dbi", "aclk_pipe", 0, + RK3568_CLKGATE_CON(12), 10, GFLAGS), + GATE(PCLK_PCIE30X1, "pclk_pcie30x1", "pclk_pipe", 0, + RK3568_CLKGATE_CON(12), 11, GFLAGS), + GATE(CLK_PCIE30X1_AUX_NDFT, "clk_pcie30x1_aux_ndft", "xin24m", 0, + RK3568_CLKGATE_CON(12), 12, GFLAGS), + GATE(ACLK_PCIE30X2_MST, "aclk_pcie30x2_mst", "aclk_pipe", 0, + RK3568_CLKGATE_CON(13), 0, GFLAGS), + GATE(ACLK_PCIE30X2_SLV, "aclk_pcie30x2_slv", "aclk_pipe", 0, + RK3568_CLKGATE_CON(13), 1, GFLAGS), + GATE(ACLK_PCIE30X2_DBI, "aclk_pcie30x2_dbi", "aclk_pipe", 0, + RK3568_CLKGATE_CON(13), 2, GFLAGS), + GATE(PCLK_PCIE30X2, "pclk_pcie30x2", "pclk_pipe", 0, + RK3568_CLKGATE_CON(13), 3, GFLAGS), + GATE(CLK_PCIE30X2_AUX_NDFT, "clk_pcie30x2_aux_ndft", "xin24m", 0, + RK3568_CLKGATE_CON(13), 4, GFLAGS), + GATE(ACLK_SATA0, "aclk_sata0", "aclk_pipe", 0, + RK3568_CLKGATE_CON(11), 0, GFLAGS), + GATE(CLK_SATA0_PMALIVE, "clk_sata0_pmalive", "gpll_20m", 0, + RK3568_CLKGATE_CON(11), 1, GFLAGS), + GATE(CLK_SATA0_RXOOB, "clk_sata0_rxoob", "cpll_50m", 0, + RK3568_CLKGATE_CON(11), 2, GFLAGS), + GATE(ACLK_SATA1, "aclk_sata1", "aclk_pipe", 0, + RK3568_CLKGATE_CON(11), 4, GFLAGS), + GATE(CLK_SATA1_PMALIVE, "clk_sata1_pmalive", "gpll_20m", 0, + RK3568_CLKGATE_CON(11), 5, GFLAGS), + GATE(CLK_SATA1_RXOOB, "clk_sata1_rxoob", "cpll_50m", 0, + RK3568_CLKGATE_CON(11), 6, GFLAGS), + GATE(ACLK_SATA2, "aclk_sata2", "aclk_pipe", 0, + RK3568_CLKGATE_CON(11), 8, GFLAGS), + GATE(CLK_SATA2_PMALIVE, "clk_sata2_pmalive", "gpll_20m", 0, + RK3568_CLKGATE_CON(11), 9, GFLAGS), + GATE(CLK_SATA2_RXOOB, "clk_sata2_rxoob", "cpll_50m", 0, + RK3568_CLKGATE_CON(11), 10, GFLAGS), + GATE(ACLK_USB3OTG0, "aclk_usb3otg0", "aclk_pipe", 0, + RK3568_CLKGATE_CON(10), 8, GFLAGS), + GATE(CLK_USB3OTG0_REF, "clk_usb3otg0_ref", "xin24m", 0, + RK3568_CLKGATE_CON(10), 9, GFLAGS), + COMPOSITE_NODIV(CLK_USB3OTG0_SUSPEND, "clk_usb3otg0_suspend", xin24m_32k_p, 0, + RK3568_CLKSEL_CON(29), 8, 1, MFLAGS, + RK3568_CLKGATE_CON(10), 10, GFLAGS), + GATE(ACLK_USB3OTG1, "aclk_usb3otg1", "aclk_pipe", 0, + RK3568_CLKGATE_CON(10), 12, GFLAGS), + GATE(CLK_USB3OTG1_REF, "clk_usb3otg1_ref", "xin24m", 0, + RK3568_CLKGATE_CON(10), 13, GFLAGS), + COMPOSITE_NODIV(CLK_USB3OTG1_SUSPEND, "clk_usb3otg1_suspend", xin24m_32k_p, 0, + RK3568_CLKSEL_CON(29), 9, 1, MFLAGS, + RK3568_CLKGATE_CON(10), 14, GFLAGS), + COMPOSITE_NODIV(CLK_XPCS_EEE, "clk_xpcs_eee", gpll200_cpll125_p, 0, + RK3568_CLKSEL_CON(29), 13, 1, MFLAGS, + RK3568_CLKGATE_CON(10), 4, GFLAGS), + GATE(PCLK_XPCS, "pclk_xpcs", "pclk_pipe", 0, + RK3568_CLKGATE_CON(13), 6, GFLAGS), + + /* PD_PHP */ + COMPOSITE_NODIV(ACLK_PHP, "aclk_php", gpll300_gpll200_gpll100_xin24m_p, 0, + RK3568_CLKSEL_CON(30), 0, 2, MFLAGS, + RK3568_CLKGATE_CON(14), 8, GFLAGS), + COMPOSITE_NODIV(HCLK_PHP, "hclk_php", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(30), 2, 2, MFLAGS, + RK3568_CLKGATE_CON(14), 9, GFLAGS), + COMPOSITE_NOMUX(PCLK_PHP, "pclk_php", "aclk_php", CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(30), 4, 4, DFLAGS, + RK3568_CLKGATE_CON(14), 10, GFLAGS), + GATE(HCLK_SDMMC0, "hclk_sdmmc0", "hclk_php", 0, + RK3568_CLKGATE_CON(15), 0, GFLAGS), + COMPOSITE_NODIV(CLK_SDMMC0, "clk_sdmmc0", clk_sdmmc_p, 0, + RK3568_CLKSEL_CON(30), 8, 3, MFLAGS, + RK3568_CLKGATE_CON(15), 1, GFLAGS), + MMC(SCLK_SDMMC0_DRV, "sdmmc0_drv", "clk_sdmmc0", RK3568_SDMMC0_CON0, 1), + MMC(SCLK_SDMMC0_SAMPLE, "sdmmc0_sample", "clk_sdmmc0", RK3568_SDMMC0_CON1, 1), + + GATE(HCLK_SDMMC1, "hclk_sdmmc1", "hclk_php", 0, + RK3568_CLKGATE_CON(15), 2, GFLAGS), + COMPOSITE_NODIV(CLK_SDMMC1, "clk_sdmmc1", clk_sdmmc_p, 0, + RK3568_CLKSEL_CON(30), 12, 3, MFLAGS, + RK3568_CLKGATE_CON(15), 3, GFLAGS), + MMC(SCLK_SDMMC1_DRV, "sdmmc1_drv", "clk_sdmmc1", RK3568_SDMMC1_CON0, 1), + MMC(SCLK_SDMMC1_SAMPLE, "sdmmc1_sample", "clk_sdmmc1", RK3568_SDMMC1_CON1, 1), + + GATE(ACLK_GMAC0, "aclk_gmac0", "aclk_php", 0, + RK3568_CLKGATE_CON(15), 5, GFLAGS), + GATE(PCLK_GMAC0, "pclk_gmac0", "pclk_php", 0, + RK3568_CLKGATE_CON(15), 6, GFLAGS), + COMPOSITE_NODIV(CLK_MAC0_2TOP, "clk_mac0_2top", clk_mac_2top_p, 0, + RK3568_CLKSEL_CON(31), 8, 2, MFLAGS, + RK3568_CLKGATE_CON(15), 7, GFLAGS), + COMPOSITE_NODIV(CLK_MAC0_OUT, "clk_mac0_out", cpll125_cpll50_cpll25_xin24m_p, 0, + RK3568_CLKSEL_CON(31), 14, 2, MFLAGS, + RK3568_CLKGATE_CON(15), 8, GFLAGS), + GATE(CLK_MAC0_REFOUT, "clk_mac0_refout", "clk_mac0_2top", 0, + RK3568_CLKGATE_CON(15), 12, GFLAGS), + COMPOSITE_NODIV(CLK_GMAC0_PTP_REF, "clk_gmac0_ptp_ref", clk_gmac_ptp_p, 0, + RK3568_CLKSEL_CON(31), 12, 2, MFLAGS, + RK3568_CLKGATE_CON(15), 4, GFLAGS), + MUX(SCLK_GMAC0, "clk_gmac0", mux_gmac0_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK3568_CLKSEL_CON(31), 2, 1, MFLAGS), + FACTOR(0, "clk_gmac0_tx_div5", "clk_gmac0", 0, 1, 5), + FACTOR(0, "clk_gmac0_tx_div50", "clk_gmac0", 0, 1, 50), + FACTOR(0, "clk_gmac0_rx_div2", "clk_gmac0", 0, 1, 2), + FACTOR(0, "clk_gmac0_rx_div20", "clk_gmac0", 0, 1, 20), + MUX(SCLK_GMAC0_RGMII_SPEED, "clk_gmac0_rgmii_speed", mux_gmac0_rgmii_speed_p, 0, + RK3568_CLKSEL_CON(31), 4, 2, MFLAGS), + MUX(SCLK_GMAC0_RMII_SPEED, "clk_gmac0_rmii_speed", mux_gmac0_rmii_speed_p, 0, + RK3568_CLKSEL_CON(31), 3, 1, MFLAGS), + MUX(SCLK_GMAC0_RX_TX, "clk_gmac0_rx_tx", mux_gmac0_rx_tx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(31), 0, 2, MFLAGS), + + /* PD_USB */ + COMPOSITE_NODIV(ACLK_USB, "aclk_usb", gpll300_gpll200_gpll100_xin24m_p, 0, + RK3568_CLKSEL_CON(32), 0, 2, MFLAGS, + RK3568_CLKGATE_CON(16), 0, GFLAGS), + COMPOSITE_NODIV(HCLK_USB, "hclk_usb", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(32), 2, 2, MFLAGS, + RK3568_CLKGATE_CON(16), 1, GFLAGS), + COMPOSITE_NOMUX(PCLK_USB, "pclk_usb", "aclk_usb", 0, + RK3568_CLKSEL_CON(32), 4, 4, DFLAGS, + RK3568_CLKGATE_CON(16), 2, GFLAGS), + GATE(HCLK_USB2HOST0, "hclk_usb2host0", "hclk_usb", 0, + RK3568_CLKGATE_CON(16), 12, GFLAGS), + GATE(HCLK_USB2HOST0_ARB, "hclk_usb2host0_arb", "hclk_usb", 0, + RK3568_CLKGATE_CON(16), 13, GFLAGS), + GATE(HCLK_USB2HOST1, "hclk_usb2host1", "hclk_usb", 0, + RK3568_CLKGATE_CON(16), 14, GFLAGS), + GATE(HCLK_USB2HOST1_ARB, "hclk_usb2host1_arb", "hclk_usb", 0, + RK3568_CLKGATE_CON(16), 15, GFLAGS), + GATE(HCLK_SDMMC2, "hclk_sdmmc2", "hclk_usb", 0, + RK3568_CLKGATE_CON(17), 0, GFLAGS), + COMPOSITE_NODIV(CLK_SDMMC2, "clk_sdmmc2", clk_sdmmc_p, 0, + RK3568_CLKSEL_CON(32), 8, 3, MFLAGS, + RK3568_CLKGATE_CON(17), 1, GFLAGS), + MMC(SCLK_SDMMC2_DRV, "sdmmc2_drv", "clk_sdmmc2", RK3568_SDMMC2_CON0, 1), + MMC(SCLK_SDMMC2_SAMPLE, "sdmmc2_sample", "clk_sdmmc2", RK3568_SDMMC2_CON1, 1), + + GATE(ACLK_GMAC1, "aclk_gmac1", "aclk_usb", 0, + RK3568_CLKGATE_CON(17), 3, GFLAGS), + GATE(PCLK_GMAC1, "pclk_gmac1", "pclk_usb", 0, + RK3568_CLKGATE_CON(17), 4, GFLAGS), + COMPOSITE_NODIV(CLK_MAC1_2TOP, "clk_mac1_2top", clk_mac_2top_p, 0, + RK3568_CLKSEL_CON(33), 8, 2, MFLAGS, + RK3568_CLKGATE_CON(17), 5, GFLAGS), + COMPOSITE_NODIV(CLK_MAC1_OUT, "clk_mac1_out", cpll125_cpll50_cpll25_xin24m_p, 0, + RK3568_CLKSEL_CON(33), 14, 2, MFLAGS, + RK3568_CLKGATE_CON(17), 6, GFLAGS), + GATE(CLK_MAC1_REFOUT, "clk_mac1_refout", "clk_mac1_2top", 0, + RK3568_CLKGATE_CON(17), 10, GFLAGS), + COMPOSITE_NODIV(CLK_GMAC1_PTP_REF, "clk_gmac1_ptp_ref", clk_gmac_ptp_p, 0, + RK3568_CLKSEL_CON(33), 12, 2, MFLAGS, + RK3568_CLKGATE_CON(17), 2, GFLAGS), + MUX(SCLK_GMAC1, "clk_gmac1", mux_gmac1_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK3568_CLKSEL_CON(33), 2, 1, MFLAGS), + FACTOR(0, "clk_gmac1_tx_div5", "clk_gmac1", 0, 1, 5), + FACTOR(0, "clk_gmac1_tx_div50", "clk_gmac1", 0, 1, 50), + FACTOR(0, "clk_gmac1_rx_div2", "clk_gmac1", 0, 1, 2), + FACTOR(0, "clk_gmac1_rx_div20", "clk_gmac1", 0, 1, 20), + MUX(SCLK_GMAC1_RGMII_SPEED, "clk_gmac1_rgmii_speed", mux_gmac1_rgmii_speed_p, 0, + RK3568_CLKSEL_CON(33), 4, 2, MFLAGS), + MUX(SCLK_GMAC1_RMII_SPEED, "clk_gmac1_rmii_speed", mux_gmac1_rmii_speed_p, 0, + RK3568_CLKSEL_CON(33), 3, 1, MFLAGS), + MUX(SCLK_GMAC1_RX_TX, "clk_gmac1_rx_tx", mux_gmac1_rx_tx_p, CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(33), 0, 2, MFLAGS), + + /* PD_PERI */ + COMPOSITE_NODIV(ACLK_PERIMID, "aclk_perimid", gpll300_gpll200_gpll100_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(10), 4, 2, MFLAGS, + RK3568_CLKGATE_CON(14), 0, GFLAGS), + COMPOSITE_NODIV(HCLK_PERIMID, "hclk_perimid", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(10), 6, 2, MFLAGS, + RK3568_CLKGATE_CON(14), 1, GFLAGS), + + /* PD_VI */ + COMPOSITE_NODIV(ACLK_VI, "aclk_vi", gpll400_gpll300_gpll200_xin24m_p, 0, + RK3568_CLKSEL_CON(34), 0, 2, MFLAGS, + RK3568_CLKGATE_CON(18), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_VI, "hclk_vi", "aclk_vi", 0, + RK3568_CLKSEL_CON(34), 4, 4, DFLAGS, + RK3568_CLKGATE_CON(18), 1, GFLAGS), + COMPOSITE_NOMUX(PCLK_VI, "pclk_vi", "aclk_vi", 0, + RK3568_CLKSEL_CON(34), 8, 4, DFLAGS, + RK3568_CLKGATE_CON(18), 2, GFLAGS), + GATE(ACLK_VICAP, "aclk_vicap", "aclk_vi", 0, + RK3568_CLKGATE_CON(18), 9, GFLAGS), + GATE(HCLK_VICAP, "hclk_vicap", "hclk_vi", 0, + RK3568_CLKGATE_CON(18), 10, GFLAGS), + COMPOSITE_NODIV(DCLK_VICAP, "dclk_vicap", cpll333_gpll300_gpll200_p, 0, + RK3568_CLKSEL_CON(34), 14, 2, MFLAGS, + RK3568_CLKGATE_CON(18), 11, GFLAGS), + GATE(ICLK_VICAP_G, "iclk_vicap_g", "iclk_vicap", 0, + RK3568_CLKGATE_CON(18), 13, GFLAGS), + GATE(ACLK_ISP, "aclk_isp", "aclk_vi", 0, + RK3568_CLKGATE_CON(19), 0, GFLAGS), + GATE(HCLK_ISP, "hclk_isp", "hclk_vi", 0, + RK3568_CLKGATE_CON(19), 1, GFLAGS), + COMPOSITE(CLK_ISP, "clk_isp", cpll_gpll_hpll_p, 0, + RK3568_CLKSEL_CON(35), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(19), 2, GFLAGS), + GATE(PCLK_CSI2HOST1, "pclk_csi2host1", "pclk_vi", 0, + RK3568_CLKGATE_CON(19), 4, GFLAGS), + COMPOSITE(CLK_CIF_OUT, "clk_cif_out", gpll_usb480m_xin24m_p, 0, + RK3568_CLKSEL_CON(35), 14, 2, MFLAGS, 8, 6, DFLAGS, + RK3568_CLKGATE_CON(19), 8, GFLAGS), + COMPOSITE(CLK_CAM0_OUT, "clk_cam0_out", gpll_usb480m_xin24m_p, 0, + RK3568_CLKSEL_CON(36), 6, 2, MFLAGS, 0, 6, DFLAGS, + RK3568_CLKGATE_CON(19), 9, GFLAGS), + COMPOSITE(CLK_CAM1_OUT, "clk_cam1_out", gpll_usb480m_xin24m_p, 0, + RK3568_CLKSEL_CON(36), 14, 2, MFLAGS, 8, 6, DFLAGS, + RK3568_CLKGATE_CON(19), 10, GFLAGS), + + /* PD_VO */ + COMPOSITE_NODIV(ACLK_VO, "aclk_vo", gpll300_cpll250_gpll100_xin24m_p, 0, + RK3568_CLKSEL_CON(37), 0, 2, MFLAGS, + RK3568_CLKGATE_CON(20), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_VO, "hclk_vo", "aclk_vo", 0, + RK3568_CLKSEL_CON(37), 8, 4, DFLAGS, + RK3568_CLKGATE_CON(20), 1, GFLAGS), + COMPOSITE_NOMUX(PCLK_VO, "pclk_vo", "aclk_vo", 0, + RK3568_CLKSEL_CON(37), 12, 4, DFLAGS, + RK3568_CLKGATE_CON(20), 2, GFLAGS), + COMPOSITE(ACLK_VOP_PRE, "aclk_vop_pre", cpll_gpll_hpll_vpll_p, 0, + RK3568_CLKSEL_CON(38), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(20), 6, GFLAGS), + GATE(ACLK_VOP, "aclk_vop", "aclk_vop_pre", 0, + RK3568_CLKGATE_CON(20), 8, GFLAGS), + GATE(HCLK_VOP, "hclk_vop", "hclk_vo", 0, + RK3568_CLKGATE_CON(20), 9, GFLAGS), + COMPOSITE(DCLK_VOP0, "dclk_vop0", hpll_vpll_gpll_cpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK3568_CLKSEL_CON(39), 10, 2, MFLAGS, 0, 8, DFLAGS, + RK3568_CLKGATE_CON(20), 10, GFLAGS), + COMPOSITE_DCLK(DCLK_VOP1, "dclk_vop1", hpll_vpll_gpll_cpll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RK3568_CLKSEL_CON(40), 10, 2, MFLAGS, 0, 8, DFLAGS, + RK3568_CLKGATE_CON(20), 11, GFLAGS, RK3568_DCLK_PARENT_MAX_PRATE), + COMPOSITE(DCLK_VOP2, "dclk_vop2", hpll_vpll_gpll_cpll_p, 0, + RK3568_CLKSEL_CON(41), 10, 2, MFLAGS, 0, 8, DFLAGS, + RK3568_CLKGATE_CON(20), 12, GFLAGS), + GATE(CLK_VOP_PWM, "clk_vop_pwm", "xin24m", 0, + RK3568_CLKGATE_CON(20), 13, GFLAGS), + GATE(ACLK_HDCP, "aclk_hdcp", "aclk_vo", 0, + RK3568_CLKGATE_CON(21), 0, GFLAGS), + GATE(HCLK_HDCP, "hclk_hdcp", "hclk_vo", 0, + RK3568_CLKGATE_CON(21), 1, GFLAGS), + GATE(PCLK_HDCP, "pclk_hdcp", "pclk_vo", 0, + RK3568_CLKGATE_CON(21), 2, GFLAGS), + GATE(PCLK_HDMI_HOST, "pclk_hdmi_host", "pclk_vo", 0, + RK3568_CLKGATE_CON(21), 3, GFLAGS), + GATE(CLK_HDMI_SFR, "clk_hdmi_sfr", "xin24m", 0, + RK3568_CLKGATE_CON(21), 4, GFLAGS), + GATE(CLK_HDMI_CEC, "clk_hdmi_cec", "clk_rtc_32k", 0, + RK3568_CLKGATE_CON(21), 5, GFLAGS), + GATE(PCLK_DSITX_0, "pclk_dsitx_0", "pclk_vo", 0, + RK3568_CLKGATE_CON(21), 6, GFLAGS), + GATE(PCLK_DSITX_1, "pclk_dsitx_1", "pclk_vo", 0, + RK3568_CLKGATE_CON(21), 7, GFLAGS), + GATE(PCLK_EDP_CTRL, "pclk_edp_ctrl", "pclk_vo", 0, + RK3568_CLKGATE_CON(21), 8, GFLAGS), + COMPOSITE_NODIV(CLK_EDP_200M, "clk_edp_200m", gpll200_gpll150_cpll125_p, 0, + RK3568_CLKSEL_CON(38), 8, 2, MFLAGS, + RK3568_CLKGATE_CON(21), 9, GFLAGS), + + /* PD_VPU */ + COMPOSITE(ACLK_VPU_PRE, "aclk_vpu_pre", gpll_cpll_p, 0, + RK3568_CLKSEL_CON(42), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(22), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_VPU_PRE, "hclk_vpu_pre", "aclk_vpu_pre", 0, + RK3568_CLKSEL_CON(42), 8, 4, DFLAGS, + RK3568_CLKGATE_CON(22), 1, GFLAGS), + GATE(ACLK_VPU, "aclk_vpu", "aclk_vpu_pre", 0, + RK3568_CLKGATE_CON(22), 4, GFLAGS), + GATE(HCLK_VPU, "hclk_vpu", "hclk_vpu_pre", 0, + RK3568_CLKGATE_CON(22), 5, GFLAGS), + + /* PD_RGA */ + COMPOSITE_NODIV(ACLK_RGA_PRE, "aclk_rga_pre", gpll300_cpll250_gpll100_xin24m_p, 0, + RK3568_CLKSEL_CON(43), 0, 2, MFLAGS, + RK3568_CLKGATE_CON(23), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_RGA_PRE, "hclk_rga_pre", "aclk_rga_pre", 0, + RK3568_CLKSEL_CON(43), 8, 4, DFLAGS, + RK3568_CLKGATE_CON(23), 1, GFLAGS), + COMPOSITE_NOMUX(PCLK_RGA_PRE, "pclk_rga_pre", "aclk_rga_pre", 0, + RK3568_CLKSEL_CON(43), 12, 4, DFLAGS, + RK3568_CLKGATE_CON(22), 12, GFLAGS), + GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, + RK3568_CLKGATE_CON(23), 4, GFLAGS), + GATE(HCLK_RGA, "hclk_rga", "hclk_rga_pre", 0, + RK3568_CLKGATE_CON(23), 5, GFLAGS), + COMPOSITE_NODIV(CLK_RGA_CORE, "clk_rga_core", gpll300_gpll200_gpll100_p, 0, + RK3568_CLKSEL_CON(43), 2, 2, MFLAGS, + RK3568_CLKGATE_CON(23), 6, GFLAGS), + GATE(ACLK_IEP, "aclk_iep", "aclk_rga_pre", 0, + RK3568_CLKGATE_CON(23), 7, GFLAGS), + GATE(HCLK_IEP, "hclk_iep", "hclk_rga_pre", 0, + RK3568_CLKGATE_CON(23), 8, GFLAGS), + COMPOSITE_NODIV(CLK_IEP_CORE, "clk_iep_core", gpll300_gpll200_gpll100_p, 0, + RK3568_CLKSEL_CON(43), 4, 2, MFLAGS, + RK3568_CLKGATE_CON(23), 9, GFLAGS), + GATE(HCLK_EBC, "hclk_ebc", "hclk_rga_pre", 0, RK3568_CLKGATE_CON(23), 10, GFLAGS), + COMPOSITE_NODIV(DCLK_EBC, "dclk_ebc", gpll400_cpll333_gpll200_p, 0, + RK3568_CLKSEL_CON(43), 6, 2, MFLAGS, + RK3568_CLKGATE_CON(23), 11, GFLAGS), + GATE(ACLK_JDEC, "aclk_jdec", "aclk_rga_pre", 0, + RK3568_CLKGATE_CON(23), 12, GFLAGS), + GATE(HCLK_JDEC, "hclk_jdec", "hclk_rga_pre", 0, + RK3568_CLKGATE_CON(23), 13, GFLAGS), + GATE(ACLK_JENC, "aclk_jenc", "aclk_rga_pre", 0, + RK3568_CLKGATE_CON(23), 14, GFLAGS), + GATE(HCLK_JENC, "hclk_jenc", "hclk_rga_pre", 0, + RK3568_CLKGATE_CON(23), 15, GFLAGS), + GATE(PCLK_EINK, "pclk_eink", "pclk_rga_pre", 0, + RK3568_CLKGATE_CON(22), 14, GFLAGS), + GATE(HCLK_EINK, "hclk_eink", "hclk_rga_pre", 0, + RK3568_CLKGATE_CON(22), 15, GFLAGS), + + /* PD_RKVENC */ + COMPOSITE(ACLK_RKVENC_PRE, "aclk_rkvenc_pre", gpll_cpll_npll_p, 0, + RK3568_CLKSEL_CON(44), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(24), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_RKVENC_PRE, "hclk_rkvenc_pre", "aclk_rkvenc_pre", 0, + RK3568_CLKSEL_CON(44), 8, 4, DFLAGS, + RK3568_CLKGATE_CON(24), 1, GFLAGS), + GATE(ACLK_RKVENC, "aclk_rkvenc", "aclk_rkvenc_pre", 0, + RK3568_CLKGATE_CON(24), 6, GFLAGS), + GATE(HCLK_RKVENC, "hclk_rkvenc", "hclk_rkvenc_pre", 0, + RK3568_CLKGATE_CON(24), 7, GFLAGS), + COMPOSITE(CLK_RKVENC_CORE, "clk_rkvenc_core", gpll_cpll_npll_vpll_p, 0, + RK3568_CLKSEL_CON(45), 14, 2, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(24), 8, GFLAGS), + COMPOSITE(ACLK_RKVDEC_PRE, "aclk_rkvdec_pre", aclk_rkvdec_pre_p, CLK_SET_RATE_NO_REPARENT, + RK3568_CLKSEL_CON(47), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(25), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_RKVDEC_PRE, "hclk_rkvdec_pre", "aclk_rkvdec_pre", 0, + RK3568_CLKSEL_CON(47), 8, 4, DFLAGS, + RK3568_CLKGATE_CON(25), 1, GFLAGS), + GATE(ACLK_RKVDEC, "aclk_rkvdec", "aclk_rkvdec_pre", 0, + RK3568_CLKGATE_CON(25), 4, GFLAGS), + GATE(HCLK_RKVDEC, "hclk_rkvdec", "hclk_rkvdec_pre", 0, + RK3568_CLKGATE_CON(25), 5, GFLAGS), + COMPOSITE(CLK_RKVDEC_CA, "clk_rkvdec_ca", gpll_cpll_npll_vpll_p, 0, + RK3568_CLKSEL_CON(48), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(25), 6, GFLAGS), + COMPOSITE(CLK_RKVDEC_CORE, "clk_rkvdec_core", clk_rkvdec_core_p, CLK_SET_RATE_NO_REPARENT, + RK3568_CLKSEL_CON(49), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK3568_CLKGATE_CON(25), 7, GFLAGS), + COMPOSITE(CLK_RKVDEC_HEVC_CA, "clk_rkvdec_hevc_ca", gpll_cpll_npll_vpll_p, 0, + RK3568_CLKSEL_CON(49), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(25), 8, GFLAGS), + + /* PD_BUS */ + COMPOSITE_NODIV(ACLK_BUS, "aclk_bus", gpll200_gpll150_gpll100_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(50), 0, 2, MFLAGS, + RK3568_CLKGATE_CON(26), 0, GFLAGS), + COMPOSITE_NODIV(PCLK_BUS, "pclk_bus", gpll100_gpll75_cpll50_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(50), 4, 2, MFLAGS, + RK3568_CLKGATE_CON(26), 1, GFLAGS), + GATE(PCLK_TSADC, "pclk_tsadc", "pclk_bus", 0, + RK3568_CLKGATE_CON(26), 4, GFLAGS), + COMPOSITE(CLK_TSADC_TSEN, "clk_tsadc_tsen", xin24m_gpll100_cpll100_p, 0, + RK3568_CLKSEL_CON(51), 4, 2, MFLAGS, 0, 3, DFLAGS, + RK3568_CLKGATE_CON(26), 5, GFLAGS), + COMPOSITE_NOMUX(CLK_TSADC, "clk_tsadc", "clk_tsadc_tsen", 0, + RK3568_CLKSEL_CON(51), 8, 7, DFLAGS, + RK3568_CLKGATE_CON(26), 6, GFLAGS), + GATE(PCLK_SARADC, "pclk_saradc", "pclk_bus", 0, + RK3568_CLKGATE_CON(26), 7, GFLAGS), + GATE(CLK_SARADC, "clk_saradc", "xin24m", 0, + RK3568_CLKGATE_CON(26), 8, GFLAGS), + GATE(PCLK_SCR, "pclk_scr", "pclk_bus", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(26), 12, GFLAGS), + GATE(PCLK_WDT_NS, "pclk_wdt_ns", "pclk_bus", 0, + RK3568_CLKGATE_CON(26), 13, GFLAGS), + GATE(TCLK_WDT_NS, "tclk_wdt_ns", "xin24m", 0, + RK3568_CLKGATE_CON(26), 14, GFLAGS), + GATE(ACLK_MCU, "aclk_mcu", "aclk_bus", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(32), 13, GFLAGS), + GATE(PCLK_INTMUX, "pclk_intmux", "pclk_bus", CLK_IGNORE_UNUSED, + RK3568_CLKGATE_CON(32), 14, GFLAGS), + GATE(PCLK_MAILBOX, "pclk_mailbox", "pclk_bus", 0, + RK3568_CLKGATE_CON(32), 15, GFLAGS), + + GATE(PCLK_UART1, "pclk_uart1", "pclk_bus", 0, + RK3568_CLKGATE_CON(27), 12, GFLAGS), + COMPOSITE(CLK_UART1_SRC, "clk_uart1_src", gpll_cpll_usb480m_p, 0, + RK3568_CLKSEL_CON(52), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(27), 13, GFLAGS), + COMPOSITE_FRACMUX(CLK_UART1_FRAC, "clk_uart1_frac", "clk_uart1_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(53), CLK_FRAC_DIVIDER_NO_LIMIT, + RK3568_CLKGATE_CON(27), 14, GFLAGS, + &rk3568_uart1_fracmux, RK3568_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART1, "sclk_uart1", "sclk_uart1_mux", 0, + RK3568_CLKGATE_CON(27), 15, GFLAGS), + + GATE(PCLK_UART2, "pclk_uart2", "pclk_bus", 0, + RK3568_CLKGATE_CON(28), 0, GFLAGS), + COMPOSITE(CLK_UART2_SRC, "clk_uart2_src", gpll_cpll_usb480m_p, 0, + RK3568_CLKSEL_CON(54), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(28), 1, GFLAGS), + COMPOSITE_FRACMUX(CLK_UART2_FRAC, "clk_uart2_frac", "clk_uart2_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(55), CLK_FRAC_DIVIDER_NO_LIMIT, + RK3568_CLKGATE_CON(28), 2, GFLAGS, + &rk3568_uart2_fracmux, RK3568_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART2, "sclk_uart2", "sclk_uart2_mux", 0, + RK3568_CLKGATE_CON(28), 3, GFLAGS), + + GATE(PCLK_UART3, "pclk_uart3", "pclk_bus", 0, + RK3568_CLKGATE_CON(28), 4, GFLAGS), + COMPOSITE(CLK_UART3_SRC, "clk_uart3_src", gpll_cpll_usb480m_p, 0, + RK3568_CLKSEL_CON(56), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(28), 5, GFLAGS), + COMPOSITE_FRACMUX(CLK_UART3_FRAC, "clk_uart3_frac", "clk_uart3_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(57), CLK_FRAC_DIVIDER_NO_LIMIT, + RK3568_CLKGATE_CON(28), 6, GFLAGS, + &rk3568_uart3_fracmux, RK3568_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART3, "sclk_uart3", "sclk_uart3_mux", 0, + RK3568_CLKGATE_CON(28), 7, GFLAGS), + + GATE(PCLK_UART4, "pclk_uart4", "pclk_bus", 0, + RK3568_CLKGATE_CON(28), 8, GFLAGS), + COMPOSITE(CLK_UART4_SRC, "clk_uart4_src", gpll_cpll_usb480m_p, 0, + RK3568_CLKSEL_CON(58), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(28), 9, GFLAGS), + COMPOSITE_FRACMUX(CLK_UART4_FRAC, "clk_uart4_frac", "clk_uart4_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(59), CLK_FRAC_DIVIDER_NO_LIMIT, + RK3568_CLKGATE_CON(28), 10, GFLAGS, + &rk3568_uart4_fracmux, RK3568_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART4, "sclk_uart4", "sclk_uart4_mux", 0, + RK3568_CLKGATE_CON(28), 11, GFLAGS), + + GATE(PCLK_UART5, "pclk_uart5", "pclk_bus", 0, + RK3568_CLKGATE_CON(28), 12, GFLAGS), + COMPOSITE(CLK_UART5_SRC, "clk_uart5_src", gpll_cpll_usb480m_p, 0, + RK3568_CLKSEL_CON(60), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(28), 13, GFLAGS), + COMPOSITE_FRACMUX(CLK_UART5_FRAC, "clk_uart5_frac", "clk_uart5_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(61), CLK_FRAC_DIVIDER_NO_LIMIT, + RK3568_CLKGATE_CON(28), 14, GFLAGS, + &rk3568_uart5_fracmux, RK3568_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART5, "sclk_uart5", "sclk_uart5_mux", 0, + RK3568_CLKGATE_CON(28), 15, GFLAGS), + + GATE(PCLK_UART6, "pclk_uart6", "pclk_bus", 0, + RK3568_CLKGATE_CON(29), 0, GFLAGS), + COMPOSITE(CLK_UART6_SRC, "clk_uart6_src", gpll_cpll_usb480m_p, 0, + RK3568_CLKSEL_CON(62), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(29), 1, GFLAGS), + COMPOSITE_FRACMUX(CLK_UART6_FRAC, "clk_uart6_frac", "clk_uart6_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(63), CLK_FRAC_DIVIDER_NO_LIMIT, + RK3568_CLKGATE_CON(29), 2, GFLAGS, + &rk3568_uart6_fracmux, RK3568_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART6, "sclk_uart6", "sclk_uart6_mux", 0, + RK3568_CLKGATE_CON(29), 3, GFLAGS), + + GATE(PCLK_UART7, "pclk_uart7", "pclk_bus", 0, + RK3568_CLKGATE_CON(29), 4, GFLAGS), + COMPOSITE(CLK_UART7_SRC, "clk_uart7_src", gpll_cpll_usb480m_p, 0, + RK3568_CLKSEL_CON(64), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(29), 5, GFLAGS), + COMPOSITE_FRACMUX(CLK_UART7_FRAC, "clk_uart7_frac", "clk_uart7_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(65), CLK_FRAC_DIVIDER_NO_LIMIT, + RK3568_CLKGATE_CON(29), 6, GFLAGS, + &rk3568_uart7_fracmux, RK3568_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART7, "sclk_uart7", "sclk_uart7_mux", 0, + RK3568_CLKGATE_CON(29), 7, GFLAGS), + + GATE(PCLK_UART8, "pclk_uart8", "pclk_bus", 0, + RK3568_CLKGATE_CON(29), 8, GFLAGS), + COMPOSITE(CLK_UART8_SRC, "clk_uart8_src", gpll_cpll_usb480m_p, 0, + RK3568_CLKSEL_CON(66), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(29), 9, GFLAGS), + COMPOSITE_FRACMUX(CLK_UART8_FRAC, "clk_uart8_frac", "clk_uart8_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(67), CLK_FRAC_DIVIDER_NO_LIMIT, + RK3568_CLKGATE_CON(29), 10, GFLAGS, + &rk3568_uart8_fracmux, RK3568_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART8, "sclk_uart8", "sclk_uart8_mux", 0, + RK3568_CLKGATE_CON(29), 11, GFLAGS), + + GATE(PCLK_UART9, "pclk_uart9", "pclk_bus", 0, + RK3568_CLKGATE_CON(29), 12, GFLAGS), + COMPOSITE(CLK_UART9_SRC, "clk_uart9_src", gpll_cpll_usb480m_p, 0, + RK3568_CLKSEL_CON(68), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_CLKGATE_CON(29), 13, GFLAGS), + COMPOSITE_FRACMUX(CLK_UART9_FRAC, "clk_uart9_frac", "clk_uart9_src", CLK_SET_RATE_PARENT, + RK3568_CLKSEL_CON(69), CLK_FRAC_DIVIDER_NO_LIMIT, + RK3568_CLKGATE_CON(29), 14, GFLAGS, + &rk3568_uart9_fracmux, RK3568_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART9, "sclk_uart9", "sclk_uart9_mux", 0, + RK3568_CLKGATE_CON(29), 15, GFLAGS), + + GATE(PCLK_CAN0, "pclk_can0", "pclk_bus", 0, + RK3568_CLKGATE_CON(27), 5, GFLAGS), + COMPOSITE(CLK_CAN0, "clk_can0", gpll_cpll_p, 0, + RK3568_CLKSEL_CON(70), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(27), 6, GFLAGS), + GATE(PCLK_CAN1, "pclk_can1", "pclk_bus", 0, + RK3568_CLKGATE_CON(27), 7, GFLAGS), + COMPOSITE(CLK_CAN1, "clk_can1", gpll_cpll_p, 0, + RK3568_CLKSEL_CON(70), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK3568_CLKGATE_CON(27), 8, GFLAGS), + GATE(PCLK_CAN2, "pclk_can2", "pclk_bus", 0, + RK3568_CLKGATE_CON(27), 9, GFLAGS), + COMPOSITE(CLK_CAN2, "clk_can2", gpll_cpll_p, 0, + RK3568_CLKSEL_CON(71), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK3568_CLKGATE_CON(27), 10, GFLAGS), + COMPOSITE_NODIV(CLK_I2C, "clk_i2c", clk_i2c_p, 0, + RK3568_CLKSEL_CON(71), 8, 2, MFLAGS, + RK3568_CLKGATE_CON(32), 10, GFLAGS), + GATE(PCLK_I2C1, "pclk_i2c1", "pclk_bus", 0, + RK3568_CLKGATE_CON(30), 0, GFLAGS), + GATE(CLK_I2C1, "clk_i2c1", "clk_i2c", 0, + RK3568_CLKGATE_CON(30), 1, GFLAGS), + GATE(PCLK_I2C2, "pclk_i2c2", "pclk_bus", 0, + RK3568_CLKGATE_CON(30), 2, GFLAGS), + GATE(CLK_I2C2, "clk_i2c2", "clk_i2c", 0, + RK3568_CLKGATE_CON(30), 3, GFLAGS), + GATE(PCLK_I2C3, "pclk_i2c3", "pclk_bus", 0, + RK3568_CLKGATE_CON(30), 4, GFLAGS), + GATE(CLK_I2C3, "clk_i2c3", "clk_i2c", 0, + RK3568_CLKGATE_CON(30), 5, GFLAGS), + GATE(PCLK_I2C4, "pclk_i2c4", "pclk_bus", 0, + RK3568_CLKGATE_CON(30), 6, GFLAGS), + GATE(CLK_I2C4, "clk_i2c4", "clk_i2c", 0, + RK3568_CLKGATE_CON(30), 7, GFLAGS), + GATE(PCLK_I2C5, "pclk_i2c5", "pclk_bus", 0, + RK3568_CLKGATE_CON(30), 8, GFLAGS), + GATE(CLK_I2C5, "clk_i2c5", "clk_i2c", 0, + RK3568_CLKGATE_CON(30), 9, GFLAGS), + GATE(PCLK_SPI0, "pclk_spi0", "pclk_bus", 0, + RK3568_CLKGATE_CON(30), 10, GFLAGS), + COMPOSITE_NODIV(CLK_SPI0, "clk_spi0", gpll200_xin24m_cpll100_p, 0, + RK3568_CLKSEL_CON(72), 0, 1, MFLAGS, + RK3568_CLKGATE_CON(30), 11, GFLAGS), + GATE(PCLK_SPI1, "pclk_spi1", "pclk_bus", 0, + RK3568_CLKGATE_CON(30), 12, GFLAGS), + COMPOSITE_NODIV(CLK_SPI1, "clk_spi1", gpll200_xin24m_cpll100_p, 0, + RK3568_CLKSEL_CON(72), 2, 1, MFLAGS, + RK3568_CLKGATE_CON(30), 13, GFLAGS), + GATE(PCLK_SPI2, "pclk_spi2", "pclk_bus", 0, + RK3568_CLKGATE_CON(30), 14, GFLAGS), + COMPOSITE_NODIV(CLK_SPI2, "clk_spi2", gpll200_xin24m_cpll100_p, 0, + RK3568_CLKSEL_CON(72), 4, 1, MFLAGS, + RK3568_CLKGATE_CON(30), 15, GFLAGS), + GATE(PCLK_SPI3, "pclk_spi3", "pclk_bus", 0, + RK3568_CLKGATE_CON(31), 0, GFLAGS), + COMPOSITE_NODIV(CLK_SPI3, "clk_spi3", gpll200_xin24m_cpll100_p, 0, + RK3568_CLKSEL_CON(72), 6, 1, MFLAGS, RK3568_CLKGATE_CON(31), 1, GFLAGS), + GATE(PCLK_PWM1, "pclk_pwm1", "pclk_bus", 0, RK3568_CLKGATE_CON(31), 10, GFLAGS), + COMPOSITE_NODIV(CLK_PWM1, "clk_pwm1", gpll100_xin24m_cpll100_p, 0, + RK3568_CLKSEL_CON(72), 8, 1, MFLAGS, + RK3568_CLKGATE_CON(31), 11, GFLAGS), + GATE(CLK_PWM1_CAPTURE, "clk_pwm1_capture", "xin24m", 0, + RK3568_CLKGATE_CON(31), 12, GFLAGS), + GATE(PCLK_PWM2, "pclk_pwm2", "pclk_bus", 0, + RK3568_CLKGATE_CON(31), 13, GFLAGS), + COMPOSITE_NODIV(CLK_PWM2, "clk_pwm2", gpll100_xin24m_cpll100_p, 0, + RK3568_CLKSEL_CON(72), 10, 1, MFLAGS, + RK3568_CLKGATE_CON(31), 14, GFLAGS), + GATE(CLK_PWM2_CAPTURE, "clk_pwm2_capture", "xin24m", 0, + RK3568_CLKGATE_CON(31), 15, GFLAGS), + GATE(PCLK_PWM3, "pclk_pwm3", "pclk_bus", 0, + RK3568_CLKGATE_CON(32), 0, GFLAGS), + COMPOSITE_NODIV(CLK_PWM3, "clk_pwm3", gpll100_xin24m_cpll100_p, 0, + RK3568_CLKSEL_CON(72), 12, 1, MFLAGS, + RK3568_CLKGATE_CON(32), 1, GFLAGS), + GATE(CLK_PWM3_CAPTURE, "clk_pwm3_capture", "xin24m", 0, + RK3568_CLKGATE_CON(32), 2, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO, "dbclk_gpio", xin24m_32k_p, 0, + RK3568_CLKSEL_CON(72), 14, 1, MFLAGS, + RK3568_CLKGATE_CON(32), 11, GFLAGS), + GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_bus", 0, + RK3568_CLKGATE_CON(31), 2, GFLAGS), + GATE(DBCLK_GPIO1, "dbclk_gpio1", "dbclk_gpio", 0, + RK3568_CLKGATE_CON(31), 3, GFLAGS), + GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_bus", 0, + RK3568_CLKGATE_CON(31), 4, GFLAGS), + GATE(DBCLK_GPIO2, "dbclk_gpio2", "dbclk_gpio", 0, + RK3568_CLKGATE_CON(31), 5, GFLAGS), + GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_bus", 0, + RK3568_CLKGATE_CON(31), 6, GFLAGS), + GATE(DBCLK_GPIO3, "dbclk_gpio3", "dbclk_gpio", 0, + RK3568_CLKGATE_CON(31), 7, GFLAGS), + GATE(PCLK_GPIO4, "pclk_gpio4", "pclk_bus", 0, + RK3568_CLKGATE_CON(31), 8, GFLAGS), + GATE(DBCLK_GPIO4, "dbclk_gpio4", "dbclk_gpio", 0, + RK3568_CLKGATE_CON(31), 9, GFLAGS), + GATE(PCLK_TIMER, "pclk_timer", "pclk_bus", 0, + RK3568_CLKGATE_CON(32), 3, GFLAGS), + GATE(CLK_TIMER0, "clk_timer0", "xin24m", 0, + RK3568_CLKGATE_CON(32), 4, GFLAGS), + GATE(CLK_TIMER1, "clk_timer1", "xin24m", 0, + RK3568_CLKGATE_CON(32), 5, GFLAGS), + GATE(CLK_TIMER2, "clk_timer2", "xin24m", 0, + RK3568_CLKGATE_CON(32), 6, GFLAGS), + GATE(CLK_TIMER3, "clk_timer3", "xin24m", 0, + RK3568_CLKGATE_CON(32), 7, GFLAGS), + GATE(CLK_TIMER4, "clk_timer4", "xin24m", 0, + RK3568_CLKGATE_CON(32), 8, GFLAGS), + GATE(CLK_TIMER5, "clk_timer5", "xin24m", 0, + RK3568_CLKGATE_CON(32), 9, GFLAGS), + + /* PD_TOP */ + COMPOSITE_NODIV(ACLK_TOP_HIGH, "aclk_top_high", cpll500_gpll400_gpll300_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(73), 0, 2, MFLAGS, + RK3568_CLKGATE_CON(33), 0, GFLAGS), + COMPOSITE_NODIV(ACLK_TOP_LOW, "aclk_top_low", gpll400_gpll300_gpll200_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(73), 4, 2, MFLAGS, + RK3568_CLKGATE_CON(33), 1, GFLAGS), + COMPOSITE_NODIV(HCLK_TOP, "hclk_top", gpll150_gpll100_gpll75_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(73), 8, 2, MFLAGS, + RK3568_CLKGATE_CON(33), 2, GFLAGS), + COMPOSITE_NODIV(PCLK_TOP, "pclk_top", gpll100_gpll75_cpll50_xin24m_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(73), 12, 2, MFLAGS, + RK3568_CLKGATE_CON(33), 3, GFLAGS), + GATE(PCLK_PCIE30PHY, "pclk_pcie30phy", "pclk_top", 0, + RK3568_CLKGATE_CON(33), 8, GFLAGS), + COMPOSITE_NODIV(CLK_OPTC_ARB, "clk_optc_arb", xin24m_cpll100_p, CLK_IS_CRITICAL, + RK3568_CLKSEL_CON(73), 15, 1, MFLAGS, + RK3568_CLKGATE_CON(33), 9, GFLAGS), + GATE(PCLK_MIPICSIPHY, "pclk_mipicsiphy", "pclk_top", 0, + RK3568_CLKGATE_CON(33), 13, GFLAGS), + GATE(PCLK_MIPIDSIPHY0, "pclk_mipidsiphy0", "pclk_top", 0, + RK3568_CLKGATE_CON(33), 14, GFLAGS), + GATE(PCLK_MIPIDSIPHY1, "pclk_mipidsiphy1", "pclk_top", 0, + RK3568_CLKGATE_CON(33), 15, GFLAGS), + GATE(PCLK_PIPEPHY0, "pclk_pipephy0", "pclk_top", 0, + RK3568_CLKGATE_CON(34), 4, GFLAGS), + GATE(PCLK_PIPEPHY1, "pclk_pipephy1", "pclk_top", 0, + RK3568_CLKGATE_CON(34), 5, GFLAGS), + GATE(PCLK_PIPEPHY2, "pclk_pipephy2", "pclk_top", 0, + RK3568_CLKGATE_CON(34), 6, GFLAGS), + GATE(PCLK_CPU_BOOST, "pclk_cpu_boost", "pclk_top", 0, + RK3568_CLKGATE_CON(34), 11, GFLAGS), + GATE(CLK_CPU_BOOST, "clk_cpu_boost", "xin24m", 0, + RK3568_CLKGATE_CON(34), 12, GFLAGS), + GATE(PCLK_OTPPHY, "pclk_otpphy", "pclk_top", 0, + RK3568_CLKGATE_CON(34), 13, GFLAGS), + GATE(PCLK_EDPPHY_GRF, "pclk_edpphy_grf", "pclk_top", 0, + RK3568_CLKGATE_CON(34), 14, GFLAGS), +}; + +static struct rockchip_clk_branch rk3568_clk_pmu_branches[] __initdata = { + /* PD_PMU */ + FACTOR(0, "ppll_ph0", "ppll", 0, 1, 2), + FACTOR(0, "ppll_ph180", "ppll", 0, 1, 2), + FACTOR(0, "hpll_ph0", "hpll", 0, 1, 2), + + MUX(CLK_PDPMU, "clk_pdpmu", clk_pdpmu_p, 0, + RK3568_PMU_CLKSEL_CON(2), 15, 1, MFLAGS), + COMPOSITE_NOMUX(PCLK_PDPMU, "pclk_pdpmu", "clk_pdpmu", CLK_IS_CRITICAL, + RK3568_PMU_CLKSEL_CON(2), 0, 5, DFLAGS, + RK3568_PMU_CLKGATE_CON(0), 2, GFLAGS), + GATE(PCLK_PMU, "pclk_pmu", "pclk_pdpmu", CLK_IS_CRITICAL, + RK3568_PMU_CLKGATE_CON(0), 6, GFLAGS), + GATE(CLK_PMU, "clk_pmu", "xin24m", CLK_IS_CRITICAL, + RK3568_PMU_CLKGATE_CON(0), 7, GFLAGS), + GATE(PCLK_I2C0, "pclk_i2c0", "pclk_pdpmu", 0, + RK3568_PMU_CLKGATE_CON(1), 0, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C0, "clk_i2c0", "clk_pdpmu", 0, + RK3568_PMU_CLKSEL_CON(3), 0, 7, DFLAGS, + RK3568_PMU_CLKGATE_CON(1), 1, GFLAGS), + GATE(PCLK_UART0, "pclk_uart0", "pclk_pdpmu", 0, + RK3568_PMU_CLKGATE_CON(1), 2, GFLAGS), + + COMPOSITE_FRACMUX(CLK_RTC32K_FRAC, "clk_rtc32k_frac", "xin24m", CLK_IGNORE_UNUSED, + RK3568_PMU_CLKSEL_CON(1), 0, + RK3568_PMU_CLKGATE_CON(0), 1, GFLAGS, + &rk3568_rtc32k_pmu_fracmux, 0), + + COMPOSITE_NOMUX(XIN_OSC0_DIV, "xin_osc0_div", "xin24m", CLK_IGNORE_UNUSED, + RK3568_PMU_CLKSEL_CON(0), 0, 5, DFLAGS, + RK3568_PMU_CLKGATE_CON(0), 0, GFLAGS), + + COMPOSITE(CLK_UART0_DIV, "sclk_uart0_div", ppll_usb480m_cpll_gpll_p, 0, + RK3568_PMU_CLKSEL_CON(4), 8, 2, MFLAGS, 0, 7, DFLAGS, + RK3568_PMU_CLKGATE_CON(1), 3, GFLAGS), + COMPOSITE_FRACMUX(CLK_UART0_FRAC, "sclk_uart0_frac", "sclk_uart0_div", CLK_SET_RATE_PARENT, + RK3568_PMU_CLKSEL_CON(5), CLK_FRAC_DIVIDER_NO_LIMIT, + RK3568_PMU_CLKGATE_CON(1), 4, GFLAGS, + &rk3568_uart0_fracmux, RK3568_UART_FRAC_MAX_PRATE), + GATE(SCLK_UART0, "sclk_uart0", "sclk_uart0_mux", 0, + RK3568_PMU_CLKGATE_CON(1), 5, GFLAGS), + + GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_pdpmu", 0, + RK3568_PMU_CLKGATE_CON(1), 9, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO0, "dbclk_gpio0", xin24m_32k_p, 0, + RK3568_PMU_CLKSEL_CON(6), 15, 1, MFLAGS, + RK3568_PMU_CLKGATE_CON(1), 10, GFLAGS), + GATE(PCLK_PWM0, "pclk_pwm0", "pclk_pdpmu", 0, + RK3568_PMU_CLKGATE_CON(1), 6, GFLAGS), + COMPOSITE(CLK_PWM0, "clk_pwm0", clk_pwm0_p, 0, + RK3568_PMU_CLKSEL_CON(6), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK3568_PMU_CLKGATE_CON(1), 7, GFLAGS), + GATE(CLK_CAPTURE_PWM0_NDFT, "clk_capture_pwm0_ndft", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(1), 8, GFLAGS), + GATE(PCLK_PMUPVTM, "pclk_pmupvtm", "pclk_pdpmu", 0, + RK3568_PMU_CLKGATE_CON(1), 11, GFLAGS), + GATE(CLK_PMUPVTM, "clk_pmupvtm", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(1), 12, GFLAGS), + GATE(CLK_CORE_PMUPVTM, "clk_core_pmupvtm", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(1), 13, GFLAGS), + COMPOSITE_NOMUX(CLK_REF24M, "clk_ref24m", "clk_pdpmu", 0, + RK3568_PMU_CLKSEL_CON(7), 0, 6, DFLAGS, + RK3568_PMU_CLKGATE_CON(2), 0, GFLAGS), + GATE(XIN_OSC0_USBPHY0_G, "xin_osc0_usbphy0_g", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(2), 1, GFLAGS), + MUX(CLK_USBPHY0_REF, "clk_usbphy0_ref", clk_usbphy0_ref_p, 0, + RK3568_PMU_CLKSEL_CON(8), 0, 1, MFLAGS), + GATE(XIN_OSC0_USBPHY1_G, "xin_osc0_usbphy1_g", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(2), 2, GFLAGS), + MUX(CLK_USBPHY1_REF, "clk_usbphy1_ref", clk_usbphy1_ref_p, 0, + RK3568_PMU_CLKSEL_CON(8), 1, 1, MFLAGS), + GATE(XIN_OSC0_MIPIDSIPHY0_G, "xin_osc0_mipidsiphy0_g", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(2), 3, GFLAGS), + MUX(CLK_MIPIDSIPHY0_REF, "clk_mipidsiphy0_ref", clk_mipidsiphy0_ref_p, 0, + RK3568_PMU_CLKSEL_CON(8), 2, 1, MFLAGS), + GATE(XIN_OSC0_MIPIDSIPHY1_G, "xin_osc0_mipidsiphy1_g", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(2), 4, GFLAGS), + MUX(CLK_MIPIDSIPHY1_REF, "clk_mipidsiphy1_ref", clk_mipidsiphy1_ref_p, 0, + RK3568_PMU_CLKSEL_CON(8), 3, 1, MFLAGS), + COMPOSITE_NOMUX(CLK_WIFI_DIV, "clk_wifi_div", "clk_pdpmu", 0, + RK3568_PMU_CLKSEL_CON(8), 8, 6, DFLAGS, + RK3568_PMU_CLKGATE_CON(2), 5, GFLAGS), + GATE(CLK_WIFI_OSC0, "clk_wifi_osc0", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(2), 6, GFLAGS), + MUX(CLK_WIFI, "clk_wifi", clk_wifi_p, CLK_SET_RATE_PARENT, + RK3568_PMU_CLKSEL_CON(8), 15, 1, MFLAGS), + COMPOSITE_NOMUX(CLK_PCIEPHY0_DIV, "clk_pciephy0_div", "ppll_ph0", 0, + RK3568_PMU_CLKSEL_CON(9), 0, 3, DFLAGS, + RK3568_PMU_CLKGATE_CON(2), 7, GFLAGS), + GATE(CLK_PCIEPHY0_OSC0, "clk_pciephy0_osc0", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(2), 8, GFLAGS), + MUX(CLK_PCIEPHY0_REF, "clk_pciephy0_ref", clk_pciephy0_ref_p, CLK_SET_RATE_PARENT, + RK3568_PMU_CLKSEL_CON(9), 3, 1, MFLAGS), + COMPOSITE_NOMUX(CLK_PCIEPHY1_DIV, "clk_pciephy1_div", "ppll_ph0", 0, + RK3568_PMU_CLKSEL_CON(9), 4, 3, DFLAGS, + RK3568_PMU_CLKGATE_CON(2), 9, GFLAGS), + GATE(CLK_PCIEPHY1_OSC0, "clk_pciephy1_osc0", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(2), 10, GFLAGS), + MUX(CLK_PCIEPHY1_REF, "clk_pciephy1_ref", clk_pciephy1_ref_p, CLK_SET_RATE_PARENT, + RK3568_PMU_CLKSEL_CON(9), 7, 1, MFLAGS), + COMPOSITE_NOMUX(CLK_PCIEPHY2_DIV, "clk_pciephy2_div", "ppll_ph0", 0, + RK3568_PMU_CLKSEL_CON(9), 8, 3, DFLAGS, + RK3568_PMU_CLKGATE_CON(2), 11, GFLAGS), + GATE(CLK_PCIEPHY2_OSC0, "clk_pciephy2_osc0", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(2), 12, GFLAGS), + MUX(CLK_PCIEPHY2_REF, "clk_pciephy2_ref", clk_pciephy2_ref_p, CLK_SET_RATE_PARENT, + RK3568_PMU_CLKSEL_CON(9), 11, 1, MFLAGS), + GATE(CLK_PCIE30PHY_REF_M, "clk_pcie30phy_ref_m", "ppll_ph0", 0, + RK3568_PMU_CLKGATE_CON(2), 13, GFLAGS), + GATE(CLK_PCIE30PHY_REF_N, "clk_pcie30phy_ref_n", "ppll_ph180", 0, + RK3568_PMU_CLKGATE_CON(2), 14, GFLAGS), + GATE(XIN_OSC0_EDPPHY_G, "xin_osc0_edpphy_g", "xin24m", 0, + RK3568_PMU_CLKGATE_CON(2), 15, GFLAGS), + MUX(CLK_HDMI_REF, "clk_hdmi_ref", clk_hdmi_ref_p, 0, + RK3568_PMU_CLKSEL_CON(8), 7, 1, MFLAGS), + + MUXPMUGRF(SCLK_32K_IOE, "clk_32k_ioe", clk_32k_ioe_p, 0, + RK3568_PMU_GRF_SOC_CON0, 0, 1, MFLAGS) +}; + +static void __iomem *rk3568_cru_base; +static void __iomem *rk3568_pmucru_base; + +static void rk3568_dump_cru(void) +{ + if (rk3568_pmucru_base) { + pr_warn("PMU CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk3568_pmucru_base, + 0x248, false); + } + if (rk3568_cru_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rk3568_cru_base, + 0x588, false); + } +} + +static void __init rk3568_pmu_clk_init(struct device_node *np) +{ + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + pr_err("%s: could not map cru pmu region\n", __func__); + return; + } + + rk3568_pmucru_base = reg_base; + + ctx = rockchip_clk_init(np, reg_base, CLKPMU_NR_CLKS); + if (IS_ERR(ctx)) { + pr_err("%s: rockchip pmu clk init failed\n", __func__); + return; + } + + rockchip_clk_register_plls(ctx, rk3568_pmu_pll_clks, + ARRAY_SIZE(rk3568_pmu_pll_clks), + RK3568_GRF_SOC_STATUS0); + + rockchip_clk_register_branches(ctx, rk3568_clk_pmu_branches, + ARRAY_SIZE(rk3568_clk_pmu_branches)); + + rockchip_register_softrst(np, 1, reg_base + RK3568_PMU_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); + + rockchip_clk_of_add_provider(np, ctx); +} + +CLK_OF_DECLARE(rk3568_cru_pmu, "rockchip,rk3568-pmucru", rk3568_pmu_clk_init); + +static void __init rk3568_clk_init(struct device_node *np) +{ + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; + struct clk **clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + pr_err("%s: could not map cru region\n", __func__); + return; + } + + rk3568_cru_base = reg_base; + + ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); + if (IS_ERR(ctx)) { + pr_err("%s: rockchip clk init failed\n", __func__); + iounmap(reg_base); + return; + } + clks = ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rk3568_pll_clks, + ARRAY_SIZE(rk3568_pll_clks), + RK3568_GRF_SOC_STATUS0); + + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", + 2, clks[PLL_APLL], clks[PLL_GPLL], + &rk3568_cpuclk_data, rk3568_cpuclk_rates, + ARRAY_SIZE(rk3568_cpuclk_rates)); + + rockchip_clk_register_branches(ctx, rk3568_clk_branches, + ARRAY_SIZE(rk3568_clk_branches)); + + rockchip_register_softrst(np, 30, reg_base + RK3568_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); + + rockchip_register_restart_notifier(ctx, RK3568_GLB_SRST_FST, NULL); + + rockchip_clk_of_add_provider(np, ctx); + + if (!rk_dump_cru) + rk_dump_cru = rk3568_dump_cru; +} + +CLK_OF_DECLARE(rk3568_cru, "rockchip,rk3568-cru", rk3568_clk_init); + +struct clk_rk3568_inits { + void (*inits)(struct device_node *np); +}; + +static const struct clk_rk3568_inits clk_rk3568_pmucru_init = { + .inits = rk3568_pmu_clk_init, +}; + +static const struct clk_rk3568_inits clk_3568_cru_init = { + .inits = rk3568_clk_init, +}; + +static const struct of_device_id clk_rk3568_match_table[] = { + { + .compatible = "rockchip,rk3568-cru", + .data = &clk_3568_cru_init, + }, { + .compatible = "rockchip,rk3568-pmucru", + .data = &clk_rk3568_pmucru_init, + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rk3568_match_table); + +static int __init clk_rk3568_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + const struct clk_rk3568_inits *init_data; + + match = of_match_device(clk_rk3568_match_table, &pdev->dev); + if (!match || !match->data) + return -EINVAL; + + init_data = match->data; + if (init_data->inits) + init_data->inits(np); + + return 0; +} + +static struct platform_driver clk_rk3568_driver = { + .driver = { + .name = "clk-rk3568", + .of_match_table = clk_rk3568_match_table, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver_probe(clk_rk3568_driver, clk_rk3568_probe); + +MODULE_DESCRIPTION("Rockchip RK3568 Clock Driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:clk-rk3568"); diff --git a/drivers/clk/rockchip/clk-rv1108.c b/drivers/clk/rockchip/clk-rv1108.c index 5947d3192866..53eaabb193a4 100644 --- a/drivers/clk/rockchip/clk-rv1108.c +++ b/drivers/clk/rockchip/clk-rv1108.c @@ -7,13 +7,17 @@ #include #include +#include #include #include +#include #include #include #include "clk.h" #define RV1108_GRF_SOC_STATUS0 0x480 +#define RV1108_I2S_FRAC_MAX_RATE 600000000 +#define RV1108_UART_FRAC_MAX_RATE 600000000 enum rv1108_plls { apll, dpll, gpll, @@ -106,9 +110,10 @@ static struct rockchip_cpuclk_rate_table rv1108_cpuclk_rates[] __initdata = { }; static const struct rockchip_cpuclk_reg_data rv1108_cpuclk_data = { - .core_reg = RV1108_CLKSEL_CON(0), - .div_core_shift = 0, - .div_core_mask = 0x1f, + .core_reg[0] = RV1108_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, .mux_core_alt = 1, .mux_core_main = 0, .mux_core_shift = 8, @@ -117,7 +122,6 @@ static const struct rockchip_cpuclk_reg_data rv1108_cpuclk_data = { PNAME(mux_pll_p) = { "xin24m", "xin24m"}; PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr", "apll_ddr" }; -PNAME(mux_armclk_p) = { "apll_core", "gpll_core", "dpll_core" }; PNAME(mux_usb480m_pre_p) = { "usbphy", "xin24m" }; PNAME(mux_hdmiphy_phy_p) = { "hdmiphy", "xin24m" }; PNAME(mux_dclk_hdmiphy_pre_p) = { "dclk_hdmiphy_src_gpll", "dclk_hdmiphy_src_dpll" }; @@ -209,7 +213,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { COMPOSITE_NOMUX(ACLK_ENMCORE, "aclkenm_core", "armclk", CLK_IGNORE_UNUSED, RV1108_CLKSEL_CON(1), 0, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, RV1108_CLKGATE_CON(0), 4, GFLAGS), - GATE(ACLK_CORE, "aclk_core", "aclkenm_core", CLK_IGNORE_UNUSED, + GATE(ACLK_CORE, "aclk_core", "aclkenm_core", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(11), 0, GFLAGS), GATE(0, "pclk_dbg", "pclken_dbg", CLK_IGNORE_UNUSED, RV1108_CLKGATE_CON(11), 1, GFLAGS), @@ -264,10 +268,10 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { RV1108_CLKGATE_CON(19), 6, GFLAGS), /* PD_PMU_wrapper */ - COMPOSITE_NOMUX(0, "pmu_24m_ena", "gpll", CLK_IGNORE_UNUSED, + COMPOSITE_NOMUX(0, "pmu_24m_ena", "gpll", CLK_IS_CRITICAL, RV1108_CLKSEL_CON(38), 0, 5, DFLAGS, RV1108_CLKGATE_CON(8), 12, GFLAGS), - GATE(0, "pclk_pmu", "pmu_24m_ena", CLK_IGNORE_UNUSED, + GATE(0, "pclk_pmu", "pmu_24m_ena", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(10), 0, GFLAGS), GATE(0, "pclk_intmem1", "pmu_24m_ena", CLK_IGNORE_UNUSED, RV1108_CLKGATE_CON(10), 1, GFLAGS), @@ -305,7 +309,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { RV1108_CLKSEL_CON(41), 0, 5, DFLAGS, RV1108_CLKGATE_CON(9), 12, GFLAGS), - GATE(0, "pclk_acodecphy", "pclk_top_pre", CLK_IGNORE_UNUSED, + GATE(0, "pclk_acodecphy", "pclk_top_pre", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(14), 6, GFLAGS), GATE(0, "pclk_usbgrf", "pclk_top_pre", CLK_IGNORE_UNUSED, RV1108_CLKGATE_CON(14), 14, GFLAGS), @@ -503,7 +507,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_src", CLK_SET_RATE_PARENT, RV1108_CLKSEL_CON(8), 0, RV1108_CLKGATE_CON(2), 1, GFLAGS, - &rv1108_i2s0_fracmux), + &rv1108_i2s0_fracmux, RV1108_I2S_FRAC_MAX_RATE), GATE(SCLK_I2S0, "sclk_i2s0", "i2s0_pre", CLK_SET_RATE_PARENT, RV1108_CLKGATE_CON(2), 2, GFLAGS), COMPOSITE_NODIV(0, "i2s_out", mux_i2s_out_p, 0, @@ -516,7 +520,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(9), 0, RK2928_CLKGATE_CON(2), 5, GFLAGS, - &rv1108_i2s1_fracmux), + &rv1108_i2s1_fracmux, RV1108_I2S_FRAC_MAX_RATE), GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT, RV1108_CLKGATE_CON(2), 6, GFLAGS), @@ -526,28 +530,28 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "i2s2_frac", "i2s2_src", CLK_SET_RATE_PARENT, RV1108_CLKSEL_CON(10), 0, RV1108_CLKGATE_CON(2), 9, GFLAGS, - &rv1108_i2s2_fracmux), + &rv1108_i2s2_fracmux, RV1108_I2S_FRAC_MAX_RATE), GATE(SCLK_I2S2, "sclk_i2s2", "i2s2_pre", CLK_SET_RATE_PARENT, RV1108_CLKGATE_CON(2), 10, GFLAGS), /* PD_BUS */ - GATE(0, "aclk_bus_src_gpll", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "aclk_bus_src_gpll", "gpll", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(1), 0, GFLAGS), - GATE(0, "aclk_bus_src_apll", "apll", CLK_IGNORE_UNUSED, + GATE(0, "aclk_bus_src_apll", "apll", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(1), 1, GFLAGS), - GATE(0, "aclk_bus_src_dpll", "dpll", CLK_IGNORE_UNUSED, + GATE(0, "aclk_bus_src_dpll", "dpll", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(1), 2, GFLAGS), - COMPOSITE_NOGATE(ACLK_PRE, "aclk_bus_pre", mux_aclk_bus_src_p, 0, + COMPOSITE_NOGATE(ACLK_PRE, "aclk_bus_pre", mux_aclk_bus_src_p, CLK_IS_CRITICAL, RV1108_CLKSEL_CON(2), 8, 2, MFLAGS, 0, 5, DFLAGS), - COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus_pre", "aclk_bus_pre", 0, + COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus_pre", "aclk_bus_pre", CLK_IS_CRITICAL, RV1108_CLKSEL_CON(3), 0, 5, DFLAGS, RV1108_CLKGATE_CON(1), 4, GFLAGS), - COMPOSITE_NOMUX(0, "pclk_bus_pre", "aclk_bus_pre", 0, + COMPOSITE_NOMUX(0, "pclk_bus_pre", "aclk_bus_pre", CLK_IS_CRITICAL, RV1108_CLKSEL_CON(3), 8, 5, DFLAGS, RV1108_CLKGATE_CON(1), 5, GFLAGS), - GATE(PCLK_BUS, "pclk_bus", "pclk_bus_pre", 0, + GATE(PCLK_BUS, "pclk_bus", "pclk_bus_pre", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(1), 6, GFLAGS), - GATE(0, "pclk_top_pre", "pclk_bus_pre", CLK_IGNORE_UNUSED, + GATE(0, "pclk_top_pre", "pclk_bus_pre", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(1), 7, GFLAGS), GATE(0, "pclk_ddr_pre", "pclk_bus_pre", CLK_IGNORE_UNUSED, RV1108_CLKGATE_CON(1), 8, GFLAGS), @@ -592,15 +596,15 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, RV1108_CLKSEL_CON(16), 0, RV1108_CLKGATE_CON(3), 2, GFLAGS, - &rv1108_uart0_fracmux), + &rv1108_uart0_fracmux, RV1108_UART_FRAC_MAX_RATE), COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, RV1108_CLKSEL_CON(17), 0, RV1108_CLKGATE_CON(3), 4, GFLAGS, - &rv1108_uart1_fracmux), + &rv1108_uart1_fracmux, RV1108_UART_FRAC_MAX_RATE), COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, RV1108_CLKSEL_CON(18), 0, RV1108_CLKGATE_CON(3), 6, GFLAGS, - &rv1108_uart2_fracmux), + &rv1108_uart2_fracmux, RV1108_UART_FRAC_MAX_RATE), GATE(PCLK_UART0, "pclk_uart0", "pclk_bus_pre", 0, RV1108_CLKGATE_CON(13), 10, GFLAGS), GATE(PCLK_UART1, "pclk_uart1", "pclk_bus_pre", 0, @@ -668,7 +672,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { RV1108_CLKGATE_CON(0), 9, GFLAGS), GATE(0, "gpll_ddr", "gpll", CLK_IGNORE_UNUSED, RV1108_CLKGATE_CON(0), 10, GFLAGS), - COMPOSITE_NOGATE(0, "clk_ddrphy_src", mux_ddrphy_p, CLK_IGNORE_UNUSED, + COMPOSITE_NOGATE(0, "clk_ddrphy_src", mux_ddrphy_p, CLK_IS_CRITICAL, RV1108_CLKSEL_CON(4), 8, 2, MFLAGS, 0, 3, DFLAGS | CLK_DIVIDER_POWER_OF_TWO), FACTOR(0, "clk_ddr", "clk_ddrphy_src", 0, 1, 2), @@ -676,9 +680,9 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { RV1108_CLKGATE_CON(10), 9, GFLAGS), GATE(0, "pclk_ddrupctl", "pclk_ddr_pre", CLK_IGNORE_UNUSED, RV1108_CLKGATE_CON(12), 4, GFLAGS), - GATE(0, "nclk_ddrupctl", "clk_ddr", CLK_IGNORE_UNUSED, + GATE(0, "nclk_ddrupctl", "clk_ddr", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(12), 5, GFLAGS), - GATE(0, "pclk_ddrmon", "pclk_ddr_pre", CLK_IGNORE_UNUSED, + GATE(0, "pclk_ddrmon", "pclk_ddr_pre", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(12), 6, GFLAGS), GATE(0, "timer_clk", "xin24m", CLK_IGNORE_UNUSED, RV1108_CLKGATE_CON(0), 11, GFLAGS), @@ -692,22 +696,22 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { */ /* PD_PERI */ - COMPOSITE_NOMUX(0, "pclk_periph_pre", "gpll", 0, + COMPOSITE_NOMUX(0, "pclk_periph_pre", "gpll", CLK_IS_CRITICAL, RV1108_CLKSEL_CON(23), 10, 5, DFLAGS, RV1108_CLKGATE_CON(4), 5, GFLAGS), - GATE(PCLK_PERI, "pclk_periph", "pclk_periph_pre", CLK_IGNORE_UNUSED, + GATE(PCLK_PERI, "pclk_periph", "pclk_periph_pre", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(15), 13, GFLAGS), - COMPOSITE_NOMUX(0, "hclk_periph_pre", "gpll", 0, + COMPOSITE_NOMUX(0, "hclk_periph_pre", "gpll", CLK_IS_CRITICAL, RV1108_CLKSEL_CON(23), 5, 5, DFLAGS, RV1108_CLKGATE_CON(4), 4, GFLAGS), - GATE(HCLK_PERI, "hclk_periph", "hclk_periph_pre", CLK_IGNORE_UNUSED, + GATE(HCLK_PERI, "hclk_periph", "hclk_periph_pre", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(15), 12, GFLAGS), - GATE(0, "aclk_peri_src_dpll", "dpll", CLK_IGNORE_UNUSED, + GATE(0, "aclk_peri_src_dpll", "dpll", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(4), 1, GFLAGS), - GATE(0, "aclk_peri_src_gpll", "gpll", CLK_IGNORE_UNUSED, + GATE(0, "aclk_peri_src_gpll", "gpll", CLK_IS_CRITICAL, RV1108_CLKGATE_CON(4), 2, GFLAGS), - COMPOSITE(ACLK_PERI, "aclk_periph", mux_aclk_peri_src_p, 0, + COMPOSITE(ACLK_PERI, "aclk_periph", mux_aclk_peri_src_p, CLK_IS_CRITICAL, RV1108_CLKSEL_CON(23), 15, 1, MFLAGS, 0, 5, DFLAGS, RV1108_CLKGATE_CON(15), 11, GFLAGS), @@ -767,24 +771,23 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = { MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "sclk_emmc", RV1108_EMMC_CON1, 1), }; -static const char *const rv1108_critical_clocks[] __initconst = { - "aclk_core", - "aclk_bus", - "hclk_bus", - "pclk_bus", - "aclk_periph", - "hclk_periph", - "pclk_periph", - "nclk_ddrupctl", - "pclk_ddrmon", - "pclk_acodecphy", - "pclk_pmu", -}; +static void __iomem *rv1108_cru_base; + +static void rv1108_dump_cru(void) +{ + if (rv1108_cru_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rv1108_cru_base, + 0x1f8, false); + } +} static void __init rv1108_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; + struct clk **clks; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -798,17 +801,16 @@ static void __init rv1108_clk_init(struct device_node *np) iounmap(reg_base); return; } + clks = ctx->clk_data.clks; rockchip_clk_register_plls(ctx, rv1108_pll_clks, ARRAY_SIZE(rv1108_pll_clks), RV1108_GRF_SOC_STATUS0); rockchip_clk_register_branches(ctx, rv1108_clk_branches, ARRAY_SIZE(rv1108_clk_branches)); - rockchip_clk_protect_critical(rv1108_critical_clocks, - ARRAY_SIZE(rv1108_critical_clocks)); rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", - mux_armclk_p, ARRAY_SIZE(mux_armclk_p), + 3, clks[PLL_APLL], clks[PLL_GPLL], &rv1108_cpuclk_data, rv1108_cpuclk_rates, ARRAY_SIZE(rv1108_cpuclk_rates)); @@ -818,5 +820,38 @@ static void __init rv1108_clk_init(struct device_node *np) rockchip_register_restart_notifier(ctx, RV1108_GLB_SRST_FST, NULL); rockchip_clk_of_add_provider(np, ctx); + + if (!rk_dump_cru) { + rv1108_cru_base = reg_base; + rk_dump_cru = rv1108_dump_cru; + } } CLK_OF_DECLARE(rv1108_cru, "rockchip,rv1108-cru", rv1108_clk_init); + +static int __init clk_rv1108_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + + rv1108_clk_init(np); + + return 0; +} + +static const struct of_device_id clk_rv1108_match_table[] = { + { + .compatible = "rockchip,rv1108-cru", + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rv1108_match_table); + +static struct platform_driver clk_rv1108_driver = { + .driver = { + .name = "clk-rv1108", + .of_match_table = clk_rv1108_match_table, + }, +}; +builtin_platform_driver_probe(clk_rv1108_driver, clk_rv1108_probe); + +MODULE_DESCRIPTION("Rockchip RV1108 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk-rv1126.c b/drivers/clk/rockchip/clk-rv1126.c new file mode 100755 index 000000000000..09a376c51c0b --- /dev/null +++ b/drivers/clk/rockchip/clk-rv1126.c @@ -0,0 +1,1586 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019 Rockchip Electronics Co. Ltd. + * Author: Finley Xiao + */ + +#include +#include +#include +#include +#include +#include +#include +#include "clk.h" + +#define RV1126_GMAC_CON 0x460 +#define RV1126_GRF_IOFUNC_CON1 0x10264 +#define RV1126_GRF_SOC_STATUS0 0x10 +#define RV1126_PMUGRF_SOC_CON0 0x100 + +#define RV1126_FRAC_MAX_PRATE 1200000000 +#define RV1126_CSIOUT_FRAC_MAX_PRATE 300000000 + +enum rv1126_pmu_plls { + gpll, +}; + +enum rv1126_plls { + apll, dpll, cpll, hpll, +}; + +static struct rockchip_pll_rate_table rv1126_pll_rates[] = { + /* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */ + RK3036_PLL_RATE(1608000000, 1, 67, 1, 1, 1, 0), + RK3036_PLL_RATE(1600000000, 3, 200, 1, 1, 1, 0), + RK3036_PLL_RATE(1584000000, 1, 132, 2, 1, 1, 0), + RK3036_PLL_RATE(1560000000, 1, 130, 2, 1, 1, 0), + RK3036_PLL_RATE(1536000000, 1, 128, 2, 1, 1, 0), + RK3036_PLL_RATE(1512000000, 1, 126, 2, 1, 1, 0), + RK3036_PLL_RATE(1488000000, 1, 124, 2, 1, 1, 0), + RK3036_PLL_RATE(1464000000, 1, 122, 2, 1, 1, 0), + RK3036_PLL_RATE(1440000000, 1, 120, 2, 1, 1, 0), + RK3036_PLL_RATE(1416000000, 1, 118, 2, 1, 1, 0), + RK3036_PLL_RATE(1400000000, 3, 350, 2, 1, 1, 0), + RK3036_PLL_RATE(1392000000, 1, 116, 2, 1, 1, 0), + RK3036_PLL_RATE(1368000000, 1, 114, 2, 1, 1, 0), + RK3036_PLL_RATE(1344000000, 1, 112, 2, 1, 1, 0), + RK3036_PLL_RATE(1320000000, 1, 110, 2, 1, 1, 0), + RK3036_PLL_RATE(1296000000, 1, 108, 2, 1, 1, 0), + RK3036_PLL_RATE(1272000000, 1, 106, 2, 1, 1, 0), + RK3036_PLL_RATE(1248000000, 1, 104, 2, 1, 1, 0), + RK3036_PLL_RATE(1200000000, 1, 100, 2, 1, 1, 0), + RK3036_PLL_RATE(1188000000, 1, 99, 2, 1, 1, 0), + RK3036_PLL_RATE(1104000000, 1, 92, 2, 1, 1, 0), + RK3036_PLL_RATE(1100000000, 3, 275, 2, 1, 1, 0), + RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0), + RK3036_PLL_RATE(1000000000, 3, 250, 2, 1, 1, 0), + RK3036_PLL_RATE(984000000, 1, 82, 2, 1, 1, 0), + RK3036_PLL_RATE(960000000, 1, 80, 2, 1, 1, 0), + RK3036_PLL_RATE(936000000, 1, 78, 2, 1, 1, 0), + RK3036_PLL_RATE(912000000, 1, 76, 2, 1, 1, 0), + RK3036_PLL_RATE(900000000, 1, 75, 2, 1, 1, 0), + RK3036_PLL_RATE(888000000, 1, 74, 2, 1, 1, 0), + RK3036_PLL_RATE(864000000, 1, 72, 2, 1, 1, 0), + RK3036_PLL_RATE(840000000, 1, 70, 2, 1, 1, 0), + RK3036_PLL_RATE(816000000, 1, 68, 2, 1, 1, 0), + RK3036_PLL_RATE(800000000, 3, 200, 2, 1, 1, 0), + RK3036_PLL_RATE(700000000, 3, 350, 4, 1, 1, 0), + RK3036_PLL_RATE(696000000, 1, 116, 4, 1, 1, 0), + RK3036_PLL_RATE(624000000, 1, 104, 4, 1, 1, 0), +#ifdef CONFIG_ROCKCHIP_LOW_PERFORMANCE + RK3036_PLL_RATE(600000000, 1, 50, 2, 1, 1, 0), +#else + RK3036_PLL_RATE(600000000, 1, 100, 4, 1, 1, 0), +#endif + RK3036_PLL_RATE(594000000, 1, 99, 4, 1, 1, 0), + RK3036_PLL_RATE(504000000, 1, 84, 4, 1, 1, 0), + RK3036_PLL_RATE(500000000, 1, 125, 6, 1, 1, 0), + RK3036_PLL_RATE(496742400, 1, 124, 6, 1, 0, 3113851), + RK3036_PLL_RATE(491520000, 1, 40, 2, 1, 0, 16106127), + RK3036_PLL_RATE(408000000, 1, 68, 2, 2, 1, 0), + RK3036_PLL_RATE(312000000, 1, 78, 6, 1, 1, 0), + RK3036_PLL_RATE(216000000, 1, 72, 4, 2, 1, 0), + RK3036_PLL_RATE(96000000, 1, 96, 6, 4, 1, 0), + { /* sentinel */ }, +}; + +#define RV1126_DIV_ACLK_CORE_MASK 0xf +#define RV1126_DIV_ACLK_CORE_SHIFT 4 +#define RV1126_DIV_PCLK_DBG_MASK 0x7 +#define RV1126_DIV_PCLK_DBG_SHIFT 0 + +#define RV1126_CLKSEL1(_aclk_core, _pclk_dbg) \ +{ \ + .reg = RV1126_CLKSEL_CON(1), \ + .val = HIWORD_UPDATE(_aclk_core, RV1126_DIV_ACLK_CORE_MASK, \ + RV1126_DIV_ACLK_CORE_SHIFT) | \ + HIWORD_UPDATE(_pclk_dbg, RV1126_DIV_PCLK_DBG_MASK, \ + RV1126_DIV_PCLK_DBG_SHIFT), \ +} + +#define RV1126_CPUCLK_RATE(_prate, _aclk_core, _pclk_dbg) \ +{ \ + .prate = _prate, \ + .divs = { \ + RV1126_CLKSEL1(_aclk_core, _pclk_dbg), \ + }, \ +} + +static struct rockchip_cpuclk_rate_table rv1126_cpuclk_rates[] __initdata = { + RV1126_CPUCLK_RATE(1608000000, 1, 7), + RV1126_CPUCLK_RATE(1584000000, 1, 7), + RV1126_CPUCLK_RATE(1560000000, 1, 7), + RV1126_CPUCLK_RATE(1536000000, 1, 7), + RV1126_CPUCLK_RATE(1512000000, 1, 7), + RV1126_CPUCLK_RATE(1488000000, 1, 5), + RV1126_CPUCLK_RATE(1464000000, 1, 5), + RV1126_CPUCLK_RATE(1440000000, 1, 5), + RV1126_CPUCLK_RATE(1416000000, 1, 5), + RV1126_CPUCLK_RATE(1392000000, 1, 5), + RV1126_CPUCLK_RATE(1368000000, 1, 5), + RV1126_CPUCLK_RATE(1344000000, 1, 5), + RV1126_CPUCLK_RATE(1320000000, 1, 5), + RV1126_CPUCLK_RATE(1296000000, 1, 5), + RV1126_CPUCLK_RATE(1272000000, 1, 5), + RV1126_CPUCLK_RATE(1248000000, 1, 5), + RV1126_CPUCLK_RATE(1224000000, 1, 5), + RV1126_CPUCLK_RATE(1200000000, 1, 5), + RV1126_CPUCLK_RATE(1104000000, 1, 5), + RV1126_CPUCLK_RATE(1008000000, 1, 5), + RV1126_CPUCLK_RATE(912000000, 1, 5), + RV1126_CPUCLK_RATE(816000000, 1, 3), + RV1126_CPUCLK_RATE(696000000, 1, 3), + RV1126_CPUCLK_RATE(600000000, 1, 3), + RV1126_CPUCLK_RATE(408000000, 1, 1), + RV1126_CPUCLK_RATE(312000000, 1, 1), + RV1126_CPUCLK_RATE(216000000, 1, 1), + RV1126_CPUCLK_RATE(96000000, 1, 1), +}; + +static const struct rockchip_cpuclk_reg_data rv1126_cpuclk_data = { + .core_reg[0] = RV1126_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, + .mux_core_alt = 0, + .mux_core_main = 2, + .mux_core_shift = 6, + .mux_core_mask = 0x3, +}; + +PNAME(mux_pll_p) = { "xin24m" }; +PNAME(mux_rtc32k_p) = { "clk_pmupvtm_divout", "xin32k", "clk_osc0_div32k" }; +PNAME(mux_clk_32k_ioe_p) = { "xin32k", "clk_rtc32k" }; +PNAME(mux_wifi_p) = { "clk_wifi_osc0", "clk_wifi_div" }; +PNAME(mux_uart1_p) = { "sclk_uart1_div", "sclk_uart1_fracdiv", "xin24m" }; +PNAME(mux_xin24m_gpll_p) = { "xin24m", "gpll" }; +PNAME(mux_gpll_xin24m_p) = { "gpll", "xin24m" }; +PNAME(mux_xin24m_32k_p) = { "xin24m", "clk_rtc32k" }; +PNAME(mux_usbphy_otg_ref_p) = { "clk_ref12m", "xin_osc0_div2_usbphyref_otg" }; +PNAME(mux_usbphy_host_ref_p) = { "clk_ref12m", "xin_osc0_div2_usbphyref_host" }; +PNAME(mux_mipidsiphy_ref_p) = { "clk_ref24m", "xin_osc0_mipiphyref" }; +PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc32k" }; +PNAME(mux_hclk_pclk_pdbus_p) = { "gpll", "dummy_cpll" }; +PNAME(mux_uart0_p) = { "sclk_uart0_div", "sclk_uart0_frac", "xin24m" }; +PNAME(mux_uart2_p) = { "sclk_uart2_div", "sclk_uart2_frac", "xin24m" }; +PNAME(mux_uart3_p) = { "sclk_uart3_div", "sclk_uart3_frac", "xin24m" }; +PNAME(mux_uart4_p) = { "sclk_uart4_div", "sclk_uart4_frac", "xin24m" }; +PNAME(mux_uart5_p) = { "sclk_uart5_div", "sclk_uart5_frac", "xin24m" }; +PNAME(mux_i2s0_tx_p) = { "mclk_i2s0_tx_div", "mclk_i2s0_tx_fracdiv", "i2s0_mclkin", "xin12m" }; +PNAME(mux_i2s0_rx_p) = { "mclk_i2s0_rx_div", "mclk_i2s0_rx_fracdiv", "i2s0_mclkin", "xin12m" }; +PNAME(mux_i2s0_tx_out2io_p) = { "mclk_i2s0_tx", "xin12m" }; +PNAME(mux_i2s0_rx_out2io_p) = { "mclk_i2s0_rx", "xin12m" }; +PNAME(mux_i2s1_p) = { "mclk_i2s1_div", "mclk_i2s1_fracdiv", "i2s1_mclkin", "xin12m" }; +PNAME(mux_i2s1_out2io_p) = { "mclk_i2s1", "xin12m" }; +PNAME(mux_i2s2_p) = { "mclk_i2s2_div", "mclk_i2s2_fracdiv", "i2s2_mclkin", "xin12m" }; +PNAME(mux_i2s2_out2io_p) = { "mclk_i2s2", "xin12m" }; +PNAME(mux_audpwm_p) = { "sclk_audpwm_div", "sclk_audpwm_fracdiv", "xin24m" }; +PNAME(mux_dclk_vop_p) = { "dclk_vop_div", "dclk_vop_fracdiv", "xin24m" }; +PNAME(mux_aclk_pdvi_p) = { "aclk_pdvi_div", "aclk_pdvi_np5" }; +PNAME(mux_clk_isp_p) = { "clk_isp_div", "clk_isp_np5" }; +PNAME(mux_gpll_usb480m_p) = { "gpll", "usb480m" }; +PNAME(mux_cif_out2io_p) = { "xin24m", "clk_cif_out2io_div", "clk_cif_out2io_fracdiv" }; +PNAME(mux_mipicsi_out2io_p) = { "xin24m", "clk_mipicsi_out2io_div", "clk_mipicsi_out2io_fracdiv" }; +PNAME(mux_aclk_pdispp_p) = { "aclk_pdispp_div", "aclk_pdispp_np5" }; +PNAME(mux_clk_ispp_p) = { "clk_ispp_div", "clk_ispp_np5" }; +PNAME(mux_usb480m_gpll_p) = { "usb480m", "gpll" }; +PNAME(clk_gmac_src_m0_p) = { "clk_gmac_div", "clk_gmac_rgmii_m0" }; +PNAME(clk_gmac_src_m1_p) = { "clk_gmac_div", "clk_gmac_rgmii_m1" }; +PNAME(mux_clk_gmac_src_p) = { "clk_gmac_src_m0", "clk_gmac_src_m1" }; +PNAME(mux_rgmii_clk_p) = { "clk_gmac_tx_div50", "clk_gmac_tx_div5", "clk_gmac_tx_src", "clk_gmac_tx_src"}; +PNAME(mux_rmii_clk_p) = { "clk_gmac_rx_div20", "clk_gmac_rx_div2" }; +PNAME(mux_gmac_tx_rx_p) = { "rgmii_mode_clk", "rmii_mode_clk" }; +PNAME(mux_dpll_gpll_p) = { "dpll", "gpll" }; +PNAME(mux_aclk_pdnpu_p) = { "aclk_pdnpu_div", "aclk_pdnpu_np5" }; +PNAME(mux_clk_npu_p) = { "clk_npu_div", "clk_npu_np5" }; + + +#ifndef CONFIG_ROCKCHIP_LOW_PERFORMANCE +PNAME(mux_gpll_usb480m_cpll_xin24m_p) = { "gpll", "usb480m", "cpll", "xin24m" }; +PNAME(mux_gpll_cpll_dpll_p) = { "gpll", "cpll", "dummy_dpll" }; +PNAME(mux_gpll_cpll_p) = { "gpll", "cpll" }; +PNAME(mux_gpll_cpll_usb480m_xin24m_p) = { "gpll", "cpll", "usb480m", "xin24m" }; +PNAME(mux_cpll_gpll_p) = { "cpll", "gpll" }; +PNAME(mux_gpll_cpll_xin24m_p) = { "gpll", "cpll", "xin24m" }; +PNAME(mux_cpll_hpll_gpll_p) = { "cpll", "hpll", "gpll" }; +PNAME(mux_cpll_gpll_hpll_p) = { "cpll", "gpll", "hpll" }; +PNAME(mux_gpll_cpll_hpll_p) = { "gpll", "cpll", "hpll" }; +PNAME(mux_gpll_cpll_apll_hpll_p) = { "gpll", "cpll", "dummy_apll", "hpll" }; +#else +PNAME(mux_gpll_usb480m_cpll_xin24m_p) = { "gpll", "usb480m", "dummy_cpll", "xin24m" }; +PNAME(mux_gpll_cpll_dpll_p) = { "gpll", "dummy_cpll", "dummy_dpll" }; +PNAME(mux_gpll_cpll_p) = { "gpll", "dummy_cpll" }; +PNAME(mux_gpll_cpll_usb480m_xin24m_p) = { "gpll", "dummy_cpll", "usb480m", "xin24m" }; +PNAME(mux_cpll_gpll_p) = { "dummy_cpll", "gpll" }; +PNAME(mux_gpll_cpll_xin24m_p) = { "gpll", "dummy_cpll", "xin24m" }; +PNAME(mux_cpll_hpll_gpll_p) = { "dummy_cpll", "dummy_hpll", "gpll" }; +PNAME(mux_cpll_gpll_hpll_p) = { "dummy_cpll", "gpll", "dummy_hpll" }; +PNAME(mux_gpll_cpll_hpll_p) = { "gpll", "dummy_cpll", "dummy_hpll" }; +PNAME(mux_gpll_cpll_apll_hpll_p) = { "gpll", "dummy_cpll", "dummy_apll", "dummy_hpll" }; +#endif + +static u32 rgmii_mux_idx[] = { 2, 3, 0, 1 }; + +static struct rockchip_pll_clock rv1126_pmu_pll_clks[] __initdata = { + [gpll] = PLL(pll_rk3328, PLL_GPLL, "gpll", mux_pll_p, + CLK_IS_CRITICAL, RV1126_PMU_PLL_CON(0), + RV1126_PMU_MODE, 0, 3, 0, rv1126_pll_rates), +}; + +static struct rockchip_pll_clock rv1126_pll_clks[] __initdata = { + [apll] = PLL(pll_rk3328, PLL_APLL, "apll", mux_pll_p, + CLK_IGNORE_UNUSED, RV1126_PLL_CON(0), + RV1126_MODE_CON, 0, 0, 0, rv1126_pll_rates), + [dpll] = PLL(pll_rk3328, PLL_DPLL, "dpll", mux_pll_p, + CLK_IGNORE_UNUSED, RV1126_PLL_CON(8), + RV1126_MODE_CON, 2, 1, 0, NULL), +#ifndef CONFIG_ROCKCHIP_LOW_PERFORMANCE + [cpll] = PLL(pll_rk3328, PLL_CPLL, "cpll", mux_pll_p, + CLK_IS_CRITICAL, RV1126_PLL_CON(16), + RV1126_MODE_CON, 4, 2, 0, rv1126_pll_rates), + [hpll] = PLL(pll_rk3328, PLL_HPLL, "hpll", mux_pll_p, + CLK_IS_CRITICAL, RV1126_PLL_CON(24), + RV1126_MODE_CON, 6, 4, 0, rv1126_pll_rates), +#else + [cpll] = PLL(pll_rk3328, PLL_CPLL, "cpll", mux_pll_p, + 0, RV1126_PLL_CON(16), + RV1126_MODE_CON, 4, 2, 0, rv1126_pll_rates), + [hpll] = PLL(pll_rk3328, PLL_HPLL, "hpll", mux_pll_p, + 0, RV1126_PLL_CON(24), + RV1126_MODE_CON, 6, 4, 0, rv1126_pll_rates), +#endif +}; + +#define MFLAGS CLK_MUX_HIWORD_MASK +#define DFLAGS CLK_DIVIDER_HIWORD_MASK +#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE) + +static struct rockchip_clk_branch rv1126_rtc32k_fracmux __initdata = + MUX(CLK_RTC32K, "clk_rtc32k", mux_rtc32k_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(0), 7, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart1_fracmux __initdata = + MUX(SCLK_UART1_MUX, "sclk_uart1_mux", mux_uart1_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(4), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart0_fracmux __initdata = + MUX(SCLK_UART0_MUX, "sclk_uart0_mux", mux_uart0_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(10), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart2_fracmux __initdata = + MUX(SCLK_UART2_MUX, "sclk_uart2_mux", mux_uart2_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(12), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart3_fracmux __initdata = + MUX(SCLK_UART3_MUX, "sclk_uart3_mux", mux_uart3_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(14), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart4_fracmux __initdata = + MUX(SCLK_UART4_MUX, "sclk_uart4_mux", mux_uart4_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(16), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart5_fracmux __initdata = + MUX(SCLK_UART5_MUX, "sclk_uart5_mux", mux_uart5_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(18), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_i2s0_tx_fracmux __initdata = + MUX(MCLK_I2S0_TX_MUX, "mclk_i2s0_tx_mux", mux_i2s0_tx_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(30), 0, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_i2s0_rx_fracmux __initdata = + MUX(MCLK_I2S0_RX_MUX, "mclk_i2s0_rx_mux", mux_i2s0_rx_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(30), 2, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_i2s1_fracmux __initdata = + MUX(MCLK_I2S1_MUX, "mclk_i2s1_mux", mux_i2s1_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(31), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_i2s2_fracmux __initdata = + MUX(MCLK_I2S2_MUX, "mclk_i2s2_mux", mux_i2s2_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(33), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_audpwm_fracmux __initdata = + MUX(SCLK_AUDPWM_MUX, "mclk_audpwm_mux", mux_audpwm_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(36), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_dclk_vop_fracmux __initdata = + MUX(DCLK_VOP_MUX, "dclk_vop_mux", mux_dclk_vop_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(47), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_cif_out2io_fracmux __initdata = + MUX(CLK_CIF_OUT_MUX, "clk_cif_out2io_mux", mux_cif_out2io_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(50), 14, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_mipicsi_out2io_fracmux __initdata = + MUX(CLK_MIPICSI_OUT_MUX, "clk_mipicsi_out2io_mux", mux_mipicsi_out2io_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(73), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_aclk_pdvi_np5 __initdata = + COMPOSITE_HALFDIV_OFFSET(ACLK_PDVI_NP5, "aclk_pdvi_np5", mux_cpll_gpll_hpll_p, 0, + RV1126_CLKSEL_CON(49), 6, 2, MFLAGS, + RV1126_CLKSEL_CON(76), 0, 5, DFLAGS, + RV1126_CLKGATE_CON(16), 13, GFLAGS); + +static struct rockchip_clk_branch rv1126_clk_isp_np5 __initdata = + COMPOSITE_HALFDIV_OFFSET(CLK_ISP_NP5, "clk_isp_np5", mux_gpll_cpll_hpll_p, 0, + RV1126_CLKSEL_CON(50), 6, 2, MFLAGS, + RV1126_CLKSEL_CON(76), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(16), 14, GFLAGS); + +static struct rockchip_clk_branch rv1126_aclk_pdispp_np5 __initdata = + COMPOSITE_HALFDIV_OFFSET(ACLK_PDISPP_NP5, "aclk_pdispp_np5", mux_cpll_gpll_hpll_p, 0, + RV1126_CLKSEL_CON(68), 6, 2, MFLAGS, + RV1126_CLKSEL_CON(77), 0, 5, DFLAGS, + RV1126_CLKGATE_CON(16), 8, GFLAGS); + +static struct rockchip_clk_branch rv1126_clk_ispp_np5 __initdata = + COMPOSITE_HALFDIV_OFFSET(CLK_ISPP_NP5, "clk_ispp_np5", mux_cpll_gpll_hpll_p, 0, + RV1126_CLKSEL_CON(69), 6, 2, MFLAGS, + RV1126_CLKSEL_CON(77), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(16), 7, GFLAGS); + +static struct rockchip_clk_branch rv1126_aclk_pdnpu_npu5 __initdata = + COMPOSITE_HALFDIV(ACLK_PDNPU_NP5, "aclk_pdnpu_np5", mux_gpll_cpll_apll_hpll_p, 0, + RV1126_CLKSEL_CON(65), 8, 2, MFLAGS, 4, 4, DFLAGS, + RV1126_CLKGATE_CON(22), 1, GFLAGS); + +static struct rockchip_clk_branch rv1126_clk_npu_np5 __initdata = + COMPOSITE_HALFDIV(CLK_NPU_NP5, "clk_npu_np5", mux_gpll_cpll_apll_hpll_p, 0, + RV1126_CLKSEL_CON(67), 8, 2, MFLAGS, 4, 4, DFLAGS, + RV1126_CLKGATE_CON(22), 10, GFLAGS); + +static struct rockchip_clk_branch rv1126_clk_pmu_branches[] __initdata = { + /* + * Clock-Architecture Diagram 2 + */ + /* PD_PMU */ + COMPOSITE_NOMUX(PCLK_PDPMU, "pclk_pdpmu", "gpll", CLK_IS_CRITICAL, + RV1126_PMU_CLKSEL_CON(1), 0, 5, DFLAGS, + RV1126_PMU_CLKGATE_CON(0), 0, GFLAGS), + + COMPOSITE_FRACMUX(CLK_OSC0_DIV32K, "clk_osc0_div32k", "xin24m", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKSEL_CON(13), 0, + RV1126_PMU_CLKGATE_CON(2), 9, GFLAGS, + &rv1126_rtc32k_fracmux, 0), + + MUXPMUGRF(CLK_32K_IOE, "clk_32k_ioe", mux_clk_32k_ioe_p, 0, + RV1126_PMUGRF_SOC_CON0, 0, 1, MFLAGS), + + COMPOSITE_NOMUX(CLK_WIFI_DIV, "clk_wifi_div", "gpll", 0, + RV1126_PMU_CLKSEL_CON(12), 0, 6, DFLAGS, + RV1126_PMU_CLKGATE_CON(2), 10, GFLAGS), + GATE(CLK_WIFI_OSC0, "clk_wifi_osc0", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(2), 11, GFLAGS), + MUX(CLK_WIFI, "clk_wifi", mux_wifi_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(12), 8, 1, MFLAGS), + + GATE(PCLK_PMU, "pclk_pmu", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(0), 1, GFLAGS), + + GATE(PCLK_UART1, "pclk_uart1", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(0), 11, GFLAGS), + COMPOSITE(SCLK_UART1_DIV, "sclk_uart1_div", mux_gpll_usb480m_cpll_xin24m_p, 0, + RV1126_PMU_CLKSEL_CON(4), 8, 2, MFLAGS, 0, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(0), 12, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART1_FRACDIV, "sclk_uart1_fracdiv", "sclk_uart1_div", CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(5), 0, + RV1126_PMU_CLKGATE_CON(0), 13, GFLAGS, + &rv1126_uart1_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(SCLK_UART1, "sclk_uart1", "sclk_uart1_mux", 0, + RV1126_PMU_CLKGATE_CON(0), 14, GFLAGS), + + GATE(PCLK_I2C0, "pclk_i2c0", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(0), 5, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C0, "clk_i2c0", "gpll", 0, + RV1126_PMU_CLKSEL_CON(2), 0, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(0), 6, GFLAGS), + GATE(PCLK_I2C2, "pclk_i2c2", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(0), 9, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C2, "clk_i2c2", "gpll", 0, + RV1126_PMU_CLKSEL_CON(3), 0, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(0), 10, GFLAGS), + + GATE(CLK_CAPTURE_PWM0, "clk_capture_pwm0", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(1), 2, GFLAGS), + GATE(PCLK_PWM0, "pclk_pwm0", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(1), 0, GFLAGS), + COMPOSITE(CLK_PWM0, "clk_pwm0", mux_xin24m_gpll_p, 0, + RV1126_PMU_CLKSEL_CON(6), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(1), 1, GFLAGS), + GATE(CLK_CAPTURE_PWM1, "clk_capture_pwm1", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(1), 5, GFLAGS), + GATE(PCLK_PWM1, "pclk_pwm1", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(1), 3, GFLAGS), + COMPOSITE(CLK_PWM1, "clk_pwm1", mux_xin24m_gpll_p, 0, + RV1126_PMU_CLKSEL_CON(6), 15, 1, MFLAGS, 8, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(1), 4, GFLAGS), + + GATE(PCLK_SPI0, "pclk_spi0", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(1), 11, GFLAGS), + COMPOSITE(CLK_SPI0, "clk_spi0", mux_gpll_xin24m_p, 0, + RV1126_PMU_CLKSEL_CON(9), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(1), 12, GFLAGS), + + GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(1), 9, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO0, "dbclk_gpio0", mux_xin24m_32k_p, 0, + RV1126_PMU_CLKSEL_CON(8), 15, 1, MFLAGS, + RV1126_PMU_CLKGATE_CON(1), 10, GFLAGS), + + GATE(PCLK_PMUPVTM, "pclk_pmupvtm", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(2), 6, GFLAGS), + GATE(CLK_PMUPVTM, "clk_pmupvtm", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(2), 5, GFLAGS), + GATE(CLK_CORE_PMUPVTM, "clk_core_pmupvtm", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(2), 7, GFLAGS), + + COMPOSITE_NOMUX(CLK_REF12M, "clk_ref12m", "gpll", 0, + RV1126_PMU_CLKSEL_CON(7), 8, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(1), 15, GFLAGS), + GATE(0, "xin_osc0_usbphyref_otg", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(1), 6, GFLAGS), + GATE(0, "xin_osc0_usbphyref_host", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(1), 7, GFLAGS), + FACTOR(0, "xin_osc0_div2_usbphyref_otg", "xin_osc0_usbphyref_otg", 0, 1, 2), + FACTOR(0, "xin_osc0_div2_usbphyref_host", "xin_osc0_usbphyref_host", 0, 1, 2), + MUX(CLK_USBPHY_OTG_REF, "clk_usbphy_otg_ref", mux_usbphy_otg_ref_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(7), 6, 1, MFLAGS), + MUX(CLK_USBPHY_HOST_REF, "clk_usbphy_host_ref", mux_usbphy_host_ref_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(7), 7, 1, MFLAGS), + + COMPOSITE_NOMUX(CLK_REF24M, "clk_ref24m", "gpll", 0, + RV1126_PMU_CLKSEL_CON(7), 0, 6, DFLAGS, + RV1126_PMU_CLKGATE_CON(1), 14, GFLAGS), + GATE(0, "xin_osc0_mipiphyref", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(1), 8, GFLAGS), + MUX(CLK_MIPIDSIPHY_REF, "clk_mipidsiphy_ref", mux_mipidsiphy_ref_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(7), 15, 1, MFLAGS), + +#ifndef CONFIG_ROCKCHIP_LOW_PERFORMANCE + GATE(CLK_PMU, "clk_pmu", "xin24m", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(0), 15, GFLAGS), + + GATE(PCLK_PMUSGRF, "pclk_pmusgrf", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(0), 4, GFLAGS), + GATE(PCLK_PMUGRF, "pclk_pmugrf", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(1), 13, GFLAGS), + GATE(PCLK_PMUCRU, "pclk_pmucru", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(2), 4, GFLAGS), + GATE(PCLK_CHIPVEROTP, "pclk_chipverotp", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(2), 0, GFLAGS), + GATE(PCLK_PDPMU_NIU, "pclk_pdpmu_niu", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(0), 2, GFLAGS), + + GATE(PCLK_SCRKEYGEN, "pclk_scrkeygen", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(0), 7, GFLAGS), +#endif +}; + +static struct rockchip_clk_branch rv1126_clk_branches[] __initdata = { + /* + * Clock-Architecture Diagram 1 + */ + MUX(USB480M, "usb480m", mux_usb480m_p, CLK_SET_RATE_PARENT, + RV1126_MODE_CON, 10, 2, MFLAGS), + FACTOR(0, "xin12m", "xin24m", 0, 1, 2), + + /* + * Clock-Architecture Diagram 3 + */ + /* PD_CORE */ + COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(1), 0, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, + RV1126_CLKGATE_CON(0), 6, GFLAGS), + GATE(CLK_CORE_CPUPVTM, "clk_core_cpupvtm", "armclk", 0, + RV1126_CLKGATE_CON(0), 12, GFLAGS), + GATE(PCLK_CPUPVTM, "pclk_cpupvtm", "pclk_dbg", 0, + RV1126_CLKGATE_CON(0), 10, GFLAGS), + GATE(CLK_CPUPVTM, "clk_cpupvtm", "xin24m", 0, + RV1126_CLKGATE_CON(0), 11, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDCORE_NIU, "hclk_pdcore_niu", "gpll", CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(0), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(0), 8, GFLAGS), + + /* + * Clock-Architecture Diagram 4 + */ + /* PD_BUS */ + COMPOSITE(0, "aclk_pdbus_pre", mux_gpll_cpll_dpll_p, CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(2), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(2), 0, GFLAGS), + GATE(ACLK_PDBUS, "aclk_pdbus", "aclk_pdbus_pre", CLK_IS_CRITICAL, + RV1126_CLKGATE_CON(2), 11, GFLAGS), + COMPOSITE(0, "hclk_pdbus_pre", mux_hclk_pclk_pdbus_p, CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(2), 15, 1, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(2), 1, GFLAGS), + GATE(HCLK_PDBUS, "hclk_pdbus", "hclk_pdbus_pre", CLK_IS_CRITICAL, + RV1126_CLKGATE_CON(2), 12, GFLAGS), + COMPOSITE(0, "pclk_pdbus_pre", mux_hclk_pclk_pdbus_p, CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(3), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(2), 2, GFLAGS), + GATE(PCLK_PDBUS, "pclk_pdbus", "pclk_pdbus_pre", CLK_IS_CRITICAL, + RV1126_CLKGATE_CON(2), 13, GFLAGS), + /* aclk_dmac is controlled by sgrf_clkgat_con. */ + SGRF_GATE(ACLK_DMAC, "aclk_dmac", "hclk_pdbus"), + GATE(ACLK_DCF, "aclk_dcf", "hclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(3), 6, GFLAGS), + GATE(PCLK_DCF, "pclk_dcf", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(3), 7, GFLAGS), + GATE(PCLK_WDT, "pclk_wdt", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(6), 14, GFLAGS), + GATE(PCLK_MAILBOX, "pclk_mailbox", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 10, GFLAGS), + + COMPOSITE(CLK_SCR1, "clk_scr1", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(3), 15, 1, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(4), 7, GFLAGS), + GATE(0, "clk_scr1_niu", "clk_scr1", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 14, GFLAGS), + GATE(CLK_SCR1_CORE, "clk_scr1_core", "clk_scr1", 0, + RV1126_CLKGATE_CON(4), 8, GFLAGS), + GATE(CLK_SCR1_RTC, "clk_scr1_rtc", "xin24m", 0, + RV1126_CLKGATE_CON(4), 9, GFLAGS), + GATE(CLK_SCR1_JTAG, "clk_scr1_jtag", "clk_scr1_jtag_io", 0, + RV1126_CLKGATE_CON(4), 10, GFLAGS), + + GATE(PCLK_UART0, "pclk_uart0", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(5), 0, GFLAGS), + COMPOSITE(SCLK_UART0_DIV, "sclk_uart0_div", mux_gpll_cpll_usb480m_xin24m_p, 0, + RV1126_CLKSEL_CON(10), 8, 2, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(5), 1, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART0_FRAC, "sclk_uart0_frac", "sclk_uart0_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(11), 0, + RV1126_CLKGATE_CON(5), 2, GFLAGS, + &rv1126_uart0_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(SCLK_UART0, "sclk_uart0", "sclk_uart0_mux", 0, + RV1126_CLKGATE_CON(5), 3, GFLAGS), + GATE(PCLK_UART2, "pclk_uart2", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(5), 4, GFLAGS), + COMPOSITE(SCLK_UART2_DIV, "sclk_uart2_div", mux_gpll_cpll_usb480m_xin24m_p, 0, + RV1126_CLKSEL_CON(12), 8, 2, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(5), 5, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART2_FRAC, "sclk_uart2_frac", "sclk_uart2_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(13), 0, + RV1126_CLKGATE_CON(5), 6, GFLAGS, + &rv1126_uart2_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(SCLK_UART2, "sclk_uart2", "sclk_uart2_mux", 0, + RV1126_CLKGATE_CON(5), 7, GFLAGS), + GATE(PCLK_UART3, "pclk_uart3", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(5), 8, GFLAGS), + COMPOSITE(SCLK_UART3_DIV, "sclk_uart3_div", mux_gpll_cpll_usb480m_xin24m_p, 0, + RV1126_CLKSEL_CON(14), 8, 2, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(5), 9, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART3_FRAC, "sclk_uart3_frac", "sclk_uart3_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(15), 0, + RV1126_CLKGATE_CON(5), 10, GFLAGS, + &rv1126_uart3_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(SCLK_UART3, "sclk_uart3", "sclk_uart3_mux", 0, + RV1126_CLKGATE_CON(5), 11, GFLAGS), + GATE(PCLK_UART4, "pclk_uart4", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(5), 12, GFLAGS), + COMPOSITE(SCLK_UART4_DIV, "sclk_uart4_div", mux_gpll_cpll_usb480m_xin24m_p, 0, + RV1126_CLKSEL_CON(16), 8, 2, MFLAGS, 0, 7, + DFLAGS, RV1126_CLKGATE_CON(5), 13, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART4_FRAC, "sclk_uart4_frac", "sclk_uart4_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(17), 0, + RV1126_CLKGATE_CON(5), 14, GFLAGS, + &rv1126_uart4_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(SCLK_UART4, "sclk_uart4", "sclk_uart4_mux", 0, + RV1126_CLKGATE_CON(5), 15, GFLAGS), + GATE(PCLK_UART5, "pclk_uart5", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(6), 0, GFLAGS), + COMPOSITE(SCLK_UART5_DIV, "sclk_uart5_div", mux_gpll_cpll_usb480m_xin24m_p, 0, + RV1126_CLKSEL_CON(18), 8, 2, MFLAGS, 0, 7, + DFLAGS, RV1126_CLKGATE_CON(6), 1, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART5_FRAC, "sclk_uart5_frac", "sclk_uart5_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(19), 0, + RV1126_CLKGATE_CON(6), 2, GFLAGS, + &rv1126_uart5_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(SCLK_UART5, "sclk_uart5", "sclk_uart5_mux", 0, + RV1126_CLKGATE_CON(6), 3, GFLAGS), + + GATE(PCLK_I2C1, "pclk_i2c1", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(3), 10, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C1, "clk_i2c1", "gpll", 0, + RV1126_CLKSEL_CON(5), 0, 7, DFLAGS, + RV1126_CLKGATE_CON(3), 11, GFLAGS), + GATE(PCLK_I2C3, "pclk_i2c3", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(3), 12, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C3, "clk_i2c3", "gpll", 0, + RV1126_CLKSEL_CON(5), 8, 7, DFLAGS, + RV1126_CLKGATE_CON(3), 13, GFLAGS), + GATE(PCLK_I2C4, "pclk_i2c4", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(3), 14, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C4, "clk_i2c4", "gpll", 0, + RV1126_CLKSEL_CON(6), 0, 7, DFLAGS, + RV1126_CLKGATE_CON(3), 15, GFLAGS), + GATE(PCLK_I2C5, "pclk_i2c5", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(4), 0, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C5, "clk_i2c5", "gpll", 0, + RV1126_CLKSEL_CON(6), 8, 7, DFLAGS, + RV1126_CLKGATE_CON(4), 1, GFLAGS), + + GATE(PCLK_SPI1, "pclk_spi1", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(4), 2, GFLAGS), + COMPOSITE(CLK_SPI1, "clk_spi1", mux_gpll_xin24m_p, 0, + RV1126_CLKSEL_CON(8), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(4), 3, GFLAGS), + + GATE(CLK_CAPTURE_PWM2, "clk_capture_pwm2", "xin24m", 0, + RV1126_CLKGATE_CON(4), 6, GFLAGS), + GATE(PCLK_PWM2, "pclk_pwm2", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(4), 4, GFLAGS), + COMPOSITE(CLK_PWM2, "clk_pwm2", mux_xin24m_gpll_p, 0, + RV1126_CLKSEL_CON(9), 15, 1, MFLAGS, 8, 7, DFLAGS, + RV1126_CLKGATE_CON(4), 5, GFLAGS), + + GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 0, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO1, "dbclk_gpio1", mux_xin24m_32k_p, 0, + RV1126_CLKSEL_CON(21), 15, 1, MFLAGS, + RV1126_CLKGATE_CON(7), 1, GFLAGS), + GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 2, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO2, "dbclk_gpio2", mux_xin24m_32k_p, 0, + RV1126_CLKSEL_CON(22), 15, 1, MFLAGS, + RV1126_CLKGATE_CON(7), 3, GFLAGS), + GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 4, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO3, "dbclk_gpio3", mux_xin24m_32k_p, 0, + RV1126_CLKSEL_CON(23), 15, 1, MFLAGS, + RV1126_CLKGATE_CON(7), 5, GFLAGS), + GATE(PCLK_GPIO4, "pclk_gpio4", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 6, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO4, "dbclk_gpio4", mux_xin24m_32k_p, 0, + RV1126_CLKSEL_CON(24), 15, 1, MFLAGS, + RV1126_CLKGATE_CON(7), 7, GFLAGS), + + GATE(PCLK_SARADC, "pclk_saradc", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(6), 4, GFLAGS), + COMPOSITE_NOMUX(CLK_SARADC, "clk_saradc", "xin24m", 0, + RV1126_CLKSEL_CON(20), 0, 11, DFLAGS, + RV1126_CLKGATE_CON(6), 5, GFLAGS), + + GATE(PCLK_TIMER, "pclk_timer", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(6), 7, GFLAGS), + GATE(CLK_TIMER0, "clk_timer0", "xin24m", 0, + RV1126_CLKGATE_CON(6), 8, GFLAGS), + GATE(CLK_TIMER1, "clk_timer1", "xin24m", 0, + RV1126_CLKGATE_CON(6), 9, GFLAGS), + GATE(CLK_TIMER2, "clk_timer2", "xin24m", 0, + RV1126_CLKGATE_CON(6), 10, GFLAGS), + GATE(CLK_TIMER3, "clk_timer3", "xin24m", 0, + RV1126_CLKGATE_CON(6), 11, GFLAGS), + GATE(CLK_TIMER4, "clk_timer4", "xin24m", 0, + RV1126_CLKGATE_CON(6), 12, GFLAGS), + GATE(CLK_TIMER5, "clk_timer5", "xin24m", 0, + RV1126_CLKGATE_CON(6), 13, GFLAGS), + + GATE(ACLK_SPINLOCK, "aclk_spinlock", "hclk_pdbus", 0, + RV1126_CLKGATE_CON(6), 6, GFLAGS), + + GATE(ACLK_DECOM, "aclk_decom", "aclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 11, GFLAGS), + GATE(PCLK_DECOM, "pclk_decom", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 12, GFLAGS), + COMPOSITE(DCLK_DECOM, "dclk_decom", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(25), 15, 1, MFLAGS, 8, 7, DFLAGS, + RV1126_CLKGATE_CON(7), 13, GFLAGS), + + GATE(PCLK_CAN, "pclk_can", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 8, GFLAGS), + COMPOSITE(CLK_CAN, "clk_can", mux_gpll_xin24m_p, 0, + RV1126_CLKSEL_CON(25), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(7), 9, GFLAGS), + /* pclk_otp and clk_otp are controlled by sgrf_clkgat_con. */ + SGRF_GATE(CLK_OTP, "clk_otp", "xin24m"), + SGRF_GATE(PCLK_OTP, "pclk_otp", "pclk_pdbus"), + + GATE(PCLK_NPU_TSADC, "pclk_npu_tsadc", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(24), 3, GFLAGS), + COMPOSITE_NOMUX(CLK_NPU_TSADC, "clk_npu_tsadc", "xin24m", 0, + RV1126_CLKSEL_CON(71), 0, 11, DFLAGS, + RV1126_CLKGATE_CON(24), 4, GFLAGS), + GATE(CLK_NPU_TSADCPHY, "clk_npu_tsadcphy", "clk_npu_tsadc", 0, + RV1126_CLKGATE_CON(24), 5, GFLAGS), + GATE(PCLK_CPU_TSADC, "pclk_cpu_tsadc", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(24), 0, GFLAGS), + COMPOSITE_NOMUX(CLK_CPU_TSADC, "clk_cpu_tsadc", "xin24m", 0, + RV1126_CLKSEL_CON(70), 0, 11, DFLAGS, + RV1126_CLKGATE_CON(24), 1, GFLAGS), + GATE(CLK_CPU_TSADCPHY, "clk_cpu_tsadcphy", "clk_cpu_tsadc", 0, + RV1126_CLKGATE_CON(24), 2, GFLAGS), + + /* + * Clock-Architecture Diagram 5 + */ + /* PD_CRYPTO */ + COMPOSITE(ACLK_PDCRYPTO, "aclk_pdcrypto", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(4), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(4), 11, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDCRYPTO, "hclk_pdcrypto", "aclk_pdcrypto", 0, + RV1126_CLKSEL_CON(4), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(4), 12, GFLAGS), + GATE(ACLK_CRYPTO, "aclk_crypto", "aclk_pdcrypto", 0, + RV1126_CLKGATE_CON(3), 2, GFLAGS), + GATE(HCLK_CRYPTO, "hclk_crypto", "hclk_pdcrypto", 0, + RV1126_CLKGATE_CON(3), 3, GFLAGS), + COMPOSITE(CLK_CRYPTO_CORE, "aclk_crypto_core", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(7), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(3), 4, GFLAGS), + COMPOSITE(CLK_CRYPTO_PKA, "aclk_crypto_pka", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(7), 15, 1, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(3), 5, GFLAGS), + + /* + * Clock-Architecture Diagram 6 + */ + /* PD_AUDIO */ + COMPOSITE_NOMUX(HCLK_PDAUDIO, "hclk_pdaudio", "gpll", 0, + RV1126_CLKSEL_CON(26), 0, 5, DFLAGS, + RV1126_CLKGATE_CON(9), 0, GFLAGS), + + GATE(HCLK_I2S0, "hclk_i2s0", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(9), 4, GFLAGS), + COMPOSITE(MCLK_I2S0_TX_DIV, "mclk_i2s0_tx_div", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(27), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(9), 5, GFLAGS), + COMPOSITE_FRACMUX(MCLK_I2S0_TX_FRACDIV, "mclk_i2s0_tx_fracdiv", "mclk_i2s0_tx_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(28), 0, + RV1126_CLKGATE_CON(9), 6, GFLAGS, + &rv1126_i2s0_tx_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(MCLK_I2S0_TX, "mclk_i2s0_tx", "mclk_i2s0_tx_mux", 0, + RV1126_CLKGATE_CON(9), 9, GFLAGS), + COMPOSITE(MCLK_I2S0_RX_DIV, "mclk_i2s0_rx_div", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(27), 15, 1, MFLAGS, 8, 7, DFLAGS, + RV1126_CLKGATE_CON(9), 7, GFLAGS), + COMPOSITE_FRACMUX(MCLK_I2S0_RX_FRACDIV, "mclk_i2s0_rx_fracdiv", "mclk_i2s0_rx_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(29), 0, + RV1126_CLKGATE_CON(9), 8, GFLAGS, + &rv1126_i2s0_rx_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(MCLK_I2S0_RX, "mclk_i2s0_rx", "mclk_i2s0_rx_mux", 0, + RV1126_CLKGATE_CON(9), 10, GFLAGS), + COMPOSITE_NODIV(MCLK_I2S0_TX_OUT2IO, "mclk_i2s0_tx_out2io", mux_i2s0_tx_out2io_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(30), 6, 1, MFLAGS, + RV1126_CLKGATE_CON(9), 13, GFLAGS), + COMPOSITE_NODIV(MCLK_I2S0_RX_OUT2IO, "mclk_i2s0_rx_out2io", mux_i2s0_rx_out2io_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(30), 8, 1, MFLAGS, + RV1126_CLKGATE_CON(9), 14, GFLAGS), + + GATE(HCLK_I2S1, "hclk_i2s1", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(10), 0, GFLAGS), + COMPOSITE(MCLK_I2S1_DIV, "mclk_i2s1_div", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(31), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(10), 1, GFLAGS), + COMPOSITE_FRACMUX(MCLK_I2S1_FRACDIV, "mclk_i2s1_fracdiv", "mclk_i2s1_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(32), 0, + RV1126_CLKGATE_CON(10), 2, GFLAGS, + &rv1126_i2s1_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(MCLK_I2S1, "mclk_i2s1", "mclk_i2s1_mux", 0, + RV1126_CLKGATE_CON(10), 3, GFLAGS), + COMPOSITE_NODIV(MCLK_I2S1_OUT2IO, "mclk_i2s1_out2io", mux_i2s1_out2io_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(31), 12, 1, MFLAGS, + RV1126_CLKGATE_CON(10), 4, GFLAGS), + GATE(HCLK_I2S2, "hclk_i2s2", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(10), 5, GFLAGS), + COMPOSITE(MCLK_I2S2_DIV, "mclk_i2s2_div", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(33), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(10), 6, GFLAGS), + COMPOSITE_FRACMUX(MCLK_I2S2_FRACDIV, "mclk_i2s2_fracdiv", "mclk_i2s2_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(34), 0, + RV1126_CLKGATE_CON(10), 7, GFLAGS, + &rv1126_i2s2_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(MCLK_I2S2, "mclk_i2s2", "mclk_i2s2_mux", 0, + RV1126_CLKGATE_CON(10), 8, GFLAGS), + COMPOSITE_NODIV(MCLK_I2S2_OUT2IO, "mclk_i2s2_out2io", mux_i2s2_out2io_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(33), 10, 1, MFLAGS, + RV1126_CLKGATE_CON(10), 9, GFLAGS), + + GATE(HCLK_PDM, "hclk_pdm", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(10), 10, GFLAGS), + COMPOSITE(MCLK_PDM, "mclk_pdm", mux_gpll_cpll_xin24m_p, 0, + RV1126_CLKSEL_CON(35), 8, 2, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(10), 11, GFLAGS), + + GATE(HCLK_AUDPWM, "hclk_audpwm", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(10), 12, GFLAGS), + COMPOSITE(SCLK_ADUPWM_DIV, "sclk_audpwm_div", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(36), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(10), 13, GFLAGS), + COMPOSITE_FRACMUX(SCLK_AUDPWM_FRACDIV, "sclk_audpwm_fracdiv", "sclk_audpwm_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(37), 0, + RV1126_CLKGATE_CON(10), 14, GFLAGS, + &rv1126_audpwm_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(SCLK_AUDPWM, "sclk_audpwm", "mclk_audpwm_mux", 0, + RV1126_CLKGATE_CON(10), 15, GFLAGS), + + GATE(PCLK_ACDCDIG, "pclk_acdcdig", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(11), 0, GFLAGS), + GATE(CLK_ACDCDIG_ADC, "clk_acdcdig_adc", "mclk_i2s0_rx", 0, + RV1126_CLKGATE_CON(11), 2, GFLAGS), + GATE(CLK_ACDCDIG_DAC, "clk_acdcdig_dac", "mclk_i2s0_tx", 0, + RV1126_CLKGATE_CON(11), 3, GFLAGS), + COMPOSITE(CLK_ACDCDIG_I2C, "clk_acdcdig_i2c", mux_gpll_xin24m_p, 0, + RV1126_CLKSEL_CON(72), 8, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(11), 1, GFLAGS), + + /* + * Clock-Architecture Diagram 7 + */ + /* PD_VEPU */ + COMPOSITE(ACLK_PDVEPU, "aclk_pdvepu", mux_cpll_hpll_gpll_p, 0, + RV1126_CLKSEL_CON(40), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(12), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDVEPU, "hclk_pdvepu", "aclk_pdvepu", 0, + RV1126_CLKSEL_CON(41), 0, 5, DFLAGS, + RV1126_CLKGATE_CON(12), 2, GFLAGS), + GATE(ACLK_VENC, "aclk_venc", "aclk_pdvepu", 0, + RV1126_CLKGATE_CON(12), 5, GFLAGS), + GATE(HCLK_VENC, "hclk_venc", "hclk_pdvepu", 0, + RV1126_CLKGATE_CON(12), 6, GFLAGS), + COMPOSITE(CLK_VENC_CORE, "clk_venc_core", mux_cpll_gpll_hpll_p, 0, + RV1126_CLKSEL_CON(40), 14, 2, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(12), 1, GFLAGS), + + /* + * Clock-Architecture Diagram 8 + */ + /* PD_VDPU */ +#if IS_ENABLED(CONFIG_ROCKCHIP_MPP_VDPU2) || IS_ENABLED(CONFIG_ROCKCHIP_MPP_RKVDEC) + COMPOSITE(ACLK_PDVDEC, "aclk_pdvdec", mux_cpll_hpll_gpll_p, CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(42), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDVDEC, "hclk_pdvdec", "aclk_pdvdec", CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(41), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 4, GFLAGS), + GATE(0, "aclk_pdvdec_niu", "aclk_pdvdec", CLK_IS_CRITICAL, + RV1126_CLKGATE_CON(13), 5, GFLAGS), + GATE(0, "hclk_pdvdec_niu", "hclk_pdvdec", CLK_IS_CRITICAL, + RV1126_CLKGATE_CON(13), 6, GFLAGS), + COMPOSITE(ACLK_PDJPEG, "aclk_pdjpeg", mux_cpll_hpll_gpll_p, CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(44), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 9, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDJPEG, "hclk_pdjpeg", "aclk_pdjpeg", CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(44), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 10, GFLAGS), + GATE(0, "aclk_pdjpeg_niu", "aclk_pdjpeg", CLK_IS_CRITICAL, + RV1126_CLKGATE_CON(13), 11, GFLAGS), + GATE(0, "hclk_pdjpeg_niu", "hclk_pdjpeg", CLK_IS_CRITICAL, + RV1126_CLKGATE_CON(13), 12, GFLAGS), +#else + COMPOSITE(ACLK_PDVDEC, "aclk_pdvdec", mux_cpll_hpll_gpll_p, 0, + RV1126_CLKSEL_CON(42), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDVDEC, "hclk_pdvdec", "aclk_pdvdec", 0, + RV1126_CLKSEL_CON(41), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 4, GFLAGS), + GATE(0, "aclk_pdvdec_niu", "aclk_pdvdec", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(13), 5, GFLAGS), + GATE(0, "hclk_pdvdec_niu", "hclk_pdvdec", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(13), 6, GFLAGS), + COMPOSITE(ACLK_PDJPEG, "aclk_pdjpeg", mux_cpll_hpll_gpll_p, 0, + RV1126_CLKSEL_CON(44), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 9, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDJPEG, "hclk_pdjpeg", "aclk_pdjpeg", 0, + RV1126_CLKSEL_CON(44), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 10, GFLAGS), + GATE(0, "aclk_pdjpeg_niu", "aclk_pdjpeg", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(13), 11, GFLAGS), + GATE(0, "hclk_pdjpeg_niu", "hclk_pdjpeg", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(13), 12, GFLAGS), +#endif + GATE(ACLK_VDEC, "aclk_vdec", "aclk_pdvdec", 0, + RV1126_CLKGATE_CON(13), 7, GFLAGS), + GATE(HCLK_VDEC, "hclk_vdec", "hclk_pdvdec", 0, + RV1126_CLKGATE_CON(13), 8, GFLAGS), + COMPOSITE(CLK_VDEC_CORE, "clk_vdec_core", mux_cpll_hpll_gpll_p, 0, + RV1126_CLKSEL_CON(42), 14, 2, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 1, GFLAGS), + COMPOSITE(CLK_VDEC_CA, "clk_vdec_ca", mux_cpll_hpll_gpll_p, 0, + RV1126_CLKSEL_CON(43), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 2, GFLAGS), + COMPOSITE(CLK_VDEC_HEVC_CA, "clk_vdec_hevc_ca", mux_cpll_hpll_gpll_p, 0, + RV1126_CLKSEL_CON(43), 14, 2, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(13), 3, GFLAGS), + GATE(ACLK_JPEG, "aclk_jpeg", "aclk_pdjpeg", 0, + RV1126_CLKGATE_CON(13), 13, GFLAGS), + GATE(HCLK_JPEG, "hclk_jpeg", "hclk_pdjpeg", 0, + RV1126_CLKGATE_CON(13), 14, GFLAGS), + + /* + * Clock-Architecture Diagram 9 + */ + /* PD_VO */ + COMPOSITE(ACLK_PDVO, "aclk_pdvo", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(45), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(14), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDVO, "hclk_pdvo", "aclk_pdvo", 0, + RV1126_CLKSEL_CON(45), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(14), 1, GFLAGS), + COMPOSITE_NOMUX(PCLK_PDVO, "pclk_pdvo", "aclk_pdvo", 0, + RV1126_CLKSEL_CON(46), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(14), 2, GFLAGS), + GATE(ACLK_RGA, "aclk_rga", "aclk_pdvo", 0, + RV1126_CLKGATE_CON(14), 6, GFLAGS), + GATE(HCLK_RGA, "hclk_rga", "hclk_pdvo", 0, + RV1126_CLKGATE_CON(14), 7, GFLAGS), + COMPOSITE(CLK_RGA_CORE, "clk_rga_core", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(46), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(14), 8, GFLAGS), + GATE(ACLK_VOP, "aclk_vop", "aclk_pdvo", 0, + RV1126_CLKGATE_CON(14), 9, GFLAGS), + GATE(HCLK_VOP, "hclk_vop", "hclk_pdvo", 0, + RV1126_CLKGATE_CON(14), 10, GFLAGS), + COMPOSITE(DCLK_VOP_DIV, "dclk_vop_div", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(47), 8, 1, MFLAGS, 0, 8, DFLAGS, + RV1126_CLKGATE_CON(14), 11, GFLAGS), + COMPOSITE_FRACMUX(DCLK_VOP_FRACDIV, "dclk_vop_fracdiv", "dclk_vop_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(48), 0, + RV1126_CLKGATE_CON(14), 12, GFLAGS, + &rv1126_dclk_vop_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(DCLK_VOP, "dclk_vop", "dclk_vop_mux", 0, + RV1126_CLKGATE_CON(14), 13, GFLAGS), + GATE(PCLK_DSIHOST, "pclk_dsihost", "pclk_pdvo", 0, + RV1126_CLKGATE_CON(14), 14, GFLAGS), + GATE(ACLK_IEP, "aclk_iep", "aclk_pdvo", 0, + RV1126_CLKGATE_CON(12), 7, GFLAGS), + GATE(HCLK_IEP, "hclk_iep", "hclk_pdvo", 0, + RV1126_CLKGATE_CON(12), 8, GFLAGS), + COMPOSITE(CLK_IEP_CORE, "clk_iep_core", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(54), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(12), 9, GFLAGS), + + /* + * Clock-Architecture Diagram 10 + */ + /* PD_VI */ + COMPOSITE_BROTHER(ACLK_PDVI_DIV, "aclk_pdvi_div", mux_cpll_gpll_hpll_p, 0, + RV1126_CLKSEL_CON(49), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(15), 0, GFLAGS, + &rv1126_aclk_pdvi_np5), + MUX(ACLK_PDVI, "aclk_pdvi", mux_aclk_pdvi_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + RV1126_CLKSEL_CON(76), 5, 1, MFLAGS), + COMPOSITE_NOMUX(HCLK_PDVI, "hclk_pdvi", "aclk_pdvi", 0, + RV1126_CLKSEL_CON(49), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(15), 1, GFLAGS), + COMPOSITE_NOMUX(PCLK_PDVI, "pclk_pdvi", "aclk_pdvi", 0, + RV1126_CLKSEL_CON(50), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(15), 2, GFLAGS), + GATE(ACLK_ISP, "aclk_isp", "aclk_pdvi", 0, + RV1126_CLKGATE_CON(15), 6, GFLAGS), + GATE(HCLK_ISP, "hclk_isp", "hclk_pdvi", 0, + RV1126_CLKGATE_CON(15), 7, GFLAGS), + COMPOSITE_BROTHER(CLK_ISP_DIV, "clk_isp_div", mux_gpll_cpll_hpll_p, 0, + RV1126_CLKSEL_CON(50), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(15), 8, GFLAGS, + &rv1126_clk_isp_np5), + MUX(CLK_ISP, "clk_isp", mux_clk_isp_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + RV1126_CLKSEL_CON(76), 13, 1, MFLAGS), + GATE(ACLK_CIF, "aclk_cif", "aclk_pdvi", 0, + RV1126_CLKGATE_CON(15), 9, GFLAGS), + GATE(HCLK_CIF, "hclk_cif", "hclk_pdvi", 0, + RV1126_CLKGATE_CON(15), 10, GFLAGS), + COMPOSITE(DCLK_CIF, "dclk_cif", mux_gpll_cpll_hpll_p, 0, + RV1126_CLKSEL_CON(51), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(15), 11, GFLAGS), + COMPOSITE(CLK_CIF_OUT_DIV, "clk_cif_out2io_div", mux_gpll_usb480m_p, 0, + RV1126_CLKSEL_CON(51), 15, 1, MFLAGS, 8, 6, DFLAGS, + RV1126_CLKGATE_CON(15), 12, GFLAGS), + COMPOSITE_FRACMUX(CLK_CIF_OUT_FRACDIV, "clk_cif_out2io_fracdiv", "clk_cif_out2io_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(52), 0, + RV1126_CLKGATE_CON(15), 13, GFLAGS, + &rv1126_cif_out2io_fracmux, RV1126_FRAC_MAX_PRATE), + GATE(CLK_CIF_OUT, "clk_cif_out2io", "clk_cif_out2io_mux", 0, + RV1126_CLKGATE_CON(15), 14, GFLAGS), + COMPOSITE(CLK_MIPICSI_OUT_DIV, "clk_mipicsi_out2io_div", mux_gpll_usb480m_p, 0, + RV1126_CLKSEL_CON(73), 8, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(23), 5, GFLAGS), + COMPOSITE_FRACMUX(CLK_MIPICSI_OUT_FRACDIV, "clk_mipicsi_out2io_fracdiv", "clk_mipicsi_out2io_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(74), 0, + RV1126_CLKGATE_CON(23), 6, GFLAGS, + &rv1126_mipicsi_out2io_fracmux, RV1126_CSIOUT_FRAC_MAX_PRATE), + GATE(CLK_MIPICSI_OUT, "clk_mipicsi_out2io", "clk_mipicsi_out2io_mux", 0, + RV1126_CLKGATE_CON(23), 7, GFLAGS), + GATE(PCLK_CSIHOST, "pclk_csihost", "pclk_pdvi", 0, + RV1126_CLKGATE_CON(15), 15, GFLAGS), + GATE(ACLK_CIFLITE, "aclk_ciflite", "aclk_pdvi", 0, + RV1126_CLKGATE_CON(16), 10, GFLAGS), + GATE(HCLK_CIFLITE, "hclk_ciflite", "hclk_pdvi", 0, + RV1126_CLKGATE_CON(16), 11, GFLAGS), + COMPOSITE(DCLK_CIFLITE, "dclk_ciflite", mux_gpll_cpll_hpll_p, 0, + RV1126_CLKSEL_CON(54), 14, 2, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(16), 12, GFLAGS), + + /* + * Clock-Architecture Diagram 11 + */ + /* PD_ISPP */ + COMPOSITE_BROTHER(ACLK_PDISPP_DIV, "aclk_pdispp_div", mux_cpll_gpll_hpll_p, 0, + RV1126_CLKSEL_CON(68), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(16), 0, GFLAGS, + &rv1126_aclk_pdispp_np5), + MUX(ACLK_PDISPP, "aclk_pdispp", mux_aclk_pdispp_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + RV1126_CLKSEL_CON(77), 5, 1, MFLAGS), + COMPOSITE_NOMUX(HCLK_PDISPP, "hclk_pdispp", "aclk_pdispp", 0, + RV1126_CLKSEL_CON(69), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(16), 1, GFLAGS), + GATE(ACLK_ISPP, "aclk_ispp", "aclk_pdispp", 0, + RV1126_CLKGATE_CON(16), 4, GFLAGS), + GATE(HCLK_ISPP, "hclk_ispp", "hclk_pdispp", 0, + RV1126_CLKGATE_CON(16), 5, GFLAGS), + COMPOSITE_BROTHER(CLK_ISPP_DIV, "clk_ispp_div", mux_cpll_gpll_hpll_p, 0, + RV1126_CLKSEL_CON(69), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(16), 6, GFLAGS, + &rv1126_clk_ispp_np5), + MUX(CLK_ISPP, "clk_ispp", mux_clk_ispp_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + RV1126_CLKSEL_CON(77), 13, 1, MFLAGS), + + /* + * Clock-Architecture Diagram 12 + */ + /* PD_PHP */ + COMPOSITE(ACLK_PDPHP, "aclk_pdphp", mux_gpll_cpll_p, CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(53), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(17), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDPHP, "hclk_pdphp", "gpll", CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(53), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(17), 1, GFLAGS), + /* PD_SDCARD */ + GATE(HCLK_PDSDMMC, "hclk_pdsdmmc", "hclk_pdphp", 0, + RV1126_CLKGATE_CON(17), 6, GFLAGS), + GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_pdsdmmc", 0, + RV1126_CLKGATE_CON(18), 4, GFLAGS), + COMPOSITE(CLK_SDMMC, "clk_sdmmc", mux_gpll_cpll_xin24m_p, 0, + RV1126_CLKSEL_CON(55), 14, 2, MFLAGS, 0, 8, + DFLAGS, RV1126_CLKGATE_CON(18), 5, GFLAGS), + MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "clk_sdmmc", RV1126_SDMMC_CON0, 1), + MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "clk_sdmmc", RV1126_SDMMC_CON1, 1), + + /* PD_SDIO */ + GATE(HCLK_PDSDIO, "hclk_pdsdio", "hclk_pdphp", 0, + RV1126_CLKGATE_CON(17), 8, GFLAGS), + GATE(HCLK_SDIO, "hclk_sdio", "hclk_pdsdio", 0, + RV1126_CLKGATE_CON(18), 6, GFLAGS), + COMPOSITE(CLK_SDIO, "clk_sdio", mux_gpll_cpll_xin24m_p, 0, + RV1126_CLKSEL_CON(56), 14, 2, MFLAGS, 0, 8, DFLAGS, + RV1126_CLKGATE_CON(18), 7, GFLAGS), + MMC(SCLK_SDIO_DRV, "sdio_drv", "clk_sdio", RV1126_SDIO_CON0, 1), + MMC(SCLK_SDIO_SAMPLE, "sdio_sample", "clk_sdio", RV1126_SDIO_CON1, 1), + + /* PD_NVM */ + GATE(HCLK_PDNVM, "hclk_pdnvm", "hclk_pdphp", 0, + RV1126_CLKGATE_CON(18), 1, GFLAGS), + GATE(HCLK_EMMC, "hclk_emmc", "hclk_pdnvm", 0, + RV1126_CLKGATE_CON(18), 8, GFLAGS), + COMPOSITE(CLK_EMMC, "clk_emmc", mux_gpll_cpll_xin24m_p, 0, + RV1126_CLKSEL_CON(57), 14, 2, MFLAGS, 0, 8, DFLAGS, + RV1126_CLKGATE_CON(18), 9, GFLAGS), + GATE(HCLK_NANDC, "hclk_nandc", "hclk_pdnvm", 0, + RV1126_CLKGATE_CON(18), 13, GFLAGS), + COMPOSITE(CLK_NANDC, "clk_nandc", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(59), 15, 1, MFLAGS, 0, 8, DFLAGS, + RV1126_CLKGATE_CON(18), 14, GFLAGS), + GATE(HCLK_SFC, "hclk_sfc", "hclk_pdnvm", 0, + RV1126_CLKGATE_CON(18), 10, GFLAGS), + GATE(HCLK_SFCXIP, "hclk_sfcxip", "hclk_pdnvm", 0, + RV1126_CLKGATE_CON(18), 11, GFLAGS), + COMPOSITE(SCLK_SFC, "sclk_sfc", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(58), 15, 1, MFLAGS, 0, 8, DFLAGS, + RV1126_CLKGATE_CON(18), 12, GFLAGS), + MMC(SCLK_EMMC_DRV, "emmc_drv", "clk_emmc", RV1126_EMMC_CON0, 1), + MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "clk_emmc", RV1126_EMMC_CON1, 1), + + /* PD_USB */ + GATE(ACLK_PDUSB, "aclk_pdusb", "aclk_pdphp", 0, + RV1126_CLKGATE_CON(19), 0, GFLAGS), + GATE(HCLK_PDUSB, "hclk_pdusb", "hclk_pdphp", 0, + RV1126_CLKGATE_CON(19), 1, GFLAGS), + GATE(HCLK_USBHOST, "hclk_usbhost", "hclk_pdusb", 0, + RV1126_CLKGATE_CON(19), 4, GFLAGS), + GATE(HCLK_USBHOST_ARB, "hclk_usbhost_arb", "hclk_pdusb", 0, + RV1126_CLKGATE_CON(19), 5, GFLAGS), +#if IS_ENABLED(CONFIG_USB_EHCI_HCD_PLATFORM) || IS_ENABLED(CONFIG_USB_OHCI_HCD_PLATFORM) + COMPOSITE(CLK_USBHOST_UTMI_OHCI, "clk_usbhost_utmi_ohci", mux_usb480m_gpll_p, CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(61), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(19), 6, GFLAGS), +#else + COMPOSITE(CLK_USBHOST_UTMI_OHCI, "clk_usbhost_utmi_ohci", mux_usb480m_gpll_p, 0, + RV1126_CLKSEL_CON(61), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(19), 6, GFLAGS), +#endif + GATE(ACLK_USBOTG, "aclk_usbotg", "aclk_pdusb", 0, + RV1126_CLKGATE_CON(19), 7, GFLAGS), + GATE(CLK_USBOTG_REF, "clk_usbotg_ref", "xin24m", 0, + RV1126_CLKGATE_CON(19), 8, GFLAGS), + /* PD_GMAC */ + GATE(ACLK_PDGMAC, "aclk_pdgmac", "aclk_pdphp", 0, + RV1126_CLKGATE_CON(20), 0, GFLAGS), + COMPOSITE_NOMUX(PCLK_PDGMAC, "pclk_pdgmac", "aclk_pdgmac", 0, + RV1126_CLKSEL_CON(63), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(20), 1, GFLAGS), + GATE(ACLK_GMAC, "aclk_gmac", "aclk_pdgmac", 0, + RV1126_CLKGATE_CON(20), 4, GFLAGS), + GATE(PCLK_GMAC, "pclk_gmac", "pclk_pdgmac", 0, + RV1126_CLKGATE_CON(20), 5, GFLAGS), + + COMPOSITE(CLK_GMAC_DIV, "clk_gmac_div", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(63), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(20), 6, GFLAGS), + GATE(CLK_GMAC_RGMII_M0, "clk_gmac_rgmii_m0", "clk_gmac_rgmii_clkin_m0", 0, + RV1126_CLKGATE_CON(20), 12, GFLAGS), + MUX(CLK_GMAC_SRC_M0, "clk_gmac_src_m0", clk_gmac_src_m0_p, CLK_SET_RATE_PARENT, + RV1126_GMAC_CON, 0, 1, MFLAGS), + GATE(CLK_GMAC_RGMII_M1, "clk_gmac_rgmii_m1", "clk_gmac_rgmii_clkin_m1", 0, + RV1126_CLKGATE_CON(20), 13, GFLAGS), + MUX(CLK_GMAC_SRC_M1, "clk_gmac_src_m1", clk_gmac_src_m1_p, CLK_SET_RATE_PARENT, + RV1126_GMAC_CON, 5, 1, MFLAGS), + MUXGRF(CLK_GMAC_SRC, "clk_gmac_src", mux_clk_gmac_src_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RV1126_GRF_IOFUNC_CON1, 12, 1, MFLAGS), + + GATE(CLK_GMAC_REF, "clk_gmac_ref", "clk_gmac_src", 0, + RV1126_CLKGATE_CON(20), 7, GFLAGS), + + GATE(CLK_GMAC_TX_SRC, "clk_gmac_tx_src", "clk_gmac_src", 0, + RV1126_CLKGATE_CON(20), 9, GFLAGS), + FACTOR(CLK_GMAC_TX_DIV5, "clk_gmac_tx_div5", "clk_gmac_tx_src", 0, 1, 5), + FACTOR(CLK_GMAC_TX_DIV50, "clk_gmac_tx_div50", "clk_gmac_tx_src", 0, 1, 50), + MUXTBL(RGMII_MODE_CLK, "rgmii_mode_clk", mux_rgmii_clk_p, CLK_SET_RATE_PARENT, + RV1126_GMAC_CON, 2, 2, MFLAGS, rgmii_mux_idx), + GATE(CLK_GMAC_RX_SRC, "clk_gmac_rx_src", "clk_gmac_src", 0, + RV1126_CLKGATE_CON(20), 8, GFLAGS), + FACTOR(CLK_GMAC_RX_DIV2, "clk_gmac_rx_div2", "clk_gmac_rx_src", 0, 1, 2), + FACTOR(CLK_GMAC_RX_DIV20, "clk_gmac_rx_div20", "clk_gmac_rx_src", 0, 1, 20), + MUX(RMII_MODE_CLK, "rmii_mode_clk", mux_rmii_clk_p, CLK_SET_RATE_PARENT, + RV1126_GMAC_CON, 1, 1, MFLAGS), + MUX(CLK_GMAC_TX_RX, "clk_gmac_tx_rx", mux_gmac_tx_rx_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + RV1126_GMAC_CON, 4, 1, MFLAGS), + + GATE(CLK_GMAC_PTPREF, "clk_gmac_ptpref", "xin24m", 0, + RV1126_CLKGATE_CON(20), 10, GFLAGS), + COMPOSITE(CLK_GMAC_ETHERNET_OUT, "clk_gmac_ethernet_out2io", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(61), 15, 1, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(20), 11, GFLAGS), + + + /* + * Clock-Architecture Diagram 14 + */ + /* PD_NPU */ + COMPOSITE_BROTHER(ACLK_PDNPU_DIV, "aclk_pdnpu_div", mux_gpll_cpll_apll_hpll_p, 0, + RV1126_CLKSEL_CON(65), 8, 2, MFLAGS, 0, 4, DFLAGS, + RV1126_CLKGATE_CON(22), 0, GFLAGS, + &rv1126_aclk_pdnpu_npu5), + MUX(ACLK_PDNPU, "aclk_pdnpu", mux_aclk_pdnpu_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + RV1126_CLKSEL_CON(65), 12, 1, MFLAGS), + COMPOSITE_NOMUX(HCLK_PDNPU, "hclk_pdnpu", "gpll", 0, + RV1126_CLKSEL_CON(66), 8, 4, DFLAGS, + RV1126_CLKGATE_CON(22), 2, GFLAGS), + COMPOSITE_NOMUX(PCLK_PDNPU, "pclk_pdnpu", "hclk_pdnpu", 0, + RV1126_CLKSEL_CON(66), 0, 5, DFLAGS, + RV1126_CLKGATE_CON(22), 3, GFLAGS), + GATE(ACLK_NPU, "aclk_npu", "aclk_pdnpu", 0, + RV1126_CLKGATE_CON(22), 7, GFLAGS), + GATE(HCLK_NPU, "hclk_npu", "hclk_pdnpu", 0, + RV1126_CLKGATE_CON(22), 8, GFLAGS), + COMPOSITE_BROTHER(CLK_NPU_DIV, "clk_npu_div", mux_gpll_cpll_apll_hpll_p, 0, + RV1126_CLKSEL_CON(67), 8, 2, MFLAGS, 0, 4, DFLAGS, + RV1126_CLKGATE_CON(22), 9, GFLAGS, + &rv1126_clk_npu_np5), + MUX(CLK_CORE_NPU, "clk_core_npu", mux_clk_npu_p, CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + RV1126_CLKSEL_CON(67), 12, 1, MFLAGS), + GATE(CLK_CORE_NPUPVTM, "clk_core_npupvtm", "clk_core_npu", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(22), 14, GFLAGS), + GATE(CLK_NPUPVTM, "clk_npupvtm", "xin24m", 0, + RV1126_CLKGATE_CON(22), 13, GFLAGS), + GATE(PCLK_NPUPVTM, "pclk_npupvtm", "pclk_pdnpu", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(22), 12, GFLAGS), + + /* + * Clock-Architecture Diagram 15 + */ + GATE(PCLK_PDTOP, "pclk_pdtop", "pclk_pdbus", CLK_IS_CRITICAL, + RV1126_CLKGATE_CON(23), 8, GFLAGS), + GATE(PCLK_DSIPHY, "pclk_dsiphy", "pclk_pdtop", 0, + RV1126_CLKGATE_CON(23), 4, GFLAGS), + GATE(PCLK_CSIPHY0, "pclk_csiphy0", "pclk_pdtop", 0, + RV1126_CLKGATE_CON(23), 2, GFLAGS), + GATE(PCLK_CSIPHY1, "pclk_csiphy1", "pclk_pdtop", 0, + RV1126_CLKGATE_CON(23), 3, GFLAGS), + GATE(PCLK_USBPHY_HOST, "pclk_usbphy_host", "pclk_pdtop", 0, + RV1126_CLKGATE_CON(19), 13, GFLAGS), + GATE(PCLK_USBPHY_OTG, "pclk_usbphy_otg", "pclk_pdtop", 0, + RV1126_CLKGATE_CON(19), 12, GFLAGS), + +#ifndef CONFIG_ROCKCHIP_LOW_PERFORMANCE + /* + * Clock-Architecture Diagram 3 + */ + /* PD_CORE */ + COMPOSITE_NOMUX(0, "aclk_core", "armclk", CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(1), 4, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, + RV1126_CLKGATE_CON(0), 2, GFLAGS), + GATE(0, "pclk_dbg_daplite", "pclk_dbg", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(0), 5, GFLAGS), + GATE(0, "clk_a7_jtag", "clk_jtag_ori", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(0), 9, GFLAGS), + GATE(0, "aclk_core_niu", "aclk_core", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(0), 3, GFLAGS), + GATE(0, "pclk_dbg_niu", "pclk_dbg", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(0), 4, GFLAGS), + /* + * Clock-Architecture Diagram 4 + */ + /* PD_BUS */ + GATE(0, "aclk_pdbus_hold_niu1", "aclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 10, GFLAGS), + GATE(0, "aclk_pdbus_niu1", "aclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 3, GFLAGS), + GATE(0, "hclk_pdbus_niu1", "hclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 4, GFLAGS), + GATE(0, "pclk_pdbus_niu1", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 5, GFLAGS), + GATE(0, "aclk_pdbus_niu2", "aclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 6, GFLAGS), + GATE(0, "hclk_pdbus_niu2", "hclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 7, GFLAGS), + GATE(0, "aclk_pdbus_niu3", "aclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 8, GFLAGS), + GATE(0, "hclk_pdbus_niu3", "hclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 9, GFLAGS), + GATE(0, "pclk_grf", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(6), 15, GFLAGS), + GATE(0, "pclk_sgrf", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(8), 4, GFLAGS), + GATE(0, "aclk_sysram", "hclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(3), 9, GFLAGS), + GATE(0, "pclk_intmux", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(7), 14, GFLAGS), + + /* + * Clock-Architecture Diagram 5 + */ + /* PD_CRYPTO */ + GATE(0, "aclk_pdcrypto_niu", "aclk_pdcrypto", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(4), 13, GFLAGS), + GATE(0, "hclk_pdcrypto_niu", "hclk_pdcrypto", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(4), 14, GFLAGS), + + /* + * Clock-Architecture Diagram 6 + */ + /* PD_AUDIO */ + GATE(0, "hclk_pdaudio_niu", "hclk_pdaudio", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(9), 2, GFLAGS), + GATE(0, "pclk_pdaudio_niu", "hclk_pdaudio", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(9), 3, GFLAGS), + + /* + * Clock-Architecture Diagram 7 + */ + /* PD_VEPU */ + GATE(0, "aclk_pdvepu_niu", "aclk_pdvepu", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(12), 3, GFLAGS), + GATE(0, "hclk_pdvepu_niu", "hclk_pdvepu", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(12), 4, GFLAGS), + + /* + * Clock-Architecture Diagram 9 + */ + /* PD_VO */ + GATE(0, "aclk_pdvo_niu", "aclk_pdvo", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(14), 3, GFLAGS), + GATE(0, "hclk_pdvo_niu", "hclk_pdvo", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(14), 4, GFLAGS), + GATE(0, "pclk_pdvo_niu", "pclk_pdvo", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(14), 5, GFLAGS), + + /* + * Clock-Architecture Diagram 10 + */ + /* PD_VI */ + GATE(0, "aclk_pdvi_niu", "aclk_pdvi", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(15), 3, GFLAGS), + GATE(0, "hclk_pdvi_niu", "hclk_pdvi", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(15), 4, GFLAGS), + GATE(0, "pclk_pdvi_niu", "pclk_pdvi", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(15), 5, GFLAGS), + /* + * Clock-Architecture Diagram 11 + */ + /* PD_ISPP */ + GATE(0, "aclk_pdispp_niu", "aclk_pdispp", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(16), 2, GFLAGS), + GATE(0, "hclk_pdispp_niu", "hclk_pdispp", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(16), 3, GFLAGS), + + /* + * Clock-Architecture Diagram 12 + */ + /* PD_PHP */ + GATE(0, "aclk_pdphpmid", "aclk_pdphp", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 2, GFLAGS), + GATE(0, "hclk_pdphpmid", "hclk_pdphp", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 3, GFLAGS), + GATE(0, "aclk_pdphpmid_niu", "aclk_pdphpmid", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 4, GFLAGS), + GATE(0, "hclk_pdphpmid_niu", "hclk_pdphpmid", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 5, GFLAGS), + + /* PD_SDCARD */ + GATE(0, "hclk_pdsdmmc_niu", "hclk_pdsdmmc", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 7, GFLAGS), + + /* PD_SDIO */ + GATE(0, "hclk_pdsdio_niu", "hclk_pdsdio", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 9, GFLAGS), + + /* PD_NVM */ + GATE(0, "hclk_pdnvm_niu", "hclk_pdnvm", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(18), 3, GFLAGS), + + /* PD_USB */ + GATE(0, "aclk_pdusb_niu", "aclk_pdusb", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(19), 2, GFLAGS), + GATE(0, "hclk_pdusb_niu", "hclk_pdusb", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(19), 3, GFLAGS), + + /* PD_GMAC */ + GATE(0, "aclk_pdgmac_niu", "aclk_pdgmac", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(20), 2, GFLAGS), + GATE(0, "pclk_pdgmac_niu", "pclk_pdgmac", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(20), 3, GFLAGS), + + /* + * Clock-Architecture Diagram 13 + */ + /* PD_DDR */ + COMPOSITE_NOMUX(0, "pclk_pdddr_pre", "gpll", CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(64), 0, 5, DFLAGS, + RV1126_CLKGATE_CON(21), 0, GFLAGS), + GATE(PCLK_PDDDR, "pclk_pdddr", "pclk_pdddr_pre", CLK_IS_CRITICAL, + RV1126_CLKGATE_CON(21), 15, GFLAGS), + GATE(0, "pclk_ddr_msch", "pclk_pdddr", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 6, GFLAGS), + COMPOSITE_NOGATE(SCLK_DDRCLK, "sclk_ddrc", mux_dpll_gpll_p, CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(64), 15, 1, MFLAGS, 8, 5, DFLAGS), + COMPOSITE(CLK_DDRPHY, "clk_ddrphy", mux_dpll_gpll_p, CLK_IS_CRITICAL, + RV1126_CLKSEL_CON(64), 15, 1, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(21), 8, GFLAGS), + GATE(0, "clk1x_phy", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 1, GFLAGS), + GATE(0, "clk_ddr_msch", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 10, GFLAGS), + GATE(0, "pclk_ddr_dfictl", "pclk_pdddr", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 2, GFLAGS), + GATE(0, "clk_ddr_dfictl", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 13, GFLAGS), + GATE(0, "pclk_ddr_standby", "pclk_pdddr", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 4, GFLAGS), + GATE(0, "clk_ddr_standby", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 14, GFLAGS), + GATE(0, "aclk_ddr_split", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 9, GFLAGS), + GATE(0, "pclk_ddr_grf", "pclk_pdddr", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 5, GFLAGS), + GATE(PCLK_DDR_MON, "pclk_ddr_mon", "pclk_pdddr", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 3, GFLAGS), + GATE(CLK_DDR_MON, "clk_ddr_mon", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(20), 15, GFLAGS), + GATE(TMCLK_DDR_MON, "tmclk_ddr_mon", "xin24m", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 7, GFLAGS), + + /* + * Clock-Architecture Diagram 14 + */ + /* PD_NPU */ + GATE(0, "aclk_pdnpu_niu", "aclk_pdnpu", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(22), 4, GFLAGS), + GATE(0, "hclk_pdnpu_niu", "hclk_pdnpu", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(22), 5, GFLAGS), + GATE(0, "pclk_pdnpu_niu", "pclk_pdnpu", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(22), 6, GFLAGS), + + /* + * Clock-Architecture Diagram 15 + */ + GATE(0, "pclk_topniu", "pclk_pdtop", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 9, GFLAGS), + GATE(PCLK_TOPCRU, "pclk_topcru", "pclk_pdtop", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 10, GFLAGS), + GATE(PCLK_TOPGRF, "pclk_topgrf", "pclk_pdtop", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 11, GFLAGS), + GATE(PCLK_CPUEMADET, "pclk_cpuemadet", "pclk_pdtop", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 12, GFLAGS), + GATE(PCLK_DDRPHY, "pclk_ddrphy", "pclk_pdtop", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 0, GFLAGS), +#endif +}; + +static void __iomem *rv1126_cru_base; +static void __iomem *rv1126_pmucru_base; + +void rv1126_dump_cru(void) +{ + if (rv1126_pmucru_base) { + pr_warn("PMU CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rv1126_pmucru_base, + 0x248, false); + } + if (rv1126_cru_base) { + pr_warn("CRU:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 32, 4, rv1126_cru_base, + 0x588, false); + } +} +EXPORT_SYMBOL_GPL(rv1126_dump_cru); + +static int rv1126_clk_panic(struct notifier_block *this, + unsigned long ev, void *ptr) +{ + rv1126_dump_cru(); + return NOTIFY_DONE; +} + +static struct notifier_block rv1126_clk_panic_block = { + .notifier_call = rv1126_clk_panic, +}; + +static struct rockchip_clk_provider *pmucru_ctx; +static void __init rv1126_pmu_clk_init(struct device_node *np) +{ + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + pr_err("%s: could not map cru pmu region\n", __func__); + return; + } + + rv1126_pmucru_base = reg_base; + + ctx = rockchip_clk_init(np, reg_base, CLKPMU_NR_CLKS); + if (IS_ERR(ctx)) { + pr_err("%s: rockchip pmu clk init failed\n", __func__); + return; + } + + rockchip_clk_register_plls(ctx, rv1126_pmu_pll_clks, + ARRAY_SIZE(rv1126_pmu_pll_clks), + RV1126_GRF_SOC_STATUS0); + + rockchip_clk_register_branches(ctx, rv1126_clk_pmu_branches, + ARRAY_SIZE(rv1126_clk_pmu_branches)); + + rockchip_register_softrst(np, 2, reg_base + RV1126_PMU_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); + + rockchip_clk_of_add_provider(np, ctx); + + pmucru_ctx = ctx; +} + +CLK_OF_DECLARE(rv1126_cru_pmu, "rockchip,rv1126-pmucru", rv1126_pmu_clk_init); + +static void __init rv1126_clk_init(struct device_node *np) +{ + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; + struct clk **cru_clks, **pmucru_clks; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + pr_err("%s: could not map cru region\n", __func__); + return; + } + + rv1126_cru_base = reg_base; + + ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); + if (IS_ERR(ctx)) { + pr_err("%s: rockchip clk init failed\n", __func__); + iounmap(reg_base); + return; + } + cru_clks = ctx->clk_data.clks; + pmucru_clks = pmucru_ctx->clk_data.clks; + + rockchip_clk_register_plls(ctx, rv1126_pll_clks, + ARRAY_SIZE(rv1126_pll_clks), + RV1126_GRF_SOC_STATUS0); + + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", + 3, cru_clks[PLL_APLL], pmucru_clks[PLL_GPLL], + &rv1126_cpuclk_data, rv1126_cpuclk_rates, + ARRAY_SIZE(rv1126_cpuclk_rates)); + + rockchip_clk_register_branches(ctx, rv1126_clk_branches, + ARRAY_SIZE(rv1126_clk_branches)); + + rockchip_register_softrst(np, 15, reg_base + RV1126_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); + + rockchip_register_restart_notifier(ctx, RV1126_GLB_SRST_FST, NULL); + + rockchip_clk_of_add_provider(np, ctx); + + atomic_notifier_chain_register(&panic_notifier_list, + &rv1126_clk_panic_block); +} + +CLK_OF_DECLARE(rv1126_cru, "rockchip,rv1126-cru", rv1126_clk_init); + +struct clk_rv1126_inits { + void (*inits)(struct device_node *np); +}; + +static const struct clk_rv1126_inits clk_rv1126_pmu_init = { + .inits = rv1126_pmu_clk_init, +}; + +static const struct clk_rv1126_inits clk_rv1126_init = { + .inits = rv1126_clk_init, +}; + +static const struct of_device_id clk_rv1126_match_table[] = { + { + .compatible = "rockchip,rv1126-cru", + .data = &clk_rv1126_init, + }, { + .compatible = "rockchip,rv1126-pmucru", + .data = &clk_rv1126_pmu_init, + }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_rv1126_match_table); + +static int __init clk_rv1126_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + const struct clk_rv1126_inits *init_data; + + match = of_match_device(clk_rv1126_match_table, &pdev->dev); + if (!match || !match->data) + return -EINVAL; + + init_data = match->data; + if (init_data->inits) + init_data->inits(np); + + return 0; +} + +static struct platform_driver clk_rv1126_driver = { + .driver = { + .name = "clk-rv1126", + .of_match_table = clk_rv1126_match_table, + }, +}; +builtin_platform_driver_probe(clk_rv1126_driver, clk_rv1126_probe); + +MODULE_DESCRIPTION("Rockchip RV1126 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c index b443169dd408..6c8e47067032 100644 --- a/drivers/clk/rockchip/clk.c +++ b/drivers/clk/rockchip/clk.c @@ -38,6 +38,7 @@ static struct clk *rockchip_clk_register_branch(const char *name, const char *const *parent_names, u8 num_parents, void __iomem *base, int muxdiv_offset, u8 mux_shift, u8 mux_width, u8 mux_flags, + u32 *mux_table, int div_offset, u8 div_shift, u8 div_width, u8 div_flags, struct clk_div_table *div_table, int gate_offset, u8 gate_shift, u8 gate_flags, unsigned long flags, @@ -60,6 +61,7 @@ static struct clk *rockchip_clk_register_branch(const char *name, mux->shift = mux_shift; mux->mask = BIT(mux_width) - 1; mux->flags = mux_flags; + mux->table = mux_table; mux->lock = lock; mux_ops = (mux_flags & CLK_MUX_READ_ONLY) ? &clk_mux_ro_ops : &clk_mux_ops; @@ -182,12 +184,43 @@ static void rockchip_fractional_approximation(struct clk_hw *hw, unsigned long p_rate, p_parent_rate; struct clk_hw *p_parent; unsigned long scale; + u32 div; p_rate = clk_hw_get_rate(clk_hw_get_parent(hw)); - if ((rate * 20 > p_rate) && (p_rate % rate != 0)) { + if (((rate * 20 > p_rate) && (p_rate % rate != 0)) || + (fd->max_prate && fd->max_prate < p_rate)) { p_parent = clk_hw_get_parent(clk_hw_get_parent(hw)); - p_parent_rate = clk_hw_get_rate(p_parent); - *parent_rate = p_parent_rate; + if (!p_parent) { + *parent_rate = p_rate; + } else { + p_parent_rate = clk_hw_get_rate(p_parent); + *parent_rate = p_parent_rate; + if (fd->max_prate && p_parent_rate > fd->max_prate) { + div = DIV_ROUND_UP(p_parent_rate, + fd->max_prate); + *parent_rate = p_parent_rate / div; + } + } + + if (*parent_rate < rate * 20) { + /* + * Fractional frequency divider to do + * integer frequency divider does not + * need 20 times the limit. + */ + if (!(*parent_rate % rate)) { + *m = 1; + *n = *parent_rate / rate; + return; + } else if (!(fd->flags & CLK_FRAC_DIVIDER_NO_LIMIT)) { + pr_warn("%s p_rate(%ld) is low than rate(%ld)*20, use integer or half-div\n", + clk_hw_get_name(hw), + *parent_rate, rate); + *m = 0; + *n = 1; + return; + } + } } /* @@ -210,7 +243,7 @@ static struct clk *rockchip_clk_register_frac_branch( void __iomem *base, int muxdiv_offset, u8 div_flags, int gate_offset, u8 gate_shift, u8 gate_flags, unsigned long flags, struct rockchip_clk_branch *child, - spinlock_t *lock) + unsigned long max_prate, spinlock_t *lock) { struct clk_hw *hw; struct rockchip_clk_frac *frac; @@ -251,6 +284,7 @@ static struct clk *rockchip_clk_register_frac_branch( div->nmask = GENMASK(div->nwidth - 1, 0) << div->nshift; div->lock = lock; div->approximation = rockchip_fractional_approximation; + div->max_prate = max_prate; div_ops = &clk_fractional_divider_ops; hw = clk_hw_register_composite(NULL, name, parent_names, num_parents, @@ -278,6 +312,8 @@ static struct clk *rockchip_clk_register_frac_branch( frac_mux->shift = child->mux_shift; frac_mux->mask = BIT(child->mux_width) - 1; frac_mux->flags = child->mux_flags; + if (child->mux_table) + frac_mux->table = child->mux_table; frac_mux->lock = lock; frac_mux->hw.init = &init; @@ -360,6 +396,61 @@ static struct clk *rockchip_clk_register_factor_branch(const char *name, return hw->clk; } +static struct clk *rockchip_clk_register_composite_brother_branch( + struct rockchip_clk_provider *ctx, const char *name, + const char *const *parent_names, u8 num_parents, + void __iomem *base, int muxdiv_offset, u8 mux_shift, + u8 mux_width, u8 mux_flags, u32 *mux_table, + int div_offset, u8 div_shift, u8 div_width, u8 div_flags, + struct clk_div_table *div_table, int gate_offset, + u8 gate_shift, u8 gate_flags, unsigned long flags, + struct rockchip_clk_branch *brother, spinlock_t *lock) +{ + struct clk *clk, *brother_clk; + struct clk_composite *composite, *brother_composite; + struct clk_hw *hw, *brother_hw; + + if (brother && brother->branch_type != branch_half_divider) { + pr_err("%s: composite brother for %s can only be a halfdiv\n", + __func__, name); + return ERR_PTR(-EINVAL); + } + + clk = rockchip_clk_register_branch(name, parent_names, num_parents, + base, muxdiv_offset, mux_shift, + mux_width, mux_flags, mux_table, + div_offset, div_shift, div_width, + div_flags, div_table, + gate_offset, gate_shift, gate_flags, + flags, lock); + if (IS_ERR(clk)) + return clk; + + brother_clk = rockchip_clk_register_halfdiv(brother->name, + brother->parent_names, brother->num_parents, + base, brother->muxdiv_offset, + brother->mux_shift, brother->mux_width, + brother->mux_flags, brother->div_offset, + brother->div_shift, brother->div_width, + brother->div_flags, brother->gate_offset, + brother->gate_shift, brother->gate_flags, + flags, lock); + if (IS_ERR(brother_clk)) + return brother_clk; + rockchip_clk_add_lookup(ctx, brother_clk, brother->id); + + hw = __clk_get_hw(clk); + brother_hw = __clk_get_hw(brother_clk); + if (hw && brother_hw) { + composite = to_clk_composite(hw); + brother_composite = to_clk_composite(brother_hw); + composite->brother_hw = brother_hw; + brother_composite->brother_hw = hw; + } + + return clk; +} + struct rockchip_clk_provider *rockchip_clk_init(struct device_node *np, void __iomem *base, unsigned long nr_clks) @@ -387,6 +478,8 @@ struct rockchip_clk_provider *rockchip_clk_init(struct device_node *np, ctx->grf = syscon_regmap_lookup_by_phandle(ctx->cru_node, "rockchip,grf"); + ctx->pmugrf = syscon_regmap_lookup_by_phandle(ctx->cru_node, + "rockchip,pmugrf"); return ctx; @@ -452,11 +545,22 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, /* catch simple muxes */ switch (list->branch_type) { case branch_mux: - clk = clk_register_mux(NULL, list->name, - list->parent_names, list->num_parents, - flags, ctx->reg_base + list->muxdiv_offset, - list->mux_shift, list->mux_width, - list->mux_flags, &ctx->lock); + if (list->mux_table) + clk = clk_register_mux_table(NULL, list->name, + list->parent_names, list->num_parents, + flags, + ctx->reg_base + list->muxdiv_offset, + list->mux_shift, + BIT(list->mux_width) - 1, + list->mux_flags, list->mux_table, + &ctx->lock); + else + clk = clk_register_mux(NULL, list->name, + list->parent_names, list->num_parents, + flags, + ctx->reg_base + list->muxdiv_offset, + list->mux_shift, list->mux_width, + list->mux_flags, &ctx->lock); break; case branch_muxgrf: clk = rockchip_clk_register_muxgrf(list->name, @@ -465,6 +569,13 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, list->mux_shift, list->mux_width, list->mux_flags); break; + case branch_muxpmugrf: + clk = rockchip_clk_register_muxgrf(list->name, + list->parent_names, list->num_parents, + flags, ctx->pmugrf, list->muxdiv_offset, + list->mux_shift, list->mux_width, + list->mux_flags); + break; case branch_divider: if (list->div_table) clk = clk_register_divider_table(NULL, @@ -488,17 +599,18 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, list->div_flags, list->gate_offset, list->gate_shift, list->gate_flags, flags, list->child, - &ctx->lock); + list->max_prate, &ctx->lock); break; case branch_half_divider: clk = rockchip_clk_register_halfdiv(list->name, list->parent_names, list->num_parents, ctx->reg_base, list->muxdiv_offset, list->mux_shift, list->mux_width, - list->mux_flags, list->div_shift, - list->div_width, list->div_flags, - list->gate_offset, list->gate_shift, - list->gate_flags, flags, &ctx->lock); + list->mux_flags, list->div_offset, + list->div_shift, list->div_width, + list->div_flags, list->gate_offset, + list->gate_shift, list->gate_flags, + flags, &ctx->lock); break; case branch_gate: flags |= CLK_SET_RATE_PARENT; @@ -514,11 +626,25 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, ctx->reg_base, list->muxdiv_offset, list->mux_shift, list->mux_width, list->mux_flags, - list->div_offset, list->div_shift, list->div_width, + list->mux_table, list->div_offset, + list->div_shift, list->div_width, list->div_flags, list->div_table, list->gate_offset, list->gate_shift, list->gate_flags, flags, &ctx->lock); break; + case branch_composite_brother: + clk = rockchip_clk_register_composite_brother_branch( + ctx, list->name, list->parent_names, + list->num_parents, ctx->reg_base, + list->muxdiv_offset, list->mux_shift, + list->mux_width, list->mux_flags, + list->mux_table, list->div_offset, + list->div_shift, list->div_width, + list->div_flags, list->div_table, + list->gate_offset, list->gate_shift, + list->gate_flags, flags, list->child, + &ctx->lock); + break; case branch_mmc: clk = rockchip_clk_register_mmc( list->name, @@ -549,7 +675,17 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, list->muxdiv_offset, list->mux_shift, list->mux_width, list->div_shift, list->div_width, list->div_flags, - ctx->reg_base, &ctx->lock); + ctx->reg_base); + break; + case branch_dclk_divider: + clk = rockchip_clk_register_dclk_branch(list->name, + list->parent_names, list->num_parents, + ctx->reg_base, list->muxdiv_offset, list->mux_shift, + list->mux_width, list->mux_flags, + list->div_offset, list->div_shift, list->div_width, + list->div_flags, list->div_table, + list->gate_offset, list->gate_shift, + list->gate_flags, flags, list->max_prate, &ctx->lock); break; } @@ -573,15 +709,17 @@ EXPORT_SYMBOL_GPL(rockchip_clk_register_branches); void rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, unsigned int lookup_id, - const char *name, const char *const *parent_names, + const char *name, u8 num_parents, + struct clk *parent, struct clk *alt_parent, const struct rockchip_cpuclk_reg_data *reg_data, const struct rockchip_cpuclk_rate_table *rates, int nrates) { struct clk *clk; - clk = rockchip_clk_register_cpuclk(name, parent_names, num_parents, + clk = rockchip_clk_register_cpuclk(name, num_parents, + parent, alt_parent, reg_data, rates, nrates, ctx->reg_base, &ctx->lock); if (IS_ERR(clk)) { @@ -594,20 +732,20 @@ void rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, } EXPORT_SYMBOL_GPL(rockchip_clk_register_armclk); -void rockchip_clk_protect_critical(const char *const clocks[], - int nclocks) -{ - int i; - - /* Protect the clocks that needs to stay on */ - for (i = 0; i < nclocks; i++) { - struct clk *clk = __clk_lookup(clocks[i]); +void (*rk_dump_cru)(void); +EXPORT_SYMBOL(rk_dump_cru); - if (clk) - clk_prepare_enable(clk); - } +static int rk_clk_panic(struct notifier_block *this, + unsigned long ev, void *ptr) +{ + if (rk_dump_cru) + rk_dump_cru(); + return NOTIFY_DONE; } -EXPORT_SYMBOL_GPL(rockchip_clk_protect_critical); + +static struct notifier_block rk_clk_panic_block = { + .notifier_call = rk_clk_panic, +}; static void __iomem *rst_base; static unsigned int reg_restart; @@ -641,5 +779,7 @@ rockchip_register_restart_notifier(struct rockchip_clk_provider *ctx, if (ret) pr_err("%s: cannot register restart handler, %d\n", __func__, ret); + atomic_notifier_chain_register(&panic_notifier_list, + &rk_clk_panic_block); } EXPORT_SYMBOL_GPL(rockchip_register_restart_notifier); diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h index 2271a84124b0..509087750eeb 100644 --- a/drivers/clk/rockchip/clk.h +++ b/drivers/clk/rockchip/clk.h @@ -37,12 +37,25 @@ struct clk; #define BOOST_SWITCH_THRESHOLD 0x0024 #define BOOST_FSM_STATUS 0x0028 #define BOOST_PLL_L_CON(x) ((x) * 0x4 + 0x2c) +#define BOOST_PLL_CON_MASK 0xffff +#define BOOST_CORE_DIV_MASK 0x1f +#define BOOST_CORE_DIV_SHIFT 0 +#define BOOST_BACKUP_PLL_MASK 0x3 +#define BOOST_BACKUP_PLL_SHIFT 8 +#define BOOST_BACKUP_PLL_USAGE_MASK 0x1 +#define BOOST_BACKUP_PLL_USAGE_SHIFT 12 +#define BOOST_BACKUP_PLL_USAGE_BORROW 0 +#define BOOST_BACKUP_PLL_USAGE_TARGET 1 +#define BOOST_ENABLE_MASK 0x1 +#define BOOST_ENABLE_SHIFT 0 #define BOOST_RECOVERY_MASK 0x1 #define BOOST_RECOVERY_SHIFT 1 #define BOOST_SW_CTRL_MASK 0x1 #define BOOST_SW_CTRL_SHIFT 2 #define BOOST_LOW_FREQ_EN_MASK 0x1 #define BOOST_LOW_FREQ_EN_SHIFT 3 +#define BOOST_STATIS_ENABLE_MASK 0x1 +#define BOOST_STATIS_ENABLE_SHIFT 4 #define BOOST_BUSY_STATE BIT(8) #define PX30_PLL_CON(x) ((x) * 0x4) @@ -79,6 +92,51 @@ struct clk; #define RV1108_EMMC_CON0 0x1e8 #define RV1108_EMMC_CON1 0x1ec +#define RV1126_PMU_MODE 0x0 +#define RV1126_PMU_PLL_CON(x) ((x) * 0x4 + 0x10) +#define RV1126_PMU_CLKSEL_CON(x) ((x) * 0x4 + 0x100) +#define RV1126_PMU_CLKGATE_CON(x) ((x) * 0x4 + 0x180) +#define RV1126_PMU_SOFTRST_CON(x) ((x) * 0x4 + 0x200) +#define RV1126_PLL_CON(x) ((x) * 0x4) +#define RV1126_MODE_CON 0x90 +#define RV1126_CLKSEL_CON(x) ((x) * 0x4 + 0x100) +#define RV1126_CLKGATE_CON(x) ((x) * 0x4 + 0x280) +#define RV1126_SOFTRST_CON(x) ((x) * 0x4 + 0x300) +#define RV1126_GLB_SRST_FST 0x408 +#define RV1126_GLB_SRST_SND 0x40c +#define RV1126_SDMMC_CON0 0x440 +#define RV1126_SDMMC_CON1 0x444 +#define RV1126_SDIO_CON0 0x448 +#define RV1126_SDIO_CON1 0x44c +#define RV1126_EMMC_CON0 0x450 +#define RV1126_EMMC_CON1 0x454 + +/* + * register positions shared by RK1808 RK2928, RK3036, + * RK3066, RK3188 and RK3228 + */ + +#define RK1808_PLL_CON(x) ((x) * 0x4) +#define RK1808_MODE_CON 0xa0 +#define RK1808_MISC_CON 0xa4 +#define RK1808_MISC1_CON 0xa8 +#define RK1808_GLB_SRST_FST 0xb8 +#define RK1808_GLB_SRST_SND 0xbc +#define RK1808_CLKSEL_CON(x) ((x) * 0x4 + 0x100) +#define RK1808_CLKGATE_CON(x) ((x) * 0x4 + 0x230) +#define RK1808_SOFTRST_CON(x) ((x) * 0x4 + 0x300) +#define RK1808_SDMMC_CON0 0x380 +#define RK1808_SDMMC_CON1 0x384 +#define RK1808_SDIO_CON0 0x388 +#define RK1808_SDIO_CON1 0x38c +#define RK1808_EMMC_CON0 0x390 +#define RK1808_EMMC_CON1 0x394 + +#define RK1808_PMU_PLL_CON(x) ((x) * 0x4 + 0x4000) +#define RK1808_PMU_MODE_CON 0x4020 +#define RK1808_PMU_CLKSEL_CON(x) ((x) * 0x4 + 0x4040) +#define RK1808_PMU_CLKGATE_CON(x) ((x) * 0x4 + 0x4080) + #define RK2928_PLL_CON(x) ((x) * 0x4) #define RK2928_MODE_CON 0x40 #define RK2928_CLKSEL_CON(x) ((x) * 0x4 + 0x44) @@ -188,6 +246,34 @@ struct clk; #define RK3399_PMU_CLKGATE_CON(x) ((x) * 0x4 + 0x100) #define RK3399_PMU_SOFTRST_CON(x) ((x) * 0x4 + 0x110) +#define RK3568_PLL_CON(x) RK2928_PLL_CON(x) +#define RK3568_MODE_CON0 0xc0 +#define RK3568_MISC_CON0 0xc4 +#define RK3568_MISC_CON1 0xc8 +#define RK3568_MISC_CON2 0xcc +#define RK3568_GLB_CNT_TH 0xd0 +#define RK3568_GLB_SRST_FST 0xd4 +#define RK3568_GLB_SRST_SND 0xd8 +#define RK3568_GLB_RST_CON 0xdc +#define RK3568_GLB_RST_ST 0xe0 +#define RK3568_CLKSEL_CON(x) ((x) * 0x4 + 0x100) +#define RK3568_CLKGATE_CON(x) ((x) * 0x4 + 0x300) +#define RK3568_SOFTRST_CON(x) ((x) * 0x4 + 0x400) +#define RK3568_SDMMC0_CON0 0x580 +#define RK3568_SDMMC0_CON1 0x584 +#define RK3568_SDMMC1_CON0 0x588 +#define RK3568_SDMMC1_CON1 0x58c +#define RK3568_SDMMC2_CON0 0x590 +#define RK3568_SDMMC2_CON1 0x594 +#define RK3568_EMMC_CON0 0x598 +#define RK3568_EMMC_CON1 0x59c + +#define RK3568_PMU_PLL_CON(x) RK2928_PLL_CON(x) +#define RK3568_PMU_MODE_CON0 0x80 +#define RK3568_PMU_CLKSEL_CON(x) ((x) * 0x4 + 0x100) +#define RK3568_PMU_CLKGATE_CON(x) ((x) * 0x4 + 0x180) +#define RK3568_PMU_SOFTRST_CON(x) ((x) * 0x4 + 0x200) + enum rockchip_pll_type { pll_rk3036, pll_rk3066, @@ -238,22 +324,30 @@ struct rockchip_clk_provider { struct clk_onecell_data clk_data; struct device_node *cru_node; struct regmap *grf; + struct regmap *pmugrf; spinlock_t lock; }; struct rockchip_pll_rate_table { unsigned long rate; - unsigned int nr; - unsigned int nf; - unsigned int no; - unsigned int nb; - /* for RK3036/RK3399 */ - unsigned int fbdiv; - unsigned int postdiv1; - unsigned int refdiv; - unsigned int postdiv2; - unsigned int dsmpd; - unsigned int frac; + union { + struct { + /* for RK3066 */ + unsigned int nr; + unsigned int nf; + unsigned int no; + unsigned int nb; + }; + struct { + /* for RK3036/RK3399 */ + unsigned int fbdiv; + unsigned int postdiv1; + unsigned int refdiv; + unsigned int postdiv2; + unsigned int dsmpd; + unsigned int frac; + }; + }; }; /** @@ -317,12 +411,21 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx, struct rockchip_pll_rate_table *rate_table, unsigned long flags, u8 clk_pll_flags); +void rockchip_boost_init(struct clk_hw *hw); + +void rockchip_boost_enable_recovery_sw_low(struct clk_hw *hw); + +void rockchip_boost_disable_recovery_sw(struct clk_hw *hw); + +void rockchip_boost_add_core_div(struct clk_hw *hw, unsigned long prate); + struct rockchip_cpuclk_clksel { int reg; u32 val; }; -#define ROCKCHIP_CPUCLK_NUM_DIVIDERS 2 +#define ROCKCHIP_CPUCLK_NUM_DIVIDERS 5 +#define ROCKCHIP_CPUCLK_MAX_CORES 4 struct rockchip_cpuclk_rate_table { unsigned long prate; struct rockchip_cpuclk_clksel divs[ROCKCHIP_CPUCLK_NUM_DIVIDERS]; @@ -330,26 +433,29 @@ struct rockchip_cpuclk_rate_table { /** * struct rockchip_cpuclk_reg_data - register offsets and masks of the cpuclock - * @core_reg: register offset of the core settings register - * @div_core_shift: core divider offset used to divide the pll value - * @div_core_mask: core divider mask - * @mux_core_alt: mux value to select alternate parent + * @core_reg[]: register offset of the cores setting register + * @div_core_shift[]: cores divider offset used to divide the pll value + * @div_core_mask[]: cores divider mask + * @num_cores: number of cpu cores * @mux_core_main: mux value to select main parent of core * @mux_core_shift: offset of the core multiplexer * @mux_core_mask: core multiplexer mask */ struct rockchip_cpuclk_reg_data { - int core_reg; - u8 div_core_shift; - u32 div_core_mask; - u8 mux_core_alt; - u8 mux_core_main; - u8 mux_core_shift; - u32 mux_core_mask; + int core_reg[ROCKCHIP_CPUCLK_MAX_CORES]; + u8 div_core_shift[ROCKCHIP_CPUCLK_MAX_CORES]; + u32 div_core_mask[ROCKCHIP_CPUCLK_MAX_CORES]; + int num_cores; + u8 mux_core_alt; + u8 mux_core_main; + u8 mux_core_shift; + u32 mux_core_mask; + const char *pll_name; }; struct clk *rockchip_clk_register_cpuclk(const char *name, - const char *const *parent_names, u8 num_parents, + u8 num_parents, + struct clk *parent, struct clk *alt_parent, const struct rockchip_cpuclk_reg_data *reg_data, const struct rockchip_cpuclk_rate_table *rates, int nrates, void __iomem *reg_base, spinlock_t *lock); @@ -361,16 +467,21 @@ struct clk *rockchip_clk_register_mmc(const char *name, /* * DDRCLK flags, including method of setting the rate * ROCKCHIP_DDRCLK_SIP: use SIP call to bl31 to change ddrclk rate. + * ROCKCHIP_DDRCLK_SCPI: use SCPI APIs to let mcu change ddrclk rate. */ #define ROCKCHIP_DDRCLK_SIP BIT(0) +#define ROCKCHIP_DDRCLK_SCPI 0x02 +#define ROCKCHIP_DDRCLK_SIP_V2 0x03 + +void rockchip_set_ddrclk_params(void __iomem *params); +void rockchip_set_ddrclk_dmcfreq_wait_complete(int (*func)(void)); struct clk *rockchip_clk_register_ddrclk(const char *name, int flags, const char *const *parent_names, u8 num_parents, int mux_offset, int mux_shift, int mux_width, int div_shift, int div_width, - int ddr_flags, void __iomem *reg_base, - spinlock_t *lock); + int ddr_flags, void __iomem *reg_base); #define ROCKCHIP_INVERTER_HIWORD_MASK BIT(0) @@ -388,8 +499,10 @@ struct clk *rockchip_clk_register_muxgrf(const char *name, enum rockchip_clk_branch_type { branch_composite, + branch_composite_brother, branch_mux, branch_muxgrf, + branch_muxpmugrf, branch_divider, branch_fraction_divider, branch_gate, @@ -398,6 +511,7 @@ enum rockchip_clk_branch_type { branch_factor, branch_ddrclk, branch_half_divider, + branch_dclk_divider, }; struct rockchip_clk_branch { @@ -411,6 +525,7 @@ struct rockchip_clk_branch { u8 mux_shift; u8 mux_width; u8 mux_flags; + u32 *mux_table; int div_offset; u8 div_shift; u8 div_width; @@ -420,6 +535,7 @@ struct rockchip_clk_branch { u8 gate_shift; u8 gate_flags; struct rockchip_clk_branch *child; + unsigned long max_prate; }; #define COMPOSITE(_id, cname, pnames, f, mo, ms, mw, mf, ds, dw,\ @@ -443,6 +559,50 @@ struct rockchip_clk_branch { .gate_flags = gf, \ } +#define COMPOSITE_BROTHER(_id, cname, pnames, f, mo, ms, mw, mf,\ + ds, dw, df, go, gs, gf, bro) \ + { \ + .id = _id, \ + .branch_type = branch_composite_brother, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = mo, \ + .mux_shift = ms, \ + .mux_width = mw, \ + .mux_flags = mf, \ + .div_shift = ds, \ + .div_width = dw, \ + .div_flags = df, \ + .gate_offset = go, \ + .gate_shift = gs, \ + .gate_flags = gf, \ + .child = bro, \ + } + +#define COMPOSITE_MUXTBL(_id, cname, pnames, f, mo, ms, mw, mf, \ + mt, ds, dw, df, go, gs, gf) \ + { \ + .id = _id, \ + .branch_type = branch_composite, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = mo, \ + .mux_shift = ms, \ + .mux_width = mw, \ + .mux_flags = mf, \ + .mux_table = mt, \ + .div_shift = ds, \ + .div_width = dw, \ + .div_flags = df, \ + .gate_offset = go, \ + .gate_shift = gs, \ + .gate_flags = gf, \ + } + #define COMPOSITE_DIV_OFFSET(_id, cname, pnames, f, mo, ms, mw, \ mf, do, ds, dw, df, go, gs, gf) \ { \ @@ -539,6 +699,26 @@ struct rockchip_clk_branch { .gate_offset = -1, \ } +#define COMPOSITE_BROTHER_NOGATE(_id, cname, pnames, f, mo, ms, \ + mw, mf, ds, dw, df, bro) \ + { \ + .id = _id, \ + .branch_type = branch_composite_brother, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = mo, \ + .mux_shift = ms, \ + .mux_width = mw, \ + .mux_flags = mf, \ + .div_shift = ds, \ + .div_width = dw, \ + .div_flags = df, \ + .gate_offset = -1, \ + .child = bro, \ + } + #define COMPOSITE_NOGATE_DIVTBL(_id, cname, pnames, f, mo, ms, \ mw, mf, ds, dw, df, dt) \ { \ @@ -559,7 +739,7 @@ struct rockchip_clk_branch { .gate_offset = -1, \ } -#define COMPOSITE_FRAC(_id, cname, pname, f, mo, df, go, gs, gf)\ +#define COMPOSITE_FRAC(_id, cname, pname, f, mo, df, go, gs, gf, prate)\ { \ .id = _id, \ .branch_type = branch_fraction_divider, \ @@ -574,9 +754,10 @@ struct rockchip_clk_branch { .gate_offset = go, \ .gate_shift = gs, \ .gate_flags = gf, \ + .max_prate = prate, \ } -#define COMPOSITE_FRACMUX(_id, cname, pname, f, mo, df, go, gs, gf, ch) \ +#define COMPOSITE_FRACMUX(_id, cname, pname, f, mo, df, go, gs, gf, ch, prate) \ { \ .id = _id, \ .branch_type = branch_fraction_divider, \ @@ -592,9 +773,10 @@ struct rockchip_clk_branch { .gate_shift = gs, \ .gate_flags = gf, \ .child = ch, \ + .max_prate = prate, \ } -#define COMPOSITE_FRACMUX_NOGATE(_id, cname, pname, f, mo, df, ch) \ +#define COMPOSITE_FRACMUX_NOGATE(_id, cname, pname, f, mo, df, ch, prate) \ { \ .id = _id, \ .branch_type = branch_fraction_divider, \ @@ -608,6 +790,7 @@ struct rockchip_clk_branch { .div_flags = df, \ .gate_offset = -1, \ .child = ch, \ + .max_prate = prate, \ } #define COMPOSITE_DDRCLK(_id, cname, pnames, f, mo, ms, mw, \ @@ -643,6 +826,22 @@ struct rockchip_clk_branch { .gate_offset = -1, \ } +#define MUXTBL(_id, cname, pnames, f, o, s, w, mf, mt) \ + { \ + .id = _id, \ + .branch_type = branch_mux, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = o, \ + .mux_shift = s, \ + .mux_width = w, \ + .mux_flags = mf, \ + .gate_offset = -1, \ + .mux_table = mt, \ + } + #define MUXGRF(_id, cname, pnames, f, o, s, w, mf) \ { \ .id = _id, \ @@ -658,6 +857,21 @@ struct rockchip_clk_branch { .gate_offset = -1, \ } +#define MUXPMUGRF(_id, cname, pnames, f, o, s, w, mf) \ + { \ + .id = _id, \ + .branch_type = branch_muxpmugrf, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = o, \ + .mux_shift = s, \ + .mux_width = w, \ + .mux_flags = mf, \ + .gate_offset = -1, \ + } + #define DIV(_id, cname, pname, f, o, s, w, df) \ { \ .id = _id, \ @@ -772,6 +986,28 @@ struct rockchip_clk_branch { .gate_flags = gf, \ } +#define COMPOSITE_HALFDIV_OFFSET(_id, cname, pnames, f, mo, ms, mw, mf, do,\ + ds, dw, df, go, gs, gf) \ + { \ + .id = _id, \ + .branch_type = branch_half_divider, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = mo, \ + .mux_shift = ms, \ + .mux_width = mw, \ + .mux_flags = mf, \ + .div_offset = do, \ + .div_shift = ds, \ + .div_width = dw, \ + .div_flags = df, \ + .gate_offset = go, \ + .gate_shift = gs, \ + .gate_flags = gf, \ + } + #define COMPOSITE_NOGATE_HALFDIV(_id, cname, pnames, f, mo, ms, mw, mf, \ ds, dw, df) \ { \ @@ -824,6 +1060,28 @@ struct rockchip_clk_branch { .gate_offset = -1, \ } +#define COMPOSITE_DCLK(_id, cname, pnames, f, mo, ms, mw, mf, ds, dw,\ + df, go, gs, gf, prate) \ + { \ + .id = _id, \ + .branch_type = branch_dclk_divider, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = mo, \ + .mux_shift = ms, \ + .mux_width = mw, \ + .mux_flags = mf, \ + .div_shift = ds, \ + .div_width = dw, \ + .div_flags = df, \ + .gate_offset = go, \ + .gate_shift = gs, \ + .gate_flags = gf, \ + .max_prate = prate, \ + } + /* SGRF clocks are only accessible from secure mode, so not controllable */ #define SGRF_GATE(_id, cname, pname) \ FACTOR(_id, cname, pname, 0, 1, 1) @@ -840,13 +1098,17 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, void rockchip_clk_register_plls(struct rockchip_clk_provider *ctx, struct rockchip_pll_clock *pll_list, unsigned int nr_pll, int grf_lock_offset); -void rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, - unsigned int lookup_id, const char *name, - const char *const *parent_names, u8 num_parents, - const struct rockchip_cpuclk_reg_data *reg_data, - const struct rockchip_cpuclk_rate_table *rates, - int nrates); -void rockchip_clk_protect_critical(const char *const clocks[], int nclocks); +void __init rockchip_clk_register_armclk(struct rockchip_clk_provider *ctx, + unsigned int lookup_id, + const char *name, + u8 num_parents, + struct clk *parent, struct clk *alt_parent, + const struct rockchip_cpuclk_reg_data *reg_data, + const struct rockchip_cpuclk_rate_table *rates, + int nrates); +int rockchip_pll_clk_rate_to_scale(struct clk *clk, unsigned long rate); +int rockchip_pll_clk_scale_to_rate(struct clk *clk, unsigned int scale); +int rockchip_pll_clk_adaptive_scaling(struct clk *clk, int sel); void rockchip_register_restart_notifier(struct rockchip_clk_provider *ctx, unsigned int reg, void (*cb)(void)); @@ -857,12 +1119,27 @@ struct clk *rockchip_clk_register_halfdiv(const char *name, u8 num_parents, void __iomem *base, int muxdiv_offset, u8 mux_shift, u8 mux_width, u8 mux_flags, - u8 div_shift, u8 div_width, - u8 div_flags, int gate_offset, - u8 gate_shift, u8 gate_flags, - unsigned long flags, + int div_offset, u8 div_shift, + u8 div_width, u8 div_flags, + int gate_offset, u8 gate_shift, + u8 gate_flags, unsigned long flags, spinlock_t *lock); +struct clk *rockchip_clk_register_dclk_branch(const char *name, + const char *const *parent_names, + u8 num_parents, + void __iomem *base, + int muxdiv_offset, u8 mux_shift, + u8 mux_width, u8 mux_flags, + int div_offset, u8 div_shift, + u8 div_width, u8 div_flags, + struct clk_div_table *div_table, + int gate_offset, + u8 gate_shift, u8 gate_flags, + unsigned long flags, + unsigned long max_prate, + spinlock_t *lock); + #ifdef CONFIG_RESET_CONTROLLER void rockchip_register_softrst(struct device_node *np, unsigned int num_regs, @@ -874,5 +1151,6 @@ static inline void rockchip_register_softrst(struct device_node *np, { } #endif +extern void (*rk_dump_cru)(void); #endif diff --git a/drivers/clk/rockchip/regmap/Kconfig b/drivers/clk/rockchip/regmap/Kconfig new file mode 100755 index 000000000000..65f691bc4141 --- /dev/null +++ b/drivers/clk/rockchip/regmap/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 + +config COMMON_CLK_ROCKCHIP_REGMAP + tristate + +config CLK_RK618 + tristate "Clock driver for Rockchip RK618" + depends on MFD_RK618 + default MFD_RK618 + select COMMON_CLK_ROCKCHIP_REGMAP + +config CLK_RK628 + tristate "Clock driver for Rockchip RK628" + depends on MFD_RK628 + default MFD_RK628 + select COMMON_CLK_ROCKCHIP_REGMAP diff --git a/drivers/clk/rockchip/regmap/Makefile b/drivers/clk/rockchip/regmap/Makefile new file mode 100755 index 000000000000..18d075d093d9 --- /dev/null +++ b/drivers/clk/rockchip/regmap/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_COMMON_CLK_ROCKCHIP_REGMAP) += clk-rockchip-regmap.o + +clk-rockchip-regmap-objs := clk-regmap-mux.o \ + clk-regmap-divider.o \ + clk-regmap-gate.o \ + clk-regmap-fractional-divider.o \ + clk-regmap-composite.o \ + clk-regmap-pll.o + +obj-$(CONFIG_CLK_RK618) += clk-rk618.o +obj-$(CONFIG_CLK_RK628) += clk-rk628.o diff --git a/drivers/clk/rockchip/regmap/clk-regmap-composite.c b/drivers/clk/rockchip/regmap/clk-regmap-composite.c new file mode 100755 index 000000000000..43d2b9a45aca --- /dev/null +++ b/drivers/clk/rockchip/regmap/clk-regmap-composite.c @@ -0,0 +1,400 @@ +/* + * Copyright (c) 2017 Rockchip Electronics Co. Ltd. + * + * Base on code in drivers/clk/clk-composite.c. + * See clk-composite.c for further copyright information. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "clk-regmap.h" + +struct clk_regmap_composite { + struct device *dev; + struct clk_hw hw; + struct clk_ops ops; + + struct clk_hw *mux_hw; + struct clk_hw *rate_hw; + struct clk_hw *gate_hw; + + const struct clk_ops *mux_ops; + const struct clk_ops *rate_ops; + const struct clk_ops *gate_ops; +}; + +#define to_clk_regmap_composite(_hw) \ + container_of(_hw, struct clk_regmap_composite, hw) + +static u8 clk_regmap_composite_get_parent(struct clk_hw *hw) +{ + struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); + const struct clk_ops *mux_ops = composite->mux_ops; + struct clk_hw *mux_hw = composite->mux_hw; + + __clk_hw_set_clk(mux_hw, hw); + + return mux_ops->get_parent(mux_hw); +} + +static int clk_regmap_composite_set_parent(struct clk_hw *hw, u8 index) +{ + struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); + const struct clk_ops *mux_ops = composite->mux_ops; + struct clk_hw *mux_hw = composite->mux_hw; + + __clk_hw_set_clk(mux_hw, hw); + + return mux_ops->set_parent(mux_hw, index); +} + +static unsigned long clk_regmap_composite_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); + const struct clk_ops *rate_ops = composite->rate_ops; + struct clk_hw *rate_hw = composite->rate_hw; + + __clk_hw_set_clk(rate_hw, hw); + + return rate_ops->recalc_rate(rate_hw, parent_rate); +} + +static int clk_regmap_composite_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); + const struct clk_ops *rate_ops = composite->rate_ops; + const struct clk_ops *mux_ops = composite->mux_ops; + struct clk_hw *rate_hw = composite->rate_hw; + struct clk_hw *mux_hw = composite->mux_hw; + struct clk_hw *parent; + unsigned long parent_rate; + long tmp_rate, best_rate = 0; + unsigned long rate_diff; + unsigned long best_rate_diff = ULONG_MAX; + long rate; + unsigned int i; + + if (rate_hw && rate_ops && rate_ops->determine_rate) { + __clk_hw_set_clk(rate_hw, hw); + return rate_ops->determine_rate(rate_hw, req); + } else if (rate_hw && rate_ops && rate_ops->round_rate && + mux_hw && mux_ops && mux_ops->set_parent) { + req->best_parent_hw = NULL; + + if (clk_hw_get_flags(hw) & CLK_SET_RATE_NO_REPARENT) { + parent = clk_hw_get_parent(mux_hw); + req->best_parent_hw = parent; + req->best_parent_rate = clk_hw_get_rate(parent); + + rate = rate_ops->round_rate(rate_hw, req->rate, + &req->best_parent_rate); + if (rate < 0) + return rate; + + req->rate = rate; + return 0; + } + + for (i = 0; i < clk_hw_get_num_parents(mux_hw); i++) { + parent = clk_hw_get_parent_by_index(mux_hw, i); + if (!parent) + continue; + + parent_rate = clk_hw_get_rate(parent); + + tmp_rate = rate_ops->round_rate(rate_hw, req->rate, + &parent_rate); + if (tmp_rate < 0) + continue; + + rate_diff = abs(req->rate - tmp_rate); + + if (!rate_diff || !req->best_parent_hw || + best_rate_diff > rate_diff) { + req->best_parent_hw = parent; + req->best_parent_rate = parent_rate; + best_rate_diff = rate_diff; + best_rate = tmp_rate; + } + + if (!rate_diff) + return 0; + } + + req->rate = best_rate; + return 0; + } else if (mux_hw && mux_ops && mux_ops->determine_rate) { + __clk_hw_set_clk(mux_hw, hw); + return mux_ops->determine_rate(mux_hw, req); + } else { + return -EINVAL; + } + + return 0; +} + +static long clk_regmap_composite_round_rate(struct clk_hw *hw, + unsigned long rate, + unsigned long *prate) +{ + struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); + const struct clk_ops *rate_ops = composite->rate_ops; + struct clk_hw *rate_hw = composite->rate_hw; + + __clk_hw_set_clk(rate_hw, hw); + + return rate_ops->round_rate(rate_hw, rate, prate); +} + +static int clk_regmap_composite_set_rate(struct clk_hw *hw, + unsigned long rate, + unsigned long parent_rate) +{ + struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); + const struct clk_ops *rate_ops = composite->rate_ops; + struct clk_hw *rate_hw = composite->rate_hw; + + __clk_hw_set_clk(rate_hw, hw); + + return rate_ops->set_rate(rate_hw, rate, parent_rate); +} + +static int clk_regmap_composite_is_prepared(struct clk_hw *hw) +{ + struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); + const struct clk_ops *gate_ops = composite->gate_ops; + struct clk_hw *gate_hw = composite->gate_hw; + + __clk_hw_set_clk(gate_hw, hw); + + return gate_ops->is_prepared(gate_hw); +} + +static int clk_regmap_composite_prepare(struct clk_hw *hw) +{ + struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); + const struct clk_ops *gate_ops = composite->gate_ops; + struct clk_hw *gate_hw = composite->gate_hw; + + __clk_hw_set_clk(gate_hw, hw); + + return gate_ops->prepare(gate_hw); +} + +static void clk_regmap_composite_unprepare(struct clk_hw *hw) +{ + struct clk_regmap_composite *composite = to_clk_regmap_composite(hw); + const struct clk_ops *gate_ops = composite->gate_ops; + struct clk_hw *gate_hw = composite->gate_hw; + + __clk_hw_set_clk(gate_hw, hw); + + gate_ops->unprepare(gate_hw); +} + +struct clk * +devm_clk_regmap_register_composite(struct device *dev, const char *name, + const char *const *parent_names, + u8 num_parents, struct regmap *regmap, + u32 mux_reg, u8 mux_shift, u8 mux_width, + u32 div_reg, u8 div_shift, u8 div_width, + u8 div_flags, + u32 gate_reg, u8 gate_shift, + unsigned long flags) +{ + struct clk_regmap_gate *gate = NULL; + struct clk_regmap_mux *mux = NULL; + struct clk_regmap_divider *div = NULL; + struct clk_regmap_fractional_divider *fd = NULL; + const struct clk_ops *mux_ops = NULL, *div_ops = NULL, *gate_ops = NULL; + const struct clk_ops *fd_ops = NULL; + struct clk_hw *mux_hw = NULL, *div_hw = NULL, *gate_hw = NULL; + struct clk_hw *fd_hw = NULL; + struct clk *clk; + struct clk_init_data init = {}; + struct clk_regmap_composite *composite; + struct clk_ops *clk_composite_ops; + + if (num_parents > 1) { + mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL); + if (!mux) + return ERR_PTR(-ENOMEM); + + mux->dev = dev; + mux->regmap = regmap; + mux->reg = mux_reg; + mux->shift = mux_shift; + mux->mask = BIT(mux_width) - 1; + mux_ops = &clk_regmap_mux_ops; + mux_hw = &mux->hw; + } + + if (gate_reg > 0) { + gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL); + if (!gate) + return ERR_PTR(-ENOMEM); + + gate->dev = dev; + gate->regmap = regmap; + gate->reg = gate_reg; + gate->shift = gate_shift; + gate_ops = &clk_regmap_gate_ops; + gate_hw = &gate->hw; + } + + if (div_reg > 0) { + if (div_flags & CLK_DIVIDER_HIWORD_MASK) { + div = devm_kzalloc(dev, sizeof(*div), GFP_KERNEL); + if (!div) + return ERR_PTR(-ENOMEM); + + div->dev = dev; + div->regmap = regmap; + div->reg = div_reg; + div->shift = div_shift; + div->width = div_width; + div_ops = &clk_regmap_divider_ops; + div_hw = &div->hw; + } else { + fd = devm_kzalloc(dev, sizeof(*fd), GFP_KERNEL); + if (!fd) + return ERR_PTR(-ENOMEM); + + fd->dev = dev; + fd->regmap = regmap; + fd->reg = div_reg; + fd->mshift = 16; + fd->mwidth = 16; + fd->mmask = GENMASK(fd->mwidth - 1, 0) << fd->mshift; + fd->nshift = 0; + fd->nwidth = 16; + fd->nmask = GENMASK(fd->nwidth - 1, 0) << fd->nshift; + fd_ops = &clk_regmap_fractional_divider_ops; + fd_hw = &fd->hw; + } + } + + composite = devm_kzalloc(dev, sizeof(*composite), GFP_KERNEL); + if (!composite) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.flags = flags; + init.parent_names = parent_names; + init.num_parents = num_parents; + + clk_composite_ops = &composite->ops; + + if (mux_hw && mux_ops) { + if (!mux_ops->get_parent) + return ERR_PTR(-EINVAL); + + composite->mux_hw = mux_hw; + composite->mux_ops = mux_ops; + clk_composite_ops->get_parent = + clk_regmap_composite_get_parent; + if (mux_ops->set_parent) + clk_composite_ops->set_parent = + clk_regmap_composite_set_parent; + if (mux_ops->determine_rate) + clk_composite_ops->determine_rate = + clk_regmap_composite_determine_rate; + } + + if (div_hw && div_ops) { + if (!div_ops->recalc_rate) + return ERR_PTR(-EINVAL); + + clk_composite_ops->recalc_rate = + clk_regmap_composite_recalc_rate; + + if (div_ops->determine_rate) + clk_composite_ops->determine_rate = + clk_regmap_composite_determine_rate; + else if (div_ops->round_rate) + clk_composite_ops->round_rate = + clk_regmap_composite_round_rate; + + /* .set_rate requires either .round_rate or .determine_rate */ + if (div_ops->set_rate) { + if (div_ops->determine_rate || div_ops->round_rate) + clk_composite_ops->set_rate = + clk_regmap_composite_set_rate; + else + WARN(1, "missing round_rate op\n"); + } + + composite->rate_hw = div_hw; + composite->rate_ops = div_ops; + } + + if (fd_hw && fd_ops) { + if (!fd_ops->recalc_rate) + return ERR_PTR(-EINVAL); + + clk_composite_ops->recalc_rate = + clk_regmap_composite_recalc_rate; + + if (fd_ops->determine_rate) + clk_composite_ops->determine_rate = + clk_regmap_composite_determine_rate; + else if (fd_ops->round_rate) + clk_composite_ops->round_rate = + clk_regmap_composite_round_rate; + + /* .set_rate requires either .round_rate or .determine_rate */ + if (fd_ops->set_rate) { + if (fd_ops->determine_rate || fd_ops->round_rate) + clk_composite_ops->set_rate = + clk_regmap_composite_set_rate; + else + WARN(1, "missing round_rate op\n"); + } + + composite->rate_hw = fd_hw; + composite->rate_ops = fd_ops; + } + + if (gate_hw && gate_ops) { + if (!gate_ops->is_prepared || !gate_ops->prepare || + !gate_ops->unprepare) + return ERR_PTR(-EINVAL); + + composite->gate_hw = gate_hw; + composite->gate_ops = gate_ops; + clk_composite_ops->is_prepared = + clk_regmap_composite_is_prepared; + clk_composite_ops->prepare = clk_regmap_composite_prepare; + clk_composite_ops->unprepare = clk_regmap_composite_unprepare; + } + + init.ops = clk_composite_ops; + composite->dev = dev; + composite->hw.init = &init; + + clk = devm_clk_register(dev, &composite->hw); + if (IS_ERR(clk)) + return clk; + + if (composite->mux_hw) + composite->mux_hw->clk = clk; + + if (composite->rate_hw) + composite->rate_hw->clk = clk; + + if (composite->gate_hw) + composite->gate_hw->clk = clk; + + return clk; +} +EXPORT_SYMBOL_GPL(devm_clk_regmap_register_composite); diff --git a/drivers/clk/rockchip/regmap/clk-regmap-divider.c b/drivers/clk/rockchip/regmap/clk-regmap-divider.c new file mode 100755 index 000000000000..cb59a3d9acd2 --- /dev/null +++ b/drivers/clk/rockchip/regmap/clk-regmap-divider.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2017 Rockchip Electronics Co. Ltd. + * + * Base on code in drivers/clk/clk-divider.c. + * See clk-divider.c for further copyright information. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "clk-regmap.h" + +#define div_mask(width) ((1 << (width)) - 1) + +#define to_clk_regmap_divider(_hw) \ + container_of(_hw, struct clk_regmap_divider, hw) + +static unsigned long +clk_regmap_divider_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +{ + struct clk_regmap_divider *divider = to_clk_regmap_divider(hw); + unsigned int val, div; + + regmap_read(divider->regmap, divider->reg, &val); + + div = val >> divider->shift; + div &= div_mask(divider->width); + + return divider_recalc_rate(hw, parent_rate, div, NULL, + CLK_DIVIDER_ROUND_CLOSEST, divider->width); +} + +static long +clk_regmap_divider_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *prate) +{ + struct clk_regmap_divider *divider = to_clk_regmap_divider(hw); + + return divider_round_rate(hw, rate, prate, NULL, divider->width, + CLK_DIVIDER_ROUND_CLOSEST); +} + +static int +clk_regmap_divider_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_regmap_divider *divider = to_clk_regmap_divider(hw); + u32 val, div; + + div = divider_get_val(rate, parent_rate, NULL, divider->width, + CLK_DIVIDER_ROUND_CLOSEST); + + dev_dbg(divider->dev, "%s: parent_rate=%ld, div=%d, rate=%ld\n", + clk_hw_get_name(hw), parent_rate, div, rate); + + val = div_mask(divider->width) << (divider->shift + 16); + val |= div << divider->shift; + + return regmap_write(divider->regmap, divider->reg, val); +} + +const struct clk_ops clk_regmap_divider_ops = { + .recalc_rate = clk_regmap_divider_recalc_rate, + .round_rate = clk_regmap_divider_round_rate, + .set_rate = clk_regmap_divider_set_rate, +}; +EXPORT_SYMBOL_GPL(clk_regmap_divider_ops); + +struct clk * +devm_clk_regmap_register_divider(struct device *dev, const char *name, + const char *parent_name, struct regmap *regmap, + u32 reg, u8 shift, u8 width, + unsigned long flags) +{ + struct clk_regmap_divider *divider; + struct clk_init_data init = {}; + + divider = devm_kzalloc(dev, sizeof(*divider), GFP_KERNEL); + if (!divider) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.ops = &clk_regmap_divider_ops; + init.flags = flags; + init.parent_names = (parent_name ? &parent_name : NULL); + init.num_parents = (parent_name ? 1 : 0); + + divider->dev = dev; + divider->regmap = regmap; + divider->reg = reg; + divider->shift = shift; + divider->width = width; + divider->hw.init = &init; + + return devm_clk_register(dev, ÷r->hw); +} +EXPORT_SYMBOL_GPL(devm_clk_regmap_register_divider); diff --git a/drivers/clk/rockchip/regmap/clk-regmap-fractional-divider.c b/drivers/clk/rockchip/regmap/clk-regmap-fractional-divider.c new file mode 100755 index 000000000000..3d5f1d2691e3 --- /dev/null +++ b/drivers/clk/rockchip/regmap/clk-regmap-fractional-divider.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019 Rockchip Electronics Co. Ltd. + * + * Base on code in drivers/clk/clk-fractional-divider.c. + * See clk-fractional-divider.c for further copyright information. + */ + +#include + +#include "clk-regmap.h" + +#define to_clk_regmap_fractional_divider(_hw) \ + container_of(_hw, struct clk_regmap_fractional_divider, hw) + +static unsigned long +clk_regmap_fractional_divider_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct clk_regmap_fractional_divider *fd = + to_clk_regmap_fractional_divider(hw); + unsigned long m, n; + u32 val; + u64 ret; + + regmap_read(fd->regmap, fd->reg, &val); + + m = (val & fd->mmask) >> fd->mshift; + n = (val & fd->nmask) >> fd->nshift; + + if (!n || !m) + return parent_rate; + + ret = (u64)parent_rate * m; + do_div(ret, n); + + return ret; +} + +static void clk_regmap_fractional_divider_approximation(struct clk_hw *hw, + unsigned long rate, unsigned long *parent_rate, + unsigned long *m, unsigned long *n) +{ + struct clk_regmap_fractional_divider *fd = + to_clk_regmap_fractional_divider(hw); + unsigned long p_rate, p_parent_rate; + struct clk_hw *p_parent; + unsigned long scale; + + p_rate = clk_hw_get_rate(clk_hw_get_parent(hw)); + if ((rate * 20 > p_rate) && (p_rate % rate != 0)) { + p_parent = clk_hw_get_parent(clk_hw_get_parent(hw)); + p_parent_rate = clk_hw_get_rate(p_parent); + *parent_rate = p_parent_rate; + } + + /* + * Get rate closer to *parent_rate to guarantee there is no overflow + * for m and n. In the result it will be the nearest rate left shifted + * by (scale - fd->nwidth) bits. + */ + scale = fls_long(*parent_rate / rate - 1); + if (scale > fd->nwidth) + rate <<= scale - fd->nwidth; + + rational_best_approximation(rate, *parent_rate, + GENMASK(fd->mwidth - 1, 0), + GENMASK(fd->nwidth - 1, 0), + m, n); +} + +static long +clk_regmap_fractional_divider_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + unsigned long m, n; + u64 ret; + + if (!rate) + return *parent_rate; + + if (rate >= *parent_rate) + return *parent_rate; + + clk_regmap_fractional_divider_approximation(hw, rate, parent_rate, + &m, &n); + + ret = (u64)*parent_rate * m; + do_div(ret, n); + + return ret; +} + +static int +clk_regmap_fractional_divider_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_regmap_fractional_divider *fd = + to_clk_regmap_fractional_divider(hw); + unsigned long m, n; + u32 val; + + rational_best_approximation(rate, parent_rate, + GENMASK(fd->mwidth - 1, 0), GENMASK(fd->nwidth - 1, 0), + &m, &n); + + dev_dbg(fd->dev, "%s: parent_rate=%ld, m=%ld, n=%ld, rate=%ld\n", + clk_hw_get_name(hw), parent_rate, m, n, rate); + + regmap_read(fd->regmap, fd->reg, &val); + val &= ~(fd->mmask | fd->nmask); + val |= (m << fd->mshift) | (n << fd->nshift); + + return regmap_write(fd->regmap, fd->reg, val); +} + +const struct clk_ops clk_regmap_fractional_divider_ops = { + .recalc_rate = clk_regmap_fractional_divider_recalc_rate, + .round_rate = clk_regmap_fractional_divider_round_rate, + .set_rate = clk_regmap_fractional_divider_set_rate, +}; +EXPORT_SYMBOL_GPL(clk_regmap_fractional_divider_ops); + +struct clk * +devm_clk_regmap_register_fractional_divider(struct device *dev, + const char *name, + const char *parent_name, + struct regmap *regmap, + u32 reg, unsigned long flags) +{ + struct clk_regmap_fractional_divider *fd; + struct clk_init_data init; + + fd = devm_kzalloc(dev, sizeof(*fd), GFP_KERNEL); + if (!fd) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.ops = &clk_regmap_fractional_divider_ops; + init.flags = flags; + init.parent_names = (parent_name ? &parent_name : NULL); + init.num_parents = (parent_name ? 1 : 0); + + fd->dev = dev; + fd->regmap = regmap; + fd->reg = reg; + fd->mshift = 16; + fd->mwidth = 16; + fd->mmask = GENMASK(fd->mwidth - 1, 0) << fd->mshift; + fd->nshift = 0; + fd->nwidth = 16; + fd->nmask = GENMASK(fd->nwidth - 1, 0) << fd->nshift; + fd->hw.init = &init; + + return devm_clk_register(dev, &fd->hw); +} +EXPORT_SYMBOL_GPL(devm_clk_regmap_register_fractional_divider); diff --git a/drivers/clk/rockchip/regmap/clk-regmap-gate.c b/drivers/clk/rockchip/regmap/clk-regmap-gate.c new file mode 100755 index 000000000000..36549b912fa2 --- /dev/null +++ b/drivers/clk/rockchip/regmap/clk-regmap-gate.c @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2017 Rockchip Electronics Co. Ltd. + * + * Base on code in drivers/clk/clk-gate.c. + * See clk-gate.c for further copyright information. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "clk-regmap.h" + +#define to_clk_regmap_gate(_hw) container_of(_hw, struct clk_regmap_gate, hw) + +static int clk_regmap_gate_prepare(struct clk_hw *hw) +{ + struct clk_regmap_gate *gate = to_clk_regmap_gate(hw); + + return regmap_write(gate->regmap, gate->reg, + 0 | BIT(gate->shift + 16)); +} + +static void clk_regmap_gate_unprepare(struct clk_hw *hw) +{ + struct clk_regmap_gate *gate = to_clk_regmap_gate(hw); + + regmap_write(gate->regmap, gate->reg, + BIT(gate->shift) | BIT(gate->shift + 16)); +} + +static int clk_regmap_gate_is_prepared(struct clk_hw *hw) +{ + struct clk_regmap_gate *gate = to_clk_regmap_gate(hw); + u32 val; + + regmap_read(gate->regmap, gate->reg, &val); + + return !(val & BIT(gate->shift)); +} + +const struct clk_ops clk_regmap_gate_ops = { + .prepare = clk_regmap_gate_prepare, + .unprepare = clk_regmap_gate_unprepare, + .is_prepared = clk_regmap_gate_is_prepared, +}; +EXPORT_SYMBOL_GPL(clk_regmap_gate_ops); + +struct clk * +devm_clk_regmap_register_gate(struct device *dev, const char *name, + const char *parent_name, + struct regmap *regmap, u32 reg, u8 shift, + unsigned long flags) +{ + struct clk_regmap_gate *gate; + struct clk_init_data init = {}; + + gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL); + if (!gate) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.ops = &clk_regmap_gate_ops; + init.flags = flags; + init.parent_names = (parent_name ? &parent_name : NULL); + init.num_parents = (parent_name ? 1 : 0); + + gate->dev = dev; + gate->regmap = regmap; + gate->reg = reg; + gate->shift = shift; + gate->hw.init = &init; + + return devm_clk_register(dev, &gate->hw); +} +EXPORT_SYMBOL_GPL(devm_clk_regmap_register_gate); diff --git a/drivers/clk/rockchip/regmap/clk-regmap-mux.c b/drivers/clk/rockchip/regmap/clk-regmap-mux.c new file mode 100755 index 000000000000..49d58b9c53ab --- /dev/null +++ b/drivers/clk/rockchip/regmap/clk-regmap-mux.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2017 Rockchip Electronics Co. Ltd. + * + * Base on code in drivers/clk/clk-mux.c. + * See clk-mux.c for further copyright information. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "clk-regmap.h" + +#define to_clk_regmap_mux(_hw) container_of(_hw, struct clk_regmap_mux, hw) + +static u8 clk_regmap_mux_get_parent(struct clk_hw *hw) +{ + struct clk_regmap_mux *mux = to_clk_regmap_mux(hw); + u8 index; + u32 val; + + regmap_read(mux->regmap, mux->reg, &val); + + index = val >> mux->shift; + index &= mux->mask; + + return index; +} + +static int clk_regmap_mux_set_parent(struct clk_hw *hw, u8 index) +{ + struct clk_regmap_mux *mux = to_clk_regmap_mux(hw); + + return regmap_write(mux->regmap, mux->reg, (index << mux->shift) | + (mux->mask << (mux->shift + 16))); +} + +const struct clk_ops clk_regmap_mux_ops = { + .set_parent = clk_regmap_mux_set_parent, + .get_parent = clk_regmap_mux_get_parent, + .determine_rate = __clk_mux_determine_rate, +}; +EXPORT_SYMBOL_GPL(clk_regmap_mux_ops); + +struct clk * +devm_clk_regmap_register_mux(struct device *dev, const char *name, + const char * const *parent_names, u8 num_parents, + struct regmap *regmap, u32 reg, u8 shift, u8 width, + unsigned long flags) +{ + struct clk_regmap_mux *mux; + struct clk_init_data init = {}; + + mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL); + if (!mux) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.ops = &clk_regmap_mux_ops; + init.flags = flags; + init.parent_names = parent_names; + init.num_parents = num_parents; + + mux->dev = dev; + mux->regmap = regmap; + mux->reg = reg; + mux->shift = shift; + mux->mask = BIT(width) - 1; + mux->hw.init = &init; + + return devm_clk_register(dev, &mux->hw); +} +EXPORT_SYMBOL_GPL(devm_clk_regmap_register_mux); diff --git a/drivers/clk/rockchip/regmap/clk-regmap-pll.c b/drivers/clk/rockchip/regmap/clk-regmap-pll.c new file mode 100755 index 000000000000..24ad7eda9d94 --- /dev/null +++ b/drivers/clk/rockchip/regmap/clk-regmap-pll.c @@ -0,0 +1,363 @@ +/* + * Copyright (c) 2017 Rockchip Electronics Co. Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "clk-regmap.h" + +#define PLLCON_OFFSET(x) (x * 4) + +#define PLL_BYPASS(x) HIWORD_UPDATE(x, 15, 15) +#define PLL_BYPASS_MASK BIT(15) +#define PLL_BYPASS_SHIFT 15 +#define PLL_POSTDIV1(x) HIWORD_UPDATE(x, 14, 12) +#define PLL_POSTDIV1_MASK GENMASK(14, 12) +#define PLL_POSTDIV1_SHIFT 12 +#define PLL_FBDIV(x) HIWORD_UPDATE(x, 11, 0) +#define PLL_FBDIV_MASK GENMASK(11, 0) +#define PLL_FBDIV_SHIFT 0 + +#define PLL_POSTDIV2(x) HIWORD_UPDATE(x, 8, 6) +#define PLL_POSTDIV2_MASK GENMASK(8, 6) +#define PLL_POSTDIV2_SHIFT 6 +#define PLL_REFDIV(x) HIWORD_UPDATE(x, 5, 0) +#define PLL_REFDIV_MASK GENMASK(5, 0) +#define PLL_REFDIV_SHIFT 0 + +#define PLL_FOUT_4PHASE_CLK_POWER_DOWN BIT(27) +#define PLL_FOUT_VCO_CLK_POWER_DOWN BIT(26) +#define PLL_FOUT_POST_DIV_POWER_DOWN BIT(25) +#define PLL_DAC_POWER_DOWN BIT(24) +#define PLL_FRAC(x) UPDATE(x, 23, 0) +#define PLL_FRAC_MASK GENMASK(23, 0) +#define PLL_FRAC_SHIFT 0 + +#define MIN_FREF_RATE 10000000UL +#define MAX_FREF_RATE 800000000UL +#define MIN_FREFDIV_RATE 1000000UL +#define MAX_FREFDIV_RATE 40000000UL +#define MIN_FVCO_RATE 400000000UL +#define MAX_FVCO_RATE 1600000000UL +#define MIN_FOUTPOSTDIV_RATE 8000000UL +#define MAX_FOUTPOSTDIV_RATE 1600000000UL + +struct clk_regmap_pll { + struct clk_hw hw; + struct device *dev; + struct regmap *regmap; + unsigned int reg; + u8 pd_shift; + u8 dsmpd_shift; + u8 lock_shift; +}; + +#define to_clk_regmap_pll(_hw) container_of(_hw, struct clk_regmap_pll, hw) + +static unsigned long +clk_regmap_pll_recalc_rate(struct clk_hw *hw, unsigned long prate) +{ + struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); + unsigned int postdiv1, fbdiv, dsmpd, postdiv2, refdiv, frac, bypass; + unsigned int con0, con1, con2; + u64 foutvco, foutpostdiv; + + regmap_read(pll->regmap, pll->reg + PLLCON_OFFSET(0), &con0); + regmap_read(pll->regmap, pll->reg + PLLCON_OFFSET(1), &con1); + regmap_read(pll->regmap, pll->reg + PLLCON_OFFSET(2), &con2); + + bypass = (con0 & PLL_BYPASS_MASK) >> PLL_BYPASS_SHIFT; + postdiv1 = (con0 & PLL_POSTDIV1_MASK) >> PLL_POSTDIV1_SHIFT; + fbdiv = (con0 & PLL_FBDIV_MASK) >> PLL_FBDIV_SHIFT; + dsmpd = (con1 & BIT(pll->dsmpd_shift)) >> pll->dsmpd_shift; + postdiv2 = (con1 & PLL_POSTDIV2_MASK) >> PLL_POSTDIV2_SHIFT; + refdiv = (con1 & PLL_REFDIV_MASK) >> PLL_REFDIV_SHIFT; + frac = (con2 & PLL_FRAC_MASK) >> PLL_FRAC_SHIFT; + + if (bypass) + return prate; + + foutvco = prate * fbdiv; + do_div(foutvco, refdiv); + + if (!dsmpd) { + u64 frac_rate = (u64)prate * frac; + + do_div(frac_rate, refdiv); + foutvco += frac_rate >> 24; + } + + foutpostdiv = foutvco; + do_div(foutpostdiv, postdiv1); + do_div(foutpostdiv, postdiv2); + + return foutpostdiv; +} + +static long clk_pll_round_rate(unsigned long fin, unsigned long fout, + u8 *refdiv, u16 *fbdiv, + u8 *postdiv1, u8 *postdiv2, + u32 *frac, u8 *dsmpd, u8 *bypass) +{ + u8 min_refdiv, max_refdiv, postdiv; + u8 _dsmpd = 1, _postdiv1 = 0, _postdiv2 = 0, _refdiv = 0; + u16 _fbdiv = 0; + u32 _frac = 0; + u64 foutvco, foutpostdiv; + + /* + * FREF : 10MHz ~ 800MHz + * FREFDIV : 1MHz ~ 40MHz + * FOUTVCO : 400MHz ~ 1.6GHz + * FOUTPOSTDIV : 8MHz ~ 1.6GHz + */ + if (fin < MIN_FREF_RATE || fin > MAX_FREF_RATE) + return -EINVAL; + + if (fout < MIN_FOUTPOSTDIV_RATE || fout > MAX_FOUTPOSTDIV_RATE) + return -EINVAL; + + if (fin == fout) { + if (bypass) + *bypass = true; + return fin; + } + + min_refdiv = DIV_ROUND_UP(fin, MAX_FREFDIV_RATE); + max_refdiv = fin / MIN_FREFDIV_RATE; + if (max_refdiv > 64) + max_refdiv = 64; + + if (fout < MIN_FVCO_RATE) { + postdiv = DIV_ROUND_UP_ULL(MIN_FVCO_RATE, fout); + + for (_postdiv2 = 1; _postdiv2 < 8; _postdiv2++) { + if (postdiv % _postdiv2) + continue; + + _postdiv1 = postdiv / _postdiv2; + + if (_postdiv1 > 0 && _postdiv1 < 8) + break; + } + + if (_postdiv2 > 7) + return -EINVAL; + + fout *= _postdiv1 * _postdiv2; + } else { + _postdiv1 = 1; + _postdiv2 = 1; + } + + for (_refdiv = min_refdiv; _refdiv <= max_refdiv; _refdiv++) { + u64 tmp, frac_rate; + + if (fin % _refdiv) + continue; + + tmp = (u64)fout * _refdiv; + do_div(tmp, fin); + _fbdiv = tmp; + if (_fbdiv < 10 || _fbdiv > 1600) + continue; + + tmp = (u64)_fbdiv * fin; + do_div(tmp, _refdiv); + if (fout < MIN_FVCO_RATE || fout > MAX_FVCO_RATE) + continue; + + frac_rate = fout - tmp; + + if (frac_rate) { + tmp = (u64)frac_rate * _refdiv; + tmp <<= 24; + do_div(tmp, fin); + _frac = tmp; + _dsmpd = 0; + } + + break; + } + + /* + * If DSMPD = 1 (DSM is disabled, "integer mode") + * FOUTVCO = FREF / REFDIV * FBDIV + * FOUTPOSTDIV = FOUTVCO / POSTDIV1 / POSTDIV2 + * + * If DSMPD = 0 (DSM is enabled, "fractional mode") + * FOUTVCO = FREF / REFDIV * (FBDIV + FRAC / 2^24) + * FOUTPOSTDIV = FOUTVCO / POSTDIV1 / POSTDIV2 + */ + foutvco = fin * _fbdiv; + do_div(foutvco, _refdiv); + + if (!_dsmpd) { + u64 frac_rate = (u64)fin * _frac; + + do_div(frac_rate, _refdiv); + foutvco += frac_rate >> 24; + } + + foutpostdiv = foutvco; + do_div(foutpostdiv, _postdiv1); + do_div(foutpostdiv, _postdiv2); + + if (refdiv) + *refdiv = _refdiv; + if (fbdiv) + *fbdiv = _fbdiv; + if (postdiv1) + *postdiv1 = _postdiv1; + if (postdiv2) + *postdiv2 = _postdiv2; + if (frac) + *frac = _frac; + if (dsmpd) + *dsmpd = _dsmpd; + if (bypass) + *bypass = false; + + return (unsigned long)foutpostdiv; +} + +static long +clk_regmap_pll_round_rate(struct clk_hw *hw, unsigned long drate, + unsigned long *prate) +{ + struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); + long rate; + + rate = clk_pll_round_rate(*prate, drate, NULL, NULL, NULL, NULL, NULL, + NULL, NULL); + + dev_dbg(pll->dev, "%s: prate=%ld, drate=%ld, rate=%ld\n", + clk_hw_get_name(hw), *prate, drate, rate); + + return rate; +} + +static int +clk_regmap_pll_set_rate(struct clk_hw *hw, unsigned long drate, + unsigned long prate) +{ + struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); + u8 refdiv, postdiv1, postdiv2, dsmpd, bypass; + u16 fbdiv; + u32 frac; + long rate; + + rate = clk_pll_round_rate(prate, drate, &refdiv, &fbdiv, &postdiv1, + &postdiv2, &frac, &dsmpd, &bypass); + if (rate < 0) + return rate; + + dev_dbg(pll->dev, "%s: rate=%ld, bypass=%d\n", + clk_hw_get_name(hw), drate, bypass); + + if (bypass) { + regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(0), + PLL_BYPASS(1)); + } else { + regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(0), + PLL_BYPASS(0) | PLL_POSTDIV1(postdiv1) | + PLL_FBDIV(fbdiv)); + regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(1), + HIWORD_UPDATE(dsmpd, pll->dsmpd_shift, pll->dsmpd_shift) | + PLL_POSTDIV2(postdiv2) | PLL_REFDIV(refdiv)); + regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(2), + PLL_FRAC(frac)); + + dev_dbg(pll->dev, "refdiv=%d, fbdiv=%d, frac=%d\n", + refdiv, fbdiv, frac); + dev_dbg(pll->dev, "postdiv1=%d, postdiv2=%d\n", + postdiv1, postdiv2); + } + + return 0; +} + +static int clk_regmap_pll_prepare(struct clk_hw *hw) +{ + struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); + u32 v; + int ret; + + regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(1), + HIWORD_UPDATE(0, pll->pd_shift, pll->pd_shift)); + + ret = regmap_read_poll_timeout(pll->regmap, + pll->reg + PLLCON_OFFSET(1), + v, v & BIT(pll->lock_shift), 50, 50000); + if (ret) + dev_err(pll->dev, "%s is not lock\n", clk_hw_get_name(hw)); + + return 0; +} + +static void clk_regmap_pll_unprepare(struct clk_hw *hw) +{ + struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); + + regmap_write(pll->regmap, pll->reg + PLLCON_OFFSET(1), + HIWORD_UPDATE(1, pll->pd_shift, pll->pd_shift)); +} + +static int clk_regmap_pll_is_prepared(struct clk_hw *hw) +{ + struct clk_regmap_pll *pll = to_clk_regmap_pll(hw); + unsigned int con1; + + regmap_read(pll->regmap, pll->reg + PLLCON_OFFSET(1), &con1); + + return !(con1 & BIT(pll->pd_shift)); +} + +static const struct clk_ops clk_regmap_pll_ops = { + .recalc_rate = clk_regmap_pll_recalc_rate, + .round_rate = clk_regmap_pll_round_rate, + .set_rate = clk_regmap_pll_set_rate, + .prepare = clk_regmap_pll_prepare, + .unprepare = clk_regmap_pll_unprepare, + .is_prepared = clk_regmap_pll_is_prepared, +}; + +struct clk * +devm_clk_regmap_register_pll(struct device *dev, const char *name, + const char *parent_name, + struct regmap *regmap, u32 reg, u8 pd_shift, + u8 dsmpd_shift, u8 lock_shift, + unsigned long flags) +{ + struct clk_regmap_pll *pll; + struct clk_init_data init = {}; + + pll = devm_kzalloc(dev, sizeof(*pll), GFP_KERNEL); + if (!pll) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.ops = &clk_regmap_pll_ops; + init.flags = flags; + init.parent_names = (parent_name ? &parent_name : NULL); + init.num_parents = (parent_name ? 1 : 0); + + pll->dev = dev; + pll->regmap = regmap; + pll->reg = reg; + pll->pd_shift = pd_shift; + pll->dsmpd_shift = dsmpd_shift; + pll->lock_shift = lock_shift; + pll->hw.init = &init; + + return devm_clk_register(dev, &pll->hw); +} +EXPORT_SYMBOL_GPL(devm_clk_regmap_register_pll); diff --git a/drivers/clk/rockchip/regmap/clk-regmap.h b/drivers/clk/rockchip/regmap/clk-regmap.h new file mode 100755 index 000000000000..255c0201a590 --- /dev/null +++ b/drivers/clk/rockchip/regmap/clk-regmap.h @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2017 Rockchip Electronics Co. Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __CLK_REGMAP_H__ +#define __CLK_REGMAP_H__ + +#include +#include +#include +#include +#include +#include +#include + +#define UPDATE(x, h, l) (((x) << (l)) & GENMASK((h), (l))) +#define HIWORD_UPDATE(v, h, l) (((v) << (l)) | (GENMASK((h), (l)) << 16)) + +struct clk_pll_data { + unsigned int id; + const char *name; + const char *parent_name; + u32 reg; + u8 pd_shift; + u8 dsmpd_shift; + u8 lock_shift; + unsigned long flags; +}; + +#define PLL(_id, _name, _parent_name, _reg, _pd_shift, _dsmpd_shift, \ + _lock_shift, _flags) \ +{ \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent_name, \ + .reg = _reg, \ + .pd_shift = _pd_shift, \ + .dsmpd_shift = _dsmpd_shift, \ + .lock_shift = _lock_shift, \ + .flags = _flags, \ +} + +#define RK618_PLL(_id, _name, _parent_name, _reg, _flags) \ + PLL(_id, _name, _parent_name, _reg, 10, 9, 15, _flags) + +#define RK628_PLL(_id, _name, _parent_name, _reg, _flags) \ + PLL(_id, _name, _parent_name, _reg, 13, 12, 10, _flags) + +struct clk_mux_data { + unsigned int id; + const char *name; + const char *const *parent_names; + u8 num_parents; + u32 reg; + u8 shift; + u8 width; + unsigned long flags; +}; + +#define MUX(_id, _name, _parent_names, _reg, _shift, _width, _flags) \ +{ \ + .id = _id, \ + .name = _name, \ + .parent_names = _parent_names, \ + .num_parents = ARRAY_SIZE(_parent_names), \ + .reg = _reg, \ + .shift = _shift, \ + .width = _width, \ + .flags = _flags, \ +} + +struct clk_gate_data { + unsigned int id; + const char *name; + const char *parent_name; + u32 reg; + u8 shift; + unsigned long flags; +}; + +#define GATE(_id, _name, _parent_name, _reg, _shift, _flags) \ +{ \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent_name, \ + .reg = _reg, \ + .shift = _shift, \ + .flags = _flags, \ +} + +struct clk_divider_data { + unsigned int id; + const char *name; + const char *parent_name; + u32 reg; + u8 shift; + u8 width; + unsigned long flags; +}; + +#define DIV(_id, _name, _parent_name, _reg, _shift, _width, _flags) \ +{ \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent_name, \ + .reg = _reg, \ + .shift = _shift, \ + .width = _width, \ + .flags = _flags, \ +} + +struct clk_composite_data { + unsigned int id; + const char *name; + const char *const *parent_names; + u8 num_parents; + u32 mux_reg; + u8 mux_shift; + u8 mux_width; + u32 div_reg; + u8 div_shift; + u8 div_width; + u8 div_flags; + u32 gate_reg; + u8 gate_shift; + unsigned long flags; +}; + +#define COMPOSITE(_id, _name, _parent_names, \ + _mux_reg, _mux_shift, _mux_width, \ + _div_reg, _div_shift, _div_width, \ + _gate_reg, _gate_shift, _flags) \ +{ \ + .id = _id, \ + .name = _name, \ + .parent_names = _parent_names, \ + .num_parents = ARRAY_SIZE(_parent_names), \ + .mux_reg = _mux_reg, \ + .mux_shift = _mux_shift, \ + .mux_width = _mux_width, \ + .div_reg = _div_reg, \ + .div_shift = _div_shift, \ + .div_width = _div_width, \ + .div_flags = CLK_DIVIDER_HIWORD_MASK, \ + .gate_reg = _gate_reg, \ + .gate_shift = _gate_shift, \ + .flags = _flags, \ +} + +#define COMPOSITE_NOMUX(_id, _name, _parent_name, \ + _div_reg, _div_shift, _div_width, \ + _gate_reg, _gate_shift, _flags) \ +{ \ + .id = _id, \ + .name = _name, \ + .parent_names = (const char *[]){ _parent_name }, \ + .num_parents = 1, \ + .div_reg = _div_reg, \ + .div_shift = _div_shift, \ + .div_width = _div_width, \ + .div_flags = CLK_DIVIDER_HIWORD_MASK, \ + .gate_reg = _gate_reg, \ + .gate_shift = _gate_shift, \ + .flags = _flags, \ +} + +#define COMPOSITE_NODIV(_id, _name, _parent_names, \ + _mux_reg, _mux_shift, _mux_width, \ + _gate_reg, _gate_shift, _flags) \ + COMPOSITE(_id, _name, _parent_names, \ + _mux_reg, _mux_shift, _mux_width, \ + 0, 0, 0, \ + _gate_reg, _gate_shift, _flags) + +#define COMPOSITE_FRAC(_id, _name, _parent_names, \ + _mux_reg, _mux_shift, _mux_width, \ + _div_reg, \ + _gate_reg, _gate_shift, _flags) \ +{ \ + .id = _id, \ + .name = _name, \ + .parent_names = _parent_names, \ + .num_parents = ARRAY_SIZE(_parent_names), \ + .mux_reg = _mux_reg, \ + .mux_shift = _mux_shift, \ + .mux_width = _mux_width, \ + .div_reg = _div_reg, \ + .gate_reg = _gate_reg, \ + .gate_shift = _gate_shift, \ + .flags = _flags, \ +} + +#define COMPOSITE_FRAC_NOMUX(_id, _name, _parent_name, \ + _div_reg, \ + _gate_reg, _gate_shift, _flags) \ +{ \ + .id = _id, \ + .name = _name, \ + .parent_names = (const char *[]){ _parent_name }, \ + .num_parents = 1, \ + .div_reg = _div_reg, \ + .gate_reg = _gate_reg, \ + .gate_shift = _gate_shift, \ + .flags = _flags, \ +} + +#define COMPOSITE_FRAC_NOGATE(_id, _name, _parent_names, \ + _mux_reg, _mux_shift, _mux_width, \ + _div_reg, \ + _flags) \ + COMPOSITE_FRAC(_id, _name, _parent_names, \ + _mux_reg, _mux_shift, _mux_width, \ + _div_reg, 0, 0, _flags) + +struct clk_regmap_fractional_divider { + struct clk_hw hw; + struct device *dev; + struct regmap *regmap; + u32 reg; + u8 mshift; + u8 mwidth; + u32 mmask; + u8 nshift; + u8 nwidth; + u32 nmask; +}; + +struct clk_regmap_divider { + struct clk_hw hw; + struct device *dev; + struct regmap *regmap; + u32 reg; + u8 shift; + u8 width; +}; + +struct clk_regmap_gate { + struct clk_hw hw; + struct device *dev; + struct regmap *regmap; + u32 reg; + u8 shift; +}; + +struct clk_regmap_mux { + struct clk_hw hw; + struct device *dev; + struct regmap *regmap; + u32 reg; + u32 mask; + u8 shift; +}; + +extern const struct clk_ops clk_regmap_mux_ops; +extern const struct clk_ops clk_regmap_divider_ops; +extern const struct clk_ops clk_regmap_gate_ops; +extern const struct clk_ops clk_regmap_fractional_divider_ops; + +struct clk * +devm_clk_regmap_register_pll(struct device *dev, const char *name, + const char *parent_name, + struct regmap *regmap, u32 reg, u8 pd_shift, + u8 dsmpd_shift, u8 lock_shift, + unsigned long flags); + +struct clk * +devm_clk_regmap_register_mux(struct device *dev, const char *name, + const char * const *parent_names, u8 num_parents, + struct regmap *regmap, u32 reg, u8 shift, u8 width, + unsigned long flags); + +struct clk * +devm_clk_regmap_register_divider(struct device *dev, const char *name, + const char *parent_name, struct regmap *regmap, + u32 reg, u8 shift, u8 width, + unsigned long flags); + +struct clk * +devm_clk_regmap_register_gate(struct device *dev, const char *name, + const char *parent_name, + struct regmap *regmap, u32 reg, u8 shift, + unsigned long flags); + +struct clk * +devm_clk_regmap_register_fractional_divider(struct device *dev, + const char *name, + const char *parent_name, + struct regmap *regmap, + u32 reg, unsigned long flags); + +struct clk * +devm_clk_regmap_register_composite(struct device *dev, const char *name, + const char *const *parent_names, + u8 num_parents, struct regmap *regmap, + u32 mux_reg, u8 mux_shift, u8 mux_width, + u32 div_reg, u8 div_shift, u8 div_width, + u8 div_flags, + u32 gate_reg, u8 gate_shift, + unsigned long flags); + +#endif diff --git a/drivers/clk/rockchip/regmap/clk-rk618.c b/drivers/clk/rockchip/regmap/clk-rk618.c new file mode 100755 index 000000000000..c780f502b354 --- /dev/null +++ b/drivers/clk/rockchip/regmap/clk-rk618.c @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2017 Rockchip Electronics Co. Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include + +#include "clk-regmap.h" + +#define RK618_CRU_CLKSEL0 0x0058 +#define RK618_CRU_CLKSEL1 0x005c +#define RK618_CRU_CLKSEL2 0x0060 +#define RK618_CRU_CLKSEL3 0x0064 +#define RK618_CRU_PLL0_CON0 0x0068 +#define RK618_CRU_PLL0_CON1 0x006c +#define RK618_CRU_PLL0_CON2 0x0070 +#define RK618_CRU_PLL1_CON0 0x0074 +#define RK618_CRU_PLL1_CON1 0x0078 +#define RK618_CRU_PLL1_CON2 0x007c + +enum { + LCDC0_CLK = 1, + LCDC1_CLK, + VIF_PLLIN_CLK, + SCALER_PLLIN_CLK, + VIF_PLL_CLK, + SCALER_PLL_CLK, + VIF0_CLK, + VIF1_CLK, + SCALER_IN_CLK, + SCALER_CLK, + DITHER_CLK, + HDMI_CLK, + MIPI_CLK, + LVDS_CLK, + LVTTL_CLK, + RGB_CLK, + VIF0_PRE_CLK, + VIF1_PRE_CLK, + CODEC_CLK, + NR_CLKS, +}; + +struct rk618_cru { + struct device *dev; + struct rk618 *parent; + struct regmap *regmap; + + struct clk_onecell_data clk_data; +}; + +static char clkin_name[32] = "dummy"; +static char lcdc0_dclkp_name[32] = "dummy"; +static char lcdc1_dclkp_name[32] = "dummy"; + +#define PNAME(x) static const char *const x[] + +PNAME(mux_pll_in_p) = { "lcdc0_clk", "lcdc1_clk", clkin_name }; +PNAME(mux_pll_src_p) = { "vif_pll_clk", "scaler_pll_clk", }; +PNAME(mux_scaler_in_src_p) = { "vif0_clk", "vif1_clk" }; +PNAME(mux_hdmi_src_p) = { "vif1_clk", "scaler_clk", "vif0_clk" }; +PNAME(mux_dither_src_p) = { "vif0_clk", "scaler_clk" }; +PNAME(mux_vif0_src_p) = { "vif0_pre_clk", lcdc0_dclkp_name }; +PNAME(mux_vif1_src_p) = { "vif1_pre_clk", lcdc1_dclkp_name }; +PNAME(mux_codec_src_p) = { "codec_pre_clk", clkin_name }; + +/* Two PLL, one for dual datarate input logic, the other for scaler */ +static const struct clk_pll_data rk618_clk_plls[] = { + RK618_PLL(VIF_PLL_CLK, "vif_pll_clk", "vif_pllin_clk", + RK618_CRU_PLL0_CON0, + 0), + RK618_PLL(SCALER_PLL_CLK, "scaler_pll_clk", "scaler_pllin_clk", + RK618_CRU_PLL1_CON0, + 0), +}; + +static const struct clk_mux_data rk618_clk_muxes[] = { + MUX(VIF_PLLIN_CLK, "vif_pllin_clk", mux_pll_in_p, + RK618_CRU_CLKSEL0, 6, 2, + 0), + MUX(SCALER_PLLIN_CLK, "scaler_pllin_clk", mux_pll_in_p, + RK618_CRU_CLKSEL0, 8, 2, + 0), + MUX(SCALER_IN_CLK, "scaler_in_clk", mux_scaler_in_src_p, + RK618_CRU_CLKSEL3, 15, 1, + 0), + MUX(DITHER_CLK, "dither_clk", mux_dither_src_p, + RK618_CRU_CLKSEL3, 14, 1, + 0), + MUX(VIF0_CLK, "vif0_clk", mux_vif0_src_p, + RK618_CRU_CLKSEL3, 1, 1, + CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT), + MUX(VIF1_CLK, "vif1_clk", mux_vif1_src_p, + RK618_CRU_CLKSEL3, 7, 1, + CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT), + MUX(CODEC_CLK, "codec_clk", mux_codec_src_p, + RK618_CRU_CLKSEL1, 1, 1, + CLK_SET_RATE_PARENT), +}; + +static const struct clk_divider_data rk618_clk_dividers[] = { + DIV(LCDC0_CLK, "lcdc0_clk", lcdc0_dclkp_name, + RK618_CRU_CLKSEL0, 0, 3, + 0), + DIV(LCDC1_CLK, "lcdc1_clk", lcdc1_dclkp_name, + RK618_CRU_CLKSEL0, 3, 3, + 0), +}; + +static const struct clk_gate_data rk618_clk_gates[] = { + GATE(MIPI_CLK, "mipi_clk", "dither_clk", + RK618_CRU_CLKSEL1, 10, + CLK_IGNORE_UNUSED), + GATE(LVDS_CLK, "lvds_clk", "dither_clk", + RK618_CRU_CLKSEL1, 9, + CLK_IGNORE_UNUSED), + GATE(LVTTL_CLK, "lvttl_clk", "dither_clk", + RK618_CRU_CLKSEL1, 12, + 0), + GATE(RGB_CLK, "rgb_clk", "dither_clk", + RK618_CRU_CLKSEL1, 11, + 0), +}; + +static const struct clk_composite_data rk618_clk_composites[] = { + COMPOSITE(SCALER_CLK, "scaler_clk", mux_pll_src_p, + RK618_CRU_CLKSEL1, 3, 1, + RK618_CRU_CLKSEL1, 5, 3, + RK618_CRU_CLKSEL1, 4, + CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT), + COMPOSITE_NODIV(HDMI_CLK, "hdmi_clk", mux_hdmi_src_p, + RK618_CRU_CLKSEL3, 12, 2, + RK618_CRU_CLKSEL1, 8, + 0), + COMPOSITE(VIF0_PRE_CLK, "vif0_pre_clk", mux_pll_src_p, + RK618_CRU_CLKSEL3, 0, 1, + RK618_CRU_CLKSEL3, 3, 3, + RK618_CRU_CLKSEL3, 2, + CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT), + COMPOSITE(VIF1_PRE_CLK, "vif1_pre_clk", mux_pll_src_p, + RK618_CRU_CLKSEL3, 6, 1, + RK618_CRU_CLKSEL3, 9, 3, + RK618_CRU_CLKSEL3, 8, + CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT), + COMPOSITE_FRAC_NOGATE(0, "codec_pre_clk", mux_pll_src_p, + RK618_CRU_CLKSEL1, 0, 1, + RK618_CRU_CLKSEL2, + 0), +}; + +static void rk618_clk_add_lookup(struct rk618_cru *cru, struct clk *clk, + unsigned int id) +{ + if (cru->clk_data.clks && id) + cru->clk_data.clks[id] = clk; +} + +static void rk618_clk_register_muxes(struct rk618_cru *cru) +{ + struct clk *clk; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk618_clk_muxes); i++) { + const struct clk_mux_data *data = &rk618_clk_muxes[i]; + + clk = devm_clk_regmap_register_mux(cru->dev, data->name, + data->parent_names, + data->num_parents, + cru->regmap, data->reg, + data->shift, data->width, + data->flags); + if (IS_ERR(clk)) { + dev_err(cru->dev, "failed to register clock %s\n", + data->name); + continue; + } + + rk618_clk_add_lookup(cru, clk, data->id); + } +} + +static void rk618_clk_register_dividers(struct rk618_cru *cru) +{ + struct clk *clk; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk618_clk_dividers); i++) { + const struct clk_divider_data *data = &rk618_clk_dividers[i]; + + clk = devm_clk_regmap_register_divider(cru->dev, data->name, + data->parent_name, + cru->regmap, data->reg, + data->shift, data->width, + data->flags); + if (IS_ERR(clk)) { + dev_err(cru->dev, "failed to register clock %s\n", + data->name); + continue; + } + + rk618_clk_add_lookup(cru, clk, data->id); + } +} + +static void rk618_clk_register_gates(struct rk618_cru *cru) +{ + struct clk *clk; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk618_clk_gates); i++) { + const struct clk_gate_data *data = &rk618_clk_gates[i]; + + clk = devm_clk_regmap_register_gate(cru->dev, data->name, + data->parent_name, + cru->regmap, + data->reg, data->shift, + data->flags); + if (IS_ERR(clk)) { + dev_err(cru->dev, "failed to register clock %s\n", + data->name); + continue; + } + + rk618_clk_add_lookup(cru, clk, data->id); + } +} + +static void rk618_clk_register_composites(struct rk618_cru *cru) +{ + struct clk *clk; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk618_clk_composites); i++) { + const struct clk_composite_data *data = + &rk618_clk_composites[i]; + + clk = devm_clk_regmap_register_composite(cru->dev, data->name, + data->parent_names, + data->num_parents, + cru->regmap, + data->mux_reg, + data->mux_shift, + data->mux_width, + data->div_reg, + data->div_shift, + data->div_width, + data->div_flags, + data->gate_reg, + data->gate_shift, + data->flags); + if (IS_ERR(clk)) { + dev_err(cru->dev, "failed to register clock %s\n", + data->name); + continue; + } + + rk618_clk_add_lookup(cru, clk, data->id); + } +} + +static void rk618_clk_register_plls(struct rk618_cru *cru) +{ + struct clk *clk; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk618_clk_plls); i++) { + const struct clk_pll_data *data = &rk618_clk_plls[i]; + + clk = devm_clk_regmap_register_pll(cru->dev, data->name, + data->parent_name, + cru->regmap, + data->reg, + data->pd_shift, + data->dsmpd_shift, + data->lock_shift, + data->flags); + if (IS_ERR(clk)) { + dev_err(cru->dev, "failed to register clock %s\n", + data->name); + continue; + } + + rk618_clk_add_lookup(cru, clk, data->id); + } +} + +static int rk618_cru_probe(struct platform_device *pdev) +{ + struct rk618 *rk618 = dev_get_drvdata(pdev->dev.parent); + struct device *dev = &pdev->dev; + struct rk618_cru *cru; + struct clk **clk_table; + const char *parent_name; + struct clk *clk; + int ret, i; + + if (!of_device_is_available(dev->of_node)) + return -ENODEV; + + cru = devm_kzalloc(dev, sizeof(*cru), GFP_KERNEL); + if (!cru) + return -ENOMEM; + + clk_table = devm_kcalloc(dev, NR_CLKS, sizeof(struct clk *), + GFP_KERNEL); + if (!clk_table) + return -ENOMEM; + + for (i = 0; i < NR_CLKS; i++) + clk_table[i] = ERR_PTR(-ENOENT); + + cru->dev = dev; + cru->parent = rk618; + cru->regmap = rk618->regmap; + cru->clk_data.clks = clk_table; + cru->clk_data.clk_num = NR_CLKS; + platform_set_drvdata(pdev, cru); + + clk = devm_clk_get(dev, "clkin"); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(dev, "failed to get clkin: %d\n", ret); + return ret; + } + + strlcpy(clkin_name, __clk_get_name(clk), sizeof(clkin_name)); + + clk = devm_clk_get(dev, "lcdc0_dclkp"); + if (IS_ERR(clk)) { + if (PTR_ERR(clk) != -ENOENT) { + ret = PTR_ERR(clk); + dev_err(dev, "failed to get lcdc0_dclkp: %d\n", ret); + return ret; + } + + clk = NULL; + } + + parent_name = __clk_get_name(clk); + if (parent_name) + strlcpy(lcdc0_dclkp_name, parent_name, + sizeof(lcdc0_dclkp_name)); + + clk = devm_clk_get(dev, "lcdc1_dclkp"); + if (IS_ERR(clk)) { + if (PTR_ERR(clk) != -ENOENT) { + ret = PTR_ERR(clk); + dev_err(dev, "failed to get lcdc1_dclkp: %d\n", ret); + return ret; + } + + clk = NULL; + } + + parent_name = __clk_get_name(clk); + if (parent_name) + strlcpy(lcdc1_dclkp_name, parent_name, + sizeof(lcdc1_dclkp_name)); + + rk618_clk_register_plls(cru); + rk618_clk_register_muxes(cru); + rk618_clk_register_dividers(cru); + rk618_clk_register_gates(cru); + rk618_clk_register_composites(cru); + + return of_clk_add_provider(dev->of_node, of_clk_src_onecell_get, + &cru->clk_data); +} + +static int rk618_cru_remove(struct platform_device *pdev) +{ + of_clk_del_provider(pdev->dev.of_node); + + return 0; +} + +static const struct of_device_id rk618_cru_of_match[] = { + { .compatible = "rockchip,rk618-cru", }, + {}, +}; +MODULE_DEVICE_TABLE(of, rk618_cru_of_match); + +static struct platform_driver rk618_cru_driver = { + .driver = { + .name = "rk618-cru", + .of_match_table = of_match_ptr(rk618_cru_of_match), + }, + .probe = rk618_cru_probe, + .remove = rk618_cru_remove, +}; +module_platform_driver(rk618_cru_driver); + +MODULE_AUTHOR("Wyon Bi "); +MODULE_DESCRIPTION("Rockchip rk618 CRU driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/rockchip/regmap/clk-rk628.c b/drivers/clk/rockchip/regmap/clk-rk628.c new file mode 100755 index 000000000000..4c3a9eac0e0c --- /dev/null +++ b/drivers/clk/rockchip/regmap/clk-rk628.c @@ -0,0 +1,569 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 Rockchip Electronics Co. Ltd. + * + * Author: Wyon Bi + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clk-regmap.h" + +#define REG(x) ((x) + 0xc0000) + +#define CRU_CPLL_CON0 REG(0x0000) +#define CRU_CPLL_CON1 REG(0x0004) +#define CRU_CPLL_CON2 REG(0x0008) +#define CRU_CPLL_CON3 REG(0x000c) +#define CRU_CPLL_CON4 REG(0x0010) +#define CRU_GPLL_CON0 REG(0x0020) +#define CRU_GPLL_CON1 REG(0x0024) +#define CRU_GPLL_CON2 REG(0x0028) +#define CRU_GPLL_CON3 REG(0x002c) +#define CRU_GPLL_CON4 REG(0x0030) +#define CRU_MODE_CON REG(0x0060) +#define CRU_CLKSEL_CON00 REG(0x0080) +#define CRU_CLKSEL_CON01 REG(0x0084) +#define CRU_CLKSEL_CON02 REG(0x0088) +#define CRU_CLKSEL_CON03 REG(0x008c) +#define CRU_CLKSEL_CON04 REG(0x0090) +#define CRU_CLKSEL_CON05 REG(0x0094) +#define CRU_CLKSEL_CON06 REG(0x0098) +#define CRU_CLKSEL_CON07 REG(0x009c) +#define CRU_CLKSEL_CON08 REG(0x00a0) +#define CRU_CLKSEL_CON09 REG(0x00a4) +#define CRU_CLKSEL_CON10 REG(0x00a8) +#define CRU_CLKSEL_CON11 REG(0x00ac) +#define CRU_CLKSEL_CON12 REG(0x00b0) +#define CRU_CLKSEL_CON13 REG(0x00b4) +#define CRU_CLKSEL_CON14 REG(0x00b8) +#define CRU_CLKSEL_CON15 REG(0x00bc) +#define CRU_CLKSEL_CON16 REG(0x00c0) +#define CRU_CLKSEL_CON17 REG(0x00c4) +#define CRU_CLKSEL_CON18 REG(0x00c8) +#define CRU_CLKSEL_CON20 REG(0x00d0) +#define CRU_CLKSEL_CON21 REG(0x00d4) +#define CRU_GATE_CON00 REG(0x0180) +#define CRU_GATE_CON01 REG(0x0184) +#define CRU_GATE_CON02 REG(0x0188) +#define CRU_GATE_CON03 REG(0x018c) +#define CRU_GATE_CON04 REG(0x0190) +#define CRU_GATE_CON05 REG(0x0194) +#define CRU_SOFTRST_CON00 REG(0x0200) +#define CRU_SOFTRST_CON01 REG(0x0204) +#define CRU_SOFTRST_CON02 REG(0x0208) +#define CRU_SOFTRST_CON04 REG(0x0210) +#define CRU_MAX_REGISTER CRU_SOFTRST_CON04 + +#define reset_to_cru(_rst) container_of(_rst, struct rk628_cru, rcdev) + +struct rk628_cru { + struct device *dev; + struct rk628 *parent; + struct regmap *regmap; + struct reset_controller_dev rcdev; + struct clk_onecell_data clk_data; +}; + +#define CNAME(x) "rk628_" x + +#define PNAME(x) static const char *const x[] + +PNAME(mux_cpll_osc_p) = { "xin_osc0_func", CNAME("clk_cpll") }; +PNAME(mux_gpll_osc_p) = { "xin_osc0_func", CNAME("clk_gpll") }; +PNAME(mux_cpll_gpll_mux_p) = { CNAME("clk_cpll_mux"), CNAME("clk_gpll_mux") }; +PNAME(mux_mclk_i2s_8ch_p) = { CNAME("clk_i2s_8ch_src"), CNAME("clk_i2s_8ch_frac"), "i2s_mclkin", "xin_osc0_half" }; +PNAME(mux_i2s_mclkout_p) = { CNAME("mclk_i2s_8ch"), "xin_osc0_half" }; + +static const struct clk_pll_data rk628_clk_plls[] = { + RK628_PLL(CGU_CLK_CPLL, CNAME("clk_cpll"), "xin_osc0_func", + CRU_CPLL_CON0, + 0), + RK628_PLL(CGU_CLK_GPLL, CNAME("clk_gpll"), "xin_osc0_func", + CRU_GPLL_CON0, + 0), +}; + +static const struct clk_mux_data rk628_clk_muxes[] = { + MUX(CGU_CLK_CPLL_MUX, CNAME("clk_cpll_mux"), mux_cpll_osc_p, + CRU_MODE_CON, 0, 1, + 0), + MUX(CGU_CLK_GPLL_MUX, CNAME("clk_gpll_mux"), mux_gpll_osc_p, + CRU_MODE_CON, 2, 1, + CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT), +}; + +static const struct clk_gate_data rk628_clk_gates[] = { + GATE(CGU_PCLK_GPIO0, CNAME("pclk_gpio0"), CNAME("pclk_logic"), + CRU_GATE_CON01, 0, + 0), + GATE(CGU_PCLK_GPIO1, CNAME("pclk_gpio1"), CNAME("pclk_logic"), + CRU_GATE_CON01, 1, + 0), + GATE(CGU_PCLK_GPIO2, CNAME("pclk_gpio2"), CNAME("pclk_logic"), + CRU_GATE_CON01, 2, + 0), + GATE(CGU_PCLK_GPIO3, CNAME("pclk_gpio3"), CNAME("pclk_logic"), + CRU_GATE_CON01, 3, + 0), + + GATE(CGU_PCLK_TXPHY_CON, CNAME("pclk_txphy_con"), CNAME("pclk_logic"), + CRU_GATE_CON02, 3, + CLK_IGNORE_UNUSED), + GATE(CGU_PCLK_EFUSE, CNAME("pclk_efuse"), CNAME("pclk_logic"), + CRU_GATE_CON00, 5, + 0), + GATE(0, CNAME("pclk_i2c2apb"), CNAME("pclk_logic"), + CRU_GATE_CON00, 3, + CLK_IGNORE_UNUSED), + GATE(0, CNAME("pclk_cru"), CNAME("pclk_logic"), + CRU_GATE_CON00, 1, + CLK_IGNORE_UNUSED), + GATE(0, CNAME("pclk_adapter"), CNAME("pclk_logic"), + CRU_GATE_CON00, 7, + CLK_IGNORE_UNUSED), + GATE(0, CNAME("pclk_regfile"), CNAME("pclk_logic"), + CRU_GATE_CON00, 2, + CLK_IGNORE_UNUSED), + GATE(CGU_PCLK_DSI0, CNAME("pclk_dsi0"), CNAME("pclk_logic"), + CRU_GATE_CON02, 6, + 0), + GATE(CGU_PCLK_DSI1, CNAME("pclk_dsi1"), CNAME("pclk_logic"), + CRU_GATE_CON02, 7, + 0), + GATE(CGU_PCLK_CSI, CNAME("pclk_csi"), CNAME("pclk_logic"), + CRU_GATE_CON02, 8, + 0), + GATE(CGU_PCLK_HDMITX, CNAME("pclk_hdmitx"), CNAME("pclk_logic"), + CRU_GATE_CON02, 4, + 0), + GATE(CGU_PCLK_RXPHY, CNAME("pclk_rxphy"), CNAME("pclk_logic"), + CRU_GATE_CON02, 0, + 0), + GATE(CGU_PCLK_HDMIRX, CNAME("pclk_hdmirx"), CNAME("pclk_logic"), + CRU_GATE_CON02, 2, + 0), + GATE(CGU_PCLK_GVIHOST, CNAME("pclk_gvihost"), CNAME("pclk_logic"), + CRU_GATE_CON02, 5, + 0), + GATE(CGU_CLK_CFG_DPHY0, CNAME("clk_cfg_dphy0"), "xin_osc0_func", + CRU_GATE_CON02, 13, + 0), + GATE(CGU_CLK_CFG_DPHY1, CNAME("clk_cfg_dphy1"), "xin_osc0_func", + CRU_GATE_CON02, 14, + 0), + GATE(CGU_CLK_TXESC, CNAME("clk_txesc"), "xin_osc0_func", + CRU_GATE_CON02, 12, + 0), +}; + +static const struct clk_composite_data rk628_clk_composites[] = { + COMPOSITE(CGU_CLK_IMODET, CNAME("clk_imodet"), mux_cpll_gpll_mux_p, + CRU_CLKSEL_CON05, 5, 1, + CRU_CLKSEL_CON05, 0, 5, + CRU_GATE_CON02, 11, + 0), + COMPOSITE(CGU_CLK_HDMIRX_AUD, CNAME("clk_hdmirx_aud"), mux_cpll_gpll_mux_p, + CRU_CLKSEL_CON05, 15, 1, + CRU_CLKSEL_CON05, 6, 8, + CRU_GATE_CON02, 10, + CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT), + COMPOSITE_FRAC_NOMUX(CGU_CLK_HDMIRX_CEC, CNAME("clk_hdmirx_cec"), "xin_osc0_func", + CRU_CLKSEL_CON12, + CRU_GATE_CON01, 15, + 0), + COMPOSITE_FRAC(CGU_CLK_RX_READ, CNAME("clk_rx_read"), mux_cpll_gpll_mux_p, + CRU_CLKSEL_CON02, 8, 1, + CRU_CLKSEL_CON14, + CRU_GATE_CON00, 11, + 0), + COMPOSITE_FRAC(CGU_SCLK_VOP, CNAME("sclk_vop"), mux_cpll_gpll_mux_p, + CRU_CLKSEL_CON02, 9, 1, + CRU_CLKSEL_CON13, + CRU_GATE_CON00, 13, + CLK_SET_RATE_NO_REPARENT), + COMPOSITE(CGU_PCLK_LOGIC, CNAME("pclk_logic"), mux_cpll_gpll_mux_p, + CRU_CLKSEL_CON00, 7, 1, + CRU_CLKSEL_CON00, 0, 5, + CRU_GATE_CON00, 0, + 0), + COMPOSITE_NOMUX(CGU_CLK_GPIO_DB0, CNAME("clk_gpio_db0"), "xin_osc0_func", + CRU_CLKSEL_CON08, 0, 10, + CRU_GATE_CON01, 4, + 0), + COMPOSITE_NOMUX(CGU_CLK_GPIO_DB1, CNAME("clk_gpio_db1"), "xin_osc0_func", + CRU_CLKSEL_CON09, 0, 10, + CRU_GATE_CON01, 5, + 0), + COMPOSITE_NOMUX(CGU_CLK_GPIO_DB2, CNAME("clk_gpio_db2"), "xin_osc0_func", + CRU_CLKSEL_CON10, 0, 10, + CRU_GATE_CON01, 6, + 0), + COMPOSITE_NOMUX(CGU_CLK_GPIO_DB3, CNAME("clk_gpio_db3"), "xin_osc0_func", + CRU_CLKSEL_CON11, 0, 10, + CRU_GATE_CON01, 7, + 0), + COMPOSITE(CGU_CLK_I2S_8CH_SRC, CNAME("clk_i2s_8ch_src"), mux_cpll_gpll_mux_p, + CRU_CLKSEL_CON03, 13, 1, + CRU_CLKSEL_CON03, 8, 5, + CRU_GATE_CON03, 9, + 0), + COMPOSITE_FRAC_NOMUX(CGU_CLK_I2S_8CH_FRAC, CNAME("clk_i2s_8ch_frac"), CNAME("clk_i2s_8ch_src"), + CRU_CLKSEL_CON04, + CRU_GATE_CON03, 10, + 0), + COMPOSITE_NODIV(CGU_MCLK_I2S_8CH, CNAME("mclk_i2s_8ch"), mux_mclk_i2s_8ch_p, + CRU_CLKSEL_CON03, 14, 2, + CRU_GATE_CON03, 11, + CLK_SET_RATE_PARENT), + COMPOSITE_NODIV(CGU_I2S_MCLKOUT, CNAME("i2s_mclkout"), mux_i2s_mclkout_p, + CRU_CLKSEL_CON03, 7, 1, + CRU_GATE_CON03, 12, + CLK_SET_RATE_PARENT), + COMPOSITE(CGU_BT1120DEC, CNAME("clk_bt1120dec"), mux_cpll_gpll_mux_p, + CRU_CLKSEL_CON02, 7, 1, + CRU_CLKSEL_CON02, 0, 5, + CRU_GATE_CON00, 12, + 0), +}; + +static void rk628_clk_add_lookup(struct rk628_cru *cru, struct clk *clk, + unsigned int id) +{ + if (cru->clk_data.clks && id) + cru->clk_data.clks[id] = clk; +} + +static void rk628_clk_register_muxes(struct rk628_cru *cru) +{ + struct clk *clk; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk628_clk_muxes); i++) { + const struct clk_mux_data *data = &rk628_clk_muxes[i]; + + clk = devm_clk_regmap_register_mux(cru->dev, data->name, + data->parent_names, + data->num_parents, + cru->regmap, data->reg, + data->shift, data->width, + data->flags); + if (IS_ERR(clk)) { + dev_err(cru->dev, "failed to register clock %s\n", + data->name); + continue; + } + + rk628_clk_add_lookup(cru, clk, data->id); + } +} + +static void rk628_clk_register_gates(struct rk628_cru *cru) +{ + struct clk *clk; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk628_clk_gates); i++) { + const struct clk_gate_data *data = &rk628_clk_gates[i]; + + clk = devm_clk_regmap_register_gate(cru->dev, data->name, + data->parent_name, + cru->regmap, + data->reg, data->shift, + data->flags); + if (IS_ERR(clk)) { + dev_err(cru->dev, "failed to register clock %s\n", + data->name); + continue; + } + + rk628_clk_add_lookup(cru, clk, data->id); + } +} + +static void rk628_clk_register_composites(struct rk628_cru *cru) +{ + struct clk *clk; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk628_clk_composites); i++) { + const struct clk_composite_data *data = + &rk628_clk_composites[i]; + + clk = devm_clk_regmap_register_composite(cru->dev, data->name, + data->parent_names, + data->num_parents, + cru->regmap, + data->mux_reg, + data->mux_shift, + data->mux_width, + data->div_reg, + data->div_shift, + data->div_width, + data->div_flags, + data->gate_reg, + data->gate_shift, + data->flags); + if (IS_ERR(clk)) { + dev_err(cru->dev, "failed to register clock %s\n", + data->name); + continue; + } + + rk628_clk_add_lookup(cru, clk, data->id); + } +} + +static void rk628_clk_register_plls(struct rk628_cru *cru) +{ + struct clk *clk; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk628_clk_plls); i++) { + const struct clk_pll_data *data = &rk628_clk_plls[i]; + + clk = devm_clk_regmap_register_pll(cru->dev, data->name, + data->parent_name, + cru->regmap, + data->reg, + data->pd_shift, + data->dsmpd_shift, + data->lock_shift, + data->flags); + if (IS_ERR(clk)) { + dev_err(cru->dev, "failed to register clock %s\n", + data->name); + continue; + } + + rk628_clk_add_lookup(cru, clk, data->id); + } +} + +struct rk628_rgu_data { + unsigned int id; + unsigned int reg; + unsigned int bit; +}; + +#define RSTGEN(_id, _reg, _bit) \ + { \ + .id = (_id), \ + .reg = (_reg), \ + .bit = (_bit), \ + } + +static const struct rk628_rgu_data rk628_rgu_data[] = { + RSTGEN(RGU_LOGIC, CRU_SOFTRST_CON00, 0), + RSTGEN(RGU_CRU, CRU_SOFTRST_CON00, 1), + RSTGEN(RGU_REGFILE, CRU_SOFTRST_CON00, 2), + RSTGEN(RGU_I2C2APB, CRU_SOFTRST_CON00, 3), + RSTGEN(RGU_EFUSE, CRU_SOFTRST_CON00, 5), + RSTGEN(RGU_ADAPTER, CRU_SOFTRST_CON00, 7), + RSTGEN(RGU_CLK_RX, CRU_SOFTRST_CON00, 11), + RSTGEN(RGU_BT1120DEC, CRU_SOFTRST_CON00, 12), + RSTGEN(RGU_VOP, CRU_SOFTRST_CON00, 13), + + RSTGEN(RGU_GPIO0, CRU_SOFTRST_CON01, 0), + RSTGEN(RGU_GPIO1, CRU_SOFTRST_CON01, 1), + RSTGEN(RGU_GPIO2, CRU_SOFTRST_CON01, 2), + RSTGEN(RGU_GPIO3, CRU_SOFTRST_CON01, 3), + RSTGEN(RGU_GPIO_DB0, CRU_SOFTRST_CON01, 4), + RSTGEN(RGU_GPIO_DB1, CRU_SOFTRST_CON01, 5), + RSTGEN(RGU_GPIO_DB2, CRU_SOFTRST_CON01, 6), + RSTGEN(RGU_GPIO_DB3, CRU_SOFTRST_CON01, 7), + + RSTGEN(RGU_RXPHY, CRU_SOFTRST_CON02, 0), + RSTGEN(RGU_HDMIRX, CRU_SOFTRST_CON02, 2), + RSTGEN(RGU_TXPHY_CON, CRU_SOFTRST_CON02, 3), + RSTGEN(RGU_HDMITX, CRU_SOFTRST_CON02, 4), + RSTGEN(RGU_GVIHOST, CRU_SOFTRST_CON02, 5), + RSTGEN(RGU_DSI0, CRU_SOFTRST_CON02, 6), + RSTGEN(RGU_DSI1, CRU_SOFTRST_CON02, 7), + RSTGEN(RGU_CSI, CRU_SOFTRST_CON02, 8), + RSTGEN(RGU_TXDATA, CRU_SOFTRST_CON02, 9), + RSTGEN(RGU_DECODER, CRU_SOFTRST_CON02, 10), + RSTGEN(RGU_ENCODER, CRU_SOFTRST_CON02, 11), + RSTGEN(RGU_HDMIRX_PON, CRU_SOFTRST_CON02, 12), + RSTGEN(RGU_TXBYTEHS, CRU_SOFTRST_CON02, 13), + RSTGEN(RGU_TXESC, CRU_SOFTRST_CON02, 14), +}; + +static int rk628_rgu_update(struct rk628_cru *cru, unsigned long id, int assert) +{ + const struct rk628_rgu_data *data = &rk628_rgu_data[id]; + + return regmap_write(cru->regmap, data->reg, + BIT(data->bit + 16) | (assert << data->bit)); +} + +static int rk628_rgu_assert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct rk628_cru *cru = reset_to_cru(rcdev); + + return rk628_rgu_update(cru, id, 1); +} + +static int rk628_rgu_deassert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct rk628_cru *cru = reset_to_cru(rcdev); + + return rk628_rgu_update(cru, id, 0); +} + +static struct reset_control_ops rk628_rgu_ops = { + .assert = rk628_rgu_assert, + .deassert = rk628_rgu_deassert, +}; + +static int rk628_reset_controller_register(struct rk628_cru *cru) +{ + struct device *dev = cru->dev; + + cru->rcdev.owner = THIS_MODULE; + cru->rcdev.nr_resets = ARRAY_SIZE(rk628_rgu_data); + cru->rcdev.of_node = dev->of_node; + cru->rcdev.ops = &rk628_rgu_ops; + + return reset_controller_register(&cru->rcdev); +} + +static const struct regmap_range rk628_cru_readable_ranges[] = { + regmap_reg_range(CRU_CPLL_CON0, CRU_CPLL_CON4), + regmap_reg_range(CRU_GPLL_CON0, CRU_GPLL_CON4), + regmap_reg_range(CRU_MODE_CON, CRU_MODE_CON), + regmap_reg_range(CRU_CLKSEL_CON00, CRU_CLKSEL_CON21), + regmap_reg_range(CRU_GATE_CON00, CRU_GATE_CON05), + regmap_reg_range(CRU_SOFTRST_CON00, CRU_SOFTRST_CON04), +}; + +static const struct regmap_access_table rk628_cru_readable_table = { + .yes_ranges = rk628_cru_readable_ranges, + .n_yes_ranges = ARRAY_SIZE(rk628_cru_readable_ranges), +}; + +static const struct regmap_config rk628_cru_regmap_config = { + .name = "cru", + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .max_register = CRU_MAX_REGISTER, + .reg_format_endian = REGMAP_ENDIAN_LITTLE, + .val_format_endian = REGMAP_ENDIAN_LITTLE, + .rd_table = &rk628_cru_readable_table, +}; + +static int rk628_cru_probe(struct platform_device *pdev) +{ + struct rk628 *rk628 = dev_get_drvdata(pdev->dev.parent); + struct device *dev = &pdev->dev; + struct rk628_cru *cru; + struct clk **clk_table; + unsigned int i; + int ret; + + cru = devm_kzalloc(dev, sizeof(*cru), GFP_KERNEL); + if (!cru) + return -ENOMEM; + + cru->regmap = devm_regmap_init_i2c(rk628->client, + &rk628_cru_regmap_config); + if (IS_ERR(cru->regmap)) { + ret = PTR_ERR(cru->regmap); + dev_err(dev, "failed to allocate register map: %d\n", ret); + return ret; + } + + /* clock switch and first set gpll almost 99MHz */ + regmap_write(cru->regmap, CRU_GPLL_CON0, 0xffff701d); + usleep_range(1000, 1100); + /* set clk_gpll_mux from gpll */ + regmap_write(cru->regmap, CRU_MODE_CON, 0xffff0004); + usleep_range(1000, 1100); + /* set pclk_logic from clk_gpll_mux and set pclk div 4 */ + regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff0080); + regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff0083); + /* set cpll almost 400MHz */ + regmap_write(cru->regmap, CRU_CPLL_CON0, 0xffff3063); + usleep_range(1000, 1100); + /* set clk_cpll_mux from clk_cpll */ + regmap_write(cru->regmap, CRU_MODE_CON, 0xffff0005); + /* set pclk use cpll, now div is 4 */ + regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff0003); + /* set pclk use cpll, now div is 12 */ + regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff000b); + /* gpll 983.04MHz */ + regmap_write(cru->regmap, CRU_GPLL_CON0, 0xffff1028); + usleep_range(1000, 1100); + /* set pclk use gpll, nuw div is 0xb */ + regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff008b); + /* set cpll 1188MHz */ + regmap_write(cru->regmap, CRU_CPLL_CON0, 0xffff1063); + usleep_range(1000, 1100); + /* set pclk use cpll, and set pclk 99MHz */ + regmap_write(cru->regmap, CRU_CLKSEL_CON00, 0xff000b); + + clk_table = devm_kcalloc(dev, CGU_NR_CLKS, sizeof(struct clk *), + GFP_KERNEL); + if (!clk_table) + return -ENOMEM; + + for (i = 0; i < CGU_NR_CLKS; i++) + clk_table[i] = ERR_PTR(-ENOENT); + + cru->dev = dev; + cru->parent = rk628; + cru->clk_data.clks = clk_table; + cru->clk_data.clk_num = CGU_NR_CLKS; + platform_set_drvdata(pdev, cru); + + rk628_clk_register_plls(cru); + rk628_clk_register_muxes(cru); + rk628_clk_register_gates(cru); + rk628_clk_register_composites(cru); + rk628_reset_controller_register(cru); + + clk_prepare_enable(clk_table[CGU_PCLK_LOGIC]); + + return of_clk_add_provider(dev->of_node, of_clk_src_onecell_get, + &cru->clk_data); +} + +static int rk628_cru_remove(struct platform_device *pdev) +{ + struct rk628_cru *cru = dev_get_drvdata(&pdev->dev); + + of_clk_del_provider(pdev->dev.of_node); + reset_controller_unregister(&cru->rcdev); + + return 0; +} + +static const struct of_device_id rk628_cru_of_match[] = { + { .compatible = "rockchip,rk628-cru", }, + {}, +}; +MODULE_DEVICE_TABLE(of, rk628_cru_of_match); + +static struct platform_driver rk628_cru_driver = { + .driver = { + .name = "rk628-cru", + .of_match_table = of_match_ptr(rk628_cru_of_match), + }, + .probe = rk628_cru_probe, + .remove = rk628_cru_remove, +}; +module_platform_driver(rk628_cru_driver); + +MODULE_AUTHOR("Wyon Bi "); +MODULE_DESCRIPTION("Rockchip RK628 CRU driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index a0c6e88bebe0..9d9cb5757913 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -85,7 +85,9 @@ config IXP4XX_TIMER Enables support for the Intel XScale IXP4xx SoC timer. config ROCKCHIP_TIMER - bool "Rockchip timer driver" if COMPILE_TEST + tristate "Rockchip timer driver" + default ARCH_ROCKCHIP + depends on ARCH_ROCKCHIP || COMPILE_TEST depends on ARM || ARM64 select TIMER_OF select CLKSRC_MMIO diff --git a/drivers/clocksource/timer-rockchip.c b/drivers/clocksource/timer-rockchip.c index 1f95d0aca08f..2f4e970d7433 100644 --- a/drivers/clocksource/timer-rockchip.c +++ b/drivers/clocksource/timer-rockchip.c @@ -8,11 +8,13 @@ #include #include #include +#include #include #include #include #include #include +#include #define TIMER_NAME "rk_timer" @@ -45,7 +47,9 @@ struct rk_clkevt { }; static struct rk_clkevt *rk_clkevt; +#ifndef MODULE static struct rk_timer *rk_clksrc; +#endif static inline struct rk_timer *rk_timer(struct clock_event_device *ce) { @@ -119,10 +123,12 @@ static irqreturn_t rk_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } +#ifndef MODULE static u64 notrace rk_timer_sched_read(void) { return ~readl_relaxed(rk_clksrc->base + TIMER_CURRENT_VALUE0); } +#endif static int __init rk_timer_probe(struct rk_timer *timer, struct device_node *np) @@ -250,6 +256,7 @@ static int __init rk_clkevt_init(struct device_node *np) return ret; } +#ifndef MODULE static int __init rk_clksrc_init(struct device_node *np) { int ret = -EINVAL; @@ -287,14 +294,17 @@ static int __init rk_clksrc_init(struct device_node *np) rk_clksrc = ERR_PTR(ret); return ret; } +#endif static int __init rk_timer_init(struct device_node *np) { if (!rk_clkevt) return rk_clkevt_init(np); +#ifndef MODULE if (!rk_clksrc) return rk_clksrc_init(np); +#endif pr_err("Too many timer definitions for '%s'\n", TIMER_NAME); return -EINVAL; @@ -302,3 +312,26 @@ static int __init rk_timer_init(struct device_node *np) TIMER_OF_DECLARE(rk3288_timer, "rockchip,rk3288-timer", rk_timer_init); TIMER_OF_DECLARE(rk3399_timer, "rockchip,rk3399-timer", rk_timer_init); + +#ifdef MODULE +static int __init rk_timer_driver_probe(struct platform_device *pdev) +{ + return rk_timer_init(pdev->dev.of_node); +} + +static const struct of_device_id rk_timer_match_table[] = { + { .compatible = "rockchip,rk3288-timer" }, + { .compatible = "rockchip,rk3399-timer" }, + { /* sentinel */ }, +}; + +static struct platform_driver rk_timer_driver = { + .driver = { + .name = TIMER_NAME, + .of_match_table = rk_timer_match_table, + }, +}; +module_platform_driver_probe(rk_timer_driver, rk_timer_driver_probe); + +MODULE_LICENSE("GPL"); +#endif diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm index 1f73fa75b1a0..0faef5fc6df3 100644 --- a/drivers/cpufreq/Kconfig.arm +++ b/drivers/cpufreq/Kconfig.arm @@ -158,6 +158,16 @@ config ARM_RASPBERRYPI_CPUFREQ If in doubt, say N. +config ARM_ROCKCHIP_CPUFREQ + tristate "Rockchip CPUfreq driver" + depends on ARCH_ROCKCHIP && CPUFREQ_DT + select PM_OPP + help + This adds the CPUFreq driver support for Rockchip SoCs, + based on cpufreq-dt. + + If in doubt, say N. + config ARM_S3C_CPUFREQ bool help diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index f1b7e3dd6e5d..2e0e827afd21 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -64,6 +64,7 @@ obj-$(CONFIG_PXA3xx) += pxa3xx-cpufreq.o obj-$(CONFIG_ARM_QCOM_CPUFREQ_HW) += qcom-cpufreq-hw.o obj-$(CONFIG_ARM_QCOM_CPUFREQ_NVMEM) += qcom-cpufreq-nvmem.o obj-$(CONFIG_ARM_RASPBERRYPI_CPUFREQ) += raspberrypi-cpufreq.o +obj-$(CONFIG_ARM_ROCKCHIP_CPUFREQ) += rockchip-cpufreq.o obj-$(CONFIG_ARM_S3C2410_CPUFREQ) += s3c2410-cpufreq.o obj-$(CONFIG_ARM_S3C2412_CPUFREQ) += s3c2412-cpufreq.o obj-$(CONFIG_ARM_S3C2416_CPUFREQ) += s3c2416-cpufreq.o diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index aea285651fba..1840409d520d 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -66,21 +66,6 @@ static const struct of_device_id whitelist[] __initconst = { { .compatible = "renesas,r8a7794", }, { .compatible = "renesas,sh73a0", }, - { .compatible = "rockchip,rk2928", }, - { .compatible = "rockchip,rk3036", }, - { .compatible = "rockchip,rk3066a", }, - { .compatible = "rockchip,rk3066b", }, - { .compatible = "rockchip,rk3188", }, - { .compatible = "rockchip,rk3228", }, - { .compatible = "rockchip,rk3288", }, - { .compatible = "rockchip,rk3328", }, - { .compatible = "rockchip,rk3366", }, - { .compatible = "rockchip,rk3368", }, - { .compatible = "rockchip,rk3399", - .data = &(struct cpufreq_dt_platform_data) - { .have_governor_per_policy = true, }, - }, - { .compatible = "st-ericsson,u8500", }, { .compatible = "st-ericsson,u8540", }, { .compatible = "st-ericsson,u9500", }, @@ -138,6 +123,28 @@ static const struct of_device_id blacklist[] __initconst = { { .compatible = "qcom,sc7180", }, { .compatible = "qcom,sdm845", }, { .compatible = "qcom,sm8150", }, + { .compatible = "rockchip,px30", }, + { .compatible = "rockchip,rk2928", }, + { .compatible = "rockchip,rk3036", }, + { .compatible = "rockchip,rk3066a", }, + { .compatible = "rockchip,rk3066b", }, + { .compatible = "rockchip,rk3126", }, + { .compatible = "rockchip,rk3128", }, + { .compatible = "rockchip,rk3188", }, + { .compatible = "rockchip,rk3228", }, + { .compatible = "rockchip,rk3229", }, + { .compatible = "rockchip,rk3288", }, + { .compatible = "rockchip,rk3288w", }, + { .compatible = "rockchip,rk3326", }, + { .compatible = "rockchip,rk3328", }, + { .compatible = "rockchip,rk3366", }, + { .compatible = "rockchip,rk3368", }, + { .compatible = "rockchip,rk3399", }, + { .compatible = "rockchip,rk3399pro", }, + { .compatible = "rockchip,rk3566", }, + { .compatible = "rockchip,rk3568", }, + { .compatible = "rockchip,rv1109", }, + { .compatible = "rockchip,rv1126", }, { .compatible = "st,stih407", }, { .compatible = "st,stih410", }, diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index e363ae04aac6..f1327e7fe361 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -23,6 +23,9 @@ #include #include "cpufreq-dt.h" +#ifdef CONFIG_ARCH_ROCKCHIP +#include "rockchip-cpufreq.h" +#endif struct private_data { struct list_head node; @@ -30,7 +33,7 @@ struct private_data { cpumask_var_t cpus; struct device *cpu_dev; struct opp_table *opp_table; - struct opp_table *reg_opp_table; + struct cpufreq_frequency_table *freq_table; bool have_static_opps; }; @@ -59,7 +62,11 @@ static int set_target(struct cpufreq_policy *policy, unsigned int index) struct private_data *priv = policy->driver_data; unsigned long freq = policy->freq_table[index].frequency; +#ifdef CONFIG_ARCH_ROCKCHIP + return rockchip_cpufreq_opp_set_rate(priv->cpu_dev, freq * 1000); +#else return dev_pm_opp_set_rate(priv->cpu_dev, freq * 1000); +#endif } /* @@ -102,7 +109,6 @@ static const char *find_supply_name(struct device *dev) static int cpufreq_init(struct cpufreq_policy *policy) { - struct cpufreq_frequency_table *freq_table; struct private_data *priv; struct device *cpu_dev; struct clk *cpu_clk; @@ -114,9 +120,7 @@ static int cpufreq_init(struct cpufreq_policy *policy) pr_err("failed to find data for cpu%d\n", policy->cpu); return -ENODEV; } - cpu_dev = priv->cpu_dev; - cpumask_copy(policy->cpus, priv->cpus); cpu_clk = clk_get(cpu_dev, NULL); if (IS_ERR(cpu_clk)) { @@ -125,67 +129,32 @@ static int cpufreq_init(struct cpufreq_policy *policy) return ret; } - /* - * Initialize OPP tables for all policy->cpus. They will be shared by - * all CPUs which have marked their CPUs shared with OPP bindings. - * - * For platforms not using operating-points-v2 bindings, we do this - * before updating policy->cpus. Otherwise, we will end up creating - * duplicate OPPs for policy->cpus. - * - * OPPs might be populated at runtime, don't check for error here - */ - if (!dev_pm_opp_of_cpumask_add_table(policy->cpus)) - priv->have_static_opps = true; - - /* - * But we need OPP table to function so if it is not there let's - * give platform code chance to provide it for us. - */ - ret = dev_pm_opp_get_opp_count(cpu_dev); - if (ret <= 0) { - dev_err(cpu_dev, "OPP table can't be empty\n"); - ret = -ENODEV; - goto out_free_opp; - } - - ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); - if (ret) { - dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); - goto out_free_opp; - } + transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); + if (!transition_latency) + transition_latency = CPUFREQ_ETERNAL; + cpumask_copy(policy->cpus, priv->cpus); policy->driver_data = priv; policy->clk = cpu_clk; - policy->freq_table = freq_table; - + policy->freq_table = priv->freq_table; policy->suspend_freq = dev_pm_opp_get_suspend_opp_freq(cpu_dev) / 1000; + policy->cpuinfo.transition_latency = transition_latency; + policy->dvfs_possible_from_any_cpu = true; /* Support turbo/boost mode */ if (policy_has_boost_freq(policy)) { /* This gets disabled by core on driver unregister */ ret = cpufreq_enable_boost_support(); if (ret) - goto out_free_cpufreq_table; + goto out_clk_put; cpufreq_dt_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs; } - transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); - if (!transition_latency) - transition_latency = CPUFREQ_ETERNAL; - - policy->cpuinfo.transition_latency = transition_latency; - policy->dvfs_possible_from_any_cpu = true; - dev_pm_opp_of_register_em(cpu_dev, policy->cpus); return 0; -out_free_cpufreq_table: - dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); -out_free_opp: - if (priv->have_static_opps) - dev_pm_opp_of_cpumask_remove_table(policy->cpus); +out_clk_put: clk_put(cpu_clk); return ret; @@ -208,11 +177,6 @@ static int cpufreq_offline(struct cpufreq_policy *policy) static int cpufreq_exit(struct cpufreq_policy *policy) { - struct private_data *priv = policy->driver_data; - - dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); - if (priv->have_static_opps) - dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); clk_put(policy->clk); return 0; } @@ -236,6 +200,7 @@ static int dt_cpufreq_early_init(struct device *dev, int cpu) { struct private_data *priv; struct device *cpu_dev; + bool fallback = false; const char *reg_name; int ret; @@ -254,68 +219,91 @@ static int dt_cpufreq_early_init(struct device *dev, int cpu) if (!alloc_cpumask_var(&priv->cpus, GFP_KERNEL)) return -ENOMEM; + cpumask_set_cpu(cpu, priv->cpus); priv->cpu_dev = cpu_dev; - /* Try to get OPP table early to ensure resources are available */ - priv->opp_table = dev_pm_opp_get_opp_table(cpu_dev); - if (IS_ERR(priv->opp_table)) { - ret = PTR_ERR(priv->opp_table); - if (ret != -EPROBE_DEFER) - dev_err(cpu_dev, "failed to get OPP table: %d\n", ret); - goto free_cpumask; - } - /* * OPP layer will be taking care of regulators now, but it needs to know * the name of the regulator first. */ reg_name = find_supply_name(cpu_dev); if (reg_name) { - priv->reg_opp_table = dev_pm_opp_set_regulators(cpu_dev, - ®_name, 1); - if (IS_ERR(priv->reg_opp_table)) { - ret = PTR_ERR(priv->reg_opp_table); + priv->opp_table = dev_pm_opp_set_regulators(cpu_dev, ®_name, + 1); + if (IS_ERR(priv->opp_table)) { + ret = PTR_ERR(priv->opp_table); if (ret != -EPROBE_DEFER) dev_err(cpu_dev, "failed to set regulators: %d\n", ret); - goto put_table; + goto free_cpumask; } } - /* Find OPP sharing information so we can fill pri->cpus here */ /* Get OPP-sharing information from "operating-points-v2" bindings */ ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, priv->cpus); if (ret) { if (ret != -ENOENT) - goto put_reg; + goto out; /* * operating-points-v2 not supported, fallback to all CPUs share * OPP for backward compatibility if the platform hasn't set * sharing CPUs. */ - if (dev_pm_opp_get_sharing_cpus(cpu_dev, priv->cpus)) { - cpumask_setall(priv->cpus); - - /* - * OPP tables are initialized only for cpu, do it for - * others as well. - */ - ret = dev_pm_opp_set_sharing_cpus(cpu_dev, priv->cpus); - if (ret) - dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", - __func__, ret); - } + if (dev_pm_opp_get_sharing_cpus(cpu_dev, priv->cpus)) + fallback = true; + } + + /* + * Initialize OPP tables for all priv->cpus. They will be shared by + * all CPUs which have marked their CPUs shared with OPP bindings. + * + * For platforms not using operating-points-v2 bindings, we do this + * before updating priv->cpus. Otherwise, we will end up creating + * duplicate OPPs for the CPUs. + * + * OPPs might be populated at runtime, don't check for error here. + */ + if (!dev_pm_opp_of_cpumask_add_table(priv->cpus)) + priv->have_static_opps = true; + + /* + * The OPP table must be initialized, statically or dynamically, by this + * point. + */ + ret = dev_pm_opp_get_opp_count(cpu_dev); + if (ret <= 0) { + dev_err(cpu_dev, "OPP table can't be empty\n"); + ret = -ENODEV; + goto out; + } + + if (fallback) { + cpumask_setall(priv->cpus); + ret = dev_pm_opp_set_sharing_cpus(cpu_dev, priv->cpus); + if (ret) + dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", + __func__, ret); + } + +#ifdef CONFIG_ARCH_ROCKCHIP + rockchip_cpufreq_adjust_power_scale(cpu_dev); +#endif + + ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &priv->freq_table); + if (ret) { + dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); + goto out; } list_add(&priv->node, &priv_list); return 0; -put_reg: - if (priv->reg_opp_table) - dev_pm_opp_put_regulators(priv->reg_opp_table); -put_table: - dev_pm_opp_put_opp_table(priv->opp_table); +out: + if (priv->have_static_opps) + dev_pm_opp_of_cpumask_remove_table(priv->cpus); + if (priv->opp_table) + dev_pm_opp_put_regulators(priv->opp_table); free_cpumask: free_cpumask_var(priv->cpus); return ret; @@ -326,9 +314,11 @@ static void dt_cpufreq_release(void) struct private_data *priv, *tmp; list_for_each_entry_safe(priv, tmp, &priv_list, node) { - if (priv->reg_opp_table) - dev_pm_opp_put_regulators(priv->reg_opp_table); - dev_pm_opp_put_opp_table(priv->opp_table); + dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &priv->freq_table); + if (priv->have_static_opps) + dev_pm_opp_of_cpumask_remove_table(priv->cpus); + if (priv->opp_table) + dev_pm_opp_put_regulators(priv->opp_table); free_cpumask_var(priv->cpus); list_del(&priv->node); } diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 58342390966b..ae35ef771900 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -688,8 +688,12 @@ static ssize_t show_##file_name \ return sprintf(buf, "%u\n", policy->object); \ } +static ssize_t show_cpuinfo_max_freq(struct cpufreq_policy *policy, char *buf) +{ + unsigned int max_freq = policy->cpuinfo.max_freq; + return sprintf(buf, "%u\n", max_freq); +} show_one(cpuinfo_min_freq, cpuinfo.min_freq); -show_one(cpuinfo_max_freq, cpuinfo.max_freq); show_one(cpuinfo_transition_latency, cpuinfo.transition_latency); show_one(scaling_min_freq, min); show_one(scaling_max_freq, max); @@ -2535,6 +2539,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, return ret; } +EXPORT_TRACEPOINT_SYMBOL_GPL(cpu_frequency_limits); /** * cpufreq_update_policy - Re-evaluate an existing cpufreq policy. diff --git a/drivers/cpufreq/cpufreq_userspace.c b/drivers/cpufreq/cpufreq_userspace.c index 50a4d7846580..1f001d281718 100644 --- a/drivers/cpufreq/cpufreq_userspace.c +++ b/drivers/cpufreq/cpufreq_userspace.c @@ -78,20 +78,18 @@ static int cpufreq_userspace_policy_start(struct cpufreq_policy *policy) mutex_lock(&userspace_mutex); per_cpu(cpu_is_managed, policy->cpu) = 1; - *setspeed = policy->cur; + if (!*setspeed) + *setspeed = policy->cur; mutex_unlock(&userspace_mutex); return 0; } static void cpufreq_userspace_policy_stop(struct cpufreq_policy *policy) { - unsigned int *setspeed = policy->governor_data; - pr_debug("managing cpu %u stopped\n", policy->cpu); mutex_lock(&userspace_mutex); per_cpu(cpu_is_managed, policy->cpu) = 0; - *setspeed = 0; mutex_unlock(&userspace_mutex); } diff --git a/drivers/cpufreq/rockchip-cpufreq.c b/drivers/cpufreq/rockchip-cpufreq.c new file mode 100755 index 000000000000..1a0e960870c1 --- /dev/null +++ b/drivers/cpufreq/rockchip-cpufreq.c @@ -0,0 +1,638 @@ +/* + * Rockchip CPUFreq Driver + * + * Copyright (C) 2017 Fuzhou Rockchip Electronics Co., Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cpufreq-dt.h" +#include "rockchip-cpufreq.h" + +struct cluster_info { + struct list_head list_head; + struct monitor_dev_info *mdev_info; + struct rockchip_opp_info opp_info; + cpumask_t cpus; + int scale; +}; +static LIST_HEAD(cluster_info_list); + +static int px30_get_soc_info(struct device *dev, struct device_node *np, + int *bin, int *process) +{ + int ret = 0; + u8 value = 0; + + if (!bin) + return 0; + + if (of_property_match_string(np, "nvmem-cell-names", + "performance") >= 0) { + ret = rockchip_nvmem_cell_read_u8(np, "performance", &value); + if (ret) { + dev_err(dev, "Failed to get soc performance value\n"); + return ret; + } + *bin = value; + } + if (*bin >= 0) + dev_info(dev, "bin=%d\n", *bin); + + return ret; +} + +static int rk3288_get_soc_info(struct device *dev, struct device_node *np, + int *bin, int *process) +{ + int ret = 0; + u8 value = 0; + char *name; + + if (!bin) + goto next; + if (of_property_match_string(np, "nvmem-cell-names", "special") >= 0) { + ret = rockchip_nvmem_cell_read_u8(np, "special", &value); + if (ret) { + dev_err(dev, "Failed to get soc special value\n"); + goto out; + } + if (value == 0xc) + *bin = 0; + else + *bin = 1; + } + + if (soc_is_rk3288w()) + name = "performance-w"; + else + name = "performance"; + + if (of_property_match_string(np, "nvmem-cell-names", name) >= 0) { + ret = rockchip_nvmem_cell_read_u8(np, name, &value); + if (ret) { + dev_err(dev, "Failed to get soc performance value\n"); + goto out; + } + if (value & 0x2) + *bin = 3; + else if (value & 0x01) + *bin = 2; + } + if (*bin >= 0) + dev_info(dev, "bin=%d\n", *bin); + +next: + if (!process) + goto out; + if (of_property_match_string(np, "nvmem-cell-names", + "process") >= 0) { + ret = rockchip_nvmem_cell_read_u8(np, "process", &value); + if (ret) { + dev_err(dev, "Failed to get soc process version\n"); + goto out; + } + if (soc_is_rk3288() && (value == 0 || value == 1)) + *process = 0; + } + if (*process >= 0) + dev_info(dev, "process=%d\n", *process); + +out: + return ret; +} + +static int rk3399_get_soc_info(struct device *dev, struct device_node *np, + int *bin, int *process) +{ + int ret = 0; + u8 value = 0; + + if (!bin) + return 0; + + if (of_property_match_string(np, "nvmem-cell-names", + "specification_serial_number") >= 0) { + ret = rockchip_nvmem_cell_read_u8(np, + "specification_serial_number", + &value); + if (ret) { + dev_err(dev, + "Failed to get specification_serial_number\n"); + goto out; + } + + if (value == 0xb) { + *bin = 0; + } else if (value == 0x1) { + if (of_property_match_string(np, "nvmem-cell-names", + "customer_demand") >= 0) { + ret = rockchip_nvmem_cell_read_u8(np, + "customer_demand", + &value); + if (ret) { + dev_err(dev, "Failed to get customer_demand\n"); + goto out; + } + if (value == 0x0) + *bin = 0; + else + *bin = 1; + } + } else if (value == 0x10) { + *bin = 1; + } + } + +out: + if (*bin >= 0) + dev_info(dev, "bin=%d\n", *bin); + + return ret; +} + +static int rk3588_cpu_set_read_margin(struct device *dev, + struct rockchip_opp_info *opp_info, + unsigned long volt) +{ + bool is_found = false; + u32 rm; + int i; + + if (!opp_info->grf || !opp_info->volt_rm_tbl) + return 0; + + for (i = 0; opp_info->volt_rm_tbl[i].rm != VOLT_RM_TABLE_END; i++) { + if (volt >= opp_info->volt_rm_tbl[i].volt) { + rm = opp_info->volt_rm_tbl[i].rm; + is_found = true; + break; + } + } + + if (!is_found) + return 0; + if (rm == opp_info->current_rm) + return 0; + + dev_dbg(dev, "set rm to %d\n", rm); + regmap_write(opp_info->grf, 0x20, 0x001c0000 | (rm << 2)); + regmap_write(opp_info->grf, 0x28, 0x003c0000 | (rm << 2)); + regmap_write(opp_info->grf, 0x2c, 0x003c0000 | (rm << 2)); + regmap_write(opp_info->grf, 0x30, 0x00200020); + udelay(1); + regmap_write(opp_info->grf, 0x30, 0x00200000); + + opp_info->current_rm = rm; + + return 0; +} + +static int rv1126_get_soc_info(struct device *dev, struct device_node *np, + int *bin, int *process) +{ + int ret = 0; + u8 value = 0; + + if (of_property_match_string(np, "nvmem-cell-names", "performance") >= 0) { + ret = rockchip_nvmem_cell_read_u8(np, "performance", &value); + if (ret) { + dev_err(dev, "Failed to get soc performance value\n"); + return ret; + } + if (value == 0x1) + *bin = 1; + else + *bin = 0; + } + if (*bin >= 0) + dev_info(dev, "bin=%d\n", *bin); + + return ret; +} + +static const struct rockchip_opp_data px30_cpu_opp_data = { + .get_soc_info = px30_get_soc_info, +}; + +static const struct rockchip_opp_data rk3288_cpu_opp_data = { + .get_soc_info = rk3288_get_soc_info, +}; + +static const struct rockchip_opp_data rk3399_cpu_opp_data = { + .get_soc_info = rk3399_get_soc_info, +}; + +static const struct rockchip_opp_data rk3588_cpu_opp_data = { + .set_read_margin = rk3588_cpu_set_read_margin, +}; + +static const struct rockchip_opp_data rv1126_cpu_opp_data = { + .get_soc_info = rv1126_get_soc_info, +}; + +static const struct of_device_id rockchip_cpufreq_of_match[] = { + { + .compatible = "rockchip,px30", + .data = (void *)&px30_cpu_opp_data, + }, + { + .compatible = "rockchip,rk3288", + .data = (void *)&rk3288_cpu_opp_data, + }, + { + .compatible = "rockchip,rk3288w", + .data = (void *)&rk3288_cpu_opp_data, + }, + { + .compatible = "rockchip,rk3326", + .data = (void *)&px30_cpu_opp_data, + }, + { + .compatible = "rockchip,rk3399", + .data = (void *)&rk3399_cpu_opp_data, + }, + { + .compatible = "rockchip,rk3588", + .data = (void *)&rk3588_cpu_opp_data, + }, + { + .compatible = "rockchip,rv1109", + .data = (void *)&rv1126_cpu_opp_data, + }, + { + .compatible = "rockchip,rv1126", + .data = (void *)&rv1126_cpu_opp_data, + }, + {}, +}; + +static struct cluster_info *rockchip_cluster_info_lookup(int cpu) +{ + struct cluster_info *cluster; + + list_for_each_entry(cluster, &cluster_info_list, list_head) { + if (cpumask_test_cpu(cpu, &cluster->cpus)) + return cluster; + } + + return NULL; +} + +static int rockchip_cpufreq_set_volt(struct device *dev, + struct regulator *reg, + struct dev_pm_opp_supply *supply, + char *reg_name) +{ + int ret; + + dev_dbg(dev, "%s: %s voltages (uV): %lu %lu %lu\n", __func__, reg_name, + supply->u_volt_min, supply->u_volt, supply->u_volt_max); + + ret = regulator_set_voltage_triplet(reg, supply->u_volt_min, + supply->u_volt, supply->u_volt_max); + if (ret) + dev_err(dev, "%s: failed to set voltage (%lu %lu %lu uV): %d\n", + __func__, supply->u_volt_min, supply->u_volt, + supply->u_volt_max, ret); + + return ret; +} + +static int cpu_opp_helper(struct dev_pm_set_opp_data *data) +{ + struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0]; + struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1]; + struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0]; + struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1]; + struct regulator *vdd_reg = data->regulators[0]; + struct regulator *mem_reg = data->regulators[1]; + struct device *dev = data->dev; + struct clk *clk = data->clk; + struct cluster_info *cluster; + struct rockchip_opp_info *opp_info; + unsigned long old_freq = data->old_opp.rate; + unsigned long new_freq = data->new_opp.rate; + int ret = 0; + + cluster = rockchip_cluster_info_lookup(dev->id); + if (!cluster) + return -EINVAL; + opp_info = &cluster->opp_info; + + /* Scaling up? Scale voltage before frequency */ + if (new_freq >= old_freq) { + ret = rockchip_cpufreq_set_volt(dev, mem_reg, new_supply_mem, + "mem"); + if (ret) + goto restore_voltage; + ret = rockchip_cpufreq_set_volt(dev, vdd_reg, new_supply_vdd, + "vdd"); + if (ret) + goto restore_voltage; + if (opp_info->data->set_read_margin) + opp_info->data->set_read_margin(dev, opp_info, + new_supply_vdd->u_volt); + } + + /* Change frequency */ + dev_dbg(dev, "%s: switching OPP: %lu Hz --> %lu Hz\n", __func__, + old_freq, new_freq); + ret = clk_set_rate(clk, new_freq); + if (ret) { + dev_err(dev, "%s: failed to set clk rate: %d\n", __func__, ret); + goto restore_rm; + } + + /* Scaling down? Scale voltage after frequency */ + if (new_freq < old_freq) { + if (opp_info->data->set_read_margin) + opp_info->data->set_read_margin(dev, opp_info, + new_supply_vdd->u_volt); + ret = rockchip_cpufreq_set_volt(dev, vdd_reg, new_supply_vdd, + "vdd"); + if (ret) + goto restore_freq; + ret = rockchip_cpufreq_set_volt(dev, mem_reg, new_supply_mem, + "mem"); + if (ret) + goto restore_freq; + } + + return 0; + +restore_freq: + if (clk_set_rate(clk, old_freq)) + dev_err(dev, "%s: failed to restore old-freq (%lu Hz)\n", + __func__, old_freq); +restore_rm: + if (opp_info->data->set_read_margin) + opp_info->data->set_read_margin(dev, opp_info, + old_supply_vdd->u_volt); +restore_voltage: + rockchip_cpufreq_set_volt(dev, mem_reg, old_supply_mem, "mem"); + rockchip_cpufreq_set_volt(dev, vdd_reg, old_supply_vdd, "vdd"); + + return ret; +} + +static int rockchip_cpufreq_cluster_init(int cpu, struct cluster_info *cluster) +{ + struct rockchip_opp_info *opp_info = &cluster->opp_info; + struct opp_table *pname_table = NULL; + struct opp_table *reg_table = NULL; + struct opp_table *opp_table; + struct device_node *np; + struct device *dev; + const char * const reg_names[] = {"cpu", "mem"}; + char *reg_name = NULL; + int bin = -EINVAL; + int process = -EINVAL; + int volt_sel = -EINVAL; + int ret = 0; + + dev = get_cpu_device(cpu); + if (!dev) + return -ENODEV; + + if (of_find_property(dev->of_node, "cpu-supply", NULL)) + reg_name = "cpu"; + else if (of_find_property(dev->of_node, "cpu0-supply", NULL)) + reg_name = "cpu0"; + else + return -ENOENT; + + np = of_parse_phandle(dev->of_node, "operating-points-v2", 0); + if (!np) { + dev_warn(dev, "OPP-v2 not supported\n"); + return -ENOENT; + } + + ret = dev_pm_opp_of_get_sharing_cpus(dev, &cluster->cpus); + if (ret) { + dev_err(dev, "Failed to get sharing cpus\n"); + goto np_err; + } + + rockchip_get_opp_data(rockchip_cpufreq_of_match, opp_info); + if (opp_info->data && opp_info->data->set_read_margin) { + opp_info->current_rm = UINT_MAX; + opp_info->grf = syscon_regmap_lookup_by_phandle(np, + "rockchip,grf"); + if (IS_ERR(opp_info->grf)) + opp_info->grf = NULL; + rockchip_get_volt_rm_table(dev, np, "volt-mem-read-margin", + &opp_info->volt_rm_tbl); + } + if (opp_info->data && opp_info->data->get_soc_info) + opp_info->data->get_soc_info(dev, np, &bin, &process); + rockchip_get_scale_volt_sel(dev, "cpu_leakage", reg_name, bin, process, + &cluster->scale, &volt_sel); + pname_table = rockchip_set_opp_prop_name(dev, process, volt_sel); + if (IS_ERR(pname_table)) { + ret = PTR_ERR(pname_table); + goto np_err; + } + + if (of_find_property(dev->of_node, "cpu-supply", NULL) && + of_find_property(dev->of_node, "mem-supply", NULL)) { + reg_table = dev_pm_opp_set_regulators(dev, reg_names, + ARRAY_SIZE(reg_names)); + if (IS_ERR(reg_table)) { + ret = PTR_ERR(reg_table); + goto pname_opp_table; + } + opp_table = dev_pm_opp_register_set_opp_helper(dev, + cpu_opp_helper); + if (IS_ERR(opp_table)) { + ret = PTR_ERR(opp_table); + goto reg_opp_table; + } + } + + of_node_put(np); + + return 0; + +reg_opp_table: + if (reg_table) + dev_pm_opp_put_regulators(reg_table); +pname_opp_table: + if (pname_table) + dev_pm_opp_put_prop_name(pname_table); +np_err: + of_node_put(np); + + return ret; +} + +int rockchip_cpufreq_adjust_power_scale(struct device *dev) +{ + struct cluster_info *cluster; + + cluster = rockchip_cluster_info_lookup(dev->id); + if (!cluster) + return -EINVAL; + rockchip_adjust_power_scale(dev, cluster->scale); + + return 0; +} +EXPORT_SYMBOL_GPL(rockchip_cpufreq_adjust_power_scale); + +int rockchip_cpufreq_opp_set_rate(struct device *dev, unsigned long target_freq) +{ + struct cluster_info *cluster; + int ret = 0; + + cluster = rockchip_cluster_info_lookup(dev->id); + if (!cluster) + return -EINVAL; + + rockchip_monitor_volt_adjust_lock(cluster->mdev_info); + ret = dev_pm_opp_set_rate(dev, target_freq); + rockchip_monitor_volt_adjust_unlock(cluster->mdev_info); + + return ret; +} +EXPORT_SYMBOL_GPL(rockchip_cpufreq_opp_set_rate); + +static int rockchip_cpufreq_suspend(struct cpufreq_policy *policy) +{ + int ret = 0; + + ret = cpufreq_generic_suspend(policy); + if (!ret) + rockchip_monitor_suspend_low_temp_adjust(policy->cpu); + + return ret; +} + +static int rockchip_cpufreq_notifier(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct device *dev; + struct cpufreq_policy *policy = data; + struct cluster_info *cluster; + struct monitor_dev_profile *mdevp = NULL; + struct monitor_dev_info *mdev_info = NULL; + + dev = get_cpu_device(policy->cpu); + if (!dev) + return NOTIFY_BAD; + + cluster = rockchip_cluster_info_lookup(policy->cpu); + if (!cluster) + return NOTIFY_BAD; + + if (event == CPUFREQ_CREATE_POLICY) { + mdevp = kzalloc(sizeof(*mdevp), GFP_KERNEL); + if (!mdevp) + return NOTIFY_BAD; + mdevp->type = MONITOR_TPYE_CPU; + mdevp->low_temp_adjust = rockchip_monitor_cpu_low_temp_adjust; + mdevp->high_temp_adjust = rockchip_monitor_cpu_high_temp_adjust; + mdevp->update_volt = rockchip_monitor_check_rate_volt; + mdevp->data = (void *)policy; + mdevp->opp_info = &cluster->opp_info; + cpumask_copy(&mdevp->allowed_cpus, policy->cpus); + mdev_info = rockchip_system_monitor_register(dev, mdevp); + if (IS_ERR(mdev_info)) { + kfree(mdevp); + dev_err(dev, "failed to register system monitor\n"); + return NOTIFY_BAD; + } + mdev_info->devp = mdevp; + cluster->mdev_info = mdev_info; + } else if (event == CPUFREQ_REMOVE_POLICY) { + if (cluster->mdev_info) { + kfree(cluster->mdev_info->devp); + rockchip_system_monitor_unregister(cluster->mdev_info); + cluster->mdev_info = NULL; + } + } + + return NOTIFY_OK; +} + +static struct notifier_block rockchip_cpufreq_notifier_block = { + .notifier_call = rockchip_cpufreq_notifier, +}; + +static int __init rockchip_cpufreq_driver_init(void) +{ + struct cluster_info *cluster, *pos; + struct cpufreq_dt_platform_data pdata = {0}; + int cpu, ret; + + for_each_possible_cpu(cpu) { + cluster = rockchip_cluster_info_lookup(cpu); + if (cluster) + continue; + + cluster = kzalloc(sizeof(*cluster), GFP_KERNEL); + if (!cluster) { + ret = -ENOMEM; + goto release_cluster_info; + } + + ret = rockchip_cpufreq_cluster_init(cpu, cluster); + if (ret) { + pr_err("Failed to initialize dvfs info cpu%d\n", cpu); + goto release_cluster_info; + } + list_add(&cluster->list_head, &cluster_info_list); + } + + pdata.have_governor_per_policy = true; + pdata.suspend = rockchip_cpufreq_suspend; + + ret = cpufreq_register_notifier(&rockchip_cpufreq_notifier_block, + CPUFREQ_POLICY_NOTIFIER); + if (ret) { + pr_err("failed to register cpufreq notifier\n"); + goto release_cluster_info; + } + + return PTR_ERR_OR_ZERO(platform_device_register_data(NULL, "cpufreq-dt", + -1, (void *)&pdata, + sizeof(struct cpufreq_dt_platform_data))); + +release_cluster_info: + list_for_each_entry_safe(cluster, pos, &cluster_info_list, list_head) { + list_del(&cluster->list_head); + kfree(cluster); + } + return ret; +} +module_init(rockchip_cpufreq_driver_init); + +MODULE_AUTHOR("Finley Xiao "); +MODULE_DESCRIPTION("Rockchip cpufreq driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cpufreq/rockchip-cpufreq.h b/drivers/cpufreq/rockchip-cpufreq.h new file mode 100755 index 000000000000..3b3dbcf73449 --- /dev/null +++ b/drivers/cpufreq/rockchip-cpufreq.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + */ +#ifndef __ROCKCHIP_CPUFREQ_H +#define __ROCKCHIP_CPUFREQ_H + +#if IS_ENABLED(CONFIG_ARM_ROCKCHIP_CPUFREQ) +int rockchip_cpufreq_adjust_power_scale(struct device *dev); +int rockchip_cpufreq_opp_set_rate(struct device *dev, unsigned long target_freq); +#else +static inline int rockchip_cpufreq_adjust_power_scale(struct device *dev) +{ + return -EOPNOTSUPP; +} + +static inline int rockchip_cpufreq_opp_set_rate(struct device *dev, + unsigned long target_freq) +{ + return -EOPNOTSUPP; +} +#endif /* CONFIG_ARM_ROCKCHIP_CPUFREQ */ + +#endif diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c index 4070e573bf43..557f59ac47a3 100644 --- a/drivers/cpuidle/driver.c +++ b/drivers/cpuidle/driver.c @@ -381,3 +381,4 @@ void cpuidle_driver_state_disabled(struct cpuidle_driver *drv, int idx, mutex_unlock(&cpuidle_lock); } +EXPORT_SYMBOL_GPL(cpuidle_driver_state_disabled); diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c index 29acaf48e575..0e51ed25665e 100644 --- a/drivers/cpuidle/governor.c +++ b/drivers/cpuidle/governor.c @@ -102,6 +102,7 @@ int cpuidle_register_governor(struct cpuidle_governor *gov) return ret; } +EXPORT_SYMBOL_GPL(cpuidle_register_governor); /** * cpuidle_governor_latency_req - Compute a latency constraint for CPU @@ -118,3 +119,4 @@ s64 cpuidle_governor_latency_req(unsigned int cpu) return (s64)device_req * NSEC_PER_USEC; } +EXPORT_SYMBOL_GPL(cpuidle_governor_latency_req); diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig index 37dc40d1fcfb..ab9c00a1b1c3 100644 --- a/drivers/devfreq/Kconfig +++ b/drivers/devfreq/Kconfig @@ -131,15 +131,20 @@ config ARM_TEGRA20_DEVFREQ It reads Memory Controller counters and adjusts the operating frequencies and voltages with OPP support. -config ARM_RK3399_DMC_DEVFREQ - tristate "ARM RK3399 DMC DEVFREQ Driver" +config ARM_ROCKCHIP_BUS_DEVFREQ + tristate "ARM ROCKCHIP BUS DEVFREQ Driver" + depends on ARCH_ROCKCHIP + help + This adds the DEVFREQ driver for the ROCKCHIP BUS. + +config ARM_ROCKCHIP_DMC_DEVFREQ + tristate "ARM ROCKCHIP DMC DEVFREQ Driver" depends on (ARCH_ROCKCHIP && HAVE_ARM_SMCCC) || \ (COMPILE_TEST && HAVE_ARM_SMCCC) select DEVFREQ_EVENT_ROCKCHIP_DFI - select DEVFREQ_GOV_SIMPLE_ONDEMAND select PM_DEVFREQ_EVENT help - This adds the DEVFREQ driver for the RK3399 DMC(Dynamic Memory Controller). + This adds the DEVFREQ driver for the ROCKCHIP DMC(Dynamic Memory Controller). It sets the frequency for the memory controller and reads the usage counts from hardware. diff --git a/drivers/devfreq/Makefile b/drivers/devfreq/Makefile index 3ca1ad0ecb97..abea7adbc7da 100644 --- a/drivers/devfreq/Makefile +++ b/drivers/devfreq/Makefile @@ -11,7 +11,8 @@ obj-$(CONFIG_DEVFREQ_GOV_PASSIVE) += governor_passive.o obj-$(CONFIG_ARM_EXYNOS_BUS_DEVFREQ) += exynos-bus.o obj-$(CONFIG_ARM_IMX_BUS_DEVFREQ) += imx-bus.o obj-$(CONFIG_ARM_IMX8M_DDRC_DEVFREQ) += imx8m-ddrc.o -obj-$(CONFIG_ARM_RK3399_DMC_DEVFREQ) += rk3399_dmc.o +obj-$(CONFIG_ARM_ROCKCHIP_BUS_DEVFREQ) += rockchip_bus.o +obj-$(CONFIG_ARM_ROCKCHIP_DMC_DEVFREQ) += rockchip_dmc.o rockchip_dmc_common.o obj-$(CONFIG_ARM_TEGRA_DEVFREQ) += tegra30-devfreq.o obj-$(CONFIG_ARM_TEGRA20_DEVFREQ) += tegra20-devfreq.o diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index c6f460550f5e..db0fe99c8d61 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -1761,6 +1761,40 @@ static ssize_t timer_store(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RW(timer); +static ssize_t load_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + int err; + struct devfreq *devfreq = to_devfreq(dev); + struct devfreq_dev_status stat = devfreq->last_status; + unsigned long freq; + ssize_t len; + + err = devfreq_update_stats(devfreq); + if (err) + return err; + + if (stat.total_time < stat.busy_time) { + err = devfreq_update_stats(devfreq); + if (err) + return err; + }; + + if (!stat.total_time) + return 0; + + len = sprintf(buf, "%lu", stat.busy_time * 100 / stat.total_time); + + if (devfreq->profile->get_cur_freq && + !devfreq->profile->get_cur_freq(devfreq->dev.parent, &freq)) + len += sprintf(buf + len, "@%luHz\n", freq); + else + len += sprintf(buf + len, "@%luHz\n", devfreq->previous_freq); + + return len; +} +static DEVICE_ATTR_RO(load); + static struct attribute *devfreq_attrs[] = { &dev_attr_name.attr, &dev_attr_governor.attr, @@ -1773,6 +1807,7 @@ static struct attribute *devfreq_attrs[] = { &dev_attr_max_freq.attr, &dev_attr_trans_stat.attr, &dev_attr_timer.attr, + &dev_attr_load.attr, NULL, }; ATTRIBUTE_GROUPS(devfreq); diff --git a/drivers/devfreq/event/Kconfig b/drivers/devfreq/event/Kconfig index 878825372f6f..4526c69c602e 100644 --- a/drivers/devfreq/event/Kconfig +++ b/drivers/devfreq/event/Kconfig @@ -39,4 +39,11 @@ config DEVFREQ_EVENT_ROCKCHIP_DFI This add the devfreq-event driver for Rockchip SoC. It provides DFI (DDR Monitor Module) driver to count ddr load. +config DEVFREQ_EVENT_ROCKCHIP_NOCP + tristate "ROCKCHIP NoC (Network On Chip) Probe DEVFREQ event Driver" + depends on ARCH_ROCKCHIP + help + This add the devfreq-event driver for Rockchip SoC. It provides NoC + (Network on Chip) Probe counters to monitor traffic statistics. + endif # PM_DEVFREQ_EVENT diff --git a/drivers/devfreq/event/Makefile b/drivers/devfreq/event/Makefile index 3c847e5d5a35..03d67f06c22e 100644 --- a/drivers/devfreq/event/Makefile +++ b/drivers/devfreq/event/Makefile @@ -4,3 +4,4 @@ obj-$(CONFIG_DEVFREQ_EVENT_EXYNOS_NOCP) += exynos-nocp.o obj-$(CONFIG_DEVFREQ_EVENT_EXYNOS_PPMU) += exynos-ppmu.o obj-$(CONFIG_DEVFREQ_EVENT_ROCKCHIP_DFI) += rockchip-dfi.o +obj-$(CONFIG_DEVFREQ_EVENT_ROCKCHIP_NOCP) += rockchip-nocp.o diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c index 9a88faaf8b27..9fd6a82f79d6 100644 --- a/drivers/devfreq/event/rockchip-dfi.c +++ b/drivers/devfreq/event/rockchip-dfi.c @@ -20,23 +20,70 @@ #include -#define RK3399_DMC_NUM_CH 2 - +#define PX30_PMUGRF_OS_REG2 0x208 +#define PX30_PMUGRF_OS_REG3 0x20c + +#define RK3128_GRF_SOC_CON0 0x140 +#define RK3128_GRF_OS_REG1 0x1cc +#define RK3128_GRF_DFI_WRNUM 0x220 +#define RK3128_GRF_DFI_RDNUM 0x224 +#define RK3128_GRF_DFI_TIMERVAL 0x22c +#define RK3128_DDR_MONITOR_EN ((1 << (16 + 6)) + (1 << 6)) +#define RK3128_DDR_MONITOR_DISB ((1 << (16 + 6)) + (0 << 6)) + +#define RK3288_PMU_SYS_REG2 0x9c +#define RK3288_GRF_SOC_CON4 0x254 +#define RK3288_GRF_SOC_STATUS(n) (0x280 + (n) * 4) +#define RK3288_DFI_EN (0x30003 << 14) +#define RK3288_DFI_DIS (0x30000 << 14) +#define RK3288_LPDDR_SEL (0x10001 << 13) +#define RK3288_DDR3_SEL (0x10000 << 13) + +#define RK3328_GRF_OS_REG2 0x5d0 + +#define RK3368_GRF_DDRC0_CON0 0x600 +#define RK3368_GRF_SOC_STATUS5 0x494 +#define RK3368_GRF_SOC_STATUS6 0x498 +#define RK3368_GRF_SOC_STATUS8 0x4a0 +#define RK3368_GRF_SOC_STATUS9 0x4a4 +#define RK3368_GRF_SOC_STATUS10 0x4a8 +#define RK3368_DFI_EN (0x30003 << 5) +#define RK3368_DFI_DIS (0x30000 << 5) + +#define MAX_DMC_NUM_CH 2 +#define READ_DRAMTYPE_INFO(n) (((n) >> 13) & 0x7) +#define READ_CH_INFO(n) (((n) >> 28) & 0x3) +#define READ_DRAMTYPE_INFO_V3(n, m) ((((n) >> 13) & 0x7) | ((((m) >> 12) & 0x3) << 3)) +#define READ_SYSREG_VERSION(m) (((m) >> 28) & 0xf) /* DDRMON_CTRL */ -#define DDRMON_CTRL 0x04 -#define CLR_DDRMON_CTRL (0x1f0000 << 0) -#define LPDDR4_EN (0x10001 << 4) -#define HARDWARE_EN (0x10001 << 3) -#define LPDDR3_EN (0x10001 << 2) -#define SOFTWARE_EN (0x10001 << 1) -#define SOFTWARE_DIS (0x10000 << 1) -#define TIME_CNT_EN (0x10001 << 0) +#define DDRMON_CTRL 0x04 +#define CLR_DDRMON_CTRL (0x3f0000 << 0) +#define DDR4_EN (0x10001 << 5) +#define LPDDR4_EN (0x10001 << 4) +#define HARDWARE_EN (0x10001 << 3) +#define LPDDR2_3_EN (0x10001 << 2) +#define SOFTWARE_EN (0x10001 << 1) +#define SOFTWARE_DIS (0x10000 << 1) +#define TIME_CNT_EN (0x10001 << 0) #define DDRMON_CH0_COUNT_NUM 0x28 #define DDRMON_CH0_DFI_ACCESS_NUM 0x2c #define DDRMON_CH1_COUNT_NUM 0x3c #define DDRMON_CH1_DFI_ACCESS_NUM 0x40 +/* pmu grf */ +#define PMUGRF_OS_REG2 0x308 + +enum { + DDR4 = 0, + DDR3 = 3, + LPDDR2 = 5, + LPDDR3 = 6, + LPDDR4 = 7, + LPDDR4X = 8, + UNUSED = 0xFF +}; + struct dmc_usage { u32 access; u32 total; @@ -50,33 +97,261 @@ struct dmc_usage { struct rockchip_dfi { struct devfreq_event_dev *edev; struct devfreq_event_desc *desc; - struct dmc_usage ch_usage[RK3399_DMC_NUM_CH]; + struct dmc_usage ch_usage[MAX_DMC_NUM_CH]; struct device *dev; void __iomem *regs; struct regmap *regmap_pmu; + struct regmap *regmap_grf; + struct regmap *regmap_pmugrf; struct clk *clk; + u32 dram_type; + /* + * available mask, 1: available, 0: not available + * each bit represent a channel + */ + u32 ch_msk; +}; + +static void rk3128_dfi_start_hardware_counter(struct devfreq_event_dev *edev) +{ + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + + regmap_write(info->regmap_grf, + RK3128_GRF_SOC_CON0, + RK3128_DDR_MONITOR_EN); +} + +static void rk3128_dfi_stop_hardware_counter(struct devfreq_event_dev *edev) +{ + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + + regmap_write(info->regmap_grf, + RK3128_GRF_SOC_CON0, + RK3128_DDR_MONITOR_DISB); +} + +static int rk3128_dfi_disable(struct devfreq_event_dev *edev) +{ + rk3128_dfi_stop_hardware_counter(edev); + + return 0; +} + +static int rk3128_dfi_enable(struct devfreq_event_dev *edev) +{ + rk3128_dfi_start_hardware_counter(edev); + + return 0; +} + +static int rk3128_dfi_set_event(struct devfreq_event_dev *edev) +{ + return 0; +} + +static int rk3128_dfi_get_event(struct devfreq_event_dev *edev, + struct devfreq_event_data *edata) +{ + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + unsigned long flags; + u32 dfi_wr, dfi_rd, dfi_timer; + + local_irq_save(flags); + + rk3128_dfi_stop_hardware_counter(edev); + + regmap_read(info->regmap_grf, RK3128_GRF_DFI_WRNUM, &dfi_wr); + regmap_read(info->regmap_grf, RK3128_GRF_DFI_RDNUM, &dfi_rd); + regmap_read(info->regmap_grf, RK3128_GRF_DFI_TIMERVAL, &dfi_timer); + + edata->load_count = (dfi_wr + dfi_rd) * 4; + edata->total_count = dfi_timer; + + rk3128_dfi_start_hardware_counter(edev); + + local_irq_restore(flags); + + return 0; +} + +static const struct devfreq_event_ops rk3128_dfi_ops = { + .disable = rk3128_dfi_disable, + .enable = rk3128_dfi_enable, + .get_event = rk3128_dfi_get_event, + .set_event = rk3128_dfi_set_event, +}; + +static void rk3288_dfi_start_hardware_counter(struct devfreq_event_dev *edev) +{ + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + + regmap_write(info->regmap_grf, RK3288_GRF_SOC_CON4, RK3288_DFI_EN); +} + +static void rk3288_dfi_stop_hardware_counter(struct devfreq_event_dev *edev) +{ + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + + regmap_write(info->regmap_grf, RK3288_GRF_SOC_CON4, RK3288_DFI_DIS); +} + +static int rk3288_dfi_disable(struct devfreq_event_dev *edev) +{ + rk3288_dfi_stop_hardware_counter(edev); + + return 0; +} + +static int rk3288_dfi_enable(struct devfreq_event_dev *edev) +{ + rk3288_dfi_start_hardware_counter(edev); + + return 0; +} + +static int rk3288_dfi_set_event(struct devfreq_event_dev *edev) +{ + return 0; +} + +static int rk3288_dfi_get_busier_ch(struct devfreq_event_dev *edev) +{ + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + u32 tmp, max = 0; + u32 i, busier_ch = 0; + u32 rd_count, wr_count, total_count; + + rk3288_dfi_stop_hardware_counter(edev); + + /* Find out which channel is busier */ + for (i = 0; i < MAX_DMC_NUM_CH; i++) { + if (!(info->ch_msk & BIT(i))) + continue; + regmap_read(info->regmap_grf, + RK3288_GRF_SOC_STATUS(11 + i * 4), &wr_count); + regmap_read(info->regmap_grf, + RK3288_GRF_SOC_STATUS(12 + i * 4), &rd_count); + regmap_read(info->regmap_grf, + RK3288_GRF_SOC_STATUS(14 + i * 4), &total_count); + info->ch_usage[i].access = (wr_count + rd_count) * 4; + info->ch_usage[i].total = total_count; + tmp = info->ch_usage[i].access; + if (tmp > max) { + busier_ch = i; + max = tmp; + } + } + rk3288_dfi_start_hardware_counter(edev); + + return busier_ch; +} + +static int rk3288_dfi_get_event(struct devfreq_event_dev *edev, + struct devfreq_event_data *edata) +{ + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + int busier_ch; + unsigned long flags; + + local_irq_save(flags); + busier_ch = rk3288_dfi_get_busier_ch(edev); + local_irq_restore(flags); + + edata->load_count = info->ch_usage[busier_ch].access; + edata->total_count = info->ch_usage[busier_ch].total; + + return 0; +} + +static const struct devfreq_event_ops rk3288_dfi_ops = { + .disable = rk3288_dfi_disable, + .enable = rk3288_dfi_enable, + .get_event = rk3288_dfi_get_event, + .set_event = rk3288_dfi_set_event, +}; + +static void rk3368_dfi_start_hardware_counter(struct devfreq_event_dev *edev) +{ + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + + regmap_write(info->regmap_grf, RK3368_GRF_DDRC0_CON0, RK3368_DFI_EN); +} + +static void rk3368_dfi_stop_hardware_counter(struct devfreq_event_dev *edev) +{ + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + + regmap_write(info->regmap_grf, RK3368_GRF_DDRC0_CON0, RK3368_DFI_DIS); +} + +static int rk3368_dfi_disable(struct devfreq_event_dev *edev) +{ + rk3368_dfi_stop_hardware_counter(edev); + + return 0; +} + +static int rk3368_dfi_enable(struct devfreq_event_dev *edev) +{ + rk3368_dfi_start_hardware_counter(edev); + + return 0; +} + +static int rk3368_dfi_set_event(struct devfreq_event_dev *edev) +{ + return 0; +} + +static int rk3368_dfi_get_event(struct devfreq_event_dev *edev, + struct devfreq_event_data *edata) +{ + struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); + unsigned long flags; + u32 dfi0_wr, dfi0_rd, dfi1_wr, dfi1_rd, dfi_timer; + + local_irq_save(flags); + + rk3368_dfi_stop_hardware_counter(edev); + + regmap_read(info->regmap_grf, RK3368_GRF_SOC_STATUS5, &dfi0_wr); + regmap_read(info->regmap_grf, RK3368_GRF_SOC_STATUS6, &dfi0_rd); + regmap_read(info->regmap_grf, RK3368_GRF_SOC_STATUS9, &dfi1_wr); + regmap_read(info->regmap_grf, RK3368_GRF_SOC_STATUS10, &dfi1_rd); + regmap_read(info->regmap_grf, RK3368_GRF_SOC_STATUS8, &dfi_timer); + + edata->load_count = (dfi0_wr + dfi0_rd + dfi1_wr + dfi1_rd) * 2; + edata->total_count = dfi_timer; + + rk3368_dfi_start_hardware_counter(edev); + + local_irq_restore(flags); + + return 0; +} + +static const struct devfreq_event_ops rk3368_dfi_ops = { + .disable = rk3368_dfi_disable, + .enable = rk3368_dfi_enable, + .get_event = rk3368_dfi_get_event, + .set_event = rk3368_dfi_set_event, }; static void rockchip_dfi_start_hardware_counter(struct devfreq_event_dev *edev) { struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); void __iomem *dfi_regs = info->regs; - u32 val; - u32 ddr_type; - - /* get ddr type */ - regmap_read(info->regmap_pmu, RK3399_PMUGRF_OS_REG2, &val); - ddr_type = (val >> RK3399_PMUGRF_DDRTYPE_SHIFT) & - RK3399_PMUGRF_DDRTYPE_MASK; /* clear DDRMON_CTRL setting */ writel_relaxed(CLR_DDRMON_CTRL, dfi_regs + DDRMON_CTRL); /* set ddr type to dfi */ - if (ddr_type == RK3399_PMUGRF_DDRTYPE_LPDDR3) - writel_relaxed(LPDDR3_EN, dfi_regs + DDRMON_CTRL); - else if (ddr_type == RK3399_PMUGRF_DDRTYPE_LPDDR4) + if (info->dram_type == LPDDR3 || info->dram_type == LPDDR2) + writel_relaxed(LPDDR2_3_EN, dfi_regs + DDRMON_CTRL); + else if (info->dram_type == LPDDR4 || info->dram_type == LPDDR4X) writel_relaxed(LPDDR4_EN, dfi_regs + DDRMON_CTRL); + else if (info->dram_type == DDR4) + writel_relaxed(DDR4_EN, dfi_regs + DDRMON_CTRL); /* enable count, use software mode */ writel_relaxed(SOFTWARE_EN, dfi_regs + DDRMON_CTRL); @@ -100,12 +375,22 @@ static int rockchip_dfi_get_busier_ch(struct devfreq_event_dev *edev) rockchip_dfi_stop_hardware_counter(edev); /* Find out which channel is busier */ - for (i = 0; i < RK3399_DMC_NUM_CH; i++) { - info->ch_usage[i].access = readl_relaxed(dfi_regs + - DDRMON_CH0_DFI_ACCESS_NUM + i * 20) * 4; + for (i = 0; i < MAX_DMC_NUM_CH; i++) { + if (!(info->ch_msk & BIT(i))) + continue; + info->ch_usage[i].total = readl_relaxed(dfi_regs + DDRMON_CH0_COUNT_NUM + i * 20); - tmp = info->ch_usage[i].access; + + /* LPDDR4 and LPDDR4X BL = 16,other DDR type BL = 8 */ + tmp = readl_relaxed(dfi_regs + + DDRMON_CH0_DFI_ACCESS_NUM + i * 20); + if (info->dram_type == LPDDR4 || info->dram_type == LPDDR4X) + tmp *= 8; + else + tmp *= 4; + info->ch_usage[i].access = tmp; + if (tmp > max) { busier_ch = i; max = tmp; @@ -121,7 +406,8 @@ static int rockchip_dfi_disable(struct devfreq_event_dev *edev) struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); rockchip_dfi_stop_hardware_counter(edev); - clk_disable_unprepare(info->clk); + if (info->clk) + clk_disable_unprepare(info->clk); return 0; } @@ -131,10 +417,13 @@ static int rockchip_dfi_enable(struct devfreq_event_dev *edev) struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); int ret; - ret = clk_prepare_enable(info->clk); - if (ret) { - dev_err(&edev->dev, "failed to enable dfi clk: %d\n", ret); - return ret; + if (info->clk) { + ret = clk_prepare_enable(info->clk); + if (ret) { + dev_err(&edev->dev, "failed to enable dfi clk: %d\n", + ret); + return ret; + } } rockchip_dfi_start_hardware_counter(edev); @@ -151,8 +440,11 @@ static int rockchip_dfi_get_event(struct devfreq_event_dev *edev, { struct rockchip_dfi *info = devfreq_event_get_drvdata(edev); int busier_ch; + unsigned long flags; + local_irq_save(flags); busier_ch = rockchip_dfi_get_busier_ch(edev); + local_irq_restore(flags); edata->load_count = info->ch_usage[busier_ch].access; edata->total_count = info->ch_usage[busier_ch].total; @@ -167,22 +459,120 @@ static const struct devfreq_event_ops rockchip_dfi_ops = { .set_event = rockchip_dfi_set_event, }; -static const struct of_device_id rockchip_dfi_id_match[] = { - { .compatible = "rockchip,rk3399-dfi" }, - { }, -}; -MODULE_DEVICE_TABLE(of, rockchip_dfi_id_match); +static __init int px30_dfi_init(struct platform_device *pdev, + struct rockchip_dfi *data, + struct devfreq_event_desc *desc) +{ + struct device_node *np = pdev->dev.of_node, *node; + struct resource *res; + u32 val_2, val_3; -static int rockchip_dfi_probe(struct platform_device *pdev) + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + data->regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(data->regs)) + return PTR_ERR(data->regs); + + node = of_parse_phandle(np, "rockchip,pmugrf", 0); + if (node) { + data->regmap_pmugrf = syscon_node_to_regmap(node); + if (IS_ERR(data->regmap_pmugrf)) + return PTR_ERR(data->regmap_pmugrf); + } + + regmap_read(data->regmap_pmugrf, PX30_PMUGRF_OS_REG2, &val_2); + regmap_read(data->regmap_pmugrf, PX30_PMUGRF_OS_REG3, &val_3); + if (READ_SYSREG_VERSION(val_3) >= 0x3) + data->dram_type = READ_DRAMTYPE_INFO_V3(val_2, val_3); + else + data->dram_type = READ_DRAMTYPE_INFO(val_2); + data->ch_msk = 1; + data->clk = NULL; + + desc->ops = &rockchip_dfi_ops; + + return 0; +} + +static __init int rk3128_dfi_init(struct platform_device *pdev, + struct rockchip_dfi *data, + struct devfreq_event_desc *desc) { - struct device *dev = &pdev->dev; - struct rockchip_dfi *data; - struct devfreq_event_desc *desc; struct device_node *np = pdev->dev.of_node, *node; - data = devm_kzalloc(dev, sizeof(struct rockchip_dfi), GFP_KERNEL); - if (!data) - return -ENOMEM; + node = of_parse_phandle(np, "rockchip,grf", 0); + if (node) { + data->regmap_grf = syscon_node_to_regmap(node); + if (IS_ERR(data->regmap_grf)) + return PTR_ERR(data->regmap_grf); + } + + desc->ops = &rk3128_dfi_ops; + + return 0; +} + +static __init int rk3288_dfi_init(struct platform_device *pdev, + struct rockchip_dfi *data, + struct devfreq_event_desc *desc) +{ + struct device_node *np = pdev->dev.of_node, *node; + u32 val; + + node = of_parse_phandle(np, "rockchip,pmu", 0); + if (node) { + data->regmap_pmu = syscon_node_to_regmap(node); + if (IS_ERR(data->regmap_pmu)) + return PTR_ERR(data->regmap_pmu); + } + + node = of_parse_phandle(np, "rockchip,grf", 0); + if (node) { + data->regmap_grf = syscon_node_to_regmap(node); + if (IS_ERR(data->regmap_grf)) + return PTR_ERR(data->regmap_grf); + } + + regmap_read(data->regmap_pmu, RK3288_PMU_SYS_REG2, &val); + data->dram_type = READ_DRAMTYPE_INFO(val); + data->ch_msk = READ_CH_INFO(val); + + if (data->dram_type == DDR3) + regmap_write(data->regmap_grf, RK3288_GRF_SOC_CON4, + RK3288_DDR3_SEL); + else + regmap_write(data->regmap_grf, RK3288_GRF_SOC_CON4, + RK3288_LPDDR_SEL); + + desc->ops = &rk3288_dfi_ops; + + return 0; +} + +static __init int rk3368_dfi_init(struct platform_device *pdev, + struct rockchip_dfi *data, + struct devfreq_event_desc *desc) +{ + struct device *dev = &pdev->dev; + + if (!dev->parent || !dev->parent->of_node) + return -EINVAL; + + data->regmap_grf = syscon_node_to_regmap(dev->parent->of_node); + if (IS_ERR(data->regmap_grf)) + return PTR_ERR(data->regmap_grf); + + desc->ops = &rk3368_dfi_ops; + + return 0; +} + +static __init int rockchip_dfi_init(struct platform_device *pdev, + struct rockchip_dfi *data, + struct devfreq_event_desc *desc) +{ + struct device *dev = &pdev->dev; + struct device_node *np = pdev->dev.of_node, *node; + u32 val; data->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(data->regs)) @@ -202,23 +592,100 @@ static int rockchip_dfi_probe(struct platform_device *pdev) if (IS_ERR(data->regmap_pmu)) return PTR_ERR(data->regmap_pmu); } - data->dev = dev; + + regmap_read(data->regmap_pmu, PMUGRF_OS_REG2, &val); + data->dram_type = READ_DRAMTYPE_INFO(val); + data->ch_msk = READ_CH_INFO(val); + + desc->ops = &rockchip_dfi_ops; + + return 0; +} + +static __init int rk3328_dfi_init(struct platform_device *pdev, + struct rockchip_dfi *data, + struct devfreq_event_desc *desc) +{ + struct device_node *np = pdev->dev.of_node, *node; + struct resource *res; + u32 val; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + data->regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(data->regs)) + return PTR_ERR(data->regs); + + node = of_parse_phandle(np, "rockchip,grf", 0); + if (node) { + data->regmap_grf = syscon_node_to_regmap(node); + if (IS_ERR(data->regmap_grf)) + return PTR_ERR(data->regmap_grf); + } + + regmap_read(data->regmap_grf, RK3328_GRF_OS_REG2, &val); + data->dram_type = READ_DRAMTYPE_INFO(val); + data->ch_msk = 1; + data->clk = NULL; + + desc->ops = &rockchip_dfi_ops; + + return 0; +} + +static const struct of_device_id rockchip_dfi_id_match[] = { + { .compatible = "rockchip,px30-dfi", .data = px30_dfi_init }, + { .compatible = "rockchip,rk1808-dfi", .data = px30_dfi_init }, + { .compatible = "rockchip,rk3128-dfi", .data = rk3128_dfi_init }, + { .compatible = "rockchip,rk3288-dfi", .data = rk3288_dfi_init }, + { .compatible = "rockchip,rk3328-dfi", .data = rk3328_dfi_init }, + { .compatible = "rockchip,rk3368-dfi", .data = rk3368_dfi_init }, + { .compatible = "rockchip,rk3399-dfi", .data = rockchip_dfi_init }, + { .compatible = "rockchip,rk3568-dfi", .data = px30_dfi_init }, + { .compatible = "rockchip,rv1126-dfi", .data = px30_dfi_init }, + { }, +}; + +static int rockchip_dfi_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rockchip_dfi *data; + struct devfreq_event_desc *desc; + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + int (*init)(struct platform_device *pdev, struct rockchip_dfi *data, + struct devfreq_event_desc *desc); + + data = devm_kzalloc(dev, sizeof(struct rockchip_dfi), GFP_KERNEL); + if (!data) + return -ENOMEM; desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL); if (!desc) return -ENOMEM; - desc->ops = &rockchip_dfi_ops; + match = of_match_node(rockchip_dfi_id_match, pdev->dev.of_node); + if (match) { + init = match->data; + if (init) { + if (init(pdev, data, desc)) + return -EINVAL; + } else { + return 0; + } + } else { + return 0; + } + desc->driver_data = data; desc->name = np->name; - data->desc = desc; - data->edev = devm_devfreq_event_add_edev(&pdev->dev, desc); + data->edev = devm_devfreq_event_add_edev(dev, desc); if (IS_ERR(data->edev)) { - dev_err(&pdev->dev, - "failed to add devfreq-event device\n"); + dev_err(dev, "failed to add devfreq-event device\n"); return PTR_ERR(data->edev); } + data->desc = desc; + data->dev = &pdev->dev; platform_set_drvdata(pdev, data); diff --git a/drivers/devfreq/event/rockchip-nocp.c b/drivers/devfreq/event/rockchip-nocp.c new file mode 100755 index 000000000000..957b84ee3290 --- /dev/null +++ b/drivers/devfreq/event/rockchip-nocp.c @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define EVENT_BYTE 0x08 +#define EVENT_CHAIN 0x10 + +#define START_EN BIT(3) +#define GLOBAL_EN BIT(0) +#define START_GO BIT(0) + +#define PROBE_MAINCTL 0x0008 +#define PROBE_CFGCTL 0x000c +#define PROBE_STATPERIOD 0x0024 +#define PROBE_STATGO 0x0028 +#define PROBE_COUNTERS_0_SRC 0x0138 +#define PROBE_COUNTERS_0_VAL 0x013c +#define PROBE_COUNTERS_1_SRC 0x014c +#define PROBE_COUNTERS_1_VAL 0x0150 + +struct rockchip_nocp { + void __iomem *reg_base; + struct device *dev; + struct devfreq_event_dev *edev; + struct devfreq_event_desc *desc; + ktime_t time; +}; + +static int rockchip_nocp_enable(struct devfreq_event_dev *edev) +{ + struct rockchip_nocp *nocp = devfreq_event_get_drvdata(edev); + void __iomem *reg_base = nocp->reg_base; + + writel_relaxed(GLOBAL_EN, reg_base + PROBE_CFGCTL); + writel_relaxed(START_EN, reg_base + PROBE_MAINCTL); + writel_relaxed(0, reg_base + PROBE_STATPERIOD); + writel_relaxed(EVENT_BYTE, reg_base + PROBE_COUNTERS_0_SRC); + writel_relaxed(EVENT_CHAIN, reg_base + PROBE_COUNTERS_1_SRC); + writel_relaxed(START_GO, reg_base + PROBE_STATGO); + + nocp->time = ktime_get(); + + return 0; +} + +static int rockchip_nocp_disable(struct devfreq_event_dev *edev) +{ + struct rockchip_nocp *nocp = devfreq_event_get_drvdata(edev); + void __iomem *reg_base = nocp->reg_base; + + writel_relaxed(0, reg_base + PROBE_STATGO); + writel_relaxed(0, reg_base + PROBE_MAINCTL); + writel_relaxed(0, reg_base + PROBE_CFGCTL); + writel_relaxed(0, reg_base + PROBE_COUNTERS_0_SRC); + writel_relaxed(0, reg_base + PROBE_COUNTERS_1_SRC); + + return 0; +} + +static int rockchip_nocp_get_event(struct devfreq_event_dev *edev, + struct devfreq_event_data *edata) +{ + struct rockchip_nocp *nocp = devfreq_event_get_drvdata(edev); + void __iomem *reg_base = nocp->reg_base; + u32 counter = 0, counter0 = 0, counter1 = 0; + int time_ms = 0; + + time_ms = ktime_to_ms(ktime_sub(ktime_get(), nocp->time)); + + counter0 = readl_relaxed(reg_base + PROBE_COUNTERS_0_VAL); + counter1 = readl_relaxed(reg_base + PROBE_COUNTERS_1_VAL); + counter = (counter0 & 0xffff) | ((counter1 & 0xffff) << 16); + counter = counter / 1000000; + if (time_ms > 0) + edata->load_count = (counter * 1000) / time_ms; + + writel_relaxed(START_GO, reg_base + PROBE_STATGO); + nocp->time = ktime_get(); + + return 0; +} + +static int rockchip_nocp_set_event(struct devfreq_event_dev *edev) +{ + return 0; +} + +static const struct devfreq_event_ops rockchip_nocp_ops = { + .disable = rockchip_nocp_disable, + .enable = rockchip_nocp_enable, + .get_event = rockchip_nocp_get_event, + .set_event = rockchip_nocp_set_event, +}; + +static const struct of_device_id rockchip_nocp_id_match[] = { + { .compatible = "rockchip,rk3288-nocp" }, + { .compatible = "rockchip,rk3368-nocp" }, + { .compatible = "rockchip,rk3399-nocp" }, + { }, +}; + +static int rockchip_nocp_probe(struct platform_device *pdev) +{ + struct resource *res; + struct rockchip_nocp *nocp; + struct devfreq_event_desc *desc; + struct device_node *np = pdev->dev.of_node; + + nocp = devm_kzalloc(&pdev->dev, sizeof(*nocp), GFP_KERNEL); + if (!nocp) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + nocp->reg_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(nocp->reg_base)) + return PTR_ERR(nocp->reg_base); + + desc = devm_kzalloc(&pdev->dev, sizeof(*desc), GFP_KERNEL); + if (!desc) + return -ENOMEM; + + desc->ops = &rockchip_nocp_ops; + desc->driver_data = nocp; + desc->name = np->name; + nocp->desc = desc; + nocp->dev = &pdev->dev; + nocp->edev = devm_devfreq_event_add_edev(&pdev->dev, desc); + if (IS_ERR(nocp->edev)) { + dev_err(&pdev->dev, "failed to add devfreq-event device\n"); + return PTR_ERR(nocp->edev); + } + + platform_set_drvdata(pdev, nocp); + + return 0; +} + +static struct platform_driver rockchip_nocp_driver = { + .probe = rockchip_nocp_probe, + .driver = { + .name = "rockchip-nocp", + .of_match_table = rockchip_nocp_id_match, + }, +}; +module_platform_driver(rockchip_nocp_driver); + +MODULE_DESCRIPTION("Rockchip NoC (Network on Chip) Probe driver"); +MODULE_AUTHOR("Finley Xiao "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/devfreq/rk3399_dmc.c b/drivers/devfreq/rk3399_dmc.c deleted file mode 100644 index 7e52375d9818..000000000000 --- a/drivers/devfreq/rk3399_dmc.c +++ /dev/null @@ -1,518 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd. - * Author: Lin Huang - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -struct dram_timing { - unsigned int ddr3_speed_bin; - unsigned int pd_idle; - unsigned int sr_idle; - unsigned int sr_mc_gate_idle; - unsigned int srpd_lite_idle; - unsigned int standby_idle; - unsigned int auto_pd_dis_freq; - unsigned int dram_dll_dis_freq; - unsigned int phy_dll_dis_freq; - unsigned int ddr3_odt_dis_freq; - unsigned int ddr3_drv; - unsigned int ddr3_odt; - unsigned int phy_ddr3_ca_drv; - unsigned int phy_ddr3_dq_drv; - unsigned int phy_ddr3_odt; - unsigned int lpddr3_odt_dis_freq; - unsigned int lpddr3_drv; - unsigned int lpddr3_odt; - unsigned int phy_lpddr3_ca_drv; - unsigned int phy_lpddr3_dq_drv; - unsigned int phy_lpddr3_odt; - unsigned int lpddr4_odt_dis_freq; - unsigned int lpddr4_drv; - unsigned int lpddr4_dq_odt; - unsigned int lpddr4_ca_odt; - unsigned int phy_lpddr4_ca_drv; - unsigned int phy_lpddr4_ck_cs_drv; - unsigned int phy_lpddr4_dq_drv; - unsigned int phy_lpddr4_odt; -}; - -struct rk3399_dmcfreq { - struct device *dev; - struct devfreq *devfreq; - struct devfreq_simple_ondemand_data ondemand_data; - struct clk *dmc_clk; - struct devfreq_event_dev *edev; - struct mutex lock; - struct dram_timing timing; - struct regulator *vdd_center; - struct regmap *regmap_pmu; - unsigned long rate, target_rate; - unsigned long volt, target_volt; - unsigned int odt_dis_freq; - int odt_pd_arg0, odt_pd_arg1; -}; - -static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq, - u32 flags) -{ - struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); - struct dev_pm_opp *opp; - unsigned long old_clk_rate = dmcfreq->rate; - unsigned long target_volt, target_rate; - struct arm_smccc_res res; - bool odt_enable = false; - int err; - - opp = devfreq_recommended_opp(dev, freq, flags); - if (IS_ERR(opp)) - return PTR_ERR(opp); - - target_rate = dev_pm_opp_get_freq(opp); - target_volt = dev_pm_opp_get_voltage(opp); - dev_pm_opp_put(opp); - - if (dmcfreq->rate == target_rate) - return 0; - - mutex_lock(&dmcfreq->lock); - - if (dmcfreq->regmap_pmu) { - if (target_rate >= dmcfreq->odt_dis_freq) - odt_enable = true; - - /* - * This makes a SMC call to the TF-A to set the DDR PD - * (power-down) timings and to enable or disable the - * ODT (on-die termination) resistors. - */ - arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, dmcfreq->odt_pd_arg0, - dmcfreq->odt_pd_arg1, - ROCKCHIP_SIP_CONFIG_DRAM_SET_ODT_PD, - odt_enable, 0, 0, 0, &res); - } - - /* - * If frequency scaling from low to high, adjust voltage first. - * If frequency scaling from high to low, adjust frequency first. - */ - if (old_clk_rate < target_rate) { - err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, - target_volt); - if (err) { - dev_err(dev, "Cannot set voltage %lu uV\n", - target_volt); - goto out; - } - } - - err = clk_set_rate(dmcfreq->dmc_clk, target_rate); - if (err) { - dev_err(dev, "Cannot set frequency %lu (%d)\n", target_rate, - err); - regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt, - dmcfreq->volt); - goto out; - } - - /* - * Check the dpll rate, - * There only two result we will get, - * 1. Ddr frequency scaling fail, we still get the old rate. - * 2. Ddr frequency scaling sucessful, we get the rate we set. - */ - dmcfreq->rate = clk_get_rate(dmcfreq->dmc_clk); - - /* If get the incorrect rate, set voltage to old value. */ - if (dmcfreq->rate != target_rate) { - dev_err(dev, "Got wrong frequency, Request %lu, Current %lu\n", - target_rate, dmcfreq->rate); - regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt, - dmcfreq->volt); - goto out; - } else if (old_clk_rate > target_rate) - err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, - target_volt); - if (err) - dev_err(dev, "Cannot set voltage %lu uV\n", target_volt); - - dmcfreq->rate = target_rate; - dmcfreq->volt = target_volt; - -out: - mutex_unlock(&dmcfreq->lock); - return err; -} - -static int rk3399_dmcfreq_get_dev_status(struct device *dev, - struct devfreq_dev_status *stat) -{ - struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); - struct devfreq_event_data edata; - int ret = 0; - - ret = devfreq_event_get_event(dmcfreq->edev, &edata); - if (ret < 0) - return ret; - - stat->current_frequency = dmcfreq->rate; - stat->busy_time = edata.load_count; - stat->total_time = edata.total_count; - - return ret; -} - -static int rk3399_dmcfreq_get_cur_freq(struct device *dev, unsigned long *freq) -{ - struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); - - *freq = dmcfreq->rate; - - return 0; -} - -static struct devfreq_dev_profile rk3399_devfreq_dmc_profile = { - .polling_ms = 200, - .target = rk3399_dmcfreq_target, - .get_dev_status = rk3399_dmcfreq_get_dev_status, - .get_cur_freq = rk3399_dmcfreq_get_cur_freq, -}; - -static __maybe_unused int rk3399_dmcfreq_suspend(struct device *dev) -{ - struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); - int ret = 0; - - ret = devfreq_event_disable_edev(dmcfreq->edev); - if (ret < 0) { - dev_err(dev, "failed to disable the devfreq-event devices\n"); - return ret; - } - - ret = devfreq_suspend_device(dmcfreq->devfreq); - if (ret < 0) { - dev_err(dev, "failed to suspend the devfreq devices\n"); - return ret; - } - - return 0; -} - -static __maybe_unused int rk3399_dmcfreq_resume(struct device *dev) -{ - struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); - int ret = 0; - - ret = devfreq_event_enable_edev(dmcfreq->edev); - if (ret < 0) { - dev_err(dev, "failed to enable the devfreq-event devices\n"); - return ret; - } - - ret = devfreq_resume_device(dmcfreq->devfreq); - if (ret < 0) { - dev_err(dev, "failed to resume the devfreq devices\n"); - return ret; - } - return ret; -} - -static SIMPLE_DEV_PM_OPS(rk3399_dmcfreq_pm, rk3399_dmcfreq_suspend, - rk3399_dmcfreq_resume); - -static int of_get_ddr_timings(struct dram_timing *timing, - struct device_node *np) -{ - int ret = 0; - - ret = of_property_read_u32(np, "rockchip,ddr3_speed_bin", - &timing->ddr3_speed_bin); - ret |= of_property_read_u32(np, "rockchip,pd_idle", - &timing->pd_idle); - ret |= of_property_read_u32(np, "rockchip,sr_idle", - &timing->sr_idle); - ret |= of_property_read_u32(np, "rockchip,sr_mc_gate_idle", - &timing->sr_mc_gate_idle); - ret |= of_property_read_u32(np, "rockchip,srpd_lite_idle", - &timing->srpd_lite_idle); - ret |= of_property_read_u32(np, "rockchip,standby_idle", - &timing->standby_idle); - ret |= of_property_read_u32(np, "rockchip,auto_pd_dis_freq", - &timing->auto_pd_dis_freq); - ret |= of_property_read_u32(np, "rockchip,dram_dll_dis_freq", - &timing->dram_dll_dis_freq); - ret |= of_property_read_u32(np, "rockchip,phy_dll_dis_freq", - &timing->phy_dll_dis_freq); - ret |= of_property_read_u32(np, "rockchip,ddr3_odt_dis_freq", - &timing->ddr3_odt_dis_freq); - ret |= of_property_read_u32(np, "rockchip,ddr3_drv", - &timing->ddr3_drv); - ret |= of_property_read_u32(np, "rockchip,ddr3_odt", - &timing->ddr3_odt); - ret |= of_property_read_u32(np, "rockchip,phy_ddr3_ca_drv", - &timing->phy_ddr3_ca_drv); - ret |= of_property_read_u32(np, "rockchip,phy_ddr3_dq_drv", - &timing->phy_ddr3_dq_drv); - ret |= of_property_read_u32(np, "rockchip,phy_ddr3_odt", - &timing->phy_ddr3_odt); - ret |= of_property_read_u32(np, "rockchip,lpddr3_odt_dis_freq", - &timing->lpddr3_odt_dis_freq); - ret |= of_property_read_u32(np, "rockchip,lpddr3_drv", - &timing->lpddr3_drv); - ret |= of_property_read_u32(np, "rockchip,lpddr3_odt", - &timing->lpddr3_odt); - ret |= of_property_read_u32(np, "rockchip,phy_lpddr3_ca_drv", - &timing->phy_lpddr3_ca_drv); - ret |= of_property_read_u32(np, "rockchip,phy_lpddr3_dq_drv", - &timing->phy_lpddr3_dq_drv); - ret |= of_property_read_u32(np, "rockchip,phy_lpddr3_odt", - &timing->phy_lpddr3_odt); - ret |= of_property_read_u32(np, "rockchip,lpddr4_odt_dis_freq", - &timing->lpddr4_odt_dis_freq); - ret |= of_property_read_u32(np, "rockchip,lpddr4_drv", - &timing->lpddr4_drv); - ret |= of_property_read_u32(np, "rockchip,lpddr4_dq_odt", - &timing->lpddr4_dq_odt); - ret |= of_property_read_u32(np, "rockchip,lpddr4_ca_odt", - &timing->lpddr4_ca_odt); - ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_ca_drv", - &timing->phy_lpddr4_ca_drv); - ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_ck_cs_drv", - &timing->phy_lpddr4_ck_cs_drv); - ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_dq_drv", - &timing->phy_lpddr4_dq_drv); - ret |= of_property_read_u32(np, "rockchip,phy_lpddr4_odt", - &timing->phy_lpddr4_odt); - - return ret; -} - -static int rk3399_dmcfreq_probe(struct platform_device *pdev) -{ - struct arm_smccc_res res; - struct device *dev = &pdev->dev; - struct device_node *np = pdev->dev.of_node, *node; - struct rk3399_dmcfreq *data; - int ret, index, size; - uint32_t *timing; - struct dev_pm_opp *opp; - u32 ddr_type; - u32 val; - - data = devm_kzalloc(dev, sizeof(struct rk3399_dmcfreq), GFP_KERNEL); - if (!data) - return -ENOMEM; - - mutex_init(&data->lock); - - data->vdd_center = devm_regulator_get(dev, "center"); - if (IS_ERR(data->vdd_center)) { - if (PTR_ERR(data->vdd_center) == -EPROBE_DEFER) - return -EPROBE_DEFER; - - dev_err(dev, "Cannot get the regulator \"center\"\n"); - return PTR_ERR(data->vdd_center); - } - - data->dmc_clk = devm_clk_get(dev, "dmc_clk"); - if (IS_ERR(data->dmc_clk)) { - if (PTR_ERR(data->dmc_clk) == -EPROBE_DEFER) - return -EPROBE_DEFER; - - dev_err(dev, "Cannot get the clk dmc_clk\n"); - return PTR_ERR(data->dmc_clk); - } - - data->edev = devfreq_event_get_edev_by_phandle(dev, "devfreq-events", 0); - if (IS_ERR(data->edev)) - return -EPROBE_DEFER; - - ret = devfreq_event_enable_edev(data->edev); - if (ret < 0) { - dev_err(dev, "failed to enable devfreq-event devices\n"); - return ret; - } - - /* - * Get dram timing and pass it to arm trust firmware, - * the dram driver in arm trust firmware will get these - * timing and to do dram initial. - */ - if (!of_get_ddr_timings(&data->timing, np)) { - timing = &data->timing.ddr3_speed_bin; - size = sizeof(struct dram_timing) / 4; - for (index = 0; index < size; index++) { - arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, *timing++, index, - ROCKCHIP_SIP_CONFIG_DRAM_SET_PARAM, - 0, 0, 0, 0, &res); - if (res.a0) { - dev_err(dev, "Failed to set dram param: %ld\n", - res.a0); - ret = -EINVAL; - goto err_edev; - } - } - } - - node = of_parse_phandle(np, "rockchip,pmu", 0); - if (!node) - goto no_pmu; - - data->regmap_pmu = syscon_node_to_regmap(node); - of_node_put(node); - if (IS_ERR(data->regmap_pmu)) { - ret = PTR_ERR(data->regmap_pmu); - goto err_edev; - } - - regmap_read(data->regmap_pmu, RK3399_PMUGRF_OS_REG2, &val); - ddr_type = (val >> RK3399_PMUGRF_DDRTYPE_SHIFT) & - RK3399_PMUGRF_DDRTYPE_MASK; - - switch (ddr_type) { - case RK3399_PMUGRF_DDRTYPE_DDR3: - data->odt_dis_freq = data->timing.ddr3_odt_dis_freq; - break; - case RK3399_PMUGRF_DDRTYPE_LPDDR3: - data->odt_dis_freq = data->timing.lpddr3_odt_dis_freq; - break; - case RK3399_PMUGRF_DDRTYPE_LPDDR4: - data->odt_dis_freq = data->timing.lpddr4_odt_dis_freq; - break; - default: - ret = -EINVAL; - goto err_edev; - }; - -no_pmu: - arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0, - ROCKCHIP_SIP_CONFIG_DRAM_INIT, - 0, 0, 0, 0, &res); - - /* - * In TF-A there is a platform SIP call to set the PD (power-down) - * timings and to enable or disable the ODT (on-die termination). - * This call needs three arguments as follows: - * - * arg0: - * bit[0-7] : sr_idle - * bit[8-15] : sr_mc_gate_idle - * bit[16-31] : standby idle - * arg1: - * bit[0-11] : pd_idle - * bit[16-27] : srpd_lite_idle - * arg2: - * bit[0] : odt enable - */ - data->odt_pd_arg0 = (data->timing.sr_idle & 0xff) | - ((data->timing.sr_mc_gate_idle & 0xff) << 8) | - ((data->timing.standby_idle & 0xffff) << 16); - data->odt_pd_arg1 = (data->timing.pd_idle & 0xfff) | - ((data->timing.srpd_lite_idle & 0xfff) << 16); - - /* - * We add a devfreq driver to our parent since it has a device tree node - * with operating points. - */ - if (dev_pm_opp_of_add_table(dev)) { - dev_err(dev, "Invalid operating-points in device tree.\n"); - ret = -EINVAL; - goto err_edev; - } - - of_property_read_u32(np, "upthreshold", - &data->ondemand_data.upthreshold); - of_property_read_u32(np, "downdifferential", - &data->ondemand_data.downdifferential); - - data->rate = clk_get_rate(data->dmc_clk); - - opp = devfreq_recommended_opp(dev, &data->rate, 0); - if (IS_ERR(opp)) { - ret = PTR_ERR(opp); - goto err_free_opp; - } - - data->rate = dev_pm_opp_get_freq(opp); - data->volt = dev_pm_opp_get_voltage(opp); - dev_pm_opp_put(opp); - - rk3399_devfreq_dmc_profile.initial_freq = data->rate; - - data->devfreq = devm_devfreq_add_device(dev, - &rk3399_devfreq_dmc_profile, - DEVFREQ_GOV_SIMPLE_ONDEMAND, - &data->ondemand_data); - if (IS_ERR(data->devfreq)) { - ret = PTR_ERR(data->devfreq); - goto err_free_opp; - } - - devm_devfreq_register_opp_notifier(dev, data->devfreq); - - data->dev = dev; - platform_set_drvdata(pdev, data); - - return 0; - -err_free_opp: - dev_pm_opp_of_remove_table(&pdev->dev); -err_edev: - devfreq_event_disable_edev(data->edev); - - return ret; -} - -static int rk3399_dmcfreq_remove(struct platform_device *pdev) -{ - struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(&pdev->dev); - - devfreq_event_disable_edev(dmcfreq->edev); - - /* - * Before remove the opp table we need to unregister the opp notifier. - */ - devm_devfreq_unregister_opp_notifier(dmcfreq->dev, dmcfreq->devfreq); - dev_pm_opp_of_remove_table(dmcfreq->dev); - - return 0; -} - -static const struct of_device_id rk3399dmc_devfreq_of_match[] = { - { .compatible = "rockchip,rk3399-dmc" }, - { }, -}; -MODULE_DEVICE_TABLE(of, rk3399dmc_devfreq_of_match); - -static struct platform_driver rk3399_dmcfreq_driver = { - .probe = rk3399_dmcfreq_probe, - .remove = rk3399_dmcfreq_remove, - .driver = { - .name = "rk3399-dmc-freq", - .pm = &rk3399_dmcfreq_pm, - .of_match_table = rk3399dmc_devfreq_of_match, - }, -}; -module_platform_driver(rk3399_dmcfreq_driver); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Lin Huang "); -MODULE_DESCRIPTION("RK3399 dmcfreq driver with devfreq framework"); diff --git a/drivers/devfreq/rockchip_bus.c b/drivers/devfreq/rockchip_bus.c new file mode 100755 index 000000000000..7032b4815c5a --- /dev/null +++ b/drivers/devfreq/rockchip_bus.c @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018, Fuzhou Rockchip Electronics Co., Ltd. + * Author: Tony Xie + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CLUSTER0 0 +#define CLUSTER1 1 +#define MAX_CLUSTERS 2 + +#define to_rockchip_bus_clk_nb(nb) \ + container_of(nb, struct rockchip_bus, clk_nb) +#define to_rockchip_bus_cpufreq_nb(nb) \ + container_of(nb, struct rockchip_bus, cpufreq_nb) + +struct busfreq_table { + unsigned long freq; + unsigned long volt; +}; + +struct rockchip_bus { + struct device *dev; + struct regulator *regulator; + struct clk *clk; + struct notifier_block clk_nb; + struct notifier_block cpufreq_nb; + struct busfreq_table *freq_table; + + unsigned int max_state; + + unsigned long cur_volt; + unsigned long cur_rate; + + /* + * Busfreq-policy-cpufreq: + * If the cpu frequency of two clusters are both less than or equal to + * cpu_high_freq, change bus rate to low_rate, otherwise change it to + * high_rate. + */ + unsigned long high_rate; + unsigned long low_rate; + unsigned int cpu_high_freq; + unsigned int cpu_freq[MAX_CLUSTERS]; +}; + +static int rockchip_sip_bus_smc_config(u32 bus_id, u32 cfg, u32 enable_msk) +{ + struct arm_smccc_res res; + + res = sip_smc_bus_config(bus_id, cfg, enable_msk); + + return res.a0; +} + +static int rockchip_bus_smc_config(struct rockchip_bus *bus) +{ + struct device *dev = bus->dev; + struct device_node *np = dev->of_node; + struct device_node *child; + unsigned int enable_msk, bus_id, cfg; + int ret; + + for_each_available_child_of_node(np, child) { + ret = of_property_read_u32_index(child, "bus-id", 0, + &bus_id); + if (ret) + continue; + + ret = of_property_read_u32_index(child, "cfg-val", 0, + &cfg); + if (ret) { + dev_info(dev, "get cfg-val error\n"); + continue; + } + + if (!cfg) { + dev_info(dev, "cfg-val invalid\n"); + continue; + } + + ret = of_property_read_u32_index(child, "enable-msk", 0, + &enable_msk); + if (ret) { + dev_info(dev, "get enable_msk error\n"); + continue; + } + + ret = rockchip_sip_bus_smc_config(bus_id, cfg, + enable_msk); + if (ret) { + dev_info(dev, "bus smc config error: %x!\n", ret); + break; + } + } + + return 0; +} + +static int rockchip_bus_set_freq_table(struct rockchip_bus *bus) +{ + struct device *dev = bus->dev; + struct dev_pm_opp *opp; + unsigned long freq; + int i, count; + + count = dev_pm_opp_get_opp_count(dev); + if (count <= 0) + return -EINVAL; + + bus->max_state = count; + bus->freq_table = devm_kcalloc(dev, + bus->max_state, + sizeof(*bus->freq_table), + GFP_KERNEL); + if (!bus->freq_table) { + bus->max_state = 0; + return -ENOMEM; + } + + for (i = 0, freq = 0; i < bus->max_state; i++, freq++) { + opp = dev_pm_opp_find_freq_ceil(dev, &freq); + if (IS_ERR(opp)) { + devm_kfree(dev, bus->freq_table); + bus->max_state = 0; + return PTR_ERR(opp); + } + bus->freq_table[i].volt = dev_pm_opp_get_voltage(opp); + bus->freq_table[i].freq = freq; + dev_pm_opp_put(opp); + } + + return 0; +} + +static int rockchip_bus_power_control_init(struct rockchip_bus *bus) +{ + struct device *dev = bus->dev; + int ret = 0; + + bus->clk = devm_clk_get(dev, "bus"); + if (IS_ERR(bus->clk)) { + dev_err(dev, "failed to get bus clock\n"); + return PTR_ERR(bus->clk); + } + + bus->regulator = devm_regulator_get(dev, "bus"); + if (IS_ERR(bus->regulator)) { + dev_err(dev, "failed to get bus regulator\n"); + return PTR_ERR(bus->regulator); + } + + ret = rockchip_init_opp_table(dev, NULL, "leakage", "pvtm"); + if (ret < 0) { + dev_err(dev, "failed to get OPP table\n"); + return ret; + } + + ret = rockchip_bus_set_freq_table(bus); + if (ret < 0) { + dev_err(dev, "failed to set bus freq table\n"); + return ret; + } + + return 0; +} + +static int rockchip_bus_clkfreq_target(struct device *dev, unsigned long freq) +{ + struct rockchip_bus *bus = dev_get_drvdata(dev); + unsigned long target_volt = bus->freq_table[bus->max_state - 1].volt; + int i; + + for (i = 0; i < bus->max_state; i++) { + if (freq <= bus->freq_table[i].freq) { + target_volt = bus->freq_table[i].volt; + break; + } + } + + if (bus->cur_volt != target_volt) { + dev_dbg(bus->dev, "target_volt: %lu\n", target_volt); + if (regulator_set_voltage(bus->regulator, target_volt, + INT_MAX)) { + dev_err(dev, "failed to set voltage %lu uV\n", + target_volt); + return -EINVAL; + } + bus->cur_volt = target_volt; + } + + return 0; +} + +static int rockchip_bus_clk_notifier(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct clk_notifier_data *ndata = data; + struct rockchip_bus *bus = to_rockchip_bus_clk_nb(nb); + int ret = 0; + + dev_dbg(bus->dev, "event %lu, old_rate %lu, new_rate: %lu\n", + event, ndata->old_rate, ndata->new_rate); + + switch (event) { + case PRE_RATE_CHANGE: + if (ndata->new_rate > ndata->old_rate) + ret = rockchip_bus_clkfreq_target(bus->dev, + ndata->new_rate); + break; + case POST_RATE_CHANGE: + if (ndata->new_rate < ndata->old_rate) + ret = rockchip_bus_clkfreq_target(bus->dev, + ndata->new_rate); + break; + case ABORT_RATE_CHANGE: + if (ndata->new_rate > ndata->old_rate) + ret = rockchip_bus_clkfreq_target(bus->dev, + ndata->old_rate); + break; + default: + break; + } + + return notifier_from_errno(ret); +} + +static int rockchip_bus_clkfreq(struct rockchip_bus *bus) +{ + struct device *dev = bus->dev; + unsigned long init_rate; + int ret = 0; + + ret = rockchip_bus_power_control_init(bus); + if (ret) { + dev_err(dev, "failed to init power control\n"); + return ret; + } + + init_rate = clk_get_rate(bus->clk); + ret = rockchip_bus_clkfreq_target(dev, init_rate); + if (ret) + return ret; + + bus->clk_nb.notifier_call = rockchip_bus_clk_notifier; + ret = clk_notifier_register(bus->clk, &bus->clk_nb); + if (ret) { + dev_err(dev, "failed to register clock notifier\n"); + return ret; + } + + return 0; +} + +static int rockchip_bus_cpufreq_target(struct device *dev, unsigned long freq, + u32 flags) +{ + struct rockchip_bus *bus = dev_get_drvdata(dev); + struct dev_pm_opp *opp; + unsigned long target_volt, target_rate = freq; + int ret = 0; + + if (!bus->regulator) { + dev_dbg(dev, "%luHz -> %luHz\n", bus->cur_rate, target_rate); + ret = clk_set_rate(bus->clk, target_rate); + if (ret) + dev_err(bus->dev, "failed to set bus rate %lu\n", + target_rate); + else + bus->cur_rate = target_rate; + return ret; + } + + opp = devfreq_recommended_opp(dev, &target_rate, flags); + if (IS_ERR(opp)) { + dev_err(dev, "failed to recommended opp %lu\n", target_rate); + return PTR_ERR(opp); + } + target_volt = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + + if (bus->cur_rate == target_rate) { + if (bus->cur_volt == target_volt) + return 0; + ret = regulator_set_voltage(bus->regulator, target_volt, + INT_MAX); + if (ret) { + dev_err(dev, "failed to set voltage %lu\n", + target_volt); + return ret; + } + bus->cur_volt = target_volt; + return 0; + } else if (!bus->cur_volt) { + bus->cur_volt = regulator_get_voltage(bus->regulator); + } + + if (bus->cur_rate < target_rate) { + ret = regulator_set_voltage(bus->regulator, target_volt, + INT_MAX); + if (ret) { + dev_err(dev, "failed to set voltage %lu\n", + target_volt); + return ret; + } + } + + ret = clk_set_rate(bus->clk, target_rate); + if (ret) { + dev_err(dev, "failed to set bus rate %lu\n", target_rate); + return ret; + } + + if (bus->cur_rate > target_rate) { + ret = regulator_set_voltage(bus->regulator, target_volt, + INT_MAX); + if (ret) { + dev_err(dev, "failed to set voltage %lu\n", + target_volt); + return ret; + } + } + + dev_dbg(dev, "%luHz %luuV -> %luHz %luuV\n", bus->cur_rate, + bus->cur_volt, target_rate, target_volt); + bus->cur_rate = target_rate; + bus->cur_volt = target_volt; + + return ret; +} + +static int rockchip_bus_cpufreq_notifier(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct rockchip_bus *bus = to_rockchip_bus_cpufreq_nb(nb); + struct cpufreq_freqs *freqs = data; + int id = topology_physical_package_id(freqs->policy->cpu); + + if (id < 0 || id >= MAX_CLUSTERS) + return NOTIFY_DONE; + + bus->cpu_freq[id] = freqs->new; + + if (!bus->cpu_freq[CLUSTER0] || !bus->cpu_freq[CLUSTER1]) + return NOTIFY_DONE; + + switch (event) { + case CPUFREQ_PRECHANGE: + if ((bus->cpu_freq[CLUSTER0] > bus->cpu_high_freq || + bus->cpu_freq[CLUSTER1] > bus->cpu_high_freq) && + bus->cur_rate != bus->high_rate) { + dev_dbg(bus->dev, "cpu%d freq=%d %d, up cci rate to %lu\n", + freqs->policy->cpu, + bus->cpu_freq[CLUSTER0], + bus->cpu_freq[CLUSTER1], + bus->high_rate); + rockchip_bus_cpufreq_target(bus->dev, bus->high_rate, + 0); + } + break; + case CPUFREQ_POSTCHANGE: + if (bus->cpu_freq[CLUSTER0] <= bus->cpu_high_freq && + bus->cpu_freq[CLUSTER1] <= bus->cpu_high_freq && + bus->cur_rate != bus->low_rate) { + dev_dbg(bus->dev, "cpu%d freq=%d %d, down cci rate to %lu\n", + freqs->policy->cpu, + bus->cpu_freq[CLUSTER0], + bus->cpu_freq[CLUSTER1], + bus->low_rate); + rockchip_bus_cpufreq_target(bus->dev, bus->low_rate, + 0); + } + break; + } + + return NOTIFY_OK; +} + +static int rockchip_bus_cpufreq(struct rockchip_bus *bus) +{ + struct device *dev = bus->dev; + struct device_node *np = dev->of_node; + unsigned int freq; + int ret = 0; + + if (of_parse_phandle(dev->of_node, "operating-points-v2", 0)) { + ret = rockchip_bus_power_control_init(bus); + if (ret) { + dev_err(dev, "failed to init power control\n"); + return ret; + } + } else { + bus->clk = devm_clk_get(dev, "bus"); + if (IS_ERR(bus->clk)) { + dev_err(dev, "failed to get bus clock\n"); + return PTR_ERR(bus->clk); + } + bus->regulator = NULL; + } + + ret = of_property_read_u32(np, "cpu-high-freq", &bus->cpu_high_freq); + if (ret) { + dev_err(dev, "failed to get cpu-high-freq\n"); + return ret; + } + ret = of_property_read_u32(np, "cci-high-freq", &freq); + if (ret) { + dev_err(dev, "failed to get cci-high-freq\n"); + return ret; + } + bus->high_rate = freq * 1000; + ret = of_property_read_u32(np, "cci-low-freq", &freq); + if (ret) { + dev_err(dev, "failed to get cci-low-freq\n"); + return ret; + } + bus->low_rate = freq * 1000; + + bus->cpufreq_nb.notifier_call = rockchip_bus_cpufreq_notifier; + ret = cpufreq_register_notifier(&bus->cpufreq_nb, + CPUFREQ_TRANSITION_NOTIFIER); + if (ret) { + dev_err(dev, "failed to register cpufreq notifier\n"); + return ret; + } + + return 0; +} + +static const struct of_device_id rockchip_busfreq_of_match[] = { + { .compatible = "rockchip,px30-bus", }, + { .compatible = "rockchip,rk1808-bus", }, + { .compatible = "rockchip,rk3288-bus", }, + { .compatible = "rockchip,rk3368-bus", }, + { .compatible = "rockchip,rk3399-bus", }, + { .compatible = "rockchip,rk3568-bus", }, + { .compatible = "rockchip,rv1126-bus", }, + { }, +}; + +MODULE_DEVICE_TABLE(of, rockchip_busfreq_of_match); + +static int rockchip_busfreq_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct rockchip_bus *bus; + const char *policy_name; + int ret = 0; + + bus = devm_kzalloc(dev, sizeof(*bus), GFP_KERNEL); + if (!bus) + return -ENOMEM; + bus->dev = dev; + platform_set_drvdata(pdev, bus); + + ret = of_property_read_string(np, "rockchip,busfreq-policy", + &policy_name); + if (ret) { + dev_info(dev, "failed to get busfreq policy\n"); + return ret; + } + + if (!strcmp(policy_name, "smc")) + ret = rockchip_bus_smc_config(bus); + else if (!strcmp(policy_name, "clkfreq")) + ret = rockchip_bus_clkfreq(bus); + else if (!strcmp(policy_name, "cpufreq")) + ret = rockchip_bus_cpufreq(bus); + + return ret; +} + +static struct platform_driver rockchip_busfreq_driver = { + .probe = rockchip_busfreq_probe, + .driver = { + .name = "rockchip,bus", + .of_match_table = rockchip_busfreq_of_match, + }, +}; + +module_platform_driver(rockchip_busfreq_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Tony Xie "); +MODULE_DESCRIPTION("rockchip busfreq driver with devfreq framework"); diff --git a/drivers/devfreq/rockchip_dmc.c b/drivers/devfreq/rockchip_dmc.c new file mode 100755 index 000000000000..2df1edb1caf3 --- /dev/null +++ b/drivers/devfreq/rockchip_dmc.c @@ -0,0 +1,3283 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Rockchip Generic dmc support. + * + * Copyright (c) 2021 Rockchip Electronics Co. Ltd. + * Author: Finley Xiao + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "governor.h" +#include "rockchip_dmc_timing.h" +#include "../clk/rockchip/clk.h" +#include "../gpu/drm/rockchip/rockchip_drm_drv.h" + +#define system_status_to_dmcfreq(nb) container_of(nb, struct rockchip_dmcfreq, \ + status_nb) +#define reboot_to_dmcfreq(nb) container_of(nb, struct rockchip_dmcfreq, \ + reboot_nb) +#define boost_to_dmcfreq(work) container_of(work, struct rockchip_dmcfreq, \ + boost_work) +#define input_hd_to_dmcfreq(hd) container_of(hd, struct rockchip_dmcfreq, \ + input_handler) + +#define VIDEO_1080P_SIZE (1920 * 1080) +#define FIQ_INIT_HANDLER (0x1) +#define FIQ_CPU_TGT_BOOT (0x0) /* to booting cpu */ +#define FIQ_NUM_FOR_DCF (143) /* NA irq map to fiq for dcf */ +#define DTS_PAR_OFFSET (4096) + +#define FALLBACK_STATIC_TEMPERATURE 55000 + +struct dmc_freq_table { + unsigned long freq; + unsigned long volt; +}; + +struct share_params { + u32 hz; + u32 lcdc_type; + u32 vop; + u32 vop_dclk_mode; + u32 sr_idle_en; + u32 addr_mcu_el3; + /* + * 1: need to wait flag1 + * 0: never wait flag1 + */ + u32 wait_flag1; + /* + * 1: need to wait flag1 + * 0: never wait flag1 + */ + u32 wait_flag0; + u32 complt_hwirq; + u32 update_drv_odt_cfg; + u32 update_deskew_cfg; + + u32 freq_count; + u32 freq_info_mhz[6]; + /* if need, add parameter after */ +}; + +static struct share_params *ddr_psci_param; + +struct rockchip_dmcfreq_ondemand_data { + unsigned int upthreshold; + unsigned int downdifferential; +}; + +struct rockchip_dmcfreq { + struct device *dev; + struct dmcfreq_common_info info; + struct rockchip_dmcfreq_ondemand_data ondemand_data; + struct clk *dmc_clk; + struct devfreq_event_dev **edev; + struct mutex lock; /* serializes access to video_info_list */ + struct dram_timing *timing; + struct regulator *vdd_center; + struct notifier_block status_nb; + struct list_head video_info_list; + struct freq_map_table *cpu_bw_tbl; + struct work_struct boost_work; + struct input_handler input_handler; + struct monitor_dev_info *mdev_info; + struct share_params *set_rate_params; + + unsigned long *nocp_bw; + unsigned long rate, target_rate; + unsigned long volt, target_volt; + unsigned long auto_min_rate; + unsigned long status_rate; + unsigned long normal_rate; + unsigned long video_1080p_rate; + unsigned long video_4k_rate; + unsigned long video_4k_10b_rate; + unsigned long performance_rate; + unsigned long hdmi_rate; + unsigned long idle_rate; + unsigned long suspend_rate; + unsigned long reboot_rate; + unsigned long boost_rate; + unsigned long fixed_rate; + unsigned long low_power_rate; + + unsigned long freq_count; + unsigned long freq_info_rate[6]; + unsigned long rate_low; + unsigned long rate_mid_low; + unsigned long rate_mid_high; + unsigned long rate_high; + + unsigned int min_cpu_freq; + unsigned int system_status_en; + unsigned int refresh; + int edev_count; + int dfi_id; + int nocp_cpu_id; + + bool is_fixed; + bool is_set_rate_direct; + + struct thermal_cooling_device *devfreq_cooling; + u32 static_coefficient; + s32 ts[4]; + struct thermal_zone_device *ddr_tz; + + unsigned int touchboostpulse_duration_val; + u64 touchboostpulse_endtime; + + int (*set_auto_self_refresh)(u32 en); +}; + +static struct pm_qos_request pm_qos; + +static inline unsigned long is_dualview(unsigned long status) +{ + return (status & SYS_STATUS_LCDC0) && (status & SYS_STATUS_LCDC1); +} + +static inline unsigned long is_isp(unsigned long status) +{ + return (status & SYS_STATUS_ISP) || + (status & SYS_STATUS_CIF0) || + (status & SYS_STATUS_CIF1); +} + +/* + * function: packaging de-skew setting to px30_ddr_dts_config_timing, + * px30_ddr_dts_config_timing will pass to trust firmware, and + * used direct to set register. + * input: de_skew + * output: tim + */ +static void px30_de_skew_set_2_reg(struct rk3328_ddr_de_skew_setting *de_skew, + struct px30_ddr_dts_config_timing *tim) +{ + u32 n; + u32 offset; + u32 shift; + + memset_io(tim->ca_skew, 0, sizeof(tim->ca_skew)); + memset_io(tim->cs0_skew, 0, sizeof(tim->cs0_skew)); + memset_io(tim->cs1_skew, 0, sizeof(tim->cs1_skew)); + + /* CA de-skew */ + for (n = 0; n < ARRAY_SIZE(de_skew->ca_de_skew); n++) { + offset = n / 2; + shift = n % 2; + /* 0 => 4; 1 => 0 */ + shift = (shift == 0) ? 4 : 0; + tim->ca_skew[offset] &= ~(0xf << shift); + tim->ca_skew[offset] |= (de_skew->ca_de_skew[n] << shift); + } + + /* CS0 data de-skew */ + for (n = 0; n < ARRAY_SIZE(de_skew->cs0_de_skew); n++) { + offset = ((n / 21) * 11) + ((n % 21) / 2); + shift = ((n % 21) % 2); + if ((n % 21) == 20) + shift = 0; + else + /* 0 => 4; 1 => 0 */ + shift = (shift == 0) ? 4 : 0; + tim->cs0_skew[offset] &= ~(0xf << shift); + tim->cs0_skew[offset] |= (de_skew->cs0_de_skew[n] << shift); + } + + /* CS1 data de-skew */ + for (n = 0; n < ARRAY_SIZE(de_skew->cs1_de_skew); n++) { + offset = ((n / 21) * 11) + ((n % 21) / 2); + shift = ((n % 21) % 2); + if ((n % 21) == 20) + shift = 0; + else + /* 0 => 4; 1 => 0 */ + shift = (shift == 0) ? 4 : 0; + tim->cs1_skew[offset] &= ~(0xf << shift); + tim->cs1_skew[offset] |= (de_skew->cs1_de_skew[n] << shift); + } +} + +/* + * function: packaging de-skew setting to rk3328_ddr_dts_config_timing, + * rk3328_ddr_dts_config_timing will pass to trust firmware, and + * used direct to set register. + * input: de_skew + * output: tim + */ +static void +rk3328_de_skew_setting_2_register(struct rk3328_ddr_de_skew_setting *de_skew, + struct rk3328_ddr_dts_config_timing *tim) +{ + u32 n; + u32 offset; + u32 shift; + + memset_io(tim->ca_skew, 0, sizeof(tim->ca_skew)); + memset_io(tim->cs0_skew, 0, sizeof(tim->cs0_skew)); + memset_io(tim->cs1_skew, 0, sizeof(tim->cs1_skew)); + + /* CA de-skew */ + for (n = 0; n < ARRAY_SIZE(de_skew->ca_de_skew); n++) { + offset = n / 2; + shift = n % 2; + /* 0 => 4; 1 => 0 */ + shift = (shift == 0) ? 4 : 0; + tim->ca_skew[offset] &= ~(0xf << shift); + tim->ca_skew[offset] |= (de_skew->ca_de_skew[n] << shift); + } + + /* CS0 data de-skew */ + for (n = 0; n < ARRAY_SIZE(de_skew->cs0_de_skew); n++) { + offset = ((n / 21) * 11) + ((n % 21) / 2); + shift = ((n % 21) % 2); + if ((n % 21) == 20) + shift = 0; + else + /* 0 => 4; 1 => 0 */ + shift = (shift == 0) ? 4 : 0; + tim->cs0_skew[offset] &= ~(0xf << shift); + tim->cs0_skew[offset] |= (de_skew->cs0_de_skew[n] << shift); + } + + /* CS1 data de-skew */ + for (n = 0; n < ARRAY_SIZE(de_skew->cs1_de_skew); n++) { + offset = ((n / 21) * 11) + ((n % 21) / 2); + shift = ((n % 21) % 2); + if ((n % 21) == 20) + shift = 0; + else + /* 0 => 4; 1 => 0 */ + shift = (shift == 0) ? 4 : 0; + tim->cs1_skew[offset] &= ~(0xf << shift); + tim->cs1_skew[offset] |= (de_skew->cs1_de_skew[n] << shift); + } +} + +static int rk_drm_get_lcdc_type(void) +{ + u32 lcdc_type = rockchip_drm_get_sub_dev_type(); + + switch (lcdc_type) { + case DRM_MODE_CONNECTOR_DPI: + case DRM_MODE_CONNECTOR_LVDS: + lcdc_type = SCREEN_LVDS; + break; + case DRM_MODE_CONNECTOR_DisplayPort: + lcdc_type = SCREEN_DP; + break; + case DRM_MODE_CONNECTOR_HDMIA: + case DRM_MODE_CONNECTOR_HDMIB: + lcdc_type = SCREEN_HDMI; + break; + case DRM_MODE_CONNECTOR_TV: + lcdc_type = SCREEN_TVOUT; + break; + case DRM_MODE_CONNECTOR_eDP: + lcdc_type = SCREEN_EDP; + break; + case DRM_MODE_CONNECTOR_DSI: + lcdc_type = SCREEN_MIPI; + break; + default: + lcdc_type = SCREEN_NULL; + break; + } + + return lcdc_type; +} + +static int rockchip_ddr_set_rate(unsigned long target_rate) +{ + struct arm_smccc_res res; + + ddr_psci_param->hz = target_rate; + ddr_psci_param->lcdc_type = rk_drm_get_lcdc_type(); + ddr_psci_param->wait_flag1 = 1; + ddr_psci_param->wait_flag0 = 1; + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_SET_RATE); + + if ((int)res.a1 == SIP_RET_SET_RATE_TIMEOUT) + rockchip_dmcfreq_wait_complete(); + + return res.a0; +} + +static int rockchip_dmcfreq_target(struct device *dev, unsigned long *freq, + u32 flags) +{ + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); + struct dev_pm_opp *opp; + struct cpufreq_policy *policy; + unsigned long old_clk_rate = dmcfreq->rate; + unsigned long target_volt, target_rate; + unsigned int cpu_cur, cpufreq_cur; + bool is_cpufreq_changed = false; + int err = 0; + + opp = devfreq_recommended_opp(dev, freq, flags); + if (IS_ERR(opp)) { + dev_err(dev, "Failed to find opp for %lu Hz\n", *freq); + return PTR_ERR(opp); + } + target_volt = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + + if (dmcfreq->is_set_rate_direct) { + target_rate = *freq; + } else { + target_rate = clk_round_rate(dmcfreq->dmc_clk, *freq); + if ((long)target_rate <= 0) + target_rate = *freq; + } + + if (dmcfreq->rate == target_rate) { + if (dmcfreq->volt == target_volt) + return 0; + err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, + INT_MAX); + if (err) { + dev_err(dev, "Cannot set voltage %lu uV\n", + target_volt); + return err; + } + dmcfreq->volt = target_volt; + return 0; + } else if (!dmcfreq->volt) { + dmcfreq->volt = regulator_get_voltage(dmcfreq->vdd_center); + } + + /* + * We need to prevent cpu hotplug from happening while a dmc freq rate + * change is happening. + * + * Do this before taking the policy rwsem to avoid deadlocks between the + * mutex that is locked/unlocked in cpu_hotplug_disable/enable. And it + * can also avoid deadlocks between the mutex that is locked/unlocked + * in get/put_online_cpus (such as store_scaling_max_freq()). + */ + get_online_cpus(); + + /* + * Go to specified cpufreq and block other cpufreq changes since + * set_rate needs to complete during vblank. + */ + cpu_cur = raw_smp_processor_id(); + policy = cpufreq_cpu_get(cpu_cur); + if (!policy) { + dev_err(dev, "cpu%d policy NULL\n", cpu_cur); + goto cpufreq; + } + down_write(&policy->rwsem); + cpufreq_cur = cpufreq_quick_get(cpu_cur); + + /* If we're thermally throttled; don't change; */ + if (dmcfreq->min_cpu_freq && cpufreq_cur < dmcfreq->min_cpu_freq) { + if (policy->max >= dmcfreq->min_cpu_freq) { + __cpufreq_driver_target(policy, dmcfreq->min_cpu_freq, + CPUFREQ_RELATION_L); + is_cpufreq_changed = true; + } else { + dev_dbg(dev, "CPU may too slow for DMC (%d MHz)\n", + policy->max); + } + } + + /* + * If frequency scaling from low to high, adjust voltage first. + * If frequency scaling from high to low, adjust frequency first. + */ + if (old_clk_rate < target_rate) { + err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, + INT_MAX); + if (err) { + dev_err(dev, "Cannot set voltage %lu uV\n", + target_volt); + goto out; + } + } + + /* + * Writer in rwsem may block readers even during its waiting in queue, + * and this may lead to a deadlock when the code path takes read sem + * twice (e.g. one in vop_lock() and another in rockchip_pmu_lock()). + * As a (suboptimal) workaround, let writer to spin until it gets the + * lock. + */ + while (!rockchip_dmcfreq_write_trylock()) + cond_resched(); + dev_dbg(dev, "%lu-->%lu\n", old_clk_rate, target_rate); + + if (dmcfreq->set_rate_params) { + dmcfreq->set_rate_params->lcdc_type = rk_drm_get_lcdc_type(); + dmcfreq->set_rate_params->wait_flag1 = 1; + dmcfreq->set_rate_params->wait_flag0 = 1; + } + + if (dmcfreq->is_set_rate_direct) + err = rockchip_ddr_set_rate(target_rate); + else + err = clk_set_rate(dmcfreq->dmc_clk, target_rate); + + rockchip_dmcfreq_write_unlock(); + if (err) { + dev_err(dev, "Cannot set frequency %lu (%d)\n", + target_rate, err); + regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt, + INT_MAX); + goto out; + } + + /* + * Check the dpll rate, + * There only two result we will get, + * 1. Ddr frequency scaling fail, we still get the old rate. + * 2. Ddr frequency scaling sucessful, we get the rate we set. + */ + dmcfreq->rate = clk_get_rate(dmcfreq->dmc_clk); + + /* If get the incorrect rate, set voltage to old value. */ + if (dmcfreq->rate != target_rate) { + dev_err(dev, "Get wrong frequency, Request %lu, Current %lu\n", + target_rate, dmcfreq->rate); + regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt, + INT_MAX); + goto out; + } else if (old_clk_rate > target_rate) { + err = regulator_set_voltage(dmcfreq->vdd_center, target_volt, + INT_MAX); + if (err) { + dev_err(dev, "Cannot set vol %lu uV\n", target_volt); + goto out; + } + } + + if (dmcfreq->info.devfreq) { + struct devfreq *devfreq = dmcfreq->info.devfreq; + + devfreq->last_status.current_frequency = *freq; + } + + dmcfreq->volt = target_volt; +out: + if (is_cpufreq_changed) + __cpufreq_driver_target(policy, cpufreq_cur, + CPUFREQ_RELATION_L); + up_write(&policy->rwsem); + cpufreq_cpu_put(policy); +cpufreq: + put_online_cpus(); + return err; +} + +static int rockchip_dmcfreq_get_dev_status(struct device *dev, + struct devfreq_dev_status *stat) +{ + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); + struct devfreq_event_data edata; + int i, ret = 0; + + if (!dmcfreq->info.auto_freq_en) + return -EINVAL; + + for (i = 0; i < dmcfreq->edev_count; i++) { + ret = devfreq_event_get_event(dmcfreq->edev[i], &edata); + if (ret < 0) { + dev_err(dev, "failed to get event %s\n", + dmcfreq->edev[i]->desc->name); + return ret; + } + if (i == dmcfreq->dfi_id) { + stat->busy_time = edata.load_count; + stat->total_time = edata.total_count; + } else { + dmcfreq->nocp_bw[i] = edata.load_count; + } + } + + return 0; +} + +static int rockchip_dmcfreq_get_cur_freq(struct device *dev, + unsigned long *freq) +{ + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); + + *freq = dmcfreq->rate; + + return 0; +} + +static struct devfreq_dev_profile rockchip_devfreq_dmc_profile = { + .polling_ms = 50, + .target = rockchip_dmcfreq_target, + .get_dev_status = rockchip_dmcfreq_get_dev_status, + .get_cur_freq = rockchip_dmcfreq_get_cur_freq, +}; + + +static inline void reset_last_status(struct devfreq *devfreq) +{ + devfreq->last_status.total_time = 1; + devfreq->last_status.busy_time = 1; +} + +static void of_get_px30_timings(struct device *dev, + struct device_node *np, uint32_t *timing) +{ + struct device_node *np_tim; + u32 *p; + struct px30_ddr_dts_config_timing *dts_timing; + struct rk3328_ddr_de_skew_setting *de_skew; + int ret = 0; + u32 i; + + dts_timing = + (struct px30_ddr_dts_config_timing *)(timing + + DTS_PAR_OFFSET / 4); + + np_tim = of_parse_phandle(np, "ddr_timing", 0); + if (!np_tim) { + ret = -EINVAL; + goto end; + } + de_skew = kmalloc(sizeof(*de_skew), GFP_KERNEL); + if (!de_skew) { + ret = -ENOMEM; + goto end; + } + p = (u32 *)dts_timing; + for (i = 0; i < ARRAY_SIZE(px30_dts_timing); i++) { + ret |= of_property_read_u32(np_tim, px30_dts_timing[i], + p + i); + } + p = (u32 *)de_skew->ca_de_skew; + for (i = 0; i < ARRAY_SIZE(rk3328_dts_ca_timing); i++) { + ret |= of_property_read_u32(np_tim, rk3328_dts_ca_timing[i], + p + i); + } + p = (u32 *)de_skew->cs0_de_skew; + for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs0_timing); i++) { + ret |= of_property_read_u32(np_tim, rk3328_dts_cs0_timing[i], + p + i); + } + p = (u32 *)de_skew->cs1_de_skew; + for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs1_timing); i++) { + ret |= of_property_read_u32(np_tim, rk3328_dts_cs1_timing[i], + p + i); + } + if (!ret) + px30_de_skew_set_2_reg(de_skew, dts_timing); + kfree(de_skew); +end: + if (!ret) { + dts_timing->available = 1; + } else { + dts_timing->available = 0; + dev_err(dev, "of_get_ddr_timings: fail\n"); + } + + of_node_put(np_tim); +} + +static void of_get_rk1808_timings(struct device *dev, + struct device_node *np, uint32_t *timing) +{ + struct device_node *np_tim; + u32 *p; + struct rk1808_ddr_dts_config_timing *dts_timing; + int ret = 0; + u32 i; + + dts_timing = + (struct rk1808_ddr_dts_config_timing *)(timing + + DTS_PAR_OFFSET / 4); + + np_tim = of_parse_phandle(np, "ddr_timing", 0); + if (!np_tim) { + ret = -EINVAL; + goto end; + } + + p = (u32 *)dts_timing; + for (i = 0; i < ARRAY_SIZE(px30_dts_timing); i++) { + ret |= of_property_read_u32(np_tim, px30_dts_timing[i], + p + i); + } + p = (u32 *)dts_timing->ca_de_skew; + for (i = 0; i < ARRAY_SIZE(rk1808_dts_ca_timing); i++) { + ret |= of_property_read_u32(np_tim, rk1808_dts_ca_timing[i], + p + i); + } + p = (u32 *)dts_timing->cs0_a_de_skew; + for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs0_a_timing); i++) { + ret |= of_property_read_u32(np_tim, rk1808_dts_cs0_a_timing[i], + p + i); + } + p = (u32 *)dts_timing->cs0_b_de_skew; + for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs0_b_timing); i++) { + ret |= of_property_read_u32(np_tim, rk1808_dts_cs0_b_timing[i], + p + i); + } + p = (u32 *)dts_timing->cs1_a_de_skew; + for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs1_a_timing); i++) { + ret |= of_property_read_u32(np_tim, rk1808_dts_cs1_a_timing[i], + p + i); + } + p = (u32 *)dts_timing->cs1_b_de_skew; + for (i = 0; i < ARRAY_SIZE(rk1808_dts_cs1_b_timing); i++) { + ret |= of_property_read_u32(np_tim, rk1808_dts_cs1_b_timing[i], + p + i); + } + +end: + if (!ret) { + dts_timing->available = 1; + } else { + dts_timing->available = 0; + dev_err(dev, "of_get_ddr_timings: fail\n"); + } + + of_node_put(np_tim); +} + +static void of_get_rk3128_timings(struct device *dev, + struct device_node *np, uint32_t *timing) +{ + struct device_node *np_tim; + u32 *p; + struct rk3128_ddr_dts_config_timing *dts_timing; + struct share_params *init_timing; + int ret = 0; + u32 i; + + init_timing = (struct share_params *)timing; + + if (of_property_read_u32(np, "vop-dclk-mode", + &init_timing->vop_dclk_mode)) + init_timing->vop_dclk_mode = 0; + + p = timing + DTS_PAR_OFFSET / 4; + np_tim = of_parse_phandle(np, "rockchip,ddr_timing", 0); + if (!np_tim) { + ret = -EINVAL; + goto end; + } + for (i = 0; i < ARRAY_SIZE(rk3128_dts_timing); i++) { + ret |= of_property_read_u32(np_tim, rk3128_dts_timing[i], + p + i); + } +end: + dts_timing = + (struct rk3128_ddr_dts_config_timing *)(timing + + DTS_PAR_OFFSET / 4); + if (!ret) { + dts_timing->available = 1; + } else { + dts_timing->available = 0; + dev_err(dev, "of_get_ddr_timings: fail\n"); + } + + of_node_put(np_tim); +} + +static uint32_t of_get_rk3228_timings(struct device *dev, + struct device_node *np, uint32_t *timing) +{ + struct device_node *np_tim; + u32 *p; + int ret = 0; + u32 i; + + p = timing + DTS_PAR_OFFSET / 4; + np_tim = of_parse_phandle(np, "rockchip,dram_timing", 0); + if (!np_tim) { + ret = -EINVAL; + goto end; + } + for (i = 0; i < ARRAY_SIZE(rk3228_dts_timing); i++) { + ret |= of_property_read_u32(np_tim, rk3228_dts_timing[i], + p + i); + } +end: + if (ret) + dev_err(dev, "of_get_ddr_timings: fail\n"); + + of_node_put(np_tim); + return ret; +} + +static void of_get_rk3288_timings(struct device *dev, + struct device_node *np, uint32_t *timing) +{ + struct device_node *np_tim; + u32 *p; + struct rk3288_ddr_dts_config_timing *dts_timing; + struct share_params *init_timing; + int ret = 0; + u32 i; + + init_timing = (struct share_params *)timing; + + if (of_property_read_u32(np, "vop-dclk-mode", + &init_timing->vop_dclk_mode)) + init_timing->vop_dclk_mode = 0; + + p = timing + DTS_PAR_OFFSET / 4; + np_tim = of_parse_phandle(np, "rockchip,ddr_timing", 0); + if (!np_tim) { + ret = -EINVAL; + goto end; + } + for (i = 0; i < ARRAY_SIZE(rk3288_dts_timing); i++) { + ret |= of_property_read_u32(np_tim, rk3288_dts_timing[i], + p + i); + } +end: + dts_timing = + (struct rk3288_ddr_dts_config_timing *)(timing + + DTS_PAR_OFFSET / 4); + if (!ret) { + dts_timing->available = 1; + } else { + dts_timing->available = 0; + dev_err(dev, "of_get_ddr_timings: fail\n"); + } + + of_node_put(np_tim); +} + +static void of_get_rk3328_timings(struct device *dev, + struct device_node *np, uint32_t *timing) +{ + struct device_node *np_tim; + u32 *p; + struct rk3328_ddr_dts_config_timing *dts_timing; + struct rk3328_ddr_de_skew_setting *de_skew; + int ret = 0; + u32 i; + + dts_timing = + (struct rk3328_ddr_dts_config_timing *)(timing + + DTS_PAR_OFFSET / 4); + + np_tim = of_parse_phandle(np, "ddr_timing", 0); + if (!np_tim) { + ret = -EINVAL; + goto end; + } + de_skew = kmalloc(sizeof(*de_skew), GFP_KERNEL); + if (!de_skew) { + ret = -ENOMEM; + goto end; + } + p = (u32 *)dts_timing; + for (i = 0; i < ARRAY_SIZE(rk3328_dts_timing); i++) { + ret |= of_property_read_u32(np_tim, rk3328_dts_timing[i], + p + i); + } + p = (u32 *)de_skew->ca_de_skew; + for (i = 0; i < ARRAY_SIZE(rk3328_dts_ca_timing); i++) { + ret |= of_property_read_u32(np_tim, rk3328_dts_ca_timing[i], + p + i); + } + p = (u32 *)de_skew->cs0_de_skew; + for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs0_timing); i++) { + ret |= of_property_read_u32(np_tim, rk3328_dts_cs0_timing[i], + p + i); + } + p = (u32 *)de_skew->cs1_de_skew; + for (i = 0; i < ARRAY_SIZE(rk3328_dts_cs1_timing); i++) { + ret |= of_property_read_u32(np_tim, rk3328_dts_cs1_timing[i], + p + i); + } + if (!ret) + rk3328_de_skew_setting_2_register(de_skew, dts_timing); + kfree(de_skew); +end: + if (!ret) { + dts_timing->available = 1; + } else { + dts_timing->available = 0; + dev_err(dev, "of_get_ddr_timings: fail\n"); + } + + of_node_put(np_tim); +} + +static void of_get_rv1126_timings(struct device *dev, + struct device_node *np, uint32_t *timing) +{ + struct device_node *np_tim; + u32 *p; + struct rk1808_ddr_dts_config_timing *dts_timing; + int ret = 0; + u32 i; + + dts_timing = + (struct rk1808_ddr_dts_config_timing *)(timing + + DTS_PAR_OFFSET / 4); + + np_tim = of_parse_phandle(np, "ddr_timing", 0); + if (!np_tim) { + ret = -EINVAL; + goto end; + } + + p = (u32 *)dts_timing; + for (i = 0; i < ARRAY_SIZE(px30_dts_timing); i++) { + ret |= of_property_read_u32(np_tim, px30_dts_timing[i], + p + i); + } + p = (u32 *)dts_timing->ca_de_skew; + for (i = 0; i < ARRAY_SIZE(rv1126_dts_ca_timing); i++) { + ret |= of_property_read_u32(np_tim, rv1126_dts_ca_timing[i], + p + i); + } + p = (u32 *)dts_timing->cs0_a_de_skew; + for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs0_a_timing); i++) { + ret |= of_property_read_u32(np_tim, rv1126_dts_cs0_a_timing[i], + p + i); + } + p = (u32 *)dts_timing->cs0_b_de_skew; + for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs0_b_timing); i++) { + ret |= of_property_read_u32(np_tim, rv1126_dts_cs0_b_timing[i], + p + i); + } + p = (u32 *)dts_timing->cs1_a_de_skew; + for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs1_a_timing); i++) { + ret |= of_property_read_u32(np_tim, rv1126_dts_cs1_a_timing[i], + p + i); + } + p = (u32 *)dts_timing->cs1_b_de_skew; + for (i = 0; i < ARRAY_SIZE(rv1126_dts_cs1_b_timing); i++) { + ret |= of_property_read_u32(np_tim, rv1126_dts_cs1_b_timing[i], + p + i); + } + +end: + if (!ret) { + dts_timing->available = 1; + } else { + dts_timing->available = 0; + dev_err(dev, "of_get_ddr_timings: fail\n"); + } + + of_node_put(np_tim); +} + +static struct rk3368_dram_timing *of_get_rk3368_timings(struct device *dev, + struct device_node *np) +{ + struct rk3368_dram_timing *timing = NULL; + struct device_node *np_tim; + int ret = 0; + + np_tim = of_parse_phandle(np, "ddr_timing", 0); + if (np_tim) { + timing = devm_kzalloc(dev, sizeof(*timing), GFP_KERNEL); + if (!timing) + goto err; + + ret |= of_property_read_u32(np_tim, "dram_spd_bin", + &timing->dram_spd_bin); + ret |= of_property_read_u32(np_tim, "sr_idle", + &timing->sr_idle); + ret |= of_property_read_u32(np_tim, "pd_idle", + &timing->pd_idle); + ret |= of_property_read_u32(np_tim, "dram_dll_disb_freq", + &timing->dram_dll_dis_freq); + ret |= of_property_read_u32(np_tim, "phy_dll_disb_freq", + &timing->phy_dll_dis_freq); + ret |= of_property_read_u32(np_tim, "dram_odt_disb_freq", + &timing->dram_odt_dis_freq); + ret |= of_property_read_u32(np_tim, "phy_odt_disb_freq", + &timing->phy_odt_dis_freq); + ret |= of_property_read_u32(np_tim, "ddr3_drv", + &timing->ddr3_drv); + ret |= of_property_read_u32(np_tim, "ddr3_odt", + &timing->ddr3_odt); + ret |= of_property_read_u32(np_tim, "lpddr3_drv", + &timing->lpddr3_drv); + ret |= of_property_read_u32(np_tim, "lpddr3_odt", + &timing->lpddr3_odt); + ret |= of_property_read_u32(np_tim, "lpddr2_drv", + &timing->lpddr2_drv); + ret |= of_property_read_u32(np_tim, "phy_clk_drv", + &timing->phy_clk_drv); + ret |= of_property_read_u32(np_tim, "phy_cmd_drv", + &timing->phy_cmd_drv); + ret |= of_property_read_u32(np_tim, "phy_dqs_drv", + &timing->phy_dqs_drv); + ret |= of_property_read_u32(np_tim, "phy_odt", + &timing->phy_odt); + ret |= of_property_read_u32(np_tim, "ddr_2t", + &timing->ddr_2t); + if (ret) { + devm_kfree(dev, timing); + goto err; + } + of_node_put(np_tim); + return timing; + } + +err: + if (timing) { + devm_kfree(dev, timing); + timing = NULL; + } + of_node_put(np_tim); + return timing; +} + +static struct rk3399_dram_timing *of_get_rk3399_timings(struct device *dev, + struct device_node *np) +{ + struct rk3399_dram_timing *timing = NULL; + struct device_node *np_tim; + int ret; + + np_tim = of_parse_phandle(np, "ddr_timing", 0); + if (np_tim) { + timing = devm_kzalloc(dev, sizeof(*timing), GFP_KERNEL); + if (!timing) + goto err; + + ret = of_property_read_u32(np_tim, "ddr3_speed_bin", + &timing->ddr3_speed_bin); + ret |= of_property_read_u32(np_tim, "pd_idle", + &timing->pd_idle); + ret |= of_property_read_u32(np_tim, "sr_idle", + &timing->sr_idle); + ret |= of_property_read_u32(np_tim, "sr_mc_gate_idle", + &timing->sr_mc_gate_idle); + ret |= of_property_read_u32(np_tim, "srpd_lite_idle", + &timing->srpd_lite_idle); + ret |= of_property_read_u32(np_tim, "standby_idle", + &timing->standby_idle); + ret |= of_property_read_u32(np_tim, "auto_lp_dis_freq", + &timing->auto_lp_dis_freq); + ret |= of_property_read_u32(np_tim, "ddr3_dll_dis_freq", + &timing->ddr3_dll_dis_freq); + ret |= of_property_read_u32(np_tim, "phy_dll_dis_freq", + &timing->phy_dll_dis_freq); + ret |= of_property_read_u32(np_tim, "ddr3_odt_dis_freq", + &timing->ddr3_odt_dis_freq); + ret |= of_property_read_u32(np_tim, "ddr3_drv", + &timing->ddr3_drv); + ret |= of_property_read_u32(np_tim, "ddr3_odt", + &timing->ddr3_odt); + ret |= of_property_read_u32(np_tim, "phy_ddr3_ca_drv", + &timing->phy_ddr3_ca_drv); + ret |= of_property_read_u32(np_tim, "phy_ddr3_dq_drv", + &timing->phy_ddr3_dq_drv); + ret |= of_property_read_u32(np_tim, "phy_ddr3_odt", + &timing->phy_ddr3_odt); + ret |= of_property_read_u32(np_tim, "lpddr3_odt_dis_freq", + &timing->lpddr3_odt_dis_freq); + ret |= of_property_read_u32(np_tim, "lpddr3_drv", + &timing->lpddr3_drv); + ret |= of_property_read_u32(np_tim, "lpddr3_odt", + &timing->lpddr3_odt); + ret |= of_property_read_u32(np_tim, "phy_lpddr3_ca_drv", + &timing->phy_lpddr3_ca_drv); + ret |= of_property_read_u32(np_tim, "phy_lpddr3_dq_drv", + &timing->phy_lpddr3_dq_drv); + ret |= of_property_read_u32(np_tim, "phy_lpddr3_odt", + &timing->phy_lpddr3_odt); + ret |= of_property_read_u32(np_tim, "lpddr4_odt_dis_freq", + &timing->lpddr4_odt_dis_freq); + ret |= of_property_read_u32(np_tim, "lpddr4_drv", + &timing->lpddr4_drv); + ret |= of_property_read_u32(np_tim, "lpddr4_dq_odt", + &timing->lpddr4_dq_odt); + ret |= of_property_read_u32(np_tim, "lpddr4_ca_odt", + &timing->lpddr4_ca_odt); + ret |= of_property_read_u32(np_tim, "phy_lpddr4_ca_drv", + &timing->phy_lpddr4_ca_drv); + ret |= of_property_read_u32(np_tim, "phy_lpddr4_ck_cs_drv", + &timing->phy_lpddr4_ck_cs_drv); + ret |= of_property_read_u32(np_tim, "phy_lpddr4_dq_drv", + &timing->phy_lpddr4_dq_drv); + ret |= of_property_read_u32(np_tim, "phy_lpddr4_odt", + &timing->phy_lpddr4_odt); + if (ret) { + devm_kfree(dev, timing); + goto err; + } + of_node_put(np_tim); + return timing; + } + +err: + if (timing) { + devm_kfree(dev, timing); + timing = NULL; + } + of_node_put(np_tim); + return timing; +} + +static int rockchip_ddr_set_auto_self_refresh(uint32_t en) +{ + struct arm_smccc_res res; + + ddr_psci_param->sr_idle_en = en; + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_SET_AT_SR); + + return res.a0; +} + +struct dmcfreq_wait_ctrl_t { + wait_queue_head_t wait_wq; + int complt_irq; + int wait_flag; + int wait_en; + int wait_time_out_ms; + int dcf_en; + struct regmap *regmap_dcf; +}; + +static struct dmcfreq_wait_ctrl_t wait_ctrl; + +static irqreturn_t wait_complete_irq(int irqno, void *dev_id) +{ + struct dmcfreq_wait_ctrl_t *ctrl = dev_id; + + ctrl->wait_flag = 0; + wake_up(&ctrl->wait_wq); + return IRQ_HANDLED; +} + +static irqreturn_t wait_dcf_complete_irq(int irqno, void *dev_id) +{ + struct arm_smccc_res res; + struct dmcfreq_wait_ctrl_t *ctrl = dev_id; + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_POST_SET_RATE); + if (res.a0) + pr_err("%s: dram post set rate error:%lx\n", __func__, res.a0); + + ctrl->wait_flag = 0; + wake_up(&ctrl->wait_wq); + return IRQ_HANDLED; +} + +int rockchip_dmcfreq_wait_complete(void) +{ + struct arm_smccc_res res; + + if (!wait_ctrl.wait_en) { + pr_err("%s: Do not support time out!\n", __func__); + return 0; + } + wait_ctrl.wait_flag = -1; + + enable_irq(wait_ctrl.complt_irq); + /* + * CPUs only enter WFI when idle to make sure that + * FIQn can quick response. + */ + cpu_latency_qos_update_request(&pm_qos, 0); + + if (wait_ctrl.dcf_en == 1) { + /* start dcf */ + regmap_update_bits(wait_ctrl.regmap_dcf, 0x0, 0x1, 0x1); + } else if (wait_ctrl.dcf_en == 2) { + res = sip_smc_dram(0, 0, ROCKCHIP_SIP_CONFIG_MCU_START); + if (res.a0) { + pr_err("rockchip_sip_config_mcu_start error:%lx\n", res.a0); + return -ENOMEM; + } + } + + wait_event_timeout(wait_ctrl.wait_wq, (wait_ctrl.wait_flag == 0), + msecs_to_jiffies(wait_ctrl.wait_time_out_ms)); + + cpu_latency_qos_update_request(&pm_qos, PM_QOS_DEFAULT_VALUE); + disable_irq(wait_ctrl.complt_irq); + + return 0; +} + +static __maybe_unused int rockchip_get_freq_info(struct rockchip_dmcfreq *dmcfreq) +{ + struct arm_smccc_res res; + struct dev_pm_opp *opp; + struct dmc_freq_table *freq_table; + unsigned long rate; + int i, j, count, ret = 0; + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_GET_FREQ_INFO); + if (res.a0) { + dev_err(dmcfreq->dev, "rockchip_sip_config_dram_get_freq_info error:%lx\n", + res.a0); + return -ENOMEM; + } + + if (ddr_psci_param->freq_count == 0 || ddr_psci_param->freq_count > 6) { + dev_err(dmcfreq->dev, "it is no available frequencies!\n"); + return -EPERM; + } + + for (i = 0; i < ddr_psci_param->freq_count; i++) + dmcfreq->freq_info_rate[i] = ddr_psci_param->freq_info_mhz[i] * 1000000; + dmcfreq->freq_count = ddr_psci_param->freq_count; + + /* update dmc_opp_table */ + count = dev_pm_opp_get_opp_count(dmcfreq->dev); + if (count <= 0) { + ret = count ? count : -ENODATA; + return ret; + } + + freq_table = kmalloc(sizeof(struct dmc_freq_table) * count, GFP_KERNEL); + for (i = 0, rate = 0; i < count; i++, rate++) { + /* find next rate */ + opp = dev_pm_opp_find_freq_ceil(dmcfreq->dev, &rate); + if (IS_ERR(opp)) { + ret = PTR_ERR(opp); + dev_err(dmcfreq->dev, "failed to find OPP for freq %lu.\n", rate); + goto out; + } + freq_table[i].freq = rate; + freq_table[i].volt = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + + for (j = 0; j < dmcfreq->freq_count; j++) { + if (rate == dmcfreq->freq_info_rate[j]) + break; + } + if (j == dmcfreq->freq_count) + dev_pm_opp_remove(dmcfreq->dev, rate); + } + + for (i = 0; i < dmcfreq->freq_count; i++) { + for (j = 0; j < count; j++) { + if (dmcfreq->freq_info_rate[i] == freq_table[j].freq) { + break; + } else if (dmcfreq->freq_info_rate[i] < freq_table[j].freq) { + dev_pm_opp_add(dmcfreq->dev, dmcfreq->freq_info_rate[i], + freq_table[j].volt); + break; + } + } + if (j == count) { + dev_err(dmcfreq->dev, "failed to match dmc_opp_table for %ld\n", + dmcfreq->freq_info_rate[i]); + if (i == 0) + ret = -EPERM; + else + dmcfreq->freq_count = i; + goto out; + } + } + +out: + kfree(freq_table); + return ret; +} + +static __maybe_unused int px30_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct arm_smccc_res res; + u32 size; + int ret; + int complt_irq; + u32 complt_hwirq; + struct irq_data *complt_irq_data; + + res = sip_smc_dram(0, 0, + ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); + dev_notice(&pdev->dev, "current ATF version 0x%lx!\n", res.a1); + if (res.a0 || res.a1 < 0x103) { + dev_err(&pdev->dev, + "trusted firmware need to update or is invalid!\n"); + return -ENXIO; + } + + dev_notice(&pdev->dev, "read tf version 0x%lx!\n", res.a1); + + /* + * first 4KB is used for interface parameters + * after 4KB * N is dts parameters + */ + size = sizeof(struct px30_ddr_dts_config_timing); + res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 4096) + 1, + SHARE_PAGE_TYPE_DDR); + if (res.a0 != 0) { + dev_err(&pdev->dev, "no ATF memory for init\n"); + return -ENOMEM; + } + ddr_psci_param = (struct share_params *)res.a1; + of_get_px30_timings(&pdev->dev, pdev->dev.of_node, + (uint32_t *)ddr_psci_param); + + init_waitqueue_head(&wait_ctrl.wait_wq); + wait_ctrl.wait_en = 1; + wait_ctrl.wait_time_out_ms = 17 * 5; + + complt_irq = platform_get_irq_byname(pdev, "complete_irq"); + if (complt_irq < 0) { + dev_err(&pdev->dev, "no IRQ for complete_irq: %d\n", + complt_irq); + return complt_irq; + } + wait_ctrl.complt_irq = complt_irq; + + ret = devm_request_irq(&pdev->dev, complt_irq, wait_complete_irq, + 0, dev_name(&pdev->dev), &wait_ctrl); + if (ret < 0) { + dev_err(&pdev->dev, "cannot request complete_irq\n"); + return ret; + } + disable_irq(complt_irq); + + complt_irq_data = irq_get_irq_data(complt_irq); + complt_hwirq = irqd_to_hwirq(complt_irq_data); + ddr_psci_param->complt_hwirq = complt_hwirq; + + dmcfreq->set_rate_params = ddr_psci_param; + rockchip_set_ddrclk_params(dmcfreq->set_rate_params); + rockchip_set_ddrclk_dmcfreq_wait_complete(rockchip_dmcfreq_wait_complete); + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_INIT); + if (res.a0) { + dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", + res.a0); + return -ENOMEM; + } + + dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; + + return 0; +} + +static __maybe_unused int rk1808_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct arm_smccc_res res; + u32 size; + int ret; + int complt_irq; + struct device_node *node; + + res = sip_smc_dram(0, 0, + ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); + dev_notice(&pdev->dev, "current ATF version 0x%lx!\n", res.a1); + if (res.a0 || res.a1 < 0x101) { + dev_err(&pdev->dev, + "trusted firmware need to update or is invalid!\n"); + return -ENXIO; + } + + /* + * first 4KB is used for interface parameters + * after 4KB * N is dts parameters + */ + size = sizeof(struct rk1808_ddr_dts_config_timing); + res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 4096) + 1, + SHARE_PAGE_TYPE_DDR); + if (res.a0 != 0) { + dev_err(&pdev->dev, "no ATF memory for init\n"); + return -ENOMEM; + } + ddr_psci_param = (struct share_params *)res.a1; + of_get_rk1808_timings(&pdev->dev, pdev->dev.of_node, + (uint32_t *)ddr_psci_param); + + /* enable start dcf in kernel after dcf ready */ + node = of_parse_phandle(pdev->dev.of_node, "dcf_reg", 0); + wait_ctrl.regmap_dcf = syscon_node_to_regmap(node); + if (IS_ERR(wait_ctrl.regmap_dcf)) + return PTR_ERR(wait_ctrl.regmap_dcf); + wait_ctrl.dcf_en = 1; + + init_waitqueue_head(&wait_ctrl.wait_wq); + wait_ctrl.wait_en = 1; + wait_ctrl.wait_time_out_ms = 17 * 5; + + complt_irq = platform_get_irq_byname(pdev, "complete_irq"); + if (complt_irq < 0) { + dev_err(&pdev->dev, "no IRQ for complete_irq: %d\n", + complt_irq); + return complt_irq; + } + wait_ctrl.complt_irq = complt_irq; + + ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, + 0, dev_name(&pdev->dev), &wait_ctrl); + if (ret < 0) { + dev_err(&pdev->dev, "cannot request complete_irq\n"); + return ret; + } + disable_irq(complt_irq); + + dmcfreq->set_rate_params = ddr_psci_param; + rockchip_set_ddrclk_params(dmcfreq->set_rate_params); + rockchip_set_ddrclk_dmcfreq_wait_complete(rockchip_dmcfreq_wait_complete); + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_INIT); + if (res.a0) { + dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", + res.a0); + return -ENOMEM; + } + + dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; + + return 0; +} + +static __maybe_unused int rk3128_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct arm_smccc_res res; + + res = sip_smc_request_share_mem(DIV_ROUND_UP(sizeof( + struct rk3128_ddr_dts_config_timing), + 4096) + 1, SHARE_PAGE_TYPE_DDR); + if (res.a0) { + dev_err(&pdev->dev, "no ATF memory for init\n"); + return -ENOMEM; + } + ddr_psci_param = (struct share_params *)res.a1; + of_get_rk3128_timings(&pdev->dev, pdev->dev.of_node, + (uint32_t *)ddr_psci_param); + + ddr_psci_param->hz = 0; + ddr_psci_param->lcdc_type = rk_drm_get_lcdc_type(); + + dmcfreq->set_rate_params = ddr_psci_param; + rockchip_set_ddrclk_params(dmcfreq->set_rate_params); + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_INIT); + + if (res.a0) { + dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", + res.a0); + return -ENOMEM; + } + + dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; + + return 0; +} + +static __maybe_unused int rk3228_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct arm_smccc_res res; + + res = sip_smc_request_share_mem(DIV_ROUND_UP(sizeof( + struct rk3228_ddr_dts_config_timing), + 4096) + 1, SHARE_PAGE_TYPE_DDR); + if (res.a0) { + dev_err(&pdev->dev, "no ATF memory for init\n"); + return -ENOMEM; + } + + ddr_psci_param = (struct share_params *)res.a1; + if (of_get_rk3228_timings(&pdev->dev, pdev->dev.of_node, + (uint32_t *)ddr_psci_param)) + return -ENOMEM; + + ddr_psci_param->hz = 0; + + dmcfreq->set_rate_params = ddr_psci_param; + rockchip_set_ddrclk_params(dmcfreq->set_rate_params); + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_INIT); + + if (res.a0) { + dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", + res.a0); + return -ENOMEM; + } + + dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; + + return 0; +} + +static __maybe_unused int rk3288_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct device *dev = &pdev->dev; + struct clk *pclk_phy, *pclk_upctl, *dmc_clk; + struct arm_smccc_res res; + int ret; + + dmc_clk = devm_clk_get(dev, "dmc_clk"); + if (IS_ERR(dmc_clk)) { + dev_err(dev, "Cannot get the clk dmc_clk\n"); + return PTR_ERR(dmc_clk); + } + ret = clk_prepare_enable(dmc_clk); + if (ret < 0) { + dev_err(dev, "failed to prepare/enable dmc_clk\n"); + return ret; + } + + pclk_phy = devm_clk_get(dev, "pclk_phy0"); + if (IS_ERR(pclk_phy)) { + dev_err(dev, "Cannot get the clk pclk_phy0\n"); + return PTR_ERR(pclk_phy); + } + ret = clk_prepare_enable(pclk_phy); + if (ret < 0) { + dev_err(dev, "failed to prepare/enable pclk_phy0\n"); + return ret; + } + pclk_upctl = devm_clk_get(dev, "pclk_upctl0"); + if (IS_ERR(pclk_upctl)) { + dev_err(dev, "Cannot get the clk pclk_upctl0\n"); + return PTR_ERR(pclk_upctl); + } + ret = clk_prepare_enable(pclk_upctl); + if (ret < 0) { + dev_err(dev, "failed to prepare/enable pclk_upctl1\n"); + return ret; + } + + pclk_phy = devm_clk_get(dev, "pclk_phy1"); + if (IS_ERR(pclk_phy)) { + dev_err(dev, "Cannot get the clk pclk_phy1\n"); + return PTR_ERR(pclk_phy); + } + ret = clk_prepare_enable(pclk_phy); + if (ret < 0) { + dev_err(dev, "failed to prepare/enable pclk_phy1\n"); + return ret; + } + pclk_upctl = devm_clk_get(dev, "pclk_upctl1"); + if (IS_ERR(pclk_upctl)) { + dev_err(dev, "Cannot get the clk pclk_upctl1\n"); + return PTR_ERR(pclk_upctl); + } + ret = clk_prepare_enable(pclk_upctl); + if (ret < 0) { + dev_err(dev, "failed to prepare/enable pclk_upctl1\n"); + return ret; + } + + res = sip_smc_request_share_mem(DIV_ROUND_UP(sizeof( + struct rk3288_ddr_dts_config_timing), + 4096) + 1, SHARE_PAGE_TYPE_DDR); + if (res.a0) { + dev_err(&pdev->dev, "no ATF memory for init\n"); + return -ENOMEM; + } + + ddr_psci_param = (struct share_params *)res.a1; + of_get_rk3288_timings(&pdev->dev, pdev->dev.of_node, + (uint32_t *)ddr_psci_param); + + ddr_psci_param->hz = 0; + ddr_psci_param->lcdc_type = rk_drm_get_lcdc_type(); + + dmcfreq->set_rate_params = ddr_psci_param; + rockchip_set_ddrclk_params(dmcfreq->set_rate_params); + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_INIT); + + if (res.a0) { + dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", + res.a0); + return -ENOMEM; + } + + dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; + + return 0; +} + +static __maybe_unused int rk3328_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct arm_smccc_res res; + u32 size; + + res = sip_smc_dram(0, 0, + ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); + dev_notice(&pdev->dev, "current ATF version 0x%lx!\n", res.a1); + if (res.a0 || (res.a1 < 0x101)) { + dev_err(&pdev->dev, + "trusted firmware need to update or is invalid!\n"); + return -ENXIO; + } + + dev_notice(&pdev->dev, "read tf version 0x%lx!\n", res.a1); + + /* + * first 4KB is used for interface parameters + * after 4KB * N is dts parameters + */ + size = sizeof(struct rk3328_ddr_dts_config_timing); + res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 4096) + 1, + SHARE_PAGE_TYPE_DDR); + if (res.a0 != 0) { + dev_err(&pdev->dev, "no ATF memory for init\n"); + return -ENOMEM; + } + ddr_psci_param = (struct share_params *)res.a1; + of_get_rk3328_timings(&pdev->dev, pdev->dev.of_node, + (uint32_t *)ddr_psci_param); + + dmcfreq->set_rate_params = ddr_psci_param; + rockchip_set_ddrclk_params(dmcfreq->set_rate_params); + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_INIT); + if (res.a0) { + dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", + res.a0); + return -ENOMEM; + } + + dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; + + return 0; +} + +static __maybe_unused int rk3368_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct device *dev = &pdev->dev; + struct device_node *np = pdev->dev.of_node; + struct arm_smccc_res res; + struct rk3368_dram_timing *dram_timing; + struct clk *pclk_phy, *pclk_upctl; + int ret; + u32 dram_spd_bin; + u32 addr_mcu_el3; + u32 dclk_mode; + u32 lcdc_type; + + pclk_phy = devm_clk_get(dev, "pclk_phy"); + if (IS_ERR(pclk_phy)) { + dev_err(dev, "Cannot get the clk pclk_phy\n"); + return PTR_ERR(pclk_phy); + } + ret = clk_prepare_enable(pclk_phy); + if (ret < 0) { + dev_err(dev, "failed to prepare/enable pclk_phy\n"); + return ret; + } + pclk_upctl = devm_clk_get(dev, "pclk_upctl"); + if (IS_ERR(pclk_upctl)) { + dev_err(dev, "Cannot get the clk pclk_upctl\n"); + return PTR_ERR(pclk_upctl); + } + ret = clk_prepare_enable(pclk_upctl); + if (ret < 0) { + dev_err(dev, "failed to prepare/enable pclk_upctl\n"); + return ret; + } + + /* + * Get dram timing and pass it to arm trust firmware, + * the dram drvier in arm trust firmware will get these + * timing and to do dram initial. + */ + dram_timing = of_get_rk3368_timings(dev, np); + if (dram_timing) { + dram_spd_bin = dram_timing->dram_spd_bin; + if (scpi_ddr_send_timing((u32 *)dram_timing, + sizeof(struct rk3368_dram_timing))) + dev_err(dev, "send ddr timing timeout\n"); + } else { + dev_err(dev, "get ddr timing from dts error\n"); + dram_spd_bin = DDR3_DEFAULT; + } + + res = sip_smc_mcu_el3fiq(FIQ_INIT_HANDLER, + FIQ_NUM_FOR_DCF, + FIQ_CPU_TGT_BOOT); + if ((res.a0) || (res.a1 == 0) || (res.a1 > 0x80000)) + dev_err(dev, "Trust version error, pls check trust version\n"); + addr_mcu_el3 = res.a1; + + if (of_property_read_u32(np, "vop-dclk-mode", &dclk_mode) == 0) + scpi_ddr_dclk_mode(dclk_mode); + + dmcfreq->set_rate_params = + devm_kzalloc(dev, sizeof(struct share_params), GFP_KERNEL); + if (!dmcfreq->set_rate_params) + return -ENOMEM; + rockchip_set_ddrclk_params(dmcfreq->set_rate_params); + + lcdc_type = rk_drm_get_lcdc_type(); + + if (scpi_ddr_init(dram_spd_bin, 0, lcdc_type, + addr_mcu_el3)) + dev_err(dev, "ddr init error\n"); + else + dev_dbg(dev, ("%s out\n"), __func__); + + dmcfreq->set_auto_self_refresh = scpi_ddr_set_auto_self_refresh; + + return 0; +} + +static int rk3399_set_msch_readlatency(unsigned int readlatency) +{ + struct arm_smccc_res res; + + arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, readlatency, 0, + ROCKCHIP_SIP_CONFIG_DRAM_SET_MSCH_RL, + 0, 0, 0, 0, &res); + + return res.a0; +} + +static __maybe_unused int rk3399_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct device *dev = &pdev->dev; + struct device_node *np = pdev->dev.of_node; + struct arm_smccc_res res; + struct rk3399_dram_timing *dram_timing; + int index, size; + u32 *timing; + + /* + * Get dram timing and pass it to arm trust firmware, + * the dram drvier in arm trust firmware will get these + * timing and to do dram initial. + */ + dram_timing = of_get_rk3399_timings(dev, np); + if (dram_timing) { + timing = (u32 *)dram_timing; + size = sizeof(struct rk3399_dram_timing) / 4; + for (index = 0; index < size; index++) { + arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, *timing++, index, + ROCKCHIP_SIP_CONFIG_DRAM_SET_PARAM, + 0, 0, 0, 0, &res); + if (res.a0) { + dev_err(dev, "Failed to set dram param: %ld\n", + res.a0); + return -EINVAL; + } + } + } + + dmcfreq->set_rate_params = + devm_kzalloc(dev, sizeof(struct share_params), GFP_KERNEL); + if (!dmcfreq->set_rate_params) + return -ENOMEM; + rockchip_set_ddrclk_params(dmcfreq->set_rate_params); + + arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0, + ROCKCHIP_SIP_CONFIG_DRAM_INIT, + 0, 0, 0, 0, &res); + + dmcfreq->info.set_msch_readlatency = rk3399_set_msch_readlatency; + + return 0; +} + +static __maybe_unused int rk3568_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct arm_smccc_res res; + int ret; + int complt_irq; + + res = sip_smc_dram(0, 0, + ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); + dev_notice(&pdev->dev, "current ATF version 0x%lx\n", res.a1); + if (res.a0 || res.a1 < 0x101) { + dev_err(&pdev->dev, "trusted firmware need update to V1.01 and above.\n"); + return -ENXIO; + } + + /* + * first 4KB is used for interface parameters + * after 4KB is dts parameters + * request share memory size 4KB * 2 + */ + res = sip_smc_request_share_mem(2, SHARE_PAGE_TYPE_DDR); + if (res.a0 != 0) { + dev_err(&pdev->dev, "no ATF memory for init\n"); + return -ENOMEM; + } + ddr_psci_param = (struct share_params *)res.a1; + /* Clear ddr_psci_param, size is 4KB * 2 */ + memset_io(ddr_psci_param, 0x0, 4096 * 2); + + /* start mcu with sip_smc_dram */ + wait_ctrl.dcf_en = 2; + + init_waitqueue_head(&wait_ctrl.wait_wq); + wait_ctrl.wait_en = 1; + wait_ctrl.wait_time_out_ms = 17 * 5; + + complt_irq = platform_get_irq_byname(pdev, "complete"); + if (complt_irq < 0) { + dev_err(&pdev->dev, "no IRQ for complt_irq: %d\n", + complt_irq); + return complt_irq; + } + wait_ctrl.complt_irq = complt_irq; + + ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, + 0, dev_name(&pdev->dev), &wait_ctrl); + if (ret < 0) { + dev_err(&pdev->dev, "cannot request complt_irq\n"); + return ret; + } + disable_irq(complt_irq); + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_INIT); + if (res.a0) { + dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", + res.a0); + return -ENOMEM; + } + + ret = rockchip_get_freq_info(dmcfreq); + if (ret < 0) { + dev_err(&pdev->dev, "cannot get frequency info\n"); + return ret; + } + dmcfreq->is_set_rate_direct = true; + + dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; + + return 0; +} + +static __maybe_unused int rk3588_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct arm_smccc_res res; + int ret; + int complt_irq; + + res = sip_smc_dram(0, 0, ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); + dev_notice(&pdev->dev, "current ATF version 0x%lx\n", res.a1); + if (res.a0) { + dev_err(&pdev->dev, "trusted firmware unsupported, please update.\n"); + return -ENXIO; + } + + /* + * first 4KB is used for interface parameters + * after 4KB is dts parameters + * request share memory size 4KB * 2 + */ + res = sip_smc_request_share_mem(2, SHARE_PAGE_TYPE_DDR); + if (res.a0 != 0) { + dev_err(&pdev->dev, "no ATF memory for init\n"); + return -ENOMEM; + } + ddr_psci_param = (struct share_params *)res.a1; + /* Clear ddr_psci_param, size is 4KB * 2 */ + memset_io(ddr_psci_param, 0x0, 4096 * 2); + + /* start mcu with sip_smc_dram */ + wait_ctrl.dcf_en = 2; + + init_waitqueue_head(&wait_ctrl.wait_wq); + wait_ctrl.wait_en = 1; + wait_ctrl.wait_time_out_ms = 17 * 5; + + complt_irq = platform_get_irq_byname(pdev, "complete"); + if (complt_irq < 0) { + dev_err(&pdev->dev, "no IRQ for complt_irq: %d\n", complt_irq); + return complt_irq; + } + wait_ctrl.complt_irq = complt_irq; + + ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, + 0, dev_name(&pdev->dev), &wait_ctrl); + if (ret < 0) { + dev_err(&pdev->dev, "cannot request complt_irq\n"); + return ret; + } + disable_irq(complt_irq); + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, ROCKCHIP_SIP_CONFIG_DRAM_INIT); + if (res.a0) { + dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", res.a0); + return -ENOMEM; + } + + ret = rockchip_get_freq_info(dmcfreq); + if (ret < 0) { + dev_err(&pdev->dev, "cannot get frequency info\n"); + return ret; + } + dmcfreq->is_set_rate_direct = true; + + dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; + + return 0; +} + +static __maybe_unused int rv1126_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + struct arm_smccc_res res; + u32 size; + int ret; + int complt_irq; + struct device_node *node; + + res = sip_smc_dram(0, 0, + ROCKCHIP_SIP_CONFIG_DRAM_GET_VERSION); + dev_notice(&pdev->dev, "current ATF version 0x%lx\n", res.a1); + if (res.a0 || res.a1 < 0x100) { + dev_err(&pdev->dev, + "trusted firmware need to update or is invalid!\n"); + return -ENXIO; + } + + /* + * first 4KB is used for interface parameters + * after 4KB * N is dts parameters + */ + size = sizeof(struct rk1808_ddr_dts_config_timing); + res = sip_smc_request_share_mem(DIV_ROUND_UP(size, 4096) + 1, + SHARE_PAGE_TYPE_DDR); + if (res.a0 != 0) { + dev_err(&pdev->dev, "no ATF memory for init\n"); + return -ENOMEM; + } + ddr_psci_param = (struct share_params *)res.a1; + of_get_rv1126_timings(&pdev->dev, pdev->dev.of_node, + (uint32_t *)ddr_psci_param); + + /* enable start dcf in kernel after dcf ready */ + node = of_parse_phandle(pdev->dev.of_node, "dcf", 0); + wait_ctrl.regmap_dcf = syscon_node_to_regmap(node); + if (IS_ERR(wait_ctrl.regmap_dcf)) + return PTR_ERR(wait_ctrl.regmap_dcf); + wait_ctrl.dcf_en = 1; + + init_waitqueue_head(&wait_ctrl.wait_wq); + wait_ctrl.wait_en = 1; + wait_ctrl.wait_time_out_ms = 17 * 5; + + complt_irq = platform_get_irq_byname(pdev, "complete"); + if (complt_irq < 0) { + dev_err(&pdev->dev, "no IRQ for complt_irq: %d\n", + complt_irq); + return complt_irq; + } + wait_ctrl.complt_irq = complt_irq; + + ret = devm_request_irq(&pdev->dev, complt_irq, wait_dcf_complete_irq, + 0, dev_name(&pdev->dev), &wait_ctrl); + if (ret < 0) { + dev_err(&pdev->dev, "cannot request complt_irq\n"); + return ret; + } + disable_irq(complt_irq); + + if (of_property_read_u32(pdev->dev.of_node, "update_drv_odt_cfg", + &ddr_psci_param->update_drv_odt_cfg)) + ddr_psci_param->update_drv_odt_cfg = 0; + + if (of_property_read_u32(pdev->dev.of_node, "update_deskew_cfg", + &ddr_psci_param->update_deskew_cfg)) + ddr_psci_param->update_deskew_cfg = 0; + + dmcfreq->set_rate_params = ddr_psci_param; + rockchip_set_ddrclk_params(dmcfreq->set_rate_params); + rockchip_set_ddrclk_dmcfreq_wait_complete(rockchip_dmcfreq_wait_complete); + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR, 0, + ROCKCHIP_SIP_CONFIG_DRAM_INIT); + if (res.a0) { + dev_err(&pdev->dev, "rockchip_sip_config_dram_init error:%lx\n", + res.a0); + return -ENOMEM; + } + + dmcfreq->set_auto_self_refresh = rockchip_ddr_set_auto_self_refresh; + + return 0; +} + +static const struct of_device_id rockchip_dmcfreq_of_match[] = { +#if IS_ENABLED(CONFIG_CPU_PX30) + { .compatible = "rockchip,px30-dmc", .data = px30_dmc_init }, +#endif +#if IS_ENABLED(CONFIG_CPU_RK1808) + { .compatible = "rockchip,rk1808-dmc", .data = rk1808_dmc_init }, +#endif +#if IS_ENABLED(CONFIG_CPU_RK312X) + { .compatible = "rockchip,rk3128-dmc", .data = rk3128_dmc_init }, +#endif +#if IS_ENABLED(CONFIG_CPU_RK322X) + { .compatible = "rockchip,rk3228-dmc", .data = rk3228_dmc_init }, +#endif +#if IS_ENABLED(CONFIG_CPU_RK3288) + { .compatible = "rockchip,rk3288-dmc", .data = rk3288_dmc_init }, +#endif +#if IS_ENABLED(CONFIG_CPU_RK3308) + { .compatible = "rockchip,rk3308-dmc", .data = NULL }, +#endif +#if IS_ENABLED(CONFIG_CPU_RK3328) + { .compatible = "rockchip,rk3328-dmc", .data = rk3328_dmc_init }, +#endif +#if IS_ENABLED(CONFIG_CPU_RK3368) + { .compatible = "rockchip,rk3368-dmc", .data = rk3368_dmc_init }, +#endif +#if IS_ENABLED(CONFIG_CPU_RK3399) + { .compatible = "rockchip,rk3399-dmc", .data = rk3399_dmc_init }, +#endif +#if IS_ENABLED(CONFIG_CPU_RK3568) + { .compatible = "rockchip,rk3568-dmc", .data = rk3568_dmc_init }, +#endif +#if IS_ENABLED(CONFIG_CPU_RK3588) + { .compatible = "rockchip,rk3588-dmc", .data = rk3588_dmc_init }, +#endif +#if IS_ENABLED(CONFIG_CPU_RV1126) + { .compatible = "rockchip,rv1126-dmc", .data = rv1126_dmc_init }, +#endif + { }, +}; +MODULE_DEVICE_TABLE(of, rockchip_dmcfreq_of_match); + +static int rockchip_get_freq_map_talbe(struct device_node *np, char *porp_name, + struct freq_map_table **table) +{ + struct freq_map_table *tbl; + const struct property *prop; + unsigned int temp_freq = 0; + int count, i; + + prop = of_find_property(np, porp_name, NULL); + if (!prop) + return -EINVAL; + + if (!prop->value) + return -ENODATA; + + count = of_property_count_u32_elems(np, porp_name); + if (count < 0) + return -EINVAL; + + if (count % 3) + return -EINVAL; + + tbl = kzalloc(sizeof(*tbl) * (count / 3 + 1), GFP_KERNEL); + if (!tbl) + return -ENOMEM; + + for (i = 0; i < count / 3; i++) { + of_property_read_u32_index(np, porp_name, 3 * i, &tbl[i].min); + of_property_read_u32_index(np, porp_name, 3 * i + 1, + &tbl[i].max); + of_property_read_u32_index(np, porp_name, 3 * i + 2, + &temp_freq); + tbl[i].freq = temp_freq * 1000; + } + + tbl[i].min = 0; + tbl[i].max = 0; + tbl[i].freq = DMCFREQ_TABLE_END; + + *table = tbl; + + return 0; +} + +static int rockchip_get_rl_map_talbe(struct device_node *np, char *porp_name, + struct rl_map_table **table) +{ + struct rl_map_table *tbl; + const struct property *prop; + int count, i; + + prop = of_find_property(np, porp_name, NULL); + if (!prop) + return -EINVAL; + + if (!prop->value) + return -ENODATA; + + count = of_property_count_u32_elems(np, porp_name); + if (count < 0) + return -EINVAL; + + if (count % 2) + return -EINVAL; + + tbl = kzalloc(sizeof(*tbl) * (count / 2 + 1), GFP_KERNEL); + if (!tbl) + return -ENOMEM; + + for (i = 0; i < count / 2; i++) { + of_property_read_u32_index(np, porp_name, 2 * i, &tbl[i].pn); + of_property_read_u32_index(np, porp_name, 2 * i + 1, + &tbl[i].rl); + } + + tbl[i].pn = 0; + tbl[i].rl = DMCFREQ_TABLE_END; + + *table = tbl; + + return 0; +} + +static int rockchip_get_system_status_rate(struct device_node *np, + char *porp_name, + struct rockchip_dmcfreq *dmcfreq) +{ + const struct property *prop; + unsigned int status = 0, freq = 0; + unsigned long temp_rate = 0; + int count, i; + + prop = of_find_property(np, porp_name, NULL); + if (!prop) + return -ENODEV; + + if (!prop->value) + return -ENODATA; + + count = of_property_count_u32_elems(np, porp_name); + if (count < 0) + return -EINVAL; + + if (count % 2) + return -EINVAL; + + for (i = 0; i < count / 2; i++) { + of_property_read_u32_index(np, porp_name, 2 * i, + &status); + of_property_read_u32_index(np, porp_name, 2 * i + 1, + &freq); + switch (status) { + case SYS_STATUS_NORMAL: + dmcfreq->normal_rate = freq * 1000; + break; + case SYS_STATUS_SUSPEND: + dmcfreq->suspend_rate = freq * 1000; + break; + case SYS_STATUS_VIDEO_1080P: + dmcfreq->video_1080p_rate = freq * 1000; + break; + case SYS_STATUS_VIDEO_4K: + dmcfreq->video_4k_rate = freq * 1000; + break; + case SYS_STATUS_VIDEO_4K_10B: + dmcfreq->video_4k_10b_rate = freq * 1000; + break; + case SYS_STATUS_PERFORMANCE: + dmcfreq->performance_rate = freq * 1000; + break; + case SYS_STATUS_HDMI: + dmcfreq->hdmi_rate = freq * 1000; + break; + case SYS_STATUS_IDLE: + dmcfreq->idle_rate = freq * 1000; + break; + case SYS_STATUS_REBOOT: + dmcfreq->reboot_rate = freq * 1000; + break; + case SYS_STATUS_BOOST: + dmcfreq->boost_rate = freq * 1000; + break; + case SYS_STATUS_ISP: + case SYS_STATUS_CIF0: + case SYS_STATUS_CIF1: + case SYS_STATUS_DUALVIEW: + temp_rate = freq * 1000; + if (dmcfreq->fixed_rate < temp_rate) + dmcfreq->fixed_rate = temp_rate; + break; + case SYS_STATUS_LOW_POWER: + dmcfreq->low_power_rate = freq * 1000; + break; + default: + break; + } + } + + return 0; +} + +static unsigned long rockchip_freq_level_2_rate(struct rockchip_dmcfreq *dmcfreq, + unsigned int level) +{ + unsigned long rate = 0; + + switch (level) { + case DMC_FREQ_LEVEL_LOW: + rate = dmcfreq->rate_low; + break; + case DMC_FREQ_LEVEL_MID_LOW: + rate = dmcfreq->rate_mid_low; + break; + case DMC_FREQ_LEVEL_MID_HIGH: + rate = dmcfreq->rate_mid_high; + break; + case DMC_FREQ_LEVEL_HIGH: + rate = dmcfreq->rate_high; + break; + default: + break; + } + + return rate; +} + +static int rockchip_get_system_status_level(struct device_node *np, + char *porp_name, + struct rockchip_dmcfreq *dmcfreq) +{ + const struct property *prop; + unsigned int status = 0, level = 0; + unsigned long temp_rate = 0; + int count, i; + + prop = of_find_property(np, porp_name, NULL); + if (!prop) + return -ENODEV; + + if (!prop->value) + return -ENODATA; + + count = of_property_count_u32_elems(np, porp_name); + if (count < 0) + return -EINVAL; + + if (count % 2) + return -EINVAL; + + if (dmcfreq->freq_count == 1) { + dmcfreq->rate_low = dmcfreq->freq_info_rate[0]; + dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[0]; + dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[0]; + dmcfreq->rate_high = dmcfreq->freq_info_rate[0]; + } else if (dmcfreq->freq_count == 2) { + dmcfreq->rate_low = dmcfreq->freq_info_rate[0]; + dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[0]; + dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[1]; + dmcfreq->rate_high = dmcfreq->freq_info_rate[1]; + } else if (dmcfreq->freq_count == 3) { + dmcfreq->rate_low = dmcfreq->freq_info_rate[0]; + dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[1]; + dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[1]; + dmcfreq->rate_high = dmcfreq->freq_info_rate[2]; + } else if (dmcfreq->freq_count == 4) { + dmcfreq->rate_low = dmcfreq->freq_info_rate[0]; + dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[1]; + dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[2]; + dmcfreq->rate_high = dmcfreq->freq_info_rate[3]; + } else if (dmcfreq->freq_count == 5 || dmcfreq->freq_count == 6) { + dmcfreq->rate_low = dmcfreq->freq_info_rate[0]; + dmcfreq->rate_mid_low = dmcfreq->freq_info_rate[1]; + dmcfreq->rate_mid_high = dmcfreq->freq_info_rate[dmcfreq->freq_count - 2]; + dmcfreq->rate_high = dmcfreq->freq_info_rate[dmcfreq->freq_count - 1]; + } else { + return -EINVAL; + } + + dmcfreq->auto_min_rate = dmcfreq->rate_low; + + for (i = 0; i < count / 2; i++) { + of_property_read_u32_index(np, porp_name, 2 * i, + &status); + of_property_read_u32_index(np, porp_name, 2 * i + 1, + &level); + switch (status) { + case SYS_STATUS_NORMAL: + dmcfreq->normal_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "normal_rate = %ld\n", dmcfreq->normal_rate); + break; + case SYS_STATUS_SUSPEND: + dmcfreq->suspend_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "suspend_rate = %ld\n", dmcfreq->suspend_rate); + break; + case SYS_STATUS_VIDEO_1080P: + dmcfreq->video_1080p_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "video_1080p_rate = %ld\n", + dmcfreq->video_1080p_rate); + break; + case SYS_STATUS_VIDEO_4K: + dmcfreq->video_4k_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "video_4k_rate = %ld\n", dmcfreq->video_4k_rate); + break; + case SYS_STATUS_VIDEO_4K_10B: + dmcfreq->video_4k_10b_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "video_4k_10b_rate = %ld\n", + dmcfreq->video_4k_10b_rate); + break; + case SYS_STATUS_PERFORMANCE: + dmcfreq->performance_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "performance_rate = %ld\n", + dmcfreq->performance_rate); + break; + case SYS_STATUS_HDMI: + dmcfreq->hdmi_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "hdmi_rate = %ld\n", dmcfreq->hdmi_rate); + break; + case SYS_STATUS_IDLE: + dmcfreq->idle_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "idle_rate = %ld\n", dmcfreq->idle_rate); + break; + case SYS_STATUS_REBOOT: + dmcfreq->reboot_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "reboot_rate = %ld\n", dmcfreq->reboot_rate); + break; + case SYS_STATUS_BOOST: + dmcfreq->boost_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "boost_rate = %ld\n", dmcfreq->boost_rate); + break; + case SYS_STATUS_ISP: + case SYS_STATUS_CIF0: + case SYS_STATUS_CIF1: + case SYS_STATUS_DUALVIEW: + temp_rate = rockchip_freq_level_2_rate(dmcfreq, level); + if (dmcfreq->fixed_rate < temp_rate) { + dmcfreq->fixed_rate = temp_rate; + dev_info(dmcfreq->dev, + "fixed_rate(isp|cif0|cif1|dualview) = %ld\n", + dmcfreq->fixed_rate); + } + break; + case SYS_STATUS_LOW_POWER: + dmcfreq->low_power_rate = rockchip_freq_level_2_rate(dmcfreq, level); + dev_info(dmcfreq->dev, "low_power_rate = %ld\n", dmcfreq->low_power_rate); + break; + default: + break; + } + } + + return 0; +} + +static void rockchip_dmcfreq_update_target(struct rockchip_dmcfreq *dmcfreq) +{ + struct devfreq *devfreq = dmcfreq->info.devfreq; + + mutex_lock(&devfreq->lock); + update_devfreq(devfreq); + mutex_unlock(&devfreq->lock); +} + +static int rockchip_dmcfreq_system_status_notifier(struct notifier_block *nb, + unsigned long status, + void *ptr) +{ + struct rockchip_dmcfreq *dmcfreq = system_status_to_dmcfreq(nb); + unsigned long target_rate = 0; + unsigned int refresh = false; + bool is_fixed = false; + + if (dmcfreq->fixed_rate && (is_dualview(status) || is_isp(status))) { + if (dmcfreq->is_fixed) + return NOTIFY_OK; + is_fixed = true; + target_rate = dmcfreq->fixed_rate; + goto next; + } + + if (dmcfreq->reboot_rate && (status & SYS_STATUS_REBOOT)) { + if (dmcfreq->info.auto_freq_en) + devfreq_monitor_stop(dmcfreq->info.devfreq); + target_rate = dmcfreq->reboot_rate; + goto next; + } + + if (dmcfreq->suspend_rate && (status & SYS_STATUS_SUSPEND)) { + target_rate = dmcfreq->suspend_rate; + refresh = true; + goto next; + } + + if (dmcfreq->low_power_rate && (status & SYS_STATUS_LOW_POWER)) { + target_rate = dmcfreq->low_power_rate; + goto next; + } + + if (dmcfreq->performance_rate && (status & SYS_STATUS_PERFORMANCE)) { + if (dmcfreq->performance_rate > target_rate) + target_rate = dmcfreq->performance_rate; + } + + if (dmcfreq->hdmi_rate && (status & SYS_STATUS_HDMI)) { + if (dmcfreq->hdmi_rate > target_rate) + target_rate = dmcfreq->hdmi_rate; + } + + if (dmcfreq->video_4k_rate && (status & SYS_STATUS_VIDEO_4K)) { + if (dmcfreq->video_4k_rate > target_rate) + target_rate = dmcfreq->video_4k_rate; + } + + if (dmcfreq->video_4k_10b_rate && (status & SYS_STATUS_VIDEO_4K_10B)) { + if (dmcfreq->video_4k_10b_rate > target_rate) + target_rate = dmcfreq->video_4k_10b_rate; + } + + if (dmcfreq->video_1080p_rate && (status & SYS_STATUS_VIDEO_1080P)) { + if (dmcfreq->video_1080p_rate > target_rate) + target_rate = dmcfreq->video_1080p_rate; + } + +next: + + dev_dbg(dmcfreq->dev, "status=0x%x\n", (unsigned int)status); + dmcfreq->is_fixed = is_fixed; + dmcfreq->status_rate = target_rate; + if (dmcfreq->refresh != refresh) { + if (dmcfreq->set_auto_self_refresh) + dmcfreq->set_auto_self_refresh(refresh); + dmcfreq->refresh = refresh; + } + rockchip_dmcfreq_update_target(dmcfreq); + + return NOTIFY_OK; +} + +static ssize_t rockchip_dmcfreq_status_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + unsigned int status = rockchip_get_system_status(); + + return sprintf(buf, "0x%x\n", status); +} + +static ssize_t rockchip_dmcfreq_status_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + if (!count) + return -EINVAL; + + rockchip_update_system_status(buf); + + return count; +} + +static DEVICE_ATTR(system_status, 0644, rockchip_dmcfreq_status_show, + rockchip_dmcfreq_status_store); + +static ssize_t upthreshold_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev->parent); + struct rockchip_dmcfreq_ondemand_data *data = &dmcfreq->ondemand_data; + + return sprintf(buf, "%d\n", data->upthreshold); +} + +static ssize_t upthreshold_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev->parent); + struct rockchip_dmcfreq_ondemand_data *data = &dmcfreq->ondemand_data; + unsigned int value; + + if (kstrtouint(buf, 10, &value)) + return -EINVAL; + + data->upthreshold = value; + + return count; +} + +static DEVICE_ATTR_RW(upthreshold); + +static ssize_t downdifferential_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev->parent); + struct rockchip_dmcfreq_ondemand_data *data = &dmcfreq->ondemand_data; + + return sprintf(buf, "%d\n", data->downdifferential); +} + +static ssize_t downdifferential_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev->parent); + struct rockchip_dmcfreq_ondemand_data *data = &dmcfreq->ondemand_data; + unsigned int value; + + if (kstrtouint(buf, 10, &value)) + return -EINVAL; + + data->downdifferential = value; + + return count; +} + +static DEVICE_ATTR_RW(downdifferential); + +static unsigned long get_nocp_req_rate(struct rockchip_dmcfreq *dmcfreq) +{ + unsigned long target = 0, cpu_bw = 0; + int i; + + if (!dmcfreq->cpu_bw_tbl || dmcfreq->nocp_cpu_id < 0) + goto out; + + cpu_bw = dmcfreq->nocp_bw[dmcfreq->nocp_cpu_id]; + + for (i = 0; dmcfreq->cpu_bw_tbl[i].freq != CPUFREQ_TABLE_END; i++) { + if (cpu_bw >= dmcfreq->cpu_bw_tbl[i].min) + target = dmcfreq->cpu_bw_tbl[i].freq; + } + +out: + return target; +} + +static int devfreq_dmc_ondemand_func(struct devfreq *df, + unsigned long *freq) +{ + int err; + struct devfreq_dev_status *stat; + unsigned long long a, b; + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(df->dev.parent); + struct rockchip_dmcfreq_ondemand_data *data = &dmcfreq->ondemand_data; + unsigned int upthreshold = data->upthreshold; + unsigned int downdifferential = data->downdifferential; + unsigned long target_freq = 0, nocp_req_rate = 0; + u64 now; + + if (dmcfreq->info.auto_freq_en && !dmcfreq->is_fixed) { + if (dmcfreq->status_rate) + target_freq = dmcfreq->status_rate; + else if (dmcfreq->auto_min_rate) + target_freq = dmcfreq->auto_min_rate; + nocp_req_rate = get_nocp_req_rate(dmcfreq); + target_freq = max3(target_freq, nocp_req_rate, + dmcfreq->info.vop_req_rate); + now = ktime_to_us(ktime_get()); + if (now < dmcfreq->touchboostpulse_endtime) + target_freq = max(target_freq, dmcfreq->boost_rate); + } else { + if (dmcfreq->status_rate) + target_freq = dmcfreq->status_rate; + else if (dmcfreq->normal_rate) + target_freq = dmcfreq->normal_rate; + if (target_freq) + *freq = target_freq; + if (dmcfreq->info.auto_freq_en && !devfreq_update_stats(df)) + return 0; + goto reset_last_status; + } + + if (!upthreshold || !downdifferential) + goto reset_last_status; + + if (upthreshold > 100 || + upthreshold < downdifferential) + goto reset_last_status; + + err = devfreq_update_stats(df); + if (err) + goto reset_last_status; + + stat = &df->last_status; + + /* Assume MAX if it is going to be divided by zero */ + if (stat->total_time == 0) { + *freq = DEVFREQ_MAX_FREQ; + return 0; + } + + /* Prevent overflow */ + if (stat->busy_time >= (1 << 24) || stat->total_time >= (1 << 24)) { + stat->busy_time >>= 7; + stat->total_time >>= 7; + } + + /* Set MAX if it's busy enough */ + if (stat->busy_time * 100 > + stat->total_time * upthreshold) { + *freq = DEVFREQ_MAX_FREQ; + return 0; + } + + /* Set MAX if we do not know the initial frequency */ + if (stat->current_frequency == 0) { + *freq = DEVFREQ_MAX_FREQ; + return 0; + } + + /* Keep the current frequency */ + if (stat->busy_time * 100 > + stat->total_time * (upthreshold - downdifferential)) { + *freq = max(target_freq, stat->current_frequency); + return 0; + } + + /* Set the desired frequency based on the load */ + a = stat->busy_time; + a *= stat->current_frequency; + b = div_u64(a, stat->total_time); + b *= 100; + b = div_u64(b, (upthreshold - downdifferential / 2)); + *freq = max_t(unsigned long, target_freq, b); + + return 0; + +reset_last_status: + reset_last_status(df); + + return 0; +} + +static int devfreq_dmc_ondemand_handler(struct devfreq *devfreq, + unsigned int event, void *data) +{ + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(devfreq->dev.parent); + + if (!dmcfreq->info.auto_freq_en) + return 0; + + switch (event) { + case DEVFREQ_GOV_START: + devfreq_monitor_start(devfreq); + break; + + case DEVFREQ_GOV_STOP: + devfreq_monitor_stop(devfreq); + break; + + case DEVFREQ_GOV_UPDATE_INTERVAL: + devfreq_update_interval(devfreq, (unsigned int *)data); + break; + + case DEVFREQ_GOV_SUSPEND: + devfreq_monitor_suspend(devfreq); + break; + + case DEVFREQ_GOV_RESUME: + devfreq_monitor_resume(devfreq); + break; + + default: + break; + } + + return 0; +} + +static struct devfreq_governor devfreq_dmc_ondemand = { + .name = "dmc_ondemand", + .get_target_freq = devfreq_dmc_ondemand_func, + .event_handler = devfreq_dmc_ondemand_handler, +}; + +static int rockchip_dmcfreq_enable_event(struct rockchip_dmcfreq *dmcfreq) +{ + int i, ret; + + if (!dmcfreq->info.auto_freq_en) + return 0; + + for (i = 0; i < dmcfreq->edev_count; i++) { + ret = devfreq_event_enable_edev(dmcfreq->edev[i]); + if (ret < 0) { + dev_err(dmcfreq->dev, + "failed to enable devfreq-event\n"); + return ret; + } + } + + return 0; +} + +static int rockchip_dmcfreq_disable_event(struct rockchip_dmcfreq *dmcfreq) +{ + int i, ret; + + if (!dmcfreq->info.auto_freq_en) + return 0; + + for (i = 0; i < dmcfreq->edev_count; i++) { + ret = devfreq_event_disable_edev(dmcfreq->edev[i]); + if (ret < 0) { + dev_err(dmcfreq->dev, + "failed to disable devfreq-event\n"); + return ret; + } + } + + return 0; +} + +static int rockchip_get_edev_id(struct rockchip_dmcfreq *dmcfreq, + const char *name) +{ + struct devfreq_event_dev *edev; + int i; + + for (i = 0; i < dmcfreq->edev_count; i++) { + edev = dmcfreq->edev[i]; + if (!strcmp(edev->desc->name, name)) + return i; + } + + return -EINVAL; +} + +static int rockchip_dmcfreq_get_event(struct rockchip_dmcfreq *dmcfreq) +{ + struct device *dev = dmcfreq->dev; + struct device_node *events_np, *np = dev->of_node; + int i, j, count, available_count = 0; + + count = devfreq_event_get_edev_count(dev, "devfreq-events"); + if (count < 0) { + dev_dbg(dev, "failed to get count of devfreq-event dev\n"); + return 0; + } + for (i = 0; i < count; i++) { + events_np = of_parse_phandle(np, "devfreq-events", i); + if (!events_np) + continue; + if (of_device_is_available(events_np)) + available_count++; + of_node_put(events_np); + } + if (!available_count) { + dev_dbg(dev, "failed to get available devfreq-event\n"); + return 0; + } + dmcfreq->edev_count = available_count; + dmcfreq->edev = devm_kzalloc(dev, + sizeof(*dmcfreq->edev) * available_count, + GFP_KERNEL); + if (!dmcfreq->edev) + return -ENOMEM; + + for (i = 0, j = 0; i < count; i++) { + events_np = of_parse_phandle(np, "devfreq-events", i); + if (!events_np) + continue; + if (of_device_is_available(events_np)) { + of_node_put(events_np); + if (j >= available_count) { + dev_err(dev, "invalid event conut\n"); + return -EINVAL; + } + dmcfreq->edev[j] = + devfreq_event_get_edev_by_phandle(dev, "devfreq-events", i); + if (IS_ERR(dmcfreq->edev[j])) + return -EPROBE_DEFER; + j++; + } else { + of_node_put(events_np); + } + } + dmcfreq->info.auto_freq_en = true; + dmcfreq->dfi_id = rockchip_get_edev_id(dmcfreq, "dfi"); + dmcfreq->nocp_cpu_id = rockchip_get_edev_id(dmcfreq, "nocp-cpu"); + dmcfreq->nocp_bw = + devm_kzalloc(dev, sizeof(*dmcfreq->nocp_bw) * available_count, + GFP_KERNEL); + if (!dmcfreq->nocp_bw) + return -ENOMEM; + + return 0; +} + +static int rockchip_dmcfreq_power_control(struct rockchip_dmcfreq *dmcfreq) +{ + struct device *dev = dmcfreq->dev; + + dmcfreq->vdd_center = devm_regulator_get_optional(dev, "center"); + if (IS_ERR(dmcfreq->vdd_center)) { + dev_err(dev, "Cannot get the regulator \"center\"\n"); + return PTR_ERR(dmcfreq->vdd_center); + } + + dmcfreq->dmc_clk = devm_clk_get(dev, "dmc_clk"); + if (IS_ERR(dmcfreq->dmc_clk)) { + dev_err(dev, "Cannot get the clk dmc_clk. If using SCMI, trusted firmware need update to V1.01 and above.\n"); + return PTR_ERR(dmcfreq->dmc_clk); + } + dmcfreq->rate = clk_get_rate(dmcfreq->dmc_clk); + + return 0; +} + +static int rockchip_dmcfreq_dmc_init(struct platform_device *pdev, + struct rockchip_dmcfreq *dmcfreq) +{ + const struct of_device_id *match; + int (*init)(struct platform_device *pdev, + struct rockchip_dmcfreq *data); + int ret; + + match = of_match_node(rockchip_dmcfreq_of_match, pdev->dev.of_node); + if (match) { + init = match->data; + if (init) { + ret = init(pdev, dmcfreq); + if (ret) + return ret; + } + } + + return 0; +} + +static void rockchip_dmcfreq_parse_dt(struct rockchip_dmcfreq *dmcfreq) +{ + struct device *dev = dmcfreq->dev; + struct device_node *np = dev->of_node; + + if (!rockchip_get_system_status_rate(np, "system-status-freq", dmcfreq)) + dmcfreq->system_status_en = true; + else if (!rockchip_get_system_status_level(np, "system-status-level", dmcfreq)) + dmcfreq->system_status_en = true; + + of_property_read_u32(np, "min-cpu-freq", &dmcfreq->min_cpu_freq); + + of_property_read_u32(np, "upthreshold", + &dmcfreq->ondemand_data.upthreshold); + of_property_read_u32(np, "downdifferential", + &dmcfreq->ondemand_data.downdifferential); + if (dmcfreq->info.auto_freq_en) + of_property_read_u32(np, "auto-freq-en", + &dmcfreq->info.auto_freq_en); + if (!dmcfreq->auto_min_rate) { + of_property_read_u32(np, "auto-min-freq", + (u32 *)&dmcfreq->auto_min_rate); + dmcfreq->auto_min_rate *= 1000; + } + + if (rockchip_get_freq_map_talbe(np, "cpu-bw-dmc-freq", + &dmcfreq->cpu_bw_tbl)) + dev_dbg(dev, "failed to get cpu bandwidth to dmc rate\n"); + if (rockchip_get_freq_map_talbe(np, "vop-frame-bw-dmc-freq", + &dmcfreq->info.vop_frame_bw_tbl)) + dev_dbg(dev, "failed to get vop frame bandwidth to dmc rate\n"); + if (rockchip_get_freq_map_talbe(np, "vop-bw-dmc-freq", + &dmcfreq->info.vop_bw_tbl)) + dev_err(dev, "failed to get vop bandwidth to dmc rate\n"); + if (rockchip_get_rl_map_talbe(np, "vop-pn-msch-readlatency", + &dmcfreq->info.vop_pn_rl_tbl)) + dev_err(dev, "failed to get vop pn to msch rl\n"); + + of_property_read_u32(np, "touchboost_duration", + (u32 *)&dmcfreq->touchboostpulse_duration_val); + if (dmcfreq->touchboostpulse_duration_val) + dmcfreq->touchboostpulse_duration_val *= USEC_PER_MSEC; + else + dmcfreq->touchboostpulse_duration_val = 500 * USEC_PER_MSEC; +} + +static int rockchip_dmcfreq_set_volt_only(struct rockchip_dmcfreq *dmcfreq) +{ + struct device *dev = dmcfreq->dev; + struct dev_pm_opp *opp; + unsigned long opp_volt, opp_rate = dmcfreq->rate; + int ret; + + opp = devfreq_recommended_opp(dev, &opp_rate, 0); + if (IS_ERR(opp)) { + dev_err(dev, "Failed to find opp for %lu Hz\n", opp_rate); + return PTR_ERR(opp); + } + opp_volt = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + + ret = regulator_set_voltage(dmcfreq->vdd_center, opp_volt, INT_MAX); + if (ret) { + dev_err(dev, "Cannot set voltage %lu uV\n", opp_volt); + return ret; + } + + return 0; +} + +static int rockchip_dmcfreq_add_devfreq(struct rockchip_dmcfreq *dmcfreq) +{ + struct devfreq_dev_profile *devp = &rockchip_devfreq_dmc_profile; + struct device *dev = dmcfreq->dev; + struct dev_pm_opp *opp; + struct devfreq *devfreq; + unsigned long opp_rate = dmcfreq->rate; + + opp = devfreq_recommended_opp(dev, &opp_rate, 0); + if (IS_ERR(opp)) { + dev_err(dev, "Failed to find opp for %lu Hz\n", opp_rate); + return PTR_ERR(opp); + } + dev_pm_opp_put(opp); + + devp->initial_freq = dmcfreq->rate; + devfreq = devm_devfreq_add_device(dev, devp, "dmc_ondemand", + &dmcfreq->ondemand_data); + if (IS_ERR(devfreq)) { + dev_err(dev, "failed to add devfreq\n"); + return PTR_ERR(devfreq); + } + + devm_devfreq_register_opp_notifier(dev, devfreq); + + devfreq->last_status.current_frequency = opp_rate; + + reset_last_status(devfreq); + + dmcfreq->info.devfreq = devfreq; + + return 0; +} + +static struct monitor_dev_profile dmc_mdevp = { + .type = MONITOR_TPYE_DEV, + .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, + .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, +}; + +static void rockchip_dmcfreq_register_notifier(struct rockchip_dmcfreq *dmcfreq) +{ + int ret; + + if (vop_register_dmc()) + dev_err(dmcfreq->dev, "fail to register notify to vop.\n"); + + dmcfreq->status_nb.notifier_call = + rockchip_dmcfreq_system_status_notifier; + ret = rockchip_register_system_status_notifier(&dmcfreq->status_nb); + if (ret) + dev_err(dmcfreq->dev, "failed to register system_status nb\n"); + + dmc_mdevp.data = dmcfreq->info.devfreq; + dmcfreq->mdev_info = rockchip_system_monitor_register(dmcfreq->dev, + &dmc_mdevp); + if (IS_ERR(dmcfreq->mdev_info)) { + dev_dbg(dmcfreq->dev, "without without system monitor\n"); + dmcfreq->mdev_info = NULL; + } +} + +static void rockchip_dmcfreq_add_interface(struct rockchip_dmcfreq *dmcfreq) +{ + struct devfreq *devfreq = dmcfreq->info.devfreq; + + if (sysfs_create_file(&devfreq->dev.kobj, &dev_attr_upthreshold.attr)) + dev_err(dmcfreq->dev, + "failed to register upthreshold sysfs file\n"); + if (sysfs_create_file(&devfreq->dev.kobj, + &dev_attr_downdifferential.attr)) + dev_err(dmcfreq->dev, + "failed to register downdifferential sysfs file\n"); + + if (!rockchip_add_system_status_interface(&devfreq->dev)) + return; + if (sysfs_create_file(&devfreq->dev.kobj, + &dev_attr_system_status.attr)) + dev_err(dmcfreq->dev, + "failed to register system_status sysfs file\n"); +} + +static void rockchip_dmcfreq_boost_work(struct work_struct *work) +{ + struct rockchip_dmcfreq *dmcfreq = boost_to_dmcfreq(work); + + rockchip_dmcfreq_update_target(dmcfreq); +} + +static void rockchip_dmcfreq_input_event(struct input_handle *handle, + unsigned int type, + unsigned int code, + int value) +{ + struct rockchip_dmcfreq *dmcfreq = handle->private; + u64 now, endtime; + + if (type != EV_ABS && type != EV_KEY) + return; + + now = ktime_to_us(ktime_get()); + endtime = now + dmcfreq->touchboostpulse_duration_val; + if (endtime < (dmcfreq->touchboostpulse_endtime + 10 * USEC_PER_MSEC)) + return; + dmcfreq->touchboostpulse_endtime = endtime; + + schedule_work(&dmcfreq->boost_work); +} + +static int rockchip_dmcfreq_input_connect(struct input_handler *handler, + struct input_dev *dev, + const struct input_device_id *id) +{ + int error; + struct input_handle *handle; + struct rockchip_dmcfreq *dmcfreq = input_hd_to_dmcfreq(handler); + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->dev = dev; + handle->handler = handler; + handle->name = "dmcfreq"; + handle->private = dmcfreq; + + error = input_register_handle(handle); + if (error) + goto err2; + + error = input_open_device(handle); + if (error) + goto err1; + + return 0; +err1: + input_unregister_handle(handle); +err2: + kfree(handle); + return error; +} + +static void rockchip_dmcfreq_input_disconnect(struct input_handle *handle) +{ + input_close_device(handle); + input_unregister_handle(handle); + kfree(handle); +} + +static const struct input_device_id rockchip_dmcfreq_input_ids[] = { + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT | + INPUT_DEVICE_ID_MATCH_ABSBIT, + .evbit = { BIT_MASK(EV_ABS) }, + .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] = + BIT_MASK(ABS_MT_POSITION_X) | + BIT_MASK(ABS_MT_POSITION_Y) }, + }, + { + .flags = INPUT_DEVICE_ID_MATCH_KEYBIT | + INPUT_DEVICE_ID_MATCH_ABSBIT, + .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) }, + .absbit = { [BIT_WORD(ABS_X)] = + BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) }, + }, + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT, + .evbit = { BIT_MASK(EV_KEY) }, + }, + { }, +}; + +static void rockchip_dmcfreq_boost_init(struct rockchip_dmcfreq *dmcfreq) +{ + if (!dmcfreq->boost_rate) + return; + INIT_WORK(&dmcfreq->boost_work, rockchip_dmcfreq_boost_work); + dmcfreq->input_handler.event = rockchip_dmcfreq_input_event; + dmcfreq->input_handler.connect = rockchip_dmcfreq_input_connect; + dmcfreq->input_handler.disconnect = rockchip_dmcfreq_input_disconnect; + dmcfreq->input_handler.name = "dmcfreq"; + dmcfreq->input_handler.id_table = rockchip_dmcfreq_input_ids; + if (input_register_handler(&dmcfreq->input_handler)) + dev_err(dmcfreq->dev, "failed to register input handler\n"); +} + +static unsigned long model_static_power(struct devfreq *devfreq, + unsigned long voltage) +{ + struct device *dev = devfreq->dev.parent; + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); + + int temperature; + unsigned long temp; + unsigned long temp_squared, temp_cubed, temp_scaling_factor; + const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 10; + + if (!IS_ERR_OR_NULL(dmcfreq->ddr_tz) && dmcfreq->ddr_tz->ops->get_temp) { + int ret; + + ret = + dmcfreq->ddr_tz->ops->get_temp(dmcfreq->ddr_tz, + &temperature); + if (ret) { + dev_warn_ratelimited(dev, + "failed to read temp for ddr thermal zone: %d\n", + ret); + temperature = FALLBACK_STATIC_TEMPERATURE; + } + } else { + temperature = FALLBACK_STATIC_TEMPERATURE; + } + + /* + * Calculate the temperature scaling factor. To be applied to the + * voltage scaled power. + */ + temp = temperature / 1000; + temp_squared = temp * temp; + temp_cubed = temp_squared * temp; + temp_scaling_factor = (dmcfreq->ts[3] * temp_cubed) + + (dmcfreq->ts[2] * temp_squared) + + (dmcfreq->ts[1] * temp) + + dmcfreq->ts[0]; + + return (((dmcfreq->static_coefficient * voltage_cubed) >> 20) + * temp_scaling_factor) / 1000000; +} + +static struct devfreq_cooling_power ddr_cooling_power_data = { + .get_static_power = model_static_power, + .dyn_power_coeff = 120, +}; + +static int ddr_power_model_simple_init(struct rockchip_dmcfreq *dmcfreq) +{ + struct device_node *power_model_node; + const char *tz_name; + u32 temp; + + power_model_node = of_get_child_by_name(dmcfreq->dev->of_node, + "ddr_power_model"); + if (!power_model_node) { + dev_err(dmcfreq->dev, "could not find power_model node\n"); + return -ENODEV; + } + + if (of_property_read_string(power_model_node, "thermal-zone", &tz_name)) { + dev_err(dmcfreq->dev, "ts in power_model not available\n"); + return -EINVAL; + } + + dmcfreq->ddr_tz = thermal_zone_get_zone_by_name(tz_name); + if (IS_ERR(dmcfreq->ddr_tz)) { + pr_warn_ratelimited + ("Error getting ddr thermal zone (%ld), not yet ready?\n", + PTR_ERR(dmcfreq->ddr_tz)); + dmcfreq->ddr_tz = NULL; + + return -EPROBE_DEFER; + } + + if (of_property_read_u32(power_model_node, "static-power-coefficient", + &dmcfreq->static_coefficient)) { + dev_err(dmcfreq->dev, + "static-power-coefficient not available\n"); + return -EINVAL; + } + if (of_property_read_u32(power_model_node, "dynamic-power-coefficient", + &temp)) { + dev_err(dmcfreq->dev, + "dynamic-power-coefficient not available\n"); + return -EINVAL; + } + ddr_cooling_power_data.dyn_power_coeff = (unsigned long)temp; + + if (of_property_read_u32_array + (power_model_node, "ts", (u32 *)dmcfreq->ts, 4)) { + dev_err(dmcfreq->dev, "ts in power_model not available\n"); + return -EINVAL; + } + + return 0; +} + +static void +rockchip_dmcfreq_register_cooling_device(struct rockchip_dmcfreq *dmcfreq) +{ + int ret; + + ret = ddr_power_model_simple_init(dmcfreq); + if (ret) + return; + dmcfreq->devfreq_cooling = + of_devfreq_cooling_register_power(dmcfreq->dev->of_node, + dmcfreq->info.devfreq, + &ddr_cooling_power_data); + if (IS_ERR(dmcfreq->devfreq_cooling)) { + ret = PTR_ERR(dmcfreq->devfreq_cooling); + dev_err(dmcfreq->dev, + "Failed to register cooling device (%d)\n", + ret); + } +} + +static int rockchip_dmcfreq_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rockchip_dmcfreq *data; + int ret; + + data = devm_kzalloc(dev, sizeof(struct rockchip_dmcfreq), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->dev = dev; + data->info.dev = dev; + mutex_init(&data->lock); + INIT_LIST_HEAD(&data->video_info_list); + + ret = rockchip_dmcfreq_get_event(data); + if (ret) + return ret; + + ret = rockchip_dmcfreq_power_control(data); + if (ret) + return ret; + + ret = rockchip_init_opp_table(dev, NULL, "ddr_leakage", "center"); + if (ret) + return ret; + + ret = rockchip_dmcfreq_dmc_init(pdev, data); + if (ret) + return ret; + + rockchip_dmcfreq_parse_dt(data); + if (!data->system_status_en && !data->info.auto_freq_en) { + dev_info(dev, "don't add devfreq feature\n"); + return rockchip_dmcfreq_set_volt_only(data); + } + + cpu_latency_qos_add_request(&pm_qos, PM_QOS_DEFAULT_VALUE); + platform_set_drvdata(pdev, data); + + ret = devfreq_add_governor(&devfreq_dmc_ondemand); + if (ret) + return ret; + ret = rockchip_dmcfreq_enable_event(data); + if (ret) + return ret; + ret = rockchip_dmcfreq_add_devfreq(data); + if (ret) { + rockchip_dmcfreq_disable_event(data); + return ret; + } + + rockchip_dmcfreq_register_notifier(data); + rockchip_dmcfreq_add_interface(data); + rockchip_dmcfreq_boost_init(data); + rockchip_dmcfreq_vop_bandwidth_init(&data->info); + rockchip_dmcfreq_register_cooling_device(data); + + rockchip_set_system_status(SYS_STATUS_NORMAL); + + return 0; +} + +static __maybe_unused int rockchip_dmcfreq_suspend(struct device *dev) +{ + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); + int ret = 0; + + if (!dmcfreq) + return 0; + + ret = rockchip_dmcfreq_disable_event(dmcfreq); + if (ret) + return ret; + + ret = devfreq_suspend_device(dmcfreq->info.devfreq); + if (ret < 0) { + dev_err(dev, "failed to suspend the devfreq devices\n"); + return ret; + } + + return 0; +} + +static __maybe_unused int rockchip_dmcfreq_resume(struct device *dev) +{ + struct rockchip_dmcfreq *dmcfreq = dev_get_drvdata(dev); + int ret = 0; + + if (!dmcfreq) + return 0; + + ret = rockchip_dmcfreq_enable_event(dmcfreq); + if (ret) + return ret; + + ret = devfreq_resume_device(dmcfreq->info.devfreq); + if (ret < 0) { + dev_err(dev, "failed to resume the devfreq devices\n"); + return ret; + } + return ret; +} + +static SIMPLE_DEV_PM_OPS(rockchip_dmcfreq_pm, rockchip_dmcfreq_suspend, + rockchip_dmcfreq_resume); +static struct platform_driver rockchip_dmcfreq_driver = { + .probe = rockchip_dmcfreq_probe, + .driver = { + .name = "rockchip-dmc", + .pm = &rockchip_dmcfreq_pm, + .of_match_table = rockchip_dmcfreq_of_match, + }, +}; +module_platform_driver(rockchip_dmcfreq_driver); + +MODULE_AUTHOR("Finley Xiao "); +MODULE_DESCRIPTION("rockchip dmcfreq driver with devfreq framework"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/devfreq/rockchip_dmc_common.c b/drivers/devfreq/rockchip_dmc_common.c new file mode 100755 index 000000000000..cacc7d02f6fc --- /dev/null +++ b/drivers/devfreq/rockchip_dmc_common.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Rockchip dmc common functions. + * + * Copyright (c) 2021 Rockchip Electronics Co. Ltd. + * Author: Finley Xiao + */ + +#include +#include + +#define msch_rl_to_dmcfreq(work) container_of(to_delayed_work(work), \ + struct rockchip_dmcfreq, \ + msch_rl_work) +#define MSCH_RL_DELAY_TIME 50 /* ms */ + +static struct dmcfreq_common_info *common_info; +static DECLARE_RWSEM(rockchip_dmcfreq_sem); + +void rockchip_dmcfreq_lock(void) +{ + down_read(&rockchip_dmcfreq_sem); +} +EXPORT_SYMBOL(rockchip_dmcfreq_lock); + +void rockchip_dmcfreq_lock_nested(void) +{ + down_read_nested(&rockchip_dmcfreq_sem, SINGLE_DEPTH_NESTING); +} +EXPORT_SYMBOL(rockchip_dmcfreq_lock_nested); + +void rockchip_dmcfreq_unlock(void) +{ + up_read(&rockchip_dmcfreq_sem); +} +EXPORT_SYMBOL(rockchip_dmcfreq_unlock); + +int rockchip_dmcfreq_write_trylock(void) +{ + return down_write_trylock(&rockchip_dmcfreq_sem); +} +EXPORT_SYMBOL(rockchip_dmcfreq_write_trylock); + +void rockchip_dmcfreq_write_unlock(void) +{ + up_write(&rockchip_dmcfreq_sem); +} +EXPORT_SYMBOL(rockchip_dmcfreq_write_unlock); + +static void set_msch_rl(unsigned int readlatency) + +{ + rockchip_dmcfreq_lock(); + dev_dbg(common_info->dev, "rl 0x%x -> 0x%x\n", + common_info->read_latency, readlatency); + if (!common_info->set_msch_readlatency(readlatency)) + common_info->read_latency = readlatency; + else + dev_err(common_info->dev, "failed to set msch rl\n"); + rockchip_dmcfreq_unlock(); +} + +static void set_msch_rl_work(struct work_struct *work) +{ + set_msch_rl(0); + common_info->is_msch_rl_work_started = false; +} + +int rockchip_dmcfreq_vop_bandwidth_init(struct dmcfreq_common_info *info) +{ + if (info->set_msch_readlatency) + INIT_DELAYED_WORK(&info->msch_rl_work, set_msch_rl_work); + common_info = info; + + return 0; +} +EXPORT_SYMBOL(rockchip_dmcfreq_vop_bandwidth_init); + +void rockchip_dmcfreq_vop_bandwidth_update(struct dmcfreq_vop_info *vop_info) +{ + unsigned long vop_last_rate, target = 0; + unsigned int readlatency = 0; + int i; + + if (!common_info) + return; + + dev_dbg(common_info->dev, "line bw=%u, frame bw=%u, pn=%u\n", + vop_info->line_bw_mbyte, vop_info->frame_bw_mbyte, + vop_info->plane_num); + + if (!common_info->vop_pn_rl_tbl || !common_info->set_msch_readlatency) + goto vop_bw_tbl; + for (i = 0; common_info->vop_pn_rl_tbl[i].rl != DMCFREQ_TABLE_END; i++) { + if (vop_info->plane_num >= common_info->vop_pn_rl_tbl[i].pn) + readlatency = common_info->vop_pn_rl_tbl[i].rl; + } + dev_dbg(common_info->dev, "pn=%u\n", vop_info->plane_num); + if (readlatency) { + cancel_delayed_work_sync(&common_info->msch_rl_work); + common_info->is_msch_rl_work_started = false; + if (common_info->read_latency != readlatency) + set_msch_rl(readlatency); + } else if (common_info->read_latency && + !common_info->is_msch_rl_work_started) { + common_info->is_msch_rl_work_started = true; + schedule_delayed_work(&common_info->msch_rl_work, + msecs_to_jiffies(MSCH_RL_DELAY_TIME)); + } + +vop_bw_tbl: + if (!common_info->auto_freq_en || !common_info->vop_bw_tbl) + goto vop_frame_bw_tbl; + + for (i = 0; common_info->vop_bw_tbl[i].freq != DMCFREQ_TABLE_END; i++) { + if (vop_info->line_bw_mbyte >= common_info->vop_bw_tbl[i].min) + target = common_info->vop_bw_tbl[i].freq; + } + +vop_frame_bw_tbl: + if (!common_info->auto_freq_en || !common_info->vop_frame_bw_tbl) + goto next; + for (i = 0; common_info->vop_frame_bw_tbl[i].freq != DMCFREQ_TABLE_END; + i++) { + if (vop_info->frame_bw_mbyte >= common_info->vop_frame_bw_tbl[i].min) { + if (target < common_info->vop_frame_bw_tbl[i].freq) + target = common_info->vop_frame_bw_tbl[i].freq; + } + } + +next: + vop_last_rate = common_info->vop_req_rate; + common_info->vop_req_rate = target; + + if (target > vop_last_rate) { + mutex_lock(&common_info->devfreq->lock); + update_devfreq(common_info->devfreq); + mutex_unlock(&common_info->devfreq->lock); + } +} +EXPORT_SYMBOL(rockchip_dmcfreq_vop_bandwidth_update); + +int rockchip_dmcfreq_vop_bandwidth_request(struct dmcfreq_vop_info *vop_info) +{ + unsigned long target = 0; + int i; + + if (!common_info || !common_info->auto_freq_en || + !common_info->vop_bw_tbl) + return 0; + + for (i = 0; common_info->vop_bw_tbl[i].freq != DMCFREQ_TABLE_END; i++) { + if (vop_info->line_bw_mbyte <= common_info->vop_bw_tbl[i].max) { + target = common_info->vop_bw_tbl[i].freq; + break; + } + } + + if (!target) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL(rockchip_dmcfreq_vop_bandwidth_request); + +MODULE_AUTHOR("Finley Xiao "); +MODULE_DESCRIPTION("rockchip dmcfreq driver with devfreq framework"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/devfreq/rockchip_dmc_dbg.c b/drivers/devfreq/rockchip_dmc_dbg.c new file mode 100755 index 000000000000..80b25e9046d1 --- /dev/null +++ b/drivers/devfreq/rockchip_dmc_dbg.c @@ -0,0 +1,1061 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020, Rockchip Electronics Co., Ltd. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "rockchip_dmc_timing.h" + +/* + * DMCDBG share memory request 4KB for delivery parameter + */ +#define DMCDBG_PAGE_NUMS (1) +#define DMCDBG_SHARE_MEM_SIZE ((DMCDBG_PAGE_NUMS) * 4096) + +#define PROC_DMCDBG_DIR_NAME "dmcdbg" +#define PROC_DMCDBG_DRAM_INFO "dmcinfo" +#define PROC_DMCDBG_POWERSAVE "powersave" +#define PROC_DMCDBG_DRVODT "drvodt" +#define PROC_DMCDBG_DESKEW "deskew" +#define PROC_DMCDBG_REGS_INFO "regsinfo" + +#define DDRDBG_FUNC_GET_VERSION (0x01) +#define DDRDBG_FUNC_GET_SUPPORTED (0x02) +#define DDRDBG_FUNC_GET_DRAM_INFO (0x03) +#define DDRDBG_FUNC_GET_DESKEW_INFO (0x04) +#define DDRDBG_FUNC_UPDATE_DESKEW (0x05) +#define DDRDBG_FUNC_DATA_TRAINING (0x06) +#define DDRDBG_FUNC_UPDATE_DESKEW_TR (0x07) +#define DDRDBG_FUNC_GET_POWERSAVE_INFO (0x08) +#define DDRDBG_FUNC_UPDATE_POWERSAVE (0x09) +#define DDRDBG_FUNC_GET_DRVODT_INFO (0x0a) +#define DDRDBG_FUNC_UPDATE_DRVODT (0x0b) +#define DDRDBG_FUNC_GET_REGISTERS_INFO (0x0c) + +#define DRV_ODT_UNKNOWN (0xffff) +#define DRV_ODT_UNSUSPEND_FIX (0x0) +#define DRV_ODT_SUSPEND_FIX (0x1) + +#define REGS_NAME_LEN_MAX (20) +#define SKEW_GROUP_NUM_MAX (6) +#define SKEW_TIMING_NUM_MAX (50) + +struct rockchip_dmcdbg { + struct device *dev; +}; + +struct proc_dir_entry *proc_dmcdbg_dir; + +struct dram_cap_info { + unsigned int rank; + unsigned int col; + unsigned int bank; + unsigned int buswidth; + unsigned int die_buswidth; + unsigned int row_3_4; + unsigned int cs0_row; + unsigned int cs1_row; + unsigned int cs0_high16bit_row; + unsigned int cs1_high16bit_row; + unsigned int bankgroup; + unsigned int size; +}; + +struct dram_info { + unsigned int version; + char dramtype[10]; + unsigned int dramfreq; + unsigned int channel_num; + struct dram_cap_info ch[2]; +}; + +static const char * const power_save_msg[] = { + "auto power down enable", + "auto power down idle cycle", + "auto self refresh enable", + "auto self refresh idle cycle", + "self refresh with clock gate idle cycle", + "self refresh and power down lite idle cycle", + "standby idle cycle", +}; + +struct power_save_info { + unsigned int pd_en; + unsigned int pd_idle; + unsigned int sr_en; + unsigned int sr_idle; + unsigned int sr_mc_gate_idle; + unsigned int srpd_lite_idle; + unsigned int standby_idle; +}; + +static const char * const drv_odt_msg[] = { + "dram side drv pull-up", + "dram side drv pull-down", + "dram side dq odt pull-up", + "dram side dq odt pull-down", + "dram side ca odt pull-up", + "dram side ca odt pull-down", + "soc side ca drv pull-up", + "soc side ca drv pull-down", + "soc side ck drv pull-up", + "soc side ck drv pull-down", + "soc side cs drv pull-up", + "soc side cs drv pull-down", + "soc side dq drv pull-up", + "soc side dq drv pull-down", + "soc side odt pull-up", + "soc side odt pull-down", + "phy vref inner", + "phy vref out", +}; + +struct drv_odt { + unsigned int value; + unsigned int ohm; + unsigned int flag; +}; + +struct drv_odt_vref { + unsigned int value; + unsigned int percen; + unsigned int flag; +}; + +struct drv_odt_info { + struct drv_odt dram_drv_up; + struct drv_odt dram_drv_down; + struct drv_odt dram_dq_odt_up; + struct drv_odt dram_dq_odt_down; + struct drv_odt dram_ca_odt_up; + struct drv_odt dram_ca_odt_down; + struct drv_odt phy_ca_drv_up; + struct drv_odt phy_ca_drv_down; + struct drv_odt phy_ck_drv_up; + struct drv_odt phy_ck_drv_down; + struct drv_odt phy_cs_drv_up; + struct drv_odt phy_cs_drv_down; + struct drv_odt phy_dq_drv_up; + struct drv_odt phy_dq_drv_down; + struct drv_odt phy_odt_up; + struct drv_odt phy_odt_down; + struct drv_odt_vref phy_vref_inner; + struct drv_odt_vref phy_vref_out; +}; + +struct dmc_registers { + char regs_name[REGS_NAME_LEN_MAX]; + unsigned int regs_addr; +}; + +struct registers_info { + unsigned int regs_num; + struct dmc_registers regs[]; +}; + +struct skew_group { + unsigned int skew_num; + unsigned int *p_skew_info; + char *p_skew_timing[SKEW_TIMING_NUM_MAX]; + char *note; +}; + +struct rockchip_dmcdbg_data { + unsigned int inited_flag; + void __iomem *share_memory; + unsigned int skew_group_num; + struct skew_group skew_group[SKEW_GROUP_NUM_MAX]; +}; + +static struct rockchip_dmcdbg_data dmcdbg_data; + +struct skew_info_rv1126 { + unsigned int ca_skew[32]; + unsigned int cs0_a_skew[44]; + unsigned int cs0_b_skew[44]; + unsigned int cs1_a_skew[44]; + unsigned int cs1_b_skew[44]; +}; + +static int dmcinfo_proc_show(struct seq_file *m, void *v) +{ + struct arm_smccc_res res; + struct dram_info *p_dram_info; + struct file *fp = NULL; + char cur_freq[20] = {0}; + char governor[20] = {0}; + loff_t pos; + u32 i; + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_GET_DRAM_INFO, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + seq_printf(m, "rockchip_sip_config_dram_debug error:%lx\n", + res.a0); + return -ENOMEM; + } + + if (!dmcdbg_data.inited_flag) { + seq_puts(m, "dmcdbg_data no int\n"); + return -EPERM; + } + p_dram_info = (struct dram_info *)dmcdbg_data.share_memory; + + /* dram type information */ + seq_printf(m, + "DramType: %s\n" + , + p_dram_info->dramtype + ); + + /* dram capacity information */ + seq_printf(m, + "\n" + "DramCapacity:\n" + ); + + for (i = 0; i < p_dram_info->channel_num; i++) { + if (p_dram_info->channel_num == 2) + seq_printf(m, + "Channel [%d]:\n" + , + i + ); + + seq_printf(m, + "CS Count: %d\n" + "Bus Width: %d bit\n" + "Column: %d\n" + "Bank: %d\n" + "CS0_Row: %d\n" + "CS1_Row: %d\n" + "DieBusWidth: %d bit\n" + "TotalSize: %d MB\n" + , + p_dram_info->ch[i].rank, + p_dram_info->ch[i].buswidth, + p_dram_info->ch[i].col, + p_dram_info->ch[i].bank, + p_dram_info->ch[i].cs0_row, + p_dram_info->ch[i].cs1_row, + p_dram_info->ch[i].die_buswidth, + p_dram_info->ch[i].size + ); + } + + /* check devfreq/dmc device */ + fp = filp_open("/sys/class/devfreq/dmc/cur_freq", O_RDONLY, 0); + if (IS_ERR(fp)) { + seq_printf(m, + "\n" + "devfreq/dmc: Disable\n" + "DramFreq: %d\n" + , + p_dram_info->dramfreq + ); + } else { + pos = 0; + kernel_read(fp, cur_freq, sizeof(cur_freq), &pos); + filp_close(fp, NULL); + + fp = filp_open("/sys/class/devfreq/dmc/governor", O_RDONLY, 0); + if (IS_ERR(fp)) { + fp = NULL; + } else { + pos = 0; + kernel_read(fp, governor, sizeof(governor), &pos); + filp_close(fp, NULL); + } + + seq_printf(m, + "\n" + "devfreq/dmc: Enable\n" + "governor: %s\n" + "cur_freq: %s\n" + , + governor, + cur_freq + ); + seq_printf(m, + "NOTE:\n" + "more information about dmc can get from /sys/class/devfreq/dmc.\n" + ); + } + + return 0; +} + +static int dmcinfo_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, dmcinfo_proc_show, NULL); +} + +static const struct file_operations dmcinfo_proc_fops = { + .open = dmcinfo_proc_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int proc_dmcinfo_init(void) +{ + /* create dmcinfo file */ + proc_create(PROC_DMCDBG_DRAM_INFO, 0644, proc_dmcdbg_dir, + &dmcinfo_proc_fops); + + return 0; +} + +static int powersave_proc_show(struct seq_file *m, void *v) +{ + struct arm_smccc_res res; + struct power_save_info *p_power; + unsigned int *p_uint; + unsigned int i = 0; + + /* get low power information */ + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, + DDRDBG_FUNC_GET_POWERSAVE_INFO, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + seq_printf(m, "rockchip_sip_config_dram_debug error:%lx\n", + res.a0); + return -ENOMEM; + } + + if (!dmcdbg_data.inited_flag) { + seq_puts(m, "dmcdbg_data no int\n"); + return -EPERM; + } + p_power = (struct power_save_info *)dmcdbg_data.share_memory; + + seq_printf(m, + "low power information:\n" + "\n" + "[number]name: value\n" + ); + + p_uint = (unsigned int *)p_power; + for (i = 0; i < ARRAY_SIZE(power_save_msg); i++) + seq_printf(m, + "[%d]%s: %d\n" + , + i, power_save_msg[i], *(p_uint + i) + ); + + seq_printf(m, + "\n" + "power save setting:\n" + "echo number=value > /proc/dmcdbg/powersave\n" + "eg: set auto power down enable to 1\n" + " echo 0=1 > /proc/dmcdbg/powersave\n" + "\n" + "Support for setting multiple parameters at the same time.\n" + "echo number=value,number=value,... > /proc/dmcdbg/powersave\n" + "eg:\n" + " echo 0=1,1=32 > /proc/dmcdbg/powersave\n" + ); + + return 0; +} + +static int powersave_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, powersave_proc_show, NULL); +} + +static ssize_t powersave_proc_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct arm_smccc_res res; + struct power_save_info *p_power; + unsigned int *p_uint; + char *buf, *cookie_pot, *p_char; + int ret = 0; + u32 loop, i, offset, value; + long long_val; + + /* get buffer data */ + buf = vzalloc(count); + cookie_pot = buf; + if (!cookie_pot) + return -ENOMEM; + + if (copy_from_user(cookie_pot, buffer, count)) { + ret = -EFAULT; + goto err; + } + + /* get power save setting information */ + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, + DDRDBG_FUNC_GET_POWERSAVE_INFO, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); + ret = -ENOMEM; + goto err; + } + + if (!dmcdbg_data.inited_flag) { + pr_err("dmcdbg_data no int\n"); + ret = -EPERM; + goto err; + } + p_power = (struct power_save_info *)dmcdbg_data.share_memory; + + loop = 0; + for (i = 0; i < count; i++) { + if (*(cookie_pot + i) == '=') + loop++; + } + + p_uint = (unsigned int *)p_power; + for (i = 0; i < loop; i++) { + p_char = strsep(&cookie_pot, "="); + ret = kstrtol(p_char, 10, &long_val); + if (ret) + goto err; + offset = long_val; + + if (i == (loop - 1)) + p_char = strsep(&cookie_pot, "\0"); + else + p_char = strsep(&cookie_pot, ","); + + ret = kstrtol(p_char, 10, &long_val); + if (ret) + goto err; + value = long_val; + + if (offset >= ARRAY_SIZE(power_save_msg)) { + ret = -EINVAL; + goto err; + } + offset = array_index_nospec(offset, ARRAY_SIZE(power_save_msg)); + + *(p_uint + offset) = value; + } + + /* update power save setting */ + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_UPDATE_POWERSAVE, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); + ret = -ENOMEM; + goto err; + } + + ret = count; +err: + vfree(buf); + return ret; +} + +static const struct file_operations powersave_proc_fops = { + .open = powersave_proc_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = powersave_proc_write, +}; + +static int proc_powersave_init(void) +{ + /* create dmcinfo file */ + proc_create(PROC_DMCDBG_POWERSAVE, 0644, proc_dmcdbg_dir, + &powersave_proc_fops); + + return 0; +} + +static int drvodt_proc_show(struct seq_file *m, void *v) +{ + struct arm_smccc_res res; + struct drv_odt_info *p_drvodt; + unsigned int *p_uint; + unsigned int i; + + /* get drive strength and odt information */ + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_GET_DRVODT_INFO, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + seq_printf(m, "rockchip_sip_config_dram_debug error:%lx\n", + res.a0); + return -ENOMEM; + } + + if (!dmcdbg_data.inited_flag) { + seq_puts(m, "dmcdbg_data no int\n"); + return -EPERM; + } + p_drvodt = (struct drv_odt_info *)dmcdbg_data.share_memory; + + seq_printf(m, + "drv and odt information:\n" + "\n" + "[number]name: value (ohm)\n" + ); + + p_uint = (unsigned int *)p_drvodt; + for (i = 0; i < ARRAY_SIZE(drv_odt_msg); i++) { + if (*(p_uint + (i * 3)) == DRV_ODT_UNKNOWN) + seq_printf(m, + "[%2d]%s: NULL (unknown) %c\n" + , + i, drv_odt_msg[i], + (*(p_uint + (i * 3) + 2) == + DRV_ODT_SUSPEND_FIX) ? '\0' : '*' + ); + else if (*(p_uint + (i * 3) + 1) == DRV_ODT_UNKNOWN) + seq_printf(m, + "[%2d]%s: %d (unknown) %c\n" + , + i, drv_odt_msg[i], *(p_uint + (i * 3)), + (*(p_uint + (i * 3) + 2) == + DRV_ODT_SUSPEND_FIX) ? '\0' : '*' + ); + else if (i < (ARRAY_SIZE(drv_odt_msg) - 2)) + seq_printf(m, + "[%2d]%s: %d (%d ohm) %c\n" + , + i, drv_odt_msg[i], *(p_uint + (i * 3)), + *(p_uint + (i * 3) + 1), + (*(p_uint + (i * 3) + 2) == + DRV_ODT_SUSPEND_FIX) ? '\0' : '*' + ); + else + seq_printf(m, + "[%2d]%s: %d (%d %%) %c\n" + , + i, drv_odt_msg[i], *(p_uint + (i * 3)), + *(p_uint + (i * 3) + 1), + (*(p_uint + (i * 3) + 2) == + DRV_ODT_SUSPEND_FIX) ? '\0' : '*' + ); + } + + seq_printf(m, + "\n" + "drvodt setting:\n" + "echo number=value > /proc/dmcdbg/drvodt\n" + "eg: set soc side ca drv up to 20\n" + " echo 6=20 > /proc/dmcdbg/drvodt\n" + "\n" + "Support for setting multiple parameters at the same time.\n" + "echo number=value,number=value,... > /proc/dmcdbg/drvodt\n" + "eg: set soc side ca drv up and down to 20\n" + " echo 6=20,7=20 > /proc/dmcdbg/drvodt\n" + "Note: Please update both up and down at the same time.\n" + " (*) mean unsupported setting value\n" + ); + + return 0; +} + +static int drvodt_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, drvodt_proc_show, NULL); +} + +static ssize_t drvodt_proc_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct arm_smccc_res res; + struct drv_odt_info *p_drvodt; + unsigned int *p_uint; + char *buf, *cookie_pot, *p_char; + int ret = 0; + u32 loop, i, offset, value; + long long_val; + + /* get buffer data */ + buf = vzalloc(count); + cookie_pot = buf; + if (!cookie_pot) + return -ENOMEM; + + if (copy_from_user(cookie_pot, buffer, count)) { + ret = -EFAULT; + goto err; + } + + /* get drv and odt setting */ + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_GET_DRVODT_INFO, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); + ret = -ENOMEM; + goto err; + } + + if (!dmcdbg_data.inited_flag) { + pr_err("dmcdbg_data no int\n"); + ret = -EPERM; + goto err; + } + p_drvodt = (struct drv_odt_info *)dmcdbg_data.share_memory; + + loop = 0; + for (i = 0; i < count; i++) { + if (*(cookie_pot + i) == '=') + loop++; + } + + p_uint = (unsigned int *)p_drvodt; + for (i = 0; i < loop; i++) { + p_char = strsep(&cookie_pot, "="); + ret = kstrtol(p_char, 10, &long_val); + if (ret) + goto err; + offset = long_val; + + if (i == (loop - 1)) + p_char = strsep(&cookie_pot, "\0"); + else + p_char = strsep(&cookie_pot, ","); + + ret = kstrtol(p_char, 10, &long_val); + if (ret) + goto err; + value = long_val; + + if (offset >= ARRAY_SIZE(drv_odt_msg)) { + ret = -EINVAL; + goto err; + } + offset *= 3; + offset = array_index_nospec(offset, ARRAY_SIZE(drv_odt_msg) * 3); + + *(p_uint + offset) = value; + } + + /* update power save setting */ + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_UPDATE_DRVODT, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); + ret = -ENOMEM; + goto err; + } + + ret = count; +err: + vfree(buf); + return ret; +} + +static const struct file_operations drvodt_proc_fops = { + .open = drvodt_proc_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = drvodt_proc_write, +}; + +static int proc_drvodt_init(void) +{ + /* create dmcinfo file */ + proc_create(PROC_DMCDBG_DRVODT, 0644, proc_dmcdbg_dir, + &drvodt_proc_fops); + + return 0; +} + +static int skew_proc_show(struct seq_file *m, void *v) +{ + struct arm_smccc_res res; + unsigned int *p_uint; + u32 group, i; + + /* get deskew information */ + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_GET_DESKEW_INFO, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + seq_printf(m, "rockchip_sip_config_dram_debug error:%lx\n", + res.a0); + return -ENOMEM; + } + + if (!dmcdbg_data.inited_flag) { + seq_puts(m, "dmcdbg_data no int\n"); + return -EPERM; + } + + seq_printf(m, + "de-skew information:\n" + "\n" + "[group_number]name: value\n" + ); + + for (group = 0; group < dmcdbg_data.skew_group_num; group++) { + if (dmcdbg_data.skew_group[group].note != NULL) + seq_printf(m, + "%s\n" + , + dmcdbg_data.skew_group[group].note + ); + p_uint = (unsigned int *)dmcdbg_data.skew_group[group].p_skew_info; + for (i = 0; i < dmcdbg_data.skew_group[group].skew_num; i++) + seq_printf(m, + "[%c%d_%d]%s: %d\n" + , + (i < 10) ? ' ' : '\0', group, i, + dmcdbg_data.skew_group[group].p_skew_timing[i], + *(p_uint + i) + ); + } + + seq_printf(m, + "\n" + "de-skew setting:\n" + "echo group_number=value > /proc/dmcdbg/deskew\n" + "eg: set a1_ddr3a14_de-skew to 8\n" + " echo 0_1=8 > /proc/dmcdbg/deskew\n" + "\n" + "Support for setting multiple parameters simultaneously.\n" + "echo group_number=value,group_number=value,... > /proc/dmcdbg/deskew\n" + "eg:\n" + " echo 0_1=8,1_2=8 > /proc/dmcdbg/deskew\n" + ); + + return 0; +} + +static int skew_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, skew_proc_show, NULL); +} + +static ssize_t skew_proc_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct arm_smccc_res res; + unsigned int *p_uint; + char *buf, *cookie_pot, *p_char; + int ret = 0; + u32 loop, i, offset_max, group, offset, value; + long long_val; + + /* get buffer data */ + buf = vzalloc(count); + cookie_pot = buf; + if (!cookie_pot) + return -ENOMEM; + + if (copy_from_user(cookie_pot, buffer, count)) { + ret = -EFAULT; + goto err; + } + + /* get skew setting */ + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_GET_DESKEW_INFO, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); + ret = -ENOMEM; + goto err; + } + + if (!dmcdbg_data.inited_flag) { + pr_err("dmcdbg_data no int\n"); + ret = -EPERM; + goto err; + } + + loop = 0; + for (i = 0; i < count; i++) { + if (*(cookie_pot + i) == '=') + loop++; + } + + for (i = 0; i < loop; i++) { + p_char = strsep(&cookie_pot, "_"); + ret = kstrtol(p_char, 10, &long_val); + if (ret) + goto err; + group = long_val; + + p_char = strsep(&cookie_pot, "="); + ret = kstrtol(p_char, 10, &long_val); + if (ret) + goto err; + offset = long_val; + + if (i == (loop - 1)) + p_char = strsep(&cookie_pot, "\0"); + else + p_char = strsep(&cookie_pot, ","); + + ret = kstrtol(p_char, 10, &long_val); + if (ret) + goto err; + value = long_val; + + if (group >= dmcdbg_data.skew_group_num) { + ret = -EINVAL; + goto err; + } + group = array_index_nospec(group, dmcdbg_data.skew_group_num); + + p_uint = (unsigned int *)dmcdbg_data.skew_group[group].p_skew_info; + offset_max = dmcdbg_data.skew_group[group].skew_num; + + if (offset >= offset_max) { + ret = -EINVAL; + goto err; + } + offset = array_index_nospec(offset, offset_max); + + *(p_uint + offset) = value; + } + + /* update power save setting */ + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, DDRDBG_FUNC_UPDATE_DESKEW, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + pr_err("rockchip_sip_config_dram_debug error:%lx\n", res.a0); + ret = -ENOMEM; + goto err; + } + + ret = count; +err: + vfree(buf); + return ret; +} + +static const struct file_operations skew_proc_fops = { + .open = skew_proc_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = skew_proc_write, +}; + +static int proc_skew_init(void) +{ + /* create dmcinfo file */ + proc_create(PROC_DMCDBG_DESKEW, 0644, proc_dmcdbg_dir, + &skew_proc_fops); + + return 0; +} + +static int regsinfo_proc_show(struct seq_file *m, void *v) +{ + struct arm_smccc_res res; + struct registers_info *p_regsinfo; + u32 i; + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDRDBG, + DDRDBG_FUNC_GET_REGISTERS_INFO, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + if (res.a0) { + seq_printf(m, "rockchip_sip_config_dram_debug error:%lx\n", + res.a0); + return -ENOMEM; + } + + if (!dmcdbg_data.inited_flag) { + seq_puts(m, "dmcdbg_data no int\n"); + return -EPERM; + } + p_regsinfo = (struct registers_info *)dmcdbg_data.share_memory; + + seq_printf(m, + "registers base address information:\n" + "\n" + ); + + for (i = 0; i < p_regsinfo->regs_num; i++) { + seq_printf(m, + "%s=0x%x\n" + , + p_regsinfo->regs[i].regs_name, + p_regsinfo->regs[i].regs_addr + ); + } + + return 0; +} + +static int regsinfo_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, regsinfo_proc_show, NULL); +} + +static const struct file_operations regsinfo_proc_fops = { + .open = regsinfo_proc_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int proc_regsinfo_init(void) +{ + /* create dmcinfo file */ + proc_create(PROC_DMCDBG_REGS_INFO, 0644, proc_dmcdbg_dir, + ®sinfo_proc_fops); + + return 0; +} + +static void rv1126_get_skew_parameter(void) +{ + struct skew_info_rv1126 *p_skew; + u32 i; + + /* get skew parameters */ + p_skew = (struct skew_info_rv1126 *)dmcdbg_data.share_memory; + dmcdbg_data.skew_group_num = 5; + + /* ca_skew parameters */ + dmcdbg_data.skew_group[0].p_skew_info = (unsigned int *)p_skew->ca_skew; + dmcdbg_data.skew_group[0].skew_num = ARRAY_SIZE(rv1126_dts_ca_timing); + for (i = 0; i < dmcdbg_data.skew_group[0].skew_num; i++) + dmcdbg_data.skew_group[0].p_skew_timing[i] = + (char *)rv1126_dts_ca_timing[i]; + dmcdbg_data.skew_group[0].note = + "(ca_skew: ddr4(pad_name)_ddr3_lpddr3_lpddr4_de-skew)"; + + /* cs0_a_skew parameters */ + dmcdbg_data.skew_group[1].p_skew_info = (unsigned int *)p_skew->cs0_a_skew; + dmcdbg_data.skew_group[1].skew_num = ARRAY_SIZE(rv1126_dts_cs0_a_timing); + for (i = 0; i < dmcdbg_data.skew_group[1].skew_num; i++) + dmcdbg_data.skew_group[1].p_skew_timing[i] = + (char *)rv1126_dts_cs0_a_timing[i]; + dmcdbg_data.skew_group[1].note = "(cs0_a_skew)"; + + /* cs0_b_skew parameters */ + dmcdbg_data.skew_group[2].p_skew_info = (unsigned int *)p_skew->cs0_b_skew; + dmcdbg_data.skew_group[2].skew_num = ARRAY_SIZE(rv1126_dts_cs0_b_timing); + for (i = 0; i < dmcdbg_data.skew_group[2].skew_num; i++) + dmcdbg_data.skew_group[2].p_skew_timing[i] = + (char *)rv1126_dts_cs0_b_timing[i]; + dmcdbg_data.skew_group[2].note = "(cs0_b_skew)"; + + /* cs1_a_skew parameters */ + dmcdbg_data.skew_group[3].p_skew_info = (unsigned int *)p_skew->cs1_a_skew; + dmcdbg_data.skew_group[3].skew_num = ARRAY_SIZE(rv1126_dts_cs1_a_timing); + for (i = 0; i < dmcdbg_data.skew_group[3].skew_num; i++) + dmcdbg_data.skew_group[3].p_skew_timing[i] = + (char *)rv1126_dts_cs1_a_timing[i]; + dmcdbg_data.skew_group[3].note = "(cs1_a_skew)"; + + /* cs1_b_skew parameters */ + dmcdbg_data.skew_group[4].p_skew_info = (unsigned int *)p_skew->cs1_b_skew; + dmcdbg_data.skew_group[4].skew_num = ARRAY_SIZE(rv1126_dts_cs1_b_timing); + for (i = 0; i < dmcdbg_data.skew_group[3].skew_num; i++) + dmcdbg_data.skew_group[4].p_skew_timing[i] = + (char *)rv1126_dts_cs1_b_timing[i]; + dmcdbg_data.skew_group[4].note = "(cs1_b_skew)"; +} + +static __maybe_unused int rv1126_dmcdbg_init(struct platform_device *pdev, + struct rockchip_dmcdbg *dmcdbg) +{ + struct arm_smccc_res res; + + /* check ddr_debug_func version */ + res = sip_smc_dram(0, DDRDBG_FUNC_GET_VERSION, + ROCKCHIP_SIP_CONFIG_DRAM_DEBUG); + dev_notice(&pdev->dev, "current ATF ddr_debug_func version 0x%lx.\n", + res.a1); + /* + * [15:8] major version, [7:0] minor version + * major version must match both kernel dmcdbg and ATF ddr_debug_func. + */ + if (res.a0 || res.a1 < 0x101 || ((res.a1 & 0xff00) != 0x100)) { + dev_err(&pdev->dev, + "version invalid,need update,the major version unmatch!\n"); + return -ENXIO; + } + + /* request share memory for pass parameter */ + res = sip_smc_request_share_mem(DMCDBG_PAGE_NUMS, + SHARE_PAGE_TYPE_DDRDBG); + if (res.a0 != 0) { + dev_err(&pdev->dev, "request share mem error\n"); + return -ENOMEM; + } + + dmcdbg_data.share_memory = (void __iomem *)res.a1; + dmcdbg_data.inited_flag = 1; + + rv1126_get_skew_parameter(); + + /* create parent dir in /proc */ + proc_dmcdbg_dir = proc_mkdir(PROC_DMCDBG_DIR_NAME, NULL); + if (!proc_dmcdbg_dir) { + dev_err(&pdev->dev, "create proc dir error!"); + return -ENOENT; + } + + proc_dmcinfo_init(); + proc_powersave_init(); + proc_drvodt_init(); + proc_skew_init(); + proc_regsinfo_init(); + return 0; +} + +static const struct of_device_id rockchip_dmcdbg_of_match[] = { + { .compatible = "rockchip,rv1126-dmcdbg", .data = rv1126_dmcdbg_init}, + { }, +}; +MODULE_DEVICE_TABLE(of, rockchip_dmcdbg_of_match); + +static int rockchip_dmcdbg_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rockchip_dmcdbg *data; + const struct of_device_id *match; + int (*init)(struct platform_device *pdev, + struct rockchip_dmcdbg *data); + int ret = 0; + + data = devm_kzalloc(dev, sizeof(struct rockchip_dmcdbg), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->dev = dev; + + /* match soc chip init */ + match = of_match_node(rockchip_dmcdbg_of_match, pdev->dev.of_node); + if (match) { + init = match->data; + if (init) { + if (init(pdev, data)) + return -EINVAL; + } + } + + return ret; +} + +static struct platform_driver rockchip_dmcdbg_driver = { + .probe = rockchip_dmcdbg_probe, + .driver = { + .name = "rockchip,dmcdbg", + .of_match_table = rockchip_dmcdbg_of_match, + }, +}; +module_platform_driver(rockchip_dmcdbg_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("YouMin Chen "); +MODULE_DESCRIPTION("rockchip dmc debug driver with devfreq framework"); diff --git a/drivers/devfreq/rockchip_dmc_timing.h b/drivers/devfreq/rockchip_dmc_timing.h new file mode 100755 index 000000000000..8f2e2c02bb90 --- /dev/null +++ b/drivers/devfreq/rockchip_dmc_timing.h @@ -0,0 +1,1307 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2020, Rockchip Electronics Co., Ltd. + */ + +#ifndef __ROCKCHIP_DMC_TIMING_H__ +#define __ROCKCHIP_DMC_TIMING_H__ + +/* hope this define can adapt all future platfor */ +static const char * const px30_dts_timing[] = { + "ddr2_speed_bin", + "ddr3_speed_bin", + "ddr4_speed_bin", + "pd_idle", + "sr_idle", + "sr_mc_gate_idle", + "srpd_lite_idle", + "standby_idle", + + "auto_pd_dis_freq", + "auto_sr_dis_freq", + "ddr2_dll_dis_freq", + "ddr3_dll_dis_freq", + "ddr4_dll_dis_freq", + "phy_dll_dis_freq", + + "ddr2_odt_dis_freq", + "phy_ddr2_odt_dis_freq", + "ddr2_drv", + "ddr2_odt", + "phy_ddr2_ca_drv", + "phy_ddr2_ck_drv", + "phy_ddr2_dq_drv", + "phy_ddr2_odt", + + "ddr3_odt_dis_freq", + "phy_ddr3_odt_dis_freq", + "ddr3_drv", + "ddr3_odt", + "phy_ddr3_ca_drv", + "phy_ddr3_ck_drv", + "phy_ddr3_dq_drv", + "phy_ddr3_odt", + + "phy_lpddr2_odt_dis_freq", + "lpddr2_drv", + "phy_lpddr2_ca_drv", + "phy_lpddr2_ck_drv", + "phy_lpddr2_dq_drv", + "phy_lpddr2_odt", + + "lpddr3_odt_dis_freq", + "phy_lpddr3_odt_dis_freq", + "lpddr3_drv", + "lpddr3_odt", + "phy_lpddr3_ca_drv", + "phy_lpddr3_ck_drv", + "phy_lpddr3_dq_drv", + "phy_lpddr3_odt", + + "lpddr4_odt_dis_freq", + "phy_lpddr4_odt_dis_freq", + "lpddr4_drv", + "lpddr4_dq_odt", + "lpddr4_ca_odt", + "phy_lpddr4_ca_drv", + "phy_lpddr4_ck_cs_drv", + "phy_lpddr4_dq_drv", + "phy_lpddr4_odt", + + "ddr4_odt_dis_freq", + "phy_ddr4_odt_dis_freq", + "ddr4_drv", + "ddr4_odt", + "phy_ddr4_ca_drv", + "phy_ddr4_ck_drv", + "phy_ddr4_dq_drv", + "phy_ddr4_odt", +}; + +struct px30_ddr_dts_config_timing { + unsigned int ddr2_speed_bin; + unsigned int ddr3_speed_bin; + unsigned int ddr4_speed_bin; + unsigned int pd_idle; + unsigned int sr_idle; + unsigned int sr_mc_gate_idle; + unsigned int srpd_lite_idle; + unsigned int standby_idle; + + unsigned int auto_pd_dis_freq; + unsigned int auto_sr_dis_freq; + /* for ddr2 only */ + unsigned int ddr2_dll_dis_freq; + /* for ddr3 only */ + unsigned int ddr3_dll_dis_freq; + /* for ddr4 only */ + unsigned int ddr4_dll_dis_freq; + unsigned int phy_dll_dis_freq; + + unsigned int ddr2_odt_dis_freq; + unsigned int phy_ddr2_odt_dis_freq; + unsigned int ddr2_drv; + unsigned int ddr2_odt; + unsigned int phy_ddr2_ca_drv; + unsigned int phy_ddr2_ck_drv; + unsigned int phy_ddr2_dq_drv; + unsigned int phy_ddr2_odt; + + unsigned int ddr3_odt_dis_freq; + unsigned int phy_ddr3_odt_dis_freq; + unsigned int ddr3_drv; + unsigned int ddr3_odt; + unsigned int phy_ddr3_ca_drv; + unsigned int phy_ddr3_ck_drv; + unsigned int phy_ddr3_dq_drv; + unsigned int phy_ddr3_odt; + + unsigned int phy_lpddr2_odt_dis_freq; + unsigned int lpddr2_drv; + unsigned int phy_lpddr2_ca_drv; + unsigned int phy_lpddr2_ck_drv; + unsigned int phy_lpddr2_dq_drv; + unsigned int phy_lpddr2_odt; + + unsigned int lpddr3_odt_dis_freq; + unsigned int phy_lpddr3_odt_dis_freq; + unsigned int lpddr3_drv; + unsigned int lpddr3_odt; + unsigned int phy_lpddr3_ca_drv; + unsigned int phy_lpddr3_ck_drv; + unsigned int phy_lpddr3_dq_drv; + unsigned int phy_lpddr3_odt; + + unsigned int lpddr4_odt_dis_freq; + unsigned int phy_lpddr4_odt_dis_freq; + unsigned int lpddr4_drv; + unsigned int lpddr4_dq_odt; + unsigned int lpddr4_ca_odt; + unsigned int phy_lpddr4_ca_drv; + unsigned int phy_lpddr4_ck_cs_drv; + unsigned int phy_lpddr4_dq_drv; + unsigned int phy_lpddr4_odt; + + unsigned int ddr4_odt_dis_freq; + unsigned int phy_ddr4_odt_dis_freq; + unsigned int ddr4_drv; + unsigned int ddr4_odt; + unsigned int phy_ddr4_ca_drv; + unsigned int phy_ddr4_ck_drv; + unsigned int phy_ddr4_dq_drv; + unsigned int phy_ddr4_odt; + + unsigned int ca_skew[15]; + unsigned int cs0_skew[44]; + unsigned int cs1_skew[44]; + + unsigned int available; +}; + +static const char * const rk1808_dts_ca_timing[] = { + "a0_ddr3a9_de-skew", + "a1_ddr3a14_de-skew", + "a2_ddr3a13_de-skew", + "a3_ddr3a11_de-skew", + "a4_ddr3a2_de-skew", + "a5_ddr3a4_de-skew", + "a6_ddr3a3_de-skew", + "a7_ddr3a6_de-skew", + "a8_ddr3a5_de-skew", + "a9_ddr3a1_de-skew", + "a10_ddr3a0_de-skew", + "a11_ddr3a7_de-skew", + "a12_ddr3casb_de-skew", + "a13_ddr3a8_de-skew", + "a14_ddr3odt0_de-skew", + "a15_ddr3ba1_de-skew", + "a16_ddr3rasb_de-skew", + "a17_ddr3null_de-skew", + "ba0_ddr3ba2_de-skew", + "ba1_ddr3a12_de-skew", + "bg0_ddr3ba0_de-skew", + "bg1_ddr3web_de-skew", + "cke_ddr3cke_de-skew", + "ck_ddr3ck_de-skew", + "ckb_ddr3ckb_de-skew", + "csb0_ddr3a10_de-skew", + "odt0_ddr3a15_de-skew", + "resetn_ddr3resetn_de-skew", + "actn_ddr3csb0_de-skew", + "csb1_ddr3csb1_de-skew", + "odt1_ddr3odt1_de-skew", +}; + +static const char * const rk1808_dts_cs0_a_timing[] = { + "cs0_dm0_rx_de-skew", + "cs0_dm0_tx_de-skew", + "cs0_dq0_rx_de-skew", + "cs0_dq0_tx_de-skew", + "cs0_dq1_rx_de-skew", + "cs0_dq1_tx_de-skew", + "cs0_dq2_rx_de-skew", + "cs0_dq2_tx_de-skew", + "cs0_dq3_rx_de-skew", + "cs0_dq3_tx_de-skew", + "cs0_dq4_rx_de-skew", + "cs0_dq4_tx_de-skew", + "cs0_dq5_rx_de-skew", + "cs0_dq5_tx_de-skew", + "cs0_dq6_rx_de-skew", + "cs0_dq6_tx_de-skew", + "cs0_dq7_rx_de-skew", + "cs0_dq7_tx_de-skew", + "cs0_dqs0p_rx_de-skew", + "cs0_dqs0p_tx_de-skew", + "cs0_dqs0n_tx_de-skew", + "cs0_dm1_rx_de-skew", + "cs0_dm1_tx_de-skew", + "cs0_dq8_rx_de-skew", + "cs0_dq8_tx_de-skew", + "cs0_dq9_rx_de-skew", + "cs0_dq9_tx_de-skew", + "cs0_dq10_rx_de-skew", + "cs0_dq10_tx_de-skew", + "cs0_dq11_rx_de-skew", + "cs0_dq11_tx_de-skew", + "cs0_dq12_rx_de-skew", + "cs0_dq12_tx_de-skew", + "cs0_dq13_rx_de-skew", + "cs0_dq13_tx_de-skew", + "cs0_dq14_rx_de-skew", + "cs0_dq14_tx_de-skew", + "cs0_dq15_rx_de-skew", + "cs0_dq15_tx_de-skew", + "cs0_dqs1p_rx_de-skew", + "cs0_dqs1p_tx_de-skew", + "cs0_dqs1n_tx_de-skew", + "cs0_dqs0n_rx_de-skew", + "cs0_dqs1n_rx_de-skew", +}; + +static const char * const rk1808_dts_cs0_b_timing[] = { + "cs0_dm2_rx_de-skew", + "cs0_dm2_tx_de-skew", + "cs0_dq16_rx_de-skew", + "cs0_dq16_tx_de-skew", + "cs0_dq17_rx_de-skew", + "cs0_dq17_tx_de-skew", + "cs0_dq18_rx_de-skew", + "cs0_dq18_tx_de-skew", + "cs0_dq19_rx_de-skew", + "cs0_dq19_tx_de-skew", + "cs0_dq20_rx_de-skew", + "cs0_dq20_tx_de-skew", + "cs0_dq21_rx_de-skew", + "cs0_dq21_tx_de-skew", + "cs0_dq22_rx_de-skew", + "cs0_dq22_tx_de-skew", + "cs0_dq23_rx_de-skew", + "cs0_dq23_tx_de-skew", + "cs0_dqs2p_rx_de-skew", + "cs0_dqs2p_tx_de-skew", + "cs0_dqs2n_tx_de-skew", + "cs0_dm3_rx_de-skew", + "cs0_dm3_tx_de-skew", + "cs0_dq24_rx_de-skew", + "cs0_dq24_tx_de-skew", + "cs0_dq25_rx_de-skew", + "cs0_dq25_tx_de-skew", + "cs0_dq26_rx_de-skew", + "cs0_dq26_tx_de-skew", + "cs0_dq27_rx_de-skew", + "cs0_dq27_tx_de-skew", + "cs0_dq28_rx_de-skew", + "cs0_dq28_tx_de-skew", + "cs0_dq29_rx_de-skew", + "cs0_dq29_tx_de-skew", + "cs0_dq30_rx_de-skew", + "cs0_dq30_tx_de-skew", + "cs0_dq31_rx_de-skew", + "cs0_dq31_tx_de-skew", + "cs0_dqs3p_rx_de-skew", + "cs0_dqs3p_tx_de-skew", + "cs0_dqs3n_tx_de-skew", + "cs0_dqs2n_rx_de-skew", + "cs0_dqs3n_rx_de-skew", +}; + +static const char * const rk1808_dts_cs1_a_timing[] = { + "cs1_dm0_rx_de-skew", + "cs1_dm0_tx_de-skew", + "cs1_dq0_rx_de-skew", + "cs1_dq0_tx_de-skew", + "cs1_dq1_rx_de-skew", + "cs1_dq1_tx_de-skew", + "cs1_dq2_rx_de-skew", + "cs1_dq2_tx_de-skew", + "cs1_dq3_rx_de-skew", + "cs1_dq3_tx_de-skew", + "cs1_dq4_rx_de-skew", + "cs1_dq4_tx_de-skew", + "cs1_dq5_rx_de-skew", + "cs1_dq5_tx_de-skew", + "cs1_dq6_rx_de-skew", + "cs1_dq6_tx_de-skew", + "cs1_dq7_rx_de-skew", + "cs1_dq7_tx_de-skew", + "cs1_dqs0p_rx_de-skew", + "cs1_dqs0p_tx_de-skew", + "cs1_dqs0n_tx_de-skew", + "cs1_dm1_rx_de-skew", + "cs1_dm1_tx_de-skew", + "cs1_dq8_rx_de-skew", + "cs1_dq8_tx_de-skew", + "cs1_dq9_rx_de-skew", + "cs1_dq9_tx_de-skew", + "cs1_dq10_rx_de-skew", + "cs1_dq10_tx_de-skew", + "cs1_dq11_rx_de-skew", + "cs1_dq11_tx_de-skew", + "cs1_dq12_rx_de-skew", + "cs1_dq12_tx_de-skew", + "cs1_dq13_rx_de-skew", + "cs1_dq13_tx_de-skew", + "cs1_dq14_rx_de-skew", + "cs1_dq14_tx_de-skew", + "cs1_dq15_rx_de-skew", + "cs1_dq15_tx_de-skew", + "cs1_dqs1p_rx_de-skew", + "cs1_dqs1p_tx_de-skew", + "cs1_dqs1n_tx_de-skew", + "cs1_dqs0n_rx_de-skew", + "cs1_dqs1n_rx_de-skew", +}; + +static const char * const rk1808_dts_cs1_b_timing[] = { + "cs1_dm2_rx_de-skew", + "cs1_dm2_tx_de-skew", + "cs1_dq16_rx_de-skew", + "cs1_dq16_tx_de-skew", + "cs1_dq17_rx_de-skew", + "cs1_dq17_tx_de-skew", + "cs1_dq18_rx_de-skew", + "cs1_dq18_tx_de-skew", + "cs1_dq19_rx_de-skew", + "cs1_dq19_tx_de-skew", + "cs1_dq20_rx_de-skew", + "cs1_dq20_tx_de-skew", + "cs1_dq21_rx_de-skew", + "cs1_dq21_tx_de-skew", + "cs1_dq22_rx_de-skew", + "cs1_dq22_tx_de-skew", + "cs1_dq23_rx_de-skew", + "cs1_dq23_tx_de-skew", + "cs1_dqs2p_rx_de-skew", + "cs1_dqs2p_tx_de-skew", + "cs1_dqs2n_tx_de-skew", + "cs1_dm3_rx_de-skew", + "cs1_dm3_tx_de-skew", + "cs1_dq24_rx_de-skew", + "cs1_dq24_tx_de-skew", + "cs1_dq25_rx_de-skew", + "cs1_dq25_tx_de-skew", + "cs1_dq26_rx_de-skew", + "cs1_dq26_tx_de-skew", + "cs1_dq27_rx_de-skew", + "cs1_dq27_tx_de-skew", + "cs1_dq28_rx_de-skew", + "cs1_dq28_tx_de-skew", + "cs1_dq29_rx_de-skew", + "cs1_dq29_tx_de-skew", + "cs1_dq30_rx_de-skew", + "cs1_dq30_tx_de-skew", + "cs1_dq31_rx_de-skew", + "cs1_dq31_tx_de-skew", + "cs1_dqs3p_rx_de-skew", + "cs1_dqs3p_tx_de-skew", + "cs1_dqs3n_tx_de-skew", + "cs1_dqs2n_rx_de-skew", + "cs1_dqs3n_rx_de-skew", +}; + +struct rk1808_ddr_dts_config_timing { + unsigned int ddr2_speed_bin; + unsigned int ddr3_speed_bin; + unsigned int ddr4_speed_bin; + unsigned int pd_idle; + unsigned int sr_idle; + unsigned int sr_mc_gate_idle; + unsigned int srpd_lite_idle; + unsigned int standby_idle; + + unsigned int auto_pd_dis_freq; + unsigned int auto_sr_dis_freq; + /* for ddr2 only */ + unsigned int ddr2_dll_dis_freq; + /* for ddr3 only */ + unsigned int ddr3_dll_dis_freq; + /* for ddr4 only */ + unsigned int ddr4_dll_dis_freq; + unsigned int phy_dll_dis_freq; + + unsigned int ddr2_odt_dis_freq; + unsigned int phy_ddr2_odt_dis_freq; + unsigned int ddr2_drv; + unsigned int ddr2_odt; + unsigned int phy_ddr2_ca_drv; + unsigned int phy_ddr2_ck_drv; + unsigned int phy_ddr2_dq_drv; + unsigned int phy_ddr2_odt; + + unsigned int ddr3_odt_dis_freq; + unsigned int phy_ddr3_odt_dis_freq; + unsigned int ddr3_drv; + unsigned int ddr3_odt; + unsigned int phy_ddr3_ca_drv; + unsigned int phy_ddr3_ck_drv; + unsigned int phy_ddr3_dq_drv; + unsigned int phy_ddr3_odt; + + unsigned int phy_lpddr2_odt_dis_freq; + unsigned int lpddr2_drv; + unsigned int phy_lpddr2_ca_drv; + unsigned int phy_lpddr2_ck_drv; + unsigned int phy_lpddr2_dq_drv; + unsigned int phy_lpddr2_odt; + + unsigned int lpddr3_odt_dis_freq; + unsigned int phy_lpddr3_odt_dis_freq; + unsigned int lpddr3_drv; + unsigned int lpddr3_odt; + unsigned int phy_lpddr3_ca_drv; + unsigned int phy_lpddr3_ck_drv; + unsigned int phy_lpddr3_dq_drv; + unsigned int phy_lpddr3_odt; + + unsigned int lpddr4_odt_dis_freq; + unsigned int phy_lpddr4_odt_dis_freq; + unsigned int lpddr4_drv; + unsigned int lpddr4_dq_odt; + unsigned int lpddr4_ca_odt; + unsigned int phy_lpddr4_ca_drv; + unsigned int phy_lpddr4_ck_cs_drv; + unsigned int phy_lpddr4_dq_drv; + unsigned int phy_lpddr4_odt; + + unsigned int ddr4_odt_dis_freq; + unsigned int phy_ddr4_odt_dis_freq; + unsigned int ddr4_drv; + unsigned int ddr4_odt; + unsigned int phy_ddr4_ca_drv; + unsigned int phy_ddr4_ck_drv; + unsigned int phy_ddr4_dq_drv; + unsigned int phy_ddr4_odt; + + unsigned int ca_de_skew[31]; + unsigned int cs0_a_de_skew[44]; + unsigned int cs0_b_de_skew[44]; + unsigned int cs1_a_de_skew[44]; + unsigned int cs1_b_de_skew[44]; + + unsigned int available; +}; + +static const char * const rk3128_dts_timing[] = { + "ddr3_speed_bin", + "pd_idle", + "sr_idle", + "auto_pd_dis_freq", + "auto_sr_dis_freq", + "ddr3_dll_dis_freq", + "lpddr2_dll_dis_freq", + "phy_dll_dis_freq", + "ddr3_odt_dis_freq", + "phy_ddr3_odt_disb_freq", + "ddr3_drv", + "ddr3_odt", + "phy_ddr3_clk_drv", + "phy_ddr3_cmd_drv", + "phy_ddr3_dqs_drv", + "phy_ddr3_odt", + "lpddr2_drv", + "phy_lpddr2_clk_drv", + "phy_lpddr2_cmd_drv", + "phy_lpddr2_dqs_drv", + "ddr_2t", +}; + +struct rk3128_ddr_dts_config_timing { + u32 ddr3_speed_bin; + u32 pd_idle; + u32 sr_idle; + u32 auto_pd_dis_freq; + u32 auto_sr_dis_freq; + u32 ddr3_dll_dis_freq; + u32 lpddr2_dll_dis_freq; + u32 phy_dll_dis_freq; + u32 ddr3_odt_dis_freq; + u32 phy_ddr3_odt_disb_freq; + u32 ddr3_drv; + u32 ddr3_odt; + u32 phy_ddr3_clk_drv; + u32 phy_ddr3_cmd_drv; + u32 phy_ddr3_dqs_drv; + u32 phy_ddr3_odt; + u32 lpddr2_drv; + u32 phy_lpddr2_clk_drv; + u32 phy_lpddr2_cmd_drv; + u32 phy_lpddr2_dqs_drv; + u32 ddr_2t; + u32 available; +}; + +static const char * const rk3228_dts_timing[] = { + "dram_spd_bin", + "sr_idle", + "pd_idle", + "dram_dll_disb_freq", + "phy_dll_disb_freq", + "dram_odt_disb_freq", + "phy_odt_disb_freq", + "ddr3_drv", + "ddr3_odt", + "lpddr3_drv", + "lpddr3_odt", + "lpddr2_drv", + "phy_ddr3_clk_drv", + "phy_ddr3_cmd_drv", + "phy_ddr3_dqs_drv", + "phy_ddr3_odt", + "phy_lp23_clk_drv", + "phy_lp23_cmd_drv", + "phy_lp23_dqs_drv", + "phy_lp3_odt" +}; + +struct rk3228_ddr_dts_config_timing { + u32 dram_spd_bin; + u32 sr_idle; + u32 pd_idle; + u32 dram_dll_dis_freq; + u32 phy_dll_dis_freq; + u32 dram_odt_dis_freq; + u32 phy_odt_dis_freq; + u32 ddr3_drv; + u32 ddr3_odt; + u32 lpddr3_drv; + u32 lpddr3_odt; + u32 lpddr2_drv; + u32 phy_ddr3_clk_drv; + u32 phy_ddr3_cmd_drv; + u32 phy_ddr3_dqs_drv; + u32 phy_ddr3_odt; + u32 phy_lp23_clk_drv; + u32 phy_lp23_cmd_drv; + u32 phy_lp23_dqs_drv; + u32 phy_lp3_odt; +}; + +static const char * const rk3288_dts_timing[] = { + "ddr3_speed_bin", + "pd_idle", + "sr_idle", + + "auto_pd_dis_freq", + "auto_sr_dis_freq", + /* for ddr3 only */ + "ddr3_dll_dis_freq", + "phy_dll_dis_freq", + + "ddr3_odt_dis_freq", + "phy_ddr3_odt_dis_freq", + "ddr3_drv", + "ddr3_odt", + "phy_ddr3_drv", + "phy_ddr3_odt", + + "lpddr2_drv", + "phy_lpddr2_drv", + + "lpddr3_odt_dis_freq", + "phy_lpddr3_odt_dis_freq", + "lpddr3_drv", + "lpddr3_odt", + "phy_lpddr3_drv", + "phy_lpddr3_odt" +}; + +struct rk3288_ddr_dts_config_timing { + unsigned int ddr3_speed_bin; + unsigned int pd_idle; + unsigned int sr_idle; + + unsigned int auto_pd_dis_freq; + unsigned int auto_sr_dis_freq; + /* for ddr3 only */ + unsigned int ddr3_dll_dis_freq; + unsigned int phy_dll_dis_freq; + + unsigned int ddr3_odt_dis_freq; + unsigned int phy_ddr3_odt_dis_freq; + unsigned int ddr3_drv; + unsigned int ddr3_odt; + unsigned int phy_ddr3_drv; + unsigned int phy_ddr3_odt; + + unsigned int lpddr2_drv; + unsigned int phy_lpddr2_drv; + + unsigned int lpddr3_odt_dis_freq; + unsigned int phy_lpddr3_odt_dis_freq; + unsigned int lpddr3_drv; + unsigned int lpddr3_odt; + unsigned int phy_lpddr3_drv; + unsigned int phy_lpddr3_odt; + + unsigned int available; +}; + +/* hope this define can adapt all future platfor */ +static const char * const rk3328_dts_timing[] = { + "ddr3_speed_bin", + "ddr4_speed_bin", + "pd_idle", + "sr_idle", + "sr_mc_gate_idle", + "srpd_lite_idle", + "standby_idle", + + "auto_pd_dis_freq", + "auto_sr_dis_freq", + "ddr3_dll_dis_freq", + "ddr4_dll_dis_freq", + "phy_dll_dis_freq", + + "ddr3_odt_dis_freq", + "phy_ddr3_odt_dis_freq", + "ddr3_drv", + "ddr3_odt", + "phy_ddr3_ca_drv", + "phy_ddr3_ck_drv", + "phy_ddr3_dq_drv", + "phy_ddr3_odt", + + "lpddr3_odt_dis_freq", + "phy_lpddr3_odt_dis_freq", + "lpddr3_drv", + "lpddr3_odt", + "phy_lpddr3_ca_drv", + "phy_lpddr3_ck_drv", + "phy_lpddr3_dq_drv", + "phy_lpddr3_odt", + + "lpddr4_odt_dis_freq", + "phy_lpddr4_odt_dis_freq", + "lpddr4_drv", + "lpddr4_dq_odt", + "lpddr4_ca_odt", + "phy_lpddr4_ca_drv", + "phy_lpddr4_ck_cs_drv", + "phy_lpddr4_dq_drv", + "phy_lpddr4_odt", + + "ddr4_odt_dis_freq", + "phy_ddr4_odt_dis_freq", + "ddr4_drv", + "ddr4_odt", + "phy_ddr4_ca_drv", + "phy_ddr4_ck_drv", + "phy_ddr4_dq_drv", + "phy_ddr4_odt", +}; + +static const char * const rk3328_dts_ca_timing[] = { + "ddr3a1_ddr4a9_de-skew", + "ddr3a0_ddr4a10_de-skew", + "ddr3a3_ddr4a6_de-skew", + "ddr3a2_ddr4a4_de-skew", + "ddr3a5_ddr4a8_de-skew", + "ddr3a4_ddr4a5_de-skew", + "ddr3a7_ddr4a11_de-skew", + "ddr3a6_ddr4a7_de-skew", + "ddr3a9_ddr4a0_de-skew", + "ddr3a8_ddr4a13_de-skew", + "ddr3a11_ddr4a3_de-skew", + "ddr3a10_ddr4cs0_de-skew", + "ddr3a13_ddr4a2_de-skew", + "ddr3a12_ddr4ba1_de-skew", + "ddr3a15_ddr4odt0_de-skew", + "ddr3a14_ddr4a1_de-skew", + "ddr3ba1_ddr4a15_de-skew", + "ddr3ba0_ddr4bg0_de-skew", + "ddr3ras_ddr4cke_de-skew", + "ddr3ba2_ddr4ba0_de-skew", + "ddr3we_ddr4bg1_de-skew", + "ddr3cas_ddr4a12_de-skew", + "ddr3ckn_ddr4ckn_de-skew", + "ddr3ckp_ddr4ckp_de-skew", + "ddr3cke_ddr4a16_de-skew", + "ddr3odt0_ddr4a14_de-skew", + "ddr3cs0_ddr4act_de-skew", + "ddr3reset_ddr4reset_de-skew", + "ddr3cs1_ddr4cs1_de-skew", + "ddr3odt1_ddr4odt1_de-skew", +}; + +static const char * const rk3328_dts_cs0_timing[] = { + "cs0_dm0_rx_de-skew", + "cs0_dm0_tx_de-skew", + "cs0_dq0_rx_de-skew", + "cs0_dq0_tx_de-skew", + "cs0_dq1_rx_de-skew", + "cs0_dq1_tx_de-skew", + "cs0_dq2_rx_de-skew", + "cs0_dq2_tx_de-skew", + "cs0_dq3_rx_de-skew", + "cs0_dq3_tx_de-skew", + "cs0_dq4_rx_de-skew", + "cs0_dq4_tx_de-skew", + "cs0_dq5_rx_de-skew", + "cs0_dq5_tx_de-skew", + "cs0_dq6_rx_de-skew", + "cs0_dq6_tx_de-skew", + "cs0_dq7_rx_de-skew", + "cs0_dq7_tx_de-skew", + "cs0_dqs0_rx_de-skew", + "cs0_dqs0p_tx_de-skew", + "cs0_dqs0n_tx_de-skew", + + "cs0_dm1_rx_de-skew", + "cs0_dm1_tx_de-skew", + "cs0_dq8_rx_de-skew", + "cs0_dq8_tx_de-skew", + "cs0_dq9_rx_de-skew", + "cs0_dq9_tx_de-skew", + "cs0_dq10_rx_de-skew", + "cs0_dq10_tx_de-skew", + "cs0_dq11_rx_de-skew", + "cs0_dq11_tx_de-skew", + "cs0_dq12_rx_de-skew", + "cs0_dq12_tx_de-skew", + "cs0_dq13_rx_de-skew", + "cs0_dq13_tx_de-skew", + "cs0_dq14_rx_de-skew", + "cs0_dq14_tx_de-skew", + "cs0_dq15_rx_de-skew", + "cs0_dq15_tx_de-skew", + "cs0_dqs1_rx_de-skew", + "cs0_dqs1p_tx_de-skew", + "cs0_dqs1n_tx_de-skew", + + "cs0_dm2_rx_de-skew", + "cs0_dm2_tx_de-skew", + "cs0_dq16_rx_de-skew", + "cs0_dq16_tx_de-skew", + "cs0_dq17_rx_de-skew", + "cs0_dq17_tx_de-skew", + "cs0_dq18_rx_de-skew", + "cs0_dq18_tx_de-skew", + "cs0_dq19_rx_de-skew", + "cs0_dq19_tx_de-skew", + "cs0_dq20_rx_de-skew", + "cs0_dq20_tx_de-skew", + "cs0_dq21_rx_de-skew", + "cs0_dq21_tx_de-skew", + "cs0_dq22_rx_de-skew", + "cs0_dq22_tx_de-skew", + "cs0_dq23_rx_de-skew", + "cs0_dq23_tx_de-skew", + "cs0_dqs2_rx_de-skew", + "cs0_dqs2p_tx_de-skew", + "cs0_dqs2n_tx_de-skew", + + "cs0_dm3_rx_de-skew", + "cs0_dm3_tx_de-skew", + "cs0_dq24_rx_de-skew", + "cs0_dq24_tx_de-skew", + "cs0_dq25_rx_de-skew", + "cs0_dq25_tx_de-skew", + "cs0_dq26_rx_de-skew", + "cs0_dq26_tx_de-skew", + "cs0_dq27_rx_de-skew", + "cs0_dq27_tx_de-skew", + "cs0_dq28_rx_de-skew", + "cs0_dq28_tx_de-skew", + "cs0_dq29_rx_de-skew", + "cs0_dq29_tx_de-skew", + "cs0_dq30_rx_de-skew", + "cs0_dq30_tx_de-skew", + "cs0_dq31_rx_de-skew", + "cs0_dq31_tx_de-skew", + "cs0_dqs3_rx_de-skew", + "cs0_dqs3p_tx_de-skew", + "cs0_dqs3n_tx_de-skew", +}; + +static const char * const rk3328_dts_cs1_timing[] = { + "cs1_dm0_rx_de-skew", + "cs1_dm0_tx_de-skew", + "cs1_dq0_rx_de-skew", + "cs1_dq0_tx_de-skew", + "cs1_dq1_rx_de-skew", + "cs1_dq1_tx_de-skew", + "cs1_dq2_rx_de-skew", + "cs1_dq2_tx_de-skew", + "cs1_dq3_rx_de-skew", + "cs1_dq3_tx_de-skew", + "cs1_dq4_rx_de-skew", + "cs1_dq4_tx_de-skew", + "cs1_dq5_rx_de-skew", + "cs1_dq5_tx_de-skew", + "cs1_dq6_rx_de-skew", + "cs1_dq6_tx_de-skew", + "cs1_dq7_rx_de-skew", + "cs1_dq7_tx_de-skew", + "cs1_dqs0_rx_de-skew", + "cs1_dqs0p_tx_de-skew", + "cs1_dqs0n_tx_de-skew", + + "cs1_dm1_rx_de-skew", + "cs1_dm1_tx_de-skew", + "cs1_dq8_rx_de-skew", + "cs1_dq8_tx_de-skew", + "cs1_dq9_rx_de-skew", + "cs1_dq9_tx_de-skew", + "cs1_dq10_rx_de-skew", + "cs1_dq10_tx_de-skew", + "cs1_dq11_rx_de-skew", + "cs1_dq11_tx_de-skew", + "cs1_dq12_rx_de-skew", + "cs1_dq12_tx_de-skew", + "cs1_dq13_rx_de-skew", + "cs1_dq13_tx_de-skew", + "cs1_dq14_rx_de-skew", + "cs1_dq14_tx_de-skew", + "cs1_dq15_rx_de-skew", + "cs1_dq15_tx_de-skew", + "cs1_dqs1_rx_de-skew", + "cs1_dqs1p_tx_de-skew", + "cs1_dqs1n_tx_de-skew", + + "cs1_dm2_rx_de-skew", + "cs1_dm2_tx_de-skew", + "cs1_dq16_rx_de-skew", + "cs1_dq16_tx_de-skew", + "cs1_dq17_rx_de-skew", + "cs1_dq17_tx_de-skew", + "cs1_dq18_rx_de-skew", + "cs1_dq18_tx_de-skew", + "cs1_dq19_rx_de-skew", + "cs1_dq19_tx_de-skew", + "cs1_dq20_rx_de-skew", + "cs1_dq20_tx_de-skew", + "cs1_dq21_rx_de-skew", + "cs1_dq21_tx_de-skew", + "cs1_dq22_rx_de-skew", + "cs1_dq22_tx_de-skew", + "cs1_dq23_rx_de-skew", + "cs1_dq23_tx_de-skew", + "cs1_dqs2_rx_de-skew", + "cs1_dqs2p_tx_de-skew", + "cs1_dqs2n_tx_de-skew", + + "cs1_dm3_rx_de-skew", + "cs1_dm3_tx_de-skew", + "cs1_dq24_rx_de-skew", + "cs1_dq24_tx_de-skew", + "cs1_dq25_rx_de-skew", + "cs1_dq25_tx_de-skew", + "cs1_dq26_rx_de-skew", + "cs1_dq26_tx_de-skew", + "cs1_dq27_rx_de-skew", + "cs1_dq27_tx_de-skew", + "cs1_dq28_rx_de-skew", + "cs1_dq28_tx_de-skew", + "cs1_dq29_rx_de-skew", + "cs1_dq29_tx_de-skew", + "cs1_dq30_rx_de-skew", + "cs1_dq30_tx_de-skew", + "cs1_dq31_rx_de-skew", + "cs1_dq31_tx_de-skew", + "cs1_dqs3_rx_de-skew", + "cs1_dqs3p_tx_de-skew", + "cs1_dqs3n_tx_de-skew", +}; + +struct rk3328_ddr_dts_config_timing { + unsigned int ddr3_speed_bin; + unsigned int ddr4_speed_bin; + unsigned int pd_idle; + unsigned int sr_idle; + unsigned int sr_mc_gate_idle; + unsigned int srpd_lite_idle; + unsigned int standby_idle; + + unsigned int auto_pd_dis_freq; + unsigned int auto_sr_dis_freq; + /* for ddr3 only */ + unsigned int ddr3_dll_dis_freq; + /* for ddr4 only */ + unsigned int ddr4_dll_dis_freq; + unsigned int phy_dll_dis_freq; + + unsigned int ddr3_odt_dis_freq; + unsigned int phy_ddr3_odt_dis_freq; + unsigned int ddr3_drv; + unsigned int ddr3_odt; + unsigned int phy_ddr3_ca_drv; + unsigned int phy_ddr3_ck_drv; + unsigned int phy_ddr3_dq_drv; + unsigned int phy_ddr3_odt; + + unsigned int lpddr3_odt_dis_freq; + unsigned int phy_lpddr3_odt_dis_freq; + unsigned int lpddr3_drv; + unsigned int lpddr3_odt; + unsigned int phy_lpddr3_ca_drv; + unsigned int phy_lpddr3_ck_drv; + unsigned int phy_lpddr3_dq_drv; + unsigned int phy_lpddr3_odt; + + unsigned int lpddr4_odt_dis_freq; + unsigned int phy_lpddr4_odt_dis_freq; + unsigned int lpddr4_drv; + unsigned int lpddr4_dq_odt; + unsigned int lpddr4_ca_odt; + unsigned int phy_lpddr4_ca_drv; + unsigned int phy_lpddr4_ck_cs_drv; + unsigned int phy_lpddr4_dq_drv; + unsigned int phy_lpddr4_odt; + + unsigned int ddr4_odt_dis_freq; + unsigned int phy_ddr4_odt_dis_freq; + unsigned int ddr4_drv; + unsigned int ddr4_odt; + unsigned int phy_ddr4_ca_drv; + unsigned int phy_ddr4_ck_drv; + unsigned int phy_ddr4_dq_drv; + unsigned int phy_ddr4_odt; + + unsigned int ca_skew[15]; + unsigned int cs0_skew[44]; + unsigned int cs1_skew[44]; + + unsigned int available; +}; + +struct rk3328_ddr_de_skew_setting { + unsigned int ca_de_skew[30]; + unsigned int cs0_de_skew[84]; + unsigned int cs1_de_skew[84]; +}; + +struct rk3368_dram_timing { + u32 dram_spd_bin; + u32 sr_idle; + u32 pd_idle; + u32 dram_dll_dis_freq; + u32 phy_dll_dis_freq; + u32 dram_odt_dis_freq; + u32 phy_odt_dis_freq; + u32 ddr3_drv; + u32 ddr3_odt; + u32 lpddr3_drv; + u32 lpddr3_odt; + u32 lpddr2_drv; + u32 phy_clk_drv; + u32 phy_cmd_drv; + u32 phy_dqs_drv; + u32 phy_odt; + u32 ddr_2t; +}; + +struct rk3399_dram_timing { + unsigned int ddr3_speed_bin; + unsigned int pd_idle; + unsigned int sr_idle; + unsigned int sr_mc_gate_idle; + unsigned int srpd_lite_idle; + unsigned int standby_idle; + unsigned int auto_lp_dis_freq; + unsigned int ddr3_dll_dis_freq; + unsigned int phy_dll_dis_freq; + unsigned int ddr3_odt_dis_freq; + unsigned int ddr3_drv; + unsigned int ddr3_odt; + unsigned int phy_ddr3_ca_drv; + unsigned int phy_ddr3_dq_drv; + unsigned int phy_ddr3_odt; + unsigned int lpddr3_odt_dis_freq; + unsigned int lpddr3_drv; + unsigned int lpddr3_odt; + unsigned int phy_lpddr3_ca_drv; + unsigned int phy_lpddr3_dq_drv; + unsigned int phy_lpddr3_odt; + unsigned int lpddr4_odt_dis_freq; + unsigned int lpddr4_drv; + unsigned int lpddr4_dq_odt; + unsigned int lpddr4_ca_odt; + unsigned int phy_lpddr4_ca_drv; + unsigned int phy_lpddr4_ck_cs_drv; + unsigned int phy_lpddr4_dq_drv; + unsigned int phy_lpddr4_odt; +}; + +struct rk3568_ddr_dts_config_timing { + unsigned int ddr2_speed_bin; + unsigned int ddr3_speed_bin; + unsigned int ddr4_speed_bin; + unsigned int pd_idle; + unsigned int sr_idle; + unsigned int sr_mc_gate_idle; + unsigned int srpd_lite_idle; + unsigned int standby_idle; + + unsigned int auto_pd_dis_freq; + unsigned int auto_sr_dis_freq; + /* for ddr2 only */ + unsigned int ddr2_dll_dis_freq; + /* for ddr3 only */ + unsigned int ddr3_dll_dis_freq; + /* for ddr4 only */ + unsigned int ddr4_dll_dis_freq; + unsigned int phy_dll_dis_freq; + + unsigned int ddr2_odt_dis_freq; + unsigned int phy_ddr2_odt_dis_freq; + unsigned int ddr2_drv; + unsigned int ddr2_odt; + unsigned int phy_ddr2_ca_drv; + unsigned int phy_ddr2_ck_drv; + unsigned int phy_ddr2_dq_drv; + unsigned int phy_ddr2_odt; + + unsigned int ddr3_odt_dis_freq; + unsigned int phy_ddr3_odt_dis_freq; + unsigned int ddr3_drv; + unsigned int ddr3_odt; + unsigned int phy_ddr3_ca_drv; + unsigned int phy_ddr3_ck_drv; + unsigned int phy_ddr3_dq_drv; + unsigned int phy_ddr3_odt; + + unsigned int phy_lpddr2_odt_dis_freq; + unsigned int lpddr2_drv; + unsigned int phy_lpddr2_ca_drv; + unsigned int phy_lpddr2_ck_drv; + unsigned int phy_lpddr2_dq_drv; + unsigned int phy_lpddr2_odt; + + unsigned int lpddr3_odt_dis_freq; + unsigned int phy_lpddr3_odt_dis_freq; + unsigned int lpddr3_drv; + unsigned int lpddr3_odt; + unsigned int phy_lpddr3_ca_drv; + unsigned int phy_lpddr3_ck_drv; + unsigned int phy_lpddr3_dq_drv; + unsigned int phy_lpddr3_odt; + + unsigned int lpddr4_odt_dis_freq; + unsigned int phy_lpddr4_odt_dis_freq; + unsigned int lpddr4_drv; + unsigned int lpddr4_dq_odt; + unsigned int lpddr4_ca_odt; + unsigned int phy_lpddr4_ca_drv; + unsigned int phy_lpddr4_ck_cs_drv; + unsigned int phy_lpddr4_dq_drv; + unsigned int phy_lpddr4_odt; + + unsigned int ddr4_odt_dis_freq; + unsigned int phy_ddr4_odt_dis_freq; + unsigned int ddr4_drv; + unsigned int ddr4_odt; + unsigned int phy_ddr4_ca_drv; + unsigned int phy_ddr4_ck_drv; + unsigned int phy_ddr4_dq_drv; + unsigned int phy_ddr4_odt; + + unsigned int available; +}; + +/* name rule: ddr4(pad_name)_ddr3_lpddr3_lpddr4_de-skew */ +static const char * const rv1126_dts_ca_timing[] = { + "a0_a3_a3_cke1-a_de-skew", + "a1_ba1_null_cke0-b_de-skew", + "a2_a9_a9_a4-a_de-skew", + "a3_a15_null_a5-b_de-skew", + "a4_a6_a6_ck-a_de-skew", + "a5_a12_null_odt0-b_de-skew", + "a6_ba2_null_a0-a_de-skew", + "a7_a4_a4_odt0-a_de-skew", + "a8_a1_a1_cke0-a_de-skew", + "a9_a5_a5_a5-a_de-skew", + "a10_a8_a8_clkb-a_de-skew", + "a11_a7_a7_ca2-a_de-skew", + "a12_rasn_null_ca1-a_de-skew", + "a13_a13_null_ca3-a_de-skew", + "a14_a14_null_csb1-b_de-skew", + "a15_a10_null_ca0-b_de-skew", + "a16_a11_null_csb0-b_de-skew", + "a17_null_null_null_de-skew", + "ba0_csb1_csb1_csb0-a_de-skew", + "ba1_wen_null_cke1-b_de-skew", + "bg0_odt1_odt1_csb1-a_de-skew", + "bg1_a2_a2_odt1-a_de-skew", + "cke0_casb_null_ca1-b_de-skew", + "ck_ck_ck_ck-b_de-skew", + "ckb_ckb_ckb_ckb-b_de-skew", + "csb0_odt0_odt0_ca2-b_de-skew", + "odt0_csb0_csb0_ca4-b_de-skew", + "resetn_resetn_null-resetn_de-skew", + "actn_cke_cke_ca3-b_de-skew", + "cke1_null_null_null_de-skew", + "csb1_ba0_null_null_de-skew", + "odt1_a0_a0_odt1-b_de-skew", +}; + +static const char * const rv1126_dts_cs0_a_timing[] = { + "cs0_dm0_rx_de-skew", + "cs0_dq0_rx_de-skew", + "cs0_dq1_rx_de-skew", + "cs0_dq2_rx_de-skew", + "cs0_dq3_rx_de-skew", + "cs0_dq4_rx_de-skew", + "cs0_dq5_rx_de-skew", + "cs0_dq6_rx_de-skew", + "cs0_dq7_rx_de-skew", + "cs0_dqs0p_rx_de-skew", + "cs0_dqs0n_rx_de-skew", + "cs0_dm1_rx_de-skew", + "cs0_dq8_rx_de-skew", + "cs0_dq9_rx_de-skew", + "cs0_dq10_rx_de-skew", + "cs0_dq11_rx_de-skew", + "cs0_dq12_rx_de-skew", + "cs0_dq13_rx_de-skew", + "cs0_dq14_rx_de-skew", + "cs0_dq15_rx_de-skew", + "cs0_dqs1p_rx_de-skew", + "cs0_dqs1n_rx_de-skew", + "cs0_dm0_tx_de-skew", + "cs0_dq0_tx_de-skew", + "cs0_dq1_tx_de-skew", + "cs0_dq2_tx_de-skew", + "cs0_dq3_tx_de-skew", + "cs0_dq4_tx_de-skew", + "cs0_dq5_tx_de-skew", + "cs0_dq6_tx_de-skew", + "cs0_dq7_tx_de-skew", + "cs0_dqs0p_tx_de-skew", + "cs0_dqs0n_tx_de-skew", + "cs0_dm1_tx_de-skew", + "cs0_dq8_tx_de-skew", + "cs0_dq9_tx_de-skew", + "cs0_dq10_tx_de-skew", + "cs0_dq11_tx_de-skew", + "cs0_dq12_tx_de-skew", + "cs0_dq13_tx_de-skew", + "cs0_dq14_tx_de-skew", + "cs0_dq15_tx_de-skew", + "cs0_dqs1p_tx_de-skew", + "cs0_dqs1n_tx_de-skew", +}; + +static const char * const rv1126_dts_cs0_b_timing[] = { + "cs0_dm2_rx_de-skew", + "cs0_dq16_rx_de-skew", + "cs0_dq17_rx_de-skew", + "cs0_dq18_rx_de-skew", + "cs0_dq19_rx_de-skew", + "cs0_dq20_rx_de-skew", + "cs0_dq21_rx_de-skew", + "cs0_dq22_rx_de-skew", + "cs0_dq23_rx_de-skew", + "cs0_dqs2p_rx_de-skew", + "cs0_dqs2n_rx_de-skew", + "cs0_dm3_rx_de-skew", + "cs0_dq24_rx_de-skew", + "cs0_dq25_rx_de-skew", + "cs0_dq26_rx_de-skew", + "cs0_dq27_rx_de-skew", + "cs0_dq28_rx_de-skew", + "cs0_dq29_rx_de-skew", + "cs0_dq30_rx_de-skew", + "cs0_dq31_rx_de-skew", + "cs0_dqs3p_rx_de-skew", + "cs0_dqs3n_rx_de-skew", + "cs0_dm2_tx_de-skew", + "cs0_dq16_tx_de-skew", + "cs0_dq17_tx_de-skew", + "cs0_dq18_tx_de-skew", + "cs0_dq19_tx_de-skew", + "cs0_dq20_tx_de-skew", + "cs0_dq21_tx_de-skew", + "cs0_dq22_tx_de-skew", + "cs0_dq23_tx_de-skew", + "cs0_dqs2p_tx_de-skew", + "cs0_dqs2n_tx_de-skew", + "cs0_dm3_tx_de-skew", + "cs0_dq24_tx_de-skew", + "cs0_dq25_tx_de-skew", + "cs0_dq26_tx_de-skew", + "cs0_dq27_tx_de-skew", + "cs0_dq28_tx_de-skew", + "cs0_dq29_tx_de-skew", + "cs0_dq30_tx_de-skew", + "cs0_dq31_tx_de-skew", + "cs0_dqs3p_tx_de-skew", + "cs0_dqs3n_tx_de-skew", +}; + +static const char * const rv1126_dts_cs1_a_timing[] = { + "cs1_dm0_rx_de-skew", + "cs1_dq0_rx_de-skew", + "cs1_dq1_rx_de-skew", + "cs1_dq2_rx_de-skew", + "cs1_dq3_rx_de-skew", + "cs1_dq4_rx_de-skew", + "cs1_dq5_rx_de-skew", + "cs1_dq6_rx_de-skew", + "cs1_dq7_rx_de-skew", + "cs1_dqs0p_rx_de-skew", + "cs1_dqs0n_rx_de-skew", + "cs1_dm1_rx_de-skew", + "cs1_dq8_rx_de-skew", + "cs1_dq9_rx_de-skew", + "cs1_dq10_rx_de-skew", + "cs1_dq11_rx_de-skew", + "cs1_dq12_rx_de-skew", + "cs1_dq13_rx_de-skew", + "cs1_dq14_rx_de-skew", + "cs1_dq15_rx_de-skew", + "cs1_dqs1p_rx_de-skew", + "cs1_dqs1n_rx_de-skew", + "cs1_dm0_tx_de-skew", + "cs1_dq0_tx_de-skew", + "cs1_dq1_tx_de-skew", + "cs1_dq2_tx_de-skew", + "cs1_dq3_tx_de-skew", + "cs1_dq4_tx_de-skew", + "cs1_dq5_tx_de-skew", + "cs1_dq6_tx_de-skew", + "cs1_dq7_tx_de-skew", + "cs1_dqs0p_tx_de-skew", + "cs1_dqs0n_tx_de-skew", + "cs1_dm1_tx_de-skew", + "cs1_dq8_tx_de-skew", + "cs1_dq9_tx_de-skew", + "cs1_dq10_tx_de-skew", + "cs1_dq11_tx_de-skew", + "cs1_dq12_tx_de-skew", + "cs1_dq13_tx_de-skew", + "cs1_dq14_tx_de-skew", + "cs1_dq15_tx_de-skew", + "cs1_dqs1p_tx_de-skew", + "cs1_dqs1n_tx_de-skew", +}; + +static const char * const rv1126_dts_cs1_b_timing[] = { + "cs1_dm2_rx_de-skew", + "cs1_dq16_rx_de-skew", + "cs1_dq17_rx_de-skew", + "cs1_dq18_rx_de-skew", + "cs1_dq19_rx_de-skew", + "cs1_dq20_rx_de-skew", + "cs1_dq21_rx_de-skew", + "cs1_dq22_rx_de-skew", + "cs1_dq23_rx_de-skew", + "cs1_dqs2p_rx_de-skew", + "cs1_dqs2n_rx_de-skew", + "cs1_dm3_rx_de-skew", + "cs1_dq24_rx_de-skew", + "cs1_dq25_rx_de-skew", + "cs1_dq26_rx_de-skew", + "cs1_dq27_rx_de-skew", + "cs1_dq28_rx_de-skew", + "cs1_dq29_rx_de-skew", + "cs1_dq30_rx_de-skew", + "cs1_dq31_rx_de-skew", + "cs1_dqs3p_rx_de-skew", + "cs1_dqs3n_rx_de-skew", + "cs1_dm2_tx_de-skew", + "cs1_dq16_tx_de-skew", + "cs1_dq17_tx_de-skew", + "cs1_dq18_tx_de-skew", + "cs1_dq19_tx_de-skew", + "cs1_dq20_tx_de-skew", + "cs1_dq21_tx_de-skew", + "cs1_dq22_tx_de-skew", + "cs1_dq23_tx_de-skew", + "cs1_dqs2p_tx_de-skew", + "cs1_dqs2n_tx_de-skew", + "cs1_dm3_tx_de-skew", + "cs1_dq24_tx_de-skew", + "cs1_dq25_tx_de-skew", + "cs1_dq26_tx_de-skew", + "cs1_dq27_tx_de-skew", + "cs1_dq28_tx_de-skew", + "cs1_dq29_tx_de-skew", + "cs1_dq30_tx_de-skew", + "cs1_dq31_tx_de-skew", + "cs1_dqs3p_tx_de-skew", + "cs1_dqs3n_tx_de-skew", +}; + +#endif /* __ROCKCHIP_DMC_TIMING_H__ */ + diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig index 594b77d8923c..c5de024009a9 100644 --- a/drivers/dma-buf/Kconfig +++ b/drivers/dma-buf/Kconfig @@ -21,7 +21,6 @@ config SW_SYNC bool "Sync File Validation Framework" default n depends on SYNC_FILE - depends on DEBUG_FS help A sync object driver that uses a 32bit counter to coordinate synchronization. Useful when there is no hardware primitive backing diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 89c10136b26f..3ef87860877a 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -32,8 +32,6 @@ #include "dma-buf-sysfs-stats.h" #include "dma-buf-process-info.h" -static inline int is_dma_buf_file(struct file *); - struct dma_buf_list { struct list_head head; struct mutex lock; @@ -41,6 +39,30 @@ struct dma_buf_list { static struct dma_buf_list db_list; +/* + * This function helps in traversing the db_list and calls the + * callback function which can extract required info out of each + * dmabuf. + */ +int get_each_dmabuf(int (*callback)(const struct dma_buf *dmabuf, + void *private), void *private) +{ + struct dma_buf *buf; + int ret = mutex_lock_interruptible(&db_list.lock); + + if (ret) + return ret; + + list_for_each_entry(buf, &db_list.head, list_node) { + ret = callback(buf, private); + if (ret) + break; + } + mutex_unlock(&db_list.lock); + return ret; +} +EXPORT_SYMBOL_GPL(get_each_dmabuf); + static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen) { struct dma_buf *dmabuf; @@ -129,6 +151,54 @@ static struct file_system_type dma_buf_fs_type = { .kill_sb = kill_anon_super, }; +#ifdef CONFIG_DMABUF_SYSFS_STATS +static void dma_buf_vma_open(struct vm_area_struct *vma) +{ + struct dma_buf *dmabuf = vma->vm_file->private_data; + + dmabuf->mmap_count++; + /* call the heap provided vma open() op */ + if (dmabuf->exp_vm_ops->open) + dmabuf->exp_vm_ops->open(vma); +} + +static void dma_buf_vma_close(struct vm_area_struct *vma) +{ + struct dma_buf *dmabuf = vma->vm_file->private_data; + + if (dmabuf->mmap_count) + dmabuf->mmap_count--; + /* call the heap provided vma close() op */ + if (dmabuf->exp_vm_ops->close) + dmabuf->exp_vm_ops->close(vma); +} + +static int dma_buf_do_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + /* call this first because the exporter might override vma->vm_ops */ + int ret = dmabuf->ops->mmap(dmabuf, vma); + + if (ret) + return ret; + + /* save the exporter provided vm_ops */ + dmabuf->exp_vm_ops = vma->vm_ops; + dmabuf->vm_ops = *(dmabuf->exp_vm_ops); + /* override open() and close() to provide buffer mmap count */ + dmabuf->vm_ops.open = dma_buf_vma_open; + dmabuf->vm_ops.close = dma_buf_vma_close; + vma->vm_ops = &dmabuf->vm_ops; + dmabuf->mmap_count++; + + return ret; +} +#else /* CONFIG_DMABUF_SYSFS_STATS */ +static int dma_buf_do_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + return dmabuf->ops->mmap(dmabuf, vma); +} +#endif /* CONFIG_DMABUF_SYSFS_STATS */ + static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) { struct dma_buf *dmabuf; @@ -147,7 +217,7 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) dmabuf->size >> PAGE_SHIFT) return -EINVAL; - return dmabuf->ops->mmap(dmabuf, vma); + return dma_buf_do_mmap(dmabuf, vma); } static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence) @@ -442,10 +512,11 @@ static const struct file_operations dma_buf_fops = { /* * is_dma_buf_file - Check if struct file* is associated with dma_buf */ -static inline int is_dma_buf_file(struct file *file) +int is_dma_buf_file(struct file *file) { return file->f_op == &dma_buf_fops; } +EXPORT_SYMBOL_GPL(is_dma_buf_file); static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags) { @@ -1132,6 +1203,30 @@ int dma_buf_begin_cpu_access(struct dma_buf *dmabuf, } EXPORT_SYMBOL_GPL(dma_buf_begin_cpu_access); +int dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction direction, + unsigned int offset, unsigned int len) +{ + int ret = 0; + + if (WARN_ON(!dmabuf)) + return -EINVAL; + + if (dmabuf->ops->begin_cpu_access_partial) + ret = dmabuf->ops->begin_cpu_access_partial(dmabuf, direction, + offset, len); + + /* Ensure that all fences are waited upon - but we first allow + * the native handler the chance to do so more efficiently if it + * chooses. A double invocation here will be reasonably cheap no-op. + */ + if (ret == 0) + ret = __dma_buf_begin_cpu_access(dmabuf, direction); + + return ret; +} +EXPORT_SYMBOL_GPL(dma_buf_begin_cpu_access_partial); + /** * dma_buf_end_cpu_access - Must be called after accessing a dma_buf from the * cpu in the kernel context. Calls end_cpu_access to allow exporter-specific @@ -1158,6 +1253,21 @@ int dma_buf_end_cpu_access(struct dma_buf *dmabuf, } EXPORT_SYMBOL_GPL(dma_buf_end_cpu_access); +int dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction direction, + unsigned int offset, unsigned int len) +{ + int ret = 0; + + WARN_ON(!dmabuf); + + if (dmabuf->ops->end_cpu_access_partial) + ret = dmabuf->ops->end_cpu_access_partial(dmabuf, direction, + offset, len); + + return ret; +} +EXPORT_SYMBOL_GPL(dma_buf_end_cpu_access_partial); /** * dma_buf_mmap - Setup up a userspace mmap with the given vma @@ -1286,6 +1396,32 @@ void dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) } EXPORT_SYMBOL_GPL(dma_buf_vunmap); +int dma_buf_get_flags(struct dma_buf *dmabuf, unsigned long *flags) +{ + int ret = 0; + + if (WARN_ON(!dmabuf) || !flags) + return -EINVAL; + + if (dmabuf->ops->get_flags) + ret = dmabuf->ops->get_flags(dmabuf, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(dma_buf_get_flags); + +int dma_buf_get_uuid(struct dma_buf *dmabuf, uuid_t *uuid) +{ + if (WARN_ON(!dmabuf) || !uuid) + return -EINVAL; + + if (!dmabuf->ops->get_uuid) + return -ENODEV; + + return dmabuf->ops->get_uuid(dmabuf, uuid); +} +EXPORT_SYMBOL_GPL(dma_buf_get_uuid); + #ifdef CONFIG_DEBUG_FS static int dma_buf_debug_show(struct seq_file *s, void *unused) { diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 7475e09b0680..d64fc03929be 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -312,22 +312,25 @@ void __dma_fence_might_wait(void) /** - * dma_fence_signal_locked - signal completion of a fence + * dma_fence_signal_timestamp_locked - signal completion of a fence * @fence: the fence to signal + * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain * * Signal completion for software callbacks on a fence, this will unblock * dma_fence_wait() calls and run all the callbacks added with * dma_fence_add_callback(). Can be called multiple times, but since a fence * can only go from the unsignaled to the signaled state and not back, it will - * only be effective the first time. + * only be effective the first time. Set the timestamp provided as the fence + * signal timestamp. * - * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock - * held. + * Unlike dma_fence_signal_timestamp(), this function must be called with + * &dma_fence.lock held. * * Returns 0 on success and a negative error value when @fence has been * signalled already. */ -int dma_fence_signal_locked(struct dma_fence *fence) +int dma_fence_signal_timestamp_locked(struct dma_fence *fence, + ktime_t timestamp) { struct dma_fence_cb *cur, *tmp; struct list_head cb_list; @@ -341,7 +344,7 @@ int dma_fence_signal_locked(struct dma_fence *fence) /* Stash the cb_list before replacing it with the timestamp */ list_replace(&fence->cb_list, &cb_list); - fence->timestamp = ktime_get(); + fence->timestamp = timestamp; set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); trace_dma_fence_signaled(fence); @@ -352,6 +355,59 @@ int dma_fence_signal_locked(struct dma_fence *fence) return 0; } +EXPORT_SYMBOL(dma_fence_signal_timestamp_locked); + +/** + * dma_fence_signal_timestamp - signal completion of a fence + * @fence: the fence to signal + * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain + * + * Signal completion for software callbacks on a fence, this will unblock + * dma_fence_wait() calls and run all the callbacks added with + * dma_fence_add_callback(). Can be called multiple times, but since a fence + * can only go from the unsignaled to the signaled state and not back, it will + * only be effective the first time. Set the timestamp provided as the fence + * signal timestamp. + * + * Returns 0 on success and a negative error value when @fence has been + * signalled already. + */ +int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp) +{ + unsigned long flags; + int ret; + + if (!fence) + return -EINVAL; + + spin_lock_irqsave(fence->lock, flags); + ret = dma_fence_signal_timestamp_locked(fence, timestamp); + spin_unlock_irqrestore(fence->lock, flags); + + return ret; +} +EXPORT_SYMBOL(dma_fence_signal_timestamp); + +/** + * dma_fence_signal_locked - signal completion of a fence + * @fence: the fence to signal + * + * Signal completion for software callbacks on a fence, this will unblock + * dma_fence_wait() calls and run all the callbacks added with + * dma_fence_add_callback(). Can be called multiple times, but since a fence + * can only go from the unsignaled to the signaled state and not back, it will + * only be effective the first time. + * + * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock + * held. + * + * Returns 0 on success and a negative error value when @fence has been + * signalled already. + */ +int dma_fence_signal_locked(struct dma_fence *fence) +{ + return dma_fence_signal_timestamp_locked(fence, ktime_get()); +} EXPORT_SYMBOL(dma_fence_signal_locked); /** @@ -379,7 +435,7 @@ int dma_fence_signal(struct dma_fence *fence) tmp = dma_fence_begin_signalling(); spin_lock_irqsave(fence->lock, flags); - ret = dma_fence_signal_locked(fence); + ret = dma_fence_signal_timestamp_locked(fence, ktime_get()); spin_unlock_irqrestore(fence->lock, flags); dma_fence_end_signalling(tmp); diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c index bbbfa28b2f6c..6c22995616a2 100644 --- a/drivers/dma-buf/dma-heap.c +++ b/drivers/dma-buf/dma-heap.c @@ -31,6 +31,7 @@ * @heap_devt heap device node * @list list head connecting to list of heaps * @heap_cdev heap char device + * @heap_dev heap device struct * * Represents a heap of memory from which buffers can be made. */ @@ -41,6 +42,8 @@ struct dma_heap { dev_t heap_devt; struct list_head list; struct cdev heap_cdev; + struct kref refcount; + struct device *heap_dev; }; static LIST_HEAD(heap_list); @@ -49,20 +52,72 @@ static dev_t dma_heap_devt; static struct class *dma_heap_class; static DEFINE_XARRAY_ALLOC(dma_heap_minors); -static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, - unsigned int fd_flags, - unsigned int heap_flags) +struct dma_heap *dma_heap_find(const char *name) { + struct dma_heap *h; + + mutex_lock(&heap_list_lock); + list_for_each_entry(h, &heap_list, list) { + if (!strcmp(h->name, name)) { + kref_get(&h->refcount); + mutex_unlock(&heap_list_lock); + return h; + } + } + mutex_unlock(&heap_list_lock); + return NULL; +} +EXPORT_SYMBOL_GPL(dma_heap_find); + + +void dma_heap_buffer_free(struct dma_buf *dmabuf) +{ + dma_buf_put(dmabuf); +} +EXPORT_SYMBOL_GPL(dma_heap_buffer_free); + +struct dma_buf *dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, + unsigned int fd_flags, + unsigned int heap_flags) +{ + if (fd_flags & ~DMA_HEAP_VALID_FD_FLAGS) + return ERR_PTR(-EINVAL); + + if (heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS) + return ERR_PTR(-EINVAL); /* * Allocations from all heaps have to begin * and end on page boundaries. */ len = PAGE_ALIGN(len); if (!len) - return -EINVAL; + return ERR_PTR(-EINVAL); return heap->ops->allocate(heap, len, fd_flags, heap_flags); } +EXPORT_SYMBOL_GPL(dma_heap_buffer_alloc); + +int dma_heap_bufferfd_alloc(struct dma_heap *heap, size_t len, + unsigned int fd_flags, + unsigned int heap_flags) +{ + struct dma_buf *dmabuf; + int fd; + + dmabuf = dma_heap_buffer_alloc(heap, len, fd_flags, heap_flags); + + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + fd = dma_buf_fd(dmabuf, fd_flags); + if (fd < 0) { + dma_buf_put(dmabuf); + /* just return, as put will call release and that will free */ + } + return fd; + +} +EXPORT_SYMBOL_GPL(dma_heap_bufferfd_alloc); static int dma_heap_open(struct inode *inode, struct file *file) { @@ -90,15 +145,9 @@ static long dma_heap_ioctl_allocate(struct file *file, void *data) if (heap_allocation->fd) return -EINVAL; - if (heap_allocation->fd_flags & ~DMA_HEAP_VALID_FD_FLAGS) - return -EINVAL; - - if (heap_allocation->heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS) - return -EINVAL; - - fd = dma_heap_buffer_alloc(heap, heap_allocation->len, - heap_allocation->fd_flags, - heap_allocation->heap_flags); + fd = dma_heap_bufferfd_alloc(heap, heap_allocation->len, + heap_allocation->fd_flags, + heap_allocation->heap_flags); if (fd < 0) return fd; @@ -191,6 +240,47 @@ void *dma_heap_get_drvdata(struct dma_heap *heap) { return heap->priv; } +EXPORT_SYMBOL_GPL(dma_heap_get_drvdata); + +static void dma_heap_release(struct kref *ref) +{ + struct dma_heap *heap = container_of(ref, struct dma_heap, refcount); + int minor = MINOR(heap->heap_devt); + + /* Note, we already holding the heap_list_lock here */ + list_del(&heap->list); + + device_destroy(dma_heap_class, heap->heap_devt); + cdev_del(&heap->heap_cdev); + xa_erase(&dma_heap_minors, minor); + + kfree(heap); +} + +void dma_heap_put(struct dma_heap *h) +{ + /* + * Take the heap_list_lock now to avoid racing with code + * scanning the list and then taking a kref. + */ + mutex_lock(&heap_list_lock); + kref_put(&h->refcount, dma_heap_release); + mutex_unlock(&heap_list_lock); +} +EXPORT_SYMBOL_GPL(dma_heap_put); + +/** + * dma_heap_get_dev() - get device struct for the heap + * @heap: DMA-Heap to retrieve device struct from + * + * Returns: + * The device struct for the heap. + */ +struct device *dma_heap_get_dev(struct dma_heap *heap) +{ + return heap->heap_dev; +} +EXPORT_SYMBOL_GPL(dma_heap_get_dev); /** * dma_heap_get_name() - get heap name @@ -203,11 +293,11 @@ const char *dma_heap_get_name(struct dma_heap *heap) { return heap->name; } +EXPORT_SYMBOL_GPL(dma_heap_get_name); struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info) { - struct dma_heap *heap, *h, *err_ret; - struct device *dev_ret; + struct dma_heap *heap, *err_ret; unsigned int minor; int ret; @@ -221,10 +311,18 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info) return ERR_PTR(-EINVAL); } + heap = dma_heap_find(exp_info->name); + if (heap) { + pr_err("dma_heap: Already registered heap named %s\n", + exp_info->name); + dma_heap_put(heap); + return ERR_PTR(-EINVAL); + } heap = kzalloc(sizeof(*heap), GFP_KERNEL); if (!heap) return ERR_PTR(-ENOMEM); + kref_init(&heap->refcount); heap->name = exp_info->name; heap->ops = exp_info->ops; heap->priv = exp_info->priv; @@ -249,28 +347,20 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info) goto err1; } - dev_ret = device_create(dma_heap_class, - NULL, - heap->heap_devt, - NULL, - heap->name); - if (IS_ERR(dev_ret)) { + heap->heap_dev = device_create(dma_heap_class, + NULL, + heap->heap_devt, + NULL, + heap->name); + if (IS_ERR(heap->heap_dev)) { pr_err("dma_heap: Unable to create device\n"); - err_ret = ERR_CAST(dev_ret); + err_ret = ERR_CAST(heap->heap_dev); goto err2; } - mutex_lock(&heap_list_lock); - /* check the name is unique */ - list_for_each_entry(h, &heap_list, list) { - if (!strcmp(h->name, exp_info->name)) { - mutex_unlock(&heap_list_lock); - pr_err("dma_heap: Already registered heap named %s\n", - exp_info->name); - err_ret = ERR_PTR(-EINVAL); - goto err3; - } - } + /* Make sure it doesn't disappear on us */ + heap->heap_dev = get_device(heap->heap_dev); + /* Add heap to the list */ list_add(&heap->list, &heap_list); @@ -288,27 +378,88 @@ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info) kfree(heap); return err_ret; } +EXPORT_SYMBOL_GPL(dma_heap_add); static char *dma_heap_devnode(struct device *dev, umode_t *mode) { return kasprintf(GFP_KERNEL, "dma_heap/%s", dev_name(dev)); } +static ssize_t total_pools_kb_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct dma_heap *heap; + u64 total_pool_size = 0; + + mutex_lock(&heap_list_lock); + list_for_each_entry(heap, &heap_list, list) { + if (heap->ops->get_pool_size) + total_pool_size += heap->ops->get_pool_size(heap); + } + mutex_unlock(&heap_list_lock); + + return sysfs_emit(buf, "%llu\n", total_pool_size / 1024); +} + +static struct kobj_attribute total_pools_kb_attr = + __ATTR_RO(total_pools_kb); + +static struct attribute *dma_heap_sysfs_attrs[] = { + &total_pools_kb_attr.attr, + NULL, +}; + +ATTRIBUTE_GROUPS(dma_heap_sysfs); + +static struct kobject *dma_heap_kobject; + +static int dma_heap_sysfs_setup(void) +{ + int ret; + + dma_heap_kobject = kobject_create_and_add("dma_heap", kernel_kobj); + if (!dma_heap_kobject) + return -ENOMEM; + + ret = sysfs_create_groups(dma_heap_kobject, dma_heap_sysfs_groups); + if (ret) { + kobject_put(dma_heap_kobject); + return ret; + } + + return 0; +} + +static void dma_heap_sysfs_teardown(void) +{ + kobject_put(dma_heap_kobject); +} + static int dma_heap_init(void) { int ret; - ret = alloc_chrdev_region(&dma_heap_devt, 0, NUM_HEAP_MINORS, DEVNAME); + ret = dma_heap_sysfs_setup(); if (ret) return ret; + ret = alloc_chrdev_region(&dma_heap_devt, 0, NUM_HEAP_MINORS, DEVNAME); + if (ret) + goto err_chrdev; + dma_heap_class = class_create(THIS_MODULE, DEVNAME); if (IS_ERR(dma_heap_class)) { - unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS); - return PTR_ERR(dma_heap_class); + ret = PTR_ERR(dma_heap_class); + goto err_class; } dma_heap_class->devnode = dma_heap_devnode; return 0; + +err_class: + unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS); +err_chrdev: + dma_heap_sysfs_teardown(); + return ret; } subsys_initcall(dma_heap_init); diff --git a/drivers/dma-buf/heaps/Kconfig b/drivers/dma-buf/heaps/Kconfig index a5eef06c4226..ff52efa83f39 100644 --- a/drivers/dma-buf/heaps/Kconfig +++ b/drivers/dma-buf/heaps/Kconfig @@ -1,12 +1,22 @@ +menuconfig DMABUF_HEAPS_DEFERRED_FREE + bool "DMA-BUF heaps deferred-free library" + help + Choose this option to enable the DMA-BUF heaps deferred-free library. + +menuconfig DMABUF_HEAPS_PAGE_POOL + bool "DMA-BUF heaps page-pool library" + help + Choose this option to enable the DMA-BUF heaps page-pool library. + config DMABUF_HEAPS_SYSTEM - bool "DMA-BUF System Heap" - depends on DMABUF_HEAPS + tristate "DMA-BUF System Heap" + depends on DMABUF_HEAPS && DMABUF_HEAPS_DEFERRED_FREE && DMABUF_HEAPS_PAGE_POOL help Choose this option to enable the system dmabuf heap. The system heap is backed by pages from the buddy allocator. If in doubt, say Y. config DMABUF_HEAPS_CMA - bool "DMA-BUF CMA Heap" + tristate "DMA-BUF CMA Heap" depends on DMABUF_HEAPS && DMA_CMA help Choose this option to enable dma-buf CMA heap. This heap is backed diff --git a/drivers/dma-buf/heaps/Makefile b/drivers/dma-buf/heaps/Makefile index 6e54cdec3da0..4d4cd94a3a4a 100644 --- a/drivers/dma-buf/heaps/Makefile +++ b/drivers/dma-buf/heaps/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += heap-helpers.o +obj-$(CONFIG_DMABUF_HEAPS_DEFERRED_FREE) += deferred-free-helper.o +obj-$(CONFIG_DMABUF_HEAPS_PAGE_POOL) += page_pool.o obj-$(CONFIG_DMABUF_HEAPS_SYSTEM) += system_heap.o obj-$(CONFIG_DMABUF_HEAPS_CMA) += cma_heap.o diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index e55384dc115b..4931578df815 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -2,76 +2,304 @@ /* * DMABUF CMA heap exporter * - * Copyright (C) 2012, 2019 Linaro Ltd. + * Copyright (C) 2012, 2019, 2020 Linaro Ltd. * Author: for ST-Ericsson. + * + * Also utilizing parts of Andrew Davis' SRAM heap: + * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew F. Davis */ - #include -#include #include #include #include #include -#include #include +#include +#include #include -#include #include -#include +#include +#include -#include "heap-helpers.h" struct cma_heap { struct dma_heap *heap; struct cma *cma; }; -static void cma_heap_free(struct heap_helper_buffer *buffer) +struct cma_heap_buffer { + struct cma_heap *heap; + struct list_head attachments; + struct mutex lock; + unsigned long len; + struct page *cma_pages; + struct page **pages; + pgoff_t pagecount; + int vmap_cnt; + void *vaddr; +}; + +struct dma_heap_attachment { + struct device *dev; + struct sg_table table; + struct list_head list; + bool mapped; +}; + +static int cma_heap_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct cma_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + int ret; + + a = kzalloc(sizeof(*a), GFP_KERNEL); + if (!a) + return -ENOMEM; + + ret = sg_alloc_table_from_pages(&a->table, buffer->pages, + buffer->pagecount, 0, + buffer->pagecount << PAGE_SHIFT, + GFP_KERNEL); + if (ret) { + kfree(a); + return ret; + } + + a->dev = attachment->dev; + INIT_LIST_HEAD(&a->list); + a->mapped = false; + + attachment->priv = a; + + mutex_lock(&buffer->lock); + list_add(&a->list, &buffer->attachments); + mutex_unlock(&buffer->lock); + + return 0; +} + +static void cma_heap_detach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct cma_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a = attachment->priv; + + mutex_lock(&buffer->lock); + list_del(&a->list); + mutex_unlock(&buffer->lock); + + sg_free_table(&a->table); + kfree(a); +} + +static struct sg_table *cma_heap_map_dma_buf(struct dma_buf_attachment *attachment, + enum dma_data_direction direction) { - struct cma_heap *cma_heap = dma_heap_get_drvdata(buffer->heap); - unsigned long nr_pages = buffer->pagecount; - struct page *cma_pages = buffer->priv_virt; + struct dma_heap_attachment *a = attachment->priv; + struct sg_table *table = &a->table; + int ret; + + ret = dma_map_sgtable(attachment->dev, table, direction, 0); + if (ret) + return ERR_PTR(-ENOMEM); + a->mapped = true; + return table; +} + +static void cma_heap_unmap_dma_buf(struct dma_buf_attachment *attachment, + struct sg_table *table, + enum dma_data_direction direction) +{ + struct dma_heap_attachment *a = attachment->priv; + + a->mapped = false; + dma_unmap_sgtable(attachment->dev, table, direction, 0); +} + +static int cma_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct cma_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + + if (buffer->vmap_cnt) + invalidate_kernel_vmap_range(buffer->vaddr, buffer->len); + + mutex_lock(&buffer->lock); + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + dma_sync_sgtable_for_cpu(a->dev, &a->table, direction); + } + mutex_unlock(&buffer->lock); + + return 0; +} + +static int cma_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct cma_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + + if (buffer->vmap_cnt) + flush_kernel_vmap_range(buffer->vaddr, buffer->len); + + mutex_lock(&buffer->lock); + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + dma_sync_sgtable_for_device(a->dev, &a->table, direction); + } + mutex_unlock(&buffer->lock); + + return 0; +} + +static vm_fault_t cma_heap_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct cma_heap_buffer *buffer = vma->vm_private_data; + + if (vmf->pgoff > buffer->pagecount) + return VM_FAULT_SIGBUS; + + vmf->page = buffer->pages[vmf->pgoff]; + get_page(vmf->page); + + return 0; +} + +static const struct vm_operations_struct dma_heap_vm_ops = { + .fault = cma_heap_vm_fault, +}; + +static int cma_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct cma_heap_buffer *buffer = dmabuf->priv; + + if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) + return -EINVAL; + + vma->vm_ops = &dma_heap_vm_ops; + vma->vm_private_data = buffer; + + return 0; +} + +static void *cma_heap_do_vmap(struct cma_heap_buffer *buffer) +{ + void *vaddr; + + vaddr = vmap(buffer->pages, buffer->pagecount, VM_MAP, PAGE_KERNEL); + if (!vaddr) + return ERR_PTR(-ENOMEM); + + return vaddr; +} + +static void *cma_heap_vmap(struct dma_buf *dmabuf) +{ + struct cma_heap_buffer *buffer = dmabuf->priv; + void *vaddr; + + mutex_lock(&buffer->lock); + if (buffer->vmap_cnt) { + buffer->vmap_cnt++; + vaddr = buffer->vaddr; + goto out; + } + + vaddr = cma_heap_do_vmap(buffer); + if (IS_ERR(vaddr)) + goto out; + + buffer->vaddr = vaddr; + buffer->vmap_cnt++; +out: + mutex_unlock(&buffer->lock); + + return vaddr; +} + +static void cma_heap_vunmap(struct dma_buf *dmabuf, void *vaddr) +{ + struct cma_heap_buffer *buffer = dmabuf->priv; + + mutex_lock(&buffer->lock); + if (!--buffer->vmap_cnt) { + vunmap(buffer->vaddr); + buffer->vaddr = NULL; + } + mutex_unlock(&buffer->lock); +} + +static void cma_heap_dma_buf_release(struct dma_buf *dmabuf) +{ + struct cma_heap_buffer *buffer = dmabuf->priv; + struct cma_heap *cma_heap = buffer->heap; + + if (buffer->vmap_cnt > 0) { + WARN(1, "%s: buffer still mapped in the kernel\n", __func__); + vunmap(buffer->vaddr); + } /* free page list */ kfree(buffer->pages); /* release memory */ - cma_release(cma_heap->cma, cma_pages, nr_pages); + cma_release(cma_heap->cma, buffer->cma_pages, buffer->pagecount); kfree(buffer); } -/* dmabuf heap CMA operations functions */ -static int cma_heap_allocate(struct dma_heap *heap, - unsigned long len, - unsigned long fd_flags, - unsigned long heap_flags) +static const struct dma_buf_ops cma_heap_buf_ops = { + .attach = cma_heap_attach, + .detach = cma_heap_detach, + .map_dma_buf = cma_heap_map_dma_buf, + .unmap_dma_buf = cma_heap_unmap_dma_buf, + .begin_cpu_access = cma_heap_dma_buf_begin_cpu_access, + .end_cpu_access = cma_heap_dma_buf_end_cpu_access, + .mmap = cma_heap_mmap, + .vmap = cma_heap_vmap, + .vunmap = cma_heap_vunmap, + .release = cma_heap_dma_buf_release, +}; + +static struct dma_buf *cma_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) { struct cma_heap *cma_heap = dma_heap_get_drvdata(heap); - struct heap_helper_buffer *helper_buffer; - struct page *cma_pages; + struct cma_heap_buffer *buffer; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); size_t size = PAGE_ALIGN(len); - unsigned long nr_pages = size >> PAGE_SHIFT; + pgoff_t pagecount = size >> PAGE_SHIFT; unsigned long align = get_order(size); + struct page *cma_pages; struct dma_buf *dmabuf; int ret = -ENOMEM; pgoff_t pg; - if (align > CONFIG_CMA_ALIGNMENT) - align = CONFIG_CMA_ALIGNMENT; + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) + return ERR_PTR(-ENOMEM); - helper_buffer = kzalloc(sizeof(*helper_buffer), GFP_KERNEL); - if (!helper_buffer) - return -ENOMEM; + INIT_LIST_HEAD(&buffer->attachments); + mutex_init(&buffer->lock); + buffer->len = size; - init_heap_helper_buffer(helper_buffer, cma_heap_free); - helper_buffer->heap = heap; - helper_buffer->size = len; + if (align > CONFIG_CMA_ALIGNMENT) + align = CONFIG_CMA_ALIGNMENT; - cma_pages = cma_alloc(cma_heap->cma, nr_pages, align, false); + cma_pages = cma_alloc(cma_heap->cma, pagecount, align, GFP_KERNEL); if (!cma_pages) - goto free_buf; + goto free_buffer; + /* Clear the cma pages */ if (PageHighMem(cma_pages)) { - unsigned long nr_clear_pages = nr_pages; + unsigned long nr_clear_pages = pagecount; struct page *page = cma_pages; while (nr_clear_pages > 0) { @@ -85,7 +313,6 @@ static int cma_heap_allocate(struct dma_heap *heap, */ if (fatal_signal_pending(current)) goto free_cma; - page++; nr_clear_pages--; } @@ -93,44 +320,41 @@ static int cma_heap_allocate(struct dma_heap *heap, memset(page_address(cma_pages), 0, size); } - helper_buffer->pagecount = nr_pages; - helper_buffer->pages = kmalloc_array(helper_buffer->pagecount, - sizeof(*helper_buffer->pages), - GFP_KERNEL); - if (!helper_buffer->pages) { + buffer->pages = kmalloc_array(pagecount, sizeof(*buffer->pages), GFP_KERNEL); + if (!buffer->pages) { ret = -ENOMEM; goto free_cma; } - for (pg = 0; pg < helper_buffer->pagecount; pg++) - helper_buffer->pages[pg] = &cma_pages[pg]; + for (pg = 0; pg < pagecount; pg++) + buffer->pages[pg] = &cma_pages[pg]; + + buffer->cma_pages = cma_pages; + buffer->heap = cma_heap; + buffer->pagecount = pagecount; /* create the dmabuf */ - dmabuf = heap_helper_export_dmabuf(helper_buffer, fd_flags); + exp_info.exp_name = dma_heap_get_name(heap); + exp_info.ops = &cma_heap_buf_ops; + exp_info.size = buffer->len; + exp_info.flags = fd_flags; + exp_info.priv = buffer; + dmabuf = dma_buf_export(&exp_info); if (IS_ERR(dmabuf)) { ret = PTR_ERR(dmabuf); goto free_pages; } - helper_buffer->dmabuf = dmabuf; - helper_buffer->priv_virt = cma_pages; - - ret = dma_buf_fd(dmabuf, fd_flags); - if (ret < 0) { - dma_buf_put(dmabuf); - /* just return, as put will call release and that will free */ - return ret; - } - - return ret; + return dmabuf; free_pages: - kfree(helper_buffer->pages); + kfree(buffer->pages); free_cma: - cma_release(cma_heap->cma, cma_pages, nr_pages); -free_buf: - kfree(helper_buffer); - return ret; + cma_release(cma_heap->cma, cma_pages, pagecount); +free_buffer: + kfree(buffer); + + return ERR_PTR(ret); } static const struct dma_heap_ops cma_heap_ops = { diff --git a/drivers/dma-buf/heaps/deferred-free-helper.c b/drivers/dma-buf/heaps/deferred-free-helper.c new file mode 100755 index 000000000000..e19c8b68dfeb --- /dev/null +++ b/drivers/dma-buf/heaps/deferred-free-helper.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Deferred dmabuf freeing helper + * + * Copyright (C) 2020 Linaro, Ltd. + * + * Based on the ION page pool code + * Copyright (C) 2011 Google, Inc. + */ + +#include +#include +#include +#include +#include + +#include "deferred-free-helper.h" + +static LIST_HEAD(free_list); +static size_t list_nr_pages; +wait_queue_head_t freelist_waitqueue; +struct task_struct *freelist_task; +static DEFINE_SPINLOCK(free_list_lock); + +void deferred_free(struct deferred_freelist_item *item, + void (*free)(struct deferred_freelist_item*, + enum df_reason), + size_t nr_pages) +{ + unsigned long flags; + + INIT_LIST_HEAD(&item->list); + item->nr_pages = nr_pages; + item->free = free; + + spin_lock_irqsave(&free_list_lock, flags); + list_add(&item->list, &free_list); + list_nr_pages += nr_pages; + spin_unlock_irqrestore(&free_list_lock, flags); + wake_up(&freelist_waitqueue); +} +EXPORT_SYMBOL_GPL(deferred_free); + +static size_t free_one_item(enum df_reason reason) +{ + unsigned long flags; + size_t nr_pages; + struct deferred_freelist_item *item; + + spin_lock_irqsave(&free_list_lock, flags); + if (list_empty(&free_list)) { + spin_unlock_irqrestore(&free_list_lock, flags); + return 0; + } + item = list_first_entry(&free_list, struct deferred_freelist_item, list); + list_del(&item->list); + nr_pages = item->nr_pages; + list_nr_pages -= nr_pages; + spin_unlock_irqrestore(&free_list_lock, flags); + + item->free(item, reason); + return nr_pages; +} + +static unsigned long get_freelist_nr_pages(void) +{ + unsigned long nr_pages; + unsigned long flags; + + spin_lock_irqsave(&free_list_lock, flags); + nr_pages = list_nr_pages; + spin_unlock_irqrestore(&free_list_lock, flags); + return nr_pages; +} + +static unsigned long freelist_shrink_count(struct shrinker *shrinker, + struct shrink_control *sc) +{ + return get_freelist_nr_pages(); +} + +static unsigned long freelist_shrink_scan(struct shrinker *shrinker, + struct shrink_control *sc) +{ + unsigned long total_freed = 0; + + if (sc->nr_to_scan == 0) + return 0; + + while (total_freed < sc->nr_to_scan) { + size_t pages_freed = free_one_item(DF_UNDER_PRESSURE); + + if (!pages_freed) + break; + + total_freed += pages_freed; + } + + return total_freed; +} + +static struct shrinker freelist_shrinker = { + .count_objects = freelist_shrink_count, + .scan_objects = freelist_shrink_scan, + .seeks = DEFAULT_SEEKS, + .batch = 0, +}; + +static int deferred_free_thread(void *data) +{ + while (true) { + wait_event_freezable(freelist_waitqueue, + get_freelist_nr_pages() > 0); + + free_one_item(DF_NORMAL); + } + + return 0; +} + +static int deferred_freelist_init(void) +{ + list_nr_pages = 0; + + init_waitqueue_head(&freelist_waitqueue); + freelist_task = kthread_run(deferred_free_thread, NULL, + "%s", "dmabuf-deferred-free-worker"); + if (IS_ERR(freelist_task)) { + pr_err("Creating thread for deferred free failed\n"); + return -1; + } + sched_set_normal(freelist_task, 19); + + return register_shrinker(&freelist_shrinker); +} +module_init(deferred_freelist_init); +MODULE_LICENSE("GPL v2"); + diff --git a/drivers/dma-buf/heaps/deferred-free-helper.h b/drivers/dma-buf/heaps/deferred-free-helper.h new file mode 100755 index 000000000000..11940328ce3f --- /dev/null +++ b/drivers/dma-buf/heaps/deferred-free-helper.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef DEFERRED_FREE_HELPER_H +#define DEFERRED_FREE_HELPER_H + +/** + * df_reason - enum for reason why item was freed + * + * This provides a reason for why the free function was called + * on the item. This is useful when deferred_free is used in + * combination with a pagepool, so under pressure the page can + * be immediately freed. + * + * DF_NORMAL: Normal deferred free + * + * DF_UNDER_PRESSURE: Free was called because the system + * is under memory pressure. Usually + * from a shrinker. Avoid allocating + * memory in the free call, as it may + * fail. + */ +enum df_reason { + DF_NORMAL, + DF_UNDER_PRESSURE, +}; + +/** + * deferred_freelist_item - item structure for deferred freelist + * + * This is to be added to the structure for whatever you want to + * defer freeing on. + * + * @nr_pages: number of pages used by item to be freed + * @free: function pointer to be called when freeing the item + * @list: list entry for the deferred list + */ +struct deferred_freelist_item { + size_t nr_pages; + void (*free)(struct deferred_freelist_item *i, + enum df_reason reason); + struct list_head list; +}; + +/** + * deferred_free - call to add item to the deferred free list + * + * @item: Pointer to deferred_freelist_item field of a structure + * @free: Function pointer to the free call + * @nr_pages: number of pages to be freed + */ +void deferred_free(struct deferred_freelist_item *item, + void (*free)(struct deferred_freelist_item *i, + enum df_reason reason), + size_t nr_pages); +#endif diff --git a/drivers/dma-buf/heaps/heap-helpers.c b/drivers/dma-buf/heaps/heap-helpers.c deleted file mode 100644 index 35aa65bbf0a7..000000000000 --- a/drivers/dma-buf/heaps/heap-helpers.c +++ /dev/null @@ -1,271 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "heap-helpers.h" - -void init_heap_helper_buffer(struct heap_helper_buffer *buffer, - void (*free)(struct heap_helper_buffer *)) -{ - buffer->priv_virt = NULL; - mutex_init(&buffer->lock); - buffer->vmap_cnt = 0; - buffer->vaddr = NULL; - buffer->pagecount = 0; - buffer->pages = NULL; - INIT_LIST_HEAD(&buffer->attachments); - buffer->free = free; -} - -struct dma_buf *heap_helper_export_dmabuf(struct heap_helper_buffer *buffer, - int fd_flags) -{ - DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - - exp_info.exp_name = dma_heap_get_name(buffer->heap); - exp_info.ops = &heap_helper_ops; - exp_info.size = buffer->size; - exp_info.flags = fd_flags; - exp_info.priv = buffer; - - return dma_buf_export(&exp_info); -} - -static void *dma_heap_map_kernel(struct heap_helper_buffer *buffer) -{ - void *vaddr; - - vaddr = vmap(buffer->pages, buffer->pagecount, VM_MAP, PAGE_KERNEL); - if (!vaddr) - return ERR_PTR(-ENOMEM); - - return vaddr; -} - -static void dma_heap_buffer_destroy(struct heap_helper_buffer *buffer) -{ - if (buffer->vmap_cnt > 0) { - WARN(1, "%s: buffer still mapped in the kernel\n", __func__); - vunmap(buffer->vaddr); - } - - buffer->free(buffer); -} - -static void *dma_heap_buffer_vmap_get(struct heap_helper_buffer *buffer) -{ - void *vaddr; - - if (buffer->vmap_cnt) { - buffer->vmap_cnt++; - return buffer->vaddr; - } - vaddr = dma_heap_map_kernel(buffer); - if (IS_ERR(vaddr)) - return vaddr; - buffer->vaddr = vaddr; - buffer->vmap_cnt++; - return vaddr; -} - -static void dma_heap_buffer_vmap_put(struct heap_helper_buffer *buffer) -{ - if (!--buffer->vmap_cnt) { - vunmap(buffer->vaddr); - buffer->vaddr = NULL; - } -} - -struct dma_heaps_attachment { - struct device *dev; - struct sg_table table; - struct list_head list; -}; - -static int dma_heap_attach(struct dma_buf *dmabuf, - struct dma_buf_attachment *attachment) -{ - struct dma_heaps_attachment *a; - struct heap_helper_buffer *buffer = dmabuf->priv; - int ret; - - a = kzalloc(sizeof(*a), GFP_KERNEL); - if (!a) - return -ENOMEM; - - ret = sg_alloc_table_from_pages(&a->table, buffer->pages, - buffer->pagecount, 0, - buffer->pagecount << PAGE_SHIFT, - GFP_KERNEL); - if (ret) { - kfree(a); - return ret; - } - - a->dev = attachment->dev; - INIT_LIST_HEAD(&a->list); - - attachment->priv = a; - - mutex_lock(&buffer->lock); - list_add(&a->list, &buffer->attachments); - mutex_unlock(&buffer->lock); - - return 0; -} - -static void dma_heap_detach(struct dma_buf *dmabuf, - struct dma_buf_attachment *attachment) -{ - struct dma_heaps_attachment *a = attachment->priv; - struct heap_helper_buffer *buffer = dmabuf->priv; - - mutex_lock(&buffer->lock); - list_del(&a->list); - mutex_unlock(&buffer->lock); - - sg_free_table(&a->table); - kfree(a); -} - -static -struct sg_table *dma_heap_map_dma_buf(struct dma_buf_attachment *attachment, - enum dma_data_direction direction) -{ - struct dma_heaps_attachment *a = attachment->priv; - struct sg_table *table = &a->table; - int ret; - - ret = dma_map_sgtable(attachment->dev, table, direction, 0); - if (ret) - table = ERR_PTR(ret); - return table; -} - -static void dma_heap_unmap_dma_buf(struct dma_buf_attachment *attachment, - struct sg_table *table, - enum dma_data_direction direction) -{ - dma_unmap_sgtable(attachment->dev, table, direction, 0); -} - -static vm_fault_t dma_heap_vm_fault(struct vm_fault *vmf) -{ - struct vm_area_struct *vma = vmf->vma; - struct heap_helper_buffer *buffer = vma->vm_private_data; - - if (vmf->pgoff > buffer->pagecount) - return VM_FAULT_SIGBUS; - - vmf->page = buffer->pages[vmf->pgoff]; - get_page(vmf->page); - - return 0; -} - -static const struct vm_operations_struct dma_heap_vm_ops = { - .fault = dma_heap_vm_fault, -}; - -static int dma_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) -{ - struct heap_helper_buffer *buffer = dmabuf->priv; - - if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) - return -EINVAL; - - vma->vm_ops = &dma_heap_vm_ops; - vma->vm_private_data = buffer; - - return 0; -} - -static void dma_heap_dma_buf_release(struct dma_buf *dmabuf) -{ - struct heap_helper_buffer *buffer = dmabuf->priv; - - dma_heap_buffer_destroy(buffer); -} - -static int dma_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, - enum dma_data_direction direction) -{ - struct heap_helper_buffer *buffer = dmabuf->priv; - struct dma_heaps_attachment *a; - int ret = 0; - - mutex_lock(&buffer->lock); - - if (buffer->vmap_cnt) - invalidate_kernel_vmap_range(buffer->vaddr, buffer->size); - - list_for_each_entry(a, &buffer->attachments, list) { - dma_sync_sg_for_cpu(a->dev, a->table.sgl, a->table.nents, - direction); - } - mutex_unlock(&buffer->lock); - - return ret; -} - -static int dma_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, - enum dma_data_direction direction) -{ - struct heap_helper_buffer *buffer = dmabuf->priv; - struct dma_heaps_attachment *a; - - mutex_lock(&buffer->lock); - - if (buffer->vmap_cnt) - flush_kernel_vmap_range(buffer->vaddr, buffer->size); - - list_for_each_entry(a, &buffer->attachments, list) { - dma_sync_sg_for_device(a->dev, a->table.sgl, a->table.nents, - direction); - } - mutex_unlock(&buffer->lock); - - return 0; -} - -static void *dma_heap_dma_buf_vmap(struct dma_buf *dmabuf) -{ - struct heap_helper_buffer *buffer = dmabuf->priv; - void *vaddr; - - mutex_lock(&buffer->lock); - vaddr = dma_heap_buffer_vmap_get(buffer); - mutex_unlock(&buffer->lock); - - return vaddr; -} - -static void dma_heap_dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) -{ - struct heap_helper_buffer *buffer = dmabuf->priv; - - mutex_lock(&buffer->lock); - dma_heap_buffer_vmap_put(buffer); - mutex_unlock(&buffer->lock); -} - -const struct dma_buf_ops heap_helper_ops = { - .map_dma_buf = dma_heap_map_dma_buf, - .unmap_dma_buf = dma_heap_unmap_dma_buf, - .mmap = dma_heap_mmap, - .release = dma_heap_dma_buf_release, - .attach = dma_heap_attach, - .detach = dma_heap_detach, - .begin_cpu_access = dma_heap_dma_buf_begin_cpu_access, - .end_cpu_access = dma_heap_dma_buf_end_cpu_access, - .vmap = dma_heap_dma_buf_vmap, - .vunmap = dma_heap_dma_buf_vunmap, -}; diff --git a/drivers/dma-buf/heaps/heap-helpers.h b/drivers/dma-buf/heaps/heap-helpers.h deleted file mode 100644 index 805d2df88024..000000000000 --- a/drivers/dma-buf/heaps/heap-helpers.h +++ /dev/null @@ -1,53 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * DMABUF Heaps helper code - * - * Copyright (C) 2011 Google, Inc. - * Copyright (C) 2019 Linaro Ltd. - */ - -#ifndef _HEAP_HELPERS_H -#define _HEAP_HELPERS_H - -#include -#include - -/** - * struct heap_helper_buffer - helper buffer metadata - * @heap: back pointer to the heap the buffer came from - * @dmabuf: backing dma-buf for this buffer - * @size: size of the buffer - * @priv_virt pointer to heap specific private value - * @lock mutext to protect the data in this structure - * @vmap_cnt count of vmap references on the buffer - * @vaddr vmap'ed virtual address - * @pagecount number of pages in the buffer - * @pages list of page pointers - * @attachments list of device attachments - * - * @free heap callback to free the buffer - */ -struct heap_helper_buffer { - struct dma_heap *heap; - struct dma_buf *dmabuf; - size_t size; - - void *priv_virt; - struct mutex lock; - int vmap_cnt; - void *vaddr; - pgoff_t pagecount; - struct page **pages; - struct list_head attachments; - - void (*free)(struct heap_helper_buffer *buffer); -}; - -void init_heap_helper_buffer(struct heap_helper_buffer *buffer, - void (*free)(struct heap_helper_buffer *)); - -struct dma_buf *heap_helper_export_dmabuf(struct heap_helper_buffer *buffer, - int fd_flags); - -extern const struct dma_buf_ops heap_helper_ops; -#endif /* _HEAP_HELPERS_H */ diff --git a/drivers/dma-buf/heaps/page_pool.c b/drivers/dma-buf/heaps/page_pool.c new file mode 100755 index 000000000000..7c34a1ba48bb --- /dev/null +++ b/drivers/dma-buf/heaps/page_pool.c @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DMA BUF page pool system + * + * Copyright (C) 2020 Linaro Ltd. + * + * Based on the ION page pool code + * Copyright (C) 2011 Google, Inc. + */ + +#include +#include +#include +#include +#include +#include "page_pool.h" + +static LIST_HEAD(pool_list); +static DEFINE_MUTEX(pool_list_lock); + +static inline +struct page *dmabuf_page_pool_alloc_pages(struct dmabuf_page_pool *pool) +{ + if (fatal_signal_pending(current)) + return NULL; + return alloc_pages(pool->gfp_mask, pool->order); +} + +static inline void dmabuf_page_pool_free_pages(struct dmabuf_page_pool *pool, + struct page *page) +{ + __free_pages(page, pool->order); +} + +static void dmabuf_page_pool_add(struct dmabuf_page_pool *pool, struct page *page) +{ + int index; + + if (PageHighMem(page)) + index = POOL_HIGHPAGE; + else + index = POOL_LOWPAGE; + + mutex_lock(&pool->mutex); + list_add_tail(&page->lru, &pool->items[index]); + pool->count[index]++; + mutex_unlock(&pool->mutex); + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + 1 << pool->order); +} + +static struct page *dmabuf_page_pool_remove(struct dmabuf_page_pool *pool, int index) +{ + struct page *page; + + mutex_lock(&pool->mutex); + page = list_first_entry_or_null(&pool->items[index], struct page, lru); + if (page) { + pool->count[index]--; + list_del(&page->lru); + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + -(1 << pool->order)); + } + mutex_unlock(&pool->mutex); + + return page; +} + +static struct page *dmabuf_page_pool_fetch(struct dmabuf_page_pool *pool) +{ + struct page *page = NULL; + + page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE); + if (!page) + page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE); + + return page; +} + +struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool) +{ + struct page *page = NULL; + + if (WARN_ON(!pool)) + return NULL; + + page = dmabuf_page_pool_fetch(pool); + + if (!page) + page = dmabuf_page_pool_alloc_pages(pool); + return page; +} +EXPORT_SYMBOL_GPL(dmabuf_page_pool_alloc); + +void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page) +{ + if (WARN_ON(pool->order != compound_order(page))) + return; + + dmabuf_page_pool_add(pool, page); +} +EXPORT_SYMBOL_GPL(dmabuf_page_pool_free); + +static int dmabuf_page_pool_total(struct dmabuf_page_pool *pool, bool high) +{ + int count = pool->count[POOL_LOWPAGE]; + + if (high) + count += pool->count[POOL_HIGHPAGE]; + + return count << pool->order; +} + +struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask, unsigned int order) +{ + struct dmabuf_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); + int i; + + if (!pool) + return NULL; + + for (i = 0; i < POOL_TYPE_SIZE; i++) { + pool->count[i] = 0; + INIT_LIST_HEAD(&pool->items[i]); + } + pool->gfp_mask = gfp_mask | __GFP_COMP; + pool->order = order; + mutex_init(&pool->mutex); + + mutex_lock(&pool_list_lock); + list_add(&pool->list, &pool_list); + mutex_unlock(&pool_list_lock); + + return pool; +} +EXPORT_SYMBOL_GPL(dmabuf_page_pool_create); + +void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool) +{ + struct page *page; + int i; + + /* Remove us from the pool list */ + mutex_lock(&pool_list_lock); + list_del(&pool->list); + mutex_unlock(&pool_list_lock); + + /* Free any remaining pages in the pool */ + for (i = 0; i < POOL_TYPE_SIZE; i++) { + while ((page = dmabuf_page_pool_remove(pool, i))) + dmabuf_page_pool_free_pages(pool, page); + } + + kfree(pool); +} +EXPORT_SYMBOL_GPL(dmabuf_page_pool_destroy); + +static int dmabuf_page_pool_do_shrink(struct dmabuf_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan) +{ + int freed = 0; + bool high; + + if (current_is_kswapd()) + high = true; + else + high = !!(gfp_mask & __GFP_HIGHMEM); + + if (nr_to_scan == 0) + return dmabuf_page_pool_total(pool, high); + + while (freed < nr_to_scan) { + struct page *page; + + /* Try to free low pages first */ + page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE); + if (!page) + page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE); + + if (!page) + break; + + dmabuf_page_pool_free_pages(pool, page); + freed += (1 << pool->order); + } + + return freed; +} + +static int dmabuf_page_pool_shrink(gfp_t gfp_mask, int nr_to_scan) +{ + struct dmabuf_page_pool *pool; + int nr_total = 0; + int nr_freed; + int only_scan = 0; + + if (!nr_to_scan) + only_scan = 1; + + mutex_lock(&pool_list_lock); + list_for_each_entry(pool, &pool_list, list) { + if (only_scan) { + nr_total += dmabuf_page_pool_do_shrink(pool, + gfp_mask, + nr_to_scan); + } else { + nr_freed = dmabuf_page_pool_do_shrink(pool, + gfp_mask, + nr_to_scan); + nr_to_scan -= nr_freed; + nr_total += nr_freed; + if (nr_to_scan <= 0) + break; + } + } + mutex_unlock(&pool_list_lock); + + return nr_total; +} + +static unsigned long dmabuf_page_pool_shrink_count(struct shrinker *shrinker, + struct shrink_control *sc) +{ + return dmabuf_page_pool_shrink(sc->gfp_mask, 0); +} + +static unsigned long dmabuf_page_pool_shrink_scan(struct shrinker *shrinker, + struct shrink_control *sc) +{ + if (sc->nr_to_scan == 0) + return 0; + return dmabuf_page_pool_shrink(sc->gfp_mask, sc->nr_to_scan); +} + +struct shrinker pool_shrinker = { + .count_objects = dmabuf_page_pool_shrink_count, + .scan_objects = dmabuf_page_pool_shrink_scan, + .seeks = DEFAULT_SEEKS, + .batch = 0, +}; + +static int dmabuf_page_pool_init_shrinker(void) +{ + return register_shrinker(&pool_shrinker); +} +module_init(dmabuf_page_pool_init_shrinker); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma-buf/heaps/page_pool.h b/drivers/dma-buf/heaps/page_pool.h new file mode 100755 index 000000000000..6b083b04f195 --- /dev/null +++ b/drivers/dma-buf/heaps/page_pool.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * DMA BUF PagePool implementation + * Based on earlier ION code by Google + * + * Copyright (C) 2011 Google, Inc. + * Copyright (C) 2020 Linaro Ltd. + */ + +#ifndef _DMABUF_PAGE_POOL_H +#define _DMABUF_PAGE_POOL_H + +#include +#include +#include +#include +#include +#include + +/* page types we track in the pool */ +enum { + POOL_LOWPAGE, /* Clean lowmem pages */ + POOL_HIGHPAGE, /* Clean highmem pages */ + + POOL_TYPE_SIZE, +}; + +/** + * struct dmabuf_page_pool - pagepool struct + * @count[]: array of number of pages of that type in the pool + * @items[]: array of list of pages of the specific type + * @mutex: lock protecting this struct and especially the count + * item list + * @gfp_mask: gfp_mask to use from alloc + * @order: order of pages in the pool + * @list: list node for list of pools + * + * Allows you to keep a pool of pre allocated pages to use + */ +struct dmabuf_page_pool { + int count[POOL_TYPE_SIZE]; + struct list_head items[POOL_TYPE_SIZE]; + struct mutex mutex; + gfp_t gfp_mask; + unsigned int order; + struct list_head list; +}; + +struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask, + unsigned int order); +void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool); +struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool); +void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page); + +#endif /* _DMABUF_PAGE_POOL_H */ diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index 0bf688e3c023..15796bc4c033 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -3,7 +3,11 @@ * DMABUF System heap exporter * * Copyright (C) 2011 Google, Inc. - * Copyright (C) 2019 Linaro Ltd. + * Copyright (C) 2019, 2020 Linaro Ltd. + * + * Portions based off of Andrew Davis' SRAM heap: + * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew F. Davis */ #include @@ -15,99 +19,546 @@ #include #include #include -#include -#include +#include + +#include "page_pool.h" +#include "deferred-free-helper.h" + +static struct dma_heap *sys_heap; +static struct dma_heap *sys_uncached_heap; + +struct system_heap_buffer { + struct dma_heap *heap; + struct list_head attachments; + struct mutex lock; + unsigned long len; + struct sg_table sg_table; + int vmap_cnt; + void *vaddr; + struct deferred_freelist_item deferred_free; + + bool uncached; +}; + +struct dma_heap_attachment { + struct device *dev; + struct sg_table *table; + struct list_head list; + bool mapped; -#include "heap-helpers.h" + bool uncached; +}; -struct dma_heap *sys_heap; +#define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \ + | __GFP_NORETRY) & ~__GFP_RECLAIM) \ + | __GFP_COMP) +#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP) +static gfp_t order_flags[] = {HIGH_ORDER_GFP, LOW_ORDER_GFP, LOW_ORDER_GFP}; +/* + * The selection of the orders used for allocation (1MB, 64K, 4K) is designed + * to match with the sizes often found in IOMMUs. Using order 4 pages instead + * of order 0 pages can significantly improve the performance of many IOMMUs + * by reducing TLB pressure and time spent updating page tables. + */ +static const unsigned int orders[] = {8, 4, 0}; +#define NUM_ORDERS ARRAY_SIZE(orders) +struct dmabuf_page_pool *pools[NUM_ORDERS]; -static void system_heap_free(struct heap_helper_buffer *buffer) +static struct sg_table *dup_sg_table(struct sg_table *table) { - pgoff_t pg; + struct sg_table *new_table; + int ret, i; + struct scatterlist *sg, *new_sg; - for (pg = 0; pg < buffer->pagecount; pg++) - __free_page(buffer->pages[pg]); - kfree(buffer->pages); - kfree(buffer); + new_table = kzalloc(sizeof(*new_table), GFP_KERNEL); + if (!new_table) + return ERR_PTR(-ENOMEM); + + ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL); + if (ret) { + kfree(new_table); + return ERR_PTR(-ENOMEM); + } + + new_sg = new_table->sgl; + for_each_sgtable_sg(table, sg, i) { + sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset); + new_sg = sg_next(new_sg); + } + + return new_table; } -static int system_heap_allocate(struct dma_heap *heap, - unsigned long len, - unsigned long fd_flags, - unsigned long heap_flags) +static int system_heap_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) { - struct heap_helper_buffer *helper_buffer; - struct dma_buf *dmabuf; - int ret = -ENOMEM; - pgoff_t pg; + struct system_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + struct sg_table *table; + + a = kzalloc(sizeof(*a), GFP_KERNEL); + if (!a) + return -ENOMEM; - helper_buffer = kzalloc(sizeof(*helper_buffer), GFP_KERNEL); - if (!helper_buffer) + table = dup_sg_table(&buffer->sg_table); + if (IS_ERR(table)) { + kfree(a); return -ENOMEM; + } + + a->table = table; + a->dev = attachment->dev; + INIT_LIST_HEAD(&a->list); + a->mapped = false; + a->uncached = buffer->uncached; + attachment->priv = a; + + mutex_lock(&buffer->lock); + list_add(&a->list, &buffer->attachments); + mutex_unlock(&buffer->lock); + + return 0; +} + +static void system_heap_detach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a = attachment->priv; + + mutex_lock(&buffer->lock); + list_del(&a->list); + mutex_unlock(&buffer->lock); + + sg_free_table(a->table); + kfree(a->table); + kfree(a); +} - init_heap_helper_buffer(helper_buffer, system_heap_free); - helper_buffer->heap = heap; - helper_buffer->size = len; +static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment, + enum dma_data_direction direction) +{ + struct dma_heap_attachment *a = attachment->priv; + struct sg_table *table = a->table; + int attr = 0; + int ret; + + if (a->uncached) + attr = DMA_ATTR_SKIP_CPU_SYNC; + + ret = dma_map_sgtable(attachment->dev, table, direction, attr); + if (ret) + return ERR_PTR(ret); + + a->mapped = true; + return table; +} + +static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment, + struct sg_table *table, + enum dma_data_direction direction) +{ + struct dma_heap_attachment *a = attachment->priv; + int attr = 0; + + if (a->uncached) + attr = DMA_ATTR_SKIP_CPU_SYNC; + a->mapped = false; + dma_unmap_sgtable(attachment->dev, table, direction, attr); +} - helper_buffer->pagecount = len / PAGE_SIZE; - helper_buffer->pages = kmalloc_array(helper_buffer->pagecount, - sizeof(*helper_buffer->pages), - GFP_KERNEL); - if (!helper_buffer->pages) { - ret = -ENOMEM; - goto err0; +static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + + mutex_lock(&buffer->lock); + + if (buffer->vmap_cnt) + invalidate_kernel_vmap_range(buffer->vaddr, buffer->len); + + if (!buffer->uncached) { + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + dma_sync_sgtable_for_cpu(a->dev, a->table, direction); + } } + mutex_unlock(&buffer->lock); + + return 0; +} + +static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + + mutex_lock(&buffer->lock); + + if (buffer->vmap_cnt) + flush_kernel_vmap_range(buffer->vaddr, buffer->len); + + if (!buffer->uncached) { + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + dma_sync_sgtable_for_device(a->dev, a->table, direction); + } + } + mutex_unlock(&buffer->lock); + + return 0; +} + +static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + struct sg_table *table = &buffer->sg_table; + unsigned long addr = vma->vm_start; + struct sg_page_iter piter; + int ret; + + if (buffer->uncached) + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + + for_each_sgtable_page(table, &piter, vma->vm_pgoff) { + struct page *page = sg_page_iter_page(&piter); + + ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE, + vma->vm_page_prot); + if (ret) + return ret; + addr += PAGE_SIZE; + if (addr >= vma->vm_end) + return 0; + } + return 0; +} + +static void *system_heap_do_vmap(struct system_heap_buffer *buffer) +{ + struct sg_table *table = &buffer->sg_table; + int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE; + struct page **pages = vmalloc(sizeof(struct page *) * npages); + struct page **tmp = pages; + struct sg_page_iter piter; + pgprot_t pgprot = PAGE_KERNEL; + void *vaddr; + + if (!pages) + return ERR_PTR(-ENOMEM); + + if (buffer->uncached) + pgprot = pgprot_writecombine(PAGE_KERNEL); + + for_each_sgtable_page(table, &piter, 0) { + WARN_ON(tmp - pages >= npages); + *tmp++ = sg_page_iter_page(&piter); + } + + vaddr = vmap(pages, npages, VM_MAP, pgprot); + vfree(pages); + + if (!vaddr) + return ERR_PTR(-ENOMEM); + + return vaddr; +} + +static void *system_heap_vmap(struct dma_buf *dmabuf) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + void *vaddr; + + mutex_lock(&buffer->lock); + if (buffer->vmap_cnt) { + buffer->vmap_cnt++; + vaddr = buffer->vaddr; + goto out; + } + + vaddr = system_heap_do_vmap(buffer); + if (IS_ERR(vaddr)) + goto out; + + buffer->vaddr = vaddr; + buffer->vmap_cnt++; +out: + mutex_unlock(&buffer->lock); - for (pg = 0; pg < helper_buffer->pagecount; pg++) { + return vaddr; +} + +static void system_heap_vunmap(struct dma_buf *dmabuf, void *vaddr) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + + mutex_lock(&buffer->lock); + if (!--buffer->vmap_cnt) { + vunmap(buffer->vaddr); + buffer->vaddr = NULL; + } + mutex_unlock(&buffer->lock); +} + +static int system_heap_zero_buffer(struct system_heap_buffer *buffer) +{ + struct sg_table *sgt = &buffer->sg_table; + struct sg_page_iter piter; + struct page *p; + void *vaddr; + int ret = 0; + + for_each_sgtable_page(sgt, &piter, 0) { + p = sg_page_iter_page(&piter); + vaddr = kmap_atomic(p); + memset(vaddr, 0, PAGE_SIZE); + kunmap_atomic(vaddr); + } + + return ret; +} + +static void system_heap_buf_free(struct deferred_freelist_item *item, + enum df_reason reason) +{ + struct system_heap_buffer *buffer; + struct sg_table *table; + struct scatterlist *sg; + int i, j; + + buffer = container_of(item, struct system_heap_buffer, deferred_free); + /* Zero the buffer pages before adding back to the pool */ + if (reason == DF_NORMAL) + if (system_heap_zero_buffer(buffer)) + reason = DF_UNDER_PRESSURE; // On failure, just free + + table = &buffer->sg_table; + for_each_sg(table->sgl, sg, table->nents, i) { + struct page *page = sg_page(sg); + + if (reason == DF_UNDER_PRESSURE) { + __free_pages(page, compound_order(page)); + } else { + for (j = 0; j < NUM_ORDERS; j++) { + if (compound_order(page) == orders[j]) + break; + } + dmabuf_page_pool_free(pools[j], page); + } + } + sg_free_table(table); + kfree(buffer); +} + +static void system_heap_dma_buf_release(struct dma_buf *dmabuf) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE; + + deferred_free(&buffer->deferred_free, system_heap_buf_free, npages); +} + +static const struct dma_buf_ops system_heap_buf_ops = { + .attach = system_heap_attach, + .detach = system_heap_detach, + .map_dma_buf = system_heap_map_dma_buf, + .unmap_dma_buf = system_heap_unmap_dma_buf, + .begin_cpu_access = system_heap_dma_buf_begin_cpu_access, + .end_cpu_access = system_heap_dma_buf_end_cpu_access, + .mmap = system_heap_mmap, + .vmap = system_heap_vmap, + .vunmap = system_heap_vunmap, + .release = system_heap_dma_buf_release, +}; + +static struct page *alloc_largest_available(unsigned long size, + unsigned int max_order) +{ + struct page *page; + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < (PAGE_SIZE << orders[i])) + continue; + if (max_order < orders[i]) + continue; + page = dmabuf_page_pool_alloc(pools[i]); + if (!page) + continue; + return page; + } + return NULL; +} + +static struct dma_buf *system_heap_do_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags, + bool uncached) +{ + struct system_heap_buffer *buffer; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + unsigned long size_remaining = len; + unsigned int max_order = orders[0]; + struct dma_buf *dmabuf; + struct sg_table *table; + struct scatterlist *sg; + struct list_head pages; + struct page *page, *tmp_page; + int i, ret = -ENOMEM; + + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&buffer->attachments); + mutex_init(&buffer->lock); + buffer->heap = heap; + buffer->len = len; + buffer->uncached = uncached; + + INIT_LIST_HEAD(&pages); + i = 0; + while (size_remaining > 0) { /* * Avoid trying to allocate memory if the process - * has been killed by by SIGKILL + * has been killed by SIGKILL */ if (fatal_signal_pending(current)) - goto err1; + goto free_buffer; + + page = alloc_largest_available(size_remaining, max_order); + if (!page) + goto free_buffer; + + list_add_tail(&page->lru, &pages); + size_remaining -= page_size(page); + max_order = compound_order(page); + i++; + } - helper_buffer->pages[pg] = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!helper_buffer->pages[pg]) - goto err1; + table = &buffer->sg_table; + if (sg_alloc_table(table, i, GFP_KERNEL)) + goto free_buffer; + + sg = table->sgl; + list_for_each_entry_safe(page, tmp_page, &pages, lru) { + sg_set_page(sg, page, page_size(page), 0); + sg = sg_next(sg); + list_del(&page->lru); } /* create the dmabuf */ - dmabuf = heap_helper_export_dmabuf(helper_buffer, fd_flags); + exp_info.exp_name = dma_heap_get_name(heap); + exp_info.ops = &system_heap_buf_ops; + exp_info.size = buffer->len; + exp_info.flags = fd_flags; + exp_info.priv = buffer; + dmabuf = dma_buf_export(&exp_info); if (IS_ERR(dmabuf)) { ret = PTR_ERR(dmabuf); - goto err1; + goto free_pages; + } + + /* + * For uncached buffers, we need to initially flush cpu cache, since + * the __GFP_ZERO on the allocation means the zeroing was done by the + * cpu and thus it is likely cached. Map (and implicitly flush) and + * unmap it now so we don't get corruption later on. + */ + if (buffer->uncached) { + dma_map_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0); + dma_unmap_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0); } - helper_buffer->dmabuf = dmabuf; + return dmabuf; - ret = dma_buf_fd(dmabuf, fd_flags); - if (ret < 0) { - dma_buf_put(dmabuf); - /* just return, as put will call release and that will free */ - return ret; +free_pages: + for_each_sgtable_sg(table, sg, i) { + struct page *p = sg_page(sg); + + __free_pages(p, compound_order(p)); } + sg_free_table(table); +free_buffer: + list_for_each_entry_safe(page, tmp_page, &pages, lru) + __free_pages(page, compound_order(page)); + kfree(buffer); - return ret; + return ERR_PTR(ret); +} -err1: - while (pg > 0) - __free_page(helper_buffer->pages[--pg]); - kfree(helper_buffer->pages); -err0: - kfree(helper_buffer); +static struct dma_buf *system_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) +{ + return system_heap_do_allocate(heap, len, fd_flags, heap_flags, false); +} - return ret; +static long system_get_pool_size(struct dma_heap *heap) +{ + int i; + long num_pages = 0; + struct dmabuf_page_pool **pool; + + pool = pools; + for (i = 0; i < NUM_ORDERS; i++, pool++) { + num_pages += ((*pool)->count[POOL_LOWPAGE] + + (*pool)->count[POOL_HIGHPAGE]) << (*pool)->order; + } + + return num_pages << PAGE_SHIFT; } static const struct dma_heap_ops system_heap_ops = { .allocate = system_heap_allocate, + .get_pool_size = system_get_pool_size, +}; + +static struct dma_buf *system_uncached_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) +{ + return system_heap_do_allocate(heap, len, fd_flags, heap_flags, true); +} + +/* Dummy function to be used until we can call coerce_mask_and_coherent */ +static struct dma_buf *system_uncached_heap_not_initialized(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) +{ + return ERR_PTR(-EBUSY); +} + +static struct dma_heap_ops system_uncached_heap_ops = { + /* After system_heap_create is complete, we will swap this */ + .allocate = system_uncached_heap_not_initialized, }; static int system_heap_create(void) { struct dma_heap_export_info exp_info; - int ret = 0; + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + pools[i] = dmabuf_page_pool_create(order_flags[i], orders[i]); + + if (!pools[i]) { + int j; + + pr_err("%s: page pool creation failed!\n", __func__); + for (j = 0; j < i; j++) + dmabuf_page_pool_destroy(pools[j]); + return -ENOMEM; + } + } exp_info.name = "system"; exp_info.ops = &system_heap_ops; @@ -115,9 +566,21 @@ static int system_heap_create(void) sys_heap = dma_heap_add(&exp_info); if (IS_ERR(sys_heap)) - ret = PTR_ERR(sys_heap); + return PTR_ERR(sys_heap); - return ret; + exp_info.name = "system-uncached"; + exp_info.ops = &system_uncached_heap_ops; + exp_info.priv = NULL; + + sys_uncached_heap = dma_heap_add(&exp_info); + if (IS_ERR(sys_uncached_heap)) + return PTR_ERR(sys_uncached_heap); + + dma_coerce_mask_and_coherent(dma_heap_get_dev(sys_uncached_heap), DMA_BIT_MASK(64)); + mb(); /* make sure we only set allocate after dma_mask is set */ + system_uncached_heap_ops.allocate = system_uncached_heap_allocate; + + return 0; } module_init(system_heap_create); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c index 348b3a9170fa..3daa6c76b8dd 100644 --- a/drivers/dma-buf/sw_sync.c +++ b/drivers/dma-buf/sw_sync.c @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include #include @@ -410,3 +412,13 @@ const struct file_operations sw_sync_debugfs_fops = { .unlocked_ioctl = sw_sync_ioctl, .compat_ioctl = compat_ptr_ioctl, }; + +static struct miscdevice sw_sync_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "sw_sync", + .fops = &sw_sync_debugfs_fops, +}; + +module_misc_device(sw_sync_dev); + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma-buf/sync_debug.c b/drivers/dma-buf/sync_debug.c index 101394f16930..a2f906741ce0 100644 --- a/drivers/dma-buf/sync_debug.c +++ b/drivers/dma-buf/sync_debug.c @@ -8,6 +8,7 @@ #include #include "sync_debug.h" +#ifdef CONFIG_DEBUG_FS static struct dentry *dbgfs; static LIST_HEAD(sync_timeline_list_head); @@ -188,3 +189,4 @@ static __init int sync_debugfs_init(void) return 0; } late_initcall(sync_debugfs_init); +#endif diff --git a/drivers/dma-buf/sync_debug.h b/drivers/dma-buf/sync_debug.h index 6176e52ba2d7..ee84997da6b4 100644 --- a/drivers/dma-buf/sync_debug.h +++ b/drivers/dma-buf/sync_debug.h @@ -62,11 +62,18 @@ struct sync_pt { struct rb_node node; }; +#ifdef CONFIG_DEBUG_FS extern const struct file_operations sw_sync_debugfs_fops; void sync_timeline_debug_add(struct sync_timeline *obj); void sync_timeline_debug_remove(struct sync_timeline *obj); void sync_file_debug_add(struct sync_file *fence); void sync_file_debug_remove(struct sync_file *fence); +#else +static inline void sync_timeline_debug_add(struct sync_timeline *obj) {} +static inline void sync_timeline_debug_remove(struct sync_timeline *obj) {} +static inline void sync_file_debug_add(struct sync_file *fence) {} +static inline void sync_file_debug_remove(struct sync_file *fence) {} +#endif #endif /* _LINUX_SYNC_H */ diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index c08968c5ddf8..d9dbfda3c5f2 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -9,7 +9,7 @@ menu "Firmware Drivers" config ARM_SCMI_PROTOCOL tristate "ARM System Control and Management Interface (SCMI) Message Protocol" depends on ARM || ARM64 || COMPILE_TEST - depends on MAILBOX || HAVE_ARM_SMCCC_DISCOVERY + depends on MAILBOX help ARM System Control and Management Interface (SCMI) protocol is a set of operating system-independent software interfaces that are @@ -251,6 +251,13 @@ config QCOM_SCM_DOWNLOAD_MODE_DEFAULT Say Y here to enable "download mode" by default. +config ROCKCHIP_SIP + tristate "Rockchip SIP interface" + depends on HAVE_ARM_SMCCC && ARCH_ROCKCHIP + help + Say Y here if you want to enable SIP callbacks for Rockchip platforms + This option enables support for communicating with the ATF. + config TI_SCI_PROTOCOL tristate "TI System Control Interface (TISCI) Message Protocol" depends on TI_MESSAGE_MANAGER diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 5e013b6a3692..850970fbacbc 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -16,6 +16,7 @@ obj-$(CONFIG_ISCSI_IBFT_FIND) += iscsi_ibft_find.o obj-$(CONFIG_ISCSI_IBFT) += iscsi_ibft.o obj-$(CONFIG_FIRMWARE_MEMMAP) += memmap.o obj-$(CONFIG_RASPBERRYPI_FIRMWARE) += raspberrypi.o +obj-$(CONFIG_ROCKCHIP_SIP) += rockchip_sip.o obj-$(CONFIG_FW_CFG_SYSFS) += qemu_fw_cfg.o obj-$(CONFIG_QCOM_SCM) += qcom_scm.o qcom_scm-smc.o qcom_scm-legacy.o obj-$(CONFIG_TI_SCI_PROTOCOL) += ti_sci.o diff --git a/drivers/firmware/rockchip_sip.c b/drivers/firmware/rockchip_sip.c new file mode 100755 index 000000000000..af98657bb83e --- /dev/null +++ b/drivers/firmware/rockchip_sip.c @@ -0,0 +1,578 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Copyright (C) 2016, Fuzhou Rockchip Electronics Co., Ltd + */ + +#include +#include +#include +#include +#include +#ifdef CONFIG_ARM +#include +#endif +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_64BIT +#define PSCI_FN_NATIVE(version, name) PSCI_##version##_FN64_##name +#else +#define PSCI_FN_NATIVE(version, name) PSCI_##version##_FN_##name +#endif + +#define SIZE_PAGE(n) ((n) << 12) + +static struct arm_smccc_res __invoke_sip_fn_smc(unsigned long function_id, + unsigned long arg0, + unsigned long arg1, + unsigned long arg2) +{ + struct arm_smccc_res res; + + arm_smccc_smc(function_id, arg0, arg1, arg2, 0, 0, 0, 0, &res); + return res; +} + +struct arm_smccc_res sip_smc_dram(u32 arg0, u32 arg1, u32 arg2) +{ + return __invoke_sip_fn_smc(SIP_DRAM_CONFIG, arg0, arg1, arg2); +} +EXPORT_SYMBOL_GPL(sip_smc_dram); + +struct arm_smccc_res sip_smc_get_atf_version(void) +{ + return __invoke_sip_fn_smc(SIP_ATF_VERSION, 0, 0, 0); +} +EXPORT_SYMBOL_GPL(sip_smc_get_atf_version); + +struct arm_smccc_res sip_smc_get_sip_version(void) +{ + return __invoke_sip_fn_smc(SIP_SIP_VERSION, 0, 0, 0); +} +EXPORT_SYMBOL_GPL(sip_smc_get_sip_version); + +int sip_smc_set_suspend_mode(u32 ctrl, u32 config1, u32 config2) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(SIP_SUSPEND_MODE, ctrl, config1, config2); + return res.a0; +} +EXPORT_SYMBOL_GPL(sip_smc_set_suspend_mode); + +struct arm_smccc_res sip_smc_get_suspend_info(u32 info) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(SIP_SUSPEND_MODE, info, 0, 0); + return res; +} +EXPORT_SYMBOL_GPL(sip_smc_get_suspend_info); + +int sip_smc_virtual_poweroff(void) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND), 0, 0, 0); + return res.a0; +} +EXPORT_SYMBOL_GPL(sip_smc_virtual_poweroff); + +int sip_smc_remotectl_config(u32 func, u32 data) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(SIP_REMOTECTL_CFG, func, data, 0); + + return res.a0; +} +EXPORT_SYMBOL_GPL(sip_smc_remotectl_config); + +u32 sip_smc_secure_reg_read(u32 addr_phy) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(SIP_ACCESS_REG, 0, addr_phy, SECURE_REG_RD); + if (res.a0) + pr_err("%s error: %d, addr phy: 0x%x\n", + __func__, (int)res.a0, addr_phy); + + return res.a1; +} +EXPORT_SYMBOL_GPL(sip_smc_secure_reg_read); + +int sip_smc_secure_reg_write(u32 addr_phy, u32 val) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(SIP_ACCESS_REG, val, addr_phy, SECURE_REG_WR); + if (res.a0) + pr_err("%s error: %d, addr phy: 0x%x\n", + __func__, (int)res.a0, addr_phy); + + return res.a0; +} +EXPORT_SYMBOL_GPL(sip_smc_secure_reg_write); + +static void *sip_map(phys_addr_t start, size_t size) +{ + struct page **pages; + phys_addr_t page_start; + unsigned int page_count; + pgprot_t prot; + unsigned int i; + void *vaddr; + + if (!pfn_valid(__phys_to_pfn(start))) + return ioremap(start, size); + + page_start = start - offset_in_page(start); + page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); + + prot = pgprot_noncached(PAGE_KERNEL); + + pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); + if (!pages) { + pr_err("%s: Failed to allocate array for %u pages\n", + __func__, page_count); + return NULL; + } + + for (i = 0; i < page_count; i++) + pages[i] = phys_to_page(page_start + i * PAGE_SIZE); + + vaddr = vmap(pages, page_count, VM_MAP, prot); + kfree(pages); + + /* + * Since vmap() uses page granularity, we must add the offset + * into the page here, to get the byte granularity address + * into the mapping to represent the actual "start" location. + */ + return vaddr + offset_in_page(start); +} + +struct arm_smccc_res sip_smc_request_share_mem(u32 page_num, + share_page_type_t page_type) +{ + struct arm_smccc_res res; + unsigned long share_mem_phy; + + res = __invoke_sip_fn_smc(SIP_SHARE_MEM, page_num, page_type, 0); + if (IS_SIP_ERROR(res.a0)) + goto error; + + share_mem_phy = res.a1; + res.a1 = (unsigned long)sip_map(share_mem_phy, SIZE_PAGE(page_num)); + +error: + return res; +} +EXPORT_SYMBOL_GPL(sip_smc_request_share_mem); + +struct arm_smccc_res sip_smc_mcu_el3fiq(u32 arg0, u32 arg1, u32 arg2) +{ + return __invoke_sip_fn_smc(SIP_MCU_EL3FIQ_CFG, arg0, arg1, arg2); +} +EXPORT_SYMBOL_GPL(sip_smc_mcu_el3fiq); + +struct arm_smccc_res sip_smc_vpu_reset(u32 arg0, u32 arg1, u32 arg2) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(PSCI_SIP_VPU_RESET, arg0, arg1, arg2); + return res; +} +EXPORT_SYMBOL_GPL(sip_smc_vpu_reset); + +struct arm_smccc_res sip_smc_bus_config(u32 arg0, u32 arg1, u32 arg2) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(SIP_BUS_CFG, arg0, arg1, arg2); + return res; +} +EXPORT_SYMBOL_GPL(sip_smc_bus_config); + +struct dram_addrmap_info *sip_smc_get_dram_map(void) +{ + struct arm_smccc_res res; + static struct dram_addrmap_info *map; + + if (map) + return map; + + /* Request share memory size 4KB */ + res = sip_smc_request_share_mem(1, SHARE_PAGE_TYPE_DDR_ADDRMAP); + if (res.a0 != 0) { + pr_err("no ATF memory for init\n"); + return NULL; + } + + map = (struct dram_addrmap_info *)res.a1; + + res = sip_smc_dram(SHARE_PAGE_TYPE_DDR_ADDRMAP, 0, + ROCKCHIP_SIP_CONFIG_DRAM_ADDRMAP_GET); + if (res.a0) { + pr_err("rockchip_sip_config_dram_init error:%lx\n", res.a0); + map = NULL; + return NULL; + } + + return map; +} +EXPORT_SYMBOL_GPL(sip_smc_get_dram_map); + +struct arm_smccc_res sip_smc_lastlog_request(void) +{ + struct arm_smccc_res res; + void __iomem *addr1, *addr2; + + res = __invoke_sip_fn_smc(SIP_LAST_LOG, local_clock(), 0, 0); + if (IS_SIP_ERROR(res.a0)) + return res; + + addr1 = sip_map(res.a1, res.a3); + if (!addr1) { + pr_err("%s: share memory buffer0 ioremap failed\n", __func__); + res.a0 = SIP_RET_INVALID_ADDRESS; + return res; + } + addr2 = sip_map(res.a2, res.a3); + if (!addr2) { + pr_err("%s: share memory buffer1 ioremap failed\n", __func__); + res.a0 = SIP_RET_INVALID_ADDRESS; + return res; + } + + res.a1 = (unsigned long)addr1; + res.a2 = (unsigned long)addr2; + + return res; +} +EXPORT_SYMBOL_GPL(sip_smc_lastlog_request); + +/************************** fiq debugger **************************************/ +/* + * AArch32 is not allowed to call SMC64(ATF framework does not support), so we + * don't change SIP_UARTDBG_FN to SIP_UARTDBG_CFG64 even when cpu is AArch32 + * mode. Let ATF support SIP_UARTDBG_CFG, and we just initialize SIP_UARTDBG_FN + * depends on compile option(CONFIG_ARM or CONFIG_ARM64). + */ +#ifdef CONFIG_ARM64 +#define SIP_UARTDBG_FN SIP_UARTDBG_CFG64 +#else +#define SIP_UARTDBG_FN SIP_UARTDBG_CFG +static int firmware_64_32bit; +#endif + +static int fiq_sip_enabled; +static int fiq_target_cpu; +static phys_addr_t ft_fiq_mem_phy; +static void __iomem *ft_fiq_mem_base; +static void (*sip_fiq_debugger_uart_irq_tf)(struct pt_regs _pt_regs, + unsigned long cpu); +int sip_fiq_debugger_is_enabled(void) +{ + return fiq_sip_enabled; +} +EXPORT_SYMBOL_GPL(sip_fiq_debugger_is_enabled); + +static struct pt_regs sip_fiq_debugger_get_pt_regs(void *reg_base, + unsigned long sp_el1) +{ + struct pt_regs fiq_pt_regs; + __maybe_unused struct sm_nsec_ctx *nsec_ctx = reg_base; + __maybe_unused struct gp_regs_ctx *gp_regs = reg_base; + +#ifdef CONFIG_ARM64 + /* + * 64-bit ATF + 64-bit kernel + */ + /* copy cpu context: x0 ~ spsr_el3 */ + memcpy(&fiq_pt_regs, reg_base, 8 * 31); + + /* copy pstate: spsr_el3 */ + memcpy(&fiq_pt_regs.pstate, reg_base + 0x110, 8); + fiq_pt_regs.sp = sp_el1; + + /* copy pc: elr_el3 */ + memcpy(&fiq_pt_regs.pc, reg_base + 0x118, 8); +#else + if (firmware_64_32bit == FIRMWARE_ATF_64BIT) { + /* + * 64-bit ATF + 32-bit kernel + */ + fiq_pt_regs.ARM_r0 = gp_regs->x0; + fiq_pt_regs.ARM_r1 = gp_regs->x1; + fiq_pt_regs.ARM_r2 = gp_regs->x2; + fiq_pt_regs.ARM_r3 = gp_regs->x3; + fiq_pt_regs.ARM_r4 = gp_regs->x4; + fiq_pt_regs.ARM_r5 = gp_regs->x5; + fiq_pt_regs.ARM_r6 = gp_regs->x6; + fiq_pt_regs.ARM_r7 = gp_regs->x7; + fiq_pt_regs.ARM_r8 = gp_regs->x8; + fiq_pt_regs.ARM_r9 = gp_regs->x9; + fiq_pt_regs.ARM_r10 = gp_regs->x10; + fiq_pt_regs.ARM_fp = gp_regs->x11; + fiq_pt_regs.ARM_ip = gp_regs->x12; + fiq_pt_regs.ARM_sp = gp_regs->x19; /* aarch32 svc_r13 */ + fiq_pt_regs.ARM_lr = gp_regs->x18; /* aarch32 svc_r14 */ + fiq_pt_regs.ARM_cpsr = gp_regs->spsr_el3; + fiq_pt_regs.ARM_pc = gp_regs->elr_el3; + } else { + /* + * 32-bit tee firmware + 32-bit kernel + */ + fiq_pt_regs.ARM_r0 = nsec_ctx->r0; + fiq_pt_regs.ARM_r1 = nsec_ctx->r1; + fiq_pt_regs.ARM_r2 = nsec_ctx->r2; + fiq_pt_regs.ARM_r3 = nsec_ctx->r3; + fiq_pt_regs.ARM_r4 = nsec_ctx->r4; + fiq_pt_regs.ARM_r5 = nsec_ctx->r5; + fiq_pt_regs.ARM_r6 = nsec_ctx->r6; + fiq_pt_regs.ARM_r7 = nsec_ctx->r7; + fiq_pt_regs.ARM_r8 = nsec_ctx->r8; + fiq_pt_regs.ARM_r9 = nsec_ctx->r9; + fiq_pt_regs.ARM_r10 = nsec_ctx->r10; + fiq_pt_regs.ARM_fp = nsec_ctx->r11; + fiq_pt_regs.ARM_ip = nsec_ctx->r12; + fiq_pt_regs.ARM_sp = nsec_ctx->svc_sp; + fiq_pt_regs.ARM_lr = nsec_ctx->svc_lr; + fiq_pt_regs.ARM_cpsr = nsec_ctx->mon_spsr; + + /* + * 'nsec_ctx->mon_lr' is not the fiq break point's PC, because it will + * be override as 'psci_fiq_debugger_uart_irq_tf_cb' for optee-os to + * jump to fiq_debugger handler. + * + * As 'nsec_ctx->und_lr' is not used for kernel, so optee-os uses it to + * deliver fiq break point's PC. + * + */ + fiq_pt_regs.ARM_pc = nsec_ctx->und_lr; + } +#endif + + return fiq_pt_regs; +} + +static void sip_fiq_debugger_uart_irq_tf_cb(unsigned long sp_el1, + unsigned long offset, + unsigned long cpu) +{ + struct pt_regs fiq_pt_regs; + char *cpu_context; + + /* calling fiq handler */ + if (ft_fiq_mem_base) { + cpu_context = (char *)ft_fiq_mem_base + offset; + fiq_pt_regs = sip_fiq_debugger_get_pt_regs(cpu_context, sp_el1); + sip_fiq_debugger_uart_irq_tf(fiq_pt_regs, cpu); + } + + /* fiq handler done, return to EL3(then EL3 return to EL1 entry) */ + __invoke_sip_fn_smc(SIP_UARTDBG_FN, 0, 0, UARTDBG_CFG_OSHDL_TO_OS); +} + +int sip_fiq_debugger_uart_irq_tf_init(u32 irq_id, void *callback_fn) +{ + struct arm_smccc_res res; + + fiq_target_cpu = 0; + + /* init fiq debugger callback */ + sip_fiq_debugger_uart_irq_tf = callback_fn; + res = __invoke_sip_fn_smc(SIP_UARTDBG_FN, irq_id, + (unsigned long)sip_fiq_debugger_uart_irq_tf_cb, + UARTDBG_CFG_INIT); + if (IS_SIP_ERROR(res.a0)) { + pr_err("%s error: %d\n", __func__, (int)res.a0); + return res.a0; + } + + /* share memory ioremap */ + if (!ft_fiq_mem_base) { + ft_fiq_mem_phy = res.a1; + ft_fiq_mem_base = sip_map(ft_fiq_mem_phy, + FIQ_UARTDBG_SHARE_MEM_SIZE); + if (!ft_fiq_mem_base) { + pr_err("%s: share memory ioremap failed\n", __func__); + return -ENOMEM; + } + } + + fiq_sip_enabled = 1; + + return SIP_RET_SUCCESS; +} +EXPORT_SYMBOL_GPL(sip_fiq_debugger_uart_irq_tf_init); + +static ulong cpu_logical_map_mpidr(u32 cpu) +{ +#ifdef MODULE + /* Empirically, local "cpu_logical_map()" for rockchip platforms */ + ulong mpidr = 0x00; + + if (cpu < 4) + /* 0x00, 0x01, 0x02, 0x03 */ + mpidr = cpu; + else if (cpu < 8) + /* 0x100, 0x101, 0x102, 0x103 */ + mpidr = 0x100 | (cpu - 4); + else + pr_err("Unsupported map cpu: %d\n", cpu); + + return mpidr; +#else + return cpu_logical_map(cpu); +#endif +} + +int sip_fiq_debugger_switch_cpu(u32 cpu) +{ + struct arm_smccc_res res; + + fiq_target_cpu = cpu; + res = __invoke_sip_fn_smc(SIP_UARTDBG_FN, cpu_logical_map_mpidr(cpu), + 0, UARTDBG_CFG_OSHDL_CPUSW); + return res.a0; +} + +int sip_fiq_debugger_sdei_switch_cpu(u32 cur_cpu, u32 target_cpu, u32 flag) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(SIP_SDEI_FIQ_DBG_SWITCH_CPU, + cur_cpu, target_cpu, flag); + return res.a0; +} + +int sip_fiq_debugger_sdei_get_event_id(u32 *fiq, u32 *sw_cpu, u32 *flag) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(SIP_SDEI_FIQ_DBG_GET_EVENT_ID, + 0, 0, 0); + *fiq = res.a1; + *sw_cpu = res.a2; + if (flag) + *flag = res.a3; + + return res.a0; +} + +EXPORT_SYMBOL_GPL(sip_fiq_debugger_switch_cpu); + +void sip_fiq_debugger_enable_debug(bool enable) +{ + unsigned long val; + + val = enable ? UARTDBG_CFG_OSHDL_DEBUG_ENABLE : + UARTDBG_CFG_OSHDL_DEBUG_DISABLE; + + __invoke_sip_fn_smc(SIP_UARTDBG_FN, 0, 0, val); +} +EXPORT_SYMBOL_GPL(sip_fiq_debugger_enable_debug); + +int sip_fiq_debugger_set_print_port(u32 port_phyaddr, u32 baudrate) +{ + struct arm_smccc_res res; + + res = __invoke_sip_fn_smc(SIP_UARTDBG_FN, port_phyaddr, baudrate, + UARTDBG_CFG_PRINT_PORT); + return res.a0; +} +EXPORT_SYMBOL_GPL(sip_fiq_debugger_set_print_port); + +int sip_fiq_debugger_request_share_memory(void) +{ + struct arm_smccc_res res; + + /* request page share memory */ + res = sip_smc_request_share_mem(FIQ_UARTDBG_PAGE_NUMS, + SHARE_PAGE_TYPE_UARTDBG); + if (IS_SIP_ERROR(res.a0)) + return res.a0; + + return SIP_RET_SUCCESS; +} +EXPORT_SYMBOL_GPL(sip_fiq_debugger_request_share_memory); + +int sip_fiq_debugger_get_target_cpu(void) +{ + return fiq_target_cpu; +} +EXPORT_SYMBOL_GPL(sip_fiq_debugger_get_target_cpu); + +void sip_fiq_debugger_enable_fiq(bool enable, uint32_t tgt_cpu) +{ + u32 en; + + fiq_target_cpu = tgt_cpu; + en = enable ? UARTDBG_CFG_FIQ_ENABEL : UARTDBG_CFG_FIQ_DISABEL; + __invoke_sip_fn_smc(SIP_UARTDBG_FN, tgt_cpu, 0, en); +} +EXPORT_SYMBOL_GPL(sip_fiq_debugger_enable_fiq); + +/******************************************************************************/ +#ifdef CONFIG_ARM +static __init int sip_firmware_init(void) +{ + struct arm_smccc_res res; + + if (!psci_smp_available()) + return 0; + + /* + * OP-TEE works on kernel 3.10 and 4.4 and we have different sip + * implement. We should tell OP-TEE the current rockchip sip version. + */ + res = __invoke_sip_fn_smc(SIP_SIP_VERSION, SIP_IMPLEMENT_V2, + SECURE_REG_WR, 0); + if (IS_SIP_ERROR(res.a0)) + pr_err("%s: set rockchip sip version v2 failed\n", __func__); + + /* + * Currently, we support: + * + * 1. 64-bit ATF + 64-bit kernel; + * 2. 64-bit ATF + 32-bit kernel; + * 3. 32-bit TEE + 32-bit kernel; + * + * We need to detect which case of above and record in firmware_64_32bit + * We get info from cpuid and compare with all supported ARMv7 cpu. + */ + switch (read_cpuid_part()) { + case ARM_CPU_PART_CORTEX_A7: + case ARM_CPU_PART_CORTEX_A8: + case ARM_CPU_PART_CORTEX_A9: + case ARM_CPU_PART_CORTEX_A12: + case ARM_CPU_PART_CORTEX_A15: + case ARM_CPU_PART_CORTEX_A17: + firmware_64_32bit = FIRMWARE_TEE_32BIT; + break; + default: + firmware_64_32bit = FIRMWARE_ATF_64BIT; + break; + } + + return 0; +} +arch_initcall(sip_firmware_init); +#endif + +MODULE_DESCRIPTION("Rockchip SIP Call"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index d1300fc003ed..9a4110ad1c20 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -495,6 +495,14 @@ config GPIO_REG A 32-bit single register GPIO fixed in/out implementation. This can be used to represent any register as a set of GPIO signals. +config GPIO_ROCKCHIP + tristate "Rockchip GPIO support" + depends on ARCH_ROCKCHIP || COMPILE_TEST + select GPIOLIB_IRQCHIP + default ARCH_ROCKCHIP + help + Say yes here to support GPIO on Rockchip SoCs. + config GPIO_SAMA5D2_PIOBU tristate "SAMA5D2 PIOBU GPIO support" depends on MFD_SYSCON diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index 09dada80ac34..ce6cafb6d3de 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -125,6 +125,7 @@ obj-$(CONFIG_GPIO_RCAR) += gpio-rcar.o obj-$(CONFIG_GPIO_RDA) += gpio-rda.o obj-$(CONFIG_GPIO_RDC321X) += gpio-rdc321x.o obj-$(CONFIG_GPIO_REG) += gpio-reg.o +obj-$(CONFIG_GPIO_ROCKCHIP) += gpio-rockchip.o obj-$(CONFIG_ARCH_SA1100) += gpio-sa1100.o obj-$(CONFIG_GPIO_SAMA5D2_PIOBU) += gpio-sama5d2-piobu.o obj-$(CONFIG_GPIO_SCH311X) += gpio-sch311x.o diff --git a/drivers/gpio/gpio-rockchip.c b/drivers/gpio/gpio-rockchip.c new file mode 100755 index 000000000000..1d1e9a64cceb --- /dev/null +++ b/drivers/gpio/gpio-rockchip.c @@ -0,0 +1,746 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020 Rockchip Electronics Co. Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../pinctrl/core.h" +#include "../pinctrl/pinctrl-rockchip.h" + +#define GPIO_TYPE_V1 (0) /* GPIO Version ID reserved */ +#define GPIO_TYPE_V2 (0x01000C2B) /* GPIO Version ID 0x01000C2B */ + +#define GPIO_BANK_PIN_NUM (32) + +static const struct rockchip_gpio_regs gpio_regs_v1 = { + .port_dr = 0x00, + .port_ddr = 0x04, + .int_en = 0x30, + .int_mask = 0x34, + .int_type = 0x38, + .int_polarity = 0x3c, + .int_status = 0x40, + .int_rawstatus = 0x44, + .debounce = 0x48, + .port_eoi = 0x4c, + .ext_port = 0x50, +}; + +static const struct rockchip_gpio_regs gpio_regs_v2 = { + .port_dr = 0x00, + .port_ddr = 0x08, + .int_en = 0x10, + .int_mask = 0x18, + .int_type = 0x20, + .int_polarity = 0x28, + .int_bothedge = 0x30, + .int_status = 0x50, + .int_rawstatus = 0x58, + .debounce = 0x38, + .dbclk_div_en = 0x40, + .dbclk_div_con = 0x48, + .port_eoi = 0x60, + .ext_port = 0x70, + .version_id = 0x78, +}; + +static inline void gpio_writel_v2(u32 val, void __iomem *reg) +{ + writel((val & 0xffff) | 0xffff0000, reg); + writel((val >> 16) | 0xffff0000, reg + 0x4); +} + +static inline u32 gpio_readl_v2(void __iomem *reg) +{ + return readl(reg + 0x4) << 16 | readl(reg); +} + +static inline void rockchip_gpio_writel(struct rockchip_pin_bank *bank, + u32 value, unsigned int offset) +{ + void __iomem *reg = bank->reg_base + offset; + + if (bank->gpio_type == GPIO_TYPE_V2) + gpio_writel_v2(value, reg); + else + writel(value, reg); +} + +static inline u32 rockchip_gpio_readl(struct rockchip_pin_bank *bank, + unsigned int offset) +{ + void __iomem *reg = bank->reg_base + offset; + u32 value; + + if (bank->gpio_type == GPIO_TYPE_V2) + value = gpio_readl_v2(reg); + else + value = readl(reg); + + return value; +} + +static inline void rockchip_gpio_writel_bit(struct rockchip_pin_bank *bank, + u32 bit, u32 value, + unsigned int offset) +{ + void __iomem *reg = bank->reg_base + offset; + u32 data; + + if (bank->gpio_type == GPIO_TYPE_V2) { + if (value) + data = BIT(bit % 16) | BIT(bit % 16 + 16); + else + data = BIT(bit % 16 + 16); + writel(data, bit >= 16 ? reg + 0x4 : reg); + } else { + data = readl(reg); + data &= ~BIT(bit); + if (value) + data |= BIT(bit); + writel(data, reg); + } +} + +static inline u32 rockchip_gpio_readl_bit(struct rockchip_pin_bank *bank, + u32 bit, unsigned int offset) +{ + void __iomem *reg = bank->reg_base + offset; + u32 data; + + if (bank->gpio_type == GPIO_TYPE_V2) { + data = readl(bit >= 16 ? reg + 0x4 : reg); + data >>= bit % 16; + } else { + data = readl(reg); + data >>= bit; + } + + return data & (0x1); +} + +static void rockchip_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) +{ + struct rockchip_pin_bank *bank = gpiochip_get_data(gc); + unsigned long flags; + + raw_spin_lock_irqsave(&bank->slock, flags); + rockchip_gpio_writel_bit(bank, offset, value, bank->gpio_regs->port_dr); + raw_spin_unlock_irqrestore(&bank->slock, flags); +} + +static int rockchip_gpio_get(struct gpio_chip *gc, unsigned int offset) +{ + struct rockchip_pin_bank *bank = gpiochip_get_data(gc); + u32 data; + + data = readl(bank->reg_base + bank->gpio_regs->ext_port); + data >>= offset; + data &= 1; + + return data; +} + +static int rockchip_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) +{ + struct rockchip_pin_bank *bank = gpiochip_get_data(chip); + u32 data; + + data = rockchip_gpio_readl_bit(bank, offset, bank->gpio_regs->port_ddr); + + return !data; +} + +static int rockchip_gpio_set_direction(struct gpio_chip *chip, unsigned int offset, bool input) +{ + struct rockchip_pin_bank *bank = gpiochip_get_data(chip); + u32 data = input ? 0 : 1; + + rockchip_gpio_writel_bit(bank, offset, data, bank->gpio_regs->port_ddr); + + return 0; +} + +static int rockchip_gpio_direction_input(struct gpio_chip *gc, unsigned int offset) +{ + return rockchip_gpio_set_direction(gc, offset, true); +} + +static int rockchip_gpio_direction_output(struct gpio_chip *gc, + unsigned int offset, int value) +{ + rockchip_gpio_set(gc, offset, value); + + return rockchip_gpio_set_direction(gc, offset, false); +} + +static int rockchip_gpio_set_debounce(struct gpio_chip *gc, + unsigned int offset, unsigned int debounce) +{ + struct rockchip_pin_bank *bank = gpiochip_get_data(gc); + const struct rockchip_gpio_regs *reg = bank->gpio_regs; + unsigned long flags, div_reg, freq, max_debounce; + bool div_debounce_support; + unsigned int cur_div_reg; + u64 div; + + if (!IS_ERR(bank->db_clk)) { + div_debounce_support = true; + freq = clk_get_rate(bank->db_clk); + max_debounce = (GENMASK(23, 0) + 1) * 2 * 1000000 / freq; + if (debounce > max_debounce) + return -EINVAL; + + div = debounce * freq; + div_reg = DIV_ROUND_CLOSEST_ULL(div, 2 * USEC_PER_SEC) - 1; + } else { + div_debounce_support = false; + } + + raw_spin_lock_irqsave(&bank->slock, flags); + + /* Only the v1 needs to configure div_en and div_con for dbclk */ + if (debounce) { + if (div_debounce_support) { + /* Configure the max debounce from consumers */ + cur_div_reg = readl(bank->reg_base + reg->dbclk_div_con); + if (cur_div_reg < div_reg) + writel(div_reg, bank->reg_base + reg->dbclk_div_con); + rockchip_gpio_writel_bit(bank, offset, 1, + reg->dbclk_div_en); + } + + rockchip_gpio_writel_bit(bank, offset, 1, reg->debounce); + } else { + if (div_debounce_support) + rockchip_gpio_writel_bit(bank, offset, 0, + reg->dbclk_div_en); + + rockchip_gpio_writel_bit(bank, offset, 0, reg->debounce); + } + + raw_spin_unlock_irqrestore(&bank->slock, flags); + + /* Enable or disable dbclk at last */ + if (div_debounce_support) { + if (debounce) + clk_prepare_enable(bank->db_clk); + else + clk_disable_unprepare(bank->db_clk); + } + + return 0; +} + +/* + * gpiolib set_config callback function. The setting of the pin + * mux function as 'gpio output' will be handled by the pinctrl subsystem + * interface. + */ +static int rockchip_gpio_set_config(struct gpio_chip *gc, unsigned int offset, + unsigned long config) +{ + enum pin_config_param param = pinconf_to_config_param(config); + unsigned int debounce = pinconf_to_config_argument(config); + int ret = 0; + + switch (param) { + case PIN_CONFIG_INPUT_DEBOUNCE: + /* + * Rockchip's gpio could only support up to one period + * of the debounce clock(pclk), which is far away from + * satisftying the requirement, as pclk is usually near + * 100MHz shared by all peripherals. So the fact is it + * has crippled debounce capability could only be useful + * to prevent any spurious glitches from waking up the system + * if the gpio is conguired as wakeup interrupt source. Let's + * still return -ENOTSUPP as before, to make sure the caller + * of gpiod_set_debounce won't change its behaviour. + */ + rockchip_gpio_set_debounce(gc, offset, debounce); + ret = -ENOTSUPP; + break; + default: + ret = -ENOTSUPP; + break; + } + + return ret; +} + +/* + * gpiolib gpio_to_irq callback function. Creates a mapping between a GPIO pin + * and a virtual IRQ, if not already present. + */ +static int rockchip_gpio_to_irq(struct gpio_chip *gc, unsigned int offset) +{ + struct rockchip_pin_bank *bank = gpiochip_get_data(gc); + unsigned int virq; + + if (!bank->domain) + return -ENXIO; + + virq = irq_create_mapping(bank->domain, offset); + + return (virq) ? : -ENXIO; +} + +static const struct gpio_chip rockchip_gpiolib_chip = { + .request = gpiochip_generic_request, + .free = gpiochip_generic_free, + .set = rockchip_gpio_set, + .get = rockchip_gpio_get, + .get_direction = rockchip_gpio_get_direction, + .direction_input = rockchip_gpio_direction_input, + .direction_output = rockchip_gpio_direction_output, + .set_config = rockchip_gpio_set_config, + .to_irq = rockchip_gpio_to_irq, + .owner = THIS_MODULE, +}; + +static void rockchip_irq_demux(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct rockchip_pin_bank *bank = irq_desc_get_handler_data(desc); + const struct rockchip_gpio_regs *reg = bank->gpio_regs; + u32 pend; + + dev_dbg(bank->dev, "got irq for bank %s\n", bank->name); + + chained_irq_enter(chip, desc); + + pend = readl_relaxed(bank->reg_base + reg->int_status); + + while (pend) { + unsigned int irq, virq; + + irq = __ffs(pend); + pend &= ~BIT(irq); + virq = irq_find_mapping(bank->domain, irq); + + if (!virq) { + dev_err(bank->dev, "unmapped irq %d\n", irq); + continue; + } + + dev_dbg(bank->dev, "handling irq %d\n", irq); + + /* + * Triggering IRQ on both rising and falling edge + * needs manual intervention. + */ + if (bank->toggle_edge_mode & BIT(irq)) { + u32 data, data_old, polarity; + unsigned long flags; + + data = readl_relaxed(bank->reg_base + reg->ext_port); + do { + raw_spin_lock_irqsave(&bank->slock, flags); + + polarity = readl_relaxed(bank->reg_base + + reg->int_polarity); + if (data & BIT(irq)) + polarity &= ~BIT(irq); + else + polarity |= BIT(irq); + writel(polarity, bank->reg_base + + reg->int_polarity); + + raw_spin_unlock_irqrestore(&bank->slock, flags); + + data_old = data; + data = readl_relaxed(bank->reg_base + + reg->ext_port); + } while ((data & BIT(irq)) != (data_old & BIT(irq))); + } + + generic_handle_irq(virq); + } + + chained_irq_exit(chip, desc); +} + +static int rockchip_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + struct rockchip_pin_bank *bank = gc->private; + u32 mask = BIT(d->hwirq); + u32 polarity; + u32 level; + u32 data; + unsigned long flags; + int ret = 0; + + raw_spin_lock_irqsave(&bank->slock, flags); + + rockchip_gpio_writel_bit(bank, d->hwirq, 0, + bank->gpio_regs->port_ddr); + + raw_spin_unlock_irqrestore(&bank->slock, flags); + + if (type & IRQ_TYPE_EDGE_BOTH) + irq_set_handler_locked(d, handle_edge_irq); + else + irq_set_handler_locked(d, handle_level_irq); + + raw_spin_lock_irqsave(&bank->slock, flags); + + level = rockchip_gpio_readl(bank, bank->gpio_regs->int_type); + polarity = rockchip_gpio_readl(bank, bank->gpio_regs->int_polarity); + + switch (type) { + case IRQ_TYPE_EDGE_BOTH: + if (bank->gpio_type == GPIO_TYPE_V2) { + bank->toggle_edge_mode &= ~mask; + rockchip_gpio_writel_bit(bank, d->hwirq, 1, + bank->gpio_regs->int_bothedge); + goto out; + } else { + bank->toggle_edge_mode |= mask; + level |= mask; + + /* + * Determine gpio state. If 1 next interrupt should be falling + * otherwise rising. + */ + data = readl(bank->reg_base + bank->gpio_regs->ext_port); + if (data & mask) + polarity &= ~mask; + else + polarity |= mask; + } + break; + case IRQ_TYPE_EDGE_RISING: + bank->toggle_edge_mode &= ~mask; + level |= mask; + polarity |= mask; + break; + case IRQ_TYPE_EDGE_FALLING: + bank->toggle_edge_mode &= ~mask; + level |= mask; + polarity &= ~mask; + break; + case IRQ_TYPE_LEVEL_HIGH: + bank->toggle_edge_mode &= ~mask; + level &= ~mask; + polarity |= mask; + break; + case IRQ_TYPE_LEVEL_LOW: + bank->toggle_edge_mode &= ~mask; + level &= ~mask; + polarity &= ~mask; + break; + default: + ret = -EINVAL; + goto out; + } + + rockchip_gpio_writel(bank, level, bank->gpio_regs->int_type); + rockchip_gpio_writel(bank, polarity, bank->gpio_regs->int_polarity); +out: + raw_spin_unlock_irqrestore(&bank->slock, flags); + + return ret; +} + +static void rockchip_irq_suspend(struct irq_data *d) +{ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + struct rockchip_pin_bank *bank = gc->private; + + bank->saved_masks = irq_reg_readl(gc, bank->gpio_regs->int_mask); + irq_reg_writel(gc, ~gc->wake_active, bank->gpio_regs->int_mask); +} + +static void rockchip_irq_resume(struct irq_data *d) +{ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + struct rockchip_pin_bank *bank = gc->private; + + irq_reg_writel(gc, bank->saved_masks, bank->gpio_regs->int_mask); +} + +static int rockchip_interrupts_register(struct rockchip_pin_bank *bank) +{ + unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; + struct irq_chip_generic *gc; + int ret; + + bank->domain = irq_domain_add_linear(bank->of_node, 32, + &irq_generic_chip_ops, NULL); + if (!bank->domain) { + dev_warn(bank->dev, "could not initialize irq domain for bank %s\n", bank->name); + return -EINVAL; + } + + ret = irq_alloc_domain_generic_chips(bank->domain, 32, 1, + bank->name, handle_level_irq, + clr, 0, 0); + if (ret) { + dev_err(bank->dev, "could not alloc generic chips for bank %s\n", bank->name); + irq_domain_remove(bank->domain); + return ret; + } + + gc = irq_get_domain_generic_chip(bank->domain, 0); + if (bank->gpio_type == GPIO_TYPE_V2) { + gc->reg_writel = gpio_writel_v2; + gc->reg_readl = gpio_readl_v2; + } + gc->reg_base = bank->reg_base; + gc->private = bank; + gc->chip_types[0].regs.mask = bank->gpio_regs->int_mask; + gc->chip_types[0].regs.ack = bank->gpio_regs->port_eoi; + gc->chip_types[0].chip.irq_ack = irq_gc_ack_set_bit; + gc->chip_types[0].chip.irq_mask = irq_gc_mask_set_bit; + gc->chip_types[0].chip.irq_unmask = irq_gc_mask_clr_bit; + gc->chip_types[0].chip.irq_enable = irq_gc_mask_clr_bit; + gc->chip_types[0].chip.irq_disable = irq_gc_mask_set_bit; + gc->chip_types[0].chip.irq_set_wake = irq_gc_set_wake; + gc->chip_types[0].chip.irq_suspend = rockchip_irq_suspend; + gc->chip_types[0].chip.irq_resume = rockchip_irq_resume; + gc->chip_types[0].chip.irq_set_type = rockchip_irq_set_type; + gc->wake_enabled = IRQ_MSK(bank->nr_pins); + + /* + * Linux assumes that all interrupts start out disabled/masked. + * Our driver only uses the concept of masked and always keeps + * things enabled, so for us that's all masked and all enabled. + */ + rockchip_gpio_writel(bank, 0xffffffff, bank->gpio_regs->int_mask); + rockchip_gpio_writel(bank, 0xffffffff, bank->gpio_regs->port_eoi); + rockchip_gpio_writel(bank, 0xffffffff, bank->gpio_regs->int_en); + gc->mask_cache = 0xffffffff; + + irq_set_chained_handler_and_data(bank->irq, + rockchip_irq_demux, bank); + + return 0; +} + +static int rockchip_gpiolib_register(struct rockchip_pin_bank *bank) +{ + struct gpio_chip *gc; + int ret; + + bank->gpio_chip = rockchip_gpiolib_chip; + + gc = &bank->gpio_chip; + gc->base = bank->pin_base; + gc->ngpio = bank->nr_pins; + gc->label = bank->name; + gc->parent = bank->dev; +#ifdef CONFIG_OF_GPIO + gc->of_node = of_node_get(bank->of_node); +#endif + + ret = gpiochip_add_data(gc, bank); + if (ret) { + dev_err(bank->dev, "failed to add gpiochip %s, %d\n", gc->label, ret); + return ret; + } + + /* + * For DeviceTree-supported systems, the gpio core checks the + * pinctrl's device node for the "gpio-ranges" property. + * If it is present, it takes care of adding the pin ranges + * for the driver. In this case the driver can skip ahead. + * + * In order to remain compatible with older, existing DeviceTree + * files which don't set the "gpio-ranges" property or systems that + * utilize ACPI the driver has to call gpiochip_add_pin_range(). + */ + if (!of_property_read_bool(bank->of_node, "gpio-ranges")) { + struct device_node *pctlnp = of_get_parent(bank->of_node); + struct pinctrl_dev *pctldev = NULL; + + if (!pctlnp) + return -ENODATA; + + pctldev = of_pinctrl_get(pctlnp); + if (!pctldev) + return -ENODEV; + + ret = gpiochip_add_pin_range(gc, dev_name(pctldev->dev), 0, gc->base, gc->ngpio); + if (ret) { + dev_err(bank->dev, "Failed to add pin range\n"); + goto fail; + } + } + + ret = rockchip_interrupts_register(bank); + if (ret) { + dev_err(bank->dev, "failed to register interrupt, %d\n", ret); + goto fail; + } + + return 0; + +fail: + gpiochip_remove(&bank->gpio_chip); + + return ret; +} + +static int rockchip_get_bank_data(struct rockchip_pin_bank *bank) +{ + struct resource res; + int id = 0; + + if (of_address_to_resource(bank->of_node, 0, &res)) + return -ENOENT; + + bank->reg_base = devm_ioremap_resource(bank->dev, &res); + if (IS_ERR(bank->reg_base)) + return PTR_ERR(bank->reg_base); + + bank->irq = irq_of_parse_and_map(bank->of_node, 0); + if (!bank->irq) + return -EINVAL; + + bank->clk = of_clk_get(bank->of_node, 0); + if (IS_ERR(bank->clk)) + return PTR_ERR(bank->clk); + + clk_prepare_enable(bank->clk); + id = readl(bank->reg_base + gpio_regs_v2.version_id); + + /* If not gpio v2, that is default to v1. */ + if (id == GPIO_TYPE_V2) { + bank->gpio_regs = &gpio_regs_v2; + bank->gpio_type = GPIO_TYPE_V2; + bank->db_clk = of_clk_get(bank->of_node, 1); + if (IS_ERR(bank->db_clk)) { + dev_err(bank->dev, "cannot find debounce clk\n"); + bank->db_clk = NULL; + return -EINVAL; + } + } else { + bank->gpio_regs = &gpio_regs_v1; + bank->gpio_type = GPIO_TYPE_V1; + } + + return 0; +} + +static struct rockchip_pin_bank *rockchip_gpio_find_bank(struct pinctrl_dev *pctldev, int id) +{ + struct rockchip_pinctrl *info; + struct rockchip_pin_bank *bank; + int i, found = 0; + + info = pinctrl_dev_get_drvdata(pctldev); + bank = info->ctrl->pin_banks; + for (i = 0; i < info->ctrl->nr_banks; i++, bank++) { + if (bank->bank_num == id) { + found = 1; + break; + } + } + + return found ? bank : NULL; +} + +static int rockchip_gpio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = pdev->dev.of_node; + struct device_node *pctlnp = of_get_parent(np); + struct pinctrl_dev *pctldev = NULL; + struct rockchip_pin_bank *bank = NULL; + static int gpio; + int id, ret; + + if (!np || !pctlnp) + return -ENODEV; + + pctldev = of_pinctrl_get(pctlnp); + if (!pctldev) + return -EPROBE_DEFER; + + id = of_alias_get_id(np, "gpio"); + if (id < 0) + id = gpio++; + + bank = rockchip_gpio_find_bank(pctldev, id); + if (!bank) + return -EINVAL; + + bank->dev = dev; + bank->of_node = dev->of_node; + + raw_spin_lock_init(&bank->slock); + + ret = rockchip_get_bank_data(bank); + if (ret) + return ret; + + ret = rockchip_gpiolib_register(bank); + if (ret) + goto err_clk; + + platform_set_drvdata(pdev, bank); + dev_info(dev, "probed %s (%s)\n", bank->name, dev_name(dev)); + + return 0; +err_clk: + clk_disable_unprepare(bank->clk); + + return ret; +} + +static int rockchip_gpio_remove(struct platform_device *pdev) +{ + struct rockchip_pin_bank *bank = platform_get_drvdata(pdev); + + clk_disable_unprepare(bank->clk); + gpiochip_remove(&bank->gpio_chip); + + return 0; +} + +static const struct of_device_id rockchip_gpio_match[] = { + { .compatible = "rockchip,gpio-bank", }, + { .compatible = "rockchip,rk3188-gpio-bank0" }, + { }, +}; + +static struct platform_driver rockchip_gpio_driver = { + .probe = rockchip_gpio_probe, + .remove = rockchip_gpio_remove, + .driver = { + .name = "rockchip-gpio", + .of_match_table = rockchip_gpio_match, + }, +}; + +static int __init rockchip_gpio_init(void) +{ + return platform_driver_register(&rockchip_gpio_driver); +} +postcore_initcall(rockchip_gpio_init); + +static void __exit rockchip_gpio_exit(void) +{ + platform_driver_unregister(&rockchip_gpio_driver); +} +module_exit(rockchip_gpio_exit); + +MODULE_DESCRIPTION("Rockchip gpio driver"); +MODULE_ALIAS("platform:rockchip-gpio"); +MODULE_LICENSE("GPL v2"); +MODULE_DEVICE_TABLE(of, rockchip_gpio_match); diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 2e63274a4c2c..ab666917b1ab 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -1046,3 +1046,14 @@ void of_gpiochip_remove(struct gpio_chip *chip) { of_node_put(chip->of_node); } + +void of_gpio_dev_init(struct gpio_chip *gc, struct gpio_device *gdev) +{ + /* If the gpiochip has an assigned OF node this takes precedence */ + if (gc->of_node) + gdev->dev.of_node = gc->of_node; + else + gc->of_node = gdev->dev.of_node; + if (gdev->dev.of_node) + gdev->dev.fwnode = of_fwnode_handle(gdev->dev.of_node); +} diff --git a/drivers/gpio/gpiolib-of.h b/drivers/gpio/gpiolib-of.h index ed26664f1537..8af2bc899aab 100644 --- a/drivers/gpio/gpiolib-of.h +++ b/drivers/gpio/gpiolib-of.h @@ -15,6 +15,7 @@ int of_gpiochip_add(struct gpio_chip *gc); void of_gpiochip_remove(struct gpio_chip *gc); int of_gpio_get_count(struct device *dev, const char *con_id); bool of_gpio_need_valid_mask(const struct gpio_chip *gc); +void of_gpio_dev_init(struct gpio_chip *gc, struct gpio_device *gdev); #else static inline struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, @@ -33,6 +34,10 @@ static inline bool of_gpio_need_valid_mask(const struct gpio_chip *gc) { return false; } +static inline void of_gpio_dev_init(struct gpio_chip *gc, + struct gpio_device *gdev) +{ +} #endif /* CONFIG_OF_GPIO */ extern struct notifier_block gpio_of_notifier; diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile index 835c88318cec..b66e520ebbdd 100644 --- a/drivers/gpu/Makefile +++ b/drivers/gpu/Makefile @@ -3,6 +3,6 @@ # taken to initialize them in the correct order. Link order is the only way # to ensure this currently. obj-$(CONFIG_TEGRA_HOST1X) += host1x/ -obj-y += drm/ vga/ +obj-y += drm/ vga/ arm/ obj-$(CONFIG_IMX_IPUV3_CORE) += ipu-v3/ obj-$(CONFIG_TRACE_GPU_MEM) += trace/ diff --git a/drivers/gpu/arm/Kbuild b/drivers/gpu/arm/Kbuild new file mode 100755 index 000000000000..b3466f167d8e --- /dev/null +++ b/drivers/gpu/arm/Kbuild @@ -0,0 +1,31 @@ +# +# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + + +# obj-$(CONFIG_MALI_MIDGARD_FOR_LINUX) += midgard_for_linux/ + +obj-$(CONFIG_MALI_MIDGARD_FOR_ANDROID) += midgard/ + +obj-$(CONFIG_MALI400) += mali400/ + +obj-$(CONFIG_MALI_BIFROST_FOR_ANDROID) += bifrost/ + +obj-$(CONFIG_MALI_BIFROST_FOR_LINUX) += bifrost_for_linux/ diff --git a/drivers/gpu/arm/Kconfig b/drivers/gpu/arm/Kconfig new file mode 100755 index 000000000000..599711c18af9 --- /dev/null +++ b/drivers/gpu/arm/Kconfig @@ -0,0 +1,48 @@ +# +# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# +# +source "drivers/gpu/arm/mali400/mali/Kconfig" + +choice + prompt "Mali Midgard driver" + +config MALI_MIDGARD_FOR_ANDROID + bool "Mali Midgard for Android" + +config MALI_MIDGARD_FOR_LINUX + bool "Mali Midgard for Linux only" + +endchoice + +source "drivers/gpu/arm/midgard/Kconfig" + +choice + prompt "Mali bifrost driver" + +config MALI_BIFROST_FOR_ANDROID + bool "Mali Bifrost for Android" + +config MALI_BIFROST_FOR_LINUX + bool "Mali Bifrost for Linux only" + +endchoice + +source "drivers/gpu/arm/bifrost/Kconfig" diff --git a/drivers/gpu/arm/bifrost/Kbuild b/drivers/gpu/arm/bifrost/Kbuild new file mode 100755 index 000000000000..c05dc8399027 --- /dev/null +++ b/drivers/gpu/arm/bifrost/Kbuild @@ -0,0 +1,229 @@ +# +# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +# Driver version string which is returned to userspace via an ioctl +MALI_RELEASE_NAME ?= "g2p0-01eac0" + +# Paths required for build + +# make $(src) as absolute path if it isn't already, by prefixing $(srctree) +src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src)) +KBASE_PATH = $(src) +KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy +UMP_PATH = $(src)/../../../base + +# Set up defaults if not defined by build system +MALI_CUSTOMER_RELEASE ?= 1 +MALI_USE_CSF ?= 0 +MALI_UNIT_TEST ?= 0 +MALI_KERNEL_TEST_API ?= 0 +MALI_COVERAGE ?= 0 +MALI_JIT_PRESSURE_LIMIT_BASE ?= 1 +CONFIG_MALI_PLATFORM_NAME ?= "devicetree" +# Experimental features (corresponding -D definition should be appended to +# DEFINES below, e.g. for MALI_EXPERIMENTAL_FEATURE, +# -DMALI_EXPERIMENTAL_FEATURE=$(MALI_EXPERIMENTAL_FEATURE) should be appended) +# +# Experimental features must default to disabled, e.g.: +# MALI_EXPERIMENTAL_FEATURE ?= 0 +MALI_INCREMENTAL_RENDERING ?= 0 + +# Set up our defines, which will be passed to gcc +DEFINES = \ + -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ + -DMALI_USE_CSF=$(MALI_USE_CSF) \ + -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ + -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ + -DMALI_COVERAGE=$(MALI_COVERAGE) \ + -DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \ + -DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \ + -DMALI_INCREMENTAL_RENDERING=$(MALI_INCREMENTAL_RENDERING) + +ifeq ($(KBUILD_EXTMOD),) +# in-tree +DEFINES +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) +else +# out-of-tree +DEFINES +=-DMALI_KBASE_PLATFORM_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) +endif + +DEFINES += -I$(srctree)/drivers/staging/android + +DEFINES += -DMALI_KBASE_BUILD + +# Use our defines when compiling +ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux +subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux + +SRC := \ + context/mali_kbase_context.c \ + debug/mali_kbase_debug_ktrace.c \ + device/mali_kbase_device.c \ + device/mali_kbase_device_hw.c \ + mali_kbase_cache_policy.c \ + mali_kbase_ccswe.c \ + mali_kbase_mem.c \ + mali_kbase_mem_pool_group.c \ + mali_kbase_native_mgm.c \ + mali_kbase_ctx_sched.c \ + mali_kbase_jm.c \ + mali_kbase_gpuprops.c \ + mali_kbase_pm.c \ + mali_kbase_config.c \ + mali_kbase_vinstr.c \ + mali_kbase_hwcnt.c \ + mali_kbase_hwcnt_backend_jm.c \ + mali_kbase_hwcnt_gpu.c \ + mali_kbase_hwcnt_legacy.c \ + mali_kbase_hwcnt_types.c \ + mali_kbase_hwcnt_virtualizer.c \ + mali_kbase_softjobs.c \ + mali_kbase_hw.c \ + mali_kbase_debug.c \ + mali_kbase_gpu_memory_debugfs.c \ + mali_kbase_mem_linux.c \ + mali_kbase_core_linux.c \ + mali_kbase_mem_profile_debugfs.c \ + mmu/mali_kbase_mmu.c \ + mmu/mali_kbase_mmu_hw_direct.c \ + mmu/mali_kbase_mmu_mode_lpae.c \ + mmu/mali_kbase_mmu_mode_aarch64.c \ + mali_kbase_disjoint_events.c \ + mali_kbase_debug_mem_view.c \ + mali_kbase_smc.c \ + mali_kbase_mem_pool.c \ + mali_kbase_mem_pool_debugfs.c \ + mali_kbase_debugfs_helper.c \ + mali_kbase_strings.c \ + mali_kbase_as_fault_debugfs.c \ + mali_kbase_regs_history_debugfs.c \ + mali_power_gpu_frequency_trace.c \ + mali_kbase_trace_gpu_mem.c \ + thirdparty/mali_kbase_mmap.c \ + tl/mali_kbase_timeline.c \ + tl/mali_kbase_timeline_io.c \ + tl/mali_kbase_tlstream.c \ + tl/mali_kbase_tracepoints.c \ + gpu/mali_kbase_gpu.c + +ifeq ($(MALI_USE_CSF),1) + SRC += \ + debug/backend/mali_kbase_debug_ktrace_csf.c \ + device/backend/mali_kbase_device_csf.c \ + device/backend/mali_kbase_device_hw_csf.c \ + gpu/backend/mali_kbase_gpu_fault_csf.c \ + tl/backend/mali_kbase_timeline_csf.c \ + mmu/backend/mali_kbase_mmu_csf.c \ + context/backend/mali_kbase_context_csf.c +else + SRC += \ + mali_kbase_dummy_job_wa.c \ + mali_kbase_debug_job_fault.c \ + mali_kbase_event.c \ + mali_kbase_jd.c \ + mali_kbase_jd_debugfs.c \ + mali_kbase_js.c \ + mali_kbase_js_ctx_attr.c \ + mali_kbase_kinstr_jm.c \ + debug/backend/mali_kbase_debug_ktrace_jm.c \ + device/backend/mali_kbase_device_jm.c \ + device/backend/mali_kbase_device_hw_jm.c \ + gpu/backend/mali_kbase_gpu_fault_jm.c \ + tl/backend/mali_kbase_timeline_jm.c \ + mmu/backend/mali_kbase_mmu_jm.c \ + context/backend/mali_kbase_context_jm.c +endif + +ifeq ($(CONFIG_MALI_CINSTR_GWT),y) + SRC += mali_kbase_gwt.c +endif + +ifeq ($(MALI_UNIT_TEST),1) + SRC += tl/mali_kbase_timeline_test.c +endif + +ifeq ($(MALI_CUSTOMER_RELEASE),0) + SRC += mali_kbase_regs_dump_debugfs.c +endif + + +ccflags-y += -I$(KBASE_PATH) -I$(KBASE_PATH)/debug \ + -I$(KBASE_PATH)/debug/backend + +# Tell the Linux build system from which .o file to create the kernel module +obj-$(CONFIG_MALI_BIFROST) += bifrost_kbase.o + +# Tell the Linux build system to enable building of our .c files +bifrost_kbase-y := $(SRC:.c=.o) + +# Kconfig passes in the name with quotes for in-tree builds - remove them. +platform_name := $(shell echo $(CONFIG_MALI_PLATFORM_NAME)) +MALI_PLATFORM_DIR := platform/$(platform_name) +ccflags-y += -I$(src)/$(MALI_PLATFORM_DIR) +include $(src)/$(MALI_PLATFORM_DIR)/Kbuild + +ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y) + ifeq ($(CONFIG_DEVFREQ_THERMAL),y) + include $(src)/ipa/Kbuild + endif +endif + +ifeq ($(MALI_USE_CSF),1) + include $(src)/csf/Kbuild +else +# empty +endif + +ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y) + include $(src)/arbiter/Kbuild +else +# empty +endif + +ifeq ($(MALI_USE_CSF),0) + bifrost_kbase-$(CONFIG_MALI_BIFROST_DMA_FENCE) += \ + mali_kbase_fence_ops.o \ + mali_kbase_dma_fence.o \ + mali_kbase_fence.o + + bifrost_kbase-$(CONFIG_SYNC_FILE) += \ + mali_kbase_fence_ops.o \ + mali_kbase_fence.o +endif + +bifrost_kbase-$(CONFIG_SYNC) += \ + mali_kbase_sync_android.o \ + mali_kbase_sync_common.o + +bifrost_kbase-$(CONFIG_SYNC_FILE) += \ + mali_kbase_fence_ops.o \ + mali_kbase_sync_file.o \ + mali_kbase_sync_common.o + +include $(src)/backend/gpu/Kbuild +bifrost_kbase-y += $(BACKEND:.c=.o) + + +ccflags-y += -I$(src)/backend/gpu +subdir-ccflags-y += -I$(src)/backend/gpu + +# For kutf and mali_kutf_irq_latency_test +obj-$(CONFIG_MALI_KUTF) += tests/ diff --git a/drivers/gpu/arm/bifrost/Kconfig b/drivers/gpu/arm/bifrost/Kconfig new file mode 100755 index 000000000000..ccb16671047a --- /dev/null +++ b/drivers/gpu/arm/bifrost/Kconfig @@ -0,0 +1,308 @@ +# +# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + + +menuconfig MALI_BIFROST + tristate "Mali Bifrost series support" + select GPU_TRACEPOINTS if ANDROID + select DMA_SHARED_BUFFER + default n + help + Enable this option to build support for a ARM Mali Bifrost GPU. + + To compile this driver as a module, choose M here: + this will generate a single module, called mali_kbase. + +config MALI_BIFROST_GATOR_SUPPORT + bool "Enable Streamline tracing support" + depends on MALI_BIFROST + default n + help + Enables kbase tracing used by the Arm Streamline Performance Analyzer. + The tracepoints are used to derive GPU activity charts in Streamline. + +config MALI_BIFROST_DVFS + bool "Enable legacy DVFS" + depends on MALI_BIFROST && !MALI_BIFROST_DEVFREQ + default n + help + Choose this option to enable legacy DVFS in the Mali Midgard DDK. + +config MALI_BIFROST_ENABLE_TRACE + bool "Enable kbase tracing" + depends on MALI_BIFROST + default y if MALI_BIFROST_DEBUG + default n + help + Enables tracing in kbase. Trace log available through + the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled + +config MALI_BIFROST_DEVFREQ + bool "devfreq support for Mali" + depends on MALI_BIFROST && PM_DEVFREQ + help + Support devfreq for Mali. + + Using the devfreq framework and, by default, the simpleondemand + governor, the frequency of Mali will be dynamically selected from the + available OPPs. + +config MALI_BIFROST_DMA_FENCE + bool "DMA_BUF fence support for Mali" + depends on MALI_BIFROST + default n + help + Support DMA_BUF fences for Mali. + + This option should only be enabled if the Linux Kernel has built in + support for DMA_BUF fences. + +config MALI_PLATFORM_NAME + depends on MALI_BIFROST + string "Platform name" + default "devicetree" + help + Enter the name of the desired platform configuration directory to + include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must + exist. + +config MALI_ARBITER_SUPPORT + bool "Enable arbiter support for Mali" + depends on MALI_BIFROST + default n + help + Enable support for the arbiter interface in the driver. + This allows an external arbiter to manage driver access + to GPU hardware in a virtualized environment + + If unsure, say N. + +# MALI_BIFROST_EXPERT configuration options + +menuconfig MALI_BIFROST_EXPERT + depends on MALI_BIFROST + bool "Enable Expert Settings" + default n + help + Enabling this option and modifying the default settings may produce a driver with performance or + other limitations. + +config MALI_CORESTACK + bool "Support controlling power to the GPU core stack" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Enabling this feature on supported GPUs will let the driver powering + on/off the GPU core stack independently without involving the Power + Domain Controller. This should only be enabled on platforms which + integration of the PDC to the Mali GPU is known to be problematic. + This feature is currently only supported on t-Six and t-HEx GPUs. + + If unsure, say N. + +config MALI_BIFROST_DEBUG + bool "Debug build" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Select this option for increased checking and reporting of errors. + +config MALI_BIFROST_FENCE_DEBUG + bool "Debug sync fence usage" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT && (SYNC || SYNC_FILE) + default y if MALI_BIFROST_DEBUG + help + Select this option to enable additional checking and reporting on the + use of sync fences in the Mali driver. + + This will add a 3s timeout to all sync fence waits in the Mali + driver, so that when work for Mali has been waiting on a sync fence + for a long time a debug message will be printed, detailing what fence + is causing the block, and which dependent Mali atoms are blocked as a + result of this. + + The timeout can be changed at runtime through the js_soft_timeout + device attribute, where the timeout is specified in milliseconds. + +config MALI_BIFROST_NO_MALI + bool "No Mali" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + This can be used to test the driver in a simulated environment + whereby the hardware is not physically present. If the hardware is physically + present it will not be used. This can be used to test the majority of the + driver without needing actual hardware or for software benchmarking. + All calls to the simulated hardware will complete immediately as if the hardware + completed the task. + +config MALI_REAL_HW + def_bool !MALI_BIFROST_NO_MALI + +config MALI_BIFROST_ERROR_INJECT + bool "Error injection" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT && MALI_BIFROST_NO_MALI + default n + help + Enables insertion of errors to test module failure and recovery mechanisms. + +config MALI_BIFROST_SYSTEM_TRACE + bool "Enable system event tracing support" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default y if MALI_BIFROST_DEBUG + default n + help + Choose this option to enable system trace events for each + kbase event. This is typically used for debugging but has + minimal overhead when not in use. Enable only if you know what + you are doing. + +config MALI_2MB_ALLOC + bool "Attempt to allocate 2MB pages" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Rather than allocating all GPU memory page-by-page, attempt to + allocate 2MB pages from the kernel. This reduces TLB pressure and + helps to prevent memory fragmentation. + + If in doubt, say N + +config MALI_PWRSOFT_765 + bool "PWRSOFT-765 ticket" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + PWRSOFT-765 fixes devfreq cooling devices issues. The fix was merged + in kernel v4.10, however if backported into the kernel then this + option must be manually selected. + + If using kernel >= v4.10 then say N, otherwise if devfreq cooling + changes have been backported say Y to avoid compilation errors. + +config MALI_MEMORY_FULLY_BACKED + bool "Memory fully physically-backed" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + This option enables full physical backing of all virtual + memory allocations in the kernel. Notice that this build + option only affects allocations of grow-on-GPU-page-fault + memory. + +config MALI_DMA_BUF_MAP_ON_DEMAND + bool "Map imported dma-bufs on demand" + depends on MALI_BIFROST + default n + help + This option caused kbase to set up the GPU mapping of imported + dma-buf when needed to run atoms. This is the legacy behaviour. + + This is intended for testing and the option will get removed in the + future. + +config MALI_DMA_BUF_LEGACY_COMPAT + bool "Enable legacy compatibility cache flush on dma-buf map" + depends on MALI_BIFROST && !MALI_DMA_BUF_MAP_ON_DEMAND + default n + help + This option enables compatibility with legacy dma-buf mapping + behavior, then the dma-buf is mapped on import, by adding cache + maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping, + including a cache flush. + + This option might work-around issues related to missing cache + flushes in other drivers. This only has an effect for clients using + UK 11.18 or older. For later UK versions it is not possible. + +config MALI_HW_ERRATA_1485982_NOT_AFFECTED + bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + This option disables the default workaround for GPU2017-1336. The + workaround keeps the L2 cache powered up except for powerdown and reset. + + The workaround introduces a limitation that will prevent the running of + protected mode content on fully coherent platforms, as the switch to IO + coherency mode requires the L2 to be turned off. + +config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE + bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED + default n + help + This option uses an alternative workaround for GPU2017-1336. Lowering + the GPU clock to a, platform specific, known good frequeuncy before + powering down the L2 cache. The clock can be specified in the device + tree using the property, opp-mali-errata-1485982. Otherwise the + slowest clock will be selected. + +config MALI_GEM5_BUILD + bool "Enable build of Mali kernel driver for GEM5" + depends on MALI_BIFROST + default n + help + This option is to do a Mali GEM5 build. + If unsure, say N. + +# Instrumentation options. + +config MALI_JOB_DUMP + bool "Enable system level support needed for job dumping" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Choose this option to enable system level support needed for + job dumping. This is typically used for instrumentation but has + minimal overhead when not in use. Enable only if you know what + you are doing. + +config MALI_BIFROST_PRFCNT_SET_SECONDARY + bool "Use secondary set of performance counters" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Select this option to use secondary set of performance counters. Kernel + features that depend on an access to the primary set of counters may + become unavailable. Enabling this option will prevent power management + from working optimally and may cause instrumentation tools to return + bogus results. + + If unsure, say N. + +config MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS + bool "Use secondary set of performance counters" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_BIFROST_PRFCNT_SET_SECONDARY && DEBUG_FS + default n + help + Select this option to make the secondary set of performance counters + available at runtime via debugfs. Kernel features that depend on an + access to the primary set of counters may become unavailable. + + This feature is unsupported and unstable, and may break at any time. + Enabling this option will prevent power management from working + optimally and may cause instrumentation tools to return bogus results. + + If unsure, say N. + +source "drivers/gpu/arm/midgard/platform/Kconfig" +# source "drivers/gpu/arm/midgard/tests/Kconfig" diff --git a/drivers/gpu/arm/bifrost/Makefile b/drivers/gpu/arm/bifrost/Makefile new file mode 100755 index 000000000000..53a12094ec14 --- /dev/null +++ b/drivers/gpu/arm/bifrost/Makefile @@ -0,0 +1,38 @@ +# +# (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + + +KDIR ?= /lib/modules/$(shell uname -r)/build + +BUSLOG_PATH_RELATIVE = $(CURDIR)/../../../.. +KBASE_PATH_RELATIVE = $(CURDIR) + +ifeq ($(CONFIG_MALI_BUSLOG),y) +#Add bus logger symbols +EXTRA_SYMBOLS += $(BUSLOG_PATH_RELATIVE)/drivers/base/bus_logger/Module.symvers +endif + +# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions +all: + $(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules + +clean: + $(MAKE) -C $(KDIR) M=$(CURDIR) clean diff --git a/drivers/gpu/arm/bifrost/Makefile.kbase b/drivers/gpu/arm/bifrost/Makefile.kbase new file mode 100755 index 000000000000..6b0f81ee76e8 --- /dev/null +++ b/drivers/gpu/arm/bifrost/Makefile.kbase @@ -0,0 +1,23 @@ +# +# (C) COPYRIGHT 2010, 2013, 2018 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +EXTRA_CFLAGS += -I$(ROOT) -I$(KBASE_PATH) -I$(KBASE_PATH)/platform_$(PLATFORM) + diff --git a/drivers/gpu/arm/bifrost/Mconfig b/drivers/gpu/arm/bifrost/Mconfig new file mode 100755 index 000000000000..99ababfc2d16 --- /dev/null +++ b/drivers/gpu/arm/bifrost/Mconfig @@ -0,0 +1,277 @@ +# +# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +menuconfig MALI_BIFROST + bool "Mali Midgard series support" + default y + help + Enable this option to build support for a ARM Mali Midgard GPU. + + To compile this driver as a module, choose M here: + this will generate a single module, called mali_kbase. + +config MALI_BIFROST_GATOR_SUPPORT + bool "Enable Streamline tracing support" + depends on MALI_BIFROST && !BACKEND_USER + default y + help + Enables kbase tracing used by the Arm Streamline Performance Analyzer. + The tracepoints are used to derive GPU activity charts in Streamline. + +config MALI_BIFROST_DVFS + bool "Enable legacy DVFS" + depends on MALI_BIFROST && !MALI_BIFROST_DEVFREQ + default n + help + Choose this option to enable legacy DVFS in the Mali Midgard DDK. + +config MALI_BIFROST_ENABLE_TRACE + bool "Enable kbase tracing" + default y if MALI_BIFROST_DEBUG + default n + help + Enables tracing in kbase. Trace log available through + the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled + +config MALI_BIFROST_DEVFREQ + bool "devfreq support for Mali" + depends on MALI_BIFROST + default y if PLATFORM_JUNO + default y if PLATFORM_CUSTOM + help + Support devfreq for Mali. + + Using the devfreq framework and, by default, the simpleondemand + governor, the frequency of Mali will be dynamically selected from the + available OPPs. + +config MALI_BIFROST_DMA_FENCE + bool "DMA_BUF fence support for Mali" + depends on MALI_BIFROST + default n + help + Support DMA_BUF fences for Mali. + + This option should only be enabled if the Linux Kernel has built in + support for DMA_BUF fences. + +config MALI_PLATFORM_NAME + depends on MALI_BIFROST + string "Platform name" + default "hisilicon" if PLATFORM_HIKEY960 + default "hisilicon" if PLATFORM_HIKEY970 + default "devicetree" + help + Enter the name of the desired platform configuration directory to + include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must + exist. + + When PLATFORM_CUSTOM is set, this needs to be set manually to + pick up the desired platform files. + +config MALI_ARBITER_SUPPORT + bool "Enable arbiter support for Mali" + depends on MALI_BIFROST && !GPU_HAS_CSF + default n + help + Enable support for the arbiter interface in the driver. + This allows an external arbiter to manage driver access + to GPU hardware in a virtualized environment + + If unsure, say N. + +# MALI_BIFROST_EXPERT configuration options + +menuconfig MALI_BIFROST_EXPERT + depends on MALI_BIFROST + bool "Enable Expert Settings" + default y + help + Enabling this option and modifying the default settings may produce a driver with performance or + other limitations. + +config MALI_CORESTACK + bool "Support controlling power to the GPU core stack" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Enabling this feature on supported GPUs will let the driver powering + on/off the GPU core stack independently without involving the Power + Domain Controller. This should only be enabled on platforms which + integration of the PDC to the Mali GPU is known to be problematic. + This feature is currently only supported on t-Six and t-HEx GPUs. + + If unsure, say N. + +config MALI_BIFROST_DEBUG + bool "Debug build" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default y if DEBUG + default n + help + Select this option for increased checking and reporting of errors. + +config MALI_BIFROST_FENCE_DEBUG + bool "Debug sync fence usage" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default y if MALI_BIFROST_DEBUG + help + Select this option to enable additional checking and reporting on the + use of sync fences in the Mali driver. + + This will add a 3s timeout to all sync fence waits in the Mali + driver, so that when work for Mali has been waiting on a sync fence + for a long time a debug message will be printed, detailing what fence + is causing the block, and which dependent Mali atoms are blocked as a + result of this. + + The timeout can be changed at runtime through the js_soft_timeout + device attribute, where the timeout is specified in milliseconds. + +choice + prompt "Error injection level" + default MALI_ERROR_INJECT_NONE + help + Enables insertion of errors to test module failure and recovery mechanisms. + +config MALI_ERROR_INJECT_NONE + bool "disabled" + help + Error injection is disabled. + +config MALI_ERROR_INJECT_TRACK_LIST + bool "error track list" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT && NO_MALI + help + Errors to inject are pre-configured by the user. + +config MALI_ERROR_INJECT_RANDOM + bool "random error injection" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT && NO_MALI + help + Injected errors are random, rather than user-driven. + +endchoice + +config MALI_ERROR_INJECT_ON + string + default "0" if MALI_ERROR_INJECT_NONE + default "1" if MALI_ERROR_INJECT_TRACK_LIST + default "2" if MALI_ERROR_INJECT_RANDOM + +config MALI_BIFROST_ERROR_INJECT + bool + default y if !MALI_ERROR_INJECT_NONE + +config MALI_BIFROST_SYSTEM_TRACE + bool "Enable system event tracing support" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default y if MALI_BIFROST_DEBUG + default n + help + Choose this option to enable system trace events for each + kbase event. This is typically used for debugging but has + minimal overhead when not in use. Enable only if you know what + you are doing. + +config MALI_2MB_ALLOC + bool "Attempt to allocate 2MB pages" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Rather than allocating all GPU memory page-by-page, attempt to + allocate 2MB pages from the kernel. This reduces TLB pressure and + helps to prevent memory fragmentation. + + If in doubt, say N + +config MALI_PWRSOFT_765 + bool "PWRSOFT-765 ticket" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + PWRSOFT-765 fixes devfreq cooling devices issues. However, they are + not merged in mainline kernel yet. So this define helps to guard those + parts of the code. + +config MALI_MEMORY_FULLY_BACKED + bool "Memory fully physically-backed" + default n + help + This option enables full backing of all virtual memory allocations + for the kernel. This only affects grow-on-GPU-page-fault memory. + +config MALI_DMA_BUF_MAP_ON_DEMAND + bool "Map imported dma-bufs on demand" + depends on MALI_BIFROST + default n + default y if !DMA_BUF_SYNC_IOCTL_SUPPORTED + help + This option caused kbase to set up the GPU mapping of imported + dma-buf when needed to run atoms. This is the legacy behaviour. + +config MALI_DMA_BUF_LEGACY_COMPAT + bool "Enable legacy compatibility cache flush on dma-buf map" + depends on MALI_BIFROST && !MALI_DMA_BUF_MAP_ON_DEMAND + default n + help + This option enables compatibility with legacy dma-buf mapping + behavior, then the dma-buf is mapped on import, by adding cache + maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping, + including a cache flush. + +config MALI_REAL_HW + bool + default y + default n if NO_MALI + +config MALI_HW_ERRATA_1485982_NOT_AFFECTED + bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + default y if PLATFORM_JUNO + help + This option disables the default workaround for GPU2017-1336. The + workaround keeps the L2 cache powered up except for powerdown and reset. + + The workaround introduces a limitation that will prevent the running of + protected mode content on fully coherent platforms, as the switch to IO + coherency mode requires the L2 to be turned off. + +config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE + bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED + default n + help + This option uses an alternative workaround for GPU2017-1336. Lowering + the GPU clock to a, platform specific, known good frequeuncy before + powering down the L2 cache. The clock can be specified in the device + tree using the property, opp-mali-errata-1485982. Otherwise the + slowest clock will be selected. + +config MALI_GEM5_BUILD + bool "Enable build of Mali kernel driver for GEM5" + depends on MALI_BIFROST + default n + help + This option is to do a Mali GEM5 build. + If unsure, say N. + +# Instrumentation options. + +# config MALI_JOB_DUMP exists in the Kernel Kconfig but is configured using CINSTR_JOB_DUMP in Mconfig. +# config MALI_BIFROST_PRFCNT_SET_SECONDARY exists in the Kernel Kconfig but is configured using CINSTR_SECONDARY_HWC in Mconfig. + +source "kernel/drivers/gpu/arm/midgard/tests/Mconfig" diff --git a/drivers/gpu/arm/bifrost/arbiter/Kbuild b/drivers/gpu/arm/bifrost/arbiter/Kbuild new file mode 100755 index 000000000000..98e47bed223a --- /dev/null +++ b/drivers/gpu/arm/bifrost/arbiter/Kbuild @@ -0,0 +1,24 @@ +# +# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +mali_kbase-y += \ + arbiter/mali_kbase_arbif.o \ + arbiter/mali_kbase_arbiter_pm.o diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c new file mode 100755 index 000000000000..ddf1a0ce0b05 --- /dev/null +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file mali_kbase_arbif.c + * Mali arbiter interface APIs to share GPU between Virtual Machines + */ + +#include +#include "mali_kbase_arbif.h" +#include +#include +#include +#include "mali_kbase_arbiter_interface.h" + +static void on_gpu_stop(struct device *dev) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED(kbdev, kbdev); + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT); +} + +static void on_gpu_granted(struct device *dev) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + KBASE_TLSTREAM_TL_ARBITER_GRANTED(kbdev, kbdev); + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT); +} + +static void on_gpu_lost(struct device *dev) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT); +} + +int kbase_arbif_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_OF + struct arbiter_if_arb_vm_ops ops; + struct arbiter_if_dev *arb_if; + struct device_node *arbiter_if_node; + struct platform_device *pdev; + int err; + + dev_dbg(kbdev->dev, "%s\n", __func__); + + arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, + "arbiter_if", 0); + if (!arbiter_if_node) { + dev_dbg(kbdev->dev, "No arbiter_if in Device Tree\n"); + /* no arbiter interface defined in device tree */ + kbdev->arb.arb_dev = NULL; + kbdev->arb.arb_if = NULL; + return 0; + } + + pdev = of_find_device_by_node(arbiter_if_node); + if (!pdev) { + dev_err(kbdev->dev, "Failed to find arbiter_if device\n"); + return -EPROBE_DEFER; + } + + if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) { + dev_err(kbdev->dev, "arbiter_if driver not available\n"); + return -EPROBE_DEFER; + } + kbdev->arb.arb_dev = &pdev->dev; + arb_if = platform_get_drvdata(pdev); + if (!arb_if) { + dev_err(kbdev->dev, "arbiter_if driver not ready\n"); + module_put(pdev->dev.driver->owner); + return -EPROBE_DEFER; + } + + kbdev->arb.arb_if = arb_if; + ops.arb_vm_gpu_stop = on_gpu_stop; + ops.arb_vm_gpu_granted = on_gpu_granted; + ops.arb_vm_gpu_lost = on_gpu_lost; + + /* register kbase arbiter_if callbacks */ + if (arb_if->vm_ops.vm_arb_register_dev) { + err = arb_if->vm_ops.vm_arb_register_dev(arb_if, + kbdev->dev, &ops); + if (err) { + dev_err(&pdev->dev, "Failed to register with arbiter\n"); + module_put(pdev->dev.driver->owner); + return err; + } + } +#else /* CONFIG_OF */ + dev_dbg(kbdev->dev, "No arbiter without Device Tree support\n"); + kbdev->arb.arb_dev = NULL; + kbdev->arb.arb_if = NULL; +#endif + return 0; +} + +void kbase_arbif_destroy(struct kbase_device *kbdev) +{ + struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; + + if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) { + dev_dbg(kbdev->dev, "%s\n", __func__); + arb_if->vm_ops.vm_arb_unregister_dev(kbdev->arb.arb_if); + } + kbdev->arb.arb_if = NULL; + if (kbdev->arb.arb_dev) + module_put(kbdev->arb.arb_dev->driver->owner); + kbdev->arb.arb_dev = NULL; +} + +void kbase_arbif_gpu_request(struct kbase_device *kbdev) +{ + struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; + + if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) { + dev_dbg(kbdev->dev, "%s\n", __func__); + arb_if->vm_ops.vm_arb_gpu_request(arb_if); + } +} + +void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required) +{ + struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; + + if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) { + dev_dbg(kbdev->dev, "%s\n", __func__); + KBASE_TLSTREAM_TL_ARBITER_STOPPED(kbdev, kbdev); + arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required); + } +} + +void kbase_arbif_gpu_active(struct kbase_device *kbdev) +{ + struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; + + if (arb_if && arb_if->vm_ops.vm_arb_gpu_active) { + dev_dbg(kbdev->dev, "%s\n", __func__); + arb_if->vm_ops.vm_arb_gpu_active(arb_if); + } +} + +void kbase_arbif_gpu_idle(struct kbase_device *kbdev) +{ + struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; + + if (arb_if && arb_if->vm_ops.vm_arb_gpu_idle) { + dev_dbg(kbdev->dev, "vm_arb_gpu_idle\n"); + arb_if->vm_ops.vm_arb_gpu_idle(arb_if); + } +} diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h new file mode 100755 index 000000000000..e7e9de76c94c --- /dev/null +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h @@ -0,0 +1,133 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * + */ + +/** + * @file + * Mali arbiter interface APIs to share GPU between Virtual Machines + */ + +#ifndef _MALI_KBASE_ARBIF_H_ +#define _MALI_KBASE_ARBIF_H_ + +/** + * enum kbase_arbif_evt - Internal Arbiter event. + * + * @KBASE_VM_GPU_INITIALIZED_EVT: KBase has finished initializing + * and can be stopped + * @KBASE_VM_GPU_STOP_EVT: Stop message received from Arbiter + * @KBASE_VM_GPU_GRANTED_EVT: Grant message received from Arbiter + * @KBASE_VM_GPU_LOST_EVT: Lost message received from Arbiter + * @KBASE_VM_GPU_IDLE_EVENT: KBase has transitioned into an inactive state. + * @KBASE_VM_REF_EVENT: KBase has transitioned into an active state. + * @KBASE_VM_OS_SUSPEND_EVENT: KBase is suspending + * @KBASE_VM_OS_RESUME_EVENT: Kbase is resuming + */ +enum kbase_arbif_evt { + KBASE_VM_GPU_INITIALIZED_EVT = 1, + KBASE_VM_GPU_STOP_EVT, + KBASE_VM_GPU_GRANTED_EVT, + KBASE_VM_GPU_LOST_EVT, + KBASE_VM_GPU_IDLE_EVENT, + KBASE_VM_REF_EVENT, + KBASE_VM_OS_SUSPEND_EVENT, + KBASE_VM_OS_RESUME_EVENT, +}; + +/** + * kbase_arbif_init() - Initialize the arbiter interface functionality. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Initialize the arbiter interface and also determines + * if Arbiter functionality is required. + * + * Return: 0 if the Arbiter interface was successfully initialized or the + * Arbiter was not required. + */ +int kbase_arbif_init(struct kbase_device *kbdev); + +/** + * kbase_arbif_destroy() - Cleanups the arbiter interface functionality. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Cleans up the arbiter interface functionality and resets the reference count + * of the arbif module used + */ +void kbase_arbif_destroy(struct kbase_device *kbdev); + +/** + * kbase_arbif_gpu_request() - Send GPU request message to the arbiter + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Sends a message to Arbiter to request GPU access. + */ +void kbase_arbif_gpu_request(struct kbase_device *kbdev); + +/** + * kbase_arbif_gpu_stopped() - Send GPU stopped message to the arbiter + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @gpu_required: true if GPU access is still required + * (Arbiter will automatically send another grant message) + * + * Sends a message to Arbiter to notify that the GPU has stopped. + * @note Once this call has been made, KBase must not attempt to access the GPU + * until the #KBASE_VM_GPU_GRANTED_EVT event has been received. + */ +void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required); + +/** + * kbase_arbif_gpu_active() - Send a GPU active message to the arbiter + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Sends a message to Arbiter to report that KBase has gone active. + */ +void kbase_arbif_gpu_active(struct kbase_device *kbdev); + +/** + * kbase_arbif_gpu_idle() - Send a GPU idle message to the arbiter + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Sends a message to Arbiter to report that KBase has gone idle. + */ +void kbase_arbif_gpu_idle(struct kbase_device *kbdev); + +#endif /* _MALI_KBASE_ARBIF_H_ */ diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_defs.h b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_defs.h new file mode 100755 index 000000000000..1f53cbf1a286 --- /dev/null +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_defs.h @@ -0,0 +1,95 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * + */ + +/** + * @file + * Mali structures define to support arbitration feature + */ + +#ifndef _MALI_KBASE_ARBITER_DEFS_H_ +#define _MALI_KBASE_ARBITER_DEFS_H_ + +#include "mali_kbase_arbiter_pm.h" + +/** + * struct kbase_arbiter_vm_state - Struct representing the state and containing the + * data of pm work + * @kbdev: Pointer to kbase device structure (must be a valid pointer) + * @vm_state_lock: The lock protecting the VM state when arbiter is used. + * This lock must also be held whenever the VM state is being + * transitioned + * @vm_state_wait: Wait queue set when GPU is granted + * @vm_state: Current state of VM + * @vm_arb_wq: Work queue for resuming or stopping work on the GPU for use + * with the Arbiter + * @vm_suspend_work: Work item for vm_arb_wq to stop current work on GPU + * @vm_resume_work: Work item for vm_arb_wq to resume current work on GPU + * @vm_arb_starting: Work queue resume in progress + * @vm_arb_stopping: Work queue suspend in progress + * @vm_arb_users_waiting: Count of users waiting for GPU + */ +struct kbase_arbiter_vm_state { + struct kbase_device *kbdev; + struct mutex vm_state_lock; + wait_queue_head_t vm_state_wait; + enum kbase_vm_state vm_state; + struct workqueue_struct *vm_arb_wq; + struct work_struct vm_suspend_work; + struct work_struct vm_resume_work; + bool vm_arb_starting; + bool vm_arb_stopping; + int vm_arb_users_waiting; +}; + +/** + * struct kbase_arbiter_device - Representing an instance of arbiter device, + * allocated from the probe method of Mali driver + * @arb_if: Pointer to the arbiter interface device + * @arb_dev: Pointer to the arbiter device + */ +struct kbase_arbiter_device { + struct arbiter_if_dev *arb_if; + struct device *arb_dev; +}; + +#endif /* _MALI_KBASE_ARBITER_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_interface.h b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_interface.h new file mode 100755 index 000000000000..5d5d8a7d2cff --- /dev/null +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_interface.h @@ -0,0 +1,181 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * + */ + +/** + * @file + * Defines the Mali arbiter interface + */ + +#ifndef _MALI_KBASE_ARBITER_INTERFACE_H_ +#define _MALI_KBASE_ARBITER_INTERFACE_H_ + +/** + * @brief Mali arbiter interface version + * + * This specifies the current version of the configuration interface. Whenever + * the arbiter interface changes, so that integration effort is required, the + * version number will be increased. Each configuration must make an effort + * to check that it implements the correct version. + * + * Version history: + * 1 - Added the Mali arbiter configuration interface. + * 2 - Strip out reference code from header + * 3 - Removed DVFS utilization interface (DVFS moved to arbiter side) + */ +#define MALI_KBASE_ARBITER_INTERFACE_VERSION 3 + +struct arbiter_if_dev; + +/** + * struct arbiter_if_arb_vm_ops - Interface to communicate messages to VM + * + * This struct contains callbacks used to deliver messages + * from the arbiter to the corresponding VM. + * + * Note that calls into these callbacks may have synchronous calls back into + * the arbiter arbiter_if_vm_arb_ops callbacks below. + * For example vm_arb_gpu_stopped() may be called as a side effect of + * arb_vm_gpu_stop() being called here. + */ +struct arbiter_if_arb_vm_ops { + /** + * arb_vm_gpu_stop() - Ask VM to stop using GPU + * @dev: The arbif kernel module device. + * + * Informs KBase to stop using the GPU as soon as possible. + * @Note: Once the driver is no longer using the GPU, a call to + * vm_arb_gpu_stopped is expected by the arbiter. + */ + void (*arb_vm_gpu_stop)(struct device *dev); + + /** + * arb_vm_gpu_granted() - GPU has been granted to VM + * @dev: The arbif kernel module device. + * + * Informs KBase that the GPU can now be used by the VM. + */ + void (*arb_vm_gpu_granted)(struct device *dev); + + /** + * arb_vm_gpu_lost() - VM has lost the GPU + * @dev: The arbif kernel module device. + * + * This is called if KBase takes too long to respond to the arbiter + * stop request. + * Once this is called, KBase will assume that access to the GPU + * has been lost and will fail all running jobs and reset its + * internal state. + * If successful, will respond with a vm_arb_gpu_stopped message. + */ + void (*arb_vm_gpu_lost)(struct device *dev); +}; + +/** + * struct arbiter_if_vm_arb_ops - Interface to communicate messages to arbiter + * + * This struct contains callbacks used to request operations + * from the VM to the arbiter + * + * Note that we must not make any synchronous calls back in to the VM + * (via arbiter_if_arb_vm_ops above) in the context of these callbacks. + */ +struct arbiter_if_vm_arb_ops { + /** + * vm_arb_register_dev() - Register VM device driver callbacks. + * @arbif_dev: The arbiter interface we are registering device callbacks + * @dev: The device structure to supply in the callbacks. + * @ops: The callbacks that the device driver supports + * (none are optional). + */ + int (*vm_arb_register_dev)(struct arbiter_if_dev *arbif_dev, + struct device *dev, struct arbiter_if_arb_vm_ops *ops); + + /** + * vm_arb_unregister_dev() - Unregister VM device driver callbacks. + * @arbif_dev: The arbiter interface we are unregistering from. + */ + void (*vm_arb_unregister_dev)(struct arbiter_if_dev *arbif_dev); + + /** + * vm_arb_gpu_request() - Ask the arbiter interface for GPU access. + * @arbif_dev: The arbiter interface we want to issue the request. + */ + void (*vm_arb_gpu_request)(struct arbiter_if_dev *arbif_dev); + + /** + * vm_arb_gpu_active() - Inform arbiter that the driver has gone active + * @arbif_dev: The arbiter interface device. + */ + void (*vm_arb_gpu_active)(struct arbiter_if_dev *arbif_dev); + + /** + * vm_arb_gpu_idle() - Inform the arbiter that the driver has gone idle + * @arbif_dev: The arbiter interface device. + */ + void (*vm_arb_gpu_idle)(struct arbiter_if_dev *arbif_dev); + + /** + * vm_arb_gpu_stopped() - Inform the arbiter that the driver has stopped + * using the GPU + * @arbif_dev: The arbiter interface device. + * @gpu_required: The GPU is still needed to do more work. + */ + void (*vm_arb_gpu_stopped)(struct arbiter_if_dev *arbif_dev, + u8 gpu_required); +}; + +/** + * struct arbiter_if_dev - Arbiter Interface + * @vm_ops: Callback functions for connecting KBase with + * arbiter interface device. + * @priv_data: Internal arbif data not used by KBASE. + * + * Arbiter Interface Kernel Module State used for linking KBase + * with an arbiter interface platform device + */ +struct arbiter_if_dev { + struct arbiter_if_vm_arb_ops vm_ops; + void *priv_data; +}; + +#endif /* _MALI_KBASE_ARBITER_INTERFACE_H_ */ diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c new file mode 100755 index 000000000000..02b5de2436ea --- /dev/null +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c @@ -0,0 +1,676 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file mali_kbase_arbiter_pm.c + * Mali arbiter power manager state machine and APIs + */ + +#include +#include +#include +#include +#include +#include +#include + +static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev); +static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld( + struct kbase_device *kbdev); + +static inline const char *kbase_arbiter_pm_vm_state_str( + enum kbase_vm_state state) +{ + switch (state) { + case KBASE_VM_STATE_INITIALIZING: + return "KBASE_VM_STATE_INITIALIZING"; + case KBASE_VM_STATE_INITIALIZING_WITH_GPU: + return "KBASE_VM_STATE_INITIALIZING_WITH_GPU"; + case KBASE_VM_STATE_SUSPENDED: + return "KBASE_VM_STATE_SUSPENDED"; + case KBASE_VM_STATE_STOPPED: + return "KBASE_VM_STATE_STOPPED"; + case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: + return "KBASE_VM_STATE_STOPPED_GPU_REQUESTED"; + case KBASE_VM_STATE_STARTING: + return "KBASE_VM_STATE_STARTING"; + case KBASE_VM_STATE_IDLE: + return "KBASE_VM_STATE_IDLE"; + case KBASE_VM_STATE_ACTIVE: + return "KBASE_VM_STATE_ACTIVE"; + case KBASE_VM_STATE_STOPPING_IDLE: + return "KBASE_VM_STATE_STOPPING_IDLE"; + case KBASE_VM_STATE_STOPPING_ACTIVE: + return "KBASE_VM_STATE_STOPPING_ACTIVE"; + case KBASE_VM_STATE_SUSPEND_PENDING: + return "KBASE_VM_STATE_SUSPEND_PENDING"; + case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: + return "KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT"; + default: + KBASE_DEBUG_ASSERT(false); + return "[UnknownState]"; + } +} + +static inline const char *kbase_arbiter_pm_vm_event_str( + enum kbase_arbif_evt evt) +{ + switch (evt) { + case KBASE_VM_GPU_INITIALIZED_EVT: + return "KBASE_VM_GPU_INITIALIZED_EVT"; + case KBASE_VM_GPU_STOP_EVT: + return "KBASE_VM_GPU_STOP_EVT"; + case KBASE_VM_GPU_GRANTED_EVT: + return "KBASE_VM_GPU_GRANTED_EVT"; + case KBASE_VM_GPU_LOST_EVT: + return "KBASE_VM_GPU_LOST_EVT"; + case KBASE_VM_OS_SUSPEND_EVENT: + return "KBASE_VM_OS_SUSPEND_EVENT"; + case KBASE_VM_OS_RESUME_EVENT: + return "KBASE_VM_OS_RESUME_EVENT"; + case KBASE_VM_GPU_IDLE_EVENT: + return "KBASE_VM_GPU_IDLE_EVENT"; + case KBASE_VM_REF_EVENT: + return "KBASE_VM_REF_EVENT"; + default: + KBASE_DEBUG_ASSERT(false); + return "[UnknownEvent]"; + } +} + +static void kbase_arbiter_pm_vm_set_state(struct kbase_device *kbdev, + enum kbase_vm_state new_state) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + dev_dbg(kbdev->dev, "VM set_state %s -> %s", + kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state), + kbase_arbiter_pm_vm_state_str(new_state)); + lockdep_assert_held(&arb_vm_state->vm_state_lock); + arb_vm_state->vm_state = new_state; + wake_up(&arb_vm_state->vm_state_wait); +} + +static void kbase_arbiter_pm_suspend_wq(struct work_struct *data) +{ + struct kbase_arbiter_vm_state *arb_vm_state = container_of(data, + struct kbase_arbiter_vm_state, + vm_suspend_work); + struct kbase_device *kbdev = arb_vm_state->kbdev; + + mutex_lock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, ">%s\n", __func__); + if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE || + arb_vm_state->vm_state == + KBASE_VM_STATE_STOPPING_ACTIVE || + arb_vm_state->vm_state == + KBASE_VM_STATE_SUSPEND_PENDING) { + mutex_unlock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, ">kbase_pm_driver_suspend\n"); + kbase_pm_driver_suspend(kbdev); + dev_dbg(kbdev->dev, "vm_state_lock); + } + mutex_unlock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, "<%s\n", __func__); +} + +static void kbase_arbiter_pm_resume_wq(struct work_struct *data) +{ + struct kbase_arbiter_vm_state *arb_vm_state = container_of(data, + struct kbase_arbiter_vm_state, + vm_resume_work); + struct kbase_device *kbdev = arb_vm_state->kbdev; + + mutex_lock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, ">%s\n", __func__); + arb_vm_state->vm_arb_starting = true; + if (arb_vm_state->vm_state == KBASE_VM_STATE_STARTING) { + mutex_unlock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, ">kbase_pm_driver_resume\n"); + kbase_pm_driver_resume(kbdev, true); + dev_dbg(kbdev->dev, "vm_state_lock); + } else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_ACTIVE) { + kbase_arbiter_pm_vm_stopped(kbdev); + } + arb_vm_state->vm_arb_starting = false; + mutex_unlock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, "<%s\n", __func__); +} + +int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) +{ + int err; + struct kbase_arbiter_vm_state *arb_vm_state = NULL; + + arb_vm_state = kmalloc(sizeof(struct kbase_arbiter_vm_state), + GFP_KERNEL); + if (arb_vm_state == NULL) + return -ENOMEM; + + arb_vm_state->kbdev = kbdev; + arb_vm_state->vm_state = KBASE_VM_STATE_INITIALIZING; + + mutex_init(&arb_vm_state->vm_state_lock); + init_waitqueue_head(&arb_vm_state->vm_state_wait); + arb_vm_state->vm_arb_wq = alloc_ordered_workqueue("kbase_vm_arb_wq", + WQ_HIGHPRI); + if (!arb_vm_state->vm_arb_wq) { + dev_err(kbdev->dev, "Failed to allocate vm_arb workqueue\n"); + return -ENOMEM; + } + INIT_WORK(&arb_vm_state->vm_suspend_work, kbase_arbiter_pm_suspend_wq); + INIT_WORK(&arb_vm_state->vm_resume_work, kbase_arbiter_pm_resume_wq); + arb_vm_state->vm_arb_starting = false; + arb_vm_state->vm_arb_users_waiting = 0; + kbdev->pm.arb_vm_state = arb_vm_state; + + err = kbase_arbif_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Failed to initialise arbif module\n"); + goto arbif_init_fail; + } + if (kbdev->arb.arb_if) { + kbase_arbif_gpu_request(kbdev); + dev_dbg(kbdev->dev, "Waiting for initial GPU assignment...\n"); + wait_event(arb_vm_state->vm_state_wait, + arb_vm_state->vm_state == + KBASE_VM_STATE_INITIALIZING_WITH_GPU); + dev_dbg(kbdev->dev, + "Waiting for initial GPU assignment - done\n"); + } + return 0; + +arbif_init_fail: + destroy_workqueue(arb_vm_state->vm_arb_wq); + kfree(arb_vm_state); + kbdev->pm.arb_vm_state = NULL; + return err; +} + +void kbase_arbiter_pm_early_term(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + mutex_lock(&arb_vm_state->vm_state_lock); + if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) { + kbase_pm_set_gpu_lost(kbdev, false); + kbase_arbif_gpu_stopped(kbdev, false); + } + mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_arbif_destroy(kbdev); + destroy_workqueue(arb_vm_state->vm_arb_wq); + arb_vm_state->vm_arb_wq = NULL; + kfree(kbdev->pm.arb_vm_state); + kbdev->pm.arb_vm_state = NULL; +} + +void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + mutex_lock(&arb_vm_state->vm_state_lock); + if (!kbdev->arb.arb_if || + arb_vm_state->vm_state > + KBASE_VM_STATE_STOPPED_GPU_REQUESTED) + kbase_release_interrupts(kbdev); + + mutex_unlock(&arb_vm_state->vm_state_lock); +} + +void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) +{ + bool request_gpu = false; + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + + if (arb_vm_state->vm_arb_users_waiting > 0 && + arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE) + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_ACTIVE); + + dev_dbg(kbdev->dev, "%s %s\n", __func__, + kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); + kbase_release_interrupts(kbdev); + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_STOPPING_ACTIVE: + request_gpu = true; + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPED_GPU_REQUESTED); + break; + case KBASE_VM_STATE_STOPPING_IDLE: + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED); + break; + case KBASE_VM_STATE_SUSPEND_PENDING: + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); + break; + default: + dev_warn(kbdev->dev, "unexpected pm_stop VM state %u", + arb_vm_state->vm_state); + break; + } + + kbase_pm_set_gpu_lost(kbdev, false); + kbase_arbif_gpu_stopped(kbdev, request_gpu); +} + +static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_INITIALIZING: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_INITIALIZING_WITH_GPU); + break; + case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING); + kbase_install_interrupts(kbdev); + queue_work(arb_vm_state->vm_arb_wq, + &arb_vm_state->vm_resume_work); + break; + case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: + kbase_pm_set_gpu_lost(kbdev, false); + kbase_arbif_gpu_stopped(kbdev, false); + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); + break; + default: + dev_warn(kbdev->dev, + "GPU_GRANTED when not expected - state %s\n", + kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); + break; + } +} + +static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) { + mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); + mutex_lock(&arb_vm_state->vm_state_lock); + } + + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_IDLE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_IDLE); + queue_work(arb_vm_state->vm_arb_wq, + &arb_vm_state->vm_suspend_work); + break; + case KBASE_VM_STATE_ACTIVE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_ACTIVE); + queue_work(arb_vm_state->vm_arb_wq, + &arb_vm_state->vm_suspend_work); + break; + case KBASE_VM_STATE_STARTING: + dev_dbg(kbdev->dev, "Got GPU_STOP event while STARTING."); + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_ACTIVE); + if (arb_vm_state->vm_arb_starting) + queue_work(arb_vm_state->vm_arb_wq, + &arb_vm_state->vm_suspend_work); + break; + case KBASE_VM_STATE_SUSPEND_PENDING: + /* Suspend finishes with a stop so nothing else to do */ + break; + default: + dev_warn(kbdev->dev, "GPU_STOP when not expected - state %s\n", + kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); + break; + } +} + +static void kbase_gpu_lost(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + bool handle_gpu_lost = false; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_STARTING: + case KBASE_VM_STATE_ACTIVE: + case KBASE_VM_STATE_IDLE: + dev_warn(kbdev->dev, "GPU lost in state %s", + kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); + kbase_arbiter_pm_vm_gpu_stop(kbdev); + handle_gpu_lost = true; + break; + case KBASE_VM_STATE_STOPPING_IDLE: + case KBASE_VM_STATE_STOPPING_ACTIVE: + case KBASE_VM_STATE_SUSPEND_PENDING: + dev_dbg(kbdev->dev, "GPU lost while stopping"); + handle_gpu_lost = true; + break; + case KBASE_VM_STATE_SUSPENDED: + case KBASE_VM_STATE_STOPPED: + case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: + dev_dbg(kbdev->dev, "GPU lost while already stopped"); + break; + case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: + dev_dbg(kbdev->dev, "GPU lost while waiting to suspend"); + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); + break; + default: + break; + } + if (handle_gpu_lost) { + /* Releasing the VM state lock here is safe because + * we are guaranteed to be in either STOPPING_IDLE, + * STOPPING_ACTIVE or SUSPEND_PENDING at this point. + * The only transitions that are valid from here are to + * STOPPED, STOPPED_GPU_REQUESTED or SUSPENDED which can + * only happen at the completion of the GPU lost handling. + */ + mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_pm_handle_gpu_lost(kbdev); + mutex_lock(&arb_vm_state->vm_state_lock); + } +} + +static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state( + struct kbase_device *kbdev) +{ + switch (kbdev->pm.arb_vm_state->vm_state) { + case KBASE_VM_STATE_SUSPENDED: + case KBASE_VM_STATE_STOPPED: + case KBASE_VM_STATE_IDLE: + case KBASE_VM_STATE_ACTIVE: + return true; + default: + return false; + } +} + +static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + enum kbase_vm_state prev_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + if (kbdev->arb.arb_if) { + if (kbdev->pm.arb_vm_state->vm_state == + KBASE_VM_STATE_SUSPENDED) + return; + } + /* Block suspend OS function until we are in a stable state + * with vm_state_lock + */ + while (!kbase_arbiter_pm_vm_os_suspend_ready_state(kbdev)) { + prev_state = arb_vm_state->vm_state; + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_STOPPING_ACTIVE: + case KBASE_VM_STATE_STOPPING_IDLE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_SUSPEND_PENDING); + break; + case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT); + break; + case KBASE_VM_STATE_STARTING: + if (!arb_vm_state->vm_arb_starting) { + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_SUSPEND_PENDING); + kbase_arbiter_pm_vm_stopped(kbdev); + } + break; + default: + break; + } + mutex_unlock(&arb_vm_state->vm_state_lock); + wait_event(arb_vm_state->vm_state_wait, + arb_vm_state->vm_state != prev_state); + mutex_lock(&arb_vm_state->vm_state_lock); + } + + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_STOPPED: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_SUSPENDED); + break; + case KBASE_VM_STATE_IDLE: + case KBASE_VM_STATE_ACTIVE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_SUSPEND_PENDING); + mutex_unlock(&arb_vm_state->vm_state_lock); + /* Ensure resume has completed fully before starting suspend */ + flush_work(&arb_vm_state->vm_resume_work); + kbase_pm_driver_suspend(kbdev); + mutex_lock(&arb_vm_state->vm_state_lock); + break; + case KBASE_VM_STATE_SUSPENDED: + break; + default: + KBASE_DEBUG_ASSERT_MSG(false, "Unexpected state to suspend"); + break; + } +} + +static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + KBASE_DEBUG_ASSERT_MSG(arb_vm_state->vm_state == + KBASE_VM_STATE_SUSPENDED, + "Unexpected state to resume"); + + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPED_GPU_REQUESTED); + kbase_arbif_gpu_request(kbdev); + + /* Release lock and block resume OS function until we have + * asynchronously received the GRANT message from the Arbiter and + * fully resumed + */ + mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); + flush_work(&arb_vm_state->vm_resume_work); + mutex_lock(&arb_vm_state->vm_state_lock); +} + +void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, + enum kbase_arbif_evt evt) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + if (!kbdev->arb.arb_if) + return; + + mutex_lock(&arb_vm_state->vm_state_lock); + dev_dbg(kbdev->dev, "%s %s\n", __func__, + kbase_arbiter_pm_vm_event_str(evt)); + + switch (evt) { + case KBASE_VM_GPU_GRANTED_EVT: + kbase_arbiter_pm_vm_gpu_start(kbdev); + break; + case KBASE_VM_GPU_STOP_EVT: + kbase_arbiter_pm_vm_gpu_stop(kbdev); + break; + case KBASE_VM_GPU_LOST_EVT: + dev_dbg(kbdev->dev, "KBASE_ARBIF_GPU_LOST_EVT!"); + kbase_gpu_lost(kbdev); + break; + case KBASE_VM_OS_SUSPEND_EVENT: + kbase_arbiter_pm_vm_os_prepare_suspend(kbdev); + break; + case KBASE_VM_OS_RESUME_EVENT: + kbase_arbiter_pm_vm_os_resume(kbdev); + break; + case KBASE_VM_GPU_IDLE_EVENT: + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_ACTIVE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_IDLE); + kbase_arbif_gpu_idle(kbdev); + break; + default: + break; + } + break; + + case KBASE_VM_REF_EVENT: + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_STARTING: + KBASE_TLSTREAM_TL_ARBITER_STARTED(kbdev, kbdev); + /* FALL THROUGH */ + case KBASE_VM_STATE_IDLE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_ACTIVE); + kbase_arbif_gpu_active(kbdev); + break; + case KBASE_VM_STATE_STOPPING_IDLE: + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_ACTIVE); + break; + default: + break; + } + break; + + case KBASE_VM_GPU_INITIALIZED_EVT: + switch (arb_vm_state->vm_state) { + case KBASE_VM_STATE_INITIALIZING_WITH_GPU: + lockdep_assert_held(&kbdev->pm.lock); + if (kbdev->pm.active_count > 0) { + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_ACTIVE); + kbase_arbif_gpu_active(kbdev); + } else { + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_IDLE); + kbase_arbif_gpu_idle(kbdev); + } + break; + default: + break; + } + break; + + default: + dev_alert(kbdev->dev, "Got Unknown Event!"); + break; + } + mutex_unlock(&arb_vm_state->vm_state_lock); +} + +KBASE_EXPORT_TEST_API(kbase_arbiter_pm_vm_event); + +static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + dev_dbg(kbdev->dev, "Waiting for GPU assignment...\n"); + wait_event(arb_vm_state->vm_state_wait, + arb_vm_state->vm_state == KBASE_VM_STATE_IDLE || + arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE); + dev_dbg(kbdev->dev, "Waiting for GPU assignment - done\n"); +} + +static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld( + struct kbase_device *kbdev) +{ + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + lockdep_assert_held(&arb_vm_state->vm_state_lock); + return (arb_vm_state->vm_state == KBASE_VM_STATE_IDLE || + arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE); +} + +int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, + enum kbase_pm_suspend_handler suspend_handler) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + int res = 0; + + if (kbdev->arb.arb_if) { + mutex_lock(&arb_vm_state->vm_state_lock); + while (!kbase_arbiter_pm_vm_gpu_assigned_lockheld(kbdev)) { + /* Update VM state since we have GPU work to do */ + if (arb_vm_state->vm_state == + KBASE_VM_STATE_STOPPING_IDLE) + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPING_ACTIVE); + else if (arb_vm_state->vm_state == + KBASE_VM_STATE_STOPPED) { + kbase_arbiter_pm_vm_set_state(kbdev, + KBASE_VM_STATE_STOPPED_GPU_REQUESTED); + kbase_arbif_gpu_request(kbdev); + } else if (arb_vm_state->vm_state == + KBASE_VM_STATE_INITIALIZING_WITH_GPU) + break; + + if (suspend_handler != + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) { + + /* In case of GPU lost, even if + * active_count > 0, we no longer have GPU + * access + */ + if (kbase_pm_is_gpu_lost(kbdev)) + res = 1; + + switch (suspend_handler) { + case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: + res = 1; + break; + case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: + if (kbdev->pm.active_count == 0) + res = 1; + break; + case KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED: + break; + default: + WARN(1, "Unknown suspend_handler\n"); + res = 1; + break; + } + break; + } + + /* Need to synchronously wait for GPU assignment */ + arb_vm_state->vm_arb_users_waiting++; + mutex_unlock(&arb_vm_state->vm_state_lock); + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + mutex_lock(&arb_vm_state->vm_state_lock); + arb_vm_state->vm_arb_users_waiting--; + } + mutex_unlock(&arb_vm_state->vm_state_lock); + } + return res; +} diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h new file mode 100755 index 000000000000..3c49eb1948c5 --- /dev/null +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h @@ -0,0 +1,159 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ + +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file + * Mali arbiter power manager state machine and APIs + */ + +#ifndef _MALI_KBASE_ARBITER_PM_H_ +#define _MALI_KBASE_ARBITER_PM_H_ + +#include "mali_kbase_arbif.h" + +/** + * enum kbase_vm_state - Current PM Arbitration state. + * + * @KBASE_VM_STATE_INITIALIZING: Special state before arbiter is initialized. + * @KBASE_VM_STATE_INITIALIZING_WITH_GPU: Initialization after GPU + * has been granted. + * @KBASE_VM_STATE_SUSPENDED: KBase is suspended by OS and GPU is not assigned. + * @KBASE_VM_STATE_STOPPED: GPU is not assigned to KBase and is not required. + * @KBASE_VM_STATE_STOPPED_GPU_REQUESTED: GPU is not assigned to KBase + * but a request has been made. + * @KBASE_VM_STATE_STARTING: GPU is assigned and KBase is getting ready to run. + * @KBASE_VM_STATE_IDLE: GPU is assigned but KBase has no work to do + * @KBASE_VM_STATE_ACTIVE: GPU is assigned and KBase is busy using it + * @KBASE_VM_STATE_SUSPEND_PENDING: OS is going into suspend mode. + * @KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: OS is going into suspend mode but GPU + * has already been requested. + * In this situation we must wait for + * the Arbiter to send a GRANTED message + * and respond immediately with + * a STOPPED message before entering + * the suspend mode. + * @KBASE_VM_STATE_STOPPING_IDLE: Arbiter has sent a stopped message and there + * is currently no work to do on the GPU. + * @KBASE_VM_STATE_STOPPING_ACTIVE: Arbiter has sent a stopped message when + * KBase has work to do. + */ +enum kbase_vm_state { + KBASE_VM_STATE_INITIALIZING, + KBASE_VM_STATE_INITIALIZING_WITH_GPU, + KBASE_VM_STATE_SUSPENDED, + KBASE_VM_STATE_STOPPED, + KBASE_VM_STATE_STOPPED_GPU_REQUESTED, + KBASE_VM_STATE_STARTING, + KBASE_VM_STATE_IDLE, + KBASE_VM_STATE_ACTIVE, + KBASE_VM_STATE_SUSPEND_PENDING, + KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT, + KBASE_VM_STATE_STOPPING_IDLE, + KBASE_VM_STATE_STOPPING_ACTIVE +}; + +/** + * kbase_arbiter_pm_early_init() - Initialize arbiter for VM Paravirtualized use + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Initialize the arbiter and other required resources during the runtime + * and request the GPU for the VM for the first time. + * + * Return: 0 if successful, otherwise a standard Linux error code + */ +int kbase_arbiter_pm_early_init(struct kbase_device *kbdev); + +/** + * kbase_arbiter_pm_early_term() - Shutdown arbiter and free resources. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Clean up all the resources + */ +void kbase_arbiter_pm_early_term(struct kbase_device *kbdev); + +/** + * kbase_arbiter_pm_release_interrupts() - Release the GPU interrupts + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Releases interrupts if needed (GPU is available) otherwise does nothing + */ +void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev); + +/** + * kbase_arbiter_pm_vm_event() - Dispatch VM event to the state machine + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * The state machine function. Receives events and transitions states + * according the event received and the current state + */ +void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, + enum kbase_arbif_evt event); + +/** + * kbase_arbiter_pm_ctx_active_handle_suspend() - Handle suspend operation for + * arbitration mode + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @suspend_handler: The handler code for how to handle a suspend + * that might occur + * + * This function handles a suspend event from the driver, + * communicating with the arbiter and waiting synchronously for the GPU + * to be granted again depending on the VM state. + * + * Return: 0 if success, 1 if failure due to system suspending/suspended + */ +int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, + enum kbase_pm_suspend_handler suspend_handler); + + +/** + * kbase_arbiter_pm_vm_stopped() - Handle stop event for the VM + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This function handles a stop event for the VM. + * It will update the VM state and forward the stop event to the driver. + */ +void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev); + +#endif /*_MALI_KBASE_ARBITER_PM_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/Kbuild b/drivers/gpu/arm/bifrost/backend/gpu/Kbuild new file mode 100755 index 000000000000..b48ab4c51875 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/Kbuild @@ -0,0 +1,65 @@ +# +# (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +BACKEND += \ + backend/gpu/mali_kbase_cache_policy_backend.c \ + backend/gpu/mali_kbase_gpuprops_backend.c \ + backend/gpu/mali_kbase_irq_linux.c \ + backend/gpu/mali_kbase_instr_backend.c \ + backend/gpu/mali_kbase_js_backend.c \ + backend/gpu/mali_kbase_pm_backend.c \ + backend/gpu/mali_kbase_pm_driver.c \ + backend/gpu/mali_kbase_pm_metrics.c \ + backend/gpu/mali_kbase_pm_ca.c \ + backend/gpu/mali_kbase_pm_always_on.c \ + backend/gpu/mali_kbase_pm_coarse_demand.c \ + backend/gpu/mali_kbase_pm_policy.c \ + backend/gpu/mali_kbase_time.c \ + backend/gpu/mali_kbase_l2_mmu_config.c \ + backend/gpu/mali_kbase_clk_rate_trace_mgr.c + +ifeq ($(MALI_USE_CSF),1) +# empty +else + BACKEND += \ + backend/gpu/mali_kbase_jm_as.c \ + backend/gpu/mali_kbase_debug_job_fault_backend.c \ + backend/gpu/mali_kbase_jm_hw.c \ + backend/gpu/mali_kbase_jm_rb.c +endif + +ifeq ($(MALI_CUSTOMER_RELEASE),0) +BACKEND += \ + backend/gpu/mali_kbase_pm_always_on_demand.c +endif + +ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y) +BACKEND += \ + backend/gpu/mali_kbase_devfreq.c +endif + +ifeq ($(CONFIG_MALI_BIFROST_NO_MALI),y) + # Dummy model + BACKEND += backend/gpu/mali_kbase_model_dummy.c + BACKEND += backend/gpu/mali_kbase_model_linux.c + # HW error simulation + BACKEND += backend/gpu/mali_kbase_model_error_generator.c +endif diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h new file mode 100755 index 000000000000..4a61f96c8c7d --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h @@ -0,0 +1,31 @@ +/* + * + * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Backend specific configuration + */ + +#ifndef _KBASE_BACKEND_CONFIG_H_ +#define _KBASE_BACKEND_CONFIG_H_ + +#endif /* _KBASE_BACKEND_CONFIG_H_ */ + diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.c new file mode 100755 index 000000000000..4e07a3f9d83f --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.c @@ -0,0 +1,34 @@ +/* + * + * (C) COPYRIGHT 2014-2016, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "backend/gpu/mali_kbase_cache_policy_backend.h" +#include + +void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, + u32 mode) +{ + kbdev->current_gpu_coherency_mode = mode; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) + kbase_reg_write(kbdev, COHERENCY_ENABLE, mode); +} + diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h new file mode 100755 index 000000000000..f78ada74f605 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +#ifndef _KBASE_CACHE_POLICY_BACKEND_H_ +#define _KBASE_CACHE_POLICY_BACKEND_H_ + +#include "mali_kbase.h" +#include "mali_base_kernel.h" + +/** + * kbase_cache_set_coherency_mode() - Sets the system coherency mode + * in the GPU. + * @kbdev: Device pointer + * @mode: Coherency mode. COHERENCY_ACE/ACE_LITE + */ +void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, + u32 mode); + +#endif /* _KBASE_CACHE_POLICY_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c new file mode 100755 index 000000000000..187d7d6f6926 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c @@ -0,0 +1,287 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Implementation of the GPU clock rate trace manager. + */ + +#include +#include +#include +#include +#include "mali_kbase_clk_rate_trace_mgr.h" + +#ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY +#include +#else +#include "mali_power_gpu_frequency_trace.h" +#endif + +#ifndef CLK_RATE_TRACE_OPS +#define CLK_RATE_TRACE_OPS (NULL) +#endif + +static int gpu_clk_rate_change_notifier(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct kbase_gpu_clk_notifier_data *ndata = data; + struct kbase_clk_data *clk_data = + container_of(nb, struct kbase_clk_data, clk_rate_change_nb); + struct kbase_clk_rate_trace_manager *clk_rtm = clk_data->clk_rtm; + unsigned long flags; + + if (WARN_ON_ONCE(clk_data->gpu_clk_handle != ndata->gpu_clk_handle)) + return NOTIFY_BAD; + + spin_lock_irqsave(&clk_rtm->lock, flags); + if (event == POST_RATE_CHANGE) { + if (!clk_rtm->gpu_idle && + (clk_data->clock_val != ndata->new_rate)) { + kbase_clk_rate_trace_manager_notify_all( + clk_rtm, clk_data->index, ndata->new_rate); + } + + clk_data->clock_val = ndata->new_rate; + } + spin_unlock_irqrestore(&clk_rtm->lock, flags); + + return NOTIFY_DONE; +} + +static int gpu_clk_data_init(struct kbase_device *kbdev, + void *gpu_clk_handle, unsigned int index) +{ + struct kbase_clk_rate_trace_op_conf *callbacks = + (struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS; + struct kbase_clk_data *clk_data; + struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; + int ret = 0; + + if (WARN_ON(!callbacks) || + WARN_ON(!gpu_clk_handle) || + WARN_ON(index >= BASE_MAX_NR_CLOCKS_REGULATORS)) + return -EINVAL; + + clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL); + if (!clk_data) { + dev_err(kbdev->dev, "Failed to allocate data for clock enumerated at index %u", index); + return -ENOMEM; + } + + clk_data->index = (u8)index; + clk_data->gpu_clk_handle = gpu_clk_handle; + /* Store the initial value of clock */ + clk_data->clock_val = + callbacks->get_gpu_clk_rate(kbdev, gpu_clk_handle); + + { + /* At the initialization time, GPU is powered off. */ + unsigned long flags; + + spin_lock_irqsave(&clk_rtm->lock, flags); + kbase_clk_rate_trace_manager_notify_all( + clk_rtm, clk_data->index, 0); + spin_unlock_irqrestore(&clk_rtm->lock, flags); + } + + clk_data->clk_rtm = clk_rtm; + clk_rtm->clks[index] = clk_data; + + clk_data->clk_rate_change_nb.notifier_call = + gpu_clk_rate_change_notifier; + + ret = callbacks->gpu_clk_notifier_register(kbdev, gpu_clk_handle, + &clk_data->clk_rate_change_nb); + if (ret) { + dev_err(kbdev->dev, "Failed to register notifier for clock enumerated at index %u", index); + kfree(clk_data); + } + + return ret; +} + +int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev) +{ + struct kbase_clk_rate_trace_op_conf *callbacks = + (struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS; + struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; + unsigned int i; + int ret = 0; + + /* Return early if no callbacks provided for clock rate tracing */ + if (!callbacks) + return 0; + + spin_lock_init(&clk_rtm->lock); + INIT_LIST_HEAD(&clk_rtm->listeners); + + clk_rtm->gpu_idle = true; + + for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { + void *gpu_clk_handle = + callbacks->enumerate_gpu_clk(kbdev, i); + + if (!gpu_clk_handle) + break; + + ret = gpu_clk_data_init(kbdev, gpu_clk_handle, i); + if (ret) + goto error; + } + + /* Activate clock rate trace manager if at least one GPU clock was + * enumerated. + */ + if (i) + WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks); + else + dev_info(kbdev->dev, "No clock(s) available for rate tracing"); + + return 0; + +error: + while (i--) { + clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister( + kbdev, clk_rtm->clks[i]->gpu_clk_handle, + &clk_rtm->clks[i]->clk_rate_change_nb); + kfree(clk_rtm->clks[i]); + } + + return ret; +} + +void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev) +{ + struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; + unsigned int i; + + WARN_ON(!list_empty(&clk_rtm->listeners)); + + if (!clk_rtm->clk_rate_trace_ops) + return; + + for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { + if (!clk_rtm->clks[i]) + break; + + clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister( + kbdev, clk_rtm->clks[i]->gpu_clk_handle, + &clk_rtm->clks[i]->clk_rate_change_nb); + kfree(clk_rtm->clks[i]); + } + + WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL); +} + +void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev) +{ + struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; + unsigned int i; + unsigned long flags; + + if (!clk_rtm->clk_rate_trace_ops) + return; + + spin_lock_irqsave(&clk_rtm->lock, flags); + + for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { + struct kbase_clk_data *clk_data = clk_rtm->clks[i]; + + if (!clk_data) + break; + + if (unlikely(!clk_data->clock_val)) + continue; + + kbase_clk_rate_trace_manager_notify_all( + clk_rtm, clk_data->index, clk_data->clock_val); + } + + clk_rtm->gpu_idle = false; + spin_unlock_irqrestore(&clk_rtm->lock, flags); +} + +void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev) +{ + struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; + unsigned int i; + unsigned long flags; + + if (!clk_rtm->clk_rate_trace_ops) + return; + + spin_lock_irqsave(&clk_rtm->lock, flags); + + for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { + struct kbase_clk_data *clk_data = clk_rtm->clks[i]; + + if (!clk_data) + break; + + if (unlikely(!clk_data->clock_val)) + continue; + + kbase_clk_rate_trace_manager_notify_all( + clk_rtm, clk_data->index, 0); + } + + clk_rtm->gpu_idle = true; + spin_unlock_irqrestore(&clk_rtm->lock, flags); +} + +void kbase_clk_rate_trace_manager_notify_all( + struct kbase_clk_rate_trace_manager *clk_rtm, + u32 clk_index, + unsigned long new_rate) +{ + struct kbase_clk_rate_listener *pos; + struct kbase_device *kbdev; + + lockdep_assert_held(&clk_rtm->lock); + + kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm); + + dev_dbg(kbdev->dev, "GPU clock %u rate changed to %lu", + clk_index, new_rate); + + /* Raise standard `power/gpu_frequency` ftrace event */ + { + unsigned long new_rate_khz = new_rate; + +#if BITS_PER_LONG == 64 + do_div(new_rate_khz, 1000); +#elif BITS_PER_LONG == 32 + new_rate_khz /= 1000; +#else +#error "unsigned long division is not supported for this architecture" +#endif + + trace_gpu_frequency(new_rate_khz, clk_index); + } + + /* Notify the listeners. */ + list_for_each_entry(pos, &clk_rtm->listeners, node) { + pos->notify(pos, clk_index, new_rate); + } +} +KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all); + diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.h new file mode 100755 index 000000000000..dcafb26ea4c0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.h @@ -0,0 +1,155 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CLK_RATE_TRACE_MGR_ +#define _KBASE_CLK_RATE_TRACE_MGR_ + +/** The index of top clock domain in kbase_clk_rate_trace_manager:clks. */ +#define KBASE_CLOCK_DOMAIN_TOP (0) + +/** The index of shader-cores clock domain in + * kbase_clk_rate_trace_manager:clks. + */ +#define KBASE_CLOCK_DOMAIN_SHADER_CORES (1) + +/** + * struct kbase_clk_data - Data stored per enumerated GPU clock. + * + * @clk_rtm: Pointer to clock rate trace manager object. + * @gpu_clk_handle: Handle unique to the enumerated GPU clock. + * @plat_private: Private data for the platform to store into + * @clk_rate_change_nb: notifier block containing the pointer to callback + * function that is invoked whenever the rate of + * enumerated GPU clock changes. + * @clock_val: Current rate of the enumerated GPU clock. + * @index: Index at which the GPU clock was enumerated. + */ +struct kbase_clk_data { + struct kbase_clk_rate_trace_manager *clk_rtm; + void *gpu_clk_handle; + void *plat_private; + struct notifier_block clk_rate_change_nb; + unsigned long clock_val; + u8 index; +}; + +/** + * kbase_clk_rate_trace_manager_init - Initialize GPU clock rate trace manager. + * + * @kbdev: Device pointer + * + * Return: 0 if success, or an error code on failure. + */ +int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev); + +/** + * kbase_clk_rate_trace_manager_term - Terminate GPU clock rate trace manager. + * + * @kbdev: Device pointer + */ +void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev); + +/** + * kbase_clk_rate_trace_manager_gpu_active - Inform GPU clock rate trace + * manager of GPU becoming active. + * + * @kbdev: Device pointer + */ +void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev); + +/** + * kbase_clk_rate_trace_manager_gpu_idle - Inform GPU clock rate trace + * manager of GPU becoming idle. + * @kbdev: Device pointer + */ +void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev); + +/** + * kbase_clk_rate_trace_manager_subscribe_no_lock() - Add freq change listener. + * + * @clk_rtm: Clock rate manager instance. + * @listener: Listener handle + * + * kbase_clk_rate_trace_manager:lock must be held by the caller. + */ +static inline void kbase_clk_rate_trace_manager_subscribe_no_lock( + struct kbase_clk_rate_trace_manager *clk_rtm, + struct kbase_clk_rate_listener *listener) +{ + lockdep_assert_held(&clk_rtm->lock); + list_add(&listener->node, &clk_rtm->listeners); +} + +/** + * kbase_clk_rate_trace_manager_subscribe() - Add freq change listener. + * + * @clk_rtm: Clock rate manager instance. + * @listener: Listener handle + */ +static inline void kbase_clk_rate_trace_manager_subscribe( + struct kbase_clk_rate_trace_manager *clk_rtm, + struct kbase_clk_rate_listener *listener) +{ + unsigned long flags; + + spin_lock_irqsave(&clk_rtm->lock, flags); + kbase_clk_rate_trace_manager_subscribe_no_lock( + clk_rtm, listener); + spin_unlock_irqrestore(&clk_rtm->lock, flags); +} + +/** + * kbase_clk_rate_trace_manager_unsubscribe() - Remove freq change listener. + * + * @clk_rtm: Clock rate manager instance. + * @listener: Listener handle + */ +static inline void kbase_clk_rate_trace_manager_unsubscribe( + struct kbase_clk_rate_trace_manager *clk_rtm, + struct kbase_clk_rate_listener *listener) +{ + unsigned long flags; + + spin_lock_irqsave(&clk_rtm->lock, flags); + list_del(&listener->node); + spin_unlock_irqrestore(&clk_rtm->lock, flags); +} + +/** + * kbase_clk_rate_trace_manager_notify_all() - Notify all clock \ + * rate listeners. + * + * @clk_rtm: Clock rate manager instance. + * @clk_index: Clock index. + * @new_rate: New clock frequency(Hz) + * + * kbase_clk_rate_trace_manager:lock must be locked. + * This function is exported to be used by clock rate trace test + * portal. + */ +void kbase_clk_rate_trace_manager_notify_all( + struct kbase_clk_rate_trace_manager *clk_rtm, + u32 clock_index, + unsigned long new_rate); + +#endif /* _KBASE_CLK_RATE_TRACE_MGR_ */ + diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c new file mode 100755 index 000000000000..3aadcb04160c --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c @@ -0,0 +1,164 @@ +/* + * + * (C) COPYRIGHT 2012-2015, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include "mali_kbase_debug_job_fault.h" + +#ifdef CONFIG_DEBUG_FS + +/*GPU_CONTROL_REG(r)*/ +static int gpu_control_reg_snapshot[] = { + GPU_ID, + SHADER_READY_LO, + SHADER_READY_HI, + TILER_READY_LO, + TILER_READY_HI, + L2_READY_LO, + L2_READY_HI +}; + +/* JOB_CONTROL_REG(r) */ +static int job_control_reg_snapshot[] = { + JOB_IRQ_MASK, + JOB_IRQ_STATUS +}; + +/* JOB_SLOT_REG(n,r) */ +static int job_slot_reg_snapshot[] = { + JS_HEAD_LO, + JS_HEAD_HI, + JS_TAIL_LO, + JS_TAIL_HI, + JS_AFFINITY_LO, + JS_AFFINITY_HI, + JS_CONFIG, + JS_STATUS, + JS_HEAD_NEXT_LO, + JS_HEAD_NEXT_HI, + JS_AFFINITY_NEXT_LO, + JS_AFFINITY_NEXT_HI, + JS_CONFIG_NEXT +}; + +/*MMU_REG(r)*/ +static int mmu_reg_snapshot[] = { + MMU_IRQ_MASK, + MMU_IRQ_STATUS +}; + +/* MMU_AS_REG(n,r) */ +static int as_reg_snapshot[] = { + AS_TRANSTAB_LO, + AS_TRANSTAB_HI, + AS_TRANSCFG_LO, + AS_TRANSCFG_HI, + AS_MEMATTR_LO, + AS_MEMATTR_HI, + AS_FAULTSTATUS, + AS_FAULTADDRESS_LO, + AS_FAULTADDRESS_HI, + AS_STATUS +}; + +bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, + int reg_range) +{ + int i, j; + int offset = 0; + int slot_number; + int as_number; + + if (kctx->reg_dump == NULL) + return false; + + slot_number = kctx->kbdev->gpu_props.num_job_slots; + as_number = kctx->kbdev->gpu_props.num_address_spaces; + + /* get the GPU control registers*/ + for (i = 0; i < sizeof(gpu_control_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + GPU_CONTROL_REG(gpu_control_reg_snapshot[i]); + offset += 2; + } + + /* get the Job control registers*/ + for (i = 0; i < sizeof(job_control_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + JOB_CONTROL_REG(job_control_reg_snapshot[i]); + offset += 2; + } + + /* get the Job Slot registers*/ + for (j = 0; j < slot_number; j++) { + for (i = 0; i < sizeof(job_slot_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + JOB_SLOT_REG(j, job_slot_reg_snapshot[i]); + offset += 2; + } + } + + /* get the MMU registers*/ + for (i = 0; i < sizeof(mmu_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = MMU_REG(mmu_reg_snapshot[i]); + offset += 2; + } + + /* get the Address space registers*/ + for (j = 0; j < as_number; j++) { + for (i = 0; i < sizeof(as_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + MMU_AS_REG(j, as_reg_snapshot[i]); + offset += 2; + } + } + + WARN_ON(offset >= (reg_range*2/4)); + + /* set the termination flag*/ + kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG; + kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG; + + dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n", + offset); + + return true; +} + +bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx) +{ + int offset = 0; + + if (kctx->reg_dump == NULL) + return false; + + while (kctx->reg_dump[offset] != REGISTER_DUMP_TERMINATION_FLAG) { + kctx->reg_dump[offset+1] = + kbase_reg_read(kctx->kbdev, + kctx->reg_dump[offset]); + offset += 2; + } + return true; +} + + +#endif diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c new file mode 100755 index 000000000000..ff561d180247 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c @@ -0,0 +1,847 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include + +#include +#include +#include +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif + +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) +#include +#else /* Linux >= 3.13 */ +/* In 3.13 the OPP include header file, types, and functions were all + * renamed. Use the old filename for the include, and define the new names to + * the old, when an old kernel is detected. + */ +#include +#define dev_pm_opp opp +#define dev_pm_opp_get_voltage opp_get_voltage +#define dev_pm_opp_get_opp_count opp_get_opp_count +#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil +#define dev_pm_opp_find_freq_floor opp_find_freq_floor +#endif /* Linux >= 3.13 */ +#include +#include +#include + +static struct devfreq_simple_ondemand_data ondemand_data; + +static struct monitor_dev_profile mali_mdevp = { + .type = MONITOR_TPYE_DEV, + .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, + .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, +}; + +/** + * opp_translate - Translate nominal OPP frequency from devicetree into real + * frequency and core mask + * @kbdev: Device pointer + * @freq: Nominal frequency + * @volt: Nominal voltage + * @core_mask: Pointer to u64 to store core mask to + * @freqs: Pointer to array of frequencies + * @volts: Pointer to array of voltages + * + * This function will only perform translation if an operating-points-v2-mali + * table is present in devicetree. If one is not present then it will return an + * untranslated frequency and all cores enabled. + */ +static void opp_translate(struct kbase_device *kbdev, unsigned long freq, + unsigned long volt, u64 *core_mask, + unsigned long *freqs, unsigned long *volts) +{ + unsigned int i; + + for (i = 0; i < kbdev->num_opps; i++) { + if (kbdev->devfreq_table[i].opp_freq == freq) { + unsigned int j; + + *core_mask = kbdev->devfreq_table[i].core_mask; + for (j = 0; j < kbdev->nr_clocks; j++) { + freqs[j] = + kbdev->devfreq_table[i].real_freqs[j]; + volts[j] = + kbdev->devfreq_table[i].opp_volts[j]; + } + + break; + } + } + + /* If failed to find OPP, return all cores enabled + * and nominal frequency + */ + if (i == kbdev->num_opps) { + *core_mask = kbdev->gpu_props.props.raw_props.shader_present; + for (i = 0; i < kbdev->nr_clocks; i++) { + freqs[i] = freq; + volts[i] = volt; + } + } +} + +static int +kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct dev_pm_opp *opp; + unsigned long nominal_freq, nominal_volt; + unsigned long freqs[BASE_MAX_NR_CLOCKS_REGULATORS] = {0}; + unsigned long old_freqs[BASE_MAX_NR_CLOCKS_REGULATORS] = {0}; + unsigned long volts[BASE_MAX_NR_CLOCKS_REGULATORS] = {0}; + unsigned int i; + u64 core_mask = 0; + + nominal_freq = *target_freq; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) + rcu_read_lock(); +#endif + opp = devfreq_recommended_opp(dev, &nominal_freq, flags); + if (IS_ERR_OR_NULL(opp)) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) + rcu_read_unlock(); +#endif + dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); + return PTR_ERR(opp); + } + nominal_volt = dev_pm_opp_get_voltage(opp); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) + rcu_read_unlock(); +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) + dev_pm_opp_put(opp); +#endif + + opp_translate(kbdev, nominal_freq, nominal_volt, &core_mask, freqs, + volts); + + /* + * Only update if there is a change of frequency + */ + if (kbdev->current_nominal_freq == nominal_freq) { + unsigned int i; + int err; + + *target_freq = nominal_freq; + +#ifdef CONFIG_REGULATOR + for (i = 0; i < kbdev->nr_regulators; i++) { + if (kbdev->current_voltages[i] == volts[i]) + continue; + + err = regulator_set_voltage(kbdev->regulators[i], + volts[i], + INT_MAX); + if (err) { + dev_err(dev, "Failed to set voltage (%d)\n", err); + return err; + } + kbdev->current_voltages[i] = volts[i]; + } +#endif + return 0; + } + dev_dbg(dev, "%lu-->%lu\n", kbdev->current_nominal_freq, nominal_freq); + +#ifdef CONFIG_REGULATOR + /* Regulators and clocks work in pairs: every clock has a regulator, + * and we never expect to have more regulators than clocks. + * + * We always need to increase the voltage before increasing + * the frequency of a regulator/clock pair, otherwise the clock + * wouldn't have enough power to perform the transition. + * + * It's always safer to decrease the frequency before decreasing + * voltage of a regulator/clock pair, otherwise the clock could have + * problems operating if it is deprived of the necessary power + * to sustain its current frequency (even if that happens for a short + * transition interval). + */ + + for (i = 0; i < kbdev->nr_clocks; i++) + old_freqs[i] = kbdev->current_freqs[i]; + + for (i = 0; i < kbdev->nr_clocks; i++) { + if (kbdev->regulators[i] && + kbdev->current_voltages[i] != volts[i] && + old_freqs[i] < freqs[i]) { + int err; + + err = regulator_set_voltage(kbdev->regulators[i], + volts[i], INT_MAX); + if (!err) { + kbdev->current_voltages[i] = volts[i]; + } else { + dev_err(dev, "Failed to increase voltage (%d) (target %lu)\n", + err, volts[i]); + return err; + } + } + } +#endif + + for (i = 0; i < kbdev->nr_clocks; i++) { + if (kbdev->clocks[i]) { + int err; + + err = clk_set_rate(kbdev->clocks[i], freqs[i]); + if (!err) { + kbdev->current_freqs[i] = freqs[i]; + } else { + dev_err(dev, "Failed to set clock %lu (target %lu)\n", + freqs[i], *target_freq); + return err; + } + } + } + +#ifdef CONFIG_REGULATOR + for (i = 0; i < kbdev->nr_clocks; i++) { + if (kbdev->regulators[i] && + kbdev->current_voltages[i] != volts[i] && + old_freqs[i] > freqs[i]) { + int err; + + err = regulator_set_voltage(kbdev->regulators[i], + volts[i], INT_MAX); + if (!err) { + kbdev->current_voltages[i] = volts[i]; + } else { + dev_err(dev, "Failed to decrease voltage (%d) (target %lu)\n", + err, volts[i]); + return err; + } + } + } +#endif + + kbase_devfreq_set_core_mask(kbdev, core_mask); + + *target_freq = nominal_freq; + kbdev->current_nominal_freq = nominal_freq; + kbdev->current_core_mask = core_mask; + if (kbdev->devfreq) + kbdev->devfreq->last_status.current_frequency = nominal_freq; + + KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(kbdev, (u64)nominal_freq); + + return 0; +} + +void kbase_devfreq_force_freq(struct kbase_device *kbdev, unsigned long freq) +{ + unsigned long target_freq = freq; + + kbase_devfreq_target(kbdev->dev, &target_freq, 0); +} + +static int +kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + *freq = kbdev->current_nominal_freq; + + return 0; +} + +static int +kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct kbasep_pm_metrics diff; + + kbase_pm_get_dvfs_metrics(kbdev, &kbdev->last_devfreq_metrics, &diff); + + stat->busy_time = diff.time_busy; + stat->total_time = diff.time_busy + diff.time_idle; + stat->current_frequency = kbdev->current_nominal_freq; + stat->private_data = NULL; + + return 0; +} + +static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, + struct devfreq_dev_profile *dp) +{ + int count; + int i = 0; + unsigned long freq; + struct dev_pm_opp *opp; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) + rcu_read_lock(); +#endif + count = dev_pm_opp_get_opp_count(kbdev->dev); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) + rcu_read_unlock(); +#endif + if (count < 0) + return count; + + dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), + GFP_KERNEL); + if (!dp->freq_table) + return -ENOMEM; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) + rcu_read_lock(); +#endif + for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) { + opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq); + if (IS_ERR(opp)) + break; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) + dev_pm_opp_put(opp); +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) */ + + dp->freq_table[i] = freq; + } +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) + rcu_read_unlock(); +#endif + + if (count != i) + dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n", + count, i); + + dp->max_state = i; + + /* Have the lowest clock as suspend clock. + * It may be overridden by 'opp-mali-errata-1485982'. + */ + if (kbdev->pm.backend.gpu_clock_slow_down_wa) { + freq = 0; + opp = dev_pm_opp_find_freq_ceil(kbdev->dev, &freq); + if (IS_ERR(opp)) { + dev_err(kbdev->dev, "failed to find slowest clock"); + return 0; + } + dev_pm_opp_put(opp); + dev_info(kbdev->dev, "suspend clock %lu from slowest", freq); + kbdev->pm.backend.gpu_clock_suspend_freq = freq; + } + + return 0; +} + +static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev) +{ + struct devfreq_dev_profile *dp = &kbdev->devfreq_profile; + + kfree(dp->freq_table); +} + +static void kbase_devfreq_term_core_mask_table(struct kbase_device *kbdev) +{ + kfree(kbdev->devfreq_table); +} + +static void kbase_devfreq_exit(struct device *dev) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + kbase_devfreq_term_freq_table(kbdev); +} + +static void kbasep_devfreq_read_suspend_clock(struct kbase_device *kbdev, + struct device_node *node) +{ + u64 freq = 0; + int err = 0; + + /* Check if this node is the opp entry having 'opp-mali-errata-1485982' + * to get the suspend clock, otherwise skip it. + */ + if (!of_property_read_bool(node, "opp-mali-errata-1485982")) + return; + + /* In kbase DevFreq, the clock will be read from 'opp-hz' + * and translated into the actual clock by opp_translate. + * + * In customer DVFS, the clock will be read from 'opp-hz-real' + * for clk driver. If 'opp-hz-real' does not exist, + * read from 'opp-hz'. + */ + if (IS_ENABLED(CONFIG_MALI_BIFROST_DEVFREQ)) + err = of_property_read_u64(node, "opp-hz", &freq); + else { + if (of_property_read_u64(node, "opp-hz-real", &freq)) + err = of_property_read_u64(node, "opp-hz", &freq); + } + + if (WARN_ON(err || !freq)) + return; + + kbdev->pm.backend.gpu_clock_suspend_freq = freq; + dev_info(kbdev->dev, + "suspend clock %llu by opp-mali-errata-1485982", freq); +} + +static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) +{ +#if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF) + /* OPP table initialization requires at least the capability to get + * regulators and clocks from the device tree, as well as parsing + * arrays of unsigned integer values. + * + * The whole initialization process shall simply be skipped if the + * minimum capability is not available. + */ + return 0; +#else + struct device_node *opp_node = of_parse_phandle(kbdev->dev->of_node, + "operating-points-v2", 0); + struct device_node *node; + int i = 0; + int count; + u64 shader_present = kbdev->gpu_props.props.raw_props.shader_present; + + if (!opp_node) + return 0; + if (!of_device_is_compatible(opp_node, "operating-points-v2-mali")) + return 0; + + count = dev_pm_opp_get_opp_count(kbdev->dev); + kbdev->devfreq_table = kmalloc_array(count, + sizeof(struct kbase_devfreq_opp), GFP_KERNEL); + if (!kbdev->devfreq_table) + return -ENOMEM; + + for_each_available_child_of_node(opp_node, node) { + const void *core_count_p; + u64 core_mask, opp_freq, + real_freqs[BASE_MAX_NR_CLOCKS_REGULATORS]; + int err; +#ifdef CONFIG_REGULATOR + u32 opp_volts[BASE_MAX_NR_CLOCKS_REGULATORS]; +#endif + + /* Read suspend clock from opp table */ + if (kbdev->pm.backend.gpu_clock_slow_down_wa) + kbasep_devfreq_read_suspend_clock(kbdev, node); + + err = of_property_read_u64(node, "opp-hz", &opp_freq); + if (err) { + dev_warn(kbdev->dev, "Failed to read opp-hz property with error %d\n", + err); + continue; + } + + +#if BASE_MAX_NR_CLOCKS_REGULATORS > 1 + err = of_property_read_u64_array(node, "opp-hz-real", + real_freqs, kbdev->nr_clocks); +#else + WARN_ON(kbdev->nr_clocks != 1); + err = of_property_read_u64(node, "opp-hz-real", real_freqs); +#endif + if (err < 0) { + dev_warn(kbdev->dev, "Failed to read opp-hz-real property with error %d\n", + err); + continue; + } +#ifdef CONFIG_REGULATOR + err = of_property_read_u32_array(node, + "opp-microvolt", opp_volts, kbdev->nr_regulators); + if (err < 0) { + dev_warn(kbdev->dev, "Failed to read opp-microvolt property with error %d\n", + err); + continue; + } +#endif + + if (of_property_read_u64(node, "opp-core-mask", &core_mask)) + core_mask = shader_present; + if (core_mask != shader_present && corestack_driver_control) { + + dev_warn(kbdev->dev, "Ignoring OPP %llu - Dynamic Core Scaling not supported on this GPU\n", + opp_freq); + continue; + } + + core_count_p = of_get_property(node, "opp-core-count", NULL); + if (core_count_p) { + u64 remaining_core_mask = + kbdev->gpu_props.props.raw_props.shader_present; + int core_count = be32_to_cpup(core_count_p); + + core_mask = 0; + + for (; core_count > 0; core_count--) { + int core = ffs(remaining_core_mask); + + if (!core) { + dev_err(kbdev->dev, "OPP has more cores than GPU\n"); + return -ENODEV; + } + + core_mask |= (1ull << (core-1)); + remaining_core_mask &= ~(1ull << (core-1)); + } + } + + if (!core_mask) { + dev_err(kbdev->dev, "OPP has invalid core mask of 0\n"); + return -ENODEV; + } + + kbdev->devfreq_table[i].opp_freq = opp_freq; + kbdev->devfreq_table[i].core_mask = core_mask; + if (kbdev->nr_clocks > 0) { + int j; + + for (j = 0; j < kbdev->nr_clocks; j++) + kbdev->devfreq_table[i].real_freqs[j] = + real_freqs[j]; + } +#ifdef CONFIG_REGULATOR + if (kbdev->nr_regulators > 0) { + int j; + + for (j = 0; j < kbdev->nr_regulators; j++) + kbdev->devfreq_table[i].opp_volts[j] = + opp_volts[j]; + } +#endif + + dev_info(kbdev->dev, "OPP %d : opp_freq=%llu core_mask=%llx\n", + i, opp_freq, core_mask); + + i++; + } + + kbdev->num_opps = i; + + return 0; +#endif /* KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE */ +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + +static const char *kbase_devfreq_req_type_name(enum kbase_devfreq_work_type type) +{ + const char *p; + + switch (type) { + case DEVFREQ_WORK_NONE: + p = "devfreq_none"; + break; + case DEVFREQ_WORK_SUSPEND: + p = "devfreq_suspend"; + break; + case DEVFREQ_WORK_RESUME: + p = "devfreq_resume"; + break; + default: + p = "Unknown devfreq_type"; + } + return p; +} + +static void kbase_devfreq_suspend_resume_worker(struct work_struct *work) +{ + struct kbase_devfreq_queue_info *info = container_of(work, + struct kbase_devfreq_queue_info, work); + struct kbase_device *kbdev = container_of(info, struct kbase_device, + devfreq_queue); + unsigned long flags; + enum kbase_devfreq_work_type type, acted_type; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + type = kbdev->devfreq_queue.req_type; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + acted_type = kbdev->devfreq_queue.acted_type; + dev_dbg(kbdev->dev, "Worker handles queued req: %s (acted: %s)\n", + kbase_devfreq_req_type_name(type), + kbase_devfreq_req_type_name(acted_type)); + switch (type) { + case DEVFREQ_WORK_SUSPEND: + case DEVFREQ_WORK_RESUME: + if (type != acted_type) { + if (type == DEVFREQ_WORK_RESUME) + devfreq_resume_device(kbdev->devfreq); + else + devfreq_suspend_device(kbdev->devfreq); + dev_dbg(kbdev->dev, "Devfreq transition occured: %s => %s\n", + kbase_devfreq_req_type_name(acted_type), + kbase_devfreq_req_type_name(type)); + kbdev->devfreq_queue.acted_type = type; + } + break; + default: + WARN_ON(1); + } +} + +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) */ + +void kbase_devfreq_enqueue_work(struct kbase_device *kbdev, + enum kbase_devfreq_work_type work_type) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + unsigned long flags; + + WARN_ON(work_type == DEVFREQ_WORK_NONE); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->devfreq_queue.req_type = work_type; + queue_work(kbdev->devfreq_queue.workq, &kbdev->devfreq_queue.work); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + dev_dbg(kbdev->dev, "Enqueuing devfreq req: %s\n", + kbase_devfreq_req_type_name(work_type)); +#endif +} + +static int kbase_devfreq_work_init(struct kbase_device *kbdev) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + kbdev->devfreq_queue.req_type = DEVFREQ_WORK_NONE; + kbdev->devfreq_queue.acted_type = DEVFREQ_WORK_RESUME; + + kbdev->devfreq_queue.workq = alloc_ordered_workqueue("devfreq_workq", 0); + if (!kbdev->devfreq_queue.workq) + return -ENOMEM; + + INIT_WORK(&kbdev->devfreq_queue.work, + kbase_devfreq_suspend_resume_worker); +#endif + return 0; +} + +static void kbase_devfreq_work_term(struct kbase_device *kbdev) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + destroy_workqueue(kbdev->devfreq_queue.workq); +#endif +} + +static unsigned long kbase_devfreq_get_static_power(struct devfreq *devfreq, + unsigned long voltage) +{ + struct device *dev = devfreq->dev.parent; + struct kbase_device *kbdev = dev_get_drvdata(dev); + + return rockchip_ipa_get_static_power(kbdev->model_data, voltage); +} + +static struct devfreq_cooling_power kbase_cooling_power = { + .get_static_power = &kbase_devfreq_get_static_power, +}; + +int kbase_devfreq_init(struct kbase_device *kbdev) +{ + struct devfreq_cooling_power *kbase_dcp = &kbase_cooling_power; + struct device_node *np = kbdev->dev->of_node; + struct devfreq_dev_profile *dp; + struct dev_pm_opp *opp; + unsigned long opp_rate; + int err; + unsigned int i; + + if (kbdev->nr_clocks == 0) { + dev_err(kbdev->dev, "Clock not available for devfreq\n"); + return -ENODEV; + } + + for (i = 0; i < kbdev->nr_clocks; i++) { + if (kbdev->clocks[i]) + kbdev->current_freqs[i] = + clk_get_rate(kbdev->clocks[i]); + else + kbdev->current_freqs[i] = 0; + } + kbdev->current_nominal_freq = kbdev->current_freqs[0]; + + dp = &kbdev->devfreq_profile; + + dp->initial_freq = kbdev->current_freqs[0]; + dp->polling_ms = 100; + dp->target = kbase_devfreq_target; + dp->get_dev_status = kbase_devfreq_status; + dp->get_cur_freq = kbase_devfreq_cur_freq; + dp->exit = kbase_devfreq_exit; + + if (kbase_devfreq_init_freq_table(kbdev, dp)) + return -EFAULT; + + if (dp->max_state > 0) { + /* Record the maximum frequency possible */ + kbdev->gpu_props.props.core_props.gpu_freq_khz_max = + dp->freq_table[0] / 1000; + }; + + err = kbase_devfreq_init_core_mask_table(kbdev); + if (err) { + kbase_devfreq_term_freq_table(kbdev); + return err; + } + + /* Initialise devfreq suspend/resume workqueue */ + err = kbase_devfreq_work_init(kbdev); + if (err) { + kbase_devfreq_term_freq_table(kbdev); + dev_err(kbdev->dev, "Devfreq initialization failed"); + return err; + } + + of_property_read_u32(np, "upthreshold", + &ondemand_data.upthreshold); + of_property_read_u32(np, "downdifferential", + &ondemand_data.downdifferential); + kbdev->devfreq = devfreq_add_device(kbdev->dev, dp, + "simple_ondemand", &ondemand_data); + if (IS_ERR(kbdev->devfreq)) { + err = PTR_ERR(kbdev->devfreq); + kbase_devfreq_work_term(kbdev); + kbase_devfreq_term_freq_table(kbdev); + return err; + } + + /* devfreq_add_device only copies a few of kbdev->dev's fields, so + * set drvdata explicitly so IPA models can access kbdev. */ + dev_set_drvdata(&kbdev->devfreq->dev, kbdev); + + err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq); + if (err) { + dev_err(kbdev->dev, + "Failed to register OPP notifier (%d)\n", err); + goto opp_notifier_failed; + } + + opp_rate = kbdev->current_freqs[0]; /* Bifrost GPU has only 1 clock. */ + opp = devfreq_recommended_opp(kbdev->dev, &opp_rate, 0); + if (!IS_ERR(opp)) + dev_pm_opp_put(opp); + kbdev->devfreq->last_status.current_frequency = opp_rate; + + mali_mdevp.data = kbdev->devfreq; + kbdev->mdev_info = rockchip_system_monitor_register(kbdev->dev, + &mali_mdevp); + if (IS_ERR(kbdev->mdev_info)) { + dev_dbg(kbdev->dev, "without system monitor\n"); + kbdev->mdev_info = NULL; + } +#ifdef CONFIG_DEVFREQ_THERMAL + if (of_find_compatible_node(kbdev->dev->of_node, NULL, + "simple-power-model")) { + of_property_read_u32(kbdev->dev->of_node, + "dynamic-power-coefficient", + (u32 *)&kbase_dcp->dyn_power_coeff); + kbdev->model_data = rockchip_ipa_power_model_init(kbdev->dev, + "gpu_leakage"); + if (IS_ERR_OR_NULL(kbdev->model_data)) { + kbdev->model_data = NULL; + dev_err(kbdev->dev, "failed to initialize power model\n"); + } else if (kbdev->model_data->dynamic_coefficient) { + kbase_dcp->dyn_power_coeff = + kbdev->model_data->dynamic_coefficient; + } + if (!kbase_dcp->dyn_power_coeff) { + err = -EINVAL; + dev_err(kbdev->dev, "failed to get dynamic-coefficient\n"); + goto cooling_failed; + } + + kbdev->devfreq_cooling = + of_devfreq_cooling_register_power(kbdev->dev->of_node, + kbdev->devfreq, + kbase_dcp); + if (IS_ERR(kbdev->devfreq_cooling)) { + err = PTR_ERR(kbdev->devfreq_cooling); + dev_err(kbdev->dev, "failed to register cooling device\n"); + goto cooling_failed; + } + } else { + err = kbase_ipa_init(kbdev); + if (err) { + dev_err(kbdev->dev, "IPA initialization failed\n"); + goto cooling_failed; + } + + kbdev->devfreq_cooling = of_devfreq_cooling_register_power( + kbdev->dev->of_node, + kbdev->devfreq, + &kbase_ipa_power_model_ops); + if (IS_ERR(kbdev->devfreq_cooling)) { + err = PTR_ERR(kbdev->devfreq_cooling); + dev_err(kbdev->dev, + "Failed to register cooling device (%d)\n", + err); + goto cooling_failed; + } + } +#endif + + return 0; + +#ifdef CONFIG_DEVFREQ_THERMAL +cooling_failed: + devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); +#endif /* CONFIG_DEVFREQ_THERMAL */ +opp_notifier_failed: + if (devfreq_remove_device(kbdev->devfreq)) + dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); + else + kbdev->devfreq = NULL; + + kbase_devfreq_work_term(kbdev); + + return err; +} + +void kbase_devfreq_term(struct kbase_device *kbdev) +{ + int err; + + dev_dbg(kbdev->dev, "Term Mali devfreq\n"); + + rockchip_system_monitor_unregister(kbdev->mdev_info); +#ifdef CONFIG_DEVFREQ_THERMAL + if (kbdev->devfreq_cooling) + devfreq_cooling_unregister(kbdev->devfreq_cooling); + + if (!kbdev->model_data) + kbase_ipa_term(kbdev); + kfree(kbdev->model_data); +#endif + + devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); + + err = devfreq_remove_device(kbdev->devfreq); + if (err) + dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); + else + kbdev->devfreq = NULL; + + kbase_devfreq_term_core_mask_table(kbdev); + + kbase_devfreq_work_term(kbdev); +} diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.h new file mode 100755 index 000000000000..7bcd47c70ef0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.h @@ -0,0 +1,47 @@ +/* + * + * (C) COPYRIGHT 2014, 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _BASE_DEVFREQ_H_ +#define _BASE_DEVFREQ_H_ + +int kbase_devfreq_init(struct kbase_device *kbdev); + +void kbase_devfreq_term(struct kbase_device *kbdev); +int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev); + +/** + * kbase_devfreq_force_freq - Set GPU frequency on L2 power on/off. + * @kbdev: Device pointer + * @freq: GPU frequency in HZ to be set when + * MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE is enabled + */ +void kbase_devfreq_force_freq(struct kbase_device *kbdev, unsigned long freq); + +/** + * kbase_devfreq_enqueue_work - Enqueue a work item for suspend/resume devfreq. + * @kbdev: Device pointer + * @work_type: The type of the devfreq work item, i.e. suspend or resume + */ +void kbase_devfreq_enqueue_work(struct kbase_device *kbdev, + enum kbase_devfreq_work_type work_type); + +#endif /* _BASE_DEVFREQ_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c new file mode 100755 index 000000000000..60ae0206d6a8 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Base kernel property query backend APIs + */ + +#include +#include +#include +#include + +int kbase_backend_gpuprops_get(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump) +{ + int i; + struct kbase_gpuprops_regdump registers; + + /* Fill regdump with the content of the relevant registers */ + registers.gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); + + registers.l2_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_FEATURES)); +#if !MALI_USE_CSF + registers.core_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(CORE_FEATURES)); +#else /* !MALI_USE_CSF */ + registers.core_features = 0; +#endif /* !MALI_USE_CSF */ + registers.tiler_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_FEATURES)); + registers.mem_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(MEM_FEATURES)); + registers.mmu_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(MMU_FEATURES)); + registers.as_present = kbase_reg_read(kbdev, + GPU_CONTROL_REG(AS_PRESENT)); +#if !MALI_USE_CSF + registers.js_present = kbase_reg_read(kbdev, + GPU_CONTROL_REG(JS_PRESENT)); +#else /* !MALI_USE_CSF */ + registers.js_present = 0; +#endif /* !MALI_USE_CSF */ + + for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) +#if !MALI_USE_CSF + registers.js_features[i] = kbase_reg_read(kbdev, + GPU_CONTROL_REG(JS_FEATURES_REG(i))); +#else /* !MALI_USE_CSF */ + registers.js_features[i] = 0; +#endif /* !MALI_USE_CSF */ + + for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) + registers.texture_features[i] = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i))); + + registers.thread_max_threads = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_MAX_THREADS)); + registers.thread_max_workgroup_size = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE)); + registers.thread_max_barrier_size = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE)); + registers.thread_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_FEATURES)); + registers.thread_tls_alloc = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_TLS_ALLOC)); + + registers.shader_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_PRESENT_LO)); + registers.shader_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_PRESENT_HI)); + + registers.tiler_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_PRESENT_LO)); + registers.tiler_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_PRESENT_HI)); + + registers.l2_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_PRESENT_LO)); + registers.l2_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_PRESENT_HI)); + + registers.stack_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(STACK_PRESENT_LO)); + registers.stack_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(STACK_PRESENT_HI)); + + if (!kbase_is_gpu_removed(kbdev)) { + *regdump = registers; + return 0; + } else + return -EIO; +} + +int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump) +{ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) { + u32 coherency_features; + + /* Ensure we can access the GPU registers */ + kbase_pm_register_access_enable(kbdev); + + coherency_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(COHERENCY_FEATURES)); + + if (kbase_is_gpu_removed(kbdev)) + return -EIO; + + regdump->coherency_features = coherency_features; + + /* We're done accessing the GPU registers for now. */ + kbase_pm_register_access_disable(kbdev); + } else { + /* Pre COHERENCY_FEATURES we only supported ACE_LITE */ + regdump->coherency_features = + COHERENCY_FEATURE_BIT(COHERENCY_NONE) | + COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); + } + + return 0; +} + +int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump) +{ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { + u32 l2_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_FEATURES)); + + if (kbase_is_gpu_removed(kbdev)) + return -EIO; + + regdump->l2_features = l2_features; + } + + return 0; +} diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c new file mode 100755 index 000000000000..54b07483dee6 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c @@ -0,0 +1,520 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * GPU backend instrumentation APIs. + */ + +#include +#include +#include +#include +#include + + +int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_instr_hwcnt_enable *enable) +{ + unsigned long flags; + int err = -EINVAL; +#if !MALI_USE_CSF + u32 irq_mask; +#endif + u32 prfcnt_config; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* alignment failure */ + if ((enable->dump_buffer == 0ULL) || (enable->dump_buffer & (2048 - 1))) + goto out_err; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { + /* Instrumentation is already enabled */ + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + goto out_err; + } + +#if !MALI_USE_CSF + /* Enable interrupt */ + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | + PRFCNT_SAMPLE_COMPLETED); +#endif + + /* In use, this context is the owner */ + kbdev->hwcnt.kctx = kctx; + /* Remember the dump address so we can reprogram it later */ + kbdev->hwcnt.addr = enable->dump_buffer; + kbdev->hwcnt.addr_bytes = enable->dump_buffer_bytes; + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + /* Configure */ + prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; +#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS + if (kbdev->hwcnt.backend.use_secondary_override) +#else + if (enable->use_secondary) +#endif + prfcnt_config |= 1 << PRFCNT_CONFIG_SETSELECT_SHIFT; + +#if MALI_USE_CSF + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CONFIG), + prfcnt_config | PRFCNT_CONFIG_MODE_OFF); + + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_LO), + enable->dump_buffer & 0xFFFFFFFF); + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_HI), + enable->dump_buffer >> 32); + + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CSHW_EN), + enable->fe_bm); + + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_SHADER_EN), + enable->shader_bm); + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_MMU_L2_EN), + enable->mmu_l2_bm); + + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_TILER_EN), + enable->tiler_bm); + + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CONFIG), + prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL); +#else + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), + prfcnt_config | PRFCNT_CONFIG_MODE_OFF); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), + enable->dump_buffer & 0xFFFFFFFF); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), + enable->dump_buffer >> 32); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), + enable->fe_bm); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), + enable->shader_bm); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), + enable->mmu_l2_bm); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), + enable->tiler_bm); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), + prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL); +#endif + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + err = 0; + + dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx); + return err; + out_err: + return err; +} + +int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) +{ + unsigned long flags, pm_flags; + int err = -EINVAL; +#if !MALI_USE_CSF + u32 irq_mask; +#endif + struct kbase_device *kbdev = kctx->kbdev; + + while (1) { + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) { + /* Instrumentation is not enabled */ + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + goto out; + } + + if (kbdev->hwcnt.kctx != kctx) { + /* Instrumentation has been setup for another context */ + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + goto out; + } + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) + break; + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + + /* Ongoing dump/setup - wait for its completion */ + wait_event(kbdev->hwcnt.backend.wait, + kbdev->hwcnt.backend.triggered != 0); + } + + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; + kbdev->hwcnt.backend.triggered = 0; + +#if MALI_USE_CSF + /* Disable the counters */ + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_CONFIG), 0); +#else + /* Disable interrupt */ + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + irq_mask & ~PRFCNT_SAMPLE_COMPLETED); + + /* Disable the counters */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0); +#endif + + kbdev->hwcnt.kctx = NULL; + kbdev->hwcnt.addr = 0ULL; + kbdev->hwcnt.addr_bytes = 0ULL; + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + + dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", + kctx); + + err = 0; + + out: + return err; +} + +int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) +{ + unsigned long flags; + int err = -EINVAL; + struct kbase_device *kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.kctx != kctx) { + /* The instrumentation has been setup for another context */ + goto unlock; + } + + if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { + /* HW counters are disabled or another dump is ongoing, or we're + * resetting */ + goto unlock; + } + + kbdev->hwcnt.backend.triggered = 0; + + /* Mark that we're dumping - the PF handler can signal that we faulted + */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; + + +#if MALI_USE_CSF + /* Reconfigure the dump address */ + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_LO), + kbdev->hwcnt.addr & 0xFFFFFFFF); + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(PRFCNT_BASE_HI), + kbdev->hwcnt.addr >> 32); +#else + /* Reconfigure the dump address */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), + kbdev->hwcnt.addr & 0xFFFFFFFF); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), + kbdev->hwcnt.addr >> 32); +#endif + + /* Start dumping */ + KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, + kbdev->hwcnt.addr); + +#if MALI_USE_CSF + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(GPU_COMMAND), + GPU_COMMAND_PRFCNT_SAMPLE); +#else + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_PRFCNT_SAMPLE); +#endif + + dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx); + + err = 0; + + unlock: + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + +#if MALI_USE_CSF + tasklet_schedule(&kbdev->hwcnt.backend.csf_hwc_irq_poll_tasklet); +#endif + + return err; +} +KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); + +bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, + bool * const success) +{ + unsigned long flags; + bool complete = false; + struct kbase_device *kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) { + *success = true; + complete = true; + } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { + *success = false; + complete = true; + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + return complete; +} +KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); + +void kbasep_cache_clean_worker(struct work_struct *data) +{ + struct kbase_device *kbdev; + unsigned long flags, pm_flags; + + kbdev = container_of(data, struct kbase_device, + hwcnt.backend.cache_clean_work); + + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + /* Clean and invalidate the caches so we're sure the mmu tables for the + * dump buffer is valid. + */ + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_REQUEST_CLEAN); + kbase_gpu_start_cache_clean_nolock(kbdev); + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + + kbase_gpu_wait_cache_clean(kbdev); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_REQUEST_CLEAN); + /* All finished and idle */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); +} + +#if MALI_USE_CSF +/** + * kbasep_hwcnt_irq_poll_tasklet - tasklet to poll MCU IRQ status register + * + * @data: tasklet parameter which pointer to kbdev + * + * This tasklet poll GPU_IRQ_STATUS register in GPU_CONTROL_MCU page to check + * PRFCNT_SAMPLE_COMPLETED bit. + * + * Tasklet is needed here since work_queue is too slow and cuased some test + * cases timeout, the poll_count variable is introduced to avoid infinite + * loop in unexpected cases, the poll_count is 1 or 2 in normal case, 128 + * should be big enough to exit the tasklet in abnormal cases. + * + * Return: void + */ +static void kbasep_hwcnt_irq_poll_tasklet(unsigned long int data) +{ + struct kbase_device *kbdev = (struct kbase_device *)data; + unsigned long flags, pm_flags; + u32 mcu_gpu_irq_raw_status = 0; + u32 poll_count = 0; + + while (1) { + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + mcu_gpu_irq_raw_status = kbase_reg_read(kbdev, + GPU_CONTROL_MCU_REG(GPU_IRQ_RAWSTAT)); + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + if (mcu_gpu_irq_raw_status & PRFCNT_SAMPLE_COMPLETED) { + kbase_reg_write(kbdev, + GPU_CONTROL_MCU_REG(GPU_IRQ_CLEAR), + PRFCNT_SAMPLE_COMPLETED); + kbase_instr_hwcnt_sample_done(kbdev); + break; + } else if (poll_count++ > 128) { + dev_err(kbdev->dev, + "Err: HWC dump timeout, count: %u", poll_count); + /* Still call sample_done to unblock waiting thread */ + kbase_instr_hwcnt_sample_done(kbdev); + break; + } + } +} +#endif + +void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) { + if (kbdev->mmu_mode->flags & KBASE_MMU_MODE_HAS_NON_CACHEABLE) { + /* All finished and idle */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + } else { + int ret; + /* Always clean and invalidate the cache after a successful dump + */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; + ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, + &kbdev->hwcnt.backend.cache_clean_work); + KBASE_DEBUG_ASSERT(ret); + } + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); +} + +int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + unsigned long flags; + int err; + + /* Wait for dump & cache clean to complete */ + wait_event(kbdev->hwcnt.backend.wait, + kbdev->hwcnt.backend.triggered != 0); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { + err = -EINVAL; + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + } else { + /* Dump done */ + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_IDLE); + err = 0; + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + return err; +} + +int kbase_instr_hwcnt_clear(struct kbase_context *kctx) +{ + unsigned long flags; + int err = -EINVAL; + struct kbase_device *kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + /* Check it's the context previously set up and we're not already + * dumping */ + if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != + KBASE_INSTR_STATE_IDLE) + goto out; + + /* Clear the counters */ + KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0); +#if MALI_USE_CSF + kbase_reg_write(kbdev, GPU_CONTROL_MCU_REG(GPU_COMMAND), + GPU_COMMAND_PRFCNT_CLEAR); +#else + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_PRFCNT_CLEAR); +#endif + + err = 0; + +out: + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + return err; +} +KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear); + +int kbase_instr_backend_init(struct kbase_device *kbdev) +{ + int ret = 0; + + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; + + init_waitqueue_head(&kbdev->hwcnt.backend.wait); + INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work, + kbasep_cache_clean_worker); + +#if MALI_USE_CSF + tasklet_init(&kbdev->hwcnt.backend.csf_hwc_irq_poll_tasklet, + kbasep_hwcnt_irq_poll_tasklet, (unsigned long int)kbdev); +#endif + + kbdev->hwcnt.backend.triggered = 0; + +#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS + kbdev->hwcnt.backend.use_secondary_override = false; +#endif + + kbdev->hwcnt.backend.cache_clean_wq = + alloc_workqueue("Mali cache cleaning workqueue", 0, 1); + if (NULL == kbdev->hwcnt.backend.cache_clean_wq) + ret = -EINVAL; + + return ret; +} + +void kbase_instr_backend_term(struct kbase_device *kbdev) +{ +#if MALI_USE_CSF + tasklet_kill(&kbdev->hwcnt.backend.csf_hwc_irq_poll_tasklet); +#endif + destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq); +} + +#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS +void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_bool("hwcnt_use_secondary", S_IRUGO | S_IWUSR, + kbdev->mali_debugfs_directory, + &kbdev->hwcnt.backend.use_secondary_override); +} +#endif diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_defs.h new file mode 100755 index 000000000000..9f785ce16e17 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_defs.h @@ -0,0 +1,63 @@ +/* + * + * (C) COPYRIGHT 2014, 2016, 2018, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Backend-specific instrumentation definitions + */ + +#ifndef _KBASE_INSTR_DEFS_H_ +#define _KBASE_INSTR_DEFS_H_ + +/* + * Instrumentation State Machine States + */ +enum kbase_instr_state { + /* State where instrumentation is not active */ + KBASE_INSTR_STATE_DISABLED = 0, + /* State machine is active and ready for a command. */ + KBASE_INSTR_STATE_IDLE, + /* Hardware is currently dumping a frame. */ + KBASE_INSTR_STATE_DUMPING, + /* We've requested a clean to occur on a workqueue */ + KBASE_INSTR_STATE_REQUEST_CLEAN, + /* An error has occured during DUMPING (page fault). */ + KBASE_INSTR_STATE_FAULT +}; + +/* Structure used for instrumentation and HW counters dumping */ +struct kbase_instr_backend { + wait_queue_head_t wait; + int triggered; +#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS + bool use_secondary_override; +#endif + + enum kbase_instr_state state; + struct workqueue_struct *cache_clean_wq; + struct work_struct cache_clean_work; +#if MALI_USE_CSF + struct tasklet_struct csf_hwc_irq_poll_tasklet; +#endif +}; + +#endif /* _KBASE_INSTR_DEFS_H_ */ + diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_internal.h new file mode 100755 index 000000000000..2254b9f30d02 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_internal.h @@ -0,0 +1,44 @@ +/* + * + * (C) COPYRIGHT 2014, 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Backend-specific HW access instrumentation APIs + */ + +#ifndef _KBASE_INSTR_INTERNAL_H_ +#define _KBASE_INSTR_INTERNAL_H_ + +/** + * kbasep_cache_clean_worker() - Workqueue for handling cache cleaning + * @data: a &struct work_struct + */ +void kbasep_cache_clean_worker(struct work_struct *data); + +/** + * kbase_instr_hwcnt_sample_done() - Dump complete interrupt received + * @kbdev: Kbase device + */ +void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev); + +#endif /* _KBASE_INSTR_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h new file mode 100755 index 000000000000..ca3c048b637a --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h @@ -0,0 +1,44 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Backend specific IRQ APIs + */ + +#ifndef _KBASE_IRQ_INTERNAL_H_ +#define _KBASE_IRQ_INTERNAL_H_ + +int kbase_install_interrupts(struct kbase_device *kbdev); + +void kbase_release_interrupts(struct kbase_device *kbdev); + +/** + * kbase_synchronize_irqs - Ensure that all IRQ handlers have completed + * execution + * @kbdev: The kbase device + */ +void kbase_synchronize_irqs(struct kbase_device *kbdev); + +int kbasep_common_test_interrupt_handlers( + struct kbase_device * const kbdev); + +#endif /* _KBASE_IRQ_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c new file mode 100755 index 000000000000..b09db552e639 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c @@ -0,0 +1,504 @@ +/* + * + * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include + +#include + +#if !defined(CONFIG_MALI_BIFROST_NO_MALI) + +/* GPU IRQ Tags */ +#define JOB_IRQ_TAG 0 +#define MMU_IRQ_TAG 1 +#define GPU_IRQ_TAG 2 + +static void *kbase_tag(void *ptr, u32 tag) +{ + return (void *)(((uintptr_t) ptr) | tag); +} + +static void *kbase_untag(void *ptr) +{ + return (void *)(((uintptr_t) ptr) & ~3); +} + +static irqreturn_t kbase_job_irq_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS)); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (!kbdev->pm.backend.driver_ready_for_irqs) + dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", + __func__, irq, val); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + if (!val) { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return IRQ_NONE; + } + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + +#if MALI_USE_CSF + /* call the csf interrupt handler */ + kbase_csf_interrupt(kbdev, val); +#else + kbase_job_done(kbdev, val); +#endif + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return IRQ_HANDLED; +} + +static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return IRQ_NONE; + } + + atomic_inc(&kbdev->faults_pending); + + val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS)); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (!kbdev->pm.backend.driver_ready_for_irqs) + dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", + __func__, irq, val); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (!val) { + atomic_dec(&kbdev->faults_pending); + return IRQ_NONE; + } + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbase_mmu_interrupt(kbdev, val); + + atomic_dec(&kbdev->faults_pending); + + return IRQ_HANDLED; +} + +static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS)); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (!kbdev->pm.backend.driver_ready_for_irqs) + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", + __func__, irq, val); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbase_gpu_interrupt(kbdev, val); + + return IRQ_HANDLED; +} + +static irq_handler_t kbase_handler_table[] = { + [JOB_IRQ_TAG] = kbase_job_irq_handler, + [MMU_IRQ_TAG] = kbase_mmu_irq_handler, + [GPU_IRQ_TAG] = kbase_gpu_irq_handler, +}; + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define JOB_IRQ_HANDLER JOB_IRQ_TAG +#define MMU_IRQ_HANDLER MMU_IRQ_TAG +#define GPU_IRQ_HANDLER GPU_IRQ_TAG + +/** + * kbase_gpu_irq_test_handler - Variant (for test) of kbase_gpu_irq_handler() + * @irq: IRQ number + * @data: Data associated with this IRQ (i.e. kbdev) + * @val: Value of the GPU_CONTROL_REG(GPU_IRQ_STATUS) + * + * Handle the GPU device interrupt source requests reflected in the + * given source bit-pattern. The test code caller is responsible for + * undertaking the required device power maintenace. + * + * Return: IRQ_HANDLED if the requests are from the GPU device, + * IRQ_NONE otherwise + */ +irqreturn_t kbase_gpu_irq_test_handler(int irq, void *data, u32 val) +{ + struct kbase_device *kbdev = kbase_untag(data); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbase_gpu_interrupt(kbdev, val); + + return IRQ_HANDLED; +} + +KBASE_EXPORT_TEST_API(kbase_gpu_irq_test_handler); + +/** + * kbase_set_custom_irq_handler - Set a custom IRQ handler + * @kbdev: Device for which the handler is to be registered + * @custom_handler: Handler to be registered + * @irq_type: Interrupt type + * + * Registers given interrupt handler for requested interrupt type + * In the case where irq handler is not specified, the default handler shall be + * registered + * + * Return: 0 case success, error code otherwise + */ +int kbase_set_custom_irq_handler(struct kbase_device *kbdev, + irq_handler_t custom_handler, + int irq_type) +{ + int result = 0; + irq_handler_t requested_irq_handler = NULL; + + KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) && + (GPU_IRQ_HANDLER >= irq_type)); + + /* Release previous handler */ + if (kbdev->irqs[irq_type].irq) + free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type)); + + requested_irq_handler = (NULL != custom_handler) ? custom_handler : + kbase_handler_table[irq_type]; + + if (0 != request_irq(kbdev->irqs[irq_type].irq, + requested_irq_handler, + kbdev->irqs[irq_type].flags | IRQF_SHARED, + dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) { + result = -EINVAL; + dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", + kbdev->irqs[irq_type].irq, irq_type); +#ifdef CONFIG_SPARSE_IRQ + dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); +#endif /* CONFIG_SPARSE_IRQ */ + } + + return result; +} + +KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler); + +/* test correct interrupt assigment and reception by cpu */ +struct kbasep_irq_test { + struct hrtimer timer; + wait_queue_head_t wait; + int triggered; + u32 timeout; +}; + +static struct kbasep_irq_test kbasep_irq_test_data; + +#define IRQ_TEST_TIMEOUT 500 + +static irqreturn_t kbase_job_irq_test_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS)); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbasep_irq_test_data.triggered = 1; + wake_up(&kbasep_irq_test_data.wait); + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val); + + return IRQ_HANDLED; +} + +static irqreturn_t kbase_mmu_irq_test_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS)); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbasep_irq_test_data.triggered = 1; + wake_up(&kbasep_irq_test_data.wait); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), val); + + return IRQ_HANDLED; +} + +static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) +{ + struct kbasep_irq_test *test_data = container_of(timer, + struct kbasep_irq_test, timer); + + test_data->timeout = 1; + test_data->triggered = 1; + wake_up(&test_data->wait); + return HRTIMER_NORESTART; +} + +static int kbasep_common_test_interrupt( + struct kbase_device * const kbdev, u32 tag) +{ + int err = 0; + irq_handler_t test_handler; + + u32 old_mask_val; + u16 mask_offset; + u16 rawstat_offset; + + switch (tag) { + case JOB_IRQ_TAG: + test_handler = kbase_job_irq_test_handler; + rawstat_offset = JOB_CONTROL_REG(JOB_IRQ_RAWSTAT); + mask_offset = JOB_CONTROL_REG(JOB_IRQ_MASK); + break; + case MMU_IRQ_TAG: + test_handler = kbase_mmu_irq_test_handler; + rawstat_offset = MMU_REG(MMU_IRQ_RAWSTAT); + mask_offset = MMU_REG(MMU_IRQ_MASK); + break; + case GPU_IRQ_TAG: + /* already tested by pm_driver - bail out */ + default: + return 0; + } + + /* store old mask */ + old_mask_val = kbase_reg_read(kbdev, mask_offset); + /* mask interrupts */ + kbase_reg_write(kbdev, mask_offset, 0x0); + + if (kbdev->irqs[tag].irq) { + /* release original handler and install test handler */ + if (kbase_set_custom_irq_handler(kbdev, test_handler, tag) != 0) { + err = -EINVAL; + } else { + kbasep_irq_test_data.timeout = 0; + hrtimer_init(&kbasep_irq_test_data.timer, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + kbasep_irq_test_data.timer.function = + kbasep_test_interrupt_timeout; + + /* trigger interrupt */ + kbase_reg_write(kbdev, mask_offset, 0x1); + kbase_reg_write(kbdev, rawstat_offset, 0x1); + + hrtimer_start(&kbasep_irq_test_data.timer, + HR_TIMER_DELAY_MSEC(IRQ_TEST_TIMEOUT), + HRTIMER_MODE_REL); + + wait_event(kbasep_irq_test_data.wait, + kbasep_irq_test_data.triggered != 0); + + if (kbasep_irq_test_data.timeout != 0) { + dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n", + kbdev->irqs[tag].irq, tag); + err = -EINVAL; + } else { + dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n", + kbdev->irqs[tag].irq, tag); + } + + hrtimer_cancel(&kbasep_irq_test_data.timer); + kbasep_irq_test_data.triggered = 0; + + /* mask interrupts */ + kbase_reg_write(kbdev, mask_offset, 0x0); + + /* release test handler */ + free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag)); + } + + /* restore original interrupt */ + if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag], + kbdev->irqs[tag].flags | IRQF_SHARED, + dev_name(kbdev->dev), kbase_tag(kbdev, tag))) { + dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n", + kbdev->irqs[tag].irq, tag); + err = -EINVAL; + } + } + /* restore old mask */ + kbase_reg_write(kbdev, mask_offset, old_mask_val); + + return err; +} + +int kbasep_common_test_interrupt_handlers( + struct kbase_device * const kbdev) +{ + int err; + + init_waitqueue_head(&kbasep_irq_test_data.wait); + kbasep_irq_test_data.triggered = 0; + + /* A suspend won't happen during startup/insmod */ + kbase_pm_context_active(kbdev); + + err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG); + if (err) { + dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n"); + goto out; + } + + err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG); + if (err) { + dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n"); + goto out; + } + + dev_dbg(kbdev->dev, "Interrupts are correctly assigned.\n"); + + out: + kbase_pm_context_idle(kbdev); + + return err; +} +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + +int kbase_install_interrupts(struct kbase_device *kbdev) +{ + u32 nr = ARRAY_SIZE(kbase_handler_table); + int err; + u32 i; + + for (i = 0; i < nr; i++) { + err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i], + kbdev->irqs[i].flags | IRQF_SHARED, + dev_name(kbdev->dev), + kbase_tag(kbdev, i)); + if (err) { + dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", + kbdev->irqs[i].irq, i); +#ifdef CONFIG_SPARSE_IRQ + dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); +#endif /* CONFIG_SPARSE_IRQ */ + goto release; + } + } + + return 0; + + release: + while (i-- > 0) + free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); + + return err; +} + +void kbase_release_interrupts(struct kbase_device *kbdev) +{ + u32 nr = ARRAY_SIZE(kbase_handler_table); + u32 i; + + for (i = 0; i < nr; i++) { + if (kbdev->irqs[i].irq) + free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); + } +} + +void kbase_synchronize_irqs(struct kbase_device *kbdev) +{ + u32 nr = ARRAY_SIZE(kbase_handler_table); + u32 i; + + for (i = 0; i < nr; i++) { + if (kbdev->irqs[i].irq) + synchronize_irq(kbdev->irqs[i].irq); + } +} + +KBASE_EXPORT_TEST_API(kbase_synchronize_irqs); + +#endif /* !defined(CONFIG_MALI_BIFROST_NO_MALI) */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c new file mode 100755 index 000000000000..9b775898dac2 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c @@ -0,0 +1,245 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * Register backend context / address space management + */ + +#include +#include +#include + +/** + * assign_and_activate_kctx_addr_space - Assign an AS to a context + * @kbdev: Kbase device + * @kctx: Kbase context + * @current_as: Address Space to assign + * + * Assign an Address Space (AS) to a context, and add the context to the Policy. + * + * This includes + * setting up the global runpool_irq structure and the context on the AS, + * Activating the MMU on the AS, + * Allowing jobs to be submitted on the AS. + * + * Context: + * kbasep_js_kctx_info.jsctx_mutex held, + * kbasep_js_device_data.runpool_mutex held, + * AS transaction mutex held, + * Runpool IRQ lock held + */ +static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_as *current_as) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + +#if !MALI_USE_CSF + /* Attribute handling */ + kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); +#endif + + /* Allow it to run jobs */ + kbasep_js_set_submit_allowed(js_devdata, kctx); + + kbase_js_runpool_inc_context_count(kbdev, kctx); +} + +bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + int i; + + if (kbdev->hwaccess.active_kctx[js] == kctx) { + /* Context is already active */ + return true; + } + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + if (kbdev->as_to_kctx[i] == kctx) { + /* Context already has ASID - mark as active */ + return true; + } + } + + /* Context does not have address space assigned */ + return false; +} + +void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + int as_nr = kctx->as_nr; + + if (as_nr == KBASEP_AS_NR_INVALID) { + WARN(1, "Attempting to release context without ASID\n"); + return; + } + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (atomic_read(&kctx->refcount) != 1) { + WARN(1, "Attempting to release active ASID\n"); + return; + } + + kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx); + + kbase_ctx_sched_release_ctx(kctx); + kbase_js_runpool_dec_context_count(kbdev, kctx); +} + +void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ +} + +int kbase_backend_find_and_release_free_address_space( + struct kbase_device *kbdev, struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + unsigned long flags; + int i; + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + struct kbasep_js_kctx_info *as_js_kctx_info; + struct kbase_context *as_kctx; + + as_kctx = kbdev->as_to_kctx[i]; + as_js_kctx_info = &as_kctx->jctx.sched_info; + + /* Don't release privileged or active contexts, or contexts with + * jobs running. + * Note that a context will have at least 1 reference (which + * was previously taken by kbasep_js_schedule_ctx()) until + * descheduled. + */ + if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && + atomic_read(&as_kctx->refcount) == 1) { + if (!kbase_ctx_sched_inc_refcount_nolock(as_kctx)) { + WARN(1, "Failed to retain active context\n"); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return KBASEP_AS_NR_INVALID; + } + + kbasep_js_clear_submit_allowed(js_devdata, as_kctx); + + /* Drop and retake locks to take the jsctx_mutex on the + * context we're about to release without violating lock + * ordering + */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + + /* Release context from address space */ + mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx); + + if (!kbase_ctx_flag(as_kctx, KCTX_SCHEDULED)) { + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, + as_kctx, + true); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); + + return i; + } + + /* Context was retained while locks were dropped, + * continue looking for free AS */ + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + } + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return KBASEP_AS_NR_INVALID; +} + +bool kbase_backend_use_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, + int as_nr) +{ + struct kbasep_js_device_data *js_devdata; + struct kbase_as *new_address_space = NULL; + int js; + + js_devdata = &kbdev->js_data; + + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + if (kbdev->hwaccess.active_kctx[js] == kctx) { + WARN(1, "Context is already scheduled in\n"); + return false; + } + } + + new_address_space = &kbdev->as[as_nr]; + + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space); + + if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { + /* We need to retain it to keep the corresponding address space + */ + kbase_ctx_sched_retain_ctx_refcount(kctx); + } + + return true; +} + diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h new file mode 100755 index 000000000000..9cccf224999e --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h @@ -0,0 +1,113 @@ +/* + * + * (C) COPYRIGHT 2014-2016, 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * Register-based HW access backend specific definitions + */ + +#ifndef _KBASE_HWACCESS_GPU_DEFS_H_ +#define _KBASE_HWACCESS_GPU_DEFS_H_ + +/* SLOT_RB_SIZE must be < 256 */ +#define SLOT_RB_SIZE 2 +#define SLOT_RB_MASK (SLOT_RB_SIZE - 1) + +/** + * struct rb_entry - Ringbuffer entry + * @katom: Atom associated with this entry + */ +struct rb_entry { + struct kbase_jd_atom *katom; +}; + +/** + * struct slot_rb - Slot ringbuffer + * @entries: Ringbuffer entries + * @last_context: The last context to submit a job on this slot + * @read_idx: Current read index of buffer + * @write_idx: Current write index of buffer + * @job_chain_flag: Flag used to implement jobchain disambiguation + */ +struct slot_rb { + struct rb_entry entries[SLOT_RB_SIZE]; + + struct kbase_context *last_context; + + u8 read_idx; + u8 write_idx; + + u8 job_chain_flag; +}; + +/** + * struct kbase_backend_data - GPU backend specific data for HW access layer + * @slot_rb: Slot ringbuffers + * @scheduling_timer: The timer tick used for rescheduling jobs + * @timer_running: Is the timer running? The runpool_mutex must be + * held whilst modifying this. + * @suspend_timer: Is the timer suspended? Set when a suspend + * occurs and cleared on resume. The runpool_mutex + * must be held whilst modifying this. + * @reset_gpu: Set to a KBASE_RESET_xxx value (see comments) + * @reset_workq: Work queue for performing the reset + * @reset_work: Work item for performing the reset + * @reset_wait: Wait event signalled when the reset is complete + * @reset_timer: Timeout for soft-stops before the reset + * @timeouts_updated: Have timeout values just been updated? + * + * The hwaccess_lock (a spinlock) must be held when accessing this structure + */ +struct kbase_backend_data { + struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS]; + +#if !MALI_USE_CSF + struct hrtimer scheduling_timer; + + bool timer_running; +#endif + bool suspend_timer; + + atomic_t reset_gpu; + +/* The GPU reset isn't pending */ +#define KBASE_RESET_GPU_NOT_PENDING 0 +/* kbase_prepare_to_reset_gpu has been called */ +#define KBASE_RESET_GPU_PREPARED 1 +/* kbase_reset_gpu has been called - the reset will now definitely happen + * within the timeout period */ +#define KBASE_RESET_GPU_COMMITTED 2 +/* The GPU reset process is currently occuring (timeout has expired or + * kbasep_try_reset_gpu_early was called) */ +#define KBASE_RESET_GPU_HAPPENING 3 +/* Reset the GPU silently, used when resetting the GPU as part of normal + * behavior (e.g. when exiting protected mode). */ +#define KBASE_RESET_GPU_SILENT 4 + struct workqueue_struct *reset_workq; + struct work_struct reset_work; + wait_queue_head_t reset_wait; + struct hrtimer reset_timer; + + bool timeouts_updated; +}; + +#endif /* _KBASE_HWACCESS_GPU_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c new file mode 100755 index 000000000000..19661c9766c6 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c @@ -0,0 +1,1462 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Base kernel job manager APIs + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev); + +static u64 kbase_job_write_affinity(struct kbase_device *kbdev, + base_jd_core_req core_req, + int js) +{ + u64 affinity; + + if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) == + BASE_JD_REQ_T) { + /* Tiler-only atom */ + /* If the hardware supports XAFFINITY then we'll only enable + * the tiler (which is the default so this is a no-op), + * otherwise enable shader core 0. + */ + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) + affinity = 1; + else + affinity = 0; + } else if ((core_req & (BASE_JD_REQ_COHERENT_GROUP | + BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) { + unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; + struct mali_base_gpu_coherent_group_info *coherency_info = + &kbdev->gpu_props.props.coherency_info; + + affinity = kbdev->pm.backend.shaders_avail & + kbdev->pm.debug_core_mask[js]; + + /* JS2 on a dual core group system targets core group 1. All + * other cases target core group 0. + */ + if (js == 2 && num_core_groups > 1) + affinity &= coherency_info->group[1].core_mask; + else + affinity &= coherency_info->group[0].core_mask; + } else { + /* Use all cores */ + affinity = kbdev->pm.backend.shaders_avail & + kbdev->pm.debug_core_mask[js]; + } + + if (unlikely(!affinity)) { +#ifdef CONFIG_MALI_BIFROST_DEBUG + u64 shaders_ready = + kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); + + WARN_ON(!(shaders_ready & kbdev->pm.backend.shaders_avail)); +#endif + + affinity = kbdev->pm.backend.shaders_avail; + } + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO), + affinity & 0xFFFFFFFF); + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI), + affinity >> 32); + + return affinity; +} + +/** + * select_job_chain() - Select which job chain to submit to the GPU + * @katom: Pointer to the atom about to be submitted to the GPU + * + * Selects one of the fragment job chains attached to the special atom at the + * end of a renderpass, or returns the address of the single job chain attached + * to any other type of atom. + * + * Which job chain is selected depends upon whether the tiling phase of the + * renderpass completed normally or was soft-stopped because it used too + * much memory. It also depends upon whether one of the fragment job chains + * has already been run as part of the same renderpass. + * + * Return: GPU virtual address of the selected job chain + */ +static u64 select_job_chain(struct kbase_jd_atom *katom) +{ + struct kbase_context *const kctx = katom->kctx; + u64 jc = katom->jc; + struct kbase_jd_renderpass *rp; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS)) + return jc; + + compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + + rp = &kctx->jctx.renderpasses[katom->renderpass_id]; + /* We can read a subset of renderpass state without holding + * higher-level locks (but not end_katom, for example). + * If the end-of-renderpass atom is running with as-yet indeterminate + * OOM state then assume that the start atom was not soft-stopped. + */ + switch (rp->state) { + case KBASE_JD_RP_OOM: + /* Tiling ran out of memory. + * Start of incremental rendering, used once. + */ + jc = katom->jc_fragment.norm_read_forced_write; + break; + case KBASE_JD_RP_START: + case KBASE_JD_RP_PEND_OOM: + /* Tiling completed successfully first time. + * Single-iteration rendering, used once. + */ + jc = katom->jc_fragment.norm_read_norm_write; + break; + case KBASE_JD_RP_RETRY_OOM: + /* Tiling ran out of memory again. + * Continuation of incremental rendering, used as + * many times as required. + */ + jc = katom->jc_fragment.forced_read_forced_write; + break; + case KBASE_JD_RP_RETRY: + case KBASE_JD_RP_RETRY_PEND_OOM: + /* Tiling completed successfully this time. + * End of incremental rendering, used once. + */ + jc = katom->jc_fragment.forced_read_norm_write; + break; + default: + WARN_ON(1); + break; + } + + dev_dbg(kctx->kbdev->dev, + "Selected job chain 0x%llx for end atom %p in state %d\n", + jc, (void *)katom, (int)rp->state); + + katom->jc = jc; + return jc; +} + +void kbase_job_hw_submit(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + int js) +{ + struct kbase_context *kctx; + u32 cfg; + u64 const jc_head = select_job_chain(katom); + u64 affinity; + + KBASE_DEBUG_ASSERT(kbdev); + KBASE_DEBUG_ASSERT(katom); + + kctx = katom->kctx; + + /* Command register must be available */ + KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx)); + + dev_dbg(kctx->kbdev->dev, "Write JS_HEAD_NEXT 0x%llx for atom %p\n", + jc_head, (void *)katom); + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), + jc_head & 0xFFFFFFFF); + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), + jc_head >> 32); + + affinity = kbase_job_write_affinity(kbdev, katom->core_req, js); + + /* start MMU, medium priority, cache clean/flush on end, clean/flush on + * start */ + cfg = kctx->as_nr; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION) && + !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) + cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; + + if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) + cfg |= JS_CONFIG_START_FLUSH_NO_ACTION; + else + cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; + + if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && + !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) + cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; + else if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CLEAN_ONLY_SAFE)) + cfg |= JS_CONFIG_END_FLUSH_CLEAN; + else + cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; + + cfg |= JS_CONFIG_THREAD_PRI(8); + + if ((katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED) || + (katom->core_req & BASE_JD_REQ_END_RENDERPASS)) + cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; + + if (kbase_hw_has_feature(kbdev, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { + if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) { + cfg |= JS_CONFIG_JOB_CHAIN_FLAG; + katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN; + kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = + true; + } else { + katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN; + kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = + false; + } + } + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_FLUSH_ID_NEXT), + katom->flush_id); + + /* Write an approximate start timestamp. + * It's approximate because there might be a job in the HEAD register. + */ + katom->start_timestamp = ktime_get(); + + /* GO ! */ + dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx", + katom, kctx, js, jc_head); + + KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, + (u32)affinity); + + KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, kctx, + js, kbase_jd_atom_id(kctx, katom), TL_JS_EVENT_START); + + KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(kbdev, katom, jc_head, + affinity, cfg); + KBASE_TLSTREAM_TL_RET_CTX_LPU( + kbdev, + kctx, + &kbdev->gpu_props.props.raw_props.js_features[ + katom->slot_nr]); + KBASE_TLSTREAM_TL_RET_ATOM_AS(kbdev, katom, &kbdev->as[kctx->as_nr]); + KBASE_TLSTREAM_TL_RET_ATOM_LPU( + kbdev, + katom, + &kbdev->gpu_props.props.raw_props.js_features[js], + "ctx_nr,atom_nr"); + kbase_kinstr_jm_atom_hw_submit(katom); +#ifdef CONFIG_GPU_TRACEPOINTS + if (!kbase_backend_nr_atoms_submitted(kbdev, js)) { + /* If this is the only job on the slot, trace it as starting */ + char js_string[16]; + + trace_gpu_sched_switch( + kbasep_make_job_slot_string(js, js_string, + sizeof(js_string)), + ktime_to_ns(katom->start_timestamp), + (u32)katom->kctx->id, 0, katom->work_id); + kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx; + } +#endif + + trace_sysgraph_gpu(SGR_SUBMIT, kctx->id, + kbase_jd_atom_id(kctx, katom), js); + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), + JS_COMMAND_START); +} + +/** + * kbasep_job_slot_update_head_start_timestamp - Update timestamp + * @kbdev: kbase device + * @js: job slot + * @end_timestamp: timestamp + * + * Update the start_timestamp of the job currently in the HEAD, based on the + * fact that we got an IRQ for the previous set of completed jobs. + * + * The estimate also takes into account the time the job was submitted, to + * work out the best estimate (which might still result in an over-estimate to + * the calculated time spent) + */ +static void kbasep_job_slot_update_head_start_timestamp( + struct kbase_device *kbdev, + int js, + ktime_t end_timestamp) +{ + ktime_t timestamp_diff; + struct kbase_jd_atom *katom; + + /* Checking the HEAD position for the job slot */ + katom = kbase_gpu_inspect(kbdev, js, 0); + if (katom != NULL) { + timestamp_diff = ktime_sub(end_timestamp, + katom->start_timestamp); + if (ktime_to_ns(timestamp_diff) >= 0) { + /* Only update the timestamp if it's a better estimate + * than what's currently stored. This is because our + * estimate that accounts for the throttle time may be + * too much of an overestimate */ + katom->start_timestamp = end_timestamp; + } + } +} + +/** + * kbasep_trace_tl_event_lpu_softstop - Call event_lpu_softstop timeline + * tracepoint + * @kbdev: kbase device + * @js: job slot + * + * Make a tracepoint call to the instrumentation module informing that + * softstop happened on given lpu (job slot). + */ +static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, + int js) +{ + KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( + kbdev, + &kbdev->gpu_props.props.raw_props.js_features[js]); +} + +void kbase_job_done(struct kbase_device *kbdev, u32 done) +{ + int i; + u32 count = 0; + ktime_t end_timestamp; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + KBASE_DEBUG_ASSERT(kbdev); + + KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ, NULL, NULL, 0, done); + + end_timestamp = ktime_get(); + + while (done) { + u32 failed = done >> 16; + + /* treat failed slots as finished slots */ + u32 finished = (done & 0xFFFF) | failed; + + /* Note: This is inherently unfair, as we always check + * for lower numbered interrupts before the higher + * numbered ones.*/ + i = ffs(finished) - 1; + KBASE_DEBUG_ASSERT(i >= 0); + + do { + int nr_done; + u32 active; + u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */ + u64 job_tail = 0; + + if (failed & (1u << i)) { + /* read out the job slot status code if the job + * slot reported failure */ + completion_code = kbase_reg_read(kbdev, + JOB_SLOT_REG(i, JS_STATUS)); + + if (completion_code == BASE_JD_EVENT_STOPPED) { + KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT( + kbdev, NULL, + i, 0, TL_JS_EVENT_SOFT_STOP); + + kbasep_trace_tl_event_lpu_softstop( + kbdev, i); + + /* Soft-stopped job - read the value of + * JS_TAIL so that the job chain can + * be resumed */ + job_tail = (u64)kbase_reg_read(kbdev, + JOB_SLOT_REG(i, JS_TAIL_LO)) | + ((u64)kbase_reg_read(kbdev, + JOB_SLOT_REG(i, JS_TAIL_HI)) + << 32); + } else if (completion_code == + BASE_JD_EVENT_NOT_STARTED) { + /* PRLAM-10673 can cause a TERMINATED + * job to come back as NOT_STARTED, but + * the error interrupt helps us detect + * it */ + completion_code = + BASE_JD_EVENT_TERMINATED; + } + + kbase_gpu_irq_evict(kbdev, i, completion_code); + + /* Some jobs that encounter a BUS FAULT may result in corrupted + * state causing future jobs to hang. Reset GPU before + * allowing any other jobs on the slot to continue. */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3076)) { + if (completion_code == BASE_JD_EVENT_JOB_BUS_FAULT) { + if (kbase_prepare_to_reset_gpu_locked(kbdev)) + kbase_reset_gpu_locked(kbdev); + } + } + } + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), + done & ((1 << i) | (1 << (i + 16)))); + active = kbase_reg_read(kbdev, + JOB_CONTROL_REG(JOB_IRQ_JS_STATE)); + + if (((active >> i) & 1) == 0 && + (((done >> (i + 16)) & 1) == 0)) { + /* There is a potential race we must work + * around: + * + * 1. A job slot has a job in both current and + * next registers + * 2. The job in current completes + * successfully, the IRQ handler reads + * RAWSTAT and calls this function with the + * relevant bit set in "done" + * 3. The job in the next registers becomes the + * current job on the GPU + * 4. Sometime before the JOB_IRQ_CLEAR line + * above the job on the GPU _fails_ + * 5. The IRQ_CLEAR clears the done bit but not + * the failed bit. This atomically sets + * JOB_IRQ_JS_STATE. However since both jobs + * have now completed the relevant bits for + * the slot are set to 0. + * + * If we now did nothing then we'd incorrectly + * assume that _both_ jobs had completed + * successfully (since we haven't yet observed + * the fail bit being set in RAWSTAT). + * + * So at this point if there are no active jobs + * left we check to see if RAWSTAT has a failure + * bit set for the job slot. If it does we know + * that there has been a new failure that we + * didn't previously know about, so we make sure + * that we record this in active (but we wait + * for the next loop to deal with it). + * + * If we were handling a job failure (i.e. done + * has the relevant high bit set) then we know + * that the value read back from + * JOB_IRQ_JS_STATE is the correct number of + * remaining jobs because the failed job will + * have prevented any futher jobs from starting + * execution. + */ + u32 rawstat = kbase_reg_read(kbdev, + JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)); + + if ((rawstat >> (i + 16)) & 1) { + /* There is a failed job that we've + * missed - add it back to active */ + active |= (1u << i); + } + } + + dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", + completion_code); + + nr_done = kbase_backend_nr_atoms_submitted(kbdev, i); + nr_done -= (active >> i) & 1; + nr_done -= (active >> (i + 16)) & 1; + + if (nr_done <= 0) { + dev_warn(kbdev->dev, "Spurious interrupt on slot %d", + i); + + goto spurious; + } + + count += nr_done; + + while (nr_done) { + if (nr_done == 1) { + kbase_gpu_complete_hw(kbdev, i, + completion_code, + job_tail, + &end_timestamp); + kbase_jm_try_kick_all(kbdev); + } else { + /* More than one job has completed. + * Since this is not the last job being + * reported this time it must have + * passed. This is because the hardware + * will not allow further jobs in a job + * slot to complete until the failed job + * is cleared from the IRQ status. + */ + kbase_gpu_complete_hw(kbdev, i, + BASE_JD_EVENT_DONE, + 0, + &end_timestamp); + } + nr_done--; + } + spurious: + done = kbase_reg_read(kbdev, + JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)); + + failed = done >> 16; + finished = (done & 0xFFFF) | failed; + if (done) + end_timestamp = ktime_get(); + } while (finished & (1 << i)); + + kbasep_job_slot_update_head_start_timestamp(kbdev, i, + end_timestamp); + } + + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_COMMITTED) { + /* If we're trying to reset the GPU then we might be able to do + * it early (without waiting for a timeout) because some jobs + * have completed + */ + kbasep_try_reset_gpu_early_locked(kbdev); + } + KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ_END, NULL, NULL, 0, count); +} + +void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, + int js, + u32 action, + base_jd_core_req core_reqs, + struct kbase_jd_atom *target_katom) +{ +#if KBASE_KTRACE_ENABLE + u32 status_reg_before; + u64 job_in_head_before; + u32 status_reg_after; + + KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK))); + + /* Check the head pointer */ + job_in_head_before = ((u64) kbase_reg_read(kbdev, + JOB_SLOT_REG(js, JS_HEAD_LO))) + | (((u64) kbase_reg_read(kbdev, + JOB_SLOT_REG(js, JS_HEAD_HI))) + << 32); + status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS)); +#endif + + if (action == JS_COMMAND_SOFT_STOP) { + if (kbase_jd_katom_is_protected(target_katom)) { +#ifdef CONFIG_MALI_BIFROST_DEBUG + dev_dbg(kbdev->dev, + "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%x", + (unsigned int)core_reqs); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + return; + } + + /* We are about to issue a soft stop, so mark the atom as having + * been soft stopped */ + target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED; + + /* Mark the point where we issue the soft-stop command */ + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(kbdev, target_katom); + + if (kbase_hw_has_feature( + kbdev, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { + action = (target_katom->atom_flags & + KBASE_KATOM_FLAGS_JOBCHAIN) ? + JS_COMMAND_SOFT_STOP_1 : + JS_COMMAND_SOFT_STOP_0; + } + } else if (action == JS_COMMAND_HARD_STOP) { + target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED; + + if (kbase_hw_has_feature( + kbdev, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { + action = (target_katom->atom_flags & + KBASE_KATOM_FLAGS_JOBCHAIN) ? + JS_COMMAND_HARD_STOP_1 : + JS_COMMAND_HARD_STOP_0; + } + } + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action); + +#if KBASE_KTRACE_ENABLE + status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS)); + if (status_reg_after == BASE_JD_EVENT_ACTIVE) { + struct kbase_jd_atom *head; + struct kbase_context *head_kctx; + + head = kbase_gpu_inspect(kbdev, js, 0); + head_kctx = head->kctx; + + if (status_reg_before == BASE_JD_EVENT_ACTIVE) + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, head, job_in_head_before, js); + else + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js); + + switch (action) { + case JS_COMMAND_SOFT_STOP: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, head_kctx, head, head->jc, js); + break; + case JS_COMMAND_SOFT_STOP_0: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, head, head->jc, js); + break; + case JS_COMMAND_SOFT_STOP_1: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, head, head->jc, js); + break; + case JS_COMMAND_HARD_STOP: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, head_kctx, head, head->jc, js); + break; + case JS_COMMAND_HARD_STOP_0: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, head, head->jc, js); + break; + case JS_COMMAND_HARD_STOP_1: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, js); + break; + default: + BUG(); + break; + } + } else { + if (status_reg_before == BASE_JD_EVENT_ACTIVE) + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, job_in_head_before, js); + else + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js); + + switch (action) { + case JS_COMMAND_SOFT_STOP: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, js); + break; + case JS_COMMAND_SOFT_STOP_0: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, 0, js); + break; + case JS_COMMAND_SOFT_STOP_1: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, 0, js); + break; + case JS_COMMAND_HARD_STOP: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, js); + break; + case JS_COMMAND_HARD_STOP_0: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, 0, js); + break; + case JS_COMMAND_HARD_STOP_1: + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js); + break; + default: + BUG(); + break; + } + } +#endif +} + +void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + kbase_job_slot_hardstop(kctx, i, NULL); +} + +/** + * kbase_is_existing_atom_submitted_later_than_ready + * @ready: sequence number of the ready atom + * @existing: sequence number of the existing atom + * + * Returns true if the existing atom has been submitted later than the + * ready atom. It is used to understand if an atom that is ready has been + * submitted earlier than the currently running atom, so that the currently + * running atom should be preempted to allow the ready atom to run. + */ +static inline bool kbase_is_existing_atom_submitted_later_than_ready(u64 ready, u64 existing) +{ + /* No seq_nr set? */ + if (!ready || !existing) + return false; + + /* Efficiently handle the unlikely case of wrapping. + * The following code assumes that the delta between the sequence number + * of the two atoms is less than INT64_MAX. + * In the extremely unlikely case where the delta is higher, the comparison + * defaults for no preemption. + * The code also assumes that the conversion from unsigned to signed types + * works because the signed integers are 2's complement. + */ + return (s64)(ready - existing) < 0; +} + +void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, + struct kbase_jd_atom *target_katom) +{ + struct kbase_device *kbdev; + int js = target_katom->slot_nr; + int priority = target_katom->sched_priority; + int seq_nr = target_katom->seq_nr; + int i; + bool stop_sent = false; + + KBASE_DEBUG_ASSERT(kctx != NULL); + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { + struct kbase_jd_atom *katom; + + katom = kbase_gpu_inspect(kbdev, js, i); + if (!katom) + continue; + + if ((kbdev->js_ctx_scheduling_mode == + KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE) && + (katom->kctx != kctx)) + continue; + + if ((katom->sched_priority > priority) || + (katom->kctx == kctx && kbase_is_existing_atom_submitted_later_than_ready(seq_nr, katom->seq_nr))) { + if (!stop_sent) + KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED( + kbdev, + target_katom); + + kbase_job_slot_softstop(kbdev, js, katom); + stop_sent = true; + } + } +} + +static int softstop_start_rp_nolock( + struct kbase_context *kctx, struct kbase_va_region *reg) +{ + struct kbase_device *const kbdev = kctx->kbdev; + struct kbase_jd_atom *katom; + struct kbase_jd_renderpass *rp; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom = kbase_gpu_inspect(kbdev, 1, 0); + + if (!katom) { + dev_dbg(kctx->kbdev->dev, "No atom on job slot\n"); + return -ESRCH; + } + + if (!(katom->core_req & BASE_JD_REQ_START_RENDERPASS)) { + dev_dbg(kctx->kbdev->dev, + "Atom %p on job slot is not start RP\n", (void *)katom); + return -EPERM; + } + + compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + + rp = &kctx->jctx.renderpasses[katom->renderpass_id]; + if (WARN_ON(rp->state != KBASE_JD_RP_START && + rp->state != KBASE_JD_RP_RETRY)) + return -EINVAL; + + dev_dbg(kctx->kbdev->dev, "OOM in state %d with region %p\n", + (int)rp->state, (void *)reg); + + if (WARN_ON(katom != rp->start_katom)) + return -EINVAL; + + dev_dbg(kctx->kbdev->dev, "Adding region %p to list %p\n", + (void *)reg, (void *)&rp->oom_reg_list); + list_move_tail(®->link, &rp->oom_reg_list); + dev_dbg(kctx->kbdev->dev, "Added region to list\n"); + + rp->state = (rp->state == KBASE_JD_RP_START ? + KBASE_JD_RP_PEND_OOM : KBASE_JD_RP_RETRY_PEND_OOM); + + kbase_job_slot_softstop(kbdev, 1, katom); + + return 0; +} + +int kbase_job_slot_softstop_start_rp(struct kbase_context *const kctx, + struct kbase_va_region *const reg) +{ + struct kbase_device *const kbdev = kctx->kbdev; + int err; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + err = softstop_start_rp_nolock(kctx, reg); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return err; +} + +void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + unsigned long timeout = msecs_to_jiffies(ZAP_TIMEOUT); + + timeout = wait_event_timeout(kctx->jctx.zero_jobs_wait, + kctx->jctx.job_nr == 0, timeout); + + if (timeout != 0) + timeout = wait_event_timeout( + kctx->jctx.sched_info.ctx.is_scheduled_wait, + !kbase_ctx_flag(kctx, KCTX_SCHEDULED), + timeout); + + /* Neither wait timed out; all done! */ + if (timeout != 0) + goto exit; + + if (kbase_prepare_to_reset_gpu(kbdev)) { + dev_err(kbdev->dev, + "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", + ZAP_TIMEOUT); + kbase_reset_gpu(kbdev); + } + + /* Wait for the reset to complete */ + kbase_reset_gpu_wait(kbdev); +exit: + dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx); + + /* Ensure that the signallers of the waitqs have finished */ + mutex_lock(&kctx->jctx.lock); + mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&kctx->jctx.lock); +} + +u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) +{ + u32 flush_id = 0; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) { + mutex_lock(&kbdev->pm.lock); + if (kbdev->pm.backend.gpu_powered) + flush_id = kbase_reg_read(kbdev, + GPU_CONTROL_REG(LATEST_FLUSH)); + mutex_unlock(&kbdev->pm.lock); + } + + return flush_id; +} + +int kbase_job_slot_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); + return 0; +} +KBASE_EXPORT_TEST_API(kbase_job_slot_init); + +void kbase_job_slot_halt(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +void kbase_job_slot_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} +KBASE_EXPORT_TEST_API(kbase_job_slot_term); + + +/** + * kbase_job_slot_softstop_swflags - Soft-stop a job with flags + * @kbdev: The kbase device + * @js: The job slot to soft-stop + * @target_katom: The job that should be soft-stopped (or NULL for any job) + * @sw_flags: Flags to pass in about the soft-stop + * + * Context: + * The job slot lock must be held when calling this function. + * The job slot must not already be in the process of being soft-stopped. + * + * Soft-stop the specified job slot, with extra information about the stop + * + * Where possible any job in the next register is evicted before the soft-stop. + */ +void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom, u32 sw_flags) +{ + dev_dbg(kbdev->dev, "Soft-stop atom %p with flags 0x%x (s:%d)\n", + target_katom, sw_flags, js); + + KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK)); + kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom, + JS_COMMAND_SOFT_STOP | sw_flags); +} + +/** + * kbase_job_slot_softstop - Soft-stop the specified job slot + * @kbdev: The kbase device + * @js: The job slot to soft-stop + * @target_katom: The job that should be soft-stopped (or NULL for any job) + * Context: + * The job slot lock must be held when calling this function. + * The job slot must not already be in the process of being soft-stopped. + * + * Where possible any job in the next register is evicted before the soft-stop. + */ +void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom) +{ + kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u); +} + +/** + * kbase_job_slot_hardstop - Hard-stop the specified job slot + * @kctx: The kbase context that contains the job(s) that should + * be hard-stopped + * @js: The job slot to hard-stop + * @target_katom: The job that should be hard-stopped (or NULL for all + * jobs from the context) + * Context: + * The job slot lock must be held when calling this function. + */ +void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom) +{ + struct kbase_device *kbdev = kctx->kbdev; + bool stopped; + + stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js, + target_katom, + JS_COMMAND_HARD_STOP); +} + +/** + * kbase_job_check_enter_disjoint - potentiall enter disjoint mode + * @kbdev: kbase device + * @action: the event which has occurred + * @core_reqs: core requirements of the atom + * @target_katom: the atom which is being affected + * + * For a certain soft-stop action, work out whether to enter disjoint + * state. + * + * This does not register multiple disjoint events if the atom has already + * started a disjoint period + * + * @core_reqs can be supplied as 0 if the atom had not started on the hardware + * (and so a 'real' soft/hard-stop was not required, but it still interrupted + * flow, perhaps on another context) + * + * kbase_job_check_leave_disjoint() should be used to end the disjoint + * state when the soft/hard-stop action is complete + */ +void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, + base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom) +{ + u32 hw_action = action & JS_COMMAND_MASK; + + /* For soft-stop, don't enter if soft-stop not allowed, or isn't + * causing disjoint. + */ + if (hw_action == JS_COMMAND_SOFT_STOP && + (kbase_jd_katom_is_protected(target_katom) || + (0 == (action & JS_COMMAND_SW_CAUSES_DISJOINT)))) + return; + + /* Nothing to do if already logged disjoint state on this atom */ + if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) + return; + + target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT; + kbase_disjoint_state_up(kbdev); +} + +/** + * kbase_job_check_enter_disjoint - potentially leave disjoint state + * @kbdev: kbase device + * @target_katom: atom which is finishing + * + * Work out whether to leave disjoint state when finishing an atom that was + * originated by kbase_job_check_enter_disjoint(). + */ +void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, + struct kbase_jd_atom *target_katom) +{ + if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) { + target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT; + kbase_disjoint_state_down(kbdev); + } +} + +static void kbase_debug_dump_registers(struct kbase_device *kbdev) +{ + int i; + + kbase_io_history_dump(kbdev); + + dev_err(kbdev->dev, "Register state:"); + dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS))); + dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x", + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)), + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE))); + for (i = 0; i < 3; i++) { + dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x", + i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS)), + i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO))); + } + dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", + kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS))); + dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)), + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK)), + kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK))); + dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1))); + dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG))); + dev_err(kbdev->dev, " TILER_CONFIG=0x%08x JM_CONFIG=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG))); +} + +static void kbasep_reset_timeout_worker(struct work_struct *data) +{ + unsigned long flags; + struct kbase_device *kbdev; + ktime_t end_timestamp = ktime_get(); + struct kbasep_js_device_data *js_devdata; + bool silent = false; + u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; + + KBASE_DEBUG_ASSERT(data); + + kbdev = container_of(data, struct kbase_device, + hwaccess.backend.reset_work); + + KBASE_DEBUG_ASSERT(kbdev); + js_devdata = &kbdev->js_data; + + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_SILENT) + silent = true; + + KBASE_KTRACE_ADD_JM(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); + + /* Disable GPU hardware counters. + * This call will block until counters are disabled. + */ + kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + + /* Make sure the timer has completed - this cannot be done from + * interrupt context, so this cannot be done within + * kbasep_try_reset_gpu_early. */ + hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); + + if (kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + /* This would re-activate the GPU. Since it's already idle, + * there's no need to reset it */ + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING); + kbase_disjoint_state_down(kbdev); + wake_up(&kbdev->hwaccess.backend.reset_wait); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return; + } + + KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + spin_lock(&kbdev->mmu_mask_change); + kbase_pm_reset_start_locked(kbdev); + + /* We're about to flush out the IRQs and their bottom half's */ + kbdev->irq_reset_flush = true; + + /* Disable IRQ to avoid IRQ handlers to kick in after releasing the + * spinlock; this also clears any outstanding interrupts */ + kbase_pm_disable_interrupts_nolock(kbdev); + + spin_unlock(&kbdev->mmu_mask_change); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Ensure that any IRQ handlers have finished + * Must be done without any locks IRQ handlers will take */ + kbase_synchronize_irqs(kbdev); + + /* Flush out any in-flight work items */ + kbase_flush_mmu_wqs(kbdev); + + /* The flush has completed so reset the active indicator */ + kbdev->irq_reset_flush = false; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { + /* Ensure that L2 is not transitioning when we send the reset + * command */ + while (--max_loops && kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_L2)) + ; + + WARN(!max_loops, "L2 power transition timed out while trying to reset\n"); + } + + mutex_lock(&kbdev->pm.lock); + /* We hold the pm lock, so there ought to be a current policy */ + KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy); + + /* All slot have been soft-stopped and we've waited + * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we + * assume that anything that is still left on the GPU is stuck there and + * we'll kill it when we reset the GPU */ + + if (!silent) + dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", + RESET_TIMEOUT); + + /* Output the state of some interesting registers to help in the + * debugging of GPU resets */ + if (!silent) + kbase_debug_dump_registers(kbdev); + + /* Complete any jobs that were still on the GPU */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->protected_mode = false; + if (!kbdev->pm.backend.protected_entry_transition_override) + kbase_backend_reset(kbdev, &end_timestamp); + kbase_pm_metrics_update(kbdev, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Reset the GPU */ + kbase_pm_init_hw(kbdev, 0); + + mutex_unlock(&kbdev->pm.lock); + + mutex_lock(&js_devdata->runpool_mutex); + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_restore_all_as(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_pm_enable_interrupts(kbdev); + + kbase_disjoint_state_down(kbdev); + + mutex_unlock(&js_devdata->runpool_mutex); + + mutex_lock(&kbdev->pm.lock); + + kbase_pm_reset_complete(kbdev); + + /* Find out what cores are required now */ + kbase_pm_update_cores_state(kbdev); + + /* Synchronously request and wait for those cores, because if + * instrumentation is enabled it would need them immediately. */ + kbase_pm_wait_for_desired_state(kbdev); + + mutex_unlock(&kbdev->pm.lock); + + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING); + + wake_up(&kbdev->hwaccess.backend.reset_wait); + if (!silent) + dev_err(kbdev->dev, "Reset complete"); + + /* Try submitting some jobs to restart processing */ + KBASE_KTRACE_ADD_JM(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0); + kbase_js_sched_all(kbdev); + + /* Process any pending slot updates */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_backend_slot_update(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_pm_context_idle(kbdev); + + /* Re-enable GPU hardware counters */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + KBASE_KTRACE_ADD_JM(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); +} + +static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) +{ + struct kbase_device *kbdev = container_of(timer, struct kbase_device, + hwaccess.backend.reset_timer); + + KBASE_DEBUG_ASSERT(kbdev); + + /* Reset still pending? */ + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) == + KBASE_RESET_GPU_COMMITTED) + queue_work(kbdev->hwaccess.backend.reset_workq, + &kbdev->hwaccess.backend.reset_work); + + return HRTIMER_NORESTART; +} + +/* + * If all jobs are evicted from the GPU then we can reset the GPU + * immediately instead of waiting for the timeout to elapse + */ + +static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) +{ + int i; + int pending_jobs = 0; + + KBASE_DEBUG_ASSERT(kbdev); + + /* Count the number of jobs */ + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i); + + if (pending_jobs > 0) { + /* There are still jobs on the GPU - wait */ + return; + } + + /* To prevent getting incorrect registers when dumping failed job, + * skip early reset. + */ + if (atomic_read(&kbdev->job_fault_debug) > 0) + return; + + /* Check that the reset has been committed to (i.e. kbase_reset_gpu has + * been called), and that no other thread beat this thread to starting + * the reset */ + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != + KBASE_RESET_GPU_COMMITTED) { + /* Reset has already occurred */ + return; + } + + queue_work(kbdev->hwaccess.backend.reset_workq, + &kbdev->hwaccess.backend.reset_work); +} + +static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbasep_try_reset_gpu_early_locked(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +/** + * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU + * @kbdev: kbase device + * + * This function just soft-stops all the slots to ensure that as many jobs as + * possible are saved. + * + * Return: + * The function returns a boolean which should be interpreted as follows: + * true - Prepared for reset, kbase_reset_gpu_locked should be called. + * false - Another thread is performing a reset, kbase_reset_gpu should + * not be called. + */ +bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) +{ + int i; + + KBASE_DEBUG_ASSERT(kbdev); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_pm_is_gpu_lost(kbdev)) { + /* GPU access has been removed, reset will be done by + * Arbiter instead + */ + return false; + } +#endif + + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING, + KBASE_RESET_GPU_PREPARED) != + KBASE_RESET_GPU_NOT_PENDING) { + /* Some other thread is already resetting the GPU */ + return false; + } + + kbase_disjoint_state_up(kbdev); + + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + kbase_job_slot_softstop(kbdev, i, NULL); + + return true; +} + +bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) +{ + unsigned long flags; + bool ret; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + ret = kbase_prepare_to_reset_gpu_locked(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return ret; +} +KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); + +/* + * This function should be called after kbase_prepare_to_reset_gpu if it + * returns true. It should never be called without a corresponding call to + * kbase_prepare_to_reset_gpu. + * + * After this function is called (or not called if kbase_prepare_to_reset_gpu + * returned false), the caller should wait for + * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset + * has completed. + */ +void kbase_reset_gpu(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + + /* Note this is an assert/atomic_set because it is a software issue for + * a race to be occuring here */ + KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_PREPARED); + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED); + + dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", + kbdev->reset_timeout_ms); + + hrtimer_start(&kbdev->hwaccess.backend.reset_timer, + HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), + HRTIMER_MODE_REL); + + /* Try resetting early */ + kbasep_try_reset_gpu_early(kbdev); +} +KBASE_EXPORT_TEST_API(kbase_reset_gpu); + +void kbase_reset_gpu_locked(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + + /* Note this is an assert/atomic_set because it is a software issue for + * a race to be occuring here */ + KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_PREPARED); + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED); + + dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", + kbdev->reset_timeout_ms); + hrtimer_start(&kbdev->hwaccess.backend.reset_timer, + HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), + HRTIMER_MODE_REL); + + /* Try resetting early */ + kbasep_try_reset_gpu_early_locked(kbdev); +} + +int kbase_reset_gpu_silent(struct kbase_device *kbdev) +{ + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING, + KBASE_RESET_GPU_SILENT) != + KBASE_RESET_GPU_NOT_PENDING) { + /* Some other thread is already resetting the GPU */ + return -EAGAIN; + } + + kbase_disjoint_state_up(kbdev); + + queue_work(kbdev->hwaccess.backend.reset_workq, + &kbdev->hwaccess.backend.reset_work); + + return 0; +} + +bool kbase_reset_gpu_is_active(struct kbase_device *kbdev) +{ + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_NOT_PENDING) + return false; + + return true; +} + +int kbase_reset_gpu_wait(struct kbase_device *kbdev) +{ + wait_event(kbdev->hwaccess.backend.reset_wait, + atomic_read(&kbdev->hwaccess.backend.reset_gpu) + == KBASE_RESET_GPU_NOT_PENDING); + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_reset_gpu_wait); + +int kbase_reset_gpu_init(struct kbase_device *kbdev) +{ + kbdev->hwaccess.backend.reset_workq = alloc_workqueue( + "Mali reset workqueue", 0, 1); + if (kbdev->hwaccess.backend.reset_workq == NULL) + return -ENOMEM; + + INIT_WORK(&kbdev->hwaccess.backend.reset_work, + kbasep_reset_timeout_worker); + + hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + kbdev->hwaccess.backend.reset_timer.function = + kbasep_reset_timer_callback; + + return 0; +} + +void kbase_reset_gpu_term(struct kbase_device *kbdev) +{ + destroy_workqueue(kbdev->hwaccess.backend.reset_workq); +} diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_internal.h new file mode 100755 index 000000000000..cd1f9794fdc4 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_internal.h @@ -0,0 +1,181 @@ +/* + * + * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Job Manager backend-specific low-level APIs. + */ + +#ifndef _KBASE_JM_HWACCESS_H_ +#define _KBASE_JM_HWACCESS_H_ + +#include +#include +#include + +#include +#include + +/** + * kbase_job_submit_nolock() - Submit a job to a certain job-slot + * @kbdev: Device pointer + * @katom: Atom to submit + * @js: Job slot to submit on + * + * The caller must check kbasep_jm_is_submit_slots_free() != false before + * calling this. + * + * The following locking conditions are made on the caller: + * - it must hold the hwaccess_lock + */ +void kbase_job_submit_nolock(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, int js); + +/** + * kbase_job_done_slot() - Complete the head job on a particular job-slot + * @kbdev: Device pointer + * @s: Job slot + * @completion_code: Completion code of job reported by GPU + * @job_tail: Job tail address reported by GPU + * @end_timestamp: Timestamp of job completion + */ +void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, + u64 job_tail, ktime_t *end_timestamp); + +#ifdef CONFIG_GPU_TRACEPOINTS +static inline char *kbasep_make_job_slot_string(int js, char *js_string, + size_t js_size) +{ + snprintf(js_string, js_size, "job_slot_%i", js); + return js_string; +} +#endif + +#if !MALI_USE_CSF +static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, + struct kbase_context *kctx) +{ + return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT)); +} +#endif + + +/** + * kbase_job_hw_submit() - Submit a job to the GPU + * @kbdev: Device pointer + * @katom: Atom to submit + * @js: Job slot to submit on + * + * The caller must check kbasep_jm_is_submit_slots_free() != false before + * calling this. + * + * The following locking conditions are made on the caller: + * - it must hold the hwaccess_lock + */ +void kbase_job_hw_submit(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + int js); + +#if !MALI_USE_CSF +/** + * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop + * on the specified atom + * @kbdev: Device pointer + * @js: Job slot to stop on + * @action: The action to perform, either JSn_COMMAND_HARD_STOP or + * JSn_COMMAND_SOFT_STOP + * @core_reqs: Core requirements of atom to stop + * @target_katom: Atom to stop + * + * The following locking conditions are made on the caller: + * - it must hold the hwaccess_lock + */ +void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, + int js, + u32 action, + base_jd_core_req core_reqs, + struct kbase_jd_atom *target_katom); +#endif /* !MALI_USE_CSF */ + +/** + * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job + * slot belonging to a given context. + * @kbdev: Device pointer + * @kctx: Context pointer. May be NULL + * @katom: Specific atom to stop. May be NULL + * @js: Job slot to hard stop + * @action: The action to perform, either JSn_COMMAND_HARD_STOP or + * JSn_COMMAND_SOFT_STOP + * + * If no context is provided then all jobs on the slot will be soft or hard + * stopped. + * + * If a katom is provided then only that specific atom will be stopped. In this + * case the kctx parameter is ignored. + * + * Jobs that are on the slot but are not yet on the GPU will be unpulled and + * returned to the job scheduler. + * + * Return: true if an atom was stopped, false otherwise + */ +bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js, + struct kbase_jd_atom *katom, + u32 action); + +/** + * kbase_job_slot_init - Initialise job slot framework + * @kbdev: Device pointer + * + * Called on driver initialisation + * + * Return: 0 on success + */ +int kbase_job_slot_init(struct kbase_device *kbdev); + +/** + * kbase_job_slot_halt - Halt the job slot framework + * @kbdev: Device pointer + * + * Should prevent any further job slot processing + */ +void kbase_job_slot_halt(struct kbase_device *kbdev); + +/** + * kbase_job_slot_term - Terminate job slot framework + * @kbdev: Device pointer + * + * Called on driver termination + */ +void kbase_job_slot_term(struct kbase_device *kbdev); + +/** + * kbase_gpu_cache_clean - Cause a GPU cache clean & flush + * @kbdev: Device pointer + * + * Caller must not be in IRQ context + */ +void kbase_gpu_cache_clean(struct kbase_device *kbdev); + +#endif /* _KBASE_JM_HWACCESS_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c new file mode 100755 index 000000000000..afaaef27883d --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c @@ -0,0 +1,1659 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * Register-based HW access backend specific APIs + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Return whether the specified ringbuffer is empty. HW access lock must be + * held */ +#define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) +/* Return number of atoms currently in the specified ringbuffer. HW access lock + * must be held */ +#define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) + +static void kbase_gpu_release_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + ktime_t *end_timestamp); + +/** + * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer + * @kbdev: Device pointer + * @katom: Atom to enqueue + * + * Context: Caller must hold the HW access lock + */ +static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; + + WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; + rb->write_idx++; + + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; +} + +/** + * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once + * it has been completed + * @kbdev: Device pointer + * @js: Job slot to remove atom from + * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in + * which case current time will be used. + * + * Context: Caller must hold the HW access lock + * + * Return: Atom removed from ringbuffer + */ +static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, + int js, + ktime_t *end_timestamp) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; + struct kbase_jd_atom *katom; + + if (SLOT_RB_EMPTY(rb)) { + WARN(1, "GPU ringbuffer unexpectedly empty\n"); + return NULL; + } + + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; + + kbase_gpu_release_atom(kbdev, katom, end_timestamp); + + rb->read_idx++; + + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; + + return katom; +} + +struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, + int idx) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if ((SLOT_RB_ENTRIES(rb) - 1) < idx) + return NULL; /* idx out of range */ + + return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; +} + +struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, + int js) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; + + if (SLOT_RB_EMPTY(rb)) + return NULL; + + return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; +} + +bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) +{ + int js; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) + return true; + } + } + return false; +} + +int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js) +{ + int nr = 0; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (katom && (katom->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED)) + nr++; + } + + return nr; +} + +int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js) +{ + int nr = 0; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + if (kbase_gpu_inspect(kbdev, js, i)) + nr++; + } + + return nr; +} + +static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js, + enum kbase_atom_gpu_rb_state min_rb_state) +{ + int nr = 0; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (katom && (katom->gpu_rb_state >= min_rb_state)) + nr++; + } + + return nr; +} + +/** + * check_secure_atom - Check if the given atom is in the given secure state and + * has a ringbuffer state of at least + * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION + * @katom: Atom pointer + * @secure: Desired secure state + * + * Return: true if atom is in the given state, false otherwise + */ +static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) +{ + if (katom->gpu_rb_state >= + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && + ((kbase_jd_katom_is_protected(katom) && secure) || + (!kbase_jd_katom_is_protected(katom) && !secure))) + return true; + + return false; +} + +/** + * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given + * secure state in the ringbuffers of at least + * state + * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE + * @kbdev: Device pointer + * @secure: Desired secure state + * + * Return: true if any atoms are in the given state, false otherwise + */ +static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, + bool secure) +{ + int js, i; + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, + js, i); + + if (katom) { + if (check_secure_atom(katom, secure)) + return true; + } + } + } + + return false; +} + +int kbase_backend_slot_free(struct kbase_device *kbdev, int js) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != + KBASE_RESET_GPU_NOT_PENDING) { + /* The GPU is being reset - so prevent submission */ + return 0; + } + + return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); +} + + +static void kbase_gpu_release_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + ktime_t *end_timestamp) +{ + struct kbase_context *kctx = katom->kctx; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + switch (katom->gpu_rb_state) { + case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: + /* Should be impossible */ + WARN(1, "Attempting to release atom not in ringbuffer\n"); + break; + + case KBASE_ATOM_GPU_RB_SUBMITTED: + kbase_kinstr_jm_atom_hw_release(katom); + /* Inform power management at start/finish of atom so it can + * update its GPU utilisation metrics. Mark atom as not + * submitted beforehand. */ + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; + kbase_pm_metrics_update(kbdev, end_timestamp); + + if (katom->core_req & BASE_JD_REQ_PERMON) + kbase_pm_release_gpu_cycle_counter_nolock(kbdev); + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, katom, + &kbdev->gpu_props.props.raw_props.js_features + [katom->slot_nr]); + KBASE_TLSTREAM_TL_NRET_ATOM_AS(kbdev, katom, &kbdev->as[kctx->as_nr]); + KBASE_TLSTREAM_TL_NRET_CTX_LPU(kbdev, kctx, + &kbdev->gpu_props.props.raw_props.js_features + [katom->slot_nr]); + + case KBASE_ATOM_GPU_RB_READY: + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: + break; + + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: + if (kbase_jd_katom_is_protected(katom) && + (katom->protected_state.enter != + KBASE_ATOM_ENTER_PROTECTED_CHECK) && + (katom->protected_state.enter != + KBASE_ATOM_ENTER_PROTECTED_HWCNT)) { + kbase_pm_protected_override_disable(kbdev); + kbase_pm_update_cores_state_nolock(kbdev); + } + if (kbase_jd_katom_is_protected(katom) && + (katom->protected_state.enter == + KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) + kbase_pm_protected_entry_override_disable(kbdev); + if (!kbase_jd_katom_is_protected(katom) && + (katom->protected_state.exit != + KBASE_ATOM_EXIT_PROTECTED_CHECK) && + (katom->protected_state.exit != + KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT)) { + kbase_pm_protected_override_disable(kbdev); + kbase_pm_update_cores_state_nolock(kbdev); + } + + if (katom->protected_state.enter != + KBASE_ATOM_ENTER_PROTECTED_CHECK || + katom->protected_state.exit != + KBASE_ATOM_EXIT_PROTECTED_CHECK) + kbdev->protected_mode_transition = false; + /* If the atom has suspended hwcnt but has not yet entered + * protected mode, then resume hwcnt now. If the GPU is now in + * protected mode then hwcnt will be resumed by GPU reset so + * don't resume it here. + */ + if (kbase_jd_katom_is_protected(katom) && + ((katom->protected_state.enter == + KBASE_ATOM_ENTER_PROTECTED_IDLE_L2) || + (katom->protected_state.enter == + KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY))) { + WARN_ON(!kbdev->protected_mode_hwcnt_disabled); + kbdev->protected_mode_hwcnt_desired = true; + if (kbdev->protected_mode_hwcnt_disabled) { + kbase_hwcnt_context_enable( + kbdev->hwcnt_gpu_ctx); + kbdev->protected_mode_hwcnt_disabled = false; + } + } + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { + if (katom->atom_flags & + KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) { + kbase_pm_protected_l2_override(kbdev, false); + katom->atom_flags &= + ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; + } + } + + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + case KBASE_ATOM_GPU_RB_RETURN_TO_JS: + break; + } + + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; + katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; +} + +static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbase_gpu_release_atom(kbdev, katom, NULL); + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; +} + +/** + * other_slots_busy - Determine if any job slots other than @js are currently + * running atoms + * @kbdev: Device pointer + * @js: Job slot + * + * Return: true if any slots other than @js are busy, false otherwise + */ +static inline bool other_slots_busy(struct kbase_device *kbdev, int js) +{ + int slot; + + for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) { + if (slot == js) + continue; + + if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot, + KBASE_ATOM_GPU_RB_SUBMITTED)) + return true; + } + + return false; +} + +static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) +{ + return kbdev->protected_mode; +} + +static void kbase_gpu_disable_coherent(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* + * When entering into protected mode, we must ensure that the + * GPU is not operating in coherent mode as well. This is to + * ensure that no protected memory can be leaked. + */ + if (kbdev->system_coherency == COHERENCY_ACE) + kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); +} + +static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) +{ + int err = -EINVAL; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ONCE(!kbdev->protected_ops, + "Cannot enter protected mode: protected callbacks not specified.\n"); + + if (kbdev->protected_ops) { + /* Switch GPU to protected mode */ + err = kbdev->protected_ops->protected_mode_enable( + kbdev->protected_dev); + + if (err) { + dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", + err); + } else { + kbdev->protected_mode = true; + kbase_ipa_protection_mode_switch_event(kbdev); + } + } + + return err; +} + +static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ONCE(!kbdev->protected_ops, + "Cannot exit protected mode: protected callbacks not specified.\n"); + + if (!kbdev->protected_ops) + return -EINVAL; + + /* The protected mode disable callback will be called as part of reset + */ + return kbase_reset_gpu_silent(kbdev); +} + +static int kbase_jm_protected_entry(struct kbase_device *kbdev, + struct kbase_jd_atom **katom, int idx, int js) +{ + int err = 0; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + err = kbase_gpu_protected_mode_enter(kbdev); + + /* + * Regardless of result before this call, we are no longer + * transitioning the GPU. + */ + + kbdev->protected_mode_transition = false; + kbase_pm_protected_override_disable(kbdev); + kbase_pm_update_cores_state_nolock(kbdev); + + KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev, kbdev); + if (err) { + /* + * Failed to switch into protected mode, resume + * GPU hwcnt and fail atom. + */ + WARN_ON(!kbdev->protected_mode_hwcnt_disabled); + kbdev->protected_mode_hwcnt_desired = true; + if (kbdev->protected_mode_hwcnt_disabled) { + kbase_hwcnt_context_enable( + kbdev->hwcnt_gpu_ctx); + kbdev->protected_mode_hwcnt_disabled = false; + } + + katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; + kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); + /* + * Only return if head atom or previous atom + * already removed - as atoms must be returned + * in order. + */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, katom[idx]); + } + + return -EINVAL; + } + + /* + * Protected mode sanity checks. + */ + KBASE_DEBUG_ASSERT_MSG( + kbase_jd_katom_is_protected(katom[idx]) == + kbase_gpu_in_protected_mode(kbdev), + "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", + kbase_jd_katom_is_protected(katom[idx]), + kbase_gpu_in_protected_mode(kbdev)); + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_READY; + + return err; +} + +static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, + struct kbase_jd_atom **katom, int idx, int js) +{ + int err = 0; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + switch (katom[idx]->protected_state.enter) { + case KBASE_ATOM_ENTER_PROTECTED_CHECK: + KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev, kbdev); + /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV + * should ensure that we are not already transitiong, and that + * there are no atoms currently on the GPU. */ + WARN_ON(kbdev->protected_mode_transition); + WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); + /* If hwcnt is disabled, it means we didn't clean up correctly + * during last exit from protected mode. + */ + WARN_ON(kbdev->protected_mode_hwcnt_disabled); + + katom[idx]->protected_state.enter = + KBASE_ATOM_ENTER_PROTECTED_HWCNT; + + kbdev->protected_mode_transition = true; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_ENTER_PROTECTED_HWCNT: + /* See if we can get away with disabling hwcnt atomically */ + kbdev->protected_mode_hwcnt_desired = false; + if (!kbdev->protected_mode_hwcnt_disabled) { + if (kbase_hwcnt_context_disable_atomic( + kbdev->hwcnt_gpu_ctx)) + kbdev->protected_mode_hwcnt_disabled = true; + } + + /* We couldn't disable atomically, so kick off a worker */ + if (!kbdev->protected_mode_hwcnt_disabled) { +#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE + queue_work(system_wq, + &kbdev->protected_mode_hwcnt_disable_work); +#else + queue_work(system_highpri_wq, + &kbdev->protected_mode_hwcnt_disable_work); +#endif + return -EAGAIN; + } + + /* Once reaching this point GPU must be + * switched to protected mode or hwcnt + * re-enabled. */ + + if (kbase_pm_protected_entry_override_enable(kbdev)) + return -EAGAIN; + + /* + * Not in correct mode, begin protected mode switch. + * Entering protected mode requires us to power down the L2, + * and drop out of fully coherent mode. + */ + katom[idx]->protected_state.enter = + KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; + + kbase_pm_protected_override_enable(kbdev); + /* + * Only if the GPU reset hasn't been initiated, there is a need + * to invoke the state machine to explicitly power down the + * shader cores and L2. + */ + if (!kbdev->pm.backend.protected_entry_transition_override) + kbase_pm_update_cores_state_nolock(kbdev); + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: + /* Avoid unnecessary waiting on non-ACE platforms. */ + if (kbdev->system_coherency == COHERENCY_ACE) { + if (kbdev->pm.backend.l2_always_on) { + /* + * If the GPU reset hasn't completed, then L2 + * could still be powered up. + */ + if (kbase_reset_gpu_is_active(kbdev)) + return -EAGAIN; + } + + if (kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_L2) || + kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_L2) || + kbase_is_gpu_removed(kbdev)) { + /* + * The L2 is still powered, wait for all + * the users to finish with it before doing + * the actual reset. + */ + return -EAGAIN; + } + } + + katom[idx]->protected_state.enter = + KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: + /* + * When entering into protected mode, we must ensure that the + * GPU is not operating in coherent mode as well. This is to + * ensure that no protected memory can be leaked. + */ + kbase_gpu_disable_coherent(kbdev); + + kbase_pm_protected_entry_override_disable(kbdev); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { + /* + * Power on L2 caches; this will also result in the + * correct value written to coherency enable register. + */ + kbase_pm_protected_l2_override(kbdev, true); + + /* + * Set the flag on the atom that additional + * L2 references are taken. + */ + katom[idx]->atom_flags |= + KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; + } + + katom[idx]->protected_state.enter = + KBASE_ATOM_ENTER_PROTECTED_FINISHED; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) + return -EAGAIN; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_ENTER_PROTECTED_FINISHED: + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { + /* + * Check that L2 caches are powered and, if so, + * enter protected mode. + */ + if (kbdev->pm.backend.l2_state == KBASE_L2_ON) { + /* + * Remove additional L2 reference and reset + * the atom flag which denotes it. + */ + if (katom[idx]->atom_flags & + KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) { + kbase_pm_protected_l2_override(kbdev, + false); + katom[idx]->atom_flags &= + ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; + } + + err = kbase_jm_protected_entry(kbdev, katom, idx, js); + + if (err) + return err; + } else { + /* + * still waiting for L2 caches to power up + */ + return -EAGAIN; + } + } else { + err = kbase_jm_protected_entry(kbdev, katom, idx, js); + + if (err) + return err; + } + } + + return 0; +} + +static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, + struct kbase_jd_atom **katom, int idx, int js) +{ + int err = 0; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + switch (katom[idx]->protected_state.exit) { + case KBASE_ATOM_EXIT_PROTECTED_CHECK: + KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev, kbdev); + /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV + * should ensure that we are not already transitiong, and that + * there are no atoms currently on the GPU. */ + WARN_ON(kbdev->protected_mode_transition); + WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); + + /* + * Exiting protected mode requires a reset, but first the L2 + * needs to be powered down to ensure it's not active when the + * reset is issued. + */ + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; + + kbdev->protected_mode_transition = true; + kbase_pm_protected_override_enable(kbdev); + kbase_pm_update_cores_state_nolock(kbdev); + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: + if (kbdev->pm.backend.l2_state != KBASE_L2_OFF) { + /* + * The L2 is still powered, wait for all the users to + * finish with it before doing the actual reset. + */ + return -EAGAIN; + } + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_RESET; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_EXIT_PROTECTED_RESET: + /* Issue the reset to the GPU */ + err = kbase_gpu_protected_mode_reset(kbdev); + + if (err == -EAGAIN) + return -EAGAIN; + + if (err) { + kbdev->protected_mode_transition = false; + kbase_pm_protected_override_disable(kbdev); + + /* Failed to exit protected mode, fail atom */ + katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; + kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); + /* Only return if head atom or previous atom + * already removed - as atoms must be returned + * in order */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, katom[idx]); + } + + /* If we're exiting from protected mode, hwcnt must have + * been disabled during entry. + */ + WARN_ON(!kbdev->protected_mode_hwcnt_disabled); + kbdev->protected_mode_hwcnt_desired = true; + if (kbdev->protected_mode_hwcnt_disabled) { + kbase_hwcnt_context_enable( + kbdev->hwcnt_gpu_ctx); + kbdev->protected_mode_hwcnt_disabled = false; + } + + return -EINVAL; + } + + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: + /* A GPU reset is issued when exiting protected mode. Once the + * reset is done all atoms' state will also be reset. For this + * reason, if the atom is still in this state we can safely + * say that the reset has not completed i.e., we have not + * finished exiting protected mode yet. + */ + return -EAGAIN; + } + + return 0; +} + +void kbase_backend_slot_update(struct kbase_device *kbdev) +{ + int js; + + lockdep_assert_held(&kbdev->hwaccess_lock); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_reset_gpu_is_active(kbdev) || + kbase_is_gpu_removed(kbdev)) +#else + if (kbase_reset_gpu_is_active(kbdev)) +#endif + return; + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + struct kbase_jd_atom *katom[2]; + int idx; + + katom[0] = kbase_gpu_inspect(kbdev, js, 0); + katom[1] = kbase_gpu_inspect(kbdev, js, 1); + WARN_ON(katom[1] && !katom[0]); + + for (idx = 0; idx < SLOT_RB_SIZE; idx++) { + bool cores_ready; + int ret; + + if (!katom[idx]) + continue; + + switch (katom[idx]->gpu_rb_state) { + case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: + /* Should be impossible */ + WARN(1, "Attempting to update atom not in ringbuffer\n"); + break; + + case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: + if (kbase_js_atom_blocked_on_x_dep(katom[idx])) + break; + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: + if (kbase_gpu_check_secure_atoms(kbdev, + !kbase_jd_katom_is_protected( + katom[idx]))) + break; + + if ((idx == 1) && (kbase_jd_katom_is_protected( + katom[0]) != + kbase_jd_katom_is_protected( + katom[1]))) + break; + + if (kbdev->protected_mode_transition) + break; + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: + + /* + * Exiting protected mode must be done before + * the references on the cores are taken as + * a power down the L2 is required which + * can't happen after the references for this + * atom are taken. + */ + + if (!kbase_gpu_in_protected_mode(kbdev) && + kbase_jd_katom_is_protected(katom[idx])) { + /* Atom needs to transition into protected mode. */ + ret = kbase_jm_enter_protected_mode(kbdev, + katom, idx, js); + if (ret) + break; + } else if (kbase_gpu_in_protected_mode(kbdev) && + !kbase_jd_katom_is_protected(katom[idx])) { + /* Atom needs to transition out of protected mode. */ + ret = kbase_jm_exit_protected_mode(kbdev, + katom, idx, js); + if (ret) + break; + } + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_CHECK; + + /* Atom needs no protected mode transition. */ + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: + if (katom[idx]->will_fail_event_code) { + kbase_gpu_mark_atom_for_return(kbdev, + katom[idx]); + /* Set EVENT_DONE so this atom will be + completed, not unpulled. */ + katom[idx]->event_code = + BASE_JD_EVENT_DONE; + /* Only return if head atom or previous + * atom already removed - as atoms must + * be returned in order. */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, katom[idx]); + } + break; + } + + cores_ready = kbase_pm_cores_requested(kbdev, + true); + + if (katom[idx]->event_code == + BASE_JD_EVENT_PM_EVENT) { + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_RETURN_TO_JS; + break; + } + + if (!cores_ready) + break; + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_READY; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_READY: + + if (idx == 1) { + /* Only submit if head atom or previous + * atom already submitted */ + if ((katom[0]->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED && + katom[0]->gpu_rb_state != + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) + break; + + /* If intra-slot serialization in use + * then don't submit atom to NEXT slot + */ + if (kbdev->serialize_jobs & + KBASE_SERIALIZE_INTRA_SLOT) + break; + } + + /* If inter-slot serialization in use then don't + * submit atom if any other slots are in use */ + if ((kbdev->serialize_jobs & + KBASE_SERIALIZE_INTER_SLOT) && + other_slots_busy(kbdev, js)) + break; + +#ifdef CONFIG_MALI_GEM5_BUILD + if (!kbasep_jm_is_js_free(kbdev, js, + katom[idx]->kctx)) + break; +#endif + /* Check if this job needs the cycle counter + * enabled before submission */ + if (katom[idx]->core_req & BASE_JD_REQ_PERMON) + kbase_pm_request_gpu_cycle_counter_l2_is_on( + kbdev); + + kbase_job_hw_submit(kbdev, katom[idx], js); + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_SUBMITTED; + + /* Inform power management at start/finish of + * atom so it can update its GPU utilisation + * metrics. */ + kbase_pm_metrics_update(kbdev, + &katom[idx]->start_timestamp); + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_SUBMITTED: + /* Atom submitted to HW, nothing else to do */ + break; + + case KBASE_ATOM_GPU_RB_RETURN_TO_JS: + /* Only return if head atom or previous atom + * already removed - as atoms must be returned + * in order */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, + katom[idx]); + } + break; + } + } + } +} + + +void kbase_backend_run_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + dev_dbg(kbdev->dev, "Backend running atom %p\n", (void *)katom); + + kbase_gpu_enqueue_atom(kbdev, katom); + kbase_backend_slot_update(kbdev); +} + +#define HAS_DEP(katom) (katom->pre_dep || katom->atom_flags & \ + (KBASE_KATOM_FLAG_X_DEP_BLOCKED | KBASE_KATOM_FLAG_FAIL_BLOCKER)) + +bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js, + u32 completion_code) +{ + struct kbase_jd_atom *katom; + struct kbase_jd_atom *next_katom; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom = kbase_gpu_inspect(kbdev, js, 0); + next_katom = kbase_gpu_inspect(kbdev, js, 1); + + if (next_katom && katom->kctx == next_katom->kctx && + next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && + (HAS_DEP(next_katom) || next_katom->sched_priority == + katom->sched_priority) && + (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO)) + != 0 || + kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI)) + != 0)) { + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), + JS_COMMAND_NOP); + next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; + + if (completion_code == BASE_JD_EVENT_STOPPED) { + KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, next_katom, + &kbdev->gpu_props.props.raw_props.js_features + [next_katom->slot_nr]); + KBASE_TLSTREAM_TL_NRET_ATOM_AS(kbdev, next_katom, &kbdev->as + [next_katom->kctx->as_nr]); + KBASE_TLSTREAM_TL_NRET_CTX_LPU(kbdev, next_katom->kctx, + &kbdev->gpu_props.props.raw_props.js_features + [next_katom->slot_nr]); + } + + if (next_katom->core_req & BASE_JD_REQ_PERMON) + kbase_pm_release_gpu_cycle_counter_nolock(kbdev); + + return true; + } + + return false; +} + +void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, + u32 completion_code, + u64 job_tail, + ktime_t *end_timestamp) +{ + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); + struct kbase_context *kctx = katom->kctx; + + dev_dbg(kbdev->dev, + "Atom %p completed on hw with code 0x%x and job_tail 0x%llx (s:%d)\n", + (void *)katom, completion_code, job_tail, js); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* + * When a hard-stop is followed close after a soft-stop, the completion + * code may be set to STOPPED, even though the job is terminated + */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) { + if (completion_code == BASE_JD_EVENT_STOPPED && + (katom->atom_flags & + KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { + completion_code = BASE_JD_EVENT_TERMINATED; + } + } + + if ((katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && + completion_code != BASE_JD_EVENT_DONE && + !(completion_code & BASE_JD_SW_EVENT)) { + /* When a job chain fails, on a T60x or when + * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not + * flushed. To prevent future evictions causing possible memory + * corruption we need to flush the cache manually before any + * affected memory gets reused. */ + katom->need_cache_flush_cores_retained = true; + } + + katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); + + if (completion_code == BASE_JD_EVENT_STOPPED) { + struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, + 0); + + /* + * Dequeue next atom from ringbuffers on same slot if required. + * This atom will already have been removed from the NEXT + * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that + * the atoms on this slot are returned in the correct order. + */ + if (next_katom && katom->kctx == next_katom->kctx && + next_katom->sched_priority == + katom->sched_priority) { + WARN_ON(next_katom->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED); + kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); + kbase_jm_return_atom_to_js(kbdev, next_katom); + } + } else if (completion_code != BASE_JD_EVENT_DONE) { + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + int i; + + if (!kbase_ctx_flag(katom->kctx, KCTX_DYING)) + dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", + js, completion_code, + kbase_gpu_exception_name( + completion_code)); + +#if KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR != 0 + KBASE_KTRACE_DUMP(kbdev); +#endif + kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); + + /* + * Remove all atoms on the same context from ringbuffers. This + * will not remove atoms that are already on the GPU, as these + * are guaranteed not to have fail dependencies on the failed + * atom. + */ + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { + struct kbase_jd_atom *katom_idx0 = + kbase_gpu_inspect(kbdev, i, 0); + struct kbase_jd_atom *katom_idx1 = + kbase_gpu_inspect(kbdev, i, 1); + + if (katom_idx0 && katom_idx0->kctx == katom->kctx && + HAS_DEP(katom_idx0) && + katom_idx0->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Dequeue katom_idx0 from ringbuffer */ + kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); + + if (katom_idx1 && + katom_idx1->kctx == katom->kctx + && HAS_DEP(katom_idx1) && + katom_idx0->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Dequeue katom_idx1 from ringbuffer */ + kbase_gpu_dequeue_atom(kbdev, i, + end_timestamp); + + katom_idx1->event_code = + BASE_JD_EVENT_STOPPED; + kbase_jm_return_atom_to_js(kbdev, + katom_idx1); + } + katom_idx0->event_code = BASE_JD_EVENT_STOPPED; + kbase_jm_return_atom_to_js(kbdev, katom_idx0); + + } else if (katom_idx1 && + katom_idx1->kctx == katom->kctx && + HAS_DEP(katom_idx1) && + katom_idx1->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Can not dequeue this atom yet - will be + * dequeued when atom at idx0 completes */ + katom_idx1->event_code = BASE_JD_EVENT_STOPPED; + kbase_gpu_mark_atom_for_return(kbdev, + katom_idx1); + } + } + } + + KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, js, completion_code); + + if (job_tail != 0 && job_tail != katom->jc) { + /* Some of the job has been executed */ + dev_dbg(kbdev->dev, + "Update job chain address of atom %p to resume from 0x%llx\n", + (void *)katom, job_tail); + + katom->jc = job_tail; + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, + katom, job_tail, js); + } + + /* Only update the event code for jobs that weren't cancelled */ + if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) + katom->event_code = (enum base_jd_event_code)completion_code; + + /* Complete the job, and start new ones + * + * Also defer remaining work onto the workqueue: + * - Re-queue Soft-stopped jobs + * - For any other jobs, queue the job back into the dependency system + * - Schedule out the parent context if necessary, and schedule a new + * one in. + */ +#ifdef CONFIG_GPU_TRACEPOINTS + { + /* The atom in the HEAD */ + struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, + 0); + + if (next_katom && next_katom->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED) { + char js_string[16]; + + trace_gpu_sched_switch(kbasep_make_job_slot_string(js, + js_string, + sizeof(js_string)), + ktime_to_ns(*end_timestamp), + (u32)next_katom->kctx->id, 0, + next_katom->work_id); + kbdev->hwaccess.backend.slot_rb[js].last_context = + next_katom->kctx; + } else { + char js_string[16]; + + trace_gpu_sched_switch(kbasep_make_job_slot_string(js, + js_string, + sizeof(js_string)), + ktime_to_ns(ktime_get()), 0, 0, + 0); + kbdev->hwaccess.backend.slot_rb[js].last_context = 0; + } + } +#endif + + if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) + kbase_reset_gpu_silent(kbdev); + + if (completion_code == BASE_JD_EVENT_STOPPED) + katom = kbase_jm_return_atom_to_js(kbdev, katom); + else + katom = kbase_jm_complete(kbdev, katom, end_timestamp); + + if (katom) { + dev_dbg(kbdev->dev, + "Cross-slot dependency %p has become runnable.\n", + (void *)katom); + + /* Check if there are lower priority jobs to soft stop */ + kbase_job_slot_ctx_priority_check_locked(kctx, katom); + + kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); + } + + /* For partial shader core off L2 cache flush */ + kbase_pm_update_state(kbdev); + + /* Job completion may have unblocked other atoms. Try to update all job + * slots */ + kbase_backend_slot_update(kbdev); +} + +void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) +{ + int js; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* Reset should always take the GPU out of protected mode */ + WARN_ON(kbase_gpu_in_protected_mode(kbdev)); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + int atom_idx = 0; + int idx; + + for (idx = 0; idx < SLOT_RB_SIZE; idx++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, + js, atom_idx); + bool keep_in_jm_rb = false; + + if (!katom) + break; + if (katom->protected_state.exit == + KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) { + /* protected mode sanity checks */ + KBASE_DEBUG_ASSERT_MSG( + kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev), + "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", + kbase_jd_katom_is_protected(katom), kbase_gpu_in_protected_mode(kbdev)); + KBASE_DEBUG_ASSERT_MSG( + (kbase_jd_katom_is_protected(katom) && js == 0) || + !kbase_jd_katom_is_protected(katom), + "Protected atom on JS%d not supported", js); + } + if ((katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) && + !kbase_ctx_flag(katom->kctx, KCTX_DYING)) + keep_in_jm_rb = true; + + kbase_gpu_release_atom(kbdev, katom, NULL); + + /* + * If the atom wasn't on HW when the reset was issued + * then leave it in the RB and next time we're kicked + * it will be processed again from the starting state. + */ + if (keep_in_jm_rb) { + katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; + /* As the atom was not removed, increment the + * index so that we read the correct atom in the + * next iteration. */ + atom_idx++; + continue; + } + + /* + * The atom was on the HW when the reset was issued + * all we can do is fail the atom. + */ + kbase_gpu_dequeue_atom(kbdev, js, NULL); + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + kbase_jm_complete(kbdev, katom, end_timestamp); + } + } + + /* Re-enable GPU hardware counters if we're resetting from protected + * mode. + */ + kbdev->protected_mode_hwcnt_desired = true; + if (kbdev->protected_mode_hwcnt_disabled) { + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbdev->protected_mode_hwcnt_disabled = false; + + KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev, kbdev); + } + + kbdev->protected_mode_transition = false; + kbase_pm_protected_override_disable(kbdev); +} + +static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, + int js, + struct kbase_jd_atom *katom, + u32 action) +{ + u32 hw_action = action & JS_COMMAND_MASK; + + kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); + kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, + katom->core_req, katom); + katom->kctx->blocked_js[js][katom->sched_priority] = true; +} + +static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + u32 action, + bool disjoint) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; + kbase_gpu_mark_atom_for_return(kbdev, katom); + katom->kctx->blocked_js[katom->slot_nr][katom->sched_priority] = true; + + if (disjoint) + kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, + katom); +} + +static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) +{ + if (katom->x_post_dep) { + struct kbase_jd_atom *dep_atom = katom->x_post_dep; + + if (dep_atom->gpu_rb_state != + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && + dep_atom->gpu_rb_state != + KBASE_ATOM_GPU_RB_RETURN_TO_JS) + return dep_atom->slot_nr; + } + return -1; +} + +bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js, + struct kbase_jd_atom *katom, + u32 action) +{ + struct kbase_jd_atom *katom_idx0; + struct kbase_jd_atom *katom_idx1; + + bool katom_idx0_valid, katom_idx1_valid; + + bool ret = false; + + int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; + int prio_idx0 = 0, prio_idx1 = 0; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); + katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); + + if (katom_idx0) + prio_idx0 = katom_idx0->sched_priority; + if (katom_idx1) + prio_idx1 = katom_idx1->sched_priority; + + if (katom) { + katom_idx0_valid = (katom_idx0 == katom); + /* If idx0 is to be removed and idx1 is on the same context, + * then idx1 must also be removed otherwise the atoms might be + * returned out of order */ + if (katom_idx1) + katom_idx1_valid = (katom_idx1 == katom) || + (katom_idx0_valid && + (katom_idx0->kctx == + katom_idx1->kctx)); + else + katom_idx1_valid = false; + } else { + katom_idx0_valid = (katom_idx0 && + (!kctx || katom_idx0->kctx == kctx)); + katom_idx1_valid = (katom_idx1 && + (!kctx || katom_idx1->kctx == kctx) && + prio_idx0 == prio_idx1); + } + + if (katom_idx0_valid) + stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); + if (katom_idx1_valid) + stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); + + if (katom_idx0_valid) { + if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Simple case - just dequeue and return */ + kbase_gpu_dequeue_atom(kbdev, js, NULL); + if (katom_idx1_valid) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + katom_idx1->event_code = + BASE_JD_EVENT_REMOVED_FROM_NEXT; + kbase_jm_return_atom_to_js(kbdev, katom_idx1); + katom_idx1->kctx->blocked_js[js][prio_idx1] = + true; + } + + katom_idx0->event_code = + BASE_JD_EVENT_REMOVED_FROM_NEXT; + kbase_jm_return_atom_to_js(kbdev, katom_idx0); + katom_idx0->kctx->blocked_js[js][prio_idx0] = true; + } else { + /* katom_idx0 is on GPU */ + if (katom_idx1_valid && katom_idx1->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* katom_idx0 and katom_idx1 are on GPU */ + + if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_COMMAND_NEXT)) == 0) { + /* idx0 has already completed - stop + * idx1 if needed*/ + if (katom_idx1_valid) { + kbase_gpu_stop_atom(kbdev, js, + katom_idx1, + action); + ret = true; + } + } else { + /* idx1 is in NEXT registers - attempt + * to remove */ + kbase_reg_write(kbdev, + JOB_SLOT_REG(js, + JS_COMMAND_NEXT), + JS_COMMAND_NOP); + + if (kbase_reg_read(kbdev, + JOB_SLOT_REG(js, + JS_HEAD_NEXT_LO)) + != 0 || + kbase_reg_read(kbdev, + JOB_SLOT_REG(js, + JS_HEAD_NEXT_HI)) + != 0) { + /* idx1 removed successfully, + * will be handled in IRQ */ + kbase_gpu_remove_atom(kbdev, + katom_idx1, + action, true); + stop_x_dep_idx1 = + should_stop_x_dep_slot(katom_idx1); + + /* stop idx0 if still on GPU */ + kbase_gpu_stop_atom(kbdev, js, + katom_idx0, + action); + ret = true; + } else if (katom_idx1_valid) { + /* idx0 has already completed, + * stop idx1 if needed */ + kbase_gpu_stop_atom(kbdev, js, + katom_idx1, + action); + ret = true; + } + } + } else if (katom_idx1_valid) { + /* idx1 not on GPU but must be dequeued*/ + + /* idx1 will be handled in IRQ */ + kbase_gpu_remove_atom(kbdev, katom_idx1, action, + false); + /* stop idx0 */ + /* This will be repeated for anything removed + * from the next registers, since their normal + * flow was also interrupted, and this function + * might not enter disjoint state e.g. if we + * don't actually do a hard stop on the head + * atom */ + kbase_gpu_stop_atom(kbdev, js, katom_idx0, + action); + ret = true; + } else { + /* no atom in idx1 */ + /* just stop idx0 */ + kbase_gpu_stop_atom(kbdev, js, katom_idx0, + action); + ret = true; + } + } + } else if (katom_idx1_valid) { + if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Mark for return */ + /* idx1 will be returned once idx0 completes */ + kbase_gpu_remove_atom(kbdev, katom_idx1, action, + false); + } else { + /* idx1 is on GPU */ + if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_COMMAND_NEXT)) == 0) { + /* idx0 has already completed - stop idx1 */ + kbase_gpu_stop_atom(kbdev, js, katom_idx1, + action); + ret = true; + } else { + /* idx1 is in NEXT registers - attempt to + * remove */ + kbase_reg_write(kbdev, JOB_SLOT_REG(js, + JS_COMMAND_NEXT), + JS_COMMAND_NOP); + + if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_HEAD_NEXT_LO)) != 0 || + kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_HEAD_NEXT_HI)) != 0) { + /* idx1 removed successfully, will be + * handled in IRQ once idx0 completes */ + kbase_gpu_remove_atom(kbdev, katom_idx1, + action, + false); + } else { + /* idx0 has already completed - stop + * idx1 */ + kbase_gpu_stop_atom(kbdev, js, + katom_idx1, + action); + ret = true; + } + } + } + } + + + if (stop_x_dep_idx0 != -1) + kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0, + NULL, action); + + if (stop_x_dep_idx1 != -1) + kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1, + NULL, action); + + return ret; +} + +void kbase_backend_cache_clean(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + if (katom->need_cache_flush_cores_retained) { + kbase_gpu_start_cache_clean(kbdev); + kbase_gpu_wait_cache_clean(kbdev); + + katom->need_cache_flush_cores_retained = false; + } +} + +void kbase_backend_complete_wq(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + /* + * If cache flush required due to HW workaround then perform the flush + * now + */ + kbase_backend_cache_clean(kbdev, katom); +} + +void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, + base_jd_core_req core_req) +{ + if (!kbdev->pm.active_count) { + mutex_lock(&kbdev->js_data.runpool_mutex); + mutex_lock(&kbdev->pm.lock); + kbase_pm_update_active(kbdev); + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&kbdev->js_data.runpool_mutex); + } +} + +void kbase_gpu_dump_slots(struct kbase_device *kbdev) +{ + unsigned long flags; + int js; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n"); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + int idx; + + for (idx = 0; idx < SLOT_RB_SIZE; idx++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, + js, + idx); + + if (katom) + dev_info(kbdev->dev, + " js%d idx%d : katom=%p gpu_rb_state=%d\n", + js, idx, katom, katom->gpu_rb_state); + else + dev_info(kbdev->dev, " js%d idx%d : empty\n", + js, idx); + } + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.h new file mode 100755 index 000000000000..c3b9f2d85536 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.h @@ -0,0 +1,83 @@ +/* + * + * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * Register-based HW access backend specific APIs + */ + +#ifndef _KBASE_HWACCESS_GPU_H_ +#define _KBASE_HWACCESS_GPU_H_ + +#include + +/** + * kbase_gpu_irq_evict - Evict an atom from a NEXT slot + * + * @kbdev: Device pointer + * @js: Job slot to evict from + * @completion_code: Event code from job that was run. + * + * Evict the atom in the NEXT slot for the specified job slot. This function is + * called from the job complete IRQ handler when the previous job has failed. + * + * Return: true if job evicted from NEXT registers, false otherwise + */ +bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js, + u32 completion_code); + +/** + * kbase_gpu_complete_hw - Complete an atom on job slot js + * + * @kbdev: Device pointer + * @js: Job slot that has completed + * @completion_code: Event code from job that has completed + * @job_tail: The tail address from the hardware if the job has partially + * completed + * @end_timestamp: Time of completion + */ +void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, + u32 completion_code, + u64 job_tail, + ktime_t *end_timestamp); + +/** + * kbase_gpu_inspect - Inspect the contents of the HW access ringbuffer + * + * @kbdev: Device pointer + * @js: Job slot to inspect + * @idx: Index into ringbuffer. 0 is the job currently running on + * the slot, 1 is the job waiting, all other values are invalid. + * Return: The atom at that position in the ringbuffer + * or NULL if no atom present + */ +struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, + int idx); + +/** + * kbase_gpu_dump_slots - Print the contents of the slot ringbuffers + * + * @kbdev: Device pointer + */ +void kbase_gpu_dump_slots(struct kbase_device *kbdev); + +#endif /* _KBASE_HWACCESS_GPU_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c new file mode 100755 index 000000000000..8187e73767be --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c @@ -0,0 +1,365 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * Register-based HW access backend specific job scheduler APIs + */ + +#include +#include +#include +#include +#include + +#if !MALI_USE_CSF +/* + * Hold the runpool_mutex for this + */ +static inline bool timer_callback_should_run(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + int nr_running_ctxs; + + lockdep_assert_held(&kbdev->js_data.runpool_mutex); + + /* Timer must stop if we are suspending */ + if (backend->suspend_timer) + return false; + + /* nr_contexts_pullable is updated with the runpool_mutex. However, the + * locking in the caller gives us a barrier that ensures + * nr_contexts_pullable is up-to-date for reading */ + nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (kbdev->js_data.softstop_always) { + /* Debug support for allowing soft-stop on a single context */ + return true; + } +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) { + /* Timeouts would have to be 4x longer (due to micro- + * architectural design) to support OpenCL conformance tests, so + * only run the timer when there's: + * - 2 or more CL contexts + * - 1 or more GLES contexts + * + * NOTE: We will treat a context that has both Compute and Non- + * Compute jobs will be treated as an OpenCL context (hence, we + * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). + */ + { + int nr_compute_ctxs = + kbasep_js_ctx_attr_count_on_runpool(kbdev, + KBASEP_JS_CTX_ATTR_COMPUTE); + int nr_noncompute_ctxs = nr_running_ctxs - + nr_compute_ctxs; + + return (bool) (nr_compute_ctxs >= 2 || + nr_noncompute_ctxs > 0); + } + } else { + /* Run the timer callback whenever you have at least 1 context + */ + return (bool) (nr_running_ctxs > 0); + } +} + +static enum hrtimer_restart timer_callback(struct hrtimer *timer) +{ + unsigned long flags; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + struct kbase_backend_data *backend; + int s; + bool reset_needed = false; + + KBASE_DEBUG_ASSERT(timer != NULL); + + backend = container_of(timer, struct kbase_backend_data, + scheduling_timer); + kbdev = container_of(backend, struct kbase_device, hwaccess.backend); + js_devdata = &kbdev->js_data; + + /* Loop through the slots */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) { + struct kbase_jd_atom *atom = NULL; + + if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) { + atom = kbase_gpu_inspect(kbdev, s, 0); + KBASE_DEBUG_ASSERT(atom != NULL); + } + + if (atom != NULL) { + /* The current version of the model doesn't support + * Soft-Stop */ + if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { + u32 ticks = atom->ticks++; + +#if !defined(CONFIG_MALI_JOB_DUMP) && !defined(CONFIG_MALI_VECTOR_DUMP) + u32 soft_stop_ticks, hard_stop_ticks, + gpu_reset_ticks; + if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { + soft_stop_ticks = + js_devdata->soft_stop_ticks_cl; + hard_stop_ticks = + js_devdata->hard_stop_ticks_cl; + gpu_reset_ticks = + js_devdata->gpu_reset_ticks_cl; + } else { + soft_stop_ticks = + js_devdata->soft_stop_ticks; + hard_stop_ticks = + js_devdata->hard_stop_ticks_ss; + gpu_reset_ticks = + js_devdata->gpu_reset_ticks_ss; + } + + /* If timeouts have been changed then ensure + * that atom tick count is not greater than the + * new soft_stop timeout. This ensures that + * atoms do not miss any of the timeouts due to + * races between this worker and the thread + * changing the timeouts. */ + if (backend->timeouts_updated && + ticks > soft_stop_ticks) + ticks = atom->ticks = soft_stop_ticks; + + /* Job is Soft-Stoppable */ + if (ticks == soft_stop_ticks) { + /* Job has been scheduled for at least + * js_devdata->soft_stop_ticks ticks. + * Soft stop the slot so we can run + * other jobs. + */ +#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS + int disjoint_threshold = + KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD; + u32 softstop_flags = 0u; + + dev_dbg(kbdev->dev, "Soft-stop"); + /* nr_user_contexts_running is updated + * with the runpool_mutex, but we can't + * take that here. + * + * However, if it's about to be + * increased then the new context can't + * run any jobs until they take the + * hwaccess_lock, so it's OK to observe + * the older value. + * + * Similarly, if it's about to be + * decreased, the last job from another + * context has already finished, so it's + * not too bad that we observe the older + * value and register a disjoint event + * when we try soft-stopping */ + if (js_devdata->nr_user_contexts_running + >= disjoint_threshold) + softstop_flags |= + JS_COMMAND_SW_CAUSES_DISJOINT; + + kbase_job_slot_softstop_swflags(kbdev, + s, atom, softstop_flags); +#endif + } else if (ticks == hard_stop_ticks) { + /* Job has been scheduled for at least + * js_devdata->hard_stop_ticks_ss ticks. + * It should have been soft-stopped by + * now. Hard stop the slot. + */ +#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS + int ms = + js_devdata->scheduling_period_ns + / 1000000u; + dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", + (unsigned long)ticks, + (unsigned long)ms); + kbase_job_slot_hardstop(atom->kctx, s, + atom); +#endif + } else if (ticks == gpu_reset_ticks) { + /* Job has been scheduled for at least + * js_devdata->gpu_reset_ticks_ss ticks. + * It should have left the GPU by now. + * Signal that the GPU needs to be + * reset. + */ + reset_needed = true; + } +#else /* !CONFIG_MALI_JOB_DUMP */ + /* NOTE: During CONFIG_MALI_JOB_DUMP, we use + * the alternate timeouts, which makes the hard- + * stop and GPU reset timeout much longer. We + * also ensure that we don't soft-stop at all. + */ + if (ticks == js_devdata->soft_stop_ticks) { + /* Job has been scheduled for at least + * js_devdata->soft_stop_ticks. We do + * not soft-stop during + * CONFIG_MALI_JOB_DUMP, however. + */ + dev_dbg(kbdev->dev, "Soft-stop"); + } else if (ticks == + js_devdata->hard_stop_ticks_dumping) { + /* Job has been scheduled for at least + * js_devdata->hard_stop_ticks_dumping + * ticks. Hard stop the slot. + */ +#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS + int ms = + js_devdata->scheduling_period_ns + / 1000000u; + dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", + (unsigned long)ticks, + (unsigned long)ms); + kbase_job_slot_hardstop(atom->kctx, s, + atom); +#endif + } else if (ticks == + js_devdata->gpu_reset_ticks_dumping) { + /* Job has been scheduled for at least + * js_devdata->gpu_reset_ticks_dumping + * ticks. It should have left the GPU by + * now. Signal that the GPU needs to be + * reset. + */ + reset_needed = true; + } +#endif /* !CONFIG_MALI_JOB_DUMP */ + } + } + } + if (reset_needed) { + dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve."); + + if (kbase_prepare_to_reset_gpu_locked(kbdev)) + kbase_reset_gpu_locked(kbdev); + } + /* the timer is re-issued if there is contexts in the run-pool */ + + if (backend->timer_running) + hrtimer_start(&backend->scheduling_timer, + HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), + HRTIMER_MODE_REL); + + backend->timeouts_updated = false; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return HRTIMER_NORESTART; +} +#endif /* !MALI_USE_CSF */ + +void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) +{ +#if !MALI_USE_CSF + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + unsigned long flags; + + lockdep_assert_held(&js_devdata->runpool_mutex); + + if (!timer_callback_should_run(kbdev)) { + /* Take spinlock to force synchronisation with timer */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + backend->timer_running = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + /* From now on, return value of timer_callback_should_run() will + * also cause the timer to not requeue itself. Its return value + * cannot change, because it depends on variables updated with + * the runpool_mutex held, which the caller of this must also + * hold */ + hrtimer_cancel(&backend->scheduling_timer); + } + + if (timer_callback_should_run(kbdev) && !backend->timer_running) { + /* Take spinlock to force synchronisation with timer */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + backend->timer_running = true; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + hrtimer_start(&backend->scheduling_timer, + HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), + HRTIMER_MODE_REL); + + KBASE_KTRACE_ADD_JM(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u); + } +#else /* !MALI_USE_CSF */ + CSTD_UNUSED(kbdev); +#endif /* !MALI_USE_CSF */ +} + +int kbase_backend_timer_init(struct kbase_device *kbdev) +{ +#if !MALI_USE_CSF + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + backend->scheduling_timer.function = timer_callback; + backend->timer_running = false; +#else /* !MALI_USE_CSF */ + CSTD_UNUSED(kbdev); +#endif /* !MALI_USE_CSF */ + + return 0; +} + +void kbase_backend_timer_term(struct kbase_device *kbdev) +{ +#if !MALI_USE_CSF + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + hrtimer_cancel(&backend->scheduling_timer); +#else /* !MALI_USE_CSF */ + CSTD_UNUSED(kbdev); +#endif /* !MALI_USE_CSF */ +} + +void kbase_backend_timer_suspend(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + backend->suspend_timer = true; + + kbase_backend_ctx_count_changed(kbdev); +} + +void kbase_backend_timer_resume(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + backend->suspend_timer = false; + + kbase_backend_ctx_count_changed(kbdev); +} + +void kbase_backend_timeouts_changed(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + backend->timeouts_updated = true; +} + diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_internal.h new file mode 100755 index 000000000000..6576e55d2e39 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_internal.h @@ -0,0 +1,74 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * Register-based HW access backend specific job scheduler APIs + */ + +#ifndef _KBASE_JS_BACKEND_H_ +#define _KBASE_JS_BACKEND_H_ + +/** + * kbase_backend_timer_init() - Initialise the JS scheduling timer + * @kbdev: Device pointer + * + * This function should be called at driver initialisation + * + * Return: 0 on success + */ +int kbase_backend_timer_init(struct kbase_device *kbdev); + +/** + * kbase_backend_timer_term() - Terminate the JS scheduling timer + * @kbdev: Device pointer + * + * This function should be called at driver termination + */ +void kbase_backend_timer_term(struct kbase_device *kbdev); + +/** + * kbase_backend_timer_suspend - Suspend is happening, stop the JS scheduling + * timer + * @kbdev: Device pointer + * + * This function should be called on suspend, after the active count has reached + * zero. This is required as the timer may have been started on job submission + * to the job scheduler, but before jobs are submitted to the GPU. + * + * Caller must hold runpool_mutex. + */ +void kbase_backend_timer_suspend(struct kbase_device *kbdev); + +/** + * kbase_backend_timer_resume - Resume is happening, re-evaluate the JS + * scheduling timer + * @kbdev: Device pointer + * + * This function should be called on resume. Note that is is not guaranteed to + * re-start the timer, only evalute whether it should be re-started. + * + * Caller must hold runpool_mutex. + */ +void kbase_backend_timer_resume(struct kbase_device *kbdev); + +#endif /* _KBASE_JS_BACKEND_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.c new file mode 100755 index 000000000000..d5526caa5899 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include "mali_kbase_l2_mmu_config.h" + +/** + * struct l2_mmu_config_limit_region + * + * @value: The default value to load into the L2_MMU_CONFIG register + * @mask: The shifted mask of the field in the L2_MMU_CONFIG register + * @shift: The shift of where the field starts in the L2_MMU_CONFIG register + * This should be the same value as the smaller of the two mask + * values + */ +struct l2_mmu_config_limit_region { + u32 value, mask, shift; +}; + +/** + * struct l2_mmu_config_limit + * + * @product_model: The GPU for which this entry applies + * @read: Values for the read limit field + * @write: Values for the write limit field + */ +struct l2_mmu_config_limit { + u32 product_model; + struct l2_mmu_config_limit_region read; + struct l2_mmu_config_limit_region write; +}; + +/* + * Zero represents no limit + * + * For LBEX TBEX TBAX TTRX and TNAX: + * The value represents the number of outstanding reads (6 bits) or writes (5 bits) + * + * For all other GPUS it is a fraction see: mali_kbase_config_defaults.h + */ +static const struct l2_mmu_config_limit limits[] = { + /* GPU, read, write */ + {GPU_ID2_PRODUCT_LBEX, + {0, GENMASK(10, 5), 5}, + {0, GENMASK(16, 12), 12} }, + {GPU_ID2_PRODUCT_TBEX, + {0, GENMASK(10, 5), 5}, + {0, GENMASK(16, 12), 12} }, + {GPU_ID2_PRODUCT_TBAX, + {0, GENMASK(10, 5), 5}, + {0, GENMASK(16, 12), 12} }, + {GPU_ID2_PRODUCT_TTRX, + {0, GENMASK(12, 7), 7}, + {0, GENMASK(17, 13), 13} }, + {GPU_ID2_PRODUCT_TNAX, + {0, GENMASK(12, 7), 7}, + {0, GENMASK(17, 13), 13} }, + {GPU_ID2_PRODUCT_TGOX, + {KBASE_3BIT_AID_32, GENMASK(14, 12), 12}, + {KBASE_3BIT_AID_32, GENMASK(17, 15), 15} }, + {GPU_ID2_PRODUCT_TNOX, + {KBASE_3BIT_AID_32, GENMASK(14, 12), 12}, + {KBASE_3BIT_AID_32, GENMASK(17, 15), 15} }, +}; + +int kbase_set_mmu_quirks(struct kbase_device *kbdev) +{ + /* All older GPUs had 2 bits for both fields, this is a default */ + struct l2_mmu_config_limit limit = { + 0, /* Any GPU not in the limits array defined above */ + {KBASE_AID_32, GENMASK(25, 24), 24}, + {KBASE_AID_32, GENMASK(27, 26), 26} + }; + u32 product_model, gpu_id; + u32 mmu_config; + int i; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + product_model = gpu_id & GPU_ID2_PRODUCT_MODEL; + + /* Limit the GPU bus bandwidth if the platform needs this. */ + for (i = 0; i < ARRAY_SIZE(limits); i++) { + if (product_model == limits[i].product_model) { + limit = limits[i]; + break; + } + } + + mmu_config = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG)); + + if (kbase_is_gpu_removed(kbdev)) + return -EIO; + + mmu_config &= ~(limit.read.mask | limit.write.mask); + /* Can't use FIELD_PREP() macro here as the mask isn't constant */ + mmu_config |= (limit.read.value << limit.read.shift) | + (limit.write.value << limit.write.shift); + + kbdev->hw_quirks_mmu = mmu_config; + + if (kbdev->system_coherency == COHERENCY_ACE) { + /* Allow memory configuration disparity to be ignored, + * we optimize the use of shared memory and thus we + * expect some disparity in the memory configuration. + */ + kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY; + } + + return 0; +} diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.h new file mode 100755 index 000000000000..0c779ac80d27 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.h @@ -0,0 +1,55 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _KBASE_L2_MMU_CONFIG_H_ +#define _KBASE_L2_MMU_CONFIG_H_ +/** + * kbase_set_mmu_quirks - Set the hw_quirks_mmu field of kbdev + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Use this function to initialise the hw_quirks_mmu field, for instance to set + * the MAX_READS and MAX_WRITES to sane defaults for each GPU. + * + * Return: Zero for succeess or a Linux error code + */ +int kbase_set_mmu_quirks(struct kbase_device *kbdev); + +#endif /* _KBASE_L2_MMU_CONFIG_H */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c new file mode 100755 index 000000000000..e33fe0b8e415 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c @@ -0,0 +1,67 @@ +/* + * + * (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * "Always on" power management policy + */ + +#include +#include + +static bool always_on_shaders_needed(struct kbase_device *kbdev) +{ + return true; +} + +static bool always_on_get_core_active(struct kbase_device *kbdev) +{ + return true; +} + +static void always_on_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static void always_on_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +/* + * The struct kbase_pm_policy structure for the demand power policy. + * + * This is the static structure that defines the demand power policy's callback + * and name. + */ +const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { + "always_on", /* name */ + always_on_init, /* init */ + always_on_term, /* term */ + always_on_shaders_needed, /* shaders_needed */ + always_on_get_core_active, /* get_core_active */ + KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ +}; + +KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.h new file mode 100755 index 000000000000..e7927cf82e5a --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.h @@ -0,0 +1,81 @@ +/* + * + * (C) COPYRIGHT 2011-2015,2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * "Always on" power management policy + */ + +#ifndef MALI_KBASE_PM_ALWAYS_ON_H +#define MALI_KBASE_PM_ALWAYS_ON_H + +/** + * DOC: + * The "Always on" power management policy has the following + * characteristics: + * + * - When KBase indicates that the GPU will be powered up, but we don't yet + * know which Job Chains are to be run: + * Shader Cores are powered up, regardless of whether or not they will be + * needed later. + * + * - When KBase indicates that Shader Cores are needed to submit the currently + * queued Job Chains: + * Shader Cores are kept powered, regardless of whether or not they will be + * needed + * + * - When KBase indicates that the GPU need not be powered: + * The Shader Cores are kept powered, regardless of whether or not they will + * be needed. The GPU itself is also kept powered, even though it is not + * needed. + * + * This policy is automatically overridden during system suspend: the desired + * core state is ignored, and the cores are forced off regardless of what the + * policy requests. After resuming from suspend, new changes to the desired + * core state made by the policy are honored. + * + * Note: + * + * - KBase indicates the GPU will be powered up when it has a User Process that + * has just started to submit Job Chains. + * + * - KBase indicates the GPU need not be powered when all the Job Chains from + * User Processes have finished, and it is waiting for a User Process to + * submit some more Job Chains. + */ + +/** + * struct kbasep_pm_policy_always_on - Private struct for policy instance data + * @dummy: unused dummy variable + * + * This contains data that is private to the particular power policy that is + * active. + */ +struct kbasep_pm_policy_always_on { + int dummy; +}; + +extern const struct kbase_pm_policy kbase_pm_always_on_policy_ops; + +#endif /* MALI_KBASE_PM_ALWAYS_ON_H */ + diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c new file mode 100755 index 000000000000..7b10d06c5fdb --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c @@ -0,0 +1,788 @@ + /* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * GPU backend implementation of base kernel power management APIs + */ + +#include +#include +#include + +#include +#if !MALI_USE_CSF +#include +#include +#include +#endif /* !MALI_USE_CSF */ +#include +#include +#include +#include +#include + +static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data); +static void kbase_pm_hwcnt_disable_worker(struct work_struct *data); +static void kbase_pm_gpu_clock_control_worker(struct work_struct *data); + +int kbase_pm_runtime_init(struct kbase_device *kbdev) +{ + struct kbase_pm_callback_conf *callbacks; + + callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; + if (callbacks) { + kbdev->pm.backend.callback_power_on = + callbacks->power_on_callback; + kbdev->pm.backend.callback_power_off = + callbacks->power_off_callback; + kbdev->pm.backend.callback_power_suspend = + callbacks->power_suspend_callback; + kbdev->pm.backend.callback_power_resume = + callbacks->power_resume_callback; + kbdev->pm.callback_power_runtime_init = + callbacks->power_runtime_init_callback; + kbdev->pm.callback_power_runtime_term = + callbacks->power_runtime_term_callback; + kbdev->pm.backend.callback_power_runtime_on = + callbacks->power_runtime_on_callback; + kbdev->pm.backend.callback_power_runtime_off = + callbacks->power_runtime_off_callback; + kbdev->pm.backend.callback_power_runtime_idle = + callbacks->power_runtime_idle_callback; + kbdev->pm.backend.callback_soft_reset = + callbacks->soft_reset_callback; + + if (callbacks->power_runtime_init_callback) + return callbacks->power_runtime_init_callback(kbdev); + else + return 0; + } + + kbdev->pm.backend.callback_power_on = NULL; + kbdev->pm.backend.callback_power_off = NULL; + kbdev->pm.backend.callback_power_suspend = NULL; + kbdev->pm.backend.callback_power_resume = NULL; + kbdev->pm.callback_power_runtime_init = NULL; + kbdev->pm.callback_power_runtime_term = NULL; + kbdev->pm.backend.callback_power_runtime_on = NULL; + kbdev->pm.backend.callback_power_runtime_off = NULL; + kbdev->pm.backend.callback_power_runtime_idle = NULL; + kbdev->pm.backend.callback_soft_reset = NULL; + + return 0; +} + +void kbase_pm_runtime_term(struct kbase_device *kbdev) +{ + if (kbdev->pm.callback_power_runtime_term) { + kbdev->pm.callback_power_runtime_term(kbdev); + } +} + +void kbase_pm_register_access_enable(struct kbase_device *kbdev) +{ + struct kbase_pm_callback_conf *callbacks; + + callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; + + if (callbacks) + callbacks->power_on_callback(kbdev); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) + dev_err(kbdev->dev, "Attempting to power on while GPU lost\n"); +#endif + + kbdev->pm.backend.gpu_powered = true; +} + +void kbase_pm_register_access_disable(struct kbase_device *kbdev) +{ + struct kbase_pm_callback_conf *callbacks; + + callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; + + if (callbacks) + callbacks->power_off_callback(kbdev); + + kbdev->pm.backend.gpu_powered = false; +} + +int kbase_hwaccess_pm_init(struct kbase_device *kbdev) +{ + int ret = 0; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + mutex_init(&kbdev->pm.lock); + + kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait", + WQ_HIGHPRI | WQ_UNBOUND, 1); + if (!kbdev->pm.backend.gpu_poweroff_wait_wq) + return -ENOMEM; + + INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, + kbase_pm_gpu_poweroff_wait_wq); + + kbdev->pm.backend.ca_cores_enabled = ~0ull; + kbdev->pm.backend.gpu_powered = false; + kbdev->pm.backend.gpu_ready = false; + kbdev->pm.suspending = false; +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbase_pm_set_gpu_lost(kbdev, false); +#endif +#ifdef CONFIG_MALI_BIFROST_DEBUG + kbdev->pm.backend.driver_ready_for_irqs = false; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); + + /* Initialise the metrics subsystem */ + ret = kbasep_pm_metrics_init(kbdev); + if (ret) + return ret; + + init_waitqueue_head(&kbdev->pm.backend.reset_done_wait); + kbdev->pm.backend.reset_done = false; + + init_waitqueue_head(&kbdev->pm.zero_active_count_wait); + kbdev->pm.active_count = 0; + + spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock); + + init_waitqueue_head(&kbdev->pm.backend.poweroff_wait); + + if (kbase_pm_ca_init(kbdev) != 0) + goto workq_fail; + + kbase_pm_policy_init(kbdev); + + if (kbase_pm_state_machine_init(kbdev) != 0) + goto pm_state_machine_fail; + + kbdev->pm.backend.hwcnt_desired = false; + kbdev->pm.backend.hwcnt_disabled = true; + INIT_WORK(&kbdev->pm.backend.hwcnt_disable_work, + kbase_pm_hwcnt_disable_worker); + kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + + if (IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED)) { + kbdev->pm.backend.l2_always_on = false; + kbdev->pm.backend.gpu_clock_slow_down_wa = false; + + return 0; + } + + /* WA1: L2 always_on for GPUs being affected by GPU2017-1336 */ + if (!IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE)) { + kbdev->pm.backend.gpu_clock_slow_down_wa = false; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336)) + kbdev->pm.backend.l2_always_on = true; + else + kbdev->pm.backend.l2_always_on = false; + + return 0; + } + + /* WA3: Clock slow down for GPUs being affected by GPU2017-1336 */ + kbdev->pm.backend.l2_always_on = false; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336)) { + kbdev->pm.backend.gpu_clock_slow_down_wa = true; + kbdev->pm.backend.gpu_clock_suspend_freq = 0; + kbdev->pm.backend.gpu_clock_slow_down_desired = true; + kbdev->pm.backend.gpu_clock_slowed_down = false; + INIT_WORK(&kbdev->pm.backend.gpu_clock_control_work, + kbase_pm_gpu_clock_control_worker); + } else + kbdev->pm.backend.gpu_clock_slow_down_wa = false; + + return 0; + +pm_state_machine_fail: + kbase_pm_policy_term(kbdev); + kbase_pm_ca_term(kbdev); +workq_fail: + kbasep_pm_metrics_term(kbdev); + return -EINVAL; +} + +void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) +{ + lockdep_assert_held(&kbdev->pm.lock); + + /* Turn clocks and interrupts on - no-op if we haven't done a previous + * kbase_pm_clock_off() */ + kbase_pm_clock_on(kbdev, is_resume); + + if (!is_resume) { + unsigned long flags; + + /* Force update of L2 state - if we have abandoned a power off + * then this may be required to power the L2 back on. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + /* Update core status as required by the policy */ + kbase_pm_update_cores_state(kbdev); + + /* NOTE: We don't wait to reach the desired state, since running atoms + * will wait for that state to be reached anyway */ +} + +static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) +{ + struct kbase_device *kbdev = container_of(data, struct kbase_device, + pm.backend.gpu_poweroff_wait_work); + struct kbase_pm_device_data *pm = &kbdev->pm; + struct kbase_pm_backend_data *backend = &pm->backend; + unsigned long flags; + +#if !MALI_USE_CSF + /* Wait for power transitions to complete. We do this with no locks held + * so that we don't deadlock with any pending workqueues. + */ + kbase_pm_wait_for_desired_state(kbdev); +#endif + + kbase_pm_lock(kbdev); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_pm_is_gpu_lost(kbdev)) + backend->poweron_required = false; +#endif + + if (!backend->poweron_required) { + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + WARN_ON(backend->shaders_state != + KBASE_SHADERS_OFF_CORESTACK_OFF || + backend->l2_state != KBASE_L2_OFF); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Disable interrupts and turn the clock off */ + if (!kbase_pm_clock_off(kbdev)) { + /* + * Page/bus faults are pending, must drop locks to + * process. Interrupts are disabled so no more faults + * should be generated at this point. + */ + kbase_pm_unlock(kbdev); + kbase_flush_mmu_wqs(kbdev); + kbase_pm_lock(kbdev); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* poweron_required may have changed while pm lock + * was released. + */ + if (kbase_pm_is_gpu_lost(kbdev)) + backend->poweron_required = false; +#endif + + /* Turn off clock now that fault have been handled. We + * dropped locks so poweron_required may have changed - + * power back on if this is the case (effectively only + * re-enabling of the interrupts would be done in this + * case, as the clocks to GPU were not withdrawn yet). + */ + if (backend->poweron_required) + kbase_pm_clock_on(kbdev, false); + else + WARN_ON(!kbase_pm_clock_off(kbdev)); + } + } + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + backend->poweroff_wait_in_progress = false; + if (backend->poweron_required) { + backend->poweron_required = false; + kbdev->pm.backend.l2_desired = true; +#if MALI_USE_CSF + kbdev->pm.backend.mcu_desired = true; +#endif + kbase_pm_update_state(kbdev); + kbase_pm_update_cores_state_nolock(kbdev); +#if !MALI_USE_CSF + kbase_backend_slot_update(kbdev); +#endif /* !MALI_USE_CSF */ + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_pm_unlock(kbdev); + + wake_up(&kbdev->pm.backend.poweroff_wait); +} + +static void kbase_pm_l2_clock_slow(struct kbase_device *kbdev) +{ +#if defined(CONFIG_MALI_BIFROST_DVFS) + struct clk *clk = kbdev->clocks[0]; +#endif + + if (!kbdev->pm.backend.gpu_clock_slow_down_wa) + return; + + /* No suspend clock is specified */ + if (WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_suspend_freq)) + return; + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) + + /* Suspend devfreq */ + devfreq_suspend_device(kbdev->devfreq); + + /* Keep the current freq to restore it upon resume */ + kbdev->previous_frequency = kbdev->current_nominal_freq; + + /* Slow down GPU clock to the suspend clock*/ + kbase_devfreq_force_freq(kbdev, + kbdev->pm.backend.gpu_clock_suspend_freq); + +#elif defined(CONFIG_MALI_BIFROST_DVFS) /* CONFIG_MALI_BIFROST_DEVFREQ */ + + if (WARN_ON_ONCE(!clk)) + return; + + /* Stop the metrics gathering framework */ + if (kbase_pm_metrics_is_active(kbdev)) + kbase_pm_metrics_stop(kbdev); + + /* Keep the current freq to restore it upon resume */ + kbdev->previous_frequency = clk_get_rate(clk); + + /* Slow down GPU clock to the suspend clock*/ + if (WARN_ON_ONCE(clk_set_rate(clk, + kbdev->pm.backend.gpu_clock_suspend_freq))) + dev_err(kbdev->dev, "Failed to set suspend freq\n"); + +#endif /* CONFIG_MALI_BIFROST_DVFS */ +} + +static void kbase_pm_l2_clock_normalize(struct kbase_device *kbdev) +{ +#if defined(CONFIG_MALI_BIFROST_DVFS) + struct clk *clk = kbdev->clocks[0]; +#endif + + if (!kbdev->pm.backend.gpu_clock_slow_down_wa) + return; + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) + + /* Restore GPU clock to the previous one */ + kbase_devfreq_force_freq(kbdev, kbdev->previous_frequency); + + /* Resume devfreq */ + devfreq_resume_device(kbdev->devfreq); + +#elif defined(CONFIG_MALI_BIFROST_DVFS) /* CONFIG_MALI_BIFROST_DEVFREQ */ + + if (WARN_ON_ONCE(!clk)) + return; + + /* Restore GPU clock */ + if (WARN_ON_ONCE(clk_set_rate(clk, kbdev->previous_frequency))) + dev_err(kbdev->dev, "Failed to restore freq (%lu)\n", + kbdev->previous_frequency); + + /* Restart the metrics gathering framework */ + kbase_pm_metrics_start(kbdev); + +#endif /* CONFIG_MALI_BIFROST_DVFS */ +} + +static void kbase_pm_gpu_clock_control_worker(struct work_struct *data) +{ + struct kbase_device *kbdev = container_of(data, struct kbase_device, + pm.backend.gpu_clock_control_work); + struct kbase_pm_device_data *pm = &kbdev->pm; + struct kbase_pm_backend_data *backend = &pm->backend; + unsigned long flags; + bool slow_down = false, normalize = false; + + /* Determine if GPU clock control is required */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (!backend->gpu_clock_slowed_down && + backend->gpu_clock_slow_down_desired) { + slow_down = true; + backend->gpu_clock_slowed_down = true; + } else if (backend->gpu_clock_slowed_down && + !backend->gpu_clock_slow_down_desired) { + normalize = true; + backend->gpu_clock_slowed_down = false; + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Control GPU clock according to the request of L2 state machine. + * The GPU clock needs to be lowered for safe L2 power down + * and restored to previous speed at L2 power up. + */ + if (slow_down) + kbase_pm_l2_clock_slow(kbdev); + else if (normalize) + kbase_pm_l2_clock_normalize(kbdev); + + /* Tell L2 state machine to transit to next state */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +static void kbase_pm_hwcnt_disable_worker(struct work_struct *data) +{ + struct kbase_device *kbdev = container_of(data, struct kbase_device, + pm.backend.hwcnt_disable_work); + struct kbase_pm_device_data *pm = &kbdev->pm; + struct kbase_pm_backend_data *backend = &pm->backend; + unsigned long flags; + + bool do_disable; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + do_disable = !backend->hwcnt_desired && !backend->hwcnt_disabled; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (!do_disable) + return; + + kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + do_disable = !backend->hwcnt_desired && !backend->hwcnt_disabled; + + if (do_disable) { + /* PM state did not change while we were doing the disable, + * so commit the work we just performed and continue the state + * machine. + */ + backend->hwcnt_disabled = true; + kbase_pm_update_state(kbdev); +#if !MALI_USE_CSF + kbase_backend_slot_update(kbdev); +#endif /* !MALI_USE_CSF */ + } else { + /* PM state was updated while we were doing the disable, + * so we need to undo the disable we just performed. + */ + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +void kbase_pm_do_poweroff(struct kbase_device *kbdev) +{ + unsigned long flags; + + lockdep_assert_held(&kbdev->pm.lock); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) + goto unlock_hwaccess; + + if (kbdev->pm.backend.poweroff_wait_in_progress) + goto unlock_hwaccess; + +#if MALI_USE_CSF + kbdev->pm.backend.mcu_desired = false; +#else + /* Force all cores off */ + kbdev->pm.backend.shaders_desired = false; +#endif + kbdev->pm.backend.l2_desired = false; + + kbdev->pm.backend.poweroff_wait_in_progress = true; + kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off = true; + + /* l2_desired being false should cause the state machine to + * start powering off the L2. When it actually is powered off, + * the interrupt handler will call kbase_pm_l2_update_state() + * again, which will trigger the kbase_pm_gpu_poweroff_wait_wq. + * Callers of this function will need to wait on poweroff_wait. + */ + kbase_pm_update_state(kbdev); + +unlock_hwaccess: + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +static bool is_poweroff_in_progress(struct kbase_device *kbdev) +{ + bool ret; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + ret = (kbdev->pm.backend.poweroff_wait_in_progress == false); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return ret; +} + +void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev) +{ + wait_event_killable(kbdev->pm.backend.poweroff_wait, + is_poweroff_in_progress(kbdev)); +} +KBASE_EXPORT_TEST_API(kbase_pm_wait_for_poweroff_complete); + +int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, + unsigned int flags) +{ + unsigned long irq_flags; + int ret; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + kbase_pm_lock(kbdev); + + /* A suspend won't happen during startup/insmod */ + KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); + + /* Power up the GPU, don't enable IRQs as we are not ready to receive + * them. */ + ret = kbase_pm_init_hw(kbdev, flags); + if (ret) { + kbase_pm_unlock(kbdev); + return ret; + } + + kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] = + kbdev->pm.debug_core_mask[1] = + kbdev->pm.debug_core_mask[2] = + kbdev->gpu_props.props.raw_props.shader_present; + + /* Pretend the GPU is active to prevent a power policy turning the GPU + * cores off */ + kbdev->pm.active_count = 1; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + /* Ensure cycle counter is off */ + kbdev->pm.backend.gpu_cycle_counter_requests = 0; + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + + /* We are ready to receive IRQ's now as power policy is set up, so + * enable them now. */ +#ifdef CONFIG_MALI_BIFROST_DEBUG + kbdev->pm.backend.driver_ready_for_irqs = true; +#endif + kbase_pm_enable_interrupts(kbdev); + + WARN_ON(!kbdev->pm.backend.gpu_powered); + /* GPU has been powered up (by kbase_pm_init_hw) and interrupts have + * been enabled, so GPU is ready for use and PM state machine can be + * exercised from this point onwards. + */ + kbdev->pm.backend.gpu_ready = true; + + /* Turn on the GPU and any cores needed by the policy */ +#if MALI_USE_CSF + /* Turn on the L2 caches, needed for firmware boot */ + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + kbdev->pm.backend.l2_desired = true; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); +#endif + kbase_pm_do_poweron(kbdev, false); + kbase_pm_unlock(kbdev); + + return 0; +} + +void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + mutex_lock(&kbdev->pm.lock); + kbase_pm_do_poweroff(kbdev); + mutex_unlock(&kbdev->pm.lock); +} + +KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); + +void kbase_hwaccess_pm_term(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0); + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0); + + cancel_work_sync(&kbdev->pm.backend.hwcnt_disable_work); + + if (kbdev->pm.backend.hwcnt_disabled) { + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + /* Free any resources the policy allocated */ + kbase_pm_state_machine_term(kbdev); + kbase_pm_policy_term(kbdev); + kbase_pm_ca_term(kbdev); + + /* Shut down the metrics subsystem */ + kbasep_pm_metrics_term(kbdev); + + destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); +} + +void kbase_pm_power_changed(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_update_state(kbdev); + +#if !MALI_USE_CSF + kbase_backend_slot_update(kbdev); +#endif /* !MALI_USE_CSF */ + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, + u64 new_core_mask_js0, u64 new_core_mask_js1, + u64 new_core_mask_js2) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&kbdev->pm.lock); + + if (kbase_dummy_job_wa_enabled(kbdev)) { + dev_warn(kbdev->dev, "Change of core mask not supported for slot 0 as dummy job WA is enabled"); + new_core_mask_js0 = kbdev->pm.debug_core_mask[0]; + } + + kbdev->pm.debug_core_mask[0] = new_core_mask_js0; + kbdev->pm.debug_core_mask[1] = new_core_mask_js1; + kbdev->pm.debug_core_mask[2] = new_core_mask_js2; + kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 | + new_core_mask_js2; + + kbase_pm_update_dynamic_cores_onoff(kbdev); +} + +void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) +{ + kbase_pm_update_active(kbdev); +} + +void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev) +{ + kbase_pm_update_active(kbdev); +} + +void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) +{ + /* Force power off the GPU and all cores (regardless of policy), only + * after the PM active count reaches zero (otherwise, we risk turning it + * off prematurely) */ + kbase_pm_lock(kbdev); + + kbase_pm_do_poweroff(kbdev); + +#if !MALI_USE_CSF + kbase_backend_timer_suspend(kbdev); +#endif /* !MALI_USE_CSF */ + + kbase_pm_unlock(kbdev); + + kbase_pm_wait_for_poweroff_complete(kbdev); + + if (kbdev->pm.backend.callback_power_suspend) + kbdev->pm.backend.callback_power_suspend(kbdev); +} + +void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) +{ + kbase_pm_lock(kbdev); + + kbdev->pm.suspending = false; +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_pm_is_gpu_lost(kbdev)) { + dev_dbg(kbdev->dev, "%s: GPU lost in progress\n", __func__); + kbase_pm_unlock(kbdev); + return; + } +#endif + kbase_pm_do_poweron(kbdev, true); + +#if !MALI_USE_CSF + kbase_backend_timer_resume(kbdev); +#endif /* !MALI_USE_CSF */ + + kbase_pm_unlock(kbdev); +} + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) +{ + unsigned long flags; + ktime_t end_timestamp = ktime_get(); + struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + + mutex_lock(&kbdev->pm.lock); + mutex_lock(&arb_vm_state->vm_state_lock); + if (kbdev->pm.backend.gpu_powered && + !kbase_pm_is_gpu_lost(kbdev)) { + kbase_pm_set_gpu_lost(kbdev, true); + + /* GPU is no longer mapped to VM. So no interrupts will + * be received and Mali registers have been replaced by + * dummy RAM + */ + WARN(!kbase_is_gpu_removed(kbdev), + "GPU is still available after GPU lost event\n"); + + /* Full GPU reset will have been done by hypervisor, so + * cancel + */ + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING); + hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); + kbase_synchronize_irqs(kbdev); + + /* Clear all jobs running on the GPU */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->protected_mode = false; + kbase_backend_reset(kbdev, &end_timestamp); + kbase_pm_metrics_update(kbdev, NULL); + kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Cancel any pending HWC dumps */ + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + } + mutex_unlock(&arb_vm_state->vm_state_lock); + mutex_unlock(&kbdev->pm.lock); +} + +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c new file mode 100755 index 000000000000..984e12503009 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c @@ -0,0 +1,114 @@ +/* + * + * (C) COPYRIGHT 2013-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Base kernel core availability APIs + */ + +#include +#include +#include +#ifdef MALI_BIFROST_NO_MALI +#include +#endif +#include + +int kbase_pm_ca_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; + + if (kbdev->current_core_mask) + pm_backend->ca_cores_enabled = kbdev->current_core_mask; + else + pm_backend->ca_cores_enabled = + kbdev->gpu_props.props.raw_props.shader_present; +#endif + + return 0; +} + +void kbase_pm_ca_term(struct kbase_device *kbdev) +{ +} + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ +void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) +{ + struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!(core_mask & kbdev->pm.debug_core_mask_all)) { + dev_err(kbdev->dev, "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n", + core_mask, kbdev->pm.debug_core_mask_all); + goto unlock; + } + + if (kbase_dummy_job_wa_enabled(kbdev)) { + dev_err(kbdev->dev, "Dynamic core scaling not supported as dummy job WA is enabled"); + goto unlock; + } + + pm_backend->ca_cores_enabled = core_mask; + + kbase_pm_update_state(kbdev); + +unlock: + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n", + pm_backend->ca_cores_enabled); +} +#endif + +u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; +#endif + + lockdep_assert_held(&kbdev->hwaccess_lock); + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + return pm_backend->ca_cores_enabled & kbdev->pm.debug_core_mask_all; +#else + return kbdev->gpu_props.props.raw_props.shader_present & + kbdev->pm.debug_core_mask_all; +#endif +} + +KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask); + +u64 kbase_pm_ca_get_instr_core_mask(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + +#ifdef CONFIG_MALI_BIFROST_NO_MALI + return (((1ull) << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1); +#elif MALI_USE_CSF + return kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); +#else + return kbdev->pm.backend.pm_shaders_core_mask; +#endif +} diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h new file mode 100755 index 000000000000..5423e96725b9 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h @@ -0,0 +1,89 @@ +/* + * + * (C) COPYRIGHT 2011-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Base kernel core availability APIs + */ + +#ifndef _KBASE_PM_CA_H_ +#define _KBASE_PM_CA_H_ + +/** + * kbase_pm_ca_init - Initialize core availability framework + * + * Must be called before calling any other core availability function + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 if the core availability framework was successfully initialized, + * -errno otherwise + */ +int kbase_pm_ca_init(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_term - Terminate core availability framework + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_ca_term(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_get_core_mask - Get currently available shaders core mask + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Returns a mask of the currently available shader cores. + * Calls into the core availability policy + * + * Return: The bit mask of available cores + */ +u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_update_core_status - Update core status + * + * @kbdev: The kbase device structure for the device (must be + * a valid pointer) + * @cores_ready: The bit mask of cores ready for job submission + * @cores_transitioning: The bit mask of cores that are transitioning power + * state + * + * Update core availability policy with current core power status + * + * Calls into the core availability policy + */ +void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, + u64 cores_transitioning); + +/** + * kbase_pm_ca_get_instr_core_mask - Get the PM state sync-ed shaders core mask + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Returns a mask of the PM state synchronised shader cores for arranging + * HW performance counter dumps + * + * Return: The bit mask of PM state synchronised cores + */ +u64 kbase_pm_ca_get_instr_core_mask(struct kbase_device *kbdev); + +#endif /* _KBASE_PM_CA_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca_devfreq.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca_devfreq.h new file mode 100755 index 000000000000..f67ec650c981 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca_devfreq.h @@ -0,0 +1,60 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * A core availability policy for use with devfreq, where core masks are + * associated with OPPs. + */ + +#ifndef MALI_KBASE_PM_CA_DEVFREQ_H +#define MALI_KBASE_PM_CA_DEVFREQ_H + +/** + * struct kbasep_pm_ca_policy_devfreq - Private structure for devfreq ca policy + * + * This contains data that is private to the devfreq core availability + * policy. + * + * @cores_desired: Cores that the policy wants to be available + * @cores_enabled: Cores that the policy is currently returning as available + * @cores_used: Cores currently powered or transitioning + */ +struct kbasep_pm_ca_policy_devfreq { + u64 cores_desired; + u64 cores_enabled; + u64 cores_used; +}; + +extern const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops; + +/** + * kbase_devfreq_set_core_mask - Set core mask for policy to use + * @kbdev: Device pointer + * @core_mask: New core mask + * + * The new core mask will have immediate effect if the GPU is powered, or will + * take effect when it is next powered on. + */ +void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask); + +#endif /* MALI_KBASE_PM_CA_DEVFREQ_H */ + diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c new file mode 100755 index 000000000000..9eef44ad877f --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c @@ -0,0 +1,66 @@ +/* + * + * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * "Coarse Demand" power management policy + */ + +#include +#include + +static bool coarse_demand_shaders_needed(struct kbase_device *kbdev) +{ + return kbase_pm_is_active(kbdev); +} + +static bool coarse_demand_get_core_active(struct kbase_device *kbdev) +{ + return kbase_pm_is_active(kbdev); +} + +static void coarse_demand_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static void coarse_demand_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +/* The struct kbase_pm_policy structure for the demand power policy. + * + * This is the static structure that defines the demand power policy's callback + * and name. + */ +const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { + "coarse_demand", /* name */ + coarse_demand_init, /* init */ + coarse_demand_term, /* term */ + coarse_demand_shaders_needed, /* shaders_needed */ + coarse_demand_get_core_active, /* get_core_active */ + KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ +}; + +KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.h new file mode 100755 index 000000000000..304e5d7fa32d --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.h @@ -0,0 +1,69 @@ +/* + * + * (C) COPYRIGHT 2012-2015,2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * "Coarse Demand" power management policy + */ + +#ifndef MALI_KBASE_PM_COARSE_DEMAND_H +#define MALI_KBASE_PM_COARSE_DEMAND_H + +/** + * DOC: + * The "Coarse" demand power management policy has the following + * characteristics: + * - When KBase indicates that the GPU will be powered up, but we don't yet + * know which Job Chains are to be run: + * - Shader Cores are powered up, regardless of whether or not they will be + * needed later. + * - When KBase indicates that Shader Cores are needed to submit the currently + * queued Job Chains: + * - Shader Cores are kept powered, regardless of whether or not they will + * be needed + * - When KBase indicates that the GPU need not be powered: + * - The Shader Cores are powered off, and the GPU itself is powered off too. + * + * @note: + * - KBase indicates the GPU will be powered up when it has a User Process that + * has just started to submit Job Chains. + * - KBase indicates the GPU need not be powered when all the Job Chains from + * User Processes have finished, and it is waiting for a User Process to + * submit some more Job Chains. + */ + +/** + * struct kbasep_pm_policy_coarse_demand - Private structure for coarse demand + * policy + * + * This contains data that is private to the coarse demand power policy. + * + * @dummy: Dummy member - no state needed + */ +struct kbasep_pm_policy_coarse_demand { + int dummy; +}; + +extern const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops; + +#endif /* MALI_KBASE_PM_COARSE_DEMAND_H */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h new file mode 100755 index 000000000000..7322c093c7b6 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h @@ -0,0 +1,560 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Backend-specific Power Manager definitions + */ + +#ifndef _KBASE_PM_HWACCESS_DEFS_H_ +#define _KBASE_PM_HWACCESS_DEFS_H_ + +#include "mali_kbase_pm_always_on.h" +#include "mali_kbase_pm_coarse_demand.h" +#if !MALI_CUSTOMER_RELEASE +#include "mali_kbase_pm_always_on_demand.h" +#endif + +/* Forward definition - see mali_kbase.h */ +struct kbase_device; +struct kbase_jd_atom; + +/** + * enum kbase_pm_core_type - The types of core in a GPU. + * + * These enumerated values are used in calls to + * - kbase_pm_get_present_cores() + * - kbase_pm_get_active_cores() + * - kbase_pm_get_trans_cores() + * - kbase_pm_get_ready_cores(). + * + * They specify which type of core should be acted on. These values are set in + * a manner that allows core_type_to_reg() function to be simpler and more + * efficient. + * + * @KBASE_PM_CORE_L2: The L2 cache + * @KBASE_PM_CORE_SHADER: Shader cores + * @KBASE_PM_CORE_TILER: Tiler cores + * @KBASE_PM_CORE_STACK: Core stacks + */ +enum kbase_pm_core_type { + KBASE_PM_CORE_L2 = L2_PRESENT_LO, + KBASE_PM_CORE_SHADER = SHADER_PRESENT_LO, + KBASE_PM_CORE_TILER = TILER_PRESENT_LO, + KBASE_PM_CORE_STACK = STACK_PRESENT_LO +}; + +/** + * enum kbase_l2_core_state - The states used for the L2 cache & tiler power + * state machine. + * + * @KBASE_L2_OFF: The L2 cache and tiler are off + * @KBASE_L2_PEND_ON: The L2 cache and tiler are powering on + * @KBASE_L2_RESTORE_CLOCKS: The GPU clock is restored. Conditionally used. + * @KBASE_L2_ON_HWCNT_ENABLE: The L2 cache and tiler are on, and hwcnt is being + * enabled + * @KBASE_L2_ON: The L2 cache and tiler are on, and hwcnt is enabled + * @KBASE_L2_ON_HWCNT_DISABLE: The L2 cache and tiler are on, and hwcnt is being + * disabled + * @KBASE_L2_SLOW_DOWN_CLOCKS: The GPU clock is set to appropriate or lowest + * clock. Conditionally used. + * @KBASE_L2_POWER_DOWN: The L2 cache and tiler are about to be powered off + * @KBASE_L2_PEND_OFF: The L2 cache and tiler are powering off + * @KBASE_L2_RESET_WAIT: The GPU is resetting, L2 cache and tiler power state + * are unknown + */ +enum kbase_l2_core_state { +#define KBASEP_L2_STATE(n) KBASE_L2_ ## n, +#include "mali_kbase_pm_l2_states.h" +#undef KBASEP_L2_STATE +}; + +#if MALI_USE_CSF +/** + * enum kbase_mcu_state - The states used for the MCU state machine. + * + * @KBASE_MCU_OFF: The MCU is powered off. + * @KBASE_MCU_PEND_ON_RELOAD: The warm boot of MCU or cold boot of MCU (with + * firmware reloading) is in progress. + * @KBASE_MCU_ON_GLB_REINIT_PEND: The MCU is enabled and Global configuration + * requests have been sent to the firmware. + * @KBASE_MCU_ON_HWCNT_ENABLE: The Global requests have completed and MCU is + * now ready for use and hwcnt is being enabled. + * @KBASE_MCU_ON: The MCU is active and hwcnt has been enabled. + * @KBASE_MCU_ON_HWCNT_DISABLE: The MCU is on and hwcnt is being disabled. + * @KBASE_MCU_ON_HALT: The MCU is on and hwcnt has been disabled, + * MCU halt would be triggered. + * @KBASE_MCU_ON_PEND_HALT: MCU halt in progress, confirmation pending. + * @KBASE_MCU_POWER_DOWN: MCU halted operations, pending being disabled. + * @KBASE_MCU_PEND_OFF: MCU is being disabled, pending on powering off. + * @KBASE_MCU_RESET_WAIT: The GPU is resetting, MCU state is unknown. + */ +enum kbase_mcu_state { +#define KBASEP_MCU_STATE(n) KBASE_MCU_ ## n, +#include "mali_kbase_pm_mcu_states.h" +#undef KBASEP_MCU_STATE +}; +#endif + +/** + * enum kbase_shader_core_state - The states used for the shaders' state machine. + * + * @KBASE_SHADERS_OFF_CORESTACK_OFF: The shaders and core stacks are off + * @KBASE_SHADERS_OFF_CORESTACK_PEND_ON: The shaders are off, core stacks have + * been requested to power on and hwcnt + * is being disabled + * @KBASE_SHADERS_PEND_ON_CORESTACK_ON: Core stacks are on, shaders have been + * requested to power on. Or after doing + * partial shader on/off, checking whether + * it's the desired state. + * @KBASE_SHADERS_ON_CORESTACK_ON: The shaders and core stacks are on, and hwcnt + * already enabled. + * @KBASE_SHADERS_ON_CORESTACK_ON_RECHECK: The shaders and core stacks + * are on, hwcnt disabled, and checks + * to powering down or re-enabling + * hwcnt. + * @KBASE_SHADERS_WAIT_OFF_CORESTACK_ON: The shaders have been requested to + * power off, but they remain on for the + * duration of the hysteresis timer + * @KBASE_SHADERS_WAIT_GPU_IDLE: The shaders partial poweroff needs to reach + * a state where jobs on the GPU are finished + * including jobs currently running and in the + * GPU queue because of GPU2017-861 + * @KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON: The hysteresis timer has expired + * @KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON: The core stacks are on and the + * level 2 cache is being flushed. + * @KBASE_SHADERS_READY_OFF_CORESTACK_ON: The core stacks are on and the shaders + * are ready to be powered off. + * @KBASE_SHADERS_PEND_OFF_CORESTACK_ON: The core stacks are on, and the shaders + * have been requested to power off + * @KBASE_SHADERS_OFF_CORESTACK_PEND_OFF: The shaders are off, and the core stacks + * have been requested to power off + * @KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF: Shaders and corestacks are + * off, but the tick timer + * cancellation is still + * pending. + * @KBASE_SHADERS_RESET_WAIT: The GPU is resetting, shader and core stack power + * states are unknown + */ +enum kbase_shader_core_state { +#define KBASEP_SHADER_STATE(n) KBASE_SHADERS_ ## n, +#include "mali_kbase_pm_shader_states.h" +#undef KBASEP_SHADER_STATE +}; + +/** + * struct kbasep_pm_metrics - Metrics data collected for use by the power + * management framework. + * + * @time_busy: number of ns the GPU was busy executing jobs since the + * @time_period_start timestamp. + * @time_idle: number of ns since time_period_start the GPU was not executing + * jobs since the @time_period_start timestamp. + * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that + * if two CL jobs were active for 400ns, this value would be updated + * with 800. + * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that + * if two GL jobs were active for 400ns, this value would be updated + * with 800. + */ +struct kbasep_pm_metrics { + u32 time_busy; + u32 time_idle; + u32 busy_cl[2]; + u32 busy_gl; +}; + +/** + * struct kbasep_pm_metrics_state - State required to collect the metrics in + * struct kbasep_pm_metrics + * @time_period_start: time at which busy/idle measurements started + * @gpu_active: true when the GPU is executing jobs. false when + * not. Updated when the job scheduler informs us a job in submitted + * or removed from a GPU slot. + * @active_cl_ctx: number of CL jobs active on the GPU. Array is per-device. + * @active_gl_ctx: number of GL jobs active on the GPU. Array is per-slot. + * @lock: spinlock protecting the kbasep_pm_metrics_data structure + * @platform_data: pointer to data controlled by platform specific code + * @kbdev: pointer to kbase device for which metrics are collected + * @values: The current values of the power management metrics. The + * kbase_pm_get_dvfs_metrics() function is used to compare these + * current values with the saved values from a previous invocation. + * @timer: timer to regularly make DVFS decisions based on the power + * management metrics. + * @timer_active: boolean indicating @timer is running + * @dvfs_last: values of the PM metrics from the last DVFS tick + * @dvfs_diff: different between the current and previous PM metrics. + */ +struct kbasep_pm_metrics_state { + ktime_t time_period_start; + bool gpu_active; + u32 active_cl_ctx[2]; + u32 active_gl_ctx[3]; + spinlock_t lock; + + void *platform_data; + struct kbase_device *kbdev; + + struct kbasep_pm_metrics values; + +#ifdef CONFIG_MALI_BIFROST_DVFS + struct hrtimer timer; + bool timer_active; + struct kbasep_pm_metrics dvfs_last; + struct kbasep_pm_metrics dvfs_diff; +#endif +}; + +/** + * struct kbasep_pm_tick_timer_state - State for the shader hysteresis timer + * @wq: Work queue to wait for the timer to stopped + * @work: Work item which cancels the timer + * @timer: Timer for powering off the shader cores + * @configured_interval: Period of GPU poweroff timer + * @configured_ticks: User-configured number of ticks to wait after the shader + * power down request is received before turning off the cores + * @remaining_ticks: Number of remaining timer ticks until shaders are powered off + * @cancel_queued: True if the cancellation work item has been queued. This is + * required to ensure that it is not queued twice, e.g. after + * a reset, which could cause the timer to be incorrectly + * cancelled later by a delayed workitem. + * @needed: Whether the timer should restart itself + */ +struct kbasep_pm_tick_timer_state { + struct workqueue_struct *wq; + struct work_struct work; + struct hrtimer timer; + + ktime_t configured_interval; + unsigned int configured_ticks; + unsigned int remaining_ticks; + + bool cancel_queued; + bool needed; +}; + +union kbase_pm_policy_data { + struct kbasep_pm_policy_always_on always_on; + struct kbasep_pm_policy_coarse_demand coarse_demand; +#if !MALI_CUSTOMER_RELEASE + struct kbasep_pm_policy_always_on_demand always_on_demand; +#endif +}; + +/** + * struct kbase_pm_backend_data - Data stored per device for power management. + * + * This structure contains data for the power management framework. There is one + * instance of this structure per device in the system. + * + * @pm_current_policy: The policy that is currently actively controlling the + * power state. + * @pm_policy_data: Private data for current PM policy + * @reset_done: Flag when a reset is complete + * @reset_done_wait: Wait queue to wait for changes to @reset_done + * @gpu_cycle_counter_requests: The reference count of active gpu cycle counter + * users + * @gpu_cycle_counter_requests_lock: Lock to protect @gpu_cycle_counter_requests + * @gpu_in_desired_state_wait: Wait queue set when the GPU is in the desired + * state according to the L2 and shader power state + * machines + * @gpu_powered: Set to true when the GPU is powered and register + * accesses are possible, false otherwise. Access to this + * variable should be protected by: both the hwaccess_lock + * spinlock and the pm.lock mutex for writes; or at least + * one of either lock for reads. + * @gpu_ready: Indicates whether the GPU is in a state in which it is + * safe to perform PM changes. When false, the PM state + * machine needs to wait before making changes to the GPU + * power policy, DevFreq or core_mask, so as to avoid these + * changing while implicit GPU resets are ongoing. + * @pm_shaders_core_mask: Shader PM state synchronised shaders core mask. It + * holds the cores enabled in a hardware counters dump, + * and may differ from @shaders_avail when under different + * states and transitions. + * @cg1_disabled: Set if the policy wants to keep the second core group + * powered off + * @driver_ready_for_irqs: Debug state indicating whether sufficient + * initialization of the driver has occurred to handle + * IRQs + * @metrics: Structure to hold metrics for the GPU + * @shader_tick_timer: Structure to hold the shader poweroff tick timer state + * @poweroff_wait_in_progress: true if a wait for GPU power off is in progress. + * hwaccess_lock must be held when accessing + * @invoke_poweroff_wait_wq_when_l2_off: flag indicating that the L2 power state + * machine should invoke the poweroff + * worker after the L2 has turned off. + * @poweron_required: true if a GPU power on is required. Should only be set + * when poweroff_wait_in_progress is true, and therefore the + * GPU can not immediately be powered on. pm.lock must be + * held when accessing + * @gpu_poweroff_wait_wq: workqueue for waiting for GPU to power off + * @gpu_poweroff_wait_work: work item for use with @gpu_poweroff_wait_wq + * @poweroff_wait: waitqueue for waiting for @gpu_poweroff_wait_work to complete + * @callback_power_on: Callback when the GPU needs to be turned on. See + * &struct kbase_pm_callback_conf + * @callback_power_off: Callback when the GPU may be turned off. See + * &struct kbase_pm_callback_conf + * @callback_power_suspend: Callback when a suspend occurs and the GPU needs to + * be turned off. See &struct kbase_pm_callback_conf + * @callback_power_resume: Callback when a resume occurs and the GPU needs to + * be turned on. See &struct kbase_pm_callback_conf + * @callback_power_runtime_on: Callback when the GPU needs to be turned on. See + * &struct kbase_pm_callback_conf + * @callback_power_runtime_off: Callback when the GPU may be turned off. See + * &struct kbase_pm_callback_conf + * @callback_power_runtime_idle: Optional callback when the GPU may be idle. See + * &struct kbase_pm_callback_conf + * @callback_soft_reset: Optional callback to software reset the GPU. See + * &struct kbase_pm_callback_conf + * @ca_cores_enabled: Cores that are currently available + * @l2_state: The current state of the L2 cache state machine. See + * &enum kbase_l2_core_state + * @l2_desired: True if the L2 cache should be powered on by the L2 cache state + * machine + * @l2_always_on: If true, disable powering down of l2 cache. + * @shaders_state: The current state of the shader state machine. + * @shaders_avail: This is updated by the state machine when it is in a state + * where it can write to the SHADER_PWRON or PWROFF registers + * to have the same set of available cores as specified by + * @shaders_desired_mask. So it would eventually have the same + * value as @shaders_desired_mask and would precisely indicate + * the cores that are currently available. This is internal to + * shader state machine and should *not* be modified elsewhere. + * @shaders_desired_mask: This is updated by the state machine when it is in + * a state where it can handle changes to the core + * availability (either by DVFS or sysfs). This is + * internal to the shader state machine and should + * *not* be modified elsewhere. + * @shaders_desired: True if the PM active count or power policy requires the + * shader cores to be on. This is used as an input to the + * shader power state machine. The current state of the + * cores may be different, but there should be transitions in + * progress that will eventually achieve this state (assuming + * that the policy doesn't change its mind in the mean time). + * @in_reset: True if a GPU is resetting and normal power manager operation is + * suspended + * @partial_shaderoff: True if we want to partial power off shader cores, + * it indicates a partial shader core off case, + * do some special operation for such case like flush + * L2 cache because of GPU2017-861 + * @protected_entry_transition_override : True if GPU reset is being used + * before entering the protected mode and so + * the reset handling behaviour is being + * overridden. + * @protected_transition_override : True if a protected mode transition is in + * progress and is overriding power manager + * behaviour. + * @protected_l2_override : Non-zero if the L2 cache is required during a + * protected mode transition. Has no effect if not + * transitioning. + * @hwcnt_desired: True if we want GPU hardware counters to be enabled. + * @hwcnt_disabled: True if GPU hardware counters are not enabled. + * @hwcnt_disable_work: Work item to disable GPU hardware counters, used if + * atomic disable is not possible. + * @gpu_clock_suspend_freq: 'opp-mali-errata-1485982' clock in opp table + * for safe L2 power cycle. + * If no opp-mali-errata-1485982 specified, + * the slowest clock will be taken. + * @gpu_clock_slow_down_wa: If true, slow down GPU clock during L2 power cycle. + * @gpu_clock_slow_down_desired: True if we want lower GPU clock + * for safe L2 power cycle. False if want GPU clock + * to back to normalized one. This is updated only + * in L2 state machine, kbase_pm_l2_update_state. + * @gpu_clock_slowed_down: During L2 power cycle, + * True if gpu clock is set at lower frequency + * for safe L2 power down, False if gpu clock gets + * restored to previous speed. This is updated only in + * work function, kbase_pm_gpu_clock_control_worker. + * @gpu_clock_control_work: work item to set GPU clock during L2 power cycle + * using gpu_clock_control + * + * Note: + * During an IRQ, @pm_current_policy can be NULL when the policy is being + * changed with kbase_pm_set_policy(). The change is protected under + * kbase_device.pm.pcower_change_lock. Direct access to this from IRQ context + * must therefore check for NULL. If NULL, then kbase_pm_set_policy() will + * re-issue the policy functions that would have been done under IRQ. + */ +struct kbase_pm_backend_data { + const struct kbase_pm_policy *pm_current_policy; + union kbase_pm_policy_data pm_policy_data; + bool reset_done; + wait_queue_head_t reset_done_wait; + int gpu_cycle_counter_requests; + spinlock_t gpu_cycle_counter_requests_lock; + + wait_queue_head_t gpu_in_desired_state_wait; + + bool gpu_powered; + bool gpu_ready; + + u64 pm_shaders_core_mask; + + bool cg1_disabled; + +#ifdef CONFIG_MALI_BIFROST_DEBUG + bool driver_ready_for_irqs; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + struct kbasep_pm_metrics_state metrics; + + struct kbasep_pm_tick_timer_state shader_tick_timer; + + bool poweroff_wait_in_progress; + bool invoke_poweroff_wait_wq_when_l2_off; + bool poweron_required; + + struct workqueue_struct *gpu_poweroff_wait_wq; + struct work_struct gpu_poweroff_wait_work; + + wait_queue_head_t poweroff_wait; + + int (*callback_power_on)(struct kbase_device *kbdev); + void (*callback_power_off)(struct kbase_device *kbdev); + void (*callback_power_suspend)(struct kbase_device *kbdev); + void (*callback_power_resume)(struct kbase_device *kbdev); + int (*callback_power_runtime_on)(struct kbase_device *kbdev); + void (*callback_power_runtime_off)(struct kbase_device *kbdev); + int (*callback_power_runtime_idle)(struct kbase_device *kbdev); + int (*callback_soft_reset)(struct kbase_device *kbdev); + + u64 ca_cores_enabled; + +#if MALI_USE_CSF + /* The current state of the micro-control unit, only applicable + * to GPUs that has such a component + */ + enum kbase_mcu_state mcu_state; +#endif + enum kbase_l2_core_state l2_state; + enum kbase_shader_core_state shaders_state; + u64 shaders_avail; + u64 shaders_desired_mask; +#if MALI_USE_CSF + /* True if the micro-control unit should be powered on */ + bool mcu_desired; +#endif + bool l2_desired; + bool l2_always_on; + bool shaders_desired; + + bool in_reset; + + bool partial_shaderoff; + + bool protected_entry_transition_override; + bool protected_transition_override; + int protected_l2_override; + + bool hwcnt_desired; + bool hwcnt_disabled; + struct work_struct hwcnt_disable_work; + + u64 gpu_clock_suspend_freq; + bool gpu_clock_slow_down_wa; + bool gpu_clock_slow_down_desired; + bool gpu_clock_slowed_down; + struct work_struct gpu_clock_control_work; +}; + + +/* List of policy IDs */ +enum kbase_pm_policy_id { + KBASE_PM_POLICY_ID_COARSE_DEMAND, +#if !MALI_CUSTOMER_RELEASE + KBASE_PM_POLICY_ID_ALWAYS_ON_DEMAND, +#endif + KBASE_PM_POLICY_ID_ALWAYS_ON +}; + +/** + * struct kbase_pm_policy - Power policy structure. + * + * Each power policy exposes a (static) instance of this structure which + * contains function pointers to the policy's methods. + * + * @name: The name of this policy + * @init: Function called when the policy is selected + * @term: Function called when the policy is unselected + * @shaders_needed: Function called to find out if shader cores are needed + * @get_core_active: Function called to get the current overall GPU power + * state + * @id: Field indicating an ID for this policy. This is not + * necessarily the same as its index in the list returned + * by kbase_pm_list_policies(). + * It is used purely for debugging. + */ +struct kbase_pm_policy { + char *name; + + /** + * Function called when the policy is selected + * + * This should initialize the kbdev->pm.pm_policy_data structure. It + * should not attempt to make any changes to hardware state. + * + * It is undefined what state the cores are in when the function is + * called. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + */ + void (*init)(struct kbase_device *kbdev); + + /** + * Function called when the policy is unselected. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + */ + void (*term)(struct kbase_device *kbdev); + + /** + * Function called to find out if shader cores are needed + * + * This needs to at least satisfy kbdev->pm.backend.shaders_desired, + * and so must never return false when shaders_desired is true. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * + * Return: true if shader cores are needed, false otherwise + */ + bool (*shaders_needed)(struct kbase_device *kbdev); + + /** + * Function called to get the current overall GPU power state + * + * This function must meet or exceed the requirements for power + * indicated by kbase_pm_is_active(). + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * + * Return: true if the GPU should be powered, false otherwise + */ + bool (*get_core_active)(struct kbase_device *kbdev); + + enum kbase_pm_policy_id id; +}; + +#endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c new file mode 100755 index 000000000000..e9e30ebadc2d --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c @@ -0,0 +1,2545 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Base kernel Power Management hardware control + */ + +#include +#include +#include +#include +#include +#include +#include +#if !MALI_USE_CSF +#include +#endif /* !MALI_USE_CSF */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + +#include + +#ifdef CONFIG_MALI_CORESTACK +bool corestack_driver_control = true; +#else +bool corestack_driver_control; /* Default value of 0/false */ +#endif +module_param(corestack_driver_control, bool, 0444); +MODULE_PARM_DESC(corestack_driver_control, + "Let the driver power on/off the GPU core stack independently " + "without involving the Power Domain Controller. This should " + "only be enabled on platforms for which integration of the PDC " + "to the Mali GPU is known to be problematic."); +KBASE_EXPORT_TEST_API(corestack_driver_control); + +/** + * enum kbasep_pm_action - Actions that can be performed on a core. + * + * This enumeration is private to the file. Its values are set to allow + * core_type_to_reg() function, which decodes this enumeration, to be simpler + * and more efficient. + * + * @ACTION_PRESENT: The cores that are present + * @ACTION_READY: The cores that are ready + * @ACTION_PWRON: Power on the cores specified + * @ACTION_PWROFF: Power off the cores specified + * @ACTION_PWRTRANS: The cores that are transitioning + * @ACTION_PWRACTIVE: The cores that are active + */ +enum kbasep_pm_action { + ACTION_PRESENT = 0, + ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO), + ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO), + ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO), + ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO), + ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO) +}; + +static u64 kbase_pm_get_state( + struct kbase_device *kbdev, + enum kbase_pm_core_type core_type, + enum kbasep_pm_action action); + +#if MALI_USE_CSF +bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (unlikely(!kbdev->csf.firmware_inited)) + return false; + + if (kbdev->csf.scheduler.pm_active_count) + return true; + + /* MCU is supposed to be ON, only when scheduler.pm_active_count is + * non zero. But for always_on policy also MCU needs to be ON. + * GPUCORE-24926 will add the proper handling for always_on + * power policy. + */ + return (kbdev->pm.backend.mcu_desired && + (kbdev->pm.backend.pm_current_policy == + &kbase_pm_always_on_policy_ops)); +} +#endif + +bool kbase_pm_is_l2_desired(struct kbase_device *kbdev) +{ + if (kbdev->pm.backend.protected_entry_transition_override) + return false; + + if (kbdev->pm.backend.protected_transition_override && + kbdev->pm.backend.protected_l2_override) + return true; + + if (kbdev->pm.backend.protected_transition_override && + !kbdev->pm.backend.shaders_desired) + return false; + + return kbdev->pm.backend.l2_desired; +} + +void kbase_pm_protected_override_enable(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->pm.backend.protected_transition_override = true; +} +void kbase_pm_protected_override_disable(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->pm.backend.protected_transition_override = false; +} + +int kbase_pm_protected_entry_override_enable(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(!kbdev->protected_mode_transition); + + if (kbdev->pm.backend.l2_always_on && + (kbdev->system_coherency == COHERENCY_ACE)) { + WARN_ON(kbdev->pm.backend.protected_entry_transition_override); + + /* + * If there is already a GPU reset pending then wait for it to + * complete before initiating a special reset for protected + * mode entry. + */ + if (kbase_reset_gpu_silent(kbdev)) + return -EAGAIN; + + kbdev->pm.backend.protected_entry_transition_override = true; + } + + return 0; +} + +void kbase_pm_protected_entry_override_disable(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(!kbdev->protected_mode_transition); + + if (kbdev->pm.backend.l2_always_on && + (kbdev->system_coherency == COHERENCY_ACE)) { + WARN_ON(!kbdev->pm.backend.protected_entry_transition_override); + + kbdev->pm.backend.protected_entry_transition_override = false; + } +} + +void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (override) { + kbdev->pm.backend.protected_l2_override++; + WARN_ON(kbdev->pm.backend.protected_l2_override <= 0); + } else { + kbdev->pm.backend.protected_l2_override--; + WARN_ON(kbdev->pm.backend.protected_l2_override < 0); + } + + kbase_pm_update_state(kbdev); +} + +/** + * core_type_to_reg - Decode a core type and action to a register. + * + * Given a core type (defined by kbase_pm_core_type) and an action (defined + * by kbasep_pm_action) this function will return the register offset that + * will perform the action on the core type. The register returned is the _LO + * register and an offset must be applied to use the _HI register. + * + * @core_type: The type of core + * @action: The type of action + * + * Return: The register offset of the _LO register that performs an action of + * type @action on a core of type @core_type. + */ +static u32 core_type_to_reg(enum kbase_pm_core_type core_type, + enum kbasep_pm_action action) +{ + if (corestack_driver_control) { + if (core_type == KBASE_PM_CORE_STACK) { + switch (action) { + case ACTION_PRESENT: + return STACK_PRESENT_LO; + case ACTION_READY: + return STACK_READY_LO; + case ACTION_PWRON: + return STACK_PWRON_LO; + case ACTION_PWROFF: + return STACK_PWROFF_LO; + case ACTION_PWRTRANS: + return STACK_PWRTRANS_LO; + default: + WARN(1, "Invalid action for core type\n"); + } + } + } + + return (u32)core_type + (u32)action; +} + +#ifdef CONFIG_ARM64 +static void mali_cci_flush_l2(struct kbase_device *kbdev) +{ + const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; + u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS; + u32 raw; + + /* + * Note that we don't take the cache flush mutex here since + * we expect to be the last user of the L2, all other L2 users + * would have dropped their references, to initiate L2 power + * down, L2 power down being the only valid place for this + * to be called from. + */ + + kbase_reg_write(kbdev, + GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAN_INV_CACHES); + + raw = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)); + + /* Wait for cache flush to complete before continuing, exit on + * gpu resets or loop expiry. */ + while (((raw & mask) == 0) && --loops) { + raw = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)); + } +} +#endif + +/** + * kbase_pm_invoke - Invokes an action on a core set + * + * This function performs the action given by @action on a set of cores of a + * type given by @core_type. It is a static function used by + * kbase_pm_transition_core_type() + * + * @kbdev: The kbase device structure of the device + * @core_type: The type of core that the action should be performed on + * @cores: A bit mask of cores to perform the action on (low 32 bits) + * @action: The action to perform on the cores + */ +static void kbase_pm_invoke(struct kbase_device *kbdev, + enum kbase_pm_core_type core_type, + u64 cores, + enum kbasep_pm_action action) +{ + u32 reg; + u32 lo = cores & 0xFFFFFFFF; + u32 hi = (cores >> 32) & 0xFFFFFFFF; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + reg = core_type_to_reg(core_type, action); + + KBASE_DEBUG_ASSERT(reg); + + if (cores) { + u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY); + + if (action == ACTION_PWRON) + state |= cores; + else if (action == ACTION_PWROFF) + state &= ~cores; + KBASE_TLSTREAM_AUX_PM_STATE(kbdev, core_type, state); + } + + /* Tracing */ + if (cores) { + if (action == ACTION_PWRON) + switch (core_type) { + case KBASE_PM_CORE_SHADER: + KBASE_KTRACE_ADD(kbdev, PM_PWRON, NULL, cores); + break; + case KBASE_PM_CORE_TILER: + KBASE_KTRACE_ADD(kbdev, PM_PWRON_TILER, NULL, cores); + break; + case KBASE_PM_CORE_L2: + KBASE_KTRACE_ADD(kbdev, PM_PWRON_L2, NULL, cores); + break; + default: + break; + } + else if (action == ACTION_PWROFF) + switch (core_type) { + case KBASE_PM_CORE_SHADER: + KBASE_KTRACE_ADD(kbdev, PM_PWROFF, NULL, cores); + break; + case KBASE_PM_CORE_TILER: + KBASE_KTRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, cores); + break; + case KBASE_PM_CORE_L2: + KBASE_KTRACE_ADD(kbdev, PM_PWROFF_L2, NULL, cores); + /* disable snoops before L2 is turned off */ + kbase_pm_cache_snoop_disable(kbdev); + break; + default: + break; + } + } + + if (kbase_dummy_job_wa_enabled(kbdev) && + action == ACTION_PWRON && + core_type == KBASE_PM_CORE_SHADER && + !(kbdev->dummy_job_wa.flags & + KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER)) { + kbase_dummy_job_wa_execute(kbdev, cores); + } else { + if (lo != 0) + kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo); + if (hi != 0) + kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi); + } +} + +/** + * kbase_pm_get_state - Get information about a core set + * + * This function gets information (chosen by @action) about a set of cores of + * a type given by @core_type. It is a static function used by + * kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and + * kbase_pm_get_ready_cores(). + * + * @kbdev: The kbase device structure of the device + * @core_type: The type of core that the should be queried + * @action: The property of the cores to query + * + * Return: A bit mask specifying the state of the cores + */ +static u64 kbase_pm_get_state(struct kbase_device *kbdev, + enum kbase_pm_core_type core_type, + enum kbasep_pm_action action) +{ + u32 reg; + u32 lo, hi; + + reg = core_type_to_reg(core_type, action); + + KBASE_DEBUG_ASSERT(reg); + + lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg)); + hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4)); + + return (((u64) hi) << 32) | ((u64) lo); +} + +/** + * kbase_pm_get_present_cores - Get the cores that are present + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of the cores that are present + */ +u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + switch (type) { + case KBASE_PM_CORE_L2: + return kbdev->gpu_props.props.raw_props.l2_present; + case KBASE_PM_CORE_SHADER: + return kbdev->gpu_props.props.raw_props.shader_present; + case KBASE_PM_CORE_TILER: + return kbdev->gpu_props.props.raw_props.tiler_present; + case KBASE_PM_CORE_STACK: + return kbdev->gpu_props.props.raw_props.stack_present; + default: + break; + } + KBASE_DEBUG_ASSERT(0); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores); + +/** + * kbase_pm_get_active_cores - Get the cores that are "active" + * (busy processing work) + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of cores that are active + */ +u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE); +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores); + +/** + * kbase_pm_get_trans_cores - Get the cores that are transitioning between + * power states + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of cores that are transitioning + */ +u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS); +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores); + +/** + * kbase_pm_get_ready_cores - Get the cores that are powered on + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of cores that are ready (powered on) + */ +u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + u64 result; + + result = kbase_pm_get_state(kbdev, type, ACTION_READY); + + switch (type) { + case KBASE_PM_CORE_SHADER: + KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED, NULL, result); + break; + case KBASE_PM_CORE_TILER: + KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, result); + break; + case KBASE_PM_CORE_L2: + KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, result); + break; + default: + break; + } + + return result; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores); + +static void kbase_pm_trigger_hwcnt_disable(struct kbase_device *kbdev) +{ + struct kbase_pm_backend_data *backend = &kbdev->pm.backend; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* See if we can get away with disabling hwcnt + * atomically, otherwise kick off a worker. + */ + if (kbase_hwcnt_context_disable_atomic(kbdev->hwcnt_gpu_ctx)) { + backend->hwcnt_disabled = true; + } else { +#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE + queue_work(system_wq, + &backend->hwcnt_disable_work); +#else + queue_work(system_highpri_wq, + &backend->hwcnt_disable_work); +#endif + } +} + +static void kbase_pm_l2_config_override(struct kbase_device *kbdev) +{ + u32 val; + + /* + * Skip if it is not supported + */ + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) + return; + + /* + * Skip if size and hash are not given explicitly, + * which means default values are used. + */ + if ((kbdev->l2_size_override == 0) && (kbdev->l2_hash_override == 0)) + return; + + val = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_CONFIG)); + + if (kbdev->l2_size_override) { + val &= ~L2_CONFIG_SIZE_MASK; + val |= (kbdev->l2_size_override << L2_CONFIG_SIZE_SHIFT); + } + + if (kbdev->l2_hash_override) { + val &= ~L2_CONFIG_HASH_MASK; + val |= (kbdev->l2_hash_override << L2_CONFIG_HASH_SHIFT); + } + + dev_dbg(kbdev->dev, "Program 0x%x to L2_CONFIG\n", val); + + /* Write L2_CONFIG to override */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_CONFIG), val); +} + +static void kbase_pm_control_gpu_clock(struct kbase_device *kbdev) +{ + struct kbase_pm_backend_data *const backend = &kbdev->pm.backend; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + queue_work(system_wq, &backend->gpu_clock_control_work); +} + +#if MALI_USE_CSF +static const char *kbase_mcu_state_to_string(enum kbase_mcu_state state) +{ + const char *const strings[] = { +#define KBASEP_MCU_STATE(n) #n, +#include "mali_kbase_pm_mcu_states.h" +#undef KBASEP_MCU_STATE + }; + if (WARN_ON((size_t)state >= ARRAY_SIZE(strings))) + return "Bad MCU state"; + else + return strings[state]; +} + +static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) +{ + struct kbase_pm_backend_data *backend = &kbdev->pm.backend; + enum kbase_mcu_state prev_state; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* + * Initial load of firmare should have been done to + * exercise the MCU state machine. + */ + if (unlikely(!kbdev->csf.firmware_inited)) { + WARN_ON(backend->mcu_state != KBASE_MCU_OFF); + return -EIO; + } + + do { + prev_state = backend->mcu_state; + + switch (backend->mcu_state) { + case KBASE_MCU_OFF: + if (kbase_pm_is_mcu_desired(kbdev) && + backend->l2_state == KBASE_L2_ON) { + kbase_csf_firmware_trigger_reload(kbdev); + backend->mcu_state = KBASE_MCU_PEND_ON_RELOAD; + } + break; + + case KBASE_MCU_PEND_ON_RELOAD: + if (kbdev->csf.firmware_reloaded) { + kbase_csf_firmware_global_reinit(kbdev); + backend->mcu_state = + KBASE_MCU_ON_GLB_REINIT_PEND; + } + break; + + case KBASE_MCU_ON_GLB_REINIT_PEND: + if (kbase_csf_firmware_global_reinit_complete(kbdev)) + backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; + break; + + case KBASE_MCU_ON_HWCNT_ENABLE: + backend->hwcnt_desired = true; + if (backend->hwcnt_disabled) { + kbase_hwcnt_context_enable( + kbdev->hwcnt_gpu_ctx); + backend->hwcnt_disabled = false; + } + backend->mcu_state = KBASE_MCU_ON; + break; + + case KBASE_MCU_ON: + if (!kbase_pm_is_mcu_desired(kbdev)) + backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; + break; + + /* ToDo. Add new state(s) if shader cores mask change for DVFS + * has to be accommodated in the MCU state machine. + */ + + case KBASE_MCU_ON_HWCNT_DISABLE: + if (kbase_pm_is_mcu_desired(kbdev)) { + backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; + break; + } + + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) + kbase_pm_trigger_hwcnt_disable(kbdev); + + if (backend->hwcnt_disabled) + backend->mcu_state = KBASE_MCU_ON_HALT; + break; + + case KBASE_MCU_ON_HALT: + if (!kbase_pm_is_mcu_desired(kbdev)) { + kbase_csf_firmware_trigger_mcu_halt(kbdev); + backend->mcu_state = KBASE_MCU_ON_PEND_HALT; + } else if (kbase_pm_is_mcu_desired(kbdev)) { + backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; + } + break; + + case KBASE_MCU_ON_PEND_HALT: + if (kbase_csf_firmware_mcu_halted(kbdev)) + backend->mcu_state = KBASE_MCU_POWER_DOWN; + break; + + case KBASE_MCU_POWER_DOWN: + kbase_csf_firmware_disable_mcu(kbdev); + backend->mcu_state = KBASE_MCU_PEND_OFF; + break; + + case KBASE_MCU_PEND_OFF: + /* wait synchronously for the MCU to get disabled */ + kbase_csf_firmware_disable_mcu_wait(kbdev); + backend->mcu_state = KBASE_MCU_OFF; + break; + + case KBASE_MCU_RESET_WAIT: + /* Reset complete */ + if (!backend->in_reset) + backend->mcu_state = KBASE_MCU_OFF; + break; + + default: + WARN(1, "Invalid state in mcu_state: %d", + backend->mcu_state); + } + + if (backend->mcu_state != prev_state) + dev_dbg(kbdev->dev, "MCU state transition: %s to %s\n", + kbase_mcu_state_to_string(prev_state), + kbase_mcu_state_to_string(backend->mcu_state)); + + } while (backend->mcu_state != prev_state); + + return 0; +} +#endif + +static const char *kbase_l2_core_state_to_string(enum kbase_l2_core_state state) +{ + const char *const strings[] = { +#define KBASEP_L2_STATE(n) #n, +#include "mali_kbase_pm_l2_states.h" +#undef KBASEP_L2_STATE + }; + if (WARN_ON((size_t)state >= ARRAY_SIZE(strings))) + return "Bad level 2 cache state"; + else + return strings[state]; +} + +static int kbase_pm_l2_update_state(struct kbase_device *kbdev) +{ + struct kbase_pm_backend_data *backend = &kbdev->pm.backend; + u64 l2_present = kbdev->gpu_props.props.raw_props.l2_present; + u64 tiler_present = kbdev->gpu_props.props.raw_props.tiler_present; + enum kbase_l2_core_state prev_state; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + do { + /* Get current state */ + u64 l2_trans = kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_L2); + u64 l2_ready = kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_L2); + u64 tiler_trans = kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_TILER); + u64 tiler_ready = kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_TILER); + + /* + * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores + * are vulnerable to corruption if gpu is lost + */ + if (kbase_is_gpu_removed(kbdev) +#ifdef CONFIG_MALI_ARBITER_SUPPORT + || kbase_pm_is_gpu_lost(kbdev)) { +#else + ) { +#endif + backend->shaders_state = + KBASE_SHADERS_OFF_CORESTACK_OFF; + backend->l2_state = KBASE_L2_OFF; + dev_dbg(kbdev->dev, "GPU lost has occurred - L2 off\n"); + break; + } + + /* mask off ready from trans in case transitions finished + * between the register reads + */ + l2_trans &= ~l2_ready; + tiler_trans &= ~tiler_ready; + + prev_state = backend->l2_state; + + switch (backend->l2_state) { + case KBASE_L2_OFF: + if (kbase_pm_is_l2_desired(kbdev)) { + /* + * Set the desired config for L2 before + * powering it on + */ + kbase_pm_l2_config_override(kbdev); + + /* L2 is required, power on. Powering on the + * tiler will also power the first L2 cache. + */ + kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, + tiler_present, ACTION_PWRON); + + /* If we have more than one L2 cache then we + * must power them on explicitly. + */ + if (l2_present != 1) + kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, + l2_present & ~1, + ACTION_PWRON); + backend->l2_state = KBASE_L2_PEND_ON; + } + break; + + case KBASE_L2_PEND_ON: + if (!l2_trans && l2_ready == l2_present && !tiler_trans + && tiler_ready == tiler_present) { + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, tiler_ready); + /* + * Ensure snoops are enabled after L2 is powered + * up. Note that kbase keeps track of the snoop + * state, so safe to repeatedly call. + */ + kbase_pm_cache_snoop_enable(kbdev); + + /* With the L2 enabled, we can now enable + * hardware counters. + */ + if (kbdev->pm.backend.gpu_clock_slow_down_wa) + backend->l2_state = + KBASE_L2_RESTORE_CLOCKS; + else + backend->l2_state = + KBASE_L2_ON_HWCNT_ENABLE; + + /* Now that the L2 is on, the shaders can start + * powering on if they're required. The obvious + * way to do this would be to call + * kbase_pm_shaders_update_state() here. + * However, that would make the two state + * machines mutually recursive, as the opposite + * would be needed for powering down. Instead, + * callers of this function should use the + * kbase_pm_update_state() wrapper, which will + * call the shader state machine immediately + * after the L2 (for power up), or + * automatically re-invoke the L2 state machine + * when the shaders power down. + */ + } + break; + + case KBASE_L2_RESTORE_CLOCKS: + /* We always assume only GPUs being affected by + * BASE_HW_ISSUE_GPU2017_1336 fall into this state + */ + WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa); + + /* If L2 not needed, we need to make sure cancellation + * of any previously issued work to restore GPU clock. + * For it, move to KBASE_L2_SLOW_DOWN_CLOCKS state. + */ + if (!kbase_pm_is_l2_desired(kbdev)) { + backend->l2_state = KBASE_L2_SLOW_DOWN_CLOCKS; + break; + } + + backend->gpu_clock_slow_down_desired = false; + if (backend->gpu_clock_slowed_down) + kbase_pm_control_gpu_clock(kbdev); + else + backend->l2_state = KBASE_L2_ON_HWCNT_ENABLE; + break; + + case KBASE_L2_ON_HWCNT_ENABLE: +#if !MALI_USE_CSF + backend->hwcnt_desired = true; + if (backend->hwcnt_disabled) { + kbase_hwcnt_context_enable( + kbdev->hwcnt_gpu_ctx); + backend->hwcnt_disabled = false; + } +#endif + backend->l2_state = KBASE_L2_ON; + break; + + case KBASE_L2_ON: + if (!kbase_pm_is_l2_desired(kbdev)) { +#if !MALI_USE_CSF + /* Do not power off L2 until the shaders and + * core stacks are off. + */ + if (backend->shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) + break; +#else + /* Do not power off L2 until the MCU has been stopped */ + if (backend->mcu_state != KBASE_MCU_OFF) + break; +#endif + + /* We need to make sure hardware counters are + * disabled before powering down the L2, to + * prevent loss of data. + * + * We waited until after the cores were powered + * down to prevent ping-ponging between hwcnt + * enabled and disabled, which would have + * happened if userspace submitted more work + * while we were trying to power down. + */ + backend->l2_state = KBASE_L2_ON_HWCNT_DISABLE; + } + break; + + case KBASE_L2_ON_HWCNT_DISABLE: +#if !MALI_USE_CSF + /* If the L2 became desired while we were waiting on the + * worker to do the actual hwcnt disable (which might + * happen if some work was submitted immediately after + * the shaders powered off), then we need to early-out + * of this state and re-enable hwcnt. + * + * If we get lucky, the hwcnt disable might not have + * actually started yet, and the logic in the hwcnt + * enable state will prevent the worker from + * performing the disable entirely, preventing loss of + * any hardware counter data. + * + * If the hwcnt disable has started, then we'll lose + * a tiny amount of hardware counter data between the + * disable and the re-enable occurring. + * + * This loss of data is preferable to the alternative, + * which is to block the shader cores from doing any + * work until we're sure hwcnt has been re-enabled. + */ + if (kbase_pm_is_l2_desired(kbdev)) { + backend->l2_state = KBASE_L2_ON_HWCNT_ENABLE; + break; + } + + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) { + kbase_pm_trigger_hwcnt_disable(kbdev); + } +#endif + + if (backend->hwcnt_disabled) { + if (kbdev->pm.backend.gpu_clock_slow_down_wa) + backend->l2_state = + KBASE_L2_SLOW_DOWN_CLOCKS; + else + backend->l2_state = KBASE_L2_POWER_DOWN; + } + break; + + case KBASE_L2_SLOW_DOWN_CLOCKS: + /* We always assume only GPUs being affected by + * BASE_HW_ISSUE_GPU2017_1336 fall into this state + */ + WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa); + + /* L2 needs to be powered up. And we need to make sure + * cancellation of any previously issued work to slow + * down GPU clock. For it, we move to the state, + * KBASE_L2_RESTORE_CLOCKS. + */ + if (kbase_pm_is_l2_desired(kbdev)) { + backend->l2_state = KBASE_L2_RESTORE_CLOCKS; + break; + } + + backend->gpu_clock_slow_down_desired = true; + if (!backend->gpu_clock_slowed_down) + kbase_pm_control_gpu_clock(kbdev); + else + backend->l2_state = KBASE_L2_POWER_DOWN; + + break; + + case KBASE_L2_POWER_DOWN: + if (!backend->l2_always_on) + /* Powering off the L2 will also power off the + * tiler. + */ + kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, + l2_present, + ACTION_PWROFF); + else + /* If L2 cache is powered then we must flush it + * before we power off the GPU. Normally this + * would have been handled when the L2 was + * powered off. + */ + kbase_gpu_start_cache_clean_nolock( + kbdev); + + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u); + + backend->l2_state = KBASE_L2_PEND_OFF; + break; + + case KBASE_L2_PEND_OFF: + if (!backend->l2_always_on) { + /* We only need to check the L2 here - if the L2 + * is off then the tiler is definitely also off. + */ + if (!l2_trans && !l2_ready) + /* L2 is now powered off */ + backend->l2_state = KBASE_L2_OFF; + } else { + if (!kbdev->cache_clean_in_progress) + backend->l2_state = KBASE_L2_OFF; + } + break; + + case KBASE_L2_RESET_WAIT: + /* Reset complete */ + if (!backend->in_reset) + backend->l2_state = KBASE_L2_OFF; + break; + + default: + WARN(1, "Invalid state in l2_state: %d", + backend->l2_state); + } + + if (backend->l2_state != prev_state) + dev_dbg(kbdev->dev, "L2 state transition: %s to %s\n", + kbase_l2_core_state_to_string(prev_state), + kbase_l2_core_state_to_string( + backend->l2_state)); + + } while (backend->l2_state != prev_state); + + if (kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off && + backend->l2_state == KBASE_L2_OFF) { + kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off = false; + queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq, + &kbdev->pm.backend.gpu_poweroff_wait_work); + } + + return 0; +} + +static void shader_poweroff_timer_stop_callback(struct work_struct *data) +{ + unsigned long flags; + struct kbasep_pm_tick_timer_state *stt = container_of(data, + struct kbasep_pm_tick_timer_state, work); + struct kbase_device *kbdev = container_of(stt, struct kbase_device, + pm.backend.shader_tick_timer); + + hrtimer_cancel(&stt->timer); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + stt->cancel_queued = false; + if (kbdev->pm.backend.gpu_powered) + kbase_pm_update_state(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +/** + * shader_poweroff_timer_queue_cancel - cancel the shader poweroff tick timer + * @kbdev: pointer to kbase device + * + * Synchronization between the shader state machine and the timer thread is + * difficult. This is because situations may arise where the state machine + * wants to start the timer, but the callback is already running, and has + * already passed the point at which it checks whether it is required, and so + * cancels itself, even though the state machine may have just tried to call + * hrtimer_start. + * + * This cannot be stopped by holding hwaccess_lock in the timer thread, + * because there are still infinitesimally small sections at the start and end + * of the callback where the lock is not held. + * + * Instead, a new state is added to the shader state machine, + * KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF. This is used to guarantee + * that when the shaders are switched off, the timer has definitely been + * cancelled. As a result, when KBASE_SHADERS_ON_CORESTACK_ON is left and the + * timer is started, it is guaranteed that either the timer is already running + * (from an availability change or cancelled timer), or hrtimer_start will + * succeed. It is critical to avoid ending up in + * KBASE_SHADERS_WAIT_OFF_CORESTACK_ON without the timer running, or it could + * hang there forever. + */ +static void shader_poweroff_timer_queue_cancel(struct kbase_device *kbdev) +{ + struct kbasep_pm_tick_timer_state *stt = + &kbdev->pm.backend.shader_tick_timer; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + stt->needed = false; + + if (hrtimer_active(&stt->timer) && !stt->cancel_queued) { + stt->cancel_queued = true; + queue_work(stt->wq, &stt->work); + } +} + +#if !MALI_USE_CSF +static const char *kbase_shader_core_state_to_string( + enum kbase_shader_core_state state) +{ + const char *const strings[] = { +#define KBASEP_SHADER_STATE(n) #n, +#include "mali_kbase_pm_shader_states.h" +#undef KBASEP_SHADER_STATE + }; + if (WARN_ON((size_t)state >= ARRAY_SIZE(strings))) + return "Bad shader core state"; + else + return strings[state]; +} + +static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) +{ + struct kbase_pm_backend_data *backend = &kbdev->pm.backend; + struct kbasep_pm_tick_timer_state *stt = + &kbdev->pm.backend.shader_tick_timer; + enum kbase_shader_core_state prev_state; + u64 stacks_avail = 0; + int err = 0; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (corestack_driver_control) + /* Always power on all the corestacks. Disabling certain + * corestacks when their respective shaders are not in the + * available bitmap is not currently supported. + */ + stacks_avail = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_STACK); + + do { + u64 shaders_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER); + u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); + u64 stacks_trans = 0; + u64 stacks_ready = 0; + + if (corestack_driver_control) { + stacks_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_STACK); + stacks_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK); + } + + /* + * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores + * are vulnerable to corruption if gpu is lost + */ + if (kbase_is_gpu_removed(kbdev) +#ifdef CONFIG_MALI_ARBITER_SUPPORT + || kbase_pm_is_gpu_lost(kbdev)) { +#else + ) { +#endif + backend->shaders_state = + KBASE_SHADERS_OFF_CORESTACK_OFF; + dev_dbg(kbdev->dev, "GPU lost has occurred - shaders off\n"); + break; + } + + /* mask off ready from trans in case transitions finished + * between the register reads + */ + shaders_trans &= ~shaders_ready; + stacks_trans &= ~stacks_ready; + + prev_state = backend->shaders_state; + + switch (backend->shaders_state) { + case KBASE_SHADERS_OFF_CORESTACK_OFF: + /* Ignore changes to the shader core availability + * except at certain points where we can handle it, + * i.e. off and SHADERS_ON_CORESTACK_ON. + */ + backend->shaders_desired_mask = + kbase_pm_ca_get_core_mask(kbdev); + backend->pm_shaders_core_mask = 0; + + if (backend->shaders_desired && + backend->l2_state == KBASE_L2_ON) { + if (backend->hwcnt_desired && + !backend->hwcnt_disabled) { + /* Trigger a hwcounter dump */ + backend->hwcnt_desired = false; + kbase_pm_trigger_hwcnt_disable(kbdev); + } + + if (backend->hwcnt_disabled) { + if (corestack_driver_control) { + kbase_pm_invoke(kbdev, + KBASE_PM_CORE_STACK, + stacks_avail, + ACTION_PWRON); + } + backend->shaders_state = + KBASE_SHADERS_OFF_CORESTACK_PEND_ON; + } + } + break; + + case KBASE_SHADERS_OFF_CORESTACK_PEND_ON: + if (!stacks_trans && stacks_ready == stacks_avail) { + backend->shaders_avail = + backend->shaders_desired_mask; + kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, + backend->shaders_avail, ACTION_PWRON); + + backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; + } + break; + + case KBASE_SHADERS_PEND_ON_CORESTACK_ON: + if (!shaders_trans && shaders_ready == backend->shaders_avail) { + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, shaders_ready); + backend->pm_shaders_core_mask = shaders_ready; + backend->hwcnt_desired = true; + if (backend->hwcnt_disabled) { + kbase_hwcnt_context_enable( + kbdev->hwcnt_gpu_ctx); + backend->hwcnt_disabled = false; + } + + backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON; + } + break; + + case KBASE_SHADERS_ON_CORESTACK_ON: + backend->shaders_desired_mask = + kbase_pm_ca_get_core_mask(kbdev); + + /* If shaders to change state, trigger a counter dump */ + if (!backend->shaders_desired || + (backend->shaders_desired_mask != shaders_ready)) { + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) + kbase_pm_trigger_hwcnt_disable(kbdev); + backend->shaders_state = + KBASE_SHADERS_ON_CORESTACK_ON_RECHECK; + } + break; + + case KBASE_SHADERS_ON_CORESTACK_ON_RECHECK: + backend->shaders_desired_mask = + kbase_pm_ca_get_core_mask(kbdev); + + if (!backend->hwcnt_disabled) { + /* Wait for being disabled */ + ; + } else if (!backend->shaders_desired) { + if (kbdev->pm.backend.protected_transition_override || +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbase_pm_is_suspending(kbdev) || + kbase_pm_is_gpu_lost(kbdev) || +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + !stt->configured_ticks || + WARN_ON(stt->cancel_queued)) { + backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; + } else { + stt->remaining_ticks = stt->configured_ticks; + stt->needed = true; + + /* The shader hysteresis timer is not + * done the obvious way, which would be + * to start an hrtimer when the shader + * power off is requested. Instead, + * use a 'tick' timer, and set the + * remaining number of ticks on a power + * off request. This avoids the + * latency of starting, then + * immediately cancelling an hrtimer + * when the shaders are re-requested + * before the timeout expires. + */ + if (!hrtimer_active(&stt->timer)) + hrtimer_start(&stt->timer, + stt->configured_interval, + HRTIMER_MODE_REL); + + backend->shaders_state = KBASE_SHADERS_WAIT_OFF_CORESTACK_ON; + } + } else if (backend->shaders_desired_mask & ~shaders_ready) { + /* set cores ready but not available to + * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON + * check pass + */ + backend->shaders_avail = + (backend->shaders_desired_mask | shaders_ready); + + kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, + backend->shaders_avail & ~shaders_ready, + ACTION_PWRON); + backend->shaders_state = + KBASE_SHADERS_PEND_ON_CORESTACK_ON; + } else if (shaders_ready & ~backend->shaders_desired_mask) { + backend->shaders_state = + KBASE_SHADERS_WAIT_GPU_IDLE; + } else { + backend->shaders_state = + KBASE_SHADERS_PEND_ON_CORESTACK_ON; + } + break; + + case KBASE_SHADERS_WAIT_OFF_CORESTACK_ON: + if (WARN_ON(!hrtimer_active(&stt->timer))) { + stt->remaining_ticks = 0; + backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; + } + + if (backend->shaders_desired) { + stt->remaining_ticks = 0; + backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK; + } else if (stt->remaining_ticks == 0) { + backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; +#ifdef CONFIG_MALI_ARBITER_SUPPORT + } else if (kbase_pm_is_suspending(kbdev) || + kbase_pm_is_gpu_lost(kbdev)) { + backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + } + break; + + case KBASE_SHADERS_WAIT_GPU_IDLE: + /* If partial shader core off need to wait the job in + * running and next register finished then flush L2 + * or it might hit GPU2017-861 + */ + if (!kbase_gpu_atoms_submitted_any(kbdev)) { + backend->partial_shaderoff = true; + backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; + } + break; + + case KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON: + shader_poweroff_timer_queue_cancel(kbdev); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) { + kbase_gpu_start_cache_clean_nolock(kbdev); + backend->shaders_state = + KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON; + } else { + backend->shaders_state = + KBASE_SHADERS_READY_OFF_CORESTACK_ON; + } + break; + + case KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON: + if (!kbdev->cache_clean_in_progress) + backend->shaders_state = + KBASE_SHADERS_READY_OFF_CORESTACK_ON; + + break; + + case KBASE_SHADERS_READY_OFF_CORESTACK_ON: + if (backend->partial_shaderoff) { + backend->partial_shaderoff = false; + /* remove cores available but not ready to + * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON + * check pass + */ + + /* shaders_desired_mask shall be a subset of + * shaders_ready + */ + WARN_ON(backend->shaders_desired_mask & ~shaders_ready); + WARN_ON(!(backend->shaders_desired_mask & shaders_ready)); + + backend->shaders_avail = + backend->shaders_desired_mask; + kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, + shaders_ready & ~backend->shaders_avail, ACTION_PWROFF); + backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, (shaders_ready & ~backend->shaders_avail)); + } else { + kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, + shaders_ready, ACTION_PWROFF); + + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u); + + backend->shaders_state = KBASE_SHADERS_PEND_OFF_CORESTACK_ON; + } + break; + + case KBASE_SHADERS_PEND_OFF_CORESTACK_ON: + if (!shaders_trans && !shaders_ready) { + if (corestack_driver_control) + kbase_pm_invoke(kbdev, KBASE_PM_CORE_STACK, + stacks_avail, ACTION_PWROFF); + + backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_PEND_OFF; + } + break; + + case KBASE_SHADERS_OFF_CORESTACK_PEND_OFF: + if (!stacks_trans && !stacks_ready) { + /* On powered off, re-enable the hwcnt */ + backend->pm_shaders_core_mask = 0; + backend->hwcnt_desired = true; + if (backend->hwcnt_disabled) { + kbase_hwcnt_context_enable( + kbdev->hwcnt_gpu_ctx); + backend->hwcnt_disabled = false; + } + backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF; + } + break; + + case KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF: + if (!hrtimer_active(&stt->timer) && !stt->cancel_queued) + backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; + break; + + case KBASE_SHADERS_RESET_WAIT: + /* Reset complete */ + if (!backend->in_reset) + backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF; + break; + } + + if (backend->shaders_state != prev_state) + dev_dbg(kbdev->dev, "Shader state transition: %s to %s\n", + kbase_shader_core_state_to_string(prev_state), + kbase_shader_core_state_to_string( + backend->shaders_state)); + + } while (backend->shaders_state != prev_state); + + return err; +} +#endif + +static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev) +{ + bool in_desired_state = true; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbase_pm_is_l2_desired(kbdev) && + kbdev->pm.backend.l2_state != KBASE_L2_ON) + in_desired_state = false; + else if (!kbase_pm_is_l2_desired(kbdev) && + kbdev->pm.backend.l2_state != KBASE_L2_OFF) + in_desired_state = false; + +#if !MALI_USE_CSF + if (kbdev->pm.backend.shaders_desired && + kbdev->pm.backend.shaders_state != KBASE_SHADERS_ON_CORESTACK_ON) + in_desired_state = false; + else if (!kbdev->pm.backend.shaders_desired && + kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) + in_desired_state = false; +#else + if (kbase_pm_is_mcu_desired(kbdev) && + kbdev->pm.backend.mcu_state != KBASE_MCU_ON) + in_desired_state = false; + else if (!kbase_pm_is_mcu_desired(kbdev) && + kbdev->pm.backend.mcu_state != KBASE_MCU_OFF) + in_desired_state = false; +#endif + + return in_desired_state; +} + +static bool kbase_pm_is_in_desired_state(struct kbase_device *kbdev) +{ + bool in_desired_state; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + in_desired_state = kbase_pm_is_in_desired_state_nolock(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return in_desired_state; +} + +static bool kbase_pm_is_in_desired_state_with_l2_powered( + struct kbase_device *kbdev) +{ + bool in_desired_state = false; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (kbase_pm_is_in_desired_state_nolock(kbdev) && + (kbdev->pm.backend.l2_state == KBASE_L2_ON)) + in_desired_state = true; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return in_desired_state; +} + +static void kbase_pm_trace_power_state(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + KBASE_TLSTREAM_AUX_PM_STATE( + kbdev, + KBASE_PM_CORE_L2, + kbase_pm_get_ready_cores( + kbdev, KBASE_PM_CORE_L2)); + KBASE_TLSTREAM_AUX_PM_STATE( + kbdev, + KBASE_PM_CORE_SHADER, + kbase_pm_get_ready_cores( + kbdev, KBASE_PM_CORE_SHADER)); + KBASE_TLSTREAM_AUX_PM_STATE( + kbdev, + KBASE_PM_CORE_TILER, + kbase_pm_get_ready_cores( + kbdev, + KBASE_PM_CORE_TILER)); + + if (corestack_driver_control) + KBASE_TLSTREAM_AUX_PM_STATE( + kbdev, + KBASE_PM_CORE_STACK, + kbase_pm_get_ready_cores( + kbdev, + KBASE_PM_CORE_STACK)); +} + +void kbase_pm_update_state(struct kbase_device *kbdev) +{ +#if !MALI_USE_CSF + enum kbase_shader_core_state prev_shaders_state = + kbdev->pm.backend.shaders_state; +#else + enum kbase_mcu_state prev_mcu_state = kbdev->pm.backend.mcu_state; +#endif + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!kbdev->pm.backend.gpu_ready) + return; /* Do nothing if the GPU is not ready */ + + if (kbase_pm_l2_update_state(kbdev)) + return; + +#if !MALI_USE_CSF + if (kbase_pm_shaders_update_state(kbdev)) + return; + + /* If the shaders just turned off, re-invoke the L2 state machine, in + * case it was waiting for the shaders to turn off before powering down + * the L2. + */ + if (prev_shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF && + kbdev->pm.backend.shaders_state == + KBASE_SHADERS_OFF_CORESTACK_OFF) { + if (kbase_pm_l2_update_state(kbdev)) + return; + } +#else + if (kbase_pm_mcu_update_state(kbdev)) + return; + + if (prev_mcu_state != KBASE_MCU_OFF && + kbdev->pm.backend.mcu_state == KBASE_MCU_OFF) { + if (kbase_pm_l2_update_state(kbdev)) + return; + } +#endif + + if (kbase_pm_is_in_desired_state_nolock(kbdev)) { + KBASE_KTRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, + kbdev->pm.backend.shaders_avail); + + kbase_pm_trace_power_state(kbdev); + + KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0); + wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); + } +} + +static enum hrtimer_restart +shader_tick_timer_callback(struct hrtimer *timer) +{ + struct kbasep_pm_tick_timer_state *stt = container_of(timer, + struct kbasep_pm_tick_timer_state, timer); + struct kbase_device *kbdev = container_of(stt, struct kbase_device, + pm.backend.shader_tick_timer); + struct kbase_pm_backend_data *backend = &kbdev->pm.backend; + unsigned long flags; + enum hrtimer_restart restart = HRTIMER_NORESTART; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (stt->remaining_ticks && + backend->shaders_state == KBASE_SHADERS_WAIT_OFF_CORESTACK_ON) { + stt->remaining_ticks--; + + /* If the remaining ticks just changed from 1 to 0, invoke the + * PM state machine to power off the shader cores. + */ + if (!stt->remaining_ticks && !backend->shaders_desired) + kbase_pm_update_state(kbdev); + } + + if (stt->needed) { + hrtimer_forward_now(timer, stt->configured_interval); + restart = HRTIMER_RESTART; + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return restart; +} + +int kbase_pm_state_machine_init(struct kbase_device *kbdev) +{ + struct kbasep_pm_tick_timer_state *stt = &kbdev->pm.backend.shader_tick_timer; + + stt->wq = alloc_workqueue("kbase_pm_shader_poweroff", WQ_HIGHPRI | WQ_UNBOUND, 1); + if (!stt->wq) + return -ENOMEM; + + INIT_WORK(&stt->work, shader_poweroff_timer_stop_callback); + + stt->needed = false; + hrtimer_init(&stt->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + stt->timer.function = shader_tick_timer_callback; + stt->configured_interval = HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS); + stt->configured_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; + + return 0; +} + +void kbase_pm_state_machine_term(struct kbase_device *kbdev) +{ + hrtimer_cancel(&kbdev->pm.backend.shader_tick_timer.timer); + destroy_workqueue(kbdev->pm.backend.shader_tick_timer.wq); +} + +void kbase_pm_reset_start_locked(struct kbase_device *kbdev) +{ + struct kbase_pm_backend_data *backend = &kbdev->pm.backend; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + backend->in_reset = true; + backend->l2_state = KBASE_L2_RESET_WAIT; +#if !MALI_USE_CSF + backend->shaders_state = KBASE_SHADERS_RESET_WAIT; +#else + /* MCU state machine is exercised only after the initial load/boot + * of the firmware. + */ + if (likely(kbdev->csf.firmware_inited)) { + backend->mcu_state = KBASE_MCU_RESET_WAIT; + kbdev->csf.firmware_reload_needed = true; + } else { + WARN_ON(backend->mcu_state != KBASE_MCU_OFF); + } +#endif + + /* We're in a reset, so hwcnt will have been synchronously disabled by + * this function's caller as part of the reset process. We therefore + * know that any call to kbase_hwcnt_context_disable_atomic, if + * required to sync the hwcnt refcount with our internal state, is + * guaranteed to succeed. + */ + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) { + WARN_ON(!kbase_hwcnt_context_disable_atomic( + kbdev->hwcnt_gpu_ctx)); + backend->hwcnt_disabled = true; + } + + shader_poweroff_timer_queue_cancel(kbdev); +} + +void kbase_pm_reset_complete(struct kbase_device *kbdev) +{ + struct kbase_pm_backend_data *backend = &kbdev->pm.backend; + unsigned long flags; + + WARN_ON(!kbase_reset_gpu_is_active(kbdev)); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* As GPU has just been reset, that results in implicit flush of L2 + * cache, can safely mark the pending cache flush operation (if there + * was any) as complete and unblock the waiter. + * No work can be submitted whilst GPU reset is ongoing. + */ + kbase_gpu_cache_clean_wait_complete(kbdev); + backend->in_reset = false; + kbase_pm_update_state(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +/* Timeout for kbase_pm_wait_for_desired_state when wait_event_killable has + * aborted due to a fatal signal. If the time spent waiting has exceeded this + * threshold then there is most likely a hardware issue. */ +#define PM_TIMEOUT_MS (5000) /* 5s */ + +static void kbase_pm_timed_out(struct kbase_device *kbdev) +{ + unsigned long flags; + + dev_err(kbdev->dev, "Power transition timed out unexpectedly\n"); +#if !MALI_USE_CSF + CSTD_UNUSED(flags); + dev_err(kbdev->dev, "Desired state :\n"); + dev_err(kbdev->dev, "\tShader=%016llx\n", + kbdev->pm.backend.shaders_desired ? kbdev->pm.backend.shaders_avail : 0); +#else + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + dev_err(kbdev->dev, "\tMCU desired = %d\n", + kbase_pm_is_mcu_desired(kbdev)); + dev_err(kbdev->dev, "\tMCU sw state = %d\n", + kbdev->pm.backend.mcu_state); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#endif + dev_err(kbdev->dev, "Current state :\n"); + dev_err(kbdev->dev, "\tShader=%08x%08x\n", + kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_READY_HI)), + kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_READY_LO))); + dev_err(kbdev->dev, "\tTiler =%08x%08x\n", + kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_READY_HI)), + kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_READY_LO))); + dev_err(kbdev->dev, "\tL2 =%08x%08x\n", + kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_READY_HI)), + kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_READY_LO))); +#if MALI_USE_CSF + dev_err(kbdev->dev, "\tMCU status = %d\n", + kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS))); +#endif + dev_err(kbdev->dev, "Cores transitioning :\n"); + dev_err(kbdev->dev, "\tShader=%08x%08x\n", + kbase_reg_read(kbdev, GPU_CONTROL_REG( + SHADER_PWRTRANS_HI)), + kbase_reg_read(kbdev, GPU_CONTROL_REG( + SHADER_PWRTRANS_LO))); + dev_err(kbdev->dev, "\tTiler =%08x%08x\n", + kbase_reg_read(kbdev, GPU_CONTROL_REG( + TILER_PWRTRANS_HI)), + kbase_reg_read(kbdev, GPU_CONTROL_REG( + TILER_PWRTRANS_LO))); + dev_err(kbdev->dev, "\tL2 =%08x%08x\n", + kbase_reg_read(kbdev, GPU_CONTROL_REG( + L2_PWRTRANS_HI)), + kbase_reg_read(kbdev, GPU_CONTROL_REG( + L2_PWRTRANS_LO))); + + dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n"); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); +} + +void kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev) +{ + unsigned long flags; + unsigned long timeout; + int err; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + timeout = jiffies + msecs_to_jiffies(PM_TIMEOUT_MS); + + /* Wait for cores */ + err = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait, + kbase_pm_is_in_desired_state_with_l2_powered(kbdev)); + + if (err < 0 && time_after(jiffies, timeout)) + kbase_pm_timed_out(kbdev); +} + +int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev) +{ + unsigned long flags; + long remaining; +#if MALI_USE_CSF + long timeout = kbase_csf_timeout_in_jiffies(PM_TIMEOUT_MS); +#else + long timeout = msecs_to_jiffies(PM_TIMEOUT_MS); +#endif + int err = 0; + + /* Let the state machine latch the most recent desired state. */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Wait for cores */ +#if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE + remaining = wait_event_killable_timeout( + kbdev->pm.backend.gpu_in_desired_state_wait, + kbase_pm_is_in_desired_state(kbdev), timeout); +#else + remaining = wait_event_timeout( + kbdev->pm.backend.gpu_in_desired_state_wait, + kbase_pm_is_in_desired_state(kbdev), timeout); +#endif + + if (!remaining) { + kbase_pm_timed_out(kbdev); + err = -ETIMEDOUT; + } else if (remaining < 0) { + dev_info(kbdev->dev, + "Wait for desired PM state got interrupted"); + err = (int)remaining; + } + + return err; +} +KBASE_EXPORT_TEST_API(kbase_pm_wait_for_desired_state); + +void kbase_pm_enable_interrupts(struct kbase_device *kbdev) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + /* + * Clear all interrupts, + * and unmask them all. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF); +#if MALI_USE_CSF + /* Enable only the Page fault bits part */ + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFF); +#else + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF); +#endif +} + +KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); + +void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(NULL != kbdev); + /* + * Mask all interrupts, + * and clear them all. + */ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0); + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF); +} + +void kbase_pm_disable_interrupts(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_disable_interrupts_nolock(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts); + +/* + * pmu layout: + * 0x0000: PMU TAG (RO) (0xCAFECAFE) + * 0x0004: PMU VERSION ID (RO) (0x00000000) + * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE) + */ +void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) +{ + bool reset_required = is_resume; + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); +#if !MALI_USE_CSF + lockdep_assert_held(&kbdev->js_data.runpool_mutex); +#endif /* !MALI_USE_CSF */ + lockdep_assert_held(&kbdev->pm.lock); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) { + dev_err(kbdev->dev, + "%s: Cannot power up while GPU lost", __func__); + return; + } +#endif + + if (kbdev->pm.backend.gpu_powered) { + /* Already turned on */ + if (kbdev->poweroff_pending) + kbase_pm_enable_interrupts(kbdev); + kbdev->poweroff_pending = false; + KBASE_DEBUG_ASSERT(!is_resume); + return; + } + + kbdev->poweroff_pending = false; + + KBASE_KTRACE_ADD(kbdev, PM_GPU_ON, NULL, 0u); + + if (is_resume && kbdev->pm.backend.callback_power_resume) { + kbdev->pm.backend.callback_power_resume(kbdev); + return; + } else if (kbdev->pm.backend.callback_power_on) { + reset_required = kbdev->pm.backend.callback_power_on(kbdev); + } + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.gpu_powered = true; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (reset_required) { + /* GPU state was lost, reset GPU to ensure it is in a + * consistent state */ + kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); + } +#ifdef CONFIG_MALI_ARBITER_SUPPORT + else { + struct kbase_arbiter_vm_state *arb_vm_state = + kbdev->pm.arb_vm_state; + + /* In the case that the GPU has just been granted by + * the Arbiter, a reset will have already been done. + * However, it is still necessary to initialize the GPU. + */ + if (arb_vm_state->vm_arb_starting) + kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS | + PM_NO_RESET); + } +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_restore_all_as(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + if (kbdev->dummy_job_wa.flags & + KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) { + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_dummy_job_wa_execute(kbdev, + kbase_pm_get_present_cores(kbdev, + KBASE_PM_CORE_SHADER)); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + /* Enable the interrupts */ + kbase_pm_enable_interrupts(kbdev); + + /* Turn on the L2 caches */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.gpu_ready = true; + kbdev->pm.backend.l2_desired = true; + kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +KBASE_EXPORT_TEST_API(kbase_pm_clock_on); + +bool kbase_pm_clock_off(struct kbase_device *kbdev) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->pm.lock); + + /* ASSERT that the cores should now be unavailable. No lock needed. */ + WARN_ON(kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF); + + kbdev->poweroff_pending = true; + + if (!kbdev->pm.backend.gpu_powered) { + /* Already turned off */ + return true; + } + + KBASE_KTRACE_ADD(kbdev, PM_GPU_OFF, NULL, 0u); + + /* Disable interrupts. This also clears any outstanding interrupts */ + kbase_pm_disable_interrupts(kbdev); + /* Ensure that any IRQ handlers have finished */ + kbase_synchronize_irqs(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (atomic_read(&kbdev->faults_pending)) { + /* Page/bus faults are still being processed. The GPU can not + * be powered off until they have completed */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return false; + } + + kbase_pm_cache_snoop_disable(kbdev); + + kbdev->pm.backend.gpu_ready = false; + + /* The GPU power may be turned off from this point */ + kbdev->pm.backend.gpu_powered = false; + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_pm_is_gpu_lost(kbdev)) { + /* Ensure we unblock any threads that are stuck waiting + * for the GPU + */ + kbase_gpu_cache_clean_wait_complete(kbdev); + } +#endif + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + + if (kbdev->pm.backend.callback_power_off) + kbdev->pm.backend.callback_power_off(kbdev); + return true; +} + +KBASE_EXPORT_TEST_API(kbase_pm_clock_off); + +struct kbasep_reset_timeout_data { + struct hrtimer timer; + bool timed_out; + struct kbase_device *kbdev; +}; + +void kbase_pm_reset_done(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + kbdev->pm.backend.reset_done = true; + wake_up(&kbdev->pm.backend.reset_done_wait); +} + +/** + * kbase_pm_wait_for_reset - Wait for a reset to happen + * + * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state. + * + * @kbdev: Kbase device + */ +static void kbase_pm_wait_for_reset(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->pm.lock); + + wait_event(kbdev->pm.backend.reset_done_wait, + (kbdev->pm.backend.reset_done)); + kbdev->pm.backend.reset_done = false; +} + +KBASE_EXPORT_TEST_API(kbase_pm_reset_done); + +static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) +{ + struct kbasep_reset_timeout_data *rtdata = + container_of(timer, struct kbasep_reset_timeout_data, timer); + + rtdata->timed_out = 1; + + /* Set the wait queue to wake up kbase_pm_init_hw even though the reset + * hasn't completed */ + kbase_pm_reset_done(rtdata->kbdev); + + return HRTIMER_NORESTART; +} + +static int kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id) +{ +#if MALI_USE_CSF + kbdev->hw_quirks_jm = kbase_reg_read(kbdev, + GPU_CONTROL_REG(CSF_CONFIG)); +#else + u32 hw_quirks_jm = kbase_reg_read(kbdev, + GPU_CONTROL_REG(JM_CONFIG)); + + if (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == GPU_ID2_PRODUCT_TMIX) { + /* Only for tMIx */ + u32 coherency_features; + + coherency_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(COHERENCY_FEATURES)); + + /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly + * documented for tMIx so force correct value here. + */ + if (coherency_features == + COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { + hw_quirks_jm |= (COHERENCY_ACE_LITE | + COHERENCY_ACE) << + JM_FORCE_COHERENCY_FEATURES_SHIFT; + } + } + + if (kbase_is_gpu_removed(kbdev)) + return -EIO; + + kbdev->hw_quirks_jm = hw_quirks_jm; + +#endif /* !MALI_USE_CSF */ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) { + int default_idvs_group_size = 0xF; + u32 tmp; + + if (of_property_read_u32(kbdev->dev->of_node, + "idvs-group-size", &tmp)) + tmp = default_idvs_group_size; + + if (tmp > IDVS_GROUP_MAX_SIZE) { + dev_err(kbdev->dev, + "idvs-group-size of %d is too large. Maximum value is %d", + tmp, IDVS_GROUP_MAX_SIZE); + tmp = default_idvs_group_size; + } + + kbdev->hw_quirks_jm |= tmp << IDVS_GROUP_SIZE_SHIFT; + } + +#define MANUAL_POWER_CONTROL ((u32)(1 << 8)) + if (corestack_driver_control) + kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL; + + return 0; +} + +static int kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id) +{ + u32 hw_quirks_sc = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_CONFIG)); + + if (kbase_is_gpu_removed(kbdev)) + return -EIO; + + if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ + hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; + else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */ + hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_2968_TTRX_3162)) + hw_quirks_sc |= SC_VAR_ALGORITHM; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_TLS_HASHING)) + hw_quirks_sc |= SC_TLS_HASH_ENABLE; + + kbdev->hw_quirks_sc = hw_quirks_sc; + + return 0; +} + +static int kbase_set_tiler_quirks(struct kbase_device *kbdev) +{ + u32 hw_quirks_tiler = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_CONFIG)); + + if (kbase_is_gpu_removed(kbdev)) + return -EIO; + + /* Set tiler clock gate override if required */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) + hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; + + kbdev->hw_quirks_tiler = hw_quirks_tiler; + + return 0; +} + +static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) +{ + struct device_node *np = kbdev->dev->of_node; + const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT; + int error = 0; + + kbdev->hw_quirks_jm = 0; + kbdev->hw_quirks_sc = 0; + kbdev->hw_quirks_tiler = 0; + kbdev->hw_quirks_mmu = 0; + + if (!of_property_read_u32(np, "quirks_jm", + &kbdev->hw_quirks_jm)) { + dev_info(kbdev->dev, + "Found quirks_jm = [0x%x] in Devicetree\n", + kbdev->hw_quirks_jm); + } else { + error = kbase_set_jm_quirks(kbdev, prod_id); + if (error) + return error; + } + + if (!of_property_read_u32(np, "quirks_sc", + &kbdev->hw_quirks_sc)) { + dev_info(kbdev->dev, + "Found quirks_sc = [0x%x] in Devicetree\n", + kbdev->hw_quirks_sc); + } else { + error = kbase_set_sc_quirks(kbdev, prod_id); + if (error) + return error; + } + + if (!of_property_read_u32(np, "quirks_tiler", + &kbdev->hw_quirks_tiler)) { + dev_info(kbdev->dev, + "Found quirks_tiler = [0x%x] in Devicetree\n", + kbdev->hw_quirks_tiler); + } else { + error = kbase_set_tiler_quirks(kbdev); + if (error) + return error; + } + + if (!of_property_read_u32(np, "quirks_mmu", + &kbdev->hw_quirks_mmu)) { + dev_info(kbdev->dev, + "Found quirks_mmu = [0x%x] in Devicetree\n", + kbdev->hw_quirks_mmu); + } else { + error = kbase_set_mmu_quirks(kbdev); + } + + return error; +} + +static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) +{ + kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), + kbdev->hw_quirks_sc); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG), + kbdev->hw_quirks_tiler); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), + kbdev->hw_quirks_mmu); +#if MALI_USE_CSF + kbase_reg_write(kbdev, GPU_CONTROL_REG(CSF_CONFIG), + kbdev->hw_quirks_jm); +#else + kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG), + kbdev->hw_quirks_jm); +#endif +} + +void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) +{ + if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && + !kbdev->cci_snoop_enabled) { +#ifdef CONFIG_ARM64 + if (kbdev->snoop_enable_smc != 0) + kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0); +#endif /* CONFIG_ARM64 */ + dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n"); + kbdev->cci_snoop_enabled = true; + } +} + +void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) +{ + if (kbdev->cci_snoop_enabled) { +#ifdef CONFIG_ARM64 + if (kbdev->snoop_disable_smc != 0) { + mali_cci_flush_l2(kbdev); + kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0); + } +#endif /* CONFIG_ARM64 */ + dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n"); + kbdev->cci_snoop_enabled = false; + } +} + +static void reenable_protected_mode_hwcnt(struct kbase_device *kbdev) +{ + unsigned long irq_flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + kbdev->protected_mode_hwcnt_desired = true; + if (kbdev->protected_mode_hwcnt_disabled) { + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbdev->protected_mode_hwcnt_disabled = false; + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); +} + +static int kbase_pm_do_reset(struct kbase_device *kbdev) +{ + struct kbasep_reset_timeout_data rtdata; + int ret; + + KBASE_KTRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, 0); + + KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev); + + if (kbdev->pm.backend.callback_soft_reset) { + ret = kbdev->pm.backend.callback_soft_reset(kbdev); + if (ret < 0) + return ret; + else if (ret > 0) + return 0; + } else { + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_SOFT_RESET); + } + + /* Unmask the reset complete interrupt only */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED); + + /* Initialize a structure for tracking the status of the reset */ + rtdata.kbdev = kbdev; + rtdata.timed_out = 0; + + /* Create a timer to use as a timeout on the reset */ + hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rtdata.timer.function = kbasep_reset_timeout; + + hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), + HRTIMER_MODE_REL); + + /* Wait for the RESET_COMPLETED interrupt to be raised */ + kbase_pm_wait_for_reset(kbdev); + + if (rtdata.timed_out == 0) { + /* GPU has been reset */ + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + return 0; + } + + /* No interrupt has been received - check if the RAWSTAT register says + * the reset has completed */ + if ((kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) & + RESET_COMPLETED)) { + /* The interrupt is set in the RAWSTAT; this suggests that the + * interrupts are not getting to the CPU */ + dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); + /* If interrupts aren't working we can't continue. */ + destroy_hrtimer_on_stack(&rtdata.timer); + return -EINVAL; + } + + if (kbase_is_gpu_removed(kbdev)) { + dev_dbg(kbdev->dev, "GPU has been removed, reset no longer needed.\n"); + destroy_hrtimer_on_stack(&rtdata.timer); + return -EINVAL; + } + + /* The GPU doesn't seem to be responding to the reset so try a hard + * reset */ + dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", + RESET_TIMEOUT); + KBASE_KTRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_HARD_RESET); + + /* Restart the timer to wait for the hard reset to complete */ + rtdata.timed_out = 0; + + hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), + HRTIMER_MODE_REL); + + /* Wait for the RESET_COMPLETED interrupt to be raised */ + kbase_pm_wait_for_reset(kbdev); + + if (rtdata.timed_out == 0) { + /* GPU has been reset */ + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + return 0; + } + + destroy_hrtimer_on_stack(&rtdata.timer); + + dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", + RESET_TIMEOUT); + + return -EINVAL; +} + +int kbase_pm_protected_mode_enable(struct kbase_device *const kbdev) +{ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_SET_PROTECTED_MODE); + return 0; +} + +int kbase_pm_protected_mode_disable(struct kbase_device *const kbdev) +{ + lockdep_assert_held(&kbdev->pm.lock); + + return kbase_pm_do_reset(kbdev); +} + +int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) +{ + unsigned long irq_flags; + int err = 0; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->pm.lock); + + /* Ensure the clock is on before attempting to access the hardware */ + if (!kbdev->pm.backend.gpu_powered) { + if (kbdev->pm.backend.callback_power_on) + kbdev->pm.backend.callback_power_on(kbdev); + + kbdev->pm.backend.gpu_powered = true; + } + + /* Ensure interrupts are off to begin with, this also clears any + * outstanding interrupts */ + kbase_pm_disable_interrupts(kbdev); + /* Ensure cache snoops are disabled before reset. */ + kbase_pm_cache_snoop_disable(kbdev); + /* Prepare for the soft-reset */ + kbdev->pm.backend.reset_done = false; + + /* The cores should be made unavailable due to the reset */ + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + if (kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); + + /* Soft reset the GPU */ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (!(flags & PM_NO_RESET)) +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + err = kbdev->protected_ops->protected_mode_disable( + kbdev->protected_dev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + kbdev->protected_mode = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); + + if (err) + goto exit; + + if (flags & PM_HW_ISSUES_DETECT) { + err = kbase_pm_hw_issues_detect(kbdev); + if (err) + goto exit; + } + + kbase_pm_hw_issues_apply(kbdev); + kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); + + /* Sanity check protected mode was left after reset */ + WARN_ON(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) & + GPU_STATUS_PROTECTED_MODE_ACTIVE); + + /* If cycle counter was in use re-enable it, enable_irqs will only be + * false when called from kbase_pm_powerup */ + if (kbdev->pm.backend.gpu_cycle_counter_requests && + (flags & PM_ENABLE_IRQS)) { + kbase_pm_enable_interrupts(kbdev); + + /* Re-enable the counters if we need to */ + spin_lock_irqsave( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + if (kbdev->pm.backend.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CYCLE_COUNT_START); + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + + kbase_pm_disable_interrupts(kbdev); + } + + if (flags & PM_ENABLE_IRQS) + kbase_pm_enable_interrupts(kbdev); + +exit: + if (!kbdev->pm.backend.protected_entry_transition_override) { + /* Re-enable GPU hardware counters if we're resetting from + * protected mode. + */ + reenable_protected_mode_hwcnt(kbdev); + } + + return err; +} + +/** + * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters + * + * Increase the count of cycle counter users and turn the cycle counters on if + * they were previously off + * + * This function is designed to be called by + * kbase_pm_request_gpu_cycle_counter() or + * kbase_pm_request_gpu_cycle_counter_l2_is_on() only + * + * When this function is called the l2 cache must be on - i.e., the GPU must be + * on. + * + * @kbdev: The kbase device structure of the device + */ +static void +kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); + + ++kbdev->pm.backend.gpu_cycle_counter_requests; + + if (1 == kbdev->pm.backend.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CYCLE_COUNT_START); + + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); +} + +void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < + INT_MAX); + + kbase_pm_request_gpu_cycle_counter_do_request(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter); + +void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < + INT_MAX); + + kbase_pm_request_gpu_cycle_counter_do_request(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on); + +void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0); + + --kbdev->pm.backend.gpu_cycle_counter_requests; + + if (0 == kbdev->pm.backend.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CYCLE_COUNT_STOP); + + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); +} + +void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_pm_release_gpu_cycle_counter_nolock(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h new file mode 100755 index 000000000000..50ca016bbd6d --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h @@ -0,0 +1,739 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Power management API definitions used internally by GPU backend + */ + +#ifndef _KBASE_BACKEND_PM_INTERNAL_H_ +#define _KBASE_BACKEND_PM_INTERNAL_H_ + +#include + +#include "mali_kbase_pm_ca.h" +#include "mali_kbase_pm_policy.h" + + +/** + * kbase_pm_dev_idle - The GPU is idle. + * + * The OS may choose to turn off idle devices + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_dev_idle(struct kbase_device *kbdev); + +/** + * kbase_pm_dev_activate - The GPU is active. + * + * The OS should avoid opportunistically turning off the GPU while it is active + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_dev_activate(struct kbase_device *kbdev); + +/** + * kbase_pm_get_present_cores - Get details of the cores that are present in + * the device. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) present in the GPU device and also a count of + * the number of cores. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of cores present + */ +u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_get_active_cores - Get details of the cores that are currently + * active in the device. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) that are actively processing work (i.e. + * turned on *and* busy). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of active cores + */ +u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_get_trans_cores - Get details of the cores that are currently + * transitioning between power states. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) that are currently transitioning between + * power states. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of transitioning cores + */ +u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_get_ready_cores - Get details of the cores that are currently + * powered and ready for jobs. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) that are powered and ready for jobs (they may + * or may not be currently executing jobs). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of ready cores + */ +u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_clock_on - Turn the clock for the device on, and enable device + * interrupts. + * + * This function can be used by a power policy to turn the clock for the GPU on. + * It should be modified during integration to perform the necessary actions to + * ensure that the GPU is fully powered and clocked. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @is_resume: true if clock on due to resume after suspend, false otherwise + */ +void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume); + +/** + * kbase_pm_clock_off - Disable device interrupts, and turn the clock for the + * device off. + * + * This function can be used by a power policy to turn the clock for the GPU + * off. It should be modified during integration to perform the necessary + * actions to turn the clock off (if this is possible in the integration). + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * + * Return: true if clock was turned off, or + * false if clock can not be turned off due to pending page/bus fault + * workers. Caller must flush MMU workqueues and retry + */ +bool kbase_pm_clock_off(struct kbase_device *kbdev); + +/** + * kbase_pm_enable_interrupts - Enable interrupts on the device. + * + * Interrupts are also enabled after a call to kbase_pm_clock_on(). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_enable_interrupts(struct kbase_device *kbdev); + +/** + * kbase_pm_disable_interrupts - Disable interrupts on the device. + * + * This prevents delivery of Power Management interrupts to the CPU so that + * kbase_pm_update_state() will not be called from the IRQ handler + * until kbase_pm_enable_interrupts() or kbase_pm_clock_on() is called. + * + * Interrupts are also disabled after a call to kbase_pm_clock_off(). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_disable_interrupts(struct kbase_device *kbdev); + +/** + * kbase_pm_disable_interrupts_nolock - Version of kbase_pm_disable_interrupts() + * that does not take the hwaccess_lock + * + * Caller must hold the hwaccess_lock. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_init_hw - Initialize the hardware. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @flags: Flags specifying the type of PM init + * + * This function checks the GPU ID register to ensure that the GPU is supported + * by the driver and performs a reset on the device so that it is in a known + * state before the device is used. + * + * Return: 0 if the device is supported and successfully reset. + */ +int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags); + +/** + * kbase_pm_reset_done - The GPU has been reset successfully. + * + * This function must be called by the GPU interrupt handler when the + * RESET_COMPLETED bit is set. It signals to the power management initialization + * code that the GPU has been successfully reset. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_reset_done(struct kbase_device *kbdev); + +#if MALI_USE_CSF +/** + * kbase_pm_wait_for_desired_state - Wait for the desired power state to be + * reached + * + * Wait for the L2 and MCU state machines to reach the states corresponding + * to the values of 'kbase_pm_is_l2_desired' and 'kbase_pm_is_mcu_desired'. + * + * The usual use-case for this is to ensure that all parts of GPU have been + * powered up after performing a GPU Reset. + * + * Unlike kbase_pm_update_state(), the caller must not hold hwaccess_lock, + * because this function will take that lock itself. + * + * NOTE: This may not wait until the correct state is reached if there is a + * power off in progress and kbase_pm_context_active() was called instead of + * kbase_csf_scheduler_pm_active(). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 on success, error code on error + */ +#else +/** + * kbase_pm_wait_for_desired_state - Wait for the desired power state to be + * reached + * + * Wait for the L2 and shader power state machines to reach the states + * corresponding to the values of 'l2_desired' and 'shaders_desired'. + * + * The usual use-case for this is to ensure cores are 'READY' after performing + * a GPU Reset. + * + * Unlike kbase_pm_update_state(), the caller must not hold hwaccess_lock, + * because this function will take that lock itself. + * + * NOTE: This may not wait until the correct state is reached if there is a + * power off in progress. To correctly wait for the desired state the caller + * must ensure that this is not the case by, for example, calling + * kbase_pm_wait_for_poweroff_complete() + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 on success, error code on error + */ +#endif +int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); + +/** + * kbase_pm_wait_for_l2_powered - Wait for the L2 cache to be powered on + * + * Wait for the L2 to be powered on, and for the L2 and the state machines of + * its dependent stack components to stabilise. + * + * kbdev->pm.active_count must be non-zero when calling this function. + * + * Unlike kbase_pm_update_state(), the caller must not hold hwaccess_lock, + * because this function will take that lock itself. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev); + +/** + * kbase_pm_update_dynamic_cores_onoff - Update the L2 and shader power state + * machines after changing shader core + * availability + * + * It can be called in any status, so need to check the l2 and shader core + * power status in this function or it will break shader/l2 state machine + * + * Caller must hold hwaccess_lock + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev); + +/** + * kbase_pm_update_cores_state_nolock - Variant of kbase_pm_update_cores_state() + * where the caller must hold + * kbase_device.hwaccess_lock + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_update_state - Update the L2 and shader power state machines + * @kbdev: Device pointer + */ +void kbase_pm_update_state(struct kbase_device *kbdev); + +/** + * kbase_pm_state_machine_init - Initialize the state machines, primarily the + * shader poweroff timer + * @kbdev: Device pointer + */ +int kbase_pm_state_machine_init(struct kbase_device *kbdev); + +/** + * kbase_pm_state_machine_term - Clean up the PM state machines' data + * @kbdev: Device pointer + */ +void kbase_pm_state_machine_term(struct kbase_device *kbdev); + +/** + * kbase_pm_update_cores_state - Update the desired state of shader cores from + * the Power Policy, and begin any power + * transitions. + * + * This function will update the desired_xx_state members of + * struct kbase_pm_device_data by calling into the current Power Policy. It will + * then begin power transitions to make the hardware acheive the desired shader + * core state. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_update_cores_state(struct kbase_device *kbdev); + +/** + * kbasep_pm_metrics_init - Initialize the metrics gathering framework. + * + * This must be called before other metric gathering APIs are called. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 on success, error code on error + */ +int kbasep_pm_metrics_init(struct kbase_device *kbdev); + +/** + * kbasep_pm_metrics_term - Terminate the metrics gathering framework. + * + * This must be called when metric gathering is no longer required. It is an + * error to call any metrics gathering function (other than + * kbasep_pm_metrics_init()) after calling this function. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbasep_pm_metrics_term(struct kbase_device *kbdev); + +/** + * kbase_pm_report_vsync - Function to be called by the frame buffer driver to + * update the vsync metric. + * + * This function should be called by the frame buffer driver to update whether + * the system is hitting the vsync target or not. buffer_updated should be true + * if the vsync corresponded with a new frame being displayed, otherwise it + * should be false. This function does not need to be called every vsync, but + * only when the value of @buffer_updated differs from a previous call. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * @buffer_updated: True if the buffer has been updated on this VSync, + * false otherwise + */ +void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated); + +/** + * kbase_pm_get_dvfs_action - Determine whether the DVFS system should change + * the clock speed of the GPU. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This function should be called regularly by the DVFS system to check whether + * the clock speed of the GPU needs updating. + */ +void kbase_pm_get_dvfs_action(struct kbase_device *kbdev); + +/** + * kbase_pm_request_gpu_cycle_counter - Mark that the GPU cycle counter is + * needed + * + * If the caller is the first caller then the GPU cycle counters will be enabled + * along with the l2 cache + * + * The GPU must be powered when calling this function (i.e. + * kbase_pm_context_active() must have been called). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev); + +/** + * kbase_pm_request_gpu_cycle_counter_l2_is_on - Mark GPU cycle counter is + * needed (l2 cache already on) + * + * This is a version of the above function + * (kbase_pm_request_gpu_cycle_counter()) suitable for being called when the + * l2 cache is known to be on and assured to be on until the subsequent call of + * kbase_pm_release_gpu_cycle_counter() such as when a job is submitted. It does + * not sleep and can be called from atomic functions. + * + * The GPU must be powered when calling this function (i.e. + * kbase_pm_context_active() must have been called) and the l2 cache must be + * powered on. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev); + +/** + * kbase_pm_release_gpu_cycle_counter - Mark that the GPU cycle counter is no + * longer in use + * + * If the caller is the last caller then the GPU cycle counters will be + * disabled. A request must have been made before a call to this. + * + * Caller must not hold the hwaccess_lock, as it will be taken in this function. + * If the caller is already holding this lock then + * kbase_pm_release_gpu_cycle_counter_nolock() must be used instead. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev); + +/** + * kbase_pm_release_gpu_cycle_counter_nolock - Version of kbase_pm_release_gpu_cycle_counter() + * that does not take hwaccess_lock + * + * Caller must hold the hwaccess_lock. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_wait_for_poweroff_complete - Wait for the poweroff workqueue to + * complete + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev); + +/** + * kbase_pm_runtime_init - Initialize runtime-pm for Mali GPU platform device + * + * Setup the power management callbacks and initialize/enable the runtime-pm + * for the Mali GPU platform device, using the callback function. This must be + * called before the kbase_pm_register_access_enable() function. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +int kbase_pm_runtime_init(struct kbase_device *kbdev); + +/** + * kbase_pm_runtime_term - Disable runtime-pm for Mali GPU platform device + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_runtime_term(struct kbase_device *kbdev); + +/** + * kbase_pm_register_access_enable - Enable access to GPU registers + * + * Enables access to the GPU registers before power management has powered up + * the GPU with kbase_pm_powerup(). + * + * This results in the power management callbacks provided in the driver + * configuration to get called to turn on power and/or clocks to the GPU. See + * kbase_pm_callback_conf. + * + * This should only be used before power management is powered up with + * kbase_pm_powerup() + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_register_access_enable(struct kbase_device *kbdev); + +/** + * kbase_pm_register_access_disable - Disable early register access + * + * Disables access to the GPU registers enabled earlier by a call to + * kbase_pm_register_access_enable(). + * + * This results in the power management callbacks provided in the driver + * configuration to get called to turn off power and/or clocks to the GPU. See + * kbase_pm_callback_conf + * + * This should only be used before power management is powered up with + * kbase_pm_powerup() + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_register_access_disable(struct kbase_device *kbdev); + +/* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline + * function */ + +/** + * kbase_pm_metrics_is_active - Check if the power management metrics + * collection is active. + * + * Note that this returns if the power management metrics collection was + * active at the time of calling, it is possible that after the call the metrics + * collection enable may have changed state. + * + * The caller must handle the consequence that the state may have changed. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * Return: true if metrics collection was active else false. + */ +bool kbase_pm_metrics_is_active(struct kbase_device *kbdev); + +/** + * kbase_pm_do_poweron - Power on the GPU, and any cores that are requested. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @is_resume: true if power on due to resume after suspend, + * false otherwise + */ +void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume); + +/** + * kbase_pm_do_poweroff - Power off the GPU, and any cores that have been + * requested. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_pm_do_poweroff(struct kbase_device *kbdev); + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) +void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, + struct kbasep_pm_metrics *last, + struct kbasep_pm_metrics *diff); +#endif /* defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) */ + +#ifdef CONFIG_MALI_BIFROST_DVFS + +/** + * kbase_platform_dvfs_event - Report utilisation to DVFS code + * + * Function provided by platform specific code when DVFS is enabled to allow + * the power management metrics system to report utilisation. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * @utilisation: The current calculated utilisation by the metrics system. + * @util_gl_share: The current calculated gl share of utilisation. + * @util_cl_share: The current calculated cl share of utilisation per core + * group. + * Return: Returns 0 on failure and non zero on success. + */ + +int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, + u32 util_gl_share, u32 util_cl_share[2]); +#endif + +void kbase_pm_power_changed(struct kbase_device *kbdev); + +/** + * kbase_pm_metrics_update - Inform the metrics system that an atom is either + * about to be run or has just completed. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @now: Pointer to the timestamp of the change, or NULL to use current time + * + * Caller must hold hwaccess_lock + */ +void kbase_pm_metrics_update(struct kbase_device *kbdev, + ktime_t *now); + +/** + * kbase_pm_cache_snoop_enable - Allow CPU snoops on the GPU + * If the GPU does not have coherency this is a no-op + * @kbdev: Device pointer + * + * This function should be called after L2 power up. + */ + +void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev); + +/** + * kbase_pm_cache_snoop_disable - Prevent CPU snoops on the GPU + * If the GPU does not have coherency this is a no-op + * @kbdev: Device pointer + * + * This function should be called before L2 power off. + */ +void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev); + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ +/** + * kbase_devfreq_set_core_mask - Set devfreq core mask + * @kbdev: Device pointer + * @core_mask: New core mask + * + * This function is used by devfreq to change the available core mask as + * required by Dynamic Core Scaling. + */ +void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask); +#endif + +/** + * kbase_pm_reset_start_locked - Signal that GPU reset has started + * @kbdev: Device pointer + * + * Normal power management operation will be suspended until the reset has + * completed. + * + * Caller must hold hwaccess_lock. + */ +void kbase_pm_reset_start_locked(struct kbase_device *kbdev); + +/** + * kbase_pm_reset_complete - Signal that GPU reset has completed + * @kbdev: Device pointer + * + * Normal power management operation will be resumed. The power manager will + * re-evaluate what cores are needed and power on or off as required. + */ +void kbase_pm_reset_complete(struct kbase_device *kbdev); + +/** + * kbase_pm_protected_override_enable - Enable the protected mode override + * @kbdev: Device pointer + * + * When the protected mode override is enabled, all shader cores are requested + * to power down, and the L2 power state can be controlled by + * kbase_pm_protected_l2_override(). + * + * Caller must hold hwaccess_lock. + */ +void kbase_pm_protected_override_enable(struct kbase_device *kbdev); + +/** + * kbase_pm_protected_override_disable - Disable the protected mode override + * @kbdev: Device pointer + * + * Caller must hold hwaccess_lock. + */ +void kbase_pm_protected_override_disable(struct kbase_device *kbdev); + +/** + * kbase_pm_protected_l2_override - Control the protected mode L2 override + * @kbdev: Device pointer + * @override: true to enable the override, false to disable + * + * When the driver is transitioning in or out of protected mode, the L2 cache is + * forced to power off. This can be overridden to force the L2 cache to power + * on. This is required to change coherency settings on some GPUs. + */ +void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override); + +/** + * kbase_pm_protected_entry_override_enable - Enable the protected mode entry + * override + * @kbdev: Device pointer + * + * Initiate a GPU reset and enable the protected mode entry override flag if + * l2_always_on WA is enabled and platform is fully coherent. If the GPU + * reset is already ongoing then protected mode entry override flag will not + * be enabled and function will have to be called again. + * + * When protected mode entry override flag is enabled to power down L2 via GPU + * reset, the GPU reset handling behavior gets changed. For example call to + * kbase_backend_reset() is skipped, Hw counters are not re-enabled and L2 + * isn't powered up again post reset. + * This is needed only as a workaround for a Hw issue where explicit power down + * of L2 causes a glitch. For entering protected mode on fully coherent + * platforms L2 needs to be powered down to switch to IO coherency mode, so to + * avoid the glitch GPU reset is used to power down L2. Hence, this function + * does nothing on systems where the glitch issue isn't present. + * + * Caller must hold hwaccess_lock. Should be only called during the transition + * to enter protected mode. + * + * Return: -EAGAIN if a GPU reset was required for the glitch workaround but + * was already ongoing, otherwise 0. + */ +int kbase_pm_protected_entry_override_enable(struct kbase_device *kbdev); + +/** + * kbase_pm_protected_entry_override_disable - Disable the protected mode entry + * override + * @kbdev: Device pointer + * + * This shall be called once L2 has powered down and switch to IO coherency + * mode has been made. As with kbase_pm_protected_entry_override_enable(), + * this function does nothing on systems where the glitch issue isn't present. + * + * Caller must hold hwaccess_lock. Should be only called during the transition + * to enter protected mode. + */ +void kbase_pm_protected_entry_override_disable(struct kbase_device *kbdev); + +/* If true, the driver should explicitly control corestack power management, + * instead of relying on the Power Domain Controller. + */ +extern bool corestack_driver_control; + +/** + * kbase_pm_is_l2_desired - Check whether l2 is desired + * + * @kbdev: Device pointer + * + * This shall be called to check whether l2 is needed to power on + * + * Return: true if l2 need to power on + */ +bool kbase_pm_is_l2_desired(struct kbase_device *kbdev); + +/** + * kbase_pm_lock - Lock all necessary mutexes to perform PM actions + * + * @kbdev: Device pointer + * + * This function locks correct mutexes independent of GPU architecture. + */ +static inline void kbase_pm_lock(struct kbase_device *kbdev) +{ +#if !MALI_USE_CSF + mutex_lock(&kbdev->js_data.runpool_mutex); +#endif /* !MALI_USE_CSF */ + mutex_lock(&kbdev->pm.lock); +} + +/** + * kbase_pm_unlock - Unlock mutexes locked by kbase_pm_lock + * + * @kbdev: Device pointer + */ +static inline void kbase_pm_unlock(struct kbase_device *kbdev) +{ + mutex_unlock(&kbdev->pm.lock); +#if !MALI_USE_CSF + mutex_unlock(&kbdev->js_data.runpool_mutex); +#endif /* !MALI_USE_CSF */ +} + +#endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_l2_states.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_l2_states.h new file mode 100755 index 000000000000..12cb051db42a --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_l2_states.h @@ -0,0 +1,38 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Backend-specific Power Manager level 2 cache state definitions. + * The function-like macro KBASEP_L2_STATE() must be defined before including + * this header file. This header file can be included multiple times in the + * same compilation unit with different definitions of KBASEP_L2_STATE(). + */ +KBASEP_L2_STATE(OFF) +KBASEP_L2_STATE(PEND_ON) +KBASEP_L2_STATE(RESTORE_CLOCKS) +KBASEP_L2_STATE(ON_HWCNT_ENABLE) +KBASEP_L2_STATE(ON) +KBASEP_L2_STATE(ON_HWCNT_DISABLE) +KBASEP_L2_STATE(SLOW_DOWN_CLOCKS) +KBASEP_L2_STATE(POWER_DOWN) +KBASEP_L2_STATE(PEND_OFF) +KBASEP_L2_STATE(RESET_WAIT) diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_mcu_states.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_mcu_states.h new file mode 100755 index 000000000000..e163bd4f4094 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_mcu_states.h @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Backend-specific Power Manager MCU state definitions. + * The function-like macro KBASEP_MCU_STATE() must be defined before including + * this header file. This header file can be included multiple times in the + * same compilation unit with different definitions of KBASEP_MCU_STATE(). + */ +KBASEP_MCU_STATE(OFF) +KBASEP_MCU_STATE(PEND_ON_RELOAD) +KBASEP_MCU_STATE(ON_GLB_REINIT_PEND) +KBASEP_MCU_STATE(ON_HWCNT_ENABLE) +KBASEP_MCU_STATE(ON) +KBASEP_MCU_STATE(ON_HWCNT_DISABLE) +KBASEP_MCU_STATE(ON_HALT) +KBASEP_MCU_STATE(ON_PEND_HALT) +KBASEP_MCU_STATE(POWER_DOWN) +KBASEP_MCU_STATE(PEND_OFF) +KBASEP_MCU_STATE(RESET_WAIT) diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c new file mode 100755 index 000000000000..b714971ba17c --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c @@ -0,0 +1,324 @@ +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Metrics for power management + */ + +#include +#include +#include +#if !MALI_USE_CSF +#include +#endif /* !MALI_USE_CSF */ +#include +#include + +/* When VSync is being hit aim for utilisation between 70-90% */ +#define KBASE_PM_VSYNC_MIN_UTILISATION 70 +#define KBASE_PM_VSYNC_MAX_UTILISATION 90 +/* Otherwise aim for 10-40% */ +#define KBASE_PM_NO_VSYNC_MIN_UTILISATION 10 +#define KBASE_PM_NO_VSYNC_MAX_UTILISATION 40 + +/* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns + * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly + * under 11s. Exceeding this will cause overflow */ +#define KBASE_PM_TIME_SHIFT 8 + +#ifdef CONFIG_MALI_BIFROST_DVFS +static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) +{ + unsigned long flags; + struct kbasep_pm_metrics_state *metrics; + + KBASE_DEBUG_ASSERT(timer != NULL); + + metrics = container_of(timer, struct kbasep_pm_metrics_state, timer); + kbase_pm_get_dvfs_action(metrics->kbdev); + + spin_lock_irqsave(&metrics->lock, flags); + + if (metrics->timer_active) + hrtimer_start(timer, + HR_TIMER_DELAY_MSEC(metrics->kbdev->pm.dvfs_period), + HRTIMER_MODE_REL); + + spin_unlock_irqrestore(&metrics->lock, flags); + + return HRTIMER_NORESTART; +} +#endif /* CONFIG_MALI_BIFROST_DVFS */ + +int kbasep_pm_metrics_init(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + kbdev->pm.backend.metrics.kbdev = kbdev; + + kbdev->pm.backend.metrics.time_period_start = ktime_get(); + kbdev->pm.backend.metrics.gpu_active = false; + kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[2] = 0; + + kbdev->pm.backend.metrics.values.time_busy = 0; + kbdev->pm.backend.metrics.values.time_idle = 0; + kbdev->pm.backend.metrics.values.busy_cl[0] = 0; + kbdev->pm.backend.metrics.values.busy_cl[1] = 0; + kbdev->pm.backend.metrics.values.busy_gl = 0; + + spin_lock_init(&kbdev->pm.backend.metrics.lock); + +#ifdef CONFIG_MALI_BIFROST_DVFS + hrtimer_init(&kbdev->pm.backend.metrics.timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + kbdev->pm.backend.metrics.timer.function = dvfs_callback; + + kbase_pm_metrics_start(kbdev); +#endif /* CONFIG_MALI_BIFROST_DVFS */ + + return 0; +} +KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init); + +void kbasep_pm_metrics_term(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_BIFROST_DVFS + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + kbdev->pm.backend.metrics.timer_active = false; + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); + + hrtimer_cancel(&kbdev->pm.backend.metrics.timer); +#endif /* CONFIG_MALI_BIFROST_DVFS */ +} + +KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); + +/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this + * function + */ +static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, + ktime_t now) +{ + ktime_t diff; + + lockdep_assert_held(&kbdev->pm.backend.metrics.lock); + + diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start); + if (ktime_to_ns(diff) < 0) + return; + + if (kbdev->pm.backend.metrics.gpu_active) { + u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); + + kbdev->pm.backend.metrics.values.time_busy += ns_time; + if (kbdev->pm.backend.metrics.active_cl_ctx[0]) + kbdev->pm.backend.metrics.values.busy_cl[0] += ns_time; + if (kbdev->pm.backend.metrics.active_cl_ctx[1]) + kbdev->pm.backend.metrics.values.busy_cl[1] += ns_time; + if (kbdev->pm.backend.metrics.active_gl_ctx[0]) + kbdev->pm.backend.metrics.values.busy_gl += ns_time; + if (kbdev->pm.backend.metrics.active_gl_ctx[1]) + kbdev->pm.backend.metrics.values.busy_gl += ns_time; + if (kbdev->pm.backend.metrics.active_gl_ctx[2]) + kbdev->pm.backend.metrics.values.busy_gl += ns_time; + } else { + kbdev->pm.backend.metrics.values.time_idle += (u32) (ktime_to_ns(diff) + >> KBASE_PM_TIME_SHIFT); + } + + kbdev->pm.backend.metrics.time_period_start = now; +} + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) +void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, + struct kbasep_pm_metrics *last, + struct kbasep_pm_metrics *diff) +{ + struct kbasep_pm_metrics *cur = &kbdev->pm.backend.metrics.values; + unsigned long flags; + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + kbase_pm_get_dvfs_utilisation_calc(kbdev, ktime_get()); + + memset(diff, 0, sizeof(*diff)); + diff->time_busy = cur->time_busy - last->time_busy; + diff->time_idle = cur->time_idle - last->time_idle; + diff->busy_cl[0] = cur->busy_cl[0] - last->busy_cl[0]; + diff->busy_cl[1] = cur->busy_cl[1] - last->busy_cl[1]; + diff->busy_gl = cur->busy_gl - last->busy_gl; + + *last = *cur; + + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); +} +KBASE_EXPORT_TEST_API(kbase_pm_get_dvfs_metrics); +#endif + +#ifdef CONFIG_MALI_BIFROST_DVFS +void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) +{ + int utilisation, util_gl_share; + int util_cl_share[2]; + int busy; + struct kbasep_pm_metrics *diff; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + diff = &kbdev->pm.backend.metrics.dvfs_diff; + + kbase_pm_get_dvfs_metrics(kbdev, &kbdev->pm.backend.metrics.dvfs_last, diff); + + utilisation = (100 * diff->time_busy) / + max(diff->time_busy + diff->time_idle, 1u); + + busy = max(diff->busy_gl + diff->busy_cl[0] + diff->busy_cl[1], 1u); + util_gl_share = (100 * diff->busy_gl) / busy; + util_cl_share[0] = (100 * diff->busy_cl[0]) / busy; + util_cl_share[1] = (100 * diff->busy_cl[1]) / busy; + + kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, util_cl_share); +} + +bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) +{ + bool isactive; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + isactive = kbdev->pm.backend.metrics.timer_active; + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); + + return isactive; +} +KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active); + +void kbase_pm_metrics_start(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + kbdev->pm.backend.metrics.timer_active = true; + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); + hrtimer_start(&kbdev->pm.backend.metrics.timer, + HR_TIMER_DELAY_MSEC(kbdev->pm.dvfs_period), + HRTIMER_MODE_REL); +} + +void kbase_pm_metrics_stop(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + kbdev->pm.backend.metrics.timer_active = false; + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); + hrtimer_cancel(&kbdev->pm.backend.metrics.timer); +} + + +#endif /* CONFIG_MALI_BIFROST_DVFS */ + +#if !MALI_USE_CSF +/** + * kbase_pm_metrics_active_calc - Update PM active counts based on currently + * running atoms + * @kbdev: Device pointer + * + * The caller must hold kbdev->pm.backend.metrics.lock + */ +static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) +{ + int js; + + lockdep_assert_held(&kbdev->pm.backend.metrics.lock); + + kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[2] = 0; + kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; + kbdev->pm.backend.metrics.gpu_active = false; + + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); + + /* Head atom may have just completed, so if it isn't running + * then try the next atom */ + if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) + katom = kbase_gpu_inspect(kbdev, js, 1); + + if (katom && katom->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED) { + if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { + int device_nr = (katom->core_req & + BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) + ? katom->device_nr : 0; + if (!WARN_ON(device_nr >= 2)) + kbdev->pm.backend.metrics. + active_cl_ctx[device_nr] = 1; + } else { + kbdev->pm.backend.metrics.active_gl_ctx[js] = 1; + trace_sysgraph(SGR_ACTIVE, 0, js); + } + kbdev->pm.backend.metrics.gpu_active = true; + } else { + trace_sysgraph(SGR_INACTIVE, 0, js); + } + } +} +#endif /* !MALI_USE_CSF */ + +/* called when job is submitted to or removed from a GPU slot */ +void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) +{ + unsigned long flags; + ktime_t now; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + + if (!timestamp) { + now = ktime_get(); + timestamp = &now; + } + + /* Track how long CL and/or GL jobs have been busy for */ + kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); + +#if !MALI_USE_CSF + kbase_pm_metrics_active_calc(kbdev); +#endif /* !MALI_USE_CSF */ + + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); +} diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c new file mode 100755 index 000000000000..48b24b1c866e --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c @@ -0,0 +1,268 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Power policy API implementations + */ + +#include +#include +#include +#include + +static const struct kbase_pm_policy *const all_policy_list[] = { +#ifdef CONFIG_MALI_BIFROST_NO_MALI + &kbase_pm_always_on_policy_ops, + &kbase_pm_coarse_demand_policy_ops, +#if !MALI_CUSTOMER_RELEASE + &kbase_pm_always_on_demand_policy_ops, +#endif +#else /* CONFIG_MALI_BIFROST_NO_MALI */ + &kbase_pm_coarse_demand_policy_ops, +#if !MALI_CUSTOMER_RELEASE + &kbase_pm_always_on_demand_policy_ops, +#endif + &kbase_pm_always_on_policy_ops +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ +}; + +void kbase_pm_policy_init(struct kbase_device *kbdev) +{ + kbdev->pm.backend.pm_current_policy = all_policy_list[0]; + kbdev->pm.backend.pm_current_policy->init(kbdev); +} + +void kbase_pm_policy_term(struct kbase_device *kbdev) +{ + kbdev->pm.backend.pm_current_policy->term(kbdev); +} + +void kbase_pm_update_active(struct kbase_device *kbdev) +{ + struct kbase_pm_device_data *pm = &kbdev->pm; + struct kbase_pm_backend_data *backend = &pm->backend; + unsigned long flags; + bool active; + + lockdep_assert_held(&pm->lock); + + /* pm_current_policy will never be NULL while pm.lock is held */ + KBASE_DEBUG_ASSERT(backend->pm_current_policy); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + active = backend->pm_current_policy->get_core_active(kbdev); + WARN((kbase_pm_is_active(kbdev) && !active), + "GPU is active but policy '%s' is indicating that it can be powered off", + kbdev->pm.backend.pm_current_policy->name); + + if (active) { + /* Power on the GPU and any cores requested by the policy */ + if (!pm->backend.invoke_poweroff_wait_wq_when_l2_off && + pm->backend.poweroff_wait_in_progress) { + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + pm->backend.poweron_required = true; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } else { + /* Cancel the invocation of + * kbase_pm_gpu_poweroff_wait_wq() from the L2 state + * machine. This is safe - it + * invoke_poweroff_wait_wq_when_l2_off is true, then + * the poweroff work hasn't even been queued yet, + * meaning we can go straight to powering on. + */ + pm->backend.invoke_poweroff_wait_wq_when_l2_off = false; + pm->backend.poweroff_wait_in_progress = false; + pm->backend.l2_desired = true; +#if MALI_USE_CSF + pm->backend.mcu_desired = true; +#endif + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_pm_do_poweron(kbdev, false); + } + } else { + /* It is an error for the power policy to power off the GPU + * when there are contexts active */ + KBASE_DEBUG_ASSERT(pm->active_count == 0); + + pm->backend.poweron_required = false; + + /* Request power off */ + if (pm->backend.gpu_powered) { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Power off the GPU immediately */ + kbase_pm_do_poweroff(kbdev); + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + } +} + +void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev) +{ + bool shaders_desired; + + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&kbdev->pm.lock); + +#if MALI_USE_CSF + /* On CSF GPUs, Host driver isn't supposed to do the power management + * for shader cores. CSF firmware will power up the cores appropriately + * and so from Driver's standpoint 'shaders_desired' flag shall always + * remain 0. + */ + return; +#endif + if (kbdev->pm.backend.pm_current_policy == NULL) + return; + if (kbdev->pm.backend.poweroff_wait_in_progress) + return; + /* In protected transition, don't allow outside shader core request + * affect transition, return directly + */ + if (kbdev->pm.backend.protected_transition_override) + return; + + shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); + + if (shaders_desired && kbase_pm_is_l2_desired(kbdev)) { + kbase_pm_update_state(kbdev); + } +} + +void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) +{ + bool shaders_desired; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->pm.backend.pm_current_policy == NULL) + return; + if (kbdev->pm.backend.poweroff_wait_in_progress) + return; + + if (kbdev->pm.backend.protected_transition_override) + /* We are trying to change in/out of protected mode - force all + * cores off so that the L2 powers down */ + shaders_desired = false; + else + shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); + +#if MALI_USE_CSF + /* On CSF GPUs, Host driver isn't supposed to do the power management + * for shader cores. CSF firmware will power up the cores appropriately + * and so from Driver's standpoint 'shaders_desired' flag shall always + * remain 0. + */ + shaders_desired = false; +#endif + if (kbdev->pm.backend.shaders_desired != shaders_desired) { + KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, kbdev->pm.backend.shaders_desired); + + kbdev->pm.backend.shaders_desired = shaders_desired; + kbase_pm_update_state(kbdev); + } +} + +void kbase_pm_update_cores_state(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_pm_update_cores_state_nolock(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +int kbase_pm_list_policies(struct kbase_device *kbdev, + const struct kbase_pm_policy * const **list) +{ + if (list) + *list = all_policy_list; + + return ARRAY_SIZE(all_policy_list); +} + +KBASE_EXPORT_TEST_API(kbase_pm_list_policies); + +const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + return kbdev->pm.backend.pm_current_policy; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_policy); + +void kbase_pm_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_policy *new_policy) +{ + const struct kbase_pm_policy *old_policy; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(new_policy != NULL); + + KBASE_KTRACE_ADD(kbdev, PM_SET_POLICY, NULL, new_policy->id); + + /* During a policy change we pretend the GPU is active */ + /* A suspend won't happen here, because we're in a syscall from a + * userspace thread */ + kbase_pm_context_active(kbdev); + + kbase_pm_lock(kbdev); + + /* Remove the policy to prevent IRQ handlers from working on it */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + old_policy = kbdev->pm.backend.pm_current_policy; + kbdev->pm.backend.pm_current_policy = NULL; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, old_policy->id); + if (old_policy->term) + old_policy->term(kbdev); + + KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, new_policy->id); + if (new_policy->init) + new_policy->init(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.pm_current_policy = new_policy; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* If any core power state changes were previously attempted, but + * couldn't be made because the policy was changing (current_policy was + * NULL), then re-try them here. */ + kbase_pm_update_active(kbdev); + kbase_pm_update_cores_state(kbdev); + + kbase_pm_unlock(kbdev); + + /* Now the policy change is finished, we release our fake context active + * reference */ + kbase_pm_context_idle(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_set_policy); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h new file mode 100755 index 000000000000..f103ef0c01e4 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h @@ -0,0 +1,106 @@ +/* + * + * (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Power policy API definitions + */ + +#ifndef _KBASE_PM_POLICY_H_ +#define _KBASE_PM_POLICY_H_ + +/** + * kbase_pm_policy_init - Initialize power policy framework + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Must be called before calling any other policy function + */ +void kbase_pm_policy_init(struct kbase_device *kbdev); + +/** + * kbase_pm_policy_term - Terminate power policy framework + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_policy_term(struct kbase_device *kbdev); + +/** + * kbase_pm_update_active - Update the active power state of the GPU + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Calls into the current power policy + */ +void kbase_pm_update_active(struct kbase_device *kbdev); + +/** + * kbase_pm_update_cores - Update the desired core state of the GPU + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Calls into the current power policy + */ +void kbase_pm_update_cores(struct kbase_device *kbdev); + +/** + * kbase_pm_cores_requested - Check that a power request has been locked into + * the HW. + * @kbdev: Kbase device + * @shader_required: true if shaders are required + * + * Called by the scheduler to check if a power on request has been locked into + * the HW. + * + * Note that there is no guarantee that the cores are actually ready, however + * when the request has been locked into the HW, then it is safe to submit work + * since the HW will wait for the transition to ready. + * + * A reference must first be taken prior to making this call. + * + * Caller must hold the hwaccess_lock. + * + * Return: true if the request to the HW was successfully made else false if the + * request is still pending. + */ +static inline bool kbase_pm_cores_requested(struct kbase_device *kbdev, + bool shader_required) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* If the L2 & tiler are not on or pending, then the tiler is not yet + * available, and shaders are definitely not powered. + */ + if (kbdev->pm.backend.l2_state != KBASE_L2_PEND_ON && + kbdev->pm.backend.l2_state != KBASE_L2_ON && + kbdev->pm.backend.l2_state != KBASE_L2_ON_HWCNT_ENABLE) + return false; + + if (shader_required && + kbdev->pm.backend.shaders_state != KBASE_SHADERS_PEND_ON_CORESTACK_ON && + kbdev->pm.backend.shaders_state != KBASE_SHADERS_ON_CORESTACK_ON && + kbdev->pm.backend.shaders_state != KBASE_SHADERS_ON_CORESTACK_ON_RECHECK) + return false; + + return true; +} + +#endif /* _KBASE_PM_POLICY_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h new file mode 100755 index 000000000000..6cafaa171962 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Backend-specific Power Manager shader core state definitions. + * The function-like macro KBASEP_SHADER_STATE() must be defined before + * including this header file. This header file can be included multiple + * times in the same compilation unit with different definitions of + * KBASEP_SHADER_STATE(). + */ +KBASEP_SHADER_STATE(OFF_CORESTACK_OFF) +KBASEP_SHADER_STATE(OFF_CORESTACK_PEND_ON) +KBASEP_SHADER_STATE(PEND_ON_CORESTACK_ON) +KBASEP_SHADER_STATE(ON_CORESTACK_ON) +KBASEP_SHADER_STATE(ON_CORESTACK_ON_RECHECK) +KBASEP_SHADER_STATE(WAIT_OFF_CORESTACK_ON) +#if !MALI_USE_CSF +KBASEP_SHADER_STATE(WAIT_GPU_IDLE) +#endif /* !MALI_USE_CSF */ +KBASEP_SHADER_STATE(WAIT_FINISHED_CORESTACK_ON) +KBASEP_SHADER_STATE(L2_FLUSHING_CORESTACK_ON) +KBASEP_SHADER_STATE(READY_OFF_CORESTACK_ON) +KBASEP_SHADER_STATE(PEND_OFF_CORESTACK_ON) +KBASEP_SHADER_STATE(OFF_CORESTACK_PEND_OFF) +KBASEP_SHADER_STATE(OFF_CORESTACK_OFF_TIMER_PEND_OFF) +KBASEP_SHADER_STATE(RESET_WAIT) diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c new file mode 100755 index 000000000000..e19f53b2cbe8 --- /dev/null +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c @@ -0,0 +1,81 @@ +/* + * + * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include + +void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev, + u64 *cycle_counter, + u64 *system_time, + struct timespec64 *ts) +{ + u32 hi1, hi2; + + if (cycle_counter) { + /* Read hi, lo, hi to ensure a coherent u64 */ + do { + hi1 = kbase_reg_read(kbdev, + GPU_CONTROL_REG(CYCLE_COUNT_HI)); + *cycle_counter = kbase_reg_read(kbdev, + GPU_CONTROL_REG(CYCLE_COUNT_LO)); + hi2 = kbase_reg_read(kbdev, + GPU_CONTROL_REG(CYCLE_COUNT_HI)); + } while (hi1 != hi2); + *cycle_counter |= (((u64) hi1) << 32); + } + + if (system_time) { + /* Read hi, lo, hi to ensure a coherent u64 */ + do { + hi1 = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TIMESTAMP_HI)); + *system_time = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TIMESTAMP_LO)); + hi2 = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TIMESTAMP_HI)); + } while (hi1 != hi2); + *system_time |= (((u64) hi1) << 32); + } + + /* Record the CPU's idea of current time */ + if (ts != NULL) +#if (KERNEL_VERSION(4, 17, 0) > LINUX_VERSION_CODE) + *ts = ktime_to_timespec64(ktime_get_raw()); +#else + ktime_get_raw_ts64(ts); +#endif +} + +void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, + u64 *system_time, struct timespec64 *ts) +{ +#if !MALI_USE_CSF + kbase_pm_request_gpu_cycle_counter(kbdev); +#endif + kbase_backend_get_gpu_time_norequest( + kbdev, cycle_counter, system_time, ts); +#if !MALI_USE_CSF + kbase_pm_release_gpu_cycle_counter(kbdev); +#endif +} diff --git a/drivers/gpu/arm/bifrost/build.bp b/drivers/gpu/arm/bifrost/build.bp new file mode 100755 index 000000000000..b9b86184f3be --- /dev/null +++ b/drivers/gpu/arm/bifrost/build.bp @@ -0,0 +1,186 @@ +/* + * + * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + +/* Kernel-side tests may include mali_kbase's headers. Therefore any config + * options which affect the sizes of any structs (e.g. adding extra members) + * must be included in these defaults, so that the structs are consistent in + * both mali_kbase and the test modules. */ +bob_defaults { + name: "mali_kbase_shared_config_defaults", + no_mali: { + kbuild_options: ["CONFIG_MALI_BIFROST_NO_MALI=y"], + }, + mali_real_hw: { + kbuild_options: ["CONFIG_MALI_REAL_HW=y"], + }, + mali_dma_fence: { + kbuild_options: ["CONFIG_MALI_BIFROST_DMA_FENCE=y"], + }, + mali_devfreq: { + kbuild_options: ["CONFIG_MALI_BIFROST_DEVFREQ=y"], + }, + mali_midgard_dvfs: { + kbuild_options: ["CONFIG_MALI_BIFROST_DVFS=y"], + }, + mali_debug: { + kbuild_options: ["CONFIG_MALI_BIFROST_DEBUG=y"], + }, + buslog: { + kbuild_options: ["CONFIG_MALI_BUSLOG=y"], + }, + cinstr_vector_dump: { + kbuild_options: ["CONFIG_MALI_VECTOR_DUMP=y"], + }, + cinstr_gwt: { + kbuild_options: ["CONFIG_MALI_CINSTR_GWT=y"], + }, + mali_gator_support: { + kbuild_options: ["CONFIG_MALI_BIFROST_GATOR_SUPPORT=y"], + }, + mali_midgard_enable_trace: { + kbuild_options: ["CONFIG_MALI_BIFROST_ENABLE_TRACE=y"], + }, + mali_system_trace: { + kbuild_options: ["CONFIG_MALI_BIFROST_SYSTEM_TRACE=y"], + }, + mali_pwrsoft_765: { + kbuild_options: ["CONFIG_MALI_PWRSOFT_765=y"], + }, + mali_memory_fully_backed: { + kbuild_options: ["CONFIG_MALI_MEMORY_FULLY_BACKED=y"], + }, + mali_dma_buf_map_on_demand: { + kbuild_options: ["CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND=y"], + }, + mali_dma_buf_legacy_compat: { + kbuild_options: ["CONFIG_MALI_DMA_BUF_LEGACY_COMPAT=y"], + }, + mali_arbiter_support: { + kbuild_options: ["CONFIG_MALI_ARBITER_SUPPORT=y"], + }, + mali_gem5_build: { + kbuild_options: ["CONFIG_MALI_GEM5_BUILD=y"], + }, + kbuild_options: [ + "MALI_UNIT_TEST={{.unit_test_code}}", + "MALI_CUSTOMER_RELEASE={{.release}}", + "MALI_USE_CSF={{.gpu_has_csf}}", + "MALI_KERNEL_TEST_API={{.debug}}", + ], + defaults: ["kernel_defaults"], +} + +bob_kernel_module { + name: "mali_kbase", + srcs: [ + "*.c", + "*.h", + "Kbuild", + "backend/gpu/*.c", + "backend/gpu/*.h", + "backend/gpu/Kbuild", + "context/*.c", + "context/*.h", + "ipa/*.c", + "ipa/*.h", + "ipa/Kbuild", + "platform/*.h", + "platform/*/*.c", + "platform/*/*.h", + "platform/*/Kbuild", + "thirdparty/*.c", + "debug/*.c", + "debug/*.h", + "device/*.c", + "device/*.h", + "gpu/*.c", + "gpu/*.h", + "tl/*.c", + "tl/*.h", + "mmu/*.c", + "mmu/*.h", + ], + kbuild_options: [ + "CONFIG_MALI_KUTF=n", + "CONFIG_MALI_MIDGARD=m", + "CONFIG_MALI_NO_MALI_DEFAULT_GPU={{.gpu}}", + "CONFIG_MALI_PLATFORM_NAME={{.mali_platform_name}}", + ], + buslog: { + extra_symbols: [ + "bus_logger", + ], + }, + mali_corestack: { + kbuild_options: ["CONFIG_MALI_CORESTACK=y"], + }, + mali_error_inject: { + kbuild_options: ["CONFIG_MALI_BIFROST_ERROR_INJECT=y"], + }, + mali_error_inject_random: { + kbuild_options: ["CONFIG_MALI_ERROR_INJECT_RANDOM=y"], + }, + cinstr_secondary_hwc: { + kbuild_options: ["CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY=y"], + }, + cinstr_secondary_hwc_via_debug_fs: { + kbuild_options: ["CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS=y"], + }, + mali_2mb_alloc: { + kbuild_options: ["CONFIG_MALI_2MB_ALLOC=y"], + }, + mali_hw_errata_1485982_not_affected: { + kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED=y"], + }, + mali_hw_errata_1485982_use_clock_alternative: { + kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE=y"], + }, + gpu_has_job_manager: { + srcs: [ + "context/backend/*_jm.c", + "debug/backend/*_jm.c", + "debug/backend/*_jm.h", + "device/backend/*_jm.c", + "gpu/backend/*_jm.c", + "gpu/backend/*_jm.h", + "jm/*.h", + "tl/backend/*_jm.c", + "mmu/backend/*_jm.c", + ], + }, + gpu_has_csf: { + srcs: [ + "context/backend/*_csf.c", + "csf/*.c", + "csf/*.h", + "csf/Kbuild", + "debug/backend/*_csf.c", + "debug/backend/*_csf.h", + "device/backend/*_csf.c", + "gpu/backend/*_csf.c", + "gpu/backend/*_csf.h", + "tl/backend/*_csf.c", + "mmu/backend/*_csf.c", + ], + }, + mali_arbiter_support: { + srcs: [ + "arbiter/*.c", + "arbiter/*.h", + "arbiter/Kbuild", + ], + }, + defaults: ["mali_kbase_shared_config_defaults"], +} diff --git a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c new file mode 100755 index 000000000000..7c68eb2f860a --- /dev/null +++ b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Base kernel context APIs for CSF GPUs + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_DEBUG_FS +#include +#include +#include +#include +#include + +void kbase_context_debugfs_init(struct kbase_context *const kctx) +{ + kbase_debug_mem_view_init(kctx); + kbase_mem_pool_debugfs_init(kctx->kctx_dentry, kctx); + kbase_jit_debugfs_init(kctx); + kbase_csf_queue_group_debugfs_init(kctx); + kbase_csf_kcpu_debugfs_init(kctx); + kbase_csf_tiler_heap_debugfs_init(kctx); +} +KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); + +void kbase_context_debugfs_term(struct kbase_context *const kctx) +{ + debugfs_remove_recursive(kctx->kctx_dentry); +} +KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); +#else +void kbase_context_debugfs_init(struct kbase_context *const kctx) +{ + CSTD_UNUSED(kctx); +} +KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); + +void kbase_context_debugfs_term(struct kbase_context *const kctx) +{ + CSTD_UNUSED(kctx); +} +KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); +#endif /* CONFIG_DEBUG_FS */ + +static const struct kbase_context_init context_init[] = { + {kbase_context_common_init, kbase_context_common_term, NULL}, + {kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, + "Memory pool goup initialization failed"}, + {kbase_mem_evictable_init, kbase_mem_evictable_deinit, + "Memory evictable initialization failed"}, + {kbase_context_mmu_init, kbase_context_mmu_term, + "MMU initialization failed"}, + {kbase_context_mem_alloc_page, kbase_context_mem_pool_free, + "Memory alloc page failed"}, + {kbase_region_tracker_init, kbase_region_tracker_term, + "Region tracker initialization failed"}, + {kbase_sticky_resource_init, kbase_context_sticky_resource_term, + "Sticky resource initialization failed"}, + {kbase_jit_init, kbase_jit_term, + "JIT initialization failed"}, + {kbase_csf_ctx_init, kbase_csf_ctx_term, + "CSF context initialization failed"}, +}; + +static void kbase_context_term_partial( + struct kbase_context *kctx, + unsigned int i) +{ + while (i-- > 0) { + if (context_init[i].term) + context_init[i].term(kctx); + } +} + +struct kbase_context *kbase_create_context(struct kbase_device *kbdev, + bool is_compat, + base_context_create_flags const flags, + unsigned long const api_version, + struct file *const filp) +{ + struct kbase_context *kctx; + unsigned int i = 0; + + if (WARN_ON(!kbdev)) + return NULL; + + /* Validate flags */ + if (WARN_ON(flags != (flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS))) + return NULL; + + /* zero-inited as lot of code assume it's zero'ed out on create */ + kctx = vzalloc(sizeof(*kctx)); + if (WARN_ON(!kctx)) + return NULL; + + kctx->kbdev = kbdev; + kctx->api_version = api_version; + kctx->filp = filp; + kctx->create_flags = flags; + + if (is_compat) + kbase_ctx_flag_set(kctx, KCTX_COMPAT); +#if defined(CONFIG_64BIT) + else + kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); +#endif /* !defined(CONFIG_64BIT) */ + + for (i = 0; i < ARRAY_SIZE(context_init); i++) { + int err = context_init[i].init(kctx); + + if (err) { + dev_err(kbdev->dev, "%s error = %d\n", + context_init[i].err_mes, err); + kbase_context_term_partial(kctx, i); + return NULL; + } + } + + return kctx; +} +KBASE_EXPORT_SYMBOL(kbase_create_context); + +void kbase_destroy_context(struct kbase_context *kctx) +{ + struct kbase_device *kbdev; + + if (WARN_ON(!kctx)) + return; + + kbdev = kctx->kbdev; + if (WARN_ON(!kbdev)) + return; + + /* Ensure the core is powered up for the destroy process + * A suspend won't happen here, because we're in a syscall + * from a userspace thread. + */ + kbase_pm_context_active(kbdev); + + kbase_mem_pool_group_mark_dying(&kctx->mem_pools); + + kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); + + kbase_pm_context_idle(kbdev); +} +KBASE_EXPORT_SYMBOL(kbase_destroy_context); diff --git a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c new file mode 100755 index 000000000000..0eb42589fe46 --- /dev/null +++ b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Base kernel context APIs for Job Manager GPUs + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_DEBUG_FS +#include +#include + +void kbase_context_debugfs_init(struct kbase_context *const kctx) +{ + kbase_debug_mem_view_init(kctx); + kbase_mem_pool_debugfs_init(kctx->kctx_dentry, kctx); + kbase_jit_debugfs_init(kctx); + kbasep_jd_debugfs_ctx_init(kctx); + kbase_debug_job_fault_context_init(kctx); +} +KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); + +void kbase_context_debugfs_term(struct kbase_context *const kctx) +{ + debugfs_remove_recursive(kctx->kctx_dentry); + kbase_debug_job_fault_context_term(kctx); +} +KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); +#else +void kbase_context_debugfs_init(struct kbase_context *const kctx) +{ + CSTD_UNUSED(kctx); +} +KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); + +void kbase_context_debugfs_term(struct kbase_context *const kctx) +{ + CSTD_UNUSED(kctx); +} +KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); +#endif /* CONFIG_DEBUG_FS */ + +static int kbase_context_kbase_kinstr_jm_init(struct kbase_context *kctx) +{ + int ret = kbase_kinstr_jm_init(&kctx->kinstr_jm); + + if (!ret) + return ret; + + return 0; +} + +static void kbase_context_kbase_kinstr_jm_term(struct kbase_context *kctx) +{ + kbase_kinstr_jm_term(kctx->kinstr_jm); +} + +static int kbase_context_kbase_timer_setup(struct kbase_context *kctx) +{ + kbase_timer_setup(&kctx->soft_job_timeout, + kbasep_soft_job_timeout_worker); + + return 0; +} + +static int kbase_context_submit_check(struct kbase_context *kctx) +{ + struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; + unsigned long irq_flags = 0; + + base_context_create_flags const flags = kctx->create_flags; + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); + + /* Translate the flags */ + if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) + kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); + + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return 0; +} + +static const struct kbase_context_init context_init[] = { + { kbase_context_common_init, kbase_context_common_term, NULL }, + { kbase_dma_fence_init, kbase_dma_fence_term, + "DMA fence initialization failed" }, + { kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, + "Memory pool goup initialization failed" }, + { kbase_mem_evictable_init, kbase_mem_evictable_deinit, + "Memory evictable initialization failed" }, + { kbase_context_mmu_init, kbase_context_mmu_term, + "MMU initialization failed" }, + { kbase_context_mem_alloc_page, kbase_context_mem_pool_free, + "Memory alloc page failed" }, + { kbase_region_tracker_init, kbase_region_tracker_term, + "Region tracker initialization failed" }, + { kbase_sticky_resource_init, kbase_context_sticky_resource_term, + "Sticky resource initialization failed" }, + { kbase_jit_init, kbase_jit_term, "JIT initialization failed" }, + { kbase_context_kbase_kinstr_jm_init, + kbase_context_kbase_kinstr_jm_term, + "JM instrumentation initialization failed" }, + { kbase_context_kbase_timer_setup, NULL, NULL }, + { kbase_event_init, kbase_event_cleanup, + "Event initialization failed" }, + { kbasep_js_kctx_init, kbasep_js_kctx_term, + "JS kctx initialization failed" }, + { kbase_jd_init, kbase_jd_exit, "JD initialization failed" }, + { kbase_context_submit_check, NULL, NULL }, +}; + +static void kbase_context_term_partial( + struct kbase_context *kctx, + unsigned int i) +{ + while (i-- > 0) { + if (context_init[i].term) + context_init[i].term(kctx); + } +} + +struct kbase_context *kbase_create_context(struct kbase_device *kbdev, + bool is_compat, + base_context_create_flags const flags, + unsigned long const api_version, + struct file *const filp) +{ + struct kbase_context *kctx; + unsigned int i = 0; + + if (WARN_ON(!kbdev)) + return NULL; + + /* Validate flags */ + if (WARN_ON(flags != (flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS))) + return NULL; + + /* zero-inited as lot of code assume it's zero'ed out on create */ + kctx = vzalloc(sizeof(*kctx)); + if (WARN_ON(!kctx)) + return NULL; + + kctx->kbdev = kbdev; + kctx->api_version = api_version; + kctx->filp = filp; + kctx->create_flags = flags; + + if (is_compat) + kbase_ctx_flag_set(kctx, KCTX_COMPAT); +#if defined(CONFIG_64BIT) + else + kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); +#endif /* !defined(CONFIG_64BIT) */ + + for (i = 0; i < ARRAY_SIZE(context_init); i++) { + int err = context_init[i].init(kctx); + + if (err) { + dev_err(kbdev->dev, "%s error = %d\n", + context_init[i].err_mes, err); + kbase_context_term_partial(kctx, i); + return NULL; + } + } + + return kctx; +} +KBASE_EXPORT_SYMBOL(kbase_create_context); + +void kbase_destroy_context(struct kbase_context *kctx) +{ + struct kbase_device *kbdev; + + if (WARN_ON(!kctx)) + return; + + kbdev = kctx->kbdev; + if (WARN_ON(!kbdev)) + return; + + /* Ensure the core is powered up for the destroy process + * A suspend won't happen here, because we're in a syscall + * from a userspace thread. + */ + kbase_pm_context_active(kbdev); + + kbase_mem_pool_group_mark_dying(&kctx->mem_pools); + + kbase_jd_zap_context(kctx); + flush_workqueue(kctx->jctx.job_done_wq); + + kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); + + kbase_pm_context_idle(kbdev); +} +KBASE_EXPORT_SYMBOL(kbase_destroy_context); diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context.c b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c new file mode 100755 index 000000000000..83182f983467 --- /dev/null +++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Base kernel context APIs + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * find_process_node - Used to traverse the process rb_tree to find if + * process exists already in process rb_tree. + * + * @node: Pointer to root node to start search. + * @tgid: Thread group PID to search for. + * + * Return: Pointer to kbase_process if exists otherwise NULL. + */ +static struct kbase_process *find_process_node(struct rb_node *node, pid_t tgid) +{ + struct kbase_process *kprcs = NULL; + + /* Check if the kctx creation request is from a existing process.*/ + while (node) { + struct kbase_process *prcs_node = + rb_entry(node, struct kbase_process, kprcs_node); + if (prcs_node->tgid == tgid) { + kprcs = prcs_node; + break; + } + + if (tgid < prcs_node->tgid) + node = node->rb_left; + else + node = node->rb_right; + } + + return kprcs; +} + +/** + * kbase_insert_kctx_to_process - Initialise kbase process context. + * + * @kctx: Pointer to kbase context. + * + * Here we initialise per process rb_tree managed by kbase_device. + * We maintain a rb_tree of each unique process that gets created. + * and Each process maintains a list of kbase context. + * This setup is currently used by kernel trace functionality + * to trace and visualise gpu memory consumption. + * + * Return: 0 on success and error number on failure. + */ +static int kbase_insert_kctx_to_process(struct kbase_context *kctx) +{ + struct rb_root *const prcs_root = &kctx->kbdev->process_root; + const pid_t tgid = kctx->tgid; + struct kbase_process *kprcs = NULL; + + lockdep_assert_held(&kctx->kbdev->kctx_list_lock); + + kprcs = find_process_node(prcs_root->rb_node, tgid); + + /* if the kctx is from new process then create a new kbase_process + * and add it to the &kbase_device->rb_tree + */ + if (!kprcs) { + struct rb_node **new = &prcs_root->rb_node, *parent = NULL; + + kprcs = kzalloc(sizeof(*kprcs), GFP_KERNEL); + if (kprcs == NULL) + return -ENOMEM; + kprcs->tgid = tgid; + INIT_LIST_HEAD(&kprcs->kctx_list); + kprcs->dma_buf_root = RB_ROOT; + kprcs->total_gpu_pages = 0; + + while (*new) { + struct kbase_process *prcs_node; + + parent = *new; + prcs_node = rb_entry(parent, struct kbase_process, + kprcs_node); + if (tgid < prcs_node->tgid) + new = &(*new)->rb_left; + else + new = &(*new)->rb_right; + } + rb_link_node(&kprcs->kprcs_node, parent, new); + rb_insert_color(&kprcs->kprcs_node, prcs_root); + } + + kctx->kprcs = kprcs; + list_add(&kctx->kprcs_link, &kprcs->kctx_list); + + return 0; +} + +int kbase_context_common_init(struct kbase_context *kctx) +{ + const unsigned long cookies_mask = KBASE_COOKIE_MASK; + int err = 0; + + /* creating a context is considered a disjoint event */ + kbase_disjoint_event(kctx->kbdev); + + kctx->as_nr = KBASEP_AS_NR_INVALID; + + atomic_set(&kctx->refcount, 0); + + spin_lock_init(&kctx->mm_update_lock); + kctx->process_mm = NULL; + atomic_set(&kctx->nonmapped_pages, 0); + atomic_set(&kctx->permanent_mapped_pages, 0); + kctx->tgid = current->tgid; + kctx->pid = current->pid; + + atomic_set(&kctx->used_pages, 0); + + mutex_init(&kctx->reg_lock); + + spin_lock_init(&kctx->mem_partials_lock); + INIT_LIST_HEAD(&kctx->mem_partials); + + spin_lock_init(&kctx->waiting_soft_jobs_lock); + INIT_LIST_HEAD(&kctx->waiting_soft_jobs); + + init_waitqueue_head(&kctx->event_queue); + atomic_set(&kctx->event_count, 0); +#if !MALI_USE_CSF + atomic_set(&kctx->event_closed, false); +#ifdef CONFIG_GPU_TRACEPOINTS + atomic_set(&kctx->jctx.work_id, 0); +#endif +#endif + + bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG); + + kctx->id = atomic_add_return(1, &(kctx->kbdev->ctx_num)) - 1; + + mutex_init(&kctx->legacy_hwcnt_lock); + + mutex_lock(&kctx->kbdev->kctx_list_lock); + list_add(&kctx->kctx_list_link, &kctx->kbdev->kctx_list); + + err = kbase_insert_kctx_to_process(kctx); + if (err) + dev_err(kctx->kbdev->dev, + "(err:%d) failed to insert kctx to kbase_process\n", err); + + KBASE_TLSTREAM_TL_KBASE_NEW_CTX(kctx->kbdev, kctx->id, + kctx->kbdev->gpu_props.props.raw_props.gpu_id); + KBASE_TLSTREAM_TL_NEW_CTX(kctx->kbdev, kctx, kctx->id, + (u32)(kctx->tgid)); + mutex_unlock(&kctx->kbdev->kctx_list_lock); + + return err; +} + +/** + * kbase_remove_kctx_from_process - remove a terminating context from + * the process list. + * + * @kctx: Pointer to kbase context. + * + * Remove the tracking of context from the list of contexts maintained under + * kbase process and if the list if empty then there no outstanding contexts + * we can remove the process node as well. + */ + +static void kbase_remove_kctx_from_process(struct kbase_context *kctx) +{ + struct kbase_process *kprcs = kctx->kprcs; + + lockdep_assert_held(&kctx->kbdev->kctx_list_lock); + list_del(&kctx->kprcs_link); + + /* if there are no outstanding contexts in current process node, + * we can remove it from the process rb_tree. + */ + if (list_empty(&kprcs->kctx_list)) { + rb_erase(&kprcs->kprcs_node, &kctx->kbdev->process_root); + /* Add checks, so that the terminating process Should not + * hold any gpu_memory. + */ + WARN_ON(kprcs->total_gpu_pages); + WARN_ON(!RB_EMPTY_ROOT(&kprcs->dma_buf_root)); + kfree(kprcs); + } +} + +void kbase_context_common_term(struct kbase_context *kctx) +{ + unsigned long flags; + int pages; + + mutex_lock(&kctx->kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + kbase_ctx_sched_remove_ctx(kctx); + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); + mutex_unlock(&kctx->kbdev->mmu_hw_mutex); + + pages = atomic_read(&kctx->used_pages); + if (pages != 0) + dev_warn(kctx->kbdev->dev, + "%s: %d pages in use!\n", __func__, pages); + + WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0); + + mutex_lock(&kctx->kbdev->kctx_list_lock); + kbase_remove_kctx_from_process(kctx); + + KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kctx->kbdev, kctx->id); + + KBASE_TLSTREAM_TL_DEL_CTX(kctx->kbdev, kctx); + list_del(&kctx->kctx_list_link); + mutex_unlock(&kctx->kbdev->kctx_list_lock); + + KBASE_KTRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, 0u); + + /* Flush the timeline stream, so the user can see the termination + * tracepoints being fired. + * The "if" statement below is for optimization. It is safe to call + * kbase_timeline_streams_flush when timeline is disabled. + */ + if (atomic_read(&kctx->kbdev->timeline_flags) != 0) + kbase_timeline_streams_flush(kctx->kbdev->timeline); + + vfree(kctx); +} + +int kbase_context_mem_pool_group_init(struct kbase_context *kctx) +{ + return kbase_mem_pool_group_init(&kctx->mem_pools, + kctx->kbdev, + &kctx->kbdev->mem_pool_defaults, + &kctx->kbdev->mem_pools); +} + +void kbase_context_mem_pool_group_term(struct kbase_context *kctx) +{ + kbase_mem_pool_group_term(&kctx->mem_pools); +} + +int kbase_context_mmu_init(struct kbase_context *kctx) +{ + kbase_mmu_init(kctx->kbdev, + &kctx->mmu, kctx, + base_context_mmu_group_id_get(kctx->create_flags)); + + return 0; +} + +void kbase_context_mmu_term(struct kbase_context *kctx) +{ + kbase_mmu_term(kctx->kbdev, &kctx->mmu); +} + +int kbase_context_mem_alloc_page(struct kbase_context *kctx) +{ + struct page *p; + + p = kbase_mem_alloc_page(&kctx->mem_pools.small[KBASE_MEM_GROUP_SINK]); + if (!p) + return -ENOMEM; + + kctx->aliasing_sink_page = as_tagged(page_to_phys(p)); + + return 0; +} + +void kbase_context_mem_pool_free(struct kbase_context *kctx) +{ + /* drop the aliasing sink page now that it can't be mapped anymore */ + kbase_mem_pool_free( + &kctx->mem_pools.small[KBASE_MEM_GROUP_SINK], + as_page(kctx->aliasing_sink_page), + false); +} + +void kbase_context_sticky_resource_term(struct kbase_context *kctx) +{ + unsigned long pending_regions_to_clean; + + kbase_gpu_vm_lock(kctx); + kbase_sticky_resource_term(kctx); + + /* free pending region setups */ + pending_regions_to_clean = KBASE_COOKIE_MASK; + bitmap_andnot(&pending_regions_to_clean, &pending_regions_to_clean, + kctx->cookies, BITS_PER_LONG); + while (pending_regions_to_clean) { + unsigned int cookie = find_first_bit(&pending_regions_to_clean, + BITS_PER_LONG); + + if (!WARN_ON(!kctx->pending_regions[cookie])) { + dev_dbg(kctx->kbdev->dev, "Freeing pending unmapped region\n"); + kbase_mem_phy_alloc_put( + kctx->pending_regions[cookie]->cpu_alloc); + kbase_mem_phy_alloc_put( + kctx->pending_regions[cookie]->gpu_alloc); + kfree(kctx->pending_regions[cookie]); + + kctx->pending_regions[cookie] = NULL; + } + + bitmap_clear(&pending_regions_to_clean, cookie, 1); + } + kbase_gpu_vm_unlock(kctx); +} diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context.h b/drivers/gpu/arm/bifrost/context/mali_kbase_context.h new file mode 100755 index 000000000000..e4ed8944bdd2 --- /dev/null +++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context.h @@ -0,0 +1,157 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2017, 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + */ + +#ifndef _KBASE_CONTEXT_H_ +#define _KBASE_CONTEXT_H_ + +#include + +/** + * kbase_context_debugfs_init - Initialize the kctx platform + * specific debugfs + * + * @kctx: kbase context + * + * This initializes some debugfs interfaces specific to the platform the source + * is compiled for. + */ +void kbase_context_debugfs_init(struct kbase_context *const kctx); + +/** + * kbase_context_debugfs_term - Terminate the kctx platform + * specific debugfs + * + * @kctx: kbase context + * + * This terminates some debugfs interfaces specific to the platform the source + * is compiled for. + */ +void kbase_context_debugfs_term(struct kbase_context *const kctx); + +/** + * kbase_create_context() - Create a kernel base context. + * + * @kbdev: Object representing an instance of GPU platform device, + * allocated from the probe method of the Mali driver. + * @is_compat: Force creation of a 32-bit context + * @flags: Flags to set, which shall be any combination of + * BASEP_CONTEXT_CREATE_KERNEL_FLAGS. + * @api_version: Application program interface version, as encoded in + * a single integer by the KBASE_API_VERSION macro. + * @filp: Pointer to the struct file corresponding to device file + * /dev/malixx instance, passed to the file's open method. + * + * Up to one context can be created for each client that opens the device file + * /dev/malixx. Context creation is deferred until a special ioctl() system call + * is made on the device file. Each context has its own GPU address space. + * + * Return: new kbase context or NULL on failure + */ +struct kbase_context * +kbase_create_context(struct kbase_device *kbdev, bool is_compat, + base_context_create_flags const flags, + unsigned long api_version, + struct file *filp); + +/** + * kbase_destroy_context - Destroy a kernel base context. + * @kctx: Context to destroy + * + * Will release all outstanding regions. + */ +void kbase_destroy_context(struct kbase_context *kctx); + +/** + * kbase_ctx_flag - Check if @flag is set on @kctx + * @kctx: Pointer to kbase context to check + * @flag: Flag to check + * + * Return: true if @flag is set on @kctx, false if not. + */ +static inline bool kbase_ctx_flag(struct kbase_context *kctx, + enum kbase_context_flags flag) +{ + return atomic_read(&kctx->flags) & flag; +} + +/** + * kbase_ctx_flag_clear - Clear @flag on @kctx + * @kctx: Pointer to kbase context + * @flag: Flag to clear + * + * Clear the @flag on @kctx. This is done atomically, so other flags being + * cleared or set at the same time will be safe. + * + * Some flags have locking requirements, check the documentation for the + * respective flags. + */ +static inline void kbase_ctx_flag_clear(struct kbase_context *kctx, + enum kbase_context_flags flag) +{ +#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE + /* + * Earlier kernel versions doesn't have atomic_andnot() or + * atomic_and(). atomic_clear_mask() was only available on some + * architectures and removed on arm in v3.13 on arm and arm64. + * + * Use a compare-exchange loop to clear the flag on pre 4.3 kernels, + * when atomic_andnot() becomes available. + */ + int old, new; + + do { + old = atomic_read(&kctx->flags); + new = old & ~flag; + + } while (atomic_cmpxchg(&kctx->flags, old, new) != old); +#else + atomic_andnot(flag, &kctx->flags); +#endif +} + +/** + * kbase_ctx_flag_set - Set @flag on @kctx + * @kctx: Pointer to kbase context + * @flag: Flag to set + * + * Set the @flag on @kctx. This is done atomically, so other flags being + * cleared or set at the same time will be safe. + * + * Some flags have locking requirements, check the documentation for the + * respective flags. + */ +static inline void kbase_ctx_flag_set(struct kbase_context *kctx, + enum kbase_context_flags flag) +{ + atomic_or(flag, &kctx->flags); +} +#endif /* _KBASE_CONTEXT_H_ */ diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context_internal.h b/drivers/gpu/arm/bifrost/context/mali_kbase_context_internal.h new file mode 100755 index 000000000000..818cdbea960d --- /dev/null +++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context_internal.h @@ -0,0 +1,60 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +#include + +typedef int kbase_context_init_method(struct kbase_context *kctx); +typedef void kbase_context_term_method(struct kbase_context *kctx); + +/** + * struct kbase_context_init - Device init/term methods. + * @init: Function pointer to a initialise method. + * @term: Function pointer to a terminate method. + * @err_mes: Error message to be printed when init method fails. + */ +struct kbase_context_init { + kbase_context_init_method *init; + kbase_context_term_method *term; + char *err_mes; +}; + +int kbase_context_common_init(struct kbase_context *kctx); +void kbase_context_common_term(struct kbase_context *kctx); + +int kbase_context_mem_pool_group_init(struct kbase_context *kctx); +void kbase_context_mem_pool_group_term(struct kbase_context *kctx); + +int kbase_context_mmu_init(struct kbase_context *kctx); +void kbase_context_mmu_term(struct kbase_context *kctx); + +int kbase_context_mem_alloc_page(struct kbase_context *kctx); +void kbase_context_mem_pool_free(struct kbase_context *kctx); + +void kbase_context_sticky_resource_term(struct kbase_context *kctx); diff --git a/drivers/gpu/arm/bifrost/csf/Kbuild b/drivers/gpu/arm/bifrost/csf/Kbuild new file mode 100755 index 000000000000..bb61811e6c85 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/Kbuild @@ -0,0 +1,40 @@ +# +# (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +mali_kbase-y += \ + csf/mali_kbase_csf_firmware_cfg.o \ + csf/mali_kbase_csf_trace_buffer.o \ + csf/mali_kbase_csf.o \ + csf/mali_kbase_csf_scheduler.o \ + csf/mali_kbase_csf_kcpu.o \ + csf/mali_kbase_csf_tiler_heap.o \ + csf/mali_kbase_csf_timeout.o \ + csf/mali_kbase_csf_tl_reader.o \ + csf/mali_kbase_csf_heap_context_alloc.o \ + csf/mali_kbase_csf_reset_gpu.o \ + csf/mali_kbase_csf_csg_debugfs.o \ + csf/mali_kbase_csf_kcpu_debugfs.o \ + csf/mali_kbase_csf_protected_memory.o \ + csf/mali_kbase_csf_tiler_heap_debugfs.o + +mali_kbase-$(CONFIG_MALI_REAL_HW) += csf/mali_kbase_csf_firmware.o + +mali_kbase-$(CONFIG_MALI_BIFROST_NO_MALI) += csf/mali_kbase_csf_firmware_no_mali.o diff --git a/drivers/gpu/arm/bifrost/csf/mali_base_csf_kernel.h b/drivers/gpu/arm/bifrost/csf/mali_base_csf_kernel.h new file mode 100755 index 000000000000..301146cbedd3 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_base_csf_kernel.h @@ -0,0 +1,598 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _BASE_CSF_KERNEL_H_ +#define _BASE_CSF_KERNEL_H_ + +/* Memory allocation, access/hint flags. + * + * See base_mem_alloc_flags. + */ + +/* IN */ +/* Read access CPU side + */ +#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) + +/* Write access CPU side + */ +#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) + +/* Read access GPU side + */ +#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) + +/* Write access GPU side + */ +#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) + +/* Execute allowed on the GPU side + */ +#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) + +/* Will be permanently mapped in kernel space. + * Flag is only allowed on allocations originating from kbase. + */ +#define BASEP_MEM_PERMANENT_KERNEL_MAPPING ((base_mem_alloc_flags)1 << 5) + +/* The allocation will completely reside within the same 4GB chunk in the GPU + * virtual space. + * Since this flag is primarily required only for the TLS memory which will + * not be used to contain executable code and also not used for Tiler heap, + * it can't be used along with BASE_MEM_PROT_GPU_EX and TILER_ALIGN_TOP flags. + */ +#define BASE_MEM_GPU_VA_SAME_4GB_PAGE ((base_mem_alloc_flags)1 << 6) + +/* Userspace is not allowed to free this memory. + * Flag is only allowed on allocations originating from kbase. + */ +#define BASEP_MEM_NO_USER_FREE ((base_mem_alloc_flags)1 << 7) + +#define BASE_MEM_RESERVED_BIT_8 ((base_mem_alloc_flags)1 << 8) + +/* Grow backing store on GPU Page Fault + */ +#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) + +/* Page coherence Outer shareable, if available + */ +#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) + +/* Page coherence Inner shareable + */ +#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) + +/* IN/OUT */ +/* Should be cached on the CPU, returned if actually cached + */ +#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) + +/* IN/OUT */ +/* Must have same VA on both the GPU and the CPU + */ +#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) + +/* OUT */ +/* Must call mmap to acquire a GPU address for the alloc + */ +#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) + +/* IN */ +/* Page coherence Outer shareable, required. + */ +#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) + +/* Protected memory + */ +#define BASE_MEM_PROTECTED ((base_mem_alloc_flags)1 << 16) + +/* Not needed physical memory + */ +#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) + +/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the + * addresses to be the same + */ +#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) + +/* CSF event memory + * + * If Outer shareable coherence is not specified or not available, then on + * allocation kbase will automatically use the uncached GPU mapping. + * There is no need for the client to specify BASE_MEM_UNCACHED_GPU + * themselves when allocating memory with the BASE_MEM_CSF_EVENT flag. + * + * This memory requires a permanent mapping + * + * See also kbase_reg_needs_kernel_mapping() + */ +#define BASE_MEM_CSF_EVENT ((base_mem_alloc_flags)1 << 19) + +#define BASE_MEM_RESERVED_BIT_20 ((base_mem_alloc_flags)1 << 20) + +/* Should be uncached on the GPU, will work only for GPUs using AARCH64 mmu + * mode. Some components within the GPU might only be able to access memory + * that is GPU cacheable. Refer to the specific GPU implementation for more + * details. The 3 shareability flags will be ignored for GPU uncached memory. + * If used while importing USER_BUFFER type memory, then the import will fail + * if the memory is not aligned to GPU and CPU cache line width. + */ +#define BASE_MEM_UNCACHED_GPU ((base_mem_alloc_flags)1 << 21) + +/* + * Bits [22:25] for group_id (0~15). + * + * base_mem_group_id_set() should be used to pack a memory group ID into a + * base_mem_alloc_flags value instead of accessing the bits directly. + * base_mem_group_id_get() should be used to extract the memory group ID from + * a base_mem_alloc_flags value. + */ +#define BASEP_MEM_GROUP_ID_SHIFT 22 +#define BASE_MEM_GROUP_ID_MASK \ + ((base_mem_alloc_flags)0xF << BASEP_MEM_GROUP_ID_SHIFT) + +/* Must do CPU cache maintenance when imported memory is mapped/unmapped + * on GPU. Currently applicable to dma-buf type only. + */ +#define BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP ((base_mem_alloc_flags)1 << 26) + +/* OUT */ +/* Kernel side cache sync ops required */ +#define BASE_MEM_KERNEL_SYNC ((base_mem_alloc_flags)1 << 28) + +/* Number of bits used as flags for base memory management + * + * Must be kept in sync with the base_mem_alloc_flags flags + */ +#define BASE_MEM_FLAGS_NR_BITS 29 + +/* A mask of all the flags which are only valid for allocations within kbase, + * and may not be passed from user space. + */ +#define BASEP_MEM_FLAGS_KERNEL_ONLY \ + (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE) + +/* A mask for all output bits, excluding IN/OUT bits. + */ +#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP + +/* A mask for all input bits, including IN/OUT bits. + */ +#define BASE_MEM_FLAGS_INPUT_MASK \ + (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) + +/* A mask of all currently reserved flags + */ +#define BASE_MEM_FLAGS_RESERVED \ + BASE_MEM_RESERVED_BIT_8 | BASE_MEM_RESERVED_BIT_20 + +#define BASEP_MEM_INVALID_HANDLE (0ull << 12) +#define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12) +#define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12) +#define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12) +#define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12) +/* reserved handles ..-47< for future special handles */ +#define BASEP_MEM_CSF_USER_REG_PAGE_HANDLE (47ul << 12) +#define BASEP_MEM_CSF_USER_IO_PAGES_HANDLE (48ul << 12) +#define BASE_MEM_COOKIE_BASE (64ul << 12) +#define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ + BASE_MEM_COOKIE_BASE) + +#define KBASE_CSF_NUM_USER_IO_PAGES_HANDLE \ + ((BASE_MEM_COOKIE_BASE - BASEP_MEM_CSF_USER_IO_PAGES_HANDLE) >> \ + LOCAL_PAGE_SHIFT) + +/** + * Valid set of just-in-time memory allocation flags + */ +#define BASE_JIT_ALLOC_VALID_FLAGS ((u8)0) + +/* Flags to pass to ::base_context_init. + * Flags can be ORed together to enable multiple things. + * + * These share the same space as BASEP_CONTEXT_FLAG_*, and so must + * not collide with them. + */ +typedef u32 base_context_create_flags; + +/* No flags set */ +#define BASE_CONTEXT_CREATE_FLAG_NONE ((base_context_create_flags)0) + +/* Base context is embedded in a cctx object (flag used for CINSTR + * software counter macros) + */ +#define BASE_CONTEXT_CCTX_EMBEDDED ((base_context_create_flags)1 << 0) + +/* Base context is a 'System Monitor' context for Hardware counters. + * + * One important side effect of this is that job submission is disabled. + */ +#define BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED \ + ((base_context_create_flags)1 << 1) + +/* Create CSF event thread. + * + * The creation of a CSF event thread is conditional and only allowed in + * unit tests for the moment, in order to avoid clashes with the existing + * Base unit tests. + */ +#define BASE_CONTEXT_CSF_EVENT_THREAD ((base_context_create_flags)1 << 2) + +/* Bit-shift used to encode a memory group ID in base_context_create_flags + */ +#define BASEP_CONTEXT_MMU_GROUP_ID_SHIFT (3) + +/* Bitmask used to encode a memory group ID in base_context_create_flags + */ +#define BASEP_CONTEXT_MMU_GROUP_ID_MASK \ + ((base_context_create_flags)0xF << BASEP_CONTEXT_MMU_GROUP_ID_SHIFT) + +/* Bitpattern describing the base_context_create_flags that can be + * passed to the kernel + */ +#define BASEP_CONTEXT_CREATE_KERNEL_FLAGS \ + (BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED | \ + BASEP_CONTEXT_MMU_GROUP_ID_MASK) + +/* Bitpattern describing the ::base_context_create_flags that can be + * passed to base_context_init() + */ +#define BASEP_CONTEXT_CREATE_ALLOWED_FLAGS \ + (BASE_CONTEXT_CCTX_EMBEDDED | \ + BASE_CONTEXT_CSF_EVENT_THREAD | \ + BASEP_CONTEXT_CREATE_KERNEL_FLAGS) + +/* Enable additional tracepoints for latency measurements (TL_ATOM_READY, + * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) + */ +#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) + +/* Indicate that job dumping is enabled. This could affect certain timers + * to account for the performance impact. + */ +#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) + +/* Enable KBase tracepoints for CSF builds */ +#define BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS (1 << 2) + +/* Enable additional CSF Firmware side tracepoints */ +#define BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS (1 << 3) + +#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ + BASE_TLSTREAM_JOB_DUMPING_ENABLED | \ + BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS | \ + BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) + +/* Number of pages mapped into the process address space for a bound GPU + * command queue. A pair of input/output pages and a Hw doorbell page + * are mapped to enable direct submission of commands to Hw. + */ +#define BASEP_QUEUE_NR_MMAP_USER_PAGES ((size_t)3) + +#define BASE_QUEUE_MAX_PRIORITY (15U) + +/* CQS Sync object is an array of u32 event_mem[2], error field index is 1 */ +#define BASEP_EVENT_VAL_INDEX (0U) +#define BASEP_EVENT_ERR_INDEX (1U) + +/* The upper limit for number of objects that could be waited/set per command. + * This limit is now enforced as internally the error inherit inputs are + * converted to 32-bit flags in a u32 variable occupying a previously padding + * field. + */ +#define BASEP_KCPU_CQS_MAX_NUM_OBJS ((size_t)32) + +/** + * enum base_kcpu_command_type - Kernel CPU queue command type. + */ +enum base_kcpu_command_type { + BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL, + BASE_KCPU_COMMAND_TYPE_FENCE_WAIT, + BASE_KCPU_COMMAND_TYPE_CQS_WAIT, + BASE_KCPU_COMMAND_TYPE_CQS_SET, + BASE_KCPU_COMMAND_TYPE_MAP_IMPORT, + BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT, + BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE, + BASE_KCPU_COMMAND_TYPE_JIT_ALLOC, + BASE_KCPU_COMMAND_TYPE_JIT_FREE, + BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND, + BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER, +}; + +/** + * enum base_queue_group_priority - Priority of a GPU Command Queue Group. + * @BASE_QUEUE_GROUP_PRIORITY_HIGH: GPU Command Queue Group is of high + * priority. + * @BASE_QUEUE_GROUP_PRIORITY_MEDIUM: GPU Command Queue Group is of medium + * priority. + * @BASE_QUEUE_GROUP_PRIORITY_LOW: GPU Command Queue Group is of low + * priority. + * @BASE_QUEUE_GROUP_PRIORITY_COUNT: Number of GPU Command Queue Group + * priority levels. + * + * Currently this is in order of highest to lowest, but if new levels are added + * then those new levels may be out of order to preserve the ABI compatibility + * with previous releases. At that point, ensure assignment to + * the 'priority' member in &kbase_queue_group is updated to ensure it remains + * a linear ordering. + * + * There should be no gaps in the enum, otherwise use of + * BASE_QUEUE_GROUP_PRIORITY_COUNT in kbase must be updated. + */ +enum base_queue_group_priority { + BASE_QUEUE_GROUP_PRIORITY_HIGH = 0, + BASE_QUEUE_GROUP_PRIORITY_MEDIUM, + BASE_QUEUE_GROUP_PRIORITY_LOW, + BASE_QUEUE_GROUP_PRIORITY_COUNT +}; + +struct base_kcpu_command_fence_info { + u64 fence; +}; + +struct base_cqs_wait { + u64 addr; + u32 val; + u32 padding; +}; + +struct base_kcpu_command_cqs_wait_info { + u64 objs; + u32 nr_objs; + u32 inherit_err_flags; +}; + +struct base_cqs_set { + u64 addr; +}; + +struct base_kcpu_command_cqs_set_info { + u64 objs; + u32 nr_objs; + u32 propagate_flags; +}; + +/** + * struct base_kcpu_command_import_info - structure which contains information + * about the imported buffer. + * + * @handle: Address of imported user buffer. + */ +struct base_kcpu_command_import_info { + u64 handle; +}; + +/** + * struct base_kcpu_command_jit_alloc_info - structure which contains + * information about jit memory allocation. + * + * @info: An array of elements of the + * struct base_jit_alloc_info type. + * @count: The number of elements in the info array. + * @padding: Padding to a multiple of 64 bits. + */ +struct base_kcpu_command_jit_alloc_info { + u64 info; + u8 count; + u8 padding[7]; +}; + +/** + * struct base_kcpu_command_jit_free_info - structure which contains + * information about jit memory which is to be freed. + * + * @ids: An array containing the JIT IDs to free. + * @count: The number of elements in the ids array. + * @padding: Padding to a multiple of 64 bits. + */ +struct base_kcpu_command_jit_free_info { + u64 ids; + u8 count; + u8 padding[7]; +}; + +/** + * struct base_kcpu_command_group_suspend_info - structure which contains + * suspend buffer data captured for a suspended queue group. + * + * @buffer: Pointer to an array of elements of the type char. + * @size: Number of elements in the @buffer array. + * @group_handle: Handle to the mapping of command stream group. + * @padding: padding to a multiple of 64 bits. + */ +struct base_kcpu_command_group_suspend_info { + u64 buffer; + u32 size; + u8 group_handle; + u8 padding[3]; +}; + +/** + * struct base_kcpu_command - kcpu command. + * + * @type: type of the kcpu command, one enum base_kcpu_command_type + * @info: structure which contains information about the kcpu command; + * actual type is determined by @p type + * @padding: padding to a multiple of 64 bits + */ +struct base_kcpu_command { + u8 type; + u8 padding[sizeof(u64) - sizeof(u8)]; + union { + struct base_kcpu_command_fence_info fence; + struct base_kcpu_command_cqs_wait_info cqs_wait; + struct base_kcpu_command_cqs_set_info cqs_set; + struct base_kcpu_command_import_info import; + struct base_kcpu_command_jit_alloc_info jit_alloc; + struct base_kcpu_command_jit_free_info jit_free; + struct base_kcpu_command_group_suspend_info suspend_buf_copy; + u64 padding[2]; /* No sub-struct should be larger */ + } info; +}; + +/** + * struct basep_cs_stream_control - Command Stream interface capabilities. + * + * @features: Features of this stream + * @padding: Padding to a multiple of 64 bits. + */ +struct basep_cs_stream_control { + u32 features; + u32 padding; +}; + +/** + * struct basep_cs_group_control - Command Stream Group interface capabilities. + * + * @features: Features of this group + * @stream_num: Number of streams in this group + * @suspend_size: Size in bytes of the suspend buffer for this group + * @padding: Padding to a multiple of 64 bits. + */ +struct basep_cs_group_control { + u32 features; + u32 stream_num; + u32 suspend_size; + u32 padding; +}; + +/** + * struct base_gpu_queue_group_error_fatal_payload - Unrecoverable fault + * error information associated with GPU command queue group. + * + * @sideband: Additional information of the unrecoverable fault. + * @status: Unrecoverable fault information. + * This consists of exception type (least significant byte) and + * data (remaining bytes). One example of exception type is + * CS_INVALID_INSTRUCTION (0x49). + * @padding: Padding to make multiple of 64bits + */ +struct base_gpu_queue_group_error_fatal_payload { + u64 sideband; + u32 status; + u32 padding; +}; + +/** + * struct base_gpu_queue_error_fatal_payload - Unrecoverable fault + * error information related to GPU command queue. + * + * @sideband: Additional information about this unrecoverable fault. + * @status: Unrecoverable fault information. + * This consists of exception type (least significant byte) and + * data (remaining bytes). One example of exception type is + * CS_INVALID_INSTRUCTION (0x49). + * @csi_index: Index of the CSF interface the queue is bound to. + * @padding: Padding to make multiple of 64bits + */ +struct base_gpu_queue_error_fatal_payload { + u64 sideband; + u32 status; + u8 csi_index; + u8 padding[3]; +}; + +/** + * enum base_gpu_queue_group_error_type - GPU Fatal error type. + * + * @BASE_GPU_QUEUE_GROUP_ERROR_FATAL: Fatal error associated with GPU + * command queue group. + * @BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL: Fatal error associated with GPU + * command queue. + * @BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT: Fatal error associated with + * progress timeout. + * @BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM: Fatal error due to running out + * of tiler heap memory. + * @BASE_GPU_QUEUE_GROUP_ERROR_FATAL_COUNT: The number of fatal error types + * + * This type is used for &struct_base_gpu_queue_group_error.error_type. + */ +enum base_gpu_queue_group_error_type { + BASE_GPU_QUEUE_GROUP_ERROR_FATAL = 0, + BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL, + BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT, + BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM, + BASE_GPU_QUEUE_GROUP_ERROR_FATAL_COUNT +}; + +/** + * struct base_gpu_queue_group_error - Unrecoverable fault information + * + * @error_type: Error type of @base_gpu_queue_group_error_type + * indicating which field in union payload is filled + * @padding: Unused bytes for 64bit boundary + * @fatal_group: Unrecoverable fault error associated with + * GPU command queue group + * @fatal_queue: Unrecoverable fault error associated with command queue + * + * @payload: Input Payload + */ +struct base_gpu_queue_group_error { + u8 error_type; + u8 padding[7]; + union { + struct base_gpu_queue_group_error_fatal_payload fatal_group; + struct base_gpu_queue_error_fatal_payload fatal_queue; + } payload; +}; + +/** + * enum base_csf_notification_type - Notification type + * + * @BASE_CSF_NOTIFICATION_EVENT: Notification with kernel event + * @BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR: Notification with GPU fatal + * error + * @BASE_CSF_NOTIFICATION_COUNT: The number of notification type + * + * This type is used for &struct_base_csf_notification.type. + */ +enum base_csf_notification_type { + BASE_CSF_NOTIFICATION_EVENT = 0, + BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, + BASE_CSF_NOTIFICATION_COUNT +}; + +/** + * struct base_csf_notification - Event or error notification + * + * @type: Notification type of @base_csf_notification_type + * @padding: Padding for 64bit boundary + * @handle: Handle of GPU command queue group associated with fatal error + * @error: Unrecoverable fault error + * @align: To fit the struct into a 64-byte cache line + * + * @payload: Input Payload + */ +struct base_csf_notification { + u8 type; + u8 padding[7]; + union { + struct { + u8 handle; + u8 padding[7]; + struct base_gpu_queue_group_error error; + } csg_error; + u8 align[56]; + } payload; +}; + +#endif /* _BASE_CSF_KERNEL_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_control_registers.h b/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_control_registers.h new file mode 100755 index 000000000000..4fff80ca4023 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_control_registers.h @@ -0,0 +1,33 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * This header was autogenerated, it should not be edited. + */ + +#ifndef _GPU_CSF_CONTROL_REGISTERS_H_ +#define _GPU_CSF_CONTROL_REGISTERS_H_ + +/* GPU_REGISTERS register offsets */ +#define GPU_CONTROL_MCU 0x3000 /* () MCU control registers */ + +#endif /* _GPU_CSF_CONTROL_REGISTERS_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_registers.h b/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_registers.h new file mode 100755 index 000000000000..5c03445f3c79 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_gpu_csf_registers.h @@ -0,0 +1,1252 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * This header was autogenerated, it should not be edited. + */ + +#ifndef _GPU_CSF_REGISTERS_H_ +#define _GPU_CSF_REGISTERS_H_ + +/* + * Begin register sets + */ + +/* DOORBELLS base address */ +#define DOORBELLS_BASE 0x0080000 +#define DOORBELLS_REG(r) (DOORBELLS_BASE + (r)) + +/* CS_KERNEL_INPUT_BLOCK base address */ +#define CS_KERNEL_INPUT_BLOCK_BASE 0x0000 +#define CS_KERNEL_INPUT_BLOCK_REG(r) (CS_KERNEL_INPUT_BLOCK_BASE + (r)) + +/* CS_KERNEL_OUTPUT_BLOCK base address */ +#define CS_KERNEL_OUTPUT_BLOCK_BASE 0x0000 +#define CS_KERNEL_OUTPUT_BLOCK_REG(r) (CS_KERNEL_OUTPUT_BLOCK_BASE + (r)) + +/* CS_USER_INPUT_BLOCK base address */ +#define CS_USER_INPUT_BLOCK_BASE 0x0000 +#define CS_USER_INPUT_BLOCK_REG(r) (CS_USER_INPUT_BLOCK_BASE + (r)) + +/* CS_USER_OUTPUT_BLOCK base address */ +#define CS_USER_OUTPUT_BLOCK_BASE 0x0000 +#define CS_USER_OUTPUT_BLOCK_REG(r) (CS_USER_OUTPUT_BLOCK_BASE + (r)) + +/* CSG_INPUT_BLOCK base address */ +#define CSG_INPUT_BLOCK_BASE 0x0000 +#define CSG_INPUT_BLOCK_REG(r) (CSG_INPUT_BLOCK_BASE + (r)) + +/* CSG_OUTPUT_BLOCK base address */ +#define CSG_OUTPUT_BLOCK_BASE 0x0000 +#define CSG_OUTPUT_BLOCK_REG(r) (CSG_OUTPUT_BLOCK_BASE + (r)) + +/* GLB_CONTROL_BLOCK base address */ +#define GLB_CONTROL_BLOCK_BASE 0x04000000 +#define GLB_CONTROL_BLOCK_REG(r) (GLB_CONTROL_BLOCK_BASE + (r)) + +/* GLB_INPUT_BLOCK base address */ +#define GLB_INPUT_BLOCK_BASE 0x0000 +#define GLB_INPUT_BLOCK_REG(r) (GLB_INPUT_BLOCK_BASE + (r)) + +/* GLB_OUTPUT_BLOCK base address */ +#define GLB_OUTPUT_BLOCK_BASE 0x0000 +#define GLB_OUTPUT_BLOCK_REG(r) (GLB_OUTPUT_BLOCK_BASE + (r)) + +/* USER base address */ +#define USER_BASE 0x0010000 +#define USER_REG(r) (USER_BASE + (r)) + +/* End register sets */ + +/* + * Begin register offsets + */ + +/* DOORBELLS register offsets */ +#define DOORBELL_0 0x0000 /* () Doorbell 0 register */ +#define DOORBELL(n) (DOORBELL_0 + (n)*65536) +#define DOORBELL_REG(n, r) (DOORBELL(n) + DOORBELL_BLOCK_REG(r)) +#define DOORBELL_COUNT 1024 + +/* DOORBELL_BLOCK register offsets */ +#define DB_BLK_DOORBELL 0x0000 /* (WO) Doorbell request */ + +/* CS_KERNEL_INPUT_BLOCK register offsets */ +#define CS_REQ 0x0000 /* () Command stream request flags */ +#define CS_CONFIG 0x0004 /* () Command stream configuration */ +#define CS_ACK_IRQ_MASK 0x000C /* () Command steam interrupt mask */ +#define CS_BASE_LO 0x0010 /* () Base pointer for the ring buffer, low word */ +#define CS_BASE_HI 0x0014 /* () Base pointer for the ring buffer, high word */ +#define CS_SIZE 0x0018 /* () Size of the ring buffer */ +#define CS_TILER_HEAP_START_LO 0x0020 /* () Pointer to heap start, low word */ +#define CS_TILER_HEAP_START_HI 0x0024 /* () Pointer to heap start, high word */ +#define CS_TILER_HEAP_END_LO 0x0028 /* () Tiler heap descriptor address, low word */ +#define CS_TILER_HEAP_END_HI 0x002C /* () Tiler heap descriptor address, high word */ +#define CS_USER_INPUT_LO 0x0030 /* () CS user mode input page address, low word */ +#define CS_USER_INPUT_HI 0x0034 /* () CS user mode input page address, high word */ +#define CS_USER_OUTPUT_LO 0x0038 /* () CS user mode input page address, low word */ +#define CS_USER_OUTPUT_HI 0x003C /* () CS user mode input page address, high word */ + +/* CS_KERNEL_OUTPUT_BLOCK register offsets */ +#define CS_ACK 0x0000 /* () Command stream acknowledge flags */ +#define CS_STATUS_CMD_PTR_LO 0x0040 /* () Program pointer current value, low word */ +#define CS_STATUS_CMD_PTR_HI 0x0044 /* () Program pointer current value, high word */ +#define CS_STATUS_WAIT 0x0048 /* () Wait condition status register */ +#define CS_STATUS_REQ_RESOURCE 0x004C /* () Indicates the resources requested by the command stream */ +#define CS_STATUS_WAIT_SYNC_POINTER_LO 0x0050 /* () Sync object pointer, low word */ +#define CS_STATUS_WAIT_SYNC_POINTER_HI 0x0054 /* () Sync object pointer, high word */ +#define CS_STATUS_WAIT_SYNC_VALUE 0x0058 /* () Sync object test value */ +#define CS_FAULT 0x0080 /* () Recoverable fault information */ +#define CS_FATAL 0x0084 /* () Unrecoverable fault information */ +#define CS_FAULT_INFO_LO 0x0088 /* () Additional information about a recoverable fault, low word */ +#define CS_FAULT_INFO_HI 0x008C /* () Additional information about a recoverable fault, high word */ +#define CS_FATAL_INFO_LO 0x0090 /* () Additional information about a non-recoverable fault, low word */ +#define CS_FATAL_INFO_HI 0x0094 /* () Additional information about a non-recoverable fault, high word */ +#define CS_HEAP_VT_START 0x00C0 /* () Number of vertex/tiling operations started */ +#define CS_HEAP_VT_END 0x00C4 /* () Number of vertex/tiling operations completed */ +#define CS_HEAP_FRAG_END 0x00CC /* () Number of fragment completed */ +#define CS_HEAP_ADDRESS_LO 0x00D0 /* () Heap address, low word */ +#define CS_HEAP_ADDRESS_HI 0x00D4 /* () Heap address, high word */ + +/* CS_USER_INPUT_BLOCK register offsets */ +#define CS_INSERT_LO 0x0000 /* () Current insert offset for ring buffer, low word */ +#define CS_INSERT_HI 0x0004 /* () Current insert offset for ring buffer, high word */ +#define CS_EXTRACT_INIT_LO 0x0008 /* () Initial extract offset for ring buffer, low word */ +#define CS_EXTRACT_INIT_HI 0x000C /* () Initial extract offset for ring buffer, high word */ + +/* CS_USER_OUTPUT_BLOCK register offsets */ +#define CS_EXTRACT_LO 0x0000 /* () Current extract offset for ring buffer, low word */ +#define CS_EXTRACT_HI 0x0004 /* () Current extract offset for ring buffer, high word */ +#define CS_ACTIVE 0x0008 /* () Initial extract offset when the command stream is started */ + +/* CSG_INPUT_BLOCK register offsets */ +#define CSG_REQ 0x0000 /* () CSG request */ +#define CSG_ACK_IRQ_MASK 0x0004 /* () Global acknowledge interrupt mask */ +#define CSG_DB_REQ 0x0008 /* () Global doorbell request */ +#define CSG_IRQ_ACK 0x000C /* () Command stream IRQ acknowledge */ +#define CSG_ALLOW_COMPUTE_LO 0x0020 /* () Allowed compute endpoints, low word */ +#define CSG_ALLOW_COMPUTE_HI 0x0024 /* () Allowed compute endpoints, high word */ +#define CSG_ALLOW_FRAGMENT_LO 0x0028 /* () Allowed fragment endpoints, low word */ +#define CSG_ALLOW_FRAGMENT_HI 0x002C /* () Allowed fragment endpoints, high word */ +#define CSG_ALLOW_OTHER 0x0030 /* () Allowed other endpoints */ +#define CSG_EP_REQ 0x0034 /* () Maximum number of endpoints allowed */ +#define CSG_SUSPEND_BUF_LO 0x0040 /* () Normal mode suspend buffer, low word */ +#define CSG_SUSPEND_BUF_HI 0x0044 /* () Normal mode suspend buffer, high word */ +#define CSG_PROTM_SUSPEND_BUF_LO 0x0048 /* () Protected mode suspend buffer, low word */ +#define CSG_PROTM_SUSPEND_BUF_HI 0x004C /* () Protected mode suspend buffer, high word */ +#define CSG_CONFIG 0x0050 /* () CSG configuration options */ + +/* CSG_OUTPUT_BLOCK register offsets */ +#define CSG_ACK 0x0000 /* () Command stream group acknowledge flags */ +#define CSG_DB_ACK 0x0008 /* () Command stream kernel doorbell acknowledge flags */ +#define CSG_IRQ_REQ 0x000C /* () Command stream interrupt request flags */ +#define CSG_STATUS_EP_CURRENT 0x0010 /* () Endpoint allocation status register */ +#define CSG_STATUS_EP_REQ 0x0014 /* () Endpoint request status register */ +#define CSG_RESOURCE_DEP 0x001C /* () Current resource dependencies */ + +/* GLB_CONTROL_BLOCK register offsets */ +#define GLB_VERSION 0x0000 /* () Global interface version */ +#define GLB_FEATURES 0x0004 /* () Global interface features */ +#define GLB_INPUT_VA 0x0008 /* () Address of GLB_INPUT_BLOCK */ +#define GLB_OUTPUT_VA 0x000C /* () Address of GLB_OUTPUT_BLOCK */ +#define GLB_GROUP_NUM 0x0010 /* () Number of CSG interfaces */ +#define GLB_GROUP_STRIDE 0x0014 /* () Stride between CSG interfaces */ +#define GLB_PRFCNT_SIZE 0x0018 /* () Size of CSF performance counters */ +#define GROUP_CONTROL_0 0x1000 /* () CSG control and capabilities */ +#define GROUP_CONTROL(n) (GROUP_CONTROL_0 + (n)*256) +#define GROUP_CONTROL_REG(n, r) (GROUP_CONTROL(n) + GROUP_CONTROL_BLOCK_REG(r)) +#define GROUP_CONTROL_COUNT 16 + +/* STREAM_CONTROL_BLOCK register offsets */ +#define STREAM_FEATURES 0x0000 /* () Command Stream interface features */ +#define STREAM_INPUT_VA 0x0004 /* () Address of CS_KERNEL_INPUT_BLOCK */ +#define STREAM_OUTPUT_VA 0x0008 /* () Address of CS_KERNEL_OUTPUT_BLOCK */ + +/* GROUP_CONTROL_BLOCK register offsets */ +#define GROUP_FEATURES 0x0000 /* () Command Stream Group interface features */ +#define GROUP_INPUT_VA 0x0004 /* () Address of CSG_INPUT_BLOCK */ +#define GROUP_OUTPUT_VA 0x0008 /* () Address of CSG_OUTPUT_BLOCK */ +#define GROUP_SUSPEND_SIZE 0x000C /* () Size of CSG suspend buffer */ +#define GROUP_PROTM_SUSPEND_SIZE 0x0010 /* () Size of CSG protected-mode suspend buffer */ +#define GROUP_STREAM_NUM 0x0014 /* () Number of CS interfaces */ +#define GROUP_STREAM_STRIDE 0x0018 /* () Stride between CS interfaces */ +#define STREAM_CONTROL_0 0x0040 /* () CS control and capabilities */ +#define STREAM_CONTROL(n) (STREAM_CONTROL_0 + (n)*12) +#define STREAM_CONTROL_REG(n, r) (STREAM_CONTROL(n) + STREAM_CONTROL_BLOCK_REG(r)) +#define STREAM_CONTROL_COUNT 16 + +/* GLB_INPUT_BLOCK register offsets */ +#define GLB_REQ 0x0000 /* () Global request */ +#define GLB_ACK_IRQ_MASK 0x0004 /* () Global acknowledge interrupt mask */ +#define GLB_DB_REQ 0x0008 /* () Global doorbell request */ +#define GLB_PROGRESS_TIMER 0x0010 /* () Global progress timeout */ +#define GLB_PWROFF_TIMER 0x0014 /* () Global shader core power off timer */ +#define GLB_ALLOC_EN_LO 0x0018 /* () Global shader core allocation enable mask, low word */ +#define GLB_ALLOC_EN_HI 0x001C /* () Global shader core allocation enable mask, high word */ +#define GLB_PROTM_COHERENCY 0x0020 /* () Configure COHERENCY_ENABLE register value to use in protected mode execution */ + +#define GLB_PRFCNT_JASID 0x0024 /* () Performance counter address space */ +#define GLB_PRFCNT_BASE_LO 0x0028 /* () Performance counter buffer address, low word */ +#define GLB_PRFCNT_BASE_HI 0x002C /* () Performance counter buffer address, high word */ +#define GLB_PRFCNT_CONFIG 0x0040 /* () Performance counter configuration */ +#define GLB_PRFCNT_CSG_SELECT 0x0044 /* () CSG performance counting enable */ +#define GLB_PRFCNT_FW_EN 0x0048 /* () Performance counter enable for firmware */ +#define GLB_PRFCNT_CSG_EN 0x004C /* () Performance counter enable for CSG */ +#define GLB_PRFCNT_CSF_EN 0x0050 /* () Performance counter enable for CSF */ +#define GLB_PRFCNT_SHADER_EN 0x0054 /* () Performance counter enable for shader cores */ +#define GLB_PRFCNT_TILER_EN 0x0058 /* () Performance counter enable for tiler */ +#define GLB_PRFCNT_MMU_L2_EN 0x005C /* () Performance counter enable for MMU/L2 cache */ + +#define GLB_DEBUG_FWUTF_DESTROY 0x0FE0 /* () Test fixture destroy function address */ +#define GLB_DEBUG_FWUTF_TEST 0x0FE4 /* () Test index */ +#define GLB_DEBUG_FWUTF_FIXTURE 0x0FE8 /* () Test fixture index */ +#define GLB_DEBUG_FWUTF_CREATE 0x0FEC /* () Test fixture create function address */ +#define GLB_DEBUG_ACK_IRQ_MASK 0x0FF8 /* () Global debug acknowledge interrupt mask */ +#define GLB_DEBUG_REQ 0x0FFC /* () Global debug request */ + +/* GLB_OUTPUT_BLOCK register offsets */ +#define GLB_ACK 0x0000 /* () Global acknowledge */ +#define GLB_DB_ACK 0x0008 /* () Global doorbell acknowledge */ +#define GLB_HALT_STATUS 0x0010 /* () Global halt status */ +#define GLB_PRFCNT_STATUS 0x0014 /* () Performance counter status */ +#define GLB_DEBUG_FWUTF_RESULT 0x0FE0 /* () Firmware debug test result */ +#define GLB_DEBUG_ACK 0x0FFC /* () Global debug acknowledge */ + +/* End register offsets */ + +/* CS_KERNEL_INPUT_BLOCK register set definitions */ + +/* CS_REQ register */ +#define CS_REQ_STATE_SHIFT 0 +#define CS_REQ_STATE_MASK (0x7 << CS_REQ_STATE_SHIFT) +#define CS_REQ_STATE_GET(reg_val) (((reg_val)&CS_REQ_STATE_MASK) >> CS_REQ_STATE_SHIFT) +#define CS_REQ_STATE_SET(reg_val, value) \ + (((reg_val) & ~CS_REQ_STATE_MASK) | (((value) << CS_REQ_STATE_SHIFT) & CS_REQ_STATE_MASK)) +/* CS_REQ_STATE values */ +#define CS_REQ_STATE_STOP 0x0 +#define CS_REQ_STATE_START 0x1 +/* End of CS_REQ_STATE values */ +#define CS_REQ_EXTRACT_EVENT_SHIFT 4 +#define CS_REQ_EXTRACT_EVENT_MASK (0x1 << CS_REQ_EXTRACT_EVENT_SHIFT) +#define CS_REQ_EXTRACT_EVENT_GET(reg_val) (((reg_val)&CS_REQ_EXTRACT_EVENT_MASK) >> CS_REQ_EXTRACT_EVENT_SHIFT) +#define CS_REQ_EXTRACT_EVENT_SET(reg_val, value) \ + (((reg_val) & ~CS_REQ_EXTRACT_EVENT_MASK) | (((value) << CS_REQ_EXTRACT_EVENT_SHIFT) & CS_REQ_EXTRACT_EVENT_MASK)) + +/* From 10.x.5, CS_REQ_ERROR_MODE is removed but TI2 bitfile upload not finished. + * Need to remove on GPUCORE-23972 + */ +#define CS_REQ_ERROR_MODE_SHIFT 5 +#define CS_REQ_ERROR_MODE_MASK (0x1 << CS_REQ_ERROR_MODE_SHIFT) +#define CS_REQ_ERROR_MODE_GET(reg_val) ((reg_val & CS_REQ_ERROR_MODE_MASK) >> CS_REQ_ERROR_MODE_SHIFT) +#define CS_REQ_ERROR_MODE_SET(reg_val, value) \ + ((reg_val & ~CS_REQ_ERROR_MODE_MASK) | ((value << CS_REQ_ERROR_MODE_SHIFT) & CS_REQ_ERROR_MODE_MASK)) + +#define CS_REQ_IDLE_SYNC_WAIT_SHIFT 8 +#define CS_REQ_IDLE_SYNC_WAIT_MASK (0x1 << CS_REQ_IDLE_SYNC_WAIT_SHIFT) +#define CS_REQ_IDLE_SYNC_WAIT_GET(reg_val) (((reg_val)&CS_REQ_IDLE_SYNC_WAIT_MASK) >> CS_REQ_IDLE_SYNC_WAIT_SHIFT) +#define CS_REQ_IDLE_SYNC_WAIT_SET(reg_val, value) \ + (((reg_val) & ~CS_REQ_IDLE_SYNC_WAIT_MASK) | \ + (((value) << CS_REQ_IDLE_SYNC_WAIT_SHIFT) & CS_REQ_IDLE_SYNC_WAIT_MASK)) +#define CS_REQ_IDLE_PROTM_PEND_SHIFT 9 +#define CS_REQ_IDLE_PROTM_PEND_MASK (0x1 << CS_REQ_IDLE_PROTM_PEND_SHIFT) +#define CS_REQ_IDLE_PROTM_PEND_GET(reg_val) (((reg_val)&CS_REQ_IDLE_PROTM_PEND_MASK) >> CS_REQ_IDLE_PROTM_PEND_SHIFT) +#define CS_REQ_IDLE_PROTM_PEND_SET(reg_val, value) \ + (((reg_val) & ~CS_REQ_IDLE_PROTM_PEND_MASK) | \ + (((value) << CS_REQ_IDLE_PROTM_PEND_SHIFT) & CS_REQ_IDLE_PROTM_PEND_MASK)) +#define CS_REQ_IDLE_EMPTY_SHIFT 10 +#define CS_REQ_IDLE_EMPTY_MASK (0x1 << CS_REQ_IDLE_EMPTY_SHIFT) +#define CS_REQ_IDLE_EMPTY_GET(reg_val) (((reg_val)&CS_REQ_IDLE_EMPTY_MASK) >> CS_REQ_IDLE_EMPTY_SHIFT) +#define CS_REQ_IDLE_EMPTY_SET(reg_val, value) \ + (((reg_val) & ~CS_REQ_IDLE_EMPTY_MASK) | (((value) << CS_REQ_IDLE_EMPTY_SHIFT) & CS_REQ_IDLE_EMPTY_MASK)) +#define CS_REQ_IDLE_RESOURCE_REQ_SHIFT 11 +#define CS_REQ_IDLE_RESOURCE_REQ_MASK (0x1 << CS_REQ_IDLE_RESOURCE_REQ_SHIFT) +#define CS_REQ_IDLE_RESOURCE_REQ_GET(reg_val) \ + (((reg_val)&CS_REQ_IDLE_RESOURCE_REQ_MASK) >> CS_REQ_IDLE_RESOURCE_REQ_SHIFT) +#define CS_REQ_IDLE_RESOURCE_REQ_SET(reg_val, value) \ + (((reg_val) & ~CS_REQ_IDLE_RESOURCE_REQ_MASK) | \ + (((value) << CS_REQ_IDLE_RESOURCE_REQ_SHIFT) & CS_REQ_IDLE_RESOURCE_REQ_MASK)) +#define CS_REQ_TILER_OOM_SHIFT 26 +#define CS_REQ_TILER_OOM_MASK (0x1 << CS_REQ_TILER_OOM_SHIFT) +#define CS_REQ_TILER_OOM_GET(reg_val) (((reg_val)&CS_REQ_TILER_OOM_MASK) >> CS_REQ_TILER_OOM_SHIFT) +#define CS_REQ_TILER_OOM_SET(reg_val, value) \ + (((reg_val) & ~CS_REQ_TILER_OOM_MASK) | (((value) << CS_REQ_TILER_OOM_SHIFT) & CS_REQ_TILER_OOM_MASK)) +#define CS_REQ_PROTM_PEND_SHIFT 27 +#define CS_REQ_PROTM_PEND_MASK (0x1 << CS_REQ_PROTM_PEND_SHIFT) +#define CS_REQ_PROTM_PEND_GET(reg_val) (((reg_val)&CS_REQ_PROTM_PEND_MASK) >> CS_REQ_PROTM_PEND_SHIFT) +#define CS_REQ_PROTM_PEND_SET(reg_val, value) \ + (((reg_val) & ~CS_REQ_PROTM_PEND_MASK) | (((value) << CS_REQ_PROTM_PEND_SHIFT) & CS_REQ_PROTM_PEND_MASK)) +#define CS_REQ_FATAL_SHIFT 30 +#define CS_REQ_FATAL_MASK (0x1 << CS_REQ_FATAL_SHIFT) +#define CS_REQ_FATAL_GET(reg_val) (((reg_val)&CS_REQ_FATAL_MASK) >> CS_REQ_FATAL_SHIFT) +#define CS_REQ_FATAL_SET(reg_val, value) \ + (((reg_val) & ~CS_REQ_FATAL_MASK) | (((value) << CS_REQ_FATAL_SHIFT) & CS_REQ_FATAL_MASK)) +#define CS_REQ_FAULT_SHIFT 31 +#define CS_REQ_FAULT_MASK (0x1 << CS_REQ_FAULT_SHIFT) +#define CS_REQ_FAULT_GET(reg_val) (((reg_val)&CS_REQ_FAULT_MASK) >> CS_REQ_FAULT_SHIFT) +#define CS_REQ_FAULT_SET(reg_val, value) \ + (((reg_val) & ~CS_REQ_FAULT_MASK) | (((value) << CS_REQ_FAULT_SHIFT) & CS_REQ_FAULT_MASK)) + +/* CS_CONFIG register */ +#define CS_CONFIG_PRIORITY_SHIFT 0 +#define CS_CONFIG_PRIORITY_MASK (0xF << CS_CONFIG_PRIORITY_SHIFT) +#define CS_CONFIG_PRIORITY_GET(reg_val) (((reg_val)&CS_CONFIG_PRIORITY_MASK) >> CS_CONFIG_PRIORITY_SHIFT) +#define CS_CONFIG_PRIORITY_SET(reg_val, value) \ + (((reg_val) & ~CS_CONFIG_PRIORITY_MASK) | (((value) << CS_CONFIG_PRIORITY_SHIFT) & CS_CONFIG_PRIORITY_MASK)) +#define CS_CONFIG_USER_DOORBELL_SHIFT 8 +#define CS_CONFIG_USER_DOORBELL_MASK (0xFF << CS_CONFIG_USER_DOORBELL_SHIFT) +#define CS_CONFIG_USER_DOORBELL_GET(reg_val) (((reg_val)&CS_CONFIG_USER_DOORBELL_MASK) >> CS_CONFIG_USER_DOORBELL_SHIFT) +#define CS_CONFIG_USER_DOORBELL_SET(reg_val, value) \ + (((reg_val) & ~CS_CONFIG_USER_DOORBELL_MASK) | \ + (((value) << CS_CONFIG_USER_DOORBELL_SHIFT) & CS_CONFIG_USER_DOORBELL_MASK)) + +/* CS_ACK_IRQ_MASK register */ +#define CS_ACK_IRQ_MASK_STATE_SHIFT 0 +#define CS_ACK_IRQ_MASK_STATE_MASK (0x7 << CS_ACK_IRQ_MASK_STATE_SHIFT) +#define CS_ACK_IRQ_MASK_STATE_GET(reg_val) (((reg_val)&CS_ACK_IRQ_MASK_STATE_MASK) >> CS_ACK_IRQ_MASK_STATE_SHIFT) +#define CS_ACK_IRQ_MASK_STATE_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_IRQ_MASK_STATE_MASK) | \ + (((value) << CS_ACK_IRQ_MASK_STATE_SHIFT) & CS_ACK_IRQ_MASK_STATE_MASK)) +/* CS_ACK_IRQ_MASK_STATE values */ +#define CS_ACK_IRQ_MASK_STATE_DISABLED 0x0 +#define CS_ACK_IRQ_MASK_STATE_ENABLED 0x7 +/* End of CS_ACK_IRQ_MASK_STATE values */ +#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT 4 +#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK (0x1 << CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT) +#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_GET(reg_val) \ + (((reg_val)&CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK) >> CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT) +#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK) | \ + (((value) << CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT) & CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK)) +#define CS_ACK_IRQ_MASK_TILER_OOM_SHIFT 26 +#define CS_ACK_IRQ_MASK_TILER_OOM_MASK (0x1 << CS_ACK_IRQ_MASK_TILER_OOM_SHIFT) +#define CS_ACK_IRQ_MASK_TILER_OOM_GET(reg_val) \ + (((reg_val)&CS_ACK_IRQ_MASK_TILER_OOM_MASK) >> CS_ACK_IRQ_MASK_TILER_OOM_SHIFT) +#define CS_ACK_IRQ_MASK_TILER_OOM_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_IRQ_MASK_TILER_OOM_MASK) | \ + (((value) << CS_ACK_IRQ_MASK_TILER_OOM_SHIFT) & CS_ACK_IRQ_MASK_TILER_OOM_MASK)) +#define CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT 27 +#define CS_ACK_IRQ_MASK_PROTM_PEND_MASK (0x1 << CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT) +#define CS_ACK_IRQ_MASK_PROTM_PEND_GET(reg_val) \ + (((reg_val)&CS_ACK_IRQ_MASK_PROTM_PEND_MASK) >> CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT) +#define CS_ACK_IRQ_MASK_PROTM_PEND_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_IRQ_MASK_PROTM_PEND_MASK) | \ + (((value) << CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT) & CS_ACK_IRQ_MASK_PROTM_PEND_MASK)) +#define CS_ACK_IRQ_MASK_FATAL_SHIFT 30 +#define CS_ACK_IRQ_MASK_FATAL_MASK (0x1 << CS_ACK_IRQ_MASK_FATAL_SHIFT) +#define CS_ACK_IRQ_MASK_FATAL_GET(reg_val) (((reg_val)&CS_ACK_IRQ_MASK_FATAL_MASK) >> CS_ACK_IRQ_MASK_FATAL_SHIFT) +#define CS_ACK_IRQ_MASK_FATAL_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_IRQ_MASK_FATAL_MASK) | \ + (((value) << CS_ACK_IRQ_MASK_FATAL_SHIFT) & CS_ACK_IRQ_MASK_FATAL_MASK)) +#define CS_ACK_IRQ_MASK_FAULT_SHIFT 31 +#define CS_ACK_IRQ_MASK_FAULT_MASK (0x1 << CS_ACK_IRQ_MASK_FAULT_SHIFT) +#define CS_ACK_IRQ_MASK_FAULT_GET(reg_val) (((reg_val)&CS_ACK_IRQ_MASK_FAULT_MASK) >> CS_ACK_IRQ_MASK_FAULT_SHIFT) +#define CS_ACK_IRQ_MASK_FAULT_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_IRQ_MASK_FAULT_MASK) | \ + (((value) << CS_ACK_IRQ_MASK_FAULT_SHIFT) & CS_ACK_IRQ_MASK_FAULT_MASK)) + +/* CS_BASE register */ +#define CS_BASE_POINTER_SHIFT 0 +#define CS_BASE_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_BASE_POINTER_SHIFT) +#define CS_BASE_POINTER_GET(reg_val) (((reg_val)&CS_BASE_POINTER_MASK) >> CS_BASE_POINTER_SHIFT) +#define CS_BASE_POINTER_SET(reg_val, value) \ + (((reg_val) & ~CS_BASE_POINTER_MASK) | (((value) << CS_BASE_POINTER_SHIFT) & CS_BASE_POINTER_MASK)) + +/* CS_SIZE register */ +#define CS_SIZE_SIZE_SHIFT 0 +#define CS_SIZE_SIZE_MASK (0xFFFFFFFF << CS_SIZE_SIZE_SHIFT) +#define CS_SIZE_SIZE_GET(reg_val) (((reg_val)&CS_SIZE_SIZE_MASK) >> CS_SIZE_SIZE_SHIFT) +#define CS_SIZE_SIZE_SET(reg_val, value) \ + (((reg_val) & ~CS_SIZE_SIZE_MASK) | (((value) << CS_SIZE_SIZE_SHIFT) & CS_SIZE_SIZE_MASK)) + +/* CS_TILER_HEAP_START register */ +#define CS_TILER_HEAP_START_POINTER_SHIFT 0 +#define CS_TILER_HEAP_START_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_TILER_HEAP_START_POINTER_SHIFT) +#define CS_TILER_HEAP_START_POINTER_GET(reg_val) \ + (((reg_val)&CS_TILER_HEAP_START_POINTER_MASK) >> CS_TILER_HEAP_START_POINTER_SHIFT) +#define CS_TILER_HEAP_START_POINTER_SET(reg_val, value) \ + (((reg_val) & ~CS_TILER_HEAP_START_POINTER_MASK) | \ + (((value) << CS_TILER_HEAP_START_POINTER_SHIFT) & CS_TILER_HEAP_START_POINTER_MASK)) +/* HeapChunkPointer nested in CS_TILER_HEAP_START_POINTER */ +/* End of HeapChunkPointer nested in CS_TILER_HEAP_START_POINTER */ + +/* CS_TILER_HEAP_END register */ +#define CS_TILER_HEAP_END_POINTER_SHIFT 0 +#define CS_TILER_HEAP_END_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_TILER_HEAP_END_POINTER_SHIFT) +#define CS_TILER_HEAP_END_POINTER_GET(reg_val) \ + (((reg_val)&CS_TILER_HEAP_END_POINTER_MASK) >> CS_TILER_HEAP_END_POINTER_SHIFT) +#define CS_TILER_HEAP_END_POINTER_SET(reg_val, value) \ + (((reg_val) & ~CS_TILER_HEAP_END_POINTER_MASK) | \ + (((value) << CS_TILER_HEAP_END_POINTER_SHIFT) & CS_TILER_HEAP_END_POINTER_MASK)) +/* HeapChunkPointer nested in CS_TILER_HEAP_END_POINTER */ +/* End of HeapChunkPointer nested in CS_TILER_HEAP_END_POINTER */ + +/* CS_USER_INPUT register */ +#define CS_USER_INPUT_POINTER_SHIFT 0 +#define CS_USER_INPUT_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_USER_INPUT_POINTER_SHIFT) +#define CS_USER_INPUT_POINTER_GET(reg_val) (((reg_val)&CS_USER_INPUT_POINTER_MASK) >> CS_USER_INPUT_POINTER_SHIFT) +#define CS_USER_INPUT_POINTER_SET(reg_val, value) \ + (((reg_val) & ~CS_USER_INPUT_POINTER_MASK) | \ + (((value) << CS_USER_INPUT_POINTER_SHIFT) & CS_USER_INPUT_POINTER_MASK)) + +/* CS_USER_OUTPUT register */ +#define CS_USER_OUTPUT_POINTER_SHIFT 0 +#define CS_USER_OUTPUT_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_USER_OUTPUT_POINTER_SHIFT) +#define CS_USER_OUTPUT_POINTER_GET(reg_val) (((reg_val)&CS_USER_OUTPUT_POINTER_MASK) >> CS_USER_OUTPUT_POINTER_SHIFT) +#define CS_USER_OUTPUT_POINTER_SET(reg_val, value) \ + (((reg_val) & ~CS_USER_OUTPUT_POINTER_MASK) | \ + (((value) << CS_USER_OUTPUT_POINTER_SHIFT) & CS_USER_OUTPUT_POINTER_MASK)) +/* End of CS_KERNEL_INPUT_BLOCK register set definitions */ + +/* CS_KERNEL_OUTPUT_BLOCK register set definitions */ + +/* CS_ACK register */ +#define CS_ACK_STATE_SHIFT 0 +#define CS_ACK_STATE_MASK (0x7 << CS_ACK_STATE_SHIFT) +#define CS_ACK_STATE_GET(reg_val) (((reg_val)&CS_ACK_STATE_MASK) >> CS_ACK_STATE_SHIFT) +#define CS_ACK_STATE_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_STATE_MASK) | (((value) << CS_ACK_STATE_SHIFT) & CS_ACK_STATE_MASK)) +/* CS_ACK_STATE values */ +#define CS_ACK_STATE_STOP 0x0 +#define CS_ACK_STATE_START 0x1 +/* End of CS_ACK_STATE values */ +#define CS_ACK_EXTRACT_EVENT_SHIFT 4 +#define CS_ACK_EXTRACT_EVENT_MASK (0x1 << CS_ACK_EXTRACT_EVENT_SHIFT) +#define CS_ACK_EXTRACT_EVENT_GET(reg_val) (((reg_val)&CS_ACK_EXTRACT_EVENT_MASK) >> CS_ACK_EXTRACT_EVENT_SHIFT) +#define CS_ACK_EXTRACT_EVENT_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_EXTRACT_EVENT_MASK) | (((value) << CS_ACK_EXTRACT_EVENT_SHIFT) & CS_ACK_EXTRACT_EVENT_MASK)) +#define CS_ACK_TILER_OOM_SHIFT 26 +#define CS_ACK_TILER_OOM_MASK (0x1 << CS_ACK_TILER_OOM_SHIFT) +#define CS_ACK_TILER_OOM_GET(reg_val) (((reg_val)&CS_ACK_TILER_OOM_MASK) >> CS_ACK_TILER_OOM_SHIFT) +#define CS_ACK_TILER_OOM_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_TILER_OOM_MASK) | (((value) << CS_ACK_TILER_OOM_SHIFT) & CS_ACK_TILER_OOM_MASK)) +#define CS_ACK_PROTM_PEND_SHIFT 27 +#define CS_ACK_PROTM_PEND_MASK (0x1 << CS_ACK_PROTM_PEND_SHIFT) +#define CS_ACK_PROTM_PEND_GET(reg_val) (((reg_val)&CS_ACK_PROTM_PEND_MASK) >> CS_ACK_PROTM_PEND_SHIFT) +#define CS_ACK_PROTM_PEND_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_PROTM_PEND_MASK) | (((value) << CS_ACK_PROTM_PEND_SHIFT) & CS_ACK_PROTM_PEND_MASK)) +#define CS_ACK_FATAL_SHIFT 30 +#define CS_ACK_FATAL_MASK (0x1 << CS_ACK_FATAL_SHIFT) +#define CS_ACK_FATAL_GET(reg_val) (((reg_val)&CS_ACK_FATAL_MASK) >> CS_ACK_FATAL_SHIFT) +#define CS_ACK_FATAL_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_FATAL_MASK) | (((value) << CS_ACK_FATAL_SHIFT) & CS_ACK_FATAL_MASK)) +#define CS_ACK_FAULT_SHIFT 31 +#define CS_ACK_FAULT_MASK (0x1 << CS_ACK_FAULT_SHIFT) +#define CS_ACK_FAULT_GET(reg_val) (((reg_val)&CS_ACK_FAULT_MASK) >> CS_ACK_FAULT_SHIFT) +#define CS_ACK_FAULT_SET(reg_val, value) \ + (((reg_val) & ~CS_ACK_FAULT_MASK) | (((value) << CS_ACK_FAULT_SHIFT) & CS_ACK_FAULT_MASK)) + +/* CS_STATUS_CMD_PTR register */ +#define CS_STATUS_CMD_PTR_POINTER_SHIFT 0 +#define CS_STATUS_CMD_PTR_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_STATUS_CMD_PTR_POINTER_SHIFT) +#define CS_STATUS_CMD_PTR_POINTER_GET(reg_val) \ + (((reg_val)&CS_STATUS_CMD_PTR_POINTER_MASK) >> CS_STATUS_CMD_PTR_POINTER_SHIFT) +#define CS_STATUS_CMD_PTR_POINTER_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_CMD_PTR_POINTER_MASK) | \ + (((value) << CS_STATUS_CMD_PTR_POINTER_SHIFT) & CS_STATUS_CMD_PTR_POINTER_MASK)) + +/* CS_STATUS_WAIT register */ +#define CS_STATUS_WAIT_SB_MASK_SHIFT 0 +#define CS_STATUS_WAIT_SB_MASK_MASK (0xFFFF << CS_STATUS_WAIT_SB_MASK_SHIFT) +#define CS_STATUS_WAIT_SB_MASK_GET(reg_val) (((reg_val)&CS_STATUS_WAIT_SB_MASK_MASK) >> CS_STATUS_WAIT_SB_MASK_SHIFT) +#define CS_STATUS_WAIT_SB_MASK_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_WAIT_SB_MASK_MASK) | \ + (((value) << CS_STATUS_WAIT_SB_MASK_SHIFT) & CS_STATUS_WAIT_SB_MASK_MASK)) +#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT 24 +#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK (0xF << CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) +#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GET(reg_val) \ + (((reg_val)&CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK) >> CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) +#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK) | \ + (((value) << CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) & CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK)) +/* CS_STATUS_WAIT_SYNC_WAIT_CONDITION values */ +#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_LE 0x0 +#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GT 0x1 +/* End of CS_STATUS_WAIT_SYNC_WAIT_CONDITION values */ +#define CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT 28 +#define CS_STATUS_WAIT_PROGRESS_WAIT_MASK (0x1 << CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) +#define CS_STATUS_WAIT_PROGRESS_WAIT_GET(reg_val) \ + (((reg_val)&CS_STATUS_WAIT_PROGRESS_WAIT_MASK) >> CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) +#define CS_STATUS_WAIT_PROGRESS_WAIT_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_WAIT_PROGRESS_WAIT_MASK) | \ + (((value) << CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) & CS_STATUS_WAIT_PROGRESS_WAIT_MASK)) +#define CS_STATUS_WAIT_PROTM_PEND_SHIFT 29 +#define CS_STATUS_WAIT_PROTM_PEND_MASK (0x1 << CS_STATUS_WAIT_PROTM_PEND_SHIFT) +#define CS_STATUS_WAIT_PROTM_PEND_GET(reg_val) \ + (((reg_val)&CS_STATUS_WAIT_PROTM_PEND_MASK) >> CS_STATUS_WAIT_PROTM_PEND_SHIFT) +#define CS_STATUS_WAIT_PROTM_PEND_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_WAIT_PROTM_PEND_MASK) | \ + (((value) << CS_STATUS_WAIT_PROTM_PEND_SHIFT) & CS_STATUS_WAIT_PROTM_PEND_MASK)) +#define CS_STATUS_WAIT_SYNC_WAIT_SHIFT 31 +#define CS_STATUS_WAIT_SYNC_WAIT_MASK (0x1 << CS_STATUS_WAIT_SYNC_WAIT_SHIFT) +#define CS_STATUS_WAIT_SYNC_WAIT_GET(reg_val) \ + (((reg_val)&CS_STATUS_WAIT_SYNC_WAIT_MASK) >> CS_STATUS_WAIT_SYNC_WAIT_SHIFT) +#define CS_STATUS_WAIT_SYNC_WAIT_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_WAIT_SYNC_WAIT_MASK) | \ + (((value) << CS_STATUS_WAIT_SYNC_WAIT_SHIFT) & CS_STATUS_WAIT_SYNC_WAIT_MASK)) + +/* CS_STATUS_REQ_RESOURCE register */ +#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT 0 +#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT) +#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_GET(reg_val) \ + (((reg_val)&CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT) +#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK) | \ + (((value) << CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK)) +#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT 1 +#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT) +#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_GET(reg_val) \ + (((reg_val)&CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT) +#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK) | \ + (((value) << CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK)) +#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT 2 +#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT) +#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_GET(reg_val) \ + (((reg_val)&CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT) +#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK) | \ + (((value) << CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK)) +#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT 3 +#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT) +#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_GET(reg_val) \ + (((reg_val)&CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT) +#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK) | \ + (((value) << CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK)) + +/* CS_STATUS_WAIT_SYNC_POINTER register */ +#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT 0 +#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT) +#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_GET(reg_val) \ + (((reg_val)&CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK) >> CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT) +#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK) | \ + (((value) << CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT) & CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK)) + +/* CS_STATUS_WAIT_SYNC_VALUE register */ +#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT 0 +#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK (0xFFFFFFFF << CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT) +#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_GET(reg_val) \ + (((reg_val)&CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK) >> CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT) +#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_SET(reg_val, value) \ + (((reg_val) & ~CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK) | \ + (((value) << CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT) & CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK)) + +/* CS_FAULT register */ +#define CS_FAULT_EXCEPTION_TYPE_SHIFT 0 +#define CS_FAULT_EXCEPTION_TYPE_MASK (0xFF << CS_FAULT_EXCEPTION_TYPE_SHIFT) +#define CS_FAULT_EXCEPTION_TYPE_GET(reg_val) (((reg_val)&CS_FAULT_EXCEPTION_TYPE_MASK) >> CS_FAULT_EXCEPTION_TYPE_SHIFT) +#define CS_FAULT_EXCEPTION_TYPE_SET(reg_val, value) \ + (((reg_val) & ~CS_FAULT_EXCEPTION_TYPE_MASK) | \ + (((value) << CS_FAULT_EXCEPTION_TYPE_SHIFT) & CS_FAULT_EXCEPTION_TYPE_MASK)) +/* CS_FAULT_EXCEPTION_TYPE values */ +#define CS_FAULT_EXCEPTION_TYPE_CS_RESOURCE_TERMINATED 0x0F +#define CS_FAULT_EXCEPTION_TYPE_CS_INHERIT_FAULT 0x4B +#define CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_PC 0x50 +#define CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_ENC 0x51 +#define CS_FAULT_EXCEPTION_TYPE_INSTR_BARRIER_FAULT 0x55 +#define CS_FAULT_EXCEPTION_TYPE_DATA_INVALID_FAULT 0x58 +#define CS_FAULT_EXCEPTION_TYPE_TILE_RANGE_FAULT 0x59 +#define CS_FAULT_EXCEPTION_TYPE_ADDR_RANGE_FAULT 0x5A +#define CS_FAULT_EXCEPTION_TYPE_IMPRECISE_FAULT 0x5B +#define CS_FAULT_EXCEPTION_TYPE_RESOURCE_EVICTION_TIMEOUT 0x69 +/* End of CS_FAULT_EXCEPTION_TYPE values */ +#define CS_FAULT_EXCEPTION_DATA_SHIFT 8 +#define CS_FAULT_EXCEPTION_DATA_MASK (0xFFFFFF << CS_FAULT_EXCEPTION_DATA_SHIFT) +#define CS_FAULT_EXCEPTION_DATA_GET(reg_val) (((reg_val)&CS_FAULT_EXCEPTION_DATA_MASK) >> CS_FAULT_EXCEPTION_DATA_SHIFT) +#define CS_FAULT_EXCEPTION_DATA_SET(reg_val, value) \ + (((reg_val) & ~CS_FAULT_EXCEPTION_DATA_MASK) | \ + (((value) << CS_FAULT_EXCEPTION_DATA_SHIFT) & CS_FAULT_EXCEPTION_DATA_MASK)) + +/* CS_FATAL register */ +#define CS_FATAL_EXCEPTION_TYPE_SHIFT 0 +#define CS_FATAL_EXCEPTION_TYPE_MASK (0xFF << CS_FATAL_EXCEPTION_TYPE_SHIFT) +#define CS_FATAL_EXCEPTION_TYPE_GET(reg_val) (((reg_val)&CS_FATAL_EXCEPTION_TYPE_MASK) >> CS_FATAL_EXCEPTION_TYPE_SHIFT) +#define CS_FATAL_EXCEPTION_TYPE_SET(reg_val, value) \ + (((reg_val) & ~CS_FATAL_EXCEPTION_TYPE_MASK) | \ + (((value) << CS_FATAL_EXCEPTION_TYPE_SHIFT) & CS_FATAL_EXCEPTION_TYPE_MASK)) +/* CS_FATAL_EXCEPTION_TYPE values */ +#define CS_FATAL_EXCEPTION_TYPE_CS_CONFIG_FAULT 0x40 +#define CS_FATAL_EXCEPTION_TYPE_CS_ENDPOINT_FAULT 0x44 +#define CS_FATAL_EXCEPTION_TYPE_CS_BUS_FAULT 0x48 +#define CS_FATAL_EXCEPTION_TYPE_CS_INVALID_INSTRUCTION 0x49 +#define CS_FATAL_EXCEPTION_TYPE_CS_CALL_STACK_OVERFLOW 0x4A +#define CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR 0x68 +/* End of CS_FATAL_EXCEPTION_TYPE values */ +#define CS_FATAL_EXCEPTION_DATA_SHIFT 8 +#define CS_FATAL_EXCEPTION_DATA_MASK (0xFFFFFF << CS_FATAL_EXCEPTION_DATA_SHIFT) +#define CS_FATAL_EXCEPTION_DATA_GET(reg_val) (((reg_val)&CS_FATAL_EXCEPTION_DATA_MASK) >> CS_FATAL_EXCEPTION_DATA_SHIFT) +#define CS_FATAL_EXCEPTION_DATA_SET(reg_val, value) \ + (((reg_val) & ~CS_FATAL_EXCEPTION_DATA_MASK) | \ + (((value) << CS_FATAL_EXCEPTION_DATA_SHIFT) & CS_FATAL_EXCEPTION_DATA_MASK)) + +/* CS_FAULT_INFO register */ +#define CS_FAULT_INFO_EXCEPTION_DATA_SHIFT 0 +#define CS_FAULT_INFO_EXCEPTION_DATA_MASK (0xFFFFFFFFFFFFFFFF << CS_FAULT_INFO_EXCEPTION_DATA_SHIFT) +#define CS_FAULT_INFO_EXCEPTION_DATA_GET(reg_val) \ + (((reg_val)&CS_FAULT_INFO_EXCEPTION_DATA_MASK) >> CS_FAULT_INFO_EXCEPTION_DATA_SHIFT) +#define CS_FAULT_INFO_EXCEPTION_DATA_SET(reg_val, value) \ + (((reg_val) & ~CS_FAULT_INFO_EXCEPTION_DATA_MASK) | \ + (((value) << CS_FAULT_INFO_EXCEPTION_DATA_SHIFT) & CS_FAULT_INFO_EXCEPTION_DATA_MASK)) + +/* CS_FATAL_INFO register */ +#define CS_FATAL_INFO_EXCEPTION_DATA_SHIFT 0 +#define CS_FATAL_INFO_EXCEPTION_DATA_MASK (0xFFFFFFFFFFFFFFFF << CS_FATAL_INFO_EXCEPTION_DATA_SHIFT) +#define CS_FATAL_INFO_EXCEPTION_DATA_GET(reg_val) \ + (((reg_val)&CS_FATAL_INFO_EXCEPTION_DATA_MASK) >> CS_FATAL_INFO_EXCEPTION_DATA_SHIFT) +#define CS_FATAL_INFO_EXCEPTION_DATA_SET(reg_val, value) \ + (((reg_val) & ~CS_FATAL_INFO_EXCEPTION_DATA_MASK) | \ + (((value) << CS_FATAL_INFO_EXCEPTION_DATA_SHIFT) & CS_FATAL_INFO_EXCEPTION_DATA_MASK)) + +/* CS_HEAP_VT_START register */ +#define CS_HEAP_VT_START_VALUE_SHIFT 0 +#define CS_HEAP_VT_START_VALUE_MASK (0xFFFFFFFF << CS_HEAP_VT_START_VALUE_SHIFT) +#define CS_HEAP_VT_START_VALUE_GET(reg_val) (((reg_val)&CS_HEAP_VT_START_VALUE_MASK) >> CS_HEAP_VT_START_VALUE_SHIFT) +#define CS_HEAP_VT_START_VALUE_SET(reg_val, value) \ + (((reg_val) & ~CS_HEAP_VT_START_VALUE_MASK) | \ + (((value) << CS_HEAP_VT_START_VALUE_SHIFT) & CS_HEAP_VT_START_VALUE_MASK)) + +/* CS_HEAP_VT_END register */ +#define CS_HEAP_VT_END_VALUE_SHIFT 0 +#define CS_HEAP_VT_END_VALUE_MASK (0xFFFFFFFF << CS_HEAP_VT_END_VALUE_SHIFT) +#define CS_HEAP_VT_END_VALUE_GET(reg_val) (((reg_val)&CS_HEAP_VT_END_VALUE_MASK) >> CS_HEAP_VT_END_VALUE_SHIFT) +#define CS_HEAP_VT_END_VALUE_SET(reg_val, value) \ + (((reg_val) & ~CS_HEAP_VT_END_VALUE_MASK) | (((value) << CS_HEAP_VT_END_VALUE_SHIFT) & CS_HEAP_VT_END_VALUE_MASK)) + +/* CS_HEAP_FRAG_END register */ +#define CS_HEAP_FRAG_END_VALUE_SHIFT 0 +#define CS_HEAP_FRAG_END_VALUE_MASK (0xFFFFFFFF << CS_HEAP_FRAG_END_VALUE_SHIFT) +#define CS_HEAP_FRAG_END_VALUE_GET(reg_val) (((reg_val)&CS_HEAP_FRAG_END_VALUE_MASK) >> CS_HEAP_FRAG_END_VALUE_SHIFT) +#define CS_HEAP_FRAG_END_VALUE_SET(reg_val, value) \ + (((reg_val) & ~CS_HEAP_FRAG_END_VALUE_MASK) | \ + (((value) << CS_HEAP_FRAG_END_VALUE_SHIFT) & CS_HEAP_FRAG_END_VALUE_MASK)) + +/* CS_HEAP_ADDRESS register */ +#define CS_HEAP_ADDRESS_POINTER_SHIFT 0 +#define CS_HEAP_ADDRESS_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_HEAP_ADDRESS_POINTER_SHIFT) +#define CS_HEAP_ADDRESS_POINTER_GET(reg_val) (((reg_val)&CS_HEAP_ADDRESS_POINTER_MASK) >> CS_HEAP_ADDRESS_POINTER_SHIFT) +#define CS_HEAP_ADDRESS_POINTER_SET(reg_val, value) \ + (((reg_val) & ~CS_HEAP_ADDRESS_POINTER_MASK) | \ + (((value) << CS_HEAP_ADDRESS_POINTER_SHIFT) & CS_HEAP_ADDRESS_POINTER_MASK)) +/* End of CS_KERNEL_OUTPUT_BLOCK register set definitions */ + +/* CS_USER_INPUT_BLOCK register set definitions */ + +/* CS_INSERT register */ +#define CS_INSERT_VALUE_SHIFT 0 +#define CS_INSERT_VALUE_MASK (0xFFFFFFFFFFFFFFFF << CS_INSERT_VALUE_SHIFT) +#define CS_INSERT_VALUE_GET(reg_val) (((reg_val)&CS_INSERT_VALUE_MASK) >> CS_INSERT_VALUE_SHIFT) +#define CS_INSERT_VALUE_SET(reg_val, value) \ + (((reg_val) & ~CS_INSERT_VALUE_MASK) | (((value) << CS_INSERT_VALUE_SHIFT) & CS_INSERT_VALUE_MASK)) + +/* CS_EXTRACT_INIT register */ +#define CS_EXTRACT_INIT_VALUE_SHIFT 0 +#define CS_EXTRACT_INIT_VALUE_MASK (0xFFFFFFFFFFFFFFFF << CS_EXTRACT_INIT_VALUE_SHIFT) +#define CS_EXTRACT_INIT_VALUE_GET(reg_val) (((reg_val)&CS_EXTRACT_INIT_VALUE_MASK) >> CS_EXTRACT_INIT_VALUE_SHIFT) +#define CS_EXTRACT_INIT_VALUE_SET(reg_val, value) \ + (((reg_val) & ~CS_EXTRACT_INIT_VALUE_MASK) | \ + (((value) << CS_EXTRACT_INIT_VALUE_SHIFT) & CS_EXTRACT_INIT_VALUE_MASK)) +/* End of CS_USER_INPUT_BLOCK register set definitions */ + +/* CS_USER_OUTPUT_BLOCK register set definitions */ + +/* CS_EXTRACT register */ +#define CS_EXTRACT_VALUE_SHIFT 0 +#define CS_EXTRACT_VALUE_MASK (0xFFFFFFFFFFFFFFFF << CS_EXTRACT_VALUE_SHIFT) +#define CS_EXTRACT_VALUE_GET(reg_val) (((reg_val)&CS_EXTRACT_VALUE_MASK) >> CS_EXTRACT_VALUE_SHIFT) +#define CS_EXTRACT_VALUE_SET(reg_val, value) \ + (((reg_val) & ~CS_EXTRACT_VALUE_MASK) | (((value) << CS_EXTRACT_VALUE_SHIFT) & CS_EXTRACT_VALUE_MASK)) + +/* CS_ACTIVE register */ +#define CS_ACTIVE_HW_ACTIVE_SHIFT 0 +#define CS_ACTIVE_HW_ACTIVE_MASK (0x1 << CS_ACTIVE_HW_ACTIVE_SHIFT) +#define CS_ACTIVE_HW_ACTIVE_GET(reg_val) (((reg_val)&CS_ACTIVE_HW_ACTIVE_MASK) >> CS_ACTIVE_HW_ACTIVE_SHIFT) +#define CS_ACTIVE_HW_ACTIVE_SET(reg_val, value) \ + (((reg_val) & ~CS_ACTIVE_HW_ACTIVE_MASK) | (((value) << CS_ACTIVE_HW_ACTIVE_SHIFT) & CS_ACTIVE_HW_ACTIVE_MASK)) +/* End of CS_USER_OUTPUT_BLOCK register set definitions */ + +/* CSG_INPUT_BLOCK register set definitions */ + +/* CSG_REQ register */ +#define CSG_REQ_STATE_SHIFT 0 +#define CSG_REQ_STATE_MASK (0x7 << CSG_REQ_STATE_SHIFT) +#define CSG_REQ_STATE_GET(reg_val) (((reg_val)&CSG_REQ_STATE_MASK) >> CSG_REQ_STATE_SHIFT) +#define CSG_REQ_STATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_REQ_STATE_MASK) | (((value) << CSG_REQ_STATE_SHIFT) & CSG_REQ_STATE_MASK)) +/* CSG_REQ_STATE values */ +#define CSG_REQ_STATE_TERMINATE 0x0 +#define CSG_REQ_STATE_START 0x1 +#define CSG_REQ_STATE_SUSPEND 0x2 +#define CSG_REQ_STATE_RESUME 0x3 +/* End of CSG_REQ_STATE values */ +#define CSG_REQ_EP_CFG_SHIFT 4 +#define CSG_REQ_EP_CFG_MASK (0x1 << CSG_REQ_EP_CFG_SHIFT) +#define CSG_REQ_EP_CFG_GET(reg_val) (((reg_val)&CSG_REQ_EP_CFG_MASK) >> CSG_REQ_EP_CFG_SHIFT) +#define CSG_REQ_EP_CFG_SET(reg_val, value) \ + (((reg_val) & ~CSG_REQ_EP_CFG_MASK) | (((value) << CSG_REQ_EP_CFG_SHIFT) & CSG_REQ_EP_CFG_MASK)) +#define CSG_REQ_STATUS_UPDATE_SHIFT 5 +#define CSG_REQ_STATUS_UPDATE_MASK (0x1 << CSG_REQ_STATUS_UPDATE_SHIFT) +#define CSG_REQ_STATUS_UPDATE_GET(reg_val) (((reg_val)&CSG_REQ_STATUS_UPDATE_MASK) >> CSG_REQ_STATUS_UPDATE_SHIFT) +#define CSG_REQ_STATUS_UPDATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_REQ_STATUS_UPDATE_MASK) | \ + (((value) << CSG_REQ_STATUS_UPDATE_SHIFT) & CSG_REQ_STATUS_UPDATE_MASK)) +#define CSG_REQ_SYNC_UPDATE_SHIFT 28 +#define CSG_REQ_SYNC_UPDATE_MASK (0x1 << CSG_REQ_SYNC_UPDATE_SHIFT) +#define CSG_REQ_SYNC_UPDATE_GET(reg_val) (((reg_val)&CSG_REQ_SYNC_UPDATE_MASK) >> CSG_REQ_SYNC_UPDATE_SHIFT) +#define CSG_REQ_SYNC_UPDATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_REQ_SYNC_UPDATE_MASK) | (((value) << CSG_REQ_SYNC_UPDATE_SHIFT) & CSG_REQ_SYNC_UPDATE_MASK)) +#define CSG_REQ_IDLE_SHIFT 29 +#define CSG_REQ_IDLE_MASK (0x1 << CSG_REQ_IDLE_SHIFT) +#define CSG_REQ_IDLE_GET(reg_val) (((reg_val)&CSG_REQ_IDLE_MASK) >> CSG_REQ_IDLE_SHIFT) +#define CSG_REQ_IDLE_SET(reg_val, value) \ + (((reg_val) & ~CSG_REQ_IDLE_MASK) | (((value) << CSG_REQ_IDLE_SHIFT) & CSG_REQ_IDLE_MASK)) +#define CSG_REQ_DOORBELL_SHIFT 30 +#define CSG_REQ_DOORBELL_MASK (0x1 << CSG_REQ_DOORBELL_SHIFT) +#define CSG_REQ_DOORBELL_GET(reg_val) (((reg_val)&CSG_REQ_DOORBELL_MASK) >> CSG_REQ_DOORBELL_SHIFT) +#define CSG_REQ_DOORBELL_SET(reg_val, value) \ + (((reg_val) & ~CSG_REQ_DOORBELL_MASK) | (((value) << CSG_REQ_DOORBELL_SHIFT) & CSG_REQ_DOORBELL_MASK)) +#define CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT 31 +#define CSG_REQ_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) +#define CSG_REQ_PROGRESS_TIMER_EVENT_GET(reg_val) \ + (((reg_val)&CSG_REQ_PROGRESS_TIMER_EVENT_MASK) >> CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) +#define CSG_REQ_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ + (((reg_val) & ~CSG_REQ_PROGRESS_TIMER_EVENT_MASK) | \ + (((value) << CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) & CSG_REQ_PROGRESS_TIMER_EVENT_MASK)) + +/* CSG_ACK_IRQ_MASK register */ +#define CSG_ACK_IRQ_MASK_STATE_SHIFT 0 +#define CSG_ACK_IRQ_MASK_STATE_MASK (0x7 << CSG_ACK_IRQ_MASK_STATE_SHIFT) +#define CSG_ACK_IRQ_MASK_STATE_GET(reg_val) (((reg_val)&CSG_ACK_IRQ_MASK_STATE_MASK) >> CSG_ACK_IRQ_MASK_STATE_SHIFT) +#define CSG_ACK_IRQ_MASK_STATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_IRQ_MASK_STATE_MASK) | \ + (((value) << CSG_ACK_IRQ_MASK_STATE_SHIFT) & CSG_ACK_IRQ_MASK_STATE_MASK)) +/* CSG_ACK_IRQ_MASK_STATE values */ +#define CSG_ACK_IRQ_MASK_STATE_DISABLED 0x0 +#define CSG_ACK_IRQ_MASK_STATE_ENABLED 0x7 +/* End of CSG_ACK_IRQ_MASK_STATE values */ +#define CSG_ACK_IRQ_MASK_EP_CFG_SHIFT 4 +#define CSG_ACK_IRQ_MASK_EP_CFG_MASK (0x1 << CSG_ACK_IRQ_MASK_EP_CFG_SHIFT) +#define CSG_ACK_IRQ_MASK_EP_CFG_GET(reg_val) (((reg_val)&CSG_ACK_IRQ_MASK_EP_CFG_MASK) >> CSG_ACK_IRQ_MASK_EP_CFG_SHIFT) +#define CSG_ACK_IRQ_MASK_EP_CFG_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_IRQ_MASK_EP_CFG_MASK) | \ + (((value) << CSG_ACK_IRQ_MASK_EP_CFG_SHIFT) & CSG_ACK_IRQ_MASK_EP_CFG_MASK)) +#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT 5 +#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK (0x1 << CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT) +#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_GET(reg_val) \ + (((reg_val)&CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK) >> CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT) +#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK) | \ + (((value) << CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT) & CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK)) +#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT 28 +#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK (0x1 << CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT) +#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_GET(reg_val) \ + (((reg_val)&CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK) >> CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT) +#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK) | \ + (((value) << CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT) & CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK)) +#define CSG_ACK_IRQ_MASK_IDLE_SHIFT 29 +#define CSG_ACK_IRQ_MASK_IDLE_MASK (0x1 << CSG_ACK_IRQ_MASK_IDLE_SHIFT) +#define CSG_ACK_IRQ_MASK_IDLE_GET(reg_val) (((reg_val)&CSG_ACK_IRQ_MASK_IDLE_MASK) >> CSG_ACK_IRQ_MASK_IDLE_SHIFT) +#define CSG_ACK_IRQ_MASK_IDLE_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_IRQ_MASK_IDLE_MASK) | \ + (((value) << CSG_ACK_IRQ_MASK_IDLE_SHIFT) & CSG_ACK_IRQ_MASK_IDLE_MASK)) +#define CSG_ACK_IRQ_MASK_DOORBELL_SHIFT 30 +#define CSG_ACK_IRQ_MASK_DOORBELL_MASK (0x1 << CSG_ACK_IRQ_MASK_DOORBELL_SHIFT) +#define CSG_ACK_IRQ_MASK_DOORBELL_GET(reg_val) \ + (((reg_val)&CSG_ACK_IRQ_MASK_DOORBELL_MASK) >> CSG_ACK_IRQ_MASK_DOORBELL_SHIFT) +#define CSG_ACK_IRQ_MASK_DOORBELL_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_IRQ_MASK_DOORBELL_MASK) | \ + (((value) << CSG_ACK_IRQ_MASK_DOORBELL_SHIFT) & CSG_ACK_IRQ_MASK_DOORBELL_MASK)) +#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT 31 +#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT) +#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_GET(reg_val) \ + (((reg_val)&CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK) >> CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT) +#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK) | \ + (((value) << CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT) & CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK)) + +/* CSG_EP_REQ register */ +#define CSG_EP_REQ_COMPUTE_EP_SHIFT 0 +#define CSG_EP_REQ_COMPUTE_EP_MASK (0xFF << CSG_EP_REQ_COMPUTE_EP_SHIFT) +#define CSG_EP_REQ_COMPUTE_EP_GET(reg_val) (((reg_val)&CSG_EP_REQ_COMPUTE_EP_MASK) >> CSG_EP_REQ_COMPUTE_EP_SHIFT) +#define CSG_EP_REQ_COMPUTE_EP_SET(reg_val, value) \ + (((reg_val) & ~CSG_EP_REQ_COMPUTE_EP_MASK) | \ + (((value) << CSG_EP_REQ_COMPUTE_EP_SHIFT) & CSG_EP_REQ_COMPUTE_EP_MASK)) +#define CSG_EP_REQ_FRAGMENT_EP_SHIFT 8 +#define CSG_EP_REQ_FRAGMENT_EP_MASK (0xFF << CSG_EP_REQ_FRAGMENT_EP_SHIFT) +#define CSG_EP_REQ_FRAGMENT_EP_GET(reg_val) (((reg_val)&CSG_EP_REQ_FRAGMENT_EP_MASK) >> CSG_EP_REQ_FRAGMENT_EP_SHIFT) +#define CSG_EP_REQ_FRAGMENT_EP_SET(reg_val, value) \ + (((reg_val) & ~CSG_EP_REQ_FRAGMENT_EP_MASK) | \ + (((value) << CSG_EP_REQ_FRAGMENT_EP_SHIFT) & CSG_EP_REQ_FRAGMENT_EP_MASK)) +#define CSG_EP_REQ_TILER_EP_SHIFT 16 +#define CSG_EP_REQ_TILER_EP_MASK (0xF << CSG_EP_REQ_TILER_EP_SHIFT) +#define CSG_EP_REQ_TILER_EP_GET(reg_val) (((reg_val)&CSG_EP_REQ_TILER_EP_MASK) >> CSG_EP_REQ_TILER_EP_SHIFT) +#define CSG_EP_REQ_TILER_EP_SET(reg_val, value) \ + (((reg_val) & ~CSG_EP_REQ_TILER_EP_MASK) | (((value) << CSG_EP_REQ_TILER_EP_SHIFT) & CSG_EP_REQ_TILER_EP_MASK)) +#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT 20 +#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK (0x1 << CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) +#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_GET(reg_val) \ + (((reg_val)&CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK) >> CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) +#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_SET(reg_val, value) \ + (((reg_val) & ~CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK) | \ + (((value) << CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) & CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK)) +#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT 21 +#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK (0x1 << CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) +#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_GET(reg_val) \ + (((reg_val)&CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) >> CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) +#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SET(reg_val, value) \ + (((reg_val) & ~CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) | \ + (((value) << CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) & CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK)) +#define CSG_EP_REQ_PRIORITY_SHIFT 28 +#define CSG_EP_REQ_PRIORITY_MASK (0xF << CSG_EP_REQ_PRIORITY_SHIFT) +#define CSG_EP_REQ_PRIORITY_GET(reg_val) (((reg_val)&CSG_EP_REQ_PRIORITY_MASK) >> CSG_EP_REQ_PRIORITY_SHIFT) +#define CSG_EP_REQ_PRIORITY_SET(reg_val, value) \ + (((reg_val) & ~CSG_EP_REQ_PRIORITY_MASK) | (((value) << CSG_EP_REQ_PRIORITY_SHIFT) & CSG_EP_REQ_PRIORITY_MASK)) + +/* CSG_SUSPEND_BUF register */ +#define CSG_SUSPEND_BUF_POINTER_SHIFT 0 +#define CSG_SUSPEND_BUF_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CSG_SUSPEND_BUF_POINTER_SHIFT) +#define CSG_SUSPEND_BUF_POINTER_GET(reg_val) (((reg_val)&CSG_SUSPEND_BUF_POINTER_MASK) >> CSG_SUSPEND_BUF_POINTER_SHIFT) +#define CSG_SUSPEND_BUF_POINTER_SET(reg_val, value) \ + (((reg_val) & ~CSG_SUSPEND_BUF_POINTER_MASK) | \ + (((value) << CSG_SUSPEND_BUF_POINTER_SHIFT) & CSG_SUSPEND_BUF_POINTER_MASK)) + +/* CSG_PROTM_SUSPEND_BUF register */ +#define CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT 0 +#define CSG_PROTM_SUSPEND_BUF_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT) +#define CSG_PROTM_SUSPEND_BUF_POINTER_GET(reg_val) \ + (((reg_val)&CSG_PROTM_SUSPEND_BUF_POINTER_MASK) >> CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT) +#define CSG_PROTM_SUSPEND_BUF_POINTER_SET(reg_val, value) \ + (((reg_val) & ~CSG_PROTM_SUSPEND_BUF_POINTER_MASK) | \ + (((value) << CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT) & CSG_PROTM_SUSPEND_BUF_POINTER_MASK)) + +/* End of CSG_INPUT_BLOCK register set definitions */ + +/* CSG_OUTPUT_BLOCK register set definitions */ + +/* CSG_ACK register */ +#define CSG_ACK_STATE_SHIFT 0 +#define CSG_ACK_STATE_MASK (0x7 << CSG_ACK_STATE_SHIFT) +#define CSG_ACK_STATE_GET(reg_val) (((reg_val)&CSG_ACK_STATE_MASK) >> CSG_ACK_STATE_SHIFT) +#define CSG_ACK_STATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_STATE_MASK) | (((value) << CSG_ACK_STATE_SHIFT) & CSG_ACK_STATE_MASK)) +/* CSG_ACK_STATE values */ +#define CSG_ACK_STATE_TERMINATE 0x0 +#define CSG_ACK_STATE_START 0x1 +#define CSG_ACK_STATE_SUSPEND 0x2 +#define CSG_ACK_STATE_RESUME 0x3 +/* End of CSG_ACK_STATE values */ +#define CSG_ACK_EP_CFG_SHIFT 4 +#define CSG_ACK_EP_CFG_MASK (0x1 << CSG_ACK_EP_CFG_SHIFT) +#define CSG_ACK_EP_CFG_GET(reg_val) (((reg_val)&CSG_ACK_EP_CFG_MASK) >> CSG_ACK_EP_CFG_SHIFT) +#define CSG_ACK_EP_CFG_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_EP_CFG_MASK) | (((value) << CSG_ACK_EP_CFG_SHIFT) & CSG_ACK_EP_CFG_MASK)) +#define CSG_ACK_STATUS_UPDATE_SHIFT 5 +#define CSG_ACK_STATUS_UPDATE_MASK (0x1 << CSG_ACK_STATUS_UPDATE_SHIFT) +#define CSG_ACK_STATUS_UPDATE_GET(reg_val) (((reg_val)&CSG_ACK_STATUS_UPDATE_MASK) >> CSG_ACK_STATUS_UPDATE_SHIFT) +#define CSG_ACK_STATUS_UPDATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_STATUS_UPDATE_MASK) | \ + (((value) << CSG_ACK_STATUS_UPDATE_SHIFT) & CSG_ACK_STATUS_UPDATE_MASK)) +#define CSG_ACK_SYNC_UPDATE_SHIFT 28 +#define CSG_ACK_SYNC_UPDATE_MASK (0x1 << CSG_ACK_SYNC_UPDATE_SHIFT) +#define CSG_ACK_SYNC_UPDATE_GET(reg_val) (((reg_val)&CSG_ACK_SYNC_UPDATE_MASK) >> CSG_ACK_SYNC_UPDATE_SHIFT) +#define CSG_ACK_SYNC_UPDATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_SYNC_UPDATE_MASK) | (((value) << CSG_ACK_SYNC_UPDATE_SHIFT) & CSG_ACK_SYNC_UPDATE_MASK)) +#define CSG_ACK_IDLE_SHIFT 29 +#define CSG_ACK_IDLE_MASK (0x1 << CSG_ACK_IDLE_SHIFT) +#define CSG_ACK_IDLE_GET(reg_val) (((reg_val)&CSG_ACK_IDLE_MASK) >> CSG_ACK_IDLE_SHIFT) +#define CSG_ACK_IDLE_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_IDLE_MASK) | (((value) << CSG_ACK_IDLE_SHIFT) & CSG_ACK_IDLE_MASK)) +#define CSG_ACK_DOORBELL_SHIFT 30 +#define CSG_ACK_DOORBELL_MASK (0x1 << CSG_ACK_DOORBELL_SHIFT) +#define CSG_ACK_DOORBELL_GET(reg_val) (((reg_val)&CSG_ACK_DOORBELL_MASK) >> CSG_ACK_DOORBELL_SHIFT) +#define CSG_ACK_DOORBELL_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_DOORBELL_MASK) | (((value) << CSG_ACK_DOORBELL_SHIFT) & CSG_ACK_DOORBELL_MASK)) +#define CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT 31 +#define CSG_ACK_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) +#define CSG_ACK_PROGRESS_TIMER_EVENT_GET(reg_val) \ + (((reg_val)&CSG_ACK_PROGRESS_TIMER_EVENT_MASK) >> CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) +#define CSG_ACK_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ + (((reg_val) & ~CSG_ACK_PROGRESS_TIMER_EVENT_MASK) | \ + (((value) << CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) & CSG_ACK_PROGRESS_TIMER_EVENT_MASK)) + +/* CSG_STATUS_EP_CURRENT register */ +#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT 0 +#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK (0xFF << CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT) +#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_GET(reg_val) \ + (((reg_val)&CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK) >> CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT) +#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_SET(reg_val, value) \ + (((reg_val) & ~CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK) | \ + (((value) << CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT) & CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK)) +#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT 8 +#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK (0xFF << CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT) +#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_GET(reg_val) \ + (((reg_val)&CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK) >> CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT) +#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SET(reg_val, value) \ + (((reg_val) & ~CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK) | \ + (((value) << CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT) & CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK)) +#define CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT 16 +#define CSG_STATUS_EP_CURRENT_TILER_EP_MASK (0xF << CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT) +#define CSG_STATUS_EP_CURRENT_TILER_EP_GET(reg_val) \ + (((reg_val)&CSG_STATUS_EP_CURRENT_TILER_EP_MASK) >> CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT) +#define CSG_STATUS_EP_CURRENT_TILER_EP_SET(reg_val, value) \ + (((reg_val) & ~CSG_STATUS_EP_CURRENT_TILER_EP_MASK) | \ + (((value) << CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT) & CSG_STATUS_EP_CURRENT_TILER_EP_MASK)) + +/* CSG_STATUS_EP_REQ register */ +#define CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT 0 +#define CSG_STATUS_EP_REQ_COMPUTE_EP_MASK (0xFF << CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT) +#define CSG_STATUS_EP_REQ_COMPUTE_EP_GET(reg_val) \ + (((reg_val)&CSG_STATUS_EP_REQ_COMPUTE_EP_MASK) >> CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT) +#define CSG_STATUS_EP_REQ_COMPUTE_EP_SET(reg_val, value) \ + (((reg_val) & ~CSG_STATUS_EP_REQ_COMPUTE_EP_MASK) | \ + (((value) << CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT) & CSG_STATUS_EP_REQ_COMPUTE_EP_MASK)) +#define CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT 8 +#define CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK (0xFF << CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT) +#define CSG_STATUS_EP_REQ_FRAGMENT_EP_GET(reg_val) \ + (((reg_val)&CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK) >> CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT) +#define CSG_STATUS_EP_REQ_FRAGMENT_EP_SET(reg_val, value) \ + (((reg_val) & ~CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK) | \ + (((value) << CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT) & CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK)) +#define CSG_STATUS_EP_REQ_TILER_EP_SHIFT 16 +#define CSG_STATUS_EP_REQ_TILER_EP_MASK (0xF << CSG_STATUS_EP_REQ_TILER_EP_SHIFT) +#define CSG_STATUS_EP_REQ_TILER_EP_GET(reg_val) \ + (((reg_val)&CSG_STATUS_EP_REQ_TILER_EP_MASK) >> CSG_STATUS_EP_REQ_TILER_EP_SHIFT) +#define CSG_STATUS_EP_REQ_TILER_EP_SET(reg_val, value) \ + (((reg_val) & ~CSG_STATUS_EP_REQ_TILER_EP_MASK) | \ + (((value) << CSG_STATUS_EP_REQ_TILER_EP_SHIFT) & CSG_STATUS_EP_REQ_TILER_EP_MASK)) +#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT 20 +#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK (0x1 << CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) +#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_GET(reg_val) \ + (((reg_val)&CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK) >> CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) +#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SET(reg_val, value) \ + (((reg_val) & ~CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK) | \ + (((value) << CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) & CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK)) +#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT 21 +#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK (0x1 << CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) +#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_GET(reg_val) \ + (((reg_val)&CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) >> CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) +#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SET(reg_val, value) \ + (((reg_val) & ~CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) | \ + (((value) << CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) & CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK)) + +/* End of CSG_OUTPUT_BLOCK register set definitions */ + +/* STREAM_CONTROL_BLOCK register set definitions */ + +/* STREAM_FEATURES register */ +#define STREAM_FEATURES_WORK_REGISTERS_SHIFT 0 +#define STREAM_FEATURES_WORK_REGISTERS_MASK (0xFF << STREAM_FEATURES_WORK_REGISTERS_SHIFT) +#define STREAM_FEATURES_WORK_REGISTERS_GET(reg_val) \ + (((reg_val)&STREAM_FEATURES_WORK_REGISTERS_MASK) >> STREAM_FEATURES_WORK_REGISTERS_SHIFT) +#define STREAM_FEATURES_WORK_REGISTERS_SET(reg_val, value) \ + (((reg_val) & ~STREAM_FEATURES_WORK_REGISTERS_MASK) | \ + (((value) << STREAM_FEATURES_WORK_REGISTERS_SHIFT) & STREAM_FEATURES_WORK_REGISTERS_MASK)) +#define STREAM_FEATURES_SCOREBOARDS_SHIFT 8 +#define STREAM_FEATURES_SCOREBOARDS_MASK (0xFF << STREAM_FEATURES_SCOREBOARDS_SHIFT) +#define STREAM_FEATURES_SCOREBOARDS_GET(reg_val) \ + (((reg_val)&STREAM_FEATURES_SCOREBOARDS_MASK) >> STREAM_FEATURES_SCOREBOARDS_SHIFT) +#define STREAM_FEATURES_SCOREBOARDS_SET(reg_val, value) \ + (((reg_val) & ~STREAM_FEATURES_SCOREBOARDS_MASK) | \ + (((value) << STREAM_FEATURES_SCOREBOARDS_SHIFT) & STREAM_FEATURES_SCOREBOARDS_MASK)) +#define STREAM_FEATURES_COMPUTE_SHIFT 16 +#define STREAM_FEATURES_COMPUTE_MASK (0x1 << STREAM_FEATURES_COMPUTE_SHIFT) +#define STREAM_FEATURES_COMPUTE_GET(reg_val) (((reg_val)&STREAM_FEATURES_COMPUTE_MASK) >> STREAM_FEATURES_COMPUTE_SHIFT) +#define STREAM_FEATURES_COMPUTE_SET(reg_val, value) \ + (((reg_val) & ~STREAM_FEATURES_COMPUTE_MASK) | \ + (((value) << STREAM_FEATURES_COMPUTE_SHIFT) & STREAM_FEATURES_COMPUTE_MASK)) +#define STREAM_FEATURES_FRAGMENT_SHIFT 17 +#define STREAM_FEATURES_FRAGMENT_MASK (0x1 << STREAM_FEATURES_FRAGMENT_SHIFT) +#define STREAM_FEATURES_FRAGMENT_GET(reg_val) \ + (((reg_val)&STREAM_FEATURES_FRAGMENT_MASK) >> STREAM_FEATURES_FRAGMENT_SHIFT) +#define STREAM_FEATURES_FRAGMENT_SET(reg_val, value) \ + (((reg_val) & ~STREAM_FEATURES_FRAGMENT_MASK) | \ + (((value) << STREAM_FEATURES_FRAGMENT_SHIFT) & STREAM_FEATURES_FRAGMENT_MASK)) +#define STREAM_FEATURES_TILER_SHIFT 18 +#define STREAM_FEATURES_TILER_MASK (0x1 << STREAM_FEATURES_TILER_SHIFT) +#define STREAM_FEATURES_TILER_GET(reg_val) (((reg_val)&STREAM_FEATURES_TILER_MASK) >> STREAM_FEATURES_TILER_SHIFT) +#define STREAM_FEATURES_TILER_SET(reg_val, value) \ + (((reg_val) & ~STREAM_FEATURES_TILER_MASK) | \ + (((value) << STREAM_FEATURES_TILER_SHIFT) & STREAM_FEATURES_TILER_MASK)) + +/* STREAM_INPUT_VA register */ +#define STREAM_INPUT_VA_VALUE_SHIFT 0 +#define STREAM_INPUT_VA_VALUE_MASK (0xFFFFFFFF << STREAM_INPUT_VA_VALUE_SHIFT) +#define STREAM_INPUT_VA_VALUE_GET(reg_val) (((reg_val)&STREAM_INPUT_VA_VALUE_MASK) >> STREAM_INPUT_VA_VALUE_SHIFT) +#define STREAM_INPUT_VA_VALUE_SET(reg_val, value) \ + (((reg_val) & ~STREAM_INPUT_VA_VALUE_MASK) | \ + (((value) << STREAM_INPUT_VA_VALUE_SHIFT) & STREAM_INPUT_VA_VALUE_MASK)) + +/* STREAM_OUTPUT_VA register */ +#define STREAM_OUTPUT_VA_VALUE_SHIFT 0 +#define STREAM_OUTPUT_VA_VALUE_MASK (0xFFFFFFFF << STREAM_OUTPUT_VA_VALUE_SHIFT) +#define STREAM_OUTPUT_VA_VALUE_GET(reg_val) (((reg_val)&STREAM_OUTPUT_VA_VALUE_MASK) >> STREAM_OUTPUT_VA_VALUE_SHIFT) +#define STREAM_OUTPUT_VA_VALUE_SET(reg_val, value) \ + (((reg_val) & ~STREAM_OUTPUT_VA_VALUE_MASK) | \ + (((value) << STREAM_OUTPUT_VA_VALUE_SHIFT) & STREAM_OUTPUT_VA_VALUE_MASK)) +/* End of STREAM_CONTROL_BLOCK register set definitions */ + +/* GLB_INPUT_BLOCK register set definitions */ + +/* GLB_REQ register */ +#define GLB_REQ_HALT_SHIFT 0 +#define GLB_REQ_HALT_MASK (0x1 << GLB_REQ_HALT_SHIFT) +#define GLB_REQ_HALT_GET(reg_val) (((reg_val)&GLB_REQ_HALT_MASK) >> GLB_REQ_HALT_SHIFT) +#define GLB_REQ_HALT_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_HALT_MASK) | (((value) << GLB_REQ_HALT_SHIFT) & GLB_REQ_HALT_MASK)) +#define GLB_REQ_CFG_PROGRESS_TIMER_SHIFT 1 +#define GLB_REQ_CFG_PROGRESS_TIMER_MASK (0x1 << GLB_REQ_CFG_PROGRESS_TIMER_SHIFT) +#define GLB_REQ_CFG_PROGRESS_TIMER_GET(reg_val) \ + (((reg_val)&GLB_REQ_CFG_PROGRESS_TIMER_MASK) >> GLB_REQ_CFG_PROGRESS_TIMER_SHIFT) +#define GLB_REQ_CFG_PROGRESS_TIMER_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_CFG_PROGRESS_TIMER_MASK) | \ + (((value) << GLB_REQ_CFG_PROGRESS_TIMER_SHIFT) & GLB_REQ_CFG_PROGRESS_TIMER_MASK)) +#define GLB_REQ_CFG_ALLOC_EN_SHIFT 2 +#define GLB_REQ_CFG_ALLOC_EN_MASK (0x1 << GLB_REQ_CFG_ALLOC_EN_SHIFT) +#define GLB_REQ_CFG_ALLOC_EN_GET(reg_val) (((reg_val)&GLB_REQ_CFG_ALLOC_EN_MASK) >> GLB_REQ_CFG_ALLOC_EN_SHIFT) +#define GLB_REQ_CFG_ALLOC_EN_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_CFG_ALLOC_EN_MASK) | (((value) << GLB_REQ_CFG_ALLOC_EN_SHIFT) & GLB_REQ_CFG_ALLOC_EN_MASK)) +#define GLB_REQ_CFG_PWROFF_TIMER_SHIFT 3 +#define GLB_REQ_CFG_PWROFF_TIMER_MASK (0x1 << GLB_REQ_CFG_PWROFF_TIMER_SHIFT) +#define GLB_REQ_CFG_PWROFF_TIMER_GET(reg_val) \ + (((reg_val)&GLB_REQ_CFG_PWROFF_TIMER_MASK) >> GLB_REQ_CFG_PWROFF_TIMER_SHIFT) +#define GLB_REQ_CFG_PWROFF_TIMER_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_CFG_PWROFF_TIMER_MASK) | \ + (((value) << GLB_REQ_CFG_PWROFF_TIMER_SHIFT) & GLB_REQ_CFG_PWROFF_TIMER_MASK)) +#define GLB_REQ_PROTM_ENTER_SHIFT 4 +#define GLB_REQ_PROTM_ENTER_MASK (0x1 << GLB_REQ_PROTM_ENTER_SHIFT) +#define GLB_REQ_PROTM_ENTER_GET(reg_val) (((reg_val)&GLB_REQ_PROTM_ENTER_MASK) >> GLB_REQ_PROTM_ENTER_SHIFT) +#define GLB_REQ_PROTM_ENTER_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_PROTM_ENTER_MASK) | (((value) << GLB_REQ_PROTM_ENTER_SHIFT) & GLB_REQ_PROTM_ENTER_MASK)) +#define GLB_REQ_PRFCNT_ENABLE_SHIFT 5 +#define GLB_REQ_PRFCNT_ENABLE_MASK (0x1 << GLB_REQ_PRFCNT_ENABLE_SHIFT) +#define GLB_REQ_PRFCNT_ENABLE_GET(reg_val) (((reg_val)&GLB_REQ_PRFCNT_ENABLE_MASK) >> GLB_REQ_PRFCNT_ENABLE_SHIFT) +#define GLB_REQ_PRFCNT_ENABLE_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_PRFCNT_ENABLE_MASK) | \ + (((value) << GLB_REQ_PRFCNT_ENABLE_SHIFT) & GLB_REQ_PRFCNT_ENABLE_MASK)) +#define GLB_REQ_PRFCNT_SAMPLE_SHIFT 6 +#define GLB_REQ_PRFCNT_SAMPLE_MASK (0x1 << GLB_REQ_PRFCNT_SAMPLE_SHIFT) +#define GLB_REQ_PRFCNT_SAMPLE_GET(reg_val) (((reg_val)&GLB_REQ_PRFCNT_SAMPLE_MASK) >> GLB_REQ_PRFCNT_SAMPLE_SHIFT) +#define GLB_REQ_PRFCNT_SAMPLE_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_PRFCNT_SAMPLE_MASK) | \ + (((value) << GLB_REQ_PRFCNT_SAMPLE_SHIFT) & GLB_REQ_PRFCNT_SAMPLE_MASK)) +#define GLB_REQ_COUNTER_ENABLE_SHIFT 7 +#define GLB_REQ_COUNTER_ENABLE_MASK (0x1 << GLB_REQ_COUNTER_ENABLE_SHIFT) +#define GLB_REQ_COUNTER_ENABLE_GET(reg_val) (((reg_val)&GLB_REQ_COUNTER_ENABLE_MASK) >> GLB_REQ_COUNTER_ENABLE_SHIFT) +#define GLB_REQ_COUNTER_ENABLE_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_COUNTER_ENABLE_MASK) | \ + (((value) << GLB_REQ_COUNTER_ENABLE_SHIFT) & GLB_REQ_COUNTER_ENABLE_MASK)) +#define GLB_REQ_PING_SHIFT 8 +#define GLB_REQ_PING_MASK (0x1 << GLB_REQ_PING_SHIFT) +#define GLB_REQ_PING_GET(reg_val) (((reg_val)&GLB_REQ_PING_MASK) >> GLB_REQ_PING_SHIFT) +#define GLB_REQ_PING_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_PING_MASK) | (((value) << GLB_REQ_PING_SHIFT) & GLB_REQ_PING_MASK)) +#define GLB_REQ_INACTIVE_COMPUTE_SHIFT 20 +#define GLB_REQ_INACTIVE_COMPUTE_MASK (0x1 << GLB_REQ_INACTIVE_COMPUTE_SHIFT) +#define GLB_REQ_INACTIVE_COMPUTE_GET(reg_val) \ + (((reg_val)&GLB_REQ_INACTIVE_COMPUTE_MASK) >> GLB_REQ_INACTIVE_COMPUTE_SHIFT) +#define GLB_REQ_INACTIVE_COMPUTE_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_INACTIVE_COMPUTE_MASK) | \ + (((value) << GLB_REQ_INACTIVE_COMPUTE_SHIFT) & GLB_REQ_INACTIVE_COMPUTE_MASK)) +#define GLB_REQ_INACTIVE_FRAGMENT_SHIFT 21 +#define GLB_REQ_INACTIVE_FRAGMENT_MASK (0x1 << GLB_REQ_INACTIVE_FRAGMENT_SHIFT) +#define GLB_REQ_INACTIVE_FRAGMENT_GET(reg_val) \ + (((reg_val)&GLB_REQ_INACTIVE_FRAGMENT_MASK) >> GLB_REQ_INACTIVE_FRAGMENT_SHIFT) +#define GLB_REQ_INACTIVE_FRAGMENT_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_INACTIVE_FRAGMENT_MASK) | \ + (((value) << GLB_REQ_INACTIVE_FRAGMENT_SHIFT) & GLB_REQ_INACTIVE_FRAGMENT_MASK)) +#define GLB_REQ_INACTIVE_TILER_SHIFT 22 +#define GLB_REQ_INACTIVE_TILER_MASK (0x1 << GLB_REQ_INACTIVE_TILER_SHIFT) +#define GLB_REQ_INACTIVE_TILER_GET(reg_val) (((reg_val)&GLB_REQ_INACTIVE_TILER_MASK) >> GLB_REQ_INACTIVE_TILER_SHIFT) +#define GLB_REQ_INACTIVE_TILER_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_INACTIVE_TILER_MASK) | \ + (((value) << GLB_REQ_INACTIVE_TILER_SHIFT) & GLB_REQ_INACTIVE_TILER_MASK)) +#define GLB_REQ_PROTM_EXIT_SHIFT 23 +#define GLB_REQ_PROTM_EXIT_MASK (0x1 << GLB_REQ_PROTM_EXIT_SHIFT) +#define GLB_REQ_PROTM_EXIT_GET(reg_val) (((reg_val)&GLB_REQ_PROTM_EXIT_MASK) >> GLB_REQ_PROTM_EXIT_SHIFT) +#define GLB_REQ_PROTM_EXIT_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_PROTM_EXIT_MASK) | (((value) << GLB_REQ_PROTM_EXIT_SHIFT) & GLB_REQ_PROTM_EXIT_MASK)) +#define GLB_REQ_DEBUG_CSF_REQ_SHIFT 30 +#define GLB_REQ_DEBUG_CSF_REQ_MASK (0x1 << GLB_REQ_DEBUG_CSF_REQ_SHIFT) +#define GLB_REQ_DEBUG_CSF_REQ_GET(reg_val) (((reg_val)&GLB_REQ_DEBUG_CSF_REQ_MASK) >> GLB_REQ_DEBUG_CSF_REQ_SHIFT) +#define GLB_REQ_DEBUG_CSF_REQ_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_DEBUG_CSF_REQ_MASK) | \ + (((value) << GLB_REQ_DEBUG_CSF_REQ_SHIFT) & GLB_REQ_DEBUG_CSF_REQ_MASK)) +#define GLB_REQ_DEBUG_HOST_REQ_SHIFT 31 +#define GLB_REQ_DEBUG_HOST_REQ_MASK (0x1 << GLB_REQ_DEBUG_HOST_REQ_SHIFT) +#define GLB_REQ_DEBUG_HOST_REQ_GET(reg_val) (((reg_val)&GLB_REQ_DEBUG_HOST_REQ_MASK) >> GLB_REQ_DEBUG_HOST_REQ_SHIFT) +#define GLB_REQ_DEBUG_HOST_REQ_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_DEBUG_HOST_REQ_MASK) | \ + (((value) << GLB_REQ_DEBUG_HOST_REQ_SHIFT) & GLB_REQ_DEBUG_HOST_REQ_MASK)) + +/* GLB_ACK_IRQ_MASK register */ +#define GLB_ACK_IRQ_MASK_HALT_SHIFT 0 +#define GLB_ACK_IRQ_MASK_HALT_MASK (0x1 << GLB_ACK_IRQ_MASK_HALT_SHIFT) +#define GLB_ACK_IRQ_MASK_HALT_GET(reg_val) (((reg_val)&GLB_ACK_IRQ_MASK_HALT_MASK) >> GLB_ACK_IRQ_MASK_HALT_SHIFT) +#define GLB_ACK_IRQ_MASK_HALT_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_HALT_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_HALT_SHIFT) & GLB_ACK_IRQ_MASK_HALT_MASK)) +#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT 1 +#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK) >> GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT) & GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK)) +#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT 2 +#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK) >> GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT) & GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK)) +#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT 3 +#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK) >> GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT) & GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK)) +#define GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT 4 +#define GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK (0x1 << GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT) +#define GLB_ACK_IRQ_MASK_PROTM_ENTER_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK) >> GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT) +#define GLB_ACK_IRQ_MASK_PROTM_ENTER_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT) & GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK)) +#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT 5 +#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK (0x1 << GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT) +#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK) >> GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT) +#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT) & GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK)) +#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT 6 +#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK (0x1 << GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT) +#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK) >> GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT) +#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT) & GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK)) +#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT 7 +#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK (0x1 << GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT) +#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK) >> GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT) +#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT) & GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK)) +#define GLB_ACK_IRQ_MASK_PING_SHIFT 8 +#define GLB_ACK_IRQ_MASK_PING_MASK (0x1 << GLB_ACK_IRQ_MASK_PING_SHIFT) +#define GLB_ACK_IRQ_MASK_PING_GET(reg_val) (((reg_val)&GLB_ACK_IRQ_MASK_PING_MASK) >> GLB_ACK_IRQ_MASK_PING_SHIFT) +#define GLB_ACK_IRQ_MASK_PING_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_PING_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_PING_SHIFT) & GLB_ACK_IRQ_MASK_PING_MASK)) +#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT 20 +#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) +#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK) >> GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) +#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) & GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK)) +#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT 21 +#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT) +#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK) >> GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT) +#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT) & GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK)) +#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT 22 +#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT) +#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK) >> GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT) +#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT) & GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK)) +#define GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT 23 +#define GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK (0x1 << GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT) +#define GLB_ACK_IRQ_MASK_PROTM_EXIT_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK) >> GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT) +#define GLB_ACK_IRQ_MASK_PROTM_EXIT_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT) & GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK)) +#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT 30 +#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK (0x1 << GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT) +#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK) >> GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT) +#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT) & GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK)) +#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT 31 +#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK (0x1 << GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT) +#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK) >> GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT) +#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT) & GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK)) + +/* GLB_PROGRESS_TIMER register */ +#define GLB_PROGRESS_TIMER_TIMEOUT_SHIFT 0 +#define GLB_PROGRESS_TIMER_TIMEOUT_MASK (0xFFFFFFFF << GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) +#define GLB_PROGRESS_TIMER_TIMEOUT_GET(reg_val) \ + (((reg_val)&GLB_PROGRESS_TIMER_TIMEOUT_MASK) >> GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) +#define GLB_PROGRESS_TIMER_TIMEOUT_SET(reg_val, value) \ + (((reg_val) & ~GLB_PROGRESS_TIMER_TIMEOUT_MASK) | \ + (((value) << GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) & GLB_PROGRESS_TIMER_TIMEOUT_MASK)) + +/* GLB_ALLOC_EN register */ +#define GLB_ALLOC_EN_MASK_SHIFT 0 +#define GLB_ALLOC_EN_MASK_MASK (0xFFFFFFFFFFFFFFFF << GLB_ALLOC_EN_MASK_SHIFT) +#define GLB_ALLOC_EN_MASK_GET(reg_val) (((reg_val)&GLB_ALLOC_EN_MASK_MASK) >> GLB_ALLOC_EN_MASK_SHIFT) +#define GLB_ALLOC_EN_MASK_SET(reg_val, value) \ + (((reg_val) & ~GLB_ALLOC_EN_MASK_MASK) | (((value) << GLB_ALLOC_EN_MASK_SHIFT) & GLB_ALLOC_EN_MASK_MASK)) + +/* GLB_PROTM_COHERENCY register */ +#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT 0 +#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK \ + (0xFFFFFFFF << GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT) +#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_GET(reg_val) \ + (((reg_val)&GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK) >> \ + GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT) +#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SET(reg_val, value) \ + (((reg_val) & ~GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK) | \ + (((value) << GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT) & \ + GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK)) +/* End of GLB_INPUT_BLOCK register set definitions */ + +/* GLB_OUTPUT_BLOCK register set definitions */ + +/* GLB_ACK register */ +#define GLB_ACK_CFG_PROGRESS_TIMER_SHIFT 1 +#define GLB_ACK_CFG_PROGRESS_TIMER_MASK (0x1 << GLB_ACK_CFG_PROGRESS_TIMER_SHIFT) +#define GLB_ACK_CFG_PROGRESS_TIMER_GET(reg_val) \ + (((reg_val)&GLB_ACK_CFG_PROGRESS_TIMER_MASK) >> GLB_ACK_CFG_PROGRESS_TIMER_SHIFT) +#define GLB_ACK_CFG_PROGRESS_TIMER_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_CFG_PROGRESS_TIMER_MASK) | \ + (((value) << GLB_ACK_CFG_PROGRESS_TIMER_SHIFT) & GLB_ACK_CFG_PROGRESS_TIMER_MASK)) +#define GLB_ACK_CFG_ALLOC_EN_SHIFT 2 +#define GLB_ACK_CFG_ALLOC_EN_MASK (0x1 << GLB_ACK_CFG_ALLOC_EN_SHIFT) +#define GLB_ACK_CFG_ALLOC_EN_GET(reg_val) (((reg_val)&GLB_ACK_CFG_ALLOC_EN_MASK) >> GLB_ACK_CFG_ALLOC_EN_SHIFT) +#define GLB_ACK_CFG_ALLOC_EN_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_CFG_ALLOC_EN_MASK) | (((value) << GLB_ACK_CFG_ALLOC_EN_SHIFT) & GLB_ACK_CFG_ALLOC_EN_MASK)) +/* End of GLB_OUTPUT_BLOCK register set definitions */ + +#endif /* _GPU_CSF_REGISTERS_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c new file mode 100755 index 000000000000..83d7513e78d9 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c @@ -0,0 +1,2547 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include "mali_kbase_csf.h" +#include "backend/gpu/mali_kbase_pm_internal.h" +#include +#include +#include "mali_gpu_csf_registers.h" +#include "mali_kbase_csf_tiler_heap.h" +#include +#include + +#define CS_REQ_EXCEPTION_MASK (CS_REQ_FAULT_MASK | CS_REQ_FATAL_MASK) +#define CS_ACK_EXCEPTION_MASK (CS_ACK_FAULT_MASK | CS_ACK_FATAL_MASK) + +/** + * struct kbase_csf_event - CSF event callback. + * + * This structure belongs to the list of events which is part of a Kbase + * context, and describes a callback function with a custom parameter to pass + * to it when a CSF event is signalled. + * + * @link: Link to the rest of the list. + * @kctx: Pointer to the Kbase context this event belongs to. + * @callback: Callback function to call when a CSF event is signalled. + * @param: Parameter to pass to the callback function. + */ +struct kbase_csf_event { + struct list_head link; + struct kbase_context *kctx; + kbase_csf_event_callback *callback; + void *param; +}; + +static void put_user_pages_mmap_handle(struct kbase_context *kctx, + struct kbase_queue *queue) +{ + unsigned long cookie_nr; + + lockdep_assert_held(&kctx->csf.lock); + + if (queue->handle == BASEP_MEM_INVALID_HANDLE) + return; + + cookie_nr = + PFN_DOWN(queue->handle - BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); + + if (!WARN_ON(kctx->csf.user_pages_info[cookie_nr] != queue)) { + /* free up cookie */ + kctx->csf.user_pages_info[cookie_nr] = NULL; + bitmap_set(kctx->csf.cookies, cookie_nr, 1); + } + + queue->handle = BASEP_MEM_INVALID_HANDLE; +} + +/* Reserve a cookie, to be returned as a handle to userspace for creating + * the CPU mapping of the pair of input/output pages and Hw doorbell page. + * Will return 0 in case of success otherwise negative on failure. + */ +static int get_user_pages_mmap_handle(struct kbase_context *kctx, + struct kbase_queue *queue) +{ + unsigned long cookie, cookie_nr; + + lockdep_assert_held(&kctx->csf.lock); + + if (bitmap_empty(kctx->csf.cookies, + KBASE_CSF_NUM_USER_IO_PAGES_HANDLE)) { + dev_err(kctx->kbdev->dev, + "No csf cookies available for allocation!"); + return -ENOMEM; + } + + /* allocate a cookie */ + cookie_nr = find_first_bit(kctx->csf.cookies, + KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); + if (kctx->csf.user_pages_info[cookie_nr]) { + dev_err(kctx->kbdev->dev, + "Inconsistent state of csf cookies!"); + return -EINVAL; + } + kctx->csf.user_pages_info[cookie_nr] = queue; + bitmap_clear(kctx->csf.cookies, cookie_nr, 1); + + /* relocate to correct base */ + cookie = cookie_nr + PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); + cookie <<= PAGE_SHIFT; + + queue->handle = (u64)cookie; + + return 0; +} + +static void gpu_munmap_user_io_pages(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + size_t num_pages = 2; + + kbase_mmu_teardown_pages(kctx->kbdev, &kctx->kbdev->csf.mcu_mmu, + reg->start_pfn, num_pages, MCU_AS_NR); + + WARN_ON(reg->flags & KBASE_REG_FREE); + + mutex_lock(&kctx->kbdev->csf.reg_lock); + kbase_remove_va_region(reg); + mutex_unlock(&kctx->kbdev->csf.reg_lock); +} + +static void init_user_output_page(struct kbase_queue *queue) +{ + u32 *addr = (u32 *)(queue->user_io_addr + PAGE_SIZE); + + addr[CS_EXTRACT_LO/4] = 0; + addr[CS_EXTRACT_HI/4] = 0; + + addr[CS_ACTIVE/4] = 0; +} + +/* Map the input/output pages in the shared interface segment of MCU firmware + * address space. + */ +static int gpu_mmap_user_io_pages(struct kbase_device *kbdev, + struct tagged_addr *phys, struct kbase_va_region *reg) +{ + unsigned long mem_flags = KBASE_REG_GPU_RD; + const size_t num_pages = 2; + int ret; + +#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ + ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ + (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) + mem_flags |= + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); +#else + if (kbdev->system_coherency == COHERENCY_NONE) { + mem_flags |= + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); + } else { + mem_flags |= KBASE_REG_SHARE_BOTH | + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED); + } +#endif + + mutex_lock(&kbdev->csf.reg_lock); + ret = kbase_add_va_region_rbtree(kbdev, reg, 0, num_pages, 1); + reg->flags &= ~KBASE_REG_FREE; + mutex_unlock(&kbdev->csf.reg_lock); + + if (ret) + return ret; + + /* Map input page */ + ret = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, + reg->start_pfn, &phys[0], + 1, mem_flags, MCU_AS_NR, + KBASE_MEM_GROUP_CSF_IO); + if (ret) + goto bad_insert; + + /* Map output page, it needs rw access */ + mem_flags |= KBASE_REG_GPU_WR; + ret = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, + reg->start_pfn + 1, &phys[1], + 1, mem_flags, MCU_AS_NR, + KBASE_MEM_GROUP_CSF_IO); + if (ret) + goto bad_insert_output_page; + + return 0; + +bad_insert_output_page: + kbase_mmu_teardown_pages(kbdev, &kbdev->csf.mcu_mmu, + reg->start_pfn, 1, MCU_AS_NR); +bad_insert: + mutex_lock(&kbdev->csf.reg_lock); + kbase_remove_va_region(reg); + mutex_unlock(&kbdev->csf.reg_lock); + + return ret; +} + +static void kernel_unmap_user_io_pages(struct kbase_context *kctx, + struct kbase_queue *queue) +{ + const size_t num_pages = 2; + + kbase_gpu_vm_lock(kctx); + + vunmap(queue->user_io_addr); + + WARN_ON(num_pages > atomic_read(&kctx->permanent_mapped_pages)); + atomic_sub(num_pages, &kctx->permanent_mapped_pages); + + kbase_gpu_vm_unlock(kctx); +} + +static int kernel_map_user_io_pages(struct kbase_context *kctx, + struct kbase_queue *queue) +{ + struct page *page_list[2]; + pgprot_t cpu_map_prot; + int ret = 0; + size_t i; + + kbase_gpu_vm_lock(kctx); + + if (ARRAY_SIZE(page_list) > (KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES - + atomic_read(&kctx->permanent_mapped_pages))) { + ret = -ENOMEM; + goto unlock; + } + + /* The pages are mapped to Userspace also, so use the same mapping + * attributes as used inside the CPU page fault handler. + */ +#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ + ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ + (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) + cpu_map_prot = pgprot_device(PAGE_KERNEL); +#else + if (kctx->kbdev->system_coherency == COHERENCY_NONE) + cpu_map_prot = pgprot_writecombine(PAGE_KERNEL); + else + cpu_map_prot = PAGE_KERNEL; +#endif + + for (i = 0; i < ARRAY_SIZE(page_list); i++) + page_list[i] = as_page(queue->phys[i]); + + queue->user_io_addr = vmap(page_list, ARRAY_SIZE(page_list), VM_MAP, cpu_map_prot); + + if (!queue->user_io_addr) + ret = -ENOMEM; + else + atomic_add(ARRAY_SIZE(page_list), &kctx->permanent_mapped_pages); + +unlock: + kbase_gpu_vm_unlock(kctx); + return ret; +} + +static void get_queue(struct kbase_queue *queue); +static void release_queue(struct kbase_queue *queue); + +/** + * kbase_csf_free_command_stream_user_pages() - Free the resources allocated + * for a queue at the time of bind. + * + * @kctx: Address of the kbase context within which the queue was created. + * @queue: Pointer to the queue to be unlinked. + * + * This function will free the pair of physical pages allocated for a GPU + * command queue, and also release the hardware doorbell page, that were mapped + * into the process address space to enable direct submission of commands to + * the hardware. Also releases the reference taken on the queue when the mapping + * was created. + * + * This function will be called only when the mapping is being removed and + * so the resources for queue will not get freed up until the mapping is + * removed even though userspace could have terminated the queue. + * Kernel will ensure that the termination of Kbase context would only be + * triggered after the mapping is removed. + * + * If an explicit or implicit unbind was missed by the userspace then the + * mapping will persist. On process exit kernel itself will remove the mapping. + */ +static void kbase_csf_free_command_stream_user_pages(struct kbase_context *kctx, + struct kbase_queue *queue) +{ + const size_t num_pages = 2; + + gpu_munmap_user_io_pages(kctx, queue->reg); + kernel_unmap_user_io_pages(kctx, queue); + + kbase_mem_pool_free_pages( + &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_IO], + num_pages, queue->phys, true, false); + + kfree(queue->reg); + queue->reg = NULL; + + /* If the queue has already been terminated by userspace + * then the ref count for queue object will drop to 0 here. + */ + release_queue(queue); +} + +int kbase_csf_alloc_command_stream_user_pages(struct kbase_context *kctx, + struct kbase_queue *queue) +{ + struct kbase_device *kbdev = kctx->kbdev; + struct kbase_va_region *reg; + const size_t num_pages = 2; + int ret; + + lockdep_assert_held(&kctx->csf.lock); + + reg = kbase_alloc_free_region(&kctx->kbdev->csf.shared_reg_rbtree, 0, + num_pages, KBASE_REG_ZONE_MCU_SHARED); + if (!reg) + return -ENOMEM; + + ret = kbase_mem_pool_alloc_pages( + &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_IO], + num_pages, queue->phys, false); + + if (ret != num_pages) + goto phys_alloc_failed; + + ret = kernel_map_user_io_pages(kctx, queue); + if (ret) + goto kernel_map_failed; + + init_user_output_page(queue); + + ret = gpu_mmap_user_io_pages(kctx->kbdev, queue->phys, reg); + if (ret) + goto gpu_mmap_failed; + + queue->reg = reg; + + mutex_lock(&kbdev->csf.reg_lock); + if (kbdev->csf.db_file_offsets > + (U32_MAX - BASEP_QUEUE_NR_MMAP_USER_PAGES + 1)) + kbdev->csf.db_file_offsets = 0; + + queue->db_file_offset = kbdev->csf.db_file_offsets; + kbdev->csf.db_file_offsets += BASEP_QUEUE_NR_MMAP_USER_PAGES; + + WARN(atomic_read(&queue->refcount) != 1, "Incorrect refcounting for queue object\n"); + /* This is the second reference taken on the queue object and + * would be dropped only when the IO mapping is removed either + * explicitly by userspace or implicitly by kernel on process exit. + */ + get_queue(queue); + queue->bind_state = KBASE_CSF_QUEUE_BOUND; + mutex_unlock(&kbdev->csf.reg_lock); + + return 0; + +gpu_mmap_failed: + kernel_unmap_user_io_pages(kctx, queue); + +kernel_map_failed: + kbase_mem_pool_free_pages( + &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_IO], + num_pages, queue->phys, false, false); + +phys_alloc_failed: + kfree(reg); + + return -ENOMEM; +} + +static struct kbase_queue_group *find_queue_group(struct kbase_context *kctx, + u8 group_handle) +{ + uint index = group_handle; + + lockdep_assert_held(&kctx->csf.lock); + + if (index < MAX_QUEUE_GROUP_NUM && kctx->csf.queue_groups[index]) { + if (WARN_ON(kctx->csf.queue_groups[index]->handle != index)) + return NULL; + return kctx->csf.queue_groups[index]; + } + + return NULL; +} + +int kbase_csf_queue_group_handle_is_valid(struct kbase_context *kctx, + u8 group_handle) +{ + struct kbase_queue_group *group; + + mutex_lock(&kctx->csf.lock); + group = find_queue_group(kctx, group_handle); + mutex_unlock(&kctx->csf.lock); + + return group ? 0 : -EINVAL; +} + +static struct kbase_queue *find_queue(struct kbase_context *kctx, u64 base_addr) +{ + struct kbase_queue *queue; + + lockdep_assert_held(&kctx->csf.lock); + + list_for_each_entry(queue, &kctx->csf.queue_list, link) { + if (base_addr == queue->base_addr) + return queue; + } + + return NULL; +} + +static void get_queue(struct kbase_queue *queue) +{ + WARN_ON(!atomic_inc_not_zero(&queue->refcount)); +} + +static void release_queue(struct kbase_queue *queue) +{ + lockdep_assert_held(&queue->kctx->csf.lock); + + WARN_ON(atomic_read(&queue->refcount) <= 0); + + if (atomic_dec_and_test(&queue->refcount)) { + /* The queue can't still be on the per context list. */ + WARN_ON(!list_empty(&queue->link)); + WARN_ON(queue->group); + kfree(queue); + } +} + +static void oom_event_worker(struct work_struct *data); +static void fault_event_worker(struct work_struct *data); + +int kbase_csf_queue_register(struct kbase_context *kctx, + struct kbase_ioctl_cs_queue_register *reg) +{ + struct kbase_queue *queue; + int ret = 0; + struct kbase_va_region *region; + u64 queue_addr = reg->buffer_gpu_addr; + size_t queue_size = reg->buffer_size >> PAGE_SHIFT; + + /* Validate the queue priority */ + if (reg->priority > BASE_QUEUE_MAX_PRIORITY) + return -EINVAL; + + mutex_lock(&kctx->csf.lock); + + /* Check if queue is already registered */ + if (find_queue(kctx, queue_addr) != NULL) { + ret = -EINVAL; + goto out; + } + + /* Check if the queue address is valid */ + kbase_gpu_vm_lock(kctx); + region = kbase_region_tracker_find_region_enclosing_address(kctx, + queue_addr); + + if (kbase_is_region_invalid_or_free(region)) { + ret = -ENOENT; + goto out_unlock_vm; + } + + if (queue_size > (region->nr_pages - + ((queue_addr >> PAGE_SHIFT) - region->start_pfn))) { + ret = -EINVAL; + goto out_unlock_vm; + } + + queue = kzalloc(sizeof(struct kbase_queue), GFP_KERNEL); + + if (!queue) { + ret = -ENOMEM; + goto out_unlock_vm; + } + + queue->kctx = kctx; + queue->base_addr = queue_addr; + queue->queue_reg = region; + queue->size = (queue_size << PAGE_SHIFT); + queue->csi_index = KBASEP_IF_NR_INVALID; + queue->enabled = false; + + queue->priority = reg->priority; + atomic_set(&queue->refcount, 1); + + queue->group = NULL; + queue->bind_state = KBASE_CSF_QUEUE_UNBOUND; + queue->handle = BASEP_MEM_INVALID_HANDLE; + queue->doorbell_nr = KBASEP_USER_DB_NR_INVALID; + + queue->status_wait = 0; + queue->sync_ptr = 0; + queue->sync_value = 0; + + INIT_LIST_HEAD(&queue->link); + INIT_LIST_HEAD(&queue->error.link); + INIT_WORK(&queue->oom_event_work, oom_event_worker); + INIT_WORK(&queue->fault_event_work, fault_event_worker); + list_add(&queue->link, &kctx->csf.queue_list); + + region->flags |= KBASE_REG_NO_USER_FREE; + +out_unlock_vm: + kbase_gpu_vm_unlock(kctx); +out: + mutex_unlock(&kctx->csf.lock); + + return ret; +} + +static void unbind_queue(struct kbase_context *kctx, + struct kbase_queue *queue); + +void kbase_csf_queue_terminate(struct kbase_context *kctx, + struct kbase_ioctl_cs_queue_terminate *term) +{ + struct kbase_queue *queue; + + mutex_lock(&kctx->csf.lock); + + queue = find_queue(kctx, term->buffer_gpu_addr); + + if (queue) { + /* As the GPU queue has been terminated by the + * user space, undo the actions that were performed when the + * queue was registered i.e. remove the queue from the per + * context list & release the initial reference. The subsequent + * lookups for the queue in find_queue() would fail. + */ + list_del_init(&queue->link); + + /* Stop the CSI to which queue was bound */ + unbind_queue(kctx, queue); + + kbase_gpu_vm_lock(kctx); + if (!WARN_ON(!queue->queue_reg)) { + /* After this the Userspace would be able to free the + * memory for GPU queue. In case the Userspace missed + * terminating the queue, the cleanup will happen on + * context termination where teardown of region tracker + * would free up the GPU queue memory. + */ + queue->queue_reg->flags &= ~KBASE_REG_NO_USER_FREE; + } + kbase_gpu_vm_unlock(kctx); + + /* Remove any pending command queue fatal from + * the per-context list. + */ + list_del_init(&queue->error.link); + + release_queue(queue); + } + + mutex_unlock(&kctx->csf.lock); +} + +int kbase_csf_queue_bind(struct kbase_context *kctx, union kbase_ioctl_cs_queue_bind *bind) +{ + struct kbase_queue *queue; + struct kbase_queue_group *group; + u8 max_streams; + int ret = -EINVAL; + + mutex_lock(&kctx->csf.lock); + + group = find_queue_group(kctx, bind->in.group_handle); + queue = find_queue(kctx, bind->in.buffer_gpu_addr); + + if (!group || !queue) + goto out; + + /* For the time being, all CSGs have the same number of CSs + * so we check CSG 0 for this number + */ + max_streams = kctx->kbdev->csf.global_iface.groups[0].stream_num; + + if (bind->in.csi_index >= max_streams) + goto out; + + if (group->run_state == KBASE_CSF_GROUP_TERMINATED) + goto out; + + if (queue->group || group->bound_queues[bind->in.csi_index]) + goto out; + + ret = get_user_pages_mmap_handle(kctx, queue); + if (ret) + goto out; + + bind->out.mmap_handle = queue->handle; + group->bound_queues[bind->in.csi_index] = queue; + queue->group = group; + queue->csi_index = bind->in.csi_index; + queue->bind_state = KBASE_CSF_QUEUE_BIND_IN_PROGRESS; + +out: + mutex_unlock(&kctx->csf.lock); + + return ret; +} + +static struct kbase_queue_group *get_bound_queue_group( + struct kbase_queue *queue) +{ + struct kbase_context *kctx = queue->kctx; + struct kbase_queue_group *group; + + if (queue->bind_state == KBASE_CSF_QUEUE_UNBOUND) + return NULL; + + if (!queue->group) + return NULL; + + if (queue->csi_index == KBASEP_IF_NR_INVALID) { + dev_warn(kctx->kbdev->dev, "CS interface index is incorrect\n"); + return NULL; + } + + group = queue->group; + + if (group->bound_queues[queue->csi_index] != queue) { + dev_warn(kctx->kbdev->dev, "Incorrect mapping between queues & queue groups\n"); + return NULL; + } + + return group; +} + +void kbase_csf_ring_csg_doorbell(struct kbase_device *kbdev, int slot) +{ + if (WARN_ON(slot < 0)) + return; + + kbase_csf_ring_csg_slots_doorbell(kbdev, (u32) (1 << slot)); +} + +void kbase_csf_ring_csg_slots_doorbell(struct kbase_device *kbdev, + u32 slot_bitmap) +{ + const struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + const u32 allowed_bitmap = + (u32) ((1U << kbdev->csf.global_iface.group_num) - 1); + u32 value; + + if (WARN_ON(slot_bitmap > allowed_bitmap)) + return; + + value = kbase_csf_firmware_global_output(global_iface, GLB_DB_ACK); + value ^= slot_bitmap; + kbase_csf_firmware_global_input_mask(global_iface, GLB_DB_REQ, value, + slot_bitmap); + + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); +} + +void kbase_csf_ring_cs_user_doorbell(struct kbase_device *kbdev, + struct kbase_queue *queue) +{ + mutex_lock(&kbdev->csf.reg_lock); + + if (queue->doorbell_nr != KBASEP_USER_DB_NR_INVALID) + kbase_csf_ring_doorbell(kbdev, queue->doorbell_nr); + + mutex_unlock(&kbdev->csf.reg_lock); +} + +void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, + struct kbase_queue *queue) +{ + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + struct kbase_queue_group *group = get_bound_queue_group(queue); + struct kbase_csf_cmd_stream_group_info *ginfo; + u32 value; + int slot; + + if (WARN_ON(!group)) + return; + + slot = kbase_csf_scheduler_group_get_slot(group); + + if (WARN_ON(slot < 0)) + return; + + ginfo = &global_iface->groups[slot]; + + value = kbase_csf_firmware_csg_output(ginfo, CSG_DB_ACK); + value ^= (1 << queue->csi_index); + kbase_csf_firmware_csg_input_mask(ginfo, CSG_DB_REQ, value, + 1 << queue->csi_index); + + kbase_csf_ring_csg_doorbell(kbdev, slot); +} + +int kbase_csf_queue_kick(struct kbase_context *kctx, + struct kbase_ioctl_cs_queue_kick *kick) +{ + struct kbase_queue_group *group; + struct kbase_queue *queue; + int err = 0; + + mutex_lock(&kctx->csf.lock); + + queue = find_queue(kctx, kick->buffer_gpu_addr); + if (!queue) + err = -EINVAL; + + if (!err) { + group = get_bound_queue_group(queue); + if (!group) { + dev_err(kctx->kbdev->dev, "queue not bound\n"); + err = -EINVAL; + } + } + + if (!err) + err = kbase_csf_scheduler_queue_start(queue); + + mutex_unlock(&kctx->csf.lock); + return err; +} + +static void unbind_stopped_queue(struct kbase_context *kctx, + struct kbase_queue *queue) +{ + lockdep_assert_held(&kctx->csf.lock); + + if (queue->bind_state != KBASE_CSF_QUEUE_UNBOUND) { + unsigned long flags; + + kbase_csf_scheduler_spin_lock(kctx->kbdev, &flags); + bitmap_clear(queue->group->protm_pending_bitmap, + queue->csi_index, 1); + queue->group->bound_queues[queue->csi_index] = NULL; + queue->group = NULL; + kbase_csf_scheduler_spin_unlock(kctx->kbdev, flags); + + put_user_pages_mmap_handle(kctx, queue); + queue->bind_state = KBASE_CSF_QUEUE_UNBOUND; + } +} +/** + * unbind_queue() - Remove the linkage between a GPU command queue and the group + * to which it was bound or being bound. + * + * @kctx: Address of the kbase context within which the queue was created. + * @queue: Pointer to the queue to be unlinked. + * + * This function will also send the stop request to firmware for the command + * stream if the group to which the GPU command queue was bound is scheduled. + * + * This function would be called when :- + * - queue is being unbound. This would happen when the IO mapping + * created on bind is removed explicitly by userspace or the process + * is getting exited. + * - queue group is being terminated which still has queues bound + * to it. This could happen on an explicit terminate request from userspace + * or when the kbase context is being terminated. + * - queue is being terminated without completing the bind operation. + * This could happen if either the queue group is terminated + * after the CS_QUEUE_BIND ioctl but before the 2nd part of bind operation + * to create the IO mapping is initiated. + * - There is a failure in executing the 2nd part of bind operation, inside the + * mmap handler, which creates the IO mapping for queue. + */ + +static void unbind_queue(struct kbase_context *kctx, struct kbase_queue *queue) +{ + lockdep_assert_held(&kctx->csf.lock); + + if (queue->bind_state != KBASE_CSF_QUEUE_UNBOUND) { + if (queue->bind_state == KBASE_CSF_QUEUE_BOUND) + kbase_csf_scheduler_queue_stop(queue); + + unbind_stopped_queue(kctx, queue); + } +} + +void kbase_csf_queue_unbind(struct kbase_queue *queue) +{ + struct kbase_context *kctx = queue->kctx; + + lockdep_assert_held(&kctx->csf.lock); + + unbind_queue(kctx, queue); + + /* Free the resources, if allocated for this queue. */ + if (queue->reg) + kbase_csf_free_command_stream_user_pages(kctx, queue); +} + +/** + * find_free_group_handle() - Find a free handle for a queue group + * + * @kctx: Address of the kbase context within which the queue group + * is to be created. + * + * Return: a queue group handle on success, or a negative error code on failure. + */ +static int find_free_group_handle(struct kbase_context *const kctx) +{ + /* find the available index in the array of CSGs per this context */ + int idx, group_handle = -ENOMEM; + + lockdep_assert_held(&kctx->csf.lock); + + for (idx = 0; + (idx != MAX_QUEUE_GROUP_NUM) && (group_handle < 0); + idx++) { + if (!kctx->csf.queue_groups[idx]) + group_handle = idx; + } + + return group_handle; +} + +/** + * iface_has_enough_streams() - Check that at least one command stream + * group supports a given number of streams + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @cs_min: Minimum number of command streams required. + * + * Return: true if at least one command stream group supports the given number + * of command streams (or more); otherwise false. + */ +static bool iface_has_enough_streams(struct kbase_device *const kbdev, + u32 const cs_min) +{ + bool has_enough = false; + struct kbase_csf_cmd_stream_group_info *const groups = + kbdev->csf.global_iface.groups; + const u32 group_num = kbdev->csf.global_iface.group_num; + u32 i; + + for (i = 0; (i < group_num) && !has_enough; i++) { + if (groups[i].stream_num >= cs_min) + has_enough = true; + } + + return has_enough; +} + +/** + * create_normal_suspend_buffer() - Create normal-mode suspend buffer per + * queue group + * + * @kctx: Pointer to kbase context where the queue group is created at + * @s_buf: Pointer to suspend buffer that is attached to queue group + * + * Return: 0 if suspend buffer is successfully allocated and reflected to GPU + * MMU page table. Otherwise -ENOMEM. + */ +static int create_normal_suspend_buffer(struct kbase_context *const kctx, + struct kbase_normal_suspend_buffer *s_buf) +{ + struct kbase_va_region *reg = NULL; + const unsigned long mem_flags = KBASE_REG_GPU_RD | KBASE_REG_GPU_WR; + const size_t nr_pages = + PFN_UP(kctx->kbdev->csf.global_iface.groups[0].suspend_size); + int err = 0; + + lockdep_assert_held(&kctx->csf.lock); + + /* Allocate and initialize Region Object */ + reg = kbase_alloc_free_region(&kctx->kbdev->csf.shared_reg_rbtree, 0, + nr_pages, KBASE_REG_ZONE_MCU_SHARED); + + if (!reg) + return -ENOMEM; + + s_buf->phy = kcalloc(nr_pages, sizeof(*s_buf->phy), GFP_KERNEL); + + if (!s_buf->phy) { + err = -ENOMEM; + goto phy_alloc_failed; + } + + /* Get physical page for a normal suspend buffer */ + err = kbase_mem_pool_alloc_pages( + &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + nr_pages, &s_buf->phy[0], false); + + if (err < 0) + goto phy_pages_alloc_failed; + + /* Insert Region Object into rbtree and make virtual address available + * to map it to physical page + */ + mutex_lock(&kctx->kbdev->csf.reg_lock); + err = kbase_add_va_region_rbtree(kctx->kbdev, reg, 0, nr_pages, 1); + reg->flags &= ~KBASE_REG_FREE; + mutex_unlock(&kctx->kbdev->csf.reg_lock); + + if (err) + goto add_va_region_failed; + + /* Update MMU table */ + err = kbase_mmu_insert_pages(kctx->kbdev, &kctx->kbdev->csf.mcu_mmu, + reg->start_pfn, &s_buf->phy[0], + nr_pages, mem_flags, + MCU_AS_NR, KBASE_MEM_GROUP_CSF_FW); + if (err) + goto mmu_insert_failed; + + s_buf->reg = reg; + + return 0; + +mmu_insert_failed: + mutex_lock(&kctx->kbdev->csf.reg_lock); + WARN_ON(kbase_remove_va_region(reg)); + mutex_unlock(&kctx->kbdev->csf.reg_lock); + +add_va_region_failed: + kbase_mem_pool_free_pages( + &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], nr_pages, + &s_buf->phy[0], false, false); + +phy_pages_alloc_failed: + kfree(s_buf->phy); +phy_alloc_failed: + kfree(reg); + + return err; +} + +/** + * create_protected_suspend_buffer() - Create protected-mode suspend buffer + * per queue group + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @s_buf: Pointer to suspend buffer that is attached to queue group + * + * Return: 0 if suspend buffer is successfully allocated and reflected to GPU + * MMU page table. Otherwise -ENOMEM. + */ +static int create_protected_suspend_buffer(struct kbase_device *const kbdev, + struct kbase_protected_suspend_buffer *s_buf) +{ + struct kbase_va_region *reg = NULL; + struct tagged_addr *phys = NULL; + const unsigned long mem_flags = KBASE_REG_GPU_RD | KBASE_REG_GPU_WR; + const size_t nr_pages = + PFN_UP(kbdev->csf.global_iface.groups[0].suspend_size); + int err = 0; + + /* Allocate and initialize Region Object */ + reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, + nr_pages, KBASE_REG_ZONE_MCU_SHARED); + + if (!reg) + return -ENOMEM; + + phys = kcalloc(nr_pages, sizeof(*phys), GFP_KERNEL); + if (!phys) { + err = -ENOMEM; + goto phy_alloc_failed; + } + + s_buf->pma = kbase_csf_protected_memory_alloc(kbdev, phys, + nr_pages); + if (s_buf->pma == NULL) { + err = -ENOMEM; + goto pma_alloc_failed; + } + + /* Insert Region Object into rbtree and make virtual address available + * to map it to physical page + */ + mutex_lock(&kbdev->csf.reg_lock); + err = kbase_add_va_region_rbtree(kbdev, reg, 0, nr_pages, 1); + reg->flags &= ~KBASE_REG_FREE; + mutex_unlock(&kbdev->csf.reg_lock); + + if (err) + goto add_va_region_failed; + + /* Update MMU table */ + err = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, + reg->start_pfn, phys, + nr_pages, mem_flags, MCU_AS_NR, + KBASE_MEM_GROUP_CSF_FW); + if (err) + goto mmu_insert_failed; + + s_buf->reg = reg; + kfree(phys); + return 0; + +mmu_insert_failed: + mutex_lock(&kbdev->csf.reg_lock); + WARN_ON(kbase_remove_va_region(reg)); + mutex_unlock(&kbdev->csf.reg_lock); + +add_va_region_failed: + kbase_csf_protected_memory_free(kbdev, s_buf->pma, nr_pages); +pma_alloc_failed: + kfree(phys); +phy_alloc_failed: + kfree(reg); + + return err; +} + +static void timer_event_worker(struct work_struct *data); +static void protm_event_worker(struct work_struct *data); +static void term_normal_suspend_buffer(struct kbase_context *const kctx, + struct kbase_normal_suspend_buffer *s_buf); + +/** + * create_suspend_buffers - Setup normal and protected mode + * suspend buffers. + * + * @kctx: Address of the kbase context within which the queue group + * is to be created. + * @group: Pointer to GPU command queue group data. + * + * Return: 0 if suspend buffers are successfully allocated. Otherwise -ENOMEM. + */ +static int create_suspend_buffers(struct kbase_context *const kctx, + struct kbase_queue_group * const group) +{ + int err = 0; + + if (create_normal_suspend_buffer(kctx, &group->normal_suspend_buf)) { + dev_err(kctx->kbdev->dev, "Failed to create normal suspend buffer\n"); + return -ENOMEM; + } + + if (kctx->kbdev->csf.pma_dev) { + err = create_protected_suspend_buffer(kctx->kbdev, + &group->protected_suspend_buf); + if (err) { + term_normal_suspend_buffer(kctx, + &group->normal_suspend_buf); + dev_err(kctx->kbdev->dev, "Failed to create protected suspend buffer\n"); + } + } else { + group->protected_suspend_buf.reg = NULL; + } + + return err; +} + +/** + * create_queue_group() - Create a queue group + * + * @kctx: Address of the kbase context within which the queue group + * is to be created. + * @create: Address of a structure which contains details of the + * queue group which is to be created. + * + * Return: a queue group handle on success, or a negative error code on failure. + */ +static int create_queue_group(struct kbase_context *const kctx, + const union kbase_ioctl_cs_queue_group_create *const create) +{ + int group_handle = find_free_group_handle(kctx); + + if (group_handle < 0) { + dev_err(kctx->kbdev->dev, + "All queue group handles are already in use\n"); + } else { + struct kbase_queue_group * const group = + kmalloc(sizeof(struct kbase_queue_group), + GFP_KERNEL); + + lockdep_assert_held(&kctx->csf.lock); + + if (!group) { + dev_err(kctx->kbdev->dev, "Failed to allocate a queue\n"); + group_handle = -ENOMEM; + } else { + int err = 0; + + group->kctx = kctx; + group->handle = group_handle; + group->csg_nr = KBASEP_CSG_NR_INVALID; + + group->tiler_mask = create->in.tiler_mask; + group->fragment_mask = create->in.fragment_mask; + group->compute_mask = create->in.compute_mask; + + group->tiler_max = create->in.tiler_max; + group->fragment_max = create->in.fragment_max; + group->compute_max = create->in.compute_max; + group->priority = create->in.priority; + group->doorbell_nr = KBASEP_USER_DB_NR_INVALID; + group->faulted = false; + + INIT_LIST_HEAD(&group->link); + INIT_LIST_HEAD(&group->link_to_schedule); + INIT_LIST_HEAD(&group->error_fatal.link); + INIT_LIST_HEAD(&group->error_timeout.link); + INIT_LIST_HEAD(&group->error_tiler_oom.link); + INIT_WORK(&group->timer_event_work, timer_event_worker); + INIT_WORK(&group->protm_event_work, protm_event_worker); + bitmap_zero(group->protm_pending_bitmap, + MAX_SUPPORTED_STREAMS_PER_GROUP); + + group->run_state = KBASE_CSF_GROUP_INACTIVE; + err = create_suspend_buffers(kctx, group); + + if (err < 0) { + kfree(group); + group_handle = err; + } else { + int j; + + kctx->csf.queue_groups[group_handle] = group; + for (j = 0; j < MAX_SUPPORTED_STREAMS_PER_GROUP; + j++) + group->bound_queues[j] = NULL; + } + } + } + + return group_handle; +} + +int kbase_csf_queue_group_create(struct kbase_context *const kctx, + union kbase_ioctl_cs_queue_group_create *const create) +{ + int err = 0; + const u32 tiler_count = hweight64(create->in.tiler_mask); + const u32 fragment_count = hweight64(create->in.fragment_mask); + const u32 compute_count = hweight64(create->in.compute_mask); + + mutex_lock(&kctx->csf.lock); + + if ((create->in.tiler_max > tiler_count) || + (create->in.fragment_max > fragment_count) || + (create->in.compute_max > compute_count)) { + dev_err(kctx->kbdev->dev, + "Invalid maximum number of endpoints for a queue group\n"); + err = -EINVAL; + } else if (create->in.priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT) { + dev_err(kctx->kbdev->dev, "Invalid queue group priority %u\n", + (unsigned int)create->in.priority); + err = -EINVAL; + } else if (!iface_has_enough_streams(kctx->kbdev, create->in.cs_min)) { + dev_err(kctx->kbdev->dev, + "No CSG has at least %d streams\n", + create->in.cs_min); + err = -EINVAL; + } else { + /* For the CSG which satisfies the condition for having + * the needed number of CSs, check whether it also conforms + * with the requirements for at least one of its CSs having + * the iterator of the needed type + * (note: for CSF v1.0 all CSs in a CSG will have access to + * the same iterators) + */ + const int group_handle = create_queue_group(kctx, create); + + if (group_handle >= 0) + create->out.group_handle = group_handle; + else + err = group_handle; + } + + mutex_unlock(&kctx->csf.lock); + + return err; +} + +/** + * term_normal_suspend_buffer() - Free normal-mode suspend buffer of queue group + * + * @kctx: Pointer to kbase context where queue group belongs to + * @s_buf: Pointer to queue group suspend buffer to be freed + */ +static void term_normal_suspend_buffer(struct kbase_context *const kctx, + struct kbase_normal_suspend_buffer *s_buf) +{ + const size_t nr_pages = + PFN_UP(kctx->kbdev->csf.global_iface.groups[0].suspend_size); + + lockdep_assert_held(&kctx->csf.lock); + + WARN_ON(kbase_mmu_teardown_pages( + kctx->kbdev, &kctx->kbdev->csf.mcu_mmu, + s_buf->reg->start_pfn, nr_pages, MCU_AS_NR)); + + WARN_ON(s_buf->reg->flags & KBASE_REG_FREE); + + mutex_lock(&kctx->kbdev->csf.reg_lock); + WARN_ON(kbase_remove_va_region(s_buf->reg)); + mutex_unlock(&kctx->kbdev->csf.reg_lock); + + kbase_mem_pool_free_pages( + &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + nr_pages, &s_buf->phy[0], false, false); + + kfree(s_buf->phy); + s_buf->phy = NULL; + kfree(s_buf->reg); + s_buf->reg = NULL; +} + +/** + * term_protected_suspend_buffer() - Free normal-mode suspend buffer of + * queue group + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @s_buf: Pointer to queue group suspend buffer to be freed + */ +static void term_protected_suspend_buffer(struct kbase_device *const kbdev, + struct kbase_protected_suspend_buffer *s_buf) +{ + const size_t nr_pages = + PFN_UP(kbdev->csf.global_iface.groups[0].suspend_size); + + WARN_ON(kbase_mmu_teardown_pages( + kbdev, &kbdev->csf.mcu_mmu, + s_buf->reg->start_pfn, nr_pages, MCU_AS_NR)); + + WARN_ON(s_buf->reg->flags & KBASE_REG_FREE); + + mutex_lock(&kbdev->csf.reg_lock); + WARN_ON(kbase_remove_va_region(s_buf->reg)); + mutex_unlock(&kbdev->csf.reg_lock); + + kbase_csf_protected_memory_free(kbdev, s_buf->pma, nr_pages); + s_buf->pma = NULL; + kfree(s_buf->reg); + s_buf->reg = NULL; +} + +void kbase_csf_term_descheduled_queue_group(struct kbase_queue_group *group) +{ + struct kbase_context *kctx = group->kctx; + + /* Currently each group supports the same number of streams */ + u32 max_streams = + kctx->kbdev->csf.global_iface.groups[0].stream_num; + u32 i; + + lockdep_assert_held(&kctx->csf.lock); + + WARN_ON(group->run_state != KBASE_CSF_GROUP_INACTIVE && + group->run_state != KBASE_CSF_GROUP_FAULT_EVICTED); + + for (i = 0; i < max_streams; i++) { + struct kbase_queue *queue = + group->bound_queues[i]; + + /* The group is already being evicted from the scheduler */ + if (queue) + unbind_stopped_queue(kctx, queue); + } + + term_normal_suspend_buffer(kctx, &group->normal_suspend_buf); + if (kctx->kbdev->csf.pma_dev) + term_protected_suspend_buffer(kctx->kbdev, + &group->protected_suspend_buf); + + group->run_state = KBASE_CSF_GROUP_TERMINATED; +} + +/** + * term_queue_group - Terminate a GPU command queue group. + * + * @group: Pointer to GPU command queue group data. + * + * Terminates a GPU command queue group. From the userspace perspective the + * group will still exist but it can't bind new queues to it. Userspace can + * still add work in queues bound to the group but it won't be executed. (This + * is because the IO mapping created upon binding such queues is still intact.) + */ +static void term_queue_group(struct kbase_queue_group *group) +{ + struct kbase_context *kctx = group->kctx; + + lockdep_assert_held(&kctx->csf.lock); + + /* Stop the group and evict it from the scheduler */ + kbase_csf_scheduler_group_deschedule(group); + + if (group->run_state == KBASE_CSF_GROUP_TERMINATED) + return; + + dev_dbg(kctx->kbdev->dev, "group %d terminating", group->handle); + + kbase_csf_term_descheduled_queue_group(group); +} + +static void cancel_queue_group_events(struct kbase_queue_group *group) +{ + cancel_work_sync(&group->timer_event_work); + cancel_work_sync(&group->protm_event_work); +} + +void kbase_csf_queue_group_terminate(struct kbase_context *kctx, + u8 group_handle) +{ + struct kbase_queue_group *group; + + mutex_lock(&kctx->csf.lock); + + group = find_queue_group(kctx, group_handle); + + if (group) { + /* Remove any pending group fatal error from the per-context list. */ + list_del_init(&group->error_tiler_oom.link); + list_del_init(&group->error_timeout.link); + list_del_init(&group->error_fatal.link); + + term_queue_group(group); + kctx->csf.queue_groups[group_handle] = NULL; + } + + mutex_unlock(&kctx->csf.lock); + + if (!group) + return; + + /* Cancel any pending event callbacks. If one is in progress + * then this thread waits synchronously for it to complete (which + * is why we must unlock the context first). We already ensured + * that no more callbacks can be enqueued by terminating the group. + */ + cancel_queue_group_events(group); + kfree(group); +} + +int kbase_csf_queue_group_suspend(struct kbase_context *kctx, + struct kbase_suspend_copy_buffer *sus_buf, + u8 group_handle) +{ + int err = -EINVAL; + struct kbase_queue_group *group; + + mutex_lock(&kctx->csf.lock); + + group = find_queue_group(kctx, group_handle); + if (group) + err = kbase_csf_scheduler_group_copy_suspend_buf(group, + sus_buf); + + mutex_unlock(&kctx->csf.lock); + return err; +} + +/** + * kbase_csf_add_fatal_error_to_kctx - Add a fatal error to per-ctx error list. + * + * @group: GPU command queue group. + * @err_payload: Error payload to report. + */ +static void kbase_csf_add_fatal_error_to_kctx( + struct kbase_queue_group *const group, + const struct base_gpu_queue_group_error *const err_payload) +{ + struct base_csf_notification error; + + if (WARN_ON(!group)) + return; + + if (WARN_ON(!err_payload)) + return; + + error = (struct base_csf_notification) { + .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, + .payload = { + .csg_error = { + .handle = group->handle, + .error = *err_payload + } + } + }; + + lockdep_assert_held(&group->kctx->csf.lock); + + /* If this group has already been in fatal error status, + * subsequent fatal error on this group should never take place. + */ + if (!WARN_ON(!list_empty(&group->error_fatal.link))) { + group->error_fatal.data = error; + list_add_tail(&group->error_fatal.link, + &group->kctx->csf.error_list); + } +} + +void kbase_csf_active_queue_groups_reset(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct list_head evicted_groups; + struct kbase_queue_group *group; + int i; + bool fatal_error_built = false; + + INIT_LIST_HEAD(&evicted_groups); + + mutex_lock(&kctx->csf.lock); + + kbase_csf_scheduler_evict_ctx_slots(kbdev, kctx, &evicted_groups); + while (!list_empty(&evicted_groups)) { + struct kbase_csf_scheduler *scheduler = + &kbdev->csf.scheduler; + unsigned long flags; + + group = list_first_entry(&evicted_groups, + struct kbase_queue_group, link); + + dev_dbg(kbdev->dev, "Context %d_%d active group %d terminated", + kctx->tgid, kctx->id, group->handle); + kbase_csf_term_descheduled_queue_group(group); + list_del_init(&group->link); + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + if ((group == scheduler->active_protm_grp) && + group->faulted) { + const struct base_gpu_queue_group_error err_payload = { + .error_type = BASE_GPU_QUEUE_GROUP_ERROR_FATAL, + .payload = { + .fatal_group = { + .status = GPU_EXCEPTION_TYPE_SW_FAULT_0, + } + } + }; + + kbase_csf_add_fatal_error_to_kctx(group, &err_payload); + fatal_error_built = true; + } + kbase_csf_scheduler_spin_unlock(kbdev, flags); + } + + if (fatal_error_built) + kbase_event_wakeup(kctx); + + /* Acting on the queue groups that are pending to be terminated. */ + for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) { + group = kctx->csf.queue_groups[i]; + if (group && + group->run_state == KBASE_CSF_GROUP_FAULT_EVICTED) + kbase_csf_term_descheduled_queue_group(group); + } + + mutex_unlock(&kctx->csf.lock); +} + +int kbase_csf_ctx_init(struct kbase_context *kctx) +{ + int err = -ENOMEM; + + INIT_LIST_HEAD(&kctx->csf.event_callback_list); + INIT_LIST_HEAD(&kctx->csf.queue_list); + INIT_LIST_HEAD(&kctx->csf.link); + INIT_LIST_HEAD(&kctx->csf.error_list); + + spin_lock_init(&kctx->csf.event_lock); + kctx->csf.user_reg_vma = NULL; + + /* Mark all the cookies as 'free' */ + bitmap_fill(kctx->csf.cookies, KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); + + kctx->csf.wq = alloc_workqueue("mali_kbase_csf_wq", + WQ_UNBOUND, 1); + + if (likely(kctx->csf.wq)) { + err = kbase_csf_scheduler_context_init(kctx); + + if (likely(!err)) { + err = kbase_csf_kcpu_queue_context_init(kctx); + + if (likely(!err)) { + err = kbase_csf_tiler_heap_context_init(kctx); + + if (likely(!err)) + mutex_init(&kctx->csf.lock); + else + kbase_csf_kcpu_queue_context_term(kctx); + } + + if (unlikely(err)) + kbase_csf_scheduler_context_term(kctx); + } + + if (unlikely(err)) + destroy_workqueue(kctx->csf.wq); + } + + return err; +} + +void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, + struct kbase_fault *fault) +{ + int gr; + bool reported = false; + struct base_gpu_queue_group_error err_payload; + + if (WARN_ON(!kctx)) + return; + + if (WARN_ON(!fault)) + return; + + err_payload = (struct base_gpu_queue_group_error) { + .error_type = BASE_GPU_QUEUE_GROUP_ERROR_FATAL, + .payload = { + .fatal_group = { + .sideband = fault->addr, + .status = fault->status, + } + } + }; + + mutex_lock(&kctx->csf.lock); + + for (gr = 0; gr < MAX_QUEUE_GROUP_NUM; gr++) { + struct kbase_queue_group *const group = + kctx->csf.queue_groups[gr]; + + if (group && group->run_state != KBASE_CSF_GROUP_TERMINATED) { + term_queue_group(group); + kbase_csf_add_fatal_error_to_kctx(group, &err_payload); + reported = true; + } + } + + mutex_unlock(&kctx->csf.lock); + + if (reported) + kbase_event_wakeup(kctx); +} + +void kbase_csf_ctx_term(struct kbase_context *kctx) +{ + u32 i; + + /* As the kbase context is terminating, its debugfs sub-directory would + * have been removed already and so would be the debugfs file created + * for queue groups & kcpu queues, hence no need to explicitly remove + * those debugfs files. + */ + kbase_csf_event_wait_remove_all(kctx); + + mutex_lock(&kctx->csf.lock); + + /* Iterate through the queue groups that were not terminated by + * userspace and issue the term request to firmware for them. + */ + for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) { + if (kctx->csf.queue_groups[i]) + term_queue_group(kctx->csf.queue_groups[i]); + } + + mutex_unlock(&kctx->csf.lock); + + /* Now that all queue groups have been terminated, there can be no + * more OoM or timer event interrupts but there can be inflight work + * items. Destroying the wq will implicitly flush those work items. + */ + destroy_workqueue(kctx->csf.wq); + + mutex_lock(&kctx->csf.lock); + + for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) + kfree(kctx->csf.queue_groups[i]); + + /* Iterate through the queues that were not terminated by + * userspace and do the required cleanup for them. + */ + while (!list_empty(&kctx->csf.queue_list)) { + struct kbase_queue *queue; + + queue = list_first_entry(&kctx->csf.queue_list, + struct kbase_queue, link); + + /* The reference held when the IO mapping was created on bind + * would have been dropped otherwise the termination of Kbase + * context itself wouldn't have kicked-in. So there shall be + * only one reference left that was taken when queue was + * registered. + */ + if (atomic_read(&queue->refcount) != 1) + dev_warn(kctx->kbdev->dev, + "Releasing queue with incorrect refcounting!\n"); + list_del_init(&queue->link); + release_queue(queue); + } + + mutex_unlock(&kctx->csf.lock); + + kbase_csf_tiler_heap_context_term(kctx); + kbase_csf_kcpu_queue_context_term(kctx); + kbase_csf_scheduler_context_term(kctx); + + mutex_destroy(&kctx->csf.lock); +} + +int kbase_csf_event_wait_add(struct kbase_context *kctx, + kbase_csf_event_callback *callback, void *param) +{ + int err = -ENOMEM; + struct kbase_csf_event *event = + kzalloc(sizeof(struct kbase_csf_event), GFP_KERNEL); + + if (event) { + unsigned long flags; + + event->kctx = kctx; + event->callback = callback; + event->param = param; + + spin_lock_irqsave(&kctx->csf.event_lock, flags); + list_add_tail(&event->link, &kctx->csf.event_callback_list); + spin_unlock_irqrestore(&kctx->csf.event_lock, flags); + + err = 0; + } + + return err; +} + +void kbase_csf_event_wait_remove(struct kbase_context *kctx, + kbase_csf_event_callback *callback, void *param) +{ + struct kbase_csf_event *event; + unsigned long flags; + + spin_lock_irqsave(&kctx->csf.event_lock, flags); + + list_for_each_entry(event, &kctx->csf.event_callback_list, link) { + if ((event->callback == callback) && (event->param == param)) { + list_del(&event->link); + kfree(event); + break; + } + } + spin_unlock_irqrestore(&kctx->csf.event_lock, flags); +} + +bool kbase_csf_read_error(struct kbase_context *kctx, + struct base_csf_notification *event_data) +{ + bool got_event = true; + struct kbase_csf_notification *error_data = NULL; + + mutex_lock(&kctx->csf.lock); + + if (likely(!list_empty(&kctx->csf.error_list))) { + error_data = list_first_entry(&kctx->csf.error_list, + struct kbase_csf_notification, link); + list_del_init(&error_data->link); + *event_data = error_data->data; + } else { + got_event = false; + } + + mutex_unlock(&kctx->csf.lock); + + return got_event; +} + +bool kbase_csf_error_pending(struct kbase_context *kctx) +{ + bool event_pended = false; + + mutex_lock(&kctx->csf.lock); + event_pended = !list_empty(&kctx->csf.error_list); + mutex_unlock(&kctx->csf.lock); + + return event_pended; +} + +void kbase_csf_event_signal(struct kbase_context *kctx, bool notify_gpu) +{ + struct kbase_csf_event *event, *next_event; + unsigned long flags; + + /* First increment the signal count and wake up event thread. + */ + atomic_set(&kctx->event_count, 1); + kbase_event_wakeup(kctx); + + /* Signal the CSF firmware. This is to ensure that pending command + * stream synch object wait operations are re-evaluated. + * Write to GLB_DOORBELL would suffice as spec says that all pending + * synch object wait operations are re-evaluated on a write to any + * CS_DOORBELL/GLB_DOORBELL register. + */ + if (notify_gpu) { + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + if (kctx->kbdev->pm.backend.gpu_powered) + kbase_csf_ring_doorbell(kctx->kbdev, CSF_KERNEL_DOORBELL_NR); + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); + } + + /* Now invoke the callbacks registered on backend side. + * Allow item removal inside the loop, if requested by the callback. + */ + spin_lock_irqsave(&kctx->csf.event_lock, flags); + + list_for_each_entry_safe( + event, next_event, &kctx->csf.event_callback_list, link) { + enum kbase_csf_event_callback_action action = + event->callback(event->param); + + if (action == KBASE_CSF_EVENT_CALLBACK_REMOVE) { + list_del(&event->link); + kfree(event); + } + } + + spin_unlock_irqrestore(&kctx->csf.event_lock, flags); +} + +void kbase_csf_event_wait_remove_all(struct kbase_context *kctx) +{ + struct kbase_csf_event *event, *next_event; + unsigned long flags; + + spin_lock_irqsave(&kctx->csf.event_lock, flags); + + list_for_each_entry_safe( + event, next_event, &kctx->csf.event_callback_list, link) { + list_del(&event->link); + kfree(event); + } + + spin_unlock_irqrestore(&kctx->csf.event_lock, flags); +} + +/** + * handle_oom_event - Handle the OoM event generated by the firmware for the + * command stream interface. + * + * This function will handle the OoM event request from the firmware for the + * command stream. It will retrieve the address of heap context and heap's + * statistics (like number of render passes in-flight) from the command + * stream's kernel output page and pass them to the tiler heap function + * to allocate a new chunk. + * It will also update the command stream's kernel input page with the address + * of a new chunk that was allocated. + * + * @kctx: Pointer to the kbase context in which the tiler heap was initialized. + * @stream: Pointer to the structure containing info provided by the firmware + * about the command stream interface. + * + * Return: 0 if successfully handled the request, otherwise a negative error + * code on failure. + */ +static int handle_oom_event(struct kbase_context *const kctx, + struct kbase_csf_cmd_stream_info const *const stream) +{ + u64 gpu_heap_va = + kbase_csf_firmware_cs_output(stream, CS_HEAP_ADDRESS_LO) | + ((u64)kbase_csf_firmware_cs_output(stream, CS_HEAP_ADDRESS_HI) << 32); + const u32 vt_start = + kbase_csf_firmware_cs_output(stream, CS_HEAP_VT_START); + const u32 vt_end = + kbase_csf_firmware_cs_output(stream, CS_HEAP_VT_END); + const u32 frag_end = + kbase_csf_firmware_cs_output(stream, CS_HEAP_FRAG_END); + u32 renderpasses_in_flight; + u64 new_chunk_ptr; + int err; + + if ((frag_end > vt_end) || (vt_end >= vt_start)) { + dev_warn(kctx->kbdev->dev, "Invalid Heap statistics provided by firmware: vt_start %d, vt_end %d, frag_end %d\n", + vt_start, vt_end, frag_end); + return -EINVAL; + } + + renderpasses_in_flight = vt_start - frag_end; + + err = kbase_csf_tiler_heap_alloc_new_chunk(kctx, + gpu_heap_va, renderpasses_in_flight, &new_chunk_ptr); + + /* It is okay to acknowledge with a NULL chunk (firmware will then wait + * for the fragment jobs to complete and release chunks) + */ + if (err == -EBUSY) + new_chunk_ptr = 0; + else if (err) + return err; + + kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_START_LO, + new_chunk_ptr & 0xFFFFFFFF); + kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_START_HI, + new_chunk_ptr >> 32); + + kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_END_LO, + new_chunk_ptr & 0xFFFFFFFF); + kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_END_HI, + new_chunk_ptr >> 32); + + return 0; +} + +/** + * report_tiler_oom_error - Report a CSG error due to a tiler heap OOM event + * + * @group: Pointer to the GPU command queue group that encountered the error + */ +static void report_tiler_oom_error(struct kbase_queue_group *group) +{ + struct base_csf_notification const + error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, + .payload = { + .csg_error = { + .handle = group->handle, + .error = { + .error_type = + BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM, + } } } }; + struct kbase_context *kctx = group->kctx; + + lockdep_assert_held(&kctx->csf.lock); + + /* Ignore this error if the previous one hasn't been reported */ + if (!WARN_ON(!list_empty(&group->error_tiler_oom.link))) { + group->error_tiler_oom.data = error; + list_add_tail(&group->error_tiler_oom.link, + &kctx->csf.error_list); + kbase_event_wakeup(kctx); + } +} + +/** + * kbase_queue_oom_event - Handle tiler out-of-memory for a GPU command queue. + * + * @queue: Pointer to queue for which out-of-memory event was received. + * + * Called with the command-stream front-end locked for the affected GPU + * virtual address space. Do not call in interrupt context. + * + * Handles tiler out-of-memory for a GPU command queue and then clears the + * notification to allow the firmware to report out-of-memory again in future. + * If the out-of-memory condition was successfully handled then this function + * rings the relevant doorbell to notify the firmware; otherwise, it terminates + * the GPU command queue group to which the queue is bound. See + * term_queue_group() for details. + */ +static void kbase_queue_oom_event(struct kbase_queue *const queue) +{ + struct kbase_context *const kctx = queue->kctx; + struct kbase_device *const kbdev = kctx->kbdev; + struct kbase_queue_group *group; + int slot_num, err; + struct kbase_csf_cmd_stream_group_info const *ginfo; + struct kbase_csf_cmd_stream_info const *stream; + u32 cs_oom_ack, cs_oom_req; + + lockdep_assert_held(&kctx->csf.lock); + + group = get_bound_queue_group(queue); + if (!group) { + dev_warn(kctx->kbdev->dev, "queue not bound\n"); + return; + } + + kbase_csf_scheduler_lock(kbdev); + + slot_num = kbase_csf_scheduler_group_get_slot(group); + + /* The group could have gone off slot before this work item got + * a chance to execute. + */ + if (slot_num < 0) + goto unlock; + + /* If the bound group is on slot yet the kctx is marked with disabled + * on address-space fault, the group is pending to be killed. So skip + * the inflight oom operation. + */ + if (kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) + goto unlock; + + ginfo = &kbdev->csf.global_iface.groups[slot_num]; + stream = &ginfo->streams[queue->csi_index]; + cs_oom_ack = kbase_csf_firmware_cs_output(stream, CS_ACK) & + CS_ACK_TILER_OOM_MASK; + cs_oom_req = kbase_csf_firmware_cs_input_read(stream, CS_REQ) & + CS_REQ_TILER_OOM_MASK; + + /* The group could have already undergone suspend-resume cycle before + * this work item got a chance to execute. On CSG resume the CS_ACK + * register is set by firmware to reflect the CS_REQ register, which + * implies that all events signaled before suspension are implicitly + * acknowledged. + * A new OoM event is expected to be generated after resume. + */ + if (cs_oom_ack == cs_oom_req) + goto unlock; + + err = handle_oom_event(kctx, stream); + + kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_oom_ack, + CS_REQ_TILER_OOM_MASK); + + if (err) { + dev_warn( + kbdev->dev, + "Queue group to be terminated, couldn't handle the OoM event\n"); + kbase_csf_scheduler_unlock(kbdev); + term_queue_group(group); + report_tiler_oom_error(group); + return; + } + + kbase_csf_ring_cs_kernel_doorbell(kbdev, queue); +unlock: + kbase_csf_scheduler_unlock(kbdev); +} + +/** + * oom_event_worker - Tiler out-of-memory handler called from a workqueue. + * + * @data: Pointer to a work_struct embedded in GPU command queue data. + * + * Handles a tiler out-of-memory condition for a GPU command queue and then + * releases a reference that was added to prevent the queue being destroyed + * while this work item was pending on a workqueue. + */ +static void oom_event_worker(struct work_struct *data) +{ + struct kbase_queue *queue = + container_of(data, struct kbase_queue, oom_event_work); + struct kbase_context *kctx = queue->kctx; + + mutex_lock(&kctx->csf.lock); + + kbase_queue_oom_event(queue); + release_queue(queue); + + mutex_unlock(&kctx->csf.lock); +} + +/** + * timer_event_worker - Timer event handler called from a workqueue. + * + * @data: Pointer to a work_struct embedded in GPU command queue group data. + * + * Notify the event notification thread of progress timeout fault + * for the GPU command queue group. + */ +static void timer_event_worker(struct work_struct *data) +{ + struct kbase_queue_group *const group = + container_of(data, struct kbase_queue_group, timer_event_work); + struct base_csf_notification const + error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, + .payload = { + .csg_error = { + .handle = group->handle, + .error = { + .error_type = + BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT, + } } } }; + struct kbase_context *const kctx = group->kctx; + + mutex_lock(&kctx->csf.lock); + + /* Ignore this error if the previous one hasn't been reported */ + if (!WARN_ON(!list_empty(&group->error_timeout.link))) { + group->error_timeout.data = error; + list_add_tail(&group->error_timeout.link, + &kctx->csf.error_list); + kbase_event_wakeup(kctx); + } + + mutex_unlock(&kctx->csf.lock); +} + +/** + * protm_event_worker - Protected mode switch request event handler + * called from a workqueue. + * + * @data: Pointer to a work_struct embedded in GPU command queue group data. + * + * Request to switch to protected mode. + */ +static void protm_event_worker(struct work_struct *data) +{ + struct kbase_queue_group *const group = + container_of(data, struct kbase_queue_group, protm_event_work); + + kbase_csf_scheduler_group_protm_enter(group); +} + +/** + * handle_fault_event - Handler for CS fault. + * + * @queue: Pointer to queue for which fault event was received. + * @stream: Pointer to the structure containing info provided by the + * firmware about the command stream interface. + * + * Prints meaningful CS fault information. + * + * Return: 0 on success, otherwise a negative system code. + */ +static int handle_fault_event(struct kbase_queue const *const queue, + struct kbase_csf_cmd_stream_info const *const stream) +{ + const u32 cs_fault = kbase_csf_firmware_cs_output(stream, CS_FAULT); + const u64 cs_fault_info = + kbase_csf_firmware_cs_output(stream, CS_FAULT_INFO_LO) | + ((u64)kbase_csf_firmware_cs_output(stream, CS_FAULT_INFO_HI) + << 32); + const u8 cs_fault_exception_type = + CS_FAULT_EXCEPTION_TYPE_GET(cs_fault); + const u32 cs_fault_exception_data = + CS_FAULT_EXCEPTION_DATA_GET(cs_fault); + const u64 cs_fault_info_exception_data = + CS_FAULT_INFO_EXCEPTION_DATA_GET(cs_fault_info); + struct kbase_device *const kbdev = queue->kctx->kbdev; + + dev_warn(kbdev->dev, "CSI: %d\n" + "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n" + "CS_FAULT.EXCEPTION_DATA: 0x%x\n" + "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n", + queue->csi_index, cs_fault_exception_type, + kbase_gpu_exception_name(cs_fault_exception_type), + cs_fault_exception_data, cs_fault_info_exception_data); + + return -EFAULT; +} + +/** + * report_queue_fatal_error - Report queue fatal error to user space + * + * @queue: Pointer to queue for which fatal event was received. + * @cs_fatal: Fault information + * @cs_fatal_info: Additional fault information + * + * If a queue has already been in fatal error status, + * subsequent fatal error on the queue should never take place. + */ +static void report_queue_fatal_error(struct kbase_queue *const queue, + u32 cs_fatal, u64 cs_fatal_info) +{ + struct base_csf_notification error = { + .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, + .payload = { + .csg_error = { + .handle = queue->group->handle, + .error = { + .error_type = + BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL, + .payload = { + .fatal_queue = { + .sideband = cs_fatal_info, + .status = cs_fatal, + .csi_index = queue->csi_index, + } + } + } + } + } + }; + + lockdep_assert_held(&queue->kctx->csf.lock); + + /* If a queue has already been in fatal error status, + * subsequent fatal error on the queue should never take place. + */ + if (!WARN_ON(!list_empty(&queue->error.link))) { + queue->error.data = error; + list_add_tail(&queue->error.link, &queue->kctx->csf.error_list); + kbase_event_wakeup(queue->kctx); + } +} + +/** + * handle_fatal_event - Handler for CS fatal. + * + * @queue: Pointer to queue for which fatal event was received. + * @stream: Pointer to the structure containing info provided by the + * firmware about the command stream interface. + * @fw_error: Return true if internal firmware fatal is handled + * + * Prints meaningful CS fatal information. + * Report queue fatal error to user space. + * + * Return: 0 on success otherwise a negative system error. + */ +static int handle_fatal_event(struct kbase_queue *const queue, + struct kbase_csf_cmd_stream_info const *const stream, + bool *fw_error) +{ + const u32 cs_fatal = kbase_csf_firmware_cs_output(stream, CS_FATAL); + const u64 cs_fatal_info = + kbase_csf_firmware_cs_output(stream, CS_FATAL_INFO_LO) | + ((u64)kbase_csf_firmware_cs_output(stream, CS_FATAL_INFO_HI) + << 32); + const u32 cs_fatal_exception_type = + CS_FATAL_EXCEPTION_TYPE_GET(cs_fatal); + const u32 cs_fatal_exception_data = + CS_FATAL_EXCEPTION_DATA_GET(cs_fatal); + const u64 cs_fatal_info_exception_data = + CS_FATAL_INFO_EXCEPTION_DATA_GET(cs_fatal_info); + struct kbase_device *const kbdev = queue->kctx->kbdev; + + lockdep_assert_held(&queue->kctx->csf.lock); + + dev_warn(kbdev->dev, + "CSG: %d, CSI: %d\n" + "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n" + "CS_FATAL.EXCEPTION_DATA: 0x%x\n" + "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n", + queue->group->handle, queue->csi_index, + cs_fatal_exception_type, + kbase_gpu_exception_name(cs_fatal_exception_type), + cs_fatal_exception_data, cs_fatal_info_exception_data); + + if (cs_fatal_exception_type == + CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR) + *fw_error = true; + else + report_queue_fatal_error(queue, cs_fatal, cs_fatal_info); + + return -EFAULT; +} + +/** + * handle_internal_firmware_fatal - Handler for CS internal firmware fault. + * + * @kbdev: Pointer to kbase device + * + * Report group fatal error to user space for all GPU command queue groups + * in the device, terminate them and reset GPU. + */ +static void handle_internal_firmware_fatal(struct kbase_device *const kbdev) +{ + int as; + + for (as = 0; as < kbdev->nr_hw_address_spaces; as++) { + struct kbase_context *kctx; + struct kbase_fault fault = { + .status = GPU_EXCEPTION_TYPE_SW_FAULT_1, + }; + + if (as == MCU_AS_NR) + continue; + + kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as); + if (!kctx) + continue; + + kbase_csf_ctx_handle_fault(kctx, &fault); + kbase_ctx_sched_release_ctx_lock(kctx); + } + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); +} + +/** + * fault_event_worker - Worker function for CS fault/fatal. + * + * @data: Pointer to a work_struct embedded in GPU command queue data. + * + * Handle the fault and fatal exception for a GPU command queue and then + * releases a reference that was added to prevent the queue being destroyed + * while this work item was pending on a workqueue. + * + * Report the fault and fatal exception for a GPU command queue and then + * clears the corresponding notification fields to allow the firmware to + * report other faults in future. + * + * It may also terminate the GPU command queue group(s) and reset GPU + * in case internal firmware CS fatal exception occurred. + */ +static void fault_event_worker(struct work_struct *const data) +{ + struct kbase_queue *const queue = + container_of(data, struct kbase_queue, fault_event_work); + + struct kbase_context *const kctx = queue->kctx; + struct kbase_device *const kbdev = kctx->kbdev; + struct kbase_queue_group *group; + int slot_num; + struct kbase_csf_cmd_stream_group_info const *ginfo; + struct kbase_csf_cmd_stream_info const *stream; + u32 cs_ack, cs_req; + int err = 0; + bool internal_fw_error = false; + + mutex_lock(&kctx->csf.lock); + kbase_csf_scheduler_lock(kbdev); + + group = get_bound_queue_group(queue); + if (!group) { + dev_warn(kbdev->dev, "queue not bound\n"); + goto unlock; + } + + slot_num = kbase_csf_scheduler_group_get_slot(group); + + /* The group could have gone off slot before this work item got + * a chance to execute. + */ + if (slot_num < 0) { + dev_warn(kbdev->dev, "invalid slot_num\n"); + goto unlock; + } + + /* If the bound group is on slot yet the kctx is marked with disabled + * on address-space fault, the group is pending to be killed. So skip + * the inflight queue exception event operation. + */ + if (kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) { + dev_warn(kbdev->dev, "kctx is already disabled on fault\n"); + goto unlock; + } + + ginfo = &kbdev->csf.global_iface.groups[slot_num]; + stream = &ginfo->streams[queue->csi_index]; + cs_ack = kbase_csf_firmware_cs_output(stream, CS_ACK); + cs_req = kbase_csf_firmware_cs_input_read(stream, CS_REQ); + + if ((cs_ack & CS_ACK_FATAL_MASK) != (cs_req & CS_REQ_FATAL_MASK)) { + err = handle_fatal_event(queue, stream, &internal_fw_error); + kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, + CS_REQ_FATAL_MASK); + } + + if ((cs_ack & CS_ACK_FAULT_MASK) != (cs_req & CS_REQ_FAULT_MASK)) { + err |= handle_fault_event(queue, stream); + kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, + CS_REQ_FAULT_MASK); + kbase_csf_ring_cs_kernel_doorbell(kbdev, queue); + } + + if (err) { + /* From 10.x.5, CS_REQ_ERROR_MODE is removed but TI2 bitfile + * upload not finished. Need to remove on GPUCORE-23972 + */ + kbase_csf_firmware_cs_input_mask(stream, CS_REQ, ~cs_ack, + CS_REQ_ERROR_MODE_MASK); + dev_dbg(kbdev->dev, "Slot-%d CSI-%d entering error mode\n", + slot_num, queue->csi_index); + } + +unlock: + release_queue(queue); + kbase_csf_scheduler_unlock(kbdev); + mutex_unlock(&kctx->csf.lock); + + if (internal_fw_error) + handle_internal_firmware_fatal(kbdev); +} + +/** + * process_cs_interrupts - Process interrupts for a command stream. + * + * @group: Pointer to GPU command queue group data. + * @ginfo: The command stream group interface provided by the firmware. + * @irqreq: CSG's IRQ request bitmask (one bit per stream). + * @irqack: CSG's IRQ acknowledge bitmask (one bit per stream). + * + * If the interrupt request bitmask differs from the acknowledge bitmask + * then the firmware is notifying the host of an event concerning those + * streams indicated by bits whose value differs. The actions required + * are then determined by examining which notification flags differ between + * the request and acknowledge registers for the individual stream(s). + */ +static void process_cs_interrupts(struct kbase_queue_group *const group, + struct kbase_csf_cmd_stream_group_info const *const ginfo, + u32 const irqreq, u32 const irqack) +{ + struct kbase_device *const kbdev = group->kctx->kbdev; + u32 remaining = irqreq ^ irqack; + bool protm_pend = false; + + kbase_csf_scheduler_spin_lock_assert_held(kbdev); + + while (remaining != 0) { + int const i = ffs(remaining) - 1; + struct kbase_queue *const queue = group->bound_queues[i]; + + /* The queue pointer can be NULL, but if it isn't NULL then it + * cannot disappear since scheduler spinlock is held and before + * freeing a bound queue it has to be first unbound which + * requires scheduler spinlock. + */ + if (queue && !WARN_ON(queue->csi_index != i)) { + struct kbase_csf_cmd_stream_info const *const stream = + &ginfo->streams[i]; + u32 const cs_req = kbase_csf_firmware_cs_input_read( + stream, CS_REQ); + u32 const cs_ack = + kbase_csf_firmware_cs_output(stream, CS_ACK); + struct workqueue_struct *wq = group->kctx->csf.wq; + + if ((cs_req & CS_REQ_EXCEPTION_MASK) ^ + (cs_ack & CS_ACK_EXCEPTION_MASK)) { + get_queue(queue); + KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_FAULT_INTERRUPT, group, queue, cs_req ^ cs_ack); + if (!queue_work(wq, &queue->fault_event_work)) + release_queue(queue); + } + + if (((cs_req & CS_REQ_TILER_OOM_MASK) ^ + (cs_ack & CS_ACK_TILER_OOM_MASK))) { + get_queue(queue); + KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_TILER_OOM_INTERRUPT, group, queue, cs_req ^ cs_ack); + if (WARN_ON(!queue_work( + wq, &queue->oom_event_work))) { + /* The work item shall not have been + * already queued, there can be only + * one pending OoM event for a + * queue. + */ + release_queue(queue); + } + } + + if ((cs_req & CS_REQ_PROTM_PEND_MASK) ^ + (cs_ack & CS_ACK_PROTM_PEND_MASK)) { + dev_dbg(kbdev->dev, + "Protected mode entry request for queue on csi %d bound to group-%d on slot %d", + queue->csi_index, group->handle, + group->csg_nr); + + bitmap_set(group->protm_pending_bitmap, i, 1); + protm_pend = true; + } + } + + remaining &= ~(1 << i); + } + + if (protm_pend) + queue_work(group->kctx->csf.wq, &group->protm_event_work); +} + +/** + * process_csg_interrupts - Process interrupts for a command stream group. + * + * @kbdev: Instance of a GPU platform device that implements a command stream + * front-end interface. + * @csg_nr: Command stream group number. + * + * Handles interrupts for a command stream group and for streams within it. + * + * If the CSG's request register value differs from its acknowledge register + * then the firmware is notifying the host of an event concerning the whole + * group. The actions required are then determined by examining which + * notification flags differ between those two register values. + * + * See process_cs_interrupts() for details of per-stream interrupt handling. + */ +static void process_csg_interrupts(struct kbase_device *const kbdev, + int const csg_nr) +{ + struct kbase_csf_cmd_stream_group_info *ginfo; + struct kbase_queue_group *group; + u32 req, ack, irqreq, irqack; + + kbase_csf_scheduler_spin_lock_assert_held(kbdev); + + if (WARN_ON(csg_nr >= kbdev->csf.global_iface.group_num)) + return; + + ginfo = &kbdev->csf.global_iface.groups[csg_nr]; + req = kbase_csf_firmware_csg_input_read(ginfo, CSG_REQ); + ack = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); + irqreq = kbase_csf_firmware_csg_output(ginfo, CSG_IRQ_REQ); + irqack = kbase_csf_firmware_csg_input_read(ginfo, CSG_IRQ_ACK); + + /* There may not be any pending CSG/CS interrupts to process */ + if ((req == ack) && (irqreq == irqack)) + return; + + /* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before + * examining the CS_ACK & CS_REQ bits. This would ensure that Host + * doesn't misses an interrupt for the CS in the race scenario where + * whilst Host is servicing an interrupt for the CS, firmware sends + * another interrupt for that CS. + */ + kbase_csf_firmware_csg_input(ginfo, CSG_IRQ_ACK, irqreq); + + group = kbase_csf_scheduler_get_group_on_slot(kbdev, csg_nr); + + /* The group pointer can be NULL here if interrupts for the group + * (like SYNC_UPDATE, IDLE notification) were delayed and arrived + * just after the suspension of group completed. However if not NULL + * then the group pointer cannot disappear even if User tries to + * terminate the group whilst this loop is running as scheduler + * spinlock is held and for freeing a group that is resident on a CSG + * slot scheduler spinlock is required. + */ + if (!group) + return; + + if (WARN_ON(kbase_csf_scheduler_group_get_slot_locked(group) != csg_nr)) + return; + + if ((req ^ ack) & CSG_REQ_SYNC_UPDATE) { + kbase_csf_firmware_csg_input_mask(ginfo, + CSG_REQ, ack, CSG_REQ_SYNC_UPDATE); + + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SYNC_UPDATE_INTERRUPT, group, req ^ ack); + kbase_csf_event_signal_cpu_only(group->kctx); + } + + /* IDLE and TILER_OOM can be safely ignored because they will be + * raised again if the group is assigned a CSG slot in future. + * TILER_OOM and PROGRESS_TIMER_EVENT may terminate the group. + */ + if (!kbase_csf_scheduler_group_events_enabled(kbdev, group)) + return; + + if ((req ^ ack) & CSG_REQ_IDLE_MASK) { + kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, ack, + CSG_REQ_IDLE_MASK); + + set_bit(csg_nr, kbdev->csf.scheduler.csg_slots_idle_mask); + + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_IDLE_INTERRUPT, group, req ^ ack); + dev_dbg(kbdev->dev, "Idle notification received for Group %u on slot %d\n", + group->handle, csg_nr); + } + + if ((req ^ ack) & CSG_REQ_PROGRESS_TIMER_EVENT_MASK) { + kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, ack, + CSG_REQ_PROGRESS_TIMER_EVENT_MASK); + + dev_dbg(kbdev->dev, "Timeout notification received for Group %u on slot %d\n", + group->handle, csg_nr); + + queue_work(group->kctx->csf.wq, &group->timer_event_work); + } + + process_cs_interrupts(group, ginfo, irqreq, irqack); +} + +void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) +{ + unsigned long flags; + u32 remaining = val; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val); + + if (val & JOB_IRQ_GLOBAL_IF) { + const struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + + kbdev->csf.interrupt_received = true; + remaining &= ~JOB_IRQ_GLOBAL_IF; + + if (!kbdev->csf.firmware_reloaded) + kbase_csf_firmware_reload_completed(kbdev); + else if (kbdev->csf.glb_init_request_pending) + kbase_pm_update_state(kbdev); + + if (global_iface->output) { + u32 glb_req, glb_ack; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + glb_req = kbase_csf_firmware_global_input_read( + global_iface, GLB_REQ); + glb_ack = kbase_csf_firmware_global_output( + global_iface, GLB_ACK); + + if ((glb_req ^ glb_ack) & GLB_REQ_PROTM_EXIT_MASK) { + dev_dbg(kbdev->dev, "Protected mode exit interrupt received"); + kbase_csf_firmware_global_input_mask( + global_iface, GLB_REQ, glb_ack, + GLB_REQ_PROTM_EXIT_MASK); + WARN_ON(!kbase_csf_scheduler_protected_mode_in_use(kbdev)); + scheduler->active_protm_grp = NULL; + KBASE_KTRACE_ADD(kbdev, SCHEDULER_EXIT_PROTM, NULL, 0u); + } + kbase_csf_scheduler_spin_unlock(kbdev, flags); + } + + if (!remaining) { + wake_up_all(&kbdev->csf.event_wait); + return; + } + } + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + while (remaining != 0) { + int const csg_nr = ffs(remaining) - 1; + + process_csg_interrupts(kbdev, csg_nr); + remaining &= ~(1 << csg_nr); + } + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + wake_up_all(&kbdev->csf.event_wait); +} + +void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev) +{ + if (kbdev->csf.db_filp) { + struct page *page = as_page(kbdev->csf.dummy_db_page); + + kbase_mem_pool_free( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + page, false); + + fput(kbdev->csf.db_filp); + } +} + +int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev) +{ + struct tagged_addr phys; + struct file *filp; + int ret; + + filp = shmem_file_setup("mali csf", MAX_LFS_FILESIZE, VM_NORESERVE); + if (IS_ERR(filp)) + return PTR_ERR(filp); + + ret = kbase_mem_pool_alloc_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + 1, &phys, false); + + if (ret <= 0) { + fput(filp); + return ret; + } + + kbdev->csf.db_filp = filp; + kbdev->csf.dummy_db_page = phys; + kbdev->csf.db_file_offsets = 0; + + return 0; +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h new file mode 100755 index 000000000000..c183d0a32302 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.h @@ -0,0 +1,444 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_H_ +#define _KBASE_CSF_H_ + +#include "mali_kbase_csf_kcpu.h" +#include "mali_kbase_csf_scheduler.h" +#include "mali_kbase_csf_firmware.h" +#include "mali_kbase_csf_protected_memory.h" + +/* Indicate invalid command stream h/w interface + */ +#define KBASEP_IF_NR_INVALID ((s8)-1) + +/* Indicate invalid command stream group number for a GPU command queue group + */ +#define KBASEP_CSG_NR_INVALID ((s8)-1) + +/* Indicate invalid user doorbell number for a GPU command queue + */ +#define KBASEP_USER_DB_NR_INVALID ((s8)-1) + +/* Waiting timeout for global request completion acknowledgment */ +#define GLB_REQ_WAIT_TIMEOUT_MS (300) /* 300 milliseconds */ + +#define CSG_REQ_EP_CFG (0x1 << CSG_REQ_EP_CFG_SHIFT) +#define CSG_REQ_SYNC_UPDATE (0x1 << CSG_REQ_SYNC_UPDATE_SHIFT) +#define FIRMWARE_PING_INTERVAL_MS (2000) /* 2 seconds */ + +/** + * enum kbase_csf_event_callback_action - return type for CSF event callbacks. + * + * @KBASE_CSF_EVENT_CALLBACK_FIRST: Never set explicitly. + * It doesn't correspond to any action or type of event callback. + * + * @KBASE_CSF_EVENT_CALLBACK_KEEP: The callback will remain registered. + * + * @KBASE_CSF_EVENT_CALLBACK_REMOVE: The callback will be removed + * immediately upon return. + * + * @KBASE_CSF_EVENT_CALLBACK_LAST: Never set explicitly. + * It doesn't correspond to any action or type of event callback. + */ +enum kbase_csf_event_callback_action { + KBASE_CSF_EVENT_CALLBACK_FIRST = 0, + KBASE_CSF_EVENT_CALLBACK_KEEP, + KBASE_CSF_EVENT_CALLBACK_REMOVE, + KBASE_CSF_EVENT_CALLBACK_LAST, +}; + +/** + * kbase_csf_event_callback_action - type for callback functions to be + * called upon CSF events. + * + * This is the type of callback functions that can be registered + * for CSF events. These function calls shall be triggered by any call + * to kbase_csf_event_signal. + * + * @param: Generic parameter to pass to the callback function. + * + * Return: KBASE_CSF_EVENT_CALLBACK_KEEP if the callback should remain + * registered, or KBASE_CSF_EVENT_CALLBACK_REMOVE if it should be removed. + */ +typedef enum kbase_csf_event_callback_action kbase_csf_event_callback(void *param); + +/** + * kbase_csf_event_wait_add - Add a CSF event callback + * + * This function adds an event callback to the list of CSF event callbacks + * belonging to a given Kbase context, to be triggered when a CSF event is + * signalled by kbase_csf_event_signal. + * + * @kctx: The Kbase context the @callback should be registered to. + * @callback: The callback function to register. + * @param: Custom parameter to be passed to the @callback function. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_event_wait_add(struct kbase_context *kctx, + kbase_csf_event_callback *callback, void *param); + +/** + * kbase_csf_event_wait_remove - Remove a CSF event callback + * + * This function removes an event callback from the list of CSF event callbacks + * belonging to a given Kbase context. + * + * @kctx: The kbase context the @callback should be removed from. + * @callback: The callback function to remove. + * @param: Custom parameter that would have been passed to the @p callback + * function. + */ +void kbase_csf_event_wait_remove(struct kbase_context *kctx, + kbase_csf_event_callback *callback, void *param); + +/** + * kbase_csf_event_wait_remove_all - Removes all CSF event callbacks + * + * This function empties the list of CSF event callbacks belonging to a given + * Kbase context. + * + * @kctx: The kbase context for which CSF event callbacks have to be removed. + */ +void kbase_csf_event_wait_remove_all(struct kbase_context *kctx); + +/** + * kbase_csf_read_error - Read command stream fatal error + * + * This function takes the command stream fatal error from context's ordered + * error_list, copies its contents to @event_data. + * + * @kctx: The kbase context to read fatal error from + * @event_data: Caller-provided buffer to copy the fatal error to + * + * Return: true if fatal error is read successfully. + */ +bool kbase_csf_read_error(struct kbase_context *kctx, + struct base_csf_notification *event_data); + +/** + * kbase_csf_error_pending - Check whether fatal error is pending + * + * @kctx: The kbase context to check fatal error upon. + * + * Return: true if fatal error is pending. + */ +bool kbase_csf_error_pending(struct kbase_context *kctx); + +/** + * kbase_csf_event_signal - Signal a CSF event + * + * This function triggers all the CSF event callbacks that are registered to + * a given Kbase context, and also signals the thread of userspace driver + * (front-end), waiting for the CSF event. + * + * @kctx: The kbase context whose CSF event callbacks shall be triggered. + * @notify_gpu: Flag to indicate if CSF firmware should be notified of the + * signaling of event that happened on the Driver side, either + * the signal came from userspace or from kcpu queues. + */ +void kbase_csf_event_signal(struct kbase_context *kctx, bool notify_gpu); + +static inline void kbase_csf_event_signal_notify_gpu(struct kbase_context *kctx) +{ + kbase_csf_event_signal(kctx, true); +} + +static inline void kbase_csf_event_signal_cpu_only(struct kbase_context *kctx) +{ + kbase_csf_event_signal(kctx, false); +} + +/** + * kbase_csf_ctx_init - Initialize the command-stream front-end for a GPU + * address space. + * + * @kctx: Pointer to the kbase context which is being initialized. + * + * Return: 0 if successful or a negative error code on failure. + */ +int kbase_csf_ctx_init(struct kbase_context *kctx); + +/** + * kbase_csf_ctx_handle_fault - Terminate queue groups & notify fault upon + * GPU bus fault, MMU page fault or similar. + * + * This function terminates all GPU command queue groups in the context and + * notifies the event notification thread of the fault. + * + * @kctx: Pointer to faulty kbase context. + * @fault: Pointer to the fault. + */ +void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, + struct kbase_fault *fault); + +/** + * kbase_csf_ctx_term - Terminate the command-stream front-end for a GPU + * address space. + * + * This function terminates any remaining CSGs and CSs which weren't destroyed + * before context termination. + * + * @kctx: Pointer to the kbase context which is being terminated. + */ +void kbase_csf_ctx_term(struct kbase_context *kctx); + +/** + * kbase_csf_queue_register - Register a GPU command queue. + * + * @kctx: Pointer to the kbase context within which the + * queue is to be registered. + * @reg: Pointer to the structure which contains details of the + * queue to be registered within the provided + * context. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_queue_register(struct kbase_context *kctx, + struct kbase_ioctl_cs_queue_register *reg); + +/** + * kbase_csf_queue_terminate - Terminate a GPU command queue. + * + * @kctx: Pointer to the kbase context within which the + * queue is to be terminated. + * @term: Pointer to the structure which identifies which + * queue is to be terminated. + */ +void kbase_csf_queue_terminate(struct kbase_context *kctx, + struct kbase_ioctl_cs_queue_terminate *term); + +/** + * kbase_csf_alloc_command_stream_user_pages - Allocate resources for a + * GPU command queue. + * + * This function allocates a pair of User mode input/output pages for a + * GPU command queue and maps them in the shared interface segment of MCU + * firmware address space. Also reserves a hardware doorbell page for the queue. + * + * @kctx: Pointer to the kbase context within which the resources + * for the queue are being allocated. + * @queue: Pointer to the queue for which to allocate resources. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_alloc_command_stream_user_pages(struct kbase_context *kctx, + struct kbase_queue *queue); + +/** + * kbase_csf_queue_bind - Bind a GPU command queue to a queue group. + * + * @kctx: The kbase context. + * @bind: Pointer to the union which specifies a queue group and a + * queue to be bound to that group. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_queue_bind(struct kbase_context *kctx, + union kbase_ioctl_cs_queue_bind *bind); + +/** + * kbase_csf_queue_unbind - Unbind a GPU command queue from a queue group + * to which it has been bound and free + * resources allocated for this queue if there + * are any. + * + * @queue: Pointer to queue to be unbound. + */ +void kbase_csf_queue_unbind(struct kbase_queue *queue); + +/** + * kbase_csf_queue_kick - Schedule a GPU command queue on the firmware + * + * @kctx: The kbase context. + * @kick: Pointer to the struct which specifies the queue + * that needs to be scheduled. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_queue_kick(struct kbase_context *kctx, + struct kbase_ioctl_cs_queue_kick *kick); + +/** Find if given the queue group handle is valid. + * + * This function is used to determine if the queue group handle is valid. + * + * @kctx: The kbase context under which the queue group exists. + * @group_handle: Handle for the group which uniquely identifies it within + * the context with which it was created. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_queue_group_handle_is_valid(struct kbase_context *kctx, + u8 group_handle); + +/** + * kbase_csf_queue_group_create - Create a GPU command queue group. + * + * @kctx: Pointer to the kbase context within which the + * queue group is to be created. + * @create: Pointer to the structure which contains details of the + * queue group which is to be created within the + * provided kbase context. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_queue_group_create(struct kbase_context *kctx, + union kbase_ioctl_cs_queue_group_create *create); + +/** + * kbase_csf_queue_group_terminate - Terminate a GPU command queue group. + * + * @kctx: Pointer to the kbase context within which the + * queue group is to be terminated. + * @group_handle: Pointer to the structure which identifies the queue + * group which is to be terminated. + */ +void kbase_csf_queue_group_terminate(struct kbase_context *kctx, + u8 group_handle); + +/** + * kbase_csf_term_descheduled_queue_group - Terminate a GPU command queue + * group that is not operational + * inside the scheduler. + * + * @group: Pointer to the structure which identifies the queue + * group to be terminated. The function assumes that the caller + * is sure that the given group is not operational inside the + * scheduler. If in doubt, use its alternative: + * @ref kbase_csf_queue_group_terminate(). + */ +void kbase_csf_term_descheduled_queue_group(struct kbase_queue_group *group); + +/** + * kbase_csf_queue_group_suspend - Suspend a GPU command queue group + * + * This function is used to suspend a queue group and copy the suspend buffer. + * + * @kctx: The kbase context for which the queue group is to be + * suspended. + * @sus_buf: Pointer to the structure which contains details of the + * user buffer and its kernel pinned pages. + * @size: The size in bytes for the user provided buffer. + * @group_handle: Handle for the group which uniquely identifies it within + * the context within which it was created. + * + * Return: 0 on success or negative value if failed to suspend + * queue group and copy suspend buffer contents. + */ +int kbase_csf_queue_group_suspend(struct kbase_context *kctx, + struct kbase_suspend_copy_buffer *sus_buf, u8 group_handle); + +/** + * kbase_csf_interrupt - Handle interrupts issued by CSF firmware. + * + * @kbdev: The kbase device to handle an IRQ for + * @val: The value of JOB IRQ status register which triggered the interrupt + */ +void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val); + +/** + * kbase_csf_doorbell_mapping_init - Initialize the bitmap of Hw doorbell pages + * used to track their availability. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev); + +void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev); + +/** + * kbase_csf_ring_csg_doorbell - ring the doorbell for a command stream group + * interface. + * + * The function kicks a notification on the command stream group interface to + * firmware. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @slot: Index of command stream group interface for ringing the door-bell. + */ +void kbase_csf_ring_csg_doorbell(struct kbase_device *kbdev, int slot); + +/** + * kbase_csf_ring_csg_slots_doorbell - ring the doorbell for a set of command + * stream group interfaces. + * + * The function kicks a notification on a set of command stream group + * interfaces to firmware. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @slot_bitmap: bitmap for the given slots, slot-0 on bit-0, etc. + */ +void kbase_csf_ring_csg_slots_doorbell(struct kbase_device *kbdev, + u32 slot_bitmap); + +/** + * kbase_csf_ring_cs_kernel_doorbell - ring the kernel doorbell for a queue + * + * The function kicks a notification to the firmware for the command stream + * interface to which the queue is bound. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @queue: Pointer to the queue for ringing the door-bell. + */ +void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, + struct kbase_queue *queue); + +/** + * kbase_csf_ring_cs_user_doorbell - ring the user doorbell allocated for a + * queue. + * + * The function kicks a notification to the firmware on the doorbell assigned + * to the queue. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @queue: Pointer to the queue for ringing the door-bell. + */ +void kbase_csf_ring_cs_user_doorbell(struct kbase_device *kbdev, + struct kbase_queue *queue); + +/** + * kbase_csf_active_queue_groups_reset - Reset the state of all active GPU + * command queue groups associated with the context. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @kctx: The kbase context. + * + * This function will iterate through all the active/scheduled GPU command + * queue groups associated with the context, deschedule and mark them as + * terminated (which will then lead to unbinding of all the queues bound to + * them) and also no more work would be allowed to execute for them. + * + * This is similar to the action taken in response to an unexpected OoM event. + */ +void kbase_csf_active_queue_groups_reset(struct kbase_device *kbdev, + struct kbase_context *kctx); + +#endif /* _KBASE_CSF_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c new file mode 100755 index 000000000000..fd8329ba9422 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c @@ -0,0 +1,460 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_csf_csg_debugfs.h" +#include +#include +#include +#include + +#ifdef CONFIG_DEBUG_FS +#include "mali_kbase_csf_tl_reader.h" + +static void kbasep_csf_scheduler_dump_active_queue_cs_status_wait( + struct seq_file *file, + u32 wait_status, + u32 wait_sync_value, + u64 wait_sync_live_value, + u64 wait_sync_pointer) +{ +#define WAITING "Waiting" +#define NOT_WAITING "Not waiting" + + seq_printf(file, "SB_MASK: %d\n", + CS_STATUS_WAIT_SB_MASK_GET(wait_status)); + seq_printf(file, "PROGRESS_WAIT: %s\n", + CS_STATUS_WAIT_PROGRESS_WAIT_GET(wait_status) ? + WAITING : NOT_WAITING); + seq_printf(file, "PROTM_PEND: %s\n", + CS_STATUS_WAIT_PROTM_PEND_GET(wait_status) ? + WAITING : NOT_WAITING); + seq_printf(file, "SYNC_WAIT: %s\n", + CS_STATUS_WAIT_SYNC_WAIT_GET(wait_status) ? + WAITING : NOT_WAITING); + seq_printf(file, "WAIT_CONDITION: %s\n", + CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GET(wait_status) ? + "greater than" : "less or equal"); + seq_printf(file, "SYNC_POINTER: 0x%llx\n", wait_sync_pointer); + seq_printf(file, "SYNC_VALUE: %d\n", wait_sync_value); + seq_printf(file, "SYNC_LIVE_VALUE: 0x%016llx\n", wait_sync_live_value); +} + +/** + * kbasep_csf_scheduler_dump_active_queue() - Print GPU command queue + * debug information + * + * @file: seq_file for printing to + * @queue: Address of a GPU command queue to examine + */ +static void kbasep_csf_scheduler_dump_active_queue(struct seq_file *file, + struct kbase_queue *queue) +{ + u32 *addr; + u64 cs_extract; + u64 cs_insert; + u32 cs_active; + u64 wait_sync_pointer; + u32 wait_status, wait_sync_value; + struct kbase_vmap_struct *mapping; + u64 *evt; + u64 wait_sync_live_value; + + if (!queue) + return; + + if (WARN_ON(queue->csi_index == KBASEP_IF_NR_INVALID || + !queue->group)) + return; + + /* Ring the doorbell to have firmware update CS_EXTRACT */ + kbase_csf_ring_cs_user_doorbell(queue->kctx->kbdev, queue); + msleep(100); + + addr = (u32 *)queue->user_io_addr; + cs_insert = addr[CS_INSERT_LO/4] | ((u64)addr[CS_INSERT_HI/4] << 32); + + addr = (u32 *)(queue->user_io_addr + PAGE_SIZE); + cs_extract = addr[CS_EXTRACT_LO/4] | ((u64)addr[CS_EXTRACT_HI/4] << 32); + cs_active = addr[CS_ACTIVE/4]; + +#define KBASEP_CSF_DEBUGFS_CS_HEADER_USER_IO \ + "Bind Idx, Ringbuf addr, Prio, Insert offset, Extract offset, Active, Doorbell\n" + + seq_printf(file, KBASEP_CSF_DEBUGFS_CS_HEADER_USER_IO "%8d, %16llx, %4u, %16llx, %16llx, %6u, %8d\n", + queue->csi_index, queue->base_addr, queue->priority, + cs_insert, cs_extract, cs_active, queue->doorbell_nr); + + /* Print status information for blocked group waiting for sync object */ + if (kbase_csf_scheduler_group_get_slot(queue->group) < 0) { + if (CS_STATUS_WAIT_SYNC_WAIT_GET(queue->status_wait)) { + wait_status = queue->status_wait; + wait_sync_value = queue->sync_value; + wait_sync_pointer = queue->sync_ptr; + + evt = (u64 *)kbase_phy_alloc_mapping_get(queue->kctx, wait_sync_pointer, &mapping); + if (evt) { + wait_sync_live_value = evt[0]; + kbase_phy_alloc_mapping_put(queue->kctx, mapping); + } else { + wait_sync_live_value = U64_MAX; + } + + kbasep_csf_scheduler_dump_active_queue_cs_status_wait( + file, wait_status, wait_sync_value, + wait_sync_live_value, wait_sync_pointer); + } + } else { + struct kbase_device const *const kbdev = + queue->group->kctx->kbdev; + struct kbase_csf_cmd_stream_group_info const *const ginfo = + &kbdev->csf.global_iface.groups[queue->group->csg_nr]; + struct kbase_csf_cmd_stream_info const *const stream = + &ginfo->streams[queue->csi_index]; + u64 cmd_ptr; + u32 req_res; + + if (WARN_ON(!stream)) + return; + + cmd_ptr = kbase_csf_firmware_cs_output(stream, + CS_STATUS_CMD_PTR_LO); + cmd_ptr |= (u64)kbase_csf_firmware_cs_output(stream, + CS_STATUS_CMD_PTR_HI) << 32; + req_res = kbase_csf_firmware_cs_output(stream, + CS_STATUS_REQ_RESOURCE); + + seq_printf(file, "CMD_PTR: 0x%llx\n", cmd_ptr); + seq_printf(file, "REQ_RESOURCE [COMPUTE]: %d\n", + CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_GET(req_res)); + seq_printf(file, "REQ_RESOURCE [FRAGMENT]: %d\n", + CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_GET(req_res)); + seq_printf(file, "REQ_RESOURCE [TILER]: %d\n", + CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_GET(req_res)); + seq_printf(file, "REQ_RESOURCE [IDVS]: %d\n", + CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_GET(req_res)); + + wait_status = kbase_csf_firmware_cs_output(stream, + CS_STATUS_WAIT); + wait_sync_value = kbase_csf_firmware_cs_output(stream, + CS_STATUS_WAIT_SYNC_VALUE); + wait_sync_pointer = kbase_csf_firmware_cs_output(stream, + CS_STATUS_WAIT_SYNC_POINTER_LO); + wait_sync_pointer |= (u64)kbase_csf_firmware_cs_output(stream, + CS_STATUS_WAIT_SYNC_POINTER_HI) << 32; + + evt = (u64 *)kbase_phy_alloc_mapping_get(queue->kctx, wait_sync_pointer, &mapping); + if (evt) { + wait_sync_live_value = evt[0]; + kbase_phy_alloc_mapping_put(queue->kctx, mapping); + } else { + wait_sync_live_value = U64_MAX; + } + + kbasep_csf_scheduler_dump_active_queue_cs_status_wait( + file, wait_status, wait_sync_value, + wait_sync_live_value, wait_sync_pointer); + } + + seq_puts(file, "\n"); +} + +/* Waiting timeout for STATUS_UPDATE acknowledgment, in milliseconds */ +#define CSF_STATUS_UPDATE_TO_MS (100) + +static void kbasep_csf_scheduler_dump_active_group(struct seq_file *file, + struct kbase_queue_group *const group) +{ + if (kbase_csf_scheduler_group_get_slot(group) >= 0) { + struct kbase_device *const kbdev = group->kctx->kbdev; + unsigned long flags; + u32 ep_c, ep_r; + char exclusive; + struct kbase_csf_cmd_stream_group_info const *const ginfo = + &kbdev->csf.global_iface.groups[group->csg_nr]; + long remaining = + kbase_csf_timeout_in_jiffies(CSF_STATUS_UPDATE_TO_MS); + u8 slot_priority = + kbdev->csf.scheduler.csg_slots[group->csg_nr].priority; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, + ~kbase_csf_firmware_csg_output(ginfo, CSG_ACK), + CSG_REQ_STATUS_UPDATE_MASK); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + kbase_csf_ring_csg_doorbell(kbdev, group->csg_nr); + + remaining = wait_event_timeout(kbdev->csf.event_wait, + !((kbase_csf_firmware_csg_input_read(ginfo, CSG_REQ) ^ + kbase_csf_firmware_csg_output(ginfo, CSG_ACK)) & + CSG_REQ_STATUS_UPDATE_MASK), remaining); + + ep_c = kbase_csf_firmware_csg_output(ginfo, + CSG_STATUS_EP_CURRENT); + ep_r = kbase_csf_firmware_csg_output(ginfo, CSG_STATUS_EP_REQ); + + if (CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_GET(ep_r)) + exclusive = 'C'; + else if (CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_GET(ep_r)) + exclusive = 'F'; + else + exclusive = '0'; + + if (!remaining) { + dev_err(kbdev->dev, + "Timed out for STATUS_UPDATE on group %d on slot %d", + group->handle, group->csg_nr); + + seq_printf(file, "*** Warn: Timed out for STATUS_UPDATE on slot %d\n", + group->csg_nr); + seq_printf(file, "*** The following group-record is likely stale\n"); + } + + seq_puts(file, "GroupID, CSG NR, CSG Prio, Run State, Priority, C_EP(Alloc/Req), F_EP(Alloc/Req), T_EP(Alloc/Req), Exclusive\n"); + seq_printf(file, "%7d, %6d, %8d, %9d, %8d, %11d/%3d, %11d/%3d, %11d/%3d, %9c\n", + group->handle, + group->csg_nr, + slot_priority, + group->run_state, + group->priority, + CSG_STATUS_EP_CURRENT_COMPUTE_EP_GET(ep_c), + CSG_STATUS_EP_REQ_COMPUTE_EP_GET(ep_r), + CSG_STATUS_EP_CURRENT_FRAGMENT_EP_GET(ep_c), + CSG_STATUS_EP_REQ_FRAGMENT_EP_GET(ep_r), + CSG_STATUS_EP_CURRENT_TILER_EP_GET(ep_c), + CSG_STATUS_EP_REQ_TILER_EP_GET(ep_r), + exclusive); + } else { + seq_puts(file, "GroupID, CSG NR, Run State, Priority\n"); + seq_printf(file, "%7d, %6d, %9d, %8d\n", + group->handle, + group->csg_nr, + group->run_state, + group->priority); + } + + if (group->run_state != KBASE_CSF_GROUP_TERMINATED) { + unsigned int i; + + seq_puts(file, "Bound queues:\n"); + + for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { + kbasep_csf_scheduler_dump_active_queue(file, + group->bound_queues[i]); + } + } + + seq_puts(file, "\n"); +} + +/** + * kbasep_csf_queue_group_debugfs_show() - Print per-context GPU command queue + * group debug information + * + * @file: The seq_file for printing to + * @data: The debugfs dentry private data, a pointer to kbase context + * + * Return: Negative error code or 0 on success. + */ +static int kbasep_csf_queue_group_debugfs_show(struct seq_file *file, + void *data) +{ + u32 gr; + struct kbase_context *const kctx = file->private; + struct kbase_device *const kbdev = kctx->kbdev; + + if (WARN_ON(!kctx)) + return -EINVAL; + + seq_printf(file, "MALI_CSF_CSG_DEBUGFS_VERSION: v%u\n", + MALI_CSF_CSG_DEBUGFS_VERSION); + + mutex_lock(&kctx->csf.lock); + kbase_csf_scheduler_lock(kbdev); + for (gr = 0; gr < MAX_QUEUE_GROUP_NUM; gr++) { + struct kbase_queue_group *const group = + kctx->csf.queue_groups[gr]; + + if (group) + kbasep_csf_scheduler_dump_active_group(file, group); + } + kbase_csf_scheduler_unlock(kbdev); + mutex_unlock(&kctx->csf.lock); + + return 0; +} + +/** + * kbasep_csf_scheduler_dump_active_groups() - Print debug info for active + * GPU command queue groups + * + * @file: The seq_file for printing to + * @data: The debugfs dentry private data, a pointer to kbase_device + * + * Return: Negative error code or 0 on success. + */ +static int kbasep_csf_scheduler_dump_active_groups(struct seq_file *file, + void *data) +{ + u32 csg_nr; + struct kbase_device *kbdev = file->private; + u32 num_groups = kbdev->csf.global_iface.group_num; + + seq_printf(file, "MALI_CSF_CSG_DEBUGFS_VERSION: v%u\n", + MALI_CSF_CSG_DEBUGFS_VERSION); + + kbase_csf_scheduler_lock(kbdev); + for (csg_nr = 0; csg_nr < num_groups; csg_nr++) { + struct kbase_queue_group *const group = + kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; + + if (!group) + continue; + + seq_printf(file, "\nCtx %d_%d\n", group->kctx->tgid, + group->kctx->id); + + kbasep_csf_scheduler_dump_active_group(file, group); + } + kbase_csf_scheduler_unlock(kbdev); + + return 0; +} + +static int kbasep_csf_queue_group_debugfs_open(struct inode *in, + struct file *file) +{ + return single_open(file, kbasep_csf_queue_group_debugfs_show, + in->i_private); +} + +static int kbasep_csf_active_queue_groups_debugfs_open(struct inode *in, + struct file *file) +{ + return single_open(file, kbasep_csf_scheduler_dump_active_groups, + in->i_private); +} + +static const struct file_operations kbasep_csf_queue_group_debugfs_fops = { + .open = kbasep_csf_queue_group_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void kbase_csf_queue_group_debugfs_init(struct kbase_context *kctx) +{ + struct dentry *file; +#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) + const mode_t mode = 0444; +#else + const mode_t mode = 0400; +#endif + + if (WARN_ON(!kctx || IS_ERR_OR_NULL(kctx->kctx_dentry))) + return; + + file = debugfs_create_file("groups", mode, + kctx->kctx_dentry, kctx, &kbasep_csf_queue_group_debugfs_fops); + + if (IS_ERR_OR_NULL(file)) { + dev_warn(kctx->kbdev->dev, + "Unable to create per context queue groups debugfs entry"); + } +} + +static const struct file_operations + kbasep_csf_active_queue_groups_debugfs_fops = { + .open = kbasep_csf_active_queue_groups_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int kbasep_csf_debugfs_scheduling_timer_enabled_get( + void *data, u64 *val) +{ + struct kbase_device *const kbdev = data; + + *val = kbase_csf_scheduler_timer_is_enabled(kbdev); + + return 0; +} + +static int kbasep_csf_debugfs_scheduling_timer_enabled_set( + void *data, u64 val) +{ + struct kbase_device *const kbdev = data; + + kbase_csf_scheduler_timer_set_enabled(kbdev, val != 0); + + return 0; +} + +static int kbasep_csf_debugfs_scheduling_timer_kick_set( + void *data, u64 val) +{ + struct kbase_device *const kbdev = data; + + kbase_csf_scheduler_kick(kbdev); + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(kbasep_csf_debugfs_scheduling_timer_enabled_fops, + &kbasep_csf_debugfs_scheduling_timer_enabled_get, + &kbasep_csf_debugfs_scheduling_timer_enabled_set, + "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(kbasep_csf_debugfs_scheduling_timer_kick_fops, + NULL, + &kbasep_csf_debugfs_scheduling_timer_kick_set, + "%llu\n"); + +void kbase_csf_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("active_groups", 0444, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_csf_active_queue_groups_debugfs_fops); + + debugfs_create_file("scheduling_timer_enabled", 0644, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_csf_debugfs_scheduling_timer_enabled_fops); + debugfs_create_file("scheduling_timer_kick", 0200, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_csf_debugfs_scheduling_timer_kick_fops); + + kbase_csf_tl_reader_debugfs_init(kbdev); + kbase_csf_firmware_trace_buffer_debugfs_init(kbdev); +} + +#else +/* + * Stub functions for when debugfs is disabled + */ +void kbase_csf_queue_group_debugfs_init(struct kbase_context *kctx) +{ +} + +void kbase_csf_debugfs_init(struct kbase_device *kbdev) +{ +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.h new file mode 100755 index 000000000000..c2e99d386f8c --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.h @@ -0,0 +1,48 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_CSG_DEBUGFS_H_ +#define _KBASE_CSF_CSG_DEBUGFS_H_ + +/* Forward declarations */ +struct kbase_device; +struct kbase_context; +struct kbase_queue_group; + +#define MALI_CSF_CSG_DEBUGFS_VERSION 0 + +/** + * kbase_csf_queue_group_debugfs_init() - Add debugfs entry for queue groups + * associated with @kctx. + * + * @kctx: Pointer to kbase_context + */ +void kbase_csf_queue_group_debugfs_init(struct kbase_context *kctx); + +/** + * kbase_csf_debugfs_init() - Add a global debugfs entry for queue groups + * + * @kbdev: Pointer to the device + */ +void kbase_csf_debugfs_init(struct kbase_device *kbdev); + +#endif /* _KBASE_CSF_CSG_DEBUGFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h new file mode 100755 index 000000000000..3829572a1aeb --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h @@ -0,0 +1,883 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* Definitions (types, defines, etcs) common to the command stream frontend. + * They are placed here to allow the hierarchy of header files to work. + */ + +#ifndef _KBASE_CSF_DEFS_H_ +#define _KBASE_CSF_DEFS_H_ + +#include +#include + +#include "mali_kbase_csf_firmware.h" + +/* Maximum number of KCPU command queues to be created per GPU address space. + */ +#define KBASEP_MAX_KCPU_QUEUES ((size_t)256) + +/* Maximum number of GPU command queue groups to be created per GPU address + * space. + */ +#define MAX_QUEUE_GROUP_NUM (256) + +/* Maximum number of GPU tiler heaps to allow to be created per GPU address + * space. + */ +#define MAX_TILER_HEAPS (128) + +/** + * enum kbase_csf_bind_state - bind state of the queue + * + * @KBASE_CSF_QUEUE_UNBOUND: Set when the queue is registered or when the link + * between queue and the group to which it was bound or being bound is removed. + * @KBASE_CSF_QUEUE_BIND_IN_PROGRESS: Set when the first part of bind operation + * has completed i.e. CS_QUEUE_BIND ioctl. + * @KBASE_CSF_QUEUE_BOUND: Set when the bind operation has completed i.e. IO + * pages have been mapped in the process address space. + */ +enum kbase_csf_queue_bind_state { + KBASE_CSF_QUEUE_UNBOUND, + KBASE_CSF_QUEUE_BIND_IN_PROGRESS, + KBASE_CSF_QUEUE_BOUND, +}; + +/** + * enum kbase_csf_reset_gpu_state - state of the gpu reset + * + * @KBASE_CSF_RESET_GPU_NOT_PENDING: Set when the GPU reset isn't pending + * @KBASE_CSF_RESET_GPU_HAPPENING: Set when the GPU reset process is occurring + * @KBASE_CSF_RESET_GPU_SILENT: Set when the GPU reset process is occurring, + * used when resetting the GPU as part of normal behavior (e.g. when exiting + * protected mode). + * @KBASE_CSF_RESET_GPU_FAILED: Set when an error is encountered during the + * GPU reset process. No more work could then be executed on GPU, unloading + * the Driver module is the only option. + */ +enum kbase_csf_reset_gpu_state { + KBASE_CSF_RESET_GPU_NOT_PENDING, + KBASE_CSF_RESET_GPU_HAPPENING, + KBASE_CSF_RESET_GPU_SILENT, + KBASE_CSF_RESET_GPU_FAILED, +}; + +/** + * enum kbase_csf_group_state - state of the GPU command queue group + * + * @KBASE_CSF_GROUP_INACTIVE: Group is inactive and won't be + * considered by scheduler for running on + * command stream group slot. + * @KBASE_CSF_GROUP_RUNNABLE: Group is in the list of runnable groups + * and is subjected to time-slice based + * scheduling. A start request would be + * sent (or already has been sent) if the + * group is assigned the command stream + * group slot for the fist time. + * @KBASE_CSF_GROUP_IDLE: Group is currently on a command stream + * group slot but all the command streams + * bound to the group have become either + * idle or waiting on sync object. + * Group could be evicted from the slot on + * the next tick if there are no spare + * slots left after scheduling non-idle + * queue groups. If the group is kept on + * slot then it would be moved to the + * RUNNABLE state, also if one of the + * queues bound to the group is kicked it + * would be moved to the RUNNABLE state. + * If the group is evicted from the slot it + * would be moved to either + * KBASE_CSF_GROUP_SUSPENDED_ON_IDLE or + * KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC + * state. + * @KBASE_CSF_GROUP_SUSPENDED: Group was evicted from the command + * stream group slot and is not running but + * is still in the list of runnable groups + * and subjected to time-slice based + * scheduling. A resume request would be + * sent when a command stream group slot is + * re-assigned to the group and once the + * resume is complete group would be moved + * back to the RUNNABLE state. + * @KBASE_CSF_GROUP_SUSPENDED_ON_IDLE: Same as KBASE_CSF_GROUP_SUSPENDED except + * that queue group also became idle before + * the suspension. This state helps + * Scheduler avoid scheduling the idle + * groups over the non-idle groups in the + * subsequent ticks. If one of the queues + * bound to the group is kicked it would be + * moved to the SUSPENDED state. + * @KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC: Same as GROUP_SUSPENDED_ON_IDLE + * except that at least one command + * stream bound to this group was + * waiting for synchronization object + * before the suspension. + * @KBASE_CSF_GROUP_FAULT_EVICTED: Group is evicted from the scheduler due + * to a fault condition, pending to be + * terminated. + * @KBASE_CSF_GROUP_TERMINATED: Group is no longer schedulable and is + * pending to be deleted by Client, all the + * queues bound to it have been unbound. + */ +enum kbase_csf_group_state { + KBASE_CSF_GROUP_INACTIVE, + KBASE_CSF_GROUP_RUNNABLE, + KBASE_CSF_GROUP_IDLE, + KBASE_CSF_GROUP_SUSPENDED, + KBASE_CSF_GROUP_SUSPENDED_ON_IDLE, + KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC, + KBASE_CSF_GROUP_FAULT_EVICTED, + KBASE_CSF_GROUP_TERMINATED, +}; + +/** + * enum kbase_csf_csg_slot_state - state of the command queue group slots under + * the scheduler control. + * + * @CSG_SLOT_READY: The slot is clean and ready to be programmed with a + * queue group. + * @CSG_SLOT_READY2RUN: The slot has been programmed with a queue group, i.e. a + * start or resume request has been sent to the firmware. + * @CSG_SLOT_RUNNING: The queue group is running on the slot, acknowledgment + * of a start or resume request has been obtained from the + * firmware. + * @CSG_SLOT_DOWN2STOP: The suspend or terminate request for the queue group on + * the slot has been sent to the firmware. + * @CSG_SLOT_STOPPED: The queue group is removed from the slot, acknowledgment + * of suspend or terminate request has been obtained from + * the firmware. + * @CSG_SLOT_READY2RUN_TIMEDOUT: The start or resume request sent on the slot + * for the queue group timed out. + * @CSG_SLOT_DOWN2STOP_TIMEDOUT: The suspend or terminate request for queue + * group on the slot timed out. + */ +enum kbase_csf_csg_slot_state { + CSG_SLOT_READY, + CSG_SLOT_READY2RUN, + CSG_SLOT_RUNNING, + CSG_SLOT_DOWN2STOP, + CSG_SLOT_STOPPED, + CSG_SLOT_READY2RUN_TIMEDOUT, + CSG_SLOT_DOWN2STOP_TIMEDOUT, +}; + +/** + * enum kbase_csf_scheduler_state - state of the scheduler operational phases. + * + * @SCHED_BUSY: The scheduler is busy performing on tick schedule + * operations, the state of command stream group slots + * can't be changed. + * @SCHED_INACTIVE: The scheduler is inactive, it is allowed to modify the + * state of command stream group slots by in-cycle + * priority scheduling. + * @SCHED_SUSPENDED: The scheduler is in low-power mode with scheduling + * operations suspended and is not holding the power + * management reference. This can happen if the GPU + * becomes idle for a duration exceeding a threshold, + * or due to a system triggered suspend action. + */ +enum kbase_csf_scheduler_state { + SCHED_BUSY, + SCHED_INACTIVE, + SCHED_SUSPENDED, +}; + +/** + * struct kbase_csf_notification - Event or error generated as part of command + * queue execution + * + * @data: Event or error data returned to userspace + * @link: Link to the linked list, &struct_kbase_csf_context.error_list. + */ +struct kbase_csf_notification { + struct base_csf_notification data; + struct list_head link; +}; + +/** + * struct kbase_queue - Object representing a GPU command queue. + * + * @kctx: Pointer to the base context with which this GPU command queue + * is associated. + * @reg: Pointer to the region allocated from the shared + * interface segment for mapping the User mode + * input/output pages in MCU firmware address space. + * @phys: Pointer to the physical pages allocated for the + * pair or User mode input/output page + * @user_io_addr: Pointer to the permanent kernel mapping of User mode + * input/output pages. The pages can be accessed through + * the mapping without any cache maintenance. + * @handle: Handle returned with bind ioctl for creating a + * contiguous User mode mapping of input/output pages & + * the hardware doorbell page. + * @doorbell_nr: Index of the hardware doorbell page assigned to the + * queue. + * @db_file_offset: File offset value that is assigned to userspace mapping + * created on bind to access the doorbell page. + * It is in page units. + * @link: Link to the linked list of GPU command queues created per + * GPU address space. + * @refcount: Reference count, stands for the number of times the queue + * has been referenced. The reference is taken when it is + * created, when it is bound to the group and also when the + * @oom_event_work or @fault_event_work work item is queued + * for it. + * @group: Pointer to the group to which this queue is bound. + * @queue_reg: Pointer to the VA region allocated for command + * stream buffer. + * @oom_event_work: Work item corresponding to the out of memory event for + * chunked tiler heap being used for this queue. + * @fault_event_work: Work item corresponding to the firmware fault event. + * @base_addr: Base address of the command stream buffer. + * @size: Size of the command stream buffer. + * @priority: Priority of this queue within the group. + * @bind_state: Bind state of the queue. + * @csi_index: The ID of the assigned command stream hardware interface. + * @enabled: Indicating whether the command stream is running, or not. + * @status_wait: Value of CS_STATUS_WAIT register of the command stream will + * be kept when the command stream gets blocked by sync wait. + * CS_STATUS_WAIT provides information on conditions queue is + * blocking on. This is set when the group, to which queue is + * bound, is suspended after getting blocked, i.e. in + * KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC state. + * @sync_ptr: Value of CS_STATUS_WAIT_SYNC_POINTER register of the command + * stream will be kept when the command stream gets blocked by + * sync wait. CS_STATUS_WAIT_SYNC_POINTER contains the address + * of synchronization object being waited on. + * Valid only when @status_wait is set. + * @sync_value: Value of CS_STATUS_WAIT_SYNC_VALUE register of the command + * stream will be kept when the command stream gets blocked by + * sync wait. CS_STATUS_WAIT_SYNC_VALUE contains the value + * tested against the synchronization object. + * Valid only when @status_wait is set. + * @error: GPU command queue fatal information to pass to user space. + */ +struct kbase_queue { + struct kbase_context *kctx; + struct kbase_va_region *reg; + struct tagged_addr phys[2]; + char *user_io_addr; + u64 handle; + int doorbell_nr; + unsigned long db_file_offset; + struct list_head link; + atomic_t refcount; + struct kbase_queue_group *group; + struct kbase_va_region *queue_reg; + struct work_struct oom_event_work; + struct work_struct fault_event_work; + u64 base_addr; + u32 size; + u8 priority; + u8 bind_state; + s8 csi_index; + bool enabled; + u32 status_wait; + u64 sync_ptr; + u32 sync_value; + struct kbase_csf_notification error; +}; + +/** + * struct kbase_normal_suspend_buffer - Object representing a normal + * suspend buffer for queue group. + * @reg: Memory region allocated for the normal-mode suspend buffer. + * @phy: Array of physical memory pages allocated for the normal- + * mode suspend buffer. + */ +struct kbase_normal_suspend_buffer { + struct kbase_va_region *reg; + struct tagged_addr *phy; +}; + +/** + * struct kbase_protected_suspend_buffer - Object representing a protected + * suspend buffer for queue group. + * @reg: Memory region allocated for the protected-mode suspend buffer. + * @pma: Array of pointer to protected mode allocations containing + * information about memory pages allocated for protected mode + * suspend buffer. + */ +struct kbase_protected_suspend_buffer { + struct kbase_va_region *reg; + struct protected_memory_allocation **pma; +}; + +/** + * struct kbase_queue_group - Object representing a GPU command queue group. + * + * @kctx: Pointer to the kbase context with which this queue group + * is associated. + * @normal_suspend_buf: Object representing the normal suspend buffer. + * Normal-mode suspend buffer that is used for + * group context switch. + * @protected_suspend_buf: Object representing the protected suspend + * buffer. Protected-mode suspend buffer that is + * used for group context switch. + * @handle: Handle which identifies this queue group. + * @csg_nr: Number/index of the command stream group to + * which this queue group is mapped; KBASEP_CSG_NR_INVALID + * indicates that the queue group is not scheduled. + * @priority: Priority of the queue group, 0 being the highest, + * BASE_QUEUE_GROUP_PRIORITY_COUNT - 1 being the lowest. + * @tiler_max: Maximum number of tiler endpoints the group is allowed + * to use. + * @fragment_max: Maximum number of fragment endpoints the group is + * allowed to use. + * @compute_max: Maximum number of compute endpoints the group is + * allowed to use. + * @tiler_mask: Mask of tiler endpoints the group is allowed to use. + * @fragment_mask: Mask of fragment endpoints the group is allowed to use. + * @compute_mask: Mask of compute endpoints the group is allowed to use. + * @link: Link to this queue group in the 'runnable_groups' list of + * the corresponding kctx. + * @link_to_schedule: Link to this queue group in the list of prepared groups + * to be scheduled, if the group is runnable/suspended. + * If the group is idle or waiting for CQS, it would be a + * link to the list of idle/blocked groups list. + * @timer_event_work: Work item corresponding to the event generated when a task + * started by a queue in this group takes too long to execute + * on an endpoint. + * @run_state: Current state of the queue group. + * @prepared_seq_num: Indicates the position of queue group in the list of + * prepared groups to be scheduled. + * @faulted: Indicates that a GPU fault occurred for the queue group. + * This flag persists until the fault has been queued to be + * reported to userspace. + * @bound_queues: Array of registered queues bound to this queue group. + * @doorbell_nr: Index of the hardware doorbell page assigned to the + * group. + * @protm_event_work: Work item corresponding to the protected mode entry + * event for this queue. + * @protm_pending_bitmap: Bit array to keep a track of command streams that + * have pending protected mode entry requests. + * @error_fatal: An error of type BASE_GPU_QUEUE_GROUP_ERROR_FATAL to be + * returned to userspace if such an error has occurred. + * @error_timeout: An error of type BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT + * to be returned to userspace if such an error has occurred. + * @error_tiler_oom: An error of type BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM + * to be returned to userspace if such an error has occurred. + */ +struct kbase_queue_group { + struct kbase_context *kctx; + struct kbase_normal_suspend_buffer normal_suspend_buf; + struct kbase_protected_suspend_buffer protected_suspend_buf; + u8 handle; + s8 csg_nr; + u8 priority; + + u8 tiler_max; + u8 fragment_max; + u8 compute_max; + + u64 tiler_mask; + u64 fragment_mask; + u64 compute_mask; + + struct list_head link; + struct list_head link_to_schedule; + struct work_struct timer_event_work; + enum kbase_csf_group_state run_state; + u32 prepared_seq_num; + bool faulted; + + struct kbase_queue *bound_queues[MAX_SUPPORTED_STREAMS_PER_GROUP]; + + int doorbell_nr; + struct work_struct protm_event_work; + DECLARE_BITMAP(protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP); + + struct kbase_csf_notification error_fatal; + struct kbase_csf_notification error_timeout; + struct kbase_csf_notification error_tiler_oom; +}; + +/** + * struct kbase_csf_kcpu_queue_context - Object representing the kernel CPU + * queues for a GPU address space. + * + * @lock: Lock preventing concurrent access to @array and the @in_use bitmap. + * @array: Array of pointers to kernel CPU command queues. + * @in_use: Bitmap which indicates which kernel CPU command queues are in use. + * @wq: Dedicated workqueue for processing kernel CPU command queues. + * @num_cmds: The number of commands that have been enqueued across + * all the KCPU command queues. This could be used as a + * timestamp to determine the command's enqueueing time. + * @jit_cmds_head: A list of the just-in-time memory commands, both + * allocate & free, in submission order, protected + * by kbase_csf_kcpu_queue_context.lock. + * @jit_blocked_queues: A list of KCPU command queues blocked by a pending + * just-in-time memory allocation command which will be + * reattempted after the impending free of other active + * allocations. + */ +struct kbase_csf_kcpu_queue_context { + struct mutex lock; + struct kbase_kcpu_command_queue *array[KBASEP_MAX_KCPU_QUEUES]; + DECLARE_BITMAP(in_use, KBASEP_MAX_KCPU_QUEUES); + struct workqueue_struct *wq; + u64 num_cmds; + + struct list_head jit_cmds_head; + struct list_head jit_blocked_queues; +}; + +/** + * struct kbase_csf_heap_context_allocator - Allocator of heap contexts + * + * Heap context structures are allocated by the kernel for use by the firmware. + * The current implementation subdivides a single GPU memory region for use as + * a sparse array. + * + * @kctx: Pointer to the kbase context with which this allocator is + * associated. + * @region: Pointer to a GPU memory region from which heap context structures + * are allocated. NULL if no heap contexts have been allocated. + * @gpu_va: GPU virtual address of the start of the region from which heap + * context structures are allocated. 0 if no heap contexts have been + * allocated. + * @lock: Lock preventing concurrent access to the @in_use bitmap. + * @in_use: Bitmap that indicates which heap context structures are currently + * allocated (in @region). + */ +struct kbase_csf_heap_context_allocator { + struct kbase_context *kctx; + struct kbase_va_region *region; + u64 gpu_va; + struct mutex lock; + DECLARE_BITMAP(in_use, MAX_TILER_HEAPS); +}; + +/** + * struct kbase_csf_tiler_heap_context - Object representing the tiler heaps + * context for a GPU address space. + * + * This contains all of the command-stream front-end state relating to chunked + * tiler heaps for one @kbase_context. It is not the same as a heap context + * structure allocated by the kernel for use by the firmware. + * + * @lock: Lock preventing concurrent access to the tiler heaps. + * @list: List of tiler heaps. + * @ctx_alloc: Allocator for heap context structures. + */ +struct kbase_csf_tiler_heap_context { + struct mutex lock; + struct list_head list; + struct kbase_csf_heap_context_allocator ctx_alloc; +}; + +/** + * struct kbase_csf_scheduler_context - Object representing the scheduler's + * context for a GPU address space. + * + * @runnable_groups: Lists of runnable GPU command queue groups in the kctx, + * one per queue group priority level. + * @num_runnable_grps: Total number of runnable groups across all priority + * levels in @runnable_groups. + * @idle_wait_groups: A list of GPU command queue groups in which all enabled + * GPU command queues are idle and at least one of them + * is blocked on a sync wait operation. + * @num_idle_wait_grps: Length of the @idle_wait_groups list. + * @sync_update_wq: Dedicated workqueue to process work items corresponding + * to the sync_update events by sync_set/sync_add + * instruction execution on command streams bound to groups + * of @idle_wait_groups list. + * @sync_update_work: work item to process the sync_update events by + * sync_set / sync_add instruction execution on command + * streams bound to groups of @idle_wait_groups list. + * @ngrp_to_schedule: Number of groups added for the context to the + * 'groups_to_schedule' list of scheduler instance. + */ +struct kbase_csf_scheduler_context { + struct list_head runnable_groups[BASE_QUEUE_GROUP_PRIORITY_COUNT]; + u32 num_runnable_grps; + struct list_head idle_wait_groups; + u32 num_idle_wait_grps; + struct workqueue_struct *sync_update_wq; + struct work_struct sync_update_work; + u32 ngrp_to_schedule; +}; + +/** + * struct kbase_csf_context - Object representing command-stream front-end + * for a GPU address space. + * + * @event_pages_head: A list of pages allocated for the event memory used by + * the synchronization objects. A separate list would help + * in the fast lookup, since the list is expected to be short + * as one page would provide the memory for up to 1K + * synchronization objects. + * KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES is the upper + * bound on the size of event memory. + * @cookies: Bitmask containing of KBASE_CSF_NUM_USER_IO_PAGES_HANDLE + * bits, used for creating the User mode CPU mapping in a + * deferred manner of a pair of User mode input/output pages + * & a hardware doorbell page. + * The pages are allocated when a GPU command queue is + * bound to a command stream group in kbase_csf_queue_bind. + * This helps returning unique handles to Userspace from + * kbase_csf_queue_bind and later retrieving the pointer to + * queue in the mmap handler. + * @user_pages_info: Array containing pointers to queue + * structures, used in conjunction with cookies bitmask for + * providing a mechansim to create a CPU mapping of + * input/output pages & hardware doorbell page. + * @lock: Serializes accesses to all members, except for ones that + * have their own locks. + * @queue_groups: Array of registered GPU command queue groups. + * @queue_list: Linked list of GPU command queues not yet deregistered. + * Note that queues can persist after deregistration if the + * userspace mapping created for them on bind operation + * hasn't been removed. + * @kcpu_queues: Kernel CPU command queues. + * @event_lock: Lock protecting access to @event_callback_list + * @event_callback_list: List of callbacks which are registered to serve CSF + * events. + * @tiler_heaps: Chunked tiler memory heaps. + * @wq: Dedicated workqueue to process work items corresponding + * to the OoM events raised for chunked tiler heaps being + * used by GPU command queues, and progress timeout events. + * @link: Link to this csf context in the 'runnable_kctxs' list of + * the scheduler instance + * @user_reg_vma: Pointer to the vma corresponding to the virtual mapping + * of the USER register page. Currently used only for sanity + * checking. + * @sched: Object representing the scheduler's context + * @error_list: List for command stream fatal errors in this context. + * Link of fatal error is + * &struct_kbase_csf_notification.link. + * @lock needs to be held to access to this list. + */ +struct kbase_csf_context { + struct list_head event_pages_head; + DECLARE_BITMAP(cookies, KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); + struct kbase_queue *user_pages_info[ + KBASE_CSF_NUM_USER_IO_PAGES_HANDLE]; + struct mutex lock; + struct kbase_queue_group *queue_groups[MAX_QUEUE_GROUP_NUM]; + struct list_head queue_list; + struct kbase_csf_kcpu_queue_context kcpu_queues; + spinlock_t event_lock; + struct list_head event_callback_list; + struct kbase_csf_tiler_heap_context tiler_heaps; + struct workqueue_struct *wq; + struct list_head link; + struct vm_area_struct *user_reg_vma; + struct kbase_csf_scheduler_context sched; + struct list_head error_list; +}; + +/** + * struct kbase_csf_reset_gpu - Object containing the members required for + * GPU reset handling. + * @workq: Workqueue to execute the GPU reset work item @work. + * @work: Work item for performing the GPU reset. + * @wait: Wait queue used to wait for the GPU reset completion. + * @state: Tracks if the GPU reset is in progress or not. + */ +struct kbase_csf_reset_gpu { + struct workqueue_struct *workq; + struct work_struct work; + wait_queue_head_t wait; + atomic_t state; +}; + +/** + * struct kbase_csf_csg_slot - Object containing members for tracking the state + * of command stream group slots. + * @resident_group: pointer to the queue group that is resident on the + * command stream group slot. + * @state: state of the slot as per enum kbase_csf_csg_slot_state. + * @trigger_jiffies: value of jiffies when change in slot state is recorded. + * @priority: dynamic priority assigned to command stream group slot. + */ +struct kbase_csf_csg_slot { + struct kbase_queue_group *resident_group; + atomic_t state; + unsigned long trigger_jiffies; + u8 priority; +}; + +/** + * struct kbase_csf_scheduler - Object representing the scheduler used for + * command-stream front-end for an instance of + * GPU platform device. + * @lock: Lock to serialize the scheduler operations and + * access to the data members. + * @interrupt_lock: Lock to protect members accessed by interrupt + * handler. + * @state: The operational phase the scheduler is in. Primarily + * used for indicating what in-cycle schedule actions + * are allowed. + * @doorbell_inuse_bitmap: Bitmap of hardware doorbell pages keeping track of + * which pages are currently available for assignment + * to clients. + * @csg_inuse_bitmap: Bitmap to keep a track of command stream group slots + * that are currently in use. + * @csg_slots: The array for tracking the state of command stream + * group slots. + * @runnable_kctxs: List of Kbase contexts that have runnable command + * queue groups. + * @groups_to_schedule: List of runnable queue groups prepared on every + * scheduler tick. The dynamic priority of the command + * stream group slot assigned to a group will depend + * upon the position of group in the list. + * @ngrp_to_schedule: Number of groups in the @groups_to_schedule list, + * incremented when a group is added to the list, used + * to record the position of group in the list. + * @num_active_address_spaces: Number of GPU address space slots that would get + * used to program the groups in @groups_to_schedule + * list on all the available command stream group + * slots. + * @num_csg_slots_for_tick: Number of command stream group slots that can be + * active in the given tick/tock. This depends on the + * value of @num_active_address_spaces. + * @idle_groups_to_schedule: List of runnable queue groups, in which all GPU + * command queues became idle or are waiting for + * synchronization object, prepared on every + * scheduler tick. The groups in this list are + * appended to the tail of @groups_to_schedule list + * after the scan out so that the idle groups aren't + * preferred for scheduling over the non-idle ones. + * @total_runnable_grps: Total number of runnable groups across all KCTXs. + * @csgs_events_enable_mask: Use for temporary masking off asynchronous events + * from firmware (such as OoM events) before a group + * is suspended. + * @csg_slots_idle_mask: Bit array for storing the mask of command stream + * group slots for which idle notification was + * received. + * @csg_slots_prio_update: Bit array for tracking slots that have an on-slot + * priority update operation. + * @last_schedule: Time in jiffies recorded when the last "tick" or + * "tock" schedule operation concluded. Used for + * evaluating the exclusion window for in-cycle + * schedule operation. + * @timer_enabled: Whether the CSF scheduler wakes itself up for + * periodic scheduling tasks. If this value is 0 + * then it will only perform scheduling under the + * influence of external factors e.g., IRQs, IOCTLs. + * @wq: Dedicated workqueue to execute the @tick_work. + * @tick_work: Work item that would perform the schedule on tick + * operation to implement the time slice based + * scheduling. + * @tock_work: Work item that would perform the schedule on tock + * operation to implement the asynchronous scheduling. + * @ping_work: Work item that would ping the firmware at regular + * intervals, only if there is a single active command + * stream group slot, to check if firmware is alive + * and would initiate a reset if the ping request + * isn't acknowledged. + * @top_ctx: Pointer to the Kbase context corresponding to the + * @top_grp. + * @top_grp: Pointer to queue group inside @groups_to_schedule + * list that was assigned the highest slot priority. + * @head_slot_priority: The dynamic slot priority to be used for the + * queue group at the head of @groups_to_schedule + * list. Once the queue group is assigned a command + * stream group slot, it is removed from the list and + * priority is decremented. + * @tock_pending_request: A "tock" request is pending: a group that is not + * currently on the GPU demands to be scheduled. + * @active_protm_grp: Indicates if firmware has been permitted to let GPU + * enter protected mode with the given group. On exit + * from protected mode the pointer is reset to NULL. + * @gpu_idle_work: Work item for facilitating the scheduler to bring + * the GPU to a low-power mode on becoming idle. + * @non_idle_suspended_grps: Count of suspended queue groups not idle. + * @pm_active_count: Count indicating if the scheduler is owning a power + * management reference count. Reference is taken when + * the count becomes 1 and is dropped when the count + * becomes 0. It is used to enable the power up of MCU + * after GPU and L2 cache have been powered up. So when + * this count is zero, MCU will not be powered up. + */ +struct kbase_csf_scheduler { + struct mutex lock; + spinlock_t interrupt_lock; + enum kbase_csf_scheduler_state state; + DECLARE_BITMAP(doorbell_inuse_bitmap, CSF_NUM_DOORBELL); + DECLARE_BITMAP(csg_inuse_bitmap, MAX_SUPPORTED_CSGS); + struct kbase_csf_csg_slot *csg_slots; + struct list_head runnable_kctxs; + struct list_head groups_to_schedule; + u32 ngrp_to_schedule; + u32 num_active_address_spaces; + u32 num_csg_slots_for_tick; + struct list_head idle_groups_to_schedule; + u32 total_runnable_grps; + DECLARE_BITMAP(csgs_events_enable_mask, MAX_SUPPORTED_CSGS); + DECLARE_BITMAP(csg_slots_idle_mask, MAX_SUPPORTED_CSGS); + DECLARE_BITMAP(csg_slots_prio_update, MAX_SUPPORTED_CSGS); + unsigned long last_schedule; + bool timer_enabled; + struct workqueue_struct *wq; + struct delayed_work tick_work; + struct delayed_work tock_work; + struct delayed_work ping_work; + struct kbase_context *top_ctx; + struct kbase_queue_group *top_grp; + u8 head_slot_priority; + bool tock_pending_request; + struct kbase_queue_group *active_protm_grp; + struct delayed_work gpu_idle_work; + atomic_t non_idle_suspended_grps; + u32 pm_active_count; +}; + +/** + * Number of GPU cycles per unit of the global progress timeout. + */ +#define GLB_PROGRESS_TIMER_TIMEOUT_SCALE ((u64)1024) + +/** + * Maximum value of the global progress timeout. + */ +#define GLB_PROGRESS_TIMER_TIMEOUT_MAX \ + ((GLB_PROGRESS_TIMER_TIMEOUT_MASK >> \ + GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) * \ + GLB_PROGRESS_TIMER_TIMEOUT_SCALE) + +/** + * struct kbase_csf - Object representing command-stream front-end for an + * instance of GPU platform device. + * + * @mcu_mmu: MMU page tables for the MCU firmware + * @firmware_interfaces: List of interfaces defined in the firmware image + * @firmware_config: List of configuration options within the firmware + * image + * @firmware_timeline_metadata: List of timeline meta-data within the firmware + * image + * @fw_cfg_kobj: Pointer to the kobject corresponding to the sysf + * directory that contains a sub-directory for each + * of the configuration option present in the + * firmware image. + * @firmware_trace_buffers: List of trace buffers described in the firmware + * image. + * @shared_interface: Pointer to the interface object containing info for + * the memory area shared between firmware & host. + * @shared_reg_rbtree: RB tree of the memory regions allocated from the + * shared interface segment in MCU firmware address + * space. + * @db_filp: Pointer to a dummy file, that alongwith + * @db_file_offsets, facilitates the use of unqiue + * file offset for the userspace mapping created + * for Hw Doorbell pages. The userspace mapping + * is made to point to this file inside the mmap + * handler. + * @db_file_offsets: Counter that is incremented every time a GPU + * command queue is bound to provide a unique file + * offset range for @db_filp file, so that pte of + * Doorbell page can be zapped through the kernel + * function unmap_mapping_range(). It is incremented + * in page units. + * @dummy_db_page: Address of the dummy page that is mapped in place + * of the real Hw doorbell page for the active GPU + * command queues after they are stopped or after the + * GPU is powered down. + * @reg_lock: Lock to serialize the MCU firmware related actions + * that affect all contexts such as allocation of + * regions from shared interface area, assignment of + * of hardware doorbell pages, assignment of CSGs, + * sending global requests. + * @event_wait: Wait queue to wait for receiving csf events, i.e. + * the interrupt from CSF firmware, or scheduler state + * changes. + * @interrupt_received: Flag set when the interrupt is received from CSF fw + * @global_iface: The result of parsing the global interface + * structure set up by the firmware, including the + * CSGs, CSs, and their properties + * @scheduler: The command stream scheduler instance. + * @reset: Contain members required for GPU reset handling. + * @progress_timeout: Maximum number of GPU clock cycles without forward + * progress to allow, for all tasks running on + * hardware endpoints (e.g. shader cores), before + * terminating a GPU command queue group. + * Must not exceed @GLB_PROGRESS_TIMER_TIMEOUT_MAX. + * @pma_dev: Pointer to protected memory allocator device. + * @firmware_inited: Flag for indicating that the cold-boot stage of + * the MCU has completed. + * @firmware_reloaded: Flag for indicating a firmware reload operation + * in GPU reset has completed. + * @firmware_reload_needed: Flag for indicating that the firmware needs to be + * reloaded as part of the GPU reset action. + * @firmware_reload_work: Work item for facilitating the procedural actions + * on reloading the firmware. + * @glb_init_request_pending: Flag to indicate that Global requests have been + * sent to the FW after MCU was re-enabled and their + * acknowledgement is pending. + */ +struct kbase_csf_device { + struct kbase_mmu_table mcu_mmu; + struct list_head firmware_interfaces; + struct list_head firmware_config; + struct list_head firmware_timeline_metadata; + struct kobject *fw_cfg_kobj; + struct kbase_csf_trace_buffers firmware_trace_buffers; + void *shared_interface; + struct rb_root shared_reg_rbtree; + struct file *db_filp; + u32 db_file_offsets; + struct tagged_addr dummy_db_page; + struct mutex reg_lock; + wait_queue_head_t event_wait; + bool interrupt_received; + struct kbase_csf_global_iface global_iface; + struct kbase_csf_scheduler scheduler; + struct kbase_csf_reset_gpu reset; + atomic64_t progress_timeout; + struct protected_memory_allocator_device *pma_dev; + bool firmware_inited; + bool firmware_reloaded; + bool firmware_reload_needed; + struct work_struct firmware_reload_work; + bool glb_init_request_pending; +}; + +/** + * struct kbase_as - Object representing an address space of GPU. + * @number: Index at which this address space structure is present + * in an array of address space structures embedded inside + * the &struct kbase_device. + * @pf_wq: Workqueue for processing work items related to + * Page fault, Bus fault and GPU fault handling. + * @work_pagefault: Work item for the Page fault handling. + * @work_busfault: Work item for the Bus fault handling. + * @work_gpufault: Work item for the GPU fault handling. + * @pf_data: Data relating to Page fault. + * @bf_data: Data relating to Bus fault. + * @gf_data: Data relating to GPU fault. + * @current_setup: Stores the MMU configuration for this address space. + */ +struct kbase_as { + int number; + struct workqueue_struct *pf_wq; + struct work_struct work_pagefault; + struct work_struct work_busfault; + struct work_struct work_gpufault; + struct kbase_fault pf_data; + struct kbase_fault bf_data; + struct kbase_fault gf_data; + struct kbase_mmu_setup current_setup; +}; + +#endif /* _KBASE_CSF_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c new file mode 100755 index 000000000000..4a924f346685 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c @@ -0,0 +1,1993 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase.h" +#include "mali_kbase_csf_firmware_cfg.h" +#include "mali_kbase_csf_trace_buffer.h" +#include "mali_kbase_csf_timeout.h" +#include "mali_kbase_mem.h" +#include +#include "mali_kbase_csf_scheduler.h" +#include "device/mali_kbase_device.h" +#include "backend/gpu/mali_kbase_pm_internal.h" +#include "tl/mali_kbase_timeline_priv.h" +#include "mali_kbase_csf_tl_reader.h" + +#include +#include +#include +#include +#include +#if (KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE) +#include +#endif +#include + +#define MALI_MAX_FIRMWARE_NAME_LEN ((size_t)20) + +static char fw_name[MALI_MAX_FIRMWARE_NAME_LEN] = "mali_csffw.bin"; +module_param_string(fw_name, fw_name, sizeof(fw_name), 0644); +MODULE_PARM_DESC(fw_name, "firmware image"); + +#ifdef CONFIG_MALI_BIFROST_DEBUG +/* Makes Driver wait indefinitely for an acknowledgment for the different + * requests it sends to firmware. Otherwise the timeouts interfere with the + * use of debugger for source-level debugging of firmware as Driver initiates + * a GPU reset when a request times out, which always happen when a debugger + * is connected. + */ +bool fw_debug; /* Default value of 0/false */ +module_param(fw_debug, bool, 0444); +MODULE_PARM_DESC(fw_debug, + "Enables effective use of a debugger for debugging firmware code."); +#endif + +#define FIRMWARE_HEADER_MAGIC (0xC3F13A6Eul) +#define FIRMWARE_HEADER_VERSION (0ul) +#define FIRMWARE_HEADER_LENGTH (0x14ul) + +#define CSF_FIRMWARE_ENTRY_READ (1ul << 0) +#define CSF_FIRMWARE_ENTRY_WRITE (1ul << 1) +#define CSF_FIRMWARE_ENTRY_EXECUTE (1ul << 2) +#define CSF_FIRMWARE_ENTRY_CACHE_MODE (3ul << 3) +#define CSF_FIRMWARE_ENTRY_PROTECTED (1ul << 5) +#define CSF_FIRMWARE_ENTRY_SHARED (1ul << 30) +#define CSF_FIRMWARE_ENTRY_ZERO (1ul << 31) + +#define CSF_FIRMWARE_ENTRY_SUPPORTED_FLAGS \ + (CSF_FIRMWARE_ENTRY_READ | \ + CSF_FIRMWARE_ENTRY_WRITE | \ + CSF_FIRMWARE_ENTRY_EXECUTE | \ + CSF_FIRMWARE_ENTRY_PROTECTED | \ + CSF_FIRMWARE_ENTRY_SHARED | \ + CSF_FIRMWARE_ENTRY_ZERO | \ + CSF_FIRMWARE_ENTRY_CACHE_MODE) + +#define CSF_FIRMWARE_ENTRY_TYPE_INTERFACE (0) +#define CSF_FIRMWARE_ENTRY_TYPE_CONFIGURATION (1) +#define CSF_FIRMWARE_ENTRY_TYPE_FUTF_TEST (2) +#define CSF_FIRMWARE_ENTRY_TYPE_TRACE_BUFFER (3) +#define CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA (4) + +#define CSF_FIRMWARE_CACHE_MODE_NONE (0ul << 3) +#define CSF_FIRMWARE_CACHE_MODE_CACHED (1ul << 3) +#define CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT (2ul << 3) +#define CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT (3ul << 3) + +#define INTERFACE_ENTRY_NAME_OFFSET (0x14) + +#define TL_METADATA_ENTRY_NAME_OFFSET (0x8) + +#define CSF_FIRMWARE_BOOT_TIMEOUT_MS (500) +#define CSF_MAX_FW_STOP_LOOPS (100000) + +#define CSF_GLB_REQ_CFG_MASK \ + (GLB_REQ_CFG_ALLOC_EN_MASK | GLB_REQ_CFG_PROGRESS_TIMER_MASK) + +static inline u32 input_page_read(const u32 *const input, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return input[offset / sizeof(u32)]; +} + +static inline void input_page_write(u32 *const input, const u32 offset, + const u32 value) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = value; +} + +static inline void input_page_partial_write(u32 *const input, const u32 offset, + u32 value, u32 mask) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = + (input_page_read(input, offset) & ~mask) | (value & mask); +} + +static inline u32 output_page_read(const u32 *const output, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return output[offset / sizeof(u32)]; +} + +static unsigned int entry_type(u32 header) +{ + return header & 0xFF; +} +static unsigned int entry_size(u32 header) +{ + return (header >> 8) & 0xFF; +} +static bool entry_optional(u32 header) +{ + return (header >> 31) & 0x1; +} + +/** + * struct firmware_interface - Represents an interface in the MCU firmware + * + * @node: Interface objects are on the kbase_device:csf.firmware_interfaces + * list using this list_head to link them + * @phys: Array of the physical (tagged) addresses making up this interface + * @name: NUL-terminated string naming the interface + * @num_pages: Number of entries in @phys (and length of the interface) + * @virtual: Virtual address that this interface is mapped at for the GPU + * @flags: bitmask of CSF_FIRMWARE_ENTRY_* conveying the interface attributes + * @data_start: Offset into firmware image at which the interface data starts + * @data_end: Offset into firmware image at which the interface data ends + * @kernel_map: A kernel mapping of the memory or NULL if not required to be + * mapped in the kernel + * @pma: Array of pointers to protected memory allocations. + */ +struct firmware_interface { + struct list_head node; + struct tagged_addr *phys; + char *name; + u32 num_pages; + u32 virtual; + u32 flags; + u32 data_start; + u32 data_end; + void *kernel_map; + struct protected_memory_allocation **pma; +}; + +/** + * Timeline metadata item within the MCU firmware + * + * @node: List head linking all timeline metadata to + * kbase_device:csf.firmware_timeline_metadata. + * @name: NUL-terminated string naming the metadata. + * @data: Metadata content. + * @size: Metadata size. + */ +struct firmware_timeline_metadata { + struct list_head node; + char *name; + char *data; + size_t size; +}; + +/* The shared interface area, used for communicating with firmware, is managed + * like a virtual memory zone. Reserve the virtual space from that zone + * corresponding to shared interface entry parsed from the firmware image. + * The shared_reg_rbtree should have been initialized before calling this + * function. + */ +static int setup_shared_iface_static_region(struct kbase_device *kbdev) +{ + struct firmware_interface *interface = kbdev->csf.shared_interface; + struct kbase_va_region *reg; + int ret = -ENOMEM; + + if (!interface) + return -EINVAL; + + reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, + interface->num_pages, KBASE_REG_ZONE_MCU_SHARED); + if (reg) { + ret = kbase_add_va_region_rbtree(kbdev, reg, + interface->virtual, interface->num_pages, 1); + if (ret) + kfree(reg); + else + reg->flags &= ~KBASE_REG_FREE; + } + + return ret; +} + +static int wait_mcu_status_value(struct kbase_device *kbdev, u32 val) +{ + u32 max_loops = CSF_MAX_FW_STOP_LOOPS; + + /* wait for the MCU_STATUS register to reach the given status value */ + while (--max_loops && + (kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS)) != val)) { + } + + return (max_loops == 0) ? -1 : 0; +} + +void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) +{ + if (wait_mcu_status_value(kbdev, MCU_CNTRL_DISABLE) < 0) + dev_err(kbdev->dev, "MCU failed to get disabled"); +} + +static void wait_for_firmware_stop(struct kbase_device *kbdev) +{ + if (wait_mcu_status_value(kbdev, MCU_CNTRL_DISABLE) < 0) { + /* This error shall go away once MIDJM-2371 is closed */ + dev_err(kbdev->dev, "Firmware failed to stop"); + } +} + +static void stop_csf_firmware(struct kbase_device *kbdev) +{ + /* Stop the MCU firmware */ + kbase_csf_firmware_disable_mcu(kbdev); + + wait_for_firmware_stop(kbdev); +} + +static void wait_for_firmware_boot(struct kbase_device *kbdev) +{ + const long wait_timeout = + kbase_csf_timeout_in_jiffies(CSF_FIRMWARE_BOOT_TIMEOUT_MS); + long remaining; + + /* Firmware will generate a global interface interrupt once booting + * is complete + */ + remaining = wait_event_timeout(kbdev->csf.event_wait, + kbdev->csf.interrupt_received == true, wait_timeout); + + if (!remaining) + dev_err(kbdev->dev, "Timed out waiting for fw boot completion"); + + kbdev->csf.interrupt_received = false; +} + +static void boot_csf_firmware(struct kbase_device *kbdev) +{ + kbase_csf_firmware_enable_mcu(kbdev); + + wait_for_firmware_boot(kbdev); +} + +static void wait_ready(struct kbase_device *kbdev) +{ + u32 max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; + u32 val; + + val = kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS)); + + /* Wait for a while for the update command to take effect */ + while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) + val = kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS)); + + if (max_loops == 0) + dev_err(kbdev->dev, "AS_ACTIVE bit stuck, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n"); +} + +static void unload_mmu_tables(struct kbase_device *kbdev) +{ + unsigned long irq_flags; + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + if (kbdev->pm.backend.gpu_powered) + kbase_mmu_disable_as(kbdev, MCU_AS_NR); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); + mutex_unlock(&kbdev->mmu_hw_mutex); +} + +static void load_mmu_tables(struct kbase_device *kbdev) +{ + unsigned long irq_flags; + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + kbase_mmu_update(kbdev, &kbdev->csf.mcu_mmu, MCU_AS_NR); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + /* Wait for a while for the update command to take effect */ + wait_ready(kbdev); +} + +/** + * convert_mem_flags() - Convert firmware memory flags to GPU region flags + * + * Return: GPU memory region flags + * + * @kbdev: Instance of GPU platform device (used to determine system coherency) + * @flags: Flags of an "interface memory setup" section in a firmware image + * @cm: appropriate cache mode chosen for the "interface memory setup" + * section, which could be different from the cache mode requested by + * firmware. + */ +static unsigned long convert_mem_flags(const struct kbase_device * const kbdev, + const u32 flags, u32 *cm) +{ + unsigned long mem_flags = 0; + u32 cache_mode = flags & CSF_FIRMWARE_ENTRY_CACHE_MODE; + bool is_shared = (flags & CSF_FIRMWARE_ENTRY_SHARED) ? true : false; + + /* The memory flags control the access permissions for the MCU, the + * shader cores/tiler are not expected to access this memory + */ + if (flags & CSF_FIRMWARE_ENTRY_READ) + mem_flags |= KBASE_REG_GPU_RD; + + if (flags & CSF_FIRMWARE_ENTRY_WRITE) + mem_flags |= KBASE_REG_GPU_WR; + + if ((flags & CSF_FIRMWARE_ENTRY_EXECUTE) == 0) + mem_flags |= KBASE_REG_GPU_NX; + + if (flags & CSF_FIRMWARE_ENTRY_PROTECTED) + mem_flags |= KBASE_REG_PROTECTED; + + /* Substitute uncached coherent memory for cached coherent memory if + * the system does not support ACE coherency. + */ + if ((cache_mode == CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT) && + (kbdev->system_coherency != COHERENCY_ACE)) + cache_mode = CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT; + + /* Substitute uncached incoherent memory for uncached coherent memory + * if the system does not support ACE-Lite coherency. + */ + if ((cache_mode == CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT) && + (kbdev->system_coherency == COHERENCY_NONE)) + cache_mode = CSF_FIRMWARE_CACHE_MODE_NONE; + + *cm = cache_mode; + + switch (cache_mode) { + case CSF_FIRMWARE_CACHE_MODE_NONE: + mem_flags |= + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); + break; + case CSF_FIRMWARE_CACHE_MODE_CACHED: + mem_flags |= + KBASE_REG_MEMATTR_INDEX( + AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY); + break; + case CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT: + case CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT: + WARN_ON(!is_shared); + mem_flags |= KBASE_REG_SHARE_BOTH | + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED); + break; + default: + dev_err(kbdev->dev, + "Firmware contains interface with unsupported cache mode\n"); + break; + } + return mem_flags; +} + +static void load_fw_image_section(struct kbase_device *kbdev, const u8 *data, + struct tagged_addr *phys, u32 num_pages, u32 flags, + u32 data_start, u32 data_end) +{ + u32 data_pos = data_start; + u32 data_len = data_end - data_start; + u32 page_num; + u32 page_limit; + + if (flags & CSF_FIRMWARE_ENTRY_ZERO) + page_limit = num_pages; + else + page_limit = (data_len + PAGE_SIZE - 1) / PAGE_SIZE; + + for (page_num = 0; page_num < page_limit; ++page_num) { + struct page *const page = as_page(phys[page_num]); + char *const p = kmap_atomic(page); + u32 const copy_len = min_t(u32, PAGE_SIZE, data_len); + + if (copy_len > 0) { + memcpy(p, data + data_pos, copy_len); + data_pos += copy_len; + data_len -= copy_len; + } + + if (flags & CSF_FIRMWARE_ENTRY_ZERO) { + u32 const zi_len = PAGE_SIZE - copy_len; + + memset(p + copy_len, 0, zi_len); + } + + kbase_sync_single_for_device(kbdev, kbase_dma_addr(page), + PAGE_SIZE, DMA_TO_DEVICE); + kunmap_atomic(p); + } +} + +static int reload_fw_data_sections(struct kbase_device *kbdev) +{ + const u32 magic = FIRMWARE_HEADER_MAGIC; + struct firmware_interface *interface; + const struct firmware *firmware; + int ret = 0; + + if (request_firmware(&firmware, fw_name, kbdev->dev) != 0) { + dev_err(kbdev->dev, + "Failed to reload firmware image '%s'\n", + fw_name); + return -ENOENT; + } + + /* Do couple of basic sanity checks */ + if (firmware->size < FIRMWARE_HEADER_LENGTH) { + dev_err(kbdev->dev, "Firmware image unexpectedly too small\n"); + ret = -EINVAL; + goto out; + } + + if (memcmp(firmware->data, &magic, sizeof(magic)) != 0) { + dev_err(kbdev->dev, "Incorrect magic value, firmware image could have been corrupted\n"); + ret = -EINVAL; + goto out; + } + + list_for_each_entry(interface, &kbdev->csf.firmware_interfaces, node) { + /* Skip reload of text & read only data sections */ + if ((interface->flags & CSF_FIRMWARE_ENTRY_EXECUTE) || + !(interface->flags & CSF_FIRMWARE_ENTRY_WRITE)) + continue; + + load_fw_image_section(kbdev, firmware->data, interface->phys, + interface->num_pages, interface->flags, + interface->data_start, interface->data_end); + } + + kbase_csf_firmware_reload_trace_buffers_data(kbdev); + +out: + release_firmware(firmware); + return ret; +} + +/** + * parse_memory_setup_entry() - Process an "interface memory setup" section + * + * Read an "interface memory setup" section from the firmware image and create + * the necessary memory region including the MMU page tables. If successful + * the interface will be added to the kbase_device:csf.firmware_interfaces list. + * + * Return: 0 if successful, negative error code on failure + * + * @kbdev: Kbase device structure + * @fw: The firmware image containing the section + * @entry: Pointer to the start of the section + * @size: Size (in bytes) of the section + */ +static int parse_memory_setup_entry(struct kbase_device *kbdev, + const struct firmware *fw, + const u32 *entry, unsigned int size) +{ + int ret = 0; + const u32 flags = entry[0]; + const u32 virtual_start = entry[1]; + const u32 virtual_end = entry[2]; + const u32 data_start = entry[3]; + const u32 data_end = entry[4]; + u32 num_pages; + char *name; + struct tagged_addr *phys = NULL; + struct firmware_interface *interface = NULL; + bool allocated_pages = false, protected_mode = false; + unsigned long mem_flags = 0; + u32 cache_mode = 0; + struct protected_memory_allocation **pma = NULL; + + if (data_end < data_start) { + dev_err(kbdev->dev, "Firmware corrupt, data_end < data_start (0x%x<0x%x)\n", + data_end, data_start); + return -EINVAL; + } + if (virtual_end < virtual_start) { + dev_err(kbdev->dev, "Firmware corrupt, virtual_end < virtual_start (0x%x<0x%x)\n", + virtual_end, virtual_start); + return -EINVAL; + } + if (data_end > fw->size) { + dev_err(kbdev->dev, "Firmware corrupt, file truncated? data_end=0x%x > fw->size=0x%zx\n", + data_end, fw->size); + return -EINVAL; + } + + if ((virtual_start & ~PAGE_MASK) != 0 || + (virtual_end & ~PAGE_MASK) != 0) { + dev_err(kbdev->dev, "Firmware corrupt: virtual addresses not page aligned: 0x%x-0x%x\n", + virtual_start, virtual_end); + return -EINVAL; + } + + if ((flags & CSF_FIRMWARE_ENTRY_SUPPORTED_FLAGS) != flags) { + dev_err(kbdev->dev, "Firmware contains interface with unsupported flags (0x%x)\n", + flags); + return -EINVAL; + } + + if (flags & CSF_FIRMWARE_ENTRY_PROTECTED) + protected_mode = true; + + if (protected_mode && kbdev->csf.pma_dev == NULL) { + dev_err(kbdev->dev, + "Protected memory allocator not found, Firmware protected mode entry will not be supported"); + return 0; + } + + num_pages = (virtual_end - virtual_start) + >> PAGE_SHIFT; + + phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); + if (!phys) + return -ENOMEM; + + if (protected_mode) { + pma = kbase_csf_protected_memory_alloc(kbdev, phys, num_pages); + + if (pma == NULL) { + ret = -ENOMEM; + goto out; + } + } else { + ret = kbase_mem_pool_alloc_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + num_pages, phys, false); + if (ret < 0) + goto out; + } + + allocated_pages = true; + load_fw_image_section(kbdev, fw->data, phys, num_pages, flags, + data_start, data_end); + + /* Allocate enough memory for the struct firmware_interface and + * the name of the interface. An extra byte is allocated to place a + * NUL-terminator in. This should already be included according to the + * specification but here we add it anyway to be robust against a + * corrupt firmware image. + */ + interface = kmalloc(sizeof(*interface) + + size - INTERFACE_ENTRY_NAME_OFFSET + 1, GFP_KERNEL); + if (!interface) { + ret = -ENOMEM; + goto out; + } + name = (void *)(interface + 1); + memcpy(name, entry + (INTERFACE_ENTRY_NAME_OFFSET / sizeof(*entry)), + size - INTERFACE_ENTRY_NAME_OFFSET); + name[size - INTERFACE_ENTRY_NAME_OFFSET] = 0; + + interface->name = name; + interface->phys = phys; + interface->num_pages = num_pages; + interface->virtual = virtual_start; + interface->kernel_map = NULL; + interface->flags = flags; + interface->data_start = data_start; + interface->data_end = data_end; + interface->pma = pma; + + mem_flags = convert_mem_flags(kbdev, flags, &cache_mode); + + if (flags & CSF_FIRMWARE_ENTRY_SHARED) { + struct page **page_list; + u32 i; + pgprot_t cpu_map_prot; + u32 mem_attr_index = KBASE_REG_MEMATTR_VALUE(mem_flags); + + /* Since SHARED memory type was used for mapping shared memory + * on GPU side, it can be mapped as cached on CPU side on both + * types of coherent platforms. + */ + if ((cache_mode == CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT) || + (cache_mode == CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT)) { + WARN_ON(mem_attr_index != + AS_MEMATTR_INDEX_SHARED); + cpu_map_prot = PAGE_KERNEL; + } else { + WARN_ON(mem_attr_index != + AS_MEMATTR_INDEX_NON_CACHEABLE); + cpu_map_prot = pgprot_writecombine(PAGE_KERNEL); + } + + page_list = kmalloc_array(num_pages, sizeof(*page_list), + GFP_KERNEL); + if (!page_list) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < num_pages; i++) + page_list[i] = as_page(phys[i]); + + interface->kernel_map = vmap(page_list, num_pages, VM_MAP, + cpu_map_prot); + + kfree(page_list); + + if (!interface->kernel_map) { + ret = -ENOMEM; + goto out; + } + } + + /* Start location of the shared interface area is fixed and is + * specified in firmware spec, and so there shall only be a + * single entry with that start address. + */ + if (virtual_start == (KBASE_REG_ZONE_MCU_SHARED_BASE << PAGE_SHIFT)) + kbdev->csf.shared_interface = interface; + + list_add(&interface->node, &kbdev->csf.firmware_interfaces); + + ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu, + virtual_start >> PAGE_SHIFT, phys, num_pages, mem_flags, + KBASE_MEM_GROUP_CSF_FW); + + if (ret != 0) { + dev_err(kbdev->dev, "Failed to insert firmware pages\n"); + /* The interface has been added to the list, so cleanup will + * be handled by firmware unloading + */ + } + + dev_dbg(kbdev->dev, "Processed section '%s'", name); + + return ret; + +out: + if (allocated_pages) { + if (protected_mode) { + if (interface) { + kbase_csf_protected_memory_free(kbdev, + interface->pma, num_pages); + } + } else { + kbase_mem_pool_free_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + num_pages, phys, false, false); + } + } + + kfree(phys); + kfree(interface); + return ret; +} + +/** + * parse_timeline_metadata_entry() - Process a "timeline metadata" section + * + * Return: 0 if successful, negative error code on failure + * + * @kbdev: Kbase device structure + * @fw: Firmware image containing the section + * @entry: Pointer to the section + * @size: Size (in bytes) of the section + */ +static int parse_timeline_metadata_entry(struct kbase_device *kbdev, + const struct firmware *fw, const u32 *entry, unsigned int size) +{ + const u32 data_start = entry[0]; + const u32 data_size = entry[1]; + const u32 data_end = data_start + data_size; + const char *name = (char *)&entry[2]; + struct firmware_timeline_metadata *metadata; + const unsigned int name_len = + size - TL_METADATA_ENTRY_NAME_OFFSET; + size_t allocation_size = sizeof(*metadata) + name_len + 1 + data_size; + + if (data_end > fw->size) { + dev_err(kbdev->dev, + "Firmware corrupt, file truncated? data_end=0x%x > fw->size=0x%zx", + data_end, fw->size); + return -EINVAL; + } + + /* Allocate enough space for firmware_timeline_metadata, + * its name and the content. + */ + metadata = kmalloc(allocation_size, GFP_KERNEL); + if (!metadata) + return -ENOMEM; + + metadata->name = (char *)(metadata + 1); + metadata->data = (char *)(metadata + 1) + name_len + 1; + metadata->size = data_size; + + memcpy(metadata->name, name, name_len); + metadata->name[name_len] = 0; + + /* Copy metadata's content. */ + memcpy(metadata->data, fw->data + data_start, data_size); + + list_add(&metadata->node, &kbdev->csf.firmware_timeline_metadata); + + dev_dbg(kbdev->dev, "Timeline metadata '%s'", metadata->name); + + return 0; +} + +/** + * load_firmware_entry() - Process an entry from a firmware image + * + * Read an entry from a firmware image and do any necessary work (e.g. loading + * the data into page accessible to the MCU). + * + * Unknown entries are ignored if the 'optional' flag is set within the entry, + * otherwise the function will fail with -EINVAL + * + * Return: 0 if successful, negative error code on failure + * + * @kbdev: Kbase device + * @fw: Firmware image containing the entry + * @offset: Byte offset within the image of the entry to load + * @header: Header word of the entry + */ +static int load_firmware_entry(struct kbase_device *kbdev, + const struct firmware *fw, + u32 offset, u32 header) +{ + const unsigned int type = entry_type(header); + unsigned int size = entry_size(header); + const bool optional = entry_optional(header); + const u32 *entry = (void *)(fw->data + offset); + + if ((offset % sizeof(*entry)) || (size % sizeof(*entry))) { + dev_err(kbdev->dev, "Firmware entry isn't 32 bit aligned, offset=0x%x size=0x%x\n", + offset, size); + return -EINVAL; + } + + if (size < sizeof(*entry)) { + dev_err(kbdev->dev, "Size field too small: %u\n", size); + return -EINVAL; + } + + /* Remove the header */ + entry++; + size -= sizeof(*entry); + + switch (type) { + case CSF_FIRMWARE_ENTRY_TYPE_INTERFACE: + /* Interface memory setup */ + if (size < INTERFACE_ENTRY_NAME_OFFSET + sizeof(*entry)) { + dev_err(kbdev->dev, "Interface memory setup entry too short (size=%u)\n", + size); + return -EINVAL; + } + return parse_memory_setup_entry(kbdev, fw, entry, size); + case CSF_FIRMWARE_ENTRY_TYPE_CONFIGURATION: + /* Configuration option */ + if (size < CONFIGURATION_ENTRY_NAME_OFFSET + sizeof(*entry)) { + dev_err(kbdev->dev, "Configuration option entry too short (size=%u)\n", + size); + return -EINVAL; + } + return kbase_csf_firmware_cfg_option_entry_parse( + kbdev, fw, entry, size); + case CSF_FIRMWARE_ENTRY_TYPE_FUTF_TEST: +#ifndef MALI_KBASE_BUILD + /* FW UTF option */ + if (size < 2*sizeof(*entry)) { + dev_err(kbdev->dev, "FW UTF entry too short (size=%u)\n", + size); + return -EINVAL; + } + return mali_kutf_process_fw_utf_entry(kbdev, fw->data, + fw->size, entry); +#endif + break; + case CSF_FIRMWARE_ENTRY_TYPE_TRACE_BUFFER: + /* Trace buffer */ + if (size < TRACE_BUFFER_ENTRY_NAME_OFFSET + sizeof(*entry)) { + dev_err(kbdev->dev, "Trace Buffer entry too short (size=%u)\n", + size); + return -EINVAL; + } + return kbase_csf_firmware_parse_trace_buffer_entry( + kbdev, entry, size); + case CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA: + /* Meta data section */ + if (size < TL_METADATA_ENTRY_NAME_OFFSET + sizeof(*entry)) { + dev_err(kbdev->dev, "Timeline metadata entry too short (size=%u)\n", + size); + return -EINVAL; + } + return parse_timeline_metadata_entry(kbdev, fw, entry, size); + } + + if (!optional) { + dev_err(kbdev->dev, + "Unsupported non-optional entry type %u in firmware\n", + type); + return -EINVAL; + } + + return 0; +} + +static void free_global_iface(struct kbase_device *kbdev) +{ + struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; + + if (iface->groups) { + unsigned int gid; + + for (gid = 0; gid < iface->group_num; ++gid) + kfree(iface->groups[gid].streams); + + kfree(iface->groups); + iface->groups = NULL; + } +} + +/** + * iface_gpu_va_to_cpu - Convert a GPU VA address within the shared interface + * region to a CPU address, using the existing mapping. + * @kbdev: Device pointer + * @gpu_va: GPU VA to convert + * + * Return: A CPU pointer to the location within the shared interface region, or + * NULL on failure. + */ +static inline void *iface_gpu_va_to_cpu(struct kbase_device *kbdev, u32 gpu_va) +{ + struct firmware_interface *interface = kbdev->csf.shared_interface; + u8 *kernel_base = interface->kernel_map; + + if (gpu_va < interface->virtual || + gpu_va >= interface->virtual + interface->num_pages * PAGE_SIZE) { + dev_err(kbdev->dev, + "Interface address 0x%x not within %u-page region at 0x%x", + gpu_va, interface->num_pages, + interface->virtual); + return NULL; + } + + return (void *)(kernel_base + (gpu_va - interface->virtual)); +} + +static int parse_cmd_stream_info(struct kbase_device *kbdev, + struct kbase_csf_cmd_stream_info *sinfo, + u32 *stream_base) +{ + sinfo->kbdev = kbdev; + sinfo->features = stream_base[STREAM_FEATURES/4]; + sinfo->input = iface_gpu_va_to_cpu(kbdev, + stream_base[STREAM_INPUT_VA/4]); + sinfo->output = iface_gpu_va_to_cpu(kbdev, + stream_base[STREAM_OUTPUT_VA/4]); + + if (sinfo->input == NULL || sinfo->output == NULL) + return -EINVAL; + + return 0; +} + +static int parse_cmd_stream_group_info(struct kbase_device *kbdev, + struct kbase_csf_cmd_stream_group_info *ginfo, + u32 *group_base, u32 group_stride) +{ + unsigned int sid; + + ginfo->kbdev = kbdev; + ginfo->features = group_base[GROUP_FEATURES/4]; + ginfo->input = iface_gpu_va_to_cpu(kbdev, + group_base[GROUP_INPUT_VA/4]); + ginfo->output = iface_gpu_va_to_cpu(kbdev, + group_base[GROUP_OUTPUT_VA/4]); + + if (ginfo->input == NULL || ginfo->output == NULL) + return -ENOMEM; + + ginfo->suspend_size = group_base[GROUP_SUSPEND_SIZE/4]; + ginfo->protm_suspend_size = group_base[GROUP_PROTM_SUSPEND_SIZE/4]; + ginfo->stream_num = group_base[GROUP_STREAM_NUM/4]; + + if (ginfo->stream_num < MIN_SUPPORTED_STREAMS_PER_GROUP || + ginfo->stream_num > MAX_SUPPORTED_STREAMS_PER_GROUP) { + dev_err(kbdev->dev, "CSG with %u streams out of range %u-%u", + ginfo->stream_num, + MIN_SUPPORTED_STREAMS_PER_GROUP, + MAX_SUPPORTED_STREAMS_PER_GROUP); + return -EINVAL; + } + + ginfo->stream_stride = group_base[GROUP_STREAM_STRIDE/4]; + + if (ginfo->stream_num * ginfo->stream_stride > group_stride) { + dev_err(kbdev->dev, + "group stride of 0x%x exceeded by %u streams with stride 0x%x", + group_stride, ginfo->stream_num, + ginfo->stream_stride); + return -EINVAL; + } + + ginfo->streams = kmalloc_array(ginfo->stream_num, + sizeof(*ginfo->streams), GFP_KERNEL); + + if (!ginfo->streams) + return -ENOMEM; + + for (sid = 0; sid < ginfo->stream_num; sid++) { + int err; + u32 *stream_base = group_base + (STREAM_CONTROL_0 + + ginfo->stream_stride * sid) / 4; + + err = parse_cmd_stream_info(kbdev, &ginfo->streams[sid], + stream_base); + if (err < 0) { + /* caller will free the memory for streams array */ + return err; + } + } + + return 0; +} + +static u32 get_firmware_version(struct kbase_device *kbdev) +{ + struct firmware_interface *interface = kbdev->csf.shared_interface; + u32 *shared_info = interface->kernel_map; + + return shared_info[GLB_VERSION/4]; +} + +static int parse_capabilities(struct kbase_device *kbdev) +{ + struct firmware_interface *interface = kbdev->csf.shared_interface; + u32 *shared_info = interface->kernel_map; + struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; + unsigned int gid; + + /* All offsets are in bytes, so divide by 4 for access via a u32 pointer + */ + + /* The version number of the global interface is expected to be a + * non-zero value. If it's not, the firmware may not have booted. + */ + iface->version = get_firmware_version(kbdev); + if (!iface->version) { + dev_err(kbdev->dev, "Version check failed. Firmware may have failed to boot."); + return -EINVAL; + } + + + iface->kbdev = kbdev; + iface->features = shared_info[GLB_FEATURES/4]; + iface->input = iface_gpu_va_to_cpu(kbdev, shared_info[GLB_INPUT_VA/4]); + iface->output = iface_gpu_va_to_cpu(kbdev, + shared_info[GLB_OUTPUT_VA/4]); + + if (iface->input == NULL || iface->output == NULL) + return -ENOMEM; + + iface->group_num = shared_info[GLB_GROUP_NUM/4]; + + if (iface->group_num < MIN_SUPPORTED_CSGS || + iface->group_num > MAX_SUPPORTED_CSGS) { + dev_err(kbdev->dev, + "Interface containing %u CSGs outside of range %u-%u", + iface->group_num, MIN_SUPPORTED_CSGS, + MAX_SUPPORTED_CSGS); + return -EINVAL; + } + + iface->group_stride = shared_info[GLB_GROUP_STRIDE/4]; + iface->prfcnt_size = shared_info[GLB_PRFCNT_SIZE/4]; + + if ((GROUP_CONTROL_0 + + (unsigned long)iface->group_num * iface->group_stride) > + (interface->num_pages * PAGE_SIZE)) { + dev_err(kbdev->dev, + "interface size of %u pages exceeded by %u CSGs with stride 0x%x", + interface->num_pages, iface->group_num, + iface->group_stride); + return -EINVAL; + } + + WARN_ON(iface->groups); + + iface->groups = kcalloc(iface->group_num, sizeof(*iface->groups), + GFP_KERNEL); + if (!iface->groups) + return -ENOMEM; + + for (gid = 0; gid < iface->group_num; gid++) { + int err; + u32 *group_base = shared_info + (GROUP_CONTROL_0 + + iface->group_stride * gid) / 4; + + err = parse_cmd_stream_group_info(kbdev, &iface->groups[gid], + group_base, iface->group_stride); + if (err < 0) { + free_global_iface(kbdev); + return err; + } + } + + return 0; +} + +static inline void access_firmware_memory(struct kbase_device *kbdev, + u32 gpu_addr, u32 *value, const bool read) +{ + struct firmware_interface *interface; + + list_for_each_entry(interface, &kbdev->csf.firmware_interfaces, node) { + if ((gpu_addr >= interface->virtual) && + (gpu_addr < interface->virtual + (interface->num_pages << PAGE_SHIFT))) { + u32 offset_bytes = gpu_addr - interface->virtual; + u32 page_num = offset_bytes >> PAGE_SHIFT; + u32 offset_in_page = offset_bytes & ~PAGE_MASK; + struct page *target_page = as_page( + interface->phys[page_num]); + u32 *cpu_addr = kmap_atomic(target_page); + + if (read) { + kbase_sync_single_for_device(kbdev, + kbase_dma_addr(target_page) + offset_in_page, + sizeof(u32), DMA_BIDIRECTIONAL); + + *value = cpu_addr[offset_in_page >> 2]; + } else { + cpu_addr[offset_in_page >> 2] = *value; + + kbase_sync_single_for_device(kbdev, + kbase_dma_addr(target_page) + offset_in_page, + sizeof(u32), DMA_BIDIRECTIONAL); + } + + kunmap_atomic(cpu_addr); + return; + } + } + dev_warn(kbdev->dev, "Invalid GPU VA %x passed\n", gpu_addr); +} + +void kbase_csf_read_firmware_memory(struct kbase_device *kbdev, + u32 gpu_addr, u32 *value) +{ + access_firmware_memory(kbdev, gpu_addr, value, true); +} + +void kbase_csf_update_firmware_memory(struct kbase_device *kbdev, + u32 gpu_addr, u32 value) +{ + access_firmware_memory(kbdev, gpu_addr, &value, false); +} + +void kbase_csf_firmware_cs_input( + const struct kbase_csf_cmd_stream_info *const info, const u32 offset, + const u32 value) +{ + const struct kbase_device * const kbdev = info->kbdev; + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value); + input_page_write(info->input, offset, value); +} + +u32 kbase_csf_firmware_cs_input_read( + const struct kbase_csf_cmd_stream_info *const info, + const u32 offset) +{ + const struct kbase_device * const kbdev = info->kbdev; + u32 const val = input_page_read(info->input, offset); + + dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val); + return val; +} + +void kbase_csf_firmware_cs_input_mask( + const struct kbase_csf_cmd_stream_info *const info, const u32 offset, + const u32 value, const u32 mask) +{ + const struct kbase_device * const kbdev = info->kbdev; + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", + offset, value, mask); + input_page_partial_write(info->input, offset, value, mask); +} + +u32 kbase_csf_firmware_cs_output( + const struct kbase_csf_cmd_stream_info *const info, const u32 offset) +{ + const struct kbase_device * const kbdev = info->kbdev; + u32 const val = output_page_read(info->output, offset); + + dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val); + return val; +} + +void kbase_csf_firmware_csg_input( + const struct kbase_csf_cmd_stream_group_info *const info, + const u32 offset, const u32 value) +{ + const struct kbase_device * const kbdev = info->kbdev; + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", + offset, value); + input_page_write(info->input, offset, value); +} + +u32 kbase_csf_firmware_csg_input_read( + const struct kbase_csf_cmd_stream_group_info *const info, + const u32 offset) +{ + const struct kbase_device * const kbdev = info->kbdev; + u32 const val = input_page_read(info->input, offset); + + dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val); + return val; +} + +void kbase_csf_firmware_csg_input_mask( + const struct kbase_csf_cmd_stream_group_info *const info, + const u32 offset, const u32 value, const u32 mask) +{ + const struct kbase_device * const kbdev = info->kbdev; + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", + offset, value, mask); + input_page_partial_write(info->input, offset, value, mask); +} + +u32 kbase_csf_firmware_csg_output( + const struct kbase_csf_cmd_stream_group_info *const info, + const u32 offset) +{ + const struct kbase_device * const kbdev = info->kbdev; + u32 const val = output_page_read(info->output, offset); + + dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val); + return val; +} + +void kbase_csf_firmware_global_input( + const struct kbase_csf_global_iface *const iface, const u32 offset, + const u32 value) +{ + const struct kbase_device * const kbdev = iface->kbdev; + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value); + input_page_write(iface->input, offset, value); +} + +void kbase_csf_firmware_global_input_mask( + const struct kbase_csf_global_iface *const iface, const u32 offset, + const u32 value, const u32 mask) +{ + const struct kbase_device * const kbdev = iface->kbdev; + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", + offset, value, mask); + input_page_partial_write(iface->input, offset, value, mask); +} + +u32 kbase_csf_firmware_global_input_read( + const struct kbase_csf_global_iface *const iface, const u32 offset) +{ + const struct kbase_device * const kbdev = iface->kbdev; + u32 const val = input_page_read(iface->input, offset); + + dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val); + return val; +} + +u32 kbase_csf_firmware_global_output( + const struct kbase_csf_global_iface *const iface, const u32 offset) +{ + const struct kbase_device * const kbdev = iface->kbdev; + u32 const val = output_page_read(iface->output, offset); + + dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val); + return val; +} + +static bool global_request_complete(struct kbase_device *const kbdev, + u32 const req_mask) +{ + struct kbase_csf_global_iface *global_iface = + &kbdev->csf.global_iface; + bool complete = false; + unsigned long flags; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + + if ((kbase_csf_firmware_global_output(global_iface, GLB_ACK) & + req_mask) == + (kbase_csf_firmware_global_input_read(global_iface, GLB_REQ) & + req_mask)) + complete = true; + + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + return complete; +} + +static int wait_for_global_request(struct kbase_device *const kbdev, + u32 const req_mask) +{ + const long wait_timeout = + kbase_csf_timeout_in_jiffies(GLB_REQ_WAIT_TIMEOUT_MS); + long remaining; + int err = 0; + + remaining = wait_event_timeout(kbdev->csf.event_wait, + global_request_complete(kbdev, req_mask), + wait_timeout); + + if (!remaining) { + dev_warn(kbdev->dev, "Timed out waiting for global request %x to complete", + req_mask); + err = -ETIMEDOUT; + } + + return err; +} + +static void set_global_request( + const struct kbase_csf_global_iface *const global_iface, + u32 const req_mask) +{ + u32 glb_req; + + kbase_csf_scheduler_spin_lock_assert_held(global_iface->kbdev); + + glb_req = kbase_csf_firmware_global_output(global_iface, GLB_ACK); + glb_req ^= req_mask; + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, glb_req, + req_mask); +} + +static void enable_endpoints_global( + const struct kbase_csf_global_iface *const global_iface, + u64 const shader_core_mask) +{ + kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_LO, + shader_core_mask & U32_MAX); + kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_HI, + shader_core_mask >> 32); + + set_global_request(global_iface, GLB_REQ_CFG_ALLOC_EN_MASK); +} + +static void set_timeout_global( + const struct kbase_csf_global_iface *const global_iface, + u64 const timeout) +{ + kbase_csf_firmware_global_input(global_iface, GLB_PROGRESS_TIMER, + timeout / GLB_PROGRESS_TIMER_TIMEOUT_SCALE); + + set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK); +} + +static void set_coherency_mode(struct kbase_device *const kbdev) +{ + const struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + u32 protected_mode_coherency = kbdev->system_coherency; + + /* GPU is supposed to use ACE-Lite coherency mode on a fully coherent + * system during protected mode execution. + */ + if (kbdev->system_coherency == COHERENCY_ACE) + protected_mode_coherency = COHERENCY_ACE_LITE; + + kbase_csf_firmware_global_input(global_iface, GLB_PROTM_COHERENCY, + protected_mode_coherency); +} + +static void global_init(struct kbase_device *const kbdev, u32 req_mask) +{ + u32 const ack_irq_mask = GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK | + GLB_ACK_IRQ_MASK_PING_MASK | + GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK | + GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK | + GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK; + + const struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + unsigned long flags; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + + /* Set the cohereny mode for protected mode execution */ + set_coherency_mode(kbdev); + + /* Enable endpoints on all present shader cores */ + enable_endpoints_global(global_iface, + kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); + + set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); + + /* Unmask the interrupts */ + kbase_csf_firmware_global_input(global_iface, + GLB_ACK_IRQ_MASK, ack_irq_mask); + + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + + kbase_csf_scheduler_spin_unlock(kbdev, flags); +} + +/** + * global_init_on_boot - Sends a global request to control various features. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * Currently only the request to enable endpoints and timeout for GPU progress + * timer is sent. + * + * Return: 0 on success, or negative on failure. + */ +static int global_init_on_boot(struct kbase_device *const kbdev) +{ + u32 const req_mask = CSF_GLB_REQ_CFG_MASK; + + global_init(kbdev, req_mask); + + return wait_for_global_request(kbdev, req_mask); +} + +void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->csf.glb_init_request_pending = true; + global_init(kbdev, CSF_GLB_REQ_CFG_MASK); +} + +bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + WARN_ON(!kbdev->csf.glb_init_request_pending); + + if (global_request_complete(kbdev, CSF_GLB_REQ_CFG_MASK)) + kbdev->csf.glb_init_request_pending = false; + + return !kbdev->csf.glb_init_request_pending; +} + +/** + * This helper function will reload the firmware image and re-enable the MCU. + * It is supposed to be called after MCU(GPU) has been reset. + * Unlike the initial boot the firmware binary image is not parsed completely. + * Only the data sections, which were loaded in memory during the initial boot, + * are re-initialized either by zeroing them or copying their data from the + * firmware binary image. The memory allocation for the firmware pages and + * MMU programming is not needed for the reboot, presuming the firmware binary + * file on the filesystem would not change. + */ +static void kbase_csf_firmware_reload_worker(struct work_struct *work) +{ + struct kbase_device *kbdev = container_of(work, struct kbase_device, + csf.firmware_reload_work); + int err; + + dev_info(kbdev->dev, "reloading firmware"); + + /* Reload just the data sections from firmware binary image */ + err = reload_fw_data_sections(kbdev); + if (err) + return; + + kbase_csf_tl_reader_reset(&kbdev->timeline->csf_tl_reader); + + /* Reboot the firmware */ + kbase_csf_firmware_enable_mcu(kbdev); +} + +void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->csf.firmware_reloaded = false; + + if (kbdev->csf.firmware_reload_needed) { + kbdev->csf.firmware_reload_needed = false; + queue_work(system_wq, &kbdev->csf.firmware_reload_work); + } else { + kbase_csf_firmware_enable_mcu(kbdev); + } +} + +void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) +{ + u32 version; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (unlikely(!kbdev->csf.firmware_inited)) + return; + + /* Check firmware rebooted properly: we do not expect + * the version number to change with a running reboot. + */ + version = get_firmware_version(kbdev); + + if (version != kbdev->csf.global_iface.version) + dev_err(kbdev->dev, "Version check failed in firmware reboot."); + + KBASE_KTRACE_ADD(kbdev, FIRMWARE_REBOOT, NULL, 0u); + + /* Tell MCU state machine to transit to next state */ + kbdev->csf.firmware_reloaded = true; + kbase_pm_update_state(kbdev); +} + +int kbase_csf_firmware_init(struct kbase_device *kbdev) +{ + const struct firmware *firmware; + const u32 magic = FIRMWARE_HEADER_MAGIC; + u8 version_major, version_minor; + u32 version_hash; + u32 entry_end_offset; + u32 entry_offset; + int ret; + + if (WARN_ON((kbdev->as_free & MCU_AS_BITMASK) == 0)) + return -EINVAL; + kbdev->as_free &= ~MCU_AS_BITMASK; + + ret = kbase_mmu_init(kbdev, &kbdev->csf.mcu_mmu, NULL, + BASE_MEM_GROUP_DEFAULT); + + if (ret != 0) { + /* Release the address space */ + kbdev->as_free |= MCU_AS_BITMASK; + return ret; + } + + init_waitqueue_head(&kbdev->csf.event_wait); + kbdev->csf.interrupt_received = false; + + INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces); + INIT_LIST_HEAD(&kbdev->csf.firmware_config); + INIT_LIST_HEAD(&kbdev->csf.firmware_timeline_metadata); + INIT_LIST_HEAD(&kbdev->csf.firmware_trace_buffers.list); + INIT_WORK(&kbdev->csf.firmware_reload_work, + kbase_csf_firmware_reload_worker); + + mutex_init(&kbdev->csf.reg_lock); + + ret = kbase_mcu_shared_interface_region_tracker_init(kbdev); + if (ret != 0) { + dev_err(kbdev->dev, "Failed to setup the rb tree for managing shared interface segment\n"); + goto error; + } + + if (request_firmware(&firmware, fw_name, kbdev->dev) != 0) { + dev_err(kbdev->dev, + "Failed to load firmware image '%s'\n", + fw_name); + ret = -ENOENT; + goto error; + } + + if (firmware->size < FIRMWARE_HEADER_LENGTH) { + dev_err(kbdev->dev, "Firmware too small\n"); + ret = -EINVAL; + goto error; + } + + if (memcmp(firmware->data, &magic, sizeof(magic)) != 0) { + dev_err(kbdev->dev, "Incorrect firmware magic\n"); + ret = -EINVAL; + goto error; + } + + version_major = firmware->data[4]; + version_minor = firmware->data[5]; + + if (version_major != FIRMWARE_HEADER_VERSION) { + dev_err(kbdev->dev, + "Firmware header version %d.%d not understood\n", + version_major, version_minor); + ret = -EINVAL; + goto error; + } + + memcpy(&version_hash, &firmware->data[8], sizeof(version_hash)); + + dev_notice(kbdev->dev, "Loading Mali firmware 0x%x", version_hash); + + memcpy(&entry_end_offset, &firmware->data[0x10], + sizeof(entry_end_offset)); + + if (entry_end_offset > firmware->size) { + dev_err(kbdev->dev, "Firmware image is truncated\n"); + ret = -EINVAL; + goto error; + } + + entry_offset = FIRMWARE_HEADER_LENGTH; + while (entry_offset < entry_end_offset) { + u32 header; + unsigned int size; + + memcpy(&header, &firmware->data[entry_offset], sizeof(header)); + + size = entry_size(header); + + ret = load_firmware_entry(kbdev, firmware, entry_offset, + header); + if (ret != 0) { + dev_err(kbdev->dev, "Failed to load firmware image\n"); + goto error; + } + entry_offset += size; + } + + if (!kbdev->csf.shared_interface) { + dev_err(kbdev->dev, "Shared interface region not found\n"); + ret = -EINVAL; + goto error; + } else { + ret = setup_shared_iface_static_region(kbdev); + if (ret != 0) { + dev_err(kbdev->dev, "Failed to insert a region for shared iface entry parsed from fw image\n"); + goto error; + } + } + + ret = kbase_csf_firmware_trace_buffers_init(kbdev); + if (ret != 0) { + dev_err(kbdev->dev, "Failed to initialize trace buffers\n"); + goto error; + } + + /* Make sure L2 cache is powered up */ + kbase_pm_wait_for_l2_powered(kbdev); + + /* Load the MMU tables into the selected address space */ + load_mmu_tables(kbdev); + + boot_csf_firmware(kbdev); + + ret = parse_capabilities(kbdev); + if (ret != 0) + goto error; + + ret = kbase_csf_doorbell_mapping_init(kbdev); + if (ret != 0) + goto error; + + ret = kbase_csf_scheduler_init(kbdev); + if (ret != 0) + goto error; + + ret = kbase_csf_timeout_init(kbdev); + if (ret != 0) + goto error; + + ret = global_init_on_boot(kbdev); + if (ret != 0) + goto error; + + ret = kbase_csf_firmware_cfg_init(kbdev); + if (ret != 0) + goto error; + + /* Firmware loaded successfully */ + release_firmware(firmware); + KBASE_KTRACE_ADD(kbdev, FIRMWARE_BOOT, NULL, + (((u64)version_hash) << 32) | + (((u64)version_major) << 8) | version_minor); + return 0; + +error: + kbase_csf_firmware_term(kbdev); + release_firmware(firmware); + return ret; +} + +void kbase_csf_firmware_term(struct kbase_device *kbdev) +{ + unsigned long flags; + int ret = 0; + + while (kbase_reset_gpu_is_active(kbdev) && !ret) + ret = kbase_reset_gpu_wait(kbdev); + + WARN(ret, "failed to wait for GPU reset"); + + /* Make sure ongoing transitions have completed */ + kbase_pm_wait_for_desired_state(kbdev); + + kbase_csf_firmware_cfg_term(kbdev); + + kbase_csf_timeout_term(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->csf.firmware_inited = false; + if (kbdev->pm.backend.mcu_state != KBASE_MCU_OFF) { + kbdev->pm.backend.mcu_state = KBASE_MCU_OFF; + stop_csf_firmware(kbdev); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + unload_mmu_tables(kbdev); + + kbase_mmu_term(kbdev, &kbdev->csf.mcu_mmu); + + kbase_csf_scheduler_term(kbdev); + + kbase_csf_doorbell_mapping_term(kbdev); + + free_global_iface(kbdev); + + /* Release the address space */ + kbdev->as_free |= MCU_AS_BITMASK; + + while (!list_empty(&kbdev->csf.firmware_interfaces)) { + struct firmware_interface *interface; + + interface = list_first_entry(&kbdev->csf.firmware_interfaces, + struct firmware_interface, node); + list_del(&interface->node); + + vunmap(interface->kernel_map); + if (interface->flags & CSF_FIRMWARE_ENTRY_PROTECTED) { + kbase_csf_protected_memory_free(kbdev, interface->pma, + interface->num_pages); + } else { + kbase_mem_pool_free_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + interface->num_pages, interface->phys, + true, false); + } + + kfree(interface->phys); + kfree(interface); + } + + while (!list_empty(&kbdev->csf.firmware_timeline_metadata)) { + struct firmware_timeline_metadata *metadata; + + metadata = list_first_entry( + &kbdev->csf.firmware_timeline_metadata, + struct firmware_timeline_metadata, + node); + list_del(&metadata->node); + + kfree(metadata); + } + + kbase_csf_firmware_trace_buffers_term(kbdev); + +#ifndef MALI_KBASE_BUILD + mali_kutf_fw_utf_entry_cleanup(kbdev); +#endif + + mutex_destroy(&kbdev->csf.reg_lock); + + /* This will also free up the region allocated for the shared interface + * entry parsed from the firmware image. + */ + kbase_mcu_shared_interface_region_tracker_term(kbdev); +} + +int kbase_csf_firmware_ping(struct kbase_device *const kbdev) +{ + const struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + unsigned long flags; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + set_global_request(global_iface, GLB_REQ_PING_MASK); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + return wait_for_global_request(kbdev, GLB_REQ_PING_MASK); +} + +int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev, + u64 const timeout) +{ + const struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + unsigned long flags; + int err; + + /* The 'reg_lock' is also taken and is held till the update is not + * complete, to ensure the update of timeout value by multiple Users + * gets serialized. + */ + mutex_lock(&kbdev->csf.reg_lock); + kbase_csf_scheduler_spin_lock(kbdev, &flags); + set_timeout_global(global_iface, timeout); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + err = wait_for_global_request(kbdev, GLB_REQ_CFG_PROGRESS_TIMER_MASK); + mutex_unlock(&kbdev->csf.reg_lock); + + return err; +} + +void kbase_csf_enter_protected_mode(struct kbase_device *kbdev) +{ + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + unsigned long flags; + unsigned int value; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + value = kbase_csf_firmware_global_output(global_iface, GLB_ACK); + value ^= GLB_REQ_PROTM_ENTER_MASK; + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, value, + GLB_REQ_PROTM_ENTER_MASK); + dev_dbg(kbdev->dev, "Sending request to enter protected mode"); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + wait_for_global_request(kbdev, GLB_REQ_PROTM_ENTER_MASK); +} + +void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev) +{ + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + unsigned long flags; + unsigned int value; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + value = kbase_csf_firmware_global_output(global_iface, GLB_ACK); + value ^= GLB_REQ_HALT_MASK; + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, value, + GLB_REQ_HALT_MASK); + dev_dbg(kbdev->dev, "Sending request to HALT MCU"); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_csf_scheduler_spin_unlock(kbdev, flags); +} + +/** + * copy_grp_and_stm - Copy command stream and/or group data + * + * @iface: Global command stream front-end interface provided by + * the firmware. + * @group_data: Pointer where to store all the group data + * (sequentially). + * @max_group_num: The maximum number of groups to be read. Can be 0, in + * which case group_data is unused. + * @stream_data: Pointer where to store all the stream data + * (sequentially). + * @max_total_stream_num: The maximum number of streams to be read. + * Can be 0, in which case stream_data is unused. + * + * Return: Total number of command streams, summed across all groups. + */ +static u32 copy_grp_and_stm( + const struct kbase_csf_global_iface * const iface, + struct basep_cs_group_control * const group_data, + u32 max_group_num, + struct basep_cs_stream_control * const stream_data, + u32 max_total_stream_num) +{ + u32 i, total_stream_num = 0; + + if (WARN_ON((max_group_num > 0) && !group_data)) + max_group_num = 0; + + if (WARN_ON((max_total_stream_num > 0) && !stream_data)) + max_total_stream_num = 0; + + for (i = 0; i < iface->group_num; i++) { + u32 j; + + if (i < max_group_num) { + group_data[i].features = iface->groups[i].features; + group_data[i].stream_num = iface->groups[i].stream_num; + group_data[i].suspend_size = + iface->groups[i].suspend_size; + } + for (j = 0; j < iface->groups[i].stream_num; j++) { + if (total_stream_num < max_total_stream_num) + stream_data[total_stream_num].features = + iface->groups[i].streams[j].features; + total_stream_num++; + } + } + + return total_stream_num; +} + +u32 kbase_csf_firmware_get_glb_iface(struct kbase_device *kbdev, + struct basep_cs_group_control *const group_data, + u32 const max_group_num, + struct basep_cs_stream_control *const stream_data, + u32 const max_total_stream_num, u32 *const glb_version, + u32 *const features, u32 *const group_num, u32 *const prfcnt_size) +{ + const struct kbase_csf_global_iface * const iface = + &kbdev->csf.global_iface; + + if (WARN_ON(!glb_version) || + WARN_ON(!features) || + WARN_ON(!group_num) || + WARN_ON(!prfcnt_size)) + return 0; + + *glb_version = iface->version; + *features = iface->features; + *group_num = iface->group_num; + *prfcnt_size = iface->prfcnt_size; + + return copy_grp_and_stm(iface, group_data, max_group_num, + stream_data, max_total_stream_num); +} + +const char *kbase_csf_firmware_get_timeline_metadata( + struct kbase_device *kbdev, const char *name, size_t *size) +{ + struct firmware_timeline_metadata *metadata; + + list_for_each_entry( + metadata, &kbdev->csf.firmware_timeline_metadata, node) { + if (!strcmp(metadata->name, name)) { + *size = metadata->size; + return metadata->data; + } + } + + *size = 0; + return NULL; +} + +int kbase_csf_firmware_mcu_shared_mapping_init( + struct kbase_device *kbdev, + unsigned int num_pages, + unsigned long cpu_map_properties, + unsigned long gpu_map_properties, + struct kbase_csf_mapping *csf_mapping) +{ + struct tagged_addr *phys; + struct kbase_va_region *va_reg; + struct page **page_list; + void *cpu_addr; + int i, ret = 0; + pgprot_t cpu_map_prot = PAGE_KERNEL; + unsigned long gpu_map_prot; + + if (cpu_map_properties & PROT_READ) + cpu_map_prot = PAGE_KERNEL_RO; + + if (kbdev->system_coherency == COHERENCY_ACE) { + gpu_map_prot = + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); + } else { + gpu_map_prot = + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); + cpu_map_prot = pgprot_writecombine(cpu_map_prot); + }; + + phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); + if (!phys) + goto out; + + page_list = kmalloc_array(num_pages, sizeof(*page_list), GFP_KERNEL); + if (!page_list) + goto page_list_alloc_error; + + ret = kbase_mem_pool_alloc_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + num_pages, phys, false); + if (ret <= 0) + goto phys_mem_pool_alloc_error; + + for (i = 0; i < num_pages; i++) + page_list[i] = as_page(phys[i]); + + cpu_addr = vmap(page_list, num_pages, VM_MAP, cpu_map_prot); + if (!cpu_addr) + goto vmap_error; + + va_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, + num_pages, KBASE_REG_ZONE_MCU_SHARED); + if (!va_reg) + goto va_region_alloc_error; + + mutex_lock(&kbdev->csf.reg_lock); + ret = kbase_add_va_region_rbtree(kbdev, va_reg, 0, num_pages, 1); + va_reg->flags &= ~KBASE_REG_FREE; + mutex_unlock(&kbdev->csf.reg_lock); + if (ret) + goto va_region_add_error; + + gpu_map_properties &= (KBASE_REG_GPU_RD | KBASE_REG_GPU_WR); + gpu_map_properties |= gpu_map_prot; + + ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu, + va_reg->start_pfn, &phys[0], num_pages, + gpu_map_properties, KBASE_MEM_GROUP_CSF_FW); + if (ret) + goto mmu_insert_pages_error; + + kfree(page_list); + csf_mapping->phys = phys; + csf_mapping->cpu_addr = cpu_addr; + csf_mapping->va_reg = va_reg; + csf_mapping->num_pages = num_pages; + + return 0; + +mmu_insert_pages_error: + mutex_lock(&kbdev->csf.reg_lock); + kbase_remove_va_region(va_reg); + mutex_unlock(&kbdev->csf.reg_lock); +va_region_add_error: + kbase_free_alloced_region(va_reg); +va_region_alloc_error: + vunmap(cpu_addr); +vmap_error: + kbase_mem_pool_free_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + num_pages, phys, false, false); + +phys_mem_pool_alloc_error: + kfree(page_list); +page_list_alloc_error: + kfree(phys); +out: + /* Zero-initialize the mapping to make sure that the termination + * function doesn't try to unmap or free random addresses. */ + csf_mapping->phys = NULL; + csf_mapping->cpu_addr = NULL; + csf_mapping->va_reg = NULL; + csf_mapping->num_pages = 0; + + return -ENOMEM; +} + +void kbase_csf_firmware_mcu_shared_mapping_term( + struct kbase_device *kbdev, struct kbase_csf_mapping *csf_mapping) +{ + if (csf_mapping->va_reg) { + mutex_lock(&kbdev->csf.reg_lock); + kbase_remove_va_region(csf_mapping->va_reg); + mutex_unlock(&kbdev->csf.reg_lock); + kbase_free_alloced_region(csf_mapping->va_reg); + } + + if (csf_mapping->phys) { + kbase_mem_pool_free_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + csf_mapping->num_pages, csf_mapping->phys, false, + false); + } + + vunmap(csf_mapping->cpu_addr); + kfree(csf_mapping->phys); +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h new file mode 100755 index 000000000000..03a5217cffb0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h @@ -0,0 +1,663 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_FIRMWARE_H_ +#define _KBASE_CSF_FIRMWARE_H_ + +#include "device/mali_kbase_device.h" +#include "mali_gpu_csf_registers.h" + +/* + * PAGE_KERNEL_RO was only defined on 32bit ARM in 4.19 in: + * Commit a3266bd49c721e2e0a71f352d83713fbd60caadb + * Author: Luis R. Rodriguez + * Date: Fri Aug 17 15:46:29 2018 -0700 + * + * mm: provide a fallback for PAGE_KERNEL_RO for architectures + * + * Some architectures do not define certain PAGE_KERNEL_* flags, this is + * either because: + * + * a) The way to implement some of these flags is *not yet ported*, or + * b) The architecture *has no way* to describe them + * + * [snip] + * + * This can be removed once support of 32bit ARM kernels predating 4.19 is no + * longer required. + */ +#ifndef PAGE_KERNEL_RO +#define PAGE_KERNEL_RO PAGE_KERNEL +#endif + +/* Address space number to claim for the firmware. */ +#define MCU_AS_NR 0 +#define MCU_AS_BITMASK (1 << MCU_AS_NR) + +/* Number of available Doorbells */ +#define CSF_NUM_DOORBELL ((u8)24) + +/* Offset to the first HW doorbell page */ +#define CSF_HW_DOORBELL_PAGE_OFFSET ((u32)0x80000) + +/* Size of HW Doorbell page, used to calculate the offset to subsequent pages */ +#define CSF_HW_DOORBELL_PAGE_SIZE ((u32)0x10000) + +/* Doorbell 0 is used by the driver. */ +#define CSF_KERNEL_DOORBELL_NR ((u32)0) + +/* Offset of name inside a trace buffer entry in the firmware image */ +#define TRACE_BUFFER_ENTRY_NAME_OFFSET (0x1C) + +/* All implementations of the host interface with major version 0 must comply + * with these restrictions: + */ +/* GLB_GROUP_NUM: At least 3 command stream groups, but no more than 31 */ +#define MIN_SUPPORTED_CSGS 3 +#define MAX_SUPPORTED_CSGS 31 +/* GROUP_STREAM_NUM: At least 8 command streams per CSG, but no more than 32 */ +#define MIN_SUPPORTED_STREAMS_PER_GROUP 8 +/* Maximum command streams per csg. */ +#define MAX_SUPPORTED_STREAMS_PER_GROUP 32 + +struct kbase_device; + + +/** + * struct kbase_csf_mapping - Memory mapping for CSF memory. + * @phys: Physical memory allocation used by the mapping. + * @cpu_addr: Starting CPU address for the mapping. + * @va_reg: GPU virtual address region for the mapping. + * @num_pages: Size of the mapping, in memory pages. + */ +struct kbase_csf_mapping { + struct tagged_addr *phys; + void *cpu_addr; + struct kbase_va_region *va_reg; + unsigned int num_pages; +}; + +/** + * struct kbase_csf_trace_buffers - List and state of firmware trace buffers. + * @list: List of trace buffers descriptors. + * @mcu_rw: Metadata for the MCU shared memory mapping used for + * GPU-readable,writable/CPU-writable variables. + * @mcu_write: Metadata for the MCU shared memory mapping used for + * GPU-writable/CPU-readable variables. + */ +struct kbase_csf_trace_buffers { + struct list_head list; + struct kbase_csf_mapping mcu_rw; + struct kbase_csf_mapping mcu_write; +}; + +/** + * struct kbase_csf_cmd_stream_info - Command stream interface provided by the + * firmware. + * + * @kbdev: Address of the instance of a GPU platform device that implements + * this interface. + * @features: Bit field of command stream features (e.g. which types of jobs + * are supported). Bits 7:0 specify the number of work registers(-1). + * Bits 11:8 specify the number of scoreboard entries(-1). + * @input: Address of command stream interface input page. + * @output: Address of command stream interface output page. + */ +struct kbase_csf_cmd_stream_info { + struct kbase_device *kbdev; + u32 features; + void *input; + void *output; +}; + +/** + * kbase_csf_firmware_cs_input() - Set a word in a command stream's input page + * + * @info: Command stream interface provided by the firmware. + * @offset: Offset of the word to be written, in bytes. + * @value: Value to be written. + */ +void kbase_csf_firmware_cs_input( + const struct kbase_csf_cmd_stream_info *info, u32 offset, u32 value); + +/** + * kbase_csf_firmware_cs_input_read() - Read a word in a command stream's input + * page + * + * Return: Value of the word read from the command stream's input page. + * + * @info: Command stream interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + */ +u32 kbase_csf_firmware_cs_input_read( + const struct kbase_csf_cmd_stream_info *const info, const u32 offset); + +/** + * kbase_csf_firmware_cs_input_mask() - Set part of a word in a command stream's + * input page + * + * @info: Command stream interface provided by the firmware. + * @offset: Offset of the word to be modified, in bytes. + * @value: Value to be written. + * @mask: Bitmask with the bits to be modified set. + */ +void kbase_csf_firmware_cs_input_mask( + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value, u32 mask); + +/** + * kbase_csf_firmware_cs_output() - Read a word in a command stream's output + * page + * + * Return: Value of the word read from the command stream's output page. + * + * @info: Command stream interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + */ +u32 kbase_csf_firmware_cs_output( + const struct kbase_csf_cmd_stream_info *info, u32 offset); +/** + * struct kbase_csf_cmd_stream_group_info - Command stream group interface + * provided by the firmware. + * + * @kbdev: Address of the instance of a GPU platform device that implements + * this interface. + * @features: Bit mask of features. Reserved bits should be 0, and should + * be ignored. + * @input: Address of global interface input page. + * @output: Address of global interface output page. + * @suspend_size: Size in bytes for normal suspend buffer for the command + * stream group. + * @protm_suspend_size: Size in bytes for protected mode suspend buffer + * for the command stream group. + * @stream_num: Number of command streams in the command stream group. + * @stream_stride: Stride in bytes in JASID0 virtual address between + * command stream capability structures. + * @streams: Address of an array of command stream capability structures. + */ +struct kbase_csf_cmd_stream_group_info { + struct kbase_device *kbdev; + u32 features; + void *input; + void *output; + u32 suspend_size; + u32 protm_suspend_size; + u32 stream_num; + u32 stream_stride; + struct kbase_csf_cmd_stream_info *streams; +}; + +/** + * kbase_csf_firmware_csg_input() - Set a word in a command stream group's + * input page + * + * @info: Command stream group interface provided by the firmware. + * @offset: Offset of the word to be written, in bytes. + * @value: Value to be written. + */ +void kbase_csf_firmware_csg_input( + const struct kbase_csf_cmd_stream_group_info *info, u32 offset, + u32 value); + +/** + * kbase_csf_firmware_csg_input_read() - Read a word in a command stream group's + * input page + * + * Return: Value of the word read from the command stream group's input page. + * + * @info: Command stream group interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + */ +u32 kbase_csf_firmware_csg_input_read( + const struct kbase_csf_cmd_stream_group_info *info, u32 offset); + +/** + * kbase_csf_firmware_csg_input_mask() - Set part of a word in a command stream + * group's input page + * + * @info: Command stream group interface provided by the firmware. + * @offset: Offset of the word to be modified, in bytes. + * @value: Value to be written. + * @mask: Bitmask with the bits to be modified set. + */ +void kbase_csf_firmware_csg_input_mask( + const struct kbase_csf_cmd_stream_group_info *info, u32 offset, + u32 value, u32 mask); + +/** + * kbase_csf_firmware_csg_output()- Read a word in a command stream group's + * output page + * + * Return: Value of the word read from the command stream group's output page. + * + * @info: Command stream group interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + */ +u32 kbase_csf_firmware_csg_output( + const struct kbase_csf_cmd_stream_group_info *info, u32 offset); + +/** + * struct kbase_csf_global_iface - Global command stream front-end interface + * provided by the firmware. + * + * @kbdev: Address of the instance of a GPU platform device that implements + * this interface. + * @version: Bits 31:16 hold the major version number and 15:0 hold the minor + * version number. A higher minor version is backwards-compatible + * with a lower minor version for the same major version. + * @features: Bit mask of features (e.g. whether certain types of job can + * be suspended). Reserved bits should be 0, and should be ignored. + * @input: Address of global interface input page. + * @output: Address of global interface output page. + * @group_num: Number of command stream groups supported. + * @group_stride: Stride in bytes in JASID0 virtual address between + * command stream group capability structures. + * @prfcnt_size: Performance counters size. + * @groups: Address of an array of command stream group capability structures. + */ +struct kbase_csf_global_iface { + struct kbase_device *kbdev; + u32 version; + u32 features; + void *input; + void *output; + u32 group_num; + u32 group_stride; + u32 prfcnt_size; + struct kbase_csf_cmd_stream_group_info *groups; +}; + +/** + * kbase_csf_firmware_global_input() - Set a word in the global input page + * + * @iface: Command stream front-end interface provided by the firmware. + * @offset: Offset of the word to be written, in bytes. + * @value: Value to be written. + */ +void kbase_csf_firmware_global_input( + const struct kbase_csf_global_iface *iface, u32 offset, u32 value); + +/** + * kbase_csf_firmware_global_input_mask() - Set part of a word in the global + * input page + * + * @iface: Command stream front-end interface provided by the firmware. + * @offset: Offset of the word to be modified, in bytes. + * @value: Value to be written. + * @mask: Bitmask with the bits to be modified set. + */ +void kbase_csf_firmware_global_input_mask( + const struct kbase_csf_global_iface *iface, u32 offset, + u32 value, u32 mask); + +/** + * kbase_csf_firmware_global_input_read() - Read a word in a global input page + * + * Return: Value of the word read from the global input page. + * + * @info: Command stream group interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + */ +u32 kbase_csf_firmware_global_input_read( + const struct kbase_csf_global_iface *info, u32 offset); + +/** + * kbase_csf_firmware_global_output() - Read a word in the global output page + * + * Return: Value of the word read from the global output page. + * + * @iface: Command stream front-end interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + */ +u32 kbase_csf_firmware_global_output( + const struct kbase_csf_global_iface *iface, u32 offset); + +/* Calculate the offset to the Hw doorbell page corresponding to the + * doorbell number. + */ +static u32 csf_doorbell_offset(int doorbell_nr) +{ + WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL); + + return CSF_HW_DOORBELL_PAGE_OFFSET + + (doorbell_nr * CSF_HW_DOORBELL_PAGE_SIZE); +} + +static inline void kbase_csf_ring_doorbell(struct kbase_device *kbdev, + int doorbell_nr) +{ + WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL); + + kbase_reg_write(kbdev, csf_doorbell_offset(doorbell_nr), (u32)1); +} + +/** + * kbase_csf_read_firmware_memory - Read a value in a GPU address + * + * This function read a value in a GPU address that belongs to + * a private firmware memory region. The function assumes that the location + * is not permanently mapped on the CPU address space, therefore it maps it + * and then unmaps it to access it independently. + * + * @kbdev: Device pointer + * @gpu_addr: GPU address to read + * @value: output pointer to which the read value will be written. + */ +void kbase_csf_read_firmware_memory(struct kbase_device *kbdev, + u32 gpu_addr, u32 *value); + +/** + * kbase_csf_update_firmware_memory - Write a value in a GPU address + * + * This function writes a given value in a GPU address that belongs to + * a private firmware memory region. The function assumes that the destination + * is not permanently mapped on the CPU address space, therefore it maps it + * and then unmaps it to access it independently. + * + * @kbdev: Device pointer + * @gpu_addr: GPU address to write + * @value: Value to write + */ +void kbase_csf_update_firmware_memory(struct kbase_device *kbdev, + u32 gpu_addr, u32 value); + +/** + * kbase_csf_firmware_init() - Load the firmware for the CSF MCU + * + * Request the firmware from user space and load it into memory. + * + * Return: 0 if successful, negative error code on failure + * + * @kbdev: Kbase device + */ +int kbase_csf_firmware_init(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_term() - Unload the firmware + * + * Frees the memory allocated by kbase_csf_firmware_init() + * + * @kbdev: Kbase device + */ +void kbase_csf_firmware_term(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_ping - Send the ping request to firmware. + * + * The function sends the ping request to firmware to confirm it is alive. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_firmware_ping(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_set_timeout - Set a hardware endpoint progress timeout. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @timeout: The maximum number of GPU cycles that is allowed to elapse + * without forward progress before the driver terminates a GPU + * command queue group. + * + * Configures the progress timeout value used by the firmware to decide + * when to report that a task is not making progress on an endpoint. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_firmware_set_timeout(struct kbase_device *kbdev, u64 timeout); + +/** + * kbase_csf_enter_protected_mode - Send the Global request to firmware to + * enter protected mode and wait for its + * completion. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +void kbase_csf_enter_protected_mode(struct kbase_device *kbdev); + +static inline bool kbase_csf_firmware_mcu_halted(struct kbase_device *kbdev) +{ +#ifndef CONFIG_MALI_BIFROST_NO_MALI + return (kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS)) == + MCU_STATUS_HALTED); +#else + return true; +#endif +} + +/** + * kbase_csf_firmware_trigger_mcu_halt - Send the Global request to firmware to + * halt its operation and bring itself + * into a known internal state for warm + * boot later. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_enable_mcu - Send the command to enable MCU + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +static inline void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev) +{ + /* Trigger the boot of MCU firmware, Use the AUTO mode as + * otherwise on fast reset, to exit protected mode, MCU will + * not reboot by itself to enter normal mode. + */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), MCU_CNTRL_AUTO); +} + +/** + * kbase_csf_firmware_disable_mcu - Send the command to disable MCU + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +static inline void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev) +{ + kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), MCU_CNTRL_DISABLE); +} + +/** + * kbase_csf_firmware_disable_mcu_wait - Wait for the MCU to reach disabled + * status. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev); + +/** + * kbase_trigger_firmware_reload - Trigger the reboot of MCU firmware, for the + * cold boot case firmware image would be + * reloaded from filesystem into memory. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_reload_completed - The reboot of MCU firmware has + * completed. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_global_reinit - Send the Global configuration requests + * after the reboot of MCU firmware. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_global_reinit_complete - Check the Global configuration + * requests, sent after the reboot of MCU firmware, have + * completed or not. + * + * Return: true if the Global configuration requests completed otherwise false. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev); + +/** + * Request the global control block of CSF interface capabilities + * + * Return: Total number of command streams, summed across all groups. + * + * @kbdev: Kbase device. + * @group_data: Pointer where to store all the group data + * (sequentially). + * @max_group_num: The maximum number of groups to be read. + * Can be 0, in which case group_data is unused. + * @stream_data: Pointer where to store all the stream data + * (sequentially). + * @max_total_stream_num: The maximum number of streams to be read. + * Can be 0, in which case stream_data is unused. + * @glb_version: Where to store the global interface version. + * Bits 31:16 hold the major version number and + * 15:0 hold the minor version number. + * A higher minor version is backwards-compatible + * with a lower minor version for the same major + * version. + * @features: Where to store a bit mask of features (e.g. + * whether certain types of job can be suspended). + * @group_num: Where to store the number of command stream groups + * supported. + * @prfcnt_size: Where to store the size of CSF performance counters, + * in bytes. Bits 31:16 hold the size of firmware + * performance counter data and 15:0 hold the size of + * hardware performance counter data. + */ +u32 kbase_csf_firmware_get_glb_iface(struct kbase_device *kbdev, + struct basep_cs_group_control *group_data, u32 max_group_num, + struct basep_cs_stream_control *stream_data, u32 max_total_stream_num, + u32 *glb_version, u32 *features, u32 *group_num, u32 *prfcnt_size); + + +/** + * Get CSF firmware header timeline metadata content + * + * Return: The firmware timeline metadata content which match @p name. + * + * @kbdev: Kbase device. + * @name: Name of the metadata which metadata content to be returned. + * @size: Metadata size if specified metadata found. + */ +const char *kbase_csf_firmware_get_timeline_metadata(struct kbase_device *kbdev, + const char *name, size_t *size); + +/** + * kbase_csf_firmware_mcu_shared_mapping_init - + * Allocate and map MCU shared memory. + * + * This helper function allocates memory and maps it on both the CPU + * and the GPU address spaces. Most of the properties of the mapping + * are implicit and will be automatically determined by the function, + * e.g. whether memory is cacheable. + * + * The client is only expected to specify whether the mapping is readable + * or writable in the CPU and the GPU address spaces; any other flag + * will be ignored by the function. + * + * Return: 0 if success, or an error code on failure. + * + * @kbdev: Kbase device the memory mapping shall belong to. + * @num_pages: Number of memory pages to map. + * @cpu_map_properties: Either PROT_READ or PROT_WRITE. + * @gpu_map_properties: Either KBASE_REG_GPU_RD or KBASE_REG_GPU_WR. + * @csf_mapping: Object where to write metadata for the memory mapping. + */ +int kbase_csf_firmware_mcu_shared_mapping_init( + struct kbase_device *kbdev, + unsigned int num_pages, + unsigned long cpu_map_properties, + unsigned long gpu_map_properties, + struct kbase_csf_mapping *csf_mapping); + +/** + * kbase_csf_firmware_mcu_shared_mapping_term - Unmap and free MCU shared memory. + * + * @kbdev: Device pointer. + * @csf_mapping: Metadata of the memory mapping to terminate. + */ +void kbase_csf_firmware_mcu_shared_mapping_term( + struct kbase_device *kbdev, struct kbase_csf_mapping *csf_mapping); + +#ifndef MALI_KBASE_BUILD +/** + * mali_kutf_process_fw_utf_entry() - Process the "Firmware UTF tests" section + * + * Read "Firmware UTF tests" section from the firmware image and create + * necessary kutf app+suite+tests. + * + * Return: 0 if successful, negative error code on failure. In both cases + * caller will have to invoke mali_kutf_fw_utf_entry_cleanup for the cleanup + * + * @kbdev: Kbase device structure + * @fw_data: Pointer to the start of firmware binary image loaded from disk + * @fw_size: Size (in bytes) of the firmware image + * @entry: Pointer to the start of the section + */ +int mali_kutf_process_fw_utf_entry(struct kbase_device *kbdev, + const void *fw_data, size_t fw_size, const u32 *entry); + +/** + * mali_kutf_fw_utf_entry_cleanup() - Remove the Fw UTF tests debugfs entries + * + * Destroy the kutf apps+suites+tests created on parsing "Firmware UTF tests" + * section from the firmware image. + * + * @kbdev: Kbase device structure + */ +void mali_kutf_fw_utf_entry_cleanup(struct kbase_device *kbdev); +#endif + +#ifdef CONFIG_MALI_BIFROST_DEBUG +extern bool fw_debug; +#endif + +static inline long kbase_csf_timeout_in_jiffies(const unsigned int msecs) +{ +#ifdef CONFIG_MALI_BIFROST_DEBUG + return (fw_debug ? MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies(msecs)); +#else + return msecs_to_jiffies(msecs); +#endif +} + +#endif diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c new file mode 100755 index 000000000000..d282d5ca7fc2 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c @@ -0,0 +1,306 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include "mali_kbase_csf_firmware_cfg.h" +#include + +#if CONFIG_SYSFS +#define CSF_FIRMWARE_CFG_SYSFS_DIR_NAME "firmware_config" + +/** + * struct firmware_config - Configuration item within the MCU firmware + * + * The firmware may expose configuration options. Each option has a name, the + * address where the option is controlled and the minimum and maximum values + * that the option can take. + * + * @node: List head linking all options to + * kbase_device:csf.firmware_config + * @kbdev: Pointer to the Kbase device + * @kobj: Kobject corresponding to the sysfs sub-directory, + * inside CSF_FIRMWARE_CFG_SYSFS_DIR_NAME directory, + * representing the configuration option @name. + * @kobj_inited: kobject initialization state + * @name: NUL-terminated string naming the option + * @address: The address in the firmware image of the configuration option + * @min: The lowest legal value of the configuration option + * @max: The maximum legal value of the configuration option + * @cur_val: The current value of the configuration option + */ +struct firmware_config { + struct list_head node; + struct kbase_device *kbdev; + struct kobject kobj; + bool kobj_inited; + char *name; + u32 address; + u32 min; + u32 max; + u32 cur_val; +}; + +#define FW_CFG_ATTR(_name, _mode) \ + struct attribute fw_cfg_attr_##_name = { \ + .name = __stringify(_name), \ + .mode = VERIFY_OCTAL_PERMISSIONS(_mode), \ + } + +static FW_CFG_ATTR(min, S_IRUGO); +static FW_CFG_ATTR(max, S_IRUGO); +static FW_CFG_ATTR(cur, S_IRUGO | S_IWUSR); + +static void fw_cfg_kobj_release(struct kobject *kobj) +{ + struct firmware_config *config = + container_of(kobj, struct firmware_config, kobj); + + kfree(config); +} + +static ssize_t show_fw_cfg(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct firmware_config *config = + container_of(kobj, struct firmware_config, kobj); + struct kbase_device *kbdev = config->kbdev; + u32 val = 0; + + if (!kbdev) + return -ENODEV; + + if (attr == &fw_cfg_attr_max) + val = config->max; + else if (attr == &fw_cfg_attr_min) + val = config->min; + else if (attr == &fw_cfg_attr_cur) { + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + val = config->cur_val; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } else { + dev_warn(kbdev->dev, + "Unexpected read from entry %s/%s", + config->name, attr->name); + return -EINVAL; + } + + return snprintf(buf, PAGE_SIZE, "%u\n", val); +} + +static ssize_t store_fw_cfg(struct kobject *kobj, + struct attribute *attr, + const char *buf, + size_t count) +{ + struct firmware_config *config = + container_of(kobj, struct firmware_config, kobj); + struct kbase_device *kbdev = config->kbdev; + + if (!kbdev) + return -ENODEV; + + if (attr == &fw_cfg_attr_cur) { + unsigned long flags; + u32 val; + int ret = kstrtouint(buf, 0, &val); + + if (ret) { + dev_err(kbdev->dev, + "Couldn't process %s/%s write operation.\n" + "Use format \n", + config->name, attr->name); + return -EINVAL; + } + + if ((val < config->min) || (val > config->max)) + return -EINVAL; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (config->cur_val == val) { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return count; + } + + /* + * If there is already a GPU reset pending then inform + * the User to retry the write. + */ + if (kbase_reset_gpu_silent(kbdev)) { + spin_unlock_irqrestore( + &kbdev->hwaccess_lock, flags); + return -EAGAIN; + } + + /* + * GPU reset request has been placed, now update the + * firmware image. GPU reset will take place only after + * hwaccess_lock is released. + * Update made to firmware image in memory would not + * be lost on GPU reset as configuration entries reside + * in the RONLY section of firmware image, which is not + * reloaded on firmware reboot due to GPU reset. + */ + kbase_csf_update_firmware_memory( + kbdev, config->address, val); + + config->cur_val = val; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Wait for the config update to take effect */ + kbase_reset_gpu_wait(kbdev); + } else { + dev_warn(kbdev->dev, + "Unexpected write to entry %s/%s", + config->name, attr->name); + return -EINVAL; + } + + return count; +} + +static const struct sysfs_ops fw_cfg_ops = { + .show = &show_fw_cfg, + .store = &store_fw_cfg, +}; + +static struct attribute *fw_cfg_attrs[] = { + &fw_cfg_attr_min, + &fw_cfg_attr_max, + &fw_cfg_attr_cur, + NULL, +}; + +static struct kobj_type fw_cfg_kobj_type = { + .release = &fw_cfg_kobj_release, + .sysfs_ops = &fw_cfg_ops, + .default_attrs = fw_cfg_attrs, +}; + +int kbase_csf_firmware_cfg_init(struct kbase_device *kbdev) +{ + struct firmware_config *config; + + kbdev->csf.fw_cfg_kobj = kobject_create_and_add( + CSF_FIRMWARE_CFG_SYSFS_DIR_NAME, &kbdev->dev->kobj); + if (!kbdev->csf.fw_cfg_kobj) { + kobject_put(kbdev->csf.fw_cfg_kobj); + dev_err(kbdev->dev, + "Creation of %s sysfs sub-directory failed\n", + CSF_FIRMWARE_CFG_SYSFS_DIR_NAME); + return -ENOMEM; + } + + list_for_each_entry(config, &kbdev->csf.firmware_config, node) { + int err; + + kbase_csf_read_firmware_memory(kbdev, config->address, + &config->cur_val); + + err = kobject_init_and_add(&config->kobj, &fw_cfg_kobj_type, + kbdev->csf.fw_cfg_kobj, "%s", config->name); + if (err) { + kobject_put(&config->kobj); + dev_err(kbdev->dev, + "Creation of %s sysfs sub-directory failed\n", + config->name); + return err; + } + + config->kobj_inited = true; + } + + return 0; +} + +void kbase_csf_firmware_cfg_term(struct kbase_device *kbdev) +{ + while (!list_empty(&kbdev->csf.firmware_config)) { + struct firmware_config *config; + + config = list_first_entry(&kbdev->csf.firmware_config, + struct firmware_config, node); + list_del(&config->node); + + if (config->kobj_inited) { + kobject_del(&config->kobj); + kobject_put(&config->kobj); + } else + kfree(config); + } + + kobject_del(kbdev->csf.fw_cfg_kobj); + kobject_put(kbdev->csf.fw_cfg_kobj); +} + +int kbase_csf_firmware_cfg_option_entry_parse(struct kbase_device *kbdev, + const struct firmware *fw, + const u32 *entry, unsigned int size) +{ + const char *name = (char *)&entry[3]; + struct firmware_config *config; + const unsigned int name_len = size - CONFIGURATION_ENTRY_NAME_OFFSET; + + /* Allocate enough space for struct firmware_config and the + * configuration option name (with NULL termination) + */ + config = kzalloc(sizeof(*config) + name_len + 1, GFP_KERNEL); + + if (!config) + return -ENOMEM; + + config->kbdev = kbdev; + config->name = (char *)(config+1); + config->address = entry[0]; + config->min = entry[1]; + config->max = entry[2]; + + memcpy(config->name, name, name_len); + config->name[name_len] = 0; + + list_add(&config->node, &kbdev->csf.firmware_config); + + dev_dbg(kbdev->dev, "Configuration option '%s' at 0x%x range %u-%u", + config->name, config->address, + config->min, config->max); + + return 0; +} +#else +int kbase_csf_firmware_cfg_init(struct kbase_device *kbdev) +{ + return 0; +} + +void kbase_csf_firmware_cfg_term(struct kbase_device *kbdev) +{ + /* !CONFIG_SYSFS: Nothing to do here */ +} + +int kbase_csf_firmware_cfg_option_entry_parse(struct kbase_device *kbdev, + const struct firmware *fw, + const u32 *entry, unsigned int size) +{ + return 0; +} +#endif /* CONFIG_SYSFS */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.h new file mode 100755 index 000000000000..ab4b6ebc5296 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.h @@ -0,0 +1,72 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_FIRMWARE_CFG_H_ +#define _KBASE_CSF_FIRMWARE_CFG_H_ + +#include +#include "mali_kbase_csf_firmware.h" +#include + +#define CONFIGURATION_ENTRY_NAME_OFFSET (0xC) + +/** + * kbase_csf_firmware_cfg_init - Create the sysfs directory for configuration + * options present in firmware image. + * + * This function would create a sysfs directory and populate it with a + * sub-directory, that would contain a file per attribute, for every + * configuration option parsed from firmware image. + * + * @kbdev: Pointer to the Kbase device + * + * Return: The initialization error code. + */ +int kbase_csf_firmware_cfg_init(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_cfg_term - Delete the sysfs directory that was created + * for firmware configuration options. + * + * @kbdev: Pointer to the Kbase device + * + */ +void kbase_csf_firmware_cfg_term(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_cfg_option_entry_parse() - Process a + * "configuration option" section. + * + * Read a "configuration option" section adding it to the + * kbase_device:csf.firmware_config list. + * + * Return: 0 if successful, negative error code on failure + * + * @kbdev: Kbase device structure + * @fw: Firmware image containing the section + * @entry: Pointer to the section + * @size: Size (in bytes) of the section + */ +int kbase_csf_firmware_cfg_option_entry_parse(struct kbase_device *kbdev, + const struct firmware *fw, + const u32 *entry, unsigned int size); +#endif /* _KBASE_CSF_FIRMWARE_CFG_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c new file mode 100755 index 000000000000..7401113c5d6a --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c @@ -0,0 +1,1012 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase.h" +#include "mali_kbase_csf_firmware.h" +#include "mali_kbase_csf_trace_buffer.h" +#include "mali_kbase_csf_timeout.h" +#include "mali_kbase_mem.h" +#include "mali_kbase_reset_gpu.h" +#include "device/mali_kbase_device.h" +#include "backend/gpu/mali_kbase_pm_internal.h" +#include "mali_kbase_csf_scheduler.h" +#include "mmu/mali_kbase_mmu.h" + +#include +#include +#include +#include +#include +#if (KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE) +#include +#endif + +#ifdef CONFIG_MALI_BIFROST_DEBUG +/* Makes Driver wait indefinitely for an acknowledgment for the different + * requests it sends to firmware. Otherwise the timeouts interfere with the + * use of debugger for source-level debugging of firmware as Driver initiates + * a GPU reset when a request times out, which always happen when a debugger + * is connected. + */ +bool fw_debug; /* Default value of 0/false */ +module_param(fw_debug, bool, 0444); +MODULE_PARM_DESC(fw_debug, + "Enables effective use of a debugger for debugging firmware code."); +#endif + +#define DUMMY_FW_PAGE_SIZE SZ_4K + +/** + * struct dummy_firmware_csi - Represents a dummy interface for MCU firmware streams + * + * @cs_kernel_input: CS kernel input memory region + * @cs_kernel_output: CS kernel output memory region + */ +struct dummy_firmware_csi { + u8 cs_kernel_input[DUMMY_FW_PAGE_SIZE]; + u8 cs_kernel_output[DUMMY_FW_PAGE_SIZE]; +}; + +/** + * struct dummy_firmware_csg - Represents a dummy interface for MCU firmware stream groups + * + * @csg_input: CSG kernel input memory region + * @csg_output: CSG kernel output memory region + * @csi: Dummy firmware CSIs + */ +struct dummy_firmware_csg { + u8 csg_input[DUMMY_FW_PAGE_SIZE]; + u8 csg_output[DUMMY_FW_PAGE_SIZE]; + struct dummy_firmware_csi csi[8]; +} dummy_firmware_csg; + +/** + * struct dummy_firmware_interface - Represents a dummy interface in the MCU firmware + * + * @global_input: Global input memory region + * @global_output: Global output memory region + * @csg: Dummy firmware CSGs + * @node: Interface objects are on the kbase_device:csf.firmware_interfaces + * list using this list_head to link them + */ +struct dummy_firmware_interface { + u8 global_input[DUMMY_FW_PAGE_SIZE]; + u8 global_output[DUMMY_FW_PAGE_SIZE]; + struct dummy_firmware_csg csg[8]; + struct list_head node; +} dummy_firmware_interface; + +#define CSF_GLB_REQ_CFG_MASK \ + (GLB_REQ_CFG_ALLOC_EN_MASK | GLB_REQ_CFG_PROGRESS_TIMER_MASK) + +static inline u32 input_page_read(const u32 *const input, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return input[offset / sizeof(u32)]; +} + +static inline void input_page_write(u32 *const input, const u32 offset, + const u32 value) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = value; +} + +static inline void input_page_partial_write(u32 *const input, const u32 offset, + u32 value, u32 mask) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = + (input_page_read(input, offset) & ~mask) | (value & mask); +} + +static inline u32 output_page_read(const u32 *const output, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return output[offset / sizeof(u32)]; +} + +static inline void output_page_write(u32 *const output, const u32 offset, + const u32 value) +{ + WARN_ON(offset % sizeof(u32)); + + output[offset / sizeof(u32)] = value; +} + +/** + * invent_memory_setup_entry() - Invent an "interface memory setup" section + * + * Invent an "interface memory setup" section similar to one from a firmware + * image. If successful the interface will be added to the + * kbase_device:csf.firmware_interfaces list. + * + * Return: 0 if successful, negative error code on failure + * + * @kbdev: Kbase device structure + */ +static int invent_memory_setup_entry(struct kbase_device *kbdev) +{ + struct dummy_firmware_interface *interface = NULL; + + /* Allocate enough memory for the struct dummy_firmware_interface. + */ + interface = kmalloc(sizeof(*interface), GFP_KERNEL); + if (!interface) + return -ENOMEM; + + kbdev->csf.shared_interface = interface; + list_add(&interface->node, &kbdev->csf.firmware_interfaces); + + /* NO_MALI: Don't insert any firmware pages */ + return 0; +} + +static void free_global_iface(struct kbase_device *kbdev) +{ + struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; + + if (iface->groups) { + unsigned int gid; + + for (gid = 0; gid < iface->group_num; ++gid) + kfree(iface->groups[gid].streams); + + kfree(iface->groups); + iface->groups = NULL; + } +} + +static int invent_cmd_stream_group_info(struct kbase_device *kbdev, + struct kbase_csf_cmd_stream_group_info *ginfo, + struct dummy_firmware_csg *csg) +{ + unsigned int sid; + + ginfo->input = csg->csg_input; + ginfo->output = csg->csg_output; + + ginfo->kbdev = kbdev; + ginfo->features = 0; + ginfo->suspend_size = 64; + ginfo->protm_suspend_size = 64; + ginfo->stream_num = ARRAY_SIZE(csg->csi); + ginfo->stream_stride = 0; + + ginfo->streams = kcalloc(ginfo->stream_num, sizeof(*ginfo->streams), GFP_KERNEL); + if (ginfo->streams == NULL) { + return -ENOMEM; + } + + for (sid = 0; sid < ginfo->stream_num; ++sid) { + struct kbase_csf_cmd_stream_info *stream = &ginfo->streams[sid]; + struct dummy_firmware_csi *csi = &csg->csi[sid]; + + stream->input = csi->cs_kernel_input; + stream->output = csi->cs_kernel_output; + + stream->kbdev = kbdev; + stream->features = + STREAM_FEATURES_WORK_REGISTERS_SET(0, 80) | + STREAM_FEATURES_SCOREBOARDS_SET(0, 8) | + STREAM_FEATURES_COMPUTE_SET(0, 1) | + STREAM_FEATURES_FRAGMENT_SET(0, 1) | + STREAM_FEATURES_TILER_SET(0, 1); + } + + return 0; +} + +static int invent_capabilities(struct kbase_device *kbdev) +{ + struct dummy_firmware_interface *interface = kbdev->csf.shared_interface; + struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; + unsigned int gid; + + iface->input = interface->global_input; + iface->output = interface->global_output; + + iface->version = 1; + iface->kbdev = kbdev; + iface->features = 0; + iface->prfcnt_size = 64; + iface->group_num = ARRAY_SIZE(interface->csg); + iface->group_stride = 0; + + iface->groups = kcalloc(iface->group_num, sizeof(*iface->groups), GFP_KERNEL); + if (iface->groups == NULL) { + return -ENOMEM; + } + + for (gid = 0; gid < iface->group_num; ++gid) { + int err; + + err = invent_cmd_stream_group_info(kbdev, &iface->groups[gid], + &interface->csg[gid]); + if (err < 0) { + free_global_iface(kbdev); + return err; + } + } + + return 0; +} + +void kbase_csf_read_firmware_memory(struct kbase_device *kbdev, + u32 gpu_addr, u32 *value) +{ + /* NO_MALI: Nothing to do here */ +} + + +void kbase_csf_update_firmware_memory(struct kbase_device *kbdev, + u32 gpu_addr, u32 value) +{ + /* NO_MALI: Nothing to do here */ +} + +void kbase_csf_firmware_cs_input( + const struct kbase_csf_cmd_stream_info *const info, const u32 offset, + const u32 value) +{ + const struct kbase_device * const kbdev = info->kbdev; + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value); + input_page_write(info->input, offset, value); + + if (offset == CS_REQ) { + /* NO_MALI: Immediately acknowledge requests */ + output_page_write(info->output, CS_ACK, value); + } +} + +u32 kbase_csf_firmware_cs_input_read( + const struct kbase_csf_cmd_stream_info *const info, + const u32 offset) +{ + const struct kbase_device * const kbdev = info->kbdev; + u32 const val = input_page_read(info->input, offset); + + dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val); + return val; +} + +void kbase_csf_firmware_cs_input_mask( + const struct kbase_csf_cmd_stream_info *const info, const u32 offset, + const u32 value, const u32 mask) +{ + const struct kbase_device * const kbdev = info->kbdev; + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", + offset, value, mask); + + /* NO_MALI: Go through kbase_csf_firmware_cs_input to capture writes */ + kbase_csf_firmware_cs_input(info, offset, (input_page_read(info->input, offset) & ~mask) | (value & mask)); +} + +u32 kbase_csf_firmware_cs_output( + const struct kbase_csf_cmd_stream_info *const info, const u32 offset) +{ + const struct kbase_device * const kbdev = info->kbdev; + u32 const val = output_page_read(info->output, offset); + + dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val); + return val; +} + +void kbase_csf_firmware_csg_input( + const struct kbase_csf_cmd_stream_group_info *const info, + const u32 offset, const u32 value) +{ + const struct kbase_device * const kbdev = info->kbdev; + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", + offset, value); + input_page_write(info->input, offset, value); + + if (offset == CSG_REQ) { + /* NO_MALI: Immediately acknowledge requests */ + output_page_write(info->output, CSG_ACK, value); + } +} + +u32 kbase_csf_firmware_csg_input_read( + const struct kbase_csf_cmd_stream_group_info *const info, + const u32 offset) +{ + const struct kbase_device * const kbdev = info->kbdev; + u32 const val = input_page_read(info->input, offset); + + dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val); + return val; +} + +void kbase_csf_firmware_csg_input_mask( + const struct kbase_csf_cmd_stream_group_info *const info, + const u32 offset, const u32 value, const u32 mask) +{ + const struct kbase_device * const kbdev = info->kbdev; + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", + offset, value, mask); + + /* NO_MALI: Go through kbase_csf_firmware_csg_input to capture writes */ + kbase_csf_firmware_csg_input(info, offset, (input_page_read(info->input, offset) & ~mask) | (value & mask)); +} + +u32 kbase_csf_firmware_csg_output( + const struct kbase_csf_cmd_stream_group_info *const info, + const u32 offset) +{ + const struct kbase_device * const kbdev = info->kbdev; + u32 const val = output_page_read(info->output, offset); + + dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val); + return val; +} + +void kbase_csf_firmware_global_input( + const struct kbase_csf_global_iface *const iface, const u32 offset, + const u32 value) +{ + const struct kbase_device * const kbdev = iface->kbdev; + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value); + input_page_write(iface->input, offset, value); + + if (offset == GLB_REQ) { + /* NO_MALI: Immediately acknowledge requests */ + output_page_write(iface->output, GLB_ACK, value); + } +} + +void kbase_csf_firmware_global_input_mask( + const struct kbase_csf_global_iface *const iface, const u32 offset, + const u32 value, const u32 mask) +{ + const struct kbase_device * const kbdev = iface->kbdev; + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", + offset, value, mask); + + /* NO_MALI: Go through kbase_csf_firmware_global_input to capture writes */ + kbase_csf_firmware_global_input(iface, offset, (input_page_read(iface->input, offset) & ~mask) | (value & mask)); +} + +u32 kbase_csf_firmware_global_input_read( + const struct kbase_csf_global_iface *const iface, const u32 offset) +{ + const struct kbase_device * const kbdev = iface->kbdev; + u32 const val = input_page_read(iface->input, offset); + + dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val); + return val; +} + +u32 kbase_csf_firmware_global_output( + const struct kbase_csf_global_iface *const iface, const u32 offset) +{ + const struct kbase_device * const kbdev = iface->kbdev; + u32 const val = output_page_read(iface->output, offset); + + dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val); + return val; +} + +static bool global_request_complete(struct kbase_device *const kbdev, + u32 const req_mask) +{ + struct kbase_csf_global_iface *global_iface = + &kbdev->csf.global_iface; + bool complete = false; + unsigned long flags; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + + if ((kbase_csf_firmware_global_output(global_iface, GLB_ACK) & + req_mask) == + (kbase_csf_firmware_global_input_read(global_iface, GLB_REQ) & + req_mask)) + complete = true; + + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + return complete; +} + +static int wait_for_global_request(struct kbase_device *const kbdev, + u32 const req_mask) +{ + const long wait_timeout = + kbase_csf_timeout_in_jiffies(GLB_REQ_WAIT_TIMEOUT_MS); + long remaining; + int err = 0; + + remaining = wait_event_timeout(kbdev->csf.event_wait, + global_request_complete(kbdev, req_mask), + wait_timeout); + + if (!remaining) { + dev_warn(kbdev->dev, "Timed out waiting for global request %x to complete", + req_mask); + err = -ETIMEDOUT; + } + + return err; +} + +static void set_global_request( + const struct kbase_csf_global_iface *const global_iface, + u32 const req_mask) +{ + u32 glb_req; + + lockdep_assert_held(&global_iface->kbdev->csf.reg_lock); + + glb_req = kbase_csf_firmware_global_output(global_iface, GLB_ACK); + glb_req ^= req_mask; + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, glb_req, + req_mask); +} + +static void enable_endpoints_global( + const struct kbase_csf_global_iface *const global_iface, + u64 const shader_core_mask) +{ + kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_LO, + shader_core_mask & U32_MAX); + kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_HI, + shader_core_mask >> 32); + + set_global_request(global_iface, GLB_REQ_CFG_ALLOC_EN_MASK); +} + +static void set_timeout_global( + const struct kbase_csf_global_iface *const global_iface, + u64 const timeout) +{ + kbase_csf_firmware_global_input(global_iface, GLB_PROGRESS_TIMER, + timeout / GLB_PROGRESS_TIMER_TIMEOUT_SCALE); + + set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK); +} + +static void global_init(struct kbase_device *const kbdev, u32 req_mask) +{ + u32 const ack_irq_mask = GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK | + GLB_ACK_IRQ_MASK_PING_MASK | + GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK | + GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK | + GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK; + + const struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + unsigned long flags; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + + /* Enable endpoints on all present shader cores */ + enable_endpoints_global(global_iface, + kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); + + set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); + + /* Unmask the interrupts */ + kbase_csf_firmware_global_input(global_iface, + GLB_ACK_IRQ_MASK, ack_irq_mask); + + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + + kbase_csf_scheduler_spin_unlock(kbdev, flags); +} + +/** + * global_init_on_boot - Sends a global request to control various features. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * Currently only the request to enable endpoints and cycle counter is sent. + * + * Return: 0 on success, or negative on failure. + */ +static int global_init_on_boot(struct kbase_device *const kbdev) +{ + u32 const req_mask = CSF_GLB_REQ_CFG_MASK; + + global_init(kbdev, req_mask); + + return wait_for_global_request(kbdev, req_mask); +} + +void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->csf.glb_init_request_pending = true; + global_init(kbdev, CSF_GLB_REQ_CFG_MASK); +} + +bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + WARN_ON(!kbdev->csf.glb_init_request_pending); + + if (global_request_complete(kbdev, CSF_GLB_REQ_CFG_MASK)) + kbdev->csf.glb_init_request_pending = false; + + return !kbdev->csf.glb_init_request_pending; +} + +static void kbase_csf_firmware_reload_worker(struct work_struct *work) +{ + struct kbase_device *kbdev = container_of(work, struct kbase_device, + csf.firmware_reload_work); + unsigned long flags; + + /* Reboot the firmware */ + kbase_csf_firmware_enable_mcu(kbdev); + + /* Tell MCU state machine to transit to next state */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->csf.firmware_reloaded = true; + kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->csf.firmware_reloaded = false; + + if (kbdev->csf.firmware_reload_needed) { + kbdev->csf.firmware_reload_needed = false; + queue_work(system_wq, &kbdev->csf.firmware_reload_work); + } else { + kbase_csf_firmware_enable_mcu(kbdev); + kbdev->csf.firmware_reloaded = true; + } +} + +void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (unlikely(!kbdev->csf.firmware_inited)) + return; + + /* Tell MCU state machine to transit to next state */ + kbdev->csf.firmware_reloaded = true; + kbase_pm_update_state(kbdev); +} + +int kbase_csf_firmware_init(struct kbase_device *kbdev) +{ + int ret; + + if (WARN_ON((kbdev->as_free & MCU_AS_BITMASK) == 0)) + return -EINVAL; + kbdev->as_free &= ~MCU_AS_BITMASK; + + ret = kbase_mmu_init(kbdev, &kbdev->csf.mcu_mmu, NULL, + BASE_MEM_GROUP_DEFAULT); + + if (ret != 0) { + /* Release the address space */ + kbdev->as_free |= MCU_AS_BITMASK; + return ret; + } + + init_waitqueue_head(&kbdev->csf.event_wait); + kbdev->csf.interrupt_received = false; + + INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces); + INIT_LIST_HEAD(&kbdev->csf.firmware_config); + INIT_LIST_HEAD(&kbdev->csf.firmware_trace_buffers.list); + INIT_WORK(&kbdev->csf.firmware_reload_work, + kbase_csf_firmware_reload_worker); + + mutex_init(&kbdev->csf.reg_lock); + + ret = kbase_mcu_shared_interface_region_tracker_init(kbdev); + if (ret != 0) { + dev_err(kbdev->dev, "Failed to setup the rb tree for managing shared interface segment\n"); + goto error; + } + + ret = invent_memory_setup_entry(kbdev); + if (ret != 0) { + dev_err(kbdev->dev, "Failed to load firmware entry\n"); + goto error; + } + + /* Make sure L2 cache is powered up */ + kbase_pm_wait_for_l2_powered(kbdev); + + /* NO_MALI: Don't init trace buffers */ + + /* NO_MALI: Don't load the MMU tables or boot CSF firmware */ + + ret = invent_capabilities(kbdev); + if (ret != 0) + goto error; + + ret = kbase_csf_doorbell_mapping_init(kbdev); + if (ret != 0) + goto error; + + ret = kbase_csf_scheduler_init(kbdev); + if (ret != 0) + goto error; + + ret = kbase_csf_timeout_init(kbdev); + if (ret != 0) + goto error; + + ret = global_init_on_boot(kbdev); + if (ret != 0) + goto error; + + return 0; + +error: + kbase_csf_firmware_term(kbdev); + return ret; +} + +void kbase_csf_firmware_term(struct kbase_device *kbdev) +{ + kbase_csf_timeout_term(kbdev); + + /* NO_MALI: Don't stop firmware or unload MMU tables */ + + kbase_mmu_term(kbdev, &kbdev->csf.mcu_mmu); + + kbase_csf_scheduler_term(kbdev); + + kbase_csf_doorbell_mapping_term(kbdev); + + free_global_iface(kbdev); + + /* Release the address space */ + kbdev->as_free |= MCU_AS_BITMASK; + + while (!list_empty(&kbdev->csf.firmware_interfaces)) { + struct dummy_firmware_interface *interface; + + interface = list_first_entry(&kbdev->csf.firmware_interfaces, + struct dummy_firmware_interface, node); + list_del(&interface->node); + + /* NO_MALI: No cleanup in dummy interface necessary */ + + kfree(interface); + } + + /* NO_MALI: No trace buffers to terminate */ + +#ifndef MALI_KBASE_BUILD + mali_kutf_fw_utf_entry_cleanup(kbdev); +#endif + + mutex_destroy(&kbdev->csf.reg_lock); + + /* This will also free up the region allocated for the shared interface + * entry parsed from the firmware image. + */ + kbase_mcu_shared_interface_region_tracker_term(kbdev); +} + +int kbase_csf_firmware_ping(struct kbase_device *const kbdev) +{ + const struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + unsigned long flags; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + set_global_request(global_iface, GLB_REQ_PING_MASK); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + return wait_for_global_request(kbdev, GLB_REQ_PING_MASK); +} + +int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev, + u64 const timeout) +{ + const struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + unsigned long flags; + int err; + + /* The 'reg_lock' is also taken and is held till the update is not + * complete, to ensure the update of timeout value by multiple Users + * gets serialized. + */ + mutex_lock(&kbdev->csf.reg_lock); + kbase_csf_scheduler_spin_lock(kbdev, &flags); + set_timeout_global(global_iface, timeout); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + err = wait_for_global_request(kbdev, GLB_REQ_CFG_PROGRESS_TIMER_MASK); + mutex_unlock(&kbdev->csf.reg_lock); + + return err; +} + +void kbase_csf_enter_protected_mode(struct kbase_device *kbdev) +{ + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + unsigned long flags; + unsigned int value; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + value = kbase_csf_firmware_global_output(global_iface, GLB_ACK); + value ^= GLB_REQ_PROTM_ENTER_MASK; + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, value, + GLB_REQ_PROTM_ENTER_MASK); + dev_dbg(kbdev->dev, "Sending request to enter protected mode"); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + wait_for_global_request(kbdev, GLB_REQ_PROTM_ENTER_MASK); +} + +void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev) +{ + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + unsigned long flags; + unsigned int value; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + value = kbase_csf_firmware_global_output(global_iface, GLB_ACK); + value ^= GLB_REQ_HALT_MASK; + kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, value, + GLB_REQ_HALT_MASK); + dev_dbg(kbdev->dev, "Sending request to HALT MCU"); + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_csf_scheduler_spin_unlock(kbdev, flags); +} + +/** + * copy_grp_and_stm - Copy command stream and/or group data + * + * @iface: Global command stream front-end interface provided by + * the firmware. + * @group_data: Pointer where to store all the group data + * (sequentially). + * @max_group_num: The maximum number of groups to be read. Can be 0, in + * which case group_data is unused. + * @stream_data: Pointer where to store all the stream data + * (sequentially). + * @max_total_stream_num: The maximum number of streams to be read. + * Can be 0, in which case stream_data is unused. + * + * Return: Total number of command streams, summed across all groups. + */ +static u32 copy_grp_and_stm( + const struct kbase_csf_global_iface * const iface, + struct basep_cs_group_control * const group_data, + u32 max_group_num, + struct basep_cs_stream_control * const stream_data, + u32 max_total_stream_num) +{ + u32 i, total_stream_num = 0; + + if (WARN_ON((max_group_num > 0) && !group_data)) + max_group_num = 0; + + if (WARN_ON((max_total_stream_num > 0) && !stream_data)) + max_total_stream_num = 0; + + for (i = 0; i < iface->group_num; i++) { + u32 j; + + if (i < max_group_num) { + group_data[i].features = iface->groups[i].features; + group_data[i].stream_num = iface->groups[i].stream_num; + } + for (j = 0; j < iface->groups[i].stream_num; j++) { + if (total_stream_num < max_total_stream_num) + stream_data[total_stream_num].features = + iface->groups[i].streams[j].features; + total_stream_num++; + } + } + + return total_stream_num; +} + +u32 kbase_csf_firmware_get_glb_iface(struct kbase_device *kbdev, + struct basep_cs_group_control *const group_data, + u32 const max_group_num, + struct basep_cs_stream_control *const stream_data, + u32 const max_total_stream_num, u32 *const glb_version, + u32 *const features, u32 *const group_num, u32 *const prfcnt_size) +{ + const struct kbase_csf_global_iface * const iface = + &kbdev->csf.global_iface; + + if (WARN_ON(!glb_version) || + WARN_ON(!features) || + WARN_ON(!group_num) || + WARN_ON(!prfcnt_size)) + return 0; + + *glb_version = iface->version; + *features = iface->features; + *group_num = iface->group_num; + *prfcnt_size = iface->prfcnt_size; + + return copy_grp_and_stm(iface, group_data, max_group_num, + stream_data, max_total_stream_num); +} + +const char *kbase_csf_firmware_get_timeline_metadata( + struct kbase_device *kbdev, const char *name, size_t *size) +{ + if (WARN_ON(!kbdev) || + WARN_ON(!name) || + WARN_ON(!size)) { + return NULL; + } + + *size = 0; + return NULL; +} + +void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) +{ + /* NO_MALI: Nothing to do here */ +} + +int kbase_csf_firmware_mcu_shared_mapping_init( + struct kbase_device *kbdev, + unsigned int num_pages, + unsigned long cpu_map_properties, + unsigned long gpu_map_properties, + struct kbase_csf_mapping *csf_mapping) +{ + struct tagged_addr *phys; + struct kbase_va_region *va_reg; + struct page **page_list; + void *cpu_addr; + int i, ret = 0; + pgprot_t cpu_map_prot = PAGE_KERNEL; + unsigned long gpu_map_prot; + + if (cpu_map_properties & PROT_READ) + cpu_map_prot = PAGE_KERNEL_RO; + + if (kbdev->system_coherency == COHERENCY_ACE) { + gpu_map_prot = + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); + } else { + gpu_map_prot = + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); + cpu_map_prot = pgprot_writecombine(cpu_map_prot); + }; + + phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); + if (!phys) + goto out; + + page_list = kmalloc_array(num_pages, sizeof(*page_list), GFP_KERNEL); + if (!page_list) + goto page_list_alloc_error; + + ret = kbase_mem_pool_alloc_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + num_pages, phys, false); + if (ret <= 0) + goto phys_mem_pool_alloc_error; + + for (i = 0; i < num_pages; i++) + page_list[i] = as_page(phys[i]); + + cpu_addr = vmap(page_list, num_pages, VM_MAP, cpu_map_prot); + if (!cpu_addr) + goto vmap_error; + + va_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, + num_pages, KBASE_REG_ZONE_MCU_SHARED); + if (!va_reg) + goto va_region_alloc_error; + + mutex_lock(&kbdev->csf.reg_lock); + ret = kbase_add_va_region_rbtree(kbdev, va_reg, 0, num_pages, 1); + va_reg->flags &= ~KBASE_REG_FREE; + mutex_unlock(&kbdev->csf.reg_lock); + if (ret) + goto va_region_add_error; + + gpu_map_properties &= (KBASE_REG_GPU_RD | KBASE_REG_GPU_WR); + gpu_map_properties |= gpu_map_prot; + + ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu, + va_reg->start_pfn, &phys[0], num_pages, + gpu_map_properties, KBASE_MEM_GROUP_CSF_FW); + if (ret) + goto mmu_insert_pages_error; + + kfree(page_list); + csf_mapping->phys = phys; + csf_mapping->cpu_addr = cpu_addr; + csf_mapping->va_reg = va_reg; + csf_mapping->num_pages = num_pages; + + return 0; + +mmu_insert_pages_error: + mutex_lock(&kbdev->csf.reg_lock); + kbase_remove_va_region(va_reg); + mutex_unlock(&kbdev->csf.reg_lock); +va_region_add_error: + kbase_free_alloced_region(va_reg); +va_region_alloc_error: + vunmap(cpu_addr); +vmap_error: + kbase_mem_pool_free_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + num_pages, phys, false, false); + +phys_mem_pool_alloc_error: + kfree(page_list); +page_list_alloc_error: + kfree(phys); +out: + /* Zero-initialize the mapping to make sure that the termination + * function doesn't try to unmap or free random addresses. */ + csf_mapping->phys = NULL; + csf_mapping->cpu_addr = NULL; + csf_mapping->va_reg = NULL; + csf_mapping->num_pages = 0; + + return -ENOMEM; +} + +void kbase_csf_firmware_mcu_shared_mapping_term( + struct kbase_device *kbdev, struct kbase_csf_mapping *csf_mapping) +{ + if (csf_mapping->va_reg) { + mutex_lock(&kbdev->csf.reg_lock); + kbase_remove_va_region(csf_mapping->va_reg); + mutex_unlock(&kbdev->csf.reg_lock); + kbase_free_alloced_region(csf_mapping->va_reg); + } + + if (csf_mapping->phys) { + kbase_mem_pool_free_pages( + &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + csf_mapping->num_pages, csf_mapping->phys, false, + false); + } + + vunmap(csf_mapping->cpu_addr); + kfree(csf_mapping->phys); +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c new file mode 100755 index 000000000000..087cc858c2b8 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c @@ -0,0 +1,196 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include "mali_kbase_csf_heap_context_alloc.h" + +/* Size of one heap context structure, in bytes. */ +#define HEAP_CTX_SIZE ((size_t)32) + +/* Total size of the GPU memory region allocated for heap contexts, in bytes. */ +#define HEAP_CTX_REGION_SIZE (MAX_TILER_HEAPS * HEAP_CTX_SIZE) + +/** + * sub_alloc - Sub-allocate a heap context from a GPU memory region + * + * @ctx_alloc: Pointer to the heap context allocator. + * + * Return: GPU virtual address of the allocated heap context or 0 on failure. + */ +static u64 sub_alloc(struct kbase_csf_heap_context_allocator *const ctx_alloc) +{ + struct kbase_context *const kctx = ctx_alloc->kctx; + int heap_nr = 0; + size_t ctx_offset = 0; + u64 heap_gpu_va = 0; + struct kbase_vmap_struct mapping; + void *ctx_ptr = NULL; + + lockdep_assert_held(&ctx_alloc->lock); + + heap_nr = find_first_zero_bit(ctx_alloc->in_use, + MAX_TILER_HEAPS); + + if (unlikely(heap_nr >= MAX_TILER_HEAPS)) { + dev_err(kctx->kbdev->dev, + "No free tiler heap contexts in the pool\n"); + return 0; + } + + ctx_offset = heap_nr * HEAP_CTX_SIZE; + heap_gpu_va = ctx_alloc->gpu_va + ctx_offset; + ctx_ptr = kbase_vmap_prot(kctx, heap_gpu_va, + HEAP_CTX_SIZE, KBASE_REG_CPU_WR, &mapping); + + if (unlikely(!ctx_ptr)) { + dev_err(kctx->kbdev->dev, + "Failed to map tiler heap context %d (0x%llX)\n", + heap_nr, heap_gpu_va); + return 0; + } + + memset(ctx_ptr, 0, HEAP_CTX_SIZE); + kbase_vunmap(ctx_ptr, &mapping); + + bitmap_set(ctx_alloc->in_use, heap_nr, 1); + + dev_dbg(kctx->kbdev->dev, "Allocated tiler heap context %d (0x%llX)\n", + heap_nr, heap_gpu_va); + + return heap_gpu_va; +} + +/** + * sub_free - Free a heap context sub-allocated from a GPU memory region + * + * @ctx_alloc: Pointer to the heap context allocator. + * @heap_gpu_va: The GPU virtual address of a heap context structure to free. + */ +static void sub_free(struct kbase_csf_heap_context_allocator *const ctx_alloc, + u64 const heap_gpu_va) +{ + struct kbase_context *const kctx = ctx_alloc->kctx; + u64 ctx_offset = 0; + unsigned int heap_nr = 0; + + lockdep_assert_held(&ctx_alloc->lock); + + if (WARN_ON(!ctx_alloc->region)) + return; + + if (WARN_ON(heap_gpu_va < ctx_alloc->gpu_va)) + return; + + ctx_offset = heap_gpu_va - ctx_alloc->gpu_va; + + if (WARN_ON(ctx_offset >= HEAP_CTX_REGION_SIZE) || + WARN_ON(ctx_offset % HEAP_CTX_SIZE)) + return; + + heap_nr = ctx_offset / HEAP_CTX_SIZE; + dev_dbg(kctx->kbdev->dev, + "Freed tiler heap context %d (0x%llX)\n", heap_nr, heap_gpu_va); + + bitmap_clear(ctx_alloc->in_use, heap_nr, 1); +} + +int kbase_csf_heap_context_allocator_init( + struct kbase_csf_heap_context_allocator *const ctx_alloc, + struct kbase_context *const kctx) +{ + /* We cannot pre-allocate GPU memory here because the + * custom VA zone may not have been created yet. + */ + ctx_alloc->kctx = kctx; + ctx_alloc->region = NULL; + ctx_alloc->gpu_va = 0; + + mutex_init(&ctx_alloc->lock); + bitmap_zero(ctx_alloc->in_use, MAX_TILER_HEAPS); + + dev_dbg(kctx->kbdev->dev, + "Initialized a tiler heap context allocator\n"); + + return 0; +} + +void kbase_csf_heap_context_allocator_term( + struct kbase_csf_heap_context_allocator *const ctx_alloc) +{ + struct kbase_context *const kctx = ctx_alloc->kctx; + + dev_dbg(kctx->kbdev->dev, + "Terminating tiler heap context allocator\n"); + + if (ctx_alloc->region) { + kbase_gpu_vm_lock(kctx); + ctx_alloc->region->flags &= ~KBASE_REG_NO_USER_FREE; + kbase_mem_free_region(kctx, ctx_alloc->region); + kbase_gpu_vm_unlock(kctx); + } + + mutex_destroy(&ctx_alloc->lock); +} + +u64 kbase_csf_heap_context_allocator_alloc( + struct kbase_csf_heap_context_allocator *const ctx_alloc) +{ + struct kbase_context *const kctx = ctx_alloc->kctx; + u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | + BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE; + u64 nr_pages = PFN_UP(HEAP_CTX_REGION_SIZE); + u64 heap_gpu_va = 0; + +#ifdef CONFIG_MALI_VECTOR_DUMP + flags |= BASE_MEM_PROT_CPU_RD; +#endif + + mutex_lock(&ctx_alloc->lock); + + /* If the pool of heap contexts wasn't already allocated then + * allocate it. + */ + if (!ctx_alloc->region) { + ctx_alloc->region = kbase_mem_alloc(kctx, nr_pages, nr_pages, + 0, &flags, &ctx_alloc->gpu_va); + } + + /* If the pool still isn't allocated then an error occurred. */ + if (unlikely(!ctx_alloc->region)) { + dev_err(kctx->kbdev->dev, "Failed to allocate a pool of tiler heap contexts\n"); + } else { + heap_gpu_va = sub_alloc(ctx_alloc); + } + + mutex_unlock(&ctx_alloc->lock); + + return heap_gpu_va; +} + +void kbase_csf_heap_context_allocator_free( + struct kbase_csf_heap_context_allocator *const ctx_alloc, + u64 const heap_gpu_va) +{ + mutex_lock(&ctx_alloc->lock); + sub_free(ctx_alloc, heap_gpu_va); + mutex_unlock(&ctx_alloc->lock); +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.h new file mode 100755 index 000000000000..f71ea01ed8c0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.h @@ -0,0 +1,76 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include + +#ifndef _KBASE_CSF_HEAP_CONTEXT_ALLOC_H_ +#define _KBASE_CSF_HEAP_CONTEXT_ALLOC_H_ + +/** + * kbase_csf_heap_context_allocator_init - Initialize an allocator for heap + * contexts + * @ctx_alloc: Pointer to the heap context allocator to initialize. + * @kctx: Pointer to the kbase context. + * + * Return: 0 if successful or a negative error code on failure. + */ +int kbase_csf_heap_context_allocator_init( + struct kbase_csf_heap_context_allocator *const ctx_alloc, + struct kbase_context *const kctx); + +/** + * kbase_csf_heap_context_allocator_term - Terminate an allocator for heap + * contexts + * @ctx_alloc: Pointer to the heap context allocator to terminate. + */ +void kbase_csf_heap_context_allocator_term( + struct kbase_csf_heap_context_allocator *const ctx_alloc); + +/** + * kbase_csf_heap_context_allocator_alloc - Allocate a heap context structure + * + * If this function is successful then it returns the address of a + * zero-initialized heap context structure for use by the firmware. + * + * @ctx_alloc: Pointer to the heap context allocator. + * + * Return: GPU virtual address of the allocated heap context or 0 on failure. + */ +u64 kbase_csf_heap_context_allocator_alloc( + struct kbase_csf_heap_context_allocator *const ctx_alloc); + +/** + * kbase_csf_heap_context_allocator_free - Free a heap context structure + * + * This function returns a heap context structure to the free pool of unused + * contexts for possible reuse by a future call to + * @kbase_csf_heap_context_allocator_alloc. + * + * @ctx_alloc: Pointer to the heap context allocator. + * @heap_gpu_va: The GPU virtual address of a heap context structure that + * was allocated for the firmware. + */ +void kbase_csf_heap_context_allocator_free( + struct kbase_csf_heap_context_allocator *const ctx_alloc, + u64 const heap_gpu_va); + +#endif /* _KBASE_CSF_HEAP_CONTEXT_ALLOC_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h new file mode 100755 index 000000000000..e9bb8d299754 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h @@ -0,0 +1,379 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_IOCTL_H_ +#define _KBASE_CSF_IOCTL_H_ + +#include +#include + +/* + * 1.0: + * - CSF IOCTL header separated from JM + */ + +#define BASE_UK_VERSION_MAJOR 1 +#define BASE_UK_VERSION_MINOR 0 + +/** + * struct kbase_ioctl_version_check - Check version compatibility between + * kernel and userspace + * + * @major: Major version number + * @minor: Minor version number + */ +struct kbase_ioctl_version_check { + __u16 major; + __u16 minor; +}; + +#define KBASE_IOCTL_VERSION_CHECK \ + _IOWR(KBASE_IOCTL_TYPE, 52, struct kbase_ioctl_version_check) + +#define KBASE_IOCTL_VERSION_CHECK_RESERVED \ + _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) + + +/** + * struct kbase_ioctl_cs_queue_register - Register a GPU command queue with the + * base back-end + * + * @buffer_gpu_addr: GPU address of the buffer backing the queue + * @buffer_size: Size of the buffer in bytes + * @priority: Priority of the queue within a group when run within a process + * @padding: Currently unused, must be zero + */ +struct kbase_ioctl_cs_queue_register { + __u64 buffer_gpu_addr; + __u32 buffer_size; + __u8 priority; + __u8 padding[3]; +}; + +#define KBASE_IOCTL_CS_QUEUE_REGISTER \ + _IOW(KBASE_IOCTL_TYPE, 36, struct kbase_ioctl_cs_queue_register) + +/** + * struct kbase_ioctl_cs_queue_kick - Kick the GPU command queue group scheduler + * to notify that a queue has been updated + * + * @buffer_gpu_addr: GPU address of the buffer backing the queue + */ +struct kbase_ioctl_cs_queue_kick { + __u64 buffer_gpu_addr; +}; + +#define KBASE_IOCTL_CS_QUEUE_KICK \ + _IOW(KBASE_IOCTL_TYPE, 37, struct kbase_ioctl_cs_queue_kick) + +/** + * union kbase_ioctl_cs_queue_bind - Bind a GPU command queue to a group + * + * @buffer_gpu_addr: GPU address of the buffer backing the queue + * @group_handle: Handle of the group to which the queue should be bound + * @csi_index: Index of the CSF interface the queue should be bound to + * @padding: Currently unused, must be zero + * @mmap_handle: Handle to be used for creating the mapping of command stream + * input/output pages + * + * @in: Input parameters + * @out: Output parameters + * + */ +union kbase_ioctl_cs_queue_bind { + struct { + __u64 buffer_gpu_addr; + __u8 group_handle; + __u8 csi_index; + __u8 padding[6]; + } in; + struct { + __u64 mmap_handle; + } out; +}; + +#define KBASE_IOCTL_CS_QUEUE_BIND \ + _IOWR(KBASE_IOCTL_TYPE, 39, union kbase_ioctl_cs_queue_bind) + +/* ioctl 40 is free to use */ + +/** + * struct kbase_ioctl_cs_queue_terminate - Terminate a GPU command queue + * + * @buffer_gpu_addr: GPU address of the buffer backing the queue + */ +struct kbase_ioctl_cs_queue_terminate { + __u64 buffer_gpu_addr; +}; + +#define KBASE_IOCTL_CS_QUEUE_TERMINATE \ + _IOW(KBASE_IOCTL_TYPE, 41, struct kbase_ioctl_cs_queue_terminate) + +/** + * union kbase_ioctl_cs_queue_group_create - Create a GPU command queue group + * + * @tiler_mask: Mask of tiler endpoints the group is allowed to use. + * @fragment_mask: Mask of fragment endpoints the group is allowed to use. + * @compute_mask: Mask of compute endpoints the group is allowed to use. + * @cs_min: Minimum number of command streams required. + * @priority: Queue group's priority within a process. + * @tiler_max: Maximum number of tiler endpoints the group is allowed + * to use. + * @fragment_max: Maximum number of fragment endpoints the group is + * allowed to use. + * @compute_max: Maximum number of compute endpoints the group is allowed + * to use. + * @padding: Currently unused, must be zero + * @group_handle: Handle of a newly created queue group. + * + * @in: Input parameters + * @out: Output parameters + * + */ +union kbase_ioctl_cs_queue_group_create { + struct { + __u64 tiler_mask; + __u64 fragment_mask; + __u64 compute_mask; + __u8 cs_min; + __u8 priority; + __u8 tiler_max; + __u8 fragment_max; + __u8 compute_max; + __u8 padding[3]; + + } in; + struct { + __u8 group_handle; + __u8 padding[7]; + } out; +}; + +#define KBASE_IOCTL_CS_QUEUE_GROUP_CREATE \ + _IOWR(KBASE_IOCTL_TYPE, 42, union kbase_ioctl_cs_queue_group_create) + +/** + * struct kbase_ioctl_cs_queue_group_term - Terminate a GPU command queue group + * + * @group_handle: Handle of the queue group to be terminated + * @padding: Padding to round up to a multiple of 8 bytes, must be zero + */ +struct kbase_ioctl_cs_queue_group_term { + __u8 group_handle; + __u8 padding[7]; +}; + +#define KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE \ + _IOW(KBASE_IOCTL_TYPE, 43, struct kbase_ioctl_cs_queue_group_term) + +#define KBASE_IOCTL_CS_EVENT_SIGNAL \ + _IO(KBASE_IOCTL_TYPE, 44) + +typedef __u8 base_kcpu_queue_id; /* We support up to 256 active KCPU queues */ + +/** + * struct kbase_ioctl_kcpu_queue_new - Create a KCPU command queue + * + * @id: ID of the new command queue returned by the kernel + * @padding: Padding to round up to a multiple of 8 bytes, must be zero + */ +struct kbase_ioctl_kcpu_queue_new { + base_kcpu_queue_id id; + __u8 padding[7]; +}; + +#define KBASE_IOCTL_KCPU_QUEUE_CREATE \ + _IOR(KBASE_IOCTL_TYPE, 45, struct kbase_ioctl_kcpu_queue_new) + +/** + * struct kbase_ioctl_kcpu_queue_delete - Destroy a KCPU command queue + * + * @id: ID of the command queue to be destroyed + * @padding: Padding to round up to a multiple of 8 bytes, must be zero + */ +struct kbase_ioctl_kcpu_queue_delete { + base_kcpu_queue_id id; + __u8 padding[7]; +}; + +#define KBASE_IOCTL_KCPU_QUEUE_DELETE \ + _IOW(KBASE_IOCTL_TYPE, 46, struct kbase_ioctl_kcpu_queue_delete) + +/** + * struct kbase_ioctl_kcpu_queue_enqueue - Enqueue commands into the KCPU queue + * + * @addr: Memory address of an array of struct base_kcpu_queue_command + * @nr_commands: Number of commands in the array + * @id: kcpu queue identifier, returned by KBASE_IOCTL_KCPU_QUEUE_CREATE ioctl + * @padding: Padding to round up to a multiple of 8 bytes, must be zero + */ +struct kbase_ioctl_kcpu_queue_enqueue { + __u64 addr; + __u32 nr_commands; + base_kcpu_queue_id id; + __u8 padding[3]; +}; + +#define KBASE_IOCTL_KCPU_QUEUE_ENQUEUE \ + _IOW(KBASE_IOCTL_TYPE, 47, struct kbase_ioctl_kcpu_queue_enqueue) + +/** + * union kbase_ioctl_cs_tiler_heap_init - Initialize chunked tiler memory heap + * + * @chunk_size: Size of each chunk. + * @initial_chunks: Initial number of chunks that heap will be created with. + * @max_chunks: Maximum number of chunks that the heap is allowed to use. + * @target_in_flight: Number of render-passes that the driver should attempt to + * keep in flight for which allocation of new chunks is + * allowed. + * @group_id: Group ID to be used for physical allocations. + * @gpu_heap_va: GPU VA (virtual address) of Heap context that was set up for + * the heap. + * @first_chunk_va: GPU VA of the first chunk allocated for the heap, actually + * points to the header of heap chunk and not to the low + * address of free memory in the chunk. + * + * @in: Input parameters + * @out: Output parameters + * + */ +union kbase_ioctl_cs_tiler_heap_init { + struct { + __u32 chunk_size; + __u32 initial_chunks; + __u32 max_chunks; + __u16 target_in_flight; + __u8 group_id; + __u8 padding; + } in; + struct { + __u64 gpu_heap_va; + __u64 first_chunk_va; + } out; +}; + +#define KBASE_IOCTL_CS_TILER_HEAP_INIT \ + _IOWR(KBASE_IOCTL_TYPE, 48, union kbase_ioctl_cs_tiler_heap_init) + +/** + * struct kbase_ioctl_cs_tiler_heap_term - Terminate a chunked tiler heap + * instance + * + * @gpu_heap_va: GPU VA of Heap context that was set up for the heap. + */ +struct kbase_ioctl_cs_tiler_heap_term { + __u64 gpu_heap_va; +}; + +#define KBASE_IOCTL_CS_TILER_HEAP_TERM \ + _IOW(KBASE_IOCTL_TYPE, 49, struct kbase_ioctl_cs_tiler_heap_term) + +/** + * union kbase_ioctl_cs_get_glb_iface - Request the global control block + * of CSF interface capabilities + * + * @max_group_num: The maximum number of groups to be read. Can be 0, in + * which case groups_ptr is unused. + * @max_total_stream_num: The maximum number of streams to be read. Can be 0, in + * which case streams_ptr is unused. + * @groups_ptr: Pointer where to store all the group data (sequentially). + * @streams_ptr: Pointer where to store all the stream data (sequentially). + * @glb_version: Global interface version. Bits 31:16 hold the major + * version number and 15:0 hold the minor version number. + * A higher minor version is backwards-compatible with a + * lower minor version for the same major version. + * @features: Bit mask of features (e.g. whether certain types of job + * can be suspended). + * @group_num: Number of command stream groups supported. + * @prfcnt_size: Size of CSF performance counters, in bytes. Bits 31:16 + * hold the size of firmware performance counter data + * and 15:0 hold the size of hardware performance counter + * data. + * @total_stream_num: Total number of command streams, summed across all groups. + * @padding: Will be zeroed. + * + * @in: Input parameters + * @out: Output parameters + * + */ +union kbase_ioctl_cs_get_glb_iface { + struct { + __u32 max_group_num; + __u32 max_total_stream_num; + __u64 groups_ptr; + __u64 streams_ptr; + } in; + struct { + __u32 glb_version; + __u32 features; + __u32 group_num; + __u32 prfcnt_size; + __u32 total_stream_num; + __u32 padding; + } out; +}; + +#define KBASE_IOCTL_CS_GET_GLB_IFACE \ + _IOWR(KBASE_IOCTL_TYPE, 51, union kbase_ioctl_cs_get_glb_iface) + +/*************** + * test ioctls * + ***************/ +#if MALI_UNIT_TEST +/* These ioctls are purely for test purposes and are not used in the production + * driver, they therefore may change without notice + */ + +/** + * struct kbase_ioctl_cs_event_memory_write - Write an event memory address + * @cpu_addr: Memory address to write + * @value: Value to write + * @padding: Currently unused, must be zero + */ +struct kbase_ioctl_cs_event_memory_write { + __u64 cpu_addr; + __u8 value; + __u8 padding[7]; +}; + +/** + * union kbase_ioctl_cs_event_memory_read - Read an event memory address + * @cpu_addr: Memory address to read + * @value: Value read + * @padding: Currently unused, must be zero + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_cs_event_memory_read { + struct { + __u64 cpu_addr; + } in; + struct { + __u8 value; + __u8 padding[7]; + } out; +}; + +#endif /* MALI_UNIT_TEST */ + +#endif /* _KBASE_CSF_IOCTL_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c new file mode 100755 index 000000000000..e1263d535918 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c @@ -0,0 +1,1737 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include "device/mali_kbase_device.h" +#include "mali_kbase_csf.h" +#include + +#ifdef CONFIG_SYNC_FILE +#include "mali_kbase_fence.h" +#include "mali_kbase_sync.h" + +static DEFINE_SPINLOCK(kbase_csf_fence_lock); +#endif + +static void kcpu_queue_process(struct kbase_kcpu_command_queue *kcpu_queue, + bool ignore_waits); + +static void kcpu_queue_process_worker(struct work_struct *data); + +static int kbase_kcpu_map_import_prepare( + struct kbase_kcpu_command_queue *kcpu_queue, + struct base_kcpu_command_import_info *import_info, + struct kbase_kcpu_command *current_command) +{ + struct kbase_context *const kctx = kcpu_queue->kctx; + struct kbase_va_region *reg; + int ret = 0; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + /* Take the processes mmap lock */ + down_read(kbase_mem_get_process_mmap_lock()); + kbase_gpu_vm_lock(kctx); + + reg = kbase_region_tracker_find_region_enclosing_address(kctx, + import_info->handle); + + if (kbase_is_region_invalid_or_free(reg) || + !kbase_mem_is_imported(reg->gpu_alloc->type)) { + ret = -EINVAL; + goto out; + } + + if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { + /* Pin the physical pages backing the user buffer while + * we are in the process context and holding the mmap lock. + * The dma mapping & GPU mapping of the pages would be done + * when the MAP_IMPORT operation is executed. + * + * Though the pages would be pinned, no reference is taken + * on the physical pages tracking object. When the last + * reference to the tracking object is dropped the pages + * would be unpinned if they weren't unpinned before. + */ + ret = kbase_jd_user_buf_pin_pages(kctx, reg); + if (ret) + goto out; + } + + current_command->type = BASE_KCPU_COMMAND_TYPE_MAP_IMPORT; + current_command->info.import.gpu_va = import_info->handle; + +out: + kbase_gpu_vm_unlock(kctx); + /* Release the processes mmap lock */ + up_read(kbase_mem_get_process_mmap_lock()); + + return ret; +} + +static int kbase_kcpu_unmap_import_prepare_internal( + struct kbase_kcpu_command_queue *kcpu_queue, + struct base_kcpu_command_import_info *import_info, + struct kbase_kcpu_command *current_command, + enum base_kcpu_command_type type) +{ + struct kbase_context *const kctx = kcpu_queue->kctx; + struct kbase_va_region *reg; + int ret = 0; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + kbase_gpu_vm_lock(kctx); + + reg = kbase_region_tracker_find_region_enclosing_address(kctx, + import_info->handle); + + if (kbase_is_region_invalid_or_free(reg) || + !kbase_mem_is_imported(reg->gpu_alloc->type)) { + ret = -EINVAL; + goto out; + } + + if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { + /* The pages should have been pinned when MAP_IMPORT + * was enqueued previously. + */ + if (reg->gpu_alloc->nents != + reg->gpu_alloc->imported.user_buf.nr_pages) { + ret = -EINVAL; + goto out; + } + } + + current_command->type = type; + current_command->info.import.gpu_va = import_info->handle; + +out: + kbase_gpu_vm_unlock(kctx); + + return ret; +} + +static int kbase_kcpu_unmap_import_prepare( + struct kbase_kcpu_command_queue *kcpu_queue, + struct base_kcpu_command_import_info *import_info, + struct kbase_kcpu_command *current_command) +{ + return kbase_kcpu_unmap_import_prepare_internal(kcpu_queue, + import_info, current_command, + BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT); +} + +static int kbase_kcpu_unmap_import_force_prepare( + struct kbase_kcpu_command_queue *kcpu_queue, + struct base_kcpu_command_import_info *import_info, + struct kbase_kcpu_command *current_command) +{ + return kbase_kcpu_unmap_import_prepare_internal(kcpu_queue, + import_info, current_command, + BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE); +} + +/** + * kbase_jit_add_to_pending_alloc_list() - Pend JIT allocation + * + * @queue: The queue containing this JIT allocation + * @cmd: The JIT allocation that is blocking this queue + */ +static void kbase_jit_add_to_pending_alloc_list( + struct kbase_kcpu_command_queue *queue, + struct kbase_kcpu_command *cmd) +{ + struct kbase_context *const kctx = queue->kctx; + struct list_head *target_list_head = + &kctx->csf.kcpu_queues.jit_blocked_queues; + struct kbase_kcpu_command_queue *blocked_queue; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + list_for_each_entry(blocked_queue, + &kctx->csf.kcpu_queues.jit_blocked_queues, + jit_blocked) { + struct kbase_kcpu_command const*const jit_alloc_cmd = + &blocked_queue->commands[blocked_queue->start_offset]; + + WARN_ON(jit_alloc_cmd->type != BASE_KCPU_COMMAND_TYPE_JIT_ALLOC); + if (cmd->enqueue_ts < jit_alloc_cmd->enqueue_ts) { + target_list_head = &blocked_queue->jit_blocked; + break; + } + } + + list_add_tail(&queue->jit_blocked, target_list_head); +} + +/** + * kbase_kcpu_jit_allocate_process() - Process JIT allocation + * + * @queue: The queue containing this JIT allocation + * @cmd: The JIT allocation command + */ +static int kbase_kcpu_jit_allocate_process( + struct kbase_kcpu_command_queue *queue, + struct kbase_kcpu_command *cmd) +{ + struct kbase_context *const kctx = queue->kctx; + struct kbase_kcpu_command_jit_alloc_info *alloc_info = + &cmd->info.jit_alloc; + struct base_jit_alloc_info *info = alloc_info->info; + struct kbase_vmap_struct mapping; + struct kbase_va_region *reg; + u32 count = alloc_info->count; + u64 *ptr, new_addr; + u32 i; + int ret; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + if (alloc_info->blocked) { + list_del(&queue->jit_blocked); + alloc_info->blocked = false; + } + + if (WARN_ON(!info)) + return -EINVAL; + + /* Check if all JIT IDs are not in use */ + for (i = 0; i < count; i++, info++) { + /* The JIT ID is still in use so fail the allocation */ + if (kctx->jit_alloc[info->id]) { + dev_warn(kctx->kbdev->dev, "JIT ID still in use\n"); + return -EINVAL; + } + } + + /* Now start the allocation loop */ + for (i = 0, info = alloc_info->info; i < count; i++, info++) { + if (kctx->jit_alloc[info->id]) { + /* The JIT ID is duplicated in this command. Roll back + * previous allocations and fail. + */ + dev_warn(kctx->kbdev->dev, "JIT ID is duplicated\n"); + ret = -EINVAL; + goto fail; + } + + /* Create a JIT allocation */ + reg = kbase_jit_allocate(kctx, info, true); + if (!reg) { + bool can_block = false; + struct kbase_kcpu_command const *jit_cmd; + + list_for_each_entry(jit_cmd, &kctx->csf.kcpu_queues.jit_cmds_head, info.jit_alloc.node) { + if (jit_cmd == cmd) + break; + + if (jit_cmd->type == BASE_KCPU_COMMAND_TYPE_JIT_FREE) { + u8 const*const free_ids = jit_cmd->info.jit_free.ids; + + if (free_ids && *free_ids && kctx->jit_alloc[*free_ids]) { + /** + * A JIT free which is active + * and submitted before this + * command. + */ + can_block = true; + break; + } + } + } + + if (!can_block) { + /** + * No prior JIT_FREE command is active. Roll + * back previous allocations and fail. + */ + dev_warn_ratelimited(kctx->kbdev->dev, "JIT alloc command failed: %p\n", cmd); + ret = -ENOMEM; + goto fail; + } + + /* There are pending frees for an active allocation + * so we should wait to see whether they free the + * memory. Add to the list of atoms for which JIT + * allocation is pending. + */ + kbase_jit_add_to_pending_alloc_list(queue, cmd); + alloc_info->blocked = true; + + /* Rollback, the whole set will be re-attempted */ + while (i-- > 0) { + info--; + kbase_jit_free(kctx, kctx->jit_alloc[info->id]); + kctx->jit_alloc[info->id] = NULL; + } + + return -EAGAIN; + } + + /* Bind it to the user provided ID. */ + kctx->jit_alloc[info->id] = reg; + } + + for (i = 0, info = alloc_info->info; i < count; i++, info++) { + /* + * Write the address of the JIT allocation to the user provided + * GPU allocation. + */ + ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr), + &mapping); + if (!ptr) { + ret = -ENOMEM; + goto fail; + } + + reg = kctx->jit_alloc[info->id]; + new_addr = reg->start_pfn << PAGE_SHIFT; + *ptr = new_addr; + kbase_vunmap(kctx, &mapping); + } + + return 0; + +fail: + /* Roll back completely */ + for (i = 0, info = alloc_info->info; i < count; i++, info++) { + /* Free the allocations that were successful. + * Mark all the allocations including the failed one and the + * other un-attempted allocations in the set, so we know they + * are in use. + */ + if (kctx->jit_alloc[info->id]) + kbase_jit_free(kctx, kctx->jit_alloc[info->id]); + + kctx->jit_alloc[info->id] = KBASE_RESERVED_REG_JIT_ALLOC; + } + + return ret; +} + +static int kbase_kcpu_jit_allocate_prepare( + struct kbase_kcpu_command_queue *kcpu_queue, + struct base_kcpu_command_jit_alloc_info *alloc_info, + struct kbase_kcpu_command *current_command) +{ + struct kbase_context *const kctx = kcpu_queue->kctx; + void __user *data = u64_to_user_ptr(alloc_info->info); + struct base_jit_alloc_info *info; + u32 count = alloc_info->count; + int ret = 0; + u32 i; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + if (!data || count > kcpu_queue->kctx->jit_max_allocations || + count > ARRAY_SIZE(kctx->jit_alloc)) { + ret = -EINVAL; + goto out; + } + + info = kmalloc_array(count, sizeof(*info), GFP_KERNEL); + if (!info) { + ret = -ENOMEM; + goto out; + } + + if (copy_from_user(info, data, sizeof(*info) * count) != 0) { + ret = -EINVAL; + goto out_free; + } + + for (i = 0; i < count; i++) { + ret = kbasep_jit_alloc_validate(kctx, &info[i]); + if (ret) + goto out_free; + } + + current_command->type = BASE_KCPU_COMMAND_TYPE_JIT_ALLOC; + list_add_tail(¤t_command->info.jit_alloc.node, + &kctx->csf.kcpu_queues.jit_cmds_head); + current_command->info.jit_alloc.info = info; + current_command->info.jit_alloc.count = count; + current_command->info.jit_alloc.blocked = false; + + return 0; +out_free: + kfree(info); +out: + return ret; +} + +/** + * kbase_kcpu_jit_allocate_finish() - Finish handling the JIT_ALLOC command + * + * @queue: The queue containing this JIT allocation + * @cmd: The JIT allocation command + */ +static void kbase_kcpu_jit_allocate_finish( + struct kbase_kcpu_command_queue *queue, + struct kbase_kcpu_command *cmd) +{ + lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); + + /* Remove this command from the jit_cmds_head list */ + list_del(&cmd->info.jit_alloc.node); + + /** + * If we get to this point we must have already cleared the blocked + * flag, otherwise it'd be a bug. + */ + if (WARN_ON(cmd->info.jit_alloc.blocked)) { + list_del(&queue->jit_blocked); + cmd->info.jit_alloc.blocked = false; + } + + kfree(cmd->info.jit_alloc.info); +} + +/** + * kbase_kcpu_jit_retry_pending_allocs() - Retry blocked JIT_ALLOC commands + * + * @kctx: The context containing the blocked JIT_ALLOC commands + */ +static void kbase_kcpu_jit_retry_pending_allocs(struct kbase_context *kctx) +{ + struct kbase_kcpu_command_queue *blocked_queue; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + /** + * Reschedule all queues blocked by JIT_ALLOC commands. + * NOTE: This code traverses the list of blocked queues directly. It + * only works as long as the queued works are not executed at the same + * time. This precondition is true since we're holding the + * kbase_csf_kcpu_queue_context.lock . + */ + list_for_each_entry(blocked_queue, + &kctx->csf.kcpu_queues.jit_blocked_queues, jit_blocked) + queue_work(kctx->csf.kcpu_queues.wq, &blocked_queue->work); +} + +static int kbase_kcpu_jit_free_process(struct kbase_context *kctx, + struct kbase_kcpu_command *const cmd) +{ + struct kbase_kcpu_command_jit_free_info *const free_info = + &cmd->info.jit_free; + u8 *ids = free_info->ids; + u32 count = free_info->count; + u32 i; + + if (WARN_ON(!ids)) + return -EINVAL; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + for (i = 0; i < count; i++, ids++) { + if ((*ids == 0) || (kctx->jit_alloc[*ids] == NULL)) { + dev_warn(kctx->kbdev->dev, "invalid JIT free ID\n"); + } else { + /* If the ID is valid but the allocation request + * failed, still succeed this command but don't + * try and free the allocation. + */ + if (kctx->jit_alloc[*ids] != + KBASE_RESERVED_REG_JIT_ALLOC) + kbase_jit_free(kctx, kctx->jit_alloc[*ids]); + + kctx->jit_alloc[*ids] = NULL; + } + } + + /* Free the list of ids */ + kfree(free_info->ids); + + /** + * Remove this command from the jit_cmds_head list and retry pending + * allocations. + */ + list_del(&cmd->info.jit_free.node); + kbase_kcpu_jit_retry_pending_allocs(kctx); + + return 0; +} + +static int kbase_kcpu_jit_free_prepare( + struct kbase_kcpu_command_queue *kcpu_queue, + struct base_kcpu_command_jit_free_info *free_info, + struct kbase_kcpu_command *current_command) +{ + struct kbase_context *const kctx = kcpu_queue->kctx; + void __user *data = u64_to_user_ptr(free_info->ids); + u8 *ids; + u32 count = free_info->count; + int ret; + u32 i; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + /* Sanity checks */ + if (!count || count > ARRAY_SIZE(kctx->jit_alloc)) { + ret = -EINVAL; + goto out; + } + + /* Copy the information for safe access and future storage */ + ids = kmalloc_array(count, sizeof(*ids), GFP_KERNEL); + if (!ids) { + ret = -ENOMEM; + goto out; + } + + if (!data) { + ret = -EINVAL; + goto out_free; + } + + if (copy_from_user(ids, data, sizeof(*ids) * count)) { + ret = -EINVAL; + goto out_free; + } + + for (i = 0; i < count; i++) { + /* Fail the command if ID sent is zero */ + if (!ids[i]) { + ret = -EINVAL; + goto out_free; + } + } + + current_command->type = BASE_KCPU_COMMAND_TYPE_JIT_FREE; + list_add_tail(¤t_command->info.jit_free.node, + &kctx->csf.kcpu_queues.jit_cmds_head); + current_command->info.jit_free.ids = ids; + current_command->info.jit_free.count = count; + + return 0; +out_free: + kfree(ids); +out: + return ret; +} + +static int kbase_csf_queue_group_suspend_prepare( + struct kbase_kcpu_command_queue *kcpu_queue, + struct base_kcpu_command_group_suspend_info *suspend_buf, + struct kbase_kcpu_command *current_command) +{ + struct kbase_context *const kctx = kcpu_queue->kctx; + struct kbase_suspend_copy_buffer *sus_buf = NULL; + u64 addr = suspend_buf->buffer; + u64 page_addr = addr & PAGE_MASK; + u64 end_addr = addr + suspend_buf->size - 1; + u64 last_page_addr = end_addr & PAGE_MASK; + int nr_pages = (last_page_addr - page_addr) / PAGE_SIZE + 1; + int pinned_pages; + int ret = 0; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + if (suspend_buf->size < + kctx->kbdev->csf.global_iface.groups[0].suspend_size) + return -EINVAL; + + ret = kbase_csf_queue_group_handle_is_valid(kctx, + suspend_buf->group_handle); + if (ret) + return ret; + + sus_buf = kzalloc(sizeof(*sus_buf), GFP_KERNEL); + if (!sus_buf) + return -ENOMEM; + + sus_buf->size = suspend_buf->size; + sus_buf->nr_pages = nr_pages; + sus_buf->offset = addr & ~PAGE_MASK; + + sus_buf->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); + if (!sus_buf->pages) { + ret = -ENOMEM; + goto out_clean_sus_buf; + } + + pinned_pages = get_user_pages_fast(page_addr, nr_pages, 1, + sus_buf->pages); + if (pinned_pages < 0) { + ret = pinned_pages; + goto out_clean_pages; + } + if (pinned_pages != nr_pages) { + ret = -EINVAL; + goto out_clean_pages; + } + + current_command->type = BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND; + current_command->info.suspend_buf_copy.sus_buf = sus_buf; + current_command->info.suspend_buf_copy.group_handle = + suspend_buf->group_handle; + return ret; + +out_clean_pages: + kfree(sus_buf->pages); +out_clean_sus_buf: + kfree(sus_buf); + return ret; +} + +static int kbase_csf_queue_group_suspend_process(struct kbase_context *kctx, + struct kbase_suspend_copy_buffer *sus_buf, + u8 group_handle) +{ + return kbase_csf_queue_group_suspend(kctx, sus_buf, group_handle); +} + +static enum kbase_csf_event_callback_action event_cqs_callback(void *param) +{ + struct kbase_kcpu_command_queue *kcpu_queue = + (struct kbase_kcpu_command_queue *)param; + struct kbase_context *const kctx = kcpu_queue->kctx; + + queue_work(kctx->csf.kcpu_queues.wq, &kcpu_queue->work); + + return KBASE_CSF_EVENT_CALLBACK_KEEP; +} + +static void cleanup_cqs_wait(struct kbase_kcpu_command_queue *queue, + struct kbase_kcpu_command_cqs_wait_info *cqs_wait) +{ + WARN_ON(!cqs_wait->nr_objs); + WARN_ON(!cqs_wait->objs); + WARN_ON(!cqs_wait->signaled); + WARN_ON(!queue->cqs_wait_count); + + if (--queue->cqs_wait_count == 0) { + kbase_csf_event_wait_remove(queue->kctx, + event_cqs_callback, queue); + } + + kfree(cqs_wait->signaled); + kfree(cqs_wait->objs); + cqs_wait->signaled = NULL; + cqs_wait->objs = NULL; +} + +static int kbase_kcpu_cqs_wait_process(struct kbase_device *kbdev, + struct kbase_kcpu_command_queue *queue, + struct kbase_kcpu_command_cqs_wait_info *cqs_wait) +{ + u32 i; + + lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); + + if (WARN_ON(!cqs_wait->nr_objs)) + return -EINVAL; + + if (WARN_ON(!cqs_wait->objs)) + return -EINVAL; + + + /* Skip the CQS waits that have already been signaled when processing */ + for (i = find_first_zero_bit(cqs_wait->signaled, cqs_wait->nr_objs); i < cqs_wait->nr_objs; i++) { + if (!test_bit(i, cqs_wait->signaled)) { + struct kbase_vmap_struct *mapping; + bool sig_set; + u32 *evt = (u32 *)kbase_phy_alloc_mapping_get(queue->kctx, + cqs_wait->objs[i].addr, &mapping); + + if (!queue->command_started) { + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( + kbdev, queue); + queue->command_started = true; + } + + if (WARN_ON(!evt)) { + queue->has_error = true; + return -EINVAL; + } + + sig_set = evt[BASEP_EVENT_VAL_INDEX] > cqs_wait->objs[i].val; + if (sig_set) { + bitmap_set(cqs_wait->signaled, i, 1); + if ((cqs_wait->inherit_err_flags & (1U << i)) && + evt[BASEP_EVENT_ERR_INDEX] > 0) + queue->has_error = true; + + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( + kbdev, queue); + queue->command_started = false; + } + + kbase_phy_alloc_mapping_put(queue->kctx, mapping); + + if (!sig_set) + break; + } + } + + /* For the queue to progress further, all cqs objects should get + * signaled. + */ + return bitmap_full(cqs_wait->signaled, cqs_wait->nr_objs); +} + +static int kbase_kcpu_cqs_wait_prepare(struct kbase_kcpu_command_queue *queue, + struct base_kcpu_command_cqs_wait_info *cqs_wait_info, + struct kbase_kcpu_command *current_command) +{ + struct base_cqs_wait *objs; + unsigned int nr_objs = cqs_wait_info->nr_objs; + + lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); + + if (cqs_wait_info->nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS) + return -EINVAL; + + objs = kcalloc(nr_objs, sizeof(*objs), GFP_KERNEL); + if (!objs) + return -ENOMEM; + + if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_info->objs), + nr_objs * sizeof(*objs))) { + kfree(objs); + return -ENOMEM; + } + + if (++queue->cqs_wait_count == 1) { + if (kbase_csf_event_wait_add(queue->kctx, + event_cqs_callback, queue)) { + kfree(objs); + return -ENOMEM; + } + } + + current_command->type = BASE_KCPU_COMMAND_TYPE_CQS_WAIT; + current_command->info.cqs_wait.nr_objs = nr_objs; + current_command->info.cqs_wait.objs = objs; + current_command->info.cqs_wait.inherit_err_flags = + cqs_wait_info->inherit_err_flags; + + current_command->info.cqs_wait.signaled = kcalloc(BITS_TO_LONGS(nr_objs), + sizeof(*current_command->info.cqs_wait.signaled), GFP_KERNEL); + if (!current_command->info.cqs_wait.signaled) + return -ENOMEM; + + return 0; +} + +static void kbase_kcpu_cqs_set_process(struct kbase_device *kbdev, + struct kbase_kcpu_command_queue *queue, + struct kbase_kcpu_command_cqs_set_info *cqs_set) +{ + unsigned int i; + + lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); + + WARN_ON(!cqs_set->nr_objs); + WARN_ON(!cqs_set->objs); + + for (i = 0; i < cqs_set->nr_objs; i++) { + struct kbase_vmap_struct *mapping; + u32 *evt = (u32 *)kbase_phy_alloc_mapping_get(queue->kctx, + cqs_set->objs[i].addr, &mapping); + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET(kbdev, queue); + if (WARN_ON(!evt)) + queue->has_error = true; + else { + if (cqs_set->propagate_flags & (1 << i)) + evt[BASEP_EVENT_ERR_INDEX] = queue->has_error; + else + evt[BASEP_EVENT_ERR_INDEX] = false; + /* Set to signaled */ + evt[BASEP_EVENT_VAL_INDEX]++; + kbase_phy_alloc_mapping_put(queue->kctx, mapping); + } + } + + kbase_csf_event_signal_notify_gpu(queue->kctx); + + kfree(cqs_set->objs); + cqs_set->objs = NULL; +} + +static int kbase_kcpu_cqs_set_prepare( + struct kbase_kcpu_command_queue *kcpu_queue, + struct base_kcpu_command_cqs_set_info *cqs_set_info, + struct kbase_kcpu_command *current_command) +{ + struct kbase_context *const kctx = kcpu_queue->kctx; + struct base_cqs_set *objs; + unsigned int nr_objs = cqs_set_info->nr_objs; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + if (cqs_set_info->nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS) + return -EINVAL; + + objs = kcalloc(nr_objs, sizeof(*objs), GFP_KERNEL); + if (!objs) + return -ENOMEM; + + if (copy_from_user(objs, u64_to_user_ptr(cqs_set_info->objs), + nr_objs * sizeof(*objs))) { + kfree(objs); + return -ENOMEM; + } + + current_command->type = BASE_KCPU_COMMAND_TYPE_CQS_SET; + current_command->info.cqs_set.nr_objs = nr_objs; + current_command->info.cqs_set.objs = objs; + current_command->info.cqs_set.propagate_flags = + cqs_set_info->propagate_flags; + + return 0; +} + +#ifdef CONFIG_SYNC_FILE +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) +static void kbase_csf_fence_wait_callback(struct fence *fence, + struct fence_cb *cb) +#else +static void kbase_csf_fence_wait_callback(struct dma_fence *fence, + struct dma_fence_cb *cb) +#endif +{ + struct kbase_kcpu_command_fence_info *fence_info = container_of(cb, + struct kbase_kcpu_command_fence_info, fence_cb); + struct kbase_kcpu_command_queue *kcpu_queue = fence_info->kcpu_queue; + struct kbase_context *const kctx = kcpu_queue->kctx; + + /* Resume kcpu command queue processing. */ + queue_work(kctx->csf.kcpu_queues.wq, &kcpu_queue->work); +} + +static void kbase_kcpu_fence_wait_cancel( + struct kbase_kcpu_command_queue *kcpu_queue, + struct kbase_kcpu_command_fence_info *fence_info) +{ + struct kbase_context *const kctx = kcpu_queue->kctx; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + if (WARN_ON(!fence_info->fence)) + return; + + if (kcpu_queue->fence_wait_processed) { + dma_fence_remove_callback(fence_info->fence, + &fence_info->fence_cb); + } + + /* Release the reference which is kept by the kcpu_queue */ + kbase_fence_put(fence_info->fence); + kcpu_queue->fence_wait_processed = false; + + fence_info->fence = NULL; +} + +/** + * kbase_kcpu_fence_wait_process() - Process the kcpu fence wait command + * + * @kcpu_queue: The queue containing the fence wait command + * @fence_info: Reference to a fence for which the command is waiting + * + * Return: 0 if fence wait is blocked, 1 if it is unblocked, negative error if + * an error has occurred and fence should no longer be waited on. + */ +static int kbase_kcpu_fence_wait_process( + struct kbase_kcpu_command_queue *kcpu_queue, + struct kbase_kcpu_command_fence_info *fence_info) +{ + int fence_status = 0; +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + lockdep_assert_held(&kcpu_queue->kctx->csf.kcpu_queues.lock); + + if (WARN_ON(!fence_info->fence)) + return -EINVAL; + + fence = fence_info->fence; + + if (kcpu_queue->fence_wait_processed) { + fence_status = dma_fence_get_status(fence); + } else { + int cb_err = dma_fence_add_callback(fence, + &fence_info->fence_cb, + kbase_csf_fence_wait_callback); + + fence_status = cb_err; + if (cb_err == 0) + kcpu_queue->fence_wait_processed = true; + else if (cb_err == -ENOENT) + fence_status = dma_fence_get_status(fence); + } + + /* + * At this point fence status can contain 3 types of values: + * - Value 0 to represent that fence in question is not signalled yet + * - Value 1 to represent that fence in question is signalled without + * errors + * - Negative error code to represent that some error has occurred such + * that waiting on it is no longer valid. + */ + + if (fence_status) + kbase_kcpu_fence_wait_cancel(kcpu_queue, fence_info); + + return fence_status; +} + +static int kbase_kcpu_fence_wait_prepare( + struct kbase_kcpu_command_queue *kcpu_queue, + struct base_kcpu_command_fence_info *fence_info, + struct kbase_kcpu_command *current_command) +{ + struct kbase_context *const kctx = kcpu_queue->kctx; +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) + struct fence *fence_in; +#else + struct dma_fence *fence_in; +#endif + struct base_fence fence; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + if (copy_from_user(&fence, u64_to_user_ptr(fence_info->fence), + sizeof(fence))) + return -ENOMEM; + + fence_in = sync_file_get_fence(fence.basep.fd); + + if (!fence_in) + return -ENOENT; + + current_command->type = BASE_KCPU_COMMAND_TYPE_FENCE_WAIT; + current_command->info.fence.fence = fence_in; + current_command->info.fence.kcpu_queue = kcpu_queue; + + return 0; +} + +static int kbase_kcpu_fence_signal_process( + struct kbase_kcpu_command_queue *kcpu_queue, + struct kbase_kcpu_command_fence_info *fence_info) +{ + struct kbase_context *const kctx = kcpu_queue->kctx; + int ret; + + if (WARN_ON(!fence_info->fence)) + return -EINVAL; + + ret = dma_fence_signal(fence_info->fence); + + if (unlikely(ret < 0)) { + dev_warn(kctx->kbdev->dev, + "fence_signal() failed with %d\n", ret); + } + + dma_fence_put(fence_info->fence); + fence_info->fence = NULL; + + return ret; +} + +static int kbase_kcpu_fence_signal_prepare( + struct kbase_kcpu_command_queue *kcpu_queue, + struct base_kcpu_command_fence_info *fence_info, + struct kbase_kcpu_command *current_command) +{ + struct kbase_context *const kctx = kcpu_queue->kctx; +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) + struct fence *fence_out; +#else + struct dma_fence *fence_out; +#endif + struct base_fence fence; + struct sync_file *sync_file; + int ret = 0; + int fd; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + if (copy_from_user(&fence, u64_to_user_ptr(fence_info->fence), + sizeof(fence))) + return -EFAULT; + + fence_out = kzalloc(sizeof(*fence_out), GFP_KERNEL); + if (!fence_out) + return -ENOMEM; + + dma_fence_init(fence_out, + &kbase_fence_ops, + &kbase_csf_fence_lock, + kcpu_queue->fence_context, + ++kcpu_queue->fence_seqno); + +#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) + /* Take an extra reference to the fence on behalf of the sync file. + * This is only needded on older kernels where sync_file_create() + * does not take its own reference. This was changed in v4.9.68 + * where sync_file_create() now takes its own reference. + */ + dma_fence_get(fence_out); +#endif + + /* create a sync_file fd representing the fence */ + sync_file = sync_file_create(fence_out); + if (!sync_file) { +#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) + dma_fence_put(fence_out); +#endif + ret = -ENOMEM; + goto file_create_fail; + } + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + ret = fd; + goto fd_flags_fail; + } + + fd_install(fd, sync_file->file); + + fence.basep.fd = fd; + + current_command->type = BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL; + current_command->info.fence.fence = fence_out; + + if (copy_to_user(u64_to_user_ptr(fence_info->fence), &fence, + sizeof(fence))) { + ret = -EFAULT; + goto fd_flags_fail; + } + + return 0; + +fd_flags_fail: + fput(sync_file->file); +file_create_fail: + dma_fence_put(fence_out); + + return ret; +} +#endif /* CONFIG_SYNC_FILE */ + +static void kcpu_queue_process_worker(struct work_struct *data) +{ + struct kbase_kcpu_command_queue *queue = container_of(data, + struct kbase_kcpu_command_queue, work); + + mutex_lock(&queue->kctx->csf.kcpu_queues.lock); + + kcpu_queue_process(queue, false); + + mutex_unlock(&queue->kctx->csf.kcpu_queues.lock); +} + +static int delete_queue(struct kbase_context *kctx, u32 id) +{ + int err = 0; + + mutex_lock(&kctx->csf.kcpu_queues.lock); + + if ((id < KBASEP_MAX_KCPU_QUEUES) && kctx->csf.kcpu_queues.array[id]) { + struct kbase_kcpu_command_queue *queue = + kctx->csf.kcpu_queues.array[id]; + + /* Drain the remaining work for this queue first and go past + * all the waits. + */ + kcpu_queue_process(queue, true); + + /* All commands should have been processed */ + WARN_ON(queue->num_pending_cmds); + + /* All CQS wait commands should have been cleaned up */ + WARN_ON(queue->cqs_wait_count); + + kctx->csf.kcpu_queues.array[id] = NULL; + bitmap_clear(kctx->csf.kcpu_queues.in_use, id, 1); + + /* Fire the tracepoint with the mutex held to enforce correct + * ordering with the summary stream. + */ + KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE(kctx->kbdev, queue); + + mutex_unlock(&kctx->csf.kcpu_queues.lock); + + cancel_work_sync(&queue->work); + + kfree(queue); + } else { + dev_warn(kctx->kbdev->dev, + "Attempt to delete a non-existent KCPU queue\n"); + mutex_unlock(&kctx->csf.kcpu_queues.lock); + err = -EINVAL; + } + return err; +} + +static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_INFO( + struct kbase_device *kbdev, + const struct kbase_kcpu_command_queue *queue, + const struct kbase_kcpu_command_jit_alloc_info *jit_alloc, + bool alloc_success) +{ + u8 i; + + KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( + kbdev, queue); + for (i = 0; i < jit_alloc->count; i++) { + const u8 id = jit_alloc->info[i].id; + const struct kbase_va_region *reg = queue->kctx->jit_alloc[id]; + u64 gpu_alloc_addr = 0; + u64 mmu_flags = 0; + + if (alloc_success && !WARN_ON(!reg) && + !WARN_ON(reg == KBASE_RESERVED_REG_JIT_ALLOC)) { +#ifdef CONFIG_MALI_VECTOR_DUMP + struct tagged_addr phy = {0}; +#endif /* CONFIG_MALI_VECTOR_DUMP */ + + gpu_alloc_addr = reg->start_pfn << PAGE_SHIFT; +#ifdef CONFIG_MALI_VECTOR_DUMP + mmu_flags = kbase_mmu_create_ate(kbdev, + phy, reg->flags, + MIDGARD_MMU_BOTTOMLEVEL, + queue->kctx->jit_group_id); +#endif /* CONFIG_MALI_VECTOR_DUMP */ + } + KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( + kbdev, queue, gpu_alloc_addr, mmu_flags); + } +} + +static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( + struct kbase_device *kbdev, + const struct kbase_kcpu_command_queue *queue) +{ + KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( + kbdev, queue); +} + +static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_INFO( + struct kbase_device *kbdev, + const struct kbase_kcpu_command_queue *queue, + const struct kbase_kcpu_command_jit_free_info *jit_free) +{ + u8 i; + + KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( + kbdev, queue); + for (i = 0; i < jit_free->count; i++) { + const u8 id = jit_free->ids[i]; + u64 pages_used = 0; + + if (id != 0) { + const struct kbase_va_region *reg = + queue->kctx->jit_alloc[id]; + if (reg && (reg != KBASE_RESERVED_REG_JIT_ALLOC)) + pages_used = reg->gpu_alloc->nents; + } + KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( + kbdev, queue, pages_used); + } +} + +static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END( + struct kbase_device *kbdev, + const struct kbase_kcpu_command_queue *queue) +{ + KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( + kbdev, queue); +} + +static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, + bool ignore_waits) +{ + struct kbase_device *kbdev = queue->kctx->kbdev; + bool process_next = true; + size_t i; + + lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); + + for (i = 0; i != queue->num_pending_cmds; ++i) { + struct kbase_kcpu_command *cmd = + &queue->commands[(u8)(queue->start_offset + i)]; + int status; + + switch (cmd->type) { + case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: + if (!queue->command_started) { + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( + kbdev, queue); + queue->command_started = true; + } + +#ifdef CONFIG_SYNC_FILE + status = 0; + + + if (ignore_waits) { + kbase_kcpu_fence_wait_cancel(queue, + &cmd->info.fence); + } else { + status = kbase_kcpu_fence_wait_process(queue, + &cmd->info.fence); + + if (status == 0) + process_next = false; + else if (status < 0) + queue->has_error = true; + } +#else + dev_warn(kbdev->dev, + "unexpected fence wait command found\n"); +#endif + + if (process_next) { + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( + kbdev, queue); + queue->command_started = false; + } + break; + case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( + kbdev, queue); + +#ifdef CONFIG_SYNC_FILE + kbase_kcpu_fence_signal_process(queue, + &cmd->info.fence); +#else + dev_warn(kbdev->dev, + "unexpected fence signal command found\n"); +#endif + + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( + kbdev, queue); + break; + case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: + status = kbase_kcpu_cqs_wait_process(kbdev, queue, + &cmd->info.cqs_wait); + + if (!status && !ignore_waits) { + process_next = false; + } else { + /* Either all CQS objects were signaled or + * there was an error or the queue itself is + * being deleted. + * In all cases can move to the next command. + * TBD: handle the error + */ + cleanup_cqs_wait(queue, &cmd->info.cqs_wait); + } + + break; + case BASE_KCPU_COMMAND_TYPE_CQS_SET: + kbase_kcpu_cqs_set_process(kbdev, queue, + &cmd->info.cqs_set); + + /* CQS sets are only traced before execution */ + break; + case BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: + /* Clear the queue's error state */ + queue->has_error = false; + break; + case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( + kbdev, queue); + + kbase_gpu_vm_lock(queue->kctx); + kbase_sticky_resource_acquire(queue->kctx, + cmd->info.import.gpu_va); + kbase_gpu_vm_unlock(queue->kctx); + + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( + kbdev, queue); + break; + case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( + kbdev, queue); + + kbase_gpu_vm_lock(queue->kctx); + kbase_sticky_resource_release(queue->kctx, NULL, + cmd->info.import.gpu_va); + kbase_gpu_vm_unlock(queue->kctx); + + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( + kbdev, queue); + break; + case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( + kbdev, queue); + + kbase_gpu_vm_lock(queue->kctx); + kbase_sticky_resource_release_force(queue->kctx, NULL, + cmd->info.import.gpu_va); + kbase_gpu_vm_unlock(queue->kctx); + + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( + kbdev, queue); + break; + case BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: + { + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( + kbdev, queue); + + status = kbase_kcpu_jit_allocate_process(queue, cmd); + if (status == -EAGAIN) { + process_next = false; + } else { + if (status != 0) + queue->has_error = true; + + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_INFO( + kbdev, queue, &cmd->info.jit_alloc, (status == 0)); + + kbase_kcpu_jit_allocate_finish(queue, cmd); + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( + kbdev, queue); + } + break; + } + case BASE_KCPU_COMMAND_TYPE_JIT_FREE: + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( + kbdev, queue); + + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_INFO( + kbdev, queue, &cmd->info.jit_free); + + status = kbase_kcpu_jit_free_process(queue->kctx, cmd); + if (status) + queue->has_error = true; + + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END( + kbdev, queue); + break; + case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: + status = kbase_csf_queue_group_suspend_process( + queue->kctx, + cmd->info.suspend_buf_copy.sus_buf, + cmd->info.suspend_buf_copy.group_handle); + if (status) + queue->has_error = true; + + kfree(cmd->info.suspend_buf_copy.sus_buf->pages); + kfree(cmd->info.suspend_buf_copy.sus_buf); + break; + default: + dev_warn(kbdev->dev, + "Unrecognized command type\n"); + break; + } /* switch */ + + /*TBD: error handling */ + + if (!process_next) + break; + } + + if (i > 0) { + queue->start_offset += i; + queue->num_pending_cmds -= i; + + /* If an attempt to enqueue commands failed then we must raise + * an event in case the client wants to retry now that there is + * free space in the buffer. + */ + if (queue->enqueue_failed) { + queue->enqueue_failed = false; + kbase_csf_event_signal_cpu_only(queue->kctx); + } + } +} + +static size_t kcpu_queue_get_space(struct kbase_kcpu_command_queue *queue) +{ + return KBASEP_KCPU_QUEUE_SIZE - queue->num_pending_cmds; +} + +static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_COMMAND( + const struct kbase_kcpu_command_queue *queue, + const struct kbase_kcpu_command *cmd) +{ + struct kbase_device *kbdev = queue->kctx->kbdev; + + switch (cmd->type) { + case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( + kbdev, queue, cmd->info.fence.fence); + break; + case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( + kbdev, queue, cmd->info.fence.fence); + break; + case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: + { + const struct base_cqs_wait *waits = cmd->info.cqs_wait.objs; + unsigned int i; + + for (i = 0; i < cmd->info.cqs_wait.nr_objs; i++) { + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( + kbdev, queue, waits[i].addr, waits[i].val); + } + break; + } + case BASE_KCPU_COMMAND_TYPE_CQS_SET: + { + const struct base_cqs_set *sets = cmd->info.cqs_set.objs; + unsigned int i; + + for (i = 0; i < cmd->info.cqs_set.nr_objs; i++) { + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( + kbdev, queue, sets[i].addr); + } + break; + } + case BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: + /* No implemented tracepoint */ + break; + case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( + kbdev, queue, cmd->info.import.gpu_va); + break; + case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( + kbdev, queue, cmd->info.import.gpu_va); + break; + case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( + kbdev, queue, cmd->info.import.gpu_va); + break; + case BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: + { + u8 i; + + KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( + kbdev, queue); + for (i = 0; i < cmd->info.jit_alloc.count; i++) { + const struct base_jit_alloc_info *info = + &cmd->info.jit_alloc.info[i]; + + KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( + kbdev, queue, + info->gpu_alloc_addr, info->va_pages, + info->commit_pages, info->extent, info->id, + info->bin_id, info->max_allocations, + info->flags, info->usage_id); + } + KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( + kbdev, queue); + break; + } + case BASE_KCPU_COMMAND_TYPE_JIT_FREE: + { + u8 i; + + KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( + kbdev, queue); + for (i = 0; i < cmd->info.jit_free.count; i++) { + KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( + kbdev, queue, cmd->info.jit_free.ids[i]); + } + KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( + kbdev, queue); + break; + } + case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: + /* No implemented tracepoint */ + break; + } +} + +int kbase_csf_kcpu_queue_enqueue(struct kbase_context *kctx, + struct kbase_ioctl_kcpu_queue_enqueue *enq) +{ + struct kbase_kcpu_command_queue *queue = NULL; + void __user *user_cmds = u64_to_user_ptr(enq->addr); + int ret = 0; + u32 i; + + /* The offset to the first command that is being processed or yet to + * be processed is of u8 type, so the number of commands inside the + * queue cannot be more than 256. + */ + BUILD_BUG_ON(KBASEP_KCPU_QUEUE_SIZE > 256); + + /* Whilst the backend interface allows enqueueing multiple commands in + * a single operation, the Base interface does not expose any mechanism + * to do so. And also right now the handling is missing for the case + * where multiple commands are submitted and the enqueue of one of the + * command in the set fails after successfully enqueuing other commands + * in the set. + */ + if (enq->nr_commands != 1) { + dev_err(kctx->kbdev->dev, + "More than one commands enqueued\n"); + return -EINVAL; + } + + mutex_lock(&kctx->csf.kcpu_queues.lock); + + if (!kctx->csf.kcpu_queues.array[enq->id]) { + ret = -EINVAL; + goto out; + } + + queue = kctx->csf.kcpu_queues.array[enq->id]; + + if (kcpu_queue_get_space(queue) < enq->nr_commands) { + ret = -EBUSY; + queue->enqueue_failed = true; + goto out; + } + + /* Copy all command's info to the command buffer. + * Note: it would be more efficient to process all commands in-line + * until we encounter an unresolved CQS_ / FENCE_WAIT, however, the + * interface allows multiple commands to be enqueued so we must account + * for the possibility to roll back. + */ + + for (i = 0; (i != enq->nr_commands) && !ret; ++i, ++kctx->csf.kcpu_queues.num_cmds) { + struct kbase_kcpu_command *kcpu_cmd = + &queue->commands[(u8)(queue->start_offset + queue->num_pending_cmds + i)]; + struct base_kcpu_command command; + unsigned int j; + + if (copy_from_user(&command, user_cmds, sizeof(command))) { + ret = -EFAULT; + goto out; + } + + user_cmds = (void __user *)((uintptr_t)user_cmds + + sizeof(struct base_kcpu_command)); + + for (j = 0; j < sizeof(command.padding); j++) { + if (command.padding[j] != 0) { + dev_dbg(kctx->kbdev->dev, + "base_kcpu_command padding not 0\n"); + ret = -EINVAL; + goto out; + } + } + + kcpu_cmd->enqueue_ts = kctx->csf.kcpu_queues.num_cmds; + switch (command.type) { + case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: +#ifdef CONFIG_SYNC_FILE + ret = kbase_kcpu_fence_wait_prepare(queue, + &command.info.fence, kcpu_cmd); +#else + ret = -EINVAL; + dev_warn(kctx->kbdev->dev, "fence wait command unsupported\n"); +#endif + break; + case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: +#ifdef CONFIG_SYNC_FILE + ret = kbase_kcpu_fence_signal_prepare(queue, + &command.info.fence, kcpu_cmd); +#else + ret = -EINVAL; + dev_warn(kctx->kbdev->dev, "fence signal command unsupported\n"); +#endif + break; + case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: + ret = kbase_kcpu_cqs_wait_prepare(queue, + &command.info.cqs_wait, kcpu_cmd); + break; + case BASE_KCPU_COMMAND_TYPE_CQS_SET: + ret = kbase_kcpu_cqs_set_prepare(queue, + &command.info.cqs_set, kcpu_cmd); + break; + case BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: + kcpu_cmd->type = BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER; + ret = 0; + break; + case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: + ret = kbase_kcpu_map_import_prepare(queue, + &command.info.import, kcpu_cmd); + break; + case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: + ret = kbase_kcpu_unmap_import_prepare(queue, + &command.info.import, kcpu_cmd); + break; + case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: + ret = kbase_kcpu_unmap_import_force_prepare(queue, + &command.info.import, kcpu_cmd); + break; + case BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: + ret = kbase_kcpu_jit_allocate_prepare(queue, + &command.info.jit_alloc, kcpu_cmd); + break; + case BASE_KCPU_COMMAND_TYPE_JIT_FREE: + ret = kbase_kcpu_jit_free_prepare(queue, + &command.info.jit_free, kcpu_cmd); + break; + case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: + ret = kbase_csf_queue_group_suspend_prepare(queue, + &command.info.suspend_buf_copy, + kcpu_cmd); + break; + + default: + dev_warn(queue->kctx->kbdev->dev, + "Unknown command type %u\n", command.type); + ret = -EINVAL; + break; + } + } + + if (!ret) { + /* We only instrument the enqueues after all commands have been + * successfully enqueued, as if we do them during the enqueue + * and there is an error, we won't be able to roll them back + * like is done for the command enqueues themselves. + */ + for (i = 0; i != enq->nr_commands; ++i) { + u8 cmd_idx = (u8)(queue->start_offset + queue->num_pending_cmds + i); + + KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_COMMAND( + queue, &queue->commands[cmd_idx]); + } + + queue->num_pending_cmds += enq->nr_commands; + kcpu_queue_process(queue, false); + } else { + /* Roll back the number of enqueued commands */ + kctx->csf.kcpu_queues.num_cmds -= i; + } + +out: + mutex_unlock(&kctx->csf.kcpu_queues.lock); + + return ret; +} + +int kbase_csf_kcpu_queue_context_init(struct kbase_context *kctx) +{ + int idx; + + bitmap_zero(kctx->csf.kcpu_queues.in_use, KBASEP_MAX_KCPU_QUEUES); + + for (idx = 0; idx < KBASEP_MAX_KCPU_QUEUES; ++idx) + kctx->csf.kcpu_queues.array[idx] = NULL; + + kctx->csf.kcpu_queues.wq = alloc_workqueue("mali_kbase_csf_kcpu", + WQ_UNBOUND | WQ_HIGHPRI, 0); + if (!kctx->csf.kcpu_queues.wq) + return -ENOMEM; + + mutex_init(&kctx->csf.kcpu_queues.lock); + + kctx->csf.kcpu_queues.num_cmds = 0; + + return 0; +} + +void kbase_csf_kcpu_queue_context_term(struct kbase_context *kctx) +{ + while (!bitmap_empty(kctx->csf.kcpu_queues.in_use, + KBASEP_MAX_KCPU_QUEUES)) { + int id = find_first_bit(kctx->csf.kcpu_queues.in_use, + KBASEP_MAX_KCPU_QUEUES); + + if (WARN_ON(!kctx->csf.kcpu_queues.array[id])) + clear_bit(id, kctx->csf.kcpu_queues.in_use); + else + (void)delete_queue(kctx, id); + } + + destroy_workqueue(kctx->csf.kcpu_queues.wq); + mutex_destroy(&kctx->csf.kcpu_queues.lock); +} + +int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx, + struct kbase_ioctl_kcpu_queue_delete *del) +{ + return delete_queue(kctx, (u32)del->id); +} + +int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, + struct kbase_ioctl_kcpu_queue_new *newq) +{ + struct kbase_kcpu_command_queue *queue; + int idx; + int ret = 0; + + /* The queue id is of u8 type and we use the index of the kcpu_queues + * array as an id, so the number of elements in the array can't be + * more than 256. + */ + BUILD_BUG_ON(KBASEP_MAX_KCPU_QUEUES > 256); + + mutex_lock(&kctx->csf.kcpu_queues.lock); + + idx = find_first_zero_bit(kctx->csf.kcpu_queues.in_use, + KBASEP_MAX_KCPU_QUEUES); + if (idx >= (int)KBASEP_MAX_KCPU_QUEUES) { + ret = -ENOMEM; + goto out; + } + + if (WARN_ON(kctx->csf.kcpu_queues.array[idx])) { + ret = -EINVAL; + goto out; + } + + queue = kzalloc(sizeof(*queue), GFP_KERNEL); + + if (!queue) { + ret = -ENOMEM; + goto out; + } + + bitmap_set(kctx->csf.kcpu_queues.in_use, idx, 1); + kctx->csf.kcpu_queues.array[idx] = queue; + queue->kctx = kctx; + queue->start_offset = 0; + queue->num_pending_cmds = 0; +#ifdef CONFIG_SYNC_FILE + queue->fence_context = dma_fence_context_alloc(1); + queue->fence_seqno = 0; + queue->fence_wait_processed = false; +#endif + queue->enqueue_failed = false; + queue->command_started = false; + INIT_LIST_HEAD(&queue->jit_blocked); + queue->has_error = false; + INIT_WORK(&queue->work, kcpu_queue_process_worker); + + newq->id = idx; + + /* Fire the tracepoint with the mutex held to enforce correct ordering + * with the summary stream. + */ + KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( + kctx->kbdev, queue, kctx->id, queue->num_pending_cmds); +out: + mutex_unlock(&kctx->csf.kcpu_queues.lock); + + return ret; +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h new file mode 100755 index 000000000000..45c76af04c0f --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h @@ -0,0 +1,305 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_KCPU_H_ +#define _KBASE_CSF_KCPU_H_ + +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) +#include +#else +#include +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ + +/* The maximum number of KCPU commands in flight, enqueueing more commands + * than this value shall block. + */ +#define KBASEP_KCPU_QUEUE_SIZE ((size_t)256) + +/** + * struct kbase_kcpu_command_import_info - Structure which holds information + * about the buffer to be imported + * + * @gpu_va: Address of the buffer to be imported. + */ +struct kbase_kcpu_command_import_info { + u64 gpu_va; +}; + +/** + * struct kbase_kcpu_command_fence_info - Structure which holds information + * about the fence object enqueued in the kcpu command queue + * + * @fence_cb: + * @fence: + * @kcpu_queue: + */ +struct kbase_kcpu_command_fence_info { +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) + struct fence_cb fence_cb; + struct fence *fence; +#else + struct dma_fence_cb fence_cb; + struct dma_fence *fence; +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ + struct kbase_kcpu_command_queue *kcpu_queue; +}; + +/** + * struct kbase_kcpu_command_cqs_set_info - Structure which holds information + * about CQS objects for the kcpu CQS set command + * + * @objs: Array of structures which define CQS objects to be used by + * the kcpu command. + * @nr_objs: Number of CQS objects in the array. + * @propagate_flags: Bit-pattern for the CQSs in the array that are set + * to propagate queue error-state to the flagged CQSs. + */ +struct kbase_kcpu_command_cqs_set_info { + struct base_cqs_set *objs; + unsigned int nr_objs; + u32 propagate_flags; +}; + +/** + * struct kbase_kcpu_command_cqs_wait_info - Structure which holds information + * about CQS objects for the kcpu CQS wait command + * + * @objs: Array of structures which define CQS objects to be used by + * the kcpu command. + * @signaled: Bit array used to report the status of the CQS wait objects. + * 1 is signaled, 0 otherwise. + * @nr_objs: Number of CQS objects in the array. + * @inherit_err_flags: Bit-pattern for the CQSs in the array who's error field + * to be served as the source for importing into the + * queue's error-state. + */ +struct kbase_kcpu_command_cqs_wait_info { + struct base_cqs_wait *objs; + unsigned long *signaled; + unsigned int nr_objs; + u32 inherit_err_flags; +}; + +/** + * struct kbase_kcpu_command_jit_alloc_info - Structure which holds information + * needed for the kcpu command for jit allocations + * + * @node Used to keep track of all JIT free/alloc commands in submission + * order. This must be located in the front of this struct to + * match that of kbase_kcpu_command_jit_free_info. + * @info: Array of objects of the struct base_jit_alloc_info type which + * specify jit allocations to be made by the kcpu command. + * @count: Number of jit alloc objects in the array. + * @blocked: Whether this allocation has been put into the pending list to + * be retried later. + */ +struct kbase_kcpu_command_jit_alloc_info { + struct list_head node; + struct base_jit_alloc_info *info; + u8 count; + bool blocked; +}; + +/** + * struct kbase_kcpu_command_jit_free_info - Structure which holds information + * needed for the kcpu jit free command + * + * @node: Used to keep track of all JIT free/alloc commands in submission + * order. This must be located in the front of this struct to + * match that of kbase_kcpu_command_jit_alloc_info. + * @ids: Array of identifiers of jit allocations which are to be freed + * by the kcpu command. + * @count: Number of elements in the array. + */ +struct kbase_kcpu_command_jit_free_info { + struct list_head node; + u8 *ids; + u8 count; +}; + +/** + * struct kbase_suspend_copy_buffer - information about the suspend buffer + * to be copied. + * + * @size: size of the suspend buffer in bytes. + * @pages: pointer to an array of pointers to the pages which contain + * the user buffer. + * @nr_pages: number of pages. + * @offset: offset into the pages + */ +struct kbase_suspend_copy_buffer { + size_t size; + struct page **pages; + int nr_pages; + size_t offset; +}; + +/** + * struct base_kcpu_command_group_suspend - structure which contains + * suspend buffer data captured for a suspended queue group. + * + * @sus_buf: Pointer to the structure which contains details of the + * user buffer and its kernel pinned pages. + * @group_handle: Handle to the mapping of command stream group. + */ +struct kbase_kcpu_command_group_suspend_info { + struct kbase_suspend_copy_buffer *sus_buf; + u8 group_handle; +}; + +/** + * struct kbase_cpu_command - Command which is to be part of the kernel + * command queue + * + * @type: Type of the command. + * @enqueue_ts: Denotes the relative time of enqueueing, a smaller value + * indicates that it has been enqueued earlier. + * @info: Structure which holds information about the command + * dependent on the command type. + */ +struct kbase_kcpu_command { + enum base_kcpu_command_type type; + u64 enqueue_ts; + union { + struct kbase_kcpu_command_fence_info fence; + struct kbase_kcpu_command_cqs_wait_info cqs_wait; + struct kbase_kcpu_command_cqs_set_info cqs_set; + struct kbase_kcpu_command_import_info import; + struct kbase_kcpu_command_jit_alloc_info jit_alloc; + struct kbase_kcpu_command_jit_free_info jit_free; + struct kbase_kcpu_command_group_suspend_info suspend_buf_copy; + } info; +}; + +/** + * struct kbase_kcpu_command_queue - a command queue executed by the kernel + * + * @kctx: The context to which this command queue belongs. + * @commands: Array of commands which have been successfully + * enqueued to this command queue. + * @work: struct work_struct which contains a pointer to + * the function which handles processing of kcpu + * commands enqueued into a kcpu command queue; + * part of kernel API for processing workqueues + * @start_offset: Index of the command to be executed next + * @num_pending_cmds: The number of commands enqueued but not yet + * executed or pending + * @cqs_wait_count: Tracks the number of CQS wait commands enqueued + * @fence_context: The dma-buf fence context number for this kcpu + * queue. A unique context number is allocated for + * each kcpu queue. + * @fence_seqno: The dma-buf fence sequence number for the fence + * that is returned on the enqueue of fence signal + * command. This is increased every time the + * fence signal command is queued. + * @fence_wait_processed: Used to avoid reprocessing of the fence wait + * command which has blocked the processing of + * commands that follow it. + * @enqueue_failed: Indicates that no space has become available in + * the buffer since an enqueue operation failed + * because of insufficient free space. + * @command_started: Indicates that the command at the front of the + * queue has been started in a previous queue + * process, but was not completed due to some + * unmet dependencies. Ensures that instrumentation + * of the execution start of these commands is only + * fired exactly once. + * @has_error: Indicates that the kcpu queue is in error mode + * or without errors since last cleaned. + * @jit_blocked: Used to keep track of command queues blocked + * by a pending JIT allocation command. + */ +struct kbase_kcpu_command_queue { + struct kbase_context *kctx; + struct kbase_kcpu_command commands[KBASEP_KCPU_QUEUE_SIZE]; + struct work_struct work; + u8 start_offset; + u16 num_pending_cmds; + u32 cqs_wait_count; + u64 fence_context; + unsigned int fence_seqno; + bool fence_wait_processed; + bool enqueue_failed; + bool command_started; + struct list_head jit_blocked; + bool has_error; +}; + +/** + * kbase_csf_kcpu_queue_new - Create new KCPU command queue. + * + * @kctx: Pointer to the kbase context within which the KCPU command + * queue will be created. + * @newq: Pointer to the structure which contains information about + * the new KCPU command queue to be created. + */ +int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, + struct kbase_ioctl_kcpu_queue_new *newq); + +/** + * kbase_csf_kcpu_queue_delete - Delete KCPU command queue. + * + * Return: 0 if successful, -EINVAL if the queue ID is invalid. + * + * @kctx: Pointer to the kbase context from which the KCPU command + * queue is to be deleted. + * @del: Pointer to the structure which specifies the KCPU command + * queue to be deleted. + */ +int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx, + struct kbase_ioctl_kcpu_queue_delete *del); + +/** + * kbase_csf_kcpu_queue_enqueue - Enqueue a KCPU command into a KCPU command + * queue. + * + * @kctx: Pointer to the kbase context within which the KCPU command + * is to be enqueued into the KCPU command queue. + * @enq: Pointer to the structure which specifies the KCPU command + * as well as the KCPU command queue into which the command + * is to be enqueued. + */ +int kbase_csf_kcpu_queue_enqueue(struct kbase_context *kctx, + struct kbase_ioctl_kcpu_queue_enqueue *enq); + +/** + * kbase_csf_kcpu_queue_context_init - Initialize the kernel CPU queues context + * for a GPU address space + * + * @kctx: Pointer to the kbase context being initialized. + * + * Return: 0 if successful or a negative error code on failure. + */ +int kbase_csf_kcpu_queue_context_init(struct kbase_context *kctx); + +/** + * kbase_csf_kcpu_queue_context_term - Terminate the kernel CPU queues context + * for a GPU address space + * + * This function deletes any kernel CPU queues that weren't deleted before + * context termination. + * + * @kctx: Pointer to the kbase context being terminated. + */ +void kbase_csf_kcpu_queue_context_term(struct kbase_context *kctx); + +#endif /* _KBASE_CSF_KCPU_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.c new file mode 100755 index 000000000000..55e3b64cbe71 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.c @@ -0,0 +1,199 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_csf_kcpu_debugfs.h" +#include +#include + +#ifdef CONFIG_SYNC_FILE +#include "mali_kbase_sync.h" +#endif + +#ifdef CONFIG_DEBUG_FS + +/** + * kbasep_csf_kcpu_debugfs_print_queue() - Print additional info for KCPU + * queues blocked on CQS wait commands. + * + * @file: The seq_file to print to + * @kctx: The context of the KCPU queue + * @waits: Pointer to the KCPU CQS wait command info + */ +static void kbasep_csf_kcpu_debugfs_print_cqs_waits(struct seq_file *file, + struct kbase_context *kctx, + struct kbase_kcpu_command_cqs_wait_info *waits) +{ + unsigned int i; + + for (i = 0; i < waits->nr_objs; i++) { + struct kbase_vmap_struct *mapping; + u32 val; + char const *msg; + u32 *const cpu_ptr = (u32 *)kbase_phy_alloc_mapping_get(kctx, + waits->objs[i].addr, &mapping); + + if (!cpu_ptr) + return; + + val = *cpu_ptr; + + kbase_phy_alloc_mapping_put(kctx, mapping); + + msg = (waits->inherit_err_flags && (1U << i)) ? "true" : + "false"; + seq_printf(file, " %llx(%u > %u, inherit_err: %s), ", + waits->objs[i].addr, val, waits->objs[i].val, msg); + } +} + +/** + * kbasep_csf_kcpu_debugfs_print_queue() - Print debug data for a KCPU queue + * + * @file: The seq_file to print to + * @kctx: The context of the KCPU queue + * @queue: Pointer to the KCPU queue + */ +static void kbasep_csf_kcpu_debugfs_print_queue(struct seq_file *file, + struct kbase_context *kctx, + struct kbase_kcpu_command_queue *queue) +{ + if (WARN_ON(!queue)) + return; + + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); + + seq_printf(file, "%16u, %11u, %7u, %13llu %8u", + queue->num_pending_cmds, queue->enqueue_failed, + queue->command_started ? 1 : 0, + queue->fence_context, queue->fence_seqno); + + if (queue->command_started) { + struct kbase_kcpu_command *cmd = + &queue->commands[queue->start_offset]; + switch (cmd->type) { +#ifdef CONFIG_SYNC_FILE + case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: + { + struct kbase_sync_fence_info info; + + kbase_sync_fence_info_get(cmd->info.fence.fence, &info); + seq_printf(file, ", Fence %p %s %s", + info.fence, info.name, + kbase_sync_status_string(info.status)); + break; + } +#endif + case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: + seq_puts(file, ", CQS "); + kbasep_csf_kcpu_debugfs_print_cqs_waits(file, kctx, + &cmd->info.cqs_wait); + break; + default: + seq_puts(file, ", U, Unknown blocking command"); + break; + } + } + + seq_puts(file, "\n"); +} + +/** + * kbasep_csf_kcpu_debugfs_show() - Print the KCPU queues debug information + * + * @file: The seq_file for printing to + * @data: The debugfs dentry private data, a pointer to kbase_context + * + * Return: Negative error code or 0 on success. + */ +static int kbasep_csf_kcpu_debugfs_show(struct seq_file *file, void *data) +{ + struct kbase_context *kctx = file->private; + unsigned long idx; + + seq_printf(file, "MALI_CSF_KCPU_DEBUGFS_VERSION: v%u\n", MALI_CSF_KCPU_DEBUGFS_VERSION); + seq_puts(file, "Queue Idx(err-mode), Pending Commands, Enqueue err, Blocked, Fence context & seqno, (Wait Type, Additional info)\n"); + mutex_lock(&kctx->csf.kcpu_queues.lock); + + idx = find_first_bit(kctx->csf.kcpu_queues.in_use, + KBASEP_MAX_KCPU_QUEUES); + + while (idx < KBASEP_MAX_KCPU_QUEUES) { + struct kbase_kcpu_command_queue *queue = + kctx->csf.kcpu_queues.array[idx]; + + seq_printf(file, "%9lu( %s ), ", idx, + queue->has_error ? "InErr" : "NoErr"); + kbasep_csf_kcpu_debugfs_print_queue(file, kctx, + kctx->csf.kcpu_queues.array[idx]); + + idx = find_next_bit(kctx->csf.kcpu_queues.in_use, + KBASEP_MAX_KCPU_QUEUES, idx + 1); + } + + mutex_unlock(&kctx->csf.kcpu_queues.lock); + return 0; +} + +static int kbasep_csf_kcpu_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_csf_kcpu_debugfs_show, in->i_private); +} + +static const struct file_operations kbasep_csf_kcpu_debugfs_fops = { + .open = kbasep_csf_kcpu_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void kbase_csf_kcpu_debugfs_init(struct kbase_context *kctx) +{ + struct dentry *file; +#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) + const mode_t mode = 0444; +#else + const mode_t mode = 0400; +#endif + + if (WARN_ON(!kctx || IS_ERR_OR_NULL(kctx->kctx_dentry))) + return; + + file = debugfs_create_file("kcpu_queues", mode, kctx->kctx_dentry, + kctx, &kbasep_csf_kcpu_debugfs_fops); + + if (IS_ERR_OR_NULL(file)) { + dev_warn(kctx->kbdev->dev, + "Unable to create KCPU debugfs entry"); + } +} + + +#else +/* + * Stub functions for when debugfs is disabled + */ +void kbase_csf_kcpu_debugfs_init(struct kbase_context *kctx) +{ +} + +#endif /* CONFIG_DEBUG_FS */ + diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.h new file mode 100755 index 000000000000..359fe2cb0168 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_debugfs.h @@ -0,0 +1,38 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_KCPU_DEBUGFS_H_ +#define _KBASE_CSF_KCPU_DEBUGFS_H_ + +/* Forward declaration */ +struct kbase_context; + +#define MALI_CSF_KCPU_DEBUGFS_VERSION 0 + +/** + * kbase_csf_kcpu_debugfs_init() - Create a debugfs entry for KCPU queues + * + * @kctx: The kbase_context for which to create the debugfs entry + */ +void kbase_csf_kcpu_debugfs_init(struct kbase_context *kctx); + +#endif /* _KBASE_CSF_KCPU_DEBUGFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c new file mode 100755 index 000000000000..987cbc2fc201 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c @@ -0,0 +1,120 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_csf_protected_memory.h" +#include + +#ifdef CONFIG_OF +#include +#endif + +int kbase_csf_protected_memory_init(struct kbase_device *const kbdev) +{ + int err = 0; + +#if CONFIG_OF + struct device_node *pma_node = of_parse_phandle(kbdev->dev->of_node, + "protected-memory-allocator", 0); + if (!pma_node) { + dev_info(kbdev->dev, "Protected memory allocator not available\n"); + } else { + struct platform_device *const pdev = + of_find_device_by_node(pma_node); + + kbdev->csf.pma_dev = NULL; + if (!pdev) { + dev_err(kbdev->dev, "Platform device for Protected memory allocator not found\n"); + } else { + kbdev->csf.pma_dev = platform_get_drvdata(pdev); + if (!kbdev->csf.pma_dev) { + dev_info(kbdev->dev, "Protected memory allocator is not ready\n"); + err = -EPROBE_DEFER; + } else if (!try_module_get(kbdev->csf.pma_dev->owner)) { + dev_err(kbdev->dev, "Failed to get Protected memory allocator module\n"); + err = -ENODEV; + } else { + dev_info(kbdev->dev, "Protected memory allocator successfully loaded\n"); + } + } + of_node_put(pma_node); + } +#endif + + return err; +} + +void kbase_csf_protected_memory_term(struct kbase_device *const kbdev) +{ + if (kbdev->csf.pma_dev) + module_put(kbdev->csf.pma_dev->owner); +} + +struct protected_memory_allocation ** + kbase_csf_protected_memory_alloc( + struct kbase_device *const kbdev, + struct tagged_addr *phys, + size_t num_pages) +{ + size_t i; + struct protected_memory_allocator_device *pma_dev = + kbdev->csf.pma_dev; + struct protected_memory_allocation **pma = + kmalloc_array(num_pages, sizeof(*pma), GFP_KERNEL); + + if (WARN_ON(!pma_dev) || WARN_ON(!phys) || !pma) + return NULL; + + for (i = 0; i < num_pages; i++) { + pma[i] = pma_dev->ops.pma_alloc_page(pma_dev, + KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER); + if (!pma[i]) + break; + + phys[i] = as_tagged(pma_dev->ops.pma_get_phys_addr(pma_dev, + pma[i])); + } + + if (i != num_pages) { + kbase_csf_protected_memory_free(kbdev, pma, i); + return NULL; + } + + return pma; +} + +void kbase_csf_protected_memory_free( + struct kbase_device *const kbdev, + struct protected_memory_allocation **pma, + size_t num_pages) +{ + size_t i; + struct protected_memory_allocator_device *pma_dev = + kbdev->csf.pma_dev; + + if (WARN_ON(!pma_dev) || WARN_ON(!pma)) + return; + + for (i = 0; i < num_pages; i++) + pma_dev->ops.pma_free_page(pma_dev, pma[i]); + + kfree(pma); +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.h new file mode 100755 index 000000000000..2b459911d834 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.h @@ -0,0 +1,72 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_PROTECTED_MEMORY_H_ +#define _KBASE_CSF_PROTECTED_MEMORY_H_ + +#include "mali_kbase.h" +/** + * kbase_csf_protected_memory_init - Initilaise protected memory allocator. + * + * @kbdev: Device pointer. + * + * Return: 0 if success, or an error code on failure. + */ +int kbase_csf_protected_memory_init(struct kbase_device *const kbdev); + +/** + * kbase_csf_protected_memory_term - Terminate prtotected memory allocator. + * + * @kbdev: Device pointer. + */ +void kbase_csf_protected_memory_term(struct kbase_device *const kbdev); + +/** + * kbase_csf_protected_memory_alloc - Allocate protected memory pages. + * + * @kbdev: Device pointer. + * @phys: Array of physical addresses to be filled in by the protected + * memory allocator. + * @num_pages: Number of pages requested to be allocated. + * + * Return: Pointer to an array of protected memory allocations on success, + * or NULL on failure. + */ +struct protected_memory_allocation ** + kbase_csf_protected_memory_alloc( + struct kbase_device *const kbdev, + struct tagged_addr *phys, + size_t num_pages); + +/** + * kbase_csf_protected_memory_free - Free the allocated + * protected memory pages + * + * @kbdev: Device pointer. + * @pma: Array of pointer to protected memory allocations. + * @num_pages: Number of pages to be freed. + */ +void kbase_csf_protected_memory_free( + struct kbase_device *const kbdev, + struct protected_memory_allocation **pma, + size_t num_pages); +#endif diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c new file mode 100755 index 000000000000..f1a318d26f43 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c @@ -0,0 +1,355 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Waiting timeout for GPU reset to complete */ +#define GPU_RESET_TIMEOUT_MS (5000) /* 5 seconds */ +#define DUMP_DWORDS_PER_LINE (4) +/* 16 characters needed for a 8 byte value in hex & 1 character for space */ +#define DUMP_HEX_CHARS_PER_DWORD ((2 * 8) + 1) +#define DUMP_HEX_CHARS_PER_LINE \ + (DUMP_DWORDS_PER_LINE * DUMP_HEX_CHARS_PER_DWORD) + +static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) +{ + kbase_io_history_dump(kbdev); + + dev_err(kbdev->dev, "Register state:"); + dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x MCU_STATUS=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS))); + dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)), + kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS))); + dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)), + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK)), + kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK))); + dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1))); + dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x TILER_CONFIG=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG)), + kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG))); +} + +static void kbase_csf_dump_firmware_trace_buffer(struct kbase_device *kbdev) +{ + u8 *buf, *line_str; + unsigned int read_size; + struct firmware_trace_buffer *tb = + kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); + + if (tb == NULL) { + dev_dbg(kbdev->dev, "Can't get the trace buffer, firmware trace dump skipped"); + return; + } + + buf = kmalloc(PAGE_SIZE + DUMP_HEX_CHARS_PER_LINE + 1, GFP_KERNEL); + if (buf == NULL) { + dev_err(kbdev->dev, "Short of memory, firmware trace dump skipped"); + return; + } + line_str = &buf[PAGE_SIZE]; + + dev_err(kbdev->dev, "Firmware trace buffer dump:"); + while ((read_size = kbase_csf_firmware_trace_buffer_read_data(tb, buf, + PAGE_SIZE))) { + u64 *ptr = (u64 *)buf; + u32 num_dwords; + + for (num_dwords = read_size / sizeof(u64); + num_dwords >= DUMP_DWORDS_PER_LINE; + num_dwords -= DUMP_DWORDS_PER_LINE) { + dev_err(kbdev->dev, "%016llx %016llx %016llx %016llx", + ptr[0], ptr[1], ptr[2], ptr[3]); + ptr += DUMP_DWORDS_PER_LINE; + } + + if (num_dwords) { + int pos = 0; + + while (num_dwords--) { + pos += snprintf(line_str + pos, + DUMP_HEX_CHARS_PER_DWORD + 1, + "%016llx ", ptr[0]); + ptr++; + } + + dev_err(kbdev->dev, "%s", line_str); + } + } + + kfree(buf); +} + +static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, + bool firmware_inited) +{ + unsigned long flags; + bool silent = false; + int err; + + if (atomic_read(&kbdev->csf.reset.state) == KBASE_CSF_RESET_GPU_SILENT) + silent = true; + + WARN_ON(kbdev->irq_reset_flush); + + /* Reset the scheduler state before disabling the interrupts as suspend of active + * CSG slots would also be done as a part of reset. + */ + if (likely(firmware_inited)) + kbase_csf_scheduler_reset(kbdev); + cancel_work_sync(&kbdev->csf.firmware_reload_work); + + /* Disable GPU hardware counters. + * This call will block until counters are disabled. + */ + kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + spin_lock(&kbdev->mmu_mask_change); + kbase_pm_reset_start_locked(kbdev); + + /* We're about to flush out the IRQs and their bottom halves */ + kbdev->irq_reset_flush = true; + + /* Disable IRQ to avoid IRQ handlers to kick in after releasing the + * spinlock; this also clears any outstanding interrupts + */ + kbase_pm_disable_interrupts_nolock(kbdev); + + spin_unlock(&kbdev->mmu_mask_change); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Ensure that any IRQ handlers have finished + * Must be done without any locks IRQ handlers will take. + */ + kbase_synchronize_irqs(kbdev); + + /* Flush out any in-flight work items */ + kbase_flush_mmu_wqs(kbdev); + + /* The flush has completed so reset the active indicator */ + kbdev->irq_reset_flush = false; + + mutex_lock(&kbdev->pm.lock); + if (!silent) + dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", + RESET_TIMEOUT); + + /* Output the state of some interesting registers to help in the + * debugging of GPU resets, and dump the firmware trace buffer + */ + if (!silent) { + kbase_csf_debug_dump_registers(kbdev); + if (likely(firmware_inited)) + kbase_csf_dump_firmware_trace_buffer(kbdev); + } + + /* Reset the GPU */ + err = kbase_pm_init_hw(kbdev, 0); + + mutex_unlock(&kbdev->pm.lock); + + if (WARN_ON(err)) + return err; + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_restore_all_as(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_pm_enable_interrupts(kbdev); + + mutex_lock(&kbdev->pm.lock); + kbase_pm_reset_complete(kbdev); + /* Synchronously wait for the reload of firmware to complete */ + err = kbase_pm_wait_for_desired_state(kbdev); + mutex_unlock(&kbdev->pm.lock); + + if (err) + return err; + + /* Re-enable GPU hardware counters */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (!silent) + dev_err(kbdev->dev, "Reset complete"); + + return 0; +} + +static void kbase_csf_reset_gpu_worker(struct work_struct *data) +{ + struct kbase_device *kbdev = container_of(data, struct kbase_device, + csf.reset.work); + bool firmware_inited; + unsigned long flags; + int err = 0; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + firmware_inited = kbdev->csf.firmware_inited; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (!kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + err = kbase_csf_reset_gpu_now(kbdev, firmware_inited); + kbase_pm_context_idle(kbdev); + } + + kbase_disjoint_state_down(kbdev); + + if (!err) { + atomic_set(&kbdev->csf.reset.state, + KBASE_CSF_RESET_GPU_NOT_PENDING); + if (likely(firmware_inited)) + kbase_csf_scheduler_enable_tick_timer(kbdev); + } else { + dev_err(kbdev->dev, "Reset failed to complete"); + atomic_set(&kbdev->csf.reset.state, + KBASE_CSF_RESET_GPU_FAILED); + } + + wake_up(&kbdev->csf.reset.wait); +} + +bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) +{ + if (atomic_cmpxchg(&kbdev->csf.reset.state, + KBASE_CSF_RESET_GPU_NOT_PENDING, + KBASE_CSF_RESET_GPU_HAPPENING) != + KBASE_CSF_RESET_GPU_NOT_PENDING) { + /* Some other thread is already resetting the GPU */ + return false; + } + + return true; +} +KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); + +bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + return kbase_prepare_to_reset_gpu(kbdev); +} + +int kbase_reset_gpu(struct kbase_device *kbdev) +{ + dev_err(kbdev->dev, "Preparing to soft-reset GPU\n"); + + kbase_disjoint_state_up(kbdev); + + queue_work(kbdev->csf.reset.workq, &kbdev->csf.reset.work); + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_reset_gpu); + +void kbase_reset_gpu_locked(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbase_reset_gpu(kbdev); +} + +int kbase_reset_gpu_silent(struct kbase_device *kbdev) +{ + if (atomic_cmpxchg(&kbdev->csf.reset.state, + KBASE_CSF_RESET_GPU_NOT_PENDING, + KBASE_CSF_RESET_GPU_SILENT) != + KBASE_CSF_RESET_GPU_NOT_PENDING) { + /* Some other thread is already resetting the GPU */ + return -EAGAIN; + } + + kbase_disjoint_state_up(kbdev); + + queue_work(kbdev->csf.reset.workq, &kbdev->csf.reset.work); + + return 0; +} + +bool kbase_reset_gpu_is_active(struct kbase_device *kbdev) +{ + if (atomic_read(&kbdev->csf.reset.state) == + KBASE_CSF_RESET_GPU_NOT_PENDING) + return false; + + return true; +} + +int kbase_reset_gpu_wait(struct kbase_device *kbdev) +{ + const long wait_timeout = + kbase_csf_timeout_in_jiffies(GPU_RESET_TIMEOUT_MS); + long remaining = wait_event_timeout(kbdev->csf.reset.wait, + (atomic_read(&kbdev->csf.reset.state) == + KBASE_CSF_RESET_GPU_NOT_PENDING) || + (atomic_read(&kbdev->csf.reset.state) == + KBASE_CSF_RESET_GPU_FAILED), + wait_timeout); + + if (!remaining) { + dev_warn(kbdev->dev, "Timed out waiting for the GPU reset to complete"); + return -ETIMEDOUT; + } else if (atomic_read(&kbdev->csf.reset.state) == + KBASE_CSF_RESET_GPU_FAILED) { + return -ENOMEM; + } + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_reset_gpu_wait); + +int kbase_reset_gpu_init(struct kbase_device *kbdev) +{ + kbdev->csf.reset.workq = alloc_workqueue("Mali reset workqueue", 0, 1); + if (kbdev->csf.reset.workq == NULL) + return -ENOMEM; + + INIT_WORK(&kbdev->csf.reset.work, kbase_csf_reset_gpu_worker); + + init_waitqueue_head(&kbdev->csf.reset.wait); + + return 0; +} + +void kbase_reset_gpu_term(struct kbase_device *kbdev) +{ + destroy_workqueue(kbdev->csf.reset.workq); +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c new file mode 100755 index 000000000000..a3017a7f25ba --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c @@ -0,0 +1,4135 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include "mali_kbase_config_defaults.h" +#include +#include +#include +#include +#include "mali_kbase_csf.h" +#include "../tl/mali_kbase_tracepoints.h" +#include "backend/gpu/mali_kbase_pm_internal.h" +#include +#include "mali_gpu_csf_registers.h" +#include + +/* Value to indicate that a queue group is not groups_to_schedule list */ +#define KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID (U32_MAX) + +/* Waiting timeout for status change acknowledgment, in milliseconds */ +#define CSF_STATE_WAIT_TIMEOUT_MS (800) /* Relaxed to 800ms from 100ms */ + +/* Waiting timeout for scheduler state change for descheduling a CSG */ +#define CSG_SCHED_STOP_TIMEOUT_MS (50) + +#define CSG_SUSPEND_ON_RESET_WAIT_TIMEOUT_MS DEFAULT_RESET_TIMEOUT_MS + +/* Maximum number of endpoints which may run tiler jobs. */ +#define CSG_TILER_MAX ((u8)1) + +/* Maximum dynamic CSG slot priority value */ +#define MAX_CSG_SLOT_PRIORITY ((u8)15) + +/* CSF scheduler time slice value */ +#define CSF_SCHEDULER_TIME_TICK_MS (100) /* 100 milliseconds */ +#define CSF_SCHEDULER_TIME_TICK_JIFFIES \ + msecs_to_jiffies(CSF_SCHEDULER_TIME_TICK_MS) + +/* + * CSF scheduler time threshold for converting "tock" requests into "tick" if + * they come too close to the end of a tick interval. This avoids scheduling + * twice in a row. + */ +#define CSF_SCHEDULER_TIME_TICK_THRESHOLD_MS \ + CSF_SCHEDULER_TIME_TICK_MS + +#define CSF_SCHEDULER_TIME_TICK_THRESHOLD_JIFFIES \ + msecs_to_jiffies(CSF_SCHEDULER_TIME_TICK_THRESHOLD_MS) + +/* Nanoseconds per millisecond */ +#define NS_PER_MS ((u64)1000 * 1000) + +/* + * CSF minimum time to reschedule for a new "tock" request. Bursts of "tock" + * requests are not serviced immediately, but shall wait for a minimum time in + * order to reduce load on the CSF scheduler thread. + */ +#define CSF_SCHEDULER_TIME_TOCK_JIFFIES 1 /* 1 jiffies-time */ + +/* Command stream suspended and is idle (empty ring buffer) */ +#define CS_IDLE_FLAG (1 << 0) + +/* Command stream suspended and is wait for a CQS condition */ +#define CS_WAIT_SYNC_FLAG (1 << 1) + +/* This is to avoid the immediate power down of GPU when then are no groups + * left for scheduling. GPUCORE-24250 would add the proper GPU idle detection + * logic. + */ +#define GPU_IDLE_POWEROFF_HYSTERESIS_DELAY msecs_to_jiffies((u32)10) + +static int scheduler_group_schedule(struct kbase_queue_group *group); +static void remove_group_from_idle_wait(struct kbase_queue_group *const group); +static +void insert_group_to_runnable(struct kbase_csf_scheduler *const scheduler, + struct kbase_queue_group *const group, + enum kbase_csf_group_state run_state); +static struct kbase_queue_group *scheduler_get_protm_enter_async_group( + struct kbase_device *const kbdev, + struct kbase_queue_group *const group); +static struct kbase_queue_group *get_tock_top_group( + struct kbase_csf_scheduler *const scheduler); +static void scheduler_enable_tick_timer_nolock(struct kbase_device *kbdev); +static int suspend_active_queue_groups(struct kbase_device *kbdev, + unsigned long *slot_mask); + +#define kctx_as_enabled(kctx) (!kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) + +static void release_doorbell(struct kbase_device *kbdev, int doorbell_nr) +{ + WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL); + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + clear_bit(doorbell_nr, kbdev->csf.scheduler.doorbell_inuse_bitmap); +} + +static int acquire_doorbell(struct kbase_device *kbdev) +{ + int doorbell_nr; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + doorbell_nr = find_first_zero_bit( + kbdev->csf.scheduler.doorbell_inuse_bitmap, + CSF_NUM_DOORBELL); + + if (doorbell_nr >= CSF_NUM_DOORBELL) + return KBASEP_USER_DB_NR_INVALID; + + set_bit(doorbell_nr, kbdev->csf.scheduler.doorbell_inuse_bitmap); + + return doorbell_nr; +} + +static void unassign_user_doorbell_from_group(struct kbase_device *kbdev, + struct kbase_queue_group *group) +{ + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + if (group->doorbell_nr != KBASEP_USER_DB_NR_INVALID) { + release_doorbell(kbdev, group->doorbell_nr); + group->doorbell_nr = KBASEP_USER_DB_NR_INVALID; + } +} + +static void unassign_user_doorbell_from_queue(struct kbase_device *kbdev, + struct kbase_queue *queue) +{ + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + mutex_lock(&kbdev->csf.reg_lock); + + if (queue->doorbell_nr != KBASEP_USER_DB_NR_INVALID) { + queue->doorbell_nr = KBASEP_USER_DB_NR_INVALID; + /* After this the dummy page would be mapped in */ + unmap_mapping_range(kbdev->csf.db_filp->f_inode->i_mapping, + queue->db_file_offset << PAGE_SHIFT, PAGE_SIZE, 1); + } + + mutex_unlock(&kbdev->csf.reg_lock); +} + +static void assign_user_doorbell_to_group(struct kbase_device *kbdev, + struct kbase_queue_group *group) +{ + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + if (group->doorbell_nr == KBASEP_USER_DB_NR_INVALID) + group->doorbell_nr = acquire_doorbell(kbdev); +} + +static void assign_user_doorbell_to_queue(struct kbase_device *kbdev, + struct kbase_queue *const queue) +{ + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + mutex_lock(&kbdev->csf.reg_lock); + + /* If bind operation for the queue hasn't completed yet, then the + * the command stream interface can't be programmed for the queue + * (even in stopped state) and so the doorbell also can't be assigned + * to it. + */ + if ((queue->bind_state == KBASE_CSF_QUEUE_BOUND) && + (queue->doorbell_nr == KBASEP_USER_DB_NR_INVALID)) { + WARN_ON(queue->group->doorbell_nr == KBASEP_USER_DB_NR_INVALID); + queue->doorbell_nr = queue->group->doorbell_nr; + + /* After this the real Hw doorbell page would be mapped in */ + unmap_mapping_range( + kbdev->csf.db_filp->f_inode->i_mapping, + queue->db_file_offset << PAGE_SHIFT, + PAGE_SIZE, 1); + } + + mutex_unlock(&kbdev->csf.reg_lock); +} + +static void scheduler_doorbell_init(struct kbase_device *kbdev) +{ + int doorbell_nr; + + bitmap_zero(kbdev->csf.scheduler.doorbell_inuse_bitmap, + CSF_NUM_DOORBELL); + + mutex_lock(&kbdev->csf.scheduler.lock); + /* Reserve doorbell 0 for use by kernel driver */ + doorbell_nr = acquire_doorbell(kbdev); + mutex_unlock(&kbdev->csf.scheduler.lock); + + WARN_ON(doorbell_nr != CSF_KERNEL_DOORBELL_NR); +} + +static u32 get_nr_active_csgs(struct kbase_device *kbdev) +{ + u32 nr_active_csgs; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + nr_active_csgs = bitmap_weight(kbdev->csf.scheduler.csg_inuse_bitmap, + kbdev->csf.global_iface.group_num); + + return nr_active_csgs; +} + +/** + * csgs_active - returns true if any of CSG slots are in use + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * Return: the interface is actively engaged flag. + */ +bool csgs_active(struct kbase_device *kbdev) +{ + u32 nr_active_csgs; + + mutex_lock(&kbdev->csf.scheduler.lock); + nr_active_csgs = get_nr_active_csgs(kbdev); + mutex_unlock(&kbdev->csf.scheduler.lock); + + /* Right now if any of the command stream group interfaces are in use + * then we need to assume that there is some work pending. + * In future when we have IDLE notifications from firmware implemented + * then we would have a better idea of the pending work. + */ + return (nr_active_csgs != 0); +} + +/** + * csg_slot_in_use - returns true if a queue group has been programmed on a + * given CSG slot. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @slot: Index/number of the CSG slot in question. + * + * Return: the interface is actively engaged flag. + * + * Note: Caller must hold the scheduler lock. + */ +static inline bool csg_slot_in_use(struct kbase_device *kbdev, int slot) +{ + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + return (kbdev->csf.scheduler.csg_slots[slot].resident_group != NULL); +} + +static bool queue_group_suspended_locked(struct kbase_queue_group *group) +{ + lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.lock); + + return (group->run_state == KBASE_CSF_GROUP_SUSPENDED || + group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE || + group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC); +} + +static bool queue_group_idle_locked(struct kbase_queue_group *group) +{ + lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.lock); + + return (group->run_state == KBASE_CSF_GROUP_IDLE || + group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE); +} + +static bool queue_group_scheduled(struct kbase_queue_group *group) +{ + return (group->run_state != KBASE_CSF_GROUP_INACTIVE && + group->run_state != KBASE_CSF_GROUP_TERMINATED && + group->run_state != KBASE_CSF_GROUP_FAULT_EVICTED); +} + +static bool queue_group_scheduled_locked(struct kbase_queue_group *group) +{ + lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.lock); + + return queue_group_scheduled(group); +} + +/** + * scheduler_timer_is_enabled_nolock() - Check if the scheduler wakes up + * automatically for periodic tasks. + * + * @kbdev: Pointer to the device + * + * This is a variant of kbase_csf_scheduler_timer_is_enabled() that assumes the + * CSF scheduler lock to already have been held. + * + * Return: true if the scheduler is configured to wake up periodically + */ +static bool scheduler_timer_is_enabled_nolock(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + return kbdev->csf.scheduler.timer_enabled; +} + +static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + + lockdep_assert_held(&scheduler->lock); + + if (scheduler->state == SCHED_SUSPENDED) { + dev_info(kbdev->dev, "Re-activating the Scheduler"); + kbase_csf_scheduler_pm_active(kbdev); + scheduler->state = SCHED_INACTIVE; + + if (kick) + scheduler_enable_tick_timer_nolock(kbdev); + } +} + +static void scheduler_suspend(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + + lockdep_assert_held(&scheduler->lock); + + if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) { + dev_dbg(kbdev->dev, "Suspending the Scheduler"); + kbase_csf_scheduler_pm_idle(kbdev); + scheduler->state = SCHED_SUSPENDED; + } +} + +/** + * update_idle_suspended_group_state() - Move the queue group to a non-idle + * suspended state. + * @group: Pointer to the queue group. + * + * This function is called to change the state of queue group to non-idle + * suspended state, if the group was suspended when all the queues bound to it + * became empty or when some queues got blocked on a sync wait & others became + * empty. The group is also moved to the runnbale list from idle wait list in + * the latter case. + * So the function gets called when a queue is kicked or sync wait condition + * gets satisfied. + */ +static void update_idle_suspended_group_state(struct kbase_queue_group *group) +{ + struct kbase_csf_scheduler *scheduler = + &group->kctx->kbdev->csf.scheduler; + + lockdep_assert_held(&scheduler->lock); + + if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) { + remove_group_from_idle_wait(group); + insert_group_to_runnable(scheduler, group, + KBASE_CSF_GROUP_SUSPENDED); + } else { + if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE) + group->run_state = KBASE_CSF_GROUP_SUSPENDED; + else + return; + } + + atomic_inc(&scheduler->non_idle_suspended_grps); +} + +int kbase_csf_scheduler_group_get_slot_locked(struct kbase_queue_group *group) +{ + struct kbase_csf_scheduler *scheduler = + &group->kctx->kbdev->csf.scheduler; + int slot_num = group->csg_nr; + + lockdep_assert_held(&scheduler->interrupt_lock); + + if (slot_num >= 0) { + if (WARN_ON(scheduler->csg_slots[slot_num].resident_group != + group)) + return -1; + } + + return slot_num; +} + +int kbase_csf_scheduler_group_get_slot(struct kbase_queue_group *group) +{ + struct kbase_csf_scheduler *scheduler = + &group->kctx->kbdev->csf.scheduler; + unsigned long flags; + int slot_num; + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + slot_num = kbase_csf_scheduler_group_get_slot_locked(group); + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); + + return slot_num; +} + +static bool kbasep_csf_scheduler_group_is_on_slot_locked( + struct kbase_queue_group *group) +{ + struct kbase_csf_scheduler *scheduler = + &group->kctx->kbdev->csf.scheduler; + int slot_num = group->csg_nr; + + lockdep_assert_held(&scheduler->lock); + + if (slot_num >= 0) { + if (!WARN_ON(scheduler->csg_slots[slot_num].resident_group != + group)) + return true; + } + + return false; +} + +bool kbase_csf_scheduler_group_events_enabled(struct kbase_device *kbdev, + struct kbase_queue_group *group) +{ + struct kbase_csf_scheduler *scheduler = + &group->kctx->kbdev->csf.scheduler; + int slot_num = group->csg_nr; + + lockdep_assert_held(&scheduler->interrupt_lock); + + if (WARN_ON(slot_num < 0)) + return false; + + return test_bit(slot_num, scheduler->csgs_events_enable_mask); +} + +struct kbase_queue_group *kbase_csf_scheduler_get_group_on_slot( + struct kbase_device *kbdev, int slot) +{ + lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); + + return kbdev->csf.scheduler.csg_slots[slot].resident_group; +} + +static int halt_stream_sync(struct kbase_queue *queue) +{ + struct kbase_queue_group *group = queue->group; + struct kbase_device *kbdev = queue->kctx->kbdev; + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + struct kbase_csf_cmd_stream_group_info *ginfo; + struct kbase_csf_cmd_stream_info *stream; + long remaining = + kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + + if (WARN_ON(!group) || + WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) + return -EINVAL; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + ginfo = &global_iface->groups[group->csg_nr]; + stream = &ginfo->streams[queue->csi_index]; + + if (CS_REQ_STATE_GET(kbase_csf_firmware_cs_input_read(stream, CS_REQ)) == + CS_REQ_STATE_START) { + + remaining = wait_event_timeout(kbdev->csf.event_wait, + (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output(stream, CS_ACK)) + == CS_ACK_STATE_START), remaining); + + if (!remaining) { + dev_warn(kbdev->dev, "Timed out waiting for queue to start on csi %d bound to group %d on slot %d", + queue->csi_index, group->handle, group->csg_nr); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + + return -ETIMEDOUT; + } + + remaining = + kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + } + + /* Set state to STOP */ + kbase_csf_firmware_cs_input_mask(stream, CS_REQ, CS_REQ_STATE_STOP, + CS_REQ_STATE_MASK); + + KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_STOP_REQUESTED, group, queue, 0u); + kbase_csf_ring_cs_kernel_doorbell(kbdev, queue); + + /* Timed wait */ + remaining = wait_event_timeout(kbdev->csf.event_wait, + (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output(stream, CS_ACK)) + == CS_ACK_STATE_STOP), remaining); + + if (!remaining) { + dev_warn(kbdev->dev, "Timed out waiting for queue to stop on csi %d bound to group %d on slot %d", + queue->csi_index, group->handle, group->csg_nr); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + } + return (remaining) ? 0 : -ETIMEDOUT; +} + +static bool can_halt_stream(struct kbase_device *kbdev, + struct kbase_queue_group *group) +{ + struct kbase_csf_csg_slot *const csg_slot = + kbdev->csf.scheduler.csg_slots; + unsigned long flags; + bool can_halt; + int slot; + + if (!queue_group_scheduled(group)) + return true; + + spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); + slot = kbase_csf_scheduler_group_get_slot_locked(group); + can_halt = (slot >= 0) && + (atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING); + spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, + flags); + + return can_halt; +} + +/** + * sched_halt_stream() - Stop a GPU queue when its queue group is not running + * on a CSG slot. + * @queue: Pointer to the GPU queue to stop. + * + * This function handles stopping gpu queues for groups that are either not on + * a command stream group slot or are on the slot but undergoing transition to + * resume or suspend states. + * It waits until the queue group is scheduled on a slot and starts running, + * which is needed as groups that were suspended may need to resume all queues + * that were enabled and running at the time of suspension. + * + * Return: 0 on success, or negative on failure. + */ +static int sched_halt_stream(struct kbase_queue *queue) +{ + struct kbase_queue_group *group = queue->group; + struct kbase_device *kbdev = queue->kctx->kbdev; + struct kbase_csf_scheduler *const scheduler = + &kbdev->csf.scheduler; + struct kbase_csf_csg_slot *const csg_slot = + kbdev->csf.scheduler.csg_slots; + bool retry_needed = false; + bool retried = false; + long remaining; + int slot; + int err = 0; + + if (WARN_ON(!group)) + return -EINVAL; + + lockdep_assert_held(&queue->kctx->csf.lock); + lockdep_assert_held(&scheduler->lock); + + slot = kbase_csf_scheduler_group_get_slot(group); + + if (slot >= 0) { + WARN_ON(atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING); + + if (atomic_read(&csg_slot[slot].state) == CSG_SLOT_READY2RUN) { + dev_dbg(kbdev->dev, "Stopping a queue on csi %d when Group-%d is in under transition to running state", + queue->csi_index, group->handle); + retry_needed = true; + } + } +retry: + /* First wait for the group to reach a stable state. IDLE state is + * an intermediate state that is only set by Scheduler at the start + * of a tick (prior to scanout) for groups that received idle + * notification, then later the idle group is moved to one of the + * suspended states or the runnable state. + */ + while (group->run_state == KBASE_CSF_GROUP_IDLE) { + mutex_unlock(&scheduler->lock); + remaining = wait_event_timeout(kbdev->csf.event_wait, + group->run_state != KBASE_CSF_GROUP_IDLE, + CSF_STATE_WAIT_TIMEOUT_MS); + mutex_lock(&scheduler->lock); + if (!remaining) { + dev_warn(kbdev->dev, + "Timed out waiting for state change of Group-%d when stopping a queue on csi %d", + group->handle, queue->csi_index); + } + } + + WARN_ON(group->run_state == KBASE_CSF_GROUP_IDLE); + /* Update the group state so that it can get scheduled soon */ + update_idle_suspended_group_state(group); + + mutex_unlock(&scheduler->lock); + + /* This function is called when the queue group is either not on a CSG + * slot or is on the slot but undergoing transition. + * + * To stop the queue, the function needs to wait either for the queue + * group to be assigned a CSG slot (and that slot has to reach the + * running state) or for the eviction of the queue group from the + * scheduler's list. + * + * In order to evaluate the latter condition, the function doesn't + * really need to lock the scheduler, as any update to the run_state + * of the queue group by sched_evict_group() would be visible due + * to implicit barriers provided by the kernel waitqueue macros. + * + * The group pointer cannot disappear meanwhile, as the high level + * CSF context is locked. Therefore, the scheduler would be + * the only one to update the run_state of the group. + */ + remaining = wait_event_timeout(kbdev->csf.event_wait, + can_halt_stream(kbdev, group), + kbase_csf_timeout_in_jiffies(20 * CSF_SCHEDULER_TIME_TICK_MS)); + + mutex_lock(&scheduler->lock); + + if (remaining && queue_group_scheduled_locked(group)) { + slot = kbase_csf_scheduler_group_get_slot(group); + + /* If the group is still on slot and slot is in running state + * then explicitly stop the command stream interface of the + * queue. Otherwise there are different cases to consider + * + * - If the queue group was already undergoing transition to + * resume/start state when this function was entered then it + * would not have disabled the command stream interface of the + * queue being stopped and the previous wait would have ended + * once the slot was in a running state with command stream + * interface still enabled. + * Now the group is going through another transition either + * to a suspend state or to a resume state (it could have + * been suspended before the scheduler lock was grabbed). + * In both scenarios need to wait again for the group to + * come on a slot and that slot to reach the running state, + * as that would guarantee that firmware will observe the + * command stream interface as disabled. + * + * - If the queue group was either off the slot or was + * undergoing transition to suspend state on entering this + * function, then the group would have been resumed with the + * queue's command stream interface in disabled state. + * So now if the group is undergoing another transition + * (after the resume) then just need to wait for the state + * bits in the ACK register of command stream interface to be + * set to STOP value. It is expected that firmware will + * process the stop/disable request of the command stream + * interface after resuming the group before it processes + * another state change request of the group. + */ + if ((slot >= 0) && + (atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING)) { + err = halt_stream_sync(queue); + } else if (retry_needed && !retried) { + retried = true; + goto retry; + } else if (slot >= 0) { + struct kbase_csf_global_iface *global_iface = + &kbdev->csf.global_iface; + struct kbase_csf_cmd_stream_group_info *ginfo = + &global_iface->groups[slot]; + struct kbase_csf_cmd_stream_info *stream = + &ginfo->streams[queue->csi_index]; + u32 cs_req = + kbase_csf_firmware_cs_input_read(stream, CS_REQ); + + if (!WARN_ON(CS_REQ_STATE_GET(cs_req) != + CS_REQ_STATE_STOP)) { + /* Timed wait */ + remaining = wait_event_timeout( + kbdev->csf.event_wait, + (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output(stream, CS_ACK)) + == CS_ACK_STATE_STOP), + CSF_STATE_WAIT_TIMEOUT_MS); + + if (!remaining) { + dev_warn(kbdev->dev, + "Timed out waiting for queue stop ack on csi %d bound to group %d on slot %d", + queue->csi_index, + group->handle, group->csg_nr); + err = -ETIMEDOUT; + } + } + } + } else if (!remaining) { + dev_warn(kbdev->dev, "Group-%d failed to get a slot for stopping the queue on csi %d", + group->handle, queue->csi_index); + err = -ETIMEDOUT; + } + + return err; +} + +static int wait_gpu_reset(struct kbase_device *kbdev) +{ + int ret = 0; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + while (kbase_reset_gpu_is_active(kbdev) && !ret) { + mutex_unlock(&kbdev->csf.scheduler.lock); + ret = kbase_reset_gpu_wait(kbdev); + mutex_lock(&kbdev->csf.scheduler.lock); + } + + return ret; +} + +int kbase_csf_scheduler_queue_stop(struct kbase_queue *queue) +{ + struct kbase_device *kbdev = queue->kctx->kbdev; + struct kbase_queue_group *group = queue->group; + bool const cs_enabled = queue->enabled; + int err = 0; + + if (WARN_ON(!group)) + return -EINVAL; + + lockdep_assert_held(&queue->kctx->csf.lock); + mutex_lock(&kbdev->csf.scheduler.lock); + + queue->enabled = false; + KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_STOP, group, queue, cs_enabled); + + wait_gpu_reset(kbdev); + + if (cs_enabled && queue_group_scheduled_locked(group)) { + struct kbase_csf_csg_slot *const csg_slot = + kbdev->csf.scheduler.csg_slots; + int slot = kbase_csf_scheduler_group_get_slot(group); + + /* Since the group needs to be resumed in order to stop the queue, + * check if GPU needs to be powered up. + */ + scheduler_wakeup(kbdev, true); + + if ((slot >= 0) && + (atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING)) + err = halt_stream_sync(queue); + else + err = sched_halt_stream(queue); + + unassign_user_doorbell_from_queue(kbdev, queue); + } + + mutex_unlock(&kbdev->csf.scheduler.lock); + return err; +} + +static void update_hw_active(struct kbase_queue *queue, bool active) +{ +#ifdef CONFIG_MALI_BIFROST_NO_MALI + if (queue && queue->enabled) { + u32 *output_addr = (u32 *)(queue->user_io_addr + PAGE_SIZE); + + output_addr[CS_ACTIVE / sizeof(u32)] = active; + } +#else + CSTD_UNUSED(queue); + CSTD_UNUSED(active); +#endif +} + +static void program_cs_extract_init(struct kbase_queue *queue) +{ + u64 *input_addr = (u64 *)queue->user_io_addr; + u64 *output_addr = (u64 *)(queue->user_io_addr + PAGE_SIZE); + + input_addr[CS_EXTRACT_INIT_LO / sizeof(u64)] = + output_addr[CS_EXTRACT_LO / sizeof(u64)]; +} + +static void program_cs(struct kbase_device *kbdev, + struct kbase_queue *queue) +{ + struct kbase_queue_group *group = queue->group; + struct kbase_csf_cmd_stream_group_info *ginfo; + struct kbase_csf_cmd_stream_info *stream; + u64 user_input; + u64 user_output; + + if (WARN_ON(!group)) + return; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) + return; + + ginfo = &kbdev->csf.global_iface.groups[group->csg_nr]; + + if (WARN_ON(queue->csi_index < 0) || + WARN_ON(queue->csi_index >= ginfo->stream_num)) + return; + + assign_user_doorbell_to_queue(kbdev, queue); + if (queue->doorbell_nr == KBASEP_USER_DB_NR_INVALID) + return; + + WARN_ON(queue->doorbell_nr != queue->group->doorbell_nr); + + if (queue->enabled && queue_group_suspended_locked(group)) + program_cs_extract_init(queue); + + stream = &ginfo->streams[queue->csi_index]; + + kbase_csf_firmware_cs_input(stream, CS_BASE_LO, + queue->base_addr & 0xFFFFFFFF); + kbase_csf_firmware_cs_input(stream, CS_BASE_HI, + queue->base_addr >> 32); + kbase_csf_firmware_cs_input(stream, CS_SIZE, + queue->size); + + user_input = (queue->reg->start_pfn << PAGE_SHIFT); + kbase_csf_firmware_cs_input(stream, CS_USER_INPUT_LO, + user_input & 0xFFFFFFFF); + kbase_csf_firmware_cs_input(stream, CS_USER_INPUT_HI, + user_input >> 32); + + user_output = ((queue->reg->start_pfn + 1) << PAGE_SHIFT); + kbase_csf_firmware_cs_input(stream, CS_USER_OUTPUT_LO, + user_output & 0xFFFFFFFF); + kbase_csf_firmware_cs_input(stream, CS_USER_OUTPUT_HI, + user_output >> 32); + + kbase_csf_firmware_cs_input(stream, CS_CONFIG, + (queue->doorbell_nr << 8) | (queue->priority & 0xF)); + + /* Enable all interrupts for now */ + kbase_csf_firmware_cs_input(stream, CS_ACK_IRQ_MASK, ~((u32)0)); + + /* + * Enable the CSG idle notification once the stream's ringbuffer + * becomes empty or the stream becomes sync_idle, waiting sync update + * or protected mode switch. + */ + kbase_csf_firmware_cs_input_mask(stream, CS_REQ, + CS_REQ_IDLE_EMPTY_MASK | CS_REQ_IDLE_SYNC_WAIT_MASK, + CS_REQ_IDLE_EMPTY_MASK | CS_REQ_IDLE_SYNC_WAIT_MASK); + + /* Set state to START/STOP */ + kbase_csf_firmware_cs_input_mask(stream, CS_REQ, + queue->enabled ? CS_REQ_STATE_START : CS_REQ_STATE_STOP, + CS_REQ_STATE_MASK); + + KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_START, group, queue, queue->enabled); + + kbase_csf_ring_cs_kernel_doorbell(kbdev, queue); + update_hw_active(queue, true); +} + +int kbase_csf_scheduler_queue_start(struct kbase_queue *queue) +{ + struct kbase_queue_group *group = queue->group; + struct kbase_device *kbdev = queue->kctx->kbdev; + bool const cs_enabled = queue->enabled; + int err = 0; + bool evicted = false; + + lockdep_assert_held(&queue->kctx->csf.lock); + + if (WARN_ON(!group || queue->bind_state != KBASE_CSF_QUEUE_BOUND)) + return -EINVAL; + + mutex_lock(&kbdev->csf.scheduler.lock); + + KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_START, group, queue, group->run_state); + err = wait_gpu_reset(kbdev); + + if (err) { + dev_warn(kbdev->dev, "Unsuccessful GPU reset detected when kicking queue (csi_index=%d) of group %d", + queue->csi_index, group->handle); + } else if (group->run_state == KBASE_CSF_GROUP_FAULT_EVICTED) { + err = -EIO; + evicted = true; + } else if ((group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) + && CS_STATUS_WAIT_SYNC_WAIT_GET(queue->status_wait)) { + dev_dbg(kbdev->dev, "blocked queue(csi_index=%d) of group %d was kicked", + queue->csi_index, group->handle); + } else { + err = scheduler_group_schedule(group); + + if (!err) { + queue->enabled = true; + if (kbasep_csf_scheduler_group_is_on_slot_locked(group)) { + if (cs_enabled) { + /* In normal situation, when a queue is + * already running, the queue update + * would be a doorbell kick on user + * side. However, if such a kick is + * shortly following a start or resume, + * the queue may actually in transition + * hence the said kick would enter the + * kernel as the hw_active flag is yet + * to be set. The sheduler needs to + * give a kick to the corresponding + * user door-bell on such a case. + */ + kbase_csf_ring_cs_user_doorbell(kbdev, queue); + } else + program_cs(kbdev, queue); + } + queue_delayed_work(system_long_wq, + &kbdev->csf.scheduler.ping_work, + msecs_to_jiffies(FIRMWARE_PING_INTERVAL_MS)); + } + } + + mutex_unlock(&kbdev->csf.scheduler.lock); + + if (evicted) + kbase_csf_term_descheduled_queue_group(group); + + return err; +} + +static enum kbase_csf_csg_slot_state update_csg_slot_status( + struct kbase_device *kbdev, s8 slot) +{ + struct kbase_csf_csg_slot *csg_slot = + &kbdev->csf.scheduler.csg_slots[slot]; + struct kbase_csf_cmd_stream_group_info *ginfo = + &kbdev->csf.global_iface.groups[slot]; + u32 state; + enum kbase_csf_csg_slot_state slot_state; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + state = CSG_ACK_STATE_GET(kbase_csf_firmware_csg_output(ginfo, + CSG_ACK)); + slot_state = atomic_read(&csg_slot->state); + + switch (slot_state) { + case CSG_SLOT_READY2RUN: + if ((state == CSG_ACK_STATE_START) || + (state == CSG_ACK_STATE_RESUME)) { + slot_state = CSG_SLOT_RUNNING; + atomic_set(&csg_slot->state, slot_state); + csg_slot->trigger_jiffies = jiffies; + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STARTED, csg_slot->resident_group, state); + dev_dbg(kbdev->dev, "Group %u running on slot %d\n", + csg_slot->resident_group->handle, slot); + } + break; + case CSG_SLOT_DOWN2STOP: + if ((state == CSG_ACK_STATE_SUSPEND) || + (state == CSG_ACK_STATE_TERMINATE)) { + slot_state = CSG_SLOT_STOPPED; + atomic_set(&csg_slot->state, slot_state); + csg_slot->trigger_jiffies = jiffies; + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOPPED, csg_slot->resident_group, state); + dev_dbg(kbdev->dev, "Group %u stopped on slot %d\n", + csg_slot->resident_group->handle, slot); + } + break; + case CSG_SLOT_DOWN2STOP_TIMEDOUT: + case CSG_SLOT_READY2RUN_TIMEDOUT: + case CSG_SLOT_READY: + case CSG_SLOT_RUNNING: + case CSG_SLOT_STOPPED: + break; + default: + dev_warn(kbdev->dev, "Unknown CSG slot state %d", slot_state); + break; + } + + return slot_state; +} + +static bool csg_slot_running(struct kbase_device *kbdev, s8 slot) +{ + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + return (update_csg_slot_status(kbdev, slot) == CSG_SLOT_RUNNING); +} + +static bool csg_slot_stopped_locked(struct kbase_device *kbdev, s8 slot) +{ + enum kbase_csf_csg_slot_state slot_state; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + slot_state = update_csg_slot_status(kbdev, slot); + + return (slot_state == CSG_SLOT_STOPPED || + slot_state == CSG_SLOT_READY); +} + +static bool csg_slot_stopped_raw(struct kbase_device *kbdev, s8 slot) +{ + struct kbase_csf_cmd_stream_group_info *ginfo = + &kbdev->csf.global_iface.groups[slot]; + u32 state; + + state = CSG_ACK_STATE_GET(kbase_csf_firmware_csg_output(ginfo, + CSG_ACK)); + + if (state == CSG_ACK_STATE_SUSPEND || state == CSG_ACK_STATE_TERMINATE) { + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOPPED, kbdev->csf.scheduler.csg_slots[slot].resident_group, state); + dev_dbg(kbdev->dev, "(raw status) slot %d stopped\n", slot); + return true; + } + + return false; +} + +static void halt_csg_slot(struct kbase_queue_group *group, bool suspend) +{ + struct kbase_device *kbdev = group->kctx->kbdev; + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + struct kbase_csf_csg_slot *csg_slot = + kbdev->csf.scheduler.csg_slots; + s8 slot; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) + return; + + slot = group->csg_nr; + + /* When in transition, wait for it to complete */ + if (atomic_read(&csg_slot[slot].state) == CSG_SLOT_READY2RUN) { + long remaining = + kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + + dev_dbg(kbdev->dev, "slot %d wait for up-running\n", slot); + remaining = wait_event_timeout(kbdev->csf.event_wait, + csg_slot_running(kbdev, slot), remaining); + if (!remaining) + dev_warn(kbdev->dev, + "slot %d timed out on up-running\n", slot); + } + + if (csg_slot_running(kbdev, slot)) { + unsigned long flags; + struct kbase_csf_cmd_stream_group_info *ginfo = + &global_iface->groups[slot]; + u32 halt_cmd = suspend ? CSG_REQ_STATE_SUSPEND : + CSG_REQ_STATE_TERMINATE; + + dev_dbg(kbdev->dev, "Halting(suspend=%d) group %d of context %d_%d on slot %d", + suspend, group->handle, group->kctx->tgid, group->kctx->id, slot); + + spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); + /* Set state to SUSPEND/TERMINATE */ + kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, halt_cmd, + CSG_REQ_STATE_MASK); + spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, + flags); + atomic_set(&csg_slot[slot].state, CSG_SLOT_DOWN2STOP); + csg_slot[slot].trigger_jiffies = jiffies; + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOP, group, halt_cmd); + + kbase_csf_ring_csg_doorbell(kbdev, slot); + } +} + +static void term_csg_slot(struct kbase_queue_group *group) +{ + halt_csg_slot(group, false); +} + +static void suspend_csg_slot(struct kbase_queue_group *group) +{ + halt_csg_slot(group, true); +} + +/** + * evaluate_sync_update() - Evaluate the sync wait condition the GPU command + * queue has been blocked on. + * + * @queue: Pointer to the GPU command queue + * + * Return: true if sync wait condition is satisfied. + */ +static bool evaluate_sync_update(struct kbase_queue *queue) +{ + enum kbase_csf_group_state run_state; + struct kbase_vmap_struct *mapping; + bool updated = false; + u32 *sync_ptr; + u32 sync_wait_cond; + + if (WARN_ON(!queue)) + return false; + + run_state = queue->group->run_state; + + if (WARN_ON((run_state != KBASE_CSF_GROUP_IDLE) && + (run_state != KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC))) + return false; + + lockdep_assert_held(&queue->kctx->kbdev->csf.scheduler.lock); + + sync_ptr = kbase_phy_alloc_mapping_get(queue->kctx, queue->sync_ptr, + &mapping); + + if (!sync_ptr) { + dev_dbg(queue->kctx->kbdev->dev, "sync memory VA 0x%016llX already freed", + queue->sync_ptr); + return false; + } + + sync_wait_cond = + CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GET(queue->status_wait); + + WARN_ON((sync_wait_cond != CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GT) && + (sync_wait_cond != CS_STATUS_WAIT_SYNC_WAIT_CONDITION_LE)); + + if (((sync_wait_cond == CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GT) && + (*sync_ptr > queue->sync_value)) || + ((sync_wait_cond == CS_STATUS_WAIT_SYNC_WAIT_CONDITION_LE) && + (*sync_ptr <= queue->sync_value))) { + /* The sync wait condition is satisfied so the group to which + * queue is bound can be re-scheduled. + */ + updated = true; + } else { + dev_dbg(queue->kctx->kbdev->dev, "sync memory not updated yet(%u)", + *sync_ptr); + } + + kbase_phy_alloc_mapping_put(queue->kctx, mapping); + + return updated; +} + +/** + * save_slot_cs() - Save the state for blocked GPU command queue. + * + * @ginfo: Pointer to the command stream group interface used by the group + * the queue is bound to. + * @queue: Pointer to the GPU command queue. + * + * This function will check if GPU command queue is blocked on a sync wait and + * evaluate the wait condition. If the wait condition isn't satisfied it would + * save the state needed to reevaluate the condition in future. + * The group to which queue is bound shall be in idle state. + * + * Return: true if the queue is blocked on a sync wait operation. + */ +static +bool save_slot_cs(struct kbase_csf_cmd_stream_group_info const *const ginfo, + struct kbase_queue *queue) +{ + struct kbase_csf_cmd_stream_info *const stream = + &ginfo->streams[queue->csi_index]; + u32 status = kbase_csf_firmware_cs_output(stream, CS_STATUS_WAIT); + bool is_waiting = false; + + WARN_ON(queue->group->run_state != KBASE_CSF_GROUP_IDLE); + + if (CS_STATUS_WAIT_SYNC_WAIT_GET(status)) { + queue->status_wait = status; + queue->sync_ptr = kbase_csf_firmware_cs_output(stream, + CS_STATUS_WAIT_SYNC_POINTER_LO); + queue->sync_ptr |= (u64)kbase_csf_firmware_cs_output(stream, + CS_STATUS_WAIT_SYNC_POINTER_HI) << 32; + queue->sync_value = kbase_csf_firmware_cs_output(stream, + CS_STATUS_WAIT_SYNC_VALUE); + + if (!evaluate_sync_update(queue)) { + is_waiting = true; + } else { + /* Sync object already got updated & met the condition + * thus it doesn't need to be reevaluated and so can + * clear the 'status_wait' here. + */ + queue->status_wait = 0; + } + } else { + /* Invalidate wait status info that would have been recorded if + * this queue was blocked when the group (in idle state) was + * suspended previously. After that the group could have been + * unblocked due to the kicking of another queue bound to it & + * so the wait status info would have stuck with this queue. + */ + queue->status_wait = 0; + } + + return is_waiting; +} + +/** + * Calculate how far in the future an event should be scheduled. + * + * The objective of this function is making sure that a minimum period of + * time is guaranteed between handling two consecutive events. + * + * This function guarantees a minimum period of time between two consecutive + * events: given the minimum period and the distance between the current time + * and the last event, the function returns the difference between the two. + * However, if more time than the minimum period has already elapsed + * since the last event, the function will return 0 to schedule work to handle + * the event with the lowest latency possible. + * + * @last_event: Timestamp of the last event, in jiffies. + * @time_now: Timestamp of the new event to handle, in jiffies. + * Must be successive to last_event. + * @period: Minimum period between two events, in jiffies. + * + * Return: Time to delay work to handle the current event, in jiffies + */ +static unsigned long get_schedule_delay(unsigned long last_event, + unsigned long time_now, + unsigned long period) +{ + const unsigned long t_distance = time_now - last_event; + const unsigned long delay_t = (t_distance < period) ? + (period - t_distance) : 0; + + return delay_t; +} + +static void schedule_in_cycle(struct kbase_queue_group *group, bool force) +{ + struct kbase_context *kctx = group->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + + lockdep_assert_held(&scheduler->lock); + + /* Only try to schedule work for this event if no requests are pending, + * otherwise the function will end up canceling previous work requests, + * and scheduler is configured to wake up periodically (or the schedule + * of work needs to be enforced in situation such as entering into + * protected mode). + */ + if ((likely(scheduler_timer_is_enabled_nolock(kbdev)) || force) && + !scheduler->tock_pending_request) { + const unsigned long delay = + get_schedule_delay(scheduler->last_schedule, jiffies, + CSF_SCHEDULER_TIME_TOCK_JIFFIES); + scheduler->tock_pending_request = true; + dev_dbg(kbdev->dev, "Kicking async for group %d\n", + group->handle); + mod_delayed_work(scheduler->wq, &scheduler->tock_work, delay); + } +} + +static +void insert_group_to_runnable(struct kbase_csf_scheduler *const scheduler, + struct kbase_queue_group *const group, + enum kbase_csf_group_state run_state) +{ + struct kbase_context *const kctx = group->kctx; + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&scheduler->lock); + + WARN_ON(group->run_state != KBASE_CSF_GROUP_INACTIVE); + + if (WARN_ON(group->priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT)) + return; + + group->run_state = run_state; + + if (run_state == KBASE_CSF_GROUP_RUNNABLE) + group->prepared_seq_num = KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID; + + list_add_tail(&group->link, + &kctx->csf.sched.runnable_groups[group->priority]); + kctx->csf.sched.num_runnable_grps++; + /* Add the kctx if not yet in runnable kctxs */ + if (kctx->csf.sched.num_runnable_grps == 1) { + /* First runnable csg, adds to the runnable_kctxs */ + INIT_LIST_HEAD(&kctx->csf.link); + list_add_tail(&kctx->csf.link, &scheduler->runnable_kctxs); + } + + scheduler->total_runnable_grps++; + + if (likely(scheduler_timer_is_enabled_nolock(kbdev)) && + (scheduler->total_runnable_grps == 1 || + scheduler->state == SCHED_SUSPENDED)) { + dev_dbg(kbdev->dev, "Kicking scheduler on first runnable group\n"); + /* Fire a scheduling to start the time-slice */ + mod_delayed_work(kbdev->csf.scheduler.wq, + &kbdev->csf.scheduler.tick_work, 0); + } else + schedule_in_cycle(group, false); + + /* Since a new group has become runnable, check if GPU needs to be + * powered up. + */ + scheduler_wakeup(kbdev, false); +} + +static +void remove_group_from_runnable(struct kbase_csf_scheduler *const scheduler, + struct kbase_queue_group *group, + enum kbase_csf_group_state run_state) +{ + struct kbase_context *kctx = group->kctx; + + lockdep_assert_held(&scheduler->lock); + + WARN_ON(!queue_group_scheduled_locked(group)); + + group->run_state = run_state; + list_del_init(&group->link); + + if (scheduler->top_grp == group) { + /* + * Note: this disables explicit rotation in the next scheduling + * cycle. However, removing the top_grp is the same as an + * implicit rotation (e.g. if we instead rotated the top_ctx + * and then remove top_grp) + * + * This implicit rotation is assumed by the scheduler rotate + * functions. + */ + scheduler->top_grp = NULL; + + /* + * Trigger a scheduling tock for a CSG containing protected + * content in case there has been any in order to minimise + * latency. + */ + group = scheduler_get_protm_enter_async_group(kctx->kbdev, + NULL); + if (group) + schedule_in_cycle(group, true); + } + + kctx->csf.sched.num_runnable_grps--; + if (kctx->csf.sched.num_runnable_grps == 0) { + /* drop the kctx */ + list_del_init(&kctx->csf.link); + if (scheduler->top_ctx == kctx) + scheduler->top_ctx = NULL; + } + + WARN_ON(scheduler->total_runnable_grps == 0); + scheduler->total_runnable_grps--; + if (!scheduler->total_runnable_grps && + scheduler->state != SCHED_SUSPENDED) { + dev_dbg(kctx->kbdev->dev, "Scheduler idle as no runnable groups"); + mod_delayed_work(system_wq, &scheduler->gpu_idle_work, + GPU_IDLE_POWEROFF_HYSTERESIS_DELAY); + } + KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, + scheduler->num_active_address_spaces | + (((u64)scheduler->total_runnable_grps) << 32)); +} + +static void insert_group_to_idle_wait(struct kbase_queue_group *const group) +{ + struct kbase_context *kctx = group->kctx; + + lockdep_assert_held(&kctx->kbdev->csf.scheduler.lock); + + WARN_ON(group->run_state != KBASE_CSF_GROUP_IDLE); + + list_add_tail(&group->link, &kctx->csf.sched.idle_wait_groups); + kctx->csf.sched.num_idle_wait_grps++; + group->run_state = KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC; + dev_dbg(kctx->kbdev->dev, + "Group-%d suspended on sync_wait, total wait_groups: %u\n", + group->handle, kctx->csf.sched.num_idle_wait_grps); +} + +static void remove_group_from_idle_wait(struct kbase_queue_group *const group) +{ + struct kbase_context *kctx = group->kctx; + + lockdep_assert_held(&kctx->kbdev->csf.scheduler.lock); + + WARN_ON(group->run_state != KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC); + + list_del_init(&group->link); + WARN_ON(kctx->csf.sched.num_idle_wait_grps == 0); + kctx->csf.sched.num_idle_wait_grps--; + group->run_state = KBASE_CSF_GROUP_INACTIVE; +} + +static void deschedule_idle_wait_group(struct kbase_csf_scheduler *scheduler, + struct kbase_queue_group *group) +{ + lockdep_assert_held(&scheduler->lock); + + if (WARN_ON(!group)) + return; + + remove_group_from_runnable(scheduler, group, KBASE_CSF_GROUP_IDLE); + insert_group_to_idle_wait(group); +} + +static bool confirm_cs_idle(struct kbase_queue *queue) +{ + u64 *input_addr = (u64 *)queue->user_io_addr; + u64 *output_addr = (u64 *)(queue->user_io_addr + PAGE_SIZE); + + return (input_addr[CS_INSERT_LO / sizeof(u64)] == + output_addr[CS_EXTRACT_LO / sizeof(u64)]); +} + +static void save_csg_slot(struct kbase_queue_group *group) +{ + struct kbase_device *kbdev = group->kctx->kbdev; + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + struct kbase_csf_cmd_stream_group_info *ginfo; + u32 state; + + lockdep_assert_held(&scheduler->lock); + + if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) + return; + + ginfo = &kbdev->csf.global_iface.groups[group->csg_nr]; + + state = + CSG_ACK_STATE_GET(kbase_csf_firmware_csg_output(ginfo, CSG_ACK)); + + if (!WARN_ON((state != CSG_ACK_STATE_SUSPEND) && + (state != CSG_ACK_STATE_TERMINATE))) { + int i; + +#ifdef CONFIG_MALI_BIFROST_NO_MALI + for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) + update_hw_active(group->bound_queues[i], false); +#endif + if (group->run_state == KBASE_CSF_GROUP_IDLE) { + bool sync_wait = false; + bool idle = true; + + /* Loop through all bound CSs & save their context */ + for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { + struct kbase_queue *const queue = + group->bound_queues[i]; + + if (queue && queue->enabled) { + if (save_slot_cs(ginfo, queue)) + sync_wait = true; + else if (idle) + idle = confirm_cs_idle(queue); + } + } + + /* Take the suspended group out of the runnable_groups + * list of the context and move it to the + * idle_wait_groups list. + */ + if (sync_wait && idle) + deschedule_idle_wait_group(scheduler, group); + else if (idle) { + group->run_state = + KBASE_CSF_GROUP_SUSPENDED_ON_IDLE; + dev_dbg(kbdev->dev, "Group-%d suspended: idle\n", + group->handle); + } else { + group->run_state = KBASE_CSF_GROUP_SUSPENDED; + atomic_inc(&scheduler->non_idle_suspended_grps); + } + } else { + group->run_state = KBASE_CSF_GROUP_SUSPENDED; + atomic_inc(&scheduler->non_idle_suspended_grps); + } + } +} + +/* Cleanup_csg_slot after it has been vacated, ready for next csg run. + * Return whether there is a kctx address fault associated with the group + * for which the clean-up is done. + */ +static bool cleanup_csg_slot(struct kbase_queue_group *group) +{ + struct kbase_context *kctx = group->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + struct kbase_csf_cmd_stream_group_info *ginfo; + s8 slot; + struct kbase_csf_csg_slot *csg_slot; + unsigned long flags; + u32 i; + bool as_fault = false; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) + return as_fault; + + slot = group->csg_nr; + csg_slot = &kbdev->csf.scheduler.csg_slots[slot]; + ginfo = &global_iface->groups[slot]; + + /* Now loop through all the bound CSs, and clean them via a stop */ + for (i = 0; i < ginfo->stream_num; i++) { + struct kbase_csf_cmd_stream_info *stream = &ginfo->streams[i]; + + if (group->bound_queues[i]) { + if (group->bound_queues[i]->enabled) { + kbase_csf_firmware_cs_input_mask(stream, + CS_REQ, CS_REQ_STATE_STOP, + CS_REQ_STATE_MASK); + } + + unassign_user_doorbell_from_queue(kbdev, + group->bound_queues[i]); + } + } + + unassign_user_doorbell_from_group(kbdev, group); + + /* The csg does not need cleanup other than drop its AS */ + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + as_fault = kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT); + kbase_ctx_sched_release_ctx(kctx); + if (unlikely(group->faulted)) + as_fault = true; + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); + + /* now marking the slot is vacant */ + spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); + kbdev->csf.scheduler.csg_slots[slot].resident_group = NULL; + group->csg_nr = KBASEP_CSG_NR_INVALID; + clear_bit(slot, kbdev->csf.scheduler.csg_slots_idle_mask); + set_bit(slot, kbdev->csf.scheduler.csgs_events_enable_mask); + clear_bit(slot, kbdev->csf.scheduler.csg_inuse_bitmap); + spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); + + csg_slot->trigger_jiffies = jiffies; + atomic_set(&csg_slot->state, CSG_SLOT_READY); + + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_CLEANED, group, slot); + dev_dbg(kbdev->dev, "Cleanup done for group %d on slot %d\n", + group->handle, slot); + + KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG(kbdev, + kbdev->gpu_props.props.raw_props.gpu_id, slot); + + return as_fault; +} + +static void update_csg_slot_priority(struct kbase_queue_group *group, u8 prio) +{ + struct kbase_device *kbdev = group->kctx->kbdev; + struct kbase_csf_csg_slot *csg_slot; + struct kbase_csf_cmd_stream_group_info *ginfo; + s8 slot; + u8 prev_prio; + u32 ep_cfg; + u32 csg_req; + unsigned long flags; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) + return; + + slot = group->csg_nr; + csg_slot = &kbdev->csf.scheduler.csg_slots[slot]; + ginfo = &kbdev->csf.global_iface.groups[slot]; + + WARN_ON(!((group->run_state == KBASE_CSF_GROUP_RUNNABLE) || + (group->run_state == KBASE_CSF_GROUP_IDLE))); + + group->run_state = KBASE_CSF_GROUP_RUNNABLE; + + if (csg_slot->priority == prio) + return; + + /* Read the csg_ep_cfg back for updating the priority field */ + ep_cfg = kbase_csf_firmware_csg_input_read(ginfo, CSG_EP_REQ); + prev_prio = CSG_EP_REQ_PRIORITY_GET(ep_cfg); + ep_cfg = CSG_EP_REQ_PRIORITY_SET(ep_cfg, prio); + kbase_csf_firmware_csg_input(ginfo, CSG_EP_REQ, ep_cfg); + + spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); + csg_req = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); + csg_req ^= CSG_REQ_EP_CFG; + kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, csg_req, + CSG_REQ_EP_CFG); + spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); + + csg_slot->priority = prio; + + dev_dbg(kbdev->dev, "Priority for group %d of context %d_%d on slot %d to be updated from %u to %u\n", + group->handle, group->kctx->tgid, group->kctx->id, slot, + prev_prio, prio); + + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_PRIO_UPDATE, group, prev_prio); + + kbase_csf_ring_csg_doorbell(kbdev, slot); + set_bit(slot, kbdev->csf.scheduler.csg_slots_prio_update); +} + +static void program_csg_slot(struct kbase_queue_group *group, s8 slot, + u8 prio) +{ + struct kbase_context *kctx = group->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + const u64 shader_core_mask = + kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER); + const u64 tiler_core_mask = + kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_TILER); + const u64 compute_mask = shader_core_mask & group->compute_mask; + const u64 fragment_mask = shader_core_mask & group->fragment_mask; + const u64 tiler_mask = tiler_core_mask & group->tiler_mask; + const u8 num_cores = kbdev->gpu_props.num_cores; + const u8 compute_max = min(num_cores, group->compute_max); + const u8 fragment_max = min(num_cores, group->fragment_max); + const u8 tiler_max = min(CSG_TILER_MAX, group->tiler_max); + struct kbase_csf_cmd_stream_group_info *ginfo; + u32 ep_cfg = 0; + u32 csg_req; + u32 state; + int i; + unsigned long flags; + const u64 normal_suspend_buf = + group->normal_suspend_buf.reg->start_pfn << PAGE_SHIFT; + struct kbase_csf_csg_slot *csg_slot = + &kbdev->csf.scheduler.csg_slots[slot]; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + if (WARN_ON(slot < 0) && + WARN_ON(slot >= global_iface->group_num)) + return; + + WARN_ON(atomic_read(&csg_slot->state) != CSG_SLOT_READY); + + ginfo = &global_iface->groups[slot]; + + /* Pick an available address space for this context */ + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_retain_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + if (kctx->as_nr == KBASEP_AS_NR_INVALID) { + dev_dbg(kbdev->dev, "Could not get a valid AS for group %d of context %d_%d on slot %d\n", + group->handle, kctx->tgid, kctx->id, slot); + return; + } + + spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); + set_bit(slot, kbdev->csf.scheduler.csg_inuse_bitmap); + kbdev->csf.scheduler.csg_slots[slot].resident_group = group; + group->csg_nr = slot; + spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); + + assign_user_doorbell_to_group(kbdev, group); + + /* Now loop through all the bound & kicked CSs, and program them */ + for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { + struct kbase_queue *queue = group->bound_queues[i]; + + if (queue) + program_cs(kbdev, queue); + } + + + /* Endpoint programming for CSG */ + kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_COMPUTE_LO, + compute_mask & U32_MAX); + kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_COMPUTE_HI, + compute_mask >> 32); + kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_FRAGMENT_LO, + fragment_mask & U32_MAX); + kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_FRAGMENT_HI, + fragment_mask >> 32); + kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_OTHER, + tiler_mask & U32_MAX); + + ep_cfg = CSG_EP_REQ_COMPUTE_EP_SET(ep_cfg, compute_max); + ep_cfg = CSG_EP_REQ_FRAGMENT_EP_SET(ep_cfg, fragment_max); + ep_cfg = CSG_EP_REQ_TILER_EP_SET(ep_cfg, tiler_max); + ep_cfg = CSG_EP_REQ_PRIORITY_SET(ep_cfg, prio); + kbase_csf_firmware_csg_input(ginfo, CSG_EP_REQ, ep_cfg); + + /* Program the address space number assigned to the context */ + kbase_csf_firmware_csg_input(ginfo, CSG_CONFIG, kctx->as_nr); + + kbase_csf_firmware_csg_input(ginfo, CSG_SUSPEND_BUF_LO, + normal_suspend_buf & U32_MAX); + kbase_csf_firmware_csg_input(ginfo, CSG_SUSPEND_BUF_HI, + normal_suspend_buf >> 32); + + if (group->protected_suspend_buf.reg) { + const u64 protm_suspend_buf = + group->protected_suspend_buf.reg->start_pfn << + PAGE_SHIFT; + kbase_csf_firmware_csg_input(ginfo, CSG_PROTM_SUSPEND_BUF_LO, + protm_suspend_buf & U32_MAX); + kbase_csf_firmware_csg_input(ginfo, CSG_PROTM_SUSPEND_BUF_HI, + protm_suspend_buf >> 32); + } + + /* Enable all interrupts for now */ + kbase_csf_firmware_csg_input(ginfo, CSG_ACK_IRQ_MASK, ~((u32)0)); + + spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); + csg_req = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); + csg_req ^= CSG_REQ_EP_CFG; + kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, csg_req, + CSG_REQ_EP_CFG); + + /* Set state to START/RESUME */ + if (queue_group_suspended_locked(group)) { + state = CSG_REQ_STATE_RESUME; + if (group->run_state == KBASE_CSF_GROUP_SUSPENDED) + atomic_dec( + &kbdev->csf.scheduler.non_idle_suspended_grps); + } else { + WARN_ON(group->run_state != KBASE_CSF_GROUP_RUNNABLE); + state = CSG_REQ_STATE_START; + } + + kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, + state, CSG_REQ_STATE_MASK); + spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); + + /* Update status before rings the door-bell, marking ready => run */ + atomic_set(&csg_slot->state, CSG_SLOT_READY2RUN); + csg_slot->trigger_jiffies = jiffies; + csg_slot->priority = prio; + + /* Trace the programming of the CSG on the slot */ + KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG(kbdev, + kbdev->gpu_props.props.raw_props.gpu_id, group->handle, slot); + + dev_dbg(kbdev->dev, "Starting group %d of context %d_%d on slot %d with priority %u\n", + group->handle, kctx->tgid, kctx->id, slot, prio); + + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_START, group, + (((u64)ep_cfg) << 32) | + ((((u32)kctx->as_nr) & 0xF) << 16) | + (state & (CSG_REQ_STATE_MASK >> CS_REQ_STATE_SHIFT))); + + kbase_csf_ring_csg_doorbell(kbdev, slot); +} + +static void remove_scheduled_group(struct kbase_device *kbdev, + struct kbase_queue_group *group) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + + lockdep_assert_held(&scheduler->lock); + + WARN_ON(group->prepared_seq_num == + KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID); + WARN_ON(list_empty(&group->link_to_schedule)); + + list_del_init(&group->link_to_schedule); + scheduler->ngrp_to_schedule--; + group->prepared_seq_num = KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID; + group->kctx->csf.sched.ngrp_to_schedule--; +} + +static void sched_evict_group(struct kbase_queue_group *group, bool fault) +{ + struct kbase_context *kctx = group->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + if (queue_group_scheduled_locked(group)) { + u32 i; + + if (group->run_state == KBASE_CSF_GROUP_SUSPENDED) + atomic_dec(&scheduler->non_idle_suspended_grps); + + for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { + if (group->bound_queues[i]) + group->bound_queues[i]->enabled = false; + } + + if (group->prepared_seq_num != + KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID) + remove_scheduled_group(kbdev, group); + + if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) + remove_group_from_idle_wait(group); + else { + remove_group_from_runnable(scheduler, group, + KBASE_CSF_GROUP_INACTIVE); + } + + WARN_ON(group->run_state != KBASE_CSF_GROUP_INACTIVE); + + if (fault) + group->run_state = KBASE_CSF_GROUP_FAULT_EVICTED; + + KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_EVICT_SCHED, group, + (((u64)scheduler->total_runnable_grps) << 32) | + ((u32)group->run_state)); + dev_dbg(kbdev->dev, "group %d exited scheduler, num_runnable_grps %d\n", + group->handle, scheduler->total_runnable_grps); + /* Notify a group has been evicted */ + wake_up_all(&kbdev->csf.event_wait); + } +} + +static int term_group_sync(struct kbase_queue_group *group) +{ + struct kbase_device *kbdev = group->kctx->kbdev; + long remaining = + kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + int err = 0; + + term_csg_slot(group); + + remaining = wait_event_timeout(kbdev->csf.event_wait, + csg_slot_stopped_locked(kbdev, group->csg_nr), remaining); + + if (!remaining) { + dev_warn(kbdev->dev, "term request timed out for group %d on slot %d", + group->handle, group->csg_nr); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + err = -ETIMEDOUT; + } + + return err; +} + +void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group) +{ + struct kbase_device *kbdev = group->kctx->kbdev; + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + long remaining = + kbase_csf_timeout_in_jiffies(CSG_SCHED_STOP_TIMEOUT_MS); + bool force = false; + + lockdep_assert_held(&group->kctx->csf.lock); + mutex_lock(&scheduler->lock); + + KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_DESCHEDULE, group, group->run_state); + while (queue_group_scheduled_locked(group)) { + u32 saved_state = scheduler->state; + bool reset = kbase_reset_gpu_is_active(kbdev); + + if (!kbasep_csf_scheduler_group_is_on_slot_locked(group)) { + sched_evict_group(group, false); + } else if (reset || saved_state == SCHED_INACTIVE || force) { + bool as_faulty; + + if (!reset) + term_group_sync(group); + /* Treat the csg been terminated */ + as_faulty = cleanup_csg_slot(group); + /* remove from the scheduler list */ + sched_evict_group(group, as_faulty); + } + + /* waiting scheduler state to change */ + if (queue_group_scheduled_locked(group)) { + mutex_unlock(&scheduler->lock); + remaining = wait_event_timeout( + kbdev->csf.event_wait, + saved_state != scheduler->state, + remaining); + if (!remaining) { + dev_warn(kbdev->dev, "Scheduler state change wait timed out for group %d on slot %d", + group->handle, group->csg_nr); + force = true; + } + mutex_lock(&scheduler->lock); + } + } + + mutex_unlock(&scheduler->lock); +} + +/** + * scheduler_group_schedule() - Schedule a GPU command queue group on firmware + * + * @group: Pointer to the queue group to be scheduled. + * + * This function would enable the scheduling of GPU command queue group on + * firmware. + * + * Return: 0 on success, or negative on failure. + */ +static int scheduler_group_schedule(struct kbase_queue_group *group) +{ + struct kbase_context *kctx = group->kctx; + struct kbase_device *kbdev = kctx->kbdev; + + lockdep_assert_held(&kctx->csf.lock); + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_SCHEDULE, group, group->run_state); + if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) + update_idle_suspended_group_state(group); + else if (queue_group_idle_locked(group)) { + WARN_ON(kctx->csf.sched.num_runnable_grps == 0); + WARN_ON(kbdev->csf.scheduler.total_runnable_grps == 0); + + if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE) + update_idle_suspended_group_state(group); + else + group->run_state = KBASE_CSF_GROUP_RUNNABLE; + } else if (!queue_group_scheduled_locked(group)) { + insert_group_to_runnable(&kbdev->csf.scheduler, group, + KBASE_CSF_GROUP_RUNNABLE); + } + + /* Since a group has become active now, check if GPU needs to be + * powered up. Also rekick the Scheduler. + */ + scheduler_wakeup(kbdev, true); + + return 0; +} + +/** + * set_max_csg_slots() - Set the number of available command stream group slots + * + * @kbdev: Pointer of the GPU device. + * + * This function would set/limit the number of command stream group slots that + * can be used in the given tick/tock. It would be less than the total command + * stream group slots supported by firmware if the number of GPU address space + * slots required to utilize all the CSG slots is more than the available + * address space slots. + */ +static inline void set_max_csg_slots(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned int total_csg_slots = kbdev->csf.global_iface.group_num; + unsigned int max_address_space_slots = kbdev->nr_hw_address_spaces - 1; + + WARN_ON(scheduler->num_active_address_spaces > total_csg_slots); + + if (likely(scheduler->num_active_address_spaces <= + max_address_space_slots)) + scheduler->num_csg_slots_for_tick = total_csg_slots; +} + +/** + * count_active_address_space() - Count the number of GPU address space slots + * + * @kbdev: Pointer of the GPU device. + * @kctx: Pointer of the Kbase context. + * + * This function would update the counter that is tracking the number of GPU + * address space slots that would be required to program the command stream + * group slots from the groups at the head of groups_to_schedule list. + */ +static inline void count_active_address_space(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned int total_csg_slots = kbdev->csf.global_iface.group_num; + unsigned int max_address_space_slots = kbdev->nr_hw_address_spaces - 1; + + if (scheduler->ngrp_to_schedule <= total_csg_slots) { + if (kctx->csf.sched.ngrp_to_schedule == 1) { + scheduler->num_active_address_spaces++; + + if (scheduler->num_active_address_spaces <= + max_address_space_slots) + scheduler->num_csg_slots_for_tick++; + } + } +} + +/** + * update_resident_groups_priority() - Update the priority of resident groups + * + * @kbdev: The GPU device. + * + * This function will update the priority of all resident queue groups + * that are at the head of groups_to_schedule list, preceding the first + * non-resident group. + * + * This function will also adjust kbase_csf_scheduler.head_slot_priority on + * the priority update. + */ +static void update_resident_groups_priority(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + u32 num_groups = scheduler->num_csg_slots_for_tick; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + while (!list_empty(&scheduler->groups_to_schedule)) { + struct kbase_queue_group *group = + list_first_entry(&scheduler->groups_to_schedule, + struct kbase_queue_group, + link_to_schedule); + bool resident = + kbasep_csf_scheduler_group_is_on_slot_locked(group); + + if ((group->prepared_seq_num >= num_groups) || !resident) + break; + + update_csg_slot_priority(group, + scheduler->head_slot_priority); + + /* Drop the head group from the list */ + remove_scheduled_group(kbdev, group); + scheduler->head_slot_priority--; + } +} + +/** + * program_group_on_vacant_csg_slot() - Program a non-resident group on the + * given vacant CSG slot. + * @kbdev: Pointer to the GPU device. + * @slot: Vacant command stream group slot number. + * + * This function will program a non-resident group at the head of + * kbase_csf_scheduler.groups_to_schedule list on the given vacant command + * stream group slot, provided the initial position of the non-resident + * group in the list is less than the number of CSG slots and there is + * an available GPU address space slot. + * kbase_csf_scheduler.head_slot_priority would also be adjusted after + * programming the slot. + */ +static void program_group_on_vacant_csg_slot(struct kbase_device *kbdev, + s8 slot) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + struct kbase_queue_group *const group = + list_empty(&scheduler->groups_to_schedule) ? NULL : + list_first_entry(&scheduler->groups_to_schedule, + struct kbase_queue_group, + link_to_schedule); + u32 num_groups = scheduler->num_csg_slots_for_tick; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + if (group && (group->prepared_seq_num < num_groups)) { + bool ret = kbasep_csf_scheduler_group_is_on_slot_locked(group); + + if (!WARN_ON(ret)) { + if (kctx_as_enabled(group->kctx) && !group->faulted) { + program_csg_slot(group, + slot, + scheduler->head_slot_priority); + + if (likely(csg_slot_in_use(kbdev, slot))) { + /* Drop the head group from the list */ + remove_scheduled_group(kbdev, group); + scheduler->head_slot_priority--; + } + } else + remove_scheduled_group(kbdev, group); + } + } +} + +/** + * program_vacant_csg_slot() - Program the vacant CSG slot with a non-resident + * group and update the priority of resident groups. + * + * @kbdev: Pointer to the GPU device. + * @slot: Vacant command stream group slot number. + * + * This function will first update the priority of all resident queue groups + * that are at the head of groups_to_schedule list, preceding the first + * non-resident group, it will then try to program the given command stream + * group slot with the non-resident group. Finally update the priority of all + * resident queue groups following the non-resident group. + * + * kbase_csf_scheduler.head_slot_priority would also be adjusted. + */ +static void program_vacant_csg_slot(struct kbase_device *kbdev, s8 slot) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + struct kbase_csf_csg_slot *const csg_slot = + scheduler->csg_slots; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + WARN_ON(atomic_read(&csg_slot[slot].state) != CSG_SLOT_READY); + + /* First update priority for already resident groups (if any) + * before the non-resident group + */ + update_resident_groups_priority(kbdev); + + /* Now consume the vacant slot for the non-resident group */ + program_group_on_vacant_csg_slot(kbdev, slot); + + /* Now update priority for already resident groups (if any) + * following the non-resident group + */ + update_resident_groups_priority(kbdev); +} + +static bool slots_state_changed(struct kbase_device *kbdev, + unsigned long *slots_mask, + bool (*state_check_func)(struct kbase_device *, s8)) +{ + u32 num_groups = kbdev->csf.global_iface.group_num; + DECLARE_BITMAP(changed_slots, MAX_SUPPORTED_CSGS) = {0}; + bool changed = false; + u32 i; + + for_each_set_bit(i, slots_mask, num_groups) { + if (state_check_func(kbdev, (s8)i)) { + set_bit(i, changed_slots); + changed = true; + } + } + + if (changed) + bitmap_copy(slots_mask, changed_slots, MAX_SUPPORTED_CSGS); + + return changed; +} + +/** + * program_suspending_csg_slots() - Program the CSG slots vacated on suspension + * of queue groups running on them. + * + * @kbdev: Pointer to the GPU device. + * + * This function will first wait for the ongoing suspension to complete on a + * command stream group slot and will then program the vacant slot with the + * non-resident queue group inside the groups_to_schedule list. + * The programming of the non-resident queue group on the vacant slot could + * fail due to unavailability of free GPU address space slot and so the + * programming is re-attempted after the ongoing suspension has completed + * for all the command stream group slots. + * The priority of resident groups before and after the non-resident group + * in the groups_to_schedule list would also be updated. + * This would be repeated for all the slots undergoing suspension. + * GPU reset would be initiated if the wait for suspend times out. + */ +static void program_suspending_csg_slots(struct kbase_device *kbdev) +{ + u32 num_groups = kbdev->csf.global_iface.group_num; + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS); + DECLARE_BITMAP(evicted_mask, MAX_SUPPORTED_CSGS) = {0}; + bool suspend_wait_failed = false; + long remaining = + kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + bitmap_complement(slot_mask, scheduler->csgs_events_enable_mask, + MAX_SUPPORTED_CSGS); + + while (!bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { + DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); + + bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); + + remaining = wait_event_timeout(kbdev->csf.event_wait, + slots_state_changed(kbdev, changed, + csg_slot_stopped_raw), + remaining); + + if (remaining) { + u32 i; + + for_each_set_bit(i, changed, num_groups) { + struct kbase_queue_group *group = + scheduler->csg_slots[i].resident_group; + + if (WARN_ON(!csg_slot_stopped_locked(kbdev, (s8)i))) { + continue; + } + /* The on slot csg is now stopped */ + clear_bit(i, slot_mask); + + if (likely(group)) { + bool as_fault; + /* Only do save/cleanup if the + * group is not terminated during + * the sleep. + */ + save_csg_slot(group); + as_fault = cleanup_csg_slot(group); + /* If AS fault detected, evict it */ + if (as_fault) { + sched_evict_group(group, true); + set_bit(i, evicted_mask); + } + } + + program_vacant_csg_slot(kbdev, (s8)i); + } + } else { + dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend, slot_mask: 0x%*pb\n", + num_groups, slot_mask); + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + suspend_wait_failed = true; + break; + } + } + + if (!bitmap_empty(evicted_mask, MAX_SUPPORTED_CSGS)) + dev_info(kbdev->dev, "Scheduler evicting slots: 0x%*pb\n", + num_groups, evicted_mask); + + if (unlikely(!suspend_wait_failed)) { + u32 i; + + while (scheduler->ngrp_to_schedule && + (scheduler->head_slot_priority > (MAX_CSG_SLOT_PRIORITY + - scheduler->num_csg_slots_for_tick))) { + i = find_first_zero_bit(scheduler->csg_inuse_bitmap, + num_groups); + if (WARN_ON(i == num_groups)) + break; + program_vacant_csg_slot(kbdev, (s8)i); + if (WARN_ON(!csg_slot_in_use(kbdev, (int)i))) + break; + } + } +} + +static void suspend_queue_group(struct kbase_queue_group *group) +{ + unsigned long flags; + struct kbase_csf_scheduler *const scheduler = + &group->kctx->kbdev->csf.scheduler; + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + clear_bit(group->csg_nr, scheduler->csgs_events_enable_mask); + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); + + /* If AS fault detected, terminate the group */ + if (!kctx_as_enabled(group->kctx) || group->faulted) + term_csg_slot(group); + else + suspend_csg_slot(group); +} + +static void wait_csg_slots_start(struct kbase_device *kbdev) +{ + u32 num_groups = kbdev->csf.global_iface.group_num; + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + long remaining = + kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; + u32 i; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + /* extract start slot flags for check */ + for (i = 0; i < num_groups; i++) { + if (atomic_read(&scheduler->csg_slots[i].state) == + CSG_SLOT_READY2RUN) + set_bit(i, slot_mask); + } + + while (!bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { + DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); + + bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); + + remaining = wait_event_timeout(kbdev->csf.event_wait, + slots_state_changed(kbdev, changed, csg_slot_running), + remaining); + + if (remaining) { + for_each_set_bit(i, changed, num_groups) { + struct kbase_queue_group *group = + scheduler->csg_slots[i].resident_group; + + /* The on slot csg is now running */ + clear_bit(i, slot_mask); + group->run_state = KBASE_CSF_GROUP_RUNNABLE; + } + } else { + dev_warn(kbdev->dev, "Timed out waiting for CSG slots to start, slots: 0x%*pb\n", + num_groups, slot_mask); + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + break; + } + } +} + +/** + * group_on_slot_is_idle() - Check if the queue group resident on a command + * stream group slot is idle. + * + * This function is called at the start of scheduling tick to check the + * idle status of a queue group resident on a command sream group slot. + * The group's idleness is determined by looping over all the bound command + * queues and checking their respective CS_STATUS_WAIT register as well as + * the insert and extract offsets. + + * This function would be simplified in future after the changes under + * consideration with MIDHARC-3065 are introduced. + * + * @kbdev: Pointer to the GPU device. + * @group: Pointer to the resident group on the given slot. + * @slot: The slot that the given group is resident on. + * + * Return: true if the group resident on slot is idle, otherwise false. + */ +static bool group_on_slot_is_idle(struct kbase_device *kbdev, + struct kbase_queue_group *group, unsigned long slot) +{ + struct kbase_csf_cmd_stream_group_info *ginfo = + &kbdev->csf.global_iface.groups[slot]; + u32 i; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { + struct kbase_queue *queue = group->bound_queues[i]; + + if (queue && queue->enabled) { + struct kbase_csf_cmd_stream_info *stream = + &ginfo->streams[queue->csi_index]; + u32 status = kbase_csf_firmware_cs_output(stream, + CS_STATUS_WAIT); + + if (!CS_STATUS_WAIT_SYNC_WAIT_GET(status) && + !confirm_cs_idle(group->bound_queues[i])) + return false; + } + } + + return true; +} + +/** + * slots_update_state_changed() - Check the handshake state of a subset of + * command group slots. + * + * Checks the state of a subset of slots selected through the slots_mask + * bit_map. Records which slots' handshake completed and send it back in the + * slots_done bit_map. + * + * @kbdev: The GPU device. + * @field_mask: The field mask for checking the state in the csg_req/ack. + * @slots_mask: A bit_map specifying the slots to check. + * @slots_done: A cleared bit_map for returning the slots that + * have finished update. + * + * Return: true if the slots_done is set for at least one slot. + * Otherwise false. + */ +static +bool slots_update_state_changed(struct kbase_device *kbdev, u32 field_mask, + const unsigned long *slots_mask, unsigned long *slots_done) +{ + u32 num_groups = kbdev->csf.global_iface.group_num; + bool changed = false; + u32 i; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + for_each_set_bit(i, slots_mask, num_groups) { + struct kbase_csf_cmd_stream_group_info const *const ginfo = + &kbdev->csf.global_iface.groups[i]; + u32 state = kbase_csf_firmware_csg_input_read(ginfo, CSG_REQ); + + state ^= kbase_csf_firmware_csg_output(ginfo, CSG_ACK); + + if (!(state & field_mask)) { + set_bit(i, slots_done); + changed = true; + } + } + + return changed; +} + +/** + * wait_csg_slots_handshake_ack - Wait the req/ack handshakes to complete on + * the specified groups. + * + * This function waits for the acknowledgement of the request that have + * already been placed for the CSG slots by the caller. Currently used for + * the CSG priority update and status update requests. + * + * @kbdev: Pointer to the GPU device. + * @field_mask: The field mask for checking the state in the csg_req/ack. + * @slot_mask: Bitmap reflecting the slots, the function will modify + * the acknowledged slots by clearing their corresponding + * bits. + * @wait_in_jiffies: Wait duration in jiffies, controlling the time-out. + * + * Return: 0 on all specified slots acknowledged; otherwise -ETIMEDOUT. For + * timed out condition with unacknowledged slots, their bits remain + * set in the slot_mask. + */ +static int wait_csg_slots_handshake_ack(struct kbase_device *kbdev, + u32 field_mask, unsigned long *slot_mask, long wait_in_jiffies) +{ + const u32 num_groups = kbdev->csf.global_iface.group_num; + long remaining = wait_in_jiffies; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + while (!bitmap_empty(slot_mask, num_groups) && + !kbase_reset_gpu_is_active(kbdev)) { + DECLARE_BITMAP(dones, MAX_SUPPORTED_CSGS) = { 0 }; + + remaining = wait_event_timeout(kbdev->csf.event_wait, + slots_update_state_changed(kbdev, field_mask, + slot_mask, dones), + remaining); + + if (remaining) + bitmap_andnot(slot_mask, slot_mask, dones, num_groups); + else + /* Timed-out on the wait */ + return -ETIMEDOUT; + } + + return 0; +} + +static void wait_csg_slots_finish_prio_update(struct kbase_device *kbdev) +{ + unsigned long *slot_mask = + kbdev->csf.scheduler.csg_slots_prio_update; + long wait_time = + kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + int ret = wait_csg_slots_handshake_ack(kbdev, CSG_REQ_EP_CFG_MASK, + slot_mask, wait_time); + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + if (ret != 0) { + /* The update timeout is not regarded as a serious + * issue, no major consequences are expected as a + * result, so just warn the case. + */ + dev_warn(kbdev->dev, "Timeout, skipping the update wait: slot mask=0x%lx", + slot_mask[0]); + } +} + +void kbase_csf_scheduler_evict_ctx_slots(struct kbase_device *kbdev, + struct kbase_context *kctx, struct list_head *evicted_groups) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + struct kbase_queue_group *group; + u32 num_groups = kbdev->csf.global_iface.group_num; + u32 slot; + DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; + DECLARE_BITMAP(terminated_slot_mask, MAX_SUPPORTED_CSGS); + long remaining = + kbase_csf_timeout_in_jiffies(DEFAULT_RESET_TIMEOUT_MS); + + lockdep_assert_held(&kctx->csf.lock); + mutex_lock(&scheduler->lock); + + KBASE_KTRACE_ADD(kbdev, EVICT_CTX_SLOTS, kctx, 0u); + for (slot = 0; slot < num_groups; slot++) { + group = kbdev->csf.scheduler.csg_slots[slot].resident_group; + if (group && group->kctx == kctx) { + term_csg_slot(group); + set_bit(slot, slot_mask); + } + } + + dev_info(kbdev->dev, "Evicting context %d_%d slots: 0x%*pb\n", + kctx->tgid, kctx->id, num_groups, slot_mask); + + bitmap_copy(terminated_slot_mask, slot_mask, MAX_SUPPORTED_CSGS); + /* Only check for GPU reset once - this thread has the scheduler lock, + * so even if the return value of kbase_reset_gpu_is_active changes, + * no reset work would be done anyway until the scheduler lock was + * released. + */ + if (!kbase_reset_gpu_is_active(kbdev)) { + while (remaining + && !bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { + DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); + + bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); + + remaining = wait_event_timeout(kbdev->csf.event_wait, + slots_state_changed(kbdev, changed, + csg_slot_stopped_raw), + remaining); + + if (remaining) + bitmap_andnot(slot_mask, slot_mask, changed, + MAX_SUPPORTED_CSGS); + } + } + + for_each_set_bit(slot, terminated_slot_mask, num_groups) { + bool as_fault; + + group = scheduler->csg_slots[slot].resident_group; + as_fault = cleanup_csg_slot(group); + /* remove the group from the scheduler list */ + sched_evict_group(group, as_fault); + /* return the evicted group to the caller */ + list_add_tail(&group->link, evicted_groups); + } + + if (!remaining) { + dev_warn(kbdev->dev, "Timeout on evicting ctx slots: 0x%*pb\n", + num_groups, slot_mask); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + } + + mutex_unlock(&scheduler->lock); +} + +/** + * scheduler_slot_protm_ack - Acknowledging the protected region requests + * from the resident group on a given slot. + * + * The function assumes that the given slot is in stable running state and + * has already been judged by the caller on that any pending protected region + * requests of the resident group should be acknowledged. + * + * @kbdev: Pointer to the GPU device. + * @group: Pointer to the resident group on the given slot. + * @slot: The slot that the given group is actively operating on. + * + * Return: true if the group has pending protm request(s) and is acknowledged. + * The caller should arrange to enter the protected mode for servicing + * it. Otherwise return false, indicating the group has no pending protm + * request. + */ +static bool scheduler_slot_protm_ack(struct kbase_device *const kbdev, + struct kbase_queue_group *const group, + const int slot) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + bool protm_ack = false; + struct kbase_csf_cmd_stream_group_info *ginfo = + &kbdev->csf.global_iface.groups[slot]; + u32 max_csi; + int i; + + if (WARN_ON(scheduler->csg_slots[slot].resident_group != group)) + return protm_ack; + + lockdep_assert_held(&scheduler->lock); + lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.interrupt_lock); + + max_csi = ginfo->stream_num; + for (i = find_first_bit(group->protm_pending_bitmap, max_csi); + i < max_csi; + i = find_next_bit(group->protm_pending_bitmap, max_csi, i + 1)) { + struct kbase_queue *queue = group->bound_queues[i]; + + clear_bit(i, group->protm_pending_bitmap); + + if (!WARN_ON(!queue) && queue->enabled) { + struct kbase_csf_cmd_stream_info *stream = + &ginfo->streams[i]; + u32 cs_protm_ack = kbase_csf_firmware_cs_output( + stream, CS_ACK) & + CS_ACK_PROTM_PEND_MASK; + u32 cs_protm_req = kbase_csf_firmware_cs_input_read( + stream, CS_REQ) & + CS_REQ_PROTM_PEND_MASK; + + if (cs_protm_ack == cs_protm_req) { + dev_dbg(kbdev->dev, + "PROTM-ack already done for queue-%d group-%d slot-%d", + queue->csi_index, group->handle, slot); + continue; + } + + kbase_csf_firmware_cs_input_mask(stream, CS_REQ, + cs_protm_ack, + CS_ACK_PROTM_PEND_MASK); + protm_ack = true; + dev_dbg(kbdev->dev, + "PROTM-ack for queue-%d, group-%d slot-%d", + queue->csi_index, group->handle, slot); + } + } + + return protm_ack; +} + +/** + * scheduler_group_check_protm_enter - Request the given group to be evaluated + * for triggering the protected mode. + * + * The function assumes the given group is either an active running group or + * the scheduler internally maintained field scheduler->top_grp. + * + * If the GPU is not already running in protected mode and the input group + * has protected region requests from its bound queues, the requests are + * acknowledged and the GPU is instructed to enter the protected mode. + * + * @kbdev: Pointer to the GPU device. + * @input_grp: Pointer to the GPU queue group. + */ +static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, + struct kbase_queue_group *const input_grp) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + lockdep_assert_held(&scheduler->lock); + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + + /* Firmware samples the PROTM_PEND ACK bit for command streams when + * Host sends PROTM_ENTER global request. So if PROTM_PEND ACK bit + * is set for a command stream after Host has sent the PROTM_ENTER + * Global request, then there is no guarantee that firmware will + * notice that prior to switching to protected mode. And firmware + * may not again raise the PROTM_PEND interrupt for that command + * stream later on. To avoid that uncertainty PROTM_PEND ACK bit + * is not set for a command stream if the request to enter protected + * mode has already been sent. It will be set later (after the exit + * from protected mode has taken place) when the group to which + * command stream is bound becomes the top group. + * + * The actual decision of entering protected mode is hinging on the + * input group is the top priority group, or, in case the previous + * top-group is evicted from the scheduler during the tick, its would + * be replacement, and that it is currently in a stable state (i.e. the + * slot state is running). + */ + if (!kbase_csf_scheduler_protected_mode_in_use(kbdev)) { + if (!WARN_ON(!input_grp)) { + const int slot = + kbase_csf_scheduler_group_get_slot_locked( + input_grp); + + /* check the input_grp is running and requesting + * protected mode + */ + if (slot >= 0 && + atomic_read( + &scheduler->csg_slots[slot].state) == + CSG_SLOT_RUNNING) { + if (kctx_as_enabled(input_grp->kctx) && + scheduler_slot_protm_ack(kbdev, + input_grp, slot)) { + /* Option of acknowledging to multiple + * CSGs from the same kctx is dropped, + * after consulting with the + * architecture team. See the comment in + * GPUCORE-21394. + */ + + /* Switch to protected mode */ + scheduler->active_protm_grp = input_grp; + KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_ENTER_PROTM, input_grp, 0u); + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); + kbase_csf_enter_protected_mode(kbdev); + return; + } + } + } + } + + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); +} + +static void scheduler_apply(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + const u32 total_csg_slots = kbdev->csf.global_iface.group_num; + const u32 available_csg_slots = scheduler->num_csg_slots_for_tick; + u32 suspend_cnt = 0; + u32 remain_cnt = 0; + u32 resident_cnt = 0; + struct kbase_queue_group *group; + u32 i; + u32 spare; + + lockdep_assert_held(&scheduler->lock); + + /* Suspend those resident groups not in the run list */ + for (i = 0; i < total_csg_slots; i++) { + group = scheduler->csg_slots[i].resident_group; + if (group) { + resident_cnt++; + if (group->prepared_seq_num >= available_csg_slots) { + suspend_queue_group(group); + suspend_cnt++; + } else + remain_cnt++; + } + } + + /* If there are spare slots, apply heads in the list */ + spare = (available_csg_slots > resident_cnt) ? + (available_csg_slots - resident_cnt) : 0; + while (!list_empty(&scheduler->groups_to_schedule)) { + group = list_first_entry(&scheduler->groups_to_schedule, + struct kbase_queue_group, + link_to_schedule); + + if (kbasep_csf_scheduler_group_is_on_slot_locked(group) && + group->prepared_seq_num < available_csg_slots) { + /* One of the resident remainders */ + update_csg_slot_priority(group, + scheduler->head_slot_priority); + } else if (spare != 0) { + s8 slot = (s8)find_first_zero_bit( + kbdev->csf.scheduler.csg_inuse_bitmap, + total_csg_slots); + + if (WARN_ON(slot >= (s8)total_csg_slots)) + break; + + if (!kctx_as_enabled(group->kctx) || group->faulted) { + /* Drop the head group and continue */ + remove_scheduled_group(kbdev, group); + continue; + } + program_csg_slot(group, slot, + scheduler->head_slot_priority); + if (unlikely(!csg_slot_in_use(kbdev, slot))) + break; + + spare--; + } else + break; + + /* Drop the head csg from the list */ + remove_scheduled_group(kbdev, group); + if (scheduler->head_slot_priority) + scheduler->head_slot_priority--; + } + + /* Dealing with groups currently going through suspend */ + program_suspending_csg_slots(kbdev); +} + +static void scheduler_ctx_scan_groups(struct kbase_device *kbdev, + struct kbase_context *kctx, int priority) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + struct kbase_queue_group *group; + + lockdep_assert_held(&scheduler->lock); + if (WARN_ON(priority < 0) || + WARN_ON(priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT)) + return; + + if (!kctx_as_enabled(kctx)) + return; + + list_for_each_entry(group, &kctx->csf.sched.runnable_groups[priority], + link) { + if (WARN_ON(!list_empty(&group->link_to_schedule))) + /* This would be a bug */ + list_del_init(&group->link_to_schedule); + + if (unlikely(group->faulted)) + continue; + + if (queue_group_idle_locked(group)) { + list_add_tail(&group->link_to_schedule, + &scheduler->idle_groups_to_schedule); + continue; + } + + if (!scheduler->ngrp_to_schedule) { + /* keep the top csg's origin */ + scheduler->top_ctx = kctx; + scheduler->top_grp = group; + } + + list_add_tail(&group->link_to_schedule, + &scheduler->groups_to_schedule); + group->prepared_seq_num = scheduler->ngrp_to_schedule++; + + kctx->csf.sched.ngrp_to_schedule++; + count_active_address_space(kbdev, kctx); + } +} + +/** + * scheduler_rotate_groups() - Rotate the runnable queue groups to provide + * fairness of scheduling within a single + * kbase_context. + * + * Since only kbase_csf_scheduler's top_grp (i.e. the queue group assigned + * the highest slot priority) is guaranteed to get the resources that it + * needs we only rotate the kbase_context corresponding to it - + * kbase_csf_scheduler's top_ctx. + * + * The priority level chosen for rotation is the one containing the previous + * scheduling cycle's kbase_csf_scheduler's top_grp. + * + * In a 'fresh-slice-cycle' this always corresponds to the highest group + * priority in use by kbase_csf_scheduler's top_ctx. That is, it's the priority + * level of the previous scheduling cycle's first runnable kbase_context. + * + * We choose this priority level because when higher priority work is + * scheduled, we should always cause the scheduler to run and do a scan. The + * scan always enumerates the highest priority work first (whether that be + * based on process priority or group priority), and thus + * kbase_csf_scheduler's top_grp will point to the first of those high priority + * groups, which necessarily must be the highest priority group in + * kbase_csf_scheduler's top_ctx. The fresh-slice-cycle will run later and pick + * up that group appropriately. + * + * If kbase_csf_scheduler's top_grp was instead evicted (and thus is NULL), + * then no explicit rotation occurs on the next fresh-slice-cycle schedule, but + * will set up kbase_csf_scheduler's top_ctx again for the next scheduling + * cycle. Implicitly, a rotation had already occurred by removing + * the kbase_csf_scheduler's top_grp + * + * If kbase_csf_scheduler's top_grp became idle and all other groups belonging + * to kbase_csf_scheduler's top_grp's priority level in kbase_csf_scheduler's + * top_ctx are also idle, then the effect of this will be to rotate idle + * groups, which might not actually become resident in the next + * scheduling slice. However this is acceptable since a queue group becoming + * idle is implicitly a rotation (as above with evicted queue groups), as it + * automatically allows a new queue group to take the maximum slot priority + * whilst the idle kbase_csf_scheduler's top_grp ends up near the back of + * the kbase_csf_scheduler's groups_to_schedule list. In this example, it will + * be for a group in the next lowest priority level or in absence of those the + * next kbase_context's queue groups. + * + * @kbdev: Pointer to the GPU device. + */ +static void scheduler_rotate_groups(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + struct kbase_context *const top_ctx = scheduler->top_ctx; + struct kbase_queue_group *const top_grp = scheduler->top_grp; + + lockdep_assert_held(&scheduler->lock); + if (top_ctx && top_grp) { + struct list_head *list = + &top_ctx->csf.sched.runnable_groups[top_grp->priority]; + + WARN_ON(top_grp->kctx != top_ctx); + if (!WARN_ON(list_empty(list))) { + list_move_tail(&top_grp->link, list); + dev_dbg(kbdev->dev, + "groups rotated for a context, num_runnable_groups: %u\n", + scheduler->top_ctx->csf.sched.num_runnable_grps); + } + } +} + +static void scheduler_rotate_ctxs(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + struct list_head *list = &scheduler->runnable_kctxs; + + lockdep_assert_held(&scheduler->lock); + if (scheduler->top_ctx) { + if (!WARN_ON(list_empty(list))) { + struct kbase_context *pos; + bool found = false; + + /* Locate the ctx on the list */ + list_for_each_entry(pos, list, csf.link) { + if (scheduler->top_ctx == pos) { + found = true; + break; + } + } + + if (!WARN_ON(!found)) { + list_move_tail(&pos->csf.link, list); + dev_dbg(kbdev->dev, "contexts rotated\n"); + } + } + } +} + +/** + * scheduler_update_idle_slots_status() - Get the status update for the command + * stream group slots for which the IDLE notification was + * received previously. + * + * This function sends a CSG status update request for all the command stream + * group slots present in the bitmap scheduler->csg_slots_idle_mask and wait + * for the request to complete. + * The bits set in the scheduler->csg_slots_idle_mask bitmap are cleared by + * this function. + * + * @kbdev: Pointer to the GPU device. + * @csg_bitmap: Bitmap of the command stream group slots for which + * the status update request completed successfully. + * @failed_csg_bitmap: Bitmap of the command stream group slots for which + * the status update request timedout. + */ +static void scheduler_update_idle_slots_status(struct kbase_device *kbdev, + unsigned long *csg_bitmap, unsigned long *failed_csg_bitmap) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + const u32 num_groups = kbdev->csf.global_iface.group_num; + struct kbase_csf_global_iface *const global_iface = + &kbdev->csf.global_iface; + unsigned long flags, i; + + lockdep_assert_held(&scheduler->lock); + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + for_each_set_bit(i, scheduler->csg_slots_idle_mask, num_groups) { + struct kbase_csf_csg_slot *csg_slot = &scheduler->csg_slots[i]; + struct kbase_queue_group *group = csg_slot->resident_group; + struct kbase_csf_cmd_stream_group_info *const ginfo = + &global_iface->groups[i]; + u32 csg_req; + + clear_bit(i, scheduler->csg_slots_idle_mask); + + if (WARN_ON(!group)) + continue; + + csg_req = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); + csg_req ^= CSG_REQ_STATUS_UPDATE_MASK; + kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, csg_req, + CSG_REQ_STATUS_UPDATE_MASK); + + set_bit(i, csg_bitmap); + } + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); + + /* The groups are aggregated into a single kernel doorbell request */ + if (!bitmap_empty(csg_bitmap, num_groups)) { + long wt = + kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + u32 db_slots = (u32)csg_bitmap[0]; + + kbase_csf_ring_csg_slots_doorbell(kbdev, db_slots); + + if (wait_csg_slots_handshake_ack(kbdev, + CSG_REQ_STATUS_UPDATE_MASK, csg_bitmap, wt)) { + dev_warn(kbdev->dev, "Timeout, treat groups as not idle: slot mask=0x%lx", + csg_bitmap[0]); + + /* Store the bitmap of timed out slots */ + bitmap_copy(failed_csg_bitmap, csg_bitmap, num_groups); + csg_bitmap[0] = ~csg_bitmap[0] & db_slots; + } else { + csg_bitmap[0] = db_slots; + } + } +} + +/** + * scheduler_handle_idle_slots() - Update the idle status of queue groups + * resident on command stream group slots for which the + * IDLE notification was received previously. + * + * This function is called at the start of scheduling tick/tock to reconfirm + * the idle status of queue groups resident on command sream group slots for + * which idle notification was received previously, i.e. all the CSG slots + * present in the bitmap scheduler->csg_slots_idle_mask. + * The confirmation is done by sending the CSG status update request to the + * firmware. The idleness of a CSG is determined by looping over all the + * bound command streams and checking their respective CS_STATUS_WAIT register + * as well as the insert and extract offset. + * The run state of the groups resident on still idle CSG slots is changed to + * KBASE_CSF_GROUP_IDLE and the bitmap scheduler->csg_slots_idle_mask is + * updated accordingly. + * The bits corresponding to slots for which the status update request timedout + * remain set in scheduler->csg_slots_idle_mask. + * + * @kbdev: Pointer to the GPU device. + */ +static void scheduler_handle_idle_slots(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + u32 num_groups = kbdev->csf.global_iface.group_num; + unsigned long flags, i; + DECLARE_BITMAP(csg_bitmap, MAX_SUPPORTED_CSGS) = { 0 }; + DECLARE_BITMAP(failed_csg_bitmap, MAX_SUPPORTED_CSGS) = { 0 }; + + lockdep_assert_held(&scheduler->lock); + + scheduler_update_idle_slots_status(kbdev, csg_bitmap, + failed_csg_bitmap); + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + for_each_set_bit(i, csg_bitmap, num_groups) { + struct kbase_csf_csg_slot *csg_slot = &scheduler->csg_slots[i]; + struct kbase_queue_group *group = csg_slot->resident_group; + + if (WARN_ON(atomic_read(&csg_slot->state) != CSG_SLOT_RUNNING)) + continue; + if (WARN_ON(!group)) + continue; + if (WARN_ON(group->run_state != KBASE_CSF_GROUP_RUNNABLE)) + continue; + if (WARN_ON(group->priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT)) + continue; + + if (group_on_slot_is_idle(kbdev, group, i)) { + group->run_state = KBASE_CSF_GROUP_IDLE; + set_bit(i, scheduler->csg_slots_idle_mask); + } + } + + bitmap_or(scheduler->csg_slots_idle_mask, + scheduler->csg_slots_idle_mask, + failed_csg_bitmap, num_groups); + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); +} + +static void scheduler_scan_idle_groups(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + struct kbase_queue_group *group, *n; + + list_for_each_entry_safe(group, n, &scheduler->idle_groups_to_schedule, + link_to_schedule) { + + WARN_ON(!queue_group_idle_locked(group)); + + if (!scheduler->ngrp_to_schedule) { + /* keep the top csg's origin */ + scheduler->top_ctx = group->kctx; + scheduler->top_grp = group; + } + + group->prepared_seq_num = scheduler->ngrp_to_schedule++; + list_move_tail(&group->link_to_schedule, + &scheduler->groups_to_schedule); + + group->kctx->csf.sched.ngrp_to_schedule++; + count_active_address_space(kbdev, group->kctx); + } +} + +static void scheduler_rotate(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + + lockdep_assert_held(&scheduler->lock); + + /* Dealing with rotation */ + scheduler_rotate_groups(kbdev); + scheduler_rotate_ctxs(kbdev); +} + +static struct kbase_queue_group *get_tock_top_group( + struct kbase_csf_scheduler *const scheduler) +{ + struct kbase_context *kctx; + int i; + + lockdep_assert_held(&scheduler->lock); + for (i = 0; i < BASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { + list_for_each_entry(kctx, + &scheduler->runnable_kctxs, csf.link) { + struct kbase_queue_group *group; + + list_for_each_entry(group, + &kctx->csf.sched.runnable_groups[i], + link) { + if (queue_group_idle_locked(group)) + continue; + + return group; + } + } + } + + return NULL; +} + +static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, + bool is_suspend) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = { 0 }; + + int ret = suspend_active_queue_groups(kbdev, slot_mask); + + if (ret) { + /* The suspend of CSGs failed, trigger the GPU reset and wait + * for it to complete to be in a deterministic state. + */ + dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend on power down, slot_mask: 0x%*pb\n", + kbdev->csf.global_iface.group_num, slot_mask); + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + + if (is_suspend) { + mutex_unlock(&scheduler->lock); + kbase_reset_gpu_wait(kbdev); + mutex_lock(&scheduler->lock); + } + return -1; + } + + /* Check if the groups became active whilst the suspend was ongoing, + * but only for the case where the system suspend is not in progress + */ + if (!is_suspend && atomic_read(&scheduler->non_idle_suspended_grps)) + return -1; + + return 0; +} + +static void gpu_idle_worker(struct work_struct *work) +{ + struct kbase_device *kbdev = container_of( + work, struct kbase_device, csf.scheduler.gpu_idle_work.work); + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + + mutex_lock(&scheduler->lock); + + if (!scheduler->total_runnable_grps) { + if (scheduler->state != SCHED_SUSPENDED) { + scheduler_suspend(kbdev); + dev_info(kbdev->dev, "Scheduler now suspended"); + } + } else { + dev_dbg(kbdev->dev, "Scheduler couldn't be suspended"); + } + + mutex_unlock(&scheduler->lock); +} + +static int scheduler_prepare(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + int i; + + lockdep_assert_held(&scheduler->lock); + + /* Empty the groups_to_schedule */ + while (!list_empty(&scheduler->groups_to_schedule)) { + struct kbase_queue_group *grp = + list_first_entry(&scheduler->groups_to_schedule, + struct kbase_queue_group, + link_to_schedule); + + remove_scheduled_group(kbdev, grp); + } + + /* Pre-scan init scheduler fields */ + if (WARN_ON(scheduler->ngrp_to_schedule != 0)) + scheduler->ngrp_to_schedule = 0; + scheduler->top_ctx = NULL; + scheduler->top_grp = NULL; + scheduler->head_slot_priority = MAX_CSG_SLOT_PRIORITY; + WARN_ON(!list_empty(&scheduler->idle_groups_to_schedule)); + scheduler->num_active_address_spaces = 0; + scheduler->num_csg_slots_for_tick = 0; + bitmap_zero(scheduler->csg_slots_prio_update, MAX_SUPPORTED_CSGS); + + /* Scan out to run groups */ + for (i = 0; i < BASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { + struct kbase_context *kctx; + + list_for_each_entry(kctx, &scheduler->runnable_kctxs, csf.link) + scheduler_ctx_scan_groups(kbdev, kctx, i); + } + + scheduler_scan_idle_groups(kbdev); + KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, + scheduler->num_active_address_spaces | + (((u64)scheduler->ngrp_to_schedule) << 32)); + set_max_csg_slots(kbdev); + dev_dbg(kbdev->dev, "prepared groups length: %u, num_active_address_spaces: %u\n", + scheduler->ngrp_to_schedule, scheduler->num_active_address_spaces); + return 0; +} + +static void scheduler_wait_protm_quit(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + long wt = kbase_csf_timeout_in_jiffies(CSF_STATE_WAIT_TIMEOUT_MS); + long remaining; + + lockdep_assert_held(&scheduler->lock); + + remaining = wait_event_timeout(kbdev->csf.event_wait, + !kbase_csf_scheduler_protected_mode_in_use(kbdev), wt); + + if (!remaining) + dev_warn(kbdev->dev, "Timeout, protm_quit wait skipped"); +} + +static void schedule_actions(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned long flags; + struct kbase_queue_group *protm_grp; + int ret; + + lockdep_assert_held(&scheduler->lock); + + ret = kbase_pm_wait_for_desired_state(kbdev); + if (ret) { + dev_err(kbdev->dev, "Wait for MCU power on failed"); + return; + } + + scheduler_handle_idle_slots(kbdev); + scheduler_prepare(kbdev); + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + protm_grp = scheduler->active_protm_grp; + + /* Avoid update if the top-group remains unchanged and in protected + * mode. For the said case, all the slots update is effectively + * competing against the active protected mode group (typically the + * top-group). If we update other slots, even on leaving the + * top-group slot untouched, the firmware would exit the protected mode + * for interacting with the host-driver. After it, as the top-group + * would again raise the request for entering protected mode, we would + * be actively doing the switching over twice without progressing the + * queue jobs. + */ + if (protm_grp && scheduler->top_grp == protm_grp) { + dev_dbg(kbdev->dev, "Scheduler keep protm exec: group-%d", + protm_grp->handle); + } else if (scheduler->top_grp) { + if (protm_grp) + dev_dbg(kbdev->dev, "Scheduler drop protm exec: group-%d", + protm_grp->handle); + + if (!bitmap_empty(scheduler->top_grp->protm_pending_bitmap, + kbdev->csf.global_iface.groups[0].stream_num)) { + dev_dbg(kbdev->dev, "Scheduler prepare protm exec: group-%d of context %d_%d", + scheduler->top_grp->handle, + scheduler->top_grp->kctx->tgid, + scheduler->top_grp->kctx->id); + + /* Due to GPUCORE-24491 only the top-group is allowed + * to be on slot and all other on slot groups have to + * be suspended before entering protected mode. + * This would change in GPUCORE-24492. + */ + scheduler->num_csg_slots_for_tick = 1; + } + + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); + + scheduler_apply(kbdev); + /* Scheduler is dropping the exec of the previous protm_grp, + * Until the protm quit completes, the GPU is effectively + * locked in the secure mode. + */ + if (protm_grp) + scheduler_wait_protm_quit(kbdev); + + wait_csg_slots_start(kbdev); + wait_csg_slots_finish_prio_update(kbdev); + + if (scheduler->num_csg_slots_for_tick == 1) { + scheduler_group_check_protm_enter(kbdev, + scheduler->top_grp); + } + + return; + } + + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); + return; +} + +static void schedule_on_tock(struct work_struct *work) +{ + struct kbase_device *kbdev = container_of(work, struct kbase_device, + csf.scheduler.tock_work.work); + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + + mutex_lock(&scheduler->lock); + + if (kbase_reset_gpu_is_active(kbdev) || + (scheduler->state == SCHED_SUSPENDED)) { + mutex_unlock(&scheduler->lock); + return; + } + + WARN_ON(!(scheduler->state == SCHED_INACTIVE)); + scheduler->state = SCHED_BUSY; + + /* Undertaking schedule action steps */ + KBASE_KTRACE_ADD(kbdev, SCHEDULER_TOCK, NULL, 0u); + schedule_actions(kbdev); + + /* Record time information */ + scheduler->last_schedule = jiffies; + + /* Tock is serviced */ + scheduler->tock_pending_request = false; + + scheduler->state = SCHED_INACTIVE; + mutex_unlock(&scheduler->lock); + + dev_dbg(kbdev->dev, + "Waking up for event after schedule-on-tock completes."); + wake_up_all(&kbdev->csf.event_wait); +} + +static void schedule_on_tick(struct work_struct *work) +{ + struct kbase_device *kbdev = container_of(work, struct kbase_device, + csf.scheduler.tick_work.work); + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + + mutex_lock(&scheduler->lock); + + if (kbase_reset_gpu_is_active(kbdev) || + (scheduler->state == SCHED_SUSPENDED)) { + mutex_unlock(&scheduler->lock); + return; + } + + scheduler->state = SCHED_BUSY; + + /* Do scheduling stuff */ + scheduler_rotate(kbdev); + + /* Undertaking schedule action steps */ + KBASE_KTRACE_ADD(kbdev, SCHEDULER_TICK, NULL, 0u); + schedule_actions(kbdev); + + /* Record time information */ + scheduler->last_schedule = jiffies; + + /* Kicking next scheduling if needed */ + if (likely(scheduler_timer_is_enabled_nolock(kbdev)) && + (scheduler->total_runnable_grps > 0)) { + mod_delayed_work(scheduler->wq, &scheduler->tick_work, + CSF_SCHEDULER_TIME_TICK_JIFFIES); + dev_dbg(kbdev->dev, "scheduling for next tick, num_runnable_groups:%u\n", + scheduler->total_runnable_grps); + } + + scheduler->state = SCHED_INACTIVE; + mutex_unlock(&scheduler->lock); + + dev_dbg(kbdev->dev, "Waking up for event after schedule-on-tick completes."); + wake_up_all(&kbdev->csf.event_wait); +} + +int wait_csg_slots_suspend(struct kbase_device *kbdev, + const unsigned long *slot_mask, + unsigned int timeout_ms) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + long remaining = kbase_csf_timeout_in_jiffies(timeout_ms); + u32 num_groups = kbdev->csf.global_iface.group_num; + int err = 0; + DECLARE_BITMAP(slot_mask_local, MAX_SUPPORTED_CSGS); + + lockdep_assert_held(&scheduler->lock); + + bitmap_copy(slot_mask_local, slot_mask, MAX_SUPPORTED_CSGS); + + while (!bitmap_empty(slot_mask_local, MAX_SUPPORTED_CSGS) + && remaining) { + DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); + + bitmap_copy(changed, slot_mask_local, MAX_SUPPORTED_CSGS); + + remaining = wait_event_timeout(kbdev->csf.event_wait, + slots_state_changed(kbdev, changed, + csg_slot_stopped_locked), + remaining); + + if (remaining) { + u32 i; + + for_each_set_bit(i, changed, num_groups) { + struct kbase_queue_group *group; + + if (WARN_ON(!csg_slot_stopped_locked(kbdev, (s8)i))) + continue; + + /* The on slot csg is now stopped */ + clear_bit(i, slot_mask_local); + + group = scheduler->csg_slots[i].resident_group; + if (likely(group)) { + /* Only do save/cleanup if the + * group is not terminated during + * the sleep. + */ + save_csg_slot(group); + if (cleanup_csg_slot(group)) + sched_evict_group(group, true); + } + } + } else { + dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend, slot_mask: 0x%*pb\n", + num_groups, slot_mask_local); + err = -ETIMEDOUT; + } + } + + return err; +} + +static int suspend_active_queue_groups(struct kbase_device *kbdev, + unsigned long *slot_mask) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + u32 num_groups = kbdev->csf.global_iface.group_num; + u32 slot_num; + int ret; + + lockdep_assert_held(&scheduler->lock); + + for (slot_num = 0; slot_num < num_groups; slot_num++) { + struct kbase_queue_group *group = + scheduler->csg_slots[slot_num].resident_group; + + if (group) { + suspend_queue_group(group); + set_bit(slot_num, slot_mask); + } + } + + ret = wait_csg_slots_suspend(kbdev, slot_mask, + CSG_SUSPEND_ON_RESET_WAIT_TIMEOUT_MS); + return ret; +} + +static int suspend_active_queue_groups_on_reset(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = { 0 }; + int ret; + + mutex_lock(&scheduler->lock); + + ret = suspend_active_queue_groups(kbdev, slot_mask); + if (ret) { + dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend before reset, slot_mask: 0x%*pb\n", + kbdev->csf.global_iface.group_num, slot_mask); + } + + if (!bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { + int ret2; + + /* Need to flush the GPU cache to ensure suspend buffer + * contents are not lost on reset of GPU. + * Do this even if suspend operation had timedout for some of + * the CSG slots. + */ + kbase_gpu_start_cache_clean(kbdev); + ret2 = kbase_gpu_wait_cache_clean_timeout(kbdev, + DEFAULT_RESET_TIMEOUT_MS); + if (ret2) { + dev_warn(kbdev->dev, "Timed out waiting for cache clean to complete before reset"); + if (!ret) + ret = ret2; + } + } + + mutex_unlock(&scheduler->lock); + + return ret; +} + +static void scheduler_inner_reset(struct kbase_device *kbdev) +{ + u32 const num_groups = kbdev->csf.global_iface.group_num; + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + WARN_ON(csgs_active(kbdev)); + + /* Cancel any potential queued delayed work(s) */ + cancel_delayed_work_sync(&scheduler->tick_work); + cancel_delayed_work_sync(&scheduler->tock_work); + cancel_delayed_work_sync(&scheduler->ping_work); + + mutex_lock(&scheduler->lock); + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + bitmap_fill(scheduler->csgs_events_enable_mask, MAX_SUPPORTED_CSGS); + scheduler->active_protm_grp = NULL; + memset(kbdev->csf.scheduler.csg_slots, 0, + num_groups * sizeof(struct kbase_csf_csg_slot)); + bitmap_zero(kbdev->csf.scheduler.csg_inuse_bitmap, num_groups); + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); + + scheduler->top_ctx = NULL; + scheduler->top_grp = NULL; + + KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, + scheduler->num_active_address_spaces | + (((u64)scheduler->total_runnable_grps) << 32)); + + mutex_unlock(&scheduler->lock); +} + +void kbase_csf_scheduler_reset(struct kbase_device *kbdev) +{ + struct kbase_context *kctx; + + WARN_ON(!kbase_reset_gpu_is_active(kbdev)); + + KBASE_KTRACE_ADD(kbdev, SCHEDULER_RESET, NULL, 0u); + if (!kbase_csf_scheduler_protected_mode_in_use(kbdev) && + !suspend_active_queue_groups_on_reset(kbdev)) { + /* As all groups have been successfully evicted from the CSG + * slots, clear out thee scheduler data fields and return + */ + scheduler_inner_reset(kbdev); + return; + } + + mutex_lock(&kbdev->kctx_list_lock); + + /* The loop to iterate over the kbase contexts is present due to lock + * ordering issue between kctx->csf.lock & kbdev->csf.scheduler.lock. + * CSF ioctls first take kctx->csf.lock which is context-specific and + * then take kbdev->csf.scheduler.lock for global actions like assigning + * a CSG slot. + * If the lock ordering constraint was not there then could have + * directly looped over the active queue groups. + */ + list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { + /* Firmware reload would reinitialize the CSG & CS interface IO + * pages, so just need to internally mark the currently active + * queue groups as terminated (similar to the unexpected OoM + * event case). + * No further work can now get executed for the active groups + * (new groups would have to be created to execute work) and + * in near future Clients would be duly informed of this + * reset. The resources (like User IO pages, GPU queue memory) + * allocated for the associated queues would be freed when the + * Clients do the teardown when they become aware of the reset. + */ + kbase_csf_active_queue_groups_reset(kbdev, kctx); + } + + mutex_unlock(&kbdev->kctx_list_lock); + + /* After queue groups reset, the scheduler data fields clear out */ + scheduler_inner_reset(kbdev); +} + +static void firmware_aliveness_monitor(struct work_struct *work) +{ + struct kbase_device *kbdev = container_of(work, struct kbase_device, + csf.scheduler.ping_work.work); + int err; + + /* Get the scheduler mutex to ensure that reset will not change while + * this function is being executed as otherwise calling kbase_reset_gpu + * when reset is already occurring is a programming error. + */ + mutex_lock(&kbdev->csf.scheduler.lock); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (fw_debug) { + /* ping requests cause distraction in firmware debugging */ + goto exit; + } +#endif + + if (kbdev->csf.scheduler.state == SCHED_SUSPENDED) + goto exit; + + if (kbase_reset_gpu_is_active(kbdev)) + goto exit; + + if (get_nr_active_csgs(kbdev) != 1) + goto exit; + + if (kbase_csf_scheduler_protected_mode_in_use(kbdev)) + goto exit; + + if (kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { + /* Suspend pending - no real need to ping */ + goto exit; + } + + kbase_pm_wait_for_desired_state(kbdev); + + err = kbase_csf_firmware_ping(kbdev); + + if (err) { + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + } else if (get_nr_active_csgs(kbdev) == 1) { + queue_delayed_work(system_long_wq, + &kbdev->csf.scheduler.ping_work, + msecs_to_jiffies(FIRMWARE_PING_INTERVAL_MS)); + } + + kbase_pm_context_idle(kbdev); +exit: + mutex_unlock(&kbdev->csf.scheduler.lock); +} + +int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, + struct kbase_suspend_copy_buffer *sus_buf) +{ + struct kbase_context *const kctx = group->kctx; + struct kbase_device *const kbdev = kctx->kbdev; + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + int err; + + lockdep_assert_held(&kctx->csf.lock); + mutex_lock(&scheduler->lock); + + err = wait_gpu_reset(kbdev); + if (err) { + dev_warn(kbdev->dev, "Error while waiting for the GPU reset to complete when suspending group %d on slot %d", + group->handle, group->csg_nr); + goto exit; + } + + if (kbasep_csf_scheduler_group_is_on_slot_locked(group)) { + DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; + + set_bit(kbase_csf_scheduler_group_get_slot(group), slot_mask); + + if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) + suspend_queue_group(group); + err = wait_csg_slots_suspend(kbdev, slot_mask, + CSF_STATE_WAIT_TIMEOUT_MS); + if (err) { + dev_warn(kbdev->dev, "Timed out waiting for the group %d to suspend on slot %d", + group->handle, group->csg_nr); + goto exit; + } + } + + if (queue_group_suspended_locked(group)) { + unsigned int target_page_nr = 0, i = 0; + u64 offset = sus_buf->offset; + size_t to_copy = sus_buf->size; + + if (scheduler->state != SCHED_SUSPENDED) { + /* Similar to the case of HW counters, need to flush + * the GPU cache before reading from the suspend buffer + * pages as they are mapped and cached on GPU side. + */ + kbase_gpu_start_cache_clean(kbdev); + kbase_gpu_wait_cache_clean(kbdev); + } else { + /* Make sure power down transitions have completed, + * i.e. L2 has been powered off as that would ensure + * its contents are flushed to memory. + * This is needed as Scheduler doesn't wait for the + * power down to finish. + */ + kbase_pm_wait_for_desired_state(kbdev); + } + + for (i = 0; i < PFN_UP(sus_buf->size) && + target_page_nr < sus_buf->nr_pages; i++) { + struct page *pg = + as_page(group->normal_suspend_buf.phy[i]); + void *sus_page = kmap(pg); + + if (sus_page) { + kbase_sync_single_for_cpu(kbdev, + kbase_dma_addr(pg), + PAGE_SIZE, DMA_BIDIRECTIONAL); + + err = kbase_mem_copy_to_pinned_user_pages( + sus_buf->pages, sus_page, + &to_copy, sus_buf->nr_pages, + &target_page_nr, offset); + kunmap(pg); + if (err) + break; + } else { + err = -ENOMEM; + break; + } + } + schedule_in_cycle(group, false); + } else { + /* If addr-space fault, the group may have been evicted */ + err = -EIO; + } + +exit: + mutex_unlock(&scheduler->lock); + return err; +} + +KBASE_EXPORT_TEST_API(kbase_csf_scheduler_group_copy_suspend_buf); + +/** + * group_sync_updated() - Evaluate sync wait condition of all blocked command + * queues of the group. + * + * @group: Pointer to the command queue group that has blocked command queue(s) + * bound to it. + * + * Return: true if sync wait condition is satisfied for at least one blocked + * queue of the group. + */ +static bool group_sync_updated(struct kbase_queue_group *group) +{ + bool updated = false; + int stream; + + WARN_ON(group->run_state != KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC); + + for (stream = 0; stream < MAX_SUPPORTED_STREAMS_PER_GROUP; ++stream) { + struct kbase_queue *const queue = group->bound_queues[stream]; + + /* To check the necessity of sync-wait evaluation, + * we rely on the cached 'status_wait' instead of reading it + * directly from shared memory as the CSG has been already + * evicted from the CSG slot, thus this CSG doesn't have + * valid information in the shared memory. + */ + if (queue && queue->enabled && + CS_STATUS_WAIT_SYNC_WAIT_GET(queue->status_wait)) + if (evaluate_sync_update(queue)) { + updated = true; + queue->status_wait = 0; + } + } + + return updated; +} + +/** + * scheduler_get_protm_enter_async_group() - Check if the GPU queue group + * can be now allowed to execute in protected mode. + * + * @kbdev: Pointer to the GPU device. + * @group: Pointer to the GPU queue group. + * + * This function is called outside the scheduling tick/tock to determine + * if the given GPU queue group can now execute in protected mode or not. + * If the group pointer passed is NULL then the evaluation is done for the + * scheduler->top_grp (or the second top-group). + * + * It returns the same group pointer, that was passed as an argument, if that + * group matches the scheduler->top_grp and has pending protected region + * requests otherwise NULL is returned. + * + * If the group pointer passed is NULL then the pointer to scheduler->top_grp + * is returned if that has pending protected region requests otherwise NULL is + * returned. + * + * If the scheduler->top_grp is NULL, which may happen when the top-group is + * evicted during the tick, the second top-group (as a replacement of the + * top-group) is used for the match check and also for the evaluation of + * pending protected region requests if the group pointer passed is NULL. + * + * Return: the pointer to queue group that can currently execute in protected + * mode or NULL. + */ +static struct kbase_queue_group *scheduler_get_protm_enter_async_group( + struct kbase_device *const kbdev, + struct kbase_queue_group *const group) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + struct kbase_queue_group *match_grp, *input_grp; + + lockdep_assert_held(&scheduler->lock); + + if (scheduler->state != SCHED_INACTIVE) + return NULL; + + match_grp = scheduler->top_grp ? scheduler->top_grp : + get_tock_top_group(scheduler); + input_grp = group ? group : match_grp; + + if (input_grp && (input_grp == match_grp)) { + struct kbase_csf_cmd_stream_group_info *ginfo = + &kbdev->csf.global_iface.groups[0]; + unsigned long *pending = + input_grp->protm_pending_bitmap; + unsigned long flags; + + spin_lock_irqsave(&scheduler->interrupt_lock, flags); + + if (kbase_csf_scheduler_protected_mode_in_use(kbdev) || + bitmap_empty(pending, ginfo->stream_num)) + input_grp = NULL; + + spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); + } else { + input_grp = NULL; + } + + return input_grp; +} + +void kbase_csf_scheduler_group_protm_enter(struct kbase_queue_group *group) +{ + struct kbase_device *const kbdev = group->kctx->kbdev; + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + + mutex_lock(&scheduler->lock); + + /* Check if the group is now eligible for execution in protected mode + * and accordingly undertake full scheduling actions as due to + * GPUCORE-24491 the on slot groups other than the top group have to + * be suspended first before entering protected mode. + */ + if (scheduler_get_protm_enter_async_group(kbdev, group)) + schedule_actions(kbdev); + + mutex_unlock(&scheduler->lock); +} + +/** + * check_group_sync_update_worker() - Check the sync wait condition for all the + * blocked queue groups + * + * @work: Pointer to the context-specific work item for evaluating the wait + * condition for all the queue groups in idle_wait_groups list. + * + * This function checks the gpu queues of all the groups present in + * idle_wait_groups list of a context. If the sync wait condition + * for at least one queue bound to the group has been satisfied then + * the group is moved to the per context list of runnable groups so + * that Scheduler can consider scheduling the group in next tick. + */ +static void check_group_sync_update_worker(struct work_struct *work) +{ + struct kbase_context *const kctx = container_of(work, + struct kbase_context, csf.sched.sync_update_work); + struct kbase_csf_scheduler *const scheduler = + &kctx->kbdev->csf.scheduler; + + mutex_lock(&scheduler->lock); + + if (kctx->csf.sched.num_idle_wait_grps != 0) { + struct kbase_queue_group *group, *temp; + + list_for_each_entry_safe(group, temp, + &kctx->csf.sched.idle_wait_groups, link) { + if (group_sync_updated(group)) { + /* Move this group back in to the runnable + * groups list of the context. + */ + update_idle_suspended_group_state(group); + KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, GROUP_SYNC_UPDATE_DONE, group, 0u); + } + } + } else { + WARN_ON(!list_empty(&kctx->csf.sched.idle_wait_groups)); + } + + mutex_unlock(&scheduler->lock); +} + +static +enum kbase_csf_event_callback_action check_group_sync_update_cb(void *param) +{ + struct kbase_context *const kctx = param; + + KBASE_KTRACE_ADD(kctx->kbdev, SYNC_UPDATE_EVENT, kctx, 0u); + queue_work(kctx->csf.sched.sync_update_wq, + &kctx->csf.sched.sync_update_work); + + return KBASE_CSF_EVENT_CALLBACK_KEEP; +} + +int kbase_csf_scheduler_context_init(struct kbase_context *kctx) +{ + int priority; + int err; + + for (priority = 0; priority < BASE_QUEUE_GROUP_PRIORITY_COUNT; + ++priority) { + INIT_LIST_HEAD(&kctx->csf.sched.runnable_groups[priority]); + } + + kctx->csf.sched.num_runnable_grps = 0; + INIT_LIST_HEAD(&kctx->csf.sched.idle_wait_groups); + kctx->csf.sched.num_idle_wait_grps = 0; + kctx->csf.sched.ngrp_to_schedule = 0; + + kctx->csf.sched.sync_update_wq = + alloc_ordered_workqueue("mali_kbase_csf_sync_update_wq", + WQ_HIGHPRI); + if (!kctx->csf.sched.sync_update_wq) { + dev_err(kctx->kbdev->dev, + "Failed to initialize scheduler context workqueue"); + return -ENOMEM; + } + + INIT_WORK(&kctx->csf.sched.sync_update_work, + check_group_sync_update_worker); + + err = kbase_csf_event_wait_add(kctx, check_group_sync_update_cb, kctx); + + if (err) { + dev_err(kctx->kbdev->dev, + "Failed to register a sync update callback"); + destroy_workqueue(kctx->csf.sched.sync_update_wq); + } + + return err; +} + +void kbase_csf_scheduler_context_term(struct kbase_context *kctx) +{ + kbase_csf_event_wait_remove(kctx, check_group_sync_update_cb, kctx); + cancel_work_sync(&kctx->csf.sched.sync_update_work); + destroy_workqueue(kctx->csf.sched.sync_update_wq); +} + +int kbase_csf_scheduler_init(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + u32 num_groups = kbdev->csf.global_iface.group_num; + + bitmap_zero(scheduler->csg_inuse_bitmap, num_groups); + bitmap_zero(scheduler->csg_slots_idle_mask, num_groups); + + scheduler->csg_slots = kcalloc(num_groups, + sizeof(*scheduler->csg_slots), GFP_KERNEL); + if (!scheduler->csg_slots) { + dev_err(kbdev->dev, + "Failed to allocate memory for csg slot status array\n"); + return -ENOMEM; + } + + scheduler->timer_enabled = true; + + scheduler->wq = alloc_ordered_workqueue("csf_scheduler_wq", WQ_HIGHPRI); + if (!scheduler->wq) { + dev_err(kbdev->dev, "Failed to allocate scheduler workqueue\n"); + + kfree(scheduler->csg_slots); + scheduler->csg_slots = NULL; + + return -ENOMEM; + } + + INIT_DEFERRABLE_WORK(&scheduler->tick_work, schedule_on_tick); + INIT_DEFERRABLE_WORK(&scheduler->tock_work, schedule_on_tock); + + INIT_DEFERRABLE_WORK(&scheduler->ping_work, firmware_aliveness_monitor); + BUILD_BUG_ON(GLB_REQ_WAIT_TIMEOUT_MS >= FIRMWARE_PING_INTERVAL_MS); + + mutex_init(&scheduler->lock); + spin_lock_init(&scheduler->interrupt_lock); + + /* Internal lists */ + INIT_LIST_HEAD(&scheduler->runnable_kctxs); + INIT_LIST_HEAD(&scheduler->groups_to_schedule); + INIT_LIST_HEAD(&scheduler->idle_groups_to_schedule); + + BUILD_BUG_ON(MAX_SUPPORTED_CSGS > + (sizeof(scheduler->csgs_events_enable_mask) * BITS_PER_BYTE)); + bitmap_fill(scheduler->csgs_events_enable_mask, MAX_SUPPORTED_CSGS); + scheduler->state = SCHED_SUSPENDED; + scheduler->pm_active_count = 0; + scheduler->ngrp_to_schedule = 0; + scheduler->total_runnable_grps = 0; + scheduler->top_ctx = NULL; + scheduler->top_grp = NULL; + scheduler->last_schedule = 0; + scheduler->tock_pending_request = false; + scheduler->active_protm_grp = NULL; + scheduler_doorbell_init(kbdev); + + INIT_DEFERRABLE_WORK(&scheduler->gpu_idle_work, gpu_idle_worker); + atomic_set(&scheduler->non_idle_suspended_grps, 0); + + return 0; +} + +void kbase_csf_scheduler_term(struct kbase_device *kbdev) +{ + if (kbdev->csf.scheduler.csg_slots) { + WARN_ON(csgs_active(kbdev)); + cancel_delayed_work_sync(&kbdev->csf.scheduler.gpu_idle_work); + cancel_delayed_work_sync(&kbdev->csf.scheduler.ping_work); + destroy_workqueue(kbdev->csf.scheduler.wq); + mutex_destroy(&kbdev->csf.scheduler.lock); + kfree(kbdev->csf.scheduler.csg_slots); + kbdev->csf.scheduler.csg_slots = NULL; + } +} + +/** + * scheduler_enable_tick_timer_nolock - Enable the scheduler tick timer. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * This function will restart the scheduler tick so that regular scheduling can + * be resumed without any explicit trigger (like kicking of GPU queues). This + * is a variant of kbase_csf_scheduler_enable_tick_timer() that assumes the + * CSF scheduler lock to already have been held. + */ +static void scheduler_enable_tick_timer_nolock(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + + lockdep_assert_held(&kbdev->csf.scheduler.lock); + + if (unlikely(!scheduler_timer_is_enabled_nolock(kbdev))) + return; + + WARN_ON((scheduler->state != SCHED_INACTIVE) && + (scheduler->state != SCHED_SUSPENDED)); + WARN_ON(delayed_work_pending(&scheduler->tick_work)); + + if (scheduler->total_runnable_grps > 0) { + mod_delayed_work(scheduler->wq, &scheduler->tick_work, 0); + dev_dbg(kbdev->dev, "Re-enabling the scheduler timer\n"); + } +} + +void kbase_csf_scheduler_enable_tick_timer(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->csf.scheduler.lock); + scheduler_enable_tick_timer_nolock(kbdev); + mutex_unlock(&kbdev->csf.scheduler.lock); +} + +bool kbase_csf_scheduler_timer_is_enabled(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + bool enabled; + + mutex_lock(&scheduler->lock); + enabled = scheduler_timer_is_enabled_nolock(kbdev); + mutex_unlock(&scheduler->lock); + + return enabled; +} + +void kbase_csf_scheduler_timer_set_enabled(struct kbase_device *kbdev, + bool enable) +{ + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + bool currently_enabled; + + mutex_lock(&scheduler->lock); + + currently_enabled = scheduler_timer_is_enabled_nolock(kbdev); + if (currently_enabled && !enable) { + scheduler->timer_enabled = false; + + cancel_delayed_work(&scheduler->tick_work); + cancel_delayed_work(&scheduler->tock_work); + } else if (!currently_enabled && enable) { + scheduler->timer_enabled = true; + + scheduler_enable_tick_timer_nolock(kbdev); + } + + mutex_unlock(&scheduler->lock); +} + +void kbase_csf_scheduler_kick(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + + mutex_lock(&scheduler->lock); + + if (unlikely(scheduler_timer_is_enabled_nolock(kbdev))) + goto out; + + if (scheduler->total_runnable_grps > 0) { + mod_delayed_work(scheduler->wq, &scheduler->tick_work, 0); + dev_dbg(kbdev->dev, "Kicking the scheduler manually\n"); + } + +out: + mutex_unlock(&scheduler->lock); +} + +void kbase_csf_scheduler_pm_suspend(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + + /* Cancel any potential queued delayed work(s) */ + cancel_delayed_work_sync(&scheduler->tick_work); + cancel_delayed_work_sync(&scheduler->tock_work); + + mutex_lock(&scheduler->lock); + + WARN_ON(!kbase_pm_is_suspending(kbdev)); + + if (scheduler->state != SCHED_SUSPENDED) { + suspend_active_groups_on_powerdown(kbdev, true); + dev_info(kbdev->dev, "Scheduler PM suspend"); + scheduler_suspend(kbdev); + } + mutex_unlock(&scheduler->lock); +} + +void kbase_csf_scheduler_pm_resume(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + + mutex_lock(&scheduler->lock); + + WARN_ON(kbase_pm_is_suspending(kbdev)); + + if (scheduler->total_runnable_grps > 0) { + WARN_ON(scheduler->state != SCHED_SUSPENDED); + dev_info(kbdev->dev, "Scheduler PM resume"); + scheduler_wakeup(kbdev, true); + } + mutex_unlock(&scheduler->lock); +} + +void kbase_csf_scheduler_pm_active(struct kbase_device *kbdev) +{ + unsigned long flags; + u32 prev_count; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + prev_count = kbdev->csf.scheduler.pm_active_count++; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* On 0 => 1, make a pm_ctx_active request */ + if (!prev_count) + kbase_pm_context_active(kbdev); + else + WARN_ON(prev_count == U32_MAX); +} + +void kbase_csf_scheduler_pm_idle(struct kbase_device *kbdev) +{ + unsigned long flags; + u32 prev_count; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + prev_count = kbdev->csf.scheduler.pm_active_count--; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (prev_count == 1) + kbase_pm_context_idle(kbdev); + else + WARN_ON(prev_count == 0); +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h new file mode 100755 index 000000000000..1b1c0681f64d --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h @@ -0,0 +1,408 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_SCHEDULER_H_ +#define _KBASE_CSF_SCHEDULER_H_ + +#include "mali_kbase_csf.h" + +/** + * kbase_csf_scheduler_queue_start() - Enable the running of GPU command queue + * on firmware. + * + * @queue: Pointer to the GPU command queue to be started. + * + * This function would enable the start of a command stream interface, within a + * command stream group, to which the @queue was bound. + * If the command stream group is already scheduled and resident, the command + * stream interface will be started right away, otherwise once the group is + * made resident. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_scheduler_queue_start(struct kbase_queue *queue); + +/** + * kbase_csf_scheduler_queue_stop() - Disable the running of GPU command queue + * on firmware. + * + * @queue: Pointer to the GPU command queue to be stopped. + * + * This function would stop the command stream interface, within a command + * stream group, to which the @queue was bound. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_scheduler_queue_stop(struct kbase_queue *queue); + +/** + * kbase_csf_scheduler_group_protm_enter - Handle the protm enter event for the + * GPU command queue group. + * + * @group: The command queue group. + * + * This function could request the firmware to enter the protected mode + * and allow the execution of protected region instructions for all the + * bound queues of the group that have protm pending bit set in their + * respective CS_ACK register. + */ +void kbase_csf_scheduler_group_protm_enter(struct kbase_queue_group *group); + +/** + * kbase_csf_scheduler_group_get_slot() - Checks if a queue group is + * programmed on a firmware Command Stream Group slot + * and returns the slot number. + * + * @group: The command queue group. + * + * Return: The slot number, if the group is programmed on a slot. + * Otherwise returns a negative number. + * + * Note: This function should not be used if the interrupt_lock is held. Use + * kbase_csf_scheduler_group_get_slot_locked() instead. + */ +int kbase_csf_scheduler_group_get_slot(struct kbase_queue_group *group); + +/** + * kbase_csf_scheduler_group_get_slot_locked() - Checks if a queue group is + * programmed on a firmware Command Stream Group slot + * and returns the slot number. + * + * @group: The command queue group. + * + * Return: The slot number, if the group is programmed on a slot. + * Otherwise returns a negative number. + * + * Note: Caller must hold the interrupt_lock. + */ +int kbase_csf_scheduler_group_get_slot_locked(struct kbase_queue_group *group); + +/** + * kbase_csf_scheduler_group_events_enabled() - Checks if interrupt events + * should be handled for a queue group. + * + * @kbdev: The device of the group. + * @group: The queue group. + * + * Return: true if interrupt events should be handled. + * + * Note: Caller must hold the interrupt_lock. + */ +bool kbase_csf_scheduler_group_events_enabled(struct kbase_device *kbdev, + struct kbase_queue_group *group); + +/** + * kbase_csf_scheduler_get_group_on_slot()- Gets the queue group that has been + * programmed to a firmware Command Stream Group slot. + * + * @kbdev: The GPU device. + * @slot: The slot for which to get the queue group. + * + * Return: Pointer to the programmed queue group. + * + * Note: Caller must hold the interrupt_lock. + */ +struct kbase_queue_group *kbase_csf_scheduler_get_group_on_slot( + struct kbase_device *kbdev, int slot); + +/** + * kbase_csf_scheduler_group_deschedule() - Deschedule a GPU command queue + * group from the firmware. + * + * @group: Pointer to the queue group to be scheduled. + * + * This function would disable the scheduling of GPU command queue group on + * firmware. + */ +void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group); + +/** + * kbase_csf_scheduler_evict_ctx_slots() - Evict all GPU command queue groups + * of a given context that are active + * running from the firmware. + * + * @kbdev: The GPU device. + * @kctx: Kbase context for the evict operation. + * @evicted_groups: List_head for returning evicted active queue groups. + * + * This function would disable the scheduling of GPU command queue groups active + * on firmware slots from the given Kbase context. The affected groups are + * added to the supplied list_head argument. + */ +void kbase_csf_scheduler_evict_ctx_slots(struct kbase_device *kbdev, + struct kbase_context *kctx, struct list_head *evicted_groups); + +/** + * kbase_csf_scheduler_context_init() - Initialize the context-specific part + * for CSF scheduler. + * + * @kctx: Pointer to kbase context that is being created. + * + * This function must be called during Kbase context creation. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_scheduler_context_init(struct kbase_context *kctx); + +/** + * kbase_csf_scheduler_init - Initialize the CSF scheduler + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * The scheduler does the arbitration for the command stream group slots + * provided by the firmware between the GPU command queue groups created + * by the Clients. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_scheduler_init(struct kbase_device *kbdev); + +/** + * kbase_csf_scheduler_context_init() - Terminate the context-specific part + * for CSF scheduler. + * + * @kctx: Pointer to kbase context that is being terminated. + * + * This function must be called during Kbase context termination. + */ +void kbase_csf_scheduler_context_term(struct kbase_context *kctx); + +/** + * kbase_csf_scheduler_term - Terminate the CSF scheduler. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * This should be called when unload of firmware is done on device + * termination. + */ +void kbase_csf_scheduler_term(struct kbase_device *kbdev); + +/** + * kbase_csf_scheduler_reset - Reset the state of all active GPU command + * queue groups. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * This function will first iterate through all the active/scheduled GPU + * command queue groups and suspend them (to avoid losing work for groups + * that are not stuck). The groups that could not get suspended would be + * descheduled and marked as terminated (which will then lead to unbinding + * of all the queues bound to them) and also no more work would be allowed + * to execute for them. + * + * This is similar to the action taken in response to an unexpected OoM event. + * No explicit re-initialization is done for CSG & CS interface I/O pages; + * instead, that happens implicitly on firmware reload. + * + * Should be called only after initiating the GPU reset. + */ +void kbase_csf_scheduler_reset(struct kbase_device *kbdev); + +/** + * kbase_csf_scheduler_enable_tick_timer - Enable the scheduler tick timer. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * This function will restart the scheduler tick so that regular scheduling can + * be resumed without any explicit trigger (like kicking of GPU queues). + */ +void kbase_csf_scheduler_enable_tick_timer(struct kbase_device *kbdev); + +/** + * kbase_csf_scheduler_group_copy_suspend_buf - Suspend a queue + * group and copy suspend buffer. + * + * This function is called to suspend a queue group and copy the suspend_buffer + * contents to the input buffer provided. + * + * @group: Pointer to the queue group to be suspended. + * @sus_buf: Pointer to the structure which contains details of the + * user buffer and its kernel pinned pages to which we need to copy + * the group suspend buffer. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, + struct kbase_suspend_copy_buffer *sus_buf); + +/** + * kbase_csf_scheduler_lock - Acquire the global Scheduler lock. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * This function will take the global scheduler lock, in order to serialize + * against the Scheduler actions, for access to CS IO pages. + */ +static inline void kbase_csf_scheduler_lock(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->csf.scheduler.lock); +} + +/** + * kbase_csf_scheduler_unlock - Release the global Scheduler lock. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +static inline void kbase_csf_scheduler_unlock(struct kbase_device *kbdev) +{ + mutex_unlock(&kbdev->csf.scheduler.lock); +} + +/** + * kbase_csf_scheduler_spin_lock - Acquire Scheduler interrupt spinlock. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @flags: Pointer to the memory location that would store the previous + * interrupt state. + * + * This function will take the global scheduler lock, in order to serialize + * against the Scheduler actions, for access to CS IO pages. + */ +static inline void kbase_csf_scheduler_spin_lock(struct kbase_device *kbdev, + unsigned long *flags) +{ + spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, *flags); +} + +/** + * kbase_csf_scheduler_spin_unlock - Release Scheduler interrupt spinlock. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @flags: Previously stored interrupt state when Scheduler interrupt + * spinlock was acquired. + */ +static inline void kbase_csf_scheduler_spin_unlock(struct kbase_device *kbdev, + unsigned long flags) +{ + spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); +} + +/** + * kbase_csf_scheduler_spin_lock_assert_held - Assert if the Scheduler + * interrupt spinlock is held. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +static inline void +kbase_csf_scheduler_spin_lock_assert_held(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); +} + +/** + * kbase_csf_scheduler_timer_is_enabled() - Check if the scheduler wakes up + * automatically for periodic tasks. + * + * @kbdev: Pointer to the device + * + * Return: true if the scheduler is configured to wake up periodically + */ +bool kbase_csf_scheduler_timer_is_enabled(struct kbase_device *kbdev); + +/** + * kbase_csf_scheduler_timer_set_enabled() - Enable/disable periodic + * scheduler tasks. + * + * @kbdev: Pointer to the device + * @enable: Whether to enable periodic scheduler tasks + */ +void kbase_csf_scheduler_timer_set_enabled(struct kbase_device *kbdev, + bool enable); + +/** + * kbase_csf_scheduler_kick - Perform pending scheduling tasks once. + * + * Note: This function is only effective if the scheduling timer is disabled. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +void kbase_csf_scheduler_kick(struct kbase_device *kbdev); + +/** + * kbase_csf_scheduler_protected_mode_in_use() - Check if the scheduler is + * running with protected mode tasks. + * + * @kbdev: Pointer to the device + * + * Return: true if the scheduler is running with protected mode tasks + */ +static inline bool kbase_csf_scheduler_protected_mode_in_use( + struct kbase_device *kbdev) +{ + return (kbdev->csf.scheduler.active_protm_grp != NULL); +} + +/** + * kbase_csf_scheduler_pm_active - Perform scheduler power active operation + * + * Note: This function will increase the scheduler's internal pm_active_count + * value, ensuring that both GPU and MCU are powered for access. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +void kbase_csf_scheduler_pm_active(struct kbase_device *kbdev); + +/** + * kbase_csf_scheduler_pm_idle - Perform the scheduler power idle operation + * + * Note: This function will decrease the scheduler's internal pm_active_count + * value. On reaching 0, the MCU and GPU could be powered off. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + */ +void kbase_csf_scheduler_pm_idle(struct kbase_device *kbdev); + +/** + * kbase_csf_scheduler_pm_resume - Reactivate the scheduler on system resume + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * This function will make the scheduler resume the scheduling of queue groups + * and take the power managemenet reference, if there are any runnable groups. + */ +void kbase_csf_scheduler_pm_resume(struct kbase_device *kbdev); + +/** + * kbase_csf_scheduler_pm_suspend - Idle the scheduler on system suspend + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * This function will make the scheduler suspend all the running queue groups + * and drop its power managemenet reference. + */ +void kbase_csf_scheduler_pm_suspend(struct kbase_device *kbdev); + +#endif /* _KBASE_CSF_SCHEDULER_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c new file mode 100755 index 000000000000..60cae15bc8ef --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c @@ -0,0 +1,584 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_csf_tiler_heap.h" +#include "mali_kbase_csf_tiler_heap_def.h" +#include "mali_kbase_csf_heap_context_alloc.h" + +/** + * encode_chunk_ptr - Encode the address and size of a chunk as an integer. + * + * The size and address of the next chunk in a list are packed into a single + * 64-bit value for storage in a chunk's header. This function returns that + * value. + * + * @chunk_size: Size of a tiler heap chunk, in bytes. + * @chunk_addr: GPU virtual address of the same tiler heap chunk. + * + * Return: Next chunk pointer suitable for writing into a chunk header. + */ +static u64 encode_chunk_ptr(u32 const chunk_size, u64 const chunk_addr) +{ + u64 encoded_size, encoded_addr; + + WARN_ON(chunk_size & ~CHUNK_SIZE_MASK); + WARN_ON(chunk_addr & ~CHUNK_ADDR_MASK); + + encoded_size = + (u64)(chunk_size >> CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT) << + CHUNK_HDR_NEXT_SIZE_POS; + + encoded_addr = + (chunk_addr >> CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT) << + CHUNK_HDR_NEXT_ADDR_POS; + + return (encoded_size & CHUNK_HDR_NEXT_SIZE_MASK) | + (encoded_addr & CHUNK_HDR_NEXT_ADDR_MASK); +} + +/** + * get_last_chunk - Get the last chunk of a tiler heap + * + * @heap: Pointer to the tiler heap. + * + * Return: The address of the most recently-linked chunk, or NULL if none. + */ +static struct kbase_csf_tiler_heap_chunk *get_last_chunk( + struct kbase_csf_tiler_heap *const heap) +{ + lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); + + if (list_empty(&heap->chunks_list)) + return NULL; + + return list_last_entry(&heap->chunks_list, + struct kbase_csf_tiler_heap_chunk, link); +} + +/** + * link_chunk - Link a chunk into a tiler heap + * + * Unless the @chunk is the first in the kernel's list of chunks belonging to + * a given tiler heap, this function stores the size and address of the @chunk + * in the header of the preceding chunk. This requires the GPU memory region + * containing the header to be be mapped temporarily, which can fail. + * + * @heap: Pointer to the tiler heap. + * @chunk: Pointer to the heap chunk to be linked. + * + * Return: 0 if successful or a negative error code on failure. + */ +static int link_chunk(struct kbase_csf_tiler_heap *const heap, + struct kbase_csf_tiler_heap_chunk *const chunk) +{ + struct kbase_csf_tiler_heap_chunk *const prev = get_last_chunk(heap); + + if (prev) { + struct kbase_context *const kctx = heap->kctx; + struct kbase_vmap_struct map; + u64 *const prev_hdr = kbase_vmap_prot(kctx, prev->gpu_va, + sizeof(*prev_hdr), KBASE_REG_CPU_WR, &map); + + if (unlikely(!prev_hdr)) { + dev_err(kctx->kbdev->dev, + "Failed to map tiler heap chunk 0x%llX\n", + prev->gpu_va); + return -ENOMEM; + } + + *prev_hdr = encode_chunk_ptr(heap->chunk_size, chunk->gpu_va); + kbase_vunmap(kctx, &map); + + dev_dbg(kctx->kbdev->dev, + "Linked tiler heap chunks, 0x%llX -> 0x%llX\n", + prev->gpu_va, chunk->gpu_va); + } + + return 0; +} + +/** + * init_chunk - Initialize and link a tiler heap chunk + * + * Zero-initialize a new chunk's header (including its pointer to the next + * chunk, which doesn't exist yet) and then update the previous chunk's + * header to link the new chunk into the chunk list. + * + * @heap: Pointer to the tiler heap. + * @chunk: Pointer to the heap chunk to be initialized and linked. + * @link_with_prev: Flag to indicate if the new chunk needs to be linked with + * the previously allocated chunk. + * + * Return: 0 if successful or a negative error code on failure. + */ +static int init_chunk(struct kbase_csf_tiler_heap *const heap, + struct kbase_csf_tiler_heap_chunk *const chunk, bool link_with_prev) +{ + struct kbase_vmap_struct map; + struct u64 *chunk_hdr = NULL; + struct kbase_context *const kctx = heap->kctx; + + if (unlikely(chunk->gpu_va & ~CHUNK_ADDR_MASK)) { + dev_err(kctx->kbdev->dev, + "Tiler heap chunk address is unusable\n"); + return -EINVAL; + } + + chunk_hdr = kbase_vmap_prot(kctx, + chunk->gpu_va, CHUNK_HDR_SIZE, KBASE_REG_CPU_WR, &map); + + if (unlikely(!chunk_hdr)) { + dev_err(kctx->kbdev->dev, + "Failed to map a tiler heap chunk header\n"); + return -ENOMEM; + } + + memset(chunk_hdr, 0, CHUNK_HDR_SIZE); + kbase_vunmap(kctx, &map); + + if (link_with_prev) + return link_chunk(heap, chunk); + else + return 0; +} + +/** + * create_chunk - Create a tiler heap chunk + * + * This function allocates a chunk of memory for a tiler heap and adds it to + * the end of the list of chunks associated with that heap. The size of the + * chunk is not a parameter because it is configured per-heap not per-chunk. + * + * @heap: Pointer to the tiler heap for which to allocate memory. + * @link_with_prev: Flag to indicate if the chunk to be allocated needs to be + * linked with the previously allocated chunk. + * + * Return: 0 if successful or a negative error code on failure. + */ +static int create_chunk(struct kbase_csf_tiler_heap *const heap, + bool link_with_prev) +{ + int err = 0; + struct kbase_context *const kctx = heap->kctx; + u64 nr_pages = PFN_UP(heap->chunk_size); + u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | + BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE | + BASE_MEM_COHERENT_LOCAL; + struct kbase_csf_tiler_heap_chunk *chunk = NULL; + + flags |= base_mem_group_id_set(kctx->jit_group_id); + +#if defined(CONFIG_MALI_BIFROST_DEBUG) || defined(CONFIG_MALI_VECTOR_DUMP) + flags |= BASE_MEM_PROT_CPU_RD; +#endif + + lockdep_assert_held(&kctx->csf.tiler_heaps.lock); + + chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); + if (unlikely(!chunk)) { + dev_err(kctx->kbdev->dev, + "No kernel memory for a new tiler heap chunk\n"); + return -ENOMEM; + } + + /* Allocate GPU memory for the new chunk. */ + INIT_LIST_HEAD(&chunk->link); + chunk->region = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, + &flags, &chunk->gpu_va); + + if (unlikely(!chunk->region)) { + dev_err(kctx->kbdev->dev, + "Failed to allocate a tiler heap chunk\n"); + err = -ENOMEM; + } else { + err = init_chunk(heap, chunk, link_with_prev); + if (unlikely(err)) { + kbase_gpu_vm_lock(kctx); + chunk->region->flags &= ~KBASE_REG_NO_USER_FREE; + kbase_mem_free_region(kctx, chunk->region); + kbase_gpu_vm_unlock(kctx); + } + } + + if (unlikely(err)) { + kfree(chunk); + } else { + list_add_tail(&chunk->link, &heap->chunks_list); + heap->chunk_count++; + + dev_dbg(kctx->kbdev->dev, "Created tiler heap chunk 0x%llX\n", + chunk->gpu_va); + } + + return err; +} + +/** + * delete_chunk - Delete a tiler heap chunk + * + * This function frees a tiler heap chunk previously allocated by @create_chunk + * and removes it from the list of chunks associated with the heap. + * + * WARNING: The deleted chunk is not unlinked from the list of chunks used by + * the GPU, therefore it is only safe to use this function when + * deleting a heap. + * + * @heap: Pointer to the tiler heap for which @chunk was allocated. + * @chunk: Pointer to a chunk to be deleted. + */ +static void delete_chunk(struct kbase_csf_tiler_heap *const heap, + struct kbase_csf_tiler_heap_chunk *const chunk) +{ + struct kbase_context *const kctx = heap->kctx; + + lockdep_assert_held(&kctx->csf.tiler_heaps.lock); + + kbase_gpu_vm_lock(kctx); + chunk->region->flags &= ~KBASE_REG_NO_USER_FREE; + kbase_mem_free_region(kctx, chunk->region); + kbase_gpu_vm_unlock(kctx); + list_del(&chunk->link); + heap->chunk_count--; + kfree(chunk); +} + +/** + * delete_all_chunks - Delete all chunks belonging to a tiler heap + * + * This function empties the list of chunks associated with a tiler heap by + * freeing all chunks previously allocated by @create_chunk. + * + * @heap: Pointer to a tiler heap. + */ +static void delete_all_chunks(struct kbase_csf_tiler_heap *heap) +{ + struct list_head *entry = NULL, *tmp = NULL; + struct kbase_context *const kctx = heap->kctx; + + lockdep_assert_held(&kctx->csf.tiler_heaps.lock); + + list_for_each_safe(entry, tmp, &heap->chunks_list) { + struct kbase_csf_tiler_heap_chunk *chunk = list_entry( + entry, struct kbase_csf_tiler_heap_chunk, link); + + delete_chunk(heap, chunk); + } +} + +/** + * create_initial_chunks - Create the initial list of chunks for a tiler heap + * + * This function allocates a given number of chunks for a tiler heap and + * adds them to the list of chunks associated with that heap. + * + * @heap: Pointer to the tiler heap for which to allocate memory. + * @nchunks: Number of chunks to create. + * + * Return: 0 if successful or a negative error code on failure. + */ +static int create_initial_chunks(struct kbase_csf_tiler_heap *const heap, + u32 const nchunks) +{ + int err = 0; + u32 i; + + for (i = 0; (i < nchunks) && likely(!err); i++) + err = create_chunk(heap, true); + + if (unlikely(err)) + delete_all_chunks(heap); + + return err; +} + +/** + * delete_heap - Delete a tiler heap + * + * This function frees any chunks allocated for a tiler heap previously + * initialized by @kbase_csf_tiler_heap_init and removes it from the list of + * heaps associated with the kbase context. The heap context structure used by + * the firmware is also freed. + * + * @heap: Pointer to a tiler heap to be deleted. + */ +static void delete_heap(struct kbase_csf_tiler_heap *heap) +{ + struct kbase_context *const kctx = heap->kctx; + + dev_dbg(kctx->kbdev->dev, "Deleting tiler heap 0x%llX\n", heap->gpu_va); + + lockdep_assert_held(&kctx->csf.tiler_heaps.lock); + + delete_all_chunks(heap); + + /* We could optimize context destruction by not freeing leaked heap + * contexts but it doesn't seem worth the extra complexity. + */ + kbase_csf_heap_context_allocator_free(&kctx->csf.tiler_heaps.ctx_alloc, + heap->gpu_va); + + list_del(&heap->link); + kfree(heap); +} + +/** + * find_tiler_heap - Find a tiler heap from the address of its heap context + * + * Each tiler heap managed by the kernel has an associated heap context + * structure used by the firmware. This function finds a tiler heap object from + * the GPU virtual address of its associated heap context. The heap context + * should have been allocated by @kbase_csf_heap_context_allocator_alloc in the + * same @kctx. + * + * @kctx: Pointer to the kbase context to search for a tiler heap. + * @heap_gpu_va: GPU virtual address of a heap context structure. + * + * Return: pointer to the tiler heap object, or NULL if not found. + */ +static struct kbase_csf_tiler_heap *find_tiler_heap( + struct kbase_context *const kctx, u64 const heap_gpu_va) +{ + struct kbase_csf_tiler_heap *heap = NULL; + + lockdep_assert_held(&kctx->csf.tiler_heaps.lock); + + list_for_each_entry(heap, &kctx->csf.tiler_heaps.list, link) { + if (heap_gpu_va == heap->gpu_va) + return heap; + } + + dev_dbg(kctx->kbdev->dev, "Tiler heap 0x%llX was not found\n", + heap_gpu_va); + + return NULL; +} + +int kbase_csf_tiler_heap_context_init(struct kbase_context *const kctx) +{ + int err = kbase_csf_heap_context_allocator_init( + &kctx->csf.tiler_heaps.ctx_alloc, kctx); + + if (unlikely(err)) + return err; + + INIT_LIST_HEAD(&kctx->csf.tiler_heaps.list); + mutex_init(&kctx->csf.tiler_heaps.lock); + + dev_dbg(kctx->kbdev->dev, "Initialized a context for tiler heaps\n"); + + return 0; +} + +void kbase_csf_tiler_heap_context_term(struct kbase_context *const kctx) +{ + struct list_head *entry = NULL, *tmp = NULL; + + dev_dbg(kctx->kbdev->dev, "Terminating a context for tiler heaps\n"); + + mutex_lock(&kctx->csf.tiler_heaps.lock); + + list_for_each_safe(entry, tmp, &kctx->csf.tiler_heaps.list) { + struct kbase_csf_tiler_heap *heap = list_entry( + entry, struct kbase_csf_tiler_heap, link); + delete_heap(heap); + } + + mutex_unlock(&kctx->csf.tiler_heaps.lock); + mutex_destroy(&kctx->csf.tiler_heaps.lock); + + kbase_csf_heap_context_allocator_term(&kctx->csf.tiler_heaps.ctx_alloc); +} + +int kbase_csf_tiler_heap_init(struct kbase_context *const kctx, + u32 const chunk_size, u32 const initial_chunks, u32 const max_chunks, + u16 const target_in_flight, u64 *const heap_gpu_va, + u64 *const first_chunk_va) +{ + int err = 0; + struct kbase_csf_tiler_heap *heap = NULL; + struct kbase_csf_heap_context_allocator *const ctx_alloc = + &kctx->csf.tiler_heaps.ctx_alloc; + + dev_dbg(kctx->kbdev->dev, + "Creating a tiler heap with %u chunks (limit: %u) of size %u\n", + initial_chunks, max_chunks, chunk_size); + + if (chunk_size == 0) + return -EINVAL; + + if (chunk_size & ~CHUNK_SIZE_MASK) + return -EINVAL; + + if (initial_chunks == 0) + return -EINVAL; + + if (initial_chunks > max_chunks) + return -EINVAL; + + if (target_in_flight == 0) + return -EINVAL; + + heap = kzalloc(sizeof(*heap), GFP_KERNEL); + if (unlikely(!heap)) { + dev_err(kctx->kbdev->dev, + "No kernel memory for a new tiler heap\n"); + return -ENOMEM; + } + + heap->kctx = kctx; + heap->chunk_size = chunk_size; + heap->max_chunks = max_chunks; + heap->target_in_flight = target_in_flight; + INIT_LIST_HEAD(&heap->chunks_list); + + heap->gpu_va = kbase_csf_heap_context_allocator_alloc(ctx_alloc); + + mutex_lock(&kctx->csf.tiler_heaps.lock); + + if (unlikely(!heap->gpu_va)) { + dev_err(kctx->kbdev->dev, + "Failed to allocate a tiler heap context\n"); + err = -ENOMEM; + } else { + err = create_initial_chunks(heap, initial_chunks); + if (unlikely(err)) { + kbase_csf_heap_context_allocator_free(ctx_alloc, + heap->gpu_va); + } + } + + if (unlikely(err)) { + kfree(heap); + } else { + struct kbase_csf_tiler_heap_chunk const *first_chunk = + list_first_entry(&heap->chunks_list, + struct kbase_csf_tiler_heap_chunk, link); + + list_add(&heap->link, &kctx->csf.tiler_heaps.list); + + *heap_gpu_va = heap->gpu_va; + *first_chunk_va = first_chunk->gpu_va; + + dev_dbg(kctx->kbdev->dev, "Created tiler heap 0x%llX\n", + heap->gpu_va); + } + + mutex_unlock(&kctx->csf.tiler_heaps.lock); + + return err; +} + +int kbase_csf_tiler_heap_term(struct kbase_context *const kctx, + u64 const heap_gpu_va) +{ + int err = 0; + struct kbase_csf_tiler_heap *heap = NULL; + + mutex_lock(&kctx->csf.tiler_heaps.lock); + + heap = find_tiler_heap(kctx, heap_gpu_va); + if (likely(heap)) + delete_heap(heap); + else + err = -EINVAL; + + mutex_unlock(&kctx->csf.tiler_heaps.lock); + + return err; +} + +/** + * alloc_new_chunk - Allocate a new chunk for the tiler heap. + * + * This function will allocate a new chunk for the chunked tiler heap depending + * on the settings provided by userspace when the heap was created and the + * heap's statistics (like number of render passes in-flight). + * + * @heap: Pointer to the tiler heap. + * @nr_in_flight: Number of render passes that are in-flight, must not be zero. + * @new_chunk_ptr: Where to store the GPU virtual address & size of the new + * chunk allocated for the heap. + * + * Return: 0 if a new chunk was allocated otherwise an appropriate negative + * error code. + */ +static int alloc_new_chunk(struct kbase_csf_tiler_heap *heap, + u32 nr_in_flight, u64 *new_chunk_ptr) +{ + int err = -ENOMEM; + + lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); + + if (!nr_in_flight) + return -EINVAL; + + if ((nr_in_flight <= heap->target_in_flight) && + (heap->chunk_count < heap->max_chunks)) { + /* Not exceeded the target number of render passes yet so be + * generous with memory. + */ + err = create_chunk(heap, false); + + if (likely(!err)) { + struct kbase_csf_tiler_heap_chunk *new_chunk = + get_last_chunk(heap); + if (!WARN_ON(!new_chunk)) { + *new_chunk_ptr = + encode_chunk_ptr(heap->chunk_size, + new_chunk->gpu_va); + return 0; + } + } + } + + /* A new chunk wasn't allocated this time, check if the allocation can + * be retried later. + */ + if (nr_in_flight > 1) { + /* Can retry as there are some ongoing fragment + * jobs which are expected to free up chunks. + */ + err = -EBUSY; + } + + return err; +} + +int kbase_csf_tiler_heap_alloc_new_chunk(struct kbase_context *kctx, + u64 gpu_heap_va, u32 nr_in_flight, u64 *new_chunk_ptr) +{ + struct kbase_csf_tiler_heap *heap; + int err = -EINVAL; + + mutex_lock(&kctx->csf.tiler_heaps.lock); + + heap = find_tiler_heap(kctx, gpu_heap_va); + + if (likely(heap)) { + err = alloc_new_chunk(heap, nr_in_flight, + new_chunk_ptr); + } + + mutex_unlock(&kctx->csf.tiler_heaps.lock); + + return err; +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.h new file mode 100755 index 000000000000..1a4729df6ca3 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.h @@ -0,0 +1,113 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_TILER_HEAP_H_ +#define _KBASE_CSF_TILER_HEAP_H_ + +#include + +/** + * kbase_csf_tiler_heap_context_init - Initialize the tiler heaps context for a + * GPU address space + * + * @kctx: Pointer to the kbase context being initialized. + * + * Return: 0 if successful or a negative error code on failure. + */ +int kbase_csf_tiler_heap_context_init(struct kbase_context *kctx); + +/** + * kbase_csf_tiler_heap_context_term - Terminate the tiler heaps context for a + * GPU address space + * + * This function deletes any chunked tiler heaps that weren't deleted before + * context termination. + * + * @kctx: Pointer to the kbase context being terminated. + */ +void kbase_csf_tiler_heap_context_term(struct kbase_context *kctx); + +/** + * kbase_csf_tiler_heap_init - Initialize a chunked tiler memory heap. + * + * @kctx: Pointer to the kbase context in which to allocate resources for the + * tiler heap. + * @chunk_size: Size of each chunk, in bytes. Must be page-aligned. + * @initial_chunks: The initial number of chunks to allocate. Must not be + * zero or greater than @max_chunks. + * @max_chunks: The maximum number of chunks that the heap should be allowed + * to use. Must not be less than @initial_chunks. + * @target_in_flight: Number of render-passes that the driver should attempt to + * keep in flight for which allocation of new chunks is + * allowed. Must not be zero. + * @gpu_heap_va: Where to store the GPU virtual address of the context that was + * set up for the tiler heap. + * @first_chunk_va: Where to store the GPU virtual address of the first chunk + * allocated for the heap. This points to the header of the + * heap chunk and not to the low address of free memory in it. + * + * Return: 0 if successful or a negative error code on failure. + */ +int kbase_csf_tiler_heap_init(struct kbase_context *kctx, + u32 chunk_size, u32 initial_chunks, u32 max_chunks, + u16 target_in_flight, u64 *gpu_heap_va, + u64 *first_chunk_va); + +/** + * kbasep_cs_tiler_heap_term - Terminate a chunked tiler memory heap. + * + * This function will terminate a chunked tiler heap and cause all the chunks + * (initial and those added during out-of-memory processing) to be freed. + * It is the caller's responsibility to ensure no further operations on this + * heap will happen before calling this function. + * + * @kctx: Pointer to the kbase context in which the tiler heap was initialized. + * @gpu_heap_va: The GPU virtual address of the context that was set up for the + * tiler heap. + * + * Return: 0 if successful or a negative error code on failure. + */ +int kbase_csf_tiler_heap_term(struct kbase_context *kctx, u64 gpu_heap_va); + +/** + * kbase_csf_tiler_heap_alloc_new_chunk - Allocate a new chunk for tiler heap. + * + * This function will allocate a new chunk for the chunked tiler heap depending + * on the settings provided by userspace when the heap was created and the + * heap's statistics (like number of render passes in-flight). + * It would return an appropriate error code if a new chunk couldn't be + * allocated. + * + * @kctx: Pointer to the kbase context in which the tiler heap was initialized. + * @gpu_heap_va: GPU virtual address of the heap context. + * @nr_in_flight: Number of render passes that are in-flight, must not be zero. + * @new_chunk_ptr: Where to store the GPU virtual address & size of the new + * chunk allocated for the heap. + * + * Return: 0 if a new chunk was allocated otherwise an appropriate negative + * error code (like -EBUSY when a free chunk is expected to be + * available upon completion of a render pass and -EINVAL when + * invalid value was passed for one of the argument). + */ +int kbase_csf_tiler_heap_alloc_new_chunk(struct kbase_context *kctx, + u64 gpu_heap_va, u32 nr_in_flight, u64 *new_chunk_ptr); +#endif diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.c new file mode 100755 index 000000000000..5d744b81fe4a --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.c @@ -0,0 +1,107 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_csf_tiler_heap_debugfs.h" +#include "mali_kbase_csf_tiler_heap_def.h" +#include +#include + +#ifdef CONFIG_DEBUG_FS + +/** + * kbasep_csf_tiler_heap_debugfs_show() - Print tiler heap information for per context + * + * @file: The seq_file for printing to + * @data: The debugfs dentry private data, a pointer to kbase_context + * + * Return: Negative error code or 0 on success. + */ +static int kbasep_csf_tiler_heap_debugfs_show(struct seq_file *file, void *data) +{ + struct kbase_context *kctx = file->private; + struct kbase_csf_tiler_heap_context *tiler_heaps_p = &kctx->csf.tiler_heaps; + struct kbase_csf_tiler_heap *heap; + struct kbase_csf_tiler_heap_chunk *chunk; + + seq_printf(file, "MALI_CSF_TILER_HEAP_DEBUGFS_VERSION: v%u\n", MALI_CSF_TILER_HEAP_DEBUGFS_VERSION); + + mutex_lock(&tiler_heaps_p->lock); + + list_for_each_entry(heap, &tiler_heaps_p->list, link) { + if (heap->kctx != kctx) + continue; + + seq_printf(file, "HEAP(gpu_va = 0x%llx):\n", heap->gpu_va); + seq_printf(file, "\tchunk_size = %u\n", heap->chunk_size); + seq_printf(file, "\tchunk_count = %u\n", heap->chunk_count); + seq_printf(file, "\tmax_chunks = %u\n", heap->max_chunks); + seq_printf(file, "\ttarget_in_flight = %u\n", heap->target_in_flight); + + list_for_each_entry(chunk, &heap->chunks_list, link) + seq_printf(file, "\t\tchunk gpu_va = 0x%llx\n", + chunk->gpu_va); + } + + mutex_unlock(&tiler_heaps_p->lock); + + return 0; +} + +static int kbasep_csf_tiler_heap_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_csf_tiler_heap_debugfs_show, in->i_private); +} + +static const struct file_operations kbasep_csf_tiler_heap_debugfs_fops = { + .open = kbasep_csf_tiler_heap_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void kbase_csf_tiler_heap_debugfs_init(struct kbase_context *kctx) +{ + struct dentry *file; + + if (WARN_ON(!kctx || IS_ERR_OR_NULL(kctx->kctx_dentry))) + return; + + file = debugfs_create_file("tiler_heaps", 0444, kctx->kctx_dentry, + kctx, &kbasep_csf_tiler_heap_debugfs_fops); + + if (IS_ERR_OR_NULL(file)) { + dev_warn(kctx->kbdev->dev, + "Unable to create tiler heap debugfs entry"); + } +} + + +#else +/* + * Stub functions for when debugfs is disabled + */ +void kbase_csf_tiler_heap_debugfs_init(struct kbase_context *kctx) +{ +} + +#endif /* CONFIG_DEBUG_FS */ + diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.h new file mode 100755 index 000000000000..44c580d82068 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_debugfs.h @@ -0,0 +1,38 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_TILER_HEAP_DEBUGFS_H_ +#define _KBASE_CSF_TILER_HEAP_DEBUGFS_H_ + +/* Forward declaration */ +struct kbase_context; + +#define MALI_CSF_TILER_HEAP_DEBUGFS_VERSION 0 + +/** + * kbase_csf_tiler_heap_debugfs_init() - Create a debugfs entry for per context tiler heap + * + * @kctx: The kbase_context for which to create the debugfs entry + */ +void kbase_csf_tiler_heap_debugfs_init(struct kbase_context *kctx); + +#endif /* _KBASE_CSF_TILER_HEAP_DEBUGFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_def.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_def.h new file mode 100755 index 000000000000..1f9e208904a9 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_def.h @@ -0,0 +1,112 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_TILER_HEAP_DEF_H_ +#define _KBASE_CSF_TILER_HEAP_DEF_H_ + +#include + +/* Size of a tiler heap chunk header, in bytes. */ +#define CHUNK_HDR_SIZE ((size_t)64) + +/* Bit-position of the next chunk's size when stored in a chunk header. */ +#define CHUNK_HDR_NEXT_SIZE_POS (0) + +/* Bit-position of the next chunk's address when stored in a chunk header. */ +#define CHUNK_HDR_NEXT_ADDR_POS (12) + +/* Bitmask of the next chunk's size when stored in a chunk header. */ +#define CHUNK_HDR_NEXT_SIZE_MASK (((u64)1 << CHUNK_HDR_NEXT_ADDR_POS) - 1u) + +/* Bitmask of the address of the next chunk when stored in a chunk header. */ +#define CHUNK_HDR_NEXT_ADDR_MASK (~CHUNK_HDR_NEXT_SIZE_MASK) + +/* Right-shift before storing the next chunk's size in a chunk header. */ +#define CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT (12) + +/* Right-shift before storing the next chunk's address in a chunk header. */ +#define CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT (12) + +/* Bitmask of valid chunk sizes. This is also the maximum chunk size, in bytes. + */ +#define CHUNK_SIZE_MASK \ + ((CHUNK_HDR_NEXT_SIZE_MASK >> CHUNK_HDR_NEXT_SIZE_POS) << \ + CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT) + +/* Bitmask of valid chunk addresses. This is also the highest address. */ +#define CHUNK_ADDR_MASK \ + ((CHUNK_HDR_NEXT_ADDR_MASK >> CHUNK_HDR_NEXT_ADDR_POS) << \ + CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT) + +/** + * struct kbase_csf_tiler_heap_chunk - A tiler heap chunk managed by the kernel + * + * Chunks are allocated upon initialization of a tiler heap or in response to + * out-of-memory events from the firmware. Chunks are always fully backed by + * physical memory to avoid the overhead of processing GPU page faults. The + * allocated GPU memory regions are linked together independent of the list of + * kernel objects of this type. + * + * @link: Link to this chunk in a list of chunks belonging to a + * @kbase_csf_tiler_heap. + * @region: Pointer to the GPU memory region allocated for the chunk. + * @gpu_va: GPU virtual address of the start of the memory region. + * This points to the header of the chunk and not to the low address + * of free memory within it. + */ +struct kbase_csf_tiler_heap_chunk { + struct list_head link; + struct kbase_va_region *region; + u64 gpu_va; +}; + +/** + * struct kbase_csf_tiler_heap - A tiler heap managed by the kernel + * + * @kctx: Pointer to the kbase context with which this heap is + * associated. + * @link: Link to this heap in a list of tiler heaps belonging to + * the @kbase_csf_tiler_heap_context. + * @chunk_size: Size of each chunk, in bytes. Must be page-aligned. + * @chunk_count: The number of chunks currently allocated. Must not be + * zero or greater than @max_chunks. + * @max_chunks: The maximum number of chunks that the heap should be + * allowed to use. Must not be less than @chunk_count. + * @target_in_flight: Number of render-passes that the driver should attempt + * to keep in flight for which allocation of new chunks is + * allowed. Must not be zero. + * @gpu_va: The GPU virtual address of the heap context structure that + * was allocated for the firmware. This is also used to + * uniquely identify the heap. + * @chunks_list: Linked list of allocated chunks. + */ +struct kbase_csf_tiler_heap { + struct kbase_context *kctx; + struct list_head link; + u32 chunk_size; + u32 chunk_count; + u32 max_chunks; + u16 target_in_flight; + u64 gpu_va; + struct list_head chunks_list; +}; +#endif /* !_KBASE_CSF_TILER_HEAP_DEF_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c new file mode 100755 index 000000000000..495ff2850500 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c @@ -0,0 +1,169 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include + +#include "mali_kbase.h" +#include "mali_kbase_config_defaults.h" +#include "mali_kbase_csf_firmware.h" +#include "mali_kbase_csf_timeout.h" +#include "backend/gpu/mali_kbase_pm_internal.h" + +/** + * set_timeout - set a new global progress timeout. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * @timeout: the maximum number of GPU cycles without forward progress to allow + * to elapse before terminating a GPU command queue group. + * + * Return: 0 on success, or negative on failure + * (e.g. -ERANGE if the requested timeout is too large). + */ +static int set_timeout(struct kbase_device *const kbdev, u64 const timeout) +{ + if (timeout > GLB_PROGRESS_TIMER_TIMEOUT_MAX) { + dev_err(kbdev->dev, "Timeout %llu is too large.\n", timeout); + return -ERANGE; + } + + dev_dbg(kbdev->dev, "New progress timeout: %llu cycles\n", timeout); + + atomic64_set(&kbdev->csf.progress_timeout, timeout); + + return 0; +} + +/** + * progress_timeout_store - Store the progress_timeout device attribute. + * @dev: The device that has the attribute. + * @attr: The attributes of the sysfs file. + * @buf: The value written to the sysfs file. + * @count: The number of bytes written to the sysfs file. + * + * This function is called when the progress_timeout sysfs file is written to. + * It checks the data written, and if valid updates the progress timeout value. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t progress_timeout_store(struct device * const dev, + struct device_attribute * const attr, const char * const buf, + size_t const count) +{ + struct kbase_device *const kbdev = dev_get_drvdata(dev); + int err; + u64 timeout; + + if (!kbdev) + return -ENODEV; + + err = kstrtou64(buf, 0, &timeout); + if (err) { + dev_err(kbdev->dev, + "Couldn't process progress_timeout write operation.\n" + "Use format \n"); + return err; + } + + err = set_timeout(kbdev, timeout); + if (!err) { + kbase_csf_scheduler_pm_active(kbdev); + + err = kbase_pm_wait_for_desired_state(kbdev); + if (!err) + err = kbase_csf_firmware_set_timeout(kbdev, timeout); + + kbase_csf_scheduler_pm_idle(kbdev); + } + + if (err) + return err; + + return count; +} + +/** + * progress_timeout_show - Show the progress_timeout device attribute. + * @dev: The device that has the attribute. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the global timeout. + * + * This function is called to get the progress timeout value. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t progress_timeout_show(struct device * const dev, + struct device_attribute * const attr, char * const buf) +{ + struct kbase_device *const kbdev = dev_get_drvdata(dev); + int err; + + if (!kbdev) + return -ENODEV; + + err = scnprintf(buf, PAGE_SIZE, "%llu\n", kbase_csf_timeout_get(kbdev)); + + return err; + +} + +static DEVICE_ATTR(progress_timeout, 0644, progress_timeout_show, + progress_timeout_store); + +int kbase_csf_timeout_init(struct kbase_device *const kbdev) +{ + u64 timeout = DEFAULT_PROGRESS_TIMEOUT; + int err; + +#ifdef CONFIG_OF + err = of_property_read_u64(kbdev->dev->of_node, + "progress_timeout", &timeout); + if (!err) + dev_info(kbdev->dev, "Found progress_timeout = %llu in Devicetree\n", + timeout); +#endif + + err = set_timeout(kbdev, timeout); + if (err) + return err; + + err = sysfs_create_file(&kbdev->dev->kobj, + &dev_attr_progress_timeout.attr); + if (err) + dev_err(kbdev->dev, "SysFS file creation failed\n"); + + return err; +} + +void kbase_csf_timeout_term(struct kbase_device * const kbdev) +{ + sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_progress_timeout.attr); +} + +u64 kbase_csf_timeout_get(struct kbase_device *const kbdev) +{ + return atomic64_read(&kbdev->csf.progress_timeout); +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.h new file mode 100755 index 000000000000..d0156c09a60f --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.h @@ -0,0 +1,69 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_TIMEOUT_H_ +#define _KBASE_CSF_TIMEOUT_H_ + +struct kbase_device; + +/** + * kbase_csf_timeout_init - Initialize the progress timeout. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. Must be zero-initialized. + * + * The progress timeout is the number of GPU clock cycles allowed to elapse + * before the driver terminates a GPU command queue group in which a task is + * making no forward progress on an endpoint (e.g. a shader core). This function + * determines the initial value and also creates a sysfs file to allow the + * timeout to be reconfigured later. + * + * Reconfigures the global firmware interface to enable the current timeout. + * + * Return: 0 on success, or negative on failure. + */ +int kbase_csf_timeout_init(struct kbase_device *kbdev); + +/** + * kbase_csf_timeout_term - Terminate the progress timeout. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * Removes the sysfs file which allowed the timeout to be reconfigured. + * Does nothing if called on a zero-initialized object. + */ +void kbase_csf_timeout_term(struct kbase_device *kbdev); + +/** + * kbase_csf_timeout_get - get the current global progress timeout. + * + * @kbdev: Instance of a GPU platform device that implements a command + * stream front-end interface. + * + * Return: the maximum number of GPU cycles that is allowed to elapse without + * forward progress before the driver terminates a GPU command queue + * group. + */ +u64 kbase_csf_timeout_get(struct kbase_device *const kbdev); + +#endif /* _KBASE_CSF_TIMEOUT_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c new file mode 100755 index 000000000000..5079a8e5af8c --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.c @@ -0,0 +1,555 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_csf_tl_reader.h" + +#include "mali_kbase_csf_trace_buffer.h" +#include "mali_kbase_reset_gpu.h" + +#include "tl/mali_kbase_tlstream.h" +#include "tl/mali_kbase_tl_serialize.h" +#include "tl/mali_kbase_tracepoints.h" + +#include "mali_kbase_pm.h" +#include "mali_kbase_hwaccess_time.h" + +#include +#include +#include + +#ifdef CONFIG_DEBUG_FS +#include "tl/mali_kbase_timeline_priv.h" +#include + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) +#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE +#endif +#endif + +/** Name of the CSFFW timeline tracebuffer. */ +#define KBASE_CSFFW_TRACEBUFFER_NAME "timeline" +/** Name of the timeline header metatadata */ +#define KBASE_CSFFW_TIMELINE_HEADER_NAME "timeline_header" + +/** + * CSFFW timeline message. + * + * @msg_id: Message ID. + * @timestamp: Timestamp of the event. + * @cycle_counter: Cycle number of the event. + * + * Contain fields that are common for all CSFFW timeline messages. + */ +struct kbase_csffw_tl_message { + u32 msg_id; + u64 timestamp; + u64 cycle_counter; +} __packed __aligned(4); + +#ifdef CONFIG_DEBUG_FS +static int kbase_csf_tl_debugfs_poll_interval_read(void *data, u64 *val) +{ + struct kbase_device *kbdev = (struct kbase_device *)data; + struct kbase_csf_tl_reader *self = &kbdev->timeline->csf_tl_reader; + + *val = self->timer_interval; + + return 0; +} + +static int kbase_csf_tl_debugfs_poll_interval_write(void *data, u64 val) +{ + struct kbase_device *kbdev = (struct kbase_device *)data; + struct kbase_csf_tl_reader *self = &kbdev->timeline->csf_tl_reader; + + if (val > KBASE_CSF_TL_READ_INTERVAL_MAX || val < KBASE_CSF_TL_READ_INTERVAL_MIN) { + return -EINVAL; + } + + self->timer_interval = (u32)val; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(kbase_csf_tl_poll_interval_fops, + kbase_csf_tl_debugfs_poll_interval_read, + kbase_csf_tl_debugfs_poll_interval_write, "%llu\n"); + +void kbase_csf_tl_reader_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("csf_tl_poll_interval_in_ms", S_IRUGO | S_IWUSR, + kbdev->debugfs_instr_directory, kbdev, + &kbase_csf_tl_poll_interval_fops); +} +#endif + +/** + * get_cpu_gpu_time() - Get current CPU and GPU timestamps. + * + * @kbdev: Kbase device. + * @cpu_ts: Output CPU timestamp. + * @gpu_ts: Output GPU timestamp. + * @gpu_cycle: Output GPU cycle counts. + */ +static void get_cpu_gpu_time( + struct kbase_device *kbdev, + u64 *cpu_ts, + u64 *gpu_ts, + u64 *gpu_cycle) +{ + struct timespec64 ts; + + kbase_pm_context_active(kbdev); + kbase_backend_get_gpu_time(kbdev, gpu_cycle, gpu_ts, &ts); + kbase_pm_context_idle(kbdev); + + if (cpu_ts) + *cpu_ts = ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; +} + +/** + * kbase_ts_converter_init() - Initialize system timestamp converter. + * + * @self: System Timestamp Converter instance. + * + * Return: Zero on success, -1 otherwise. + */ +static int kbase_ts_converter_init( + struct kbase_ts_converter *self, + struct kbase_device *kbdev) +{ + u64 cpu_ts = 0; + u64 gpu_ts = 0; + u64 freq; + u64 common_factor; + + get_cpu_gpu_time(kbdev, &cpu_ts, &gpu_ts, NULL); + freq = arch_timer_get_cntfrq(); + + if (!freq) { + dev_warn(kbdev->dev, "arch_timer_get_rate() is zero!"); + return -1; + } + + common_factor = gcd(NSEC_PER_SEC, freq); + + self->multiplier = div64_u64(NSEC_PER_SEC, common_factor); + self->divisor = div64_u64(freq, common_factor); + self->offset = + cpu_ts - div64_u64(gpu_ts * self->multiplier, self->divisor); + + return 0; +} + +/** + * kbase_ts_converter_convert() - Convert GPU timestamp to CPU timestamp. + * + * @self: System Timestamp Converter instance. + * @gpu_ts: System timestamp value to converter. + * + * Return: The CPU timestamp. + */ +void kbase_ts_converter_convert( + const struct kbase_ts_converter *self, + u64 *gpu_ts) +{ + u64 old_gpu_ts = *gpu_ts; + *gpu_ts = div64_u64(old_gpu_ts * self->multiplier, + self->divisor) + self->offset; +} + +/** + * tl_reader_overflow_notify() - Emit stream overflow tracepoint. + * + * @self: CSFFW TL Reader instance. + * @msg_buf_start: Start of the message. + * @msg_buf_end: End of the message buffer. + */ +static void tl_reader_overflow_notify( + const struct kbase_csf_tl_reader *self, + u8 *const msg_buf_start, + u8 *const msg_buf_end) +{ + struct kbase_device *kbdev = self->kbdev; + struct kbase_csffw_tl_message message = {0}; + + /* Reuse the timestamp and cycle count from current event if possible */ + if (msg_buf_start + sizeof(message) <= msg_buf_end) + memcpy(&message, msg_buf_start, sizeof(message)); + + KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( + kbdev, message.timestamp, message.cycle_counter); +} + +/** + * tl_reader_overflow_check() - Check if an overflow has happened + * + * @self: CSFFW TL Reader instance. + * @event_id: Incoming event id. + * + * Return: True, if an overflow has happened, False otherwise. + */ +static bool tl_reader_overflow_check( + struct kbase_csf_tl_reader *self, + u16 event_id) +{ + struct kbase_device *kbdev = self->kbdev; + bool has_overflow = false; + + /* 0 is a special event_id and reserved for the very first tracepoint + * after reset, we should skip overflow check when reset happened. + */ + if (event_id != 0) { + has_overflow = self->got_first_event + && self->expected_event_id != event_id; + + if (has_overflow) + dev_warn(kbdev->dev, + "CSFFW overflow, event_id: %u, expected: %u.", + event_id, self->expected_event_id); + } + + self->got_first_event = true; + self->expected_event_id = event_id + 1; + /* When event_id reaches its max value, it skips 0 and wraps to 1. */ + if (self->expected_event_id == 0) + self->expected_event_id++; + + return has_overflow; +} + +/** + * tl_reader_reset() - Reset timeline tracebuffer reader state machine. + * + * @self: CSFFW TL Reader instance. + * + * Reset the reader to the default state, i.e. set all the + * mutable fields to zero. + */ +static void tl_reader_reset(struct kbase_csf_tl_reader *self) +{ + self->got_first_event = false; + self->is_active = false; + self->expected_event_id = 0; + self->tl_header.btc = 0; +} + +void kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader *self) +{ + struct kbase_device *kbdev = self->kbdev; + struct kbase_tlstream *stream = self->stream; + + u8 *read_buffer = self->read_buffer; + const size_t read_buffer_size = sizeof(self->read_buffer); + + u32 bytes_read; + u8 *csffw_data_begin; + u8 *csffw_data_end; + u8 *csffw_data_it; + + unsigned long flags; + + spin_lock_irqsave(&self->read_lock, flags); + + /* If not running, early exit. */ + if (!self->is_active) { + spin_unlock_irqrestore(&self->read_lock, flags); + return; + } + + /* Copying the whole buffer in a single shot. We assume + * that the buffer will not contain partially written messages. + */ + bytes_read = kbase_csf_firmware_trace_buffer_read_data( + self->trace_buffer, read_buffer, read_buffer_size); + csffw_data_begin = read_buffer; + csffw_data_end = read_buffer + bytes_read; + + for (csffw_data_it = csffw_data_begin; + csffw_data_it < csffw_data_end;) { + u32 event_header; + u16 event_id; + u16 event_size; + unsigned long acq_flags; + char *buffer; + + /* Can we safely read event_id? */ + if (csffw_data_it + sizeof(event_header) > csffw_data_end) { + dev_warn( + kbdev->dev, + "Unable to parse CSFFW tracebuffer event header."); + break; + } + + /* Read and parse the event header. */ + memcpy(&event_header, csffw_data_it, sizeof(event_header)); + event_id = (event_header >> 0) & 0xFFFF; + event_size = (event_header >> 16) & 0xFFFF; + csffw_data_it += sizeof(event_header); + + /* Detect if an overflow has happened. */ + if (tl_reader_overflow_check(self, event_id)) + tl_reader_overflow_notify(self, + csffw_data_it, + csffw_data_end); + + /* Can we safely read the message body? */ + if (csffw_data_it + event_size > csffw_data_end) { + dev_warn(kbdev->dev, + "event_id: %u, can't read with event_size: %u.", + event_id, event_size); + break; + } + + /* Convert GPU timestamp to CPU timestamp. */ + { + struct kbase_csffw_tl_message *msg = + (struct kbase_csffw_tl_message *) csffw_data_it; + kbase_ts_converter_convert( + &self->ts_converter, + &msg->timestamp); + } + + /* Copy the message out to the tl_stream. */ + buffer = kbase_tlstream_msgbuf_acquire( + stream, event_size, &acq_flags); + kbasep_serialize_bytes(buffer, 0, csffw_data_it, event_size); + kbase_tlstream_msgbuf_release(stream, acq_flags); + csffw_data_it += event_size; + } + + spin_unlock_irqrestore(&self->read_lock, flags); +} + +static void kbasep_csf_tl_reader_read_callback(struct timer_list *timer) +{ + struct kbase_csf_tl_reader *self = + container_of(timer, struct kbase_csf_tl_reader, read_timer); + + int rcode; + + kbase_csf_tl_reader_flush_buffer(self); + + rcode = mod_timer(&self->read_timer, + jiffies + msecs_to_jiffies(self->timer_interval)); + + CSTD_UNUSED(rcode); +} + +/** + * tl_reader_init_late() - Late CSFFW TL Reader initialization. + * + * @self: CSFFW TL Reader instance. + * @kbdev: Kbase device. + * + * Late initialization is done once at kbase_csf_tl_reader_start() time. + * This is because the firmware image is not parsed + * by the kbase_csf_tl_reader_init() time. + * + * Return: Zero on success, -1 otherwise. + */ +static int tl_reader_init_late( + struct kbase_csf_tl_reader *self, + struct kbase_device *kbdev) +{ + struct firmware_trace_buffer *tb; + size_t hdr_size = 0; + const char *hdr = NULL; + + if (self->kbdev) + return 0; + + tb = kbase_csf_firmware_get_trace_buffer( + kbdev, KBASE_CSFFW_TRACEBUFFER_NAME); + hdr = kbase_csf_firmware_get_timeline_metadata( + kbdev, KBASE_CSFFW_TIMELINE_HEADER_NAME, &hdr_size); + + if (!tb) { + dev_warn( + kbdev->dev, + "'%s' tracebuffer is not present in the firmware image.", + KBASE_CSFFW_TRACEBUFFER_NAME); + return -1; + } + + if (!hdr) { + dev_warn( + kbdev->dev, + "'%s' timeline metadata is not present in the firmware image.", + KBASE_CSFFW_TIMELINE_HEADER_NAME); + return -1; + } + + if (kbase_ts_converter_init(&self->ts_converter, kbdev)) { + return -1; + } + + self->kbdev = kbdev; + self->trace_buffer = tb; + self->tl_header.data = hdr; + self->tl_header.size = hdr_size; + + return 0; +} + +/** + * tl_reader_update_enable_bit() - Update the first bit of a CSFFW tracebuffer. + * + * @self: CSFFW TL Reader instance. + * @value: The value to set. + * + * Update the first bit of a CSFFW tracebufer and then reset the GPU. + * This is to make these changes visible to the MCU. + * + * Return: 0 on success, -EAGAIN if a GPU reset was in progress. + */ +static int tl_reader_update_enable_bit( + struct kbase_csf_tl_reader *self, + bool value) +{ + struct kbase_device *kbdev = self->kbdev; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* If there is already a GPU reset pending then inform + * the User to retry the update. + */ + if (kbase_reset_gpu_silent(kbdev)) { + spin_unlock_irqrestore( + &kbdev->hwaccess_lock, flags); + dev_warn( + kbdev->dev, + "GPU reset already in progress when enabling firmware timeline."); + return -EAGAIN; + } + + /* GPU reset request has been placed, now update the + * firmware image. GPU reset will take place only after + * hwaccess_lock is released. + */ + kbase_csf_firmware_trace_buffer_update_trace_enable_bit( + self->trace_buffer, 0, value); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return 0; +} + +void kbase_csf_tl_reader_init(struct kbase_csf_tl_reader *self, + struct kbase_tlstream *stream) +{ + self->timer_interval = KBASE_CSF_TL_READ_INTERVAL_DEFAULT; + + kbase_timer_setup(&self->read_timer, + kbasep_csf_tl_reader_read_callback); + + self->stream = stream; + + /* This will be initialized by tl_reader_init_late() */ + self->kbdev = NULL; + self->trace_buffer = NULL; + self->tl_header.data = NULL; + self->tl_header.size = 0; + + spin_lock_init(&self->read_lock); + + tl_reader_reset(self); +} + +void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self) +{ + del_timer_sync(&self->read_timer); +} + +int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, + struct kbase_device *kbdev) +{ + int rcode; + + /* If already running, early exit. */ + if (self->is_active) + return 0; + + if (tl_reader_init_late(self, kbdev)) { +#if defined(CONFIG_MALI_BIFROST_NO_MALI) + dev_warn( + kbdev->dev, + "CSFFW timeline is not available for MALI_BIFROST_NO_MALI builds!"); + return 0; +#else + return -EINVAL; +#endif + } + + tl_reader_reset(self); + + self->is_active = true; + /* Set bytes to copy to the header size. This is to trigger copying + * of the header to the user space. + */ + self->tl_header.btc = self->tl_header.size; + + /* Enable the tracebuffer on the CSFFW side. */ + rcode = tl_reader_update_enable_bit(self, true); + if (rcode != 0) + return rcode; + + rcode = mod_timer(&self->read_timer, + jiffies + msecs_to_jiffies(self->timer_interval)); + + return 0; +} + +void kbase_csf_tl_reader_stop(struct kbase_csf_tl_reader *self) +{ + unsigned long flags; + + /* If is not running, early exit. */ + if (!self->is_active) + return; + + /* Disable the tracebuffer on the CSFFW side. */ + tl_reader_update_enable_bit(self, false); + + del_timer_sync(&self->read_timer); + + spin_lock_irqsave(&self->read_lock, flags); + + tl_reader_reset(self); + + spin_unlock_irqrestore(&self->read_lock, flags); +} + +void kbase_csf_tl_reader_reset(struct kbase_csf_tl_reader *self) +{ + u64 gpu_cycle = 0; + struct kbase_device *kbdev = self->kbdev; + + if (!kbdev) + return; + + kbase_csf_tl_reader_flush_buffer(self); + + get_cpu_gpu_time(kbdev, NULL, NULL, &gpu_cycle); + KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET(kbdev, gpu_cycle); +} diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.h new file mode 100755 index 000000000000..f5ce9d629f55 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tl_reader.h @@ -0,0 +1,181 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSFFW_TL_READER_H_ +#define _KBASE_CSFFW_TL_READER_H_ + +#include +#include +#include + +/** The number of pages used for CSFFW trace buffer. Can be tweaked. */ +#define KBASE_CSF_TL_BUFFER_NR_PAGES 4 +/** CSFFW Timeline read polling minimum period in milliseconds. */ +#define KBASE_CSF_TL_READ_INTERVAL_MIN 20 +/** CSFFW Timeline read polling default period in milliseconds. */ +#define KBASE_CSF_TL_READ_INTERVAL_DEFAULT 200 +/** CSFFW Timeline read polling maximum period in milliseconds. */ +#define KBASE_CSF_TL_READ_INTERVAL_MAX (60*1000) + +struct firmware_trace_buffer; +struct kbase_tlstream; +struct kbase_device; + +/** + * System timestamp to CPU timestamp converter state. + * + * @multiplier: Numerator of the converter's fraction. + * @divisor: Denominator of the converter's fraction. + * @offset: Converter's offset term. + * + * According to Generic timer spec, system timer: + * - Increments at a fixed frequency + * - Starts operating from zero + * + * Hence CPU time is a linear function of System Time. + * + * CPU_ts = alpha * SYS_ts + beta + * + * Where + * - alpha = 10^9/SYS_ts_freq + * - beta is calculated by two timer samples taken at the same time: + * beta = CPU_ts_s - SYS_ts_s * alpha + * + * Since alpha is a rational number, we minimizing possible + * rounding error by simplifying the ratio. Thus alpha is stored + * as a simple `multiplier / divisor` ratio. + * + */ +struct kbase_ts_converter { + u64 multiplier; + u64 divisor; + s64 offset; +}; + +/** + * struct kbase_csf_tl_reader - CSFFW timeline reader state. + * + * @read_timer: Timer used for periodical tracebufer reading. + * @timer_interval: Timer polling period in milliseconds. + * @stream: Timeline stream where to the tracebuffer content + * is copied. + * @kbdev: KBase device. + * @trace_buffer: CSF Firmware timeline tracebuffer. + * @tl_header.data: CSFFW Timeline header content. + * @tl_header.size: CSFFW Timeline header size. + * @tl_header.btc: CSFFW Timeline header remaining bytes to copy to + * the user space. + * @ts_converter: Timestamp converter state. + * @got_first_event: True, if a CSFFW timelime session has been enabled + * and the first event was received. + * @is_active: True, if a CSFFW timelime session has been enabled. + * @expected_event_id: The last 16 bit event ID received from CSFFW. It + * is only valid when got_first_event is true. + * @read_buffer: Temporary buffer used for CSFFW timeline data + * reading from the tracebufer. + */ +struct kbase_csf_tl_reader { + struct timer_list read_timer; + u32 timer_interval; + struct kbase_tlstream *stream; + + struct kbase_device *kbdev; + struct firmware_trace_buffer *trace_buffer; + struct { + const char *data; + size_t size; + size_t btc; + } tl_header; + struct kbase_ts_converter ts_converter; + + bool got_first_event; + bool is_active; + u16 expected_event_id; + + u8 read_buffer[PAGE_SIZE * KBASE_CSF_TL_BUFFER_NR_PAGES]; + spinlock_t read_lock; +}; + +/** + * kbase_csf_tl_reader_init() - Initialize CSFFW Timelime Stream Reader. + * + * @self: CSFFW TL Reader instance. + * @stream: Destination timeline stream. + */ +void kbase_csf_tl_reader_init(struct kbase_csf_tl_reader *self, + struct kbase_tlstream *stream); + +/** + * kbase_csf_tl_reader_term() - Terminate CSFFW Timelime Stream Reader. + * + * @self: CSFFW TL Reader instance. + */ +void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self); + +/** + * kbase_csf_tl_reader_flush_buffer() - + * Flush trace from buffer into CSFFW timeline stream. + * + * @self: CSFFW TL Reader instance. + */ + +void kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader *self); + +/** + * kbase_csf_tl_reader_start() - + * Start asynchronous copying of CSFFW timeline stream. + * + * @self: CSFFW TL Reader instance. + * @kbdev: Kbase device. + * + * Return: zero on success, a negative error code otherwise. + */ +int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, + struct kbase_device *kbdev); + +/** + * kbase_csf_tl_reader_stop() - + * Stop asynchronous copying of CSFFW timeline stream. + * + * @self: CSFFW TL Reader instance. + */ +void kbase_csf_tl_reader_stop(struct kbase_csf_tl_reader *self); + +#ifdef CONFIG_DEBUG_FS +/** + * kbase_csf_tl_reader_debugfs_init() - + * Initialize debugfs for CSFFW Timelime Stream Reader. + * + * @kbdev: Kbase device. + */ +void kbase_csf_tl_reader_debugfs_init(struct kbase_device *kbdev); +#endif + +/** + * kbase_csf_tl_reader_reset() - + * Reset CSFFW timeline reader, it should be called before reset CSFFW. + * + * @self: CSFFW TL Reader instance. + */ +void kbase_csf_tl_reader_reset(struct kbase_csf_tl_reader *self); + +#endif /* _KBASE_CSFFW_TL_READER_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c new file mode 100755 index 000000000000..4d68766b8b9a --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c @@ -0,0 +1,623 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase.h" +#include "mali_kbase_defs.h" +#include "mali_kbase_csf_firmware.h" +#include "mali_kbase_csf_trace_buffer.h" +#include "mali_kbase_reset_gpu.h" +#include "mali_kbase_csf_tl_reader.h" + +#include +#include + +/** + * struct firmware_trace_buffer - Trace Buffer within the MCU firmware + * + * The firmware relays information to the host by writing on memory buffers + * which are allocated and partially configured by the host. These buffers + * are called Trace Buffers: each of them has a specific purpose and is + * identified by a name and a set of memory addresses where the host can + * set pointers to host-allocated structures. + * + * @kbdev: Pointer to the Kbase device. + * @node: List head linking all trace buffers to + * kbase_device:csf.firmware_trace_buffers + * @data_mapping: MCU shared memory mapping used for the data buffer. + * @type: The type of the trace buffer. + * @trace_enable_entry_count: Number of Trace Enable bits. + * @gpu_va: Structure containing all the Firmware addresses + * that are accessed by the MCU. + * @size_address: The address where the MCU shall read the size of + * the data buffer. + * @insert_address: The address that shall be dereferenced by the MCU + * to write the Insert offset. + * @extract_address: The address that shall be dereferenced by the MCU + * to read the Extract offset. + * @data_address: The address that shall be dereferenced by the MCU + * to write the Trace Buffer. + * @trace_enable: The address where the MCU shall read the array of + * Trace Enable bits describing which trace points + * and features shall be enabled. + * @cpu_va: Structure containing CPU addresses of variables which + * are permanently mapped on the CPU address space. + * @insert_cpu_va: CPU virtual address of the Insert variable. + * @extract_cpu_va: CPU virtual address of the Extract variable. + * @num_pages: Size of the data buffer, in pages. + * @trace_enable_init_mask: Initial value for the trace enable bit mask. + * @name: NULL terminated string which contains the name of the trace buffer. + */ +struct firmware_trace_buffer { + struct kbase_device *kbdev; + struct list_head node; + struct kbase_csf_mapping data_mapping; + u32 type; + u32 trace_enable_entry_count; + struct gpu_va { + u32 size_address; + u32 insert_address; + u32 extract_address; + u32 data_address; + u32 trace_enable; + } gpu_va; + struct cpu_va { + u32 *insert_cpu_va; + u32 *extract_cpu_va; + } cpu_va; + u32 num_pages; + u32 trace_enable_init_mask[CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX]; + char name[1]; /* this field must be last */ +}; + +/** + * struct firmware_trace_buffer_data - Configuration data for trace buffers + * + * Describe how to set up a trace buffer interface. + * Trace buffers are identified by name and they require a data buffer and + * an initial mask of values for the trace enable bits. + * + * @name: Name identifier of the trace buffer + * @trace_enable_init_mask: Initial value to assign to the trace enable bits + * @size: Size of the data buffer to allocate for the trace buffer, in pages. + * The size of a data buffer must always be a power of 2. + */ +struct firmware_trace_buffer_data { + char name[64]; + u32 trace_enable_init_mask[CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX]; + size_t size; +}; + +/** + * Table of configuration data for trace buffers. + * + * This table contains the configuration data for the trace buffers that are + * expected to be parsed from the firmware. + */ +static const struct firmware_trace_buffer_data +trace_buffer_data[] = { +#ifndef MALI_KBASE_BUILD + { "fwutf", {0}, 1 }, +#endif + { FW_TRACE_BUF_NAME, {0}, 4 }, + { "benchmark", {0}, 2 }, + { "timeline", {0}, KBASE_CSF_TL_BUFFER_NR_PAGES }, +}; + +int kbase_csf_firmware_trace_buffers_init(struct kbase_device *kbdev) +{ + struct firmware_trace_buffer *trace_buffer; + int ret = 0; + u32 mcu_rw_offset = 0, mcu_write_offset = 0; + const u32 cache_line_alignment = kbase_get_cache_line_alignment(kbdev); + + if (list_empty(&kbdev->csf.firmware_trace_buffers.list)) { + dev_dbg(kbdev->dev, "No trace buffers to initialise\n"); + return 0; + } + + /* GPU-readable,writable memory used for Extract variables */ + ret = kbase_csf_firmware_mcu_shared_mapping_init( + kbdev, 1, PROT_WRITE, + KBASE_REG_GPU_RD | KBASE_REG_GPU_WR, + &kbdev->csf.firmware_trace_buffers.mcu_rw); + if (ret != 0) { + dev_err(kbdev->dev, "Failed to map GPU-rw MCU shared memory\n"); + goto out; + } + + /* GPU-writable memory used for Insert variables */ + ret = kbase_csf_firmware_mcu_shared_mapping_init( + kbdev, 1, PROT_READ, KBASE_REG_GPU_WR, + &kbdev->csf.firmware_trace_buffers.mcu_write); + if (ret != 0) { + dev_err(kbdev->dev, "Failed to map GPU-writable MCU shared memory\n"); + goto out; + } + + list_for_each_entry(trace_buffer, &kbdev->csf.firmware_trace_buffers.list, node) { + u32 extract_gpu_va, insert_gpu_va, data_buffer_gpu_va, + trace_enable_size_dwords; + u32 *extract_cpu_va, *insert_cpu_va; + unsigned int i; + + /* GPU-writable data buffer for the individual trace buffer */ + ret = kbase_csf_firmware_mcu_shared_mapping_init( + kbdev, trace_buffer->num_pages, PROT_READ, KBASE_REG_GPU_WR, + &trace_buffer->data_mapping); + if (ret) { + dev_err(kbdev->dev, "Failed to map GPU-writable MCU shared memory for a trace buffer\n"); + goto out; + } + + extract_gpu_va = + (kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg->start_pfn << PAGE_SHIFT) + + mcu_rw_offset; + extract_cpu_va = (u32*)( + kbdev->csf.firmware_trace_buffers.mcu_rw.cpu_addr + + mcu_rw_offset); + insert_gpu_va = + (kbdev->csf.firmware_trace_buffers.mcu_write.va_reg->start_pfn << PAGE_SHIFT) + + mcu_write_offset; + insert_cpu_va = (u32*)( + kbdev->csf.firmware_trace_buffers.mcu_write.cpu_addr + + mcu_write_offset); + data_buffer_gpu_va = + (trace_buffer->data_mapping.va_reg->start_pfn << PAGE_SHIFT); + + /* Initialize the Extract variable */ + *extract_cpu_va = 0; + + /* Each FW address shall be mapped and set individually, as we can't + * assume anything about their location in the memory address space. + */ + kbase_csf_update_firmware_memory( + kbdev, trace_buffer->gpu_va.data_address, data_buffer_gpu_va); + kbase_csf_update_firmware_memory( + kbdev, trace_buffer->gpu_va.insert_address, insert_gpu_va); + kbase_csf_update_firmware_memory( + kbdev, trace_buffer->gpu_va.extract_address, extract_gpu_va); + kbase_csf_update_firmware_memory( + kbdev, trace_buffer->gpu_va.size_address, + trace_buffer->num_pages << PAGE_SHIFT); + + trace_enable_size_dwords = + (trace_buffer->trace_enable_entry_count + 31) >> 5; + + for (i = 0; i < trace_enable_size_dwords; i++) { + kbase_csf_update_firmware_memory( + kbdev, trace_buffer->gpu_va.trace_enable + i*4, + trace_buffer->trace_enable_init_mask[i]); + } + + /* Store CPU virtual addresses for permanently mapped variables */ + trace_buffer->cpu_va.insert_cpu_va = insert_cpu_va; + trace_buffer->cpu_va.extract_cpu_va = extract_cpu_va; + + /* Update offsets */ + mcu_write_offset += cache_line_alignment; + mcu_rw_offset += cache_line_alignment; + } + +out: + return ret; +} + +void kbase_csf_firmware_trace_buffers_term(struct kbase_device *kbdev) +{ + if (list_empty(&kbdev->csf.firmware_trace_buffers.list)) + return; + + while (!list_empty(&kbdev->csf.firmware_trace_buffers.list)) { + struct firmware_trace_buffer *trace_buffer; + + trace_buffer = list_first_entry(&kbdev->csf.firmware_trace_buffers.list, + struct firmware_trace_buffer, node); + kbase_csf_firmware_mcu_shared_mapping_term(kbdev, &trace_buffer->data_mapping); + list_del(&trace_buffer->node); + + kfree(trace_buffer); + } + + kbase_csf_firmware_mcu_shared_mapping_term( + kbdev, &kbdev->csf.firmware_trace_buffers.mcu_rw); + kbase_csf_firmware_mcu_shared_mapping_term( + kbdev, &kbdev->csf.firmware_trace_buffers.mcu_write); +} + +int kbase_csf_firmware_parse_trace_buffer_entry(struct kbase_device *kbdev, + const u32 *entry, unsigned int size) +{ + const char *name = (char *)&entry[7]; + const unsigned int name_len = size - TRACE_BUFFER_ENTRY_NAME_OFFSET; + struct firmware_trace_buffer *trace_buffer; + unsigned int i; + + /* Allocate enough space for struct firmware_trace_buffer and the + * trace buffer name (with NULL termination). + */ + trace_buffer = + kmalloc(sizeof(*trace_buffer) + name_len + 1, GFP_KERNEL); + + if (!trace_buffer) + return -ENOMEM; + + memcpy(&trace_buffer->name, name, name_len); + trace_buffer->name[name_len] = '\0'; + + for (i = 0; i < ARRAY_SIZE(trace_buffer_data); i++) { + if (!strcmp(trace_buffer_data[i].name, trace_buffer->name)) { + unsigned int j; + + trace_buffer->kbdev = kbdev; + trace_buffer->type = entry[0]; + trace_buffer->gpu_va.size_address = entry[1]; + trace_buffer->gpu_va.insert_address = entry[2]; + trace_buffer->gpu_va.extract_address = entry[3]; + trace_buffer->gpu_va.data_address = entry[4]; + trace_buffer->gpu_va.trace_enable = entry[5]; + trace_buffer->trace_enable_entry_count = entry[6]; + trace_buffer->num_pages = trace_buffer_data[i].size; + + for (j = 0; j < CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX; j++) { + trace_buffer->trace_enable_init_mask[j] = + trace_buffer_data[i].trace_enable_init_mask[j]; + } + break; + } + } + + if (i < ARRAY_SIZE(trace_buffer_data)) { + list_add(&trace_buffer->node, &kbdev->csf.firmware_trace_buffers.list); + dev_dbg(kbdev->dev, "Trace buffer '%s'", trace_buffer->name); + } else { + dev_dbg(kbdev->dev, "Unknown trace buffer '%s'", trace_buffer->name); + kfree(trace_buffer); + } + + return 0; +} + +void kbase_csf_firmware_reload_trace_buffers_data(struct kbase_device *kbdev) +{ + struct firmware_trace_buffer *trace_buffer; + u32 mcu_rw_offset = 0, mcu_write_offset = 0; + const u32 cache_line_alignment = kbase_get_cache_line_alignment(kbdev); + + list_for_each_entry(trace_buffer, &kbdev->csf.firmware_trace_buffers.list, node) { + u32 extract_gpu_va, insert_gpu_va, data_buffer_gpu_va, + trace_enable_size_dwords; + u32 *extract_cpu_va, *insert_cpu_va; + unsigned int i; + + /* Rely on the fact that all required mappings already exist */ + extract_gpu_va = + (kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg->start_pfn << PAGE_SHIFT) + + mcu_rw_offset; + extract_cpu_va = (u32*)( + kbdev->csf.firmware_trace_buffers.mcu_rw.cpu_addr + + mcu_rw_offset); + insert_gpu_va = + (kbdev->csf.firmware_trace_buffers.mcu_write.va_reg->start_pfn << PAGE_SHIFT) + + mcu_write_offset; + insert_cpu_va = (u32*)( + kbdev->csf.firmware_trace_buffers.mcu_write.cpu_addr + + mcu_write_offset); + data_buffer_gpu_va = + (trace_buffer->data_mapping.va_reg->start_pfn << PAGE_SHIFT); + + /* Notice that the function only re-updates firmware memory locations + * with information that allows access to the trace buffers without + * really resetting their state. For instance, the Insert offset will + * not change and, as a consequence, the Extract offset is not going + * to be reset to keep consistency. + */ + + /* Each FW address shall be mapped and set individually, as we can't + * assume anything about their location in the memory address space. + */ + kbase_csf_update_firmware_memory( + kbdev, trace_buffer->gpu_va.data_address, data_buffer_gpu_va); + kbase_csf_update_firmware_memory( + kbdev, trace_buffer->gpu_va.insert_address, insert_gpu_va); + kbase_csf_update_firmware_memory( + kbdev, trace_buffer->gpu_va.extract_address, extract_gpu_va); + kbase_csf_update_firmware_memory( + kbdev, trace_buffer->gpu_va.size_address, + trace_buffer->num_pages << PAGE_SHIFT); + + trace_enable_size_dwords = + (trace_buffer->trace_enable_entry_count + 31) >> 5; + + for (i = 0; i < trace_enable_size_dwords; i++) { + kbase_csf_update_firmware_memory( + kbdev, trace_buffer->gpu_va.trace_enable + i*4, + trace_buffer->trace_enable_init_mask[i]); + } + + /* Store CPU virtual addresses for permanently mapped variables, + * as they might have slightly changed. + */ + trace_buffer->cpu_va.insert_cpu_va = insert_cpu_va; + trace_buffer->cpu_va.extract_cpu_va = extract_cpu_va; + + /* Update offsets */ + mcu_write_offset += cache_line_alignment; + mcu_rw_offset += cache_line_alignment; + } +} + +struct firmware_trace_buffer *kbase_csf_firmware_get_trace_buffer( + struct kbase_device *kbdev, const char *name) +{ + struct firmware_trace_buffer *trace_buffer; + + list_for_each_entry(trace_buffer, &kbdev->csf.firmware_trace_buffers.list, node) { + if (!strcmp(trace_buffer->name, name)) + return trace_buffer; + } + + return NULL; +} +EXPORT_SYMBOL(kbase_csf_firmware_get_trace_buffer); + +unsigned int kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count( + const struct firmware_trace_buffer *trace_buffer) +{ + return trace_buffer->trace_enable_entry_count; +} +EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count); + +void kbase_csf_firmware_trace_buffer_update_trace_enable_bit( + struct firmware_trace_buffer *tb, unsigned int bit, bool value) +{ + if (bit < tb->trace_enable_entry_count) { + unsigned int trace_enable_reg_offset = bit >> 5; + u32 trace_enable_bit_mask = 1u << (bit & 0x1F); + + if (value) { + tb->trace_enable_init_mask[trace_enable_reg_offset] |= + trace_enable_bit_mask; + } else { + tb->trace_enable_init_mask[trace_enable_reg_offset] &= + ~trace_enable_bit_mask; + } + + /* This is not strictly needed as the caller is supposed to + * reload the firmware image (through GPU reset) after updating + * the bitmask. Otherwise there is no guarantee that firmware + * will take into account the updated bitmask for all types of + * trace buffers, since firmware could continue to use the + * value of bitmask it cached after the boot. + */ + kbase_csf_update_firmware_memory(tb->kbdev, + tb->gpu_va.trace_enable + trace_enable_reg_offset*4, + tb->trace_enable_init_mask[trace_enable_reg_offset]); + } +} +EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_update_trace_enable_bit); + +bool kbase_csf_firmware_trace_buffer_is_empty( + const struct firmware_trace_buffer *trace_buffer) +{ + return *(trace_buffer->cpu_va.insert_cpu_va) == + *(trace_buffer->cpu_va.extract_cpu_va); +} +EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_is_empty); + +unsigned int kbase_csf_firmware_trace_buffer_read_data( + struct firmware_trace_buffer *trace_buffer, u8 *data, unsigned int num_bytes) +{ + unsigned int bytes_copied; + u8 *data_cpu_va = trace_buffer->data_mapping.cpu_addr; + u32 extract_offset = *(trace_buffer->cpu_va.extract_cpu_va); + u32 insert_offset = *(trace_buffer->cpu_va.insert_cpu_va); + u32 buffer_size = trace_buffer->num_pages << PAGE_SHIFT; + + if (insert_offset >= extract_offset) { + bytes_copied = min_t(unsigned int, num_bytes, + (insert_offset - extract_offset)); + memcpy(data, &data_cpu_va[extract_offset], bytes_copied); + extract_offset += bytes_copied; + } else { + unsigned int bytes_copied_head, bytes_copied_tail; + + bytes_copied_tail = min_t(unsigned int, num_bytes, + (buffer_size - extract_offset)); + memcpy(data, &data_cpu_va[extract_offset], bytes_copied_tail); + + bytes_copied_head = min_t(unsigned int, + (num_bytes - bytes_copied_tail), insert_offset); + memcpy(&data[bytes_copied_tail], data_cpu_va, bytes_copied_head); + + bytes_copied = bytes_copied_head + bytes_copied_tail; + extract_offset += bytes_copied; + if (extract_offset >= buffer_size) + extract_offset = bytes_copied_head; + } + + *(trace_buffer->cpu_va.extract_cpu_va) = extract_offset; + + return bytes_copied; +} +EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_read_data); + +#ifdef CONFIG_DEBUG_FS + +#define U32_BITS 32 +static u64 get_trace_buffer_active_mask64(struct firmware_trace_buffer *tb) +{ + u64 active_mask = tb->trace_enable_init_mask[0]; + + if (tb->trace_enable_entry_count > U32_BITS) + active_mask |= (u64)tb->trace_enable_init_mask[1] << U32_BITS; + + return active_mask; +} + +static void update_trace_buffer_active_mask64(struct firmware_trace_buffer *tb, + u64 mask) +{ + unsigned int i; + + for (i = 0; i < tb->trace_enable_entry_count; i++) + kbase_csf_firmware_trace_buffer_update_trace_enable_bit(tb, i, + (mask >> i) & 1); +} + +static int set_trace_buffer_active_mask64(struct firmware_trace_buffer *tb, + u64 mask) +{ + struct kbase_device *kbdev = tb->kbdev; + unsigned long flags; + int err = 0; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + /* If there is already a GPU reset pending, need a retry */ + if (kbase_reset_gpu_silent(kbdev)) + err = -EAGAIN; + else + update_trace_buffer_active_mask64(tb, mask); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return err; +} + +static int kbase_csf_firmware_trace_enable_mask_read(void *data, u64 *val) +{ + struct kbase_device *kbdev = (struct kbase_device *)data; + struct firmware_trace_buffer *tb = + kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); + + if (tb == NULL) { + dev_err(kbdev->dev, "Couldn't get the firmware trace buffer"); + return -EIO; + } + /* The enabled traces limited to u64 here, regarded practical */ + *val = get_trace_buffer_active_mask64(tb); + return 0; +} + +static int kbase_csf_firmware_trace_enable_mask_write(void *data, u64 val) +{ + struct kbase_device *kbdev = (struct kbase_device *)data; + struct firmware_trace_buffer *tb = + kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); + u64 new_mask; + unsigned int enable_bits_count; + + if (tb == NULL) { + dev_err(kbdev->dev, "Couldn't get the firmware trace buffer"); + return -EIO; + } + + /* Ignore unsupported types */ + enable_bits_count = + kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count(tb); + if (enable_bits_count > 64) { + dev_dbg(kbdev->dev, "Limit enabled bits count from %u to 64", + enable_bits_count); + enable_bits_count = 64; + } + new_mask = val & ((1 << enable_bits_count) - 1); + + if (new_mask != get_trace_buffer_active_mask64(tb)) + return set_trace_buffer_active_mask64(tb, new_mask); + else + return 0; +} + +static int kbasep_csf_firmware_trace_debugfs_open(struct inode *in, + struct file *file) +{ + struct kbase_device *kbdev = in->i_private; + + file->private_data = kbdev; + dev_dbg(kbdev->dev, "Opened firmware trace buffer dump debugfs file"); + + return 0; +} + +static ssize_t kbasep_csf_firmware_trace_debugfs_read(struct file *file, + char __user *buf, size_t size, loff_t *ppos) +{ + struct kbase_device *kbdev = file->private_data; + u8 *pbyte; + unsigned int n_read; + unsigned long not_copied; + /* Limit the kernel buffer to no more than two pages */ + size_t mem = MIN(size, 2 * PAGE_SIZE); + unsigned long flags; + + struct firmware_trace_buffer *tb = + kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); + + if (tb == NULL) { + dev_err(kbdev->dev, "Couldn't get the firmware trace buffer"); + return -EIO; + } + + pbyte = kmalloc(mem, GFP_KERNEL); + if (pbyte == NULL) { + dev_err(kbdev->dev, "Couldn't allocate memory for trace buffer dump"); + return -ENOMEM; + } + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + n_read = kbase_csf_firmware_trace_buffer_read_data(tb, pbyte, mem); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Do the copy, if we have obtained some trace data */ + not_copied = (n_read) ? copy_to_user(buf, pbyte, n_read) : 0; + kfree(pbyte); + + if (!not_copied) { + *ppos += n_read; + return n_read; + } + + dev_err(kbdev->dev, "Couldn't copy trace buffer data to user space buffer"); + return -EFAULT; +} + + +DEFINE_SIMPLE_ATTRIBUTE(kbase_csf_firmware_trace_enable_mask_fops, + kbase_csf_firmware_trace_enable_mask_read, + kbase_csf_firmware_trace_enable_mask_write, "%llx\n"); + +static const struct file_operations kbasep_csf_firmware_trace_debugfs_fops = { + .owner = THIS_MODULE, + .open = kbasep_csf_firmware_trace_debugfs_open, + .read = kbasep_csf_firmware_trace_debugfs_read, + .llseek = no_llseek, +}; + +void kbase_csf_firmware_trace_buffer_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("fw_trace_enable_mask", 0644, + kbdev->mali_debugfs_directory, kbdev, + &kbase_csf_firmware_trace_enable_mask_fops); + + debugfs_create_file("fw_traces", 0444, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_csf_firmware_trace_debugfs_fops); +} +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h new file mode 100755 index 000000000000..2cac55e0664d --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.h @@ -0,0 +1,177 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CSF_TRACE_BUFFER_H_ +#define _KBASE_CSF_TRACE_BUFFER_H_ + +#include + +#define CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX (4) +#define FW_TRACE_BUF_NAME "fwlog" + +/* Forward declarations */ +struct firmware_trace_buffer; +struct kbase_device; + +/** + * kbase_csf_firmware_trace_buffers_init - Initialize trace buffers + * + * Allocate resources for trace buffers. In particular: + * - One memory page of GPU-readable, CPU-writable memory is used for + * the Extract variables of all trace buffers. + * - One memory page of GPU-writable, CPU-readable memory is used for + * the Insert variables of all trace buffers. + * - A data buffer of GPU-writable, CPU-readable memory is allocated + * for each trace buffer. + * + * After that, firmware addresses are written with pointers to the + * insert, extract and data buffer variables. The size and the trace + * enable bits are not dereferenced by the GPU and shall be written + * in the firmware addresses directly. + * + * This function relies on the assumption that the list of + * firmware_trace_buffer elements in the device has already been + * populated with data from the firmware image parsing. + * + * Return: 0 if success, or an error code on failure. + * + * @kbdev: Device pointer + */ +int kbase_csf_firmware_trace_buffers_init(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_trace_buffer_term - Terminate trace buffers + * + * @kbdev: Device pointer + */ +void kbase_csf_firmware_trace_buffers_term(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_parse_trace_buffer_entry - Process a "trace buffer" section + * + * Read a "trace buffer" section adding metadata for the related trace buffer + * to the kbase_device:csf.firmware_trace_buffers list. + * + * Unexpected trace buffers will not be parsed and, as a consequence, + * will not be initialized. + * + * Return: 0 if successful, negative error code on failure. + * + * @kbdev: Kbase device structure + * @entry: Pointer to the section + * @size: Size (in bytes) of the section + */ +int kbase_csf_firmware_parse_trace_buffer_entry(struct kbase_device *kbdev, + const u32 *entry, unsigned int size); + +/** + * kbase_csf_firmware_reload_trace_buffers_data - + * Reload trace buffers data for firmware reboot + * + * Helper function used when rebooting the firmware to reload the initial setup + * for all the trace buffers which have been previously parsed and initialized. + * + * Almost all of the operations done in the initialization process are + * replicated, with the difference that they might be done in a different order + * and that the variables of a given trace buffer may be mapped to different + * offsets within the same existing mappings. + * + * In other words, the re-initialization done by this function will be + * equivalent but not necessarily identical to the original initialization. + * + * @kbdev: Device pointer + */ +void kbase_csf_firmware_reload_trace_buffers_data(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_get_trace_buffer - Get a trace buffer + * + * Return: handle to a trace buffer, given the name, or NULL if a trace buffer + * with that name couldn't be found. + * + * @kbdev: Device pointer + * @name: Name of the trace buffer to find + */ +struct firmware_trace_buffer *kbase_csf_firmware_get_trace_buffer( + struct kbase_device *kbdev, const char *name); + +/** + * kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count - + * Get number of trace enable bits for a trace buffer + * + * Return: Number of trace enable bits in a trace buffer. + * + * @trace_buffer: Trace buffer handle + */ +unsigned int kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count( + const struct firmware_trace_buffer *trace_buffer); + +/** + * kbase_csf_firmware_trace_buffer_update_trace_enable_bit - + * Update a trace enable bit + * + * Update the value of a given trace enable bit. + * + * @trace_buffer: Trace buffer handle + * @bit: Bit to update + * @value: New value for the given bit + */ +void kbase_csf_firmware_trace_buffer_update_trace_enable_bit( + struct firmware_trace_buffer *trace_buffer, unsigned int bit, bool value); + +/** + * kbase_csf_firmware_trace_buffer_is_empty - Empty trace buffer predicate + * + * Return: True if the trace buffer is empty, or false otherwise. + * + * @trace_buffer: Trace buffer handle + */ +bool kbase_csf_firmware_trace_buffer_is_empty( + const struct firmware_trace_buffer *trace_buffer); + +/** + * kbase_csf_firmware_trace_buffer_read_data - Read data from a trace buffer + * + * Read available data from a trace buffer. The client provides a data buffer + * of a given size and the maximum number of bytes to read. + * + * Return: Number of bytes read from the trace buffer. + * + * @trace_buffer: Trace buffer handle + * @data: Pointer to a client-allocated where data shall be written. + * @num_bytes: Maximum number of bytes to read from the trace buffer. + */ +unsigned int kbase_csf_firmware_trace_buffer_read_data( + struct firmware_trace_buffer *trace_buffer, u8 *data, unsigned int num_bytes); + +#ifdef CONFIG_DEBUG_FS +/** + * kbase_csf_fw_trace_buffer_debugfs_init() - Add debugfs entries for setting + * enable mask and dumping the binary + * firmware trace buffer + * + * @kbdev: Pointer to the device + */ +void kbase_csf_firmware_trace_buffer_debugfs_init(struct kbase_device *kbdev); +#endif /* CONFIG_DEBUG_FS */ + +#endif /* _KBASE_CSF_TRACE_BUFFER_H_ */ diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h new file mode 100755 index 000000000000..32181d711193 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h @@ -0,0 +1,116 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** + * ***** DO NOT INCLUDE DIRECTLY ***** + * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** + */ + +/* + * The purpose of this header file is just to contain a list of trace code + * identifiers + * + * When updating this file, also remember to update + * mali_kbase_debug_linux_ktrace_csf.h + * + * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT + * DESCRIBED IN mali_kbase_debug_ktrace_codes.h + */ + +#if 0 /* Dummy section to avoid breaking formatting */ +int dummy_array[] = { +#endif + /* + * Generic CSF events + */ + KBASE_KTRACE_CODE_MAKE_CODE(EVICT_CTX_SLOTS), + /* info_val[0:7] == fw version_minor + * info_val[15:8] == fw version_major + * info_val[63:32] == fw version_hash + */ + KBASE_KTRACE_CODE_MAKE_CODE(FIRMWARE_BOOT), + KBASE_KTRACE_CODE_MAKE_CODE(FIRMWARE_REBOOT), + KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TOCK), + KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TICK), + KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_RESET), + KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_EXIT_PROTM), + KBASE_KTRACE_CODE_MAKE_CODE(SYNC_UPDATE_EVENT), + + /* + * Group events + */ + /* info_val[2:0] == CSG_REQ state issued + * info_val[19:16] == as_nr + * info_val[63:32] == endpoint config (max number of endpoints allowed) + */ + KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_START), + /* info_val == CSG_REQ state issued */ + KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_STOP), + /* info_val == CSG_ACK state */ + KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_STARTED), + /* info_val == CSG_ACK state */ + KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_STOPPED), + /* info_val == slot cleaned */ + KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_CLEANED), + /* info_val == previous priority */ + KBASE_KTRACE_CODE_MAKE_CODE(CSG_PRIO_UPDATE), + /* info_val == CSG_REQ ^ CSG_ACK */ + KBASE_KTRACE_CODE_MAKE_CODE(CSG_SYNC_UPDATE_INTERRUPT), + /* info_val == CSG_REQ ^ CSG_ACK */ + KBASE_KTRACE_CODE_MAKE_CODE(CSG_IDLE_INTERRUPT), + KBASE_KTRACE_CODE_MAKE_CODE(GROUP_SYNC_UPDATE_DONE), + /* info_val == run state of the group */ + KBASE_KTRACE_CODE_MAKE_CODE(GROUP_DESCHEDULE), + /* info_val == run state of the group */ + KBASE_KTRACE_CODE_MAKE_CODE(GROUP_SCHEDULE), + /* info_val[31:0] == new run state of the evicted group + * info_val[63:32] == number of runnable groups + */ + KBASE_KTRACE_CODE_MAKE_CODE(GROUP_EVICT_SCHED), + KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_ENTER_PROTM), + /* info_val[31:0] == number of GPU address space slots in use + * info_val[63:32] == number of runnable groups + */ + KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TOP_GRP), + + /* + * Group + Queue events + */ + /* info_val == queue->enabled */ + KBASE_KTRACE_CODE_MAKE_CODE(CSI_START), + /* info_val == queue->enabled before stop */ + KBASE_KTRACE_CODE_MAKE_CODE(CSI_STOP), + KBASE_KTRACE_CODE_MAKE_CODE(CSI_STOP_REQUESTED), + /* info_val == CS_REQ ^ CS_ACK */ + KBASE_KTRACE_CODE_MAKE_CODE(CSI_FAULT_INTERRUPT), + /* info_val == CS_REQ ^ CS_ACK */ + KBASE_KTRACE_CODE_MAKE_CODE(CSI_TILER_OOM_INTERRUPT), + /* info_val == group->run_State (for group the queue is bound to) */ + KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_START), + KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_STOP), + +#if 0 /* Dummy section to avoid breaking formatting */ +}; +#endif + +/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_jm.h new file mode 100755 index 000000000000..b201e49bd0f2 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_jm.h @@ -0,0 +1,173 @@ +/* + * + * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** + * ***** DO NOT INCLUDE DIRECTLY ***** + * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** + */ + +/* + * The purpose of this header file is just to contain a list of trace code + * identifiers + * + * When updating this file, also remember to update + * mali_kbase_debug_linux_ktrace_jm.h + * + * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT + * DESCRIBED IN mali_kbase_debug_ktrace_codes.h + */ + +#if 0 /* Dummy section to avoid breaking formatting */ +int dummy_array[] = { +#endif + + /* + * Job Slot management events + */ + /* info_val==irq rawstat at start */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ), + /* info_val==jobs processed */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ_END), + /* In the following: + * + * - ctx is set if a corresponding job found (NULL otherwise, e.g. some + * soft-stop cases) + * - uatom==kernel-side mapped uatom address (for correlation with + * user-side) + */ + /* info_val==exit code; gpu_addr==chain gpuaddr */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_JOB_DONE), + /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of + * affinity + */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT), + /* gpu_addr is as follows: + * - If JS_STATUS active after soft-stop, val==gpu addr written to + * JS_HEAD on submit + * - otherwise gpu_addr==0 + */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP), + KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), + KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), + /* gpu_addr==JS_HEAD read */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP), + /* gpu_addr==JS_HEAD read */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), + /* gpu_addr==JS_HEAD read */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), + /* gpu_addr==JS_TAIL read */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), + /* gpu_addr is as follows: + * - If JS_STATUS active before soft-stop, val==JS_HEAD + * - otherwise gpu_addr==0 + */ + /* gpu_addr==JS_HEAD read */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), + KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), + KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), + /* info_val == is_scheduled */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), + /* info_val == is_scheduled */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), + KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_DONE), + /* info_val == nr jobs submitted */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), + /* gpu_addr==JS_HEAD_NEXT last written */ + KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), + KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), + KBASE_KTRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), + KBASE_KTRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), + /* + * Job dispatch events + */ + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), + /* gpu_addr==0, info_val==0, uatom==0 */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), + /* + * Scheduler Core events + */ + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_ADD_JOB), + /* gpu_addr==last value written/would be written to JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), + KBASE_KTRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), + KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), + /* info_val == lower 32 bits of affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), + /* info_val == lower 32 bits of affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), + /* info_val == lower 32 bits of affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), + /* info_val == lower 32 bits of rechecked affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), + /* info_val == lower 32 bits of rechecked affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), + /* info_val == lower 32 bits of affinity */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), + /* info_val == the ctx attribute now on ctx */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), + /* info_val == the ctx attribute now on runpool */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), + /* info_val == the ctx attribute now off ctx */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), + /* info_val == the ctx attribute now off runpool */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), + /* + * Scheduler Policy events + */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), + /* info_val == whether it was evicted */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), + /* gpu_addr==JS_HEAD to write if the job were run */ + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), + KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), + +#if 0 /* Dummy section to avoid breaking formatting */ +}; +#endif + +/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c new file mode 100755 index 000000000000..2ea901b666c2 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c @@ -0,0 +1,143 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ +#include +#include "debug/mali_kbase_debug_ktrace_internal.h" +#include "debug/backend/mali_kbase_debug_ktrace_csf.h" + +#if KBASE_KTRACE_TARGET_RBUF + +void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) +{ + *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), + "group,slot,prio,csi"), 0); +} + +void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, + char *buffer, int sz, s32 *written) +{ + const struct kbase_ktrace_backend * const be_msg = &trace_msg->backend; + /* At present, no need to check for KBASE_KTRACE_FLAG_BACKEND, as the + * other backend-specific flags currently imply this anyway + */ + + /* group parts */ + if (be_msg->flags & KBASE_KTRACE_FLAG_CSF_GROUP) { + const s8 slot = be_msg->csg_nr; + /* group,slot, */ + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + "%u,%d,", be_msg->group_handle, slot), 0); + + /* prio */ + if (slot >= 0) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + "%u", be_msg->slot_prio), 0); + + /* , */ + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + ","), 0); + } else { + /* No group,slot,prio fields, but ensure ending with "," */ + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + ",,,"), 0); + } + + /* queue parts: csi */ + if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_CSF_QUEUE) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + "%d", be_msg->csi_index), 0); + + /* Don't end with a trailing "," - this is a 'standalone' formatted + * msg, caller will handle the delimiters + */ +} + +void kbasep_ktrace_add_csf(struct kbase_device *kbdev, + enum kbase_ktrace_code code, struct kbase_queue_group *group, + struct kbase_queue *queue, kbase_ktrace_flag_t flags, + u64 info_val) +{ + unsigned long irqflags; + struct kbase_ktrace_msg *trace_msg; + struct kbase_context *kctx = NULL; + + spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); + + /* Reserve and update indices */ + trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); + + /* Determine the kctx */ + if (group) + kctx = group->kctx; + else if (queue) + kctx = queue->kctx; + + /* Fill the common part of the message (including backend.flags) */ + kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, + info_val); + + /* Indicate to the common code that backend-specific parts will be + * valid + */ + trace_msg->backend.flags |= KBASE_KTRACE_FLAG_BACKEND; + + /* Fill the CSF-specific parts of the message + * + * Generally, no need to use default initializers when queue/group not + * present - can usually check the flags instead. + */ + + if (queue) { + trace_msg->backend.flags |= KBASE_KTRACE_FLAG_CSF_QUEUE; + trace_msg->backend.csi_index = queue->csi_index; + } + + if (group) { + const s8 slot = group->csg_nr; + + trace_msg->backend.flags |= KBASE_KTRACE_FLAG_CSF_GROUP; + + trace_msg->backend.csg_nr = slot; + + if (slot >= 0) { + struct kbase_csf_csg_slot *csg_slot = &kbdev->csf.scheduler.csg_slots[slot]; + + trace_msg->backend.slot_prio = csg_slot->priority; + } + /* slot >=0 indicates whether slot_prio valid, so no need to + * initialize in the case where it's invalid + */ + + trace_msg->backend.group_handle = group->handle; + } + + WARN_ON((trace_msg->backend.flags & ~KBASE_KTRACE_FLAG_ALL)); + + /* Done */ + spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); +} + +#endif /* KBASE_KTRACE_TARGET_RBUF */ diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.h new file mode 100755 index 000000000000..b055ff82a116 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.h @@ -0,0 +1,148 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_KTRACE_CSF_H_ +#define _KBASE_DEBUG_KTRACE_CSF_H_ + +/* + * KTrace target for internal ringbuffer + */ +#if KBASE_KTRACE_TARGET_RBUF +/** + * kbasep_ktrace_add_csf - internal function to add trace about Command Stream + * Frontend + * @kbdev: kbase device + * @code: trace code + * @group: queue group, or NULL if no queue group + * @queue: queue, or NULL if no queue + * @flags: flags about the message + * @info_val: generic information about @code to add to the trace + * + * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_CSF() instead. + */ + +void kbasep_ktrace_add_csf(struct kbase_device *kbdev, + enum kbase_ktrace_code code, struct kbase_queue_group *group, + struct kbase_queue *queue, kbase_ktrace_flag_t flags, + u64 info_val); + +#define KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, group, queue, flags, info_val) \ + kbasep_ktrace_add_csf(kbdev, KBASE_KTRACE_CODE(code), group, queue, \ + flags, info_val) + +#else /* KBASE_KTRACE_TARGET_RBUF */ + +#define KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, group, queue, flags, info_val) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(group);\ + CSTD_UNUSED(queue); \ + CSTD_UNUSED(flags);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#endif /* KBASE_KTRACE_TARGET_RBUF */ + +/* + * KTrace target for Linux's ftrace + * + * Note: the header file(s) that define the trace_mali_<...> tracepoints are + * included by the parent header file + */ +#if KBASE_KTRACE_TARGET_FTRACE + +#define KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, group, queue, info_val) \ + trace_mali_##code(kbdev, group, queue, info_val) + +#else /* KBASE_KTRACE_TARGET_FTRACE */ + +#define KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, group, queue, info_val) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(group);\ + CSTD_UNUSED(queue);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#endif /* KBASE_KTRACE_TARGET_FTRACE */ + +/* + * Master set of macros to route KTrace to any of the targets + */ + +/** + * KBASE_KTRACE_ADD_CSF_GRP - Add trace values about a group, with info + * @kbdev: kbase device + * @code: trace code + * @group: queue group, or NULL if no queue group + * @info_val: generic information about @code to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_CSF_GRP(kbdev, code, group, info_val) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + struct kbase_queue_group *__group = group; \ + u64 __info_val = info_val; \ + KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, __group, NULL, 0u, \ + __info_val); \ + KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, __group, NULL, \ + __info_val); \ + } while (0) + +/** + * KBASE_KTRACE_ADD_CSF_GRP_Q - Add trace values about a group, queue, with info + * @kbdev: kbase device + * @code: trace code + * @group: queue group, or NULL if no queue group + * @queue: queue, or NULL if no queue + * @info_val: generic information about @code to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, code, group, queue, info_val) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + struct kbase_queue_group *__group = group; \ + struct kbase_queue *__queue = queue; \ + u64 __info_val = info_val; \ + KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, __group, __queue, 0u, \ + __info_val); \ + KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, __group, \ + __queue, __info_val); \ + } while (0) + +#endif /* _KBASE_DEBUG_KTRACE_CSF_H_ */ diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_csf.h new file mode 100755 index 000000000000..f265fe9a9753 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_csf.h @@ -0,0 +1,85 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_KTRACE_DEFS_CSF_H_ +#define _KBASE_DEBUG_KTRACE_DEFS_CSF_H_ + +#if KBASE_KTRACE_TARGET_RBUF +/** + * DOC: KTrace version history, CSF variant + * + * 1.0: + * First version, with version information in the header. + * + * 1.1: + * kctx field is no longer a pointer, and is now an ID of the format %d_%u as + * used by kctx directories in mali debugfs entries: (tgid creating the kctx), + * (unique kctx id) + * + * ftrace backend now outputs kctx field (as %d_%u format). + * + * Add fields group, slot, prio, csi into backend-specific part. + */ +#define KBASE_KTRACE_VERSION_MAJOR 1 +#define KBASE_KTRACE_VERSION_MINOR 1 + +/* indicates if the trace message has valid queue-group related info. */ +#define KBASE_KTRACE_FLAG_CSF_GROUP (((kbase_ktrace_flag_t)1) << 0) + +/* indicates if the trace message has valid queue related info. */ +#define KBASE_KTRACE_FLAG_CSF_QUEUE (((kbase_ktrace_flag_t)1) << 1) + +/* Collect all the flags together for debug checking */ +#define KBASE_KTRACE_FLAG_BACKEND_ALL \ + (KBASE_KTRACE_FLAG_CSF_GROUP | KBASE_KTRACE_FLAG_CSF_QUEUE) + + +/** + * struct kbase_ktrace_backend - backend specific part of a trace message + * + * @code: Identifies the event, refer to enum kbase_ktrace_code. + * @flags: indicates information about the trace message itself. Used + * during dumping of the message. + * @group_handle: Handle identifying the associated queue group. Only valid + * when @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. + * @csg_nr: Number/index of the associated queue group's command stream + * group to which it is mapped, or negative if none associated. + * Only valid when @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. + * @slot_prio: The priority of the slot for the associated group, if it was + * scheduled. Hence, only valid when @csg_nr >=0 and @flags + * contains KBASE_KTRACE_FLAG_CSF_GROUP. + * @csi_index: ID of the associated queue's Command Stream HW interface. + * Only valid when @flags contains KBASE_KTRACE_FLAG_CSF_QUEUE. + */ +struct kbase_ktrace_backend { + /* Place 64 and 32-bit members together */ + /* Pack smaller members together */ + kbase_ktrace_code_t code; + kbase_ktrace_flag_t flags; + u8 group_handle; + s8 csg_nr; + u8 slot_prio; + s8 csi_index; +}; + +#endif /* KBASE_KTRACE_TARGET_RBUF */ +#endif /* _KBASE_DEBUG_KTRACE_DEFS_CSF_H_ */ diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h new file mode 100755 index 000000000000..ea8e01a87f3f --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h @@ -0,0 +1,102 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_KTRACE_DEFS_JM_H_ +#define _KBASE_DEBUG_KTRACE_DEFS_JM_H_ + +#if KBASE_KTRACE_TARGET_RBUF +/** + * DOC: KTrace version history, JM variant + * + * 1.0: + * Original version (implicit, header did not carry version information). + * + * 2.0: + * Introduced version information into the header. + * + * Some changes of parameter names in header. + * + * Trace now uses all 64-bits of info_val. + * + * Non-JM specific parts moved to using info_val instead of refcount/gpu_addr. + * + * 2.1: + * kctx field is no longer a pointer, and is now an ID of the format %d_%u as + * used by kctx directories in mali debugfs entries: (tgid creating the kctx), + * (unique kctx id). + * + * ftrace backend now outputs kctx field (as %d_%u format). + * + */ +#define KBASE_KTRACE_VERSION_MAJOR 2 +#define KBASE_KTRACE_VERSION_MINOR 1 +#endif /* KBASE_KTRACE_TARGET_RBUF */ + +/* + * Note: mali_kbase_debug_ktrace_jm.h needs these value even if the RBUF target + * is disabled (they get discarded with CSTD_UNUSED(), but they're still + * referenced) + */ + +/* indicates if the trace message has a valid refcount member */ +#define KBASE_KTRACE_FLAG_JM_REFCOUNT (((kbase_ktrace_flag_t)1) << 0) +/* indicates if the trace message has a valid jobslot member */ +#define KBASE_KTRACE_FLAG_JM_JOBSLOT (((kbase_ktrace_flag_t)1) << 1) +/* indicates if the trace message has valid atom related info. */ +#define KBASE_KTRACE_FLAG_JM_ATOM (((kbase_ktrace_flag_t)1) << 2) + +#if KBASE_KTRACE_TARGET_RBUF +/* Collect all the flags together for debug checking */ +#define KBASE_KTRACE_FLAG_BACKEND_ALL \ + (KBASE_KTRACE_FLAG_JM_REFCOUNT | KBASE_KTRACE_FLAG_JM_JOBSLOT \ + | KBASE_KTRACE_FLAG_JM_ATOM) + +/** + * struct kbase_ktrace_backend - backend specific part of a trace message + * + * @atom_udata: Copy of the user data sent for the atom in base_jd_submit. + * Only valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags + * @gpu_addr: GPU address, usually of the job-chain represented by an atom. + * @atom_number: id of the atom for which trace message was added. Only valid + * if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags + * @code: Identifies the event, refer to enum kbase_ktrace_code. + * @flags: indicates information about the trace message itself. Used + * during dumping of the message. + * @jobslot: job-slot for which trace message was added, valid only for + * job-slot management events. + * @refcount: reference count for the context, valid for certain events + * related to scheduler core and policy. + */ +struct kbase_ktrace_backend { + /* Place 64 and 32-bit members together */ + u64 atom_udata[2]; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ + u64 gpu_addr; + int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ + /* Pack smaller members together */ + kbase_ktrace_code_t code; + kbase_ktrace_flag_t flags; + u8 jobslot; + u8 refcount; +}; +#endif /* KBASE_KTRACE_TARGET_RBUF */ + +#endif /* _KBASE_DEBUG_KTRACE_DEFS_JM_H_ */ diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c new file mode 100755 index 000000000000..1b821281f09f --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c @@ -0,0 +1,115 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ +#include +#include "debug/mali_kbase_debug_ktrace_internal.h" +#include "debug/backend/mali_kbase_debug_ktrace_jm.h" + +#if KBASE_KTRACE_TARGET_RBUF + +void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) +{ + *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), + "katom,gpu_addr,jobslot,refcount"), 0); +} + +void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, + char *buffer, int sz, s32 *written) +{ + /* katom */ + if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_ATOM) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + "atom %d (ud: 0x%llx 0x%llx)", + trace_msg->backend.atom_number, + trace_msg->backend.atom_udata[0], + trace_msg->backend.atom_udata[1]), 0); + + /* gpu_addr */ + if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_BACKEND) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + ",%.8llx,", trace_msg->backend.gpu_addr), 0); + else + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + ",,"), 0); + + /* jobslot */ + if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + "%d", trace_msg->backend.jobslot), 0); + + *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), + ","), 0); + + /* refcount */ + if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) + *written += MAX(snprintf(buffer + *written, + MAX(sz - *written, 0), + "%d", trace_msg->backend.refcount), 0); +} + +void kbasep_ktrace_add_jm(struct kbase_device *kbdev, + enum kbase_ktrace_code code, struct kbase_context *kctx, + struct kbase_jd_atom *katom, u64 gpu_addr, + kbase_ktrace_flag_t flags, int refcount, int jobslot, + u64 info_val) +{ + unsigned long irqflags; + struct kbase_ktrace_msg *trace_msg; + + spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); + + /* Reserve and update indices */ + trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); + + /* Fill the common part of the message (including backend.flags) */ + kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, + info_val); + + /* Indicate to the common code that backend-specific parts will be + * valid + */ + trace_msg->backend.flags |= KBASE_KTRACE_FLAG_BACKEND; + + /* Fill the JM-specific parts of the message */ + if (katom) { + trace_msg->backend.flags |= KBASE_KTRACE_FLAG_JM_ATOM; + + trace_msg->backend.atom_number = kbase_jd_atom_id(katom->kctx, katom); + trace_msg->backend.atom_udata[0] = katom->udata.blob[0]; + trace_msg->backend.atom_udata[1] = katom->udata.blob[1]; + } + + trace_msg->backend.gpu_addr = gpu_addr; + trace_msg->backend.jobslot = jobslot; + /* Clamp refcount */ + trace_msg->backend.refcount = MIN((unsigned int)refcount, 0xFF); + + WARN_ON((trace_msg->backend.flags & ~KBASE_KTRACE_FLAG_ALL)); + + /* Done */ + spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); +} + +#endif /* KBASE_KTRACE_TARGET_RBUF */ diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h new file mode 100755 index 000000000000..adfcb1aa556e --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h @@ -0,0 +1,309 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_KTRACE_JM_H_ +#define _KBASE_DEBUG_KTRACE_JM_H_ + +/* + * KTrace target for internal ringbuffer + */ +#if KBASE_KTRACE_TARGET_RBUF +/** + * kbasep_ktrace_add_jm - internal function to add trace about Job Management + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @flags: flags about the message + * @refcount: reference count information to add to the trace + * @jobslot: jobslot information to add to the trace + * @info_val: generic information about @code to add to the trace + * + * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_JM() instead. + */ +void kbasep_ktrace_add_jm(struct kbase_device *kbdev, + enum kbase_ktrace_code code, struct kbase_context *kctx, + struct kbase_jd_atom *katom, u64 gpu_addr, + kbase_ktrace_flag_t flags, int refcount, int jobslot, + u64 info_val); + +#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, \ + refcount, jobslot, info_val) \ + kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ + gpu_addr, flags, refcount, jobslot, info_val) + +#else /* KBASE_KTRACE_TARGET_RBUF */ + +#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, \ + refcount, jobslot, info_val) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(flags);\ + CSTD_UNUSED(refcount);\ + CSTD_UNUSED(jobslot);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) +#endif /* KBASE_KTRACE_TARGET_RBUF */ + +/* + * KTrace target for Linux's ftrace + * + * Note: the header file(s) that define the trace_mali_<...> tracepoints are + * included by the parent header file + */ +#if KBASE_KTRACE_TARGET_FTRACE +#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ + jobslot) \ + trace_mali_##code(kctx, jobslot, 0) + +#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, jobslot, info_val) \ + trace_mali_##code(kctx, jobslot, info_val) + +#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \ + gpu_addr, refcount) \ + trace_mali_##code(kctx, refcount, 0) + +#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, refcount, info_val) \ + trace_mali_##code(kctx, refcount, info_val) + +#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \ + info_val) \ + trace_mali_##code(kctx, gpu_addr, info_val) +#else /* KBASE_KTRACE_TARGET_FTRACE */ +#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ + jobslot) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(jobslot);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, jobslot, info_val) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(jobslot);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \ + gpu_addr, refcount) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(refcount);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, refcount, info_val) \ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \ + info_val)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(kctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) +#endif /* KBASE_KTRACE_TARGET_FTRACE */ + +/* + * Master set of macros to route KTrace to any of the targets + */ + +/** + * KBASE_KTRACE_ADD_JM_SLOT - Add trace values about a job-slot + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @jobslot: jobslot information to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ + jobslot) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __gpu_addr = gpu_addr; \ + int __jobslot = jobslot; \ + KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ + KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, \ + 0); \ + KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \ + } while (0) + +/** + * KBASE_KTRACE_ADD_JM_SLOT_INFO - Add trace values about a job-slot, with info + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @jobslot: jobslot information to add to the trace + * @info_val: generic information about @code to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \ + jobslot, info_val) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __gpu_addr = gpu_addr; \ + int __jobslot = jobslot; \ + u64 __info_val = info_val; \ + KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ + KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, \ + __info_val); \ + KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, __jobslot, __info_val); \ + } while (0) + +/** + * KBASE_KTRACE_ADD_JM_REFCOUNT - Add trace values about a kctx refcount + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @refcount: reference count information to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \ + refcount) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __gpu_addr = gpu_addr; \ + int __refcount = refcount; \ + KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ + KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, \ + 0u); \ + KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount); \ + } while (0) + +/** + * KBASE_KTRACE_ADD_JM_REFCOUNT_INFO - Add trace values about a kctx refcount, + * and info + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @refcount: reference count information to add to the trace + * @info_val: generic information about @code to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ + gpu_addr, refcount, info_val) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __gpu_addr = gpu_addr; \ + int __refcount = refcount; \ + u64 __info_val = info_val; \ + KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ + KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, \ + __info_val); \ + KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount, __info_val); \ + } while (0) + +/** + * KBASE_KTRACE_ADD_JM - Add trace values (no slot or refcount) + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @katom: kbase atom, or NULL if no atom + * @gpu_addr: GPU address, usually related to @katom + * @info_val: generic information about @code to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __gpu_addr = gpu_addr; \ + u64 __info_val = info_val; \ + KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ + 0u, 0, 0, __info_val); \ + KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \ + } while (0) + +#endif /* _KBASE_DEBUG_KTRACE_JM_H_ */ diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h new file mode 100755 index 000000000000..d103e5766456 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h @@ -0,0 +1,147 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * NOTE: This must **only** be included through mali_linux_trace.h, + * otherwise it will fail to setup tracepoints correctly + */ + +#if !defined(_KBASE_DEBUG_LINUX_KTRACE_CSF_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _KBASE_DEBUG_LINUX_KTRACE_CSF_H_ + +/* + * Generic CSF events - using the common DEFINE_MALI_ADD_EVENT + */ +DEFINE_MALI_ADD_EVENT(EVICT_CTX_SLOTS); +DEFINE_MALI_ADD_EVENT(FIRMWARE_BOOT); +DEFINE_MALI_ADD_EVENT(FIRMWARE_REBOOT); +DEFINE_MALI_ADD_EVENT(SCHEDULER_TOCK); +DEFINE_MALI_ADD_EVENT(SCHEDULER_TICK); +DEFINE_MALI_ADD_EVENT(SCHEDULER_RESET); +DEFINE_MALI_ADD_EVENT(SCHEDULER_EXIT_PROTM); +DEFINE_MALI_ADD_EVENT(SYNC_UPDATE_EVENT); + +DECLARE_EVENT_CLASS(mali_csf_grp_q_template, + TP_PROTO(struct kbase_device *kbdev, struct kbase_queue_group *group, + struct kbase_queue *queue, u64 info_val), + TP_ARGS(kbdev, group, queue, info_val), + TP_STRUCT__entry( + __field(u64, info_val) + __field(pid_t, kctx_tgid) + __field(u32, kctx_id) + __field(u8, group_handle) + __field(s8, csg_nr) + __field(u8, slot_prio) + __field(s8, csi_index) + ), + TP_fast_assign( + { + struct kbase_context *kctx = NULL; + + __entry->info_val = info_val; + /* Note: if required in future, we could record some + * flags in __entry about whether the group/queue parts + * are valid, and add that to the trace message e.g. + * by using __print_flags()/__print_symbolic() + */ + if (queue) { + /* Note: kctx overridden by group->kctx later if group is valid */ + kctx = queue->kctx; + __entry->csi_index = queue->csi_index; + } else { + __entry->csi_index = -1; + } + + if (group) { + kctx = group->kctx; + __entry->group_handle = group->handle; + __entry->csg_nr = group->csg_nr; + if (group->csg_nr >= 0) + __entry->slot_prio = kbdev->csf.scheduler.csg_slots[group->csg_nr].priority; + else + __entry->slot_prio = 0u; + } else { + __entry->group_handle = 0u; + __entry->csg_nr = -1; + __entry->slot_prio = 0u; + } + __entry->kctx_id = (kctx) ? kctx->id : 0u; + __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; + } + + ), + TP_printk("kctx=%d_%u group=%u slot=%d prio=%u csi=%d info=0x%llx", + __entry->kctx_tgid, __entry->kctx_id, + __entry->group_handle, __entry->csg_nr, + __entry->slot_prio, __entry->csi_index, + __entry->info_val) +); + +/* + * Group events + */ +#define DEFINE_MALI_CSF_GRP_EVENT(name) \ + DEFINE_EVENT_PRINT(mali_csf_grp_q_template, mali_##name, \ + TP_PROTO(struct kbase_device *kbdev, struct kbase_queue_group *group, \ + struct kbase_queue *queue, u64 info_val), \ + TP_ARGS(kbdev, group, queue, info_val), \ + TP_printk("kctx=%d_%u group=%u slot=%d prio=%u info=0x%llx", \ + __entry->kctx_tgid, __entry->kctx_id, __entry->group_handle, \ + __entry->csg_nr, __entry->slot_prio, __entry->info_val)) + +DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_START); +DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_STOP); +DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_STARTED); +DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_STOPPED); +DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_CLEANED); +DEFINE_MALI_CSF_GRP_EVENT(CSG_PRIO_UPDATE); +DEFINE_MALI_CSF_GRP_EVENT(CSG_SYNC_UPDATE_INTERRUPT); +DEFINE_MALI_CSF_GRP_EVENT(CSG_IDLE_INTERRUPT); +DEFINE_MALI_CSF_GRP_EVENT(GROUP_SYNC_UPDATE_DONE); +DEFINE_MALI_CSF_GRP_EVENT(GROUP_DESCHEDULE); +DEFINE_MALI_CSF_GRP_EVENT(GROUP_SCHEDULE); +DEFINE_MALI_CSF_GRP_EVENT(GROUP_EVICT_SCHED); +DEFINE_MALI_CSF_GRP_EVENT(SCHEDULER_ENTER_PROTM); +DEFINE_MALI_CSF_GRP_EVENT(SCHEDULER_TOP_GRP); + +#undef DEFINE_MALI_CSF_GRP_EVENT + +/* + * Group + Queue events + */ +#define DEFINE_MALI_CSF_GRP_Q_EVENT(name) \ + DEFINE_EVENT(mali_csf_grp_q_template, mali_##name, \ + TP_PROTO(struct kbase_device *kbdev, struct kbase_queue_group *group, \ + struct kbase_queue *queue, u64 info_val), \ + TP_ARGS(kbdev, group, queue, info_val)) + +DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_START); +DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_STOP); +DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_STOP_REQUESTED); +DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_FAULT_INTERRUPT); +DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_TILER_OOM_INTERRUPT); +DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_START); +DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_STOP); + +#undef DEFINE_MALI_CSF_GRP_Q_EVENT + +#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_CSF_H_) || defined(TRACE_HEADER_MULTI_READ) */ diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h new file mode 100755 index 000000000000..037b1edecd8e --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h @@ -0,0 +1,165 @@ +/* + * + * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * NOTE: This must **only** be included through mali_linux_trace.h, + * otherwise it will fail to setup tracepoints correctly + */ + +#if !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _KBASE_DEBUG_LINUX_KTRACE_JM_H_ + +DECLARE_EVENT_CLASS(mali_jm_slot_template, + TP_PROTO(struct kbase_context *kctx, int jobslot, u64 info_val), + TP_ARGS(kctx, jobslot, info_val), + TP_STRUCT__entry( + __field(pid_t, kctx_tgid) + __field(u32, kctx_id) + __field(unsigned int, jobslot) + __field(u64, info_val) + ), + TP_fast_assign( + __entry->kctx_id = (kctx) ? kctx->id : 0u; + __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; + __entry->jobslot = jobslot; + __entry->info_val = info_val; + ), + TP_printk("kctx=%d_%u jobslot=%u info=0x%llx", __entry->kctx_tgid, + __entry->kctx_id, __entry->jobslot, __entry->info_val) +); + +#define DEFINE_MALI_JM_SLOT_EVENT(name) \ +DEFINE_EVENT(mali_jm_slot_template, mali_##name, \ + TP_PROTO(struct kbase_context *kctx, int jobslot, u64 info_val), \ + TP_ARGS(kctx, jobslot, info_val)) +DEFINE_MALI_JM_SLOT_EVENT(JM_SUBMIT); +DEFINE_MALI_JM_SLOT_EVENT(JM_JOB_DONE); +DEFINE_MALI_JM_SLOT_EVENT(JM_UPDATE_HEAD); +DEFINE_MALI_JM_SLOT_EVENT(JM_CHECK_HEAD); +DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP); +DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_0); +DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_1); +DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP); +DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_0); +DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_1); +DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); +DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_EVICT); +DEFINE_MALI_JM_SLOT_EVENT(JM_BEGIN_RESET_WORKER); +DEFINE_MALI_JM_SLOT_EVENT(JM_END_RESET_WORKER); +DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); +DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); +DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_CURRENT); +DEFINE_MALI_JM_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); +DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); +DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); +DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); +DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); +DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); +DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); +DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); +DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); +#undef DEFINE_MALI_JM_SLOT_EVENT + +DECLARE_EVENT_CLASS(mali_jm_refcount_template, + TP_PROTO(struct kbase_context *kctx, int refcount, u64 info_val), + TP_ARGS(kctx, refcount, info_val), + TP_STRUCT__entry( + __field(pid_t, kctx_tgid) + __field(u32, kctx_id) + __field(unsigned int, refcount) + __field(u64, info_val) + ), + TP_fast_assign( + __entry->kctx_id = (kctx) ? kctx->id : 0u; + __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; + __entry->refcount = refcount; + __entry->info_val = info_val; + ), + TP_printk("kctx=%d_%u refcount=%u info=0x%llx", __entry->kctx_tgid, + __entry->kctx_id, __entry->refcount, __entry->info_val) +); + +#define DEFINE_MALI_JM_REFCOUNT_EVENT(name) \ +DEFINE_EVENT(mali_jm_refcount_template, mali_##name, \ + TP_PROTO(struct kbase_context *kctx, int refcount, u64 info_val), \ + TP_ARGS(kctx, refcount, info_val)) +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_ADD_JOB); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_REMOVE_JOB); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); +DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); +#undef DEFINE_MALI_JM_REFCOUNT_EVENT + +DECLARE_EVENT_CLASS(mali_jm_add_template, + TP_PROTO(struct kbase_context *kctx, u64 gpu_addr, u64 info_val), + TP_ARGS(kctx, gpu_addr, info_val), + TP_STRUCT__entry( + __field(pid_t, kctx_tgid) + __field(u32, kctx_id) + __field(u64, gpu_addr) + __field(u64, info_val) + ), + TP_fast_assign( + __entry->kctx_id = (kctx) ? kctx->id : 0u; + __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; + __entry->gpu_addr = gpu_addr; + __entry->info_val = info_val; + ), + TP_printk("kctx=%d_%u gpu_addr=0x%llx info=0x%llx", __entry->kctx_tgid, + __entry->kctx_id, __entry->gpu_addr, __entry->info_val) +); + +#define DEFINE_MALI_JM_ADD_EVENT(name) \ +DEFINE_EVENT(mali_jm_add_template, mali_##name, \ + TP_PROTO(struct kbase_context *kctx, u64 gpu_addr, u64 info_val), \ + TP_ARGS(kctx, gpu_addr, info_val)) +DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER); +DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER_END); +DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL_WORKER); +DEFINE_MALI_JM_ADD_EVENT(JD_DONE); +DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL); +DEFINE_MALI_JM_ADD_EVENT(JD_ZAP_CONTEXT); +DEFINE_MALI_JM_ADD_EVENT(JM_IRQ); +DEFINE_MALI_JM_ADD_EVENT(JM_IRQ_END); +DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS); +DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS_DONE); +DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_NON_SCHEDULED); +DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_SCHEDULED); +DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_DONE); +DEFINE_MALI_JM_ADD_EVENT(JM_SUBMIT_AFTER_RESET); +DEFINE_MALI_JM_ADD_EVENT(JM_JOB_COMPLETE); +DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); +DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); +DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); +DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); +DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_END); +DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_START); +DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); +#undef DEFINE_MALI_JM_ADD_EVENT + +#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ)*/ diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c new file mode 100755 index 000000000000..a13c0ba20c94 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c @@ -0,0 +1,356 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ +#include +#include "debug/mali_kbase_debug_ktrace_internal.h" + +int kbase_ktrace_init(struct kbase_device *kbdev) +{ +#if KBASE_KTRACE_TARGET_RBUF + struct kbase_ktrace_msg *rbuf; + + rbuf = kmalloc_array(KBASE_KTRACE_SIZE, sizeof(*rbuf), GFP_KERNEL); + + if (!rbuf) + return -EINVAL; + + kbdev->ktrace.rbuf = rbuf; + spin_lock_init(&kbdev->ktrace.lock); +#endif /* KBASE_KTRACE_TARGET_RBUF */ + return 0; +} + +void kbase_ktrace_term(struct kbase_device *kbdev) +{ +#if KBASE_KTRACE_TARGET_RBUF + kfree(kbdev->ktrace.rbuf); +#endif /* KBASE_KTRACE_TARGET_RBUF */ +} + +void kbase_ktrace_hook_wrapper(void *param) +{ + struct kbase_device *kbdev = (struct kbase_device *)param; + + KBASE_KTRACE_DUMP(kbdev); +} + +#if KBASE_KTRACE_TARGET_RBUF + +static const char * const kbasep_ktrace_code_string[] = { + /* + * IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE + * THIS MUST BE USED AT THE START OF THE ARRAY + */ +#define KBASE_KTRACE_CODE_MAKE_CODE(X) # X +#include "debug/mali_kbase_debug_ktrace_codes.h" +#undef KBASE_KTRACE_CODE_MAKE_CODE +}; + +static void kbasep_ktrace_format_header(char *buffer, int sz, s32 written) +{ + written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + "secs,thread_id,cpu,code,kctx,"), 0); + + kbasep_ktrace_backend_format_header(buffer, sz, &written); + + written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + ",info_val,ktrace_version=%u.%u", + KBASE_KTRACE_VERSION_MAJOR, + KBASE_KTRACE_VERSION_MINOR), 0); + + buffer[sz - 1] = 0; +} + +static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, + char *buffer, int sz) +{ + s32 written = 0; + + /* Initial part of message: + * + * secs,thread_id,cpu,code, + */ + written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + "%d.%.6d,%d,%d,%s,", + (int)trace_msg->timestamp.tv_sec, + (int)(trace_msg->timestamp.tv_nsec / 1000), + trace_msg->thread_id, trace_msg->cpu, + kbasep_ktrace_code_string[trace_msg->backend.code]), 0); + + /* kctx part: */ + if (trace_msg->kctx_tgid) { + written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + "%d_%u", + trace_msg->kctx_tgid, trace_msg->kctx_id), 0); + } + /* Trailing comma */ + written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + ","), 0); + + /* Backend parts */ + kbasep_ktrace_backend_format_msg(trace_msg, buffer, sz, + &written); + + /* Rest of message: + * + * ,info_val + * + * Note that the last column is empty, it's simply to hold the ktrace + * version in the header + */ + written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + ",0x%.16llx", + (unsigned long long)trace_msg->info_val), 0); + buffer[sz - 1] = 0; +} + +static void kbasep_ktrace_dump_msg(struct kbase_device *kbdev, + struct kbase_ktrace_msg *trace_msg) +{ + char buffer[KTRACE_DUMP_MESSAGE_SIZE]; + + lockdep_assert_held(&kbdev->ktrace.lock); + + kbasep_ktrace_format_msg(trace_msg, buffer, sizeof(buffer)); + dev_dbg(kbdev->dev, "%s", buffer); +} + +struct kbase_ktrace_msg *kbasep_ktrace_reserve(struct kbase_ktrace *ktrace) +{ + struct kbase_ktrace_msg *trace_msg; + + lockdep_assert_held(&ktrace->lock); + + trace_msg = &ktrace->rbuf[ktrace->next_in]; + + /* Update the ringbuffer indices */ + ktrace->next_in = (ktrace->next_in + 1) & KBASE_KTRACE_MASK; + if (ktrace->next_in == ktrace->first_out) + ktrace->first_out = (ktrace->first_out + 1) & KBASE_KTRACE_MASK; + + return trace_msg; +} +void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace, + struct kbase_ktrace_msg *trace_msg, enum kbase_ktrace_code code, + struct kbase_context *kctx, kbase_ktrace_flag_t flags, + u64 info_val) +{ + lockdep_assert_held(&ktrace->lock); + + trace_msg->thread_id = task_pid_nr(current); + trace_msg->cpu = task_cpu(current); + + ktime_get_real_ts64(&trace_msg->timestamp); + + /* No need to store a flag about whether there was a kctx, tgid==0 is + * sufficient + */ + if (kctx) { + trace_msg->kctx_tgid = kctx->tgid; + trace_msg->kctx_id = kctx->id; + } else { + trace_msg->kctx_tgid = 0; + trace_msg->kctx_id = 0; + } + trace_msg->info_val = info_val; + trace_msg->backend.code = code; + trace_msg->backend.flags = flags; +} + +void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code, + struct kbase_context *kctx, kbase_ktrace_flag_t flags, + u64 info_val) +{ + unsigned long irqflags; + struct kbase_ktrace_msg *trace_msg; + + WARN_ON((flags & ~KBASE_KTRACE_FLAG_COMMON_ALL)); + + spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); + + /* Reserve and update indices */ + trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); + + /* Fill the common part of the message (including backend.flags) */ + kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, + info_val); + + /* Done */ + spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); +} + +static void kbasep_ktrace_clear_locked(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->ktrace.lock); + kbdev->ktrace.first_out = kbdev->ktrace.next_in; +} +void kbasep_ktrace_clear(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->ktrace.lock, flags); + kbasep_ktrace_clear_locked(kbdev); + spin_unlock_irqrestore(&kbdev->ktrace.lock, flags); +} + +void kbasep_ktrace_dump(struct kbase_device *kbdev) +{ + unsigned long flags; + u32 start; + u32 end; + char buffer[KTRACE_DUMP_MESSAGE_SIZE] = "Dumping trace:\n"; + + kbasep_ktrace_format_header(buffer, sizeof(buffer), strlen(buffer)); + dev_dbg(kbdev->dev, "%s", buffer); + + spin_lock_irqsave(&kbdev->ktrace.lock, flags); + start = kbdev->ktrace.first_out; + end = kbdev->ktrace.next_in; + + while (start != end) { + struct kbase_ktrace_msg *trace_msg = &kbdev->ktrace.rbuf[start]; + + kbasep_ktrace_dump_msg(kbdev, trace_msg); + + start = (start + 1) & KBASE_KTRACE_MASK; + } + dev_dbg(kbdev->dev, "TRACE_END"); + + kbasep_ktrace_clear_locked(kbdev); + + spin_unlock_irqrestore(&kbdev->ktrace.lock, flags); +} + +#ifdef CONFIG_DEBUG_FS +struct trace_seq_state { + struct kbase_ktrace_msg trace_buf[KBASE_KTRACE_SIZE]; + u32 start; + u32 end; +}; + +static void *kbasep_ktrace_seq_start(struct seq_file *s, loff_t *pos) +{ + struct trace_seq_state *state = s->private; + int i; + + if (*pos == 0) + /* See Documentation/filesystems/seq_file.txt */ + return SEQ_START_TOKEN; + + if (*pos > KBASE_KTRACE_SIZE) + return NULL; + i = state->start + *pos; + if ((state->end >= state->start && i >= state->end) || + i >= state->end + KBASE_KTRACE_SIZE) + return NULL; + + i &= KBASE_KTRACE_MASK; + + return &state->trace_buf[i]; +} + +static void kbasep_ktrace_seq_stop(struct seq_file *s, void *data) +{ +} + +static void *kbasep_ktrace_seq_next(struct seq_file *s, void *data, loff_t *pos) +{ + struct trace_seq_state *state = s->private; + int i; + + if (data != SEQ_START_TOKEN) + (*pos)++; + + i = (state->start + *pos) & KBASE_KTRACE_MASK; + if (i == state->end) + return NULL; + + return &state->trace_buf[i]; +} + +static int kbasep_ktrace_seq_show(struct seq_file *s, void *data) +{ + struct kbase_ktrace_msg *trace_msg = data; + char buffer[KTRACE_DUMP_MESSAGE_SIZE]; + + /* If this is the start, print a header */ + if (data == SEQ_START_TOKEN) + kbasep_ktrace_format_header(buffer, sizeof(buffer), 0); + else + kbasep_ktrace_format_msg(trace_msg, buffer, sizeof(buffer)); + + seq_printf(s, "%s\n", buffer); + return 0; +} + +static const struct seq_operations kbasep_ktrace_seq_ops = { + .start = kbasep_ktrace_seq_start, + .next = kbasep_ktrace_seq_next, + .stop = kbasep_ktrace_seq_stop, + .show = kbasep_ktrace_seq_show, +}; + +static int kbasep_ktrace_debugfs_open(struct inode *inode, struct file *file) +{ + struct kbase_device *kbdev = inode->i_private; + unsigned long flags; + + struct trace_seq_state *state; + + state = __seq_open_private(file, &kbasep_ktrace_seq_ops, + sizeof(*state)); + if (!state) + return -ENOMEM; + + spin_lock_irqsave(&kbdev->ktrace.lock, flags); + state->start = kbdev->ktrace.first_out; + state->end = kbdev->ktrace.next_in; + memcpy(state->trace_buf, kbdev->ktrace.rbuf, sizeof(state->trace_buf)); + spin_unlock_irqrestore(&kbdev->ktrace.lock, flags); + + return 0; +} + +static const struct file_operations kbasep_ktrace_debugfs_fops = { + .owner = THIS_MODULE, + .open = kbasep_ktrace_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + +void kbase_ktrace_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("mali_trace", 0444, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_ktrace_debugfs_fops); +} +#endif /* CONFIG_DEBUG_FS */ + +#else /* KBASE_KTRACE_TARGET_RBUF */ + +#ifdef CONFIG_DEBUG_FS +void kbase_ktrace_debugfs_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* KBASE_KTRACE_TARGET_RBUF */ diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.h new file mode 100755 index 000000000000..e4e2e8c35001 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.h @@ -0,0 +1,226 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * DOC: Kbase's own trace, 'KTrace' + * + * Low overhead trace specific to kbase, aimed at: + * - common use-cases for tracing kbase specific functionality to do with + * running work on the GPU + * - easy 1-line addition of new types of trace + * + * KTrace can be recorded in one or more of the following targets: + * - KBASE_KTRACE_TARGET_RBUF: low overhead ringbuffer protected by an + * irq-spinlock, output available via dev_dbg() and debugfs file + * - KBASE_KTRACE_TARGET_FTRACE: ftrace based tracepoints under 'mali' events + */ + +#ifndef _KBASE_DEBUG_KTRACE_H_ +#define _KBASE_DEBUG_KTRACE_H_ + +#if KBASE_KTRACE_TARGET_FTRACE +#include "mali_linux_trace.h" +#endif + +#if MALI_USE_CSF +#include "debug/backend/mali_kbase_debug_ktrace_csf.h" +#else +#include "debug/backend/mali_kbase_debug_ktrace_jm.h" +#endif + +/** + * kbase_ktrace_init - initialize kbase ktrace. + * @kbdev: kbase device + */ +int kbase_ktrace_init(struct kbase_device *kbdev); + +/** + * kbase_ktrace_term - terminate kbase ktrace. + * @kbdev: kbase device + */ +void kbase_ktrace_term(struct kbase_device *kbdev); + +/** + * kbase_ktrace_hook_wrapper - wrapper so that dumping ktrace can be done via a + * callback. + * @param: kbase device, cast to void pointer + */ +void kbase_ktrace_hook_wrapper(void *param); + +#ifdef CONFIG_DEBUG_FS +/** + * kbase_ktrace_debugfs_init - initialize kbase ktrace for debugfs usage, if + * the selected targets support it. + * @kbdev: kbase device + * + * There is no matching 'term' call, debugfs_remove_recursive() is sufficient. + */ +void kbase_ktrace_debugfs_init(struct kbase_device *kbdev); +#endif /* CONFIG_DEBUG_FS */ + +/* + * KTrace target for internal ringbuffer + */ +#if KBASE_KTRACE_TARGET_RBUF +/** + * kbasep_ktrace_add - internal function to add trace to the ringbuffer. + * @kbdev: kbase device + * @code: ktrace code + * @kctx: kbase context, or NULL if no context + * @flags: flags about the message + * @info_val: generic information about @code to add to the trace + * + * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD() instead. + */ +void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code, + struct kbase_context *kctx, kbase_ktrace_flag_t flags, + u64 info_val); + +/** + * kbasep_ktrace_clear - clear the trace ringbuffer + * @kbdev: kbase device + * + * PRIVATE: do not use directly. Use KBASE_KTRACE_CLEAR() instead. + */ +void kbasep_ktrace_clear(struct kbase_device *kbdev); + +/** + * kbasep_ktrace_dump - dump ktrace ringbuffer to dev_dbg(), then clear it + * @kbdev: kbase device + * + * PRIVATE: do not use directly. Use KBASE_KTRACE_DUMP() instead. + */ +void kbasep_ktrace_dump(struct kbase_device *kbdev); + +#define KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, info_val) \ + kbasep_ktrace_add(kbdev, KBASE_KTRACE_CODE(code), kctx, 0, \ + info_val) \ + +#define KBASE_KTRACE_RBUF_CLEAR(kbdev) \ + kbasep_ktrace_clear(kbdev) + +#define KBASE_KTRACE_RBUF_DUMP(kbdev) \ + kbasep_ktrace_dump(kbdev) + +#else /* KBASE_KTRACE_TARGET_RBUF */ + +#define KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, info_val) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(code); \ + CSTD_UNUSED(kctx); \ + CSTD_UNUSED(info_val); \ + CSTD_NOP(0); \ + } while (0) + +#define KBASE_KTRACE_RBUF_CLEAR(kbdev) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(0); \ + } while (0) +#define KBASE_KTRACE_RBUF_DUMP(kbdev) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(0); \ + } while (0) +#endif /* KBASE_KTRACE_TARGET_RBUF */ + +/* + * KTrace target for Linux's ftrace + */ +#if KBASE_KTRACE_TARGET_FTRACE + +#define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \ + trace_mali_##code(kctx, info_val) + +#else /* KBASE_KTRACE_TARGET_FTRACE */ +#define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(code); \ + CSTD_UNUSED(kctx); \ + CSTD_UNUSED(info_val); \ + CSTD_NOP(0); \ + } while (0) +#endif /* KBASE_KTRACE_TARGET_FTRACE */ + +/* No 'clear' implementation for ftrace yet */ +#define KBASE_KTRACE_FTRACE_CLEAR(kbdev) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(0); \ + } while (0) + +/* No 'dump' implementation for ftrace yet */ +#define KBASE_KTRACE_FTRACE_DUMP(kbdev) \ + do { \ + CSTD_UNUSED(kbdev); \ + CSTD_NOP(0); \ + } while (0) + +/* + * Master set of macros to route KTrace to any of the targets + */ + +/** + * KBASE_KTRACE_ADD - Add trace values + * @kbdev: kbase device + * @code: trace code + * @kctx: kbase context, or NULL if no context + * @info_val: generic information about @code to add to the trace + * + * Note: Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when + * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied + * to this macro must: + * a) be static or static inline, and + * b) just return 0 and have no other statements present in the body. + */ +#define KBASE_KTRACE_ADD(kbdev, code, kctx, info_val) \ + do { \ + /* capture values that could come from non-pure function calls */ \ + u64 __info_val = info_val; \ + KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, __info_val); \ + KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, __info_val); \ + } while (0) + +/** + * KBASE_KTRACE_CLEAR - Clear the trace, if applicable to the target(s) + * @kbdev: kbase device + */ +#define KBASE_KTRACE_CLEAR(kbdev) \ + do { \ + KBASE_KTRACE_RBUF_CLEAR(kbdev); \ + KBASE_KTRACE_FTRACE_CLEAR(kbdev); \ + } while (0) + +/** + * KBASE_KTRACE_DUMP - Dump the trace, if applicable to the target(s) + * @kbdev: kbase device + */ +#define KBASE_KTRACE_DUMP(kbdev) \ + do { \ + KBASE_KTRACE_RBUF_DUMP(kbdev); \ + KBASE_KTRACE_FTRACE_DUMP(kbdev); \ + } while (0) + +#endif /* _KBASE_DEBUG_KTRACE_H_ */ diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h new file mode 100755 index 000000000000..b50bceee4244 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h @@ -0,0 +1,165 @@ +/* + * + * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** + * ***** DO NOT INCLUDE DIRECTLY ***** + * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** + */ + +/* + * The purpose of this header file is just to contain a list of trace code + * identifiers + * + * When updating this file, also remember to update + * mali_kbase_debug_linux_ktrace.h + * + * Each identifier is wrapped in a macro, so that its string form and enum form + * can be created + * + * Each macro is separated with a comma, to allow insertion into an array + * initializer or enum definition block. + * + * This allows automatic creation of an enum and a corresponding array of + * strings + * + * Before #including, the includer MUST #define KBASE_KTRACE_CODE_MAKE_CODE. + * After #including, the includer MUST #under KBASE_KTRACE_CODE_MAKE_CODE. + * + * e.g.: + * #define KBASE_KTRACE_CODE( X ) KBASE_KTRACE_CODE_ ## X + * typedef enum + * { + * #define KBASE_KTRACE_CODE_MAKE_CODE( X ) KBASE_KTRACE_CODE( X ) + * #include "mali_kbase_debug_ktrace_codes.h" + * #undef KBASE_KTRACE_CODE_MAKE_CODE + * } kbase_ktrace_code; + * + * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE + * + * + * The use of the macro here is: + * - KBASE_KTRACE_CODE_MAKE_CODE( X ) + * + * Which produces: + * - For an enum, KBASE_KTRACE_CODE_X + * - For a string, "X" + * + * + * For example: + * - KBASE_KTRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to: + * - KBASE_KTRACE_CODE_JM_JOB_COMPLETE for the enum + * - "JM_JOB_COMPLETE" for the string + * - To use it to trace an event, do: + * - KBASE_KTRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val ); + */ + +#if 0 /* Dummy section to avoid breaking formatting */ +int dummy_array[] = { +#endif + + /* + * Core events + */ + /* no info_val */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), + /* no info_val */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM), + /* info_val == GPU_IRQ_STATUS register */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ), + /* info_val == bits cleared */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR), + /* info_val == GPU_IRQ_STATUS register */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE), + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET), + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET), + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR), + /* info_val == dump address */ + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE), + KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES), + + /* + * Power Management Events + */ + KBASE_KTRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP), + KBASE_KTRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON_L2), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF_L2), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER), + KBASE_KTRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_NEEDED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_NEEDED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED), + KBASE_KTRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS), + /* info_val == kbdev->pm.active_count*/ + KBASE_KTRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE), + /* info_val == kbdev->pm.active_count*/ + KBASE_KTRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE), + KBASE_KTRACE_CODE_MAKE_CODE(PM_GPU_ON), + KBASE_KTRACE_CODE_MAKE_CODE(PM_GPU_OFF), + /* info_val == policy number, or -1 for "Already changing" */ + KBASE_KTRACE_CODE_MAKE_CODE(PM_SET_POLICY), + KBASE_KTRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY), + /* info_val == policy number */ + KBASE_KTRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT), + /* info_val == policy number */ + KBASE_KTRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM), + + /* + * Context Scheduler events + */ + /* info_val == kctx->refcount */ + KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RETAIN_CTX_NOLOCK), + /* info_val == kctx->refcount */ + KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RELEASE_CTX), + + +#if MALI_USE_CSF +#include "debug/backend/mali_kbase_debug_ktrace_codes_csf.h" +#else +#include "debug/backend/mali_kbase_debug_ktrace_codes_jm.h" +#endif + /* + * Unused code just to make it easier to not have a comma at the end. + * All other codes MUST come before this + */ + KBASE_KTRACE_CODE_MAKE_CODE(DUMMY) + +#if 0 /* Dummy section to avoid breaking formatting */ +}; +#endif + +/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_defs.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_defs.h new file mode 100755 index 000000000000..c680feb86387 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_defs.h @@ -0,0 +1,183 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_KTRACE_DEFS_H_ +#define _KBASE_DEBUG_KTRACE_DEFS_H_ + +/* Enable SW tracing when set */ +#if defined(CONFIG_MALI_BIFROST_ENABLE_TRACE) || defined(CONFIG_MALI_BIFROST_SYSTEM_TRACE) +#define KBASE_KTRACE_ENABLE 1 +#endif + +#ifndef KBASE_KTRACE_ENABLE +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define KBASE_KTRACE_ENABLE 1 +#else /* CONFIG_MALI_BIFROST_DEBUG */ +#define KBASE_KTRACE_ENABLE 0 +#endif /* CONFIG_MALI_BIFROST_DEBUG */ +#endif /* KBASE_KTRACE_ENABLE */ + +/* Select targets for recording of trace: + * + */ +#if KBASE_KTRACE_ENABLE + +#ifdef CONFIG_MALI_BIFROST_SYSTEM_TRACE +#define KBASE_KTRACE_TARGET_FTRACE 1 +#else /* CONFIG_MALI_BIFROST_SYSTEM_TRACE */ +#define KBASE_KTRACE_TARGET_FTRACE 0 +#endif /* CONFIG_MALI_BIFROST_SYSTEM_TRACE */ + +#ifdef CONFIG_MALI_BIFROST_ENABLE_TRACE +#define KBASE_KTRACE_TARGET_RBUF 1 +#else /* CONFIG_MALI_BIFROST_ENABLE_TRACE*/ +#define KBASE_KTRACE_TARGET_RBUF 0 +#endif /* CONFIG_MALI_BIFROST_ENABLE_TRACE */ + +#else /* KBASE_KTRACE_ENABLE */ +#define KBASE_KTRACE_TARGET_FTRACE 0 +#define KBASE_KTRACE_TARGET_RBUF 0 +#endif /* KBASE_KTRACE_ENABLE */ + +/* + * Note: Some backends define flags in this type even if the RBUF target is + * disabled (they get discarded with CSTD_UNUSED(), but they're still + * referenced) + */ +typedef u8 kbase_ktrace_flag_t; + +#if KBASE_KTRACE_TARGET_RBUF +typedef u8 kbase_ktrace_code_t; + +/* + * NOTE: KBASE_KTRACE_VERSION_MAJOR, KBASE_KTRACE_VERSION_MINOR are kept in + * the backend, since updates can be made to one backend in a way that doesn't + * affect the other. + * + * However, modifying the common part could require both backend versions to be + * updated. + */ + +/* + * struct kbase_ktrace_backend - backend specific part of a trace message + * + * At the very least, this must contain a kbase_ktrace_code_t 'code' member and + * a kbase_ktrace_flag_t 'flags' member + */ +struct kbase_ktrace_backend; +#endif /* KBASE_KTRACE_TARGET_RBUF */ + +#if MALI_USE_CSF +#include "debug/backend/mali_kbase_debug_ktrace_defs_csf.h" +#else +#include "debug/backend/mali_kbase_debug_ktrace_defs_jm.h" +#endif + +#if KBASE_KTRACE_TARGET_RBUF +/* Indicates if the trace message has backend related info. + * + * If not set, consider the &kbase_ktrace_backend part of a &kbase_ktrace_msg + * as uninitialized, apart from the mandatory parts: + * - code + * - flags + */ +#define KBASE_KTRACE_FLAG_BACKEND (((kbase_ktrace_flag_t)1) << 7) + +/* Collect all the common flags together for debug checking */ +#define KBASE_KTRACE_FLAG_COMMON_ALL \ + (KBASE_KTRACE_FLAG_BACKEND) + +#define KBASE_KTRACE_FLAG_ALL \ + (KBASE_KTRACE_FLAG_COMMON_ALL | KBASE_KTRACE_FLAG_BACKEND_ALL) + +#define KBASE_KTRACE_SHIFT 8 /* 256 entries */ +#define KBASE_KTRACE_SIZE (1 << KBASE_KTRACE_SHIFT) +#define KBASE_KTRACE_MASK ((1 << KBASE_KTRACE_SHIFT)-1) + +#define KBASE_KTRACE_CODE(X) KBASE_KTRACE_CODE_ ## X + +/* Note: compiletime_assert() about this against kbase_ktrace_code_t is in + * kbase_ktrace_init() + */ +enum kbase_ktrace_code { + /* + * IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE + * THIS MUST BE USED AT THE START OF THE ENUM + */ +#define KBASE_KTRACE_CODE_MAKE_CODE(X) KBASE_KTRACE_CODE(X) +#include +#undef KBASE_KTRACE_CODE_MAKE_CODE + /* Comma on its own, to extend the list */ + , + /* Must be the last in the enum */ + KBASE_KTRACE_CODE_COUNT +}; + +/** + * struct kbase_ktrace - object representing a trace message added to trace + * buffer trace_rbuf in &kbase_device + * @timestamp: CPU timestamp at which the trace message was added. + * @thread_id: id of the thread in the context of which trace message was + * added. + * @cpu: indicates which CPU the @thread_id was scheduled on when the + * trace message was added. + * @kctx_tgid: Thread group ID of the &kbase_context associated with the + * message, or 0 if none associated. + * @kctx_id: Unique identifier of the &kbase_context associated with the + * message. Only valid if @kctx_tgid != 0. + * @info_val: value specific to the type of event being traced. Refer to the + * specific code in enum kbase_ktrace_code + * @backend: backend-specific trace information. All backends must implement + * a minimum common set of members + */ +struct kbase_ktrace_msg { + struct timespec64 timestamp; + u32 thread_id; + u32 cpu; + pid_t kctx_tgid; + u32 kctx_id; + u64 info_val; + + struct kbase_ktrace_backend backend; +}; + +struct kbase_ktrace { + spinlock_t lock; + u16 first_out; + u16 next_in; + struct kbase_ktrace_msg *rbuf; +}; + + +static inline void kbase_ktrace_compiletime_asserts(void) +{ + /* See also documentation of enum kbase_ktrace_code */ + compiletime_assert(sizeof(kbase_ktrace_code_t) == sizeof(unsigned long long) || + KBASE_KTRACE_CODE_COUNT <= (1ull << (sizeof(kbase_ktrace_code_t) * BITS_PER_BYTE)), + "kbase_ktrace_code_t not wide enough for KBASE_KTRACE_CODE_COUNT"); + compiletime_assert((KBASE_KTRACE_FLAG_BACKEND_ALL & KBASE_KTRACE_FLAG_COMMON_ALL) == 0, + "KTrace backend flags intersect with KTrace common flags"); + +} + +#endif /* KBASE_KTRACE_TARGET_RBUF */ +#endif /* _KBASE_DEBUG_KTRACE_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_internal.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_internal.h new file mode 100755 index 000000000000..e450760e3426 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_internal.h @@ -0,0 +1,89 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_KTRACE_INTERNAL_H_ +#define _KBASE_DEBUG_KTRACE_INTERNAL_H_ + +#if KBASE_KTRACE_TARGET_RBUF + +#define KTRACE_DUMP_MESSAGE_SIZE 256 + +/** + * kbasep_ktrace_backend_format_header - format the backend part of the header + * @buffer: buffer to write to + * @sz: size of @buffer in bytes + * @written: pointer to storage for updating bytes written so far to @buffer + * + * The backend must format only the non-common backend specific parts of the + * header. It must format them as though they were standalone. The caller will + * handle adding any delimiters around this. + */ +void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written); + +/** + * kbasep_ktrace_backend_format_msg - format the backend part of the message + * @trace_msg: ktrace message + * @buffer: buffer to write to + * @sz: size of @buffer in bytes + * @written: pointer to storage for updating bytes written so far to @buffer + * + * The backend must format only the non-common backend specific parts of the + * message. It must format them as though they were standalone. The caller will + * handle adding any delimiters around this. + * + * A caller may have the flags member of @trace_msg with + * %KBASE_KTRACE_FLAG_BACKEND clear. The backend must handle that setting + * appropriately. + */ +void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, + char *buffer, int sz, s32 *written); + + +/** + * kbasep_ktrace_reserve - internal function to reserve space for a ktrace + * message + * @ktrace: kbase device's ktrace + * + * This may also empty the oldest entry in the ringbuffer to make space. + */ +struct kbase_ktrace_msg *kbasep_ktrace_reserve(struct kbase_ktrace *ktrace); + +/** + * kbasep_ktrace_msg_init - internal function to initialize just the common + * part of a ktrace message + * @ktrace: kbase device's ktrace + * @trace_msg: ktrace message to initialize + * @code: ktrace code + * @kctx: kbase context, or NULL if no context + * @flags: flags about the message + * @info_val: generic information about @code to add to the trace + * + * The common part includes the mandatory parts of the backend part + */ +void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace, + struct kbase_ktrace_msg *trace_msg, enum kbase_ktrace_code code, + struct kbase_context *kctx, kbase_ktrace_flag_t flags, + u64 info_val); + +#endif /* KBASE_KTRACE_TARGET_RBUF */ + +#endif /* _KBASE_DEBUG_KTRACE_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h new file mode 100755 index 000000000000..27f687faf072 --- /dev/null +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h @@ -0,0 +1,111 @@ +/* + * + * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * NOTE: This must **only** be included through mali_linux_trace.h, + * otherwise it will fail to setup tracepoints correctly + */ + +#if !defined(_KBASE_DEBUG_LINUX_KTRACE_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _KBASE_DEBUG_LINUX_KTRACE_H_ + +#if KBASE_KTRACE_TARGET_FTRACE + +DECLARE_EVENT_CLASS(mali_add_template, + TP_PROTO(struct kbase_context *kctx, u64 info_val), + TP_ARGS(kctx, info_val), + TP_STRUCT__entry( + __field(pid_t, kctx_tgid) + __field(u32, kctx_id) + __field(u64, info_val) + ), + TP_fast_assign( + __entry->kctx_id = (kctx) ? kctx->id : 0u; + __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; + __entry->info_val = info_val; + ), + TP_printk("kctx=%d_%u info=0x%llx", __entry->kctx_tgid, + __entry->kctx_id, __entry->info_val) +); + +/* DEFINE_MALI_ADD_EVENT is available also to backends for backend-specific + * simple trace codes + */ +#define DEFINE_MALI_ADD_EVENT(name) \ +DEFINE_EVENT(mali_add_template, mali_##name, \ + TP_PROTO(struct kbase_context *kctx, u64 info_val), \ + TP_ARGS(kctx, info_val)) +DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY); +DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE); +DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET); +DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET); +DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE); +DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR); +DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED); +DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP); +DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP); +DEFINE_MALI_ADD_EVENT(PM_PWRON); +DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER); +DEFINE_MALI_ADD_EVENT(PM_PWRON_L2); +DEFINE_MALI_ADD_EVENT(PM_PWROFF); +DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER); +DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2); +DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED); +DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER); +DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_TILER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_TILER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE); +DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER); +DEFINE_MALI_ADD_EVENT(PM_GPU_ON); +DEFINE_MALI_ADD_EVENT(PM_GPU_OFF); +DEFINE_MALI_ADD_EVENT(PM_SET_POLICY); +DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT); +DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM); +DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY); +DEFINE_MALI_ADD_EVENT(PM_CONTEXT_ACTIVE); +DEFINE_MALI_ADD_EVENT(PM_CONTEXT_IDLE); +DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); +DEFINE_MALI_ADD_EVENT(SCHED_RETAIN_CTX_NOLOCK); +DEFINE_MALI_ADD_EVENT(SCHED_RELEASE_CTX); + +#if MALI_USE_CSF +#include "mali_kbase_debug_linux_ktrace_csf.h" +#else +#include "mali_kbase_debug_linux_ktrace_jm.h" +#endif + +#undef DEFINE_MALI_ADD_EVENT + +#endif /* KBASE_KTRACE_TARGET_FTRACE */ + +#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_H_) || defined(TRACE_HEADER_MULTI_READ) */ diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c new file mode 100755 index 000000000000..d8b3fff6a214 --- /dev/null +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "../mali_kbase_device_internal.h" +#include "../mali_kbase_device.h" + +#include +#include +#include +#include +#include + +#ifdef CONFIG_MALI_BIFROST_NO_MALI +#include +#endif + +#include +#include +#include +#include +#include + +static void kbase_device_csf_firmware_term(struct kbase_device *kbdev) +{ + kbase_clk_rate_trace_manager_term(kbdev); + kbase_csf_firmware_term(kbdev); +} + +static int kbase_device_csf_firmware_init(struct kbase_device *kbdev) +{ + int err = kbase_csf_firmware_init(kbdev); + + if (!err) { + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.mcu_state = KBASE_MCU_ON; + kbdev->csf.firmware_inited = true; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + /* Post firmware init, idle condition is restored. Note this is + * a deferral action step from the late init stage for CSF. + */ + kbase_pm_context_idle(kbdev); + + if (!err) + kbase_clk_rate_trace_manager_init(kbdev); + + return err; +} + +/** + * kbase_backend_late_init - Perform any backend-specific initialization. + * @kbdev: Device pointer + * + * Return: 0 on success, or an error code on failure. + */ +static int kbase_backend_late_init(struct kbase_device *kbdev) +{ + int err; + + err = kbase_hwaccess_pm_init(kbdev); + if (err) + return err; + + err = kbase_reset_gpu_init(kbdev); + if (err) + goto fail_reset_gpu_init; + + err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); + if (err) + goto fail_pm_powerup; + + err = kbase_backend_timer_init(kbdev); + if (err) + goto fail_timer; + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#ifndef CONFIG_MALI_BIFROST_NO_MALI + if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { + dev_err(kbdev->dev, "Interrupt assignment check failed.\n"); + err = -EINVAL; + goto fail_interrupt_test; + } +#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + /* Do the initialisation of devfreq. + * Devfreq needs backend_timer_init() for completion of its + * initialisation and it also needs to catch the first callback + * occurrence of the runtime_suspend event for maintaining state + * coherence with the backend power management, hence needs to be + * placed before the kbase_pm_context_idle(). + */ + err = kbase_backend_devfreq_init(kbdev); + if (err) + goto fail_devfreq_init; + + /* Update gpuprops with L2_FEATURES if applicable */ + err = kbase_gpuprops_update_l2_features(kbdev); + if (err) + goto fail_update_l2_features; + + init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); + + return 0; + +fail_update_l2_features: +fail_devfreq_init: + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#ifndef CONFIG_MALI_BIFROST_NO_MALI +fail_interrupt_test: +#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + kbase_backend_timer_term(kbdev); +fail_timer: + kbase_hwaccess_pm_halt(kbdev); +fail_pm_powerup: + kbase_reset_gpu_term(kbdev); +fail_reset_gpu_init: + kbase_hwaccess_pm_term(kbdev); + + return err; +} + +/** + * kbase_backend_late_term - Perform any backend-specific termination. + * @kbdev: Device pointer + */ +static void kbase_backend_late_term(struct kbase_device *kbdev) +{ + kbase_backend_devfreq_term(kbdev); + kbase_hwaccess_pm_halt(kbdev); + kbase_reset_gpu_term(kbdev); + kbase_hwaccess_pm_term(kbdev); +} + +static const struct kbase_device_init dev_init[] = { +#ifdef CONFIG_MALI_BIFROST_NO_MALI + {kbase_gpu_device_create, kbase_gpu_device_destroy, + "Dummy model initialization failed"}, +#else + {assign_irqs, NULL, + "IRQ search failed"}, + {registers_map, registers_unmap, + "Register map failed"}, +#endif + {power_control_init, power_control_term, + "Power control initialization failed"}, + {kbase_device_io_history_init, kbase_device_io_history_term, + "Register access history initialization failed"}, + {kbase_device_early_init, kbase_device_early_term, + "Early device initialization failed"}, + {kbase_device_populate_max_freq, NULL, + "Populating max frequency failed"}, + {kbase_device_misc_init, kbase_device_misc_term, + "Miscellaneous device initialization failed"}, + {kbase_ctx_sched_init, kbase_ctx_sched_term, + "Context scheduler initialization failed"}, + {kbase_mem_init, kbase_mem_term, + "Memory subsystem initialization failed"}, + {kbase_csf_protected_memory_init, kbase_csf_protected_memory_term, + "Protected memory allocator initialization failed"}, + {kbase_device_coherency_init, NULL, + "Device coherency init failed"}, + {kbase_protected_mode_init, kbase_protected_mode_term, + "Protected mode subsystem initialization failed"}, + {kbase_device_list_init, kbase_device_list_term, + "Device list setup failed"}, + {kbase_device_timeline_init, kbase_device_timeline_term, + "Timeline stream initialization failed"}, + {kbase_clk_rate_trace_manager_init, + kbase_clk_rate_trace_manager_term, + "Clock rate trace manager initialization failed"}, + {kbase_device_hwcnt_backend_jm_init, + kbase_device_hwcnt_backend_jm_term, + "GPU hwcnt backend creation failed"}, + {kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term, + "GPU hwcnt context initialization failed"}, + {kbase_device_hwcnt_virtualizer_init, + kbase_device_hwcnt_virtualizer_term, + "GPU hwcnt virtualizer initialization failed"}, + {kbase_device_vinstr_init, kbase_device_vinstr_term, + "Virtual instrumentation initialization failed"}, + {kbase_backend_late_init, kbase_backend_late_term, + "Late backend initialization failed"}, + {kbase_device_csf_firmware_init, kbase_device_csf_firmware_term, + "Firmware initialization failed"}, +#ifdef MALI_KBASE_BUILD + {kbase_device_debugfs_init, kbase_device_debugfs_term, + "DebugFS initialization failed"}, + /* Sysfs init needs to happen before registering the device with + * misc_register(), otherwise it causes a race condition between + * registering the device and a uevent event being generated for + * userspace, causing udev rules to run which might expect certain + * sysfs attributes present. As a result of the race condition + * we avoid, some Mali sysfs entries may have appeared to udev + * to not exist. + * For more information, see + * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the + * paragraph that starts with "Word of warning", currently the + * second-last paragraph. + */ + {kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed"}, + {kbase_device_misc_register, kbase_device_misc_deregister, + "Misc device registration failed"}, +#ifdef CONFIG_MALI_BUSLOG + {buslog_init, buslog_term, "Bus log client registration failed"}, +#endif + {kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer, + "GPU property population failed"}, +#endif +}; + +static void kbase_device_term_partial(struct kbase_device *kbdev, + unsigned int i) +{ + while (i-- > 0) { + if (dev_init[i].term) + dev_init[i].term(kbdev); + } +} + +void kbase_device_term(struct kbase_device *kbdev) +{ + kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init)); + kbase_mem_halt(kbdev); +} + +int kbase_device_init(struct kbase_device *kbdev) +{ + int err = 0; + unsigned int i = 0; + + dev_info(kbdev->dev, "Kernel DDK version %s", MALI_RELEASE_NAME); + + kbase_device_id_init(kbdev); + kbase_disjoint_init(kbdev); + + for (i = 0; i < ARRAY_SIZE(dev_init); i++) { + err = dev_init[i].init(kbdev); + if (err) { + dev_err(kbdev->dev, "%s error = %d\n", + dev_init[i].err_mes, err); + kbase_device_term_partial(kbdev, i); + break; + } + } + + return err; +} diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c new file mode 100755 index 000000000000..97bcc1d23aa3 --- /dev/null +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * kbase_report_gpu_fault - Report a GPU fault of the device. + * + * @kbdev: Kbase device pointer + * @status: Fault status + * @as_nr: Faulty address space + * @as_valid: true if address space is valid + * + * This function is called from the interrupt handler when a GPU fault occurs. + */ +static void kbase_report_gpu_fault(struct kbase_device *kbdev, u32 status, + u32 as_nr, bool as_valid) +{ + u64 address = (u64) kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32; + + address |= kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTADDRESS_LO)); + + /* Report GPU fault for all contexts in case either + * the address space is invalid or it's MCU address space. + */ + kbase_mmu_gpu_fault_interrupt(kbdev, status, as_nr, address, as_valid); +} + +static bool kbase_gpu_fault_interrupt(struct kbase_device *kbdev) +{ + const u32 status = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTSTATUS)); + const bool as_valid = status & GPU_FAULTSTATUS_JASID_VALID_FLAG; + const u32 as_nr = (status & GPU_FAULTSTATUS_JASID_MASK) >> + GPU_FAULTSTATUS_JASID_SHIFT; + bool bus_fault = (status & GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) == + GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_BUS_FAULT; + bool clear_gpu_fault = true; + + if (bus_fault) { + /* If as_valid, reset gpu when ASID is for MCU. */ + if (!as_valid || (as_nr == MCU_AS_NR)) { + kbase_report_gpu_fault(kbdev, status, as_nr, as_valid); + + dev_err(kbdev->dev, "GPU bus fault triggering gpu-reset ...\n"); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + } else { + /* Handle Bus fault */ + if (kbase_mmu_bus_fault_interrupt(kbdev, status, as_nr)) + clear_gpu_fault = false; + } + } else + kbase_report_gpu_fault(kbdev, status, as_nr, as_valid); + + return clear_gpu_fault; +} + +void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) +{ + bool clear_gpu_fault = false; + + KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); + if (val & GPU_FAULT) + clear_gpu_fault = kbase_gpu_fault_interrupt(kbdev); + + if (val & GPU_PROTECTED_FAULT) { + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + dev_err_ratelimited(kbdev->dev, "GPU fault in protected mode"); + + /* Mask the protected fault interrupt to avoid the potential + * deluge of such interrupts. It will be unmasked on GPU reset. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + GPU_IRQ_REG_ALL & ~GPU_PROTECTED_FAULT); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + if (!WARN_ON(!kbase_csf_scheduler_protected_mode_in_use(kbdev))) + scheduler->active_protm_grp->faulted = true; + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + } + + if (val & RESET_COMPLETED) + kbase_pm_reset_done(kbdev); + + KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val); + + /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must + * be called after the IRQ has been cleared. This is because it might + * trigger further power transitions and we don't want to miss the + * interrupt raised to notify us that these further transitions have + * finished. The same applies to kbase_clean_caches_done() - if another + * clean was queued, it might trigger another clean, which might + * generate another interrupt which shouldn't be missed. + */ + + if (val & CLEAN_CACHES_COMPLETED) + kbase_clean_caches_done(kbdev); + + if (val & (POWER_CHANGED_ALL | MCU_STATUS_GPU_IRQ)) { + kbase_pm_power_changed(kbdev); + } else if (val & CLEAN_CACHES_COMPLETED) { + /* If cache line evict messages can be lost when shader cores + * power down then we need to flush the L2 cache before powering + * down cores. When the flush completes, the shaders' state + * machine needs to be re-invoked to proceed with powering down + * cores. + */ + if (kbdev->pm.backend.l2_always_on || + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) + kbase_pm_power_changed(kbdev); + } + + if (clear_gpu_fault) { + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAR_FAULT); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); +} diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c new file mode 100755 index 000000000000..a11d778071b5 --- /dev/null +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include +#include +#include + +/** + * kbase_report_gpu_fault - Report a GPU fault. + * @kbdev: Kbase device pointer + * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS + * was also set + * + * This function is called from the interrupt handler when a GPU fault occurs. + * It reports the details of the fault using dev_warn(). + */ +static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) +{ + u32 status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS)); + u64 address = (u64) kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32; + + address |= kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTADDRESS_LO)); + + dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx", + status, + kbase_gpu_exception_name(status & 0xFF), + address); + if (multiple) + dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n"); +} + +void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) +{ + KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); + if (val & GPU_FAULT) + kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS); + + if (val & RESET_COMPLETED) + kbase_pm_reset_done(kbdev); + + if (val & PRFCNT_SAMPLE_COMPLETED) + kbase_instr_hwcnt_sample_done(kbdev); + + KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val); + + /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must + * be called after the IRQ has been cleared. This is because it might + * trigger further power transitions and we don't want to miss the + * interrupt raised to notify us that these further transitions have + * finished. The same applies to kbase_clean_caches_done() - if another + * clean was queued, it might trigger another clean, which might + * generate another interrupt which shouldn't be missed. + */ + + if (val & CLEAN_CACHES_COMPLETED) + kbase_clean_caches_done(kbdev); + + if (val & POWER_CHANGED_ALL) { + kbase_pm_power_changed(kbdev); + } else if (val & CLEAN_CACHES_COMPLETED) { + /* If cache line evict messages can be lost when shader cores + * power down then we need to flush the L2 cache before powering + * down cores. When the flush completes, the shaders' state + * machine needs to be re-invoked to proceed with powering down + * cores. + */ + if (kbdev->pm.backend.l2_always_on || + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) + kbase_pm_power_changed(kbdev); + } + + KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); +} diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c new file mode 100755 index 000000000000..8e853eb82fa1 --- /dev/null +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "../mali_kbase_device_internal.h" +#include "../mali_kbase_device.h" + +#include +#include +#include +#include + +#ifdef CONFIG_MALI_BIFROST_NO_MALI +#include +#endif + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +/** + * kbase_backend_late_init - Perform any backend-specific initialization. + * @kbdev: Device pointer + * + * Return: 0 on success, or an error code on failure. + */ +static int kbase_backend_late_init(struct kbase_device *kbdev) +{ + int err; + + err = kbase_hwaccess_pm_init(kbdev); + if (err) + return err; + + err = kbase_reset_gpu_init(kbdev); + if (err) + goto fail_reset_gpu_init; + + err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); + if (err) + goto fail_pm_powerup; + + err = kbase_backend_timer_init(kbdev); + if (err) + goto fail_timer; + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#ifndef CONFIG_MALI_BIFROST_NO_MALI + if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { + dev_err(kbdev->dev, "Interrupt assignment check failed.\n"); + err = -EINVAL; + goto fail_interrupt_test; + } +#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + err = kbase_job_slot_init(kbdev); + if (err) + goto fail_job_slot; + + /* Do the initialisation of devfreq. + * Devfreq needs backend_timer_init() for completion of its + * initialisation and it also needs to catch the first callback + * occurrence of the runtime_suspend event for maintaining state + * coherence with the backend power management, hence needs to be + * placed before the kbase_pm_context_idle(). + */ + err = kbase_backend_devfreq_init(kbdev); + if (err) + goto fail_devfreq_init; + + /* Idle the GPU and/or cores, if the policy wants it to */ + kbase_pm_context_idle(kbdev); + + /* Update gpuprops with L2_FEATURES if applicable */ + err = kbase_gpuprops_update_l2_features(kbdev); + if (err) + goto fail_update_l2_features; + + init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); + + return 0; + +fail_update_l2_features: +fail_devfreq_init: + kbase_job_slot_term(kbdev); +fail_job_slot: + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#ifndef CONFIG_MALI_BIFROST_NO_MALI +fail_interrupt_test: +#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + kbase_backend_timer_term(kbdev); +fail_timer: + kbase_hwaccess_pm_halt(kbdev); +fail_pm_powerup: + kbase_reset_gpu_term(kbdev); +fail_reset_gpu_init: + kbase_hwaccess_pm_term(kbdev); + + return err; +} + +/** + * kbase_backend_late_term - Perform any backend-specific termination. + * @kbdev: Device pointer + */ +static void kbase_backend_late_term(struct kbase_device *kbdev) +{ + kbase_backend_devfreq_term(kbdev); + kbase_job_slot_halt(kbdev); + kbase_job_slot_term(kbdev); + kbase_backend_timer_term(kbdev); + kbase_hwaccess_pm_halt(kbdev); + kbase_reset_gpu_term(kbdev); + kbase_hwaccess_pm_term(kbdev); +} + +static const struct kbase_device_init dev_init[] = { +#ifdef CONFIG_MALI_BIFROST_NO_MALI + {kbase_gpu_device_create, kbase_gpu_device_destroy, + "Dummy model initialization failed"}, +#else + {assign_irqs, NULL, + "IRQ search failed"}, + {registers_map, registers_unmap, + "Register map failed"}, +#endif + {kbase_device_io_history_init, kbase_device_io_history_term, + "Register access history initialization failed"}, + {kbase_device_pm_init, kbase_device_pm_term, + "Power management initialization failed"}, + {kbase_device_early_init, kbase_device_early_term, + "Early device initialization failed"}, + {kbase_device_populate_max_freq, NULL, + "Populating max frequency failed"}, + {kbase_device_misc_init, kbase_device_misc_term, + "Miscellaneous device initialization failed"}, + {kbase_ctx_sched_init, kbase_ctx_sched_term, + "Context scheduler initialization failed"}, + {kbase_mem_init, kbase_mem_term, + "Memory subsystem initialization failed"}, + {kbase_device_coherency_init, NULL, + "Device coherency init failed"}, + {kbase_protected_mode_init, kbase_protected_mode_term, + "Protected mode subsystem initialization failed"}, + {kbase_device_list_init, kbase_device_list_term, + "Device list setup failed"}, + {kbasep_js_devdata_init, kbasep_js_devdata_term, + "Job JS devdata initialization failed"}, + {kbase_device_timeline_init, kbase_device_timeline_term, + "Timeline stream initialization failed"}, + {kbase_clk_rate_trace_manager_init, + kbase_clk_rate_trace_manager_term, + "Clock rate trace manager initialization failed"}, + {kbase_device_hwcnt_backend_jm_init, + kbase_device_hwcnt_backend_jm_term, + "GPU hwcnt backend creation failed"}, + {kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term, + "GPU hwcnt context initialization failed"}, + {kbase_device_hwcnt_virtualizer_init, + kbase_device_hwcnt_virtualizer_term, + "GPU hwcnt virtualizer initialization failed"}, + {kbase_device_vinstr_init, kbase_device_vinstr_term, + "Virtual instrumentation initialization failed"}, + {kbase_backend_late_init, kbase_backend_late_term, + "Late backend initialization failed"}, +#ifdef MALI_KBASE_BUILD + {kbase_debug_job_fault_dev_init, kbase_debug_job_fault_dev_term, + "Job fault debug initialization failed"}, + {kbase_device_debugfs_init, kbase_device_debugfs_term, + "DebugFS initialization failed"}, + /* Sysfs init needs to happen before registering the device with + * misc_register(), otherwise it causes a race condition between + * registering the device and a uevent event being generated for + * userspace, causing udev rules to run which might expect certain + * sysfs attributes present. As a result of the race condition + * we avoid, some Mali sysfs entries may have appeared to udev + * to not exist. + * For more information, see + * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the + * paragraph that starts with "Word of warning", currently the + * second-last paragraph. + */ + {kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed"}, + {kbase_device_misc_register, kbase_device_misc_deregister, + "Misc device registration failed"}, +#ifdef CONFIG_MALI_BUSLOG + {buslog_init, buslog_term, "Bus log client registration failed"}, +#endif + {kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer, + "GPU property population failed"}, +#endif + {kbase_dummy_job_wa_load, kbase_dummy_job_wa_cleanup, + "Dummy job workaround load failed"}, +}; + +static void kbase_device_term_partial(struct kbase_device *kbdev, + unsigned int i) +{ + while (i-- > 0) { + if (dev_init[i].term) + dev_init[i].term(kbdev); + } +} + +void kbase_device_term(struct kbase_device *kbdev) +{ + kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init)); + kbasep_js_devdata_halt(kbdev); + kbase_mem_halt(kbdev); +} + +int kbase_device_init(struct kbase_device *kbdev) +{ + int err = 0; + unsigned int i = 0; + + dev_info(kbdev->dev, "Kernel DDK version %s", MALI_RELEASE_NAME); + + kbase_device_id_init(kbdev); + kbase_disjoint_init(kbdev); + + for (i = 0; i < ARRAY_SIZE(dev_init); i++) { + err = dev_init[i].init(kbdev); + if (err) { + dev_err(kbdev->dev, "%s error = %d\n", + dev_init[i].err_mes, err); + kbase_device_term_partial(kbdev, i); + break; + } + } + + return err; +} diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device.c b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c new file mode 100755 index 000000000000..76fb33a5e881 --- /dev/null +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Base kernel device APIs + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include "mali_kbase_vinstr.h" +#include "mali_kbase_hwcnt_context.h" +#include "mali_kbase_hwcnt_virtualizer.h" + +#include "mali_kbase_device.h" +#include "backend/gpu/mali_kbase_pm_internal.h" +#include "backend/gpu/mali_kbase_irq_internal.h" +#include "mali_kbase_regs_history_debugfs.h" + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include "arbiter/mali_kbase_arbiter_pm.h" +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + +/* NOTE: Magic - 0x45435254 (TRCE in ASCII). + * Supports tracing feature provided in the base module. + * Please keep it in sync with the value of base module. + */ +#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254 + +/* Number of register accesses for the buffer that we allocate during + * initialization time. The buffer size can be changed later via debugfs. + */ +#define KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ((u16)512) + +static DEFINE_MUTEX(kbase_dev_list_lock); +static LIST_HEAD(kbase_dev_list); +static int kbase_dev_nr; + +struct kbase_device *kbase_device_alloc(void) +{ + return kzalloc(sizeof(struct kbase_device), GFP_KERNEL); +} + +/** + * kbase_device_all_as_init() - Initialise address space objects of the device. + * + * @kbdev: Pointer to kbase device. + * + * Return: 0 on success otherwise non-zero. + */ +static int kbase_device_all_as_init(struct kbase_device *kbdev) +{ + int i, err = 0; + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + err = kbase_mmu_as_init(kbdev, i); + if (err) + break; + } + + if (err) { + while (i-- > 0) + kbase_mmu_as_term(kbdev, i); + } + + return err; +} + +static void kbase_device_all_as_term(struct kbase_device *kbdev) +{ + int i; + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) + kbase_mmu_as_term(kbdev, i); +} + +int kbase_device_misc_init(struct kbase_device * const kbdev) +{ + int err; +#ifdef CONFIG_ARM64 + struct device_node *np = NULL; +#endif /* CONFIG_ARM64 */ + + spin_lock_init(&kbdev->mmu_mask_change); + mutex_init(&kbdev->mmu_hw_mutex); +#ifdef CONFIG_ARM64 + kbdev->cci_snoop_enabled = false; + np = kbdev->dev->of_node; + if (np != NULL) { + if (of_property_read_u32(np, "snoop_enable_smc", + &kbdev->snoop_enable_smc)) + kbdev->snoop_enable_smc = 0; + if (of_property_read_u32(np, "snoop_disable_smc", + &kbdev->snoop_disable_smc)) + kbdev->snoop_disable_smc = 0; + /* Either both or none of the calls should be provided. */ + if (!((kbdev->snoop_disable_smc == 0 + && kbdev->snoop_enable_smc == 0) + || (kbdev->snoop_disable_smc != 0 + && kbdev->snoop_enable_smc != 0))) { + WARN_ON(1); + err = -EINVAL; + goto fail; + } + } +#endif /* CONFIG_ARM64 */ + /* Get the list of workarounds for issues on the current HW + * (identified by the GPU_ID register) + */ + err = kbase_hw_set_issues_mask(kbdev); + if (err) + goto fail; + + /* Set the list of features available on the current HW + * (identified by the GPU_ID register) + */ + kbase_hw_set_features_mask(kbdev); + + err = kbase_gpuprops_set_features(kbdev); + if (err) + goto fail; + + /* On Linux 4.0+, dma coherency is determined from device tree */ +#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) + set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops); +#endif + + /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our + * device structure was created by device-tree + */ + if (!kbdev->dev->dma_mask) + kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask; + + err = dma_set_mask(kbdev->dev, + DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); + if (err) + goto dma_set_mask_failed; + + err = dma_set_coherent_mask(kbdev->dev, + DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); + if (err) + goto dma_set_mask_failed; + + kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces; + + err = kbase_device_all_as_init(kbdev); + if (err) + goto dma_set_mask_failed; + + spin_lock_init(&kbdev->hwcnt.lock); + + err = kbase_ktrace_init(kbdev); + if (err) + goto term_as; + + init_waitqueue_head(&kbdev->cache_clean_wait); + + kbase_debug_assert_register_hook(&kbase_ktrace_hook_wrapper, kbdev); + + atomic_set(&kbdev->ctx_num, 0); + + err = kbase_instr_backend_init(kbdev); + if (err) + goto term_trace; + + kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; + + kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); + else + kbdev->mmu_mode = kbase_mmu_mode_get_lpae(); + + mutex_init(&kbdev->kctx_list_lock); + INIT_LIST_HEAD(&kbdev->kctx_list); + + spin_lock_init(&kbdev->hwaccess_lock); + + return 0; +term_trace: + kbase_ktrace_term(kbdev); +term_as: + kbase_device_all_as_term(kbdev); +dma_set_mask_failed: +fail: + return err; +} + +void kbase_device_misc_term(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + + WARN_ON(!list_empty(&kbdev->kctx_list)); + +#if KBASE_KTRACE_ENABLE + kbase_debug_assert_register_hook(NULL, NULL); +#endif + + kbase_instr_backend_term(kbdev); + + kbase_ktrace_term(kbdev); + + kbase_device_all_as_term(kbdev); +} + +void kbase_device_free(struct kbase_device *kbdev) +{ + kfree(kbdev); +} + +void kbase_device_id_init(struct kbase_device *kbdev) +{ + scnprintf(kbdev->devname, DEVNAME_SIZE, "%s%d", kbase_drv_name, + kbase_dev_nr); + kbdev->id = kbase_dev_nr; +} + +void kbase_increment_device_id(void) +{ + kbase_dev_nr++; +} + +int kbase_device_hwcnt_backend_jm_init(struct kbase_device *kbdev) +{ + return kbase_hwcnt_backend_jm_create(kbdev, &kbdev->hwcnt_gpu_iface); +} + +void kbase_device_hwcnt_backend_jm_term(struct kbase_device *kbdev) +{ + kbase_hwcnt_backend_jm_destroy(&kbdev->hwcnt_gpu_iface); +} + +int kbase_device_hwcnt_context_init(struct kbase_device *kbdev) +{ + return kbase_hwcnt_context_init(&kbdev->hwcnt_gpu_iface, + &kbdev->hwcnt_gpu_ctx); +} + +void kbase_device_hwcnt_context_term(struct kbase_device *kbdev) +{ + kbase_hwcnt_context_term(kbdev->hwcnt_gpu_ctx); +} + +int kbase_device_hwcnt_virtualizer_init(struct kbase_device *kbdev) +{ + return kbase_hwcnt_virtualizer_init(kbdev->hwcnt_gpu_ctx, + KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS, + &kbdev->hwcnt_gpu_virt); +} + +void kbase_device_hwcnt_virtualizer_term(struct kbase_device *kbdev) +{ + kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt); +} + +int kbase_device_timeline_init(struct kbase_device *kbdev) +{ + atomic_set(&kbdev->timeline_flags, 0); + return kbase_timeline_init(&kbdev->timeline, &kbdev->timeline_flags); +} + +void kbase_device_timeline_term(struct kbase_device *kbdev) +{ + kbase_timeline_term(kbdev->timeline); +} + +int kbase_device_vinstr_init(struct kbase_device *kbdev) +{ + return kbase_vinstr_init(kbdev->hwcnt_gpu_virt, &kbdev->vinstr_ctx); +} + +void kbase_device_vinstr_term(struct kbase_device *kbdev) +{ + kbase_vinstr_term(kbdev->vinstr_ctx); +} + +int kbase_device_io_history_init(struct kbase_device *kbdev) +{ + return kbase_io_history_init(&kbdev->io_history, + KBASEP_DEFAULT_REGISTER_HISTORY_SIZE); +} + +void kbase_device_io_history_term(struct kbase_device *kbdev) +{ + kbase_io_history_term(&kbdev->io_history); +} + +int kbase_device_misc_register(struct kbase_device *kbdev) +{ + return misc_register(&kbdev->mdev); +} + +void kbase_device_misc_deregister(struct kbase_device *kbdev) +{ + misc_deregister(&kbdev->mdev); +} + +int kbase_device_list_init(struct kbase_device *kbdev) +{ + const struct list_head *dev_list; + + dev_list = kbase_device_get_list(); + list_add(&kbdev->entry, &kbase_dev_list); + kbase_device_put_list(dev_list); + + return 0; +} + +void kbase_device_list_term(struct kbase_device *kbdev) +{ + const struct list_head *dev_list; + + dev_list = kbase_device_get_list(); + list_del(&kbdev->entry); + kbase_device_put_list(dev_list); +} + +const struct list_head *kbase_device_get_list(void) +{ + mutex_lock(&kbase_dev_list_lock); + return &kbase_dev_list; +} +KBASE_EXPORT_TEST_API(kbase_device_get_list); + +void kbase_device_put_list(const struct list_head *dev_list) +{ + mutex_unlock(&kbase_dev_list_lock); +} +KBASE_EXPORT_TEST_API(kbase_device_put_list); + +int kbase_device_early_init(struct kbase_device *kbdev) +{ + int err; + + err = kbasep_platform_device_init(kbdev); + if (err) + return err; + + err = kbase_pm_runtime_init(kbdev); + if (err) + goto fail_runtime_pm; + + /* Ensure we can access the GPU registers */ + kbase_pm_register_access_enable(kbdev); + + /* Find out GPU properties based on the GPU feature registers */ + kbase_gpuprops_set(kbdev); + + /* We're done accessing the GPU registers for now. */ + kbase_pm_register_access_disable(kbdev); + + err = kbase_install_interrupts(kbdev); + if (err) + goto fail_interrupts; + + return 0; + +fail_interrupts: + kbase_pm_runtime_term(kbdev); +fail_runtime_pm: + kbasep_platform_device_term(kbdev); + + return err; +} + +void kbase_device_early_term(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbdev->arb.arb_if) + kbase_arbiter_pm_release_interrupts(kbdev); + else + kbase_release_interrupts(kbdev); +#else + kbase_release_interrupts(kbdev); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + kbase_pm_runtime_term(kbdev); + kbasep_platform_device_term(kbdev); +} diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device.h b/drivers/gpu/arm/bifrost/device/mali_kbase_device.h new file mode 100755 index 000000000000..33264bcc0464 --- /dev/null +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device.h @@ -0,0 +1,177 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include + +/** + * kbase_device_get_list - get device list. + * + * Get access to device list. + * + * Return: Pointer to the linked list head. + */ +const struct list_head *kbase_device_get_list(void); + +/** + * kbase_device_put_list - put device list. + * + * @dev_list: head of linked list containing device list. + * + * Put access to the device list. + */ +void kbase_device_put_list(const struct list_head *dev_list); + +/** + * Kbase_increment_device_id - increment device id. + * + * Used to increment device id on successful initialization of the device. + */ +void kbase_increment_device_id(void); + +/** + * kbase_device_init - Device initialisation. + * + * This is called from device probe to initialise various other + * components needed. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 on success and non-zero value on failure. + */ +int kbase_device_init(struct kbase_device *kbdev); + +/** + * kbase_device_term - Device termination. + * + * This is called from device remove to terminate various components that + * were initialised during kbase_device_init. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + */ +void kbase_device_term(struct kbase_device *kbdev); + +/** + * kbase_reg_write - write to GPU register + * @kbdev: Kbase device pointer + * @offset: Offset of register + * @value: Value to write + * + * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). + */ +void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value); + +/** + * kbase_reg_read - read from GPU register + * @kbdev: Kbase device pointer + * @offset: Offset of register + * + * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). + * + * Return: Value in desired register + */ +u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset); + +/** + * kbase_is_gpu_removed() - Has the GPU been removed. + * @kbdev: Kbase device pointer + * + * When Kbase takes too long to give up the GPU, the Arbiter + * can remove it. This will then be followed by a GPU lost event. + * This function will return true if the GPU has been removed. + * When this happens register reads will be zero. A zero GPU_ID is + * invalid so this is used to detect when GPU is removed. + * + * Return: True if GPU removed + */ +bool kbase_is_gpu_removed(struct kbase_device *kbdev); + +/** + * kbase_gpu_start_cache_clean - Start a cache clean + * @kbdev: Kbase device + * + * Issue a cache clean and invalidate command to hardware. This function will + * take hwaccess_lock. + */ +void kbase_gpu_start_cache_clean(struct kbase_device *kbdev); + +/** + * kbase_gpu_start_cache_clean_nolock - Start a cache clean + * @kbdev: Kbase device + * + * Issue a cache clean and invalidate command to hardware. hwaccess_lock + * must be held by the caller. + */ +void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev); + +/** + * kbase_gpu_wait_cache_clean - Wait for cache cleaning to finish + * @kbdev: Kbase device + * + * This function will take hwaccess_lock, and may sleep. + */ +void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev); + +/** + * kbase_gpu_wait_cache_clean_timeout - Wait for certain time for cache + * cleaning to finish + * @kbdev: Kbase device + * @wait_timeout_ms: Time in milliseconds, to wait for cache clean to complete. + * + * This function will take hwaccess_lock, and may sleep. This is supposed to be + * called from paths (like GPU reset) where an indefinite wait for the + * completion of cache clean operation can cause deadlock, as the operation may + * never complete. + * + * Return: 0 if successful or a negative error code on failure. + */ +int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev, + unsigned int wait_timeout_ms); + +/** + * kbase_gpu_cache_clean_wait_complete - Called after the cache cleaning is + * finished. Would also be called after + * the GPU reset. + * @kbdev: Kbase device + * + * Caller must hold the hwaccess_lock. + */ +void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev); + +/** + * kbase_clean_caches_done - Issue preiously queued cache clean request or + * wake up the requester that issued cache clean. + * @kbdev: Kbase device + * + * Caller must hold the hwaccess_lock. + */ +void kbase_clean_caches_done(struct kbase_device *kbdev); + +/** + * kbase_gpu_interrupt - GPU interrupt handler + * @kbdev: Kbase device pointer + * @val: The value of the GPU IRQ status register which triggered the call + * + * This function is called from the interrupt handler when a GPU irq is to be + * handled. + */ +void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c b/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c new file mode 100755 index 000000000000..3a75c6c05cfa --- /dev/null +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2014-2016, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#if !defined(CONFIG_MALI_BIFROST_NO_MALI) +void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value) +{ + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + KBASE_DEBUG_ASSERT(kbdev->dev != NULL); + + writel(value, kbdev->reg + offset); + +#ifdef CONFIG_DEBUG_FS + if (unlikely(kbdev->io_history.enabled)) + kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, + value, 1); +#endif /* CONFIG_DEBUG_FS */ + dev_dbg(kbdev->dev, "w: reg %08x val %08x", offset, value); +} + +KBASE_EXPORT_TEST_API(kbase_reg_write); + +u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset) +{ + u32 val; + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + KBASE_DEBUG_ASSERT(kbdev->dev != NULL); + + val = readl(kbdev->reg + offset); + +#ifdef CONFIG_DEBUG_FS + if (unlikely(kbdev->io_history.enabled)) + kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, + val, 0); +#endif /* CONFIG_DEBUG_FS */ + dev_dbg(kbdev->dev, "r: reg %08x val %08x", offset, val); + + return val; +} + +KBASE_EXPORT_TEST_API(kbase_reg_read); + +bool kbase_is_gpu_removed(struct kbase_device *kbdev) +{ + u32 val; + + val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); + + return val == 0; +} +#endif /* !defined(CONFIG_MALI_BIFROST_NO_MALI) */ + +void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev) +{ + u32 irq_mask; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->cache_clean_in_progress) { + /* If this is called while another clean is in progress, we + * can't rely on the current one to flush any new changes in + * the cache. Instead, trigger another cache clean immediately + * after this one finishes. + */ + kbdev->cache_clean_queued = true; + return; + } + + /* Enable interrupt */ + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + irq_mask | CLEAN_CACHES_COMPLETED); + + KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAN_INV_CACHES); + + kbdev->cache_clean_in_progress = true; +} + +void kbase_gpu_start_cache_clean(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_gpu_start_cache_clean_nolock(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->cache_clean_queued = false; + kbdev->cache_clean_in_progress = false; + wake_up(&kbdev->cache_clean_wait); +} + +void kbase_clean_caches_done(struct kbase_device *kbdev) +{ + u32 irq_mask; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (kbdev->cache_clean_queued) { + kbdev->cache_clean_queued = false; + + KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAN_INV_CACHES); + } else { + /* Disable interrupt */ + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + irq_mask & ~CLEAN_CACHES_COMPLETED); + + kbase_gpu_cache_clean_wait_complete(kbdev); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +static inline bool get_cache_clean_flag(struct kbase_device *kbdev) +{ + bool cache_clean_in_progress; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + cache_clean_in_progress = kbdev->cache_clean_in_progress; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return cache_clean_in_progress; +} + +void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev) +{ + while (get_cache_clean_flag(kbdev)) { + wait_event_interruptible(kbdev->cache_clean_wait, + !kbdev->cache_clean_in_progress); + } +} + +int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev, + unsigned int wait_timeout_ms) +{ + long remaining = msecs_to_jiffies(wait_timeout_ms); + + while (remaining && get_cache_clean_flag(kbdev)) { + remaining = wait_event_timeout(kbdev->cache_clean_wait, + !kbdev->cache_clean_in_progress, + remaining); + } + + return (remaining ? 0 : -ETIMEDOUT); +} diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device_internal.h b/drivers/gpu/arm/bifrost/device/mali_kbase_device_internal.h new file mode 100755 index 000000000000..54644582eac5 --- /dev/null +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device_internal.h @@ -0,0 +1,78 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include + +typedef int kbase_device_init_method(struct kbase_device *kbdev); +typedef void kbase_device_term_method(struct kbase_device *kbdev); + +/** + * struct kbase_device_init - Device init/term methods. + * @init: Function pointer to a initialise method. + * @term: Function pointer to a terminate method. + * @err_mes: Error message to be printed when init method fails. + */ +struct kbase_device_init { + kbase_device_init_method *init; + kbase_device_term_method *term; + char *err_mes; +}; + +int kbase_device_vinstr_init(struct kbase_device *kbdev); +void kbase_device_vinstr_term(struct kbase_device *kbdev); + +int kbase_device_timeline_init(struct kbase_device *kbdev); +void kbase_device_timeline_term(struct kbase_device *kbdev); + +int kbase_device_hwcnt_backend_jm_init(struct kbase_device *kbdev); +void kbase_device_hwcnt_backend_jm_term(struct kbase_device *kbdev); + +int kbase_device_hwcnt_context_init(struct kbase_device *kbdev); +void kbase_device_hwcnt_context_term(struct kbase_device *kbdev); + +int kbase_device_hwcnt_virtualizer_init(struct kbase_device *kbdev); +void kbase_device_hwcnt_virtualizer_term(struct kbase_device *kbdev); + +int kbase_device_list_init(struct kbase_device *kbdev); +void kbase_device_list_term(struct kbase_device *kbdev); + +int kbase_device_io_history_init(struct kbase_device *kbdev); +void kbase_device_io_history_term(struct kbase_device *kbdev); + +int kbase_device_misc_register(struct kbase_device *kbdev); +void kbase_device_misc_deregister(struct kbase_device *kbdev); + +void kbase_device_id_init(struct kbase_device *kbdev); + +/** + * kbase_device_early_init - Perform any device-specific initialization. + * @kbdev: Device pointer + * + * Return: 0 on success, or an error code on failure. + */ +int kbase_device_early_init(struct kbase_device *kbdev); + +/** + * kbase_device_early_term - Perform any device-specific termination. + * @kbdev: Device pointer + */ +void kbase_device_early_term(struct kbase_device *kbdev); diff --git a/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c new file mode 100755 index 000000000000..f7e9b125ba8b --- /dev/null +++ b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c @@ -0,0 +1,105 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include "csf/mali_gpu_csf_registers.h" +#include "../mali_kbase_gpu_fault.h" + +const char *kbase_gpu_exception_name(u32 const exception_code) +{ + const char *e; + + switch (exception_code) { + /* Command Stream exceptions */ + case CS_FAULT_EXCEPTION_TYPE_CS_RESOURCE_TERMINATED: + e = "CS_RESOURCE_TERMINATED"; + break; + case CS_FAULT_EXCEPTION_TYPE_CS_INHERIT_FAULT: + e = "CS_INHERIT_FAULT"; + break; + /* Command Stream fatal exceptions */ + case CS_FATAL_EXCEPTION_TYPE_CS_CONFIG_FAULT: + e = "CS_CONFIG_FAULT"; + break; + case CS_FATAL_EXCEPTION_TYPE_CS_ENDPOINT_FAULT: + e = "FATAL_CS_ENDPOINT_FAULT"; + break; + case CS_FATAL_EXCEPTION_TYPE_CS_BUS_FAULT: + e = "FATAL_CS_BUS_FAULT"; + break; + case CS_FATAL_EXCEPTION_TYPE_CS_INVALID_INSTRUCTION: + e = "FATAL_CS_INVALID_INSTRUCTION"; + break; + case CS_FATAL_EXCEPTION_TYPE_CS_CALL_STACK_OVERFLOW: + e = "FATAL_CS_CALL_STACK_OVERFLOW"; + break; + /* Shader exceptions */ + case CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_PC: + e = "INSTR_INVALID_PC"; + break; + case CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_ENC: + e = "INSTR_INVALID_ENC"; + break; + case CS_FAULT_EXCEPTION_TYPE_INSTR_BARRIER_FAULT: + e = "INSTR_BARRIER_FAULT"; + break; + /* Misc exceptions */ + case CS_FAULT_EXCEPTION_TYPE_DATA_INVALID_FAULT: + e = "DATA_INVALID_FAULT"; + break; + case CS_FAULT_EXCEPTION_TYPE_TILE_RANGE_FAULT: + e = "TILE_RANGE_FAULT"; + break; + case CS_FAULT_EXCEPTION_TYPE_ADDR_RANGE_FAULT: + e = "ADDR_RANGE_FAULT"; + break; + case CS_FAULT_EXCEPTION_TYPE_IMPRECISE_FAULT: + e = "IMPRECISE_FAULT"; + break; + /* FW exceptions */ + case CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR: + e = "FIRMWARE_INTERNAL_ERROR"; + break; + case CS_FAULT_EXCEPTION_TYPE_RESOURCE_EVICTION_TIMEOUT: + e = "RESOURCE_EVICTION_TIMEOUT"; + break; + /* GPU Fault */ + case GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_BUS_FAULT: + e = "GPU_BUS_FAULT"; + break; + case GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_SHAREABILITY_FAULT: + e = "GPU_SHAREABILITY_FAULT"; + break; + case GPU_FAULTSTATUS_EXCEPTION_TYPE_SYSTEM_SHAREABILITY_FAULT: + e = "SYSTEM_SHAREABILITY_FAULT"; + break; + case GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_CACHEABILITY_FAULT: + e = "GPU_CACHEABILITY_FAULT"; + break; + /* Any other exception code is unknown */ + default: + e = "UNKNOWN"; + break; + } + + return e; +} diff --git a/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_jm.c b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_jm.c new file mode 100755 index 000000000000..56f541516489 --- /dev/null +++ b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_jm.c @@ -0,0 +1,177 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include + +#include "../mali_kbase_gpu_fault.h" + +const char *kbase_gpu_exception_name(u32 const exception_code) +{ + const char *e; + + switch (exception_code) { + /* Non-Fault Status code */ + case 0x00: + e = "NOT_STARTED/IDLE/OK"; + break; + case 0x01: + e = "DONE"; + break; + case 0x02: + e = "INTERRUPTED"; + break; + case 0x03: + e = "STOPPED"; + break; + case 0x04: + e = "TERMINATED"; + break; + case 0x08: + e = "ACTIVE"; + break; + /* Job exceptions */ + case 0x40: + e = "JOB_CONFIG_FAULT"; + break; + case 0x41: + e = "JOB_POWER_FAULT"; + break; + case 0x42: + e = "JOB_READ_FAULT"; + break; + case 0x43: + e = "JOB_WRITE_FAULT"; + break; + case 0x44: + e = "JOB_AFFINITY_FAULT"; + break; + case 0x48: + e = "JOB_BUS_FAULT"; + break; + case 0x50: + e = "INSTR_INVALID_PC"; + break; + case 0x51: + e = "INSTR_INVALID_ENC"; + break; + case 0x52: + e = "INSTR_TYPE_MISMATCH"; + break; + case 0x53: + e = "INSTR_OPERAND_FAULT"; + break; + case 0x54: + e = "INSTR_TLS_FAULT"; + break; + case 0x55: + e = "INSTR_BARRIER_FAULT"; + break; + case 0x56: + e = "INSTR_ALIGN_FAULT"; + break; + case 0x58: + e = "DATA_INVALID_FAULT"; + break; + case 0x59: + e = "TILE_RANGE_FAULT"; + break; + case 0x5A: + e = "ADDR_RANGE_FAULT"; + break; + case 0x60: + e = "OUT_OF_MEMORY"; + break; + /* GPU exceptions */ + case 0x80: + e = "DELAYED_BUS_FAULT"; + break; + case 0x88: + e = "SHAREABILITY_FAULT"; + break; + /* MMU exceptions */ + case 0xC0: + case 0xC1: + case 0xC2: + case 0xC3: + case 0xC4: + case 0xC5: + case 0xC6: + case 0xC7: + e = "TRANSLATION_FAULT"; + break; + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + e = "PERMISSION_FAULT"; + break; + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD5: + case 0xD6: + case 0xD7: + e = "TRANSTAB_BUS_FAULT"; + break; + case 0xD8: + case 0xD9: + case 0xDA: + case 0xDB: + case 0xDC: + case 0xDD: + case 0xDE: + case 0xDF: + e = "ACCESS_FLAG"; + break; + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + e = "ADDRESS_SIZE_FAULT"; + break; + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + e = "MEMORY_ATTRIBUTES_FAULT"; + break; + default: + e = "UNKNOWN"; + break; + }; + + return e; +} diff --git a/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_csf.h b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_csf.h new file mode 100755 index 000000000000..ff6e4ae47184 --- /dev/null +++ b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_csf.h @@ -0,0 +1,297 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_GPU_REGMAP_CSF_H_ +#define _KBASE_GPU_REGMAP_CSF_H_ + +#if !MALI_USE_CSF +#error "Cannot be compiled with JM" +#endif + +#include "csf/mali_gpu_csf_control_registers.h" +#define GPU_CONTROL_MCU_REG(r) (GPU_CONTROL_MCU + (r)) + + +/* Set to implementation defined, outer caching */ +#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull +/* Set to write back memory, outer caching */ +#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull +/* Set to inner non-cacheable, outer-non-cacheable + * Setting defined by the alloc bits is ignored, but set to a valid encoding: + * - no-alloc on read + * - no alloc on write + */ +#define AS_MEMATTR_AARCH64_NON_CACHEABLE 0x4Cull +/* Set to shared memory, that is inner cacheable on ACE and inner or outer + * shared, otherwise inner non-cacheable. + * Outer cacheable if inner or outer shared, otherwise outer non-cacheable. + */ +#define AS_MEMATTR_AARCH64_SHARED 0x8ull + +/* Symbols for default MEMATTR to use + * Default is - HW implementation defined caching + */ +#define AS_MEMATTR_INDEX_DEFAULT 0 +#define AS_MEMATTR_INDEX_DEFAULT_ACE 3 + +/* HW implementation defined caching */ +#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0 +/* Force cache on */ +#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1 +/* Write-alloc */ +#define AS_MEMATTR_INDEX_WRITE_ALLOC 2 +/* Outer coherent, inner implementation defined policy */ +#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3 +/* Outer coherent, write alloc inner */ +#define AS_MEMATTR_INDEX_OUTER_WA 4 +/* Normal memory, inner non-cacheable, outer non-cacheable (ARMv8 mode only) */ +#define AS_MEMATTR_INDEX_NON_CACHEABLE 5 +/* Normal memory, shared between MCU and Host */ +#define AS_MEMATTR_INDEX_SHARED 6 + +/* Configuration bits for the Command Stream Frontend. */ +#define CSF_CONFIG 0xF00 + +/* CSF_CONFIG register */ +#define CSF_CONFIG_FORCE_COHERENCY_FEATURES_SHIFT 2 + +/* GPU control registers */ +#define MCU_CONTROL 0x700 +#define MCU_STATUS 0x704 + +#define MCU_CNTRL_ENABLE (1 << 0) +#define MCU_CNTRL_AUTO (1 << 1) +#define MCU_CNTRL_DISABLE (0) + +#define MCU_STATUS_HALTED (1 << 1) + +#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory + * region base address, low word + */ +#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory + * region base address, high word + */ +#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter + * configuration + */ + +#define PRFCNT_CSHW_EN 0x06C /* (RW) Performance counter + * enable for Command Stream Hardware + */ + +#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable + * flags for shader cores + */ +#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable + * flags for tiler + */ +#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable + * flags for MMU/L2 cache + */ + +/* JOB IRQ flags */ +#define JOB_IRQ_GLOBAL_IF (1 << 31) /* Global interface interrupt received */ + +/* GPU_COMMAND codes */ +#define GPU_COMMAND_CODE_NOP 0x00 /* No operation, nothing happens */ +#define GPU_COMMAND_CODE_RESET 0x01 /* Reset the GPU */ +#define GPU_COMMAND_CODE_PRFCNT 0x02 /* Clear or sample performance counters */ +#define GPU_COMMAND_CODE_TIME 0x03 /* Configure time sources */ +#define GPU_COMMAND_CODE_FLUSH_CACHES 0x04 /* Flush caches */ +#define GPU_COMMAND_CODE_SET_PROTECTED_MODE 0x05 /* Places the GPU in protected mode */ +#define GPU_COMMAND_CODE_FINISH_HALT 0x06 /* Halt CSF */ +#define GPU_COMMAND_CODE_CLEAR_FAULT 0x07 /* Clear GPU_FAULTSTATUS and GPU_FAULTADDRESS, TODX */ + +/* GPU_COMMAND_RESET payloads */ + +/* This will leave the state of active jobs UNDEFINED, but will leave the external bus in a defined and idle state. + * Power domains will remain powered on. + */ +#define GPU_COMMAND_RESET_PAYLOAD_FAST_RESET 0x00 + +/* This will leave the state of active command streams UNDEFINED, but will leave the external bus in a defined and + * idle state. + */ +#define GPU_COMMAND_RESET_PAYLOAD_SOFT_RESET 0x01 + +/* This reset will leave the state of currently active streams UNDEFINED, will likely lose data, and may leave + * the system bus in an inconsistent state. Use only as a last resort when nothing else works. + */ +#define GPU_COMMAND_RESET_PAYLOAD_HARD_RESET 0x02 + +/* GPU_COMMAND_PRFCNT payloads */ +#define GPU_COMMAND_PRFCNT_PAYLOAD_SAMPLE 0x01 /* Sample performance counters */ +#define GPU_COMMAND_PRFCNT_PAYLOAD_CLEAR 0x02 /* Clear performance counters */ + +/* GPU_COMMAND_TIME payloads */ +#define GPU_COMMAND_TIME_DISABLE 0x00 /* Disable cycle counter */ +#define GPU_COMMAND_TIME_ENABLE 0x01 /* Enable cycle counter */ + +/* GPU_COMMAND_FLUSH_CACHES payloads */ +#define GPU_COMMAND_FLUSH_PAYLOAD_NONE 0x00 /* No flush */ +#define GPU_COMMAND_FLUSH_PAYLOAD_CLEAN 0x01 /* Clean the caches */ +#define GPU_COMMAND_FLUSH_PAYLOAD_INVALIDATE 0x02 /* Invalidate the caches */ +#define GPU_COMMAND_FLUSH_PAYLOAD_CLEAN_INVALIDATE 0x03 /* Clean and invalidate the caches */ + +/* GPU_COMMAND command + payload */ +#define GPU_COMMAND_CODE_PAYLOAD(opcode, payload) \ + ((u32)opcode | ((u32)payload << 8)) + +/* Final GPU_COMMAND form */ +/* No operation, nothing happens */ +#define GPU_COMMAND_NOP \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_NOP, 0) + +/* Stop all external bus interfaces, and then reset the entire GPU. */ +#define GPU_COMMAND_SOFT_RESET \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_RESET, GPU_COMMAND_RESET_PAYLOAD_SOFT_RESET) + +/* Immediately reset the entire GPU. */ +#define GPU_COMMAND_HARD_RESET \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_RESET, GPU_COMMAND_RESET_PAYLOAD_HARD_RESET) + +/* Clear all performance counters, setting them all to zero. */ +#define GPU_COMMAND_PRFCNT_CLEAR \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_PRFCNT, GPU_COMMAND_PRFCNT_PAYLOAD_CLEAR) + +/* Sample all performance counters, writing them out to memory */ +#define GPU_COMMAND_PRFCNT_SAMPLE \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_PRFCNT, GPU_COMMAND_PRFCNT_PAYLOAD_SAMPLE) + +/* Starts the cycle counter, and system timestamp propagation */ +#define GPU_COMMAND_CYCLE_COUNT_START \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_TIME, GPU_COMMAND_TIME_ENABLE) + +/* Stops the cycle counter, and system timestamp propagation */ +#define GPU_COMMAND_CYCLE_COUNT_STOP \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_TIME, GPU_COMMAND_TIME_DISABLE) + +/* Clean all caches */ +#define GPU_COMMAND_CLEAN_CACHES \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_FLUSH_CACHES, GPU_COMMAND_FLUSH_PAYLOAD_CLEAN) + +/* Clean and invalidate all caches */ +#define GPU_COMMAND_CLEAN_INV_CACHES \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_FLUSH_CACHES, GPU_COMMAND_FLUSH_PAYLOAD_CLEAN_INVALIDATE) + +/* Places the GPU in protected mode */ +#define GPU_COMMAND_SET_PROTECTED_MODE \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_SET_PROTECTED_MODE, 0) + +/* Halt CSF */ +#define GPU_COMMAND_FINISH_HALT \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_FINISH_HALT, 0) + +/* Clear GPU faults */ +#define GPU_COMMAND_CLEAR_FAULT \ + GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_CLEAR_FAULT, 0) + +/* End Command Values */ + +/* GPU_FAULTSTATUS register */ +#define GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT 0 +#define GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK (0xFFul) +#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GET(reg_val) \ + (((reg_val)&GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) \ + >> GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) +#define GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT 8 +#define GPU_FAULTSTATUS_ACCESS_TYPE_MASK \ + (0x3ul << GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT) + +#define GPU_FAULTSTATUS_ADDR_VALID_SHIFT 10 +#define GPU_FAULTSTATUS_ADDR_VALID_FLAG \ + (1ul << GPU_FAULTSTATUS_ADDR_VALID_SHIFT) + +#define GPU_FAULTSTATUS_JASID_VALID_SHIFT 11 +#define GPU_FAULTSTATUS_JASID_VALID_FLAG \ + (1ul << GPU_FAULTSTATUS_JASID_VALID_SHIFT) + +#define GPU_FAULTSTATUS_JASID_SHIFT 12 +#define GPU_FAULTSTATUS_JASID_MASK (0xF << GPU_FAULTSTATUS_JASID_SHIFT) +#define GPU_FAULTSTATUS_JASID_GET(reg_val) \ + (((reg_val)&GPU_FAULTSTATUS_JASID_MASK) >> GPU_FAULTSTATUS_JASID_SHIFT) +#define GPU_FAULTSTATUS_JASID_SET(reg_val, value) \ + (((reg_val) & ~GPU_FAULTSTATUS_JASID_MASK) | \ + (((value) << GPU_FAULTSTATUS_JASID_SHIFT) & GPU_FAULTSTATUS_JASID_MASK)) + +#define GPU_FAULTSTATUS_SOURCE_ID_SHIFT 16 +#define GPU_FAULTSTATUS_SOURCE_ID_MASK \ + (0xFFFFul << GPU_FAULTSTATUS_SOURCE_ID_SHIFT) +/* End GPU_FAULTSTATUS register */ + +/* GPU_FAULTSTATUS_ACCESS_TYPE values */ +#define GPU_FAULTSTATUS_ACCESS_TYPE_ATOMIC 0x0 +#define GPU_FAULTSTATUS_ACCESS_TYPE_EXECUTE 0x1 +#define GPU_FAULTSTATUS_ACCESS_TYPE_READ 0x2 +#define GPU_FAULTSTATUS_ACCESS_TYPE_WRITE 0x3 +/* End of GPU_FAULTSTATUS_ACCESS_TYPE values */ + +/* TODO: Remove once 10.x.6 headers became available */ +#define GPU_EXCEPTION_TYPE_SW_FAULT_0 ((u8)0x70) +#define GPU_EXCEPTION_TYPE_SW_FAULT_1 ((u8)0x71) + +/* GPU_FAULTSTATUS_EXCEPTION_TYPE values */ +#define GPU_FAULTSTATUS_EXCEPTION_TYPE_OK 0x00 +#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_BUS_FAULT 0x80 +#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_SHAREABILITY_FAULT 0x88 +#define GPU_FAULTSTATUS_EXCEPTION_TYPE_SYSTEM_SHAREABILITY_FAULT 0x89 +#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_CACHEABILITY_FAULT 0x8A +/* End of GPU_FAULTSTATUS_EXCEPTION_TYPE values */ + +#define GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT GPU_U(10) +#define GPU_FAULTSTATUS_ADDRESS_VALID_MASK (GPU_U(0x1) << GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT) +#define GPU_FAULTSTATUS_ADDRESS_VALID_GET(reg_val) \ + (((reg_val)&GPU_FAULTSTATUS_ADDRESS_VALID_MASK) >> GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT) +#define GPU_FAULTSTATUS_ADDRESS_VALID_SET(reg_val, value) \ + (((reg_val) & ~GPU_FAULTSTATUS_ADDRESS_VALID_MASK) | \ + (((value) << GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT) & GPU_FAULTSTATUS_ADDRESS_VALID_MASK)) + +/* IRQ flags */ +#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ +#define GPU_PROTECTED_FAULT (1 << 1) /* A GPU fault has occurred in protected mode */ +#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. */ +#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ +#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down. */ +#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ +#define DOORBELL_MIRROR (1 << 18) /* Mirrors the doorbell interrupt line to the CPU */ +#define MCU_STATUS_GPU_IRQ (1 << 19) /* MCU requires attention */ + +/* + * In Debug build, + * GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE is used to clear and unmask interupts sources of GPU_IRQ + * by writing it onto GPU_IRQ_CLEAR/MASK registers. + * + * In Release build, + * GPU_IRQ_REG_COMMON is used. + * + * Note: + * CLEAN_CACHES_COMPLETED - Used separately for cache operation. + * DOORBELL_MIRROR - Do not have it included for GPU_IRQ_REG_COMMON + * as it can't be cleared by GPU_IRQ_CLEAR, thus interrupt storm might happen + */ +#define GPU_IRQ_REG_COMMON (GPU_FAULT | GPU_PROTECTED_FAULT | RESET_COMPLETED \ + | POWER_CHANGED_ALL | MCU_STATUS_GPU_IRQ) + +/* GPU_CONTROL_MCU.GPU_IRQ_RAWSTAT */ +#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when performance count sample has completed */ + +#endif /* _KBASE_GPU_REGMAP_CSF_H_ */ diff --git a/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_jm.h b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_jm.h new file mode 100755 index 000000000000..c9c2fbd49058 --- /dev/null +++ b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_jm.h @@ -0,0 +1,288 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_GPU_REGMAP_JM_H_ +#define _KBASE_GPU_REGMAP_JM_H_ + +#if MALI_USE_CSF +#error "Cannot be compiled with CSF" +#endif + +/* Set to implementation defined, outer caching */ +#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull +/* Set to write back memory, outer caching */ +#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull +/* Set to inner non-cacheable, outer-non-cacheable + * Setting defined by the alloc bits is ignored, but set to a valid encoding: + * - no-alloc on read + * - no alloc on write + */ +#define AS_MEMATTR_AARCH64_NON_CACHEABLE 0x4Cull + +/* Symbols for default MEMATTR to use + * Default is - HW implementation defined caching + */ +#define AS_MEMATTR_INDEX_DEFAULT 0 +#define AS_MEMATTR_INDEX_DEFAULT_ACE 3 + +/* HW implementation defined caching */ +#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0 +/* Force cache on */ +#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1 +/* Write-alloc */ +#define AS_MEMATTR_INDEX_WRITE_ALLOC 2 +/* Outer coherent, inner implementation defined policy */ +#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3 +/* Outer coherent, write alloc inner */ +#define AS_MEMATTR_INDEX_OUTER_WA 4 +/* Normal memory, inner non-cacheable, outer non-cacheable (ARMv8 mode only) */ +#define AS_MEMATTR_INDEX_NON_CACHEABLE 5 + +/* GPU control registers */ + +#define CORE_FEATURES 0x008 /* (RO) Shader Core Features */ +#define JS_PRESENT 0x01C /* (RO) Job slots present */ +#define LATEST_FLUSH 0x038 /* (RO) Flush ID of latest + * clean-and-invalidate operation + */ + +#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory + * region base address, low word + */ +#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory + * region base address, high word + */ +#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter + * configuration + */ +#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable + * flags for Job Manager + */ +#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable + * flags for shader cores + */ +#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable + * flags for tiler + */ +#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable + * flags for MMU/L2 cache + */ + +#define JS0_FEATURES 0x0C0 /* (RO) Features of job slot 0 */ +#define JS1_FEATURES 0x0C4 /* (RO) Features of job slot 1 */ +#define JS2_FEATURES 0x0C8 /* (RO) Features of job slot 2 */ +#define JS3_FEATURES 0x0CC /* (RO) Features of job slot 3 */ +#define JS4_FEATURES 0x0D0 /* (RO) Features of job slot 4 */ +#define JS5_FEATURES 0x0D4 /* (RO) Features of job slot 5 */ +#define JS6_FEATURES 0x0D8 /* (RO) Features of job slot 6 */ +#define JS7_FEATURES 0x0DC /* (RO) Features of job slot 7 */ +#define JS8_FEATURES 0x0E0 /* (RO) Features of job slot 8 */ +#define JS9_FEATURES 0x0E4 /* (RO) Features of job slot 9 */ +#define JS10_FEATURES 0x0E8 /* (RO) Features of job slot 10 */ +#define JS11_FEATURES 0x0EC /* (RO) Features of job slot 11 */ +#define JS12_FEATURES 0x0F0 /* (RO) Features of job slot 12 */ +#define JS13_FEATURES 0x0F4 /* (RO) Features of job slot 13 */ +#define JS14_FEATURES 0x0F8 /* (RO) Features of job slot 14 */ +#define JS15_FEATURES 0x0FC /* (RO) Features of job slot 15 */ + +#define JS_FEATURES_REG(n) GPU_CONTROL_REG(JS0_FEATURES + ((n) << 2)) + +#define JM_CONFIG 0xF00 /* (RW) Job manager configuration (implementation-specific) */ + +/* Job control registers */ + +#define JOB_IRQ_JS_STATE 0x010 /* status==active and _next == busy snapshot from last JOB_IRQ_CLEAR */ +#define JOB_IRQ_THROTTLE 0x014 /* cycles to delay delivering an interrupt externally. The JOB_IRQ_STATUS is NOT affected by this, just the delivery of the interrupt. */ + +#define JOB_SLOT0 0x800 /* Configuration registers for job slot 0 */ +#define JOB_SLOT1 0x880 /* Configuration registers for job slot 1 */ +#define JOB_SLOT2 0x900 /* Configuration registers for job slot 2 */ +#define JOB_SLOT3 0x980 /* Configuration registers for job slot 3 */ +#define JOB_SLOT4 0xA00 /* Configuration registers for job slot 4 */ +#define JOB_SLOT5 0xA80 /* Configuration registers for job slot 5 */ +#define JOB_SLOT6 0xB00 /* Configuration registers for job slot 6 */ +#define JOB_SLOT7 0xB80 /* Configuration registers for job slot 7 */ +#define JOB_SLOT8 0xC00 /* Configuration registers for job slot 8 */ +#define JOB_SLOT9 0xC80 /* Configuration registers for job slot 9 */ +#define JOB_SLOT10 0xD00 /* Configuration registers for job slot 10 */ +#define JOB_SLOT11 0xD80 /* Configuration registers for job slot 11 */ +#define JOB_SLOT12 0xE00 /* Configuration registers for job slot 12 */ +#define JOB_SLOT13 0xE80 /* Configuration registers for job slot 13 */ +#define JOB_SLOT14 0xF00 /* Configuration registers for job slot 14 */ +#define JOB_SLOT15 0xF80 /* Configuration registers for job slot 15 */ + +#define JOB_SLOT_REG(n, r) (JOB_CONTROL_REG(JOB_SLOT0 + ((n) << 7)) + (r)) + +#define JS_HEAD_LO 0x00 /* (RO) Job queue head pointer for job slot n, low word */ +#define JS_HEAD_HI 0x04 /* (RO) Job queue head pointer for job slot n, high word */ +#define JS_TAIL_LO 0x08 /* (RO) Job queue tail pointer for job slot n, low word */ +#define JS_TAIL_HI 0x0C /* (RO) Job queue tail pointer for job slot n, high word */ +#define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ +#define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ +#define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ +#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job + slot n */ + +#define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ +#define JS_STATUS 0x24 /* (RO) Status register for job slot n */ + +#define JS_HEAD_NEXT_LO 0x40 /* (RW) Next job queue head pointer for job slot n, low word */ +#define JS_HEAD_NEXT_HI 0x44 /* (RW) Next job queue head pointer for job slot n, high word */ + +#define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ +#define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ +#define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ +#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for + job slot n */ + +#define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ + +#define JS_FLUSH_ID_NEXT 0x70 /* (RW) Next job slot n cache flush ID */ + +/* No JM-specific MMU control registers */ +/* No JM-specific MMU address space control registers */ + +/* JS_COMMAND register commands */ +#define JS_COMMAND_NOP 0x00 /* NOP Operation. Writing this value is ignored */ +#define JS_COMMAND_START 0x01 /* Start processing a job chain. Writing this value is ignored */ +#define JS_COMMAND_SOFT_STOP 0x02 /* Gently stop processing a job chain */ +#define JS_COMMAND_HARD_STOP 0x03 /* Rudely stop processing a job chain */ +#define JS_COMMAND_SOFT_STOP_0 0x04 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */ +#define JS_COMMAND_HARD_STOP_0 0x05 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */ +#define JS_COMMAND_SOFT_STOP_1 0x06 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */ +#define JS_COMMAND_HARD_STOP_1 0x07 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */ + +#define JS_COMMAND_MASK 0x07 /* Mask of bits currently in use by the HW */ + +/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */ +#define JS_CONFIG_START_FLUSH_NO_ACTION (0u << 0) +#define JS_CONFIG_START_FLUSH_CLEAN (1u << 8) +#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE (3u << 8) +#define JS_CONFIG_START_MMU (1u << 10) +#define JS_CONFIG_JOB_CHAIN_FLAG (1u << 11) +#define JS_CONFIG_END_FLUSH_NO_ACTION JS_CONFIG_START_FLUSH_NO_ACTION +#define JS_CONFIG_END_FLUSH_CLEAN (1u << 12) +#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE (3u << 12) +#define JS_CONFIG_ENABLE_FLUSH_REDUCTION (1u << 14) +#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK (1u << 15) +#define JS_CONFIG_THREAD_PRI(n) ((n) << 16) + +/* JS_XAFFINITY register values */ +#define JS_XAFFINITY_XAFFINITY_ENABLE (1u << 0) +#define JS_XAFFINITY_TILER_ENABLE (1u << 8) +#define JS_XAFFINITY_CACHE_ENABLE (1u << 16) + +/* JS_STATUS register values */ + +/* NOTE: Please keep this values in sync with enum base_jd_event_code in mali_base_kernel.h. + * The values are separated to avoid dependency of userspace and kernel code. + */ + +/* Group of values representing the job status instead of a particular fault */ +#define JS_STATUS_NO_EXCEPTION_BASE 0x00 +#define JS_STATUS_INTERRUPTED (JS_STATUS_NO_EXCEPTION_BASE + 0x02) /* 0x02 means INTERRUPTED */ +#define JS_STATUS_STOPPED (JS_STATUS_NO_EXCEPTION_BASE + 0x03) /* 0x03 means STOPPED */ +#define JS_STATUS_TERMINATED (JS_STATUS_NO_EXCEPTION_BASE + 0x04) /* 0x04 means TERMINATED */ + +/* General fault values */ +#define JS_STATUS_FAULT_BASE 0x40 +#define JS_STATUS_CONFIG_FAULT (JS_STATUS_FAULT_BASE) /* 0x40 means CONFIG FAULT */ +#define JS_STATUS_POWER_FAULT (JS_STATUS_FAULT_BASE + 0x01) /* 0x41 means POWER FAULT */ +#define JS_STATUS_READ_FAULT (JS_STATUS_FAULT_BASE + 0x02) /* 0x42 means READ FAULT */ +#define JS_STATUS_WRITE_FAULT (JS_STATUS_FAULT_BASE + 0x03) /* 0x43 means WRITE FAULT */ +#define JS_STATUS_AFFINITY_FAULT (JS_STATUS_FAULT_BASE + 0x04) /* 0x44 means AFFINITY FAULT */ +#define JS_STATUS_BUS_FAULT (JS_STATUS_FAULT_BASE + 0x08) /* 0x48 means BUS FAULT */ + +/* Instruction or data faults */ +#define JS_STATUS_INSTRUCTION_FAULT_BASE 0x50 +#define JS_STATUS_INSTR_INVALID_PC (JS_STATUS_INSTRUCTION_FAULT_BASE) /* 0x50 means INSTR INVALID PC */ +#define JS_STATUS_INSTR_INVALID_ENC (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x01) /* 0x51 means INSTR INVALID ENC */ +#define JS_STATUS_INSTR_TYPE_MISMATCH (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x02) /* 0x52 means INSTR TYPE MISMATCH */ +#define JS_STATUS_INSTR_OPERAND_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x03) /* 0x53 means INSTR OPERAND FAULT */ +#define JS_STATUS_INSTR_TLS_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x04) /* 0x54 means INSTR TLS FAULT */ +#define JS_STATUS_INSTR_BARRIER_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x05) /* 0x55 means INSTR BARRIER FAULT */ +#define JS_STATUS_INSTR_ALIGN_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x06) /* 0x56 means INSTR ALIGN FAULT */ +/* NOTE: No fault with 0x57 code defined in spec. */ +#define JS_STATUS_DATA_INVALID_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x08) /* 0x58 means DATA INVALID FAULT */ +#define JS_STATUS_TILE_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x09) /* 0x59 means TILE RANGE FAULT */ +#define JS_STATUS_ADDRESS_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x0A) /* 0x5A means ADDRESS RANGE FAULT */ + +/* Other faults */ +#define JS_STATUS_MEMORY_FAULT_BASE 0x60 +#define JS_STATUS_OUT_OF_MEMORY (JS_STATUS_MEMORY_FAULT_BASE) /* 0x60 means OUT OF MEMORY */ +#define JS_STATUS_UNKNOWN 0x7F /* 0x7F means UNKNOWN */ + +/* JS_FEATURES register */ +#define JS_FEATURE_NULL_JOB (1u << 1) +#define JS_FEATURE_SET_VALUE_JOB (1u << 2) +#define JS_FEATURE_CACHE_FLUSH_JOB (1u << 3) +#define JS_FEATURE_COMPUTE_JOB (1u << 4) +#define JS_FEATURE_VERTEX_JOB (1u << 5) +#define JS_FEATURE_GEOMETRY_JOB (1u << 6) +#define JS_FEATURE_TILER_JOB (1u << 7) +#define JS_FEATURE_FUSED_JOB (1u << 8) +#define JS_FEATURE_FRAGMENT_JOB (1u << 9) + +/* JM_CONFIG register */ +#define JM_TIMESTAMP_OVERRIDE (1ul << 0) +#define JM_CLOCK_GATE_OVERRIDE (1ul << 1) +#define JM_JOB_THROTTLE_ENABLE (1ul << 2) +#define JM_JOB_THROTTLE_LIMIT_SHIFT (3) +#define JM_MAX_JOB_THROTTLE_LIMIT (0x3F) +#define JM_FORCE_COHERENCY_FEATURES_SHIFT (2) + +/* GPU_COMMAND values */ +#define GPU_COMMAND_NOP 0x00 /* No operation, nothing happens */ +#define GPU_COMMAND_SOFT_RESET 0x01 /* Stop all external bus interfaces, and then reset the entire GPU. */ +#define GPU_COMMAND_HARD_RESET 0x02 /* Immediately reset the entire GPU. */ +#define GPU_COMMAND_PRFCNT_CLEAR 0x03 /* Clear all performance counters, setting them all to zero. */ +#define GPU_COMMAND_PRFCNT_SAMPLE 0x04 /* Sample all performance counters, writing them out to memory */ +#define GPU_COMMAND_CYCLE_COUNT_START 0x05 /* Starts the cycle counter, and system timestamp propagation */ +#define GPU_COMMAND_CYCLE_COUNT_STOP 0x06 /* Stops the cycle counter, and system timestamp propagation */ +#define GPU_COMMAND_CLEAN_CACHES 0x07 /* Clean all caches */ +#define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */ +#define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */ + +/* IRQ flags */ +#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ +#define MULTIPLE_GPU_FAULTS (1 << 7) /* More than one GPU Fault occurred. */ +#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. */ +#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ +#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down. */ +#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ +#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ + +/* + * In Debug build, + * GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE is used to clear and enable interupts sources of GPU_IRQ + * by writing it onto GPU_IRQ_CLEAR/MASK registers. + * + * In Release build, + * GPU_IRQ_REG_COMMON is used. + * + * Note: + * CLEAN_CACHES_COMPLETED - Used separately for cache operation. + */ +#define GPU_IRQ_REG_COMMON (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ + | POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED) + +#endif /* _KBASE_GPU_REGMAP_JM_H_ */ diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.c b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.c new file mode 100755 index 000000000000..3128db4cabfc --- /dev/null +++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.c @@ -0,0 +1,41 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include + +const char *kbase_gpu_access_type_name(u32 fault_status) +{ + switch (AS_FAULTSTATUS_ACCESS_TYPE_GET(fault_status)) { + case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: + return "ATOMIC"; + case AS_FAULTSTATUS_ACCESS_TYPE_READ: + return "READ"; + case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: + return "WRITE"; + case AS_FAULTSTATUS_ACCESS_TYPE_EX: + return "EXECUTE"; + default: + WARN_ON(1); + return NULL; + } +} diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.h b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.h new file mode 100755 index 000000000000..9516e56eda01 --- /dev/null +++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.h @@ -0,0 +1,31 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_GPU_H_ +#define _KBASE_GPU_H_ + +#include "mali_kbase_gpu_regmap.h" +#include "mali_kbase_gpu_fault.h" +#include "mali_kbase_gpu_coherency.h" +#include "mali_kbase_gpu_id.h" + +#endif /* _KBASE_GPU_H_ */ diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h new file mode 100755 index 000000000000..bb2b1613aa47 --- /dev/null +++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h @@ -0,0 +1,31 @@ +/* + * + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_GPU_COHERENCY_H_ +#define _KBASE_GPU_COHERENCY_H_ + +#define COHERENCY_ACE_LITE 0 +#define COHERENCY_ACE 1 +#define COHERENCY_NONE 31 +#define COHERENCY_FEATURE_BIT(x) (1 << (x)) + +#endif /* _KBASE_GPU_COHERENCY_H_ */ diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_fault.h b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_fault.h new file mode 100755 index 000000000000..e63c3881a3ca --- /dev/null +++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_fault.h @@ -0,0 +1,48 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_GPU_FAULT_H_ +#define _KBASE_GPU_FAULT_H_ + +/** Returns the name associated with a Mali exception code + * + * @exception_code: exception code + * + * This function is called from the interrupt handler when a GPU fault occurs. + * + * Return: name associated with the exception code + */ +const char *kbase_gpu_exception_name(u32 exception_code); + +/** + * kbase_gpu_access_type_name - Convert MMU_AS_CONTROL.FAULTSTATUS.ACCESS_TYPE + * into string. + * @fault_status: value of FAULTSTATUS register. + * + * After MMU fault, this function can be used to get readable information about + * access_type of the MMU fault. + * + * Return: String of the access type. + */ +const char *kbase_gpu_access_type_name(u32 fault_status); + +#endif /* _KBASE_GPU_FAULT_H_ */ diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h new file mode 100755 index 000000000000..31d55264c67f --- /dev/null +++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h @@ -0,0 +1,119 @@ +/* + * + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_GPU_ID_H_ +#define _KBASE_GPU_ID_H_ + +/* GPU_ID register */ +#define GPU_ID_VERSION_STATUS_SHIFT 0 +#define GPU_ID_VERSION_MINOR_SHIFT 4 +#define GPU_ID_VERSION_MAJOR_SHIFT 12 +#define GPU_ID_VERSION_PRODUCT_ID_SHIFT 16 +#define GPU_ID_VERSION_STATUS (0xFu << GPU_ID_VERSION_STATUS_SHIFT) +#define GPU_ID_VERSION_MINOR (0xFFu << GPU_ID_VERSION_MINOR_SHIFT) +#define GPU_ID_VERSION_MAJOR (0xFu << GPU_ID_VERSION_MAJOR_SHIFT) +#define GPU_ID_VERSION_PRODUCT_ID (0xFFFFu << GPU_ID_VERSION_PRODUCT_ID_SHIFT) + +#define GPU_ID2_VERSION_STATUS_SHIFT 0 +#define GPU_ID2_VERSION_MINOR_SHIFT 4 +#define GPU_ID2_VERSION_MAJOR_SHIFT 12 +#define GPU_ID2_PRODUCT_MAJOR_SHIFT 16 +#define GPU_ID2_ARCH_REV_SHIFT 20 +#define GPU_ID2_ARCH_MINOR_SHIFT 24 +#define GPU_ID2_ARCH_MAJOR_SHIFT 28 +#define GPU_ID2_VERSION_STATUS (0xFu << GPU_ID2_VERSION_STATUS_SHIFT) +#define GPU_ID2_VERSION_MINOR (0xFFu << GPU_ID2_VERSION_MINOR_SHIFT) +#define GPU_ID2_VERSION_MAJOR (0xFu << GPU_ID2_VERSION_MAJOR_SHIFT) +#define GPU_ID2_PRODUCT_MAJOR (0xFu << GPU_ID2_PRODUCT_MAJOR_SHIFT) +#define GPU_ID2_ARCH_REV (0xFu << GPU_ID2_ARCH_REV_SHIFT) +#define GPU_ID2_ARCH_MINOR (0xFu << GPU_ID2_ARCH_MINOR_SHIFT) +#define GPU_ID2_ARCH_MAJOR (0xFu << GPU_ID2_ARCH_MAJOR_SHIFT) +#define GPU_ID2_PRODUCT_MODEL (GPU_ID2_ARCH_MAJOR | GPU_ID2_PRODUCT_MAJOR) +#define GPU_ID2_VERSION (GPU_ID2_VERSION_MAJOR | \ + GPU_ID2_VERSION_MINOR | \ + GPU_ID2_VERSION_STATUS) + +/* Helper macro to create a partial GPU_ID (new format) that defines + a product ignoring its version. */ +#define GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, product_major) \ + ((((u32)arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ + (((u32)arch_minor) << GPU_ID2_ARCH_MINOR_SHIFT) | \ + (((u32)arch_rev) << GPU_ID2_ARCH_REV_SHIFT) | \ + (((u32)product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) + +/* Helper macro to create a partial GPU_ID (new format) that specifies the + revision (major, minor, status) of a product */ +#define GPU_ID2_VERSION_MAKE(version_major, version_minor, version_status) \ + ((((u32)version_major) << GPU_ID2_VERSION_MAJOR_SHIFT) | \ + (((u32)version_minor) << GPU_ID2_VERSION_MINOR_SHIFT) | \ + (((u32)version_status) << GPU_ID2_VERSION_STATUS_SHIFT)) + +/* Helper macro to create a complete GPU_ID (new format) */ +#define GPU_ID2_MAKE(arch_major, arch_minor, arch_rev, product_major, \ + version_major, version_minor, version_status) \ + (GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, \ + product_major) | \ + GPU_ID2_VERSION_MAKE(version_major, version_minor, \ + version_status)) + +/* Helper macro to create a partial GPU_ID (new format) that identifies + a particular GPU model by its arch_major and product_major. */ +#define GPU_ID2_MODEL_MAKE(arch_major, product_major) \ + ((((u32)arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ + (((u32)product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) + +/* Strip off the non-relevant bits from a product_id value and make it suitable + for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU + model. */ +#define GPU_ID2_MODEL_MATCH_VALUE(product_id) \ + ((((u32)product_id) << GPU_ID2_PRODUCT_MAJOR_SHIFT) & \ + GPU_ID2_PRODUCT_MODEL) + +#define GPU_ID2_PRODUCT_TMIX GPU_ID2_MODEL_MAKE(6, 0) +#define GPU_ID2_PRODUCT_THEX GPU_ID2_MODEL_MAKE(6, 1) +#define GPU_ID2_PRODUCT_TSIX GPU_ID2_MODEL_MAKE(7, 0) +#define GPU_ID2_PRODUCT_TDVX GPU_ID2_MODEL_MAKE(7, 3) +#define GPU_ID2_PRODUCT_TNOX GPU_ID2_MODEL_MAKE(7, 1) +#define GPU_ID2_PRODUCT_TGOX GPU_ID2_MODEL_MAKE(7, 2) +#define GPU_ID2_PRODUCT_TTRX GPU_ID2_MODEL_MAKE(9, 0) +#define GPU_ID2_PRODUCT_TNAX GPU_ID2_MODEL_MAKE(9, 1) +#define GPU_ID2_PRODUCT_TBEX GPU_ID2_MODEL_MAKE(9, 2) +#define GPU_ID2_PRODUCT_LBEX GPU_ID2_MODEL_MAKE(9, 4) +#define GPU_ID2_PRODUCT_TBAX GPU_ID2_MODEL_MAKE(9, 5) +#define GPU_ID2_PRODUCT_TDUX GPU_ID2_MODEL_MAKE(10, 1) +#define GPU_ID2_PRODUCT_TODX GPU_ID2_MODEL_MAKE(10, 2) +#define GPU_ID2_PRODUCT_TGRX GPU_ID2_MODEL_MAKE(10, 3) +#define GPU_ID2_PRODUCT_TVAX GPU_ID2_MODEL_MAKE(10, 4) +#define GPU_ID2_PRODUCT_LODX GPU_ID2_MODEL_MAKE(10, 7) +#define GPU_ID2_PRODUCT_TTUX GPU_ID2_MODEL_MAKE(11, 2) +#define GPU_ID2_PRODUCT_LTUX GPU_ID2_MODEL_MAKE(11, 3) +#define GPU_ID2_PRODUCT_TE2X GPU_ID2_MODEL_MAKE(11, 1) + +/* Helper macro to create a GPU_ID assuming valid values for id, major, + minor, status */ +#define GPU_ID_MAKE(id, major, minor, status) \ + ((((u32)id) << GPU_ID_VERSION_PRODUCT_ID_SHIFT) | \ + (((u32)major) << GPU_ID_VERSION_MAJOR_SHIFT) | \ + (((u32)minor) << GPU_ID_VERSION_MINOR_SHIFT) | \ + (((u32)status) << GPU_ID_VERSION_STATUS_SHIFT)) + +#endif /* _KBASE_GPU_ID_H_ */ diff --git a/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_regmap.h b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_regmap.h new file mode 100755 index 000000000000..d8066f43768b --- /dev/null +++ b/drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_regmap.h @@ -0,0 +1,428 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_GPU_REGMAP_H_ +#define _KBASE_GPU_REGMAP_H_ + +#include "mali_kbase_gpu_coherency.h" +#include "mali_kbase_gpu_id.h" +#if MALI_USE_CSF +#include "backend/mali_kbase_gpu_regmap_csf.h" +#else +#include "backend/mali_kbase_gpu_regmap_jm.h" +#endif + +/* Begin Register Offsets */ +/* GPU control registers */ + +#define GPU_CONTROL_BASE 0x0000 +#define GPU_CONTROL_REG(r) (GPU_CONTROL_BASE + (r)) +#define GPU_ID 0x000 /* (RO) GPU and revision identifier */ +#define L2_FEATURES 0x004 /* (RO) Level 2 cache features */ +#define TILER_FEATURES 0x00C /* (RO) Tiler Features */ +#define MEM_FEATURES 0x010 /* (RO) Memory system features */ +#define MMU_FEATURES 0x014 /* (RO) MMU features */ +#define AS_PRESENT 0x018 /* (RO) Address space slots present */ +#define GPU_IRQ_RAWSTAT 0x020 /* (RW) */ +#define GPU_IRQ_CLEAR 0x024 /* (WO) */ +#define GPU_IRQ_MASK 0x028 /* (RW) */ +#define GPU_IRQ_STATUS 0x02C /* (RO) */ + +#define GPU_COMMAND 0x030 /* (WO) */ +#define GPU_STATUS 0x034 /* (RO) */ + +#define GPU_DBGEN (1 << 8) /* DBGEN wire status */ + +#define GPU_FAULTSTATUS 0x03C /* (RO) GPU exception type and fault status */ +#define GPU_FAULTADDRESS_LO 0x040 /* (RO) GPU exception fault address, low word */ +#define GPU_FAULTADDRESS_HI 0x044 /* (RO) GPU exception fault address, high word */ + +#define L2_CONFIG 0x048 /* (RW) Level 2 cache configuration */ + +#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */ +#define SUPER_L2_COHERENT (1 << 1) /* Shader cores within a core + * supergroup are l2 coherent + */ + +#define PWR_KEY 0x050 /* (WO) Power manager key register */ +#define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */ +#define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */ + +#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */ +#define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */ +#define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */ +#define TIMESTAMP_HI 0x09C /* (RO) Global time stamp counter, high word */ + +#define THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */ +#define THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */ +#define THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */ +#define THREAD_FEATURES 0x0AC /* (RO) Thread features */ +#define THREAD_TLS_ALLOC 0x310 /* (RO) Number of threads per core that TLS must be allocated for */ + +#define TEXTURE_FEATURES_0 0x0B0 /* (RO) Support flags for indexed texture formats 0..31 */ +#define TEXTURE_FEATURES_1 0x0B4 /* (RO) Support flags for indexed texture formats 32..63 */ +#define TEXTURE_FEATURES_2 0x0B8 /* (RO) Support flags for indexed texture formats 64..95 */ +#define TEXTURE_FEATURES_3 0x0BC /* (RO) Support flags for texture order */ + +#define TEXTURE_FEATURES_REG(n) GPU_CONTROL_REG(TEXTURE_FEATURES_0 + ((n) << 2)) + +#define SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */ +#define SHADER_PRESENT_HI 0x104 /* (RO) Shader core present bitmap, high word */ + +#define TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */ +#define TILER_PRESENT_HI 0x114 /* (RO) Tiler core present bitmap, high word */ + +#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */ +#define L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */ + +#define STACK_PRESENT_LO 0xE00 /* (RO) Core stack present bitmap, low word */ +#define STACK_PRESENT_HI 0xE04 /* (RO) Core stack present bitmap, high word */ + +#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */ +#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */ + +#define TILER_READY_LO 0x150 /* (RO) Tiler core ready bitmap, low word */ +#define TILER_READY_HI 0x154 /* (RO) Tiler core ready bitmap, high word */ + +#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */ +#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */ + +#define STACK_READY_LO 0xE10 /* (RO) Core stack ready bitmap, low word */ +#define STACK_READY_HI 0xE14 /* (RO) Core stack ready bitmap, high word */ + +#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */ +#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */ + +#define TILER_PWRON_LO 0x190 /* (WO) Tiler core power on bitmap, low word */ +#define TILER_PWRON_HI 0x194 /* (WO) Tiler core power on bitmap, high word */ + +#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */ +#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */ + +#define STACK_PWRON_LO 0xE20 /* (RO) Core stack power on bitmap, low word */ +#define STACK_PWRON_HI 0xE24 /* (RO) Core stack power on bitmap, high word */ + +#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */ +#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */ + +#define TILER_PWROFF_LO 0x1D0 /* (WO) Tiler core power off bitmap, low word */ +#define TILER_PWROFF_HI 0x1D4 /* (WO) Tiler core power off bitmap, high word */ + +#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */ +#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */ + +#define STACK_PWROFF_LO 0xE30 /* (RO) Core stack power off bitmap, low word */ +#define STACK_PWROFF_HI 0xE34 /* (RO) Core stack power off bitmap, high word */ + +#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */ +#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */ + +#define TILER_PWRTRANS_LO 0x210 /* (RO) Tiler core power transition bitmap, low word */ +#define TILER_PWRTRANS_HI 0x214 /* (RO) Tiler core power transition bitmap, high word */ + +#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */ +#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */ + +#define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */ +#define STACK_PWRTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */ + +#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */ +#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */ + +#define TILER_PWRACTIVE_LO 0x250 /* (RO) Tiler core active bitmap, low word */ +#define TILER_PWRACTIVE_HI 0x254 /* (RO) Tiler core active bitmap, high word */ + +#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */ +#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */ + +#define COHERENCY_FEATURES 0x300 /* (RO) Coherency features present */ +#define COHERENCY_ENABLE 0x304 /* (RW) Coherency enable */ + +#define SHADER_CONFIG 0xF04 /* (RW) Shader core configuration (implementation-specific) */ +#define TILER_CONFIG 0xF08 /* (RW) Tiler core configuration (implementation-specific) */ +#define L2_MMU_CONFIG 0xF0C /* (RW) L2 cache and MMU configuration (implementation-specific) */ + +/* Job control registers */ + +#define JOB_CONTROL_BASE 0x1000 + +#define JOB_CONTROL_REG(r) (JOB_CONTROL_BASE + (r)) + +#define JOB_IRQ_RAWSTAT 0x000 /* Raw interrupt status register */ +#define JOB_IRQ_CLEAR 0x004 /* Interrupt clear register */ +#define JOB_IRQ_MASK 0x008 /* Interrupt mask register */ +#define JOB_IRQ_STATUS 0x00C /* Interrupt status register */ + +/* MMU control registers */ + +#define MEMORY_MANAGEMENT_BASE 0x2000 +#define MMU_REG(r) (MEMORY_MANAGEMENT_BASE + (r)) + +#define MMU_IRQ_RAWSTAT 0x000 /* (RW) Raw interrupt status register */ +#define MMU_IRQ_CLEAR 0x004 /* (WO) Interrupt clear register */ +#define MMU_IRQ_MASK 0x008 /* (RW) Interrupt mask register */ +#define MMU_IRQ_STATUS 0x00C /* (RO) Interrupt status register */ + +#define MMU_AS0 0x400 /* Configuration registers for address space 0 */ +#define MMU_AS1 0x440 /* Configuration registers for address space 1 */ +#define MMU_AS2 0x480 /* Configuration registers for address space 2 */ +#define MMU_AS3 0x4C0 /* Configuration registers for address space 3 */ +#define MMU_AS4 0x500 /* Configuration registers for address space 4 */ +#define MMU_AS5 0x540 /* Configuration registers for address space 5 */ +#define MMU_AS6 0x580 /* Configuration registers for address space 6 */ +#define MMU_AS7 0x5C0 /* Configuration registers for address space 7 */ +#define MMU_AS8 0x600 /* Configuration registers for address space 8 */ +#define MMU_AS9 0x640 /* Configuration registers for address space 9 */ +#define MMU_AS10 0x680 /* Configuration registers for address space 10 */ +#define MMU_AS11 0x6C0 /* Configuration registers for address space 11 */ +#define MMU_AS12 0x700 /* Configuration registers for address space 12 */ +#define MMU_AS13 0x740 /* Configuration registers for address space 13 */ +#define MMU_AS14 0x780 /* Configuration registers for address space 14 */ +#define MMU_AS15 0x7C0 /* Configuration registers for address space 15 */ + +/* MMU address space control registers */ + +#define MMU_AS_REG(n, r) (MMU_REG(MMU_AS0 + ((n) << 6)) + (r)) + +#define AS_TRANSTAB_LO 0x00 /* (RW) Translation Table Base Address for address space n, low word */ +#define AS_TRANSTAB_HI 0x04 /* (RW) Translation Table Base Address for address space n, high word */ +#define AS_MEMATTR_LO 0x08 /* (RW) Memory attributes for address space n, low word. */ +#define AS_MEMATTR_HI 0x0C /* (RW) Memory attributes for address space n, high word. */ +#define AS_LOCKADDR_LO 0x10 /* (RW) Lock region address for address space n, low word */ +#define AS_LOCKADDR_HI 0x14 /* (RW) Lock region address for address space n, high word */ +#define AS_COMMAND 0x18 /* (WO) MMU command register for address space n */ +#define AS_FAULTSTATUS 0x1C /* (RO) MMU fault status register for address space n */ +#define AS_FAULTADDRESS_LO 0x20 /* (RO) Fault Address for address space n, low word */ +#define AS_FAULTADDRESS_HI 0x24 /* (RO) Fault Address for address space n, high word */ +#define AS_STATUS 0x28 /* (RO) Status flags for address space n */ + +/* (RW) Translation table configuration for address space n, low word */ +#define AS_TRANSCFG_LO 0x30 +/* (RW) Translation table configuration for address space n, high word */ +#define AS_TRANSCFG_HI 0x34 +/* (RO) Secondary fault address for address space n, low word */ +#define AS_FAULTEXTRA_LO 0x38 +/* (RO) Secondary fault address for address space n, high word */ +#define AS_FAULTEXTRA_HI 0x3C + +/* End Register Offsets */ + +/* Include POWER_CHANGED_SINGLE in debug builds for use in irq latency test. */ +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE) +#else /* CONFIG_MALI_BIFROST_DEBUG */ +#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON) +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + +/* + * MMU_IRQ_RAWSTAT register values. Values are valid also for + * MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers. + */ + +#define MMU_PAGE_FAULT_FLAGS 16 + +/* Macros returning a bitmask to retrieve page fault or bus error flags from + * MMU registers */ +#define MMU_PAGE_FAULT(n) (1UL << (n)) +#define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS)) + +/* + * Begin LPAE MMU TRANSTAB register values + */ +#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK 0xfffff000 +#define AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED (0u << 0) +#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY (1u << 1) +#define AS_TRANSTAB_LPAE_ADRMODE_TABLE (3u << 0) +#define AS_TRANSTAB_LPAE_READ_INNER (1u << 2) +#define AS_TRANSTAB_LPAE_SHARE_OUTER (1u << 4) + +#define AS_TRANSTAB_LPAE_ADRMODE_MASK 0x00000003 + +/* + * Begin AARCH64 MMU TRANSTAB register values + */ +#define MMU_HW_OUTA_BITS 40 +#define AS_TRANSTAB_BASE_MASK ((1ULL << MMU_HW_OUTA_BITS) - (1ULL << 4)) + +/* + * Begin MMU STATUS register values + */ +#define AS_STATUS_AS_ACTIVE 0x01 + +#define AS_FAULTSTATUS_EXCEPTION_CODE_MASK (0x7<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT (0x0<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT (0x1<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT (0x2<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG (0x3<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT (0x4<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT (0x5<<3) + +#define AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT 0 +#define AS_FAULTSTATUS_EXCEPTION_TYPE_MASK (0xFF << AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) +#define AS_FAULTSTATUS_EXCEPTION_TYPE_GET(reg_val) \ + (((reg_val)&AS_FAULTSTATUS_EXCEPTION_TYPE_MASK) >> AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) +#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_0 0xC0 + +#define AS_FAULTSTATUS_ACCESS_TYPE_SHIFT 8 +#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3 << AS_FAULTSTATUS_ACCESS_TYPE_SHIFT) +#define AS_FAULTSTATUS_ACCESS_TYPE_GET(reg_val) \ + (((reg_val)&AS_FAULTSTATUS_ACCESS_TYPE_MASK) >> AS_FAULTSTATUS_ACCESS_TYPE_SHIFT) + +#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0) +#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1) +#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2) +#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3) + +#define AS_FAULTSTATUS_SOURCE_ID_SHIFT 16 +#define AS_FAULTSTATUS_SOURCE_ID_MASK (0xFFFF << AS_FAULTSTATUS_SOURCE_ID_SHIFT) +#define AS_FAULTSTATUS_SOURCE_ID_GET(reg_val) \ + (((reg_val)&AS_FAULTSTATUS_SOURCE_ID_MASK) >> AS_FAULTSTATUS_SOURCE_ID_SHIFT) + +/* + * Begin MMU TRANSCFG register values + */ +#define AS_TRANSCFG_ADRMODE_LEGACY 0 +#define AS_TRANSCFG_ADRMODE_UNMAPPED 1 +#define AS_TRANSCFG_ADRMODE_IDENTITY 2 +#define AS_TRANSCFG_ADRMODE_AARCH64_4K 6 +#define AS_TRANSCFG_ADRMODE_AARCH64_64K 8 + +#define AS_TRANSCFG_ADRMODE_MASK 0xF + +/* + * Begin TRANSCFG register values + */ +#define AS_TRANSCFG_PTW_MEMATTR_MASK (3ull << 24) +#define AS_TRANSCFG_PTW_MEMATTR_NON_CACHEABLE (1ull << 24) +#define AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK (2ull << 24) + +#define AS_TRANSCFG_PTW_SH_MASK ((3ull << 28)) +#define AS_TRANSCFG_PTW_SH_OS (2ull << 28) +#define AS_TRANSCFG_PTW_SH_IS (3ull << 28) +#define AS_TRANSCFG_R_ALLOCATE (1ull << 30) + +/* + * Begin Command Values + */ + +/* AS_COMMAND register commands */ +#define AS_COMMAND_NOP 0x00 /* NOP Operation */ +#define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */ +#define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */ +#define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */ +#define AS_COMMAND_FLUSH 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs + (deprecated - only for use with T60x) */ +#define AS_COMMAND_FLUSH_PT 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs */ +#define AS_COMMAND_FLUSH_MEM 0x05 /* Wait for memory accesses to complete, flush all the L1s cache then + flush all L2 caches then issue a flush region command to all MMUs */ + +/* GPU_STATUS values */ +#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ +#define GPU_STATUS_PROTECTED_MODE_ACTIVE (1 << 7) /* Set if protected mode is active */ + +/* PRFCNT_CONFIG register values */ +#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ +#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ +#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ + +/* The performance counters are disabled. */ +#define PRFCNT_CONFIG_MODE_OFF 0 +/* The performance counters are enabled, but are only written out when a + * PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. + */ +#define PRFCNT_CONFIG_MODE_MANUAL 1 +/* The performance counters are enabled, and are written out each time a tile + * finishes rendering. + */ +#define PRFCNT_CONFIG_MODE_TILE 2 + +/* AS_MEMATTR values from MMU_MEMATTR_STAGE1: */ +/* Use GPU implementation-defined caching policy. */ +#define AS_MEMATTR_IMPL_DEF_CACHE_POLICY 0x88ull +/* The attribute set to force all resources to be cached. */ +#define AS_MEMATTR_FORCE_TO_CACHE_ALL 0x8Full +/* Inner write-alloc cache setup, no outer caching */ +#define AS_MEMATTR_WRITE_ALLOC 0x8Dull + +/* Use GPU implementation-defined caching policy. */ +#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull +/* The attribute set to force all resources to be cached. */ +#define AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL 0x4Full +/* Inner write-alloc cache setup, no outer caching */ +#define AS_MEMATTR_LPAE_WRITE_ALLOC 0x4Dull +/* Set to implementation defined, outer caching */ +#define AS_MEMATTR_LPAE_OUTER_IMPL_DEF 0x88ull +/* Set to write back memory, outer caching */ +#define AS_MEMATTR_LPAE_OUTER_WA 0x8Dull +/* There is no LPAE support for non-cacheable, since the memory type is always + * write-back. + * Marking this setting as reserved for LPAE + */ +#define AS_MEMATTR_LPAE_NON_CACHEABLE_RESERVED + +/* L2_MMU_CONFIG register */ +#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT (23) +#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT) + +/* End L2_MMU_CONFIG register */ + +/* THREAD_* registers */ + +/* THREAD_FEATURES IMPLEMENTATION_TECHNOLOGY values */ +#define IMPLEMENTATION_UNSPECIFIED 0 +#define IMPLEMENTATION_SILICON 1 +#define IMPLEMENTATION_FPGA 2 +#define IMPLEMENTATION_MODEL 3 + +/* Default values when registers are not supported by the implemented hardware */ +#define THREAD_MT_DEFAULT 256 +#define THREAD_MWS_DEFAULT 256 +#define THREAD_MBS_DEFAULT 256 +#define THREAD_MR_DEFAULT 1024 +#define THREAD_MTQ_DEFAULT 4 +#define THREAD_MTGS_DEFAULT 10 + +/* End THREAD_* registers */ + +/* SHADER_CONFIG register */ +#define SC_LS_ALLOW_ATTR_TYPES (1ul << 16) +#define SC_TLS_HASH_ENABLE (1ul << 17) +#define SC_LS_ATTR_CHECK_DISABLE (1ul << 18) +#define SC_VAR_ALGORITHM (1ul << 29) +/* End SHADER_CONFIG register */ + +/* TILER_CONFIG register */ +#define TC_CLOCK_GATE_OVERRIDE (1ul << 0) +/* End TILER_CONFIG register */ + +/* L2_CONFIG register */ +#define L2_CONFIG_SIZE_SHIFT 16 +#define L2_CONFIG_SIZE_MASK (0xFFul << L2_CONFIG_SIZE_SHIFT) +#define L2_CONFIG_HASH_SHIFT 24 +#define L2_CONFIG_HASH_MASK (0xFFul << L2_CONFIG_HASH_SHIFT) +/* End L2_CONFIG register */ + +/* IDVS_GROUP register */ +#define IDVS_GROUP_SIZE_SHIFT (16) +#define IDVS_GROUP_MAX_SIZE (0x3F) + +#endif /* _KBASE_GPU_REGMAP_H_ */ diff --git a/drivers/gpu/arm/bifrost/ipa/Kbuild b/drivers/gpu/arm/bifrost/ipa/Kbuild new file mode 100755 index 000000000000..04aa9d82d7c5 --- /dev/null +++ b/drivers/gpu/arm/bifrost/ipa/Kbuild @@ -0,0 +1,28 @@ +# +# (C) COPYRIGHT 2016-2018 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +bifrost_kbase-y += \ + ipa/mali_kbase_ipa_simple.o \ + ipa/mali_kbase_ipa.o \ + ipa/mali_kbase_ipa_vinstr_g7x.o \ + ipa/mali_kbase_ipa_vinstr_common.o + +bifrost_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c new file mode 100755 index 000000000000..67adb65306dd --- /dev/null +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c @@ -0,0 +1,672 @@ +/* + * + * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ +#include +#include +#include +#include "mali_kbase.h" +#include "mali_kbase_ipa.h" +#include "mali_kbase_ipa_debugfs.h" +#include "mali_kbase_ipa_simple.h" +#include "backend/gpu/mali_kbase_pm_internal.h" + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) +#include +#else +#include +#define dev_pm_opp_find_freq_exact opp_find_freq_exact +#define dev_pm_opp_get_voltage opp_get_voltage +#define dev_pm_opp opp +#endif + +#define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model" + +static const struct kbase_ipa_model_ops *kbase_ipa_all_model_ops[] = { + &kbase_simple_ipa_model_ops, + &kbase_g71_ipa_model_ops, + &kbase_g72_ipa_model_ops, + &kbase_g76_ipa_model_ops, + &kbase_g52_ipa_model_ops, + &kbase_g52_r1_ipa_model_ops, + &kbase_g51_ipa_model_ops, + &kbase_g77_ipa_model_ops, + &kbase_tnax_ipa_model_ops, + &kbase_tbex_ipa_model_ops, + &kbase_tbax_ipa_model_ops +}; + +int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) +{ + int err = 0; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (model->ops->recalculate) { + err = model->ops->recalculate(model); + if (err) { + dev_err(model->kbdev->dev, + "recalculation of power model %s returned error %d\n", + model->ops->name, err); + } + } + + return err; +} + +const struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, + const char *name) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(kbase_ipa_all_model_ops); ++i) { + const struct kbase_ipa_model_ops *ops = kbase_ipa_all_model_ops[i]; + + if (!strcmp(ops->name, name)) + return ops; + } + + dev_err(kbdev->dev, "power model \'%s\' not found\n", name); + + return NULL; +} +KBASE_EXPORT_TEST_API(kbase_ipa_model_ops_find); + +const char *kbase_ipa_model_name_from_id(u32 gpu_id) +{ + const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + switch (GPU_ID2_MODEL_MATCH_VALUE(prod_id)) { + case GPU_ID2_PRODUCT_TMIX: + return "mali-g71-power-model"; + case GPU_ID2_PRODUCT_THEX: + return "mali-g72-power-model"; + case GPU_ID2_PRODUCT_TNOX: + return "mali-g76-power-model"; + case GPU_ID2_PRODUCT_TSIX: + return "mali-g51-power-model"; + case GPU_ID2_PRODUCT_TGOX: + if ((gpu_id & GPU_ID2_VERSION_MAJOR) == + (0 << GPU_ID2_VERSION_MAJOR_SHIFT)) + /* g52 aliased to g76 power-model's ops */ + return "mali-g52-power-model"; + else + return "mali-g52_r1-power-model"; + case GPU_ID2_PRODUCT_TNAX: + return "mali-tnax-power-model"; + case GPU_ID2_PRODUCT_TTRX: + return "mali-g77-power-model"; + case GPU_ID2_PRODUCT_TBEX: + return "mali-tbex-power-model"; + case GPU_ID2_PRODUCT_TBAX: + return "mali-tbax-power-model"; + default: + return KBASE_IPA_FALLBACK_MODEL_NAME; + } +} +KBASE_EXPORT_TEST_API(kbase_ipa_model_name_from_id); + +static struct device_node *get_model_dt_node(struct kbase_ipa_model *model, + bool dt_required) +{ + struct device_node *model_dt_node; + char compat_string[64]; + + snprintf(compat_string, sizeof(compat_string), "arm,%s", + model->ops->name); + + /* of_find_compatible_node() will call of_node_put() on the root node, + * so take a reference on it first. + */ + of_node_get(model->kbdev->dev->of_node); + model_dt_node = of_find_compatible_node(model->kbdev->dev->of_node, + NULL, compat_string); + if (!model_dt_node && !model->missing_dt_node_warning) { + if (dt_required) + dev_warn(model->kbdev->dev, + "Couldn't find power_model DT node matching \'%s\'\n", + compat_string); + model->missing_dt_node_warning = true; + } + + return model_dt_node; +} + +int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, + const char *name, s32 *addr, + size_t num_elems, bool dt_required) +{ + int err, i; + struct device_node *model_dt_node = get_model_dt_node(model, + dt_required); + char *origin; + + err = of_property_read_u32_array(model_dt_node, name, addr, num_elems); + /* We're done with model_dt_node now, so drop the reference taken in + * get_model_dt_node()/of_find_compatible_node(). + */ + of_node_put(model_dt_node); + + if (err && dt_required) { + memset(addr, 0, sizeof(s32) * num_elems); + dev_warn(model->kbdev->dev, + "Error %d, no DT entry: %s.%s = %zu*[0]\n", + err, model->ops->name, name, num_elems); + origin = "zero"; + } else if (err && !dt_required) { + origin = "default"; + } else /* !err */ { + origin = "DT"; + } + + /* Create a unique debugfs entry for each element */ + for (i = 0; i < num_elems; ++i) { + char elem_name[32]; + + if (num_elems == 1) + snprintf(elem_name, sizeof(elem_name), "%s", name); + else + snprintf(elem_name, sizeof(elem_name), "%s.%d", + name, i); + + dev_dbg(model->kbdev->dev, "%s.%s = %d (%s)\n", + model->ops->name, elem_name, addr[i], origin); + + err = kbase_ipa_model_param_add(model, elem_name, + &addr[i], sizeof(s32), + PARAM_TYPE_S32); + if (err) + goto exit; + } +exit: + return err; +} + +int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, + const char *name, char *addr, + size_t size, bool dt_required) +{ + int err; + struct device_node *model_dt_node = get_model_dt_node(model, + dt_required); + const char *string_prop_value; + char *origin; + + err = of_property_read_string(model_dt_node, name, + &string_prop_value); + + /* We're done with model_dt_node now, so drop the reference taken in + * get_model_dt_node()/of_find_compatible_node(). + */ + of_node_put(model_dt_node); + + if (err && dt_required) { + strncpy(addr, "", size - 1); + dev_warn(model->kbdev->dev, + "Error %d, no DT entry: %s.%s = \'%s\'\n", + err, model->ops->name, name, addr); + err = 0; + origin = "zero"; + } else if (err && !dt_required) { + origin = "default"; + } else /* !err */ { + strncpy(addr, string_prop_value, size - 1); + origin = "DT"; + } + + addr[size - 1] = '\0'; + + dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (%s)\n", + model->ops->name, name, string_prop_value, origin); + + err = kbase_ipa_model_param_add(model, name, addr, size, + PARAM_TYPE_STRING); + return err; +} + +void kbase_ipa_term_model(struct kbase_ipa_model *model) +{ + if (!model) + return; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (model->ops->term) + model->ops->term(model); + + kbase_ipa_model_param_free_all(model); + + kfree(model); +} +KBASE_EXPORT_TEST_API(kbase_ipa_term_model); + +struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, + const struct kbase_ipa_model_ops *ops) +{ + struct kbase_ipa_model *model; + int err; + + lockdep_assert_held(&kbdev->ipa.lock); + + if (!ops || !ops->name) + return NULL; + + model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL); + if (!model) + return NULL; + + model->kbdev = kbdev; + model->ops = ops; + INIT_LIST_HEAD(&model->params); + + err = model->ops->init(model); + if (err) { + dev_err(kbdev->dev, + "init of power model \'%s\' returned error %d\n", + ops->name, err); + kfree(model); + return NULL; + } + + err = kbase_ipa_model_recalculate(model); + if (err) { + kbase_ipa_term_model(model); + return NULL; + } + + return model; +} +KBASE_EXPORT_TEST_API(kbase_ipa_init_model); + +static void kbase_ipa_term_locked(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->ipa.lock); + + /* Clean up the models */ + if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) + kbase_ipa_term_model(kbdev->ipa.configured_model); + kbase_ipa_term_model(kbdev->ipa.fallback_model); + + kbdev->ipa.configured_model = NULL; + kbdev->ipa.fallback_model = NULL; +} + +int kbase_ipa_init(struct kbase_device *kbdev) +{ + + const char *model_name; + const struct kbase_ipa_model_ops *ops; + struct kbase_ipa_model *default_model = NULL; + int err; + + mutex_init(&kbdev->ipa.lock); + /* + * Lock during init to avoid warnings from lockdep_assert_held (there + * shouldn't be any concurrent access yet). + */ + mutex_lock(&kbdev->ipa.lock); + + /* The simple IPA model must *always* be present.*/ + ops = kbase_ipa_model_ops_find(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME); + + default_model = kbase_ipa_init_model(kbdev, ops); + if (!default_model) { + err = -EINVAL; + goto end; + } + + kbdev->ipa.fallback_model = default_model; + err = of_property_read_string(kbdev->dev->of_node, + "ipa-model", + &model_name); + if (err) { + /* Attempt to load a match from GPU-ID */ + u32 gpu_id; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + model_name = kbase_ipa_model_name_from_id(gpu_id); + dev_dbg(kbdev->dev, + "Inferring model from GPU ID 0x%x: \'%s\'\n", + gpu_id, model_name); + err = 0; + } else { + dev_dbg(kbdev->dev, + "Using ipa-model parameter from DT: \'%s\'\n", + model_name); + } + + if (strcmp(KBASE_IPA_FALLBACK_MODEL_NAME, model_name) != 0) { + ops = kbase_ipa_model_ops_find(kbdev, model_name); + kbdev->ipa.configured_model = kbase_ipa_init_model(kbdev, ops); + if (!kbdev->ipa.configured_model) { + dev_warn(kbdev->dev, + "Failed to initialize ipa-model: \'%s\'\n" + "Falling back on default model\n", + model_name); + kbdev->ipa.configured_model = default_model; + } + } else { + kbdev->ipa.configured_model = default_model; + } + +end: + if (err) + kbase_ipa_term_locked(kbdev); + else + dev_info(kbdev->dev, + "Using configured power model %s, and fallback %s\n", + kbdev->ipa.configured_model->ops->name, + kbdev->ipa.fallback_model->ops->name); + + mutex_unlock(&kbdev->ipa.lock); + return err; +} +KBASE_EXPORT_TEST_API(kbase_ipa_init); + +void kbase_ipa_term(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->ipa.lock); + kbase_ipa_term_locked(kbdev); + mutex_unlock(&kbdev->ipa.lock); + + mutex_destroy(&kbdev->ipa.lock); +} +KBASE_EXPORT_TEST_API(kbase_ipa_term); + +/** + * kbase_scale_dynamic_power() - Scale a dynamic power coefficient to an OPP + * @c: Dynamic model coefficient, in pW/(Hz V^2). Should be in range + * 0 < c < 2^26 to prevent overflow. + * @freq: Frequency, in Hz. Range: 2^23 < freq < 2^30 (~8MHz to ~1GHz) + * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) + * + * Keep a record of the approximate range of each value at every stage of the + * calculation, to ensure we don't overflow. This makes heavy use of the + * approximations 1000 = 2^10 and 1000000 = 2^20, but does the actual + * calculations in decimal for increased accuracy. + * + * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) + */ +static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq, + const u32 voltage) +{ + /* Range: 2^8 < v2 < 2^16 m(V^2) */ + const u32 v2 = (voltage * voltage) / 1000; + + /* Range: 2^3 < f_MHz < 2^10 MHz */ + const u32 f_MHz = freq / 1000000; + + /* Range: 2^11 < v2f_big < 2^26 kHz V^2 */ + const u32 v2f_big = v2 * f_MHz; + + /* Range: 2^1 < v2f < 2^16 MHz V^2 */ + const u32 v2f = v2f_big / 1000; + + /* Range (working backwards from next line): 0 < v2fc < 2^23 uW. + * Must be < 2^42 to avoid overflowing the return value. */ + const u64 v2fc = (u64) c * (u64) v2f; + + /* Range: 0 < v2fc / 1000 < 2^13 mW */ + return div_u64(v2fc, 1000); +} + +/** + * kbase_scale_static_power() - Scale a static power coefficient to an OPP + * @c: Static model coefficient, in uW/V^3. Should be in range + * 0 < c < 2^32 to prevent overflow. + * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) + * + * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) + */ +u32 kbase_scale_static_power(const u32 c, const u32 voltage) +{ + /* Range: 2^8 < v2 < 2^16 m(V^2) */ + const u32 v2 = (voltage * voltage) / 1000; + + /* Range: 2^17 < v3_big < 2^29 m(V^2) mV */ + const u32 v3_big = v2 * voltage; + + /* Range: 2^7 < v3 < 2^19 m(V^3) */ + const u32 v3 = v3_big / 1000; + + /* + * Range (working backwards from next line): 0 < v3c_big < 2^33 nW. + * The result should be < 2^52 to avoid overflowing the return value. + */ + const u64 v3c_big = (u64) c * (u64) v3; + + /* Range: 0 < v3c_big / 1000000 < 2^13 mW */ + return div_u64(v3c_big, 1000000); +} + +void kbase_ipa_protection_mode_switch_event(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* Record the event of GPU entering protected mode. */ + kbdev->ipa_protection_mode_switched = true; +} + +static struct kbase_ipa_model *get_current_model(struct kbase_device *kbdev) +{ + struct kbase_ipa_model *model; + unsigned long flags; + + lockdep_assert_held(&kbdev->ipa.lock); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (kbdev->ipa_protection_mode_switched || + kbdev->ipa.force_fallback_model) + model = kbdev->ipa.fallback_model; + else + model = kbdev->ipa.configured_model; + + /* + * Having taken cognizance of the fact that whether GPU earlier + * protected mode or not, the event can be now reset (if GPU is not + * currently in protected mode) so that configured model is used + * for the next sample. + */ + if (!kbdev->protected_mode) + kbdev->ipa_protection_mode_switched = false; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return model; +} + +static u32 get_static_power_locked(struct kbase_device *kbdev, + struct kbase_ipa_model *model, + unsigned long voltage) +{ + u32 power = 0; + int err; + u32 power_coeff; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (!model->ops->get_static_coeff) + model = kbdev->ipa.fallback_model; + + if (model->ops->get_static_coeff) { + err = model->ops->get_static_coeff(model, &power_coeff); + if (!err) + power = kbase_scale_static_power(power_coeff, + (u32) voltage); + } + + return power; +} + +#if defined(CONFIG_MALI_PWRSOFT_765) || \ + LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static unsigned long kbase_get_static_power(struct devfreq *df, + unsigned long voltage) +#else +static unsigned long kbase_get_static_power(unsigned long voltage) +#endif +{ + struct kbase_ipa_model *model; + u32 power = 0; +#if defined(CONFIG_MALI_PWRSOFT_765) || \ + LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + struct kbase_device *kbdev = dev_get_drvdata(&df->dev); +#else + struct kbase_device *kbdev = kbase_find_device(-1); +#endif + + if (!kbdev) + return 0ul; + + mutex_lock(&kbdev->ipa.lock); + + model = get_current_model(kbdev); + power = get_static_power_locked(kbdev, model, voltage); + + mutex_unlock(&kbdev->ipa.lock); + +#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ + LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) + kbase_release_device(kbdev); +#endif + + return power; +} + +#if defined(CONFIG_MALI_PWRSOFT_765) || \ + LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static unsigned long kbase_get_dynamic_power(struct devfreq *df, + unsigned long freq, + unsigned long voltage) +#else +static unsigned long kbase_get_dynamic_power(unsigned long freq, + unsigned long voltage) +#endif +{ + struct kbase_ipa_model *model; + u32 power_coeff = 0, power = 0; + int err = 0; +#if defined(CONFIG_MALI_PWRSOFT_765) || \ + LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + struct kbase_device *kbdev = dev_get_drvdata(&df->dev); +#else + struct kbase_device *kbdev = kbase_find_device(-1); +#endif + + if (!kbdev) + return 0ul; + + mutex_lock(&kbdev->ipa.lock); + + model = kbdev->ipa.fallback_model; + + err = model->ops->get_dynamic_coeff(model, &power_coeff); + + if (!err) + power = kbase_scale_dynamic_power(power_coeff, freq, voltage); + else + dev_err_ratelimited(kbdev->dev, + "Model %s returned error code %d\n", + model->ops->name, err); + + mutex_unlock(&kbdev->ipa.lock); + +#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ + LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) + kbase_release_device(kbdev); +#endif + + return power; +} + +int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, + unsigned long freq, + unsigned long voltage) +{ + struct kbase_ipa_model *model; + u32 power_coeff = 0; + int err = 0; + struct kbasep_pm_metrics diff; + u64 total_time; + + lockdep_assert_held(&kbdev->ipa.lock); + + kbase_pm_get_dvfs_metrics(kbdev, &kbdev->ipa.last_metrics, &diff); + + model = get_current_model(kbdev); + + err = model->ops->get_dynamic_coeff(model, &power_coeff); + + /* If the counter model returns an error (e.g. switching back to + * protected mode and failing to read counters, or a counter sample + * with too few cycles), revert to the fallback model. + */ + if (err && model != kbdev->ipa.fallback_model) { + model = kbdev->ipa.fallback_model; + err = model->ops->get_dynamic_coeff(model, &power_coeff); + } + + if (err) + return err; + + *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); + + /* time_busy / total_time cannot be >1, so assigning the 64-bit + * result of div_u64 to *power cannot overflow. + */ + total_time = diff.time_busy + (u64) diff.time_idle; + *power = div_u64(*power * (u64) diff.time_busy, + max(total_time, 1ull)); + + *power += get_static_power_locked(kbdev, model, voltage); + + return err; +} +KBASE_EXPORT_TEST_API(kbase_get_real_power_locked); + +int kbase_get_real_power(struct devfreq *df, u32 *power, + unsigned long freq, + unsigned long voltage) +{ + int ret; + struct kbase_device *kbdev = dev_get_drvdata(&df->dev); + + if (!kbdev) + return -ENODEV; + + mutex_lock(&kbdev->ipa.lock); + ret = kbase_get_real_power_locked(kbdev, power, freq, voltage); + mutex_unlock(&kbdev->ipa.lock); + + return ret; +} +KBASE_EXPORT_TEST_API(kbase_get_real_power); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +struct devfreq_cooling_ops kbase_ipa_power_model_ops = { +#else +struct devfreq_cooling_power kbase_ipa_power_model_ops = { +#endif + .get_static_power = &kbase_get_static_power, + .get_dynamic_power = &kbase_get_dynamic_power, +}; +KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops); diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h new file mode 100755 index 000000000000..f43f3d9416b4 --- /dev/null +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h @@ -0,0 +1,254 @@ +/* + * + * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_IPA_H_ +#define _KBASE_IPA_H_ + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + +struct devfreq; + +/** + * struct kbase_ipa_model - Object describing a particular IPA model. + * @kbdev: pointer to kbase device + * @model_data: opaque pointer to model specific data, accessed + * only by model specific methods. + * @ops: pointer to object containing model specific methods. + * @params: head of the list of debugfs params added for model + * @missing_dt_node_warning: flag to limit the matching power model DT not found + * warning to once. + */ +struct kbase_ipa_model { + struct kbase_device *kbdev; + void *model_data; + const struct kbase_ipa_model_ops *ops; + struct list_head params; + bool missing_dt_node_warning; +}; + +/** + * kbase_ipa_model_add_param_s32 - Add an integer model parameter + * @model: pointer to IPA model + * @name: name of corresponding debugfs entry + * @addr: address where the value is stored + * @num_elems: number of elements (1 if not an array) + * @dt_required: if false, a corresponding devicetree entry is not required, + * and the current value will be used. If true, a warning is + * output and the data is zeroed + * + * Return: 0 on success, or an error code + */ +int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, + const char *name, s32 *addr, + size_t num_elems, bool dt_required); + +/** + * kbase_ipa_model_add_param_string - Add a string model parameter + * @model: pointer to IPA model + * @name: name of corresponding debugfs entry + * @addr: address where the value is stored + * @size: size, in bytes, of the value storage (so the maximum string + * length is size - 1) + * @dt_required: if false, a corresponding devicetree entry is not required, + * and the current value will be used. If true, a warning is + * output and the data is zeroed + * + * Return: 0 on success, or an error code + */ +int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, + const char *name, char *addr, + size_t size, bool dt_required); + +struct kbase_ipa_model_ops { + char *name; + /* The init, recalculate and term ops on the default model are always + * called. However, all the other models are only invoked if the model + * is selected in the device tree. Otherwise they are never + * initialized. Additional resources can be acquired by models in + * init(), however they must be terminated in the term(). + */ + int (*init)(struct kbase_ipa_model *model); + /* Called immediately after init(), or when a parameter is changed, so + * that any coefficients derived from model parameters can be + * recalculated. */ + int (*recalculate)(struct kbase_ipa_model *model); + void (*term)(struct kbase_ipa_model *model); + /* + * get_dynamic_coeff() - calculate dynamic power coefficient + * @model: pointer to model + * @coeffp: pointer to return value location + * + * Calculate a dynamic power coefficient, with units pW/(Hz V^2), which + * is then scaled by the IPA framework according to the current OPP's + * frequency and voltage. + * + * Return: 0 on success, or an error code. + */ + int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp); + /* + * get_static_coeff() - calculate static power coefficient + * @model: pointer to model + * @coeffp: pointer to return value location + * + * Calculate a static power coefficient, with units uW/(V^3), which is + * scaled by the IPA framework according to the current OPP's voltage. + * + * Return: 0 on success, or an error code. + */ + int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); +}; + +/** + * kbase_ipa_init - Initialize the IPA feature + * @kbdev: pointer to kbase device + * + * simple IPA power model is initialized as a fallback model and if that + * initialization fails then IPA is not used. + * The device tree is read for the name of ipa model to be used, by using the + * property string "ipa-model". If that ipa model is supported then it is + * initialized but if the initialization fails then simple power model is used. + * + * Return: 0 on success, negative -errno on error + */ +int kbase_ipa_init(struct kbase_device *kbdev); + +/** + * kbase_ipa_term - Terminate the IPA feature + * @kbdev: pointer to kbase device + * + * Both simple IPA power model and model retrieved from device tree are + * terminated. + */ +void kbase_ipa_term(struct kbase_device *kbdev); + +/** + * kbase_ipa_model_recalculate - Recalculate the model coefficients + * @model: pointer to the IPA model object, already initialized + * + * It shall be called immediately after the model has been initialized + * or when the model parameter has changed, so that any coefficients + * derived from parameters can be recalculated. + * Its a wrapper for the module specific recalculate() method. + * + * Return: 0 on success, negative -errno on error + */ +int kbase_ipa_model_recalculate(struct kbase_ipa_model *model); + +/** + * kbase_ipa_model_ops_find - Lookup an IPA model using its name + * @kbdev: pointer to kbase device + * @name: name of model to lookup + * + * Return: Pointer to model's 'ops' structure, or NULL if the lookup failed. + */ +const struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, + const char *name); + +/** + * kbase_ipa_model_name_from_id - Find the best model for a given GPU ID + * @gpu_id: GPU ID of GPU the model will be used for + * + * Return: The name of the appropriate counter-based model, or the name of the + * fallback model if no counter model exists. + */ +const char *kbase_ipa_model_name_from_id(u32 gpu_id); + +/** + * kbase_ipa_init_model - Initilaize the particular IPA model + * @kbdev: pointer to kbase device + * @ops: pointer to object containing model specific methods. + * + * Initialize the model corresponding to the @ops pointer passed. + * The init() method specified in @ops would be called. + * + * Return: pointer to kbase_ipa_model on success, NULL on error + */ +struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, + const struct kbase_ipa_model_ops *ops); +/** + * kbase_ipa_term_model - Terminate the particular IPA model + * @model: pointer to the IPA model object, already initialized + * + * Terminate the model, using the term() method. + * Module specific parameters would be freed. + */ +void kbase_ipa_term_model(struct kbase_ipa_model *model); + +/** + * kbase_ipa_protection_mode_switch_event - Inform IPA of the GPU's entry into + * protected mode + * @kbdev: pointer to kbase device + * + * Makes IPA aware of the GPU switching to protected mode. + */ +void kbase_ipa_protection_mode_switch_event(struct kbase_device *kbdev); + +extern const struct kbase_ipa_model_ops kbase_g71_ipa_model_ops; +extern const struct kbase_ipa_model_ops kbase_g72_ipa_model_ops; +extern const struct kbase_ipa_model_ops kbase_g76_ipa_model_ops; +extern const struct kbase_ipa_model_ops kbase_g52_ipa_model_ops; +extern const struct kbase_ipa_model_ops kbase_g52_r1_ipa_model_ops; +extern const struct kbase_ipa_model_ops kbase_g51_ipa_model_ops; +extern const struct kbase_ipa_model_ops kbase_g77_ipa_model_ops; +extern const struct kbase_ipa_model_ops kbase_tnax_ipa_model_ops; +extern const struct kbase_ipa_model_ops kbase_tbex_ipa_model_ops; +extern const struct kbase_ipa_model_ops kbase_tbax_ipa_model_ops; + +/** + * kbase_get_real_power() - get the real power consumption of the GPU + * @df: dynamic voltage and frequency scaling information for the GPU. + * @power: where to store the power consumption, in mW. + * @freq: a frequency, in HZ. + * @voltage: a voltage, in mV. + * + * The returned value incorporates both static and dynamic power consumption. + * + * Return: 0 on success, or an error code. + */ +int kbase_get_real_power(struct devfreq *df, u32 *power, + unsigned long freq, + unsigned long voltage); + +#if MALI_UNIT_TEST +/* Called by kbase_get_real_power() to invoke the power models. + * Must be called with kbdev->ipa.lock held. + * This function is only exposed for use by unit tests. + */ +int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, + unsigned long freq, + unsigned long voltage); +#endif /* MALI_UNIT_TEST */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +extern struct devfreq_cooling_ops kbase_ipa_power_model_ops; +#else +extern struct devfreq_cooling_power kbase_ipa_power_model_ops; +#endif + +#else /* !(defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ + +static inline void kbase_ipa_protection_mode_switch_event(struct kbase_device *kbdev) +{ } + +#endif /* (defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ + +#endif diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c new file mode 100755 index 000000000000..30a3b7d1b3be --- /dev/null +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c @@ -0,0 +1,322 @@ +/* + * + * (C) COPYRIGHT 2017-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include + +#include "mali_kbase.h" +#include "mali_kbase_ipa.h" +#include "mali_kbase_ipa_debugfs.h" + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)) +#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE +#endif + +struct kbase_ipa_model_param { + char *name; + union { + void *voidp; + s32 *s32p; + char *str; + } addr; + size_t size; + enum kbase_ipa_model_param_type type; + struct kbase_ipa_model *model; + struct list_head link; +}; + +static int param_int_get(void *data, u64 *val) +{ + struct kbase_ipa_model_param *param = data; + + mutex_lock(¶m->model->kbdev->ipa.lock); + *(s64 *) val = *param->addr.s32p; + mutex_unlock(¶m->model->kbdev->ipa.lock); + + return 0; +} + +static int param_int_set(void *data, u64 val) +{ + struct kbase_ipa_model_param *param = data; + struct kbase_ipa_model *model = param->model; + s64 sval = (s64) val; + s32 old_val; + int err = 0; + + if (sval < S32_MIN || sval > S32_MAX) + return -ERANGE; + + mutex_lock(¶m->model->kbdev->ipa.lock); + old_val = *param->addr.s32p; + *param->addr.s32p = val; + err = kbase_ipa_model_recalculate(model); + if (err < 0) + *param->addr.s32p = old_val; + mutex_unlock(¶m->model->kbdev->ipa.lock); + + return err; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_s32, param_int_get, param_int_set, "%lld\n"); + +static ssize_t param_string_get(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct kbase_ipa_model_param *param = file->private_data; + ssize_t ret; + size_t len; + + mutex_lock(¶m->model->kbdev->ipa.lock); + len = strnlen(param->addr.str, param->size - 1) + 1; + ret = simple_read_from_buffer(user_buf, count, ppos, + param->addr.str, len); + mutex_unlock(¶m->model->kbdev->ipa.lock); + + return ret; +} + +static ssize_t param_string_set(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct kbase_ipa_model_param *param = file->private_data; + struct kbase_ipa_model *model = param->model; + char *old_str = NULL; + ssize_t ret = count; + size_t buf_size; + int err; + + mutex_lock(&model->kbdev->ipa.lock); + + if (count > param->size) { + ret = -EINVAL; + goto end; + } + + old_str = kstrndup(param->addr.str, param->size, GFP_KERNEL); + if (!old_str) { + ret = -ENOMEM; + goto end; + } + + buf_size = min(param->size - 1, count); + if (copy_from_user(param->addr.str, user_buf, buf_size)) { + ret = -EFAULT; + goto end; + } + + param->addr.str[buf_size] = '\0'; + + err = kbase_ipa_model_recalculate(model); + if (err < 0) { + ret = err; + strlcpy(param->addr.str, old_str, param->size); + } + +end: + kfree(old_str); + mutex_unlock(&model->kbdev->ipa.lock); + + return ret; +} + +static const struct file_operations fops_string = { + .owner = THIS_MODULE, + .read = param_string_get, + .write = param_string_set, + .open = simple_open, + .llseek = default_llseek, +}; + +int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, + void *addr, size_t size, + enum kbase_ipa_model_param_type type) +{ + struct kbase_ipa_model_param *param; + + param = kzalloc(sizeof(*param), GFP_KERNEL); + + if (!param) + return -ENOMEM; + + /* 'name' is stack-allocated for array elements, so copy it into + * heap-allocated storage */ + param->name = kstrdup(name, GFP_KERNEL); + + if (!param->name) { + kfree(param); + return -ENOMEM; + } + + param->addr.voidp = addr; + param->size = size; + param->type = type; + param->model = model; + + list_add(¶m->link, &model->params); + + return 0; +} + +void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_param *param_p, *param_n; + + list_for_each_entry_safe(param_p, param_n, &model->params, link) { + list_del(¶m_p->link); + kfree(param_p->name); + kfree(param_p); + } +} + +static int force_fallback_model_get(void *data, u64 *val) +{ + struct kbase_device *kbdev = data; + + mutex_lock(&kbdev->ipa.lock); + *val = kbdev->ipa.force_fallback_model; + mutex_unlock(&kbdev->ipa.lock); + + return 0; +} + +static int force_fallback_model_set(void *data, u64 val) +{ + struct kbase_device *kbdev = data; + + mutex_lock(&kbdev->ipa.lock); + kbdev->ipa.force_fallback_model = (val ? true : false); + mutex_unlock(&kbdev->ipa.lock); + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(force_fallback_model, + force_fallback_model_get, + force_fallback_model_set, + "%llu\n"); + +static int current_power_get(void *data, u64 *val) +{ + struct kbase_device *kbdev = data; + struct devfreq *df = kbdev->devfreq; + u32 power; + + kbase_pm_context_active(kbdev); + /* The current model assumes that there's no more than one voltage + * regulator currently available in the system. + */ + kbase_get_real_power(df, &power, + kbdev->current_nominal_freq, + (kbdev->current_voltages[0] / 1000)); + kbase_pm_context_idle(kbdev); + + *val = power; + + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(current_power, current_power_get, NULL, "%llu\n"); + +static void kbase_ipa_model_debugfs_init(struct kbase_ipa_model *model) +{ + struct list_head *it; + struct dentry *dir; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + dir = debugfs_create_dir(model->ops->name, + model->kbdev->mali_debugfs_directory); + + if (!dir) { + dev_err(model->kbdev->dev, + "Couldn't create mali debugfs %s directory", + model->ops->name); + return; + } + + list_for_each(it, &model->params) { + struct kbase_ipa_model_param *param = + list_entry(it, + struct kbase_ipa_model_param, + link); + const struct file_operations *fops = NULL; + + switch (param->type) { + case PARAM_TYPE_S32: + fops = &fops_s32; + break; + case PARAM_TYPE_STRING: + fops = &fops_string; + break; + } + + if (unlikely(!fops)) { + dev_err(model->kbdev->dev, + "Type not set for %s parameter %s\n", + model->ops->name, param->name); + } else { + debugfs_create_file(param->name, S_IRUGO | S_IWUSR, + dir, param, fops); + } + } +} + +void kbase_ipa_model_param_set_s32(struct kbase_ipa_model *model, + const char *name, s32 val) +{ + struct kbase_ipa_model_param *param; + + mutex_lock(&model->kbdev->ipa.lock); + + list_for_each_entry(param, &model->params, link) { + if (!strcmp(param->name, name)) { + if (param->type == PARAM_TYPE_S32) { + *param->addr.s32p = val; + } else { + dev_err(model->kbdev->dev, + "Wrong type for %s parameter %s\n", + model->ops->name, param->name); + } + break; + } + } + + mutex_unlock(&model->kbdev->ipa.lock); +} +KBASE_EXPORT_TEST_API(kbase_ipa_model_param_set_s32); + +void kbase_ipa_debugfs_init(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->ipa.lock); + + if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) + kbase_ipa_model_debugfs_init(kbdev->ipa.configured_model); + kbase_ipa_model_debugfs_init(kbdev->ipa.fallback_model); + + debugfs_create_file("ipa_current_power", 0444, + kbdev->mali_debugfs_directory, kbdev, ¤t_power); + debugfs_create_file("ipa_force_fallback_model", 0644, + kbdev->mali_debugfs_directory, kbdev, &force_fallback_model); + + mutex_unlock(&kbdev->ipa.lock); +} diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.h b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.h new file mode 100755 index 000000000000..a983d9c14216 --- /dev/null +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.h @@ -0,0 +1,68 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_IPA_DEBUGFS_H_ +#define _KBASE_IPA_DEBUGFS_H_ + +enum kbase_ipa_model_param_type { + PARAM_TYPE_S32 = 1, + PARAM_TYPE_STRING, +}; + +#ifdef CONFIG_DEBUG_FS + +void kbase_ipa_debugfs_init(struct kbase_device *kbdev); +int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, + void *addr, size_t size, + enum kbase_ipa_model_param_type type); +void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model); + +/** + * kbase_ipa_model_param_set_s32 - Set an integer model parameter + * + * @model: pointer to IPA model + * @name: name of corresponding debugfs entry + * @val: new value of the parameter + * + * This function is only exposed for use by unit tests running in + * kernel space. Normally it is expected that parameter values will + * instead be set via debugfs. + */ +void kbase_ipa_model_param_set_s32(struct kbase_ipa_model *model, + const char *name, s32 val); + +#else /* CONFIG_DEBUG_FS */ + +static inline int kbase_ipa_model_param_add(struct kbase_ipa_model *model, + const char *name, void *addr, + size_t size, + enum kbase_ipa_model_param_type type) +{ + return 0; +} + +static inline void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) +{ } + +#endif /* CONFIG_DEBUG_FS */ + +#endif /* _KBASE_IPA_DEBUGFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c new file mode 100755 index 000000000000..9a11ee5b1e74 --- /dev/null +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c @@ -0,0 +1,356 @@ +/* + * + * (C) COPYRIGHT 2016-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif +#include +#include +#include + +#include "mali_kbase.h" +#include "mali_kbase_defs.h" +#include "mali_kbase_ipa_simple.h" +#include "mali_kbase_ipa_debugfs.h" + +#if MALI_UNIT_TEST + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +static unsigned long dummy_temp; + +static int kbase_simple_power_model_get_dummy_temp( + struct thermal_zone_device *tz, + unsigned long *temp) +{ + *temp = READ_ONCE(dummy_temp); + return 0; +} + +#else +static int dummy_temp; + +static int kbase_simple_power_model_get_dummy_temp( + struct thermal_zone_device *tz, + int *temp) +{ + *temp = READ_ONCE(dummy_temp); + return 0; +} +#endif + +/* Intercept calls to the kernel function using a macro */ +#ifdef thermal_zone_get_temp +#undef thermal_zone_get_temp +#endif +#define thermal_zone_get_temp(tz, temp) \ + kbase_simple_power_model_get_dummy_temp(tz, temp) + +void kbase_simple_power_model_set_dummy_temp(int temp) +{ + WRITE_ONCE(dummy_temp, temp); +} +KBASE_EXPORT_TEST_API(kbase_simple_power_model_set_dummy_temp); + +#endif /* MALI_UNIT_TEST */ + +/* + * This model is primarily designed for the Juno platform. It may not be + * suitable for other platforms. The additional resources in this model + * should preferably be minimal, as this model is rarely used when a dynamic + * model is available. + */ + +/** + * struct kbase_ipa_model_simple_data - IPA context per device + * @dynamic_coefficient: dynamic coefficient of the model + * @static_coefficient: static coefficient of the model + * @ts: Thermal scaling coefficients of the model + * @tz_name: Thermal zone name + * @gpu_tz: thermal zone device + * @poll_temperature_thread: Handle for temperature polling thread + * @current_temperature: Most recent value of polled temperature + * @temperature_poll_interval_ms: How often temperature should be checked, in ms + */ + +struct kbase_ipa_model_simple_data { + u32 dynamic_coefficient; + u32 static_coefficient; + s32 ts[4]; + char tz_name[THERMAL_NAME_LENGTH]; + struct thermal_zone_device *gpu_tz; + struct task_struct *poll_temperature_thread; + int current_temperature; + int temperature_poll_interval_ms; +}; +#define FALLBACK_STATIC_TEMPERATURE 55000 + +/** + * calculate_temp_scaling_factor() - Calculate temperature scaling coefficient + * @ts: Signed coefficients, in order t^0 to t^3, with units Deg^-N + * @t: Temperature, in mDeg C. Range: -2^17 < t < 2^17 + * + * Scale the temperature according to a cubic polynomial whose coefficients are + * provided in the device tree. The result is used to scale the static power + * coefficient, where 1000000 means no change. + * + * Return: Temperature scaling factor. Range 0 <= ret <= 10,000,000. + */ +static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t) +{ + /* Range: -2^24 < t2 < 2^24 m(Deg^2) */ + const s64 t2 = div_s64((t * t), 1000); + + /* Range: -2^31 < t3 < 2^31 m(Deg^3) */ + const s64 t3 = div_s64((t * t2), 1000); + + /* + * Sum the parts. t^[1-3] are in m(Deg^N), but the coefficients are in + * Deg^-N, so we need to multiply the last coefficient by 1000. + * Range: -2^63 < res_big < 2^63 + */ + const s64 res_big = ts[3] * t3 /* +/- 2^62 */ + + ts[2] * t2 /* +/- 2^55 */ + + ts[1] * t /* +/- 2^48 */ + + ts[0] * (s64)1000; /* +/- 2^41 */ + + /* Range: -2^60 < res_unclamped < 2^60 */ + s64 res_unclamped = div_s64(res_big, 1000); + + /* Clamp to range of 0x to 10x the static power */ + return clamp(res_unclamped, (s64) 0, (s64) 10000000); +} + +/* We can't call thermal_zone_get_temp() directly in model_static_coeff(), + * because we don't know if tz->lock is held in the same thread. So poll it in + * a separate thread to get around this. */ +static int poll_temperature(void *data) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *) data; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) + unsigned long temp; +#else + int temp; +#endif + + set_freezable(); + + while (!kthread_should_stop()) { + struct thermal_zone_device *tz = READ_ONCE(model_data->gpu_tz); + + if (tz) { + int ret; + + ret = thermal_zone_get_temp(tz, &temp); + if (ret) { + pr_warn_ratelimited("Error reading temperature for gpu thermal zone: %d\n", + ret); + temp = FALLBACK_STATIC_TEMPERATURE; + } + } else { + temp = FALLBACK_STATIC_TEMPERATURE; + } + + WRITE_ONCE(model_data->current_temperature, temp); + + msleep_interruptible(READ_ONCE(model_data->temperature_poll_interval_ms)); + + try_to_freeze(); + } + + return 0; +} + +static int model_static_coeff(struct kbase_ipa_model *model, u32 *coeffp) +{ + u32 temp_scaling_factor; + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *) model->model_data; + u64 coeff_big; + int temp; + + temp = READ_ONCE(model_data->current_temperature); + + /* Range: 0 <= temp_scaling_factor < 2^24 */ + temp_scaling_factor = calculate_temp_scaling_factor(model_data->ts, + temp); + + /* + * Range: 0 <= coeff_big < 2^52 to avoid overflowing *coeffp. This + * means static_coefficient must be in range + * 0 <= static_coefficient < 2^28. + */ + coeff_big = (u64) model_data->static_coefficient * (u64) temp_scaling_factor; + *coeffp = div_u64(coeff_big, 1000000); + + return 0; +} + +static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *) model->model_data; + + *coeffp = model_data->dynamic_coefficient; + + return 0; +} + +static int add_params(struct kbase_ipa_model *model) +{ + int err = 0; + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *)model->model_data; + + err = kbase_ipa_model_add_param_s32(model, "static-coefficient", + &model_data->static_coefficient, + 1, true); + if (err) + goto end; + + err = kbase_ipa_model_add_param_s32(model, "dynamic-coefficient", + &model_data->dynamic_coefficient, + 1, true); + if (err) + goto end; + + err = kbase_ipa_model_add_param_s32(model, "ts", + model_data->ts, 4, true); + if (err) + goto end; + + err = kbase_ipa_model_add_param_string(model, "thermal-zone", + model_data->tz_name, + sizeof(model_data->tz_name), true); + if (err) + goto end; + + model_data->temperature_poll_interval_ms = 200; + err = kbase_ipa_model_add_param_s32(model, "temp-poll-interval-ms", + &model_data->temperature_poll_interval_ms, + 1, false); + +end: + return err; +} + +static int kbase_simple_power_model_init(struct kbase_ipa_model *model) +{ + int err; + struct kbase_ipa_model_simple_data *model_data; + + model_data = kzalloc(sizeof(struct kbase_ipa_model_simple_data), + GFP_KERNEL); + if (!model_data) + return -ENOMEM; + + model->model_data = (void *) model_data; + + model_data->current_temperature = FALLBACK_STATIC_TEMPERATURE; + model_data->poll_temperature_thread = kthread_run(poll_temperature, + (void *) model_data, + "mali-simple-power-model-temp-poll"); + if (IS_ERR(model_data->poll_temperature_thread)) { + err = PTR_ERR(model_data->poll_temperature_thread); + kfree(model_data); + return err; + } + + err = add_params(model); + if (err) { + kbase_ipa_model_param_free_all(model); + kthread_stop(model_data->poll_temperature_thread); + kfree(model_data); + } + + return err; +} + +static int kbase_simple_power_model_recalculate(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *)model->model_data; + struct thermal_zone_device *tz; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (!strnlen(model_data->tz_name, sizeof(model_data->tz_name))) { + model_data->gpu_tz = NULL; + } else { + char tz_name[THERMAL_NAME_LENGTH]; + + strlcpy(tz_name, model_data->tz_name, sizeof(tz_name)); + + /* Release ipa.lock so that thermal_list_lock is not acquired + * with ipa.lock held, thereby avoid lock ordering violation + * lockdep warning. The warning comes as a chain of locks + * ipa.lock --> thermal_list_lock --> tz->lock gets formed + * on registering devfreq cooling device when probe method + * of mali platform driver is invoked. + */ + mutex_unlock(&model->kbdev->ipa.lock); + tz = thermal_zone_get_zone_by_name(tz_name); + mutex_lock(&model->kbdev->ipa.lock); + + if (IS_ERR_OR_NULL(tz)) { + pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n", + PTR_ERR(tz), tz_name); + return -EPROBE_DEFER; + } + + /* Check if another thread raced against us & updated the + * thermal zone name string. Update the gpu_tz pointer only if + * the name string did not change whilst we retrieved the new + * thermal_zone_device pointer, otherwise model_data->tz_name & + * model_data->gpu_tz would become inconsistent with each other. + * The below check will succeed only for the thread which last + * updated the name string. + */ + if (strncmp(tz_name, model_data->tz_name, sizeof(tz_name)) == 0) + model_data->gpu_tz = tz; + } + + return 0; +} + +static void kbase_simple_power_model_term(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *)model->model_data; + + kthread_stop(model_data->poll_temperature_thread); + + kfree(model_data); +} + +struct kbase_ipa_model_ops kbase_simple_ipa_model_ops = { + .name = "mali-simple-power-model", + .init = &kbase_simple_power_model_init, + .recalculate = &kbase_simple_power_model_recalculate, + .term = &kbase_simple_power_model_term, + .get_dynamic_coeff = &model_dynamic_coeff, + .get_static_coeff = &model_static_coeff, +}; +KBASE_EXPORT_TEST_API(kbase_simple_ipa_model_ops); diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.h b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.h new file mode 100755 index 000000000000..84534e07ec55 --- /dev/null +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_IPA_SIMPLE_H_ +#define _KBASE_IPA_SIMPLE_H_ + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + +extern struct kbase_ipa_model_ops kbase_simple_ipa_model_ops; + +#if MALI_UNIT_TEST +/** + * kbase_simple_power_model_set_dummy_temp() - set a dummy temperature value + * @temp: Temperature of the thermal zone, in millidegrees celsius. + * + * This is only intended for use in unit tests, to ensure that the temperature + * values used by the simple power model are predictable. Deterministic + * behavior is necessary to allow validation of the static power values + * computed by this model. + */ +void kbase_simple_power_model_set_dummy_temp(int temp); +#endif /* MALI_UNIT_TEST */ + +#endif /* (defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ + +#endif /* _KBASE_IPA_SIMPLE_H_ */ diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c new file mode 100755 index 000000000000..702db1623101 --- /dev/null +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c @@ -0,0 +1,349 @@ +/* + * + * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_ipa_vinstr_common.h" +#include "mali_kbase_ipa_debugfs.h" + +#define DEFAULT_SCALING_FACTOR 5 + +/* If the value of GPU_ACTIVE is below this, use the simple model + * instead, to avoid extrapolating small amounts of counter data across + * large sample periods. + */ +#define DEFAULT_MIN_SAMPLE_CYCLES 10000 + +/** + * read_hwcnt() - read a counter value + * @model_data: pointer to model data + * @offset: offset, in bytes, into vinstr buffer + * + * Return: A 32-bit counter value. Range: 0 < value < 2^27 (worst case would be + * incrementing every cycle over a ~100ms sample period at a high frequency, + * e.g. 1 GHz: 2^30 * 0.1seconds ~= 2^27. + */ +static inline u32 kbase_ipa_read_hwcnt( + struct kbase_ipa_model_vinstr_data *model_data, + u32 offset) +{ + u8 *p = (u8 *)model_data->dump_buf.dump_buf; + + return *(u32 *)&p[offset]; +} + +static inline s64 kbase_ipa_add_saturate(s64 a, s64 b) +{ + s64 rtn; + + if (a > 0 && (S64_MAX - a) < b) + rtn = S64_MAX; + else if (a < 0 && (S64_MIN - a) > b) + rtn = S64_MIN; + else + rtn = a + b; + + return rtn; +} + +s64 kbase_ipa_sum_all_shader_cores( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, u32 counter) +{ + struct kbase_device *kbdev = model_data->kbdev; + u64 core_mask; + u32 base = 0; + s64 ret = 0; + + core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; + while (core_mask != 0ull) { + if ((core_mask & 1ull) != 0ull) { + /* 0 < counter_value < 2^27 */ + u32 counter_value = kbase_ipa_read_hwcnt(model_data, + base + counter); + + /* 0 < ret < 2^27 * max_num_cores = 2^32 */ + ret = kbase_ipa_add_saturate(ret, counter_value); + } + base += KBASE_IPA_NR_BYTES_PER_BLOCK; + core_mask >>= 1; + } + + /* Range: -2^54 < ret * coeff < 2^54 */ + return ret * coeff; +} + +s64 kbase_ipa_sum_all_memsys_blocks( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, u32 counter) +{ + struct kbase_device *kbdev = model_data->kbdev; + const u32 num_blocks = kbdev->gpu_props.props.l2_props.num_l2_slices; + u32 base = 0; + s64 ret = 0; + u32 i; + + for (i = 0; i < num_blocks; i++) { + /* 0 < counter_value < 2^27 */ + u32 counter_value = kbase_ipa_read_hwcnt(model_data, + base + counter); + + /* 0 < ret < 2^27 * max_num_memsys_blocks = 2^29 */ + ret = kbase_ipa_add_saturate(ret, counter_value); + base += KBASE_IPA_NR_BYTES_PER_BLOCK; + } + + /* Range: -2^51 < ret * coeff < 2^51 */ + return ret * coeff; +} + +s64 kbase_ipa_single_counter( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, u32 counter) +{ + /* Range: 0 < counter_value < 2^27 */ + const u32 counter_value = kbase_ipa_read_hwcnt(model_data, counter); + + /* Range: -2^49 < ret < 2^49 */ + return counter_value * (s64) coeff; +} + +int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) +{ + int errcode; + struct kbase_device *kbdev = model_data->kbdev; + struct kbase_hwcnt_virtualizer *hvirt = kbdev->hwcnt_gpu_virt; + struct kbase_hwcnt_enable_map enable_map; + const struct kbase_hwcnt_metadata *metadata = + kbase_hwcnt_virtualizer_metadata(hvirt); + + if (!metadata) + return -1; + + errcode = kbase_hwcnt_enable_map_alloc(metadata, &enable_map); + if (errcode) { + dev_err(kbdev->dev, "Failed to allocate IPA enable map"); + return errcode; + } + + kbase_hwcnt_enable_map_enable_all(&enable_map); + + /* Disable cycle counter only. */ + enable_map.clk_enable_map = 0; + + errcode = kbase_hwcnt_virtualizer_client_create( + hvirt, &enable_map, &model_data->hvirt_cli); + kbase_hwcnt_enable_map_free(&enable_map); + if (errcode) { + dev_err(kbdev->dev, "Failed to register IPA with virtualizer"); + model_data->hvirt_cli = NULL; + return errcode; + } + + errcode = kbase_hwcnt_dump_buffer_alloc( + metadata, &model_data->dump_buf); + if (errcode) { + dev_err(kbdev->dev, "Failed to allocate IPA dump buffer"); + kbase_hwcnt_virtualizer_client_destroy(model_data->hvirt_cli); + model_data->hvirt_cli = NULL; + return errcode; + } + + return 0; +} + +void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) +{ + if (model_data->hvirt_cli) { + kbase_hwcnt_virtualizer_client_destroy(model_data->hvirt_cli); + kbase_hwcnt_dump_buffer_free(&model_data->dump_buf); + model_data->hvirt_cli = NULL; + } +} + +int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) +{ + struct kbase_ipa_model_vinstr_data *model_data = + (struct kbase_ipa_model_vinstr_data *)model->model_data; + s64 energy = 0; + size_t i; + u64 coeff = 0, coeff_mul = 0; + u64 start_ts_ns, end_ts_ns; + u32 active_cycles; + int err = 0; + + err = kbase_hwcnt_virtualizer_client_dump(model_data->hvirt_cli, + &start_ts_ns, &end_ts_ns, &model_data->dump_buf); + if (err) + goto err0; + + /* Range: 0 (GPU not used at all), to the max sampling interval, say + * 1s, * max GPU frequency (GPU 100% utilized). + * 0 <= active_cycles <= 1 * ~2GHz + * 0 <= active_cycles < 2^31 + */ + active_cycles = model_data->get_active_cycles(model_data); + + if (active_cycles < (u32) max(model_data->min_sample_cycles, 0)) { + err = -ENODATA; + goto err0; + } + + /* Range: 1 <= active_cycles < 2^31 */ + active_cycles = max(1u, active_cycles); + + /* Range of 'energy' is +/- 2^54 * number of IPA groups (~8), so around + * -2^57 < energy < 2^57 + */ + for (i = 0; i < model_data->groups_def_num; i++) { + const struct kbase_ipa_group *group = &model_data->groups_def[i]; + s32 coeff = model_data->group_values[i]; + s64 group_energy = group->op(model_data, coeff, + group->counter_block_offset); + + energy = kbase_ipa_add_saturate(energy, group_energy); + } + + /* Range: 0 <= coeff < 2^57 */ + if (energy > 0) + coeff = energy; + + /* Range: 0 <= coeff < 2^57 (because active_cycles >= 1). However, this + * can be constrained further: Counter values can only be increased by + * a theoretical maximum of about 64k per clock cycle. Beyond this, + * we'd have to sample every 1ms to avoid them overflowing at the + * lowest clock frequency (say 100MHz). Therefore, we can write the + * range of 'coeff' in terms of active_cycles: + * + * coeff = SUM(coeffN * counterN * num_cores_for_counterN) + * coeff <= SUM(coeffN * counterN) * max_num_cores + * coeff <= num_IPA_groups * max_coeff * max_counter * max_num_cores + * (substitute max_counter = 2^16 * active_cycles) + * coeff <= num_IPA_groups * max_coeff * 2^16 * active_cycles * max_num_cores + * coeff <= 2^3 * 2^22 * 2^16 * active_cycles * 2^5 + * coeff <= 2^46 * active_cycles + * + * So after the division: 0 <= coeff <= 2^46 + */ + coeff = div_u64(coeff, active_cycles); + + /* Not all models were derived at the same reference voltage. Voltage + * scaling is done by multiplying by V^2, so we need to *divide* by + * Vref^2 here. + * Range: 0 <= coeff <= 2^49 + */ + coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); + /* Range: 0 <= coeff <= 2^52 */ + coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); + + /* Scale by user-specified integer factor. + * Range: 0 <= coeff_mul < 2^57 + */ + coeff_mul = coeff * model_data->scaling_factor; + + /* The power models have results with units + * mW/(MHz V^2), i.e. nW/(Hz V^2). With precision of 1/1000000, this + * becomes fW/(Hz V^2), which are the units of coeff_mul. However, + * kbase_scale_dynamic_power() expects units of pW/(Hz V^2), so divide + * by 1000. + * Range: 0 <= coeff_mul < 2^47 + */ + coeff_mul = div_u64(coeff_mul, 1000u); + +err0: + /* Clamp to a sensible range - 2^16 gives about 14W at 400MHz/750mV */ + *coeffp = clamp(coeff_mul, (u64) 0, (u64) 1 << 16); + return err; +} + +int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, + const struct kbase_ipa_group *ipa_groups_def, + size_t ipa_group_size, + kbase_ipa_get_active_cycles_callback get_active_cycles, + s32 reference_voltage) +{ + int err = 0; + size_t i; + struct kbase_ipa_model_vinstr_data *model_data; + + if (!model || !ipa_groups_def || !ipa_group_size || !get_active_cycles) + return -EINVAL; + + model_data = kzalloc(sizeof(*model_data), GFP_KERNEL); + if (!model_data) + return -ENOMEM; + + model_data->kbdev = model->kbdev; + model_data->groups_def = ipa_groups_def; + model_data->groups_def_num = ipa_group_size; + model_data->get_active_cycles = get_active_cycles; + + model->model_data = (void *) model_data; + + for (i = 0; i < model_data->groups_def_num; ++i) { + const struct kbase_ipa_group *group = &model_data->groups_def[i]; + + model_data->group_values[i] = group->default_value; + err = kbase_ipa_model_add_param_s32(model, group->name, + &model_data->group_values[i], + 1, false); + if (err) + goto exit; + } + + model_data->scaling_factor = DEFAULT_SCALING_FACTOR; + err = kbase_ipa_model_add_param_s32(model, "scale", + &model_data->scaling_factor, + 1, false); + if (err) + goto exit; + + model_data->min_sample_cycles = DEFAULT_MIN_SAMPLE_CYCLES; + err = kbase_ipa_model_add_param_s32(model, "min_sample_cycles", + &model_data->min_sample_cycles, + 1, false); + if (err) + goto exit; + + model_data->reference_voltage = reference_voltage; + err = kbase_ipa_model_add_param_s32(model, "reference_voltage", + &model_data->reference_voltage, + 1, false); + if (err) + goto exit; + + err = kbase_ipa_attach_vinstr(model_data); + +exit: + if (err) { + kbase_ipa_model_param_free_all(model); + kfree(model_data); + } + return err; +} + +void kbase_ipa_vinstr_common_model_term(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_vinstr_data *model_data = + (struct kbase_ipa_model_vinstr_data *)model->model_data; + + kbase_ipa_detach_vinstr(model_data); + kfree(model_data); +} diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h new file mode 100755 index 000000000000..46e3cd4bc6e1 --- /dev/null +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h @@ -0,0 +1,217 @@ +/* + * + * (C) COPYRIGHT 2017-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_IPA_VINSTR_COMMON_H_ +#define _KBASE_IPA_VINSTR_COMMON_H_ + +#include "mali_kbase.h" +#include "mali_kbase_hwcnt_virtualizer.h" +#include "mali_kbase_hwcnt_types.h" + +/* Maximum number of IPA groups for an IPA model. */ +#define KBASE_IPA_MAX_GROUP_DEF_NUM 16 + +/* Number of bytes per hardware counter in a vinstr_buffer. */ +#define KBASE_IPA_NR_BYTES_PER_CNT 4 + +/* Number of hardware counters per block in a vinstr_buffer. */ +#define KBASE_IPA_NR_CNT_PER_BLOCK 64 + +/* Number of bytes per block in a vinstr_buffer. */ +#define KBASE_IPA_NR_BYTES_PER_BLOCK \ + (KBASE_IPA_NR_CNT_PER_BLOCK * KBASE_IPA_NR_BYTES_PER_CNT) + +struct kbase_ipa_model_vinstr_data; + +typedef u32 (*kbase_ipa_get_active_cycles_callback)(struct kbase_ipa_model_vinstr_data *); + +/** + * struct kbase_ipa_model_vinstr_data - IPA context per device + * @kbdev: pointer to kbase device + * @groups_def: Array of IPA groups. + * @groups_def_num: Number of elements in the array of IPA groups. + * @get_active_cycles: Callback to return number of active cycles during + * counter sample period + * @hvirt_cli: hardware counter virtualizer client handle + * @dump_buf: buffer to dump hardware counters onto + * @reference_voltage: voltage, in mV, of the operating point used when + * deriving the power model coefficients. Range approx + * 0.1V - 5V (~= 8V): 2^7 <= reference_voltage <= 2^13 + * @scaling_factor: User-specified power scaling factor. This is an + * integer, which is multiplied by the power coefficient + * just before OPP scaling. + * Range approx 0-32: 0 < scaling_factor < 2^5 + * @min_sample_cycles: If the value of the GPU_ACTIVE counter (the number of + * cycles the GPU was working) is less than + * min_sample_cycles, the counter model will return an + * error, causing the IPA framework to approximate using + * the cached simple model results instead. This may be + * more accurate than extrapolating using a very small + * counter dump. + */ +struct kbase_ipa_model_vinstr_data { + struct kbase_device *kbdev; + s32 group_values[KBASE_IPA_MAX_GROUP_DEF_NUM]; + const struct kbase_ipa_group *groups_def; + size_t groups_def_num; + kbase_ipa_get_active_cycles_callback get_active_cycles; + struct kbase_hwcnt_virtualizer_client *hvirt_cli; + struct kbase_hwcnt_dump_buffer dump_buf; + s32 reference_voltage; + s32 scaling_factor; + s32 min_sample_cycles; +}; + +/** + * struct ipa_group - represents a single IPA group + * @name: name of the IPA group + * @default_value: default value of coefficient for IPA group. + * Coefficients are interpreted as fractions where the + * denominator is 1000000. + * @op: which operation to be performed on the counter values + * @counter_block_offset: block offset in bytes of the counter used to calculate energy for IPA group + */ +struct kbase_ipa_group { + const char *name; + s32 default_value; + s64 (*op)(struct kbase_ipa_model_vinstr_data *, s32, u32); + u32 counter_block_offset; +}; + +/** + * kbase_ipa_sum_all_shader_cores() - sum a counter over all cores + * @model_data: pointer to model data + * @coeff: model coefficient. Unity is ~2^20, so range approx + * +/- 4.0: -2^22 < coeff < 2^22 + * @counter offset in bytes of the counter used to calculate energy + * for IPA group + * + * Calculate energy estimation based on hardware counter `counter' + * across all shader cores. + * + * Return: Sum of counter values. Range: -2^54 < ret < 2^54 + */ +s64 kbase_ipa_sum_all_shader_cores( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, u32 counter); + +/** + * kbase_ipa_sum_all_memsys_blocks() - sum a counter over all mem system blocks + * @model_data: pointer to model data + * @coeff: model coefficient. Unity is ~2^20, so range approx + * +/- 4.0: -2^22 < coeff < 2^22 + * @counter: offset in bytes of the counter used to calculate energy + * for IPA group + * + * Calculate energy estimation based on hardware counter `counter' across all + * memory system blocks. + * + * Return: Sum of counter values. Range: -2^51 < ret < 2^51 + */ +s64 kbase_ipa_sum_all_memsys_blocks( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, u32 counter); + +/** + * kbase_ipa_single_counter() - sum a single counter + * @model_data: pointer to model data + * @coeff: model coefficient. Unity is ~2^20, so range approx + * +/- 4.0: -2^22 < coeff < 2^22 + * @counter: offset in bytes of the counter used to calculate energy + * for IPA group + * + * Calculate energy estimation based on hardware counter `counter'. + * + * Return: Counter value. Range: -2^49 < ret < 2^49 + */ +s64 kbase_ipa_single_counter( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, u32 counter); + +/** + * attach_vinstr() - attach a vinstr_buffer to an IPA model. + * @model_data pointer to model data + * + * Attach a vinstr_buffer to an IPA model. The vinstr_buffer + * allows access to the hardware counters used to calculate + * energy consumption. + * + * Return: 0 on success, or an error code. + */ +int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); + +/** + * detach_vinstr() - detach a vinstr_buffer from an IPA model. + * @model_data pointer to model data + * + * Detach a vinstr_buffer from an IPA model. + */ +void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); + +/** + * kbase_ipa_vinstr_dynamic_coeff() - calculate dynamic power based on HW counters + * @model: pointer to instantiated model + * @coeffp: pointer to location where calculated power, in + * pW/(Hz V^2), is stored. + * + * This is a GPU-agnostic implementation of the get_dynamic_coeff() + * function of an IPA model. It relies on the model being populated + * with GPU-specific attributes at initialization time. + * + * Return: 0 on success, or an error code. + */ +int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp); + +/** + * kbase_ipa_vinstr_common_model_init() - initialize ipa power model + * @model: ipa power model to initialize + * @ipa_groups_def: array of ipa groups which sets coefficients for + * the corresponding counters used in the ipa model + * @ipa_group_size: number of elements in the array @ipa_groups_def + * @get_active_cycles: callback to return the number of cycles the GPU was + * active during the counter sample period. + * @reference_voltage: voltage, in mV, of the operating point used when + * deriving the power model coefficients. + * + * This initialization function performs initialization steps common + * for ipa models based on counter values. In each call, the model + * passes its specific coefficient values per ipa counter group via + * @ipa_groups_def array. + * + * Return: 0 on success, error code otherwise + */ +int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, + const struct kbase_ipa_group *ipa_groups_def, + size_t ipa_group_size, + kbase_ipa_get_active_cycles_callback get_active_cycles, + s32 reference_voltage); + +/** + * kbase_ipa_vinstr_common_model_term() - terminate ipa power model + * @model: ipa power model to terminate + * + * This function performs all necessary steps to terminate ipa power model + * including clean up of resources allocated to hold model data. + */ +void kbase_ipa_vinstr_common_model_term(struct kbase_ipa_model *model); + +#endif /* _KBASE_IPA_VINSTR_COMMON_H_ */ diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c new file mode 100755 index 000000000000..83174eb66ded --- /dev/null +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c @@ -0,0 +1,490 @@ +/* + * + * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ +#include + +#include "mali_kbase_ipa_vinstr_common.h" +#include "mali_kbase.h" + + +/* Performance counter blocks base offsets */ +#define JM_BASE (0 * KBASE_IPA_NR_BYTES_PER_BLOCK) +#define TILER_BASE (1 * KBASE_IPA_NR_BYTES_PER_BLOCK) +#define MEMSYS_BASE (2 * KBASE_IPA_NR_BYTES_PER_BLOCK) + +/* JM counter block offsets */ +#define JM_GPU_ACTIVE (KBASE_IPA_NR_BYTES_PER_CNT * 6) + +/* Tiler counter block offsets */ +#define TILER_ACTIVE (KBASE_IPA_NR_BYTES_PER_CNT * 45) + +/* MEMSYS counter block offsets */ +#define MEMSYS_L2_ANY_LOOKUP (KBASE_IPA_NR_BYTES_PER_CNT * 25) + +/* SC counter block offsets */ +#define SC_FRAG_ACTIVE (KBASE_IPA_NR_BYTES_PER_CNT * 4) +#define SC_EXEC_CORE_ACTIVE (KBASE_IPA_NR_BYTES_PER_CNT * 26) +#define SC_EXEC_INSTR_FMA (KBASE_IPA_NR_BYTES_PER_CNT * 27) +#define SC_EXEC_INSTR_COUNT (KBASE_IPA_NR_BYTES_PER_CNT * 28) +#define SC_EXEC_INSTR_MSG (KBASE_IPA_NR_BYTES_PER_CNT * 30) +#define SC_TEX_FILT_NUM_OPERATIONS (KBASE_IPA_NR_BYTES_PER_CNT * 39) +#define SC_TEX_COORD_ISSUE (KBASE_IPA_NR_BYTES_PER_CNT * 40) +#define SC_TEX_TFCH_NUM_OPERATIONS (KBASE_IPA_NR_BYTES_PER_CNT * 42) +#define SC_VARY_INSTR (KBASE_IPA_NR_BYTES_PER_CNT * 49) +#define SC_VARY_SLOT_32 (KBASE_IPA_NR_BYTES_PER_CNT * 50) +#define SC_VARY_SLOT_16 (KBASE_IPA_NR_BYTES_PER_CNT * 51) +#define SC_BEATS_RD_LSC (KBASE_IPA_NR_BYTES_PER_CNT * 56) +#define SC_BEATS_WR_LSC (KBASE_IPA_NR_BYTES_PER_CNT * 61) +#define SC_BEATS_WR_TIB (KBASE_IPA_NR_BYTES_PER_CNT * 62) + +/** + * get_jm_counter() - get performance counter offset inside the Job Manager block + * @model_data: pointer to GPU model data. + * @counter_block_offset: offset in bytes of the performance counter inside the Job Manager block. + * + * Return: Block offset in bytes of the required performance counter. + */ +static u32 kbase_g7x_power_model_get_jm_counter(struct kbase_ipa_model_vinstr_data *model_data, + u32 counter_block_offset) +{ + return JM_BASE + counter_block_offset; +} + +/** + * get_memsys_counter() - get performance counter offset inside the Memory System block + * @model_data: pointer to GPU model data. + * @counter_block_offset: offset in bytes of the performance counter inside the (first) Memory System block. + * + * Return: Block offset in bytes of the required performance counter. + */ +static u32 kbase_g7x_power_model_get_memsys_counter(struct kbase_ipa_model_vinstr_data *model_data, + u32 counter_block_offset) +{ + /* The base address of Memory System performance counters is always the same, although their number + * may vary based on the number of cores. For the moment it's ok to return a constant. + */ + return MEMSYS_BASE + counter_block_offset; +} + +/** + * get_sc_counter() - get performance counter offset inside the Shader Cores block + * @model_data: pointer to GPU model data. + * @counter_block_offset: offset in bytes of the performance counter inside the (first) Shader Cores block. + * + * Return: Block offset in bytes of the required performance counter. + */ +static u32 kbase_g7x_power_model_get_sc_counter(struct kbase_ipa_model_vinstr_data *model_data, + u32 counter_block_offset) +{ + const u32 sc_base = MEMSYS_BASE + + (model_data->kbdev->gpu_props.props.l2_props.num_l2_slices * + KBASE_IPA_NR_BYTES_PER_BLOCK); + + return sc_base + counter_block_offset; +} + +/** + * memsys_single_counter() - calculate energy for a single Memory System performance counter. + * @model_data: pointer to GPU model data. + * @coeff: default value of coefficient for IPA group. + * @offset: offset in bytes of the counter inside the block it belongs to. + * + * Return: Energy estimation for a single Memory System performance counter. + */ +static s64 kbase_g7x_sum_all_memsys_blocks( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, + u32 offset) +{ + u32 counter; + + counter = kbase_g7x_power_model_get_memsys_counter(model_data, offset); + return kbase_ipa_sum_all_memsys_blocks(model_data, coeff, counter); +} + +/** + * sum_all_shader_cores() - calculate energy for a Shader Cores performance counter for all cores. + * @model_data: pointer to GPU model data. + * @coeff: default value of coefficient for IPA group. + * @counter_block_offset: offset in bytes of the counter inside the block it belongs to. + * + * Return: Energy estimation for a Shader Cores performance counter for all cores. + */ +static s64 kbase_g7x_sum_all_shader_cores( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, + u32 counter_block_offset) +{ + u32 counter; + + counter = kbase_g7x_power_model_get_sc_counter(model_data, + counter_block_offset); + return kbase_ipa_sum_all_shader_cores(model_data, coeff, counter); +} + +/** + * jm_single_counter() - calculate energy for a single Job Manager performance counter. + * @model_data: pointer to GPU model data. + * @coeff: default value of coefficient for IPA group. + * @counter_block_offset: offset in bytes of the counter inside the block it belongs to. + * + * Return: Energy estimation for a single Job Manager performance counter. + */ +static s64 kbase_g7x_jm_single_counter( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, + u32 counter_block_offset) +{ + u32 counter; + + counter = kbase_g7x_power_model_get_jm_counter(model_data, + counter_block_offset); + return kbase_ipa_single_counter(model_data, coeff, counter); +} + +/** + * get_active_cycles() - return the GPU_ACTIVE counter + * @model_data: pointer to GPU model data. + * + * Return: the number of cycles the GPU was active during the counter sampling + * period. + */ +static u32 kbase_g7x_get_active_cycles( + struct kbase_ipa_model_vinstr_data *model_data) +{ + u32 counter = kbase_g7x_power_model_get_jm_counter(model_data, JM_GPU_ACTIVE); + + /* Counters are only 32-bit, so we can safely multiply by 1 then cast + * the 64-bit result back to a u32. + */ + return kbase_ipa_single_counter(model_data, 1, counter); +} + +/** Table of IPA group definitions. + * + * For each IPA group, this table defines a function to access the given performance block counter (or counters, + * if the operation needs to be iterated on multiple blocks) and calculate energy estimation. + */ + +static const struct kbase_ipa_group ipa_groups_def_g71[] = { + { + .name = "l2_access", + .default_value = 526300, + .op = kbase_g7x_sum_all_memsys_blocks, + .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, + }, + { + .name = "exec_instr_count", + .default_value = 301100, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_COUNT, + }, + { + .name = "tex_issue", + .default_value = 197400, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_TEX_COORD_ISSUE, + }, + { + .name = "tile_wb", + .default_value = -156400, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_BEATS_WR_TIB, + }, + { + .name = "gpu_active", + .default_value = 115800, + .op = kbase_g7x_jm_single_counter, + .counter_block_offset = JM_GPU_ACTIVE, + }, +}; + +static const struct kbase_ipa_group ipa_groups_def_g72[] = { + { + .name = "l2_access", + .default_value = 393000, + .op = kbase_g7x_sum_all_memsys_blocks, + .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, + }, + { + .name = "exec_instr_count", + .default_value = 227000, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_COUNT, + }, + { + .name = "tex_issue", + .default_value = 181900, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_TEX_COORD_ISSUE, + }, + { + .name = "tile_wb", + .default_value = -120200, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_BEATS_WR_TIB, + }, + { + .name = "gpu_active", + .default_value = 133100, + .op = kbase_g7x_jm_single_counter, + .counter_block_offset = JM_GPU_ACTIVE, + }, +}; + +static const struct kbase_ipa_group ipa_groups_def_g76[] = { + { + .name = "gpu_active", + .default_value = 122000, + .op = kbase_g7x_jm_single_counter, + .counter_block_offset = JM_GPU_ACTIVE, + }, + { + .name = "exec_instr_count", + .default_value = 488900, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_COUNT, + }, + { + .name = "vary_instr", + .default_value = 212100, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_VARY_INSTR, + }, + { + .name = "tex_tfch_num_operations", + .default_value = 288000, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, + }, + { + .name = "l2_access", + .default_value = 378100, + .op = kbase_g7x_sum_all_memsys_blocks, + .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, + }, +}; + +static const struct kbase_ipa_group ipa_groups_def_g52_r1[] = { + { + .name = "gpu_active", + .default_value = 224200, + .op = kbase_g7x_jm_single_counter, + .counter_block_offset = JM_GPU_ACTIVE, + }, + { + .name = "exec_instr_count", + .default_value = 384700, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_COUNT, + }, + { + .name = "vary_instr", + .default_value = 271900, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_VARY_INSTR, + }, + { + .name = "tex_tfch_num_operations", + .default_value = 477700, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, + }, + { + .name = "l2_access", + .default_value = 551400, + .op = kbase_g7x_sum_all_memsys_blocks, + .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, + }, +}; + +static const struct kbase_ipa_group ipa_groups_def_g51[] = { + { + .name = "gpu_active", + .default_value = 201400, + .op = kbase_g7x_jm_single_counter, + .counter_block_offset = JM_GPU_ACTIVE, + }, + { + .name = "exec_instr_count", + .default_value = 392700, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_COUNT, + }, + { + .name = "vary_instr", + .default_value = 274000, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_VARY_INSTR, + }, + { + .name = "tex_tfch_num_operations", + .default_value = 528000, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, + }, + { + .name = "l2_access", + .default_value = 506400, + .op = kbase_g7x_sum_all_memsys_blocks, + .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, + }, +}; + +static const struct kbase_ipa_group ipa_groups_def_g77[] = { + { + .name = "l2_access", + .default_value = 710800, + .op = kbase_g7x_sum_all_memsys_blocks, + .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, + }, + { + .name = "exec_instr_msg", + .default_value = 2375300, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_MSG, + }, + { + .name = "exec_instr_fma", + .default_value = 656100, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_FMA, + }, + { + .name = "tex_filt_num_operations", + .default_value = 318800, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, + }, + { + .name = "gpu_active", + .default_value = 172800, + .op = kbase_g7x_jm_single_counter, + .counter_block_offset = JM_GPU_ACTIVE, + }, +}; + +static const struct kbase_ipa_group ipa_groups_def_tbex[] = { + { + .name = "l2_access", + .default_value = 599800, + .op = kbase_g7x_sum_all_memsys_blocks, + .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, + }, + { + .name = "exec_instr_msg", + .default_value = 1830200, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_MSG, + }, + { + .name = "exec_instr_fma", + .default_value = 407300, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_FMA, + }, + { + .name = "tex_filt_num_operations", + .default_value = 224500, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, + }, + { + .name = "gpu_active", + .default_value = 153800, + .op = kbase_g7x_jm_single_counter, + .counter_block_offset = JM_GPU_ACTIVE, + }, +}; + +static const struct kbase_ipa_group ipa_groups_def_tbax[] = { + { + .name = "l2_access", + .default_value = 599800, + .op = kbase_g7x_sum_all_memsys_blocks, + .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, + }, + { + .name = "exec_instr_msg", + .default_value = 1830200, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_MSG, + }, + { + .name = "exec_instr_fma", + .default_value = 407300, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_EXEC_INSTR_FMA, + }, + { + .name = "tex_filt_num_operations", + .default_value = 224500, + .op = kbase_g7x_sum_all_shader_cores, + .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, + }, + { + .name = "gpu_active", + .default_value = 153800, + .op = kbase_g7x_jm_single_counter, + .counter_block_offset = JM_GPU_ACTIVE, + }, +}; + + +#define IPA_POWER_MODEL_OPS(gpu, init_token) \ + const struct kbase_ipa_model_ops kbase_ ## gpu ## _ipa_model_ops = { \ + .name = "mali-" #gpu "-power-model", \ + .init = kbase_ ## init_token ## _power_model_init, \ + .term = kbase_ipa_vinstr_common_model_term, \ + .get_dynamic_coeff = kbase_ipa_vinstr_dynamic_coeff, \ + }; \ + KBASE_EXPORT_TEST_API(kbase_ ## gpu ## _ipa_model_ops) + +#define STANDARD_POWER_MODEL(gpu, reference_voltage) \ + static int kbase_ ## gpu ## _power_model_init(\ + struct kbase_ipa_model *model) \ + { \ + BUILD_BUG_ON(ARRAY_SIZE(ipa_groups_def_ ## gpu) > \ + KBASE_IPA_MAX_GROUP_DEF_NUM); \ + return kbase_ipa_vinstr_common_model_init(model, \ + ipa_groups_def_ ## gpu, \ + ARRAY_SIZE(ipa_groups_def_ ## gpu), \ + kbase_g7x_get_active_cycles, \ + (reference_voltage)); \ + } \ + IPA_POWER_MODEL_OPS(gpu, gpu) + +#define ALIAS_POWER_MODEL(gpu, as_gpu) \ + IPA_POWER_MODEL_OPS(gpu, as_gpu) + +STANDARD_POWER_MODEL(g71, 800); +STANDARD_POWER_MODEL(g72, 800); +STANDARD_POWER_MODEL(g76, 800); +STANDARD_POWER_MODEL(g52_r1, 1000); +STANDARD_POWER_MODEL(g51, 1000); +STANDARD_POWER_MODEL(g77, 1000); +STANDARD_POWER_MODEL(tbex, 1000); +STANDARD_POWER_MODEL(tbax, 1000); + +/* g52 is an alias of g76 (TNOX) for IPA */ +ALIAS_POWER_MODEL(g52, g76); +/* tnax is an alias of g77 (TTRX) for IPA */ +ALIAS_POWER_MODEL(tnax, g77); diff --git a/drivers/gpu/arm/bifrost/jm/mali_base_jm_kernel.h b/drivers/gpu/arm/bifrost/jm/mali_base_jm_kernel.h new file mode 100755 index 000000000000..9367cc5431cf --- /dev/null +++ b/drivers/gpu/arm/bifrost/jm/mali_base_jm_kernel.h @@ -0,0 +1,1079 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ +#ifndef _BASE_JM_KERNEL_H_ +#define _BASE_JM_KERNEL_H_ + +/* Memory allocation, access/hint flags. + * + * See base_mem_alloc_flags. + */ + +/* IN */ +/* Read access CPU side + */ +#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) + +/* Write access CPU side + */ +#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) + +/* Read access GPU side + */ +#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) + +/* Write access GPU side + */ +#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) + +/* Execute allowed on the GPU side + */ +#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) + +/* Will be permanently mapped in kernel space. + * Flag is only allowed on allocations originating from kbase. + */ +#define BASEP_MEM_PERMANENT_KERNEL_MAPPING ((base_mem_alloc_flags)1 << 5) + +/* The allocation will completely reside within the same 4GB chunk in the GPU + * virtual space. + * Since this flag is primarily required only for the TLS memory which will + * not be used to contain executable code and also not used for Tiler heap, + * it can't be used along with BASE_MEM_PROT_GPU_EX and TILER_ALIGN_TOP flags. + */ +#define BASE_MEM_GPU_VA_SAME_4GB_PAGE ((base_mem_alloc_flags)1 << 6) + +/* Userspace is not allowed to free this memory. + * Flag is only allowed on allocations originating from kbase. + */ +#define BASEP_MEM_NO_USER_FREE ((base_mem_alloc_flags)1 << 7) + +#define BASE_MEM_RESERVED_BIT_8 ((base_mem_alloc_flags)1 << 8) + +/* Grow backing store on GPU Page Fault + */ +#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) + +/* Page coherence Outer shareable, if available + */ +#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) + +/* Page coherence Inner shareable + */ +#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) + +/* IN/OUT */ +/* Should be cached on the CPU, returned if actually cached + */ +#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) + +/* IN/OUT */ +/* Must have same VA on both the GPU and the CPU + */ +#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) + +/* OUT */ +/* Must call mmap to acquire a GPU address for the allocation + */ +#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) + +/* IN */ +/* Page coherence Outer shareable, required. + */ +#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) + +/* Protected memory + */ +#define BASE_MEM_PROTECTED ((base_mem_alloc_flags)1 << 16) + +/* Not needed physical memory + */ +#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) + +/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the + * addresses to be the same + */ +#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) + +/** + * Bit 19 is reserved. + * + * Do not remove, use the next unreserved bit for new flags + */ +#define BASE_MEM_RESERVED_BIT_19 ((base_mem_alloc_flags)1 << 19) + +/** + * Memory starting from the end of the initial commit is aligned to 'extent' + * pages, where 'extent' must be a power of 2 and no more than + * BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES + */ +#define BASE_MEM_TILER_ALIGN_TOP ((base_mem_alloc_flags)1 << 20) + +/* Should be uncached on the GPU, will work only for GPUs using AARCH64 mmu + * mode. Some components within the GPU might only be able to access memory + * that is GPU cacheable. Refer to the specific GPU implementation for more + * details. The 3 shareability flags will be ignored for GPU uncached memory. + * If used while importing USER_BUFFER type memory, then the import will fail + * if the memory is not aligned to GPU and CPU cache line width. + */ +#define BASE_MEM_UNCACHED_GPU ((base_mem_alloc_flags)1 << 21) + +/* + * Bits [22:25] for group_id (0~15). + * + * base_mem_group_id_set() should be used to pack a memory group ID into a + * base_mem_alloc_flags value instead of accessing the bits directly. + * base_mem_group_id_get() should be used to extract the memory group ID from + * a base_mem_alloc_flags value. + */ +#define BASEP_MEM_GROUP_ID_SHIFT 22 +#define BASE_MEM_GROUP_ID_MASK \ + ((base_mem_alloc_flags)0xF << BASEP_MEM_GROUP_ID_SHIFT) + +/* Must do CPU cache maintenance when imported memory is mapped/unmapped + * on GPU. Currently applicable to dma-buf type only. + */ +#define BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP ((base_mem_alloc_flags)1 << 26) + +/* Use the GPU VA chosen by the kernel client */ +#define BASE_MEM_FLAG_MAP_FIXED ((base_mem_alloc_flags)1 << 27) + +/* OUT */ +/* Kernel side cache sync ops required */ +#define BASE_MEM_KERNEL_SYNC ((base_mem_alloc_flags)1 << 28) + +/* Force trimming of JIT allocations when creating a new allocation */ +#define BASEP_MEM_PERFORM_JIT_TRIM ((base_mem_alloc_flags)1 << 29) + +/* Number of bits used as flags for base memory management + * + * Must be kept in sync with the base_mem_alloc_flags flags + */ +#define BASE_MEM_FLAGS_NR_BITS 30 + +/* A mask of all the flags which are only valid for allocations within kbase, + * and may not be passed from user space. + */ +#define BASEP_MEM_FLAGS_KERNEL_ONLY \ + (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE | \ + BASE_MEM_FLAG_MAP_FIXED | BASEP_MEM_PERFORM_JIT_TRIM) + +/* A mask for all output bits, excluding IN/OUT bits. + */ +#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP + +/* A mask for all input bits, including IN/OUT bits. + */ +#define BASE_MEM_FLAGS_INPUT_MASK \ + (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) + +/* A mask of all currently reserved flags + */ +#define BASE_MEM_FLAGS_RESERVED \ + (BASE_MEM_RESERVED_BIT_8 | BASE_MEM_RESERVED_BIT_19) + +#define BASEP_MEM_INVALID_HANDLE (0ull << 12) +#define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12) +#define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12) +#define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12) +#define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12) +/* reserved handles ..-47< for future special handles */ +#define BASE_MEM_COOKIE_BASE (64ul << 12) +#define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ + BASE_MEM_COOKIE_BASE) + +/* Similar to BASE_MEM_TILER_ALIGN_TOP, memory starting from the end of the + * initial commit is aligned to 'extent' pages, where 'extent' must be a power + * of 2 and no more than BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES + */ +#define BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP (1 << 0) + +/** + * If set, the heap info address points to a u32 holding the used size in bytes; + * otherwise it points to a u64 holding the lowest address of unused memory. + */ +#define BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE (1 << 1) + +/** + * Valid set of just-in-time memory allocation flags + * + * Note: BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE cannot be set if heap_info_gpu_addr + * in %base_jit_alloc_info is 0 (atom with BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE set + * and heap_info_gpu_addr being 0 will be rejected). + */ +#define BASE_JIT_ALLOC_VALID_FLAGS \ + (BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP | BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) + +/** + * typedef base_context_create_flags - Flags to pass to ::base_context_init. + * + * Flags can be ORed together to enable multiple things. + * + * These share the same space as BASEP_CONTEXT_FLAG_*, and so must + * not collide with them. + */ +typedef u32 base_context_create_flags; + +/* No flags set */ +#define BASE_CONTEXT_CREATE_FLAG_NONE ((base_context_create_flags)0) + +/* Base context is embedded in a cctx object (flag used for CINSTR + * software counter macros) + */ +#define BASE_CONTEXT_CCTX_EMBEDDED ((base_context_create_flags)1 << 0) + +/* Base context is a 'System Monitor' context for Hardware counters. + * + * One important side effect of this is that job submission is disabled. + */ +#define BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED \ + ((base_context_create_flags)1 << 1) + +/* Bit-shift used to encode a memory group ID in base_context_create_flags + */ +#define BASEP_CONTEXT_MMU_GROUP_ID_SHIFT (3) + +/* Bitmask used to encode a memory group ID in base_context_create_flags + */ +#define BASEP_CONTEXT_MMU_GROUP_ID_MASK \ + ((base_context_create_flags)0xF << BASEP_CONTEXT_MMU_GROUP_ID_SHIFT) + +/* Bitpattern describing the base_context_create_flags that can be + * passed to the kernel + */ +#define BASEP_CONTEXT_CREATE_KERNEL_FLAGS \ + (BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED | \ + BASEP_CONTEXT_MMU_GROUP_ID_MASK) + +/* Bitpattern describing the ::base_context_create_flags that can be + * passed to base_context_init() + */ +#define BASEP_CONTEXT_CREATE_ALLOWED_FLAGS \ + (BASE_CONTEXT_CCTX_EMBEDDED | BASEP_CONTEXT_CREATE_KERNEL_FLAGS) + +/* + * Private flags used on the base context + * + * These start at bit 31, and run down to zero. + * + * They share the same space as base_context_create_flags, and so must + * not collide with them. + */ + +/* Private flag tracking whether job descriptor dumping is disabled */ +#define BASEP_CONTEXT_FLAG_JOB_DUMP_DISABLED \ + ((base_context_create_flags)(1 << 31)) + +/* Enable additional tracepoints for latency measurements (TL_ATOM_READY, + * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) + */ +#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) + +/* Indicate that job dumping is enabled. This could affect certain timers + * to account for the performance impact. + */ +#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) + +#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ + BASE_TLSTREAM_JOB_DUMPING_ENABLED) +/* + * Dependency stuff, keep it private for now. May want to expose it if + * we decide to make the number of semaphores a configurable + * option. + */ +#define BASE_JD_ATOM_COUNT 256 + +/* Maximum number of concurrent render passes. + */ +#define BASE_JD_RP_COUNT (256) + +/* Set/reset values for a software event */ +#define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) +#define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) + +/** + * struct base_jd_udata - Per-job data + * + * This structure is used to store per-job data, and is completely unused + * by the Base driver. It can be used to store things such as callback + * function pointer, data to handle job completion. It is guaranteed to be + * untouched by the Base driver. + * + * @blob: per-job data array + */ +struct base_jd_udata { + u64 blob[2]; +}; + +/** + * typedef base_jd_dep_type - Job dependency type. + * + * A flags field will be inserted into the atom structure to specify whether a + * dependency is a data or ordering dependency (by putting it before/after + * 'core_req' in the structure it should be possible to add without changing + * the structure size). + * When the flag is set for a particular dependency to signal that it is an + * ordering only dependency then errors will not be propagated. + */ +typedef u8 base_jd_dep_type; + +#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */ +#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */ +#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */ + +/** + * typedef base_jd_core_req - Job chain hardware requirements. + * + * A job chain must specify what GPU features it needs to allow the + * driver to schedule the job correctly. By not specifying the + * correct settings can/will cause an early job termination. Multiple + * values can be ORed together to specify multiple requirements. + * Special case is ::BASE_JD_REQ_DEP, which is used to express complex + * dependencies, and that doesn't execute anything on the hardware. + */ +typedef u32 base_jd_core_req; + +/* Requirements that come from the HW */ + +/* No requirement, dependency only + */ +#define BASE_JD_REQ_DEP ((base_jd_core_req)0) + +/* Requires fragment shaders + */ +#define BASE_JD_REQ_FS ((base_jd_core_req)1 << 0) + +/* Requires compute shaders + * + * This covers any of the following GPU job types: + * - Vertex Shader Job + * - Geometry Shader Job + * - An actual Compute Shader Job + * + * Compare this with BASE_JD_REQ_ONLY_COMPUTE, which specifies that the + * job is specifically just the "Compute Shader" job type, and not the "Vertex + * Shader" nor the "Geometry Shader" job type. + */ +#define BASE_JD_REQ_CS ((base_jd_core_req)1 << 1) + +/* Requires tiling */ +#define BASE_JD_REQ_T ((base_jd_core_req)1 << 2) + +/* Requires cache flushes */ +#define BASE_JD_REQ_CF ((base_jd_core_req)1 << 3) + +/* Requires value writeback */ +#define BASE_JD_REQ_V ((base_jd_core_req)1 << 4) + +/* SW-only requirements - the HW does not expose these as part of the job slot + * capabilities + */ + +/* Requires fragment job with AFBC encoding */ +#define BASE_JD_REQ_FS_AFBC ((base_jd_core_req)1 << 13) + +/* SW-only requirement: coalesce completion events. + * If this bit is set then completion of this atom will not cause an event to + * be sent to userspace, whether successful or not; completion events will be + * deferred until an atom completes which does not have this bit set. + * + * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES. + */ +#define BASE_JD_REQ_EVENT_COALESCE ((base_jd_core_req)1 << 5) + +/* SW Only requirement: the job chain requires a coherent core group. We don't + * mind which coherent core group is used. + */ +#define BASE_JD_REQ_COHERENT_GROUP ((base_jd_core_req)1 << 6) + +/* SW Only requirement: The performance counters should be enabled only when + * they are needed, to reduce power consumption. + */ +#define BASE_JD_REQ_PERMON ((base_jd_core_req)1 << 7) + +/* SW Only requirement: External resources are referenced by this atom. + * + * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE and + * BASE_JD_REQ_SOFT_EVENT_WAIT. + */ +#define BASE_JD_REQ_EXTERNAL_RESOURCES ((base_jd_core_req)1 << 8) + +/* SW Only requirement: Software defined job. Jobs with this bit set will not be + * submitted to the hardware but will cause some action to happen within the + * driver + */ +#define BASE_JD_REQ_SOFT_JOB ((base_jd_core_req)1 << 9) + +#define BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME (BASE_JD_REQ_SOFT_JOB | 0x1) +#define BASE_JD_REQ_SOFT_FENCE_TRIGGER (BASE_JD_REQ_SOFT_JOB | 0x2) +#define BASE_JD_REQ_SOFT_FENCE_WAIT (BASE_JD_REQ_SOFT_JOB | 0x3) + +/* 0x4 RESERVED for now */ + +/* SW only requirement: event wait/trigger job. + * + * - BASE_JD_REQ_SOFT_EVENT_WAIT: this job will block until the event is set. + * - BASE_JD_REQ_SOFT_EVENT_SET: this job sets the event, thus unblocks the + * other waiting jobs. It completes immediately. + * - BASE_JD_REQ_SOFT_EVENT_RESET: this job resets the event, making it + * possible for other jobs to wait upon. It completes immediately. + */ +#define BASE_JD_REQ_SOFT_EVENT_WAIT (BASE_JD_REQ_SOFT_JOB | 0x5) +#define BASE_JD_REQ_SOFT_EVENT_SET (BASE_JD_REQ_SOFT_JOB | 0x6) +#define BASE_JD_REQ_SOFT_EVENT_RESET (BASE_JD_REQ_SOFT_JOB | 0x7) + +#define BASE_JD_REQ_SOFT_DEBUG_COPY (BASE_JD_REQ_SOFT_JOB | 0x8) + +/* SW only requirement: Just In Time allocation + * + * This job requests a single or multiple just-in-time allocations through a + * list of base_jit_alloc_info structure which is passed via the jc element of + * the atom. The number of base_jit_alloc_info structures present in the + * list is passed via the nr_extres element of the atom + * + * It should be noted that the id entry in base_jit_alloc_info must not + * be reused until it has been released via BASE_JD_REQ_SOFT_JIT_FREE. + * + * Should this soft job fail it is expected that a BASE_JD_REQ_SOFT_JIT_FREE + * soft job to free the JIT allocation is still made. + * + * The job will complete immediately. + */ +#define BASE_JD_REQ_SOFT_JIT_ALLOC (BASE_JD_REQ_SOFT_JOB | 0x9) + +/* SW only requirement: Just In Time free + * + * This job requests a single or multiple just-in-time allocations created by + * BASE_JD_REQ_SOFT_JIT_ALLOC to be freed. The ID list of the just-in-time + * allocations is passed via the jc element of the atom. + * + * The job will complete immediately. + */ +#define BASE_JD_REQ_SOFT_JIT_FREE (BASE_JD_REQ_SOFT_JOB | 0xa) + +/* SW only requirement: Map external resource + * + * This job requests external resource(s) are mapped once the dependencies + * of the job have been satisfied. The list of external resources are + * passed via the jc element of the atom which is a pointer to a + * base_external_resource_list. + */ +#define BASE_JD_REQ_SOFT_EXT_RES_MAP (BASE_JD_REQ_SOFT_JOB | 0xb) + +/* SW only requirement: Unmap external resource + * + * This job requests external resource(s) are unmapped once the dependencies + * of the job has been satisfied. The list of external resources are + * passed via the jc element of the atom which is a pointer to a + * base_external_resource_list. + */ +#define BASE_JD_REQ_SOFT_EXT_RES_UNMAP (BASE_JD_REQ_SOFT_JOB | 0xc) + +/* HW Requirement: Requires Compute shaders (but not Vertex or Geometry Shaders) + * + * This indicates that the Job Chain contains GPU jobs of the 'Compute + * Shaders' type. + * + * In contrast to BASE_JD_REQ_CS, this does not indicate that the Job + * Chain contains 'Geometry Shader' or 'Vertex Shader' jobs. + */ +#define BASE_JD_REQ_ONLY_COMPUTE ((base_jd_core_req)1 << 10) + +/* HW Requirement: Use the base_jd_atom::device_nr field to specify a + * particular core group + * + * If both BASE_JD_REQ_COHERENT_GROUP and this flag are set, this flag + * takes priority + * + * This is only guaranteed to work for BASE_JD_REQ_ONLY_COMPUTE atoms. + * + * If the core availability policy is keeping the required core group turned + * off, then the job will fail with a BASE_JD_EVENT_PM_EVENT error code. + */ +#define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ((base_jd_core_req)1 << 11) + +/* SW Flag: If this bit is set then the successful completion of this atom + * will not cause an event to be sent to userspace + */ +#define BASE_JD_REQ_EVENT_ONLY_ON_FAILURE ((base_jd_core_req)1 << 12) + +/* SW Flag: If this bit is set then completion of this atom will not cause an + * event to be sent to userspace, whether successful or not. + */ +#define BASEP_JD_REQ_EVENT_NEVER ((base_jd_core_req)1 << 14) + +/* SW Flag: Skip GPU cache clean and invalidation before starting a GPU job. + * + * If this bit is set then the GPU's cache will not be cleaned and invalidated + * until a GPU job starts which does not have this bit set or a job completes + * which does not have the BASE_JD_REQ_SKIP_CACHE_END bit set. Do not use + * if the CPU may have written to memory addressed by the job since the last job + * without this bit set was submitted. + */ +#define BASE_JD_REQ_SKIP_CACHE_START ((base_jd_core_req)1 << 15) + +/* SW Flag: Skip GPU cache clean and invalidation after a GPU job completes. + * + * If this bit is set then the GPU's cache will not be cleaned and invalidated + * until a GPU job completes which does not have this bit set or a job starts + * which does not have the BASE_JD_REQ_SKIP_CACHE_START bit set. Do not use + * if the CPU may read from or partially overwrite memory addressed by the job + * before the next job without this bit set completes. + */ +#define BASE_JD_REQ_SKIP_CACHE_END ((base_jd_core_req)1 << 16) + +/* Request the atom be executed on a specific job slot. + * + * When this flag is specified, it takes precedence over any existing job slot + * selection logic. + */ +#define BASE_JD_REQ_JOB_SLOT ((base_jd_core_req)1 << 17) + +/* SW-only requirement: The atom is the start of a renderpass. + * + * If this bit is set then the job chain will be soft-stopped if it causes the + * GPU to write beyond the end of the physical pages backing the tiler heap, and + * committing more memory to the heap would exceed an internal threshold. It may + * be resumed after running one of the job chains attached to an atom with + * BASE_JD_REQ_END_RENDERPASS set and the same renderpass ID. It may be + * resumed multiple times until it completes without memory usage exceeding the + * threshold. + * + * Usually used with BASE_JD_REQ_T. + */ +#define BASE_JD_REQ_START_RENDERPASS ((base_jd_core_req)1 << 18) + +/* SW-only requirement: The atom is the end of a renderpass. + * + * If this bit is set then the atom incorporates the CPU address of a + * base_jd_fragment object instead of the GPU address of a job chain. + * + * Which job chain is run depends upon whether the atom with the same renderpass + * ID and the BASE_JD_REQ_START_RENDERPASS bit set completed normally or + * was soft-stopped when it exceeded an upper threshold for tiler heap memory + * usage. + * + * It also depends upon whether one of the job chains attached to the atom has + * already been run as part of the same renderpass (in which case it would have + * written unresolved multisampled and otherwise-discarded output to temporary + * buffers that need to be read back). The job chain for doing a forced read and + * forced write (from/to temporary buffers) is run as many times as necessary. + * + * Usually used with BASE_JD_REQ_FS. + */ +#define BASE_JD_REQ_END_RENDERPASS ((base_jd_core_req)1 << 19) + +/* These requirement bits are currently unused in base_jd_core_req + */ +#define BASEP_JD_REQ_RESERVED \ + (~(BASE_JD_REQ_ATOM_TYPE | BASE_JD_REQ_EXTERNAL_RESOURCES | \ + BASE_JD_REQ_EVENT_ONLY_ON_FAILURE | BASEP_JD_REQ_EVENT_NEVER | \ + BASE_JD_REQ_EVENT_COALESCE | \ + BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | \ + BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ + BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END | \ + BASE_JD_REQ_JOB_SLOT | BASE_JD_REQ_START_RENDERPASS | \ + BASE_JD_REQ_END_RENDERPASS)) + +/* Mask of all bits in base_jd_core_req that control the type of the atom. + * + * This allows dependency only atoms to have flags set + */ +#define BASE_JD_REQ_ATOM_TYPE \ + (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T | BASE_JD_REQ_CF | \ + BASE_JD_REQ_V | BASE_JD_REQ_SOFT_JOB | BASE_JD_REQ_ONLY_COMPUTE) + +/** + * Mask of all bits in base_jd_core_req that control the type of a soft job. + */ +#define BASE_JD_REQ_SOFT_JOB_TYPE (BASE_JD_REQ_SOFT_JOB | 0x1f) + +/* Returns non-zero value if core requirements passed define a soft job or + * a dependency only job. + */ +#define BASE_JD_REQ_SOFT_JOB_OR_DEP(core_req) \ + (((core_req) & BASE_JD_REQ_SOFT_JOB) || \ + ((core_req) & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) + +/** + * enum kbase_jd_atom_state + * + * @KBASE_JD_ATOM_STATE_UNUSED: Atom is not used. + * @KBASE_JD_ATOM_STATE_QUEUED: Atom is queued in JD. + * @KBASE_JD_ATOM_STATE_IN_JS: Atom has been given to JS (is runnable/running). + * @KBASE_JD_ATOM_STATE_HW_COMPLETED: Atom has been completed, but not yet + * handed back to job dispatcher for + * dependency resolution. + * @KBASE_JD_ATOM_STATE_COMPLETED: Atom has been completed, but not yet handed + * back to userspace. + */ +enum kbase_jd_atom_state { + KBASE_JD_ATOM_STATE_UNUSED, + KBASE_JD_ATOM_STATE_QUEUED, + KBASE_JD_ATOM_STATE_IN_JS, + KBASE_JD_ATOM_STATE_HW_COMPLETED, + KBASE_JD_ATOM_STATE_COMPLETED +}; + +/** + * typedef base_atom_id - Type big enough to store an atom number in. + */ +typedef u8 base_atom_id; + +/** + * struct base_dependency - + * + * @atom_id: An atom number + * @dependency_type: Dependency type + */ +struct base_dependency { + base_atom_id atom_id; + base_jd_dep_type dependency_type; +}; + +/** + * struct base_jd_fragment - Set of GPU fragment job chains used for rendering. + * + * @norm_read_norm_write: Job chain for full rendering. + * GPU address of a fragment job chain to render in the + * circumstance where the tiler job chain did not exceed + * its memory usage threshold and no fragment job chain + * was previously run for the same renderpass. + * It is used no more than once per renderpass. + * @norm_read_forced_write: Job chain for starting incremental + * rendering. + * GPU address of a fragment job chain to render in + * the circumstance where the tiler job chain exceeded + * its memory usage threshold for the first time and + * no fragment job chain was previously run for the + * same renderpass. + * Writes unresolved multisampled and normally- + * discarded output to temporary buffers that must be + * read back by a subsequent forced_read job chain + * before the renderpass is complete. + * It is used no more than once per renderpass. + * @forced_read_forced_write: Job chain for continuing incremental + * rendering. + * GPU address of a fragment job chain to render in + * the circumstance where the tiler job chain + * exceeded its memory usage threshold again + * and a fragment job chain was previously run for + * the same renderpass. + * Reads unresolved multisampled and + * normally-discarded output from temporary buffers + * written by a previous forced_write job chain and + * writes the same to temporary buffers again. + * It is used as many times as required until + * rendering completes. + * @forced_read_norm_write: Job chain for ending incremental rendering. + * GPU address of a fragment job chain to render in the + * circumstance where the tiler job chain did not + * exceed its memory usage threshold this time and a + * fragment job chain was previously run for the same + * renderpass. + * Reads unresolved multisampled and normally-discarded + * output from temporary buffers written by a previous + * forced_write job chain in order to complete a + * renderpass. + * It is used no more than once per renderpass. + * + * This structure is referenced by the main atom structure if + * BASE_JD_REQ_END_RENDERPASS is set in the base_jd_core_req. + */ +struct base_jd_fragment { + u64 norm_read_norm_write; + u64 norm_read_forced_write; + u64 forced_read_forced_write; + u64 forced_read_norm_write; +}; + +/** + * typedef base_jd_prio - Base Atom priority. + * + * Only certain priority levels are actually implemented, as specified by the + * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority + * level that is not one of those defined below. + * + * Priority levels only affect scheduling after the atoms have had dependencies + * resolved. For example, a low priority atom that has had its dependencies + * resolved might run before a higher priority atom that has not had its + * dependencies resolved. + * + * In general, fragment atoms do not affect non-fragment atoms with + * lower priorities, and vice versa. One exception is that there is only one + * priority value for each context. So a high-priority (e.g.) fragment atom + * could increase its context priority, causing its non-fragment atoms to also + * be scheduled sooner. + * + * The atoms are scheduled as follows with respect to their priorities: + * * Let atoms 'X' and 'Y' be for the same job slot who have dependencies + * resolved, and atom 'X' has a higher priority than atom 'Y' + * * If atom 'Y' is currently running on the HW, then it is interrupted to + * allow atom 'X' to run soon after + * * If instead neither atom 'Y' nor atom 'X' are running, then when choosing + * the next atom to run, atom 'X' will always be chosen instead of atom 'Y' + * * Any two atoms that have the same priority could run in any order with + * respect to each other. That is, there is no ordering constraint between + * atoms of the same priority. + * + * The sysfs file 'js_ctx_scheduling_mode' is used to control how atoms are + * scheduled between contexts. The default value, 0, will cause higher-priority + * atoms to be scheduled first, regardless of their context. The value 1 will + * use a round-robin algorithm when deciding which context's atoms to schedule + * next, so higher-priority atoms can only preempt lower priority atoms within + * the same context. See KBASE_JS_SYSTEM_PRIORITY_MODE and + * KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE for more details. + */ +typedef u8 base_jd_prio; + +/* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */ +#define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0) +/* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and + * BASE_JD_PRIO_LOW + */ +#define BASE_JD_PRIO_HIGH ((base_jd_prio)1) +/* Low atom priority. */ +#define BASE_JD_PRIO_LOW ((base_jd_prio)2) + +/* Count of the number of priority levels. This itself is not a valid + * base_jd_prio setting + */ +#define BASE_JD_NR_PRIO_LEVELS 3 + +/** + * struct base_jd_atom_v2 - Node of a dependency graph used to submit a + * GPU job chain or soft-job to the kernel driver. + * + * @jc: GPU address of a job chain or (if BASE_JD_REQ_END_RENDERPASS + * is set in the base_jd_core_req) the CPU address of a + * base_jd_fragment object. + * @udata: User data. + * @extres_list: List of external resources. + * @nr_extres: Number of external resources or JIT allocations. + * @jit_id: Zero-terminated array of IDs of just-in-time memory + * allocations written to by the atom. When the atom + * completes, the value stored at the + * &struct_base_jit_alloc_info.heap_info_gpu_addr of + * each allocation is read in order to enforce an + * overall physical memory usage limit. + * @pre_dep: Pre-dependencies. One need to use SETTER function to assign + * this field; this is done in order to reduce possibility of + * improper assignment of a dependency field. + * @atom_number: Unique number to identify the atom. + * @prio: Atom priority. Refer to base_jd_prio for more details. + * @device_nr: Core group when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP + * specified. + * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. + * @core_req: Core requirements. + * @renderpass_id: Renderpass identifier used to associate an atom that has + * BASE_JD_REQ_START_RENDERPASS set in its core requirements + * with an atom that has BASE_JD_REQ_END_RENDERPASS set. + * @padding: Unused. Must be zero. + * + * This structure has changed since UK 10.2 for which base_jd_core_req was a + * u16 value. + * + * In UK 10.3 a core_req field of a u32 type was added to the end of the + * structure, and the place in the structure previously occupied by u16 + * core_req was kept but renamed to compat_core_req. + * + * From UK 11.20 - compat_core_req is now occupied by u8 jit_id[2]. + * Compatibility with UK 10.x from UK 11.y is not handled because + * the major version increase prevents this. + * + * For UK 11.20 jit_id[2] must be initialized to zero. + */ +struct base_jd_atom_v2 { + u64 jc; + struct base_jd_udata udata; + u64 extres_list; + u16 nr_extres; + u8 jit_id[2]; + struct base_dependency pre_dep[2]; + base_atom_id atom_number; + base_jd_prio prio; + u8 device_nr; + u8 jobslot; + base_jd_core_req core_req; + u8 renderpass_id; + u8 padding[7]; +}; + +/** + * struct base_jd_atom - Same as base_jd_atom_v2, but has an extra seq_nr + * at the beginning. + * + * @seq_nr: Sequence number of logical grouping of atoms. + * @jc: GPU address of a job chain or (if BASE_JD_REQ_END_RENDERPASS + * is set in the base_jd_core_req) the CPU address of a + * base_jd_fragment object. + * @udata: User data. + * @extres_list: List of external resources. + * @nr_extres: Number of external resources or JIT allocations. + * @jit_id: Zero-terminated array of IDs of just-in-time memory + * allocations written to by the atom. When the atom + * completes, the value stored at the + * &struct_base_jit_alloc_info.heap_info_gpu_addr of + * each allocation is read in order to enforce an + * overall physical memory usage limit. + * @pre_dep: Pre-dependencies. One need to use SETTER function to assign + * this field; this is done in order to reduce possibility of + * improper assignment of a dependency field. + * @atom_number: Unique number to identify the atom. + * @prio: Atom priority. Refer to base_jd_prio for more details. + * @device_nr: Core group when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP + * specified. + * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. + * @core_req: Core requirements. + * @renderpass_id: Renderpass identifier used to associate an atom that has + * BASE_JD_REQ_START_RENDERPASS set in its core requirements + * with an atom that has BASE_JD_REQ_END_RENDERPASS set. + * @padding: Unused. Must be zero. + */ +typedef struct base_jd_atom { + u64 seq_nr; + u64 jc; + struct base_jd_udata udata; + u64 extres_list; + u16 nr_extres; + u8 jit_id[2]; + struct base_dependency pre_dep[2]; + base_atom_id atom_number; + base_jd_prio prio; + u8 device_nr; + u8 jobslot; + base_jd_core_req core_req; + u8 renderpass_id; + u8 padding[7]; +} base_jd_atom; + +/* Job chain event code bits + * Defines the bits used to create ::base_jd_event_code + */ +enum { + BASE_JD_SW_EVENT_KERNEL = (1u << 15), /* Kernel side event */ + BASE_JD_SW_EVENT = (1u << 14), /* SW defined event */ + /* Event indicates success (SW events only) */ + BASE_JD_SW_EVENT_SUCCESS = (1u << 13), + BASE_JD_SW_EVENT_JOB = (0u << 11), /* Job related event */ + BASE_JD_SW_EVENT_BAG = (1u << 11), /* Bag related event */ + BASE_JD_SW_EVENT_INFO = (2u << 11), /* Misc/info event */ + BASE_JD_SW_EVENT_RESERVED = (3u << 11), /* Reserved event type */ + /* Mask to extract the type from an event code */ + BASE_JD_SW_EVENT_TYPE_MASK = (3u << 11) +}; + +/** + * enum base_jd_event_code - Job chain event codes + * + * @BASE_JD_EVENT_RANGE_HW_NONFAULT_START: Start of hardware non-fault status + * codes. + * Obscurely, BASE_JD_EVENT_TERMINATED + * indicates a real fault, because the + * job was hard-stopped. + * @BASE_JD_EVENT_NOT_STARTED: Can't be seen by userspace, treated as + * 'previous job done'. + * @BASE_JD_EVENT_STOPPED: Can't be seen by userspace, becomes + * TERMINATED, DONE or JOB_CANCELLED. + * @BASE_JD_EVENT_TERMINATED: This is actually a fault status code - the job + * was hard stopped. + * @BASE_JD_EVENT_ACTIVE: Can't be seen by userspace, jobs only returned on + * complete/fail/cancel. + * @BASE_JD_EVENT_RANGE_HW_NONFAULT_END: End of hardware non-fault status codes. + * Obscurely, BASE_JD_EVENT_TERMINATED + * indicates a real fault, + * because the job was hard-stopped. + * @BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START: Start of hardware fault and + * software error status codes. + * @BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END: End of hardware fault and + * software error status codes. + * @BASE_JD_EVENT_RANGE_SW_SUCCESS_START: Start of software success status + * codes. + * @BASE_JD_EVENT_RANGE_SW_SUCCESS_END: End of software success status codes. + * @BASE_JD_EVENT_RANGE_KERNEL_ONLY_START: Start of kernel-only status codes. + * Such codes are never returned to + * user-space. + * @BASE_JD_EVENT_RANGE_KERNEL_ONLY_END: End of kernel-only status codes. + * + * HW and low-level SW events are represented by event codes. + * The status of jobs which succeeded are also represented by + * an event code (see @BASE_JD_EVENT_DONE). + * Events are usually reported as part of a &struct base_jd_event. + * + * The event codes are encoded in the following way: + * * 10:0 - subtype + * * 12:11 - type + * * 13 - SW success (only valid if the SW bit is set) + * * 14 - SW event (HW event if not set) + * * 15 - Kernel event (should never be seen in userspace) + * + * Events are split up into ranges as follows: + * * BASE_JD_EVENT_RANGE__START + * * BASE_JD_EVENT_RANGE__END + * + * code is in 's range when: + * BASE_JD_EVENT_RANGE__START <= code < + * BASE_JD_EVENT_RANGE__END + * + * Ranges can be asserted for adjacency by testing that the END of the previous + * is equal to the START of the next. This is useful for optimizing some tests + * for range. + * + * A limitation is that the last member of this enum must explicitly be handled + * (with an assert-unreachable statement) in switch statements that use + * variables of this type. Otherwise, the compiler warns that we have not + * handled that enum value. + */ +enum base_jd_event_code { + /* HW defined exceptions */ + BASE_JD_EVENT_RANGE_HW_NONFAULT_START = 0, + + /* non-fatal exceptions */ + BASE_JD_EVENT_NOT_STARTED = 0x00, + BASE_JD_EVENT_DONE = 0x01, + BASE_JD_EVENT_STOPPED = 0x03, + BASE_JD_EVENT_TERMINATED = 0x04, + BASE_JD_EVENT_ACTIVE = 0x08, + + BASE_JD_EVENT_RANGE_HW_NONFAULT_END = 0x40, + BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START = 0x40, + + /* job exceptions */ + BASE_JD_EVENT_JOB_CONFIG_FAULT = 0x40, + BASE_JD_EVENT_JOB_POWER_FAULT = 0x41, + BASE_JD_EVENT_JOB_READ_FAULT = 0x42, + BASE_JD_EVENT_JOB_WRITE_FAULT = 0x43, + BASE_JD_EVENT_JOB_AFFINITY_FAULT = 0x44, + BASE_JD_EVENT_JOB_BUS_FAULT = 0x48, + BASE_JD_EVENT_INSTR_INVALID_PC = 0x50, + BASE_JD_EVENT_INSTR_INVALID_ENC = 0x51, + BASE_JD_EVENT_INSTR_TYPE_MISMATCH = 0x52, + BASE_JD_EVENT_INSTR_OPERAND_FAULT = 0x53, + BASE_JD_EVENT_INSTR_TLS_FAULT = 0x54, + BASE_JD_EVENT_INSTR_BARRIER_FAULT = 0x55, + BASE_JD_EVENT_INSTR_ALIGN_FAULT = 0x56, + BASE_JD_EVENT_DATA_INVALID_FAULT = 0x58, + BASE_JD_EVENT_TILE_RANGE_FAULT = 0x59, + BASE_JD_EVENT_STATE_FAULT = 0x5A, + BASE_JD_EVENT_OUT_OF_MEMORY = 0x60, + BASE_JD_EVENT_UNKNOWN = 0x7F, + + /* GPU exceptions */ + BASE_JD_EVENT_DELAYED_BUS_FAULT = 0x80, + BASE_JD_EVENT_SHAREABILITY_FAULT = 0x88, + + /* MMU exceptions */ + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1 = 0xC1, + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2 = 0xC2, + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3 = 0xC3, + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4 = 0xC4, + BASE_JD_EVENT_PERMISSION_FAULT = 0xC8, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1 = 0xD1, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2 = 0xD2, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3 = 0xD3, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4 = 0xD4, + BASE_JD_EVENT_ACCESS_FLAG = 0xD8, + + /* SW defined exceptions */ + BASE_JD_EVENT_MEM_GROWTH_FAILED = + BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x000, + BASE_JD_EVENT_TIMED_OUT = + BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x001, + BASE_JD_EVENT_JOB_CANCELLED = + BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x002, + BASE_JD_EVENT_JOB_INVALID = + BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x003, + BASE_JD_EVENT_PM_EVENT = + BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x004, + + BASE_JD_EVENT_BAG_INVALID = + BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_BAG | 0x003, + + BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END = BASE_JD_SW_EVENT | + BASE_JD_SW_EVENT_RESERVED | 0x3FF, + + BASE_JD_EVENT_RANGE_SW_SUCCESS_START = BASE_JD_SW_EVENT | + BASE_JD_SW_EVENT_SUCCESS | 0x000, + + BASE_JD_EVENT_PROGRESS_REPORT = BASE_JD_SW_EVENT | + BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_JOB | 0x000, + BASE_JD_EVENT_BAG_DONE = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | + BASE_JD_SW_EVENT_BAG | 0x000, + BASE_JD_EVENT_DRV_TERMINATED = BASE_JD_SW_EVENT | + BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_INFO | 0x000, + + BASE_JD_EVENT_RANGE_SW_SUCCESS_END = BASE_JD_SW_EVENT | + BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_RESERVED | 0x3FF, + + BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | + BASE_JD_SW_EVENT_KERNEL | 0x000, + BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | + BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x000, + BASE_JD_EVENT_END_RP_DONE = BASE_JD_SW_EVENT | + BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x001, + + BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | + BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_RESERVED | 0x3FF +}; + +/** + * struct base_jd_event_v2 - Event reporting structure + * + * @event_code: event code. + * @atom_number: the atom number that has completed. + * @udata: user data. + * + * This structure is used by the kernel driver to report information + * about GPU events. They can either be HW-specific events or low-level + * SW events, such as job-chain completion. + * + * The event code contains an event type field which can be extracted + * by ANDing with BASE_JD_SW_EVENT_TYPE_MASK. + */ +struct base_jd_event_v2 { + enum base_jd_event_code event_code; + base_atom_id atom_number; + struct base_jd_udata udata; +}; + +/** + * struct base_dump_cpu_gpu_counters - Structure for + * BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS + * jobs. + * + * This structure is stored into the memory pointed to by the @jc field + * of &struct base_jd_atom. + * + * It must not occupy the same CPU cache line(s) as any neighboring data. + * This is to avoid cases where access to pages containing the structure + * is shared between cached and un-cached memory regions, which would + * cause memory corruption. + */ + +struct base_dump_cpu_gpu_counters { + u64 system_time; + u64 cycle_counter; + u64 sec; + u32 usec; + u8 padding[36]; +}; + +#endif /* _BASE_JM_KERNEL_H_ */ diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h new file mode 100755 index 000000000000..4fb5d1d9c410 --- /dev/null +++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h @@ -0,0 +1,844 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Definitions (types, defines, etcs) specific to Job Manager Kbase. + * They are placed here to allow the hierarchy of header files to work. + */ + +#ifndef _KBASE_JM_DEFS_H_ +#define _KBASE_JM_DEFS_H_ + +#include "mali_kbase_js_defs.h" + +/* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */ +#define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1 + +/* + * Number of milliseconds before resetting the GPU when a job cannot be "zapped" + * from the hardware. Note that the time is actually + * ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and + * the GPU actually being reset to give other contexts time for their jobs + * to be soft-stopped and removed from the hardware before resetting. + */ +#define ZAP_TIMEOUT 1000 + +/* + * Prevent soft-stops from occurring in scheduling situations + * + * This is not due to HW issues, but when scheduling is desired to be more + * predictable. + * + * Therefore, soft stop may still be disabled due to HW issues. + * + * Soft stop will still be used for non-scheduling purposes e.g. when + * terminating a context. + * + * if not in use, define this value to 0 instead of being undefined. + */ +#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0 + +/* + * Prevent hard-stops from occurring in scheduling situations + * + * This is not due to HW issues, but when scheduling is desired to be more + * predictable. + * + * Hard stop will still be used for non-scheduling purposes e.g. when + * terminating a context. + * + * if not in use, define this value to 0 instead of being undefined. + */ +#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 + +/* Atom has been previously soft-stopped */ +#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1<<1) +/* Atom has been previously retried to execute */ +#define KBASE_KATOM_FLAGS_RERUN (1<<2) +/* Atom submitted with JOB_CHAIN_FLAG bit set in JS_CONFIG_NEXT register, helps + * to disambiguate short-running job chains during soft/hard stopping of jobs + */ +#define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3) +/* Atom has been previously hard-stopped. */ +#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4) +/* Atom has caused us to enter disjoint state */ +#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5) +/* Atom blocked on cross-slot dependency */ +#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7) +/* Atom has fail dependency on cross-slot dependency */ +#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8) +/* Atom is currently in the list of atoms blocked on cross-slot dependencies */ +#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9) +/* Atom is currently holding a context reference */ +#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10) +/* Atom requires GPU to be in protected mode */ +#define KBASE_KATOM_FLAG_PROTECTED (1<<11) +/* Atom has been stored in runnable_tree */ +#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12) +/* Atom is waiting for L2 caches to power up in order to enter protected mode */ +#define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1<<13) + +/* SW related flags about types of JS_COMMAND action + * NOTE: These must be masked off by JS_COMMAND_MASK + */ + +/* This command causes a disjoint event */ +#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100 + +/* Bitmask of all SW related flags */ +#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT) + +#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK) +#error "JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK." \ + "Must update JS_COMMAND_SW_<..> bitmasks" +#endif + +/* Soft-stop command that causes a Disjoint event. This of course isn't + * entirely masked off by JS_COMMAND_MASK + */ +#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \ + (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP) + +#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT + +/* Serialize atoms within a slot (ie only one atom per job slot) */ +#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0) +/* Serialize atoms between slots (ie only one job slot running at any time) */ +#define KBASE_SERIALIZE_INTER_SLOT (1 << 1) +/* Reset the GPU after each atom completion */ +#define KBASE_SERIALIZE_RESET (1 << 2) + +#ifdef CONFIG_DEBUG_FS +/** + * struct base_job_fault_event - keeps track of the atom which faulted or which + * completed after the faulty atom but before the + * debug data for faulty atom was dumped. + * + * @event_code: event code for the atom, should != BASE_JD_EVENT_DONE for + * the atom which faulted. + * @katom: pointer to the atom for which job fault occurred or which + * completed after the faulty atom. + * @job_fault_work: work item, queued only for the faulty atom, which waits for + * the dumping to get completed and then does the bottom half + * of job done for the atoms which followed the faulty atom. + * @head: List head used to store the atom in the global list of + * faulty atoms or context specific list of atoms which got + * completed during the dump. + * @reg_offset: offset of the register to be dumped next, only applicable + * for the faulty atom. + */ +struct base_job_fault_event { + + u32 event_code; + struct kbase_jd_atom *katom; + struct work_struct job_fault_work; + struct list_head head; + int reg_offset; +}; +#endif + +/** + * struct kbase_jd_atom_dependency - Contains the dependency info for an atom. + * @atom: pointer to the dependee atom. + * @dep_type: type of dependency on the dependee @atom, i.e. order or data + * dependency. BASE_JD_DEP_TYPE_INVALID indicates no dependency. + */ +struct kbase_jd_atom_dependency { + struct kbase_jd_atom *atom; + u8 dep_type; +}; + +/** + * kbase_jd_katom_dep_atom - Retrieves a read-only reference to the + * dependee atom. + * @dep: pointer to the dependency info structure. + * + * Return: readonly reference to dependee atom. + */ +static inline const struct kbase_jd_atom * +kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep) +{ + LOCAL_ASSERT(dep != NULL); + + return (const struct kbase_jd_atom *)(dep->atom); +} + +/** + * kbase_jd_katom_dep_type - Retrieves the dependency type info + * + * @dep: pointer to the dependency info structure. + * + * Return: the type of dependency there is on the dependee atom. + */ +static inline u8 kbase_jd_katom_dep_type( + const struct kbase_jd_atom_dependency *dep) +{ + LOCAL_ASSERT(dep != NULL); + + return dep->dep_type; +} + +/** + * kbase_jd_katom_dep_set - sets up the dependency info structure + * as per the values passed. + * @const_dep: pointer to the dependency info structure to be setup. + * @a: pointer to the dependee atom. + * @type: type of dependency there is on the dependee atom. + */ +static inline void kbase_jd_katom_dep_set( + const struct kbase_jd_atom_dependency *const_dep, + struct kbase_jd_atom *a, u8 type) +{ + struct kbase_jd_atom_dependency *dep; + + LOCAL_ASSERT(const_dep != NULL); + + dep = (struct kbase_jd_atom_dependency *)const_dep; + + dep->atom = a; + dep->dep_type = type; +} + +/** + * kbase_jd_katom_dep_clear - resets the dependency info structure + * + * @const_dep: pointer to the dependency info structure to be setup. + */ +static inline void kbase_jd_katom_dep_clear( + const struct kbase_jd_atom_dependency *const_dep) +{ + struct kbase_jd_atom_dependency *dep; + + LOCAL_ASSERT(const_dep != NULL); + + dep = (struct kbase_jd_atom_dependency *)const_dep; + + dep->atom = NULL; + dep->dep_type = BASE_JD_DEP_TYPE_INVALID; +} + +/** + * enum kbase_atom_gpu_rb_state - The state of an atom, pertinent after it + * becomes runnable, with respect to job slot + * ringbuffer/fifo. + * @KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: Atom not currently present in slot fifo, + * which implies that either atom has not become + * runnable due to dependency or has completed + * the execution on GPU. + * @KBASE_ATOM_GPU_RB_WAITING_BLOCKED: Atom has been added to slot fifo but is + * blocked due to cross slot dependency, + * can't be submitted to GPU. + * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: Atom has been added to slot + * fifo but is waiting for the completion of + * previously added atoms in current & other + * slots, as their protected mode requirements + * do not match with the current atom. + * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: Atom is in slot fifo + * and is waiting for completion of protected + * mode transition, needed before the atom is + * submitted to GPU. + * @KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: Atom is in slot fifo but is + * waiting for the cores, which are needed to + * execute the job chain represented by the atom, + * to become available + * @KBASE_ATOM_GPU_RB_READY: Atom is in slot fifo and can be submitted to + * GPU. + * @KBASE_ATOM_GPU_RB_SUBMITTED: Atom is in slot fifo and has been submitted + * to GPU. + * @KBASE_ATOM_GPU_RB_RETURN_TO_JS: Atom must be returned to JS due to some + * failure, but only after the previously added + * atoms in fifo have completed or have also + * been returned to JS. + */ +enum kbase_atom_gpu_rb_state { + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB, + KBASE_ATOM_GPU_RB_WAITING_BLOCKED, + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV, + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION, + KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE, + KBASE_ATOM_GPU_RB_READY, + KBASE_ATOM_GPU_RB_SUBMITTED, + KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1 +}; + +/** + * enum kbase_atom_enter_protected_state - The state of an atom with respect to + * the preparation for GPU's entry into protected mode, + * becomes pertinent only after atom's state with respect + * to slot ringbuffer is + * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION + * @KBASE_ATOM_ENTER_PROTECTED_CHECK: Starting state. Check if there are any + * atoms currently submitted to GPU and protected mode + * transition is not already in progress. + * @KBASE_ATOM_ENTER_PROTECTED_HWCNT: Wait for hardware counter context to + * become disabled before entry into protected mode. + * @KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: Wait for the L2 to become idle in + * preparation for the coherency change. L2 shall be + * powered down and GPU shall come out of fully + * coherent mode before entering protected mode. + * @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change; + * for BASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on + * so that coherency register contains correct value when + * GPU enters protected mode. + * @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for + * BASE_HW_ISSUE_TGOX_R1_1234 check + * that L2 is powered up and switch GPU to protected mode. + */ +enum kbase_atom_enter_protected_state { + /* + * NOTE: The integer value of this must match + * KBASE_ATOM_EXIT_PROTECTED_CHECK. + */ + KBASE_ATOM_ENTER_PROTECTED_CHECK = 0, + KBASE_ATOM_ENTER_PROTECTED_HWCNT, + KBASE_ATOM_ENTER_PROTECTED_IDLE_L2, + KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY, + KBASE_ATOM_ENTER_PROTECTED_FINISHED, +}; + +/** + * enum kbase_atom_exit_protected_state - The state of an atom with respect to + * the preparation for GPU's exit from protected mode, + * becomes pertinent only after atom's state with respect + * to slot ngbuffer is + * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION + * @KBASE_ATOM_EXIT_PROTECTED_CHECK: Starting state. Check if there are any + * atoms currently submitted to GPU and protected mode + * transition is not already in progress. + * @KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: Wait for the L2 to become idle in + * preparation for the reset, as exiting protected mode + * requires a reset. + * @KBASE_ATOM_EXIT_PROTECTED_RESET: Issue the reset to trigger exit from + * protected mode + * @KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: End state, Wait for the reset to + * complete + */ +enum kbase_atom_exit_protected_state { + /* + * NOTE: The integer value of this must match + * KBASE_ATOM_ENTER_PROTECTED_CHECK. + */ + KBASE_ATOM_EXIT_PROTECTED_CHECK = 0, + KBASE_ATOM_EXIT_PROTECTED_IDLE_L2, + KBASE_ATOM_EXIT_PROTECTED_RESET, + KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT, +}; + +/** + * struct kbase_ext_res - Contains the info for external resources referred + * by an atom, which have been mapped on GPU side. + * @gpu_address: Start address of the memory region allocated for + * the resource from GPU virtual address space. + * @alloc: pointer to physical pages tracking object, set on + * mapping the external resource on GPU side. + */ +struct kbase_ext_res { + u64 gpu_address; + struct kbase_mem_phy_alloc *alloc; +}; + +/** + * struct kbase_jd_atom - object representing the atom, containing the complete + * state and attributes of an atom. + * @work: work item for the bottom half processing of the atom, + * by JD or JS, after it got executed on GPU or the + * input fence got signaled + * @start_timestamp: time at which the atom was submitted to the GPU, by + * updating the JS_HEAD_NEXTn register. + * @udata: copy of the user data sent for the atom in + * base_jd_submit. + * @kctx: Pointer to the base context with which the atom is + * associated. + * @dep_head: Array of 2 list heads, pointing to the two list of + * atoms + * which are blocked due to dependency on this atom. + * @dep_item: Array of 2 list heads, used to store the atom in the + * list of other atoms depending on the same dependee + * atom. + * @dep: Array containing the dependency info for the 2 atoms + * on which the atom depends upon. + * @jd_item: List head used during job dispatch job_done + * processing - as dependencies may not be entirely + * resolved at this point, + * we need to use a separate list head. + * @in_jd_list: flag set to true if atom's @jd_item is currently on + * a list, prevents atom being processed twice. + * @jit_ids: Zero-terminated array of IDs of just-in-time memory + * allocations written to by the atom. When the atom + * completes, the value stored at the + * &struct_base_jit_alloc_info.heap_info_gpu_addr of + * each allocation is read in order to enforce an + * overall physical memory usage limit. + * @nr_extres: number of external resources referenced by the atom. + * @extres: pointer to the location containing info about + * @nr_extres external resources referenced by the atom. + * @device_nr: indicates the coregroup with which the atom is + * associated, when + * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified. + * @jc: GPU address of the job-chain. + * @softjob_data: Copy of data read from the user space buffer that @jc + * points to. + * @fence: Stores either an input or output sync fence, + * depending on soft-job type + * @sync_waiter: Pointer to the sync fence waiter structure passed to + * the callback function on signaling of the input + * fence. + * @dma_fence: object containing pointers to both input & output + * fences and other related members used for explicit + * sync through soft jobs and for the implicit + * synchronization required on access to external + * resources. + * @event_code: Event code for the job chain represented by the atom, + * both HW and low-level SW events are represented by + * event codes. + * @core_req: bitmask of BASE_JD_REQ_* flags specifying either + * Hw or Sw requirements for the job chain represented + * by the atom. + * @ticks: Number of scheduling ticks for which atom has been + * running on the GPU. + * @sched_priority: Priority of the atom for Job scheduling, as per the + * KBASE_JS_ATOM_SCHED_PRIO_*. + * @completed: Wait queue to wait upon for the completion of atom. + * @status: Indicates at high level at what stage the atom is in, + * as per KBASE_JD_ATOM_STATE_*, that whether it is not + * in use or its queued in JD or given to JS or + * submitted to Hw or it completed the execution on Hw. + * @work_id: used for GPU tracepoints, its a snapshot of the + * 'work_id' counter in kbase_jd_context which is + * incremented on every call to base_jd_submit. + * @slot_nr: Job slot chosen for the atom. + * @atom_flags: bitmask of KBASE_KATOM_FLAG* flags capturing the + * excat low level state of the atom. + * @gpu_rb_state: bitmnask of KBASE_ATOM_GPU_RB_* flags, precisely + * tracking atom's state after it has entered + * Job scheduler on becoming runnable. Atom + * could be blocked due to cross slot dependency + * or waiting for the shader cores to become available + * or waiting for protected mode transitions to + * complete. + * @need_cache_flush_cores_retained: flag indicating that manual flush of GPU + * cache is needed for the atom and the shader cores + * used for atom have been kept on. + * @blocked: flag indicating that atom's resubmission to GPU is + * blocked till the work item is scheduled to return the + * atom to JS. + * @pre_dep: Pointer to atom that this atom has same-slot + * dependency on + * @post_dep: Pointer to atom that has same-slot dependency on + * this atom + * @x_pre_dep: Pointer to atom that this atom has cross-slot + * dependency on + * @x_post_dep: Pointer to atom that has cross-slot dependency on + * this atom + * @flush_id: The GPU's flush count recorded at the time of + * submission, + * used for the cache flush optimization + * @fault_event: Info for dumping the debug data on Job fault. + * @queue: List head used for 4 different purposes : + * Adds atom to the list of dma-buf fence waiting atoms. + * Adds atom to the list of atoms blocked due to cross + * slot dependency. + * Adds atom to the list of softjob atoms for which JIT + * allocation has been deferred + * Adds atom to the list of softjob atoms waiting for + * the signaling of fence. + * @jit_node: Used to keep track of all JIT free/alloc jobs in + * submission order + * @jit_blocked: Flag indicating that JIT allocation requested through + * softjob atom will be reattempted after the impending + * free of other active JIT allocations. + * @will_fail_event_code: If non-zero, this indicates that the atom will fail + * with the set event_code when the atom is processed. + * Used for special handling of atoms, which have a data + * dependency on the failed atoms. + * @protected_state: State of the atom, as per + * KBASE_ATOM_(ENTER|EXIT)_PROTECTED_*, + * when transitioning into or out of protected mode. + * Atom will be either entering or exiting the + * protected mode. + * @runnable_tree_node: The node added to context's job slot specific rb tree + * when the atom becomes runnable. + * @age: Age of atom relative to other atoms in the context, + * is snapshot of the age_count counter in kbase + * context. + */ +struct kbase_jd_atom { + struct work_struct work; + ktime_t start_timestamp; + + struct base_jd_udata udata; + struct kbase_context *kctx; + + struct list_head dep_head[2]; + struct list_head dep_item[2]; + const struct kbase_jd_atom_dependency dep[2]; + struct list_head jd_item; + bool in_jd_list; + +#if MALI_JIT_PRESSURE_LIMIT_BASE + u8 jit_ids[2]; +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + u16 nr_extres; + struct kbase_ext_res *extres; + + u32 device_nr; + u64 jc; + void *softjob_data; +#if defined(CONFIG_SYNC) + struct sync_fence *fence; + struct sync_fence_waiter sync_waiter; +#endif /* CONFIG_SYNC */ +#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + struct { + /* Use the functions/API defined in mali_kbase_fence.h to + * when working with this sub struct + */ +#if defined(CONFIG_SYNC_FILE) + /* Input fence */ +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) + struct fence *fence_in; +#else + struct dma_fence *fence_in; +#endif +#endif + /* This points to the dma-buf output fence for this atom. If + * this is NULL then there is no fence for this atom and the + * following fields related to dma_fence may have invalid data. + * + * The context and seqno fields contain the details for this + * fence. + * + * This fence is signaled when the katom is completed, + * regardless of the event_code of the katom (signal also on + * failure). + */ +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + /* The dma-buf fence context number for this atom. A unique + * context number is allocated to each katom in the context on + * context creation. + */ + unsigned int context; + /* The dma-buf fence sequence number for this atom. This is + * increased every time this katom uses dma-buf fence. + */ + atomic_t seqno; + /* This contains a list of all callbacks set up to wait on + * other fences. This atom must be held back from JS until all + * these callbacks have been called and dep_count have reached + * 0. The initial value of dep_count must be equal to the + * number of callbacks on this list. + * + * This list is protected by jctx.lock. Callbacks are added to + * this list when the atom is built and the wait are set up. + * All the callbacks then stay on the list until all callbacks + * have been called and the atom is queued, or cancelled, and + * then all callbacks are taken off the list and freed. + */ + struct list_head callbacks; + /* Atomic counter of number of outstandind dma-buf fence + * dependencies for this atom. When dep_count reaches 0 the + * atom may be queued. + * + * The special value "-1" may only be set after the count + * reaches 0, while holding jctx.lock. This indicates that the + * atom has been handled, either queued in JS or cancelled. + * + * If anyone but the dma-fence worker sets this to -1 they must + * ensure that any potentially queued worker must have + * completed before allowing the atom to be marked as unused. + * This can be done by flushing the fence work queue: + * kctx->dma_fence.wq. + */ + atomic_t dep_count; + } dma_fence; +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || CONFIG_SYNC_FILE */ + + /* Note: refer to kbasep_js_atom_retained_state, which will take a copy + * of some of the following members + */ + enum base_jd_event_code event_code; + base_jd_core_req core_req; + u8 jobslot; + u8 renderpass_id; + struct base_jd_fragment jc_fragment; + + u32 ticks; + int sched_priority; + + wait_queue_head_t completed; + enum kbase_jd_atom_state status; +#ifdef CONFIG_GPU_TRACEPOINTS + int work_id; +#endif + int slot_nr; + + u32 atom_flags; + + int retry_count; + + enum kbase_atom_gpu_rb_state gpu_rb_state; + + bool need_cache_flush_cores_retained; + + atomic_t blocked; + + /* user-space sequence number, to order atoms in some temporal order */ + u64 seq_nr; + + struct kbase_jd_atom *pre_dep; + struct kbase_jd_atom *post_dep; + + struct kbase_jd_atom *x_pre_dep; + struct kbase_jd_atom *x_post_dep; + + u32 flush_id; + +#ifdef CONFIG_DEBUG_FS + struct base_job_fault_event fault_event; +#endif + struct list_head queue; + + struct list_head jit_node; + bool jit_blocked; + + enum base_jd_event_code will_fail_event_code; + + union { + enum kbase_atom_enter_protected_state enter; + enum kbase_atom_exit_protected_state exit; + } protected_state; + + struct rb_node runnable_tree_node; + + u32 age; +}; + +static inline bool kbase_jd_katom_is_protected( + const struct kbase_jd_atom *katom) +{ + return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED); +} + +/* + * Theory of operations: + * + * Atom objects are statically allocated within the context structure. + * + * Each atom is the head of two lists, one for the "left" set of dependencies, + * one for the "right" set. + */ + +#define KBASE_JD_DEP_QUEUE_SIZE 256 + +/** + * enum kbase_jd_renderpass_state - State of a renderpass + * @KBASE_JD_RP_COMPLETE: Unused or completed renderpass. Can only transition to + * START. + * @KBASE_JD_RP_START: Renderpass making a first attempt at tiling. + * Can transition to PEND_OOM or COMPLETE. + * @KBASE_JD_RP_PEND_OOM: Renderpass whose first attempt at tiling used too much + * memory and has a soft-stop pending. Can transition to + * OOM or COMPLETE. + * @KBASE_JD_RP_OOM: Renderpass whose first attempt at tiling used too much + * memory and therefore switched to incremental + * rendering. The fragment job chain is forced to run. + * Can only transition to RETRY. + * @KBASE_JD_RP_RETRY: Renderpass making a second or subsequent attempt at + * tiling. Can transition to RETRY_PEND_OOM or COMPLETE. + * @KBASE_JD_RP_RETRY_PEND_OOM: Renderpass whose second or subsequent attempt at + * tiling used too much memory again and has a + * soft-stop pending. Can transition to RETRY_OOM + * or COMPLETE. + * @KBASE_JD_RP_RETRY_OOM: Renderpass whose second or subsequent attempt at + * tiling used too much memory again. The fragment job + * chain is forced to run. Can only transition to RETRY. + * + * A state machine is used to control incremental rendering. + */ +enum kbase_jd_renderpass_state { + KBASE_JD_RP_COMPLETE, /* COMPLETE => START */ + KBASE_JD_RP_START, /* START => PEND_OOM or COMPLETE */ + KBASE_JD_RP_PEND_OOM, /* PEND_OOM => OOM or COMPLETE */ + KBASE_JD_RP_OOM, /* OOM => RETRY */ + KBASE_JD_RP_RETRY, /* RETRY => RETRY_PEND_OOM or + * COMPLETE + */ + KBASE_JD_RP_RETRY_PEND_OOM, /* RETRY_PEND_OOM => RETRY_OOM or + * COMPLETE + */ + KBASE_JD_RP_RETRY_OOM, /* RETRY_OOM => RETRY */ +}; + +/** + * struct kbase_jd_renderpass - Data for a renderpass + * @state: Current state of the renderpass. If KBASE_JD_RP_COMPLETE then + * all other members are invalid. + * Both the job dispatcher context and hwaccess_lock must be + * locked to modify this so that it can be read with either + * (or both) locked. + * @start_katom: Address of the atom that is the start of a renderpass. + * Both the job dispatcher context and hwaccess_lock must be + * locked to modify this so that it can be read with either + * (or both) locked. + * @end_katom: Address of the atom that is the end of a renderpass, or NULL + * if that atom hasn't been added to the job scheduler yet. + * The job dispatcher context and hwaccess_lock must be + * locked to modify this so that it can be read with either + * (or both) locked. + * @oom_reg_list: A list of region structures which triggered out-of-memory. + * The hwaccess_lock must be locked to access this. + * + * Atoms tagged with BASE_JD_REQ_START_RENDERPASS or BASE_JD_REQ_END_RENDERPASS + * are associated with an object of this type, which is created and maintained + * by kbase to keep track of each renderpass. + */ +struct kbase_jd_renderpass { + enum kbase_jd_renderpass_state state; + struct kbase_jd_atom *start_katom; + struct kbase_jd_atom *end_katom; + struct list_head oom_reg_list; +}; + +/** + * struct kbase_jd_context - per context object encapsulating all the + * Job dispatcher related state. + * @lock: lock to serialize the updates made to the + * Job dispatcher state and kbase_jd_atom objects. + * @sched_info: Structure encapsulating all the Job scheduling + * info. + * @atoms: Array of the objects representing atoms, + * containing the complete state and attributes + * of an atom. + * @renderpasses: Array of renderpass state for incremental + * rendering, indexed by user-specified renderpass + * ID. + * @job_nr: Tracks the number of atoms being processed by the + * kbase. This includes atoms that are not tracked by + * scheduler: 'not ready to run' & 'dependency-only' + * jobs. + * @zero_jobs_wait: Waitq that reflects whether there are no jobs + * (including SW-only dependency jobs). This is set + * when no jobs are present on the ctx, and clear + * when there are jobs. + * This must be updated atomically with @job_nr. + * note: Job Dispatcher knows about more jobs than + * the Job Scheduler as it is unaware of jobs that + * are blocked on dependencies and SW-only dependency + * jobs. This waitq can be waited upon to find out + * when the context jobs are all done/cancelled + * (including those that might've been blocked + * on dependencies) - and so, whether it can be + * terminated. However, it should only be terminated + * once it is not present in the run-pool. + * Since the waitq is only set under @lock, + * the waiter should also briefly obtain and drop + * @lock to guarantee that the setter has completed + * its work on the kbase_context + * @job_done_wq: Workqueue to which the per atom work item is + * queued for bottom half processing when the + * atom completes + * execution on GPU or the input fence get signaled. + * @tb_lock: Lock to serialize the write access made to @tb to + * to store the register access trace messages. + * @tb: Pointer to the Userspace accessible buffer storing + * the trace messages for register read/write + * accesses made by the Kbase. The buffer is filled + * in circular fashion. + * @tb_wrap_offset: Offset to the end location in the trace buffer, + * the write pointer is moved to the beginning on + * reaching this offset. + * @work_id: atomic variable used for GPU tracepoints, + * incremented on every call to base_jd_submit. + * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both + * allocate & free, in submission order, protected + * by kbase_jd_context.lock. + * @jit_pending_alloc: A list of just-in-time memory allocation + * soft-jobs which will be reattempted after the + * impending free of other active allocations. + */ +struct kbase_jd_context { + struct mutex lock; + struct kbasep_js_kctx_info sched_info; + struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; + struct kbase_jd_renderpass renderpasses[BASE_JD_RP_COUNT]; + struct workqueue_struct *job_done_wq; + + wait_queue_head_t zero_jobs_wait; + spinlock_t tb_lock; + u32 *tb; + u32 job_nr; + size_t tb_wrap_offset; + +#ifdef CONFIG_GPU_TRACEPOINTS + atomic_t work_id; +#endif + + struct list_head jit_atoms_head; + struct list_head jit_pending_alloc; +}; + +/** + * struct jsctx_queue - JS context atom queue + * @runnable_tree: Root of RB-tree containing currently runnable atoms on this + * job slot. + * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot + * dependencies. Atoms on this list will be moved to the + * runnable_tree when the blocking atom completes. + * + * hwaccess_lock must be held when accessing this structure. + */ +struct jsctx_queue { + struct rb_root runnable_tree; + struct list_head x_dep_head; +}; + +/** + * struct kbase_as - Object representing an address space of GPU. + * @number: Index at which this address space structure is present + * in an array of address space structures embedded inside + * the &struct kbase_device. + * @pf_wq: Workqueue for processing work items related to + * Page fault and Bus fault handling. + * @work_pagefault: Work item for the Page fault handling. + * @work_busfault: Work item for the Bus fault handling. + * @pf_data: Data relating to Page fault. + * @bf_data: Data relating to Bus fault. + * @current_setup: Stores the MMU configuration for this address space. + */ +struct kbase_as { + int number; + struct workqueue_struct *pf_wq; + struct work_struct work_pagefault; + struct work_struct work_busfault; + struct kbase_fault pf_data; + struct kbase_fault bf_data; + struct kbase_mmu_setup current_setup; +}; + +#endif /* _KBASE_JM_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h new file mode 100755 index 000000000000..305a9eb221ae --- /dev/null +++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h @@ -0,0 +1,216 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_JM_IOCTL_H_ +#define _KBASE_JM_IOCTL_H_ + +#include +#include + +/* + * 11.1: + * - Add BASE_MEM_TILER_ALIGN_TOP under base_mem_alloc_flags + * 11.2: + * - KBASE_MEM_QUERY_FLAGS can return KBASE_REG_PF_GROW and KBASE_REG_PROTECTED, + * which some user-side clients prior to 11.2 might fault if they received + * them + * 11.3: + * - New ioctls KBASE_IOCTL_STICKY_RESOURCE_MAP and + * KBASE_IOCTL_STICKY_RESOURCE_UNMAP + * 11.4: + * - New ioctl KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET + * 11.5: + * - New ioctl: KBASE_IOCTL_MEM_JIT_INIT (old ioctl renamed to _OLD) + * 11.6: + * - Added flags field to base_jit_alloc_info structure, which can be used to + * specify pseudo chunked tiler alignment for JIT allocations. + * 11.7: + * - Removed UMP support + * 11.8: + * - Added BASE_MEM_UNCACHED_GPU under base_mem_alloc_flags + * 11.9: + * - Added BASE_MEM_PERMANENT_KERNEL_MAPPING and BASE_MEM_FLAGS_KERNEL_ONLY + * under base_mem_alloc_flags + * 11.10: + * - Enabled the use of nr_extres field of base_jd_atom_v2 structure for + * JIT_ALLOC and JIT_FREE type softjobs to enable multiple JIT allocations + * with one softjob. + * 11.11: + * - Added BASE_MEM_GPU_VA_SAME_4GB_PAGE under base_mem_alloc_flags + * 11.12: + * - Removed ioctl: KBASE_IOCTL_GET_PROFILING_CONTROLS + * 11.13: + * - New ioctl: KBASE_IOCTL_MEM_EXEC_INIT + * 11.14: + * - Add BASE_MEM_GROUP_ID_MASK, base_mem_group_id_get, base_mem_group_id_set + * under base_mem_alloc_flags + * 11.15: + * - Added BASEP_CONTEXT_MMU_GROUP_ID_MASK under base_context_create_flags. + * - Require KBASE_IOCTL_SET_FLAGS before BASE_MEM_MAP_TRACKING_HANDLE can be + * passed to mmap(). + * 11.16: + * - Extended ioctl KBASE_IOCTL_MEM_SYNC to accept imported dma-buf. + * - Modified (backwards compatible) ioctl KBASE_IOCTL_MEM_IMPORT behavior for + * dma-buf. Now, buffers are mapped on GPU when first imported, no longer + * requiring external resource or sticky resource tracking. UNLESS, + * CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND is enabled. + * 11.17: + * - Added BASE_JD_REQ_JOB_SLOT. + * - Reused padding field in base_jd_atom_v2 to pass job slot number. + * - New ioctl: KBASE_IOCTL_GET_CPU_GPU_TIMEINFO + * 11.18: + * - Added BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP under base_mem_alloc_flags + * 11.19: + * - Extended base_jd_atom_v2 to allow a renderpass ID to be specified. + * 11.20: + * - Added new phys_pages member to kbase_ioctl_mem_jit_init for + * KBASE_IOCTL_MEM_JIT_INIT, previous variants of this renamed to use _10_2 + * (replacing '_OLD') and _11_5 suffixes + * - Replaced compat_core_req (deprecated in 10.3) with jit_id[2] in + * base_jd_atom_v2. It must currently be initialized to zero. + * - Added heap_info_gpu_addr to base_jit_alloc_info, and + * BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE allowable in base_jit_alloc_info's + * flags member. Previous variants of this structure are kept and given _10_2 + * and _11_5 suffixes. + * - The above changes are checked for safe values in usual builds + * 11.21: + * - v2.0 of mali_trace debugfs file, which now versions the file separately + * 11.22: + * - Added base_jd_atom (v3), which is seq_nr + base_jd_atom_v2. + * KBASE_IOCTL_JOB_SUBMIT supports both in parallel. + * 11.23: + * - Modified KBASE_IOCTL_MEM_COMMIT behavior to reject requests to modify + * the physical memory backing of JIT allocations. This was not supposed + * to be a valid use case, but it was allowed by the previous implementation. + * 11.24: + * - Added a sysfs file 'serialize_jobs' inside a new sub-directory + * 'scheduling'. + * 11.25: + * - Enabled JIT pressure limit in base/kbase by default + * 11.26: + * - Added kinstr_jm API + * 11.27: + * - Backwards compatible extension to HWC ioctl. + * 11.28: + * - Added kernel side cache ops needed hint + * 11.29: + * - Reserve ioctl 52 + */ +#define BASE_UK_VERSION_MAJOR 11 +#define BASE_UK_VERSION_MINOR 29 + +/** + * struct kbase_ioctl_version_check - Check version compatibility between + * kernel and userspace + * + * @major: Major version number + * @minor: Minor version number + */ +struct kbase_ioctl_version_check { + __u16 major; + __u16 minor; +}; + +#define KBASE_IOCTL_VERSION_CHECK \ + _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) + +#define KBASE_IOCTL_VERSION_CHECK_RESERVED \ + _IOWR(KBASE_IOCTL_TYPE, 52, struct kbase_ioctl_version_check) + +/** + * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel + * + * @addr: Memory address of an array of struct base_jd_atom_v2 or v3 + * @nr_atoms: Number of entries in the array + * @stride: sizeof(struct base_jd_atom_v2) or sizeof(struct base_jd_atom) + */ +struct kbase_ioctl_job_submit { + __u64 addr; + __u32 nr_atoms; + __u32 stride; +}; + +#define KBASE_IOCTL_JOB_SUBMIT \ + _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit) + +#define KBASE_IOCTL_POST_TERM \ + _IO(KBASE_IOCTL_TYPE, 4) + +/** + * struct kbase_ioctl_soft_event_update - Update the status of a soft-event + * @event: GPU address of the event which has been updated + * @new_status: The new status to set + * @flags: Flags for future expansion + */ +struct kbase_ioctl_soft_event_update { + __u64 event; + __u32 new_status; + __u32 flags; +}; + +#define KBASE_IOCTL_SOFT_EVENT_UPDATE \ + _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) + +/** + * struct kbase_kinstr_jm_fd_out - Explains the compatibility information for + * the `struct kbase_kinstr_jm_atom_state_change` structure returned from the + * kernel + * + * @size: The size of the `struct kbase_kinstr_jm_atom_state_change` + * @version: Represents a breaking change in the + * `struct kbase_kinstr_jm_atom_state_change` + * @padding: Explicit padding to get the structure up to 64bits. See + * https://www.kernel.org/doc/Documentation/ioctl/botching-up-ioctls.rst + * + * The `struct kbase_kinstr_jm_atom_state_change` may have extra members at the + * end of the structure that older user space might not understand. If the + * `version` is the same, the structure is still compatible with newer kernels. + * The `size` can be used to cast the opaque memory returned from the kernel. + */ +struct kbase_kinstr_jm_fd_out { + __u16 size; + __u8 version; + __u8 padding[5]; +}; + +/** + * struct kbase_kinstr_jm_fd_in - Options when creating the file descriptor + * + * @count: Number of atom states that can be stored in the kernel circular + * buffer. Must be a power of two + * @padding: Explicit padding to get the structure up to 64bits. See + * https://www.kernel.org/doc/Documentation/ioctl/botching-up-ioctls.rst + */ +struct kbase_kinstr_jm_fd_in { + __u16 count; + __u8 padding[6]; +}; + +union kbase_kinstr_jm_fd { + struct kbase_kinstr_jm_fd_in in; + struct kbase_kinstr_jm_fd_out out; +}; + +#define KBASE_IOCTL_KINSTR_JM_FD \ + _IOWR(KBASE_IOCTL_TYPE, 51, union kbase_kinstr_jm_fd) + +#endif /* _KBASE_JM_IOCTL_H_ */ diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h new file mode 100755 index 000000000000..6c222ceae8ee --- /dev/null +++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h @@ -0,0 +1,892 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Job Scheduler Interface. + * These interfaces are Internal to KBase. + */ + +#ifndef _KBASE_JM_JS_H_ +#define _KBASE_JM_JS_H_ + +#include "mali_kbase_js_ctx_attr.h" + +/** + * kbasep_js_devdata_init - Initialize the Job Scheduler + * + * The struct kbasep_js_device_data sub-structure of kbdev must be zero + * initialized before passing to the kbasep_js_devdata_init() function. This is + * to give efficient error path code. + */ +int kbasep_js_devdata_init(struct kbase_device * const kbdev); + +/** + * kbasep_js_devdata_halt - Halt the Job Scheduler. + * + * It is safe to call this on kbdev even if it the kbasep_js_device_data + * sub-structure was never initialized/failed initialization, to give efficient + * error-path code. + * + * For this to work, the struct kbasep_js_device_data sub-structure of kbdev + * must be zero initialized before passing to the kbasep_js_devdata_init() + * function. This is to give efficient error path code. + * + * It is a programming error to call this whilst there are still kbase_context + * structures registered with this scheduler. + * + */ +void kbasep_js_devdata_halt(struct kbase_device *kbdev); + +/** + * kbasep_js_devdata_term - Terminate the Job Scheduler + * + * It is safe to call this on kbdev even if it the kbasep_js_device_data + * sub-structure was never initialized/failed initialization, to give efficient + * error-path code. + * + * For this to work, the struct kbasep_js_device_data sub-structure of kbdev + * must be zero initialized before passing to the kbasep_js_devdata_init() + * function. This is to give efficient error path code. + * + * It is a programming error to call this whilst there are still kbase_context + * structures registered with this scheduler. + */ +void kbasep_js_devdata_term(struct kbase_device *kbdev); + +/** + * kbasep_js_kctx_init - Initialize the Scheduling Component of a + * struct kbase_context on the Job Scheduler. + * + * This effectively registers a struct kbase_context with a Job Scheduler. + * + * It does not register any jobs owned by the struct kbase_context with + * the scheduler. Those must be separately registered by kbasep_js_add_job(). + * + * The struct kbase_context must be zero initialized before passing to the + * kbase_js_init() function. This is to give efficient error path code. + */ +int kbasep_js_kctx_init(struct kbase_context *const kctx); + +/** + * kbasep_js_kctx_term - Terminate the Scheduling Component of a + * struct kbase_context on the Job Scheduler + * + * This effectively de-registers a struct kbase_context from its Job Scheduler + * + * It is safe to call this on a struct kbase_context that has never had or + * failed initialization of its jctx.sched_info member, to give efficient + * error-path code. + * + * For this to work, the struct kbase_context must be zero intitialized before + * passing to the kbase_js_init() function. + * + * It is a Programming Error to call this whilst there are still jobs + * registered with this context. + */ +void kbasep_js_kctx_term(struct kbase_context *kctx); + +/** + * kbasep_js_add_job - Add a job chain to the Job Scheduler, + * and take necessary actions to + * schedule the context/run the job. + * + * This atomically does the following: + * * Update the numbers of jobs information + * * Add the job to the run pool if necessary (part of init_job) + * + * Once this is done, then an appropriate action is taken: + * * If the ctx is scheduled, it attempts to start the next job (which might be + * this added job) + * * Otherwise, and if this is the first job on the context, it enqueues it on + * the Policy Queue + * + * The Policy's Queue can be updated by this in the following ways: + * * In the above case that this is the first job on the context + * * If the context is high priority and the context is not scheduled, then it + * could cause the Policy to schedule out a low-priority context, allowing + * this context to be scheduled in. + * + * If the context is already scheduled on the RunPool, then adding a job to it + * is guaranteed not to update the Policy Queue. And so, the caller is + * guaranteed to not need to try scheduling a context from the Run Pool - it + * can safely assert that the result is false. + * + * It is a programming error to have more than U32_MAX jobs in flight at a time. + * + * The following locking conditions are made on the caller: + * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * * it must not hold hwaccess_lock (as this will be obtained internally) + * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * * it must not hold kbasep_jd_device_data::queue_mutex (again, it's used + * internally). + * + * Return: true indicates that the Policy Queue was updated, and so the + * caller will need to try scheduling a context onto the Run Pool, + * false indicates that no updates were made to the Policy Queue, + * so no further action is required from the caller. This is always returned + * when the context is currently scheduled. + */ +bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom); + +/** + * kbasep_js_remove_job - Remove a job chain from the Job Scheduler, + * except for its 'retained state'. + * + * Completely removing a job requires several calls: + * * kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of + * the atom + * * kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler + * * kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the + * remaining state held as part of the job having been run. + * + * In the common case of atoms completing normally, this set of actions is more + * optimal for spinlock purposes than having kbasep_js_remove_job() handle all + * of the actions. + * + * In the case of canceling atoms, it is easier to call + * kbasep_js_remove_cancelled_job(), which handles all the necessary actions. + * + * It is a programming error to call this when: + * * a atom is not a job belonging to kctx. + * * a atom has already been removed from the Job Scheduler. + * * a atom is still in the runpool + * + * Do not use this for removing jobs being killed by kbase_jd_cancel() - use + * kbasep_js_remove_cancelled_job() instead. + * + * The following locking conditions are made on the caller: + * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * + */ +void kbasep_js_remove_job(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_jd_atom *atom); + +/** + * kbasep_js_remove_cancelled_job - Completely remove a job chain from the + * Job Scheduler, in the case + * where the job chain was cancelled. + * + * This is a variant of kbasep_js_remove_job() that takes care of removing all + * of the retained state too. This is generally useful for cancelled atoms, + * which need not be handled in an optimal way. + * + * It is a programming error to call this when: + * * a atom is not a job belonging to kctx. + * * a atom has already been removed from the Job Scheduler. + * * a atom is still in the runpool: + * * it is not being killed with kbasep_jd_cancel() + * + * The following locking conditions are made on the caller: + * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * * it must not hold the hwaccess_lock, (as this will be obtained + * internally) + * * it must not hold kbasep_js_device_data::runpool_mutex (as this could be + * obtained internally) + * + * Return: true indicates that ctx attributes have changed and the caller + * should call kbase_js_sched_all() to try to run more jobs and + * false otherwise. + */ +bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +/** + * kbasep_js_runpool_requeue_or_kill_ctx - Handling the requeuing/killing of a + * context that was evicted from the + * policy queue or runpool. + * + * This should be used whenever handing off a context that has been evicted + * from the policy queue or the runpool: + * * If the context is not dying and has jobs, it gets re-added to the policy + * queue + * * Otherwise, it is not added + * + * In addition, if the context is dying the jobs are killed asynchronously. + * + * In all cases, the Power Manager active reference is released + * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. + * has_pm_ref must be set to false whenever the context was not previously in + * the runpool and does not hold a Power Manager active refcount. Note that + * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an + * active refcount even though they weren't in the runpool. + * + * The following locking conditions are made on the caller: + * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * * it must not hold kbasep_jd_device_data::queue_mutex (as this will be + * obtained internally) + */ +void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, bool has_pm_ref); + +/** + * kbasep_js_runpool_release_ctx - Release a refcount of a context being busy, + * allowing it to be scheduled out. + * + * When the refcount reaches zero and the context might be scheduled out + * (depending on whether the Scheduling Policy has deemed it so, or if it has + * run out of jobs). + * + * If the context does get scheduled out, then The following actions will be + * taken as part of deschduling a context: + * For the context being descheduled: + * * If the context is in the processing of dying (all the jobs are being + * removed from it), then descheduling also kills off any jobs remaining in the + * context. + * * If the context is not dying, and any jobs remain after descheduling the + * context then it is re-enqueued to the Policy's Queue. + * * Otherwise, the context is still known to the scheduler, but remains absent + * from the Policy Queue until a job is next added to it. + * * In all descheduling cases, the Power Manager active reference (obtained + * during kbasep_js_try_schedule_head_ctx()) is released + * (kbase_pm_context_idle()). + * + * Whilst the context is being descheduled, this also handles actions that + * cause more atoms to be run: + * * Attempt submitting atoms when the Context Attributes on the Runpool have + * changed. This is because the context being scheduled out could mean that + * there are more opportunities to run atoms. + * * Attempt submitting to a slot that was previously blocked due to affinity + * restrictions. This is usually only necessary when releasing a context + * happens as part of completing a previous job, but is harmless nonetheless. + * * Attempt scheduling in a new context (if one is available), and if + * necessary, running a job from that new context. + * + * Unlike retaining a context in the runpool, this function cannot be called + * from IRQ context. + * + * It is a programming error to call this on a kctx that is not currently + * scheduled, or that already has a zero refcount. + * + * The following locking conditions are made on the caller: + * * it must not hold the hwaccess_lock, because it will be used internally. + * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * * it must not hold the kbase_device::mmu_hw_mutex (as this will be + * obtained internally) + * * it must not hold kbasep_jd_device_data::queue_mutex (as this will be + * obtained internally) + * + */ +void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbasep_js_runpool_release_ctx_and_katom_retained_state - Variant of + * kbasep_js_runpool_release_ctx() that handles additional + * actions from completing an atom. + * + * This is usually called as part of completing an atom and releasing the + * refcount on the context held by the atom. + * + * Therefore, the extra actions carried out are part of handling actions queued + * on a completed atom, namely: + * * Releasing the atom's context attributes + * * Retrying the submission on a particular slot, because we couldn't submit + * on that slot from an IRQ handler. + * + * The locking conditions of this function are the same as those for + * kbasep_js_runpool_release_ctx() + */ +void kbasep_js_runpool_release_ctx_and_katom_retained_state( + struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state); + +/** + * kbasep_js_runpool_release_ctx_nolock - Variant of + * kbase_js_runpool_release_ctx() that assumes that + * kbasep_js_device_data::runpool_mutex and + * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not + * attempt to schedule new contexts. + */ +void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbasep_js_schedule_privileged_ctx - Schedule in a privileged context + * + * This schedules a context in regardless of the context priority. + * If the runpool is full, a context will be forced out of the runpool and the + * function will wait for the new context to be scheduled in. + * The context will be kept scheduled in (and the corresponding address space + * reserved) until kbasep_js_release_privileged_ctx is called). + * + * The following locking conditions are made on the caller: + * * it must not hold the hwaccess_lock, because it will be used internally. + * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * * it must not hold the kbase_device::mmu_hw_mutex (as this will be + * obtained internally) + * * it must not hold kbasep_jd_device_data::queue_mutex (again, it's used + * internally). + * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will + * be used internally. + * + */ +void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbasep_js_release_privileged_ctx - Release a privileged context, + * allowing it to be scheduled out. + * + * See kbasep_js_runpool_release_ctx for potential side effects. + * + * The following locking conditions are made on the caller: + * * it must not hold the hwaccess_lock, because it will be used internally. + * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * * it must not hold the kbase_device::mmu_hw_mutex (as this will be + * obtained internally) + * + */ +void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbase_js_try_run_jobs - Try to submit the next job on each slot + * + * The following locks may be used: + * * kbasep_js_device_data::runpool_mutex + * * hwaccess_lock + */ +void kbase_js_try_run_jobs(struct kbase_device *kbdev); + +/** + * kbasep_js_suspend - Suspend the job scheduler during a Power Management + * Suspend event. + * + * Causes all contexts to be removed from the runpool, and prevents any + * contexts from (re)entering the runpool. + * + * This does not handle suspending the one privileged context: the caller must + * instead do this by by suspending the GPU HW Counter Instrumentation. + * + * This will eventually cause all Power Management active references held by + * contexts on the runpool to be released, without running any more atoms. + * + * The caller must then wait for all Power Management active refcount to become + * zero before completing the suspend. + * + * The emptying mechanism may take some time to complete, since it can wait for + * jobs to complete naturally instead of forcing them to end quickly. However, + * this is bounded by the Job Scheduler's Job Timeouts. Hence, this + * function is guaranteed to complete in a finite time. + */ +void kbasep_js_suspend(struct kbase_device *kbdev); + +/** + * kbasep_js_resume - Resume the Job Scheduler after a Power Management + * Resume event. + * + * This restores the actions from kbasep_js_suspend(): + * * Schedules contexts back into the runpool + * * Resumes running atoms on the GPU + */ +void kbasep_js_resume(struct kbase_device *kbdev); + +/** + * kbase_js_dep_resolved_submit - Submit an atom to the job scheduler. + * + * @kctx: Context pointer + * @atom: Pointer to the atom to submit + * + * The atom is enqueued on the context's ringbuffer. The caller must have + * ensured that all dependencies can be represented in the ringbuffer. + * + * Caller must hold jctx->lock + * + * Return: true if the context requires to be enqueued, otherwise false. + */ +bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +/** + * jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer. + * @kctx: Context Pointer + * @prio: Priority (specifies the queue together with js). + * @js: Job slot (specifies the queue together with prio). + * + * Pushes all possible atoms from the linked list to the ringbuffer. + * Number of atoms are limited to free space in the ringbuffer and + * number of available atoms in the linked list. + * + */ +void jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js); + +/** + * kbase_js_pull - Pull an atom from a context in the job scheduler for + * execution. + * + * @kctx: Context to pull from + * @js: Job slot to pull from + * + * The atom will not be removed from the ringbuffer at this stage. + * + * The HW access lock must be held when calling this function. + * + * Return: a pointer to an atom, or NULL if there are no atoms for this + * slot that can be currently run. + */ +struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js); + +/** + * kbase_js_unpull - Return an atom to the job scheduler ringbuffer. + * + * @kctx: Context pointer + * @atom: Pointer to the atom to unpull + * + * An atom is 'unpulled' if execution is stopped but intended to be returned to + * later. The most common reason for this is that the atom has been + * soft-stopped. Another reason is if an end-of-renderpass atom completed + * but will need to be run again as part of the same renderpass. + * + * Note that if multiple atoms are to be 'unpulled', they must be returned in + * the reverse order to which they were originally pulled. It is a programming + * error to return atoms in any other order. + * + * The HW access lock must be held when calling this function. + * + */ +void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom); + +/** + * kbase_js_complete_atom_wq - Complete an atom from jd_done_worker(), + * removing it from the job + * scheduler ringbuffer. + * @kctx: Context pointer + * @katom: Pointer to the atom to complete + * + * If the atom failed then all dependee atoms marked for failure propagation + * will also fail. + * + * Return: true if the context is now idle (no jobs pulled) false otherwise. + */ +bool kbase_js_complete_atom_wq(struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +/** + * kbase_js_complete_atom - Complete an atom. + * + * @katom: Pointer to the atom to complete + * @end_timestamp: The time that the atom completed (may be NULL) + * + * Most of the work required to complete an atom will be performed by + * jd_done_worker(). + * + * The HW access lock must be held when calling this function. + * + * Return: a atom that has now been unblocked and can now be run, or NULL + * if none + */ +struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, + ktime_t *end_timestamp); + +/** + * kbase_js_atom_blocked_on_x_dep - Decide whether to ignore a cross-slot + * dependency + * @katom: Pointer to an atom in the slot ringbuffer + * + * A cross-slot dependency is ignored if necessary to unblock incremental + * rendering. If the atom at the start of a renderpass used too much memory + * and was soft-stopped then the atom at the end of a renderpass is submitted + * to hardware regardless of its dependency on the start-of-renderpass atom. + * This can happen multiple times for the same pair of atoms. + * + * Return: true to block the atom or false to allow it to be submitted to + * hardware. + */ +bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *katom); + +/** + * kbase_js_sched - Submit atoms from all available contexts. + * + * @kbdev: Device pointer + * @js_mask: Mask of job slots to submit to + * + * This will attempt to submit as many jobs as possible to the provided job + * slots. It will exit when either all job slots are full, or all contexts have + * been used. + * + */ +void kbase_js_sched(struct kbase_device *kbdev, int js_mask); + +/** + * kbase_jd_zap_context - Attempt to deschedule a context that is being + * destroyed + * @kctx: Context pointer + * + * This will attempt to remove a context from any internal job scheduler queues + * and perform any other actions to ensure a context will not be submitted + * from. + * + * If the context is currently scheduled, then the caller must wait for all + * pending jobs to complete before taking any further action. + */ +void kbase_js_zap_context(struct kbase_context *kctx); + +/** + * kbase_js_is_atom_valid - Validate an atom + * + * @kbdev: Device pointer + * @katom: Atom to validate + * + * This will determine whether the atom can be scheduled onto the GPU. Atoms + * with invalid combinations of core requirements will be rejected. + * + * Return: true if atom is valid false otherwise. + */ +bool kbase_js_is_atom_valid(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_js_set_timeouts - update all JS timeouts with user specified data + * + * @kbdev: Device pointer + * + * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is + * set to a positive number then that becomes the new value used, if a timeout + * is negative then the default is set. + */ +void kbase_js_set_timeouts(struct kbase_device *kbdev); + +/** + * kbase_js_set_ctx_priority - set the context priority + * + * @kctx: Context pointer + * @new_priority: New priority value for the Context + * + * The context priority is set to a new value and it is moved to the + * pullable/unpullable list as per the new priority. + */ +void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority); + + +/** + * kbase_js_update_ctx_priority - update the context priority + * + * @kctx: Context pointer + * + * The context priority gets updated as per the priority of atoms currently in + * use for that context, but only if system priority mode for context scheduling + * is being used. + */ +void kbase_js_update_ctx_priority(struct kbase_context *kctx); + +/* + * Helpers follow + */ + +/** + * kbasep_js_is_submit_allowed - Check that a context is allowed to submit + * jobs on this policy + * + * The purpose of this abstraction is to hide the underlying data size, + * and wrap up the long repeated line of code. + * + * As with any bool, never test the return value with true. + * + * The caller must hold hwaccess_lock. + */ +static inline bool kbasep_js_is_submit_allowed( + struct kbasep_js_device_data *js_devdata, + struct kbase_context *kctx) +{ + u16 test_bit; + bool is_allowed; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + test_bit = (u16) (1u << kctx->as_nr); + + is_allowed = (bool) (js_devdata->runpool_irq.submit_allowed & test_bit); + dev_dbg(kctx->kbdev->dev, "JS: submit %s allowed on %p (as=%d)", + is_allowed ? "is" : "isn't", (void *)kctx, kctx->as_nr); + return is_allowed; +} + +/** + * kbasep_js_set_submit_allowed - Allow a context to submit jobs on this policy + * + * The purpose of this abstraction is to hide the underlying data size, + * and wrap up the long repeated line of code. + * + * The caller must hold hwaccess_lock. + */ +static inline void kbasep_js_set_submit_allowed( + struct kbasep_js_device_data *js_devdata, + struct kbase_context *kctx) +{ + u16 set_bit; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + set_bit = (u16) (1u << kctx->as_nr); + + dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)", + kctx, kctx->as_nr); + + js_devdata->runpool_irq.submit_allowed |= set_bit; +} + +/** + * kbasep_js_clear_submit_allowed - Prevent a context from submitting more + * jobs on this policy + * + * The purpose of this abstraction is to hide the underlying data size, + * and wrap up the long repeated line of code. + * + * The caller must hold hwaccess_lock. + */ +static inline void kbasep_js_clear_submit_allowed( + struct kbasep_js_device_data *js_devdata, + struct kbase_context *kctx) +{ + u16 clear_bit; + u16 clear_mask; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + clear_bit = (u16) (1u << kctx->as_nr); + clear_mask = ~clear_bit; + + dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)", + kctx, kctx->as_nr); + + js_devdata->runpool_irq.submit_allowed &= clear_mask; +} + +/** + * Create an initial 'invalid' atom retained state, that requires no + * atom-related work to be done on releasing with + * kbasep_js_runpool_release_ctx_and_katom_retained_state() + */ +static inline void kbasep_js_atom_retained_state_init_invalid( + struct kbasep_js_atom_retained_state *retained_state) +{ + retained_state->event_code = BASE_JD_EVENT_NOT_STARTED; + retained_state->core_req = + KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID; +} + +/** + * Copy atom state that can be made available after jd_done_nolock() is called + * on that atom. + */ +static inline void kbasep_js_atom_retained_state_copy( + struct kbasep_js_atom_retained_state *retained_state, + const struct kbase_jd_atom *katom) +{ + retained_state->event_code = katom->event_code; + retained_state->core_req = katom->core_req; + retained_state->sched_priority = katom->sched_priority; + retained_state->device_nr = katom->device_nr; +} + +/** + * kbasep_js_has_atom_finished - Determine whether an atom has finished + * (given its retained state), + * and so should be given back to + * userspace/removed from the system. + * + * @katom_retained_state: the retained state of the atom to check + * + * Reasons for an atom not finishing include: + * * Being soft-stopped (and so, the atom should be resubmitted sometime later) + * * It is an end of renderpass atom that was run to consume the output of a + * start-of-renderpass atom that was soft-stopped because it used too much + * memory. In this case, it will have to be run again later. + * + * Return: false if the atom has not finished, true otherwise. + */ +static inline bool kbasep_js_has_atom_finished( + const struct kbasep_js_atom_retained_state *katom_retained_state) +{ + return (bool) (katom_retained_state->event_code != + BASE_JD_EVENT_STOPPED && + katom_retained_state->event_code != + BASE_JD_EVENT_REMOVED_FROM_NEXT && + katom_retained_state->event_code != + BASE_JD_EVENT_END_RP_DONE); +} + +/** + * kbasep_js_atom_retained_state_is_valid - Determine whether a struct + * kbasep_js_atom_retained_state + * is valid + * @katom_retained_state the atom's retained state to check + * + * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates + * that the code should just ignore it. + * + * Return: false if the retained state is invalid, true otherwise. + */ +static inline bool kbasep_js_atom_retained_state_is_valid( + const struct kbasep_js_atom_retained_state *katom_retained_state) +{ + return (bool) (katom_retained_state->core_req != + KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID); +} + +/** + * kbase_js_runpool_inc_context_count - Increment number of running contexts. + * + * The following locking conditions are made on the caller: + * * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. + * * The caller must hold the kbasep_js_device_data::runpool_mutex + */ +static inline void kbase_js_runpool_inc_context_count( + struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + + /* Track total contexts */ + KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX); + ++(js_devdata->nr_all_contexts_running); + + if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + /* Track contexts that can submit jobs */ + KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running < + S8_MAX); + ++(js_devdata->nr_user_contexts_running); + } +} + +/** + * kbase_js_runpool_dec_context_count - decrement number of running contexts. + * + * The following locking conditions are made on the caller: + * * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. + * * The caller must hold the kbasep_js_device_data::runpool_mutex + */ +static inline void kbase_js_runpool_dec_context_count( + struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + + /* Track total contexts */ + --(js_devdata->nr_all_contexts_running); + KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0); + + if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + /* Track contexts that can submit jobs */ + --(js_devdata->nr_user_contexts_running); + KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0); + } +} + +/** + * kbase_js_sched_all - Submit atoms from all available contexts to all + * job slots. + * + * @kbdev: Device pointer + * + * This will attempt to submit as many jobs as possible. It will exit when + * either all job slots are full, or all contexts have been used. + */ +static inline void kbase_js_sched_all(struct kbase_device *kbdev) +{ + kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); +} + +extern const int +kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS]; + +extern const base_jd_prio +kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + +/** + * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio) + * to relative ordering + * @atom_prio: Priority ID to translate. + * + * Atom priority values for @ref base_jd_prio cannot be compared directly to + * find out which are higher or lower. + * + * This function will convert base_jd_prio values for successively lower + * priorities into a monotonically increasing sequence. That is, the lower the + * base_jd_prio priority, the higher the value produced by this function. This + * is in accordance with how the rest of the kernel treats priority. + * + * The mapping is 1:1 and the size of the valid input range is the same as the + * size of the valid output range, i.e. + * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS + * + * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions + * + * Return: On success: a value in the inclusive range + * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure: + * KBASE_JS_ATOM_SCHED_PRIO_INVALID + */ +static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio) +{ + if (atom_prio >= BASE_JD_NR_PRIO_LEVELS) + return KBASE_JS_ATOM_SCHED_PRIO_INVALID; + + return kbasep_js_atom_priority_to_relative[atom_prio]; +} + +static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio) +{ + unsigned int prio_idx; + + KBASE_DEBUG_ASSERT(sched_prio >= 0 && + sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT); + + prio_idx = (unsigned int)sched_prio; + + return kbasep_js_relative_priority_to_atom[prio_idx]; +} + +#endif /* _KBASE_JM_JS_H_ */ diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_js_defs.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_js_defs.h new file mode 100755 index 000000000000..900ecd2c1b8d --- /dev/null +++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_js_defs.h @@ -0,0 +1,409 @@ +/* + * + * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/** + * @file mali_kbase_js.h + * Job Scheduler Type Definitions + */ + +#ifndef _KBASE_JS_DEFS_H_ +#define _KBASE_JS_DEFS_H_ + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_js + * @{ + */ +/* Forward decls */ +struct kbase_device; +struct kbase_jd_atom; + + +typedef u32 kbase_context_flags; + +/** Callback function run on all of a context's jobs registered with the Job + * Scheduler */ +typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd_atom *katom); + +/** + * @brief Maximum number of jobs that can be submitted to a job slot whilst + * inside the IRQ handler. + * + * This is important because GPU NULL jobs can complete whilst the IRQ handler + * is running. Otherwise, it potentially allows an unlimited number of GPU NULL + * jobs to be submitted inside the IRQ handler, which increases IRQ latency. + */ +#define KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ 2 + +/** + * @brief Context attributes + * + * Each context attribute can be thought of as a boolean value that caches some + * state information about either the runpool, or the context: + * - In the case of the runpool, it is a cache of "Do any contexts owned by + * the runpool have attribute X?" + * - In the case of a context, it is a cache of "Do any atoms owned by the + * context have attribute X?" + * + * The boolean value of the context attributes often affect scheduling + * decisions, such as affinities to use and job slots to use. + * + * To accomodate changes of state in the context, each attribute is refcounted + * in the context, and in the runpool for all running contexts. Specifically: + * - The runpool holds a refcount of how many contexts in the runpool have this + * attribute. + * - The context holds a refcount of how many atoms have this attribute. + */ +enum kbasep_js_ctx_attr { + /** Attribute indicating a context that contains Compute jobs. That is, + * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE + * + * @note A context can be both 'Compute' and 'Non Compute' if it contains + * both types of jobs. + */ + KBASEP_JS_CTX_ATTR_COMPUTE, + + /** Attribute indicating a context that contains Non-Compute jobs. That is, + * the context has some jobs that are \b not of type @ref + * BASE_JD_REQ_ONLY_COMPUTE. + * + * @note A context can be both 'Compute' and 'Non Compute' if it contains + * both types of jobs. + */ + KBASEP_JS_CTX_ATTR_NON_COMPUTE, + + /** Attribute indicating that a context contains compute-job atoms that + * aren't restricted to a coherent group, and can run on all cores. + * + * Specifically, this is when the atom's \a core_req satisfy: + * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 + * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups + * + * Such atoms could be blocked from running if one of the coherent groups + * is being used by another job slot, so tracking this context attribute + * allows us to prevent such situations. + * + * @note This doesn't take into account the 1-coregroup case, where all + * compute atoms would effectively be able to run on 'all cores', but + * contexts will still not always get marked with this attribute. Instead, + * it is the caller's responsibility to take into account the number of + * coregroups when interpreting this attribute. + * + * @note Whilst Tiler atoms are normally combined with + * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without + * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy + * enough to handle anyway. + */ + KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, + + /** Must be the last in the enum */ + KBASEP_JS_CTX_ATTR_COUNT +}; + +enum { + /** Bit indicating that new atom should be started because this atom completed */ + KBASE_JS_ATOM_DONE_START_NEW_ATOMS = (1u << 0), + /** Bit indicating that the atom was evicted from the JS_NEXT registers */ + KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT = (1u << 1) +}; + +/** Combination of KBASE_JS_ATOM_DONE_<...> bits */ +typedef u32 kbasep_js_atom_done_code; + +/* + * Context scheduling mode defines for kbase_device::js_ctx_scheduling_mode + */ +enum { + /* + * In this mode, higher priority atoms will be scheduled first, + * regardless of the context they belong to. Newly-runnable higher + * priority atoms can preempt lower priority atoms currently running on + * the GPU, even if they belong to a different context. + */ + KBASE_JS_SYSTEM_PRIORITY_MODE = 0, + + /* + * In this mode, the highest-priority atom will be chosen from each + * context in turn using a round-robin algorithm, so priority only has + * an effect within the context an atom belongs to. Newly-runnable + * higher priority atoms can preempt the lower priority atoms currently + * running on the GPU, but only if they belong to the same context. + */ + KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE, + + /* Must be the last in the enum */ + KBASE_JS_PRIORITY_MODE_COUNT, +}; + +/* + * Internal atom priority defines for kbase_jd_atom::sched_prio + */ +enum { + KBASE_JS_ATOM_SCHED_PRIO_HIGH = 0, + KBASE_JS_ATOM_SCHED_PRIO_MED, + KBASE_JS_ATOM_SCHED_PRIO_LOW, + KBASE_JS_ATOM_SCHED_PRIO_COUNT, +}; + +/* Invalid priority for kbase_jd_atom::sched_prio */ +#define KBASE_JS_ATOM_SCHED_PRIO_INVALID -1 + +/* Default priority in the case of contexts with no atoms, or being lenient + * about invalid priorities from userspace. + */ +#define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED + +/** + * @brief KBase Device Data Job Scheduler sub-structure + * + * This encapsulates the current context of the Job Scheduler on a particular + * device. This context is global to the device, and is not tied to any + * particular struct kbase_context running on the device. + * + * nr_contexts_running and as_free are optimized for packing together (by making + * them smaller types than u32). The operations on them should rarely involve + * masking. The use of signed types for arithmetic indicates to the compiler that + * the value will not rollover (which would be undefined behavior), and so under + * the Total License model, it is free to make optimizations based on that (i.e. + * to remove masking). + */ +struct kbasep_js_device_data { + /* Sub-structure to collect together Job Scheduling data used in IRQ + * context. The hwaccess_lock must be held when accessing. */ + struct runpool_irq { + /** Bitvector indicating whether a currently scheduled context is allowed to submit jobs. + * When bit 'N' is set in this, it indicates whether the context bound to address space + * 'N' is allowed to submit jobs. + */ + u16 submit_allowed; + + /** Context Attributes: + * Each is large enough to hold a refcount of the number of contexts + * that can fit into the runpool. This is currently BASE_MAX_NR_AS + * + * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store + * the refcount. Hence, it's not worthwhile reducing this to + * bit-manipulation on u32s to save space (where in contrast, 4 bit + * sub-fields would be easy to do and would save space). + * + * Whilst this must not become negative, the sign bit is used for: + * - error detection in debug builds + * - Optimization: it is undefined for a signed int to overflow, and so + * the compiler can optimize for that never happening (thus, no masking + * is required on updating the variable) */ + s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; + + /* + * Affinity management and tracking + */ + /** Bitvector to aid affinity checking. Element 'n' bit 'i' indicates + * that slot 'n' is using core i (i.e. slot_affinity_refcount[n][i] > 0) */ + u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; + /** Refcount for each core owned by each slot. Used to generate the + * slot_affinities array of bitvectors + * + * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, + * because it is refcounted only when a job is definitely about to be + * submitted to a slot, and is de-refcounted immediately after a job + * finishes */ + s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; + } runpool_irq; + + /** + * Scheduling semaphore. This must be held when calling + * kbase_jm_kick() + */ + struct semaphore schedule_sem; + + /** + * List of contexts that can currently be pulled from + */ + struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + /** + * List of contexts that can not currently be pulled from, but have + * jobs currently running. + */ + struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + + /** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */ + s8 nr_user_contexts_running; + /** Number of currently scheduled contexts (including ones that are not submitting jobs) */ + s8 nr_all_contexts_running; + + /** Core Requirements to match up with base_js_atom's core_req memeber + * @note This is a write-once member, and so no locking is required to read */ + base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; + + u32 scheduling_period_ns; /*< Value for JS_SCHEDULING_PERIOD_NS */ + u32 soft_stop_ticks; /*< Value for JS_SOFT_STOP_TICKS */ + u32 soft_stop_ticks_cl; /*< Value for JS_SOFT_STOP_TICKS_CL */ + u32 hard_stop_ticks_ss; /*< Value for JS_HARD_STOP_TICKS_SS */ + u32 hard_stop_ticks_cl; /*< Value for JS_HARD_STOP_TICKS_CL */ + u32 hard_stop_ticks_dumping; /*< Value for JS_HARD_STOP_TICKS_DUMPING */ + u32 gpu_reset_ticks_ss; /*< Value for JS_RESET_TICKS_SS */ + u32 gpu_reset_ticks_cl; /*< Value for JS_RESET_TICKS_CL */ + u32 gpu_reset_ticks_dumping; /*< Value for JS_RESET_TICKS_DUMPING */ + u32 ctx_timeslice_ns; /**< Value for JS_CTX_TIMESLICE_NS */ + + /** List of suspended soft jobs */ + struct list_head suspended_soft_jobs_list; + +#ifdef CONFIG_MALI_BIFROST_DEBUG + /* Support soft-stop on a single context */ + bool softstop_always; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + /** The initalized-flag is placed at the end, to avoid cache-pollution (we should + * only be using this during init/term paths). + * @note This is a write-once member, and so no locking is required to read */ + int init_status; + + /* Number of contexts that can currently be pulled from */ + u32 nr_contexts_pullable; + + /* Number of contexts that can either be pulled from or are currently + * running */ + atomic_t nr_contexts_runnable; + + /** Value for JS_SOFT_JOB_TIMEOUT */ + atomic_t soft_job_timeout_ms; + + /** + * Queue Lock, used to access the Policy's queue of contexts + * independently of the Run Pool. + * + * Of course, you don't need the Run Pool lock to access this. + */ + struct mutex queue_mutex; + + /** + * Run Pool mutex, for managing contexts within the runpool. + * Unless otherwise specified, you must hold this lock whilst accessing + * any members that follow + * + * In addition, this is used to access: + * * the kbasep_js_kctx_info::runpool substructure + */ + struct mutex runpool_mutex; +}; + +/** + * @brief KBase Context Job Scheduling information structure + * + * This is a substructure in the struct kbase_context that encapsulates all the + * scheduling information. + */ +struct kbasep_js_kctx_info { + + /** + * Job Scheduler Context information sub-structure. These members are + * accessed regardless of whether the context is: + * - In the Policy's Run Pool + * - In the Policy's Queue + * - Not queued nor in the Run Pool. + * + * You must obtain the jsctx_mutex before accessing any other members of + * this substructure. + * + * You may not access any of these members from IRQ context. + */ + struct kbase_jsctx { + struct mutex jsctx_mutex; /**< Job Scheduler Context lock */ + + /** Number of jobs ready to run - does \em not include the jobs waiting in + * the dispatcher, and dependency-only jobs. See kbase_jd_context::job_nr + * for such jobs*/ + u32 nr_jobs; + + /** Context Attributes: + * Each is large enough to hold a refcount of the number of atoms on + * the context. **/ + u32 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; + + /** + * Wait queue to wait for KCTX_SHEDULED flag state changes. + * */ + wait_queue_head_t is_scheduled_wait; + + /** Link implementing JS queues. Context can be present on one + * list per job slot + */ + struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS]; + } ctx; + + /* The initalized-flag is placed at the end, to avoid cache-pollution (we should + * only be using this during init/term paths) */ + int init_status; +}; + +/** Subset of atom state that can be available after jd_done_nolock() is called + * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), + * because the original atom could disappear. */ +struct kbasep_js_atom_retained_state { + /** Event code - to determine whether the atom has finished */ + enum base_jd_event_code event_code; + /** core requirements */ + base_jd_core_req core_req; + /* priority */ + int sched_priority; + /* Core group atom was executed on */ + u32 device_nr; + +}; + +/** + * Value signifying 'no retry on a slot required' for: + * - kbase_js_atom_retained_state::retry_submit_on_slot + * - kbase_jd_atom::retry_submit_on_slot + */ +#define KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID (-1) + +/** + * base_jd_core_req value signifying 'invalid' for a kbase_jd_atom_retained_state. + * + * @see kbase_atom_retained_state_is_valid() + */ +#define KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID BASE_JD_REQ_DEP + +/** + * @brief The JS timer resolution, in microseconds + * + * Any non-zero difference in time will be at least this size. + */ +#define KBASEP_JS_TICK_RESOLUTION_US 1 + + + /** @} *//* end group kbase_js */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_JS_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h new file mode 100755 index 000000000000..0dc08381bee6 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h @@ -0,0 +1,515 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, + * please update base/tools/hwconfig_generator/hwc_{issues,features}.py + * For more information see base/tools/hwconfig_generator/README + */ + +#ifndef _BASE_HWCONFIG_FEATURES_H_ +#define _BASE_HWCONFIG_FEATURES_H_ + +enum base_hw_feature { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_TLS_HASHING, + BASE_HW_FEATURE_THREAD_GROUP_SPLIT, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_IDVS_GROUP_SIZE, + BASE_HW_FEATURE_L2_CONFIG, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_generic[] = { + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tMIx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_THREAD_GROUP_SPLIT, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tHEx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_THREAD_GROUP_SPLIT, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tSIx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_THREAD_GROUP_SPLIT, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tDVx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_THREAD_GROUP_SPLIT, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tNOx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_THREAD_GROUP_SPLIT, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_TLS_HASHING, + BASE_HW_FEATURE_IDVS_GROUP_SIZE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tGOx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_THREAD_GROUP_SPLIT, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_TLS_HASHING, + BASE_HW_FEATURE_IDVS_GROUP_SIZE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tTRx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_IDVS_GROUP_SIZE, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tNAx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_IDVS_GROUP_SIZE, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tBEx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_IDVS_GROUP_SIZE, + BASE_HW_FEATURE_L2_CONFIG, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tBAx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_IDVS_GROUP_SIZE, + BASE_HW_FEATURE_L2_CONFIG, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tDUx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_IDVS_GROUP_SIZE, + BASE_HW_FEATURE_L2_CONFIG, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tODx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_L2_CONFIG, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tGRx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_L2_CONFIG, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tVAx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_L2_CONFIG, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tTUx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_L2_CONFIG, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tE2x[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_IDVS_GROUP_SIZE, + BASE_HW_FEATURE_L2_CONFIG, + BASE_HW_FEATURE_CLEAN_ONLY_SAFE, + BASE_HW_FEATURE_END +}; + +#endif /* _BASE_HWCONFIG_FEATURES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h new file mode 100755 index 000000000000..c1ad3ac40705 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h @@ -0,0 +1,684 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, + * please update base/tools/hwconfig_generator/hwc_{issues,features}.py + * For more information see base/tools/hwconfig_generator/README + */ + +#ifndef _BASE_HWCONFIG_ISSUES_H_ +#define _BASE_HWCONFIG_ISSUES_H_ + +enum base_hw_issue { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_TNOX_1194, + BASE_HW_ISSUE_TGOX_R1_1234, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TSIX_1792, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_3076, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_generic[] = { + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tMIx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tHEx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TSIX_1792, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TSIX_1792, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tSIx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tDVx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TNOX_1194, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tNOx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TNOX_1194, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tGOx_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TGOX_R1_1234, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tGOx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_3076, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_3076, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tTRx_r0p2[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_3076, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tTRx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_3076, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tNAx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_3076, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tNAx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tBEx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tBEx_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tBEx_r1p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tBEx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TTRX_3485, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tBAx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tBAx_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tBAx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tDUx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tDUx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tODx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tGRx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tVAx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tTUx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tE2x_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tE2x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_GPU2019_3212, + BASE_HW_ISSUE_END +}; + +#endif /* _BASE_HWCONFIG_ISSUES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_base_kernel.h b/drivers/gpu/arm/bifrost/mali_base_kernel.h new file mode 100755 index 000000000000..086171adb6e5 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_base_kernel.h @@ -0,0 +1,807 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Base structures shared with the kernel. + */ + +#ifndef _BASE_KERNEL_H_ +#define _BASE_KERNEL_H_ + +struct base_mem_handle { + struct { + u64 handle; + } basep; +}; + +#include "mali_base_mem_priv.h" +#include "gpu/mali_kbase_gpu_coherency.h" +#include "gpu/mali_kbase_gpu_id.h" + +#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 4 + +#define BASE_MAX_COHERENT_GROUPS 16 + +#if defined CDBG_ASSERT +#define LOCAL_ASSERT CDBG_ASSERT +#elif defined KBASE_DEBUG_ASSERT +#define LOCAL_ASSERT KBASE_DEBUG_ASSERT +#else +#error assert macro not defined! +#endif + +#if defined(PAGE_MASK) && defined(PAGE_SHIFT) +#define LOCAL_PAGE_SHIFT PAGE_SHIFT +#define LOCAL_PAGE_LSB ~PAGE_MASK +#else +#include + +#if defined OSU_CONFIG_CPU_PAGE_SIZE_LOG2 +#define LOCAL_PAGE_SHIFT OSU_CONFIG_CPU_PAGE_SIZE_LOG2 +#define LOCAL_PAGE_LSB ((1ul << OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - 1) +#else +#error Failed to find page size +#endif +#endif + +/* Physical memory group ID for normal usage. + */ +#define BASE_MEM_GROUP_DEFAULT (0) + +/* Number of physical memory groups. + */ +#define BASE_MEM_GROUP_COUNT (16) + +/** + * typedef base_mem_alloc_flags - Memory allocation, access/hint flags. + * + * A combination of MEM_PROT/MEM_HINT flags must be passed to each allocator + * in order to determine the best cache policy. Some combinations are + * of course invalid (e.g. MEM_PROT_CPU_WR | MEM_HINT_CPU_RD), + * which defines a write-only region on the CPU side, which is + * heavily read by the CPU... + * Other flags are only meaningful to a particular allocator. + * More flags can be added to this list, as long as they don't clash + * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). + */ +typedef u32 base_mem_alloc_flags; + +/* A mask for all the flags which are modifiable via the base_mem_set_flags + * interface. + */ +#define BASE_MEM_FLAGS_MODIFIABLE \ + (BASE_MEM_DONT_NEED | BASE_MEM_COHERENT_SYSTEM | \ + BASE_MEM_COHERENT_LOCAL) + +/* A mask of all the flags that can be returned via the base_mem_get_flags() + * interface. + */ +#define BASE_MEM_FLAGS_QUERYABLE \ + (BASE_MEM_FLAGS_INPUT_MASK & ~(BASE_MEM_SAME_VA | \ + BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_DONT_NEED | \ + BASE_MEM_IMPORT_SHARED | BASE_MEM_FLAGS_RESERVED | \ + BASEP_MEM_FLAGS_KERNEL_ONLY)) + +/** + * enum base_mem_import_type - Memory types supported by @a base_mem_import + * + * @BASE_MEM_IMPORT_TYPE_INVALID: Invalid type + * @BASE_MEM_IMPORT_TYPE_UMM: UMM import. Handle type is a file descriptor (int) + * @BASE_MEM_IMPORT_TYPE_USER_BUFFER: User buffer import. Handle is a + * base_mem_import_user_buffer + * + * Each type defines what the supported handle type is. + * + * If any new type is added here ARM must be contacted + * to allocate a numeric value for it. + * Do not just add a new type without synchronizing with ARM + * as future releases from ARM might include other new types + * which could clash with your custom types. + */ +enum base_mem_import_type { + BASE_MEM_IMPORT_TYPE_INVALID = 0, + /** + * Import type with value 1 is deprecated. + */ + BASE_MEM_IMPORT_TYPE_UMM = 2, + BASE_MEM_IMPORT_TYPE_USER_BUFFER = 3 +}; + +/** + * struct base_mem_import_user_buffer - Handle of an imported user buffer + * + * @ptr: address of imported user buffer + * @length: length of imported user buffer in bytes + * + * This structure is used to represent a handle of an imported user buffer. + */ + +struct base_mem_import_user_buffer { + u64 ptr; + u64 length; +}; + +/* Mask to detect 4GB boundary alignment */ +#define BASE_MEM_MASK_4GB 0xfffff000UL +/* Mask to detect 4GB boundary (in page units) alignment */ +#define BASE_MEM_PFN_MASK_4GB (BASE_MEM_MASK_4GB >> LOCAL_PAGE_SHIFT) + +/* Limit on the 'extent' parameter for an allocation with the + * BASE_MEM_TILER_ALIGN_TOP flag set + * + * This is the same as the maximum limit for a Buffer Descriptor's chunk size + */ +#define BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES_LOG2 \ + (21u - (LOCAL_PAGE_SHIFT)) +#define BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES \ + (1ull << (BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES_LOG2)) + +/* Bit mask of cookies used for for memory allocation setup */ +#define KBASE_COOKIE_MASK ~1UL /* bit 0 is reserved */ + +/* Maximum size allowed in a single KBASE_IOCTL_MEM_ALLOC call */ +#define KBASE_MEM_ALLOC_MAX_SIZE ((8ull << 30) >> PAGE_SHIFT) /* 8 GB */ + +/** + * struct base_fence - Cross-device synchronisation fence. + * + * A fence is used to signal when the GPU has finished accessing a resource that + * may be shared with other devices, and also to delay work done asynchronously + * by the GPU until other devices have finished accessing a shared resource. + */ +struct base_fence { + struct { + int fd; + int stream_fd; + } basep; +}; + +/** + * struct base_mem_aliasing_info - Memory aliasing info + * + * Describes a memory handle to be aliased. + * A subset of the handle can be chosen for aliasing, given an offset and a + * length. + * A special handle BASE_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a + * region where a special page is mapped with a write-alloc cache setup, + * typically used when the write result of the GPU isn't needed, but the GPU + * must write anyway. + * + * Offset and length are specified in pages. + * Offset must be within the size of the handle. + * Offset+length must not overrun the size of the handle. + * + * @handle: Handle to alias, can be BASE_MEM_WRITE_ALLOC_PAGES_HANDLE + * @offset: Offset within the handle to start aliasing from, in pages. + * Not used with BASE_MEM_WRITE_ALLOC_PAGES_HANDLE. + * @length: Length to alias, in pages. For BASE_MEM_WRITE_ALLOC_PAGES_HANDLE + * specifies the number of times the special page is needed. + */ +struct base_mem_aliasing_info { + struct base_mem_handle handle; + u64 offset; + u64 length; +}; + +/* Maximum percentage of just-in-time memory allocation trimming to perform + * on free. + */ +#define BASE_JIT_MAX_TRIM_LEVEL (100) + +/* Maximum number of concurrent just-in-time memory allocations. + */ +#define BASE_JIT_ALLOC_COUNT (255) + +/* base_jit_alloc_info in use for kernel driver versions 10.2 to early 11.5 + * + * jit_version is 1 + * + * Due to the lack of padding specified, user clients between 32 and 64-bit + * may have assumed a different size of the struct + * + * An array of structures was not supported + */ +struct base_jit_alloc_info_10_2 { + u64 gpu_alloc_addr; + u64 va_pages; + u64 commit_pages; + u64 extent; + u8 id; +}; + +/* base_jit_alloc_info introduced by kernel driver version 11.5, and in use up + * to 11.19 + * + * This structure had a number of modifications during and after kernel driver + * version 11.5, but remains size-compatible throughout its version history, and + * with earlier variants compatible with future variants by requiring + * zero-initialization to the unused space in the structure. + * + * jit_version is 2 + * + * Kernel driver version history: + * 11.5: Initial introduction with 'usage_id' and padding[5]. All padding bytes + * must be zero. Kbase minor version was not incremented, so some + * versions of 11.5 do not have this change. + * 11.5: Added 'bin_id' and 'max_allocations', replacing 2 padding bytes (Kbase + * minor version not incremented) + * 11.6: Added 'flags', replacing 1 padding byte + * 11.10: Arrays of this structure are supported + */ +struct base_jit_alloc_info_11_5 { + u64 gpu_alloc_addr; + u64 va_pages; + u64 commit_pages; + u64 extent; + u8 id; + u8 bin_id; + u8 max_allocations; + u8 flags; + u8 padding[2]; + u16 usage_id; +}; + +/** + * struct base_jit_alloc_info - Structure which describes a JIT allocation + * request. + * @gpu_alloc_addr: The GPU virtual address to write the JIT + * allocated GPU virtual address to. + * @va_pages: The minimum number of virtual pages required. + * @commit_pages: The minimum number of physical pages which + * should back the allocation. + * @extent: Granularity of physical pages to grow the + * allocation by during a fault. + * @id: Unique ID provided by the caller, this is used + * to pair allocation and free requests. + * Zero is not a valid value. + * @bin_id: The JIT allocation bin, used in conjunction with + * @max_allocations to limit the number of each + * type of JIT allocation. + * @max_allocations: The maximum number of allocations allowed within + * the bin specified by @bin_id. Should be the same + * for all allocations within the same bin. + * @flags: flags specifying the special requirements for + * the JIT allocation, see + * %BASE_JIT_ALLOC_VALID_FLAGS + * @padding: Expansion space - should be initialised to zero + * @usage_id: A hint about which allocation should be reused. + * The kernel should attempt to use a previous + * allocation with the same usage_id + * @heap_info_gpu_addr: Pointer to an object in GPU memory describing + * the actual usage of the region. + * + * jit_version is 3. + * + * When modifications are made to this structure, it is still compatible with + * jit_version 3 when: a) the size is unchanged, and b) new members only + * replace the padding bytes. + * + * Previous jit_version history: + * jit_version == 1, refer to &base_jit_alloc_info_10_2 + * jit_version == 2, refer to &base_jit_alloc_info_11_5 + * + * Kbase version history: + * 11.20: added @heap_info_gpu_addr + */ +struct base_jit_alloc_info { + u64 gpu_alloc_addr; + u64 va_pages; + u64 commit_pages; + u64 extent; + u8 id; + u8 bin_id; + u8 max_allocations; + u8 flags; + u8 padding[2]; + u16 usage_id; + u64 heap_info_gpu_addr; +}; + +enum base_external_resource_access { + BASE_EXT_RES_ACCESS_SHARED, + BASE_EXT_RES_ACCESS_EXCLUSIVE +}; + +struct base_external_resource { + u64 ext_resource; +}; + + +/** + * The maximum number of external resources which can be mapped/unmapped + * in a single request. + */ +#define BASE_EXT_RES_COUNT_MAX 10 + +/** + * struct base_external_resource_list - Structure which describes a list of + * external resources. + * @count: The number of resources. + * @ext_res: Array of external resources which is + * sized at allocation time. + */ +struct base_external_resource_list { + u64 count; + struct base_external_resource ext_res[1]; +}; + +struct base_jd_debug_copy_buffer { + u64 address; + u64 size; + struct base_external_resource extres; +}; + +#define GPU_MAX_JOB_SLOTS 16 + +/** + * User-side Base GPU Property Queries + * + * The User-side Base GPU Property Query interface encapsulates two + * sub-modules: + * + * - "Dynamic GPU Properties" + * - "Base Platform Config GPU Properties" + * + * Base only deals with properties that vary between different GPU + * implementations - the Dynamic GPU properties and the Platform Config + * properties. + * + * For properties that are constant for the GPU Architecture, refer to the + * GPU module. However, we will discuss their relevance here just to + * provide background information. + * + * About the GPU Properties in Base and GPU modules + * + * The compile-time properties (Platform Config, GPU Compile-time + * properties) are exposed as pre-processor macros. + * + * Complementing the compile-time properties are the Dynamic GPU + * Properties, which act as a conduit for the GPU Configuration + * Discovery. + * + * In general, the dynamic properties are present to verify that the platform + * has been configured correctly with the right set of Platform Config + * Compile-time Properties. + * + * As a consistent guide across the entire DDK, the choice for dynamic or + * compile-time should consider the following, in order: + * 1. Can the code be written so that it doesn't need to know the + * implementation limits at all? + * 2. If you need the limits, get the information from the Dynamic Property + * lookup. This should be done once as you fetch the context, and then cached + * as part of the context data structure, so it's cheap to access. + * 3. If there's a clear and arguable inefficiency in using Dynamic Properties, + * then use a Compile-Time Property (Platform Config, or GPU Compile-time + * property). Examples of where this might be sensible follow: + * - Part of a critical inner-loop + * - Frequent re-use throughout the driver, causing significant extra load + * instructions or control flow that would be worthwhile optimizing out. + * + * We cannot provide an exhaustive set of examples, neither can we provide a + * rule for every possible situation. Use common sense, and think about: what + * the rest of the driver will be doing; how the compiler might represent the + * value if it is a compile-time constant; whether an OEM shipping multiple + * devices would benefit much more from a single DDK binary, instead of + * insignificant micro-optimizations. + * + * Dynamic GPU Properties + * + * Dynamic GPU properties are presented in two sets: + * 1. the commonly used properties in @ref base_gpu_props, which have been + * unpacked from GPU register bitfields. + * 2. The full set of raw, unprocessed properties in gpu_raw_gpu_props + * (also a member of base_gpu_props). All of these are presented in + * the packed form, as presented by the GPU registers themselves. + * + * The raw properties in gpu_raw_gpu_props are necessary to + * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device + * behaving differently?". In this case, all information about the + * configuration is potentially useful, but it does not need to be processed + * by the driver. Instead, the raw registers can be processed by the Mali + * Tools software on the host PC. + * + * The properties returned extend the GPU Configuration Discovery + * registers. For example, GPU clock speed is not specified in the GPU + * Architecture, but is necessary for OpenCL's clGetDeviceInfo() function. + * + * The GPU properties are obtained by a call to + * base_get_gpu_props(). This simply returns a pointer to a const + * base_gpu_props structure. It is constant for the life of a base + * context. Multiple calls to base_get_gpu_props() to a base context + * return the same pointer to a constant structure. This avoids cache pollution + * of the common data. + * + * This pointer must not be freed, because it does not point to the start of a + * region allocated by the memory allocator; instead, just close the @ref + * base_context. + * + * + * Kernel Operation + * + * During Base Context Create time, user-side makes a single kernel call: + * - A call to fill user memory with GPU information structures + * + * The kernel-side will fill the provided the entire processed base_gpu_props + * structure, because this information is required in both + * user and kernel side; it does not make sense to decode it twice. + * + * Coherency groups must be derived from the bitmasks, but this can be done + * kernel side, and just once at kernel startup: Coherency groups must already + * be known kernel-side, to support chains that specify a 'Only Coherent Group' + * SW requirement, or 'Only Coherent Group with Tiler' SW requirement. + * + * Coherency Group calculation + * + * Creation of the coherent group data is done at device-driver startup, and so + * is one-time. This will most likely involve a loop with CLZ, shifting, and + * bit clearing on the L2_PRESENT mask, depending on whether the + * system is L2 Coherent. The number of shader cores is done by a + * population count, since faulty cores may be disabled during production, + * producing a non-contiguous mask. + * + * The memory requirements for this algorithm can be determined either by a u64 + * population count on the L2_PRESENT mask (a LUT helper already is + * required for the above), or simple assumption that there can be no more than + * 16 coherent groups, since core groups are typically 4 cores. + */ + +#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 4 + +#define BASE_MAX_COHERENT_GROUPS 16 + +struct mali_base_gpu_core_props { + /** + * Product specific value. + */ + u32 product_id; + + /** + * Status of the GPU release. + * No defined values, but starts at 0 and increases by one for each + * release status (alpha, beta, EAC, etc.). + * 4 bit values (0-15). + */ + u16 version_status; + + /** + * Minor release number of the GPU. "P" part of an "RnPn" release number. + * 8 bit values (0-255). + */ + u16 minor_revision; + + /** + * Major release number of the GPU. "R" part of an "RnPn" release number. + * 4 bit values (0-15). + */ + u16 major_revision; + + u16 padding; + + /* The maximum GPU frequency. Reported to applications by + * clGetDeviceInfo() + */ + u32 gpu_freq_khz_max; + + /** + * Size of the shader program counter, in bits. + */ + u32 log2_program_counter_size; + + /** + * TEXTURE_FEATURES_x registers, as exposed by the GPU. This is a + * bitpattern where a set bit indicates that the format is supported. + * + * Before using a texture format, it is recommended that the corresponding + * bit be checked. + */ + u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; + + /** + * Theoretical maximum memory available to the GPU. It is unlikely that a + * client will be able to allocate all of this memory for their own + * purposes, but this at least provides an upper bound on the memory + * available to the GPU. + * + * This is required for OpenCL's clGetDeviceInfo() call when + * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The + * client will not be expecting to allocate anywhere near this value. + */ + u64 gpu_available_memory_size; + + /** + * The number of execution engines. + */ + u8 num_exec_engines; +}; + +/** + * + * More information is possible - but associativity and bus width are not + * required by upper-level apis. + */ +struct mali_base_gpu_l2_cache_props { + u8 log2_line_size; + u8 log2_cache_size; + u8 num_l2_slices; /* Number of L2C slices. 1 or higher */ + u8 padding[5]; +}; + +struct mali_base_gpu_tiler_props { + u32 bin_size_bytes; /* Max is 4*2^15 */ + u32 max_active_levels; /* Max is 2^15 */ +}; + +/** + * GPU threading system details. + */ +struct mali_base_gpu_thread_props { + u32 max_threads; /* Max. number of threads per core */ + u32 max_workgroup_size; /* Max. number of threads per workgroup */ + u32 max_barrier_size; /* Max. number of threads that can synchronize on a simple barrier */ + u16 max_registers; /* Total size [1..65535] of the register file available per core. */ + u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */ + u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */ + u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */ + u8 padding[3]; + u32 tls_alloc; /* Number of threads per core that TLS must + * be allocated for + */ +}; + +/** + * struct mali_base_gpu_coherent_group - descriptor for a coherent group + * + * \c core_mask exposes all cores in that coherent group, and \c num_cores + * provides a cached population-count for that mask. + * + * @note Whilst all cores are exposed in the mask, not all may be available to + * the application, depending on the Kernel Power policy. + * + * @note if u64s must be 8-byte aligned, then this structure has 32-bits of wastage. + */ +struct mali_base_gpu_coherent_group { + u64 core_mask; /**< Core restriction mask required for the group */ + u16 num_cores; /**< Number of cores in the group */ + u16 padding[3]; +}; + +/** + * struct mali_base_gpu_coherent_group_info - Coherency group information + * + * Note that the sizes of the members could be reduced. However, the \c group + * member might be 8-byte aligned to ensure the u64 core_mask is 8-byte + * aligned, thus leading to wastage if the other members sizes were reduced. + * + * The groups are sorted by core mask. The core masks are non-repeating and do + * not intersect. + */ +struct mali_base_gpu_coherent_group_info { + u32 num_groups; + + /** + * Number of core groups (coherent or not) in the GPU. Equivalent to the number of L2 Caches. + * + * The GPU Counter dumping writes 2048 bytes per core group, regardless of + * whether the core groups are coherent or not. Hence this member is needed + * to calculate how much memory is required for dumping. + * + * @note Do not use it to work out how many valid elements are in the + * group[] member. Use num_groups instead. + */ + u32 num_core_groups; + + /** + * Coherency features of the memory, accessed by gpu_mem_features + * methods + */ + u32 coherency; + + u32 padding; + + /** + * Descriptors of coherent groups + */ + struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS]; +}; + +/** + * struct gpu_raw_gpu_props - A complete description of the GPU's Hardware + * Configuration Discovery registers. + * + * The information is presented inefficiently for access. For frequent access, + * the values should be better expressed in an unpacked form in the + * base_gpu_props structure. + * + * The raw properties in gpu_raw_gpu_props are necessary to + * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device + * behaving differently?". In this case, all information about the + * configuration is potentially useful, but it does not need to be processed + * by the driver. Instead, the raw registers can be processed by the Mali + * Tools software on the host PC. + * + */ +struct gpu_raw_gpu_props { + u64 shader_present; + u64 tiler_present; + u64 l2_present; + u64 stack_present; + + u32 l2_features; + u32 core_features; + u32 mem_features; + u32 mmu_features; + + u32 as_present; + + u32 js_present; + u32 js_features[GPU_MAX_JOB_SLOTS]; + u32 tiler_features; + u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; + + u32 gpu_id; + + u32 thread_max_threads; + u32 thread_max_workgroup_size; + u32 thread_max_barrier_size; + u32 thread_features; + + /* + * Note: This is the _selected_ coherency mode rather than the + * available modes as exposed in the coherency_features register. + */ + u32 coherency_mode; + + u32 thread_tls_alloc; +}; + +/** + * struct base_gpu_props - Return structure for base_get_gpu_props(). + * + * NOTE: the raw_props member in this data structure contains the register + * values from which the value of the other members are derived. The derived + * members exist to allow for efficient access and/or shielding the details + * of the layout of the registers. + * + * @unused_1: Keep for backwards compatibility. + * @raw_props: This member is large, likely to be 128 bytes. + * @coherency_info: This must be last member of the structure. + */ +struct base_gpu_props { + struct mali_base_gpu_core_props core_props; + struct mali_base_gpu_l2_cache_props l2_props; + u64 unused_1; + struct mali_base_gpu_tiler_props tiler_props; + struct mali_base_gpu_thread_props thread_props; + struct gpu_raw_gpu_props raw_props; + struct mali_base_gpu_coherent_group_info coherency_info; +}; + +#if MALI_USE_CSF +#include "csf/mali_base_csf_kernel.h" +#else +#include "jm/mali_base_jm_kernel.h" +#endif + +/** + * base_mem_group_id_get() - Get group ID from flags + * @flags: Flags to pass to base_mem_alloc + * + * This inline function extracts the encoded group ID from flags + * and converts it into numeric value (0~15). + * + * Return: group ID(0~15) extracted from the parameter + */ +static inline int base_mem_group_id_get(base_mem_alloc_flags flags) +{ + LOCAL_ASSERT((flags & ~BASE_MEM_FLAGS_INPUT_MASK) == 0); + return (int)((flags & BASE_MEM_GROUP_ID_MASK) >> + BASEP_MEM_GROUP_ID_SHIFT); +} + +/** + * base_mem_group_id_set() - Set group ID into base_mem_alloc_flags + * @id: group ID(0~15) you want to encode + * + * This inline function encodes specific group ID into base_mem_alloc_flags. + * Parameter 'id' should lie in-between 0 to 15. + * + * Return: base_mem_alloc_flags with the group ID (id) encoded + * + * The return value can be combined with other flags against base_mem_alloc + * to identify a specific memory group. + */ +static inline base_mem_alloc_flags base_mem_group_id_set(int id) +{ + if ((id < 0) || (id >= BASE_MEM_GROUP_COUNT)) { + /* Set to default value when id is out of range. */ + id = BASE_MEM_GROUP_DEFAULT; + } + + return ((base_mem_alloc_flags)id << BASEP_MEM_GROUP_ID_SHIFT) & + BASE_MEM_GROUP_ID_MASK; +} + +/** + * base_context_mmu_group_id_set - Encode a memory group ID in + * base_context_create_flags + * + * Memory allocated for GPU page tables will come from the specified group. + * + * @group_id: Physical memory group ID. Range is 0..(BASE_MEM_GROUP_COUNT-1). + * + * Return: Bitmask of flags to pass to base_context_init. + */ +static inline base_context_create_flags base_context_mmu_group_id_set( + int const group_id) +{ + LOCAL_ASSERT(group_id >= 0); + LOCAL_ASSERT(group_id < BASE_MEM_GROUP_COUNT); + return BASEP_CONTEXT_MMU_GROUP_ID_MASK & + ((base_context_create_flags)group_id << + BASEP_CONTEXT_MMU_GROUP_ID_SHIFT); +} + +/** + * base_context_mmu_group_id_get - Decode a memory group ID from + * base_context_create_flags + * + * Memory allocated for GPU page tables will come from the returned group. + * + * @flags: Bitmask of flags to pass to base_context_init. + * + * Return: Physical memory group ID. Valid range is 0..(BASE_MEM_GROUP_COUNT-1). + */ +static inline int base_context_mmu_group_id_get( + base_context_create_flags const flags) +{ + LOCAL_ASSERT(flags == (flags & BASEP_CONTEXT_CREATE_ALLOWED_FLAGS)); + return (int)((flags & BASEP_CONTEXT_MMU_GROUP_ID_MASK) >> + BASEP_CONTEXT_MMU_GROUP_ID_SHIFT); +} + +/* + * A number of bit flags are defined for requesting cpu_gpu_timeinfo. These + * flags are also used, where applicable, for specifying which fields + * are valid following the request operation. + */ + +/* For monotonic (counter) timefield */ +#define BASE_TIMEINFO_MONOTONIC_FLAG (1UL << 0) +/* For system wide timestamp */ +#define BASE_TIMEINFO_TIMESTAMP_FLAG (1UL << 1) +/* For GPU cycle counter */ +#define BASE_TIMEINFO_CYCLE_COUNTER_FLAG (1UL << 2) +/* Specify kernel GPU register timestamp */ +#define BASE_TIMEINFO_KERNEL_SOURCE_FLAG (1UL << 30) +/* Specify userspace cntvct_el0 timestamp source */ +#define BASE_TIMEINFO_USER_SOURCE_FLAG (1UL << 31) + +#define BASE_TIMEREQUEST_ALLOWED_FLAGS (\ + BASE_TIMEINFO_MONOTONIC_FLAG | \ + BASE_TIMEINFO_TIMESTAMP_FLAG | \ + BASE_TIMEINFO_CYCLE_COUNTER_FLAG | \ + BASE_TIMEINFO_KERNEL_SOURCE_FLAG | \ + BASE_TIMEINFO_USER_SOURCE_FLAG) + +#endif /* _BASE_KERNEL_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_base_mem_priv.h b/drivers/gpu/arm/bifrost/mali_base_mem_priv.h new file mode 100755 index 000000000000..844a025b715d --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_base_mem_priv.h @@ -0,0 +1,57 @@ +/* + * + * (C) COPYRIGHT 2010-2015, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#ifndef _BASE_MEM_PRIV_H_ +#define _BASE_MEM_PRIV_H_ + +#define BASE_SYNCSET_OP_MSYNC (1U << 0) +#define BASE_SYNCSET_OP_CSYNC (1U << 1) + +/* + * This structure describe a basic memory coherency operation. + * It can either be: + * @li a sync from CPU to Memory: + * - type = ::BASE_SYNCSET_OP_MSYNC + * - mem_handle = a handle to the memory object on which the operation + * is taking place + * - user_addr = the address of the range to be synced + * - size = the amount of data to be synced, in bytes + * - offset is ignored. + * @li a sync from Memory to CPU: + * - type = ::BASE_SYNCSET_OP_CSYNC + * - mem_handle = a handle to the memory object on which the operation + * is taking place + * - user_addr = the address of the range to be synced + * - size = the amount of data to be synced, in bytes. + * - offset is ignored. + */ +struct basep_syncset { + struct base_mem_handle mem_handle; + u64 user_addr; + u64 size; + u8 type; + u8 padding[7]; +}; + +#endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase.h b/drivers/gpu/arm/bifrost/mali_kbase.h new file mode 100755 index 000000000000..8189d02ab910 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase.h @@ -0,0 +1,614 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#ifndef _KBASE_H_ +#define _KBASE_H_ + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) +#include +#endif +#include +#include +#include +#include +#include +#include + +#include "mali_base_kernel.h" +#include + +/* + * Include mali_kbase_defs.h first as this provides types needed by other local + * header files. + */ +#include "mali_kbase_defs.h" + +#include "debug/mali_kbase_debug_ktrace.h" +#include "context/mali_kbase_context.h" +#include "mali_kbase_strings.h" +#include "mali_kbase_mem_lowlevel.h" +#include "mali_kbase_utility.h" +#include "mali_kbase_mem.h" +#include "mmu/mali_kbase_mmu.h" +#include "mali_kbase_gpu_memory_debugfs.h" +#include "mali_kbase_mem_profile_debugfs.h" +#include "mali_kbase_gpuprops.h" +#include "mali_kbase_ioctl.h" +#if !MALI_USE_CSF +#include "mali_kbase_debug_job_fault.h" +#include "mali_kbase_jd_debugfs.h" +#include "mali_kbase_jm.h" +#include "mali_kbase_js.h" +#endif /* !MALI_USE_CSF */ + +#include "ipa/mali_kbase_ipa.h" + +#ifdef CONFIG_GPU_TRACEPOINTS +#include +#endif + +#include "mali_linux_trace.h" + +#if MALI_USE_CSF +#include "csf/mali_kbase_csf.h" +#endif + +#ifndef u64_to_user_ptr +/* Introduced in Linux v4.6 */ +#define u64_to_user_ptr(x) ((void __user *)(uintptr_t)x) +#endif + +#if MALI_USE_CSF +/* Physical memory group ID for command stream frontend user I/O. + */ +#define KBASE_MEM_GROUP_CSF_IO BASE_MEM_GROUP_DEFAULT + +/* Physical memory group ID for command stream frontend firmware. + */ +#define KBASE_MEM_GROUP_CSF_FW BASE_MEM_GROUP_DEFAULT +#endif + +/* Physical memory group ID for a special page which can alias several regions. + */ +#define KBASE_MEM_GROUP_SINK BASE_MEM_GROUP_DEFAULT + +/* + * Kernel-side Base (KBase) APIs + */ + +struct kbase_device *kbase_device_alloc(void); +/* +* note: configuration attributes member of kbdev needs to have +* been setup before calling kbase_device_init +*/ + +int kbase_device_misc_init(struct kbase_device *kbdev); +void kbase_device_misc_term(struct kbase_device *kbdev); +void kbase_device_free(struct kbase_device *kbdev); +int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature); + +/* Needed for gator integration and for reporting vsync information */ +struct kbase_device *kbase_find_device(int minor); +void kbase_release_device(struct kbase_device *kbdev); + +/** + * kbase_context_get_unmapped_area() - get an address range which is currently + * unmapped. + * @kctx: A kernel base context (which has its own GPU address space). + * @addr: CPU mapped address (set to 0 since MAP_FIXED mapping is not allowed + * as Mali GPU driver decides about the mapping). + * @len: Length of the address range. + * @pgoff: Page offset within the GPU address space of the kbase context. + * @flags: Flags for the allocation. + * + * Finds the unmapped address range which satisfies requirements specific to + * GPU and those provided by the call parameters. + * + * 1) Requirement for allocations greater than 2MB: + * - alignment offset is set to 2MB and the alignment mask to 2MB decremented + * by 1. + * + * 2) Requirements imposed for the shader memory alignment: + * - alignment is decided by the number of GPU pc bits which can be read from + * GPU properties of the device associated with this kbase context; alignment + * offset is set to this value in bytes and the alignment mask to the offset + * decremented by 1. + * - allocations must not to be at 4GB boundaries. Such cases are indicated + * by the flag KBASE_REG_GPU_NX not being set (check the flags of the kbase + * region). 4GB boundaries can be checked against @ref BASE_MEM_MASK_4GB. + * + * 3) Requirements imposed for tiler memory alignment, cases indicated by + * the flag @ref KBASE_REG_TILER_ALIGN_TOP (check the flags of the kbase + * region): + * - alignment offset is set to the difference between the kbase region + * extent (converted from the original value in pages to bytes) and the kbase + * region initial_commit (also converted from the original value in pages to + * bytes); alignment mask is set to the kbase region extent in bytes and + * decremented by 1. + * + * Return: if successful, address of the unmapped area aligned as required; + * error code (negative) in case of failure; + */ +unsigned long kbase_context_get_unmapped_area(struct kbase_context *kctx, + const unsigned long addr, const unsigned long len, + const unsigned long pgoff, const unsigned long flags); + + +int assign_irqs(struct kbase_device *kbdev); + +int kbase_sysfs_init(struct kbase_device *kbdev); +void kbase_sysfs_term(struct kbase_device *kbdev); + + +int kbase_protected_mode_init(struct kbase_device *kbdev); +void kbase_protected_mode_term(struct kbase_device *kbdev); + +/** + * kbase_device_pm_init() - Performs power management initialization and + * Verifies device tree configurations. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 if successful, otherwise a standard Linux error code + */ +int kbase_device_pm_init(struct kbase_device *kbdev); + +/** + * kbase_device_pm_term() - Performs power management deinitialization and + * Free resources. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Clean up all the resources + */ +void kbase_device_pm_term(struct kbase_device *kbdev); + + +int power_control_init(struct kbase_device *kbdev); +void power_control_term(struct kbase_device *kbdev); + +#ifdef CONFIG_DEBUG_FS +void kbase_device_debugfs_term(struct kbase_device *kbdev); +int kbase_device_debugfs_init(struct kbase_device *kbdev); +#else /* CONFIG_DEBUG_FS */ +static inline int kbase_device_debugfs_init(struct kbase_device *kbdev) +{ + return 0; +} + +static inline void kbase_device_debugfs_term(struct kbase_device *kbdev) { } +#endif /* CONFIG_DEBUG_FS */ + +int registers_map(struct kbase_device *kbdev); +void registers_unmap(struct kbase_device *kbdev); + +int kbase_device_coherency_init(struct kbase_device *kbdev); + +#ifdef CONFIG_MALI_BUSLOG +int buslog_init(struct kbase_device *kbdev); +void buslog_term(struct kbase_device *kbdev); +#endif + +#if !MALI_USE_CSF +int kbase_jd_init(struct kbase_context *kctx); +void kbase_jd_exit(struct kbase_context *kctx); + +/** + * kbase_jd_submit - Submit atoms to the job dispatcher + * + * @kctx: The kbase context to submit to + * @user_addr: The address in user space of the struct base_jd_atom array + * @nr_atoms: The number of atoms in the array + * @stride: sizeof(struct base_jd_atom) + * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) + * + * Return: 0 on success or error code + */ +int kbase_jd_submit(struct kbase_context *kctx, + void __user *user_addr, u32 nr_atoms, u32 stride, + bool uk6_atom); + +/** + * kbase_jd_done_worker - Handle a job completion + * @data: a &struct work_struct + * + * This function requeues the job from the runpool (if it was soft-stopped or + * removed from NEXT registers). + * + * Removes it from the system if it finished/failed/was cancelled. + * + * Resolves dependencies to add dependent jobs to the context, potentially + * starting them if necessary (which may add more references to the context) + * + * Releases the reference to the context from the no-longer-running job. + * + * Handles retrying submission outside of IRQ context if it failed from within + * IRQ context. + */ +void kbase_jd_done_worker(struct work_struct *data); + +void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp, + kbasep_js_atom_done_code done_code); +void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom); +void kbase_jd_zap_context(struct kbase_context *kctx); +bool jd_done_nolock(struct kbase_jd_atom *katom, + struct list_head *completed_jobs_ctx); +void kbase_jd_free_external_resources(struct kbase_jd_atom *katom); +void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom); + +/** + * kbase_job_done - Process completed jobs from job interrupt + * @kbdev: Pointer to the kbase device. + * @done: Bitmask of done or failed jobs, from JOB_IRQ_STAT register + * + * This function processes the completed, or failed, jobs from the GPU job + * slots, for the bits set in the @done bitmask. + * + * The hwaccess_lock must be held when calling this function. + */ +void kbase_job_done(struct kbase_device *kbdev, u32 done); + +/** + * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms + * and soft stop them + * @kctx: Pointer to context to check. + * @katom: Pointer to priority atom. + * + * Atoms from @kctx on the same job slot as @katom, which have lower priority + * than @katom will be soft stopped and put back in the queue, so that atoms + * with higher priority can run. + * + * The hwaccess_lock must be held when calling this function. + */ +void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +/** + * kbase_job_slot_softstop_start_rp() - Soft-stop the atom at the start + * of a renderpass. + * @kctx: Pointer to a kernel base context. + * @reg: Reference of a growable GPU memory region in the same context. + * Takes ownership of the reference if successful. + * + * Used to switch to incremental rendering if we have nearly run out of + * virtual address space in a growable memory region and the atom currently + * executing on a job slot is the tiler job chain at the start of a renderpass. + * + * Return 0 if successful, otherwise a negative error code. + */ +int kbase_job_slot_softstop_start_rp(struct kbase_context *kctx, + struct kbase_va_region *reg); + +void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom); +void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom, u32 sw_flags); +void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom); +void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, + base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom); +void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, + struct kbase_jd_atom *target_katom); + +#endif /* !MALI_USE_CSF */ + +void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event); +#if !MALI_USE_CSF +int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent); +#endif /* !MALI_USE_CSF */ +int kbase_event_pending(struct kbase_context *ctx); +int kbase_event_init(struct kbase_context *kctx); +void kbase_event_close(struct kbase_context *kctx); +void kbase_event_cleanup(struct kbase_context *kctx); +void kbase_event_wakeup(struct kbase_context *kctx); + +/** + * kbasep_jit_alloc_validate() - Validate the JIT allocation info. + * + * @kctx: Pointer to the kbase context within which the JIT + * allocation is to be validated. + * @info: Pointer to struct @base_jit_alloc_info + * which is to be validated. + * @return: 0 if jit allocation is valid; negative error code otherwise + */ +int kbasep_jit_alloc_validate(struct kbase_context *kctx, + struct base_jit_alloc_info *info); + +/** + * kbase_jit_retry_pending_alloc() - Retry blocked just-in-time memory + * allocations. + * + * @kctx: Pointer to the kbase context within which the just-in-time + * memory allocations are to be retried. + */ +void kbase_jit_retry_pending_alloc(struct kbase_context *kctx); + +/** + * kbase_free_user_buffer() - Free memory allocated for struct + * @kbase_debug_copy_buffer. + * + * @buffer: Pointer to the memory location allocated for the object + * of the type struct @kbase_debug_copy_buffer. + */ +static inline void kbase_free_user_buffer( + struct kbase_debug_copy_buffer *buffer) +{ + struct page **pages = buffer->extres_pages; + int nr_pages = buffer->nr_extres_pages; + + if (pages) { + int i; + + for (i = 0; i < nr_pages; i++) { + struct page *pg = pages[i]; + + if (pg) + put_page(pg); + } + kfree(pages); + } +} + +/** + * kbase_mem_copy_from_extres() - Copy from external resources. + * + * @kctx: kbase context within which the copying is to take place. + * @buf_data: Pointer to the information about external resources: + * pages pertaining to the external resource, number of + * pages to copy. + */ +int kbase_mem_copy_from_extres(struct kbase_context *kctx, + struct kbase_debug_copy_buffer *buf_data); +#if !MALI_USE_CSF +int kbase_process_soft_job(struct kbase_jd_atom *katom); +int kbase_prepare_soft_job(struct kbase_jd_atom *katom); +void kbase_finish_soft_job(struct kbase_jd_atom *katom); +void kbase_cancel_soft_job(struct kbase_jd_atom *katom); +void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev); +void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom); +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom); +#endif +int kbase_soft_event_update(struct kbase_context *kctx, + u64 event, + unsigned char new_status); + +void kbasep_soft_job_timeout_worker(struct timer_list *timer); +void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt); +#endif /* !MALI_USE_CSF */ + +void kbasep_as_do_poke(struct work_struct *work); + +/** + * Check whether a system suspend is in progress, or has already been suspended + * + * The caller should ensure that either kbdev->pm.active_count_lock is held, or + * a dmb was executed recently (to ensure the value is most + * up-to-date). However, without a lock the value could change afterwards. + * + * @return false if a suspend is not in progress + * @return !=false otherwise + */ +static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) +{ + return kbdev->pm.suspending; +} + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +/* + * Check whether a gpu lost is in progress + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Indicates whether a gpu lost has been received and jobs are no longer + * being scheduled + * + * Return: false if gpu is lost + * Return: != false otherwise + */ +static inline bool kbase_pm_is_gpu_lost(struct kbase_device *kbdev) +{ + return (atomic_read(&kbdev->pm.gpu_lost) == 0 ? false : true); +} + +/* + * Set or clear gpu lost state + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @gpu_lost: true to activate GPU lost state, FALSE is deactive it + * + * Puts power management code into gpu lost state or takes it out of the + * state. Once in gpu lost state new GPU jobs will no longer be + * scheduled. + */ +static inline void kbase_pm_set_gpu_lost(struct kbase_device *kbdev, + bool gpu_lost) +{ + atomic_set(&kbdev->pm.gpu_lost, (gpu_lost ? 1 : 0)); +} +#endif + +/** + * kbase_pm_is_active - Determine whether the GPU is active + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This takes into account whether there is an active context reference. + * + * Return: true if the GPU is active, false otherwise + */ +static inline bool kbase_pm_is_active(struct kbase_device *kbdev) +{ + return kbdev->pm.active_count > 0; +} + +/** + * kbase_pm_metrics_start - Start the utilization metrics timer + * @kbdev: Pointer to the kbase device for which to start the utilization + * metrics calculation thread. + * + * Start the timer that drives the metrics calculation, runs the custom DVFS. + */ +void kbase_pm_metrics_start(struct kbase_device *kbdev); + +/** + * kbase_pm_metrics_stop - Stop the utilization metrics timer + * @kbdev: Pointer to the kbase device for which to stop the utilization + * metrics calculation thread. + * + * Stop the timer that drives the metrics calculation, runs the custom DVFS. + */ +void kbase_pm_metrics_stop(struct kbase_device *kbdev); + +#if !MALI_USE_CSF +/** + * Return the atom's ID, as was originally supplied by userspace in + * base_jd_atom::atom_number + */ +static inline int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + int result; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(katom->kctx == kctx); + + result = katom - &kctx->jctx.atoms[0]; + KBASE_DEBUG_ASSERT(result >= 0 && result <= BASE_JD_ATOM_COUNT); + return result; +} + +/** + * kbase_jd_atom_from_id - Return the atom structure for the given atom ID + * @kctx: Context pointer + * @id: ID of atom to retrieve + * + * Return: Pointer to struct kbase_jd_atom associated with the supplied ID + */ +static inline struct kbase_jd_atom *kbase_jd_atom_from_id( + struct kbase_context *kctx, int id) +{ + return &kctx->jctx.atoms[id]; +} +#endif /* !MALI_USE_CSF */ + +/** + * Initialize the disjoint state + * + * The disjoint event count and state are both set to zero. + * + * Disjoint functions usage: + * + * The disjoint event count should be incremented whenever a disjoint event occurs. + * + * There are several cases which are regarded as disjoint behavior. Rather than just increment + * the counter during disjoint events we also increment the counter when jobs may be affected + * by what the GPU is currently doing. To facilitate this we have the concept of disjoint state. + * + * Disjoint state is entered during GPU reset. Increasing the disjoint state also increases + * the count of disjoint events. + * + * The disjoint state is then used to increase the count of disjoint events during job submission + * and job completion. Any atom submitted or completed while the disjoint state is greater than + * zero is regarded as a disjoint event. + * + * The disjoint event counter is also incremented immediately whenever a job is soft stopped + * and during context creation. + * + * @param kbdev The kbase device + * + * Return: 0 on success and non-zero value on failure. + */ +void kbase_disjoint_init(struct kbase_device *kbdev); + +/** + * Increase the count of disjoint events + * called when a disjoint event has happened + * + * @param kbdev The kbase device + */ +void kbase_disjoint_event(struct kbase_device *kbdev); + +/** + * Increase the count of disjoint events only if the GPU is in a disjoint state + * + * This should be called when something happens which could be disjoint if the GPU + * is in a disjoint state. The state refcount keeps track of this. + * + * @param kbdev The kbase device + */ +void kbase_disjoint_event_potential(struct kbase_device *kbdev); + +/** + * Returns the count of disjoint events + * + * @param kbdev The kbase device + * @return the count of disjoint events + */ +u32 kbase_disjoint_event_get(struct kbase_device *kbdev); + +/** + * Increment the refcount state indicating that the GPU is in a disjoint state. + * + * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) + * eventually after the disjoint state has completed @ref kbase_disjoint_state_down + * should be called + * + * @param kbdev The kbase device + */ +void kbase_disjoint_state_up(struct kbase_device *kbdev); + +/** + * Decrement the refcount state + * + * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) + * + * Called after @ref kbase_disjoint_state_up once the disjoint state is over + * + * @param kbdev The kbase device + */ +void kbase_disjoint_state_down(struct kbase_device *kbdev); + +/** + * If a job is soft stopped and the number of contexts is >= this value + * it is reported as a disjoint event + */ +#define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2 + +#if !defined(UINT64_MAX) + #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL) +#endif + +#endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.c new file mode 100755 index 000000000000..76bbfffe03a0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.c @@ -0,0 +1,113 @@ +/* + * + * (C) COPYRIGHT 2016-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include + +#include +#include +#include + +#ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_MALI_BIFROST_DEBUG + +static int kbase_as_fault_read(struct seq_file *sfile, void *data) +{ + uintptr_t as_no = (uintptr_t) sfile->private; + + struct list_head *entry; + const struct list_head *kbdev_list; + struct kbase_device *kbdev = NULL; + + kbdev_list = kbase_device_get_list(); + + list_for_each(entry, kbdev_list) { + kbdev = list_entry(entry, struct kbase_device, entry); + + if (kbdev->debugfs_as_read_bitmap & (1ULL << as_no)) { + + /* don't show this one again until another fault occors */ + kbdev->debugfs_as_read_bitmap &= ~(1ULL << as_no); + + /* output the last page fault addr */ + seq_printf(sfile, "%llu\n", + (u64) kbdev->as[as_no].pf_data.addr); + } + + } + + kbase_device_put_list(kbdev_list); + + return 0; +} + +static int kbase_as_fault_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbase_as_fault_read, in->i_private); +} + +static const struct file_operations as_fault_fops = { + .owner = THIS_MODULE, + .open = kbase_as_fault_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +#endif /* CONFIG_MALI_BIFROST_DEBUG */ +#endif /* CONFIG_DEBUG_FS */ + +/* + * Initialize debugfs entry for each address space + */ +void kbase_as_fault_debugfs_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_MALI_BIFROST_DEBUG + uint i; + char as_name[64]; + struct dentry *debugfs_directory; + + kbdev->debugfs_as_read_bitmap = 0ULL; + + KBASE_DEBUG_ASSERT(kbdev->nr_hw_address_spaces); + KBASE_DEBUG_ASSERT(sizeof(kbdev->as[0].pf_data.addr) == sizeof(u64)); + + debugfs_directory = debugfs_create_dir("address_spaces", + kbdev->mali_debugfs_directory); + + if (debugfs_directory) { + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + snprintf(as_name, ARRAY_SIZE(as_name), "as%u", i); + debugfs_create_file(as_name, S_IRUGO, + debugfs_directory, + (void *)(uintptr_t)i, + &as_fault_fops); + } + } else { + dev_warn(kbdev->dev, + "unable to create address_spaces debugfs directory"); + } + +#endif /* CONFIG_MALI_BIFROST_DEBUG */ +#endif /* CONFIG_DEBUG_FS */ + return; +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h new file mode 100755 index 000000000000..58d7fcf030a4 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h @@ -0,0 +1,50 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_AS_FAULT_DEBUG_FS_H +#define _KBASE_AS_FAULT_DEBUG_FS_H + +/** + * kbase_as_fault_debugfs_init() - Add debugfs files for reporting page faults + * + * @kbdev: Pointer to kbase_device + */ +void kbase_as_fault_debugfs_init(struct kbase_device *kbdev); + +/** + * kbase_as_fault_debugfs_new() - make the last fault available on debugfs + * + * @kbdev: Pointer to kbase_device + * @as_no: The address space the fault occurred on + */ +static inline void +kbase_as_fault_debugfs_new(struct kbase_device *kbdev, int as_no) +{ +#ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_MALI_BIFROST_DEBUG + kbdev->debugfs_as_read_bitmap |= (1ULL << as_no); +#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + return; +} + +#endif /*_KBASE_AS_FAULT_DEBUG_FS_H*/ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_bits.h b/drivers/gpu/arm/bifrost/mali_kbase_bits.h new file mode 100755 index 000000000000..2c110937a792 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_bits.h @@ -0,0 +1,41 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +#ifndef _KBASE_BITS_H_ +#define _KBASE_BITS_H_ + +#if (KERNEL_VERSION(4, 19, 0) <= LINUX_VERSION_CODE) +#include +#else +#include +#endif + +#endif /* _KBASE_BITS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c new file mode 100755 index 000000000000..27a03cf02138 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c @@ -0,0 +1,67 @@ +/* + * + * (C) COPYRIGHT 2012-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Cache Policy API. + */ + +#include "mali_kbase_cache_policy.h" + +/* + * The output flags should be a combination of the following values: + * KBASE_REG_CPU_CACHED: CPU cache should be enabled + * KBASE_REG_GPU_CACHED: GPU cache should be enabled + * + * NOTE: Some components within the GPU might only be able to access memory + * that is KBASE_REG_GPU_CACHED. Refer to the specific GPU implementation for + * more details. + */ +u32 kbase_cache_enabled(u32 flags, u32 nr_pages) +{ + u32 cache_flags = 0; + + CSTD_UNUSED(nr_pages); + + if (!(flags & BASE_MEM_UNCACHED_GPU)) + cache_flags |= KBASE_REG_GPU_CACHED; + + if (flags & BASE_MEM_CACHED_CPU) + cache_flags |= KBASE_REG_CPU_CACHED; + + return cache_flags; +} + + +void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir) +{ + dma_sync_single_for_device(kbdev->dev, handle, size, dir); +} + + +void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir) +{ + dma_sync_single_for_cpu(kbdev->dev, handle, size, dir); +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h new file mode 100755 index 000000000000..8a1e5291bf5f --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h @@ -0,0 +1,50 @@ +/* + * + * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Cache Policy API. + */ + +#ifndef _KBASE_CACHE_POLICY_H_ +#define _KBASE_CACHE_POLICY_H_ + +#include "mali_kbase.h" +#include "mali_base_kernel.h" + +/** + * kbase_cache_enabled - Choose the cache policy for a specific region + * @flags: flags describing attributes of the region + * @nr_pages: total number of pages (backed or not) for the region + * + * Tells whether the CPU and GPU caches should be enabled or not for a specific + * region. + * This function can be modified to customize the cache policy depending on the + * flags and size of the region. + * + * Return: a combination of %KBASE_REG_CPU_CACHED and %KBASE_REG_GPU_CACHED + * depending on the cache policy + */ +u32 kbase_cache_enabled(u32 flags, u32 nr_pages); + +#endif /* _KBASE_CACHE_POLICY_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_caps.h b/drivers/gpu/arm/bifrost/mali_kbase_caps.h new file mode 100755 index 000000000000..b201a60fa6e3 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_caps.h @@ -0,0 +1,65 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/** + * @file mali_kbase_caps.h + * + * Driver Capability Queries. + */ + +#ifndef _KBASE_CAPS_H_ +#define _KBASE_CAPS_H_ + +#include + +typedef enum mali_kbase_cap { + MALI_KBASE_CAP_SYSTEM_MONITOR = 0, + MALI_KBASE_CAP_JIT_PRESSURE_LIMIT, + MALI_KBASE_CAP_MEM_GROW_ON_GPF, + MALI_KBASE_CAP_MEM_PROTECTED, + MALI_KBASE_NUM_CAPS +} mali_kbase_cap; + +extern bool mali_kbase_supports_cap(unsigned long api_version, mali_kbase_cap cap); + +static inline bool mali_kbase_supports_system_monitor(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_SYSTEM_MONITOR); +} + +static inline bool mali_kbase_supports_jit_pressure_limit(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_JIT_PRESSURE_LIMIT); +} + +static inline bool mali_kbase_supports_mem_grow_on_gpf(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_GROW_ON_GPF); +} + +static inline bool mali_kbase_supports_mem_protected(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_PROTECTED); +} + +#endif /* __KBASE_CAPS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ccswe.c b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.c new file mode 100755 index 000000000000..87d5aaa6bb5d --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.c @@ -0,0 +1,105 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_ccswe.h" +#include "mali_kbase_linux.h" + +#include +#include + +static u64 kbasep_ccswe_cycle_at_no_lock( + struct kbase_ccswe *self, u64 timestamp_ns) +{ + s64 diff_s, diff_ns; + u32 gpu_freq; + + lockdep_assert_held(&self->access); + + diff_ns = timestamp_ns - self->timestamp_ns; + gpu_freq = diff_ns > 0 ? self->gpu_freq : self->prev_gpu_freq; + + diff_s = div_s64(diff_ns, NSEC_PER_SEC); + diff_ns -= diff_s * NSEC_PER_SEC; + + return self->cycles_elapsed + diff_s * gpu_freq + + div_s64(diff_ns * gpu_freq, NSEC_PER_SEC); +} + +void kbase_ccswe_init(struct kbase_ccswe *self) +{ + memset(self, 0, sizeof(*self)); + + spin_lock_init(&self->access); +} +KBASE_EXPORT_TEST_API(kbase_ccswe_init); + +u64 kbase_ccswe_cycle_at(struct kbase_ccswe *self, u64 timestamp_ns) +{ + unsigned long flags; + u64 result; + + spin_lock_irqsave(&self->access, flags); + result = kbasep_ccswe_cycle_at_no_lock(self, timestamp_ns); + spin_unlock_irqrestore(&self->access, flags); + + return result; +} +KBASE_EXPORT_TEST_API(kbase_ccswe_cycle_at); + +void kbase_ccswe_freq_change( + struct kbase_ccswe *self, u64 timestamp_ns, u32 gpu_freq) +{ + unsigned long flags; + + spin_lock_irqsave(&self->access, flags); + + /* The time must go only forward. */ + if (WARN_ON(timestamp_ns < self->timestamp_ns)) + goto exit; + + /* If this is the first frequency change, cycles_elapsed is zero. */ + if (self->timestamp_ns) + self->cycles_elapsed = kbasep_ccswe_cycle_at_no_lock( + self, timestamp_ns); + + self->timestamp_ns = timestamp_ns; + self->prev_gpu_freq = self->gpu_freq; + self->gpu_freq = gpu_freq; +exit: + spin_unlock_irqrestore(&self->access, flags); +} +KBASE_EXPORT_TEST_API(kbase_ccswe_freq_change); + +void kbase_ccswe_reset(struct kbase_ccswe *self) +{ + unsigned long flags; + + spin_lock_irqsave(&self->access, flags); + + self->timestamp_ns = 0; + self->cycles_elapsed = 0; + self->gpu_freq = 0; + self->prev_gpu_freq = 0; + + spin_unlock_irqrestore(&self->access, flags); +} + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ccswe.h b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.h new file mode 100755 index 000000000000..3a7cf73d9eac --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.h @@ -0,0 +1,97 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CCSWE_H_ +#define _KBASE_CCSWE_H_ + +#include + +/** + * struct kbase_ccswe - Cycle count software estimator. + * + * @access: Spinlock protecting this structure access. + * @timestamp_ns: Timestamp(ns) when the last frequency change + * occurred. + * @cycles_elapsed: Number of cycles elapsed before the last frequency + * change + * @gpu_freq: Current GPU frequency(Hz) value. + * @prev_gpu_freq: Previous GPU frequency(Hz) before the last frequency + * change. + */ +struct kbase_ccswe { + spinlock_t access; + u64 timestamp_ns; + u64 cycles_elapsed; + u32 gpu_freq; + u32 prev_gpu_freq; +}; + +/** + * kbase_ccswe_init() - initialize the cycle count estimator. + * + * @self: Cycles count software estimator instance. + */ +void kbase_ccswe_init(struct kbase_ccswe *self); + + +/** + * kbase_ccswe_cycle_at() - Estimate cycle count at given timestamp. + * + * @self: Cycles count software estimator instance. + * @timestamp_ns: The timestamp(ns) for cycle count estimation. + * + * The timestamp must be bigger than the timestamp of the penultimate + * frequency change. If only one frequency change occurred, the + * timestamp must be bigger than the timestamp of the frequency change. + * This is to allow the following code to be executed w/o synchronization. + * If lines below executed atomically, it is safe to assume that only + * one frequency change may happen in between. + * + * u64 ts = ktime_get_raw_ns(); + * u64 cycle = kbase_ccswe_cycle_at(&ccswe, ts) + * + * Returns: estimated value of cycle count at a given time. + */ +u64 kbase_ccswe_cycle_at(struct kbase_ccswe *self, u64 timestamp_ns); + +/** + * kbase_ccswe_freq_change() - update GPU frequency. + * + * @self: Cycles count software estimator instance. + * @timestamp_ns: Timestamp(ns) when frequency change occurred. + * @gpu_freq: New GPU frequency value. + * + * The timestamp must be bigger than the timestamp of the previous + * frequency change. The function is to be called at the frequency + * change moment (not later). + */ +void kbase_ccswe_freq_change( + struct kbase_ccswe *self, u64 timestamp_ns, u32 gpu_freq); + +/** + * kbase_ccswe_reset() - reset estimator state + * + * @self: Cycles count software estimator instance. + */ +void kbase_ccswe_reset(struct kbase_ccswe *self); + +#endif /* _KBASE_CCSWE_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config.c b/drivers/gpu/arm/bifrost/mali_kbase_config.c new file mode 100755 index 000000000000..ce7070d1d634 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_config.c @@ -0,0 +1,48 @@ +/* + * + * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#include +#include +#include + +int kbasep_platform_device_init(struct kbase_device *kbdev) +{ + struct kbase_platform_funcs_conf *platform_funcs_p; + + platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; + if (platform_funcs_p && platform_funcs_p->platform_init_func) + return platform_funcs_p->platform_init_func(kbdev); + + return 0; +} + +void kbasep_platform_device_term(struct kbase_device *kbdev) +{ + struct kbase_platform_funcs_conf *platform_funcs_p; + + platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; + if (platform_funcs_p && platform_funcs_p->platform_term_func) + platform_funcs_p->platform_term_func(kbdev); +} + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config.h b/drivers/gpu/arm/bifrost/mali_kbase_config.h new file mode 100755 index 000000000000..57456e2b90db --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_config.h @@ -0,0 +1,393 @@ +/* + * + * (C) COPYRIGHT 2010-2017, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_config.h + * Configuration API and Attributes for KBase + */ + +#ifndef _KBASE_CONFIG_H_ +#define _KBASE_CONFIG_H_ + +#include +#include +#include +#include + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_config Configuration API and Attributes + * @{ + */ + +/* Forward declaration of struct kbase_device */ +struct kbase_device; + +/** + * kbase_platform_funcs_conf - Specifies platform init/term function pointers + * + * Specifies the functions pointers for platform specific initialization and + * termination. By default no functions are required. No additional platform + * specific control is necessary. + */ +struct kbase_platform_funcs_conf { + /** + * platform_init_func - platform specific init function pointer + * @kbdev - kbase_device pointer + * + * Returns 0 on success, negative error code otherwise. + * + * Function pointer for platform specific initialization or NULL if no + * initialization function is required. At the point this the GPU is + * not active and its power and clocks are in unknown (platform specific + * state) as kbase doesn't yet have control of power and clocks. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed (and possibly initialized) in here. + */ + int (*platform_init_func)(struct kbase_device *kbdev); + /** + * platform_term_func - platform specific termination function pointer + * @kbdev - kbase_device pointer + * + * Function pointer for platform specific termination or NULL if no + * termination function is required. At the point this the GPU will be + * idle but still powered and clocked. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed (and possibly terminated) in here. + */ + void (*platform_term_func)(struct kbase_device *kbdev); +}; + +/* + * @brief Specifies the callbacks for power management + * + * By default no callbacks will be made and the GPU must not be powered off. + */ +struct kbase_pm_callback_conf { + /** Callback for when the GPU is idle and the power to it can be switched off. + * + * The system integrator can decide whether to either do nothing, just switch off + * the clocks to the GPU, or to completely power down the GPU. + * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the + * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). + */ + void (*power_off_callback)(struct kbase_device *kbdev); + + /** Callback for when the GPU is about to become active and power must be supplied. + * + * This function must not return until the GPU is powered and clocked sufficiently for register access to + * succeed. The return value specifies whether the GPU was powered down since the call to power_off_callback. + * If the GPU state has been lost then this function must return 1, otherwise it should return 0. + * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the + * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). + * + * The return value of the first call to this function is ignored. + * + * @return 1 if the GPU state may have been lost, 0 otherwise. + */ + int (*power_on_callback)(struct kbase_device *kbdev); + + /** Callback for when the system is requesting a suspend and GPU power + * must be switched off. + * + * Note that if this callback is present, then this may be called + * without a preceding call to power_off_callback. Therefore this + * callback must be able to take any action that might otherwise happen + * in power_off_callback. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed and modified in here. It is the platform \em + * callbacks responsibility to initialize and terminate this pointer if + * used (see @ref kbase_platform_funcs_conf). + */ + void (*power_suspend_callback)(struct kbase_device *kbdev); + + /** Callback for when the system is resuming from a suspend and GPU + * power must be switched on. + * + * Note that if this callback is present, then this may be called + * without a following call to power_on_callback. Therefore this + * callback must be able to take any action that might otherwise happen + * in power_on_callback. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed and modified in here. It is the platform \em + * callbacks responsibility to initialize and terminate this pointer if + * used (see @ref kbase_platform_funcs_conf). + */ + void (*power_resume_callback)(struct kbase_device *kbdev); + + /** Callback for handling runtime power management initialization. + * + * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback + * will become active from calls made to the OS from within this function. + * The runtime calls can be triggered by calls from @ref power_off_callback and @ref power_on_callback. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * + * @return 0 on success, else int error code. + */ + int (*power_runtime_init_callback)(struct kbase_device *kbdev); + + /** Callback for handling runtime power management termination. + * + * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback + * should no longer be called by the OS on completion of this function. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + */ + void (*power_runtime_term_callback)(struct kbase_device *kbdev); + + /** Callback for runtime power-off power management callback + * + * For linux this callback will be called by the kernel runtime_suspend callback. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * + * @return 0 on success, else OS error code. + */ + void (*power_runtime_off_callback)(struct kbase_device *kbdev); + + /** Callback for runtime power-on power management callback + * + * For linux this callback will be called by the kernel runtime_resume callback. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + */ + int (*power_runtime_on_callback)(struct kbase_device *kbdev); + + /* + * Optional callback for checking if GPU can be suspended when idle + * + * This callback will be called by the runtime power management core + * when the reference count goes to 0 to provide notification that the + * GPU now seems idle. + * + * If this callback finds that the GPU can't be powered off, or handles + * suspend by powering off directly or queueing up a power off, a + * non-zero value must be returned to prevent the runtime PM core from + * also triggering a suspend. + * + * Returning 0 will cause the runtime PM core to conduct a regular + * autosuspend. + * + * This callback is optional and if not provided regular autosuspend + * will be triggered. + * + * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use + * this feature. + * + * Return 0 if GPU can be suspended, positive value if it can not be + * suspeneded by runtime PM, else OS error code + */ + int (*power_runtime_idle_callback)(struct kbase_device *kbdev); + + /* + * Optional callback for software reset + * + * This callback will be called by the power management core to trigger + * a GPU soft reset. + * + * Return 0 if the soft reset was successful and the RESET_COMPLETED + * interrupt will be raised, or a positive value if the interrupt won't + * be raised. On error, return the corresponding OS error code. + */ + int (*soft_reset_callback)(struct kbase_device *kbdev); +}; + +/* struct kbase_gpu_clk_notifier_data - Data for clock rate change notifier. + * + * Pointer to this structure is supposed to be passed to the gpu clock rate + * change notifier function. This structure is deliberately aligned with the + * common clock framework notification structure 'struct clk_notifier_data' + * and such alignment should be maintained. + * + * @gpu_clk_handle: Handle of the GPU clock for which notifier was registered. + * @old_rate: Previous rate of this GPU clock. + * @new_rate: New rate of this GPU clock. + */ +struct kbase_gpu_clk_notifier_data { + void *gpu_clk_handle; + unsigned long old_rate; + unsigned long new_rate; +}; + +/** + * kbase_clk_rate_trace_op_conf - Specifies GPU clock rate trace operations. + * + * Specifies the functions pointers for platform specific GPU clock rate trace + * operations. By default no functions are required. + */ +struct kbase_clk_rate_trace_op_conf { + /** + * enumerate_gpu_clk - Enumerate a GPU clock on the given index + * @kbdev - kbase_device pointer + * @index - GPU clock index + * + * Returns a handle unique to the given GPU clock, or NULL if the clock + * array has been exhausted at the given index value. + * + * Kbase will use this function pointer to enumerate the existence of a + * GPU clock on the given index. + */ + void *(*enumerate_gpu_clk)(struct kbase_device *kbdev, + unsigned int index); + + /** + * get_gpu_clk_rate - Get the current rate for an enumerated clock. + * @kbdev - kbase_device pointer + * @gpu_clk_handle - Handle unique to the enumerated GPU clock + * + * Returns current rate of the GPU clock in unit of Hz. + */ + unsigned long (*get_gpu_clk_rate)(struct kbase_device *kbdev, + void *gpu_clk_handle); + + /** + * gpu_clk_notifier_register - Register a clock rate change notifier. + * @kbdev - kbase_device pointer + * @gpu_clk_handle - Handle unique to the enumerated GPU clock + * @nb - notifier block containing the callback function + * pointer + * + * Returns 0 on success, negative error code otherwise. + * + * This function pointer is used to register a callback function that + * is supposed to be invoked whenever the rate of clock corresponding + * to @gpu_clk_handle changes. + * @nb contains the pointer to callback function. + * The callback function expects the pointer of type + * 'struct kbase_gpu_clk_notifier_data' as the third argument. + */ + int (*gpu_clk_notifier_register)(struct kbase_device *kbdev, + void *gpu_clk_handle, struct notifier_block *nb); + + /** + * gpu_clk_notifier_unregister - Unregister clock rate change notifier + * @kbdev - kbase_device pointer + * @gpu_clk_handle - Handle unique to the enumerated GPU clock + * @nb - notifier block containing the callback function + * pointer + * + * This function pointer is used to unregister a callback function that + * was previously registered to get notified of the change in rate + * of clock corresponding to @gpu_clk_handle. + */ + void (*gpu_clk_notifier_unregister)(struct kbase_device *kbdev, + void *gpu_clk_handle, struct notifier_block *nb); +}; + +#ifdef CONFIG_OF +struct kbase_platform_config { +}; +#else + +/* + * @brief Specifies start and end of I/O memory region. + */ +struct kbase_io_memory_region { + u64 start; + u64 end; +}; + +/* + * @brief Specifies I/O related resources like IRQs and memory region for I/O operations. + */ +struct kbase_io_resources { + u32 job_irq_number; + u32 mmu_irq_number; + u32 gpu_irq_number; + struct kbase_io_memory_region io_memory_region; +}; + +struct kbase_platform_config { + const struct kbase_io_resources *io_resources; +}; + +#endif /* CONFIG_OF */ + +/** + * @brief Gets the pointer to platform config. + * + * @return Pointer to the platform config + */ +struct kbase_platform_config *kbase_get_platform_config(void); + +/** + * kbasep_platform_device_init: - Platform specific call to initialize hardware + * @kbdev: kbase device pointer + * + * Function calls a platform defined routine if specified in the configuration + * attributes. The routine can initialize any hardware and context state that + * is required for the GPU block to function. + * + * Return: 0 if no errors have been found in the config. + * Negative error code otherwise. + */ +int kbasep_platform_device_init(struct kbase_device *kbdev); + +/** + * kbasep_platform_device_term - Platform specific call to terminate hardware + * @kbdev: Kbase device pointer + * + * Function calls a platform defined routine if specified in the configuration + * attributes. The routine can destroy any platform specific context state and + * shut down any hardware functionality that are outside of the Power Management + * callbacks. + * + */ +void kbasep_platform_device_term(struct kbase_device *kbdev); + +#ifndef CONFIG_OF +/** + * kbase_platform_register - Register a platform device for the GPU + * + * This can be used to register a platform device on systems where device tree + * is not enabled and the platform initialisation code in the kernel doesn't + * create the GPU device. Where possible device tree should be used instead. + * + * Return: 0 for success, any other fail causes module initialisation to fail + */ +int kbase_platform_register(void); + +/** + * kbase_platform_unregister - Unregister a fake platform device + * + * Unregister the platform device created with kbase_platform_register() + */ +void kbase_platform_unregister(void); +#endif + + /** @} *//* end group kbase_config */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_CONFIG_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h new file mode 100755 index 000000000000..e079281127ab --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h @@ -0,0 +1,213 @@ +/* + * + * (C) COPYRIGHT 2013-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file mali_kbase_config_defaults.h + * + * Default values for configuration settings + * + */ + +#ifndef _KBASE_CONFIG_DEFAULTS_H_ +#define _KBASE_CONFIG_DEFAULTS_H_ + +/* Include mandatory definitions per platform */ +#include + +enum { + /** + * Use unrestricted Address ID width on the AXI bus. + */ + KBASE_AID_32 = 0x0, + + /** + * Restrict GPU to a half of maximum Address ID count. + * This will reduce performance, but reduce bus load due to GPU. + */ + KBASE_AID_16 = 0x3, + + /** + * Restrict GPU to a quarter of maximum Address ID count. + * This will reduce performance, but reduce bus load due to GPU. + */ + KBASE_AID_8 = 0x2, + + /** + * Restrict GPU to an eighth of maximum Address ID count. + * This will reduce performance, but reduce bus load due to GPU. + */ + KBASE_AID_4 = 0x1 +}; + +enum { + /** + * Use unrestricted Address ID width on the AXI bus. + * Restricting ID width will reduce performance & bus load due to GPU. + */ + KBASE_3BIT_AID_32 = 0x0, + + /* Restrict GPU to 7/8 of maximum Address ID count. */ + KBASE_3BIT_AID_28 = 0x1, + + /* Restrict GPU to 3/4 of maximum Address ID count. */ + KBASE_3BIT_AID_24 = 0x2, + + /* Restrict GPU to 5/8 of maximum Address ID count. */ + KBASE_3BIT_AID_20 = 0x3, + + /* Restrict GPU to 1/2 of maximum Address ID count. */ + KBASE_3BIT_AID_16 = 0x4, + + /* Restrict GPU to 3/8 of maximum Address ID count. */ + KBASE_3BIT_AID_12 = 0x5, + + /* Restrict GPU to 1/4 of maximum Address ID count. */ + KBASE_3BIT_AID_8 = 0x6, + + /* Restrict GPU to 1/8 of maximum Address ID count. */ + KBASE_3BIT_AID_4 = 0x7 +}; + +/** + * Default period for DVFS sampling + */ +#define DEFAULT_PM_DVFS_PERIOD 100 /* 100ms */ + +/** + * Power Management poweroff tick granuality. This is in nanoseconds to + * allow HR timer support. + * + * On each scheduling tick, the power manager core may decide to: + * -# Power off one or more shader cores + * -# Power off the entire GPU + */ +#define DEFAULT_PM_GPU_POWEROFF_TICK_NS (400000) /* 400us */ + +/** + * Power Manager number of ticks before shader cores are powered off + */ +#define DEFAULT_PM_POWEROFF_TICK_SHADER (2) /* 400-800us */ + +/** + * Default scheduling tick granuality + */ +#define DEFAULT_JS_SCHEDULING_PERIOD_NS (100000000u) /* 100ms */ + +/** + * Default minimum number of scheduling ticks before jobs are soft-stopped. + * + * This defines the time-slice for a job (which may be different from that of a + * context) + */ +#define DEFAULT_JS_SOFT_STOP_TICKS (1) /* 100ms-200ms */ + +/** + * Default minimum number of scheduling ticks before CL jobs are soft-stopped. + */ +#define DEFAULT_JS_SOFT_STOP_TICKS_CL (1) /* 100ms-200ms */ + +/** + * Default minimum number of scheduling ticks before jobs are hard-stopped + */ +#define DEFAULT_JS_HARD_STOP_TICKS_SS (50) /* 5s */ + +/** + * Default minimum number of scheduling ticks before CL jobs are hard-stopped. + */ +#define DEFAULT_JS_HARD_STOP_TICKS_CL (50) /* 5s */ + +/** + * Default minimum number of scheduling ticks before jobs are hard-stopped + * during dumping + */ +#define DEFAULT_JS_HARD_STOP_TICKS_DUMPING (15000) /* 1500s */ + +/** + * Default timeout for some software jobs, after which the software event wait + * jobs will be cancelled. + */ +#define DEFAULT_JS_SOFT_JOB_TIMEOUT (3000) /* 3s */ + +/** + * Default minimum number of scheduling ticks before the GPU is reset to clear a + * "stuck" job + */ +#define DEFAULT_JS_RESET_TICKS_SS (55) /* 5.5s */ + +/** + * Default minimum number of scheduling ticks before the GPU is reset to clear a + * "stuck" CL job. + */ +#define DEFAULT_JS_RESET_TICKS_CL (55) /* 5.5s */ + +/** + * Default minimum number of scheduling ticks before the GPU is reset to clear a + * "stuck" job during dumping. + */ +#define DEFAULT_JS_RESET_TICKS_DUMPING (15020) /* 1502s */ + +/** + * Default number of milliseconds given for other jobs on the GPU to be + * soft-stopped when the GPU needs to be reset. + */ +#define DEFAULT_RESET_TIMEOUT_MS (3000) /* 3s */ + +/** + * Default timeslice that a context is scheduled in for, in nanoseconds. + * + * When a context has used up this amount of time across its jobs, it is + * scheduled out to let another run. + * + * @note the resolution is nanoseconds (ns) here, because that's the format + * often used by the OS. + */ +#define DEFAULT_JS_CTX_TIMESLICE_NS (50000000) /* 50ms */ + +/** + * Maximum frequency (in kHz) that the GPU can be clocked. For some platforms + * this isn't available, so we simply define a dummy value here. If devfreq + * is enabled the value will be read from there, otherwise this should be + * overridden by defining GPU_FREQ_KHZ_MAX in the platform file. + */ +#define DEFAULT_GPU_FREQ_KHZ_MAX (5000) + +/** + * Default timeout for task execution on an endpoint + * + * Number of GPU clock cycles before the driver terminates a task that is + * making no forward progress on an endpoint (e.g. shader core). + * Value chosen is equivalent to the time after which a job is hard stopped + * which is 5 seconds (assuming the GPU is usually clocked at ~500 MHZ). + */ +#define DEFAULT_PROGRESS_TIMEOUT ((u64)5 * 500 * 1024 * 1024) + +/** + * Default threshold at which to switch to incremental rendering + * + * Fraction of the maximum size of an allocation that grows on GPU page fault + * that can be used up before the driver switches to incremental rendering, + * in 256ths. 0 means disable incremental rendering. + */ +#define DEFAULT_IR_THRESHOLD (192) + +#endif /* _KBASE_CONFIG_DEFAULTS_H_ */ + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c new file mode 100755 index 000000000000..071b9236dee0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c @@ -0,0 +1,5001 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include +#ifdef CONFIG_MALI_BIFROST_DEVFREQ +#include +#include +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif /* CONFIG_DEVFREQ_THERMAL */ +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ +#ifdef CONFIG_MALI_BIFROST_NO_MALI +#include "mali_kbase_model_linux.h" +#include +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ +#include "mali_kbase_mem_profile_debugfs_buf_size.h" +#include "mali_kbase_debug_mem_view.h" +#include "mali_kbase_mem.h" +#include "mali_kbase_mem_pool_debugfs.h" +#include "mali_kbase_debugfs_helper.h" +#if !MALI_CUSTOMER_RELEASE +#include "mali_kbase_regs_dump_debugfs.h" +#endif /* !MALI_CUSTOMER_RELEASE */ +#include "mali_kbase_regs_history_debugfs.h" +#include +#include +#if !MALI_USE_CSF +#include +#endif /* !MALI_USE_CSF */ +#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS +#include +#endif +#include +#include +#include "mali_kbase_ioctl.h" +#if !MALI_USE_CSF +#include "mali_kbase_kinstr_jm.h" +#endif +#include "mali_kbase_hwcnt_context.h" +#include "mali_kbase_hwcnt_virtualizer.h" +#include "mali_kbase_hwcnt_legacy.h" +#include "mali_kbase_vinstr.h" +#if MALI_USE_CSF +#include "csf/mali_kbase_csf_firmware.h" +#include "csf/mali_kbase_csf_tiler_heap.h" +#include "csf/mali_kbase_csf_kcpu_debugfs.h" +#include "csf/mali_kbase_csf_csg_debugfs.h" +#endif +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include "arbiter/mali_kbase_arbiter_pm.h" +#endif + +#include "mali_kbase_cs_experimental.h" + +#ifdef CONFIG_MALI_CINSTR_GWT +#include "mali_kbase_gwt.h" +#endif +#include "mali_kbase_pm_internal.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* is_compat_task/in_compat_syscall */ +#include +#include +#include +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ +#include +#include +#include +#include + +#include + + +#if (KERNEL_VERSION(3, 13, 0) <= LINUX_VERSION_CODE) +#include +#include +#else +#include +#endif + +#include + +#include + +#include +#include +#include + +#include + +/* GPU IRQ Tags */ +#define JOB_IRQ_TAG 0 +#define MMU_IRQ_TAG 1 +#define GPU_IRQ_TAG 2 + +#define KERNEL_SIDE_DDK_VERSION_STRING "K:" MALI_RELEASE_NAME "(GPL)" + +/** + * Kernel min/maj <=> API Version + */ +#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \ + (((minor) & 0xFFF) << 8) | \ + ((0 & 0xFF) << 0)) + +#define KBASE_API_MIN(api_version) ((api_version >> 8) & 0xFFF) +#define KBASE_API_MAJ(api_version) ((api_version >> 20) & 0xFFF) + +/** + * mali_kbase_api_version_to_maj_min - convert an api_version to a min/maj pair + * + * @api_version: API version to convert + * @major: Major version number (must not exceed 12 bits) + * @minor: Major version number (must not exceed 12 bits) + */ +void mali_kbase_api_version_to_maj_min(unsigned long api_version, u16 *maj, u16 *min) +{ + if (WARN_ON(!maj)) + return; + + if (WARN_ON(!min)) + return; + + *maj = KBASE_API_MAJ(api_version); + *min = KBASE_API_MIN(api_version); +} + +/** + * kbase capabilities table + */ +typedef struct mali_kbase_capability_def { + u16 required_major; + u16 required_minor; +} mali_kbase_capability_def; + +/** + * This must be kept in-sync with mali_kbase_cap + * + * TODO: The alternative approach would be to embed the cap enum values + * in the table. Less efficient but potentially safer. + */ +static mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CAPS] = { +#if MALI_USE_CSF + { 1, 0 }, /* SYSTEM_MONITOR */ + { 1, 0 }, /* JIT_PRESSURE_LIMIT */ + { 1, 0 }, /* MEM_GROW_ON_GPF */ + { 1, 0 } /* MEM_PROTECTED */ +#else + { 11, 15 }, /* SYSTEM_MONITOR */ + { 11, 25 }, /* JIT_PRESSURE_LIMIT */ + { 11, 2 }, /* MEM_GROW_ON_GPF */ + { 11, 2 } /* MEM_PROTECTED */ +#endif +}; + +/** + * mali_kbase_supports_cap - Query whether a kbase capability is supported + * + * @api_version: API version to convert + * @cap: Capability to query for - see mali_kbase_caps.h + */ +bool mali_kbase_supports_cap(unsigned long api_version, mali_kbase_cap cap) +{ + bool supported = false; + unsigned long required_ver; + + mali_kbase_capability_def const *cap_def; + + if (WARN_ON(cap < 0)) + return false; + + if (WARN_ON(cap >= MALI_KBASE_NUM_CAPS)) + return false; + + cap_def = &kbase_caps_table[(int)cap]; + required_ver = KBASE_API_VERSION(cap_def->required_major, cap_def->required_minor); + supported = (api_version >= required_ver); + + return supported; +} + +/** + * kbase_file_new - Create an object representing a device file + * + * @kbdev: An instance of the GPU platform device, allocated from the probe + * method of the driver. + * @filp: Pointer to the struct file corresponding to device file + * /dev/malixx instance, passed to the file's open method. + * + * In its initial state, the device file has no context (i.e. no GPU + * address space) and no API version number. Both must be assigned before + * kbase_file_get_kctx_if_setup_complete() can be used successfully. + * + * @return Address of an object representing a simulated device file, or NULL + * on failure. + */ +static struct kbase_file *kbase_file_new(struct kbase_device *const kbdev, + struct file *const filp) +{ + struct kbase_file *const kfile = kmalloc(sizeof(*kfile), GFP_KERNEL); + + if (kfile) { + kfile->kbdev = kbdev; + kfile->filp = filp; + kfile->kctx = NULL; + kfile->api_version = 0; + atomic_set(&kfile->setup_state, KBASE_FILE_NEED_VSN); + } + return kfile; +} + +/** + * kbase_file_set_api_version - Set the application programmer interface version + * + * @kfile: A device file created by kbase_file_new() + * @major: Major version number (must not exceed 12 bits) + * @minor: Major version number (must not exceed 12 bits) + * + * An application programmer interface (API) version must be specified + * before calling kbase_file_create_kctx(), otherwise an error is returned. + * + * If a version number was already set for the given @kfile (or is in the + * process of being set by another thread) then an error is returned. + * + * Return: 0 if successful, otherwise a negative error code. + */ +static int kbase_file_set_api_version(struct kbase_file *const kfile, + u16 const major, u16 const minor) +{ + if (WARN_ON(!kfile)) + return -EINVAL; + + /* setup pending, try to signal that we'll do the setup, + * if setup was already in progress, err this call + */ + if (atomic_cmpxchg(&kfile->setup_state, KBASE_FILE_NEED_VSN, + KBASE_FILE_VSN_IN_PROGRESS) != KBASE_FILE_NEED_VSN) + return -EPERM; + + /* save the proposed version number for later use */ + kfile->api_version = KBASE_API_VERSION(major, minor); + + atomic_set(&kfile->setup_state, KBASE_FILE_NEED_CTX); + return 0; +} + +/** + * kbase_file_get_api_version - Get the application programmer interface version + * + * @kfile: A device file created by kbase_file_new() + * + * Return: The version number (encoded with KBASE_API_VERSION) or 0 if none has + * been set. + */ +static unsigned long kbase_file_get_api_version(struct kbase_file *const kfile) +{ + if (WARN_ON(!kfile)) + return 0; + + if (atomic_read(&kfile->setup_state) < KBASE_FILE_NEED_CTX) + return 0; + + return kfile->api_version; +} + +/** + * kbase_file_create_kctx - Create a kernel base context + * + * @kfile: A device file created by kbase_file_new() + * @flags: Flags to set, which can be any combination of + * BASEP_CONTEXT_CREATE_KERNEL_FLAGS. + * + * This creates a new context for the GPU platform device instance that was + * specified when kbase_file_new() was called. Each context has its own GPU + * address space. If a context was already created for the given @kfile (or is + * in the process of being created for it by another thread) then an error is + * returned. + * + * An API version number must have been set by kbase_file_set_api_version() + * before calling this function, otherwise an error is returned. + * + * Return: 0 if a new context was created, otherwise a negative error code. + */ +static int kbase_file_create_kctx(struct kbase_file *kfile, + base_context_create_flags flags); + +/** + * kbase_file_get_kctx_if_setup_complete - Get a kernel base context + * pointer from a device file + * + * @kfile: A device file created by kbase_file_new() + * + * This function returns an error code (encoded with ERR_PTR) if no context + * has been created for the given @kfile. This makes it safe to use in + * circumstances where the order of initialization cannot be enforced, but + * only if the caller checks the return value. + * + * Return: Address of the kernel base context associated with the @kfile, or + * NULL if no context exists. + */ +static struct kbase_context *kbase_file_get_kctx_if_setup_complete( + struct kbase_file *const kfile) +{ + if (WARN_ON(!kfile) || + atomic_read(&kfile->setup_state) != KBASE_FILE_COMPLETE || + WARN_ON(!kfile->kctx)) + return NULL; + + return kfile->kctx; +} + +/** + * kbase_file_delete - Destroy an object representing a device file + * + * @kfile: A device file created by kbase_file_new() + * + * If any context was created for the @kfile then it is destroyed. + */ +static void kbase_file_delete(struct kbase_file *const kfile) +{ + struct kbase_device *kbdev = NULL; + + if (WARN_ON(!kfile)) + return; + + kfile->filp->private_data = NULL; + kbdev = kfile->kbdev; + + if (atomic_read(&kfile->setup_state) == KBASE_FILE_COMPLETE) { + struct kbase_context *kctx = kfile->kctx; + +#ifdef CONFIG_DEBUG_FS + kbasep_mem_profile_debugfs_remove(kctx); +#endif + + mutex_lock(&kctx->legacy_hwcnt_lock); + /* If this client was performing hardware counter dumping and + * did not explicitly detach itself, destroy it now + */ + kbase_hwcnt_legacy_client_destroy(kctx->legacy_hwcnt_cli); + kctx->legacy_hwcnt_cli = NULL; + mutex_unlock(&kctx->legacy_hwcnt_lock); + + kbase_context_debugfs_term(kctx); + + kbase_destroy_context(kctx); + + dev_dbg(kbdev->dev, "deleted base context\n"); + } + + kbase_release_device(kbdev); + + kfree(kfile); +} + +static int kbase_api_handshake(struct kbase_file *kfile, + struct kbase_ioctl_version_check *version) +{ + int err = 0; + + switch (version->major) { + case BASE_UK_VERSION_MAJOR: + /* set minor to be the lowest common */ + version->minor = min_t(int, BASE_UK_VERSION_MINOR, + (int)version->minor); + break; + default: + /* We return our actual version regardless if it + * matches the version returned by userspace - + * userspace can bail if it can't handle this + * version + */ + version->major = BASE_UK_VERSION_MAJOR; + version->minor = BASE_UK_VERSION_MINOR; + break; + } + + /* save the proposed version number for later use */ + err = kbase_file_set_api_version(kfile, version->major, version->minor); + if (unlikely(err)) + return err; + + /* For backward compatibility, we may need to create the context before + * the flags have been set. Originally it was created on file open + * (with job submission disabled) but we don't support that usage. + */ + if (!mali_kbase_supports_system_monitor(kbase_file_get_api_version(kfile))) + err = kbase_file_create_kctx(kfile, + BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED); + + return err; +} + +static int kbase_api_handshake_dummy(struct kbase_file *kfile, + struct kbase_ioctl_version_check *version) +{ + return -EPERM; +} + +/** + * enum mali_error - Mali error codes shared with userspace + * + * This is subset of those common Mali errors that can be returned to userspace. + * Values of matching user and kernel space enumerators MUST be the same. + * MALI_ERROR_NONE is guaranteed to be 0. + * + * @MALI_ERROR_NONE: Success + * @MALI_ERROR_OUT_OF_GPU_MEMORY: Not used in the kernel driver + * @MALI_ERROR_OUT_OF_MEMORY: Memory allocation failure + * @MALI_ERROR_FUNCTION_FAILED: Generic error code + */ +enum mali_error { + MALI_ERROR_NONE = 0, + MALI_ERROR_OUT_OF_GPU_MEMORY, + MALI_ERROR_OUT_OF_MEMORY, + MALI_ERROR_FUNCTION_FAILED, +}; + +static struct kbase_device *to_kbase_device(struct device *dev) +{ + return dev_get_drvdata(dev); +} + +int assign_irqs(struct kbase_device *kbdev) +{ + struct platform_device *pdev; + int i; + + if (!kbdev) + return -ENODEV; + + pdev = to_platform_device(kbdev->dev); + /* 3 IRQ resources */ + for (i = 0; i < 3; i++) { + struct resource *irq_res; + int irqtag; + + irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, i); + if (!irq_res) { + dev_err(kbdev->dev, "No IRQ resource at index %d\n", i); + return -ENOENT; + } + +#ifdef CONFIG_OF + if (!strncasecmp(irq_res->name, "JOB", 4)) { + irqtag = JOB_IRQ_TAG; + } else if (!strncasecmp(irq_res->name, "MMU", 4)) { + irqtag = MMU_IRQ_TAG; + } else if (!strncasecmp(irq_res->name, "GPU", 4)) { + irqtag = GPU_IRQ_TAG; + } else { + dev_err(&pdev->dev, "Invalid irq res name: '%s'\n", + irq_res->name); + return -EINVAL; + } +#else + irqtag = i; +#endif /* CONFIG_OF */ + kbdev->irqs[irqtag].irq = irq_res->start; + kbdev->irqs[irqtag].flags = irq_res->flags & IRQF_TRIGGER_MASK; + } + + return 0; +} + +/* Find a particular kbase device (as specified by minor number), or find the "first" device if -1 is specified */ +struct kbase_device *kbase_find_device(int minor) +{ + struct kbase_device *kbdev = NULL; + struct list_head *entry; + const struct list_head *dev_list = kbase_device_get_list(); + + list_for_each(entry, dev_list) { + struct kbase_device *tmp; + + tmp = list_entry(entry, struct kbase_device, entry); + if (tmp->mdev.minor == minor || minor == -1) { + kbdev = tmp; + get_device(kbdev->dev); + break; + } + } + kbase_device_put_list(dev_list); + + return kbdev; +} +EXPORT_SYMBOL(kbase_find_device); + +void kbase_release_device(struct kbase_device *kbdev) +{ + put_device(kbdev->dev); +} +EXPORT_SYMBOL(kbase_release_device); + +#ifdef CONFIG_DEBUG_FS +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && \ + !(LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 28) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)) +/* + * Older versions, before v4.6, of the kernel doesn't have + * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28 + */ +static int kstrtobool_from_user(const char __user *s, size_t count, bool *res) +{ + char buf[4]; + + count = min(count, sizeof(buf) - 1); + + if (copy_from_user(buf, s, count)) + return -EFAULT; + buf[count] = '\0'; + + return strtobool(buf, res); +} +#endif + +static ssize_t write_ctx_infinite_cache(struct file *f, const char __user *ubuf, size_t size, loff_t *off) +{ + struct kbase_context *kctx = f->private_data; + int err; + bool value; + + err = kstrtobool_from_user(ubuf, size, &value); + if (err) + return err; + + if (value) + kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); + else + kbase_ctx_flag_clear(kctx, KCTX_INFINITE_CACHE); + + return size; +} + +static ssize_t read_ctx_infinite_cache(struct file *f, char __user *ubuf, size_t size, loff_t *off) +{ + struct kbase_context *kctx = f->private_data; + char buf[32]; + int count; + bool value; + + value = kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE); + + count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); + + return simple_read_from_buffer(ubuf, size, off, buf, count); +} + +static const struct file_operations kbase_infinite_cache_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = write_ctx_infinite_cache, + .read = read_ctx_infinite_cache, +}; + +static ssize_t write_ctx_force_same_va(struct file *f, const char __user *ubuf, + size_t size, loff_t *off) +{ + struct kbase_context *kctx = f->private_data; + int err; + bool value; + + err = kstrtobool_from_user(ubuf, size, &value); + if (err) + return err; + + if (value) { +#if defined(CONFIG_64BIT) + /* 32-bit clients cannot force SAME_VA */ + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + return -EINVAL; + kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); +#else /* defined(CONFIG_64BIT) */ + /* 32-bit clients cannot force SAME_VA */ + return -EINVAL; +#endif /* defined(CONFIG_64BIT) */ + } else { + kbase_ctx_flag_clear(kctx, KCTX_FORCE_SAME_VA); + } + + return size; +} + +static ssize_t read_ctx_force_same_va(struct file *f, char __user *ubuf, + size_t size, loff_t *off) +{ + struct kbase_context *kctx = f->private_data; + char buf[32]; + int count; + bool value; + + value = kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA); + + count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); + + return simple_read_from_buffer(ubuf, size, off, buf, count); +} + +static const struct file_operations kbase_force_same_va_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = write_ctx_force_same_va, + .read = read_ctx_force_same_va, +}; +#endif /* CONFIG_DEBUG_FS */ + +static int kbase_file_create_kctx(struct kbase_file *const kfile, + base_context_create_flags const flags) +{ + struct kbase_device *kbdev = NULL; + struct kbase_context *kctx = NULL; +#ifdef CONFIG_DEBUG_FS + char kctx_name[64]; +#endif + + if (WARN_ON(!kfile)) + return -EINVAL; + + /* setup pending, try to signal that we'll do the setup, + * if setup was already in progress, err this call + */ + if (atomic_cmpxchg(&kfile->setup_state, KBASE_FILE_NEED_CTX, + KBASE_FILE_CTX_IN_PROGRESS) != KBASE_FILE_NEED_CTX) + return -EPERM; + + kbdev = kfile->kbdev; + +#if (KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE) + kctx = kbase_create_context(kbdev, in_compat_syscall(), + flags, kfile->api_version, kfile->filp); +#else + kctx = kbase_create_context(kbdev, is_compat_task(), + flags, kfile->api_version, kfile->filp); +#endif /* (KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE) */ + + /* if bad flags, will stay stuck in setup mode */ + if (!kctx) + return -ENOMEM; + + if (kbdev->infinite_cache_active_default) + kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); + +#ifdef CONFIG_DEBUG_FS + snprintf(kctx_name, 64, "%d_%d", kctx->tgid, kctx->id); + + mutex_init(&kctx->mem_profile_lock); + + kctx->kctx_dentry = debugfs_create_dir(kctx_name, + kbdev->debugfs_ctx_directory); + + if (IS_ERR_OR_NULL(kctx->kctx_dentry)) { + /* we don't treat this as a fail - just warn about it */ + dev_warn(kbdev->dev, "couldn't create debugfs dir for kctx\n"); + } else { +#if (KERNEL_VERSION(4, 7, 0) > LINUX_VERSION_CODE) + /* prevent unprivileged use of debug file system + * in old kernel version + */ + debugfs_create_file("infinite_cache", 0600, kctx->kctx_dentry, + kctx, &kbase_infinite_cache_fops); +#else + debugfs_create_file("infinite_cache", 0644, kctx->kctx_dentry, + kctx, &kbase_infinite_cache_fops); +#endif + debugfs_create_file("force_same_va", 0600, kctx->kctx_dentry, + kctx, &kbase_force_same_va_fops); + + kbase_context_debugfs_init(kctx); + } +#endif /* CONFIG_DEBUG_FS */ + + dev_dbg(kbdev->dev, "created base context\n"); + + kfile->kctx = kctx; + atomic_set(&kfile->setup_state, KBASE_FILE_COMPLETE); + + return 0; +} + +static int kbase_open(struct inode *inode, struct file *filp) +{ + struct kbase_device *kbdev = NULL; + struct kbase_file *kfile; + int ret = 0; + + kbdev = kbase_find_device(iminor(inode)); + + if (!kbdev) + return -ENODEV; + + kfile = kbase_file_new(kbdev, filp); + if (!kfile) { + ret = -ENOMEM; + goto out; + } + + filp->private_data = kfile; + filp->f_mode |= FMODE_UNSIGNED_OFFSET; + + return 0; + + out: + kbase_release_device(kbdev); + return ret; +} + +static int kbase_release(struct inode *inode, struct file *filp) +{ + struct kbase_file *const kfile = filp->private_data; + + kbase_file_delete(kfile); + return 0; +} + +static int kbase_api_set_flags(struct kbase_file *kfile, + struct kbase_ioctl_set_flags *flags) +{ + int err = 0; + unsigned long const api_version = kbase_file_get_api_version(kfile); + struct kbase_context *kctx = NULL; + + /* Validate flags */ + if (flags->create_flags != + (flags->create_flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS)) + return -EINVAL; + + /* For backward compatibility, the context may have been created before + * the flags were set. + */ + if (mali_kbase_supports_system_monitor(api_version)) { + err = kbase_file_create_kctx(kfile, flags->create_flags); + } else { +#if !MALI_USE_CSF + struct kbasep_js_kctx_info *js_kctx_info = NULL; + unsigned long irq_flags = 0; +#endif + + /* If setup is incomplete (e.g. because the API version + * wasn't set) then we have to give up. + */ + kctx = kbase_file_get_kctx_if_setup_complete(kfile); + if (unlikely(!kctx)) + return -EPERM; + +#if MALI_USE_CSF + /* On CSF GPUs Job Manager interface isn't used to submit jobs + * (there are no job slots). So the legacy job manager path to + * submit jobs needs to remain disabled for CSF GPUs. + */ +#else + js_kctx_info = &kctx->jctx.sched_info; + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); + /* Translate the flags */ + if ((flags->create_flags & + BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) + kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); + + + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); +#endif + } + + return err; +} + +#if !MALI_USE_CSF +static int kbase_api_job_submit(struct kbase_context *kctx, + struct kbase_ioctl_job_submit *submit) +{ + return kbase_jd_submit(kctx, u64_to_user_ptr(submit->addr), + submit->nr_atoms, + submit->stride, false); +} +#endif /* !MALI_USE_CSF */ + +static int kbase_api_get_gpuprops(struct kbase_context *kctx, + struct kbase_ioctl_get_gpuprops *get_props) +{ + struct kbase_gpu_props *kprops = &kctx->kbdev->gpu_props; + int err; + + if (get_props->flags != 0) { + dev_err(kctx->kbdev->dev, "Unsupported flags to get_gpuprops"); + return -EINVAL; + } + + if (get_props->size == 0) + return kprops->prop_buffer_size; + if (get_props->size < kprops->prop_buffer_size) + return -EINVAL; + + err = copy_to_user(u64_to_user_ptr(get_props->buffer), + kprops->prop_buffer, + kprops->prop_buffer_size); + if (err) + return -EFAULT; + return kprops->prop_buffer_size; +} + +#if !MALI_USE_CSF +static int kbase_api_post_term(struct kbase_context *kctx) +{ + kbase_event_close(kctx); + return 0; +} +#endif /* !MALI_USE_CSF */ + +static int kbase_api_mem_alloc(struct kbase_context *kctx, + union kbase_ioctl_mem_alloc *alloc) +{ + struct kbase_va_region *reg; + u64 flags = alloc->in.flags; + u64 gpu_va; + + rcu_read_lock(); + /* Don't allow memory allocation until user space has set up the + * tracking page (which sets kctx->process_mm). Also catches when we've + * forked. + */ + if (rcu_dereference(kctx->process_mm) != current->mm) { + rcu_read_unlock(); + return -EINVAL; + } + rcu_read_unlock(); + + if (flags & BASEP_MEM_FLAGS_KERNEL_ONLY) + return -ENOMEM; + + /* Force SAME_VA if a 64-bit client. + * The only exception is GPU-executable memory if an EXEC_VA zone + * has been initialized. In that case, GPU-executable memory may + * or may not be SAME_VA. + */ + if ((!kbase_ctx_flag(kctx, KCTX_COMPAT)) && + kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA)) { + if (!(flags & BASE_MEM_PROT_GPU_EX) || !kbase_has_exec_va_zone(kctx)) + flags |= BASE_MEM_SAME_VA; + } + +#if MALI_USE_CSF + /* If CSF event memory allocation, need to force certain flags. + * SAME_VA - GPU address needs to be used as a CPU address, explicit + * mmap has to be avoided. + * CACHED_CPU - Frequent access to the event memory by CPU. + * COHERENT_SYSTEM - No explicit cache maintenance around the access + * to event memory so need to leverage the coherency support. + */ + if (flags & BASE_MEM_CSF_EVENT) { + flags |= (BASE_MEM_SAME_VA | + BASE_MEM_CACHED_CPU | + BASE_MEM_COHERENT_SYSTEM); + } +#endif + + reg = kbase_mem_alloc(kctx, alloc->in.va_pages, + alloc->in.commit_pages, + alloc->in.extent, + &flags, &gpu_va); + + if (!reg) + return -ENOMEM; + + alloc->out.flags = flags; + alloc->out.gpu_va = gpu_va; + + return 0; +} + +static int kbase_api_mem_query(struct kbase_context *kctx, + union kbase_ioctl_mem_query *query) +{ + return kbase_mem_query(kctx, query->in.gpu_addr, + query->in.query, &query->out.value); +} + +static int kbase_api_mem_free(struct kbase_context *kctx, + struct kbase_ioctl_mem_free *free) +{ + return kbase_mem_free(kctx, free->gpu_addr); +} + +#if !MALI_USE_CSF +static int kbase_api_kinstr_jm_fd(struct kbase_context *kctx, + union kbase_kinstr_jm_fd *arg) +{ + return kbase_kinstr_jm_get_fd(kctx->kinstr_jm, arg); +} +#endif + +static int kbase_api_hwcnt_reader_setup(struct kbase_context *kctx, + struct kbase_ioctl_hwcnt_reader_setup *setup) +{ + return kbase_vinstr_hwcnt_reader_setup(kctx->kbdev->vinstr_ctx, setup); +} + +static int kbase_api_hwcnt_enable(struct kbase_context *kctx, + struct kbase_ioctl_hwcnt_enable *enable) +{ + int ret; + + mutex_lock(&kctx->legacy_hwcnt_lock); + if (enable->dump_buffer != 0) { + /* Non-zero dump buffer, so user wants to create the client */ + if (kctx->legacy_hwcnt_cli == NULL) { + ret = kbase_hwcnt_legacy_client_create( + kctx->kbdev->hwcnt_gpu_virt, + enable, + &kctx->legacy_hwcnt_cli); + } else { + /* This context already has a client */ + ret = -EBUSY; + } + } else { + /* Zero dump buffer, so user wants to destroy the client */ + if (kctx->legacy_hwcnt_cli != NULL) { + kbase_hwcnt_legacy_client_destroy( + kctx->legacy_hwcnt_cli); + kctx->legacy_hwcnt_cli = NULL; + ret = 0; + } else { + /* This context has no client to destroy */ + ret = -EINVAL; + } + } + mutex_unlock(&kctx->legacy_hwcnt_lock); + + return ret; +} + +static int kbase_api_hwcnt_dump(struct kbase_context *kctx) +{ + int ret; + + mutex_lock(&kctx->legacy_hwcnt_lock); + ret = kbase_hwcnt_legacy_client_dump(kctx->legacy_hwcnt_cli); + mutex_unlock(&kctx->legacy_hwcnt_lock); + + return ret; +} + +static int kbase_api_hwcnt_clear(struct kbase_context *kctx) +{ + int ret; + + mutex_lock(&kctx->legacy_hwcnt_lock); + ret = kbase_hwcnt_legacy_client_clear(kctx->legacy_hwcnt_cli); + mutex_unlock(&kctx->legacy_hwcnt_lock); + + return ret; +} + +static int kbase_api_get_cpu_gpu_timeinfo(struct kbase_context *kctx, + union kbase_ioctl_get_cpu_gpu_timeinfo *timeinfo) +{ + u32 flags = timeinfo->in.request_flags; + struct timespec64 ts; + u64 timestamp; + u64 cycle_cnt; + + kbase_pm_context_active(kctx->kbdev); + + kbase_backend_get_gpu_time(kctx->kbdev, + (flags & BASE_TIMEINFO_CYCLE_COUNTER_FLAG) ? &cycle_cnt : NULL, + (flags & BASE_TIMEINFO_TIMESTAMP_FLAG) ? ×tamp : NULL, + (flags & BASE_TIMEINFO_MONOTONIC_FLAG) ? &ts : NULL); + + if (flags & BASE_TIMEINFO_TIMESTAMP_FLAG) + timeinfo->out.timestamp = timestamp; + + if (flags & BASE_TIMEINFO_CYCLE_COUNTER_FLAG) + timeinfo->out.cycle_counter = cycle_cnt; + + if (flags & BASE_TIMEINFO_MONOTONIC_FLAG) { + timeinfo->out.sec = ts.tv_sec; + timeinfo->out.nsec = ts.tv_nsec; + } + + kbase_pm_context_idle(kctx->kbdev); + + return 0; +} + +#ifdef CONFIG_MALI_BIFROST_NO_MALI +static int kbase_api_hwcnt_set(struct kbase_context *kctx, + struct kbase_ioctl_hwcnt_values *values) +{ + gpu_model_set_dummy_prfcnt_sample( + (u32 __user *)(uintptr_t)values->data, + values->size); + + return 0; +} +#endif + +static int kbase_api_disjoint_query(struct kbase_context *kctx, + struct kbase_ioctl_disjoint_query *query) +{ + query->counter = kbase_disjoint_event_get(kctx->kbdev); + + return 0; +} + +static int kbase_api_get_ddk_version(struct kbase_context *kctx, + struct kbase_ioctl_get_ddk_version *version) +{ + int ret; + int len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); + + if (version->version_buffer == 0) + return len; + + if (version->size < len) + return -EOVERFLOW; + + ret = copy_to_user(u64_to_user_ptr(version->version_buffer), + KERNEL_SIDE_DDK_VERSION_STRING, + sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); + + if (ret) + return -EFAULT; + + return len; +} + +/* Defaults for legacy just-in-time memory allocator initialization + * kernel calls + */ +#define DEFAULT_MAX_JIT_ALLOCATIONS 255 +#define JIT_LEGACY_TRIM_LEVEL (0) /* No trimming */ + +static int kbase_api_mem_jit_init_10_2(struct kbase_context *kctx, + struct kbase_ioctl_mem_jit_init_10_2 *jit_init) +{ + kctx->jit_version = 1; + + /* since no phys_pages parameter, use the maximum: va_pages */ + return kbase_region_tracker_init_jit(kctx, jit_init->va_pages, + DEFAULT_MAX_JIT_ALLOCATIONS, + JIT_LEGACY_TRIM_LEVEL, BASE_MEM_GROUP_DEFAULT, + jit_init->va_pages); +} + +static int kbase_api_mem_jit_init_11_5(struct kbase_context *kctx, + struct kbase_ioctl_mem_jit_init_11_5 *jit_init) +{ + int i; + + kctx->jit_version = 2; + + for (i = 0; i < sizeof(jit_init->padding); i++) { + /* Ensure all padding bytes are 0 for potential future + * extension + */ + if (jit_init->padding[i]) + return -EINVAL; + } + + /* since no phys_pages parameter, use the maximum: va_pages */ + return kbase_region_tracker_init_jit(kctx, jit_init->va_pages, + jit_init->max_allocations, jit_init->trim_level, + jit_init->group_id, jit_init->va_pages); +} + +static int kbase_api_mem_jit_init(struct kbase_context *kctx, + struct kbase_ioctl_mem_jit_init *jit_init) +{ + int i; + + kctx->jit_version = 3; + + for (i = 0; i < sizeof(jit_init->padding); i++) { + /* Ensure all padding bytes are 0 for potential future + * extension + */ + if (jit_init->padding[i]) + return -EINVAL; + } + + return kbase_region_tracker_init_jit(kctx, jit_init->va_pages, + jit_init->max_allocations, jit_init->trim_level, + jit_init->group_id, jit_init->phys_pages); +} + +static int kbase_api_mem_exec_init(struct kbase_context *kctx, + struct kbase_ioctl_mem_exec_init *exec_init) +{ + return kbase_region_tracker_init_exec(kctx, exec_init->va_pages); +} + +static int kbase_api_mem_sync(struct kbase_context *kctx, + struct kbase_ioctl_mem_sync *sync) +{ + struct basep_syncset sset = { + .mem_handle.basep.handle = sync->handle, + .user_addr = sync->user_addr, + .size = sync->size, + .type = sync->type + }; + + return kbase_sync_now(kctx, &sset); +} + +static int kbase_api_mem_find_cpu_offset(struct kbase_context *kctx, + union kbase_ioctl_mem_find_cpu_offset *find) +{ + return kbasep_find_enclosing_cpu_mapping_offset( + kctx, + find->in.cpu_addr, + find->in.size, + &find->out.offset); +} + +static int kbase_api_mem_find_gpu_start_and_offset(struct kbase_context *kctx, + union kbase_ioctl_mem_find_gpu_start_and_offset *find) +{ + return kbasep_find_enclosing_gpu_mapping_start_and_offset( + kctx, + find->in.gpu_addr, + find->in.size, + &find->out.start, + &find->out.offset); +} + +static int kbase_api_get_context_id(struct kbase_context *kctx, + struct kbase_ioctl_get_context_id *info) +{ + info->id = kctx->id; + + return 0; +} + +static int kbase_api_tlstream_acquire(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_acquire *acquire) +{ + return kbase_timeline_io_acquire(kctx->kbdev, acquire->flags); +} + +static int kbase_api_tlstream_flush(struct kbase_context *kctx) +{ + kbase_timeline_streams_flush(kctx->kbdev->timeline); + + return 0; +} + +static int kbase_api_mem_commit(struct kbase_context *kctx, + struct kbase_ioctl_mem_commit *commit) +{ + return kbase_mem_commit(kctx, commit->gpu_addr, commit->pages); +} + +static int kbase_api_mem_alias(struct kbase_context *kctx, + union kbase_ioctl_mem_alias *alias) +{ + struct base_mem_aliasing_info *ai; + u64 flags; + int err; + + if (alias->in.nents == 0 || alias->in.nents > 2048) + return -EINVAL; + + if (alias->in.stride > (U64_MAX / 2048)) + return -EINVAL; + + ai = vmalloc(sizeof(*ai) * alias->in.nents); + if (!ai) + return -ENOMEM; + + err = copy_from_user(ai, + u64_to_user_ptr(alias->in.aliasing_info), + sizeof(*ai) * alias->in.nents); + if (err) { + vfree(ai); + return -EFAULT; + } + + flags = alias->in.flags; + if (flags & BASEP_MEM_FLAGS_KERNEL_ONLY) { + vfree(ai); + return -EINVAL; + } + + alias->out.gpu_va = kbase_mem_alias(kctx, &flags, + alias->in.stride, alias->in.nents, + ai, &alias->out.va_pages); + + alias->out.flags = flags; + + vfree(ai); + + if (alias->out.gpu_va == 0) + return -ENOMEM; + + return 0; +} + +static int kbase_api_mem_import(struct kbase_context *kctx, + union kbase_ioctl_mem_import *import) +{ + int ret; + u64 flags = import->in.flags; + + if (flags & BASEP_MEM_FLAGS_KERNEL_ONLY) + return -ENOMEM; + + ret = kbase_mem_import(kctx, + import->in.type, + u64_to_user_ptr(import->in.phandle), + import->in.padding, + &import->out.gpu_va, + &import->out.va_pages, + &flags); + + import->out.flags = flags; + + return ret; +} + +static int kbase_api_mem_flags_change(struct kbase_context *kctx, + struct kbase_ioctl_mem_flags_change *change) +{ + if (change->flags & BASEP_MEM_FLAGS_KERNEL_ONLY) + return -ENOMEM; + + return kbase_mem_flags_change(kctx, change->gpu_va, + change->flags, change->mask); +} + +static int kbase_api_stream_create(struct kbase_context *kctx, + struct kbase_ioctl_stream_create *stream) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + int fd, ret; + + /* Name must be NULL-terminated and padded with NULLs, so check last + * character is NULL + */ + if (stream->name[sizeof(stream->name)-1] != 0) + return -EINVAL; + + ret = kbase_sync_fence_stream_create(stream->name, &fd); + + if (ret) + return ret; + return fd; +#else + return -ENOENT; +#endif +} + +static int kbase_api_fence_validate(struct kbase_context *kctx, + struct kbase_ioctl_fence_validate *validate) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + return kbase_sync_fence_validate(validate->fd); +#else + return -ENOENT; +#endif +} + +static int kbase_api_mem_profile_add(struct kbase_context *kctx, + struct kbase_ioctl_mem_profile_add *data) +{ + char *buf; + int err; + + if (data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { + dev_err(kctx->kbdev->dev, "mem_profile_add: buffer too big\n"); + return -EINVAL; + } + + buf = kmalloc(data->len, GFP_KERNEL); + if (ZERO_OR_NULL_PTR(buf)) + return -ENOMEM; + + err = copy_from_user(buf, u64_to_user_ptr(data->buffer), + data->len); + if (err) { + kfree(buf); + return -EFAULT; + } + + return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); +} + +#if !MALI_USE_CSF +static int kbase_api_soft_event_update(struct kbase_context *kctx, + struct kbase_ioctl_soft_event_update *update) +{ + if (update->flags != 0) + return -EINVAL; + + return kbase_soft_event_update(kctx, update->event, update->new_status); +} +#endif /* !MALI_USE_CSF */ + +static int kbase_api_sticky_resource_map(struct kbase_context *kctx, + struct kbase_ioctl_sticky_resource_map *map) +{ + int ret; + u64 i; + u64 gpu_addr[BASE_EXT_RES_COUNT_MAX]; + + if (!map->count || map->count > BASE_EXT_RES_COUNT_MAX) + return -EOVERFLOW; + + ret = copy_from_user(gpu_addr, u64_to_user_ptr(map->address), + sizeof(u64) * map->count); + + if (ret != 0) + return -EFAULT; + + kbase_gpu_vm_lock(kctx); + + for (i = 0; i < map->count; i++) { + if (!kbase_sticky_resource_acquire(kctx, gpu_addr[i])) { + /* Invalid resource */ + ret = -EINVAL; + break; + } + } + + if (ret != 0) { + while (i > 0) { + i--; + kbase_sticky_resource_release_force(kctx, NULL, gpu_addr[i]); + } + } + + kbase_gpu_vm_unlock(kctx); + + return ret; +} + +static int kbase_api_sticky_resource_unmap(struct kbase_context *kctx, + struct kbase_ioctl_sticky_resource_unmap *unmap) +{ + int ret; + u64 i; + u64 gpu_addr[BASE_EXT_RES_COUNT_MAX]; + + if (!unmap->count || unmap->count > BASE_EXT_RES_COUNT_MAX) + return -EOVERFLOW; + + ret = copy_from_user(gpu_addr, u64_to_user_ptr(unmap->address), + sizeof(u64) * unmap->count); + + if (ret != 0) + return -EFAULT; + + kbase_gpu_vm_lock(kctx); + + for (i = 0; i < unmap->count; i++) { + if (!kbase_sticky_resource_release_force(kctx, NULL, gpu_addr[i])) { + /* Invalid resource, but we keep going anyway */ + ret = -EINVAL; + } + } + + kbase_gpu_vm_unlock(kctx); + + return ret; +} + +#if MALI_UNIT_TEST +static int kbase_api_tlstream_test(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_test *test) +{ + kbase_timeline_test( + kctx->kbdev, + test->tpw_count, + test->msg_delay, + test->msg_count, + test->aux_msg); + + return 0; +} + +static int kbase_api_tlstream_stats(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_stats *stats) +{ + kbase_timeline_stats(kctx->kbdev->timeline, + &stats->bytes_collected, + &stats->bytes_generated); + + return 0; +} +#endif /* MALI_UNIT_TEST */ + +#if MALI_USE_CSF +static int kbasep_cs_event_signal(struct kbase_context *kctx) +{ + kbase_csf_event_signal_notify_gpu(kctx); + return 0; +} + +static int kbasep_cs_queue_register(struct kbase_context *kctx, + struct kbase_ioctl_cs_queue_register *reg) +{ + kctx->jit_group_id = BASE_MEM_GROUP_DEFAULT; + + return kbase_csf_queue_register(kctx, reg); +} + +static int kbasep_cs_queue_terminate(struct kbase_context *kctx, + struct kbase_ioctl_cs_queue_terminate *term) +{ + kbase_csf_queue_terminate(kctx, term); + + return 0; +} + +static int kbasep_cs_queue_bind(struct kbase_context *kctx, + union kbase_ioctl_cs_queue_bind *bind) +{ + return kbase_csf_queue_bind(kctx, bind); +} + +static int kbasep_cs_queue_kick(struct kbase_context *kctx, + struct kbase_ioctl_cs_queue_kick *kick) +{ + return kbase_csf_queue_kick(kctx, kick); +} + +static int kbasep_cs_queue_group_create(struct kbase_context *kctx, + union kbase_ioctl_cs_queue_group_create *create) +{ + return kbase_csf_queue_group_create(kctx, create); +} + +static int kbasep_cs_queue_group_terminate(struct kbase_context *kctx, + struct kbase_ioctl_cs_queue_group_term *term) +{ + kbase_csf_queue_group_terminate(kctx, term->group_handle); + + return 0; +} + +static int kbasep_kcpu_queue_new(struct kbase_context *kctx, + struct kbase_ioctl_kcpu_queue_new *new) +{ + return kbase_csf_kcpu_queue_new(kctx, new); +} + +static int kbasep_kcpu_queue_delete(struct kbase_context *kctx, + struct kbase_ioctl_kcpu_queue_delete *delete) +{ + return kbase_csf_kcpu_queue_delete(kctx, delete); +} + +static int kbasep_kcpu_queue_enqueue(struct kbase_context *kctx, + struct kbase_ioctl_kcpu_queue_enqueue *enqueue) +{ + return kbase_csf_kcpu_queue_enqueue(kctx, enqueue); +} + +static int kbasep_cs_tiler_heap_init(struct kbase_context *kctx, + union kbase_ioctl_cs_tiler_heap_init *heap_init) +{ + kctx->jit_group_id = heap_init->in.group_id; + + return kbase_csf_tiler_heap_init(kctx, heap_init->in.chunk_size, + heap_init->in.initial_chunks, heap_init->in.max_chunks, + heap_init->in.target_in_flight, + &heap_init->out.gpu_heap_va, &heap_init->out.first_chunk_va); +} + +static int kbasep_cs_tiler_heap_term(struct kbase_context *kctx, + struct kbase_ioctl_cs_tiler_heap_term *heap_term) +{ + return kbase_csf_tiler_heap_term(kctx, heap_term->gpu_heap_va); +} + +static int kbase_ioctl_cs_get_glb_iface(struct kbase_context *kctx, + union kbase_ioctl_cs_get_glb_iface *param) +{ + struct basep_cs_stream_control *stream_data = NULL; + struct basep_cs_group_control *group_data = NULL; + void __user *user_groups, *user_streams; + int err = 0; + u32 const max_group_num = param->in.max_group_num; + u32 const max_total_stream_num = param->in.max_total_stream_num; + + if (max_group_num > MAX_SUPPORTED_CSGS) + return -EINVAL; + + if (max_total_stream_num > + MAX_SUPPORTED_CSGS * MAX_SUPPORTED_STREAMS_PER_GROUP) + return -EINVAL; + + user_groups = u64_to_user_ptr(param->in.groups_ptr); + user_streams = u64_to_user_ptr(param->in.streams_ptr); + + if (max_group_num > 0) { + if (!user_groups) + err = -EINVAL; + else { + group_data = kcalloc(max_group_num, + sizeof(*group_data), GFP_KERNEL); + if (!group_data) + err = -ENOMEM; + } + } + + if (max_total_stream_num > 0) { + if (!user_streams) + err = -EINVAL; + else { + stream_data = kcalloc(max_total_stream_num, + sizeof(*stream_data), GFP_KERNEL); + if (!stream_data) + err = -ENOMEM; + } + } + + if (!err) { + param->out.total_stream_num = + kbase_csf_firmware_get_glb_iface(kctx->kbdev, + group_data, max_group_num, + stream_data, max_total_stream_num, + ¶m->out.glb_version, ¶m->out.features, + ¶m->out.group_num, ¶m->out.prfcnt_size); + + param->out.padding = 0; + + if (copy_to_user(user_groups, group_data, + MIN(max_group_num, param->out.group_num) * + sizeof(*group_data))) + err = -EFAULT; + } + + if (!err) + if (copy_to_user(user_streams, stream_data, + MIN(max_total_stream_num, param->out.total_stream_num) * + sizeof(*stream_data))) + err = -EFAULT; + + kfree(group_data); + kfree(stream_data); + return err; +} +#endif /* MALI_USE_CSF */ + +#define KBASE_HANDLE_IOCTL(cmd, function, arg) \ + do { \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ + return function(arg); \ + } while (0) + +#define KBASE_HANDLE_IOCTL_IN(cmd, function, type, arg) \ + do { \ + type param; \ + int err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return function(arg, ¶m); \ + } while (0) + +#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + memset(¶m, 0, sizeof(param)); \ + ret = function(arg, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return ret; \ + } while (0) + +#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + ret = function(arg, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return ret; \ + } while (0) + +static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct kbase_file *const kfile = filp->private_data; + struct kbase_context *kctx = NULL; + struct kbase_device *kbdev = kfile->kbdev; + void __user *uarg = (void __user *)arg; + + /* Only these ioctls are available until setup is complete */ + switch (cmd) { + case KBASE_IOCTL_VERSION_CHECK: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK, + kbase_api_handshake, + struct kbase_ioctl_version_check, + kfile); + break; + + case KBASE_IOCTL_VERSION_CHECK_RESERVED: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK_RESERVED, + kbase_api_handshake_dummy, + struct kbase_ioctl_version_check, + kfile); + break; + + case KBASE_IOCTL_SET_FLAGS: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_FLAGS, + kbase_api_set_flags, + struct kbase_ioctl_set_flags, + kfile); + break; + } + + kctx = kbase_file_get_kctx_if_setup_complete(kfile); + if (unlikely(!kctx)) + return -EPERM; + + /* Normal ioctls */ + switch (cmd) { +#if !MALI_USE_CSF + case KBASE_IOCTL_JOB_SUBMIT: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, + kbase_api_job_submit, + struct kbase_ioctl_job_submit, + kctx); + break; +#endif /* !MALI_USE_CSF */ + case KBASE_IOCTL_GET_GPUPROPS: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_GPUPROPS, + kbase_api_get_gpuprops, + struct kbase_ioctl_get_gpuprops, + kctx); + break; +#if !MALI_USE_CSF + case KBASE_IOCTL_POST_TERM: + KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, + kbase_api_post_term, + kctx); + break; +#endif /* !MALI_USE_CSF */ + case KBASE_IOCTL_MEM_ALLOC: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, + kbase_api_mem_alloc, + union kbase_ioctl_mem_alloc, + kctx); + break; + case KBASE_IOCTL_MEM_QUERY: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY, + kbase_api_mem_query, + union kbase_ioctl_mem_query, + kctx); + break; + case KBASE_IOCTL_MEM_FREE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FREE, + kbase_api_mem_free, + struct kbase_ioctl_mem_free, + kctx); + break; + case KBASE_IOCTL_DISJOINT_QUERY: + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_DISJOINT_QUERY, + kbase_api_disjoint_query, + struct kbase_ioctl_disjoint_query, + kctx); + break; + case KBASE_IOCTL_GET_DDK_VERSION: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_DDK_VERSION, + kbase_api_get_ddk_version, + struct kbase_ioctl_get_ddk_version, + kctx); + break; + case KBASE_IOCTL_MEM_JIT_INIT_10_2: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT_10_2, + kbase_api_mem_jit_init_10_2, + struct kbase_ioctl_mem_jit_init_10_2, + kctx); + break; + case KBASE_IOCTL_MEM_JIT_INIT_11_5: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT_11_5, + kbase_api_mem_jit_init_11_5, + struct kbase_ioctl_mem_jit_init_11_5, + kctx); + break; + case KBASE_IOCTL_MEM_JIT_INIT: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT, + kbase_api_mem_jit_init, + struct kbase_ioctl_mem_jit_init, + kctx); + break; + case KBASE_IOCTL_MEM_EXEC_INIT: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_EXEC_INIT, + kbase_api_mem_exec_init, + struct kbase_ioctl_mem_exec_init, + kctx); + break; + case KBASE_IOCTL_MEM_SYNC: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_SYNC, + kbase_api_mem_sync, + struct kbase_ioctl_mem_sync, + kctx); + break; + case KBASE_IOCTL_MEM_FIND_CPU_OFFSET: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_CPU_OFFSET, + kbase_api_mem_find_cpu_offset, + union kbase_ioctl_mem_find_cpu_offset, + kctx); + break; + case KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET, + kbase_api_mem_find_gpu_start_and_offset, + union kbase_ioctl_mem_find_gpu_start_and_offset, + kctx); + break; + case KBASE_IOCTL_GET_CONTEXT_ID: + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_GET_CONTEXT_ID, + kbase_api_get_context_id, + struct kbase_ioctl_get_context_id, + kctx); + break; + case KBASE_IOCTL_TLSTREAM_ACQUIRE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_ACQUIRE, + kbase_api_tlstream_acquire, + struct kbase_ioctl_tlstream_acquire, + kctx); + break; + case KBASE_IOCTL_TLSTREAM_FLUSH: + KBASE_HANDLE_IOCTL(KBASE_IOCTL_TLSTREAM_FLUSH, + kbase_api_tlstream_flush, + kctx); + break; + case KBASE_IOCTL_MEM_COMMIT: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_COMMIT, + kbase_api_mem_commit, + struct kbase_ioctl_mem_commit, + kctx); + break; + case KBASE_IOCTL_MEM_ALIAS: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALIAS, + kbase_api_mem_alias, + union kbase_ioctl_mem_alias, + kctx); + break; + case KBASE_IOCTL_MEM_IMPORT: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_IMPORT, + kbase_api_mem_import, + union kbase_ioctl_mem_import, + kctx); + break; + case KBASE_IOCTL_MEM_FLAGS_CHANGE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FLAGS_CHANGE, + kbase_api_mem_flags_change, + struct kbase_ioctl_mem_flags_change, + kctx); + break; + case KBASE_IOCTL_STREAM_CREATE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STREAM_CREATE, + kbase_api_stream_create, + struct kbase_ioctl_stream_create, + kctx); + break; + case KBASE_IOCTL_FENCE_VALIDATE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_FENCE_VALIDATE, + kbase_api_fence_validate, + struct kbase_ioctl_fence_validate, + kctx); + break; + case KBASE_IOCTL_MEM_PROFILE_ADD: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_PROFILE_ADD, + kbase_api_mem_profile_add, + struct kbase_ioctl_mem_profile_add, + kctx); + break; + +#if !MALI_USE_CSF + case KBASE_IOCTL_SOFT_EVENT_UPDATE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, + kbase_api_soft_event_update, + struct kbase_ioctl_soft_event_update, + kctx); + break; +#endif /* !MALI_USE_CSF */ + + case KBASE_IOCTL_STICKY_RESOURCE_MAP: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STICKY_RESOURCE_MAP, + kbase_api_sticky_resource_map, + struct kbase_ioctl_sticky_resource_map, + kctx); + break; + case KBASE_IOCTL_STICKY_RESOURCE_UNMAP: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STICKY_RESOURCE_UNMAP, + kbase_api_sticky_resource_unmap, + struct kbase_ioctl_sticky_resource_unmap, + kctx); + break; + + /* Instrumentation. */ +#if !MALI_USE_CSF + case KBASE_IOCTL_KINSTR_JM_FD: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_KINSTR_JM_FD, + kbase_api_kinstr_jm_fd, + union kbase_kinstr_jm_fd, + kctx); + break; +#endif + case KBASE_IOCTL_HWCNT_READER_SETUP: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_READER_SETUP, + kbase_api_hwcnt_reader_setup, + struct kbase_ioctl_hwcnt_reader_setup, + kctx); + break; + case KBASE_IOCTL_HWCNT_ENABLE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_ENABLE, + kbase_api_hwcnt_enable, + struct kbase_ioctl_hwcnt_enable, + kctx); + break; + case KBASE_IOCTL_HWCNT_DUMP: + KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_DUMP, + kbase_api_hwcnt_dump, + kctx); + break; + case KBASE_IOCTL_HWCNT_CLEAR: + KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_CLEAR, + kbase_api_hwcnt_clear, + kctx); + break; + case KBASE_IOCTL_GET_CPU_GPU_TIMEINFO: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_GET_CPU_GPU_TIMEINFO, + kbase_api_get_cpu_gpu_timeinfo, + union kbase_ioctl_get_cpu_gpu_timeinfo, + kctx); + break; +#ifdef CONFIG_MALI_BIFROST_NO_MALI + case KBASE_IOCTL_HWCNT_SET: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_SET, + kbase_api_hwcnt_set, + struct kbase_ioctl_hwcnt_values, + kctx); + break; +#endif +#ifdef CONFIG_MALI_CINSTR_GWT + case KBASE_IOCTL_CINSTR_GWT_START: + KBASE_HANDLE_IOCTL(KBASE_IOCTL_CINSTR_GWT_START, + kbase_gpu_gwt_start, + kctx); + break; + case KBASE_IOCTL_CINSTR_GWT_STOP: + KBASE_HANDLE_IOCTL(KBASE_IOCTL_CINSTR_GWT_STOP, + kbase_gpu_gwt_stop, + kctx); + break; + case KBASE_IOCTL_CINSTR_GWT_DUMP: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CINSTR_GWT_DUMP, + kbase_gpu_gwt_dump, + union kbase_ioctl_cinstr_gwt_dump, + kctx); + break; +#endif +#if MALI_USE_CSF + case KBASE_IOCTL_CS_EVENT_SIGNAL: + KBASE_HANDLE_IOCTL(KBASE_IOCTL_CS_EVENT_SIGNAL, + kbasep_cs_event_signal, + kctx); + break; + case KBASE_IOCTL_CS_QUEUE_REGISTER: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_REGISTER, + kbasep_cs_queue_register, + struct kbase_ioctl_cs_queue_register, + kctx); + break; + case KBASE_IOCTL_CS_QUEUE_TERMINATE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_TERMINATE, + kbasep_cs_queue_terminate, + struct kbase_ioctl_cs_queue_terminate, + kctx); + break; + case KBASE_IOCTL_CS_QUEUE_BIND: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_QUEUE_BIND, + kbasep_cs_queue_bind, + union kbase_ioctl_cs_queue_bind, + kctx); + break; + case KBASE_IOCTL_CS_QUEUE_KICK: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_KICK, + kbasep_cs_queue_kick, + struct kbase_ioctl_cs_queue_kick, + kctx); + break; + case KBASE_IOCTL_CS_QUEUE_GROUP_CREATE: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_QUEUE_GROUP_CREATE, + kbasep_cs_queue_group_create, + union kbase_ioctl_cs_queue_group_create, + kctx); + break; + case KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE, + kbasep_cs_queue_group_terminate, + struct kbase_ioctl_cs_queue_group_term, + kctx); + break; + case KBASE_IOCTL_KCPU_QUEUE_CREATE: + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_KCPU_QUEUE_CREATE, + kbasep_kcpu_queue_new, + struct kbase_ioctl_kcpu_queue_new, + kctx); + break; + case KBASE_IOCTL_KCPU_QUEUE_DELETE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_KCPU_QUEUE_DELETE, + kbasep_kcpu_queue_delete, + struct kbase_ioctl_kcpu_queue_delete, + kctx); + break; + case KBASE_IOCTL_KCPU_QUEUE_ENQUEUE: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_KCPU_QUEUE_ENQUEUE, + kbasep_kcpu_queue_enqueue, + struct kbase_ioctl_kcpu_queue_enqueue, + kctx); + break; + case KBASE_IOCTL_CS_TILER_HEAP_INIT: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_TILER_HEAP_INIT, + kbasep_cs_tiler_heap_init, + union kbase_ioctl_cs_tiler_heap_init, + kctx); + break; + case KBASE_IOCTL_CS_TILER_HEAP_TERM: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_TILER_HEAP_TERM, + kbasep_cs_tiler_heap_term, + struct kbase_ioctl_cs_tiler_heap_term, + kctx); + break; + case KBASE_IOCTL_CS_GET_GLB_IFACE: + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_GET_GLB_IFACE, + kbase_ioctl_cs_get_glb_iface, + union kbase_ioctl_cs_get_glb_iface, + kctx); + break; +#endif /* MALI_USE_CSF */ +#if MALI_UNIT_TEST + case KBASE_IOCTL_TLSTREAM_TEST: + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_TEST, + kbase_api_tlstream_test, + struct kbase_ioctl_tlstream_test, + kctx); + break; + case KBASE_IOCTL_TLSTREAM_STATS: + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, + kbase_api_tlstream_stats, + struct kbase_ioctl_tlstream_stats, + kctx); + break; +#endif /* MALI_UNIT_TEST */ + } + + dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); + + return -ENOIOCTLCMD; +} + +#if MALI_USE_CSF +static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) +{ + struct kbase_file *const kfile = filp->private_data; + struct kbase_context *const kctx = + kbase_file_get_kctx_if_setup_complete(kfile); + struct base_csf_notification event_data = { + .type = BASE_CSF_NOTIFICATION_EVENT }; + const size_t data_size = sizeof(event_data); + bool read_event = false, read_error = false; + + if (unlikely(!kctx)) + return -EPERM; + + if (atomic_read(&kctx->event_count)) + read_event = true; + else + read_error = kbase_csf_read_error(kctx, &event_data); + + if (!read_event && !read_error) { + /* This condition is not treated as an error. + * It is possible that event handling thread was woken up due + * to a fault/error that occurred for a queue group, but before + * the corresponding fault data was read by the thread the + * queue group was already terminated by the userspace. + */ + dev_dbg(kctx->kbdev->dev, "Neither event nor error signaled"); + } + + if (copy_to_user(buf, &event_data, data_size) != 0) { + dev_warn(kctx->kbdev->dev, + "Failed to copy data\n"); + return -EFAULT; + } + + if (read_event) + atomic_set(&kctx->event_count, 0); + + return data_size; +} +#else /* MALI_USE_CSF */ +static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) +{ + struct kbase_file *const kfile = filp->private_data; + struct kbase_context *const kctx = + kbase_file_get_kctx_if_setup_complete(kfile); + struct base_jd_event_v2 uevent; + int out_count = 0; + + if (unlikely(!kctx)) + return -EPERM; + + if (count < sizeof(uevent)) + return -ENOBUFS; + + do { + while (kbase_event_dequeue(kctx, &uevent)) { + if (out_count > 0) + goto out; + + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; + + if (wait_event_interruptible(kctx->event_queue, + kbase_event_pending(kctx)) != 0) + return -ERESTARTSYS; + } + if (uevent.event_code == BASE_JD_EVENT_DRV_TERMINATED) { + if (out_count == 0) + return -EPIPE; + goto out; + } + + if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) + return -EFAULT; + + buf += sizeof(uevent); + out_count++; + count -= sizeof(uevent); + } while (count >= sizeof(uevent)); + + out: + return out_count * sizeof(uevent); +} +#endif /* MALI_USE_CSF */ + +static unsigned int kbase_poll(struct file *filp, poll_table *wait) +{ + struct kbase_file *const kfile = filp->private_data; + struct kbase_context *const kctx = + kbase_file_get_kctx_if_setup_complete(kfile); + + if (unlikely(!kctx)) + return POLLERR; + + poll_wait(filp, &kctx->event_queue, wait); + if (kbase_event_pending(kctx)) + return POLLIN | POLLRDNORM; + + return 0; +} + +void kbase_event_wakeup(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx); + + wake_up_interruptible(&kctx->event_queue); +} + +KBASE_EXPORT_TEST_API(kbase_event_wakeup); + +#if MALI_USE_CSF +int kbase_event_pending(struct kbase_context *ctx) +{ + WARN_ON_ONCE(!ctx); + + return (atomic_read(&ctx->event_count) != 0) || + kbase_csf_error_pending(ctx); +} +#else +int kbase_event_pending(struct kbase_context *ctx) +{ + KBASE_DEBUG_ASSERT(ctx); + + return (atomic_read(&ctx->event_count) != 0) || + (atomic_read(&ctx->event_closed) != 0); +} +#endif + +KBASE_EXPORT_TEST_API(kbase_event_pending); + +static int kbase_mmap(struct file *const filp, struct vm_area_struct *const vma) +{ + struct kbase_file *const kfile = filp->private_data; + struct kbase_context *const kctx = + kbase_file_get_kctx_if_setup_complete(kfile); + + if (unlikely(!kctx)) + return -EPERM; + + return kbase_context_mmap(kctx, vma); +} + +static int kbase_check_flags(int flags) +{ + /* Enforce that the driver keeps the O_CLOEXEC flag so that execve() always + * closes the file descriptor in a child process. + */ + if (0 == (flags & O_CLOEXEC)) + return -EINVAL; + + return 0; +} + +static unsigned long kbase_get_unmapped_area(struct file *const filp, + const unsigned long addr, const unsigned long len, + const unsigned long pgoff, const unsigned long flags) +{ + struct kbase_file *const kfile = filp->private_data; + struct kbase_context *const kctx = + kbase_file_get_kctx_if_setup_complete(kfile); + + if (unlikely(!kctx)) + return -EPERM; + + return kbase_context_get_unmapped_area(kctx, addr, len, pgoff, flags); +} + +static const struct file_operations kbase_fops = { + .owner = THIS_MODULE, + .open = kbase_open, + .release = kbase_release, + .read = kbase_read, + .poll = kbase_poll, + .unlocked_ioctl = kbase_ioctl, + .compat_ioctl = kbase_ioctl, + .mmap = kbase_mmap, + .check_flags = kbase_check_flags, + .get_unmapped_area = kbase_get_unmapped_area, +}; + +/** + * show_policy - Show callback for the power_policy sysfs file. + * + * This function is called to get the contents of the power_policy sysfs + * file. This is a list of the available policies with the currently active one + * surrounded by square brackets. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf) +{ + struct kbase_device *kbdev; + const struct kbase_pm_policy *current_policy; + const struct kbase_pm_policy *const *policy_list; + int policy_count; + int i; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + current_policy = kbase_pm_get_policy(kbdev); + + policy_count = kbase_pm_list_policies(kbdev, &policy_list); + + for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { + if (policy_list[i] == current_policy) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); + else + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); + } + + if (ret < PAGE_SIZE - 1) { + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); + } else { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/** + * set_policy - Store callback for the power_policy sysfs file. + * + * This function is called when the power_policy sysfs file is written to. + * It matches the requested policy against the available policies and if a + * matching policy is found calls kbase_pm_set_policy() to change the + * policy. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes to write to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + const struct kbase_pm_policy *new_policy = NULL; + const struct kbase_pm_policy *const *policy_list; + int policy_count; + int i; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + policy_count = kbase_pm_list_policies(kbdev, &policy_list); + + for (i = 0; i < policy_count; i++) { + if (sysfs_streq(policy_list[i]->name, buf)) { + new_policy = policy_list[i]; + break; + } + } + + if (!new_policy) { + dev_err(dev, "power_policy: policy not found\n"); + return -EINVAL; + } + + kbase_pm_set_policy(kbdev, new_policy); + + return count; +} + +/* + * The sysfs file power_policy. + * + * This is used for obtaining information about the available policies, + * determining which policy is currently active, and changing the active + * policy. + */ +static DEVICE_ATTR(power_policy, S_IRUGO | S_IWUSR, show_policy, set_policy); + +/* + * show_core_mask - Show callback for the core_mask sysfs file. + * + * This function is called to get the contents of the core_mask sysfs file. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Current core mask (JS0) : 0x%llX\n", + kbdev->pm.debug_core_mask[0]); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Current core mask (JS1) : 0x%llX\n", + kbdev->pm.debug_core_mask[1]); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Current core mask (JS2) : 0x%llX\n", + kbdev->pm.debug_core_mask[2]); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Available core mask : 0x%llX\n", + kbdev->gpu_props.props.raw_props.shader_present); + + return ret; +} + +/** + * set_core_mask - Store callback for the core_mask sysfs file. + * + * This function is called when the core_mask sysfs file is written to. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes to write to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + u64 new_core_mask[3]; + int items, i; + ssize_t err = count; + unsigned long flags; + u64 shader_present, group0_core_mask; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + items = sscanf(buf, "%llx %llx %llx", + &new_core_mask[0], &new_core_mask[1], + &new_core_mask[2]); + + if (items != 1 && items != 3) { + dev_err(kbdev->dev, "Couldn't process core mask write operation.\n" + "Use format \n" + "or \n"); + err = -EINVAL; + goto end; + } + + if (items == 1) + new_core_mask[1] = new_core_mask[2] = new_core_mask[0]; + + mutex_lock(&kbdev->pm.lock); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + shader_present = kbdev->gpu_props.props.raw_props.shader_present; + group0_core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; + + for (i = 0; i < 3; ++i) { + if ((new_core_mask[i] & shader_present) != new_core_mask[i]) { + dev_err(dev, "Invalid core mask 0x%llX for JS %d: Includes non-existent cores (present = 0x%llX)", + new_core_mask[i], i, shader_present); + err = -EINVAL; + goto unlock; + + } else if (!(new_core_mask[i] & shader_present & kbdev->pm.backend.ca_cores_enabled)) { + dev_err(dev, "Invalid core mask 0x%llX for JS %d: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX\n", + new_core_mask[i], i, + kbdev->gpu_props.props.raw_props.shader_present, + kbdev->pm.backend.ca_cores_enabled); + err = -EINVAL; + goto unlock; + + } else if (!(new_core_mask[i] & group0_core_mask)) { + dev_err(dev, "Invalid core mask 0x%llX for JS %d: No intersection with group 0 core mask 0x%llX\n", + new_core_mask[i], i, group0_core_mask); + err = -EINVAL; + goto unlock; + } + } + + if (kbdev->pm.debug_core_mask[0] != new_core_mask[0] || + kbdev->pm.debug_core_mask[1] != + new_core_mask[1] || + kbdev->pm.debug_core_mask[2] != + new_core_mask[2]) { + + kbase_pm_set_debug_core_mask(kbdev, new_core_mask[0], + new_core_mask[1], new_core_mask[2]); + } + +unlock: + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->pm.lock); +end: + return err; +} + +/* + * The sysfs file core_mask. + * + * This is used to restrict shader core availability for debugging purposes. + * Reading it will show the current core mask and the mask of cores available. + * Writing to it will set the current core mask. + */ +static DEVICE_ATTR(core_mask, S_IRUGO | S_IWUSR, show_core_mask, set_core_mask); + +#if !MALI_USE_CSF +/** + * set_soft_job_timeout - Store callback for the soft_job_timeout sysfs + * file. + * + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The value written to the sysfs file. + * @count: The number of bytes to write to the sysfs file. + * + * This allows setting the timeout for software jobs. Waiting soft event wait + * jobs will be cancelled after this period expires, while soft fence wait jobs + * will print debug information if the fence debug feature is enabled. + * + * This is expressed in milliseconds. + * + * Return: count if the function succeeded. An error code on failure. + */ +static ssize_t set_soft_job_timeout(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int soft_job_timeout_ms; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + if ((kstrtoint(buf, 0, &soft_job_timeout_ms) != 0) || + (soft_job_timeout_ms <= 0)) + return -EINVAL; + + atomic_set(&kbdev->js_data.soft_job_timeout_ms, + soft_job_timeout_ms); + + return count; +} + +/** + * show_soft_job_timeout - Show callback for the soft_job_timeout sysfs + * file. + * + * This will return the timeout for the software jobs. + * + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer for the sysfs file contents. + * + * Return: The number of bytes output to buf. + */ +static ssize_t show_soft_job_timeout(struct device *dev, + struct device_attribute *attr, + char * const buf) +{ + struct kbase_device *kbdev; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + return scnprintf(buf, PAGE_SIZE, "%i\n", + atomic_read(&kbdev->js_data.soft_job_timeout_ms)); +} + +static DEVICE_ATTR(soft_job_timeout, S_IRUGO | S_IWUSR, + show_soft_job_timeout, set_soft_job_timeout); + +static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, + int default_ticks, u32 old_ticks) +{ + if (timeout_ms > 0) { + u64 ticks = timeout_ms * 1000000ULL; + do_div(ticks, kbdev->js_data.scheduling_period_ns); + if (!ticks) + return 1; + return ticks; + } else if (timeout_ms < 0) { + return default_ticks; + } else { + return old_ticks; + } +} + +/** + * set_js_timeouts - Store callback for the js_timeouts sysfs file. + * + * This function is called to get the contents of the js_timeouts sysfs + * file. This file contains five values separated by whitespace. The values + * are basically the same as %JS_SOFT_STOP_TICKS, %JS_HARD_STOP_TICKS_SS, + * %JS_HARD_STOP_TICKS_DUMPING, %JS_RESET_TICKS_SS, %JS_RESET_TICKS_DUMPING + * configuration values (in that order), with the difference that the js_timeout + * values are expressed in MILLISECONDS. + * + * The js_timeouts sysfile file allows the current values in + * use by the job scheduler to get override. Note that a value needs to + * be other than 0 for it to override the current job scheduler value. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes to write to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_js_timeouts(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int items; + long js_soft_stop_ms; + long js_soft_stop_ms_cl; + long js_hard_stop_ms_ss; + long js_hard_stop_ms_cl; + long js_hard_stop_ms_dumping; + long js_reset_ms_ss; + long js_reset_ms_cl; + long js_reset_ms_dumping; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + items = sscanf(buf, "%ld %ld %ld %ld %ld %ld %ld %ld", + &js_soft_stop_ms, &js_soft_stop_ms_cl, + &js_hard_stop_ms_ss, &js_hard_stop_ms_cl, + &js_hard_stop_ms_dumping, &js_reset_ms_ss, + &js_reset_ms_cl, &js_reset_ms_dumping); + + if (items == 8) { + struct kbasep_js_device_data *js_data = &kbdev->js_data; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + +#define UPDATE_TIMEOUT(ticks_name, ms_name, default) do {\ + js_data->ticks_name = timeout_ms_to_ticks(kbdev, ms_name, \ + default, js_data->ticks_name); \ + dev_dbg(kbdev->dev, "Overriding " #ticks_name \ + " with %lu ticks (%lu ms)\n", \ + (unsigned long)js_data->ticks_name, \ + ms_name); \ + } while (0) + + UPDATE_TIMEOUT(soft_stop_ticks, js_soft_stop_ms, + DEFAULT_JS_SOFT_STOP_TICKS); + UPDATE_TIMEOUT(soft_stop_ticks_cl, js_soft_stop_ms_cl, + DEFAULT_JS_SOFT_STOP_TICKS_CL); + UPDATE_TIMEOUT(hard_stop_ticks_ss, js_hard_stop_ms_ss, + DEFAULT_JS_HARD_STOP_TICKS_SS); + UPDATE_TIMEOUT(hard_stop_ticks_cl, js_hard_stop_ms_cl, + DEFAULT_JS_HARD_STOP_TICKS_CL); + UPDATE_TIMEOUT(hard_stop_ticks_dumping, + js_hard_stop_ms_dumping, + DEFAULT_JS_HARD_STOP_TICKS_DUMPING); + UPDATE_TIMEOUT(gpu_reset_ticks_ss, js_reset_ms_ss, + DEFAULT_JS_RESET_TICKS_SS); + UPDATE_TIMEOUT(gpu_reset_ticks_cl, js_reset_ms_cl, + DEFAULT_JS_RESET_TICKS_CL); + UPDATE_TIMEOUT(gpu_reset_ticks_dumping, js_reset_ms_dumping, + DEFAULT_JS_RESET_TICKS_DUMPING); + + kbase_js_set_timeouts(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return count; + } + + dev_err(kbdev->dev, "Couldn't process js_timeouts write operation.\n" + "Use format \n" + "Write 0 for no change, -1 to restore default timeout\n"); + return -EINVAL; +} + +static unsigned long get_js_timeout_in_ms( + u32 scheduling_period_ns, + u32 ticks) +{ + u64 ms = (u64)ticks * scheduling_period_ns; + + do_div(ms, 1000000UL); + return ms; +} + +/** + * show_js_timeouts - Show callback for the js_timeouts sysfs file. + * + * This function is called to get the contents of the js_timeouts sysfs + * file. It returns the last set values written to the js_timeouts sysfs file. + * If the file didn't get written yet, the values will be current setting in + * use. + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_js_timeouts(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + unsigned long js_soft_stop_ms; + unsigned long js_soft_stop_ms_cl; + unsigned long js_hard_stop_ms_ss; + unsigned long js_hard_stop_ms_cl; + unsigned long js_hard_stop_ms_dumping; + unsigned long js_reset_ms_ss; + unsigned long js_reset_ms_cl; + unsigned long js_reset_ms_dumping; + u32 scheduling_period_ns; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + scheduling_period_ns = kbdev->js_data.scheduling_period_ns; + +#define GET_TIMEOUT(name) get_js_timeout_in_ms(\ + scheduling_period_ns, \ + kbdev->js_data.name) + + js_soft_stop_ms = GET_TIMEOUT(soft_stop_ticks); + js_soft_stop_ms_cl = GET_TIMEOUT(soft_stop_ticks_cl); + js_hard_stop_ms_ss = GET_TIMEOUT(hard_stop_ticks_ss); + js_hard_stop_ms_cl = GET_TIMEOUT(hard_stop_ticks_cl); + js_hard_stop_ms_dumping = GET_TIMEOUT(hard_stop_ticks_dumping); + js_reset_ms_ss = GET_TIMEOUT(gpu_reset_ticks_ss); + js_reset_ms_cl = GET_TIMEOUT(gpu_reset_ticks_cl); + js_reset_ms_dumping = GET_TIMEOUT(gpu_reset_ticks_dumping); + +#undef GET_TIMEOUT + + ret = scnprintf(buf, PAGE_SIZE, "%lu %lu %lu %lu %lu %lu %lu %lu\n", + js_soft_stop_ms, js_soft_stop_ms_cl, + js_hard_stop_ms_ss, js_hard_stop_ms_cl, + js_hard_stop_ms_dumping, js_reset_ms_ss, + js_reset_ms_cl, js_reset_ms_dumping); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/* + * The sysfs file js_timeouts. + * + * This is used to override the current job scheduler values for + * JS_STOP_STOP_TICKS_SS + * JS_STOP_STOP_TICKS_CL + * JS_HARD_STOP_TICKS_SS + * JS_HARD_STOP_TICKS_CL + * JS_HARD_STOP_TICKS_DUMPING + * JS_RESET_TICKS_SS + * JS_RESET_TICKS_CL + * JS_RESET_TICKS_DUMPING. + */ +static DEVICE_ATTR(js_timeouts, S_IRUGO | S_IWUSR, show_js_timeouts, set_js_timeouts); + +static u32 get_new_js_timeout( + u32 old_period, + u32 old_ticks, + u32 new_scheduling_period_ns) +{ + u64 ticks = (u64)old_period * (u64)old_ticks; + do_div(ticks, new_scheduling_period_ns); + return ticks?ticks:1; +} + +/** + * set_js_scheduling_period - Store callback for the js_scheduling_period sysfs + * file + * @dev: The device the sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes to write to the sysfs file + * + * This function is called when the js_scheduling_period sysfs file is written + * to. It checks the data written, and if valid updates the js_scheduling_period + * value + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_js_scheduling_period(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + unsigned int js_scheduling_period; + u32 new_scheduling_period_ns; + u32 old_period; + struct kbasep_js_device_data *js_data; + unsigned long flags; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + js_data = &kbdev->js_data; + + ret = kstrtouint(buf, 0, &js_scheduling_period); + if (ret || !js_scheduling_period) { + dev_err(kbdev->dev, "Couldn't process js_scheduling_period write operation.\n" + "Use format \n"); + return -EINVAL; + } + + new_scheduling_period_ns = js_scheduling_period * 1000000; + + /* Update scheduling timeouts */ + mutex_lock(&js_data->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* If no contexts have been scheduled since js_timeouts was last written + * to, the new timeouts might not have been latched yet. So check if an + * update is pending and use the new values if necessary. */ + + /* Use previous 'new' scheduling period as a base if present. */ + old_period = js_data->scheduling_period_ns; + +#define SET_TIMEOUT(name) \ + (js_data->name = get_new_js_timeout(\ + old_period, \ + kbdev->js_data.name, \ + new_scheduling_period_ns)) + + SET_TIMEOUT(soft_stop_ticks); + SET_TIMEOUT(soft_stop_ticks_cl); + SET_TIMEOUT(hard_stop_ticks_ss); + SET_TIMEOUT(hard_stop_ticks_cl); + SET_TIMEOUT(hard_stop_ticks_dumping); + SET_TIMEOUT(gpu_reset_ticks_ss); + SET_TIMEOUT(gpu_reset_ticks_cl); + SET_TIMEOUT(gpu_reset_ticks_dumping); + +#undef SET_TIMEOUT + + js_data->scheduling_period_ns = new_scheduling_period_ns; + + kbase_js_set_timeouts(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_data->runpool_mutex); + + dev_dbg(kbdev->dev, "JS scheduling period: %dms\n", + js_scheduling_period); + + return count; +} + +/** + * show_js_scheduling_period - Show callback for the js_scheduling_period sysfs + * entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current period used for the JS scheduling + * period. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_js_scheduling_period(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + u32 period; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + period = kbdev->js_data.scheduling_period_ns; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", + period / 1000000); + + return ret; +} + +static DEVICE_ATTR(js_scheduling_period, S_IRUGO | S_IWUSR, + show_js_scheduling_period, set_js_scheduling_period); + + +#ifdef CONFIG_MALI_BIFROST_DEBUG +static ssize_t set_js_softstop_always(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + int softstop_always; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtoint(buf, 0, &softstop_always); + if (ret || ((softstop_always != 0) && (softstop_always != 1))) { + dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->js_data.softstop_always = (bool) softstop_always; + dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", + (kbdev->js_data.softstop_always) ? + "Enabled" : "Disabled"); + return count; +} + +static ssize_t show_js_softstop_always(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->js_data.softstop_always); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/* + * By default, soft-stops are disabled when only a single context is present. + * The ability to enable soft-stop when only a single context is present can be + * used for debug and unit-testing purposes. + * (see CL t6xx_stress_1 unit-test as an example whereby this feature is used.) + */ +static DEVICE_ATTR(js_softstop_always, S_IRUGO | S_IWUSR, show_js_softstop_always, set_js_softstop_always); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ +#endif /* !MALI_USE_CSF */ + +#ifdef CONFIG_MALI_BIFROST_DEBUG +typedef void (kbasep_debug_command_func) (struct kbase_device *); + +enum kbasep_debug_command_code { + KBASEP_DEBUG_COMMAND_DUMPTRACE, + + /* This must be the last enum */ + KBASEP_DEBUG_COMMAND_COUNT +}; + +struct kbasep_debug_command { + char *str; + kbasep_debug_command_func *func; +}; + +void kbasep_ktrace_dump_wrapper(struct kbase_device *kbdev) +{ + KBASE_KTRACE_DUMP(kbdev); +} + +/* Debug commands supported by the driver */ +static const struct kbasep_debug_command debug_commands[] = { + { + .str = "dumptrace", + .func = &kbasep_ktrace_dump_wrapper, + } +}; + +/** + * show_debug - Show callback for the debug_command sysfs file. + * + * This function is called to get the contents of the debug_command sysfs + * file. This is a list of the available debug commands, separated by newlines. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_debug(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + int i; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT && ret < PAGE_SIZE; i++) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s\n", debug_commands[i].str); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/** + * issue_debug - Store callback for the debug_command sysfs file. + * + * This function is called when the debug_command sysfs file is written to. + * It matches the requested command against the available commands, and if + * a matching command is found calls the associated function from + * @debug_commands to issue the command. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t issue_debug(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int i; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT; i++) { + if (sysfs_streq(debug_commands[i].str, buf)) { + debug_commands[i].func(kbdev); + return count; + } + } + + /* Debug Command not found */ + dev_err(dev, "debug_command: command not known\n"); + return -EINVAL; +} + +/* The sysfs file debug_command. + * + * This is used to issue general debug commands to the device driver. + * Reading it will produce a list of debug commands, separated by newlines. + * Writing to it with one of those commands will issue said command. + */ +static DEVICE_ATTR(debug_command, S_IRUGO | S_IWUSR, show_debug, issue_debug); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + +/** + * kbase_show_gpuinfo - Show callback for the gpuinfo sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get a description of the present Mali + * GPU via the gpuinfo sysfs entry. This includes the GPU family, the + * number of cores, the hardware version and the raw product id. For + * example + * + * Mali-T60x MP4 r0p0 0x6956 + * + * Return: The number of bytes output to @buf. + */ +static ssize_t kbase_show_gpuinfo(struct device *dev, + struct device_attribute *attr, char *buf) +{ + static const struct gpu_product_id_name { + unsigned id; + char *name; + } gpu_product_id_names[] = { + { .id = GPU_ID2_PRODUCT_TMIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G71" }, + { .id = GPU_ID2_PRODUCT_THEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G72" }, + { .id = GPU_ID2_PRODUCT_TSIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G51" }, + { .id = GPU_ID2_PRODUCT_TNOX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G76" }, + { .id = GPU_ID2_PRODUCT_TDVX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G31" }, + { .id = GPU_ID2_PRODUCT_TGOX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G52" }, + { .id = GPU_ID2_PRODUCT_TTRX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G77" }, + { .id = GPU_ID2_PRODUCT_TBEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G78" }, + { .id = GPU_ID2_PRODUCT_TBAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-TBAX" }, + { .id = GPU_ID2_PRODUCT_LBEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G68" }, + { .id = GPU_ID2_PRODUCT_TNAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G57" }, + { .id = GPU_ID2_PRODUCT_TODX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-TODX" }, + { .id = GPU_ID2_PRODUCT_TGRX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-TGRX" }, + { .id = GPU_ID2_PRODUCT_TVAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-TVAX" }, + { .id = GPU_ID2_PRODUCT_LODX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-LODX" }, + { .id = GPU_ID2_PRODUCT_TTUX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-TTUX" }, + { .id = GPU_ID2_PRODUCT_LTUX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-LTUX" }, + { .id = GPU_ID2_PRODUCT_TE2X >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-TE2X" }, + }; + const char *product_name = "(Unknown Mali GPU)"; + struct kbase_device *kbdev; + u32 gpu_id; + unsigned product_id, product_id_mask; + unsigned i; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + product_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; + product_id_mask = GPU_ID2_PRODUCT_MODEL >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + for (i = 0; i < ARRAY_SIZE(gpu_product_id_names); ++i) { + const struct gpu_product_id_name *p = &gpu_product_id_names[i]; + + if ((p->id & product_id_mask) == + (product_id & product_id_mask)) { + product_name = p->name; + break; + } + } + + return scnprintf(buf, PAGE_SIZE, "%s %d cores r%dp%d 0x%04X\n", + product_name, kbdev->gpu_props.num_cores, + (gpu_id & GPU_ID_VERSION_MAJOR) >> GPU_ID_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID_VERSION_MINOR) >> GPU_ID_VERSION_MINOR_SHIFT, + product_id); +} +static DEVICE_ATTR(gpuinfo, S_IRUGO, kbase_show_gpuinfo, NULL); + +/** + * set_dvfs_period - Store callback for the dvfs_period sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the dvfs_period sysfs file is written to. It + * checks the data written, and if valid updates the DVFS period variable, + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_dvfs_period(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + int dvfs_period; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtoint(buf, 0, &dvfs_period); + if (ret || dvfs_period <= 0) { + dev_err(kbdev->dev, "Couldn't process dvfs_period write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->pm.dvfs_period = dvfs_period; + dev_dbg(kbdev->dev, "DVFS period: %dms\n", dvfs_period); + + return count; +} + +/** + * show_dvfs_period - Show callback for the dvfs_period sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current period used for the DVFS sample + * timer. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_dvfs_period(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->pm.dvfs_period); + + return ret; +} + +static DEVICE_ATTR(dvfs_period, S_IRUGO | S_IWUSR, show_dvfs_period, + set_dvfs_period); + +/** + * set_pm_poweroff - Store callback for the pm_poweroff sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the pm_poweroff sysfs file is written to. + * + * This file contains three values separated by whitespace. The values + * are gpu_poweroff_time (the period of the poweroff timer, in ns), + * poweroff_shader_ticks (the number of poweroff timer ticks before an idle + * shader is powered off), and poweroff_gpu_ticks (the number of poweroff timer + * ticks before the GPU is powered off), in that order. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_pm_poweroff(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + struct kbasep_pm_tick_timer_state *stt; + int items; + u64 gpu_poweroff_time; + unsigned int poweroff_shader_ticks, poweroff_gpu_ticks; + unsigned long flags; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + items = sscanf(buf, "%llu %u %u", &gpu_poweroff_time, + &poweroff_shader_ticks, + &poweroff_gpu_ticks); + if (items != 3) { + dev_err(kbdev->dev, "Couldn't process pm_poweroff write operation.\n" + "Use format \n"); + return -EINVAL; + } + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + stt = &kbdev->pm.backend.shader_tick_timer; + stt->configured_interval = HR_TIMER_DELAY_NSEC(gpu_poweroff_time); + stt->configured_ticks = poweroff_shader_ticks; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (poweroff_gpu_ticks != 0) + dev_warn(kbdev->dev, "Separate GPU poweroff delay no longer supported.\n"); + + return count; +} + +/** + * show_pm_poweroff - Show callback for the pm_poweroff sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current period used for the DVFS sample + * timer. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_pm_poweroff(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + struct kbasep_pm_tick_timer_state *stt; + ssize_t ret; + unsigned long flags; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + stt = &kbdev->pm.backend.shader_tick_timer; + ret = scnprintf(buf, PAGE_SIZE, "%llu %u 0\n", + ktime_to_ns(stt->configured_interval), + stt->configured_ticks); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return ret; +} + +static DEVICE_ATTR(pm_poweroff, S_IRUGO | S_IWUSR, show_pm_poweroff, + set_pm_poweroff); + +/** + * set_reset_timeout - Store callback for the reset_timeout sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the reset_timeout sysfs file is written to. It + * checks the data written, and if valid updates the reset timeout. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_reset_timeout(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + int reset_timeout; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtoint(buf, 0, &reset_timeout); + if (ret || reset_timeout <= 0) { + dev_err(kbdev->dev, "Couldn't process reset_timeout write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->reset_timeout_ms = reset_timeout; + dev_dbg(kbdev->dev, "Reset timeout: %dms\n", reset_timeout); + + return count; +} + +/** + * show_reset_timeout - Show callback for the reset_timeout sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current reset timeout. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_reset_timeout(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->reset_timeout_ms); + + return ret; +} + +static DEVICE_ATTR(reset_timeout, S_IRUGO | S_IWUSR, show_reset_timeout, + set_reset_timeout); + + +static ssize_t show_mem_pool_size(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *const kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, + kbdev->mem_pools.small, MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_debugfs_size); +} + +static ssize_t set_mem_pool_size(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *const kbdev = to_kbase_device(dev); + int err; + + if (!kbdev) + return -ENODEV; + + err = kbase_debugfs_helper_set_attr_from_string(buf, + kbdev->mem_pools.small, MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_debugfs_trim); + + return err ? err : count; +} + +static DEVICE_ATTR(mem_pool_size, S_IRUGO | S_IWUSR, show_mem_pool_size, + set_mem_pool_size); + +static ssize_t show_mem_pool_max_size(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *const kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, + kbdev->mem_pools.small, MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_debugfs_max_size); +} + +static ssize_t set_mem_pool_max_size(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *const kbdev = to_kbase_device(dev); + int err; + + if (!kbdev) + return -ENODEV; + + err = kbase_debugfs_helper_set_attr_from_string(buf, + kbdev->mem_pools.small, MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_debugfs_set_max_size); + + return err ? err : count; +} + +static DEVICE_ATTR(mem_pool_max_size, S_IRUGO | S_IWUSR, show_mem_pool_max_size, + set_mem_pool_max_size); + +/** + * show_lp_mem_pool_size - Show size of the large memory pages pool. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the pool size. + * + * This function is called to get the number of large memory pages which currently populate the kbdev pool. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_lp_mem_pool_size(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *const kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, + kbdev->mem_pools.large, MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_debugfs_size); +} + +/** + * set_lp_mem_pool_size - Set size of the large memory pages pool. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The value written to the sysfs file. + * @count: The number of bytes written to the sysfs file. + * + * This function is called to set the number of large memory pages which should populate the kbdev pool. + * This may cause existing pages to be removed from the pool, or new pages to be created and then added to the pool. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_lp_mem_pool_size(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *const kbdev = to_kbase_device(dev); + int err; + + if (!kbdev) + return -ENODEV; + + err = kbase_debugfs_helper_set_attr_from_string(buf, + kbdev->mem_pools.large, MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_debugfs_trim); + + return err ? err : count; +} + +static DEVICE_ATTR(lp_mem_pool_size, S_IRUGO | S_IWUSR, show_lp_mem_pool_size, + set_lp_mem_pool_size); + +/** + * show_lp_mem_pool_max_size - Show maximum size of the large memory pages pool. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the pool size. + * + * This function is called to get the maximum number of large memory pages that the kbdev pool can possibly contain. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_lp_mem_pool_max_size(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *const kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, + kbdev->mem_pools.large, MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_debugfs_max_size); +} + +/** + * set_lp_mem_pool_max_size - Set maximum size of the large memory pages pool. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The value written to the sysfs file. + * @count: The number of bytes written to the sysfs file. + * + * This function is called to set the maximum number of large memory pages that the kbdev pool can possibly contain. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_lp_mem_pool_max_size(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *const kbdev = to_kbase_device(dev); + int err; + + if (!kbdev) + return -ENODEV; + + err = kbase_debugfs_helper_set_attr_from_string(buf, + kbdev->mem_pools.large, MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_debugfs_set_max_size); + + return err ? err : count; +} + +static DEVICE_ATTR(lp_mem_pool_max_size, S_IRUGO | S_IWUSR, show_lp_mem_pool_max_size, + set_lp_mem_pool_max_size); + +#if !MALI_USE_CSF +/** + * show_js_ctx_scheduling_mode - Show callback for js_ctx_scheduling_mode sysfs + * entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the context scheduling mode information. + * + * This function is called to get the context scheduling mode being used by JS. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_js_ctx_scheduling_mode(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + return scnprintf(buf, PAGE_SIZE, "%u\n", kbdev->js_ctx_scheduling_mode); +} + +/** + * set_js_ctx_scheduling_mode - Set callback for js_ctx_scheduling_mode sysfs + * entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The value written to the sysfs file. + * @count: The number of bytes written to the sysfs file. + * + * This function is called when the js_ctx_scheduling_mode sysfs file is written + * to. It checks the data written, and if valid updates the ctx scheduling mode + * being by JS. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_js_ctx_scheduling_mode(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_context *kctx; + u32 new_js_ctx_scheduling_mode; + struct kbase_device *kbdev; + unsigned long flags; + int ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtouint(buf, 0, &new_js_ctx_scheduling_mode); + if (ret || new_js_ctx_scheduling_mode >= KBASE_JS_PRIORITY_MODE_COUNT) { + dev_err(kbdev->dev, "Couldn't process js_ctx_scheduling_mode" + " write operation.\n" + "Use format \n"); + return -EINVAL; + } + + if (new_js_ctx_scheduling_mode == kbdev->js_ctx_scheduling_mode) + return count; + + mutex_lock(&kbdev->kctx_list_lock); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Update the context priority mode */ + kbdev->js_ctx_scheduling_mode = new_js_ctx_scheduling_mode; + + /* Adjust priority of all the contexts as per the new mode */ + list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) + kbase_js_update_ctx_priority(kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->kctx_list_lock); + + dev_dbg(kbdev->dev, "JS ctx scheduling mode: %u\n", new_js_ctx_scheduling_mode); + + return count; +} + +static DEVICE_ATTR(js_ctx_scheduling_mode, S_IRUGO | S_IWUSR, + show_js_ctx_scheduling_mode, + set_js_ctx_scheduling_mode); + +#ifdef MALI_KBASE_BUILD + +/* Number of entries in serialize_jobs_settings[] */ +#define NR_SERIALIZE_JOBS_SETTINGS 5 +/* Maximum string length in serialize_jobs_settings[].name */ +#define MAX_SERIALIZE_JOBS_NAME_LEN 16 + +static struct +{ + char *name; + u8 setting; +} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { + {"none", 0}, + {"intra-slot", KBASE_SERIALIZE_INTRA_SLOT}, + {"inter-slot", KBASE_SERIALIZE_INTER_SLOT}, + {"full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT}, + {"full-reset", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | + KBASE_SERIALIZE_RESET} +}; + +/** + * update_serialize_jobs_setting - Update the serialization setting for the + * submission of GPU jobs. + * + * This function is called when the serialize_jobs sysfs/debugfs file is + * written to. It matches the requested setting against the available settings + * and if a matching setting is found updates kbdev->serialize_jobs. + * + * @kbdev: An instance of the GPU platform device, allocated from the probe + * method of the driver. + * @buf: Buffer containing the value written to the sysfs/debugfs file. + * @count: The number of bytes to write to the sysfs/debugfs file. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t update_serialize_jobs_setting(struct kbase_device *kbdev, + const char *buf, size_t count) +{ + int i; + bool valid = false; + + for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { + if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { + kbdev->serialize_jobs = + serialize_jobs_settings[i].setting; + valid = true; + break; + } + } + + if (!valid) { + dev_err(kbdev->dev, "serialize_jobs: invalid setting"); + return -EINVAL; + } + + return count; +} + +#ifdef CONFIG_DEBUG_FS +/** + * kbasep_serialize_jobs_seq_debugfs_show - Show callback for the serialize_jobs + * debugfs file + * @sfile: seq_file pointer + * @data: Private callback data + * + * This function is called to get the contents of the serialize_jobs debugfs + * file. This is a list of the available settings with the currently active one + * surrounded by square brackets. + * + * Return: 0 on success, or an error code on error + */ +static int kbasep_serialize_jobs_seq_debugfs_show(struct seq_file *sfile, + void *data) +{ + struct kbase_device *kbdev = sfile->private; + int i; + + CSTD_UNUSED(data); + + for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { + if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) + seq_printf(sfile, "[%s] ", + serialize_jobs_settings[i].name); + else + seq_printf(sfile, "%s ", + serialize_jobs_settings[i].name); + } + + seq_puts(sfile, "\n"); + + return 0; +} + +/** + * kbasep_serialize_jobs_debugfs_write - Store callback for the serialize_jobs + * debugfs file. + * @file: File pointer + * @ubuf: User buffer containing data to store + * @count: Number of bytes in user buffer + * @ppos: File position + * + * This function is called when the serialize_jobs debugfs file is written to. + * It matches the requested setting against the available settings and if a + * matching setting is found updates kbdev->serialize_jobs. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct kbase_device *kbdev = s->private; + char buf[MAX_SERIALIZE_JOBS_NAME_LEN]; + + CSTD_UNUSED(ppos); + + count = min_t(size_t, sizeof(buf) - 1, count); + if (copy_from_user(buf, ubuf, count)) + return -EFAULT; + + buf[count] = 0; + + return update_serialize_jobs_setting(kbdev, buf, count); +} + +/** + * kbasep_serialize_jobs_debugfs_open - Open callback for the serialize_jobs + * debugfs file + * @in: inode pointer + * @file: file pointer + * + * Return: Zero on success, error code on failure + */ +static int kbasep_serialize_jobs_debugfs_open(struct inode *in, + struct file *file) +{ + return single_open(file, kbasep_serialize_jobs_seq_debugfs_show, + in->i_private); +} + +static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { + .owner = THIS_MODULE, + .open = kbasep_serialize_jobs_debugfs_open, + .read = seq_read, + .write = kbasep_serialize_jobs_debugfs_write, + .llseek = seq_lseek, + .release = single_release, +}; + +#endif /* CONFIG_DEBUG_FS */ + +/** + * show_serialize_jobs_sysfs - Show callback for serialize_jobs sysfs file. + * + * This function is called to get the contents of the serialize_jobs sysfs + * file. This is a list of the available settings with the currently active + * one surrounded by square brackets. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_serialize_jobs_sysfs(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + ssize_t ret = 0; + int i; + + for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { + if (kbdev->serialize_jobs == + serialize_jobs_settings[i].setting) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s]", + serialize_jobs_settings[i].name); + else + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", + serialize_jobs_settings[i].name); + } + + if (ret < PAGE_SIZE - 1) { + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); + } else { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/** + * store_serialize_jobs_sysfs - Store callback for serialize_jobs sysfs file. + * + * This function is called when the serialize_jobs sysfs file is written to. + * It matches the requested setting against the available settings and if a + * matching setting is found updates kbdev->serialize_jobs. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes to write to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t store_serialize_jobs_sysfs(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + return update_serialize_jobs_setting(to_kbase_device(dev), buf, count); +} + +static DEVICE_ATTR(serialize_jobs, 0600, show_serialize_jobs_sysfs, + store_serialize_jobs_sysfs); +#endif /* MALI_KBASE_BUILD */ +#endif /* !MALI_USE_CSF */ + +static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) +{ + struct kbase_device *kbdev = container_of(data, struct kbase_device, + protected_mode_hwcnt_disable_work); + unsigned long flags; + + bool do_disable; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + do_disable = !kbdev->protected_mode_hwcnt_desired && + !kbdev->protected_mode_hwcnt_disabled; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (!do_disable) + return; + + kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + do_disable = !kbdev->protected_mode_hwcnt_desired && + !kbdev->protected_mode_hwcnt_disabled; + + if (do_disable) { + /* Protected mode state did not change while we were doing the + * disable, so commit the work we just performed and continue + * the state machine. + */ + kbdev->protected_mode_hwcnt_disabled = true; +#if !MALI_USE_CSF + kbase_backend_slot_update(kbdev); +#endif /* !MALI_USE_CSF */ + } else { + /* Protected mode state was updated while we were doing the + * disable, so we need to undo the disable we just performed. + */ + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +static int kbasep_protected_mode_enable(struct protected_mode_device *pdev) +{ + struct kbase_device *kbdev = pdev->data; + + return kbase_pm_protected_mode_enable(kbdev); +} + +static int kbasep_protected_mode_disable(struct protected_mode_device *pdev) +{ + struct kbase_device *kbdev = pdev->data; + + return kbase_pm_protected_mode_disable(kbdev); +} + +static const struct protected_mode_ops kbasep_native_protected_ops = { + .protected_mode_enable = kbasep_protected_mode_enable, + .protected_mode_disable = kbasep_protected_mode_disable +}; + +int kbase_protected_mode_init(struct kbase_device *kbdev) +{ + /* Use native protected ops */ + kbdev->protected_dev = kzalloc(sizeof(*kbdev->protected_dev), + GFP_KERNEL); + if (!kbdev->protected_dev) + return -ENOMEM; + kbdev->protected_dev->data = kbdev; + kbdev->protected_ops = &kbasep_native_protected_ops; + INIT_WORK(&kbdev->protected_mode_hwcnt_disable_work, + kbasep_protected_mode_hwcnt_disable_worker); + kbdev->protected_mode_hwcnt_desired = true; + kbdev->protected_mode_hwcnt_disabled = false; + return 0; +} + +void kbase_protected_mode_term(struct kbase_device *kbdev) +{ + cancel_work_sync(&kbdev->protected_mode_hwcnt_disable_work); + kfree(kbdev->protected_dev); +} + +#ifdef CONFIG_MALI_BIFROST_NO_MALI +static int kbase_common_reg_map(struct kbase_device *kbdev) +{ + return 0; +} +static void kbase_common_reg_unmap(struct kbase_device * const kbdev) +{ +} +#else /* CONFIG_MALI_BIFROST_NO_MALI */ +static int kbase_common_reg_map(struct kbase_device *kbdev) +{ + int err = 0; + + if (!request_mem_region(kbdev->reg_start, kbdev->reg_size, dev_name(kbdev->dev))) { + dev_err(kbdev->dev, "Register window unavailable\n"); + err = -EIO; + goto out_region; + } + + kbdev->reg = ioremap(kbdev->reg_start, kbdev->reg_size); + if (!kbdev->reg) { + dev_err(kbdev->dev, "Can't remap register window\n"); + err = -EINVAL; + goto out_ioremap; + } + + return err; + +out_ioremap: + release_mem_region(kbdev->reg_start, kbdev->reg_size); +out_region: + return err; +} + +static void kbase_common_reg_unmap(struct kbase_device * const kbdev) +{ + if (kbdev->reg) { + iounmap(kbdev->reg); + release_mem_region(kbdev->reg_start, kbdev->reg_size); + kbdev->reg = NULL; + kbdev->reg_start = 0; + kbdev->reg_size = 0; + } +} +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ + +int registers_map(struct kbase_device * const kbdev) +{ + /* the first memory resource is the physical address of the GPU + * registers. + */ + struct platform_device *pdev = to_platform_device(kbdev->dev); + struct resource *reg_res; + int err; + + reg_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!reg_res) { + dev_err(kbdev->dev, "Invalid register resource\n"); + return -ENOENT; + } + + kbdev->reg_start = reg_res->start; + kbdev->reg_size = resource_size(reg_res); + +#if MALI_USE_CSF + if (kbdev->reg_size < + (CSF_HW_DOORBELL_PAGE_OFFSET + + CSF_NUM_DOORBELL * CSF_HW_DOORBELL_PAGE_SIZE)) { + dev_err(kbdev->dev, "Insufficient register space, will override to the required size\n"); + kbdev->reg_size = CSF_HW_DOORBELL_PAGE_OFFSET + + CSF_NUM_DOORBELL * CSF_HW_DOORBELL_PAGE_SIZE; + } +#endif + + err = kbase_common_reg_map(kbdev); + if (err) { + dev_err(kbdev->dev, "Failed to map registers\n"); + return err; + } + + return 0; +} + +void registers_unmap(struct kbase_device *kbdev) +{ + kbase_common_reg_unmap(kbdev); +} + +#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) + +static bool kbase_is_pm_enabled(const struct device_node *gpu_node) +{ + const struct device_node *power_model_node; + const void *cooling_cells_node; + const void *operating_point_node; + bool is_pm_enable = false; + + power_model_node = of_get_child_by_name(gpu_node, + "power_model"); + if (power_model_node) + is_pm_enable = true; + + cooling_cells_node = of_get_property(gpu_node, + "#cooling-cells", NULL); + if (cooling_cells_node) + is_pm_enable = true; + + operating_point_node = of_get_property(gpu_node, + "operating-points", NULL); + if (operating_point_node) + is_pm_enable = true; + + return is_pm_enable; +} + +static bool kbase_is_pv_enabled(const struct device_node *gpu_node) +{ + const void *arbiter_if_node; + + arbiter_if_node = of_get_property(gpu_node, + "arbiter_if", NULL); + + return arbiter_if_node ? true : false; +} + +static bool kbase_is_full_coherency_enabled(const struct device_node *gpu_node) +{ + const void *coherency_dts; + u32 coherency; + + coherency_dts = of_get_property(gpu_node, + "system-coherency", + NULL); + if (coherency_dts) { + coherency = be32_to_cpup(coherency_dts); + if (coherency == COHERENCY_ACE) + return true; + } + return false; +} + +#endif /* CONFIG_MALI_ARBITER_SUPPORT && CONFIG_OF */ + +int kbase_device_pm_init(struct kbase_device *kbdev) +{ + int err = 0; + +#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) + + u32 gpu_id; + u32 product_id; + u32 gpu_model_id; + + if (kbase_is_pv_enabled(kbdev->dev->of_node)) { + if (kbase_is_pm_enabled(kbdev->dev->of_node)) { + /* Arbitration AND power management invalid */ + dev_err(kbdev->dev, "Invalid combination of arbitration AND power management\n"); + return -EPERM; + } + if (kbase_is_full_coherency_enabled(kbdev->dev->of_node)) { + /* Arbitration AND full coherency invalid */ + dev_err(kbdev->dev, "Invalid combination of arbitration AND full coherency\n"); + return -EPERM; + } + err = kbase_arbiter_pm_early_init(kbdev); + if (err == 0) { + /* Check if Arbitration is running on + * supported GPU platform + */ + kbase_pm_register_access_enable(kbdev); + gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); + kbase_pm_register_access_disable(kbdev); + product_id = KBASE_UBFX32(gpu_id, + GPU_ID_VERSION_PRODUCT_ID_SHIFT, 16); + gpu_model_id = GPU_ID2_MODEL_MATCH_VALUE(product_id); + + if (gpu_model_id != GPU_ID2_PRODUCT_TGOX + && gpu_model_id != GPU_ID2_PRODUCT_TNOX) { + kbase_arbiter_pm_early_term(kbdev); + dev_err(kbdev->dev, "GPU platform not suitable for arbitration\n"); + return -EPERM; + } + } + } else { + err = power_control_init(kbdev); + } +#else + err = power_control_init(kbdev); +#endif /* CONFIG_MALI_ARBITER_SUPPORT && CONFIG_OF */ + return err; +} + +void kbase_device_pm_term(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#ifdef CONFIG_OF + if (kbase_is_pv_enabled(kbdev->dev->of_node)) + kbase_arbiter_pm_early_term(kbdev); + else + power_control_term(kbdev); +#endif /* CONFIG_OF */ +#else + power_control_term(kbdev); +#endif +} + +int power_control_init(struct kbase_device *kbdev) +{ +#if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF) + /* Power control initialization requires at least the capability to get + * regulators and clocks from the device tree, as well as parsing + * arrays of unsigned integer values. + * + * The whole initialization process shall simply be skipped if the + * minimum capability is not available. + */ + return 0; +#else + struct platform_device *pdev; + int err = 0; + unsigned int i; +#if defined(CONFIG_REGULATOR) + static const char *regulator_names[] = { + "mali", "shadercores" + }; + BUILD_BUG_ON(ARRAY_SIZE(regulator_names) < BASE_MAX_NR_CLOCKS_REGULATORS); +#endif /* CONFIG_REGULATOR */ + + if (!kbdev) + return -ENODEV; + + pdev = to_platform_device(kbdev->dev); + +#if defined(CONFIG_REGULATOR) + /* Since the error code EPROBE_DEFER causes the entire probing + * procedure to be restarted from scratch at a later time, + * all regulators will be released before returning. + * + * Any other error is ignored and the driver will continue + * operating with a partial initialization of regulators. + */ + for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { + kbdev->regulators[i] = regulator_get_optional(kbdev->dev, + regulator_names[i]); + if (IS_ERR_OR_NULL(kbdev->regulators[i])) { + err = PTR_ERR(kbdev->regulators[i]); + kbdev->regulators[i] = NULL; + break; + } + } + if (err == -EPROBE_DEFER) { + while ((i > 0) && (i < BASE_MAX_NR_CLOCKS_REGULATORS)) + regulator_put(kbdev->regulators[--i]); + return err; + } + + kbdev->nr_regulators = i; + dev_dbg(&pdev->dev, "Regulators probed: %u\n", kbdev->nr_regulators); +#endif + + /* Having more clocks than regulators is acceptable, while the + * opposite shall not happen. + * + * Since the error code EPROBE_DEFER causes the entire probing + * procedure to be restarted from scratch at a later time, + * all clocks and regulators will be released before returning. + * + * Any other error is ignored and the driver will continue + * operating with a partial initialization of clocks. + */ + for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { + kbdev->clocks[i] = of_clk_get(kbdev->dev->of_node, i); + if (IS_ERR_OR_NULL(kbdev->clocks[i])) { + err = PTR_ERR(kbdev->clocks[i]); + kbdev->clocks[i] = NULL; + break; + } + + err = clk_prepare(kbdev->clocks[i]); + if (err) { + dev_err(kbdev->dev, + "Failed to prepare and enable clock (%d)\n", + err); + clk_put(kbdev->clocks[i]); + break; + } + } + if (err == -EPROBE_DEFER) { + while ((i > 0) && (i < BASE_MAX_NR_CLOCKS_REGULATORS)) { + clk_unprepare(kbdev->clocks[--i]); + clk_put(kbdev->clocks[i]); + } + goto clocks_probe_defer; + } + + kbdev->nr_clocks = i; + dev_dbg(&pdev->dev, "Clocks probed: %u\n", kbdev->nr_clocks); + +#if defined(CONFIG_PM_OPP) +#if ((KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) && \ + defined(CONFIG_REGULATOR)) + if (kbdev->nr_regulators > 0) { + kbdev->opp_table = dev_pm_opp_set_regulators(kbdev->dev, + regulator_names, BASE_MAX_NR_CLOCKS_REGULATORS); + } +#endif /* (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE */ +#ifdef CONFIG_ARCH_ROCKCHIP + err = kbase_platform_rk_init_opp_table(kbdev); + if (err) + dev_err(kbdev->dev, "Failed to init_opp_table (%d)\n", err); +#else + err = dev_pm_opp_of_add_table(kbdev->dev); + CSTD_UNUSED(err); +#endif +#endif /* CONFIG_PM_OPP */ + +#endif /* KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE */ + return 0; + +clocks_probe_defer: +#if defined(CONFIG_REGULATOR) + for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) + regulator_put(kbdev->regulators[i]); +#endif + return err; +} + +void power_control_term(struct kbase_device *kbdev) +{ + unsigned int i; + +#if (KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE && \ + !defined(LSK_OPPV2_BACKPORT)) +#if KERNEL_VERSION(3, 19, 0) <= LINUX_VERSION_CODE + of_free_opp_table(kbdev->dev); +#endif +#else + +#if defined(CONFIG_PM_OPP) + dev_pm_opp_of_remove_table(kbdev->dev); +#if ((KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) && \ + defined(CONFIG_REGULATOR)) + if (!IS_ERR_OR_NULL(kbdev->opp_table)) + dev_pm_opp_put_regulators(kbdev->opp_table); +#endif /* (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE */ +#endif /* CONFIG_PM_OPP */ + +#endif /* KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE */ + + for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { + if (kbdev->clocks[i]) { + clk_unprepare(kbdev->clocks[i]); + clk_put(kbdev->clocks[i]); + kbdev->clocks[i] = NULL; + } else + break; + } + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ + && defined(CONFIG_REGULATOR) + for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { + if (kbdev->regulators[i]) { + regulator_put(kbdev->regulators[i]); + kbdev->regulators[i] = NULL; + } + } +#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ +} + +#ifdef MALI_KBASE_BUILD +#ifdef CONFIG_DEBUG_FS + +static void trigger_reset(struct kbase_device *kbdev) +{ + kbase_pm_context_active(kbdev); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + kbase_pm_context_idle(kbdev); +} + +#define MAKE_QUIRK_ACCESSORS(type) \ +static int type##_quirks_set(void *data, u64 val) \ +{ \ + struct kbase_device *kbdev; \ + kbdev = (struct kbase_device *)data; \ + kbdev->hw_quirks_##type = (u32)val; \ + trigger_reset(kbdev); \ + return 0;\ +} \ +\ +static int type##_quirks_get(void *data, u64 *val) \ +{ \ + struct kbase_device *kbdev;\ + kbdev = (struct kbase_device *)data;\ + *val = kbdev->hw_quirks_##type;\ + return 0;\ +} \ +DEFINE_SIMPLE_ATTRIBUTE(fops_##type##_quirks, type##_quirks_get,\ + type##_quirks_set, "%llu\n") + +MAKE_QUIRK_ACCESSORS(sc); +MAKE_QUIRK_ACCESSORS(tiler); +MAKE_QUIRK_ACCESSORS(mmu); +MAKE_QUIRK_ACCESSORS(jm); + +static ssize_t kbase_device_debugfs_reset_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *ppos) +{ + struct kbase_device *kbdev = file->private_data; + CSTD_UNUSED(ubuf); + CSTD_UNUSED(count); + CSTD_UNUSED(ppos); + + trigger_reset(kbdev); + + return count; +} + +static const struct file_operations fops_trigger_reset = { + .owner = THIS_MODULE, + .open = simple_open, + .write = kbase_device_debugfs_reset_write, + .llseek = default_llseek, +}; + +/** + * debugfs_protected_debug_mode_read - "protected_debug_mode" debugfs read + * @file: File object to read is for + * @buf: User buffer to populate with data + * @len: Length of user buffer + * @ppos: Offset within file object + * + * Retrieves the current status of protected debug mode + * (0 = disabled, 1 = enabled) + * + * Return: Number of bytes added to user buffer + */ +static ssize_t debugfs_protected_debug_mode_read(struct file *file, + char __user *buf, size_t len, loff_t *ppos) +{ + struct kbase_device *kbdev = (struct kbase_device *)file->private_data; + u32 gpu_status; + ssize_t ret_val; + + kbase_pm_context_active(kbdev); + gpu_status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)); + kbase_pm_context_idle(kbdev); + + if (gpu_status & GPU_DBGEN) + ret_val = simple_read_from_buffer(buf, len, ppos, "1\n", 2); + else + ret_val = simple_read_from_buffer(buf, len, ppos, "0\n", 2); + + return ret_val; +} + +/* + * struct fops_protected_debug_mode - "protected_debug_mode" debugfs fops + * + * Contains the file operations for the "protected_debug_mode" debugfs file + */ +static const struct file_operations fops_protected_debug_mode = { + .owner = THIS_MODULE, + .open = simple_open, + .read = debugfs_protected_debug_mode_read, + .llseek = default_llseek, +}; + +static int kbase_device_debugfs_mem_pool_max_size_show(struct seq_file *sfile, + void *data) +{ + CSTD_UNUSED(data); + return kbase_debugfs_helper_seq_read(sfile, + MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_config_debugfs_max_size); +} + +static ssize_t kbase_device_debugfs_mem_pool_max_size_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *ppos) +{ + int err = 0; + + CSTD_UNUSED(ppos); + err = kbase_debugfs_helper_seq_write(file, ubuf, count, + MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_config_debugfs_set_max_size); + + return err ? err : count; +} + +static int kbase_device_debugfs_mem_pool_max_size_open(struct inode *in, + struct file *file) +{ + return single_open(file, kbase_device_debugfs_mem_pool_max_size_show, + in->i_private); +} + +static const struct file_operations + kbase_device_debugfs_mem_pool_max_size_fops = { + .owner = THIS_MODULE, + .open = kbase_device_debugfs_mem_pool_max_size_open, + .read = seq_read, + .write = kbase_device_debugfs_mem_pool_max_size_write, + .llseek = seq_lseek, + .release = single_release, +}; + +int kbase_device_debugfs_init(struct kbase_device *kbdev) +{ + struct dentry *debugfs_ctx_defaults_directory; + int err; + /* prevent unprivileged use of debug file system + * in old kernel version + */ +#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) + /* only for newer kernel version debug file system is safe */ + const mode_t mode = 0644; +#else + const mode_t mode = 0600; +#endif + + kbdev->mali_debugfs_directory = debugfs_create_dir(kbdev->devname, + NULL); + if (!kbdev->mali_debugfs_directory) { + dev_err(kbdev->dev, "Couldn't create mali debugfs directory\n"); + err = -ENOMEM; + goto out; + } + + kbdev->debugfs_ctx_directory = debugfs_create_dir("ctx", + kbdev->mali_debugfs_directory); + if (!kbdev->debugfs_ctx_directory) { + dev_err(kbdev->dev, "Couldn't create mali debugfs ctx directory\n"); + err = -ENOMEM; + goto out; + } + + kbdev->debugfs_instr_directory = debugfs_create_dir("instrumentation", + kbdev->mali_debugfs_directory); + if (!kbdev->debugfs_instr_directory) { + dev_err(kbdev->dev, "Couldn't create mali debugfs instrumentation directory\n"); + err = -ENOMEM; + goto out; + } + + debugfs_ctx_defaults_directory = debugfs_create_dir("defaults", + kbdev->debugfs_ctx_directory); + if (!debugfs_ctx_defaults_directory) { + dev_err(kbdev->dev, "Couldn't create mali debugfs ctx defaults directory\n"); + err = -ENOMEM; + goto out; + } + +#if !MALI_CUSTOMER_RELEASE + kbasep_regs_dump_debugfs_init(kbdev); +#endif /* !MALI_CUSTOMER_RELEASE */ + kbasep_regs_history_debugfs_init(kbdev); + +#if !MALI_USE_CSF + kbase_debug_job_fault_debugfs_init(kbdev); +#endif /* !MALI_USE_CSF */ + + kbasep_gpu_memory_debugfs_init(kbdev); + kbase_as_fault_debugfs_init(kbdev); +#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS + kbase_instr_backend_debugfs_init(kbdev); +#endif + /* fops_* variables created by invocations of macro + * MAKE_QUIRK_ACCESSORS() above. */ + debugfs_create_file("quirks_sc", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_sc_quirks); + debugfs_create_file("quirks_tiler", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_tiler_quirks); + debugfs_create_file("quirks_mmu", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_mmu_quirks); + debugfs_create_file("quirks_jm", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_jm_quirks); + + debugfs_create_bool("infinite_cache", mode, + debugfs_ctx_defaults_directory, + &kbdev->infinite_cache_active_default); + + debugfs_create_file("mem_pool_max_size", mode, + debugfs_ctx_defaults_directory, + &kbdev->mem_pool_defaults.small, + &kbase_device_debugfs_mem_pool_max_size_fops); + + debugfs_create_file("lp_mem_pool_max_size", mode, + debugfs_ctx_defaults_directory, + &kbdev->mem_pool_defaults.large, + &kbase_device_debugfs_mem_pool_max_size_fops); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { + debugfs_create_file("protected_debug_mode", S_IRUGO, + kbdev->mali_debugfs_directory, kbdev, + &fops_protected_debug_mode); + } + + debugfs_create_file("reset", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_trigger_reset); + + kbase_ktrace_debugfs_init(kbdev); + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ +#ifdef CONFIG_DEVFREQ_THERMAL + if (kbdev->devfreq && !kbdev->model_data) + kbase_ipa_debugfs_init(kbdev); +#endif /* CONFIG_DEVFREQ_THERMAL */ +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + +#if MALI_USE_CSF + kbase_csf_debugfs_init(kbdev); +#else + debugfs_create_file("serialize_jobs", S_IRUGO | S_IWUSR, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_serialize_jobs_debugfs_fops); +#endif + + return 0; + +out: + debugfs_remove_recursive(kbdev->mali_debugfs_directory); + return err; +} + +void kbase_device_debugfs_term(struct kbase_device *kbdev) +{ + debugfs_remove_recursive(kbdev->mali_debugfs_directory); +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* MALI_KBASE_BUILD */ + +int kbase_device_coherency_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_OF + u32 supported_coherency_bitmap = + kbdev->gpu_props.props.raw_props.coherency_mode; + const void *coherency_override_dts; + u32 override_coherency, gpu_id; + unsigned int prod_id; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + gpu_id &= GPU_ID_VERSION_PRODUCT_ID; + prod_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + /* Only for tMIx : + * (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly + * documented for tMIx so force correct value here. + */ + if (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == + GPU_ID2_PRODUCT_TMIX) + if (supported_coherency_bitmap == + COHERENCY_FEATURE_BIT(COHERENCY_ACE)) + supported_coherency_bitmap |= + COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); + +#endif /* CONFIG_OF */ + + kbdev->system_coherency = COHERENCY_NONE; + + /* device tree may override the coherency */ +#ifdef CONFIG_OF + coherency_override_dts = of_get_property(kbdev->dev->of_node, + "system-coherency", + NULL); + if (coherency_override_dts) { + + override_coherency = be32_to_cpup(coherency_override_dts); + + if ((override_coherency <= COHERENCY_NONE) && + (supported_coherency_bitmap & + COHERENCY_FEATURE_BIT(override_coherency))) { + + kbdev->system_coherency = override_coherency; + + dev_info(kbdev->dev, + "Using coherency mode %u set from dtb", + override_coherency); + } else + dev_warn(kbdev->dev, + "Ignoring unsupported coherency mode %u set from dtb", + override_coherency); + } + +#endif /* CONFIG_OF */ + + kbdev->gpu_props.props.raw_props.coherency_mode = + kbdev->system_coherency; + + return 0; +} + +#ifdef CONFIG_MALI_BUSLOG + +/* Callback used by the kbase bus logger client, to initiate a GPU reset + * when the bus log is restarted. GPU reset is used as reference point + * in HW bus log analyses. + */ +static void kbase_logging_started_cb(void *data) +{ + struct kbase_device *kbdev = (struct kbase_device *)data; + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + dev_info(kbdev->dev, "KBASE - Bus logger restarted\n"); +} + +int buslog_init(struct kbase_device *kbdev) +{ + int err = 0; + + err = bl_core_client_register(kbdev->devname, + kbase_logging_started_cb, + kbdev, &kbdev->buslogger, + THIS_MODULE, NULL); + if (err == 0) + bl_core_set_threshold(kbdev->buslogger, 1024*1024*1024); + + return err; +} + +void buslog_term(struct kbase_device *kbdev) +{ + bl_core_client_unregister(kbdev->buslogger); +} +#endif + +static struct attribute *kbase_scheduling_attrs[] = { +#if !MALI_USE_CSF + &dev_attr_serialize_jobs.attr, +#endif /* !MALI_USE_CSF */ + NULL +}; + +static struct attribute *kbase_attrs[] = { +#ifdef CONFIG_MALI_BIFROST_DEBUG + &dev_attr_debug_command.attr, +#if !MALI_USE_CSF + &dev_attr_js_softstop_always.attr, +#endif /* !MALI_USE_CSF */ +#endif +#if !MALI_USE_CSF + &dev_attr_js_timeouts.attr, + &dev_attr_soft_job_timeout.attr, +#endif /* !MALI_USE_CSF */ + &dev_attr_gpuinfo.attr, + &dev_attr_dvfs_period.attr, + &dev_attr_pm_poweroff.attr, + &dev_attr_reset_timeout.attr, +#if !MALI_USE_CSF + &dev_attr_js_scheduling_period.attr, +#endif /* !MALI_USE_CSF */ + &dev_attr_power_policy.attr, + &dev_attr_core_mask.attr, + &dev_attr_mem_pool_size.attr, + &dev_attr_mem_pool_max_size.attr, + &dev_attr_lp_mem_pool_size.attr, + &dev_attr_lp_mem_pool_max_size.attr, +#if !MALI_USE_CSF + &dev_attr_js_ctx_scheduling_mode.attr, +#endif /* !MALI_USE_CSF */ + NULL +}; + +#define SYSFS_SCHEDULING_GROUP "scheduling" +static const struct attribute_group kbase_scheduling_attr_group = { + .name = SYSFS_SCHEDULING_GROUP, + .attrs = kbase_scheduling_attrs, +}; + +static const struct attribute_group kbase_attr_group = { + .attrs = kbase_attrs, +}; + +int kbase_sysfs_init(struct kbase_device *kbdev) +{ + int err = 0; + + kbdev->mdev.minor = MISC_DYNAMIC_MINOR; + kbdev->mdev.name = kbdev->devname; + kbdev->mdev.fops = &kbase_fops; + kbdev->mdev.parent = get_device(kbdev->dev); + kbdev->mdev.mode = 0666; + + err = sysfs_create_group(&kbdev->dev->kobj, &kbase_attr_group); + if (!err) { + err = sysfs_create_group(&kbdev->dev->kobj, + &kbase_scheduling_attr_group); + if (err) { + dev_err(kbdev->dev, "Creation of %s sysfs group failed", + SYSFS_SCHEDULING_GROUP); + sysfs_remove_group(&kbdev->dev->kobj, + &kbase_attr_group); + } + } + + return err; +} + +void kbase_sysfs_term(struct kbase_device *kbdev) +{ + sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); + sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); + put_device(kbdev->dev); +} + +static int kbase_platform_device_remove(struct platform_device *pdev) +{ + struct kbase_device *kbdev = to_kbase_device(&pdev->dev); + + if (!kbdev) + return -ENODEV; + + kbase_device_term(kbdev); + dev_set_drvdata(kbdev->dev, NULL); + kbase_device_free(kbdev); + + return 0; +} + +void kbase_backend_devfreq_term(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + if (kbdev->devfreq) + kbase_devfreq_term(kbdev); +#endif +} + +int kbase_backend_devfreq_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + /* Devfreq uses hardware counters, so must be initialized after it. */ + int err = kbase_devfreq_init(kbdev); + + if (err) + dev_err(kbdev->dev, "Continuing without devfreq\n"); +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + return 0; +} + +static int kbase_platform_device_probe(struct platform_device *pdev) +{ + struct kbase_device *kbdev; + int err = 0; + + mali_kbase_print_cs_experimental(); + + kbdev = kbase_device_alloc(); + if (!kbdev) { + dev_err(&pdev->dev, "Allocate device failed\n"); + return -ENOMEM; + } + + kbdev->dev = &pdev->dev; + dev_set_drvdata(kbdev->dev, kbdev); + + err = kbase_device_init(kbdev); + + if (err) { + if (err == -EPROBE_DEFER) + dev_err(kbdev->dev, "Device initialization Deferred\n"); + else + dev_err(kbdev->dev, "Device initialization failed\n"); + + dev_set_drvdata(kbdev->dev, NULL); + kbase_device_free(kbdev); + } else { +#ifdef MALI_KBASE_BUILD + dev_info(kbdev->dev, + "Probed as %s\n", dev_name(kbdev->mdev.this_device)); +#endif /* MALI_KBASE_BUILD */ + kbase_increment_device_id(); +#ifdef CONFIG_MALI_ARBITER_SUPPORT + mutex_lock(&kbdev->pm.lock); + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_INITIALIZED_EVT); + mutex_unlock(&kbdev->pm.lock); +#endif + } + + return err; +} + +#undef KBASEP_DEFAULT_REGISTER_HISTORY_SIZE + +/** + * kbase_device_suspend - Suspend callback from the OS. + * + * This is called by Linux when the device should suspend. + * + * @dev: The device to suspend + * + * Return: A standard Linux error code + */ +static int kbase_device_suspend(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + kbase_pm_suspend(kbdev); + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + dev_dbg(dev, "Callback %s\n", __func__); + if (kbdev->devfreq) { + kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_SUSPEND); + flush_workqueue(kbdev->devfreq_queue.workq); + } +#endif + return 0; +} + +/** + * kbase_device_resume - Resume callback from the OS. + * + * This is called by Linux when the device should resume from suspension. + * + * @dev: The device to resume + * + * Return: A standard Linux error code + */ +static int kbase_device_resume(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + kbase_pm_resume(kbdev); + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + dev_dbg(dev, "Callback %s\n", __func__); + if (kbdev->devfreq) { + mutex_lock(&kbdev->pm.lock); + if (kbdev->pm.active_count > 0) + kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_RESUME); + mutex_unlock(&kbdev->pm.lock); + flush_workqueue(kbdev->devfreq_queue.workq); + } +#endif + return 0; +} + +/** + * kbase_device_runtime_suspend - Runtime suspend callback from the OS. + * + * This is called by Linux when the device should prepare for a condition in + * which it will not be able to communicate with the CPU(s) and RAM due to + * power management. + * + * @dev: The device to suspend + * + * Return: A standard Linux error code + */ +#ifdef KBASE_PM_RUNTIME +static int kbase_device_runtime_suspend(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + if (kbdev->devfreq) + kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_SUSPEND); +#endif + + if (kbdev->pm.backend.callback_power_runtime_off) { + kbdev->pm.backend.callback_power_runtime_off(kbdev); + dev_dbg(dev, "runtime suspend\n"); + } + return 0; +} +#endif /* KBASE_PM_RUNTIME */ + +/** + * kbase_device_runtime_resume - Runtime resume callback from the OS. + * + * This is called by Linux when the device should go into a fully active state. + * + * @dev: The device to suspend + * + * Return: A standard Linux error code + */ + +#ifdef KBASE_PM_RUNTIME +static int kbase_device_runtime_resume(struct device *dev) +{ + int ret = 0; + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + dev_dbg(dev, "Callback %s\n", __func__); + if (kbdev->pm.backend.callback_power_runtime_on) { + ret = kbdev->pm.backend.callback_power_runtime_on(kbdev); + dev_dbg(dev, "runtime resume\n"); + } + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + if (kbdev->devfreq) + kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_RESUME); +#endif + + return ret; +} +#endif /* KBASE_PM_RUNTIME */ + + +#ifdef KBASE_PM_RUNTIME +/** + * kbase_device_runtime_idle - Runtime idle callback from the OS. + * @dev: The device to suspend + * + * This is called by Linux when the device appears to be inactive and it might + * be placed into a low power state. + * + * Return: 0 if device can be suspended, non-zero to avoid runtime autosuspend, + * otherwise a standard Linux error code + */ +static int kbase_device_runtime_idle(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + dev_dbg(dev, "Callback %s\n", __func__); + /* Use platform specific implementation if it exists. */ + if (kbdev->pm.backend.callback_power_runtime_idle) + return kbdev->pm.backend.callback_power_runtime_idle(kbdev); + + /* Just need to update the device's last busy mark. Kernel will respect + * the autosuspend delay and so won't suspend the device immediately. + */ + pm_runtime_mark_last_busy(kbdev->dev); + return 0; +} +#endif /* KBASE_PM_RUNTIME */ + +/* The power management operations for the platform driver. + */ +static const struct dev_pm_ops kbase_pm_ops = { + .suspend = kbase_device_suspend, + .resume = kbase_device_resume, +#ifdef KBASE_PM_RUNTIME + .runtime_suspend = kbase_device_runtime_suspend, + .runtime_resume = kbase_device_runtime_resume, + .runtime_idle = kbase_device_runtime_idle, +#endif /* KBASE_PM_RUNTIME */ +}; + +#ifdef CONFIG_OF +static const struct of_device_id kbase_dt_ids[] = { + { .compatible = "arm,mali-bifrost" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, kbase_dt_ids); +#endif + +static struct platform_driver kbase_platform_driver = { + .probe = kbase_platform_device_probe, + .remove = kbase_platform_device_remove, + .driver = { + .name = kbase_drv_name, + .owner = THIS_MODULE, + .pm = &kbase_pm_ops, + .of_match_table = of_match_ptr(kbase_dt_ids), + }, +}; + +/* + * The driver will not provide a shortcut to create the Mali platform device + * anymore when using Device Tree. + */ +#ifdef CONFIG_OF +module_platform_driver(kbase_platform_driver); +#else + +static int __init kbase_driver_init(void) +{ + int ret; + + ret = kbase_platform_register(); + if (ret) + return ret; + + ret = platform_driver_register(&kbase_platform_driver); + + if (ret) + kbase_platform_unregister(); + + return ret; +} + +static void __exit kbase_driver_exit(void) +{ + platform_driver_unregister(&kbase_platform_driver); + kbase_platform_unregister(); +} + +module_init(kbase_driver_init); +module_exit(kbase_driver_exit); + +#endif /* CONFIG_OF */ + +MODULE_LICENSE("GPL"); +MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \ + __stringify(BASE_UK_VERSION_MAJOR) "." \ + __stringify(BASE_UK_VERSION_MINOR) ")"); + +#define CREATE_TRACE_POINTS +/* Create the trace points (otherwise we just get code to call a tracepoint) */ +#include "mali_linux_trace.h" + +#ifdef CONFIG_MALI_BIFROST_GATOR_SUPPORT +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_job_slots_event); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_status); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_page_fault_insert_pages); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_total_alloc_pages_change); + +void kbase_trace_mali_pm_status(u32 dev_id, u32 event, u64 value) +{ + trace_mali_pm_status(dev_id, event, value); +} + +void kbase_trace_mali_job_slots_event(u32 dev_id, u32 event, const struct kbase_context *kctx, u8 atom_id) +{ + trace_mali_job_slots_event(dev_id, event, + (kctx != NULL ? kctx->tgid : 0), + (kctx != NULL ? kctx->pid : 0), + atom_id); +} + +void kbase_trace_mali_page_fault_insert_pages(u32 dev_id, int event, u32 value) +{ + trace_mali_page_fault_insert_pages(dev_id, event, value); +} + +void kbase_trace_mali_total_alloc_pages_change(u32 dev_id, long long int event) +{ + trace_mali_total_alloc_pages_change(dev_id, event); +} +#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h b/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h new file mode 100755 index 000000000000..caba2cd7a0e3 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h @@ -0,0 +1,51 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ + +/* + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + */ + +#ifndef _KBASE_CS_EXPERIMENTAL_H_ +#define _KBASE_CS_EXPERIMENTAL_H_ + +#include + +/** + * mali_kbase_print_cs_experimental() - Print a string for every Core Services + * experimental feature that is enabled + */ +static inline void mali_kbase_print_cs_experimental(void) +{ +#if MALI_INCREMENTAL_RENDERING + pr_info("mali_kbase: INCREMENTAL_RENDERING (experimental) enabled"); +#endif /* MALI_INCREMENTAL_RENDERING */ +} + +#endif /* _KBASE_CS_EXPERIMENTAL_H_ */ + + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c new file mode 100755 index 000000000000..750dbd8c3924 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c @@ -0,0 +1,355 @@ +/* + * + * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include + +#include +#include "mali_kbase_ctx_sched.h" +#include "tl/mali_kbase_tracepoints.h" + +/* Helper for ktrace */ +#if KBASE_KTRACE_ENABLE +static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) +{ + return atomic_read(&kctx->refcount); +} +#else /* KBASE_KTRACE_ENABLE */ +static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) +{ + CSTD_UNUSED(kctx); + return 0; +} +#endif /* KBASE_KTRACE_ENABLE */ + +int kbase_ctx_sched_init(struct kbase_device *kbdev) +{ + int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; + + /* These two must be recalculated if nr_hw_address_spaces changes + * (e.g. for HW workarounds) */ + kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; + kbdev->as_free = as_present; /* All ASs initially free */ + + memset(kbdev->as_to_kctx, 0, sizeof(kbdev->as_to_kctx)); + + return 0; +} + +void kbase_ctx_sched_term(struct kbase_device *kbdev) +{ + s8 i; + + /* Sanity checks */ + for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { + WARN_ON(kbdev->as_to_kctx[i] != NULL); + WARN_ON(!(kbdev->as_free & (1u << i))); + } +} + +/* kbasep_ctx_sched_find_as_for_ctx - Find a free address space + * + * @kbdev: The context for which to find a free address space + * + * Return: A valid AS if successful, otherwise KBASEP_AS_NR_INVALID + * + * This function returns an address space available for use. It would prefer + * returning an AS that has been previously assigned to the context to + * avoid having to reprogram the MMU. + */ +static int kbasep_ctx_sched_find_as_for_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + int free_as; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* First check if the previously assigned AS is available */ + if ((kctx->as_nr != KBASEP_AS_NR_INVALID) && + (kbdev->as_free & (1u << kctx->as_nr))) + return kctx->as_nr; + + /* The previously assigned AS was taken, we'll be returning any free + * AS at this point. + */ + free_as = ffs(kbdev->as_free) - 1; + if (free_as >= 0 && free_as < kbdev->nr_hw_address_spaces) + return free_as; + + return KBASEP_AS_NR_INVALID; +} + +int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(!kbdev->pm.backend.gpu_powered); + + if (atomic_inc_return(&kctx->refcount) == 1) { + int const free_as = kbasep_ctx_sched_find_as_for_ctx(kctx); + + if (free_as != KBASEP_AS_NR_INVALID) { + kbdev->as_free &= ~(1u << free_as); + /* Only program the MMU if the context has not been + * assigned the same address space before. + */ + if (free_as != kctx->as_nr) { + struct kbase_context *const prev_kctx = + kbdev->as_to_kctx[free_as]; + + if (prev_kctx) { + WARN_ON(atomic_read(&prev_kctx->refcount) != 0); + kbase_mmu_disable(prev_kctx); + KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( + kbdev, prev_kctx->id); + prev_kctx->as_nr = KBASEP_AS_NR_INVALID; + } + + kctx->as_nr = free_as; + kbdev->as_to_kctx[free_as] = kctx; + KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( + kbdev, kctx->id, free_as); + kbase_mmu_update(kbdev, &kctx->mmu, + kctx->as_nr); + } + } else { + atomic_dec(&kctx->refcount); + + /* Failed to find an available address space, we must + * be returning an error at this point. + */ + WARN_ON(kctx->as_nr != KBASEP_AS_NR_INVALID); + } + } + + return kctx->as_nr; +} + +void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->hwaccess_lock); + WARN_ON(atomic_read(&kctx->refcount) == 0); + WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID); + WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx); + + atomic_inc(&kctx->refcount); +} + +void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + int new_ref_count; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + new_ref_count = atomic_dec_return(&kctx->refcount); + if (new_ref_count == 0) { + kbdev->as_free |= (1u << kctx->as_nr); + if (kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) { + KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( + kbdev, kctx->id); + kbdev->as_to_kctx[kctx->as_nr] = NULL; + kctx->as_nr = KBASEP_AS_NR_INVALID; + kbase_ctx_flag_clear(kctx, KCTX_AS_DISABLED_ON_FAULT); + } + } + + KBASE_KTRACE_ADD(kbdev, SCHED_RELEASE_CTX, kctx, new_ref_count); +} + +void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(atomic_read(&kctx->refcount) != 0); + + if (kctx->as_nr != KBASEP_AS_NR_INVALID) { + if (kbdev->pm.backend.gpu_powered) + kbase_mmu_disable(kctx); + + KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(kbdev, kctx->id); + kbdev->as_to_kctx[kctx->as_nr] = NULL; + kctx->as_nr = KBASEP_AS_NR_INVALID; + } +} + +void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) +{ + s8 i; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(!kbdev->pm.backend.gpu_powered); + + for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { + struct kbase_context *kctx; + +#if MALI_USE_CSF + if ((i == MCU_AS_NR) && kbdev->csf.firmware_inited) { + kbase_mmu_update(kbdev, &kbdev->csf.mcu_mmu, + MCU_AS_NR); + continue; + } +#endif + kctx = kbdev->as_to_kctx[i]; + if (kctx) { + if (atomic_read(&kctx->refcount)) { + WARN_ON(kctx->as_nr != i); + + kbase_mmu_update(kbdev, &kctx->mmu, + kctx->as_nr); + kbase_ctx_flag_clear(kctx, + KCTX_AS_DISABLED_ON_FAULT); + } else { + /* This context might have been assigned an + * AS before, clear it. + */ + if (kctx->as_nr != KBASEP_AS_NR_INVALID) { + KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( + kbdev, kctx->id); + kbdev->as_to_kctx[kctx->as_nr] = NULL; + kctx->as_nr = KBASEP_AS_NR_INVALID; + } + } + } else { + kbase_mmu_disable_as(kbdev, i); + } + } +} + +struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount( + struct kbase_device *kbdev, size_t as_nr) +{ + unsigned long flags; + struct kbase_context *found_kctx = NULL; + + if (WARN_ON(kbdev == NULL)) + return NULL; + + if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) + return NULL; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + found_kctx = kbdev->as_to_kctx[as_nr]; + + if (found_kctx != NULL) + kbase_ctx_sched_retain_ctx_refcount(found_kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return found_kctx; +} + +struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, + size_t as_nr) +{ + unsigned long flags; + struct kbase_context *found_kctx; + + if (WARN_ON(kbdev == NULL)) + return NULL; + + if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) + return NULL; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + found_kctx = kbdev->as_to_kctx[as_nr]; + + if (found_kctx) { + if (WARN_ON(atomic_read(&found_kctx->refcount) <= 0)) + found_kctx = NULL; + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return found_kctx; +} + +bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx) +{ + bool result = false; + int as_nr; + + if (WARN_ON(kctx == NULL)) + return result; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + as_nr = kctx->as_nr; + if (atomic_read(&kctx->refcount) > 0) { + KBASE_DEBUG_ASSERT(as_nr >= 0); + + kbase_ctx_sched_retain_ctx_refcount(kctx); + KBASE_KTRACE_ADD(kctx->kbdev, SCHED_RETAIN_CTX_NOLOCK, kctx, + kbase_ktrace_get_ctx_refcnt(kctx)); + result = true; + } + + return result; +} + +bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx) +{ + unsigned long flags; + bool result = false; + + if (WARN_ON(kctx == NULL)) + return result; + + if (WARN_ON(kctx->kbdev == NULL)) + return result; + + mutex_lock(&kctx->kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + result = kbase_ctx_sched_inc_refcount_nolock(kctx); + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); + mutex_unlock(&kctx->kbdev->mmu_hw_mutex); + + return result; +} + +void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx) +{ + unsigned long flags; + + if (WARN_ON(!kctx)) + return; + + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + + if (!WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID) && + !WARN_ON(atomic_read(&kctx->refcount) <= 0)) + kbase_ctx_sched_release_ctx(kctx); + + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h new file mode 100755 index 000000000000..1affa719e6dc --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h @@ -0,0 +1,209 @@ +/* + * + * (C) COPYRIGHT 2017-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_CTX_SCHED_H_ +#define _KBASE_CTX_SCHED_H_ + +#include + +/** + * The Context Scheduler manages address space assignment and reference + * counting to kbase_context. The interface has been designed to minimise + * interactions between the Job Scheduler and Power Management/MMU to support + * the existing Job Scheduler interface. + * + * The initial implementation of the Context Scheduler does not schedule + * contexts. Instead it relies on the Job Scheduler to make decisions of + * when to schedule/evict contexts if address spaces are starved. In the + * future, once an interface between the CS and JS has been devised to + * provide enough information about how each context is consuming GPU resources, + * those decisions can be made in the CS itself, thereby reducing duplicated + * code. + */ + +/** + * kbase_ctx_sched_init - Initialise the context scheduler + * @kbdev: The device for which the context scheduler needs to be initialised + * + * This must be called during device initialisation. The number of hardware + * address spaces must already be established before calling this function. + * + * Return: 0 for success, otherwise failure + */ +int kbase_ctx_sched_init(struct kbase_device *kbdev); + +/** + * kbase_ctx_sched_term - Terminate the context scheduler + * @kbdev: The device for which the context scheduler needs to be terminated + * + * This must be called during device termination after all contexts have been + * destroyed. + */ +void kbase_ctx_sched_term(struct kbase_device *kbdev); + +/** + * kbase_ctx_sched_retain_ctx - Retain a reference to the @ref kbase_context + * @kctx: The context to which to retain a reference + * + * This function should be called whenever an address space should be assigned + * to a context and programmed onto the MMU. It should typically be called + * when jobs are ready to be submitted to the GPU. + * + * It can be called as many times as necessary. The address space will be + * assigned to the context for as long as there is a reference to said context. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + * + * Return: The address space that the context has been assigned to or + * KBASEP_AS_NR_INVALID if no address space was available. + */ +int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx); + +/** + * kbase_ctx_sched_retain_ctx_refcount + * @kctx: The context to which to retain a reference + * + * This function only retains a reference to the context. It must be called + * only when the context already has a reference. + * + * This is typically called inside an atomic session where we know the context + * is already scheduled in but want to take an extra reference to ensure that + * it doesn't get descheduled. + * + * The kbase_device::hwaccess_lock must be held whilst calling this function + */ +void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx); + +/** + * kbase_ctx_sched_release_ctx - Release a reference to the @ref kbase_context + * @kctx: The context from which to release a reference + * + * This function should be called whenever an address space could be unassigned + * from a context. When there are no more references to said context, the + * address space previously assigned to this context shall be reassigned to + * other contexts as needed. + * + * The kbase_device::hwaccess_lock must be held whilst calling this function + */ +void kbase_ctx_sched_release_ctx(struct kbase_context *kctx); + +/** + * kbase_ctx_sched_remove_ctx - Unassign previously assigned address space + * @kctx: The context to be removed + * + * This function should be called when a context is being destroyed. The + * context must no longer have any reference. If it has been assigned an + * address space before then the AS will be unprogrammed. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); + +/** + * kbase_ctx_sched_restore_all_as - Reprogram all address spaces + * @kbdev: The device for which address spaces to be reprogrammed + * + * This function shall reprogram all address spaces previously assigned to + * contexts. It can be used after the GPU is reset. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev); + +/** + * kbase_ctx_sched_as_to_ctx_refcount - Lookup a context based on its current + * address space and ensure that is stays scheduled in + * @kbdev: The device for which the returned context must belong + * @as_nr: address space assigned to the context of interest + * + * The context is refcounted as being busy to prevent it from scheduling + * out. It must be released with kbase_ctx_sched_release_ctx() when it is no + * longer required to stay scheduled in. + * + * This function can safely be called from IRQ context. + * + * The following locking conditions are made on the caller: + * * it must not hold the kbase_device::hwaccess_lock, because it will be used + * internally. + * + * Return: a valid struct kbase_context on success, which has been refcounted + * as being busy or return NULL on failure, indicating that no context was found + * in as_nr. + */ +struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount( + struct kbase_device *kbdev, size_t as_nr); + +/** + * kbase_ctx_sched_as_to_ctx - Lookup a context based on its current address + * space + * @kbdev: The device for which the returned context must belong + * @as_nr: address space assigned to the context of interest + * + * Return: a valid struct kbase_context on success or NULL on failure, + * indicating that no context was found in as_nr. + */ +struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, + size_t as_nr); + +/** + * kbase_ctx_sched_inc_refcount_nolock - Refcount a context as being busy, + * preventing it from being scheduled out. + * @kctx: Context to be refcounted + * + * The following locks must be held by the caller: + * * kbase_device::mmu_hw_mutex + * * kbase_device::hwaccess_lock + * + * Return: true if refcount succeeded, and the context will not be scheduled + * out, false if the refcount failed (because the context is being/has been + * scheduled out). + */ +bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx); + +/** + * kbase_ctx_sched_inc_refcount - Refcount a context as being busy, preventing + * it from being scheduled out. + * @kctx: Context to be refcounted + * + * The following locking conditions are made on the caller: + * * it must not hold kbase_device::mmu_hw_mutex and + * kbase_device::hwaccess_lock, because they will be used internally. + * + * Return: true if refcount succeeded, and the context will not be scheduled + * out, false if the refcount failed (because the context is being/has been + * scheduled out). + */ +bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx); + +/** + * kbase_ctx_sched_release_ctx_lock - Release a reference count of a context + * @kctx: Context for which refcount should be decreased + * + * Effectivelly, this is a wrapper for kbase_ctx_sched_release_ctx, but + * kbase_device::hwaccess_lock is required NOT to be locked. + */ +void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx); + +#endif /* _KBASE_CTX_SCHED_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug.c b/drivers/gpu/arm/bifrost/mali_kbase_debug.c new file mode 100755 index 000000000000..118f787fb74c --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug.c @@ -0,0 +1,44 @@ +/* + * + * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#include + +static struct kbasep_debug_assert_cb kbasep_debug_assert_registered_cb = { + NULL, + NULL +}; + +void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param) +{ + kbasep_debug_assert_registered_cb.func = func; + kbasep_debug_assert_registered_cb.param = param; +} + +void kbasep_debug_assert_call_hook(void) +{ + if (kbasep_debug_assert_registered_cb.func != NULL) + kbasep_debug_assert_registered_cb.func(kbasep_debug_assert_registered_cb.param); +} +KBASE_EXPORT_SYMBOL(kbasep_debug_assert_call_hook); + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug.h b/drivers/gpu/arm/bifrost/mali_kbase_debug.h new file mode 100755 index 000000000000..f33413908405 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug.h @@ -0,0 +1,169 @@ +/* + * + * (C) COPYRIGHT 2012-2015, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#ifndef _KBASE_DEBUG_H +#define _KBASE_DEBUG_H + +#include + +/** @brief If equals to 0, a trace containing the file, line, and function will be displayed before each message. */ +#define KBASE_DEBUG_SKIP_TRACE 0 + +/** @brief If different from 0, the trace will only contain the file and line. */ +#define KBASE_DEBUG_SKIP_FUNCTION_NAME 0 + +/** @brief Disable the asserts tests if set to 1. Default is to disable the asserts in release. */ +#ifndef KBASE_DEBUG_DISABLE_ASSERTS +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define KBASE_DEBUG_DISABLE_ASSERTS 0 +#else +#define KBASE_DEBUG_DISABLE_ASSERTS 1 +#endif +#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ + +/** Function type that is called on an KBASE_DEBUG_ASSERT() or KBASE_DEBUG_ASSERT_MSG() */ +typedef void (kbase_debug_assert_hook) (void *); + +struct kbasep_debug_assert_cb { + kbase_debug_assert_hook *func; + void *param; +}; + +/** + * @def KBASEP_DEBUG_PRINT_TRACE + * @brief Private macro containing the format of the trace to display before every message + * @sa KBASE_DEBUG_SKIP_TRACE, KBASE_DEBUG_SKIP_FUNCTION_NAME + */ +#if !KBASE_DEBUG_SKIP_TRACE +#define KBASEP_DEBUG_PRINT_TRACE \ + "In file: " __FILE__ " line: " CSTD_STR2(__LINE__) +#if !KBASE_DEBUG_SKIP_FUNCTION_NAME +#define KBASEP_DEBUG_PRINT_FUNCTION __func__ +#else +#define KBASEP_DEBUG_PRINT_FUNCTION "" +#endif +#else +#define KBASEP_DEBUG_PRINT_TRACE "" +#endif + +/** + * @def KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) + * @brief (Private) system printing function associated to the @ref KBASE_DEBUG_ASSERT_MSG event. + * @param trace location in the code from where the message is printed + * @param function function from where the message is printed + * @param ... Format string followed by format arguments. + * @note function parameter cannot be concatenated with other strings + */ +/* Select the correct system output function*/ +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...)\ + do { \ + pr_err("Mali: %s function:%s ", trace, function);\ + pr_err(__VA_ARGS__);\ + pr_err("\n");\ + } while (false) +#else +#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) CSTD_NOP() +#endif + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define KBASE_CALL_ASSERT_HOOK() kbasep_debug_assert_call_hook() +#else +#define KBASE_CALL_ASSERT_HOOK() CSTD_NOP() +#endif + +/** + * @def KBASE_DEBUG_ASSERT(expr) + * @brief Calls @ref KBASE_PRINT_ASSERT and prints the expression @a expr if @a expr is false + * + * @note This macro does nothing if the flag @ref KBASE_DEBUG_DISABLE_ASSERTS is set to 1 + * + * @param expr Boolean expression + */ +#define KBASE_DEBUG_ASSERT(expr) \ + KBASE_DEBUG_ASSERT_MSG(expr, #expr) + +#if KBASE_DEBUG_DISABLE_ASSERTS +#define KBASE_DEBUG_ASSERT_MSG(expr, ...) CSTD_NOP() +#else + /** + * @def KBASE_DEBUG_ASSERT_MSG(expr, ...) + * @brief Calls @ref KBASEP_DEBUG_ASSERT_OUT and prints the given message if @a expr is false + * + * @note This macro does nothing if the flag @ref KBASE_DEBUG_DISABLE_ASSERTS is set to 1 + * + * @param expr Boolean expression + * @param ... Message to display when @a expr is false, as a format string followed by format arguments. + */ +#define KBASE_DEBUG_ASSERT_MSG(expr, ...) \ + do { \ + if (!(expr)) { \ + KBASEP_DEBUG_ASSERT_OUT(KBASEP_DEBUG_PRINT_TRACE, KBASEP_DEBUG_PRINT_FUNCTION, __VA_ARGS__);\ + KBASE_CALL_ASSERT_HOOK();\ + BUG();\ + } \ + } while (false) +#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ + +/** + * @def KBASE_DEBUG_CODE( X ) + * @brief Executes the code inside the macro only in debug mode + * + * @param X Code to compile only in debug mode. + */ +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define KBASE_DEBUG_CODE(X) X +#else +#define KBASE_DEBUG_CODE(X) CSTD_NOP() +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + +/** @} */ + +/** + * @brief Register a function to call on ASSERT + * + * Such functions will \b only be called during Debug mode, and for debugging + * features \b only. Do not rely on them to be called in general use. + * + * To disable the hook, supply NULL to \a func. + * + * @note This function is not thread-safe, and should only be used to + * register/deregister once in the module's lifetime. + * + * @param[in] func the function to call when an assert is triggered. + * @param[in] param the parameter to pass to \a func when calling it + */ +void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param); + +/** + * @brief Call a debug assert hook previously registered with kbase_debug_assert_register_hook() + * + * @note This function is not thread-safe with respect to multiple threads + * registering functions and parameters with + * kbase_debug_assert_register_hook(). Otherwise, thread safety is the + * responsibility of the registered hook. + */ +void kbasep_debug_assert_call_hook(void); + +#endif /* _KBASE_DEBUG_H */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.c b/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.c new file mode 100755 index 000000000000..dbc774d56ab4 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.c @@ -0,0 +1,566 @@ +/* + * + * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include + +#ifdef CONFIG_DEBUG_FS + +static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev) +{ + struct list_head *event_list = &kbdev->job_fault_event_list; + unsigned long flags; + bool ret; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + ret = !list_empty(event_list); + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + return ret; +} + +static void kbase_ctx_remove_pending_event(struct kbase_context *kctx) +{ + struct list_head *event_list = &kctx->kbdev->job_fault_event_list; + struct base_job_fault_event *event; + unsigned long flags; + + spin_lock_irqsave(&kctx->kbdev->job_fault_event_lock, flags); + list_for_each_entry(event, event_list, head) { + if (event->katom->kctx == kctx) { + list_del(&event->head); + spin_unlock_irqrestore(&kctx->kbdev->job_fault_event_lock, flags); + + wake_up(&kctx->kbdev->job_fault_resume_wq); + flush_work(&event->job_fault_work); + + /* job_fault_event_list can only have a single atom for + * each context. + */ + return; + } + } + spin_unlock_irqrestore(&kctx->kbdev->job_fault_event_lock, flags); +} + +static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + struct list_head *event_list = &kctx->kbdev->job_fault_event_list; + struct base_job_fault_event *event; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + if (list_empty(event_list)) { + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + return true; + } + list_for_each_entry(event, event_list, head) { + if (event->katom->kctx == kctx) { + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, + flags); + return false; + } + } + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + return true; +} + +static int wait_for_job_fault(struct kbase_device *kbdev) +{ +#if KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE && \ + KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE + int ret = wait_event_interruptible_timeout(kbdev->job_fault_wq, + kbase_is_job_fault_event_pending(kbdev), + msecs_to_jiffies(2000)); + if (ret == 0) + return -EAGAIN; + else if (ret > 0) + return 0; + else + return ret; +#else + return wait_event_interruptible(kbdev->job_fault_wq, + kbase_is_job_fault_event_pending(kbdev)); +#endif +} + +/* wait until the fault happen and copy the event */ +static int kbase_job_fault_event_wait(struct kbase_device *kbdev, + struct base_job_fault_event *event) +{ + struct list_head *event_list = &kbdev->job_fault_event_list; + struct base_job_fault_event *event_in; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + while (list_empty(event_list)) { + int err; + + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + err = wait_for_job_fault(kbdev); + if (err) + return err; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + } + + event_in = list_entry(event_list->next, + struct base_job_fault_event, head); + event->event_code = event_in->event_code; + event->katom = event_in->katom; + + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + return 0; + +} + +/* remove the event from the queue */ +static struct base_job_fault_event *kbase_job_fault_event_dequeue( + struct kbase_device *kbdev, struct list_head *event_list) +{ + struct base_job_fault_event *event; + + event = list_entry(event_list->next, + struct base_job_fault_event, head); + list_del(event_list->next); + + return event; + +} + +/* Remove all the following atoms after the failed atom in the same context + * Call the postponed bottom half of job done. + * Then, this context could be rescheduled. + */ +static void kbase_job_fault_resume_event_cleanup(struct kbase_context *kctx) +{ + struct list_head *event_list = &kctx->job_fault_resume_event_list; + + while (!list_empty(event_list)) { + struct base_job_fault_event *event; + + event = kbase_job_fault_event_dequeue(kctx->kbdev, + &kctx->job_fault_resume_event_list); + kbase_jd_done_worker(&event->katom->work); + } + +} + +static void kbase_job_fault_resume_worker(struct work_struct *data) +{ + struct base_job_fault_event *event = container_of(data, + struct base_job_fault_event, job_fault_work); + struct kbase_context *kctx; + struct kbase_jd_atom *katom; + + katom = event->katom; + kctx = katom->kctx; + + dev_info(kctx->kbdev->dev, "Job dumping wait\n"); + + /* When it was waked up, it need to check if queue is empty or the + * failed atom belongs to different context. If yes, wake up. Both + * of them mean the failed job has been dumped. Please note, it + * should never happen that the job_fault_event_list has the two + * atoms belong to the same context. + */ + wait_event(kctx->kbdev->job_fault_resume_wq, + kbase_ctx_has_no_event_pending(kctx)); + + atomic_set(&kctx->job_fault_count, 0); + kbase_jd_done_worker(&katom->work); + + /* In case the following atoms were scheduled during failed job dump + * the job_done_worker was held. We need to rerun it after the dump + * was finished + */ + kbase_job_fault_resume_event_cleanup(kctx); + + dev_info(kctx->kbdev->dev, "Job dumping finish, resume scheduler\n"); +} + +static struct base_job_fault_event *kbase_job_fault_event_queue( + struct list_head *event_list, + struct kbase_jd_atom *atom, + u32 completion_code) +{ + struct base_job_fault_event *event; + + event = &atom->fault_event; + + event->katom = atom; + event->event_code = completion_code; + + list_add_tail(&event->head, event_list); + + return event; + +} + +static void kbase_job_fault_event_post(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, u32 completion_code) +{ + struct base_job_fault_event *event; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list, + katom, completion_code); + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + wake_up_interruptible(&kbdev->job_fault_wq); + + INIT_WORK(&event->job_fault_work, kbase_job_fault_resume_worker); + queue_work(kbdev->job_fault_resume_workq, &event->job_fault_work); + + dev_info(katom->kctx->kbdev->dev, "Job fault happen, start dump: %d_%d", + katom->kctx->tgid, katom->kctx->id); + +} + +/* + * This function will process the job fault + * Get the register copy + * Send the failed job dump event + * Create a Wait queue to wait until the job dump finish + */ + +bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, + u32 completion_code) +{ + struct kbase_context *kctx = katom->kctx; + + /* Check if dumping is in the process + * only one atom of each context can be dumped at the same time + * If the atom belongs to different context, it can be dumped + */ + if (atomic_read(&kctx->job_fault_count) > 0) { + kbase_job_fault_event_queue( + &kctx->job_fault_resume_event_list, + katom, completion_code); + dev_info(kctx->kbdev->dev, "queue:%d\n", + kbase_jd_atom_id(kctx, katom)); + return true; + } + + if (kbase_ctx_flag(kctx, KCTX_DYING)) + return false; + + if (atomic_read(&kctx->kbdev->job_fault_debug) > 0) { + + if (completion_code != BASE_JD_EVENT_DONE) { + + if (kbase_job_fault_get_reg_snapshot(kctx) == false) { + dev_warn(kctx->kbdev->dev, "get reg dump failed\n"); + return false; + } + + kbase_job_fault_event_post(kctx->kbdev, katom, + completion_code); + atomic_inc(&kctx->job_fault_count); + dev_info(kctx->kbdev->dev, "post:%d\n", + kbase_jd_atom_id(kctx, katom)); + return true; + + } + } + return false; + +} + +static int debug_job_fault_show(struct seq_file *m, void *v) +{ + struct kbase_device *kbdev = m->private; + struct base_job_fault_event *event = (struct base_job_fault_event *)v; + struct kbase_context *kctx = event->katom->kctx; + int i; + + dev_info(kbdev->dev, "debug job fault seq show:%d_%d, %d", + kctx->tgid, kctx->id, event->reg_offset); + + if (kctx->reg_dump == NULL) { + dev_warn(kbdev->dev, "reg dump is NULL"); + return -1; + } + + if (kctx->reg_dump[event->reg_offset] == + REGISTER_DUMP_TERMINATION_FLAG) { + /* Return the error here to stop the read. And the + * following next() will not be called. The stop can + * get the real event resource and release it + */ + return -1; + } + + if (event->reg_offset == 0) + seq_printf(m, "%d_%d\n", kctx->tgid, kctx->id); + + for (i = 0; i < 50; i++) { + if (kctx->reg_dump[event->reg_offset] == + REGISTER_DUMP_TERMINATION_FLAG) { + break; + } + seq_printf(m, "%08x: %08x\n", + kctx->reg_dump[event->reg_offset], + kctx->reg_dump[1+event->reg_offset]); + event->reg_offset += 2; + + } + + + return 0; +} +static void *debug_job_fault_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct kbase_device *kbdev = m->private; + struct base_job_fault_event *event = (struct base_job_fault_event *)v; + + dev_info(kbdev->dev, "debug job fault seq next:%d, %d", + event->reg_offset, (int)*pos); + + return event; +} + +static void *debug_job_fault_start(struct seq_file *m, loff_t *pos) +{ + struct kbase_device *kbdev = m->private; + struct base_job_fault_event *event; + + dev_info(kbdev->dev, "fault job seq start:%d", (int)*pos); + + /* The condition is trick here. It needs make sure the + * fault hasn't happened and the dumping hasn't been started, + * or the dumping has finished + */ + if (*pos == 0) { + event = kmalloc(sizeof(*event), GFP_KERNEL); + if (!event) + return NULL; + event->reg_offset = 0; + if (kbase_job_fault_event_wait(kbdev, event)) { + kfree(event); + return NULL; + } + + /* The cache flush workaround is called in bottom half of + * job done but we delayed it. Now we should clean cache + * earlier. Then the GPU memory dump should be correct. + */ + kbase_backend_cache_clean(kbdev, event->katom); + } else + return NULL; + + return event; +} + +static void debug_job_fault_stop(struct seq_file *m, void *v) +{ + struct kbase_device *kbdev = m->private; + + /* here we wake up the kbase_jd_done_worker after stop, it needs + * get the memory dump before the register dump in debug daemon, + * otherwise, the memory dump may be incorrect. + */ + + if (v != NULL) { + kfree(v); + dev_info(kbdev->dev, "debug job fault seq stop stage 1"); + + } else { + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + if (!list_empty(&kbdev->job_fault_event_list)) { + kbase_job_fault_event_dequeue(kbdev, + &kbdev->job_fault_event_list); + wake_up(&kbdev->job_fault_resume_wq); + } + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + dev_info(kbdev->dev, "debug job fault seq stop stage 2"); + } + +} + +static const struct seq_operations ops = { + .start = debug_job_fault_start, + .next = debug_job_fault_next, + .stop = debug_job_fault_stop, + .show = debug_job_fault_show, +}; + +static int debug_job_fault_open(struct inode *in, struct file *file) +{ + struct kbase_device *kbdev = in->i_private; + + if (atomic_cmpxchg(&kbdev->job_fault_debug, 0, 1) == 1) { + dev_warn(kbdev->dev, "debug job fault is busy, only a single client is allowed"); + return -EBUSY; + } + + seq_open(file, &ops); + + ((struct seq_file *)file->private_data)->private = kbdev; + dev_info(kbdev->dev, "debug job fault seq open"); + + + return 0; + +} + +static int debug_job_fault_release(struct inode *in, struct file *file) +{ + struct kbase_device *kbdev = in->i_private; + struct list_head *event_list = &kbdev->job_fault_event_list; + unsigned long flags; + + seq_release(in, file); + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + + /* Disable job fault dumping. This will let kbase run jobs as normal, + * without blocking waiting for a job_fault client to read failed jobs. + * + * After this a new client may open the file, and may re-enable job + * fault dumping, but the job_fault_event_lock we hold here will block + * that from interfering until after we've completed the cleanup. + */ + atomic_dec(&kbdev->job_fault_debug); + + /* Clean the unprocessed job fault. After that, all the suspended + * contexts could be rescheduled. Remove all the failed atoms that + * belong to different contexts Resume all the contexts that were + * suspend due to failed job. + */ + while (!list_empty(event_list)) { + kbase_job_fault_event_dequeue(kbdev, event_list); + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + wake_up(&kbdev->job_fault_resume_wq); + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + } + + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + dev_info(kbdev->dev, "debug job fault seq close"); + + return 0; +} + +static const struct file_operations kbasep_debug_job_fault_fops = { + .owner = THIS_MODULE, + .open = debug_job_fault_open, + .read = seq_read, + .llseek = seq_lseek, + .release = debug_job_fault_release, +}; + +/* + * Initialize debugfs entry for job fault dump + */ +void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("job_fault", 0400, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_debug_job_fault_fops); +} + + +int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) +{ + + INIT_LIST_HEAD(&kbdev->job_fault_event_list); + + init_waitqueue_head(&(kbdev->job_fault_wq)); + init_waitqueue_head(&(kbdev->job_fault_resume_wq)); + spin_lock_init(&kbdev->job_fault_event_lock); + + kbdev->job_fault_resume_workq = alloc_workqueue( + "kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1); + if (!kbdev->job_fault_resume_workq) + return -ENOMEM; + + atomic_set(&kbdev->job_fault_debug, 0); + + return 0; +} + +/* + * Release the relevant resource per device + */ +void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) +{ + destroy_workqueue(kbdev->job_fault_resume_workq); +} + + +/* + * Initialize the relevant data structure per context + */ +void kbase_debug_job_fault_context_init(struct kbase_context *kctx) +{ + + /* We need allocate double size register range + * Because this memory will keep the register address and value + */ + kctx->reg_dump = vmalloc(0x4000 * 2); + if (kctx->reg_dump == NULL) + return; + + if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { + vfree(kctx->reg_dump); + kctx->reg_dump = NULL; + } + INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); + atomic_set(&kctx->job_fault_count, 0); + +} + +/* + * release the relevant resource per context + */ +void kbase_debug_job_fault_context_term(struct kbase_context *kctx) +{ + vfree(kctx->reg_dump); +} + +void kbase_debug_job_fault_kctx_unblock(struct kbase_context *kctx) +{ + WARN_ON(!kbase_ctx_flag(kctx, KCTX_DYING)); + + kbase_ctx_remove_pending_event(kctx); +} + +#else /* CONFIG_DEBUG_FS */ + +int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) +{ + return 0; +} + +void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) +{ +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h b/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h new file mode 100755 index 000000000000..ef69627cdce8 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h @@ -0,0 +1,116 @@ +/* + * + * (C) COPYRIGHT 2012-2016, 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_JOB_FAULT_H +#define _KBASE_DEBUG_JOB_FAULT_H + +#include +#include + +#define REGISTER_DUMP_TERMINATION_FLAG 0xFFFFFFFF + +/** + * kbase_debug_job_fault_dev_init - Create the fault event wait queue + * per device and initialize the required lists. + * @kbdev: Device pointer + * + * Return: Zero on success or a negative error code. + */ +int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev); + +/** + * kbase_debug_job_fault_debugfs_init - Initialize job fault debug sysfs + * @kbdev: Device pointer + */ +void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev); + +/** + * kbase_debug_job_fault_dev_term - Clean up resources created in + * kbase_debug_job_fault_dev_init. + * @kbdev: Device pointer + */ +void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); + +/** + * kbase_debug_job_fault_context_init - Initialize the relevant + * data structure per context + * @kctx: KBase context pointer + */ +void kbase_debug_job_fault_context_init(struct kbase_context *kctx); + +/** + * kbase_debug_job_fault_context_term - Release the relevant + * resource per context + * @kctx: KBase context pointer + */ +void kbase_debug_job_fault_context_term(struct kbase_context *kctx); + +/** + * kbase_debug_job_fault_kctx_unblock - Unblock the atoms blocked on job fault + * dumping on context termination. + * + * This function is called during context termination to unblock the atom for + * which the job fault occurred and also the atoms following it. This is needed + * otherwise the wait for zero jobs could timeout (leading to an assertion + * failure, kernel panic in debug builds) in the pathological case where + * although the thread/daemon capturing the job fault events is running, + * but for some reasons has stopped consuming the events. + * + * @kctx: KBase context pointer + */ +void kbase_debug_job_fault_kctx_unblock(struct kbase_context *kctx); + +/** + * kbase_debug_job_fault_process - Process the failed job. + * It will send a event and wake up the job fault waiting queue + * Then create a work queue to wait for job dump finish + * This function should be called in the interrupt handler and before + * jd_done that make sure the jd_done_worker will be delayed until the + * job dump finish + * @katom: The failed atom pointer + * @completion_code: the job status + * @return true if dump is going on + */ +bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, + u32 completion_code); + + +/** + * kbase_debug_job_fault_reg_snapshot_init - Set the interested registers + * address during the job fault process, the relevant registers will + * be saved when a job fault happen + * @kctx: KBase context pointer + * @reg_range: Maximum register address space + * @return true if initializing successfully + */ +bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, + int reg_range); + +/** + * kbase_job_fault_get_reg_snapshot - Read the interested registers for + * failed job dump + * @kctx: KBase context pointer + * @return true if getting registers successfully + */ +bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx); + +#endif /*_KBASE_DEBUG_JOB_FAULT_H*/ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c new file mode 100755 index 000000000000..478813705a41 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c @@ -0,0 +1,313 @@ +/* + * + * (C) COPYRIGHT 2013-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Debugfs interface to dump the memory visible to the GPU + */ + +#include "mali_kbase_debug_mem_view.h" +#include "mali_kbase.h" + +#include +#include + +#ifdef CONFIG_DEBUG_FS + +#if (KERNEL_VERSION(4, 1, 0) > LINUX_VERSION_CODE) +#define get_file_rcu(x) atomic_long_inc_not_zero(&(x)->f_count) +#endif + +struct debug_mem_mapping { + struct list_head node; + + struct kbase_mem_phy_alloc *alloc; + unsigned long flags; + + u64 start_pfn; + size_t nr_pages; +}; + +struct debug_mem_data { + struct list_head mapping_list; + struct kbase_context *kctx; +}; + +struct debug_mem_seq_off { + struct list_head *lh; + size_t offset; +}; + +static void *debug_mem_start(struct seq_file *m, loff_t *_pos) +{ + struct debug_mem_data *mem_data = m->private; + struct debug_mem_seq_off *data; + struct debug_mem_mapping *map; + loff_t pos = *_pos; + + list_for_each_entry(map, &mem_data->mapping_list, node) { + if (pos >= map->nr_pages) { + pos -= map->nr_pages; + } else { + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + data->lh = &map->node; + data->offset = pos; + return data; + } + } + + /* Beyond the end */ + return NULL; +} + +static void debug_mem_stop(struct seq_file *m, void *v) +{ + kfree(v); +} + +static void *debug_mem_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct debug_mem_data *mem_data = m->private; + struct debug_mem_seq_off *data = v; + struct debug_mem_mapping *map; + + map = list_entry(data->lh, struct debug_mem_mapping, node); + + if (data->offset < map->nr_pages - 1) { + data->offset++; + ++*pos; + return data; + } + + if (list_is_last(data->lh, &mem_data->mapping_list)) { + kfree(data); + return NULL; + } + + data->lh = data->lh->next; + data->offset = 0; + ++*pos; + + return data; +} + +static int debug_mem_show(struct seq_file *m, void *v) +{ + struct debug_mem_data *mem_data = m->private; + struct debug_mem_seq_off *data = v; + struct debug_mem_mapping *map; + int i, j; + struct page *page; + uint32_t *mapping; + pgprot_t prot = PAGE_KERNEL; + + map = list_entry(data->lh, struct debug_mem_mapping, node); + + kbase_gpu_vm_lock(mem_data->kctx); + + if (data->offset >= map->alloc->nents) { + seq_printf(m, "%016llx: Unbacked page\n\n", (map->start_pfn + + data->offset) << PAGE_SHIFT); + goto out; + } + + if (!(map->flags & KBASE_REG_CPU_CACHED)) + prot = pgprot_writecombine(prot); + + page = as_page(map->alloc->pages[data->offset]); + mapping = vmap(&page, 1, VM_MAP, prot); + if (!mapping) + goto out; + + for (i = 0; i < PAGE_SIZE; i += 4*sizeof(*mapping)) { + seq_printf(m, "%016llx:", i + ((map->start_pfn + + data->offset) << PAGE_SHIFT)); + + for (j = 0; j < 4*sizeof(*mapping); j += sizeof(*mapping)) + seq_printf(m, " %08x", mapping[(i+j)/sizeof(*mapping)]); + seq_putc(m, '\n'); + } + + vunmap(mapping); + + seq_putc(m, '\n'); + +out: + kbase_gpu_vm_unlock(mem_data->kctx); + return 0; +} + +static const struct seq_operations ops = { + .start = debug_mem_start, + .next = debug_mem_next, + .stop = debug_mem_stop, + .show = debug_mem_show, +}; + +static int debug_mem_zone_open(struct rb_root *rbtree, + struct debug_mem_data *mem_data) +{ + int ret = 0; + struct rb_node *p; + struct kbase_va_region *reg; + struct debug_mem_mapping *mapping; + + for (p = rb_first(rbtree); p; p = rb_next(p)) { + reg = rb_entry(p, struct kbase_va_region, rblink); + + if (reg->gpu_alloc == NULL) + /* Empty region - ignore */ + continue; + + mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) { + ret = -ENOMEM; + goto out; + } + + mapping->alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + mapping->start_pfn = reg->start_pfn; + mapping->nr_pages = reg->nr_pages; + mapping->flags = reg->flags; + list_add_tail(&mapping->node, &mem_data->mapping_list); + } + +out: + return ret; +} + +static int debug_mem_open(struct inode *i, struct file *file) +{ + struct kbase_context *const kctx = i->i_private; + struct debug_mem_data *mem_data; + int ret; + + if (get_file_rcu(kctx->filp) == 0) + return -ENOENT; + + ret = seq_open(file, &ops); + if (ret) + goto open_fail; + + mem_data = kmalloc(sizeof(*mem_data), GFP_KERNEL); + if (!mem_data) { + ret = -ENOMEM; + goto out; + } + + mem_data->kctx = kctx; + + INIT_LIST_HEAD(&mem_data->mapping_list); + + kbase_gpu_vm_lock(kctx); + + ret = debug_mem_zone_open(&kctx->reg_rbtree_same, mem_data); + if (0 != ret) { + kbase_gpu_vm_unlock(kctx); + goto out; + } + + ret = debug_mem_zone_open(&kctx->reg_rbtree_custom, mem_data); + if (0 != ret) { + kbase_gpu_vm_unlock(kctx); + goto out; + } + + ret = debug_mem_zone_open(&kctx->reg_rbtree_exec, mem_data); + if (0 != ret) { + kbase_gpu_vm_unlock(kctx); + goto out; + } + + kbase_gpu_vm_unlock(kctx); + + ((struct seq_file *)file->private_data)->private = mem_data; + + return 0; + +out: + if (mem_data) { + while (!list_empty(&mem_data->mapping_list)) { + struct debug_mem_mapping *mapping; + + mapping = list_first_entry(&mem_data->mapping_list, + struct debug_mem_mapping, node); + kbase_mem_phy_alloc_put(mapping->alloc); + list_del(&mapping->node); + kfree(mapping); + } + kfree(mem_data); + } + seq_release(i, file); +open_fail: + fput(kctx->filp); + + return ret; +} + +static int debug_mem_release(struct inode *inode, struct file *file) +{ + struct kbase_context *const kctx = inode->i_private; + struct seq_file *sfile = file->private_data; + struct debug_mem_data *mem_data = sfile->private; + struct debug_mem_mapping *mapping; + + seq_release(inode, file); + + while (!list_empty(&mem_data->mapping_list)) { + mapping = list_first_entry(&mem_data->mapping_list, + struct debug_mem_mapping, node); + kbase_mem_phy_alloc_put(mapping->alloc); + list_del(&mapping->node); + kfree(mapping); + } + + kfree(mem_data); + + fput(kctx->filp); + + return 0; +} + +static const struct file_operations kbase_debug_mem_view_fops = { + .owner = THIS_MODULE, + .open = debug_mem_open, + .release = debug_mem_release, + .read = seq_read, + .llseek = seq_lseek +}; + +void kbase_debug_mem_view_init(struct kbase_context *const kctx) +{ + /* Caller already ensures this, but we keep the pattern for + * maintenance safety. + */ + if (WARN_ON(!kctx) || + WARN_ON(IS_ERR_OR_NULL(kctx->kctx_dentry))) + return; + + debugfs_create_file("mem_view", 0400, kctx->kctx_dentry, kctx, + &kbase_debug_mem_view_fops); +} + +#endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.h b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.h new file mode 100755 index 000000000000..b948b7cd9dd4 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.h @@ -0,0 +1,40 @@ +/* + * + * (C) COPYRIGHT 2013-2015, 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUG_MEM_VIEW_H +#define _KBASE_DEBUG_MEM_VIEW_H + +#include + +/** + * kbase_debug_mem_view_init - Initialize the mem_view sysfs file + * @kctx: Pointer to kernel base context + * + * This function creates a "mem_view" file which can be used to get a view of + * the context's memory as the GPU sees it (i.e. using the GPU's page tables). + * + * The file is cleaned up by a call to debugfs_remove_recursive() deleting the + * parent directory. + */ +void kbase_debug_mem_view_init(struct kbase_context *kctx); + +#endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c b/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c new file mode 100755 index 000000000000..37e507b164c5 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c @@ -0,0 +1,183 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include + +#include "mali_kbase_debugfs_helper.h" + +/* Arbitrary maximum size to prevent user space allocating too much kernel + * memory + */ +#define DEBUGFS_MEM_POOLS_MAX_WRITE_SIZE (256u) + +/** + * set_attr_from_string - Parse a string to set elements of an array + * + * This is the core of the implementation of + * kbase_debugfs_helper_set_attr_from_string. The only difference between the + * two functions is that this one requires the input string to be writable. + * + * @buf: Input string to parse. Must be nul-terminated! + * @array: Address of an object that can be accessed like an array. + * @nelems: Number of elements in the array. + * @set_attr_fn: Function to be called back for each array element. + * + * Return: 0 if success, negative error code otherwise. + */ +static int set_attr_from_string( + char *const buf, + void *const array, size_t const nelems, + kbase_debugfs_helper_set_attr_fn const set_attr_fn) +{ + size_t index, err = 0; + char *ptr = buf; + + for (index = 0; index < nelems && *ptr; ++index) { + unsigned long new_size; + size_t len; + char sep; + + /* Drop leading spaces */ + while (*ptr == ' ') + ptr++; + + len = strcspn(ptr, "\n "); + if (len == 0) { + /* No more values (allow this) */ + break; + } + + /* Substitute a nul terminator for a space character + * to make the substring valid for kstrtoul. + */ + sep = ptr[len]; + if (sep == ' ') + ptr[len++] = '\0'; + + err = kstrtoul(ptr, 0, &new_size); + if (err) + break; + + /* Skip the substring (including any premature nul terminator) + */ + ptr += len; + + set_attr_fn(array, index, new_size); + } + + return err; +} + +int kbase_debugfs_helper_set_attr_from_string( + const char *const buf, void *const array, size_t const nelems, + kbase_debugfs_helper_set_attr_fn const set_attr_fn) +{ + char *const wbuf = kstrdup(buf, GFP_KERNEL); + int err = 0; + + if (!wbuf) + return -ENOMEM; + + err = set_attr_from_string(wbuf, array, nelems, + set_attr_fn); + + kfree(wbuf); + return err; +} + +ssize_t kbase_debugfs_helper_get_attr_to_string( + char *const buf, size_t const size, + void *const array, size_t const nelems, + kbase_debugfs_helper_get_attr_fn const get_attr_fn) +{ + ssize_t total = 0; + size_t index; + + for (index = 0; index < nelems; ++index) { + const char *postfix = " "; + + if (index == (nelems-1)) + postfix = "\n"; + + total += scnprintf(buf + total, size - total, "%zu%s", + get_attr_fn(array, index), postfix); + } + + return total; +} + +int kbase_debugfs_helper_seq_write(struct file *const file, + const char __user *const ubuf, size_t const count, + size_t const nelems, + kbase_debugfs_helper_set_attr_fn const set_attr_fn) +{ + const struct seq_file *const sfile = file->private_data; + void *const array = sfile->private; + int err = 0; + char *buf; + + if (WARN_ON(!array)) + return -EINVAL; + + if (WARN_ON(count > DEBUGFS_MEM_POOLS_MAX_WRITE_SIZE)) + return -EINVAL; + + buf = kmalloc(count + 1, GFP_KERNEL); + if (buf == NULL) + return -ENOMEM; + + if (copy_from_user(buf, ubuf, count)) { + kfree(buf); + return -EFAULT; + } + + buf[count] = '\0'; + err = set_attr_from_string(buf, + array, nelems, set_attr_fn); + kfree(buf); + + return err; +} + +int kbase_debugfs_helper_seq_read(struct seq_file *const sfile, + size_t const nelems, + kbase_debugfs_helper_get_attr_fn const get_attr_fn) +{ + void *const array = sfile->private; + size_t index; + + if (WARN_ON(!array)) + return -EINVAL; + + for (index = 0; index < nelems; ++index) { + const char *postfix = " "; + + if (index == (nelems-1)) + postfix = "\n"; + + seq_printf(sfile, "%zu%s", get_attr_fn(array, index), postfix); + } + return 0; +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.h b/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.h new file mode 100755 index 000000000000..c3c9efa14e65 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.h @@ -0,0 +1,141 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DEBUGFS_HELPER_H_ +#define _KBASE_DEBUGFS_HELPER_H_ + +/** + * typedef kbase_debugfs_helper_set_attr_fn - Type of function to set an + * attribute value from an array + * + * @array: Address of an object that can be accessed like an array. + * @index: An element index. The valid range depends on the use-case. + * @value: Attribute value to be set. + */ +typedef void (*kbase_debugfs_helper_set_attr_fn)( + void *array, size_t index, size_t value); + +/** + * kbase_debugfs_helper_set_attr_from_string - Parse a string to reconfigure an + * array + * + * The given function is called once for each attribute value found in the + * input string. It is not an error if the string specifies fewer attribute + * values than the specified number of array elements. + * + * The number base of each attribute value is detected automatically + * according to the standard rules (e.g. prefix "0x" for hexadecimal). + * Attribute values are separated by one or more space characters. + * Additional leading and trailing spaces are ignored. + * + * @buf: Input string to parse. Must be nul-terminated! + * @array: Address of an object that can be accessed like an array. + * @nelems: Number of elements in the array. + * @set_attr_fn: Function to be called back for each array element. + * + * Return: 0 if success, negative error code otherwise. + */ +int kbase_debugfs_helper_set_attr_from_string( + const char *buf, void *array, size_t nelems, + kbase_debugfs_helper_set_attr_fn set_attr_fn); + +/** + * typedef kbase_debugfs_helper_get_attr_fn - Type of function to get an + * attribute value from an array + * + * @array: Address of an object that can be accessed like an array. + * @index: An element index. The valid range depends on the use-case. + * + * Return: Value of attribute. + */ +typedef size_t (*kbase_debugfs_helper_get_attr_fn)( + void *array, size_t index); + +/** + * kbase_debugfs_helper_get_attr_to_string - Construct a formatted string + * from elements in an array + * + * The given function is called once for each array element to get the + * value of the attribute to be inspected. The attribute values are + * written to the buffer as a formatted string of decimal numbers + * separated by spaces and terminated by a linefeed. + * + * @buf: Buffer in which to store the formatted output string. + * @size: The size of the buffer, in bytes. + * @array: Address of an object that can be accessed like an array. + * @nelems: Number of elements in the array. + * @get_attr_fn: Function to be called back for each array element. + * + * Return: Number of characters written excluding the nul terminator. + */ +ssize_t kbase_debugfs_helper_get_attr_to_string( + char *buf, size_t size, void *array, size_t nelems, + kbase_debugfs_helper_get_attr_fn get_attr_fn); + +/** + * kbase_debugfs_helper_seq_read - Implements reads from a virtual file for an + * array + * + * The virtual file must have been opened by calling single_open and passing + * the address of an object that can be accessed like an array. + * + * The given function is called once for each array element to get the + * value of the attribute to be inspected. The attribute values are + * written to the buffer as a formatted string of decimal numbers + * separated by spaces and terminated by a linefeed. + * + * @sfile: A virtual file previously opened by calling single_open. + * @nelems: Number of elements in the array. + * @get_attr_fn: Function to be called back for each array element. + * + * Return: 0 if success, negative error code otherwise. + */ +int kbase_debugfs_helper_seq_read( + struct seq_file *const sfile, size_t const nelems, + kbase_debugfs_helper_get_attr_fn const get_attr_fn); + +/** + * kbase_debugfs_helper_seq_write - Implements writes to a virtual file for an + * array + * + * The virtual file must have been opened by calling single_open and passing + * the address of an object that can be accessed like an array. + * + * The given function is called once for each attribute value found in the + * data written to the virtual file. For further details, refer to the + * description of set_attr_from_string. + * + * @file: A virtual file previously opened by calling single_open. + * @ubuf: Source address in user space. + * @count: Number of bytes written to the virtual file. + * @nelems: Number of elements in the array. + * @set_attr_fn: Function to be called back for each array element. + * + * Return: 0 if success, negative error code otherwise. + */ +int kbase_debugfs_helper_seq_write(struct file *const file, + const char __user *const ubuf, size_t const count, + size_t const nelems, + kbase_debugfs_helper_set_attr_fn const set_attr_fn); + +#endif /*_KBASE_DEBUGFS_HELPER_H_ */ + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_defs.h new file mode 100755 index 000000000000..980cf09500ef --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_defs.h @@ -0,0 +1,1807 @@ +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_defs.h + * + * Defintions (types, defines, etcs) common to Kbase. They are placed here to + * allow the hierarchy of header files to work. + */ + +#ifndef _KBASE_DEFS_H_ +#define _KBASE_DEFS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifdef CONFIG_MALI_BUSLOG +#include +#endif + +#if defined(CONFIG_SYNC) +#include +#else +#include "mali_kbase_fence_defs.h" +#endif + +#ifdef CONFIG_DEBUG_FS +#include +#endif /* CONFIG_DEBUG_FS */ + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ +#include +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + +#include +#include +#include + +#if defined(CONFIG_PM_RUNTIME) || \ + (defined(CONFIG_PM) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) +#define KBASE_PM_RUNTIME 1 +#endif + +#include "debug/mali_kbase_debug_ktrace_defs.h" + +/** Number of milliseconds before we time out on a GPU soft/hard reset */ +#define RESET_TIMEOUT 500 + +/** + * The maximum number of Job Slots to support in the Hardware. + * + * You can optimize this down if your target devices will only ever support a + * small number of job slots. + */ +#define BASE_JM_MAX_NR_SLOTS 3 + +/** + * The maximum number of Address Spaces to support in the Hardware. + * + * You can optimize this down if your target devices will only ever support a + * small number of Address Spaces + */ +#define BASE_MAX_NR_AS 16 + +/* mmu */ +#define MIDGARD_MMU_LEVEL(x) (x) + +#define MIDGARD_MMU_TOPLEVEL MIDGARD_MMU_LEVEL(0) + +#define MIDGARD_MMU_BOTTOMLEVEL MIDGARD_MMU_LEVEL(3) + +#define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW | KBASE_REG_GPU_WR) + +/** setting in kbase_context::as_nr that indicates it's invalid */ +#define KBASEP_AS_NR_INVALID (-1) + +/** + * Maximum size in bytes of a MMU lock region, as a logarithm + */ +#define KBASE_LOCK_REGION_MAX_SIZE_LOG2 (64) + +/** + * Minimum size in bytes of a MMU lock region, as a logarithm + */ +#define KBASE_LOCK_REGION_MIN_SIZE_LOG2 (15) + +#include "mali_kbase_hwaccess_defs.h" + +/* Maximum number of pages of memory that require a permanent mapping, per + * kbase_context + */ +#define KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES ((32 * 1024ul * 1024ul) >> \ + PAGE_SHIFT) +/* Minimum threshold period for hwcnt dumps between different hwcnt virtualizer + * clients, to reduce undesired system load. + * If a virtualizer client requests a dump within this threshold period after + * some other client has performed a dump, a new dump won't be performed and + * the accumulated counter values for that client will be returned instead. + */ +#define KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS (200 * NSEC_PER_USEC) + +/* Maximum number of clock/regulator pairs that may be referenced by + * the device node. + * This is dependent on support for of_property_read_u64_array() in the + * kernel. + */ +#if (KERNEL_VERSION(4, 0, 0) <= LINUX_VERSION_CODE) || \ + defined(LSK_OPPV2_BACKPORT) +#define BASE_MAX_NR_CLOCKS_REGULATORS (2) +#else +#define BASE_MAX_NR_CLOCKS_REGULATORS (1) +#endif + +/* Forward declarations */ +struct kbase_context; +struct kbase_device; +struct kbase_as; +struct kbase_mmu_setup; +struct kbase_ipa_model_vinstr_data; +struct kbase_kinstr_jm; + +/** + * struct kbase_io_access - holds information about 1 register access + * + * @addr: first bit indicates r/w (r=0, w=1) + * @value: value written or read + */ +struct kbase_io_access { + uintptr_t addr; + u32 value; +}; + +/** + * struct kbase_io_history - keeps track of all recent register accesses + * + * @enabled: true if register accesses are recorded, false otherwise + * @lock: spinlock protecting kbase_io_access array + * @count: number of registers read/written + * @size: number of elements in kbase_io_access array + * @buf: array of kbase_io_access + */ +struct kbase_io_history { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) + bool enabled; +#else + u32 enabled; +#endif + + spinlock_t lock; + size_t count; + u16 size; + struct kbase_io_access *buf; +}; + +/** + * struct kbase_debug_copy_buffer - information about the buffer to be copied. + * + * @size: size of the buffer in bytes + * @pages: pointer to an array of pointers to the pages which contain + * the buffer + * @is_vmalloc: true if @pages was allocated with vzalloc. false if @pages was + * allocated with kcalloc + * @nr_pages: number of pages + * @offset: offset into the pages + * @gpu_alloc: pointer to physical memory allocated by the GPU + * @extres_pages: array of pointers to the pages containing external resources + * for this buffer + * @nr_extres_pages: number of pages in @extres_pages + */ +struct kbase_debug_copy_buffer { + size_t size; + struct page **pages; + bool is_vmalloc; + int nr_pages; + size_t offset; + struct kbase_mem_phy_alloc *gpu_alloc; + + struct page **extres_pages; + int nr_extres_pages; +}; + +struct kbase_device_info { + u32 features; +}; + +struct kbase_mmu_setup { + u64 transtab; + u64 memattr; + u64 transcfg; +}; + +/** + * struct kbase_fault - object containing data relating to a page or bus fault. + * @addr: Records the faulting address. + * @extra_addr: Records the secondary fault address. + * @status: Records the fault status as reported by Hw. + * @protected_mode: Flag indicating whether the fault occurred in protected mode + * or not. + */ +struct kbase_fault { + u64 addr; + u64 extra_addr; + u32 status; + bool protected_mode; +}; + +/** + * struct kbase_mmu_table - object representing a set of GPU page tables + * @mmu_teardown_pages: Buffer of 4 Pages in size, used to cache the entries + * of top & intermediate level page tables to avoid + * repeated calls to kmap_atomic during the MMU teardown. + * @mmu_lock: Lock to serialize the accesses made to multi level GPU + * page tables + * @pgd: Physical address of the page allocated for the top + * level page table of the context, this is used for + * MMU HW programming as the address translation will + * start from the top level page table. + * @group_id: A memory group ID to be passed to a platform-specific + * memory group manager. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @kctx: If this set of MMU tables belongs to a context then + * this is a back-reference to the context, otherwise + * it is NULL + */ +struct kbase_mmu_table { + u64 *mmu_teardown_pages; + struct mutex mmu_lock; + phys_addr_t pgd; + u8 group_id; + struct kbase_context *kctx; +}; + +#if MALI_USE_CSF +#include "csf/mali_kbase_csf_defs.h" +#else +#include "jm/mali_kbase_jm_defs.h" +#endif + +static inline int kbase_as_has_bus_fault(struct kbase_as *as, + struct kbase_fault *fault) +{ + return (fault == &as->bf_data); +} + +static inline int kbase_as_has_page_fault(struct kbase_as *as, + struct kbase_fault *fault) +{ + return (fault == &as->pf_data); +} + +/** + * struct kbasep_mem_device - Data stored per device for memory allocation + * + * @used_pages: Tracks usage of OS shared memory. Updated when OS memory is + * allocated/freed. + * @ir_threshold: Fraction of the maximum size of an allocation that grows + * on GPU page fault that can be used before the driver + * switches to incremental rendering, in 1/256ths. + * 0 means disabled. + */ +struct kbasep_mem_device { + atomic_t used_pages; + atomic_t ir_threshold; +}; + +struct kbase_clk_rate_listener; + +/** + * kbase_clk_rate_listener_on_change_t() - Frequency change callback + * + * @listener: Clock frequency change listener. + * @clk_index: Index of the clock for which the change has occurred. + * @clk_rate_hz: Clock frequency(Hz). + * + * A callback to call when clock rate changes. The function must not + * sleep. No clock rate manager functions must be called from here, as + * its lock is taken. + */ +typedef void (*kbase_clk_rate_listener_on_change_t)( + struct kbase_clk_rate_listener *listener, + u32 clk_index, + u32 clk_rate_hz); + +/** + * struct kbase_clk_rate_listener - Clock frequency listener + * + * @node: List node. + * @notify: Callback to be called when GPU frequency changes. + */ +struct kbase_clk_rate_listener { + struct list_head node; + kbase_clk_rate_listener_on_change_t notify; +}; + +/** + * struct kbase_clk_rate_trace_manager - Data stored per device for GPU clock + * rate trace manager. + * + * @gpu_idle: Tracks the idle state of GPU. + * @clks: Array of pointer to structures storing data for every + * enumerated GPU clock. + * @clk_rate_trace_ops: Pointer to the platform specific GPU clock rate trace + * operations. + * @gpu_clk_rate_trace_write: Pointer to the function that would emit the + * tracepoint for the clock rate change. + * @listeners: List of listener attached. + * @lock: Lock to serialize the actions of GPU clock rate trace + * manager. + */ +struct kbase_clk_rate_trace_manager { + bool gpu_idle; + struct kbase_clk_data *clks[BASE_MAX_NR_CLOCKS_REGULATORS]; + struct kbase_clk_rate_trace_op_conf *clk_rate_trace_ops; + struct list_head listeners; + spinlock_t lock; +}; + +/** + * Data stored per device for power management. + * + * This structure contains data for the power management framework. There is one + * instance of this structure per device in the system. + */ +struct kbase_pm_device_data { + /** + * The lock protecting Power Management structures accessed outside of + * IRQ. + * + * This lock must also be held whenever the GPU is being powered on or + * off. + */ + struct mutex lock; + + /** + * The reference count of active contexts on this device. Note that + * some code paths keep shaders/the tiler powered whilst this is 0. Use + * kbase_pm_is_active() instead to check for such cases. + */ + int active_count; + /** Flag indicating suspending/suspended */ + bool suspending; +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* Flag indicating gpu lost */ + atomic_t gpu_lost; +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + /* Wait queue set when active_count == 0 */ + wait_queue_head_t zero_active_count_wait; + + /** + * Bit masks identifying the available shader cores that are specified + * via sysfs. One mask per job slot. + */ + u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS]; + u64 debug_core_mask_all; + + /** + * Callback for initializing the runtime power management. + * + * @param kbdev The kbase device + * + * @return 0 on success, else error code + */ + int (*callback_power_runtime_init)(struct kbase_device *kbdev); + + /** + * Callback for terminating the runtime power management. + * + * @param kbdev The kbase device + */ + void (*callback_power_runtime_term)(struct kbase_device *kbdev); + + /* Time in milliseconds between each dvfs sample */ + u32 dvfs_period; + + struct kbase_pm_backend_data backend; + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /** + * The state of the arbiter VM machine + */ + struct kbase_arbiter_vm_state *arb_vm_state; +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + + /** + * The state of the GPU clock rate trace manager + */ + struct kbase_clk_rate_trace_manager clk_rtm; +}; + +/** + * struct kbase_mem_pool - Page based memory pool for kctx/kbdev + * @kbdev: Kbase device where memory is used + * @cur_size: Number of free pages currently in the pool (may exceed + * @max_size in some corner cases) + * @max_size: Maximum number of free pages in the pool + * @order: order = 0 refers to a pool of 4 KB pages + * order = 9 refers to a pool of 2 MB pages (2^9 * 4KB = 2 MB) + * @group_id: A memory group ID to be passed to a platform-specific + * memory group manager, if present. Immutable. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @pool_lock: Lock protecting the pool - must be held when modifying + * @cur_size and @page_list + * @page_list: List of free pages in the pool + * @reclaim: Shrinker for kernel reclaim of free pages + * @next_pool: Pointer to next pool where pages can be allocated when this + * pool is empty. Pages will spill over to the next pool when + * this pool is full. Can be NULL if there is no next pool. + * @dying: true if the pool is being terminated, and any ongoing + * operations should be abandoned + * @dont_reclaim: true if the shrinker is forbidden from reclaiming memory from + * this pool, eg during a grow operation + */ +struct kbase_mem_pool { + struct kbase_device *kbdev; + size_t cur_size; + size_t max_size; + u8 order; + u8 group_id; + spinlock_t pool_lock; + struct list_head page_list; + struct shrinker reclaim; + + struct kbase_mem_pool *next_pool; + + bool dying; + bool dont_reclaim; +}; + +/** + * struct kbase_mem_pool_group - a complete set of physical memory pools. + * + * Memory pools are used to allow efficient reallocation of previously-freed + * physical pages. A pair of memory pools is initialized for each physical + * memory group: one for 4 KiB pages and one for 2 MiB pages. These arrays + * should be indexed by physical memory group ID, the meaning of which is + * defined by the systems integrator. + * + * @small: Array of objects containing the state for pools of 4 KiB size + * physical pages. + * @large: Array of objects containing the state for pools of 2 MiB size + * physical pages. + */ +struct kbase_mem_pool_group { + struct kbase_mem_pool small[MEMORY_GROUP_MANAGER_NR_GROUPS]; + struct kbase_mem_pool large[MEMORY_GROUP_MANAGER_NR_GROUPS]; +}; + +/** + * struct kbase_mem_pool_config - Initial configuration for a physical memory + * pool + * + * @max_size: Maximum number of free pages that the pool can hold. + */ +struct kbase_mem_pool_config { + size_t max_size; +}; + +/** + * struct kbase_mem_pool_group_config - Initial configuration for a complete + * set of physical memory pools + * + * This array should be indexed by physical memory group ID, the meaning + * of which is defined by the systems integrator. + * + * @small: Array of initial configuration for pools of 4 KiB pages. + * @large: Array of initial configuration for pools of 2 MiB pages. + */ +struct kbase_mem_pool_group_config { + struct kbase_mem_pool_config small[MEMORY_GROUP_MANAGER_NR_GROUPS]; + struct kbase_mem_pool_config large[MEMORY_GROUP_MANAGER_NR_GROUPS]; +}; + +/** + * struct kbase_devfreq_opp - Lookup table for converting between nominal OPP + * frequency, real frequencies and core mask + * @real_freqs: Real GPU frequencies. + * @opp_volts: OPP voltages. + * @opp_freq: Nominal OPP frequency + * @core_mask: Shader core mask + */ +struct kbase_devfreq_opp { + u64 opp_freq; + u64 core_mask; + u64 real_freqs[BASE_MAX_NR_CLOCKS_REGULATORS]; + u32 opp_volts[BASE_MAX_NR_CLOCKS_REGULATORS]; +}; + +/* MMU mode flags */ +#define KBASE_MMU_MODE_HAS_NON_CACHEABLE (1ul << 0) /* Has NON_CACHEABLE MEMATTR */ + +/** + * struct kbase_mmu_mode - object containing pointer to methods invoked for + * programming the MMU, as per the MMU mode supported + * by Hw. + * @update: enable & setup/configure one of the GPU address space. + * @get_as_setup: retrieve the configuration of one of the GPU address space. + * @disable_as: disable one of the GPU address space. + * @pte_to_phy_addr: retrieve the physical address encoded in the page table entry. + * @ate_is_valid: check if the pte is a valid address translation entry + * encoding the physical address of the actual mapped page. + * @pte_is_valid: check if the pte is a valid entry encoding the physical + * address of the next lower level page table. + * @entry_set_ate: program the pte to be a valid address translation entry to + * encode the physical address of the actual page being mapped. + * @entry_set_pte: program the pte to be a valid entry to encode the physical + * address of the next lower level page table. + * @entry_invalidate: clear out or invalidate the pte. + * @flags: bitmask of MMU mode flags. Refer to KBASE_MMU_MODE_ constants. + */ +struct kbase_mmu_mode { + void (*update)(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, + int as_nr); + void (*get_as_setup)(struct kbase_mmu_table *mmut, + struct kbase_mmu_setup * const setup); + void (*disable_as)(struct kbase_device *kbdev, int as_nr); + phys_addr_t (*pte_to_phy_addr)(u64 entry); + int (*ate_is_valid)(u64 ate, int level); + int (*pte_is_valid)(u64 pte, int level); + void (*entry_set_ate)(u64 *entry, struct tagged_addr phy, + unsigned long flags, int level); + void (*entry_set_pte)(u64 *entry, phys_addr_t phy); + void (*entry_invalidate)(u64 *entry); + unsigned long flags; +}; + +struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void); +struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void); + +#define DEVNAME_SIZE 16 + +/** + * enum kbase_devfreq_work_type - The type of work to perform in the devfreq + * suspend/resume worker. + * @DEVFREQ_WORK_NONE: Initilisation state. + * @DEVFREQ_WORK_SUSPEND: Call devfreq_suspend_device(). + * @DEVFREQ_WORK_RESUME: Call devfreq_resume_device(). + */ +enum kbase_devfreq_work_type { + DEVFREQ_WORK_NONE, + DEVFREQ_WORK_SUSPEND, + DEVFREQ_WORK_RESUME +}; + +/** + * struct kbase_devfreq_queue_info - Object representing an instance for managing + * the queued devfreq suspend/resume works. + * @workq: Workqueue for devfreq suspend/resume requests + * @work: Work item for devfreq suspend & resume + * @req_type: Requested work type to be performed by the devfreq + * suspend/resume worker + * @acted_type: Work type has been acted on by the worker, i.e. the + * internal recorded state of the suspend/resume + */ +struct kbase_devfreq_queue_info { + struct workqueue_struct *workq; + struct work_struct work; + enum kbase_devfreq_work_type req_type; + enum kbase_devfreq_work_type acted_type; +}; + +/** + * struct kbase_process - Representing an object of a kbase process instantiated + * when the first kbase context is created under it. + * @tgid: Thread group ID. + * @total_gpu_pages: Total gpu pages allocated across all the contexts + * of this process, it accounts for both native allocations + * and dma_buf imported allocations. + * @kctx_list: List of kbase contexts created for the process. + * @kprcs_node: Node to a rb_tree, kbase_device will maintain a rb_tree + * based on key tgid, kprcs_node is the node link to + * &struct_kbase_device.process_root. + * @dma_buf_root: RB tree of the dma-buf imported allocations, imported + * across all the contexts created for this process. + * Used to ensure that pages of allocation are accounted + * only once for the process, even if the allocation gets + * imported multiple times for the process. + */ +struct kbase_process { + pid_t tgid; + size_t total_gpu_pages; + struct list_head kctx_list; + + struct rb_node kprcs_node; + struct rb_root dma_buf_root; +}; + +/** + * struct kbase_device - Object representing an instance of GPU platform device, + * allocated from the probe method of mali driver. + * @hw_quirks_sc: Configuration to be used for the shader cores as per + * the HW issues present in the GPU. + * @hw_quirks_tiler: Configuration to be used for the Tiler as per the HW + * issues present in the GPU. + * @hw_quirks_mmu: Configuration to be used for the MMU as per the HW + * issues present in the GPU. + * @hw_quirks_jm: Configuration to be used for the Job Manager as per + * the HW issues present in the GPU. + * @entry: Links the device instance to the global list of GPU + * devices. The list would have as many entries as there + * are GPU device instances. + * @dev: Pointer to the kernel's generic/base representation + * of the GPU platform device. + * @mdev: Pointer to the miscellaneous device registered to + * provide Userspace access to kernel driver through the + * device file /dev/malixx. + * @reg_start: Base address of the region in physical address space + * where GPU registers have been mapped. + * @reg_size: Size of the region containing GPU registers + * @reg: Kernel virtual address of the region containing GPU + * registers, using which Driver will access the registers. + * @irqs: Array containing IRQ resource info for 3 types of + * interrupts : Job scheduling, MMU & GPU events (like + * power management, cache etc.) + * @clocks: Pointer to the input clock resources referenced by + * the GPU device node. + * @nr_clocks: Number of clocks set in the clocks array. + * @regulators: Pointer to the structs corresponding to the + * regulators referenced by the GPU device node. + * @nr_regulators: Number of regulators set in the regulators array. + * @opp_table: Pointer to the device OPP structure maintaining the + * link to OPPs attached to a device. This is obtained + * after setting regulator names for the device. + * @devname: string containing the name used for GPU device instance, + * miscellaneous device is registered using the same name. + * @id: Unique identifier for the device, indicates the number of + * devices which have been created so far. + * @model: Pointer, valid only when Driver is compiled to not access + * the real GPU Hw, to the dummy model which tries to mimic + * to some extent the state & behavior of GPU Hw in response + * to the register accesses made by the Driver. + * @irq_slab: slab cache for allocating the work items queued when + * model mimics raising of IRQ to cause an interrupt on CPU. + * @irq_workq: workqueue for processing the irq work items. + * @serving_job_irq: function to execute work items queued when model mimics + * the raising of JS irq, mimics the interrupt handler + * processing JS interrupts. + * @serving_gpu_irq: function to execute work items queued when model mimics + * the raising of GPU irq, mimics the interrupt handler + * processing GPU interrupts. + * @serving_mmu_irq: function to execute work items queued when model mimics + * the raising of MMU irq, mimics the interrupt handler + * processing MMU interrupts. + * @reg_op_lock: lock used by model to serialize the handling of register + * accesses made by the driver. + * @pm: Per device object for storing data for power management + * framework. + * @js_data: Per device object encapsulating the current context of + * Job Scheduler, which is global to the device and is not + * tied to any particular struct kbase_context running on + * the device + * @mem_pools: Global pools of free physical memory pages which can + * be used by all the contexts. + * @memdev: keeps track of the in use physical pages allocated by + * the Driver. + * @mmu_mode: Pointer to the object containing methods for programming + * the MMU, depending on the type of MMU supported by Hw. + * @mgm_dev: Pointer to the memory group manager device attached + * to the GPU device. This points to an internal memory + * group manager if no platform-specific memory group + * manager was retrieved through device tree. + * @as: Array of objects representing address spaces of GPU. + * @as_free: Bitpattern of free/available GPU address spaces. + * @as_to_kctx: Array of pointers to struct kbase_context, having + * GPU adrress spaces assigned to them. + * @mmu_mask_change: Lock to serialize the access to MMU interrupt mask + * register used in the handling of Bus & Page faults. + * @gpu_props: Object containing complete information about the + * configuration/properties of GPU HW device in use. + * @hw_issues_mask: List of SW workarounds for HW issues + * @hw_features_mask: List of available HW features. + * @disjoint_event: struct for keeping track of the disjoint information, + * that whether the GPU is in a disjoint state and the + * number of disjoint events that have occurred on GPU. + * @nr_hw_address_spaces: Number of address spaces actually available in the + * GPU, remains constant after driver initialisation. + * @nr_user_address_spaces: Number of address spaces available to user contexts + * @hwcnt: Structure used for instrumentation and HW counters + * dumping + * @hwcnt_gpu_iface: Backend interface for GPU hardware counter access. + * @hwcnt_gpu_ctx: Context for GPU hardware counter access. + * @hwaccess_lock must be held when calling + * kbase_hwcnt_context_enable() with @hwcnt_gpu_ctx. + * @hwcnt_gpu_virt: Virtualizer for GPU hardware counters. + * @vinstr_ctx: vinstr context created per device. + * @timeline_flags: Bitmask defining which sets of timeline tracepoints + * are enabled. If zero, there is no timeline client and + * therefore timeline is disabled. + * @timeline: Timeline context created per device. + * @trace_lock: Lock to serialize the access to trace buffer. + * @trace_first_out: Index/offset in the trace buffer at which the first + * unread message is present. + * @trace_next_in: Index/offset in the trace buffer at which the new + * message will be written. + * @trace_rbuf: Pointer to the buffer storing debug messages/prints + * tracing the various events in Driver. + * The buffer is filled in circular fashion. + * @reset_timeout_ms: Number of milliseconds to wait for the soft stop to + * complete for the GPU jobs before proceeding with the + * GPU reset. + * @cache_clean_in_progress: Set when a cache clean has been started, and + * cleared when it has finished. This prevents multiple + * cache cleans being done simultaneously. + * @cache_clean_queued: Set if a cache clean is invoked while another is in + * progress. If this happens, another cache clean needs + * to be triggered immediately after completion of the + * current one. + * @cache_clean_wait: Signalled when a cache clean has finished. + * @platform_context: Platform specific private data to be accessed by + * platform specific config files only. + * @kctx_list: List of kbase_contexts created for the device, + * including any contexts that might be created for + * hardware counters. + * @kctx_list_lock: Lock protecting concurrent accesses to @kctx_list. + * @devfreq_profile: Describes devfreq profile for the Mali GPU device, passed + * to devfreq_add_device() to add devfreq feature to Mali + * GPU device. + * @devfreq: Pointer to devfreq structure for Mali GPU device, + * returned on the call to devfreq_add_device(). + * @current_freqs: The real frequencies, corresponding to + * @current_nominal_freq, at which the Mali GPU device + * is currently operating, as retrieved from + * @devfreq_table in the target callback of + * @devfreq_profile. + * @current_nominal_freq: The nominal frequency currently used for the Mali GPU + * device as retrieved through devfreq_recommended_opp() + * using the freq value passed as an argument to target + * callback of @devfreq_profile + * @current_voltages: The voltages corresponding to @current_nominal_freq, + * as retrieved from @devfreq_table in the target + * callback of @devfreq_profile. + * @current_core_mask: bitmask of shader cores that are currently desired & + * enabled, corresponding to @current_nominal_freq as + * retrieved from @devfreq_table in the target callback + * of @devfreq_profile. + * @devfreq_table: Pointer to the lookup table for converting between + * nominal OPP (operating performance point) frequency, + * and real frequency and core mask. This table is + * constructed according to operating-points-v2-mali + * table in devicetree. + * @num_opps: Number of operating performance points available for the Mali + * GPU device. + * @devfreq_queue: Per device object for storing data that manages devfreq + * suspend & resume request queue and the related items. + * @devfreq_cooling: Pointer returned on registering devfreq cooling device + * corresponding to @devfreq. + * @ipa_protection_mode_switched: is set to TRUE when GPU is put into protected + * mode. It is a sticky flag which is cleared by IPA + * once it has made use of information that GPU had + * previously entered protected mode. + * @ipa: Top level structure for IPA, containing pointers to both + * configured & fallback models. + * @previous_frequency: Previous frequency of GPU clock used for + * BASE_HW_ISSUE_GPU2017_1336 workaround, This clock is + * restored when L2 is powered on. + * @job_fault_debug: Flag to control the dumping of debug data for job faults, + * set when the 'job_fault' debugfs file is opened. + * @mali_debugfs_directory: Root directory for the debugfs files created by the driver + * @debugfs_ctx_directory: Directory inside the @mali_debugfs_directory containing + * a sub-directory for every context. + * @debugfs_as_read_bitmap: bitmap of address spaces for which the bus or page fault + * has occurred. + * @job_fault_wq: Waitqueue to block the job fault dumping daemon till the + * occurrence of a job fault. + * @job_fault_resume_wq: Waitqueue on which every context with a faulty job wait + * for the job fault dumping to complete before they can + * do bottom half of job done for the atoms which followed + * the faulty atom. + * @job_fault_resume_workq: workqueue to process the work items queued for the faulty + * atoms, whereby the work item function waits for the dumping + * to get completed. + * @job_fault_event_list: List of atoms, each belonging to a different context, which + * generated a job fault. + * @job_fault_event_lock: Lock to protect concurrent accesses to @job_fault_event_list + * @regs_dump_debugfs_data: Contains the offset of register to be read through debugfs + * file "read_register". + * @ctx_num: Total number of contexts created for the device. + * @io_history: Pointer to an object keeping a track of all recent + * register accesses. The history of register accesses + * can be read through "regs_history" debugfs file. + * @hwaccess: Contains a pointer to active kbase context and GPU + * backend specific data for HW access layer. + * @faults_pending: Count of page/bus faults waiting for bottom half processing + * via workqueues. + * @poweroff_pending: Set when power off operation for GPU is started, reset when + * power on for GPU is started. + * @infinite_cache_active_default: Set to enable using infinite cache for all the + * allocations of a new context. + * @mem_pool_defaults: Default configuration for the group of memory pools + * created for a new context. + * @current_gpu_coherency_mode: coherency mode in use, which can be different + * from @system_coherency, when using protected mode. + * @system_coherency: coherency mode as retrieved from the device tree. + * @cci_snoop_enabled: Flag to track when CCI snoops have been enabled. + * @snoop_enable_smc: SMC function ID to call into Trusted firmware to + * enable cache snooping. Value of 0 indicates that it + * is not used. + * @snoop_disable_smc: SMC function ID to call disable cache snooping. + * @protected_ops: Pointer to the methods for switching in or out of the + * protected mode, as per the @protected_dev being used. + * @protected_dev: Pointer to the protected mode switcher device attached + * to the GPU device retrieved through device tree if + * GPU do not support protected mode switching natively. + * @protected_mode: set to TRUE when GPU is put into protected mode + * @protected_mode_transition: set to TRUE when GPU is transitioning into or + * out of protected mode. + * @protected_mode_hwcnt_desired: True if we want GPU hardware counters to be + * enabled. Counters must be disabled before transition + * into protected mode. + * @protected_mode_hwcnt_disabled: True if GPU hardware counters are not + * enabled. + * @protected_mode_hwcnt_disable_work: Work item to disable GPU hardware + * counters, used if atomic disable is not possible. + * @buslogger: Pointer to the structure required for interfacing + * with the bus logger module to set the size of buffer + * used by the module for capturing bus logs. + * @irq_reset_flush: Flag to indicate that GPU reset is in-flight and flush of + * IRQ + bottom half is being done, to prevent the writes + * to MMU_IRQ_CLEAR & MMU_IRQ_MASK registers. + * @inited_subsys: Bitmap of inited sub systems at the time of device probe. + * Used during device remove or for handling error in probe. + * @hwaccess_lock: Lock, which can be taken from IRQ context, to serialize + * the updates made to Job dispatcher + scheduler states. + * @mmu_hw_mutex: Protects access to MMU operations and address space + * related state. + * @serialize_jobs: Currently used mode for serialization of jobs, both + * intra & inter slots serialization is supported. + * @backup_serialize_jobs: Copy of the original value of @serialize_jobs taken + * when GWT is enabled. Used to restore the original value + * on disabling of GWT. + * @js_ctx_scheduling_mode: Context scheduling mode currently being used by + * Job Scheduler + * @l2_size_override: Used to set L2 cache size via device tree blob + * @l2_hash_override: Used to set L2 cache hash via device tree blob + * @process_root: rb_tree root node for maintaining a rb_tree of + * kbase_process based on key tgid(thread group ID). + * @dma_buf_root: rb_tree root node for maintaining a rb_tree of + * &struct kbase_dma_buf based on key dma_buf. + * We maintain a rb_tree of dma_buf mappings under + * kbase_device and kbase_process, one indicates a + * mapping and gpu memory usage at device level and + * other one at process level. + * @total_gpu_pages: Total GPU pages used for the complete GPU device. + * @dma_buf_lock: This mutex should be held while accounting for + * @total_gpu_pages from imported dma buffers. + * @gpu_mem_usage_lock: This spinlock should be held while accounting + * @total_gpu_pages for both native and dma-buf imported + * allocations. + */ +struct kbase_device { + u32 hw_quirks_sc; + u32 hw_quirks_tiler; + u32 hw_quirks_mmu; + u32 hw_quirks_jm; + + struct list_head entry; + struct device *dev; + struct miscdevice mdev; + u64 reg_start; + size_t reg_size; + void __iomem *reg; + + struct { + int irq; + int flags; + } irqs[3]; + + struct clk *clocks[BASE_MAX_NR_CLOCKS_REGULATORS]; + unsigned int nr_clocks; +#ifdef CONFIG_REGULATOR + struct regulator *regulators[BASE_MAX_NR_CLOCKS_REGULATORS]; + unsigned int nr_regulators; +#if (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) + struct opp_table *opp_table; +#endif /* (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE */ +#endif /* CONFIG_REGULATOR */ + char devname[DEVNAME_SIZE]; + u32 id; + +#ifdef CONFIG_MALI_BIFROST_NO_MALI + void *model; + struct kmem_cache *irq_slab; + struct workqueue_struct *irq_workq; + atomic_t serving_job_irq; + atomic_t serving_gpu_irq; + atomic_t serving_mmu_irq; + spinlock_t reg_op_lock; +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ + + struct kbase_pm_device_data pm; + + struct kbase_mem_pool_group mem_pools; + struct kbasep_mem_device memdev; + struct kbase_mmu_mode const *mmu_mode; + + struct memory_group_manager_device *mgm_dev; + + struct kbase_as as[BASE_MAX_NR_AS]; + u16 as_free; /* Bitpattern of free Address Spaces */ + struct kbase_context *as_to_kctx[BASE_MAX_NR_AS]; + + spinlock_t mmu_mask_change; + + struct kbase_gpu_props gpu_props; + + unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; + unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; + + struct { + atomic_t count; + atomic_t state; + } disjoint_event; + + s8 nr_hw_address_spaces; + s8 nr_user_address_spaces; + + struct kbase_hwcnt { + /* The lock should be used when accessing any of the following members */ + spinlock_t lock; + + struct kbase_context *kctx; + u64 addr; + u64 addr_bytes; + + struct kbase_instr_backend backend; + } hwcnt; + + struct kbase_hwcnt_backend_interface hwcnt_gpu_iface; + struct kbase_hwcnt_context *hwcnt_gpu_ctx; + struct kbase_hwcnt_virtualizer *hwcnt_gpu_virt; + struct kbase_vinstr_context *vinstr_ctx; + + atomic_t timeline_flags; + struct kbase_timeline *timeline; + +#if KBASE_KTRACE_TARGET_RBUF + struct kbase_ktrace ktrace; +#endif + u32 reset_timeout_ms; + + bool cache_clean_in_progress; + bool cache_clean_queued; + wait_queue_head_t cache_clean_wait; + + void *platform_context; + + struct list_head kctx_list; + struct mutex kctx_list_lock; + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + struct devfreq_dev_profile devfreq_profile; + struct devfreq *devfreq; + unsigned long current_freqs[BASE_MAX_NR_CLOCKS_REGULATORS]; + unsigned long current_nominal_freq; + unsigned long current_voltages[BASE_MAX_NR_CLOCKS_REGULATORS]; + u64 current_core_mask; + struct kbase_devfreq_opp *devfreq_table; + int num_opps; + struct kbasep_pm_metrics last_devfreq_metrics; + struct monitor_dev_info *mdev_info; + struct ipa_power_model_data *model_data; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + struct kbase_devfreq_queue_info devfreq_queue; +#endif + +#ifdef CONFIG_DEVFREQ_THERMAL +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + struct devfreq_cooling_device *devfreq_cooling; +#else + struct thermal_cooling_device *devfreq_cooling; +#endif + bool ipa_protection_mode_switched; + struct { + /* Access to this struct must be with ipa.lock held */ + struct mutex lock; + struct kbase_ipa_model *configured_model; + struct kbase_ipa_model *fallback_model; + + /* Values of the PM utilization metrics from last time the + * power model was invoked. The utilization is calculated as + * the difference between last_metrics and the current values. + */ + struct kbasep_pm_metrics last_metrics; + /* Model data to pass to ipa_gpu_active/idle() */ + struct kbase_ipa_model_vinstr_data *model_data; + + /* true if use of fallback model has been forced by the User */ + bool force_fallback_model; + } ipa; +#endif /* CONFIG_DEVFREQ_THERMAL */ +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + unsigned long previous_frequency; + + atomic_t job_fault_debug; + +#ifdef CONFIG_DEBUG_FS + struct dentry *mali_debugfs_directory; + struct dentry *debugfs_ctx_directory; + struct dentry *debugfs_instr_directory; + +#ifdef CONFIG_MALI_BIFROST_DEBUG + u64 debugfs_as_read_bitmap; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + wait_queue_head_t job_fault_wq; + wait_queue_head_t job_fault_resume_wq; + struct workqueue_struct *job_fault_resume_workq; + struct list_head job_fault_event_list; + spinlock_t job_fault_event_lock; + +#if !MALI_CUSTOMER_RELEASE + struct { + u16 reg_offset; + } regs_dump_debugfs_data; +#endif /* !MALI_CUSTOMER_RELEASE */ +#endif /* CONFIG_DEBUG_FS */ + + atomic_t ctx_num; + +#ifdef CONFIG_DEBUG_FS + struct kbase_io_history io_history; +#endif /* CONFIG_DEBUG_FS */ + + struct kbase_hwaccess_data hwaccess; + + atomic_t faults_pending; + + bool poweroff_pending; + + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) + bool infinite_cache_active_default; +#else + u32 infinite_cache_active_default; +#endif + struct kbase_mem_pool_group_config mem_pool_defaults; + + u32 current_gpu_coherency_mode; + u32 system_coherency; + + bool cci_snoop_enabled; + + u32 snoop_enable_smc; + u32 snoop_disable_smc; + + const struct protected_mode_ops *protected_ops; + + struct protected_mode_device *protected_dev; + + bool protected_mode; + + bool protected_mode_transition; + + bool protected_mode_hwcnt_desired; + + bool protected_mode_hwcnt_disabled; + + struct work_struct protected_mode_hwcnt_disable_work; + +#ifdef CONFIG_MALI_BUSLOG + struct bus_logger_client *buslogger; +#endif + + bool irq_reset_flush; + + u32 inited_subsys; + + spinlock_t hwaccess_lock; + + struct mutex mmu_hw_mutex; + + u8 l2_size_override; + u8 l2_hash_override; + +#if MALI_USE_CSF + /* Command-stream front-end for the device. */ + struct kbase_csf_device csf; +#else + struct kbasep_js_device_data js_data; + + /* See KBASE_JS_*_PRIORITY_MODE for details. */ + u32 js_ctx_scheduling_mode; + + /* See KBASE_SERIALIZE_* for details */ + u8 serialize_jobs; + +#ifdef CONFIG_MALI_CINSTR_GWT + u8 backup_serialize_jobs; +#endif /* CONFIG_MALI_CINSTR_GWT */ + +#endif /* MALI_USE_CSF */ + + struct rb_root process_root; + struct rb_root dma_buf_root; + + size_t total_gpu_pages; + struct mutex dma_buf_lock; + spinlock_t gpu_mem_usage_lock; + + struct { + struct kbase_context *ctx; + u64 jc; + int slot; + u64 flags; + } dummy_job_wa; + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* Pointer to the arbiter device */ + struct kbase_arbiter_device arb; +#endif +}; + +/** + * enum kbase_file_state - Initialization state of a file opened by @kbase_open + * + * @KBASE_FILE_NEED_VSN: Initial state, awaiting API version. + * @KBASE_FILE_VSN_IN_PROGRESS: Indicates if setting an API version is in + * progress and other setup calls shall be + * rejected. + * @KBASE_FILE_NEED_CTX: Indicates if the API version handshake has + * completed, awaiting context creation flags. + * @KBASE_FILE_CTX_IN_PROGRESS: Indicates if the context's setup is in progress + * and other setup calls shall be rejected. + * @KBASE_FILE_COMPLETE: Indicates if the setup for context has + * completed, i.e. flags have been set for the + * context. + * + * The driver allows only limited interaction with user-space until setup + * is complete. + */ +enum kbase_file_state { + KBASE_FILE_NEED_VSN, + KBASE_FILE_VSN_IN_PROGRESS, + KBASE_FILE_NEED_CTX, + KBASE_FILE_CTX_IN_PROGRESS, + KBASE_FILE_COMPLETE +}; + +/** + * struct kbase_file - Object representing a file opened by @kbase_open + * + * @kbdev: Object representing an instance of GPU platform device, + * allocated from the probe method of the Mali driver. + * @filp: Pointer to the struct file corresponding to device file + * /dev/malixx instance, passed to the file's open method. + * @kctx: Object representing an entity, among which GPU is + * scheduled and which gets its own GPU address space. + * Invalid until @setup_state is KBASE_FILE_COMPLETE. + * @api_version: Contains the version number for User/kernel interface, + * used for compatibility check. Invalid until + * @setup_state is KBASE_FILE_NEED_CTX. + * @setup_state: Initialization state of the file. Values come from + * the kbase_file_state enumeration. + */ +struct kbase_file { + struct kbase_device *kbdev; + struct file *filp; + struct kbase_context *kctx; + unsigned long api_version; + atomic_t setup_state; +}; + +/** + * enum kbase_context_flags - Flags for kbase contexts + * + * @KCTX_COMPAT: Set when the context process is a compat process, 32-bit + * process on a 64-bit kernel. + * + * @KCTX_RUNNABLE_REF: Set when context is counted in + * kbdev->js_data.nr_contexts_runnable. Must hold queue_mutex when accessing. + * + * @KCTX_ACTIVE: Set when the context is active. + * + * @KCTX_PULLED: Set when last kick() caused atoms to be pulled from this + * context. + * + * @KCTX_MEM_PROFILE_INITIALIZED: Set when the context's memory profile has been + * initialized. + * + * @KCTX_INFINITE_CACHE: Set when infinite cache is to be enabled for new + * allocations. Existing allocations will not change. + * + * @KCTX_SUBMIT_DISABLED: Set to prevent context from submitting any jobs. + * + * @KCTX_PRIVILEGED:Set if the context uses an address space and should be kept + * scheduled in. + * + * @KCTX_SCHEDULED: Set when the context is scheduled on the Run Pool. + * This is only ever updated whilst the jsctx_mutex is held. + * + * @KCTX_DYING: Set when the context process is in the process of being evicted. + * + * @KCTX_NO_IMPLICIT_SYNC: Set when explicit Android fences are in use on this + * context, to disable use of implicit dma-buf fences. This is used to avoid + * potential synchronization deadlocks. + * + * @KCTX_FORCE_SAME_VA: Set when BASE_MEM_SAME_VA should be forced on memory + * allocations. For 64-bit clients it is enabled by default, and disabled by + * default on 32-bit clients. Being able to clear this flag is only used for + * testing purposes of the custom zone allocation on 64-bit user-space builds, + * where we also require more control than is available through e.g. the JIT + * allocation mechanism. However, the 64-bit user-space client must still + * reserve a JIT region using KBASE_IOCTL_MEM_JIT_INIT + * + * @KCTX_PULLED_SINCE_ACTIVE_JS0: Set when the context has had an atom pulled + * from it for job slot 0. This is reset when the context first goes active or + * is re-activated on that slot. + * + * @KCTX_PULLED_SINCE_ACTIVE_JS1: Set when the context has had an atom pulled + * from it for job slot 1. This is reset when the context first goes active or + * is re-activated on that slot. + * + * @KCTX_PULLED_SINCE_ACTIVE_JS2: Set when the context has had an atom pulled + * from it for job slot 2. This is reset when the context first goes active or + * is re-activated on that slot. + * + * @KCTX_AS_DISABLED_ON_FAULT: Set when the GPU address space is disabled for + * the context due to unhandled page(or bus) fault. It is cleared when the + * refcount for the context drops to 0 or on when the address spaces are + * re-enabled on GPU reset or power cycle. + * + * All members need to be separate bits. This enum is intended for use in a + * bitmask where multiple values get OR-ed together. + */ +enum kbase_context_flags { + KCTX_COMPAT = 1U << 0, + KCTX_RUNNABLE_REF = 1U << 1, + KCTX_ACTIVE = 1U << 2, + KCTX_PULLED = 1U << 3, + KCTX_MEM_PROFILE_INITIALIZED = 1U << 4, + KCTX_INFINITE_CACHE = 1U << 5, + KCTX_SUBMIT_DISABLED = 1U << 6, + KCTX_PRIVILEGED = 1U << 7, + KCTX_SCHEDULED = 1U << 8, + KCTX_DYING = 1U << 9, + KCTX_NO_IMPLICIT_SYNC = 1U << 10, + KCTX_FORCE_SAME_VA = 1U << 11, + KCTX_PULLED_SINCE_ACTIVE_JS0 = 1U << 12, + KCTX_PULLED_SINCE_ACTIVE_JS1 = 1U << 13, + KCTX_PULLED_SINCE_ACTIVE_JS2 = 1U << 14, + KCTX_AS_DISABLED_ON_FAULT = 1U << 15, +#if MALI_JIT_PRESSURE_LIMIT_BASE + /* + * Set when JIT physical page limit is less than JIT virtual address + * page limit, so we must take care to not exceed the physical limit + */ + KCTX_JPL_ENABLED = 1U << 16, +#endif /* !MALI_JIT_PRESSURE_LIMIT_BASE */ +}; + +struct kbase_sub_alloc { + struct list_head link; + struct page *page; + DECLARE_BITMAP(sub_pages, SZ_2M / SZ_4K); +}; + +/** + * struct kbase_context - Kernel base context + * + * @filp: Pointer to the struct file corresponding to device file + * /dev/malixx instance, passed to the file's open method. + * @kbdev: Pointer to the Kbase device for which the context is created. + * @kctx_list_link: Node into Kbase device list of contexts. + * @mmu: Structure holding details of the MMU tables for this + * context + * @id: Unique identifier for the context, indicates the number of + * contexts which have been created for the device so far. + * @api_version: contains the version number for User/kernel interface, + * used for compatibility check. + * @event_list: list of posted events about completed atoms, to be sent to + * event handling thread of Userpsace. + * @event_coalesce_list: list containing events corresponding to successive atoms + * which have requested deferred delivery of the completion + * events to Userspace. + * @event_mutex: Lock to protect the concurrent access to @event_list & + * @event_mutex. + * @event_closed: Flag set through POST_TERM ioctl, indicates that Driver + * should stop posting events and also inform event handling + * thread that context termination is in progress. + * @event_workq: Workqueue for processing work items corresponding to atoms + * that do not return an event to userspace. + * @event_count: Count of the posted events to be consumed by Userspace. + * @event_coalesce_count: Count of the events present in @event_coalesce_list. + * @flags: bitmap of enums from kbase_context_flags, indicating the + * state & attributes for the context. + * @aliasing_sink_page: Special page used for KBASE_MEM_TYPE_ALIAS allocations, + * which can alias number of memory regions. The page is + * represent a region where it is mapped with a write-alloc + * cache setup, typically used when the write result of the + * GPU isn't needed, but the GPU must write anyway. + * @mem_partials_lock: Lock for protecting the operations done on the elements + * added to @mem_partials list. + * @mem_partials: List head for the list of large pages, 2MB in size, which + * which have been split into 4 KB pages and are used + * partially for the allocations >= 2 MB in size. + * @reg_lock: Lock used for GPU virtual address space management operations, + * like adding/freeing a memory region in the address space. + * Can be converted to a rwlock ?. + * @reg_rbtree_same: RB tree of the memory regions allocated from the SAME_VA + * zone of the GPU virtual address space. Used for allocations + * having the same value for GPU & CPU virtual address. + * @reg_rbtree_custom: RB tree of the memory regions allocated from the CUSTOM_VA + * zone of the GPU virtual address space. + * @reg_rbtree_exec: RB tree of the memory regions allocated from the EXEC_VA + * zone of the GPU virtual address space. Used for GPU-executable + * allocations which don't need the SAME_VA property. + * @cookies: Bitmask containing of BITS_PER_LONG bits, used mainly for + * SAME_VA allocations to defer the reservation of memory region + * (from the GPU virtual address space) from base_mem_alloc + * ioctl to mmap system call. This helps returning unique + * handles, disguised as GPU VA, to Userspace from base_mem_alloc + * and later retrieving the pointer to memory region structure + * in the mmap handler. + * @pending_regions: Array containing pointers to memory region structures, + * used in conjunction with @cookies bitmask mainly for + * providing a mechansim to have the same value for CPU & + * GPU virtual address. + * @event_queue: Wait queue used for blocking the thread, which consumes + * the base_jd_event corresponding to an atom, when there + * are no more posted events. + * @tgid: Thread group ID of the process whose thread created + * the context (by calling KBASE_IOCTL_VERSION_CHECK or + * KBASE_IOCTL_SET_FLAGS, depending on the @api_version). + * This is usually, but not necessarily, the same as the + * process whose thread opened the device file + * /dev/malixx instance. + * @pid: ID of the thread, corresponding to process @tgid, + * which actually created the context. This is usually, + * but not necessarily, the same as the thread which + * opened the device file /dev/malixx instance. + * @jctx: object encapsulating all the Job dispatcher related state, + * including the array of atoms. + * @used_pages: Keeps a track of the number of 4KB physical pages in use + * for the context. + * @nonmapped_pages: Updated in the same way as @used_pages, except for the case + * when special tracking page is freed by userspace where it + * is reset to 0. + * @permanent_mapped_pages: Usage count of permanently mapped memory + * @mem_pools: Context-specific pools of free physical memory pages. + * @reclaim: Shrinker object registered with the kernel containing + * the pointer to callback function which is invoked under + * low memory conditions. In the callback function Driver + * frees up the memory for allocations marked as + * evictable/reclaimable. + * @evict_list: List head for the list containing the allocations which + * can be evicted or freed up in the shrinker callback. + * @waiting_soft_jobs: List head for the list containing softjob atoms, which + * are either waiting for the event set operation, or waiting + * for the signaling of input fence or waiting for the GPU + * device to powered on so as to dump the CPU/GPU timestamps. + * @waiting_soft_jobs_lock: Lock to protect @waiting_soft_jobs list from concurrent + * accesses. + * @dma_fence: Object containing list head for the list of dma-buf fence + * waiting atoms and the waitqueue to process the work item + * queued for the atoms blocked on the signaling of dma-buf + * fences. + * @as_nr: id of the address space being used for the scheduled in + * context. This is effectively part of the Run Pool, because + * it only has a valid setting (!=KBASEP_AS_NR_INVALID) whilst + * the context is scheduled in. The hwaccess_lock must be held + * whilst accessing this. + * If the context relating to this value of as_nr is required, + * then the context must be retained to ensure that it doesn't + * disappear whilst it is being used. Alternatively, hwaccess_lock + * can be held to ensure the context doesn't disappear (but this + * has restrictions on what other locks can be taken simutaneously). + * @refcount: Keeps track of the number of users of this context. A user + * can be a job that is available for execution, instrumentation + * needing to 'pin' a context for counter collection, etc. + * If the refcount reaches 0 then this context is considered + * inactive and the previously programmed AS might be cleared + * at any point. + * Generally the reference count is incremented when the context + * is scheduled in and an atom is pulled from the context's per + * slot runnable tree in JM GPU or GPU command queue + * group is programmed on CSG slot in CSF GPU. + * @mm_update_lock: lock used for handling of special tracking page. + * @process_mm: Pointer to the memory descriptor of the process which + * created the context. Used for accounting the physical + * pages used for GPU allocations, done for the context, + * to the memory consumed by the process. + * @same_va_end: End address of the SAME_VA zone (in 4KB page units) + * @exec_va_start: Start address of the EXEC_VA zone (in 4KB page units) + * or U64_MAX if the EXEC_VA zone is uninitialized. + * @gpu_va_end: End address of the GPU va space (in 4KB page units) + * @jit_va: Indicates if a JIT_VA zone has been created. + * @mem_profile_data: Buffer containing the profiling information provided by + * Userspace, can be read through the mem_profile debugfs file. + * @mem_profile_size: Size of the @mem_profile_data. + * @mem_profile_lock: Lock to serialize the operations related to mem_profile + * debugfs file. + * @kctx_dentry: Pointer to the debugfs directory created for every context, + * inside kbase_device::debugfs_ctx_directory, containing + * context specific files. + * @reg_dump: Buffer containing a register offset & value pair, used + * for dumping job fault debug info. + * @job_fault_count: Indicates that a job fault occurred for the context and + * dumping of its debug info is in progress. + * @job_fault_resume_event_list: List containing atoms completed after the faulty + * atom but before the debug data for faulty atom was dumped. + * @jsctx_queue: Per slot & priority arrays of object containing the root + * of RB-tree holding currently runnable atoms on the job slot + * and the head item of the linked list of atoms blocked on + * cross-slot dependencies. + * @atoms_pulled: Total number of atoms currently pulled from the context. + * @atoms_pulled_slot: Per slot count of the number of atoms currently pulled + * from the context. + * @atoms_pulled_slot_pri: Per slot & priority count of the number of atoms currently + * pulled from the context. hwaccess_lock shall be held when + * accessing it. + * @blocked_js: Indicates if the context is blocked from submitting atoms + * on a slot at a given priority. This is set to true, when + * the atom corresponding to context is soft/hard stopped or + * removed from the HEAD_NEXT register in response to + * soft/hard stop. + * @slots_pullable: Bitmask of slots, indicating the slots for which the + * context has pullable atoms in the runnable tree. + * @work: Work structure used for deferred ASID assignment. + * @legacy_hwcnt_cli: Pointer to the legacy userspace hardware counters + * client, there can be only such client per kbase + * context. + * @legacy_hwcnt_lock: Lock used to prevent concurrent access to + * @legacy_hwcnt_cli. + * @completed_jobs: List containing completed atoms for which base_jd_event is + * to be posted. + * @work_count: Number of work items, corresponding to atoms, currently + * pending on job_done workqueue of @jctx. + * @soft_job_timeout: Timer object used for failing/cancelling the waiting + * soft-jobs which have been blocked for more than the + * timeout value used for the soft-jobs + * @jit_alloc: Array of 256 pointers to GPU memory regions, used for + * just-in-time memory allocations. + * @jit_max_allocations: Maximum allowed number of in-flight + * just-in-time memory allocations. + * @jit_current_allocations: Current number of in-flight just-in-time + * memory allocations. + * @jit_current_allocations_per_bin: Current number of in-flight just-in-time + * memory allocations per bin. + * @jit_version: Version number indicating whether userspace is using + * old or new version of interface for just-in-time + * memory allocations. + * 1 -> client used KBASE_IOCTL_MEM_JIT_INIT_10_2 + * 2 -> client used KBASE_IOCTL_MEM_JIT_INIT_11_5 + * 3 -> client used KBASE_IOCTL_MEM_JIT_INIT + * @jit_group_id: A memory group ID to be passed to a platform-specific + * memory group manager. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @jit_phys_pages_limit: Limit of physical pages to apply across all + * just-in-time memory allocations, applied to + * @jit_current_phys_pressure. + * @jit_current_phys_pressure: Current 'pressure' on physical pages, which is + * the sum of the worst case estimate of pages that + * could be used (i.e. the + * &struct_kbase_va_region.nr_pages for all in-use + * just-in-time memory regions that have not yet had + * a usage report) and the actual number of pages + * that were used (i.e. the + * &struct_kbase_va_region.used_pages for regions + * that have had a usage report). + * @jit_phys_pages_to_be_allocated: Count of the physical pages that are being + * now allocated for just-in-time memory + * allocations of a context (across all the + * threads). This is supposed to be updated + * with @reg_lock held before allocating + * the backing pages. This helps ensure that + * total physical memory usage for just in + * time memory allocation remains within the + * @jit_phys_pages_limit in multi-threaded + * scenarios. + * @jit_active_head: List containing the just-in-time memory allocations + * which are in use. + * @jit_pool_head: List containing the just-in-time memory allocations + * which have been freed up by userspace and so not being + * used by them. + * Driver caches them to quickly fulfill requests for new + * JIT allocations. They are released in case of memory + * pressure as they are put on the @evict_list when they + * are freed up by userspace. + * @jit_destroy_head: List containing the just-in-time memory allocations + * which were moved to it from @jit_pool_head, in the + * shrinker callback, after freeing their backing + * physical pages. + * @jit_evict_lock: Lock used for operations done on just-in-time memory + * allocations and also for accessing @evict_list. + * @jit_work: Work item queued to defer the freeing of a memory + * region when a just-in-time memory allocation is moved + * to @jit_destroy_head. + * @ext_res_meta_head: A list of sticky external resources which were requested to + * be mapped on GPU side, through a softjob atom of type + * EXT_RES_MAP or STICKY_RESOURCE_MAP ioctl. + * @age_count: Counter incremented on every call to jd_submit_atom, + * atom is assigned the snapshot of this counter, which + * is used to determine the atom's age when it is added to + * the runnable RB-tree. + * @trim_level: Level of JIT allocation trimming to perform on free (0-100%) + * @kprcs: Reference to @struct kbase_process that the current + * kbase_context belongs to. + * @kprcs_link: List link for the list of kbase context maintained + * under kbase_process. + * @gwt_enabled: Indicates if tracking of GPU writes is enabled, protected by + * kbase_context.reg_lock. + * @gwt_was_enabled: Simple sticky bit flag to know if GWT was ever enabled. + * @gwt_current_list: A list of addresses for which GPU has generated write faults, + * after the last snapshot of it was sent to userspace. + * @gwt_snapshot_list: Snapshot of the @gwt_current_list for sending to user space. + * @priority: Indicates the context priority. Used along with @atoms_count + * for context scheduling, protected by hwaccess_lock. + * @atoms_count: Number of GPU atoms currently in use, per priority + * @create_flags: Flags used in context creation. + * @kinstr_jm: Kernel job manager instrumentation context handle + * + * A kernel base context is an entity among which the GPU is scheduled. + * Each context has its own GPU address space. + * Up to one context can be created for each client that opens the device file + * /dev/malixx. Context creation is deferred until a special ioctl() system call + * is made on the device file. + */ +struct kbase_context { + struct file *filp; + struct kbase_device *kbdev; + struct list_head kctx_list_link; + struct kbase_mmu_table mmu; + + u32 id; + unsigned long api_version; + struct list_head event_list; + struct list_head event_coalesce_list; + struct mutex event_mutex; +#if !MALI_USE_CSF + atomic_t event_closed; +#endif + struct workqueue_struct *event_workq; + atomic_t event_count; + int event_coalesce_count; + + atomic_t flags; + + struct tagged_addr aliasing_sink_page; + + spinlock_t mem_partials_lock; + struct list_head mem_partials; + + struct mutex reg_lock; + + struct rb_root reg_rbtree_same; + struct rb_root reg_rbtree_custom; + struct rb_root reg_rbtree_exec; + +#if MALI_USE_CSF + struct kbase_csf_context csf; +#else + struct kbase_jd_context jctx; + struct jsctx_queue jsctx_queue + [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS]; + + struct list_head completed_jobs; + atomic_t work_count; + struct timer_list soft_job_timeout; + + atomic_t atoms_pulled; + atomic_t atoms_pulled_slot[BASE_JM_MAX_NR_SLOTS]; + int atoms_pulled_slot_pri[BASE_JM_MAX_NR_SLOTS][ + KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + int priority; + bool blocked_js[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + s16 atoms_count[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + u32 slots_pullable; + u32 age_count; +#endif /* MALI_USE_CSF */ + + DECLARE_BITMAP(cookies, BITS_PER_LONG); + struct kbase_va_region *pending_regions[BITS_PER_LONG]; + + wait_queue_head_t event_queue; + pid_t tgid; + pid_t pid; + atomic_t used_pages; + atomic_t nonmapped_pages; + atomic_t permanent_mapped_pages; + + struct kbase_mem_pool_group mem_pools; + + struct shrinker reclaim; + struct list_head evict_list; + + struct list_head waiting_soft_jobs; + spinlock_t waiting_soft_jobs_lock; +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + struct { + struct list_head waiting_resource; + struct workqueue_struct *wq; + } dma_fence; +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + int as_nr; + + atomic_t refcount; + + spinlock_t mm_update_lock; + struct mm_struct __rcu *process_mm; + u64 same_va_end; + u64 exec_va_start; + u64 gpu_va_end; + bool jit_va; + +#ifdef CONFIG_DEBUG_FS + char *mem_profile_data; + size_t mem_profile_size; + struct mutex mem_profile_lock; + struct dentry *kctx_dentry; + + unsigned int *reg_dump; + atomic_t job_fault_count; + struct list_head job_fault_resume_event_list; + +#endif /* CONFIG_DEBUG_FS */ + + struct kbase_hwcnt_legacy_client *legacy_hwcnt_cli; + struct mutex legacy_hwcnt_lock; + + struct kbase_va_region *jit_alloc[1 + BASE_JIT_ALLOC_COUNT]; + u8 jit_max_allocations; + u8 jit_current_allocations; + u8 jit_current_allocations_per_bin[256]; + u8 jit_version; + u8 jit_group_id; +#if MALI_JIT_PRESSURE_LIMIT_BASE + u64 jit_phys_pages_limit; + u64 jit_current_phys_pressure; + u64 jit_phys_pages_to_be_allocated; +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + struct list_head jit_active_head; + struct list_head jit_pool_head; + struct list_head jit_destroy_head; + struct mutex jit_evict_lock; + struct work_struct jit_work; + + struct list_head ext_res_meta_head; + + u8 trim_level; + + struct kbase_process *kprcs; + struct list_head kprcs_link; + +#ifdef CONFIG_MALI_CINSTR_GWT + bool gwt_enabled; + bool gwt_was_enabled; + struct list_head gwt_current_list; + struct list_head gwt_snapshot_list; +#endif + + base_context_create_flags create_flags; + +#if !MALI_USE_CSF + struct kbase_kinstr_jm *kinstr_jm; +#endif +}; + +#ifdef CONFIG_MALI_CINSTR_GWT +/** + * struct kbasep_gwt_list_element - Structure used to collect GPU + * write faults. + * @link: List head for adding write faults. + * @region: Details of the region where we have the + * faulting page address. + * @page_addr: Page address where GPU write fault occurred. + * @num_pages: The number of pages modified. + * + * Using this structure all GPU write faults are stored in a list. + */ +struct kbasep_gwt_list_element { + struct list_head link; + struct kbase_va_region *region; + u64 page_addr; + u64 num_pages; +}; + +#endif + +/** + * struct kbase_ctx_ext_res_meta - Structure which binds an external resource + * to a @kbase_context. + * @ext_res_node: List head for adding the metadata to a + * @kbase_context. + * @alloc: The physical memory allocation structure + * which is mapped. + * @gpu_addr: The GPU virtual address the resource is + * mapped to. + * @ref: Reference count. + * + * External resources can be mapped into multiple contexts as well as the same + * context multiple times. + * As kbase_va_region itself isn't refcounted we can't attach our extra + * information to it as it could be removed under our feet leaving external + * resources pinned. + * This metadata structure binds a single external resource to a single + * context, ensuring that per context mapping is tracked separately so it can + * be overridden when needed and abuses by the application (freeing the resource + * multiple times) don't effect the refcount of the physical allocation. + */ +struct kbase_ctx_ext_res_meta { + struct list_head ext_res_node; + struct kbase_mem_phy_alloc *alloc; + u64 gpu_addr; + u32 ref; +}; + +enum kbase_reg_access_type { + REG_READ, + REG_WRITE +}; + +enum kbase_share_attr_bits { + /* (1ULL << 8) bit is reserved */ + SHARE_BOTH_BITS = (2ULL << 8), /* inner and outer shareable coherency */ + SHARE_INNER_BITS = (3ULL << 8) /* inner shareable coherency */ +}; + +/** + * kbase_device_is_cpu_coherent - Returns if the device is CPU coherent. + * @kbdev: kbase device + * + * Return: true if the device access are coherent, false if not. + */ +static inline bool kbase_device_is_cpu_coherent(struct kbase_device *kbdev) +{ + if ((kbdev->system_coherency == COHERENCY_ACE_LITE) || + (kbdev->system_coherency == COHERENCY_ACE)) + return true; + + return false; +} + +/* Conversion helpers for setting up high resolution timers */ +#define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x))*1000000U)) +#define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x)) + +/* Maximum number of loops polling the GPU for a cache flush before we assume it must have completed */ +#define KBASE_CLEAN_CACHE_MAX_LOOPS 100000 +/* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */ +#define KBASE_AS_INACTIVE_MAX_LOOPS 100000000 + +/* JobDescriptorHeader - taken from the architecture specifications, the layout + * is currently identical for all GPU archs. */ +struct job_descriptor_header { + u32 exception_status; + u32 first_incomplete_task; + u64 fault_pointer; + u8 job_descriptor_size : 1; + u8 job_type : 7; + u8 job_barrier : 1; + u8 _reserved_01 : 1; + u8 _reserved_1 : 1; + u8 _reserved_02 : 1; + u8 _reserved_03 : 1; + u8 _reserved_2 : 1; + u8 _reserved_04 : 1; + u8 _reserved_05 : 1; + u16 job_index; + u16 job_dependency_index_1; + u16 job_dependency_index_2; + union { + u64 _64; + u32 _32; + } next_job; +}; + +#endif /* _KBASE_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c b/drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c new file mode 100755 index 000000000000..b5ac414b1223 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c @@ -0,0 +1,81 @@ +/* + * + * (C) COPYRIGHT 2014, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Base kernel disjoint events helper functions + */ + +#include + +void kbase_disjoint_init(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + atomic_set(&kbdev->disjoint_event.count, 0); + atomic_set(&kbdev->disjoint_event.state, 0); +} + +/* increment the disjoint event count */ +void kbase_disjoint_event(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + atomic_inc(&kbdev->disjoint_event.count); +} + +/* increment the state and the event counter */ +void kbase_disjoint_state_up(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + atomic_inc(&kbdev->disjoint_event.state); + + kbase_disjoint_event(kbdev); +} + +/* decrement the state */ +void kbase_disjoint_state_down(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(atomic_read(&kbdev->disjoint_event.state) > 0); + + kbase_disjoint_event(kbdev); + + atomic_dec(&kbdev->disjoint_event.state); +} + +/* increments the count only if the state is > 0 */ +void kbase_disjoint_event_potential(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + if (atomic_read(&kbdev->disjoint_event.state)) + kbase_disjoint_event(kbdev); +} + +u32 kbase_disjoint_event_get(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + return atomic_read(&kbdev->disjoint_event.count); +} +KBASE_EXPORT_TEST_API(kbase_disjoint_event_get); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.c b/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.c new file mode 100755 index 000000000000..1fac5e3e68f1 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.c @@ -0,0 +1,473 @@ +/* + * + * (C) COPYRIGHT 2011-2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* Include mali_kbase_dma_fence.h before checking for CONFIG_MALI_BIFROST_DMA_FENCE as + * it will be set there. + */ +#include "mali_kbase_dma_fence.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void +kbase_dma_fence_work(struct work_struct *pwork); + +static void +kbase_dma_fence_waiters_add(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + list_add_tail(&katom->queue, &kctx->dma_fence.waiting_resource); +} + +static void +kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom) +{ + list_del(&katom->queue); +} + +static int +kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info, + struct ww_acquire_ctx *ctx) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) + struct reservation_object *content_res = NULL; +#else + struct dma_resv *content_res = NULL; +#endif + unsigned int content_res_idx = 0; + unsigned int r; + int err = 0; + + ww_acquire_init(ctx, &reservation_ww_class); + +retry: + for (r = 0; r < info->dma_fence_resv_count; r++) { + if (info->resv_objs[r] == content_res) { + content_res = NULL; + continue; + } + + err = ww_mutex_lock(&info->resv_objs[r]->lock, ctx); + if (err) + goto error; + } + + ww_acquire_done(ctx); + return err; + +error: + content_res_idx = r; + + /* Unlock the locked one ones */ + while (r--) + ww_mutex_unlock(&info->resv_objs[r]->lock); + + if (content_res) + ww_mutex_unlock(&content_res->lock); + + /* If we deadlock try with lock_slow and retry */ + if (err == -EDEADLK) { + content_res = info->resv_objs[content_res_idx]; + ww_mutex_lock_slow(&content_res->lock, ctx); + goto retry; + } + + /* If we are here the function failed */ + ww_acquire_fini(ctx); + return err; +} + +static void +kbase_dma_fence_unlock_reservations(struct kbase_dma_fence_resv_info *info, + struct ww_acquire_ctx *ctx) +{ + unsigned int r; + + for (r = 0; r < info->dma_fence_resv_count; r++) + ww_mutex_unlock(&info->resv_objs[r]->lock); + ww_acquire_fini(ctx); +} + + + +/** + * kbase_dma_fence_queue_work() - Queue work to handle @katom + * @katom: Pointer to atom for which to queue work + * + * Queue kbase_dma_fence_work() for @katom to clean up the fence callbacks and + * submit the atom. + */ +static void +kbase_dma_fence_queue_work(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + bool ret; + + INIT_WORK(&katom->work, kbase_dma_fence_work); + ret = queue_work(kctx->dma_fence.wq, &katom->work); + /* Warn if work was already queued, that should not happen. */ + WARN_ON(!ret); +} + +/** + * kbase_dma_fence_cancel_atom() - Cancels waiting on an atom + * @katom: Katom to cancel + * + * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. + */ +static void +kbase_dma_fence_cancel_atom(struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&katom->kctx->jctx.lock); + + /* Cancel callbacks and clean up. */ + kbase_fence_free_callbacks(katom); + + /* Mark the atom as handled in case all fences signaled just before + * canceling the callbacks and the worker was queued. + */ + kbase_fence_dep_count_set(katom, -1); + + /* Prevent job_done_nolock from being called twice on an atom when + * there is a race between job completion and cancellation. + */ + + if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { + /* Wait was cancelled - zap the atom */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); + } +} + +/** + * kbase_dma_fence_work() - Worker thread called when a fence is signaled + * @pwork: work_struct containing a pointer to a katom + * + * This function will clean and mark all dependencies as satisfied + */ +static void +kbase_dma_fence_work(struct work_struct *pwork) +{ + struct kbase_jd_atom *katom; + struct kbase_jd_context *ctx; + + katom = container_of(pwork, struct kbase_jd_atom, work); + ctx = &katom->kctx->jctx; + + mutex_lock(&ctx->lock); + if (kbase_fence_dep_count_read(katom) != 0) + goto out; + + kbase_fence_dep_count_set(katom, -1); + + /* Remove atom from list of dma-fence waiting atoms. */ + kbase_dma_fence_waiters_remove(katom); + /* Cleanup callbacks. */ + kbase_fence_free_callbacks(katom); + /* + * Queue atom on GPU, unless it has already completed due to a failing + * dependency. Run jd_done_nolock() on the katom if it is completed. + */ + if (unlikely(katom->status == KBASE_JD_ATOM_STATE_COMPLETED)) + jd_done_nolock(katom, NULL); + else + kbase_jd_dep_clear_locked(katom); + +out: + mutex_unlock(&ctx->lock); +} + +static void +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) +#else +kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) +#endif +{ + struct kbase_fence_cb *kcb = container_of(cb, + struct kbase_fence_cb, + fence_cb); + struct kbase_jd_atom *katom = kcb->katom; + + /* If the atom is zapped dep_count will be forced to a negative number + * preventing this callback from ever scheduling work. Which in turn + * would reschedule the atom. + */ + + if (kbase_fence_dep_count_dec_and_test(katom)) + kbase_dma_fence_queue_work(katom); +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +static int +kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, + struct reservation_object *resv, + bool exclusive) +#else +static int +kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, + struct dma_resv *resv, + bool exclusive) +#endif +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *excl_fence = NULL; + struct fence **shared_fences = NULL; +#else + struct dma_fence *excl_fence = NULL; + struct dma_fence **shared_fences = NULL; +#endif + unsigned int shared_count = 0; + int err, i; + + err = reservation_object_get_fences_rcu(resv, + &excl_fence, + &shared_count, + &shared_fences); + if (err) + return err; + + if (excl_fence) { + err = kbase_fence_add_callback(katom, + excl_fence, + kbase_dma_fence_cb); + + /* Release our reference, taken by reservation_object_get_fences_rcu(), + * to the fence. We have set up our callback (if that was possible), + * and it's the fence's owner is responsible for singling the fence + * before allowing it to disappear. + */ + dma_fence_put(excl_fence); + + if (err) + goto out; + } + + if (exclusive) { + for (i = 0; i < shared_count; i++) { + err = kbase_fence_add_callback(katom, + shared_fences[i], + kbase_dma_fence_cb); + if (err) + goto out; + } + } + + /* Release all our references to the shared fences, taken by + * reservation_object_get_fences_rcu(). We have set up our callback (if + * that was possible), and it's the fence's owner is responsible for + * signaling the fence before allowing it to disappear. + */ +out: + for (i = 0; i < shared_count; i++) + dma_fence_put(shared_fences[i]); + kfree(shared_fences); + + if (err) { + /* + * On error, cancel and clean up all callbacks that was set up + * before the error. + */ + kbase_fence_free_callbacks(katom); + } + + return err; +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +void kbase_dma_fence_add_reservation(struct reservation_object *resv, + struct kbase_dma_fence_resv_info *info, + bool exclusive) +#else +void kbase_dma_fence_add_reservation(struct dma_resv *resv, + struct kbase_dma_fence_resv_info *info, + bool exclusive) +#endif +{ + unsigned int i; + + for (i = 0; i < info->dma_fence_resv_count; i++) { + /* Duplicate resource, ignore */ + if (info->resv_objs[i] == resv) + return; + } + + info->resv_objs[info->dma_fence_resv_count] = resv; + if (exclusive) + set_bit(info->dma_fence_resv_count, + info->dma_fence_excl_bitmap); + (info->dma_fence_resv_count)++; +} + +int kbase_dma_fence_wait(struct kbase_jd_atom *katom, + struct kbase_dma_fence_resv_info *info) +{ + int err, i; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + struct ww_acquire_ctx ww_ctx; + + lockdep_assert_held(&katom->kctx->jctx.lock); + + fence = kbase_fence_out_new(katom); + if (!fence) { + err = -ENOMEM; + dev_err(katom->kctx->kbdev->dev, + "Error %d creating fence.\n", err); + return err; + } + + kbase_fence_dep_count_set(katom, 1); + + err = kbase_dma_fence_lock_reservations(info, &ww_ctx); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d locking reservations.\n", err); + kbase_fence_dep_count_set(katom, -1); + kbase_fence_out_remove(katom); + return err; + } + + for (i = 0; i < info->dma_fence_resv_count; i++) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) + struct reservation_object *obj = info->resv_objs[i]; +#else + struct dma_resv *obj = info->resv_objs[i]; +#endif + if (!test_bit(i, info->dma_fence_excl_bitmap)) { + err = reservation_object_reserve_shared(obj); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d reserving space for shared fence.\n", err); + goto end; + } + + err = kbase_dma_fence_add_reservation_callback(katom, obj, false); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d adding reservation to callback.\n", err); + goto end; + } + + reservation_object_add_shared_fence(obj, fence); + } else { + err = kbase_dma_fence_add_reservation_callback(katom, obj, true); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d adding reservation to callback.\n", err); + goto end; + } + + reservation_object_add_excl_fence(obj, fence); + } + } + +end: + kbase_dma_fence_unlock_reservations(info, &ww_ctx); + + if (likely(!err)) { + /* Test if the callbacks are already triggered */ + if (kbase_fence_dep_count_dec_and_test(katom)) { + kbase_fence_dep_count_set(katom, -1); + kbase_fence_free_callbacks(katom); + } else { + /* Add katom to the list of dma-buf fence waiting atoms + * only if it is still waiting. + */ + kbase_dma_fence_waiters_add(katom); + } + } else { + /* There was an error, cancel callbacks, set dep_count to -1 to + * indicate that the atom has been handled (the caller will + * kill it for us), signal the fence, free callbacks and the + * fence. + */ + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + kbase_dma_fence_signal(katom); + } + + return err; +} + +void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx) +{ + struct list_head *list = &kctx->dma_fence.waiting_resource; + + while (!list_empty(list)) { + struct kbase_jd_atom *katom; + + katom = list_first_entry(list, struct kbase_jd_atom, queue); + kbase_dma_fence_waiters_remove(katom); + kbase_dma_fence_cancel_atom(katom); + } +} + +void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom) +{ + /* Cancel callbacks and clean up. */ + if (kbase_fence_free_callbacks(katom)) + kbase_dma_fence_queue_work(katom); +} + +void kbase_dma_fence_signal(struct kbase_jd_atom *katom) +{ + if (!katom->dma_fence.fence) + return; + + /* Signal the atom's fence. */ + dma_fence_signal(katom->dma_fence.fence); + + kbase_fence_out_remove(katom); + + kbase_fence_free_callbacks(katom); +} + +void kbase_dma_fence_term(struct kbase_context *kctx) +{ + destroy_workqueue(kctx->dma_fence.wq); + kctx->dma_fence.wq = NULL; +} + +int kbase_dma_fence_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->dma_fence.waiting_resource); + + kctx->dma_fence.wq = alloc_workqueue("mali-fence-%d", + WQ_UNBOUND, 1, kctx->pid); + if (!kctx->dma_fence.wq) + return -ENOMEM; + + return 0; +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.h b/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.h new file mode 100755 index 000000000000..3ac8186328a1 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_dma_fence.h @@ -0,0 +1,144 @@ +/* + * + * (C) COPYRIGHT 2010-2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DMA_FENCE_H_ +#define _KBASE_DMA_FENCE_H_ + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + +#include +#include +#include + +/* Forward declaration from mali_kbase_defs.h */ +struct kbase_jd_atom; +struct kbase_context; + +/** + * struct kbase_dma_fence_resv_info - Structure with list of reservation objects + * @resv_objs: Array of reservation objects to attach the + * new fence to. + * @dma_fence_resv_count: Number of reservation objects in the array. + * @dma_fence_excl_bitmap: Specifies which resv_obj are exclusive. + * + * This is used by some functions to pass around a collection of data about + * reservation objects. + */ +struct kbase_dma_fence_resv_info { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) + struct reservation_object **resv_objs; +#else + struct dma_resv **resv_objs; +#endif + unsigned int dma_fence_resv_count; + unsigned long *dma_fence_excl_bitmap; +}; + +/** + * kbase_dma_fence_add_reservation() - Adds a resv to the array of resv_objs + * @resv: Reservation object to add to the array. + * @info: Pointer to struct with current reservation info + * @exclusive: Boolean indicating if exclusive access is needed + * + * The function adds a new reservation_object to an existing array of + * reservation_objects. At the same time keeps track of which objects require + * exclusive access in dma_fence_excl_bitmap. + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) +void kbase_dma_fence_add_reservation(struct reservation_object *resv, + struct kbase_dma_fence_resv_info *info, + bool exclusive); +#else +void kbase_dma_fence_add_reservation(struct dma_resv *resv, + struct kbase_dma_fence_resv_info *info, + bool exclusive); +#endif + +/** + * kbase_dma_fence_wait() - Creates a new fence and attaches it to the resv_objs + * @katom: Katom with the external dependency. + * @info: Pointer to struct with current reservation info + * + * Return: An error code or 0 if succeeds + */ +int kbase_dma_fence_wait(struct kbase_jd_atom *katom, + struct kbase_dma_fence_resv_info *info); + +/** + * kbase_dma_fence_cancel_ctx() - Cancel all dma-fences blocked atoms on kctx + * @kctx: Pointer to kbase context + * + * This function will cancel and clean up all katoms on @kctx that is waiting + * on dma-buf fences. + * + * Locking: jctx.lock needs to be held when calling this function. + */ +void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx); + +/** + * kbase_dma_fence_cancel_callbacks() - Cancel only callbacks on katom + * @katom: Pointer to katom whose callbacks are to be canceled + * + * This function cancels all dma-buf fence callbacks on @katom, but does not + * cancel the katom itself. + * + * The caller is responsible for ensuring that jd_done_nolock is called on + * @katom. + * + * Locking: jctx.lock must be held when calling this function. + */ +void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom); + +/** + * kbase_dma_fence_signal() - Signal katom's fence and clean up after wait + * @katom: Pointer to katom to signal and clean up + * + * This function will signal the @katom's fence, if it has one, and clean up + * the callback data from the katom's wait on earlier fences. + * + * Locking: jctx.lock must be held while calling this function. + */ +void kbase_dma_fence_signal(struct kbase_jd_atom *katom); + +/** + * kbase_dma_fence_term() - Terminate Mali dma-fence context + * @kctx: kbase context to terminate + */ +void kbase_dma_fence_term(struct kbase_context *kctx); + +/** + * kbase_dma_fence_init() - Initialize Mali dma-fence context + * @kctx: kbase context to initialize + */ +int kbase_dma_fence_init(struct kbase_context *kctx); + +#else /* CONFIG_MALI_BIFROST_DMA_FENCE */ +/* Dummy functions for when dma-buf fence isn't enabled. */ + +static inline int kbase_dma_fence_init(struct kbase_context *kctx) +{ + return 0; +} + +static inline void kbase_dma_fence_term(struct kbase_context *kctx) {} +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ +#endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c new file mode 100755 index 000000000000..a5a7ad744a8e --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c @@ -0,0 +1,442 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Implementation of the dummy job execution workaround for the GPU hang issue. + */ + +#include +#include +#include + +#include +#include + +#define DUMMY_JOB_WA_BINARY_NAME "valhall-1691526.wa" + +struct wa_header { + u16 signature; + u16 version; + u32 info_offset; +} __packed; + +struct wa_v2_info { + u64 jc; + u32 js; + u32 blob_offset; + u64 flags; +} __packed; + +struct wa_blob { + u64 base; + u32 size; + u32 map_flags; + u32 payload_offset; + u32 blob_offset; +} __packed; + +static bool in_range(const u8 *base, const u8 *end, off_t off, size_t sz) +{ + return !(end - base - off < sz); +} + +static u32 wait_any(struct kbase_device *kbdev, off_t offset, u32 bits) +{ + int loop; + const int timeout = 100; + u32 val; + + for (loop = 0; loop < timeout; loop++) { + val = kbase_reg_read(kbdev, offset); + if (val & bits) + break; + udelay(10); + } + + if (loop == timeout) { + dev_err(kbdev->dev, + "Timeout reading register 0x%lx, bits 0x%lx, last read was 0x%lx\n", + (unsigned long)offset, (unsigned long)bits, + (unsigned long)val); + } + + return (val & bits); +} + +static int wait(struct kbase_device *kbdev, off_t offset, u32 bits, bool set) +{ + int loop; + const int timeout = 100; + u32 val; + u32 target = 0; + + if (set) + target = bits; + + for (loop = 0; loop < timeout; loop++) { + val = kbase_reg_read(kbdev, (offset)); + if ((val & bits) == target) + break; + + udelay(10); + } + + if (loop == timeout) { + dev_err(kbdev->dev, + "Timeout reading register 0x%lx, bits 0x%lx, last read was 0x%lx\n", + (unsigned long)offset, (unsigned long)bits, + (unsigned long)val); + return -ETIMEDOUT; + } + + return 0; +} + +static inline int run_job(struct kbase_device *kbdev, int as, int slot, + u64 cores, u64 jc) +{ + u32 done; + + /* setup job */ + kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_HEAD_NEXT_LO), + jc & U32_MAX); + kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_HEAD_NEXT_HI), + jc >> 32); + kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_AFFINITY_NEXT_LO), + cores & U32_MAX); + kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_AFFINITY_NEXT_HI), + cores >> 32); + kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_CONFIG_NEXT), + JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK | as); + + /* go */ + kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_COMMAND_NEXT), + JS_COMMAND_START); + + /* wait for the slot to finish (done, error) */ + done = wait_any(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), + (1ul << (16+slot)) | (1ul << slot)); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), done); + + if (done != (1ul << slot)) { + dev_err(kbdev->dev, + "Failed to run WA job on slot %d cores 0x%llx: done 0x%lx\n", + slot, (unsigned long long)cores, + (unsigned long)done); + dev_err(kbdev->dev, "JS_STATUS on failure: 0x%x\n", + kbase_reg_read(kbdev, JOB_SLOT_REG(slot, JS_STATUS))); + + return -EFAULT; + } else { + return 0; + } +} + +/* To be called after power up & MMU init, but before everything else */ +int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores) +{ + int as; + int slot; + u64 jc; + int failed = 0; + int runs = 0; + u32 old_gpu_mask; + u32 old_job_mask; + + if (!kbdev) + return -EFAULT; + + if (!kbdev->dummy_job_wa.ctx) + return -EFAULT; + + as = kbdev->dummy_job_wa.ctx->as_nr; + slot = kbdev->dummy_job_wa.slot; + jc = kbdev->dummy_job_wa.jc; + + /* mask off all but MMU IRQs */ + old_gpu_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); + old_job_mask = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK)); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0); + + /* power up requested cores */ + kbase_reg_write(kbdev, SHADER_PWRON_LO, (cores & U32_MAX)); + kbase_reg_write(kbdev, SHADER_PWRON_HI, (cores >> 32)); + + if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP) { + /* wait for power-ups */ + wait(kbdev, SHADER_READY_LO, (cores & U32_MAX), true); + if (cores >> 32) + wait(kbdev, SHADER_READY_HI, (cores >> 32), true); + } + + if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE) { + int i; + + /* do for each requested core */ + for (i = 0; i < sizeof(cores) * 8; i++) { + u64 affinity; + + affinity = 1ull << i; + + if (!(cores & affinity)) + continue; + + if (run_job(kbdev, as, slot, affinity, jc)) + failed++; + runs++; + } + + } else { + if (run_job(kbdev, as, slot, cores, jc)) + failed++; + runs++; + } + + if (kbdev->dummy_job_wa.flags & + KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) { + /* power off shader cores (to reduce any dynamic leakage) */ + kbase_reg_write(kbdev, SHADER_PWROFF_LO, (cores & U32_MAX)); + kbase_reg_write(kbdev, SHADER_PWROFF_HI, (cores >> 32)); + + /* wait for power off complete */ + wait(kbdev, SHADER_READY_LO, (cores & U32_MAX), false); + wait(kbdev, SHADER_PWRTRANS_LO, (cores & U32_MAX), false); + if (cores >> 32) { + wait(kbdev, SHADER_READY_HI, (cores >> 32), false); + wait(kbdev, SHADER_PWRTRANS_HI, (cores >> 32), false); + } + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), U32_MAX); + } + + /* restore IRQ masks */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), old_gpu_mask); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), old_job_mask); + + if (failed) + dev_err(kbdev->dev, + "WA complete with %d failures out of %d runs\n", failed, + runs); + + return failed ? -EFAULT : 0; +} + +static ssize_t show_dummy_job_wa_info(struct device * const dev, + struct device_attribute * const attr, char * const buf) +{ + struct kbase_device *const kbdev = dev_get_drvdata(dev); + int err; + + if (!kbdev || !kbdev->dummy_job_wa.ctx) + return -ENODEV; + + err = scnprintf(buf, PAGE_SIZE, "slot %u flags %llx\n", + kbdev->dummy_job_wa.slot, kbdev->dummy_job_wa.flags); + + return err; +} + +static DEVICE_ATTR(dummy_job_wa_info, 0444, show_dummy_job_wa_info, NULL); + +static bool wa_blob_load_needed(struct kbase_device *kbdev) +{ + if (of_machine_is_compatible("arm,juno")) + return false; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3485)) + return true; + + return false; +} + +int kbase_dummy_job_wa_load(struct kbase_device *kbdev) +{ + const struct firmware *firmware; + static const char wa_name[] = DUMMY_JOB_WA_BINARY_NAME; + const u32 signature = 0x4157; + const u32 version = 2; + const u8 *fw_end; + const u8 *fw; + const struct wa_header *header; + const struct wa_v2_info *v2_info; + u32 blob_offset; + int err; + struct kbase_context *kctx; + + if (!wa_blob_load_needed(kbdev)) + return 0; + + /* load the wa */ + err = request_firmware(&firmware, wa_name, kbdev->dev); + + if (err) { + dev_err(kbdev->dev, "WA blob missing. Please refer to the Arm Mali DDK Valhall Release Notes, " + "Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed"); + return -ENODEV; + } + + kctx = kbase_create_context(kbdev, true, + BASE_CONTEXT_CREATE_FLAG_NONE, 0, + NULL); + + if (!kctx) { + dev_err(kbdev->dev, "Failed to create WA context\n"); + goto no_ctx; + } + + fw = firmware->data; + fw_end = fw + firmware->size; + + dev_dbg(kbdev->dev, "Loaded firmware of size %zu bytes\n", + firmware->size); + + if (!in_range(fw, fw_end, 0, sizeof(*header))) { + dev_err(kbdev->dev, "WA too small\n"); + goto bad_fw; + } + + header = (const struct wa_header *)(fw + 0); + + if (header->signature != signature) { + dev_err(kbdev->dev, "WA signature failure: 0x%lx\n", + (unsigned long)header->signature); + goto bad_fw; + } + + if (header->version != version) { + dev_err(kbdev->dev, "WA version 0x%lx not supported\n", + (unsigned long)header->version); + goto bad_fw; + } + + if (!in_range(fw, fw_end, header->info_offset, sizeof(*v2_info))) { + dev_err(kbdev->dev, "WA info offset out of bounds\n"); + goto bad_fw; + } + + v2_info = (const struct wa_v2_info *)(fw + header->info_offset); + + if (v2_info->flags & ~KBASE_DUMMY_JOB_WA_FLAGS) { + dev_err(kbdev->dev, "Unsupported WA flag(s): 0x%llx\n", + (unsigned long long)v2_info->flags); + goto bad_fw; + } + + kbdev->dummy_job_wa.slot = v2_info->js; + kbdev->dummy_job_wa.jc = v2_info->jc; + kbdev->dummy_job_wa.flags = v2_info->flags; + + blob_offset = v2_info->blob_offset; + + while (blob_offset) { + const struct wa_blob *blob; + size_t nr_pages; + u64 flags; + u64 gpu_va; + struct kbase_va_region *va_region; + + if (!in_range(fw, fw_end, blob_offset, sizeof(*blob))) { + dev_err(kbdev->dev, "Blob offset out-of-range: 0x%lx\n", + (unsigned long)blob_offset); + goto bad_fw; + } + + blob = (const struct wa_blob *)(fw + blob_offset); + if (!in_range(fw, fw_end, blob->payload_offset, blob->size)) { + dev_err(kbdev->dev, "Payload out-of-bounds\n"); + goto bad_fw; + } + + gpu_va = blob->base; + if (PAGE_ALIGN(gpu_va) != gpu_va) { + dev_err(kbdev->dev, "blob not page aligned\n"); + goto bad_fw; + } + nr_pages = PFN_UP(blob->size); + flags = blob->map_flags | BASE_MEM_FLAG_MAP_FIXED; + + va_region = kbase_mem_alloc(kctx, nr_pages, nr_pages, + 0, &flags, &gpu_va); + + if (!va_region) { + dev_err(kbdev->dev, "Failed to allocate for blob\n"); + } else { + struct kbase_vmap_struct vmap = { 0 }; + const u8 *payload; + void *dst; + + /* copy the payload, */ + payload = fw + blob->payload_offset; + + dst = kbase_vmap(kctx, + va_region->start_pfn << PAGE_SHIFT, + nr_pages << PAGE_SHIFT, &vmap); + + if (dst) { + memcpy(dst, payload, blob->size); + kbase_vunmap(kctx, &vmap); + } else { + dev_err(kbdev->dev, + "Failed to copy payload\n"); + } + + } + blob_offset = blob->blob_offset; /* follow chain */ + } + + release_firmware(firmware); + + kbasep_js_schedule_privileged_ctx(kbdev, kctx); + + kbdev->dummy_job_wa.ctx = kctx; + + err = sysfs_create_file(&kbdev->dev->kobj, + &dev_attr_dummy_job_wa_info.attr); + if (err) + dev_err(kbdev->dev, "SysFS file creation for dummy job wa failed\n"); + + return 0; + +bad_fw: + kbase_destroy_context(kctx); +no_ctx: + release_firmware(firmware); + return -EFAULT; +} + +void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) +{ + struct kbase_context *wa_ctx; + + /* Can be safely called even if the file wasn't created on probe */ + sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_dummy_job_wa_info.attr); + + wa_ctx = READ_ONCE(kbdev->dummy_job_wa.ctx); + WRITE_ONCE(kbdev->dummy_job_wa.ctx, NULL); + /* make this write visible before we tear down the ctx */ + smp_mb(); + + if (wa_ctx) { + kbasep_js_release_privileged_ctx(kbdev, wa_ctx); + kbase_destroy_context(wa_ctx); + } +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.h b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.h new file mode 100755 index 000000000000..e19495055b48 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.h @@ -0,0 +1,74 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_DUMMY_JOB_WORKAROUND_ +#define _KBASE_DUMMY_JOB_WORKAROUND_ + +#define KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE (1ull << 0) +#define KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP (1ull << 1) +#define KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER (1ull << 2) + +#define KBASE_DUMMY_JOB_WA_FLAGS (KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE | \ + KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP | \ + KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) + +#if MALI_USE_CSF + +static inline int kbase_dummy_job_wa_load(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); + return 0; +} + +static inline void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static inline int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, + u64 cores) +{ + CSTD_UNUSED(kbdev); + CSTD_UNUSED(cores); + return 0; +} + +static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); + return false; +} + +#else + +int kbase_dummy_job_wa_load(struct kbase_device *kbdev); +void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev); +int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores); + +static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev) +{ + return (kbdev->dummy_job_wa.ctx != NULL); +} + +#endif /* MALI_USE_CSF */ + +#endif /* _KBASE_DUMMY_JOB_WORKAROUND_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_event.c b/drivers/gpu/arm/bifrost/mali_kbase_event.c new file mode 100755 index 000000000000..5adb80f9bbd2 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_event.c @@ -0,0 +1,274 @@ +/* + * + * (C) COPYRIGHT 2010-2016,2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#include +#include +#include +#include + +static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + struct base_jd_udata data; + struct kbase_device *kbdev; + + lockdep_assert_held(&kctx->jctx.lock); + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(katom != NULL); + KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); + + kbdev = kctx->kbdev; + data = katom->udata; + + KBASE_TLSTREAM_TL_NRET_ATOM_CTX(kbdev, katom, kctx); + KBASE_TLSTREAM_TL_DEL_ATOM(kbdev, katom); + + katom->status = KBASE_JD_ATOM_STATE_UNUSED; + dev_dbg(kbdev->dev, "Atom %p status to unused\n", (void *)katom); + wake_up(&katom->completed); + + return data; +} + +int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent) +{ + struct kbase_jd_atom *atom; + + KBASE_DEBUG_ASSERT(ctx); + + mutex_lock(&ctx->event_mutex); + + if (list_empty(&ctx->event_list)) { + if (!atomic_read(&ctx->event_closed)) { + mutex_unlock(&ctx->event_mutex); + return -1; + } + + /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */ + mutex_unlock(&ctx->event_mutex); + uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED; + memset(&uevent->udata, 0, sizeof(uevent->udata)); + dev_dbg(ctx->kbdev->dev, + "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n", + BASE_JD_EVENT_DRV_TERMINATED); + return 0; + } + + /* normal event processing */ + atomic_dec(&ctx->event_count); + atom = list_entry(ctx->event_list.next, struct kbase_jd_atom, dep_item[0]); + list_del(ctx->event_list.next); + + mutex_unlock(&ctx->event_mutex); + + dev_dbg(ctx->kbdev->dev, "event dequeuing %p\n", (void *)atom); + uevent->event_code = atom->event_code; + + uevent->atom_number = (atom - ctx->jctx.atoms); + + if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) + kbase_jd_free_external_resources(atom); + + mutex_lock(&ctx->jctx.lock); + uevent->udata = kbase_event_process(ctx, atom); + mutex_unlock(&ctx->jctx.lock); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_event_dequeue); + +/** + * kbase_event_process_noreport_worker - Worker for processing atoms that do not + * return an event but do have external + * resources + * @data: Work structure + */ +static void kbase_event_process_noreport_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) + kbase_jd_free_external_resources(katom); + + mutex_lock(&kctx->jctx.lock); + kbase_event_process(kctx, katom); + mutex_unlock(&kctx->jctx.lock); +} + +/** + * kbase_event_process_noreport - Process atoms that do not return an event + * @kctx: Context pointer + * @katom: Atom to be processed + * + * Atoms that do not have external resources will be processed immediately. + * Atoms that do have external resources will be processed on a workqueue, in + * order to avoid locking issues. + */ +static void kbase_event_process_noreport(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + INIT_WORK(&katom->work, kbase_event_process_noreport_worker); + queue_work(kctx->event_workq, &katom->work); + } else { + kbase_event_process(kctx, katom); + } +} + +/** + * kbase_event_coalesce - Move pending events to the main event list + * @kctx: Context pointer + * + * kctx->event_list and kctx->event_coalesce_count must be protected + * by a lock unless this is the last thread using them + * (and we're about to terminate the lock). + * + * Return: The number of pending events moved to the main event list + */ +static int kbase_event_coalesce(struct kbase_context *kctx) +{ + const int event_count = kctx->event_coalesce_count; + + /* Join the list of pending events onto the tail of the main list + and reset it */ + list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); + kctx->event_coalesce_count = 0; + + /* Return the number of events moved */ + return event_count; +} + +void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom) +{ + struct kbase_device *kbdev = ctx->kbdev; + + dev_dbg(kbdev->dev, "Posting event for atom %p\n", (void *)atom); + + if (WARN_ON(atom->status != KBASE_JD_ATOM_STATE_COMPLETED)) { + dev_warn(kbdev->dev, + "%s: Atom %d (%p) not completed (status %d)\n", + __func__, + kbase_jd_atom_id(atom->kctx, atom), + atom->kctx, + atom->status); + return; + } + + if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) { + if (atom->event_code == BASE_JD_EVENT_DONE) { + dev_dbg(kbdev->dev, "Suppressing event (atom done)\n"); + kbase_event_process_noreport(ctx, atom); + return; + } + } + + if (atom->core_req & BASEP_JD_REQ_EVENT_NEVER) { + dev_dbg(kbdev->dev, "Suppressing event (never)\n"); + kbase_event_process_noreport(ctx, atom); + return; + } + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, atom, TL_ATOM_STATE_POSTED); + if (atom->core_req & BASE_JD_REQ_EVENT_COALESCE) { + /* Don't report the event until other event(s) have completed */ + dev_dbg(kbdev->dev, "Deferring event (coalesced)\n"); + mutex_lock(&ctx->event_mutex); + list_add_tail(&atom->dep_item[0], &ctx->event_coalesce_list); + ++ctx->event_coalesce_count; + mutex_unlock(&ctx->event_mutex); + } else { + /* Report the event and any pending events now */ + int event_count = 1; + + mutex_lock(&ctx->event_mutex); + event_count += kbase_event_coalesce(ctx); + list_add_tail(&atom->dep_item[0], &ctx->event_list); + atomic_add(event_count, &ctx->event_count); + mutex_unlock(&ctx->event_mutex); + dev_dbg(kbdev->dev, "Reporting %d events\n", event_count); + + kbase_event_wakeup(ctx); + + /* Post-completion latency */ + trace_sysgraph(SGR_POST, ctx->id, + kbase_jd_atom_id(ctx, atom)); + } +} +KBASE_EXPORT_TEST_API(kbase_event_post); + +void kbase_event_close(struct kbase_context *kctx) +{ + mutex_lock(&kctx->event_mutex); + atomic_set(&kctx->event_closed, true); + mutex_unlock(&kctx->event_mutex); + kbase_event_wakeup(kctx); +} + +int kbase_event_init(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx); + + INIT_LIST_HEAD(&kctx->event_list); + INIT_LIST_HEAD(&kctx->event_coalesce_list); + mutex_init(&kctx->event_mutex); + kctx->event_coalesce_count = 0; + kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); + + if (NULL == kctx->event_workq) + return -EINVAL; + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_event_init); + +void kbase_event_cleanup(struct kbase_context *kctx) +{ + int event_count; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(kctx->event_workq); + + flush_workqueue(kctx->event_workq); + destroy_workqueue(kctx->event_workq); + + /* We use kbase_event_dequeue to remove the remaining events as that + * deals with all the cleanup needed for the atoms. + * + * Note: use of kctx->event_list without a lock is safe because this must be the last + * thread using it (because we're about to terminate the lock) + */ + event_count = kbase_event_coalesce(kctx); + atomic_add(event_count, &kctx->event_count); + + while (!list_empty(&kctx->event_list)) { + struct base_jd_event_v2 event; + + kbase_event_dequeue(kctx, &event); + } +} + +KBASE_EXPORT_TEST_API(kbase_event_cleanup); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence.c b/drivers/gpu/arm/bifrost/mali_kbase_fence.c new file mode 100755 index 000000000000..5e04acf87892 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_fence.c @@ -0,0 +1,154 @@ +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include + +/* Spin lock protecting all Mali fences as fence->lock. */ +static DEFINE_SPINLOCK(kbase_fence_lock); + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +struct fence * +kbase_fence_out_new(struct kbase_jd_atom *katom) +#else +struct dma_fence * +kbase_fence_out_new(struct kbase_jd_atom *katom) +#endif +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + WARN_ON(katom->dma_fence.fence); + + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return NULL; + + dma_fence_init(fence, + &kbase_fence_ops, + &kbase_fence_lock, + katom->dma_fence.context, + atomic_inc_return(&katom->dma_fence.seqno)); + + katom->dma_fence.fence = fence; + + return fence; +} + +bool +kbase_fence_free_callbacks(struct kbase_jd_atom *katom) +{ + struct kbase_fence_cb *cb, *tmp; + bool res = false; + + lockdep_assert_held(&katom->kctx->jctx.lock); + + /* Clean up and free callbacks. */ + list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) { + bool ret; + + /* Cancel callbacks that hasn't been called yet. */ + ret = dma_fence_remove_callback(cb->fence, &cb->fence_cb); + if (ret) { + int ret; + + /* Fence had not signaled, clean up after + * canceling. + */ + ret = atomic_dec_return(&katom->dma_fence.dep_count); + + if (unlikely(ret == 0)) + res = true; + } + + /* + * Release the reference taken in + * kbase_fence_add_callback(). + */ + dma_fence_put(cb->fence); + list_del(&cb->node); + kfree(cb); + } + + return res; +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +int +kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct fence *fence, + fence_func_t callback) +#else +int +kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct dma_fence *fence, + dma_fence_func_t callback) +#endif +{ + int err = 0; + struct kbase_fence_cb *kbase_fence_cb; + + if (!fence) + return -EINVAL; + + kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL); + if (!kbase_fence_cb) + return -ENOMEM; + + kbase_fence_cb->fence = fence; + kbase_fence_cb->katom = katom; + INIT_LIST_HEAD(&kbase_fence_cb->node); + atomic_inc(&katom->dma_fence.dep_count); + + err = dma_fence_add_callback(fence, &kbase_fence_cb->fence_cb, + callback); + if (err == -ENOENT) { + /* Fence signaled, get the completion result */ + err = dma_fence_get_status(fence); + + /* remap success completion to err code */ + if (err == 1) + err = 0; + + kfree(kbase_fence_cb); + atomic_dec(&katom->dma_fence.dep_count); + } else if (err) { + kfree(kbase_fence_cb); + atomic_dec(&katom->dma_fence.dep_count); + } else { + /* + * Get reference to fence that will be kept until callback gets + * cleaned up in kbase_fence_free_callbacks(). + */ + dma_fence_get(fence); + /* Add callback to katom's list of callbacks */ + list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks); + } + + return err; +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence.h b/drivers/gpu/arm/bifrost/mali_kbase_fence.h new file mode 100755 index 000000000000..f319d9e1dce6 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_fence.h @@ -0,0 +1,284 @@ +/* + * + * (C) COPYRIGHT 2010-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_FENCE_H_ +#define _KBASE_FENCE_H_ + +/* + * mali_kbase_fence.[hc] has common fence code used by both + * - CONFIG_MALI_BIFROST_DMA_FENCE - implicit DMA fences + * - CONFIG_SYNC_FILE - explicit fences beginning with 4.9 kernel + */ + +#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + +#include +#include "mali_kbase_fence_defs.h" +#include "mali_kbase.h" + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +extern const struct fence_ops kbase_fence_ops; +#else +extern const struct dma_fence_ops kbase_fence_ops; +#endif + +/** +* struct kbase_fence_cb - Mali dma-fence callback data struct +* @fence_cb: Callback function +* @katom: Pointer to katom that is waiting on this callback +* @fence: Pointer to the fence object on which this callback is waiting +* @node: List head for linking this callback to the katom +*/ +struct kbase_fence_cb { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence_cb fence_cb; + struct fence *fence; +#else + struct dma_fence_cb fence_cb; + struct dma_fence *fence; +#endif + struct kbase_jd_atom *katom; + struct list_head node; +}; + +/** + * kbase_fence_out_new() - Creates a new output fence and puts it on the atom + * @katom: Atom to create an output fence for + * + * return: A new fence object on success, NULL on failure. + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); +#else +struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); +#endif + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_fence_in_set() - Assign input fence to atom + * @katom: Atom to assign input fence to + * @fence: Input fence to assign to atom + * + * This function will take ownership of one fence reference! + */ +#define kbase_fence_fence_in_set(katom, fence) \ + do { \ + WARN_ON((katom)->dma_fence.fence_in); \ + (katom)->dma_fence.fence_in = fence; \ + } while (0) +#endif + + +#if !MALI_USE_CSF +/** + * kbase_fence_out_remove() - Removes the output fence from atom + * @katom: Atom to remove output fence for + * + * This will also release the reference to this fence which the atom keeps + */ +static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom) +{ + if (katom->dma_fence.fence) { + dma_fence_put(katom->dma_fence.fence); + katom->dma_fence.fence = NULL; + } +} + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_out_remove() - Removes the input fence from atom + * @katom: Atom to remove input fence for + * + * This will also release the reference to this fence which the atom keeps + */ +static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom) +{ + if (katom->dma_fence.fence_in) { + dma_fence_put(katom->dma_fence.fence_in); + katom->dma_fence.fence_in = NULL; + } +} +#endif + +/** + * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us + * @katom: Atom to check output fence for + * + * Return: true if fence exists and is valid, otherwise false + */ +static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom) +{ + return katom->dma_fence.fence && + katom->dma_fence.fence->ops == &kbase_fence_ops; +} + +/** + * kbase_fence_out_signal() - Signal output fence of atom + * @katom: Atom to signal output fence for + * @status: Status to signal with (0 for success, < 0 for error) + * + * Return: 0 on success, < 0 on error + */ +static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, + int status) +{ + if (status) { +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ + KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE) + fence_set_error(katom->dma_fence.fence, status); +#elif (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) + dma_fence_set_error(katom->dma_fence.fence, status); +#else + katom->dma_fence.fence->status = status; +#endif + } + return dma_fence_signal(katom->dma_fence.fence); +} + +/** + * kbase_fence_add_callback() - Add callback on @fence to block @katom + * @katom: Pointer to katom that will be blocked by @fence + * @fence: Pointer to fence on which to set up the callback + * @callback: Pointer to function to be called when fence is signaled + * + * Caller needs to hold a reference to @fence when calling this function, and + * the caller is responsible for releasing that reference. An additional + * reference to @fence will be taken when the callback was successfully set up + * and @fence needs to be kept valid until the callback has been called and + * cleanup have been done. + * + * Return: 0 on success: fence was either already signaled, or callback was + * set up. Negative error code is returned on error. + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +int kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct fence *fence, + fence_func_t callback); +#else +int kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct dma_fence *fence, + dma_fence_func_t callback); +#endif + +/** + * kbase_fence_dep_count_set() - Set dep_count value on atom to specified value + * @katom: Atom to set dep_count for + * @val: value to set dep_count to + * + * The dep_count is available to the users of this module so that they can + * synchronize completion of the wait with cancellation and adding of more + * callbacks. For instance, a user could do the following: + * + * dep_count set to 1 + * callback #1 added, dep_count is increased to 2 + * callback #1 happens, dep_count decremented to 1 + * since dep_count > 0, no completion is done + * callback #2 is added, dep_count is increased to 2 + * dep_count decremented to 1 + * callback #2 happens, dep_count decremented to 0 + * since dep_count now is zero, completion executes + * + * The dep_count can also be used to make sure that the completion only + * executes once. This is typically done by setting dep_count to -1 for the + * thread that takes on this responsibility. + */ +static inline void +kbase_fence_dep_count_set(struct kbase_jd_atom *katom, int val) +{ + atomic_set(&katom->dma_fence.dep_count, val); +} + +/** + * kbase_fence_dep_count_dec_and_test() - Decrements dep_count + * @katom: Atom to decrement dep_count for + * + * See @kbase_fence_dep_count_set for general description about dep_count + * + * Return: true if value was decremented to zero, otherwise false + */ +static inline bool +kbase_fence_dep_count_dec_and_test(struct kbase_jd_atom *katom) +{ + return atomic_dec_and_test(&katom->dma_fence.dep_count); +} + +/** + * kbase_fence_dep_count_read() - Returns the current dep_count value + * @katom: Pointer to katom + * + * See @kbase_fence_dep_count_set for general description about dep_count + * + * Return: The current dep_count value + */ +static inline int kbase_fence_dep_count_read(struct kbase_jd_atom *katom) +{ + return atomic_read(&katom->dma_fence.dep_count); +} + +/** + * kbase_fence_free_callbacks() - Free dma-fence callbacks on a katom + * @katom: Pointer to katom + * + * This function will free all fence callbacks on the katom's list of + * callbacks. Callbacks that have not yet been called, because their fence + * hasn't yet signaled, will first be removed from the fence. + * + * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. + * + * Return: true if dep_count reached 0, otherwise false. + */ +bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom); + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_in_get() - Retrieve input fence for atom. + * @katom: Atom to get input fence from + * + * A ref will be taken for the fence, so use @kbase_fence_put() to release it + * + * Return: The fence, or NULL if there is no input fence for atom + */ +#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in) +#endif + +/** + * kbase_fence_out_get() - Retrieve output fence for atom. + * @katom: Atom to get output fence from + * + * A ref will be taken for the fence, so use @kbase_fence_put() to release it + * + * Return: The fence, or NULL if there is no output fence for atom + */ +#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) + +#endif /* !MALI_USE_CSF */ + +/** + * kbase_fence_put() - Releases a reference to a fence + * @fence: Fence to release reference for. + */ +#define kbase_fence_put(fence) dma_fence_put(fence) + + +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || defined(CONFIG_SYNC_FILE */ + +#endif /* _KBASE_FENCE_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_fence_defs.h new file mode 100755 index 000000000000..303029639d38 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_fence_defs.h @@ -0,0 +1,64 @@ +/* + * + * (C) COPYRIGHT 2010-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_FENCE_DEFS_H_ +#define _KBASE_FENCE_DEFS_H_ + +/* + * There was a big rename in the 4.10 kernel (fence* -> dma_fence*) + * This file hides the compatibility issues with this for the rest the driver + */ + +#include + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + +#include + +#define dma_fence_context_alloc(a) fence_context_alloc(a) +#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e) +#define dma_fence_get(a) fence_get(a) +#define dma_fence_put(a) fence_put(a) +#define dma_fence_signal(a) fence_signal(a) +#define dma_fence_is_signaled(a) fence_is_signaled(a) +#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c) +#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b) + +#if (KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE) +#define dma_fence_get_status(a) (fence_is_signaled(a) ? (a)->error ?: 1 : 0) +#else +#define dma_fence_get_status(a) (fence_is_signaled(a) ? (a)->status ?: 1 : 0) +#endif + +#else + +#include + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) +#define dma_fence_get_status(a) (dma_fence_is_signaled(a) ? \ + (a)->status ?: 1 \ + : 0) +#endif + +#endif /* < 4.10.0 */ + +#endif /* _KBASE_FENCE_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c b/drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c new file mode 100755 index 000000000000..c4703748bec6 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c @@ -0,0 +1,84 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include + +static const char * +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_get_driver_name(struct fence *fence) +#else +kbase_fence_get_driver_name(struct dma_fence *fence) +#endif +{ + return kbase_drv_name; +} + +static const char * +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_get_timeline_name(struct fence *fence) +#else +kbase_fence_get_timeline_name(struct dma_fence *fence) +#endif +{ + return kbase_timeline_name; +} + +static bool +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_enable_signaling(struct fence *fence) +#else +kbase_fence_enable_signaling(struct dma_fence *fence) +#endif +{ + return true; +} + +static void +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_fence_value_str(struct fence *fence, char *str, int size) +#else +kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) +#endif +{ +#if (KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE) + snprintf(str, size, "%u", fence->seqno); +#else + snprintf(str, size, "%llu", fence->seqno); +#endif +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +const struct fence_ops kbase_fence_ops = { + .wait = fence_default_wait, +#else +const struct dma_fence_ops kbase_fence_ops = { + .wait = dma_fence_default_wait, +#endif + .get_driver_name = kbase_fence_get_driver_name, + .get_timeline_name = kbase_fence_get_timeline_name, + .enable_signaling = kbase_fence_enable_signaling, + .fence_value_str = kbase_fence_fence_value_str +}; + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gator.h b/drivers/gpu/arm/bifrost/mali_kbase_gator.h new file mode 100755 index 000000000000..579c7b6ff3aa --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_gator.h @@ -0,0 +1,53 @@ +/* + * + * (C) COPYRIGHT 2011-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* NB taken from gator */ +/* + * List of possible actions to be controlled by DS-5 Streamline. + * The following numbers are used by gator to control the frame buffer dumping + * and s/w counter reporting. We cannot use the enums in mali_uk_types.h because + * they are unknown inside gator. + */ + +#ifndef _KBASE_GATOR_H_ +#define _KBASE_GATOR_H_ + +#include + +#define GATOR_JOB_SLOT_START 1 +#define GATOR_JOB_SLOT_STOP 2 +#define GATOR_JOB_SLOT_SOFT_STOPPED 3 + +#ifdef CONFIG_MALI_BIFROST_GATOR_SUPPORT + +#define GATOR_MAKE_EVENT(type, number) (((type) << 24) | ((number) << 16)) + +struct kbase_context; + +void kbase_trace_mali_job_slots_event(u32 dev_id, u32 event, const struct kbase_context *kctx, u8 atom_id); +void kbase_trace_mali_pm_status(u32 dev_id, u32 event, u64 value); +void kbase_trace_mali_page_fault_insert_pages(u32 dev_id, int event, u32 value); +void kbase_trace_mali_total_alloc_pages_change(u32 dev_id, long long int event); + +#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ + +#endif /* _KBASE_GATOR_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.c new file mode 100755 index 000000000000..569abd920fde --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.c @@ -0,0 +1,104 @@ +/* + * + * (C) COPYRIGHT 2012-2017, 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include + +#ifdef CONFIG_DEBUG_FS +/** Show callback for the @c gpu_memory debugfs file. + * + * This function is called to get the contents of the @c gpu_memory debugfs + * file. This is a report of current gpu memory usage. + * + * @param sfile The debugfs entry + * @param data Data associated with the entry + * + * @return 0 if successfully prints data in debugfs entry file + * -1 if it encountered an error + */ + +static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data) +{ + struct list_head *entry; + const struct list_head *kbdev_list; + + kbdev_list = kbase_device_get_list(); + list_for_each(entry, kbdev_list) { + struct kbase_device *kbdev = NULL; + struct kbase_context *kctx; + + kbdev = list_entry(entry, struct kbase_device, entry); + /* output the total memory usage and cap for this device */ + seq_printf(sfile, "%-16s %10u\n", + kbdev->devname, + atomic_read(&(kbdev->memdev.used_pages))); + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { + /* output the memory usage and cap for each kctx + * opened on this device */ + seq_printf(sfile, " %s-0x%p %10u\n", + "kctx", + kctx, + atomic_read(&(kctx->used_pages))); + } + mutex_unlock(&kbdev->kctx_list_lock); + } + kbase_device_put_list(kbdev_list); + return 0; +} + +/* + * File operations related to debugfs entry for gpu_memory + */ +static int kbasep_gpu_memory_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_gpu_memory_seq_show, NULL); +} + +static const struct file_operations kbasep_gpu_memory_debugfs_fops = { + .owner = THIS_MODULE, + .open = kbasep_gpu_memory_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* + * Initialize debugfs entry for gpu_memory + */ +void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("gpu_memory", S_IRUGO, + kbdev->mali_debugfs_directory, NULL, + &kbasep_gpu_memory_debugfs_fops); + return; +} + +#else +/* + * Stub functions for when debugfs is disabled + */ +void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) +{ + return; +} +#endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h new file mode 100755 index 000000000000..a45dabbb680f --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h @@ -0,0 +1,54 @@ +/* + * + * (C) COPYRIGHT 2012-2014, 2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_gpu_memory_debugfs.h + * Header file for gpu_memory entry in debugfs + * + */ + +#ifndef _KBASE_GPU_MEMORY_DEBUGFS_H +#define _KBASE_GPU_MEMORY_DEBUGFS_H + +#include +#include + +/* kbase_io_history_add - add new entry to the register access history + * + * @h: Pointer to the history data structure + * @addr: Register address + * @value: The value that is either read from or written to the register + * @write: 1 if it's a register write, 0 if it's a read + */ +void kbase_io_history_add(struct kbase_io_history *h, void __iomem const *addr, + u32 value, u8 write); + +/** + * kbasep_gpu_memory_debugfs_init - Initialize gpu_memory debugfs entry + * + * @kbdev: Device pointer + */ +void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev); + +#endif /*_KBASE_GPU_MEMORY_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c new file mode 100755 index 000000000000..020b5d853608 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c @@ -0,0 +1,649 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Base kernel property query APIs + */ + +#include +#include +#include +#include +#include +#include "mali_kbase_ioctl.h" +#include +#include +#include +#include + + +static void kbase_gpuprops_construct_coherent_groups( + struct base_gpu_props * const props) +{ + struct mali_base_gpu_coherent_group *current_group; + u64 group_present; + u64 group_mask; + u64 first_set, first_set_prev; + u32 num_groups = 0; + + KBASE_DEBUG_ASSERT(NULL != props); + + props->coherency_info.coherency = props->raw_props.mem_features; + props->coherency_info.num_core_groups = hweight64(props->raw_props.l2_present); + + if (props->coherency_info.coherency & GROUPS_L2_COHERENT) { + /* Group is l2 coherent */ + group_present = props->raw_props.l2_present; + } else { + /* Group is l1 coherent */ + group_present = props->raw_props.shader_present; + } + + /* + * The coherent group mask can be computed from the l2 present + * register. + * + * For the coherent group n: + * group_mask[n] = (first_set[n] - 1) & ~(first_set[n-1] - 1) + * where first_set is group_present with only its nth set-bit kept + * (i.e. the position from where a new group starts). + * + * For instance if the groups are l2 coherent and l2_present=0x0..01111: + * The first mask is: + * group_mask[1] = (first_set[1] - 1) & ~(first_set[0] - 1) + * = (0x0..010 - 1) & ~(0x0..01 - 1) + * = 0x0..00f + * The second mask is: + * group_mask[2] = (first_set[2] - 1) & ~(first_set[1] - 1) + * = (0x0..100 - 1) & ~(0x0..010 - 1) + * = 0x0..0f0 + * And so on until all the bits from group_present have been cleared + * (i.e. there is no group left). + */ + + current_group = props->coherency_info.group; + first_set = group_present & ~(group_present - 1); + + while (group_present != 0 && num_groups < BASE_MAX_COHERENT_GROUPS) { + group_present -= first_set; /* Clear the current group bit */ + first_set_prev = first_set; + + first_set = group_present & ~(group_present - 1); + group_mask = (first_set - 1) & ~(first_set_prev - 1); + + /* Populate the coherent_group structure for each group */ + current_group->core_mask = group_mask & props->raw_props.shader_present; + current_group->num_cores = hweight64(current_group->core_mask); + + num_groups++; + current_group++; + } + + if (group_present != 0) + pr_warn("Too many coherent groups (keeping only %d groups).\n", BASE_MAX_COHERENT_GROUPS); + + props->coherency_info.num_groups = num_groups; +} + +/** + * kbase_gpuprops_get_props - Get the GPU configuration + * @gpu_props: The &struct base_gpu_props structure + * @kbdev: The &struct kbase_device structure for the device + * + * Fill the &struct base_gpu_props structure with values from the GPU + * configuration registers. Only the raw properties are filled in this function. + * + * Return: Zero on success, Linux error code on failure + */ +static int kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props, + struct kbase_device *kbdev) +{ + struct kbase_gpuprops_regdump regdump; + int i; + int err; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(NULL != gpu_props); + + /* Dump relevant registers */ + err = kbase_backend_gpuprops_get(kbdev, ®dump); + if (err) + return err; + + gpu_props->raw_props.gpu_id = regdump.gpu_id; + gpu_props->raw_props.tiler_features = regdump.tiler_features; + gpu_props->raw_props.mem_features = regdump.mem_features; + gpu_props->raw_props.mmu_features = regdump.mmu_features; + gpu_props->raw_props.l2_features = regdump.l2_features; + gpu_props->raw_props.core_features = regdump.core_features; + + gpu_props->raw_props.as_present = regdump.as_present; + gpu_props->raw_props.js_present = regdump.js_present; + gpu_props->raw_props.shader_present = + ((u64) regdump.shader_present_hi << 32) + + regdump.shader_present_lo; + gpu_props->raw_props.tiler_present = + ((u64) regdump.tiler_present_hi << 32) + + regdump.tiler_present_lo; + gpu_props->raw_props.l2_present = + ((u64) regdump.l2_present_hi << 32) + + regdump.l2_present_lo; + gpu_props->raw_props.stack_present = + ((u64) regdump.stack_present_hi << 32) + + regdump.stack_present_lo; + + for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) + gpu_props->raw_props.js_features[i] = regdump.js_features[i]; + + for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) + gpu_props->raw_props.texture_features[i] = regdump.texture_features[i]; + + gpu_props->raw_props.thread_max_barrier_size = regdump.thread_max_barrier_size; + gpu_props->raw_props.thread_max_threads = regdump.thread_max_threads; + gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size; + gpu_props->raw_props.thread_features = regdump.thread_features; + gpu_props->raw_props.thread_tls_alloc = regdump.thread_tls_alloc; + + return 0; +} + +void kbase_gpuprops_update_core_props_gpu_id( + struct base_gpu_props * const gpu_props) +{ + gpu_props->core_props.version_status = + KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4); + gpu_props->core_props.minor_revision = + KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8); + gpu_props->core_props.major_revision = + KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4); + gpu_props->core_props.product_id = + KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16); +} + +/** + * kbase_gpuprops_calculate_props - Calculate the derived properties + * @gpu_props: The &struct base_gpu_props structure + * @kbdev: The &struct kbase_device structure for the device + * + * Fill the &struct base_gpu_props structure with values derived from the GPU + * configuration registers + */ +static void kbase_gpuprops_calculate_props( + struct base_gpu_props * const gpu_props, struct kbase_device *kbdev) +{ + int i; + u32 gpu_id; + u32 product_id; + + /* Populate the base_gpu_props structure */ + kbase_gpuprops_update_core_props_gpu_id(gpu_props); + gpu_props->core_props.log2_program_counter_size = KBASE_GPU_PC_SIZE_LOG2; +#if KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE + gpu_props->core_props.gpu_available_memory_size = totalram_pages << PAGE_SHIFT; +#else + gpu_props->core_props.gpu_available_memory_size = + totalram_pages() << PAGE_SHIFT; +#endif + + gpu_props->core_props.num_exec_engines = + KBASE_UBFX32(gpu_props->raw_props.core_features, 0, 4); + + for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) + gpu_props->core_props.texture_features[i] = gpu_props->raw_props.texture_features[i]; + + gpu_props->l2_props.log2_line_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 0U, 8); + gpu_props->l2_props.log2_cache_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); + + /* Field with number of l2 slices is added to MEM_FEATURES register + * since t76x. Below code assumes that for older GPU reserved bits will + * be read as zero. */ + gpu_props->l2_props.num_l2_slices = + KBASE_UBFX32(gpu_props->raw_props.mem_features, 8U, 4) + 1; + + gpu_props->tiler_props.bin_size_bytes = 1 << KBASE_UBFX32(gpu_props->raw_props.tiler_features, 0U, 6); + gpu_props->tiler_props.max_active_levels = KBASE_UBFX32(gpu_props->raw_props.tiler_features, 8U, 4); + + if (gpu_props->raw_props.thread_max_threads == 0) + gpu_props->thread_props.max_threads = THREAD_MT_DEFAULT; + else + gpu_props->thread_props.max_threads = gpu_props->raw_props.thread_max_threads; + + if (gpu_props->raw_props.thread_max_workgroup_size == 0) + gpu_props->thread_props.max_workgroup_size = THREAD_MWS_DEFAULT; + else + gpu_props->thread_props.max_workgroup_size = gpu_props->raw_props.thread_max_workgroup_size; + + if (gpu_props->raw_props.thread_max_barrier_size == 0) + gpu_props->thread_props.max_barrier_size = THREAD_MBS_DEFAULT; + else + gpu_props->thread_props.max_barrier_size = gpu_props->raw_props.thread_max_barrier_size; + + if (gpu_props->raw_props.thread_tls_alloc == 0) + gpu_props->thread_props.tls_alloc = + gpu_props->thread_props.max_threads; + else + gpu_props->thread_props.tls_alloc = + gpu_props->raw_props.thread_tls_alloc; + + /* MIDHARC-2364 was intended for tULx. + * Workaround for the incorrectly applied THREAD_FEATURES to tDUx. + */ + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; + product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; + +#if MALI_USE_CSF + gpu_props->thread_props.max_registers = + KBASE_UBFX32(gpu_props->raw_props.thread_features, + 0U, 22); + gpu_props->thread_props.impl_tech = + KBASE_UBFX32(gpu_props->raw_props.thread_features, + 22U, 2); + gpu_props->thread_props.max_task_queue = + KBASE_UBFX32(gpu_props->raw_props.thread_features, + 24U, 8); + gpu_props->thread_props.max_thread_group_split = 0; +#else + if ((gpu_id & GPU_ID2_PRODUCT_MODEL) == GPU_ID2_PRODUCT_TDUX) { + gpu_props->thread_props.max_registers = + KBASE_UBFX32(gpu_props->raw_props.thread_features, + 0U, 22); + gpu_props->thread_props.impl_tech = + KBASE_UBFX32(gpu_props->raw_props.thread_features, + 22U, 2); + gpu_props->thread_props.max_task_queue = + KBASE_UBFX32(gpu_props->raw_props.thread_features, + 24U, 8); + gpu_props->thread_props.max_thread_group_split = 0; + } else { + gpu_props->thread_props.max_registers = + KBASE_UBFX32(gpu_props->raw_props.thread_features, + 0U, 16); + gpu_props->thread_props.max_task_queue = + KBASE_UBFX32(gpu_props->raw_props.thread_features, + 16U, 8); + gpu_props->thread_props.max_thread_group_split = + KBASE_UBFX32(gpu_props->raw_props.thread_features, + 24U, 6); + gpu_props->thread_props.impl_tech = + KBASE_UBFX32(gpu_props->raw_props.thread_features, + 30U, 2); + } +#endif + + /* If values are not specified, then use defaults */ + if (gpu_props->thread_props.max_registers == 0) { + gpu_props->thread_props.max_registers = THREAD_MR_DEFAULT; + gpu_props->thread_props.max_task_queue = THREAD_MTQ_DEFAULT; + gpu_props->thread_props.max_thread_group_split = THREAD_MTGS_DEFAULT; + } + /* Initialize the coherent_group structure for each group */ + kbase_gpuprops_construct_coherent_groups(gpu_props); +} + +void kbase_gpuprops_set(struct kbase_device *kbdev) +{ + struct kbase_gpu_props *gpu_props; + struct gpu_raw_gpu_props *raw; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + gpu_props = &kbdev->gpu_props; + raw = &gpu_props->props.raw_props; + + /* Initialize the base_gpu_props structure from the hardware */ + kbase_gpuprops_get_props(&gpu_props->props, kbdev); + + /* Populate the derived properties */ + kbase_gpuprops_calculate_props(&gpu_props->props, kbdev); + + /* Populate kbase-only fields */ + gpu_props->l2_props.associativity = KBASE_UBFX32(raw->l2_features, 8U, 8); + gpu_props->l2_props.external_bus_width = KBASE_UBFX32(raw->l2_features, 24U, 8); + + gpu_props->mem.core_group = KBASE_UBFX32(raw->mem_features, 0U, 1); + + gpu_props->mmu.va_bits = KBASE_UBFX32(raw->mmu_features, 0U, 8); + gpu_props->mmu.pa_bits = KBASE_UBFX32(raw->mmu_features, 8U, 8); + + gpu_props->num_cores = hweight64(raw->shader_present); + gpu_props->num_core_groups = hweight64(raw->l2_present); + gpu_props->num_address_spaces = hweight32(raw->as_present); + gpu_props->num_job_slots = hweight32(raw->js_present); +} + +int kbase_gpuprops_set_features(struct kbase_device *kbdev) +{ + struct base_gpu_props *gpu_props; + struct kbase_gpuprops_regdump regdump; + int err; + + gpu_props = &kbdev->gpu_props.props; + + /* Dump relevant registers */ + err = kbase_backend_gpuprops_get_features(kbdev, ®dump); + if (err) + return err; + + /* + * Copy the raw value from the register, later this will get turned + * into the selected coherency mode. + * Additionally, add non-coherent mode, as this is always supported. + */ + gpu_props->raw_props.coherency_mode = regdump.coherency_features | + COHERENCY_FEATURE_BIT(COHERENCY_NONE); + + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_THREAD_GROUP_SPLIT)) + gpu_props->thread_props.max_thread_group_split = 0; + + return err; +} + +/* + * Module parameters to allow the L2 size and hash configuration to be + * overridden. + * + * These parameters must be set on insmod to take effect, and are not visible + * in sysfs. + */ +static u8 override_l2_size; +module_param(override_l2_size, byte, 0); +MODULE_PARM_DESC(override_l2_size, "Override L2 size config for testing"); + +static u8 override_l2_hash; +module_param(override_l2_hash, byte, 0); +MODULE_PARM_DESC(override_l2_hash, "Override L2 hash config for testing"); + +/** + * kbase_read_l2_config_from_dt - Read L2 configuration + * @kbdev: The kbase device for which to get the L2 configuration. + * + * Check for L2 configuration overrides in module parameters and device tree. + * Override values in module parameters take priority over override values in + * device tree. + * + * Return: true if either size or hash was overridden, false if no overrides + * were found. + */ +static bool kbase_read_l2_config_from_dt(struct kbase_device * const kbdev) +{ + struct device_node *np = kbdev->dev->of_node; + + if (!np) + return false; + + if (override_l2_size) + kbdev->l2_size_override = override_l2_size; + else if (of_property_read_u8(np, "l2-size", &kbdev->l2_size_override)) + kbdev->l2_size_override = 0; + + if (override_l2_hash) + kbdev->l2_hash_override = override_l2_hash; + else if (of_property_read_u8(np, "l2-hash", &kbdev->l2_hash_override)) + kbdev->l2_hash_override = 0; + + if (kbdev->l2_size_override || kbdev->l2_hash_override) + return true; + + return false; +} + +int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) +{ + int err = 0; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { + struct kbase_gpuprops_regdump regdump; + struct base_gpu_props *gpu_props = &kbdev->gpu_props.props; + + /* Check for L2 cache size & hash overrides */ + if (!kbase_read_l2_config_from_dt(kbdev)) + return 0; + + /* Need L2 to get powered to reflect to L2_FEATURES */ + kbase_pm_context_active(kbdev); + + /* Wait for the completion of L2 power transition */ + kbase_pm_wait_for_l2_powered(kbdev); + + /* Dump L2_FEATURES register */ + err = kbase_backend_gpuprops_get_l2_features(kbdev, ®dump); + if (err) + goto idle_gpu; + + dev_info(kbdev->dev, "Reflected L2_FEATURES is 0x%x\n", + regdump.l2_features); + + /* Update gpuprops with reflected L2_FEATURES */ + gpu_props->raw_props.l2_features = regdump.l2_features; + gpu_props->l2_props.log2_cache_size = + KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); + +idle_gpu: + /* Let GPU idle */ + kbase_pm_context_idle(kbdev); + } + + return err; +} + +static struct { + u32 type; + size_t offset; + int size; +} gpu_property_mapping[] = { +#define PROP(name, member) \ + {KBASE_GPUPROP_ ## name, offsetof(struct base_gpu_props, member), \ + sizeof(((struct base_gpu_props *)0)->member)} + PROP(PRODUCT_ID, core_props.product_id), + PROP(VERSION_STATUS, core_props.version_status), + PROP(MINOR_REVISION, core_props.minor_revision), + PROP(MAJOR_REVISION, core_props.major_revision), + PROP(GPU_FREQ_KHZ_MAX, core_props.gpu_freq_khz_max), + PROP(LOG2_PROGRAM_COUNTER_SIZE, core_props.log2_program_counter_size), + PROP(TEXTURE_FEATURES_0, core_props.texture_features[0]), + PROP(TEXTURE_FEATURES_1, core_props.texture_features[1]), + PROP(TEXTURE_FEATURES_2, core_props.texture_features[2]), + PROP(TEXTURE_FEATURES_3, core_props.texture_features[3]), + PROP(GPU_AVAILABLE_MEMORY_SIZE, core_props.gpu_available_memory_size), + PROP(NUM_EXEC_ENGINES, core_props.num_exec_engines), + + PROP(L2_LOG2_LINE_SIZE, l2_props.log2_line_size), + PROP(L2_LOG2_CACHE_SIZE, l2_props.log2_cache_size), + PROP(L2_NUM_L2_SLICES, l2_props.num_l2_slices), + + PROP(TILER_BIN_SIZE_BYTES, tiler_props.bin_size_bytes), + PROP(TILER_MAX_ACTIVE_LEVELS, tiler_props.max_active_levels), + + PROP(MAX_THREADS, thread_props.max_threads), + PROP(MAX_WORKGROUP_SIZE, thread_props.max_workgroup_size), + PROP(MAX_BARRIER_SIZE, thread_props.max_barrier_size), + PROP(MAX_REGISTERS, thread_props.max_registers), + PROP(MAX_TASK_QUEUE, thread_props.max_task_queue), + PROP(MAX_THREAD_GROUP_SPLIT, thread_props.max_thread_group_split), + PROP(IMPL_TECH, thread_props.impl_tech), + PROP(TLS_ALLOC, thread_props.tls_alloc), + + PROP(RAW_SHADER_PRESENT, raw_props.shader_present), + PROP(RAW_TILER_PRESENT, raw_props.tiler_present), + PROP(RAW_L2_PRESENT, raw_props.l2_present), + PROP(RAW_STACK_PRESENT, raw_props.stack_present), + PROP(RAW_L2_FEATURES, raw_props.l2_features), + PROP(RAW_CORE_FEATURES, raw_props.core_features), + PROP(RAW_MEM_FEATURES, raw_props.mem_features), + PROP(RAW_MMU_FEATURES, raw_props.mmu_features), + PROP(RAW_AS_PRESENT, raw_props.as_present), + PROP(RAW_JS_PRESENT, raw_props.js_present), + PROP(RAW_JS_FEATURES_0, raw_props.js_features[0]), + PROP(RAW_JS_FEATURES_1, raw_props.js_features[1]), + PROP(RAW_JS_FEATURES_2, raw_props.js_features[2]), + PROP(RAW_JS_FEATURES_3, raw_props.js_features[3]), + PROP(RAW_JS_FEATURES_4, raw_props.js_features[4]), + PROP(RAW_JS_FEATURES_5, raw_props.js_features[5]), + PROP(RAW_JS_FEATURES_6, raw_props.js_features[6]), + PROP(RAW_JS_FEATURES_7, raw_props.js_features[7]), + PROP(RAW_JS_FEATURES_8, raw_props.js_features[8]), + PROP(RAW_JS_FEATURES_9, raw_props.js_features[9]), + PROP(RAW_JS_FEATURES_10, raw_props.js_features[10]), + PROP(RAW_JS_FEATURES_11, raw_props.js_features[11]), + PROP(RAW_JS_FEATURES_12, raw_props.js_features[12]), + PROP(RAW_JS_FEATURES_13, raw_props.js_features[13]), + PROP(RAW_JS_FEATURES_14, raw_props.js_features[14]), + PROP(RAW_JS_FEATURES_15, raw_props.js_features[15]), + PROP(RAW_TILER_FEATURES, raw_props.tiler_features), + PROP(RAW_TEXTURE_FEATURES_0, raw_props.texture_features[0]), + PROP(RAW_TEXTURE_FEATURES_1, raw_props.texture_features[1]), + PROP(RAW_TEXTURE_FEATURES_2, raw_props.texture_features[2]), + PROP(RAW_TEXTURE_FEATURES_3, raw_props.texture_features[3]), + PROP(RAW_GPU_ID, raw_props.gpu_id), + PROP(RAW_THREAD_MAX_THREADS, raw_props.thread_max_threads), + PROP(RAW_THREAD_MAX_WORKGROUP_SIZE, + raw_props.thread_max_workgroup_size), + PROP(RAW_THREAD_MAX_BARRIER_SIZE, raw_props.thread_max_barrier_size), + PROP(RAW_THREAD_FEATURES, raw_props.thread_features), + PROP(RAW_THREAD_TLS_ALLOC, raw_props.thread_tls_alloc), + PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode), + + PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups), + PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups), + PROP(COHERENCY_COHERENCY, coherency_info.coherency), + PROP(COHERENCY_GROUP_0, coherency_info.group[0].core_mask), + PROP(COHERENCY_GROUP_1, coherency_info.group[1].core_mask), + PROP(COHERENCY_GROUP_2, coherency_info.group[2].core_mask), + PROP(COHERENCY_GROUP_3, coherency_info.group[3].core_mask), + PROP(COHERENCY_GROUP_4, coherency_info.group[4].core_mask), + PROP(COHERENCY_GROUP_5, coherency_info.group[5].core_mask), + PROP(COHERENCY_GROUP_6, coherency_info.group[6].core_mask), + PROP(COHERENCY_GROUP_7, coherency_info.group[7].core_mask), + PROP(COHERENCY_GROUP_8, coherency_info.group[8].core_mask), + PROP(COHERENCY_GROUP_9, coherency_info.group[9].core_mask), + PROP(COHERENCY_GROUP_10, coherency_info.group[10].core_mask), + PROP(COHERENCY_GROUP_11, coherency_info.group[11].core_mask), + PROP(COHERENCY_GROUP_12, coherency_info.group[12].core_mask), + PROP(COHERENCY_GROUP_13, coherency_info.group[13].core_mask), + PROP(COHERENCY_GROUP_14, coherency_info.group[14].core_mask), + PROP(COHERENCY_GROUP_15, coherency_info.group[15].core_mask), + +#undef PROP +}; + +int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev) +{ + struct kbase_gpu_props *kprops = &kbdev->gpu_props; + struct base_gpu_props *props = &kprops->props; + u32 count = ARRAY_SIZE(gpu_property_mapping); + u32 i; + u32 size = 0; + u8 *p; + + for (i = 0; i < count; i++) { + /* 4 bytes for the ID, and the size of the property */ + size += 4 + gpu_property_mapping[i].size; + } + + kprops->prop_buffer_size = size; + kprops->prop_buffer = kmalloc(size, GFP_KERNEL); + + if (!kprops->prop_buffer) { + kprops->prop_buffer_size = 0; + return -ENOMEM; + } + + p = kprops->prop_buffer; + +#define WRITE_U8(v) (*p++ = (v) & 0xFF) +#define WRITE_U16(v) do { WRITE_U8(v); WRITE_U8((v) >> 8); } while (0) +#define WRITE_U32(v) do { WRITE_U16(v); WRITE_U16((v) >> 16); } while (0) +#define WRITE_U64(v) do { WRITE_U32(v); WRITE_U32((v) >> 32); } while (0) + + for (i = 0; i < count; i++) { + u32 type = gpu_property_mapping[i].type; + u8 type_size; + void *field = ((u8 *)props) + gpu_property_mapping[i].offset; + + switch (gpu_property_mapping[i].size) { + case 1: + type_size = KBASE_GPUPROP_VALUE_SIZE_U8; + break; + case 2: + type_size = KBASE_GPUPROP_VALUE_SIZE_U16; + break; + case 4: + type_size = KBASE_GPUPROP_VALUE_SIZE_U32; + break; + case 8: + type_size = KBASE_GPUPROP_VALUE_SIZE_U64; + break; + default: + dev_err(kbdev->dev, + "Invalid gpu_property_mapping type=%d size=%d", + type, gpu_property_mapping[i].size); + return -EINVAL; + } + + WRITE_U32((type<<2) | type_size); + + switch (type_size) { + case KBASE_GPUPROP_VALUE_SIZE_U8: + WRITE_U8(*((u8 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U16: + WRITE_U16(*((u16 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U32: + WRITE_U32(*((u32 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U64: + WRITE_U64(*((u64 *)field)); + break; + default: /* Cannot be reached */ + WARN_ON(1); + return -EINVAL; + } + } + + return 0; +} + +void kbase_gpuprops_free_user_buffer(struct kbase_device *kbdev) +{ + kfree(kbdev->gpu_props.prop_buffer); +} + +int kbase_device_populate_max_freq(struct kbase_device *kbdev) +{ + struct mali_base_gpu_core_props *core_props; + + /* obtain max configured gpu frequency, if devfreq is enabled then + * this will be overridden by the highest operating point found + */ + core_props = &(kbdev->gpu_props.props.core_props); +#ifdef GPU_FREQ_KHZ_MAX + core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX; +#else + core_props->gpu_freq_khz_max = DEFAULT_GPU_FREQ_KHZ_MAX; +#endif + + return 0; +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h new file mode 100755 index 000000000000..5eee7948381a --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h @@ -0,0 +1,135 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2011-2015, 2017, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_gpuprops.h + * Base kernel property query APIs + */ + +#ifndef _KBASE_GPUPROPS_H_ +#define _KBASE_GPUPROPS_H_ + +#include "mali_kbase_gpuprops_types.h" + +/* Forward definition - see mali_kbase.h */ +struct kbase_device; + +/** + * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield. + * @value: The value from which to extract bits. + * @offset: The first bit to extract (0 being the LSB). + * @size: The number of bits to extract. + * + * Context: @offset + @size <= 32. + * + * Return: Bits [@offset, @offset + @size) from @value. + */ +/* from mali_cdsb.h */ +#define KBASE_UBFX32(value, offset, size) \ + (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1)) + +/** + * @brief Set up Kbase GPU properties. + * + * Set up Kbase GPU properties with information from the GPU registers + * + * @param kbdev The struct kbase_device structure for the device + */ +void kbase_gpuprops_set(struct kbase_device *kbdev); + +/** + * kbase_gpuprops_set_features - Set up Kbase GPU properties + * @kbdev: Device pointer + * + * This function sets up GPU properties that are dependent on the hardware + * features bitmask. This function must be preceeded by a call to + * kbase_hw_set_features_mask(). + * + * Return: Zero on success, Linux error code on failure + */ +int kbase_gpuprops_set_features(struct kbase_device *kbdev); + +/** + * kbase_gpuprops_update_l2_features - Update GPU property of L2_FEATURES + * @kbdev: Device pointer + * + * This function updates l2_features and the log2 cache size. + * + * Return: Zero on success, Linux error code for failure + */ +int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev); + +/** + * kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer + * @kbdev: The kbase device + * + * Fills prop_buffer with the GPU properties for user space to read. + */ +int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev); + +/** + * kbase_gpuprops_free_user_buffer - Free the GPU properties buffer. + * @kbdev: kbase device pointer + * + * Free the GPU properties buffer allocated from + * kbase_gpuprops_populate_user_buffer. + */ +void kbase_gpuprops_free_user_buffer(struct kbase_device *kbdev); + +/** + * kbase_device_populate_max_freq - Populate max gpu frequency. + * @kbdev: kbase device pointer + * + * Populate the maximum gpu frequency to be used when devfreq is disabled. + * + * Return: 0 on success and non-zero value on failure. + */ +int kbase_device_populate_max_freq(struct kbase_device *kbdev); + +/** + * kbase_gpuprops_update_core_props_gpu_id - break down gpu id value + * @gpu_props: the &base_gpu_props structure + * + * Break down gpu_id value stored in base_gpu_props::raw_props.gpu_id into + * separate fields (version_status, minor_revision, major_revision, product_id) + * stored in base_gpu_props::core_props. + */ +void kbase_gpuprops_update_core_props_gpu_id( + struct base_gpu_props * const gpu_props); + +#endif /* _KBASE_GPUPROPS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops_types.h b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops_types.h new file mode 100755 index 000000000000..ec6f1c39ccb0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops_types.h @@ -0,0 +1,98 @@ +/* + * + * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_gpuprops_types.h + * Base kernel property query APIs + */ + +#ifndef _KBASE_GPUPROPS_TYPES_H_ +#define _KBASE_GPUPROPS_TYPES_H_ + +#include "mali_base_kernel.h" + +#define KBASE_GPU_SPEED_MHZ 123 +#define KBASE_GPU_PC_SIZE_LOG2 24U + +struct kbase_gpuprops_regdump { + u32 gpu_id; + u32 l2_features; + u32 core_features; + u32 tiler_features; + u32 mem_features; + u32 mmu_features; + u32 as_present; + u32 js_present; + u32 thread_max_threads; + u32 thread_max_workgroup_size; + u32 thread_max_barrier_size; + u32 thread_features; + u32 thread_tls_alloc; + u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; + u32 js_features[GPU_MAX_JOB_SLOTS]; + u32 shader_present_lo; + u32 shader_present_hi; + u32 tiler_present_lo; + u32 tiler_present_hi; + u32 l2_present_lo; + u32 l2_present_hi; + u32 stack_present_lo; + u32 stack_present_hi; + u32 coherency_features; +}; + +struct kbase_gpu_cache_props { + u8 associativity; + u8 external_bus_width; +}; + +struct kbase_gpu_mem_props { + u8 core_group; +}; + +struct kbase_gpu_mmu_props { + u8 va_bits; + u8 pa_bits; +}; + +struct kbase_gpu_props { + /* kernel-only properties */ + u8 num_cores; + u8 num_core_groups; + u8 num_address_spaces; + u8 num_job_slots; + + struct kbase_gpu_cache_props l2_props; + + struct kbase_gpu_mem_props mem; + struct kbase_gpu_mmu_props mmu; + + /* Properties shared with userspace */ + struct base_gpu_props props; + + u32 prop_buffer_size; + void *prop_buffer; +}; + +#endif /* _KBASE_GPUPROPS_TYPES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gwt.c b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c new file mode 100755 index 000000000000..91dc4dbc0800 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c @@ -0,0 +1,273 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_gwt.h" +#include + +static inline void kbase_gpu_gwt_setup_page_permission( + struct kbase_context *kctx, + unsigned long flag, + struct rb_node *node) +{ + struct rb_node *rbnode = node; + + while (rbnode) { + struct kbase_va_region *reg; + int err = 0; + + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + if (reg->nr_pages && !kbase_is_region_invalid_or_free(reg) && + (reg->flags & KBASE_REG_GPU_WR)) { + err = kbase_mmu_update_pages(kctx, reg->start_pfn, + kbase_get_gpu_phy_pages(reg), + reg->gpu_alloc->nents, + reg->flags & flag, + reg->gpu_alloc->group_id); + if (err) + dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages failure\n"); + } + + rbnode = rb_next(rbnode); + } +} + +static void kbase_gpu_gwt_setup_pages(struct kbase_context *kctx, + unsigned long flag) +{ + kbase_gpu_gwt_setup_page_permission(kctx, flag, + rb_first(&(kctx->reg_rbtree_same))); + kbase_gpu_gwt_setup_page_permission(kctx, flag, + rb_first(&(kctx->reg_rbtree_custom))); +} + + +int kbase_gpu_gwt_start(struct kbase_context *kctx) +{ + kbase_gpu_vm_lock(kctx); + if (kctx->gwt_enabled) { + kbase_gpu_vm_unlock(kctx); + return -EBUSY; + } + + INIT_LIST_HEAD(&kctx->gwt_current_list); + INIT_LIST_HEAD(&kctx->gwt_snapshot_list); + +#if !MALI_USE_CSF + /* If GWT is enabled using new vector dumping format + * from user space, back up status of the job serialization flag and + * use full serialisation of jobs for dumping. + * Status will be restored on end of dumping in gwt_stop. + */ + kctx->kbdev->backup_serialize_jobs = kctx->kbdev->serialize_jobs; + kctx->kbdev->serialize_jobs = KBASE_SERIALIZE_INTRA_SLOT | + KBASE_SERIALIZE_INTER_SLOT; + +#endif + /* Mark gwt enabled before making pages read only in case a + write page fault is triggered while we're still in this loop. + (kbase_gpu_vm_lock() doesn't prevent this!) + */ + kctx->gwt_enabled = true; + kctx->gwt_was_enabled = true; + + kbase_gpu_gwt_setup_pages(kctx, ~KBASE_REG_GPU_WR); + + kbase_gpu_vm_unlock(kctx); + return 0; +} + +int kbase_gpu_gwt_stop(struct kbase_context *kctx) +{ + struct kbasep_gwt_list_element *pos, *n; + + kbase_gpu_vm_lock(kctx); + if (!kctx->gwt_enabled) { + kbase_gpu_vm_unlock(kctx); + return -EINVAL; + } + + list_for_each_entry_safe(pos, n, &kctx->gwt_current_list, link) { + list_del(&pos->link); + kfree(pos); + } + + list_for_each_entry_safe(pos, n, &kctx->gwt_snapshot_list, link) { + list_del(&pos->link); + kfree(pos); + } + +#if !MALI_USE_CSF + kctx->kbdev->serialize_jobs = kctx->kbdev->backup_serialize_jobs; +#endif + + kbase_gpu_gwt_setup_pages(kctx, ~0UL); + + kctx->gwt_enabled = false; + kbase_gpu_vm_unlock(kctx); + return 0; +} + + +static int list_cmp_function(void *priv, struct list_head *a, + struct list_head *b) +{ + struct kbasep_gwt_list_element *elementA = container_of(a, + struct kbasep_gwt_list_element, link); + struct kbasep_gwt_list_element *elementB = container_of(b, + struct kbasep_gwt_list_element, link); + + CSTD_UNUSED(priv); + + if (elementA->page_addr > elementB->page_addr) + return 1; + return -1; +} + +static void kbase_gpu_gwt_collate(struct kbase_context *kctx, + struct list_head *snapshot_list) +{ + struct kbasep_gwt_list_element *pos, *n; + struct kbasep_gwt_list_element *collated = NULL; + + /* Sort the list */ + list_sort(NULL, snapshot_list, list_cmp_function); + + /* Combine contiguous areas. */ + list_for_each_entry_safe(pos, n, snapshot_list, link) { + if (collated == NULL || collated->region != + pos->region || + (collated->page_addr + + (collated->num_pages * PAGE_SIZE)) != + pos->page_addr) { + /* This is the first time through, a new region or + * is not contiguous - start collating to this element + */ + collated = pos; + } else { + /* contiguous so merge */ + collated->num_pages += pos->num_pages; + /* remove element from list */ + list_del(&pos->link); + kfree(pos); + } + } +} + +int kbase_gpu_gwt_dump(struct kbase_context *kctx, + union kbase_ioctl_cinstr_gwt_dump *gwt_dump) +{ + const u32 ubuf_size = gwt_dump->in.len; + u32 ubuf_count = 0; + __user void *user_addr = (__user void *) + (uintptr_t)gwt_dump->in.addr_buffer; + __user void *user_sizes = (__user void *) + (uintptr_t)gwt_dump->in.size_buffer; + + kbase_gpu_vm_lock(kctx); + + if (!kctx->gwt_enabled) { + kbase_gpu_vm_unlock(kctx); + /* gwt_dump shouldn't be called when gwt is disabled */ + return -EPERM; + } + + if (!gwt_dump->in.len || !gwt_dump->in.addr_buffer + || !gwt_dump->in.size_buffer) { + kbase_gpu_vm_unlock(kctx); + /* We don't have any valid user space buffer to copy the + * write modified addresses. + */ + return -EINVAL; + } + + if (list_empty(&kctx->gwt_snapshot_list) && + !list_empty(&kctx->gwt_current_list)) { + + list_replace_init(&kctx->gwt_current_list, + &kctx->gwt_snapshot_list); + + /* We have collected all write faults so far + * and they will be passed on to user space. + * Reset the page flags state to allow collection of + * further write faults. + */ + kbase_gpu_gwt_setup_pages(kctx, ~KBASE_REG_GPU_WR); + + /* Sort and combine consecutive pages in the dump list*/ + kbase_gpu_gwt_collate(kctx, &kctx->gwt_snapshot_list); + } + + while ((!list_empty(&kctx->gwt_snapshot_list))) { + u64 addr_buffer[32]; + u64 num_page_buffer[32]; + u32 count = 0; + int err; + struct kbasep_gwt_list_element *dump_info, *n; + + list_for_each_entry_safe(dump_info, n, + &kctx->gwt_snapshot_list, link) { + addr_buffer[count] = dump_info->page_addr; + num_page_buffer[count] = dump_info->num_pages; + count++; + list_del(&dump_info->link); + kfree(dump_info); + if (ARRAY_SIZE(addr_buffer) == count || + ubuf_size == (ubuf_count + count)) + break; + } + + if (count) { + err = copy_to_user((user_addr + + (ubuf_count * sizeof(u64))), + (void *)addr_buffer, + count * sizeof(u64)); + if (err) { + dev_err(kctx->kbdev->dev, "Copy to user failure\n"); + kbase_gpu_vm_unlock(kctx); + return err; + } + err = copy_to_user((user_sizes + + (ubuf_count * sizeof(u64))), + (void *)num_page_buffer, + count * sizeof(u64)); + if (err) { + dev_err(kctx->kbdev->dev, "Copy to user failure\n"); + kbase_gpu_vm_unlock(kctx); + return err; + } + + ubuf_count += count; + } + + if (ubuf_count == ubuf_size) + break; + } + + if (!list_empty(&kctx->gwt_snapshot_list)) + gwt_dump->out.more_data_available = 1; + else + gwt_dump->out.more_data_available = 0; + + gwt_dump->out.no_of_addr_collected = ubuf_count; + kbase_gpu_vm_unlock(kctx); + return 0; +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gwt.h b/drivers/gpu/arm/bifrost/mali_kbase_gwt.h new file mode 100755 index 000000000000..7e7746e64915 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_gwt.h @@ -0,0 +1,55 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#if !defined(_KBASE_GWT_H) +#define _KBASE_GWT_H + +#include +#include + +/** + * kbase_gpu_gwt_start - Start the GPU write tracking + * @kctx: Pointer to kernel context + * + * @return 0 on success, error on failure. + */ +int kbase_gpu_gwt_start(struct kbase_context *kctx); + +/** + * kbase_gpu_gwt_stop - Stop the GPU write tracking + * @kctx: Pointer to kernel context + * + * @return 0 on success, error on failure. + */ +int kbase_gpu_gwt_stop(struct kbase_context *kctx); + +/** + * kbase_gpu_gwt_dump - Pass page address of faulting addresses to user space. + * @kctx: Pointer to kernel context + * @gwt_dump: User space data to be passed. + * + * @return 0 on success, error on failure. + */ +int kbase_gpu_gwt_dump(struct kbase_context *kctx, + union kbase_ioctl_cinstr_gwt_dump *gwt_dump); + +#endif /* _KBASE_GWT_H */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hw.c b/drivers/gpu/arm/bifrost/mali_kbase_hw.c new file mode 100755 index 000000000000..dc58ffb931be --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hw.c @@ -0,0 +1,437 @@ +/* + * + * (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Run-time work-arounds helpers + */ + +#include +#include +#include "gpu/mali_kbase_gpu_regmap.h" +#include "mali_kbase.h" +#include "mali_kbase_hw.h" + +void kbase_hw_set_features_mask(struct kbase_device *kbdev) +{ + const enum base_hw_feature *features; + u32 gpu_id; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + + switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { + case GPU_ID2_PRODUCT_TMIX: + features = base_hw_features_tMIx; + break; + case GPU_ID2_PRODUCT_THEX: + features = base_hw_features_tHEx; + break; + case GPU_ID2_PRODUCT_TSIX: + features = base_hw_features_tSIx; + break; + case GPU_ID2_PRODUCT_TDVX: + features = base_hw_features_tDVx; + break; + case GPU_ID2_PRODUCT_TNOX: + features = base_hw_features_tNOx; + break; + case GPU_ID2_PRODUCT_TGOX: + features = base_hw_features_tGOx; + break; + case GPU_ID2_PRODUCT_TTRX: + features = base_hw_features_tTRx; + break; + case GPU_ID2_PRODUCT_TNAX: + features = base_hw_features_tNAx; + break; + case GPU_ID2_PRODUCT_LBEX: + case GPU_ID2_PRODUCT_TBEX: + features = base_hw_features_tBEx; + break; + case GPU_ID2_PRODUCT_TBAX: + features = base_hw_features_tBAx; + break; + case GPU_ID2_PRODUCT_TDUX: + features = base_hw_features_tDUx; + break; + case GPU_ID2_PRODUCT_TODX: + case GPU_ID2_PRODUCT_LODX: + features = base_hw_features_tODx; + break; + case GPU_ID2_PRODUCT_TGRX: + features = base_hw_features_tGRx; + break; + case GPU_ID2_PRODUCT_TVAX: + features = base_hw_features_tVAx; + break; + case GPU_ID2_PRODUCT_TTUX: + /* Fallthrough */ + case GPU_ID2_PRODUCT_LTUX: + features = base_hw_features_tTUx; + break; + case GPU_ID2_PRODUCT_TE2X: + features = base_hw_features_tE2x; + break; + default: + features = base_hw_features_generic; + break; + } + + for (; *features != BASE_HW_FEATURE_END; features++) + set_bit(*features, &kbdev->hw_features_mask[0]); + +#if defined(CONFIG_MALI_VECTOR_DUMP) + /* When dumping is enabled, need to disable flush reduction optimization + * for GPUs on which it is safe to have only cache clean operation at + * the end of job chain. + * This is required to make vector dump work. There is some discrepancy + * in the implementation of flush reduction optimization due to + * unclear or ambiguous ARCH spec. + */ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CLEAN_ONLY_SAFE)) + clear_bit(BASE_HW_FEATURE_FLUSH_REDUCTION, + &kbdev->hw_features_mask[0]); +#endif +} + +/** + * kbase_hw_get_issues_for_new_id - Get the hardware issues for a new GPU ID + * @kbdev: Device pointer + * + * Return: pointer to an array of hardware issues, terminated by + * BASE_HW_ISSUE_END. + * + * In debugging versions of the driver, unknown versions of a known GPU will + * be treated as the most recent known version not later than the actual + * version. In such circumstances, the GPU ID in @kbdev will also be replaced + * with the most recent known version. + * + * Note: The GPU configuration must have been read by kbase_gpuprops_get_props() + * before calling this function. + */ +static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( + struct kbase_device *kbdev) +{ + const enum base_hw_issue *issues = NULL; + + struct base_hw_product { + u32 product_model; + struct { + u32 version; + const enum base_hw_issue *issues; + } map[7]; + }; + + static const struct base_hw_product base_hw_products[] = { + {GPU_ID2_PRODUCT_TMIX, + {{GPU_ID2_VERSION_MAKE(0, 0, 1), + base_hw_issues_tMIx_r0p0_05dev0}, + {GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tMIx_r0p1}, + {U32_MAX /* sentinel value */, NULL} } }, + + {GPU_ID2_PRODUCT_THEX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1}, + {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tHEx_r0p1}, + {GPU_ID2_VERSION_MAKE(0, 2, 0), base_hw_issues_tHEx_r0p2}, + {GPU_ID2_VERSION_MAKE(0, 3, 0), base_hw_issues_tHEx_r0p3}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TSIX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1}, + {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0}, + {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_tSIx_r1p1}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TDVX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tDVx_r0p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TNOX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tNOx_r0p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TGOX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tGOx_r0p0}, + {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tGOx_r1p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TTRX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTRx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tTRx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tTRx_r0p1}, + {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tTRx_r0p1}, + {GPU_ID2_VERSION_MAKE(0, 2, 0), base_hw_issues_tTRx_r0p2}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TNAX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tNAx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tNAx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 4), base_hw_issues_tNAx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 5), base_hw_issues_tNAx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tNAx_r0p1}, + {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tNAx_r0p1}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_LBEX, + {{GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_lBEx_r1p0}, + {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_lBEx_r1p1}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TBEX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tBEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tBEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tBEx_r0p1}, + {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tBEx_r1p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TBAX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tBAx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tBAx_r0p0}, + {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tBAx_r1p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TDUX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tDUx_r0p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TODX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tODx_r0p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_LODX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tODx_r0p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TGRX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tGRx_r0p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TVAX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tVAx_r0p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TTUX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTUx_r0p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_LTUX, + {{GPU_ID2_VERSION_MAKE(3, 0, 0), base_hw_issues_tTUx_r0p0}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TE2X, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tE2x_r0p0}, + {U32_MAX, NULL} } }, + }; + + u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + const u32 product_model = gpu_id & GPU_ID2_PRODUCT_MODEL; + const struct base_hw_product *product = NULL; + size_t p; + + /* Stop when we reach the end of the products array. */ + for (p = 0; p < ARRAY_SIZE(base_hw_products); ++p) { + if (product_model == base_hw_products[p].product_model) { + product = &base_hw_products[p]; + break; + } + } + + if (product != NULL) { + /* Found a matching product. */ + const u32 version = gpu_id & GPU_ID2_VERSION; + u32 fallback_version = 0; + const enum base_hw_issue *fallback_issues = NULL; + size_t v; + + /* Stop when we reach the end of the map. */ + for (v = 0; product->map[v].version != U32_MAX; ++v) { + + if (version == product->map[v].version) { + /* Exact match so stop. */ + issues = product->map[v].issues; + break; + } + + /* Check whether this is a candidate for most recent + known version not later than the actual + version. */ + if ((version > product->map[v].version) && + (product->map[v].version >= fallback_version)) { +#if MALI_CUSTOMER_RELEASE + /* Match on version's major and minor fields */ + if (((version ^ product->map[v].version) >> + GPU_ID2_VERSION_MINOR_SHIFT) == 0) +#endif + { + fallback_version = product->map[v].version; + fallback_issues = product->map[v].issues; + } + } + } + + if ((issues == NULL) && (fallback_issues != NULL)) { + /* Fall back to the issue set of the most recent known + version not later than the actual version. */ + issues = fallback_issues; + +#if MALI_CUSTOMER_RELEASE + dev_warn(kbdev->dev, + "GPU hardware issue table may need updating:\n" +#else + dev_info(kbdev->dev, +#endif + "r%dp%d status %d is unknown; treating as r%dp%d status %d", + (gpu_id & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT, + (fallback_version & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (fallback_version & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (fallback_version & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT); + + gpu_id &= ~GPU_ID2_VERSION; + gpu_id |= fallback_version; + kbdev->gpu_props.props.raw_props.gpu_id = gpu_id; + + kbase_gpuprops_update_core_props_gpu_id( + &kbdev->gpu_props.props); + } + } + return issues; +} + +int kbase_hw_set_issues_mask(struct kbase_device *kbdev) +{ + const enum base_hw_issue *issues; + u32 gpu_id; + u32 impl_tech; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + impl_tech = kbdev->gpu_props.props.thread_props.impl_tech; + + if (impl_tech != IMPLEMENTATION_MODEL) { + issues = kbase_hw_get_issues_for_new_id(kbdev); + if (issues == NULL) { + dev_err(kbdev->dev, + "Unknown GPU ID %x", gpu_id); + return -EINVAL; + } + +#if !MALI_CUSTOMER_RELEASE + /* The GPU ID might have been replaced with the last + known version of the same GPU. */ + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; +#endif + } else { + /* Software model */ + switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { + case GPU_ID2_PRODUCT_TMIX: + issues = base_hw_issues_model_tMIx; + break; + case GPU_ID2_PRODUCT_THEX: + issues = base_hw_issues_model_tHEx; + break; + case GPU_ID2_PRODUCT_TSIX: + issues = base_hw_issues_model_tSIx; + break; + case GPU_ID2_PRODUCT_TDVX: + issues = base_hw_issues_model_tDVx; + break; + case GPU_ID2_PRODUCT_TNOX: + issues = base_hw_issues_model_tNOx; + break; + case GPU_ID2_PRODUCT_TGOX: + issues = base_hw_issues_model_tGOx; + break; + case GPU_ID2_PRODUCT_TTRX: + issues = base_hw_issues_model_tTRx; + break; + case GPU_ID2_PRODUCT_TNAX: + issues = base_hw_issues_model_tNAx; + break; + case GPU_ID2_PRODUCT_LBEX: + case GPU_ID2_PRODUCT_TBEX: + issues = base_hw_issues_model_tBEx; + break; + case GPU_ID2_PRODUCT_TBAX: + issues = base_hw_issues_model_tBAx; + break; + case GPU_ID2_PRODUCT_TDUX: + issues = base_hw_issues_model_tDUx; + break; + case GPU_ID2_PRODUCT_TODX: + case GPU_ID2_PRODUCT_LODX: + issues = base_hw_issues_model_tODx; + break; + case GPU_ID2_PRODUCT_TGRX: + issues = base_hw_issues_model_tGRx; + break; + case GPU_ID2_PRODUCT_TVAX: + issues = base_hw_issues_model_tVAx; + break; + case GPU_ID2_PRODUCT_TTUX: + case GPU_ID2_PRODUCT_LTUX: + issues = base_hw_issues_model_tTUx; + break; + case GPU_ID2_PRODUCT_TE2X: + issues = base_hw_issues_model_tE2x; + break; + default: + dev_err(kbdev->dev, + "Unknown GPU ID %x", gpu_id); + return -EINVAL; + } + } + + dev_info(kbdev->dev, + "GPU identified as 0x%x arch %d.%d.%d r%dp%d status %d", + (gpu_id & GPU_ID2_PRODUCT_MAJOR) >> + GPU_ID2_PRODUCT_MAJOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_MAJOR) >> + GPU_ID2_ARCH_MAJOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_MINOR) >> + GPU_ID2_ARCH_MINOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_REV) >> + GPU_ID2_ARCH_REV_SHIFT, + (gpu_id & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT); + + for (; *issues != BASE_HW_ISSUE_END; issues++) + set_bit(*issues, &kbdev->hw_issues_mask[0]); + + return 0; +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hw.h b/drivers/gpu/arm/bifrost/mali_kbase_hw.h new file mode 100755 index 000000000000..f386b1624317 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hw.h @@ -0,0 +1,70 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file + * Run-time work-arounds helpers + */ + +#ifndef _KBASE_HW_H_ +#define _KBASE_HW_H_ + +#include "mali_kbase_defs.h" + +/** + * @brief Tell whether a work-around should be enabled + */ +#define kbase_hw_has_issue(kbdev, issue)\ + test_bit(issue, &(kbdev)->hw_issues_mask[0]) + +/** + * @brief Tell whether a feature is supported + */ +#define kbase_hw_has_feature(kbdev, feature)\ + test_bit(feature, &(kbdev)->hw_features_mask[0]) + +/** + * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID + * @kbdev: Device pointer + * + * Return: 0 if the GPU ID was recognized, otherwise -EINVAL. + * + * The GPU ID is read from the @kbdev. + * + * In debugging versions of the driver, unknown versions of a known GPU with a + * new-format ID will be treated as the most recent known version not later + * than the actual version. In such circumstances, the GPU ID in @kbdev will + * also be replaced with the most recent known version. + * + * Note: The GPU configuration must have been read by + * kbase_gpuprops_get_props() before calling this function. + */ +int kbase_hw_set_issues_mask(struct kbase_device *kbdev); + +/** + * @brief Set the features mask depending on the GPU ID + */ +void kbase_hw_set_features_mask(struct kbase_device *kbdev); + +#endif /* _KBASE_HW_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_backend.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_backend.h new file mode 100755 index 000000000000..89df2519ab97 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_backend.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2014-2015, 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * HW access backend common APIs + */ + +#ifndef _KBASE_HWACCESS_BACKEND_H_ +#define _KBASE_HWACCESS_BACKEND_H_ + +/** + * kbase_backend_devfreq_init - Perform backend devfreq related initialization. + * @kbdev: Device pointer + * + * Return: 0 on success, or an error code on failure. + */ +int kbase_backend_devfreq_init(struct kbase_device *kbdev); + +/** + * kbase_backend_devfreq_term - Perform backend-devfreq termination. + * @kbdev: Device pointer + */ +void kbase_backend_devfreq_term(struct kbase_device *kbdev); + +#endif /* _KBASE_HWACCESS_BACKEND_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h new file mode 100755 index 000000000000..124a2d9cf0c3 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h @@ -0,0 +1,51 @@ +/* + * + * (C) COPYRIGHT 2014, 2016, 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/** + * @file mali_kbase_hwaccess_gpu_defs.h + * HW access common definitions + */ + +#ifndef _KBASE_HWACCESS_DEFS_H_ +#define _KBASE_HWACCESS_DEFS_H_ + +#include + +/** + * struct kbase_hwaccess_data - object encapsulating the GPU backend specific + * data for the HW access layer. + * hwaccess_lock (a spinlock) must be held when + * accessing this structure. + * @active_kctx: pointer to active kbase context which last submitted an + * atom to GPU and while the context is active it can + * submit new atoms to GPU from the irq context also, without + * going through the bottom half of job completion path. + * @backend: GPU backend specific data for HW access layer + */ +struct kbase_hwaccess_data { + struct kbase_context *active_kctx[BASE_JM_MAX_NR_SLOTS]; + + struct kbase_backend_data backend; +}; + +#endif /* _KBASE_HWACCESS_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_gpuprops.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_gpuprops.h new file mode 100755 index 000000000000..3ae0dbe6886d --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_gpuprops.h @@ -0,0 +1,87 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * (C) COPYRIGHT 2014-2015, 2018, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +/** + * Base kernel property query backend APIs + */ + +#ifndef _KBASE_HWACCESS_GPUPROPS_H_ +#define _KBASE_HWACCESS_GPUPROPS_H_ + +/** + * kbase_backend_gpuprops_get() - Fill @regdump with GPU properties read from + * GPU + * @kbdev: Device pointer + * @regdump: Pointer to struct kbase_gpuprops_regdump structure + * + * The caller should ensure that GPU remains powered-on during this function. + * + * Return: Zero for succeess or a Linux error code + */ +int kbase_backend_gpuprops_get(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump); + +/** + * kbase_backend_gpuprops_get_features - Fill @regdump with GPU properties read + * from GPU + * @kbdev: Device pointer + * @regdump: Pointer to struct kbase_gpuprops_regdump structure + * + * This function reads GPU properties that are dependent on the hardware + * features bitmask. It will power-on the GPU if required. + * + * Return: Zero for succeess or a Linux error code + */ +int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump); + +/** + * kbase_backend_gpuprops_get_l2_features - Fill @regdump with L2_FEATURES read + * from GPU + * @kbdev: Device pointer + * @regdump: Pointer to struct kbase_gpuprops_regdump structure + * + * This function reads L2_FEATURES register that is dependent on the hardware + * features bitmask. It will power-on the GPU if required. + * + * Return: Zero on success, Linux error code on failure + */ +int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump); + + +#endif /* _KBASE_HWACCESS_GPUPROPS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_instr.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_instr.h new file mode 100755 index 000000000000..4fd2e3549268 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_instr.h @@ -0,0 +1,151 @@ +/* + * + * (C) COPYRIGHT 2014-2015, 2017-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * HW Access instrumentation common APIs + */ + +#ifndef _KBASE_HWACCESS_INSTR_H_ +#define _KBASE_HWACCESS_INSTR_H_ + +#include + +/** + * struct kbase_instr_hwcnt_enable - Enable hardware counter collection. + * @dump_buffer: GPU address to write counters to. + * @dump_buffer_bytes: Size in bytes of the buffer pointed to by dump_buffer. + * @fe_bm: counters selection bitmask (Front End). + * @shader_bm: counters selection bitmask (Shader). + * @tiler_bm: counters selection bitmask (Tiler). + * @mmu_l2_bm: counters selection bitmask (MMU_L2). + * @use_secondary: use secondary performance counters set for applicable + * counter blocks. + */ +struct kbase_instr_hwcnt_enable { + u64 dump_buffer; + u64 dump_buffer_bytes; + u32 fe_bm; + u32 shader_bm; + u32 tiler_bm; + u32 mmu_l2_bm; + bool use_secondary; +}; + +/** + * kbase_instr_hwcnt_enable_internal() - Enable HW counters collection + * @kbdev: Kbase device + * @kctx: Kbase context + * @enable: HW counter setup parameters + * + * Context: might sleep, waiting for reset to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_instr_hwcnt_enable *enable); + +/** + * kbase_instr_hwcnt_disable_internal() - Disable HW counters collection + * @kctx: Kbase context + * + * Context: might sleep, waiting for an ongoing dump to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx); + +/** + * kbase_instr_hwcnt_request_dump() - Request HW counter dump from GPU + * @kctx: Kbase context + * + * Caller must either wait for kbase_instr_hwcnt_dump_complete() to return true, + * of call kbase_instr_hwcnt_wait_for_dump(). + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx); + +/** + * kbase_instr_hwcnt_wait_for_dump() - Wait until pending HW counter dump has + * completed. + * @kctx: Kbase context + * + * Context: will sleep, waiting for dump to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx); + +/** + * kbase_instr_hwcnt_dump_complete - Tell whether the HW counters dump has + * completed + * @kctx: Kbase context + * @success: Set to true if successful + * + * Context: does not sleep. + * + * Return: true if the dump is complete + */ +bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, + bool * const success); + +/** + * kbase_instr_hwcnt_clear() - Clear HW counters + * @kctx: Kbase context + * + * Context: might sleep, waiting for reset to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_clear(struct kbase_context *kctx); + +/** + * kbase_instr_backend_init() - Initialise the instrumentation backend + * @kbdev: Kbase device + * + * This function should be called during driver initialization. + * + * Return: 0 on success + */ +int kbase_instr_backend_init(struct kbase_device *kbdev); + +/** + * kbase_instr_backend_init() - Terminate the instrumentation backend + * @kbdev: Kbase device + * + * This function should be called during driver termination. + */ +void kbase_instr_backend_term(struct kbase_device *kbdev); + +#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS +/** + * kbase_instr_backend_debugfs_init() - Add a debugfs entry for the + * hardware counter set. + * @kbdev: kbase device + */ +void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev); +#endif + +#endif /* _KBASE_HWACCESS_INSTR_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h new file mode 100755 index 000000000000..f6ce17e4180f --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h @@ -0,0 +1,304 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * HW access job manager common APIs + */ + +#ifndef _KBASE_HWACCESS_JM_H_ +#define _KBASE_HWACCESS_JM_H_ + +/** + * kbase_backend_run_atom() - Run an atom on the GPU + * @kbdev: Device pointer + * @atom: Atom to run + * + * Caller must hold the HW access lock + */ +void kbase_backend_run_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_backend_slot_update - Update state based on slot ringbuffers + * + * @kbdev: Device pointer + * + * Inspect the jobs in the slot ringbuffers and update state. + * + * This will cause jobs to be submitted to hardware if they are unblocked + */ +void kbase_backend_slot_update(struct kbase_device *kbdev); + +/** + * kbase_backend_find_and_release_free_address_space() - Release a free AS + * @kbdev: Device pointer + * @kctx: Context pointer + * + * This function can evict an idle context from the runpool, freeing up the + * address space it was using. + * + * The address space is marked as in use. The caller must either assign a + * context using kbase_gpu_use_ctx(), or release it using + * kbase_ctx_sched_release() + * + * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none + * available + */ +int kbase_backend_find_and_release_free_address_space( + struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the + * provided address space. + * @kbdev: Device pointer + * @kctx: Context pointer. May be NULL + * @as_nr: Free address space to use + * + * kbase_gpu_next_job() will pull atoms from the active context. + * + * Return: true if successful, false if ASID not assigned. + */ +bool kbase_backend_use_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, + int as_nr); + +/** + * kbase_backend_use_ctx_sched() - Activate a context. + * @kbdev: Device pointer + * @kctx: Context pointer + * @js: Job slot to activate context on + * + * kbase_gpu_next_job() will pull atoms from the active context. + * + * The context must already be scheduled and assigned to an address space. If + * the context is not scheduled, then kbase_gpu_use_ctx() should be used + * instead. + * + * Caller must hold hwaccess_lock + * + * Return: true if context is now active, false otherwise (ie if context does + * not have an address space assigned) + */ +bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, + struct kbase_context *kctx, int js); + +/** + * kbase_backend_release_ctx_irq - Release a context from the GPU. This will + * de-assign the assigned address space. + * @kbdev: Device pointer + * @kctx: Context pointer + * + * Caller must hold kbase_device->mmu_hw_mutex and hwaccess_lock + */ +void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbase_backend_release_ctx_noirq - Release a context from the GPU. This will + * de-assign the assigned address space. + * @kbdev: Device pointer + * @kctx: Context pointer + * + * Caller must hold kbase_device->mmu_hw_mutex + * + * This function must perform any operations that could not be performed in IRQ + * context by kbase_backend_release_ctx_irq(). + */ +void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbase_backend_cache_clean - Perform a cache clean if the given atom requires + * one + * @kbdev: Device pointer + * @katom: Pointer to the failed atom + * + * On some GPUs, the GPU cache must be cleaned following a failed atom. This + * function performs a clean if it is required by @katom. + */ +void kbase_backend_cache_clean(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + + +/** + * kbase_backend_complete_wq() - Perform backend-specific actions required on + * completing an atom. + * @kbdev: Device pointer + * @katom: Pointer to the atom to complete + * + * This function should only be called from kbase_jd_done_worker() or + * js_return_worker(). + * + * Return: true if atom has completed, false if atom should be re-submitted + */ +void kbase_backend_complete_wq(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +#if !MALI_USE_CSF +/** + * kbase_backend_complete_wq_post_sched - Perform backend-specific actions + * required on completing an atom, after + * any scheduling has taken place. + * @kbdev: Device pointer + * @core_req: Core requirements of atom + * + * This function should only be called from kbase_jd_done_worker() or + * js_return_worker(). + */ +void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, + base_jd_core_req core_req); +#endif /* !MALI_USE_CSF */ + +/** + * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU + * and remove any others from the ringbuffers. + * @kbdev: Device pointer + * @end_timestamp: Timestamp of reset + */ +void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp); + +/** + * kbase_backend_inspect_tail - Return the atom currently at the tail of slot + * @js + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Atom currently at the head of slot @js, or NULL + */ +struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, + int js); + +/** + * kbase_backend_nr_atoms_on_slot() - Return the number of atoms currently on a + * slot. + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Number of atoms currently on slot + */ +int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js); + +/** + * kbase_backend_nr_atoms_submitted() - Return the number of atoms on a slot + * that are currently on the GPU. + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Number of atoms currently on slot @js that are currently on the GPU. + */ +int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js); + +/** + * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs + * has changed. + * @kbdev: Device pointer + * + * Perform any required backend-specific actions (eg starting/stopping + * scheduling timers). + */ +void kbase_backend_ctx_count_changed(struct kbase_device *kbdev); + +/** + * kbase_backend_timeouts_changed() - Job Scheduler timeouts have changed. + * @kbdev: Device pointer + * + * Perform any required backend-specific actions (eg updating timeouts of + * currently running atoms). + */ +void kbase_backend_timeouts_changed(struct kbase_device *kbdev); + +/** + * kbase_backend_slot_free() - Return the number of jobs that can be currently + * submitted to slot @js. + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Number of jobs that can be submitted. + */ +int kbase_backend_slot_free(struct kbase_device *kbdev, int js); + +/** + * kbase_job_check_enter_disjoint - potentially leave disjoint state + * @kbdev: kbase device + * @target_katom: atom which is finishing + * + * Work out whether to leave disjoint state when finishing an atom that was + * originated by kbase_job_check_enter_disjoint(). + */ +void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, + struct kbase_jd_atom *target_katom); + +/** + * kbase_backend_jm_kill_running_jobs_from_kctx - Kill all jobs that are + * currently running on GPU from a context + * @kctx: Context pointer + * + * This is used in response to a page fault to remove all jobs from the faulting + * context from the hardware. + * + * Caller must hold hwaccess_lock. + */ +void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx); + +/** + * kbase_jm_wait_for_zero_jobs - Wait for context to have zero jobs running, and + * to be descheduled. + * @kctx: Context pointer + * + * This should be called following kbase_js_zap_context(), to ensure the context + * can be safely destroyed. + */ +void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx); + +/** + * kbase_backend_get_current_flush_id - Return the current flush ID + * + * @kbdev: Device pointer + * + * Return: the current flush ID to be recorded for each job chain + */ +u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev); + +/** + * kbase_job_slot_hardstop - Hard-stop the specified job slot + * @kctx: The kbase context that contains the job(s) that should + * be hard-stopped + * @js: The job slot to hard-stop + * @target_katom: The job that should be hard-stopped (or NULL for all + * jobs from the context) + * Context: + * The job slot lock must be held when calling this function. + */ +void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom); + +/** + * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms + * currently on the GPU + * @kbdev: Device pointer + * + * Return: true if there are any atoms on the GPU, false otherwise + */ +bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev); + +#endif /* _KBASE_HWACCESS_JM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h new file mode 100755 index 000000000000..bbaf6eaf8d88 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h @@ -0,0 +1,229 @@ +/* + * + * (C) COPYRIGHT 2014-2015, 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/** + * @file mali_kbase_hwaccess_pm.h + * HW access power manager common APIs + */ + +#ifndef _KBASE_HWACCESS_PM_H_ +#define _KBASE_HWACCESS_PM_H_ + +#include +#include + +#include + +/* Forward definition - see mali_kbase.h */ +struct kbase_device; + +/* Functions common to all HW access backends */ + +/** + * Initialize the power management framework. + * + * Must be called before any other power management function + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 if the power management framework was successfully initialized. + */ +int kbase_hwaccess_pm_init(struct kbase_device *kbdev); + +/** + * Terminate the power management framework. + * + * No power management functions may be called after this + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_hwaccess_pm_term(struct kbase_device *kbdev); + +/** + * kbase_hwaccess_pm_powerup - Power up the GPU. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @flags: Flags to pass on to kbase_pm_init_hw + * + * Power up GPU after all modules have been initialized and interrupt handlers + * installed. + * + * Return: 0 if powerup was successful. + */ +int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, + unsigned int flags); + +/** + * Halt the power management framework. + * + * Should ensure that no new interrupts are generated, but allow any currently + * running interrupt handlers to complete successfully. The GPU is forced off by + * the time this function returns, regardless of whether or not the active power + * policy asks for the GPU to be powered off. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_halt(struct kbase_device *kbdev); + +/** + * Perform any backend-specific actions to suspend the GPU + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev); + +/** + * Perform any backend-specific actions to resume the GPU from a suspend + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_resume(struct kbase_device *kbdev); + +/** + * Perform any required actions for activating the GPU. Called when the first + * context goes active. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); + +/** + * Perform any required actions for idling the GPU. Called when the last + * context goes idle. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); + + +/** + * Set the debug core mask. + * + * This determines which cores the power manager is allowed to use. + * + * @param kbdev The kbase device structure for the device (must be a + * valid pointer) + * @param new_core_mask_js0 The core mask to use for job slot 0 + * @param new_core_mask_js0 The core mask to use for job slot 1 + * @param new_core_mask_js0 The core mask to use for job slot 2 + */ +void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, + u64 new_core_mask_js0, u64 new_core_mask_js1, + u64 new_core_mask_js2); + + +/** + * Get the current policy. + * + * Returns the policy that is currently active. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * + * @return The current policy + */ +const struct kbase_pm_ca_policy +*kbase_pm_ca_get_policy(struct kbase_device *kbdev); + +/** + * Change the policy to the one specified. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * @param policy The policy to change to (valid pointer returned from + * @ref kbase_pm_ca_list_policies) + */ +void kbase_pm_ca_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_ca_policy *policy); + +/** + * Retrieve a static list of the available policies. + * + * @param[out] policies An array pointer to take the list of policies. This may + * be NULL. The contents of this array must not be + * modified. + * + * @return The number of policies + */ +int +kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **policies); + + +/** + * Get the current policy. + * + * Returns the policy that is currently active. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * + * @return The current policy + */ +const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev); + +/** + * Change the policy to the one specified. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * @param policy The policy to change to (valid pointer returned from + * @ref kbase_pm_list_policies) + */ +void kbase_pm_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_policy *policy); + +/** + * kbase_pm_list_policies - Retrieve a static list of the available policies. + * + * @kbdev: The kbase device structure for the device. + * @list: An array pointer to take the list of policies. This may be NULL. + * The contents of this array must not be modified. + * + * Return: The number of policies + */ +int kbase_pm_list_policies(struct kbase_device *kbdev, + const struct kbase_pm_policy * const **list); + +/** + * kbase_protected_most_enable - Enable protected mode + * + * @kbdev: Address of the instance of a GPU platform device. + * + * Return: Zero on success or an error code + */ +int kbase_pm_protected_mode_enable(struct kbase_device *kbdev); + +/** + * kbase_protected_mode_disable - Disable protected mode + * + * @kbdev: Address of the instance of a GPU platform device. + * + * Return: Zero on success or an error code + */ +int kbase_pm_protected_mode_disable(struct kbase_device *kbdev); + +#endif /* _KBASE_HWACCESS_PM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h new file mode 100755 index 000000000000..94b7551b865e --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h @@ -0,0 +1,56 @@ +/* + * + * (C) COPYRIGHT 2014,2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/** + * + */ + +#ifndef _KBASE_BACKEND_TIME_H_ +#define _KBASE_BACKEND_TIME_H_ + +/** + * kbase_backend_get_gpu_time() - Get current GPU time + * @kbdev: Device pointer + * @cycle_counter: Pointer to u64 to store cycle counter in + * @system_time: Pointer to u64 to store system time in + * @ts: Pointer to struct timespec to store current monotonic + * time in + */ +void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, + u64 *system_time, struct timespec64 *ts); + +/** + * kbase_backend_get_gpu_time_norequest() - Get current GPU time without + * request/release cycle counter + * @kbdev: Device pointer + * @cycle_counter: Pointer to u64 to store cycle counter in + * @system_time: Pointer to u64 to store system time in + * @ts: Pointer to struct timespec to store current monotonic + * time in + */ +void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev, + u64 *cycle_counter, + u64 *system_time, + struct timespec64 *ts); + +#endif /* _KBASE_BACKEND_TIME_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt.c new file mode 100755 index 000000000000..2708af78b292 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt.c @@ -0,0 +1,794 @@ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Implementation of hardware counter context and accumulator APIs. + */ + +#include "mali_kbase_hwcnt_context.h" +#include "mali_kbase_hwcnt_accumulator.h" +#include "mali_kbase_hwcnt_backend.h" +#include "mali_kbase_hwcnt_types.h" +#include "mali_malisw.h" +#include "mali_kbase_debug.h" +#include "mali_kbase_linux.h" + +#include +#include +#include + +/** + * enum kbase_hwcnt_accum_state - Hardware counter accumulator states. + * @ACCUM_STATE_ERROR: Error state, where all accumulator operations fail. + * @ACCUM_STATE_DISABLED: Disabled state, where dumping is always disabled. + * @ACCUM_STATE_ENABLED: Enabled state, where dumping is enabled if there are + * any enabled counters. + */ +enum kbase_hwcnt_accum_state { + ACCUM_STATE_ERROR, + ACCUM_STATE_DISABLED, + ACCUM_STATE_ENABLED +}; + +/** + * struct kbase_hwcnt_accumulator - Hardware counter accumulator structure. + * @backend: Pointer to created counter backend. + * @state: The current state of the accumulator. + * - State transition from disabled->enabled or + * disabled->error requires state_lock. + * - State transition from enabled->disabled or + * enabled->error requires both accum_lock and + * state_lock. + * - Error state persists until next disable. + * @enable_map: The current set of enabled counters. + * - Must only be modified while holding both + * accum_lock and state_lock. + * - Can be read while holding either lock. + * - Must stay in sync with enable_map_any_enabled. + * @enable_map_any_enabled: True if any counters in the map are enabled, else + * false. If true, and state is ACCUM_STATE_ENABLED, + * then the counter backend will be enabled. + * - Must only be modified while holding both + * accum_lock and state_lock. + * - Can be read while holding either lock. + * - Must stay in sync with enable_map. + * @scratch_map: Scratch enable map, used as temporary enable map + * storage during dumps. + * - Must only be read or modified while holding + * accum_lock. + * @accum_buf: Accumulation buffer, where dumps will be accumulated + * into on transition to a disable state. + * - Must only be read or modified while holding + * accum_lock. + * @accumulated: True if the accumulation buffer has been accumulated + * into and not subsequently read from yet, else false. + * - Must only be read or modified while holding + * accum_lock. + * @ts_last_dump_ns: Timestamp (ns) of the end time of the most recent + * dump that was requested by the user. + * - Must only be read or modified while holding + * accum_lock. + */ +struct kbase_hwcnt_accumulator { + struct kbase_hwcnt_backend *backend; + enum kbase_hwcnt_accum_state state; + struct kbase_hwcnt_enable_map enable_map; + bool enable_map_any_enabled; + struct kbase_hwcnt_enable_map scratch_map; + struct kbase_hwcnt_dump_buffer accum_buf; + bool accumulated; + u64 ts_last_dump_ns; +}; + +/** + * struct kbase_hwcnt_context - Hardware counter context structure. + * @iface: Pointer to hardware counter backend interface. + * @state_lock: Spinlock protecting state. + * @disable_count: Disable count of the context. Initialised to 1. + * Decremented when the accumulator is acquired, and incremented + * on release. Incremented on calls to + * kbase_hwcnt_context_disable[_atomic], and decremented on + * calls to kbase_hwcnt_context_enable. + * - Must only be read or modified while holding state_lock. + * @accum_lock: Mutex protecting accumulator. + * @accum_inited: Flag to prevent concurrent accumulator initialisation and/or + * termination. Set to true before accumulator initialisation, + * and false after accumulator termination. + * - Must only be modified while holding both accum_lock and + * state_lock. + * - Can be read while holding either lock. + * @accum: Hardware counter accumulator structure. + */ +struct kbase_hwcnt_context { + const struct kbase_hwcnt_backend_interface *iface; + spinlock_t state_lock; + size_t disable_count; + struct mutex accum_lock; + bool accum_inited; + struct kbase_hwcnt_accumulator accum; +}; + +int kbase_hwcnt_context_init( + const struct kbase_hwcnt_backend_interface *iface, + struct kbase_hwcnt_context **out_hctx) +{ + struct kbase_hwcnt_context *hctx = NULL; + + if (!iface || !out_hctx) + return -EINVAL; + + hctx = kzalloc(sizeof(*hctx), GFP_KERNEL); + if (!hctx) + return -ENOMEM; + + hctx->iface = iface; + spin_lock_init(&hctx->state_lock); + hctx->disable_count = 1; + mutex_init(&hctx->accum_lock); + hctx->accum_inited = false; + + *out_hctx = hctx; + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_context_init); + +void kbase_hwcnt_context_term(struct kbase_hwcnt_context *hctx) +{ + if (!hctx) + return; + + /* Make sure we didn't leak the accumulator */ + WARN_ON(hctx->accum_inited); + kfree(hctx); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_context_term); + +/** + * kbasep_hwcnt_accumulator_term() - Terminate the accumulator for the context. + * @hctx: Non-NULL pointer to hardware counter context. + */ +static void kbasep_hwcnt_accumulator_term(struct kbase_hwcnt_context *hctx) +{ + WARN_ON(!hctx); + WARN_ON(!hctx->accum_inited); + + kbase_hwcnt_enable_map_free(&hctx->accum.scratch_map); + kbase_hwcnt_dump_buffer_free(&hctx->accum.accum_buf); + kbase_hwcnt_enable_map_free(&hctx->accum.enable_map); + hctx->iface->term(hctx->accum.backend); + memset(&hctx->accum, 0, sizeof(hctx->accum)); +} + +/** + * kbasep_hwcnt_accumulator_init() - Initialise the accumulator for the context. + * @hctx: Non-NULL pointer to hardware counter context. + * + * Return: 0 on success, else error code. + */ +static int kbasep_hwcnt_accumulator_init(struct kbase_hwcnt_context *hctx) +{ + int errcode; + + WARN_ON(!hctx); + WARN_ON(!hctx->accum_inited); + + errcode = hctx->iface->init( + hctx->iface->info, &hctx->accum.backend); + if (errcode) + goto error; + + hctx->accum.state = ACCUM_STATE_ERROR; + + errcode = kbase_hwcnt_enable_map_alloc( + hctx->iface->metadata, &hctx->accum.enable_map); + if (errcode) + goto error; + + hctx->accum.enable_map_any_enabled = false; + + errcode = kbase_hwcnt_dump_buffer_alloc( + hctx->iface->metadata, &hctx->accum.accum_buf); + if (errcode) + goto error; + + errcode = kbase_hwcnt_enable_map_alloc( + hctx->iface->metadata, &hctx->accum.scratch_map); + if (errcode) + goto error; + + hctx->accum.accumulated = false; + + hctx->accum.ts_last_dump_ns = + hctx->iface->timestamp_ns(hctx->accum.backend); + + return 0; + +error: + kbasep_hwcnt_accumulator_term(hctx); + return errcode; +} + +/** + * kbasep_hwcnt_accumulator_disable() - Transition the accumulator into the + * disabled state, from the enabled or + * error states. + * @hctx: Non-NULL pointer to hardware counter context. + * @accumulate: True if we should accumulate before disabling, else false. + */ +static void kbasep_hwcnt_accumulator_disable( + struct kbase_hwcnt_context *hctx, bool accumulate) +{ + int errcode = 0; + bool backend_enabled = false; + struct kbase_hwcnt_accumulator *accum; + unsigned long flags; + u64 dump_time_ns; + + WARN_ON(!hctx); + lockdep_assert_held(&hctx->accum_lock); + WARN_ON(!hctx->accum_inited); + + accum = &hctx->accum; + + spin_lock_irqsave(&hctx->state_lock, flags); + + WARN_ON(hctx->disable_count != 0); + WARN_ON(hctx->accum.state == ACCUM_STATE_DISABLED); + + if ((hctx->accum.state == ACCUM_STATE_ENABLED) && + (accum->enable_map_any_enabled)) + backend_enabled = true; + + if (!backend_enabled) + hctx->accum.state = ACCUM_STATE_DISABLED; + + spin_unlock_irqrestore(&hctx->state_lock, flags); + + /* Early out if the backend is not already enabled */ + if (!backend_enabled) + return; + + if (!accumulate) + goto disable; + + /* Try and accumulate before disabling */ + errcode = hctx->iface->dump_request(accum->backend, &dump_time_ns); + if (errcode) + goto disable; + + errcode = hctx->iface->dump_wait(accum->backend); + if (errcode) + goto disable; + + errcode = hctx->iface->dump_get(accum->backend, + &accum->accum_buf, &accum->enable_map, accum->accumulated); + if (errcode) + goto disable; + + accum->accumulated = true; + +disable: + hctx->iface->dump_disable(accum->backend); + + /* Regardless of any errors during the accumulate, put the accumulator + * in the disabled state. + */ + spin_lock_irqsave(&hctx->state_lock, flags); + + hctx->accum.state = ACCUM_STATE_DISABLED; + + spin_unlock_irqrestore(&hctx->state_lock, flags); +} + +/** + * kbasep_hwcnt_accumulator_enable() - Transition the accumulator into the + * enabled state, from the disabled state. + * @hctx: Non-NULL pointer to hardware counter context. + */ +static void kbasep_hwcnt_accumulator_enable(struct kbase_hwcnt_context *hctx) +{ + int errcode = 0; + struct kbase_hwcnt_accumulator *accum; + + WARN_ON(!hctx); + lockdep_assert_held(&hctx->state_lock); + WARN_ON(!hctx->accum_inited); + WARN_ON(hctx->accum.state != ACCUM_STATE_DISABLED); + + accum = &hctx->accum; + + /* The backend only needs enabling if any counters are enabled */ + if (accum->enable_map_any_enabled) + errcode = hctx->iface->dump_enable_nolock( + accum->backend, &accum->enable_map); + + if (!errcode) + accum->state = ACCUM_STATE_ENABLED; + else + accum->state = ACCUM_STATE_ERROR; +} + +/** + * kbasep_hwcnt_accumulator_dump() - Perform a dump with the most up-to-date + * values of enabled counters possible, and + * optionally update the set of enabled + * counters. + * @hctx : Non-NULL pointer to the hardware counter context + * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will + * be written out to on success + * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will + * be written out to on success + * @dump_buf: Pointer to the buffer where the dump will be written out to on + * success. If non-NULL, must have the same metadata as the + * accumulator. If NULL, the dump will be discarded. + * @new_map: Pointer to the new counter enable map. If non-NULL, must have + * the same metadata as the accumulator. If NULL, the set of + * enabled counters will be unchanged. + */ +static int kbasep_hwcnt_accumulator_dump( + struct kbase_hwcnt_context *hctx, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf, + const struct kbase_hwcnt_enable_map *new_map) +{ + int errcode = 0; + unsigned long flags; + enum kbase_hwcnt_accum_state state; + bool dump_requested = false; + bool dump_written = false; + bool cur_map_any_enabled; + struct kbase_hwcnt_enable_map *cur_map; + bool new_map_any_enabled = false; + u64 dump_time_ns; + struct kbase_hwcnt_accumulator *accum; + + WARN_ON(!hctx); + WARN_ON(!ts_start_ns); + WARN_ON(!ts_end_ns); + WARN_ON(dump_buf && (dump_buf->metadata != hctx->iface->metadata)); + WARN_ON(new_map && (new_map->metadata != hctx->iface->metadata)); + WARN_ON(!hctx->accum_inited); + lockdep_assert_held(&hctx->accum_lock); + + accum = &hctx->accum; + cur_map = &accum->scratch_map; + + /* Save out info about the current enable map */ + cur_map_any_enabled = accum->enable_map_any_enabled; + kbase_hwcnt_enable_map_copy(cur_map, &accum->enable_map); + + if (new_map) + new_map_any_enabled = + kbase_hwcnt_enable_map_any_enabled(new_map); + + /* + * We're holding accum_lock, so the accumulator state might transition + * from disabled to enabled during this function (as enabling is lock + * free), but it will never disable (as disabling needs to hold the + * accum_lock), nor will it ever transition from enabled to error (as + * an enable while we're already enabled is impossible). + * + * If we're already disabled, we'll only look at the accumulation buffer + * rather than do a real dump, so a concurrent enable does not affect + * us. + * + * If a concurrent enable fails, we might transition to the error + * state, but again, as we're only looking at the accumulation buffer, + * it's not an issue. + */ + spin_lock_irqsave(&hctx->state_lock, flags); + + state = accum->state; + + /* + * Update the new map now, such that if an enable occurs during this + * dump then that enable will set the new map. If we're already enabled, + * then we'll do it ourselves after the dump. + */ + if (new_map) { + kbase_hwcnt_enable_map_copy( + &accum->enable_map, new_map); + accum->enable_map_any_enabled = new_map_any_enabled; + } + + spin_unlock_irqrestore(&hctx->state_lock, flags); + + /* Error state, so early out. No need to roll back any map updates */ + if (state == ACCUM_STATE_ERROR) + return -EIO; + + /* Initiate the dump if the backend is enabled. */ + if ((state == ACCUM_STATE_ENABLED) && cur_map_any_enabled) { + if (dump_buf) { + errcode = hctx->iface->dump_request( + accum->backend, &dump_time_ns); + dump_requested = true; + } else { + dump_time_ns = hctx->iface->timestamp_ns( + accum->backend); + errcode = hctx->iface->dump_clear(accum->backend); + } + + if (errcode) + goto error; + } else { + dump_time_ns = hctx->iface->timestamp_ns(accum->backend); + } + + /* Copy any accumulation into the dest buffer */ + if (accum->accumulated && dump_buf) { + kbase_hwcnt_dump_buffer_copy( + dump_buf, &accum->accum_buf, cur_map); + dump_written = true; + } + + /* Wait for any requested dumps to complete */ + if (dump_requested) { + WARN_ON(state != ACCUM_STATE_ENABLED); + errcode = hctx->iface->dump_wait(accum->backend); + if (errcode) + goto error; + } + + /* If we're enabled and there's a new enable map, change the enabled set + * as soon after the dump has completed as possible. + */ + if ((state == ACCUM_STATE_ENABLED) && new_map) { + /* Backend is only enabled if there were any enabled counters */ + if (cur_map_any_enabled) + hctx->iface->dump_disable(accum->backend); + + /* (Re-)enable the backend if the new map has enabled counters. + * No need to acquire the spinlock, as concurrent enable while + * we're already enabled and holding accum_lock is impossible. + */ + if (new_map_any_enabled) { + errcode = hctx->iface->dump_enable( + accum->backend, new_map); + if (errcode) + goto error; + } + } + + /* Copy, accumulate, or zero into the dest buffer to finish */ + if (dump_buf) { + /* If we dumped, copy or accumulate it into the destination */ + if (dump_requested) { + WARN_ON(state != ACCUM_STATE_ENABLED); + errcode = hctx->iface->dump_get( + accum->backend, + dump_buf, + cur_map, + dump_written); + if (errcode) + goto error; + dump_written = true; + } + + /* If we've not written anything into the dump buffer so far, it + * means there was nothing to write. Zero any enabled counters. + */ + if (!dump_written) + kbase_hwcnt_dump_buffer_zero(dump_buf, cur_map); + } + + /* Write out timestamps */ + *ts_start_ns = accum->ts_last_dump_ns; + *ts_end_ns = dump_time_ns; + + accum->accumulated = false; + accum->ts_last_dump_ns = dump_time_ns; + + return 0; +error: + /* An error was only physically possible if the backend was enabled */ + WARN_ON(state != ACCUM_STATE_ENABLED); + + /* Disable the backend, and transition to the error state */ + hctx->iface->dump_disable(accum->backend); + spin_lock_irqsave(&hctx->state_lock, flags); + + accum->state = ACCUM_STATE_ERROR; + + spin_unlock_irqrestore(&hctx->state_lock, flags); + + return errcode; +} + +/** + * kbasep_hwcnt_context_disable() - Increment the disable count of the context. + * @hctx: Non-NULL pointer to hardware counter context. + * @accumulate: True if we should accumulate before disabling, else false. + */ +static void kbasep_hwcnt_context_disable( + struct kbase_hwcnt_context *hctx, bool accumulate) +{ + unsigned long flags; + + WARN_ON(!hctx); + lockdep_assert_held(&hctx->accum_lock); + + if (!kbase_hwcnt_context_disable_atomic(hctx)) { + kbasep_hwcnt_accumulator_disable(hctx, accumulate); + + spin_lock_irqsave(&hctx->state_lock, flags); + + /* Atomic disable failed and we're holding the mutex, so current + * disable count must be 0. + */ + WARN_ON(hctx->disable_count != 0); + hctx->disable_count++; + + spin_unlock_irqrestore(&hctx->state_lock, flags); + } +} + +int kbase_hwcnt_accumulator_acquire( + struct kbase_hwcnt_context *hctx, + struct kbase_hwcnt_accumulator **accum) +{ + int errcode = 0; + unsigned long flags; + + if (!hctx || !accum) + return -EINVAL; + + mutex_lock(&hctx->accum_lock); + spin_lock_irqsave(&hctx->state_lock, flags); + + if (!hctx->accum_inited) + /* Set accum initing now to prevent concurrent init */ + hctx->accum_inited = true; + else + /* Already have an accum, or already being inited */ + errcode = -EBUSY; + + spin_unlock_irqrestore(&hctx->state_lock, flags); + mutex_unlock(&hctx->accum_lock); + + if (errcode) + return errcode; + + errcode = kbasep_hwcnt_accumulator_init(hctx); + + if (errcode) { + mutex_lock(&hctx->accum_lock); + spin_lock_irqsave(&hctx->state_lock, flags); + + hctx->accum_inited = false; + + spin_unlock_irqrestore(&hctx->state_lock, flags); + mutex_unlock(&hctx->accum_lock); + + return errcode; + } + + spin_lock_irqsave(&hctx->state_lock, flags); + + WARN_ON(hctx->disable_count == 0); + WARN_ON(hctx->accum.enable_map_any_enabled); + + /* Decrement the disable count to allow the accumulator to be accessible + * now that it's fully constructed. + */ + hctx->disable_count--; + + /* + * Make sure the accumulator is initialised to the correct state. + * Regardless of initial state, counters don't need to be enabled via + * the backend, as the initial enable map has no enabled counters. + */ + hctx->accum.state = (hctx->disable_count == 0) ? + ACCUM_STATE_ENABLED : + ACCUM_STATE_DISABLED; + + spin_unlock_irqrestore(&hctx->state_lock, flags); + + *accum = &hctx->accum; + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_acquire); + +void kbase_hwcnt_accumulator_release(struct kbase_hwcnt_accumulator *accum) +{ + unsigned long flags; + struct kbase_hwcnt_context *hctx; + + if (!accum) + return; + + hctx = container_of(accum, struct kbase_hwcnt_context, accum); + + mutex_lock(&hctx->accum_lock); + + /* Double release is a programming error */ + WARN_ON(!hctx->accum_inited); + + /* Disable the context to ensure the accumulator is inaccesible while + * we're destroying it. This performs the corresponding disable count + * increment to the decrement done during acquisition. + */ + kbasep_hwcnt_context_disable(hctx, false); + + mutex_unlock(&hctx->accum_lock); + + kbasep_hwcnt_accumulator_term(hctx); + + mutex_lock(&hctx->accum_lock); + spin_lock_irqsave(&hctx->state_lock, flags); + + hctx->accum_inited = false; + + spin_unlock_irqrestore(&hctx->state_lock, flags); + mutex_unlock(&hctx->accum_lock); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_release); + +void kbase_hwcnt_context_disable(struct kbase_hwcnt_context *hctx) +{ + if (WARN_ON(!hctx)) + return; + + /* Try and atomically disable first, so we can avoid locking the mutex + * if we don't need to. + */ + if (kbase_hwcnt_context_disable_atomic(hctx)) + return; + + mutex_lock(&hctx->accum_lock); + + kbasep_hwcnt_context_disable(hctx, true); + + mutex_unlock(&hctx->accum_lock); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_context_disable); + +bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx) +{ + unsigned long flags; + bool atomic_disabled = false; + + if (WARN_ON(!hctx)) + return false; + + spin_lock_irqsave(&hctx->state_lock, flags); + + if (!WARN_ON(hctx->disable_count == SIZE_MAX)) { + /* + * If disable count is non-zero, we can just bump the disable + * count. + * + * Otherwise, we can't disable in an atomic context. + */ + if (hctx->disable_count != 0) { + hctx->disable_count++; + atomic_disabled = true; + } + } + + spin_unlock_irqrestore(&hctx->state_lock, flags); + + return atomic_disabled; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_context_disable_atomic); + +void kbase_hwcnt_context_enable(struct kbase_hwcnt_context *hctx) +{ + unsigned long flags; + + if (WARN_ON(!hctx)) + return; + + spin_lock_irqsave(&hctx->state_lock, flags); + + if (!WARN_ON(hctx->disable_count == 0)) { + if (hctx->disable_count == 1) + kbasep_hwcnt_accumulator_enable(hctx); + + hctx->disable_count--; + } + + spin_unlock_irqrestore(&hctx->state_lock, flags); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_context_enable); + +const struct kbase_hwcnt_metadata *kbase_hwcnt_context_metadata( + struct kbase_hwcnt_context *hctx) +{ + if (!hctx) + return NULL; + + return hctx->iface->metadata; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_context_metadata); + +int kbase_hwcnt_accumulator_set_counters( + struct kbase_hwcnt_accumulator *accum, + const struct kbase_hwcnt_enable_map *new_map, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf) +{ + int errcode; + struct kbase_hwcnt_context *hctx; + + if (!accum || !new_map || !ts_start_ns || !ts_end_ns) + return -EINVAL; + + hctx = container_of(accum, struct kbase_hwcnt_context, accum); + + if ((new_map->metadata != hctx->iface->metadata) || + (dump_buf && (dump_buf->metadata != hctx->iface->metadata))) + return -EINVAL; + + mutex_lock(&hctx->accum_lock); + + errcode = kbasep_hwcnt_accumulator_dump( + hctx, ts_start_ns, ts_end_ns, dump_buf, new_map); + + mutex_unlock(&hctx->accum_lock); + + return errcode; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_set_counters); + +int kbase_hwcnt_accumulator_dump( + struct kbase_hwcnt_accumulator *accum, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf) +{ + int errcode; + struct kbase_hwcnt_context *hctx; + + if (!accum || !ts_start_ns || !ts_end_ns) + return -EINVAL; + + hctx = container_of(accum, struct kbase_hwcnt_context, accum); + + if (dump_buf && (dump_buf->metadata != hctx->iface->metadata)) + return -EINVAL; + + mutex_lock(&hctx->accum_lock); + + errcode = kbasep_hwcnt_accumulator_dump( + hctx, ts_start_ns, ts_end_ns, dump_buf, NULL); + + mutex_unlock(&hctx->accum_lock); + + return errcode; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_dump); + +u64 kbase_hwcnt_accumulator_timestamp_ns(struct kbase_hwcnt_accumulator *accum) +{ + struct kbase_hwcnt_context *hctx; + + if (WARN_ON(!accum)) + return 0; + + hctx = container_of(accum, struct kbase_hwcnt_context, accum); + return hctx->iface->timestamp_ns(accum->backend); +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_accumulator.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_accumulator.h new file mode 100755 index 000000000000..eb82ea4bfd14 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_accumulator.h @@ -0,0 +1,146 @@ +/* + * + * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Hardware counter accumulator API. + */ + +#ifndef _KBASE_HWCNT_ACCUMULATOR_H_ +#define _KBASE_HWCNT_ACCUMULATOR_H_ + +#include + +struct kbase_hwcnt_context; +struct kbase_hwcnt_accumulator; +struct kbase_hwcnt_enable_map; +struct kbase_hwcnt_dump_buffer; + +/** + * kbase_hwcnt_accumulator_acquire() - Acquire the hardware counter accumulator + * for a hardware counter context. + * @hctx: Non-NULL pointer to a hardware counter context. + * @accum: Non-NULL pointer to where the pointer to the created accumulator + * will be stored on success. + * + * There can exist at most one instance of the hardware counter accumulator per + * context at a time. + * + * If multiple clients need access to the hardware counters at the same time, + * then an abstraction built on top of the single instance to the hardware + * counter accumulator is required. + * + * No counters will be enabled with the returned accumulator. A subsequent call + * to kbase_hwcnt_accumulator_set_counters must be used to turn them on. + * + * There are four components to a hardware counter dump: + * - A set of enabled counters + * - A start time + * - An end time + * - A dump buffer containing the accumulated counter values for all enabled + * counters between the start and end times. + * + * For each dump, it is guaranteed that all enabled counters were active for the + * entirety of the period between the start and end times. + * + * It is also guaranteed that the start time of dump "n" is always equal to the + * end time of dump "n - 1". + * + * For all dumps, the values of any counters that were not enabled is undefined. + * + * Return: 0 on success or error code. + */ +int kbase_hwcnt_accumulator_acquire( + struct kbase_hwcnt_context *hctx, + struct kbase_hwcnt_accumulator **accum); + +/** + * kbase_hwcnt_accumulator_release() - Release a hardware counter accumulator. + * @accum: Non-NULL pointer to the hardware counter accumulator. + * + * The accumulator must be released before the context the accumulator was + * created from is terminated. + */ +void kbase_hwcnt_accumulator_release(struct kbase_hwcnt_accumulator *accum); + +/** + * kbase_hwcnt_accumulator_set_counters() - Perform a dump of the currently + * enabled counters, and enable a new + * set of counters that will be used + * for subsequent dumps. + * @accum: Non-NULL pointer to the hardware counter accumulator. + * @new_map: Non-NULL pointer to the new counter enable map. Must have the + * same metadata as the accumulator. + * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will + * be written out to on success. + * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will + * be written out to on success. + * @dump_buf: Pointer to the buffer where the dump will be written out to on + * success. If non-NULL, must have the same metadata as the + * accumulator. If NULL, the dump will be discarded. + * + * If this function fails for some unexpected reason (i.e. anything other than + * invalid args), then the accumulator will be put into the error state until + * the parent context is next disabled. + * + * Return: 0 on success or error code. + */ +int kbase_hwcnt_accumulator_set_counters( + struct kbase_hwcnt_accumulator *accum, + const struct kbase_hwcnt_enable_map *new_map, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf); + +/** + * kbase_hwcnt_accumulator_dump() - Perform a dump of the currently enabled + * counters. + * @accum: Non-NULL pointer to the hardware counter accumulator. + * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will + * be written out to on success. + * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will + * be written out to on success. + * @dump_buf: Pointer to the buffer where the dump will be written out to on + * success. If non-NULL, must have the same metadata as the + * accumulator. If NULL, the dump will be discarded. + * + * If this function fails for some unexpected reason (i.e. anything other than + * invalid args), then the accumulator will be put into the error state until + * the parent context is next disabled. + * + * Return: 0 on success or error code. + */ +int kbase_hwcnt_accumulator_dump( + struct kbase_hwcnt_accumulator *accum, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf); + +/** + * kbase_hwcnt_accumulator_timestamp_ns() - Get the current accumulator backend + * timestamp. + * @accum: Non-NULL pointer to the hardware counter accumulator. + * + * Return: Accumulator backend timestamp in nanoseconds. + */ +u64 kbase_hwcnt_accumulator_timestamp_ns(struct kbase_hwcnt_accumulator *accum); + +#endif /* _KBASE_HWCNT_ACCUMULATOR_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend.h new file mode 100755 index 000000000000..3a921b754b55 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend.h @@ -0,0 +1,220 @@ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Virtual interface for hardware counter backends. + */ + +#ifndef _KBASE_HWCNT_BACKEND_H_ +#define _KBASE_HWCNT_BACKEND_H_ + +#include + +struct kbase_hwcnt_metadata; +struct kbase_hwcnt_enable_map; +struct kbase_hwcnt_dump_buffer; + +/* + * struct kbase_hwcnt_backend_info - Opaque pointer to information used to + * create an instance of a hardware counter + * backend. + */ +struct kbase_hwcnt_backend_info; + +/* + * struct kbase_hwcnt_backend_info - Opaque pointer to a hardware counter + * backend, used to perform dumps. + */ +struct kbase_hwcnt_backend; + +/** + * typedef kbase_hwcnt_backend_init_fn - Initialise a counter backend. + * @info: Non-NULL pointer to backend info. + * @out_backend: Non-NULL pointer to where backend is stored on success. + * + * All uses of the created hardware counter backend must be externally + * synchronised. + * + * Return: 0 on success, else error code. + */ +typedef int (*kbase_hwcnt_backend_init_fn)( + const struct kbase_hwcnt_backend_info *info, + struct kbase_hwcnt_backend **out_backend); + +/** + * typedef kbase_hwcnt_backend_term_fn - Terminate a counter backend. + * @backend: Pointer to backend to be terminated. + */ +typedef void (*kbase_hwcnt_backend_term_fn)( + struct kbase_hwcnt_backend *backend); + +/** + * typedef kbase_hwcnt_backend_timestamp_ns_fn - Get the current backend + * timestamp. + * @backend: Non-NULL pointer to backend. + * + * Return: Backend timestamp in nanoseconds. + */ +typedef u64 (*kbase_hwcnt_backend_timestamp_ns_fn)( + struct kbase_hwcnt_backend *backend); + +/** + * typedef kbase_hwcnt_backend_dump_enable_fn - Start counter dumping with the + * backend. + * @backend: Non-NULL pointer to backend. + * @enable_map: Non-NULL pointer to enable map specifying enabled counters. + * + * The enable_map must have been created using the interface's metadata. + * If the backend has already been enabled, an error is returned. + * + * May be called in an atomic context. + * + * Return: 0 on success, else error code. + */ +typedef int (*kbase_hwcnt_backend_dump_enable_fn)( + struct kbase_hwcnt_backend *backend, + const struct kbase_hwcnt_enable_map *enable_map); + +/** + * typedef kbase_hwcnt_backend_dump_enable_nolock_fn - Start counter dumping + * with the backend. + * @backend: Non-NULL pointer to backend. + * @enable_map: Non-NULL pointer to enable map specifying enabled counters. + * + * Exactly the same as kbase_hwcnt_backend_dump_enable_fn(), except must be + * called in an atomic context with the spinlock documented by the specific + * backend interface held. + * + * Return: 0 on success, else error code. + */ +typedef int (*kbase_hwcnt_backend_dump_enable_nolock_fn)( + struct kbase_hwcnt_backend *backend, + const struct kbase_hwcnt_enable_map *enable_map); + +/** + * typedef kbase_hwcnt_backend_dump_disable_fn - Disable counter dumping with + * the backend. + * @backend: Non-NULL pointer to backend. + * + * If the backend is already disabled, does nothing. + * Any undumped counter values since the last dump get will be lost. + */ +typedef void (*kbase_hwcnt_backend_dump_disable_fn)( + struct kbase_hwcnt_backend *backend); + +/** + * typedef kbase_hwcnt_backend_dump_clear_fn - Reset all the current undumped + * counters. + * @backend: Non-NULL pointer to backend. + * + * If the backend is not enabled, returns an error. + * + * Return: 0 on success, else error code. + */ +typedef int (*kbase_hwcnt_backend_dump_clear_fn)( + struct kbase_hwcnt_backend *backend); + +/** + * typedef kbase_hwcnt_backend_dump_request_fn - Request an asynchronous counter + * dump. + * @backend: Non-NULL pointer to backend. + * @dump_time_ns: Non-NULL pointer where the timestamp of when the dump was + * requested will be written out to on success. + * + * If the backend is not enabled or another dump is already in progress, + * returns an error. + * + * Return: 0 on success, else error code. + */ +typedef int (*kbase_hwcnt_backend_dump_request_fn)( + struct kbase_hwcnt_backend *backend, + u64 *dump_time_ns); + +/** + * typedef kbase_hwcnt_backend_dump_wait_fn - Wait until the last requested + * counter dump has completed. + * @backend: Non-NULL pointer to backend. + * + * If the backend is not enabled, returns an error. + * + * Return: 0 on success, else error code. + */ +typedef int (*kbase_hwcnt_backend_dump_wait_fn)( + struct kbase_hwcnt_backend *backend); + +/** + * typedef kbase_hwcnt_backend_dump_get_fn - Copy or accumulate enable the + * counters dumped after the last dump + * request into the dump buffer. + * @backend: Non-NULL pointer to backend. + * @dump_buffer: Non-NULL pointer to destination dump buffer. + * @enable_map: Non-NULL pointer to enable map specifying enabled values. + * @accumulate: True if counters should be accumulated into dump_buffer, rather + * than copied. + * + * If the backend is not enabled, returns an error. + * If a dump is in progress (i.e. dump_wait has not yet returned successfully) + * then the resultant contents of the dump buffer will be undefined. + * + * Return: 0 on success, else error code. + */ +typedef int (*kbase_hwcnt_backend_dump_get_fn)( + struct kbase_hwcnt_backend *backend, + struct kbase_hwcnt_dump_buffer *dump_buffer, + const struct kbase_hwcnt_enable_map *enable_map, + bool accumulate); + +/** + * struct kbase_hwcnt_backend_interface - Hardware counter backend virtual + * interface. + * @metadata: Immutable hardware counter metadata. + * @info: Immutable info used to initialise an instance of the + * backend. + * @init: Function ptr to initialise an instance of the backend. + * @term: Function ptr to terminate an instance of the backend. + * @timestamp_ns: Function ptr to get the current backend timestamp. + * @dump_enable: Function ptr to enable dumping. + * @dump_enable_nolock: Function ptr to enable dumping while the + * backend-specific spinlock is already held. + * @dump_disable: Function ptr to disable dumping. + * @dump_clear: Function ptr to clear counters. + * @dump_request: Function ptr to request a dump. + * @dump_wait: Function ptr to wait until dump to complete. + * @dump_get: Function ptr to copy or accumulate dump into a dump + * buffer. + */ +struct kbase_hwcnt_backend_interface { + const struct kbase_hwcnt_metadata *metadata; + const struct kbase_hwcnt_backend_info *info; + kbase_hwcnt_backend_init_fn init; + kbase_hwcnt_backend_term_fn term; + kbase_hwcnt_backend_timestamp_ns_fn timestamp_ns; + kbase_hwcnt_backend_dump_enable_fn dump_enable; + kbase_hwcnt_backend_dump_enable_nolock_fn dump_enable_nolock; + kbase_hwcnt_backend_dump_disable_fn dump_disable; + kbase_hwcnt_backend_dump_clear_fn dump_clear; + kbase_hwcnt_backend_dump_request_fn dump_request; + kbase_hwcnt_backend_dump_wait_fn dump_wait; + kbase_hwcnt_backend_dump_get_fn dump_get; +}; + +#endif /* _KBASE_HWCNT_BACKEND_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.c new file mode 100755 index 000000000000..9f65de41694f --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.c @@ -0,0 +1,736 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_hwcnt_backend_jm.h" +#include "mali_kbase_hwcnt_gpu.h" +#include "mali_kbase_hwcnt_types.h" +#include "mali_kbase.h" +#include "mali_kbase_pm_ca.h" +#include "mali_kbase_hwaccess_instr.h" +#include "mali_kbase_hwaccess_time.h" +#include "mali_kbase_ccswe.h" + +#ifdef CONFIG_MALI_BIFROST_NO_MALI +#include "backend/gpu/mali_kbase_model_dummy.h" +#endif +#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" + +#if MALI_USE_CSF +#include "mali_kbase_ctx_sched.h" +#else +#include "backend/gpu/mali_kbase_pm_internal.h" +#endif + +/** + * struct kbase_hwcnt_backend_jm_info - Information used to create an instance + * of a JM hardware counter backend. + * @kbdev: KBase device. + * @use_secondary: True if secondary performance counters should be used, + * else false. Ignored if secondary counters are not supported. + * @metadata: Hardware counter metadata. + * @dump_bytes: Bytes of GPU memory required to perform a + * hardware counter dump. + */ +struct kbase_hwcnt_backend_jm_info { + struct kbase_device *kbdev; + bool use_secondary; + const struct kbase_hwcnt_metadata *metadata; + size_t dump_bytes; +}; + +/** + * struct kbase_hwcnt_backend_jm - Instance of a JM hardware counter backend. + * @info: Info used to create the backend. + * @kctx: KBase context used for GPU memory allocation and + * counter dumping. + * @gpu_dump_va: GPU hardware counter dump buffer virtual address. + * @cpu_dump_va: CPU mapping of gpu_dump_va. + * @vmap: Dump buffer vmap. + * @enabled: True if dumping has been enabled, else false. + * @pm_core_mask: PM state sync-ed shaders core mask for the enabled + * dumping. + * @clk_enable_map: The enable map specifying enabled clock domains. + * @cycle_count_elapsed: + * Cycle count elapsed for a given sample period. + * The top clock cycle, index 0, is read directly from + * hardware, but the other clock domains need to be + * calculated with software estimation. + * @prev_cycle_count: Previous cycle count to calculate the cycle count for + * sample period. + * @rate_listener: Clock rate listener callback state. + * @ccswe_shader_cores: Shader cores cycle count software estimator. + */ +struct kbase_hwcnt_backend_jm { + const struct kbase_hwcnt_backend_jm_info *info; + struct kbase_context *kctx; + u64 gpu_dump_va; + void *cpu_dump_va; + struct kbase_vmap_struct *vmap; + bool enabled; + u64 pm_core_mask; + u64 clk_enable_map; + u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS]; + u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS]; + struct kbase_clk_rate_listener rate_listener; + struct kbase_ccswe ccswe_shader_cores; +}; + +/** + * kbasep_hwcnt_backend_jm_on_freq_change() - On freq change callback + * + * @rate_listener: Callback state + * @clk_index: Clock index + * @clk_rate_hz: Clock frequency(hz) + */ +static void kbasep_hwcnt_backend_jm_on_freq_change( + struct kbase_clk_rate_listener *rate_listener, + u32 clk_index, + u32 clk_rate_hz) +{ + struct kbase_hwcnt_backend_jm *backend_jm = container_of( + rate_listener, struct kbase_hwcnt_backend_jm, rate_listener); + u64 timestamp_ns; + + if (clk_index != KBASE_CLOCK_DOMAIN_SHADER_CORES) + return; + + timestamp_ns = ktime_get_raw_ns(); + kbase_ccswe_freq_change( + &backend_jm->ccswe_shader_cores, timestamp_ns, clk_rate_hz); +} + +/** + * kbasep_hwcnt_backend_jm_cc_enable() - Enable cycle count tracking + * + * @backend: Non-NULL pointer to backend. + * @enable_map: Non-NULL pointer to enable map specifying enabled counters. + * @timestamp_ns: Timestamp(ns) when HWCNT were enabled. + */ +static void kbasep_hwcnt_backend_jm_cc_enable( + struct kbase_hwcnt_backend_jm *backend_jm, + const struct kbase_hwcnt_enable_map *enable_map, + u64 timestamp_ns) +{ + struct kbase_device *kbdev = backend_jm->kctx->kbdev; + u64 clk_enable_map = enable_map->clk_enable_map; + u64 cycle_count; + + if (kbase_hwcnt_clk_enable_map_enabled( + clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { +#if !MALI_USE_CSF + /* turn on the cycle counter */ + kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev); +#endif + /* Read cycle count for top clock domain. */ + kbase_backend_get_gpu_time_norequest( + kbdev, &cycle_count, NULL, NULL); + + backend_jm->prev_cycle_count[KBASE_CLOCK_DOMAIN_TOP] = + cycle_count; + } + + if (kbase_hwcnt_clk_enable_map_enabled( + clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { + /* software estimation for non-top clock domains */ + struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; + const struct kbase_clk_data *clk_data = + rtm->clks[KBASE_CLOCK_DOMAIN_SHADER_CORES]; + u32 cur_freq; + unsigned long flags; + + spin_lock_irqsave(&rtm->lock, flags); + + cur_freq = (u32) clk_data->clock_val; + kbase_ccswe_reset(&backend_jm->ccswe_shader_cores); + kbase_ccswe_freq_change( + &backend_jm->ccswe_shader_cores, + timestamp_ns, + cur_freq); + + kbase_clk_rate_trace_manager_subscribe_no_lock( + rtm, &backend_jm->rate_listener); + + spin_unlock_irqrestore(&rtm->lock, flags); + + /* ccswe was reset. The estimated cycle is zero. */ + backend_jm->prev_cycle_count[ + KBASE_CLOCK_DOMAIN_SHADER_CORES] = 0; + } + + /* Keep clk_enable_map for dump_request. */ + backend_jm->clk_enable_map = clk_enable_map; +} + +/** + * kbasep_hwcnt_backend_jm_cc_disable() - Disable cycle count tracking + * + * @backend: Non-NULL pointer to backend. + */ +static void kbasep_hwcnt_backend_jm_cc_disable( + struct kbase_hwcnt_backend_jm *backend_jm) +{ + struct kbase_device *kbdev = backend_jm->kctx->kbdev; + struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; + u64 clk_enable_map = backend_jm->clk_enable_map; + +#if !MALI_USE_CSF + if (kbase_hwcnt_clk_enable_map_enabled( + clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { + /* turn off the cycle counter */ + kbase_pm_release_gpu_cycle_counter(kbdev); + } +#endif + if (kbase_hwcnt_clk_enable_map_enabled( + clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { + + kbase_clk_rate_trace_manager_unsubscribe( + rtm, &backend_jm->rate_listener); + } +} + + +/* JM backend implementation of kbase_hwcnt_backend_timestamp_ns_fn */ +static u64 kbasep_hwcnt_backend_jm_timestamp_ns( + struct kbase_hwcnt_backend *backend) +{ + (void)backend; + return ktime_get_raw_ns(); +} + +/* JM backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ +static int kbasep_hwcnt_backend_jm_dump_enable_nolock( + struct kbase_hwcnt_backend *backend, + const struct kbase_hwcnt_enable_map *enable_map) +{ + int errcode; + struct kbase_hwcnt_backend_jm *backend_jm = + (struct kbase_hwcnt_backend_jm *)backend; + struct kbase_context *kctx; + struct kbase_device *kbdev; + struct kbase_hwcnt_physical_enable_map phys; + struct kbase_instr_hwcnt_enable enable; + u64 timestamp_ns; + + if (!backend_jm || !enable_map || backend_jm->enabled || + (enable_map->metadata != backend_jm->info->metadata)) + return -EINVAL; + + kctx = backend_jm->kctx; + kbdev = backend_jm->kctx->kbdev; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbase_hwcnt_gpu_enable_map_to_physical(&phys, enable_map); + + enable.fe_bm = phys.fe_bm; + enable.shader_bm = phys.shader_bm; + enable.tiler_bm = phys.tiler_bm; + enable.mmu_l2_bm = phys.mmu_l2_bm; + enable.use_secondary = backend_jm->info->use_secondary; + enable.dump_buffer = backend_jm->gpu_dump_va; + enable.dump_buffer_bytes = backend_jm->info->dump_bytes; + + timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); + + errcode = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &enable); + if (errcode) + goto error; + + backend_jm->pm_core_mask = kbase_pm_ca_get_instr_core_mask(kbdev); + backend_jm->enabled = true; + + kbasep_hwcnt_backend_jm_cc_enable(backend_jm, enable_map, timestamp_ns); + + return 0; +error: + return errcode; +} + +/* JM backend implementation of kbase_hwcnt_backend_dump_enable_fn */ +static int kbasep_hwcnt_backend_jm_dump_enable( + struct kbase_hwcnt_backend *backend, + const struct kbase_hwcnt_enable_map *enable_map) +{ + unsigned long flags; + int errcode; + struct kbase_hwcnt_backend_jm *backend_jm = + (struct kbase_hwcnt_backend_jm *)backend; + struct kbase_device *kbdev; + + if (!backend_jm) + return -EINVAL; + + kbdev = backend_jm->kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + errcode = kbasep_hwcnt_backend_jm_dump_enable_nolock( + backend, enable_map); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return errcode; +} + +/* JM backend implementation of kbase_hwcnt_backend_dump_disable_fn */ +static void kbasep_hwcnt_backend_jm_dump_disable( + struct kbase_hwcnt_backend *backend) +{ + int errcode; + struct kbase_hwcnt_backend_jm *backend_jm = + (struct kbase_hwcnt_backend_jm *)backend; + + if (WARN_ON(!backend_jm) || !backend_jm->enabled) + return; + + kbasep_hwcnt_backend_jm_cc_disable(backend_jm); + + errcode = kbase_instr_hwcnt_disable_internal(backend_jm->kctx); + WARN_ON(errcode); + + backend_jm->enabled = false; +} + +/* JM backend implementation of kbase_hwcnt_backend_dump_clear_fn */ +static int kbasep_hwcnt_backend_jm_dump_clear( + struct kbase_hwcnt_backend *backend) +{ + struct kbase_hwcnt_backend_jm *backend_jm = + (struct kbase_hwcnt_backend_jm *)backend; + + if (!backend_jm || !backend_jm->enabled) + return -EINVAL; + + return kbase_instr_hwcnt_clear(backend_jm->kctx); +} + +/* JM backend implementation of kbase_hwcnt_backend_dump_request_fn */ +static int kbasep_hwcnt_backend_jm_dump_request( + struct kbase_hwcnt_backend *backend, + u64 *dump_time_ns) +{ + struct kbase_hwcnt_backend_jm *backend_jm = + (struct kbase_hwcnt_backend_jm *)backend; + struct kbase_device *kbdev; + const struct kbase_hwcnt_metadata *metadata; + u64 current_cycle_count; + size_t clk; + int ret; + + if (!backend_jm || !backend_jm->enabled) + return -EINVAL; + + kbdev = backend_jm->kctx->kbdev; + metadata = backend_jm->info->metadata; + + /* Disable pre-emption, to make the timestamp as accurate as possible */ + preempt_disable(); + { + *dump_time_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); + ret = kbase_instr_hwcnt_request_dump(backend_jm->kctx); + + kbase_hwcnt_metadata_for_each_clock(metadata, clk) { + if (!kbase_hwcnt_clk_enable_map_enabled( + backend_jm->clk_enable_map, clk)) + continue; + + if (clk == KBASE_CLOCK_DOMAIN_TOP) { + /* Read cycle count for top clock domain. */ + kbase_backend_get_gpu_time_norequest( + kbdev, ¤t_cycle_count, + NULL, NULL); + } else { + /* + * Estimate cycle count for non-top clock + * domain. + */ + current_cycle_count = kbase_ccswe_cycle_at( + &backend_jm->ccswe_shader_cores, + *dump_time_ns); + } + backend_jm->cycle_count_elapsed[clk] = + current_cycle_count - + backend_jm->prev_cycle_count[clk]; + + /* + * Keep the current cycle count for later calculation. + */ + backend_jm->prev_cycle_count[clk] = current_cycle_count; + } + } + preempt_enable(); + + return ret; +} + +/* JM backend implementation of kbase_hwcnt_backend_dump_wait_fn */ +static int kbasep_hwcnt_backend_jm_dump_wait( + struct kbase_hwcnt_backend *backend) +{ + struct kbase_hwcnt_backend_jm *backend_jm = + (struct kbase_hwcnt_backend_jm *)backend; + + if (!backend_jm || !backend_jm->enabled) + return -EINVAL; + + return kbase_instr_hwcnt_wait_for_dump(backend_jm->kctx); +} + +/* JM backend implementation of kbase_hwcnt_backend_dump_get_fn */ +static int kbasep_hwcnt_backend_jm_dump_get( + struct kbase_hwcnt_backend *backend, + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_enable_map *dst_enable_map, + bool accumulate) +{ + struct kbase_hwcnt_backend_jm *backend_jm = + (struct kbase_hwcnt_backend_jm *)backend; + size_t clk; + + if (!backend_jm || !dst || !dst_enable_map || + (backend_jm->info->metadata != dst->metadata) || + (dst_enable_map->metadata != dst->metadata)) + return -EINVAL; + + /* Invalidate the kernel buffer before reading from it. */ + kbase_sync_mem_regions( + backend_jm->kctx, backend_jm->vmap, KBASE_SYNC_TO_CPU); + + kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) { + if (!kbase_hwcnt_clk_enable_map_enabled( + dst_enable_map->clk_enable_map, clk)) + continue; + + /* Extract elapsed cycle count for each clock domain. */ + dst->clk_cnt_buf[clk] = backend_jm->cycle_count_elapsed[clk]; + } + + return kbase_hwcnt_gpu_dump_get( + dst, backend_jm->cpu_dump_va, dst_enable_map, + backend_jm->pm_core_mask, accumulate); +} + +/** + * kbasep_hwcnt_backend_jm_dump_alloc() - Allocate a GPU dump buffer. + * @info: Non-NULL pointer to JM backend info. + * @kctx: Non-NULL pointer to kbase context. + * @gpu_dump_va: Non-NULL pointer to where GPU dump buffer virtual address + * is stored on success. + * + * Return: 0 on success, else error code. + */ +static int kbasep_hwcnt_backend_jm_dump_alloc( + const struct kbase_hwcnt_backend_jm_info *info, + struct kbase_context *kctx, + u64 *gpu_dump_va) +{ + struct kbase_va_region *reg; + u64 flags; + u64 nr_pages; + + WARN_ON(!info); + WARN_ON(!kctx); + WARN_ON(!gpu_dump_va); + + flags = BASE_MEM_PROT_CPU_RD | + BASE_MEM_PROT_GPU_WR | + BASEP_MEM_PERMANENT_KERNEL_MAPPING | + BASE_MEM_CACHED_CPU; + + if (kctx->kbdev->mmu_mode->flags & KBASE_MMU_MODE_HAS_NON_CACHEABLE) + flags |= BASE_MEM_UNCACHED_GPU; + + nr_pages = PFN_UP(info->dump_bytes); + + reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, gpu_dump_va); + + if (!reg) + return -ENOMEM; + + return 0; +} + +/** + * kbasep_hwcnt_backend_jm_dump_free() - Free an allocated GPU dump buffer. + * @kctx: Non-NULL pointer to kbase context. + * @gpu_dump_va: GPU dump buffer virtual address. + */ +static void kbasep_hwcnt_backend_jm_dump_free( + struct kbase_context *kctx, + u64 gpu_dump_va) +{ + WARN_ON(!kctx); + if (gpu_dump_va) + kbase_mem_free(kctx, gpu_dump_va); +} + +/** + * kbasep_hwcnt_backend_jm_destroy() - Destroy a JM backend. + * @backend: Pointer to JM backend to destroy. + * + * Can be safely called on a backend in any state of partial construction. + */ +static void kbasep_hwcnt_backend_jm_destroy( + struct kbase_hwcnt_backend_jm *backend) +{ + if (!backend) + return; + + if (backend->kctx) { +#if MALI_USE_CSF + unsigned long flags; +#endif + struct kbase_context *kctx = backend->kctx; + struct kbase_device *kbdev = kctx->kbdev; + + if (backend->cpu_dump_va) + kbase_phy_alloc_mapping_put(kctx, backend->vmap); + + if (backend->gpu_dump_va) + kbasep_hwcnt_backend_jm_dump_free( + kctx, backend->gpu_dump_va); + +#if MALI_USE_CSF + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_release_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#else + kbasep_js_release_privileged_ctx(kbdev, kctx); +#endif + kbase_destroy_context(kctx); + } + + kfree(backend); +} + +/** + * kbasep_hwcnt_backend_jm_create() - Create a JM backend. + * @info: Non-NULL pointer to backend info. + * @out_backend: Non-NULL pointer to where backend is stored on success. + * + * Return: 0 on success, else error code. + */ +static int kbasep_hwcnt_backend_jm_create( + const struct kbase_hwcnt_backend_jm_info *info, + struct kbase_hwcnt_backend_jm **out_backend) +{ +#if MALI_USE_CSF + unsigned long flags; +#endif + int errcode; + struct kbase_device *kbdev; + struct kbase_hwcnt_backend_jm *backend = NULL; + + WARN_ON(!info); + WARN_ON(!out_backend); + + kbdev = info->kbdev; + + backend = kzalloc(sizeof(*backend), GFP_KERNEL); + if (!backend) + goto alloc_error; + + backend->info = info; + + backend->kctx = kbase_create_context(kbdev, true, + BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED, 0, NULL); + if (!backend->kctx) + goto alloc_error; + +#if MALI_USE_CSF + kbase_pm_context_active(kbdev); + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_retain_ctx(backend->kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + kbase_pm_context_idle(kbdev); +#else + kbasep_js_schedule_privileged_ctx(kbdev, backend->kctx); +#endif + + errcode = kbasep_hwcnt_backend_jm_dump_alloc( + info, backend->kctx, &backend->gpu_dump_va); + if (errcode) + goto error; + + backend->cpu_dump_va = kbase_phy_alloc_mapping_get(backend->kctx, + backend->gpu_dump_va, &backend->vmap); + if (!backend->cpu_dump_va) + goto alloc_error; + + kbase_ccswe_init(&backend->ccswe_shader_cores); + backend->rate_listener.notify = kbasep_hwcnt_backend_jm_on_freq_change; + +#ifdef CONFIG_MALI_BIFROST_NO_MALI + /* The dummy model needs the CPU mapping. */ + gpu_model_set_dummy_prfcnt_base_cpu(backend->cpu_dump_va); +#endif + + *out_backend = backend; + return 0; + +alloc_error: + errcode = -ENOMEM; +error: + kbasep_hwcnt_backend_jm_destroy(backend); + return errcode; +} + +/* JM backend implementation of kbase_hwcnt_backend_init_fn */ +static int kbasep_hwcnt_backend_jm_init( + const struct kbase_hwcnt_backend_info *info, + struct kbase_hwcnt_backend **out_backend) +{ + int errcode; + struct kbase_hwcnt_backend_jm *backend = NULL; + + if (!info || !out_backend) + return -EINVAL; + + errcode = kbasep_hwcnt_backend_jm_create( + (const struct kbase_hwcnt_backend_jm_info *) info, &backend); + if (errcode) + return errcode; + + *out_backend = (struct kbase_hwcnt_backend *)backend; + + return 0; +} + +/* JM backend implementation of kbase_hwcnt_backend_term_fn */ +static void kbasep_hwcnt_backend_jm_term(struct kbase_hwcnt_backend *backend) +{ + if (!backend) + return; + + kbasep_hwcnt_backend_jm_dump_disable(backend); + kbasep_hwcnt_backend_jm_destroy( + (struct kbase_hwcnt_backend_jm *)backend); +} + +/** + * kbasep_hwcnt_backend_jm_info_destroy() - Destroy a JM backend info. + * @info: Pointer to info to destroy. + * + * Can be safely called on a backend info in any state of partial construction. + */ +static void kbasep_hwcnt_backend_jm_info_destroy( + const struct kbase_hwcnt_backend_jm_info *info) +{ + if (!info) + return; + + kbase_hwcnt_gpu_metadata_destroy(info->metadata); + kfree(info); +} + +/** + * kbasep_hwcnt_backend_jm_info_create() - Create a JM backend info. + * @kbdev: Non_NULL pointer to kbase device. + * @out_info: Non-NULL pointer to where info is stored on success. + * + * Return 0 on success, else error code. + */ +static int kbasep_hwcnt_backend_jm_info_create( + struct kbase_device *kbdev, + const struct kbase_hwcnt_backend_jm_info **out_info) +{ + int errcode = -ENOMEM; + struct kbase_hwcnt_gpu_info hwcnt_gpu_info; + struct kbase_hwcnt_backend_jm_info *info = NULL; + + WARN_ON(!kbdev); + WARN_ON(!out_info); + + errcode = kbase_hwcnt_gpu_info_init(kbdev, &hwcnt_gpu_info); + if (errcode) + return errcode; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + goto error; + + info->kbdev = kbdev; + +#ifdef CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY + info->use_secondary = true; +#else + info->use_secondary = false; +#endif + + errcode = kbase_hwcnt_gpu_metadata_create( + &hwcnt_gpu_info, info->use_secondary, + &info->metadata, + &info->dump_bytes); + if (errcode) + goto error; + + *out_info = info; + + return 0; +error: + kbasep_hwcnt_backend_jm_info_destroy(info); + return errcode; +} + +int kbase_hwcnt_backend_jm_create( + struct kbase_device *kbdev, + struct kbase_hwcnt_backend_interface *iface) +{ + int errcode; + const struct kbase_hwcnt_backend_jm_info *info = NULL; + + if (!kbdev || !iface) + return -EINVAL; + + errcode = kbasep_hwcnt_backend_jm_info_create(kbdev, &info); + + if (errcode) + return errcode; + + iface->metadata = info->metadata; + iface->info = (struct kbase_hwcnt_backend_info *)info; + iface->init = kbasep_hwcnt_backend_jm_init; + iface->term = kbasep_hwcnt_backend_jm_term; + iface->timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns; + iface->dump_enable = kbasep_hwcnt_backend_jm_dump_enable; + iface->dump_enable_nolock = kbasep_hwcnt_backend_jm_dump_enable_nolock; + iface->dump_disable = kbasep_hwcnt_backend_jm_dump_disable; + iface->dump_clear = kbasep_hwcnt_backend_jm_dump_clear; + iface->dump_request = kbasep_hwcnt_backend_jm_dump_request; + iface->dump_wait = kbasep_hwcnt_backend_jm_dump_wait; + iface->dump_get = kbasep_hwcnt_backend_jm_dump_get; + + return 0; +} + +void kbase_hwcnt_backend_jm_destroy( + struct kbase_hwcnt_backend_interface *iface) +{ + if (!iface) + return; + + kbasep_hwcnt_backend_jm_info_destroy( + (const struct kbase_hwcnt_backend_jm_info *)iface->info); + memset(iface, 0, sizeof(*iface)); +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.h new file mode 100755 index 000000000000..f15faeba704a --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_backend_jm.h @@ -0,0 +1,61 @@ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Concrete implementation of mali_kbase_hwcnt_backend interface for JM + * backend. + */ + +#ifndef _KBASE_HWCNT_BACKEND_JM_H_ +#define _KBASE_HWCNT_BACKEND_JM_H_ + +#include "mali_kbase_hwcnt_backend.h" + +struct kbase_device; + +/** + * kbase_hwcnt_backend_jm_create() - Create a JM hardware counter backend + * interface. + * @kbdev: Non-NULL pointer to kbase device. + * @iface: Non-NULL pointer to backend interface structure that is filled in + * on creation success. + * + * Calls to iface->dump_enable_nolock() require kbdev->hwaccess_lock held. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_backend_jm_create( + struct kbase_device *kbdev, + struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_jm_destroy() - Destroy a JM hardware counter backend + * interface. + * @iface: Pointer to interface to destroy. + * + * Can be safely called on an all-zeroed interface, or on an already destroyed + * interface. + */ +void kbase_hwcnt_backend_jm_destroy( + struct kbase_hwcnt_backend_interface *iface); + +#endif /* _KBASE_HWCNT_BACKEND_JM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_context.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_context.h new file mode 100755 index 000000000000..bc50ad12c2f4 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_context.h @@ -0,0 +1,119 @@ +/* + * + * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Hardware counter context API. + */ + +#ifndef _KBASE_HWCNT_CONTEXT_H_ +#define _KBASE_HWCNT_CONTEXT_H_ + +#include + +struct kbase_hwcnt_backend_interface; +struct kbase_hwcnt_context; + +/** + * kbase_hwcnt_context_init() - Initialise a hardware counter context. + * @iface: Non-NULL pointer to a hardware counter backend interface. + * @out_hctx: Non-NULL pointer to where the pointer to the created context will + * be stored on success. + * + * On creation, the disable count of the context will be 0. + * A hardware counter accumulator can be acquired using a created context. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_context_init( + const struct kbase_hwcnt_backend_interface *iface, + struct kbase_hwcnt_context **out_hctx); + +/** + * kbase_hwcnt_context_term() - Terminate a hardware counter context. + * @hctx: Pointer to context to be terminated. + */ +void kbase_hwcnt_context_term(struct kbase_hwcnt_context *hctx); + +/** + * kbase_hwcnt_context_metadata() - Get the hardware counter metadata used by + * the context, so related counter data + * structures can be created. + * @hctx: Non-NULL pointer to the hardware counter context. + * + * Return: Non-NULL pointer to metadata, or NULL on error. + */ +const struct kbase_hwcnt_metadata *kbase_hwcnt_context_metadata( + struct kbase_hwcnt_context *hctx); + +/** + * kbase_hwcnt_context_disable() - Increment the disable count of the context. + * @hctx: Pointer to the hardware counter context. + * + * If a call to this function increments the disable count from 0 to 1, and + * an accumulator has been acquired, then a counter dump will be performed + * before counters are disabled via the backend interface. + * + * Subsequent dumps via the accumulator while counters are disabled will first + * return the accumulated dump, then will return dumps with zeroed counters. + * + * After this function call returns, it is guaranteed that counters will not be + * enabled via the backend interface. + */ +void kbase_hwcnt_context_disable(struct kbase_hwcnt_context *hctx); + +/** + * kbase_hwcnt_context_disable_atomic() - Increment the disable count of the + * context if possible in an atomic + * context. + * @hctx: Pointer to the hardware counter context. + * + * This function will only succeed if hardware counters are effectively already + * disabled, i.e. there is no accumulator, the disable count is already + * non-zero, or the accumulator has no counters set. + * + * After this function call returns true, it is guaranteed that counters will + * not be enabled via the backend interface. + * + * Return: True if the disable count was incremented, else False. + */ +bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx); + +/** + * kbase_hwcnt_context_enable() - Decrement the disable count of the context. + * @hctx: Pointer to the hardware counter context. + * + * If a call to this function decrements the disable count from 1 to 0, and + * an accumulator has been acquired, then counters will be re-enabled via the + * backend interface. + * + * If an accumulator has been acquired and enabling counters fails for some + * reason, the accumulator will be placed into an error state. + * + * It is only valid to call this function one time for each prior returned call + * to kbase_hwcnt_context_disable. + * + * The spinlock documented in the backend interface that was passed in to + * kbase_hwcnt_context_init() must be held before calling this function. + */ +void kbase_hwcnt_context_enable(struct kbase_hwcnt_context *hctx); + +#endif /* _KBASE_HWCNT_CONTEXT_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c new file mode 100755 index 000000000000..499f3bc23bec --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c @@ -0,0 +1,571 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_hwcnt_gpu.h" +#include "mali_kbase_hwcnt_types.h" +#include "mali_kbase.h" +#ifdef CONFIG_MALI_BIFROST_NO_MALI +#include "backend/gpu/mali_kbase_model_dummy.h" +#endif + +#define KBASE_HWCNT_V5_BLOCK_TYPE_COUNT 4 +#define KBASE_HWCNT_V5_HEADERS_PER_BLOCK 4 +#define KBASE_HWCNT_V5_COUNTERS_PER_BLOCK 60 +#define KBASE_HWCNT_V5_VALUES_PER_BLOCK \ + (KBASE_HWCNT_V5_HEADERS_PER_BLOCK + KBASE_HWCNT_V5_COUNTERS_PER_BLOCK) +/* Index of the PRFCNT_EN header into a V5 counter block */ +#define KBASE_HWCNT_V5_PRFCNT_EN_HEADER 2 + +/** + * kbasep_hwcnt_backend_gpu_metadata_v5_create() - Create hardware counter + * metadata for a v5 GPU. + * @v5_info: Non-NULL pointer to hwcnt info for a v5 GPU. + * @use_secondary: True if secondary performance counters should be used, else + * false. Ignored if secondary counters are not supported. + * @metadata: Non-NULL pointer to where created metadata is stored + * on success. + * + * Return: 0 on success, else error code. + */ +static int kbasep_hwcnt_backend_gpu_metadata_v5_create( + const struct kbase_hwcnt_gpu_v5_info *v5_info, + bool use_secondary, + const struct kbase_hwcnt_metadata **metadata) +{ + struct kbase_hwcnt_description desc; + struct kbase_hwcnt_group_description group; + struct kbase_hwcnt_block_description + blks[KBASE_HWCNT_V5_BLOCK_TYPE_COUNT]; + size_t non_sc_block_count; + size_t sc_block_count; + + WARN_ON(!v5_info); + WARN_ON(!metadata); + + /* Calculate number of block instances that aren't shader cores */ + non_sc_block_count = 2 + v5_info->l2_count; + /* Calculate number of block instances that are shader cores */ + sc_block_count = fls64(v5_info->core_mask); + + /* + * A system can have up to 64 shader cores, but the 64-bit + * availability mask can't physically represent that many cores as well + * as the other hardware blocks. + * Error out if there are more blocks than our implementation can + * support. + */ + if ((sc_block_count + non_sc_block_count) > KBASE_HWCNT_AVAIL_MASK_BITS) + return -EINVAL; + + /* One Job Manager block */ + blks[0].type = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM; + blks[0].inst_cnt = 1; + blks[0].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; + blks[0].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; + + /* One Tiler block */ + blks[1].type = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER; + blks[1].inst_cnt = 1; + blks[1].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; + blks[1].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; + + /* l2_count memsys blks */ + blks[2].type = use_secondary ? + KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2 : + KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS; + blks[2].inst_cnt = v5_info->l2_count; + blks[2].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; + blks[2].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; + + /* + * There are as many shader cores in the system as there are bits set in + * the core mask. However, the dump buffer memory requirements need to + * take into account the fact that the core mask may be non-contiguous. + * + * For example, a system with a core mask of 0b1011 has the same dump + * buffer memory requirements as a system with 0b1111, but requires more + * memory than a system with 0b0111. However, core 2 of the system with + * 0b1011 doesn't physically exist, and the dump buffer memory that + * accounts for that core will never be written to when we do a counter + * dump. + * + * We find the core mask's last set bit to determine the memory + * requirements, and embed the core mask into the availability mask so + * we can determine later which shader cores physically exist. + */ + blks[3].type = use_secondary ? + KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2 : + KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC; + blks[3].inst_cnt = sc_block_count; + blks[3].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; + blks[3].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; + + WARN_ON(KBASE_HWCNT_V5_BLOCK_TYPE_COUNT != 4); + + group.type = KBASE_HWCNT_GPU_GROUP_TYPE_V5; + group.blk_cnt = KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; + group.blks = blks; + + desc.grp_cnt = 1; + desc.grps = &group; + desc.clk_cnt = v5_info->clk_cnt; + + /* The JM, Tiler, and L2s are always available, and are before cores */ + desc.avail_mask = (1ull << non_sc_block_count) - 1; + /* Embed the core mask directly in the availability mask */ + desc.avail_mask |= (v5_info->core_mask << non_sc_block_count); + + return kbase_hwcnt_metadata_create(&desc, metadata); +} + +/** + * kbasep_hwcnt_backend_gpu_v5_dump_bytes() - Get the raw dump buffer size for a + * V5 GPU. + * @v5_info: Non-NULL pointer to hwcnt info for a v5 GPU. + * + * Return: Size of buffer the V5 GPU needs to perform a counter dump. + */ +static size_t kbasep_hwcnt_backend_gpu_v5_dump_bytes( + const struct kbase_hwcnt_gpu_v5_info *v5_info) +{ + WARN_ON(!v5_info); + return (2 + v5_info->l2_count + fls64(v5_info->core_mask)) * + KBASE_HWCNT_V5_VALUES_PER_BLOCK * + KBASE_HWCNT_VALUE_BYTES; +} + +int kbase_hwcnt_gpu_info_init( + struct kbase_device *kbdev, + struct kbase_hwcnt_gpu_info *info) +{ + size_t clk; + + if (!kbdev || !info) + return -EINVAL; + +#ifdef CONFIG_MALI_BIFROST_NO_MALI + /* NO_MALI uses V5 layout, regardless of the underlying platform. */ + info->type = KBASE_HWCNT_GPU_GROUP_TYPE_V5; + info->v5.l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS; + info->v5.core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1; +#else + { + const struct base_gpu_props *props = &kbdev->gpu_props.props; + const size_t l2_count = props->l2_props.num_l2_slices; + const size_t core_mask = + props->coherency_info.group[0].core_mask; + + info->type = KBASE_HWCNT_GPU_GROUP_TYPE_V5; + info->v5.l2_count = l2_count; + info->v5.core_mask = core_mask; + } +#endif + + /* Determine the number of available clock domains. */ + for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) { + if (kbdev->pm.clk_rtm.clks[clk] == NULL) + break; + } + info->v5.clk_cnt = clk; + + return 0; +} + +int kbase_hwcnt_gpu_metadata_create( + const struct kbase_hwcnt_gpu_info *info, + bool use_secondary, + const struct kbase_hwcnt_metadata **out_metadata, + size_t *out_dump_bytes) +{ + int errcode; + const struct kbase_hwcnt_metadata *metadata; + size_t dump_bytes; + + if (!info || !out_metadata || !out_dump_bytes) + return -EINVAL; + + if (info->type == KBASE_HWCNT_GPU_GROUP_TYPE_V5) { + dump_bytes = kbasep_hwcnt_backend_gpu_v5_dump_bytes(&info->v5); + errcode = kbasep_hwcnt_backend_gpu_metadata_v5_create( + &info->v5, use_secondary, &metadata); + } else { + return -EINVAL; + } + if (errcode) + return errcode; + + /* + * Dump abstraction size should be exactly the same size and layout as + * the physical dump size, for backwards compatibility. + */ + WARN_ON(dump_bytes != metadata->dump_buf_bytes); + + *out_metadata = metadata; + *out_dump_bytes = dump_bytes; + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_metadata_create); + +void kbase_hwcnt_gpu_metadata_destroy( + const struct kbase_hwcnt_metadata *metadata) +{ + if (!metadata) + return; + + kbase_hwcnt_metadata_destroy(metadata); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_metadata_destroy); + +static bool is_block_type_shader( + const u64 grp_type, + const u64 blk_type, + const size_t blk) +{ + bool is_shader = false; + + /* Warn on unknown group type */ + if (WARN_ON(grp_type != KBASE_HWCNT_GPU_GROUP_TYPE_V5)) + return false; + + if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC || + blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2) + is_shader = true; + + return is_shader; +} + +int kbase_hwcnt_gpu_dump_get( + struct kbase_hwcnt_dump_buffer *dst, + void *src, + const struct kbase_hwcnt_enable_map *dst_enable_map, + u64 pm_core_mask, + bool accumulate) +{ + const struct kbase_hwcnt_metadata *metadata; + const u32 *dump_src; + size_t src_offset, grp, blk, blk_inst; + u64 core_mask = pm_core_mask; + + if (!dst || !src || !dst_enable_map || + (dst_enable_map->metadata != dst->metadata)) + return -EINVAL; + + metadata = dst->metadata; + dump_src = (const u32 *)src; + src_offset = 0; + + kbase_hwcnt_metadata_for_each_block( + metadata, grp, blk, blk_inst) { + const size_t hdr_cnt = + kbase_hwcnt_metadata_block_headers_count( + metadata, grp, blk); + const size_t ctr_cnt = + kbase_hwcnt_metadata_block_counters_count( + metadata, grp, blk); + const u64 blk_type = kbase_hwcnt_metadata_block_type( + metadata, grp, blk); + const bool is_shader_core = is_block_type_shader( + kbase_hwcnt_metadata_group_type(metadata, grp), + blk_type, blk); + + /* Early out if no values in the dest block are enabled */ + if (kbase_hwcnt_enable_map_block_enabled( + dst_enable_map, grp, blk, blk_inst)) { + u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( + dst, grp, blk, blk_inst); + const u32 *src_blk = dump_src + src_offset; + + if (!is_shader_core || (core_mask & 1)) { + if (accumulate) { + kbase_hwcnt_dump_buffer_block_accumulate( + dst_blk, src_blk, hdr_cnt, + ctr_cnt); + } else { + kbase_hwcnt_dump_buffer_block_copy( + dst_blk, src_blk, + (hdr_cnt + ctr_cnt)); + } + } else if (!accumulate) { + kbase_hwcnt_dump_buffer_block_zero( + dst_blk, (hdr_cnt + ctr_cnt)); + } + } + + src_offset += (hdr_cnt + ctr_cnt); + if (is_shader_core) + core_mask = core_mask >> 1; + } + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_dump_get); + +/** + * kbasep_hwcnt_backend_gpu_block_map_to_physical() - Convert from a block + * enable map abstraction to + * a physical block enable + * map. + * @lo: Low 64 bits of block enable map abstraction. + * @hi: High 64 bits of block enable map abstraction. + * + * The abstraction uses 128 bits to enable 128 block values, whereas the + * physical uses just 32 bits, as bit n enables values [n*4, n*4+3]. + * Therefore, this conversion is lossy. + * + * Return: 32-bit physical block enable map. + */ +static inline u32 kbasep_hwcnt_backend_gpu_block_map_to_physical( + u64 lo, + u64 hi) +{ + u32 phys = 0; + u64 dwords[2] = {lo, hi}; + size_t dword_idx; + + for (dword_idx = 0; dword_idx < 2; dword_idx++) { + const u64 dword = dwords[dword_idx]; + u16 packed = 0; + + size_t hword_bit; + + for (hword_bit = 0; hword_bit < 16; hword_bit++) { + const size_t dword_bit = hword_bit * 4; + const u16 mask = + ((dword >> (dword_bit + 0)) & 0x1) | + ((dword >> (dword_bit + 1)) & 0x1) | + ((dword >> (dword_bit + 2)) & 0x1) | + ((dword >> (dword_bit + 3)) & 0x1); + packed |= (mask << hword_bit); + } + phys |= ((u32)packed) << (16 * dword_idx); + } + return phys; +} + +/** + * kbasep_hwcnt_backend_gpu_block_map_from_physical() - Convert from a physical + * block enable map to a + * block enable map + * abstraction. + * @phys: Physical 32-bit block enable map + * @lo: Non-NULL pointer to where low 64 bits of block enable map abstraction + * will be stored. + * @hi: Non-NULL pointer to where high 64 bits of block enable map abstraction + * will be stored. + */ +static inline void kbasep_hwcnt_backend_gpu_block_map_from_physical( + u32 phys, + u64 *lo, + u64 *hi) +{ + u64 dwords[2] = {0, 0}; + + size_t dword_idx; + + for (dword_idx = 0; dword_idx < 2; dword_idx++) { + const u16 packed = phys >> (16 * dword_idx); + u64 dword = 0; + + size_t hword_bit; + + for (hword_bit = 0; hword_bit < 16; hword_bit++) { + const size_t dword_bit = hword_bit * 4; + const u64 mask = (packed >> (hword_bit)) & 0x1; + + dword |= mask << (dword_bit + 0); + dword |= mask << (dword_bit + 1); + dword |= mask << (dword_bit + 2); + dword |= mask << (dword_bit + 3); + } + dwords[dword_idx] = dword; + } + *lo = dwords[0]; + *hi = dwords[1]; +} + +void kbase_hwcnt_gpu_enable_map_to_physical( + struct kbase_hwcnt_physical_enable_map *dst, + const struct kbase_hwcnt_enable_map *src) +{ + const struct kbase_hwcnt_metadata *metadata; + + u64 fe_bm = 0; + u64 shader_bm = 0; + u64 tiler_bm = 0; + u64 mmu_l2_bm = 0; + + size_t grp, blk, blk_inst; + + if (WARN_ON(!src) || WARN_ON(!dst)) + return; + + metadata = src->metadata; + + kbase_hwcnt_metadata_for_each_block( + metadata, grp, blk, blk_inst) { + const u64 grp_type = kbase_hwcnt_metadata_group_type( + metadata, grp); + const u64 blk_type = kbase_hwcnt_metadata_block_type( + metadata, grp, blk); + const size_t blk_val_cnt = + kbase_hwcnt_metadata_block_values_count( + metadata, grp, blk); + const u64 *blk_map = kbase_hwcnt_enable_map_block_instance( + src, grp, blk, blk_inst); + + if ((enum kbase_hwcnt_gpu_group_type)grp_type == + KBASE_HWCNT_GPU_GROUP_TYPE_V5) { + WARN_ON(blk_val_cnt != KBASE_HWCNT_V5_VALUES_PER_BLOCK); + switch ((enum kbase_hwcnt_gpu_v5_block_type)blk_type) { + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM: + fe_bm |= *blk_map; + break; + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: + tiler_bm |= *blk_map; + break; + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: + shader_bm |= *blk_map; + break; + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2: + mmu_l2_bm |= *blk_map; + break; + default: + WARN_ON(true); + } + } else { + WARN_ON(true); + } + } + + dst->fe_bm = + kbasep_hwcnt_backend_gpu_block_map_to_physical(fe_bm, 0); + dst->shader_bm = + kbasep_hwcnt_backend_gpu_block_map_to_physical(shader_bm, 0); + dst->tiler_bm = + kbasep_hwcnt_backend_gpu_block_map_to_physical(tiler_bm, 0); + dst->mmu_l2_bm = + kbasep_hwcnt_backend_gpu_block_map_to_physical(mmu_l2_bm, 0); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_enable_map_to_physical); + +void kbase_hwcnt_gpu_enable_map_from_physical( + struct kbase_hwcnt_enable_map *dst, + const struct kbase_hwcnt_physical_enable_map *src) +{ + const struct kbase_hwcnt_metadata *metadata; + + u64 ignored_hi; + u64 fe_bm; + u64 shader_bm; + u64 tiler_bm; + u64 mmu_l2_bm; + size_t grp, blk, blk_inst; + + if (WARN_ON(!src) || WARN_ON(!dst)) + return; + + metadata = dst->metadata; + + kbasep_hwcnt_backend_gpu_block_map_from_physical( + src->fe_bm, &fe_bm, &ignored_hi); + kbasep_hwcnt_backend_gpu_block_map_from_physical( + src->shader_bm, &shader_bm, &ignored_hi); + kbasep_hwcnt_backend_gpu_block_map_from_physical( + src->tiler_bm, &tiler_bm, &ignored_hi); + kbasep_hwcnt_backend_gpu_block_map_from_physical( + src->mmu_l2_bm, &mmu_l2_bm, &ignored_hi); + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { + const u64 grp_type = kbase_hwcnt_metadata_group_type( + metadata, grp); + const u64 blk_type = kbase_hwcnt_metadata_block_type( + metadata, grp, blk); + const size_t blk_val_cnt = + kbase_hwcnt_metadata_block_values_count( + metadata, grp, blk); + u64 *blk_map = kbase_hwcnt_enable_map_block_instance( + dst, grp, blk, blk_inst); + + if ((enum kbase_hwcnt_gpu_group_type)grp_type == + KBASE_HWCNT_GPU_GROUP_TYPE_V5) { + WARN_ON(blk_val_cnt != KBASE_HWCNT_V5_VALUES_PER_BLOCK); + switch ((enum kbase_hwcnt_gpu_v5_block_type)blk_type) { + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM: + *blk_map = fe_bm; + break; + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: + *blk_map = tiler_bm; + break; + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: + *blk_map = shader_bm; + break; + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2: + *blk_map = mmu_l2_bm; + break; + default: + WARN_ON(true); + } + } else { + WARN_ON(true); + } + } +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_enable_map_from_physical); + +void kbase_hwcnt_gpu_patch_dump_headers( + struct kbase_hwcnt_dump_buffer *buf, + const struct kbase_hwcnt_enable_map *enable_map) +{ + const struct kbase_hwcnt_metadata *metadata; + size_t grp, blk, blk_inst; + + if (WARN_ON(!buf) || WARN_ON(!enable_map) || + WARN_ON(buf->metadata != enable_map->metadata)) + return; + + metadata = buf->metadata; + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { + const u64 grp_type = + kbase_hwcnt_metadata_group_type(metadata, grp); + u32 *buf_blk = kbase_hwcnt_dump_buffer_block_instance( + buf, grp, blk, blk_inst); + const u64 *blk_map = kbase_hwcnt_enable_map_block_instance( + enable_map, grp, blk, blk_inst); + const u32 prfcnt_en = + kbasep_hwcnt_backend_gpu_block_map_to_physical( + blk_map[0], 0); + + if ((enum kbase_hwcnt_gpu_group_type)grp_type == + KBASE_HWCNT_GPU_GROUP_TYPE_V5) { + buf_blk[KBASE_HWCNT_V5_PRFCNT_EN_HEADER] = prfcnt_en; + } else { + WARN_ON(true); + } + } +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_patch_dump_headers); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.h new file mode 100755 index 000000000000..f0d51763f7f7 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.h @@ -0,0 +1,217 @@ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_HWCNT_GPU_H_ +#define _KBASE_HWCNT_GPU_H_ + +#include + +struct kbase_device; +struct kbase_hwcnt_metadata; +struct kbase_hwcnt_enable_map; +struct kbase_hwcnt_dump_buffer; + +/** + * enum kbase_hwcnt_gpu_group_type - GPU hardware counter group types, used to + * identify metadata groups. + * @KBASE_HWCNT_GPU_GROUP_TYPE_V5: GPU V5 group type. + */ +enum kbase_hwcnt_gpu_group_type { + KBASE_HWCNT_GPU_GROUP_TYPE_V5 = 0x10, +}; + +/** + * enum kbase_hwcnt_gpu_v5_block_type - GPU V5 hardware counter block types, + * used to identify metadata blocks. + * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM: Job Manager block. + * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: Tiler block. + * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: Shader Core block. + * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: Secondary Shader Core block. + * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: Memsys block. + * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2: Secondary Memsys block. + */ +enum kbase_hwcnt_gpu_v5_block_type { + KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM = 0x40, + KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER, + KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC, + KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2, + KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS, + KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2, +}; + +/** + * struct kbase_hwcnt_physical_enable_map - Representation of enable map + * directly used by GPU. + * @fe_bm: Front end (JM/CSHW) counters selection bitmask. + * @shader_bm: Shader counters selection bitmask. + * @tiler_bm: Tiler counters selection bitmask. + * @mmu_l2_bm: MMU_L2 counters selection bitmask. + */ +struct kbase_hwcnt_physical_enable_map { + u32 fe_bm; + u32 shader_bm; + u32 tiler_bm; + u32 mmu_l2_bm; +}; + +/** + * struct kbase_hwcnt_gpu_v5_info - Information about hwcnt blocks on v5 GPUs. + * @l2_count: L2 cache count. + * @core_mask: Shader core mask. May be sparse. + * @clk_cnt: Number of clock domains available. + */ +struct kbase_hwcnt_gpu_v5_info { + size_t l2_count; + u64 core_mask; + u8 clk_cnt; +}; + +/** + * struct kbase_hwcnt_gpu_info - Tagged union with information about the current + * GPU's hwcnt blocks. + * @type: GPU type. + * @v5: Info filled in if a v5 GPU. + */ +struct kbase_hwcnt_gpu_info { + enum kbase_hwcnt_gpu_group_type type; + struct kbase_hwcnt_gpu_v5_info v5; +}; + +/** + * kbase_hwcnt_gpu_info_init() - Initialise an info structure used to create the + * hwcnt metadata. + * @kbdev: Non-NULL pointer to kbase device. + * @info: Non-NULL pointer to data structure to be filled in. + * + * The initialised info struct will only be valid for use while kbdev is valid. + */ +int kbase_hwcnt_gpu_info_init( + struct kbase_device *kbdev, + struct kbase_hwcnt_gpu_info *info); + +/** + * kbase_hwcnt_gpu_metadata_create() - Create hardware counter metadata for the + * current GPU. + * @info: Non-NULL pointer to info struct initialised by + * kbase_hwcnt_gpu_info_init. + * @use_secondary: True if secondary performance counters should be used, else + * false. Ignored if secondary counters are not supported. + * @out_metadata: Non-NULL pointer to where created metadata is stored on + * success. + * @out_dump_bytes: Non-NULL pointer to where the size of the GPU counter dump + * buffer is stored on success. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_gpu_metadata_create( + const struct kbase_hwcnt_gpu_info *info, + bool use_secondary, + const struct kbase_hwcnt_metadata **out_metadata, + size_t *out_dump_bytes); + +/** + * kbase_hwcnt_gpu_metadata_destroy() - Destroy GPU hardware counter metadata. + * @metadata: Pointer to metadata to destroy. + */ +void kbase_hwcnt_gpu_metadata_destroy( + const struct kbase_hwcnt_metadata *metadata); + +/** + * kbase_hwcnt_gpu_dump_get() - Copy or accumulate enabled counters from the raw + * dump buffer in src into the dump buffer + * abstraction in dst. + * @dst: Non-NULL pointer to dst dump buffer. + * @src: Non-NULL pointer to src raw dump buffer, of same length + * as returned in out_dump_bytes parameter of + * kbase_hwcnt_gpu_metadata_create. + * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. + * @pm_core_mask: PM state synchronized shaders core mask with the dump. + * @accumulate: True if counters in src should be accumulated into dst, + * rather than copied. + * + * The dst and dst_enable_map MUST have been created from the same metadata as + * returned from the call to kbase_hwcnt_gpu_metadata_create as was used to get + * the length of src. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_gpu_dump_get( + struct kbase_hwcnt_dump_buffer *dst, + void *src, + const struct kbase_hwcnt_enable_map *dst_enable_map, + const u64 pm_core_mask, + bool accumulate); + +/** + * kbase_hwcnt_gpu_enable_map_to_physical() - Convert an enable map abstraction + * into a physical enable map. + * @dst: Non-NULL pointer to dst physical enable map. + * @src: Non-NULL pointer to src enable map abstraction. + * + * The src must have been created from a metadata returned from a call to + * kbase_hwcnt_gpu_metadata_create. + * + * This is a lossy conversion, as the enable map abstraction has one bit per + * individual counter block value, but the physical enable map uses 1 bit for + * every 4 counters, shared over all instances of a block. + */ +void kbase_hwcnt_gpu_enable_map_to_physical( + struct kbase_hwcnt_physical_enable_map *dst, + const struct kbase_hwcnt_enable_map *src); + +/** + * kbase_hwcnt_gpu_enable_map_from_physical() - Convert a physical enable map to + * an enable map abstraction. + * @dst: Non-NULL pointer to dst enable map abstraction. + * @src: Non-NULL pointer to src physical enable map. + * + * The dst must have been created from a metadata returned from a call to + * kbase_hwcnt_gpu_metadata_create. + * + * This is a lossy conversion, as the physical enable map can technically + * support counter blocks with 128 counters each, but no hardware actually uses + * more than 64, so the enable map abstraction has nowhere to store the enable + * information for the 64 non-existent counters. + */ +void kbase_hwcnt_gpu_enable_map_from_physical( + struct kbase_hwcnt_enable_map *dst, + const struct kbase_hwcnt_physical_enable_map *src); + +/** + * kbase_hwcnt_gpu_patch_dump_headers() - Patch all the performance counter + * enable headers in a dump buffer to + * reflect the specified enable map. + * @buf: Non-NULL pointer to dump buffer to patch. + * @enable_map: Non-NULL pointer to enable map. + * + * The buf and enable_map must have been created from a metadata returned from + * a call to kbase_hwcnt_gpu_metadata_create. + * + * This function should be used before handing off a dump buffer over the + * kernel-user boundary, to ensure the header is accurate for the enable map + * used by the user. + */ +void kbase_hwcnt_gpu_patch_dump_headers( + struct kbase_hwcnt_dump_buffer *buf, + const struct kbase_hwcnt_enable_map *enable_map); + +#endif /* _KBASE_HWCNT_GPU_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.c new file mode 100755 index 000000000000..794ef39e365c --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.c @@ -0,0 +1,152 @@ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_hwcnt_legacy.h" +#include "mali_kbase_hwcnt_virtualizer.h" +#include "mali_kbase_hwcnt_types.h" +#include "mali_kbase_hwcnt_gpu.h" +#include "mali_kbase_ioctl.h" + +#include +#include + +/** + * struct kbase_hwcnt_legacy_client - Legacy hardware counter client. + * @user_dump_buf: Pointer to a non-NULL user buffer, where dumps are returned. + * @enable_map: Counter enable map. + * @dump_buf: Dump buffer used to manipulate dumps before copied to user. + * @hvcli: Hardware counter virtualizer client. + */ +struct kbase_hwcnt_legacy_client { + void __user *user_dump_buf; + struct kbase_hwcnt_enable_map enable_map; + struct kbase_hwcnt_dump_buffer dump_buf; + struct kbase_hwcnt_virtualizer_client *hvcli; +}; + +int kbase_hwcnt_legacy_client_create( + struct kbase_hwcnt_virtualizer *hvirt, + struct kbase_ioctl_hwcnt_enable *enable, + struct kbase_hwcnt_legacy_client **out_hlcli) +{ + int errcode; + struct kbase_hwcnt_legacy_client *hlcli; + const struct kbase_hwcnt_metadata *metadata; + struct kbase_hwcnt_physical_enable_map phys_em; + + if (!hvirt || !enable || !enable->dump_buffer || !out_hlcli) + return -EINVAL; + + metadata = kbase_hwcnt_virtualizer_metadata(hvirt); + + hlcli = kzalloc(sizeof(*hlcli), GFP_KERNEL); + if (!hlcli) + return -ENOMEM; + + hlcli->user_dump_buf = (void __user *)(uintptr_t)enable->dump_buffer; + + errcode = kbase_hwcnt_enable_map_alloc(metadata, &hlcli->enable_map); + if (errcode) + goto error; + + /* Translate from the ioctl enable map to the internal one */ + phys_em.fe_bm = enable->fe_bm; + phys_em.shader_bm = enable->shader_bm; + phys_em.tiler_bm = enable->tiler_bm; + phys_em.mmu_l2_bm = enable->mmu_l2_bm; + kbase_hwcnt_gpu_enable_map_from_physical(&hlcli->enable_map, &phys_em); + + errcode = kbase_hwcnt_dump_buffer_alloc(metadata, &hlcli->dump_buf); + if (errcode) + goto error; + + errcode = kbase_hwcnt_virtualizer_client_create( + hvirt, &hlcli->enable_map, &hlcli->hvcli); + if (errcode) + goto error; + + *out_hlcli = hlcli; + return 0; + +error: + kbase_hwcnt_legacy_client_destroy(hlcli); + return errcode; +} + +void kbase_hwcnt_legacy_client_destroy(struct kbase_hwcnt_legacy_client *hlcli) +{ + if (!hlcli) + return; + + kbase_hwcnt_virtualizer_client_destroy(hlcli->hvcli); + kbase_hwcnt_dump_buffer_free(&hlcli->dump_buf); + kbase_hwcnt_enable_map_free(&hlcli->enable_map); + kfree(hlcli); +} + +int kbase_hwcnt_legacy_client_dump(struct kbase_hwcnt_legacy_client *hlcli) +{ + int errcode; + u64 ts_start_ns; + u64 ts_end_ns; + + if (!hlcli) + return -EINVAL; + + /* Dump into the kernel buffer */ + errcode = kbase_hwcnt_virtualizer_client_dump(hlcli->hvcli, + &ts_start_ns, &ts_end_ns, &hlcli->dump_buf); + if (errcode) + return errcode; + + /* Patch the dump buf headers, to hide the counters that other hwcnt + * clients are using. + */ + kbase_hwcnt_gpu_patch_dump_headers( + &hlcli->dump_buf, &hlcli->enable_map); + + /* Zero all non-enabled counters (current values are undefined) */ + kbase_hwcnt_dump_buffer_zero_non_enabled( + &hlcli->dump_buf, &hlcli->enable_map); + + /* Copy into the user's buffer */ + errcode = copy_to_user(hlcli->user_dump_buf, hlcli->dump_buf.dump_buf, + hlcli->dump_buf.metadata->dump_buf_bytes); + /* Non-zero errcode implies user buf was invalid or too small */ + if (errcode) + return -EFAULT; + + return 0; +} + +int kbase_hwcnt_legacy_client_clear(struct kbase_hwcnt_legacy_client *hlcli) +{ + u64 ts_start_ns; + u64 ts_end_ns; + + if (!hlcli) + return -EINVAL; + + /* Dump with a NULL buffer to clear this client's counters */ + return kbase_hwcnt_virtualizer_client_dump(hlcli->hvcli, + &ts_start_ns, &ts_end_ns, NULL); +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.h new file mode 100755 index 000000000000..7a610ae378a2 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_legacy.h @@ -0,0 +1,94 @@ +/* + * + * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Legacy hardware counter interface, giving userspace clients simple, + * synchronous access to hardware counters. + * + * Any functions operating on an single legacy hardware counter client instance + * must be externally synchronised. + * Different clients may safely be used concurrently. + */ + +#ifndef _KBASE_HWCNT_LEGACY_H_ +#define _KBASE_HWCNT_LEGACY_H_ + +struct kbase_hwcnt_legacy_client; +struct kbase_ioctl_hwcnt_enable; +struct kbase_hwcnt_virtualizer; + +/** + * kbase_hwcnt_legacy_client_create() - Create a legacy hardware counter client. + * @hvirt: Non-NULL pointer to hardware counter virtualizer the client + * should be attached to. + * @enable: Non-NULL pointer to hwcnt_enable structure, containing a valid + * pointer to a user dump buffer large enough to hold a dump, and + * the counters that should be enabled. + * @out_hlcli: Non-NULL pointer to where the pointer to the created client will + * be stored on success. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_legacy_client_create( + struct kbase_hwcnt_virtualizer *hvirt, + struct kbase_ioctl_hwcnt_enable *enable, + struct kbase_hwcnt_legacy_client **out_hlcli); + +/** + * kbase_hwcnt_legacy_client_destroy() - Destroy a legacy hardware counter + * client. + * @hlcli: Pointer to the legacy hardware counter client. + * + * Will safely destroy a client in any partial state of construction. + */ +void kbase_hwcnt_legacy_client_destroy(struct kbase_hwcnt_legacy_client *hlcli); + +/** + * kbase_hwcnt_legacy_client_dump() - Perform a hardware counter dump into the + * client's user buffer. + * @hlcli: Non-NULL pointer to the legacy hardware counter client. + * + * This function will synchronously dump hardware counters into the user buffer + * specified on client creation, with the counters specified on client creation. + * + * The counters are automatically cleared after each dump, such that the next + * dump performed will return the counter values accumulated between the time of + * this function call and the next dump. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_legacy_client_dump(struct kbase_hwcnt_legacy_client *hlcli); + +/** + * kbase_hwcnt_legacy_client_clear() - Perform and discard a hardware counter + * dump. + * @hlcli: Non-NULL pointer to the legacy hardware counter client. + * + * This function will synchronously clear the hardware counters, such that the + * next dump performed will return the counter values accumulated between the + * time of this function call and the next dump. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_legacy_client_clear(struct kbase_hwcnt_legacy_client *hlcli); + +#endif /* _KBASE_HWCNT_LEGACY_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_reader.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_reader.h new file mode 100755 index 000000000000..8cd3835595f7 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_reader.h @@ -0,0 +1,106 @@ +/* + * + * (C) COPYRIGHT 2015, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_HWCNT_READER_H_ +#define _KBASE_HWCNT_READER_H_ + +#include + +/* The ids of ioctl commands. */ +#define KBASE_HWCNT_READER 0xBE +#define KBASE_HWCNT_READER_GET_HWVER _IOR(KBASE_HWCNT_READER, 0x00, u32) +#define KBASE_HWCNT_READER_GET_BUFFER_SIZE _IOR(KBASE_HWCNT_READER, 0x01, u32) +#define KBASE_HWCNT_READER_DUMP _IOW(KBASE_HWCNT_READER, 0x10, u32) +#define KBASE_HWCNT_READER_CLEAR _IOW(KBASE_HWCNT_READER, 0x11, u32) +#define KBASE_HWCNT_READER_GET_BUFFER _IOC(_IOC_READ, KBASE_HWCNT_READER, 0x20,\ + offsetof(struct kbase_hwcnt_reader_metadata, cycles)) +#define KBASE_HWCNT_READER_GET_BUFFER_WITH_CYCLES _IOR(KBASE_HWCNT_READER, 0x20,\ + struct kbase_hwcnt_reader_metadata) +#define KBASE_HWCNT_READER_PUT_BUFFER _IOC(_IOC_WRITE, KBASE_HWCNT_READER, 0x21,\ + offsetof(struct kbase_hwcnt_reader_metadata, cycles)) +#define KBASE_HWCNT_READER_PUT_BUFFER_WITH_CYCLES _IOW(KBASE_HWCNT_READER, 0x21,\ + struct kbase_hwcnt_reader_metadata) +#define KBASE_HWCNT_READER_SET_INTERVAL _IOW(KBASE_HWCNT_READER, 0x30, u32) +#define KBASE_HWCNT_READER_ENABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x40, u32) +#define KBASE_HWCNT_READER_DISABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x41, u32) +#define KBASE_HWCNT_READER_GET_API_VERSION _IOW(KBASE_HWCNT_READER, 0xFF, u32) +#define KBASE_HWCNT_READER_GET_API_VERSION_WITH_FEATURES \ + _IOW(KBASE_HWCNT_READER, 0xFF, \ + struct kbase_hwcnt_reader_api_version) + +/** + * struct kbase_hwcnt_reader_metadata_cycles - GPU clock cycles + * @top: the number of cycles associated with the main clock for the + * GPU + * @shader_cores: the cycles that have elapsed on the GPU shader cores + */ +struct kbase_hwcnt_reader_metadata_cycles { + u64 top; + u64 shader_cores; +}; + +/** + * struct kbase_hwcnt_reader_metadata - hwcnt reader sample buffer metadata + * @timestamp: time when sample was collected + * @event_id: id of an event that triggered sample collection + * @buffer_idx: position in sampling area where sample buffer was stored + * @cycles: the GPU cycles that occurred since the last sample + */ +struct kbase_hwcnt_reader_metadata { + u64 timestamp; + u32 event_id; + u32 buffer_idx; + struct kbase_hwcnt_reader_metadata_cycles cycles; +}; + +/** + * enum base_hwcnt_reader_event - hwcnt dumping events + * @BASE_HWCNT_READER_EVENT_MANUAL: manual request for dump + * @BASE_HWCNT_READER_EVENT_PERIODIC: periodic dump + * @BASE_HWCNT_READER_EVENT_PREJOB: prejob dump request + * @BASE_HWCNT_READER_EVENT_POSTJOB: postjob dump request + * @BASE_HWCNT_READER_EVENT_COUNT: number of supported events + */ +enum base_hwcnt_reader_event { + BASE_HWCNT_READER_EVENT_MANUAL, + BASE_HWCNT_READER_EVENT_PERIODIC, + BASE_HWCNT_READER_EVENT_PREJOB, + BASE_HWCNT_READER_EVENT_POSTJOB, + + BASE_HWCNT_READER_EVENT_COUNT +}; + +/** + * struct kbase_hwcnt_reader_api_version - hwcnt reader API version + * @versoin: API version + * @features: available features in this API version + */ +#define KBASE_HWCNT_READER_API_VERSION_NO_FEATURE (0) +#define KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_TOP (1 << 0) +#define KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_SHADER_CORES (1 << 1) +struct kbase_hwcnt_reader_api_version { + u32 version; + u32 features; +}; + +#endif /* _KBASE_HWCNT_READER_H_ */ + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.c new file mode 100755 index 000000000000..2b9fe02acd75 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.c @@ -0,0 +1,604 @@ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_hwcnt_types.h" +#include "mali_kbase.h" + +/* Minimum alignment of each block of hardware counters */ +#define KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT \ + (KBASE_HWCNT_BITFIELD_BITS * KBASE_HWCNT_VALUE_BYTES) + +/** + * KBASE_HWCNT_ALIGN_UPWARDS() - Align a value to an alignment. + * @value: The value to align upwards. + * @alignment: The alignment. + * + * Return: A number greater than or equal to value that is aligned to alignment. + */ +#define KBASE_HWCNT_ALIGN_UPWARDS(value, alignment) \ + (value + ((alignment - (value % alignment)) % alignment)) + +int kbase_hwcnt_metadata_create( + const struct kbase_hwcnt_description *desc, + const struct kbase_hwcnt_metadata **out_metadata) +{ + char *buf; + struct kbase_hwcnt_metadata *metadata; + struct kbase_hwcnt_group_metadata *grp_mds; + size_t grp; + size_t enable_map_count; /* Number of u64 bitfields (inc padding) */ + size_t dump_buf_count; /* Number of u32 values (inc padding) */ + size_t avail_mask_bits; /* Number of availability mask bits */ + + size_t size; + size_t offset; + + if (!desc || !out_metadata) + return -EINVAL; + + /* The maximum number of clock domains is 64. */ + if (desc->clk_cnt > (sizeof(u64) * BITS_PER_BYTE)) + return -EINVAL; + + /* Calculate the bytes needed to tightly pack the metadata */ + + /* Top level metadata */ + size = 0; + size += sizeof(struct kbase_hwcnt_metadata); + + /* Group metadata */ + size += sizeof(struct kbase_hwcnt_group_metadata) * desc->grp_cnt; + + /* Block metadata */ + for (grp = 0; grp < desc->grp_cnt; grp++) { + size += sizeof(struct kbase_hwcnt_block_metadata) * + desc->grps[grp].blk_cnt; + } + + /* Single allocation for the entire metadata */ + buf = kmalloc(size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + /* Use the allocated memory for the metadata and its members */ + + /* Bump allocate the top level metadata */ + offset = 0; + metadata = (struct kbase_hwcnt_metadata *)(buf + offset); + offset += sizeof(struct kbase_hwcnt_metadata); + + /* Bump allocate the group metadata */ + grp_mds = (struct kbase_hwcnt_group_metadata *)(buf + offset); + offset += sizeof(struct kbase_hwcnt_group_metadata) * desc->grp_cnt; + + enable_map_count = 0; + dump_buf_count = 0; + avail_mask_bits = 0; + + for (grp = 0; grp < desc->grp_cnt; grp++) { + size_t blk; + + const struct kbase_hwcnt_group_description *grp_desc = + desc->grps + grp; + struct kbase_hwcnt_group_metadata *grp_md = grp_mds + grp; + + size_t group_enable_map_count = 0; + size_t group_dump_buffer_count = 0; + size_t group_avail_mask_bits = 0; + + /* Bump allocate this group's block metadata */ + struct kbase_hwcnt_block_metadata *blk_mds = + (struct kbase_hwcnt_block_metadata *)(buf + offset); + offset += sizeof(struct kbase_hwcnt_block_metadata) * + grp_desc->blk_cnt; + + /* Fill in each block in the group's information */ + for (blk = 0; blk < grp_desc->blk_cnt; blk++) { + const struct kbase_hwcnt_block_description *blk_desc = + grp_desc->blks + blk; + struct kbase_hwcnt_block_metadata *blk_md = + blk_mds + blk; + const size_t n_values = + blk_desc->hdr_cnt + blk_desc->ctr_cnt; + + blk_md->type = blk_desc->type; + blk_md->inst_cnt = blk_desc->inst_cnt; + blk_md->hdr_cnt = blk_desc->hdr_cnt; + blk_md->ctr_cnt = blk_desc->ctr_cnt; + blk_md->enable_map_index = group_enable_map_count; + blk_md->enable_map_stride = + kbase_hwcnt_bitfield_count(n_values); + blk_md->dump_buf_index = group_dump_buffer_count; + blk_md->dump_buf_stride = + KBASE_HWCNT_ALIGN_UPWARDS( + n_values, + (KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT / + KBASE_HWCNT_VALUE_BYTES)); + blk_md->avail_mask_index = group_avail_mask_bits; + + group_enable_map_count += + blk_md->enable_map_stride * blk_md->inst_cnt; + group_dump_buffer_count += + blk_md->dump_buf_stride * blk_md->inst_cnt; + group_avail_mask_bits += blk_md->inst_cnt; + } + + /* Fill in the group's information */ + grp_md->type = grp_desc->type; + grp_md->blk_cnt = grp_desc->blk_cnt; + grp_md->blk_metadata = blk_mds; + grp_md->enable_map_index = enable_map_count; + grp_md->dump_buf_index = dump_buf_count; + grp_md->avail_mask_index = avail_mask_bits; + + enable_map_count += group_enable_map_count; + dump_buf_count += group_dump_buffer_count; + avail_mask_bits += group_avail_mask_bits; + } + + /* Fill in the top level metadata's information */ + metadata->grp_cnt = desc->grp_cnt; + metadata->grp_metadata = grp_mds; + metadata->enable_map_bytes = + enable_map_count * KBASE_HWCNT_BITFIELD_BYTES; + metadata->dump_buf_bytes = dump_buf_count * KBASE_HWCNT_VALUE_BYTES; + metadata->avail_mask = desc->avail_mask; + metadata->clk_cnt = desc->clk_cnt; + + WARN_ON(size != offset); + /* Due to the block alignment, there should be exactly one enable map + * bit per 4 bytes in the dump buffer. + */ + WARN_ON(metadata->dump_buf_bytes != + (metadata->enable_map_bytes * + BITS_PER_BYTE * KBASE_HWCNT_VALUE_BYTES)); + + *out_metadata = metadata; + return 0; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_metadata_create); + +void kbase_hwcnt_metadata_destroy(const struct kbase_hwcnt_metadata *metadata) +{ + kfree(metadata); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_metadata_destroy); + +int kbase_hwcnt_enable_map_alloc( + const struct kbase_hwcnt_metadata *metadata, + struct kbase_hwcnt_enable_map *enable_map) +{ + u64 *enable_map_buf; + + if (!metadata || !enable_map) + return -EINVAL; + + if (metadata->enable_map_bytes > 0) { + enable_map_buf = + kzalloc(metadata->enable_map_bytes, GFP_KERNEL); + if (!enable_map_buf) + return -ENOMEM; + } else { + enable_map_buf = NULL; + } + + enable_map->metadata = metadata; + enable_map->hwcnt_enable_map = enable_map_buf; + return 0; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_enable_map_alloc); + +void kbase_hwcnt_enable_map_free(struct kbase_hwcnt_enable_map *enable_map) +{ + if (!enable_map) + return; + + kfree(enable_map->hwcnt_enable_map); + enable_map->hwcnt_enable_map = NULL; + enable_map->metadata = NULL; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_enable_map_free); + +int kbase_hwcnt_dump_buffer_alloc( + const struct kbase_hwcnt_metadata *metadata, + struct kbase_hwcnt_dump_buffer *dump_buf) +{ + size_t dump_buf_bytes; + size_t clk_cnt_buf_bytes; + u8 *buf; + + if (!metadata || !dump_buf) + return -EINVAL; + + dump_buf_bytes = metadata->dump_buf_bytes; + clk_cnt_buf_bytes = sizeof(*dump_buf->clk_cnt_buf) * metadata->clk_cnt; + + /* Make a single allocation for both dump_buf and clk_cnt_buf. */ + buf = kmalloc(dump_buf_bytes + clk_cnt_buf_bytes, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + dump_buf->metadata = metadata; + dump_buf->dump_buf = (u32 *)buf; + dump_buf->clk_cnt_buf = (u64 *)(buf + dump_buf_bytes); + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_alloc); + +void kbase_hwcnt_dump_buffer_free(struct kbase_hwcnt_dump_buffer *dump_buf) +{ + if (!dump_buf) + return; + + kfree(dump_buf->dump_buf); + memset(dump_buf, 0, sizeof(*dump_buf)); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_free); + +int kbase_hwcnt_dump_buffer_array_alloc( + const struct kbase_hwcnt_metadata *metadata, + size_t n, + struct kbase_hwcnt_dump_buffer_array *dump_bufs) +{ + struct kbase_hwcnt_dump_buffer *buffers; + size_t buf_idx; + unsigned int order; + unsigned long addr; + size_t dump_buf_bytes; + size_t clk_cnt_buf_bytes; + + if (!metadata || !dump_bufs) + return -EINVAL; + + dump_buf_bytes = metadata->dump_buf_bytes; + clk_cnt_buf_bytes = + sizeof(*dump_bufs->bufs->clk_cnt_buf) * metadata->clk_cnt; + + /* Allocate memory for the dump buffer struct array */ + buffers = kmalloc_array(n, sizeof(*buffers), GFP_KERNEL); + if (!buffers) + return -ENOMEM; + + /* Allocate pages for the actual dump buffers, as they tend to be fairly + * large. + */ + order = get_order((dump_buf_bytes + clk_cnt_buf_bytes) * n); + addr = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + + if (!addr) { + kfree(buffers); + return -ENOMEM; + } + + dump_bufs->page_addr = addr; + dump_bufs->page_order = order; + dump_bufs->buf_cnt = n; + dump_bufs->bufs = buffers; + + /* Set the buffer of each dump buf */ + for (buf_idx = 0; buf_idx < n; buf_idx++) { + const size_t dump_buf_offset = dump_buf_bytes * buf_idx; + const size_t clk_cnt_buf_offset = + (dump_buf_bytes * n) + (clk_cnt_buf_bytes * buf_idx); + + buffers[buf_idx].metadata = metadata; + buffers[buf_idx].dump_buf = (u32 *)(addr + dump_buf_offset); + buffers[buf_idx].clk_cnt_buf = + (u64 *)(addr + clk_cnt_buf_offset); + } + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_array_alloc); + +void kbase_hwcnt_dump_buffer_array_free( + struct kbase_hwcnt_dump_buffer_array *dump_bufs) +{ + if (!dump_bufs) + return; + + kfree(dump_bufs->bufs); + free_pages(dump_bufs->page_addr, dump_bufs->page_order); + memset(dump_bufs, 0, sizeof(*dump_bufs)); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_array_free); + +void kbase_hwcnt_dump_buffer_zero( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_enable_map *dst_enable_map) +{ + const struct kbase_hwcnt_metadata *metadata; + size_t grp, blk, blk_inst; + + if (WARN_ON(!dst) || + WARN_ON(!dst_enable_map) || + WARN_ON(dst->metadata != dst_enable_map->metadata)) + return; + + metadata = dst->metadata; + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { + u32 *dst_blk; + size_t val_cnt; + + if (!kbase_hwcnt_enable_map_block_enabled( + dst_enable_map, grp, blk, blk_inst)) + continue; + + dst_blk = kbase_hwcnt_dump_buffer_block_instance( + dst, grp, blk, blk_inst); + val_cnt = kbase_hwcnt_metadata_block_values_count( + metadata, grp, blk); + + kbase_hwcnt_dump_buffer_block_zero(dst_blk, val_cnt); + } + + memset(dst->clk_cnt_buf, 0, + sizeof(*dst->clk_cnt_buf) * metadata->clk_cnt); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero); + +void kbase_hwcnt_dump_buffer_zero_strict( + struct kbase_hwcnt_dump_buffer *dst) +{ + if (WARN_ON(!dst)) + return; + + memset(dst->dump_buf, 0, dst->metadata->dump_buf_bytes); + + memset(dst->clk_cnt_buf, 0, + sizeof(*dst->clk_cnt_buf) * dst->metadata->clk_cnt); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero_strict); + +void kbase_hwcnt_dump_buffer_zero_non_enabled( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_enable_map *dst_enable_map) +{ + const struct kbase_hwcnt_metadata *metadata; + size_t grp, blk, blk_inst; + + if (WARN_ON(!dst) || + WARN_ON(!dst_enable_map) || + WARN_ON(dst->metadata != dst_enable_map->metadata)) + return; + + metadata = dst->metadata; + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { + u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( + dst, grp, blk, blk_inst); + const u64 *blk_em = kbase_hwcnt_enable_map_block_instance( + dst_enable_map, grp, blk, blk_inst); + size_t val_cnt = kbase_hwcnt_metadata_block_values_count( + metadata, grp, blk); + + /* Align upwards to include padding bytes */ + val_cnt = KBASE_HWCNT_ALIGN_UPWARDS(val_cnt, + (KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT / + KBASE_HWCNT_VALUE_BYTES)); + + if (kbase_hwcnt_metadata_block_instance_avail( + metadata, grp, blk, blk_inst)) { + /* Block available, so only zero non-enabled values */ + kbase_hwcnt_dump_buffer_block_zero_non_enabled( + dst_blk, blk_em, val_cnt); + } else { + /* Block not available, so zero the entire thing */ + kbase_hwcnt_dump_buffer_block_zero(dst_blk, val_cnt); + } + } +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero_non_enabled); + +void kbase_hwcnt_dump_buffer_copy( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_dump_buffer *src, + const struct kbase_hwcnt_enable_map *dst_enable_map) +{ + const struct kbase_hwcnt_metadata *metadata; + size_t grp, blk, blk_inst; + size_t clk; + + if (WARN_ON(!dst) || + WARN_ON(!src) || + WARN_ON(!dst_enable_map) || + WARN_ON(dst == src) || + WARN_ON(dst->metadata != src->metadata) || + WARN_ON(dst->metadata != dst_enable_map->metadata)) + return; + + metadata = dst->metadata; + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { + u32 *dst_blk; + const u32 *src_blk; + size_t val_cnt; + + if (!kbase_hwcnt_enable_map_block_enabled( + dst_enable_map, grp, blk, blk_inst)) + continue; + + dst_blk = kbase_hwcnt_dump_buffer_block_instance( + dst, grp, blk, blk_inst); + src_blk = kbase_hwcnt_dump_buffer_block_instance( + src, grp, blk, blk_inst); + val_cnt = kbase_hwcnt_metadata_block_values_count( + metadata, grp, blk); + + kbase_hwcnt_dump_buffer_block_copy(dst_blk, src_blk, val_cnt); + } + + kbase_hwcnt_metadata_for_each_clock(metadata, clk) { + if (kbase_hwcnt_clk_enable_map_enabled( + dst_enable_map->clk_enable_map, clk)) + dst->clk_cnt_buf[clk] = src->clk_cnt_buf[clk]; + } +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_copy); + +void kbase_hwcnt_dump_buffer_copy_strict( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_dump_buffer *src, + const struct kbase_hwcnt_enable_map *dst_enable_map) +{ + const struct kbase_hwcnt_metadata *metadata; + size_t grp, blk, blk_inst; + size_t clk; + + if (WARN_ON(!dst) || + WARN_ON(!src) || + WARN_ON(!dst_enable_map) || + WARN_ON(dst == src) || + WARN_ON(dst->metadata != src->metadata) || + WARN_ON(dst->metadata != dst_enable_map->metadata)) + return; + + metadata = dst->metadata; + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { + u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( + dst, grp, blk, blk_inst); + const u32 *src_blk = kbase_hwcnt_dump_buffer_block_instance( + src, grp, blk, blk_inst); + const u64 *blk_em = kbase_hwcnt_enable_map_block_instance( + dst_enable_map, grp, blk, blk_inst); + size_t val_cnt = kbase_hwcnt_metadata_block_values_count( + metadata, grp, blk); + /* Align upwards to include padding bytes */ + val_cnt = KBASE_HWCNT_ALIGN_UPWARDS(val_cnt, + (KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT / + KBASE_HWCNT_VALUE_BYTES)); + + kbase_hwcnt_dump_buffer_block_copy_strict( + dst_blk, src_blk, blk_em, val_cnt); + } + + kbase_hwcnt_metadata_for_each_clock(metadata, clk) { + bool clk_enabled = + kbase_hwcnt_clk_enable_map_enabled( + dst_enable_map->clk_enable_map, clk); + + dst->clk_cnt_buf[clk] = clk_enabled ? src->clk_cnt_buf[clk] : 0; + } +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_copy_strict); + +void kbase_hwcnt_dump_buffer_accumulate( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_dump_buffer *src, + const struct kbase_hwcnt_enable_map *dst_enable_map) +{ + const struct kbase_hwcnt_metadata *metadata; + size_t grp, blk, blk_inst; + size_t clk; + + if (WARN_ON(!dst) || + WARN_ON(!src) || + WARN_ON(!dst_enable_map) || + WARN_ON(dst == src) || + WARN_ON(dst->metadata != src->metadata) || + WARN_ON(dst->metadata != dst_enable_map->metadata)) + return; + + metadata = dst->metadata; + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { + u32 *dst_blk; + const u32 *src_blk; + size_t hdr_cnt; + size_t ctr_cnt; + + if (!kbase_hwcnt_enable_map_block_enabled( + dst_enable_map, grp, blk, blk_inst)) + continue; + + dst_blk = kbase_hwcnt_dump_buffer_block_instance( + dst, grp, blk, blk_inst); + src_blk = kbase_hwcnt_dump_buffer_block_instance( + src, grp, blk, blk_inst); + hdr_cnt = kbase_hwcnt_metadata_block_headers_count( + metadata, grp, blk); + ctr_cnt = kbase_hwcnt_metadata_block_counters_count( + metadata, grp, blk); + + kbase_hwcnt_dump_buffer_block_accumulate( + dst_blk, src_blk, hdr_cnt, ctr_cnt); + } + + kbase_hwcnt_metadata_for_each_clock(metadata, clk) { + if (kbase_hwcnt_clk_enable_map_enabled( + dst_enable_map->clk_enable_map, clk)) + dst->clk_cnt_buf[clk] += src->clk_cnt_buf[clk]; + } +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_accumulate); + +void kbase_hwcnt_dump_buffer_accumulate_strict( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_dump_buffer *src, + const struct kbase_hwcnt_enable_map *dst_enable_map) +{ + const struct kbase_hwcnt_metadata *metadata; + size_t grp, blk, blk_inst; + size_t clk; + + if (WARN_ON(!dst) || + WARN_ON(!src) || + WARN_ON(!dst_enable_map) || + WARN_ON(dst == src) || + WARN_ON(dst->metadata != src->metadata) || + WARN_ON(dst->metadata != dst_enable_map->metadata)) + return; + + metadata = dst->metadata; + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { + u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( + dst, grp, blk, blk_inst); + const u32 *src_blk = kbase_hwcnt_dump_buffer_block_instance( + src, grp, blk, blk_inst); + const u64 *blk_em = kbase_hwcnt_enable_map_block_instance( + dst_enable_map, grp, blk, blk_inst); + size_t hdr_cnt = kbase_hwcnt_metadata_block_headers_count( + metadata, grp, blk); + size_t ctr_cnt = kbase_hwcnt_metadata_block_counters_count( + metadata, grp, blk); + /* Align upwards to include padding bytes */ + ctr_cnt = KBASE_HWCNT_ALIGN_UPWARDS(hdr_cnt + ctr_cnt, + (KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT / + KBASE_HWCNT_VALUE_BYTES) - hdr_cnt); + + kbase_hwcnt_dump_buffer_block_accumulate_strict( + dst_blk, src_blk, blk_em, hdr_cnt, ctr_cnt); + } + + kbase_hwcnt_metadata_for_each_clock(metadata, clk) { + if (kbase_hwcnt_clk_enable_map_enabled( + dst_enable_map->clk_enable_map, clk)) + dst->clk_cnt_buf[clk] += src->clk_cnt_buf[clk]; + else + dst->clk_cnt_buf[clk] = 0; + } +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_accumulate_strict); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.h new file mode 100755 index 000000000000..3394b1271cc8 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_types.h @@ -0,0 +1,1142 @@ +/* + * + * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Hardware counter types. + * Contains structures for describing the physical layout of hardware counter + * dump buffers and enable maps within a system. + * + * Also contains helper functions for manipulation of these dump buffers and + * enable maps. + * + * Through use of these structures and functions, hardware counters can be + * enabled, copied, accumulated, and generally manipulated in a generic way, + * regardless of the physical counter dump layout. + * + * Terminology: + * + * Hardware Counter System: + * A collection of hardware counter groups, making a full hardware counter + * system. + * Hardware Counter Group: + * A group of Hardware Counter Blocks (e.g. a t62x might have more than one + * core group, so has one counter group per core group, where each group + * may have a different number and layout of counter blocks). + * Hardware Counter Block: + * A block of hardware counters (e.g. shader block, tiler block). + * Hardware Counter Block Instance: + * An instance of a Hardware Counter Block (e.g. an MP4 GPU might have + * 4 shader block instances). + * + * Block Header: + * A header value inside a counter block. Headers don't count anything, + * so it is only valid to copy or zero them. Headers are always the first + * values in the block. + * Block Counter: + * A counter value inside a counter block. Counters can be zeroed, copied, + * or accumulated. Counters are always immediately after the headers in the + * block. + * Block Value: + * A catch-all term for block headers and block counters. + * + * Enable Map: + * An array of u64 bitfields, where each bit either enables exactly one + * block value, or is unused (padding). + * Dump Buffer: + * An array of u32 values, where each u32 corresponds either to one block + * value, or is unused (padding). + * Availability Mask: + * A bitfield, where each bit corresponds to whether a block instance is + * physically available (e.g. an MP3 GPU may have a sparse core mask of + * 0b1011, meaning it only has 3 cores but for hardware counter dumps has the + * same dump buffer layout as an MP4 GPU with a core mask of 0b1111. In this + * case, the availability mask might be 0b1011111 (the exact layout will + * depend on the specific hardware architecture), with the 3 extra early bits + * corresponding to other block instances in the hardware counter system). + * Metadata: + * Structure describing the physical layout of the enable map and dump buffers + * for a specific hardware counter system. + * + */ + +#ifndef _KBASE_HWCNT_TYPES_H_ +#define _KBASE_HWCNT_TYPES_H_ + +#include +#include +#include +#include +#include "mali_malisw.h" + +/* Number of bytes in each bitfield */ +#define KBASE_HWCNT_BITFIELD_BYTES (sizeof(u64)) + +/* Number of bits in each bitfield */ +#define KBASE_HWCNT_BITFIELD_BITS (KBASE_HWCNT_BITFIELD_BYTES * BITS_PER_BYTE) + +/* Number of bytes for each counter value */ +#define KBASE_HWCNT_VALUE_BYTES (sizeof(u32)) + +/* Number of bits in an availability mask (i.e. max total number of block + * instances supported in a Hardware Counter System) + */ +#define KBASE_HWCNT_AVAIL_MASK_BITS (sizeof(u64) * BITS_PER_BYTE) + +/** + * struct kbase_hwcnt_block_description - Description of one or more identical, + * contiguous, Hardware Counter Blocks. + * @type: The arbitrary identifier used to identify the type of the block. + * @inst_cnt: The number of Instances of the block. + * @hdr_cnt: The number of 32-bit Block Headers in the block. + * @ctr_cnt: The number of 32-bit Block Counters in the block. + */ +struct kbase_hwcnt_block_description { + u64 type; + size_t inst_cnt; + size_t hdr_cnt; + size_t ctr_cnt; +}; + +/** + * struct kbase_hwcnt_group_description - Description of one or more identical, + * contiguous Hardware Counter Groups. + * @type: The arbitrary identifier used to identify the type of the group. + * @blk_cnt: The number of types of Hardware Counter Block in the group. + * @blks: Non-NULL pointer to an array of blk_cnt block descriptions, + * describing each type of Hardware Counter Block in the group. + */ +struct kbase_hwcnt_group_description { + u64 type; + size_t blk_cnt; + const struct kbase_hwcnt_block_description *blks; +}; + +/** + * struct kbase_hwcnt_description - Description of a Hardware Counter System. + * @grp_cnt: The number of Hardware Counter Groups. + * @grps: Non-NULL pointer to an array of grp_cnt group descriptions, + * describing each Hardware Counter Group in the system. + * @avail_mask: Flat Availability Mask for all block instances in the system. + * @clk_cnt: The number of clock domains in the system. The maximum is 64. + */ +struct kbase_hwcnt_description { + size_t grp_cnt; + const struct kbase_hwcnt_group_description *grps; + u64 avail_mask; + u8 clk_cnt; +}; + +/** + * struct kbase_hwcnt_block_metadata - Metadata describing the physical layout + * of a block in a Hardware Counter System's + * Dump Buffers and Enable Maps. + * @type: The arbitrary identifier used to identify the type of the + * block. + * @inst_cnt: The number of Instances of the block. + * @hdr_cnt: The number of 32-bit Block Headers in the block. + * @ctr_cnt: The number of 32-bit Block Counters in the block. + * @enable_map_index: Index in u64s into the parent's Enable Map where the + * Enable Map bitfields of the Block Instances described by + * this metadata start. + * @enable_map_stride: Stride in u64s between the Enable Maps of each of the + * Block Instances described by this metadata. + * @dump_buf_index: Index in u32s into the parent's Dump Buffer where the + * Dump Buffers of the Block Instances described by this + * metadata start. + * @dump_buf_stride: Stride in u32s between the Dump Buffers of each of the + * Block Instances described by this metadata. + * @avail_mask_index: Index in bits into the parent's Availability Mask where + * the Availability Masks of the Block Instances described + * by this metadata start. + */ +struct kbase_hwcnt_block_metadata { + u64 type; + size_t inst_cnt; + size_t hdr_cnt; + size_t ctr_cnt; + size_t enable_map_index; + size_t enable_map_stride; + size_t dump_buf_index; + size_t dump_buf_stride; + size_t avail_mask_index; +}; + +/** + * struct kbase_hwcnt_group_metadata - Metadata describing the physical layout + * of a group of blocks in a Hardware + * Counter System's Dump Buffers and Enable + * Maps. + * @type: The arbitrary identifier used to identify the type of the + * group. + * @blk_cnt: The number of types of Hardware Counter Block in the + * group. + * @blk_metadata: Non-NULL pointer to an array of blk_cnt block metadata, + * describing the physical layout of each type of Hardware + * Counter Block in the group. + * @enable_map_index: Index in u64s into the parent's Enable Map where the + * Enable Maps of the blocks within the group described by + * this metadata start. + * @dump_buf_index: Index in u32s into the parent's Dump Buffer where the + * Dump Buffers of the blocks within the group described by + * metadata start. + * @avail_mask_index: Index in bits into the parent's Availability Mask where + * the Availability Masks of the blocks within the group + * described by this metadata start. + */ +struct kbase_hwcnt_group_metadata { + u64 type; + size_t blk_cnt; + const struct kbase_hwcnt_block_metadata *blk_metadata; + size_t enable_map_index; + size_t dump_buf_index; + size_t avail_mask_index; +}; + +/** + * struct kbase_hwcnt_metadata - Metadata describing the physical layout + * of Dump Buffers and Enable Maps within a + * Hardware Counter System. + * @grp_cnt: The number of Hardware Counter Groups. + * @grp_metadata: Non-NULL pointer to an array of grp_cnt group metadata, + * describing the physical layout of each Hardware Counter + * Group in the system. + * @enable_map_bytes: The size in bytes of an Enable Map needed for the system. + * @dump_buf_bytes: The size in bytes of a Dump Buffer needed for the system. + * @avail_mask: The Availability Mask for the system. + * @clk_cnt: The number of clock domains in the system. + */ +struct kbase_hwcnt_metadata { + size_t grp_cnt; + const struct kbase_hwcnt_group_metadata *grp_metadata; + size_t enable_map_bytes; + size_t dump_buf_bytes; + u64 avail_mask; + u8 clk_cnt; +}; + +/** + * struct kbase_hwcnt_enable_map - Hardware Counter Enable Map. Array of u64 + * bitfields. + * @metadata: Non-NULL pointer to metadata used to identify, and to describe + * the layout of the enable map. + * @hwcnt_enable_map: Non-NULL pointer of size metadata->enable_map_bytes to an + * array of u64 bitfields, each bit of which enables one hardware + * counter. + * @clk_enable_map: An array of u64 bitfields, each bit of which enables cycle + * counter for a given clock domain. + */ +struct kbase_hwcnt_enable_map { + const struct kbase_hwcnt_metadata *metadata; + u64 *hwcnt_enable_map; + u64 clk_enable_map; +}; + +/** + * struct kbase_hwcnt_dump_buffer - Hardware Counter Dump Buffer. Array of u32 + * values. + * @metadata: Non-NULL pointer to metadata used to identify, and to describe + * the layout of the Dump Buffer. + * @dump_buf: Non-NULL pointer of size metadata->dump_buf_bytes to an array + * of u32 values. + * @clk_cnt_buf: A pointer to an array of u64 values for cycle count elapsed + * for each clock domain. + */ +struct kbase_hwcnt_dump_buffer { + const struct kbase_hwcnt_metadata *metadata; + u32 *dump_buf; + u64 *clk_cnt_buf; +}; + +/** + * struct kbase_hwcnt_dump_buffer_array - Hardware Counter Dump Buffer array. + * @page_addr: Address of allocated pages. A single allocation is used for all + * Dump Buffers in the array. + * @page_order: The allocation order of the pages. + * @buf_cnt: The number of allocated Dump Buffers. + * @bufs: Non-NULL pointer to the array of Dump Buffers. + */ +struct kbase_hwcnt_dump_buffer_array { + unsigned long page_addr; + unsigned int page_order; + size_t buf_cnt; + struct kbase_hwcnt_dump_buffer *bufs; +}; + +/** + * kbase_hwcnt_metadata_create() - Create a hardware counter metadata object + * from a description. + * @desc: Non-NULL pointer to a hardware counter description. + * @metadata: Non-NULL pointer to where created metadata will be stored on + * success. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_metadata_create( + const struct kbase_hwcnt_description *desc, + const struct kbase_hwcnt_metadata **metadata); + +/** + * kbase_hwcnt_metadata_destroy() - Destroy a hardware counter metadata object. + * @metadata: Pointer to hardware counter metadata + */ +void kbase_hwcnt_metadata_destroy(const struct kbase_hwcnt_metadata *metadata); + +/** + * kbase_hwcnt_metadata_group_count() - Get the number of groups. + * @metadata: Non-NULL pointer to metadata. + * + * Return: Number of hardware counter groups described by metadata. + */ +#define kbase_hwcnt_metadata_group_count(metadata) \ + ((metadata)->grp_cnt) + +/** + * kbase_hwcnt_metadata_group_type() - Get the arbitrary type of a group. + * @metadata: Non-NULL pointer to metadata. + * @grp: Index of the group in the metadata. + * + * Return: Type of the group grp. + */ +#define kbase_hwcnt_metadata_group_type(metadata, grp) \ + ((metadata)->grp_metadata[(grp)].type) + +/** + * kbase_hwcnt_metadata_block_count() - Get the number of blocks in a group. + * @metadata: Non-NULL pointer to metadata. + * @grp: Index of the group in the metadata. + * + * Return: Number of blocks in group grp. + */ +#define kbase_hwcnt_metadata_block_count(metadata, grp) \ + ((metadata)->grp_metadata[(grp)].blk_cnt) + +/** + * kbase_hwcnt_metadata_block_type() - Get the arbitrary type of a block. + * @metadata: Non-NULL pointer to metadata. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * + * Return: Type of the block blk in group grp. + */ +#define kbase_hwcnt_metadata_block_type(metadata, grp, blk) \ + ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].type) + +/** + * kbase_hwcnt_metadata_block_instance_count() - Get the number of instances of + * a block. + * @metadata: Non-NULL pointer to metadata. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * + * Return: Number of instances of block blk in group grp. + */ +#define kbase_hwcnt_metadata_block_instance_count(metadata, grp, blk) \ + ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].inst_cnt) + +/** + * kbase_hwcnt_metadata_block_headers_count() - Get the number of counter + * headers. + * @metadata: Non-NULL pointer to metadata. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * + * Return: Number of u32 counter headers in each instance of block blk in + * group grp. + */ +#define kbase_hwcnt_metadata_block_headers_count(metadata, grp, blk) \ + ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].hdr_cnt) + +/** + * kbase_hwcnt_metadata_block_counters_count() - Get the number of counters. + * @metadata: Non-NULL pointer to metadata. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * + * Return: Number of u32 counters in each instance of block blk in group + * grp. + */ +#define kbase_hwcnt_metadata_block_counters_count(metadata, grp, blk) \ + ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].ctr_cnt) + +/** + * kbase_hwcnt_metadata_block_values_count() - Get the number of values. + * @metadata: Non-NULL pointer to metadata. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * + * Return: Number of u32 headers plus counters in each instance of block blk + * in group grp. + */ +#define kbase_hwcnt_metadata_block_values_count(metadata, grp, blk) \ + (kbase_hwcnt_metadata_block_counters_count((metadata), (grp), (blk)) \ + + kbase_hwcnt_metadata_block_headers_count((metadata), (grp), (blk))) + +/** + * kbase_hwcnt_metadata_for_each_block() - Iterate over each block instance in + * the metadata. + * @md: Non-NULL pointer to metadata. + * @grp: size_t variable used as group iterator. + * @blk: size_t variable used as block iterator. + * @blk_inst: size_t variable used as block instance iterator. + * + * Iteration order is group, then block, then block instance (i.e. linearly + * through memory). + */ +#define kbase_hwcnt_metadata_for_each_block(md, grp, blk, blk_inst) \ + for ((grp) = 0; (grp) < kbase_hwcnt_metadata_group_count((md)); (grp)++) \ + for ((blk) = 0; (blk) < kbase_hwcnt_metadata_block_count((md), (grp)); (blk)++) \ + for ((blk_inst) = 0; (blk_inst) < kbase_hwcnt_metadata_block_instance_count((md), (grp), (blk)); (blk_inst)++) + +/** + * kbase_hwcnt_metadata_block_avail_bit() - Get the bit index into the avail + * mask corresponding to the block. + * @metadata: Non-NULL pointer to metadata. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * + * Return: The bit index into the avail mask for the block. + */ +static inline size_t kbase_hwcnt_metadata_block_avail_bit( + const struct kbase_hwcnt_metadata *metadata, + size_t grp, + size_t blk) +{ + const size_t bit = + metadata->grp_metadata[grp].avail_mask_index + + metadata->grp_metadata[grp].blk_metadata[blk].avail_mask_index; + + return bit; +} + +/** + * kbase_hwcnt_metadata_block_instance_avail() - Check if a block instance is + * available. + * @metadata: Non-NULL pointer to metadata. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * @blk_inst: Index of the block instance in the block. + * + * Return: true if the block instance is available, else false. + */ +static inline bool kbase_hwcnt_metadata_block_instance_avail( + const struct kbase_hwcnt_metadata *metadata, + size_t grp, + size_t blk, + size_t blk_inst) +{ + const size_t bit = kbase_hwcnt_metadata_block_avail_bit( + metadata, grp, blk) + blk_inst; + const u64 mask = 1ull << bit; + + return (metadata->avail_mask & mask) != 0; +} + +/** + * kbase_hwcnt_enable_map_alloc() - Allocate an enable map. + * @metadata: Non-NULL pointer to metadata describing the system. + * @enable_map: Non-NULL pointer to enable map to be initialised. Will be + * initialised to all zeroes (i.e. all counters disabled). + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_enable_map_alloc( + const struct kbase_hwcnt_metadata *metadata, + struct kbase_hwcnt_enable_map *enable_map); + +/** + * kbase_hwcnt_enable_map_free() - Free an enable map. + * @enable_map: Enable map to be freed. + * + * Can be safely called on an all-zeroed enable map structure, or on an already + * freed enable map. + */ +void kbase_hwcnt_enable_map_free(struct kbase_hwcnt_enable_map *enable_map); + +/** + * kbase_hwcnt_enable_map_block_instance() - Get the pointer to a block + * instance's enable map. + * @map: Non-NULL pointer to (const) enable map. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * @blk_inst: Index of the block instance in the block. + * + * Return: (const) u64* to the bitfield(s) used as the enable map for the + * block instance. + */ +#define kbase_hwcnt_enable_map_block_instance(map, grp, blk, blk_inst) \ + ((map)->hwcnt_enable_map + \ + (map)->metadata->grp_metadata[(grp)].enable_map_index + \ + (map)->metadata->grp_metadata[(grp)].blk_metadata[(blk)].enable_map_index + \ + (map)->metadata->grp_metadata[(grp)].blk_metadata[(blk)].enable_map_stride * (blk_inst)) + +/** + * kbase_hwcnt_bitfield_count() - Calculate the number of u64 bitfields required + * to have at minimum one bit per value. + * @val_cnt: Number of values. + * + * Return: Number of required bitfields. + */ +static inline size_t kbase_hwcnt_bitfield_count(size_t val_cnt) +{ + return (val_cnt + KBASE_HWCNT_BITFIELD_BITS - 1) / + KBASE_HWCNT_BITFIELD_BITS; +} + +/** + * kbase_hwcnt_enable_map_block_disable_all() - Disable all values in a block. + * @dst: Non-NULL pointer to enable map. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * @blk_inst: Index of the block instance in the block. + */ +static inline void kbase_hwcnt_enable_map_block_disable_all( + struct kbase_hwcnt_enable_map *dst, + size_t grp, + size_t blk, + size_t blk_inst) +{ + const size_t val_cnt = kbase_hwcnt_metadata_block_values_count( + dst->metadata, grp, blk); + const size_t bitfld_cnt = kbase_hwcnt_bitfield_count(val_cnt); + u64 *block_enable_map = kbase_hwcnt_enable_map_block_instance( + dst, grp, blk, blk_inst); + + memset(block_enable_map, 0, bitfld_cnt * KBASE_HWCNT_BITFIELD_BYTES); +} + +/** + * kbase_hwcnt_enable_map_disable_all() - Disable all values in the enable map. + * @dst: Non-NULL pointer to enable map to zero. + */ +static inline void kbase_hwcnt_enable_map_disable_all( + struct kbase_hwcnt_enable_map *dst) +{ + if (dst->hwcnt_enable_map != NULL) + memset(dst->hwcnt_enable_map, 0, + dst->metadata->enable_map_bytes); + + dst->clk_enable_map = 0; +} + +/** + * kbase_hwcnt_enable_map_block_enable_all() - Enable all values in a block. + * @dst: Non-NULL pointer to enable map. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * @blk_inst: Index of the block instance in the block. + */ +static inline void kbase_hwcnt_enable_map_block_enable_all( + struct kbase_hwcnt_enable_map *dst, + size_t grp, + size_t blk, + size_t blk_inst) +{ + const size_t val_cnt = kbase_hwcnt_metadata_block_values_count( + dst->metadata, grp, blk); + const size_t bitfld_cnt = kbase_hwcnt_bitfield_count(val_cnt); + u64 *block_enable_map = kbase_hwcnt_enable_map_block_instance( + dst, grp, blk, blk_inst); + + size_t bitfld_idx; + + for (bitfld_idx = 0; bitfld_idx < bitfld_cnt; bitfld_idx++) { + const u64 remaining_values = val_cnt - + (bitfld_idx * KBASE_HWCNT_BITFIELD_BITS); + u64 block_enable_map_mask = U64_MAX; + + if (remaining_values < KBASE_HWCNT_BITFIELD_BITS) + block_enable_map_mask = (1ull << remaining_values) - 1; + + block_enable_map[bitfld_idx] = block_enable_map_mask; + } +} + +/** + * kbase_hwcnt_enable_map_block_enable_all() - Enable all values in an enable + * map. + * @dst: Non-NULL pointer to enable map. + */ +static inline void kbase_hwcnt_enable_map_enable_all( + struct kbase_hwcnt_enable_map *dst) +{ + size_t grp, blk, blk_inst; + + kbase_hwcnt_metadata_for_each_block(dst->metadata, grp, blk, blk_inst) + kbase_hwcnt_enable_map_block_enable_all( + dst, grp, blk, blk_inst); + + dst->clk_enable_map = (1ull << dst->metadata->clk_cnt) - 1; +} + +/** + * kbase_hwcnt_enable_map_copy() - Copy an enable map to another. + * @dst: Non-NULL pointer to destination enable map. + * @src: Non-NULL pointer to source enable map. + * + * The dst and src MUST have been created from the same metadata. + */ +static inline void kbase_hwcnt_enable_map_copy( + struct kbase_hwcnt_enable_map *dst, + const struct kbase_hwcnt_enable_map *src) +{ + if (dst->hwcnt_enable_map != NULL) { + memcpy(dst->hwcnt_enable_map, + src->hwcnt_enable_map, + dst->metadata->enable_map_bytes); + } + + dst->clk_enable_map = src->clk_enable_map; +} + +/** + * kbase_hwcnt_enable_map_union() - Union dst and src enable maps into dst. + * @dst: Non-NULL pointer to destination enable map. + * @src: Non-NULL pointer to source enable map. + * + * The dst and src MUST have been created from the same metadata. + */ +static inline void kbase_hwcnt_enable_map_union( + struct kbase_hwcnt_enable_map *dst, + const struct kbase_hwcnt_enable_map *src) +{ + const size_t bitfld_count = + dst->metadata->enable_map_bytes / KBASE_HWCNT_BITFIELD_BYTES; + size_t i; + + if (dst->hwcnt_enable_map != NULL) { + for (i = 0; i < bitfld_count; i++) + dst->hwcnt_enable_map[i] |= src->hwcnt_enable_map[i]; + } + + dst->clk_enable_map |= src->clk_enable_map; +} + +/** + * kbase_hwcnt_enable_map_block_enabled() - Check if any values in a block + * instance are enabled. + * @enable_map: Non-NULL pointer to enable map. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * @blk_inst: Index of the block instance in the block. + * + * Return: true if any values in the block are enabled, else false. + */ +static inline bool kbase_hwcnt_enable_map_block_enabled( + const struct kbase_hwcnt_enable_map *enable_map, + size_t grp, + size_t blk, + size_t blk_inst) +{ + bool any_enabled = false; + const size_t val_cnt = kbase_hwcnt_metadata_block_values_count( + enable_map->metadata, grp, blk); + const size_t bitfld_cnt = kbase_hwcnt_bitfield_count(val_cnt); + const u64 *block_enable_map = kbase_hwcnt_enable_map_block_instance( + enable_map, grp, blk, blk_inst); + + size_t bitfld_idx; + + for (bitfld_idx = 0; bitfld_idx < bitfld_cnt; bitfld_idx++) { + const u64 remaining_values = val_cnt - + (bitfld_idx * KBASE_HWCNT_BITFIELD_BITS); + u64 block_enable_map_mask = U64_MAX; + + if (remaining_values < KBASE_HWCNT_BITFIELD_BITS) + block_enable_map_mask = (1ull << remaining_values) - 1; + + any_enabled = any_enabled || + (block_enable_map[bitfld_idx] & block_enable_map_mask); + } + + return any_enabled; +} + +/** + * kbase_hwcnt_enable_map_any_enabled() - Check if any values are enabled. + * @enable_map: Non-NULL pointer to enable map. + * + * Return: true if any values are enabled, else false. + */ +static inline bool kbase_hwcnt_enable_map_any_enabled( + const struct kbase_hwcnt_enable_map *enable_map) +{ + size_t grp, blk, blk_inst; + const u64 clk_enable_map_mask = + (1ull << enable_map->metadata->clk_cnt) - 1; + + if (enable_map->metadata->clk_cnt > 0 && + (enable_map->clk_enable_map & clk_enable_map_mask)) + return true; + + kbase_hwcnt_metadata_for_each_block( + enable_map->metadata, grp, blk, blk_inst) { + if (kbase_hwcnt_enable_map_block_enabled( + enable_map, grp, blk, blk_inst)) + return true; + } + + return false; +} + +/** + * kbase_hwcnt_enable_map_block_value_enabled() - Check if a value in a block + * instance is enabled. + * @bitfld: Non-NULL pointer to the block bitfield(s) obtained from a call to + * kbase_hwcnt_enable_map_block_instance. + * @val_idx: Index of the value to check in the block instance. + * + * Return: true if the value was enabled, else false. + */ +static inline bool kbase_hwcnt_enable_map_block_value_enabled( + const u64 *bitfld, + size_t val_idx) +{ + const size_t idx = val_idx / KBASE_HWCNT_BITFIELD_BITS; + const size_t bit = val_idx % KBASE_HWCNT_BITFIELD_BITS; + const u64 mask = 1ull << bit; + + return (bitfld[idx] & mask) != 0; +} + +/** + * kbase_hwcnt_enable_map_block_enable_value() - Enable a value in a block + * instance. + * @bitfld: Non-NULL pointer to the block bitfield(s) obtained from a call to + * kbase_hwcnt_enable_map_block_instance. + * @val_idx: Index of the value to enable in the block instance. + */ +static inline void kbase_hwcnt_enable_map_block_enable_value( + u64 *bitfld, + size_t val_idx) +{ + const size_t idx = val_idx / KBASE_HWCNT_BITFIELD_BITS; + const size_t bit = val_idx % KBASE_HWCNT_BITFIELD_BITS; + const u64 mask = 1ull << bit; + + bitfld[idx] |= mask; +} + +/** + * kbase_hwcnt_enable_map_block_disable_value() - Disable a value in a block + * instance. + * @bitfld: Non-NULL pointer to the block bitfield(s) obtained from a call to + * kbase_hwcnt_enable_map_block_instance. + * @val_idx: Index of the value to disable in the block instance. + */ +static inline void kbase_hwcnt_enable_map_block_disable_value( + u64 *bitfld, + size_t val_idx) +{ + const size_t idx = val_idx / KBASE_HWCNT_BITFIELD_BITS; + const size_t bit = val_idx % KBASE_HWCNT_BITFIELD_BITS; + const u64 mask = 1ull << bit; + + bitfld[idx] &= ~mask; +} + +/** + * kbase_hwcnt_dump_buffer_alloc() - Allocate a dump buffer. + * @metadata: Non-NULL pointer to metadata describing the system. + * @dump_buf: Non-NULL pointer to dump buffer to be initialised. Will be + * initialised to undefined values, so must be used as a copy dest, + * or cleared before use. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_dump_buffer_alloc( + const struct kbase_hwcnt_metadata *metadata, + struct kbase_hwcnt_dump_buffer *dump_buf); + +/** + * kbase_hwcnt_dump_buffer_free() - Free a dump buffer. + * @dump_buf: Dump buffer to be freed. + * + * Can be safely called on an all-zeroed dump buffer structure, or on an already + * freed dump buffer. + */ +void kbase_hwcnt_dump_buffer_free(struct kbase_hwcnt_dump_buffer *dump_buf); + +/** + * kbase_hwcnt_dump_buffer_array_alloc() - Allocate an array of dump buffers. + * @metadata: Non-NULL pointer to metadata describing the system. + * @n: Number of dump buffers to allocate + * @dump_bufs: Non-NULL pointer to dump buffer array to be initialised. Each + * dump buffer in the array will be initialised to undefined values, + * so must be used as a copy dest, or cleared before use. + * + * A single zeroed contiguous page allocation will be used for all of the + * buffers inside the array, where: + * dump_bufs[n].dump_buf == page_addr + n * metadata.dump_buf_bytes + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_dump_buffer_array_alloc( + const struct kbase_hwcnt_metadata *metadata, + size_t n, + struct kbase_hwcnt_dump_buffer_array *dump_bufs); + +/** + * kbase_hwcnt_dump_buffer_array_free() - Free a dump buffer array. + * @dump_bufs: Dump buffer array to be freed. + * + * Can be safely called on an all-zeroed dump buffer array structure, or on an + * already freed dump buffer array. + */ +void kbase_hwcnt_dump_buffer_array_free( + struct kbase_hwcnt_dump_buffer_array *dump_bufs); + +/** + * kbase_hwcnt_dump_buffer_block_instance() - Get the pointer to a block + * instance's dump buffer. + * @buf: Non-NULL pointer to (const) dump buffer. + * @grp: Index of the group in the metadata. + * @blk: Index of the block in the group. + * @blk_inst: Index of the block instance in the block. + * + * Return: (const) u32* to the dump buffer for the block instance. + */ +#define kbase_hwcnt_dump_buffer_block_instance(buf, grp, blk, blk_inst) \ + ((buf)->dump_buf + \ + (buf)->metadata->grp_metadata[(grp)].dump_buf_index + \ + (buf)->metadata->grp_metadata[(grp)].blk_metadata[(blk)].dump_buf_index + \ + (buf)->metadata->grp_metadata[(grp)].blk_metadata[(blk)].dump_buf_stride * (blk_inst)) + +/** + * kbase_hwcnt_dump_buffer_zero() - Zero all enabled values in dst. + * After the operation, all non-enabled values + * will be undefined. + * @dst: Non-NULL pointer to dump buffer. + * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. + * + * The dst and dst_enable_map MUST have been created from the same metadata. + */ +void kbase_hwcnt_dump_buffer_zero( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_enable_map *dst_enable_map); + +/** + * kbase_hwcnt_dump_buffer_block_zero() - Zero all values in a block. + * @dst_blk: Non-NULL pointer to dst block obtained from a call to + * kbase_hwcnt_dump_buffer_block_instance. + * @val_cnt: Number of values in the block. + */ +static inline void kbase_hwcnt_dump_buffer_block_zero( + u32 *dst_blk, + size_t val_cnt) +{ + memset(dst_blk, 0, (val_cnt * KBASE_HWCNT_VALUE_BYTES)); +} + +/** + * kbase_hwcnt_dump_buffer_zero_strict() - Zero all values in dst. + * After the operation, all values + * (including padding bytes) will be + * zero. + * Slower than the non-strict variant. + * @dst: Non-NULL pointer to dump buffer. + */ +void kbase_hwcnt_dump_buffer_zero_strict( + struct kbase_hwcnt_dump_buffer *dst); + +/** + * kbase_hwcnt_dump_buffer_zero_non_enabled() - Zero all non-enabled values in + * dst (including padding bytes and + * unavailable blocks). + * After the operation, all enabled + * values will be unchanged. + * @dst: Non-NULL pointer to dump buffer. + * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. + * + * The dst and dst_enable_map MUST have been created from the same metadata. + */ +void kbase_hwcnt_dump_buffer_zero_non_enabled( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_enable_map *dst_enable_map); + +/** + * kbase_hwcnt_dump_buffer_block_zero_non_enabled() - Zero all non-enabled + * values in a block. + * After the operation, all + * enabled values will be + * unchanged. + * @dst_blk: Non-NULL pointer to dst block obtained from a call to + * kbase_hwcnt_dump_buffer_block_instance. + * @blk_em: Non-NULL pointer to the block bitfield(s) obtained from a call to + * kbase_hwcnt_enable_map_block_instance. + * @val_cnt: Number of values in the block. + */ +static inline void kbase_hwcnt_dump_buffer_block_zero_non_enabled( + u32 *dst_blk, + const u64 *blk_em, + size_t val_cnt) +{ + size_t val; + + for (val = 0; val < val_cnt; val++) { + if (!kbase_hwcnt_enable_map_block_value_enabled(blk_em, val)) + dst_blk[val] = 0; + } +} + +/** + * kbase_hwcnt_dump_buffer_copy() - Copy all enabled values from src to dst. + * After the operation, all non-enabled values + * will be undefined. + * @dst: Non-NULL pointer to dst dump buffer. + * @src: Non-NULL pointer to src dump buffer. + * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. + * + * The dst, src, and dst_enable_map MUST have been created from the same + * metadata. + */ +void kbase_hwcnt_dump_buffer_copy( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_dump_buffer *src, + const struct kbase_hwcnt_enable_map *dst_enable_map); + +/** + * kbase_hwcnt_dump_buffer_block_copy() - Copy all block values from src to dst. + * @dst_blk: Non-NULL pointer to dst block obtained from a call to + * kbase_hwcnt_dump_buffer_block_instance. + * @src_blk: Non-NULL pointer to src block obtained from a call to + * kbase_hwcnt_dump_buffer_block_instance. + * @val_cnt: Number of values in the block. + */ +static inline void kbase_hwcnt_dump_buffer_block_copy( + u32 *dst_blk, + const u32 *src_blk, + size_t val_cnt) +{ + /* Copy all the counters in the block instance. + * Values of non-enabled counters are undefined. + */ + memcpy(dst_blk, src_blk, (val_cnt * KBASE_HWCNT_VALUE_BYTES)); +} + +/** + * kbase_hwcnt_dump_buffer_copy_strict() - Copy all enabled values from src to + * dst. + * After the operation, all non-enabled + * values (including padding bytes) will + * be zero. + * Slower than the non-strict variant. + * @dst: Non-NULL pointer to dst dump buffer. + * @src: Non-NULL pointer to src dump buffer. + * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. + * + * The dst, src, and dst_enable_map MUST have been created from the same + * metadata. + */ +void kbase_hwcnt_dump_buffer_copy_strict( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_dump_buffer *src, + const struct kbase_hwcnt_enable_map *dst_enable_map); + +/** + * kbase_hwcnt_dump_buffer_block_copy_strict() - Copy all enabled block values + * from src to dst. + * After the operation, all + * non-enabled values will be + * zero. + * @dst_blk: Non-NULL pointer to dst block obtained from a call to + * kbase_hwcnt_dump_buffer_block_instance. + * @src_blk: Non-NULL pointer to src block obtained from a call to + * kbase_hwcnt_dump_buffer_block_instance. + * @blk_em: Non-NULL pointer to the block bitfield(s) obtained from a call to + * kbase_hwcnt_enable_map_block_instance. + * @val_cnt: Number of values in the block. + * + * After the copy, any disabled values in dst will be zero. + */ +static inline void kbase_hwcnt_dump_buffer_block_copy_strict( + u32 *dst_blk, + const u32 *src_blk, + const u64 *blk_em, + size_t val_cnt) +{ + size_t val; + + for (val = 0; val < val_cnt; val++) { + bool val_enabled = kbase_hwcnt_enable_map_block_value_enabled( + blk_em, val); + + dst_blk[val] = val_enabled ? src_blk[val] : 0; + } +} + +/** + * kbase_hwcnt_dump_buffer_accumulate() - Copy all enabled headers and + * accumulate all enabled counters from + * src to dst. + * After the operation, all non-enabled + * values will be undefined. + * @dst: Non-NULL pointer to dst dump buffer. + * @src: Non-NULL pointer to src dump buffer. + * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. + * + * The dst, src, and dst_enable_map MUST have been created from the same + * metadata. + */ +void kbase_hwcnt_dump_buffer_accumulate( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_dump_buffer *src, + const struct kbase_hwcnt_enable_map *dst_enable_map); + +/** + * kbase_hwcnt_dump_buffer_block_accumulate() - Copy all block headers and + * accumulate all block counters + * from src to dst. + * @dst_blk: Non-NULL pointer to dst block obtained from a call to + * kbase_hwcnt_dump_buffer_block_instance. + * @src_blk: Non-NULL pointer to src block obtained from a call to + * kbase_hwcnt_dump_buffer_block_instance. + * @hdr_cnt: Number of headers in the block. + * @ctr_cnt: Number of counters in the block. + */ +static inline void kbase_hwcnt_dump_buffer_block_accumulate( + u32 *dst_blk, + const u32 *src_blk, + size_t hdr_cnt, + size_t ctr_cnt) +{ + size_t ctr; + /* Copy all the headers in the block instance. + * Values of non-enabled headers are undefined. + */ + memcpy(dst_blk, src_blk, hdr_cnt * KBASE_HWCNT_VALUE_BYTES); + + /* Accumulate all the counters in the block instance. + * Values of non-enabled counters are undefined. + */ + for (ctr = hdr_cnt; ctr < ctr_cnt + hdr_cnt; ctr++) { + u32 *dst_ctr = dst_blk + ctr; + const u32 *src_ctr = src_blk + ctr; + + const u32 src_counter = *src_ctr; + const u32 dst_counter = *dst_ctr; + + /* Saturating add */ + u32 accumulated = src_counter + dst_counter; + + if (accumulated < src_counter) + accumulated = U32_MAX; + + *dst_ctr = accumulated; + } +} + +/** + * kbase_hwcnt_dump_buffer_accumulate_strict() - Copy all enabled headers and + * accumulate all enabled counters + * from src to dst. + * After the operation, all + * non-enabled values (including + * padding bytes) will be zero. + * Slower than the non-strict + * variant. + * @dst: Non-NULL pointer to dst dump buffer. + * @src: Non-NULL pointer to src dump buffer. + * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. + * + * The dst, src, and dst_enable_map MUST have been created from the same + * metadata. + */ +void kbase_hwcnt_dump_buffer_accumulate_strict( + struct kbase_hwcnt_dump_buffer *dst, + const struct kbase_hwcnt_dump_buffer *src, + const struct kbase_hwcnt_enable_map *dst_enable_map); + +/** + * kbase_hwcnt_dump_buffer_block_accumulate_strict() - Copy all enabled block + * headers and accumulate + * all block counters from + * src to dst. + * After the operation, all + * non-enabled values will + * be zero. + * @dst_blk: Non-NULL pointer to dst block obtained from a call to + * kbase_hwcnt_dump_buffer_block_instance. + * @src_blk: Non-NULL pointer to src block obtained from a call to + * kbase_hwcnt_dump_buffer_block_instance. + * @blk_em: Non-NULL pointer to the block bitfield(s) obtained from a call to + * kbase_hwcnt_enable_map_block_instance. + * @hdr_cnt: Number of headers in the block. + * @ctr_cnt: Number of counters in the block. + */ +static inline void kbase_hwcnt_dump_buffer_block_accumulate_strict( + u32 *dst_blk, + const u32 *src_blk, + const u64 *blk_em, + size_t hdr_cnt, + size_t ctr_cnt) +{ + size_t ctr; + + kbase_hwcnt_dump_buffer_block_copy_strict( + dst_blk, src_blk, blk_em, hdr_cnt); + + for (ctr = hdr_cnt; ctr < ctr_cnt + hdr_cnt; ctr++) { + bool ctr_enabled = kbase_hwcnt_enable_map_block_value_enabled( + blk_em, ctr); + + u32 *dst_ctr = dst_blk + ctr; + const u32 *src_ctr = src_blk + ctr; + + const u32 src_counter = *src_ctr; + const u32 dst_counter = *dst_ctr; + + /* Saturating add */ + u32 accumulated = src_counter + dst_counter; + + if (accumulated < src_counter) + accumulated = U32_MAX; + + *dst_ctr = ctr_enabled ? accumulated : 0; + } +} + +/** + * @brief Iterate over each clock domain in the metadata. + * + * @param[in] md Non-NULL pointer to metadata. + * @param[in] clk size_t variable used as clock iterator. + */ +#define kbase_hwcnt_metadata_for_each_clock(md, clk) \ + for ((clk) = 0; (clk) < (md)->clk_cnt; (clk)++) + +/** + * kbase_hwcnt_clk_enable_map_enabled() - Check if the given index is enabled + * in clk_enable_map. + * @clk_enable_map: An enable map for clock domains. + * @index: Index of the enable map for clock domain. + * + * Return: true if the index of the clock domain is enabled, else false. + */ +static inline bool kbase_hwcnt_clk_enable_map_enabled( + const u64 clk_enable_map, const size_t index) +{ + if (clk_enable_map & (1ull << index)) + return true; + return false; +} + +#endif /* _KBASE_HWCNT_TYPES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.c new file mode 100755 index 000000000000..917e47cda0f9 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.c @@ -0,0 +1,790 @@ +/* + * + * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_hwcnt_virtualizer.h" +#include "mali_kbase_hwcnt_accumulator.h" +#include "mali_kbase_hwcnt_context.h" +#include "mali_kbase_hwcnt_types.h" +#include "mali_malisw.h" +#include "mali_kbase_debug.h" +#include "mali_kbase_linux.h" + +#include +#include + +/** + * struct kbase_hwcnt_virtualizer - Hardware counter virtualizer structure. + * @hctx: Hardware counter context being virtualized. + * @dump_threshold_ns: Minimum threshold period for dumps between different + * clients where a new accumulator dump will not be + * performed, and instead accumulated values will be used. + * If 0, rate limiting is disabled. + * @metadata: Hardware counter metadata. + * @lock: Lock acquired at all entrypoints, to protect mutable + * state. + * @client_count: Current number of virtualizer clients. + * @clients: List of virtualizer clients. + * @accum: Hardware counter accumulator. NULL if no clients. + * @scratch_map: Enable map used as scratch space during counter changes. + * @scratch_buf: Dump buffer used as scratch space during dumps. + * @ts_last_dump_ns: End time of most recent dump across all clients. + */ +struct kbase_hwcnt_virtualizer { + struct kbase_hwcnt_context *hctx; + u64 dump_threshold_ns; + const struct kbase_hwcnt_metadata *metadata; + struct mutex lock; + size_t client_count; + struct list_head clients; + struct kbase_hwcnt_accumulator *accum; + struct kbase_hwcnt_enable_map scratch_map; + struct kbase_hwcnt_dump_buffer scratch_buf; + u64 ts_last_dump_ns; +}; + +/** + * struct kbase_hwcnt_virtualizer_client - Virtualizer client structure. + * @node: List node used for virtualizer client list. + * @hvirt: Hardware counter virtualizer. + * @enable_map: Enable map with client's current enabled counters. + * @accum_buf: Dump buffer with client's current accumulated counters. + * @has_accum: True if accum_buf contains any accumulated counters. + * @ts_start_ns: Counter collection start time of current dump. + */ +struct kbase_hwcnt_virtualizer_client { + struct list_head node; + struct kbase_hwcnt_virtualizer *hvirt; + struct kbase_hwcnt_enable_map enable_map; + struct kbase_hwcnt_dump_buffer accum_buf; + bool has_accum; + u64 ts_start_ns; +}; + +const struct kbase_hwcnt_metadata *kbase_hwcnt_virtualizer_metadata( + struct kbase_hwcnt_virtualizer *hvirt) +{ + if (!hvirt) + return NULL; + + return hvirt->metadata; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_metadata); + +/** + * kbasep_hwcnt_virtualizer_client_free - Free a virtualizer client's memory. + * @hvcli: Pointer to virtualizer client. + * + * Will safely free a client in any partial state of construction. + */ +static void kbasep_hwcnt_virtualizer_client_free( + struct kbase_hwcnt_virtualizer_client *hvcli) +{ + if (!hvcli) + return; + + kbase_hwcnt_dump_buffer_free(&hvcli->accum_buf); + kbase_hwcnt_enable_map_free(&hvcli->enable_map); + kfree(hvcli); +} + +/** + * kbasep_hwcnt_virtualizer_client_alloc - Allocate memory for a virtualizer + * client. + * @metadata: Non-NULL pointer to counter metadata. + * @out_hvcli: Non-NULL pointer to where created client will be stored on + * success. + * + * Return: 0 on success, else error code. + */ +static int kbasep_hwcnt_virtualizer_client_alloc( + const struct kbase_hwcnt_metadata *metadata, + struct kbase_hwcnt_virtualizer_client **out_hvcli) +{ + int errcode; + struct kbase_hwcnt_virtualizer_client *hvcli = NULL; + + WARN_ON(!metadata); + WARN_ON(!out_hvcli); + + hvcli = kzalloc(sizeof(*hvcli), GFP_KERNEL); + if (!hvcli) + return -ENOMEM; + + errcode = kbase_hwcnt_enable_map_alloc(metadata, &hvcli->enable_map); + if (errcode) + goto error; + + errcode = kbase_hwcnt_dump_buffer_alloc(metadata, &hvcli->accum_buf); + if (errcode) + goto error; + + *out_hvcli = hvcli; + return 0; +error: + kbasep_hwcnt_virtualizer_client_free(hvcli); + return errcode; +} + +/** + * kbasep_hwcnt_virtualizer_client_accumulate - Accumulate a dump buffer into a + * client's accumulation buffer. + * @hvcli: Non-NULL pointer to virtualizer client. + * @dump_buf: Non-NULL pointer to dump buffer to accumulate from. + */ +static void kbasep_hwcnt_virtualizer_client_accumulate( + struct kbase_hwcnt_virtualizer_client *hvcli, + const struct kbase_hwcnt_dump_buffer *dump_buf) +{ + WARN_ON(!hvcli); + WARN_ON(!dump_buf); + lockdep_assert_held(&hvcli->hvirt->lock); + + if (hvcli->has_accum) { + /* If already some accumulation, accumulate */ + kbase_hwcnt_dump_buffer_accumulate( + &hvcli->accum_buf, dump_buf, &hvcli->enable_map); + } else { + /* If no accumulation, copy */ + kbase_hwcnt_dump_buffer_copy( + &hvcli->accum_buf, dump_buf, &hvcli->enable_map); + } + hvcli->has_accum = true; +} + +/** + * kbasep_hwcnt_virtualizer_accumulator_term - Terminate the hardware counter + * accumulator after final client + * removal. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * + * Will safely terminate the accumulator in any partial state of initialisation. + */ +static void kbasep_hwcnt_virtualizer_accumulator_term( + struct kbase_hwcnt_virtualizer *hvirt) +{ + WARN_ON(!hvirt); + lockdep_assert_held(&hvirt->lock); + WARN_ON(hvirt->client_count); + + kbase_hwcnt_dump_buffer_free(&hvirt->scratch_buf); + kbase_hwcnt_enable_map_free(&hvirt->scratch_map); + kbase_hwcnt_accumulator_release(hvirt->accum); + hvirt->accum = NULL; +} + +/** + * kbasep_hwcnt_virtualizer_accumulator_init - Initialise the hardware counter + * accumulator before first client + * addition. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * + * Return: 0 on success, else error code. + */ +static int kbasep_hwcnt_virtualizer_accumulator_init( + struct kbase_hwcnt_virtualizer *hvirt) +{ + int errcode; + + WARN_ON(!hvirt); + lockdep_assert_held(&hvirt->lock); + WARN_ON(hvirt->client_count); + WARN_ON(hvirt->accum); + + errcode = kbase_hwcnt_accumulator_acquire( + hvirt->hctx, &hvirt->accum); + if (errcode) + goto error; + + errcode = kbase_hwcnt_enable_map_alloc( + hvirt->metadata, &hvirt->scratch_map); + if (errcode) + goto error; + + errcode = kbase_hwcnt_dump_buffer_alloc( + hvirt->metadata, &hvirt->scratch_buf); + if (errcode) + goto error; + + return 0; +error: + kbasep_hwcnt_virtualizer_accumulator_term(hvirt); + return errcode; +} + +/** + * kbasep_hwcnt_virtualizer_client_add - Add a newly allocated client to the + * virtualizer. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * @hvcli: Non-NULL pointer to the virtualizer client to add. + * @enable_map: Non-NULL pointer to client's initial enable map. + * + * Return: 0 on success, else error code. + */ +static int kbasep_hwcnt_virtualizer_client_add( + struct kbase_hwcnt_virtualizer *hvirt, + struct kbase_hwcnt_virtualizer_client *hvcli, + const struct kbase_hwcnt_enable_map *enable_map) +{ + int errcode = 0; + u64 ts_start_ns; + u64 ts_end_ns; + + WARN_ON(!hvirt); + WARN_ON(!hvcli); + WARN_ON(!enable_map); + lockdep_assert_held(&hvirt->lock); + + if (hvirt->client_count == 0) + /* First client added, so initialise the accumulator */ + errcode = kbasep_hwcnt_virtualizer_accumulator_init(hvirt); + if (errcode) + return errcode; + + hvirt->client_count += 1; + + if (hvirt->client_count == 1) { + /* First client, so just pass the enable map onwards as is */ + errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, + enable_map, &ts_start_ns, &ts_end_ns, NULL); + } else { + struct kbase_hwcnt_virtualizer_client *pos; + + /* Make the scratch enable map the union of all enable maps */ + kbase_hwcnt_enable_map_copy( + &hvirt->scratch_map, enable_map); + list_for_each_entry(pos, &hvirt->clients, node) + kbase_hwcnt_enable_map_union( + &hvirt->scratch_map, &pos->enable_map); + + /* Set the counters with the new union enable map */ + errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, + &hvirt->scratch_map, + &ts_start_ns, &ts_end_ns, + &hvirt->scratch_buf); + /* Accumulate into only existing clients' accumulation bufs */ + if (!errcode) + list_for_each_entry(pos, &hvirt->clients, node) + kbasep_hwcnt_virtualizer_client_accumulate( + pos, &hvirt->scratch_buf); + } + if (errcode) + goto error; + + list_add(&hvcli->node, &hvirt->clients); + hvcli->hvirt = hvirt; + kbase_hwcnt_enable_map_copy(&hvcli->enable_map, enable_map); + hvcli->has_accum = false; + hvcli->ts_start_ns = ts_end_ns; + + /* Store the most recent dump time for rate limiting */ + hvirt->ts_last_dump_ns = ts_end_ns; + + return 0; +error: + hvirt->client_count -= 1; + if (hvirt->client_count == 0) + kbasep_hwcnt_virtualizer_accumulator_term(hvirt); + return errcode; +} + +/** + * kbasep_hwcnt_virtualizer_client_remove - Remove a client from the + * virtualizer. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * @hvcli: Non-NULL pointer to the virtualizer client to remove. + */ +static void kbasep_hwcnt_virtualizer_client_remove( + struct kbase_hwcnt_virtualizer *hvirt, + struct kbase_hwcnt_virtualizer_client *hvcli) +{ + int errcode = 0; + u64 ts_start_ns; + u64 ts_end_ns; + + WARN_ON(!hvirt); + WARN_ON(!hvcli); + lockdep_assert_held(&hvirt->lock); + + list_del(&hvcli->node); + hvirt->client_count -= 1; + + if (hvirt->client_count == 0) { + /* Last client removed, so terminate the accumulator */ + kbasep_hwcnt_virtualizer_accumulator_term(hvirt); + } else { + struct kbase_hwcnt_virtualizer_client *pos; + /* Make the scratch enable map the union of all enable maps */ + kbase_hwcnt_enable_map_disable_all(&hvirt->scratch_map); + list_for_each_entry(pos, &hvirt->clients, node) + kbase_hwcnt_enable_map_union( + &hvirt->scratch_map, &pos->enable_map); + /* Set the counters with the new union enable map */ + errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, + &hvirt->scratch_map, + &ts_start_ns, &ts_end_ns, + &hvirt->scratch_buf); + /* Accumulate into remaining clients' accumulation bufs */ + if (!errcode) + list_for_each_entry(pos, &hvirt->clients, node) + kbasep_hwcnt_virtualizer_client_accumulate( + pos, &hvirt->scratch_buf); + + /* Store the most recent dump time for rate limiting */ + hvirt->ts_last_dump_ns = ts_end_ns; + } + WARN_ON(errcode); +} + +/** + * kbasep_hwcnt_virtualizer_client_set_counters - Perform a dump of the client's + * currently enabled counters, + * and enable a new set of + * counters that will be used for + * subsequent dumps. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * @hvcli: Non-NULL pointer to the virtualizer client. + * @enable_map: Non-NULL pointer to the new counter enable map for the client. + * Must have the same metadata as the virtualizer. + * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will + * be written out to on success. + * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will + * be written out to on success. + * @dump_buf: Pointer to the buffer where the dump will be written out to on + * success. If non-NULL, must have the same metadata as the + * accumulator. If NULL, the dump will be discarded. + * + * Return: 0 on success or error code. + */ +static int kbasep_hwcnt_virtualizer_client_set_counters( + struct kbase_hwcnt_virtualizer *hvirt, + struct kbase_hwcnt_virtualizer_client *hvcli, + const struct kbase_hwcnt_enable_map *enable_map, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf) +{ + int errcode; + struct kbase_hwcnt_virtualizer_client *pos; + + WARN_ON(!hvirt); + WARN_ON(!hvcli); + WARN_ON(!enable_map); + WARN_ON(!ts_start_ns); + WARN_ON(!ts_end_ns); + WARN_ON(enable_map->metadata != hvirt->metadata); + WARN_ON(dump_buf && (dump_buf->metadata != hvirt->metadata)); + lockdep_assert_held(&hvirt->lock); + + /* Make the scratch enable map the union of all enable maps */ + kbase_hwcnt_enable_map_copy(&hvirt->scratch_map, enable_map); + list_for_each_entry(pos, &hvirt->clients, node) + /* Ignore the enable map of the selected client */ + if (pos != hvcli) + kbase_hwcnt_enable_map_union( + &hvirt->scratch_map, &pos->enable_map); + + /* Set the counters with the new union enable map */ + errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, + &hvirt->scratch_map, ts_start_ns, ts_end_ns, + &hvirt->scratch_buf); + if (errcode) + return errcode; + + /* Accumulate into all accumulation bufs except the selected client's */ + list_for_each_entry(pos, &hvirt->clients, node) + if (pos != hvcli) + kbasep_hwcnt_virtualizer_client_accumulate( + pos, &hvirt->scratch_buf); + + /* Finally, write into the dump buf */ + if (dump_buf) { + const struct kbase_hwcnt_dump_buffer *src = &hvirt->scratch_buf; + + if (hvcli->has_accum) { + kbase_hwcnt_dump_buffer_accumulate( + &hvcli->accum_buf, src, &hvcli->enable_map); + src = &hvcli->accum_buf; + } + kbase_hwcnt_dump_buffer_copy(dump_buf, src, &hvcli->enable_map); + } + hvcli->has_accum = false; + + /* Update the selected client's enable map */ + kbase_hwcnt_enable_map_copy(&hvcli->enable_map, enable_map); + + /* Fix up the timestamps */ + *ts_start_ns = hvcli->ts_start_ns; + hvcli->ts_start_ns = *ts_end_ns; + + /* Store the most recent dump time for rate limiting */ + hvirt->ts_last_dump_ns = *ts_end_ns; + + return errcode; +} + +int kbase_hwcnt_virtualizer_client_set_counters( + struct kbase_hwcnt_virtualizer_client *hvcli, + const struct kbase_hwcnt_enable_map *enable_map, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf) +{ + int errcode; + struct kbase_hwcnt_virtualizer *hvirt; + + if (!hvcli || !enable_map || !ts_start_ns || !ts_end_ns) + return -EINVAL; + + hvirt = hvcli->hvirt; + + if ((enable_map->metadata != hvirt->metadata) || + (dump_buf && (dump_buf->metadata != hvirt->metadata))) + return -EINVAL; + + mutex_lock(&hvirt->lock); + + if ((hvirt->client_count == 1) && (!hvcli->has_accum)) { + /* + * If there's only one client with no prior accumulation, we can + * completely skip the virtualize and just pass through the call + * to the accumulator, saving a fair few copies and + * accumulations. + */ + errcode = kbase_hwcnt_accumulator_set_counters( + hvirt->accum, enable_map, + ts_start_ns, ts_end_ns, dump_buf); + + if (!errcode) { + /* Update the selected client's enable map */ + kbase_hwcnt_enable_map_copy( + &hvcli->enable_map, enable_map); + + /* Fix up the timestamps */ + *ts_start_ns = hvcli->ts_start_ns; + hvcli->ts_start_ns = *ts_end_ns; + + /* Store the most recent dump time for rate limiting */ + hvirt->ts_last_dump_ns = *ts_end_ns; + } + } else { + /* Otherwise, do the full virtualize */ + errcode = kbasep_hwcnt_virtualizer_client_set_counters( + hvirt, hvcli, enable_map, + ts_start_ns, ts_end_ns, dump_buf); + } + + mutex_unlock(&hvirt->lock); + + return errcode; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_set_counters); + +/** + * kbasep_hwcnt_virtualizer_client_dump - Perform a dump of the client's + * currently enabled counters. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * @hvcli: Non-NULL pointer to the virtualizer client. + * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will + * be written out to on success. + * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will + * be written out to on success. + * @dump_buf: Pointer to the buffer where the dump will be written out to on + * success. If non-NULL, must have the same metadata as the + * accumulator. If NULL, the dump will be discarded. + * + * Return: 0 on success or error code. + */ +static int kbasep_hwcnt_virtualizer_client_dump( + struct kbase_hwcnt_virtualizer *hvirt, + struct kbase_hwcnt_virtualizer_client *hvcli, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf) +{ + int errcode; + struct kbase_hwcnt_virtualizer_client *pos; + + WARN_ON(!hvirt); + WARN_ON(!hvcli); + WARN_ON(!ts_start_ns); + WARN_ON(!ts_end_ns); + WARN_ON(dump_buf && (dump_buf->metadata != hvirt->metadata)); + lockdep_assert_held(&hvirt->lock); + + /* Perform the dump */ + errcode = kbase_hwcnt_accumulator_dump(hvirt->accum, + ts_start_ns, ts_end_ns, &hvirt->scratch_buf); + if (errcode) + return errcode; + + /* Accumulate into all accumulation bufs except the selected client's */ + list_for_each_entry(pos, &hvirt->clients, node) + if (pos != hvcli) + kbasep_hwcnt_virtualizer_client_accumulate( + pos, &hvirt->scratch_buf); + + /* Finally, write into the dump buf */ + if (dump_buf) { + const struct kbase_hwcnt_dump_buffer *src = &hvirt->scratch_buf; + + if (hvcli->has_accum) { + kbase_hwcnt_dump_buffer_accumulate( + &hvcli->accum_buf, src, &hvcli->enable_map); + src = &hvcli->accum_buf; + } + kbase_hwcnt_dump_buffer_copy(dump_buf, src, &hvcli->enable_map); + } + hvcli->has_accum = false; + + /* Fix up the timestamps */ + *ts_start_ns = hvcli->ts_start_ns; + hvcli->ts_start_ns = *ts_end_ns; + + /* Store the most recent dump time for rate limiting */ + hvirt->ts_last_dump_ns = *ts_end_ns; + + return errcode; +} + +/** + * kbasep_hwcnt_virtualizer_client_dump_rate_limited - Perform a dump of the + * client's currently enabled counters + * if it hasn't been rate limited, + * otherwise return the client's most + * recent accumulation. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * @hvcli: Non-NULL pointer to the virtualizer client. + * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will + * be written out to on success. + * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will + * be written out to on success. + * @dump_buf: Pointer to the buffer where the dump will be written out to on + * success. If non-NULL, must have the same metadata as the + * accumulator. If NULL, the dump will be discarded. + * + * Return: 0 on success or error code. + */ +static int kbasep_hwcnt_virtualizer_client_dump_rate_limited( + struct kbase_hwcnt_virtualizer *hvirt, + struct kbase_hwcnt_virtualizer_client *hvcli, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf) +{ + bool rate_limited = true; + + WARN_ON(!hvirt); + WARN_ON(!hvcli); + WARN_ON(!ts_start_ns); + WARN_ON(!ts_end_ns); + WARN_ON(dump_buf && (dump_buf->metadata != hvirt->metadata)); + lockdep_assert_held(&hvirt->lock); + + if (hvirt->dump_threshold_ns == 0) { + /* Threshold == 0, so rate limiting disabled */ + rate_limited = false; + } else if (hvirt->ts_last_dump_ns == hvcli->ts_start_ns) { + /* Last dump was performed by this client, and dumps from an + * individual client are never rate limited + */ + rate_limited = false; + } else { + const u64 ts_ns = + kbase_hwcnt_accumulator_timestamp_ns(hvirt->accum); + const u64 time_since_last_dump_ns = + ts_ns - hvirt->ts_last_dump_ns; + + /* Dump period equals or exceeds the threshold */ + if (time_since_last_dump_ns >= hvirt->dump_threshold_ns) + rate_limited = false; + } + + if (!rate_limited) + return kbasep_hwcnt_virtualizer_client_dump( + hvirt, hvcli, ts_start_ns, ts_end_ns, dump_buf); + + /* If we've gotten this far, the client must have something accumulated + * otherwise it is a logic error + */ + WARN_ON(!hvcli->has_accum); + + if (dump_buf) + kbase_hwcnt_dump_buffer_copy( + dump_buf, &hvcli->accum_buf, &hvcli->enable_map); + hvcli->has_accum = false; + + *ts_start_ns = hvcli->ts_start_ns; + *ts_end_ns = hvirt->ts_last_dump_ns; + hvcli->ts_start_ns = hvirt->ts_last_dump_ns; + + return 0; +} + +int kbase_hwcnt_virtualizer_client_dump( + struct kbase_hwcnt_virtualizer_client *hvcli, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf) +{ + int errcode; + struct kbase_hwcnt_virtualizer *hvirt; + + if (!hvcli || !ts_start_ns || !ts_end_ns) + return -EINVAL; + + hvirt = hvcli->hvirt; + + if (dump_buf && (dump_buf->metadata != hvirt->metadata)) + return -EINVAL; + + mutex_lock(&hvirt->lock); + + if ((hvirt->client_count == 1) && (!hvcli->has_accum)) { + /* + * If there's only one client with no prior accumulation, we can + * completely skip the virtualize and just pass through the call + * to the accumulator, saving a fair few copies and + * accumulations. + */ + errcode = kbase_hwcnt_accumulator_dump( + hvirt->accum, ts_start_ns, ts_end_ns, dump_buf); + + if (!errcode) { + /* Fix up the timestamps */ + *ts_start_ns = hvcli->ts_start_ns; + hvcli->ts_start_ns = *ts_end_ns; + + /* Store the most recent dump time for rate limiting */ + hvirt->ts_last_dump_ns = *ts_end_ns; + } + } else { + /* Otherwise, do the full virtualize */ + errcode = kbasep_hwcnt_virtualizer_client_dump_rate_limited( + hvirt, hvcli, ts_start_ns, ts_end_ns, dump_buf); + } + + mutex_unlock(&hvirt->lock); + + return errcode; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_dump); + +int kbase_hwcnt_virtualizer_client_create( + struct kbase_hwcnt_virtualizer *hvirt, + const struct kbase_hwcnt_enable_map *enable_map, + struct kbase_hwcnt_virtualizer_client **out_hvcli) +{ + int errcode; + struct kbase_hwcnt_virtualizer_client *hvcli; + + if (!hvirt || !enable_map || !out_hvcli || + (enable_map->metadata != hvirt->metadata)) + return -EINVAL; + + errcode = kbasep_hwcnt_virtualizer_client_alloc( + hvirt->metadata, &hvcli); + if (errcode) + return errcode; + + mutex_lock(&hvirt->lock); + + errcode = kbasep_hwcnt_virtualizer_client_add(hvirt, hvcli, enable_map); + + mutex_unlock(&hvirt->lock); + + if (errcode) { + kbasep_hwcnt_virtualizer_client_free(hvcli); + return errcode; + } + + *out_hvcli = hvcli; + return 0; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_create); + +void kbase_hwcnt_virtualizer_client_destroy( + struct kbase_hwcnt_virtualizer_client *hvcli) +{ + if (!hvcli) + return; + + mutex_lock(&hvcli->hvirt->lock); + + kbasep_hwcnt_virtualizer_client_remove(hvcli->hvirt, hvcli); + + mutex_unlock(&hvcli->hvirt->lock); + + kbasep_hwcnt_virtualizer_client_free(hvcli); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_destroy); + +int kbase_hwcnt_virtualizer_init( + struct kbase_hwcnt_context *hctx, + u64 dump_threshold_ns, + struct kbase_hwcnt_virtualizer **out_hvirt) +{ + struct kbase_hwcnt_virtualizer *virt; + const struct kbase_hwcnt_metadata *metadata; + + if (!hctx || !out_hvirt) + return -EINVAL; + + metadata = kbase_hwcnt_context_metadata(hctx); + if (!metadata) + return -EINVAL; + + virt = kzalloc(sizeof(*virt), GFP_KERNEL); + if (!virt) + return -ENOMEM; + + virt->hctx = hctx; + virt->dump_threshold_ns = dump_threshold_ns; + virt->metadata = metadata; + + mutex_init(&virt->lock); + INIT_LIST_HEAD(&virt->clients); + + *out_hvirt = virt; + return 0; +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_init); + +void kbase_hwcnt_virtualizer_term( + struct kbase_hwcnt_virtualizer *hvirt) +{ + if (!hvirt) + return; + + /* Non-zero client count implies client leak */ + if (WARN_ON(hvirt->client_count != 0)) { + struct kbase_hwcnt_virtualizer_client *pos, *n; + + list_for_each_entry_safe(pos, n, &hvirt->clients, node) + kbase_hwcnt_virtualizer_client_destroy(pos); + } + + WARN_ON(hvirt->client_count != 0); + WARN_ON(hvirt->accum); + + kfree(hvirt); +} +KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_term); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.h b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.h new file mode 100755 index 000000000000..8f628c3306fc --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_virtualizer.h @@ -0,0 +1,145 @@ +/* + * + * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Hardware counter virtualizer API. + * + * Virtualizes a hardware counter context, so multiple clients can access + * a single hardware counter resource as though each was the exclusive user. + */ + +#ifndef _KBASE_HWCNT_VIRTUALIZER_H_ +#define _KBASE_HWCNT_VIRTUALIZER_H_ + +#include + +struct kbase_hwcnt_context; +struct kbase_hwcnt_virtualizer; +struct kbase_hwcnt_virtualizer_client; +struct kbase_hwcnt_enable_map; +struct kbase_hwcnt_dump_buffer; + +/** + * kbase_hwcnt_virtualizer_init - Initialise a hardware counter virtualizer. + * @hctx: Non-NULL pointer to the hardware counter context to + * virtualize. + * @dump_threshold_ns: Minimum threshold period for dumps between different + * clients where a new accumulator dump will not be + * performed, and instead accumulated values will be used. + * If 0, rate limiting will be disabled. + * @out_hvirt: Non-NULL pointer to where the pointer to the created + * virtualizer will be stored on success. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_virtualizer_init( + struct kbase_hwcnt_context *hctx, + u64 dump_threshold_ns, + struct kbase_hwcnt_virtualizer **out_hvirt); + +/** + * kbase_hwcnt_virtualizer_term - Terminate a hardware counter virtualizer. + * @hvirt: Pointer to virtualizer to be terminated. + */ +void kbase_hwcnt_virtualizer_term( + struct kbase_hwcnt_virtualizer *hvirt); + +/** + * kbase_hwcnt_virtualizer_metadata - Get the hardware counter metadata used by + * the virtualizer, so related counter data + * structures can be created. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * + * Return: Non-NULL pointer to metadata, or NULL on error. + */ +const struct kbase_hwcnt_metadata *kbase_hwcnt_virtualizer_metadata( + struct kbase_hwcnt_virtualizer *hvirt); + +/** + * kbase_hwcnt_virtualizer_client_create - Create a new virtualizer client. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * @enable_map: Non-NULL pointer to the enable map for the client. Must have the + * same metadata as the virtualizer. + * @out_hvcli: Non-NULL pointer to where the pointer to the created client will + * be stored on success. + * + * Return: 0 on success, else error code. + */ +int kbase_hwcnt_virtualizer_client_create( + struct kbase_hwcnt_virtualizer *hvirt, + const struct kbase_hwcnt_enable_map *enable_map, + struct kbase_hwcnt_virtualizer_client **out_hvcli); + +/** + * kbase_hwcnt_virtualizer_client_destroy() - Destroy a virtualizer client. + * @hvcli: Pointer to the hardware counter client. + */ +void kbase_hwcnt_virtualizer_client_destroy( + struct kbase_hwcnt_virtualizer_client *hvcli); + +/** + * kbase_hwcnt_virtualizer_client_set_counters - Perform a dump of the client's + * currently enabled counters, and + * enable a new set of counters + * that will be used for + * subsequent dumps. + * @hvcli: Non-NULL pointer to the virtualizer client. + * @enable_map: Non-NULL pointer to the new counter enable map for the client. + * Must have the same metadata as the virtualizer. + * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will + * be written out to on success. + * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will + * be written out to on success. + * @dump_buf: Pointer to the buffer where the dump will be written out to on + * success. If non-NULL, must have the same metadata as the + * accumulator. If NULL, the dump will be discarded. + * + * Return: 0 on success or error code. + */ +int kbase_hwcnt_virtualizer_client_set_counters( + struct kbase_hwcnt_virtualizer_client *hvcli, + const struct kbase_hwcnt_enable_map *enable_map, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf); + +/** + * kbase_hwcnt_virtualizer_client_dump - Perform a dump of the client's + * currently enabled counters. + * @hvcli: Non-NULL pointer to the virtualizer client. + * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will + * be written out to on success. + * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will + * be written out to on success. + * @dump_buf: Pointer to the buffer where the dump will be written out to on + * success. If non-NULL, must have the same metadata as the + * accumulator. If NULL, the dump will be discarded. + * + * Return: 0 on success or error code. + */ +int kbase_hwcnt_virtualizer_client_dump( + struct kbase_hwcnt_virtualizer_client *hvcli, + u64 *ts_start_ns, + u64 *ts_end_ns, + struct kbase_hwcnt_dump_buffer *dump_buf); + +#endif /* _KBASE_HWCNT_VIRTUALIZER_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h b/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h new file mode 100755 index 000000000000..fed45100b4be --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h @@ -0,0 +1,838 @@ +/* + * + * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_IOCTL_H_ +#define _KBASE_IOCTL_H_ + +#ifdef __cpluscplus +extern "C" { +#endif + +#include +#include + +#if MALI_USE_CSF +#include "csf/mali_kbase_csf_ioctl.h" +#else +#include "jm/mali_kbase_jm_ioctl.h" +#endif /* MALI_USE_CSF */ + +#define KBASE_IOCTL_TYPE 0x80 + +/** + * struct kbase_ioctl_set_flags - Set kernel context creation flags + * + * @create_flags: Flags - see base_context_create_flags + */ +struct kbase_ioctl_set_flags { + __u32 create_flags; +}; + +#define KBASE_IOCTL_SET_FLAGS \ + _IOW(KBASE_IOCTL_TYPE, 1, struct kbase_ioctl_set_flags) + +/** + * struct kbase_ioctl_get_gpuprops - Read GPU properties from the kernel + * + * @buffer: Pointer to the buffer to store properties into + * @size: Size of the buffer + * @flags: Flags - must be zero for now + * + * The ioctl will return the number of bytes stored into @buffer or an error + * on failure (e.g. @size is too small). If @size is specified as 0 then no + * data will be written but the return value will be the number of bytes needed + * for all the properties. + * + * @flags may be used in the future to request a different format for the + * buffer. With @flags == 0 the following format is used. + * + * The buffer will be filled with pairs of values, a u32 key identifying the + * property followed by the value. The size of the value is identified using + * the bottom bits of the key. The value then immediately followed the key and + * is tightly packed (there is no padding). All keys and values are + * little-endian. + * + * 00 = u8 + * 01 = u16 + * 10 = u32 + * 11 = u64 + */ +struct kbase_ioctl_get_gpuprops { + __u64 buffer; + __u32 size; + __u32 flags; +}; + +#define KBASE_IOCTL_GET_GPUPROPS \ + _IOW(KBASE_IOCTL_TYPE, 3, struct kbase_ioctl_get_gpuprops) + +/** + * union kbase_ioctl_mem_alloc - Allocate memory on the GPU + * + * @va_pages: The number of pages of virtual address space to reserve + * @commit_pages: The number of physical pages to allocate + * @extent: The number of extra pages to allocate on each GPU fault which grows + * the region + * @flags: Flags + * @gpu_va: The GPU virtual address which is allocated + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_alloc { + struct { + __u64 va_pages; + __u64 commit_pages; + __u64 extent; + __u64 flags; + } in; + struct { + __u64 flags; + __u64 gpu_va; + } out; +}; + +#define KBASE_IOCTL_MEM_ALLOC \ + _IOWR(KBASE_IOCTL_TYPE, 5, union kbase_ioctl_mem_alloc) + +/** + * struct kbase_ioctl_mem_query - Query properties of a GPU memory region + * @gpu_addr: A GPU address contained within the region + * @query: The type of query + * @value: The result of the query + * + * Use a %KBASE_MEM_QUERY_xxx flag as input for @query. + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_query { + struct { + __u64 gpu_addr; + __u64 query; + } in; + struct { + __u64 value; + } out; +}; + +#define KBASE_IOCTL_MEM_QUERY \ + _IOWR(KBASE_IOCTL_TYPE, 6, union kbase_ioctl_mem_query) + +#define KBASE_MEM_QUERY_COMMIT_SIZE ((u64)1) +#define KBASE_MEM_QUERY_VA_SIZE ((u64)2) +#define KBASE_MEM_QUERY_FLAGS ((u64)3) + +/** + * struct kbase_ioctl_mem_free - Free a memory region + * @gpu_addr: Handle to the region to free + */ +struct kbase_ioctl_mem_free { + __u64 gpu_addr; +}; + +#define KBASE_IOCTL_MEM_FREE \ + _IOW(KBASE_IOCTL_TYPE, 7, struct kbase_ioctl_mem_free) + +/** + * struct kbase_ioctl_hwcnt_reader_setup - Setup HWC dumper/reader + * @buffer_count: requested number of dumping buffers + * @fe_bm: counters selection bitmask (Front end) + * @shader_bm: counters selection bitmask (Shader) + * @tiler_bm: counters selection bitmask (Tiler) + * @mmu_l2_bm: counters selection bitmask (MMU_L2) + * + * A fd is returned from the ioctl if successful, or a negative value on error + */ +struct kbase_ioctl_hwcnt_reader_setup { + __u32 buffer_count; + __u32 fe_bm; + __u32 shader_bm; + __u32 tiler_bm; + __u32 mmu_l2_bm; +}; + +#define KBASE_IOCTL_HWCNT_READER_SETUP \ + _IOW(KBASE_IOCTL_TYPE, 8, struct kbase_ioctl_hwcnt_reader_setup) + +/** + * struct kbase_ioctl_hwcnt_enable - Enable hardware counter collection + * @dump_buffer: GPU address to write counters to + * @fe_bm: counters selection bitmask (Front end) + * @shader_bm: counters selection bitmask (Shader) + * @tiler_bm: counters selection bitmask (Tiler) + * @mmu_l2_bm: counters selection bitmask (MMU_L2) + */ +struct kbase_ioctl_hwcnt_enable { + __u64 dump_buffer; + __u32 fe_bm; + __u32 shader_bm; + __u32 tiler_bm; + __u32 mmu_l2_bm; +}; + +#define KBASE_IOCTL_HWCNT_ENABLE \ + _IOW(KBASE_IOCTL_TYPE, 9, struct kbase_ioctl_hwcnt_enable) + +#define KBASE_IOCTL_HWCNT_DUMP \ + _IO(KBASE_IOCTL_TYPE, 10) + +#define KBASE_IOCTL_HWCNT_CLEAR \ + _IO(KBASE_IOCTL_TYPE, 11) + +/** + * struct kbase_ioctl_hwcnt_values - Values to set dummy the dummy counters to. + * @data: Counter samples for the dummy model. + * @size: Size of the counter sample data. + * @padding: Padding. + */ +struct kbase_ioctl_hwcnt_values { + __u64 data; + __u32 size; + __u32 padding; +}; + +#define KBASE_IOCTL_HWCNT_SET \ + _IOW(KBASE_IOCTL_TYPE, 32, struct kbase_ioctl_hwcnt_values) + +/** + * struct kbase_ioctl_disjoint_query - Query the disjoint counter + * @counter: A counter of disjoint events in the kernel + */ +struct kbase_ioctl_disjoint_query { + __u32 counter; +}; + +#define KBASE_IOCTL_DISJOINT_QUERY \ + _IOR(KBASE_IOCTL_TYPE, 12, struct kbase_ioctl_disjoint_query) + +/** + * struct kbase_ioctl_get_ddk_version - Query the kernel version + * @version_buffer: Buffer to receive the kernel version string + * @size: Size of the buffer + * @padding: Padding + * + * The ioctl will return the number of bytes written into version_buffer + * (which includes a NULL byte) or a negative error code + * + * The ioctl request code has to be _IOW because the data in ioctl struct is + * being copied to the kernel, even though the kernel then writes out the + * version info to the buffer specified in the ioctl. + */ +struct kbase_ioctl_get_ddk_version { + __u64 version_buffer; + __u32 size; + __u32 padding; +}; + +#define KBASE_IOCTL_GET_DDK_VERSION \ + _IOW(KBASE_IOCTL_TYPE, 13, struct kbase_ioctl_get_ddk_version) + +/** + * struct kbase_ioctl_mem_jit_init_10_2 - Initialize the just-in-time memory + * allocator (between kernel driver + * version 10.2--11.4) + * @va_pages: Number of VA pages to reserve for JIT + * + * Note that depending on the VA size of the application and GPU, the value + * specified in @va_pages may be ignored. + * + * New code should use KBASE_IOCTL_MEM_JIT_INIT instead, this is kept for + * backwards compatibility. + */ +struct kbase_ioctl_mem_jit_init_10_2 { + __u64 va_pages; +}; + +#define KBASE_IOCTL_MEM_JIT_INIT_10_2 \ + _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init_10_2) + +/** + * struct kbase_ioctl_mem_jit_init_11_5 - Initialize the just-in-time memory + * allocator (between kernel driver + * version 11.5--11.19) + * @va_pages: Number of VA pages to reserve for JIT + * @max_allocations: Maximum number of concurrent allocations + * @trim_level: Level of JIT allocation trimming to perform on free (0 - 100%) + * @group_id: Group ID to be used for physical allocations + * @padding: Currently unused, must be zero + * + * Note that depending on the VA size of the application and GPU, the value + * specified in @va_pages may be ignored. + * + * New code should use KBASE_IOCTL_MEM_JIT_INIT instead, this is kept for + * backwards compatibility. + */ +struct kbase_ioctl_mem_jit_init_11_5 { + __u64 va_pages; + __u8 max_allocations; + __u8 trim_level; + __u8 group_id; + __u8 padding[5]; +}; + +#define KBASE_IOCTL_MEM_JIT_INIT_11_5 \ + _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init_11_5) + +/** + * struct kbase_ioctl_mem_jit_init - Initialize the just-in-time memory + * allocator + * @va_pages: Number of GPU virtual address pages to reserve for just-in-time + * memory allocations + * @max_allocations: Maximum number of concurrent allocations + * @trim_level: Level of JIT allocation trimming to perform on free (0 - 100%) + * @group_id: Group ID to be used for physical allocations + * @padding: Currently unused, must be zero + * @phys_pages: Maximum number of physical pages to allocate just-in-time + * + * Note that depending on the VA size of the application and GPU, the value + * specified in @va_pages may be ignored. + */ +struct kbase_ioctl_mem_jit_init { + __u64 va_pages; + __u8 max_allocations; + __u8 trim_level; + __u8 group_id; + __u8 padding[5]; + __u64 phys_pages; +}; + +#define KBASE_IOCTL_MEM_JIT_INIT \ + _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init) + +/** + * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory + * + * @handle: GPU memory handle (GPU VA) + * @user_addr: The address where it is mapped in user space + * @size: The number of bytes to synchronise + * @type: The direction to synchronise: 0 is sync to memory (clean), + * 1 is sync from memory (invalidate). Use the BASE_SYNCSET_OP_xxx constants. + * @padding: Padding to round up to a multiple of 8 bytes, must be zero + */ +struct kbase_ioctl_mem_sync { + __u64 handle; + __u64 user_addr; + __u64 size; + __u8 type; + __u8 padding[7]; +}; + +#define KBASE_IOCTL_MEM_SYNC \ + _IOW(KBASE_IOCTL_TYPE, 15, struct kbase_ioctl_mem_sync) + +/** + * union kbase_ioctl_mem_find_cpu_offset - Find the offset of a CPU pointer + * + * @gpu_addr: The GPU address of the memory region + * @cpu_addr: The CPU address to locate + * @size: A size in bytes to validate is contained within the region + * @offset: The offset from the start of the memory region to @cpu_addr + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_find_cpu_offset { + struct { + __u64 gpu_addr; + __u64 cpu_addr; + __u64 size; + } in; + struct { + __u64 offset; + } out; +}; + +#define KBASE_IOCTL_MEM_FIND_CPU_OFFSET \ + _IOWR(KBASE_IOCTL_TYPE, 16, union kbase_ioctl_mem_find_cpu_offset) + +/** + * struct kbase_ioctl_get_context_id - Get the kernel context ID + * + * @id: The kernel context ID + */ +struct kbase_ioctl_get_context_id { + __u32 id; +}; + +#define KBASE_IOCTL_GET_CONTEXT_ID \ + _IOR(KBASE_IOCTL_TYPE, 17, struct kbase_ioctl_get_context_id) + +/** + * struct kbase_ioctl_tlstream_acquire - Acquire a tlstream fd + * + * @flags: Flags + * + * The ioctl returns a file descriptor when successful + */ +struct kbase_ioctl_tlstream_acquire { + __u32 flags; +}; + +#define KBASE_IOCTL_TLSTREAM_ACQUIRE \ + _IOW(KBASE_IOCTL_TYPE, 18, struct kbase_ioctl_tlstream_acquire) + +#define KBASE_IOCTL_TLSTREAM_FLUSH \ + _IO(KBASE_IOCTL_TYPE, 19) + +/** + * struct kbase_ioctl_mem_commit - Change the amount of memory backing a region + * + * @gpu_addr: The memory region to modify + * @pages: The number of physical pages that should be present + * + * The ioctl may return on the following error codes or 0 for success: + * -ENOMEM: Out of memory + * -EINVAL: Invalid arguments + */ +struct kbase_ioctl_mem_commit { + __u64 gpu_addr; + __u64 pages; +}; + +#define KBASE_IOCTL_MEM_COMMIT \ + _IOW(KBASE_IOCTL_TYPE, 20, struct kbase_ioctl_mem_commit) + +/** + * union kbase_ioctl_mem_alias - Create an alias of memory regions + * @flags: Flags, see BASE_MEM_xxx + * @stride: Bytes between start of each memory region + * @nents: The number of regions to pack together into the alias + * @aliasing_info: Pointer to an array of struct base_mem_aliasing_info + * @gpu_va: Address of the new alias + * @va_pages: Size of the new alias + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_alias { + struct { + __u64 flags; + __u64 stride; + __u64 nents; + __u64 aliasing_info; + } in; + struct { + __u64 flags; + __u64 gpu_va; + __u64 va_pages; + } out; +}; + +#define KBASE_IOCTL_MEM_ALIAS \ + _IOWR(KBASE_IOCTL_TYPE, 21, union kbase_ioctl_mem_alias) + +/** + * union kbase_ioctl_mem_import - Import memory for use by the GPU + * @flags: Flags, see BASE_MEM_xxx + * @phandle: Handle to the external memory + * @type: Type of external memory, see base_mem_import_type + * @padding: Amount of extra VA pages to append to the imported buffer + * @gpu_va: Address of the new alias + * @va_pages: Size of the new alias + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_import { + struct { + __u64 flags; + __u64 phandle; + __u32 type; + __u32 padding; + } in; + struct { + __u64 flags; + __u64 gpu_va; + __u64 va_pages; + } out; +}; + +#define KBASE_IOCTL_MEM_IMPORT \ + _IOWR(KBASE_IOCTL_TYPE, 22, union kbase_ioctl_mem_import) + +/** + * struct kbase_ioctl_mem_flags_change - Change the flags for a memory region + * @gpu_va: The GPU region to modify + * @flags: The new flags to set + * @mask: Mask of the flags to modify + */ +struct kbase_ioctl_mem_flags_change { + __u64 gpu_va; + __u64 flags; + __u64 mask; +}; + +#define KBASE_IOCTL_MEM_FLAGS_CHANGE \ + _IOW(KBASE_IOCTL_TYPE, 23, struct kbase_ioctl_mem_flags_change) + +/** + * struct kbase_ioctl_stream_create - Create a synchronisation stream + * @name: A name to identify this stream. Must be NULL-terminated. + * + * Note that this is also called a "timeline", but is named stream to avoid + * confusion with other uses of the word. + * + * Unused bytes in @name (after the first NULL byte) must be also be NULL bytes. + * + * The ioctl returns a file descriptor. + */ +struct kbase_ioctl_stream_create { + char name[32]; +}; + +#define KBASE_IOCTL_STREAM_CREATE \ + _IOW(KBASE_IOCTL_TYPE, 24, struct kbase_ioctl_stream_create) + +/** + * struct kbase_ioctl_fence_validate - Validate a fd refers to a fence + * @fd: The file descriptor to validate + */ +struct kbase_ioctl_fence_validate { + int fd; +}; + +#define KBASE_IOCTL_FENCE_VALIDATE \ + _IOW(KBASE_IOCTL_TYPE, 25, struct kbase_ioctl_fence_validate) + +/** + * struct kbase_ioctl_mem_profile_add - Provide profiling information to kernel + * @buffer: Pointer to the information + * @len: Length + * @padding: Padding + * + * The data provided is accessible through a debugfs file + */ +struct kbase_ioctl_mem_profile_add { + __u64 buffer; + __u32 len; + __u32 padding; +}; + +#define KBASE_IOCTL_MEM_PROFILE_ADD \ + _IOW(KBASE_IOCTL_TYPE, 27, struct kbase_ioctl_mem_profile_add) + +/** + * struct kbase_ioctl_sticky_resource_map - Permanently map an external resource + * @count: Number of resources + * @address: Array of u64 GPU addresses of the external resources to map + */ +struct kbase_ioctl_sticky_resource_map { + __u64 count; + __u64 address; +}; + +#define KBASE_IOCTL_STICKY_RESOURCE_MAP \ + _IOW(KBASE_IOCTL_TYPE, 29, struct kbase_ioctl_sticky_resource_map) + +/** + * struct kbase_ioctl_sticky_resource_map - Unmap a resource mapped which was + * previously permanently mapped + * @count: Number of resources + * @address: Array of u64 GPU addresses of the external resources to unmap + */ +struct kbase_ioctl_sticky_resource_unmap { + __u64 count; + __u64 address; +}; + +#define KBASE_IOCTL_STICKY_RESOURCE_UNMAP \ + _IOW(KBASE_IOCTL_TYPE, 30, struct kbase_ioctl_sticky_resource_unmap) + +/** + * union kbase_ioctl_mem_find_gpu_start_and_offset - Find the start address of + * the GPU memory region for + * the given gpu address and + * the offset of that address + * into the region + * + * @gpu_addr: GPU virtual address + * @size: Size in bytes within the region + * @start: Address of the beginning of the memory region enclosing @gpu_addr + * for the length of @offset bytes + * @offset: The offset from the start of the memory region to @gpu_addr + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_find_gpu_start_and_offset { + struct { + __u64 gpu_addr; + __u64 size; + } in; + struct { + __u64 start; + __u64 offset; + } out; +}; + +#define KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET \ + _IOWR(KBASE_IOCTL_TYPE, 31, union kbase_ioctl_mem_find_gpu_start_and_offset) + + +#define KBASE_IOCTL_CINSTR_GWT_START \ + _IO(KBASE_IOCTL_TYPE, 33) + +#define KBASE_IOCTL_CINSTR_GWT_STOP \ + _IO(KBASE_IOCTL_TYPE, 34) + +/** + * union kbase_ioctl_gwt_dump - Used to collect all GPU write fault addresses. + * @addr_buffer: Address of buffer to hold addresses of gpu modified areas. + * @size_buffer: Address of buffer to hold size of modified areas (in pages) + * @len: Number of addresses the buffers can hold. + * @more_data_available: Status indicating if more addresses are available. + * @no_of_addr_collected: Number of addresses collected into addr_buffer. + * + * @in: Input parameters + * @out: Output parameters + * + * This structure is used when performing a call to dump GPU write fault + * addresses. + */ +union kbase_ioctl_cinstr_gwt_dump { + struct { + __u64 addr_buffer; + __u64 size_buffer; + __u32 len; + __u32 padding; + + } in; + struct { + __u32 no_of_addr_collected; + __u8 more_data_available; + __u8 padding[27]; + } out; +}; + +#define KBASE_IOCTL_CINSTR_GWT_DUMP \ + _IOWR(KBASE_IOCTL_TYPE, 35, union kbase_ioctl_cinstr_gwt_dump) + +/** + * struct kbase_ioctl_mem_exec_init - Initialise the EXEC_VA memory zone + * + * @va_pages: Number of VA pages to reserve for EXEC_VA + */ +struct kbase_ioctl_mem_exec_init { + __u64 va_pages; +}; + +#define KBASE_IOCTL_MEM_EXEC_INIT \ + _IOW(KBASE_IOCTL_TYPE, 38, struct kbase_ioctl_mem_exec_init) + +/** + * union kbase_ioctl_get_cpu_gpu_timeinfo - Request zero or more types of + * cpu/gpu time (counter values) + * + * @request_flags: Bit-flags indicating the requested types. + * @paddings: Unused, size alignment matching the out. + * @sec: Integer field of the monotonic time, unit in seconds. + * @nsec: Fractional sec of the monotonic time, in nano-seconds. + * @padding: Unused, for u64 alignment + * @timestamp: System wide timestamp (counter) value. + * @cycle_counter: GPU cycle counter value. + * + * @in: Input parameters + * @out: Output parameters + * + */ +union kbase_ioctl_get_cpu_gpu_timeinfo { + struct { + __u32 request_flags; + __u32 paddings[7]; + } in; + struct { + __u64 sec; + __u32 nsec; + __u32 padding; + __u64 timestamp; + __u64 cycle_counter; + } out; +}; + +#define KBASE_IOCTL_GET_CPU_GPU_TIMEINFO \ + _IOWR(KBASE_IOCTL_TYPE, 50, union kbase_ioctl_get_cpu_gpu_timeinfo) + +/*************** + * test ioctls * + ***************/ +#if MALI_UNIT_TEST +/* These ioctls are purely for test purposes and are not used in the production + * driver, they therefore may change without notice + */ + +#define KBASE_IOCTL_TEST_TYPE (KBASE_IOCTL_TYPE + 1) + +/** + * struct kbase_ioctl_tlstream_test - Start a timeline stream test + * + * @tpw_count: number of trace point writers in each context + * @msg_delay: time delay between tracepoints from one writer in milliseconds + * @msg_count: number of trace points written by one writer + * @aux_msg: if non-zero aux messages will be included + */ +struct kbase_ioctl_tlstream_test { + __u32 tpw_count; + __u32 msg_delay; + __u32 msg_count; + __u32 aux_msg; +}; + +#define KBASE_IOCTL_TLSTREAM_TEST \ + _IOW(KBASE_IOCTL_TEST_TYPE, 1, struct kbase_ioctl_tlstream_test) + +/** + * struct kbase_ioctl_tlstream_stats - Read tlstream stats for test purposes + * @bytes_collected: number of bytes read by user + * @bytes_generated: number of bytes generated by tracepoints + */ +struct kbase_ioctl_tlstream_stats { + __u32 bytes_collected; + __u32 bytes_generated; +}; + +#define KBASE_IOCTL_TLSTREAM_STATS \ + _IOR(KBASE_IOCTL_TEST_TYPE, 2, struct kbase_ioctl_tlstream_stats) + +#endif /* MALI_UNIT_TEST */ + +/* Customer extension range */ +#define KBASE_IOCTL_EXTRA_TYPE (KBASE_IOCTL_TYPE + 2) + +/* If the integration needs extra ioctl add them there + * like this: + * + * struct my_ioctl_args { + * .... + * } + * + * #define KBASE_IOCTL_MY_IOCTL \ + * _IOWR(KBASE_IOCTL_EXTRA_TYPE, 0, struct my_ioctl_args) + */ + + +/********************************** + * Definitions for GPU properties * + **********************************/ +#define KBASE_GPUPROP_VALUE_SIZE_U8 (0x0) +#define KBASE_GPUPROP_VALUE_SIZE_U16 (0x1) +#define KBASE_GPUPROP_VALUE_SIZE_U32 (0x2) +#define KBASE_GPUPROP_VALUE_SIZE_U64 (0x3) + +#define KBASE_GPUPROP_PRODUCT_ID 1 +#define KBASE_GPUPROP_VERSION_STATUS 2 +#define KBASE_GPUPROP_MINOR_REVISION 3 +#define KBASE_GPUPROP_MAJOR_REVISION 4 +/* 5 previously used for GPU speed */ +#define KBASE_GPUPROP_GPU_FREQ_KHZ_MAX 6 +/* 7 previously used for minimum GPU speed */ +#define KBASE_GPUPROP_LOG2_PROGRAM_COUNTER_SIZE 8 +#define KBASE_GPUPROP_TEXTURE_FEATURES_0 9 +#define KBASE_GPUPROP_TEXTURE_FEATURES_1 10 +#define KBASE_GPUPROP_TEXTURE_FEATURES_2 11 +#define KBASE_GPUPROP_GPU_AVAILABLE_MEMORY_SIZE 12 + +#define KBASE_GPUPROP_L2_LOG2_LINE_SIZE 13 +#define KBASE_GPUPROP_L2_LOG2_CACHE_SIZE 14 +#define KBASE_GPUPROP_L2_NUM_L2_SLICES 15 + +#define KBASE_GPUPROP_TILER_BIN_SIZE_BYTES 16 +#define KBASE_GPUPROP_TILER_MAX_ACTIVE_LEVELS 17 + +#define KBASE_GPUPROP_MAX_THREADS 18 +#define KBASE_GPUPROP_MAX_WORKGROUP_SIZE 19 +#define KBASE_GPUPROP_MAX_BARRIER_SIZE 20 +#define KBASE_GPUPROP_MAX_REGISTERS 21 +#define KBASE_GPUPROP_MAX_TASK_QUEUE 22 +#define KBASE_GPUPROP_MAX_THREAD_GROUP_SPLIT 23 +#define KBASE_GPUPROP_IMPL_TECH 24 + +#define KBASE_GPUPROP_RAW_SHADER_PRESENT 25 +#define KBASE_GPUPROP_RAW_TILER_PRESENT 26 +#define KBASE_GPUPROP_RAW_L2_PRESENT 27 +#define KBASE_GPUPROP_RAW_STACK_PRESENT 28 +#define KBASE_GPUPROP_RAW_L2_FEATURES 29 +#define KBASE_GPUPROP_RAW_CORE_FEATURES 30 +#define KBASE_GPUPROP_RAW_MEM_FEATURES 31 +#define KBASE_GPUPROP_RAW_MMU_FEATURES 32 +#define KBASE_GPUPROP_RAW_AS_PRESENT 33 +#define KBASE_GPUPROP_RAW_JS_PRESENT 34 +#define KBASE_GPUPROP_RAW_JS_FEATURES_0 35 +#define KBASE_GPUPROP_RAW_JS_FEATURES_1 36 +#define KBASE_GPUPROP_RAW_JS_FEATURES_2 37 +#define KBASE_GPUPROP_RAW_JS_FEATURES_3 38 +#define KBASE_GPUPROP_RAW_JS_FEATURES_4 39 +#define KBASE_GPUPROP_RAW_JS_FEATURES_5 40 +#define KBASE_GPUPROP_RAW_JS_FEATURES_6 41 +#define KBASE_GPUPROP_RAW_JS_FEATURES_7 42 +#define KBASE_GPUPROP_RAW_JS_FEATURES_8 43 +#define KBASE_GPUPROP_RAW_JS_FEATURES_9 44 +#define KBASE_GPUPROP_RAW_JS_FEATURES_10 45 +#define KBASE_GPUPROP_RAW_JS_FEATURES_11 46 +#define KBASE_GPUPROP_RAW_JS_FEATURES_12 47 +#define KBASE_GPUPROP_RAW_JS_FEATURES_13 48 +#define KBASE_GPUPROP_RAW_JS_FEATURES_14 49 +#define KBASE_GPUPROP_RAW_JS_FEATURES_15 50 +#define KBASE_GPUPROP_RAW_TILER_FEATURES 51 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_0 52 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_1 53 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_2 54 +#define KBASE_GPUPROP_RAW_GPU_ID 55 +#define KBASE_GPUPROP_RAW_THREAD_MAX_THREADS 56 +#define KBASE_GPUPROP_RAW_THREAD_MAX_WORKGROUP_SIZE 57 +#define KBASE_GPUPROP_RAW_THREAD_MAX_BARRIER_SIZE 58 +#define KBASE_GPUPROP_RAW_THREAD_FEATURES 59 +#define KBASE_GPUPROP_RAW_COHERENCY_MODE 60 + +#define KBASE_GPUPROP_COHERENCY_NUM_GROUPS 61 +#define KBASE_GPUPROP_COHERENCY_NUM_CORE_GROUPS 62 +#define KBASE_GPUPROP_COHERENCY_COHERENCY 63 +#define KBASE_GPUPROP_COHERENCY_GROUP_0 64 +#define KBASE_GPUPROP_COHERENCY_GROUP_1 65 +#define KBASE_GPUPROP_COHERENCY_GROUP_2 66 +#define KBASE_GPUPROP_COHERENCY_GROUP_3 67 +#define KBASE_GPUPROP_COHERENCY_GROUP_4 68 +#define KBASE_GPUPROP_COHERENCY_GROUP_5 69 +#define KBASE_GPUPROP_COHERENCY_GROUP_6 70 +#define KBASE_GPUPROP_COHERENCY_GROUP_7 71 +#define KBASE_GPUPROP_COHERENCY_GROUP_8 72 +#define KBASE_GPUPROP_COHERENCY_GROUP_9 73 +#define KBASE_GPUPROP_COHERENCY_GROUP_10 74 +#define KBASE_GPUPROP_COHERENCY_GROUP_11 75 +#define KBASE_GPUPROP_COHERENCY_GROUP_12 76 +#define KBASE_GPUPROP_COHERENCY_GROUP_13 77 +#define KBASE_GPUPROP_COHERENCY_GROUP_14 78 +#define KBASE_GPUPROP_COHERENCY_GROUP_15 79 + +#define KBASE_GPUPROP_TEXTURE_FEATURES_3 80 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_3 81 + +#define KBASE_GPUPROP_NUM_EXEC_ENGINES 82 + +#define KBASE_GPUPROP_RAW_THREAD_TLS_ALLOC 83 +#define KBASE_GPUPROP_TLS_ALLOC 84 + +#ifdef __cpluscplus +} +#endif + +#endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd.c b/drivers/gpu/arm/bifrost/mali_kbase_jd.c new file mode 100755 index 000000000000..d0674d1bd8f4 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_jd.c @@ -0,0 +1,1819 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#include +#ifdef CONFIG_COMPAT +#include +#endif +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "mali_kbase_dma_fence.h" +#include + +#include + +#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) +/* random32 was renamed to prandom_u32 in 3.8 */ +#define prandom_u32 random32 +#endif + +/* Return whether katom will run on the GPU or not. Currently only soft jobs and + * dependency-only atoms do not run on the GPU */ +#define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ + ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \ + BASE_JD_REQ_DEP))) + +/* + * This is the kernel side of the API. Only entry points are: + * - kbase_jd_submit(): Called from userspace to submit a single bag + * - kbase_jd_done(): Called from interrupt context to track the + * completion of a job. + * Callouts: + * - to the job manager (enqueue a job) + * - to the event subsystem (signals the completion/failure of bag/job-chains). + */ + +static void __user * +get_compat_pointer(struct kbase_context *kctx, const u64 p) +{ +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + return compat_ptr(p); +#endif + return u64_to_user_ptr(p); +} + +/* Mark an atom as complete, and trace it in kinstr_jm */ +static void jd_mark_atom_complete(struct kbase_jd_atom *katom) +{ + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + kbase_kinstr_jm_atom_complete(katom); + dev_dbg(katom->kctx->kbdev->dev, "Atom %p status to completed\n", + (void *)katom); +} + +/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs + * + * Returns whether the JS needs a reschedule. + * + * Note that the caller must also check the atom status and + * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock + */ +static bool jd_run_atom(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + dev_dbg(kctx->kbdev->dev, "JD run atom %p in kctx %p\n", + (void *)katom, (void *)kctx); + + KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); + + if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { + /* Dependency only atom */ + trace_sysgraph(SGR_SUBMIT, kctx->id, + kbase_jd_atom_id(katom->kctx, katom)); + jd_mark_atom_complete(katom); + return 0; + } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { + /* Soft-job */ + if (katom->will_fail_event_code) { + kbase_finish_soft_job(katom); + jd_mark_atom_complete(katom); + return 0; + } + if (kbase_process_soft_job(katom) == 0) { + kbase_finish_soft_job(katom); + jd_mark_atom_complete(katom); + } + return 0; + } + + katom->status = KBASE_JD_ATOM_STATE_IN_JS; + dev_dbg(kctx->kbdev->dev, "Atom %p status to in JS\n", (void *)katom); + /* Queue an action about whether we should try scheduling a context */ + return kbasep_js_add_job(kctx, katom); +} + +void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom) +{ + struct kbase_device *kbdev; + + KBASE_DEBUG_ASSERT(katom); + kbdev = katom->kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev); + + /* Check whether the atom's other dependencies were already met. If + * katom is a GPU atom then the job scheduler may be able to represent + * the dependencies, hence we may attempt to submit it before they are + * met. Other atoms must have had both dependencies resolved. + */ + if (IS_GPU_ATOM(katom) || + (!kbase_jd_katom_dep_atom(&katom->dep[0]) && + !kbase_jd_katom_dep_atom(&katom->dep[1]))) { + /* katom dep complete, attempt to run it */ + bool resched = false; + + resched = jd_run_atom(katom); + + if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) { + /* The atom has already finished */ + resched |= jd_done_nolock(katom, NULL); + } + + if (resched) + kbase_js_sched_all(kbdev); + } +} + +void kbase_jd_free_external_resources(struct kbase_jd_atom *katom) +{ +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + /* Flush dma-fence workqueue to ensure that any callbacks that may have + * been queued are done before continuing. + * Any successfully completed atom would have had all it's callbacks + * completed before the atom was run, so only flush for failed atoms. + */ + if (katom->event_code != BASE_JD_EVENT_DONE) + flush_workqueue(katom->kctx->dma_fence.wq); +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ +} + +static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) +{ + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + kbase_dma_fence_signal(katom); +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + kbase_gpu_vm_lock(katom->kctx); + /* only roll back if extres is non-NULL */ + if (katom->extres) { + u32 res_no; + + res_no = katom->nr_extres; + while (res_no-- > 0) { + struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; + struct kbase_va_region *reg; + + reg = kbase_region_tracker_find_region_base_address( + katom->kctx, + katom->extres[res_no].gpu_address); + kbase_unmap_external_resource(katom->kctx, reg, alloc); + } + kfree(katom->extres); + katom->extres = NULL; + } + kbase_gpu_vm_unlock(katom->kctx); +} + +/* + * Set up external resources needed by this job. + * + * jctx.lock must be held when this is called. + */ + +static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom *user_atom) +{ + int err_ret_val = -EINVAL; + u32 res_no; +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + struct kbase_dma_fence_resv_info info = { + .resv_objs = NULL, + .dma_fence_resv_count = 0, + .dma_fence_excl_bitmap = NULL + }; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + /* + * When both dma-buf fence and Android native sync is enabled, we + * disable dma-buf fence for contexts that are using Android native + * fences. + */ + const bool implicit_sync = !kbase_ctx_flag(katom->kctx, + KCTX_NO_IMPLICIT_SYNC); +#else /* CONFIG_SYNC || CONFIG_SYNC_FILE*/ + const bool implicit_sync = true; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + struct base_external_resource *input_extres; + + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); + + /* no resources encoded, early out */ + if (!katom->nr_extres) + return -EINVAL; + + katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL); + if (!katom->extres) + return -ENOMEM; + + /* copy user buffer to the end of our real buffer. + * Make sure the struct sizes haven't changed in a way + * we don't support */ + BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres)); + input_extres = (struct base_external_resource *) + (((unsigned char *)katom->extres) + + (sizeof(*katom->extres) - sizeof(*input_extres)) * + katom->nr_extres); + + if (copy_from_user(input_extres, + get_compat_pointer(katom->kctx, user_atom->extres_list), + sizeof(*input_extres) * katom->nr_extres) != 0) { + err_ret_val = -EINVAL; + goto early_err_out; + } + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (implicit_sync) { + info.resv_objs = kmalloc_array(katom->nr_extres, +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) + sizeof(struct reservation_object *), +#else + sizeof(struct dma_resv *), +#endif + GFP_KERNEL); + if (!info.resv_objs) { + err_ret_val = -ENOMEM; + goto early_err_out; + } + + info.dma_fence_excl_bitmap = + kcalloc(BITS_TO_LONGS(katom->nr_extres), + sizeof(unsigned long), GFP_KERNEL); + if (!info.dma_fence_excl_bitmap) { + err_ret_val = -ENOMEM; + goto early_err_out; + } + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + /* Take the processes mmap lock */ + down_read(kbase_mem_get_process_mmap_lock()); + + /* need to keep the GPU VM locked while we set up UMM buffers */ + kbase_gpu_vm_lock(katom->kctx); + for (res_no = 0; res_no < katom->nr_extres; res_no++) { + struct base_external_resource *res = &input_extres[res_no]; + struct kbase_va_region *reg; + struct kbase_mem_phy_alloc *alloc; +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + bool exclusive; + exclusive = (res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE) + ? true : false; +#endif + reg = kbase_region_tracker_find_region_enclosing_address( + katom->kctx, + res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE); + /* did we find a matching region object? */ + if (kbase_is_region_invalid_or_free(reg)) { + /* roll back */ + goto failed_loop; + } + + if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) && + (reg->flags & KBASE_REG_PROTECTED)) { + katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED; + } + + alloc = kbase_map_external_resource(katom->kctx, reg, + current->mm); + if (!alloc) { + err_ret_val = -EINVAL; + goto failed_loop; + } + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (implicit_sync && + reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) + struct reservation_object *resv; +#else + struct dma_resv *resv; +#endif + resv = reg->gpu_alloc->imported.umm.dma_buf->resv; + if (resv) + kbase_dma_fence_add_reservation(resv, &info, + exclusive); + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + /* finish with updating out array with the data we found */ + /* NOTE: It is important that this is the last thing we do (or + * at least not before the first write) as we overwrite elements + * as we loop and could be overwriting ourself, so no writes + * until the last read for an element. + * */ + katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */ + katom->extres[res_no].alloc = alloc; + } + /* successfully parsed the extres array */ + /* drop the vm lock now */ + kbase_gpu_vm_unlock(katom->kctx); + + /* Release the processes mmap lock */ + up_read(kbase_mem_get_process_mmap_lock()); + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (implicit_sync) { + if (info.dma_fence_resv_count) { + int ret; + + ret = kbase_dma_fence_wait(katom, &info); + if (ret < 0) + goto failed_dma_fence_setup; + } + + kfree(info.resv_objs); + kfree(info.dma_fence_excl_bitmap); + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + /* all done OK */ + return 0; + +/* error handling section */ + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE +failed_dma_fence_setup: + /* Lock the processes mmap lock */ + down_read(kbase_mem_get_process_mmap_lock()); + + /* lock before we unmap */ + kbase_gpu_vm_lock(katom->kctx); +#endif + + failed_loop: + /* undo the loop work */ + while (res_no-- > 0) { + struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; + + kbase_unmap_external_resource(katom->kctx, NULL, alloc); + } + kbase_gpu_vm_unlock(katom->kctx); + + /* Release the processes mmap lock */ + up_read(kbase_mem_get_process_mmap_lock()); + + early_err_out: + kfree(katom->extres); + katom->extres = NULL; +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (implicit_sync) { + kfree(info.resv_objs); + kfree(info.dma_fence_excl_bitmap); + } +#endif + return err_ret_val; +} + +static inline void jd_resolve_dep(struct list_head *out_list, + struct kbase_jd_atom *katom, + u8 d, bool ctx_is_dying) +{ + u8 other_d = !d; + + while (!list_empty(&katom->dep_head[d])) { + struct kbase_jd_atom *dep_atom; + struct kbase_jd_atom *other_dep_atom; + u8 dep_type; + + dep_atom = list_entry(katom->dep_head[d].next, + struct kbase_jd_atom, dep_item[d]); + list_del(katom->dep_head[d].next); + + dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]); + kbase_jd_katom_dep_clear(&dep_atom->dep[d]); + + if (katom->event_code != BASE_JD_EVENT_DONE && + (dep_type != BASE_JD_DEP_TYPE_ORDER)) { +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + kbase_dma_fence_cancel_callbacks(dep_atom); +#endif + + dep_atom->event_code = katom->event_code; + KBASE_DEBUG_ASSERT(dep_atom->status != + KBASE_JD_ATOM_STATE_UNUSED); + + dep_atom->will_fail_event_code = dep_atom->event_code; + } + other_dep_atom = (struct kbase_jd_atom *) + kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]); + + if (!dep_atom->in_jd_list && (!other_dep_atom || + (IS_GPU_ATOM(dep_atom) && !ctx_is_dying && + !dep_atom->will_fail_event_code && + !other_dep_atom->will_fail_event_code))) { + bool dep_satisfied = true; +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + int dep_count; + + dep_count = kbase_fence_dep_count_read(dep_atom); + if (likely(dep_count == -1)) { + dep_satisfied = true; + } else { + /* + * There are either still active callbacks, or + * all fences for this @dep_atom has signaled, + * but the worker that will queue the atom has + * not yet run. + * + * Wait for the fences to signal and the fence + * worker to run and handle @dep_atom. If + * @dep_atom was completed due to error on + * @katom, then the fence worker will pick up + * the complete status and error code set on + * @dep_atom above. + */ + dep_satisfied = false; + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + if (dep_satisfied) { + trace_sysgraph(SGR_DEP_RES, + dep_atom->kctx->id, + kbase_jd_atom_id(katom->kctx, dep_atom)); + dep_atom->in_jd_list = true; + list_add_tail(&dep_atom->jd_item, out_list); + } + } + } +} + +/** + * is_dep_valid - Validate that a dependency is valid for early dependency + * submission + * @katom: Dependency atom to validate + * + * A dependency is valid if any of the following are true : + * - It does not exist (a non-existent dependency does not block submission) + * - It is in the job scheduler + * - It has completed, does not have a failure event code, and has not been + * marked to fail in the future + * + * Return: true if valid, false otherwise + */ +static bool is_dep_valid(struct kbase_jd_atom *katom) +{ + /* If there's no dependency then this is 'valid' from the perspective of + * early dependency submission */ + if (!katom) + return true; + + /* Dependency must have reached the job scheduler */ + if (katom->status < KBASE_JD_ATOM_STATE_IN_JS) + return false; + + /* If dependency has completed and has failed or will fail then it is + * not valid */ + if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && + (katom->event_code != BASE_JD_EVENT_DONE || + katom->will_fail_event_code)) + return false; + + return true; +} + +static void jd_try_submitting_deps(struct list_head *out_list, + struct kbase_jd_atom *node) +{ + int i; + + for (i = 0; i < 2; i++) { + struct list_head *pos; + + list_for_each(pos, &node->dep_head[i]) { + struct kbase_jd_atom *dep_atom = list_entry(pos, + struct kbase_jd_atom, dep_item[i]); + + if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) { + /*Check if atom deps look sane*/ + bool dep0_valid = is_dep_valid( + dep_atom->dep[0].atom); + bool dep1_valid = is_dep_valid( + dep_atom->dep[1].atom); + bool dep_satisfied = true; +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + int dep_count; + + dep_count = kbase_fence_dep_count_read( + dep_atom); + if (likely(dep_count == -1)) { + dep_satisfied = true; + } else { + /* + * There are either still active callbacks, or + * all fences for this @dep_atom has signaled, + * but the worker that will queue the atom has + * not yet run. + * + * Wait for the fences to signal and the fence + * worker to run and handle @dep_atom. If + * @dep_atom was completed due to error on + * @katom, then the fence worker will pick up + * the complete status and error code set on + * @dep_atom above. + */ + dep_satisfied = false; + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + if (dep0_valid && dep1_valid && dep_satisfied) { + trace_sysgraph(SGR_DEP_RES, + dep_atom->kctx->id, + kbase_jd_atom_id(dep_atom->kctx, + dep_atom)); + dep_atom->in_jd_list = true; + list_add(&dep_atom->jd_item, out_list); + } + } + } + } +} + +#if MALI_JIT_PRESSURE_LIMIT_BASE +/** + * jd_update_jit_usage - Update just-in-time physical memory usage for an atom. + * + * @katom: An atom that has just finished. + * + * Read back actual just-in-time memory region usage from atoms that provide + * this information, and update the current physical page pressure. + * + * The caller must hold the kbase_jd_context.lock. + */ +static void jd_update_jit_usage(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct kbase_va_region *reg; + struct kbase_vmap_struct mapping; + u64 *ptr; + u64 used_pages; + unsigned int idx; + + lockdep_assert_held(&kctx->jctx.lock); + + /* If this atom wrote to JIT memory, find out how much it has written + * and update the usage information in the region. + */ + for (idx = 0; + idx < ARRAY_SIZE(katom->jit_ids) && katom->jit_ids[idx]; + idx++) { + enum heap_pointer { LOW = 0, HIGH, COUNT }; + size_t size_to_read; + u64 read_val; + + reg = kctx->jit_alloc[katom->jit_ids[idx]]; + + if (!reg) { + dev_warn(kctx->kbdev->dev, + "%s: JIT id[%u]=%u has no region\n", + __func__, idx, katom->jit_ids[idx]); + continue; + } + + if (reg == KBASE_RESERVED_REG_JIT_ALLOC) { + dev_warn(kctx->kbdev->dev, + "%s: JIT id[%u]=%u has failed to allocate a region\n", + __func__, idx, katom->jit_ids[idx]); + continue; + } + + if (!reg->heap_info_gpu_addr) + continue; + + size_to_read = sizeof(*ptr); + if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) + size_to_read = sizeof(u32); + else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) + size_to_read = sizeof(u64[COUNT]); + + ptr = kbase_vmap(kctx, reg->heap_info_gpu_addr, size_to_read, + &mapping); + + if (!ptr) { + dev_warn(kctx->kbdev->dev, + "%s: JIT id[%u]=%u start=0x%llx unable to map end marker %llx\n", + __func__, idx, katom->jit_ids[idx], + reg->start_pfn << PAGE_SHIFT, + reg->heap_info_gpu_addr); + continue; + } + + if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) { + read_val = READ_ONCE(*(u32 *)ptr); + used_pages = PFN_UP(read_val); + } else { + u64 addr_end; + + if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { + const unsigned long extent_bytes = reg->extent + << PAGE_SHIFT; + const u64 low_ptr = ptr[LOW]; + const u64 high_ptr = ptr[HIGH]; + + /* As either the low or high pointer could + * consume their partition and move onto the + * next chunk, we need to account for both. + * In the case where nothing has been allocated + * from the high pointer the whole chunk could + * be backed unnecessarily - but the granularity + * is the chunk size anyway and any non-zero + * offset of low pointer from the start of the + * chunk would result in the whole chunk being + * backed. + */ + read_val = max(high_ptr, low_ptr); + + /* kbase_check_alloc_sizes() already satisfies + * this, but here to avoid future maintenance + * hazards + */ + WARN_ON(!is_power_of_2(extent_bytes)); + addr_end = ALIGN(read_val, extent_bytes); + } else { + addr_end = read_val = READ_ONCE(*ptr); + } + + if (addr_end >= (reg->start_pfn << PAGE_SHIFT)) + used_pages = PFN_UP(addr_end) - reg->start_pfn; + else + used_pages = reg->used_pages; + } + + trace_mali_jit_report(katom, reg, idx, read_val, used_pages); + kbase_trace_jit_report_gpu_mem(kctx, reg, 0u); + + /* We can never have used more pages than the VA size of the + * region + */ + if (used_pages > reg->nr_pages) { + dev_warn(kctx->kbdev->dev, + "%s: JIT id[%u]=%u start=0x%llx used_pages %llx > %zx (read 0x%llx as %s%s)\n", + __func__, idx, katom->jit_ids[idx], + reg->start_pfn << PAGE_SHIFT, + used_pages, reg->nr_pages, read_val, + (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) ? + "size" : "addr", + (reg->flags & KBASE_REG_TILER_ALIGN_TOP) ? + " with align" : ""); + used_pages = reg->nr_pages; + } + /* Note: one real use case has an atom correctly reporting 0 + * pages in use. This happens in normal use-cases but may only + * happen for a few of the application's frames. + */ + + kbase_vunmap(kctx, &mapping); + + kbase_jit_report_update_pressure(kctx, reg, used_pages, 0u); + } + + kbase_jit_retry_pending_alloc(kctx); +} +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +/* + * Perform the necessary handling of an atom that has finished running + * on the GPU. + * + * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller + * is responsible for calling kbase_finish_soft_job *before* calling this function. + * + * The caller must hold the kbase_jd_context.lock. + */ +bool jd_done_nolock(struct kbase_jd_atom *katom, + struct list_head *completed_jobs_ctx) +{ + struct kbase_context *kctx = katom->kctx; + struct list_head completed_jobs; + struct list_head runnable_jobs; + bool need_to_try_schedule_context = false; + int i; + + INIT_LIST_HEAD(&completed_jobs); + INIT_LIST_HEAD(&runnable_jobs); + + KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); + +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (kbase_ctx_flag(kctx, KCTX_JPL_ENABLED)) + jd_update_jit_usage(katom); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + /* This is needed in case an atom is failed due to being invalid, this + * can happen *before* the jobs that the atom depends on have completed */ + for (i = 0; i < 2; i++) { + if (kbase_jd_katom_dep_atom(&katom->dep[i])) { + list_del(&katom->dep_item[i]); + kbase_jd_katom_dep_clear(&katom->dep[i]); + } + } + + jd_mark_atom_complete(katom); + list_add_tail(&katom->jd_item, &completed_jobs); + + while (!list_empty(&completed_jobs)) { + katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item); + list_del(completed_jobs.prev); + KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); + + for (i = 0; i < 2; i++) + jd_resolve_dep(&runnable_jobs, katom, i, + kbase_ctx_flag(kctx, KCTX_DYING)); + + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) + kbase_jd_post_external_resources(katom); + + while (!list_empty(&runnable_jobs)) { + struct kbase_jd_atom *node; + + node = list_entry(runnable_jobs.next, + struct kbase_jd_atom, jd_item); + list_del(runnable_jobs.next); + node->in_jd_list = false; + + dev_dbg(kctx->kbdev->dev, "List node %p has status %d\n", + node, node->status); + + KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); + if (node->status == KBASE_JD_ATOM_STATE_IN_JS) + continue; + + if (node->status != KBASE_JD_ATOM_STATE_COMPLETED && + !kbase_ctx_flag(kctx, KCTX_DYING)) { + need_to_try_schedule_context |= jd_run_atom(node); + } else { + node->event_code = katom->event_code; + + if (node->core_req & + BASE_JD_REQ_SOFT_JOB) { + WARN_ON(!list_empty(&node->queue)); + kbase_finish_soft_job(node); + } + node->status = KBASE_JD_ATOM_STATE_COMPLETED; + } + + if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) { + list_add_tail(&node->jd_item, &completed_jobs); + } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS && + !node->will_fail_event_code) { + /* Node successfully submitted, try submitting + * dependencies as they may now be representable + * in JS */ + jd_try_submitting_deps(&runnable_jobs, node); + } + } + + /* Register a completed job as a disjoint event when the GPU + * is in a disjoint state (ie. being reset). + */ + kbase_disjoint_event_potential(kctx->kbdev); + if (completed_jobs_ctx) + list_add_tail(&katom->jd_item, completed_jobs_ctx); + else + kbase_event_post(kctx, katom); + + /* Decrement and check the TOTAL number of jobs. This includes + * those not tracked by the scheduler: 'not ready to run' and + * 'dependency-only' jobs. */ + if (--kctx->jctx.job_nr == 0) + wake_up(&kctx->jctx.zero_jobs_wait); /* All events are safely queued now, and we can signal any waiter + * that we've got no more jobs (so we can be safely terminated) */ + } + + return need_to_try_schedule_context; +} + +KBASE_EXPORT_TEST_API(jd_done_nolock); + +#ifdef CONFIG_GPU_TRACEPOINTS +enum { + CORE_REQ_DEP_ONLY, + CORE_REQ_SOFT, + CORE_REQ_COMPUTE, + CORE_REQ_FRAGMENT, + CORE_REQ_VERTEX, + CORE_REQ_TILER, + CORE_REQ_FRAGMENT_VERTEX, + CORE_REQ_FRAGMENT_VERTEX_TILER, + CORE_REQ_FRAGMENT_TILER, + CORE_REQ_VERTEX_TILER, + CORE_REQ_UNKNOWN +}; +static const char * const core_req_strings[] = { + "Dependency Only Job", + "Soft Job", + "Compute Shader Job", + "Fragment Shader Job", + "Vertex/Geometry Shader Job", + "Tiler Job", + "Fragment Shader + Vertex/Geometry Shader Job", + "Fragment Shader + Vertex/Geometry Shader Job + Tiler Job", + "Fragment Shader + Tiler Job", + "Vertex/Geometry Shader Job + Tiler Job", + "Unknown Job" +}; +static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req) +{ + if (core_req & BASE_JD_REQ_SOFT_JOB) + return core_req_strings[CORE_REQ_SOFT]; + if (core_req & BASE_JD_REQ_ONLY_COMPUTE) + return core_req_strings[CORE_REQ_COMPUTE]; + switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) { + case BASE_JD_REQ_DEP: + return core_req_strings[CORE_REQ_DEP_ONLY]; + case BASE_JD_REQ_FS: + return core_req_strings[CORE_REQ_FRAGMENT]; + case BASE_JD_REQ_CS: + return core_req_strings[CORE_REQ_VERTEX]; + case BASE_JD_REQ_T: + return core_req_strings[CORE_REQ_TILER]; + case (BASE_JD_REQ_FS | BASE_JD_REQ_CS): + return core_req_strings[CORE_REQ_FRAGMENT_VERTEX]; + case (BASE_JD_REQ_FS | BASE_JD_REQ_T): + return core_req_strings[CORE_REQ_FRAGMENT_TILER]; + case (BASE_JD_REQ_CS | BASE_JD_REQ_T): + return core_req_strings[CORE_REQ_VERTEX_TILER]; + case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T): + return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER]; + } + return core_req_strings[CORE_REQ_UNKNOWN]; +} +#endif + +/* Trace an atom submission. */ +static void jd_trace_atom_submit(struct kbase_context *const kctx, + struct kbase_jd_atom *const katom, + int *priority) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + KBASE_TLSTREAM_TL_NEW_ATOM(kbdev, katom, kbase_jd_atom_id(kctx, katom)); + KBASE_TLSTREAM_TL_RET_ATOM_CTX(kbdev, katom, kctx); + if (priority) + KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(kbdev, katom, *priority); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_IDLE); + kbase_kinstr_jm_atom_queue(katom); +} + +static bool jd_submit_atom(struct kbase_context *const kctx, + const struct base_jd_atom *const user_atom, + const struct base_jd_fragment *const user_jc_incr, + struct kbase_jd_atom *const katom) +{ + struct kbase_device *kbdev = kctx->kbdev; + struct kbase_jd_context *jctx = &kctx->jctx; + int queued = 0; + int i; + int sched_prio; + bool will_fail = false; + unsigned long flags; + enum kbase_jd_atom_state status; + + dev_dbg(kbdev->dev, "User did JD submit atom %p\n", (void *)katom); + + /* Update the TOTAL number of jobs. This includes those not tracked by + * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ + jctx->job_nr++; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + katom->start_timestamp.tv64 = 0; +#else + katom->start_timestamp = 0; +#endif + katom->udata = user_atom->udata; + katom->kctx = kctx; + katom->nr_extres = user_atom->nr_extres; + katom->extres = NULL; + katom->device_nr = user_atom->device_nr; + katom->jc = user_atom->jc; + katom->core_req = user_atom->core_req; + katom->jobslot = user_atom->jobslot; + katom->seq_nr = user_atom->seq_nr; + katom->atom_flags = 0; + katom->retry_count = 0; + katom->need_cache_flush_cores_retained = 0; + katom->pre_dep = NULL; + katom->post_dep = NULL; + katom->x_pre_dep = NULL; + katom->x_post_dep = NULL; + katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; + katom->softjob_data = NULL; + + trace_sysgraph(SGR_ARRIVE, kctx->id, user_atom->atom_number); + +#if MALI_JIT_PRESSURE_LIMIT_BASE + /* Older API version atoms might have random values where jit_id now + * lives, but we must maintain backwards compatibility - handle the + * issue. + */ + if (!mali_kbase_supports_jit_pressure_limit(kctx->api_version)) { + katom->jit_ids[0] = 0; + katom->jit_ids[1] = 0; + } else { + katom->jit_ids[0] = user_atom->jit_id[0]; + katom->jit_ids[1] = user_atom->jit_id[1]; + } +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + katom->renderpass_id = user_atom->renderpass_id; + + /* Implicitly sets katom->protected_state.enter as well. */ + katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; + + katom->age = kctx->age_count++; + + INIT_LIST_HEAD(&katom->queue); + INIT_LIST_HEAD(&katom->jd_item); +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + kbase_fence_dep_count_set(katom, -1); +#endif + + /* Don't do anything if there is a mess up with dependencies. + This is done in a separate cycle to check both the dependencies at ones, otherwise + it will be extra complexity to deal with 1st dependency ( just added to the list ) + if only the 2nd one has invalid config. + */ + for (i = 0; i < 2; i++) { + int dep_atom_number = user_atom->pre_dep[i].atom_id; + base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type; + + if (dep_atom_number) { + if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER && + dep_atom_type != BASE_JD_DEP_TYPE_DATA) { + katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + dev_dbg(kbdev->dev, + "Atom %p status to completed\n", + (void *)katom); + + /* Wrong dependency setup. Atom will be sent + * back to user space. Do not record any + * dependencies. */ + jd_trace_atom_submit(kctx, katom, NULL); + + return jd_done_nolock(katom, NULL); + } + } + } + + /* Add dependencies */ + for (i = 0; i < 2; i++) { + int dep_atom_number = user_atom->pre_dep[i].atom_id; + base_jd_dep_type dep_atom_type; + struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; + + dep_atom_type = user_atom->pre_dep[i].dependency_type; + kbase_jd_katom_dep_clear(&katom->dep[i]); + + if (!dep_atom_number) + continue; + + if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || + dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) { + + if (dep_atom->event_code == BASE_JD_EVENT_DONE) + continue; + /* don't stop this atom if it has an order dependency + * only to the failed one, try to submit it through + * the normal path + */ + if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER && + dep_atom->event_code > BASE_JD_EVENT_ACTIVE) { + continue; + } + + /* Atom has completed, propagate the error code if any */ + katom->event_code = dep_atom->event_code; + katom->status = KBASE_JD_ATOM_STATE_QUEUED; + dev_dbg(kbdev->dev, "Atom %p status to queued\n", + (void *)katom); + + /* This atom will be sent back to user space. + * Do not record any dependencies. + */ + jd_trace_atom_submit(kctx, katom, NULL); + + will_fail = true; + + } else { + /* Atom is in progress, add this atom to the list */ + list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]); + kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type); + queued = 1; + } + } + + if (will_fail) { + if (!queued) { + if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { + /* This softjob has failed due to a previous + * dependency, however we should still run the + * prepare & finish functions + */ + int err = kbase_prepare_soft_job(katom); + + if (err >= 0) + kbase_finish_soft_job(katom); + } + + return jd_done_nolock(katom, NULL); + } + } + + /* These must occur after the above loop to ensure that an atom + * that depends on a previous atom with the same number behaves + * as expected + */ + katom->event_code = BASE_JD_EVENT_DONE; + katom->status = KBASE_JD_ATOM_STATE_QUEUED; + dev_dbg(kbdev->dev, "Atom %p status to queued\n", (void *)katom); + + /* For invalid priority, be most lenient and choose the default */ + sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); + if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) + sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; + katom->sched_priority = sched_prio; + + /* Create a new atom. */ + jd_trace_atom_submit(kctx, katom, &katom->sched_priority); + +#if !MALI_INCREMENTAL_RENDERING + /* Reject atoms for incremental rendering if not supported */ + if (katom->core_req & + (BASE_JD_REQ_START_RENDERPASS|BASE_JD_REQ_END_RENDERPASS)) { + dev_err(kctx->kbdev->dev, + "Rejecting atom with unsupported core_req 0x%x\n", + katom->core_req); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return jd_done_nolock(katom, NULL); + } +#endif /* !MALI_INCREMENTAL_RENDERING */ + + if (katom->core_req & BASE_JD_REQ_END_RENDERPASS) { + WARN_ON(katom->jc != 0); + katom->jc_fragment = *user_jc_incr; + } else if (!katom->jc && + (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { + /* Reject atoms with job chain = NULL, as these cause issues + * with soft-stop + */ + dev_err(kctx->kbdev->dev, "Rejecting atom with jc = NULL\n"); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return jd_done_nolock(katom, NULL); + } + + /* Reject atoms with an invalid device_nr */ + if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) && + (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) { + dev_err(kctx->kbdev->dev, + "Rejecting atom with invalid device_nr %d\n", + katom->device_nr); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return jd_done_nolock(katom, NULL); + } + + /* Reject atoms with invalid core requirements */ + if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && + (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) { + dev_err(kctx->kbdev->dev, + "Rejecting atom with invalid core requirements\n"); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE; + return jd_done_nolock(katom, NULL); + } + + /* Reject soft-job atom of certain types from accessing external resources */ + if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && + (((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_FENCE_WAIT) || + ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_JIT_ALLOC) || + ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_JIT_FREE))) { + dev_err(kctx->kbdev->dev, + "Rejecting soft-job atom accessing external resources\n"); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return jd_done_nolock(katom, NULL); + } + + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + /* handle what we need to do to access the external resources */ + if (kbase_jd_pre_external_resources(katom, user_atom) != 0) { + /* setup failed (no access, bad resource, unknown resource types, etc.) */ + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return jd_done_nolock(katom, NULL); + } + } + +#if !MALI_JIT_PRESSURE_LIMIT_BASE + if (mali_kbase_supports_jit_pressure_limit(kctx->api_version) && + (user_atom->jit_id[0] || user_atom->jit_id[1])) { + /* JIT pressure limit is disabled, but we are receiving non-0 + * JIT IDs - atom is invalid. + */ + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return jd_done_nolock(katom, NULL); + } +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + /* Validate the atom. Function will return error if the atom is + * malformed. + * + * Soft-jobs never enter the job scheduler but have their own initialize method. + * + * If either fail then we immediately complete the atom with an error. + */ + if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) { + if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return jd_done_nolock(katom, NULL); + } + } else { + /* Soft-job */ + if (kbase_prepare_soft_job(katom) != 0) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return jd_done_nolock(katom, NULL); + } + } + +#ifdef CONFIG_GPU_TRACEPOINTS + katom->work_id = atomic_inc_return(&jctx->work_id); + trace_gpu_job_enqueue(kctx->id, katom->work_id, + kbasep_map_core_reqs_to_string(katom->core_req)); +#endif + + if (queued && !IS_GPU_ATOM(katom)) + return false; + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (kbase_fence_dep_count_read(katom) != -1) + return false; + +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { + if (kbase_process_soft_job(katom) == 0) { + kbase_finish_soft_job(katom); + return jd_done_nolock(katom, NULL); + } + return false; + } + + if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { + bool need_to_try_schedule_context; + + katom->status = KBASE_JD_ATOM_STATE_IN_JS; + dev_dbg(kctx->kbdev->dev, "Atom %p status to in JS\n", + (void *)katom); + + need_to_try_schedule_context = kbasep_js_add_job(kctx, katom); + /* If job was cancelled then resolve immediately */ + if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) + return need_to_try_schedule_context; + + /* Synchronize with backend reset */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + status = katom->status; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + if (status == KBASE_JD_ATOM_STATE_HW_COMPLETED) { + dev_dbg(kctx->kbdev->dev, + "Atom %d cancelled on HW\n", + kbase_jd_atom_id(katom->kctx, katom)); + return need_to_try_schedule_context; + } + } + + /* This is a pure dependency. Resolve it immediately */ + return jd_done_nolock(katom, NULL); +} + +int kbase_jd_submit(struct kbase_context *kctx, + void __user *user_addr, u32 nr_atoms, u32 stride, + bool uk6_atom) +{ + struct kbase_jd_context *jctx = &kctx->jctx; + int err = 0; + int i; + bool need_to_try_schedule_context = false; + struct kbase_device *kbdev; + u32 latest_flush; + + bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2) || + stride == offsetof(struct base_jd_atom_v2, renderpass_id)); + + /* + * kbase_jd_submit isn't expected to fail and so all errors with the + * jobs are reported by immediately failing them (through event system) + */ + kbdev = kctx->kbdev; + + beenthere(kctx, "%s", "Enter"); + + if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it\n"); + return -EINVAL; + } + + if (stride != offsetof(struct base_jd_atom_v2, renderpass_id) && + stride != sizeof(struct base_jd_atom_v2) && + stride != offsetof(struct base_jd_atom, renderpass_id) && + stride != sizeof(struct base_jd_atom)) { + dev_err(kbdev->dev, + "Stride %u passed to job_submit isn't supported by the kernel\n", + stride); + return -EINVAL; + } + + /* All atoms submitted in this call have the same flush ID */ + latest_flush = kbase_backend_get_current_flush_id(kbdev); + + for (i = 0; i < nr_atoms; i++) { + struct base_jd_atom user_atom; + struct base_jd_fragment user_jc_incr; + struct kbase_jd_atom *katom; + + if (unlikely(jd_atom_is_v2)) { + if (copy_from_user(&user_atom.jc, user_addr, sizeof(struct base_jd_atom_v2)) != 0) { + dev_err(kbdev->dev, + "Invalid atom address %p passed to job_submit\n", + user_addr); + err = -EFAULT; + break; + } + + /* no seq_nr in v2 */ + user_atom.seq_nr = 0; + } else { + if (copy_from_user(&user_atom, user_addr, stride) != 0) { + dev_err(kbdev->dev, + "Invalid atom address %p passed to job_submit\n", + user_addr); + err = -EFAULT; + break; + } + } + + if (stride == offsetof(struct base_jd_atom_v2, renderpass_id)) { + dev_dbg(kbdev->dev, "No renderpass ID: use 0\n"); + user_atom.renderpass_id = 0; + } else { + /* Ensure all padding bytes are 0 for potential future + * extension + */ + size_t j; + + dev_dbg(kbdev->dev, "Renderpass ID is %d\n", + user_atom.renderpass_id); + for (j = 0; j < sizeof(user_atom.padding); j++) { + if (user_atom.padding[j]) { + dev_err(kbdev->dev, + "Bad padding byte %zu: %d\n", + j, user_atom.padding[j]); + err = -EINVAL; + break; + } + } + if (err) + break; + } + + /* In this case 'jc' is the CPU address of a struct + * instead of a GPU address of a job chain. + */ + if (user_atom.core_req & BASE_JD_REQ_END_RENDERPASS) { + if (copy_from_user(&user_jc_incr, + u64_to_user_ptr(user_atom.jc), + sizeof(user_jc_incr))) { + dev_err(kbdev->dev, + "Invalid jc address 0x%llx passed to job_submit\n", + user_atom.jc); + err = -EFAULT; + break; + } + dev_dbg(kbdev->dev, "Copied IR jobchain addresses\n"); + user_atom.jc = 0; + } + + user_addr = (void __user *)((uintptr_t) user_addr + stride); + + mutex_lock(&jctx->lock); +#ifndef compiletime_assert +#define compiletime_assert_defined +#define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \ +while (false) +#endif + compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) == + BASE_JD_ATOM_COUNT, + "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); + compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) == + sizeof(user_atom.atom_number), + "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); +#ifdef compiletime_assert_defined +#undef compiletime_assert +#undef compiletime_assert_defined +#endif + katom = &jctx->atoms[user_atom.atom_number]; + + /* Record the flush ID for the cache flush optimisation */ + katom->flush_id = latest_flush; + + while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) { + /* Atom number is already in use, wait for the atom to + * complete + */ + mutex_unlock(&jctx->lock); + + /* This thread will wait for the atom to complete. Due + * to thread scheduling we are not sure that the other + * thread that owns the atom will also schedule the + * context, so we force the scheduler to be active and + * hence eventually schedule this context at some point + * later. + */ + kbase_js_sched_all(kbdev); + + if (wait_event_killable(katom->completed, + katom->status == + KBASE_JD_ATOM_STATE_UNUSED) != 0) { + /* We're being killed so the result code + * doesn't really matter + */ + return 0; + } + mutex_lock(&jctx->lock); + } + + need_to_try_schedule_context |= jd_submit_atom(kctx, &user_atom, + &user_jc_incr, katom); + + /* Register a completed job as a disjoint event when the GPU is in a disjoint state + * (ie. being reset). + */ + kbase_disjoint_event_potential(kbdev); + + mutex_unlock(&jctx->lock); + } + + if (need_to_try_schedule_context) + kbase_js_sched_all(kbdev); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_jd_submit); + +void kbase_jd_done_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); + struct kbase_jd_context *jctx; + struct kbase_context *kctx; + struct kbasep_js_kctx_info *js_kctx_info; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + u64 cache_jc = katom->jc; + struct kbasep_js_atom_retained_state katom_retained_state; + bool context_idle; + base_jd_core_req core_req = katom->core_req; + + /* Soft jobs should never reach this function */ + KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); + + kctx = katom->kctx; + jctx = &kctx->jctx; + kbdev = kctx->kbdev; + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + dev_dbg(kbdev->dev, "Enter atom %p done worker for kctx %p\n", + (void *)katom, (void *)kctx); + + KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); + + kbase_backend_complete_wq(kbdev, katom); + + /* + * Begin transaction on JD context and JS context + */ + mutex_lock(&jctx->lock); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_DONE); + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + /* This worker only gets called on contexts that are scheduled *in*. This is + * because it only happens in response to an IRQ from a job that was + * running. + */ + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (katom->event_code == BASE_JD_EVENT_STOPPED) { + unsigned long flags; + + dev_dbg(kbdev->dev, "Atom %p has been promoted to stopped\n", + (void *)katom); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + katom->status = KBASE_JD_ATOM_STATE_IN_JS; + dev_dbg(kctx->kbdev->dev, "Atom %p status to in JS\n", + (void *)katom); + kbase_js_unpull(kctx, katom); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&jctx->lock); + + return; + } + + if ((katom->event_code != BASE_JD_EVENT_DONE) && + (!kbase_ctx_flag(katom->kctx, KCTX_DYING))) + dev_err(kbdev->dev, + "t6xx: GPU fault 0x%02lx from job slot %d\n", + (unsigned long)katom->event_code, + katom->slot_nr); + + /* Retain state before the katom disappears */ + kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); + + context_idle = kbase_js_complete_atom_wq(kctx, katom); + + KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state)); + + kbasep_js_remove_job(kbdev, kctx, katom); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; + /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */ + jd_done_nolock(katom, &kctx->completed_jobs); + + /* katom may have been freed now, do not use! */ + + if (context_idle) { + unsigned long flags; + + context_idle = false; + mutex_lock(&js_devdata->queue_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* If kbase_sched() has scheduled this context back in then + * KCTX_ACTIVE will have been set after we marked it as + * inactive, and another pm reference will have been taken, so + * drop our reference. But do not call kbase_jm_idle_ctx(), as + * the context is active and fast-starting is allowed. + * + * If an atom has been fast-started then kctx->atoms_pulled will + * be non-zero but KCTX_ACTIVE will still be false (as the + * previous pm reference has been inherited). Do NOT drop our + * reference, as it has been re-used, and leave the context as + * active. + * + * If no new atoms have been started then KCTX_ACTIVE will still + * be false and atoms_pulled will be zero, so drop the reference + * and call kbase_jm_idle_ctx(). + * + * As the checks are done under both the queue_mutex and + * hwaccess_lock is should be impossible for this to race + * with the scheduler code. + */ + if (kbase_ctx_flag(kctx, KCTX_ACTIVE) || + !atomic_read(&kctx->atoms_pulled)) { + /* Calling kbase_jm_idle_ctx() here will ensure that + * atoms are not fast-started when we drop the + * hwaccess_lock. This is not performed if + * KCTX_ACTIVE is set as in that case another pm + * reference has been taken and a fast-start would be + * valid. + */ + if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) + kbase_jm_idle_ctx(kbdev, kctx); + context_idle = true; + } else { + kbase_ctx_flag_set(kctx, KCTX_ACTIVE); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->queue_mutex); + } + + /* + * Transaction complete + */ + mutex_unlock(&jctx->lock); + + /* Job is now no longer running, so can now safely release the context + * reference, and handle any actions that were logged against the atom's retained state */ + + kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); + + kbase_js_sched_all(kbdev); + + if (!atomic_dec_return(&kctx->work_count)) { + /* If worker now idle then post all events that jd_done_nolock() + * has queued */ + mutex_lock(&jctx->lock); + while (!list_empty(&kctx->completed_jobs)) { + struct kbase_jd_atom *atom = list_entry( + kctx->completed_jobs.next, + struct kbase_jd_atom, jd_item); + list_del(kctx->completed_jobs.next); + + kbase_event_post(kctx, atom); + } + mutex_unlock(&jctx->lock); + } + + kbase_backend_complete_wq_post_sched(kbdev, core_req); + + if (context_idle) + kbase_pm_context_idle(kbdev); + + KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); + + dev_dbg(kbdev->dev, "Leave atom %p done worker for kctx %p\n", + (void *)katom, (void *)kctx); +} + +/** + * jd_cancel_worker - Work queue job cancel function. + * @data: a &struct work_struct + * + * Only called as part of 'Zapping' a context (which occurs on termination). + * Operates serially with the kbase_jd_done_worker() on the work queue. + * + * This can only be called on contexts that aren't scheduled. + * + * We don't need to release most of the resources that would occur on + * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be + * running (by virtue of only being called on contexts that aren't + * scheduled). + */ +static void jd_cancel_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); + struct kbase_jd_context *jctx; + struct kbase_context *kctx; + struct kbasep_js_kctx_info *js_kctx_info; + bool need_to_try_schedule_context; + bool attr_state_changed; + struct kbase_device *kbdev; + + /* Soft jobs should never reach this function */ + KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); + + kctx = katom->kctx; + kbdev = kctx->kbdev; + jctx = &kctx->jctx; + js_kctx_info = &kctx->jctx.sched_info; + + KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); + + /* This only gets called on contexts that are scheduled out. Hence, we must + * make sure we don't de-ref the number of running jobs (there aren't + * any), nor must we try to schedule out the context (it's already + * scheduled out). + */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + /* Scheduler: Remove the job from the system */ + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + mutex_lock(&jctx->lock); + + need_to_try_schedule_context = jd_done_nolock(katom, NULL); + /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to + * schedule the context. There's also no need for the jsctx_mutex to have been taken + * around this too. */ + KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); + + /* katom may have been freed now, do not use! */ + mutex_unlock(&jctx->lock); + + if (attr_state_changed) + kbase_js_sched_all(kbdev); +} + +/** + * kbase_jd_done - Complete a job that has been removed from the Hardware + * @katom: atom which has been completed + * @slot_nr: slot the atom was on + * @end_timestamp: completion time + * @done_code: completion code + * + * This must be used whenever a job has been removed from the Hardware, e.g.: + * An IRQ indicates that the job finished (for both error and 'done' codes), or + * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop. + * + * Some work is carried out immediately, and the rest is deferred onto a + * workqueue + * + * Context: + * This can be called safely from atomic context. + * The caller must hold kbdev->hwaccess_lock + */ +void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, + ktime_t *end_timestamp, kbasep_js_atom_done_code done_code) +{ + struct kbase_context *kctx; + struct kbase_device *kbdev; + + KBASE_DEBUG_ASSERT(katom); + kctx = katom->kctx; + KBASE_DEBUG_ASSERT(kctx); + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev); + + if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) + katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; + + KBASE_KTRACE_ADD_JM(kbdev, JD_DONE, kctx, katom, katom->jc, 0); + + kbase_job_check_leave_disjoint(kbdev, katom); + + katom->slot_nr = slot_nr; + + atomic_inc(&kctx->work_count); + +#ifdef CONFIG_DEBUG_FS + /* a failed job happened and is waiting for dumping*/ + if (!katom->will_fail_event_code && + kbase_debug_job_fault_process(katom, katom->event_code)) + return; +#endif + + WARN_ON(work_pending(&katom->work)); + INIT_WORK(&katom->work, kbase_jd_done_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + +KBASE_EXPORT_TEST_API(kbase_jd_done); + +void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(NULL != katom); + kctx = katom->kctx; + KBASE_DEBUG_ASSERT(NULL != kctx); + + dev_dbg(kbdev->dev, "JD: cancelling atom %p\n", (void *)katom); + KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); + + /* This should only be done from a context that is not scheduled */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + WARN_ON(work_pending(&katom->work)); + + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + INIT_WORK(&katom->work, jd_cancel_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + + +void kbase_jd_zap_context(struct kbase_context *kctx) +{ + struct kbase_jd_atom *katom; + struct list_head *entry, *tmp; + struct kbase_device *kbdev; + + KBASE_DEBUG_ASSERT(kctx); + + kbdev = kctx->kbdev; + + KBASE_KTRACE_ADD_JM(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); + + kbase_js_zap_context(kctx); + + mutex_lock(&kctx->jctx.lock); + + /* + * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are + * queued outside the job scheduler. + */ + + del_timer_sync(&kctx->soft_job_timeout); + list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { + katom = list_entry(entry, struct kbase_jd_atom, queue); + kbase_cancel_soft_job(katom); + } + + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + kbase_dma_fence_cancel_all_atoms(kctx); +#endif + + mutex_unlock(&kctx->jctx.lock); + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + /* Flush dma-fence workqueue to ensure that any callbacks that may have + * been queued are done before continuing. + */ + flush_workqueue(kctx->dma_fence.wq); +#endif + +#ifdef CONFIG_DEBUG_FS + kbase_debug_job_fault_kctx_unblock(kctx); +#endif + + kbase_jm_wait_for_zero_jobs(kctx); +} + +KBASE_EXPORT_TEST_API(kbase_jd_zap_context); + +int kbase_jd_init(struct kbase_context *kctx) +{ + int i; + int mali_err = 0; + + KBASE_DEBUG_ASSERT(kctx); + + kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", + WQ_HIGHPRI | WQ_UNBOUND, 1); + if (NULL == kctx->jctx.job_done_wq) { + mali_err = -ENOMEM; + goto out1; + } + + for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { + init_waitqueue_head(&kctx->jctx.atoms[i].completed); + + INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]); + INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]); + + /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */ + kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; + kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; + +#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + kctx->jctx.atoms[i].dma_fence.context = + dma_fence_context_alloc(1); + atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0); + INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks); +#endif + } + + for (i = 0; i < BASE_JD_RP_COUNT; i++) + kctx->jctx.renderpasses[i].state = KBASE_JD_RP_COMPLETE; + + mutex_init(&kctx->jctx.lock); + + init_waitqueue_head(&kctx->jctx.zero_jobs_wait); + + spin_lock_init(&kctx->jctx.tb_lock); + + kctx->jctx.job_nr = 0; + INIT_LIST_HEAD(&kctx->completed_jobs); + atomic_set(&kctx->work_count, 0); + + return 0; + + out1: + return mali_err; +} + +KBASE_EXPORT_TEST_API(kbase_jd_init); + +void kbase_jd_exit(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx); + + /* Work queue is emptied by this */ + destroy_workqueue(kctx->jctx.job_done_wq); +} + +KBASE_EXPORT_TEST_API(kbase_jd_exit); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c new file mode 100755 index 000000000000..6b0c36d6b93f --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c @@ -0,0 +1,250 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifdef CONFIG_DEBUG_FS + +#include +#include +#include +#include +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include +#endif +#include + +struct kbase_jd_debugfs_depinfo { + u8 id; + char type; +}; + +static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, + struct seq_file *sfile) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + struct kbase_sync_fence_info info; + int res; + + switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + res = kbase_sync_fence_out_info_get(atom, &info); + if (res == 0) + seq_printf(sfile, "Sa([%p]%d) ", + info.fence, info.status); + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + res = kbase_sync_fence_in_info_get(atom, &info); + if (res == 0) + seq_printf(sfile, "Wa([%p]%d) ", + info.fence, info.status); + break; + default: + break; + } +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + struct kbase_fence_cb *cb; + + if (atom->dma_fence.fence) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = atom->dma_fence.fence; +#else + struct dma_fence *fence = atom->dma_fence.fence; +#endif + + seq_printf(sfile, +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + "Sd(%u#%u: %s) ", +#else + "Sd(%llu#%u: %s) ", +#endif + fence->context, + fence->seqno, + dma_fence_is_signaled(fence) ? + "signaled" : "active"); + } + + list_for_each_entry(cb, &atom->dma_fence.callbacks, + node) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = cb->fence; +#else + struct dma_fence *fence = cb->fence; +#endif + + seq_printf(sfile, +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + "Wd(%u#%u: %s) ", +#else + "Wd(%llu#%u: %s) ", +#endif + fence->context, + fence->seqno, + dma_fence_is_signaled(fence) ? + "signaled" : "active"); + } + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + +} + +static void kbasep_jd_debugfs_atom_deps( + struct kbase_jd_debugfs_depinfo *deps, + struct kbase_jd_atom *atom) +{ + struct kbase_context *kctx = atom->kctx; + int i; + + for (i = 0; i < 2; i++) { + deps[i].id = (unsigned)(atom->dep[i].atom ? + kbase_jd_atom_id(kctx, atom->dep[i].atom) : 0); + + switch (atom->dep[i].dep_type) { + case BASE_JD_DEP_TYPE_INVALID: + deps[i].type = ' '; + break; + case BASE_JD_DEP_TYPE_DATA: + deps[i].type = 'D'; + break; + case BASE_JD_DEP_TYPE_ORDER: + deps[i].type = '>'; + break; + default: + deps[i].type = '?'; + break; + } + } +} +/** + * kbasep_jd_debugfs_atoms_show - Show callback for the JD atoms debugfs file. + * @sfile: The debugfs entry + * @data: Data associated with the entry + * + * This function is called to get the contents of the JD atoms debugfs file. + * This is a report of all atoms managed by kbase_jd_context.atoms + * + * Return: 0 if successfully prints data in debugfs entry file, failure + * otherwise + */ +static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) +{ + struct kbase_context *kctx = sfile->private; + struct kbase_jd_atom *atoms; + unsigned long irq_flags; + int i; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + /* Print version */ + seq_printf(sfile, "v%u\n", MALI_JD_DEBUGFS_VERSION); + + /* Print U/K API version */ + seq_printf(sfile, "ukv%u.%u\n", BASE_UK_VERSION_MAJOR, + BASE_UK_VERSION_MINOR); + + /* Print table heading */ + seq_puts(sfile, " ID, Core req, St, CR, Predeps, Start time, Additional info...\n"); + + atoms = kctx->jctx.atoms; + /* General atom states */ + mutex_lock(&kctx->jctx.lock); + /* JS-related states */ + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); + for (i = 0; i != BASE_JD_ATOM_COUNT; ++i) { + struct kbase_jd_atom *atom = &atoms[i]; + s64 start_timestamp = 0; + struct kbase_jd_debugfs_depinfo deps[2]; + + if (atom->status == KBASE_JD_ATOM_STATE_UNUSED) + continue; + + /* start_timestamp is cleared as soon as the atom leaves UNUSED state + * and set before a job is submitted to the h/w, a non-zero value means + * it is valid */ + if (ktime_to_ns(atom->start_timestamp)) + start_timestamp = ktime_to_ns( + ktime_sub(ktime_get(), atom->start_timestamp)); + + kbasep_jd_debugfs_atom_deps(deps, atom); + + seq_printf(sfile, + "%3u, %8x, %2u, %c%3u %c%3u, %20lld, ", + i, atom->core_req, atom->status, + deps[0].type, deps[0].id, + deps[1].type, deps[1].id, + start_timestamp); + + + kbase_jd_debugfs_fence_info(atom, sfile); + + seq_puts(sfile, "\n"); + } + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); + mutex_unlock(&kctx->jctx.lock); + + return 0; +} + + +/** + * kbasep_jd_debugfs_atoms_open - open operation for atom debugfs file + * @in: &struct inode pointer + * @file: &struct file pointer + * + * Return: file descriptor + */ +static int kbasep_jd_debugfs_atoms_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_jd_debugfs_atoms_show, in->i_private); +} + +static const struct file_operations kbasep_jd_debugfs_atoms_fops = { + .owner = THIS_MODULE, + .open = kbasep_jd_debugfs_atoms_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) +{ +#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) + const mode_t mode = S_IRUGO; +#else + const mode_t mode = S_IRUSR; +#endif + + /* Caller already ensures this, but we keep the pattern for + * maintenance safety. + */ + if (WARN_ON(!kctx) || + WARN_ON(IS_ERR_OR_NULL(kctx->kctx_dentry))) + return; + + /* Expose all atoms */ + debugfs_create_file("atoms", mode, kctx->kctx_dentry, kctx, + &kbasep_jd_debugfs_atoms_fops); + +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.h new file mode 100755 index 000000000000..697bdef4d434 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file mali_kbase_jd_debugfs.h + * Header file for job dispatcher-related entries in debugfs + */ + +#ifndef _KBASE_JD_DEBUGFS_H +#define _KBASE_JD_DEBUGFS_H + +#include + +#define MALI_JD_DEBUGFS_VERSION 3 + +/* Forward declarations */ +struct kbase_context; + +/** + * kbasep_jd_debugfs_ctx_init() - Add debugfs entries for JD system + * + * @kctx Pointer to kbase_context + */ +void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx); + +#endif /*_KBASE_JD_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_jm.c new file mode 100755 index 000000000000..fb15a8c1727a --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_jm.c @@ -0,0 +1,155 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * HW access job manager common APIs + */ + +#include +#include "mali_kbase_hwaccess_jm.h" +#include "mali_kbase_jm.h" + +#if !MALI_USE_CSF +/** + * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot + * @js on the active context. + * @kbdev: Device pointer + * @js: Job slot to run on + * @nr_jobs_to_submit: Number of jobs to attempt to submit + * + * Return: true if slot can still be submitted on, false if slot is now full. + */ +static bool kbase_jm_next_job(struct kbase_device *kbdev, int js, + int nr_jobs_to_submit) +{ + struct kbase_context *kctx; + int i; + + kctx = kbdev->hwaccess.active_kctx[js]; + dev_dbg(kbdev->dev, + "Trying to run the next %d jobs in kctx %p (s:%d)\n", + nr_jobs_to_submit, (void *)kctx, js); + + if (!kctx) + return true; + + for (i = 0; i < nr_jobs_to_submit; i++) { + struct kbase_jd_atom *katom = kbase_js_pull(kctx, js); + + if (!katom) + return true; /* Context has no jobs on this slot */ + + kbase_backend_run_atom(kbdev, katom); + } + + dev_dbg(kbdev->dev, "Slot ringbuffer should now be full (s:%d)\n", js); + return false; +} + +u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask) +{ + u32 ret_mask = 0; + + lockdep_assert_held(&kbdev->hwaccess_lock); + dev_dbg(kbdev->dev, "JM kick slot mask 0x%x\n", js_mask); + + while (js_mask) { + int js = ffs(js_mask) - 1; + int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js); + + if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit)) + ret_mask |= (1 << js); + + js_mask &= ~(1 << js); + } + + dev_dbg(kbdev->dev, "Can still submit to mask 0x%x\n", ret_mask); + return ret_mask; +} + +void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!down_trylock(&js_devdata->schedule_sem)) { + kbase_jm_kick(kbdev, js_mask); + up(&js_devdata->schedule_sem); + } +} + +void kbase_jm_try_kick_all(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!down_trylock(&js_devdata->schedule_sem)) { + kbase_jm_kick_all(kbdev); + up(&js_devdata->schedule_sem); + } +} +#endif /* !MALI_USE_CSF */ + +void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + int js; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + if (kbdev->hwaccess.active_kctx[js] == kctx) { + dev_dbg(kbdev->dev, "Marking kctx %p as inactive (s:%d)\n", + (void *)kctx, js); + kbdev->hwaccess.active_kctx[js] = NULL; + } + } +} + +#if !MALI_USE_CSF +struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + dev_dbg(kbdev->dev, "Atom %p is returning with event code 0x%x\n", + (void *)katom, katom->event_code); + + if (katom->event_code != BASE_JD_EVENT_STOPPED && + katom->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT) { + return kbase_js_complete_atom(katom, NULL); + } else { + kbase_js_unpull(katom->kctx, katom); + return NULL; + } +} + +struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, ktime_t *end_timestamp) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + return kbase_js_complete_atom(katom, end_timestamp); +} +#endif /* !MALI_USE_CSF */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_jm.h new file mode 100755 index 000000000000..b3fd421a1ff3 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_jm.h @@ -0,0 +1,119 @@ +/* + * + * (C) COPYRIGHT 2014, 2016, 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +/* + * Job manager common APIs + */ + +#ifndef _KBASE_JM_H_ +#define _KBASE_JM_H_ + +#if !MALI_USE_CSF +/** + * kbase_jm_kick() - Indicate that there are jobs ready to run. + * @kbdev: Device pointer + * @js_mask: Mask of the job slots that can be pulled from. + * + * Caller must hold the hwaccess_lock and schedule_sem semaphore + * + * Return: Mask of the job slots that can still be submitted to. + */ +u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask); + +/** + * kbase_jm_kick_all() - Indicate that there are jobs ready to run on all job + * slots. + * @kbdev: Device pointer + * + * Caller must hold the hwaccess_lock and schedule_sem semaphore + * + * Return: Mask of the job slots that can still be submitted to. + */ +static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev) +{ + return kbase_jm_kick(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); +} + +/** + * kbase_jm_try_kick - Attempt to call kbase_jm_kick + * @kbdev: Device pointer + * @js_mask: Mask of the job slots that can be pulled from + * Context: Caller must hold hwaccess_lock + * + * If schedule_sem can be immediately obtained then this function will call + * kbase_jm_kick() otherwise it will do nothing. + */ +void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask); + +/** + * kbase_jm_try_kick_all() - Attempt to call kbase_jm_kick_all + * @kbdev: Device pointer + * Context: Caller must hold hwaccess_lock + * + * If schedule_sem can be immediately obtained then this function will call + * kbase_jm_kick_all() otherwise it will do nothing. + */ +void kbase_jm_try_kick_all(struct kbase_device *kbdev); +#endif /* !MALI_USE_CSF */ + +/** + * kbase_jm_idle_ctx() - Mark a context as idle. + * @kbdev: Device pointer + * @kctx: Context to mark as idle + * + * No more atoms will be pulled from this context until it is marked as active + * by kbase_js_use_ctx(). + * + * The context should have no atoms currently pulled from it + * (kctx->atoms_pulled == 0). + * + * Caller must hold the hwaccess_lock + */ +void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +#if !MALI_USE_CSF +/** + * kbase_jm_return_atom_to_js() - Return an atom to the job scheduler that has + * been soft-stopped or will fail due to a + * dependency + * @kbdev: Device pointer + * @katom: Atom that has been stopped or will be failed + * + * Return: Atom that has now been unblocked and can now be run, or NULL if none + */ +struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_jm_complete() - Complete an atom + * @kbdev: Device pointer + * @katom: Atom that has completed + * @end_timestamp: Timestamp of atom completion + * + * Return: Atom that has now been unblocked and can now be run, or NULL if none + */ +struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, ktime_t *end_timestamp); +#endif /* !MALI_USE_CSF */ + +#endif /* _KBASE_JM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js.c b/drivers/gpu/arm/bifrost/mali_kbase_js.c new file mode 100755 index 000000000000..9b338eb66531 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_js.c @@ -0,0 +1,3760 @@ +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/* + * Job Scheduler Implementation + */ +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "mali_kbase_jm.h" +#include "mali_kbase_hwaccess_jm.h" + +/* + * Private types + */ + +/* Bitpattern indicating the result of releasing a context */ +enum { + /* The context was descheduled - caller should try scheduling in a new + * one to keep the runpool full */ + KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), + /* Ctx attributes were changed - caller should try scheduling all + * contexts */ + KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) +}; + +typedef u32 kbasep_js_release_result; + +const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { + KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ + KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ + KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */ +}; + +const base_jd_prio +kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { + BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ + BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ + BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ +}; + + +/* + * Private function prototypes + */ +static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( + struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state); + +static int kbase_js_get_slot(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, + kbasep_js_ctx_job_cb callback); + +/* Helper for ktrace */ +#if KBASE_KTRACE_ENABLE +static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) +{ + return atomic_read(&kctx->refcount); +} +#else /* KBASE_KTRACE_ENABLE */ +static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) +{ + CSTD_UNUSED(kctx); + return 0; +} +#endif /* KBASE_KTRACE_ENABLE */ + +/* + * Private functions + */ + +/** + * core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements + * @features: JSn_FEATURE register value + * + * Given a JSn_FEATURE register value returns the core requirements that match + * + * Return: Core requirement bit mask + */ +static base_jd_core_req core_reqs_from_jsn_features(u16 features) +{ + base_jd_core_req core_req = 0u; + + if ((features & JS_FEATURE_SET_VALUE_JOB) != 0) + core_req |= BASE_JD_REQ_V; + + if ((features & JS_FEATURE_CACHE_FLUSH_JOB) != 0) + core_req |= BASE_JD_REQ_CF; + + if ((features & JS_FEATURE_COMPUTE_JOB) != 0) + core_req |= BASE_JD_REQ_CS; + + if ((features & JS_FEATURE_TILER_JOB) != 0) + core_req |= BASE_JD_REQ_T; + + if ((features & JS_FEATURE_FRAGMENT_JOB) != 0) + core_req |= BASE_JD_REQ_FS; + + return core_req; +} + +static void kbase_js_sync_timers(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->js_data.runpool_mutex); + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&kbdev->js_data.runpool_mutex); +} + +/** + * jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * @prio: Priority to check. + * + * Return true if there are no atoms to pull. There may be running atoms in the + * ring buffer even if there are no atoms to pull. It is also possible for the + * ring buffer to be full (with running atoms) when this functions returns + * true. + * + * Return: true if there are no atoms to pull, false otherwise. + */ +static inline bool +jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio) +{ + bool none_to_pull; + struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + none_to_pull = RB_EMPTY_ROOT(&rb->runnable_tree); + + dev_dbg(kctx->kbdev->dev, + "Slot %d (prio %d) is %spullable in kctx %p\n", + js, prio, none_to_pull ? "not " : "", kctx); + + return none_to_pull; +} + +/** + * jsctx_rb_none_to_pull(): - Check if all priority ring buffers have no + * pullable atoms + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * + * Caller must hold hwaccess_lock + * + * Return: true if the ring buffers for all priorities have no pullable atoms, + * false otherwise. + */ +static inline bool +jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) +{ + int prio; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; + prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { + if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) + return false; + } + + return true; +} + +/** + * jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue. + * @kctx: Pointer to kbase context with the queue. + * @js: Job slot id to iterate. + * @prio: Priority id to iterate. + * @callback: Function pointer to callback. + * + * Iterate over a queue and invoke @callback for each entry in the queue, and + * remove the entry from the queue. + * + * If entries are added to the queue while this is running those entries may, or + * may not be covered. To ensure that all entries in the buffer have been + * enumerated when this function returns jsctx->lock must be held when calling + * this function. + * + * The HW access lock must always be held when calling this function. + */ +static void +jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, + kbasep_js_ctx_job_cb callback) +{ + struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + while (!RB_EMPTY_ROOT(&queue->runnable_tree)) { + struct rb_node *node = rb_first(&queue->runnable_tree); + struct kbase_jd_atom *entry = rb_entry(node, + struct kbase_jd_atom, runnable_tree_node); + + rb_erase(node, &queue->runnable_tree); + callback(kctx->kbdev, entry); + + /* Runnable end-of-renderpass atoms can also be in the linked + * list of atoms blocked on cross-slot dependencies. Remove them + * to avoid calling the callback twice. + */ + if (entry->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) { + WARN_ON(!(entry->core_req & + BASE_JD_REQ_END_RENDERPASS)); + dev_dbg(kctx->kbdev->dev, + "Del runnable atom %p from X_DEP list\n", + (void *)entry); + + list_del(&entry->queue); + entry->atom_flags &= + ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; + } + } + + while (!list_empty(&queue->x_dep_head)) { + struct kbase_jd_atom *entry = list_entry(queue->x_dep_head.next, + struct kbase_jd_atom, queue); + + WARN_ON(!(entry->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); + dev_dbg(kctx->kbdev->dev, + "Del blocked atom %p from X_DEP list\n", + (void *)entry); + + list_del(queue->x_dep_head.next); + entry->atom_flags &= + ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; + + callback(kctx->kbdev, entry); + } +} + +/** + * jsctx_queue_foreach(): - Execute callback for each entry in every queue + * @kctx: Pointer to kbase context with queue. + * @js: Job slot id to iterate. + * @callback: Function pointer to callback. + * + * Iterate over all the different priorities, and for each call + * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback + * for each entry, and remove the entry from the queue. + */ +static inline void +jsctx_queue_foreach(struct kbase_context *kctx, int js, + kbasep_js_ctx_job_cb callback) +{ + int prio; + + for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; + prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) + jsctx_queue_foreach_prio(kctx, js, prio, callback); +} + +/** + * jsctx_rb_peek_prio(): - Check buffer and get next atom + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * @prio: Priority id to check. + * + * Check the ring buffer for the specified @js and @prio and return a pointer to + * the next atom, unless the ring buffer is empty. + * + * Return: Pointer to next atom in buffer, or NULL if there is no atom. + */ +static inline struct kbase_jd_atom * +jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) +{ + struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + struct rb_node *node; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + dev_dbg(kctx->kbdev->dev, + "Peeking runnable tree of kctx %p for prio %d (s:%d)\n", + (void *)kctx, prio, js); + + node = rb_first(&rb->runnable_tree); + if (!node) { + dev_dbg(kctx->kbdev->dev, "Tree is empty\n"); + return NULL; + } + + return rb_entry(node, struct kbase_jd_atom, runnable_tree_node); +} + +/** + * jsctx_rb_peek(): - Check all priority buffers and get next atom + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * + * Check the ring buffers for all priorities, starting from + * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a + * pointer to the next atom, unless all the priority's ring buffers are empty. + * + * Caller must hold the hwaccess_lock. + * + * Return: Pointer to next atom in buffer, or NULL if there is no atom. + */ +static inline struct kbase_jd_atom * +jsctx_rb_peek(struct kbase_context *kctx, int js) +{ + int prio; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; + prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { + struct kbase_jd_atom *katom; + + katom = jsctx_rb_peek_prio(kctx, js, prio); + if (katom) + return katom; + } + + return NULL; +} + +/** + * jsctx_rb_pull(): - Mark atom in list as running + * @kctx: Pointer to kbase context with ring buffer. + * @katom: Pointer to katom to pull. + * + * Mark an atom previously obtained from jsctx_rb_peek() as running. + * + * @katom must currently be at the head of the ring buffer. + */ +static inline void +jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + dev_dbg(kctx->kbdev->dev, "Erasing atom %p from runnable tree of kctx %p\n", + (void *)katom, (void *)kctx); + + /* Atoms must be pulled in the correct order. */ + WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio)); + + rb_erase(&katom->runnable_tree_node, &rb->runnable_tree); +} + +#define LESS_THAN_WRAP(a, b) ((s32)(a - b) < 0) + +static void +jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + struct kbase_device *kbdev = kctx->kbdev; + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; + struct rb_node **new = &(queue->runnable_tree.rb_node), *parent = NULL; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + dev_dbg(kbdev->dev, "Adding atom %p to runnable tree of kctx %p (s:%d)\n", + (void *)katom, (void *)kctx, js); + + while (*new) { + struct kbase_jd_atom *entry = container_of(*new, + struct kbase_jd_atom, runnable_tree_node); + + parent = *new; + if (LESS_THAN_WRAP(katom->age, entry->age)) + new = &((*new)->rb_left); + else + new = &((*new)->rb_right); + } + + /* Add new node and rebalance tree. */ + rb_link_node(&katom->runnable_tree_node, parent, new); + rb_insert_color(&katom->runnable_tree_node, &queue->runnable_tree); + + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_READY); +} + +/** + * jsctx_rb_unpull(): - Undo marking of atom in list as running + * @kctx: Pointer to kbase context with ring buffer. + * @katom: Pointer to katom to unpull. + * + * Undo jsctx_rb_pull() and put @katom back in the queue. + * + * jsctx_rb_unpull() must be called on atoms in the same order the atoms were + * pulled. + */ +static inline void +jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + jsctx_tree_add(kctx, katom); +} + +static bool kbase_js_ctx_pullable(struct kbase_context *kctx, + int js, + bool is_scheduled); +static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js); +static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js); + +/* + * Functions private to KBase ('Protected' functions) + */ +int kbasep_js_devdata_init(struct kbase_device * const kbdev) +{ + struct kbasep_js_device_data *jsdd; + int i, j; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + jsdd = &kbdev->js_data; + +#ifdef CONFIG_MALI_BIFROST_DEBUG + /* Soft-stop will be disabled on a single context by default unless + * softstop_always is set */ + jsdd->softstop_always = false; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + jsdd->nr_all_contexts_running = 0; + jsdd->nr_user_contexts_running = 0; + jsdd->nr_contexts_pullable = 0; + atomic_set(&jsdd->nr_contexts_runnable, 0); + /* No ctx allowed to submit */ + jsdd->runpool_irq.submit_allowed = 0u; + memset(jsdd->runpool_irq.ctx_attr_ref_count, 0, + sizeof(jsdd->runpool_irq.ctx_attr_ref_count)); + memset(jsdd->runpool_irq.slot_affinities, 0, + sizeof(jsdd->runpool_irq.slot_affinities)); + memset(jsdd->runpool_irq.slot_affinity_refcount, 0, + sizeof(jsdd->runpool_irq.slot_affinity_refcount)); + INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list); + + /* Config attributes */ + jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; + jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; + jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; + jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; + jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; + jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING; + jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; + jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; + jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING; + jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS; + atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT); + + dev_dbg(kbdev->dev, "JS Config Attribs: "); + dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u", + jsdd->scheduling_period_ns); + dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u", + jsdd->soft_stop_ticks); + dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u", + jsdd->soft_stop_ticks_cl); + dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u", + jsdd->hard_stop_ticks_ss); + dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u", + jsdd->hard_stop_ticks_cl); + dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u", + jsdd->hard_stop_ticks_dumping); + dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u", + jsdd->gpu_reset_ticks_ss); + dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u", + jsdd->gpu_reset_ticks_cl); + dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u", + jsdd->gpu_reset_ticks_dumping); + dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u", + jsdd->ctx_timeslice_ns); + dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i", + atomic_read(&jsdd->soft_job_timeout_ms)); + + if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss && + jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss && + jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping && + jsdd->hard_stop_ticks_dumping < + jsdd->gpu_reset_ticks_dumping)) { + dev_err(kbdev->dev, "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n"); + return -EINVAL; + } + +#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS + dev_dbg(kbdev->dev, "Job Scheduling Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.", + jsdd->soft_stop_ticks, + jsdd->scheduling_period_ns); +#endif +#if KBASE_DISABLE_SCHEDULING_HARD_STOPS + dev_dbg(kbdev->dev, "Job Scheduling Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.", + jsdd->hard_stop_ticks_ss, + jsdd->hard_stop_ticks_dumping, + jsdd->scheduling_period_ns); +#endif +#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS + dev_dbg(kbdev->dev, "Note: The JS tick timer (if coded) will still be run, but do nothing."); +#endif + + for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) + jsdd->js_reqs[i] = core_reqs_from_jsn_features( + kbdev->gpu_props.props.raw_props.js_features[i]); + + /* On error, we could continue on: providing none of the below resources + * rely on the ones above */ + + mutex_init(&jsdd->runpool_mutex); + mutex_init(&jsdd->queue_mutex); + sema_init(&jsdd->schedule_sem, 1); + + for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { + for (j = 0; j < KBASE_JS_ATOM_SCHED_PRIO_COUNT; ++j) { + INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i][j]); + INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i][j]); + } + } + + return 0; +} + +void kbasep_js_devdata_halt(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +void kbasep_js_devdata_term(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata; + s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { 0, }; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + js_devdata = &kbdev->js_data; + + /* The caller must de-register all contexts before calling this + */ + KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0); + KBASE_DEBUG_ASSERT(memcmp( + js_devdata->runpool_irq.ctx_attr_ref_count, + zero_ctx_attr_ref_count, + sizeof(zero_ctx_attr_ref_count)) == 0); + CSTD_UNUSED(zero_ctx_attr_ref_count); +} + +int kbasep_js_kctx_init(struct kbase_context *const kctx) +{ + struct kbase_device *kbdev; + struct kbasep_js_kctx_info *js_kctx_info; + int i, j; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) + INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]); + + js_kctx_info = &kctx->jctx.sched_info; + + kctx->slots_pullable = 0; + js_kctx_info->ctx.nr_jobs = 0; + kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); + kbase_ctx_flag_clear(kctx, KCTX_DYING); + memset(js_kctx_info->ctx.ctx_attr_ref_count, 0, + sizeof(js_kctx_info->ctx.ctx_attr_ref_count)); + + /* Initially, the context is disabled from submission until the create + * flags are set */ + kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); + + /* On error, we could continue on: providing none of the below resources + * rely on the ones above */ + mutex_init(&js_kctx_info->ctx.jsctx_mutex); + + init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); + + for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { + for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { + INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); + kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; + } + } + + return 0; +} + +void kbasep_js_kctx_term(struct kbase_context *kctx) +{ + struct kbase_device *kbdev; + struct kbasep_js_kctx_info *js_kctx_info; + int js; + bool update_ctx_count = false; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + js_kctx_info = &kctx->jctx.sched_info; + + /* The caller must de-register all jobs before calling this */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0); + + mutex_lock(&kbdev->js_data.queue_mutex); + mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)) { + WARN_ON(atomic_read(&kbdev->js_data.nr_contexts_runnable) <= 0); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + update_ctx_count = true; + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + } + + mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&kbdev->js_data.queue_mutex); + + if (update_ctx_count) { + mutex_lock(&kbdev->js_data.runpool_mutex); + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&kbdev->js_data.runpool_mutex); + } +} + +/** + * kbase_js_ctx_list_add_pullable_nolock - Variant of + * kbase_jd_ctx_list_add_pullable() + * where the caller must hold + * hwaccess_lock + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + dev_dbg(kbdev->dev, "Add pullable tail kctx %p (s:%d)\n", + (void *)kctx, js); + + if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_pullable[js][kctx->priority]); + + if (!kctx->slots_pullable) { + kbdev->js_data.nr_contexts_pullable++; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); + atomic_inc(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable |= (1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_add_pullable_head_nolock - Variant of + * kbase_js_ctx_list_add_pullable_head() + * where the caller must hold + * hwaccess_lock + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_pullable_head_nolock( + struct kbase_device *kbdev, struct kbase_context *kctx, int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + dev_dbg(kbdev->dev, "Add pullable head kctx %p (s:%d)\n", + (void *)kctx, js); + + if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_pullable[js][kctx->priority]); + + if (!kctx->slots_pullable) { + kbdev->js_data.nr_contexts_pullable++; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); + atomic_inc(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable |= (1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_add_pullable_head - Add context to the head of the + * per-slot pullable context queue + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * If the context is on either the pullable or unpullable queues, then it is + * removed before being added to the head. + * + * This function should be used when a context has been scheduled, but no jobs + * can currently be pulled from it. + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + ret = kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return ret; +} + +/** + * kbase_js_ctx_list_add_unpullable_nolock - Add context to the tail of the + * per-slot unpullable context queue + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * The context must already be on the per-slot pullable queue. It will be + * removed from the pullable queue before being added to the unpullable queue. + * + * This function should be used when a context has been pulled from, and there + * are no jobs remaining on the specified slot. + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + dev_dbg(kbdev->dev, "Add unpullable tail kctx %p (s:%d)\n", + (void *)kctx, js); + + list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_unpullable[js][kctx->priority]); + + if (kctx->slots_pullable == (1 << js)) { + kbdev->js_data.nr_contexts_pullable--; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable &= ~(1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_remove_nolock - Remove context from the per-slot pullable + * or unpullable context queues + * @kbdev: Device pointer + * @kctx: Context to remove from queue + * @js: Job slot to use + * + * The context must already be on one of the queues. + * + * This function should be used when a context has no jobs on the GPU, and no + * jobs remaining for the specified slot. + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])); + + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + if (kctx->slots_pullable == (1 << js)) { + kbdev->js_data.nr_contexts_pullable--; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable &= ~(1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_pop_head_nolock - Variant of kbase_js_ctx_list_pop_head() + * where the caller must hold + * hwaccess_lock + * @kbdev: Device pointer + * @js: Job slot to use + * + * Caller must hold hwaccess_lock + * + * Return: Context to use for specified slot. + * NULL if no contexts present for specified slot + */ +static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( + struct kbase_device *kbdev, + int js) +{ + struct kbase_context *kctx; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { + if (list_empty(&kbdev->js_data.ctx_list_pullable[js][i])) + continue; + + kctx = list_entry(kbdev->js_data.ctx_list_pullable[js][i].next, + struct kbase_context, + jctx.sched_info.ctx.ctx_list_entry[js]); + + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + dev_dbg(kbdev->dev, + "Popped %p from the pullable queue (s:%d)\n", + (void *)kctx, js); + return kctx; + } + return NULL; +} + +/** + * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable + * queue. + * @kbdev: Device pointer + * @js: Job slot to use + * + * Return: Context to use for specified slot. + * NULL if no contexts present for specified slot + */ +static struct kbase_context *kbase_js_ctx_list_pop_head( + struct kbase_device *kbdev, int js) +{ + struct kbase_context *kctx; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kctx = kbase_js_ctx_list_pop_head_nolock(kbdev, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return kctx; +} + +/** + * kbase_js_ctx_pullable - Return if a context can be pulled from on the + * specified slot + * @kctx: Context pointer + * @js: Job slot to use + * @is_scheduled: true if the context is currently scheduled + * + * Caller must hold hwaccess_lock + * + * Return: true if context can be pulled from on specified slot + * false otherwise + */ +static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js, + bool is_scheduled) +{ + struct kbasep_js_device_data *js_devdata; + struct kbase_jd_atom *katom; + struct kbase_device *kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + js_devdata = &kbdev->js_data; + + if (is_scheduled) { + if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) { + dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %p\n", + (void *)kctx); + return false; + } + } + katom = jsctx_rb_peek(kctx, js); + if (!katom) { + dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %p (s:%d)\n", + (void *)kctx, js); + return false; /* No pullable atoms */ + } + if (kctx->blocked_js[js][katom->sched_priority]) { + dev_dbg(kbdev->dev, + "JS: kctx %p is blocked from submitting atoms at priority %d (s:%d)\n", + (void *)kctx, katom->sched_priority, js); + return false; + } + if (atomic_read(&katom->blocked)) { + dev_dbg(kbdev->dev, "JS: Atom %p is blocked in js_ctx_pullable\n", + (void *)katom); + return false; /* next atom blocked */ + } + if (kbase_js_atom_blocked_on_x_dep(katom)) { + if (katom->x_pre_dep->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || + katom->x_pre_dep->will_fail_event_code) { + dev_dbg(kbdev->dev, + "JS: X pre-dep %p is not present in slot FIFO or will fail\n", + (void *)katom->x_pre_dep); + return false; + } + if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && + kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) { + dev_dbg(kbdev->dev, + "JS: Atom %p has cross-slot fail dependency and atoms on slot (s:%d)\n", + (void *)katom, js); + return false; + } + } + + dev_dbg(kbdev->dev, "JS: Atom %p is pullable in kctx %p (s:%d)\n", + (void *)katom, (void *)kctx, js); + + return true; +} + +static bool kbase_js_dep_validate(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + struct kbase_device *kbdev = kctx->kbdev; + bool ret = true; + bool has_dep = false, has_x_dep = false; + int js = kbase_js_get_slot(kbdev, katom); + int prio = katom->sched_priority; + int i; + + for (i = 0; i < 2; i++) { + struct kbase_jd_atom *dep_atom = katom->dep[i].atom; + + if (dep_atom) { + int dep_js = kbase_js_get_slot(kbdev, dep_atom); + int dep_prio = dep_atom->sched_priority; + + dev_dbg(kbdev->dev, + "Checking dep %d of atom %p (s:%d) on %p (s:%d)\n", + i, (void *)katom, js, (void *)dep_atom, dep_js); + + /* Dependent atom must already have been submitted */ + if (!(dep_atom->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { + dev_dbg(kbdev->dev, + "Blocker not submitted yet\n"); + ret = false; + break; + } + + /* Dependencies with different priorities can't + be represented in the ringbuffer */ + if (prio != dep_prio) { + dev_dbg(kbdev->dev, + "Different atom priorities\n"); + ret = false; + break; + } + + if (js == dep_js) { + /* Only one same-slot dependency can be + * represented in the ringbuffer */ + if (has_dep) { + dev_dbg(kbdev->dev, + "Too many same-slot deps\n"); + ret = false; + break; + } + /* Each dependee atom can only have one + * same-slot dependency */ + if (dep_atom->post_dep) { + dev_dbg(kbdev->dev, + "Too many same-slot successors\n"); + ret = false; + break; + } + has_dep = true; + } else { + /* Only one cross-slot dependency can be + * represented in the ringbuffer */ + if (has_x_dep) { + dev_dbg(kbdev->dev, + "Too many cross-slot deps\n"); + ret = false; + break; + } + /* Each dependee atom can only have one + * cross-slot dependency */ + if (dep_atom->x_post_dep) { + dev_dbg(kbdev->dev, + "Too many cross-slot successors\n"); + ret = false; + break; + } + /* The dependee atom can not already be in the + * HW access ringbuffer */ + if (dep_atom->gpu_rb_state != + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + dev_dbg(kbdev->dev, + "Blocker already in ringbuffer (state:%d)\n", + dep_atom->gpu_rb_state); + ret = false; + break; + } + /* The dependee atom can not already have + * completed */ + if (dep_atom->status != + KBASE_JD_ATOM_STATE_IN_JS) { + dev_dbg(kbdev->dev, + "Blocker already completed (status:%d)\n", + dep_atom->status); + ret = false; + break; + } + + has_x_dep = true; + } + + /* Dependency can be represented in ringbuffers */ + } + } + + /* If dependencies can be represented by ringbuffer then clear them from + * atom structure */ + if (ret) { + for (i = 0; i < 2; i++) { + struct kbase_jd_atom *dep_atom = katom->dep[i].atom; + + if (dep_atom) { + int dep_js = kbase_js_get_slot(kbdev, dep_atom); + + dev_dbg(kbdev->dev, + "Clearing dep %d of atom %p (s:%d) on %p (s:%d)\n", + i, (void *)katom, js, (void *)dep_atom, + dep_js); + + if ((js != dep_js) && + (dep_atom->status != + KBASE_JD_ATOM_STATE_COMPLETED) + && (dep_atom->status != + KBASE_JD_ATOM_STATE_HW_COMPLETED) + && (dep_atom->status != + KBASE_JD_ATOM_STATE_UNUSED)) { + + katom->atom_flags |= + KBASE_KATOM_FLAG_X_DEP_BLOCKED; + + dev_dbg(kbdev->dev, "Set X_DEP flag on atom %p\n", + (void *)katom); + + katom->x_pre_dep = dep_atom; + dep_atom->x_post_dep = katom; + if (kbase_jd_katom_dep_type( + &katom->dep[i]) == + BASE_JD_DEP_TYPE_DATA) + katom->atom_flags |= + KBASE_KATOM_FLAG_FAIL_BLOCKER; + } + if ((kbase_jd_katom_dep_type(&katom->dep[i]) + == BASE_JD_DEP_TYPE_DATA) && + (js == dep_js)) { + katom->pre_dep = dep_atom; + dep_atom->post_dep = katom; + } + + list_del(&katom->dep_item[i]); + kbase_jd_katom_dep_clear(&katom->dep[i]); + } + } + } else { + dev_dbg(kbdev->dev, + "Deps of atom %p (s:%d) could not be represented\n", + (void *)katom, js); + } + + return ret; +} + +void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority) +{ + struct kbase_device *kbdev = kctx->kbdev; + int js; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* Move kctx to the pullable/upullable list as per the new priority */ + if (new_priority != kctx->priority) { + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + if (kctx->slots_pullable & (1 << js)) + list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_pullable[js][new_priority]); + else + list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_unpullable[js][new_priority]); + } + + kctx->priority = new_priority; + } +} + +void kbase_js_update_ctx_priority(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + int new_priority = KBASE_JS_ATOM_SCHED_PRIO_LOW; + int prio; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->js_ctx_scheduling_mode == KBASE_JS_SYSTEM_PRIORITY_MODE) { + /* Determine the new priority for context, as per the priority + * of currently in-use atoms. + */ + for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; + prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { + if (kctx->atoms_count[prio]) { + new_priority = prio; + break; + } + } + } + + kbase_js_set_ctx_priority(kctx, new_priority); +} + +/** + * js_add_start_rp() - Add an atom that starts a renderpass to the job scheduler + * @start_katom: Pointer to the atom to be added. + * Return: 0 if successful or a negative value on failure. + */ +static int js_add_start_rp(struct kbase_jd_atom *const start_katom) +{ + struct kbase_context *const kctx = start_katom->kctx; + struct kbase_jd_renderpass *rp; + struct kbase_device *const kbdev = kctx->kbdev; + unsigned long flags; + + lockdep_assert_held(&kctx->jctx.lock); + + if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) + return -EINVAL; + + if (start_katom->core_req & BASE_JD_REQ_END_RENDERPASS) + return -EINVAL; + + compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + + rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; + + if (rp->state != KBASE_JD_RP_COMPLETE) + return -EINVAL; + + dev_dbg(kctx->kbdev->dev, "JS add start atom %p of RP %d\n", + (void *)start_katom, start_katom->renderpass_id); + + /* The following members are read when updating the job slot + * ringbuffer/fifo therefore they require additional locking. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + rp->state = KBASE_JD_RP_START; + rp->start_katom = start_katom; + rp->end_katom = NULL; + INIT_LIST_HEAD(&rp->oom_reg_list); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return 0; +} + +/** + * js_add_end_rp() - Add an atom that ends a renderpass to the job scheduler + * @end_katom: Pointer to the atom to be added. + * Return: 0 if successful or a negative value on failure. + */ +static int js_add_end_rp(struct kbase_jd_atom *const end_katom) +{ + struct kbase_context *const kctx = end_katom->kctx; + struct kbase_jd_renderpass *rp; + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kctx->jctx.lock); + + if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) + return -EINVAL; + + if (end_katom->core_req & BASE_JD_REQ_START_RENDERPASS) + return -EINVAL; + + compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + + rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; + + dev_dbg(kbdev->dev, "JS add end atom %p in state %d of RP %d\n", + (void *)end_katom, (int)rp->state, end_katom->renderpass_id); + + if (rp->state == KBASE_JD_RP_COMPLETE) + return -EINVAL; + + if (rp->end_katom == NULL) { + /* We can't be in a retry state until the fragment job chain + * has completed. + */ + unsigned long flags; + + WARN_ON(rp->state == KBASE_JD_RP_RETRY); + WARN_ON(rp->state == KBASE_JD_RP_RETRY_PEND_OOM); + WARN_ON(rp->state == KBASE_JD_RP_RETRY_OOM); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + rp->end_katom = end_katom; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } else + WARN_ON(rp->end_katom != end_katom); + + return 0; +} + +bool kbasep_js_add_job(struct kbase_context *kctx, + struct kbase_jd_atom *atom) +{ + unsigned long flags; + struct kbasep_js_kctx_info *js_kctx_info; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + int err = 0; + + bool enqueue_required = false; + bool timer_sync = false; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(atom != NULL); + lockdep_assert_held(&kctx->jctx.lock); + + kbdev = kctx->kbdev; + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + if (atom->core_req & BASE_JD_REQ_START_RENDERPASS) + err = js_add_start_rp(atom); + else if (atom->core_req & BASE_JD_REQ_END_RENDERPASS) + err = js_add_end_rp(atom); + + if (err < 0) { + atom->event_code = BASE_JD_EVENT_JOB_INVALID; + atom->status = KBASE_JD_ATOM_STATE_COMPLETED; + goto out_unlock; + } + + /* + * Begin Runpool transaction + */ + mutex_lock(&js_devdata->runpool_mutex); + + /* Refcount ctx.nr_jobs */ + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX); + ++(js_kctx_info->ctx.nr_jobs); + dev_dbg(kbdev->dev, "Add atom %p to kctx %p; now %d in ctx\n", + (void *)atom, (void *)kctx, js_kctx_info->ctx.nr_jobs); + + /* Lock for state available during IRQ */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (++kctx->atoms_count[atom->sched_priority] == 1) + kbase_js_update_ctx_priority(kctx); + + if (!kbase_js_dep_validate(kctx, atom)) { + /* Dependencies could not be represented */ + --(js_kctx_info->ctx.nr_jobs); + dev_dbg(kbdev->dev, + "Remove atom %p from kctx %p; now %d in ctx\n", + (void *)atom, (void *)kctx, js_kctx_info->ctx.nr_jobs); + + /* Setting atom status back to queued as it still has unresolved + * dependencies */ + atom->status = KBASE_JD_ATOM_STATE_QUEUED; + dev_dbg(kbdev->dev, "Atom %p status to queued\n", (void *)atom); + + /* Undo the count, as the atom will get added again later but + * leave the context priority adjusted or boosted, in case if + * this was the first higher priority atom received for this + * context. + * This will prevent the scenario of priority inversion, where + * another context having medium priority atoms keeps getting + * scheduled over this context, which is having both lower and + * higher priority atoms, but higher priority atoms are blocked + * due to dependency on lower priority atoms. With priority + * boost the high priority atom will get to run at earliest. + */ + kctx->atoms_count[atom->sched_priority]--; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->runpool_mutex); + + goto out_unlock; + } + + enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); + + KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, + kbase_ktrace_get_ctx_refcnt(kctx)); + + /* Context Attribute Refcounting */ + kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); + + if (enqueue_required) { + if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false)) + timer_sync = kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, atom->slot_nr); + else + timer_sync = kbase_js_ctx_list_add_unpullable_nolock( + kbdev, kctx, atom->slot_nr); + } + /* If this context is active and the atom is the first on its slot, + * kick the job manager to attempt to fast-start the atom */ + if (enqueue_required && kctx == + kbdev->hwaccess.active_kctx[atom->slot_nr]) + kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); + /* End runpool transaction */ + + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + /* A job got added while/after kbase_job_zap_context() + * was called on a non-scheduled context. Kill that job + * by killing the context. */ + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, + false); + } else if (js_kctx_info->ctx.nr_jobs == 1) { + /* Handle Refcount going from 0 to 1: schedule the + * context on the Queue */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx); + + /* Queue was updated - caller must try to + * schedule the head context */ + WARN_ON(!enqueue_required); + } + } +out_unlock: + dev_dbg(kbdev->dev, "Enqueue of kctx %p is %srequired\n", + kctx, enqueue_required ? "" : "not "); + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + mutex_unlock(&js_devdata->queue_mutex); + + return enqueue_required; +} + +void kbasep_js_remove_job(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_jd_atom *atom) +{ + struct kbasep_js_kctx_info *js_kctx_info; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(atom != NULL); + + js_kctx_info = &kctx->jctx.sched_info; + + KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, + kbase_ktrace_get_ctx_refcnt(kctx)); + + /* De-refcount ctx.nr_jobs */ + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); + --(js_kctx_info->ctx.nr_jobs); + dev_dbg(kbdev->dev, + "Remove atom %p from kctx %p; now %d in ctx\n", + (void *)atom, (void *)kctx, js_kctx_info->ctx.nr_jobs); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (--kctx->atoms_count[atom->sched_priority] == 0) + kbase_js_update_ctx_priority(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + unsigned long flags; + struct kbasep_js_atom_retained_state katom_retained_state; + bool attr_state_changed; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(katom != NULL); + + kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); + kbasep_js_remove_job(kbdev, kctx, katom); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* The atom has 'finished' (will not be re-run), so no need to call + * kbasep_js_has_atom_finished(). + * + * This is because it returns false for soft-stopped atoms, but we + * want to override that, because we're cancelling an atom regardless of + * whether it was soft-stopped or not */ + attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, + &katom_retained_state); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return attr_state_changed; +} + +/** + * kbasep_js_run_jobs_after_ctx_and_atom_release - Try running more jobs after + * releasing a context and/or atom + * @kbdev: The kbase_device to operate on + * @kctx: The kbase_context to operate on + * @katom_retained_state: Retained state from the atom + * @runpool_ctx_attr_change: True if the runpool context attributes have changed + * + * This collates a set of actions that must happen whilst hwaccess_lock is held. + * + * This includes running more jobs when: + * - The previously released kctx caused a ctx attribute change, + * - The released atom caused a ctx attribute change, + * - Slots were previously blocked due to affinity restrictions, + * - Submission during IRQ handling failed. + * + * Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were + * changed. The caller should try scheduling all contexts + */ +static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( + struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state, + bool runpool_ctx_attr_change) +{ + struct kbasep_js_device_data *js_devdata; + kbasep_js_release_result result = 0; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(katom_retained_state != NULL); + js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (js_devdata->nr_user_contexts_running != 0 && runpool_ctx_attr_change) { + /* A change in runpool ctx attributes might mean we can + * run more jobs than before */ + result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; + + KBASE_KTRACE_ADD_JM_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, + kctx, NULL, 0u, 0); + } + return result; +} + +/** + * kbasep_js_runpool_release_ctx_internal - Internal function to release the reference + * on a ctx and an atom's "retained state", only + * taking the runpool and as transaction mutexes + * @kbdev: The kbase_device to operate on + * @kctx: The kbase_context to operate on + * @katom_retained_state: Retained state from the atom + * + * This also starts more jobs running in the case of an ctx-attribute state change + * + * This does none of the followup actions for scheduling: + * - It does not schedule in a new context + * - It does not requeue or handle dying contexts + * + * For those tasks, just call kbasep_js_runpool_release_ctx() instead + * + * Has following requirements + * - Context is scheduled in, and kctx->as_nr matches kctx_as_nr + * - Context has a non-zero refcount + * - Caller holds js_kctx_info->ctx.jsctx_mutex + * - Caller holds js_devdata->runpool_mutex + * + * Return: A bitpattern, containing KBASEP_JS_RELEASE_RESULT_* flags, indicating + * the result of releasing a context that whether the caller should try + * scheduling a new context or should try scheduling all contexts. + */ +static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( + struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state) +{ + unsigned long flags; + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + + kbasep_js_release_result release_result = 0u; + bool runpool_ctx_attr_change = false; + int kctx_as_nr; + int new_ref_count; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + kctx_as_nr = kctx->as_nr; + KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); + + /* + * Transaction begins on AS and runpool_irq + * + * Assert about out calling contract + */ + mutex_lock(&kbdev->pm.lock); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr); + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); + + /* Update refcount */ + kbase_ctx_sched_release_ctx(kctx); + new_ref_count = atomic_read(&kctx->refcount); + + /* Release the atom if it finished (i.e. wasn't soft-stopped) */ + if (kbasep_js_has_atom_finished(katom_retained_state)) + runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom( + kbdev, kctx, katom_retained_state); + + if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && +#ifdef CONFIG_MALI_ARBITER_SUPPORT + !kbase_pm_is_gpu_lost(kbdev) && +#endif + !kbase_pm_is_suspending(kbdev)) { + /* Context is kept scheduled into an address space even when + * there are no jobs, in this case we have to handle the + * situation where all jobs have been evicted from the GPU and + * submission is disabled. + * + * At this point we re-enable submission to allow further jobs + * to be executed + */ + kbasep_js_set_submit_allowed(js_devdata, kctx); + } + + /* Make a set of checks to see if the context should be scheduled out. + * Note that there'll always be at least 1 reference to the context + * which was previously acquired by kbasep_js_schedule_ctx(). */ + if (new_ref_count == 1 && + (!kbasep_js_is_submit_allowed(js_devdata, kctx) || +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbase_pm_is_gpu_lost(kbdev) || +#endif + kbase_pm_is_suspending(kbdev))) { + int num_slots = kbdev->gpu_props.num_job_slots; + int slot; + + /* Last reference, and we've been told to remove this context + * from the Run Pool */ + dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d", + kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, + kbasep_js_is_submit_allowed(js_devdata, kctx)); + + KBASE_TLSTREAM_TL_NRET_AS_CTX(kbdev, &kbdev->as[kctx->as_nr], kctx); + + kbase_backend_release_ctx_irq(kbdev, kctx); + + for (slot = 0; slot < num_slots; slot++) { + if (kbdev->hwaccess.active_kctx[slot] == kctx) { + dev_dbg(kbdev->dev, "Marking kctx %p as inactive (s:%d)\n", + (void *)kctx, slot); + kbdev->hwaccess.active_kctx[slot] = NULL; + } + } + + /* Ctx Attribute handling + * + * Releasing atoms attributes must either happen before this, or + * after the KCTX_SHEDULED flag is changed, otherwise we + * double-decount the attributes + */ + runpool_ctx_attr_change |= + kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); + + /* Releasing the context and katom retained state can allow + * more jobs to run */ + release_result |= + kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, + kctx, katom_retained_state, + runpool_ctx_attr_change); + + /* + * Transaction ends on AS and runpool_irq: + * + * By this point, the AS-related data is now clear and ready + * for re-use. + * + * Since releases only occur once for each previous successful + * retain, and no more retains are allowed on this context, no + * other thread will be operating in this + * code whilst we are + */ + + /* Recalculate pullable status for all slots */ + for (slot = 0; slot < num_slots; slot++) { + if (kbase_js_ctx_pullable(kctx, slot, false)) + kbase_js_ctx_list_add_pullable_nolock(kbdev, + kctx, slot); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_backend_release_ctx_noirq(kbdev, kctx); + + mutex_unlock(&kbdev->pm.lock); + + /* Note: Don't reuse kctx_as_nr now */ + + /* Synchronize with any timers */ + kbase_backend_ctx_count_changed(kbdev); + + /* update book-keeping info */ + kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); + /* Signal any waiter that the context is not scheduled, so is + * safe for termination - once the jsctx_mutex is also dropped, + * and jobs have finished. */ + wake_up(&js_kctx_info->ctx.is_scheduled_wait); + + /* Queue an action to occur after we've dropped the lock */ + release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED | + KBASEP_JS_RELEASE_RESULT_SCHED_ALL; + } else { + kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, + katom_retained_state, runpool_ctx_attr_change); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->pm.lock); + } + + return release_result; +} + +void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_atom_retained_state katom_retained_state; + + /* Setup a dummy katom_retained_state */ + kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); + + kbasep_js_runpool_release_ctx_internal(kbdev, kctx, + &katom_retained_state); +} + +void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, bool has_pm_ref) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + /* This is called if and only if you've you've detached the context from + * the Runpool Queue, and not added it back to the Runpool + */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + /* Dying: don't requeue, but kill all jobs on the context. This + * happens asynchronously */ + dev_dbg(kbdev->dev, + "JS: ** Killing Context %p on RunPool Remove **", kctx); + kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); + } +} + +void kbasep_js_runpool_release_ctx_and_katom_retained_state( + struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + kbasep_js_release_result release_result; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, + katom_retained_state); + + /* Drop the runpool mutex to allow requeing kctx */ + mutex_unlock(&js_devdata->runpool_mutex); + + if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); + + /* Drop the jsctx_mutex to allow scheduling in a new context */ + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL) + kbase_js_sched_all(kbdev); +} + +void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_atom_retained_state katom_retained_state; + + kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); + + kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, + &katom_retained_state); +} + +/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into + * kbase_js_sched_all() */ +static void kbasep_js_runpool_release_ctx_no_schedule( + struct kbase_device *kbdev, struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + kbasep_js_release_result release_result; + struct kbasep_js_atom_retained_state katom_retained_state_struct; + struct kbasep_js_atom_retained_state *katom_retained_state = + &katom_retained_state_struct; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + kbasep_js_atom_retained_state_init_invalid(katom_retained_state); + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, + katom_retained_state); + + /* Drop the runpool mutex to allow requeing kctx */ + mutex_unlock(&js_devdata->runpool_mutex); + if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); + + /* Drop the jsctx_mutex to allow scheduling in a new context */ + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + /* NOTE: could return release_result if the caller would like to know + * whether it should schedule a new context, but currently no callers do + */ +} + +void kbase_js_set_timeouts(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbase_backend_timeouts_changed(kbdev); +} + +static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + unsigned long flags; + bool kctx_suspended = false; + int as_nr; + + dev_dbg(kbdev->dev, "Scheduling kctx %p (s:%d)\n", kctx, js); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + /* Pick available address space for this context */ + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + as_nr = kbase_ctx_sched_retain_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + if (as_nr == KBASEP_AS_NR_INVALID) { + as_nr = kbase_backend_find_and_release_free_address_space( + kbdev, kctx); + if (as_nr != KBASEP_AS_NR_INVALID) { + /* Attempt to retain the context again, this should + * succeed */ + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + as_nr = kbase_ctx_sched_retain_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + WARN_ON(as_nr == KBASEP_AS_NR_INVALID); + } + } + if (as_nr == KBASEP_AS_NR_INVALID) + return false; /* No address spaces currently available */ + + /* + * Atomic transaction on the Context and Run Pool begins + */ + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Check to see if context is dying due to kbase_job_zap_context() */ + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + /* Roll back the transaction so far and return */ + kbase_ctx_sched_release_ctx(kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return false; + } + + KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, + 0u, + kbase_ktrace_get_ctx_refcnt(kctx)); + + kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); + + /* Assign context to previously chosen address space */ + if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { + /* Roll back the transaction so far and return */ + kbase_ctx_sched_release_ctx(kctx); + kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return false; + } + + kbdev->hwaccess.active_kctx[js] = kctx; + + KBASE_TLSTREAM_TL_RET_AS_CTX(kbdev, &kbdev->as[kctx->as_nr], kctx); + + /* Cause any future waiter-on-termination to wait until the context is + * descheduled */ + wake_up(&js_kctx_info->ctx.is_scheduled_wait); + + /* Re-check for suspending: a suspend could've occurred, and all the + * contexts could've been removed from the runpool before we took this + * lock. In this case, we don't want to allow this context to run jobs, + * we just want it out immediately. + * + * The DMB required to read the suspend flag was issued recently as part + * of the hwaccess_lock locking. If a suspend occurs *after* that lock + * was taken (i.e. this condition doesn't execute), then the + * kbasep_js_suspend() code will cleanup this context instead (by virtue + * of it being called strictly after the suspend flag is set, and will + * wait for this lock to drop) */ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { +#else + if (kbase_pm_is_suspending(kbdev)) { +#endif + /* Cause it to leave at some later point */ + bool retained; + + retained = kbase_ctx_sched_inc_refcount_nolock(kctx); + KBASE_DEBUG_ASSERT(retained); + + kbasep_js_clear_submit_allowed(js_devdata, kctx); + kctx_suspended = true; + } + + kbase_ctx_flag_clear(kctx, KCTX_PULLED_SINCE_ACTIVE_JS0 << js); + + /* Transaction complete */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + /* Synchronize with any timers */ + kbase_backend_ctx_count_changed(kbdev); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + /* Note: after this point, the context could potentially get scheduled + * out immediately */ + + if (kctx_suspended) { + /* Finishing forcing out the context due to a suspend. Use a + * variant of kbasep_js_runpool_release_ctx() that doesn't + * schedule a new context, to prevent a risk of recursion back + * into this function */ + kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); + return false; + } + return true; +} + +static bool kbase_js_use_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_backend_use_ctx_sched(kbdev, kctx, js)) { + + dev_dbg(kbdev->dev, + "kctx %p already has ASID - mark as active (s:%d)\n", + (void *)kctx, js); + + if (kbdev->hwaccess.active_kctx[js] != kctx) { + kbdev->hwaccess.active_kctx[js] = kctx; + kbase_ctx_flag_clear(kctx, + KCTX_PULLED_SINCE_ACTIVE_JS0 << js); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return true; /* Context already scheduled */ + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return kbasep_js_schedule_ctx(kbdev, kctx, js); +} + +void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_kctx_info *js_kctx_info; + struct kbasep_js_device_data *js_devdata; + bool is_scheduled; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* This should only happen in response to a system call + * from a user-space thread. + * In a non-arbitrated environment this can never happen + * whilst suspending. + * + * In an arbitrated environment, user-space threads can run + * while we are suspended (for example GPU not available + * to this VM), however in that case we will block on + * the wait event for KCTX_SCHEDULED, since no context + * can be scheduled until we have the GPU again. + */ + if (kbdev->arb.arb_if == NULL) + if (WARN_ON(kbase_pm_is_suspending(kbdev))) + return; +#else + /* This should only happen in response to a system call + * from a user-space thread. + * In a non-arbitrated environment this can never happen + * whilst suspending. + */ + if (WARN_ON(kbase_pm_is_suspending(kbdev))) + return; +#endif + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + /* Mark the context as privileged */ + kbase_ctx_flag_set(kctx, KCTX_PRIVILEGED); + + is_scheduled = kbase_ctx_flag(kctx, KCTX_SCHEDULED); + if (!is_scheduled) { + /* Add the context to the pullable list */ + if (kbase_js_ctx_list_add_pullable_head(kbdev, kctx, 0)) + kbase_js_sync_timers(kbdev); + + /* Fast-starting requires the jsctx_mutex to be dropped, + * because it works on multiple ctxs */ + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + /* Try to schedule the context in */ + kbase_js_sched_all(kbdev); + + /* Wait for the context to be scheduled in */ + wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, + kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + } else { + /* Already scheduled in - We need to retain it to keep the + * corresponding address space */ + WARN_ON(!kbase_ctx_sched_inc_refcount(kctx)); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + } +} +KBASE_EXPORT_TEST_API(kbasep_js_schedule_privileged_ctx); + +void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + + /* We don't need to use the address space anymore */ + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + kbase_ctx_flag_clear(kctx, KCTX_PRIVILEGED); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + /* Release the context - it will be scheduled out */ + kbasep_js_runpool_release_ctx(kbdev, kctx); + + kbase_js_sched_all(kbdev); +} +KBASE_EXPORT_TEST_API(kbasep_js_release_privileged_ctx); + +void kbasep_js_suspend(struct kbase_device *kbdev) +{ + unsigned long flags; + struct kbasep_js_device_data *js_devdata; + int i; + u16 retained = 0u; + + KBASE_DEBUG_ASSERT(kbdev); + KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev)); + js_devdata = &kbdev->js_data; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Prevent all contexts from submitting */ + js_devdata->runpool_irq.submit_allowed = 0; + + /* Retain each of the contexts, so we can cause it to leave even if it + * had no refcount to begin with */ + for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { + struct kbase_context *kctx = kbdev->as_to_kctx[i]; + + retained = retained << 1; + + if (kctx && !(kbdev->as_free & (1u << i))) { + kbase_ctx_sched_retain_ctx_refcount(kctx); + retained |= 1u; + /* This loop will not have an effect on the privileged + * contexts as they would have an extra ref count + * compared to the normal contexts, so they will hold + * on to their address spaces. MMU will re-enabled for + * them on resume. + */ + } + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* De-ref the previous retain to ensure each context gets pulled out + * sometime later. */ + for (i = 0; + i < BASE_MAX_NR_AS; + ++i, retained = retained >> 1) { + struct kbase_context *kctx = kbdev->as_to_kctx[i]; + + if (retained & 1u) + kbasep_js_runpool_release_ctx(kbdev, kctx); + } + + /* Caller must wait for all Power Manager active references to be + * dropped */ +} + +void kbasep_js_resume(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata; + int js, prio; + + KBASE_DEBUG_ASSERT(kbdev); + js_devdata = &kbdev->js_data; + KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); + + mutex_lock(&js_devdata->queue_mutex); + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; + prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { + struct kbase_context *kctx, *n; + unsigned long flags; + +#ifndef CONFIG_MALI_ARBITER_SUPPORT + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + list_for_each_entry_safe(kctx, n, + &kbdev->js_data.ctx_list_unpullable[js][prio], + jctx.sched_info.ctx.ctx_list_entry[js]) { + struct kbasep_js_kctx_info *js_kctx_info; + bool timer_sync = false; + + /* Drop lock so we can take kctx mutexes */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_js_ctx_pullable(kctx, js, false)) + timer_sync = + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + /* Take lock before accessing list again */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#else + bool timer_sync = false; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + list_for_each_entry_safe(kctx, n, + &kbdev->js_data.ctx_list_unpullable[js][prio], + jctx.sched_info.ctx.ctx_list_entry[js]) { + + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_js_ctx_pullable(kctx, js, false)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (timer_sync) { + mutex_lock(&js_devdata->runpool_mutex); + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); + } +#endif + } + } + mutex_unlock(&js_devdata->queue_mutex); + + /* Restart atom processing */ + kbase_js_sched_all(kbdev); + + /* JS Resume complete */ +} + +bool kbase_js_is_atom_valid(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + if ((katom->core_req & BASE_JD_REQ_FS) && + (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | + BASE_JD_REQ_T))) + return false; + + if ((katom->core_req & BASE_JD_REQ_JOB_SLOT) && + (katom->jobslot >= BASE_JM_MAX_NR_SLOTS)) + return false; + + return true; +} + +static int kbase_js_get_slot(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + if (katom->core_req & BASE_JD_REQ_JOB_SLOT) + return katom->jobslot; + + if (katom->core_req & BASE_JD_REQ_FS) + return 0; + + if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { + if (katom->device_nr == 1 && + kbdev->gpu_props.num_core_groups == 2) + return 2; + } + + return 1; +} + +bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + bool enqueue_required, add_required = true; + + katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + lockdep_assert_held(&kctx->jctx.lock); + + /* If slot will transition from unpullable to pullable then add to + * pullable list */ + if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) { + enqueue_required = true; + } else { + enqueue_required = false; + } + + if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || + (katom->pre_dep && (katom->pre_dep->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; + + dev_dbg(kctx->kbdev->dev, "Add atom %p to X_DEP list (s:%d)\n", + (void *)katom, js); + + list_add_tail(&katom->queue, &queue->x_dep_head); + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; + if (kbase_js_atom_blocked_on_x_dep(katom)) { + enqueue_required = false; + add_required = false; + } + } else { + dev_dbg(kctx->kbdev->dev, "Atom %p not added to X_DEP list\n", + (void *)katom); + } + + if (add_required) { + /* Check if there are lower priority jobs to soft stop */ + kbase_job_slot_ctx_priority_check_locked(kctx, katom); + + /* Add atom to ring buffer. */ + jsctx_tree_add(kctx, katom); + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; + } + + dev_dbg(kctx->kbdev->dev, + "Enqueue of kctx %p is %srequired to submit atom %p\n", + kctx, enqueue_required ? "" : "not ", katom); + + return enqueue_required; +} + +/** + * kbase_js_move_to_tree - Move atom (and any dependent atoms) to the + * runnable_tree, ready for execution + * @katom: Atom to submit + * + * It is assumed that @katom does not have KBASE_KATOM_FLAG_X_DEP_BLOCKED set, + * but is still present in the x_dep list. If @katom has a same-slot dependent + * atom then that atom (and any dependents) will also be moved. + */ +static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) +{ + struct kbase_context *const kctx = katom->kctx; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + while (katom) { + WARN_ON(!(katom->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); + + if (!kbase_js_atom_blocked_on_x_dep(katom)) { + dev_dbg(kctx->kbdev->dev, + "Del atom %p from X_DEP list in js_move_to_tree\n", + (void *)katom); + + list_del(&katom->queue); + katom->atom_flags &= + ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; + /* For incremental rendering, an end-of-renderpass atom + * may have had its dependency on start-of-renderpass + * ignored and may therefore already be in the tree. + */ + if (!(katom->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { + jsctx_tree_add(kctx, katom); + katom->atom_flags |= + KBASE_KATOM_FLAG_JSCTX_IN_TREE; + } + } else { + dev_dbg(kctx->kbdev->dev, + "Atom %p blocked on x-dep in js_move_to_tree\n", + (void *)katom); + break; + } + + katom = katom->post_dep; + } +} + + +/** + * kbase_js_evict_deps - Evict dependencies of a failed atom. + * @kctx: Context pointer + * @katom: Pointer to the atom that has failed. + * @js: The job slot the katom was run on. + * @prio: Priority of the katom. + * + * Remove all post dependencies of an atom from the context ringbuffers. + * + * The original atom's event_code will be propogated to all dependent atoms. + * + * Context: Caller must hold the HW access lock + */ +static void kbase_js_evict_deps(struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js, int prio) +{ + struct kbase_jd_atom *x_dep = katom->x_post_dep; + struct kbase_jd_atom *next_katom = katom->post_dep; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + if (next_katom) { + KBASE_DEBUG_ASSERT(next_katom->status != + KBASE_JD_ATOM_STATE_HW_COMPLETED); + next_katom->will_fail_event_code = katom->event_code; + + } + + /* Has cross slot depenency. */ + if (x_dep && (x_dep->atom_flags & (KBASE_KATOM_FLAG_JSCTX_IN_TREE | + KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { + /* Remove dependency.*/ + x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; + trace_sysgraph(SGR_DEP_RES, kctx->id, + kbase_jd_atom_id(kctx, x_dep)); + + dev_dbg(kctx->kbdev->dev, "Cleared X_DEP flag on atom %p\n", + (void *)x_dep); + + /* Fail if it had a data dependency. */ + if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) { + x_dep->will_fail_event_code = katom->event_code; + } + if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) + kbase_js_move_to_tree(x_dep); + } +} + +struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) +{ + struct kbase_jd_atom *katom; + struct kbasep_js_device_data *js_devdata; + struct kbase_device *kbdev; + int pulled; + + KBASE_DEBUG_ASSERT(kctx); + + kbdev = kctx->kbdev; + dev_dbg(kbdev->dev, "JS: pulling an atom from kctx %p (s:%d)\n", + (void *)kctx, js); + + js_devdata = &kbdev->js_data; + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) { + dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %p\n", + (void *)kctx); + return NULL; + } +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) +#else + if (kbase_pm_is_suspending(kbdev)) +#endif + return NULL; + + katom = jsctx_rb_peek(kctx, js); + if (!katom) { + dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %p (s:%d)\n", + (void *)kctx, js); + return NULL; + } + if (kctx->blocked_js[js][katom->sched_priority]) { + dev_dbg(kbdev->dev, + "JS: kctx %p is blocked from submitting atoms at priority %d (s:%d)\n", + (void *)kctx, katom->sched_priority, js); + return NULL; + } + if (atomic_read(&katom->blocked)) { + dev_dbg(kbdev->dev, "JS: Atom %p is blocked in js_pull\n", + (void *)katom); + return NULL; + } + + /* Due to ordering restrictions when unpulling atoms on failure, we do + * not allow multiple runs of fail-dep atoms from the same context to be + * present on the same slot */ + if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { + struct kbase_jd_atom *prev_atom = + kbase_backend_inspect_tail(kbdev, js); + + if (prev_atom && prev_atom->kctx != kctx) + return NULL; + } + + if (kbase_js_atom_blocked_on_x_dep(katom)) { + if (katom->x_pre_dep->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || + katom->x_pre_dep->will_fail_event_code) { + dev_dbg(kbdev->dev, + "JS: X pre-dep %p is not present in slot FIFO or will fail\n", + (void *)katom->x_pre_dep); + return NULL; + } + if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && + kbase_backend_nr_atoms_on_slot(kbdev, js)) { + dev_dbg(kbdev->dev, + "JS: Atom %p has cross-slot fail dependency and atoms on slot (s:%d)\n", + (void *)katom, js); + return NULL; + } + } + + kbase_ctx_flag_set(kctx, KCTX_PULLED); + kbase_ctx_flag_set(kctx, (KCTX_PULLED_SINCE_ACTIVE_JS0 << js)); + + pulled = atomic_inc_return(&kctx->atoms_pulled); + if (pulled == 1 && !kctx->slots_pullable) { + WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); + atomic_inc(&kbdev->js_data.nr_contexts_runnable); + } + atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]); + kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++; + jsctx_rb_pull(kctx, katom); + + kbase_ctx_sched_retain_ctx_refcount(kctx); + + katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF; + + katom->ticks = 0; + + dev_dbg(kbdev->dev, "JS: successfully pulled atom %p from kctx %p (s:%d)\n", + (void *)katom, (void *)kctx, js); + + return katom; +} + +/** + * js_return_of_start_rp() - Handle soft-stop of an atom that starts a + * renderpass + * @start_katom: Pointer to the start-of-renderpass atom that was soft-stopped + * + * This function is called to switch to incremental rendering if the tiler job + * chain at the start of a renderpass has used too much memory. It prevents the + * tiler job being pulled for execution in the job scheduler again until the + * next phase of incremental rendering is complete. + * + * If the end-of-renderpass atom is already in the job scheduler (because a + * previous attempt at tiling used too much memory during the same renderpass) + * then it is unblocked; otherwise, it is run by handing it to the scheduler. + */ +static void js_return_of_start_rp(struct kbase_jd_atom *const start_katom) +{ + struct kbase_context *const kctx = start_katom->kctx; + struct kbase_device *const kbdev = kctx->kbdev; + struct kbase_jd_renderpass *rp; + struct kbase_jd_atom *end_katom; + unsigned long flags; + + lockdep_assert_held(&kctx->jctx.lock); + + if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) + return; + + compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + + rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; + + if (WARN_ON(rp->start_katom != start_katom)) + return; + + dev_dbg(kctx->kbdev->dev, + "JS return start atom %p in state %d of RP %d\n", + (void *)start_katom, (int)rp->state, + start_katom->renderpass_id); + + if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) + return; + + /* The tiler job might have been soft-stopped for some reason other + * than running out of memory. + */ + if (rp->state == KBASE_JD_RP_START || rp->state == KBASE_JD_RP_RETRY) { + dev_dbg(kctx->kbdev->dev, + "JS return isn't OOM in state %d of RP %d\n", + (int)rp->state, start_katom->renderpass_id); + return; + } + + dev_dbg(kctx->kbdev->dev, + "JS return confirm OOM in state %d of RP %d\n", + (int)rp->state, start_katom->renderpass_id); + + if (WARN_ON(rp->state != KBASE_JD_RP_PEND_OOM && + rp->state != KBASE_JD_RP_RETRY_PEND_OOM)) + return; + + /* Prevent the tiler job being pulled for execution in the + * job scheduler again. + */ + dev_dbg(kbdev->dev, "Blocking start atom %p\n", + (void *)start_katom); + atomic_inc(&start_katom->blocked); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + rp->state = (rp->state == KBASE_JD_RP_PEND_OOM) ? + KBASE_JD_RP_OOM : KBASE_JD_RP_RETRY_OOM; + + /* Was the fragment job chain submitted to kbase yet? */ + end_katom = rp->end_katom; + if (end_katom) { + dev_dbg(kctx->kbdev->dev, "JS return add end atom %p\n", + (void *)end_katom); + + if (rp->state == KBASE_JD_RP_RETRY_OOM) { + /* Allow the end of the renderpass to be pulled for + * execution again to continue incremental rendering. + */ + dev_dbg(kbdev->dev, "Unblocking end atom %p\n", + (void *)end_katom); + atomic_dec(&end_katom->blocked); + WARN_ON(!(end_katom->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_TREE)); + WARN_ON(end_katom->status != KBASE_JD_ATOM_STATE_IN_JS); + + kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, + end_katom->slot_nr); + + /* Expect the fragment job chain to be scheduled without + * further action because this function is called when + * returning an atom to the job scheduler ringbuffer. + */ + end_katom = NULL; + } else { + WARN_ON(end_katom->status != + KBASE_JD_ATOM_STATE_QUEUED && + end_katom->status != KBASE_JD_ATOM_STATE_IN_JS); + } + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (end_katom) + kbase_jd_dep_clear_locked(end_katom); +} + +/** + * js_return_of_end_rp() - Handle completion of an atom that ends a renderpass + * @end_katom: Pointer to the end-of-renderpass atom that was completed + * + * This function is called to continue incremental rendering if the tiler job + * chain at the start of a renderpass used too much memory. It resets the + * mechanism for detecting excessive memory usage then allows the soft-stopped + * tiler job chain to be pulled for execution again. + * + * The start-of-renderpass atom must already been submitted to kbase. + */ +static void js_return_of_end_rp(struct kbase_jd_atom *const end_katom) +{ + struct kbase_context *const kctx = end_katom->kctx; + struct kbase_device *const kbdev = kctx->kbdev; + struct kbase_jd_renderpass *rp; + struct kbase_jd_atom *start_katom; + unsigned long flags; + + lockdep_assert_held(&kctx->jctx.lock); + + if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) + return; + + compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + + rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; + + if (WARN_ON(rp->end_katom != end_katom)) + return; + + dev_dbg(kctx->kbdev->dev, + "JS return end atom %p in state %d of RP %d\n", + (void *)end_katom, (int)rp->state, end_katom->renderpass_id); + + if (WARN_ON(rp->state != KBASE_JD_RP_OOM && + rp->state != KBASE_JD_RP_RETRY_OOM)) + return; + + /* Reduce the number of mapped pages in the memory regions that + * triggered out-of-memory last time so that we can detect excessive + * memory usage again. + */ + kbase_gpu_vm_lock(kctx); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + while (!list_empty(&rp->oom_reg_list)) { + struct kbase_va_region *reg = + list_first_entry(&rp->oom_reg_list, + struct kbase_va_region, link); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + dev_dbg(kbdev->dev, + "Reset backing to %zu pages for region %p\n", + reg->threshold_pages, (void *)reg); + + if (!WARN_ON(reg->flags & KBASE_REG_VA_FREED)) + kbase_mem_shrink(kctx, reg, reg->threshold_pages); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + dev_dbg(kbdev->dev, "Deleting region %p from list\n", + (void *)reg); + list_del_init(®->link); + kbase_va_region_alloc_put(kctx, reg); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_gpu_vm_unlock(kctx); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + rp->state = KBASE_JD_RP_RETRY; + dev_dbg(kbdev->dev, "Changed state to %d for retry\n", rp->state); + + /* Allow the start of the renderpass to be pulled for execution again + * to begin/continue incremental rendering. + */ + start_katom = rp->start_katom; + if (!WARN_ON(!start_katom)) { + dev_dbg(kbdev->dev, "Unblocking start atom %p\n", + (void *)start_katom); + atomic_dec(&start_katom->blocked); + (void)kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, + start_katom->slot_nr); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +static void js_return_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; + struct kbasep_js_atom_retained_state retained_state; + int js = katom->slot_nr; + int prio = katom->sched_priority; + bool timer_sync = false; + bool context_idle = false; + unsigned long flags; + base_jd_core_req core_req = katom->core_req; + + dev_dbg(kbdev->dev, "%s for atom %p with event code 0x%x\n", + __func__, (void *)katom, katom->event_code); + + if (katom->event_code != BASE_JD_EVENT_END_RP_DONE) + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(kbdev, katom); + + kbase_backend_complete_wq(kbdev, katom); + + kbasep_js_atom_retained_state_copy(&retained_state, katom); + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + atomic_dec(&kctx->atoms_pulled); + atomic_dec(&kctx->atoms_pulled_slot[js]); + + if (katom->event_code != BASE_JD_EVENT_END_RP_DONE) + atomic_dec(&katom->blocked); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kctx->atoms_pulled_slot_pri[js][katom->sched_priority]--; + + if (!atomic_read(&kctx->atoms_pulled_slot[js]) && + jsctx_rb_none_to_pull(kctx, js)) + timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); + + /* If this slot has been blocked due to soft-stopped atoms, and all + * atoms have now been processed, then unblock the slot */ + if (!kctx->atoms_pulled_slot_pri[js][prio] && + kctx->blocked_js[js][prio]) { + kctx->blocked_js[js][prio] = false; + + /* Only mark the slot as pullable if the context is not idle - + * that case is handled below */ + if (atomic_read(&kctx->atoms_pulled) && + kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + + if (!atomic_read(&kctx->atoms_pulled)) { + dev_dbg(kbdev->dev, + "No atoms currently pulled from context %p\n", + (void *)kctx); + + if (!kctx->slots_pullable) { + dev_dbg(kbdev->dev, + "Context %p %s counted as runnable\n", + (void *)kctx, + kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF) ? + "is" : "isn't"); + + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + timer_sync = true; + } + + if (kctx->as_nr != KBASEP_AS_NR_INVALID && + !kbase_ctx_flag(kctx, KCTX_DYING)) { + int num_slots = kbdev->gpu_props.num_job_slots; + int slot; + + if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) + kbasep_js_set_submit_allowed(js_devdata, kctx); + + for (slot = 0; slot < num_slots; slot++) { + if (kbase_js_ctx_pullable(kctx, slot, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, slot); + } + } + + kbase_jm_idle_ctx(kbdev, kctx); + + context_idle = true; + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (context_idle) { + dev_dbg(kbdev->dev, + "Context %p %s counted as active\n", + (void *)kctx, + kbase_ctx_flag(kctx, KCTX_ACTIVE) ? + "is" : "isn't"); + WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + kbase_pm_context_idle(kbdev); + } + + if (timer_sync) + kbase_js_sync_timers(kbdev); + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + if (katom->core_req & BASE_JD_REQ_START_RENDERPASS) { + mutex_lock(&kctx->jctx.lock); + js_return_of_start_rp(katom); + mutex_unlock(&kctx->jctx.lock); + } else if (katom->event_code == BASE_JD_EVENT_END_RP_DONE) { + mutex_lock(&kctx->jctx.lock); + js_return_of_end_rp(katom); + mutex_unlock(&kctx->jctx.lock); + } + + katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; + dev_dbg(kbdev->dev, "JS: retained state %s finished", + kbasep_js_has_atom_finished(&retained_state) ? + "has" : "hasn't"); + + WARN_ON(kbasep_js_has_atom_finished(&retained_state)); + + kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, + &retained_state); + + kbase_js_sched_all(kbdev); + + kbase_backend_complete_wq_post_sched(kbdev, core_req); + + dev_dbg(kbdev->dev, "Leaving %s for atom %p\n", + __func__, (void *)katom); +} + +void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + dev_dbg(kctx->kbdev->dev, "Unpulling atom %p in kctx %p\n", + (void *)katom, (void *)kctx); + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + jsctx_rb_unpull(kctx, katom); + + WARN_ON(work_pending(&katom->work)); + + /* Block re-submission until workqueue has run */ + atomic_inc(&katom->blocked); + + kbase_job_check_leave_disjoint(kctx->kbdev, katom); + + INIT_WORK(&katom->work, js_return_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + +/** + * js_complete_start_rp() - Handle completion of atom that starts a renderpass + * @kctx: Context pointer + * @start_katom: Pointer to the atom that completed + * + * Put any references to virtual memory regions that might have been added by + * kbase_job_slot_softstop_start_rp() because the tiler job chain completed + * despite any pending soft-stop request. + * + * If the atom that just completed was soft-stopped during a previous attempt to + * run it then there should be a blocked end-of-renderpass atom waiting for it, + * which we must unblock to process the output of the tiler job chain. + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool js_complete_start_rp(struct kbase_context *kctx, + struct kbase_jd_atom *const start_katom) +{ + struct kbase_device *const kbdev = kctx->kbdev; + struct kbase_jd_renderpass *rp; + bool timer_sync = false; + + lockdep_assert_held(&kctx->jctx.lock); + + if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) + return false; + + compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + + rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; + + if (WARN_ON(rp->start_katom != start_katom)) + return false; + + dev_dbg(kctx->kbdev->dev, + "Start atom %p is done in state %d of RP %d\n", + (void *)start_katom, (int)rp->state, + start_katom->renderpass_id); + + if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) + return false; + + if (rp->state == KBASE_JD_RP_PEND_OOM || + rp->state == KBASE_JD_RP_RETRY_PEND_OOM) { + unsigned long flags; + + dev_dbg(kctx->kbdev->dev, + "Start atom %p completed before soft-stop\n", + (void *)start_katom); + + kbase_gpu_vm_lock(kctx); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + while (!list_empty(&rp->oom_reg_list)) { + struct kbase_va_region *reg = + list_first_entry(&rp->oom_reg_list, + struct kbase_va_region, link); + + WARN_ON(reg->flags & KBASE_REG_VA_FREED); + dev_dbg(kctx->kbdev->dev, "Deleting region %p from list\n", + (void *)reg); + list_del_init(®->link); + kbase_va_region_alloc_put(kctx, reg); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_gpu_vm_unlock(kctx); + } else { + dev_dbg(kctx->kbdev->dev, + "Start atom %p did not exceed memory threshold\n", + (void *)start_katom); + + WARN_ON(rp->state != KBASE_JD_RP_START && + rp->state != KBASE_JD_RP_RETRY); + } + + if (rp->state == KBASE_JD_RP_RETRY || + rp->state == KBASE_JD_RP_RETRY_PEND_OOM) { + struct kbase_jd_atom *const end_katom = rp->end_katom; + + if (!WARN_ON(!end_katom)) { + unsigned long flags; + + /* Allow the end of the renderpass to be pulled for + * execution again to continue incremental rendering. + */ + dev_dbg(kbdev->dev, "Unblocking end atom %p!\n", + (void *)end_katom); + atomic_dec(&end_katom->blocked); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + timer_sync = kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, end_katom->slot_nr); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + } + + return timer_sync; +} + +/** + * js_complete_end_rp() - Handle final completion of atom that ends a renderpass + * @kctx: Context pointer + * @end_katom: Pointer to the atom that completed for the last time + * + * This function must only be called if the renderpass actually completed + * without the tiler job chain at the start using too much memory; otherwise + * completion of the end-of-renderpass atom is handled similarly to a soft-stop. + */ +static void js_complete_end_rp(struct kbase_context *kctx, + struct kbase_jd_atom *const end_katom) +{ + struct kbase_device *const kbdev = kctx->kbdev; + unsigned long flags; + struct kbase_jd_renderpass *rp; + + lockdep_assert_held(&kctx->jctx.lock); + + if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) + return; + + compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + + rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; + + if (WARN_ON(rp->end_katom != end_katom)) + return; + + dev_dbg(kbdev->dev, "End atom %p is done in state %d of RP %d\n", + (void *)end_katom, (int)rp->state, end_katom->renderpass_id); + + if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE) || + WARN_ON(rp->state == KBASE_JD_RP_OOM) || + WARN_ON(rp->state == KBASE_JD_RP_RETRY_OOM)) + return; + + /* Rendering completed without running out of memory. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + WARN_ON(!list_empty(&rp->oom_reg_list)); + rp->state = KBASE_JD_RP_COMPLETE; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + dev_dbg(kbdev->dev, "Renderpass %d is complete\n", + end_katom->renderpass_id); +} + +bool kbase_js_complete_atom_wq(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + struct kbasep_js_kctx_info *js_kctx_info; + struct kbasep_js_device_data *js_devdata; + struct kbase_device *kbdev; + unsigned long flags; + bool timer_sync = false; + int atom_slot; + bool context_idle = false; + int prio = katom->sched_priority; + + kbdev = kctx->kbdev; + atom_slot = katom->slot_nr; + + dev_dbg(kbdev->dev, "%s for atom %p (s:%d)\n", + __func__, (void *)katom, atom_slot); + + /* Update the incremental rendering state machine. + */ + if (katom->core_req & BASE_JD_REQ_START_RENDERPASS) + timer_sync |= js_complete_start_rp(kctx, katom); + else if (katom->core_req & BASE_JD_REQ_END_RENDERPASS) + js_complete_end_rp(kctx, katom); + + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { + dev_dbg(kbdev->dev, "Atom %p is in runnable_tree\n", + (void *)katom); + + context_idle = !atomic_dec_return(&kctx->atoms_pulled); + atomic_dec(&kctx->atoms_pulled_slot[atom_slot]); + kctx->atoms_pulled_slot_pri[atom_slot][prio]--; + + if (!atomic_read(&kctx->atoms_pulled) && + !kctx->slots_pullable) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + timer_sync = true; + } + + /* If this slot has been blocked due to soft-stopped atoms, and + * all atoms have now been processed, then unblock the slot */ + if (!kctx->atoms_pulled_slot_pri[atom_slot][prio] + && kctx->blocked_js[atom_slot][prio]) { + dev_dbg(kbdev->dev, + "kctx %p is no longer blocked from submitting on slot %d at priority %d\n", + (void *)kctx, atom_slot, prio); + + kctx->blocked_js[atom_slot][prio] = false; + if (kbase_js_ctx_pullable(kctx, atom_slot, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, atom_slot); + } + } + WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); + + if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) && + jsctx_rb_none_to_pull(kctx, atom_slot)) { + if (!list_empty( + &kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot])) + timer_sync |= kbase_js_ctx_list_remove_nolock( + kctx->kbdev, kctx, atom_slot); + } + + /* + * If submission is disabled on this context (most likely due to an + * atom failure) and there are now no atoms left in the system then + * re-enable submission so that context can be scheduled again. + */ + if (!kbasep_js_is_submit_allowed(js_devdata, kctx) && + !atomic_read(&kctx->atoms_pulled) && + !kbase_ctx_flag(kctx, KCTX_DYING)) { + int js; + + kbasep_js_set_submit_allowed(js_devdata, kctx); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + if (kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + } else if (katom->x_post_dep && + kbasep_js_is_submit_allowed(js_devdata, kctx)) { + int js; + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + if (kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + } + + /* Mark context as inactive. The pm reference will be dropped later in + * jd_done_worker(). + */ + if (context_idle) { + dev_dbg(kbdev->dev, "kctx %p is no longer active\n", + (void *)kctx); + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); + + dev_dbg(kbdev->dev, "Leaving %s\n", __func__); + return context_idle; +} + +/** + * js_end_rp_is_complete() - Check whether an atom that ends a renderpass has + * completed for the last time. + * + * @end_katom: Pointer to the atom that completed on the hardware. + * + * An atom that ends a renderpass may be run on the hardware several times + * before notifying userspace or allowing dependent atoms to be executed. + * + * This function is used to decide whether or not to allow end-of-renderpass + * atom completion. It only returns false if the atom at the start of the + * renderpass was soft-stopped because it used too much memory during the most + * recent attempt at tiling. + * + * Return: True if the atom completed for the last time. + */ +static bool js_end_rp_is_complete(struct kbase_jd_atom *const end_katom) +{ + struct kbase_context *const kctx = end_katom->kctx; + struct kbase_device *const kbdev = kctx->kbdev; + struct kbase_jd_renderpass *rp; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) + return true; + + compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + + rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; + + if (WARN_ON(rp->end_katom != end_katom)) + return true; + + dev_dbg(kbdev->dev, + "JS complete end atom %p in state %d of RP %d\n", + (void *)end_katom, (int)rp->state, + end_katom->renderpass_id); + + if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) + return true; + + /* Failure of end-of-renderpass atoms must not return to the + * start of the renderpass. + */ + if (end_katom->event_code != BASE_JD_EVENT_DONE) + return true; + + if (rp->state != KBASE_JD_RP_OOM && + rp->state != KBASE_JD_RP_RETRY_OOM) + return true; + + dev_dbg(kbdev->dev, "Suppressing end atom completion\n"); + return false; +} + +struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, + ktime_t *end_timestamp) +{ + struct kbase_device *kbdev; + struct kbase_context *kctx = katom->kctx; + struct kbase_jd_atom *x_dep = katom->x_post_dep; + + kbdev = kctx->kbdev; + dev_dbg(kbdev->dev, "Atom %p complete in kctx %p (post-dep %p)\n", + (void *)katom, (void *)kctx, (void *)x_dep); + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + if ((katom->core_req & BASE_JD_REQ_END_RENDERPASS) && + !js_end_rp_is_complete(katom)) { + katom->event_code = BASE_JD_EVENT_END_RP_DONE; + kbase_js_unpull(kctx, katom); + return NULL; + } + + if (katom->will_fail_event_code) + katom->event_code = katom->will_fail_event_code; + + katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; + dev_dbg(kbdev->dev, "Atom %p status to HW completed\n", (void *)katom); + + if (katom->event_code != BASE_JD_EVENT_DONE) { + kbase_js_evict_deps(kctx, katom, katom->slot_nr, + katom->sched_priority); + } + + KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, NULL, + katom->slot_nr, 0, TL_JS_EVENT_STOP); + + trace_sysgraph_gpu(SGR_COMPLETE, kctx->id, + kbase_jd_atom_id(katom->kctx, katom), katom->slot_nr); + + kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); + + /* Unblock cross dependency if present */ + if (x_dep && (katom->event_code == BASE_JD_EVENT_DONE || + !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) && + (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)) { + bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, + false); + x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; + trace_sysgraph(SGR_DEP_RES, kctx->id, + kbase_jd_atom_id(katom->kctx, x_dep)); + dev_dbg(kbdev->dev, "Cleared X_DEP flag on atom %p\n", + (void *)x_dep); + + kbase_js_move_to_tree(x_dep); + + if (!was_pullable && kbase_js_ctx_pullable(kctx, x_dep->slot_nr, + false)) + kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, + x_dep->slot_nr); + + if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { + dev_dbg(kbdev->dev, "Atom %p is in runnable tree\n", + (void *)x_dep); + return x_dep; + } + } else { + dev_dbg(kbdev->dev, + "No cross-slot dep to unblock for atom %p\n", + (void *)katom); + } + + return NULL; +} + +/** + * kbase_js_atom_blocked_on_x_dep - Decide whether to ignore a cross-slot + * dependency + * @katom: Pointer to an atom in the slot ringbuffer + * + * A cross-slot dependency is ignored if necessary to unblock incremental + * rendering. If the atom at the start of a renderpass used too much memory + * and was soft-stopped then the atom at the end of a renderpass is submitted + * to hardware regardless of its dependency on the start-of-renderpass atom. + * This can happen multiple times for the same pair of atoms. + * + * Return: true to block the atom or false to allow it to be submitted to + * hardware + */ +bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *const katom) +{ + struct kbase_context *const kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct kbase_jd_renderpass *rp; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!(katom->atom_flags & + KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { + dev_dbg(kbdev->dev, "Atom %p is not blocked on a cross-slot dependency", + (void *)katom); + return false; + } + + if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS)) { + dev_dbg(kbdev->dev, "Atom %p is blocked on a cross-slot dependency", + (void *)katom); + return true; + } + + compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= + ARRAY_SIZE(kctx->jctx.renderpasses), + "Should check invalid access to renderpasses"); + + rp = &kctx->jctx.renderpasses[katom->renderpass_id]; + /* We can read a subset of renderpass state without holding + * higher-level locks (but not end_katom, for example). + */ + + WARN_ON(rp->state == KBASE_JD_RP_COMPLETE); + + dev_dbg(kbdev->dev, "End atom has cross-slot dep in state %d\n", + (int)rp->state); + + if (rp->state != KBASE_JD_RP_OOM && rp->state != KBASE_JD_RP_RETRY_OOM) + return true; + + /* Tiler ran out of memory so allow the fragment job chain to run + * if it only depends on the tiler job chain. + */ + if (katom->x_pre_dep != rp->start_katom) { + dev_dbg(kbdev->dev, "Dependency is on %p not start atom %p\n", + (void *)katom->x_pre_dep, (void *)rp->start_katom); + return true; + } + + dev_dbg(kbdev->dev, "Ignoring cross-slot dep on atom %p\n", + (void *)katom->x_pre_dep); + + return false; +} + +void kbase_js_sched(struct kbase_device *kbdev, int js_mask) +{ + struct kbasep_js_device_data *js_devdata; + struct kbase_context *last_active[BASE_JM_MAX_NR_SLOTS]; + bool timer_sync = false; + bool ctx_waiting[BASE_JM_MAX_NR_SLOTS]; + int js; + + dev_dbg(kbdev->dev, "%s kbdev %p mask 0x%x\n", + __func__, (void *)kbdev, (unsigned int)js_mask); + + js_devdata = &kbdev->js_data; + + down(&js_devdata->schedule_sem); + mutex_lock(&js_devdata->queue_mutex); + + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + last_active[js] = kbdev->hwaccess.active_kctx[js]; + ctx_waiting[js] = false; + } + + while (js_mask) { + js = ffs(js_mask) - 1; + + while (1) { + struct kbase_context *kctx; + unsigned long flags; + bool context_idle = false; + + kctx = kbase_js_ctx_list_pop_head(kbdev, js); + + if (!kctx) { + js_mask &= ~(1 << js); + dev_dbg(kbdev->dev, + "No kctx on pullable list (s:%d)\n", + js); + break; + } + + if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) { + context_idle = true; + + dev_dbg(kbdev->dev, + "kctx %p is not active (s:%d)\n", + (void *)kctx, js); + + if (kbase_pm_context_active_handle_suspend( + kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { + dev_dbg(kbdev->dev, + "Suspend pending (s:%d)\n", js); + /* Suspend pending - return context to + * queue and stop scheduling */ + mutex_lock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + if (kbase_js_ctx_list_add_pullable_head( + kctx->kbdev, kctx, js)) + kbase_js_sync_timers(kbdev); + mutex_unlock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + up(&js_devdata->schedule_sem); + return; + } + kbase_ctx_flag_set(kctx, KCTX_ACTIVE); + } + + if (!kbase_js_use_ctx(kbdev, kctx, js)) { + mutex_lock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + + dev_dbg(kbdev->dev, + "kctx %p cannot be used at this time\n", + kctx); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (kbase_js_ctx_pullable(kctx, js, false) + || kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) + timer_sync |= + kbase_js_ctx_list_add_pullable_head_nolock( + kctx->kbdev, kctx, js); + else + timer_sync |= + kbase_js_ctx_list_add_unpullable_nolock( + kctx->kbdev, kctx, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + mutex_unlock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + if (context_idle) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + kbase_pm_context_idle(kbdev); + } + + /* No more jobs can be submitted on this slot */ + js_mask &= ~(1 << js); + break; + } + mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_ctx_flag_clear(kctx, KCTX_PULLED); + + if (!kbase_jm_kick(kbdev, 1 << js)) { + dev_dbg(kbdev->dev, + "No more jobs can be submitted (s:%d)\n", + js); + js_mask &= ~(1 << js); + } + if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { + bool pullable; + + dev_dbg(kbdev->dev, + "No atoms pulled from kctx %p (s:%d)\n", + (void *)kctx, js); + + pullable = kbase_js_ctx_pullable(kctx, js, + true); + + /* Failed to pull jobs - push to head of list. + * Unless this context is already 'active', in + * which case it's effectively already scheduled + * so push it to the back of the list. */ + if (pullable && kctx == last_active[js] && + kbase_ctx_flag(kctx, + (KCTX_PULLED_SINCE_ACTIVE_JS0 << + js))) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kctx->kbdev, + kctx, js); + else if (pullable) + timer_sync |= + kbase_js_ctx_list_add_pullable_head_nolock( + kctx->kbdev, + kctx, js); + else + timer_sync |= + kbase_js_ctx_list_add_unpullable_nolock( + kctx->kbdev, + kctx, js); + + /* If this context is not the active context, + * but the active context is pullable on this + * slot, then we need to remove the active + * marker to prevent it from submitting atoms in + * the IRQ handler, which would prevent this + * context from making progress. */ + if (last_active[js] && kctx != last_active[js] + && kbase_js_ctx_pullable( + last_active[js], js, true)) + ctx_waiting[js] = true; + + if (context_idle) { + kbase_jm_idle_ctx(kbdev, kctx); + spin_unlock_irqrestore( + &kbdev->hwaccess_lock, + flags); + WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + kbase_pm_context_idle(kbdev); + } else { + spin_unlock_irqrestore( + &kbdev->hwaccess_lock, + flags); + } + mutex_unlock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + + js_mask &= ~(1 << js); + break; /* Could not run atoms on this slot */ + } + + dev_dbg(kbdev->dev, "Push kctx %p to back of list\n", + (void *)kctx); + if (kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kctx->kbdev, kctx, js); + else + timer_sync |= + kbase_js_ctx_list_add_unpullable_nolock( + kctx->kbdev, kctx, js); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + } + } + + if (timer_sync) + kbase_js_sync_timers(kbdev); + + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + if (kbdev->hwaccess.active_kctx[js] == last_active[js] && + ctx_waiting[js]) { + dev_dbg(kbdev->dev, "Marking kctx %p as inactive (s:%d)\n", + (void *)last_active[js], js); + kbdev->hwaccess.active_kctx[js] = NULL; + } + } + + mutex_unlock(&js_devdata->queue_mutex); + up(&js_devdata->schedule_sem); +} + +void kbase_js_zap_context(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; + + /* + * Critical assumption: No more submission is possible outside of the + * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs) + * whilst the struct kbase_context is terminating. + */ + + /* First, atomically do the following: + * - mark the context as dying + * - try to evict it from the queue */ + mutex_lock(&kctx->jctx.lock); + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + kbase_ctx_flag_set(kctx, KCTX_DYING); + + dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx); + + /* + * At this point we know: + * - If eviction succeeded, it was in the queue, but now no + * longer is + * - We must cancel the jobs here. No Power Manager active reference to + * release. + * - This happens asynchronously - kbase_jd_zap_context() will wait for + * those jobs to be killed. + * - If eviction failed, then it wasn't in the queue. It is one + * of the following: + * - a. it didn't have any jobs, and so is not in the Queue or + * the Run Pool (not scheduled) + * - Hence, no more work required to cancel jobs. No Power Manager + * active reference to release. + * - b. it was in the middle of a scheduling transaction (and thus must + * have at least 1 job). This can happen from a syscall or a + * kernel thread. We still hold the jsctx_mutex, and so the thread + * must be waiting inside kbasep_js_try_schedule_head_ctx(), + * before checking whether the runpool is full. That thread will + * continue after we drop the mutex, and will notice the context + * is dying. It will rollback the transaction, killing all jobs at + * the same time. kbase_jd_zap_context() will wait for those jobs + * to be killed. + * - Hence, no more work required to cancel jobs, or to release the + * Power Manager active reference. + * - c. it is scheduled, and may or may not be running jobs + * - We must cause it to leave the runpool by stopping it from + * submitting any more jobs. When it finally does leave, + * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs + * (because it is dying), release the Power Manager active reference, + * and will not requeue the context in the queue. + * kbase_jd_zap_context() will wait for those jobs to be killed. + * - Hence, work required just to make it leave the runpool. Cancelling + * jobs and releasing the Power manager active reference will be + * handled when it leaves the runpool. + */ + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { + unsigned long flags; + int js; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + if (!list_empty( + &kctx->jctx.sched_info.ctx.ctx_list_entry[js])) + list_del_init( + &kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* The following events require us to kill off remaining jobs + * and update PM book-keeping: + * - we evicted it correctly (it must have jobs to be in the + * Queue) + * + * These events need no action, but take this path anyway: + * - Case a: it didn't have any jobs, and was never in the Queue + * - Case b: scheduling transaction will be partially rolled- + * back (this already cancels the jobs) + */ + + KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx); + + /* Only cancel jobs when we evicted from the + * queue. No Power Manager active reference was held. + * + * Having is_dying set ensures that this kills, and + * doesn't requeue */ + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + mutex_unlock(&kctx->jctx.lock); + } else { + unsigned long flags; + bool was_retained; + + /* Case c: didn't evict, but it is scheduled - it's in the Run + * Pool */ + KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx); + + /* Disable the ctx from submitting any more jobs */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbasep_js_clear_submit_allowed(js_devdata, kctx); + + /* Retain and (later) release the context whilst it is is now + * disallowed from submitting jobs - ensures that someone + * somewhere will be removing the context later on */ + was_retained = kbase_ctx_sched_inc_refcount_nolock(kctx); + + /* Since it's scheduled and we have the jsctx_mutex, it must be + * retained successfully */ + KBASE_DEBUG_ASSERT(was_retained); + + dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx); + + /* Cancel any remaining running jobs for this kctx - if any. + * Submit is disallowed which takes effect immediately, so no + * more new jobs will appear after we do this. */ + kbase_backend_jm_kill_running_jobs_from_kctx(kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + mutex_unlock(&kctx->jctx.lock); + + dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", + kctx); + + kbasep_js_runpool_release_ctx(kbdev, kctx); + } + + KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); + + /* After this, you must wait on both the + * kbase_jd_context::zero_jobs_wait and the + * kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs + * to be destroyed, and the context to be de-scheduled (if it was on the + * runpool). + * + * kbase_jd_zap_context() will do this. */ +} + +static inline int trace_get_refcnt(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + return atomic_read(&kctx->refcount); +} + +/** + * kbase_js_foreach_ctx_job(): - Call a function on all jobs in context + * @kctx: Pointer to context. + * @callback: Pointer to function to call for each job. + * + * Call a function on all jobs belonging to a non-queued, non-running + * context, and detach the jobs from the context as it goes. + * + * Due to the locks that might be held at the time of the call, the callback + * may need to defer work on a workqueue to complete its actions (e.g. when + * cancelling jobs) + * + * Atoms will be removed from the queue, so this must only be called when + * cancelling jobs (which occurs as part of context destruction). + * + * The locking conditions on the caller are as follows: + * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex. + */ +static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, + kbasep_js_ctx_job_cb callback) +{ + struct kbase_device *kbdev; + unsigned long flags; + u32 js; + + kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, + 0u, trace_get_refcnt(kbdev, kctx)); + + /* Invoke callback on jobs on each slot in turn */ + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) + jsctx_queue_foreach(kctx, js, callback); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js.h b/drivers/gpu/arm/bifrost/mali_kbase_js.h new file mode 100755 index 000000000000..541acd4afed7 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_js.h @@ -0,0 +1,40 @@ +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_js.h + * Job Scheduler APIs. + */ + +#ifndef _KBASE_JS_H_ +#define _KBASE_JS_H_ + +#include "context/mali_kbase_context.h" +#include "mali_kbase_defs.h" +#include "mali_kbase_debug.h" +#include +#include "jm/mali_kbase_jm_js.h" +#include "jm/mali_kbase_js_defs.h" + +#endif /* _KBASE_JS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.c b/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.c new file mode 100755 index 000000000000..141d04a385cb --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.c @@ -0,0 +1,283 @@ +/* + * + * (C) COPYRIGHT 2012-2016, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +#include +#include + +/* + * Private functions follow + */ + +/** + * @brief Check whether a ctx has a certain attribute, and if so, retain that + * attribute on the runpool. + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx is scheduled on the runpool + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { + KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX); + ++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); + + if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) { + /* First refcount indicates a state change */ + runpool_state_changed = true; + KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute); + } + } + + return runpool_state_changed; +} + +/** + * @brief Check whether a ctx has a certain attribute, and if so, release that + * attribute on the runpool. + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx is scheduled on the runpool + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { + KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0); + --(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); + + if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) { + /* Last de-refcount indicates a state change */ + runpool_state_changed = true; + KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute); + } + } + + return runpool_state_changed; +} + +/** + * @brief Retain a certain attribute on a ctx, also retaining it on the runpool + * if the context is scheduled. + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + * + * @return true indicates a change in ctx attributes state of the runpool. + * This may allow the scheduler to submit more jobs than previously. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX); + + ++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); + + if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { + /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */ + KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); + runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute); + } + + return runpool_state_changed; +} + +/* + * @brief Release a certain attribute on a ctx, also releasing it from the runpool + * if the context is scheduled. + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + * + * @return true indicates a change in ctx attributes state of the runpool. + * This may allow the scheduler to submit more jobs than previously. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0); + + if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { + lockdep_assert_held(&kbdev->hwaccess_lock); + /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */ + runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute); + KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); + } + + /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */ + --(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); + + return runpool_state_changed; +} + +/* + * More commonly used public functions + */ + +void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + bool runpool_state_changed; + int i; + + /* Retain any existing attributes */ + for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { + /* The context is being scheduled in, so update the runpool with the new attributes */ + runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); + + /* We don't need to know about state changed, because retaining a + * context occurs on scheduling it, and that itself will also try + * to run new atoms */ + CSTD_UNUSED(runpool_state_changed); + } + } +} + +bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + bool runpool_state_changed = false; + int i; + + /* Release any existing attributes */ + for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { + /* The context is being scheduled out, so update the runpool on the removed attributes */ + runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); + } + } + + return runpool_state_changed; +} + +void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + bool runpool_state_changed = false; + base_jd_core_req core_req; + + KBASE_DEBUG_ASSERT(katom); + core_req = katom->core_req; + + if (core_req & BASE_JD_REQ_ONLY_COMPUTE) + runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); + else + runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); + + if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { + /* Atom that can run on slot1 or slot2, and can use all cores */ + runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); + } + + /* We don't need to know about state changed, because retaining an + * atom occurs on adding it, and that itself will also try to run + * new atoms */ + CSTD_UNUSED(runpool_state_changed); +} + +bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state) +{ + bool runpool_state_changed = false; + base_jd_core_req core_req; + + KBASE_DEBUG_ASSERT(katom_retained_state); + core_req = katom_retained_state->core_req; + + /* No-op for invalid atoms */ + if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false) + return false; + + if (core_req & BASE_JD_REQ_ONLY_COMPUTE) + runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); + else + runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); + + if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { + /* Atom that can run on slot1 or slot2, and can use all cores */ + runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); + } + + return runpool_state_changed; +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.h b/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.h new file mode 100755 index 000000000000..25fd39787c71 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_js_ctx_attr.h @@ -0,0 +1,155 @@ +/* + * + * (C) COPYRIGHT 2012-2015, 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_js_ctx_attr.h + * Job Scheduler Context Attribute APIs + */ + +#ifndef _KBASE_JS_CTX_ATTR_H_ +#define _KBASE_JS_CTX_ATTR_H_ + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_js + * @{ + */ + +/** + * Retain all attributes of a context + * + * This occurs on scheduling in the context on the runpool (but after + * is_scheduled is set) + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx->is_scheduled is true + */ +void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * Release all attributes of a context + * + * This occurs on scheduling out the context from the runpool (but before + * is_scheduled is cleared) + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx->is_scheduled is true + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * Retain all attributes of an atom + * + * This occurs on adding an atom to a context + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + */ +void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); + +/** + * Release all attributes of an atom, given its retained state. + * + * This occurs after (permanently) removing an atom from a context + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + * + * This is a no-op when \a katom_retained_state is invalid. + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); + +/** + * Requires: + * - runpool_irq spinlock + */ +static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_device_data *js_devdata; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_devdata = &kbdev->js_data; + + return js_devdata->runpool_irq.ctx_attr_ref_count[attribute]; +} + +/** + * Requires: + * - runpool_irq spinlock + */ +static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) +{ + /* In general, attributes are 'on' when they have a non-zero refcount (note: the refcount will never be < 0) */ + return (bool) kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute); +} + +/** + * Requires: + * - jsctx mutex + */ +static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_kctx_info = &kctx->jctx.sched_info; + + /* In general, attributes are 'on' when they have a refcount (which should never be < 0) */ + return (bool) (js_kctx_info->ctx.ctx_attr_ref_count[attribute]); +} + + /** @} *//* end group kbase_js */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_JS_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c new file mode 100755 index 000000000000..fd1ea8815b16 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c @@ -0,0 +1,895 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * mali_kbase_kinstr_jm.c + * Kernel driver public interface to job manager atom tracing + */ + +#include "mali_kbase_kinstr_jm.h" +#include "mali_kbase_kinstr_jm_reader.h" + +#include "mali_kbase.h" +#include "mali_kbase_linux.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if KERNEL_VERSION(5, 1, 0) <= LINUX_VERSION_CODE +#include +#else +// Stringify the expression if no message is given. +#define static_assert(e, ...) __static_assert(e, #__VA_ARGS__, #e) +#define __static_assert(e, msg, ...) _Static_assert(e, msg) +#endif + +#if KERNEL_VERSION(4, 16, 0) >= LINUX_VERSION_CODE +typedef unsigned int __poll_t; +#endif + +#ifndef ENOTSUP +#define ENOTSUP EOPNOTSUPP +#endif + +/* The module printing prefix */ +#define PR_ "mali_kbase_kinstr_jm: " + +/* Allows us to perform ASM goto for the tracing + * https://www.kernel.org/doc/Documentation/static-keys.txt + */ +#if KERNEL_VERSION(4, 3, 0) <= LINUX_VERSION_CODE +DEFINE_STATIC_KEY_FALSE(basep_kinstr_jm_reader_static_key); +#else +struct static_key basep_kinstr_jm_reader_static_key = STATIC_KEY_INIT_FALSE; +#define static_branch_inc(key) static_key_slow_inc(key) +#define static_branch_dec(key) static_key_slow_dec(key) +#endif /* KERNEL_VERSION(4 ,3, 0) <= LINUX_VERSION_CODE */ + +#define KBASE_KINSTR_JM_VERSION 1 + +/** + * struct kbase_kinstr_jm - The context for the kernel job manager atom tracing + * @readers: a bitlocked list of opened readers. Readers are attached to the + * private data of a file descriptor that the user opens with the + * KBASE_IOCTL_KINSTR_JM_FD IO control call. + * @refcount: reference count for the context. Any reader will have a link + * back to the context so that they can remove themselves from the + * list. + * + * This is opaque outside this compilation unit + */ +struct kbase_kinstr_jm { + struct hlist_bl_head readers; + struct kref refcount; +}; + +/** + * struct kbase_kinstr_jm_atom_state_change - Represents an atom changing to a + * new state + * @timestamp: Raw monotonic nanoseconds of the state change + * @state: The state that the atom has moved to + * @atom: The atom number that has changed state + * @flags: Flags associated with the state change. See + * KBASE_KINSTR_JM_ATOM_STATE_FLAG_* defines. + * @reserved: Reserved for future use. + * @data: Extra data for the state change. Active member depends on state. + * + * We can add new fields to the structure and old user code will gracefully + * ignore the new fields. + * + * We can change the size of the structure and old user code will gracefully + * skip over the new size via `struct kbase_kinstr_jm_fd_out->size`. + * + * If we remove fields, the version field in `struct + * kbase_kinstr_jm_fd_out->version` will be incremented and old user code will + * gracefully fail and tell the user that the kernel API is too new and has + * backwards-incompatible changes. Note that one userspace can opt to handle + * multiple kernel major versions of the structure. + * + * If we need to change the _meaning_ of one of the fields, i.e. the state + * machine has had a incompatible change, we can keep the same members in the + * structure and update the version as above. User code will no longer + * recognise that it has the supported field and can gracefully explain to the + * user that the kernel API is no longer supported. + * + * When making changes to this structure, make sure they are either: + * - additions to the end (for minor version bumps (i.e. only a size increase)) + * such that the layout of existing fields doesn't change, or; + * - update the version reported to userspace so that it can fail explicitly. + */ +struct kbase_kinstr_jm_atom_state_change { + u64 timestamp; + s8 state; /* enum kbase_kinstr_jm_reader_atom_state */ + u8 atom; + u8 flags; + u8 reserved[1]; + /* Tagged union based on state. Ensure members are aligned correctly! */ + union { + struct { + u8 slot; + } start; + u8 padding[4]; + } data; +}; +static_assert( + ((1 << 8 * sizeof(((struct kbase_kinstr_jm_atom_state_change *)0)->state)) - 1) >= + KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT); + +#define KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW BIT(0) + +/** + * struct reader_changes - The circular buffer of kernel atom state changes + * @data: The allocated buffer. This is allocated when the user requests + * the reader file descriptor. It is released when the user calls + * close() on the fd. When accessing this, lock the producer spin + * lock to prevent races on the allocated memory. The consume lock + * does not need to be held because newly-inserted data will always + * be outside the currenly-read range. + * @producer: The producing spinlock which allows us to push changes into the + * buffer at the same time as a user read occurring. This needs to + * be locked when saving/restoring the IRQ because we can receive an + * interrupt from the GPU when an atom completes. The CPU could have + * a task preempted that is holding this lock. + * @consumer: The consuming mutex which locks around the user read(). + * Must be held when updating the tail of the circular buffer. + * @head: The head of the circular buffer. Can be used with Linux @c CIRC_ + * helpers. The producer should lock and update this with an SMP + * store when a new change lands. The consumer can read with an + * SMP load. This allows the producer to safely insert new changes + * into the circular buffer. + * @tail: The tail of the circular buffer. Can be used with Linux @c CIRC_ + * helpers. The producer should do a READ_ONCE load and the consumer + * should SMP store. + * @size: The number of changes that are allowed in @c data. Can be used + * with Linux @c CIRC_ helpers. Will always be a power of two. The + * producer lock should be held when updating this and stored with + * an SMP release memory barrier. This means that the consumer can + * do an SMP load. + * @threshold: The number of changes above which threads polling on the reader + * file descriptor will be woken up. + */ +struct reader_changes { + struct kbase_kinstr_jm_atom_state_change *data; + spinlock_t producer; + struct mutex consumer; + u32 head; + u32 tail; + u32 size; + u32 threshold; +}; + +/** + * reader_changes_is_valid_size() - Determines if requested changes buffer size + * is valid. + * @size: The requested memory size + * + * We have a constraint that the underlying physical buffer must be a + * power of two so that we can use the efficient circular buffer helpers that + * the kernel provides. It also needs to be representable within a u32. + * + * Return: + * * true - the size is valid + * * false - the size is invalid + */ +static inline bool reader_changes_is_valid_size(const size_t size) +{ + typedef struct reader_changes changes_t; + const size_t elem_size = sizeof(*((changes_t *)0)->data); + const size_t size_size = sizeof(((changes_t *)0)->size); + const size_t size_max = (1ull << (size_size * 8)) - 1; + + return is_power_of_2(size) && /* Is a power of two */ + ((size / elem_size) <= size_max); /* Small enough */ +} + +/** + * reader_changes_init() - Initializes the reader changes and allocates the + * changes buffer + * @changes: The context pointer, must point to a zero-inited allocated reader + * changes structure. We may support allocating the structure in the + * future. + * @size: The requested changes buffer size + * + * Return: + * (0, U16_MAX] - the number of data elements allocated + * -EINVAL - a pointer was invalid + * -ENOTSUP - we do not support allocation of the context + * -ERANGE - the requested memory size was invalid + * -ENOMEM - could not allocate the memory + * -EADDRINUSE - the buffer memory was already allocated + */ +static int reader_changes_init(struct reader_changes *const changes, + const size_t size) +{ + BUILD_BUG_ON((PAGE_SIZE % sizeof(*changes->data)) != 0); + + if (!reader_changes_is_valid_size(size)) { + pr_warn(PR_ "invalid size %zu\n", size); + return -ERANGE; + } + + changes->data = vmalloc(size); + if (!changes->data) + return -ENOMEM; + + spin_lock_init(&changes->producer); + mutex_init(&changes->consumer); + + changes->size = size / sizeof(*changes->data); + changes->threshold = min(((size_t)(changes->size)) / 4, + ((size_t)(PAGE_SIZE)) / sizeof(*changes->data)); + + return changes->size; +} + +/** + * reader_changes_term() - Cleans up a reader changes structure + * @changes: The context to clean up + * + * Releases the allocated state changes memory + */ +static void reader_changes_term(struct reader_changes *const changes) +{ + struct kbase_kinstr_jm_atom_state_change *data = NULL; + unsigned long irq; + + /* + * Although changes->data is used on the consumer side, too, no active + * consumer is possible by the time we clean up the reader changes, so + * no need to take the consumer lock. However, we do need the producer + * lock because the list removal can race with list traversal. + */ + spin_lock_irqsave(&changes->producer, irq); + swap(changes->data, data); + spin_unlock_irqrestore(&changes->producer, irq); + + mutex_destroy(&changes->consumer); + vfree(data); +} + +/** + * reader_changes_count_locked() - Retrieves the count of state changes from the + * tail to the physical end of the buffer + * @changes: The state changes context + * + * The consumer mutex must be held. Uses the CIRC_CNT_TO_END macro to + * determine the count, so there may be more items. However, that's the maximum + * number that can be read in one contiguous read. + * + * Return: the number of changes in the circular buffer until the end of the + * allocation + */ +static u32 reader_changes_count_locked(struct reader_changes *const changes) +{ + u32 head; + + lockdep_assert_held_once(&changes->consumer); + + head = smp_load_acquire(&changes->head); + + return CIRC_CNT_TO_END(head, changes->tail, changes->size); +} + +/** + * reader_changes_count() - Retrieves the count of state changes from the + * tail to the physical end of the buffer + * @changes: The state changes context + * + * Return: the number of changes in the circular buffer until the end of the + * allocation + */ +static u32 reader_changes_count(struct reader_changes *const changes) +{ + u32 ret; + + mutex_lock(&changes->consumer); + ret = reader_changes_count_locked(changes); + mutex_unlock(&changes->consumer); + return ret; +} + +/** + * reader_changes_push() - Pushes a change into the reader circular buffer. + * @changes: The buffer to insert the change into + * @change: Kernel atom change to insert + * @wait_queue: The queue to be kicked when changes should be read from + * userspace. Kicked when a threshold is reached or there is + * overflow. + */ +static void reader_changes_push( + struct reader_changes *const changes, + const struct kbase_kinstr_jm_atom_state_change *const change, + wait_queue_head_t *const wait_queue) +{ + u32 head, tail, size, space; + unsigned long irq; + struct kbase_kinstr_jm_atom_state_change *data; + + spin_lock_irqsave(&changes->producer, irq); + + /* We may be called for a reader_changes that's awaiting cleanup. */ + data = changes->data; + if (!data) + goto unlock; + + size = changes->size; + head = changes->head; + tail = smp_load_acquire(&changes->tail); + + space = CIRC_SPACE(head, tail, size); + if (space >= 1) { + data[head] = *change; + if (space == 1) { + data[head].flags |= + KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW; + pr_warn(PR_ "overflow of circular buffer\n"); + } + smp_store_release(&changes->head, (head + 1) & (size - 1)); + } + + /* Wake for either overflow or over-threshold cases. */ + if (CIRC_CNT(head + 1, tail, size) >= changes->threshold) + wake_up_interruptible(wait_queue); + +unlock: + spin_unlock_irqrestore(&changes->producer, irq); +} + +/** + * struct reader - Allows the kernel state changes to be read by user space. + * @node: The node in the @c readers locked list + * @rcu_head: storage for the RCU callback to free this reader (see kfree_rcu) + * @changes: The circular buffer of user changes + * @wait_queue: A wait queue for poll + * @context: a pointer to the parent context that created this reader. Can be + * used to remove the reader from the list of readers. Reference + * counted. + * + * The reader is a circular buffer in kernel space. State changes are pushed + * into the buffer. The flow from user space is: + * + * * Request file descriptor with KBASE_IOCTL_KINSTR_JM_FD. This will + * allocate the kernel side circular buffer with a size specified in the + * ioctl argument. + * * The user will then poll the file descriptor for data + * * Upon receiving POLLIN, perform a read() on the file descriptor to get + * the data out. + * * The buffer memory will be freed when the file descriptor is closed + */ +struct reader { + struct hlist_bl_node node; + struct rcu_head rcu_head; + struct reader_changes changes; + wait_queue_head_t wait_queue; + struct kbase_kinstr_jm *context; +}; + +static struct kbase_kinstr_jm * +kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx); +static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx); +static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, + struct reader *const reader); +static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, + struct reader *const reader); + +/** + * reader_term() - Terminate a instrumentation job manager reader context. + * @reader: Pointer to context to be terminated. + */ +static void reader_term(struct reader *const reader) +{ + if (!reader) + return; + + kbase_kinstr_jm_readers_del(reader->context, reader); + reader_changes_term(&reader->changes); + kbase_kinstr_jm_ref_put(reader->context); + + kfree_rcu(reader, rcu_head); +} + +/** + * reader_init() - Initialise a instrumentation job manager reader context. + * @out_reader: Non-NULL pointer to where the pointer to the created context + * will be stored on success. + * @ctx: the pointer to the parent context. Reference count will be + * increased if initialization is successful + * @num_changes: The number of changes to allocate a buffer for + * + * Return: 0 on success, else error code. + */ +static int reader_init(struct reader **const out_reader, + struct kbase_kinstr_jm *const ctx, + size_t const num_changes) +{ + struct reader *reader = NULL; + const size_t change_size = sizeof(struct kbase_kinstr_jm_atom_state_change); + int status; + + if (!out_reader || !ctx || !num_changes) + return -EINVAL; + + reader = kzalloc(sizeof(*reader), GFP_KERNEL); + if (!reader) + return -ENOMEM; + + INIT_HLIST_BL_NODE(&reader->node); + init_waitqueue_head(&reader->wait_queue); + + reader->context = kbase_kinstr_jm_ref_get(ctx); + + status = reader_changes_init(&reader->changes, num_changes * change_size); + if (status < 0) + goto fail; + + status = kbase_kinstr_jm_readers_add(ctx, reader); + if (status < 0) + goto fail; + + *out_reader = reader; + + return 0; + +fail: + kbase_kinstr_jm_ref_put(reader->context); + kfree(reader); + return status; +} + +/** + * reader_release() - Invoked when the reader file descriptor is released + * @node: The inode that the file descriptor that the file corresponds to. In + * our case our reader file descriptor is backed by an anonymous node so + * not much is in this. + * @file: the file data. Our reader context is held in the private data + * Return: zero on success + */ +static int reader_release(struct inode *const node, struct file *const file) +{ + struct reader *const reader = file->private_data; + + reader_term(reader); + file->private_data = NULL; + + return 0; +} + +/** + * reader_changes_copy_to_user() - Copy any changes from a changes structure to + * the user-provided buffer. + * @changes: The changes structure from which to copy. + * @buffer: The user buffer to copy the data to. + * @buffer_size: The number of bytes in the buffer. + * Return: The number of bytes copied or negative errno on failure. + */ +static ssize_t reader_changes_copy_to_user(struct reader_changes *const changes, + char __user *buffer, + size_t buffer_size) +{ + ssize_t ret = 0; + struct kbase_kinstr_jm_atom_state_change const *src_buf = READ_ONCE( + changes->data); + size_t const entry_size = sizeof(*src_buf); + size_t changes_tail, changes_count, read_size; + + /* Needed for the quick buffer capacity calculation below. + * Note that we can't use is_power_of_2() since old compilers don't + * understand it's a constant expression. + */ +#define is_power_of_two(x) ((x) && !((x) & ((x) - 1))) + static_assert(is_power_of_two( + sizeof(struct kbase_kinstr_jm_atom_state_change))); +#undef is_power_of_two + + lockdep_assert_held_once(&changes->consumer); + + /* Read continuously until either: + * - we've filled the output buffer, or + * - there are no changes when we check. + * + * If more changes arrive while we're copying to the user, we can copy + * those as well, space permitting. + */ + do { + changes_tail = changes->tail; + changes_count = reader_changes_count_locked(changes); + read_size = min(changes_count * entry_size, + buffer_size & ~(entry_size - 1)); + + if (!read_size) + break; + + if (copy_to_user(buffer, &(src_buf[changes_tail]), read_size)) + return -EFAULT; + + buffer += read_size; + buffer_size -= read_size; + ret += read_size; + changes_tail = (changes_tail + read_size / entry_size) & + (changes->size - 1); + smp_store_release(&changes->tail, changes_tail); + } while (read_size); + + return ret; +} + +/** + * reader_read() - Handles a read call on the reader file descriptor + * + * @filp: The file that the read was performed on + * @buffer: The destination buffer + * @buffer_size: The maximum number of bytes to read + * @offset: The offset into the 'file' to read from. + * + * Note the destination buffer needs to be fully mapped in userspace or the read + * will fault. + * + * Return: + * * The number of bytes read or: + * * -EBADF - the file descriptor did not have an attached reader + * * -EFAULT - memory access fault + * * -EAGAIN - if the file is set to nonblocking reads with O_NONBLOCK and there + * is no data available + * + * Note: The number of bytes read will always be a multiple of the size of an + * entry. + */ +static ssize_t reader_read(struct file *const filp, + char __user *const buffer, + size_t const buffer_size, + loff_t *const offset) +{ + struct reader *const reader = filp->private_data; + struct reader_changes *changes; + ssize_t ret; + + if (!reader) + return -EBADF; + + if (buffer_size < sizeof(struct kbase_kinstr_jm_atom_state_change)) + return -ENOBUFS; + +#if KERNEL_VERSION(5, 0, 0) <= LINUX_VERSION_CODE + if (!access_ok(buffer, buffer_size)) + return -EIO; +#else + if (!access_ok(VERIFY_WRITE, buffer, buffer_size)) + return -EIO; +#endif + + changes = &reader->changes; + + mutex_lock(&changes->consumer); + if (!reader_changes_count_locked(changes)) { + if (filp->f_flags & O_NONBLOCK) { + ret = -EAGAIN; + goto exit; + } + + if (wait_event_interruptible( + reader->wait_queue, + !!reader_changes_count_locked(changes))) { + ret = -EINTR; + goto exit; + } + } + + ret = reader_changes_copy_to_user(changes, buffer, buffer_size); + +exit: + mutex_unlock(&changes->consumer); + return ret; +} + +/** + * reader_poll() - Handles a poll call on the reader file descriptor + * @file: The file that the poll was performed on + * @wait: The poll table + * + * The results of the poll will be unreliable if there is no mapped memory as + * there is no circular buffer to push atom state changes into. + * + * Return: + * * 0 - no data ready + * * POLLIN - state changes have been buffered + * * -EBADF - the file descriptor did not have an attached reader + * * -EINVAL - the IO control arguments were invalid + */ +static __poll_t reader_poll(struct file *const file, + struct poll_table_struct *const wait) +{ + struct reader *reader; + struct reader_changes *changes; + + if (unlikely(!file || !wait)) + return -EINVAL; + + reader = file->private_data; + if (unlikely(!reader)) + return -EBADF; + + changes = &reader->changes; + + if (reader_changes_count(changes) >= changes->threshold) + return POLLIN; + + poll_wait(file, &reader->wait_queue, wait); + + return (reader_changes_count(changes) > 0) ? POLLIN : 0; +} + +/* The file operations virtual function table */ +static const struct file_operations file_operations = { + .owner = THIS_MODULE, + .llseek = no_llseek, + .read = reader_read, + .poll = reader_poll, + .release = reader_release +}; + +/* The maximum amount of readers that can be created on a context. */ +static const size_t kbase_kinstr_jm_readers_max = 16; + +/** + * kbasep_kinstr_jm_release() - Invoked when the reference count is dropped + * @ref: the context reference count + */ +static void kbase_kinstr_jm_release(struct kref *const ref) +{ + struct kbase_kinstr_jm *const ctx = + container_of(ref, struct kbase_kinstr_jm, refcount); + + kfree(ctx); +} + +/** + * kbase_kinstr_jm_ref_get() - Reference counts the instrumentation context + * @ctx: the context to reference count + * Return: the reference counted context + */ +static struct kbase_kinstr_jm * +kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx) +{ + if (likely(ctx)) + kref_get(&ctx->refcount); + return ctx; +} + +/** + * kbase_kinstr_jm_ref_put() - Dereferences the instrumentation context + * @ctx: the context to lower the reference count on + */ +static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx) +{ + if (likely(ctx)) + kref_put(&ctx->refcount, kbase_kinstr_jm_release); +} + +/** + * kbase_kinstr_jm_readers_add() - Adds a reader to the list of readers + * @ctx: the instrumentation context + * @reader: the reader to add + * + * Return: + * 0 - success + * -ENOMEM - too many readers already added. + */ +static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, + struct reader *const reader) +{ + struct hlist_bl_head *const readers = &ctx->readers; + struct hlist_bl_node *node; + struct reader *temp; + size_t count = 0; + + hlist_bl_lock(readers); + + hlist_bl_for_each_entry_rcu(temp, node, readers, node) + ++count; + + if (kbase_kinstr_jm_readers_max < count) { + hlist_bl_unlock(readers); + return -ENOMEM; + } + + hlist_bl_add_head_rcu(&reader->node, readers); + + hlist_bl_unlock(readers); + + static_branch_inc(&basep_kinstr_jm_reader_static_key); + + return 0; +} + +/** + * readers_del() - Deletes a reader from the list of readers + * @ctx: the instrumentation context + * @reader: the reader to delete + */ +static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, + struct reader *const reader) +{ + struct hlist_bl_head *const readers = &ctx->readers; + + hlist_bl_lock(readers); + hlist_bl_del_rcu(&reader->node); + hlist_bl_unlock(readers); + + static_branch_dec(&basep_kinstr_jm_reader_static_key); +} + +int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, + union kbase_kinstr_jm_fd *jm_fd_arg) +{ + struct kbase_kinstr_jm_fd_in const *in; + struct reader *reader; + size_t const change_size = sizeof(struct + kbase_kinstr_jm_atom_state_change); + int status; + int fd; + int i; + + if (!ctx || !jm_fd_arg) + return -EINVAL; + + in = &jm_fd_arg->in; + + if (!is_power_of_2(in->count)) + return -EINVAL; + + for (i = 0; i < sizeof(in->padding); ++i) + if (in->padding[i]) + return -EINVAL; + + status = reader_init(&reader, ctx, in->count); + if (status < 0) + return status; + + jm_fd_arg->out.version = KBASE_KINSTR_JM_VERSION; + jm_fd_arg->out.size = change_size; + memset(&jm_fd_arg->out.padding, 0, sizeof(jm_fd_arg->out.padding)); + + fd = anon_inode_getfd("[mali_kinstr_jm]", &file_operations, reader, + O_CLOEXEC); + if (fd < 0) + reader_term(reader); + + return fd; +} + +int kbase_kinstr_jm_init(struct kbase_kinstr_jm **const out_ctx) +{ + struct kbase_kinstr_jm *ctx = NULL; + + if (!out_ctx) + return -EINVAL; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + INIT_HLIST_BL_HEAD(&ctx->readers); + kref_init(&ctx->refcount); + + *out_ctx = ctx; + + return 0; +} + +void kbase_kinstr_jm_term(struct kbase_kinstr_jm *const ctx) +{ + kbase_kinstr_jm_ref_put(ctx); +} + +void kbasep_kinstr_jm_atom_state( + struct kbase_jd_atom *const katom, + const enum kbase_kinstr_jm_reader_atom_state state) +{ + struct kbase_context *const kctx = katom->kctx; + struct kbase_kinstr_jm *const ctx = kctx->kinstr_jm; + const u8 id = kbase_jd_atom_id(kctx, katom); + struct kbase_kinstr_jm_atom_state_change change = { + .timestamp = ktime_get_raw_ns(), .atom = id, .state = state + }; + struct reader *reader; + struct hlist_bl_node *node; + + WARN(KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT < state || 0 > state, + PR_ "unsupported katom (%u) state (%i)", id, state); + + switch (state) { + case KBASE_KINSTR_JM_READER_ATOM_STATE_START: + change.data.start.slot = katom->jobslot; + break; + default: + break; + } + + rcu_read_lock(); + hlist_bl_for_each_entry_rcu(reader, node, &ctx->readers, node) + reader_changes_push( + &reader->changes, &change, &reader->wait_queue); + rcu_read_unlock(); +} + +KBASE_EXPORT_TEST_API(kbasep_kinstr_jm_atom_state); + +void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const katom) +{ + struct kbase_context *const kctx = katom->kctx; + struct kbase_device *const kbdev = kctx->kbdev; + const int slot = katom->slot_nr; + struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); + + BUILD_BUG_ON(SLOT_RB_SIZE != 2); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (WARN_ON(slot < 0 || slot >= GPU_MAX_JOB_SLOTS)) + return; + if (WARN_ON(!submitted)) + return; + + if (submitted == katom) + kbase_kinstr_jm_atom_state_start(katom); +} + +void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const katom) +{ + struct kbase_context *const kctx = katom->kctx; + struct kbase_device *const kbdev = kctx->kbdev; + const int slot = katom->slot_nr; + struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); + struct kbase_jd_atom *const queued = kbase_gpu_inspect(kbdev, slot, 1); + + BUILD_BUG_ON(SLOT_RB_SIZE != 2); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (WARN_ON(slot < 0 || slot >= GPU_MAX_JOB_SLOTS)) + return; + if (WARN_ON(!submitted)) + return; + if (WARN_ON((submitted != katom) && (queued != katom))) + return; + + if (queued == katom) + return; + + if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) + kbase_kinstr_jm_atom_state_stop(katom); + if (queued && queued->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) + kbase_kinstr_jm_atom_state_start(queued); +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.h new file mode 100755 index 000000000000..555edfeef77c --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.h @@ -0,0 +1,283 @@ +/* + * + * (C) COPYRIGHT 2019,2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * mali_kbase_kinstr_jm.h + * Kernel driver public interface to job manager atom tracing. This API provides + * a method to get the atom state changes into user space. + * + * The flow of operation is: + * + * | kernel | user | + * | ----------------------------------- | ----------------------------------- | + * | Initialize API with | | + * | kbase_kinstr_jm_init() | | + * | | | + * | Kernel code injects states with | | + * | kbase_kinstr_jm_atom_state_*() APIs | | + * | | Call ioctl() to get file descriptor | + * | | via KBASE_IOCTL_KINSTR_JM_FD | + * | Allocates a reader attached to FD | | + * | Allocates circular buffer and | | + * | patches, via ASM goto, the | | + * | kbase_kinstr_jm_atom_state_*() | | + * | | loop: | + * | | Call poll() on FD for POLLIN | + * | When threshold of changes is hit, | | + * | the poll is interrupted with | | + * | POLLIN. If circular buffer is | | + * | full then store the missed count | | + * | and interrupt poll | Call read() to get data from | + * | | circular buffer via the fd | + * | Kernel advances tail of circular | | + * | buffer | | + * | | Close file descriptor | + * | Deallocates circular buffer | | + * | | | + * | Terminate API with | | + * | kbase_kinstr_jm_term() | | + * + * All tracepoints are guarded on a static key. The static key is activated when + * a user space reader gets created. This means that there is negligible cost + * inserting the tracepoints into code when there are no readers. + */ + +#ifndef _KBASE_KINSTR_JM_H_ +#define _KBASE_KINSTR_JM_H_ + +#include "mali_kbase_kinstr_jm_reader.h" + +#ifdef __KERNEL__ +#include +#include +#else +/* empty wrapper macros for userspace */ +#define static_branch_unlikely(key) (1) +#define KERNEL_VERSION(a, b, c) (0) +#define LINUX_VERSION_CODE (1) +#endif /* __KERNEL__ */ + +/* Forward declarations */ +struct kbase_context; +struct kbase_kinstr_jm; +struct kbase_jd_atom; +union kbase_kinstr_jm_fd; + +/** + * kbase_kinstr_jm_init() - Initialise an instrumentation job manager context. + * @ctx: Non-NULL pointer to where the pointer to the created context will + * be stored on success. + * + * Return: 0 on success, else error code. + */ +int kbase_kinstr_jm_init(struct kbase_kinstr_jm **ctx); + +/** + * kbase_kinstr_jm_term() - Terminate an instrumentation job manager context. + * @ctx: Pointer to context to be terminated. + */ +void kbase_kinstr_jm_term(struct kbase_kinstr_jm *ctx); + +/** + * kbase_kinstr_jm_get_fd() - Retrieves a file descriptor that can be used to + * read the atom state changes from userspace + * + * @ctx: Pointer to the initialized context + * @jm_fd_arg: Pointer to the union containing the in/out params + * Return: -1 on failure, valid file descriptor on success + */ +int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, + union kbase_kinstr_jm_fd *jm_fd_arg); + +/** + * kbasep_kinstr_jm_atom_state() - Signifies that an atom has changed state + * @atom: The atom that has changed state + * @state: The new state of the atom + * + * This performs the actual storage of the state ready for user space to + * read the data. It is only called when the static key is enabled from + * kbase_kinstr_jm_atom_state(). There is almost never a need to invoke this + * function directly. + */ +void kbasep_kinstr_jm_atom_state( + struct kbase_jd_atom *const atom, + const enum kbase_kinstr_jm_reader_atom_state state); + +/* Allows ASM goto patching to reduce tracing overhead. This is + * incremented/decremented when readers are created and terminated. This really + * shouldn't be changed externally, but if you do, make sure you use + * a static_key_inc()/static_key_dec() pair. + */ +#if KERNEL_VERSION(4, 3, 0) <= LINUX_VERSION_CODE +extern struct static_key_false basep_kinstr_jm_reader_static_key; +#else +/* Pre-4.3 kernels have a different API for static keys, but work + * mostly the same with less type safety. */ +extern struct static_key basep_kinstr_jm_reader_static_key; +#define static_branch_unlikely(key) static_key_false(key) +#endif /* KERNEL_VERSION(4, 3, 0) <= LINUX_VERSION_CODE */ + +/** + * kbase_kinstr_jm_atom_state() - Signifies that an atom has changed state + * @atom: The atom that has changed state + * @state: The new state of the atom + * + * This uses a static key to reduce overhead when tracing is disabled + */ +static inline void kbase_kinstr_jm_atom_state( + struct kbase_jd_atom *const atom, + const enum kbase_kinstr_jm_reader_atom_state state) +{ + if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) + kbasep_kinstr_jm_atom_state(atom, state); +} + +/** + * kbase_kinstr_jm_atom_state_queue() - Signifies that an atom has entered a + * hardware or software queue. + * @atom: The atom that has changed state + */ +static inline void kbase_kinstr_jm_atom_state_queue( + struct kbase_jd_atom *const atom) +{ + kbase_kinstr_jm_atom_state( + atom, KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE); +} + +/** + * kbase_kinstr_jm_atom_state_start() - Signifies that work has started on an + * atom + * @atom: The atom that has changed state + */ +static inline void kbase_kinstr_jm_atom_state_start( + struct kbase_jd_atom *const atom) +{ + kbase_kinstr_jm_atom_state( + atom, KBASE_KINSTR_JM_READER_ATOM_STATE_START); +} + +/** + * kbase_kinstr_jm_atom_state_stop() - Signifies that work has stopped on an + * atom + * @atom: The atom that has changed state + */ +static inline void kbase_kinstr_jm_atom_state_stop( + struct kbase_jd_atom *const atom) +{ + kbase_kinstr_jm_atom_state( + atom, KBASE_KINSTR_JM_READER_ATOM_STATE_STOP); +} + +/** + * kbase_kinstr_jm_atom_state_complete() - Signifies that all work has completed + * on an atom + * @atom: The atom that has changed state + */ +static inline void kbase_kinstr_jm_atom_state_complete( + struct kbase_jd_atom *const atom) +{ + kbase_kinstr_jm_atom_state( + atom, KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE); +} + +/** + * kbase_kinstr_jm_atom_queue() - A software *or* hardware atom is queued for + * execution + * @atom: The atom that has changed state + */ +static inline void kbase_kinstr_jm_atom_queue(struct kbase_jd_atom *const atom) +{ + kbase_kinstr_jm_atom_state_queue(atom); +} + +/** + * kbase_kinstr_jm_atom_complete() - A software *or* hardware atom is fully + * completed + * @atom: The atom that has changed state + */ +static inline void kbase_kinstr_jm_atom_complete( + struct kbase_jd_atom *const atom) +{ + kbase_kinstr_jm_atom_state_complete(atom); +} + +/** + * kbase_kinstr_jm_atom_sw_start() - A software atom has started work + * @atom: The atom that has changed state + */ +static inline void kbase_kinstr_jm_atom_sw_start( + struct kbase_jd_atom *const atom) +{ + kbase_kinstr_jm_atom_state_start(atom); +} + +/** + * kbase_kinstr_jm_atom_sw_stop() - A software atom has stopped work + * @atom: The atom that has changed state + */ +static inline void kbase_kinstr_jm_atom_sw_stop( + struct kbase_jd_atom *const atom) +{ + kbase_kinstr_jm_atom_state_stop(atom); +} + +/** + * kbasep_kinstr_jm_atom_hw_submit() - A hardware atom has been submitted + * @atom: The atom that has been submitted + * + * This private implementation should not be called directly, it is protected + * by a static key in kbase_kinstr_jm_atom_hw_submit(). Use that instead. + */ +void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const atom); + +/** + * kbase_kinstr_jm_atom_hw_submit() - A hardware atom has been submitted + * @atom: The atom that has been submitted + */ +static inline void kbase_kinstr_jm_atom_hw_submit( + struct kbase_jd_atom *const atom) +{ + if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) + kbasep_kinstr_jm_atom_hw_submit(atom); +} + +/** + * kbasep_kinstr_jm_atom_hw_release() - A hardware atom has been released + * @atom: The atom that has been released + * + * This private implementation should not be called directly, it is protected + * by a static key in kbase_kinstr_jm_atom_hw_release(). Use that instead. + */ +void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const atom); + +/** + * kbase_kinstr_jm_atom_hw_release() - A hardware atom has been released + * @atom: The atom that has been released + */ +static inline void kbase_kinstr_jm_atom_hw_release( + struct kbase_jd_atom *const atom) +{ + if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) + kbasep_kinstr_jm_atom_hw_release(atom); +} + +#endif /* _KBASE_KINSTR_JM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm_reader.h b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm_reader.h new file mode 100755 index 000000000000..e267e6bc44de --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm_reader.h @@ -0,0 +1,70 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * mali_kbase_kinstr_jm_reader.h + * Provides an ioctl API to read kernel atom state changes. The flow of the + * API is: + * 1. Obtain the file descriptor with ``KBASE_IOCTL_KINSTR_JM_FD`` + * 2. Determine the buffer structure layout via the above ioctl's returned + * size and version fields in ``struct kbase_kinstr_jm_fd_out`` + * 4. Poll the file descriptor for ``POLLIN`` + * 5. Get data with read() on the fd + * 6. Use the structure version to understand how to read the data from the + * buffer + * 7. Repeat 4-6 + * 8. Close the file descriptor + */ + +#ifndef _KBASE_KINSTR_JM_READER_H_ +#define _KBASE_KINSTR_JM_READER_H_ + +/** + * enum kbase_kinstr_jm_reader_atom_state - Determines the work state of an atom + * @KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE: Signifies that an atom has + * entered a hardware queue + * @KBASE_KINSTR_JM_READER_ATOM_STATE_START: Signifies that work has started + * on an atom + * @KBASE_KINSTR_JM_READER_ATOM_STATE_STOP: Signifies that work has stopped + * on an atom + * @KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE: Signifies that work has + * completed on an atom + * @KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT: The number of state enumerations + * + * We can add new states to the end of this if they do not break the existing + * state machine. Old user mode code can gracefully ignore states they do not + * understand. + * + * If we need to make a breaking change to the state machine, we can do that by + * changing the version reported by KBASE_IOCTL_KINSTR_JM_FD. This will + * mean that old user mode code will fail to understand the new state field in + * the structure and gracefully not use the state change API. + */ +enum kbase_kinstr_jm_reader_atom_state { + KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE, + KBASE_KINSTR_JM_READER_ATOM_STATE_START, + KBASE_KINSTR_JM_READER_ATOM_STATE_STOP, + KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE, + KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT +}; + +#endif /* _KBASE_KINSTR_JM_READER_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_linux.h b/drivers/gpu/arm/bifrost/mali_kbase_linux.h new file mode 100755 index 000000000000..003ac9e68a76 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_linux.h @@ -0,0 +1,48 @@ +/* + * + * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_linux.h + * Base kernel APIs, Linux implementation. + */ + +#ifndef _KBASE_LINUX_H_ +#define _KBASE_LINUX_H_ + +/* All things that are needed for the Linux port. */ +#include +#include +#include +#include +#include + +#if (defined(MALI_KERNEL_TEST_API) && (1 == MALI_KERNEL_TEST_API)) + #define KBASE_EXPORT_TEST_API(func) EXPORT_SYMBOL(func) +#else + #define KBASE_EXPORT_TEST_API(func) +#endif + +#define KBASE_EXPORT_SYMBOL(func) EXPORT_SYMBOL(func) + +#endif /* _KBASE_LINUX_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.c b/drivers/gpu/arm/bifrost/mali_kbase_mem.c new file mode 100755 index 000000000000..82a799c2d673 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.c @@ -0,0 +1,4734 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Base kernel memory APIs + */ +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_OF +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Alignment of objects allocated by the GPU inside a just-in-time memory + * region whose size is given by an end address + * + * This is the alignment of objects allocated by the GPU, but possibly not + * fully written to. When taken into account with + * KBASE_GPU_ALLOCATED_OBJECT_MAX_BYTES it gives the maximum number of bytes + * that the JIT memory report size can exceed the actual backed memory size. + */ +#define KBASE_GPU_ALLOCATED_OBJECT_ALIGN_BYTES (128u) + +/* + * Maximum size of objects allocated by the GPU inside a just-in-time memory + * region whose size is given by an end address + * + * This is the maximum size of objects allocated by the GPU, but possibly not + * fully written to. When taken into account with + * KBASE_GPU_ALLOCATED_OBJECT_ALIGN_BYTES it gives the maximum number of bytes + * that the JIT memory report size can exceed the actual backed memory size. + */ +#define KBASE_GPU_ALLOCATED_OBJECT_MAX_BYTES (512u) + + +/* Forward declarations */ +static void free_partial_locked(struct kbase_context *kctx, + struct kbase_mem_pool *pool, struct tagged_addr tp); + +static size_t kbase_get_num_cpu_va_bits(struct kbase_context *kctx) +{ +#if defined(CONFIG_ARM64) + /* VA_BITS can be as high as 48 bits, but all bits are available for + * both user and kernel. + */ + size_t cpu_va_bits = VA_BITS; +#elif defined(CONFIG_X86_64) + /* x86_64 can access 48 bits of VA, but the 48th is used to denote + * kernel (1) vs userspace (0), so the max here is 47. + */ + size_t cpu_va_bits = 47; +#elif defined(CONFIG_ARM) || defined(CONFIG_X86_32) + size_t cpu_va_bits = sizeof(void *) * BITS_PER_BYTE; +#else +#error "Unknown CPU VA width for this architecture" +#endif + +#ifdef CONFIG_64BIT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + cpu_va_bits = 32; +#endif + + return cpu_va_bits; +} + +/* This function finds out which RB tree the given pfn from the GPU VA belongs + * to based on the memory zone the pfn refers to */ +static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, + u64 gpu_pfn) +{ + struct rb_root *rbtree = NULL; + + /* The gpu_pfn can only be greater than the starting pfn of the EXEC_VA + * zone if this has been initialized. + */ + if (gpu_pfn >= kctx->exec_va_start) + rbtree = &kctx->reg_rbtree_exec; + else { + u64 same_va_end; + +#ifdef CONFIG_64BIT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) +#endif /* CONFIG_64BIT */ + same_va_end = KBASE_REG_ZONE_CUSTOM_VA_BASE; +#ifdef CONFIG_64BIT + else + same_va_end = kctx->same_va_end; +#endif /* CONFIG_64BIT */ + + if (gpu_pfn >= same_va_end) + rbtree = &kctx->reg_rbtree_custom; + else + rbtree = &kctx->reg_rbtree_same; + } + + return rbtree; +} + +/* This function inserts a region into the tree. */ +static void kbase_region_tracker_insert(struct kbase_va_region *new_reg) +{ + u64 start_pfn = new_reg->start_pfn; + struct rb_node **link = NULL; + struct rb_node *parent = NULL; + struct rb_root *rbtree = NULL; + + rbtree = new_reg->rbtree; + + link = &(rbtree->rb_node); + /* Find the right place in the tree using tree search */ + while (*link) { + struct kbase_va_region *old_reg; + + parent = *link; + old_reg = rb_entry(parent, struct kbase_va_region, rblink); + + /* RBTree requires no duplicate entries. */ + KBASE_DEBUG_ASSERT(old_reg->start_pfn != start_pfn); + + if (old_reg->start_pfn > start_pfn) + link = &(*link)->rb_left; + else + link = &(*link)->rb_right; + } + + /* Put the new node there, and rebalance tree */ + rb_link_node(&(new_reg->rblink), parent, link); + + rb_insert_color(&(new_reg->rblink), rbtree); +} + +static struct kbase_va_region *find_region_enclosing_range_rbtree( + struct rb_root *rbtree, u64 start_pfn, size_t nr_pages) +{ + struct rb_node *rbnode; + struct kbase_va_region *reg; + u64 end_pfn = start_pfn + nr_pages; + + rbnode = rbtree->rb_node; + + while (rbnode) { + u64 tmp_start_pfn, tmp_end_pfn; + + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + tmp_start_pfn = reg->start_pfn; + tmp_end_pfn = reg->start_pfn + reg->nr_pages; + + /* If start is lower than this, go left. */ + if (start_pfn < tmp_start_pfn) + rbnode = rbnode->rb_left; + /* If end is higher than this, then go right. */ + else if (end_pfn > tmp_end_pfn) + rbnode = rbnode->rb_right; + else /* Enclosing */ + return reg; + } + + return NULL; +} + +struct kbase_va_region *kbase_find_region_enclosing_address( + struct rb_root *rbtree, u64 gpu_addr) +{ + u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; + struct rb_node *rbnode; + struct kbase_va_region *reg; + + rbnode = rbtree->rb_node; + + while (rbnode) { + u64 tmp_start_pfn, tmp_end_pfn; + + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + tmp_start_pfn = reg->start_pfn; + tmp_end_pfn = reg->start_pfn + reg->nr_pages; + + /* If start is lower than this, go left. */ + if (gpu_pfn < tmp_start_pfn) + rbnode = rbnode->rb_left; + /* If end is higher than this, then go right. */ + else if (gpu_pfn >= tmp_end_pfn) + rbnode = rbnode->rb_right; + else /* Enclosing */ + return reg; + } + + return NULL; +} + +/* Find region enclosing given address. */ +struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address( + struct kbase_context *kctx, u64 gpu_addr) +{ + u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; + struct rb_root *rbtree = NULL; + + KBASE_DEBUG_ASSERT(NULL != kctx); + + lockdep_assert_held(&kctx->reg_lock); + + rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); + + return kbase_find_region_enclosing_address(rbtree, gpu_addr); +} + +KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address); + +struct kbase_va_region *kbase_find_region_base_address( + struct rb_root *rbtree, u64 gpu_addr) +{ + u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; + struct rb_node *rbnode = NULL; + struct kbase_va_region *reg = NULL; + + rbnode = rbtree->rb_node; + + while (rbnode) { + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + if (reg->start_pfn > gpu_pfn) + rbnode = rbnode->rb_left; + else if (reg->start_pfn < gpu_pfn) + rbnode = rbnode->rb_right; + else + return reg; + } + + return NULL; +} + +/* Find region with given base address */ +struct kbase_va_region *kbase_region_tracker_find_region_base_address( + struct kbase_context *kctx, u64 gpu_addr) +{ + u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; + struct rb_root *rbtree = NULL; + + lockdep_assert_held(&kctx->reg_lock); + + rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); + + return kbase_find_region_base_address(rbtree, gpu_addr); +} + +KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address); + +/* Find region meeting given requirements */ +static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs( + struct kbase_va_region *reg_reqs, + size_t nr_pages, size_t align_offset, size_t align_mask, + u64 *out_start_pfn) +{ + struct rb_node *rbnode = NULL; + struct kbase_va_region *reg = NULL; + struct rb_root *rbtree = NULL; + + /* Note that this search is a linear search, as we do not have a target + address in mind, so does not benefit from the rbtree search */ + rbtree = reg_reqs->rbtree; + + for (rbnode = rb_first(rbtree); rbnode; rbnode = rb_next(rbnode)) { + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + if ((reg->nr_pages >= nr_pages) && + (reg->flags & KBASE_REG_FREE)) { + /* Check alignment */ + u64 start_pfn = reg->start_pfn; + + /* When align_offset == align, this sequence is + * equivalent to: + * (start_pfn + align_mask) & ~(align_mask) + * + * Otherwise, it aligns to n*align + offset, for the + * lowest value n that makes this still >start_pfn */ + start_pfn += align_mask; + start_pfn -= (start_pfn - align_offset) & (align_mask); + + if (!(reg_reqs->flags & KBASE_REG_GPU_NX)) { + /* Can't end at 4GB boundary */ + if (0 == ((start_pfn + nr_pages) & BASE_MEM_PFN_MASK_4GB)) + start_pfn += align_offset; + + /* Can't start at 4GB boundary */ + if (0 == (start_pfn & BASE_MEM_PFN_MASK_4GB)) + start_pfn += align_offset; + + if (!((start_pfn + nr_pages) & BASE_MEM_PFN_MASK_4GB) || + !(start_pfn & BASE_MEM_PFN_MASK_4GB)) + continue; + } else if (reg_reqs->flags & + KBASE_REG_GPU_VA_SAME_4GB_PAGE) { + u64 end_pfn = start_pfn + nr_pages - 1; + + if ((start_pfn & ~BASE_MEM_PFN_MASK_4GB) != + (end_pfn & ~BASE_MEM_PFN_MASK_4GB)) + start_pfn = end_pfn & ~BASE_MEM_PFN_MASK_4GB; + } + + if ((start_pfn >= reg->start_pfn) && + (start_pfn <= (reg->start_pfn + reg->nr_pages - 1)) && + ((start_pfn + nr_pages - 1) <= (reg->start_pfn + reg->nr_pages - 1))) { + *out_start_pfn = start_pfn; + return reg; + } + } + } + + return NULL; +} + +/** + * @brief Remove a region object from the global list. + * + * The region reg is removed, possibly by merging with other free and + * compatible adjacent regions. It must be called with the context + * region lock held. The associated memory is not released (see + * kbase_free_alloced_region). Internal use only. + */ +int kbase_remove_va_region(struct kbase_va_region *reg) +{ + struct rb_node *rbprev; + struct kbase_va_region *prev = NULL; + struct rb_node *rbnext; + struct kbase_va_region *next = NULL; + struct rb_root *reg_rbtree = NULL; + + int merged_front = 0; + int merged_back = 0; + int err = 0; + + reg_rbtree = reg->rbtree; + + /* Try to merge with the previous block first */ + rbprev = rb_prev(&(reg->rblink)); + if (rbprev) { + prev = rb_entry(rbprev, struct kbase_va_region, rblink); + if (prev->flags & KBASE_REG_FREE) { + /* We're compatible with the previous VMA, + * merge with it */ + WARN_ON((prev->flags & KBASE_REG_ZONE_MASK) != + (reg->flags & KBASE_REG_ZONE_MASK)); + prev->nr_pages += reg->nr_pages; + rb_erase(&(reg->rblink), reg_rbtree); + reg = prev; + merged_front = 1; + } + } + + /* Try to merge with the next block second */ + /* Note we do the lookup here as the tree may have been rebalanced. */ + rbnext = rb_next(&(reg->rblink)); + if (rbnext) { + /* We're compatible with the next VMA, merge with it */ + next = rb_entry(rbnext, struct kbase_va_region, rblink); + if (next->flags & KBASE_REG_FREE) { + WARN_ON((next->flags & KBASE_REG_ZONE_MASK) != + (reg->flags & KBASE_REG_ZONE_MASK)); + next->start_pfn = reg->start_pfn; + next->nr_pages += reg->nr_pages; + rb_erase(&(reg->rblink), reg_rbtree); + merged_back = 1; + if (merged_front) { + /* We already merged with prev, free it */ + kfree(reg); + } + } + } + + /* If we failed to merge then we need to add a new block */ + if (!(merged_front || merged_back)) { + /* + * We didn't merge anything. Add a new free + * placeholder and remove the original one. + */ + struct kbase_va_region *free_reg; + + free_reg = kbase_alloc_free_region(reg_rbtree, + reg->start_pfn, reg->nr_pages, + reg->flags & KBASE_REG_ZONE_MASK); + if (!free_reg) { + err = -ENOMEM; + goto out; + } + rb_replace_node(&(reg->rblink), &(free_reg->rblink), reg_rbtree); + } + + out: + return err; +} + +KBASE_EXPORT_TEST_API(kbase_remove_va_region); + +/** + * kbase_insert_va_region_nolock - Insert a VA region to the list, + * replacing the existing one. + * + * @new_reg: The new region to insert + * @at_reg: The region to replace + * @start_pfn: The Page Frame Number to insert at + * @nr_pages: The number of pages of the region + */ +static int kbase_insert_va_region_nolock(struct kbase_va_region *new_reg, + struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages) +{ + struct rb_root *reg_rbtree = NULL; + int err = 0; + + reg_rbtree = at_reg->rbtree; + + /* Must be a free region */ + KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0); + /* start_pfn should be contained within at_reg */ + KBASE_DEBUG_ASSERT((start_pfn >= at_reg->start_pfn) && (start_pfn < at_reg->start_pfn + at_reg->nr_pages)); + /* at least nr_pages from start_pfn should be contained within at_reg */ + KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= at_reg->start_pfn + at_reg->nr_pages); + + new_reg->start_pfn = start_pfn; + new_reg->nr_pages = nr_pages; + + /* Regions are a whole use, so swap and delete old one. */ + if (at_reg->start_pfn == start_pfn && at_reg->nr_pages == nr_pages) { + rb_replace_node(&(at_reg->rblink), &(new_reg->rblink), + reg_rbtree); + kfree(at_reg); + } + /* New region replaces the start of the old one, so insert before. */ + else if (at_reg->start_pfn == start_pfn) { + at_reg->start_pfn += nr_pages; + KBASE_DEBUG_ASSERT(at_reg->nr_pages >= nr_pages); + at_reg->nr_pages -= nr_pages; + + kbase_region_tracker_insert(new_reg); + } + /* New region replaces the end of the old one, so insert after. */ + else if ((at_reg->start_pfn + at_reg->nr_pages) == (start_pfn + nr_pages)) { + at_reg->nr_pages -= nr_pages; + + kbase_region_tracker_insert(new_reg); + } + /* New region splits the old one, so insert and create new */ + else { + struct kbase_va_region *new_front_reg; + + new_front_reg = kbase_alloc_free_region(reg_rbtree, + at_reg->start_pfn, + start_pfn - at_reg->start_pfn, + at_reg->flags & KBASE_REG_ZONE_MASK); + + if (new_front_reg) { + at_reg->nr_pages -= nr_pages + new_front_reg->nr_pages; + at_reg->start_pfn = start_pfn + nr_pages; + + kbase_region_tracker_insert(new_front_reg); + kbase_region_tracker_insert(new_reg); + } else { + err = -ENOMEM; + } + } + + return err; +} + +/** + * kbase_add_va_region - Add a VA region to the region list for a context. + * + * @kctx: kbase context containing the region + * @reg: the region to add + * @addr: the address to insert the region at + * @nr_pages: the number of pages in the region + * @align: the minimum alignment in pages + */ +int kbase_add_va_region(struct kbase_context *kctx, + struct kbase_va_region *reg, u64 addr, + size_t nr_pages, size_t align) +{ + int err = 0; + struct kbase_device *kbdev = kctx->kbdev; + int cpu_va_bits = kbase_get_num_cpu_va_bits(kctx); + int gpu_pc_bits = + kbdev->gpu_props.props.core_props.log2_program_counter_size; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != reg); + + lockdep_assert_held(&kctx->reg_lock); + + /* The executable allocation from the SAME_VA zone would already have an + * appropriately aligned GPU VA chosen for it. + * Also the executable allocation from EXEC_VA zone doesn't need the + * special alignment. + */ + if (!(reg->flags & KBASE_REG_GPU_NX) && !addr && + ((reg->flags & KBASE_REG_ZONE_MASK) != KBASE_REG_ZONE_EXEC_VA)) { + if (cpu_va_bits > gpu_pc_bits) { + align = max(align, (size_t)((1ULL << gpu_pc_bits) + >> PAGE_SHIFT)); + } + } + + do { + err = kbase_add_va_region_rbtree(kbdev, reg, addr, nr_pages, + align); + if (err != -ENOMEM) + break; + + /* + * If the allocation is not from the same zone as JIT + * then don't retry, we're out of VA and there is + * nothing which can be done about it. + */ + if ((reg->flags & KBASE_REG_ZONE_MASK) != + KBASE_REG_ZONE_CUSTOM_VA) + break; + } while (kbase_jit_evict(kctx)); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_add_va_region); + +/** + * kbase_add_va_region_rbtree - Insert a region into its corresponding rbtree + * + * Insert a region into the rbtree that was specified when the region was + * created. If addr is 0 a free area in the rbtree is used, otherwise the + * specified address is used. + * + * @kbdev: The kbase device + * @reg: The region to add + * @addr: The address to add the region at, or 0 to map at any available address + * @nr_pages: The size of the region in pages + * @align: The minimum alignment in pages + */ +int kbase_add_va_region_rbtree(struct kbase_device *kbdev, + struct kbase_va_region *reg, + u64 addr, size_t nr_pages, size_t align) +{ + struct device *const dev = kbdev->dev; + struct rb_root *rbtree = NULL; + struct kbase_va_region *tmp; + u64 gpu_pfn = addr >> PAGE_SHIFT; + int err = 0; + + rbtree = reg->rbtree; + + if (!align) + align = 1; + + /* must be a power of 2 */ + KBASE_DEBUG_ASSERT(is_power_of_2(align)); + KBASE_DEBUG_ASSERT(nr_pages > 0); + + /* Path 1: Map a specific address. Find the enclosing region, + * which *must* be free. + */ + if (gpu_pfn) { + KBASE_DEBUG_ASSERT(!(gpu_pfn & (align - 1))); + + tmp = find_region_enclosing_range_rbtree(rbtree, gpu_pfn, + nr_pages); + if (kbase_is_region_invalid(tmp)) { + dev_warn(dev, "Enclosing region not found or invalid: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages); + err = -ENOMEM; + goto exit; + } else if (!kbase_is_region_free(tmp)) { + dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", + tmp->start_pfn, tmp->flags, + tmp->nr_pages, gpu_pfn, nr_pages); + err = -ENOMEM; + goto exit; + } + + err = kbase_insert_va_region_nolock(reg, tmp, gpu_pfn, + nr_pages); + if (err) { + dev_warn(dev, "Failed to insert va region"); + err = -ENOMEM; + } + } else { + /* Path 2: Map any free address which meets the requirements. */ + u64 start_pfn; + size_t align_offset = align; + size_t align_mask = align - 1; + +#if !MALI_USE_CSF + if ((reg->flags & KBASE_REG_TILER_ALIGN_TOP)) { + WARN(align > 1, "%s with align %lx might not be honored for KBASE_REG_TILER_ALIGN_TOP memory", + __func__, + (unsigned long)align); + align_mask = reg->extent - 1; + align_offset = reg->extent - reg->initial_commit; + } +#endif /* !MALI_USE_CSF */ + + tmp = kbase_region_tracker_find_region_meeting_reqs(reg, + nr_pages, align_offset, align_mask, + &start_pfn); + if (tmp) { + err = kbase_insert_va_region_nolock(reg, tmp, + start_pfn, nr_pages); + if (unlikely(err)) { + dev_warn(dev, "Failed to insert region: 0x%08llx start_pfn, %zu nr_pages", + start_pfn, nr_pages); + } + } else { + dev_dbg(dev, "Failed to find a suitable region: %zu nr_pages, %zu align_offset, %zu align_mask\n", + nr_pages, align_offset, align_mask); + err = -ENOMEM; + } + } + +exit: + return err; +} + +/** + * @brief Initialize the internal region tracker data structure. + */ +static void kbase_region_tracker_ds_init(struct kbase_context *kctx, + struct kbase_va_region *same_va_reg, + struct kbase_va_region *custom_va_reg) +{ + kctx->reg_rbtree_same = RB_ROOT; + kbase_region_tracker_insert(same_va_reg); + + /* Although custom_va_reg and exec_va_reg don't always exist, + * initialize unconditionally because of the mem_view debugfs + * implementation which relies on them being empty. + * + * The difference between the two is that the EXEC_VA region + * is never initialized at this stage. + */ + kctx->reg_rbtree_custom = RB_ROOT; + kctx->reg_rbtree_exec = RB_ROOT; + + if (custom_va_reg) + kbase_region_tracker_insert(custom_va_reg); +} + +static void kbase_region_tracker_erase_rbtree(struct rb_root *rbtree) +{ + struct rb_node *rbnode; + struct kbase_va_region *reg; + + do { + rbnode = rb_first(rbtree); + if (rbnode) { + rb_erase(rbnode, rbtree); + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + WARN_ON(reg->va_refcnt != 1); + /* Reset the start_pfn - as the rbtree is being + * destroyed and we've already erased this region, there + * is no further need to attempt to remove it. + * This won't affect the cleanup if the region was + * being used as a sticky resource as the cleanup + * related to sticky resources anyways need to be + * performed before the term of region tracker. + */ + reg->start_pfn = 0; + kbase_free_alloced_region(reg); + } + } while (rbnode); +} + +void kbase_region_tracker_term(struct kbase_context *kctx) +{ + kbase_gpu_vm_lock(kctx); + kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_same); + kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_custom); + kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_exec); +#if MALI_USE_CSF + WARN_ON(!list_empty(&kctx->csf.event_pages_head)); +#endif + kbase_gpu_vm_unlock(kctx); +} + +void kbase_region_tracker_term_rbtree(struct rb_root *rbtree) +{ + kbase_region_tracker_erase_rbtree(rbtree); +} + +static size_t kbase_get_same_va_bits(struct kbase_context *kctx) +{ + return min(kbase_get_num_cpu_va_bits(kctx), + (size_t) kctx->kbdev->gpu_props.mmu.va_bits); +} + +int kbase_region_tracker_init(struct kbase_context *kctx) +{ + struct kbase_va_region *same_va_reg; + struct kbase_va_region *custom_va_reg = NULL; + size_t same_va_bits = kbase_get_same_va_bits(kctx); + u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE; + u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT; + u64 same_va_pages; + int err; + + /* Take the lock as kbase_free_alloced_region requires it */ + kbase_gpu_vm_lock(kctx); + + same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; + /* all have SAME_VA */ + same_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 1, + same_va_pages, + KBASE_REG_ZONE_SAME_VA); + + if (!same_va_reg) { + err = -ENOMEM; + goto fail_unlock; + } + +#ifdef CONFIG_64BIT + /* 32-bit clients have custom VA zones */ + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { +#endif + if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) { + err = -EINVAL; + goto fail_free_same_va; + } + /* If the current size of TMEM is out of range of the + * virtual address space addressable by the MMU then + * we should shrink it to fit + */ + if ((KBASE_REG_ZONE_CUSTOM_VA_BASE + KBASE_REG_ZONE_CUSTOM_VA_SIZE) >= gpu_va_limit) + custom_va_size = gpu_va_limit - KBASE_REG_ZONE_CUSTOM_VA_BASE; + + custom_va_reg = kbase_alloc_free_region( + &kctx->reg_rbtree_custom, + KBASE_REG_ZONE_CUSTOM_VA_BASE, + custom_va_size, KBASE_REG_ZONE_CUSTOM_VA); + + if (!custom_va_reg) { + err = -ENOMEM; + goto fail_free_same_va; + } +#ifdef CONFIG_64BIT + } else { + custom_va_size = 0; + } +#endif + + kbase_region_tracker_ds_init(kctx, same_va_reg, custom_va_reg); + + kctx->same_va_end = same_va_pages + 1; + kctx->gpu_va_end = kctx->same_va_end + custom_va_size; + kctx->exec_va_start = U64_MAX; + kctx->jit_va = false; + +#if MALI_USE_CSF + INIT_LIST_HEAD(&kctx->csf.event_pages_head); +#endif + + kbase_gpu_vm_unlock(kctx); + return 0; + +fail_free_same_va: + kbase_free_alloced_region(same_va_reg); +fail_unlock: + kbase_gpu_vm_unlock(kctx); + return err; +} + +#ifdef CONFIG_64BIT +static int kbase_region_tracker_init_jit_64(struct kbase_context *kctx, + u64 jit_va_pages) +{ + struct kbase_va_region *same_va; + struct kbase_va_region *custom_va_reg; + + lockdep_assert_held(&kctx->reg_lock); + + /* First verify that a JIT_VA zone has not been created already. */ + if (kctx->jit_va) + return -EINVAL; + + /* + * Modify the same VA free region after creation. Be careful to ensure + * that allocations haven't been made as they could cause an overlap + * to happen with existing same VA allocations and the custom VA zone. + */ + same_va = kbase_region_tracker_find_region_base_address(kctx, + PAGE_SIZE); + if (!same_va) + return -ENOMEM; + + if (same_va->nr_pages < jit_va_pages || kctx->same_va_end < jit_va_pages) + return -ENOMEM; + + /* It's safe to adjust the same VA zone now */ + same_va->nr_pages -= jit_va_pages; + kctx->same_va_end -= jit_va_pages; + + /* + * Create a custom VA zone at the end of the VA for allocations which + * JIT can use so it doesn't have to allocate VA from the kernel. + */ + custom_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_custom, + kctx->same_va_end, + jit_va_pages, + KBASE_REG_ZONE_CUSTOM_VA); + + /* + * The context will be destroyed if we fail here so no point + * reverting the change we made to same_va. + */ + if (!custom_va_reg) + return -ENOMEM; + + kbase_region_tracker_insert(custom_va_reg); + return 0; +} +#endif + +int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, + int max_allocations, int trim_level, int group_id, + u64 phys_pages_limit) +{ + int err = 0; + + if (trim_level < 0 || trim_level > BASE_JIT_MAX_TRIM_LEVEL) + return -EINVAL; + + if (group_id < 0 || group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS) + return -EINVAL; + + if (phys_pages_limit > jit_va_pages) + return -EINVAL; + +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (phys_pages_limit != jit_va_pages) + kbase_ctx_flag_set(kctx, KCTX_JPL_ENABLED); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + kbase_gpu_vm_lock(kctx); + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) + err = kbase_region_tracker_init_jit_64(kctx, jit_va_pages); +#endif + /* + * Nothing to do for 32-bit clients, JIT uses the existing + * custom VA zone. + */ + + if (!err) { + kctx->jit_max_allocations = max_allocations; + kctx->trim_level = trim_level; + kctx->jit_va = true; + kctx->jit_group_id = group_id; +#if MALI_JIT_PRESSURE_LIMIT_BASE + kctx->jit_phys_pages_limit = phys_pages_limit; + dev_dbg(kctx->kbdev->dev, "phys_pages_limit set to %llu\n", + phys_pages_limit); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + } + + kbase_gpu_vm_unlock(kctx); + + return err; +} + +int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages) +{ + struct kbase_va_region *shrinking_va_reg; + struct kbase_va_region *exec_va_reg; + u64 exec_va_start, exec_va_base_addr; + int err; + + /* The EXEC_VA zone shall be created by making space at the end of the + * address space. Firstly, verify that the number of EXEC_VA pages + * requested by the client is reasonable and then make sure that it is + * not greater than the address space itself before calculating the base + * address of the new zone. + */ + if (exec_va_pages == 0 || exec_va_pages > KBASE_REG_ZONE_EXEC_VA_MAX_PAGES) + return -EINVAL; + + kbase_gpu_vm_lock(kctx); + + /* First verify that a JIT_VA zone has not been created already. */ + if (kctx->jit_va) { + err = -EPERM; + goto exit_unlock; + } + + if (exec_va_pages > kctx->gpu_va_end) { + err = -ENOMEM; + goto exit_unlock; + } + + exec_va_start = kctx->gpu_va_end - exec_va_pages; + exec_va_base_addr = exec_va_start << PAGE_SHIFT; + + shrinking_va_reg = kbase_region_tracker_find_region_enclosing_address(kctx, + exec_va_base_addr); + if (!shrinking_va_reg) { + err = -ENOMEM; + goto exit_unlock; + } + + /* Make sure that the EXEC_VA region is still uninitialized */ + if ((shrinking_va_reg->flags & KBASE_REG_ZONE_MASK) == + KBASE_REG_ZONE_EXEC_VA) { + err = -EPERM; + goto exit_unlock; + } + + if (shrinking_va_reg->nr_pages <= exec_va_pages) { + err = -ENOMEM; + goto exit_unlock; + } + + exec_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_exec, + exec_va_start, + exec_va_pages, + KBASE_REG_ZONE_EXEC_VA); + if (!exec_va_reg) { + err = -ENOMEM; + goto exit_unlock; + } + + shrinking_va_reg->nr_pages -= exec_va_pages; +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) + kctx->same_va_end -= exec_va_pages; +#endif + kctx->exec_va_start = exec_va_start; + + kbase_region_tracker_insert(exec_va_reg); + err = 0; + +exit_unlock: + kbase_gpu_vm_unlock(kctx); + return err; +} + +#if MALI_USE_CSF +void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev) +{ + kbase_region_tracker_term_rbtree(&kbdev->csf.shared_reg_rbtree); +} + +int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev) +{ + struct kbase_va_region *shared_reg; + u64 shared_reg_start_pfn; + u64 shared_reg_size; + + shared_reg_start_pfn = KBASE_REG_ZONE_MCU_SHARED_BASE; + shared_reg_size = KBASE_REG_ZONE_MCU_SHARED_SIZE; + + kbdev->csf.shared_reg_rbtree = RB_ROOT; + + shared_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, + shared_reg_start_pfn, + shared_reg_size, + KBASE_REG_ZONE_MCU_SHARED); + if (!shared_reg) + return -ENOMEM; + + kbase_region_tracker_insert(shared_reg); + return 0; +} +#endif + +int kbase_mem_init(struct kbase_device *kbdev) +{ + int err = 0; + struct kbasep_mem_device *memdev; +#ifdef CONFIG_OF + struct device_node *mgm_node = NULL; +#endif + + KBASE_DEBUG_ASSERT(kbdev); + + memdev = &kbdev->memdev; + + kbase_mem_pool_group_config_set_max_size(&kbdev->mem_pool_defaults, + KBASE_MEM_POOL_MAX_SIZE_KCTX); + + /* Initialize memory usage */ + atomic_set(&memdev->used_pages, 0); + + spin_lock_init(&kbdev->gpu_mem_usage_lock); + kbdev->total_gpu_pages = 0; + kbdev->process_root = RB_ROOT; + kbdev->dma_buf_root = RB_ROOT; + mutex_init(&kbdev->dma_buf_lock); + +#ifdef IR_THRESHOLD + atomic_set(&memdev->ir_threshold, IR_THRESHOLD); +#else + atomic_set(&memdev->ir_threshold, DEFAULT_IR_THRESHOLD); +#endif + + kbdev->mgm_dev = &kbase_native_mgm_dev; + +#ifdef CONFIG_OF + /* Check to see whether or not a platform-specific memory group manager + * is configured and available. + */ + mgm_node = of_parse_phandle(kbdev->dev->of_node, + "physical-memory-group-manager", 0); + if (!mgm_node) { + dev_info(kbdev->dev, + "No memory group manager is configured\n"); + } else { + struct platform_device *const pdev = + of_find_device_by_node(mgm_node); + + if (!pdev) { + dev_err(kbdev->dev, + "The configured memory group manager was not found\n"); + } else { + kbdev->mgm_dev = platform_get_drvdata(pdev); + if (!kbdev->mgm_dev) { + dev_info(kbdev->dev, + "Memory group manager is not ready\n"); + err = -EPROBE_DEFER; + } else if (!try_module_get(kbdev->mgm_dev->owner)) { + dev_err(kbdev->dev, + "Failed to get memory group manger module\n"); + err = -ENODEV; + kbdev->mgm_dev = NULL; + } else { + dev_info(kbdev->dev, + "Memory group manager successfully loaded\n"); + } + } + of_node_put(mgm_node); + } +#endif + + if (likely(!err)) { + struct kbase_mem_pool_group_config mem_pool_defaults; + + kbase_mem_pool_group_config_set_max_size(&mem_pool_defaults, + KBASE_MEM_POOL_MAX_SIZE_KBDEV); + + err = kbase_mem_pool_group_init(&kbdev->mem_pools, kbdev, + &mem_pool_defaults, NULL); + } + + return err; +} + +void kbase_mem_halt(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +void kbase_mem_term(struct kbase_device *kbdev) +{ + struct kbasep_mem_device *memdev; + int pages; + + KBASE_DEBUG_ASSERT(kbdev); + + memdev = &kbdev->memdev; + + pages = atomic_read(&memdev->used_pages); + if (pages != 0) + dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); + + kbase_mem_pool_group_term(&kbdev->mem_pools); + + WARN_ON(kbdev->total_gpu_pages); + WARN_ON(!RB_EMPTY_ROOT(&kbdev->process_root)); + WARN_ON(!RB_EMPTY_ROOT(&kbdev->dma_buf_root)); + mutex_destroy(&kbdev->dma_buf_lock); + + if (kbdev->mgm_dev) + module_put(kbdev->mgm_dev->owner); +} +KBASE_EXPORT_TEST_API(kbase_mem_term); + +/** + * @brief Allocate a free region object. + * + * The allocated object is not part of any list yet, and is flagged as + * KBASE_REG_FREE. No mapping is allocated yet. + * + * zone is KBASE_REG_ZONE_CUSTOM_VA or KBASE_REG_ZONE_SAME_VA. + * + */ +struct kbase_va_region *kbase_alloc_free_region(struct rb_root *rbtree, + u64 start_pfn, size_t nr_pages, int zone) +{ + struct kbase_va_region *new_reg; + + KBASE_DEBUG_ASSERT(rbtree != NULL); + + /* zone argument should only contain zone related region flags */ + KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0); + KBASE_DEBUG_ASSERT(nr_pages > 0); + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE)); + + new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL); + + if (!new_reg) + return NULL; + + new_reg->va_refcnt = 1; + new_reg->cpu_alloc = NULL; /* no alloc bound yet */ + new_reg->gpu_alloc = NULL; /* no alloc bound yet */ + new_reg->rbtree = rbtree; + new_reg->flags = zone | KBASE_REG_FREE; + + new_reg->flags |= KBASE_REG_GROWABLE; + + new_reg->start_pfn = start_pfn; + new_reg->nr_pages = nr_pages; + + INIT_LIST_HEAD(&new_reg->jit_node); + INIT_LIST_HEAD(&new_reg->link); + + return new_reg; +} + +KBASE_EXPORT_TEST_API(kbase_alloc_free_region); + +static struct kbase_context *kbase_reg_flags_to_kctx( + struct kbase_va_region *reg) +{ + struct kbase_context *kctx = NULL; + struct rb_root *rbtree = reg->rbtree; + + switch (reg->flags & KBASE_REG_ZONE_MASK) { + case KBASE_REG_ZONE_CUSTOM_VA: + kctx = container_of(rbtree, struct kbase_context, + reg_rbtree_custom); + break; + case KBASE_REG_ZONE_SAME_VA: + kctx = container_of(rbtree, struct kbase_context, + reg_rbtree_same); + break; + case KBASE_REG_ZONE_EXEC_VA: + kctx = container_of(rbtree, struct kbase_context, + reg_rbtree_exec); + break; + default: + WARN(1, "Unknown zone in region: flags=0x%lx\n", reg->flags); + break; + } + + return kctx; +} + +/** + * @brief Free a region object. + * + * The described region must be freed of any mapping. + * + * If the region is not flagged as KBASE_REG_FREE, the region's + * alloc object will be released. + * It is a bug if no alloc object exists for non-free regions. + * + */ +void kbase_free_alloced_region(struct kbase_va_region *reg) +{ +#if MALI_USE_CSF + if ((reg->flags & KBASE_REG_ZONE_MASK) == + KBASE_REG_ZONE_MCU_SHARED) { + kfree(reg); + return; + } +#endif + if (!(reg->flags & KBASE_REG_FREE)) { + struct kbase_context *kctx = kbase_reg_flags_to_kctx(reg); + + if (WARN_ON(!kctx)) + return; + + if (WARN_ON(kbase_is_region_invalid(reg))) + return; + + dev_dbg(kctx->kbdev->dev, "Freeing memory region %p\n", + (void *)reg); +#if MALI_USE_CSF + if (reg->flags & KBASE_REG_CSF_EVENT) + kbase_unlink_event_mem_page(kctx, reg); +#endif + + mutex_lock(&kctx->jit_evict_lock); + + /* + * The physical allocation should have been removed from the + * eviction list before this function is called. However, in the + * case of abnormal process termination or the app leaking the + * memory kbase_mem_free_region is not called so it can still be + * on the list at termination time of the region tracker. + */ + if (!list_empty(®->gpu_alloc->evict_node)) { + mutex_unlock(&kctx->jit_evict_lock); + + /* + * Unlink the physical allocation before unmaking it + * evictable so that the allocation isn't grown back to + * its last backed size as we're going to unmap it + * anyway. + */ + reg->cpu_alloc->reg = NULL; + if (reg->cpu_alloc != reg->gpu_alloc) + reg->gpu_alloc->reg = NULL; + + /* + * If a region has been made evictable then we must + * unmake it before trying to free it. + * If the memory hasn't been reclaimed it will be + * unmapped and freed below, if it has been reclaimed + * then the operations below are no-ops. + */ + if (reg->flags & KBASE_REG_DONT_NEED) { + KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == + KBASE_MEM_TYPE_NATIVE); + kbase_mem_evictable_unmake(reg->gpu_alloc); + } + } else { + mutex_unlock(&kctx->jit_evict_lock); + } + + /* + * Remove the region from the sticky resource metadata + * list should it be there. + */ + kbase_sticky_resource_release_force(kctx, NULL, + reg->start_pfn << PAGE_SHIFT); + + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); + + reg->flags |= KBASE_REG_VA_FREED; + kbase_va_region_alloc_put(kctx, reg); + } else { + kfree(reg); + } +} + +KBASE_EXPORT_TEST_API(kbase_free_alloced_region); + +int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align) +{ + int err; + size_t i = 0; + unsigned long attr; + unsigned long mask = ~KBASE_REG_MEMATTR_MASK; + unsigned long gwt_mask = ~0; + int group_id; + struct kbase_mem_phy_alloc *alloc; + +#ifdef CONFIG_MALI_CINSTR_GWT + if (kctx->gwt_enabled) + gwt_mask = ~KBASE_REG_GPU_WR; +#endif + + if ((kctx->kbdev->system_coherency == COHERENCY_ACE) && + (reg->flags & KBASE_REG_SHARE_BOTH)) + attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA); + else + attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC); + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != reg); + + err = kbase_add_va_region(kctx, reg, addr, nr_pages, align); + if (err) + return err; + + alloc = reg->gpu_alloc; + group_id = alloc->group_id; + + if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { + u64 const stride = alloc->imported.alias.stride; + + KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); + for (i = 0; i < alloc->imported.alias.nents; i++) { + if (alloc->imported.alias.aliased[i].alloc) { + err = kbase_mmu_insert_pages(kctx->kbdev, + &kctx->mmu, + reg->start_pfn + (i * stride), + alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset, + alloc->imported.alias.aliased[i].length, + reg->flags & gwt_mask, + kctx->as_nr, + group_id); + if (err) + goto bad_insert; + + kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc); + } else { + err = kbase_mmu_insert_single_page(kctx, + reg->start_pfn + i * stride, + kctx->aliasing_sink_page, + alloc->imported.alias.aliased[i].length, + (reg->flags & mask & gwt_mask) | attr, + group_id); + + if (err) + goto bad_insert; + } + } + } else { + err = kbase_mmu_insert_pages(kctx->kbdev, + &kctx->mmu, + reg->start_pfn, + kbase_get_gpu_phy_pages(reg), + kbase_reg_current_backed_size(reg), + reg->flags & gwt_mask, + kctx->as_nr, + group_id); + if (err) + goto bad_insert; + kbase_mem_phy_alloc_gpu_mapped(alloc); + } + + if (reg->flags & KBASE_REG_IMPORT_PAD && + !WARN_ON(reg->nr_pages < reg->gpu_alloc->nents) && + reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM && + reg->gpu_alloc->imported.umm.current_mapping_usage_count) { + /* For padded imported dma-buf memory, map the dummy aliasing + * page from the end of the dma-buf pages, to the end of the + * region using a read only mapping. + * + * Only map when it's imported dma-buf memory that is currently + * mapped. + * + * Assume reg->gpu_alloc->nents is the number of actual pages + * in the dma-buf memory. + */ + err = kbase_mmu_insert_single_page(kctx, + reg->start_pfn + reg->gpu_alloc->nents, + kctx->aliasing_sink_page, + reg->nr_pages - reg->gpu_alloc->nents, + (reg->flags | KBASE_REG_GPU_RD) & + ~KBASE_REG_GPU_WR, + KBASE_MEM_GROUP_SINK); + if (err) + goto bad_insert; + } + + return err; + +bad_insert: + kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, + reg->start_pfn, reg->nr_pages, + kctx->as_nr); + + if (alloc->type == KBASE_MEM_TYPE_ALIAS) { + KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); + while (i--) + if (alloc->imported.alias.aliased[i].alloc) + kbase_mem_phy_alloc_gpu_unmapped(alloc->imported.alias.aliased[i].alloc); + } + + kbase_remove_va_region(reg); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_gpu_mmap); + +static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc, bool writeable); + +int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + int err = 0; + size_t i; + + if (reg->start_pfn == 0) + return 0; + + if (!reg->gpu_alloc) + return -EINVAL; + + /* Tear down down GPU page tables, depending on memory type. */ + switch (reg->gpu_alloc->type) { + case KBASE_MEM_TYPE_ALIAS: /* Fall-through */ + case KBASE_MEM_TYPE_IMPORTED_UMM: + err = kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, + reg->start_pfn, reg->nr_pages, kctx->as_nr); + break; + default: + err = kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, + reg->start_pfn, kbase_reg_current_backed_size(reg), + kctx->as_nr); + break; + } + + /* Update tracking, and other cleanup, depending on memory type. */ + switch (reg->gpu_alloc->type) { + case KBASE_MEM_TYPE_ALIAS: + KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); + for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++) + if (reg->gpu_alloc->imported.alias.aliased[i].alloc) + kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); + break; + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { + struct kbase_alloc_import_user_buf *user_buf = + ®->gpu_alloc->imported.user_buf; + + if (user_buf->current_mapping_usage_count & PINNED_ON_IMPORT) { + user_buf->current_mapping_usage_count &= + ~PINNED_ON_IMPORT; + + /* The allocation could still have active mappings. */ + if (user_buf->current_mapping_usage_count == 0) { + kbase_jd_user_buf_unmap(kctx, reg->gpu_alloc, + (reg->flags & KBASE_REG_GPU_WR)); + } + } + } + /* Fall-through */ + default: + kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc); + break; + } + + return err; +} + +static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( + struct kbase_context *kctx, + unsigned long uaddr, size_t size, u64 *offset) +{ + struct vm_area_struct *vma; + struct kbase_cpu_mapping *map; + unsigned long vm_pgoff_in_region; + unsigned long vm_off_in_region; + unsigned long map_start; + size_t map_size; + + lockdep_assert_held(kbase_mem_get_process_mmap_lock()); + + if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */ + return NULL; + + vma = find_vma_intersection(current->mm, uaddr, uaddr+size); + + if (!vma || vma->vm_start > uaddr) + return NULL; + if (vma->vm_ops != &kbase_vm_ops) + /* Not ours! */ + return NULL; + + map = vma->vm_private_data; + + if (map->kctx != kctx) + /* Not from this context! */ + return NULL; + + vm_pgoff_in_region = vma->vm_pgoff - map->region->start_pfn; + vm_off_in_region = vm_pgoff_in_region << PAGE_SHIFT; + map_start = vma->vm_start - vm_off_in_region; + map_size = map->region->nr_pages << PAGE_SHIFT; + + if ((uaddr + size) > (map_start + map_size)) + /* Not within the CPU mapping */ + return NULL; + + *offset = (uaddr - vma->vm_start) + vm_off_in_region; + + return map; +} + +int kbasep_find_enclosing_cpu_mapping_offset( + struct kbase_context *kctx, + unsigned long uaddr, size_t size, u64 *offset) +{ + struct kbase_cpu_mapping *map; + + kbase_os_mem_map_lock(kctx); + + map = kbasep_find_enclosing_cpu_mapping(kctx, uaddr, size, offset); + + kbase_os_mem_map_unlock(kctx); + + if (!map) + return -EINVAL; + + return 0; +} + +KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset); + +int kbasep_find_enclosing_gpu_mapping_start_and_offset(struct kbase_context *kctx, + u64 gpu_addr, size_t size, u64 *start, u64 *offset) +{ + struct kbase_va_region *region; + + kbase_gpu_vm_lock(kctx); + + region = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr); + + if (!region) { + kbase_gpu_vm_unlock(kctx); + return -EINVAL; + } + + *start = region->start_pfn << PAGE_SHIFT; + + *offset = gpu_addr - *start; + + if (((region->start_pfn + region->nr_pages) << PAGE_SHIFT) < (gpu_addr + size)) { + kbase_gpu_vm_unlock(kctx); + return -EINVAL; + } + + kbase_gpu_vm_unlock(kctx); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbasep_find_enclosing_gpu_mapping_start_and_offset); + +void kbase_sync_single(struct kbase_context *kctx, + struct tagged_addr t_cpu_pa, struct tagged_addr t_gpu_pa, + off_t offset, size_t size, enum kbase_sync_type sync_fn) +{ + struct page *cpu_page; + phys_addr_t cpu_pa = as_phys_addr_t(t_cpu_pa); + phys_addr_t gpu_pa = as_phys_addr_t(t_gpu_pa); + + cpu_page = pfn_to_page(PFN_DOWN(cpu_pa)); + + if (likely(cpu_pa == gpu_pa)) { + dma_addr_t dma_addr; + + BUG_ON(!cpu_page); + BUG_ON(offset + size > PAGE_SIZE); + + dma_addr = kbase_dma_addr(cpu_page) + offset; + if (sync_fn == KBASE_SYNC_TO_CPU) + dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr, + size, DMA_BIDIRECTIONAL); + else if (sync_fn == KBASE_SYNC_TO_DEVICE) + dma_sync_single_for_device(kctx->kbdev->dev, dma_addr, + size, DMA_BIDIRECTIONAL); + } else { + void *src = NULL; + void *dst = NULL; + struct page *gpu_page; + + if (WARN(!gpu_pa, "No GPU PA found for infinite cache op")) + return; + + gpu_page = pfn_to_page(PFN_DOWN(gpu_pa)); + + if (sync_fn == KBASE_SYNC_TO_DEVICE) { + src = ((unsigned char *)kmap(cpu_page)) + offset; + dst = ((unsigned char *)kmap(gpu_page)) + offset; + } else if (sync_fn == KBASE_SYNC_TO_CPU) { + dma_sync_single_for_cpu(kctx->kbdev->dev, + kbase_dma_addr(gpu_page) + offset, + size, DMA_BIDIRECTIONAL); + src = ((unsigned char *)kmap(gpu_page)) + offset; + dst = ((unsigned char *)kmap(cpu_page)) + offset; + } + memcpy(dst, src, size); + kunmap(gpu_page); + kunmap(cpu_page); + if (sync_fn == KBASE_SYNC_TO_DEVICE) + dma_sync_single_for_device(kctx->kbdev->dev, + kbase_dma_addr(gpu_page) + offset, + size, DMA_BIDIRECTIONAL); + } +} + +static int kbase_do_syncset(struct kbase_context *kctx, + struct basep_syncset *sset, enum kbase_sync_type sync_fn) +{ + int err = 0; + struct kbase_va_region *reg; + struct kbase_cpu_mapping *map; + unsigned long start; + size_t size; + struct tagged_addr *cpu_pa; + struct tagged_addr *gpu_pa; + u64 page_off, page_count; + u64 i; + u64 offset; + + kbase_os_mem_map_lock(kctx); + kbase_gpu_vm_lock(kctx); + + /* find the region where the virtual address is contained */ + reg = kbase_region_tracker_find_region_enclosing_address(kctx, + sset->mem_handle.basep.handle); + if (kbase_is_region_invalid_or_free(reg)) { + dev_warn(kctx->kbdev->dev, "Can't find a valid region at VA 0x%016llX", + sset->mem_handle.basep.handle); + err = -EINVAL; + goto out_unlock; + } + + /* + * Handle imported memory before checking for KBASE_REG_CPU_CACHED. The + * CPU mapping cacheability is defined by the owner of the imported + * memory, and not by kbase, therefore we must assume that any imported + * memory may be cached. + */ + if (kbase_mem_is_imported(reg->gpu_alloc->type)) { + err = kbase_mem_do_sync_imported(kctx, reg, sync_fn); + goto out_unlock; + } + + if (!(reg->flags & KBASE_REG_CPU_CACHED)) + goto out_unlock; + + start = (uintptr_t)sset->user_addr; + size = (size_t)sset->size; + + map = kbasep_find_enclosing_cpu_mapping(kctx, start, size, &offset); + if (!map) { + dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX", + start, sset->mem_handle.basep.handle); + err = -EINVAL; + goto out_unlock; + } + + page_off = offset >> PAGE_SHIFT; + offset &= ~PAGE_MASK; + page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + cpu_pa = kbase_get_cpu_phy_pages(reg); + gpu_pa = kbase_get_gpu_phy_pages(reg); + + if (page_off > reg->nr_pages || + page_off + page_count > reg->nr_pages) { + /* Sync overflows the region */ + err = -EINVAL; + goto out_unlock; + } + + /* Sync first page */ + if (as_phys_addr_t(cpu_pa[page_off])) { + size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); + + kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off], + offset, sz, sync_fn); + } + + /* Sync middle pages (if any) */ + for (i = 1; page_count > 2 && i < page_count - 1; i++) { + /* we grow upwards, so bail on first non-present page */ + if (!as_phys_addr_t(cpu_pa[page_off + i])) + break; + + kbase_sync_single(kctx, cpu_pa[page_off + i], + gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn); + } + + /* Sync last page (if any) */ + if (page_count > 1 && + as_phys_addr_t(cpu_pa[page_off + page_count - 1])) { + size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1; + + kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1], + gpu_pa[page_off + page_count - 1], 0, sz, + sync_fn); + } + +out_unlock: + kbase_gpu_vm_unlock(kctx); + kbase_os_mem_map_unlock(kctx); + return err; +} + +int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) +{ + int err = -EINVAL; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(sset != NULL); + + if (sset->mem_handle.basep.handle & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, + "mem_handle: passed parameter is invalid"); + return -EINVAL; + } + + switch (sset->type) { + case BASE_SYNCSET_OP_MSYNC: + err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_DEVICE); + break; + + case BASE_SYNCSET_OP_CSYNC: + err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_CPU); + break; + + default: + dev_warn(kctx->kbdev->dev, "Unknown msync op %d\n", sset->type); + break; + } + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_sync_now); + +/* vm lock must be held */ +int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + int err; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != reg); + dev_dbg(kctx->kbdev->dev, "%s %p in kctx %p\n", + __func__, (void *)reg, (void *)kctx); + lockdep_assert_held(&kctx->reg_lock); + + if (reg->flags & KBASE_REG_NO_USER_FREE) { + dev_warn(kctx->kbdev->dev, "Attempt to free GPU memory whose freeing by user space is forbidden!\n"); + return -EINVAL; + } + + /* + * Unlink the physical allocation before unmaking it evictable so + * that the allocation isn't grown back to its last backed size + * as we're going to unmap it anyway. + */ + reg->cpu_alloc->reg = NULL; + if (reg->cpu_alloc != reg->gpu_alloc) + reg->gpu_alloc->reg = NULL; + + /* + * If a region has been made evictable then we must unmake it + * before trying to free it. + * If the memory hasn't been reclaimed it will be unmapped and freed + * below, if it has been reclaimed then the operations below are no-ops. + */ + if (reg->flags & KBASE_REG_DONT_NEED) { + KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == + KBASE_MEM_TYPE_NATIVE); + kbase_mem_evictable_unmake(reg->gpu_alloc); + } + + err = kbase_gpu_munmap(kctx, reg); + if (err) { + dev_warn(kctx->kbdev->dev, "Could not unmap from the GPU...\n"); + goto out; + } + + /* This will also free the physical pages */ + kbase_free_alloced_region(reg); + + out: + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mem_free_region); + +/** + * @brief Free the region from the GPU and unregister it. + * + * This function implements the free operation on a memory segment. + * It will loudly fail if called with outstanding mappings. + */ +int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) +{ + int err = 0; + struct kbase_va_region *reg; + + KBASE_DEBUG_ASSERT(kctx != NULL); + dev_dbg(kctx->kbdev->dev, "%s 0x%llx in kctx %p\n", + __func__, gpu_addr, (void *)kctx); + + if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) { + dev_warn(kctx->kbdev->dev, "kbase_mem_free: gpu_addr parameter is invalid"); + return -EINVAL; + } + + if (0 == gpu_addr) { + dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); + return -EINVAL; + } + kbase_gpu_vm_lock(kctx); + + if (gpu_addr >= BASE_MEM_COOKIE_BASE && + gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) { + int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); + + reg = kctx->pending_regions[cookie]; + if (!reg) { + err = -EINVAL; + goto out_unlock; + } + + /* ask to unlink the cookie as we'll free it */ + + kctx->pending_regions[cookie] = NULL; + bitmap_set(kctx->cookies, cookie, 1); + + kbase_free_alloced_region(reg); + } else { + /* A real GPU va */ + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (kbase_is_region_invalid_or_free(reg)) { + dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX", + gpu_addr); + err = -EINVAL; + goto out_unlock; + } + + if ((reg->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA) { + /* SAME_VA must be freed through munmap */ + dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__, + gpu_addr); + err = -EINVAL; + goto out_unlock; + } + err = kbase_mem_free_region(kctx, reg); + } + + out_unlock: + kbase_gpu_vm_unlock(kctx); + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mem_free); + +int kbase_update_region_flags(struct kbase_context *kctx, + struct kbase_va_region *reg, unsigned long flags) +{ + KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0); + + reg->flags |= kbase_cache_enabled(flags, reg->nr_pages); + /* all memory is now growable */ + reg->flags |= KBASE_REG_GROWABLE; + + if (flags & BASE_MEM_GROW_ON_GPF) + reg->flags |= KBASE_REG_PF_GROW; + + if (flags & BASE_MEM_PROT_CPU_WR) + reg->flags |= KBASE_REG_CPU_WR; + + if (flags & BASE_MEM_PROT_CPU_RD) + reg->flags |= KBASE_REG_CPU_RD; + + if (flags & BASE_MEM_PROT_GPU_WR) + reg->flags |= KBASE_REG_GPU_WR; + + if (flags & BASE_MEM_PROT_GPU_RD) + reg->flags |= KBASE_REG_GPU_RD; + + if (0 == (flags & BASE_MEM_PROT_GPU_EX)) + reg->flags |= KBASE_REG_GPU_NX; + + if (!kbase_device_is_cpu_coherent(kctx->kbdev)) { + if (flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED && + !(flags & BASE_MEM_UNCACHED_GPU)) + return -EINVAL; + } else if (flags & (BASE_MEM_COHERENT_SYSTEM | + BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { + reg->flags |= KBASE_REG_SHARE_BOTH; + } + + if (!(reg->flags & KBASE_REG_SHARE_BOTH) && + flags & BASE_MEM_COHERENT_LOCAL) { + reg->flags |= KBASE_REG_SHARE_IN; + } + +#if !MALI_USE_CSF + if (flags & BASE_MEM_TILER_ALIGN_TOP) + reg->flags |= KBASE_REG_TILER_ALIGN_TOP; +#endif /* !MALI_USE_CSF */ + +#if MALI_USE_CSF + if (flags & BASE_MEM_CSF_EVENT) { + reg->flags |= KBASE_REG_CSF_EVENT; + reg->flags |= KBASE_REG_PERMANENT_KERNEL_MAPPING; + + if (!(reg->flags & KBASE_REG_SHARE_BOTH)) { + /* On non coherent platforms need to map as uncached on + * both sides. + */ + reg->flags &= ~KBASE_REG_CPU_CACHED; + reg->flags &= ~KBASE_REG_GPU_CACHED; + } + } +#endif + + /* Set up default MEMATTR usage */ + if (!(reg->flags & KBASE_REG_GPU_CACHED)) { + if (kctx->kbdev->mmu_mode->flags & + KBASE_MMU_MODE_HAS_NON_CACHEABLE) { + /* Override shareability, and MEMATTR for uncached */ + reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); + reg->flags |= KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); + } else { + dev_warn(kctx->kbdev->dev, + "Can't allocate GPU uncached memory due to MMU in Legacy Mode\n"); + return -EINVAL; + } +#if MALI_USE_CSF + } else if (reg->flags & KBASE_REG_CSF_EVENT) { + WARN_ON(!(reg->flags & KBASE_REG_SHARE_BOTH)); + + reg->flags |= + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED); +#endif + } else if (kctx->kbdev->system_coherency == COHERENCY_ACE && + (reg->flags & KBASE_REG_SHARE_BOTH)) { + reg->flags |= + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); + } else { + reg->flags |= + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT); + } + + if (flags & BASEP_MEM_PERMANENT_KERNEL_MAPPING) + reg->flags |= KBASE_REG_PERMANENT_KERNEL_MAPPING; + + if (flags & BASEP_MEM_NO_USER_FREE) + reg->flags |= KBASE_REG_NO_USER_FREE; + + if (flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) + reg->flags |= KBASE_REG_GPU_VA_SAME_4GB_PAGE; + + return 0; +} + +int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, + size_t nr_pages_requested) +{ + int new_page_count __maybe_unused; + size_t nr_left = nr_pages_requested; + int res; + struct kbase_context *kctx; + struct kbase_device *kbdev; + struct tagged_addr *tp; + + if (WARN_ON(alloc->type != KBASE_MEM_TYPE_NATIVE) || + WARN_ON(alloc->imported.native.kctx == NULL) || + WARN_ON(alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { + return -EINVAL; + } + + if (alloc->reg) { + if (nr_pages_requested > alloc->reg->nr_pages - alloc->nents) + goto invalid_request; + } + + kctx = alloc->imported.native.kctx; + kbdev = kctx->kbdev; + + if (nr_pages_requested == 0) + goto done; /*nothing to do*/ + + new_page_count = atomic_add_return( + nr_pages_requested, &kctx->used_pages); + atomic_add(nr_pages_requested, + &kctx->kbdev->memdev.used_pages); + + /* Increase mm counters before we allocate pages so that this + * allocation is visible to the OOM killer */ + kbase_process_page_usage_inc(kctx, nr_pages_requested); + + tp = alloc->pages + alloc->nents; + +#ifdef CONFIG_MALI_2MB_ALLOC + /* Check if we have enough pages requested so we can allocate a large + * page (512 * 4KB = 2MB ) + */ + if (nr_left >= (SZ_2M / SZ_4K)) { + int nr_lp = nr_left / (SZ_2M / SZ_4K); + + res = kbase_mem_pool_alloc_pages( + &kctx->mem_pools.large[alloc->group_id], + nr_lp * (SZ_2M / SZ_4K), + tp, + true); + + if (res > 0) { + nr_left -= res; + tp += res; + } + + if (nr_left) { + struct kbase_sub_alloc *sa, *temp_sa; + + spin_lock(&kctx->mem_partials_lock); + + list_for_each_entry_safe(sa, temp_sa, + &kctx->mem_partials, link) { + int pidx = 0; + + while (nr_left) { + pidx = find_next_zero_bit(sa->sub_pages, + SZ_2M / SZ_4K, + pidx); + bitmap_set(sa->sub_pages, pidx, 1); + *tp++ = as_tagged_tag(page_to_phys(sa->page + + pidx), + FROM_PARTIAL); + nr_left--; + + if (bitmap_full(sa->sub_pages, SZ_2M / SZ_4K)) { + /* unlink from partial list when full */ + list_del_init(&sa->link); + break; + } + } + } + spin_unlock(&kctx->mem_partials_lock); + } + + /* only if we actually have a chunk left <512. If more it indicates + * that we couldn't allocate a 2MB above, so no point to retry here. + */ + if (nr_left > 0 && nr_left < (SZ_2M / SZ_4K)) { + /* create a new partial and suballocate the rest from it */ + struct page *np = NULL; + + do { + int err; + + np = kbase_mem_pool_alloc( + &kctx->mem_pools.large[ + alloc->group_id]); + if (np) + break; + + err = kbase_mem_pool_grow( + &kctx->mem_pools.large[alloc->group_id], + 1); + if (err) + break; + } while (1); + + if (np) { + int i; + struct kbase_sub_alloc *sa; + struct page *p; + + sa = kmalloc(sizeof(*sa), GFP_KERNEL); + if (!sa) { + kbase_mem_pool_free( + &kctx->mem_pools.large[ + alloc->group_id], + np, + false); + goto no_new_partial; + } + + /* store pointers back to the control struct */ + np->lru.next = (void *)sa; + for (p = np; p < np + SZ_2M / SZ_4K; p++) + p->lru.prev = (void *)np; + INIT_LIST_HEAD(&sa->link); + bitmap_zero(sa->sub_pages, SZ_2M / SZ_4K); + sa->page = np; + + for (i = 0; i < nr_left; i++) + *tp++ = as_tagged_tag(page_to_phys(np + i), FROM_PARTIAL); + + bitmap_set(sa->sub_pages, 0, nr_left); + nr_left = 0; + + /* expose for later use */ + spin_lock(&kctx->mem_partials_lock); + list_add(&sa->link, &kctx->mem_partials); + spin_unlock(&kctx->mem_partials_lock); + } + } + } +no_new_partial: +#endif + + if (nr_left) { + res = kbase_mem_pool_alloc_pages( + &kctx->mem_pools.small[alloc->group_id], + nr_left, tp, false); + if (res <= 0) + goto alloc_failed; + } + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kbdev, + kctx->id, + (u64)new_page_count); + + alloc->nents += nr_pages_requested; + + kbase_trace_gpu_mem_usage_inc(kctx->kbdev, kctx, nr_pages_requested); + +done: + return 0; + +alloc_failed: + /* rollback needed if got one or more 2MB but failed later */ + if (nr_left != nr_pages_requested) { + size_t nr_pages_to_free = nr_pages_requested - nr_left; + + alloc->nents += nr_pages_to_free; + + kbase_process_page_usage_inc(kctx, nr_pages_to_free); + atomic_add(nr_pages_to_free, &kctx->used_pages); + atomic_add(nr_pages_to_free, + &kctx->kbdev->memdev.used_pages); + + kbase_free_phy_pages_helper(alloc, nr_pages_to_free); + } + + kbase_process_page_usage_dec(kctx, nr_pages_requested); + atomic_sub(nr_pages_requested, &kctx->used_pages); + atomic_sub(nr_pages_requested, + &kctx->kbdev->memdev.used_pages); + +invalid_request: + return -ENOMEM; +} + +struct tagged_addr *kbase_alloc_phy_pages_helper_locked( + struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, + size_t nr_pages_requested, + struct kbase_sub_alloc **prealloc_sa) +{ + int new_page_count __maybe_unused; + size_t nr_left = nr_pages_requested; + int res; + struct kbase_context *kctx; + struct kbase_device *kbdev; + struct tagged_addr *tp; + struct tagged_addr *new_pages = NULL; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); + KBASE_DEBUG_ASSERT(alloc->imported.native.kctx); + + lockdep_assert_held(&pool->pool_lock); + +#if !defined(CONFIG_MALI_2MB_ALLOC) + WARN_ON(pool->order); +#endif + + if (alloc->reg) { + if (nr_pages_requested > alloc->reg->nr_pages - alloc->nents) + goto invalid_request; + } + + kctx = alloc->imported.native.kctx; + kbdev = kctx->kbdev; + + lockdep_assert_held(&kctx->mem_partials_lock); + + if (nr_pages_requested == 0) + goto done; /*nothing to do*/ + + new_page_count = atomic_add_return( + nr_pages_requested, &kctx->used_pages); + atomic_add(nr_pages_requested, + &kctx->kbdev->memdev.used_pages); + + /* Increase mm counters before we allocate pages so that this + * allocation is visible to the OOM killer + */ + kbase_process_page_usage_inc(kctx, nr_pages_requested); + + tp = alloc->pages + alloc->nents; + new_pages = tp; + +#ifdef CONFIG_MALI_2MB_ALLOC + if (pool->order) { + int nr_lp = nr_left / (SZ_2M / SZ_4K); + + res = kbase_mem_pool_alloc_pages_locked(pool, + nr_lp * (SZ_2M / SZ_4K), + tp); + + if (res > 0) { + nr_left -= res; + tp += res; + } + + if (nr_left) { + struct kbase_sub_alloc *sa, *temp_sa; + + list_for_each_entry_safe(sa, temp_sa, + &kctx->mem_partials, link) { + int pidx = 0; + + while (nr_left) { + pidx = find_next_zero_bit(sa->sub_pages, + SZ_2M / SZ_4K, + pidx); + bitmap_set(sa->sub_pages, pidx, 1); + *tp++ = as_tagged_tag(page_to_phys( + sa->page + pidx), + FROM_PARTIAL); + nr_left--; + + if (bitmap_full(sa->sub_pages, + SZ_2M / SZ_4K)) { + /* unlink from partial list when + * full + */ + list_del_init(&sa->link); + break; + } + } + } + } + + /* only if we actually have a chunk left <512. If more it + * indicates that we couldn't allocate a 2MB above, so no point + * to retry here. + */ + if (nr_left > 0 && nr_left < (SZ_2M / SZ_4K)) { + /* create a new partial and suballocate the rest from it + */ + struct page *np = NULL; + + np = kbase_mem_pool_alloc_locked(pool); + + if (np) { + int i; + struct kbase_sub_alloc *const sa = *prealloc_sa; + struct page *p; + + /* store pointers back to the control struct */ + np->lru.next = (void *)sa; + for (p = np; p < np + SZ_2M / SZ_4K; p++) + p->lru.prev = (void *)np; + INIT_LIST_HEAD(&sa->link); + bitmap_zero(sa->sub_pages, SZ_2M / SZ_4K); + sa->page = np; + + for (i = 0; i < nr_left; i++) + *tp++ = as_tagged_tag( + page_to_phys(np + i), + FROM_PARTIAL); + + bitmap_set(sa->sub_pages, 0, nr_left); + nr_left = 0; + /* Indicate to user that we'll free this memory + * later. + */ + *prealloc_sa = NULL; + + /* expose for later use */ + list_add(&sa->link, &kctx->mem_partials); + } + } + if (nr_left) + goto alloc_failed; + } else { +#endif + res = kbase_mem_pool_alloc_pages_locked(pool, + nr_left, + tp); + if (res <= 0) + goto alloc_failed; +#ifdef CONFIG_MALI_2MB_ALLOC + } +#endif + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kbdev, + kctx->id, + (u64)new_page_count); + + alloc->nents += nr_pages_requested; + + kbase_trace_gpu_mem_usage_inc(kctx->kbdev, kctx, nr_pages_requested); + +done: + return new_pages; + +alloc_failed: + /* rollback needed if got one or more 2MB but failed later */ + if (nr_left != nr_pages_requested) { + size_t nr_pages_to_free = nr_pages_requested - nr_left; + + struct tagged_addr *start_free = alloc->pages + alloc->nents; + +#ifdef CONFIG_MALI_2MB_ALLOC + if (pool->order) { + while (nr_pages_to_free) { + if (is_huge_head(*start_free)) { + kbase_mem_pool_free_pages_locked( + pool, 512, + start_free, + false, /* not dirty */ + true); /* return to pool */ + nr_pages_to_free -= 512; + start_free += 512; + } else if (is_partial(*start_free)) { + free_partial_locked(kctx, pool, + *start_free); + nr_pages_to_free--; + start_free++; + } + } + } else { +#endif + kbase_mem_pool_free_pages_locked(pool, + nr_pages_to_free, + start_free, + false, /* not dirty */ + true); /* return to pool */ +#ifdef CONFIG_MALI_2MB_ALLOC + } +#endif + } + + kbase_process_page_usage_dec(kctx, nr_pages_requested); + atomic_sub(nr_pages_requested, &kctx->used_pages); + atomic_sub(nr_pages_requested, &kctx->kbdev->memdev.used_pages); + +invalid_request: + return NULL; +} + +static void free_partial(struct kbase_context *kctx, int group_id, struct + tagged_addr tp) +{ + struct page *p, *head_page; + struct kbase_sub_alloc *sa; + + p = as_page(tp); + head_page = (struct page *)p->lru.prev; + sa = (struct kbase_sub_alloc *)head_page->lru.next; + spin_lock(&kctx->mem_partials_lock); + clear_bit(p - head_page, sa->sub_pages); + if (bitmap_empty(sa->sub_pages, SZ_2M / SZ_4K)) { + list_del(&sa->link); + kbase_mem_pool_free( + &kctx->mem_pools.large[group_id], + head_page, + true); + kfree(sa); + } else if (bitmap_weight(sa->sub_pages, SZ_2M / SZ_4K) == + SZ_2M / SZ_4K - 1) { + /* expose the partial again */ + list_add(&sa->link, &kctx->mem_partials); + } + spin_unlock(&kctx->mem_partials_lock); +} + +int kbase_free_phy_pages_helper( + struct kbase_mem_phy_alloc *alloc, + size_t nr_pages_to_free) +{ + struct kbase_context *kctx = alloc->imported.native.kctx; + struct kbase_device *kbdev = kctx->kbdev; + bool syncback; + bool reclaimed = (alloc->evicted != 0); + struct tagged_addr *start_free; + int new_page_count __maybe_unused; + size_t freed = 0; + + if (WARN_ON(alloc->type != KBASE_MEM_TYPE_NATIVE) || + WARN_ON(alloc->imported.native.kctx == NULL) || + WARN_ON(alloc->nents < nr_pages_to_free) || + WARN_ON(alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { + return -EINVAL; + } + + /* early out if nothing to do */ + if (0 == nr_pages_to_free) + return 0; + + start_free = alloc->pages + alloc->nents - nr_pages_to_free; + + syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; + + /* pad start_free to a valid start location */ + while (nr_pages_to_free && is_huge(*start_free) && + !is_huge_head(*start_free)) { + nr_pages_to_free--; + start_free++; + } + + while (nr_pages_to_free) { + if (is_huge_head(*start_free)) { + /* This is a 2MB entry, so free all the 512 pages that + * it points to + */ + kbase_mem_pool_free_pages( + &kctx->mem_pools.large[alloc->group_id], + 512, + start_free, + syncback, + reclaimed); + nr_pages_to_free -= 512; + start_free += 512; + freed += 512; + } else if (is_partial(*start_free)) { + free_partial(kctx, alloc->group_id, *start_free); + nr_pages_to_free--; + start_free++; + freed++; + } else { + struct tagged_addr *local_end_free; + + local_end_free = start_free; + while (nr_pages_to_free && + !is_huge(*local_end_free) && + !is_partial(*local_end_free)) { + local_end_free++; + nr_pages_to_free--; + } + kbase_mem_pool_free_pages( + &kctx->mem_pools.small[alloc->group_id], + local_end_free - start_free, + start_free, + syncback, + reclaimed); + freed += local_end_free - start_free; + start_free += local_end_free - start_free; + } + } + + alloc->nents -= freed; + + /* + * If the allocation was not evicted (i.e. evicted == 0) then + * the page accounting needs to be done. + */ + if (!reclaimed) { + kbase_process_page_usage_dec(kctx, freed); + new_page_count = atomic_sub_return(freed, + &kctx->used_pages); + atomic_sub(freed, + &kctx->kbdev->memdev.used_pages); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kbdev, + kctx->id, + (u64)new_page_count); + + kbase_trace_gpu_mem_usage_dec(kctx->kbdev, kctx, freed); + } + + return 0; +} + +static void free_partial_locked(struct kbase_context *kctx, + struct kbase_mem_pool *pool, struct tagged_addr tp) +{ + struct page *p, *head_page; + struct kbase_sub_alloc *sa; + + lockdep_assert_held(&pool->pool_lock); + lockdep_assert_held(&kctx->mem_partials_lock); + + p = as_page(tp); + head_page = (struct page *)p->lru.prev; + sa = (struct kbase_sub_alloc *)head_page->lru.next; + clear_bit(p - head_page, sa->sub_pages); + if (bitmap_empty(sa->sub_pages, SZ_2M / SZ_4K)) { + list_del(&sa->link); + kbase_mem_pool_free_locked(pool, head_page, true); + kfree(sa); + } else if (bitmap_weight(sa->sub_pages, SZ_2M / SZ_4K) == + SZ_2M / SZ_4K - 1) { + /* expose the partial again */ + list_add(&sa->link, &kctx->mem_partials); + } +} + +void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, + struct kbase_mem_pool *pool, struct tagged_addr *pages, + size_t nr_pages_to_free) +{ + struct kbase_context *kctx = alloc->imported.native.kctx; + struct kbase_device *kbdev = kctx->kbdev; + bool syncback; + bool reclaimed = (alloc->evicted != 0); + struct tagged_addr *start_free; + size_t freed = 0; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); + KBASE_DEBUG_ASSERT(alloc->imported.native.kctx); + KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free); + + lockdep_assert_held(&pool->pool_lock); + lockdep_assert_held(&kctx->mem_partials_lock); + + /* early out if nothing to do */ + if (!nr_pages_to_free) + return; + + start_free = pages; + + syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; + + /* pad start_free to a valid start location */ + while (nr_pages_to_free && is_huge(*start_free) && + !is_huge_head(*start_free)) { + nr_pages_to_free--; + start_free++; + } + + while (nr_pages_to_free) { + if (is_huge_head(*start_free)) { + /* This is a 2MB entry, so free all the 512 pages that + * it points to + */ + WARN_ON(!pool->order); + kbase_mem_pool_free_pages_locked(pool, + 512, + start_free, + syncback, + reclaimed); + nr_pages_to_free -= 512; + start_free += 512; + freed += 512; + } else if (is_partial(*start_free)) { + WARN_ON(!pool->order); + free_partial_locked(kctx, pool, *start_free); + nr_pages_to_free--; + start_free++; + freed++; + } else { + struct tagged_addr *local_end_free; + + WARN_ON(pool->order); + local_end_free = start_free; + while (nr_pages_to_free && + !is_huge(*local_end_free) && + !is_partial(*local_end_free)) { + local_end_free++; + nr_pages_to_free--; + } + kbase_mem_pool_free_pages_locked(pool, + local_end_free - start_free, + start_free, + syncback, + reclaimed); + freed += local_end_free - start_free; + start_free += local_end_free - start_free; + } + } + + alloc->nents -= freed; + + /* + * If the allocation was not evicted (i.e. evicted == 0) then + * the page accounting needs to be done. + */ + if (!reclaimed) { + int new_page_count; + + kbase_process_page_usage_dec(kctx, freed); + new_page_count = atomic_sub_return(freed, + &kctx->used_pages); + atomic_sub(freed, + &kctx->kbdev->memdev.used_pages); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kbdev, + kctx->id, + (u64)new_page_count); + + kbase_trace_gpu_mem_usage_dec(kctx->kbdev, kctx, freed); + } +} + +#if MALI_USE_CSF +/** + * kbase_jd_user_buf_unpin_pages - Release the pinned pages of a user buffer. + * @alloc: The allocation for the imported user buffer. + */ +static void kbase_jd_user_buf_unpin_pages(struct kbase_mem_phy_alloc *alloc); +#endif + +void kbase_mem_kref_free(struct kref *kref) +{ + struct kbase_mem_phy_alloc *alloc; + + alloc = container_of(kref, struct kbase_mem_phy_alloc, kref); + + switch (alloc->type) { + case KBASE_MEM_TYPE_NATIVE: { + + if (!WARN_ON(!alloc->imported.native.kctx)) { + if (alloc->permanent_map) + kbase_phy_alloc_mapping_term( + alloc->imported.native.kctx, + alloc); + + /* + * The physical allocation must have been removed from + * the eviction list before trying to free it. + */ + mutex_lock( + &alloc->imported.native.kctx->jit_evict_lock); + WARN_ON(!list_empty(&alloc->evict_node)); + mutex_unlock( + &alloc->imported.native.kctx->jit_evict_lock); + + kbase_process_page_usage_dec( + alloc->imported.native.kctx, + alloc->imported.native.nr_struct_pages); + } + kbase_free_phy_pages_helper(alloc, alloc->nents); + break; + } + case KBASE_MEM_TYPE_ALIAS: { + /* just call put on the underlying phy allocs */ + size_t i; + struct kbase_aliased *aliased; + + aliased = alloc->imported.alias.aliased; + if (aliased) { + for (i = 0; i < alloc->imported.alias.nents; i++) + if (aliased[i].alloc) + kbase_mem_phy_alloc_put(aliased[i].alloc); + vfree(aliased); + } + break; + } + case KBASE_MEM_TYPE_RAW: + /* raw pages, external cleanup */ + break; + case KBASE_MEM_TYPE_IMPORTED_UMM: + if (!IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND)) { + WARN_ONCE(alloc->imported.umm.current_mapping_usage_count != 1, + "WARNING: expected excatly 1 mapping, got %d", + alloc->imported.umm.current_mapping_usage_count); + dma_buf_unmap_attachment( + alloc->imported.umm.dma_attachment, + alloc->imported.umm.sgt, + DMA_BIDIRECTIONAL); + kbase_remove_dma_buf_usage(alloc->imported.umm.kctx, + alloc); + } + dma_buf_detach(alloc->imported.umm.dma_buf, + alloc->imported.umm.dma_attachment); + dma_buf_put(alloc->imported.umm.dma_buf); + break; + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: +#if MALI_USE_CSF + kbase_jd_user_buf_unpin_pages(alloc); +#endif + if (alloc->imported.user_buf.mm) + mmdrop(alloc->imported.user_buf.mm); + if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) + vfree(alloc->imported.user_buf.pages); + else + kfree(alloc->imported.user_buf.pages); + break; + default: + WARN(1, "Unexecpted free of type %d\n", alloc->type); + break; + } + + /* Free based on allocation type */ + if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) + vfree(alloc); + else + kfree(alloc); +} + +KBASE_EXPORT_TEST_API(kbase_mem_kref_free); + +int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size) +{ + KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT(vsize > 0); + + /* validate user provided arguments */ + if (size > vsize || vsize > reg->nr_pages) + goto out_term; + + /* Prevent vsize*sizeof from wrapping around. + * For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail. + */ + if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages))) + goto out_term; + + KBASE_DEBUG_ASSERT(0 != vsize); + + if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0) + goto out_term; + + reg->cpu_alloc->reg = reg; + if (reg->cpu_alloc != reg->gpu_alloc) { + if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0) + goto out_rollback; + reg->gpu_alloc->reg = reg; + } + + return 0; + +out_rollback: + kbase_free_phy_pages_helper(reg->cpu_alloc, size); +out_term: + return -1; +} + +KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages); + +bool kbase_check_alloc_flags(unsigned long flags) +{ + /* Only known input flags should be set. */ + if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) + return false; + + /* At least one flag should be set */ + if (flags == 0) + return false; + + /* Either the GPU or CPU must be reading from the allocated memory */ + if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0) + return false; + + /* Either the GPU or CPU must be writing to the allocated memory */ + if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0) + return false; + + /* GPU executable memory cannot: + * - Be written by the GPU + * - Be grown on GPU page fault + */ + if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & + (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) + return false; + +#if !MALI_USE_CSF + /* GPU executable memory also cannot have the top of its initial + * commit aligned to 'extent' + */ + if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & + BASE_MEM_TILER_ALIGN_TOP)) + return false; +#endif /* !MALI_USE_CSF */ + + /* To have an allocation lie within a 4GB chunk is required only for + * TLS memory, which will never be used to contain executable code. + */ + if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & + BASE_MEM_PROT_GPU_EX)) + return false; + +#if !MALI_USE_CSF + /* TLS memory should also not be used for tiler heap */ + if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & + BASE_MEM_TILER_ALIGN_TOP)) + return false; +#endif /* !MALI_USE_CSF */ + + /* GPU should have at least read or write access otherwise there is no + reason for allocating. */ + if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) + return false; + + /* BASE_MEM_IMPORT_SHARED is only valid for imported memory */ + if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED) + return false; + + /* BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP is only valid for imported + * memory */ + if ((flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) == + BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) + return false; + + /* Should not combine BASE_MEM_COHERENT_LOCAL with + * BASE_MEM_COHERENT_SYSTEM */ + if ((flags & (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) == + (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) + return false; + + return true; +} + +bool kbase_check_import_flags(unsigned long flags) +{ + /* Only known input flags should be set. */ + if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) + return false; + + /* At least one flag should be set */ + if (flags == 0) + return false; + + /* Imported memory cannot be GPU executable */ + if (flags & BASE_MEM_PROT_GPU_EX) + return false; + + /* Imported memory cannot grow on page fault */ + if (flags & BASE_MEM_GROW_ON_GPF) + return false; + +#if !MALI_USE_CSF + /* Imported memory cannot be aligned to the end of its initial commit */ + if (flags & BASE_MEM_TILER_ALIGN_TOP) + return false; +#endif /* !MALI_USE_CSF */ + + /* GPU should have at least read or write access otherwise there is no + reason for importing. */ + if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) + return false; + + /* Protected memory cannot be read by the CPU */ + if ((flags & BASE_MEM_PROTECTED) && (flags & BASE_MEM_PROT_CPU_RD)) + return false; + + return true; +} + +int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, + u64 va_pages, u64 commit_pages, u64 large_extent) +{ + struct device *dev = kctx->kbdev->dev; + int gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; + u64 gpu_pc_pages_max = 1ULL << gpu_pc_bits >> PAGE_SHIFT; + struct kbase_va_region test_reg; + + /* kbase_va_region's extent member can be of variable size, so check against that type */ + test_reg.extent = large_extent; + +#define KBASE_MSG_PRE "GPU allocation attempted with " + + if (0 == va_pages) { + dev_warn(dev, KBASE_MSG_PRE "0 va_pages!"); + return -EINVAL; + } + + if (va_pages > KBASE_MEM_ALLOC_MAX_SIZE) { + dev_warn(dev, KBASE_MSG_PRE "va_pages==%lld larger than KBASE_MEM_ALLOC_MAX_SIZE!", + (unsigned long long)va_pages); + return -ENOMEM; + } + + /* Note: commit_pages is checked against va_pages during + * kbase_alloc_phy_pages() */ + + /* Limit GPU executable allocs to GPU PC size */ + if ((flags & BASE_MEM_PROT_GPU_EX) && (va_pages > gpu_pc_pages_max)) { + dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_PROT_GPU_EX and va_pages==%lld larger than GPU PC range %lld", + (unsigned long long)va_pages, + (unsigned long long)gpu_pc_pages_max); + + return -EINVAL; + } + + if ((flags & BASE_MEM_GROW_ON_GPF) && (test_reg.extent == 0)) { + dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GROW_ON_GPF but extent == 0\n"); + return -EINVAL; + } + +#if !MALI_USE_CSF + if ((flags & BASE_MEM_TILER_ALIGN_TOP) && (test_reg.extent == 0)) { + dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP but extent == 0\n"); + return -EINVAL; + } + + if (!(flags & (BASE_MEM_GROW_ON_GPF | BASE_MEM_TILER_ALIGN_TOP)) && + test_reg.extent != 0) { + dev_warn(dev, KBASE_MSG_PRE "neither BASE_MEM_GROW_ON_GPF nor BASE_MEM_TILER_ALIGN_TOP set but extent != 0\n"); + return -EINVAL; + } +#else + if (!(flags & BASE_MEM_GROW_ON_GPF) && test_reg.extent != 0) { + dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GROW_ON_GPF not set but extent != 0\n"); + return -EINVAL; + } +#endif /* !MALI_USE_CSF */ + +#if !MALI_USE_CSF + /* BASE_MEM_TILER_ALIGN_TOP memory has a number of restrictions */ + if (flags & BASE_MEM_TILER_ALIGN_TOP) { +#define KBASE_MSG_PRE_FLAG KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP and " + unsigned long small_extent; + + if (large_extent > BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES) { + dev_warn(dev, KBASE_MSG_PRE_FLAG "extent==%lld pages exceeds limit %lld", + (unsigned long long)large_extent, + BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES); + return -EINVAL; + } + /* For use with is_power_of_2, which takes unsigned long, so + * must ensure e.g. on 32-bit kernel it'll fit in that type */ + small_extent = (unsigned long)large_extent; + + if (!is_power_of_2(small_extent)) { + dev_warn(dev, KBASE_MSG_PRE_FLAG "extent==%ld not a non-zero power of 2", + small_extent); + return -EINVAL; + } + + if (commit_pages > large_extent) { + dev_warn(dev, KBASE_MSG_PRE_FLAG "commit_pages==%ld exceeds extent==%ld", + (unsigned long)commit_pages, + (unsigned long)large_extent); + return -EINVAL; + } +#undef KBASE_MSG_PRE_FLAG + } +#endif /* !MALI_USE_CSF */ + + if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && + (va_pages > (BASE_MEM_PFN_MASK_4GB + 1))) { + dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GPU_VA_SAME_4GB_PAGE and va_pages==%lld greater than that needed for 4GB space", + (unsigned long long)va_pages); + return -EINVAL; + } + + return 0; +#undef KBASE_MSG_PRE +} + +/** + * @brief Acquire the per-context region list lock + */ +void kbase_gpu_vm_lock(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx != NULL); + mutex_lock(&kctx->reg_lock); +} + +KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); + +/** + * @brief Release the per-context region list lock + */ +void kbase_gpu_vm_unlock(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx != NULL); + mutex_unlock(&kctx->reg_lock); +} + +KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); + +#ifdef CONFIG_DEBUG_FS +struct kbase_jit_debugfs_data { + int (*func)(struct kbase_jit_debugfs_data *); + struct mutex lock; + struct kbase_context *kctx; + u64 active_value; + u64 pool_value; + u64 destroy_value; + char buffer[50]; +}; + +static int kbase_jit_debugfs_common_open(struct inode *inode, + struct file *file, int (*func)(struct kbase_jit_debugfs_data *)) +{ + struct kbase_jit_debugfs_data *data; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->func = func; + mutex_init(&data->lock); + data->kctx = (struct kbase_context *) inode->i_private; + + file->private_data = data; + + return nonseekable_open(inode, file); +} + +static ssize_t kbase_jit_debugfs_common_read(struct file *file, + char __user *buf, size_t len, loff_t *ppos) +{ + struct kbase_jit_debugfs_data *data; + size_t size; + int ret; + + data = (struct kbase_jit_debugfs_data *) file->private_data; + mutex_lock(&data->lock); + + if (*ppos) { + size = strnlen(data->buffer, sizeof(data->buffer)); + } else { + if (!data->func) { + ret = -EACCES; + goto out_unlock; + } + + if (data->func(data)) { + ret = -EACCES; + goto out_unlock; + } + + size = scnprintf(data->buffer, sizeof(data->buffer), + "%llu,%llu,%llu", data->active_value, + data->pool_value, data->destroy_value); + } + + ret = simple_read_from_buffer(buf, len, ppos, data->buffer, size); + +out_unlock: + mutex_unlock(&data->lock); + return ret; +} + +static int kbase_jit_debugfs_common_release(struct inode *inode, + struct file *file) +{ + kfree(file->private_data); + return 0; +} + +#define KBASE_JIT_DEBUGFS_DECLARE(__fops, __func) \ +static int __fops ## _open(struct inode *inode, struct file *file) \ +{ \ + return kbase_jit_debugfs_common_open(inode, file, __func); \ +} \ +static const struct file_operations __fops = { \ + .owner = THIS_MODULE, \ + .open = __fops ## _open, \ + .release = kbase_jit_debugfs_common_release, \ + .read = kbase_jit_debugfs_common_read, \ + .write = NULL, \ + .llseek = generic_file_llseek, \ +} + +static int kbase_jit_debugfs_count_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct list_head *tmp; + + mutex_lock(&kctx->jit_evict_lock); + list_for_each(tmp, &kctx->jit_active_head) { + data->active_value++; + } + + list_for_each(tmp, &kctx->jit_pool_head) { + data->pool_value++; + } + + list_for_each(tmp, &kctx->jit_destroy_head) { + data->destroy_value++; + } + mutex_unlock(&kctx->jit_evict_lock); + + return 0; +} +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_count_fops, + kbase_jit_debugfs_count_get); + +static int kbase_jit_debugfs_vm_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct kbase_va_region *reg; + + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { + data->active_value += reg->nr_pages; + } + + list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { + data->pool_value += reg->nr_pages; + } + + list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { + data->destroy_value += reg->nr_pages; + } + mutex_unlock(&kctx->jit_evict_lock); + + return 0; +} +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_vm_fops, + kbase_jit_debugfs_vm_get); + +static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct kbase_va_region *reg; + + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { + data->active_value += reg->gpu_alloc->nents; + } + + list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { + data->pool_value += reg->gpu_alloc->nents; + } + + list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { + data->destroy_value += reg->gpu_alloc->nents; + } + mutex_unlock(&kctx->jit_evict_lock); + + return 0; +} +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops, + kbase_jit_debugfs_phys_get); + +#if MALI_JIT_PRESSURE_LIMIT_BASE +static int kbase_jit_debugfs_used_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct kbase_va_region *reg; + +#if !MALI_USE_CSF + mutex_lock(&kctx->jctx.lock); +#endif /* !MALI_USE_CSF */ + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { + data->active_value += reg->used_pages; + } + mutex_unlock(&kctx->jit_evict_lock); +#if !MALI_USE_CSF + mutex_unlock(&kctx->jctx.lock); +#endif /* !MALI_USE_CSF */ + + return 0; +} + +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_used_fops, + kbase_jit_debugfs_used_get); + +static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, + struct kbase_va_region *reg, size_t pages_needed, + size_t *freed, bool shrink); + +static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct kbase_va_region *reg; + +#if !MALI_USE_CSF + mutex_lock(&kctx->jctx.lock); +#endif /* !MALI_USE_CSF */ + kbase_gpu_vm_lock(kctx); + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { + int err; + size_t freed = 0u; + + err = kbase_mem_jit_trim_pages_from_region(kctx, reg, + SIZE_MAX, &freed, false); + + if (err) { + /* Failed to calculate, try the next region */ + continue; + } + + data->active_value += freed; + } + mutex_unlock(&kctx->jit_evict_lock); + kbase_gpu_vm_unlock(kctx); +#if !MALI_USE_CSF + mutex_unlock(&kctx->jctx.lock); +#endif /* !MALI_USE_CSF */ + + return 0; +} + +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_trim_fops, + kbase_jit_debugfs_trim_get); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +void kbase_jit_debugfs_init(struct kbase_context *kctx) +{ + /* prevent unprivileged use of debug file system + * in old kernel version + */ +#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) + /* only for newer kernel version debug file system is safe */ + const mode_t mode = 0444; +#else + const mode_t mode = 0400; +#endif + + /* Caller already ensures this, but we keep the pattern for + * maintenance safety. + */ + if (WARN_ON(!kctx) || + WARN_ON(IS_ERR_OR_NULL(kctx->kctx_dentry))) + return; + + + + /* Debugfs entry for getting the number of JIT allocations. */ + debugfs_create_file("mem_jit_count", mode, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_count_fops); + + /* + * Debugfs entry for getting the total number of virtual pages + * used by JIT allocations. + */ + debugfs_create_file("mem_jit_vm", mode, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_vm_fops); + + /* + * Debugfs entry for getting the number of physical pages used + * by JIT allocations. + */ + debugfs_create_file("mem_jit_phys", mode, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_phys_fops); +#if MALI_JIT_PRESSURE_LIMIT_BASE + /* + * Debugfs entry for getting the number of pages used + * by JIT allocations for estimating the physical pressure + * limit. + */ + debugfs_create_file("mem_jit_used", mode, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_used_fops); + + /* + * Debugfs entry for getting the number of pages that could + * be trimmed to free space for more JIT allocations. + */ + debugfs_create_file("mem_jit_trim", mode, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_trim_fops); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ +} +#endif /* CONFIG_DEBUG_FS */ + +/** + * kbase_jit_destroy_worker - Deferred worker which frees JIT allocations + * @work: Work item + * + * This function does the work of freeing JIT allocations whose physical + * backing has been released. + */ +static void kbase_jit_destroy_worker(struct work_struct *work) +{ + struct kbase_context *kctx; + struct kbase_va_region *reg; + + kctx = container_of(work, struct kbase_context, jit_work); + do { + mutex_lock(&kctx->jit_evict_lock); + if (list_empty(&kctx->jit_destroy_head)) { + mutex_unlock(&kctx->jit_evict_lock); + break; + } + + reg = list_first_entry(&kctx->jit_destroy_head, + struct kbase_va_region, jit_node); + + list_del(®->jit_node); + mutex_unlock(&kctx->jit_evict_lock); + + kbase_gpu_vm_lock(kctx); + reg->flags &= ~KBASE_REG_NO_USER_FREE; + kbase_mem_free_region(kctx, reg); + kbase_gpu_vm_unlock(kctx); + } while (1); +} + +int kbase_jit_init(struct kbase_context *kctx) +{ + mutex_lock(&kctx->jit_evict_lock); + INIT_LIST_HEAD(&kctx->jit_active_head); + INIT_LIST_HEAD(&kctx->jit_pool_head); + INIT_LIST_HEAD(&kctx->jit_destroy_head); + INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); + +#if MALI_USE_CSF + INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_cmds_head); + INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_blocked_queues); +#else /* !MALI_USE_CSF */ + INIT_LIST_HEAD(&kctx->jctx.jit_atoms_head); + INIT_LIST_HEAD(&kctx->jctx.jit_pending_alloc); +#endif /* MALI_USE_CSF */ + mutex_unlock(&kctx->jit_evict_lock); + + kctx->jit_max_allocations = 0; + kctx->jit_current_allocations = 0; + kctx->trim_level = 0; + + return 0; +} + +/* Check if the allocation from JIT pool is of the same size as the new JIT + * allocation and also, if BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP is set, meets + * the alignment requirements. + */ +static bool meet_size_and_tiler_align_top_requirements( + const struct kbase_va_region *walker, + const struct base_jit_alloc_info *info) +{ + bool meet_reqs = true; + + if (walker->nr_pages != info->va_pages) + meet_reqs = false; + +#if !MALI_USE_CSF + if (meet_reqs && (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP)) { + size_t align = info->extent; + size_t align_mask = align - 1; + + if ((walker->start_pfn + info->commit_pages) & align_mask) + meet_reqs = false; + } +#endif /* !MALI_USE_CSF */ + + return meet_reqs; +} + +#if MALI_JIT_PRESSURE_LIMIT_BASE +/* Function will guarantee *@freed will not exceed @pages_needed + */ +static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, + struct kbase_va_region *reg, size_t pages_needed, + size_t *freed, bool shrink) +{ + int err = 0; + size_t available_pages = 0u; + const size_t old_pages = kbase_reg_current_backed_size(reg); + size_t new_pages = old_pages; + size_t to_free = 0u; + size_t max_allowed_pages = old_pages; + +#if !MALI_USE_CSF + lockdep_assert_held(&kctx->jctx.lock); +#endif /* !MALI_USE_CSF */ + lockdep_assert_held(&kctx->reg_lock); + + /* Is this a JIT allocation that has been reported on? */ + if (reg->used_pages == reg->nr_pages) + goto out; + + if (!(reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE)) { + /* For address based memory usage calculation, the GPU + * allocates objects of up to size 's', but aligns every object + * to alignment 'a', with a < s. + * + * It also doesn't have to write to all bytes in an object of + * size 's'. + * + * Hence, we can observe the GPU's address for the end of used + * memory being up to (s - a) bytes into the first unallocated + * page. + * + * We allow for this and only warn when it exceeds this bound + * (rounded up to page sized units). Note, this is allowed to + * exceed reg->nr_pages. + */ + max_allowed_pages += PFN_UP( + KBASE_GPU_ALLOCATED_OBJECT_MAX_BYTES - + KBASE_GPU_ALLOCATED_OBJECT_ALIGN_BYTES); + } else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { + /* The GPU could report being ready to write to the next + * 'extent' sized chunk, but didn't actually write to it, so we + * can report up to 'extent' size pages more than the backed + * size. + * + * Note, this is allowed to exceed reg->nr_pages. + */ + max_allowed_pages += reg->extent; + + /* Also note that in these GPUs, the GPU may make a large (>1 + * page) initial allocation but not actually write out to all + * of it. Hence it might report that a much higher amount of + * memory was used than actually was written to. This does not + * result in a real warning because on growing this memory we + * round up the size of the allocation up to an 'extent' sized + * chunk, hence automatically bringing the backed size up to + * the reported size. + */ + } + + if (old_pages < reg->used_pages) { + /* Prevent overflow on available_pages, but only report the + * problem if it's in a scenario where used_pages should have + * been consistent with the backed size + * + * Note: In case of a size-based report, this legitimately + * happens in common use-cases: we allow for up to this size of + * memory being used, but depending on the content it doesn't + * have to use all of it. + * + * Hence, we're much more quiet about that in the size-based + * report case - it's not indicating a real problem, it's just + * for information + */ + if (max_allowed_pages < reg->used_pages) { + if (!(reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE)) + dev_warn(kctx->kbdev->dev, + "%s: current backed pages %zu < reported used pages %zu (allowed to be up to %zu) on JIT 0x%llx vapages %zu\n", + __func__, + old_pages, reg->used_pages, + max_allowed_pages, + reg->start_pfn << PAGE_SHIFT, + reg->nr_pages); + else + dev_dbg(kctx->kbdev->dev, + "%s: no need to trim, current backed pages %zu < reported used pages %zu on size-report for JIT 0x%llx vapages %zu\n", + __func__, + old_pages, reg->used_pages, + reg->start_pfn << PAGE_SHIFT, + reg->nr_pages); + } + /* In any case, no error condition to report here, caller can + * try other regions + */ + + goto out; + } + available_pages = old_pages - reg->used_pages; + to_free = min(available_pages, pages_needed); + + if (shrink) { + new_pages -= to_free; + + err = kbase_mem_shrink(kctx, reg, new_pages); + } +out: + trace_mali_jit_trim_from_region(reg, to_free, old_pages, + available_pages, new_pages); + *freed = to_free; + return err; +} + + +/** + * kbase_mem_jit_trim_pages - Trim JIT regions until sufficient pages have been + * freed + * @kctx: Pointer to the kbase context whose active JIT allocations will be + * checked. + * @pages_needed: The maximum number of pages to trim. + * + * This functions checks all active JIT allocations in @kctx for unused pages + * at the end, and trim the backed memory regions of those allocations down to + * the used portion and free the unused pages into the page pool. + * + * Specifying @pages_needed allows us to stop early when there's enough + * physical memory freed to sufficiently bring down the total JIT physical page + * usage (e.g. to below the pressure limit) + * + * Return: Total number of successfully freed pages + */ +static size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, + size_t pages_needed) +{ + struct kbase_va_region *reg, *tmp; + size_t total_freed = 0; + +#if !MALI_USE_CSF + lockdep_assert_held(&kctx->jctx.lock); +#endif /* !MALI_USE_CSF */ + lockdep_assert_held(&kctx->reg_lock); + lockdep_assert_held(&kctx->jit_evict_lock); + + list_for_each_entry_safe(reg, tmp, &kctx->jit_active_head, jit_node) { + int err; + size_t freed = 0u; + + err = kbase_mem_jit_trim_pages_from_region(kctx, reg, + pages_needed, &freed, true); + + if (err) { + /* Failed to trim, try the next region */ + continue; + } + + total_freed += freed; + WARN_ON(freed > pages_needed); + pages_needed -= freed; + if (!pages_needed) + break; + } + + trace_mali_jit_trim(total_freed); + + return total_freed; +} +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +static int kbase_jit_grow(struct kbase_context *kctx, + const struct base_jit_alloc_info *info, + struct kbase_va_region *reg, + struct kbase_sub_alloc **prealloc_sas) +{ + size_t delta; + size_t pages_required; + size_t old_size; + struct kbase_mem_pool *pool; + int ret = -ENOMEM; + struct tagged_addr *gpu_pages; + + if (info->commit_pages > reg->nr_pages) { + /* Attempted to grow larger than maximum size */ + return -EINVAL; + } + + lockdep_assert_held(&kctx->reg_lock); + + /* Make the physical backing no longer reclaimable */ + if (!kbase_mem_evictable_unmake(reg->gpu_alloc)) + goto update_failed; + + if (reg->gpu_alloc->nents >= info->commit_pages) + goto done; + + /* Grow the backing */ + old_size = reg->gpu_alloc->nents; + + /* Allocate some more pages */ + delta = info->commit_pages - reg->gpu_alloc->nents; + pages_required = delta; + +#ifdef CONFIG_MALI_2MB_ALLOC + if (pages_required >= (SZ_2M / SZ_4K)) { + pool = &kctx->mem_pools.large[kctx->jit_group_id]; + /* Round up to number of 2 MB pages required */ + pages_required += ((SZ_2M / SZ_4K) - 1); + pages_required /= (SZ_2M / SZ_4K); + } else { +#endif + pool = &kctx->mem_pools.small[kctx->jit_group_id]; +#ifdef CONFIG_MALI_2MB_ALLOC + } +#endif + + if (reg->cpu_alloc != reg->gpu_alloc) + pages_required *= 2; + + spin_lock(&kctx->mem_partials_lock); + kbase_mem_pool_lock(pool); + + /* As we can not allocate memory from the kernel with the vm_lock held, + * grow the pool to the required size with the lock dropped. We hold the + * pool lock to prevent another thread from allocating from the pool + * between the grow and allocation. + */ + while (kbase_mem_pool_size(pool) < pages_required) { + int pool_delta = pages_required - kbase_mem_pool_size(pool); + int ret; + + kbase_mem_pool_unlock(pool); + spin_unlock(&kctx->mem_partials_lock); + + kbase_gpu_vm_unlock(kctx); + ret = kbase_mem_pool_grow(pool, pool_delta); + kbase_gpu_vm_lock(kctx); + + if (ret) + goto update_failed; + + spin_lock(&kctx->mem_partials_lock); + kbase_mem_pool_lock(pool); + } + + gpu_pages = kbase_alloc_phy_pages_helper_locked(reg->gpu_alloc, pool, + delta, &prealloc_sas[0]); + if (!gpu_pages) { + kbase_mem_pool_unlock(pool); + spin_unlock(&kctx->mem_partials_lock); + goto update_failed; + } + + if (reg->cpu_alloc != reg->gpu_alloc) { + struct tagged_addr *cpu_pages; + + cpu_pages = kbase_alloc_phy_pages_helper_locked(reg->cpu_alloc, + pool, delta, &prealloc_sas[1]); + if (!cpu_pages) { + kbase_free_phy_pages_helper_locked(reg->gpu_alloc, + pool, gpu_pages, delta); + kbase_mem_pool_unlock(pool); + spin_unlock(&kctx->mem_partials_lock); + goto update_failed; + } + } + kbase_mem_pool_unlock(pool); + spin_unlock(&kctx->mem_partials_lock); + + ret = kbase_mem_grow_gpu_mapping(kctx, reg, info->commit_pages, + old_size); + /* + * The grow failed so put the allocation back in the + * pool and return failure. + */ + if (ret) + goto update_failed; + +done: + ret = 0; + + /* Update attributes of JIT allocation taken from the pool */ + reg->initial_commit = info->commit_pages; + reg->extent = info->extent; + +update_failed: + return ret; +} + +static void trace_jit_stats(struct kbase_context *kctx, + u32 bin_id, u32 max_allocations) +{ + const u32 alloc_count = + kctx->jit_current_allocations_per_bin[bin_id]; + struct kbase_device *kbdev = kctx->kbdev; + + struct kbase_va_region *walker; + u32 va_pages = 0; + u32 ph_pages = 0; + + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(walker, &kctx->jit_active_head, jit_node) { + if (walker->jit_bin_id != bin_id) + continue; + + va_pages += walker->nr_pages; + ph_pages += walker->gpu_alloc->nents; + } + mutex_unlock(&kctx->jit_evict_lock); + + KBASE_TLSTREAM_AUX_JIT_STATS(kbdev, kctx->id, bin_id, + max_allocations, alloc_count, va_pages, ph_pages); +} + +#if MALI_JIT_PRESSURE_LIMIT_BASE +/** + * get_jit_phys_backing() - calculate the physical backing of all JIT + * allocations + * + * @kctx: Pointer to the kbase context whose active JIT allocations will be + * checked + * + * Return: number of pages that are committed by JIT allocations + */ +static size_t get_jit_phys_backing(struct kbase_context *kctx) +{ + struct kbase_va_region *walker; + size_t backing = 0; + + lockdep_assert_held(&kctx->jit_evict_lock); + + list_for_each_entry(walker, &kctx->jit_active_head, jit_node) { + backing += kbase_reg_current_backed_size(walker); + } + + return backing; +} + +void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, + size_t needed_pages) +{ + size_t jit_backing = 0; + size_t pages_to_trim = 0; + +#if !MALI_USE_CSF + lockdep_assert_held(&kctx->jctx.lock); +#endif /* !MALI_USE_CSF */ + lockdep_assert_held(&kctx->reg_lock); + lockdep_assert_held(&kctx->jit_evict_lock); + + jit_backing = get_jit_phys_backing(kctx); + + /* It is possible that this is the case - if this is the first + * allocation after "ignore_pressure_limit" allocation. + */ + if (jit_backing > kctx->jit_phys_pages_limit) { + pages_to_trim += (jit_backing - kctx->jit_phys_pages_limit) + + needed_pages; + } else { + size_t backed_diff = kctx->jit_phys_pages_limit - jit_backing; + + if (needed_pages > backed_diff) + pages_to_trim += needed_pages - backed_diff; + } + + if (pages_to_trim) { + size_t trimmed_pages = + kbase_mem_jit_trim_pages(kctx, pages_to_trim); + + /* This should never happen - we already asserted that + * we are not violating JIT pressure limit in earlier + * checks, which means that in-flight JIT allocations + * must have enough unused pages to satisfy the new + * allocation + */ + WARN_ON(trimmed_pages < pages_to_trim); + } +} +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +/** + * jit_allow_allocate() - check whether basic conditions are satisfied to allow + * a new JIT allocation + * + * @kctx: Pointer to the kbase context + * @info: Pointer to JIT allocation information for the new allocation + * @ignore_pressure_limit: Flag to indicate whether JIT pressure limit check + * should be ignored + * + * Return: true if allocation can be executed, false otherwise + */ +static bool jit_allow_allocate(struct kbase_context *kctx, + const struct base_jit_alloc_info *info, + bool ignore_pressure_limit) +{ +#if MALI_USE_CSF + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); +#else + lockdep_assert_held(&kctx->jctx.lock); +#endif + +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (!ignore_pressure_limit && + ((kctx->jit_phys_pages_limit <= kctx->jit_current_phys_pressure) || + (info->va_pages > (kctx->jit_phys_pages_limit - kctx->jit_current_phys_pressure)))) { + dev_dbg(kctx->kbdev->dev, + "Max JIT page allocations limit reached: active pages %llu, max pages %llu\n", + kctx->jit_current_phys_pressure + info->va_pages, + kctx->jit_phys_pages_limit); + return false; + } +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + if (kctx->jit_current_allocations >= kctx->jit_max_allocations) { + /* Too many current allocations */ + dev_dbg(kctx->kbdev->dev, + "Max JIT allocations limit reached: active allocations %d, max allocations %d\n", + kctx->jit_current_allocations, + kctx->jit_max_allocations); + return false; + } + + if (info->max_allocations > 0 && + kctx->jit_current_allocations_per_bin[info->bin_id] >= + info->max_allocations) { + /* Too many current allocations in this bin */ + dev_dbg(kctx->kbdev->dev, + "Per bin limit of max JIT allocations reached: bin_id %d, active allocations %d, max allocations %d\n", + info->bin_id, + kctx->jit_current_allocations_per_bin[info->bin_id], + info->max_allocations); + return false; + } + + return true; +} + +static struct kbase_va_region * +find_reasonable_region(const struct base_jit_alloc_info *info, + struct list_head *pool_head, bool ignore_usage_id) +{ + struct kbase_va_region *closest_reg = NULL; + struct kbase_va_region *walker; + size_t current_diff = SIZE_MAX; + + list_for_each_entry(walker, pool_head, jit_node) { + if ((ignore_usage_id || + walker->jit_usage_id == info->usage_id) && + walker->jit_bin_id == info->bin_id && + meet_size_and_tiler_align_top_requirements(walker, info)) { + size_t min_size, max_size, diff; + + /* + * The JIT allocations VA requirements have been met, + * it's suitable but other allocations might be a + * better fit. + */ + min_size = min_t(size_t, walker->gpu_alloc->nents, + info->commit_pages); + max_size = max_t(size_t, walker->gpu_alloc->nents, + info->commit_pages); + diff = max_size - min_size; + + if (current_diff > diff) { + current_diff = diff; + closest_reg = walker; + } + + /* The allocation is an exact match */ + if (current_diff == 0) + break; + } + } + + return closest_reg; +} + +struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, + const struct base_jit_alloc_info *info, + bool ignore_pressure_limit) +{ + struct kbase_va_region *reg = NULL; + struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; + int i; + +#if MALI_USE_CSF + lockdep_assert_held(&kctx->csf.kcpu_queues.lock); +#else + lockdep_assert_held(&kctx->jctx.lock); +#endif + + if (!jit_allow_allocate(kctx, info, ignore_pressure_limit)) + return NULL; + +#ifdef CONFIG_MALI_2MB_ALLOC + /* Preallocate memory for the sub-allocation structs */ + for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { + prealloc_sas[i] = kmalloc(sizeof(*prealloc_sas[i]), GFP_KERNEL); + if (!prealloc_sas[i]) + goto end; + } +#endif + + kbase_gpu_vm_lock(kctx); + mutex_lock(&kctx->jit_evict_lock); + + /* + * Scan the pool for an existing allocation which meets our + * requirements and remove it. + */ + if (info->usage_id != 0) + /* First scan for an allocation with the same usage ID */ + reg = find_reasonable_region(info, &kctx->jit_pool_head, false); + + if (!reg) + /* No allocation with the same usage ID, or usage IDs not in + * use. Search for an allocation we can reuse. + */ + reg = find_reasonable_region(info, &kctx->jit_pool_head, true); + + if (reg) { +#if MALI_JIT_PRESSURE_LIMIT_BASE + size_t needed_pages = 0; +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + int ret; + + /* + * Remove the found region from the pool and add it to the + * active list. + */ + list_move(®->jit_node, &kctx->jit_active_head); + + WARN_ON(reg->gpu_alloc->evicted); + + /* + * Remove the allocation from the eviction list as it's no + * longer eligible for eviction. This must be done before + * dropping the jit_evict_lock + */ + list_del_init(®->gpu_alloc->evict_node); + +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (!ignore_pressure_limit) { + if (info->commit_pages > reg->gpu_alloc->nents) + needed_pages = info->commit_pages - + reg->gpu_alloc->nents; + + /* Update early the recycled JIT region's estimate of + * used_pages to ensure it doesn't get trimmed + * undesirably. This is needed as the recycled JIT + * region has been added to the active list but the + * number of used pages for it would be zero, so it + * could get trimmed instead of other allocations only + * to be regrown later resulting in a breach of the JIT + * physical pressure limit. + * Also that trimming would disturb the accounting of + * physical pages, i.e. the VM stats, as the number of + * backing pages would have changed when the call to + * kbase_mem_evictable_unmark_reclaim is made. + * + * The second call to update pressure at the end of + * this function would effectively be a nop. + */ + kbase_jit_report_update_pressure( + kctx, reg, info->va_pages, + KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); + + kbase_jit_request_phys_increase_locked(kctx, + needed_pages); + } +#endif + mutex_unlock(&kctx->jit_evict_lock); + + /* kbase_jit_grow() can release & reacquire 'kctx->reg_lock', + * so any state protected by that lock might need to be + * re-evaluated if more code is added here in future. + */ + ret = kbase_jit_grow(kctx, info, reg, prealloc_sas); + +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (!ignore_pressure_limit) + kbase_jit_done_phys_increase(kctx, needed_pages); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + kbase_gpu_vm_unlock(kctx); + + if (ret < 0) { + /* + * An update to an allocation from the pool failed, + * chances are slim a new allocation would fair any + * better so return the allocation to the pool and + * return the function with failure. + */ + dev_dbg(kctx->kbdev->dev, + "JIT allocation resize failed: va_pages 0x%llx, commit_pages 0x%llx\n", + info->va_pages, info->commit_pages); +#if MALI_JIT_PRESSURE_LIMIT_BASE + /* Undo the early change made to the recycled JIT + * region's estimate of used_pages. + */ + if (!ignore_pressure_limit) { + kbase_jit_report_update_pressure( + kctx, reg, 0, + KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); + } +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + mutex_lock(&kctx->jit_evict_lock); + list_move(®->jit_node, &kctx->jit_pool_head); + mutex_unlock(&kctx->jit_evict_lock); + reg = NULL; + goto end; + } + } else { + /* No suitable JIT allocation was found so create a new one */ + u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | + BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF | + BASE_MEM_COHERENT_LOCAL | + BASEP_MEM_NO_USER_FREE; + u64 gpu_addr; + +#if !MALI_USE_CSF + if (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP) + flags |= BASE_MEM_TILER_ALIGN_TOP; +#endif /* !MALI_USE_CSF */ + + flags |= base_mem_group_id_set(kctx->jit_group_id); +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (!ignore_pressure_limit) { + flags |= BASEP_MEM_PERFORM_JIT_TRIM; + /* The corresponding call to 'done_phys_increase' would + * be made inside the kbase_mem_alloc(). + */ + kbase_jit_request_phys_increase_locked( + kctx, info->commit_pages); + } +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + mutex_unlock(&kctx->jit_evict_lock); + kbase_gpu_vm_unlock(kctx); + + reg = kbase_mem_alloc(kctx, info->va_pages, info->commit_pages, + info->extent, &flags, &gpu_addr); + if (!reg) { + /* Most likely not enough GPU virtual space left for + * the new JIT allocation. + */ + dev_dbg(kctx->kbdev->dev, + "Failed to allocate JIT memory: va_pages 0x%llx, commit_pages 0x%llx\n", + info->va_pages, info->commit_pages); + goto end; + } + + if (!ignore_pressure_limit) { + /* Due to enforcing of pressure limit, kbase_mem_alloc + * was instructed to perform the trimming which in turn + * would have ensured that the new JIT allocation is + * already in the jit_active_head list, so nothing to + * do here. + */ + WARN_ON(list_empty(®->jit_node)); + } else { + mutex_lock(&kctx->jit_evict_lock); + list_add(®->jit_node, &kctx->jit_active_head); + mutex_unlock(&kctx->jit_evict_lock); + } + } + + trace_mali_jit_alloc(reg, info->id); + + kctx->jit_current_allocations++; + kctx->jit_current_allocations_per_bin[info->bin_id]++; + + trace_jit_stats(kctx, info->bin_id, info->max_allocations); + + reg->jit_usage_id = info->usage_id; + reg->jit_bin_id = info->bin_id; + reg->flags |= KBASE_REG_ACTIVE_JIT_ALLOC; +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (info->flags & BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) + reg->flags = reg->flags | KBASE_REG_HEAP_INFO_IS_SIZE; + reg->heap_info_gpu_addr = info->heap_info_gpu_addr; + kbase_jit_report_update_pressure(kctx, reg, info->va_pages, + KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +end: + for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) + kfree(prealloc_sas[i]); + + return reg; +} + +void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + u64 old_pages; + + /* JIT id not immediately available here, so use 0u */ + trace_mali_jit_free(reg, 0u); + + /* Get current size of JIT region */ + old_pages = kbase_reg_current_backed_size(reg); + if (reg->initial_commit < old_pages) { + /* Free trim_level % of region, but don't go below initial + * commit size + */ + u64 new_size = MAX(reg->initial_commit, + div_u64(old_pages * (100 - kctx->trim_level), 100)); + u64 delta = old_pages - new_size; + + if (delta) + kbase_mem_shrink(kctx, reg, old_pages - delta); + } + +#if MALI_JIT_PRESSURE_LIMIT_BASE + reg->heap_info_gpu_addr = 0; + kbase_jit_report_update_pressure(kctx, reg, 0, + KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + kctx->jit_current_allocations--; + kctx->jit_current_allocations_per_bin[reg->jit_bin_id]--; + + trace_jit_stats(kctx, reg->jit_bin_id, UINT_MAX); + + kbase_mem_evictable_mark_reclaim(reg->gpu_alloc); + + kbase_gpu_vm_lock(kctx); + reg->flags |= KBASE_REG_DONT_NEED; + reg->flags &= ~KBASE_REG_ACTIVE_JIT_ALLOC; + kbase_mem_shrink_cpu_mapping(kctx, reg, 0, reg->gpu_alloc->nents); + kbase_gpu_vm_unlock(kctx); + + /* + * Add the allocation to the eviction list and the jit pool, after this + * point the shrink can reclaim it, or it may be reused. + */ + mutex_lock(&kctx->jit_evict_lock); + + /* This allocation can't already be on a list. */ + WARN_ON(!list_empty(®->gpu_alloc->evict_node)); + list_add(®->gpu_alloc->evict_node, &kctx->evict_list); + + list_move(®->jit_node, &kctx->jit_pool_head); + + mutex_unlock(&kctx->jit_evict_lock); +} + +void kbase_jit_backing_lost(struct kbase_va_region *reg) +{ + struct kbase_context *kctx = kbase_reg_flags_to_kctx(reg); + + if (WARN_ON(!kctx)) + return; + + lockdep_assert_held(&kctx->jit_evict_lock); + + /* + * JIT allocations will always be on a list, if the region + * is not on a list then it's not a JIT allocation. + */ + if (list_empty(®->jit_node)) + return; + + /* + * Freeing the allocation requires locks we might not be able + * to take now, so move the allocation to the free list and kick + * the worker which will do the freeing. + */ + list_move(®->jit_node, &kctx->jit_destroy_head); + + schedule_work(&kctx->jit_work); +} + +bool kbase_jit_evict(struct kbase_context *kctx) +{ + struct kbase_va_region *reg = NULL; + + lockdep_assert_held(&kctx->reg_lock); + + /* Free the oldest allocation from the pool */ + mutex_lock(&kctx->jit_evict_lock); + if (!list_empty(&kctx->jit_pool_head)) { + reg = list_entry(kctx->jit_pool_head.prev, + struct kbase_va_region, jit_node); + list_del(®->jit_node); + list_del_init(®->gpu_alloc->evict_node); + } + mutex_unlock(&kctx->jit_evict_lock); + + if (reg) { + reg->flags &= ~KBASE_REG_NO_USER_FREE; + kbase_mem_free_region(kctx, reg); + } + + return (reg != NULL); +} + +void kbase_jit_term(struct kbase_context *kctx) +{ + struct kbase_va_region *walker; + + /* Free all allocations for this context */ + + kbase_gpu_vm_lock(kctx); + mutex_lock(&kctx->jit_evict_lock); + /* Free all allocations from the pool */ + while (!list_empty(&kctx->jit_pool_head)) { + walker = list_first_entry(&kctx->jit_pool_head, + struct kbase_va_region, jit_node); + list_del(&walker->jit_node); + list_del_init(&walker->gpu_alloc->evict_node); + mutex_unlock(&kctx->jit_evict_lock); + walker->flags &= ~KBASE_REG_NO_USER_FREE; + kbase_mem_free_region(kctx, walker); + mutex_lock(&kctx->jit_evict_lock); + } + + /* Free all allocations from active list */ + while (!list_empty(&kctx->jit_active_head)) { + walker = list_first_entry(&kctx->jit_active_head, + struct kbase_va_region, jit_node); + list_del(&walker->jit_node); + list_del_init(&walker->gpu_alloc->evict_node); + mutex_unlock(&kctx->jit_evict_lock); + walker->flags &= ~KBASE_REG_NO_USER_FREE; + kbase_mem_free_region(kctx, walker); + mutex_lock(&kctx->jit_evict_lock); + } +#if MALI_JIT_PRESSURE_LIMIT_BASE + WARN_ON(kctx->jit_phys_pages_to_be_allocated); +#endif + mutex_unlock(&kctx->jit_evict_lock); + kbase_gpu_vm_unlock(kctx); + + /* + * Flush the freeing of allocations whose backing has been freed + * (i.e. everything in jit_destroy_head). + */ + cancel_work_sync(&kctx->jit_work); +} + +#if MALI_JIT_PRESSURE_LIMIT_BASE +void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, + struct kbase_va_region *reg, unsigned int flags) +{ + /* Offset to the location used for a JIT report within the GPU memory + * + * This constants only used for this debugging function - not useful + * anywhere else in kbase + */ + const u64 jit_report_gpu_mem_offset = sizeof(u64)*2; + + u64 addr_start; + struct kbase_vmap_struct mapping; + u64 *ptr; + + if (reg->heap_info_gpu_addr == 0ull) + goto out; + + /* Nothing else to trace in the case the memory just contains the + * size. Other tracepoints already record the relevant area of memory. + */ + if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) + goto out; + + addr_start = reg->heap_info_gpu_addr - jit_report_gpu_mem_offset; + + ptr = kbase_vmap(kctx, addr_start, KBASE_JIT_REPORT_GPU_MEM_SIZE, + &mapping); + if (!ptr) { + dev_warn(kctx->kbdev->dev, + "%s: JIT start=0x%llx unable to map memory near end pointer %llx\n", + __func__, reg->start_pfn << PAGE_SHIFT, + addr_start); + goto out; + } + + trace_mali_jit_report_gpu_mem(addr_start, reg->start_pfn << PAGE_SHIFT, + ptr, flags); + + kbase_vunmap(kctx, &mapping); +out: + return; +} +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +#if MALI_JIT_PRESSURE_LIMIT_BASE +void kbase_jit_report_update_pressure(struct kbase_context *kctx, + struct kbase_va_region *reg, u64 new_used_pages, + unsigned int flags) +{ + u64 diff; + +#if !MALI_USE_CSF + lockdep_assert_held(&kctx->jctx.lock); +#endif /* !MALI_USE_CSF */ + + trace_mali_jit_report_pressure(reg, new_used_pages, + kctx->jit_current_phys_pressure + new_used_pages - + reg->used_pages, + flags); + + if (WARN_ON(new_used_pages > reg->nr_pages)) + return; + + if (reg->used_pages > new_used_pages) { + /* We reduced the number of used pages */ + diff = reg->used_pages - new_used_pages; + + if (!WARN_ON(diff > kctx->jit_current_phys_pressure)) + kctx->jit_current_phys_pressure -= diff; + + reg->used_pages = new_used_pages; + } else { + /* We increased the number of used pages */ + diff = new_used_pages - reg->used_pages; + + if (!WARN_ON(diff > U64_MAX - kctx->jit_current_phys_pressure)) + kctx->jit_current_phys_pressure += diff; + + reg->used_pages = new_used_pages; + } + +} +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +bool kbase_has_exec_va_zone(struct kbase_context *kctx) +{ + bool has_exec_va_zone; + + kbase_gpu_vm_lock(kctx); + has_exec_va_zone = (kctx->exec_va_start != U64_MAX); + kbase_gpu_vm_unlock(kctx); + + return has_exec_va_zone; +} + +#if MALI_USE_CSF +static void kbase_jd_user_buf_unpin_pages(struct kbase_mem_phy_alloc *alloc) +{ + if (alloc->nents) { + struct page **pages = alloc->imported.user_buf.pages; + long i; + + WARN_ON(alloc->nents != alloc->imported.user_buf.nr_pages); + + for (i = 0; i < alloc->nents; i++) + put_page(pages[i]); + } +} +#endif + +int kbase_jd_user_buf_pin_pages(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; + struct page **pages = alloc->imported.user_buf.pages; + unsigned long address = alloc->imported.user_buf.address; + struct mm_struct *mm = alloc->imported.user_buf.mm; + long pinned_pages; + long i; + + if (WARN_ON(alloc->type != KBASE_MEM_TYPE_IMPORTED_USER_BUF)) + return -EINVAL; + + if (alloc->nents) { + if (WARN_ON(alloc->nents != alloc->imported.user_buf.nr_pages)) + return -EINVAL; + else + return 0; + } + + if (WARN_ON(reg->gpu_alloc->imported.user_buf.mm != current->mm)) + return -EINVAL; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + pinned_pages = get_user_pages(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, +#if KERNEL_VERSION(4, 4, 168) <= LINUX_VERSION_CODE && \ +KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL); +#else + reg->flags & KBASE_REG_GPU_WR, + 0, pages, NULL); +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + pinned_pages = get_user_pages_remote(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR, + 0, pages, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + pinned_pages = get_user_pages_remote(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) + pinned_pages = get_user_pages_remote(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL, NULL); +#else + pinned_pages = get_user_pages_remote(mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL, NULL); +#endif + + if (pinned_pages <= 0) + return pinned_pages; + + if (pinned_pages != alloc->imported.user_buf.nr_pages) { + for (i = 0; i < pinned_pages; i++) + put_page(pages[i]); + return -ENOMEM; + } + + alloc->nents = pinned_pages; + + return 0; +} + +static int kbase_jd_user_buf_map(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + long pinned_pages; + struct kbase_mem_phy_alloc *alloc; + struct page **pages; + struct tagged_addr *pa; + long i; + unsigned long address; + struct device *dev; + unsigned long offset; + unsigned long local_size; + unsigned long gwt_mask = ~0; + int err = kbase_jd_user_buf_pin_pages(kctx, reg); + + if (err) + return err; + + alloc = reg->gpu_alloc; + pa = kbase_get_gpu_phy_pages(reg); + address = alloc->imported.user_buf.address; + pinned_pages = alloc->nents; + pages = alloc->imported.user_buf.pages; + dev = kctx->kbdev->dev; + offset = address & ~PAGE_MASK; + local_size = alloc->imported.user_buf.size; + + for (i = 0; i < pinned_pages; i++) { + dma_addr_t dma_addr; + unsigned long min; + + min = MIN(PAGE_SIZE - offset, local_size); + dma_addr = dma_map_page(dev, pages[i], + offset, min, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) + goto unwind; + + alloc->imported.user_buf.dma_addrs[i] = dma_addr; + pa[i] = as_tagged(page_to_phys(pages[i])); + + local_size -= min; + offset = 0; + } + +#ifdef CONFIG_MALI_CINSTR_GWT + if (kctx->gwt_enabled) + gwt_mask = ~KBASE_REG_GPU_WR; +#endif + + err = kbase_mmu_insert_pages(kctx->kbdev, &kctx->mmu, reg->start_pfn, + pa, kbase_reg_current_backed_size(reg), + reg->flags & gwt_mask, kctx->as_nr, + alloc->group_id); + if (err == 0) + return 0; + + /* fall down */ +unwind: + alloc->nents = 0; + while (i--) { + dma_unmap_page(kctx->kbdev->dev, + alloc->imported.user_buf.dma_addrs[i], + PAGE_SIZE, DMA_BIDIRECTIONAL); + } + + while (++i < pinned_pages) { + put_page(pages[i]); + pages[i] = NULL; + } + + return err; +} + +/* This function would also perform the work of unpinning pages on Job Manager + * GPUs, which implies that a call to kbase_jd_user_buf_pin_pages() will NOT + * have a corresponding call to kbase_jd_user_buf_unpin_pages(). + */ +static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc, bool writeable) +{ + long i; + struct page **pages; + unsigned long size = alloc->imported.user_buf.size; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); + pages = alloc->imported.user_buf.pages; + for (i = 0; i < alloc->imported.user_buf.nr_pages; i++) { + unsigned long local_size; + dma_addr_t dma_addr = alloc->imported.user_buf.dma_addrs[i]; + + local_size = MIN(size, PAGE_SIZE - (dma_addr & ~PAGE_MASK)); + dma_unmap_page(kctx->kbdev->dev, dma_addr, local_size, + DMA_BIDIRECTIONAL); + if (writeable) + set_page_dirty_lock(pages[i]); +#if !MALI_USE_CSF + put_page(pages[i]); + pages[i] = NULL; +#endif + + size -= local_size; + } +#if !MALI_USE_CSF + alloc->nents = 0; +#endif +} + +int kbase_mem_copy_to_pinned_user_pages(struct page **dest_pages, + void *src_page, size_t *to_copy, unsigned int nr_pages, + unsigned int *target_page_nr, size_t offset) +{ + void *target_page = kmap(dest_pages[*target_page_nr]); + size_t chunk = PAGE_SIZE-offset; + + if (!target_page) { + pr_err("%s: kmap failure", __func__); + return -ENOMEM; + } + + chunk = min(chunk, *to_copy); + + memcpy(target_page + offset, src_page, chunk); + *to_copy -= chunk; + + kunmap(dest_pages[*target_page_nr]); + + *target_page_nr += 1; + if (*target_page_nr >= nr_pages || *to_copy == 0) + return 0; + + target_page = kmap(dest_pages[*target_page_nr]); + if (!target_page) { + pr_err("%s: kmap failure", __func__); + return -ENOMEM; + } + + KBASE_DEBUG_ASSERT(target_page); + + chunk = min(offset, *to_copy); + memcpy(target_page, src_page + PAGE_SIZE-offset, chunk); + *to_copy -= chunk; + + kunmap(dest_pages[*target_page_nr]); + + return 0; +} + +struct kbase_mem_phy_alloc *kbase_map_external_resource( + struct kbase_context *kctx, struct kbase_va_region *reg, + struct mm_struct *locked_mm) +{ + int err; + + lockdep_assert_held(&kctx->reg_lock); + + /* decide what needs to happen for this resource */ + switch (reg->gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { + if ((reg->gpu_alloc->imported.user_buf.mm != locked_mm) && + (!reg->gpu_alloc->nents)) + goto exit; + + reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++; + if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) { + err = kbase_jd_user_buf_map(kctx, reg); + if (err) { + reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--; + goto exit; + } + } + } + break; + case KBASE_MEM_TYPE_IMPORTED_UMM: { + err = kbase_mem_umm_map(kctx, reg); + if (err) + goto exit; + break; + } + default: + goto exit; + } + + return kbase_mem_phy_alloc_get(reg->gpu_alloc); +exit: + return NULL; +} + +void kbase_unmap_external_resource(struct kbase_context *kctx, + struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc) +{ + switch (alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_UMM: { + kbase_mem_umm_unmap(kctx, reg, alloc); + } + break; + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { + alloc->imported.user_buf.current_mapping_usage_count--; + + if (0 == alloc->imported.user_buf.current_mapping_usage_count) { + bool writeable = true; + + if (!kbase_is_region_invalid_or_free(reg) && + reg->gpu_alloc == alloc) + kbase_mmu_teardown_pages( + kctx->kbdev, + &kctx->mmu, + reg->start_pfn, + kbase_reg_current_backed_size(reg), + kctx->as_nr); + + if (reg && ((reg->flags & KBASE_REG_GPU_WR) == 0)) + writeable = false; + + kbase_jd_user_buf_unmap(kctx, alloc, writeable); + } + } + break; + default: + break; + } + kbase_mem_phy_alloc_put(alloc); +} + +struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( + struct kbase_context *kctx, u64 gpu_addr) +{ + struct kbase_ctx_ext_res_meta *meta = NULL; + struct kbase_ctx_ext_res_meta *walker; + + lockdep_assert_held(&kctx->reg_lock); + + /* + * Walk the per context external resource metadata list for the + * metadata which matches the region which is being acquired. + */ + list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) { + if (walker->gpu_addr == gpu_addr) { + meta = walker; + meta->ref++; + break; + } + } + + /* No metadata exists so create one. */ + if (!meta) { + struct kbase_va_region *reg; + + /* Find the region */ + reg = kbase_region_tracker_find_region_enclosing_address( + kctx, gpu_addr); + if (kbase_is_region_invalid_or_free(reg)) + goto failed; + + /* Allocate the metadata object */ + meta = kzalloc(sizeof(*meta), GFP_KERNEL); + if (!meta) + goto failed; + + /* + * Fill in the metadata object and acquire a reference + * for the physical resource. + */ + meta->alloc = kbase_map_external_resource(kctx, reg, NULL); + meta->ref = 1; + + if (!meta->alloc) + goto fail_map; + + meta->gpu_addr = reg->start_pfn << PAGE_SHIFT; + + list_add(&meta->ext_res_node, &kctx->ext_res_meta_head); + } + + return meta; + +fail_map: + kfree(meta); +failed: + return NULL; +} + +static struct kbase_ctx_ext_res_meta * +find_sticky_resource_meta(struct kbase_context *kctx, u64 gpu_addr) +{ + struct kbase_ctx_ext_res_meta *walker; + + lockdep_assert_held(&kctx->reg_lock); + + /* + * Walk the per context external resource metadata list for the + * metadata which matches the region which is being released. + */ + list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) + if (walker->gpu_addr == gpu_addr) + return walker; + + return NULL; +} + +static void release_sticky_resource_meta(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta) +{ + struct kbase_va_region *reg; + + /* Drop the physical memory reference and free the metadata. */ + reg = kbase_region_tracker_find_region_enclosing_address( + kctx, + meta->gpu_addr); + + kbase_unmap_external_resource(kctx, reg, meta->alloc); + list_del(&meta->ext_res_node); + kfree(meta); +} + +bool kbase_sticky_resource_release(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) +{ + lockdep_assert_held(&kctx->reg_lock); + + /* Search of the metadata if one isn't provided. */ + if (!meta) + meta = find_sticky_resource_meta(kctx, gpu_addr); + + /* No metadata so just return. */ + if (!meta) + return false; + + if (--meta->ref != 0) + return true; + + release_sticky_resource_meta(kctx, meta); + + return true; +} + +bool kbase_sticky_resource_release_force(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) +{ + lockdep_assert_held(&kctx->reg_lock); + + /* Search of the metadata if one isn't provided. */ + if (!meta) + meta = find_sticky_resource_meta(kctx, gpu_addr); + + /* No metadata so just return. */ + if (!meta) + return false; + + release_sticky_resource_meta(kctx, meta); + + return true; +} + +int kbase_sticky_resource_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->ext_res_meta_head); + + return 0; +} + +void kbase_sticky_resource_term(struct kbase_context *kctx) +{ + struct kbase_ctx_ext_res_meta *walker; + + lockdep_assert_held(&kctx->reg_lock); + + /* + * Free any sticky resources which haven't been unmapped. + * + * Note: + * We don't care about refcounts at this point as no future + * references to the meta data will be made. + * Region termination would find these if we didn't free them + * here, but it's more efficient if we do the clean up here. + */ + while (!list_empty(&kctx->ext_res_meta_head)) { + walker = list_first_entry(&kctx->ext_res_meta_head, + struct kbase_ctx_ext_res_meta, ext_res_node); + + kbase_sticky_resource_release_force(kctx, walker, 0); + } +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.h b/drivers/gpu/arm/bifrost/mali_kbase_mem.h new file mode 100755 index 000000000000..2238fbfe9e99 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.h @@ -0,0 +1,1962 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_mem.h + * Base kernel memory APIs + */ + +#ifndef _KBASE_MEM_H_ +#define _KBASE_MEM_H_ + +#ifndef _KBASE_H_ +#error "Don't include this file directly, use mali_kbase.h instead" +#endif + +#include +#include "mali_base_kernel.h" +#include +#include "mali_kbase_pm.h" +#include "mali_kbase_defs.h" +/* Required for kbase_mem_evictable_unmake */ +#include "mali_kbase_mem_linux.h" + +static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, + int pages); + +/* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */ +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */ + +/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages. +The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and +page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table +updates and generates duplicate page faults as the page table information used by the MMU is not valid. */ +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */ + +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */ + +/* This must always be a power of 2 */ +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2) +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316) +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630) +/** + * A CPU mapping + */ +struct kbase_cpu_mapping { + struct list_head mappings_list; + struct kbase_mem_phy_alloc *alloc; + struct kbase_context *kctx; + struct kbase_va_region *region; + int count; + int free_on_close; +}; + +enum kbase_memory_type { + KBASE_MEM_TYPE_NATIVE, + KBASE_MEM_TYPE_IMPORTED_UMM, + KBASE_MEM_TYPE_IMPORTED_USER_BUF, + KBASE_MEM_TYPE_ALIAS, + KBASE_MEM_TYPE_RAW +}; + +/* internal structure, mirroring base_mem_aliasing_info, + * but with alloc instead of a gpu va (handle) */ +struct kbase_aliased { + struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */ + u64 offset; /* in pages */ + u64 length; /* in pages */ +}; + +/** + * @brief Physical pages tracking object properties + */ +#define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED (1u << 0) +#define KBASE_MEM_PHY_ALLOC_LARGE (1u << 1) + +/* struct kbase_mem_phy_alloc - Physical pages tracking object. + * + * Set up to track N pages. + * N not stored here, the creator holds that info. + * This object only tracks how many elements are actually valid (present). + * Changing of nents or *pages should only happen if the kbase_mem_phy_alloc + * is not shared with another region or client. CPU mappings are OK to + * exist when changing, as long as the tracked mappings objects are + * updated as part of the change. + * + * @kref: number of users of this alloc + * @gpu_mappings: count number of times mapped on the GPU + * @nents: 0..N + * @pages: N elements, only 0..nents are valid + * @mappings: List of CPU mappings of this physical memory allocation. + * @evict_node: Node used to store this allocation on the eviction list + * @evicted: Physical backing size when the pages where evicted + * @reg: Back reference to the region structure which created this + * allocation, or NULL if it has been freed. + * @type: type of buffer + * @permanent_map: Kernel side mapping of the alloc, shall never be + * referred directly. kbase_phy_alloc_mapping_get() & + * kbase_phy_alloc_mapping_put() pair should be used + * around access to the kernel-side CPU mapping so that + * mapping doesn't disappear whilst it is being accessed. + * @properties: Bitmask of properties, e.g. KBASE_MEM_PHY_ALLOC_LARGE. + * @group_id: A memory group ID to be passed to a platform-specific + * memory group manager, if present. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @imported: member in union valid based on @a type + */ +struct kbase_mem_phy_alloc { + struct kref kref; + atomic_t gpu_mappings; + size_t nents; + struct tagged_addr *pages; + struct list_head mappings; + struct list_head evict_node; + size_t evicted; + struct kbase_va_region *reg; + enum kbase_memory_type type; + struct kbase_vmap_struct *permanent_map; + u8 properties; + u8 group_id; + + union { + struct { + struct kbase_context *kctx; + struct dma_buf *dma_buf; + struct dma_buf_attachment *dma_attachment; + unsigned int current_mapping_usage_count; + struct sg_table *sgt; + bool need_sync; + } umm; + struct { + u64 stride; + size_t nents; + struct kbase_aliased *aliased; + } alias; + struct { + struct kbase_context *kctx; + /* Number of pages in this structure, including *pages. + * Used for kernel memory tracking. + */ + size_t nr_struct_pages; + } native; + struct kbase_alloc_import_user_buf { + unsigned long address; + unsigned long size; + unsigned long nr_pages; + struct page **pages; + /* top bit (1<<31) of current_mapping_usage_count + * specifies that this import was pinned on import + * See PINNED_ON_IMPORT + */ + u32 current_mapping_usage_count; + struct mm_struct *mm; + dma_addr_t *dma_addrs; + } user_buf; + } imported; +}; + +/* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is + * used to signify that a buffer was pinned when it was imported. Since the + * reference count is limited by the number of atoms that can be submitted at + * once there should be no danger of overflowing into this bit. + * Stealing the top bit also has the benefit that + * current_mapping_usage_count != 0 if and only if the buffer is mapped. + */ +#define PINNED_ON_IMPORT (1<<31) + +/** + * enum kbase_jit_report_flags - Flags for just-in-time memory allocation + * pressure limit functions + * @KBASE_JIT_REPORT_ON_ALLOC_OR_FREE: Notifying about an update happening due + * to a just-in-time memory allocation or free + * + * Used to control flow within pressure limit related functions, or to provide + * extra debugging information + */ +enum kbase_jit_report_flags { + KBASE_JIT_REPORT_ON_ALLOC_OR_FREE = (1u << 0) +}; + +static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) +{ + KBASE_DEBUG_ASSERT(alloc); + /* we only track mappings of NATIVE buffers */ + if (alloc->type == KBASE_MEM_TYPE_NATIVE) + atomic_inc(&alloc->gpu_mappings); +} + +static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc *alloc) +{ + KBASE_DEBUG_ASSERT(alloc); + /* we only track mappings of NATIVE buffers */ + if (alloc->type == KBASE_MEM_TYPE_NATIVE) + if (0 > atomic_dec_return(&alloc->gpu_mappings)) { + pr_err("Mismatched %s:\n", __func__); + dump_stack(); + } +} + +/** + * kbase_mem_is_imported - Indicate whether a memory type is imported + * + * @type: the memory type + * + * Return: true if the memory type is imported, false otherwise + */ +static inline bool kbase_mem_is_imported(enum kbase_memory_type type) +{ + return (type == KBASE_MEM_TYPE_IMPORTED_UMM) || + (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); +} + +void kbase_mem_kref_free(struct kref *kref); + +int kbase_mem_init(struct kbase_device *kbdev); +void kbase_mem_halt(struct kbase_device *kbdev); +void kbase_mem_term(struct kbase_device *kbdev); + +static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc) +{ + kref_get(&alloc->kref); + return alloc; +} + +static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc) +{ + kref_put(&alloc->kref, kbase_mem_kref_free); + return NULL; +} + +/** + * A GPU memory region, and attributes for CPU mappings. + * + * @rblink: Node in a red-black tree of memory regions within the same zone of + * the GPU's virtual address space. + * @link: Links to neighboring items in a list of growable memory regions + * that triggered incremental rendering by growing too much. + * @rbtree: Backlink to the red-black tree of memory regions. + * @start_pfn: The Page Frame Number in GPU virtual address space. + * @nr_pages: The size of the region in pages. + * @initial_commit: Initial commit, for aligning the start address and + * correctly growing KBASE_REG_TILER_ALIGN_TOP regions. + * @threshold_pages: If non-zero and the amount of memory committed to a region + * that can grow on page fault exceeds this number of pages + * then the driver switches to incremental rendering. + * @extent: Number of pages allocated on page fault. + * @cpu_alloc: The physical memory we mmap to the CPU when mapping this region. + * @gpu_alloc: The physical memory we mmap to the GPU when mapping this region. + * @jit_node: Links to neighboring regions in the just-in-time memory pool. + * @jit_usage_id: The last just-in-time memory usage ID for this region. + * @jit_bin_id: The just-in-time memory bin this region came from. + * @va_refcnt: Number of users of this region. Protected by reg_lock. + */ +struct kbase_va_region { + struct rb_node rblink; + struct list_head link; + struct rb_root *rbtree; + u64 start_pfn; + size_t nr_pages; + size_t initial_commit; + size_t threshold_pages; + +/* Free region */ +#define KBASE_REG_FREE (1ul << 0) +/* CPU write access */ +#define KBASE_REG_CPU_WR (1ul << 1) +/* GPU write access */ +#define KBASE_REG_GPU_WR (1ul << 2) +/* No eXecute flag */ +#define KBASE_REG_GPU_NX (1ul << 3) +/* Is CPU cached? */ +#define KBASE_REG_CPU_CACHED (1ul << 4) +/* Is GPU cached? + * Some components within the GPU might only be able to access memory that is + * GPU cacheable. Refer to the specific GPU implementation for more details. + */ +#define KBASE_REG_GPU_CACHED (1ul << 5) + +#define KBASE_REG_GROWABLE (1ul << 6) +/* Can grow on pf? */ +#define KBASE_REG_PF_GROW (1ul << 7) + +/* Allocation doesn't straddle the 4GB boundary in GPU virtual space */ +#define KBASE_REG_GPU_VA_SAME_4GB_PAGE (1ul << 8) + +/* inner shareable coherency */ +#define KBASE_REG_SHARE_IN (1ul << 9) +/* inner & outer shareable coherency */ +#define KBASE_REG_SHARE_BOTH (1ul << 10) + +/* Space for 4 different zones */ +#define KBASE_REG_ZONE_MASK (3ul << 11) +#define KBASE_REG_ZONE(x) (((x) & 3) << 11) + +/* GPU read access */ +#define KBASE_REG_GPU_RD (1ul<<13) +/* CPU read access */ +#define KBASE_REG_CPU_RD (1ul<<14) + +/* Index of chosen MEMATTR for this region (0..7) */ +#define KBASE_REG_MEMATTR_MASK (7ul << 16) +#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16) +#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16) + +#define KBASE_REG_PROTECTED (1ul << 19) + +#define KBASE_REG_DONT_NEED (1ul << 20) + +/* Imported buffer is padded? */ +#define KBASE_REG_IMPORT_PAD (1ul << 21) + +#if MALI_USE_CSF +/* CSF event memory */ +#define KBASE_REG_CSF_EVENT (1ul << 22) +#else +/* Bit 22 is reserved. + * + * Do not remove, use the next unreserved bit for new flags + */ +#define KBASE_REG_RESERVED_BIT_22 (1ul << 22) +#endif + +#if !MALI_USE_CSF +/* The top of the initial commit is aligned to extent pages. + * Extent must be a power of 2 */ +#define KBASE_REG_TILER_ALIGN_TOP (1ul << 23) +#else +/* Bit 23 is reserved. + * + * Do not remove, use the next unreserved bit for new flags + */ +#define KBASE_REG_RESERVED_BIT_23 (1ul << 23) +#endif /* !MALI_USE_CSF */ + +/* Whilst this flag is set the GPU allocation is not supposed to be freed by + * user space. The flag will remain set for the lifetime of JIT allocations. + */ +#define KBASE_REG_NO_USER_FREE (1ul << 24) + +/* Memory has permanent kernel side mapping */ +#define KBASE_REG_PERMANENT_KERNEL_MAPPING (1ul << 25) + +/* GPU VA region has been freed by the userspace, but still remains allocated + * due to the reference held by CPU mappings created on the GPU VA region. + * + * A region with this flag set has had kbase_gpu_munmap() called on it, but can + * still be looked-up in the region tracker as a non-free region. Hence must + * not create or update any more GPU mappings on such regions because they will + * not be unmapped when the region is finally destroyed. + * + * Since such regions are still present in the region tracker, new allocations + * attempted with BASE_MEM_SAME_VA might fail if their address intersects with + * a region with this flag set. + * + * In addition, this flag indicates the gpu_alloc member might no longer valid + * e.g. in infinite cache simulation. + */ +#define KBASE_REG_VA_FREED (1ul << 26) + +/* If set, the heap info address points to a u32 holding the used size in bytes; + * otherwise it points to a u64 holding the lowest address of unused memory. + */ +#define KBASE_REG_HEAP_INFO_IS_SIZE (1ul << 27) + +/* Allocation is actively used for JIT memory */ +#define KBASE_REG_ACTIVE_JIT_ALLOC (1ul << 28) + +#define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0) + +/* only used with 32-bit clients */ +/* + * On a 32bit platform, custom VA should be wired from 4GB + * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface + * limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference). + * So we put the default limit to the maximum possible on Linux and shrink + * it down, if required by the GPU, during initialization. + */ + +#define KBASE_REG_ZONE_CUSTOM_VA KBASE_REG_ZONE(1) +#define KBASE_REG_ZONE_CUSTOM_VA_BASE (0x100000000ULL >> PAGE_SHIFT) +#define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) +/* end 32-bit clients only */ + +/* The starting address and size of the GPU-executable zone are dynamic + * and depend on the platform and the number of pages requested by the + * user process, with an upper limit of 4 GB. + */ +#define KBASE_REG_ZONE_EXEC_VA KBASE_REG_ZONE(2) +#define KBASE_REG_ZONE_EXEC_VA_MAX_PAGES ((1ULL << 32) >> PAGE_SHIFT) /* 4 GB */ + +#if MALI_USE_CSF +#define KBASE_REG_ZONE_MCU_SHARED KBASE_REG_ZONE(3) +#define KBASE_REG_ZONE_MCU_SHARED_BASE (0x04000000ULL >> PAGE_SHIFT) +#define KBASE_REG_ZONE_MCU_SHARED_SIZE (((0x08000000ULL) >> PAGE_SHIFT) - \ + KBASE_REG_ZONE_MCU_SHARED_BASE) +#endif + + unsigned long flags; + size_t extent; + struct kbase_mem_phy_alloc *cpu_alloc; + struct kbase_mem_phy_alloc *gpu_alloc; + struct list_head jit_node; + u16 jit_usage_id; + u8 jit_bin_id; +#if MALI_JIT_PRESSURE_LIMIT_BASE + /* Pointer to an object in GPU memory defining an end of an allocated + * region + * + * The object can be one of: + * - u32 value defining the size of the region + * - u64 pointer first unused byte in the region + * + * The interpretation of the object depends on + * BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE flag in jit_info_flags - if it is + * set, the heap info object should be interpreted as size. + */ + u64 heap_info_gpu_addr; + + /* The current estimate of the number of pages used, which in normal + * use is either: + * - the initial estimate == va_pages + * - the actual pages used, as found by a JIT usage report + * + * Note that since the value is calculated from GPU memory after a JIT + * usage report, at any point in time it is allowed to take a random + * value that is no greater than va_pages (e.g. it may be greater than + * gpu_alloc->nents) + */ + size_t used_pages; +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + int va_refcnt; +}; + +/* Special marker for failed JIT allocations that still must be marked as + * in-use + */ +#define KBASE_RESERVED_REG_JIT_ALLOC ((struct kbase_va_region *)-1) + +static inline bool kbase_is_region_free(struct kbase_va_region *reg) +{ + return (!reg || reg->flags & KBASE_REG_FREE); +} + +static inline bool kbase_is_region_invalid(struct kbase_va_region *reg) +{ + return (!reg || reg->flags & KBASE_REG_VA_FREED); +} + +static inline bool kbase_is_region_invalid_or_free(struct kbase_va_region *reg) +{ + /* Possibly not all functions that find regions would be using this + * helper, so they need to be checked when maintaining this function. + */ + return (kbase_is_region_invalid(reg) || kbase_is_region_free(reg)); +} + +int kbase_remove_va_region(struct kbase_va_region *reg); +static inline void kbase_region_refcnt_free(struct kbase_va_region *reg) +{ + /* If region was mapped then remove va region*/ + if (reg->start_pfn) + kbase_remove_va_region(reg); + + /* To detect use-after-free in debug builds */ + KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE); + kfree(reg); +} + +static inline struct kbase_va_region *kbase_va_region_alloc_get( + struct kbase_context *kctx, struct kbase_va_region *region) +{ + lockdep_assert_held(&kctx->reg_lock); + + WARN_ON(!region->va_refcnt); + + /* non-atomic as kctx->reg_lock is held */ + dev_dbg(kctx->kbdev->dev, "va_refcnt %d before get %p\n", + region->va_refcnt, (void *)region); + region->va_refcnt++; + + return region; +} + +static inline struct kbase_va_region *kbase_va_region_alloc_put( + struct kbase_context *kctx, struct kbase_va_region *region) +{ + lockdep_assert_held(&kctx->reg_lock); + + WARN_ON(region->va_refcnt <= 0); + WARN_ON(region->flags & KBASE_REG_FREE); + + /* non-atomic as kctx->reg_lock is held */ + region->va_refcnt--; + dev_dbg(kctx->kbdev->dev, "va_refcnt %d after put %p\n", + region->va_refcnt, (void *)region); + if (!region->va_refcnt) + kbase_region_refcnt_free(region); + + return NULL; +} + +/* Common functions */ +static inline struct tagged_addr *kbase_get_cpu_phy_pages( + struct kbase_va_region *reg) +{ + KBASE_DEBUG_ASSERT(reg); + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); + + return reg->cpu_alloc->pages; +} + +static inline struct tagged_addr *kbase_get_gpu_phy_pages( + struct kbase_va_region *reg) +{ + KBASE_DEBUG_ASSERT(reg); + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); + + return reg->gpu_alloc->pages; +} + +static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg) +{ + KBASE_DEBUG_ASSERT(reg); + /* if no alloc object the backed size naturally is 0 */ + if (!reg->cpu_alloc) + return 0; + + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); + + return reg->cpu_alloc->nents; +} + +#define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */ + +static inline struct kbase_mem_phy_alloc *kbase_alloc_create( + struct kbase_context *kctx, size_t nr_pages, + enum kbase_memory_type type, int group_id) +{ + struct kbase_mem_phy_alloc *alloc; + size_t alloc_size = sizeof(*alloc) + sizeof(*alloc->pages) * nr_pages; + size_t per_page_size = sizeof(*alloc->pages); + + /* Imported pages may have page private data already in use */ + if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { + alloc_size += nr_pages * + sizeof(*alloc->imported.user_buf.dma_addrs); + per_page_size += sizeof(*alloc->imported.user_buf.dma_addrs); + } + + /* + * Prevent nr_pages*per_page_size + sizeof(*alloc) from + * wrapping around. + */ + if (nr_pages > ((((size_t) -1) - sizeof(*alloc)) + / per_page_size)) + return ERR_PTR(-ENOMEM); + + /* Allocate based on the size to reduce internal fragmentation of vmem */ + if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) + alloc = vzalloc(alloc_size); + else + alloc = kzalloc(alloc_size, GFP_KERNEL); + + if (!alloc) + return ERR_PTR(-ENOMEM); + + if (type == KBASE_MEM_TYPE_NATIVE) { + alloc->imported.native.nr_struct_pages = + (alloc_size + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + kbase_process_page_usage_inc(kctx, + alloc->imported.native.nr_struct_pages); + } + + /* Store allocation method */ + if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) + alloc->properties |= KBASE_MEM_PHY_ALLOC_LARGE; + + kref_init(&alloc->kref); + atomic_set(&alloc->gpu_mappings, 0); + alloc->nents = 0; + alloc->pages = (void *)(alloc + 1); + INIT_LIST_HEAD(&alloc->mappings); + alloc->type = type; + alloc->group_id = group_id; + + if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) + alloc->imported.user_buf.dma_addrs = + (void *) (alloc->pages + nr_pages); + + return alloc; +} + +static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, + struct kbase_context *kctx, int group_id) +{ + KBASE_DEBUG_ASSERT(reg); + KBASE_DEBUG_ASSERT(!reg->cpu_alloc); + KBASE_DEBUG_ASSERT(!reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE); + + reg->cpu_alloc = kbase_alloc_create(kctx, reg->nr_pages, + KBASE_MEM_TYPE_NATIVE, group_id); + if (IS_ERR(reg->cpu_alloc)) + return PTR_ERR(reg->cpu_alloc); + else if (!reg->cpu_alloc) + return -ENOMEM; + + reg->cpu_alloc->imported.native.kctx = kctx; + if (kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE) + && (reg->flags & KBASE_REG_CPU_CACHED)) { + reg->gpu_alloc = kbase_alloc_create(kctx, reg->nr_pages, + KBASE_MEM_TYPE_NATIVE, group_id); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) { + kbase_mem_phy_alloc_put(reg->cpu_alloc); + return -ENOMEM; + } + reg->gpu_alloc->imported.native.kctx = kctx; + } else { + reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + } + + mutex_lock(&kctx->jit_evict_lock); + INIT_LIST_HEAD(®->cpu_alloc->evict_node); + INIT_LIST_HEAD(®->gpu_alloc->evict_node); + mutex_unlock(&kctx->jit_evict_lock); + + reg->flags &= ~KBASE_REG_FREE; + + return 0; +} + +/* + * Max size for kbdev memory pool (in pages) + */ +#define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT) + +/* + * Max size for kctx memory pool (in pages) + */ +#define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) + +/* + * The order required for a 2MB page allocation (2^order * 4KB = 2MB) + */ +#define KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER 9 + +/* + * The order required for a 4KB page allocation + */ +#define KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER 0 + +/** + * kbase_mem_pool_config_set_max_size - Set maximum number of free pages in + * initial configuration of a memory pool + * + * @config: Initial configuration for a physical memory pool + * @max_size: Maximum number of free pages that a pool created from + * @config can hold + */ +static inline void kbase_mem_pool_config_set_max_size( + struct kbase_mem_pool_config *const config, size_t const max_size) +{ + WRITE_ONCE(config->max_size, max_size); +} + +/** + * kbase_mem_pool_config_get_max_size - Get maximum number of free pages from + * initial configuration of a memory pool + * + * @config: Initial configuration for a physical memory pool + * + * Return: Maximum number of free pages that a pool created from @config + * can hold + */ +static inline size_t kbase_mem_pool_config_get_max_size( + const struct kbase_mem_pool_config *const config) +{ + return READ_ONCE(config->max_size); +} + +/** + * kbase_mem_pool_init - Create a memory pool for a kbase device + * @pool: Memory pool to initialize + * @config: Initial configuration for the memory pool + * @order: Page order for physical page size (order=0=>4kB, order=9=>2MB) + * @group_id: A memory group ID to be passed to a platform-specific + * memory group manager, if present. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @kbdev: Kbase device where memory is used + * @next_pool: Pointer to the next pool or NULL. + * + * Allocations from @pool are in whole pages. Each @pool has a free list where + * pages can be quickly allocated from. The free list is initially empty and + * filled whenever pages are freed back to the pool. The number of free pages + * in the pool will in general not exceed @max_size, but the pool may in + * certain corner cases grow above @max_size. + * + * If @next_pool is not NULL, we will allocate from @next_pool before going to + * the memory group manager. Similarly pages can spill over to @next_pool when + * @pool is full. Pages are zeroed before they spill over to another pool, to + * prevent leaking information between applications. + * + * A shrinker is registered so that Linux mm can reclaim pages from the pool as + * needed. + * + * Return: 0 on success, negative -errno on error + */ +int kbase_mem_pool_init(struct kbase_mem_pool *pool, + const struct kbase_mem_pool_config *config, + unsigned int order, + int group_id, + struct kbase_device *kbdev, + struct kbase_mem_pool *next_pool); + +/** + * kbase_mem_pool_term - Destroy a memory pool + * @pool: Memory pool to destroy + * + * Pages in the pool will spill over to @next_pool (if available) or freed to + * the kernel. + */ +void kbase_mem_pool_term(struct kbase_mem_pool *pool); + +/** + * kbase_mem_pool_alloc - Allocate a page from memory pool + * @pool: Memory pool to allocate from + * + * Allocations from the pool are made as follows: + * 1. If there are free pages in the pool, allocate a page from @pool. + * 2. Otherwise, if @next_pool is not NULL and has free pages, allocate a page + * from @next_pool. + * 3. Return NULL if no memory in the pool + * + * Return: Pointer to allocated page, or NULL if allocation failed. + * + * Note : This function should not be used if the pool lock is held. Use + * kbase_mem_pool_alloc_locked() instead. + */ +struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool); + +/** + * kbase_mem_pool_alloc_locked - Allocate a page from memory pool + * @pool: Memory pool to allocate from + * + * If there are free pages in the pool, this function allocates a page from + * @pool. This function does not use @next_pool. + * + * Return: Pointer to allocated page, or NULL if allocation failed. + * + * Note : Caller must hold the pool lock. + */ +struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool); + +/** + * kbase_mem_pool_free - Free a page to memory pool + * @pool: Memory pool where page should be freed + * @page: Page to free to the pool + * @dirty: Whether some of the page may be dirty in the cache. + * + * Pages are freed to the pool as follows: + * 1. If @pool is not full, add @page to @pool. + * 2. Otherwise, if @next_pool is not NULL and not full, add @page to + * @next_pool. + * 3. Finally, free @page to the kernel. + * + * Note : This function should not be used if the pool lock is held. Use + * kbase_mem_pool_free_locked() instead. + */ +void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page, + bool dirty); + +/** + * kbase_mem_pool_free_locked - Free a page to memory pool + * @pool: Memory pool where page should be freed + * @p: Page to free to the pool + * @dirty: Whether some of the page may be dirty in the cache. + * + * If @pool is not full, this function adds @page to @pool. Otherwise, @page is + * freed to the kernel. This function does not use @next_pool. + * + * Note : Caller must hold the pool lock. + */ +void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, + bool dirty); + +/** + * kbase_mem_pool_alloc_pages - Allocate pages from memory pool + * @pool: Memory pool to allocate from + * @nr_4k_pages: Number of pages to allocate + * @pages: Pointer to array where the physical address of the allocated + * pages will be stored. + * @partial_allowed: If fewer pages allocated is allowed + * + * Like kbase_mem_pool_alloc() but optimized for allocating many pages. + * + * Return: + * On success number of pages allocated (could be less than nr_pages if + * partial_allowed). + * On error an error code. + * + * Note : This function should not be used if the pool lock is held. Use + * kbase_mem_pool_alloc_pages_locked() instead. + * + * The caller must not hold vm_lock, as this could cause a deadlock if + * the kernel OoM killer runs. If the caller must allocate pages while holding + * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead. + */ +int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, + struct tagged_addr *pages, bool partial_allowed); + +/** + * kbase_mem_pool_alloc_pages_locked - Allocate pages from memory pool + * @pool: Memory pool to allocate from + * @nr_4k_pages: Number of pages to allocate + * @pages: Pointer to array where the physical address of the allocated + * pages will be stored. + * + * Like kbase_mem_pool_alloc() but optimized for allocating many pages. This + * version does not allocate new pages from the kernel, and therefore will never + * trigger the OoM killer. Therefore, it can be run while the vm_lock is held. + * + * As new pages can not be allocated, the caller must ensure there are + * sufficient pages in the pool. Usage of this function should look like : + * + * kbase_gpu_vm_lock(kctx); + * kbase_mem_pool_lock(pool) + * while (kbase_mem_pool_size(pool) < pages_required) { + * kbase_mem_pool_unlock(pool) + * kbase_gpu_vm_unlock(kctx); + * kbase_mem_pool_grow(pool) + * kbase_gpu_vm_lock(kctx); + * kbase_mem_pool_lock(pool) + * } + * kbase_mem_pool_alloc_pages_locked(pool) + * kbase_mem_pool_unlock(pool) + * Perform other processing that requires vm_lock... + * kbase_gpu_vm_unlock(kctx); + * + * This ensures that the pool can be grown to the required size and that the + * allocation can complete without another thread using the newly grown pages. + * + * Return: + * On success number of pages allocated. + * On error an error code. + * + * Note : Caller must hold the pool lock. + */ +int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, + size_t nr_4k_pages, struct tagged_addr *pages); + +/** + * kbase_mem_pool_free_pages - Free pages to memory pool + * @pool: Memory pool where pages should be freed + * @nr_pages: Number of pages to free + * @pages: Pointer to array holding the physical addresses of the pages to + * free. + * @dirty: Whether any pages may be dirty in the cache. + * @reclaimed: Whether the pages where reclaimable and thus should bypass + * the pool and go straight to the kernel. + * + * Like kbase_mem_pool_free() but optimized for freeing many pages. + */ +void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, + struct tagged_addr *pages, bool dirty, bool reclaimed); + +/** + * kbase_mem_pool_free_pages_locked - Free pages to memory pool + * @pool: Memory pool where pages should be freed + * @nr_pages: Number of pages to free + * @pages: Pointer to array holding the physical addresses of the pages to + * free. + * @dirty: Whether any pages may be dirty in the cache. + * @reclaimed: Whether the pages where reclaimable and thus should bypass + * the pool and go straight to the kernel. + * + * Like kbase_mem_pool_free() but optimized for freeing many pages. + */ +void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, + size_t nr_pages, struct tagged_addr *pages, bool dirty, + bool reclaimed); + +/** + * kbase_mem_pool_size - Get number of free pages in memory pool + * @pool: Memory pool to inspect + * + * Note: the size of the pool may in certain corner cases exceed @max_size! + * + * Return: Number of free pages in the pool + */ +static inline size_t kbase_mem_pool_size(struct kbase_mem_pool *pool) +{ + return READ_ONCE(pool->cur_size); +} + +/** + * kbase_mem_pool_max_size - Get maximum number of free pages in memory pool + * @pool: Memory pool to inspect + * + * Return: Maximum number of free pages in the pool + */ +static inline size_t kbase_mem_pool_max_size(struct kbase_mem_pool *pool) +{ + return pool->max_size; +} + + +/** + * kbase_mem_pool_set_max_size - Set maximum number of free pages in memory pool + * @pool: Memory pool to inspect + * @max_size: Maximum number of free pages the pool can hold + * + * If @max_size is reduced, the pool will be shrunk to adhere to the new limit. + * For details see kbase_mem_pool_shrink(). + */ +void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size); + +/** + * kbase_mem_pool_grow - Grow the pool + * @pool: Memory pool to grow + * @nr_to_grow: Number of pages to add to the pool + * + * Adds @nr_to_grow pages to the pool. Note that this may cause the pool to + * become larger than the maximum size specified. + * + * Returns: 0 on success, -ENOMEM if unable to allocate sufficent pages + */ +int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow); + +/** + * kbase_mem_pool_trim - Grow or shrink the pool to a new size + * @pool: Memory pool to trim + * @new_size: New number of pages in the pool + * + * If @new_size > @cur_size, fill the pool with new pages from the kernel, but + * not above the max_size for the pool. + * If @new_size < @cur_size, shrink the pool by freeing pages to the kernel. + */ +void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size); + +/** + * kbase_mem_pool_mark_dying - Mark that this pool is dying + * @pool: Memory pool + * + * This will cause any ongoing allocation operations (eg growing on page fault) + * to be terminated. + */ +void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool); + +/** + * kbase_mem_alloc_page - Allocate a new page for a device + * @pool: Memory pool to allocate a page from + * + * Most uses should use kbase_mem_pool_alloc to allocate a page. However that + * function can fail in the event the pool is empty. + * + * Return: A new page or NULL if no memory + */ +struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool); + +/** + * kbase_region_tracker_init - Initialize the region tracker data structure + * @kctx: kbase context + * + * Return: 0 if success, negative error code otherwise. + */ +int kbase_region_tracker_init(struct kbase_context *kctx); + +/** + * kbase_region_tracker_init_jit - Initialize the just-in-time memory + * allocation region + * @kctx: Kbase context. + * @jit_va_pages: Size of the JIT region in pages. + * @max_allocations: Maximum number of allocations allowed for the JIT region. + * Valid range is 0..%BASE_JIT_ALLOC_COUNT. + * @trim_level: Trim level for the JIT region. + * Valid range is 0..%BASE_JIT_MAX_TRIM_LEVEL. + * @group_id: The physical group ID from which to allocate JIT memory. + * Valid range is 0..(%MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @phys_pages_limit: Maximum number of physical pages to use to back the JIT + * region. Must not exceed @jit_va_pages. + * + * Return: 0 if success, negative error code otherwise. + */ +int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, + int max_allocations, int trim_level, int group_id, + u64 phys_pages_limit); + +/** + * kbase_region_tracker_init_exec - Initialize the GPU-executable memory region + * @kctx: kbase context + * @exec_va_pages: Size of the JIT region in pages. + * It must not be greater than 4 GB. + * + * Return: 0 if success, negative error code otherwise. + */ +int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages); + +/** + * kbase_region_tracker_term - Terminate the JIT region + * @kctx: kbase context + */ +void kbase_region_tracker_term(struct kbase_context *kctx); + +/** + * kbase_region_tracker_term_rbtree - Free memory for a region tracker + * + * This will free all the regions within the region tracker + * + * @rbtree: Region tracker tree root + */ +void kbase_region_tracker_term_rbtree(struct rb_root *rbtree); + +struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address( + struct kbase_context *kctx, u64 gpu_addr); +struct kbase_va_region *kbase_find_region_enclosing_address( + struct rb_root *rbtree, u64 gpu_addr); + +/** + * @brief Check that a pointer is actually a valid region. + * + * Must be called with context lock held. + */ +struct kbase_va_region *kbase_region_tracker_find_region_base_address( + struct kbase_context *kctx, u64 gpu_addr); +struct kbase_va_region *kbase_find_region_base_address(struct rb_root *rbtree, + u64 gpu_addr); + +struct kbase_va_region *kbase_alloc_free_region(struct rb_root *rbtree, + u64 start_pfn, size_t nr_pages, int zone); +void kbase_free_alloced_region(struct kbase_va_region *reg); +int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, + u64 addr, size_t nr_pages, size_t align); +int kbase_add_va_region_rbtree(struct kbase_device *kbdev, + struct kbase_va_region *reg, u64 addr, size_t nr_pages, + size_t align); + +bool kbase_check_alloc_flags(unsigned long flags); +bool kbase_check_import_flags(unsigned long flags); + +/** + * kbase_check_alloc_sizes - check user space sizes parameters for an + * allocation + * + * @kctx: kbase context + * @flags: The flags passed from user space + * @va_pages: The size of the requested region, in pages. + * @commit_pages: Number of pages to commit initially. + * @extent: Number of pages to grow by on GPU page fault and/or alignment + * (depending on flags) + * + * Makes checks on the size parameters passed in from user space for a memory + * allocation call, with respect to the flags requested. + * + * Return: 0 if sizes are valid for these flags, negative error code otherwise + */ +int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, + u64 va_pages, u64 commit_pages, u64 extent); + +/** + * kbase_update_region_flags - Convert user space flags to kernel region flags + * + * @kctx: kbase context + * @reg: The region to update the flags on + * @flags: The flags passed from user space + * + * The user space flag BASE_MEM_COHERENT_SYSTEM_REQUIRED will be rejected and + * this function will fail if the system does not support system coherency. + * + * Return: 0 if successful, -EINVAL if the flags are not supported + */ +int kbase_update_region_flags(struct kbase_context *kctx, + struct kbase_va_region *reg, unsigned long flags); + +void kbase_gpu_vm_lock(struct kbase_context *kctx); +void kbase_gpu_vm_unlock(struct kbase_context *kctx); + +int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size); + +/** + * @brief Register region and map it on the GPU. + * + * Call kbase_add_va_region() and map the region on the GPU. + */ +int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); + +/** + * @brief Remove the region from the GPU and unregister it. + * + * Must be called with context lock held. + */ +int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg); + +/** + * kbase_mmu_update - Configure an address space on the GPU to the specified + * MMU tables + * + * The caller has the following locking conditions: + * - It must hold kbase_device->mmu_hw_mutex + * - It must hold the hwaccess_lock + * + * @kbdev: Kbase device structure + * @mmut: The set of MMU tables to be configured on the address space + * @as_nr: The address space to be configured + */ +void kbase_mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, + int as_nr); + +/** + * kbase_mmu_disable() - Disable the MMU for a previously active kbase context. + * @kctx: Kbase context + * + * Disable and perform the required cache maintenance to remove the all + * data from provided kbase context from the GPU caches. + * + * The caller has the following locking conditions: + * - It must hold kbase_device->mmu_hw_mutex + * - It must hold the hwaccess_lock + */ +void kbase_mmu_disable(struct kbase_context *kctx); + +/** + * kbase_mmu_disable_as() - Set the MMU to unmapped mode for the specified + * address space. + * @kbdev: Kbase device + * @as_nr: The address space number to set to unmapped. + * + * This function must only be called during reset/power-up and it used to + * ensure the registers are in a known state. + * + * The caller must hold kbdev->mmu_hw_mutex. + */ +void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr); + +void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); + +/** + * kbase_mmu_dump() - Dump the MMU tables to a buffer. + * + * This function allocates a buffer (of @c nr_pages pages) to hold a dump + * of the MMU tables and fills it. If the buffer is too small + * then the return value will be NULL. + * + * The GPU vm lock must be held when calling this function. + * + * The buffer returned should be freed with @ref vfree when it is no longer + * required. + * + * @kctx: The kbase context to dump + * @nr_pages: The number of pages to allocate for the buffer. + * + * Return: The address of the buffer containing the MMU dump or NULL on error + * (including if the @c nr_pages is too small) + */ +void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); + +/** + * kbase_sync_now - Perform cache maintenance on a memory region + * + * @kctx: The kbase context of the region + * @sset: A syncset structure describing the region and direction of the + * synchronisation required + * + * Return: 0 on success or error code + */ +int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset); +void kbase_sync_single(struct kbase_context *kctx, struct tagged_addr cpu_pa, + struct tagged_addr gpu_pa, off_t offset, size_t size, + enum kbase_sync_type sync_fn); + +/* OS specific functions */ +int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr); +int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg); +void kbase_os_mem_map_lock(struct kbase_context *kctx); +void kbase_os_mem_map_unlock(struct kbase_context *kctx); + +/** + * kbasep_os_process_page_usage_update() - Update the memory allocation + * counters for the current process. + * + * OS specific call to updates the current memory allocation counters + * for the current process with the supplied delta. + * + * @kctx: The kbase context + * @pages: The desired delta to apply to the memory usage counters. + */ + +void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages); + +/** + * kbase_process_page_usage_inc() - Add to the memory allocation counters for + * the current process + * + * OS specific call to add to the current memory allocation counters for + * the current process by the supplied amount. + * + * @kctx: The kernel base context used for the allocation. + * @pages: The desired delta to apply to the memory usage counters. + */ + +static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages) +{ + kbasep_os_process_page_usage_update(kctx, pages); +} + +/** + * kbase_process_page_usage_dec() - Subtract from the memory allocation + * counters for the current process. + * + * OS specific call to subtract from the current memory allocation counters + * for the current process by the supplied amount. + * + * @kctx: The kernel base context used for the allocation. + * @pages: The desired delta to apply to the memory usage counters. + */ + +static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages) +{ + kbasep_os_process_page_usage_update(kctx, 0 - pages); +} + +/** + * kbasep_find_enclosing_cpu_mapping_offset() - Find the offset of the CPU + * mapping of a memory allocation containing a given address range + * + * Searches for a CPU mapping of any part of any region that fully encloses the + * CPU virtual address range specified by @uaddr and @size. Returns a failure + * indication if only part of the address range lies within a CPU mapping. + * + * @kctx: The kernel base context used for the allocation. + * @uaddr: Start of the CPU virtual address range. + * @size: Size of the CPU virtual address range (in bytes). + * @offset: The offset from the start of the allocation to the specified CPU + * virtual address. + * + * Return: 0 if offset was obtained successfully. Error code otherwise. + */ +int kbasep_find_enclosing_cpu_mapping_offset( + struct kbase_context *kctx, + unsigned long uaddr, size_t size, u64 *offset); + +/** + * kbasep_find_enclosing_gpu_mapping_start_and_offset() - Find the address of + * the start of GPU virtual memory region which encloses @gpu_addr for the + * @size length in bytes + * + * Searches for the memory region in GPU virtual memory space which contains + * the region defined by the @gpu_addr and @size, where @gpu_addr is the + * beginning and @size the length in bytes of the provided region. If found, + * the location of the start address of the GPU virtual memory region is + * passed in @start pointer and the location of the offset of the region into + * the GPU virtual memory region is passed in @offset pointer. + * + * @kctx: The kernel base context within which the memory is searched. + * @gpu_addr: GPU virtual address for which the region is sought; defines + * the beginning of the provided region. + * @size: The length (in bytes) of the provided region for which the + * GPU virtual memory region is sought. + * @start: Pointer to the location where the address of the start of + * the found GPU virtual memory region is. + * @offset: Pointer to the location where the offset of @gpu_addr into + * the found GPU virtual memory region is. + */ +int kbasep_find_enclosing_gpu_mapping_start_and_offset( + struct kbase_context *kctx, + u64 gpu_addr, size_t size, u64 *start, u64 *offset); + +/** + * kbase_alloc_phy_pages_helper - Allocates physical pages. + * @alloc: allocation object to add pages to + * @nr_pages_requested: number of physical pages to allocate + * + * Allocates \a nr_pages_requested and updates the alloc object. + * + * Return: 0 if all pages have been successfully allocated. Error code otherwise + * + * Note : The caller must not hold vm_lock, as this could cause a deadlock if + * the kernel OoM killer runs. If the caller must allocate pages while holding + * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead. + * + * This function cannot be used from interrupt context + */ +int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, + size_t nr_pages_requested); + +/** + * kbase_alloc_phy_pages_helper_locked - Allocates physical pages. + * @alloc: allocation object to add pages to + * @pool: Memory pool to allocate from + * @nr_pages_requested: number of physical pages to allocate + * @prealloc_sa: Information about the partial allocation if the amount + * of memory requested is not a multiple of 2MB. One + * instance of struct kbase_sub_alloc must be allocated by + * the caller iff CONFIG_MALI_2MB_ALLOC is enabled. + * + * Allocates \a nr_pages_requested and updates the alloc object. This function + * does not allocate new pages from the kernel, and therefore will never trigger + * the OoM killer. Therefore, it can be run while the vm_lock is held. + * + * As new pages can not be allocated, the caller must ensure there are + * sufficient pages in the pool. Usage of this function should look like : + * + * kbase_gpu_vm_lock(kctx); + * kbase_mem_pool_lock(pool) + * while (kbase_mem_pool_size(pool) < pages_required) { + * kbase_mem_pool_unlock(pool) + * kbase_gpu_vm_unlock(kctx); + * kbase_mem_pool_grow(pool) + * kbase_gpu_vm_lock(kctx); + * kbase_mem_pool_lock(pool) + * } + * kbase_alloc_phy_pages_helper_locked(pool) + * kbase_mem_pool_unlock(pool) + * Perform other processing that requires vm_lock... + * kbase_gpu_vm_unlock(kctx); + * + * This ensures that the pool can be grown to the required size and that the + * allocation can complete without another thread using the newly grown pages. + * + * If CONFIG_MALI_2MB_ALLOC is defined and the allocation is >= 2MB, then + * @pool must be alloc->imported.native.kctx->lp_mem_pool. Otherwise it must be + * alloc->imported.native.kctx->mem_pool. + * @prealloc_sa is used to manage the non-2MB sub-allocation. It has to be + * pre-allocated because we must not sleep (due to the usage of kmalloc()) + * whilst holding pool->pool_lock. + * @prealloc_sa shall be set to NULL if it has been consumed by this function + * to indicate that the caller must not free it. + * + * Return: Pointer to array of allocated pages. NULL on failure. + * + * Note : Caller must hold pool->pool_lock + */ +struct tagged_addr *kbase_alloc_phy_pages_helper_locked( + struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, + size_t nr_pages_requested, + struct kbase_sub_alloc **prealloc_sa); + +/** + * kbase_free_phy_pages_helper() - Free physical pages. + * + * Frees \a nr_pages and updates the alloc object. + * + * @alloc: allocation object to free pages from + * @nr_pages_to_free: number of physical pages to free + * + * Return: 0 on success, otherwise a negative error code + */ +int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free); + +/** + * kbase_free_phy_pages_helper_locked - Free pages allocated with + * kbase_alloc_phy_pages_helper_locked() + * @alloc: Allocation object to free pages from + * @pool: Memory pool to return freed pages to + * @pages: Pages allocated by kbase_alloc_phy_pages_helper_locked() + * @nr_pages_to_free: Number of physical pages to free + * + * This function atomically frees pages allocated with + * kbase_alloc_phy_pages_helper_locked(). @pages is the pointer to the page + * array that is returned by that function. @pool must be the pool that the + * pages were originally allocated from. + * + * If the mem_pool has been unlocked since the allocation then + * kbase_free_phy_pages_helper() should be used instead. + */ +void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, + struct kbase_mem_pool *pool, struct tagged_addr *pages, + size_t nr_pages_to_free); + +static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) +{ + SetPagePrivate(p); + if (sizeof(dma_addr_t) > sizeof(p->private)) { + /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the + * private field stays the same. So we have to be clever and + * use the fact that we only store DMA addresses of whole pages, + * so the low bits should be zero */ + KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1))); + set_page_private(p, dma_addr >> PAGE_SHIFT); + } else { + set_page_private(p, dma_addr); + } +} + +static inline dma_addr_t kbase_dma_addr(struct page *p) +{ + if (sizeof(dma_addr_t) > sizeof(p->private)) + return ((dma_addr_t)page_private(p)) << PAGE_SHIFT; + + return (dma_addr_t)page_private(p); +} + +static inline void kbase_clear_dma_addr(struct page *p) +{ + ClearPagePrivate(p); +} + +/** + * kbase_flush_mmu_wqs() - Flush MMU workqueues. + * @kbdev: Device pointer. + * + * This function will cause any outstanding page or bus faults to be processed. + * It should be called prior to powering off the GPU. + */ +void kbase_flush_mmu_wqs(struct kbase_device *kbdev); + +/** + * kbase_sync_single_for_device - update physical memory and give GPU ownership + * @kbdev: Device pointer + * @handle: DMA address of region + * @size: Size of region to sync + * @dir: DMA data direction + */ + +void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir); + +/** + * kbase_sync_single_for_cpu - update physical memory and give CPU ownership + * @kbdev: Device pointer + * @handle: DMA address of region + * @size: Size of region to sync + * @dir: DMA data direction + */ + +void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir); + +#ifdef CONFIG_DEBUG_FS +/** + * kbase_jit_debugfs_init - Add per context debugfs entry for JIT. + * @kctx: kbase context + */ +void kbase_jit_debugfs_init(struct kbase_context *kctx); +#endif /* CONFIG_DEBUG_FS */ + +/** + * kbase_jit_init - Initialize the JIT memory pool management + * @kctx: kbase context + * + * Returns zero on success or negative error number on failure. + */ +int kbase_jit_init(struct kbase_context *kctx); + +/** + * kbase_jit_allocate - Allocate JIT memory + * @kctx: kbase context + * @info: JIT allocation information + * @ignore_pressure_limit: Whether the JIT memory pressure limit is ignored + * + * Return: JIT allocation on success or NULL on failure. + */ +struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, + const struct base_jit_alloc_info *info, + bool ignore_pressure_limit); + +/** + * kbase_jit_free - Free a JIT allocation + * @kctx: kbase context + * @reg: JIT allocation + * + * Frees a JIT allocation and places it into the free pool for later reuse. + */ +void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg); + +/** + * kbase_jit_backing_lost - Inform JIT that an allocation has lost backing + * @reg: JIT allocation + */ +void kbase_jit_backing_lost(struct kbase_va_region *reg); + +/** + * kbase_jit_evict - Evict a JIT allocation from the pool + * @kctx: kbase context + * + * Evict the least recently used JIT allocation from the pool. This can be + * required if normal VA allocations are failing due to VA exhaustion. + * + * Return: True if a JIT allocation was freed, false otherwise. + */ +bool kbase_jit_evict(struct kbase_context *kctx); + +/** + * kbase_jit_term - Terminate the JIT memory pool management + * @kctx: kbase context + */ +void kbase_jit_term(struct kbase_context *kctx); + +#if MALI_JIT_PRESSURE_LIMIT_BASE +/** + * kbase_trace_jit_report_gpu_mem_trace_enabled - variant of + * kbase_trace_jit_report_gpu_mem() that should only be called once the + * corresponding tracepoint is verified to be enabled + * @kctx: kbase context + * @reg: Just-in-time memory region to trace + * @flags: combination of values from enum kbase_jit_report_flags + */ +void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, + struct kbase_va_region *reg, unsigned int flags); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +/** + * kbase_trace_jit_report_gpu_mem - Trace information about the GPU memory used + * to make a JIT report + * @kctx: kbase context + * @reg: Just-in-time memory region to trace + * @flags: combination of values from enum kbase_jit_report_flags + * + * Information is traced using the trace_mali_jit_report_gpu_mem() tracepoint. + * + * In case that tracepoint is not enabled, this function should have the same + * low overheads as a tracepoint itself (i.e. use of 'jump labels' to avoid + * conditional branches) + * + * This can take the reg_lock on @kctx, do not use in places where this lock is + * already held. + * + * Note: this has to be a macro because at this stage the tracepoints have not + * been included. Also gives no opportunity for the compiler to mess up + * inlining it. + */ +#if MALI_JIT_PRESSURE_LIMIT_BASE +#define kbase_trace_jit_report_gpu_mem(kctx, reg, flags) \ + do { \ + if (trace_mali_jit_report_gpu_mem_enabled()) \ + kbase_trace_jit_report_gpu_mem_trace_enabled( \ + (kctx), (reg), (flags)); \ + } while (0) +#else +#define kbase_trace_jit_report_gpu_mem(kctx, reg, flags) \ + CSTD_NOP(kctx, reg, flags) +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +#if MALI_JIT_PRESSURE_LIMIT_BASE +/** + * kbase_jit_report_update_pressure - safely update the JIT physical page + * pressure and JIT region's estimate of used_pages + * @kctx: kbase context, to update the current physical pressure + * @reg: Just-in-time memory region to update with @new_used_pages + * @new_used_pages: new value of number of pages used in the JIT region + * @flags: combination of values from enum kbase_jit_report_flags + * + * Takes care of: + * - correctly updating the pressure given the current reg->used_pages and + * new_used_pages + * - then updating the %kbase_va_region used_pages member + * + * Precondition: + * - new_used_pages <= reg->nr_pages + */ +void kbase_jit_report_update_pressure(struct kbase_context *kctx, + struct kbase_va_region *reg, u64 new_used_pages, + unsigned int flags); + +/** + * jit_trim_necessary_pages() - calculate and trim the least pages possible to + * satisfy a new JIT allocation + * + * @kctx: Pointer to the kbase context + * @needed_pages: Number of JIT physical pages by which trimming is requested. + * The actual number of pages trimmed could differ. + * + * Before allocating a new just-in-time memory region or reusing a previous + * one, ensure that the total JIT physical page usage also will not exceed the + * pressure limit. + * + * If there are no reported-on allocations, then we already guarantee this will + * be the case - because our current pressure then only comes from the va_pages + * of each JIT region, hence JIT physical page usage is guaranteed to be + * bounded by this. + * + * However as soon as JIT allocations become "reported on", the pressure is + * lowered to allow new JIT regions to be allocated. It is after such a point + * that the total JIT physical page usage could (either now or in the future on + * a grow-on-GPU-page-fault) exceed the pressure limit, but only on newly + * allocated JIT regions. Hence, trim any "reported on" regions. + * + * Any pages freed will go into the pool and be allocated from there in + * kbase_mem_alloc(). + */ +void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, + size_t needed_pages); + +/* + * Same as kbase_jit_request_phys_increase(), except that Caller is supposed + * to take jit_evict_lock also on @kctx before calling this function. + */ +static inline void +kbase_jit_request_phys_increase_locked(struct kbase_context *kctx, + size_t needed_pages) +{ +#if !MALI_USE_CSF + lockdep_assert_held(&kctx->jctx.lock); +#endif /* !MALI_USE_CSF */ + lockdep_assert_held(&kctx->reg_lock); + lockdep_assert_held(&kctx->jit_evict_lock); + + kctx->jit_phys_pages_to_be_allocated += needed_pages; + + kbase_jit_trim_necessary_pages(kctx, + kctx->jit_phys_pages_to_be_allocated); +} + +/** + * kbase_jit_request_phys_increase() - Increment the backing pages count and do + * the required trimming before allocating pages for a JIT allocation. + * + * @kctx: Pointer to the kbase context + * @needed_pages: Number of pages to be allocated for the JIT allocation. + * + * This function needs to be called before allocating backing pages for a + * just-in-time memory region. The backing pages are currently allocated when, + * + * - A new JIT region is created. + * - An old JIT region is reused from the cached pool. + * - GPU page fault occurs for the active JIT region. + * - Backing is grown for the JIT region through the commit ioctl. + * + * This function would ensure that the total JIT physical page usage does not + * exceed the pressure limit even when the backing pages get allocated + * simultaneously for multiple JIT allocations from different threads. + * + * There should be a matching call to kbase_jit_done_phys_increase(), after + * the pages have been allocated and accounted against the active JIT + * allocation. + * + * Caller is supposed to take reg_lock on @kctx before calling this function. + */ +static inline void kbase_jit_request_phys_increase(struct kbase_context *kctx, + size_t needed_pages) +{ +#if !MALI_USE_CSF + lockdep_assert_held(&kctx->jctx.lock); +#endif /* !MALI_USE_CSF */ + lockdep_assert_held(&kctx->reg_lock); + + mutex_lock(&kctx->jit_evict_lock); + kbase_jit_request_phys_increase_locked(kctx, needed_pages); + mutex_unlock(&kctx->jit_evict_lock); +} + +/** + * kbase_jit_done_phys_increase() - Decrement the backing pages count after the + * allocation of pages for a JIT allocation. + * + * @kctx: Pointer to the kbase context + * @needed_pages: Number of pages that were allocated for the JIT allocation. + * + * This function should be called after backing pages have been allocated and + * accounted against the active JIT allocation. + * The call should be made when the following have been satisfied: + * when the allocation is on the jit_active_head. + * when additional needed_pages have been allocated. + * kctx->reg_lock was held during the above and has not yet been unlocked. + * Failure to call this function before unlocking the kctx->reg_lock when + * either the above have changed may result in over-accounting the memory. + * This ensures kbase_jit_trim_necessary_pages() gets a consistent count of + * the memory. + * + * A matching call to kbase_jit_request_phys_increase() should have been made, + * before the allocation of backing pages. + * + * Caller is supposed to take reg_lock on @kctx before calling this function. + */ +static inline void kbase_jit_done_phys_increase(struct kbase_context *kctx, + size_t needed_pages) +{ + lockdep_assert_held(&kctx->reg_lock); + + WARN_ON(kctx->jit_phys_pages_to_be_allocated < needed_pages); + + kctx->jit_phys_pages_to_be_allocated -= needed_pages; +} +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +/** + * kbase_has_exec_va_zone - EXEC_VA zone predicate + * + * Determine whether an EXEC_VA zone has been created for the GPU address space + * of the given kbase context. + * + * @kctx: kbase context + * + * Return: True if the kbase context has an EXEC_VA zone. + */ +bool kbase_has_exec_va_zone(struct kbase_context *kctx); + +/** + * kbase_map_external_resource - Map an external resource to the GPU. + * @kctx: kbase context. + * @reg: The region to map. + * @locked_mm: The mm_struct which has been locked for this operation. + * + * Return: The physical allocation which backs the region on success or NULL + * on failure. + */ +struct kbase_mem_phy_alloc *kbase_map_external_resource( + struct kbase_context *kctx, struct kbase_va_region *reg, + struct mm_struct *locked_mm); + +/** + * kbase_unmap_external_resource - Unmap an external resource from the GPU. + * @kctx: kbase context. + * @reg: The region to unmap or NULL if it has already been released. + * @alloc: The physical allocation being unmapped. + */ +void kbase_unmap_external_resource(struct kbase_context *kctx, + struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); + + +/** + * kbase_jd_user_buf_pin_pages - Pin the pages of a user buffer. + * @kctx: kbase context. + * @reg: The region associated with the imported user buffer. + * + * To successfully pin the pages for a user buffer the current mm_struct must + * be the same as the mm_struct of the user buffer. After successfully pinning + * the pages further calls to this function succeed without doing work. + * + * Return: zero on success or negative number on failure. + */ +int kbase_jd_user_buf_pin_pages(struct kbase_context *kctx, + struct kbase_va_region *reg); + +/** + * kbase_sticky_resource_init - Initialize sticky resource management. + * @kctx: kbase context + * + * Returns zero on success or negative error number on failure. + */ +int kbase_sticky_resource_init(struct kbase_context *kctx); + +/** + * kbase_sticky_resource_acquire - Acquire a reference on a sticky resource. + * @kctx: kbase context. + * @gpu_addr: The GPU address of the external resource. + * + * Return: The metadata object which represents the binding between the + * external resource and the kbase context on success or NULL on failure. + */ +struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( + struct kbase_context *kctx, u64 gpu_addr); + +/** + * kbase_sticky_resource_release - Release a reference on a sticky resource. + * @kctx: kbase context. + * @meta: Binding metadata. + * @gpu_addr: GPU address of the external resource. + * + * If meta is NULL then gpu_addr will be used to scan the metadata list and + * find the matching metadata (if any), otherwise the provided meta will be + * used and gpu_addr will be ignored. + * + * Return: True if the release found the metadata and the reference was dropped. + */ +bool kbase_sticky_resource_release(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); + +/** + * kbase_sticky_resource_release_force - Release a sticky resource. + * @kctx: kbase context. + * @meta: Binding metadata. + * @gpu_addr: GPU address of the external resource. + * + * If meta is NULL then gpu_addr will be used to scan the metadata list and + * find the matching metadata (if any), otherwise the provided meta will be + * used and gpu_addr will be ignored. + * + * Return: True if the release found the metadata and the resource was + * released. + */ +bool kbase_sticky_resource_release_force(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); + +/** + * kbase_sticky_resource_term - Terminate sticky resource management. + * @kctx: kbase context + */ +void kbase_sticky_resource_term(struct kbase_context *kctx); + +/** + * kbase_mem_pool_lock - Lock a memory pool + * @pool: Memory pool to lock + */ +static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool) +{ + spin_lock(&pool->pool_lock); +} + +/** + * kbase_mem_pool_lock - Release a memory pool + * @pool: Memory pool to lock + */ +static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) +{ + spin_unlock(&pool->pool_lock); +} + +/** + * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable. + * @alloc: The physical allocation + */ +void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc); + +#if MALI_USE_CSF +/** + * kbase_link_event_mem_page - Add the new event memory region to the per + * context list of event pages. + * @kctx: Pointer to kbase context + * @reg: Pointer to the region allocated for event memory. + * + * The region being linked shouldn't have been marked as free and should + * have KBASE_REG_CSF_EVENT flag set for it. + */ +static inline void kbase_link_event_mem_page(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + lockdep_assert_held(&kctx->reg_lock); + + WARN_ON(reg->flags & KBASE_REG_FREE); + WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT)); + + list_add(®->link, &kctx->csf.event_pages_head); +} + +/** + * kbase_unlink_event_mem_page - Remove the event memory region from the per + * context list of event pages. + * @kctx: Pointer to kbase context + * @reg: Pointer to the region allocated for event memory. + * + * The region being un-linked shouldn't have been marked as free and should + * have KBASE_REG_CSF_EVENT flag set for it. + */ +static inline void kbase_unlink_event_mem_page(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + lockdep_assert_held(&kctx->reg_lock); + + WARN_ON(reg->flags & KBASE_REG_FREE); + WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT)); + + list_del(®->link); +} + +/** + * kbase_mcu_shared_interface_region_tracker_init - Initialize the rb tree to + * manage the shared interface segment of MCU firmware address space. + * @kbdev: Pointer to the kbase device + * + * Returns zero on success or negative error number on failure. + */ +int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev); + +/** + * kbase_mcu_shared_interface_region_tracker_term - Teardown the rb tree + * managing the shared interface segment of MCU firmware address space. + * @kbdev: Pointer to the kbase device + */ +void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev); +#endif + +/** + * kbase_mem_umm_map - Map dma-buf + * @kctx: Pointer to the kbase context + * @reg: Pointer to the region of the imported dma-buf to map + * + * Map a dma-buf on the GPU. The mappings are reference counted. + * + * Returns 0 on success, or a negative error code. + */ +int kbase_mem_umm_map(struct kbase_context *kctx, + struct kbase_va_region *reg); + +/** + * kbase_mem_umm_unmap - Unmap dma-buf + * @kctx: Pointer to the kbase context + * @reg: Pointer to the region of the imported dma-buf to unmap + * @alloc: Pointer to the alloc to release + * + * Unmap a dma-buf from the GPU. The mappings are reference counted. + * + * @reg must be the original region with GPU mapping of @alloc; or NULL. If + * @reg is NULL, or doesn't match @alloc, the GPU page table entries matching + * @reg will not be updated. + * + * @alloc must be a valid physical allocation of type + * KBASE_MEM_TYPE_IMPORTED_UMM that was previously mapped by + * kbase_mem_umm_map(). The dma-buf attachment referenced by @alloc will + * release it's mapping reference, and if the refcount reaches 0, also be be + * unmapped, regardless of the value of @reg. + */ +void kbase_mem_umm_unmap(struct kbase_context *kctx, + struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); + +/** + * kbase_mem_do_sync_imported - Sync caches for imported memory + * @kctx: Pointer to the kbase context + * @reg: Pointer to the region with imported memory to sync + * @sync_fn: The type of sync operation to perform + * + * Sync CPU caches for supported (currently only dma-buf (UMM)) memory. + * Attempting to sync unsupported imported memory types will result in an error + * code, -EINVAL. + * + * Return: 0 on success, or a negative error code. + */ +int kbase_mem_do_sync_imported(struct kbase_context *kctx, + struct kbase_va_region *reg, enum kbase_sync_type sync_fn); + +/** + * kbase_mem_copy_to_pinned_user_pages - Memcpy from source input page to + * an unaligned address at a given offset from the start of a target page. + * + * @dest_pages: Pointer to the array of pages to which the content is + * to be copied from the provided @src_page. + * @src_page: Pointer to the page which correspond to the source page + * from which the copying will take place. + * @to_copy: Total number of bytes pending to be copied from + * @src_page to @target_page_nr within @dest_pages. + * This will get decremented by number of bytes we + * managed to copy from source page to target pages. + * @nr_pages: Total number of pages present in @dest_pages. + * @target_page_nr: Target page number to which @src_page needs to be + * copied. This will get incremented by one if + * we are successful in copying from source page. + * @offset: Offset in bytes into the target pages from which the + * copying is to be performed. + * + * Return: 0 on success, or a negative error code. + */ +int kbase_mem_copy_to_pinned_user_pages(struct page **dest_pages, + void *src_page, size_t *to_copy, unsigned int nr_pages, + unsigned int *target_page_nr, size_t offset); + +#endif /* _KBASE_MEM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c new file mode 100755 index 000000000000..99b5b852667e --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c @@ -0,0 +1,3425 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_mem_linux.c + * Base kernel memory APIs, Linux implementation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) +#include +#endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#if ((KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) || \ + (KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE)) +/* Enable workaround for ion for kernels prior to v5.0.0 and from v5.3.0 + * onwards. + * + * For kernels prior to v4.12, workaround is needed as ion lacks the cache + * maintenance in begin_cpu_access and end_cpu_access methods. + * + * For kernels prior to v4.17.2, workaround is needed to avoid the potentially + * disruptive warnings which can come if begin_cpu_access and end_cpu_access + * methods are not called in pairs. + * Note that some long term maintenance kernel versions (e.g. 4.9.x, 4.14.x) + * only require this workaround on their earlier releases. However it is still + * safe to use it on such releases, and it simplifies the version check. + * + * For kernels later than v4.17.2, workaround is needed as ion can potentially + * end up calling dma_sync_sg_for_* for a dma-buf importer that hasn't mapped + * the attachment. This would result in a kernel panic as ion populates the + * dma_address when the attachment is mapped and kernel derives the physical + * address for cache maintenance from the dma_address. + * With some multi-threaded tests it has been seen that the same dma-buf memory + * gets imported twice on Mali DDK side and so the problem of sync happening + * with an importer having an unmapped attachment comes at the time of 2nd + * import. The same problem can if there is another importer of dma-buf + * memory. + * + * Workaround can be safely disabled for kernels between v5.0.0 and v5.2.2, + * as all the above stated issues are not there. + * + * dma_sync_sg_for_* calls will be made directly as a workaround using the + * Kbase's attachment to dma-buf that was previously mapped. + */ +#define KBASE_MEM_ION_SYNC_WORKAROUND +#endif + +#define IR_THRESHOLD_STEPS (256u) + +#if MALI_USE_CSF +static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, + struct vm_area_struct *vma); +static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, + struct vm_area_struct *vma); +#endif + +static int kbase_vmap_phy_pages(struct kbase_context *kctx, + struct kbase_va_region *reg, u64 offset_bytes, size_t size, + struct kbase_vmap_struct *map); +static void kbase_vunmap_phy_pages(struct kbase_context *kctx, + struct kbase_vmap_struct *map); + +static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma); + +static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages); + +/* Retrieve the associated region pointer if the GPU address corresponds to + * one of the event memory pages. The enclosing region, if found, shouldn't + * have been marked as free. + */ +static struct kbase_va_region *kbase_find_event_mem_region( + struct kbase_context *kctx, u64 gpu_addr) +{ +#if MALI_USE_CSF + u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; + struct kbase_va_region *reg; + + lockdep_assert_held(&kctx->reg_lock); + + list_for_each_entry(reg, &kctx->csf.event_pages_head, link) { + if ((reg->start_pfn <= gpu_pfn) && + (gpu_pfn < (reg->start_pfn + reg->nr_pages))) { + if (WARN_ON(reg->flags & KBASE_REG_FREE)) + return NULL; + + if (WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT))) + return NULL; + + return reg; + } + } +#endif + + return NULL; +} + +/** + * kbase_phy_alloc_mapping_init - Initialize the kernel side permanent mapping + * of the physical allocation belonging to a + * region + * @kctx: The kernel base context @reg belongs to. + * @reg: The region whose physical allocation is to be mapped + * @vsize: The size of the requested region, in pages + * @size: The size in pages initially committed to the region + * + * Return: 0 on success, otherwise an error code indicating failure + * + * Maps the physical allocation backing a non-free @reg, so it may be + * accessed directly from the kernel. This is only supported for physical + * allocations of type KBASE_MEM_TYPE_NATIVE, and will fail for other types of + * physical allocation. + * + * The mapping is stored directly in the allocation that backs @reg. The + * refcount is not incremented at this point. Instead, use of the mapping should + * be surrounded by kbase_phy_alloc_mapping_get() and + * kbase_phy_alloc_mapping_put() to ensure it does not disappear whilst the + * client is accessing it. + * + * Both cached and uncached regions are allowed, but any sync operations are the + * responsibility of the client using the permanent mapping. + * + * A number of checks are made to ensure that a region that needs a permanent + * mapping can actually be supported: + * - The region must be created as fully backed + * - The region must not be growable + * + * This function will fail if those checks are not satisfied. + * + * On success, the region will also be forced into a certain kind: + * - It will no longer be growable + */ +static int kbase_phy_alloc_mapping_init(struct kbase_context *kctx, + struct kbase_va_region *reg, size_t vsize, size_t size) +{ + size_t size_bytes = (size << PAGE_SHIFT); + struct kbase_vmap_struct *kern_mapping; + int err = 0; + + /* Can only map in regions that are always fully committed + * Don't setup the mapping twice + * Only support KBASE_MEM_TYPE_NATIVE allocations + */ + if (vsize != size || reg->cpu_alloc->permanent_map != NULL || + reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) + return -EINVAL; + + if (size > (KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES - + atomic_read(&kctx->permanent_mapped_pages))) { + dev_warn(kctx->kbdev->dev, "Request for %llu more pages mem needing a permanent mapping would breach limit %lu, currently at %d pages", + (u64)size, + KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES, + atomic_read(&kctx->permanent_mapped_pages)); + return -ENOMEM; + } + + kern_mapping = kzalloc(sizeof(*kern_mapping), GFP_KERNEL); + if (!kern_mapping) + return -ENOMEM; + + err = kbase_vmap_phy_pages(kctx, reg, 0u, size_bytes, kern_mapping); + if (err < 0) + goto vmap_fail; + + /* No support for growing or shrinking mapped regions */ + reg->flags &= ~KBASE_REG_GROWABLE; + + reg->cpu_alloc->permanent_map = kern_mapping; + atomic_add(size, &kctx->permanent_mapped_pages); + + return 0; +vmap_fail: + kfree(kern_mapping); + return err; +} + +void kbase_phy_alloc_mapping_term(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc) +{ + WARN_ON(!alloc->permanent_map); + kbase_vunmap_phy_pages(kctx, alloc->permanent_map); + kfree(alloc->permanent_map); + + alloc->permanent_map = NULL; + + /* Mappings are only done on cpu_alloc, so don't need to worry about + * this being reduced a second time if a separate gpu_alloc is + * freed + */ + WARN_ON(alloc->nents > atomic_read(&kctx->permanent_mapped_pages)); + atomic_sub(alloc->nents, &kctx->permanent_mapped_pages); +} + +void *kbase_phy_alloc_mapping_get(struct kbase_context *kctx, + u64 gpu_addr, + struct kbase_vmap_struct **out_kern_mapping) +{ + struct kbase_va_region *reg; + void *kern_mem_ptr = NULL; + struct kbase_vmap_struct *kern_mapping; + u64 mapping_offset; + + WARN_ON(!kctx); + WARN_ON(!out_kern_mapping); + + kbase_gpu_vm_lock(kctx); + + /* First do a quick lookup in the list of event memory regions */ + reg = kbase_find_event_mem_region(kctx, gpu_addr); + + if (!reg) { + reg = kbase_region_tracker_find_region_enclosing_address( + kctx, gpu_addr); + } + + if (kbase_is_region_invalid_or_free(reg)) + goto out_unlock; + + kern_mapping = reg->cpu_alloc->permanent_map; + if (kern_mapping == NULL) + goto out_unlock; + + mapping_offset = gpu_addr - (reg->start_pfn << PAGE_SHIFT); + + /* Refcount the allocations to prevent them disappearing */ + WARN_ON(reg->cpu_alloc != kern_mapping->cpu_alloc); + WARN_ON(reg->gpu_alloc != kern_mapping->gpu_alloc); + (void)kbase_mem_phy_alloc_get(kern_mapping->cpu_alloc); + (void)kbase_mem_phy_alloc_get(kern_mapping->gpu_alloc); + + kern_mem_ptr = (void *)(uintptr_t)((uintptr_t)kern_mapping->addr + mapping_offset); + *out_kern_mapping = kern_mapping; +out_unlock: + kbase_gpu_vm_unlock(kctx); + return kern_mem_ptr; +} + +void kbase_phy_alloc_mapping_put(struct kbase_context *kctx, + struct kbase_vmap_struct *kern_mapping) +{ + WARN_ON(!kctx); + WARN_ON(!kern_mapping); + + WARN_ON(kctx != kern_mapping->cpu_alloc->imported.native.kctx); + WARN_ON(kern_mapping != kern_mapping->cpu_alloc->permanent_map); + + kbase_mem_phy_alloc_put(kern_mapping->cpu_alloc); + kbase_mem_phy_alloc_put(kern_mapping->gpu_alloc); + + /* kern_mapping and the gpu/cpu phy allocs backing it must not be used + * from now on + */ +} + +struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, + u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, + u64 *gpu_va) +{ + int zone; + struct kbase_va_region *reg; + struct rb_root *rbtree; + struct device *dev; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(flags); + KBASE_DEBUG_ASSERT(gpu_va); + + dev = kctx->kbdev->dev; + dev_dbg(dev, "Allocating %lld va_pages, %lld commit_pages, %lld extent, 0x%llX flags\n", + va_pages, commit_pages, extent, *flags); + +#if MALI_USE_CSF + *gpu_va = 0; /* return 0 on failure */ +#else + if (!(*flags & BASE_MEM_FLAG_MAP_FIXED)) + *gpu_va = 0; /* return 0 on failure */ + else + dev_err(dev, + "Keeping requested GPU VA of 0x%llx\n", + (unsigned long long)*gpu_va); +#endif + + if (!kbase_check_alloc_flags(*flags)) { + dev_warn(dev, + "kbase_mem_alloc called with bad flags (%llx)", + (unsigned long long)*flags); + goto bad_flags; + } + +#ifdef CONFIG_DEBUG_FS + if (unlikely(kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE))) { + /* Mask coherency flags if infinite cache is enabled to prevent + * the skipping of syncs from BASE side. + */ + *flags &= ~(BASE_MEM_COHERENT_SYSTEM_REQUIRED | + BASE_MEM_COHERENT_SYSTEM); + } +#endif + + if ((*flags & BASE_MEM_UNCACHED_GPU) != 0 && + (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0) { + /* Remove COHERENT_SYSTEM_REQUIRED flag if uncached GPU mapping is requested */ + *flags &= ~BASE_MEM_COHERENT_SYSTEM_REQUIRED; + } + if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable"); + goto bad_flags; + } + if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ + *flags &= ~BASE_MEM_COHERENT_SYSTEM; + } + + if (kbase_check_alloc_sizes(kctx, *flags, va_pages, commit_pages, extent)) + goto bad_sizes; + +#ifdef CONFIG_MALI_MEMORY_FULLY_BACKED + /* Ensure that memory is fully physically-backed. */ + if (*flags & BASE_MEM_GROW_ON_GPF) + commit_pages = va_pages; +#endif + + /* find out which VA zone to use */ + if (*flags & BASE_MEM_SAME_VA) { + rbtree = &kctx->reg_rbtree_same; + zone = KBASE_REG_ZONE_SAME_VA; + } else if ((*flags & BASE_MEM_PROT_GPU_EX) && kbase_has_exec_va_zone(kctx)) { + rbtree = &kctx->reg_rbtree_exec; + zone = KBASE_REG_ZONE_EXEC_VA; + } else { + rbtree = &kctx->reg_rbtree_custom; + zone = KBASE_REG_ZONE_CUSTOM_VA; + } + + reg = kbase_alloc_free_region(rbtree, PFN_DOWN(*gpu_va), + va_pages, zone); + + if (!reg) { + dev_err(dev, "Failed to allocate free region"); + goto no_region; + } + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + if (kbase_reg_prepare_native(reg, kctx, + base_mem_group_id_get(*flags)) != 0) { + dev_err(dev, "Failed to prepare region"); + goto prepare_failed; + } + + if (unlikely(reg->cpu_alloc != reg->gpu_alloc)) + *flags |= BASE_MEM_KERNEL_SYNC; + + /* make sure base knows if the memory is actually cached or not */ + if (reg->flags & KBASE_REG_CPU_CACHED) + *flags |= BASE_MEM_CACHED_CPU; + else + *flags &= ~BASE_MEM_CACHED_CPU; + + if (*flags & BASE_MEM_GROW_ON_GPF) { + unsigned int const ir_threshold = atomic_read( + &kctx->kbdev->memdev.ir_threshold); + + reg->threshold_pages = ((va_pages * ir_threshold) + + (IR_THRESHOLD_STEPS / 2)) / IR_THRESHOLD_STEPS; + } else + reg->threshold_pages = 0; + + if (*flags & BASE_MEM_GROW_ON_GPF) { + /* kbase_check_alloc_sizes() already checks extent is valid for + * assigning to reg->extent */ + reg->extent = extent; +#if !MALI_USE_CSF + } else if (*flags & BASE_MEM_TILER_ALIGN_TOP) { + reg->extent = extent; +#endif /* !MALI_USE_CSF */ + } else { + reg->extent = 0; + } + + if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) { + dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)", + (unsigned long long)commit_pages, + (unsigned long long)va_pages); + goto no_mem; + } + reg->initial_commit = commit_pages; + + kbase_gpu_vm_lock(kctx); + + if (reg->flags & KBASE_REG_PERMANENT_KERNEL_MAPPING) { + /* Permanent kernel mappings must happen as soon as + * reg->cpu_alloc->pages is ready. Currently this happens after + * kbase_alloc_phy_pages(). If we move that to setup pages + * earlier, also move this call too + */ + int err = kbase_phy_alloc_mapping_init(kctx, reg, va_pages, + commit_pages); + if (err < 0) { + kbase_gpu_vm_unlock(kctx); + goto no_kern_mapping; + } + } + +#if MALI_USE_CSF + if (reg->flags & KBASE_REG_CSF_EVENT) { + WARN_ON(!(*flags & BASE_MEM_SAME_VA)); + + kbase_link_event_mem_page(kctx, reg); + } +#endif + + /* mmap needed to setup VA? */ + if (*flags & BASE_MEM_SAME_VA) { + unsigned long cookie, cookie_nr; + + /* Bind to a cookie */ + if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) { + dev_err(dev, "No cookies available for allocation!"); + kbase_gpu_vm_unlock(kctx); + goto no_cookie; + } + /* return a cookie */ + cookie_nr = find_first_bit(kctx->cookies, BITS_PER_LONG); + bitmap_clear(kctx->cookies, cookie_nr, 1); + BUG_ON(kctx->pending_regions[cookie_nr]); + kctx->pending_regions[cookie_nr] = reg; + + /* relocate to correct base */ + cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE); + cookie <<= PAGE_SHIFT; + + *gpu_va = (u64) cookie; + } else /* we control the VA */ { + if (kbase_gpu_mmap(kctx, reg, *gpu_va, va_pages, 1) != 0) { + dev_warn(dev, "Failed to map memory on GPU"); + kbase_gpu_vm_unlock(kctx); + goto no_mmap; + } + /* return real GPU VA */ + *gpu_va = reg->start_pfn << PAGE_SHIFT; + } + +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (*flags & BASEP_MEM_PERFORM_JIT_TRIM) { + kbase_jit_done_phys_increase(kctx, commit_pages); + + mutex_lock(&kctx->jit_evict_lock); + WARN_ON(!list_empty(®->jit_node)); + list_add(®->jit_node, &kctx->jit_active_head); + mutex_unlock(&kctx->jit_evict_lock); + } +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + kbase_gpu_vm_unlock(kctx); + return reg; + +no_mmap: +no_cookie: +#if MALI_USE_CSF + if (reg->flags & KBASE_REG_CSF_EVENT) { + kbase_gpu_vm_lock(kctx); + kbase_unlink_event_mem_page(kctx, reg); + kbase_gpu_vm_unlock(kctx); + } +#endif +no_kern_mapping: +no_mem: +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (*flags & BASEP_MEM_PERFORM_JIT_TRIM) { + kbase_gpu_vm_lock(kctx); + kbase_jit_done_phys_increase(kctx, commit_pages); + kbase_gpu_vm_unlock(kctx); + } +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +invalid_flags: +prepare_failed: + kfree(reg); +no_region: +bad_sizes: +bad_flags: + return NULL; +} +KBASE_EXPORT_TEST_API(kbase_mem_alloc); + +int kbase_mem_query(struct kbase_context *kctx, + u64 gpu_addr, u64 query, u64 * const out) +{ + struct kbase_va_region *reg; + int ret = -EINVAL; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(out); + + if (gpu_addr & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, "mem_query: gpu_addr: passed parameter is invalid"); + return -EINVAL; + } + + kbase_gpu_vm_lock(kctx); + + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (kbase_is_region_invalid_or_free(reg)) + goto out_unlock; + + switch (query) { + case KBASE_MEM_QUERY_COMMIT_SIZE: + if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) { + *out = kbase_reg_current_backed_size(reg); + } else { + size_t i; + struct kbase_aliased *aliased; + *out = 0; + aliased = reg->cpu_alloc->imported.alias.aliased; + for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++) + *out += aliased[i].length; + } + break; + case KBASE_MEM_QUERY_VA_SIZE: + *out = reg->nr_pages; + break; + case KBASE_MEM_QUERY_FLAGS: + { + *out = 0; + if (KBASE_REG_CPU_WR & reg->flags) + *out |= BASE_MEM_PROT_CPU_WR; + if (KBASE_REG_CPU_RD & reg->flags) + *out |= BASE_MEM_PROT_CPU_RD; + if (KBASE_REG_CPU_CACHED & reg->flags) + *out |= BASE_MEM_CACHED_CPU; + if (KBASE_REG_GPU_WR & reg->flags) + *out |= BASE_MEM_PROT_GPU_WR; + if (KBASE_REG_GPU_RD & reg->flags) + *out |= BASE_MEM_PROT_GPU_RD; + if (!(KBASE_REG_GPU_NX & reg->flags)) + *out |= BASE_MEM_PROT_GPU_EX; + if (KBASE_REG_SHARE_BOTH & reg->flags) + *out |= BASE_MEM_COHERENT_SYSTEM; + if (KBASE_REG_SHARE_IN & reg->flags) + *out |= BASE_MEM_COHERENT_LOCAL; + if (mali_kbase_supports_mem_grow_on_gpf(kctx->api_version)) { + /* Prior to this version, this was known about by + * user-side but we did not return them. Returning + * it caused certain clients that were not expecting + * it to fail, so we omit it as a special-case for + * compatibility reasons + */ + if (KBASE_REG_PF_GROW & reg->flags) + *out |= BASE_MEM_GROW_ON_GPF; + } + if (mali_kbase_supports_mem_protected(kctx->api_version)) { + /* Prior to this version, this was known about by + * user-side but we did not return them. Returning + * it caused certain clients that were not expecting + * it to fail, so we omit it as a special-case for + * compatibility reasons + */ + if (KBASE_REG_PROTECTED & reg->flags) + *out |= BASE_MEM_PROTECTED; + } +#if !MALI_USE_CSF + if (KBASE_REG_TILER_ALIGN_TOP & reg->flags) + *out |= BASE_MEM_TILER_ALIGN_TOP; +#endif /* !MALI_USE_CSF */ + if (!(KBASE_REG_GPU_CACHED & reg->flags)) + *out |= BASE_MEM_UNCACHED_GPU; +#if MALI_USE_CSF + if (KBASE_REG_CSF_EVENT & reg->flags) + *out |= BASE_MEM_CSF_EVENT; +#endif + if (KBASE_REG_GPU_VA_SAME_4GB_PAGE & reg->flags) + *out |= BASE_MEM_GPU_VA_SAME_4GB_PAGE; + + *out |= base_mem_group_id_set(reg->cpu_alloc->group_id); + + WARN(*out & ~BASE_MEM_FLAGS_QUERYABLE, + "BASE_MEM_FLAGS_QUERYABLE needs updating\n"); + *out &= BASE_MEM_FLAGS_QUERYABLE; + break; + } + default: + *out = 0; + goto out_unlock; + } + + ret = 0; + +out_unlock: + kbase_gpu_vm_unlock(kctx); + return ret; +} + +/** + * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the + * Ephemeral memory eviction list. + * @s: Shrinker + * @sc: Shrinker control + * + * Return: Number of pages which can be freed. + */ +static +unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_context *kctx; + struct kbase_mem_phy_alloc *alloc; + unsigned long pages = 0; + + kctx = container_of(s, struct kbase_context, reclaim); + + mutex_lock(&kctx->jit_evict_lock); + + list_for_each_entry(alloc, &kctx->evict_list, evict_node) + pages += alloc->nents; + + mutex_unlock(&kctx->jit_evict_lock); + return pages; +} + +/** + * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction + * list for pages and try to reclaim them. + * @s: Shrinker + * @sc: Shrinker control + * + * Return: Number of pages freed (can be less then requested) or -1 if the + * shrinker failed to free pages in its pool. + * + * Note: + * This function accesses region structures without taking the region lock, + * this is required as the OOM killer can call the shrinker after the region + * lock has already been held. + * This is safe as we can guarantee that a region on the eviction list will + * not be freed (kbase_mem_free_region removes the allocation from the list + * before destroying it), or modified by other parts of the driver. + * The eviction list itself is guarded by the eviction lock and the MMU updates + * are protected by their own lock. + */ +static +unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_context *kctx; + struct kbase_mem_phy_alloc *alloc; + struct kbase_mem_phy_alloc *tmp; + unsigned long freed = 0; + + kctx = container_of(s, struct kbase_context, reclaim); + mutex_lock(&kctx->jit_evict_lock); + + list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { + int err; + + err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg, + 0, alloc->nents); + if (err != 0) { + /* + * Failed to remove GPU mapping, tell the shrinker + * to stop trying to shrink our slab even though we + * have pages in it. + */ + freed = -1; + goto out_unlock; + } + + /* + * Update alloc->evicted before freeing the backing so the + * helper can determine that it needs to bypass the accounting + * and memory pool. + */ + alloc->evicted = alloc->nents; + + kbase_free_phy_pages_helper(alloc, alloc->evicted); + freed += alloc->evicted; + list_del_init(&alloc->evict_node); + + /* + * Inform the JIT allocator this region has lost backing + * as it might need to free the allocation. + */ + kbase_jit_backing_lost(alloc->reg); + + /* Enough pages have been freed so stop now */ + if (freed > sc->nr_to_scan) + break; + } +out_unlock: + mutex_unlock(&kctx->jit_evict_lock); + + return freed; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s, + struct shrink_control *sc) +{ + if (sc->nr_to_scan == 0) + return kbase_mem_evictable_reclaim_count_objects(s, sc); + + return kbase_mem_evictable_reclaim_scan_objects(s, sc); +} +#endif + +int kbase_mem_evictable_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->evict_list); + mutex_init(&kctx->jit_evict_lock); + + /* Register shrinker */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) + kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink; +#else + kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; + kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; +#endif + kctx->reclaim.seeks = DEFAULT_SEEKS; + /* Kernel versions prior to 3.1 : + * struct shrinker does not define batch */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) + kctx->reclaim.batch = 0; +#endif + register_shrinker(&kctx->reclaim); + return 0; +} + +void kbase_mem_evictable_deinit(struct kbase_context *kctx) +{ + unregister_shrinker(&kctx->reclaim); +} + +/** + * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable. + * @alloc: The physical allocation + */ +void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc) +{ + struct kbase_context *kctx = alloc->imported.native.kctx; + struct kbase_device *kbdev = kctx->kbdev; + int __maybe_unused new_page_count; + + kbase_process_page_usage_dec(kctx, alloc->nents); + new_page_count = atomic_sub_return(alloc->nents, + &kctx->used_pages); + atomic_sub(alloc->nents, &kctx->kbdev->memdev.used_pages); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kbdev, + kctx->id, + (u64)new_page_count); + kbase_trace_gpu_mem_usage_dec(kbdev, kctx, alloc->nents); +} + +/** + * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable. + * @alloc: The physical allocation + */ +static +void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) +{ + struct kbase_context *kctx = alloc->imported.native.kctx; + struct kbase_device *kbdev = kctx->kbdev; + int __maybe_unused new_page_count; + + new_page_count = atomic_add_return(alloc->nents, + &kctx->used_pages); + atomic_add(alloc->nents, &kctx->kbdev->memdev.used_pages); + + /* Increase mm counters so that the allocation is accounted for + * against the process and thus is visible to the OOM killer, + */ + kbase_process_page_usage_inc(kctx, alloc->nents); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kbdev, + kctx->id, + (u64)new_page_count); + kbase_trace_gpu_mem_usage_inc(kbdev, kctx, alloc->nents); +} + +int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) +{ + struct kbase_context *kctx = gpu_alloc->imported.native.kctx; + + lockdep_assert_held(&kctx->reg_lock); + + kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg, + 0, gpu_alloc->nents); + + mutex_lock(&kctx->jit_evict_lock); + /* This allocation can't already be on a list. */ + WARN_ON(!list_empty(&gpu_alloc->evict_node)); + + /* + * Add the allocation to the eviction list, after this point the shrink + * can reclaim it. + */ + list_add(&gpu_alloc->evict_node, &kctx->evict_list); + mutex_unlock(&kctx->jit_evict_lock); + kbase_mem_evictable_mark_reclaim(gpu_alloc); + + gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED; + return 0; +} + +bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) +{ + struct kbase_context *kctx = gpu_alloc->imported.native.kctx; + int err = 0; + + lockdep_assert_held(&kctx->reg_lock); + + mutex_lock(&kctx->jit_evict_lock); + /* + * First remove the allocation from the eviction list as it's no + * longer eligible for eviction. + */ + list_del_init(&gpu_alloc->evict_node); + mutex_unlock(&kctx->jit_evict_lock); + + if (gpu_alloc->evicted == 0) { + /* + * The backing is still present, update the VM stats as it's + * in use again. + */ + kbase_mem_evictable_unmark_reclaim(gpu_alloc); + } else { + /* If the region is still alive ... */ + if (gpu_alloc->reg) { + /* ... allocate replacement backing ... */ + err = kbase_alloc_phy_pages_helper(gpu_alloc, + gpu_alloc->evicted); + + /* + * ... and grow the mapping back to its + * pre-eviction size. + */ + if (!err) + err = kbase_mem_grow_gpu_mapping(kctx, + gpu_alloc->reg, + gpu_alloc->evicted, 0); + + gpu_alloc->evicted = 0; + } + } + + /* If the region is still alive remove the DONT_NEED attribute. */ + if (gpu_alloc->reg) + gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED; + + return (err == 0); +} + +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask) +{ + struct kbase_va_region *reg; + int ret = -EINVAL; + unsigned int real_flags = 0; + unsigned int new_flags = 0; + bool prev_needed, new_needed; + + KBASE_DEBUG_ASSERT(kctx); + + if (!gpu_addr) + return -EINVAL; + + if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) + return -EINVAL; + + /* nuke other bits */ + flags &= mask; + + /* check for only supported flags */ + if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE)) + goto out; + + /* mask covers bits we don't support? */ + if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE)) + goto out; + + /* convert flags */ + if (BASE_MEM_COHERENT_SYSTEM & flags) + real_flags |= KBASE_REG_SHARE_BOTH; + else if (BASE_MEM_COHERENT_LOCAL & flags) + real_flags |= KBASE_REG_SHARE_IN; + + /* now we can lock down the context, and find the region */ + down_write(kbase_mem_get_process_mmap_lock()); + kbase_gpu_vm_lock(kctx); + + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (kbase_is_region_invalid_or_free(reg)) + goto out_unlock; + + /* Is the region being transitioning between not needed and needed? */ + prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED; + new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED; + if (prev_needed != new_needed) { + /* Aliased allocations can't be made ephemeral */ + if (atomic_read(®->cpu_alloc->gpu_mappings) > 1) + goto out_unlock; + + if (new_needed) { + /* Only native allocations can be marked not needed */ + if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) { + ret = -EINVAL; + goto out_unlock; + } + ret = kbase_mem_evictable_make(reg->gpu_alloc); + if (ret) + goto out_unlock; + } else { + kbase_mem_evictable_unmake(reg->gpu_alloc); + } + } + + /* limit to imported memory */ + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM) + goto out_unlock; + + /* shareability flags are ignored for GPU uncached memory */ + if (!(reg->flags & KBASE_REG_GPU_CACHED)) { + ret = 0; + goto out_unlock; + } + + /* no change? */ + if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) { + ret = 0; + goto out_unlock; + } + + new_flags = reg->flags & ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); + new_flags |= real_flags; + + /* Currently supporting only imported memory */ + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM) { + ret = -EINVAL; + goto out_unlock; + } + + if (IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND)) { + /* Future use will use the new flags, existing mapping + * will NOT be updated as memory should not be in use + * by the GPU when updating the flags. + */ + WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count); + ret = 0; + } else if (reg->gpu_alloc->imported.umm.current_mapping_usage_count) { + /* + * When CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND is not enabled the + * dma-buf GPU mapping should always be present, check that + * this is the case and warn and skip the page table update if + * not. + * + * Then update dma-buf GPU mapping with the new flags. + * + * Note: The buffer must not be in use on the GPU when + * changing flags. If the buffer is in active use on + * the GPU, there is a risk that the GPU may trigger a + * shareability fault, as it will see the same + * addresses from buffer with different shareability + * properties. + */ + dev_dbg(kctx->kbdev->dev, + "Updating page tables on mem flag change\n"); + ret = kbase_mmu_update_pages(kctx, reg->start_pfn, + kbase_get_gpu_phy_pages(reg), + kbase_reg_current_backed_size(reg), + new_flags, + reg->gpu_alloc->group_id); + if (ret) + dev_warn(kctx->kbdev->dev, + "Failed to update GPU page tables on flag change: %d\n", + ret); + } else + WARN_ON(!reg->gpu_alloc->imported.umm.current_mapping_usage_count); + + /* If everything is good, then set the new flags on the region. */ + if (!ret) + reg->flags = new_flags; + +out_unlock: + kbase_gpu_vm_unlock(kctx); + up_write(kbase_mem_get_process_mmap_lock()); +out: + return ret; +} + +#define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS) + +int kbase_mem_do_sync_imported(struct kbase_context *kctx, + struct kbase_va_region *reg, enum kbase_sync_type sync_fn) +{ + int ret = -EINVAL; + struct dma_buf *dma_buf; + enum dma_data_direction dir = DMA_BIDIRECTIONAL; + + lockdep_assert_held(&kctx->reg_lock); + + /* We assume that the same physical allocation object is used for both + * GPU and CPU for imported buffers. + */ + WARN_ON(reg->cpu_alloc != reg->gpu_alloc); + + /* Currently only handle dma-bufs */ + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM) + return ret; + /* + * Attempting to sync with CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND + * enabled can expose us to a Linux Kernel issue between v4.6 and + * v4.19. We will not attempt to support cache syncs on dma-bufs that + * are mapped on demand (i.e. not on import), even on pre-4.6, neither + * on 4.20 or newer kernels, because this makes it difficult for + * userspace to know when they can rely on the cache sync. + * Instead, only support syncing when we always map dma-bufs on import, + * or if the particular buffer is mapped right now. + */ + if (IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND) && + !reg->gpu_alloc->imported.umm.current_mapping_usage_count) + return ret; + + dma_buf = reg->gpu_alloc->imported.umm.dma_buf; + + switch (sync_fn) { + case KBASE_SYNC_TO_DEVICE: + dev_dbg(kctx->kbdev->dev, + "Syncing imported buffer at GPU VA %llx to GPU\n", + reg->start_pfn); +#ifdef KBASE_MEM_ION_SYNC_WORKAROUND + if (!WARN_ON(!reg->gpu_alloc->imported.umm.dma_attachment)) { + struct dma_buf_attachment *attachment = reg->gpu_alloc->imported.umm.dma_attachment; + struct sg_table *sgt = reg->gpu_alloc->imported.umm.sgt; + + dma_sync_sg_for_device(attachment->dev, sgt->sgl, + sgt->nents, dir); + ret = 0; + } +#else + /* Though the below version check could be superfluous depending upon the version condition + * used for enabling KBASE_MEM_ION_SYNC_WORKAROUND, we still keep this check here to allow + * ease of modification for non-ION systems or systems where ION has been patched. + */ +#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS) + dma_buf_end_cpu_access(dma_buf, + 0, dma_buf->size, + dir); + ret = 0; +#else + ret = dma_buf_end_cpu_access(dma_buf, + dir); +#endif +#endif /* KBASE_MEM_ION_SYNC_WORKAROUND */ + break; + case KBASE_SYNC_TO_CPU: + dev_dbg(kctx->kbdev->dev, + "Syncing imported buffer at GPU VA %llx to CPU\n", + reg->start_pfn); +#ifdef KBASE_MEM_ION_SYNC_WORKAROUND + if (!WARN_ON(!reg->gpu_alloc->imported.umm.dma_attachment)) { + struct dma_buf_attachment *attachment = reg->gpu_alloc->imported.umm.dma_attachment; + struct sg_table *sgt = reg->gpu_alloc->imported.umm.sgt; + + dma_sync_sg_for_cpu(attachment->dev, sgt->sgl, + sgt->nents, dir); + ret = 0; + } +#else + ret = dma_buf_begin_cpu_access(dma_buf, +#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS) + 0, dma_buf->size, +#endif + dir); +#endif /* KBASE_MEM_ION_SYNC_WORKAROUND */ + break; + }; + + if (unlikely(ret)) + dev_warn(kctx->kbdev->dev, + "Failed to sync mem region %pK at GPU VA %llx: %d\n", + reg, reg->start_pfn, ret); + + return ret; +} + +/** + * kbase_mem_umm_unmap_attachment - Unmap dma-buf attachment + * @kctx: Pointer to kbase context + * @alloc: Pointer to allocation with imported dma-buf memory to unmap + * + * This will unmap a dma-buf. Must be called after the GPU page tables for the + * region have been torn down. + */ +static void kbase_mem_umm_unmap_attachment(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc) +{ + struct tagged_addr *pa = alloc->pages; + + dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, + alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); + alloc->imported.umm.sgt = NULL; + + kbase_remove_dma_buf_usage(kctx, alloc); + + memset(pa, 0xff, sizeof(*pa) * alloc->nents); + alloc->nents = 0; +} + +/* to replace sg_dma_len. */ +#define MALI_SG_DMA_LEN(sg) ((sg)->length) + +/** + * kbase_mem_umm_map_attachment - Prepare attached dma-buf for GPU mapping + * @kctx: Pointer to kbase context + * @reg: Pointer to region with imported dma-buf memory to map + * + * Map the dma-buf and prepare the page array with the tagged Mali physical + * addresses for GPU mapping. + * + * Return: 0 on success, or negative error code + */ +static int kbase_mem_umm_map_attachment(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + struct sg_table *sgt; + struct scatterlist *s; + int i; + struct tagged_addr *pa; + int err; + size_t count = 0; + struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; + + WARN_ON_ONCE(alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM); + WARN_ON_ONCE(alloc->imported.umm.sgt); + + sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, + DMA_BIDIRECTIONAL); + if (IS_ERR_OR_NULL(sgt)) + return -EINVAL; + + /* save for later */ + alloc->imported.umm.sgt = sgt; + + pa = kbase_get_gpu_phy_pages(reg); + + for_each_sg(sgt->sgl, s, sgt->nents, i) { + size_t j, pages = PFN_UP(MALI_SG_DMA_LEN(s)); + + WARN_ONCE(MALI_SG_DMA_LEN(s) & (PAGE_SIZE-1), + "MALI_SG_DMA_LEN(s)=%u is not a multiple of PAGE_SIZE\n", + MALI_SG_DMA_LEN(s)); + + WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE-1), + "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n", + (unsigned long long) sg_dma_address(s)); + + for (j = 0; (j < pages) && (count < reg->nr_pages); j++, count++) + *pa++ = as_tagged(sg_dma_address(s) + + (j << PAGE_SHIFT)); + WARN_ONCE(j < pages, + "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n", + alloc->imported.umm.dma_buf->size); + } + + if (!(reg->flags & KBASE_REG_IMPORT_PAD) && + WARN_ONCE(count < reg->nr_pages, + "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n", + alloc->imported.umm.dma_buf->size)) { + err = -EINVAL; + goto err_unmap_attachment; + } + + /* Update nents as we now have pages to map */ + alloc->nents = count; + kbase_add_dma_buf_usage(kctx, alloc); + + return 0; + +err_unmap_attachment: + kbase_mem_umm_unmap_attachment(kctx, alloc); + + return err; +} + +int kbase_mem_umm_map(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + int err; + struct kbase_mem_phy_alloc *alloc; + unsigned long gwt_mask = ~0; + + lockdep_assert_held(&kctx->reg_lock); + + alloc = reg->gpu_alloc; + + alloc->imported.umm.current_mapping_usage_count++; + if (alloc->imported.umm.current_mapping_usage_count != 1) { + if (IS_ENABLED(CONFIG_MALI_DMA_BUF_LEGACY_COMPAT) || + alloc->imported.umm.need_sync) { + if (!kbase_is_region_invalid_or_free(reg)) { + err = kbase_mem_do_sync_imported(kctx, reg, + KBASE_SYNC_TO_DEVICE); + WARN_ON_ONCE(err); + } + } + return 0; + } + + err = kbase_mem_umm_map_attachment(kctx, reg); + if (err) + goto bad_map_attachment; + +#ifdef CONFIG_MALI_CINSTR_GWT + if (kctx->gwt_enabled) + gwt_mask = ~KBASE_REG_GPU_WR; +#endif + + err = kbase_mmu_insert_pages(kctx->kbdev, + &kctx->mmu, + reg->start_pfn, + kbase_get_gpu_phy_pages(reg), + kbase_reg_current_backed_size(reg), + reg->flags & gwt_mask, + kctx->as_nr, + alloc->group_id); + if (err) + goto bad_insert; + + if (reg->flags & KBASE_REG_IMPORT_PAD && + !WARN_ON(reg->nr_pages < alloc->nents)) { + /* For padded imported dma-buf memory, map the dummy aliasing + * page from the end of the dma-buf pages, to the end of the + * region using a read only mapping. + * + * Assume alloc->nents is the number of actual pages in the + * dma-buf memory. + */ + err = kbase_mmu_insert_single_page(kctx, + reg->start_pfn + alloc->nents, + kctx->aliasing_sink_page, + reg->nr_pages - alloc->nents, + (reg->flags | KBASE_REG_GPU_RD) & + ~KBASE_REG_GPU_WR, + KBASE_MEM_GROUP_SINK); + if (err) + goto bad_pad_insert; + } + + return 0; + +bad_pad_insert: + kbase_mmu_teardown_pages(kctx->kbdev, + &kctx->mmu, + reg->start_pfn, + alloc->nents, + kctx->as_nr); +bad_insert: + kbase_mem_umm_unmap_attachment(kctx, alloc); +bad_map_attachment: + alloc->imported.umm.current_mapping_usage_count--; + + return err; +} + +void kbase_mem_umm_unmap(struct kbase_context *kctx, + struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc) +{ + alloc->imported.umm.current_mapping_usage_count--; + if (alloc->imported.umm.current_mapping_usage_count) { + if (IS_ENABLED(CONFIG_MALI_DMA_BUF_LEGACY_COMPAT) || + alloc->imported.umm.need_sync) { + if (!kbase_is_region_invalid_or_free(reg)) { + int err = kbase_mem_do_sync_imported(kctx, reg, + KBASE_SYNC_TO_CPU); + WARN_ON_ONCE(err); + } + } + return; + } + + if (!kbase_is_region_invalid_or_free(reg) && reg->gpu_alloc == alloc) { + int err; + + err = kbase_mmu_teardown_pages(kctx->kbdev, + &kctx->mmu, + reg->start_pfn, + reg->nr_pages, + kctx->as_nr); + WARN_ON(err); + } + + kbase_mem_umm_unmap_attachment(kctx, alloc); +} + +static int get_umm_memory_group_id(struct kbase_context *kctx, + struct dma_buf *dma_buf) +{ + int group_id = BASE_MEM_GROUP_DEFAULT; + + if (kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_id) { + struct memory_group_manager_import_data mgm_import_data; + + mgm_import_data.type = + MEMORY_GROUP_MANAGER_IMPORT_TYPE_DMA_BUF; + mgm_import_data.u.dma_buf = dma_buf; + + group_id = kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_id( + kctx->kbdev->mgm_dev, &mgm_import_data); + } + + return group_id; +} + +/** + * kbase_mem_from_umm - Import dma-buf memory into kctx + * @kctx: Pointer to kbase context to import memory into + * @fd: File descriptor of dma-buf to import + * @va_pages: Pointer where virtual size of the region will be output + * @flags: Pointer to memory flags + * @padding: Number of read only padding pages to be inserted at the end of the + * GPU mapping of the dma-buf + * + * Return: Pointer to new kbase_va_region object of the imported dma-buf, or + * NULL on error. + * + * This function imports a dma-buf into kctx, and created a kbase_va_region + * object that wraps the dma-buf. + */ +static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, + int fd, u64 *va_pages, u64 *flags, u32 padding) +{ + struct kbase_va_region *reg; + struct dma_buf *dma_buf; + struct dma_buf_attachment *dma_attachment; + bool shared_zone = false; + bool need_sync = false; + int group_id; + + /* 64-bit address range is the max */ + if (*va_pages > (U64_MAX / PAGE_SIZE)) + return NULL; + + dma_buf = dma_buf_get(fd); + if (IS_ERR_OR_NULL(dma_buf)) + return NULL; + + dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev); + if (IS_ERR_OR_NULL(dma_attachment)) { + dma_buf_put(dma_buf); + return NULL; + } + + *va_pages = (PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT) + padding; + if (!*va_pages) { + dma_buf_detach(dma_buf, dma_attachment); + dma_buf_put(dma_buf); + return NULL; + } + + /* ignore SAME_VA */ + *flags &= ~BASE_MEM_SAME_VA; + + /* + * Force CPU cached flag. + * + * We can't query the dma-buf exporter to get details about the CPU + * cache attributes of CPU mappings, so we have to assume that the + * buffer may be cached, and call into the exporter for cache + * maintenance, and rely on the exporter to do the right thing when + * handling our calls. + */ + *flags |= BASE_MEM_CACHED_CPU; + + if (*flags & BASE_MEM_IMPORT_SHARED) + shared_zone = true; + + if (*flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) + need_sync = true; + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* + * 64-bit tasks require us to reserve VA on the CPU that we use + * on the GPU. + */ + shared_zone = true; + } +#endif + + if (shared_zone) { + *flags |= BASE_MEM_NEED_MMAP; + reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, + 0, *va_pages, KBASE_REG_ZONE_SAME_VA); + } else { + reg = kbase_alloc_free_region(&kctx->reg_rbtree_custom, + 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); + } + + if (!reg) { + dma_buf_detach(dma_buf, dma_attachment); + dma_buf_put(dma_buf); + return NULL; + } + + group_id = get_umm_memory_group_id(kctx, dma_buf); + + reg->gpu_alloc = kbase_alloc_create(kctx, *va_pages, + KBASE_MEM_TYPE_IMPORTED_UMM, group_id); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto error_out; + + /* No pages to map yet */ + reg->gpu_alloc->nents = 0; + + reg->flags &= ~KBASE_REG_FREE; + reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */ + reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */ + + if (*flags & BASE_MEM_PROTECTED) + reg->flags |= KBASE_REG_PROTECTED; + + if (padding) + reg->flags |= KBASE_REG_IMPORT_PAD; + + reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM; + reg->gpu_alloc->imported.umm.sgt = NULL; + reg->gpu_alloc->imported.umm.dma_buf = dma_buf; + reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment; + reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0; + reg->gpu_alloc->imported.umm.need_sync = need_sync; + reg->gpu_alloc->imported.umm.kctx = kctx; + reg->extent = 0; + + if (!IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND)) { + int err; + + reg->gpu_alloc->imported.umm.current_mapping_usage_count = 1; + + err = kbase_mem_umm_map_attachment(kctx, reg); + if (err) { + dev_warn(kctx->kbdev->dev, + "Failed to map dma-buf %pK on GPU: %d\n", + dma_buf, err); + goto error_out; + } + + *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; + } + + return reg; + +error_out: + kbase_mem_phy_alloc_put(reg->gpu_alloc); + kbase_mem_phy_alloc_put(reg->cpu_alloc); +no_alloc: + kfree(reg); + + return NULL; +} + +u32 kbase_get_cache_line_alignment(struct kbase_device *kbdev) +{ + u32 cpu_cache_line_size = cache_line_size(); + u32 gpu_cache_line_size = + (1UL << kbdev->gpu_props.props.l2_props.log2_line_size); + + return ((cpu_cache_line_size > gpu_cache_line_size) ? + cpu_cache_line_size : + gpu_cache_line_size); +} + +static struct kbase_va_region *kbase_mem_from_user_buffer( + struct kbase_context *kctx, unsigned long address, + unsigned long size, u64 *va_pages, u64 *flags) +{ + long i; + struct kbase_va_region *reg; + struct rb_root *rbtree; + long faulted_pages; + int zone = KBASE_REG_ZONE_CUSTOM_VA; + bool shared_zone = false; + u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx->kbdev); + struct kbase_alloc_import_user_buf *user_buf; + struct page **pages = NULL; + + /* Flag supported only for dma-buf imported memory */ + if (*flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) + return NULL; + + if ((address & (cache_line_alignment - 1)) != 0 || + (size & (cache_line_alignment - 1)) != 0) { + if (*flags & BASE_MEM_UNCACHED_GPU) { + dev_warn(kctx->kbdev->dev, + "User buffer is not cache line aligned and marked as GPU uncached\n"); + goto bad_size; + } + + /* Coherency must be enabled to handle partial cache lines */ + if (*flags & (BASE_MEM_COHERENT_SYSTEM | + BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { + /* Force coherent system required flag, import will + * then fail if coherency isn't available + */ + *flags |= BASE_MEM_COHERENT_SYSTEM_REQUIRED; + } else { + dev_warn(kctx->kbdev->dev, + "User buffer is not cache line aligned and no coherency enabled\n"); + goto bad_size; + } + } + + *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) - + PFN_DOWN(address); + if (!*va_pages) + goto bad_size; + + if (*va_pages > (UINT64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + /* SAME_VA generally not supported with imported memory (no known use cases) */ + *flags &= ~BASE_MEM_SAME_VA; + + if (*flags & BASE_MEM_IMPORT_SHARED) + shared_zone = true; + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* + * 64-bit tasks require us to reserve VA on the CPU that we use + * on the GPU. + */ + shared_zone = true; + } +#endif + + if (shared_zone) { + *flags |= BASE_MEM_NEED_MMAP; + zone = KBASE_REG_ZONE_SAME_VA; + rbtree = &kctx->reg_rbtree_same; + } else + rbtree = &kctx->reg_rbtree_custom; + + reg = kbase_alloc_free_region(rbtree, 0, *va_pages, zone); + + if (!reg) + goto no_region; + + reg->gpu_alloc = kbase_alloc_create( + kctx, *va_pages, KBASE_MEM_TYPE_IMPORTED_USER_BUF, + BASE_MEM_GROUP_DEFAULT); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc_obj; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + reg->flags &= ~KBASE_REG_FREE; + reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */ + reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */ + + user_buf = ®->gpu_alloc->imported.user_buf; + + user_buf->size = size; + user_buf->address = address; + user_buf->nr_pages = *va_pages; + user_buf->mm = current->mm; +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE + atomic_inc(¤t->mm->mm_count); +#else + mmgrab(current->mm); +#endif + if (reg->gpu_alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) + user_buf->pages = vmalloc(*va_pages * sizeof(struct page *)); + else + user_buf->pages = kmalloc_array(*va_pages, + sizeof(struct page *), GFP_KERNEL); + + if (!user_buf->pages) + goto no_page_array; + + /* If the region is coherent with the CPU then the memory is imported + * and mapped onto the GPU immediately. + * Otherwise get_user_pages is called as a sanity check, but with + * NULL as the pages argument which will fault the pages, but not + * pin them. The memory will then be pinned only around the jobs that + * specify the region as an external resource. + */ + if (reg->flags & KBASE_REG_SHARE_BOTH) { + pages = user_buf->pages; + *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; + } + + down_read(kbase_mem_get_process_mmap_lock()); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + faulted_pages = get_user_pages(current, current->mm, address, *va_pages, +#if KERNEL_VERSION(4, 4, 168) <= LINUX_VERSION_CODE && \ +KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL); +#else + reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + faulted_pages = get_user_pages(address, *va_pages, + reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); +#else + faulted_pages = get_user_pages(address, *va_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL); +#endif + + up_read(kbase_mem_get_process_mmap_lock()); + + if (faulted_pages != *va_pages) + goto fault_mismatch; + + reg->gpu_alloc->nents = 0; + reg->extent = 0; + + if (pages) { + struct device *dev = kctx->kbdev->dev; + unsigned long local_size = user_buf->size; + unsigned long offset = user_buf->address & ~PAGE_MASK; + struct tagged_addr *pa = kbase_get_gpu_phy_pages(reg); + + /* Top bit signifies that this was pinned on import */ + user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT; + + for (i = 0; i < faulted_pages; i++) { + dma_addr_t dma_addr; + unsigned long min; + + min = MIN(PAGE_SIZE - offset, local_size); + dma_addr = dma_map_page(dev, pages[i], + offset, min, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) + goto unwind_dma_map; + + user_buf->dma_addrs[i] = dma_addr; + pa[i] = as_tagged(page_to_phys(pages[i])); + + local_size -= min; + offset = 0; + } + + reg->gpu_alloc->nents = faulted_pages; + } + + return reg; + +unwind_dma_map: + while (i--) { + dma_unmap_page(kctx->kbdev->dev, + user_buf->dma_addrs[i], + PAGE_SIZE, DMA_BIDIRECTIONAL); + } +fault_mismatch: + if (pages) { + for (i = 0; i < faulted_pages; i++) + put_page(pages[i]); + } +no_page_array: +invalid_flags: + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +no_alloc_obj: + kfree(reg); +no_region: +bad_size: + return NULL; + +} + + +u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, + u64 nents, struct base_mem_aliasing_info *ai, + u64 *num_pages) +{ + struct kbase_va_region *reg; + u64 gpu_va; + size_t i; + bool coherent; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(flags); + KBASE_DEBUG_ASSERT(ai); + KBASE_DEBUG_ASSERT(num_pages); + + /* mask to only allowed flags */ + *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | + BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL | + BASE_MEM_PROT_CPU_RD | BASE_MEM_COHERENT_SYSTEM_REQUIRED); + + if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) { + dev_warn(kctx->kbdev->dev, + "kbase_mem_alias called with bad flags (%llx)", + (unsigned long long)*flags); + goto bad_flags; + } + coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 || + (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0; + + if (!stride) + goto bad_stride; + + if (!nents) + goto bad_nents; + + if ((nents * stride) > (U64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + /* calculate the number of pages this alias will cover */ + *num_pages = nents * stride; + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* 64-bit tasks must MMAP anyway, but not expose this address to + * clients */ + *flags |= BASE_MEM_NEED_MMAP; + reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 0, + *num_pages, + KBASE_REG_ZONE_SAME_VA); + } else { +#else + if (1) { +#endif + reg = kbase_alloc_free_region(&kctx->reg_rbtree_custom, + 0, *num_pages, + KBASE_REG_ZONE_CUSTOM_VA); + } + + if (!reg) + goto no_reg; + + /* zero-sized page array, as we don't need one/can support one */ + reg->gpu_alloc = kbase_alloc_create(kctx, 0, KBASE_MEM_TYPE_ALIAS, + BASE_MEM_GROUP_DEFAULT); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc_obj; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + reg->gpu_alloc->imported.alias.nents = nents; + reg->gpu_alloc->imported.alias.stride = stride; + reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents); + if (!reg->gpu_alloc->imported.alias.aliased) + goto no_aliased_array; + + kbase_gpu_vm_lock(kctx); + + /* validate and add src handles */ + for (i = 0; i < nents; i++) { + if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) { + if (ai[i].handle.basep.handle != + BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) + goto bad_handle; /* unsupported magic handle */ + if (!ai[i].length) + goto bad_handle; /* must be > 0 */ + if (ai[i].length > stride) + goto bad_handle; /* can't be larger than the + stride */ + reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; + } else { + struct kbase_va_region *aliasing_reg; + struct kbase_mem_phy_alloc *alloc; + + aliasing_reg = kbase_region_tracker_find_region_base_address( + kctx, + (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); + + /* validate found region */ + if (kbase_is_region_invalid_or_free(aliasing_reg)) + goto bad_handle; /* Not found/already free */ + if (aliasing_reg->flags & KBASE_REG_DONT_NEED) + goto bad_handle; /* Ephemeral region */ + if (!(aliasing_reg->flags & KBASE_REG_GPU_CACHED)) + goto bad_handle; /* GPU uncached memory */ + if (!aliasing_reg->gpu_alloc) + goto bad_handle; /* No alloc */ + if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) + goto bad_handle; /* Not a native alloc */ + if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0)) + goto bad_handle; + /* Non-coherent memory cannot alias + coherent memory, and vice versa.*/ + + /* check size against stride */ + if (!ai[i].length) + goto bad_handle; /* must be > 0 */ + if (ai[i].length > stride) + goto bad_handle; /* can't be larger than the + stride */ + + alloc = aliasing_reg->gpu_alloc; + + /* check against the alloc's size */ + if (ai[i].offset > alloc->nents) + goto bad_handle; /* beyond end */ + if (ai[i].offset + ai[i].length > alloc->nents) + goto bad_handle; /* beyond end */ + + reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc); + reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; + reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset; + } + } + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* Bind to a cookie */ + if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) { + dev_err(kctx->kbdev->dev, "No cookies available for allocation!"); + goto no_cookie; + } + /* return a cookie */ + gpu_va = find_first_bit(kctx->cookies, BITS_PER_LONG); + bitmap_clear(kctx->cookies, gpu_va, 1); + BUG_ON(kctx->pending_regions[gpu_va]); + kctx->pending_regions[gpu_va] = reg; + + /* relocate to correct base */ + gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); + gpu_va <<= PAGE_SHIFT; + } else /* we control the VA */ { +#else + if (1) { +#endif + if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) { + dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU"); + goto no_mmap; + } + /* return real GPU VA */ + gpu_va = reg->start_pfn << PAGE_SHIFT; + } + + reg->flags &= ~KBASE_REG_FREE; + reg->flags &= ~KBASE_REG_GROWABLE; + + kbase_gpu_vm_unlock(kctx); + + return gpu_va; + +#ifdef CONFIG_64BIT +no_cookie: +#endif +no_mmap: +bad_handle: + kbase_gpu_vm_unlock(kctx); +no_aliased_array: +invalid_flags: + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +no_alloc_obj: + kfree(reg); +no_reg: +bad_size: +bad_nents: +bad_stride: +bad_flags: + return 0; +} + +int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, + u64 *flags) +{ + struct kbase_va_region *reg; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(gpu_va); + KBASE_DEBUG_ASSERT(va_pages); + KBASE_DEBUG_ASSERT(flags); + + if ((!kbase_ctx_flag(kctx, KCTX_COMPAT)) && + kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA)) + *flags |= BASE_MEM_SAME_VA; + + if (!kbase_check_import_flags(*flags)) { + dev_warn(kctx->kbdev->dev, + "kbase_mem_import called with bad flags (%llx)", + (unsigned long long)*flags); + goto bad_flags; + } + + if ((*flags & BASE_MEM_UNCACHED_GPU) != 0 && + (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0) { + /* Remove COHERENT_SYSTEM_REQUIRED flag if uncached GPU mapping is requested */ + *flags &= ~BASE_MEM_COHERENT_SYSTEM_REQUIRED; + } + if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + dev_warn(kctx->kbdev->dev, + "kbase_mem_import call required coherent mem when unavailable"); + goto bad_flags; + } + if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ + *flags &= ~BASE_MEM_COHERENT_SYSTEM; + } + + if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) { + dev_warn(kctx->kbdev->dev, + "padding is only supported for UMM"); + goto bad_flags; + } + + switch (type) { + case BASE_MEM_IMPORT_TYPE_UMM: { + int fd; + + if (get_user(fd, (int __user *)phandle)) + reg = NULL; + else + reg = kbase_mem_from_umm(kctx, fd, va_pages, flags, + padding); + } + break; + case BASE_MEM_IMPORT_TYPE_USER_BUFFER: { + struct base_mem_import_user_buffer user_buffer; + void __user *uptr; + + if (copy_from_user(&user_buffer, phandle, + sizeof(user_buffer))) { + reg = NULL; + } else { +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + uptr = compat_ptr(user_buffer.ptr); + else +#endif + uptr = u64_to_user_ptr(user_buffer.ptr); + + reg = kbase_mem_from_user_buffer(kctx, + (unsigned long)uptr, user_buffer.length, + va_pages, flags); + } + break; + } + default: { + reg = NULL; + break; + } + } + + if (!reg) + goto no_reg; + + kbase_gpu_vm_lock(kctx); + + /* mmap needed to setup VA? */ + if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) { + /* Bind to a cookie */ + if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) + goto no_cookie; + /* return a cookie */ + *gpu_va = find_first_bit(kctx->cookies, BITS_PER_LONG); + bitmap_clear(kctx->cookies, *gpu_va, 1); + BUG_ON(kctx->pending_regions[*gpu_va]); + kctx->pending_regions[*gpu_va] = reg; + + /* relocate to correct base */ + *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); + *gpu_va <<= PAGE_SHIFT; + + } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) { + /* we control the VA, mmap now to the GPU */ + if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0) + goto no_gpu_va; + /* return real GPU VA */ + *gpu_va = reg->start_pfn << PAGE_SHIFT; + } else { + /* we control the VA, but nothing to mmap yet */ + if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0) + goto no_gpu_va; + /* return real GPU VA */ + *gpu_va = reg->start_pfn << PAGE_SHIFT; + } + + /* clear out private flags */ + *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1); + + kbase_gpu_vm_unlock(kctx); + + return 0; + +no_gpu_va: +no_cookie: + kbase_gpu_vm_unlock(kctx); + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); + kfree(reg); +no_reg: +bad_flags: + *gpu_va = 0; + *va_pages = 0; + *flags = 0; + return -ENOMEM; +} + +int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages) +{ + struct tagged_addr *phy_pages; + u64 delta = new_pages - old_pages; + int ret = 0; + + lockdep_assert_held(&kctx->reg_lock); + + /* Map the new pages into the GPU */ + phy_pages = kbase_get_gpu_phy_pages(reg); + ret = kbase_mmu_insert_pages(kctx->kbdev, &kctx->mmu, + reg->start_pfn + old_pages, phy_pages + old_pages, delta, + reg->flags, kctx->as_nr, reg->gpu_alloc->group_id); + + return ret; +} + +void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages) +{ + u64 gpu_va_start = reg->start_pfn; + + if (new_pages == old_pages) + /* Nothing to do */ + return; + + unmap_mapping_range(kctx->filp->f_inode->i_mapping, + (gpu_va_start + new_pages)<kbdev, &kctx->mmu, + reg->start_pfn + new_pages, delta, kctx->as_nr); + + return ret; +} + +int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) +{ + u64 old_pages; + u64 delta = 0; + int res = -EINVAL; + struct kbase_va_region *reg; + bool read_locked = false; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(gpu_addr != 0); + + if (gpu_addr & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid"); + return -EINVAL; + } + + down_write(kbase_mem_get_process_mmap_lock()); + kbase_gpu_vm_lock(kctx); + + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (kbase_is_region_invalid_or_free(reg)) + goto out_unlock; + + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) + goto out_unlock; + + if (0 == (reg->flags & KBASE_REG_GROWABLE)) + goto out_unlock; + + if (reg->flags & KBASE_REG_ACTIVE_JIT_ALLOC) + goto out_unlock; + + /* Would overflow the VA region */ + if (new_pages > reg->nr_pages) + goto out_unlock; + + /* can't be mapped more than once on the GPU */ + if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) + goto out_unlock; + /* can't grow regions which are ephemeral */ + if (reg->flags & KBASE_REG_DONT_NEED) + goto out_unlock; + +#ifdef CONFIG_MALI_MEMORY_FULLY_BACKED + /* Reject resizing commit size */ + if (reg->flags & KBASE_REG_PF_GROW) + new_pages = reg->nr_pages; +#endif + + if (new_pages == reg->gpu_alloc->nents) { + /* no change */ + res = 0; + goto out_unlock; + } + + old_pages = kbase_reg_current_backed_size(reg); + if (new_pages > old_pages) { + delta = new_pages - old_pages; + + /* + * No update to the mm so downgrade the writer lock to a read + * lock so other readers aren't blocked after this point. + */ + downgrade_write(kbase_mem_get_process_mmap_lock()); + read_locked = true; + + /* Allocate some more pages */ + if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) { + res = -ENOMEM; + goto out_unlock; + } + if (reg->cpu_alloc != reg->gpu_alloc) { + if (kbase_alloc_phy_pages_helper( + reg->gpu_alloc, delta) != 0) { + res = -ENOMEM; + kbase_free_phy_pages_helper(reg->cpu_alloc, + delta); + goto out_unlock; + } + } + + /* No update required for CPU mappings, that's done on fault. */ + + /* Update GPU mapping. */ + res = kbase_mem_grow_gpu_mapping(kctx, reg, + new_pages, old_pages); + + /* On error free the new pages */ + if (res) { + kbase_free_phy_pages_helper(reg->cpu_alloc, delta); + if (reg->cpu_alloc != reg->gpu_alloc) + kbase_free_phy_pages_helper(reg->gpu_alloc, + delta); + res = -ENOMEM; + goto out_unlock; + } + } else { + res = kbase_mem_shrink(kctx, reg, new_pages); + if (res) + res = -ENOMEM; + } + +out_unlock: + kbase_gpu_vm_unlock(kctx); + if (read_locked) + up_read(kbase_mem_get_process_mmap_lock()); + else + up_write(kbase_mem_get_process_mmap_lock()); + + return res; +} + +int kbase_mem_shrink(struct kbase_context *const kctx, + struct kbase_va_region *const reg, u64 const new_pages) +{ + u64 delta, old_pages; + int err; + + lockdep_assert_held(&kctx->reg_lock); + + if (WARN_ON(!kctx)) + return -EINVAL; + + if (WARN_ON(!reg)) + return -EINVAL; + + old_pages = kbase_reg_current_backed_size(reg); + if (WARN_ON(old_pages < new_pages)) + return -EINVAL; + + delta = old_pages - new_pages; + + /* Update the GPU mapping */ + err = kbase_mem_shrink_gpu_mapping(kctx, reg, + new_pages, old_pages); + if (err >= 0) { + /* Update all CPU mapping(s) */ + kbase_mem_shrink_cpu_mapping(kctx, reg, + new_pages, old_pages); + + kbase_free_phy_pages_helper(reg->cpu_alloc, delta); + if (reg->cpu_alloc != reg->gpu_alloc) + kbase_free_phy_pages_helper(reg->gpu_alloc, delta); + } + + return err; +} + + +static void kbase_cpu_vm_open(struct vm_area_struct *vma) +{ + struct kbase_cpu_mapping *map = vma->vm_private_data; + + KBASE_DEBUG_ASSERT(map); + KBASE_DEBUG_ASSERT(map->count > 0); + /* non-atomic as we're under Linux' mm lock */ + map->count++; +} + +static void kbase_cpu_vm_close(struct vm_area_struct *vma) +{ + struct kbase_cpu_mapping *map = vma->vm_private_data; + + KBASE_DEBUG_ASSERT(map); + KBASE_DEBUG_ASSERT(map->count > 0); + + /* non-atomic as we're under Linux' mm lock */ + if (--map->count) + return; + + KBASE_DEBUG_ASSERT(map->kctx); + KBASE_DEBUG_ASSERT(map->alloc); + + kbase_gpu_vm_lock(map->kctx); + + if (map->free_on_close) { + KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) == + KBASE_REG_ZONE_SAME_VA); + /* Avoid freeing memory on the process death which results in + * GPU Page Fault. Memory will be freed in kbase_destroy_context + */ + if (!(current->flags & PF_EXITING)) + kbase_mem_free_region(map->kctx, map->region); + } + + list_del(&map->mappings_list); + + kbase_va_region_alloc_put(map->kctx, map->region); + kbase_gpu_vm_unlock(map->kctx); + + kbase_mem_phy_alloc_put(map->alloc); + kfree(map); +} + +static struct kbase_aliased *get_aliased_alloc(struct vm_area_struct *vma, + struct kbase_va_region *reg, + pgoff_t *start_off, + size_t nr_pages) +{ + struct kbase_aliased *aliased = + reg->cpu_alloc->imported.alias.aliased; + + if (!reg->cpu_alloc->imported.alias.stride || + reg->nr_pages < (*start_off + nr_pages)) { + return NULL; + } + + while (*start_off >= reg->cpu_alloc->imported.alias.stride) { + aliased++; + *start_off -= reg->cpu_alloc->imported.alias.stride; + } + + if (!aliased->alloc) { + /* sink page not available for dumping map */ + return NULL; + } + + if ((*start_off + nr_pages) > aliased->length) { + /* not fully backed by physical pages */ + return NULL; + } + + return aliased; +} + +#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) +static vm_fault_t kbase_cpu_vm_fault(struct vm_area_struct *vma, + struct vm_fault *vmf) +{ +#else +static vm_fault_t kbase_cpu_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; +#endif + struct kbase_cpu_mapping *map = vma->vm_private_data; + pgoff_t map_start_pgoff; + pgoff_t fault_pgoff; + size_t i; + pgoff_t addr; + size_t nents; + struct tagged_addr *pages; + vm_fault_t ret = VM_FAULT_SIGBUS; + struct memory_group_manager_device *mgm_dev; + + KBASE_DEBUG_ASSERT(map); + KBASE_DEBUG_ASSERT(map->count > 0); + KBASE_DEBUG_ASSERT(map->kctx); + KBASE_DEBUG_ASSERT(map->alloc); + + map_start_pgoff = vma->vm_pgoff - map->region->start_pfn; + + kbase_gpu_vm_lock(map->kctx); + if (unlikely(map->region->cpu_alloc->type == KBASE_MEM_TYPE_ALIAS)) { + struct kbase_aliased *aliased = + get_aliased_alloc(vma, map->region, &map_start_pgoff, 1); + + if (!aliased) + goto exit; + + nents = aliased->length; + pages = aliased->alloc->pages + aliased->offset; + } else { + nents = map->alloc->nents; + pages = map->alloc->pages; + } + + fault_pgoff = map_start_pgoff + (vmf->pgoff - vma->vm_pgoff); + + if (fault_pgoff >= nents) + goto exit; + + /* Fault on access to DONT_NEED regions */ + if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED)) + goto exit; + + /* We are inserting all valid pages from the start of CPU mapping and + * not from the fault location (the mmap handler was previously doing + * the same). + */ + i = map_start_pgoff; + addr = (pgoff_t)(vma->vm_start >> PAGE_SHIFT); + mgm_dev = map->kctx->kbdev->mgm_dev; + while (i < nents && (addr < vma->vm_end >> PAGE_SHIFT)) { + + ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, + map->alloc->group_id, vma, addr << PAGE_SHIFT, + PFN_DOWN(as_phys_addr_t(pages[i])), vma->vm_page_prot); + + if (ret != VM_FAULT_NOPAGE) + goto exit; + + i++; addr++; + } + +exit: + kbase_gpu_vm_unlock(map->kctx); + return ret; +} + +const struct vm_operations_struct kbase_vm_ops = { + .open = kbase_cpu_vm_open, + .close = kbase_cpu_vm_close, + .fault = kbase_cpu_vm_fault +}; + +static int kbase_cpu_mmap(struct kbase_context *kctx, + struct kbase_va_region *reg, + struct vm_area_struct *vma, + void *kaddr, + size_t nr_pages, + unsigned long aligned_offset, + int free_on_close) +{ + struct kbase_cpu_mapping *map; + int err = 0; + + map = kzalloc(sizeof(*map), GFP_KERNEL); + + if (!map) { + WARN_ON(1); + err = -ENOMEM; + goto out; + } + + /* + * VM_DONTCOPY - don't make this mapping available in fork'ed processes + * VM_DONTEXPAND - disable mremap on this region + * VM_IO - disables paging + * VM_DONTDUMP - Don't include in core dumps (3.7 only) + * VM_MIXEDMAP - Support mixing struct page*s and raw pfns. + * This is needed to support using the dedicated and + * the OS based memory backends together. + */ + /* + * This will need updating to propagate coherency flags + * See MIDBASE-1057 + */ + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) + vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; +#else + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; +#endif + vma->vm_ops = &kbase_vm_ops; + vma->vm_private_data = map; + + if (reg->cpu_alloc->type == KBASE_MEM_TYPE_ALIAS && nr_pages) { + pgoff_t rel_pgoff = vma->vm_pgoff - reg->start_pfn + + (aligned_offset >> PAGE_SHIFT); + struct kbase_aliased *aliased = + get_aliased_alloc(vma, reg, &rel_pgoff, nr_pages); + + if (!aliased) { + err = -EINVAL; + kfree(map); + goto out; + } + } + + if (!(reg->flags & KBASE_REG_CPU_CACHED) && + (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) { + /* We can't map vmalloc'd memory uncached. + * Other memory will have been returned from + * kbase_mem_pool which would be + * suitable for mapping uncached. + */ + BUG_ON(kaddr); + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + } + + if (!kaddr) { + vma->vm_flags |= VM_PFNMAP; + } else { + WARN_ON(aligned_offset); + /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */ + vma->vm_flags |= VM_MIXEDMAP; + /* vmalloc remaping is easy... */ + err = remap_vmalloc_range(vma, kaddr, 0); + WARN_ON(err); + } + + if (err) { + kfree(map); + goto out; + } + + map->region = kbase_va_region_alloc_get(kctx, reg); + map->free_on_close = free_on_close; + map->kctx = kctx; + map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + map->count = 1; /* start with one ref */ + + if (reg->flags & KBASE_REG_CPU_CACHED) + map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; + + list_add(&map->mappings_list, &map->alloc->mappings); + + out: + return err; +} + +#ifdef CONFIG_MALI_VECTOR_DUMP +static void kbase_free_unused_jit_allocations(struct kbase_context *kctx) +{ + /* Free all cached/unused JIT allocations as their contents are not + * really needed for the replay. The GPU writes to them would already + * have been captured through the GWT mechanism. + * This considerably reduces the size of mmu-snapshot-file and it also + * helps avoid segmentation fault issue during vector dumping of + * complex contents when the unused JIT allocations are accessed to + * dump their contents (as they appear in the page tables snapshot) + * but they got freed by the shrinker under low memory scenarios + * (which do occur with complex contents). + */ + while (kbase_jit_evict(kctx)) + ; +} +#endif + +static int kbase_mmu_dump_mmap(struct kbase_context *kctx, + struct vm_area_struct *vma, + struct kbase_va_region **const reg, + void **const kmap_addr) +{ + struct kbase_va_region *new_reg; + void *kaddr; + u32 nr_pages; + size_t size; + int err = 0; + + dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n"); + size = (vma->vm_end - vma->vm_start); + nr_pages = size >> PAGE_SHIFT; + +#ifdef CONFIG_MALI_VECTOR_DUMP + kbase_free_unused_jit_allocations(kctx); +#endif + + kaddr = kbase_mmu_dump(kctx, nr_pages); + + if (!kaddr) { + err = -ENOMEM; + goto out; + } + + new_reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 0, nr_pages, + KBASE_REG_ZONE_SAME_VA); + if (!new_reg) { + err = -ENOMEM; + WARN_ON(1); + goto out; + } + + new_reg->cpu_alloc = kbase_alloc_create(kctx, 0, KBASE_MEM_TYPE_RAW, + BASE_MEM_GROUP_DEFAULT); + if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { + err = -ENOMEM; + new_reg->cpu_alloc = NULL; + WARN_ON(1); + goto out_no_alloc; + } + + new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); + + new_reg->flags &= ~KBASE_REG_FREE; + new_reg->flags |= KBASE_REG_CPU_CACHED; + if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { + err = -ENOMEM; + WARN_ON(1); + goto out_va_region; + } + + *kmap_addr = kaddr; + *reg = new_reg; + + dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n"); + return 0; + +out_no_alloc: +out_va_region: + kbase_free_alloced_region(new_reg); +out: + return err; +} + + +void kbase_os_mem_map_lock(struct kbase_context *kctx) +{ + (void)kctx; + down_read(kbase_mem_get_process_mmap_lock()); +} + +void kbase_os_mem_map_unlock(struct kbase_context *kctx) +{ + (void)kctx; + up_read(kbase_mem_get_process_mmap_lock()); +} + +static int kbasep_reg_mmap(struct kbase_context *kctx, + struct vm_area_struct *vma, + struct kbase_va_region **regm, + size_t *nr_pages, size_t *aligned_offset) + +{ + int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); + struct kbase_va_region *reg; + int err = 0; + + *aligned_offset = 0; + + dev_dbg(kctx->kbdev->dev, "in kbasep_reg_mmap\n"); + + /* SAME_VA stuff, fetch the right region */ + reg = kctx->pending_regions[cookie]; + if (!reg) { + err = -ENOMEM; + goto out; + } + + if ((reg->flags & KBASE_REG_GPU_NX) && (reg->nr_pages != *nr_pages)) { + /* incorrect mmap size */ + /* leave the cookie for a potential later + * mapping, or to be reclaimed later when the + * context is freed */ + err = -ENOMEM; + goto out; + } + + if ((vma->vm_flags & VM_READ && !(reg->flags & KBASE_REG_CPU_RD)) || + (vma->vm_flags & VM_WRITE && !(reg->flags & KBASE_REG_CPU_WR))) { + /* VM flags inconsistent with region flags */ + err = -EPERM; + dev_err(kctx->kbdev->dev, "%s:%d inconsistent VM flags\n", + __FILE__, __LINE__); + goto out; + } + + /* adjust down nr_pages to what we have physically */ + *nr_pages = kbase_reg_current_backed_size(reg); + + if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset, + reg->nr_pages, 1) != 0) { + dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__); + /* Unable to map in GPU space. */ + WARN_ON(1); + err = -ENOMEM; + goto out; + } + /* no need for the cookie anymore */ + kctx->pending_regions[cookie] = NULL; + bitmap_set(kctx->cookies, cookie, 1); + + /* + * Overwrite the offset with the region start_pfn, so we effectively + * map from offset 0 in the region. However subtract the aligned + * offset so that when user space trims the mapping the beginning of + * the trimmed VMA has the correct vm_pgoff; + */ + vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT); +out: + *regm = reg; + dev_dbg(kctx->kbdev->dev, "kbasep_reg_mmap done\n"); + + return err; +} + +int kbase_context_mmap(struct kbase_context *const kctx, + struct vm_area_struct *const vma) +{ + struct kbase_va_region *reg = NULL; + void *kaddr = NULL; + size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + int err = 0; + int free_on_close = 0; + struct device *dev = kctx->kbdev->dev; + size_t aligned_offset = 0; + + dev_dbg(dev, "kbase_mmap\n"); + + if (!(vma->vm_flags & VM_READ)) + vma->vm_flags &= ~VM_MAYREAD; + if (!(vma->vm_flags & VM_WRITE)) + vma->vm_flags &= ~VM_MAYWRITE; + + if (0 == nr_pages) { + err = -EINVAL; + goto out; + } + + if (!(vma->vm_flags & VM_SHARED)) { + err = -EINVAL; + goto out; + } + + kbase_gpu_vm_lock(kctx); + + if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) { + /* The non-mapped tracking helper page */ + err = kbase_tracking_page_setup(kctx, vma); + goto out_unlock; + } + + /* if not the MTP, verify that the MTP has been mapped */ + rcu_read_lock(); + /* catches both when the special page isn't present or + * when we've forked */ + if (rcu_dereference(kctx->process_mm) != current->mm) { + err = -EINVAL; + rcu_read_unlock(); + goto out_unlock; + } + rcu_read_unlock(); + + switch (vma->vm_pgoff) { + case PFN_DOWN(BASEP_MEM_INVALID_HANDLE): + case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE): + /* Illegal handle for direct map */ + err = -EINVAL; + goto out_unlock; + case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE): + /* MMU dump */ + err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr); + if (0 != err) + goto out_unlock; + /* free the region on munmap */ + free_on_close = 1; + break; +#if MALI_USE_CSF + case PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE): + kbase_gpu_vm_unlock(kctx); + err = kbase_csf_cpu_mmap_user_reg_page(kctx, vma); + goto out; + case PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE) ... + PFN_DOWN(BASE_MEM_COOKIE_BASE) - 1: { + kbase_gpu_vm_unlock(kctx); + mutex_lock(&kctx->csf.lock); + err = kbase_csf_cpu_mmap_user_io_pages(kctx, vma); + mutex_unlock(&kctx->csf.lock); + goto out; + } +#endif + case PFN_DOWN(BASE_MEM_COOKIE_BASE) ... + PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { + err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, + &aligned_offset); + if (0 != err) + goto out_unlock; + /* free the region on munmap */ + free_on_close = 1; + break; + } + default: { + reg = kbase_region_tracker_find_region_enclosing_address(kctx, + (u64)vma->vm_pgoff << PAGE_SHIFT); + + if (!kbase_is_region_invalid_or_free(reg)) { + /* will this mapping overflow the size of the region? */ + if (nr_pages > (reg->nr_pages - + (vma->vm_pgoff - reg->start_pfn))) { + err = -ENOMEM; + goto out_unlock; + } + + if ((vma->vm_flags & VM_READ && + !(reg->flags & KBASE_REG_CPU_RD)) || + (vma->vm_flags & VM_WRITE && + !(reg->flags & KBASE_REG_CPU_WR))) { + /* VM flags inconsistent with region flags */ + err = -EPERM; + dev_err(dev, "%s:%d inconsistent VM flags\n", + __FILE__, __LINE__); + goto out_unlock; + } + + if (KBASE_MEM_TYPE_IMPORTED_UMM == + reg->cpu_alloc->type) { + if (0 != (vma->vm_pgoff - reg->start_pfn)) { + err = -EINVAL; + dev_warn(dev, "%s:%d attempt to do a partial map in a dma_buf: non-zero offset to dma_buf mapping!\n", + __FILE__, __LINE__); + goto out_unlock; + } + err = dma_buf_mmap( + reg->cpu_alloc->imported.umm.dma_buf, + vma, vma->vm_pgoff - reg->start_pfn); + goto out_unlock; + } + + if (reg->cpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { + /* initial params check for aliased dumping map */ + if (nr_pages > reg->gpu_alloc->imported.alias.stride || + !reg->gpu_alloc->imported.alias.stride || + !nr_pages) { + err = -EINVAL; + dev_warn(dev, "mmap aliased: invalid params!\n"); + goto out_unlock; + } + } + else if (reg->cpu_alloc->nents < + (vma->vm_pgoff - reg->start_pfn + nr_pages)) { + /* limit what we map to the amount currently backed */ + if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents) + nr_pages = 0; + else + nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn); + } + } else { + err = -ENOMEM; + goto out_unlock; + } + } /* default */ + } /* switch */ + + err = kbase_cpu_mmap(kctx, reg, vma, kaddr, nr_pages, aligned_offset, + free_on_close); + + if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) { + /* MMU dump - userspace should now have a reference on + * the pages, so we can now free the kernel mapping */ + vfree(kaddr); + } + +out_unlock: + kbase_gpu_vm_unlock(kctx); +out: + if (err) + dev_err(dev, "mmap failed %d\n", err); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_context_mmap); + +void kbase_sync_mem_regions(struct kbase_context *kctx, + struct kbase_vmap_struct *map, enum kbase_sync_type dest) +{ + size_t i; + off_t const offset = map->offset_in_page; + size_t const page_count = PFN_UP(offset + map->size); + + /* Sync first page */ + size_t sz = MIN(((size_t) PAGE_SIZE - offset), map->size); + struct tagged_addr cpu_pa = map->cpu_pages[0]; + struct tagged_addr gpu_pa = map->gpu_pages[0]; + + kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz, dest); + + /* Sync middle pages (if any) */ + for (i = 1; page_count > 2 && i < page_count - 1; i++) { + cpu_pa = map->cpu_pages[i]; + gpu_pa = map->gpu_pages[i]; + kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE, dest); + } + + /* Sync last page (if any) */ + if (page_count > 1) { + cpu_pa = map->cpu_pages[page_count - 1]; + gpu_pa = map->gpu_pages[page_count - 1]; + sz = ((offset + map->size - 1) & ~PAGE_MASK) + 1; + kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, dest); + } +} + +static int kbase_vmap_phy_pages(struct kbase_context *kctx, + struct kbase_va_region *reg, u64 offset_bytes, size_t size, + struct kbase_vmap_struct *map) +{ + unsigned long page_index; + unsigned int offset_in_page = offset_bytes & ~PAGE_MASK; + size_t page_count = PFN_UP(offset_in_page + size); + struct tagged_addr *page_array; + struct page **pages; + void *cpu_addr = NULL; + pgprot_t prot; + size_t i; + + if (!size || !map || !reg->cpu_alloc || !reg->gpu_alloc) + return -EINVAL; + + /* check if page_count calculation will wrap */ + if (size > ((size_t)-1 / PAGE_SIZE)) + return -EINVAL; + + page_index = offset_bytes >> PAGE_SHIFT; + + /* check if page_index + page_count will wrap */ + if (-1UL - page_count < page_index) + return -EINVAL; + + if (page_index + page_count > kbase_reg_current_backed_size(reg)) + return -ENOMEM; + + if (reg->flags & KBASE_REG_DONT_NEED) + return -EINVAL; + + prot = PAGE_KERNEL; + if (!(reg->flags & KBASE_REG_CPU_CACHED)) { + /* Map uncached */ + prot = pgprot_writecombine(prot); + } + + page_array = kbase_get_cpu_phy_pages(reg); + if (!page_array) + return -ENOMEM; + + pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); + if (!pages) + return -ENOMEM; + + for (i = 0; i < page_count; i++) + pages[i] = as_page(page_array[page_index + i]); + + /* Note: enforcing a RO prot_request onto prot is not done, since: + * - CPU-arch-specific integration required + * - kbase_vmap() requires no access checks to be made/enforced */ + + cpu_addr = vmap(pages, page_count, VM_MAP, prot); + + kfree(pages); + + if (!cpu_addr) + return -ENOMEM; + + map->offset_in_page = offset_in_page; + map->cpu_alloc = reg->cpu_alloc; + map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index]; + map->gpu_alloc = reg->gpu_alloc; + map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index]; + map->addr = (void *)((uintptr_t)cpu_addr + offset_in_page); + map->size = size; + map->sync_needed = ((reg->flags & KBASE_REG_CPU_CACHED) != 0) && + !kbase_mem_is_imported(map->gpu_alloc->type); + + if (map->sync_needed) + kbase_sync_mem_regions(kctx, map, KBASE_SYNC_TO_CPU); + + return 0; +} + +void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, + unsigned long prot_request, struct kbase_vmap_struct *map) +{ + struct kbase_va_region *reg; + void *addr = NULL; + u64 offset_bytes; + struct kbase_mem_phy_alloc *cpu_alloc; + struct kbase_mem_phy_alloc *gpu_alloc; + int err; + + kbase_gpu_vm_lock(kctx); + + reg = kbase_region_tracker_find_region_enclosing_address(kctx, + gpu_addr); + if (kbase_is_region_invalid_or_free(reg)) + goto out_unlock; + + /* check access permissions can be satisfied + * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} + */ + if ((reg->flags & prot_request) != prot_request) + goto out_unlock; + + offset_bytes = gpu_addr - (reg->start_pfn << PAGE_SHIFT); + cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + err = kbase_vmap_phy_pages(kctx, reg, offset_bytes, size, map); + if (err < 0) + goto fail_vmap_phy_pages; + + addr = map->addr; + +out_unlock: + kbase_gpu_vm_unlock(kctx); + return addr; + +fail_vmap_phy_pages: + kbase_gpu_vm_unlock(kctx); + kbase_mem_phy_alloc_put(cpu_alloc); + kbase_mem_phy_alloc_put(gpu_alloc); + + return NULL; +} + +void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, + struct kbase_vmap_struct *map) +{ + /* 0 is specified for prot_request to indicate no access checks should + * be made. + * + * As mentioned in kbase_vmap_prot() this means that a kernel-side + * CPU-RO mapping is not enforced to allow this to work */ + return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map); +} +KBASE_EXPORT_TEST_API(kbase_vmap); + +static void kbase_vunmap_phy_pages(struct kbase_context *kctx, + struct kbase_vmap_struct *map) +{ + void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK); + vunmap(addr); + + if (map->sync_needed) + kbase_sync_mem_regions(kctx, map, KBASE_SYNC_TO_DEVICE); + + map->offset_in_page = 0; + map->cpu_pages = NULL; + map->gpu_pages = NULL; + map->addr = NULL; + map->size = 0; + map->sync_needed = false; +} + +void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map) +{ + kbase_vunmap_phy_pages(kctx, map); + map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc); + map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc); +} +KBASE_EXPORT_TEST_API(kbase_vunmap); + +static void kbasep_add_mm_counter(struct mm_struct *mm, int member, long value) +{ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)) + /* To avoid the build breakage due to an unexported kernel symbol + * 'mm_trace_rss_stat' from later kernels, i.e. from V4.19.0 onwards, + * we inline here the equivalent of 'add_mm_counter()' from linux + * kernel V5.4.0~8. + */ + atomic_long_add(value, &mm->rss_stat.count[member]); +#else + add_mm_counter(mm, member, value); +#endif +} + +void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages) +{ + struct mm_struct *mm; + + rcu_read_lock(); + mm = rcu_dereference(kctx->process_mm); + if (mm) { + atomic_add(pages, &kctx->nonmapped_pages); +#ifdef SPLIT_RSS_COUNTING + kbasep_add_mm_counter(mm, MM_FILEPAGES, pages); +#else + spin_lock(&mm->page_table_lock); + kbasep_add_mm_counter(mm, MM_FILEPAGES, pages); + spin_unlock(&mm->page_table_lock); +#endif + } + rcu_read_unlock(); +} + +static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx) +{ + int pages; + struct mm_struct *mm; + + spin_lock(&kctx->mm_update_lock); + mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock)); + if (!mm) { + spin_unlock(&kctx->mm_update_lock); + return; + } + + rcu_assign_pointer(kctx->process_mm, NULL); + spin_unlock(&kctx->mm_update_lock); + synchronize_rcu(); + + pages = atomic_xchg(&kctx->nonmapped_pages, 0); +#ifdef SPLIT_RSS_COUNTING + kbasep_add_mm_counter(mm, MM_FILEPAGES, -pages); +#else + spin_lock(&mm->page_table_lock); + kbasep_add_mm_counter(mm, MM_FILEPAGES, -pages); + spin_unlock(&mm->page_table_lock); +#endif +} + +static void kbase_special_vm_close(struct vm_area_struct *vma) +{ + struct kbase_context *kctx; + + kctx = vma->vm_private_data; + kbasep_os_process_page_usage_drain(kctx); +} + +static const struct vm_operations_struct kbase_vm_special_ops = { + .close = kbase_special_vm_close, +}; + +static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma) +{ + /* check that this is the only tracking page */ + spin_lock(&kctx->mm_update_lock); + if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) { + spin_unlock(&kctx->mm_update_lock); + return -EFAULT; + } + + rcu_assign_pointer(kctx->process_mm, current->mm); + + spin_unlock(&kctx->mm_update_lock); + + /* no real access */ + vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; +#else + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; +#endif + vma->vm_ops = &kbase_vm_special_ops; + vma->vm_private_data = kctx; + + return 0; +} + +#if MALI_USE_CSF +static unsigned long get_queue_doorbell_pfn(struct kbase_device *kbdev, + struct kbase_queue *queue) +{ + lockdep_assert_held(&kbdev->csf.reg_lock); + + /* Return the real Hw doorbell page if queue has been + * assigned one, otherwise a dummy page. Always return the + * dummy page in no mali builds. + */ + if ((queue->doorbell_nr == KBASEP_USER_DB_NR_INVALID) || + IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI)) + return PFN_DOWN(as_phys_addr_t(kbdev->csf.dummy_db_page)); + + return (PFN_DOWN(kbdev->reg_start + CSF_HW_DOORBELL_PAGE_OFFSET + + (u64)queue->doorbell_nr * CSF_HW_DOORBELL_PAGE_SIZE)); +} + +static void kbase_csf_user_io_pages_vm_open(struct vm_area_struct *vma) +{ + WARN(1, "Unexpected attempt to clone private vma\n"); + vma->vm_private_data = NULL; +} + +static void kbase_csf_user_io_pages_vm_close(struct vm_area_struct *vma) +{ + struct kbase_queue *queue = vma->vm_private_data; + struct kbase_context *kctx; + + if (WARN_ON(!queue)) + return; + + kctx = queue->kctx; + + mutex_lock(&kctx->csf.lock); + kbase_csf_queue_unbind(queue); + mutex_unlock(&kctx->csf.lock); + + /* Now as the vma is closed, drop the reference on mali device file */ + fput(kctx->filp); +} + +#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) +static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_area_struct *vma, + struct vm_fault *vmf) +{ +#else +static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; +#endif + struct kbase_queue *queue = vma->vm_private_data; + unsigned long doorbell_cpu_addr, input_cpu_addr, output_cpu_addr; + unsigned long doorbell_page_pfn, input_page_pfn, output_page_pfn; + pgprot_t doorbell_pgprot, input_page_pgprot, output_page_pgprot; + size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start); + vm_fault_t ret; + struct kbase_device *kbdev; + struct memory_group_manager_device *mgm_dev; + + /* Few sanity checks up front */ + if ((nr_pages != BASEP_QUEUE_NR_MMAP_USER_PAGES) || + (vma->vm_pgoff != queue->db_file_offset)) + return VM_FAULT_SIGBUS; + + mutex_lock(&queue->kctx->csf.lock); + kbdev = queue->kctx->kbdev; + mgm_dev = kbdev->mgm_dev; + + /* Always map the doorbell page as uncached */ + doorbell_pgprot = pgprot_device(vma->vm_page_prot); + +#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ + ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ + (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) + vma->vm_page_prot = doorbell_pgprot; + input_page_pgprot = doorbell_pgprot; + output_page_pgprot = doorbell_pgprot; +#else + if (kbdev->system_coherency == COHERENCY_NONE) { + input_page_pgprot = pgprot_writecombine(vma->vm_page_prot); + output_page_pgprot = pgprot_writecombine(vma->vm_page_prot); + } else { + input_page_pgprot = vma->vm_page_prot; + output_page_pgprot = vma->vm_page_prot; + } +#endif + + doorbell_cpu_addr = vma->vm_start; + +#if KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE + if ((unsigned long)vmf->virtual_address == doorbell_cpu_addr) { +#else + if (vmf->address == doorbell_cpu_addr) { +#endif + mutex_lock(&kbdev->csf.reg_lock); + doorbell_page_pfn = get_queue_doorbell_pfn(kbdev, queue); + ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, + KBASE_MEM_GROUP_CSF_IO, vma, doorbell_cpu_addr, + doorbell_page_pfn, doorbell_pgprot); + mutex_unlock(&kbdev->csf.reg_lock); + } else { + /* Map the Input page */ + input_cpu_addr = doorbell_cpu_addr + PAGE_SIZE; + input_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[0])); + ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, + KBASE_MEM_GROUP_CSF_IO, vma, input_cpu_addr, + input_page_pfn, input_page_pgprot); + if (ret != VM_FAULT_NOPAGE) + goto exit; + + /* Map the Output page */ + output_cpu_addr = input_cpu_addr + PAGE_SIZE; + output_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[1])); + ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, + KBASE_MEM_GROUP_CSF_IO, vma, output_cpu_addr, + output_page_pfn, output_page_pgprot); + } + +exit: + mutex_unlock(&queue->kctx->csf.lock); + return ret; +} + +static const struct vm_operations_struct kbase_csf_user_io_pages_vm_ops = { + .open = kbase_csf_user_io_pages_vm_open, + .close = kbase_csf_user_io_pages_vm_close, + .fault = kbase_csf_user_io_pages_vm_fault +}; + +/* Program the client process's page table entries to map the pair of + * input/output pages & Hw doorbell page. The caller should have validated that + * vma->vm_pgoff maps to the range of csf cookies. + */ +static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, + struct vm_area_struct *vma) +{ + unsigned long cookie = + vma->vm_pgoff - PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); + size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + struct kbase_queue *queue; + int err = 0; + + lockdep_assert_held(&kctx->csf.lock); + + queue = kctx->csf.user_pages_info[cookie]; + + /* Looks like the bind has been aborted */ + if (!queue) + return -EINVAL; + + if (WARN_ON(test_bit(cookie, kctx->csf.cookies))) + return -EINVAL; + + /* no need for the cookie anymore */ + kctx->csf.user_pages_info[cookie] = NULL; + bitmap_set(kctx->csf.cookies, cookie, 1); + + /* Reset the handle to avoid (re)freeing the cookie (which can + * now get re-assigned) on unbind. + */ + queue->handle = BASEP_MEM_INVALID_HANDLE; + + if (nr_pages != BASEP_QUEUE_NR_MMAP_USER_PAGES) { + err = -EINVAL; + goto map_failed; + } + + err = kbase_csf_alloc_command_stream_user_pages(kctx, queue); + if (err) + goto map_failed; + +#if (KERNEL_VERSION(3, 7, 0) <= LINUX_VERSION_CODE) + vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; +#else + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; +#endif + /* TODO use VM_MIXEDMAP, since it is more appropriate as both types of + * memory with and without "struct page" backing are being inserted here. + * Hw Doorbell pages comes from the device register area so kernel does + * not use "struct page" for them. + */ + vma->vm_flags |= VM_PFNMAP; + + vma->vm_ops = &kbase_csf_user_io_pages_vm_ops; + vma->vm_private_data = queue; + + /* Make vma point to the special internal file, but don't drop the + * reference on mali device file (that would be done later when the + * vma is closed). + */ + vma->vm_file = kctx->kbdev->csf.db_filp; + get_file(vma->vm_file); + /* Also adjust the vm_pgoff */ + vma->vm_pgoff = queue->db_file_offset; + + return 0; + +map_failed: + kbase_csf_queue_unbind(queue); + + return err; +} + +static void kbase_csf_user_reg_vm_close(struct vm_area_struct *vma) +{ + struct kbase_context *kctx = vma->vm_private_data; + + WARN_ON(!kctx->csf.user_reg_vma); + + kctx->csf.user_reg_vma = NULL; +} + +#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) +static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_area_struct *vma, + struct vm_fault *vmf) +{ +#else +static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; +#endif + struct kbase_context *kctx = vma->vm_private_data; + struct kbase_device *kbdev = kctx->kbdev; + unsigned long pfn = PFN_DOWN(kbdev->reg_start + USER_BASE); + size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start); + + /* Few sanity checks up front */ + if (WARN_ON(nr_pages != 1) || + WARN_ON(vma != kctx->csf.user_reg_vma) || + WARN_ON(vma->vm_pgoff != + PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE))) + return VM_FAULT_SIGBUS; + + /* TODO: check PM state here and don't map in the actual register page + * if GPU is powered down or is about to be powered down. + */ + + return vmf_insert_pfn_prot(vma, vma->vm_start, pfn, vma->vm_page_prot); +} + +static const struct vm_operations_struct kbase_csf_user_reg_vm_ops = { + .close = kbase_csf_user_reg_vm_close, + .fault = kbase_csf_user_reg_vm_fault +}; + +static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, + struct vm_area_struct *vma) +{ + size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start); + + /* Few sanity checks */ + if (kctx->csf.user_reg_vma) + return -EBUSY; + + if (nr_pages != 1) + return -EINVAL; + + if (vma->vm_flags & (VM_WRITE | VM_MAYWRITE)) + return -EPERM; + + /* Map uncached */ + vma->vm_page_prot = pgprot_device(vma->vm_page_prot); + + vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; + + /* User register page comes from the device register area so + * "struct page" isn't available for it. + */ + vma->vm_flags |= VM_PFNMAP; + + kctx->csf.user_reg_vma = vma; + + vma->vm_ops = &kbase_csf_user_reg_vm_ops; + vma->vm_private_data = kctx; + + return 0; +} + +#endif /* MALI_USE_CSF */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h new file mode 100755 index 000000000000..85e030ab751a --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h @@ -0,0 +1,478 @@ +/* + * + * (C) COPYRIGHT 2010, 2012-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_mem_linux.h + * Base kernel memory APIs, Linux implementation. + */ + +#ifndef _KBASE_MEM_LINUX_H_ +#define _KBASE_MEM_LINUX_H_ + +/** A HWC dump mapping */ +struct kbase_hwc_dma_mapping { + void *cpu_va; + dma_addr_t dma_pa; + size_t size; +}; + +/** + * kbase_mem_alloc - Create a new allocation for GPU + * + * @kctx: The kernel context + * @va_pages: The number of pages of virtual address space to reserve + * @commit_pages: The number of physical pages to allocate upfront + * @extent: The number of extra pages to allocate on each GPU fault which + * grows the region. + * @flags: bitmask of BASE_MEM_* flags to convey special requirements & + * properties for the new allocation. + * @gpu_va: Start address of the memory region which was allocated from GPU + * virtual address space. + * + * Return: 0 on success or error code + */ +struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, + u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, + u64 *gpu_va); + +/** + * kbase_mem_query - Query properties of a GPU memory region + * + * @kctx: The kernel context + * @gpu_addr: A GPU address contained within the memory region + * @query: The type of query, from KBASE_MEM_QUERY_* flags, which could be + * regarding the amount of backing physical memory allocated so far + * for the region or the size of the region or the flags associated + * with the region. + * @out: Pointer to the location to store the result of query. + * + * Return: 0 on success or error code + */ +int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, + u64 *const out); + +/** + * kbase_mem_import - Import the external memory for use by the GPU + * + * @kctx: The kernel context + * @type: Type of external memory + * @phandle: Handle to the external memory interpreted as per the type. + * @padding: Amount of extra VA pages to append to the imported buffer + * @gpu_va: GPU address assigned to the imported external memory + * @va_pages: Size of the memory region reserved from the GPU address space + * @flags: bitmask of BASE_MEM_* flags to convey special requirements & + * properties for the new allocation representing the external + * memory. + * Return: 0 on success or error code + */ +int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, + u64 *flags); + +/** + * kbase_mem_alias - Create a new allocation for GPU, aliasing one or more + * memory regions + * + * @kctx: The kernel context + * @flags: bitmask of BASE_MEM_* flags. + * @stride: Bytes between start of each memory region + * @nents: The number of regions to pack together into the alias + * @ai: Pointer to the struct containing the memory aliasing info + * @num_pages: Number of pages the alias will cover + * + * Return: 0 on failure or otherwise the GPU VA for the alias + */ +u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages); + +/** + * kbase_mem_flags_change - Change the flags for a memory region + * + * @kctx: The kernel context + * @gpu_addr: A GPU address contained within the memory region to modify. + * @flags: The new flags to set + * @mask: Mask of the flags, from BASE_MEM_*, to modify. + * + * Return: 0 on success or error code + */ +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask); + +/** + * kbase_mem_commit - Change the physical backing size of a region + * + * @kctx: The kernel context + * @gpu_addr: Handle to the memory region + * @new_pages: Number of physical pages to back the region with + * + * Return: 0 on success or error code + */ +int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages); + +/** + * kbase_mem_shrink - Shrink the physical backing size of a region + * + * @kctx: The kernel context + * @reg: The GPU region + * @new_pages: Number of physical pages to back the region with + * + * Return: 0 on success or error code + */ +int kbase_mem_shrink(struct kbase_context *kctx, + struct kbase_va_region *reg, u64 new_pages); + +/** + * kbase_context_mmap - Memory map method, gets invoked when mmap system call is + * issued on device file /dev/malixx. + * @kctx: The kernel context + * @vma: Pointer to the struct containing the info where the GPU allocation + * will be mapped in virtual address space of CPU. + * + * Return: 0 on success or error code + */ +int kbase_context_mmap(struct kbase_context *kctx, struct vm_area_struct *vma); + +/** + * kbase_mem_evictable_init - Initialize the Ephemeral memory eviction + * mechanism. + * @kctx: The kbase context to initialize. + * + * Return: Zero on success or -errno on failure. + */ +int kbase_mem_evictable_init(struct kbase_context *kctx); + +/** + * kbase_mem_evictable_deinit - De-initialize the Ephemeral memory eviction + * mechanism. + * @kctx: The kbase context to de-initialize. + */ +void kbase_mem_evictable_deinit(struct kbase_context *kctx); + +/** + * kbase_mem_grow_gpu_mapping - Grow the GPU mapping of an allocation + * @kctx: Context the region belongs to + * @reg: The GPU region + * @new_pages: The number of pages after the grow + * @old_pages: The number of pages before the grow + * + * Return: 0 on success, -errno on error. + * + * Expand the GPU mapping to encompass the new psychical pages which have + * been added to the allocation. + * + * Note: Caller must be holding the region lock. + */ +int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages); + +/** + * kbase_mem_evictable_make - Make a physical allocation eligible for eviction + * @gpu_alloc: The physical allocation to make evictable + * + * Return: 0 on success, -errno on error. + * + * Take the provided region and make all the physical pages within it + * reclaimable by the kernel, updating the per-process VM stats as well. + * Remove any CPU mappings (as these can't be removed in the shrinker callback + * as mmap_sem/mmap_lock might already be taken) but leave the GPU mapping + * intact as and until the shrinker reclaims the allocation. + * + * Note: Must be called with the region lock of the containing context. + */ +int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc); + +/** + * kbase_mem_evictable_unmake - Remove a physical allocations eligibility for + * eviction. + * @alloc: The physical allocation to remove eviction eligibility from. + * + * Return: True if the allocation had its backing restored and false if + * it hasn't. + * + * Make the physical pages in the region no longer reclaimable and update the + * per-process stats, if the shrinker has already evicted the memory then + * re-allocate it if the region is still alive. + * + * Note: Must be called with the region lock of the containing context. + */ +bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *alloc); + +struct kbase_vmap_struct { + off_t offset_in_page; + struct kbase_mem_phy_alloc *cpu_alloc; + struct kbase_mem_phy_alloc *gpu_alloc; + struct tagged_addr *cpu_pages; + struct tagged_addr *gpu_pages; + void *addr; + size_t size; + bool sync_needed; +}; + + +/** + * kbase_vmap_prot - Map a GPU VA range into the kernel safely, only if the + * requested access permissions are supported + * @kctx: Context the VA range belongs to + * @gpu_addr: Start address of VA range + * @size: Size of VA range + * @prot_request: Flags indicating how the caller will then access the memory + * @map: Structure to be given to kbase_vunmap() on freeing + * + * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error + * + * Map a GPU VA Range into the kernel. The VA range must be contained within a + * GPU memory region. Appropriate CPU cache-flushing operations are made as + * required, dependent on the CPU mapping for the memory region. + * + * This is safer than using kmap() on the pages directly, + * because the pages here are refcounted to prevent freeing (and hence reuse + * elsewhere in the system) until an kbase_vunmap() + * + * The flags in @prot_request should use KBASE_REG_{CPU,GPU}_{RD,WR}, to check + * whether the region should allow the intended access, and return an error if + * disallowed. This is essential for security of imported memory, particularly + * a user buf from SHM mapped into the process as RO. In that case, write + * access must be checked if the intention is for kernel to write to the + * memory. + * + * The checks are also there to help catch access errors on memory where + * security is not a concern: imported memory that is always RW, and memory + * that was allocated and owned by the process attached to @kctx. In this case, + * it helps to identify memory that was was mapped with the wrong access type. + * + * Note: KBASE_REG_GPU_{RD,WR} flags are currently supported for legacy cases + * where either the security of memory is solely dependent on those flags, or + * when userspace code was expecting only the GPU to access the memory (e.g. HW + * workarounds). + * + * All cache maintenance operations shall be ignored if the + * memory region has been imported. + * + */ +void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, + unsigned long prot_request, struct kbase_vmap_struct *map); + +/** + * kbase_vmap - Map a GPU VA range into the kernel safely + * @kctx: Context the VA range belongs to + * @gpu_addr: Start address of VA range + * @size: Size of VA range + * @map: Structure to be given to kbase_vunmap() on freeing + * + * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error + * + * Map a GPU VA Range into the kernel. The VA range must be contained within a + * GPU memory region. Appropriate CPU cache-flushing operations are made as + * required, dependent on the CPU mapping for the memory region. + * + * This is safer than using kmap() on the pages directly, + * because the pages here are refcounted to prevent freeing (and hence reuse + * elsewhere in the system) until an kbase_vunmap() + * + * kbase_vmap_prot() should be used in preference, since kbase_vmap() makes no + * checks to ensure the security of e.g. imported user bufs from RO SHM. + * + * Note: All cache maintenance operations shall be ignored if the memory region + * has been imported. + */ +void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, + struct kbase_vmap_struct *map); + +/** + * kbase_vunmap - Unmap a GPU VA range from the kernel + * @kctx: Context the VA range belongs to + * @map: Structure describing the mapping from the corresponding kbase_vmap() + * call + * + * Unmaps a GPU VA range from the kernel, given its @map structure obtained + * from kbase_vmap(). Appropriate CPU cache-flushing operations are made as + * required, dependent on the CPU mapping for the memory region. + * + * The reference taken on pages during kbase_vmap() is released. + * + * Note: All cache maintenance operations shall be ignored if the memory region + * has been imported. + */ +void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map); + +extern const struct vm_operations_struct kbase_vm_ops; + +/** + * kbase_sync_mem_regions - Perform the cache maintenance for the kernel mode + * CPU mapping. + * @kctx: Context the CPU mapping belongs to. + * @map: Structure describing the CPU mapping, setup previously by the + * kbase_vmap() call. + * @dest: Indicates the type of maintenance required (i.e. flush or invalidate) + * + * Note: The caller shall ensure that CPU mapping is not revoked & remains + * active whilst the maintenance is in progress. + */ +void kbase_sync_mem_regions(struct kbase_context *kctx, + struct kbase_vmap_struct *map, enum kbase_sync_type dest); + +/** + * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation + * @kctx: Context the region belongs to + * @reg: The GPU region + * @new_pages: The number of pages after the shrink + * @old_pages: The number of pages before the shrink + * + * Shrink (or completely remove) all CPU mappings which reference the shrunk + * part of the allocation. + */ +void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages); + +/** + * kbase_phy_alloc_mapping_term - Terminate the kernel side mapping of a + * physical allocation + * @kctx: The kernel base context associated with the mapping + * @alloc: Pointer to the allocation to terminate + * + * This function will unmap the kernel mapping, and free any structures used to + * track it. + */ +void kbase_phy_alloc_mapping_term(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc); + +/** + * kbase_phy_alloc_mapping_get - Get a kernel-side CPU pointer to the permanent + * mapping of a physical allocation + * @kctx: The kernel base context @gpu_addr will be looked up in + * @gpu_addr: The gpu address to lookup for the kernel-side CPU mapping + * @out_kern_mapping: Pointer to storage for a struct kbase_vmap_struct pointer + * which will be used for a call to + * kbase_phy_alloc_mapping_put() + * + * Return: Pointer to a kernel-side accessible location that directly + * corresponds to @gpu_addr, or NULL on failure + * + * Looks up @gpu_addr to retrieve the CPU pointer that can be used to access + * that location kernel-side. Only certain kinds of memory have a permanent + * kernel mapping, refer to the internal functions + * kbase_reg_needs_kernel_mapping() and kbase_phy_alloc_mapping_init() for more + * information. + * + * If this function succeeds, a CPU access to the returned pointer will access + * the actual location represented by @gpu_addr. That is, the return value does + * not require any offset added to it to access the location specified in + * @gpu_addr + * + * The client must take care to either apply any necessary sync operations when + * accessing the data, or ensure that the enclosing region was coherent with + * the GPU, or uncached in the CPU. + * + * The refcount on the physical allocations backing the region are taken, so + * that they do not disappear whilst the client is accessing it. Once the + * client has finished accessing the memory, it must be released with a call to + * kbase_phy_alloc_mapping_put() + * + * Whilst this is expected to execute quickly (the mapping was already setup + * when the physical allocation was created), the call is not IRQ-safe due to + * the region lookup involved. + * + * An error code may indicate that: + * - a userside process has freed the allocation, and so @gpu_addr is no longer + * valid + * - the region containing @gpu_addr does not support a permanent kernel mapping + */ +void *kbase_phy_alloc_mapping_get(struct kbase_context *kctx, u64 gpu_addr, + struct kbase_vmap_struct **out_kern_mapping); + +/** + * kbase_phy_alloc_mapping_put - Put a reference to the kernel-side mapping of a + * physical allocation + * @kctx: The kernel base context associated with the mapping + * @kern_mapping: Pointer to a struct kbase_phy_alloc_mapping pointer obtained + * from a call to kbase_phy_alloc_mapping_get() + * + * Releases the reference to the allocations backing @kern_mapping that was + * obtained through a call to kbase_phy_alloc_mapping_get(). This must be used + * when the client no longer needs to access the kernel-side CPU pointer. + * + * If this was the last reference on the underlying physical allocations, they + * will go through the normal allocation free steps, which also includes an + * unmap of the permanent kernel mapping for those allocations. + * + * Due to these operations, the function is not IRQ-safe. However it is + * expected to execute quickly in the normal case, i.e. when the region holding + * the physical allocation is still present. + */ +void kbase_phy_alloc_mapping_put(struct kbase_context *kctx, + struct kbase_vmap_struct *kern_mapping); + +/** + * kbase_get_cache_line_alignment - Return cache line alignment + * + * Helper function to return the maximum cache line alignment considering + * both CPU and GPU cache sizes. + * + * Return: CPU and GPU cache line alignment, in bytes. + * + * @kbdev: Device pointer. + */ +u32 kbase_get_cache_line_alignment(struct kbase_device *kbdev); + +#if (KERNEL_VERSION(4, 20, 0) > LINUX_VERSION_CODE) +static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, pgprot_t pgprot) +{ + int err; + +#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ + ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ + (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) + if (pgprot_val(pgprot) != pgprot_val(vma->vm_page_prot)) + return VM_FAULT_SIGBUS; + + err = vm_insert_pfn(vma, addr, pfn); +#else + err = vm_insert_pfn_prot(vma, addr, pfn, pgprot); +#endif + + if (unlikely(err == -ENOMEM)) + return VM_FAULT_OOM; + if (unlikely(err < 0 && err != -EBUSY)) + return VM_FAULT_SIGBUS; + + return VM_FAULT_NOPAGE; +} +#endif + +/** + * kbase_mem_get_process_mmap_lock - Return the mmap lock for the current process + * + * Return: the mmap lock for the current process + */ +static inline struct rw_semaphore *kbase_mem_get_process_mmap_lock(void) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) + return ¤t->mm->mmap_sem; +#else /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) */ + return ¤t->mm->mmap_lock; +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) */ +} + +#endif /* _KBASE_MEM_LINUX_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h new file mode 100755 index 000000000000..70116030f233 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h @@ -0,0 +1,166 @@ +/* + * + * (C) COPYRIGHT 2012-2014,2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#ifndef _KBASE_MEM_LOWLEVEL_H +#define _KBASE_MEM_LOWLEVEL_H + +#ifndef _KBASE_H_ +#error "Don't include this file directly, use mali_kbase.h instead" +#endif + +#include + +/** + * @brief Flags for kbase_phy_allocator_pages_alloc + */ +#define KBASE_PHY_PAGES_FLAG_DEFAULT (0) /** Default allocation flag */ +#define KBASE_PHY_PAGES_FLAG_CLEAR (1 << 0) /** Clear the pages after allocation */ +#define KBASE_PHY_PAGES_FLAG_POISON (1 << 1) /** Fill the memory with a poison value */ + +#define KBASE_PHY_PAGES_SUPPORTED_FLAGS (KBASE_PHY_PAGES_FLAG_DEFAULT|KBASE_PHY_PAGES_FLAG_CLEAR|KBASE_PHY_PAGES_FLAG_POISON) + +#define KBASE_PHY_PAGES_POISON_VALUE 0xFD /** Value to fill the memory with when KBASE_PHY_PAGES_FLAG_POISON is set */ + +enum kbase_sync_type { + KBASE_SYNC_TO_CPU, + KBASE_SYNC_TO_DEVICE +}; + +struct tagged_addr { phys_addr_t tagged_addr; }; + +#define HUGE_PAGE (1u << 0) +#define HUGE_HEAD (1u << 1) +#define FROM_PARTIAL (1u << 2) + +/* + * Note: if macro for converting physical address to page is not defined + * in the kernel itself, it is defined hereby. This is to avoid build errors + * which are reported during builds for some architectures. + */ +#ifndef phys_to_page +#define phys_to_page(phys) (pfn_to_page((phys) >> PAGE_SHIFT)) +#endif + +/** + * as_phys_addr_t - Retrieve the physical address from tagged address by + * masking the lower order 12 bits. + * @t: tagged address to be translated. + * + * Return: physical address corresponding to tagged address. + */ +static inline phys_addr_t as_phys_addr_t(struct tagged_addr t) +{ + return t.tagged_addr & PAGE_MASK; +} + +/** + * as_page - Retrieve the struct page from a tagged address + * @t: tagged address to be translated. + * + * Return: pointer to struct page corresponding to tagged address. + */ +static inline struct page *as_page(struct tagged_addr t) +{ + return phys_to_page(as_phys_addr_t(t)); +} + +/** + * as_tagged - Convert the physical address to tagged address type though + * there is no tag info present, the lower order 12 bits will be 0 + * @phys: physical address to be converted to tagged type + * + * This is used for 4KB physical pages allocated by the Driver or imported pages + * and is needed as physical pages tracking object stores the reference for + * physical pages using tagged address type in lieu of the type generally used + * for physical addresses. + * + * Return: address of tagged address type. + */ +static inline struct tagged_addr as_tagged(phys_addr_t phys) +{ + struct tagged_addr t; + + t.tagged_addr = phys & PAGE_MASK; + return t; +} + +/** + * as_tagged_tag - Form the tagged address by storing the tag or metadata in the + * lower order 12 bits of physial address + * @phys: physical address to be converted to tagged address + * @tag: tag to be stored along with the physical address. + * + * The tag info is used while freeing up the pages + * + * Return: tagged address storing physical address & tag. + */ +static inline struct tagged_addr as_tagged_tag(phys_addr_t phys, int tag) +{ + struct tagged_addr t; + + t.tagged_addr = (phys & PAGE_MASK) | (tag & ~PAGE_MASK); + return t; +} + +/** + * is_huge - Check if the physical page is one of the 512 4KB pages of the + * large page which was not split to be used partially + * @t: tagged address storing the tag in the lower order bits. + * + * Return: true if page belongs to large page, or false + */ +static inline bool is_huge(struct tagged_addr t) +{ + return t.tagged_addr & HUGE_PAGE; +} + +/** + * is_huge_head - Check if the physical page is the first 4KB page of the + * 512 4KB pages within a large page which was not split + * to be used partially + * @t: tagged address storing the tag in the lower order bits. + * + * Return: true if page is the first page of a large page, or false + */ +static inline bool is_huge_head(struct tagged_addr t) +{ + int mask = HUGE_HEAD | HUGE_PAGE; + + return mask == (t.tagged_addr & mask); +} + +/** + * is_partial - Check if the physical page is one of the 512 pages of the + * large page which was split in 4KB pages to be used + * partially for allocations >= 2 MB in size. + * @t: tagged address storing the tag in the lower order bits. + * + * Return: true if page was taken from large page used partially, or false + */ +static inline bool is_partial(struct tagged_addr t) +{ + return t.tagged_addr & FROM_PARTIAL; +} + +#endif /* _KBASE_LOWLEVEL_H */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c new file mode 100755 index 000000000000..0723e32e2003 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c @@ -0,0 +1,856 @@ +/* + * + * (C) COPYRIGHT 2015-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define pool_dbg(pool, format, ...) \ + dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \ + (pool->next_pool) ? "kctx" : "kbdev", \ + kbase_mem_pool_size(pool), \ + kbase_mem_pool_max_size(pool), \ + ##__VA_ARGS__) + +#define NOT_DIRTY false +#define NOT_RECLAIMED false + +static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool) +{ + ssize_t max_size = kbase_mem_pool_max_size(pool); + ssize_t cur_size = kbase_mem_pool_size(pool); + + return max(max_size - cur_size, (ssize_t)0); +} + +static bool kbase_mem_pool_is_full(struct kbase_mem_pool *pool) +{ + return kbase_mem_pool_size(pool) >= kbase_mem_pool_max_size(pool); +} + +static bool kbase_mem_pool_is_empty(struct kbase_mem_pool *pool) +{ + return kbase_mem_pool_size(pool) == 0; +} + +static void kbase_mem_pool_add_locked(struct kbase_mem_pool *pool, + struct page *p) +{ + lockdep_assert_held(&pool->pool_lock); + + list_add(&p->lru, &pool->page_list); + pool->cur_size++; + + pool_dbg(pool, "added page\n"); +} + +static void kbase_mem_pool_add(struct kbase_mem_pool *pool, struct page *p) +{ + kbase_mem_pool_lock(pool); + kbase_mem_pool_add_locked(pool, p); + kbase_mem_pool_unlock(pool); +} + +static void kbase_mem_pool_add_list_locked(struct kbase_mem_pool *pool, + struct list_head *page_list, size_t nr_pages) +{ + lockdep_assert_held(&pool->pool_lock); + + list_splice(page_list, &pool->page_list); + pool->cur_size += nr_pages; + + pool_dbg(pool, "added %zu pages\n", nr_pages); +} + +static void kbase_mem_pool_add_list(struct kbase_mem_pool *pool, + struct list_head *page_list, size_t nr_pages) +{ + kbase_mem_pool_lock(pool); + kbase_mem_pool_add_list_locked(pool, page_list, nr_pages); + kbase_mem_pool_unlock(pool); +} + +static struct page *kbase_mem_pool_remove_locked(struct kbase_mem_pool *pool) +{ + struct page *p; + + lockdep_assert_held(&pool->pool_lock); + + if (kbase_mem_pool_is_empty(pool)) + return NULL; + + p = list_first_entry(&pool->page_list, struct page, lru); + list_del_init(&p->lru); + pool->cur_size--; + + pool_dbg(pool, "removed page\n"); + + return p; +} + +static struct page *kbase_mem_pool_remove(struct kbase_mem_pool *pool) +{ + struct page *p; + + kbase_mem_pool_lock(pool); + p = kbase_mem_pool_remove_locked(pool); + kbase_mem_pool_unlock(pool); + + return p; +} + +static void kbase_mem_pool_sync_page(struct kbase_mem_pool *pool, + struct page *p) +{ + struct device *dev = pool->kbdev->dev; + dma_sync_single_for_device(dev, kbase_dma_addr(p), + (PAGE_SIZE << pool->order), DMA_BIDIRECTIONAL); +} + +static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool, + struct page *p) +{ + int i; + + for (i = 0; i < (1U << pool->order); i++) + clear_highpage(p+i); + + kbase_mem_pool_sync_page(pool, p); +} + +static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, + struct page *p) +{ + /* Zero page before spilling */ + kbase_mem_pool_zero_page(next_pool, p); + + kbase_mem_pool_add(next_pool, p); +} + +struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) +{ + struct page *p; + gfp_t gfp; + struct kbase_device *const kbdev = pool->kbdev; + struct device *const dev = kbdev->dev; + dma_addr_t dma_addr; + int i; + +#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) + /* DMA cache sync fails for HIGHMEM before 3.5 on ARM */ + gfp = GFP_USER | __GFP_ZERO; +#else + gfp = GFP_HIGHUSER | __GFP_ZERO; +#endif + + /* don't warn on higher order failures */ + if (pool->order) + gfp |= __GFP_NOWARN; + + p = kbdev->mgm_dev->ops.mgm_alloc_page(kbdev->mgm_dev, + pool->group_id, gfp, pool->order); + if (!p) + return NULL; + + dma_addr = dma_map_page(dev, p, 0, (PAGE_SIZE << pool->order), + DMA_BIDIRECTIONAL); + + if (dma_mapping_error(dev, dma_addr)) { + kbdev->mgm_dev->ops.mgm_free_page(kbdev->mgm_dev, + pool->group_id, p, pool->order); + return NULL; + } + + WARN_ON(dma_addr != page_to_phys(p)); + for (i = 0; i < (1u << pool->order); i++) + kbase_set_dma_addr(p+i, dma_addr + PAGE_SIZE * i); + + return p; +} + +static void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, + struct page *p) +{ + struct kbase_device *const kbdev = pool->kbdev; + struct device *const dev = kbdev->dev; + dma_addr_t dma_addr = kbase_dma_addr(p); + int i; + + dma_unmap_page(dev, dma_addr, (PAGE_SIZE << pool->order), + DMA_BIDIRECTIONAL); + for (i = 0; i < (1u << pool->order); i++) + kbase_clear_dma_addr(p+i); + + kbdev->mgm_dev->ops.mgm_free_page(kbdev->mgm_dev, + pool->group_id, p, pool->order); + + pool_dbg(pool, "freed page to kernel\n"); +} + +static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool, + size_t nr_to_shrink) +{ + struct page *p; + size_t i; + + lockdep_assert_held(&pool->pool_lock); + + for (i = 0; i < nr_to_shrink && !kbase_mem_pool_is_empty(pool); i++) { + p = kbase_mem_pool_remove_locked(pool); + kbase_mem_pool_free_page(pool, p); + } + + return i; +} + +static size_t kbase_mem_pool_shrink(struct kbase_mem_pool *pool, + size_t nr_to_shrink) +{ + size_t nr_freed; + + kbase_mem_pool_lock(pool); + nr_freed = kbase_mem_pool_shrink_locked(pool, nr_to_shrink); + kbase_mem_pool_unlock(pool); + + return nr_freed; +} + +int kbase_mem_pool_grow(struct kbase_mem_pool *pool, + size_t nr_to_grow) +{ + struct page *p; + size_t i; + + kbase_mem_pool_lock(pool); + + pool->dont_reclaim = true; + for (i = 0; i < nr_to_grow; i++) { + if (pool->dying) { + pool->dont_reclaim = false; + kbase_mem_pool_shrink_locked(pool, nr_to_grow); + kbase_mem_pool_unlock(pool); + + return -ENOMEM; + } + kbase_mem_pool_unlock(pool); + + p = kbase_mem_alloc_page(pool); + if (!p) { + kbase_mem_pool_lock(pool); + pool->dont_reclaim = false; + kbase_mem_pool_unlock(pool); + + return -ENOMEM; + } + + kbase_mem_pool_lock(pool); + kbase_mem_pool_add_locked(pool, p); + } + pool->dont_reclaim = false; + kbase_mem_pool_unlock(pool); + + return 0; +} + +void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size) +{ + size_t cur_size; + int err = 0; + + cur_size = kbase_mem_pool_size(pool); + + if (new_size > pool->max_size) + new_size = pool->max_size; + + if (new_size < cur_size) + kbase_mem_pool_shrink(pool, cur_size - new_size); + else if (new_size > cur_size) + err = kbase_mem_pool_grow(pool, new_size - cur_size); + + if (err) { + size_t grown_size = kbase_mem_pool_size(pool); + + dev_warn(pool->kbdev->dev, + "Mem pool not grown to the required size of %zu bytes, grown for additional %zu bytes instead!\n", + (new_size - cur_size), (grown_size - cur_size)); + } +} + +void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size) +{ + size_t cur_size; + size_t nr_to_shrink; + + kbase_mem_pool_lock(pool); + + pool->max_size = max_size; + + cur_size = kbase_mem_pool_size(pool); + if (max_size < cur_size) { + nr_to_shrink = cur_size - max_size; + kbase_mem_pool_shrink_locked(pool, nr_to_shrink); + } + + kbase_mem_pool_unlock(pool); +} + + +static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_mem_pool *pool; + size_t pool_size; + + pool = container_of(s, struct kbase_mem_pool, reclaim); + + kbase_mem_pool_lock(pool); + if (pool->dont_reclaim && !pool->dying) { + kbase_mem_pool_unlock(pool); + return 0; + } + pool_size = kbase_mem_pool_size(pool); + kbase_mem_pool_unlock(pool); + + return pool_size; +} + +static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_mem_pool *pool; + unsigned long freed; + + pool = container_of(s, struct kbase_mem_pool, reclaim); + + kbase_mem_pool_lock(pool); + if (pool->dont_reclaim && !pool->dying) { + kbase_mem_pool_unlock(pool); + return 0; + } + + pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan); + + freed = kbase_mem_pool_shrink_locked(pool, sc->nr_to_scan); + + kbase_mem_pool_unlock(pool); + + pool_dbg(pool, "reclaim freed %ld pages\n", freed); + + return freed; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static int kbase_mem_pool_reclaim_shrink(struct shrinker *s, + struct shrink_control *sc) +{ + if (sc->nr_to_scan == 0) + return kbase_mem_pool_reclaim_count_objects(s, sc); + + return kbase_mem_pool_reclaim_scan_objects(s, sc); +} +#endif + +int kbase_mem_pool_init(struct kbase_mem_pool *pool, + const struct kbase_mem_pool_config *config, + unsigned int order, + int group_id, + struct kbase_device *kbdev, + struct kbase_mem_pool *next_pool) +{ + if (WARN_ON(group_id < 0) || + WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { + return -EINVAL; + } + + pool->cur_size = 0; + pool->max_size = kbase_mem_pool_config_get_max_size(config); + pool->order = order; + pool->group_id = group_id; + pool->kbdev = kbdev; + pool->next_pool = next_pool; + pool->dying = false; + + spin_lock_init(&pool->pool_lock); + INIT_LIST_HEAD(&pool->page_list); + + /* Register shrinker */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) + pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink; +#else + pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects; + pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects; +#endif + pool->reclaim.seeks = DEFAULT_SEEKS; + /* Kernel versions prior to 3.1 : + * struct shrinker does not define batch */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) + pool->reclaim.batch = 0; +#endif + register_shrinker(&pool->reclaim); + + pool_dbg(pool, "initialized\n"); + + return 0; +} + +void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool) +{ + kbase_mem_pool_lock(pool); + pool->dying = true; + kbase_mem_pool_unlock(pool); +} + +void kbase_mem_pool_term(struct kbase_mem_pool *pool) +{ + struct kbase_mem_pool *next_pool = pool->next_pool; + struct page *p, *tmp; + size_t nr_to_spill = 0; + LIST_HEAD(spill_list); + LIST_HEAD(free_list); + int i; + + pool_dbg(pool, "terminate()\n"); + + unregister_shrinker(&pool->reclaim); + + kbase_mem_pool_lock(pool); + pool->max_size = 0; + + if (next_pool && !kbase_mem_pool_is_full(next_pool)) { + /* Spill to next pool (may overspill) */ + nr_to_spill = kbase_mem_pool_capacity(next_pool); + nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill); + + /* Zero pages first without holding the next_pool lock */ + for (i = 0; i < nr_to_spill; i++) { + p = kbase_mem_pool_remove_locked(pool); + list_add(&p->lru, &spill_list); + } + } + + while (!kbase_mem_pool_is_empty(pool)) { + /* Free remaining pages to kernel */ + p = kbase_mem_pool_remove_locked(pool); + list_add(&p->lru, &free_list); + } + + kbase_mem_pool_unlock(pool); + + if (next_pool && nr_to_spill) { + list_for_each_entry(p, &spill_list, lru) + kbase_mem_pool_zero_page(pool, p); + + /* Add new page list to next_pool */ + kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill); + + pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill); + } + + list_for_each_entry_safe(p, tmp, &free_list, lru) { + list_del_init(&p->lru); + kbase_mem_pool_free_page(pool, p); + } + + pool_dbg(pool, "terminated\n"); +} + +struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool) +{ + struct page *p; + + do { + pool_dbg(pool, "alloc()\n"); + p = kbase_mem_pool_remove(pool); + + if (p) + return p; + + pool = pool->next_pool; + } while (pool); + + return NULL; +} + +struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool) +{ + struct page *p; + + lockdep_assert_held(&pool->pool_lock); + + pool_dbg(pool, "alloc_locked()\n"); + p = kbase_mem_pool_remove_locked(pool); + + if (p) + return p; + + return NULL; +} + +void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, + bool dirty) +{ + struct kbase_mem_pool *next_pool = pool->next_pool; + + pool_dbg(pool, "free()\n"); + + if (!kbase_mem_pool_is_full(pool)) { + /* Add to our own pool */ + if (dirty) + kbase_mem_pool_sync_page(pool, p); + + kbase_mem_pool_add(pool, p); + } else if (next_pool && !kbase_mem_pool_is_full(next_pool)) { + /* Spill to next pool */ + kbase_mem_pool_spill(next_pool, p); + } else { + /* Free page */ + kbase_mem_pool_free_page(pool, p); + } +} + +void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, + bool dirty) +{ + pool_dbg(pool, "free_locked()\n"); + + lockdep_assert_held(&pool->pool_lock); + + if (!kbase_mem_pool_is_full(pool)) { + /* Add to our own pool */ + if (dirty) + kbase_mem_pool_sync_page(pool, p); + + kbase_mem_pool_add_locked(pool, p); + } else { + /* Free page */ + kbase_mem_pool_free_page(pool, p); + } +} + +int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, + struct tagged_addr *pages, bool partial_allowed) +{ + struct page *p; + size_t nr_from_pool; + size_t i = 0; + int err = -ENOMEM; + size_t nr_pages_internal; + + nr_pages_internal = nr_4k_pages / (1u << (pool->order)); + + if (nr_pages_internal * (1u << pool->order) != nr_4k_pages) + return -EINVAL; + + pool_dbg(pool, "alloc_pages(4k=%zu):\n", nr_4k_pages); + pool_dbg(pool, "alloc_pages(internal=%zu):\n", nr_pages_internal); + + /* Get pages from this pool */ + kbase_mem_pool_lock(pool); + nr_from_pool = min(nr_pages_internal, kbase_mem_pool_size(pool)); + while (nr_from_pool--) { + int j; + p = kbase_mem_pool_remove_locked(pool); + if (pool->order) { + pages[i++] = as_tagged_tag(page_to_phys(p), + HUGE_HEAD | HUGE_PAGE); + for (j = 1; j < (1u << pool->order); j++) + pages[i++] = as_tagged_tag(page_to_phys(p) + + PAGE_SIZE * j, + HUGE_PAGE); + } else { + pages[i++] = as_tagged(page_to_phys(p)); + } + } + kbase_mem_pool_unlock(pool); + + if (i != nr_4k_pages && pool->next_pool) { + /* Allocate via next pool */ + err = kbase_mem_pool_alloc_pages(pool->next_pool, + nr_4k_pages - i, pages + i, partial_allowed); + + if (err < 0) + goto err_rollback; + + i += err; + } else { + /* Get any remaining pages from kernel */ + while (i != nr_4k_pages) { + p = kbase_mem_alloc_page(pool); + if (!p) { + if (partial_allowed) + goto done; + else + goto err_rollback; + } + + if (pool->order) { + int j; + + pages[i++] = as_tagged_tag(page_to_phys(p), + HUGE_PAGE | + HUGE_HEAD); + for (j = 1; j < (1u << pool->order); j++) { + phys_addr_t phys; + + phys = page_to_phys(p) + PAGE_SIZE * j; + pages[i++] = as_tagged_tag(phys, + HUGE_PAGE); + } + } else { + pages[i++] = as_tagged(page_to_phys(p)); + } + } + } + +done: + pool_dbg(pool, "alloc_pages(%zu) done\n", i); + return i; + +err_rollback: + kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED); + return err; +} + +int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, + size_t nr_4k_pages, struct tagged_addr *pages) +{ + struct page *p; + size_t i; + size_t nr_pages_internal; + + lockdep_assert_held(&pool->pool_lock); + + nr_pages_internal = nr_4k_pages / (1u << (pool->order)); + + if (nr_pages_internal * (1u << pool->order) != nr_4k_pages) + return -EINVAL; + + pool_dbg(pool, "alloc_pages_locked(4k=%zu):\n", nr_4k_pages); + pool_dbg(pool, "alloc_pages_locked(internal=%zu):\n", + nr_pages_internal); + + if (kbase_mem_pool_size(pool) < nr_pages_internal) { + pool_dbg(pool, "Failed alloc\n"); + return -ENOMEM; + } + + for (i = 0; i < nr_pages_internal; i++) { + int j; + + p = kbase_mem_pool_remove_locked(pool); + if (pool->order) { + *pages++ = as_tagged_tag(page_to_phys(p), + HUGE_HEAD | HUGE_PAGE); + for (j = 1; j < (1u << pool->order); j++) { + *pages++ = as_tagged_tag(page_to_phys(p) + + PAGE_SIZE * j, + HUGE_PAGE); + } + } else { + *pages++ = as_tagged(page_to_phys(p)); + } + } + + return nr_4k_pages; +} + +static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, + size_t nr_pages, struct tagged_addr *pages, + bool zero, bool sync) +{ + struct page *p; + size_t nr_to_pool = 0; + LIST_HEAD(new_page_list); + size_t i; + + if (!nr_pages) + return; + + pool_dbg(pool, "add_array(%zu, zero=%d, sync=%d):\n", + nr_pages, zero, sync); + + /* Zero/sync pages first without holding the pool lock */ + for (i = 0; i < nr_pages; i++) { + if (unlikely(!as_phys_addr_t(pages[i]))) + continue; + + if (is_huge_head(pages[i]) || !is_huge(pages[i])) { + p = as_page(pages[i]); + if (zero) + kbase_mem_pool_zero_page(pool, p); + else if (sync) + kbase_mem_pool_sync_page(pool, p); + + list_add(&p->lru, &new_page_list); + nr_to_pool++; + } + pages[i] = as_tagged(0); + } + + /* Add new page list to pool */ + kbase_mem_pool_add_list(pool, &new_page_list, nr_to_pool); + + pool_dbg(pool, "add_array(%zu) added %zu pages\n", + nr_pages, nr_to_pool); +} + +static void kbase_mem_pool_add_array_locked(struct kbase_mem_pool *pool, + size_t nr_pages, struct tagged_addr *pages, + bool zero, bool sync) +{ + struct page *p; + size_t nr_to_pool = 0; + LIST_HEAD(new_page_list); + size_t i; + + lockdep_assert_held(&pool->pool_lock); + + if (!nr_pages) + return; + + pool_dbg(pool, "add_array_locked(%zu, zero=%d, sync=%d):\n", + nr_pages, zero, sync); + + /* Zero/sync pages first */ + for (i = 0; i < nr_pages; i++) { + if (unlikely(!as_phys_addr_t(pages[i]))) + continue; + + if (is_huge_head(pages[i]) || !is_huge(pages[i])) { + p = as_page(pages[i]); + if (zero) + kbase_mem_pool_zero_page(pool, p); + else if (sync) + kbase_mem_pool_sync_page(pool, p); + + list_add(&p->lru, &new_page_list); + nr_to_pool++; + } + pages[i] = as_tagged(0); + } + + /* Add new page list to pool */ + kbase_mem_pool_add_list_locked(pool, &new_page_list, nr_to_pool); + + pool_dbg(pool, "add_array_locked(%zu) added %zu pages\n", + nr_pages, nr_to_pool); +} + +void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, + struct tagged_addr *pages, bool dirty, bool reclaimed) +{ + struct kbase_mem_pool *next_pool = pool->next_pool; + struct page *p; + size_t nr_to_pool; + LIST_HEAD(to_pool_list); + size_t i = 0; + + pool_dbg(pool, "free_pages(%zu):\n", nr_pages); + + if (!reclaimed) { + /* Add to this pool */ + nr_to_pool = kbase_mem_pool_capacity(pool); + nr_to_pool = min(nr_pages, nr_to_pool); + + kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty); + + i += nr_to_pool; + + if (i != nr_pages && next_pool) { + /* Spill to next pool (may overspill) */ + nr_to_pool = kbase_mem_pool_capacity(next_pool); + nr_to_pool = min(nr_pages - i, nr_to_pool); + + kbase_mem_pool_add_array(next_pool, nr_to_pool, + pages + i, true, dirty); + i += nr_to_pool; + } + } + + /* Free any remaining pages to kernel */ + for (; i < nr_pages; i++) { + if (unlikely(!as_phys_addr_t(pages[i]))) + continue; + + if (is_huge(pages[i]) && !is_huge_head(pages[i])) { + pages[i] = as_tagged(0); + continue; + } + + p = as_page(pages[i]); + + kbase_mem_pool_free_page(pool, p); + pages[i] = as_tagged(0); + } + + pool_dbg(pool, "free_pages(%zu) done\n", nr_pages); +} + + +void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, + size_t nr_pages, struct tagged_addr *pages, bool dirty, + bool reclaimed) +{ + struct page *p; + size_t nr_to_pool; + LIST_HEAD(to_pool_list); + size_t i = 0; + + lockdep_assert_held(&pool->pool_lock); + + pool_dbg(pool, "free_pages_locked(%zu):\n", nr_pages); + + if (!reclaimed) { + /* Add to this pool */ + nr_to_pool = kbase_mem_pool_capacity(pool); + nr_to_pool = min(nr_pages, nr_to_pool); + + kbase_mem_pool_add_array_locked(pool, nr_pages, pages, false, + dirty); + + i += nr_to_pool; + } + + /* Free any remaining pages to kernel */ + for (; i < nr_pages; i++) { + if (unlikely(!as_phys_addr_t(pages[i]))) + continue; + + if (is_huge(pages[i]) && !is_huge_head(pages[i])) { + pages[i] = as_tagged(0); + continue; + } + + p = as_page(pages[i]); + + kbase_mem_pool_free_page(pool, p); + pages[i] = as_tagged(0); + } + + pool_dbg(pool, "free_pages_locked(%zu) done\n", nr_pages); +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.c new file mode 100755 index 000000000000..5879fdf85b1d --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.c @@ -0,0 +1,191 @@ +/* + * + * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include + +#include "mali_kbase_mem_pool_debugfs.h" +#include "mali_kbase_debugfs_helper.h" + +void kbase_mem_pool_debugfs_trim(void *const array, size_t const index, + size_t const value) +{ + struct kbase_mem_pool *const mem_pools = array; + + if (WARN_ON(!mem_pools) || + WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + return; + + kbase_mem_pool_trim(&mem_pools[index], value); +} + +void kbase_mem_pool_debugfs_set_max_size(void *const array, + size_t const index, size_t const value) +{ + struct kbase_mem_pool *const mem_pools = array; + + if (WARN_ON(!mem_pools) || + WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + return; + + kbase_mem_pool_set_max_size(&mem_pools[index], value); +} + +size_t kbase_mem_pool_debugfs_size(void *const array, size_t const index) +{ + struct kbase_mem_pool *const mem_pools = array; + + if (WARN_ON(!mem_pools) || + WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + return 0; + + return kbase_mem_pool_size(&mem_pools[index]); +} + +size_t kbase_mem_pool_debugfs_max_size(void *const array, size_t const index) +{ + struct kbase_mem_pool *const mem_pools = array; + + if (WARN_ON(!mem_pools) || + WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + return 0; + + return kbase_mem_pool_max_size(&mem_pools[index]); +} + +void kbase_mem_pool_config_debugfs_set_max_size(void *const array, + size_t const index, size_t const value) +{ + struct kbase_mem_pool_config *const configs = array; + + if (WARN_ON(!configs) || + WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + return; + + kbase_mem_pool_config_set_max_size(&configs[index], value); +} + +size_t kbase_mem_pool_config_debugfs_max_size(void *const array, + size_t const index) +{ + struct kbase_mem_pool_config *const configs = array; + + if (WARN_ON(!configs) || + WARN_ON(index >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + return 0; + + return kbase_mem_pool_config_get_max_size(&configs[index]); +} + +static int kbase_mem_pool_debugfs_size_show(struct seq_file *sfile, void *data) +{ + CSTD_UNUSED(data); + return kbase_debugfs_helper_seq_read(sfile, + MEMORY_GROUP_MANAGER_NR_GROUPS, kbase_mem_pool_debugfs_size); +} + +static ssize_t kbase_mem_pool_debugfs_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *ppos) +{ + int err; + + CSTD_UNUSED(ppos); + err = kbase_debugfs_helper_seq_write(file, ubuf, count, + MEMORY_GROUP_MANAGER_NR_GROUPS, kbase_mem_pool_debugfs_trim); + return err ? err : count; +} + +static int kbase_mem_pool_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbase_mem_pool_debugfs_size_show, + in->i_private); +} + +static const struct file_operations kbase_mem_pool_debugfs_fops = { + .owner = THIS_MODULE, + .open = kbase_mem_pool_debugfs_open, + .read = seq_read, + .write = kbase_mem_pool_debugfs_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static int kbase_mem_pool_debugfs_max_size_show(struct seq_file *sfile, + void *data) +{ + CSTD_UNUSED(data); + return kbase_debugfs_helper_seq_read(sfile, + MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_debugfs_max_size); +} + +static ssize_t kbase_mem_pool_debugfs_max_size_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *ppos) +{ + int err; + + CSTD_UNUSED(ppos); + err = kbase_debugfs_helper_seq_write(file, ubuf, count, + MEMORY_GROUP_MANAGER_NR_GROUPS, + kbase_mem_pool_debugfs_set_max_size); + return err ? err : count; +} + +static int kbase_mem_pool_debugfs_max_size_open(struct inode *in, + struct file *file) +{ + return single_open(file, kbase_mem_pool_debugfs_max_size_show, + in->i_private); +} + +static const struct file_operations kbase_mem_pool_debugfs_max_size_fops = { + .owner = THIS_MODULE, + .open = kbase_mem_pool_debugfs_max_size_open, + .read = seq_read, + .write = kbase_mem_pool_debugfs_max_size_write, + .llseek = seq_lseek, + .release = single_release, +}; + +void kbase_mem_pool_debugfs_init(struct dentry *parent, + struct kbase_context *kctx) +{ + /* prevent unprivileged use of debug file in old kernel version */ +#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) + /* only for newer kernel version debug file system is safe */ + const mode_t mode = 0644; +#else + const mode_t mode = 0600; +#endif + + debugfs_create_file("mem_pool_size", mode, parent, + &kctx->mem_pools.small, &kbase_mem_pool_debugfs_fops); + + debugfs_create_file("mem_pool_max_size", mode, parent, + &kctx->mem_pools.small, &kbase_mem_pool_debugfs_max_size_fops); + + debugfs_create_file("lp_mem_pool_size", mode, parent, + &kctx->mem_pools.large, &kbase_mem_pool_debugfs_fops); + + debugfs_create_file("lp_mem_pool_max_size", mode, parent, + &kctx->mem_pools.large, &kbase_mem_pool_debugfs_max_size_fops); +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.h new file mode 100755 index 000000000000..2932945b3185 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_debugfs.h @@ -0,0 +1,123 @@ +/* + * + * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_MEM_POOL_DEBUGFS_H_ +#define _KBASE_MEM_POOL_DEBUGFS_H_ + +#include + +/** + * kbase_mem_pool_debugfs_init - add debugfs knobs for @pool + * @parent: Parent debugfs dentry + * @kctx: The kbase context + * + * Adds four debugfs files under @parent: + * - mem_pool_size: get/set the current sizes of @kctx: mem_pools + * - mem_pool_max_size: get/set the max sizes of @kctx: mem_pools + * - lp_mem_pool_size: get/set the current sizes of @kctx: lp_mem_pool + * - lp_mem_pool_max_size: get/set the max sizes of @kctx:lp_mem_pool + */ +void kbase_mem_pool_debugfs_init(struct dentry *parent, + struct kbase_context *kctx); + +/** + * kbase_mem_pool_debugfs_trim - Grow or shrink a memory pool to a new size + * + * @array: Address of the first in an array of physical memory pools. + * @index: A memory group ID to be used as an index into the array of memory + * pools. Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @value: New number of pages in the pool. + * + * If @value > current size, fill the pool with new pages from the kernel, but + * not above the max_size for the pool. + * If @value < current size, shrink the pool by freeing pages to the kernel. + */ +void kbase_mem_pool_debugfs_trim(void *array, size_t index, size_t value); + +/** + * kbase_mem_pool_debugfs_set_max_size - Set maximum number of free pages in + * memory pool + * + * @array: Address of the first in an array of physical memory pools. + * @index: A memory group ID to be used as an index into the array of memory + * pools. Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @value: Maximum number of free pages the pool can hold. + * + * If the maximum size is reduced, the pool will be shrunk to adhere to the + * new limit. For details see kbase_mem_pool_shrink(). + */ +void kbase_mem_pool_debugfs_set_max_size(void *array, size_t index, + size_t value); + +/** + * kbase_mem_pool_debugfs_size - Get number of free pages in a memory pool + * + * @array: Address of the first in an array of physical memory pools. + * @index: A memory group ID to be used as an index into the array of memory + * pools. Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * + * Note: the size of the pool may in certain corner cases exceed @max_size! + * + * Return: Number of free pages in the pool + */ +size_t kbase_mem_pool_debugfs_size(void *array, size_t index); + +/** + * kbase_mem_pool_debugfs_max_size - Get maximum number of free pages in a + * memory pool + * + * @array: Address of the first in an array of physical memory pools. + * @index: A memory group ID to be used as an index into the array of memory + * pools. Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * + * Return: Maximum number of free pages in the pool + */ +size_t kbase_mem_pool_debugfs_max_size(void *array, size_t index); + +/** + * kbase_mem_pool_config_debugfs_set_max_size - Set maximum number of free pages + * in initial configuration of pool + * + * @array: Array of initial configurations for a set of physical memory pools. + * @index: A memory group ID to be used as an index into the array. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @value : Maximum number of free pages that a memory pool created from the + * selected configuration can hold. + */ +void kbase_mem_pool_config_debugfs_set_max_size(void *array, size_t index, + size_t value); + +/** + * kbase_mem_pool_config_debugfs_max_size - Get maximum number of free pages + * from initial configuration of pool + * + * @array: Array of initial configurations for a set of physical memory pools. + * @index: A memory group ID to be used as an index into the array. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * + * Return: Maximum number of free pages that a memory pool created from the + * selected configuration can hold. + */ +size_t kbase_mem_pool_config_debugfs_max_size(void *array, size_t index); + +#endif /*_KBASE_MEM_POOL_DEBUGFS_H_ */ + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c new file mode 100755 index 000000000000..aa2554805b5b --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c @@ -0,0 +1,115 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include + +#include + +void kbase_mem_pool_group_config_set_max_size( + struct kbase_mem_pool_group_config *const configs, + size_t const max_size) +{ + size_t const large_max_size = max_size >> + (KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER - + KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER); + int gid; + + for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { + kbase_mem_pool_config_set_max_size(&configs->small[gid], + max_size); + + kbase_mem_pool_config_set_max_size(&configs->large[gid], + large_max_size); + } +} + +int kbase_mem_pool_group_init( + struct kbase_mem_pool_group *const mem_pools, + struct kbase_device *const kbdev, + const struct kbase_mem_pool_group_config *const configs, + struct kbase_mem_pool_group *next_pools) +{ + int gid, err = 0; + + for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { + err = kbase_mem_pool_init(&mem_pools->small[gid], + &configs->small[gid], + KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER, + gid, + kbdev, + next_pools ? &next_pools->small[gid] : NULL); + + if (!err) { + err = kbase_mem_pool_init(&mem_pools->large[gid], + &configs->large[gid], + KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, + gid, + kbdev, + next_pools ? &next_pools->large[gid] : NULL); + if (err) + kbase_mem_pool_term(&mem_pools->small[gid]); + } + + /* Break out of the loop early to avoid incrementing the count + * of memory pool pairs successfully initialized. + */ + if (err) + break; + } + + if (err) { + /* gid gives the number of memory pool pairs successfully + * initialized, which is one greater than the array index of the + * last group. + */ + while (gid-- > 0) { + kbase_mem_pool_term(&mem_pools->small[gid]); + kbase_mem_pool_term(&mem_pools->large[gid]); + } + } + + return err; +} + +void kbase_mem_pool_group_mark_dying( + struct kbase_mem_pool_group *const mem_pools) +{ + int gid; + + for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { + kbase_mem_pool_mark_dying(&mem_pools->small[gid]); + kbase_mem_pool_mark_dying(&mem_pools->large[gid]); + } +} + +void kbase_mem_pool_group_term( + struct kbase_mem_pool_group *const mem_pools) +{ + int gid; + + for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { + kbase_mem_pool_term(&mem_pools->small[gid]); + kbase_mem_pool_term(&mem_pools->large[gid]); + } +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h new file mode 100755 index 000000000000..0484f5940ad1 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h @@ -0,0 +1,92 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_MEM_POOL_GROUP_H_ +#define _KBASE_MEM_POOL_GROUP_H_ + +#include + +/** + * kbase_mem_pool_group_config_init - Set the initial configuration for a + * set of memory pools + * + * This function sets the initial configuration for every memory pool so that + * the maximum amount of free memory that each pool can hold is identical. + * The equivalent number of 2 MiB pages is calculated automatically for the + * purpose of configuring the large page pools. + * + * @configs: Initial configuration for the set of memory pools + * @max_size: Maximum number of free 4 KiB pages each pool can hold + */ +void kbase_mem_pool_group_config_set_max_size( + struct kbase_mem_pool_group_config *configs, size_t max_size); + +/** + * kbase_mem_pool_group_init - Initialize a set of memory pools + * + * Initializes a complete set of physical memory pools. Memory pools are used to + * allow efficient reallocation of previously-freed physical pages. A pair of + * memory pools is initialized for each physical memory group: one for 4 KiB + * pages and one for 2 MiB pages. + * + * If @next_pools is not NULL then a request to allocate memory from an + * empty pool in @mem_pools will attempt to allocate from the equivalent pool + * in @next_pools before going to the memory group manager. Similarly + * pages can spill over to the equivalent pool in @next_pools when a pool + * is full in @mem_pools. Pages are zeroed before they spill over to another + * pool, to prevent leaking information between applications. + * + * @mem_pools: Set of memory pools to initialize + * @kbdev: Kbase device where memory is used + * @configs: Initial configuration for the set of memory pools + * @next_pools: Set of memory pools from which to allocate memory if there + * is no free memory in one of the @mem_pools + * + * Return: 0 on success, otherwise a negative error code + */ +int kbase_mem_pool_group_init(struct kbase_mem_pool_group *mem_pools, + struct kbase_device *kbdev, + const struct kbase_mem_pool_group_config *configs, + struct kbase_mem_pool_group *next_pools); + +/** + * kbase_mem_pool_group_term - Mark a set of memory pools as dying + * + * Marks a complete set of physical memory pools previously initialized by + * @kbase_mem_pool_group_init as dying. This will cause any ongoing allocation + * operations (eg growing on page fault) to be terminated. + * + * @mem_pools: Set of memory pools to mark + */ +void kbase_mem_pool_group_mark_dying(struct kbase_mem_pool_group *mem_pools); + +/** + * kbase_mem_pool_group_term - Terminate a set of memory pools + * + * Terminates a complete set of physical memory pools previously initialized by + * @kbase_mem_pool_group_init. + * + * @mem_pools: Set of memory pools to terminate + */ +void kbase_mem_pool_group_term(struct kbase_mem_pool_group *mem_pools); + +#endif /* _KBASE_MEM_POOL_GROUP_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.c new file mode 100755 index 000000000000..85723f825054 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.c @@ -0,0 +1,134 @@ +/* + * + * (C) COPYRIGHT 2012-2017, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include + +#ifdef CONFIG_DEBUG_FS + +/** Show callback for the @c mem_profile debugfs file. + * + * This function is called to get the contents of the @c mem_profile debugfs + * file. This is a report of current memory usage and distribution in userspace. + * + * @param sfile The debugfs entry + * @param data Data associated with the entry + * + * @return 0 if it successfully prints data in debugfs entry file, non-zero otherwise + */ +static int kbasep_mem_profile_seq_show(struct seq_file *sfile, void *data) +{ + struct kbase_context *kctx = sfile->private; + + mutex_lock(&kctx->mem_profile_lock); + + seq_write(sfile, kctx->mem_profile_data, kctx->mem_profile_size); + + seq_putc(sfile, '\n'); + + mutex_unlock(&kctx->mem_profile_lock); + + return 0; +} + +/* + * File operations related to debugfs entry for mem_profile + */ +static int kbasep_mem_profile_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_mem_profile_seq_show, in->i_private); +} + +static const struct file_operations kbasep_mem_profile_debugfs_fops = { + .owner = THIS_MODULE, + .open = kbasep_mem_profile_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, + size_t size) +{ +#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) + const mode_t mode = 0444; +#else + const mode_t mode = 0400; +#endif + int err = 0; + + mutex_lock(&kctx->mem_profile_lock); + + dev_dbg(kctx->kbdev->dev, "initialised: %d", + kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); + + if (!kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { + if (IS_ERR_OR_NULL(kctx->kctx_dentry)) { + err = -ENOMEM; + } else if (!debugfs_create_file("mem_profile", mode, + kctx->kctx_dentry, kctx, + &kbasep_mem_profile_debugfs_fops)) { + err = -EAGAIN; + } else { + kbase_ctx_flag_set(kctx, + KCTX_MEM_PROFILE_INITIALIZED); + } + } + + if (kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { + kfree(kctx->mem_profile_data); + kctx->mem_profile_data = data; + kctx->mem_profile_size = size; + } else { + kfree(data); + } + + dev_dbg(kctx->kbdev->dev, "returning: %d, initialised: %d", + err, kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); + + mutex_unlock(&kctx->mem_profile_lock); + + return err; +} + +void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx) +{ + mutex_lock(&kctx->mem_profile_lock); + + dev_dbg(kctx->kbdev->dev, "initialised: %d", + kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); + + kfree(kctx->mem_profile_data); + kctx->mem_profile_data = NULL; + kctx->mem_profile_size = 0; + + mutex_unlock(&kctx->mem_profile_lock); +} + +#else /* CONFIG_DEBUG_FS */ + +int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, + size_t size) +{ + kfree(data); + return 0; +} +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h new file mode 100755 index 000000000000..1462247c3bca --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h @@ -0,0 +1,64 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_mem_profile_debugfs.h + * Header file for mem profiles entries in debugfs + * + */ + +#ifndef _KBASE_MEM_PROFILE_DEBUGFS_H +#define _KBASE_MEM_PROFILE_DEBUGFS_H + +#include +#include + +/** + * @brief Remove entry from Mali memory profile debugfs + */ +void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx); + +/** + * @brief Insert @p data to the debugfs file so it can be read by userspace + * + * The function takes ownership of @p data and frees it later when new data + * is inserted. + * + * If the debugfs entry corresponding to the @p kctx doesn't exist, + * an attempt will be made to create it. + * + * @param kctx The context whose debugfs file @p data should be inserted to + * @param data A NULL-terminated string to be inserted to the debugfs file, + * without the trailing new line character + * @param size The length of the @p data string + * @return 0 if @p data inserted correctly + * -EAGAIN in case of error + * @post @ref mem_profile_initialized will be set to @c true + * the first time this function succeeds. + */ +int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, + size_t size); + +#endif /*_KBASE_MEM_PROFILE_DEBUGFS_H*/ + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h new file mode 100755 index 000000000000..d55cc854c415 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2014, 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file mali_kbase_mem_profile_debugfs_buf_size.h + * Header file for the size of the buffer to accumulate the histogram report text in + */ + +#ifndef _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ +#define _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ + +/** + * The size of the buffer to accumulate the histogram report text in + * @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT + */ +#define KBASE_MEM_PROFILE_MAX_BUF_SIZE \ + ((size_t) (64 + ((80 + (56 * 64)) * 53) + 56)) + +#endif /*_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/ + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mipe_gen_header.h b/drivers/gpu/arm/bifrost/mali_kbase_mipe_gen_header.h new file mode 100755 index 000000000000..72acadfae993 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mipe_gen_header.h @@ -0,0 +1,219 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. + * DO NOT EDIT. + */ + +/* clang-format off */ + +#include "mali_kbase_mipe_proto.h" + +/** + * This header generates MIPE tracepoint declaration BLOB at + * compile time. + * + * It is intentional that there is no header guard. + * The header could be included multiple times for + * different blobs compilation. + * + * Before including this header MIPE_HEADER_* parameters must be + * defined. See documentation below: + */ + +/** + * The name of the variable where the result BLOB will be stored. + */ +#if !defined(MIPE_HEADER_BLOB_VAR_NAME) +#error "MIPE_HEADER_BLOB_VAR_NAME must be defined!" +#endif + +/** + * A compiler attribute for the BLOB variable. + * + * e.g. __attribute__((section("my_section"))) + * + * Default value is no attribute. + */ +#if !defined(MIPE_HEADER_BLOB_VAR_ATTRIBUTE) +#define MIPE_HEADER_BLOB_VAR_ATTRIBUTE +#endif + +/** + * MIPE stream id. + * + * See enum tl_stream_id. + */ +#if !defined(MIPE_HEADER_STREAM_ID) +#error "MIPE_HEADER_STREAM_ID must be defined!" +#endif + +/** + * MIPE packet class. + * + * See enum tl_packet_class. + */ +#if !defined(MIPE_HEADER_PKT_CLASS) +#error "MIPE_HEADER_PKT_CLASS must be defined!" +#endif + +/** + * The list of tracepoints to process. + * + * It should be defined as follows: + * + * #define MIPE_HEADER_TRACEPOINT_LIST \ + * TRACEPOINT_DESC(FIRST_TRACEPOINT, "Some description", "@II", "first_arg,second_arg") \ + * TRACEPOINT_DESC(SECOND_TRACEPOINT, "Some description", "@II", "first_arg,second_arg") \ + * etc. + * + * Where the first argument is tracepoints name, the second + * argument is a short tracepoint description, the third argument + * argument types (see MIPE documentation), and the fourth argument + * is comma separated argument names. + */ +#if !defined(MIPE_HEADER_TRACEPOINT_LIST) +#error "MIPE_HEADER_TRACEPOINT_LIST must be defined!" +#endif + +/** + * The number of entries in MIPE_HEADER_TRACEPOINT_LIST. + */ +#if !defined(MIPE_HEADER_TRACEPOINT_LIST_SIZE) +#error "MIPE_HEADER_TRACEPOINT_LIST_SIZE must be defined!" +#endif + +/** + * The list of enums to process. + * + * It should be defined as follows: + * + * #define MIPE_HEADER_ENUM_LIST \ + * ENUM_DESC(enum_arg_name, enum_value) \ + * ENUM_DESC(enum_arg_name, enum_value) \ + * etc. + * + * Where enum_arg_name is the name of a tracepoint argument being used with + * this enum. enum_value is a valid C enum value. + * + * Default value is an empty list. + */ +#if defined(MIPE_HEADER_ENUM_LIST) + +/** + * Tracepoint message ID used for enums declaration. + */ +#if !defined(MIPE_HEADER_ENUM_MSG_ID) +#error "MIPE_HEADER_ENUM_MSG_ID must be defined!" +#endif + +#else +#define MIPE_HEADER_ENUM_LIST +#endif + +/** + * The MIPE tracepoint declaration BLOB. + */ +const struct +{ + u32 _mipe_w0; + u32 _mipe_w1; + u8 _protocol_version; + u8 _pointer_size; + u32 _tp_count; +#define TRACEPOINT_DESC(name, desc, arg_types, arg_names) \ + struct { \ + u32 _name; \ + u32 _size_string_name; \ + char _string_name[sizeof(#name)]; \ + u32 _size_desc; \ + char _desc[sizeof(desc)]; \ + u32 _size_arg_types; \ + char _arg_types[sizeof(arg_types)]; \ + u32 _size_arg_names; \ + char _arg_names[sizeof(arg_names)]; \ + } __attribute__ ((__packed__)) __ ## name; + +#define ENUM_DESC(arg_name, value) \ + struct { \ + u32 _msg_id; \ + u32 _arg_name_len; \ + char _arg_name[sizeof(#arg_name)]; \ + u32 _value; \ + u32 _value_str_len; \ + char _value_str[sizeof(#value)]; \ + } __attribute__ ((__packed__)) __ ## arg_name ## _ ## value; + + MIPE_HEADER_TRACEPOINT_LIST + MIPE_HEADER_ENUM_LIST +#undef TRACEPOINT_DESC +#undef ENUM_DESC +} __attribute__((packed)) MIPE_HEADER_BLOB_VAR_NAME MIPE_HEADER_BLOB_VAR_ATTRIBUTE = { + ._mipe_w0 = MIPE_PACKET_HEADER_W0( + TL_PACKET_FAMILY_TL, + MIPE_HEADER_PKT_CLASS, + TL_PACKET_TYPE_HEADER, + MIPE_HEADER_STREAM_ID), + ._mipe_w1 = MIPE_PACKET_HEADER_W1( + sizeof(MIPE_HEADER_BLOB_VAR_NAME) - PACKET_HEADER_SIZE, + 0), + ._protocol_version = SWTRACE_VERSION, + ._pointer_size = sizeof(void *), + ._tp_count = MIPE_HEADER_TRACEPOINT_LIST_SIZE, +#define TRACEPOINT_DESC(name, desc, arg_types, arg_names) \ + .__ ## name = { \ + ._name = name, \ + ._size_string_name = sizeof(#name), \ + ._string_name = #name, \ + ._size_desc = sizeof(desc), \ + ._desc = desc, \ + ._size_arg_types = sizeof(arg_types), \ + ._arg_types = arg_types, \ + ._size_arg_names = sizeof(arg_names), \ + ._arg_names = arg_names \ + }, +#define ENUM_DESC(arg_name, value) \ + .__ ## arg_name ## _ ## value = { \ + ._msg_id = MIPE_HEADER_ENUM_MSG_ID, \ + ._arg_name_len = sizeof(#arg_name), \ + ._arg_name = #arg_name, \ + ._value = value, \ + ._value_str_len = sizeof(#value), \ + ._value_str = #value \ + }, + + MIPE_HEADER_TRACEPOINT_LIST + MIPE_HEADER_ENUM_LIST +#undef TRACEPOINT_DESC +#undef ENUM_DESC +}; + +#undef MIPE_HEADER_BLOB_VAR_NAME +#undef MIPE_HEADER_BLOB_VAR_ATTRIBUTE +#undef MIPE_HEADER_STREAM_ID +#undef MIPE_HEADER_PKT_CLASS +#undef MIPE_HEADER_TRACEPOINT_LIST +#undef MIPE_HEADER_TRACEPOINT_LIST_SIZE +#undef MIPE_HEADER_ENUM_LIST +#undef MIPE_HEADER_ENUM_MSG_ID + +/* clang-format on */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mipe_proto.h b/drivers/gpu/arm/bifrost/mali_kbase_mipe_proto.h new file mode 100755 index 000000000000..54667cfc6304 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_mipe_proto.h @@ -0,0 +1,127 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. + * DO NOT EDIT. + */ + +/* clang-format off */ + +#if !defined(_KBASE_MIPE_PROTO_H) +#define _KBASE_MIPE_PROTO_H + +#define _BITFIELD_MASK_FIELD(pos, len) \ + (((1u << len) - 1) << pos) + +#define _BITFIELD_SET_FIELD(pos, len, value) \ + (_BITFIELD_MASK_FIELD(pos, len) & (((u32) value) << pos)) + +#define BITFIELD_SET(field_name, value) \ + _BITFIELD_SET_FIELD(field_name ## _POS, field_name ## _LEN, value) + +/* The version of swtrace protocol used in timeline stream. */ +#define SWTRACE_VERSION 3 + +/* Packet header - first word. + * These values must be defined according to MIPE documentation. + */ +#define PACKET_STREAMID_POS 0 +#define PACKET_STREAMID_LEN 8 +#define PACKET_RSVD1_POS (PACKET_STREAMID_POS + PACKET_STREAMID_LEN) +#define PACKET_RSVD1_LEN 8 +#define PACKET_TYPE_POS (PACKET_RSVD1_POS + PACKET_RSVD1_LEN) +#define PACKET_TYPE_LEN 3 +#define PACKET_CLASS_POS (PACKET_TYPE_POS + PACKET_TYPE_LEN) +#define PACKET_CLASS_LEN 7 +#define PACKET_FAMILY_POS (PACKET_CLASS_POS + PACKET_CLASS_LEN) +#define PACKET_FAMILY_LEN 6 + +/* Packet header - second word + * These values must be defined according to MIPE documentation. + */ +#define PACKET_LENGTH_POS 0 +#define PACKET_LENGTH_LEN 24 +#define PACKET_SEQBIT_POS (PACKET_LENGTH_POS + PACKET_LENGTH_LEN) +#define PACKET_SEQBIT_LEN 1 +#define PACKET_RSVD2_POS (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN) +#define PACKET_RSVD2_LEN 7 + +/* First word of a MIPE packet */ +#define MIPE_PACKET_HEADER_W0(pkt_family, pkt_class, pkt_type, stream_id) \ + (0 \ + | BITFIELD_SET(PACKET_FAMILY, pkt_family) \ + | BITFIELD_SET(PACKET_CLASS, pkt_class) \ + | BITFIELD_SET(PACKET_TYPE, pkt_type) \ + | BITFIELD_SET(PACKET_STREAMID, stream_id)) + +/* Second word of a MIPE packet */ +#define MIPE_PACKET_HEADER_W1(packet_length, seqbit) \ + (0 \ + | BITFIELD_SET(PACKET_LENGTH, packet_length) \ + | BITFIELD_SET(PACKET_SEQBIT, seqbit)) + +/* The number of bytes reserved for packet header. + * These value must be defined according to MIPE documentation. + */ +#define PACKET_HEADER_SIZE 8 /* bytes */ + +/* The number of bytes reserved for packet sequence number. + * These value must be defined according to MIPE documentation. + */ +#define PACKET_NUMBER_SIZE 4 /* bytes */ + +/* Timeline packet family ids. + * Values are significant! Check MIPE documentation. + */ +enum tl_packet_family { + TL_PACKET_FAMILY_CTRL = 0, /* control packets */ + TL_PACKET_FAMILY_TL = 1, /* timeline packets */ + TL_PACKET_FAMILY_COUNT +}; + +/* Packet classes used in timeline streams. + * Values are significant! Check MIPE documentation. + */ +enum tl_packet_class { + TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */ + TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */ +}; + +/* Packet types used in timeline streams. + * Values are significant! Check MIPE documentation. + */ +enum tl_packet_type { + TL_PACKET_TYPE_HEADER = 0, /* stream's header/directory */ + TL_PACKET_TYPE_BODY = 1, /* stream's body */ + TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */ +}; + +/* Stream ID types (timeline family). */ +enum tl_stream_id { + TL_STREAM_ID_USER = 0, /* User-space driver Timeline stream. */ + TL_STREAM_ID_KERNEL = 1, /* Kernel-space driver Timeline stream. */ + TL_STREAM_ID_CSFFW = 2, /* CSF firmware driver Timeline stream. */ +}; + +#endif /* _KBASE_MIPE_PROTO_H */ + +/* clang-format on */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c new file mode 100755 index 000000000000..38ae46e0ddf1 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c @@ -0,0 +1,153 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include + +#include +#include + +/** + * kbase_native_mgm_alloc - Native physical memory allocation method + * + * @mgm_dev: The memory group manager the request is being made through. + * @group_id: A physical memory group ID, which must be valid but is not used. + * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. + * @gfp_mask: Bitmask of Get Free Page flags affecting allocator behavior. + * @order: Page order for physical page size (order=0 means 4 KiB, + * order=9 means 2 MiB). + * + * Delegates all memory allocation requests to the kernel's alloc_pages + * function. + * + * Return: Pointer to allocated page, or NULL if allocation failed. + */ +static struct page *kbase_native_mgm_alloc( + struct memory_group_manager_device *mgm_dev, int group_id, + gfp_t gfp_mask, unsigned int order) +{ + /* + * Check that the base and the mgm defines, from separate header files, + * for the max number of memory groups are compatible. + */ + BUILD_BUG_ON(BASE_MEM_GROUP_COUNT != MEMORY_GROUP_MANAGER_NR_GROUPS); + /* + * Check that the mask used for storing the memory group ID is big + * enough for the largest possible memory group ID. + */ + BUILD_BUG_ON((BASEP_CONTEXT_MMU_GROUP_ID_MASK + >> BASEP_CONTEXT_MMU_GROUP_ID_SHIFT) + < (BASE_MEM_GROUP_COUNT - 1)); + + CSTD_UNUSED(mgm_dev); + CSTD_UNUSED(group_id); + + return alloc_pages(gfp_mask, order); +} + +/** + * kbase_native_mgm_free - Native physical memory freeing method + * + * @mgm_dev: The memory group manager the request is being made through. + * @group_id: A physical memory group ID, which must be valid but is not used. + * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. + * @page: Address of the struct associated with a page of physical + * memory that was allocated by calling kbase_native_mgm_alloc + * with the same argument values. + * @order: Page order for physical page size (order=0 means 4 KiB, + * order=9 means 2 MiB). + * + * Delegates all memory freeing requests to the kernel's __free_pages function. + */ +static void kbase_native_mgm_free(struct memory_group_manager_device *mgm_dev, + int group_id, struct page *page, unsigned int order) +{ + CSTD_UNUSED(mgm_dev); + CSTD_UNUSED(group_id); + + __free_pages(page, order); +} + +/** + * kbase_native_mgm_vmf_insert_pfn_prot - Native method to map a page on the CPU + * + * @mgm_dev: The memory group manager the request is being made through. + * @group_id: A physical memory group ID, which must be valid but is not used. + * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. + * @vma: The virtual memory area to insert the page into. + * @addr: An address contained in @vma to assign to the inserted page. + * @pfn: The kernel Page Frame Number to insert at @addr in @vma. + * @pgprot: Protection flags for the inserted page. + * + * Called from a CPU virtual memory page fault handler. Delegates all memory + * mapping requests to the kernel's vmf_insert_pfn_prot function. + * + * Return: Type of fault that occurred or VM_FAULT_NOPAGE if the page table + * entry was successfully installed. + */ +static vm_fault_t kbase_native_mgm_vmf_insert_pfn_prot( + struct memory_group_manager_device *mgm_dev, int group_id, + struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, pgprot_t pgprot) +{ + CSTD_UNUSED(mgm_dev); + CSTD_UNUSED(group_id); + + return vmf_insert_pfn_prot(vma, addr, pfn, pgprot); +} + +/** + * kbase_native_mgm_update_gpu_pte - Native method to modify a GPU page table + * entry + * + * @mgm_dev: The memory group manager the request is being made through. + * @group_id: A physical memory group ID, which must be valid but is not used. + * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. + * @mmu_level: The level of the MMU page table where the page is getting mapped. + * @pte: The prepared page table entry. + * + * This function simply returns the @pte without modification. + * + * Return: A GPU page table entry to be stored in a page table. + */ +static u64 +kbase_native_mgm_update_gpu_pte(struct memory_group_manager_device *mgm_dev, + int group_id, int mmu_level, u64 pte) +{ + CSTD_UNUSED(mgm_dev); + CSTD_UNUSED(group_id); + CSTD_UNUSED(mmu_level); + + return pte; +} + +struct memory_group_manager_device kbase_native_mgm_dev = { + .ops = { + .mgm_alloc_page = kbase_native_mgm_alloc, + .mgm_free_page = kbase_native_mgm_free, + .mgm_get_import_memory_id = NULL, + .mgm_vmf_insert_pfn_prot = kbase_native_mgm_vmf_insert_pfn_prot, + .mgm_update_gpu_pte = kbase_native_mgm_update_gpu_pte, + }, + .data = NULL +}; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.h b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.h new file mode 100755 index 000000000000..431b1f4cb5db --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.h @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_NATIVE_MGM_H_ +#define _KBASE_NATIVE_MGM_H_ + +#include + +/** + * kbase_native_mgm_dev - Native memory group manager device + * + * An implementation of the memory group manager interface that is intended for + * internal use when no platform-specific memory group manager is available. + * + * It ignores the specified group ID and delegates to the kernel's physical + * memory allocation and freeing functions. + */ +extern struct memory_group_manager_device kbase_native_mgm_dev; + +#endif /* _KBASE_NATIVE_MGM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_platform_fake.c b/drivers/gpu/arm/bifrost/mali_kbase_platform_fake.c new file mode 100755 index 000000000000..fbb090e6c21f --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_platform_fake.c @@ -0,0 +1,124 @@ +/* + * + * (C) COPYRIGHT 2011-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include + + +/* + * This file is included only for type definitions and functions belonging to + * specific platform folders. Do not add dependencies with symbols that are + * defined somewhere else. + */ +#include + +#define PLATFORM_CONFIG_RESOURCE_COUNT 4 +#define PLATFORM_CONFIG_IRQ_RES_COUNT 3 + +static struct platform_device *mali_device; + +#ifndef CONFIG_OF +/** + * @brief Convert data in struct kbase_io_resources struct to Linux-specific resources + * + * Function converts data in struct kbase_io_resources struct to an array of Linux resource structures. Note that function + * assumes that size of linux_resource array is at least PLATFORM_CONFIG_RESOURCE_COUNT. + * Resources are put in fixed order: I/O memory region, job IRQ, MMU IRQ, GPU IRQ. + * + * @param[in] io_resource Input IO resource data + * @param[out] linux_resources Pointer to output array of Linux resource structures + */ +static void kbasep_config_parse_io_resources(const struct kbase_io_resources *io_resources, struct resource *const linux_resources) +{ + if (!io_resources || !linux_resources) { + pr_err("%s: couldn't find proper resources\n", __func__); + return; + } + + memset(linux_resources, 0, PLATFORM_CONFIG_RESOURCE_COUNT * sizeof(struct resource)); + + linux_resources[0].start = io_resources->io_memory_region.start; + linux_resources[0].end = io_resources->io_memory_region.end; + linux_resources[0].flags = IORESOURCE_MEM; + + linux_resources[1].start = io_resources->job_irq_number; + linux_resources[1].end = io_resources->job_irq_number; + linux_resources[1].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; + + linux_resources[2].start = io_resources->mmu_irq_number; + linux_resources[2].end = io_resources->mmu_irq_number; + linux_resources[2].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; + + linux_resources[3].start = io_resources->gpu_irq_number; + linux_resources[3].end = io_resources->gpu_irq_number; + linux_resources[3].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; +} +#endif /* CONFIG_OF */ + +int kbase_platform_register(void) +{ + struct kbase_platform_config *config; +#ifndef CONFIG_OF + struct resource resources[PLATFORM_CONFIG_RESOURCE_COUNT]; +#endif + int err; + + config = kbase_get_platform_config(); /* declared in midgard/mali_kbase_config.h but defined in platform folder */ + if (config == NULL) { + pr_err("%s: couldn't get platform config\n", __func__); + return -ENODEV; + } + + mali_device = platform_device_alloc("mali", 0); + if (mali_device == NULL) + return -ENOMEM; + +#ifndef CONFIG_OF + kbasep_config_parse_io_resources(config->io_resources, resources); + err = platform_device_add_resources(mali_device, resources, PLATFORM_CONFIG_RESOURCE_COUNT); + if (err) { + platform_device_put(mali_device); + mali_device = NULL; + return err; + } +#endif /* CONFIG_OF */ + + err = platform_device_add(mali_device); + if (err) { + platform_device_unregister(mali_device); + mali_device = NULL; + return err; + } + + return 0; +} +EXPORT_SYMBOL(kbase_platform_register); + +void kbase_platform_unregister(void) +{ + if (mali_device) + platform_device_unregister(mali_device); +} +EXPORT_SYMBOL(kbase_platform_unregister); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.c b/drivers/gpu/arm/bifrost/mali_kbase_pm.c new file mode 100755 index 000000000000..630ab1550045 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.c @@ -0,0 +1,292 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_pm.c + * Base kernel power management APIs + */ + +#include +#include +#include +#include + +#include +#include + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +#include +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + +#include + +int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) +{ + return kbase_hwaccess_pm_powerup(kbdev, flags); +} + +void kbase_pm_halt(struct kbase_device *kbdev) +{ + kbase_hwaccess_pm_halt(kbdev); +} + +void kbase_pm_context_active(struct kbase_device *kbdev) +{ + (void)kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); +} + +int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, + enum kbase_pm_suspend_handler suspend_handler) +{ + int c; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + dev_dbg(kbdev->dev, "%s - reason = %d, pid = %d\n", __func__, + suspend_handler, current->pid); + kbase_pm_lock(kbdev); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_arbiter_pm_ctx_active_handle_suspend(kbdev, + suspend_handler)) { + kbase_pm_unlock(kbdev); + return 1; + } +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + + if (kbase_pm_is_suspending(kbdev)) { + switch (suspend_handler) { + case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: + if (kbdev->pm.active_count != 0) + break; + /* FALLTHROUGH */ + case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: + kbase_pm_unlock(kbdev); + return 1; + + case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE: + /* FALLTHROUGH */ + default: + KBASE_DEBUG_ASSERT_MSG(false, "unreachable"); + break; + } + } + c = ++kbdev->pm.active_count; + KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_ACTIVE, NULL, c); + + if (c == 1) { + /* First context active: Power on the GPU and + * any cores requested by the policy + */ + kbase_hwaccess_pm_gpu_active(kbdev); +#ifdef CONFIG_MALI_ARBITER_SUPPORT + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_REF_EVENT); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + kbase_clk_rate_trace_manager_gpu_active(kbdev); + } + + kbase_pm_unlock(kbdev); + dev_dbg(kbdev->dev, "%s %d\n", __func__, kbdev->pm.active_count); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_pm_context_active); + +void kbase_pm_context_idle(struct kbase_device *kbdev) +{ + int c; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + + kbase_pm_lock(kbdev); + + c = --kbdev->pm.active_count; + KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_IDLE, NULL, c); + + KBASE_DEBUG_ASSERT(c >= 0); + + if (c == 0) { + /* Last context has gone idle */ + kbase_hwaccess_pm_gpu_idle(kbdev); + kbase_clk_rate_trace_manager_gpu_idle(kbdev); + + /* Wake up anyone waiting for this to become 0 (e.g. suspend). + * The waiters must synchronize with us by locking the pm.lock + * after waiting. + */ + wake_up(&kbdev->pm.zero_active_count_wait); + } + + kbase_pm_unlock(kbdev); + dev_dbg(kbdev->dev, "%s %d (pid = %d)\n", __func__, + kbdev->pm.active_count, current->pid); +} + +KBASE_EXPORT_TEST_API(kbase_pm_context_idle); + +void kbase_pm_driver_suspend(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + + /* Suspend vinstr. This blocks until the vinstr worker and timer are + * no longer running. + */ + kbase_vinstr_suspend(kbdev->vinstr_ctx); + + /* Disable GPU hardware counters. + * This call will block until counters are disabled. + */ + kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + + mutex_lock(&kbdev->pm.lock); + if (WARN_ON(kbase_pm_is_suspending(kbdev))) { + mutex_unlock(&kbdev->pm.lock); + return; + } + kbdev->pm.suspending = true; + mutex_unlock(&kbdev->pm.lock); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbdev->arb.arb_if) { + int i; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->js_data.runpool_irq.submit_allowed = 0; + kbase_disjoint_state_up(kbdev); + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + kbase_job_slot_softstop(kbdev, i, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + + /* From now on, the active count will drop towards zero. Sometimes, + * it'll go up briefly before going down again. However, once + * it reaches zero it will stay there - guaranteeing that we've idled + * all pm references + */ + +#if !MALI_USE_CSF + /* Suspend job scheduler and associated components, so that it releases all + * the PM active count references */ + kbasep_js_suspend(kbdev); +#else + kbase_csf_scheduler_pm_suspend(kbdev); +#endif + + /* Wait for the active count to reach zero. This is not the same as + * waiting for a power down, since not all policies power down when this + * reaches zero. + */ + dev_dbg(kbdev->dev, ">wait_event - waiting for active_count == 0 (pid = %d)\n", + current->pid); + wait_event(kbdev->pm.zero_active_count_wait, + kbdev->pm.active_count == 0); + dev_dbg(kbdev->dev, ">wait_event - waiting done\n"); + + /* NOTE: We synchronize with anything that was just finishing a + * kbase_pm_context_idle() call by locking the pm.lock below + */ + kbase_hwaccess_pm_suspend(kbdev); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbdev->arb.arb_if) { + mutex_lock(&kbdev->pm.arb_vm_state->vm_state_lock); + kbase_arbiter_pm_vm_stopped(kbdev); + mutex_unlock(&kbdev->pm.arb_vm_state->vm_state_lock); + } +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ +} + +void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) +{ + unsigned long flags; + + /* MUST happen before any pm_context_active calls occur */ + kbase_hwaccess_pm_resume(kbdev); + + /* Initial active call, to power on the GPU/cores if needed */ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbase_pm_context_active_handle_suspend(kbdev, + (arb_gpu_start ? + KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED : + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE))) + return; +#else + kbase_pm_context_active(kbdev); +#endif + +#if !MALI_USE_CSF + /* Resume any blocked atoms (which may cause contexts to be scheduled in + * and dependent atoms to run) + */ + kbase_resume_suspended_soft_jobs(kbdev); + + /* Resume the Job Scheduler and associated components, and start running + * atoms + */ + kbasep_js_resume(kbdev); +#else + kbase_csf_scheduler_pm_resume(kbdev); +#endif + + /* Matching idle call, to power off the GPU/cores if we didn't actually + * need it and the policy doesn't want it on + */ + kbase_pm_context_idle(kbdev); + + /* Re-enable GPU hardware counters */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Resume vinstr */ + kbase_vinstr_resume(kbdev->vinstr_ctx); +} + +void kbase_pm_suspend(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbdev->arb.arb_if) + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_SUSPEND_EVENT); + else + kbase_pm_driver_suspend(kbdev); +#else + kbase_pm_driver_suspend(kbdev); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ +} + +void kbase_pm_resume(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_ARBITER_SUPPORT + if (kbdev->arb.arb_if) + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_RESUME_EVENT); + else + kbase_pm_driver_resume(kbdev, false); +#else + kbase_pm_driver_resume(kbdev, false); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.h b/drivers/gpu/arm/bifrost/mali_kbase_pm.h new file mode 100755 index 000000000000..13565186c11f --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.h @@ -0,0 +1,251 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_kbase_pm.h + * Power management API definitions + */ + +#ifndef _KBASE_PM_H_ +#define _KBASE_PM_H_ + +#include "mali_kbase_hwaccess_pm.h" + +#define PM_ENABLE_IRQS 0x01 +#define PM_HW_ISSUES_DETECT 0x02 + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +/* In the case that the GPU was granted by the Arbiter, it will have + * already been reset. The following flag ensures it is not reset + * twice. + */ +#define PM_NO_RESET 0x04 +#endif + +/** Initialize the power management framework. + * + * Must be called before any other power management function + * + * @param kbdev The kbase device structure for the device + * (must be a valid pointer) + * + * @return 0 if the power management framework was successfully initialized. + */ +int kbase_pm_init(struct kbase_device *kbdev); + +/** Power up GPU after all modules have been initialized and interrupt handlers installed. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + * + * @param flags Flags to pass on to kbase_pm_init_hw + * + * @return 0 if powerup was successful. + */ +int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags); + +/** + * Halt the power management framework. + * Should ensure that no new interrupts are generated, + * but allow any currently running interrupt handlers to complete successfully. + * The GPU is forced off by the time this function returns, regardless of + * whether or not the active power policy asks for the GPU to be powered off. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_halt(struct kbase_device *kbdev); + +/** Terminate the power management framework. + * + * No power management functions may be called after this + * (except @ref kbase_pm_init) + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_term(struct kbase_device *kbdev); + +/** Increment the count of active contexts. + * + * This function should be called when a context is about to submit a job. + * It informs the active power policy that the GPU is going to be in use shortly + * and the policy is expected to start turning on the GPU. + * + * This function will block until the GPU is available. + * + * This function ASSERTS if a suspend is occuring/has occurred whilst this is + * in use. Use kbase_pm_contect_active_unless_suspending() instead. + * + * @note a Suspend is only visible to Kernel threads; user-space threads in a + * syscall cannot witness a suspend, because they are frozen before the suspend + * begins. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_context_active(struct kbase_device *kbdev); + + +/** Handler codes for doing kbase_pm_context_active_handle_suspend() */ +enum kbase_pm_suspend_handler { + /** A suspend is not expected/not possible - this is the same as + * kbase_pm_context_active() + */ + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, + /** If we're suspending, fail and don't increase the active count */ + KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE, + /** If we're suspending, succeed and allow the active count to increase + * if it didn't go from 0->1 (i.e., we didn't re-activate the GPU). + * + * This should only be used when there is a bounded time on the activation + * (e.g. guarantee it's going to be idled very soon after) + */ + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE, +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /** Special case when Arbiter has notified we can use GPU. + * Active count should always start at 0 in this case. + */ + KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED, +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ +}; + +/** Suspend 'safe' variant of kbase_pm_context_active() + * + * If a suspend is in progress, this allows for various different ways of + * handling the suspend. Refer to @ref enum kbase_pm_suspend_handler for details. + * + * We returns a status code indicating whether we're allowed to keep the GPU + * active during the suspend, depending on the handler code. If the status code + * indicates a failure, the caller must abort whatever operation it was + * attempting, and potentially queue it up for after the OS has resumed. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + * @param suspend_handler The handler code for how to handle a suspend that might occur + * @return zero Indicates success + * @return non-zero Indicates failure due to the system being suspending/suspended. + */ +int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler); + +/** Decrement the reference count of active contexts. + * + * This function should be called when a context becomes idle. + * After this call the GPU may be turned off by the power policy so the calling + * code should ensure that it does not access the GPU's registers. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_context_idle(struct kbase_device *kbdev); + +/* NOTE: kbase_pm_is_active() is in mali_kbase.h, because it is an inline + * function + */ + +/** + * Suspend the GPU and prevent any further register accesses to it from Kernel + * threads. + * + * This is called in response to an OS suspend event, and calls into the various + * kbase components to complete the suspend. + * + * @note the mechanisms used here rely on all user-space threads being frozen + * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up + * the GPU e.g. via atom submission. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_suspend(struct kbase_device *kbdev); + +/** + * Resume the GPU, allow register accesses to it, and resume running atoms on + * the GPU. + * + * This is called in response to an OS resume event, and calls into the various + * kbase components to complete the resume. + * + * Also called when using VM arbiter, when GPU access has been granted. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_resume(struct kbase_device *kbdev); + +/** + * kbase_pm_vsync_callback - vsync callback + * + * @buffer_updated: 1 if a new frame was displayed, 0 otherwise + * @data: Pointer to the kbase device as returned by kbase_find_device() + * + * Callback function used to notify the power management code that a vsync has + * occurred on the display. + */ +void kbase_pm_vsync_callback(int buffer_updated, void *data); + +/** + * kbase_pm_driver_suspend() - Put GPU and driver in suspend state + * @param kbdev The kbase device structure for the device + * (must be a valid pointer) + * + * Suspend the GPU and prevent any further register accesses to it from Kernel + * threads. + * + * This is called in response to an OS suspend event, and calls into the various + * kbase components to complete the suspend. + * + * Despite kbase_pm_suspend(), it will ignore to update Arbiter + * status if MALI_ARBITER_SUPPORT is enabled. + * + * @note the mechanisms used here rely on all user-space threads being frozen + * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up + * the GPU e.g. via atom submission. + */ +void kbase_pm_driver_suspend(struct kbase_device *kbdev); + +/** + * kbase_pm_driver_resume() - Put GPU and driver in resume + * @param kbdev The kbase device structure for the device + * (must be a valid pointer) + * + * Resume the GPU, allow register accesses to it, and resume running atoms on + * the GPU. + * + * This is called in response to an OS resume event, and calls into the various + * kbase components to complete the resume. + * + * Also called when using VM arbiter, when GPU access has been granted. + * + * Despite kbase_pm_resume(), it will ignore to update Arbiter + * status if MALI_ARBITER_SUPPORT is enabled. + */ +void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start); + +#ifdef CONFIG_MALI_ARBITER_SUPPORT +/** + * kbase_pm_handle_gpu_lost() - Handle GPU Lost for the VM + * @kbdev: Device pointer + * + * Handles the case that the Arbiter has forced the GPU away from the VM, + * so that interrupts will not be received and registers are no longer + * accessible because replaced by dummy RAM. + * Kill any running tasks and put the driver into a GPU powered-off state. + */ +void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev); +#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + +#endif /* _KBASE_PM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c new file mode 100755 index 000000000000..7b86c58440db --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c @@ -0,0 +1,245 @@ +/* + * + * (C) COPYRIGHT 2014, 2016, 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase.h" +#include "mali_kbase_regs_history_debugfs.h" + +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) + +#include + +/** + * kbase_io_history_resize - resize the register access history buffer. + * + * @h: Pointer to a valid register history to resize + * @new_size: Number of accesses the buffer could hold + * + * A successful resize will clear all recent register accesses. + * If resizing fails for any reason (e.g., could not allocate memory, invalid + * buffer size) then the original buffer will be kept intact. + * + * @return 0 if the buffer was resized, failure otherwise + */ +static int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size) +{ + struct kbase_io_access *old_buf; + struct kbase_io_access *new_buf; + unsigned long flags; + + if (!new_size) + goto out_err; /* The new size must not be 0 */ + + new_buf = vmalloc(new_size * sizeof(*h->buf)); + if (!new_buf) + goto out_err; + + spin_lock_irqsave(&h->lock, flags); + + old_buf = h->buf; + + /* Note: we won't bother with copying the old data over. The dumping + * logic wouldn't work properly as it relies on 'count' both as a + * counter and as an index to the buffer which would have changed with + * the new array. This is a corner case that we don't need to support. + */ + h->count = 0; + h->size = new_size; + h->buf = new_buf; + + spin_unlock_irqrestore(&h->lock, flags); + + vfree(old_buf); + + return 0; + +out_err: + return -1; +} + +int kbase_io_history_init(struct kbase_io_history *h, u16 n) +{ + h->enabled = false; + spin_lock_init(&h->lock); + h->count = 0; + h->size = 0; + h->buf = NULL; + if (kbase_io_history_resize(h, n)) + return -1; + + return 0; +} + +void kbase_io_history_term(struct kbase_io_history *h) +{ + vfree(h->buf); + h->buf = NULL; +} + +void kbase_io_history_add(struct kbase_io_history *h, + void __iomem const *addr, u32 value, u8 write) +{ + struct kbase_io_access *io; + unsigned long flags; + + spin_lock_irqsave(&h->lock, flags); + + io = &h->buf[h->count % h->size]; + io->addr = (uintptr_t)addr | write; + io->value = value; + ++h->count; + /* If count overflows, move the index by the buffer size so the entire + * buffer will still be dumped later + */ + if (unlikely(!h->count)) + h->count = h->size; + + spin_unlock_irqrestore(&h->lock, flags); +} + +void kbase_io_history_dump(struct kbase_device *kbdev) +{ + struct kbase_io_history *const h = &kbdev->io_history; + u16 i; + size_t iters; + unsigned long flags; + + if (!unlikely(h->enabled)) + return; + + spin_lock_irqsave(&h->lock, flags); + + dev_err(kbdev->dev, "Register IO History:"); + iters = (h->size > h->count) ? h->count : h->size; + dev_err(kbdev->dev, "Last %zu register accesses of %zu total:\n", iters, + h->count); + for (i = 0; i < iters; ++i) { + struct kbase_io_access *io = + &h->buf[(h->count - iters + i) % h->size]; + char const access = (io->addr & 1) ? 'w' : 'r'; + + dev_err(kbdev->dev, "%6i: %c: reg 0x%016lx val %08x\n", i, + access, (unsigned long)(io->addr & ~0x1), io->value); + } + + spin_unlock_irqrestore(&h->lock, flags); +} + +static int regs_history_size_get(void *data, u64 *val) +{ + struct kbase_io_history *const h = data; + + *val = h->size; + + return 0; +} + +static int regs_history_size_set(void *data, u64 val) +{ + struct kbase_io_history *const h = data; + + return kbase_io_history_resize(h, (u16)val); +} + + +DEFINE_SIMPLE_ATTRIBUTE(regs_history_size_fops, + regs_history_size_get, + regs_history_size_set, + "%llu\n"); + + +/** + * regs_history_show - show callback for the register access history file. + * + * @sfile: The debugfs entry + * @data: Data associated with the entry + * + * This function is called to dump all recent accesses to the GPU registers. + * + * @return 0 if successfully prints data in debugfs entry file, failure + * otherwise + */ +static int regs_history_show(struct seq_file *sfile, void *data) +{ + struct kbase_io_history *const h = sfile->private; + u16 i; + size_t iters; + unsigned long flags; + + if (!h->enabled) { + seq_puts(sfile, "The register access history is disabled\n"); + goto out; + } + + spin_lock_irqsave(&h->lock, flags); + + iters = (h->size > h->count) ? h->count : h->size; + seq_printf(sfile, "Last %zu register accesses of %zu total:\n", iters, + h->count); + for (i = 0; i < iters; ++i) { + struct kbase_io_access *io = + &h->buf[(h->count - iters + i) % h->size]; + char const access = (io->addr & 1) ? 'w' : 'r'; + + seq_printf(sfile, "%6i: %c: reg 0x%016lx val %08x\n", i, access, + (unsigned long)(io->addr & ~0x1), io->value); + } + + spin_unlock_irqrestore(&h->lock, flags); + +out: + return 0; +} + +/** + * regs_history_open - open operation for regs_history debugfs file + * + * @in: &struct inode pointer + * @file: &struct file pointer + * + * @return file descriptor + */ +static int regs_history_open(struct inode *in, struct file *file) +{ + return single_open(file, ®s_history_show, in->i_private); +} + +static const struct file_operations regs_history_fops = { + .owner = THIS_MODULE, + .open = ®s_history_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_bool("regs_history_enabled", S_IRUGO | S_IWUSR, + kbdev->mali_debugfs_directory, + &kbdev->io_history.enabled); + debugfs_create_file("regs_history_size", S_IRUGO | S_IWUSR, + kbdev->mali_debugfs_directory, + &kbdev->io_history, ®s_history_size_fops); + debugfs_create_file("regs_history", S_IRUGO, + kbdev->mali_debugfs_directory, &kbdev->io_history, + ®s_history_fops); +} +#endif /* defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.h new file mode 100755 index 000000000000..200c0c2d8de8 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.h @@ -0,0 +1,85 @@ +/* + * + * (C) COPYRIGHT 2014, 2016, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Header file for register access history support via debugfs + * + * This interface is made available via /sys/kernel/debug/mali#/regs_history*. + * + * Usage: + * - regs_history_enabled: whether recording of register accesses is enabled. + * Write 'y' to enable, 'n' to disable. + * - regs_history_size: size of the register history buffer, must be > 0 + * - regs_history: return the information about last accesses to the registers. + */ + +#ifndef _KBASE_REGS_HISTORY_DEBUGFS_H +#define _KBASE_REGS_HISTORY_DEBUGFS_H + +struct kbase_device; + +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) + +/** + * kbase_io_history_init - initialize data struct for register access history + * + * @h: The register history to initialize + * @n: The number of register accesses that the buffer could hold + * + * @return 0 if successfully initialized, failure otherwise + */ +int kbase_io_history_init(struct kbase_io_history *h, u16 n); + +/** + * kbase_io_history_term - uninit all resources for the register access history + * + * @h: The register history to terminate + */ +void kbase_io_history_term(struct kbase_io_history *h); + +/** + * kbase_io_history_dump - print the register history to the kernel ring buffer + * + * @kbdev: Pointer to kbase_device containing the register history to dump + */ +void kbase_io_history_dump(struct kbase_device *kbdev); + +/** + * kbasep_regs_history_debugfs_init - add debugfs entries for register history + * + * @kbdev: Pointer to kbase_device containing the register history + */ +void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev); + +#else /* defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) */ + +#define kbase_io_history_init(...) ((int)0) + +#define kbase_io_history_term CSTD_NOP + +#define kbase_io_history_dump CSTD_NOP + +#define kbasep_regs_history_debugfs_init CSTD_NOP + +#endif /* defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) */ + +#endif /*_KBASE_REGS_HISTORY_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h b/drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h new file mode 100755 index 000000000000..61bbb0b48490 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h @@ -0,0 +1,129 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_RESET_GPU_H_ +#define _KBASE_RESET_GPU_H_ + +/** + * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU. + * @kbdev: Device pointer + * + * Caller is expected to hold the kbdev->hwaccess_lock. + * + * Return: a boolean which should be interpreted as follows: + * - true - Prepared for reset, kbase_reset_gpu should be called. + * - false - Another thread is performing a reset, kbase_reset_gpu should + * not be called. + */ +bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev); + +/** + * kbase_prepare_to_reset_gpu - Prepare for resetting the GPU. + * @kbdev: Device pointer + * + * Return: a boolean which should be interpreted as follows: + * - true - Prepared for reset, kbase_reset_gpu should be called. + * - false - Another thread is performing a reset, kbase_reset_gpu should + * not be called. + */ +bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu - Reset the GPU + * @kbdev: Device pointer + * + * This function should be called after kbase_prepare_to_reset_gpu if it returns + * true. It should never be called without a corresponding call to + * kbase_prepare_to_reset_gpu (only on Job Manager GPUs). + * + * After this function is called the caller should call kbase_reset_gpu_wait() + * to know when the reset has completed. + */ +void kbase_reset_gpu(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_locked - Reset the GPU + * @kbdev: Device pointer + * + * This function should be called after kbase_prepare_to_reset_gpu_locked if it + * returns true. It should never be called without a corresponding call to + * kbase_prepare_to_reset_gpu (only on Job Manager GPUs). + * Caller is expected to hold the kbdev->hwaccess_lock. + * + * After this function is called, the caller should call kbase_reset_gpu_wait() + * to know when the reset has completed. + */ +void kbase_reset_gpu_locked(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_silent - Reset the GPU silently + * @kbdev: Device pointer + * + * Reset the GPU without trying to cancel jobs (applicable to Job Manager GPUs) + * and don't emit messages into the kernel log while doing the reset. + * + * This function should be used in cases where we are doing a controlled reset + * of the GPU as part of normal processing (e.g. exiting protected mode) where + * the driver will have ensured the scheduler has been idled and all other + * users of the GPU (e.g. instrumentation) have been suspended. + * + * Return: 0 if the reset was started successfully + * -EAGAIN if another reset is currently in progress + */ +int kbase_reset_gpu_silent(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_is_active - Reports if the GPU is being reset + * @kbdev: Device pointer + * + * Return: True if the GPU is in the process of being reset (or if the reset of + * GPU failed, not applicable to Job Manager GPUs). + */ +bool kbase_reset_gpu_is_active(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_wait - Wait for a GPU reset to complete + * @kbdev: Device pointer + * + * This function may wait indefinitely. + * + * Return: 0 if successful or a negative error code on failure. + */ +int kbase_reset_gpu_wait(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_init - Initialize the GPU reset handling mechanism. + * + * @kbdev: Device pointer + * + * Return: 0 if successful or a negative error code on failure. + */ +int kbase_reset_gpu_init(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_term - Terminate the GPU reset handling mechanism. + * + * @kbdev: Device pointer + */ +void kbase_reset_gpu_term(struct kbase_device *kbdev); + +#endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase_smc.c b/drivers/gpu/arm/bifrost/mali_kbase_smc.c new file mode 100755 index 000000000000..b5c7b1289846 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_smc.c @@ -0,0 +1,91 @@ +/* + * + * (C) COPYRIGHT 2015, 2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifdef CONFIG_ARM64 + +#include +#include + +#include + +/* __asmeq is not available on Kernel versions >= 4.20 */ +#ifndef __asmeq +/* + * This is used to ensure the compiler did actually allocate the register we + * asked it for some inline assembly sequences. Apparently we can't trust the + * compiler from one version to another so a bit of paranoia won't hurt. This + * string is meant to be concatenated with the inline asm string and will + * cause compilation to stop on mismatch. (for details, see gcc PR 15089) + */ +#define __asmeq(x, y) ".ifnc " x "," y " ; .err ; .endif\n\t" +#endif + +static noinline u64 invoke_smc_fid(u64 function_id, + u64 arg0, u64 arg1, u64 arg2) +{ + register u64 x0 asm("x0") = function_id; + register u64 x1 asm("x1") = arg0; + register u64 x2 asm("x2") = arg1; + register u64 x3 asm("x3") = arg2; + + asm volatile( + __asmeq("%0", "x0") + __asmeq("%1", "x1") + __asmeq("%2", "x2") + __asmeq("%3", "x3") + "smc #0\n" + : "+r" (x0) + : "r" (x1), "r" (x2), "r" (x3)); + + return x0; +} + +u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2) +{ + /* Is fast call (bit 31 set) */ + KBASE_DEBUG_ASSERT(fid & ~SMC_FAST_CALL); + /* bits 16-23 must be zero for fast calls */ + KBASE_DEBUG_ASSERT((fid & (0xFF << 16)) == 0); + + return invoke_smc_fid(fid, arg0, arg1, arg2); +} + +u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, + u64 arg0, u64 arg1, u64 arg2) +{ + u32 fid = 0; + + /* Only the six bits allowed should be used. */ + KBASE_DEBUG_ASSERT((oen & ~SMC_OEN_MASK) == 0); + + fid |= SMC_FAST_CALL; /* Bit 31: Fast call */ + if (smc64) + fid |= SMC_64; /* Bit 30: 1=SMC64, 0=SMC32 */ + fid |= oen; /* Bit 29:24: OEN */ + /* Bit 23:16: Must be zero for fast calls */ + fid |= (function_number); /* Bit 15:0: function number */ + + return kbase_invoke_smc_fid(fid, arg0, arg1, arg2); +} + +#endif /* CONFIG_ARM64 */ + diff --git a/drivers/gpu/arm/bifrost/mali_kbase_smc.h b/drivers/gpu/arm/bifrost/mali_kbase_smc.h new file mode 100755 index 000000000000..221eb21a8c7f --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_smc.h @@ -0,0 +1,72 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#ifndef _KBASE_SMC_H_ +#define _KBASE_SMC_H_ + +#ifdef CONFIG_ARM64 + +#include + +#define SMC_FAST_CALL (1 << 31) +#define SMC_64 (1 << 30) + +#define SMC_OEN_OFFSET 24 +#define SMC_OEN_MASK (0x3F << SMC_OEN_OFFSET) /* 6 bits */ +#define SMC_OEN_SIP (2 << SMC_OEN_OFFSET) +#define SMC_OEN_STD (4 << SMC_OEN_OFFSET) + + +/** + * kbase_invoke_smc_fid - Perform a secure monitor call + * @fid: The SMC function to call, see SMC Calling convention. + * @arg0: First argument to the SMC. + * @arg1: Second argument to the SMC. + * @arg2: Third argument to the SMC. + * + * See SMC Calling Convention for details. + * + * Return: the return value from the SMC. + */ +u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2); + +/** + * kbase_invoke_smc_fid - Perform a secure monitor call + * @oen: Owning Entity number (SIP, STD etc). + * @function_number: The function number within the OEN. + * @smc64: use SMC64 calling convention instead of SMC32. + * @arg0: First argument to the SMC. + * @arg1: Second argument to the SMC. + * @arg2: Third argument to the SMC. + * + * See SMC Calling Convention for details. + * + * Return: the return value from the SMC call. + */ +u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, + u64 arg0, u64 arg1, u64 arg2); + +#endif /* CONFIG_ARM64 */ + +#endif /* _KBASE_SMC_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c new file mode 100755 index 000000000000..c164719b3d7b --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c @@ -0,0 +1,1805 @@ +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#include + +#include +#include +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !MALI_USE_CSF +/** + * @file mali_kbase_softjobs.c + * + * This file implements the logic behind software only jobs that are + * executed within the driver rather than being handed over to the GPU. + */ + +static void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + unsigned long lflags; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_add_tail(&katom->queue, &kctx->waiting_soft_jobs); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + unsigned long lflags; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_del(&katom->queue); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +static void kbasep_add_waiting_with_timeout(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + /* Record the start time of this atom so we could cancel it at + * the right time. + */ + katom->start_timestamp = ktime_get(); + + /* Add the atom to the waiting list before the timer is + * (re)started to make sure that it gets processed. + */ + kbasep_add_waiting_soft_job(katom); + + /* Schedule timeout of this atom after a period if it is not active */ + if (!timer_pending(&kctx->soft_job_timeout)) { + int timeout_ms = atomic_read( + &kctx->kbdev->js_data.soft_job_timeout_ms); + mod_timer(&kctx->soft_job_timeout, + jiffies + msecs_to_jiffies(timeout_ms)); + } +} + +static int kbasep_read_soft_event_status( + struct kbase_context *kctx, u64 evt, unsigned char *status) +{ + unsigned char *mapped_evt; + struct kbase_vmap_struct map; + + mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); + if (!mapped_evt) + return -EFAULT; + + *status = *mapped_evt; + + kbase_vunmap(kctx, &map); + + return 0; +} + +static int kbasep_write_soft_event_status( + struct kbase_context *kctx, u64 evt, unsigned char new_status) +{ + unsigned char *mapped_evt; + struct kbase_vmap_struct map; + + if ((new_status != BASE_JD_SOFT_EVENT_SET) && + (new_status != BASE_JD_SOFT_EVENT_RESET)) + return -EINVAL; + + mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); + if (!mapped_evt) + return -EFAULT; + + *mapped_evt = new_status; + + kbase_vunmap(kctx, &map); + + return 0; +} + +static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) +{ + struct kbase_vmap_struct map; + void *user_result; + struct timespec64 ts; + struct base_dump_cpu_gpu_counters data; + u64 system_time; + u64 cycle_counter; + u64 jc = katom->jc; + struct kbase_context *kctx = katom->kctx; + int pm_active_err; + + memset(&data, 0, sizeof(data)); + + /* Take the PM active reference as late as possible - otherwise, it could + * delay suspend until we process the atom (which may be at the end of a + * long chain of dependencies */ + pm_active_err = kbase_pm_context_active_handle_suspend(kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); + if (pm_active_err) { + struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; + + /* We're suspended - queue this on the list of suspended jobs + * Use dep_item[1], because dep_item[0] was previously in use + * for 'waiting_soft_jobs'. + */ + mutex_lock(&js_devdata->runpool_mutex); + list_add_tail(&katom->dep_item[1], &js_devdata->suspended_soft_jobs_list); + mutex_unlock(&js_devdata->runpool_mutex); + + /* Also adding this to the list of waiting soft job */ + kbasep_add_waiting_soft_job(katom); + + return pm_active_err; + } + + kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, + &ts); + + kbase_pm_context_idle(kctx->kbdev); + + data.sec = ts.tv_sec; + data.usec = ts.tv_nsec / 1000; + data.system_time = system_time; + data.cycle_counter = cycle_counter; + + /* Assume this atom will be cancelled until we know otherwise */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* GPU_WR access is checked on the range for returning the result to + * userspace for the following reasons: + * - security, this is currently how imported user bufs are checked. + * - userspace ddk guaranteed to assume region was mapped as GPU_WR */ + user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); + if (!user_result) + return 0; + + memcpy(user_result, &data, sizeof(data)); + + kbase_vunmap(kctx, &map); + + /* Atom was fine - mark it as done */ + katom->event_code = BASE_JD_EVENT_DONE; + + return 0; +} + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +/* Called by the explicit fence mechanism when a fence wait has completed */ +void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + mutex_lock(&kctx->jctx.lock); + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(kctx->kbdev); + mutex_unlock(&kctx->jctx.lock); +} +#endif + +static void kbasep_soft_event_complete_job(struct work_struct *work) +{ + struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + int resched; + + mutex_lock(&kctx->jctx.lock); + resched = jd_done_nolock(katom, NULL); + mutex_unlock(&kctx->jctx.lock); + + if (resched) + kbase_js_sched_all(kctx->kbdev); +} + +void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt) +{ + int cancel_timer = 1; + struct list_head *entry, *tmp; + unsigned long lflags; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { + struct kbase_jd_atom *katom = list_entry( + entry, struct kbase_jd_atom, queue); + + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_EVENT_WAIT: + if (katom->jc == evt) { + list_del(&katom->queue); + + katom->event_code = BASE_JD_EVENT_DONE; + INIT_WORK(&katom->work, + kbasep_soft_event_complete_job); + queue_work(kctx->jctx.job_done_wq, + &katom->work); + } else { + /* There are still other waiting jobs, we cannot + * cancel the timer yet. + */ + cancel_timer = 0; + } + break; +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG + case BASE_JD_REQ_SOFT_FENCE_WAIT: + /* Keep the timer running if fence debug is enabled and + * there are waiting fence jobs. + */ + cancel_timer = 0; + break; +#endif + } + } + + if (cancel_timer) + del_timer(&kctx->soft_job_timeout); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG +static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct device *dev = kctx->kbdev->dev; + int i; + + for (i = 0; i < 2; i++) { + struct kbase_jd_atom *dep; + + list_for_each_entry(dep, &katom->dep_head[i], dep_item[i]) { + if (dep->status == KBASE_JD_ATOM_STATE_UNUSED || + dep->status == KBASE_JD_ATOM_STATE_COMPLETED) + continue; + + if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) + == BASE_JD_REQ_SOFT_FENCE_TRIGGER) { + /* Found blocked trigger fence. */ + struct kbase_sync_fence_info info; + + if (!kbase_sync_fence_in_info_get(dep, &info)) { + dev_warn(dev, + "\tVictim trigger atom %d fence [%p] %s: %s\n", + kbase_jd_atom_id(kctx, dep), + info.fence, + info.name, + kbase_sync_status_string(info.status)); + } + } + + kbase_fence_debug_check_atom(dep); + } + } +} + +static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct device *dev = katom->kctx->kbdev->dev; + int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); + unsigned long lflags; + struct kbase_sync_fence_info info; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + + if (kbase_sync_fence_in_info_get(katom, &info)) { + /* Fence must have signaled just after timeout. */ + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); + return; + } + + dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%p] after %dms\n", + kctx->tgid, kctx->id, + kbase_jd_atom_id(kctx, katom), + info.fence, timeout_ms); + dev_warn(dev, "\tGuilty fence [%p] %s: %s\n", + info.fence, info.name, + kbase_sync_status_string(info.status)); + + /* Search for blocked trigger atoms */ + kbase_fence_debug_check_atom(katom); + + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); + + kbase_sync_fence_in_dump(katom); +} + +struct kbase_fence_debug_work { + struct kbase_jd_atom *katom; + struct work_struct work; +}; + +static void kbase_fence_debug_wait_timeout_worker(struct work_struct *work) +{ + struct kbase_fence_debug_work *w = container_of(work, + struct kbase_fence_debug_work, work); + struct kbase_jd_atom *katom = w->katom; + struct kbase_context *kctx = katom->kctx; + + mutex_lock(&kctx->jctx.lock); + kbase_fence_debug_wait_timeout(katom); + mutex_unlock(&kctx->jctx.lock); + + kfree(w); +} + +static void kbase_fence_debug_timeout(struct kbase_jd_atom *katom) +{ + struct kbase_fence_debug_work *work; + struct kbase_context *kctx = katom->kctx; + + /* Enqueue fence debug worker. Use job_done_wq to get + * debug print ordered with job completion. + */ + work = kzalloc(sizeof(struct kbase_fence_debug_work), GFP_ATOMIC); + /* Ignore allocation failure. */ + if (work) { + work->katom = katom; + INIT_WORK(&work->work, kbase_fence_debug_wait_timeout_worker); + queue_work(kctx->jctx.job_done_wq, &work->work); + } +} +#endif /* CONFIG_MALI_BIFROST_FENCE_DEBUG */ + +void kbasep_soft_job_timeout_worker(struct timer_list *timer) +{ + struct kbase_context *kctx = container_of(timer, struct kbase_context, + soft_job_timeout); + u32 timeout_ms = (u32)atomic_read( + &kctx->kbdev->js_data.soft_job_timeout_ms); + ktime_t cur_time = ktime_get(); + bool restarting = false; + unsigned long lflags; + struct list_head *entry, *tmp; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { + struct kbase_jd_atom *katom = list_entry(entry, + struct kbase_jd_atom, queue); + s64 elapsed_time = ktime_to_ms(ktime_sub(cur_time, + katom->start_timestamp)); + + if (elapsed_time < (s64)timeout_ms) { + restarting = true; + continue; + } + + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_EVENT_WAIT: + /* Take it out of the list to ensure that it + * will be cancelled in all cases + */ + list_del(&katom->queue); + + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + INIT_WORK(&katom->work, kbasep_soft_event_complete_job); + queue_work(kctx->jctx.job_done_wq, &katom->work); + break; +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG + case BASE_JD_REQ_SOFT_FENCE_WAIT: + kbase_fence_debug_timeout(katom); + break; +#endif + } + } + + if (restarting) + mod_timer(timer, jiffies + msecs_to_jiffies(timeout_ms)); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +static int kbasep_soft_event_wait(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + unsigned char status; + + /* The status of this soft-job is stored in jc */ + if (kbasep_read_soft_event_status(kctx, katom->jc, &status)) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + return 0; + } + + if (status == BASE_JD_SOFT_EVENT_SET) + return 0; /* Event already set, nothing to do */ + + kbasep_add_waiting_with_timeout(katom); + + return 1; +} + +static void kbasep_soft_event_update_locked(struct kbase_jd_atom *katom, + unsigned char new_status) +{ + /* Complete jobs waiting on the same event */ + struct kbase_context *kctx = katom->kctx; + + if (kbasep_write_soft_event_status(kctx, katom->jc, new_status) != 0) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + return; + } + + if (new_status == BASE_JD_SOFT_EVENT_SET) + kbasep_complete_triggered_soft_events(kctx, katom->jc); +} + +/** + * kbase_soft_event_update() - Update soft event state + * @kctx: Pointer to context + * @event: Event to update + * @new_status: New status value of event + * + * Update the event, and wake up any atoms waiting for the event. + * + * Return: 0 on success, a negative error code on failure. + */ +int kbase_soft_event_update(struct kbase_context *kctx, + u64 event, + unsigned char new_status) +{ + int err = 0; + + mutex_lock(&kctx->jctx.lock); + + if (kbasep_write_soft_event_status(kctx, event, new_status)) { + err = -ENOENT; + goto out; + } + + if (new_status == BASE_JD_SOFT_EVENT_SET) + kbasep_complete_triggered_soft_events(kctx, event); + +out: + mutex_unlock(&kctx->jctx.lock); + + return err; +} + +static void kbasep_soft_event_cancel_job(struct kbase_jd_atom *katom) +{ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +static void kbase_debug_copy_finish(struct kbase_jd_atom *katom) +{ + struct kbase_debug_copy_buffer *buffers = katom->softjob_data; + unsigned int i; + unsigned int nr = katom->nr_extres; + + if (!buffers) + return; + + kbase_gpu_vm_lock(katom->kctx); + for (i = 0; i < nr; i++) { + int p; + struct kbase_mem_phy_alloc *gpu_alloc = buffers[i].gpu_alloc; + + if (!buffers[i].pages) + break; + for (p = 0; p < buffers[i].nr_pages; p++) { + struct page *pg = buffers[i].pages[p]; + + if (pg) + put_page(pg); + } + if (buffers[i].is_vmalloc) + vfree(buffers[i].pages); + else + kfree(buffers[i].pages); + if (gpu_alloc) { + switch (gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + { + kbase_free_user_buffer(&buffers[i]); + break; + } + default: + /* Nothing to be done. */ + break; + } + kbase_mem_phy_alloc_put(gpu_alloc); + } + } + kbase_gpu_vm_unlock(katom->kctx); + kfree(buffers); + + katom->softjob_data = NULL; +} + +static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) +{ + struct kbase_debug_copy_buffer *buffers; + struct base_jd_debug_copy_buffer *user_buffers = NULL; + unsigned int i; + unsigned int nr = katom->nr_extres; + int ret = 0; + void __user *user_structs = (void __user *)(uintptr_t)katom->jc; + + if (!user_structs) + return -EINVAL; + + buffers = kcalloc(nr, sizeof(*buffers), GFP_KERNEL); + if (!buffers) { + ret = -ENOMEM; + goto out_cleanup; + } + katom->softjob_data = buffers; + + user_buffers = kmalloc_array(nr, sizeof(*user_buffers), GFP_KERNEL); + + if (!user_buffers) { + ret = -ENOMEM; + goto out_cleanup; + } + + ret = copy_from_user(user_buffers, user_structs, + sizeof(*user_buffers)*nr); + if (ret) { + ret = -EFAULT; + goto out_cleanup; + } + + for (i = 0; i < nr; i++) { + u64 addr = user_buffers[i].address; + u64 page_addr = addr & PAGE_MASK; + u64 end_page_addr = addr + user_buffers[i].size - 1; + u64 last_page_addr = end_page_addr & PAGE_MASK; + int nr_pages = (last_page_addr-page_addr)/PAGE_SIZE+1; + int pinned_pages; + struct kbase_va_region *reg; + struct base_external_resource user_extres; + + if (!addr) + continue; + + if (last_page_addr < page_addr) { + ret = -EINVAL; + goto out_cleanup; + } + + buffers[i].nr_pages = nr_pages; + buffers[i].offset = addr & ~PAGE_MASK; + if (buffers[i].offset >= PAGE_SIZE) { + ret = -EINVAL; + goto out_cleanup; + } + buffers[i].size = user_buffers[i].size; + + if (nr_pages > (KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD / + sizeof(struct page *))) { + buffers[i].is_vmalloc = true; + buffers[i].pages = vzalloc(nr_pages * + sizeof(struct page *)); + } else { + buffers[i].is_vmalloc = false; + buffers[i].pages = kcalloc(nr_pages, + sizeof(struct page *), GFP_KERNEL); + } + + if (!buffers[i].pages) { + ret = -ENOMEM; + goto out_cleanup; + } + + pinned_pages = get_user_pages_fast(page_addr, + nr_pages, + 1, /* Write */ + buffers[i].pages); + if (pinned_pages < 0) { + /* get_user_pages_fast has failed - page array is not + * valid. Don't try to release any pages. + */ + buffers[i].nr_pages = 0; + + ret = pinned_pages; + goto out_cleanup; + } + if (pinned_pages != nr_pages) { + /* Adjust number of pages, so that we only attempt to + * release pages in the array that we know are valid. + */ + buffers[i].nr_pages = pinned_pages; + + ret = -EINVAL; + goto out_cleanup; + } + + user_extres = user_buffers[i].extres; + if (user_extres.ext_resource == 0ULL) { + ret = -EINVAL; + goto out_cleanup; + } + + kbase_gpu_vm_lock(katom->kctx); + reg = kbase_region_tracker_find_region_enclosing_address( + katom->kctx, user_extres.ext_resource & + ~BASE_EXT_RES_ACCESS_EXCLUSIVE); + + if (kbase_is_region_invalid_or_free(reg) || + reg->gpu_alloc == NULL) { + ret = -EINVAL; + goto out_unlock; + } + + buffers[i].gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + buffers[i].nr_extres_pages = reg->nr_pages; + + if (reg->nr_pages*PAGE_SIZE != buffers[i].size) + dev_warn(katom->kctx->kbdev->dev, "Copy buffer is not of same size as the external resource to copy.\n"); + + switch (reg->gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + { + struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; + unsigned long nr_pages = + alloc->imported.user_buf.nr_pages; + + if (alloc->imported.user_buf.mm != current->mm) { + ret = -EINVAL; + goto out_unlock; + } + buffers[i].extres_pages = kcalloc(nr_pages, + sizeof(struct page *), GFP_KERNEL); + if (!buffers[i].extres_pages) { + ret = -ENOMEM; + goto out_unlock; + } + + ret = get_user_pages_fast( + alloc->imported.user_buf.address, + nr_pages, 0, + buffers[i].extres_pages); + if (ret != nr_pages) { + /* Adjust number of pages, so that we only + * attempt to release pages in the array that we + * know are valid. + */ + if (ret < 0) + buffers[i].nr_extres_pages = 0; + else + buffers[i].nr_extres_pages = ret; + + goto out_unlock; + } + ret = 0; + break; + } + default: + /* Nothing to be done. */ + break; + } + kbase_gpu_vm_unlock(katom->kctx); + } + kfree(user_buffers); + + return ret; + +out_unlock: + kbase_gpu_vm_unlock(katom->kctx); + +out_cleanup: + /* Frees allocated memory for kbase_debug_copy_job struct, including + * members, and sets jc to 0 */ + kbase_debug_copy_finish(katom); + kfree(user_buffers); + + return ret; +} +#endif /* !MALI_USE_CSF */ + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) +static void *dma_buf_kmap_page(struct kbase_mem_phy_alloc *gpu_alloc, + unsigned long page_num, struct page **page) +{ + struct sg_table *sgt = gpu_alloc->imported.umm.sgt; + struct sg_page_iter sg_iter; + unsigned long page_index = 0; + + if (WARN_ON(gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) + return NULL; + + if (!sgt) + return NULL; + + if (WARN_ON(page_num >= gpu_alloc->nents)) + return NULL; + + for_each_sg_page(sgt->sgl, &sg_iter, sgt->nents, 0) { + if (page_index == page_num) { + *page = sg_page_iter_page(&sg_iter); + + return kmap(*page); + } + page_index++; + } + + return NULL; +} +#endif + +int kbase_mem_copy_from_extres(struct kbase_context *kctx, + struct kbase_debug_copy_buffer *buf_data) +{ + unsigned int i; + unsigned int target_page_nr = 0; + struct page **pages = buf_data->pages; + u64 offset = buf_data->offset; + size_t extres_size = buf_data->nr_extres_pages*PAGE_SIZE; + size_t to_copy = min(extres_size, buf_data->size); + struct kbase_mem_phy_alloc *gpu_alloc = buf_data->gpu_alloc; + int ret = 0; + size_t dma_to_copy; + + KBASE_DEBUG_ASSERT(pages != NULL); + + kbase_gpu_vm_lock(kctx); + if (!gpu_alloc) { + ret = -EINVAL; + goto out_unlock; + } + + switch (gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + { + for (i = 0; i < buf_data->nr_extres_pages && + target_page_nr < buf_data->nr_pages; i++) { + struct page *pg = buf_data->extres_pages[i]; + void *extres_page = kmap(pg); + + if (extres_page) { + ret = kbase_mem_copy_to_pinned_user_pages( + pages, extres_page, &to_copy, + buf_data->nr_pages, + &target_page_nr, offset); + kunmap(pg); + if (ret) + goto out_unlock; + } + } + } + break; + case KBASE_MEM_TYPE_IMPORTED_UMM: { + struct dma_buf *dma_buf = gpu_alloc->imported.umm.dma_buf; + + KBASE_DEBUG_ASSERT(dma_buf != NULL); + if (dma_buf->size > buf_data->nr_extres_pages * PAGE_SIZE) + dev_warn(kctx->kbdev->dev, "External resources buffer size mismatch"); + + dma_to_copy = min(dma_buf->size, + (size_t)(buf_data->nr_extres_pages * PAGE_SIZE)); + ret = dma_buf_begin_cpu_access(dma_buf, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) + 0, dma_to_copy, +#endif + DMA_FROM_DEVICE); + if (ret) + goto out_unlock; + + for (i = 0; i < dma_to_copy/PAGE_SIZE && + target_page_nr < buf_data->nr_pages; i++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) + struct page *pg; + void *extres_page = dma_buf_kmap_page(gpu_alloc, i, &pg); +#else + void *extres_page = dma_buf_kmap(dma_buf, i); +#endif + if (extres_page) { + ret = kbase_mem_copy_to_pinned_user_pages( + pages, extres_page, &to_copy, + buf_data->nr_pages, + &target_page_nr, offset); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) + kunmap(pg); +#else + dma_buf_kunmap(dma_buf, i, extres_page); +#endif + if (ret) + goto out_unlock; + } + } + dma_buf_end_cpu_access(dma_buf, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) + 0, dma_to_copy, +#endif + DMA_FROM_DEVICE); + break; + } + default: + ret = -EINVAL; + } +out_unlock: + kbase_gpu_vm_unlock(kctx); + return ret; +} + +#if !MALI_USE_CSF +static int kbase_debug_copy(struct kbase_jd_atom *katom) +{ + struct kbase_debug_copy_buffer *buffers = katom->softjob_data; + unsigned int i; + + if (WARN_ON(!buffers)) + return -EINVAL; + + for (i = 0; i < katom->nr_extres; i++) { + int res = kbase_mem_copy_from_extres(katom->kctx, &buffers[i]); + + if (res) + return res; + } + + return 0; +} +#endif /* !MALI_USE_CSF */ + +#define KBASEP_JIT_ALLOC_GPU_ADDR_ALIGNMENT ((u32)0x7) + +int kbasep_jit_alloc_validate(struct kbase_context *kctx, + struct base_jit_alloc_info *info) +{ + int j; + /* If the ID is zero, then fail the job */ + if (info->id == 0) + return -EINVAL; + + /* Sanity check that the PA fits within the VA */ + if (info->va_pages < info->commit_pages) + return -EINVAL; + + /* Ensure the GPU address is correctly aligned */ + if ((info->gpu_alloc_addr & KBASEP_JIT_ALLOC_GPU_ADDR_ALIGNMENT) != 0) + return -EINVAL; + + /* Interface version 2 (introduced with kernel driver version 11.5) + * onward has padding and a flags member to validate. + * + * Note: To support earlier versions the extra bytes will have been set + * to 0 by the caller. + */ + + /* Check padding is all zeroed */ + for (j = 0; j < sizeof(info->padding); j++) { + if (info->padding[j] != 0) + return -EINVAL; + } + + /* Only valid flags shall be set */ + if (info->flags & ~(BASE_JIT_ALLOC_VALID_FLAGS)) + return -EINVAL; + +#if !MALI_JIT_PRESSURE_LIMIT_BASE + /* If just-in-time memory allocation pressure limit feature is disabled, + * heap_info_gpu_addr must be zeroed-out + */ + if (info->heap_info_gpu_addr) + return -EINVAL; +#endif + +#if !MALI_USE_CSF + /* If BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE is set, heap_info_gpu_addr + * cannot be 0 + */ + if ((info->flags & BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) && + !info->heap_info_gpu_addr) + return -EINVAL; +#endif /* !MALI_USE_CSF */ + + return 0; +} + +#if !MALI_USE_CSF + +#if (KERNEL_VERSION(3, 18, 63) > LINUX_VERSION_CODE) +#define offsetofend(TYPE, MEMBER) \ + (offsetof(TYPE, MEMBER) + sizeof(((TYPE *)0)->MEMBER)) +#endif + +/* + * Sizes of user data to copy for each just-in-time memory interface version + * + * In interface version 2 onwards this is the same as the struct size, allowing + * copying of arrays of structures from userspace. + * + * In interface version 1 the structure size was variable, and hence arrays of + * structures cannot be supported easily, and were not a feature present in + * version 1 anyway. + */ +static const size_t jit_info_copy_size_for_jit_version[] = { + /* in jit_version 1, the structure did not have any end padding, hence + * it could be a different size on 32 and 64-bit clients. We therefore + * do not copy past the last member + */ + [1] = offsetofend(struct base_jit_alloc_info_10_2, id), + [2] = sizeof(struct base_jit_alloc_info_11_5), + [3] = sizeof(struct base_jit_alloc_info) +}; + +static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) +{ + __user u8 *data = (__user u8 *)(uintptr_t) katom->jc; + struct base_jit_alloc_info *info; + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + u32 count; + int ret; + u32 i; + size_t jit_info_user_copy_size; + + WARN_ON(kctx->jit_version >= + ARRAY_SIZE(jit_info_copy_size_for_jit_version)); + jit_info_user_copy_size = + jit_info_copy_size_for_jit_version[kctx->jit_version]; + WARN_ON(jit_info_user_copy_size > sizeof(*info)); + + /* For backwards compatibility, and to prevent reading more than 1 jit + * info struct on jit version 1 + */ + if (katom->nr_extres == 0 || kctx->jit_version == 1) + katom->nr_extres = 1; + count = katom->nr_extres; + + /* Sanity checks */ + if (!data || count > kctx->jit_max_allocations || + count > ARRAY_SIZE(kctx->jit_alloc)) { + ret = -EINVAL; + goto fail; + } + + /* Copy the information for safe access and future storage */ + info = kmalloc_array(count, sizeof(*info), GFP_KERNEL); + if (!info) { + ret = -ENOMEM; + goto fail; + } + + katom->softjob_data = info; + + for (i = 0; i < count; i++, info++, data += jit_info_user_copy_size) { + if (copy_from_user(info, data, jit_info_user_copy_size) != 0) { + ret = -EINVAL; + goto free_info; + } + /* Clear any remaining bytes when user struct is smaller than + * kernel struct. For jit version 1, this also clears the + * padding bytes + */ + memset(((u8 *)info) + jit_info_user_copy_size, 0, + sizeof(*info) - jit_info_user_copy_size); + + ret = kbasep_jit_alloc_validate(kctx, info); + if (ret) + goto free_info; + KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO(kbdev, katom, + info->va_pages, info->commit_pages, info->extent, + info->id, info->bin_id, info->max_allocations, + info->flags, info->usage_id); + } + + katom->jit_blocked = false; + + lockdep_assert_held(&kctx->jctx.lock); + list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head); + + /* + * Note: + * The provided info->gpu_alloc_addr isn't validated here as + * userland can cache allocations which means that even + * though the region is valid it doesn't represent the + * same thing it used to. + * + * Complete validation of va_pages, commit_pages and extent + * isn't done here as it will be done during the call to + * kbase_mem_alloc. + */ + return 0; + +free_info: + kfree(katom->softjob_data); + katom->softjob_data = NULL; +fail: + return ret; +} + +static u8 *kbase_jit_free_get_ids(struct kbase_jd_atom *katom) +{ + if (WARN_ON((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) != + BASE_JD_REQ_SOFT_JIT_FREE)) + return NULL; + + return (u8 *) katom->softjob_data; +} + +static void kbase_jit_add_to_pending_alloc_list(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct list_head *target_list_head = NULL; + struct kbase_jd_atom *entry; + + list_for_each_entry(entry, &kctx->jctx.jit_pending_alloc, queue) { + if (katom->age < entry->age) { + target_list_head = &entry->queue; + break; + } + } + + if (target_list_head == NULL) + target_list_head = &kctx->jctx.jit_pending_alloc; + + list_add_tail(&katom->queue, target_list_head); +} + +static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct base_jit_alloc_info *info; + struct kbase_va_region *reg; + struct kbase_vmap_struct mapping; + u64 *ptr, new_addr; + u32 count = katom->nr_extres; + u32 i; + bool ignore_pressure_limit = false; + + trace_sysgraph(SGR_SUBMIT, kctx->id, + kbase_jd_atom_id(kctx, katom)); + + if (katom->jit_blocked) { + list_del(&katom->queue); + katom->jit_blocked = false; + } + + info = katom->softjob_data; + if (WARN_ON(!info)) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return 0; + } + + for (i = 0; i < count; i++, info++) { + /* The JIT ID is still in use so fail the allocation */ + if (kctx->jit_alloc[info->id]) { + katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; + return 0; + } + } + +#if MALI_JIT_PRESSURE_LIMIT_BASE + /** + * If this is the only JIT_ALLOC atom in-flight or if JIT pressure limit + * is disabled at the context scope, then bypass JIT pressure limit + * logic in kbase_jit_allocate(). + */ + if (!kbase_ctx_flag(kctx, KCTX_JPL_ENABLED) + || (kctx->jit_current_allocations == 0)) { + ignore_pressure_limit = true; + } +#else + ignore_pressure_limit = true; +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + + for (i = 0, info = katom->softjob_data; i < count; i++, info++) { + if (kctx->jit_alloc[info->id]) { + /* The JIT ID is duplicated in this atom. Roll back + * previous allocations and fail. + */ + u32 j; + + info = katom->softjob_data; + for (j = 0; j < i; j++, info++) { + kbase_jit_free(kctx, kctx->jit_alloc[info->id]); + kctx->jit_alloc[info->id] = + KBASE_RESERVED_REG_JIT_ALLOC; + } + + katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; + return 0; + } + + /* Create a JIT allocation */ + reg = kbase_jit_allocate(kctx, info, ignore_pressure_limit); + if (!reg) { + struct kbase_jd_atom *jit_atom; + bool can_block = false; + + lockdep_assert_held(&kctx->jctx.lock); + + list_for_each_entry(jit_atom, &kctx->jctx.jit_atoms_head, jit_node) { + if (jit_atom == katom) + break; + + if ((jit_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == + BASE_JD_REQ_SOFT_JIT_FREE) { + u8 *free_ids = kbase_jit_free_get_ids(jit_atom); + + if (free_ids && *free_ids && + kctx->jit_alloc[*free_ids]) { + /* A JIT free which is active and + * submitted before this atom + */ + can_block = true; + break; + } + } + } + + if (!can_block) { + /* Mark the failed allocation as well as the + * other un-attempted allocations in the set, + * so we know they are in use even if the + * allocation itself failed. + */ + for (; i < count; i++, info++) { + kctx->jit_alloc[info->id] = + KBASE_RESERVED_REG_JIT_ALLOC; + } + + katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; + dev_warn_ratelimited(kbdev->dev, "JIT alloc softjob failed: atom id %d\n", + kbase_jd_atom_id(kctx, katom)); + return 0; + } + + /* There are pending frees for an active allocation + * so we should wait to see whether they free the + * memory. Add to the list of atoms for which JIT + * allocation is pending. + */ + kbase_jit_add_to_pending_alloc_list(katom); + katom->jit_blocked = true; + + /* Rollback, the whole set will be re-attempted */ + while (i-- > 0) { + info--; + kbase_jit_free(kctx, kctx->jit_alloc[info->id]); + kctx->jit_alloc[info->id] = NULL; + } + + return 1; + } + + /* Bind it to the user provided ID. */ + kctx->jit_alloc[info->id] = reg; + } + + for (i = 0, info = katom->softjob_data; i < count; i++, info++) { + u64 entry_mmu_flags = 0; + /* + * Write the address of the JIT allocation to the user provided + * GPU allocation. + */ + ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr), + &mapping); + if (!ptr) { + /* + * Leave the allocations "live" as the JIT free atom + * will be submitted anyway. + */ + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return 0; + } + + reg = kctx->jit_alloc[info->id]; + new_addr = reg->start_pfn << PAGE_SHIFT; + *ptr = new_addr; + +#if defined(CONFIG_MALI_VECTOR_DUMP) + /* + * Retrieve the mmu flags for JIT allocation + * only if dumping is enabled + */ + entry_mmu_flags = kbase_mmu_create_ate(kbdev, + (struct tagged_addr){ 0 }, reg->flags, + MIDGARD_MMU_BOTTOMLEVEL, kctx->jit_group_id); +#endif + + KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(kbdev, katom, + info->gpu_alloc_addr, new_addr, info->flags, + entry_mmu_flags, info->id, info->commit_pages, + info->extent, info->va_pages); + kbase_vunmap(kctx, &mapping); + + kbase_trace_jit_report_gpu_mem(kctx, reg, + KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); + } + + katom->event_code = BASE_JD_EVENT_DONE; + + return 0; +} + +static void kbase_jit_allocate_finish(struct kbase_jd_atom *katom) +{ + struct base_jit_alloc_info *info; + + lockdep_assert_held(&katom->kctx->jctx.lock); + + if (WARN_ON(!katom->softjob_data)) + return; + + /* Remove atom from jit_atoms_head list */ + list_del(&katom->jit_node); + + if (katom->jit_blocked) { + list_del(&katom->queue); + katom->jit_blocked = false; + } + + info = katom->softjob_data; + /* Free the info structure */ + kfree(info); +} + +static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + __user void *data = (__user void *)(uintptr_t) katom->jc; + u8 *ids; + u32 count = MAX(katom->nr_extres, 1); + u32 i; + int ret; + + /* Sanity checks */ + if (count > ARRAY_SIZE(kctx->jit_alloc)) { + ret = -EINVAL; + goto fail; + } + + /* Copy the information for safe access and future storage */ + ids = kmalloc_array(count, sizeof(*ids), GFP_KERNEL); + if (!ids) { + ret = -ENOMEM; + goto fail; + } + + lockdep_assert_held(&kctx->jctx.lock); + katom->softjob_data = ids; + + /* For backwards compatibility */ + if (katom->nr_extres) { + /* Fail the job if there is no list of ids */ + if (!data) { + ret = -EINVAL; + goto free_info; + } + + if (copy_from_user(ids, data, sizeof(*ids)*count) != 0) { + ret = -EINVAL; + goto free_info; + } + } else { + katom->nr_extres = 1; + *ids = (u8)katom->jc; + } + for (i = 0; i < count; i++) + KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO(kbdev, katom, ids[i]); + + list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head); + + return 0; + +free_info: + kfree(katom->softjob_data); + katom->softjob_data = NULL; +fail: + return ret; +} + +static void kbase_jit_free_process(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + u8 *ids = kbase_jit_free_get_ids(katom); + u32 count = katom->nr_extres; + u32 i; + + if (ids == NULL) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return; + } + + for (i = 0; i < count; i++, ids++) { + /* + * If the ID is zero or it is not in use yet then fail the job. + */ + if ((*ids == 0) || (kctx->jit_alloc[*ids] == NULL)) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return; + } + } +} + +static void kbasep_jit_finish_worker(struct work_struct *work) +{ + struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + int resched; + + mutex_lock(&kctx->jctx.lock); + kbase_finish_soft_job(katom); + resched = jd_done_nolock(katom, NULL); + mutex_unlock(&kctx->jctx.lock); + + if (resched) + kbase_js_sched_all(kctx->kbdev); +} + +void kbase_jit_retry_pending_alloc(struct kbase_context *kctx) +{ + LIST_HEAD(jit_pending_alloc_list); + struct list_head *i, *tmp; + + list_splice_tail_init(&kctx->jctx.jit_pending_alloc, + &jit_pending_alloc_list); + + list_for_each_safe(i, tmp, &jit_pending_alloc_list) { + struct kbase_jd_atom *pending_atom = list_entry(i, + struct kbase_jd_atom, queue); + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kctx->kbdev, pending_atom); + kbase_kinstr_jm_atom_sw_start(pending_atom); + if (kbase_jit_allocate_process(pending_atom) == 0) { + /* Atom has completed */ + INIT_WORK(&pending_atom->work, + kbasep_jit_finish_worker); + queue_work(kctx->jctx.job_done_wq, &pending_atom->work); + } + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END(kctx->kbdev, pending_atom); + kbase_kinstr_jm_atom_sw_stop(pending_atom); + } +} + +static void kbase_jit_free_finish(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + u8 *ids; + size_t j; + + lockdep_assert_held(&kctx->jctx.lock); + + ids = kbase_jit_free_get_ids(katom); + if (WARN_ON(ids == NULL)) { + return; + } + + /* Remove this atom from the jit_atoms_head list */ + list_del(&katom->jit_node); + + for (j = 0; j != katom->nr_extres; ++j) { + if ((ids[j] != 0) && (kctx->jit_alloc[ids[j]] != NULL)) { + /* + * If the ID is valid but the allocation request failed + * still succeed this soft job but don't try and free + * the allocation. + */ + if (kctx->jit_alloc[ids[j]] != + KBASE_RESERVED_REG_JIT_ALLOC) { + KBASE_TLSTREAM_TL_JIT_USEDPAGES(kctx->kbdev, + kctx->jit_alloc[ids[j]]-> + gpu_alloc->nents, ids[j]); + kbase_jit_free(kctx, kctx->jit_alloc[ids[j]]); + } + kctx->jit_alloc[ids[j]] = NULL; + } + } + /* Free the list of ids */ + kfree(ids); + + kbase_jit_retry_pending_alloc(kctx); +} + +static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) +{ + __user struct base_external_resource_list *user_ext_res; + struct base_external_resource_list *ext_res; + u64 count = 0; + size_t copy_size; + int ret; + + user_ext_res = (__user struct base_external_resource_list *) + (uintptr_t) katom->jc; + + /* Fail the job if there is no info structure */ + if (!user_ext_res) { + ret = -EINVAL; + goto fail; + } + + if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) { + ret = -EINVAL; + goto fail; + } + + /* Is the number of external resources in range? */ + if (!count || count > BASE_EXT_RES_COUNT_MAX) { + ret = -EINVAL; + goto fail; + } + + /* Copy the information for safe access and future storage */ + copy_size = sizeof(*ext_res); + copy_size += sizeof(struct base_external_resource) * (count - 1); + ext_res = kzalloc(copy_size, GFP_KERNEL); + if (!ext_res) { + ret = -ENOMEM; + goto fail; + } + + if (copy_from_user(ext_res, user_ext_res, copy_size) != 0) { + ret = -EINVAL; + goto free_info; + } + + /* + * Overwrite the count with the first value incase it was changed + * after the fact. + */ + ext_res->count = count; + + katom->softjob_data = ext_res; + + return 0; + +free_info: + kfree(ext_res); +fail: + return ret; +} + +static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) +{ + struct base_external_resource_list *ext_res; + int i; + bool failed = false; + + ext_res = katom->softjob_data; + if (!ext_res) + goto failed_jc; + + kbase_gpu_vm_lock(katom->kctx); + + for (i = 0; i < ext_res->count; i++) { + u64 gpu_addr; + + gpu_addr = ext_res->ext_res[i].ext_resource & + ~BASE_EXT_RES_ACCESS_EXCLUSIVE; + if (map) { + if (!kbase_sticky_resource_acquire(katom->kctx, + gpu_addr)) + goto failed_loop; + } else + if (!kbase_sticky_resource_release_force(katom->kctx, NULL, + gpu_addr)) + failed = true; + } + + /* + * In the case of unmap we continue unmapping other resources in the + * case of failure but will always report failure if _any_ unmap + * request fails. + */ + if (failed) + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + else + katom->event_code = BASE_JD_EVENT_DONE; + + kbase_gpu_vm_unlock(katom->kctx); + + return; + +failed_loop: + while (i > 0) { + u64 const gpu_addr = ext_res->ext_res[i - 1].ext_resource & + ~BASE_EXT_RES_ACCESS_EXCLUSIVE; + + kbase_sticky_resource_release_force(katom->kctx, NULL, gpu_addr); + + --i; + } + + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + kbase_gpu_vm_unlock(katom->kctx); + +failed_jc: + return; +} + +static void kbase_ext_res_finish(struct kbase_jd_atom *katom) +{ + struct base_external_resource_list *ext_res; + + ext_res = katom->softjob_data; + /* Free the info structure */ + kfree(ext_res); +} + +int kbase_process_soft_job(struct kbase_jd_atom *katom) +{ + int ret = 0; + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kbdev, katom); + kbase_kinstr_jm_atom_sw_start(katom); + + trace_sysgraph(SGR_SUBMIT, kctx->id, + kbase_jd_atom_id(kctx, katom)); + + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: + ret = kbase_dump_cpu_gpu_time(katom); + break; + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + katom->event_code = kbase_sync_fence_out_trigger(katom, + katom->event_code == BASE_JD_EVENT_DONE ? + 0 : -EFAULT); + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + { + ret = kbase_sync_fence_in_wait(katom); + + if (ret == 1) { +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG + kbasep_add_waiting_with_timeout(katom); +#else + kbasep_add_waiting_soft_job(katom); +#endif + } + break; + } +#endif + case BASE_JD_REQ_SOFT_EVENT_WAIT: + ret = kbasep_soft_event_wait(katom); + break; + case BASE_JD_REQ_SOFT_EVENT_SET: + kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_SET); + break; + case BASE_JD_REQ_SOFT_EVENT_RESET: + kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_RESET); + break; + case BASE_JD_REQ_SOFT_DEBUG_COPY: + { + int res = kbase_debug_copy(katom); + + if (res) + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + break; + } + case BASE_JD_REQ_SOFT_JIT_ALLOC: + ret = kbase_jit_allocate_process(katom); + break; + case BASE_JD_REQ_SOFT_JIT_FREE: + kbase_jit_free_process(katom); + break; + case BASE_JD_REQ_SOFT_EXT_RES_MAP: + kbase_ext_res_process(katom, true); + break; + case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: + kbase_ext_res_process(katom, false); + break; + } + + /* Atom is complete */ + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END(kbdev, katom); + kbase_kinstr_jm_atom_sw_stop(katom); + return ret; +} + +void kbase_cancel_soft_job(struct kbase_jd_atom *katom) +{ + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_WAIT: + kbase_sync_fence_in_cancel_wait(katom); + break; +#endif + case BASE_JD_REQ_SOFT_EVENT_WAIT: + kbasep_soft_event_cancel_job(katom); + break; + default: + /* This soft-job doesn't support cancellation! */ + KBASE_DEBUG_ASSERT(0); + } +} + +int kbase_prepare_soft_job(struct kbase_jd_atom *katom) +{ + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: + { + if (!IS_ALIGNED(katom->jc, cache_line_size())) + return -EINVAL; + } + break; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + { + struct base_fence fence; + int fd; + + if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) + return -EINVAL; + + fd = kbase_sync_fence_out_create(katom, + fence.basep.stream_fd); + if (fd < 0) + return -EINVAL; + + fence.basep.fd = fd; + if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) { + kbase_sync_fence_out_remove(katom); + kbase_sync_fence_close_fd(fd); + fence.basep.fd = -EINVAL; + return -EINVAL; + } + } + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + { + struct base_fence fence; + int ret; + + if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) + return -EINVAL; + + /* Get a reference to the fence object */ + ret = kbase_sync_fence_in_from_fd(katom, + fence.basep.fd); + if (ret < 0) + return ret; + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + /* + * Set KCTX_NO_IMPLICIT_FENCE in the context the first + * time a soft fence wait job is observed. This will + * prevent the implicit dma-buf fence to conflict with + * the Android native sync fences. + */ + if (!kbase_ctx_flag(katom->kctx, KCTX_NO_IMPLICIT_SYNC)) + kbase_ctx_flag_set(katom->kctx, KCTX_NO_IMPLICIT_SYNC); +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + } + break; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + case BASE_JD_REQ_SOFT_JIT_ALLOC: + return kbase_jit_allocate_prepare(katom); + case BASE_JD_REQ_SOFT_JIT_FREE: + return kbase_jit_free_prepare(katom); + case BASE_JD_REQ_SOFT_EVENT_WAIT: + case BASE_JD_REQ_SOFT_EVENT_SET: + case BASE_JD_REQ_SOFT_EVENT_RESET: + if (katom->jc == 0) + return -EINVAL; + break; + case BASE_JD_REQ_SOFT_DEBUG_COPY: + return kbase_debug_copy_prepare(katom); + case BASE_JD_REQ_SOFT_EXT_RES_MAP: + return kbase_ext_res_prepare(katom); + case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: + return kbase_ext_res_prepare(katom); + default: + /* Unsupported soft-job */ + return -EINVAL; + } + return 0; +} + +void kbase_finish_soft_job(struct kbase_jd_atom *katom) +{ + trace_sysgraph(SGR_COMPLETE, katom->kctx->id, + kbase_jd_atom_id(katom->kctx, katom)); + + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: + /* Nothing to do */ + break; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + /* If fence has not yet been signaled, do it now */ + kbase_sync_fence_out_trigger(katom, katom->event_code == + BASE_JD_EVENT_DONE ? 0 : -EFAULT); + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + /* Release katom's reference to fence object */ + kbase_sync_fence_in_remove(katom); + break; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + case BASE_JD_REQ_SOFT_DEBUG_COPY: + kbase_debug_copy_finish(katom); + break; + case BASE_JD_REQ_SOFT_JIT_ALLOC: + kbase_jit_allocate_finish(katom); + break; + case BASE_JD_REQ_SOFT_EXT_RES_MAP: + kbase_ext_res_finish(katom); + break; + case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: + kbase_ext_res_finish(katom); + break; + case BASE_JD_REQ_SOFT_JIT_FREE: + kbase_jit_free_finish(katom); + break; + } +} + +void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) +{ + LIST_HEAD(local_suspended_soft_jobs); + struct kbase_jd_atom *tmp_iter; + struct kbase_jd_atom *katom_iter; + struct kbasep_js_device_data *js_devdata; + bool resched = false; + + KBASE_DEBUG_ASSERT(kbdev); + + js_devdata = &kbdev->js_data; + + /* Move out the entire list */ + mutex_lock(&js_devdata->runpool_mutex); + list_splice_init(&js_devdata->suspended_soft_jobs_list, + &local_suspended_soft_jobs); + mutex_unlock(&js_devdata->runpool_mutex); + + /* + * Each atom must be detached from the list and ran separately - + * it could be re-added to the old list, but this is unlikely + */ + list_for_each_entry_safe(katom_iter, tmp_iter, + &local_suspended_soft_jobs, dep_item[1]) { + struct kbase_context *kctx = katom_iter->kctx; + + mutex_lock(&kctx->jctx.lock); + + /* Remove from the global list */ + list_del(&katom_iter->dep_item[1]); + /* Remove from the context's list of waiting soft jobs */ + kbasep_remove_waiting_soft_job(katom_iter); + + if (kbase_process_soft_job(katom_iter) == 0) { + kbase_finish_soft_job(katom_iter); + resched |= jd_done_nolock(katom_iter, NULL); + } + mutex_unlock(&kctx->jctx.lock); + } + + if (resched) + kbase_js_sched_all(kbdev); +} +#endif /* !MALI_USE_CSF */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_strings.c b/drivers/gpu/arm/bifrost/mali_kbase_strings.c new file mode 100755 index 000000000000..22caa4a6d814 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_strings.c @@ -0,0 +1,28 @@ + /* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ +#include "mali_kbase_strings.h" + +#define KBASE_DRV_NAME "mali" +#define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline" + +const char kbase_drv_name[] = KBASE_DRV_NAME; +const char kbase_timeline_name[] = KBASE_TIMELINE_NAME; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_strings.h b/drivers/gpu/arm/bifrost/mali_kbase_strings.h new file mode 100755 index 000000000000..d2f1825314fe --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_strings.h @@ -0,0 +1,24 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +extern const char kbase_drv_name[]; +extern const char kbase_timeline_name[]; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_sync.h b/drivers/gpu/arm/bifrost/mali_kbase_sync.h new file mode 100755 index 000000000000..4e5ab3ca557a --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_sync.h @@ -0,0 +1,231 @@ +/* + * + * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file mali_kbase_sync.h + * + * This file contains our internal "API" for explicit fences. + * It hides the implementation details of the actual explicit fence mechanism + * used (Android fences or sync file with DMA fences). + */ + +#ifndef MALI_KBASE_SYNC_H +#define MALI_KBASE_SYNC_H + +#include +#ifdef CONFIG_SYNC +#include +#endif +#ifdef CONFIG_SYNC_FILE +#include "mali_kbase_fence_defs.h" +#include +#endif + +#include "mali_kbase.h" + +/** + * struct kbase_sync_fence_info - Information about a fence + * @fence: Pointer to fence (type is void*, as underlaying struct can differ) + * @name: The name given to this fence when it was created + * @status: < 0 means error, 0 means active, 1 means signaled + * + * Use kbase_sync_fence_in_info_get() or kbase_sync_fence_out_info_get() + * to get the information. + */ +struct kbase_sync_fence_info { + void *fence; + char name[32]; + int status; +}; + +/** + * kbase_sync_fence_stream_create() - Create a stream object + * @name: Name of stream (only used to ease debugging/visualization) + * @out_fd: A file descriptor representing the created stream object + * + * Can map down to a timeline implementation in some implementations. + * Exposed as a file descriptor. + * Life-time controlled via the file descriptor: + * - dup to add a ref + * - close to remove a ref + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_stream_create(const char *name, int *const out_fd); + +#if !MALI_USE_CSF +/** + * kbase_sync_fence_out_create Create an explicit output fence to specified atom + * @katom: Atom to assign the new explicit fence to + * @stream_fd: File descriptor for stream object to create fence on + * + * return: Valid file descriptor to fence or < 0 on error + */ +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); + +/** + * kbase_sync_fence_in_from_fd() Assigns an existing fence to specified atom + * @katom: Atom to assign the existing explicit fence to + * @fd: File descriptor to an existing fence + * + * Assigns an explicit input fence to atom. + * This can later be waited for by calling @kbase_sync_fence_in_wait + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); +#endif /* !MALI_USE_CSF */ + +/** + * kbase_sync_fence_validate() - Validate a fd to be a valid fence + * @fd: File descriptor to check + * + * This function is only usable to catch unintentional user errors early, + * it does not stop malicious code changing the fd after this function returns. + * + * return 0: if fd is for a valid fence, < 0 if invalid + */ +int kbase_sync_fence_validate(int fd); + +#if !MALI_USE_CSF +/** + * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom + * @katom: Atom with an explicit fence to signal + * @result: < 0 means signal with error, 0 >= indicates success + * + * Signal output fence attached on katom and remove the fence from the atom. + * + * return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE + */ +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result); + +/** + * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled + * @katom: Atom with explicit fence to wait for + * + * If the fence is already signaled, then 0 is returned, and the caller must + * continue processing of the katom. + * + * If the fence isn't already signaled, then this kbase_sync framework will + * take responsibility to continue the processing once the fence is signaled. + * + * return: 0 if already signaled, otherwise 1 + */ +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits + * @katom: Atom to cancel wait for + * + * This function is fully responsible for continuing processing of this atom + * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all) + */ +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_in_remove() - Remove the input fence from the katom + * @katom: Atom to remove explicit input fence for + * + * This will also release the corresponding reference. + */ +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_out_remove() - Remove the output fence from the katom + * @katom: Atom to remove explicit output fence for + * + * This will also release the corresponding reference. + */ +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); +#endif /* !MALI_USE_CSF */ + +/** + * kbase_sync_fence_close_fd() - Close a file descriptor representing a fence + * @fd: File descriptor to close + */ +static inline void kbase_sync_fence_close_fd(int fd) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + ksys_close(fd); +#else + sys_close(fd); +#endif +} + +#if !MALI_USE_CSF +/** + * kbase_sync_fence_in_info_get() - Retrieves information about input fence + * @katom: Atom to get fence information from + * @info: Struct to be filled with fence information + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info); + +/** + * kbase_sync_fence_out_info_get() - Retrieves information about output fence + * @katom: Atom to get fence information from + * @info: Struct to be filled with fence information + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info); +#endif /* !MALI_USE_CSF */ + +#if defined(CONFIG_SYNC_FILE) +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) +void kbase_sync_fence_info_get(struct fence *fence, + struct kbase_sync_fence_info *info); +#else +void kbase_sync_fence_info_get(struct dma_fence *fence, + struct kbase_sync_fence_info *info); +#endif +#endif + +/** + * kbase_sync_status_string() - Get string matching @status + * @status: Value of fence status. + * + * return: Pointer to string describing @status. + */ +const char *kbase_sync_status_string(int status); + + +#if !MALI_USE_CSF +/* + * Internal worker used to continue processing of atom. + */ +void kbase_sync_fence_wait_worker(struct work_struct *data); + +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG +/** + * kbase_sync_fence_in_dump() Trigger a debug dump of atoms input fence state + * @katom: Atom to trigger fence debug dump for + */ +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); +#endif +#endif /* !MALI_USE_CSF */ + +#endif /* MALI_KBASE_SYNC_H */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_sync_android.c b/drivers/gpu/arm/bifrost/mali_kbase_sync_android.c new file mode 100755 index 000000000000..41f740a7bc8c --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_sync_android.c @@ -0,0 +1,542 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Code for supporting explicit Android fences (CONFIG_SYNC) + * Known to be good for kernels 4.5 and earlier. + * Replaced with CONFIG_SYNC_FILE for 4.9 and later kernels + * (see mali_kbase_sync_file.c) + */ + +#include +#include +#include +#include +#include +#include +#include +#include "sync.h" +#include +#include + +struct mali_sync_timeline { + struct sync_timeline timeline; + atomic_t counter; + atomic_t signaled; +}; + +struct mali_sync_pt { + struct sync_pt pt; + int order; + int result; +}; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +/* For backwards compatibility with kernels before 3.17. After 3.17 + * sync_pt_parent is included in the kernel. */ +static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) +{ + return pt->parent; +} +#endif + +static struct mali_sync_timeline *to_mali_sync_timeline( + struct sync_timeline *timeline) +{ + return container_of(timeline, struct mali_sync_timeline, timeline); +} + +static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) +{ + return container_of(pt, struct mali_sync_pt, pt); +} + +static struct sync_pt *timeline_dup(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_pt *new_mpt; + struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), + sizeof(struct mali_sync_pt)); + + if (!new_pt) + return NULL; + + new_mpt = to_mali_sync_pt(new_pt); + new_mpt->order = mpt->order; + new_mpt->result = mpt->result; + + return new_pt; +} + +static int timeline_has_signaled(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_timeline *mtl = to_mali_sync_timeline( + sync_pt_parent(pt)); + int result = mpt->result; + + int diff = atomic_read(&mtl->signaled) - mpt->order; + + if (diff >= 0) + return (result < 0) ? result : 1; + + return 0; +} + +static int timeline_compare(struct sync_pt *a, struct sync_pt *b) +{ + struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt); + struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt); + + int diff = ma->order - mb->order; + + if (diff == 0) + return 0; + + return (diff < 0) ? -1 : 1; +} + +static void timeline_value_str(struct sync_timeline *timeline, char *str, + int size) +{ + struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline); + + snprintf(str, size, "%d", atomic_read(&mtl->signaled)); +} + +static void pt_value_str(struct sync_pt *pt, char *str, int size) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + + snprintf(str, size, "%d(%d)", mpt->order, mpt->result); +} + +static struct sync_timeline_ops mali_timeline_ops = { + .driver_name = "Mali", + .dup = timeline_dup, + .has_signaled = timeline_has_signaled, + .compare = timeline_compare, + .timeline_value_str = timeline_value_str, + .pt_value_str = pt_value_str, +}; + +/* Allocates a timeline for Mali + * + * One timeline should be allocated per API context. + */ +static struct sync_timeline *mali_sync_timeline_alloc(const char *name) +{ + struct sync_timeline *tl; + struct mali_sync_timeline *mtl; + + tl = sync_timeline_create(&mali_timeline_ops, + sizeof(struct mali_sync_timeline), name); + if (!tl) + return NULL; + + /* Set the counter in our private struct */ + mtl = to_mali_sync_timeline(tl); + atomic_set(&mtl->counter, 0); + atomic_set(&mtl->signaled, 0); + + return tl; +} + +static int kbase_stream_close(struct inode *inode, struct file *file) +{ + struct sync_timeline *tl; + + tl = (struct sync_timeline *)file->private_data; + sync_timeline_destroy(tl); + return 0; +} + +static const struct file_operations stream_fops = { + .owner = THIS_MODULE, + .release = kbase_stream_close, +}; + +int kbase_sync_fence_stream_create(const char *name, int *const out_fd) +{ + struct sync_timeline *tl; + + if (!out_fd) + return -EINVAL; + + tl = mali_sync_timeline_alloc(name); + if (!tl) + return -EINVAL; + + *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY|O_CLOEXEC); + + if (*out_fd < 0) { + sync_timeline_destroy(tl); + return -EINVAL; + } + + return 0; +} + +/* Allocates a sync point within the timeline. + * + * The timeline must be the one allocated by kbase_sync_timeline_alloc + * + * Sync points must be triggered in *exactly* the same order as they are + * allocated. + */ +static struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent) +{ + struct sync_pt *pt = sync_pt_create(parent, + sizeof(struct mali_sync_pt)); + struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent); + struct mali_sync_pt *mpt; + + if (!pt) + return NULL; + + mpt = to_mali_sync_pt(pt); + mpt->order = atomic_inc_return(&mtl->counter); + mpt->result = 0; + + return pt; +} + +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) +{ + struct sync_timeline *tl; + struct sync_pt *pt; + struct sync_fence *fence; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) + struct files_struct *files; + struct fdtable *fdt; +#endif + int fd; + struct file *tl_file; + + tl_file = fget(tl_fd); + if (tl_file == NULL) + return -EBADF; + + if (tl_file->f_op != &stream_fops) { + fd = -EBADF; + goto out; + } + + tl = tl_file->private_data; + + pt = kbase_sync_pt_alloc(tl); + if (!pt) { + fd = -EFAULT; + goto out; + } + + fence = sync_fence_create("mali_fence", pt); + if (!fence) { + sync_pt_free(pt); + fd = -EFAULT; + goto out; + } + + /* from here the fence owns the sync_pt */ + + /* create a fd representing the fence */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) + fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); + if (fd < 0) { + sync_fence_put(fence); + goto out; + } +#else + fd = get_unused_fd(); + if (fd < 0) { + sync_fence_put(fence); + goto out; + } + + files = current->files; + spin_lock(&files->file_lock); + fdt = files_fdtable(files); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) + __set_close_on_exec(fd, fdt); +#else + FD_SET(fd, fdt->close_on_exec); +#endif + spin_unlock(&files->file_lock); +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ + + /* bind fence to the new fd */ + sync_fence_install(fence, fd); + + katom->fence = sync_fence_fdget(fd); + if (katom->fence == NULL) { + /* The only way the fence can be NULL is if userspace closed it + * for us, so we don't need to clear it up */ + fd = -EINVAL; + goto out; + } + +out: + fput(tl_file); + + return fd; +} + +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) +{ + katom->fence = sync_fence_fdget(fd); + return katom->fence ? 0 : -ENOENT; +} + +int kbase_sync_fence_validate(int fd) +{ + struct sync_fence *fence; + + fence = sync_fence_fdget(fd); + if (!fence) + return -EINVAL; + + sync_fence_put(fence); + return 0; +} + +/* Returns true if the specified timeline is allocated by Mali */ +static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) +{ + return timeline->ops == &mali_timeline_ops; +} + +/* Signals a particular sync point + * + * Sync points must be triggered in *exactly* the same order as they are + * allocated. + * + * If they are signaled in the wrong order then a message will be printed in + * debug builds and otherwise attempts to signal order sync_pts will be ignored. + * + * result can be negative to indicate error, any other value is interpreted as + * success. + */ +static void kbase_sync_signal_pt(struct sync_pt *pt, int result) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_timeline *mtl = to_mali_sync_timeline( + sync_pt_parent(pt)); + int signaled; + int diff; + + mpt->result = result; + + do { + signaled = atomic_read(&mtl->signaled); + + diff = signaled - mpt->order; + + if (diff > 0) { + /* The timeline is already at or ahead of this point. + * This should not happen unless userspace has been + * signaling fences out of order, so warn but don't + * violate the sync_pt API. + * The warning is only in debug builds to prevent + * a malicious user being able to spam dmesg. + */ +#ifdef CONFIG_MALI_BIFROST_DEBUG + pr_err("Fences were triggered in a different order to allocation!"); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + return; + } + } while (atomic_cmpxchg(&mtl->signaled, + signaled, mpt->order) != signaled); +} + +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) +{ + struct sync_pt *pt; + struct sync_timeline *timeline; + + if (!katom->fence) + return BASE_JD_EVENT_JOB_CANCELLED; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + if (!list_is_singular(&katom->fence->pt_list_head)) { +#else + if (katom->fence->num_fences != 1) { +#endif + /* Not exactly one item in the list - so it didn't (directly) + * come from us */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + pt = list_first_entry(&katom->fence->pt_list_head, + struct sync_pt, pt_list); +#else + pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); +#endif + timeline = sync_pt_parent(pt); + + if (!kbase_sync_timeline_is_ours(timeline)) { + /* Fence has a sync_pt which isn't ours! */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + + kbase_sync_signal_pt(pt, result); + + sync_timeline_signal(timeline); + + kbase_sync_fence_out_remove(katom); + + return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; +} + +static inline int kbase_fence_get_status(struct sync_fence *fence) +{ + if (!fence) + return -ENOENT; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + return fence->status; +#else + return atomic_read(&fence->status); +#endif +} + +static void kbase_fence_wait_callback(struct sync_fence *fence, + struct sync_fence_waiter *waiter) +{ + struct kbase_jd_atom *katom = container_of(waiter, + struct kbase_jd_atom, sync_waiter); + struct kbase_context *kctx = katom->kctx; + + /* Propagate the fence status to the atom. + * If negative then cancel this atom and its dependencies. + */ + if (kbase_fence_get_status(fence) < 0) + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* To prevent a potential deadlock we schedule the work onto the + * job_done_wq workqueue + * + * The issue is that we may signal the timeline while holding + * kctx->jctx.lock and the callbacks are run synchronously from + * sync_timeline_signal. So we simply defer the work. + */ + + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) +{ + int ret; + + sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback); + + ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter); + + if (ret == 1) { + /* Already signaled */ + return 0; + } + + if (ret < 0) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + /* We should cause the dependent jobs in the bag to be failed, + * to do this we schedule the work queue to complete this job */ + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(katom->kctx->jctx.job_done_wq, &katom->work); + } + + return 1; +} + +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) +{ + if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { + /* The wait wasn't cancelled - leave the cleanup for + * kbase_fence_wait_callback */ + return; + } + + /* Wait was cancelled - zap the atoms */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) +{ + if (katom->fence) { + sync_fence_put(katom->fence); + katom->fence = NULL; + } +} + +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) +{ + if (katom->fence) { + sync_fence_put(katom->fence); + katom->fence = NULL; + } +} + +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ + if (!katom->fence) + return -ENOENT; + + info->fence = katom->fence; + info->status = kbase_fence_get_status(katom->fence); + strlcpy(info->name, katom->fence->name, sizeof(info->name)); + + return 0; +} + +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ + if (!katom->fence) + return -ENOENT; + + info->fence = katom->fence; + info->status = kbase_fence_get_status(katom->fence); + strlcpy(info->name, katom->fence->name, sizeof(info->name)); + + return 0; +} + +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) +{ + /* Dump out the full state of all the Android sync fences. + * The function sync_dump() isn't exported to modules, so force + * sync_fence_wait() to time out to trigger sync_dump(). + */ + if (katom->fence) + sync_fence_wait(katom->fence, 1); +} +#endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase_sync_common.c b/drivers/gpu/arm/bifrost/mali_kbase_sync_common.c new file mode 100755 index 000000000000..866894bd0f94 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_sync_common.c @@ -0,0 +1,51 @@ +/* + * + * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * @file mali_kbase_sync_common.c + * + * Common code for our explicit fence functionality + */ + +#include +#include "mali_kbase.h" +#include "mali_kbase_sync.h" + +#if !MALI_USE_CSF +void kbase_sync_fence_wait_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom; + + katom = container_of(data, struct kbase_jd_atom, work); + kbase_soft_event_wait_callback(katom); +} +#endif /* !MALI_USE_CSF */ + +const char *kbase_sync_status_string(int status) +{ + if (status == 0) + return "active"; + else if (status > 0) + return "signaled"; + else + return "error"; +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_sync_file.c b/drivers/gpu/arm/bifrost/mali_kbase_sync_file.c new file mode 100755 index 000000000000..271873b9fe29 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_sync_file.c @@ -0,0 +1,372 @@ +/* + * + * (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Code for supporting explicit Linux fences (CONFIG_SYNC_FILE) + * Introduced in kernel 4.9. + * Android explicit fences (CONFIG_SYNC) can be used for older kernels + * (see mali_kbase_sync_android.c) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mali_kbase_fence_defs.h" +#include "mali_kbase_sync.h" +#include "mali_kbase_fence.h" +#include "mali_kbase.h" + +static const struct file_operations stream_fops = { + .owner = THIS_MODULE +}; + +int kbase_sync_fence_stream_create(const char *name, int *const out_fd) +{ + if (!out_fd) + return -EINVAL; + + *out_fd = anon_inode_getfd(name, &stream_fops, NULL, + O_RDONLY | O_CLOEXEC); + if (*out_fd < 0) + return -EINVAL; + + return 0; +} + +#if !MALI_USE_CSF +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + struct sync_file *sync_file; + int fd; + + fence = kbase_fence_out_new(katom); + if (!fence) + return -ENOMEM; + +#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) + /* Take an extra reference to the fence on behalf of the sync_file. + * This is only needed on older kernels where sync_file_create() + * does not take its own reference. This was changed in v4.9.68, + * where sync_file_create() now takes its own reference. + */ + dma_fence_get(fence); +#endif + + /* create a sync_file fd representing the fence */ + sync_file = sync_file_create(fence); + if (!sync_file) { +#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) + dma_fence_put(fence); +#endif + kbase_fence_out_remove(katom); + return -ENOMEM; + } + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + fput(sync_file->file); + kbase_fence_out_remove(katom); + return fd; + } + + fd_install(fd, sync_file->file); + + return fd; +} + +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = sync_file_get_fence(fd); +#else + struct dma_fence *fence = sync_file_get_fence(fd); +#endif + + if (!fence) + return -ENOENT; + + kbase_fence_fence_in_set(katom, fence); + + return 0; +} +#endif /* !MALI_USE_CSF */ + +int kbase_sync_fence_validate(int fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = sync_file_get_fence(fd); +#else + struct dma_fence *fence = sync_file_get_fence(fd); +#endif + + if (!fence) + return -EINVAL; + + dma_fence_put(fence); + + return 0; /* valid */ +} + +#if !MALI_USE_CSF +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) +{ + int res; + + if (!kbase_fence_out_is_ours(katom)) { + /* Not our fence */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + + res = kbase_fence_out_signal(katom, result); + if (unlikely(res < 0)) { + dev_warn(katom->kctx->kbdev->dev, + "fence_signal() failed with %d\n", res); + } + + kbase_sync_fence_out_remove(katom); + + return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +static void kbase_fence_wait_callback(struct fence *fence, + struct fence_cb *cb) +#else +static void kbase_fence_wait_callback(struct dma_fence *fence, + struct dma_fence_cb *cb) +#endif +{ + struct kbase_fence_cb *kcb = container_of(cb, + struct kbase_fence_cb, + fence_cb); + struct kbase_jd_atom *katom = kcb->katom; + struct kbase_context *kctx = katom->kctx; + + /* Cancel atom if fence is erroneous */ +#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ + (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ + KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) + if (dma_fence_is_signaled(kcb->fence) && kcb->fence->error < 0) +#else + if (dma_fence_is_signaled(kcb->fence) && kcb->fence->status < 0) +#endif + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + if (kbase_fence_dep_count_dec_and_test(katom)) { + /* We take responsibility of handling this */ + kbase_fence_dep_count_set(katom, -1); + + /* To prevent a potential deadlock we schedule the work onto the + * job_done_wq workqueue + * + * The issue is that we may signal the timeline while holding + * kctx->jctx.lock and the callbacks are run synchronously from + * sync_timeline_signal. So we simply defer the work. + */ + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); + } +} + +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) +{ + int err; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_in_get(katom); + if (!fence) + return 0; /* no input fence to wait for, good to go! */ + + kbase_fence_dep_count_set(katom, 1); + + err = kbase_fence_add_callback(katom, fence, kbase_fence_wait_callback); + + kbase_fence_put(fence); + + if (likely(!err)) { + /* Test if the callbacks are already triggered */ + if (kbase_fence_dep_count_dec_and_test(katom)) { + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + return 0; /* Already signaled, good to go right now */ + } + + /* Callback installed, so we just need to wait for it... */ + } else { + /* Failure */ + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* We should cause the dependent jobs in the bag to be failed, + * to do this we schedule the work queue to complete this job */ + + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(katom->kctx->jctx.job_done_wq, &katom->work); + } + + return 1; /* completion to be done later by callback/worker */ +} + +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) +{ + if (!kbase_fence_free_callbacks(katom)) { + /* The wait wasn't cancelled - + * leave the cleanup for kbase_fence_wait_callback */ + return; + } + + /* Take responsibility of completion */ + kbase_fence_dep_count_set(katom, -1); + + /* Wait was cancelled - zap the atoms */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) +{ + kbase_fence_out_remove(katom); +} + +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) +{ + kbase_fence_free_callbacks(katom); + kbase_fence_in_remove(katom); +} +#endif /* !MALI_USE_CSF */ + +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) +void kbase_sync_fence_info_get(struct fence *fence, + struct kbase_sync_fence_info *info) +#else +void kbase_sync_fence_info_get(struct dma_fence *fence, + struct kbase_sync_fence_info *info) +#endif +{ + info->fence = fence; + + /* translate into CONFIG_SYNC status: + * < 0 : error + * 0 : active + * 1 : signaled + */ + if (dma_fence_is_signaled(fence)) { +#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ + (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ + KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) + int status = fence->error; +#else + int status = fence->status; +#endif + if (status < 0) + info->status = status; /* signaled with error */ + else + info->status = 1; /* signaled with success */ + } else { + info->status = 0; /* still active (unsignaled) */ + } + +#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE) + scnprintf(info->name, sizeof(info->name), "%u#%u", + fence->context, fence->seqno); +#elif (KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE) + scnprintf(info->name, sizeof(info->name), "%llu#%u", + fence->context, fence->seqno); +#else + scnprintf(info->name, sizeof(info->name), "%llu#%llu", + fence->context, fence->seqno); +#endif +} + +#if !MALI_USE_CSF +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_in_get(katom); + if (!fence) + return -ENOENT; + + kbase_sync_fence_info_get(fence, info); + + kbase_fence_put(fence); + + return 0; +} + +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_out_get(katom); + if (!fence) + return -ENOENT; + + kbase_sync_fence_info_get(fence, info); + + kbase_fence_put(fence); + + return 0; +} + + +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) +{ + /* Not implemented */ +} +#endif +#endif /* !MALI_USE_CSF*/ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.c b/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.c new file mode 100755 index 000000000000..7669895b3c5d --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.c @@ -0,0 +1,227 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include + +/** + * struct kbase_dma_buf - Object instantiated when a dma-buf imported allocation + * is mapped to GPU for the first time within a process. + * Another instantiation is done for the case when that + * allocation is mapped for the first time to GPU. + * + * @dma_buf: Reference to dma_buf been imported. + * @dma_buf_node: Link node to maintain a rb_tree of kbase_dma_buf. + * @import_count: The number of times the dma_buf was imported. + */ +struct kbase_dma_buf { + struct dma_buf *dma_buf; + struct rb_node dma_buf_node; + u32 import_count; +}; + +/** + * kbase_delete_dma_buf_mapping - Delete a dma buffer mapping. + * + * @kctx: Pointer to kbase context. + * @dma_buf: Pointer to a dma buffer mapping. + * @tree: Pointer to root of rb_tree containing the dma_buf's mapped. + * + * when we un-map any dma mapping we need to remove them from rb_tree, + * rb_tree is maintained at kbase_device level and kbase_process level + * by passing the root of kbase_device or kbase_process we can remove + * the node from the tree. + */ +static bool kbase_delete_dma_buf_mapping(struct kbase_context *kctx, + struct dma_buf *dma_buf, + struct rb_root *tree) +{ + struct kbase_dma_buf *buf_node = NULL; + struct rb_node *node = tree->rb_node; + bool mapping_removed = false; + + lockdep_assert_held(&kctx->kbdev->dma_buf_lock); + + while (node) { + buf_node = rb_entry(node, struct kbase_dma_buf, dma_buf_node); + + if (dma_buf == buf_node->dma_buf) { + WARN_ON(!buf_node->import_count); + + buf_node->import_count--; + + if (!buf_node->import_count) { + rb_erase(&buf_node->dma_buf_node, tree); + kfree(buf_node); + mapping_removed = true; + } + + break; + } + + if (dma_buf < buf_node->dma_buf) + node = node->rb_left; + else + node = node->rb_right; + } + + WARN_ON(!buf_node); + return mapping_removed; +} + +/** + * kbase_capture_dma_buf_mapping - capture a dma buffer mapping. + * + * @kctx: Pointer to kbase context. + * @dma_buf: Pointer to a dma buffer mapping. + * @root: Pointer to root of rb_tree containing the dma_buf's. + * + * We maintain a kbase_device level and kbase_process level rb_tree + * of all unique dma_buf's mapped to gpu memory. So when attach any + * dma_buf add it the rb_tree's. To add the unique mapping we need + * check if the mapping is not a duplicate and then add them. + */ +static bool kbase_capture_dma_buf_mapping(struct kbase_context *kctx, + struct dma_buf *dma_buf, + struct rb_root *root) +{ + struct kbase_dma_buf *buf_node = NULL; + struct rb_node *node = root->rb_node; + bool unique_buf_imported = true; + + lockdep_assert_held(&kctx->kbdev->dma_buf_lock); + + while (node) { + buf_node = rb_entry(node, struct kbase_dma_buf, dma_buf_node); + + if (dma_buf == buf_node->dma_buf) { + unique_buf_imported = false; + break; + } + + if (dma_buf < buf_node->dma_buf) + node = node->rb_left; + else + node = node->rb_right; + } + + if (unique_buf_imported) { + struct kbase_dma_buf *buf_node = + kzalloc(sizeof(*buf_node), GFP_KERNEL); + + if (buf_node == NULL) { + dev_err(kctx->kbdev->dev, "Error allocating memory for kbase_dma_buf\n"); + /* Dont account for it if we fail to allocate memory */ + unique_buf_imported = false; + } else { + struct rb_node **new = &(root->rb_node), *parent = NULL; + + buf_node->dma_buf = dma_buf; + buf_node->import_count = 1; + while (*new) { + struct kbase_dma_buf *node; + + parent = *new; + node = rb_entry(parent, struct kbase_dma_buf, + dma_buf_node); + if (dma_buf < node->dma_buf) + new = &(*new)->rb_left; + else + new = &(*new)->rb_right; + } + rb_link_node(&buf_node->dma_buf_node, parent, new); + rb_insert_color(&buf_node->dma_buf_node, root); + } + } else if (!WARN_ON(!buf_node)) { + buf_node->import_count++; + } + + return unique_buf_imported; +} + +void kbase_remove_dma_buf_usage(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc) +{ + struct kbase_device *kbdev = kctx->kbdev; + bool dev_mapping_removed, prcs_mapping_removed; + + mutex_lock(&kbdev->dma_buf_lock); + + dev_mapping_removed = kbase_delete_dma_buf_mapping( + kctx, alloc->imported.umm.dma_buf, &kbdev->dma_buf_root); + + prcs_mapping_removed = kbase_delete_dma_buf_mapping( + kctx, alloc->imported.umm.dma_buf, &kctx->kprcs->dma_buf_root); + + WARN_ON(dev_mapping_removed && !prcs_mapping_removed); + + spin_lock(&kbdev->gpu_mem_usage_lock); + if (dev_mapping_removed) + kbdev->total_gpu_pages -= alloc->nents; + + if (prcs_mapping_removed) + kctx->kprcs->total_gpu_pages -= alloc->nents; + + if (dev_mapping_removed || prcs_mapping_removed) + kbase_trace_gpu_mem_usage(kbdev, kctx); + spin_unlock(&kbdev->gpu_mem_usage_lock); + + mutex_unlock(&kbdev->dma_buf_lock); +} + +void kbase_add_dma_buf_usage(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc) +{ + struct kbase_device *kbdev = kctx->kbdev; + bool unique_dev_dmabuf, unique_prcs_dmabuf; + + mutex_lock(&kbdev->dma_buf_lock); + + /* add dma_buf to device and process. */ + unique_dev_dmabuf = kbase_capture_dma_buf_mapping( + kctx, alloc->imported.umm.dma_buf, &kbdev->dma_buf_root); + + unique_prcs_dmabuf = kbase_capture_dma_buf_mapping( + kctx, alloc->imported.umm.dma_buf, &kctx->kprcs->dma_buf_root); + + WARN_ON(unique_dev_dmabuf && !unique_prcs_dmabuf); + + spin_lock(&kbdev->gpu_mem_usage_lock); + if (unique_dev_dmabuf) + kbdev->total_gpu_pages += alloc->nents; + + if (unique_prcs_dmabuf) + kctx->kprcs->total_gpu_pages += alloc->nents; + + if (unique_prcs_dmabuf || unique_dev_dmabuf) + kbase_trace_gpu_mem_usage(kbdev, kctx); + spin_unlock(&kbdev->gpu_mem_usage_lock); + + mutex_unlock(&kbdev->dma_buf_lock); +} + +#if !defined(CONFIG_TRACE_GPU_MEM) && !MALI_CUSTOMER_RELEASE +#define CREATE_TRACE_POINTS +#include "mali_gpu_mem_trace.h" +#endif diff --git a/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.h b/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.h new file mode 100755 index 000000000000..7e95956f3132 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_trace_gpu_mem.h @@ -0,0 +1,103 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_TRACE_GPU_MEM_H_ +#define _KBASE_TRACE_GPU_MEM_H_ + +#ifdef CONFIG_TRACE_GPU_MEM +#include +#elif !MALI_CUSTOMER_RELEASE +#include "mali_gpu_mem_trace.h" +#endif + +#define DEVICE_TGID ((u32) 0U) + +static void kbase_trace_gpu_mem_usage(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + lockdep_assert_held(&kbdev->gpu_mem_usage_lock); + +#if defined(CONFIG_TRACE_GPU_MEM) || !MALI_CUSTOMER_RELEASE + trace_gpu_mem_total(kbdev->id, DEVICE_TGID, + kbdev->total_gpu_pages << PAGE_SHIFT); + + if (likely(kctx)) + trace_gpu_mem_total(kbdev->id, kctx->kprcs->tgid, + kctx->kprcs->total_gpu_pages << PAGE_SHIFT); +#endif +} + +static inline void kbase_trace_gpu_mem_usage_dec(struct kbase_device *kbdev, + struct kbase_context *kctx, size_t pages) +{ + spin_lock(&kbdev->gpu_mem_usage_lock); + + if (likely(kctx)) + kctx->kprcs->total_gpu_pages -= pages; + + kbdev->total_gpu_pages -= pages; + + kbase_trace_gpu_mem_usage(kbdev, kctx); + + spin_unlock(&kbdev->gpu_mem_usage_lock); +} + +static inline void kbase_trace_gpu_mem_usage_inc(struct kbase_device *kbdev, + struct kbase_context *kctx, size_t pages) +{ + spin_lock(&kbdev->gpu_mem_usage_lock); + + if (likely(kctx)) + kctx->kprcs->total_gpu_pages += pages; + + kbdev->total_gpu_pages += pages; + + kbase_trace_gpu_mem_usage(kbdev, kctx); + + spin_unlock(&kbdev->gpu_mem_usage_lock); +} + +/** + * kbase_remove_dma_buf_usage - Remove a dma-buf entry captured. + * + * @kctx: Pointer to the kbase context + * @alloc: Pointer to the alloc to unmap + * + * Remove reference to dma buf been unmapped from kbase_device level + * rb_tree and Kbase_process level dma buf rb_tree. + */ +void kbase_remove_dma_buf_usage(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc); + +/** + * kbase_add_dma_buf_usage - Add a dma-buf entry captured. + * + * @kctx: Pointer to the kbase context + * @alloc: Pointer to the alloc to map in + * + * Add reference to dma buf been mapped to kbase_device level + * rb_tree and Kbase_process level dma buf rb_tree. + */ +void kbase_add_dma_buf_usage(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc); + +#endif /* _KBASE_TRACE_GPU_MEM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_utility.h b/drivers/gpu/arm/bifrost/mali_kbase_utility.h new file mode 100755 index 000000000000..8d4f044376a9 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_utility.h @@ -0,0 +1,55 @@ +/* + * + * (C) COPYRIGHT 2012-2013, 2015, 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#ifndef _KBASE_UTILITY_H +#define _KBASE_UTILITY_H + +#ifndef _KBASE_H_ +#error "Don't include this file directly, use mali_kbase.h instead" +#endif + +static inline void kbase_timer_setup(struct timer_list *timer, + void (*callback)(struct timer_list *timer)) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) + setup_timer(timer, (void (*)(unsigned long)) callback, + (unsigned long) timer); +#else + timer_setup(timer, callback, 0); +#endif +} + +#ifndef WRITE_ONCE + #ifdef ASSIGN_ONCE + #define WRITE_ONCE(x, val) ASSIGN_ONCE(val, x) + #else + #define WRITE_ONCE(x, val) (ACCESS_ONCE(x) = (val)) + #endif +#endif + +#ifndef READ_ONCE + #define READ_ONCE(x) ACCESS_ONCE(x) +#endif + +#endif /* _KBASE_UTILITY_H */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_vinstr.c b/drivers/gpu/arm/bifrost/mali_kbase_vinstr.c new file mode 100755 index 000000000000..3b0e2d6855ce --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_vinstr.c @@ -0,0 +1,1083 @@ +/* + * + * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_vinstr.h" +#include "mali_kbase_hwcnt_virtualizer.h" +#include "mali_kbase_hwcnt_types.h" +#include "mali_kbase_hwcnt_reader.h" +#include "mali_kbase_hwcnt_gpu.h" +#include "mali_kbase_ioctl.h" +#include "mali_malisw.h" +#include "mali_kbase_debug.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Hwcnt reader API version */ +#define HWCNT_READER_API 1 + +/* The minimum allowed interval between dumps (equivalent to 10KHz) */ +#define DUMP_INTERVAL_MIN_NS (100 * NSEC_PER_USEC) + +/* The maximum allowed buffers per client */ +#define MAX_BUFFER_COUNT 32 + +/** + * struct kbase_vinstr_context - IOCTL interface for userspace hardware + * counters. + * @hvirt: Hardware counter virtualizer used by vinstr. + * @metadata: Hardware counter metadata provided by virtualizer. + * @lock: Lock protecting all vinstr state. + * @suspend_count: Suspend reference count. If non-zero, timer and worker are + * prevented from being re-scheduled. + * @client_count: Number of vinstr clients. + * @clients: List of vinstr clients. + * @dump_timer: Timer that enqueues dump_work to a workqueue. + * @dump_work: Worker for performing periodic counter dumps. + */ +struct kbase_vinstr_context { + struct kbase_hwcnt_virtualizer *hvirt; + const struct kbase_hwcnt_metadata *metadata; + struct mutex lock; + size_t suspend_count; + size_t client_count; + struct list_head clients; + struct hrtimer dump_timer; + struct work_struct dump_work; +}; + +/** + * struct kbase_vinstr_client - A vinstr client attached to a vinstr context. + * @vctx: Vinstr context client is attached to. + * @hvcli: Hardware counter virtualizer client. + * @node: Node used to attach this client to list in vinstr + * context. + * @dump_interval_ns: Interval between periodic dumps. If 0, not a periodic + * client. + * @next_dump_time_ns: Time in ns when this client's next periodic dump must + * occur. If 0, not a periodic client. + * @enable_map: Counters enable map. + * @tmp_buf: Temporary buffer to use before handing dump to client. + * @dump_bufs: Array of dump buffers allocated by this client. + * @dump_bufs_meta: Metadata of dump buffers. + * @meta_idx: Index of metadata being accessed by userspace. + * @read_idx: Index of buffer read by userspace. + * @write_idx: Index of buffer being written by dump worker. + * @waitq: Client's notification queue. + */ +struct kbase_vinstr_client { + struct kbase_vinstr_context *vctx; + struct kbase_hwcnt_virtualizer_client *hvcli; + struct list_head node; + u64 next_dump_time_ns; + u32 dump_interval_ns; + struct kbase_hwcnt_enable_map enable_map; + struct kbase_hwcnt_dump_buffer tmp_buf; + struct kbase_hwcnt_dump_buffer_array dump_bufs; + struct kbase_hwcnt_reader_metadata *dump_bufs_meta; + atomic_t meta_idx; + atomic_t read_idx; + atomic_t write_idx; + wait_queue_head_t waitq; +}; + +static unsigned int kbasep_vinstr_hwcnt_reader_poll( + struct file *filp, + poll_table *wait); + +static long kbasep_vinstr_hwcnt_reader_ioctl( + struct file *filp, + unsigned int cmd, + unsigned long arg); + +static int kbasep_vinstr_hwcnt_reader_mmap( + struct file *filp, + struct vm_area_struct *vma); + +static int kbasep_vinstr_hwcnt_reader_release( + struct inode *inode, + struct file *filp); + +/* Vinstr client file operations */ +static const struct file_operations vinstr_client_fops = { + .owner = THIS_MODULE, + .poll = kbasep_vinstr_hwcnt_reader_poll, + .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, + .compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, + .mmap = kbasep_vinstr_hwcnt_reader_mmap, + .release = kbasep_vinstr_hwcnt_reader_release, +}; + +/** + * kbasep_vinstr_timestamp_ns() - Get the current time in nanoseconds. + * + * Return: Current time in nanoseconds. + */ +static u64 kbasep_vinstr_timestamp_ns(void) +{ + return ktime_get_raw_ns(); +} + +/** + * kbasep_vinstr_next_dump_time_ns() - Calculate the next periodic dump time. + * @cur_ts_ns: Current time in nanoseconds. + * @interval: Interval between dumps in nanoseconds. + * + * Return: 0 if interval is 0 (i.e. a non-periodic client), or the next dump + * time that occurs after cur_ts_ns. + */ +static u64 kbasep_vinstr_next_dump_time_ns(u64 cur_ts_ns, u32 interval) +{ + /* Non-periodic client */ + if (interval == 0) + return 0; + + /* + * Return the next interval after the current time relative to t=0. + * This means multiple clients with the same period will synchronise, + * regardless of when they were started, allowing the worker to be + * scheduled less frequently. + */ + do_div(cur_ts_ns, interval); + return (cur_ts_ns + 1) * interval; +} + +/** + * kbasep_vinstr_client_dump() - Perform a dump for a client. + * @vcli: Non-NULL pointer to a vinstr client. + * @event_id: Event type that triggered the dump. + * + * Return: 0 on success, else error code. + */ +static int kbasep_vinstr_client_dump( + struct kbase_vinstr_client *vcli, + enum base_hwcnt_reader_event event_id) +{ + int errcode; + u64 ts_start_ns; + u64 ts_end_ns; + unsigned int write_idx; + unsigned int read_idx; + struct kbase_hwcnt_dump_buffer *tmp_buf; + struct kbase_hwcnt_dump_buffer *dump_buf; + struct kbase_hwcnt_reader_metadata *meta; + u8 clk_cnt; + + WARN_ON(!vcli); + lockdep_assert_held(&vcli->vctx->lock); + + write_idx = atomic_read(&vcli->write_idx); + read_idx = atomic_read(&vcli->read_idx); + + /* Check if there is a place to copy HWC block into. */ + if (write_idx - read_idx == vcli->dump_bufs.buf_cnt) + return -EBUSY; + write_idx %= vcli->dump_bufs.buf_cnt; + + dump_buf = &vcli->dump_bufs.bufs[write_idx]; + meta = &vcli->dump_bufs_meta[write_idx]; + tmp_buf = &vcli->tmp_buf; + + errcode = kbase_hwcnt_virtualizer_client_dump( + vcli->hvcli, &ts_start_ns, &ts_end_ns, tmp_buf); + if (errcode) + return errcode; + + /* Patch the dump buf headers, to hide the counters that other hwcnt + * clients are using. + */ + kbase_hwcnt_gpu_patch_dump_headers(tmp_buf, &vcli->enable_map); + + /* Copy the temp buffer to the userspace visible buffer. The strict + * variant will explicitly zero any non-enabled counters to ensure + * nothing except exactly what the user asked for is made visible. + */ + kbase_hwcnt_dump_buffer_copy_strict( + dump_buf, tmp_buf, &vcli->enable_map); + + clk_cnt = vcli->vctx->metadata->clk_cnt; + + meta->timestamp = ts_end_ns; + meta->event_id = event_id; + meta->buffer_idx = write_idx; + meta->cycles.top = (clk_cnt > 0) ? dump_buf->clk_cnt_buf[0] : 0; + meta->cycles.shader_cores = + (clk_cnt > 1) ? dump_buf->clk_cnt_buf[1] : 0; + + /* Notify client. Make sure all changes to memory are visible. */ + wmb(); + atomic_inc(&vcli->write_idx); + wake_up_interruptible(&vcli->waitq); + return 0; +} + +/** + * kbasep_vinstr_client_clear() - Reset all the client's counters to zero. + * @vcli: Non-NULL pointer to a vinstr client. + * + * Return: 0 on success, else error code. + */ +static int kbasep_vinstr_client_clear(struct kbase_vinstr_client *vcli) +{ + u64 ts_start_ns; + u64 ts_end_ns; + + WARN_ON(!vcli); + lockdep_assert_held(&vcli->vctx->lock); + + /* A virtualizer dump with a NULL buffer will just clear the virtualizer + * client's buffer. + */ + return kbase_hwcnt_virtualizer_client_dump( + vcli->hvcli, &ts_start_ns, &ts_end_ns, NULL); +} + +/** + * kbasep_vinstr_reschedule_worker() - Update next dump times for all periodic + * vinstr clients, then reschedule the dump + * worker appropriately. + * @vctx: Non-NULL pointer to the vinstr context. + * + * If there are no periodic clients, then the dump worker will not be + * rescheduled. Else, the dump worker will be rescheduled for the next periodic + * client dump. + */ +static void kbasep_vinstr_reschedule_worker(struct kbase_vinstr_context *vctx) +{ + u64 cur_ts_ns; + u64 earliest_next_ns = U64_MAX; + struct kbase_vinstr_client *pos; + + WARN_ON(!vctx); + lockdep_assert_held(&vctx->lock); + + cur_ts_ns = kbasep_vinstr_timestamp_ns(); + + /* + * Update each client's next dump time, and find the earliest next + * dump time if any of the clients have a non-zero interval. + */ + list_for_each_entry(pos, &vctx->clients, node) { + const u64 cli_next_ns = + kbasep_vinstr_next_dump_time_ns( + cur_ts_ns, pos->dump_interval_ns); + + /* Non-zero next dump time implies a periodic client */ + if ((cli_next_ns != 0) && (cli_next_ns < earliest_next_ns)) + earliest_next_ns = cli_next_ns; + + pos->next_dump_time_ns = cli_next_ns; + } + + /* Cancel the timer if it is already pending */ + hrtimer_cancel(&vctx->dump_timer); + + /* Start the timer if there are periodic clients and vinstr is not + * suspended. + */ + if ((earliest_next_ns != U64_MAX) && + (vctx->suspend_count == 0) && + !WARN_ON(earliest_next_ns < cur_ts_ns)) + hrtimer_start( + &vctx->dump_timer, + ns_to_ktime(earliest_next_ns - cur_ts_ns), + HRTIMER_MODE_REL); +} + +/** + * kbasep_vinstr_dump_worker()- Dump worker, that dumps all periodic clients + * that need to be dumped, then reschedules itself. + * @work: Work structure. + */ +static void kbasep_vinstr_dump_worker(struct work_struct *work) +{ + struct kbase_vinstr_context *vctx = + container_of(work, struct kbase_vinstr_context, dump_work); + struct kbase_vinstr_client *pos; + u64 cur_time_ns; + + mutex_lock(&vctx->lock); + + cur_time_ns = kbasep_vinstr_timestamp_ns(); + + /* Dump all periodic clients whose next dump time is before the current + * time. + */ + list_for_each_entry(pos, &vctx->clients, node) { + if ((pos->next_dump_time_ns != 0) && + (pos->next_dump_time_ns < cur_time_ns)) + kbasep_vinstr_client_dump( + pos, BASE_HWCNT_READER_EVENT_PERIODIC); + } + + /* Update the next dump times of all periodic clients, then reschedule + * this worker at the earliest next dump time. + */ + kbasep_vinstr_reschedule_worker(vctx); + + mutex_unlock(&vctx->lock); +} + +/** + * kbasep_vinstr_dump_timer() - Dump timer that schedules the dump worker for + * execution as soon as possible. + * @timer: Timer structure. + */ +static enum hrtimer_restart kbasep_vinstr_dump_timer(struct hrtimer *timer) +{ + struct kbase_vinstr_context *vctx = + container_of(timer, struct kbase_vinstr_context, dump_timer); + + /* We don't need to check vctx->suspend_count here, as the suspend + * function will ensure that any worker enqueued here is immediately + * cancelled, and the worker itself won't reschedule this timer if + * suspend_count != 0. + */ +#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE + queue_work(system_wq, &vctx->dump_work); +#else + queue_work(system_highpri_wq, &vctx->dump_work); +#endif + return HRTIMER_NORESTART; +} + +/** + * kbasep_vinstr_client_destroy() - Destroy a vinstr client. + * @vcli: vinstr client. Must not be attached to a vinstr context. + */ +static void kbasep_vinstr_client_destroy(struct kbase_vinstr_client *vcli) +{ + if (!vcli) + return; + + kbase_hwcnt_virtualizer_client_destroy(vcli->hvcli); + kfree(vcli->dump_bufs_meta); + kbase_hwcnt_dump_buffer_array_free(&vcli->dump_bufs); + kbase_hwcnt_dump_buffer_free(&vcli->tmp_buf); + kbase_hwcnt_enable_map_free(&vcli->enable_map); + kfree(vcli); +} + +/** + * kbasep_vinstr_client_create() - Create a vinstr client. Does not attach to + * the vinstr context. + * @vctx: Non-NULL pointer to vinstr context. + * @setup: Non-NULL pointer to hardware counter ioctl setup structure. + * setup->buffer_count must not be 0. + * @out_vcli: Non-NULL pointer to where created client will be stored on + * success. + * + * Return: 0 on success, else error code. + */ +static int kbasep_vinstr_client_create( + struct kbase_vinstr_context *vctx, + struct kbase_ioctl_hwcnt_reader_setup *setup, + struct kbase_vinstr_client **out_vcli) +{ + int errcode; + struct kbase_vinstr_client *vcli; + struct kbase_hwcnt_physical_enable_map phys_em; + + WARN_ON(!vctx); + WARN_ON(!setup); + WARN_ON(setup->buffer_count == 0); + + vcli = kzalloc(sizeof(*vcli), GFP_KERNEL); + if (!vcli) + return -ENOMEM; + + vcli->vctx = vctx; + + errcode = kbase_hwcnt_enable_map_alloc( + vctx->metadata, &vcli->enable_map); + if (errcode) + goto error; + + phys_em.fe_bm = setup->fe_bm; + phys_em.shader_bm = setup->shader_bm; + phys_em.tiler_bm = setup->tiler_bm; + phys_em.mmu_l2_bm = setup->mmu_l2_bm; + kbase_hwcnt_gpu_enable_map_from_physical(&vcli->enable_map, &phys_em); + + errcode = kbase_hwcnt_dump_buffer_alloc(vctx->metadata, &vcli->tmp_buf); + if (errcode) + goto error; + + /* Enable all the available clk_enable_map. */ + vcli->enable_map.clk_enable_map = (1ull << vctx->metadata->clk_cnt) - 1; + + errcode = kbase_hwcnt_dump_buffer_array_alloc( + vctx->metadata, setup->buffer_count, &vcli->dump_bufs); + if (errcode) + goto error; + + errcode = -ENOMEM; + vcli->dump_bufs_meta = kmalloc_array( + setup->buffer_count, sizeof(*vcli->dump_bufs_meta), GFP_KERNEL); + if (!vcli->dump_bufs_meta) + goto error; + + errcode = kbase_hwcnt_virtualizer_client_create( + vctx->hvirt, &vcli->enable_map, &vcli->hvcli); + if (errcode) + goto error; + + init_waitqueue_head(&vcli->waitq); + + *out_vcli = vcli; + return 0; +error: + kbasep_vinstr_client_destroy(vcli); + return errcode; +} + +int kbase_vinstr_init( + struct kbase_hwcnt_virtualizer *hvirt, + struct kbase_vinstr_context **out_vctx) +{ + struct kbase_vinstr_context *vctx; + const struct kbase_hwcnt_metadata *metadata; + + if (!hvirt || !out_vctx) + return -EINVAL; + + metadata = kbase_hwcnt_virtualizer_metadata(hvirt); + if (!metadata) + return -EINVAL; + + vctx = kzalloc(sizeof(*vctx), GFP_KERNEL); + if (!vctx) + return -ENOMEM; + + vctx->hvirt = hvirt; + vctx->metadata = metadata; + + mutex_init(&vctx->lock); + INIT_LIST_HEAD(&vctx->clients); + hrtimer_init(&vctx->dump_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + vctx->dump_timer.function = kbasep_vinstr_dump_timer; + INIT_WORK(&vctx->dump_work, kbasep_vinstr_dump_worker); + + *out_vctx = vctx; + return 0; +} + +void kbase_vinstr_term(struct kbase_vinstr_context *vctx) +{ + if (!vctx) + return; + + cancel_work_sync(&vctx->dump_work); + + /* Non-zero client count implies client leak */ + if (WARN_ON(vctx->client_count != 0)) { + struct kbase_vinstr_client *pos, *n; + + list_for_each_entry_safe(pos, n, &vctx->clients, node) { + list_del(&pos->node); + vctx->client_count--; + kbasep_vinstr_client_destroy(pos); + } + } + + WARN_ON(vctx->client_count != 0); + kfree(vctx); +} + +void kbase_vinstr_suspend(struct kbase_vinstr_context *vctx) +{ + if (WARN_ON(!vctx)) + return; + + mutex_lock(&vctx->lock); + + if (!WARN_ON(vctx->suspend_count == SIZE_MAX)) + vctx->suspend_count++; + + mutex_unlock(&vctx->lock); + + /* Always sync cancel the timer and then the worker, regardless of the + * new suspend count. + * + * This ensures concurrent calls to kbase_vinstr_suspend() always block + * until vinstr is fully suspended. + * + * The timer is cancelled before the worker, as the timer + * unconditionally re-enqueues the worker, but the worker checks the + * suspend_count that we just incremented before rescheduling the timer. + * + * Therefore if we cancel the worker first, the timer might re-enqueue + * the worker before we cancel the timer, but the opposite is not + * possible. + */ + hrtimer_cancel(&vctx->dump_timer); + cancel_work_sync(&vctx->dump_work); +} + +void kbase_vinstr_resume(struct kbase_vinstr_context *vctx) +{ + if (WARN_ON(!vctx)) + return; + + mutex_lock(&vctx->lock); + + if (!WARN_ON(vctx->suspend_count == 0)) { + vctx->suspend_count--; + + /* Last resume, so re-enqueue the worker if we have any periodic + * clients. + */ + if (vctx->suspend_count == 0) { + struct kbase_vinstr_client *pos; + bool has_periodic_clients = false; + + list_for_each_entry(pos, &vctx->clients, node) { + if (pos->dump_interval_ns != 0) { + has_periodic_clients = true; + break; + } + } + + if (has_periodic_clients) +#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE + queue_work(system_wq, &vctx->dump_work); +#else + queue_work(system_highpri_wq, &vctx->dump_work); +#endif + } + } + + mutex_unlock(&vctx->lock); +} + +int kbase_vinstr_hwcnt_reader_setup( + struct kbase_vinstr_context *vctx, + struct kbase_ioctl_hwcnt_reader_setup *setup) +{ + int errcode; + int fd; + struct kbase_vinstr_client *vcli = NULL; + + if (!vctx || !setup || + (setup->buffer_count == 0) || + (setup->buffer_count > MAX_BUFFER_COUNT)) + return -EINVAL; + + errcode = kbasep_vinstr_client_create(vctx, setup, &vcli); + if (errcode) + goto error; + + /* Add the new client. No need to reschedule worker, as not periodic */ + mutex_lock(&vctx->lock); + + vctx->client_count++; + list_add(&vcli->node, &vctx->clients); + + mutex_unlock(&vctx->lock); + + /* Expose to user-space only once the client is fully initialized */ + errcode = anon_inode_getfd( + "[mali_vinstr_desc]", + &vinstr_client_fops, + vcli, + O_RDONLY | O_CLOEXEC); + if (errcode < 0) + goto client_installed_error; + + fd = errcode; + + return fd; + +client_installed_error: + mutex_lock(&vctx->lock); + + vctx->client_count--; + list_del(&vcli->node); + + mutex_unlock(&vctx->lock); +error: + kbasep_vinstr_client_destroy(vcli); + return errcode; +} + +/** + * kbasep_vinstr_hwcnt_reader_buffer_ready() - Check if client has ready + * buffers. + * @cli: Non-NULL pointer to vinstr client. + * + * Return: Non-zero if client has at least one dumping buffer filled that was + * not notified to user yet. + */ +static int kbasep_vinstr_hwcnt_reader_buffer_ready( + struct kbase_vinstr_client *cli) +{ + WARN_ON(!cli); + return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx); +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_dump() - Dump ioctl command. + * @cli: Non-NULL pointer to vinstr client. + * + * Return: 0 on success, else error code. + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_dump( + struct kbase_vinstr_client *cli) +{ + int errcode; + + mutex_lock(&cli->vctx->lock); + + errcode = kbasep_vinstr_client_dump( + cli, BASE_HWCNT_READER_EVENT_MANUAL); + + mutex_unlock(&cli->vctx->lock); + return errcode; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_clear() - Clear ioctl command. + * @cli: Non-NULL pointer to vinstr client. + * + * Return: 0 on success, else error code. + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_clear( + struct kbase_vinstr_client *cli) +{ + int errcode; + + mutex_lock(&cli->vctx->lock); + + errcode = kbasep_vinstr_client_clear(cli); + + mutex_unlock(&cli->vctx->lock); + return errcode; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer() - Get buffer ioctl command. + * @cli: Non-NULL pointer to vinstr client. + * @buffer: Non-NULL pointer to userspace buffer. + * @size: Size of buffer. + * + * Return: 0 on success, else error code. + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( + struct kbase_vinstr_client *cli, + void __user *buffer, + size_t size) +{ + unsigned int meta_idx = atomic_read(&cli->meta_idx); + unsigned int idx = meta_idx % cli->dump_bufs.buf_cnt; + + struct kbase_hwcnt_reader_metadata *meta = &cli->dump_bufs_meta[idx]; + const size_t meta_size = sizeof(struct kbase_hwcnt_reader_metadata); + const size_t min_size = min(size, meta_size); + + /* Metadata sanity check. */ + WARN_ON(idx != meta->buffer_idx); + + /* Check if there is any buffer available. */ + if (unlikely(atomic_read(&cli->write_idx) == meta_idx)) + return -EAGAIN; + + /* Check if previously taken buffer was put back. */ + if (unlikely(atomic_read(&cli->read_idx) != meta_idx)) + return -EBUSY; + + /* Clear user buffer to zero. */ + if (unlikely(meta_size < size && clear_user(buffer, size))) + return -EFAULT; + + /* Copy next available buffer's metadata to user. */ + if (unlikely(copy_to_user(buffer, meta, min_size))) + return -EFAULT; + + atomic_inc(&cli->meta_idx); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer() - Put buffer ioctl command. + * @cli: Non-NULL pointer to vinstr client. + * @buffer: Non-NULL pointer to userspace buffer. + * @size: Size of buffer. + * + * Return: 0 on success, else error code. + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( + struct kbase_vinstr_client *cli, + void __user *buffer, + size_t size) +{ + unsigned int read_idx = atomic_read(&cli->read_idx); + unsigned int idx = read_idx % cli->dump_bufs.buf_cnt; + + struct kbase_hwcnt_reader_metadata *meta; + const size_t meta_size = sizeof(struct kbase_hwcnt_reader_metadata); + const size_t max_size = max(size, meta_size); + int ret = 0; + u8 stack_kbuf[64]; + u8 *kbuf = NULL; + size_t i; + + /* Check if any buffer was taken. */ + if (unlikely(atomic_read(&cli->meta_idx) == read_idx)) + return -EPERM; + + if (likely(max_size <= sizeof(stack_kbuf))) { + /* Use stack buffer when the size is small enough. */ + if (unlikely(meta_size > size)) + memset(stack_kbuf, 0, sizeof(stack_kbuf)); + kbuf = stack_kbuf; + } else { + kbuf = kzalloc(max_size, GFP_KERNEL); + if (unlikely(!kbuf)) + return -ENOMEM; + } + + /* + * Copy user buffer to zero cleared kernel buffer which has enough + * space for both user buffer and kernel metadata. + */ + if (unlikely(copy_from_user(kbuf, buffer, size))) { + ret = -EFAULT; + goto out; + } + + /* + * Make sure any "extra" data passed from userspace is zero. + * It's meaningful only in case meta_size < size. + */ + for (i = meta_size; i < size; i++) { + /* Check if user data beyond meta size is zero. */ + if (unlikely(kbuf[i] != 0)) { + ret = -EINVAL; + goto out; + } + } + + /* Check if correct buffer is put back. */ + meta = (struct kbase_hwcnt_reader_metadata *)kbuf; + if (unlikely(idx != meta->buffer_idx)) { + ret = -EINVAL; + goto out; + } + + atomic_inc(&cli->read_idx); +out: + if (unlikely(kbuf != stack_kbuf)) + kfree(kbuf); + return ret; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_set_interval() - Set interval ioctl command. + * @cli: Non-NULL pointer to vinstr client. + * @interval: Periodic dumping interval (disable periodic dumping if 0). + * + * Return: 0 always. + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval( + struct kbase_vinstr_client *cli, + u32 interval) +{ + mutex_lock(&cli->vctx->lock); + + if ((interval != 0) && (interval < DUMP_INTERVAL_MIN_NS)) + interval = DUMP_INTERVAL_MIN_NS; + /* Update the interval, and put in a dummy next dump time */ + cli->dump_interval_ns = interval; + cli->next_dump_time_ns = 0; + + /* + * If it's a periodic client, kick off the worker early to do a proper + * timer reschedule. Return value is ignored, as we don't care if the + * worker is already queued. + */ + if ((interval != 0) && (cli->vctx->suspend_count == 0)) +#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE + queue_work(system_wq, &cli->vctx->dump_work); +#else + queue_work(system_highpri_wq, &cli->vctx->dump_work); +#endif + + mutex_unlock(&cli->vctx->lock); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_enable_event() - Enable event ioctl command. + * @cli: Non-NULL pointer to vinstr client. + * @event_id: ID of event to enable. + * + * Return: 0 always. + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event( + struct kbase_vinstr_client *cli, + enum base_hwcnt_reader_event event_id) +{ + /* No-op, as events aren't supported */ + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_disable_event() - Disable event ioctl + * command. + * @cli: Non-NULL pointer to vinstr client. + * @event_id: ID of event to disable. + * + * Return: 0 always. + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event( + struct kbase_vinstr_client *cli, + enum base_hwcnt_reader_event event_id) +{ + /* No-op, as events aren't supported */ + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver() - Get HW version ioctl command. + * @cli: Non-NULL pointer to vinstr client. + * @hwver: Non-NULL pointer to user buffer where HW version will be stored. + * + * Return: 0 on success, else error code. + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( + struct kbase_vinstr_client *cli, + u32 __user *hwver) +{ + u32 ver = 5; + const enum kbase_hwcnt_gpu_group_type type = + kbase_hwcnt_metadata_group_type(cli->vctx->metadata, 0); + + if (WARN_ON(type != KBASE_HWCNT_GPU_GROUP_TYPE_V5)) + return -EINVAL; + + return put_user(ver, hwver); +} + +/** + * The hwcnt reader's ioctl command - get API version. + * @cli: The non-NULL pointer to the client + * @arg: Command's argument. + * @size: Size of arg. + * + * @return 0 on success, else error code. + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_get_api_version( + struct kbase_vinstr_client *cli, unsigned long arg, size_t size) +{ + long ret = -EINVAL; + u8 clk_cnt = cli->vctx->metadata->clk_cnt; + + if (size == sizeof(u32)) { + ret = put_user(HWCNT_READER_API, (u32 __user *)arg); + } else if (size == sizeof(struct kbase_hwcnt_reader_api_version)) { + struct kbase_hwcnt_reader_api_version api_version = { + .version = HWCNT_READER_API, + .features = KBASE_HWCNT_READER_API_VERSION_NO_FEATURE, + }; + + if (clk_cnt > 0) + api_version.features |= + KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_TOP; + if (clk_cnt > 1) + api_version.features |= + KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_SHADER_CORES; + + ret = copy_to_user( + (void __user *)arg, &api_version, sizeof(api_version)); + } + return ret; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl() - hwcnt reader's ioctl. + * @filp: Non-NULL pointer to file structure. + * @cmd: User command. + * @arg: Command's argument. + * + * Return: 0 on success, else error code. + */ +static long kbasep_vinstr_hwcnt_reader_ioctl( + struct file *filp, + unsigned int cmd, + unsigned long arg) +{ + long rcode; + struct kbase_vinstr_client *cli; + + if (!filp || (_IOC_TYPE(cmd) != KBASE_HWCNT_READER)) + return -EINVAL; + + cli = filp->private_data; + if (!cli) + return -EINVAL; + + switch (_IOC_NR(cmd)) { + case _IOC_NR(KBASE_HWCNT_READER_GET_API_VERSION): + rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_api_version( + cli, arg, _IOC_SIZE(cmd)); + break; + case _IOC_NR(KBASE_HWCNT_READER_GET_HWVER): + rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( + cli, (u32 __user *)arg); + break; + case _IOC_NR(KBASE_HWCNT_READER_GET_BUFFER_SIZE): + rcode = put_user( + (u32)cli->vctx->metadata->dump_buf_bytes, + (u32 __user *)arg); + break; + case _IOC_NR(KBASE_HWCNT_READER_DUMP): + rcode = kbasep_vinstr_hwcnt_reader_ioctl_dump(cli); + break; + case _IOC_NR(KBASE_HWCNT_READER_CLEAR): + rcode = kbasep_vinstr_hwcnt_reader_ioctl_clear(cli); + break; + case _IOC_NR(KBASE_HWCNT_READER_GET_BUFFER): + rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( + cli, (void __user *)arg, _IOC_SIZE(cmd)); + break; + case _IOC_NR(KBASE_HWCNT_READER_PUT_BUFFER): + rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( + cli, (void __user *)arg, _IOC_SIZE(cmd)); + break; + case _IOC_NR(KBASE_HWCNT_READER_SET_INTERVAL): + rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval( + cli, (u32)arg); + break; + case _IOC_NR(KBASE_HWCNT_READER_ENABLE_EVENT): + rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event( + cli, (enum base_hwcnt_reader_event)arg); + break; + case _IOC_NR(KBASE_HWCNT_READER_DISABLE_EVENT): + rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event( + cli, (enum base_hwcnt_reader_event)arg); + break; + default: + pr_warn("Unknown HWCNT ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); + rcode = -EINVAL; + break; + } + + return rcode; +} + +/** + * kbasep_vinstr_hwcnt_reader_poll() - hwcnt reader's poll. + * @filp: Non-NULL pointer to file structure. + * @wait: Non-NULL pointer to poll table. + * + * Return: POLLIN if data can be read without blocking, 0 if data can not be + * read without blocking, else error code. + */ +static unsigned int kbasep_vinstr_hwcnt_reader_poll( + struct file *filp, + poll_table *wait) +{ + struct kbase_vinstr_client *cli; + + if (!filp || !wait) + return -EINVAL; + + cli = filp->private_data; + if (!cli) + return -EINVAL; + + poll_wait(filp, &cli->waitq, wait); + if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli)) + return POLLIN; + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_mmap() - hwcnt reader's mmap. + * @filp: Non-NULL pointer to file structure. + * @vma: Non-NULL pointer to vma structure. + * + * Return: 0 on success, else error code. + */ +static int kbasep_vinstr_hwcnt_reader_mmap( + struct file *filp, + struct vm_area_struct *vma) +{ + struct kbase_vinstr_client *cli; + unsigned long vm_size, size, addr, pfn, offset; + + if (!filp || !vma) + return -EINVAL; + + cli = filp->private_data; + if (!cli) + return -EINVAL; + + vm_size = vma->vm_end - vma->vm_start; + size = cli->dump_bufs.buf_cnt * cli->vctx->metadata->dump_buf_bytes; + + if (vma->vm_pgoff > (size >> PAGE_SHIFT)) + return -EINVAL; + + offset = vma->vm_pgoff << PAGE_SHIFT; + if (vm_size > size - offset) + return -EINVAL; + + addr = __pa(cli->dump_bufs.page_addr + offset); + pfn = addr >> PAGE_SHIFT; + + return remap_pfn_range( + vma, vma->vm_start, pfn, vm_size, vma->vm_page_prot); +} + +/** + * kbasep_vinstr_hwcnt_reader_release() - hwcnt reader's release. + * @inode: Non-NULL pointer to inode structure. + * @filp: Non-NULL pointer to file structure. + * + * Return: 0 always. + */ +static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode, + struct file *filp) +{ + struct kbase_vinstr_client *vcli = filp->private_data; + + mutex_lock(&vcli->vctx->lock); + + vcli->vctx->client_count--; + list_del(&vcli->node); + + mutex_unlock(&vcli->vctx->lock); + + kbasep_vinstr_client_destroy(vcli); + + return 0; +} diff --git a/drivers/gpu/arm/bifrost/mali_kbase_vinstr.h b/drivers/gpu/arm/bifrost/mali_kbase_vinstr.h new file mode 100755 index 000000000000..81d315f95567 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_vinstr.h @@ -0,0 +1,91 @@ +/* + * + * (C) COPYRIGHT 2015-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * Vinstr, used to provide an ioctl for userspace access to periodic hardware + * counters. + */ + +#ifndef _KBASE_VINSTR_H_ +#define _KBASE_VINSTR_H_ + +struct kbase_vinstr_context; +struct kbase_hwcnt_virtualizer; +struct kbase_ioctl_hwcnt_reader_setup; + +/** + * kbase_vinstr_init() - Initialise a vinstr context. + * @hvirt: Non-NULL pointer to the hardware counter virtualizer. + * @out_vctx: Non-NULL pointer to where the pointer to the created vinstr + * context will be stored on success. + * + * On creation, the suspend count of the context will be 0. + * + * Return: 0 on success, else error code. + */ +int kbase_vinstr_init( + struct kbase_hwcnt_virtualizer *hvirt, + struct kbase_vinstr_context **out_vctx); + +/** + * kbase_vinstr_term() - Terminate a vinstr context. + * @vctx: Pointer to the vinstr context to be terminated. + */ +void kbase_vinstr_term(struct kbase_vinstr_context *vctx); + +/** + * kbase_vinstr_suspend() - Increment the suspend count of the context. + * @vctx: Non-NULL pointer to the vinstr context to be suspended. + * + * After this function call returns, it is guaranteed that all timers and + * workers in vinstr will be cancelled, and will not be re-triggered until + * after the context has been resumed. In effect, this means no new counter + * dumps will occur for any existing or subsequently added periodic clients. + */ +void kbase_vinstr_suspend(struct kbase_vinstr_context *vctx); + +/** + * kbase_vinstr_resume() - Decrement the suspend count of the context. + * @vctx: Non-NULL pointer to the vinstr context to be resumed. + * + * If a call to this function decrements the suspend count from 1 to 0, then + * normal operation of vinstr will be resumed (i.e. counter dumps will once + * again be automatically triggered for all periodic clients). + * + * It is only valid to call this function one time for each prior returned call + * to kbase_vinstr_suspend. + */ +void kbase_vinstr_resume(struct kbase_vinstr_context *vctx); + +/** + * kbase_vinstr_hwcnt_reader_setup() - Set up a new hardware counter reader + * client. + * @vinstr_ctx: Non-NULL pointer to the vinstr context. + * @setup: Non-NULL pointer to the hwcnt reader configuration. + * + * Return: file descriptor on success, else a (negative) error code. + */ +int kbase_vinstr_hwcnt_reader_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_ioctl_hwcnt_reader_setup *setup); + +#endif /* _KBASE_VINSTR_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_linux_trace.h b/drivers/gpu/arm/bifrost/mali_linux_trace.h new file mode 100755 index 000000000000..be812f62c862 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_linux_trace.h @@ -0,0 +1,552 @@ +/* + * + * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mali + +#if !defined(_TRACE_MALI_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_MALI_H + +#include + +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) +#define MALI_JOB_SLOTS_EVENT_CHANGED + +/** + * mali_job_slots_event - Reports change of job slot status. + * @gpu_id: Kbase device id + * @event_id: ORed together bitfields representing a type of event, + * made with the GATOR_MAKE_EVENT() macro. + */ +TRACE_EVENT(mali_job_slots_event, + TP_PROTO(u32 gpu_id, u32 event_id, u32 tgid, u32 pid, + u8 job_id), + TP_ARGS(gpu_id, event_id, tgid, pid, job_id), + TP_STRUCT__entry( + __field(u32, gpu_id) + __field(u32, event_id) + __field(u32, tgid) + __field(u32, pid) + __field(u8, job_id) + ), + TP_fast_assign( + __entry->gpu_id = gpu_id; + __entry->event_id = event_id; + __entry->tgid = tgid; + __entry->pid = pid; + __entry->job_id = job_id; + ), + TP_printk("gpu=%u event=%u tgid=%u pid=%u job_id=%u", + __entry->gpu_id, __entry->event_id, + __entry->tgid, __entry->pid, __entry->job_id) +); + +/** + * mali_pm_status - Reports change of power management status. + * @gpu_id: Kbase device id + * @event_id: Core type (shader, tiler, L2 cache) + * @value: 64bits bitmask reporting either power status of + * the cores (1-ON, 0-OFF) + */ +TRACE_EVENT(mali_pm_status, + TP_PROTO(u32 gpu_id, u32 event_id, u64 value), + TP_ARGS(gpu_id, event_id, value), + TP_STRUCT__entry( + __field(u32, gpu_id) + __field(u32, event_id) + __field(u64, value) + ), + TP_fast_assign( + __entry->gpu_id = gpu_id; + __entry->event_id = event_id; + __entry->value = value; + ), + TP_printk("gpu=%u event %u = %llu", + __entry->gpu_id, __entry->event_id, __entry->value) +); + +/** + * mali_page_fault_insert_pages - Reports an MMU page fault + * resulting in new pages being mapped. + * @gpu_id: Kbase device id + * @event_id: MMU address space number + * @value: Number of newly allocated pages + */ +TRACE_EVENT(mali_page_fault_insert_pages, + TP_PROTO(u32 gpu_id, s32 event_id, u64 value), + TP_ARGS(gpu_id, event_id, value), + TP_STRUCT__entry( + __field(u32, gpu_id) + __field(s32, event_id) + __field(u64, value) + ), + TP_fast_assign( + __entry->gpu_id = gpu_id; + __entry->event_id = event_id; + __entry->value = value; + ), + TP_printk("gpu=%u event %d = %llu", + __entry->gpu_id, __entry->event_id, __entry->value) +); + +/** + * mali_total_alloc_pages_change - Reports that the total number of + * allocated pages has changed. + * @gpu_id: Kbase device id + * @event_id: Total number of pages allocated + */ +TRACE_EVENT(mali_total_alloc_pages_change, + TP_PROTO(u32 gpu_id, s64 event_id), + TP_ARGS(gpu_id, event_id), + TP_STRUCT__entry( + __field(u32, gpu_id) + __field(s64, event_id) + ), + TP_fast_assign( + __entry->gpu_id = gpu_id; + __entry->event_id = event_id; + ), + TP_printk("gpu=%u event=%lld", __entry->gpu_id, __entry->event_id) +); +#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ + +/* + * MMU subsystem tracepoints + */ + +/* Fault status and exception code helpers + * + * Must be macros to allow use by user-side tracepoint tools + * + * bits 0:1 masked off code, and used for the level + * + * Tracepoint files get included more than once - protect against multiple + * definition + */ +#ifndef __TRACE_MALI_MMU_HELPERS +#define __TRACE_MALI_MMU_HELPERS +/* Complex macros should be enclosed in parenthesis. + * + * We need to have those parentheses removed for our arrays of symbolic look-ups + * for __print_symbolic() whilst also being able to use them outside trace code + */ +#define _ENSURE_PARENTHESIS(args...) args + +#define KBASE_MMU_FAULT_CODE_EXCEPTION_NAME_PRINT(code) \ + (!KBASE_MMU_FAULT_CODE_VALID(code) ? "UNKNOWN,level=" : \ + __print_symbolic(((code) & ~3u), \ + KBASE_MMU_FAULT_CODE_SYMBOLIC_STRINGS)) +#define KBASE_MMU_FAULT_CODE_LEVEL(code) \ + (((((code) & ~0x3u) == 0xC4) ? 4 : 0) + ((code) & 0x3u)) + +#define KBASE_MMU_FAULT_STATUS_CODE(status) \ + ((status) & 0xFFu) +#define KBASE_MMU_FAULT_STATUS_DECODED_STRING(status) \ + (((status) & (1u << 10)) ? "DECODER_FAULT" : "SLAVE_FAULT") + +#define KBASE_MMU_FAULT_STATUS_EXCEPTION_NAME_PRINT(status) \ + KBASE_MMU_FAULT_CODE_EXCEPTION_NAME_PRINT( \ + KBASE_MMU_FAULT_STATUS_CODE(status)) + +#define KBASE_MMU_FAULT_STATUS_LEVEL(status) \ + KBASE_MMU_FAULT_CODE_LEVEL(KBASE_MMU_FAULT_STATUS_CODE(status)) + +#define KBASE_MMU_FAULT_STATUS_ACCESS(status) \ + ((status) & AS_FAULTSTATUS_ACCESS_TYPE_MASK) +#define KBASE_MMU_FAULT_ACCESS_SYMBOLIC_STRINGS _ENSURE_PARENTHESIS(\ + {AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC, "ATOMIC" }, \ + {AS_FAULTSTATUS_ACCESS_TYPE_EX, "EXECUTE"}, \ + {AS_FAULTSTATUS_ACCESS_TYPE_READ, "READ" }, \ + {AS_FAULTSTATUS_ACCESS_TYPE_WRITE, "WRITE" }) +#define KBASE_MMU_FAULT_STATUS_ACCESS_PRINT(status) \ + __print_symbolic(KBASE_MMU_FAULT_STATUS_ACCESS(status), \ + KBASE_MMU_FAULT_ACCESS_SYMBOLIC_STRINGS) + +#if MALI_USE_CSF +#define KBASE_MMU_FAULT_CODE_VALID(code) \ + ((code >= 0xC0 && code <= 0xEB) && \ + (!(code >= 0xC5 && code <= 0xC7)) && \ + (!(code >= 0xCC && code <= 0xD8)) && \ + (!(code >= 0xDC && code <= 0xDF)) && \ + (!(code >= 0xE1 && code <= 0xE3))) +#define KBASE_MMU_FAULT_CODE_SYMBOLIC_STRINGS _ENSURE_PARENTHESIS(\ + {0xC0, "TRANSLATION_FAULT_" }, \ + {0xC4, "TRANSLATION_FAULT_" }, \ + {0xC8, "PERMISSION_FAULT_" }, \ + {0xD0, "TRANSTAB_BUS_FAULT_" }, \ + {0xD8, "ACCESS_FLAG_" }, \ + {0xE0, "ADDRESS_SIZE_FAULT_IN" }, \ + {0xE4, "ADDRESS_SIZE_FAULT_OUT" }, \ + {0xE8, "MEMORY_ATTRIBUTES_FAULT_" }) +#else /* MALI_USE_CSF */ +#define KBASE_MMU_FAULT_CODE_VALID(code) \ + ((code >= 0xC0 && code <= 0xEF) && \ + (!(code >= 0xC5 && code <= 0xC6)) && \ + (!(code >= 0xCC && code <= 0xCF)) && \ + (!(code >= 0xD4 && code <= 0xD7)) && \ + (!(code >= 0xDC && code <= 0xDF))) +#define KBASE_MMU_FAULT_CODE_SYMBOLIC_STRINGS _ENSURE_PARENTHESIS(\ + {0xC0, "TRANSLATION_FAULT_" }, \ + {0xC4, "TRANSLATION_FAULT(_7==_IDENTITY)_" }, \ + {0xC8, "PERMISSION_FAULT_" }, \ + {0xD0, "TRANSTAB_BUS_FAULT_" }, \ + {0xD8, "ACCESS_FLAG_" }, \ + {0xE0, "ADDRESS_SIZE_FAULT_IN" }, \ + {0xE4, "ADDRESS_SIZE_FAULT_OUT" }, \ + {0xE8, "MEMORY_ATTRIBUTES_FAULT_" }, \ + {0xEC, "MEMORY_ATTRIBUTES_NONCACHEABLE_" }) +#endif /* MALI_USE_CSF */ +#endif /* __TRACE_MALI_MMU_HELPERS */ + +/* trace_mali_mmu_page_fault_grow + * + * Tracepoint about a successful grow of a region due to a GPU page fault + */ +TRACE_EVENT(mali_mmu_page_fault_grow, + TP_PROTO(struct kbase_va_region *reg, struct kbase_fault *fault, + size_t new_pages), + TP_ARGS(reg, fault, new_pages), + TP_STRUCT__entry( + __field(u64, start_addr) + __field(u64, fault_addr) + __field(u64, fault_extra_addr) + __field(size_t, new_pages) + __field(u32, status) + ), + TP_fast_assign( + __entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; + __entry->fault_addr = fault->addr; + __entry->fault_extra_addr = fault->extra_addr; + __entry->new_pages = new_pages; + __entry->status = fault->status; + ), + TP_printk("start=0x%llx fault_addr=0x%llx fault_extra_addr=0x%llx new_pages=%zu raw_fault_status=0x%x decoded_faultstatus=%s exception_type=0x%x,%s%u access_type=0x%x,%s source_id=0x%x", + __entry->start_addr, __entry->fault_addr, + __entry->fault_extra_addr, __entry->new_pages, + __entry->status, + KBASE_MMU_FAULT_STATUS_DECODED_STRING(__entry->status), + KBASE_MMU_FAULT_STATUS_CODE(__entry->status), + KBASE_MMU_FAULT_STATUS_EXCEPTION_NAME_PRINT(__entry->status), + KBASE_MMU_FAULT_STATUS_LEVEL(__entry->status), + KBASE_MMU_FAULT_STATUS_ACCESS(__entry->status) >> 8, + KBASE_MMU_FAULT_STATUS_ACCESS_PRINT(__entry->status), + __entry->status >> 16) +); + + + + +/* + * Just-in-time memory allocation subsystem tracepoints + */ + +/* Just-in-time memory allocation soft-job template. Override the TP_printk + * further if need be. jit_id can be 0. + */ +DECLARE_EVENT_CLASS(mali_jit_softjob_template, + TP_PROTO(struct kbase_va_region *reg, u8 jit_id), + TP_ARGS(reg, jit_id), + TP_STRUCT__entry( + __field(u64, start_addr) + __field(size_t, nr_pages) + __field(size_t, backed_pages) + __field(u8, jit_id) + ), + TP_fast_assign( + __entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; + __entry->nr_pages = reg->nr_pages; + __entry->backed_pages = kbase_reg_current_backed_size(reg); + __entry->jit_id = jit_id; + ), + TP_printk("jit_id=%u start=0x%llx va_pages=0x%zx backed_size=0x%zx", + __entry->jit_id, __entry->start_addr, __entry->nr_pages, + __entry->backed_pages) +); + +/* trace_mali_jit_alloc() + * + * Tracepoint about a just-in-time memory allocation soft-job successfully + * allocating memory + */ +DEFINE_EVENT(mali_jit_softjob_template, mali_jit_alloc, + TP_PROTO(struct kbase_va_region *reg, u8 jit_id), + TP_ARGS(reg, jit_id)); + +/* trace_mali_jit_free() + * + * Tracepoint about memory that was allocated just-in-time being freed + * (which may happen either on free soft-job, or during rollback error + * paths of an allocation soft-job, etc) + * + * Free doesn't immediately have the just-in-time memory allocation ID so + * it's currently suppressed from the output - set jit_id to 0 + */ +DEFINE_EVENT_PRINT(mali_jit_softjob_template, mali_jit_free, + TP_PROTO(struct kbase_va_region *reg, u8 jit_id), + TP_ARGS(reg, jit_id), + TP_printk("start=0x%llx va_pages=0x%zx backed_size=0x%zx", + __entry->start_addr, __entry->nr_pages, __entry->backed_pages)); + +#if !MALI_USE_CSF +#if MALI_JIT_PRESSURE_LIMIT_BASE +/* trace_mali_jit_report + * + * Tracepoint about the GPU data structure read to form a just-in-time memory + * allocation report, and its calculated physical page usage + */ +TRACE_EVENT(mali_jit_report, + TP_PROTO(struct kbase_jd_atom *katom, struct kbase_va_region *reg, + unsigned int id_idx, u64 read_val, u64 used_pages), + TP_ARGS(katom, reg, id_idx, read_val, used_pages), + TP_STRUCT__entry( + __field(u64, start_addr) + __field(u64, read_val) + __field(u64, used_pages) + __field(unsigned long, flags) + __field(u8, id_idx) + __field(u8, jit_id) + ), + TP_fast_assign( + __entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; + __entry->read_val = read_val; + __entry->used_pages = used_pages; + __entry->flags = reg->flags; + __entry->id_idx = id_idx; + __entry->jit_id = katom->jit_ids[id_idx]; + ), + TP_printk("start=0x%llx jit_ids[%u]=%u read_type='%s' read_val=0x%llx used_pages=%llu", + __entry->start_addr, __entry->id_idx, __entry->jit_id, + __print_symbolic(__entry->flags, + { 0, "address"}, + { KBASE_REG_TILER_ALIGN_TOP, "address with align" }, + { KBASE_REG_HEAP_INFO_IS_SIZE, "size" }, + { KBASE_REG_HEAP_INFO_IS_SIZE | + KBASE_REG_TILER_ALIGN_TOP, + "size with align (invalid)" } + ), + __entry->read_val, __entry->used_pages) +); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ +#endif /* !MALI_USE_CSF */ + +#if (KERNEL_VERSION(4, 1, 0) <= LINUX_VERSION_CODE) +TRACE_DEFINE_ENUM(KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); +#endif + +#if MALI_JIT_PRESSURE_LIMIT_BASE +/* trace_mali_jit_report_pressure + * + * Tracepoint about change in physical memory pressure, due to the information + * about a region changing. Examples include: + * - a report on a region that was allocated just-in-time + * - just-in-time allocation of a region + * - free of a region that was allocated just-in-time + */ +TRACE_EVENT(mali_jit_report_pressure, + TP_PROTO(struct kbase_va_region *reg, u64 new_used_pages, + u64 new_pressure, unsigned int flags), + TP_ARGS(reg, new_used_pages, new_pressure, flags), + TP_STRUCT__entry( + __field(u64, start_addr) + __field(u64, used_pages) + __field(u64, new_used_pages) + __field(u64, new_pressure) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; + __entry->used_pages = reg->used_pages; + __entry->new_used_pages = new_used_pages; + __entry->new_pressure = new_pressure; + __entry->flags = flags; + ), + TP_printk("start=0x%llx old_used_pages=%llu new_used_pages=%llu new_pressure=%llu report_flags=%s", + __entry->start_addr, __entry->used_pages, + __entry->new_used_pages, __entry->new_pressure, + __print_flags(__entry->flags, "|", + { KBASE_JIT_REPORT_ON_ALLOC_OR_FREE, + "HAPPENED_ON_ALLOC_OR_FREE" })) +); +#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ + +#ifndef __TRACE_SYSGRAPH_ENUM +#define __TRACE_SYSGRAPH_ENUM +/* Enum of sysgraph message IDs */ +enum sysgraph_msg { + SGR_ARRIVE, + SGR_DEP_RES, + SGR_SUBMIT, + SGR_COMPLETE, + SGR_POST, + SGR_ACTIVE, + SGR_INACTIVE +}; +#endif /* __TRACE_SYSGRAPH_ENUM */ + +/* A template for SYSGRAPH events + * + * Most of the sysgraph events contain only one input argument + * which is atom_id therefore they will be using a common template + */ +TRACE_EVENT(sysgraph, + TP_PROTO(enum sysgraph_msg message, unsigned int proc_id, + unsigned int atom_id), + TP_ARGS(message, proc_id, atom_id), + TP_STRUCT__entry( + __field(unsigned int, proc_id) + __field(enum sysgraph_msg, message) + __field(unsigned int, atom_id) + ), + TP_fast_assign( + __entry->proc_id = proc_id; + __entry->message = message; + __entry->atom_id = atom_id; + ), + TP_printk("msg=%u proc_id=%u, param1=%d\n", __entry->message, + __entry->proc_id, __entry->atom_id) +); + +/* A template for SYSGRAPH GPU events + * + * Sysgraph events that record start/complete events + * on GPU also record a js value in addition to the + * atom id. + */ +TRACE_EVENT(sysgraph_gpu, + TP_PROTO(enum sysgraph_msg message, unsigned int proc_id, + unsigned int atom_id, unsigned int js), + TP_ARGS(message, proc_id, atom_id, js), + TP_STRUCT__entry( + __field(unsigned int, proc_id) + __field(enum sysgraph_msg, message) + __field(unsigned int, atom_id) + __field(unsigned int, js) + ), + TP_fast_assign( + __entry->proc_id = proc_id; + __entry->message = message; + __entry->atom_id = atom_id; + __entry->js = js; + ), + TP_printk("msg=%u proc_id=%u, param1=%d, param2=%d\n", + __entry->message, __entry->proc_id, + __entry->atom_id, __entry->js) +); + +/* Tracepoint files get included more than once - protect against multiple + * definition + */ +#undef KBASE_JIT_REPORT_GPU_MEM_SIZE + +/* Size in bytes of the memory surrounding the location used for a just-in-time + * memory allocation report + */ +#define KBASE_JIT_REPORT_GPU_MEM_SIZE (4 * sizeof(u64)) + +/* trace_mali_jit_report_gpu_mem + * + * Tracepoint about the GPU memory nearby the location used for a just-in-time + * memory allocation report + */ +TRACE_EVENT(mali_jit_report_gpu_mem, + TP_PROTO(u64 base_addr, u64 reg_addr, u64 *gpu_mem, unsigned int flags), + TP_ARGS(base_addr, reg_addr, gpu_mem, flags), + TP_STRUCT__entry( + __field(u64, base_addr) + __field(u64, reg_addr) + __array(u64, mem_values, + KBASE_JIT_REPORT_GPU_MEM_SIZE / sizeof(u64)) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->base_addr = base_addr; + __entry->reg_addr = reg_addr; + memcpy(__entry->mem_values, gpu_mem, + sizeof(__entry->mem_values)); + __entry->flags = flags; + ), + TP_printk("start=0x%llx read GPU memory base=0x%llx values=%s report_flags=%s", + __entry->reg_addr, __entry->base_addr, + __print_array(__entry->mem_values, + ARRAY_SIZE(__entry->mem_values), sizeof(u64)), + __print_flags(__entry->flags, "|", + { KBASE_JIT_REPORT_ON_ALLOC_OR_FREE, + "HAPPENED_ON_ALLOC_OR_FREE" })) +); + +/* trace_mali_jit_trim_from_region + * + * Tracepoint about trimming physical pages from a region + */ +TRACE_EVENT(mali_jit_trim_from_region, + TP_PROTO(struct kbase_va_region *reg, size_t freed_pages, + size_t old_pages, size_t available_pages, size_t new_pages), + TP_ARGS(reg, freed_pages, old_pages, available_pages, new_pages), + TP_STRUCT__entry( + __field(u64, start_addr) + __field(size_t, freed_pages) + __field(size_t, old_pages) + __field(size_t, available_pages) + __field(size_t, new_pages) + ), + TP_fast_assign( + __entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; + __entry->freed_pages = freed_pages; + __entry->old_pages = old_pages; + __entry->available_pages = available_pages; + __entry->new_pages = new_pages; + ), + TP_printk("start=0x%llx freed_pages=%zu old_pages=%zu available_pages=%zu new_pages=%zu", + __entry->start_addr, __entry->freed_pages, __entry->old_pages, + __entry->available_pages, __entry->new_pages) +); + +/* trace_mali_jit_trim + * + * Tracepoint about total trimmed physical pages + */ +TRACE_EVENT(mali_jit_trim, + TP_PROTO(size_t freed_pages), + TP_ARGS(freed_pages), + TP_STRUCT__entry( + __field(size_t, freed_pages) + ), + TP_fast_assign( + __entry->freed_pages = freed_pages; + ), + TP_printk("freed_pages=%zu", __entry->freed_pages) +); + +#include "mali_kbase_debug_linux_ktrace.h" + +#endif /* _TRACE_MALI_H */ + +#undef TRACE_INCLUDE_PATH +/* lwn.net/Articles/383362 suggests this should remain as '.', and instead + * extend CFLAGS + */ +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE mali_linux_trace + +/* This part must be outside protection */ +#include diff --git a/drivers/gpu/arm/bifrost/mali_malisw.h b/drivers/gpu/arm/bifrost/mali_malisw.h new file mode 100755 index 000000000000..3a4db10bdb3d --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_malisw.h @@ -0,0 +1,109 @@ +/* + * + * (C) COPYRIGHT 2014-2015, 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Kernel-wide include for common macros and types. + */ + +#ifndef _MALISW_H_ +#define _MALISW_H_ + +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#define U8_MAX ((u8)~0U) +#define S8_MAX ((s8)(U8_MAX>>1)) +#define S8_MIN ((s8)(-S8_MAX - 1)) +#define U16_MAX ((u16)~0U) +#define S16_MAX ((s16)(U16_MAX>>1)) +#define S16_MIN ((s16)(-S16_MAX - 1)) +#define U32_MAX ((u32)~0U) +#define S32_MAX ((s32)(U32_MAX>>1)) +#define S32_MIN ((s32)(-S32_MAX - 1)) +#define U64_MAX ((u64)~0ULL) +#define S64_MAX ((s64)(U64_MAX>>1)) +#define S64_MIN ((s64)(-S64_MAX - 1)) +#endif /* LINUX_VERSION_CODE */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) +#define SIZE_MAX (~(size_t)0) +#endif /* LINUX_VERSION_CODE */ + +/** + * MIN - Return the lesser of two values. + * + * As a macro it may evaluate its arguments more than once. + * Refer to MAX macro for more details + */ +#define MIN(x, y) ((x) < (y) ? (x) : (y)) + +/** + * MAX - Return the greater of two values. + * + * As a macro it may evaluate its arguments more than once. + * If called on the same two arguments as MIN it is guaranteed to return + * the one that MIN didn't return. This is significant for types where not + * all values are comparable e.g. NaNs in floating-point types. But if you want + * to retrieve the min and max of two values, consider using a conditional swap + * instead. + */ +#define MAX(x, y) ((x) < (y) ? (y) : (x)) + +/** + * @hideinitializer + * Function-like macro for suppressing unused variable warnings. Where possible + * such variables should be removed; this macro is present for cases where we + * much support API backwards compatibility. + */ +#define CSTD_UNUSED(x) ((void)(x)) + +/** + * @hideinitializer + * Function-like macro for use where "no behavior" is desired. This is useful + * when compile time macros turn a function-like macro in to a no-op, but + * where having no statement is otherwise invalid. + */ +#define CSTD_NOP(...) ((void)#__VA_ARGS__) + +/** + * @hideinitializer + * Function-like macro for stringizing a single level macro. + * @code + * #define MY_MACRO 32 + * CSTD_STR1( MY_MACRO ) + * > "MY_MACRO" + * @endcode + */ +#define CSTD_STR1(x) #x + +/** + * @hideinitializer + * Function-like macro for stringizing a macro's value. This should not be used + * if the macro is defined in a way which may have no value; use the + * alternative @c CSTD_STR2N macro should be used instead. + * @code + * #define MY_MACRO 32 + * CSTD_STR2( MY_MACRO ) + * > "32" + * @endcode + */ +#define CSTD_STR2(x) CSTD_STR1(x) + +#endif /* _MALISW_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.c b/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.c new file mode 100755 index 000000000000..b6fb5a094fab --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.c @@ -0,0 +1,27 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* Create the trace point if not configured in kernel */ +#ifndef CONFIG_TRACE_POWER_GPU_FREQUENCY +#define CREATE_TRACE_POINTS +#include "mali_power_gpu_frequency_trace.h" +#endif diff --git a/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.h b/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.h new file mode 100755 index 000000000000..3b90ae437db9 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_power_gpu_frequency_trace.h @@ -0,0 +1,69 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _TRACE_POWER_GPU_FREQUENCY_MALI +#define _TRACE_POWER_GPU_FREQUENCY_MALI +#endif + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM power +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE mali_power_gpu_frequency_trace +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#if !defined(_TRACE_POWER_GPU_FREQUENCY_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_POWER_GPU_FREQUENCY_H + +#include + +DECLARE_EVENT_CLASS(gpu, + + TP_PROTO(unsigned int state, unsigned int gpu_id), + + TP_ARGS(state, gpu_id), + + TP_STRUCT__entry( + __field( u32, state ) + __field( u32, gpu_id ) + ), + + TP_fast_assign( + __entry->state = state; + __entry->gpu_id = gpu_id; + ), + + TP_printk("state=%lu gpu_id=%lu", (unsigned long)__entry->state, + (unsigned long)__entry->gpu_id) +); + +DEFINE_EVENT(gpu, gpu_frequency, + + TP_PROTO(unsigned int frequency, unsigned int gpu_id), + + TP_ARGS(frequency, gpu_id) +); + +#endif /* _TRACE_POWER_GPU_FREQUENCY_H */ + +/* This part must be outside protection */ +#include diff --git a/drivers/gpu/arm/bifrost/mali_uk.h b/drivers/gpu/arm/bifrost/mali_uk.h new file mode 100755 index 000000000000..701f3909042f --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_uk.h @@ -0,0 +1,84 @@ +/* + * + * (C) COPYRIGHT 2010, 2012-2015, 2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +/** + * @file mali_uk.h + * Types and definitions that are common across OSs for both the user + * and kernel side of the User-Kernel interface. + */ + +#ifndef _UK_H_ +#define _UK_H_ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @defgroup uk_api User-Kernel Interface API + * + * The User-Kernel Interface abstracts the communication mechanism between the user and kernel-side code of device + * drivers developed as part of the Midgard DDK. Currently that includes the Base driver. + * + * It exposes an OS independent API to user-side code (UKU) which routes functions calls to an OS-independent + * kernel-side API (UKK) via an OS-specific communication mechanism. + * + * This API is internal to the Midgard DDK and is not exposed to any applications. + * + * @{ + */ + +/** + * These are identifiers for kernel-side drivers implementing a UK interface, aka UKK clients. The + * UK module maps this to an OS specific device name, e.g. "gpu_base" -> "GPU0:". Specify this + * identifier to select a UKK client to the uku_open() function. + * + * When a new UKK client driver is created a new identifier needs to be added to the uk_client_id + * enumeration and the uku_open() implemenation for the various OS ports need to be updated to + * provide a mapping of the identifier to the OS specific device name. + * + */ +enum uk_client_id { + /** + * Value used to identify the Base driver UK client. + */ + UK_CLIENT_MALI_T600_BASE, + + /** The number of uk clients supported. This must be the last member of the enum */ + UK_CLIENT_COUNT +}; + +/** @} end group uk_api */ + +/** @} *//* end group base_api */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif /* _UK_H_ */ diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c new file mode 100755 index 000000000000..1d106999228a --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c @@ -0,0 +1,532 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Base kernel MMU management specific for CSF GPU. + */ + +#include +#include +#include +#include +#include +#include +#include "../mali_kbase_mmu_internal.h" + +void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, + struct kbase_mmu_setup * const setup) +{ + /* Set up the required caching policies at the correct indices + * in the memattr register. + */ + setup->memattr = + (AS_MEMATTR_IMPL_DEF_CACHE_POLICY << + (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | + (AS_MEMATTR_FORCE_TO_CACHE_ALL << + (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | + (AS_MEMATTR_WRITE_ALLOC << + (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | + (AS_MEMATTR_AARCH64_OUTER_IMPL_DEF << + (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | + (AS_MEMATTR_AARCH64_OUTER_WA << + (AS_MEMATTR_INDEX_OUTER_WA * 8)) | + (AS_MEMATTR_AARCH64_NON_CACHEABLE << + (AS_MEMATTR_INDEX_NON_CACHEABLE * 8)) | + (AS_MEMATTR_AARCH64_SHARED << + (AS_MEMATTR_INDEX_SHARED * 8)); + + setup->transtab = (u64)mmut->pgd & AS_TRANSTAB_BASE_MASK; + setup->transcfg = AS_TRANSCFG_ADRMODE_AARCH64_4K; +} + +/** + * submit_work_pagefault() - Submit a work for MMU page fault. + * + * @kbdev: Kbase device pointer + * @as_nr: Faulty address space + * @fault: Data relating to the fault + * + * This function submits a work for reporting the details of MMU fault. + */ +static void submit_work_pagefault(struct kbase_device *kbdev, u32 as_nr, + struct kbase_fault *fault) +{ + struct kbase_as *const as = &kbdev->as[as_nr]; + + as->pf_data = (struct kbase_fault) { + .status = fault->status, + .addr = fault->addr, + }; + + if (kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_nr)) { + WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); + atomic_inc(&kbdev->faults_pending); + } +} + +void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, + struct kbase_fault *fault) +{ + /* decode the fault status */ + u32 exception_type = fault->status & 0xFF; + u32 access_type = (fault->status >> 8) & 0x3; + u32 source_id = (fault->status >> 16); + int as_no; + + /* terminal fault, print info about the fault */ + dev_err(kbdev->dev, + "Unexpected Page fault in firmware address space at VA 0x%016llX\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X\n", + fault->addr, + fault->status, + exception_type, kbase_gpu_exception_name(exception_type), + access_type, kbase_gpu_access_type_name(fault->status), + source_id); + + /* Report MMU fault for all address spaces (except MCU_AS_NR) */ + for (as_no = 1; as_no < kbdev->nr_hw_address_spaces; as_no++) + if (kbase_ctx_sched_as_to_ctx(kbdev, as_no)) + submit_work_pagefault(kbdev, as_no, fault); + + /* GPU reset is required to recover */ + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); +} +KBASE_EXPORT_TEST_API(kbase_mmu_report_mcu_as_fault_and_reset); + +void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, + struct kbase_as *as, struct kbase_fault *fault) +{ + struct kbase_device *kbdev = kctx->kbdev; + u32 const status = fault->status; + int exception_type = (status & GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) >> + GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT; + int access_type = (status & GPU_FAULTSTATUS_ACCESS_TYPE_MASK) >> + GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT; + int source_id = (status & GPU_FAULTSTATUS_SOURCE_ID_MASK) >> + GPU_FAULTSTATUS_SOURCE_ID_SHIFT; + const char *addr_valid = (status & GPU_FAULTSTATUS_ADDR_VALID_FLAG) ? + "true" : "false"; + int as_no = as->number; + unsigned long flags; + + /* terminal fault, print info about the fault */ + dev_err(kbdev->dev, + "GPU bus fault in AS%d at VA 0x%016llX\n" + "VA_VALID: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X\n" + "pid: %d\n", + as_no, fault->addr, + addr_valid, + status, + exception_type, kbase_gpu_exception_name(exception_type), + access_type, kbase_gpu_access_type_name(access_type), + source_id, + kctx->pid); + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_disable(kctx); + kbase_ctx_flag_set(kctx, KCTX_AS_DISABLED_ON_FAULT); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + /* Switching to UNMAPPED mode above would have enabled the firmware to + * recover from the fault (if the memory access was made by firmware) + * and it can then respond to CSG termination requests to be sent now. + * All GPU command queue groups associated with the context would be + * affected as they use the same GPU address space. + */ + kbase_csf_ctx_handle_fault(kctx, fault); + + /* Now clear the GPU fault */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAR_FAULT); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +/** + * The caller must ensure it's retained the ctx to prevent it from being + * scheduled out whilst it's being worked on. + */ +void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, + struct kbase_as *as, const char *reason_str, + struct kbase_fault *fault) +{ + unsigned long flags; + unsigned int exception_type; + unsigned int access_type; + unsigned int source_id; + int as_no; + struct kbase_device *kbdev; + const u32 status = fault->status; + + as_no = as->number; + kbdev = kctx->kbdev; + + /* Make sure the context was active */ + if (WARN_ON(atomic_read(&kctx->refcount) <= 0)) + return; + + /* decode the fault status */ + exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); + access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); + source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); + + /* terminal fault, print info about the fault */ + dev_err(kbdev->dev, + "Unhandled Page fault in AS%d at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X\n" + "pid: %d\n", + as_no, fault->addr, + reason_str, + status, + exception_type, kbase_gpu_exception_name(exception_type), + access_type, kbase_gpu_access_type_name(status), + source_id, + kctx->pid); + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + + /* switch to UNMAPPED mode, + * will abort all jobs and stop any hw counter dumping + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_disable(kctx); + kbase_ctx_flag_set(kctx, KCTX_AS_DISABLED_ON_FAULT); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + + /* Switching to UNMAPPED mode above would have enabled the firmware to + * recover from the fault (if the memory access was made by firmware) + * and it can then respond to CSG termination requests to be sent now. + * All GPU command queue groups associated with the context would be + * affected as they use the same GPU address space. + */ + kbase_csf_ctx_handle_fault(kctx, fault); + + /* Clear down the fault */ + kbase_mmu_hw_clear_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); +} + +/** + * kbase_mmu_interrupt_process() - Process a bus or page fault. + * @kbdev: The kbase_device the fault happened on + * @kctx: The kbase_context for the faulting address space if one was + * found. + * @as: The address space that has the fault + * @fault: Data relating to the fault + * + * This function will process a fault on a specific address space + */ +static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_as *as, + struct kbase_fault *fault) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!kctx) { + dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Spurious IRQ or SW Design Error?\n", + kbase_as_has_bus_fault(as, fault) ? + "Bus error" : "Page fault", + as->number, fault->addr); + + /* Since no ctx was found, the MMU must be disabled. */ + WARN_ON(as->current_setup.transtab); + + if (kbase_as_has_bus_fault(as, fault)) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAR_FAULT); + else if (kbase_as_has_page_fault(as, fault)) { + kbase_mmu_hw_clear_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + } + + return; + } + + if (kbase_as_has_bus_fault(as, fault)) { + /* + * We need to switch to UNMAPPED mode - but we do this in a + * worker so that we can sleep + */ + WARN_ON(!queue_work(as->pf_wq, &as->work_busfault)); + atomic_inc(&kbdev->faults_pending); + } else { + WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); + atomic_inc(&kbdev->faults_pending); + } +} + +int kbase_mmu_bus_fault_interrupt(struct kbase_device *kbdev, + u32 status, u32 as_nr) +{ + struct kbase_context *kctx; + unsigned long flags; + struct kbase_as *as; + struct kbase_fault *fault; + + if (WARN_ON(as_nr == MCU_AS_NR)) + return -EINVAL; + + if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) + return -EINVAL; + + as = &kbdev->as[as_nr]; + fault = &as->bf_data; + fault->status = status; + fault->addr = (u64) kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32; + fault->addr |= kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTADDRESS_LO)); + fault->protected_mode = false; + + /* report the fault to debugfs */ + kbase_as_fault_debugfs_new(kbdev, as_nr); + + kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_nr); + + /* Process the bus fault interrupt for this address space */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_interrupt_process(kbdev, kctx, as, fault); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return 0; +} + +void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) +{ + const int num_as = 16; + const int pf_shift = 0; + const unsigned long as_bit_mask = (1UL << num_as) - 1; + unsigned long flags; + u32 new_mask; + u32 tmp; + u32 pf_bits = ((irq_stat >> pf_shift) & as_bit_mask); + + /* remember current mask */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)); + /* mask interrupts for now */ + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0); + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); + + while (pf_bits) { + struct kbase_context *kctx; + int as_no = ffs(pf_bits) - 1; + struct kbase_as *as = &kbdev->as[as_no]; + struct kbase_fault *fault = &as->pf_data; + + /* find faulting address */ + fault->addr = kbase_reg_read(kbdev, MMU_AS_REG(as_no, + AS_FAULTADDRESS_HI)); + fault->addr <<= 32; + fault->addr |= kbase_reg_read(kbdev, MMU_AS_REG(as_no, + AS_FAULTADDRESS_LO)); + + /* Mark the fault protected or not */ + fault->protected_mode = false; + + /* report the fault to debugfs */ + kbase_as_fault_debugfs_new(kbdev, as_no); + + /* record the fault status */ + fault->status = kbase_reg_read(kbdev, MMU_AS_REG(as_no, + AS_FAULTSTATUS)); + + fault->extra_addr = kbase_reg_read(kbdev, + MMU_AS_REG(as_no, AS_FAULTEXTRA_HI)); + fault->extra_addr <<= 32; + fault->extra_addr |= kbase_reg_read(kbdev, + MMU_AS_REG(as_no, AS_FAULTEXTRA_LO)); + + /* Mark page fault as handled */ + pf_bits &= ~(1UL << as_no); + + /* remove the queued PF from the mask */ + new_mask &= ~MMU_PAGE_FAULT(as_no); + + if (as_no == MCU_AS_NR) { + kbase_mmu_report_mcu_as_fault_and_reset(kbdev, fault); + /* Pointless to handle remaining faults */ + break; + } + + /* + * Refcount the kctx - it shouldn't disappear anyway, since + * Page faults _should_ only occur whilst GPU commands are + * executing, and a command causing the Page fault shouldn't + * complete until the MMU is updated. + * Reference is released at the end of bottom half of page + * fault handling. + */ + kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_no); + + /* Process the interrupt for this address space */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_interrupt_process(kbdev, kctx, as, fault); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + /* reenable interrupts */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)); + new_mask |= tmp; + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask); + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); +} + +int kbase_mmu_switch_to_ir(struct kbase_context *const kctx, + struct kbase_va_region *const reg) +{ + /* Can't soft-stop the provoking job */ + return -EPERM; +} + +/** + * kbase_mmu_gpu_fault_worker() - Process a GPU fault for the device. + * + * @data: work_struct passed by queue_work() + * + * Report a GPU fatal error for all GPU command queue groups that are + * using the address space and terminate them. + */ +static void kbase_mmu_gpu_fault_worker(struct work_struct *data) +{ + struct kbase_as *const faulting_as = container_of(data, struct kbase_as, + work_gpufault); + const u32 as_nr = faulting_as->number; + struct kbase_device *const kbdev = container_of(faulting_as, struct + kbase_device, as[as_nr]); + struct kbase_fault *fault; + struct kbase_context *kctx; + u32 status; + u64 address; + u32 as_valid; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + fault = &faulting_as->gf_data; + status = fault->status; + as_valid = status & GPU_FAULTSTATUS_JASID_VALID_FLAG; + address = fault->addr; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + dev_warn(kbdev->dev, + "GPU Fault 0x%08x (%s) in AS%u at 0x%016llx\n" + "ASID_VALID: %s, ADDRESS_VALID: %s\n", + status, + kbase_gpu_exception_name( + GPU_FAULTSTATUS_EXCEPTION_TYPE_GET(status)), + as_nr, address, + as_valid ? "true" : "false", + status & GPU_FAULTSTATUS_ADDR_VALID_FLAG ? "true" : "false"); + + kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_nr); + kbase_csf_ctx_handle_fault(kctx, fault); + kbase_ctx_sched_release_ctx_lock(kctx); + + atomic_dec(&kbdev->faults_pending); +} + +/** + * submit_work_gpufault() - Submit a work for GPU fault. + * + * @kbdev: Kbase device pointer + * @status: GPU fault status + * @as_nr: Faulty address space + * @address: GPU fault address + * + * This function submits a work for reporting the details of GPU fault. + */ +static void submit_work_gpufault(struct kbase_device *kbdev, u32 status, + u32 as_nr, u64 address) +{ + unsigned long flags; + struct kbase_as *const as = &kbdev->as[as_nr]; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + as->gf_data = (struct kbase_fault) { + .status = status, + .addr = address, + }; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_nr)) { + WARN_ON(!queue_work(as->pf_wq, &as->work_gpufault)); + atomic_inc(&kbdev->faults_pending); + } +} + +void kbase_mmu_gpu_fault_interrupt(struct kbase_device *kbdev, u32 status, + u32 as_nr, u64 address, bool as_valid) +{ + if (!as_valid || (as_nr == MCU_AS_NR)) { + int as; + + /* Report GPU fault for all contexts (except MCU_AS_NR) in case either + * the address space is invalid or it's MCU address space. + */ + for (as = 1; as < kbdev->nr_hw_address_spaces; as++) + submit_work_gpufault(kbdev, status, as, address); + } else + submit_work_gpufault(kbdev, status, as_nr, address); +} +KBASE_EXPORT_TEST_API(kbase_mmu_gpu_fault_interrupt); + +int kbase_mmu_as_init(struct kbase_device *kbdev, int i) +{ + kbdev->as[i].number = i; + kbdev->as[i].bf_data.addr = 0ULL; + kbdev->as[i].pf_data.addr = 0ULL; + kbdev->as[i].gf_data.addr = 0ULL; + + kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%d", 0, 1, i); + if (!kbdev->as[i].pf_wq) + return -ENOMEM; + + INIT_WORK(&kbdev->as[i].work_pagefault, kbase_mmu_page_fault_worker); + INIT_WORK(&kbdev->as[i].work_busfault, kbase_mmu_bus_fault_worker); + INIT_WORK(&kbdev->as[i].work_gpufault, kbase_mmu_gpu_fault_worker); + + return 0; +} diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c new file mode 100755 index 000000000000..b0187a46b733 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c @@ -0,0 +1,440 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Base kernel MMU management specific for Job Manager GPU. + */ + +#include +#include +#include +#include +#include +#include "../mali_kbase_mmu_internal.h" + +void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, + struct kbase_mmu_setup * const setup) +{ + /* Set up the required caching policies at the correct indices + * in the memattr register. + */ + setup->memattr = + (AS_MEMATTR_IMPL_DEF_CACHE_POLICY << + (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | + (AS_MEMATTR_FORCE_TO_CACHE_ALL << + (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | + (AS_MEMATTR_WRITE_ALLOC << + (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | + (AS_MEMATTR_AARCH64_OUTER_IMPL_DEF << + (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | + (AS_MEMATTR_AARCH64_OUTER_WA << + (AS_MEMATTR_INDEX_OUTER_WA * 8)) | + (AS_MEMATTR_AARCH64_NON_CACHEABLE << + (AS_MEMATTR_INDEX_NON_CACHEABLE * 8)); + + setup->transtab = (u64)mmut->pgd & AS_TRANSTAB_BASE_MASK; + setup->transcfg = AS_TRANSCFG_ADRMODE_AARCH64_4K; +} + +void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, + struct kbase_as *as, struct kbase_fault *fault) +{ + struct kbase_device *const kbdev = kctx->kbdev; + u32 const status = fault->status; + u32 const exception_type = (status & 0xFF); + u32 const exception_data = (status >> 8) & 0xFFFFFF; + int const as_no = as->number; + unsigned long flags; + + /* terminal fault, print info about the fault */ + dev_err(kbdev->dev, + "GPU bus fault in AS%d at VA 0x%016llX\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "exception data 0x%X\n" + "pid: %d\n", + as_no, fault->addr, + status, + exception_type, kbase_gpu_exception_name(exception_type), + exception_data, + kctx->pid); + + /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter + * dumping AS transaction begin + */ + mutex_lock(&kbdev->mmu_hw_mutex); + + /* Set the MMU into unmapped mode */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_disable(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + + kbase_mmu_hw_clear_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); +} + +/** + * The caller must ensure it's retained the ctx to prevent it from being + * scheduled out whilst it's being worked on. + */ +void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, + struct kbase_as *as, const char *reason_str, + struct kbase_fault *fault) +{ + unsigned long flags; + u32 exception_type; + u32 access_type; + u32 source_id; + int as_no; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + + as_no = as->number; + kbdev = kctx->kbdev; + js_devdata = &kbdev->js_data; + + /* Make sure the context was active */ + if (WARN_ON(atomic_read(&kctx->refcount) <= 0)) + return; + + /* decode the fault status */ + exception_type = fault->status & 0xFF; + access_type = (fault->status >> 8) & 0x3; + source_id = (fault->status >> 16); + + /* terminal fault, print info about the fault */ + dev_err(kbdev->dev, + "Unhandled Page fault in AS%d at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X\n" + "pid: %d\n", + as_no, fault->addr, + reason_str, + fault->status, + exception_type, kbase_gpu_exception_name(exception_type), + access_type, kbase_gpu_access_type_name(fault->status), + source_id, + kctx->pid); + + /* hardware counters dump fault handling */ + if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && + (kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_DUMPING)) { + if ((fault->addr >= kbdev->hwcnt.addr) && + (fault->addr < (kbdev->hwcnt.addr + + kbdev->hwcnt.addr_bytes))) + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; + } + + /* Stop the kctx from submitting more jobs and cause it to be scheduled + * out/rescheduled - this will occur on releasing the context's refcount + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbasep_js_clear_submit_allowed(js_devdata, kctx); + + /* Kill any running jobs from the context. Submit is disallowed, so no + * more jobs from this context can appear in the job slots from this + * point on + */ + kbase_backend_jm_kill_running_jobs_from_kctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + + /* switch to UNMAPPED mode, will abort all jobs and stop + * any hw counter dumping + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_disable(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->mmu_hw_mutex); + + /* AS transaction end */ + /* Clear down the fault */ + kbase_mmu_hw_clear_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); +} + +/** + * kbase_mmu_interrupt_process() - Process a bus or page fault. + * @kbdev: The kbase_device the fault happened on + * @kctx: The kbase_context for the faulting address space if one was + * found. + * @as: The address space that has the fault + * @fault: Data relating to the fault + * + * This function will process a fault on a specific address space + */ +static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_as *as, + struct kbase_fault *fault) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + dev_dbg(kbdev->dev, + "Entering %s kctx %p, as %p\n", + __func__, (void *)kctx, (void *)as); + + if (!kctx) { + dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Spurious IRQ or SW Design Error?\n", + kbase_as_has_bus_fault(as, fault) ? + "Bus error" : "Page fault", + as->number, fault->addr); + + /* Since no ctx was found, the MMU must be disabled. */ + WARN_ON(as->current_setup.transtab); + + if (kbase_as_has_bus_fault(as, fault)) { + kbase_mmu_hw_clear_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + } else if (kbase_as_has_page_fault(as, fault)) { + kbase_mmu_hw_clear_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + } + + return; + } + + if (kbase_as_has_bus_fault(as, fault)) { + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + /* + * hw counters dumping in progress, signal the + * other thread that it failed + */ + if ((kbdev->hwcnt.kctx == kctx) && + (kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_DUMPING)) + kbdev->hwcnt.backend.state = + KBASE_INSTR_STATE_FAULT; + + /* + * Stop the kctx from submitting more jobs and cause it + * to be scheduled out/rescheduled when all references + * to it are released + */ + kbasep_js_clear_submit_allowed(js_devdata, kctx); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + dev_warn(kbdev->dev, + "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", + as->number, fault->addr, + fault->extra_addr); + else + dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", + as->number, fault->addr); + + /* + * We need to switch to UNMAPPED mode - but we do this in a + * worker so that we can sleep + */ + WARN_ON(!queue_work(as->pf_wq, &as->work_busfault)); + atomic_inc(&kbdev->faults_pending); + } else { + WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); + atomic_inc(&kbdev->faults_pending); + } + + dev_dbg(kbdev->dev, + "Leaving %s kctx %p, as %p\n", + __func__, (void *)kctx, (void *)as); +} + +static void validate_protected_page_fault(struct kbase_device *kbdev) +{ + /* GPUs which support (native) protected mode shall not report page + * fault addresses unless it has protected debug mode and protected + * debug mode is turned on + */ + u32 protected_debug_mode = 0; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { + protected_debug_mode = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_STATUS)) & GPU_DBGEN; + } + + if (!protected_debug_mode) { + /* fault_addr should never be reported in protected mode. + * However, we just continue by printing an error message + */ + dev_err(kbdev->dev, "Fault address reported in protected mode\n"); + } +} + +void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) +{ + const int num_as = 16; + const int busfault_shift = MMU_PAGE_FAULT_FLAGS; + const int pf_shift = 0; + const unsigned long as_bit_mask = (1UL << num_as) - 1; + unsigned long flags; + u32 new_mask; + u32 tmp, bf_bits, pf_bits; + + dev_dbg(kbdev->dev, "Entering %s irq_stat %u\n", + __func__, irq_stat); + /* bus faults */ + bf_bits = (irq_stat >> busfault_shift) & as_bit_mask; + /* page faults (note: Ignore ASes with both pf and bf) */ + pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits; + + if (WARN_ON(kbdev == NULL)) + return; + + /* remember current mask */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)); + /* mask interrupts for now */ + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0); + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); + + while (bf_bits | pf_bits) { + struct kbase_as *as; + int as_no; + struct kbase_context *kctx; + struct kbase_fault *fault; + + /* + * the while logic ensures we have a bit set, no need to check + * for not-found here + */ + as_no = ffs(bf_bits | pf_bits) - 1; + as = &kbdev->as[as_no]; + + /* find the fault type */ + if (bf_bits & (1 << as_no)) + fault = &as->bf_data; + else + fault = &as->pf_data; + + /* + * Refcount the kctx ASAP - it shouldn't disappear anyway, since + * Bus/Page faults _should_ only occur whilst jobs are running, + * and a job causing the Bus/Page fault shouldn't complete until + * the MMU is updated + */ + kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_no); + + /* find faulting address */ + fault->addr = kbase_reg_read(kbdev, MMU_AS_REG(as_no, + AS_FAULTADDRESS_HI)); + fault->addr <<= 32; + fault->addr |= kbase_reg_read(kbdev, MMU_AS_REG(as_no, + AS_FAULTADDRESS_LO)); + /* Mark the fault protected or not */ + fault->protected_mode = kbdev->protected_mode; + + if (kbdev->protected_mode && fault->addr) { + /* check if address reporting is allowed */ + validate_protected_page_fault(kbdev); + } + + /* report the fault to debugfs */ + kbase_as_fault_debugfs_new(kbdev, as_no); + + /* record the fault status */ + fault->status = kbase_reg_read(kbdev, MMU_AS_REG(as_no, + AS_FAULTSTATUS)); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { + fault->extra_addr = kbase_reg_read(kbdev, + MMU_AS_REG(as_no, AS_FAULTEXTRA_HI)); + fault->extra_addr <<= 32; + fault->extra_addr |= kbase_reg_read(kbdev, + MMU_AS_REG(as_no, AS_FAULTEXTRA_LO)); + } + + if (kbase_as_has_bus_fault(as, fault)) { + /* Mark bus fault as handled. + * Note that a bus fault is processed first in case + * where both a bus fault and page fault occur. + */ + bf_bits &= ~(1UL << as_no); + + /* remove the queued BF (and PF) from the mask */ + new_mask &= ~(MMU_BUS_ERROR(as_no) | + MMU_PAGE_FAULT(as_no)); + } else { + /* Mark page fault as handled */ + pf_bits &= ~(1UL << as_no); + + /* remove the queued PF from the mask */ + new_mask &= ~MMU_PAGE_FAULT(as_no); + } + + /* Process the interrupt for this address space */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_interrupt_process(kbdev, kctx, as, fault); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + /* reenable interrupts */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)); + new_mask |= tmp; + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask); + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); + + dev_dbg(kbdev->dev, "Leaving %s irq_stat %u\n", + __func__, irq_stat); +} + +int kbase_mmu_switch_to_ir(struct kbase_context *const kctx, + struct kbase_va_region *const reg) +{ + dev_dbg(kctx->kbdev->dev, + "Switching to incremental rendering for region %p\n", + (void *)reg); + return kbase_job_slot_softstop_start_rp(kctx, reg); +} + +int kbase_mmu_as_init(struct kbase_device *kbdev, int i) +{ + kbdev->as[i].number = i; + kbdev->as[i].bf_data.addr = 0ULL; + kbdev->as[i].pf_data.addr = 0ULL; + + kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%d", 0, 1, i); + if (!kbdev->as[i].pf_wq) + return -ENOMEM; + + INIT_WORK(&kbdev->as[i].work_pagefault, kbase_mmu_page_fault_worker); + INIT_WORK(&kbdev->as[i].work_busfault, kbase_mmu_bus_fault_worker); + + return 0; +} diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c new file mode 100755 index 000000000000..a5cda009426d --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c @@ -0,0 +1,2275 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * @file mali_kbase_mmu.c + * Base kernel MMU management. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#define KBASE_MMU_PAGE_ENTRIES 512 + +/** + * kbase_mmu_flush_invalidate() - Flush and invalidate the GPU caches. + * @kctx: The KBase context. + * @vpfn: The virtual page frame number to start the flush on. + * @nr: The number of pages to flush. + * @sync: Set if the operation should be synchronous or not. + * + * Issue a cache flush + invalidate to the GPU caches and invalidate the TLBs. + * + * If sync is not set then transactions still in flight when the flush is issued + * may use the old page tables and the data they write will not be written out + * to memory, this function returns after the flush has been issued but + * before all accesses which might effect the flushed region have completed. + * + * If sync is set then accesses in the flushed region will be drained + * before data is flush and invalidated through L1, L2 and into memory, + * after which point this function will return. + */ +static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, + u64 vpfn, size_t nr, bool sync); + +/** + * kbase_mmu_flush_invalidate_no_ctx() - Flush and invalidate the GPU caches. + * @kbdev: Device pointer. + * @vpfn: The virtual page frame number to start the flush on. + * @nr: The number of pages to flush. + * @sync: Set if the operation should be synchronous or not. + * @as_nr: GPU address space number for which flush + invalidate is required. + * + * This is used for MMU tables which do not belong to a user space context. + */ +static void kbase_mmu_flush_invalidate_no_ctx(struct kbase_device *kbdev, + u64 vpfn, size_t nr, bool sync, int as_nr); + +/** + * kbase_mmu_sync_pgd - sync page directory to memory + * @kbdev: Device pointer. + * @handle: Address of DMA region. + * @size: Size of the region to sync. + * + * This should be called after each page directory update. + */ + +static void kbase_mmu_sync_pgd(struct kbase_device *kbdev, + dma_addr_t handle, size_t size) +{ + /* If page table is not coherent then ensure the gpu can read + * the pages from memory + */ + if (kbdev->system_coherency != COHERENCY_ACE) + dma_sync_single_for_device(kbdev->dev, handle, size, + DMA_TO_DEVICE); +} + +/* + * Definitions: + * - PGD: Page Directory. + * - PTE: Page Table Entry. A 64bit value pointing to the next + * level of translation + * - ATE: Address Translation Entry. A 64bit value pointing to + * a 4kB physical page. + */ + +static int kbase_mmu_update_pages_no_flush(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags, int group_id); + +/** + * reg_grow_calc_extra_pages() - Calculate the number of backed pages to add to + * a region on a GPU page fault + * + * @reg: The region that will be backed with more pages + * @fault_rel_pfn: PFN of the fault relative to the start of the region + * + * This calculates how much to increase the backing of a region by, based on + * where a GPU page fault occurred and the flags in the region. + * + * This can be more than the minimum number of pages that would reach + * @fault_rel_pfn, for example to reduce the overall rate of page fault + * interrupts on a region, or to ensure that the end address is aligned. + * + * Return: the number of backed pages to increase by + */ +static size_t reg_grow_calc_extra_pages(struct kbase_device *kbdev, + struct kbase_va_region *reg, size_t fault_rel_pfn) +{ + size_t multiple = reg->extent; + size_t reg_current_size = kbase_reg_current_backed_size(reg); + size_t minimum_extra = fault_rel_pfn - reg_current_size + 1; + size_t remainder; + + if (!multiple) { + dev_warn(kbdev->dev, + "VA Region 0x%llx extent was 0, allocator needs to set this properly for KBASE_REG_PF_GROW\n", + ((unsigned long long)reg->start_pfn) << PAGE_SHIFT); + return minimum_extra; + } + + /* Calculate the remainder to subtract from minimum_extra to make it + * the desired (rounded down) multiple of the extent. + * Depending on reg's flags, the base used for calculating multiples is + * different + */ + + /* multiple is based from the current backed size, even if the + * current backed size/pfn for end of committed memory are not + * themselves aligned to multiple + */ + remainder = minimum_extra % multiple; + +#if !MALI_USE_CSF + if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { + /* multiple is based from the top of the initial commit, which + * has been allocated in such a way that (start_pfn + + * initial_commit) is already aligned to multiple. Hence the + * pfn for the end of committed memory will also be aligned to + * multiple + */ + size_t initial_commit = reg->initial_commit; + + if (fault_rel_pfn < initial_commit) { + /* this case is just to catch in case it's been + * recommitted by userspace to be smaller than the + * initial commit + */ + minimum_extra = initial_commit - reg_current_size; + remainder = 0; + } else { + /* same as calculating + * (fault_rel_pfn - initial_commit + 1) + */ + size_t pages_after_initial = minimum_extra + + reg_current_size - initial_commit; + + remainder = pages_after_initial % multiple; + } + } +#endif /* !MALI_USE_CSF */ + + if (remainder == 0) + return minimum_extra; + + return minimum_extra + multiple - remainder; +} + +#ifdef CONFIG_MALI_CINSTR_GWT +static void kbase_gpu_mmu_handle_write_faulting_as( + struct kbase_device *kbdev, + struct kbase_as *faulting_as, + u64 start_pfn, size_t nr, u32 op) +{ + mutex_lock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_clear_fault(kbdev, faulting_as, + KBASE_MMU_FAULT_TYPE_PAGE); + kbase_mmu_hw_do_operation(kbdev, faulting_as, start_pfn, + nr, op, 1); + + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_enable_fault(kbdev, faulting_as, + KBASE_MMU_FAULT_TYPE_PAGE); +} + +static void kbase_gpu_mmu_handle_write_fault(struct kbase_context *kctx, + struct kbase_as *faulting_as) +{ + struct kbasep_gwt_list_element *pos; + struct kbase_va_region *region; + struct kbase_device *kbdev; + struct kbase_fault *fault; + u64 fault_pfn, pfn_offset; + u32 op; + int ret; + int as_no; + + as_no = faulting_as->number; + kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); + fault = &faulting_as->pf_data; + fault_pfn = fault->addr >> PAGE_SHIFT; + + kbase_gpu_vm_lock(kctx); + + /* Find region and check if it should be writable. */ + region = kbase_region_tracker_find_region_enclosing_address(kctx, + fault->addr); + if (kbase_is_region_invalid_or_free(region)) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Memory is not mapped on the GPU", + &faulting_as->pf_data); + return; + } + + if (!(region->flags & KBASE_REG_GPU_WR)) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Region does not have write permissions", + &faulting_as->pf_data); + return; + } + + /* Capture addresses of faulting write location + * for job dumping if write tracking is enabled. + */ + if (kctx->gwt_enabled) { + u64 page_addr = fault->addr & PAGE_MASK; + bool found = false; + /* Check if this write was already handled. */ + list_for_each_entry(pos, &kctx->gwt_current_list, link) { + if (page_addr == pos->page_addr) { + found = true; + break; + } + } + + if (!found) { + pos = kmalloc(sizeof(*pos), GFP_KERNEL); + if (pos) { + pos->region = region; + pos->page_addr = page_addr; + pos->num_pages = 1; + list_add(&pos->link, &kctx->gwt_current_list); + } else { + dev_warn(kbdev->dev, "kmalloc failure"); + } + } + } + + pfn_offset = fault_pfn - region->start_pfn; + /* Now make this faulting page writable to GPU. */ + ret = kbase_mmu_update_pages_no_flush(kctx, fault_pfn, + &kbase_get_gpu_phy_pages(region)[pfn_offset], + 1, region->flags, region->gpu_alloc->group_id); + + /* flush L2 and unlock the VA (resumes the MMU) */ + op = AS_COMMAND_FLUSH_PT; + + kbase_gpu_mmu_handle_write_faulting_as(kbdev, faulting_as, + fault_pfn, 1, op); + + kbase_gpu_vm_unlock(kctx); +} + +static void kbase_gpu_mmu_handle_permission_fault(struct kbase_context *kctx, + struct kbase_as *faulting_as) +{ + struct kbase_fault *fault = &faulting_as->pf_data; + + switch (AS_FAULTSTATUS_ACCESS_TYPE_GET(fault->status)) { + case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: + case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: + kbase_gpu_mmu_handle_write_fault(kctx, faulting_as); + break; + case AS_FAULTSTATUS_ACCESS_TYPE_EX: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Execute Permission fault", fault); + break; + case AS_FAULTSTATUS_ACCESS_TYPE_READ: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Read Permission fault", fault); + break; + default: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown Permission fault", fault); + break; + } +} +#endif + +#define MAX_POOL_LEVEL 2 + +/** + * page_fault_try_alloc - Try to allocate memory from a context pool + * @kctx: Context pointer + * @region: Region to grow + * @new_pages: Number of 4 kB pages to allocate + * @pages_to_grow: Pointer to variable to store number of outstanding pages on + * failure. This can be either 4 kB or 2 MB pages, depending on + * the number of pages requested. + * @grow_2mb_pool: Pointer to variable to store which pool needs to grow - true + * for 2 MB, false for 4 kB. + * @prealloc_sas: Pointer to kbase_sub_alloc structures + * + * This function will try to allocate as many pages as possible from the context + * pool, then if required will try to allocate the remaining pages from the + * device pool. + * + * This function will not allocate any new memory beyond that that is already + * present in the context or device pools. This is because it is intended to be + * called with the vm_lock held, which could cause recursive locking if the + * allocation caused the out-of-memory killer to run. + * + * If 2 MB pages are enabled and new_pages is >= 2 MB then pages_to_grow will be + * a count of 2 MB pages, otherwise it will be a count of 4 kB pages. + * + * Return: true if successful, false on failure + */ +static bool page_fault_try_alloc(struct kbase_context *kctx, + struct kbase_va_region *region, size_t new_pages, + int *pages_to_grow, bool *grow_2mb_pool, + struct kbase_sub_alloc **prealloc_sas) +{ + struct tagged_addr *gpu_pages[MAX_POOL_LEVEL] = {NULL}; + struct tagged_addr *cpu_pages[MAX_POOL_LEVEL] = {NULL}; + size_t pages_alloced[MAX_POOL_LEVEL] = {0}; + struct kbase_mem_pool *pool, *root_pool; + int pool_level = 0; + bool alloc_failed = false; + size_t pages_still_required; + + if (WARN_ON(region->gpu_alloc->group_id >= + MEMORY_GROUP_MANAGER_NR_GROUPS)) { + /* Do not try to grow the memory pool */ + *pages_to_grow = 0; + return false; + } + +#ifdef CONFIG_MALI_2MB_ALLOC + if (new_pages >= (SZ_2M / SZ_4K)) { + root_pool = &kctx->mem_pools.large[region->gpu_alloc->group_id]; + *grow_2mb_pool = true; + } else { +#endif + root_pool = &kctx->mem_pools.small[region->gpu_alloc->group_id]; + *grow_2mb_pool = false; +#ifdef CONFIG_MALI_2MB_ALLOC + } +#endif + + if (region->gpu_alloc != region->cpu_alloc) + new_pages *= 2; + + pages_still_required = new_pages; + + /* Determine how many pages are in the pools before trying to allocate. + * Don't attempt to allocate & free if the allocation can't succeed. + */ + for (pool = root_pool; pool != NULL; pool = pool->next_pool) { + size_t pool_size_4k; + + kbase_mem_pool_lock(pool); + + pool_size_4k = kbase_mem_pool_size(pool) << pool->order; + if (pool_size_4k >= pages_still_required) + pages_still_required = 0; + else + pages_still_required -= pool_size_4k; + + kbase_mem_pool_unlock(pool); + + if (!pages_still_required) + break; + } + + if (pages_still_required) { + /* Insufficient pages in pools. Don't try to allocate - just + * request a grow. + */ + *pages_to_grow = pages_still_required; + + return false; + } + + /* Since we've dropped the pool locks, the amount of memory in the pools + * may change between the above check and the actual allocation. + */ + pool = root_pool; + for (pool_level = 0; pool_level < MAX_POOL_LEVEL; pool_level++) { + size_t pool_size_4k; + size_t pages_to_alloc_4k; + size_t pages_to_alloc_4k_per_alloc; + + kbase_mem_pool_lock(pool); + + /* Allocate as much as possible from this pool*/ + pool_size_4k = kbase_mem_pool_size(pool) << pool->order; + pages_to_alloc_4k = MIN(new_pages, pool_size_4k); + if (region->gpu_alloc == region->cpu_alloc) + pages_to_alloc_4k_per_alloc = pages_to_alloc_4k; + else + pages_to_alloc_4k_per_alloc = pages_to_alloc_4k >> 1; + + pages_alloced[pool_level] = pages_to_alloc_4k; + if (pages_to_alloc_4k) { + gpu_pages[pool_level] = + kbase_alloc_phy_pages_helper_locked( + region->gpu_alloc, pool, + pages_to_alloc_4k_per_alloc, + &prealloc_sas[0]); + + if (!gpu_pages[pool_level]) { + alloc_failed = true; + } else if (region->gpu_alloc != region->cpu_alloc) { + cpu_pages[pool_level] = + kbase_alloc_phy_pages_helper_locked( + region->cpu_alloc, pool, + pages_to_alloc_4k_per_alloc, + &prealloc_sas[1]); + + if (!cpu_pages[pool_level]) + alloc_failed = true; + } + } + + kbase_mem_pool_unlock(pool); + + if (alloc_failed) { + WARN_ON(!new_pages); + WARN_ON(pages_to_alloc_4k >= new_pages); + WARN_ON(pages_to_alloc_4k_per_alloc >= new_pages); + break; + } + + new_pages -= pages_to_alloc_4k; + + if (!new_pages) + break; + + pool = pool->next_pool; + if (!pool) + break; + } + + if (new_pages) { + /* Allocation was unsuccessful */ + int max_pool_level = pool_level; + + pool = root_pool; + + /* Free memory allocated so far */ + for (pool_level = 0; pool_level <= max_pool_level; + pool_level++) { + kbase_mem_pool_lock(pool); + + if (region->gpu_alloc != region->cpu_alloc) { + if (pages_alloced[pool_level] && + cpu_pages[pool_level]) + kbase_free_phy_pages_helper_locked( + region->cpu_alloc, + pool, cpu_pages[pool_level], + pages_alloced[pool_level]); + } + + if (pages_alloced[pool_level] && gpu_pages[pool_level]) + kbase_free_phy_pages_helper_locked( + region->gpu_alloc, + pool, gpu_pages[pool_level], + pages_alloced[pool_level]); + + kbase_mem_pool_unlock(pool); + + pool = pool->next_pool; + } + + /* + * If the allocation failed despite there being enough memory in + * the pool, then just fail. Otherwise, try to grow the memory + * pool. + */ + if (alloc_failed) + *pages_to_grow = 0; + else + *pages_to_grow = new_pages; + + return false; + } + + /* Allocation was successful. No pages to grow, return success. */ + *pages_to_grow = 0; + + return true; +} + +/* Small wrapper function to factor out GPU-dependent context releasing */ +static void release_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ +#if MALI_USE_CSF + CSTD_UNUSED(kbdev); + kbase_ctx_sched_release_ctx_lock(kctx); +#else /* MALI_USE_CSF */ + kbasep_js_runpool_release_ctx(kbdev, kctx); +#endif /* MALI_USE_CSF */ +} + +void kbase_mmu_page_fault_worker(struct work_struct *data) +{ + u64 fault_pfn; + u32 fault_status; + size_t new_pages; + size_t fault_rel_pfn; + struct kbase_as *faulting_as; + int as_no; + struct kbase_context *kctx; + struct kbase_device *kbdev; + struct kbase_va_region *region; + struct kbase_fault *fault; + int err; + bool grown = false; + int pages_to_grow; + bool grow_2mb_pool; + struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; + int i; + size_t current_backed_size; +#if MALI_JIT_PRESSURE_LIMIT_BASE + size_t pages_trimmed = 0; +#endif + + faulting_as = container_of(data, struct kbase_as, work_pagefault); + fault = &faulting_as->pf_data; + fault_pfn = fault->addr >> PAGE_SHIFT; + as_no = faulting_as->number; + + kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); + dev_dbg(kbdev->dev, + "Entering %s %p, fault_pfn %lld, as_no %d\n", + __func__, (void *)data, fault_pfn, as_no); + + /* Grab the context that was already refcounted in kbase_mmu_interrupt() + * Therefore, it cannot be scheduled out of this AS until we explicitly + * release it + */ + kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_no); + if (!kctx) { + atomic_dec(&kbdev->faults_pending); + return; + } + + KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev); + +#if MALI_JIT_PRESSURE_LIMIT_BASE +#if !MALI_USE_CSF + mutex_lock(&kctx->jctx.lock); +#endif +#endif + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* check if we still have GPU */ + if (unlikely(kbase_is_gpu_removed(kbdev))) { + dev_dbg(kbdev->dev, + "%s: GPU has been removed\n", __func__); + goto fault_done; + } +#endif + + if (unlikely(fault->protected_mode)) { + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Protected mode fault", fault); + kbase_mmu_hw_clear_fault(kbdev, faulting_as, + KBASE_MMU_FAULT_TYPE_PAGE); + + goto fault_done; + } + + fault_status = fault->status; + switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) { + + case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT: + /* need to check against the region to handle this one */ + break; + + case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT: +#ifdef CONFIG_MALI_CINSTR_GWT + /* If GWT was ever enabled then we need to handle + * write fault pages even if the feature was disabled later. + */ + if (kctx->gwt_was_enabled) { + kbase_gpu_mmu_handle_permission_fault(kctx, + faulting_as); + goto fault_done; + } +#endif + + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Permission failure", fault); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Translation table bus fault", fault); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG: + /* nothing to do, but we don't expect this fault currently */ + dev_warn(kbdev->dev, "Access flag unexpectedly set"); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Address size fault", fault); + else + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code", fault); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Memory attributes fault", fault); + else + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code", fault); + goto fault_done; + + default: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code", fault); + goto fault_done; + } + +#ifdef CONFIG_MALI_2MB_ALLOC + /* Preallocate memory for the sub-allocation structs if necessary */ + for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { + prealloc_sas[i] = kmalloc(sizeof(*prealloc_sas[i]), GFP_KERNEL); + if (!prealloc_sas[i]) { + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Failed pre-allocating memory for sub-allocations' metadata", + fault); + goto fault_done; + } + } +#endif /* CONFIG_MALI_2MB_ALLOC */ + +page_fault_retry: + /* so we have a translation fault, + * let's see if it is for growable memory + */ + kbase_gpu_vm_lock(kctx); + + region = kbase_region_tracker_find_region_enclosing_address(kctx, + fault->addr); + if (kbase_is_region_invalid_or_free(region)) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Memory is not mapped on the GPU", fault); + goto fault_done; + } + + if (region->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "DMA-BUF is not mapped on the GPU", fault); + goto fault_done; + } + + if (region->gpu_alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Bad physical memory group ID", fault); + goto fault_done; + } + + if ((region->flags & GROWABLE_FLAGS_REQUIRED) + != GROWABLE_FLAGS_REQUIRED) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Memory is not growable", fault); + goto fault_done; + } + + if ((region->flags & KBASE_REG_DONT_NEED)) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Don't need memory can't be grown", fault); + goto fault_done; + } + + /* find the size we need to grow it by + * we know the result fit in a size_t due to + * kbase_region_tracker_find_region_enclosing_address + * validating the fault_address to be within a size_t from the start_pfn + */ + fault_rel_pfn = fault_pfn - region->start_pfn; + + current_backed_size = kbase_reg_current_backed_size(region); + + if (fault_rel_pfn < current_backed_size) { + dev_dbg(kbdev->dev, + "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring", + fault->addr, region->start_pfn, + region->start_pfn + + current_backed_size); + + mutex_lock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_clear_fault(kbdev, faulting_as, + KBASE_MMU_FAULT_TYPE_PAGE); + /* [1] in case another page fault occurred while we were + * handling the (duplicate) page fault we need to ensure we + * don't loose the other page fault as result of us clearing + * the MMU IRQ. Therefore, after we clear the MMU IRQ we send + * an UNLOCK command that will retry any stalled memory + * transaction (which should cause the other page fault to be + * raised again). + */ + kbase_mmu_hw_do_operation(kbdev, faulting_as, 0, 0, + AS_COMMAND_UNLOCK, 1); + + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_enable_fault(kbdev, faulting_as, + KBASE_MMU_FAULT_TYPE_PAGE); + kbase_gpu_vm_unlock(kctx); + + goto fault_done; + } + + new_pages = reg_grow_calc_extra_pages(kbdev, region, fault_rel_pfn); + + /* cap to max vsize */ + new_pages = min(new_pages, region->nr_pages - current_backed_size); + dev_dbg(kctx->kbdev->dev, "Allocate %zu pages on page fault\n", + new_pages); + + if (new_pages == 0) { + mutex_lock(&kbdev->mmu_hw_mutex); + + /* Duplicate of a fault we've already handled, nothing to do */ + kbase_mmu_hw_clear_fault(kbdev, faulting_as, + KBASE_MMU_FAULT_TYPE_PAGE); + /* See comment [1] about UNLOCK usage */ + kbase_mmu_hw_do_operation(kbdev, faulting_as, 0, 0, + AS_COMMAND_UNLOCK, 1); + + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_enable_fault(kbdev, faulting_as, + KBASE_MMU_FAULT_TYPE_PAGE); + kbase_gpu_vm_unlock(kctx); + goto fault_done; + } + + pages_to_grow = 0; + +#if MALI_JIT_PRESSURE_LIMIT_BASE + if ((region->flags & KBASE_REG_ACTIVE_JIT_ALLOC) && !pages_trimmed) { + kbase_jit_request_phys_increase(kctx, new_pages); + pages_trimmed = new_pages; + } +#endif + + spin_lock(&kctx->mem_partials_lock); + grown = page_fault_try_alloc(kctx, region, new_pages, &pages_to_grow, + &grow_2mb_pool, prealloc_sas); + spin_unlock(&kctx->mem_partials_lock); + + if (grown) { + u64 pfn_offset; + u32 op; + + /* alloc success */ + WARN_ON(kbase_reg_current_backed_size(region) > + region->nr_pages); + + /* set up the new pages */ + pfn_offset = kbase_reg_current_backed_size(region) - new_pages; + /* + * Note: + * Issuing an MMU operation will unlock the MMU and cause the + * translation to be replayed. If the page insertion fails then + * rather then trying to continue the context should be killed + * so the no_flush version of insert_pages is used which allows + * us to unlock the MMU as we see fit. + */ + err = kbase_mmu_insert_pages_no_flush(kbdev, &kctx->mmu, + region->start_pfn + pfn_offset, + &kbase_get_gpu_phy_pages(region)[pfn_offset], + new_pages, region->flags, region->gpu_alloc->group_id); + if (err) { + kbase_free_phy_pages_helper(region->gpu_alloc, + new_pages); + if (region->gpu_alloc != region->cpu_alloc) + kbase_free_phy_pages_helper(region->cpu_alloc, + new_pages); + kbase_gpu_vm_unlock(kctx); + /* The locked VA region will be unlocked and the cache + * invalidated in here + */ + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Page table update failure", fault); + goto fault_done; + } + KBASE_TLSTREAM_AUX_PAGEFAULT(kbdev, kctx->id, as_no, + (u64)new_pages); + trace_mali_mmu_page_fault_grow(region, fault, new_pages); + +#if MALI_INCREMENTAL_RENDERING + /* Switch to incremental rendering if we have nearly run out of + * memory in a JIT memory allocation. + */ + if (region->threshold_pages && + kbase_reg_current_backed_size(region) > + region->threshold_pages) { + + dev_dbg(kctx->kbdev->dev, + "%zu pages exceeded IR threshold %zu\n", + new_pages + current_backed_size, + region->threshold_pages); + + if (kbase_mmu_switch_to_ir(kctx, region) >= 0) { + dev_dbg(kctx->kbdev->dev, + "Get region %p for IR\n", + (void *)region); + kbase_va_region_alloc_get(kctx, region); + } + } +#endif + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + + /* flush L2 and unlock the VA (resumes the MMU) */ + op = AS_COMMAND_FLUSH_PT; + + /* clear MMU interrupt - this needs to be done after updating + * the page tables but before issuing a FLUSH command. The + * FLUSH cmd has a side effect that it restarts stalled memory + * transactions in other address spaces which may cause + * another fault to occur. If we didn't clear the interrupt at + * this stage a new IRQ might not be raised when the GPU finds + * a MMU IRQ is already pending. + */ + kbase_mmu_hw_clear_fault(kbdev, faulting_as, + KBASE_MMU_FAULT_TYPE_PAGE); + + kbase_mmu_hw_do_operation(kbdev, faulting_as, + fault->addr >> PAGE_SHIFT, + new_pages, op, 1); + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + + /* reenable this in the mask */ + kbase_mmu_hw_enable_fault(kbdev, faulting_as, + KBASE_MMU_FAULT_TYPE_PAGE); + +#ifdef CONFIG_MALI_CINSTR_GWT + if (kctx->gwt_enabled) { + /* GWT also tracks growable regions. */ + struct kbasep_gwt_list_element *pos; + + pos = kmalloc(sizeof(*pos), GFP_KERNEL); + if (pos) { + pos->region = region; + pos->page_addr = (region->start_pfn + + pfn_offset) << + PAGE_SHIFT; + pos->num_pages = new_pages; + list_add(&pos->link, + &kctx->gwt_current_list); + } else { + dev_warn(kbdev->dev, "kmalloc failure"); + } + } +#endif + +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (pages_trimmed) { + kbase_jit_done_phys_increase(kctx, pages_trimmed); + pages_trimmed = 0; + } +#endif + kbase_gpu_vm_unlock(kctx); + } else { + int ret = -ENOMEM; + + kbase_gpu_vm_unlock(kctx); + + /* If the memory pool was insufficient then grow it and retry. + * Otherwise fail the allocation. + */ + if (pages_to_grow > 0) { +#ifdef CONFIG_MALI_2MB_ALLOC + if (grow_2mb_pool) { + /* Round page requirement up to nearest 2 MB */ + struct kbase_mem_pool *const lp_mem_pool = + &kctx->mem_pools.large[ + region->gpu_alloc->group_id]; + + pages_to_grow = (pages_to_grow + + ((1 << lp_mem_pool->order) - 1)) + >> lp_mem_pool->order; + + ret = kbase_mem_pool_grow(lp_mem_pool, + pages_to_grow); + } else { +#endif + struct kbase_mem_pool *const mem_pool = + &kctx->mem_pools.small[ + region->gpu_alloc->group_id]; + + ret = kbase_mem_pool_grow(mem_pool, + pages_to_grow); +#ifdef CONFIG_MALI_2MB_ALLOC + } +#endif + } + if (ret < 0) { + /* failed to extend, handle as a normal PF */ + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Page allocation failure", fault); + } else { + dev_dbg(kbdev->dev, "Try again after pool_grow\n"); + goto page_fault_retry; + } + } + +fault_done: +#if MALI_JIT_PRESSURE_LIMIT_BASE + if (pages_trimmed) { + kbase_gpu_vm_lock(kctx); + kbase_jit_done_phys_increase(kctx, pages_trimmed); + kbase_gpu_vm_unlock(kctx); + } +#if !MALI_USE_CSF + mutex_unlock(&kctx->jctx.lock); +#endif +#endif + + for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) + kfree(prealloc_sas[i]); + + /* + * By this point, the fault was handled in some way, + * so release the ctx refcount + */ + release_ctx(kbdev, kctx); + + atomic_dec(&kbdev->faults_pending); + dev_dbg(kbdev->dev, "Leaving page_fault_worker %p\n", (void *)data); +} + +static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut) +{ + u64 *page; + int i; + struct page *p; + + p = kbase_mem_pool_alloc(&kbdev->mem_pools.small[mmut->group_id]); + if (!p) + return 0; + + page = kmap(p); + if (page == NULL) + goto alloc_free; + + /* If the MMU tables belong to a context then account the memory usage + * to that context, otherwise the MMU tables are device wide and are + * only accounted to the device. + */ + if (mmut->kctx) { + int new_page_count; + + new_page_count = atomic_add_return(1, + &mmut->kctx->used_pages); + KBASE_TLSTREAM_AUX_PAGESALLOC( + kbdev, + mmut->kctx->id, + (u64)new_page_count); + kbase_process_page_usage_inc(mmut->kctx, 1); + } + + atomic_add(1, &kbdev->memdev.used_pages); + + kbase_trace_gpu_mem_usage_inc(kbdev, mmut->kctx, 1); + + for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) + kbdev->mmu_mode->entry_invalidate(&page[i]); + + kbase_mmu_sync_pgd(kbdev, kbase_dma_addr(p), PAGE_SIZE); + + kunmap(p); + return page_to_phys(p); + +alloc_free: + kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], p, + false); + + return 0; +} + +/* Given PGD PFN for level N, return PGD PFN for level N+1, allocating the + * new table from the pool if needed and possible + */ +static int mmu_get_next_pgd(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, + phys_addr_t *pgd, u64 vpfn, int level) +{ + u64 *page; + phys_addr_t target_pgd; + struct page *p; + + KBASE_DEBUG_ASSERT(*pgd); + + lockdep_assert_held(&mmut->mmu_lock); + + /* + * Architecture spec defines level-0 as being the top-most. + * This is a bit unfortunate here, but we keep the same convention. + */ + vpfn >>= (3 - level) * 9; + vpfn &= 0x1FF; + + p = pfn_to_page(PFN_DOWN(*pgd)); + page = kmap(p); + if (page == NULL) { + dev_warn(kbdev->dev, "%s: kmap failure\n", __func__); + return -EINVAL; + } + + target_pgd = kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]); + + if (!target_pgd) { + target_pgd = kbase_mmu_alloc_pgd(kbdev, mmut); + if (!target_pgd) { + dev_dbg(kbdev->dev, "%s: kbase_mmu_alloc_pgd failure\n", + __func__); + kunmap(p); + return -ENOMEM; + } + + kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd); + + kbase_mmu_sync_pgd(kbdev, kbase_dma_addr(p), PAGE_SIZE); + /* Rely on the caller to update the address space flags. */ + } + + kunmap(p); + *pgd = target_pgd; + + return 0; +} + +/* + * Returns the PGD for the specified level of translation + */ +static int mmu_get_pgd_at_level(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, + u64 vpfn, + int level, + phys_addr_t *out_pgd) +{ + phys_addr_t pgd; + int l; + + lockdep_assert_held(&mmut->mmu_lock); + pgd = mmut->pgd; + + for (l = MIDGARD_MMU_TOPLEVEL; l < level; l++) { + int err = mmu_get_next_pgd(kbdev, mmut, &pgd, vpfn, l); + /* Handle failure condition */ + if (err) { + dev_dbg(kbdev->dev, + "%s: mmu_get_next_pgd failure at level %d\n", + __func__, l); + return err; + } + } + + *out_pgd = pgd; + + return 0; +} + +static int mmu_get_bottom_pgd(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, + u64 vpfn, + phys_addr_t *out_pgd) +{ + return mmu_get_pgd_at_level(kbdev, mmut, vpfn, MIDGARD_MMU_BOTTOMLEVEL, + out_pgd); +} + +static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, + u64 from_vpfn, u64 to_vpfn) +{ + phys_addr_t pgd; + u64 vpfn = from_vpfn; + struct kbase_mmu_mode const *mmu_mode; + + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + KBASE_DEBUG_ASSERT(from_vpfn <= to_vpfn); + + lockdep_assert_held(&mmut->mmu_lock); + + mmu_mode = kbdev->mmu_mode; + + while (vpfn < to_vpfn) { + unsigned int i; + unsigned int idx = vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - idx; + unsigned int pcount = 0; + unsigned int left = to_vpfn - vpfn; + int level; + u64 *page; + + if (count > left) + count = left; + + /* need to check if this is a 2MB page or a 4kB */ + pgd = mmut->pgd; + + for (level = MIDGARD_MMU_TOPLEVEL; + level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { + idx = (vpfn >> ((3 - level) * 9)) & 0x1FF; + page = kmap(phys_to_page(pgd)); + if (mmu_mode->ate_is_valid(page[idx], level)) + break; /* keep the mapping */ + kunmap(phys_to_page(pgd)); + pgd = mmu_mode->pte_to_phy_addr(page[idx]); + } + + switch (level) { + case MIDGARD_MMU_LEVEL(2): + /* remap to single entry to update */ + pcount = 1; + break; + case MIDGARD_MMU_BOTTOMLEVEL: + /* page count is the same as the logical count */ + pcount = count; + break; + default: + dev_warn(kbdev->dev, "%sNo support for ATEs at level %d\n", + __func__, level); + goto next; + } + + /* Invalidate the entries we added */ + for (i = 0; i < pcount; i++) + mmu_mode->entry_invalidate(&page[idx + i]); + + kbase_mmu_sync_pgd(kbdev, + kbase_dma_addr(phys_to_page(pgd)) + 8 * idx, + 8 * pcount); + kunmap(phys_to_page(pgd)); + +next: + vpfn += count; + } +} + +/* + * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn' + */ +int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr phys, size_t nr, + unsigned long flags, int const group_id) +{ + phys_addr_t pgd; + u64 *pgd_page; + /* In case the insert_single_page only partially completes + * we need to be able to recover + */ + bool recover_required = false; + u64 start_vpfn = vpfn; + size_t recover_count = 0; + size_t remain = nr; + int err; + struct kbase_device *kbdev; + + if (WARN_ON(kctx == NULL)) + return -EINVAL; + + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + + kbdev = kctx->kbdev; + + /* Early out if there is nothing to do */ + if (nr == 0) + return 0; + + mutex_lock(&kctx->mmu.mmu_lock); + + while (remain) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; + struct page *p; + + if (count > remain) + count = remain; + + /* + * Repeatedly calling mmu_get_bottom_pte() is clearly + * suboptimal. We don't have to re-parse the whole tree + * each time (just cache the l0-l2 sequence). + * On the other hand, it's only a gain when we map more than + * 256 pages at once (on average). Do we really care? + */ + do { + err = mmu_get_bottom_pgd(kbdev, &kctx->mmu, + vpfn, &pgd); + if (err != -ENOMEM) + break; + /* Fill the memory pool with enough pages for + * the page walk to succeed + */ + mutex_unlock(&kctx->mmu.mmu_lock); + err = kbase_mem_pool_grow( + &kbdev->mem_pools.small[ + kctx->mmu.group_id], + MIDGARD_MMU_BOTTOMLEVEL); + mutex_lock(&kctx->mmu.mmu_lock); + } while (!err); + if (err) { + dev_warn(kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n"); + if (recover_required) { + /* Invalidate the pages we have partially + * completed + */ + mmu_insert_pages_failure_recovery(kbdev, + &kctx->mmu, + start_vpfn, + start_vpfn + recover_count); + } + goto fail_unlock; + } + + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap(p); + if (!pgd_page) { + dev_warn(kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n"); + if (recover_required) { + /* Invalidate the pages we have partially + * completed + */ + mmu_insert_pages_failure_recovery(kbdev, + &kctx->mmu, + start_vpfn, + start_vpfn + recover_count); + } + err = -ENOMEM; + goto fail_unlock; + } + + for (i = 0; i < count; i++) { + unsigned int ofs = index + i; + + /* Fail if the current page is a valid ATE entry */ + KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); + + pgd_page[ofs] = kbase_mmu_create_ate(kbdev, + phys, flags, MIDGARD_MMU_BOTTOMLEVEL, group_id); + } + + vpfn += count; + remain -= count; + + kbase_mmu_sync_pgd(kbdev, + kbase_dma_addr(p) + (index * sizeof(u64)), + count * sizeof(u64)); + + kunmap(p); + /* We have started modifying the page table. + * If further pages need inserting and fail we need to undo what + * has already taken place + */ + recover_required = true; + recover_count += count; + } + mutex_unlock(&kctx->mmu.mmu_lock); + kbase_mmu_flush_invalidate(kctx, start_vpfn, nr, false); + return 0; + +fail_unlock: + mutex_unlock(&kctx->mmu.mmu_lock); + kbase_mmu_flush_invalidate(kctx, start_vpfn, nr, false); + return err; +} + +static inline void cleanup_empty_pte(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, u64 *pte) +{ + phys_addr_t tmp_pgd; + struct page *tmp_p; + + tmp_pgd = kbdev->mmu_mode->pte_to_phy_addr(*pte); + tmp_p = phys_to_page(tmp_pgd); + kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], + tmp_p, false); + + /* If the MMU tables belong to a context then we accounted the memory + * usage to that context, so decrement here. + */ + if (mmut->kctx) { + kbase_process_page_usage_dec(mmut->kctx, 1); + atomic_sub(1, &mmut->kctx->used_pages); + } + atomic_sub(1, &kbdev->memdev.used_pages); + + kbase_trace_gpu_mem_usage_dec(kbdev, mmut->kctx, 1); +} + +u64 kbase_mmu_create_ate(struct kbase_device *const kbdev, + struct tagged_addr const phy, unsigned long const flags, + int const level, int const group_id) +{ + u64 entry; + + kbdev->mmu_mode->entry_set_ate(&entry, phy, flags, level); + return kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, + group_id, level, entry); +} + +int kbase_mmu_insert_pages_no_flush(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, + const u64 start_vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags, + int const group_id) +{ + phys_addr_t pgd; + u64 *pgd_page; + u64 insert_vpfn = start_vpfn; + size_t remain = nr; + int err; + struct kbase_mmu_mode const *mmu_mode; + + /* Note that 0 is a valid start_vpfn */ + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(start_vpfn <= (U64_MAX / PAGE_SIZE)); + + mmu_mode = kbdev->mmu_mode; + + /* Early out if there is nothing to do */ + if (nr == 0) + return 0; + + mutex_lock(&mmut->mmu_lock); + + while (remain) { + unsigned int i; + unsigned int vindex = insert_vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - vindex; + struct page *p; + int cur_level; + + if (count > remain) + count = remain; + + if (!vindex && is_huge_head(*phys)) + cur_level = MIDGARD_MMU_LEVEL(2); + else + cur_level = MIDGARD_MMU_BOTTOMLEVEL; + + /* + * Repeatedly calling mmu_get_pgd_at_level() is clearly + * suboptimal. We don't have to re-parse the whole tree + * each time (just cache the l0-l2 sequence). + * On the other hand, it's only a gain when we map more than + * 256 pages at once (on average). Do we really care? + */ + do { + err = mmu_get_pgd_at_level(kbdev, mmut, insert_vpfn, + cur_level, &pgd); + if (err != -ENOMEM) + break; + /* Fill the memory pool with enough pages for + * the page walk to succeed + */ + mutex_unlock(&mmut->mmu_lock); + err = kbase_mem_pool_grow( + &kbdev->mem_pools.small[mmut->group_id], + cur_level); + mutex_lock(&mmut->mmu_lock); + } while (!err); + + if (err) { + dev_warn(kbdev->dev, + "%s: mmu_get_bottom_pgd failure\n", __func__); + if (insert_vpfn != start_vpfn) { + /* Invalidate the pages we have partially + * completed + */ + mmu_insert_pages_failure_recovery(kbdev, + mmut, start_vpfn, insert_vpfn); + } + goto fail_unlock; + } + + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap(p); + if (!pgd_page) { + dev_warn(kbdev->dev, "%s: kmap failure\n", + __func__); + if (insert_vpfn != start_vpfn) { + /* Invalidate the pages we have partially + * completed + */ + mmu_insert_pages_failure_recovery(kbdev, + mmut, start_vpfn, insert_vpfn); + } + err = -ENOMEM; + goto fail_unlock; + } + + if (cur_level == MIDGARD_MMU_LEVEL(2)) { + int level_index = (insert_vpfn >> 9) & 0x1FF; + u64 *target = &pgd_page[level_index]; + + if (mmu_mode->pte_is_valid(*target, cur_level)) + cleanup_empty_pte(kbdev, mmut, target); + *target = kbase_mmu_create_ate(kbdev, *phys, flags, + cur_level, group_id); + } else { + for (i = 0; i < count; i++) { + unsigned int ofs = vindex + i; + u64 *target = &pgd_page[ofs]; + + /* Warn if the current page is a valid ATE + * entry. The page table shouldn't have anything + * in the place where we are trying to put a + * new entry. Modification to page table entries + * should be performed with + * kbase_mmu_update_pages() + */ + WARN_ON((*target & 1UL) != 0); + + *target = kbase_mmu_create_ate(kbdev, + phys[i], flags, cur_level, group_id); + } + } + + phys += count; + insert_vpfn += count; + remain -= count; + + kbase_mmu_sync_pgd(kbdev, + kbase_dma_addr(p) + (vindex * sizeof(u64)), + count * sizeof(u64)); + + kunmap(p); + } + + err = 0; + +fail_unlock: + mutex_unlock(&mmut->mmu_lock); + return err; +} + +/* + * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn' for GPU address space + * number 'as_nr'. + */ +int kbase_mmu_insert_pages(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags, int as_nr, int const group_id) +{ + int err; + + err = kbase_mmu_insert_pages_no_flush(kbdev, mmut, vpfn, + phys, nr, flags, group_id); + + if (mmut->kctx) + kbase_mmu_flush_invalidate(mmut->kctx, vpfn, nr, false); + else + kbase_mmu_flush_invalidate_no_ctx(kbdev, vpfn, nr, false, + as_nr); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages); + +/** + * kbase_mmu_flush_invalidate_noretain() - Flush and invalidate the GPU caches + * without retaining the kbase context. + * @kctx: The KBase context. + * @vpfn: The virtual page frame number to start the flush on. + * @nr: The number of pages to flush. + * @sync: Set if the operation should be synchronous or not. + * + * As per kbase_mmu_flush_invalidate but doesn't retain the kctx or do any + * other locking. + */ +static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx, + u64 vpfn, size_t nr, bool sync) +{ + struct kbase_device *kbdev = kctx->kbdev; + int err; + u32 op; + + /* Early out if there is nothing to do */ + if (nr == 0) + return; + + if (sync) + op = AS_COMMAND_FLUSH_MEM; + else + op = AS_COMMAND_FLUSH_PT; + + err = kbase_mmu_hw_do_operation(kbdev, + &kbdev->as[kctx->as_nr], + vpfn, nr, op, 0); + if (err) { + /* Flush failed to complete, assume the + * GPU has hung and perform a reset to recover + */ + dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n"); + + if (kbase_prepare_to_reset_gpu_locked(kbdev)) + kbase_reset_gpu_locked(kbdev); + } +} + +/* Perform a flush/invalidate on a particular address space + */ +static void kbase_mmu_flush_invalidate_as(struct kbase_device *kbdev, + struct kbase_as *as, + u64 vpfn, size_t nr, bool sync) +{ + int err; + u32 op; + + if (kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + /* GPU is off so there's no need to perform flush/invalidate */ + return; + } + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + + if (sync) + op = AS_COMMAND_FLUSH_MEM; + else + op = AS_COMMAND_FLUSH_PT; + + err = kbase_mmu_hw_do_operation(kbdev, + as, vpfn, nr, op, 0); + + if (err) { + /* Flush failed to complete, assume the GPU has hung and + * perform a reset to recover + */ + dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + } + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + + kbase_pm_context_idle(kbdev); +} + +static void kbase_mmu_flush_invalidate_no_ctx(struct kbase_device *kbdev, + u64 vpfn, size_t nr, bool sync, int as_nr) +{ + /* Skip if there is nothing to do */ + if (nr) { + kbase_mmu_flush_invalidate_as(kbdev, &kbdev->as[as_nr], vpfn, + nr, sync); + } +} + +static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, + u64 vpfn, size_t nr, bool sync) +{ + struct kbase_device *kbdev; + bool ctx_is_in_runpool; + + /* Early out if there is nothing to do */ + if (nr == 0) + return; + + kbdev = kctx->kbdev; +#if !MALI_USE_CSF + mutex_lock(&kbdev->js_data.queue_mutex); +#endif /* !MALI_USE_CSF */ + ctx_is_in_runpool = kbase_ctx_sched_inc_refcount(kctx); +#if !MALI_USE_CSF + mutex_unlock(&kbdev->js_data.queue_mutex); +#endif /* !MALI_USE_CSF */ + + if (ctx_is_in_runpool) { + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + + kbase_mmu_flush_invalidate_as(kbdev, &kbdev->as[kctx->as_nr], + vpfn, nr, sync); + + release_ctx(kbdev, kctx); + } +} + +void kbase_mmu_update(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, + int as_nr) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&kbdev->mmu_hw_mutex); + KBASE_DEBUG_ASSERT(as_nr != KBASEP_AS_NR_INVALID); + + kbdev->mmu_mode->update(kbdev, mmut, as_nr); +} +KBASE_EXPORT_TEST_API(kbase_mmu_update); + +void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&kbdev->mmu_hw_mutex); + + kbdev->mmu_mode->disable_as(kbdev, as_nr); +} + +void kbase_mmu_disable(struct kbase_context *kctx) +{ + /* ASSERT that the context has a valid as_nr, which is only the case + * when it's scheduled in. + * + * as_nr won't change because the caller has the hwaccess_lock + */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + /* + * The address space is being disabled, drain all knowledge of it out + * from the caches as pages and page tables might be freed after this. + * + * The job scheduler code will already be holding the locks and context + * so just do the flush. + */ + kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true); + + kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr); +} +KBASE_EXPORT_TEST_API(kbase_mmu_disable); + +/* + * We actually only discard the ATE, and not the page table + * pages. There is a potential DoS here, as we'll leak memory by + * having PTEs that are potentially unused. Will require physical + * page accounting, so MMU pages are part of the process allocation. + * + * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is + * currently scheduled into the runpool, and so potentially uses a lot of locks. + * These locks must be taken in the correct order with respect to others + * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more + * information. + */ +int kbase_mmu_teardown_pages(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, u64 vpfn, size_t nr, int as_nr) +{ + phys_addr_t pgd; + u64 start_vpfn = vpfn; + size_t requested_nr = nr; + struct kbase_mmu_mode const *mmu_mode; + int err = -EFAULT; + + if (nr == 0) { + /* early out if nothing to do */ + return 0; + } + + mutex_lock(&mmut->mmu_lock); + + mmu_mode = kbdev->mmu_mode; + + while (nr) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; + unsigned int pcount; + int level; + u64 *page; + + if (count > nr) + count = nr; + + /* need to check if this is a 2MB or a 4kB page */ + pgd = mmut->pgd; + + for (level = MIDGARD_MMU_TOPLEVEL; + level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { + phys_addr_t next_pgd; + + index = (vpfn >> ((3 - level) * 9)) & 0x1FF; + page = kmap(phys_to_page(pgd)); + if (mmu_mode->ate_is_valid(page[index], level)) + break; /* keep the mapping */ + else if (!mmu_mode->pte_is_valid(page[index], level)) { + /* nothing here, advance */ + switch (level) { + case MIDGARD_MMU_LEVEL(0): + count = 134217728; + break; + case MIDGARD_MMU_LEVEL(1): + count = 262144; + break; + case MIDGARD_MMU_LEVEL(2): + count = 512; + break; + case MIDGARD_MMU_LEVEL(3): + count = 1; + break; + } + if (count > nr) + count = nr; + goto next; + } + next_pgd = mmu_mode->pte_to_phy_addr(page[index]); + kunmap(phys_to_page(pgd)); + pgd = next_pgd; + } + + switch (level) { + case MIDGARD_MMU_LEVEL(0): + case MIDGARD_MMU_LEVEL(1): + dev_warn(kbdev->dev, + "%s: No support for ATEs at level %d\n", + __func__, level); + kunmap(phys_to_page(pgd)); + goto out; + case MIDGARD_MMU_LEVEL(2): + /* can only teardown if count >= 512 */ + if (count >= 512) { + pcount = 1; + } else { + dev_warn(kbdev->dev, + "%s: limiting teardown as it tries to do a partial 2MB teardown, need 512, but have %d to tear down\n", + __func__, count); + pcount = 0; + } + break; + case MIDGARD_MMU_BOTTOMLEVEL: + /* page count is the same as the logical count */ + pcount = count; + break; + default: + dev_err(kbdev->dev, + "%s: found non-mapped memory, early out\n", + __func__); + vpfn += count; + nr -= count; + continue; + } + + /* Invalidate the entries we added */ + for (i = 0; i < pcount; i++) + mmu_mode->entry_invalidate(&page[index + i]); + + kbase_mmu_sync_pgd(kbdev, + kbase_dma_addr(phys_to_page(pgd)) + + 8 * index, 8*pcount); + +next: + kunmap(phys_to_page(pgd)); + vpfn += count; + nr -= count; + } + err = 0; +out: + mutex_unlock(&mmut->mmu_lock); + + if (mmut->kctx) + kbase_mmu_flush_invalidate(mmut->kctx, start_vpfn, requested_nr, + true); + else + kbase_mmu_flush_invalidate_no_ctx(kbdev, start_vpfn, requested_nr, + true, as_nr); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages); + +/** + * kbase_mmu_update_pages_no_flush() - Update page table entries on the GPU + * + * This will update page table entries that already exist on the GPU based on + * the new flags that are passed. It is used as a response to the changes of + * the memory attributes + * + * The caller is responsible for validating the memory attributes + * + * @kctx: Kbase context + * @vpfn: Virtual PFN (Page Frame Number) of the first page to update + * @phys: Tagged physical addresses of the physical pages to replace the + * current mappings + * @nr: Number of pages to update + * @flags: Flags + * @group_id: The physical memory group in which the page was allocated. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + */ +static int kbase_mmu_update_pages_no_flush(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags, int const group_id) +{ + phys_addr_t pgd; + u64 *pgd_page; + int err; + struct kbase_device *kbdev; + + if (WARN_ON(kctx == NULL)) + return -EINVAL; + + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + + /* Early out if there is nothing to do */ + if (nr == 0) + return 0; + + mutex_lock(&kctx->mmu.mmu_lock); + + kbdev = kctx->kbdev; + + while (nr) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + size_t count = KBASE_MMU_PAGE_ENTRIES - index; + struct page *p; + + if (count > nr) + count = nr; + + do { + err = mmu_get_bottom_pgd(kbdev, &kctx->mmu, + vpfn, &pgd); + if (err != -ENOMEM) + break; + /* Fill the memory pool with enough pages for + * the page walk to succeed + */ + mutex_unlock(&kctx->mmu.mmu_lock); + err = kbase_mem_pool_grow( + &kbdev->mem_pools.small[ + kctx->mmu.group_id], + MIDGARD_MMU_BOTTOMLEVEL); + mutex_lock(&kctx->mmu.mmu_lock); + } while (!err); + if (err) { + dev_warn(kbdev->dev, + "mmu_get_bottom_pgd failure\n"); + goto fail_unlock; + } + + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap(p); + if (!pgd_page) { + dev_warn(kbdev->dev, "kmap failure\n"); + err = -ENOMEM; + goto fail_unlock; + } + + for (i = 0; i < count; i++) + pgd_page[index + i] = kbase_mmu_create_ate(kbdev, + phys[i], flags, MIDGARD_MMU_BOTTOMLEVEL, + group_id); + + phys += count; + vpfn += count; + nr -= count; + + kbase_mmu_sync_pgd(kbdev, + kbase_dma_addr(p) + (index * sizeof(u64)), + count * sizeof(u64)); + + kunmap(pfn_to_page(PFN_DOWN(pgd))); + } + + mutex_unlock(&kctx->mmu.mmu_lock); + return 0; + +fail_unlock: + mutex_unlock(&kctx->mmu.mmu_lock); + return err; +} + +int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags, int const group_id) +{ + int err; + + err = kbase_mmu_update_pages_no_flush(kctx, vpfn, phys, nr, flags, + group_id); + kbase_mmu_flush_invalidate(kctx, vpfn, nr, true); + return err; +} + +static void mmu_teardown_level(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, phys_addr_t pgd, + int level, u64 *pgd_page_buffer) +{ + phys_addr_t target_pgd; + struct page *p; + u64 *pgd_page; + int i; + struct kbase_mmu_mode const *mmu_mode; + + lockdep_assert_held(&mmut->mmu_lock); + + pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); + /* kmap_atomic should NEVER fail. */ + if (WARN_ON(pgd_page == NULL)) + return; + /* Copy the page to our preallocated buffer so that we can minimize + * kmap_atomic usage + */ + memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE); + kunmap_atomic(pgd_page); + pgd_page = pgd_page_buffer; + + mmu_mode = kbdev->mmu_mode; + + for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { + target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]); + + if (target_pgd) { + if (mmu_mode->pte_is_valid(pgd_page[i], level)) { + mmu_teardown_level(kbdev, mmut, + target_pgd, + level + 1, + pgd_page_buffer + + (PAGE_SIZE / sizeof(u64))); + } + } + } + + p = pfn_to_page(PFN_DOWN(pgd)); + + kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], + p, true); + + atomic_sub(1, &kbdev->memdev.used_pages); + + /* If MMU tables belong to a context then pages will have been accounted + * against it, so we must decrement the usage counts here. + */ + if (mmut->kctx) { + kbase_process_page_usage_dec(mmut->kctx, 1); + atomic_sub(1, &mmut->kctx->used_pages); + } + + kbase_trace_gpu_mem_usage_dec(kbdev, mmut->kctx, 1); +} + +int kbase_mmu_init(struct kbase_device *const kbdev, + struct kbase_mmu_table *const mmut, struct kbase_context *const kctx, + int const group_id) +{ + if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS) || + WARN_ON(group_id < 0)) + return -EINVAL; + + mmut->group_id = group_id; + mutex_init(&mmut->mmu_lock); + mmut->kctx = kctx; + + /* Preallocate MMU depth of four pages for mmu_teardown_level to use */ + mmut->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); + + if (mmut->mmu_teardown_pages == NULL) + return -ENOMEM; + + mmut->pgd = 0; + /* We allocate pages into the kbdev memory pool, then + * kbase_mmu_alloc_pgd will allocate out of that pool. This is done to + * avoid allocations from the kernel happening with the lock held. + */ + while (!mmut->pgd) { + int err; + + err = kbase_mem_pool_grow( + &kbdev->mem_pools.small[mmut->group_id], + MIDGARD_MMU_BOTTOMLEVEL); + if (err) { + kbase_mmu_term(kbdev, mmut); + return -ENOMEM; + } + + mutex_lock(&mmut->mmu_lock); + mmut->pgd = kbase_mmu_alloc_pgd(kbdev, mmut); + mutex_unlock(&mmut->mmu_lock); + } + + return 0; +} + +void kbase_mmu_term(struct kbase_device *kbdev, struct kbase_mmu_table *mmut) +{ + if (mmut->pgd) { + mutex_lock(&mmut->mmu_lock); + mmu_teardown_level(kbdev, mmut, mmut->pgd, MIDGARD_MMU_TOPLEVEL, + mmut->mmu_teardown_pages); + mutex_unlock(&mmut->mmu_lock); + + if (mmut->kctx) + KBASE_TLSTREAM_AUX_PAGESALLOC(kbdev, mmut->kctx->id, 0); + } + + kfree(mmut->mmu_teardown_pages); + mutex_destroy(&mmut->mmu_lock); +} + +void kbase_mmu_as_term(struct kbase_device *kbdev, int i) +{ + destroy_workqueue(kbdev->as[i].pf_wq); +} + +static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, + int level, char ** const buffer, size_t *size_left) +{ + phys_addr_t target_pgd; + u64 *pgd_page; + int i; + size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64); + size_t dump_size; + struct kbase_device *kbdev; + struct kbase_mmu_mode const *mmu_mode; + + if (WARN_ON(kctx == NULL)) + return 0; + lockdep_assert_held(&kctx->mmu.mmu_lock); + + kbdev = kctx->kbdev; + mmu_mode = kbdev->mmu_mode; + + pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd))); + if (!pgd_page) { + dev_warn(kbdev->dev, "%s: kmap failure\n", __func__); + return 0; + } + + if (*size_left >= size) { + /* A modified physical address that contains + * the page table level + */ + u64 m_pgd = pgd | level; + + /* Put the modified physical address in the output buffer */ + memcpy(*buffer, &m_pgd, sizeof(m_pgd)); + *buffer += sizeof(m_pgd); + + /* Followed by the page table itself */ + memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES); + *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES; + + *size_left -= size; + } + + if (level < MIDGARD_MMU_BOTTOMLEVEL) { + for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { + if (mmu_mode->pte_is_valid(pgd_page[i], level)) { + target_pgd = mmu_mode->pte_to_phy_addr( + pgd_page[i]); + + dump_size = kbasep_mmu_dump_level(kctx, + target_pgd, level + 1, + buffer, size_left); + if (!dump_size) { + kunmap(pfn_to_page(PFN_DOWN(pgd))); + return 0; + } + size += dump_size; + } + } + } + + kunmap(pfn_to_page(PFN_DOWN(pgd))); + + return size; +} + +void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) +{ + void *kaddr; + size_t size_left; + + KBASE_DEBUG_ASSERT(kctx); + + if (nr_pages == 0) { + /* can't dump in a 0 sized buffer, early out */ + return NULL; + } + + size_left = nr_pages * PAGE_SIZE; + + if (WARN_ON(size_left == 0)) + return NULL; + kaddr = vmalloc_user(size_left); + + mutex_lock(&kctx->mmu.mmu_lock); + + if (kaddr) { + u64 end_marker = 0xFFULL; + char *buffer; + char *mmu_dump_buffer; + u64 config[3]; + size_t dump_size, size = 0; + struct kbase_mmu_setup as_setup; + + buffer = (char *)kaddr; + mmu_dump_buffer = buffer; + + kctx->kbdev->mmu_mode->get_as_setup(&kctx->mmu, + &as_setup); + config[0] = as_setup.transtab; + config[1] = as_setup.memattr; + config[2] = as_setup.transcfg; + memcpy(buffer, &config, sizeof(config)); + mmu_dump_buffer += sizeof(config); + size_left -= sizeof(config); + size += sizeof(config); + + dump_size = kbasep_mmu_dump_level(kctx, + kctx->mmu.pgd, + MIDGARD_MMU_TOPLEVEL, + &mmu_dump_buffer, + &size_left); + + if (!dump_size) + goto fail_free; + + size += dump_size; + + /* Add on the size for the end marker */ + size += sizeof(u64); + + if (size > (nr_pages * PAGE_SIZE)) { + /* The buffer isn't big enough - free the memory and + * return failure + */ + goto fail_free; + } + + /* Add the end marker */ + memcpy(mmu_dump_buffer, &end_marker, sizeof(u64)); + } + + mutex_unlock(&kctx->mmu.mmu_lock); + return kaddr; + +fail_free: + vfree(kaddr); + mutex_unlock(&kctx->mmu.mmu_lock); + return NULL; +} +KBASE_EXPORT_TEST_API(kbase_mmu_dump); + +void kbase_mmu_bus_fault_worker(struct work_struct *data) +{ + struct kbase_as *faulting_as; + int as_no; + struct kbase_context *kctx; + struct kbase_device *kbdev; + struct kbase_fault *fault; + + faulting_as = container_of(data, struct kbase_as, work_busfault); + fault = &faulting_as->bf_data; + + /* Ensure that any pending page fault worker has completed */ + flush_work(&faulting_as->work_pagefault); + + as_no = faulting_as->number; + + kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); + + /* Grab the context, already refcounted in kbase_mmu_interrupt() on + * flagging of the bus-fault. Therefore, it cannot be scheduled out of + * this AS until we explicitly release it + */ + kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_no); + if (!kctx) { + atomic_dec(&kbdev->faults_pending); + return; + } + +#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* check if we still have GPU */ + if (unlikely(kbase_is_gpu_removed(kbdev))) { + dev_dbg(kbdev->dev, + "%s: GPU has been removed\n", __func__); + release_ctx(kbdev, kctx); + atomic_dec(&kbdev->faults_pending); + return; + } +#endif + + if (unlikely(fault->protected_mode)) { + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Permission failure", fault); + kbase_mmu_hw_clear_fault(kbdev, faulting_as, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + release_ctx(kbdev, kctx); + atomic_dec(&kbdev->faults_pending); + return; + + } + + /* NOTE: If GPU already powered off for suspend, + * we don't need to switch to unmapped + */ + if (!kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + kbase_gpu_report_bus_fault_and_kill(kctx, faulting_as, fault); + kbase_pm_context_idle(kbdev); + } + + release_ctx(kbdev, kctx); + + atomic_dec(&kbdev->faults_pending); +} + +void kbase_flush_mmu_wqs(struct kbase_device *kbdev) +{ + int i; + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + struct kbase_as *as = &kbdev->as[i]; + + flush_workqueue(as->pf_wq); + } +} diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h new file mode 100755 index 000000000000..f2613e881dac --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h @@ -0,0 +1,156 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_MMU_H_ +#define _KBASE_MMU_H_ + +/** + * kbase_mmu_as_init() - Initialising GPU address space object. + * + * This is called from device probe to initialise an address space object + * of the device. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer). + * @i: Array index of address space object. + * + * Return: 0 on success and non-zero value on failure. + */ +int kbase_mmu_as_init(struct kbase_device *kbdev, int i); + +/** + * kbase_mmu_as_term() - Terminate address space object. + * + * This is called upon device termination to destroy + * the address space object of the device. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer). + * @i: Array index of address space object. + */ +void kbase_mmu_as_term(struct kbase_device *kbdev, int i); + +/** + * kbase_mmu_init - Initialise an object representing GPU page tables + * + * The structure should be terminated using kbase_mmu_term() + * + * @kbdev: Instance of GPU platform device, allocated from the probe method. + * @mmut: GPU page tables to be initialized. + * @kctx: Optional kbase context, may be NULL if this set of MMU tables + * is not associated with a context. + * @group_id: The physical group ID from which to allocate GPU page tables. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * + * Return: 0 if successful, otherwise a negative error code. + */ +int kbase_mmu_init(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, + struct kbase_context *kctx, int group_id); + +/** + * kbase_mmu_interrupt - Process an MMU interrupt. + * + * Process the MMU interrupt that was reported by the &kbase_device. + * + * @kbdev: Pointer to the kbase device for which the interrupt happened. + * @irq_stat: Value of the MMU_IRQ_STATUS register. + */ +void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); + +/** + * kbase_mmu_term - Terminate an object representing GPU page tables + * + * This will free any page tables that have been allocated + * + * @kbdev: Instance of GPU platform device, allocated from the probe method. + * @mmut: GPU page tables to be destroyed. + */ +void kbase_mmu_term(struct kbase_device *kbdev, struct kbase_mmu_table *mmut); + +/** + * kbase_mmu_create_ate - Create an address translation entry + * + * @kbdev: Instance of GPU platform device, allocated from the probe method. + * @phy: Physical address of the page to be mapped for GPU access. + * @flags: Bitmask of attributes of the GPU memory region being mapped. + * @level: Page table level for which to build an address translation entry. + * @group_id: The physical memory group in which the page was allocated. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * + * This function creates an address translation entry to encode the physical + * address of a page to be mapped for access by the GPU, along with any extra + * attributes required for the GPU memory region. + * + * Return: An address translation entry, either in LPAE or AArch64 format + * (depending on the driver's configuration). + */ +u64 kbase_mmu_create_ate(struct kbase_device *kbdev, + struct tagged_addr phy, unsigned long flags, int level, int group_id); + +int kbase_mmu_insert_pages_no_flush(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, + const u64 start_vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags, int group_id); +int kbase_mmu_insert_pages(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags, int as_nr, int group_id); +int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr phys, size_t nr, + unsigned long flags, int group_id); + +int kbase_mmu_teardown_pages(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, u64 vpfn, + size_t nr, int as_nr); +int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags, int const group_id); + +/** + * kbase_mmu_bus_fault_interrupt - Process a bus fault interrupt. + * + * Process the bus fault interrupt that was reported for a particular GPU + * address space. + * + * @kbdev: Pointer to the kbase device for which bus fault was reported. + * @status: Value of the GPU_FAULTSTATUS register. + * @as_nr: GPU address space for which the bus fault occurred. + * + * Return: zero if the operation was successful, non-zero otherwise. + */ +int kbase_mmu_bus_fault_interrupt(struct kbase_device *kbdev, u32 status, + u32 as_nr); + +/** + * kbase_mmu_gpu_fault_interrupt() - Report a GPU fault. + * @kbdev: Kbase device pointer + * @status: GPU fault status + * @as_nr: Faulty address space + * @address: GPU fault address + * @as_valid: true if address space is valid + * + * This function builds GPU fault information to submit a work + * for reporting the details of the fault. + */ +void kbase_mmu_gpu_fault_interrupt(struct kbase_device *kbdev, u32 status, + u32 as_nr, u64 address, bool as_valid); + +#endif /* _KBASE_MMU_H_ */ diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h new file mode 100755 index 000000000000..e6eef86d7ac0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h @@ -0,0 +1,107 @@ +/* + * + * (C) COPYRIGHT 2014-2015, 2018-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * DOC: Interface file for accessing MMU hardware functionality + * + * This module provides an abstraction for accessing the functionality provided + * by the midgard MMU and thus allows all MMU HW access to be contained within + * one common place and allows for different backends (implementations) to + * be provided. + */ + +#ifndef _KBASE_MMU_HW_H_ +#define _KBASE_MMU_HW_H_ + +/* Forward declarations */ +struct kbase_device; +struct kbase_as; +struct kbase_context; + +/** + * enum kbase_mmu_fault_type - MMU fault type descriptor. + */ +enum kbase_mmu_fault_type { + KBASE_MMU_FAULT_TYPE_UNKNOWN = 0, + KBASE_MMU_FAULT_TYPE_PAGE, + KBASE_MMU_FAULT_TYPE_BUS, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED +}; + +/** + * kbase_mmu_hw_configure - Configure an address space for use. + * @kbdev: kbase device to configure. + * @as: address space to configure. + * + * Configure the MMU using the address space details setup in the + * kbase_context structure. + */ +void kbase_mmu_hw_configure(struct kbase_device *kbdev, + struct kbase_as *as); + +/** + * kbase_mmu_hw_do_operation - Issue an operation to the MMU. + * @kbdev: kbase device to issue the MMU operation on. + * @as: address space to issue the MMU operation on. + * @vpfn: MMU Virtual Page Frame Number to start the operation on. + * @nr: Number of pages to work on. + * @type: Operation type (written to ASn_COMMAND). + * @handling_irq: Is this operation being called during the handling + * of an interrupt? + * + * Issue an operation (MMU invalidate, MMU flush, etc) on the address space that + * is associated with the provided kbase_context over the specified range + * + * Return: Zero if the operation was successful, non-zero otherwise. + */ +int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, + u64 vpfn, u32 nr, u32 type, + unsigned int handling_irq); + +/** + * kbase_mmu_hw_clear_fault - Clear a fault that has been previously reported by + * the MMU. + * @kbdev: kbase device to clear the fault from. + * @as: address space to clear the fault from. + * @type: The type of fault that needs to be cleared. + * + * Clear a bus error or page fault that has been reported by the MMU. + */ +void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, + enum kbase_mmu_fault_type type); + +/** + * kbase_mmu_hw_enable_fault - Enable fault that has been previously reported by + * the MMU. + * @kbdev: kbase device to again enable the fault from. + * @as: address space to again enable the fault from. + * @type: The type of fault that needs to be enabled again. + * + * After a page fault or bus error has been reported by the MMU these + * will be disabled. After these are handled this function needs to be + * called to enable the page fault or bus error fault again. + */ +void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, + enum kbase_mmu_fault_type type); + +#endif /* _KBASE_MMU_HW_H_ */ diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c new file mode 100755 index 000000000000..a820ab24ac05 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c @@ -0,0 +1,274 @@ +/* + * + * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include +#include +#include + +/** + * lock_region() - Generate lockaddr to lock memory region in MMU + * @pfn: Starting page frame number of the region to lock + * @num_pages: Number of pages to lock. It must be greater than 0. + * @lockaddr: Address and size of memory region to lock + * + * The lockaddr value is a combination of the starting address and + * the size of the region that encompasses all the memory pages to lock. + * + * The size is expressed as a logarithm: it is represented in a way + * that is compatible with the HW specification and it also determines + * how many of the lowest bits of the address are cleared. + * + * Return: 0 if success, or an error code on failure. + */ +static int lock_region(u64 pfn, u32 num_pages, u64 *lockaddr) +{ + const u64 lockaddr_base = pfn << PAGE_SHIFT; + u64 lockaddr_size_log2, region_frame_number_start, + region_frame_number_end; + + if (num_pages == 0) + return -EINVAL; + + /* The size is expressed as a logarithm and should take into account + * the possibility that some pages might spill into the next region. + */ + lockaddr_size_log2 = fls(num_pages) + PAGE_SHIFT - 1; + + /* Round up if the number of pages is not a power of 2. */ + if (num_pages != ((u32)1 << (lockaddr_size_log2 - PAGE_SHIFT))) + lockaddr_size_log2 += 1; + + /* Round up if some memory pages spill into the next region. */ + region_frame_number_start = pfn >> (lockaddr_size_log2 - PAGE_SHIFT); + region_frame_number_end = + (pfn + num_pages - 1) >> (lockaddr_size_log2 - PAGE_SHIFT); + + if (region_frame_number_start < region_frame_number_end) + lockaddr_size_log2 += 1; + + /* Represent the size according to the HW specification. */ + lockaddr_size_log2 = MAX(lockaddr_size_log2, + KBASE_LOCK_REGION_MIN_SIZE_LOG2); + + if (lockaddr_size_log2 > KBASE_LOCK_REGION_MAX_SIZE_LOG2) + return -EINVAL; + + /* The lowest bits are cleared and then set to size - 1 to represent + * the size in a way that is compatible with the HW specification. + */ + *lockaddr = lockaddr_base & ~((1ull << lockaddr_size_log2) - 1); + *lockaddr |= lockaddr_size_log2 - 1; + + return 0; +} + +static int wait_ready(struct kbase_device *kbdev, + unsigned int as_nr) +{ + unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; + u32 val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS)); + + /* Wait for the MMU status to indicate there is no active command, in + * case one is pending. Do not log remaining register accesses. + */ + while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) + val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS)); + + if (max_loops == 0) { + dev_err(kbdev->dev, "AS_ACTIVE bit stuck, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n"); + return -1; + } + + /* If waiting in loop was performed, log last read value. */ + if (KBASE_AS_INACTIVE_MAX_LOOPS - 1 > max_loops) + kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS)); + + return 0; +} + +static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd) +{ + int status; + + /* write AS_COMMAND when MMU is ready to accept another command */ + status = wait_ready(kbdev, as_nr); + if (status == 0) + kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd); + + return status; +} + +void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) +{ + struct kbase_mmu_setup *current_setup = &as->current_setup; + u64 transcfg = 0; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { + transcfg = current_setup->transcfg; + + /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK + * Clear PTW_MEMATTR bits + */ + transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; + /* Enable correct PTW_MEMATTR bits */ + transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; + /* Ensure page-tables reads use read-allocate cache-policy in + * the L2 + */ + transcfg |= AS_TRANSCFG_R_ALLOCATE; + + if (kbdev->system_coherency == COHERENCY_ACE) { + /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) + * Clear PTW_SH bits + */ + transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); + /* Enable correct PTW_SH bits */ + transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); + } + + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), + transcfg); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), + (transcfg >> 32) & 0xFFFFFFFFUL); + } else { + if (kbdev->system_coherency == COHERENCY_ACE) + current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER; + } + + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO), + current_setup->transtab & 0xFFFFFFFFUL); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI), + (current_setup->transtab >> 32) & 0xFFFFFFFFUL); + + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO), + current_setup->memattr & 0xFFFFFFFFUL); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI), + (current_setup->memattr >> 32) & 0xFFFFFFFFUL); + + KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(kbdev, as, + current_setup->transtab, + current_setup->memattr, + transcfg); + + write_cmd(kbdev, as->number, AS_COMMAND_UPDATE); +} + +int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, + u64 vpfn, u32 nr, u32 op, + unsigned int handling_irq) +{ + int ret; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + + if (op == AS_COMMAND_UNLOCK) { + /* Unlock doesn't require a lock first */ + ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK); + } else { + u64 lock_addr; + + ret = lock_region(vpfn, nr, &lock_addr); + + if (!ret) { + /* Lock the region that needs to be updated */ + kbase_reg_write(kbdev, + MMU_AS_REG(as->number, AS_LOCKADDR_LO), + lock_addr & 0xFFFFFFFFUL); + kbase_reg_write(kbdev, + MMU_AS_REG(as->number, AS_LOCKADDR_HI), + (lock_addr >> 32) & 0xFFFFFFFFUL); + write_cmd(kbdev, as->number, AS_COMMAND_LOCK); + + /* Run the MMU operation */ + write_cmd(kbdev, as->number, op); + + /* Wait for the flush to complete */ + ret = wait_ready(kbdev, as->number); + } + } + + return ret; +} + +void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, + enum kbase_mmu_fault_type type) +{ + unsigned long flags; + u32 pf_bf_mask; + + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + + /* + * A reset is in-flight and we're flushing the IRQ + bottom half + * so don't update anything as it could race with the reset code. + */ + if (kbdev->irq_reset_flush) + goto unlock; + + /* Clear the page (and bus fault IRQ as well in case one occurred) */ + pf_bf_mask = MMU_PAGE_FAULT(as->number); +#if !MALI_USE_CSF + if (type == KBASE_MMU_FAULT_TYPE_BUS || + type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) + pf_bf_mask |= MMU_BUS_ERROR(as->number); +#endif + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask); + +unlock: + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); +} + +void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, + enum kbase_mmu_fault_type type) +{ + unsigned long flags; + u32 irq_mask; + + /* Enable the page fault IRQ + * (and bus fault IRQ as well in case one occurred) + */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + + /* + * A reset is in-flight and we're flushing the IRQ + bottom half + * so don't update anything as it could race with the reset code. + */ + if (kbdev->irq_reset_flush) + goto unlock; + + irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)) | + MMU_PAGE_FAULT(as->number); + +#if !MALI_USE_CSF + if (type == KBASE_MMU_FAULT_TYPE_BUS || + type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) + irq_mask |= MMU_BUS_ERROR(as->number); +#endif + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask); + +unlock: + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); +} diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h new file mode 100755 index 000000000000..8ecb14d72327 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h @@ -0,0 +1,73 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KBASE_MMU_INTERNAL_H_ +#define _KBASE_MMU_INTERNAL_H_ + +void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, + struct kbase_mmu_setup * const setup); + +/** + * kbase_mmu_report_mcu_as_fault_and_reset - Report page fault for all + * address spaces and reset the GPU. + * @kbdev: The kbase_device the fault happened on + * @fault: Data relating to the fault + */ +void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, + struct kbase_fault *fault); + +void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, + struct kbase_as *as, struct kbase_fault *fault); + +void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, + struct kbase_as *as, const char *reason_str, + struct kbase_fault *fault); + +/** + * kbase_mmu_switch_to_ir() - Switch to incremental rendering if possible + * @kctx: kbase_context for the faulting address space. + * @reg: of a growable GPU memory region in the same context. + * Takes ownership of the reference if successful. + * + * Used to switch to incremental rendering if we have nearly run out of + * virtual address space in a growable memory region. + * + * Return 0 if successful, otherwise a negative error code. + */ +int kbase_mmu_switch_to_ir(struct kbase_context *kctx, + struct kbase_va_region *reg); + +/** + * kbase_mmu_page_fault_worker() - Process a page fault. + * + * @data: work_struct passed by queue_work() + */ +void kbase_mmu_page_fault_worker(struct work_struct *data); + +/** + * kbase_mmu_bus_fault_worker() - Process a bus fault. + * + * @data: work_struct passed by queue_work() + */ +void kbase_mmu_bus_fault_worker(struct work_struct *data); + +#endif /* _KBASE_MMU_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c new file mode 100755 index 000000000000..02493e9b2621 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c @@ -0,0 +1,200 @@ +/* + * + * (C) COPYRIGHT 2010-2014, 2016-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase.h" +#include +#include "mali_kbase_defs.h" +#include +#include + +#define ENTRY_TYPE_MASK 3ULL +/* For valid ATEs bit 1 = ((level == 3) ? 1 : 0). + * Valid ATE entries at level 3 are flagged with the value 3. + * Valid ATE entries at level 0-2 are flagged with the value 1. + */ +#define ENTRY_IS_ATE_L3 3ULL +#define ENTRY_IS_ATE_L02 1ULL +#define ENTRY_IS_INVAL 2ULL +#define ENTRY_IS_PTE 3ULL + +#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ +#define ENTRY_ACCESS_RW (1ULL << 6) /* bits 6:7 */ +#define ENTRY_ACCESS_RO (3ULL << 6) +#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ +#define ENTRY_ACCESS_BIT (1ULL << 10) +#define ENTRY_NX_BIT (1ULL << 54) + +/* Helper Function to perform assignment of page table entries, to + * ensure the use of strd, which is required on LPAE systems. + */ +static inline void page_table_entry_set(u64 *pte, u64 phy) +{ +#if KERNEL_VERSION(3, 18, 13) <= LINUX_VERSION_CODE + WRITE_ONCE(*pte, phy); +#else +#ifdef CONFIG_64BIT + barrier(); + *pte = phy; + barrier(); +#elif defined(CONFIG_ARM) + barrier(); + asm volatile("ldrd r0, [%1]\n\t" + "strd r0, %0\n\t" + : "=m" (*pte) + : "r" (&phy) + : "r0", "r1"); + barrier(); +#else +#error "64-bit atomic write must be implemented for your architecture" +#endif +#endif +} + +static void mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, + int as_nr) +{ + struct kbase_as *as; + struct kbase_mmu_setup *current_setup; + + if (WARN_ON(as_nr == KBASEP_AS_NR_INVALID)) + return; + + as = &kbdev->as[as_nr]; + current_setup = &as->current_setup; + + kbase_mmu_get_as_setup(mmut, current_setup); + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as); +} + +static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) +{ + struct kbase_as * const as = &kbdev->as[as_nr]; + struct kbase_mmu_setup * const current_setup = &as->current_setup; + + current_setup->transtab = 0ULL; + current_setup->transcfg = AS_TRANSCFG_ADRMODE_UNMAPPED; + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as); +} + +static phys_addr_t pte_to_phy_addr(u64 entry) +{ + if (!(entry & 1)) + return 0; + + return entry & ~0xFFF; +} + +static int ate_is_valid(u64 ate, int const level) +{ + if (level == MIDGARD_MMU_BOTTOMLEVEL) + return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE_L3); + else + return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE_L02); +} + +static int pte_is_valid(u64 pte, int const level) +{ + /* PTEs cannot exist at the bottom level */ + if (level == MIDGARD_MMU_BOTTOMLEVEL) + return false; + return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); +} + +/* + * Map KBASE_REG flags to MMU flags + */ +static u64 get_mmu_flags(unsigned long flags) +{ + u64 mmu_flags; + + /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ + mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; + + /* Set access flags - note that AArch64 stage 1 does not support + * write-only access, so we use read/write instead + */ + if (flags & KBASE_REG_GPU_WR) + mmu_flags |= ENTRY_ACCESS_RW; + else if (flags & KBASE_REG_GPU_RD) + mmu_flags |= ENTRY_ACCESS_RO; + + /* nx if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; + + if (flags & KBASE_REG_SHARE_BOTH) { + /* inner and outer shareable */ + mmu_flags |= SHARE_BOTH_BITS; + } else if (flags & KBASE_REG_SHARE_IN) { + /* inner shareable coherency */ + mmu_flags |= SHARE_INNER_BITS; + } + + return mmu_flags; +} + +static void entry_set_ate(u64 *entry, + struct tagged_addr phy, + unsigned long flags, + int const level) +{ + if (level == MIDGARD_MMU_BOTTOMLEVEL) + page_table_entry_set(entry, as_phys_addr_t(phy) | + get_mmu_flags(flags) | + ENTRY_ACCESS_BIT | ENTRY_IS_ATE_L3); + else + page_table_entry_set(entry, as_phys_addr_t(phy) | + get_mmu_flags(flags) | + ENTRY_ACCESS_BIT | ENTRY_IS_ATE_L02); +} + +static void entry_set_pte(u64 *entry, phys_addr_t phy) +{ + page_table_entry_set(entry, (phy & PAGE_MASK) | + ENTRY_ACCESS_BIT | ENTRY_IS_PTE); +} + +static void entry_invalidate(u64 *entry) +{ + page_table_entry_set(entry, ENTRY_IS_INVAL); +} + +static struct kbase_mmu_mode const aarch64_mode = { + .update = mmu_update, + .get_as_setup = kbase_mmu_get_as_setup, + .disable_as = mmu_disable_as, + .pte_to_phy_addr = pte_to_phy_addr, + .ate_is_valid = ate_is_valid, + .pte_is_valid = pte_is_valid, + .entry_set_ate = entry_set_ate, + .entry_set_pte = entry_set_pte, + .entry_invalidate = entry_invalidate, + .flags = KBASE_MMU_MODE_HAS_NON_CACHEABLE +}; + +struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void) +{ + return &aarch64_mode; +} diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_lpae.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_lpae.c new file mode 100755 index 000000000000..91a2d7ac4dcb --- /dev/null +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_lpae.c @@ -0,0 +1,215 @@ +/* + * + * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + +#include "mali_kbase.h" +#include +#include "mali_kbase_defs.h" + +#define ENTRY_TYPE_MASK 3ULL +#define ENTRY_IS_ATE 1ULL +#define ENTRY_IS_INVAL 2ULL +#define ENTRY_IS_PTE 3ULL + +#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ +#define ENTRY_RD_BIT (1ULL << 6) +#define ENTRY_WR_BIT (1ULL << 7) +#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ +#define ENTRY_ACCESS_BIT (1ULL << 10) +#define ENTRY_NX_BIT (1ULL << 54) + +#define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | \ + ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT) + +/* Helper Function to perform assignment of page table entries, to + * ensure the use of strd, which is required on LPAE systems. + */ +static inline void page_table_entry_set(u64 *pte, u64 phy) +{ +#if KERNEL_VERSION(3, 18, 13) <= LINUX_VERSION_CODE + WRITE_ONCE(*pte, phy); +#else +#ifdef CONFIG_64BIT + barrier(); + *pte = phy; + barrier(); +#elif defined(CONFIG_ARM) + barrier(); + asm volatile("ldrd r0, [%1]\n\t" + "strd r0, %0\n\t" + : "=m" (*pte) + : "r" (&phy) + : "r0", "r1"); + barrier(); +#else +#error "64-bit atomic write must be implemented for your architecture" +#endif +#endif +} + +static void mmu_get_as_setup(struct kbase_mmu_table *mmut, + struct kbase_mmu_setup * const setup) +{ + /* Set up the required caching policies at the correct indices + * in the memattr register. + */ + setup->memattr = + (AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY << + (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | + (AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL << + (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | + (AS_MEMATTR_LPAE_WRITE_ALLOC << + (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | + (AS_MEMATTR_LPAE_OUTER_IMPL_DEF << + (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | + (AS_MEMATTR_LPAE_OUTER_WA << + (AS_MEMATTR_INDEX_OUTER_WA * 8)) | + 0; /* The other indices are unused for now */ + + setup->transtab = ((u64)mmut->pgd & + ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_LPAE_ADDR_SPACE_MASK)) | + AS_TRANSTAB_LPAE_ADRMODE_TABLE | + AS_TRANSTAB_LPAE_READ_INNER; + + setup->transcfg = 0; +} + +static void mmu_update(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut, + int as_nr) +{ + struct kbase_as *as; + struct kbase_mmu_setup *current_setup; + + if (WARN_ON(as_nr == KBASEP_AS_NR_INVALID)) + return; + + as = &kbdev->as[as_nr]; + current_setup = &as->current_setup; + + mmu_get_as_setup(mmut, current_setup); + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as); +} + +static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) +{ + struct kbase_as * const as = &kbdev->as[as_nr]; + struct kbase_mmu_setup * const current_setup = &as->current_setup; + + current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED; + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as); +} + +static phys_addr_t pte_to_phy_addr(u64 entry) +{ + if (!(entry & 1)) + return 0; + + return entry & ~0xFFF; +} + +static int ate_is_valid(u64 ate, int const level) +{ + return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); +} + +static int pte_is_valid(u64 pte, int const level) +{ + return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); +} + +/* + * Map KBASE_REG flags to MMU flags + */ +static u64 get_mmu_flags(unsigned long flags) +{ + u64 mmu_flags; + unsigned long memattr_idx; + + memattr_idx = KBASE_REG_MEMATTR_VALUE(flags); + if (WARN(memattr_idx == AS_MEMATTR_INDEX_NON_CACHEABLE, + "Legacy Mode MMU cannot honor GPU non-cachable memory, will use default instead\n")) + memattr_idx = AS_MEMATTR_INDEX_DEFAULT; + /* store mem_attr index as 4:2, noting that: + * - macro called above ensures 3 bits already + * - all AS_MEMATTR_INDEX_<...> macros only use 3 bits + */ + mmu_flags = memattr_idx << 2; + + /* write perm if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0; + /* read perm if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0; + /* nx if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; + + if (flags & KBASE_REG_SHARE_BOTH) { + /* inner and outer shareable */ + mmu_flags |= SHARE_BOTH_BITS; + } else if (flags & KBASE_REG_SHARE_IN) { + /* inner shareable coherency */ + mmu_flags |= SHARE_INNER_BITS; + } + + return mmu_flags; +} + +static void entry_set_ate(u64 *entry, + struct tagged_addr phy, + unsigned long flags, + int const level) +{ + page_table_entry_set(entry, as_phys_addr_t(phy) | get_mmu_flags(flags) | + ENTRY_IS_ATE); +} + +static void entry_set_pte(u64 *entry, phys_addr_t phy) +{ + page_table_entry_set(entry, (phy & ~0xFFF) | ENTRY_IS_PTE); +} + +static void entry_invalidate(u64 *entry) +{ + page_table_entry_set(entry, ENTRY_IS_INVAL); +} + +static struct kbase_mmu_mode const lpae_mode = { + .update = mmu_update, + .get_as_setup = mmu_get_as_setup, + .disable_as = mmu_disable_as, + .pte_to_phy_addr = pte_to_phy_addr, + .ate_is_valid = ate_is_valid, + .pte_is_valid = pte_is_valid, + .entry_set_ate = entry_set_ate, + .entry_set_pte = entry_set_pte, + .entry_invalidate = entry_invalidate, + .flags = 0 +}; + +struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void) +{ + return &lpae_mode; +} diff --git a/drivers/gpu/arm/bifrost/platform/Kconfig b/drivers/gpu/arm/bifrost/platform/Kconfig new file mode 100755 index 000000000000..ef9fb963ecf5 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/Kconfig @@ -0,0 +1,30 @@ +# +# (C) COPYRIGHT 2012-2013, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + + + +# Add your platform specific Kconfig file here +# +# "drivers/gpu/arm/midgard/platform/xxx/Kconfig" +# +# Where xxx is the platform name is the name set in MALI_PLATFORM_NAME +# + diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/Kbuild b/drivers/gpu/arm/bifrost/platform/devicetree/Kbuild new file mode 100755 index 000000000000..78343c0570d1 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/devicetree/Kbuild @@ -0,0 +1,25 @@ +# +# (C) COPYRIGHT 2012-2017, 2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +mali_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_devicetree.o \ + $(MALI_PLATFORM_DIR)/mali_kbase_runtime_pm.o \ + $(MALI_PLATFORM_DIR)/mali_kbase_clk_rate_trace.o diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_clk_rate_trace.c b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_clk_rate_trace.c new file mode 100755 index 000000000000..11a8b77dca06 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_clk_rate_trace.c @@ -0,0 +1,68 @@ +/* + * + * (C) COPYRIGHT 2015, 2017-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include "mali_kbase_config_platform.h" + +static void *enumerate_gpu_clk(struct kbase_device *kbdev, + unsigned int index) +{ + if (index >= kbdev->nr_clocks) + return NULL; + + return kbdev->clocks[index]; +} + +static unsigned long get_gpu_clk_rate(struct kbase_device *kbdev, + void *gpu_clk_handle) +{ + return clk_get_rate((struct clk *)gpu_clk_handle); +} + +static int gpu_clk_notifier_register(struct kbase_device *kbdev, + void *gpu_clk_handle, struct notifier_block *nb) +{ + compiletime_assert(offsetof(struct clk_notifier_data, clk) == + offsetof(struct kbase_gpu_clk_notifier_data, gpu_clk_handle), + "mismatch in the offset of clk member"); + + compiletime_assert(sizeof(((struct clk_notifier_data *)0)->clk) == + sizeof(((struct kbase_gpu_clk_notifier_data *)0)->gpu_clk_handle), + "mismatch in the size of clk member"); + + return clk_notifier_register((struct clk *)gpu_clk_handle, nb); +} + +static void gpu_clk_notifier_unregister(struct kbase_device *kbdev, + void *gpu_clk_handle, struct notifier_block *nb) +{ + clk_notifier_unregister((struct clk *)gpu_clk_handle, nb); +} + +struct kbase_clk_rate_trace_op_conf clk_rate_trace_ops = { + .get_gpu_clk_rate = get_gpu_clk_rate, + .enumerate_gpu_clk = enumerate_gpu_clk, + .gpu_clk_notifier_register = gpu_clk_notifier_register, + .gpu_clk_notifier_unregister = gpu_clk_notifier_unregister, +}; diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_devicetree.c b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_devicetree.c new file mode 100755 index 000000000000..ccefddf882fd --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_devicetree.c @@ -0,0 +1,41 @@ +/* + * + * (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include + +static struct kbase_platform_config dummy_platform_config; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &dummy_platform_config; +} + +#ifndef CONFIG_OF +int kbase_platform_register(void) +{ + return 0; +} + +void kbase_platform_unregister(void) +{ +} +#endif diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_platform.h new file mode 100755 index 000000000000..2137b425c1ab --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_config_platform.h @@ -0,0 +1,49 @@ +/* + * + * (C) COPYRIGHT 2014-2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +#define CLK_RATE_TRACE_OPS (&clk_rate_trace_ops) + +extern struct kbase_pm_callback_conf pm_callbacks; +extern struct kbase_clk_rate_trace_op_conf clk_rate_trace_ops; + +/** + * Autosuspend delay + * + * The delay time (in milliseconds) to be used for autosuspend + */ +#define AUTO_SUSPEND_DELAY (100) diff --git a/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c new file mode 100755 index 000000000000..8772edb56f73 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/devicetree/mali_kbase_runtime_pm.c @@ -0,0 +1,185 @@ +/* + * + * (C) COPYRIGHT 2015, 2017-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include +#include +#include +#include "mali_kbase_config_platform.h" + +static void enable_gpu_power_control(struct kbase_device *kbdev) +{ + unsigned int i; + +#if defined(CONFIG_REGULATOR) + for (i = 0; i < kbdev->nr_regulators; i++) { + if (WARN_ON(kbdev->regulators[i] == NULL)) + ; + else if (!regulator_is_enabled(kbdev->regulators[i])) + WARN_ON(regulator_enable(kbdev->regulators[i])); + } +#endif + + for (i = 0; i < kbdev->nr_clocks; i++) { + if (WARN_ON(kbdev->clocks[i] == NULL)) + ; + else if (!__clk_is_enabled(kbdev->clocks[i])) + WARN_ON(clk_prepare_enable(kbdev->clocks[i])); + } +} + +static void disable_gpu_power_control(struct kbase_device *kbdev) +{ + unsigned int i; + + for (i = 0; i < kbdev->nr_clocks; i++) { + if (WARN_ON(kbdev->clocks[i] == NULL)) + ; + else if (__clk_is_enabled(kbdev->clocks[i])) { + clk_disable_unprepare(kbdev->clocks[i]); + WARN_ON(__clk_is_enabled(kbdev->clocks[i])); + } + + } + +#if defined(CONFIG_REGULATOR) + for (i = 0; i < kbdev->nr_regulators; i++) { + if (WARN_ON(kbdev->regulators[i] == NULL)) + ; + else if (regulator_is_enabled(kbdev->regulators[i])) + WARN_ON(regulator_disable(kbdev->regulators[i])); + } +#endif +} + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + int ret = 1; /* Assume GPU has been powered off */ + int error; + + dev_dbg(kbdev->dev, "pm_callback_power_on %p\n", + (void *)kbdev->dev->pm_domain); + + enable_gpu_power_control(kbdev); + + error = pm_runtime_get_sync(kbdev->dev); + if (error == 1) { + /* + * Let core know that the chip has not been + * powered off, so we can save on re-initialization. + */ + ret = 0; + } + + dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); + + return ret; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "pm_callback_power_off\n"); + + pm_runtime_mark_last_busy(kbdev->dev); + pm_runtime_put_autosuspend(kbdev->dev); + +#ifndef KBASE_PM_RUNTIME + disable_gpu_power_control(kbdev); +#endif +} + +#ifdef KBASE_PM_RUNTIME +static int kbase_device_runtime_init(struct kbase_device *kbdev) +{ + int ret = 0; + + dev_dbg(kbdev->dev, "kbase_device_runtime_init\n"); + + pm_runtime_set_autosuspend_delay(kbdev->dev, AUTO_SUSPEND_DELAY); + pm_runtime_use_autosuspend(kbdev->dev); + + pm_runtime_set_active(kbdev->dev); + pm_runtime_enable(kbdev->dev); + + if (!pm_runtime_enabled(kbdev->dev)) { + dev_warn(kbdev->dev, "pm_runtime not enabled"); + ret = -ENOSYS; + } + + return ret; +} + +static void kbase_device_runtime_disable(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "kbase_device_runtime_disable\n"); + pm_runtime_disable(kbdev->dev); +} +#endif + +static int pm_callback_runtime_on(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "pm_callback_runtime_on\n"); + + enable_gpu_power_control(kbdev); + return 0; +} + +static void pm_callback_runtime_off(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "pm_callback_runtime_off\n"); + + disable_gpu_power_control(kbdev); +} + +static void pm_callback_resume(struct kbase_device *kbdev) +{ + int ret = pm_callback_runtime_on(kbdev); + + WARN_ON(ret); +} + +static void pm_callback_suspend(struct kbase_device *kbdev) +{ + pm_callback_runtime_off(kbdev); +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = pm_callback_suspend, + .power_resume_callback = pm_callback_resume, +#ifdef KBASE_PM_RUNTIME + .power_runtime_init_callback = kbase_device_runtime_init, + .power_runtime_term_callback = kbase_device_runtime_disable, + .power_runtime_on_callback = pm_callback_runtime_on, + .power_runtime_off_callback = pm_callback_runtime_off, +#else /* KBASE_PM_RUNTIME */ + .power_runtime_init_callback = NULL, + .power_runtime_term_callback = NULL, + .power_runtime_on_callback = NULL, + .power_runtime_off_callback = NULL, +#endif /* KBASE_PM_RUNTIME */ +}; + + diff --git a/drivers/gpu/arm/bifrost/platform/rk/Kbuild b/drivers/gpu/arm/bifrost/platform/rk/Kbuild new file mode 100755 index 000000000000..7cc6c59d969f --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/rk/Kbuild @@ -0,0 +1,17 @@ +# +# (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + +bifrost_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_rk.o \ + diff --git a/drivers/gpu/arm/bifrost/platform/rk/custom_log.h b/drivers/gpu/arm/bifrost/platform/rk/custom_log.h new file mode 100755 index 000000000000..5de70ee13d25 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/rk/custom_log.h @@ -0,0 +1,192 @@ +/* + * (C) COPYRIGHT RockChip Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +#ifndef __CUSTOM_LOG_H__ +#define __CUSTOM_LOG_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/* ----------------------------------------------------------------------------- + * Include Files + * ----------------------------------------------------------------------------- + */ +#include +#include + +/* ----------------------------------------------------------------------------- + * Macros Definition + * ----------------------------------------------------------------------------- + */ + +/** 若下列 macro 有被定义, æ‰ ä½¿èƒ½ log 输出. */ +/* #define ENABLE_DEBUG_LOG */ + +/*----------------------------------------------------------------------------*/ + +#ifdef ENABLE_VERBOSE_LOG +/** Verbose log. */ +#define V(fmt, args...) \ + pr_debug("V : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) +#else +#define V(...) ((void)0) +#endif + +#ifdef ENABLE_DEBUG_LOG +/** Debug log. */ +#define D(fmt, args...) \ + pr_info("D : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) +#else +#define D(...) ((void)0) +#endif + +#define I(fmt, args...) \ + pr_info("I : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +#define W(fmt, args...) \ + pr_warn("W : [File] : %s; [Line] : %d; [Func] : %s(); " \ + fmt "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +#define E(fmt, args...) \ + pr_err("E : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +/*-------------------------------------------------------*/ + +/** 使用 D(), 以åè¿›åˆ¶çš„å½¢å¼æ‰“å°å˜é‡ 'var' çš„ value. */ +#define D_DEC(var) D(#var " = %d.", var) + +#define E_DEC(var) E(#var " = %d.", var) + +/** 使用 D(), 以åå…­è¿›åˆ¶çš„å½¢å¼æ‰“å°å˜é‡ 'var' çš„ value. */ +#define D_HEX(var) D(#var " = 0x%x.", var) + +#define E_HEX(var) E(#var " = 0x%x.", var) + +/** + * 使用 D(), 以å六进制的形å¼, + * æ‰“å°æŒ‡é’ˆç±»åž‹å˜é‡ 'ptr' çš„ value. + */ +#define D_PTR(ptr) D(#ptr " = %p.", ptr) + +#define E_PTR(ptr) E(#ptr " = %p.", ptr) + +/** 使用 D(), æ‰“å° char 字串. */ +#define D_STR(p_str) \ +do { \ + if (!p_str) { \ + D(#p_str " = NULL."); \ + else \ + D(#p_str " = '%s'.", p_str); \ +} while (0) + +#define E_STR(p_str) \ +do { \ + if (!p_str) \ + E(#p_str " = NULL."); \ + else \ + E(#p_str " = '%s'.", p_str); \ +} while (0) + +#ifdef ENABLE_DEBUG_LOG +/** + * log 从 'p_start' 地å€å¼€å§‹çš„ 'len' 个字节的数æ®. + */ +#define D_MEM(p_start, len) \ +do { \ + int i = 0; \ + char *p = (char *)(p_start); \ + D("dump memory from addr of '" #p_start "', from %p, length %d' : ", \ + (p_start), \ + (len)); \ + pr_debug("\t\t"); \ + for (i = 0; i < (len); i++) \ + pr_debug("0x%02x, ", p[i]); \ + pr_debug("\n"); \ +} while (0) +#else +#define D_MEM(...) ((void)0) +#endif + +/*-------------------------------------------------------*/ + +/** + * 在特定æ¡ä»¶ä¸‹, 判定 error å‘生, + * å°†å˜é‡ 'ret_var' 设置 'err_code', + * log 输出对应的 Error Caution, + * ç„¶åŽè·³è½¬ 'label' 指定的代ç å¤„执行. + * @param msg + * 纯字串形å¼çš„æç¤ºä¿¡æ¯. + * @param ret_var + * æ ‡è¯†å‡½æ•°æ‰§è¡ŒçŠ¶æ€æˆ–者结果的å˜é‡, + * 将被设置具体的 Error Code. + * 通常是 'ret' or 'result'. + * @param err_code + * 表å¾ç‰¹å®š error 的常数标识, + * 通常是 å®çš„å½¢æ€. + * @param label + * 程åºå°†è¦è·³è½¬åˆ°çš„错误处ç†ä»£ç çš„æ ‡å·, + * 通常就是 'EXIT'. + * @param args... + * 对应 'msg_fmt' 实å‚中, + * '%s', '%d', ... 等转æ¢è¯´æ˜Žç¬¦çš„具体å¯å˜é•¿å®žå‚. + */ +#define SET_ERROR_AND_JUMP(msg_fmt, ret_var, err_code, label, args...) \ +do { \ + E("To set '" #ret_var "' to %d('" #err_code "'), because : " msg_fmt, \ + (err_code), \ + ## args); \ + (ret_var) = (err_code); \ + goto label; \ +} while (0) + +/* ----------------------------------------------------------------------------- + * Types and Structures Definition + * ----------------------------------------------------------------------------- + */ + +/* ----------------------------------------------------------------------------- + * Global Functions' Prototype + * ----------------------------------------------------------------------------- + */ + +/* ----------------------------------------------------------------------------- + * Inline Functions Implementation + * ----------------------------------------------------------------------------- + */ + +#ifdef __cplusplus +} +#endif + +#endif /* __CUSTOM_LOG_H__ */ diff --git a/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_platform.h new file mode 100755 index 000000000000..07c5b6f8a760 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_platform.h @@ -0,0 +1,88 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +/** + * @file mali_kbase_config_platform.h + * 声明 platform_config_of_rk (platform_rk çš„ platform_config). + */ + +/** + * Maximum frequency GPU will be clocked at. + * Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX (5000) + +/** + * Minimum frequency GPU will be clocked at. + * Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN (5000) + +/** + * CPU_SPEED_FUNC + * - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz + * - see kbase_cpu_clk_speed_func for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (NULL) + +/** + * GPU_SPEED_FUNC + * - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz + * - see kbase_gpu_clk_speed_func for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: + * pointer to @ref kbase_pm_callback_conf + * Default value: + * See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) +extern struct kbase_pm_callback_conf pm_callbacks; + +/** + * Platform specific configuration functions + * + * Attached value: + * pointer to @ref kbase_platform_funcs_conf + * Default value: + * See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (&platform_funcs) +extern struct kbase_platform_funcs_conf platform_funcs; + +/** + * Secure mode switch + * + * Attached value: pointer to @ref kbase_secure_ops + */ +#define SECURE_CALLBACKS (NULL) + diff --git a/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c new file mode 100755 index 000000000000..e73ef450d135 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c @@ -0,0 +1,459 @@ +/* + * (C) COPYRIGHT RockChip Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +/* #define ENABLE_DEBUG_LOG */ +#include "custom_log.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mali_kbase_rk.h" + +/** + * @file mali_kbase_config_rk.c + * 对 platform_config_of_rk 的具体实现. + * + * mali_device_driver 包å«ä¸¤éƒ¨åˆ† : + * .DP : platform_dependent_part_in_mdd : + * ä¾èµ– platform 部分, + * æºç åœ¨ /platform// + * 在 mali_device_driver 内部, + * 记为 platform_dependent_part, + * 也被记为 platform_specific_code. + * .DP : common_parts_in_mdd : + * arm 实现的通用的部分, + * æºç åœ¨ / 下. + * 在 mali_device_driver 内部, 记为 common_parts. + */ + +/*---------------------------------------------------------------------------*/ + +#ifdef CONFIG_REGULATOR +static int rk_pm_enable_regulator(struct kbase_device *kbdev); +static void rk_pm_disable_regulator(struct kbase_device *kbdev); +#else +static inline int rk_pm_enable_regulator(struct kbase_device *kbdev) +{ + return 0; +} + +static inline void rk_pm_disable_regulator(struct kbase_device *kbdev) +{ +} +#endif + +static int rk_pm_enable_clk(struct kbase_device *kbdev); + +static void rk_pm_disable_clk(struct kbase_device *kbdev); + +static int kbase_platform_rk_create_sysfs_files(struct device *dev); + +static void kbase_platform_rk_remove_sysfs_files(struct device *dev); + +/*---------------------------------------------------------------------------*/ + +static void rk_pm_power_off_delay_work(struct work_struct *work) +{ + struct rk_context *platform = + container_of(to_delayed_work(work), struct rk_context, work); + struct kbase_device *kbdev = platform->kbdev; + + if (!platform->is_powered) { + D("mali_dev is already powered off."); + return; + } + + if (pm_runtime_enabled(kbdev->dev)) { + D("to put_sync_suspend mali_dev."); + pm_runtime_put_sync_suspend(kbdev->dev); + } + + rk_pm_disable_regulator(kbdev); + + platform->is_powered = false; + wake_unlock(&platform->wake_lock); +} + +static int kbase_platform_rk_init(struct kbase_device *kbdev) +{ + int ret = 0; + struct rk_context *platform; + + platform = kzalloc(sizeof(*platform), GFP_KERNEL); + if (!platform) { + E("err."); + return -ENOMEM; + } + + platform->is_powered = false; + platform->kbdev = kbdev; + + platform->delay_ms = 200; + if (of_property_read_u32(kbdev->dev->of_node, "power-off-delay-ms", + &platform->delay_ms)) + W("power-off-delay-ms not available."); + + platform->power_off_wq = create_freezable_workqueue("gpu_power_off_wq"); + if (!platform->power_off_wq) { + E("couldn't create workqueue"); + ret = -ENOMEM; + goto err_wq; + } + INIT_DEFERRABLE_WORK(&platform->work, rk_pm_power_off_delay_work); + + wake_lock_init(&platform->wake_lock, WAKE_LOCK_SUSPEND, "gpu"); + + platform->utilisation_period = DEFAULT_UTILISATION_PERIOD_IN_MS; + + ret = kbase_platform_rk_create_sysfs_files(kbdev->dev); + if (ret) { + E("fail to create sysfs_files. ret = %d.", ret); + goto err_sysfs_files; + } + + kbdev->platform_context = (void *)platform; + pm_runtime_enable(kbdev->dev); + + return 0; + +err_sysfs_files: + wake_lock_destroy(&platform->wake_lock); + destroy_workqueue(platform->power_off_wq); +err_wq: + return ret; +} + +static void kbase_platform_rk_term(struct kbase_device *kbdev) +{ + struct rk_context *platform = + (struct rk_context *)kbdev->platform_context; + + pm_runtime_disable(kbdev->dev); + kbdev->platform_context = NULL; + + if (platform) { + cancel_delayed_work_sync(&platform->work); + wake_lock_destroy(&platform->wake_lock); + destroy_workqueue(platform->power_off_wq); + platform->is_powered = false; + platform->kbdev = NULL; + kfree(platform); + } + kbase_platform_rk_remove_sysfs_files(kbdev->dev); +} + +struct kbase_platform_funcs_conf platform_funcs = { + .platform_init_func = &kbase_platform_rk_init, + .platform_term_func = &kbase_platform_rk_term, +}; + +/*---------------------------------------------------------------------------*/ + +static int rk_pm_callback_runtime_on(struct kbase_device *kbdev) +{ + return 0; +} + +static void rk_pm_callback_runtime_off(struct kbase_device *kbdev) +{ +} + +static int rk_pm_callback_power_on(struct kbase_device *kbdev) +{ + int ret = 1; /* Assume GPU has been powered off */ + int err = 0; + struct rk_context *platform = get_rk_context(kbdev); + + cancel_delayed_work_sync(&platform->work); + + err = rk_pm_enable_clk(kbdev); + if (err) { + E("failed to enable clk: %d", err); + return err; + } + + if (platform->is_powered) { + D("mali_device is already powered."); + return 0; + } + + /* we must enable vdd_gpu before pd_gpu_in_chip. */ + err = rk_pm_enable_regulator(kbdev); + if (err) { + E("fail to enable regulator, err : %d.", err); + return err; + } + + /* è‹¥ mali_dev çš„ runtime_pm 是 enabled çš„, 则... */ + if (pm_runtime_enabled(kbdev->dev)) { + D("to resume mali_dev syncly."); + /* 对 pd_in_chip çš„ on æ“作, + * 将在 pm_domain çš„ runtime_pm_callbacks 中完æˆ. + */ + err = pm_runtime_get_sync(kbdev->dev); + if (err < 0) { + E("failed to runtime resume device: %d.", err); + return err; + } else if (err == 1) { /* runtime_pm_status is still active */ + D("chip has NOT been powered off, no need to re-init."); + ret = 0; + } + } + + platform->is_powered = true; + wake_lock(&platform->wake_lock); + + return ret; +} + +static void rk_pm_callback_power_off(struct kbase_device *kbdev) +{ + struct rk_context *platform = get_rk_context(kbdev); + + rk_pm_disable_clk(kbdev); + queue_delayed_work(platform->power_off_wq, &platform->work, + msecs_to_jiffies(platform->delay_ms)); +} + +int rk_kbase_device_runtime_init(struct kbase_device *kbdev) +{ + return 0; +} + +void rk_kbase_device_runtime_disable(struct kbase_device *kbdev) +{ +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = rk_pm_callback_power_on, + .power_off_callback = rk_pm_callback_power_off, +#ifdef CONFIG_PM + .power_runtime_init_callback = rk_kbase_device_runtime_init, + .power_runtime_term_callback = rk_kbase_device_runtime_disable, + .power_runtime_on_callback = rk_pm_callback_runtime_on, + .power_runtime_off_callback = rk_pm_callback_runtime_off, +#else /* CONFIG_PM */ + .power_runtime_init_callback = NULL, + .power_runtime_term_callback = NULL, + .power_runtime_on_callback = NULL, + .power_runtime_off_callback = NULL, +#endif /* CONFIG_PM */ +}; + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} + +/*---------------------------------------------------------------------------*/ + +void kbase_platform_rk_shutdown(struct kbase_device *kbdev) +{ + I("to make vdd_gpu enabled for turning off pd_gpu in pm_framework."); + rk_pm_enable_regulator(kbdev); +} + +/*---------------------------------------------------------------------------*/ + +#ifdef CONFIG_REGULATOR +static int rk_pm_enable_regulator(struct kbase_device *kbdev) +{ + int ret = 0; + unsigned int i; + + for (i = 0; i < kbdev->nr_regulators; i++) { + struct regulator *regulator = kbdev->regulators[i]; + if (!regulator) { + W("no mali regulator control, no need to enable."); + goto EXIT; + } + + D("to enable regulator."); + ret = regulator_enable(regulator); + if (ret) { + E("fail to enable regulator, ret : %d.", ret); + goto EXIT; + } + } + +EXIT: + return ret; +} + +static void rk_pm_disable_regulator(struct kbase_device *kbdev) +{ + unsigned int i; + + for (i = 0; i < kbdev->nr_regulators; i++) { + struct regulator *regulator = kbdev->regulators[i]; + + if (!regulator) { + W("no mali regulator control, no need to disable."); + return; + } + + D("to disable regulator."); + regulator_disable(regulator); + } +} +#endif + +static int rk_pm_enable_clk(struct kbase_device *kbdev) +{ + int err = 0; + unsigned int i; + + for (i = 0; i < kbdev->nr_clocks; i++) { + struct clk *clock = kbdev->clocks[i]; + + if (!clock) { + W("no mali clock control, no need to enable."); + } else { + D("to enable clk."); + err = clk_enable(clock); + if (err) + E("failed to enable clk: %d.", err); + } + } + + return err; +} + +static void rk_pm_disable_clk(struct kbase_device *kbdev) +{ + unsigned int i; + + for (i = 0; i < kbdev->nr_clocks; i++) { + struct clk *clock = kbdev->clocks[i]; + + if (!clock) { + W("no mali clock control, no need to disable."); + } else { + D("to disable clk."); + clk_disable(clock); + } + } +} + +/*---------------------------------------------------------------------------*/ + +static ssize_t utilisation_period_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct rk_context *platform = get_rk_context(kbdev); + ssize_t ret = 0; + + ret += snprintf(buf, PAGE_SIZE, "%u\n", platform->utilisation_period); + + return ret; +} + +static ssize_t utilisation_period_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct rk_context *platform = get_rk_context(kbdev); + int ret = 0; + + ret = kstrtouint(buf, 0, &platform->utilisation_period); + if (ret) { + E("invalid input period : %s.", buf); + return ret; + } + D("set utilisation_period to '%d'.", platform->utilisation_period); + + return count; +} + +static ssize_t utilisation_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct rk_context *platform = get_rk_context(kbdev); + ssize_t ret = 0; + unsigned long period_in_us = platform->utilisation_period * 1000; + u32 utilisation; + struct kbasep_pm_metrics metrics_when_start; + struct kbasep_pm_metrics metrics_diff; /* between start and end. */ + u32 total_time = 0; + u32 busy_time = 0; + + /* get current metrics data. */ + kbase_pm_get_dvfs_metrics(kbdev, &metrics_when_start, &metrics_diff); + /* sleep for 'period_in_us'. */ + usleep_range(period_in_us, period_in_us + 100); + /* get metrics data between start and end. */ + kbase_pm_get_dvfs_metrics(kbdev, &metrics_when_start, &metrics_diff); + + total_time = metrics_diff.time_busy + metrics_diff.time_idle; + busy_time = metrics_diff.time_busy; + D("total_time : %u, busy_time : %u.", total_time, busy_time); + + utilisation = busy_time * 100 / total_time; + ret += snprintf(buf, PAGE_SIZE, "%d\n", utilisation); + + return ret; +} + +static DEVICE_ATTR_RW(utilisation_period); +static DEVICE_ATTR_RO(utilisation); + +static int kbase_platform_rk_create_sysfs_files(struct device *dev) +{ + int ret = 0; + + ret = device_create_file(dev, &dev_attr_utilisation_period); + if (ret) { + E("fail to create sysfs file 'utilisation_period'."); + goto out; + } + + ret = device_create_file(dev, &dev_attr_utilisation); + if (ret) { + E("fail to create sysfs file 'utilisation'."); + goto remove_utilisation_period; + } + + return 0; + +remove_utilisation_period: + device_remove_file(dev, &dev_attr_utilisation_period); +out: + return ret; +} + +static void kbase_platform_rk_remove_sysfs_files(struct device *dev) +{ + device_remove_file(dev, &dev_attr_utilisation_period); + device_remove_file(dev, &dev_attr_utilisation); +} + +int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev) +{ + return rockchip_init_opp_table(kbdev->dev, NULL, + "gpu_leakage", "mali"); +} diff --git a/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_rk.h b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_rk.h new file mode 100755 index 000000000000..6eab25014d21 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_rk.h @@ -0,0 +1,62 @@ +/* drivers/gpu/t6xx/kbase/src/platform/rk/mali_kbase_platform.h + * Rockchip SoC Mali-Midgard platform-dependent codes + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software FoundatIon. + */ + +/** + * @file mali_kbase_rk.h + * + * defines work_context type of platform_dependent_part. + */ + +#ifndef _MALI_KBASE_RK_H_ +#define _MALI_KBASE_RK_H_ + +#include + +/*---------------------------------------------------------------------------*/ + +#define DEFAULT_UTILISATION_PERIOD_IN_MS (100) + +/*---------------------------------------------------------------------------*/ + +/* + * struct rk_context - work_context of platform_dependent_part_of_rk. + */ +struct rk_context { + /* + * record the status of common_parts calling 'power_on_callback' + * and 'power_off_callback'. + */ + bool is_powered; + + struct kbase_device *kbdev; + + struct workqueue_struct *power_off_wq; + /* delayed_work_to_power_off_gpu. */ + struct delayed_work work; + unsigned int delay_ms; + + /* + * WAKE_LOCK_SUSPEND for ensuring to run + * delayed_work_to_power_off_gpu before suspend. + */ + struct wake_lock wake_lock; + + /* debug only, the period in ms to count gpu_utilisation. */ + unsigned int utilisation_period; +}; + +/*---------------------------------------------------------------------------*/ + +static inline struct rk_context *get_rk_context( + const struct kbase_device *kbdev) +{ + return (struct rk_context *)(kbdev->platform_context); +} + +#endif /* _MALI_KBASE_RK_H_ */ + diff --git a/drivers/gpu/arm/bifrost/platform/vexpress/Kbuild b/drivers/gpu/arm/bifrost/platform/vexpress/Kbuild new file mode 100755 index 000000000000..6780e4c9433b --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/vexpress/Kbuild @@ -0,0 +1,24 @@ +# +# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +mali_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ + mali_kbase_platform_fake.o diff --git a/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_platform.h new file mode 100755 index 000000000000..fac3cd52182f --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_platform.h @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_vexpress.c new file mode 100755 index 000000000000..d165ce262814 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/vexpress/mali_kbase_config_vexpress.c @@ -0,0 +1,69 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#include +#include +#include +#include +#include "mali_kbase_config_platform.h" + +#ifndef CONFIG_OF +static struct kbase_io_resources io_resources = { + .job_irq_number = 68, + .mmu_irq_number = 69, + .gpu_irq_number = 70, + .io_memory_region = { + .start = 0xFC010000, + .end = 0xFC010000 + (4096 * 4) - 1 + } +}; +#endif /* CONFIG_OF */ + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ + return 1; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = NULL, + .power_resume_callback = NULL +}; + +static struct kbase_platform_config versatile_platform_config = { +#ifndef CONFIG_OF + .io_resources = &io_resources +#endif +}; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &versatile_platform_config; +} diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/Kbuild b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/Kbuild new file mode 100755 index 000000000000..51b408efd48a --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/Kbuild @@ -0,0 +1,24 @@ +# +# (C) COPYRIGHT 2013-2014, 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +mali_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ + mali_kbase_platform_fake.o diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h new file mode 100755 index 000000000000..fac3cd52182f --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c new file mode 100755 index 000000000000..efca0a5b3493 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c @@ -0,0 +1,65 @@ +/* + * + * (C) COPYRIGHT 2011-2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include +#include + +#ifndef CONFIG_OF +static struct kbase_io_resources io_resources = { + .job_irq_number = 68, + .mmu_irq_number = 69, + .gpu_irq_number = 70, + .io_memory_region = { + .start = 0x2f010000, + .end = 0x2f010000 + (4096 * 4) - 1} +}; +#endif + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ + return 1; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = NULL, + .power_resume_callback = NULL +}; + +static struct kbase_platform_config versatile_platform_config = { +#ifndef CONFIG_OF + .io_resources = &io_resources +#endif +}; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &versatile_platform_config; +} diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/Kbuild b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/Kbuild new file mode 100755 index 000000000000..e07709c9b1a5 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/Kbuild @@ -0,0 +1,25 @@ +# +# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +mali_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ + $(MALI_PLATFORM_DIR)/mali_kbase_cpu_vexpress.o \ + mali_kbase_platform_fake.o diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h new file mode 100755 index 000000000000..fac3cd52182f --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c new file mode 100755 index 000000000000..b6714b95b776 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c @@ -0,0 +1,67 @@ +/* + * + * (C) COPYRIGHT 2011-2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + + + +#include +#include +#include +#include + +#ifndef CONFIG_OF +static struct kbase_io_resources io_resources = { + .job_irq_number = 75, + .mmu_irq_number = 76, + .gpu_irq_number = 77, + .io_memory_region = { + .start = 0x2F000000, + .end = 0x2F000000 + (4096 * 4) - 1} +}; +#endif + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ + return 1; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = NULL, + .power_resume_callback = NULL +}; + +static struct kbase_platform_config versatile_platform_config = { +#ifndef CONFIG_OF + .io_resources = &io_resources +#endif +}; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &versatile_platform_config; +} diff --git a/drivers/gpu/arm/bifrost/protected_mode_switcher.h b/drivers/gpu/arm/bifrost/protected_mode_switcher.h new file mode 100755 index 000000000000..8778d812aea0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/protected_mode_switcher.h @@ -0,0 +1,69 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _PROTECTED_MODE_SWITCH_H_ +#define _PROTECTED_MODE_SWITCH_H_ + +struct protected_mode_device; + +/** + * struct protected_mode_ops - Callbacks for protected mode switch operations + * + * @protected_mode_enable: Callback to enable protected mode for device + * @protected_mode_disable: Callback to disable protected mode for device + */ +struct protected_mode_ops { + /** + * protected_mode_enable() - Enable protected mode on device + * @dev: The struct device + * + * Return: 0 on success, non-zero on error + */ + int (*protected_mode_enable)( + struct protected_mode_device *protected_dev); + + /** + * protected_mode_disable() - Disable protected mode on device, and + * reset device + * @dev: The struct device + * + * Return: 0 on success, non-zero on error + */ + int (*protected_mode_disable)( + struct protected_mode_device *protected_dev); +}; + +/** + * struct protected_mode_device - Device structure for protected mode devices + * + * @ops - Callbacks associated with this device + * @data - Pointer to device private data + * + * This structure should be registered with the platform device using + * platform_set_drvdata(). + */ +struct protected_mode_device { + struct protected_mode_ops ops; + void *data; +}; + +#endif /* _PROTECTED_MODE_SWITCH_H_ */ diff --git a/drivers/gpu/arm/bifrost/tests/Kbuild b/drivers/gpu/arm/bifrost/tests/Kbuild new file mode 100755 index 000000000000..c26bef780781 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/Kbuild @@ -0,0 +1,24 @@ +# +# (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +obj-$(CONFIG_MALI_KUTF) += kutf/ +obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test/ +obj-$(CONFIG_MALI_CLK_RATE_TRACE_PORTAL) += mali_kutf_clk_rate_trace/kernel/ diff --git a/drivers/gpu/arm/bifrost/tests/Kconfig b/drivers/gpu/arm/bifrost/tests/Kconfig new file mode 100755 index 000000000000..83a4d7764a50 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/Kconfig @@ -0,0 +1,24 @@ +# +# (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +source "drivers/gpu/arm/midgard/tests/kutf/Kconfig" +source "drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig" +source "drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/Kconfig" diff --git a/drivers/gpu/arm/bifrost/tests/Mconfig b/drivers/gpu/arm/bifrost/tests/Mconfig new file mode 100755 index 000000000000..bba96b3d9e48 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/Mconfig @@ -0,0 +1,38 @@ +# +# (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# + +config UNIT_TEST_KERNEL_MODULES + bool + default y if UNIT_TEST_CODE && BUILD_KERNEL_MODULES + default n + +config BUILD_IPA_TESTS + bool + default y if UNIT_TEST_KERNEL_MODULES && MALI_BIFROST_DEVFREQ + default n + +config BUILD_IPA_UNIT_TESTS + bool + default y if NO_MALI && BUILD_IPA_TESTS + default n + +config BUILD_CSF_TESTS + bool + default y if UNIT_TEST_KERNEL_MODULES && GPU_HAS_CSF + default n + +config BUILD_ARBIF_TESTS + bool + default y if UNIT_TEST_KERNEL_MODULES && MALI_ARBITER_SUPPORT + default n + diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers.h new file mode 100755 index 000000000000..858b9c38b49a --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers.h @@ -0,0 +1,85 @@ +/* + * + * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KERNEL_UTF_HELPERS_H_ +#define _KERNEL_UTF_HELPERS_H_ + +/* kutf_helpers.h + * Test helper functions for the kernel UTF test infrastructure. + * + * These functions provide methods for enqueuing/dequeuing lines of text sent + * by user space. They are used to implement the transfer of "userdata" from + * user space to kernel. + */ + +#include + +/** + * kutf_helper_pending_input() - Check any pending lines sent by user space + * @context: KUTF context + * + * Return: true if there are pending lines, otherwise false + */ +bool kutf_helper_pending_input(struct kutf_context *context); + +/** + * kutf_helper_input_dequeue() - Dequeue a line sent by user space + * @context: KUTF context + * @str_size: Pointer to an integer to receive the size of the string + * + * If no line is available then this function will wait (interruptibly) until + * a line is available. + * + * Return: The line dequeued, ERR_PTR(-EINTR) if interrupted or NULL on end + * of data. + */ +char *kutf_helper_input_dequeue(struct kutf_context *context, size_t *str_size); + +/** + * kutf_helper_input_enqueue() - Enqueue a line sent by user space + * @context: KUTF context + * @str: The user space address of the line + * @size: The length in bytes of the string + * + * This function will use copy_from_user to copy the string out of user space. + * The string need not be NULL-terminated (@size should not include the NULL + * termination). + * + * As a special case @str==NULL and @size==0 is valid to mark the end of input, + * but callers should use kutf_helper_input_enqueue_end_of_data() instead. + * + * Return: 0 on success, -EFAULT if the line cannot be copied from user space, + * -ENOMEM if out of memory. + */ +int kutf_helper_input_enqueue(struct kutf_context *context, + const char __user *str, size_t size); + +/** + * kutf_helper_input_enqueue_end_of_data() - Signal no more data is to be sent + * @context: KUTF context + * + * After this function has been called, kutf_helper_input_dequeue() will always + * return NULL. + */ +void kutf_helper_input_enqueue_end_of_data(struct kutf_context *context); + +#endif /* _KERNEL_UTF_HELPERS_H_ */ diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers_user.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers_user.h new file mode 100755 index 000000000000..3b1300e1ce6f --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_helpers_user.h @@ -0,0 +1,179 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KERNEL_UTF_HELPERS_USER_H_ +#define _KERNEL_UTF_HELPERS_USER_H_ + +/* kutf_helpers.h + * Test helper functions for the kernel UTF test infrastructure, whose + * implementation mirrors that of similar functions for kutf-userside + */ + +#include +#include + + +#define KUTF_HELPER_MAX_VAL_NAME_LEN 255 + +enum kutf_helper_valtype { + KUTF_HELPER_VALTYPE_INVALID, + KUTF_HELPER_VALTYPE_U64, + KUTF_HELPER_VALTYPE_STR, + + KUTF_HELPER_VALTYPE_COUNT /* Must be last */ +}; + +struct kutf_helper_named_val { + enum kutf_helper_valtype type; + char *val_name; + union { + u64 val_u64; + char *val_str; + } u; +}; + +/* Extra error values for certain helpers when we want to distinguish between + * Linux's own error values too. + * + * These can only be used on certain functions returning an int type that are + * documented as returning one of these potential values, they cannot be used + * from functions return a ptr type, since we can't decode it with PTR_ERR + * + * No negative values are used - Linux error codes should be used instead, and + * indicate a problem in accessing the data file itself (are generally + * unrecoverable) + * + * Positive values indicate correct access but invalid parsing (can be + * recovered from assuming data in the future is correct) */ +enum kutf_helper_err { + /* No error - must be zero */ + KUTF_HELPER_ERR_NONE = 0, + /* Named value parsing encountered an invalid name */ + KUTF_HELPER_ERR_INVALID_NAME, + /* Named value parsing of string or u64 type encountered extra + * characters after the value (after the last digit for a u64 type or + * after the string end delimiter for string type) */ + KUTF_HELPER_ERR_CHARS_AFTER_VAL, + /* Named value parsing of string type couldn't find the string end + * delimiter. + * + * This cannot be encountered when the NAME="value" message exceeds the + * textbuf's maximum line length, because such messages are not checked + * for an end string delimiter */ + KUTF_HELPER_ERR_NO_END_DELIMITER, + /* Named value didn't parse as any of the known types */ + KUTF_HELPER_ERR_INVALID_VALUE, +}; + + +/* Send named NAME=value pair, u64 value + * + * NAME must match [A-Z0-9_]\+ and can be up to MAX_VAL_NAME_LEN characters long + * + * Any failure will be logged on the suite's current test fixture + * + * Returns 0 on success, non-zero on failure + */ +int kutf_helper_send_named_u64(struct kutf_context *context, + const char *val_name, u64 val); + +/* Get the maximum length of a string that can be represented as a particular + * NAME="value" pair without string-value truncation in the kernel's buffer + * + * Given val_name and the kernel buffer's size, this can be used to determine + * the maximum length of a string that can be sent as val_name="value" pair + * without having the string value truncated. Any string longer than this will + * be truncated at some point during communication to this size. + * + * It is assumed that val_name is a valid name for + * kutf_helper_send_named_str(), and no checking will be made to + * ensure this. + * + * Returns the maximum string length that can be represented, or a negative + * value if the NAME="value" encoding itself wouldn't fit in kern_buf_sz + */ +int kutf_helper_max_str_len_for_kern(const char *val_name, int kern_buf_sz); + +/* Send named NAME="str" pair + * + * no escaping allowed in str. Any of the following characters will terminate + * the string: '"' '\\' '\n' + * + * NAME must match [A-Z0-9_]\+ and can be up to MAX_VAL_NAME_LEN characters long + * + * Any failure will be logged on the suite's current test fixture + * + * Returns 0 on success, non-zero on failure */ +int kutf_helper_send_named_str(struct kutf_context *context, + const char *val_name, const char *val_str); + +/* Receive named NAME=value pair + * + * This can receive u64 and string values - check named_val->type + * + * If you are not planning on dynamic handling of the named value's name and + * type, then kutf_helper_receive_check_val() is more useful as a + * convenience function. + * + * String members of named_val will come from memory allocated on the fixture's mempool + * + * Returns 0 on success. Negative value on failure to receive from the 'run' + * file, positive value indicates an enum kutf_helper_err value for correct + * reception of data but invalid parsing */ +int kutf_helper_receive_named_val( + struct kutf_context *context, + struct kutf_helper_named_val *named_val); + +/* Receive and validate NAME=value pair + * + * As with kutf_helper_receive_named_val, but validate that the + * name and type are as expected, as a convenience for a common pattern found + * in tests. + * + * NOTE: this only returns an error value if there was actually a problem + * receiving data. + * + * NOTE: If the underlying data was received correctly, but: + * - isn't of the expected name + * - isn't the expected type + * - isn't correctly parsed for the type + * then the following happens: + * - failure result is recorded + * - named_val->type will be KUTF_HELPER_VALTYPE_INVALID + * - named_val->u will contain some default value that should be relatively + * harmless for the test, including being writable in the case of string + * values + * - return value will be 0 to indicate success + * + * The rationale behind this is that we'd prefer to continue the rest of the + * test with failures propagated, rather than hitting a timeout */ +int kutf_helper_receive_check_val( + struct kutf_helper_named_val *named_val, + struct kutf_context *context, + const char *expect_val_name, + enum kutf_helper_valtype expect_val_type); + +/* Output a named value to kmsg */ +void kutf_helper_output_named_val(struct kutf_helper_named_val *named_val); + + +#endif /* _KERNEL_UTF_HELPERS_USER_H_ */ diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_mem.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_mem.h new file mode 100755 index 000000000000..988559de1edf --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_mem.h @@ -0,0 +1,73 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KERNEL_UTF_MEM_H_ +#define _KERNEL_UTF_MEM_H_ + +/* kutf_mem.h + * Functions for management of memory pools in the kernel. + * + * This module implements a memory pool allocator, allowing a test + * implementation to allocate linked allocations which can then be freed by a + * single free which releases all of the resources held by the entire pool. + * + * Note that it is not possible to free single resources within the pool once + * allocated. + */ + +#include +#include + +/** + * struct kutf_mempool - the memory pool context management structure + * @head: list head on which the allocations in this context are added to + * @lock: mutex for concurrent allocation from multiple threads + * + */ +struct kutf_mempool { + struct list_head head; + struct mutex lock; +}; + +/** + * kutf_mempool_init() - Initialize a memory pool. + * @pool: Memory pool structure to initialize, provided by the user + * + * Return: zero on success + */ +int kutf_mempool_init(struct kutf_mempool *pool); + +/** + * kutf_mempool_alloc() - Allocate memory from a pool + * @pool: Memory pool to allocate from + * @size: Size of memory wanted in number of bytes + * + * Return: Pointer to memory on success, NULL on failure. + */ +void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size); + +/** + * kutf_mempool_destroy() - Destroy a memory pool, freeing all memory within it. + * @pool: The memory pool to free + */ +void kutf_mempool_destroy(struct kutf_mempool *pool); +#endif /* _KERNEL_UTF_MEM_H_ */ diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_resultset.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_resultset.h new file mode 100755 index 000000000000..49ebeb4ec546 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_resultset.h @@ -0,0 +1,181 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KERNEL_UTF_RESULTSET_H_ +#define _KERNEL_UTF_RESULTSET_H_ + +/* kutf_resultset.h + * Functions and structures for handling test results and result sets. + * + * This section of the kernel UTF contains structures and functions used for the + * management of Results and Result Sets. + */ + +/** + * enum kutf_result_status - Status values for a single Test error. + * @KUTF_RESULT_BENCHMARK: Result is a meta-result containing benchmark + * results. + * @KUTF_RESULT_SKIP: The test was skipped. + * @KUTF_RESULT_UNKNOWN: The test has an unknown result. + * @KUTF_RESULT_PASS: The test result passed. + * @KUTF_RESULT_DEBUG: The test result passed, but raised a debug + * message. + * @KUTF_RESULT_INFO: The test result passed, but raised + * an informative message. + * @KUTF_RESULT_WARN: The test result passed, but raised a warning + * message. + * @KUTF_RESULT_FAIL: The test result failed with a non-fatal error. + * @KUTF_RESULT_FATAL: The test result failed with a fatal error. + * @KUTF_RESULT_ABORT: The test result failed due to a non-UTF + * assertion failure. + * @KUTF_RESULT_USERDATA: User data is ready to be read, + * this is not seen outside the kernel + * @KUTF_RESULT_USERDATA_WAIT: Waiting for user data to be sent, + * this is not seen outside the kernel + * @KUTF_RESULT_TEST_FINISHED: The test has finished, no more results will + * be produced. This is not seen outside kutf + */ +enum kutf_result_status { + KUTF_RESULT_BENCHMARK = -3, + KUTF_RESULT_SKIP = -2, + KUTF_RESULT_UNKNOWN = -1, + + KUTF_RESULT_PASS = 0, + KUTF_RESULT_DEBUG = 1, + KUTF_RESULT_INFO = 2, + KUTF_RESULT_WARN = 3, + KUTF_RESULT_FAIL = 4, + KUTF_RESULT_FATAL = 5, + KUTF_RESULT_ABORT = 6, + + KUTF_RESULT_USERDATA = 7, + KUTF_RESULT_USERDATA_WAIT = 8, + KUTF_RESULT_TEST_FINISHED = 9 +}; + +/* The maximum size of a kutf_result_status result when + * converted to a string + */ +#define KUTF_ERROR_MAX_NAME_SIZE 21 + +#ifdef __KERNEL__ + +#include +#include + +struct kutf_context; + +/** + * struct kutf_result - Represents a single test result. + * @node: Next result in the list of results. + * @status: The status summary (pass / warn / fail / etc). + * @message: A more verbose status message. + */ +struct kutf_result { + struct list_head node; + enum kutf_result_status status; + const char *message; +}; + +/** + * KUTF_RESULT_SET_WAITING_FOR_INPUT - Test is waiting for user data + * + * This flag is set within a struct kutf_result_set whenever the test is blocked + * waiting for user data. Attempts to dequeue results when this flag is set + * will cause a dummy %KUTF_RESULT_USERDATA_WAIT result to be produced. This + * is used to output a warning message and end of file. + */ +#define KUTF_RESULT_SET_WAITING_FOR_INPUT 1 + +/** + * struct kutf_result_set - Represents a set of results. + * @results: List head of a struct kutf_result list for storing the results + * @waitq: Wait queue signalled whenever new results are added. + * @flags: Flags see %KUTF_RESULT_SET_WAITING_FOR_INPUT + */ +struct kutf_result_set { + struct list_head results; + wait_queue_head_t waitq; + int flags; +}; + +/** + * kutf_create_result_set() - Create a new result set + * to which results can be added. + * + * Return: The created result set. + */ +struct kutf_result_set *kutf_create_result_set(void); + +/** + * kutf_add_result() - Add a result to the end of an existing result set. + * + * @context: The kutf context + * @status: The result status to add. + * @message: The result message to add. + * + * Return: 0 if the result is successfully added. -ENOMEM if allocation fails. + */ +int kutf_add_result(struct kutf_context *context, + enum kutf_result_status status, const char *message); + +/** + * kutf_remove_result() - Remove a result from the head of a result set. + * @set: The result set. + * + * This function will block until there is a result to read. The wait is + * interruptible, so this function will return with an ERR_PTR if interrupted. + * + * Return: result or ERR_PTR if interrupted + */ +struct kutf_result *kutf_remove_result( + struct kutf_result_set *set); + +/** + * kutf_destroy_result_set() - Free a previously created result set. + * + * @results: The result set whose resources to free. + */ +void kutf_destroy_result_set(struct kutf_result_set *results); + +/** + * kutf_set_waiting_for_input() - The test is waiting for userdata + * + * @set: The result set to update + * + * Causes the result set to always have results and return a fake + * %KUTF_RESULT_USERDATA_WAIT result. + */ +void kutf_set_waiting_for_input(struct kutf_result_set *set); + +/** + * kutf_clear_waiting_for_input() - The test is no longer waiting for userdata + * + * @set: The result set to update + * + * Cancels the effect of kutf_set_waiting_for_input() + */ +void kutf_clear_waiting_for_input(struct kutf_result_set *set); + +#endif /* __KERNEL__ */ + +#endif /* _KERNEL_UTF_RESULTSET_H_ */ diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_suite.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_suite.h new file mode 100755 index 000000000000..8d75f506f9eb --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_suite.h @@ -0,0 +1,569 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KERNEL_UTF_SUITE_H_ +#define _KERNEL_UTF_SUITE_H_ + +/* kutf_suite.h + * Functions for management of test suites. + * + * This collection of data structures, macros, and functions are used to + * create Test Suites, Tests within those Test Suites, and Fixture variants + * of each test. + */ + +#include +#include +#include + +#include +#include + +/* Arbitrary maximum size to prevent user space allocating too much kernel + * memory + */ +#define KUTF_MAX_LINE_LENGTH (1024u) + +/** + * Pseudo-flag indicating an absence of any specified test class. Note that + * tests should not be annotated with this constant as it is simply a zero + * value; tests without a more specific class must be marked with the flag + * KUTF_F_TEST_GENERIC. + */ +#define KUTF_F_TEST_NONE ((unsigned int)(0)) + +/** + * Class indicating this test is a smoke test. + * A given set of smoke tests should be quick to run, enabling rapid turn-around + * of "regress-on-commit" test runs. + */ +#define KUTF_F_TEST_SMOKETEST ((unsigned int)(1 << 1)) + +/** + * Class indicating this test is a performance test. + * These tests typically produce a performance metric, such as "time to run" or + * "frames per second", + */ +#define KUTF_F_TEST_PERFORMANCE ((unsigned int)(1 << 2)) + +/** + * Class indicating that this test is a deprecated test. + * These tests have typically been replaced by an alternative test which is + * more efficient, or has better coverage. + */ +#define KUTF_F_TEST_DEPRECATED ((unsigned int)(1 << 3)) + +/** + * Class indicating that this test is a known failure. + * These tests have typically been run and failed, but marking them as a known + * failure means it is easier to triage results. + * + * It is typically more convenient to triage known failures using the + * results database and web UI, as this means there is no need to modify the + * test code. + */ +#define KUTF_F_TEST_EXPECTED_FAILURE ((unsigned int)(1 << 4)) + +/** + * Class indicating that this test is a generic test, which is not a member of + * a more specific test class. Tests which are not created with a specific set + * of filter flags by the user are assigned this test class by default. + */ +#define KUTF_F_TEST_GENERIC ((unsigned int)(1 << 5)) + +/** + * Class indicating this test is a resource allocation failure test. + * A resource allocation failure test will test that an error code is + * correctly propagated when an allocation fails. + */ +#define KUTF_F_TEST_RESFAIL ((unsigned int)(1 << 6)) + +/** + * Additional flag indicating that this test is an expected failure when + * run in resource failure mode. These tests are never run when running + * the low resource mode. + */ +#define KUTF_F_TEST_EXPECTED_FAILURE_RF ((unsigned int)(1 << 7)) + +/** + * Flag reserved for user-defined filter zero. + */ +#define KUTF_F_TEST_USER_0 ((unsigned int)(1 << 24)) + +/** + * Flag reserved for user-defined filter one. + */ +#define KUTF_F_TEST_USER_1 ((unsigned int)(1 << 25)) + +/** + * Flag reserved for user-defined filter two. + */ +#define KUTF_F_TEST_USER_2 ((unsigned int)(1 << 26)) + +/** + * Flag reserved for user-defined filter three. + */ +#define KUTF_F_TEST_USER_3 ((unsigned int)(1 << 27)) + +/** + * Flag reserved for user-defined filter four. + */ +#define KUTF_F_TEST_USER_4 ((unsigned int)(1 << 28)) + +/** + * Flag reserved for user-defined filter five. + */ +#define KUTF_F_TEST_USER_5 ((unsigned int)(1 << 29)) + +/** + * Flag reserved for user-defined filter six. + */ +#define KUTF_F_TEST_USER_6 ((unsigned int)(1 << 30)) + +/** + * Flag reserved for user-defined filter seven. + */ +#define KUTF_F_TEST_USER_7 ((unsigned int)(1 << 31)) + +/** + * Pseudo-flag indicating that all test classes should be executed. + */ +#define KUTF_F_TEST_ALL ((unsigned int)(0xFFFFFFFFU)) + +/** + * union kutf_callback_data - Union used to store test callback data + * @ptr_value: pointer to the location where test callback data + * are stored + * @u32_value: a number which represents test callback data + */ +union kutf_callback_data { + void *ptr_value; + u32 u32_value; +}; + +/** + * struct kutf_userdata_line - A line of user data to be returned to the user + * @node: struct list_head to link this into a list + * @str: The line of user data to return to user space + * @size: The number of bytes within @str + */ +struct kutf_userdata_line { + struct list_head node; + char *str; + size_t size; +}; + +/** + * KUTF_USERDATA_WARNING_OUTPUT - Flag specifying that a warning has been output + * + * If user space reads the "run" file while the test is waiting for user data, + * then the framework will output a warning message and set this flag within + * struct kutf_userdata. A subsequent read will then simply return an end of + * file condition rather than outputting the warning again. The upshot of this + * is that simply running 'cat' on a test which requires user data will produce + * the warning followed by 'cat' exiting due to EOF - which is much more user + * friendly than blocking indefinitely waiting for user data. + */ +#define KUTF_USERDATA_WARNING_OUTPUT 1 + +/** + * struct kutf_userdata - Structure holding user data + * @flags: See %KUTF_USERDATA_WARNING_OUTPUT + * @input_head: List of struct kutf_userdata_line containing user data + * to be read by the kernel space test. + * @input_waitq: Wait queue signalled when there is new user data to be + * read by the kernel space test. + */ +struct kutf_userdata { + unsigned long flags; + struct list_head input_head; + wait_queue_head_t input_waitq; +}; + +/** + * struct kutf_context - Structure representing a kernel test context + * @kref: Refcount for number of users of this context + * @suite: Convenience pointer to the suite this context + * is running + * @test_fix: The fixture that is being run in this context + * @fixture_pool: The memory pool used for the duration of + * the fixture/text context. + * @fixture: The user provided fixture structure. + * @fixture_index: The index (id) of the current fixture. + * @fixture_name: The name of the current fixture (or NULL if unnamed). + * @test_data: Any user private data associated with this test + * @result_set: All the results logged by this test context + * @status: The status of the currently running fixture. + * @expected_status: The expected status on exist of the currently + * running fixture. + * @work: Work item to enqueue onto the work queue to run the test + * @userdata: Structure containing the user data for the test to read + */ +struct kutf_context { + struct kref kref; + struct kutf_suite *suite; + struct kutf_test_fixture *test_fix; + struct kutf_mempool fixture_pool; + void *fixture; + unsigned int fixture_index; + const char *fixture_name; + union kutf_callback_data test_data; + struct kutf_result_set *result_set; + enum kutf_result_status status; + enum kutf_result_status expected_status; + + struct work_struct work; + struct kutf_userdata userdata; +}; + +/** + * struct kutf_suite - Structure representing a kernel test suite + * @app: The application this suite belongs to. + * @name: The name of this suite. + * @suite_data: Any user private data associated with this + * suite. + * @create_fixture: Function used to create a new fixture instance + * @remove_fixture: Function used to destroy a new fixture instance + * @fixture_variants: The number of variants (must be at least 1). + * @suite_default_flags: Suite global filter flags which are set on + * all tests. + * @node: List node for suite_list + * @dir: The debugfs directory for this suite + * @test_list: List head to store all the tests which are + * part of this suite + */ +struct kutf_suite { + struct kutf_application *app; + const char *name; + union kutf_callback_data suite_data; + void *(*create_fixture)(struct kutf_context *context); + void (*remove_fixture)(struct kutf_context *context); + unsigned int fixture_variants; + unsigned int suite_default_flags; + struct list_head node; + struct dentry *dir; + struct list_head test_list; +}; + +/* ============================================================================ + Application functions +============================================================================ */ + +/** + * kutf_create_application() - Create an in kernel test application. + * @name: The name of the test application. + * + * Return: pointer to the kutf_application on success or NULL + * on failure + */ +struct kutf_application *kutf_create_application(const char *name); + +/** + * kutf_destroy_application() - Destroy an in kernel test application. + * + * @app: The test application to destroy. + */ +void kutf_destroy_application(struct kutf_application *app); + +/* ============================================================================ + Suite functions +============================================================================ */ + +/** + * kutf_create_suite() - Create a kernel test suite. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * + * Suite names must be unique. Should two suites with the same name be + * registered with the same application then this function will fail, if they + * are registered with different applications then the function will not detect + * this and the call will succeed. + * + * Return: pointer to the created kutf_suite on success or NULL + * on failure + */ +struct kutf_suite *kutf_create_suite( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context)); + +/** + * kutf_create_suite_with_filters() - Create a kernel test suite with user + * defined default filters. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * @filters: Filters to apply to a test if it doesn't provide its own + * + * Suite names must be unique. Should two suites with the same name be + * registered with the same application then this function will fail, if they + * are registered with different applications then the function will not detect + * this and the call will succeed. + * + * Return: pointer to the created kutf_suite on success or NULL on failure + */ +struct kutf_suite *kutf_create_suite_with_filters( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters); + +/** + * kutf_create_suite_with_filters_and_data() - Create a kernel test suite with + * user defined default filters. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * @filters: Filters to apply to a test if it doesn't provide its own + * @suite_data: Suite specific callback data, provided during the + * running of the test in the kutf_context + * + * Return: pointer to the created kutf_suite on success or NULL + * on failure + */ +struct kutf_suite *kutf_create_suite_with_filters_and_data( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data suite_data); + +/** + * kutf_add_test() - Add a test to a kernel test suite. + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * + * Note: As no filters are provided the test will use the suite filters instead + */ +void kutf_add_test(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context)); + +/** + * kutf_add_test_with_filters() - Add a test to a kernel test suite with filters + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * @filters: A set of filtering flags, assigning test categories. + */ +void kutf_add_test_with_filters(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters); + +/** + * kutf_add_test_with_filters_and_data() - Add a test to a kernel test suite + * with filters. + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * @filters: A set of filtering flags, assigning test categories. + * @test_data: Test specific callback data, provided during the + * running of the test in the kutf_context + */ +void kutf_add_test_with_filters_and_data( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data test_data); + + +/* ============================================================================ + Test functions +============================================================================ */ +/** + * kutf_test_log_result_external() - Log a result which has been created + * externally into a in a standard form + * recognized by the log parser. + * @context: The test context the test is running in + * @message: The message for this result + * @new_status: The result status of this log message + */ +void kutf_test_log_result_external( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status); + +/** + * kutf_test_expect_abort() - Tell the kernel that you expect the current + * fixture to produce an abort. + * @context: The test context this test is running in. + */ +void kutf_test_expect_abort(struct kutf_context *context); + +/** + * kutf_test_expect_fatal() - Tell the kernel that you expect the current + * fixture to produce a fatal error. + * @context: The test context this test is running in. + */ +void kutf_test_expect_fatal(struct kutf_context *context); + +/** + * kutf_test_expect_fail() - Tell the kernel that you expect the current + * fixture to fail. + * @context: The test context this test is running in. + */ +void kutf_test_expect_fail(struct kutf_context *context); + +/** + * kutf_test_expect_warn() - Tell the kernel that you expect the current + * fixture to produce a warning. + * @context: The test context this test is running in. + */ +void kutf_test_expect_warn(struct kutf_context *context); + +/** + * kutf_test_expect_pass() - Tell the kernel that you expect the current + * fixture to pass. + * @context: The test context this test is running in. + */ +void kutf_test_expect_pass(struct kutf_context *context); + +/** + * kutf_test_skip() - Tell the kernel that the test should be skipped. + * @context: The test context this test is running in. + */ +void kutf_test_skip(struct kutf_context *context); + +/** + * kutf_test_skip_msg() - Tell the kernel that this test has been skipped, + * supplying a reason string. + * @context: The test context this test is running in. + * @message: A message string containing the reason for the skip. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a prebaked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_skip_msg(struct kutf_context *context, const char *message); + +/** + * kutf_test_pass() - Tell the kernel that this test has passed. + * @context: The test context this test is running in. + * @message: A message string containing the reason for the pass. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_pass(struct kutf_context *context, char const *message); + +/** + * kutf_test_debug() - Send a debug message + * @context: The test context this test is running in. + * @message: A message string containing the debug information. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_debug(struct kutf_context *context, char const *message); + +/** + * kutf_test_info() - Send an information message + * @context: The test context this test is running in. + * @message: A message string containing the information message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_info(struct kutf_context *context, char const *message); + +/** + * kutf_test_warn() - Send a warning message + * @context: The test context this test is running in. + * @message: A message string containing the warning message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_warn(struct kutf_context *context, char const *message); + +/** + * kutf_test_fail() - Tell the kernel that a test has failed + * @context: The test context this test is running in. + * @message: A message string containing the failure message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_fail(struct kutf_context *context, char const *message); + +/** + * kutf_test_fatal() - Tell the kernel that a test has triggered a fatal error + * @context: The test context this test is running in. + * @message: A message string containing the fatal error message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_fatal(struct kutf_context *context, char const *message); + +/** + * kutf_test_abort() - Tell the kernel that a test triggered an abort in the test + * + * @context: The test context this test is running in. + */ +void kutf_test_abort(struct kutf_context *context); + +#endif /* _KERNEL_UTF_SUITE_H_ */ diff --git a/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_utils.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_utils.h new file mode 100755 index 000000000000..25b8285500d7 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_utils.h @@ -0,0 +1,60 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KERNEL_UTF_UTILS_H_ +#define _KERNEL_UTF_UTILS_H_ + +/* kutf_utils.h + * Utilities for the kernel UTF test infrastructure. + * + * This collection of library functions are provided for use by kernel UTF + * and users of kernel UTF which don't directly fit within the other + * code modules. + */ + +#include + +/** + * Maximum size of the message strings within kernel UTF, messages longer then + * this will be truncated. + */ +#define KUTF_MAX_DSPRINTF_LEN 1024 + +/** + * kutf_dsprintf() - dynamic sprintf + * @pool: memory pool to allocate from + * @fmt: The format string describing the string to document. + * @... The parameters to feed in to the format string. + * + * This function implements sprintf which dynamically allocates memory to store + * the string. The library will free the memory containing the string when the + * result set is cleared or destroyed. + * + * Note The returned string may be truncated to fit an internal temporary + * buffer, which is KUTF_MAX_DSPRINTF_LEN bytes in length. + * + * Return: Returns pointer to allocated string, or NULL on error. + */ +const char *kutf_dsprintf(struct kutf_mempool *pool, + const char *fmt, ...); + +#endif /* _KERNEL_UTF_UTILS_H_ */ diff --git a/drivers/gpu/arm/bifrost/tests/kutf/Kbuild b/drivers/gpu/arm/bifrost/tests/kutf/Kbuild new file mode 100755 index 000000000000..2531d41ca28d --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/Kbuild @@ -0,0 +1,26 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +ccflags-y += -I$(src)/../include + +obj-$(CONFIG_MALI_KUTF) += kutf.o + +kutf-y := kutf_mem.o kutf_resultset.o kutf_suite.o kutf_utils.o kutf_helpers.o kutf_helpers_user.o diff --git a/drivers/gpu/arm/bifrost/tests/kutf/Kconfig b/drivers/gpu/arm/bifrost/tests/kutf/Kconfig new file mode 100755 index 000000000000..0cdb474c06a3 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/Kconfig @@ -0,0 +1,28 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + + +config MALI_KUTF + tristate "Mali Kernel Unit Test Framework" + default m + help + Enables MALI testing framework. To compile it as a module, + choose M here - this will generate a single module called kutf. diff --git a/drivers/gpu/arm/bifrost/tests/kutf/Makefile b/drivers/gpu/arm/bifrost/tests/kutf/Makefile new file mode 100755 index 000000000000..d848e8774bd0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/Makefile @@ -0,0 +1,35 @@ +# +# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +# linux build system bootstrap for out-of-tree module + +# default to building for the host +ARCH ?= $(shell uname -m) + +ifeq ($(KDIR),) +$(error Must specify KDIR to point to the kernel to target)) +endif + +all: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS=-I$(CURDIR)/../include modules + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean diff --git a/drivers/gpu/arm/bifrost/tests/kutf/build.bp b/drivers/gpu/arm/bifrost/tests/kutf/build.bp new file mode 100755 index 000000000000..32eab143e669 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/build.bp @@ -0,0 +1,36 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + +bob_kernel_module { + name: "kutf", + defaults: [ + "kernel_defaults", + "kutf_includes", + ], + srcs: [ + "Kbuild", + "kutf_helpers.c", + "kutf_helpers_user.c", + "kutf_mem.c", + "kutf_resultset.c", + "kutf_suite.c", + "kutf_utils.c", + ], + kbuild_options: ["CONFIG_MALI_KUTF=m"], + enabled: false, + base_build_kutf: { + enabled: true, + }, +} diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers.c new file mode 100755 index 000000000000..4463b04792f5 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers.c @@ -0,0 +1,131 @@ +/* + * + * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* Kernel UTF test helpers */ +#include + +#include +#include +#include +#include +#include +#include +#include + +static DEFINE_SPINLOCK(kutf_input_lock); + +bool kutf_helper_pending_input(struct kutf_context *context) +{ + bool input_pending; + + spin_lock(&kutf_input_lock); + + input_pending = !list_empty(&context->userdata.input_head); + + spin_unlock(&kutf_input_lock); + + return input_pending; +} +EXPORT_SYMBOL(kutf_helper_pending_input); + +char *kutf_helper_input_dequeue(struct kutf_context *context, size_t *str_size) +{ + struct kutf_userdata_line *line; + + spin_lock(&kutf_input_lock); + + while (list_empty(&context->userdata.input_head)) { + int err; + + kutf_set_waiting_for_input(context->result_set); + + spin_unlock(&kutf_input_lock); + + err = wait_event_interruptible(context->userdata.input_waitq, + kutf_helper_pending_input(context)); + + if (err) + return ERR_PTR(-EINTR); + + spin_lock(&kutf_input_lock); + } + + line = list_first_entry(&context->userdata.input_head, + struct kutf_userdata_line, node); + if (line->str) { + /* + * Unless it is the end-of-input marker, + * remove it from the list + */ + list_del(&line->node); + } + + spin_unlock(&kutf_input_lock); + + if (str_size) + *str_size = line->size; + return line->str; +} + +int kutf_helper_input_enqueue(struct kutf_context *context, + const char __user *str, size_t size) +{ + struct kutf_userdata_line *line; + + line = kutf_mempool_alloc(&context->fixture_pool, + sizeof(*line) + size + 1); + if (!line) + return -ENOMEM; + if (str) { + unsigned long bytes_not_copied; + + line->size = size; + line->str = (void *)(line + 1); + bytes_not_copied = copy_from_user(line->str, str, size); + if (bytes_not_copied != 0) + return -EFAULT; + /* Zero terminate the string */ + line->str[size] = '\0'; + } else { + /* This is used to mark the end of input */ + WARN_ON(size); + line->size = 0; + line->str = NULL; + } + + spin_lock(&kutf_input_lock); + + list_add_tail(&line->node, &context->userdata.input_head); + + kutf_clear_waiting_for_input(context->result_set); + + spin_unlock(&kutf_input_lock); + + wake_up(&context->userdata.input_waitq); + + return 0; +} + +void kutf_helper_input_enqueue_end_of_data(struct kutf_context *context) +{ + kutf_helper_input_enqueue(context, NULL, 0); +} diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c new file mode 100755 index 000000000000..108fa82d9b21 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c @@ -0,0 +1,468 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* Kernel UTF test helpers that mirror those for kutf-userside */ +#include +#include +#include + +#include +#include +#include + +const char *valtype_names[] = { + "INVALID", + "U64", + "STR", +}; + +static const char *get_val_type_name(enum kutf_helper_valtype valtype) +{ + /* enums can be signed or unsigned (implementation dependant), so + * enforce it to prevent: + * a) "<0 comparison on unsigned type" warning - if we did both upper + * and lower bound check + * b) incorrect range checking if it was a signed type - if we did + * upper bound check only */ + unsigned int type_idx = (unsigned int)valtype; + + if (type_idx >= (unsigned int)KUTF_HELPER_VALTYPE_COUNT) + type_idx = (unsigned int)KUTF_HELPER_VALTYPE_INVALID; + + return valtype_names[type_idx]; +} + +/* Check up to str_len chars of val_str to see if it's a valid value name: + * + * - Has between 1 and KUTF_HELPER_MAX_VAL_NAME_LEN characters before the \0 terminator + * - And, each char is in the character set [A-Z0-9_] */ +static int validate_val_name(const char *val_str, int str_len) +{ + int i = 0; + + for (i = 0; str_len && i <= KUTF_HELPER_MAX_VAL_NAME_LEN && val_str[i] != '\0'; ++i, --str_len) { + char val_chr = val_str[i]; + + if (val_chr >= 'A' && val_chr <= 'Z') + continue; + if (val_chr >= '0' && val_chr <= '9') + continue; + if (val_chr == '_') + continue; + + /* Character not in the set [A-Z0-9_] - report error */ + return 1; + } + + /* Names of 0 length are not valid */ + if (i == 0) + return 1; + /* Length greater than KUTF_HELPER_MAX_VAL_NAME_LEN not allowed */ + if (i > KUTF_HELPER_MAX_VAL_NAME_LEN || (i == KUTF_HELPER_MAX_VAL_NAME_LEN && val_str[i] != '\0')) + return 1; + + return 0; +} + +/* Find the length of the valid part of the string when it will be in quotes + * e.g. "str" + * + * That is, before any '\\', '\n' or '"' characters. This is so we don't have + * to escape the string */ +static int find_quoted_string_valid_len(const char *str) +{ + char *ptr; + const char *check_chars = "\\\n\""; + + ptr = strpbrk(str, check_chars); + if (ptr) + return (int)(ptr-str); + + return (int)strlen(str); +} + +static int kutf_helper_userdata_enqueue(struct kutf_context *context, + const char *str) +{ + char *str_copy; + size_t len; + int err; + + len = strlen(str)+1; + + str_copy = kutf_mempool_alloc(&context->fixture_pool, len); + if (!str_copy) + return -ENOMEM; + + strcpy(str_copy, str); + + err = kutf_add_result(context, KUTF_RESULT_USERDATA, str_copy); + + return err; +} + +#define MAX_U64_HEX_LEN 16 +/* (Name size) + ("=0x" size) + (64-bit hex value size) + (terminator) */ +#define NAMED_U64_VAL_BUF_SZ (KUTF_HELPER_MAX_VAL_NAME_LEN + 3 + MAX_U64_HEX_LEN + 1) + +int kutf_helper_send_named_u64(struct kutf_context *context, + const char *val_name, u64 val) +{ + int ret = 1; + char msgbuf[NAMED_U64_VAL_BUF_SZ]; + const char *errmsg = NULL; + + if (validate_val_name(val_name, KUTF_HELPER_MAX_VAL_NAME_LEN + 1)) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send u64 value named '%s': Invalid value name", val_name); + goto out_err; + } + + ret = snprintf(msgbuf, NAMED_U64_VAL_BUF_SZ, "%s=0x%llx", val_name, val); + if (ret >= NAMED_U64_VAL_BUF_SZ || ret < 0) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send u64 value named '%s': snprintf() problem buffer size==%d ret=%d", + val_name, NAMED_U64_VAL_BUF_SZ, ret); + goto out_err; + } + + ret = kutf_helper_userdata_enqueue(context, msgbuf); + if (ret) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send u64 value named '%s': send returned %d", + val_name, ret); + goto out_err; + } + + return ret; +out_err: + kutf_test_fail(context, errmsg); + return ret; +} +EXPORT_SYMBOL(kutf_helper_send_named_u64); + +#define NAMED_VALUE_SEP "=" +#define NAMED_STR_START_DELIM NAMED_VALUE_SEP "\"" +#define NAMED_STR_END_DELIM "\"" + +int kutf_helper_max_str_len_for_kern(const char *val_name, + int kern_buf_sz) +{ + const int val_name_len = strlen(val_name); + const int start_delim_len = strlen(NAMED_STR_START_DELIM); + const int end_delim_len = strlen(NAMED_STR_END_DELIM); + int max_msg_len = kern_buf_sz; + int max_str_len; + + max_str_len = max_msg_len - val_name_len - start_delim_len - + end_delim_len; + + return max_str_len; +} +EXPORT_SYMBOL(kutf_helper_max_str_len_for_kern); + +int kutf_helper_send_named_str(struct kutf_context *context, + const char *val_name, + const char *val_str) +{ + int val_str_len; + int str_buf_sz; + char *str_buf = NULL; + int ret = 1; + char *copy_ptr; + int val_name_len; + int start_delim_len = strlen(NAMED_STR_START_DELIM); + int end_delim_len = strlen(NAMED_STR_END_DELIM); + const char *errmsg = NULL; + + if (validate_val_name(val_name, KUTF_HELPER_MAX_VAL_NAME_LEN + 1)) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send u64 value named '%s': Invalid value name", val_name); + goto out_err; + } + val_name_len = strlen(val_name); + + val_str_len = find_quoted_string_valid_len(val_str); + + /* (name length) + ("=\"" length) + (val_str len) + ("\"" length) + terminator */ + str_buf_sz = val_name_len + start_delim_len + val_str_len + end_delim_len + 1; + + /* Using kmalloc() here instead of mempool since we know we need to free + * before we return */ + str_buf = kmalloc(str_buf_sz, GFP_KERNEL); + if (!str_buf) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send str value named '%s': kmalloc failed, str_buf_sz=%d", + val_name, str_buf_sz); + goto out_err; + } + copy_ptr = str_buf; + + /* Manually copy each string component instead of snprintf because + * val_str may need to end early, and less error path handling */ + + /* name */ + memcpy(copy_ptr, val_name, val_name_len); + copy_ptr += val_name_len; + + /* str start delimiter */ + memcpy(copy_ptr, NAMED_STR_START_DELIM, start_delim_len); + copy_ptr += start_delim_len; + + /* str value */ + memcpy(copy_ptr, val_str, val_str_len); + copy_ptr += val_str_len; + + /* str end delimiter */ + memcpy(copy_ptr, NAMED_STR_END_DELIM, end_delim_len); + copy_ptr += end_delim_len; + + /* Terminator */ + *copy_ptr = '\0'; + + ret = kutf_helper_userdata_enqueue(context, str_buf); + + if (ret) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send str value named '%s': send returned %d", + val_name, ret); + goto out_err; + } + + kfree(str_buf); + return ret; + +out_err: + kutf_test_fail(context, errmsg); + kfree(str_buf); + return ret; +} +EXPORT_SYMBOL(kutf_helper_send_named_str); + +int kutf_helper_receive_named_val( + struct kutf_context *context, + struct kutf_helper_named_val *named_val) +{ + size_t recv_sz; + char *recv_str; + char *search_ptr; + char *name_str = NULL; + int name_len; + int strval_len; + enum kutf_helper_valtype type = KUTF_HELPER_VALTYPE_INVALID; + char *strval = NULL; + u64 u64val = 0; + int err = KUTF_HELPER_ERR_INVALID_VALUE; + + recv_str = kutf_helper_input_dequeue(context, &recv_sz); + if (!recv_str) + return -EBUSY; + else if (IS_ERR(recv_str)) + return PTR_ERR(recv_str); + + /* Find the '=', grab the name and validate it */ + search_ptr = strnchr(recv_str, recv_sz, NAMED_VALUE_SEP[0]); + if (search_ptr) { + name_len = search_ptr - recv_str; + if (!validate_val_name(recv_str, name_len)) { + /* no need to reallocate - just modify string in place */ + name_str = recv_str; + name_str[name_len] = '\0'; + + /* Move until after the '=' */ + recv_str += (name_len + 1); + recv_sz -= (name_len + 1); + } + } + if (!name_str) { + pr_err("Invalid name part for received string '%s'\n", + recv_str); + return KUTF_HELPER_ERR_INVALID_NAME; + } + + /* detect value type */ + if (*recv_str == NAMED_STR_START_DELIM[1]) { + /* string delimiter start*/ + ++recv_str; + --recv_sz; + + /* Find end of string */ + search_ptr = strnchr(recv_str, recv_sz, NAMED_STR_END_DELIM[0]); + if (search_ptr) { + strval_len = search_ptr - recv_str; + /* Validate the string to ensure it contains no quotes */ + if (strval_len == find_quoted_string_valid_len(recv_str)) { + /* no need to reallocate - just modify string in place */ + strval = recv_str; + strval[strval_len] = '\0'; + + /* Move until after the end delimiter */ + recv_str += (strval_len + 1); + recv_sz -= (strval_len + 1); + type = KUTF_HELPER_VALTYPE_STR; + } else { + pr_err("String value contains invalid characters in rest of received string '%s'\n", recv_str); + err = KUTF_HELPER_ERR_CHARS_AFTER_VAL; + } + } else { + pr_err("End of string delimiter not found in rest of received string '%s'\n", recv_str); + err = KUTF_HELPER_ERR_NO_END_DELIMITER; + } + } else { + /* possibly a number value - strtoull will parse it */ + err = kstrtoull(recv_str, 0, &u64val); + /* unlike userspace can't get an end ptr, but if kstrtoull() + * reads characters after the number it'll report -EINVAL */ + if (!err) { + int len_remain = strnlen(recv_str, recv_sz); + + type = KUTF_HELPER_VALTYPE_U64; + recv_str += len_remain; + recv_sz -= len_remain; + } else { + /* special case: not a number, report as such */ + pr_err("Rest of received string was not a numeric value or quoted string value: '%s'\n", recv_str); + } + } + + if (type == KUTF_HELPER_VALTYPE_INVALID) + return err; + + /* Any remaining characters - error */ + if (strnlen(recv_str, recv_sz) != 0) { + pr_err("Characters remain after value of type %s: '%s'\n", + get_val_type_name(type), recv_str); + return KUTF_HELPER_ERR_CHARS_AFTER_VAL; + } + + /* Success - write into the output structure */ + switch (type) { + case KUTF_HELPER_VALTYPE_U64: + named_val->u.val_u64 = u64val; + break; + case KUTF_HELPER_VALTYPE_STR: + named_val->u.val_str = strval; + break; + default: + pr_err("Unreachable, fix kutf_helper_receive_named_val\n"); + /* Coding error, report as though 'run' file failed */ + return -EINVAL; + } + + named_val->val_name = name_str; + named_val->type = type; + + return KUTF_HELPER_ERR_NONE; +} +EXPORT_SYMBOL(kutf_helper_receive_named_val); + +#define DUMMY_MSG "" +int kutf_helper_receive_check_val( + struct kutf_helper_named_val *named_val, + struct kutf_context *context, + const char *expect_val_name, + enum kutf_helper_valtype expect_val_type) +{ + int err; + + err = kutf_helper_receive_named_val(context, named_val); + if (err < 0) { + const char *msg = kutf_dsprintf(&context->fixture_pool, + "Failed to receive value named '%s'", + expect_val_name); + kutf_test_fail(context, msg); + return err; + } else if (err > 0) { + const char *msg = kutf_dsprintf(&context->fixture_pool, + "Named-value parse error when expecting value named '%s'", + expect_val_name); + kutf_test_fail(context, msg); + goto out_fail_and_fixup; + } + + if (strcmp(named_val->val_name, expect_val_name) != 0) { + const char *msg = kutf_dsprintf(&context->fixture_pool, + "Expecting to receive value named '%s' but got '%s'", + expect_val_name, named_val->val_name); + kutf_test_fail(context, msg); + goto out_fail_and_fixup; + } + + + if (named_val->type != expect_val_type) { + const char *msg = kutf_dsprintf(&context->fixture_pool, + "Expecting value named '%s' to be of type %s but got %s", + expect_val_name, get_val_type_name(expect_val_type), + get_val_type_name(named_val->type)); + kutf_test_fail(context, msg); + goto out_fail_and_fixup; + } + + return err; + +out_fail_and_fixup: + /* Produce a valid but incorrect value */ + switch (expect_val_type) { + case KUTF_HELPER_VALTYPE_U64: + named_val->u.val_u64 = 0ull; + break; + case KUTF_HELPER_VALTYPE_STR: + { + char *str = kutf_mempool_alloc(&context->fixture_pool, sizeof(DUMMY_MSG)); + + if (!str) + return -1; + + strcpy(str, DUMMY_MSG); + named_val->u.val_str = str; + break; + } + default: + break; + } + + /* Indicate that this is invalid */ + named_val->type = KUTF_HELPER_VALTYPE_INVALID; + + /* But at least allow the caller to continue in the test with failures */ + return 0; +} +EXPORT_SYMBOL(kutf_helper_receive_check_val); + +void kutf_helper_output_named_val(struct kutf_helper_named_val *named_val) +{ + switch (named_val->type) { + case KUTF_HELPER_VALTYPE_U64: + pr_warn("%s=0x%llx\n", named_val->val_name, named_val->u.val_u64); + break; + case KUTF_HELPER_VALTYPE_STR: + pr_warn("%s=\"%s\"\n", named_val->val_name, named_val->u.val_str); + break; + case KUTF_HELPER_VALTYPE_INVALID: + pr_warn("%s is invalid\n", named_val->val_name); + break; + default: + pr_warn("%s has unknown type %d\n", named_val->val_name, named_val->type); + break; + } +} +EXPORT_SYMBOL(kutf_helper_output_named_val); diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_mem.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_mem.c new file mode 100755 index 000000000000..fd98beaeb84a --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_mem.c @@ -0,0 +1,108 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* Kernel UTF memory management functions */ + +#include +#include +#include + +#include + + +/** + * struct kutf_alloc_entry - Structure representing an allocation. + * @node: List node for use with kutf_mempool. + * @data: Data area of the allocation + */ +struct kutf_alloc_entry { + struct list_head node; + u8 data[0]; +}; + +int kutf_mempool_init(struct kutf_mempool *pool) +{ + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + return -1; + } + + INIT_LIST_HEAD(&pool->head); + mutex_init(&pool->lock); + + return 0; +} +EXPORT_SYMBOL(kutf_mempool_init); + +void kutf_mempool_destroy(struct kutf_mempool *pool) +{ + struct list_head *remove; + struct list_head *tmp; + + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + return; + } + + mutex_lock(&pool->lock); + list_for_each_safe(remove, tmp, &pool->head) { + struct kutf_alloc_entry *remove_alloc; + + remove_alloc = list_entry(remove, struct kutf_alloc_entry, node); + list_del(&remove_alloc->node); + kfree(remove_alloc); + } + mutex_unlock(&pool->lock); + +} +EXPORT_SYMBOL(kutf_mempool_destroy); + +void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size) +{ + struct kutf_alloc_entry *ret; + + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + goto fail_pool; + } + + mutex_lock(&pool->lock); + + ret = kmalloc(sizeof(*ret) + size, GFP_KERNEL); + if (!ret) { + pr_err("Failed to allocate memory\n"); + goto fail_alloc; + } + + INIT_LIST_HEAD(&ret->node); + list_add(&ret->node, &pool->head); + + mutex_unlock(&pool->lock); + + return &ret->data[0]; + +fail_alloc: + mutex_unlock(&pool->lock); +fail_pool: + return NULL; +} +EXPORT_SYMBOL(kutf_mempool_alloc); diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_resultset.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_resultset.c new file mode 100755 index 000000000000..94ecfa4421e1 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_resultset.c @@ -0,0 +1,164 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* Kernel UTF result management functions */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +/* Lock to protect all result structures */ +static DEFINE_SPINLOCK(kutf_result_lock); + +struct kutf_result_set *kutf_create_result_set(void) +{ + struct kutf_result_set *set; + + set = kmalloc(sizeof(*set), GFP_KERNEL); + if (!set) { + pr_err("Failed to allocate resultset"); + goto fail_alloc; + } + + INIT_LIST_HEAD(&set->results); + init_waitqueue_head(&set->waitq); + set->flags = 0; + + return set; + +fail_alloc: + return NULL; +} + +int kutf_add_result(struct kutf_context *context, + enum kutf_result_status status, + const char *message) +{ + struct kutf_mempool *mempool = &context->fixture_pool; + struct kutf_result_set *set = context->result_set; + /* Create the new result */ + struct kutf_result *new_result; + + BUG_ON(set == NULL); + + new_result = kutf_mempool_alloc(mempool, sizeof(*new_result)); + if (!new_result) { + pr_err("Result allocation failed\n"); + return -ENOMEM; + } + + INIT_LIST_HEAD(&new_result->node); + new_result->status = status; + new_result->message = message; + + spin_lock(&kutf_result_lock); + + list_add_tail(&new_result->node, &set->results); + + spin_unlock(&kutf_result_lock); + + wake_up(&set->waitq); + + return 0; +} + +void kutf_destroy_result_set(struct kutf_result_set *set) +{ + if (!list_empty(&set->results)) + pr_err("kutf_destroy_result_set: Unread results from test\n"); + + kfree(set); +} + +static bool kutf_has_result(struct kutf_result_set *set) +{ + bool has_result; + + spin_lock(&kutf_result_lock); + if (set->flags & KUTF_RESULT_SET_WAITING_FOR_INPUT) + /* Pretend there are results if waiting for input */ + has_result = true; + else + has_result = !list_empty(&set->results); + spin_unlock(&kutf_result_lock); + + return has_result; +} + +struct kutf_result *kutf_remove_result(struct kutf_result_set *set) +{ + struct kutf_result *result = NULL; + int ret; + + do { + ret = wait_event_interruptible(set->waitq, + kutf_has_result(set)); + + if (ret) + return ERR_PTR(ret); + + spin_lock(&kutf_result_lock); + + if (!list_empty(&set->results)) { + result = list_first_entry(&set->results, + struct kutf_result, + node); + list_del(&result->node); + } else if (set->flags & KUTF_RESULT_SET_WAITING_FOR_INPUT) { + /* Return a fake result */ + static struct kutf_result waiting = { + .status = KUTF_RESULT_USERDATA_WAIT + }; + result = &waiting; + } + /* If result == NULL then there was a race with the event + * being removed between the check in kutf_has_result and + * the lock being obtained. In this case we retry + */ + + spin_unlock(&kutf_result_lock); + } while (result == NULL); + + return result; +} + +void kutf_set_waiting_for_input(struct kutf_result_set *set) +{ + spin_lock(&kutf_result_lock); + set->flags |= KUTF_RESULT_SET_WAITING_FOR_INPUT; + spin_unlock(&kutf_result_lock); + + wake_up(&set->waitq); +} + +void kutf_clear_waiting_for_input(struct kutf_result_set *set) +{ + spin_lock(&kutf_result_lock); + set->flags &= ~KUTF_RESULT_SET_WAITING_FOR_INPUT; + spin_unlock(&kutf_result_lock); +} diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c new file mode 100755 index 000000000000..9dc6e2b4bad4 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c @@ -0,0 +1,1224 @@ +/* + * + * (C) COPYRIGHT 2014, 2017-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* Kernel UTF suite, test and fixture management including user to kernel + * interaction */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +/** + * struct kutf_application - Structure which represents kutf application + * @name: The name of this test application. + * @dir: The debugfs directory for this test + * @suite_list: List head to store all the suites which are part of this + * application + */ +struct kutf_application { + const char *name; + struct dentry *dir; + struct list_head suite_list; +}; + +/** + * struct kutf_test_function - Structure which represents kutf test function + * @suite: Back reference to the suite this test function + * belongs to + * @filters: Filters that apply to this test function + * @test_id: Test ID + * @execute: Function to run for this test + * @test_data: Static data for this test + * @node: List node for test_list + * @variant_list: List head to store all the variants which can run on + * this function + * @dir: debugfs directory for this test function + */ +struct kutf_test_function { + struct kutf_suite *suite; + unsigned int filters; + unsigned int test_id; + void (*execute)(struct kutf_context *context); + union kutf_callback_data test_data; + struct list_head node; + struct list_head variant_list; + struct dentry *dir; +}; + +/** + * struct kutf_test_fixture - Structure which holds information on the kutf + * test fixture + * @test_func: Test function this fixture belongs to + * @fixture_index: Index of this fixture + * @node: List node for variant_list + * @dir: debugfs directory for this test fixture + */ +struct kutf_test_fixture { + struct kutf_test_function *test_func; + unsigned int fixture_index; + struct list_head node; + struct dentry *dir; +}; + +static struct dentry *base_dir; +static struct workqueue_struct *kutf_workq; + +/** + * struct kutf_convert_table - Structure which keeps test results + * @result_name: Status of the test result + * @result: Status value for a single test + */ +struct kutf_convert_table { + char result_name[50]; + enum kutf_result_status result; +}; + +struct kutf_convert_table kutf_convert[] = { +#define ADD_UTF_RESULT(_name) \ +{ \ + #_name, \ + _name, \ +}, +ADD_UTF_RESULT(KUTF_RESULT_BENCHMARK) +ADD_UTF_RESULT(KUTF_RESULT_SKIP) +ADD_UTF_RESULT(KUTF_RESULT_UNKNOWN) +ADD_UTF_RESULT(KUTF_RESULT_PASS) +ADD_UTF_RESULT(KUTF_RESULT_DEBUG) +ADD_UTF_RESULT(KUTF_RESULT_INFO) +ADD_UTF_RESULT(KUTF_RESULT_WARN) +ADD_UTF_RESULT(KUTF_RESULT_FAIL) +ADD_UTF_RESULT(KUTF_RESULT_FATAL) +ADD_UTF_RESULT(KUTF_RESULT_ABORT) +}; + +#define UTF_CONVERT_SIZE (ARRAY_SIZE(kutf_convert)) + +/** + * kutf_create_context() - Create a test context in which a specific fixture + * of an application will be run and its results + * reported back to the user + * @test_fix: Test fixture to be run. + * + * The context's refcount will be initialized to 1. + * + * Return: Returns the created test context on success or NULL on failure + */ +static struct kutf_context *kutf_create_context( + struct kutf_test_fixture *test_fix); + +/** + * kutf_destroy_context() - Destroy a previously created test context, only + * once its refcount has become zero + * @kref: pointer to kref member within the context + * + * This should only be used via a kref_put() call on the context's kref member + */ +static void kutf_destroy_context(struct kref *kref); + +/** + * kutf_context_get() - increment refcount on a context + * @context: the kutf context + * + * This must be used when the lifetime of the context might exceed that of the + * thread creating @context + */ +static void kutf_context_get(struct kutf_context *context); + +/** + * kutf_context_put() - decrement refcount on a context, destroying it when it + * reached zero + * @context: the kutf context + * + * This must be used only after a corresponding kutf_context_get() call on + * @context, and the caller no longer needs access to @context. + */ +static void kutf_context_put(struct kutf_context *context); + +/** + * kutf_set_result() - Set the test result against the specified test context + * @context: Test context + * @status: Result status + */ +static void kutf_set_result(struct kutf_context *context, + enum kutf_result_status status); + +/** + * kutf_set_expected_result() - Set the expected test result for the specified + * test context + * @context: Test context + * @expected_status: Expected result status + */ +static void kutf_set_expected_result(struct kutf_context *context, + enum kutf_result_status expected_status); + +/** + * kutf_result_to_string() - Converts a KUTF result into a string + * @result_str: Output result string + * @result: Result status to convert + * + * Return: 1 if test result was successfully converted to string, 0 otherwise + */ +static int kutf_result_to_string(char **result_str, + enum kutf_result_status result) +{ + int i; + int ret = 0; + + for (i = 0; i < UTF_CONVERT_SIZE; i++) { + if (result == kutf_convert[i].result) { + *result_str = kutf_convert[i].result_name; + ret = 1; + } + } + return ret; +} + +/** + * kutf_debugfs_const_string_read() - Simple debugfs read callback which + * returns a constant string + * @file: Opened file to read from + * @buf: User buffer to write the data into + * @len: Amount of data to read + * @ppos: Offset into file to read from + * + * Return: On success, the number of bytes read and offset @ppos advanced by + * this number; on error, negative value + */ +static ssize_t kutf_debugfs_const_string_read(struct file *file, + char __user *buf, size_t len, loff_t *ppos) +{ + char *str = file->private_data; + + return simple_read_from_buffer(buf, len, ppos, str, strlen(str)); +} + +static const struct file_operations kutf_debugfs_const_string_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = kutf_debugfs_const_string_read, + .llseek = default_llseek, +}; + +/** + * kutf_add_explicit_result() - Check if an explicit result needs to be added + * @context: KUTF test context + */ +static void kutf_add_explicit_result(struct kutf_context *context) +{ + switch (context->expected_status) { + case KUTF_RESULT_UNKNOWN: + break; + + case KUTF_RESULT_WARN: + if (context->status == KUTF_RESULT_WARN) + kutf_test_pass(context, + "Pass (expected warn occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected warn missing)"); + break; + + case KUTF_RESULT_FAIL: + if (context->status == KUTF_RESULT_FAIL) + kutf_test_pass(context, + "Pass (expected fail occurred)"); + else if (context->status != KUTF_RESULT_SKIP) { + /* Force the expected status so the fail gets logged */ + context->expected_status = KUTF_RESULT_PASS; + kutf_test_fail(context, + "Fail (expected fail missing)"); + } + break; + + case KUTF_RESULT_FATAL: + if (context->status == KUTF_RESULT_FATAL) + kutf_test_pass(context, + "Pass (expected fatal occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected fatal missing)"); + break; + + case KUTF_RESULT_ABORT: + if (context->status == KUTF_RESULT_ABORT) + kutf_test_pass(context, + "Pass (expected abort occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected abort missing)"); + break; + default: + break; + } +} + +static void kutf_run_test(struct work_struct *data) +{ + struct kutf_context *test_context = container_of(data, + struct kutf_context, work); + struct kutf_suite *suite = test_context->suite; + struct kutf_test_function *test_func; + + test_func = test_context->test_fix->test_func; + + /* + * Call the create fixture function if required before the + * fixture is run + */ + if (suite->create_fixture) + test_context->fixture = suite->create_fixture(test_context); + + /* Only run the test if the fixture was created (if required) */ + if ((suite->create_fixture && test_context->fixture) || + (!suite->create_fixture)) { + /* Run this fixture */ + test_func->execute(test_context); + + if (suite->remove_fixture) + suite->remove_fixture(test_context); + + kutf_add_explicit_result(test_context); + } + + kutf_add_result(test_context, KUTF_RESULT_TEST_FINISHED, NULL); + + kutf_context_put(test_context); +} + +/** + * kutf_debugfs_run_open() Debugfs open callback for the "run" entry. + * @inode: inode of the opened file + * @file: Opened file to read from + * + * This function creates a KUTF context and queues it onto a workqueue to be + * run asynchronously. The resulting file descriptor can be used to communicate + * userdata to the test and to read back the results of the test execution. + * + * Return: 0 on success + */ +static int kutf_debugfs_run_open(struct inode *inode, struct file *file) +{ + struct kutf_test_fixture *test_fix = inode->i_private; + struct kutf_context *test_context; + int err = 0; + + test_context = kutf_create_context(test_fix); + if (!test_context) { + err = -ENOMEM; + goto finish; + } + + file->private_data = test_context; + + /* This reference is release by the kutf_run_test */ + kutf_context_get(test_context); + + queue_work(kutf_workq, &test_context->work); + +finish: + return err; +} + +#define USERDATA_WARNING_MESSAGE "WARNING: This test requires userdata\n" + +/** + * kutf_debugfs_run_read() - Debugfs read callback for the "run" entry. + * @file: Opened file to read from + * @buf: User buffer to write the data into + * @len: Amount of data to read + * @ppos: Offset into file to read from + * + * This function emits the results of the test, blocking until they are + * available. + * + * If the test involves user data then this will also return user data records + * to user space. If the test is waiting for user data then this function will + * output a message (to make the likes of 'cat' display it), followed by + * returning 0 to mark the end of file. + * + * Results will be emitted one at a time, once all the results have been read + * 0 will be returned to indicate there is no more data. + * + * Return: Number of bytes read. + */ +static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, + size_t len, loff_t *ppos) +{ + struct kutf_context *test_context = file->private_data; + struct kutf_result *res; + unsigned long bytes_not_copied; + ssize_t bytes_copied = 0; + char *kutf_str_ptr = NULL; + size_t kutf_str_len = 0; + size_t message_len = 0; + char separator = ':'; + char terminator = '\n'; + + res = kutf_remove_result(test_context->result_set); + + if (IS_ERR(res)) + return PTR_ERR(res); + + /* + * Handle 'fake' results - these results are converted to another + * form before being returned from the kernel + */ + switch (res->status) { + case KUTF_RESULT_TEST_FINISHED: + return 0; + case KUTF_RESULT_USERDATA_WAIT: + if (test_context->userdata.flags & + KUTF_USERDATA_WARNING_OUTPUT) { + /* + * Warning message already output, + * signal end-of-file + */ + return 0; + } + + message_len = sizeof(USERDATA_WARNING_MESSAGE)-1; + if (message_len > len) + message_len = len; + + bytes_not_copied = copy_to_user(buf, + USERDATA_WARNING_MESSAGE, + message_len); + if (bytes_not_copied != 0) + return -EFAULT; + test_context->userdata.flags |= KUTF_USERDATA_WARNING_OUTPUT; + return message_len; + case KUTF_RESULT_USERDATA: + message_len = strlen(res->message); + if (message_len > len-1) { + message_len = len-1; + pr_warn("User data truncated, read not long enough\n"); + } + bytes_not_copied = copy_to_user(buf, res->message, + message_len); + if (bytes_not_copied != 0) { + pr_warn("Failed to copy data to user space buffer\n"); + return -EFAULT; + } + /* Finally the terminator */ + bytes_not_copied = copy_to_user(&buf[message_len], + &terminator, 1); + if (bytes_not_copied != 0) { + pr_warn("Failed to copy data to user space buffer\n"); + return -EFAULT; + } + return message_len+1; + default: + /* Fall through - this is a test result */ + break; + } + + /* Note: This code assumes a result is read completely */ + kutf_result_to_string(&kutf_str_ptr, res->status); + if (kutf_str_ptr) + kutf_str_len = strlen(kutf_str_ptr); + + if (res->message) + message_len = strlen(res->message); + + if ((kutf_str_len + 1 + message_len + 1) > len) { + pr_err("Not enough space in user buffer for a single result"); + return 0; + } + + /* First copy the result string */ + if (kutf_str_ptr) { + bytes_not_copied = copy_to_user(&buf[0], kutf_str_ptr, + kutf_str_len); + bytes_copied += kutf_str_len - bytes_not_copied; + if (bytes_not_copied) + goto exit; + } + + /* Then the separator */ + bytes_not_copied = copy_to_user(&buf[bytes_copied], + &separator, 1); + bytes_copied += 1 - bytes_not_copied; + if (bytes_not_copied) + goto exit; + + /* Finally Next copy the result string */ + if (res->message) { + bytes_not_copied = copy_to_user(&buf[bytes_copied], + res->message, message_len); + bytes_copied += message_len - bytes_not_copied; + if (bytes_not_copied) + goto exit; + } + + /* Finally the terminator */ + bytes_not_copied = copy_to_user(&buf[bytes_copied], + &terminator, 1); + bytes_copied += 1 - bytes_not_copied; + +exit: + return bytes_copied; +} + +/** + * kutf_debugfs_run_write() Debugfs write callback for the "run" entry. + * @file: Opened file to write to + * @buf: User buffer to read the data from + * @len: Amount of data to write + * @ppos: Offset into file to write to + * + * This function allows user and kernel to exchange extra data necessary for + * the test fixture. + * + * The data is added to the first struct kutf_context running the fixture + * + * Return: Number of bytes written + */ +static ssize_t kutf_debugfs_run_write(struct file *file, + const char __user *buf, size_t len, loff_t *ppos) +{ + int ret = 0; + struct kutf_context *test_context = file->private_data; + + if (len > KUTF_MAX_LINE_LENGTH) + return -EINVAL; + + ret = kutf_helper_input_enqueue(test_context, buf, len); + if (ret < 0) + return ret; + + return len; +} + +/** + * kutf_debugfs_run_release() - Debugfs release callback for the "run" entry. + * @inode: File entry representation + * @file: A specific opening of the file + * + * Release any resources that were created during the opening of the file + * + * Note that resources may not be released immediately, that might only happen + * later when other users of the kutf_context release their refcount. + * + * Return: 0 on success + */ +static int kutf_debugfs_run_release(struct inode *inode, struct file *file) +{ + struct kutf_context *test_context = file->private_data; + + kutf_helper_input_enqueue_end_of_data(test_context); + + kutf_context_put(test_context); + return 0; +} + +static const struct file_operations kutf_debugfs_run_ops = { + .owner = THIS_MODULE, + .open = kutf_debugfs_run_open, + .read = kutf_debugfs_run_read, + .write = kutf_debugfs_run_write, + .release = kutf_debugfs_run_release, + .llseek = default_llseek, +}; + +/** + * create_fixture_variant() - Creates a fixture variant for the specified + * test function and index and the debugfs entries + * that represent it. + * @test_func: Test function + * @fixture_index: Fixture index + * + * Return: 0 on success, negative value corresponding to error code in failure + */ +static int create_fixture_variant(struct kutf_test_function *test_func, + unsigned int fixture_index) +{ + struct kutf_test_fixture *test_fix; + char name[11]; /* Enough to print the MAX_UINT32 + the null terminator */ + struct dentry *tmp; + int err; + + test_fix = kmalloc(sizeof(*test_fix), GFP_KERNEL); + if (!test_fix) { + pr_err("Failed to create debugfs directory when adding fixture\n"); + err = -ENOMEM; + goto fail_alloc; + } + + test_fix->test_func = test_func; + test_fix->fixture_index = fixture_index; + + snprintf(name, sizeof(name), "%d", fixture_index); + test_fix->dir = debugfs_create_dir(name, test_func->dir); + if (!test_func->dir) { + pr_err("Failed to create debugfs directory when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_dir; + } + + tmp = debugfs_create_file("type", S_IROTH, test_fix->dir, "fixture\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_file; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) + tmp = debugfs_create_file_unsafe( +#else + tmp = debugfs_create_file( +#endif + "run", 0600, test_fix->dir, + test_fix, + &kutf_debugfs_run_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"run\" when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_file; + } + + list_add(&test_fix->node, &test_func->variant_list); + return 0; + +fail_file: + debugfs_remove_recursive(test_fix->dir); +fail_dir: + kfree(test_fix); +fail_alloc: + return err; +} + +/** + * kutf_remove_test_variant() - Destroy a previously created fixture variant. + * @test_fix: Test fixture + */ +static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix) +{ + debugfs_remove_recursive(test_fix->dir); + kfree(test_fix); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) +/* Adapting to the upstream debugfs_create_x32() change */ +static int ktufp_u32_get(void *data, u64 *val) +{ + *val = *(u32 *)data; + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(kutfp_fops_x32_ro, ktufp_u32_get, NULL, "0x%08llx\n"); +#endif + +void kutf_add_test_with_filters_and_data( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data test_data) +{ + struct kutf_test_function *test_func; + struct dentry *tmp; + unsigned int i; + + test_func = kmalloc(sizeof(*test_func), GFP_KERNEL); + if (!test_func) { + pr_err("Failed to allocate memory when adding test %s\n", name); + goto fail_alloc; + } + + INIT_LIST_HEAD(&test_func->variant_list); + + test_func->dir = debugfs_create_dir(name, suite->dir); + if (!test_func->dir) { + pr_err("Failed to create debugfs directory when adding test %s\n", name); + goto fail_dir; + } + + tmp = debugfs_create_file("type", S_IROTH, test_func->dir, "test\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); + goto fail_file; + } + + test_func->filters = filters; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) + tmp = debugfs_create_file_unsafe("filters", S_IROTH, test_func->dir, + &test_func->filters, &kutfp_fops_x32_ro); +#else + tmp = debugfs_create_x32("filters", S_IROTH, test_func->dir, + &test_func->filters); +#endif + if (!tmp) { + pr_err("Failed to create debugfs file \"filters\" when adding test %s\n", name); + goto fail_file; + } + + test_func->test_id = id; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) + debugfs_create_u32("test_id", S_IROTH, test_func->dir, + &test_func->test_id); +#else + tmp = debugfs_create_u32("test_id", S_IROTH, test_func->dir, + &test_func->test_id); + if (!tmp) { + pr_err("Failed to create debugfs file \"test_id\" when adding test %s\n", name); + goto fail_file; + } +#endif + + for (i = 0; i < suite->fixture_variants; i++) { + if (create_fixture_variant(test_func, i)) { + pr_err("Failed to create fixture %d when adding test %s\n", i, name); + goto fail_file; + } + } + + test_func->suite = suite; + test_func->execute = execute; + test_func->test_data = test_data; + + list_add(&test_func->node, &suite->test_list); + return; + +fail_file: + debugfs_remove_recursive(test_func->dir); +fail_dir: + kfree(test_func); +fail_alloc: + return; +} +EXPORT_SYMBOL(kutf_add_test_with_filters_and_data); + +void kutf_add_test_with_filters( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + + kutf_add_test_with_filters_and_data(suite, + id, + name, + execute, + suite->suite_default_flags, + data); +} +EXPORT_SYMBOL(kutf_add_test_with_filters); + +void kutf_add_test(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context)) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + + kutf_add_test_with_filters_and_data(suite, + id, + name, + execute, + suite->suite_default_flags, + data); +} +EXPORT_SYMBOL(kutf_add_test); + +/** + * kutf_remove_test(): Remove a previously added test function. + * @test_func: Test function + */ +static void kutf_remove_test(struct kutf_test_function *test_func) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &test_func->variant_list) { + struct kutf_test_fixture *test_fix; + + test_fix = list_entry(pos, struct kutf_test_fixture, node); + kutf_remove_test_variant(test_fix); + } + + list_del(&test_func->node); + debugfs_remove_recursive(test_func->dir); + kfree(test_func); +} + +struct kutf_suite *kutf_create_suite_with_filters_and_data( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data suite_data) +{ + struct kutf_suite *suite; + struct dentry *tmp; + + suite = kmalloc(sizeof(*suite), GFP_KERNEL); + if (!suite) { + pr_err("Failed to allocate memory when creating suite %s\n", name); + goto fail_kmalloc; + } + + suite->dir = debugfs_create_dir(name, app->dir); + if (!suite->dir) { + pr_err("Failed to create debugfs directory when adding test %s\n", name); + goto fail_debugfs; + } + + tmp = debugfs_create_file("type", S_IROTH, suite->dir, "suite\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); + goto fail_file; + } + + INIT_LIST_HEAD(&suite->test_list); + suite->app = app; + suite->name = name; + suite->fixture_variants = fixture_count; + suite->create_fixture = create_fixture; + suite->remove_fixture = remove_fixture; + suite->suite_default_flags = filters; + suite->suite_data = suite_data; + + list_add(&suite->node, &app->suite_list); + + return suite; + +fail_file: + debugfs_remove_recursive(suite->dir); +fail_debugfs: + kfree(suite); +fail_kmalloc: + return NULL; +} +EXPORT_SYMBOL(kutf_create_suite_with_filters_and_data); + +struct kutf_suite *kutf_create_suite_with_filters( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + return kutf_create_suite_with_filters_and_data(app, + name, + fixture_count, + create_fixture, + remove_fixture, + filters, + data); +} +EXPORT_SYMBOL(kutf_create_suite_with_filters); + +struct kutf_suite *kutf_create_suite( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context)) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + return kutf_create_suite_with_filters_and_data(app, + name, + fixture_count, + create_fixture, + remove_fixture, + KUTF_F_TEST_GENERIC, + data); +} +EXPORT_SYMBOL(kutf_create_suite); + +/** + * kutf_destroy_suite() - Destroy a previously added test suite. + * @suite: Test suite + */ +static void kutf_destroy_suite(struct kutf_suite *suite) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &suite->test_list) { + struct kutf_test_function *test_func; + + test_func = list_entry(pos, struct kutf_test_function, node); + kutf_remove_test(test_func); + } + + list_del(&suite->node); + debugfs_remove_recursive(suite->dir); + kfree(suite); +} + +struct kutf_application *kutf_create_application(const char *name) +{ + struct kutf_application *app; + struct dentry *tmp; + + app = kmalloc(sizeof(*app), GFP_KERNEL); + if (!app) { + pr_err("Failed to create allocate memory when creating application %s\n", name); + goto fail_kmalloc; + } + + app->dir = debugfs_create_dir(name, base_dir); + if (!app->dir) { + pr_err("Failed to create debugfs direcotry when creating application %s\n", name); + goto fail_debugfs; + } + + tmp = debugfs_create_file("type", S_IROTH, app->dir, "application\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when creating application %s\n", name); + goto fail_file; + } + + INIT_LIST_HEAD(&app->suite_list); + app->name = name; + + return app; + +fail_file: + debugfs_remove_recursive(app->dir); +fail_debugfs: + kfree(app); +fail_kmalloc: + return NULL; +} +EXPORT_SYMBOL(kutf_create_application); + +void kutf_destroy_application(struct kutf_application *app) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &app->suite_list) { + struct kutf_suite *suite; + + suite = list_entry(pos, struct kutf_suite, node); + kutf_destroy_suite(suite); + } + + debugfs_remove_recursive(app->dir); + kfree(app); +} +EXPORT_SYMBOL(kutf_destroy_application); + +static struct kutf_context *kutf_create_context( + struct kutf_test_fixture *test_fix) +{ + struct kutf_context *new_context; + + new_context = kmalloc(sizeof(*new_context), GFP_KERNEL); + if (!new_context) { + pr_err("Failed to allocate test context"); + goto fail_alloc; + } + + new_context->result_set = kutf_create_result_set(); + if (!new_context->result_set) { + pr_err("Failed to create result set"); + goto fail_result_set; + } + + new_context->test_fix = test_fix; + /* Save the pointer to the suite as the callbacks will require it */ + new_context->suite = test_fix->test_func->suite; + new_context->status = KUTF_RESULT_UNKNOWN; + new_context->expected_status = KUTF_RESULT_UNKNOWN; + + kutf_mempool_init(&new_context->fixture_pool); + new_context->fixture = NULL; + new_context->fixture_index = test_fix->fixture_index; + new_context->fixture_name = NULL; + new_context->test_data = test_fix->test_func->test_data; + + new_context->userdata.flags = 0; + INIT_LIST_HEAD(&new_context->userdata.input_head); + init_waitqueue_head(&new_context->userdata.input_waitq); + + INIT_WORK(&new_context->work, kutf_run_test); + + kref_init(&new_context->kref); + + return new_context; + +fail_result_set: + kfree(new_context); +fail_alloc: + return NULL; +} + +static void kutf_destroy_context(struct kref *kref) +{ + struct kutf_context *context; + + context = container_of(kref, struct kutf_context, kref); + kutf_destroy_result_set(context->result_set); + kutf_mempool_destroy(&context->fixture_pool); + kfree(context); +} + +static void kutf_context_get(struct kutf_context *context) +{ + kref_get(&context->kref); +} + +static void kutf_context_put(struct kutf_context *context) +{ + kref_put(&context->kref, kutf_destroy_context); +} + + +static void kutf_set_result(struct kutf_context *context, + enum kutf_result_status status) +{ + context->status = status; +} + +static void kutf_set_expected_result(struct kutf_context *context, + enum kutf_result_status expected_status) +{ + context->expected_status = expected_status; +} + +/** + * kutf_test_log_result() - Log a result for the specified test context + * @context: Test context + * @message: Result string + * @new_status: Result status + */ +static void kutf_test_log_result( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status) +{ + if (context->status < new_status) + context->status = new_status; + + if (context->expected_status != new_status) + kutf_add_result(context, new_status, message); +} + +void kutf_test_log_result_external( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status) +{ + kutf_test_log_result(context, message, new_status); +} +EXPORT_SYMBOL(kutf_test_log_result_external); + +void kutf_test_expect_abort(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_ABORT); +} +EXPORT_SYMBOL(kutf_test_expect_abort); + +void kutf_test_expect_fatal(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_FATAL); +} +EXPORT_SYMBOL(kutf_test_expect_fatal); + +void kutf_test_expect_fail(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_FAIL); +} +EXPORT_SYMBOL(kutf_test_expect_fail); + +void kutf_test_expect_warn(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_WARN); +} +EXPORT_SYMBOL(kutf_test_expect_warn); + +void kutf_test_expect_pass(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_PASS); +} +EXPORT_SYMBOL(kutf_test_expect_pass); + +void kutf_test_skip(struct kutf_context *context) +{ + kutf_set_result(context, KUTF_RESULT_SKIP); + kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); + + kutf_test_log_result(context, "Test skipped", KUTF_RESULT_SKIP); +} +EXPORT_SYMBOL(kutf_test_skip); + +void kutf_test_skip_msg(struct kutf_context *context, const char *message) +{ + kutf_set_result(context, KUTF_RESULT_SKIP); + kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); + + kutf_test_log_result(context, kutf_dsprintf(&context->fixture_pool, + "Test skipped: %s", message), KUTF_RESULT_SKIP); + kutf_test_log_result(context, "!!!Test skipped!!!", KUTF_RESULT_SKIP); +} +EXPORT_SYMBOL(kutf_test_skip_msg); + +void kutf_test_debug(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_DEBUG); +} +EXPORT_SYMBOL(kutf_test_debug); + +void kutf_test_pass(struct kutf_context *context, char const *message) +{ + static const char explicit_message[] = "(explicit pass)"; + + if (!message) + message = explicit_message; + + kutf_test_log_result(context, message, KUTF_RESULT_PASS); +} +EXPORT_SYMBOL(kutf_test_pass); + +void kutf_test_info(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_INFO); +} +EXPORT_SYMBOL(kutf_test_info); + +void kutf_test_warn(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_WARN); +} +EXPORT_SYMBOL(kutf_test_warn); + +void kutf_test_fail(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_FAIL); +} +EXPORT_SYMBOL(kutf_test_fail); + +void kutf_test_fatal(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_FATAL); +} +EXPORT_SYMBOL(kutf_test_fatal); + +void kutf_test_abort(struct kutf_context *context) +{ + kutf_test_log_result(context, "", KUTF_RESULT_ABORT); +} +EXPORT_SYMBOL(kutf_test_abort); + +#ifdef CONFIG_DEBUG_FS + +/** + * init_kutf_core() - Module entry point. + * + * Create the base entry point in debugfs. + */ +static int __init init_kutf_core(void) +{ + kutf_workq = alloc_workqueue("kutf workq", WQ_UNBOUND, 1); + if (!kutf_workq) + return -ENOMEM; + + base_dir = debugfs_create_dir("kutf_tests", NULL); + if (!base_dir) { + destroy_workqueue(kutf_workq); + kutf_workq = NULL; + return -ENOMEM; + } + + return 0; +} + +/** + * exit_kutf_core() - Module exit point. + * + * Remove the base entry point in debugfs. + */ +static void __exit exit_kutf_core(void) +{ + debugfs_remove_recursive(base_dir); + + if (kutf_workq) + destroy_workqueue(kutf_workq); +} + +#else /* CONFIG_DEBUG_FS */ + +/** + * init_kutf_core() - Module entry point. + * + * Stub for when build against a kernel without debugfs support + */ +static int __init init_kutf_core(void) +{ + pr_debug("KUTF requires a kernel with debug fs support"); + + return -ENODEV; +} + +/** + * exit_kutf_core() - Module exit point. + * + * Stub for when build against a kernel without debugfs support + */ +static void __exit exit_kutf_core(void) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +MODULE_LICENSE("GPL"); + +module_init(init_kutf_core); +module_exit(exit_kutf_core); diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c new file mode 100755 index 000000000000..7f5ac517fdb4 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c @@ -0,0 +1,76 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* Kernel UTF utility functions */ + +#include +#include +#include +#include + +#include +#include + +static char tmp_buffer[KUTF_MAX_DSPRINTF_LEN]; + +DEFINE_MUTEX(buffer_lock); + +const char *kutf_dsprintf(struct kutf_mempool *pool, + const char *fmt, ...) +{ + va_list args; + int len; + int size; + void *buffer; + + mutex_lock(&buffer_lock); + va_start(args, fmt); + len = vsnprintf(tmp_buffer, sizeof(tmp_buffer), fmt, args); + va_end(args); + + if (len < 0) { + pr_err("kutf_dsprintf: Bad format dsprintf format %s\n", fmt); + goto fail_format; + } + + if (len >= sizeof(tmp_buffer)) { + pr_warn("kutf_dsprintf: Truncated dsprintf message %s\n", fmt); + size = sizeof(tmp_buffer); + } else { + size = len + 1; + } + + buffer = kutf_mempool_alloc(pool, size); + if (!buffer) + goto fail_alloc; + + memcpy(buffer, tmp_buffer, size); + mutex_unlock(&buffer_lock); + + return buffer; + +fail_alloc: +fail_format: + mutex_unlock(&buffer_lock); + return NULL; +} +EXPORT_SYMBOL(kutf_dsprintf); diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kbuild b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kbuild new file mode 100755 index 000000000000..f5565d30f9cf --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kbuild @@ -0,0 +1,26 @@ +# +# (C) COPYRIGHT 2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android + +obj-$(CONFIG_MALI_CLK_RATE_TRACE_PORTAL) += mali_kutf_clk_rate_trace_test_portal.o + +mali_kutf_clk_rate_trace_test_portal-y := mali_kutf_clk_rate_trace_test.o diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kconfig b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kconfig new file mode 100755 index 000000000000..8196e4cc6b37 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Kconfig @@ -0,0 +1,30 @@ +# +# (C) COPYRIGHT 2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +config CONFIG_MALI_CLK_RATE_TRACE_PORTAL + tristate "Mali GPU Clock Trace Test portal" + depends on MALI_BIFROST && MALI_BIFROST_DEBUG && MALI_KUTF + default m + help + This option will build a test module mali_kutf_clk_rate_trace_test_portal + that can test the clocks integration into the platform and exercise some + basic trace test in the system. Choosing M here will generate a single + module called mali_kutf_clk_rate_trace_test_portal. diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Makefile b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Makefile new file mode 100755 index 000000000000..71c78b84830c --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/Makefile @@ -0,0 +1,57 @@ +# +# (C) COPYRIGHT 2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +ifneq ($(KERNELRELEASE),) + +ccflags-y := \ + -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ + -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ + -I$(src)/../../include \ + -I$(src)/../../../../../../../include \ + -I$(src)/../../../../ \ + -I$(src)/../../../ \ + -I$(src)/../../../backend/gpu \ + -I$(src)/../../../debug \ + -I$(src)/../../../debug/backend \ + -I$(src)/ \ + -I$(srctree)/drivers/staging/android \ + -I$(srctree)/include/linux + +obj-m := mali_kutf_clk_rate_trace_test_portal.o +mali_kutf_clk_rate_trace_test_portal-y := mali_kutf_clk_rate_trace_test.o + +else +# linux build system bootstrap for out-of-tree module + +# default to building for the host +ARCH ?= $(shell uname -m) + +ifeq ($(KDIR),) +$(error Must specify KDIR to point to the kernel to target)) +endif + +all: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../../kutf/Module.symvers $(CURDIR)/../../../Module.symvers" modules + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean + +endif diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/build.bp b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/build.bp new file mode 100755 index 000000000000..0cc2904db542 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/build.bp @@ -0,0 +1,34 @@ +/* + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +bob_kernel_module { + name: "mali_kutf_clk_rate_trace_test_portal", + defaults: [ + "mali_kbase_shared_config_defaults", + "kernel_test_includes", + ], + srcs: [ + "../mali_kutf_clk_rate_trace_test.h", + "Makefile", + "mali_kutf_clk_rate_trace_test.c", + ], + extra_symbols: [ + "mali_kbase", + "kutf", + ], + enabled: false, + base_build_kutf: { + enabled: true, + kbuild_options: ["CONFIG_MALI_CLK_RATE_TRACE_PORTAL=m"], + }, +} diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c new file mode 100755 index 000000000000..d74a278bffa7 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c @@ -0,0 +1,890 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include + +#include +#include +#include +#include +#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) +#include +#else +#include +#endif +#include "mali_kbase.h" +#include "mali_kbase_irq_internal.h" +#include "mali_kbase_pm_internal.h" +#include "mali_kbase_clk_rate_trace_mgr.h" + +#include +#include +#include +#include + +#include "../mali_kutf_clk_rate_trace_test.h" + +#define MINOR_FOR_FIRST_KBASE_DEV (-1) + +/* KUTF test application pointer for this test */ +struct kutf_application *kutf_app; + +enum portal_server_state { + PORTAL_STATE_NO_CLK, + PORTAL_STATE_LIVE, + PORTAL_STATE_CLOSING, +}; + +/** + * struct clk_trace_snapshot - Trace info data on a clock. + * @previous_rate: Snapshot start point clock rate. + * @current_rate: End point clock rate. It becomes the start rate of the + * next trace snapshot. + * @rate_up_cnt: Count in the snapshot duration when the clock trace + * write is a rate of higher value than the last. + * @rate_down_cnt: Count in the snapshot duration when the clock trace write + * is a rate of lower value than the last. + */ +struct clk_trace_snapshot { + unsigned long previous_rate; + unsigned long current_rate; + u32 rate_up_cnt; + u32 rate_down_cnt; +}; + +/** + * struct kutf_clk_rate_trace_fixture_data - Fixture data for the test. + * @kbdev: kbase device for the GPU. + * @listener: Clock rate change listener structure. + * @invoke_notify: When true, invoke notify command is being executed. + * @snapshot: Clock trace update snapshot data array. A snapshot + * for each clock contains info accumulated beteen two + * GET_TRACE_SNAPSHOT requests. + * @nclks: Number of clocks visible to the trace portal. + * @pm_ctx_cnt: Net count of PM (Power Management) context INC/DEC + * PM_CTX_CNT requests made to the portal. On change from + * 0 to 1 (INC), or, 1 to 0 (DEC), a PM context action is + * triggered. + * @total_update_cnt: Total number of received trace write callbacks. + * @server_state: Portal server operational state. + * @result_msg: Message for the test result. + * @test_status: Portal test reslt status. + */ +struct kutf_clk_rate_trace_fixture_data { + struct kbase_device *kbdev; + struct kbase_clk_rate_listener listener; + bool invoke_notify; + struct clk_trace_snapshot snapshot[BASE_MAX_NR_CLOCKS_REGULATORS]; + unsigned int nclks; + unsigned int pm_ctx_cnt; + unsigned int total_update_cnt; + enum portal_server_state server_state; + char const *result_msg; + enum kutf_result_status test_status; +}; + +struct clk_trace_portal_input { + struct kutf_helper_named_val cmd_input; + enum kbasep_clk_rate_trace_req portal_cmd; + int named_val_err; +}; + +struct kbasep_cmd_name_pair { + enum kbasep_clk_rate_trace_req cmd; + const char *name; +}; + +struct kbasep_cmd_name_pair kbasep_portal_cmd_name_map[] = { + {PORTAL_CMD_GET_CLK_RATE_MGR, GET_CLK_RATE_MGR}, + {PORTAL_CMD_GET_CLK_RATE_TRACE, GET_CLK_RATE_TRACE}, + {PORTAL_CMD_GET_TRACE_SNAPSHOT, GET_TRACE_SNAPSHOT}, + {PORTAL_CMD_INC_PM_CTX_CNT, INC_PM_CTX_CNT}, + {PORTAL_CMD_DEC_PM_CTX_CNT, DEC_PM_CTX_CNT}, + {PORTAL_CMD_CLOSE_PORTAL, CLOSE_PORTAL}, + {PORTAL_CMD_INVOKE_NOTIFY_42KHZ, INVOKE_NOTIFY_42KHZ}, + }; + +/* Global pointer for the kutf_portal_trace_write() to use. When + * this pointer is engaged, new requests for create fixture will fail + * hence limiting the use of the portal at any time to a singleton. + */ +struct kutf_clk_rate_trace_fixture_data *g_ptr_portal_data; + +#define PORTAL_MSG_LEN (KUTF_MAX_LINE_LENGTH - MAX_REPLY_NAME_LEN) +static char portal_msg_buf[PORTAL_MSG_LEN]; + +static void kutf_portal_trace_write( + struct kbase_clk_rate_listener *listener, + u32 index, u32 new_rate) +{ + struct clk_trace_snapshot *snapshot; + struct kutf_clk_rate_trace_fixture_data *data = container_of( + listener, struct kutf_clk_rate_trace_fixture_data, listener); + + lockdep_assert_held(&data->kbdev->pm.clk_rtm.lock); + + if (WARN_ON(g_ptr_portal_data == NULL)) + return; + if (WARN_ON(index >= g_ptr_portal_data->nclks)) + return; + + /* This callback is triggered by invoke notify command, skipping */ + if (data->invoke_notify) + return; + + snapshot = &g_ptr_portal_data->snapshot[index]; + if (new_rate > snapshot->current_rate) + snapshot->rate_up_cnt++; + else + snapshot->rate_down_cnt++; + snapshot->current_rate = new_rate; + g_ptr_portal_data->total_update_cnt++; +} + +static void kutf_set_pm_ctx_active(struct kutf_context *context) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + + if (WARN_ON(data->pm_ctx_cnt != 1)) + return; + + kbase_pm_context_active(data->kbdev); + kbase_pm_wait_for_desired_state(data->kbdev); +#if !MALI_USE_CSF + kbase_pm_request_gpu_cycle_counter(data->kbdev); +#endif +} + +static void kutf_set_pm_ctx_idle(struct kutf_context *context) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + + if (WARN_ON(data->pm_ctx_cnt > 0)) + return; + + kbase_pm_context_idle(data->kbdev); +#if !MALI_USE_CSF + kbase_pm_release_gpu_cycle_counter(data->kbdev); +#endif +} + +static char const *kutf_clk_trace_do_change_pm_ctx(struct kutf_context *context, + struct clk_trace_portal_input *cmd) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + int seq = cmd->cmd_input.u.val_u64 & 0xFF; + const unsigned int cnt = data->pm_ctx_cnt; + const enum kbasep_clk_rate_trace_req req = cmd->portal_cmd; + char const *errmsg = NULL; + + WARN_ON(req != PORTAL_CMD_INC_PM_CTX_CNT && + req != PORTAL_CMD_DEC_PM_CTX_CNT); + + if (req == PORTAL_CMD_INC_PM_CTX_CNT && cnt < UINT_MAX) { + data->pm_ctx_cnt++; + if (data->pm_ctx_cnt == 1) + kutf_set_pm_ctx_active(context); + } + + if (req == PORTAL_CMD_DEC_PM_CTX_CNT && cnt > 0) { + data->pm_ctx_cnt--; + if (data->pm_ctx_cnt == 0) + kutf_set_pm_ctx_idle(context); + } + + /* Skip the length check, no chance of overflow for two ints */ + snprintf(portal_msg_buf, PORTAL_MSG_LEN, + "{SEQ:%d, PM_CTX_CNT:%u}", seq, data->pm_ctx_cnt); + + if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { + pr_warn("Error in sending ack for adjusting pm_ctx_cnt\n"); + errmsg = kutf_dsprintf(&context->fixture_pool, + "Error in sending ack for adjusting pm_ctx_cnt"); + } + + return errmsg; +} + +static char const *kutf_clk_trace_do_get_rate(struct kutf_context *context, + struct clk_trace_portal_input *cmd) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + int seq = cmd->cmd_input.u.val_u64 & 0xFF; + unsigned long rate; + bool idle; + int ret; + int i; + char const *errmsg = NULL; + + WARN_ON((cmd->portal_cmd != PORTAL_CMD_GET_CLK_RATE_MGR) && + (cmd->portal_cmd != PORTAL_CMD_GET_CLK_RATE_TRACE)); + + ret = snprintf(portal_msg_buf, PORTAL_MSG_LEN, + "{SEQ:%d, RATE:[", seq); + + for (i = 0; i < data->nclks; i++) { + spin_lock(&kbdev->pm.clk_rtm.lock); + if (cmd->portal_cmd == PORTAL_CMD_GET_CLK_RATE_MGR) + rate = kbdev->pm.clk_rtm.clks[i]->clock_val; + else + rate = data->snapshot[i].current_rate; + idle = kbdev->pm.clk_rtm.gpu_idle; + spin_unlock(&kbdev->pm.clk_rtm.lock); + + if ((i + 1) == data->nclks) + ret += snprintf(portal_msg_buf + ret, + PORTAL_MSG_LEN - ret, "0x%lx], GPU_IDLE:%d}", + rate, idle); + else + ret += snprintf(portal_msg_buf + ret, + PORTAL_MSG_LEN - ret, "0x%lx, ", rate); + + if (ret >= PORTAL_MSG_LEN) { + pr_warn("Message buf overflow with rate array data\n"); + return kutf_dsprintf(&context->fixture_pool, + "Message buf overflow with rate array data"); + } + } + + if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { + pr_warn("Error in sending back rate array\n"); + errmsg = kutf_dsprintf(&context->fixture_pool, + "Error in sending rate array"); + } + + return errmsg; +} + +/** + * kutf_clk_trace_do_get_snapshot() - Send back the current snapshot + * @context: KUTF context + * @cmd: The decoded portal input request + * + * The accumulated clock rate trace information is kept inside as an snapshot + * record. A user request of getting the snapshot marks the closure of the + * current snapshot record, and the start of the next one. The response + * message contains the current snapshot record, with each clock's + * data sequentially placed inside (array marker) [ ]. + */ +static char const *kutf_clk_trace_do_get_snapshot(struct kutf_context *context, + struct clk_trace_portal_input *cmd) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + struct clk_trace_snapshot snapshot; + int seq = cmd->cmd_input.u.val_u64 & 0xFF; + int ret; + int i; + char const *fmt; + char const *errmsg = NULL; + + WARN_ON(cmd->portal_cmd != PORTAL_CMD_GET_TRACE_SNAPSHOT); + + ret = snprintf(portal_msg_buf, PORTAL_MSG_LEN, + "{SEQ:%d, SNAPSHOT_ARRAY:[", seq); + + for (i = 0; i < data->nclks; i++) { + spin_lock(&data->kbdev->pm.clk_rtm.lock); + /* copy out the snapshot of the clock */ + snapshot = data->snapshot[i]; + /* Set the next snapshot start condition */ + data->snapshot[i].previous_rate = snapshot.current_rate; + data->snapshot[i].rate_up_cnt = 0; + data->snapshot[i].rate_down_cnt = 0; + spin_unlock(&data->kbdev->pm.clk_rtm.lock); + + /* Check i corresponding to the last clock */ + if ((i + 1) == data->nclks) + fmt = "(0x%lx, 0x%lx, %u, %u)]}"; + else + fmt = "(0x%lx, 0x%lx, %u, %u), "; + ret += snprintf(portal_msg_buf + ret, PORTAL_MSG_LEN - ret, + fmt, snapshot.previous_rate, snapshot.current_rate, + snapshot.rate_up_cnt, snapshot.rate_down_cnt); + if (ret >= PORTAL_MSG_LEN) { + pr_warn("Message buf overflow with snapshot data\n"); + return kutf_dsprintf(&context->fixture_pool, + "Message buf overflow with snapshot data"); + } + } + + if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { + pr_warn("Error in sending back snapshot array\n"); + errmsg = kutf_dsprintf(&context->fixture_pool, + "Error in sending snapshot array"); + } + + return errmsg; +} + +/** + * kutf_clk_trace_do_invoke_notify_42k() - Invokes the stored notification callback + * @context: KUTF context + * @cmd: The decoded portal input request + * + * Invokes frequency change notification callbacks with a fake + * GPU frequency 42 kHz for the top clock domain. + */ +static char const *kutf_clk_trace_do_invoke_notify_42k( + struct kutf_context *context, + struct clk_trace_portal_input *cmd) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + int seq = cmd->cmd_input.u.val_u64 & 0xFF; + const unsigned long new_rate_hz = 42000; + int ret; + char const *errmsg = NULL; + struct kbase_clk_rate_trace_manager *clk_rtm = &data->kbdev->pm.clk_rtm; + + WARN_ON(cmd->portal_cmd != PORTAL_CMD_INVOKE_NOTIFY_42KHZ); + + spin_lock(&clk_rtm->lock); + + data->invoke_notify = true; + kbase_clk_rate_trace_manager_notify_all( + clk_rtm, 0, new_rate_hz); + data->invoke_notify = false; + + spin_unlock(&clk_rtm->lock); + + ret = snprintf(portal_msg_buf, PORTAL_MSG_LEN, + "{SEQ:%d, HZ:%lu}", seq, new_rate_hz); + + if (ret >= PORTAL_MSG_LEN) { + pr_warn("Message buf overflow with invoked data\n"); + return kutf_dsprintf(&context->fixture_pool, + "Message buf overflow with invoked data"); + } + + if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { + pr_warn("Error in sending ack for " INVOKE_NOTIFY_42KHZ "request\n"); + errmsg = kutf_dsprintf(&context->fixture_pool, + "Error in sending ack for " INVOKE_NOTIFY_42KHZ "request"); + } + + return errmsg; +} + +static char const *kutf_clk_trace_do_close_portal(struct kutf_context *context, + struct clk_trace_portal_input *cmd) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + int seq = cmd->cmd_input.u.val_u64 & 0xFF; + char const *errmsg = NULL; + + WARN_ON(cmd->portal_cmd != PORTAL_CMD_CLOSE_PORTAL); + + data->server_state = PORTAL_STATE_CLOSING; + + /* Skip the length check, no chance of overflow for two ints */ + snprintf(portal_msg_buf, PORTAL_MSG_LEN, + "{SEQ:%d, PM_CTX_CNT:%u}", seq, data->pm_ctx_cnt); + + if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { + pr_warn("Error in sending ack for " CLOSE_PORTAL "reuquest\n"); + errmsg = kutf_dsprintf(&context->fixture_pool, + "Error in sending ack for " CLOSE_PORTAL "reuquest"); + } + + return errmsg; +} + +static bool kutf_clk_trace_dequeue_portal_cmd(struct kutf_context *context, + struct clk_trace_portal_input *cmd) +{ + int i; + int err = kutf_helper_receive_named_val(context, &cmd->cmd_input); + + cmd->named_val_err = err; + if (err == KUTF_HELPER_ERR_NONE && + cmd->cmd_input.type == KUTF_HELPER_VALTYPE_U64) { + /* All portal request commands are of format (named u64): + * CMD_NAME=1234 + * where, 1234 is a (variable) sequence number tag. + */ + for (i = 0; i < PORTAL_TOTAL_CMDS; i++) { + if (strcmp(cmd->cmd_input.val_name, + kbasep_portal_cmd_name_map[i].name)) + continue; + + cmd->portal_cmd = kbasep_portal_cmd_name_map[i].cmd; + return true; + } + } + + cmd->portal_cmd = PORTAL_CMD_INVALID; + return false; +} + +static void kutf_clk_trace_flag_result(struct kutf_context *context, + enum kutf_result_status result, char const *msg) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + + if (result > data->test_status) { + data->test_status = result; + if (msg) + data->result_msg = msg; + if (data->server_state == PORTAL_STATE_LIVE && + result > KUTF_RESULT_WARN) { + data->server_state = PORTAL_STATE_CLOSING; + } + } +} + +static bool kutf_clk_trace_process_portal_cmd(struct kutf_context *context, + struct clk_trace_portal_input *cmd) +{ + char const *errmsg = NULL; + + BUILD_BUG_ON(ARRAY_SIZE(kbasep_portal_cmd_name_map) != + PORTAL_TOTAL_CMDS); + WARN_ON(cmd->portal_cmd == PORTAL_CMD_INVALID); + + switch (cmd->portal_cmd) { + case PORTAL_CMD_GET_CLK_RATE_MGR: + /* Fall through */ + case PORTAL_CMD_GET_CLK_RATE_TRACE: + errmsg = kutf_clk_trace_do_get_rate(context, cmd); + break; + case PORTAL_CMD_GET_TRACE_SNAPSHOT: + errmsg = kutf_clk_trace_do_get_snapshot(context, cmd); + break; + case PORTAL_CMD_INC_PM_CTX_CNT: + /* Fall through */ + case PORTAL_CMD_DEC_PM_CTX_CNT: + errmsg = kutf_clk_trace_do_change_pm_ctx(context, cmd); + break; + case PORTAL_CMD_CLOSE_PORTAL: + errmsg = kutf_clk_trace_do_close_portal(context, cmd); + break; + case PORTAL_CMD_INVOKE_NOTIFY_42KHZ: + errmsg = kutf_clk_trace_do_invoke_notify_42k(context, cmd); + break; + default: + pr_warn("Don't know how to handle portal_cmd: %d, abort session.\n", + cmd->portal_cmd); + errmsg = kutf_dsprintf(&context->fixture_pool, + "Don't know how to handle portal_cmd: %d", + cmd->portal_cmd); + break; + } + + if (errmsg) + kutf_clk_trace_flag_result(context, KUTF_RESULT_FAIL, errmsg); + + return (errmsg == NULL); +} + +/** + * kutf_clk_trace_do_nack_response() - respond a NACK to erroneous input + * @context: KUTF context + * @cmd: The erroneous input request + * + * This function deal with an erroneous input request, and respond with + * a proper 'NACK' message. + */ +static int kutf_clk_trace_do_nack_response(struct kutf_context *context, + struct clk_trace_portal_input *cmd) +{ + int seq; + int err; + char const *errmsg = NULL; + + WARN_ON(cmd->portal_cmd != PORTAL_CMD_INVALID); + + if (cmd->named_val_err == KUTF_HELPER_ERR_NONE && + cmd->cmd_input.type == KUTF_HELPER_VALTYPE_U64) { + /* Keep seq number as % 256 */ + seq = cmd->cmd_input.u.val_u64 & 255; + snprintf(portal_msg_buf, PORTAL_MSG_LEN, + "{SEQ:%d, MSG: Unknown command '%s'.}", seq, + cmd->cmd_input.val_name); + err = kutf_helper_send_named_str(context, "NACK", + portal_msg_buf); + } else + err = kutf_helper_send_named_str(context, "NACK", + "Wrong portal cmd format (Ref example: CMD_NAME=0X16)"); + + if (err) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send portal NACK response"); + kutf_clk_trace_flag_result(context, KUTF_RESULT_FAIL, errmsg); + } + + return err; +} + +/** + * kutf_clk_trace_barebone_check() - Sanity test on the clock tracing + * @context: KUTF context + * + * This function carries out some basic test on the tracing operation: + * 1). GPU idle on test start, trace rate should be 0 (low power state) + * 2). Make sure GPU is powered up, the trace rate should match + * that from the clcok manager's internal recorded rate + * 3). If the GPU active transition occurs following 2), there + * must be rate change event from tracing. + */ +void kutf_clk_trace_barebone_check(struct kutf_context *context) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + bool fail = false; + bool idle[2] = { false }; + char const *msg = NULL; + int i; + + /* Check consistency if gpu happens to be idle */ + spin_lock(&kbdev->pm.clk_rtm.lock); + idle[0] = kbdev->pm.clk_rtm.gpu_idle; + if (kbdev->pm.clk_rtm.gpu_idle) { + for (i = 0; i < data->nclks; i++) { + if (data->snapshot[i].current_rate) { + /* Idle should have a rate 0 */ + fail = true; + break; + } + } + } + spin_unlock(&kbdev->pm.clk_rtm.lock); + if (fail) { + msg = kutf_dsprintf(&context->fixture_pool, + "GPU Idle not yielding 0-rate"); + pr_err("Trace did not see idle rate\n"); + } else { + /* Make local PM active if not done so yet */ + if (data->pm_ctx_cnt == 0) { + /* Ensure the GPU is powered */ + data->pm_ctx_cnt++; + kutf_set_pm_ctx_active(context); + } + /* Checking the rate is consistent */ + spin_lock(&kbdev->pm.clk_rtm.lock); + idle[1] = kbdev->pm.clk_rtm.gpu_idle; + for (i = 0; i < data->nclks; i++) { + /* Rate match between the manager and the trace */ + if (kbdev->pm.clk_rtm.clks[i]->clock_val != + data->snapshot[i].current_rate) { + fail = true; + break; + } + } + spin_unlock(&kbdev->pm.clk_rtm.lock); + + if (idle[1]) { + msg = kutf_dsprintf(&context->fixture_pool, + "GPU still idle after set_pm_ctx_active"); + pr_err("GPU still idle after set_pm_ctx_active\n"); + } + + if (!msg && fail) { + msg = kutf_dsprintf(&context->fixture_pool, + "Trace rate not matching Clk manager's read"); + pr_err("Trace rate not matching Clk manager's read\n"); + } + } + + if (!msg && idle[0] && !idle[1] && !data->total_update_cnt) { + msg = kutf_dsprintf(&context->fixture_pool, + "Trace update did not occur"); + pr_err("Trace update did not occur\n"); + } + if (msg) + kutf_clk_trace_flag_result(context, KUTF_RESULT_FAIL, msg); + else if (!data->total_update_cnt) { + msg = kutf_dsprintf(&context->fixture_pool, + "No trace update seen during the test!"); + kutf_clk_trace_flag_result(context, KUTF_RESULT_WARN, msg); + } +} + +static bool kutf_clk_trace_end_of_stream(struct clk_trace_portal_input *cmd) +{ + return (cmd->named_val_err == -EBUSY); +} + +void kutf_clk_trace_no_clks_dummy(struct kutf_context *context) +{ + struct clk_trace_portal_input cmd; + unsigned long timeout = jiffies + HZ * 2; + bool has_cmd; + + while (time_before(jiffies, timeout)) { + if (kutf_helper_pending_input(context)) { + has_cmd = kutf_clk_trace_dequeue_portal_cmd(context, + &cmd); + if (!has_cmd && kutf_clk_trace_end_of_stream(&cmd)) + break; + + kutf_helper_send_named_str(context, "NACK", + "Fatal! No clocks visible, aborting"); + } + msleep(20); + } + + kutf_clk_trace_flag_result(context, KUTF_RESULT_FATAL, + "No clocks visble to the portal"); +} + +/** + * mali_kutf_clk_rate_trace_test_portal() - Service portal input + * @context: KUTF context + * + * The test portal operates on input requests. If the input request is one + * of the recognized portal commands, it handles it accordingly. Otherwise + * a negative response 'NACK' is returned. The portal service terminates + * when a 'CLOSE_PORTAL' request is received, or due to an internal error. + * Both case would result in the server_state transitioned to CLOSING. + * + * If the portal is closed on request, a sanity test on the clock rate + * trace operation is undertaken via function: + * kutf_clk_trace_barebone_check(); + */ +static void mali_kutf_clk_rate_trace_test_portal(struct kutf_context *context) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + struct clk_trace_portal_input new_cmd; + + pr_debug("Test portal service start\n"); + + while (data->server_state == PORTAL_STATE_LIVE) { + if (kutf_clk_trace_dequeue_portal_cmd(context, &new_cmd)) + kutf_clk_trace_process_portal_cmd(context, &new_cmd); + else if (kutf_clk_trace_end_of_stream(&new_cmd)) + /* Dequeue on portal input, end of stream */ + data->server_state = PORTAL_STATE_CLOSING; + else + kutf_clk_trace_do_nack_response(context, &new_cmd); + } + + /* Closing, exhausting all the pending inputs with NACKs. */ + if (data->server_state == PORTAL_STATE_CLOSING) { + while (kutf_helper_pending_input(context) && + (kutf_clk_trace_dequeue_portal_cmd(context, &new_cmd) || + !kutf_clk_trace_end_of_stream(&new_cmd))) { + kutf_helper_send_named_str(context, "NACK", + "Portal closing down"); + } + } + + /* If no portal error, do a barebone test here irrespective + * whatever the portal live session has been testing, which + * is entirely driven by the user-side via portal requests. + */ + if (data->test_status <= KUTF_RESULT_WARN) { + if (data->server_state != PORTAL_STATE_NO_CLK) + kutf_clk_trace_barebone_check(context); + else { + /* No clocks case, NACK 2-sec for the fatal situation */ + kutf_clk_trace_no_clks_dummy(context); + } + } + + /* If we have changed pm_ctx count, drop it back */ + if (data->pm_ctx_cnt) { + /* Although we count on portal requests, it only has material + * impact when from 0 -> 1. So the reverse is a simple one off. + */ + data->pm_ctx_cnt = 0; + kutf_set_pm_ctx_idle(context); + } + + /* Finally log the test result line */ + if (data->test_status < KUTF_RESULT_WARN) + kutf_test_pass(context, data->result_msg); + else if (data->test_status == KUTF_RESULT_WARN) + kutf_test_warn(context, data->result_msg); + else if (data->test_status == KUTF_RESULT_FATAL) + kutf_test_fatal(context, data->result_msg); + else + kutf_test_fail(context, data->result_msg); + + pr_debug("Test end\n"); +} + +/** + * mali_kutf_clk_rate_trace_create_fixture() - Creates the fixture data + * required for mali_kutf_clk_rate_trace_test_portal. + * @context: KUTF context. + * + * Return: Fixture data created on success or NULL on failure + */ +static void *mali_kutf_clk_rate_trace_create_fixture( + struct kutf_context *context) +{ + struct kutf_clk_rate_trace_fixture_data *data; + struct kbase_device *kbdev; + unsigned long rate; + int i; + + /* Acquire the kbase device */ + pr_debug("Finding device\n"); + kbdev = kbase_find_device(MINOR_FOR_FIRST_KBASE_DEV); + if (kbdev == NULL) { + kutf_test_fail(context, "Failed to find kbase device"); + return NULL; + } + + pr_debug("Creating fixture\n"); + data = kutf_mempool_alloc(&context->fixture_pool, + sizeof(struct kutf_clk_rate_trace_fixture_data)); + if (!data) + return NULL; + + *data = (const struct kutf_clk_rate_trace_fixture_data) { 0 }; + pr_debug("Hooking up the test portal to kbdev clk rate trace\n"); + spin_lock(&kbdev->pm.clk_rtm.lock); + + if (g_ptr_portal_data != NULL) { + pr_warn("Test portal is already in use, run aborted\n"); + kutf_test_fail(context, "Portal allows single session only"); + spin_unlock(&kbdev->pm.clk_rtm.lock); + return NULL; + } + + for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { + if (kbdev->pm.clk_rtm.clks[i]) { + data->nclks++; + if (kbdev->pm.clk_rtm.gpu_idle) + rate = 0; + else + rate = kbdev->pm.clk_rtm.clks[i]->clock_val; + data->snapshot[i].previous_rate = rate; + data->snapshot[i].current_rate = rate; + } + } + + spin_unlock(&kbdev->pm.clk_rtm.lock); + + if (data->nclks) { + /* Subscribe this test server portal */ + data->listener.notify = kutf_portal_trace_write; + data->invoke_notify = false; + + kbase_clk_rate_trace_manager_subscribe( + &kbdev->pm.clk_rtm, &data->listener); + /* Update the kutf_server_portal fixture_data pointer */ + g_ptr_portal_data = data; + } + + data->kbdev = kbdev; + data->result_msg = NULL; + data->test_status = KUTF_RESULT_PASS; + + if (data->nclks == 0) { + data->server_state = PORTAL_STATE_NO_CLK; + pr_debug("Kbdev has no clocks for rate trace"); + } else + data->server_state = PORTAL_STATE_LIVE; + + pr_debug("Created fixture\n"); + + return data; +} + +/** + * Destroy fixture data previously created by + * mali_kutf_clk_rate_trace_create_fixture. + * + * @context: KUTF context. + */ +static void mali_kutf_clk_rate_trace_remove_fixture( + struct kutf_context *context) +{ + struct kutf_clk_rate_trace_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + + if (data->nclks) { + /* Clean up the portal trace write arrangement */ + g_ptr_portal_data = NULL; + + kbase_clk_rate_trace_manager_unsubscribe( + &kbdev->pm.clk_rtm, &data->listener); + } + pr_debug("Destroying fixture\n"); + kbase_release_device(kbdev); + pr_debug("Destroyed fixture\n"); +} + +/** + * mali_kutf_clk_rate_trace_test_module_init() - Entry point for test mdoule. + */ +int mali_kutf_clk_rate_trace_test_module_init(void) +{ + struct kutf_suite *suite; + unsigned int filters; + union kutf_callback_data suite_data = { 0 }; + + pr_debug("Creating app\n"); + + g_ptr_portal_data = NULL; + kutf_app = kutf_create_application(CLK_RATE_TRACE_APP_NAME); + + if (!kutf_app) { + pr_warn("Creation of app " CLK_RATE_TRACE_APP_NAME + " failed!\n"); + return -ENOMEM; + } + + pr_debug("Create suite %s\n", CLK_RATE_TRACE_SUITE_NAME); + suite = kutf_create_suite_with_filters_and_data( + kutf_app, CLK_RATE_TRACE_SUITE_NAME, 1, + mali_kutf_clk_rate_trace_create_fixture, + mali_kutf_clk_rate_trace_remove_fixture, + KUTF_F_TEST_GENERIC, + suite_data); + + if (!suite) { + pr_warn("Creation of suite %s failed!\n", + CLK_RATE_TRACE_SUITE_NAME); + kutf_destroy_application(kutf_app); + return -ENOMEM; + } + + filters = suite->suite_default_flags; + kutf_add_test_with_filters( + suite, 0x0, CLK_RATE_TRACE_PORTAL, + mali_kutf_clk_rate_trace_test_portal, + filters); + + pr_debug("Init complete\n"); + return 0; +} + +/** + * mali_kutf_clk_rate_trace_test_module_exit() - Module exit point for this + * test. + */ +void mali_kutf_clk_rate_trace_test_module_exit(void) +{ + pr_debug("Exit start\n"); + kutf_destroy_application(kutf_app); + pr_debug("Exit complete\n"); +} + + +module_init(mali_kutf_clk_rate_trace_test_module_init); +module_exit(mali_kutf_clk_rate_trace_test_module_exit); + +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h new file mode 100755 index 000000000000..f46afd5086bd --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h @@ -0,0 +1,148 @@ +/* + * + * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#ifndef _KUTF_CLK_RATE_TRACE_TEST_H_ +#define _KUTF_CLK_RATE_TRACE_TEST_H_ + +#define CLK_RATE_TRACE_APP_NAME "clk_rate_trace" +#define CLK_RATE_TRACE_SUITE_NAME "rate_trace" +#define CLK_RATE_TRACE_PORTAL "portal" + +/** + * enum kbasep_clk_rate_trace_req - request command to the clock rate trace + * service portal. + * + * @PORTAL_CMD_GET_CLK_RATE_MGR: Request the clock trace manager internal + * data record. On a positive acknowledgement + * the prevailing clock rates and the GPU idle + * condition flag are returned. + * @PORTAL_CMD_GET_CLK_RATE_TRACE: Request the clock trace portal to return its + * data record. On a positive acknowledgement + * the last trace recorded clock rates and the + * GPU idle condition flag are returned. + * @PORTAL_CMD_GET_TRACE_SNAPSHOT: Request the clock trace portal to return its + * current snapshot data record. On a positive + * acknowledgement the snapshot array matching + * the number of clocks are returned. It also + * starts a fresh snapshot inside the clock + * trace portal. + * @PORTAL_CMD_INC_PM_CTX_CNT: Request the clock trace portal to increase + * its internal PM_CTX_COUNT. If this increase + * yielded a count of 0 -> 1 change, the portal + * will initiate a PM_CTX_ACTIVE call to the + * Kbase power management. Futher increase + * requests will limit to only affect the + * portal internal count value. + * @PORTAL_CMD_DEC_PM_CTX_CNT: Request the clock trace portal to decrease + * its internal PM_CTX_COUNT. If this decrease + * yielded a count of 1 -> 0 change, the portal + * will initiate a PM_CTX_IDLE call to the + * Kbase power management. + * @PORTAL_CMD_CLOSE_PORTAL: Inform the clock trace portal service the + * client has completed its session. The portal + * will start the close down action. If no + * error has occurred during the dynamic + * interactive session, an inherent basic test + * carrying out some sanity check on the clock + * trace is undertaken. + * @PORTAL_CMD_INVOKE_NOTIFY_42KHZ: Invokes all clock rate trace manager callbacks + * for the top clock domain with a new GPU frequency + * set to 42 kHZ. + * @PORTAL_CMD_INVALID: Valid commands termination marker. Must be + * the highest enumeration value, as it + * represents valid command array size. + * @PORTAL_TOTAL_CMDS: Alias of PORTAL_CMD_INVALID. + */ +/* PORTAL_CMD_INVALID must be the last one, serving the size */ +enum kbasep_clk_rate_trace_req { + PORTAL_CMD_GET_CLK_RATE_MGR, + PORTAL_CMD_GET_CLK_RATE_TRACE, + PORTAL_CMD_GET_TRACE_SNAPSHOT, + PORTAL_CMD_INC_PM_CTX_CNT, + PORTAL_CMD_DEC_PM_CTX_CNT, + PORTAL_CMD_CLOSE_PORTAL, + PORTAL_CMD_INVOKE_NOTIFY_42KHZ, + PORTAL_CMD_INVALID, + PORTAL_TOTAL_CMDS = PORTAL_CMD_INVALID, +}; + +/** + * Portal service request command names. The portal request consists of a kutf + * named u64-value. For those above enumerated PORTAL_CMD, the names defined + * here are used to mark the name and then followed with a sequence number + * value. Example (manual script here for illustration): + * exec 5<>run # open the portal kutf run as fd-5 + * echo GET_CLK_RATE_MGR=1 >&5 # send the cmd and sequence number 1 + * head -n 1 <&5 # read back the 1-line server reseponse + * ACK="{SEQ:1, RATE:[0x1ad27480], GPU_IDLE:1}" # response string + * echo GET_TRACE_SNAPSHOT=1 >&5 # send the cmd and sequence number 1 + * head -n 1 <&5 # read back the 1-line server reseponse + * ACK="{SEQ:1, SNAPSHOT_ARRAY:[(0x0, 0x1ad27480, 1, 0)]}" + * echo CLOSE_PORTAL=1 >&5 # close the portal + * cat <&5 # read back all the response lines + * ACK="{SEQ:1, PM_CTX_CNT:0}" # response to close command + * KUTF_RESULT_PASS:(explicit pass) # internal sanity test passed. + * exec 5>&- # close the service portal fd. + * + * Expected request command return format: + * GET_CLK_RATE_MGR: ACK="{SEQ:12, RATE:[1080, 1280], GPU_IDLE:1}" + * Note, the above contains 2-clock with rates in [], GPU idle + * GET_CLK_RATE_TRACE: ACK="{SEQ:6, RATE:[0x1ad27480], GPU_IDLE:0}" + * Note, 1-clock with rate in [], GPU not idle + * GET_TRACE_SNAPSHOT: ACK="{SEQ:8, SNAPSHOT_ARRAY:[(0x0, 0x1ad27480, 1, 0)]}" + * Note, 1-clock, (start_rate : 0, last_rate : 0x1ad27480, + * trace_rate_up_count: 1, trace_rate_down_count : 0) + * For the specific sample case here, there is a single rate_trace event + * that yielded a rate increase change. No rate drop event recorded in the + * reporting snapshot duration. + * INC_PM_CTX_CNT: ACK="{SEQ:1, PM_CTX_CNT:1}" + * Note, after the increment, M_CTX_CNT is 1. (i.e. 0 -> 1) + * DEC_PM_CTX_CNT: ACK="{SEQ:3, PM_CTX_CNT:0}" + * Note, after the decrement, PM_CTX_CNT is 0. (i.e. 1 -> 0) + * CLOSE_PORTAL: ACK="{SEQ:1, PM_CTX_CNT:1}" + * Note, at the close, PM_CTX_CNT is 1. The PM_CTX_CNT will internally be + * dropped down to 0 as part of the portal close clean up. + */ +#define GET_CLK_RATE_MGR "GET_CLK_RATE_MGR" +#define GET_CLK_RATE_TRACE "GET_CLK_RATE_TRACE" +#define GET_TRACE_SNAPSHOT "GET_TRACE_SNAPSHOT" +#define INC_PM_CTX_CNT "INC_PM_CTX_CNT" +#define DEC_PM_CTX_CNT "DEC_PM_CTX_CNT" +#define CLOSE_PORTAL "CLOSE_PORTAL" +#define INVOKE_NOTIFY_42KHZ "INVOKE_NOTIFY_42KHZ" + +/** + * Portal service response tag names. The response consists of a kutf + * named string-value. In case of a 'NACK' (negative acknowledgement), it + * can be one of the two formats: + * 1. NACK="{SEQ:2, MSG:xyzed}" # NACK on command with sequence tag-2. + * Note, the portal has received a valid name and valid sequence number + * but can't carry-out the request, reason in the MSG field. + * 2. NACK="Failing-message" + * Note, unable to parse a valid name or valid sequence number, + * or some internal error condition. Reason in the quoted string. + */ +#define ACK "ACK" +#define NACK "NACK" +#define MAX_REPLY_NAME_LEN 32 + +#endif /* _KUTF_CLK_RATE_TRACE_TEST_H_ */ diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kbuild b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kbuild new file mode 100755 index 000000000000..ca8c51273b4c --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kbuild @@ -0,0 +1,26 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android + +obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test.o + +mali_kutf_irq_test-y := mali_kutf_irq_test_main.o diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kconfig b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kconfig new file mode 100755 index 000000000000..78283307713d --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Kconfig @@ -0,0 +1,29 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +config MALI_IRQ_LATENCY + tristate "Mali GPU IRQ latency measurement" + depends on MALI_BIFROST && MALI_BIFROST_DEBUG && MALI_KUTF + default m + help + This option will build a test module mali_kutf_irq_test that + can determine the latency of the Mali GPU IRQ on your system. + Choosing M here will generate a single module called mali_kutf_irq_test. diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile new file mode 100755 index 000000000000..bc4d654a90ca --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile @@ -0,0 +1,51 @@ +# +# (C) COPYRIGHT 2015, 2017-2018, 2020 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +# linux build system bootstrap for out-of-tree module + +# default to building for the host +ARCH ?= $(shell uname -m) + +ifeq ($(KDIR),) +$(error Must specify KDIR to point to the kernel to target)) +endif + +TEST_CCFLAGS := \ + -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ + -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ + -DMALI_USE_CSF=$(MALI_USE_CSF) \ + $(SCONS_CFLAGS) \ + -I$(CURDIR)/../include \ + -I$(CURDIR)/../../../../../../include \ + -I$(CURDIR)/../../../ \ + -I$(CURDIR)/../../ \ + -I$(CURDIR)/../../backend/gpu \ + -I$(CURDIR)/../../debug \ + -I$(CURDIR)/../../debug/backend \ + -I$(CURDIR)/ \ + -I$(srctree)/drivers/staging/android \ + -I$(srctree)/include/linux + +all: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS="$(TEST_CCFLAGS)" KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../kutf/Module.symvers $(CURDIR)/../../Module.symvers" modules + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/build.bp b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/build.bp new file mode 100755 index 000000000000..90efdcf9ad9c --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/build.bp @@ -0,0 +1,35 @@ +/* + * + * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + +bob_kernel_module { + name: "mali_kutf_irq_test", + defaults: [ + "mali_kbase_shared_config_defaults", + "kernel_test_includes", + ], + srcs: [ + "Kbuild", + "mali_kutf_irq_test_main.c", + ], + extra_symbols: [ + "mali_kbase", + "kutf", + ], + enabled: false, + base_build_kutf: { + enabled: true, + kbuild_options: ["CONFIG_MALI_IRQ_LATENCY=m"], + }, +} diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c new file mode 100755 index 000000000000..5f27c3a7e9b2 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c @@ -0,0 +1,278 @@ +/* + * + * (C) COPYRIGHT 2016-2018, 2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include + +#include "mali_kbase.h" +#include +#include + +#include +#include + +/* + * This file contains the code which is used for measuring interrupt latency + * of the Mali GPU IRQ. In particular, function mali_kutf_irq_latency() is + * used with this purpose and it is called within KUTF framework - a kernel + * unit test framework. The measured latency provided by this test should + * be representative for the latency of the Mali JOB/MMU IRQs as well. + */ + +/* KUTF test application pointer for this test */ +struct kutf_application *irq_app; + +/** + * struct kutf_irq_fixture data - test fixture used by the test functions. + * @kbdev: kbase device for the GPU. + * + */ +struct kutf_irq_fixture_data { + struct kbase_device *kbdev; +}; + +#define SEC_TO_NANO(s) ((s)*1000000000LL) + +/* ID for the GPU IRQ */ +#define GPU_IRQ_HANDLER 2 + +#define NR_TEST_IRQS ((u32)1000000) + +/* IRQ for the test to trigger. Currently POWER_CHANGED_SINGLE as it is + * otherwise unused in the DDK + */ +#define TEST_IRQ POWER_CHANGED_SINGLE + +#define IRQ_TIMEOUT HZ + +/* Kernel API for setting irq throttle hook callback and irq time in us*/ +extern int kbase_set_custom_irq_handler(struct kbase_device *kbdev, + irq_handler_t custom_handler, + int irq_type); +extern irqreturn_t kbase_gpu_irq_test_handler(int irq, void *data, u32 val); + +static DECLARE_WAIT_QUEUE_HEAD(wait); +static bool triggered; +static u64 irq_time; + +static void *kbase_untag(void *ptr) +{ + return (void *)(((uintptr_t) ptr) & ~3); +} + +/** + * kbase_gpu_irq_custom_handler - Custom IRQ throttle handler + * @irq: IRQ number + * @data: Data associated with this IRQ + * + * Return: state of the IRQ + */ +static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) +{ + struct kbase_device *kbdev = kbase_untag(data); + u32 val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS)); + irqreturn_t result; + u64 tval; + bool has_test_irq = val & TEST_IRQ; + + if (has_test_irq) { + tval = ktime_get_real_ns(); + /* Clear the test source only here */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), + TEST_IRQ); + /* Remove the test IRQ status bit */ + val = val ^ TEST_IRQ; + } + + result = kbase_gpu_irq_test_handler(irq, data, val); + + if (has_test_irq) { + irq_time = tval; + triggered = true; + wake_up(&wait); + result = IRQ_HANDLED; + } + + return result; +} + +/** + * mali_kutf_irq_default_create_fixture() - Creates the fixture data required + * for all the tests in the irq suite. + * @context: KUTF context. + * + * Return: Fixture data created on success or NULL on failure + */ +static void *mali_kutf_irq_default_create_fixture( + struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data; + + data = kutf_mempool_alloc(&context->fixture_pool, + sizeof(struct kutf_irq_fixture_data)); + + if (!data) + goto fail; + + /* Acquire the kbase device */ + data->kbdev = kbase_find_device(-1); + if (data->kbdev == NULL) { + kutf_test_fail(context, "Failed to find kbase device"); + goto fail; + } + + return data; + +fail: + return NULL; +} + +/** + * mali_kutf_irq_default_remove_fixture() - Destroy fixture data previously + * created by mali_kutf_irq_default_create_fixture. + * + * @context: KUTF context. + */ +static void mali_kutf_irq_default_remove_fixture( + struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + + kbase_release_device(kbdev); +} + +/** + * mali_kutf_irq_latency() - measure GPU IRQ latency + * @context: kutf context within which to perform the test + * + * The test triggers IRQs manually, and measures the + * time between triggering the IRQ and the IRQ handler being executed. + * + * This is not a traditional test, in that the pass/fail status has little + * meaning (other than indicating that the IRQ handler executed at all). Instead + * the results are in the latencies provided with the test result. There is no + * meaningful pass/fail result that can be obtained here, instead the latencies + * are provided for manual analysis only. + */ +static void mali_kutf_irq_latency(struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + u64 min_time = U64_MAX, max_time = 0, average_time = 0; + u32 i; + const char *results; + + /* Force GPU to be powered */ + kbase_pm_context_active(kbdev); + kbase_pm_wait_for_desired_state(kbdev); + + kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler, + GPU_IRQ_HANDLER); + + for (i = 1; i <= NR_TEST_IRQS; i++) { + u64 start_time = ktime_get_real_ns(); + + triggered = false; + + /* Trigger fake IRQ */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), + TEST_IRQ); + + if (wait_event_timeout(wait, triggered, IRQ_TIMEOUT) == 0) { + /* Wait extra time to see if it would come */ + wait_event_timeout(wait, triggered, 10 * IRQ_TIMEOUT); + break; + } + + if ((irq_time - start_time) < min_time) + min_time = irq_time - start_time; + if ((irq_time - start_time) > max_time) + max_time = irq_time - start_time; + average_time += irq_time - start_time; + + udelay(10); + } + + /* Go back to default handler */ + kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_HANDLER); + + kbase_pm_context_idle(kbdev); + + if (i > NR_TEST_IRQS) { + do_div(average_time, NR_TEST_IRQS); + results = kutf_dsprintf(&context->fixture_pool, + "Min latency = %lldns, Max latency = %lldns, Average latency = %lldns\n", + min_time, max_time, average_time); + kutf_test_pass(context, results); + } else { + results = kutf_dsprintf(&context->fixture_pool, + "Timed out for the %u-th IRQ (loop_limit: %u), triggered late: %d\n", + i, NR_TEST_IRQS, triggered); + kutf_test_fail(context, results); + } +} + +/** + * Module entry point for this test. + */ +int mali_kutf_irq_test_main_init(void) +{ + struct kutf_suite *suite; + + irq_app = kutf_create_application("irq"); + + if (NULL == irq_app) { + pr_warn("Creation of test application failed!\n"); + return -ENOMEM; + } + + suite = kutf_create_suite(irq_app, "irq_default", + 1, mali_kutf_irq_default_create_fixture, + mali_kutf_irq_default_remove_fixture); + + if (NULL == suite) { + pr_warn("Creation of test suite failed!\n"); + kutf_destroy_application(irq_app); + return -ENOMEM; + } + + kutf_add_test(suite, 0x0, "irq_latency", + mali_kutf_irq_latency); + return 0; +} + +/** + * Module exit point for this test. + */ +void mali_kutf_irq_test_main_exit(void) +{ + kutf_destroy_application(irq_app); +} + +module_init(mali_kutf_irq_test_main_init); +module_exit(mali_kutf_irq_test_main_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("ARM Ltd."); +MODULE_VERSION("1.0"); diff --git a/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c b/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c new file mode 100755 index 000000000000..cd90ea0ec285 --- /dev/null +++ b/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c @@ -0,0 +1,368 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + *//* + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "linux/mman.h" +#include "../mali_kbase.h" + +/* mali_kbase_mmap.c + * + * This file contains Linux specific implementation of + * kbase_context_get_unmapped_area() interface. + */ + + +/** + * align_and_check() - Align the specified pointer to the provided alignment and + * check that it is still in range. + * @gap_end: Highest possible start address for allocation (end of gap in + * address space) + * @gap_start: Start address of current memory area / gap in address space + * @info: vm_unmapped_area_info structure passed to caller, containing + * alignment, length and limits for the allocation + * @is_shader_code: True if the allocation is for shader code (which has + * additional alignment requirements) + * @is_same_4gb_page: True if the allocation needs to reside completely within + * a 4GB chunk + * + * Return: true if gap_end is now aligned correctly and is still in range, + * false otherwise + */ +static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, + struct vm_unmapped_area_info *info, bool is_shader_code, + bool is_same_4gb_page) +{ + /* Compute highest gap address at the desired alignment */ + (*gap_end) -= info->length; + (*gap_end) -= (*gap_end - info->align_offset) & info->align_mask; + + if (is_shader_code) { + /* Check for 4GB boundary */ + if (0 == (*gap_end & BASE_MEM_MASK_4GB)) + (*gap_end) -= (info->align_offset ? info->align_offset : + info->length); + if (0 == ((*gap_end + info->length) & BASE_MEM_MASK_4GB)) + (*gap_end) -= (info->align_offset ? info->align_offset : + info->length); + + if (!(*gap_end & BASE_MEM_MASK_4GB) || !((*gap_end + + info->length) & BASE_MEM_MASK_4GB)) + return false; + } else if (is_same_4gb_page) { + unsigned long start = *gap_end; + unsigned long end = *gap_end + info->length; + unsigned long mask = ~((unsigned long)U32_MAX); + + /* Check if 4GB boundary is straddled */ + if ((start & mask) != ((end - 1) & mask)) { + unsigned long offset = end - (end & mask); + /* This is to ensure that alignment doesn't get + * disturbed in an attempt to prevent straddling at + * 4GB boundary. The GPU VA is aligned to 2MB when the + * allocation size is > 2MB and there is enough CPU & + * GPU virtual space. + */ + unsigned long rounded_offset = + ALIGN(offset, info->align_mask + 1); + + start -= rounded_offset; + end -= rounded_offset; + + *gap_end = start; + + /* The preceding 4GB boundary shall not get straddled, + * even after accounting for the alignment, as the + * size of allocation is limited to 4GB and the initial + * start location was already aligned. + */ + WARN_ON((start & mask) != ((end - 1) & mask)); + } + } + + + if ((*gap_end < info->low_limit) || (*gap_end < gap_start)) + return false; + + + return true; +} + +/** + * kbase_unmapped_area_topdown() - allocates new areas top-down from + * below the stack limit. + * @info: Information about the memory area to allocate. + * @is_shader_code: Boolean which denotes whether the allocated area is + * intended for the use by shader core in which case a + * special alignment requirements apply. + * @is_same_4gb_page: Boolean which indicates whether the allocated area needs + * to reside completely within a 4GB chunk. + * + * The unmapped_area_topdown() function in the Linux kernel is not exported + * using EXPORT_SYMBOL_GPL macro. To allow us to call this function from a + * module and also make use of the fact that some of the requirements for + * the unmapped area are known in advance, we implemented an extended version + * of this function and prefixed it with 'kbase_'. + * + * The difference in the call parameter list comes from the fact that + * kbase_unmapped_area_topdown() is called with additional parameters which + * are provided to indicate whether the allocation is for a shader core memory, + * which has additional alignment requirements, and whether the allocation can + * straddle a 4GB boundary. + * + * The modification of the original Linux function lies in how the computation + * of the highest gap address at the desired alignment is performed once the + * gap with desirable properties is found. For this purpose a special function + * is introduced (@ref align_and_check()) which beside computing the gap end + * at the desired alignment also performs additional alignment checks for the + * case when the memory is executable shader core memory, for which it is + * ensured that the gap does not end on a 4GB boundary, and for the case when + * memory needs to be confined within a 4GB chunk. + * + * Return: address of the found gap end (high limit) if area is found; + * -ENOMEM if search is unsuccessful +*/ + +static unsigned long kbase_unmapped_area_topdown(struct vm_unmapped_area_info + *info, bool is_shader_code, bool is_same_4gb_page) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + unsigned long length, low_limit, high_limit, gap_start, gap_end; + + /* Adjust search length to account for worst case alignment overhead */ + length = info->length + info->align_mask; + if (length < info->length) + return -ENOMEM; + + /* + * Adjust search limits by the desired length. + * See implementation comment at top of unmapped_area(). + */ + gap_end = info->high_limit; + if (gap_end < length) + return -ENOMEM; + high_limit = gap_end - length; + + if (info->low_limit > high_limit) + return -ENOMEM; + low_limit = info->low_limit + length; + + /* Check highest gap, which does not precede any rbtree node */ + gap_start = mm->highest_vm_end; + if (gap_start <= high_limit) { + if (align_and_check(&gap_end, gap_start, info, + is_shader_code, is_same_4gb_page)) + return gap_end; + } + + /* Check if rbtree root looks promising */ + if (RB_EMPTY_ROOT(&mm->mm_rb)) + return -ENOMEM; + vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); + if (vma->rb_subtree_gap < length) + return -ENOMEM; + + while (true) { + /* Visit right subtree if it looks promising */ + gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0; + if (gap_start <= high_limit && vma->vm_rb.rb_right) { + struct vm_area_struct *right = + rb_entry(vma->vm_rb.rb_right, + struct vm_area_struct, vm_rb); + if (right->rb_subtree_gap >= length) { + vma = right; + continue; + } + } + +check_current: + /* Check if current node has a suitable gap */ + gap_end = vma->vm_start; + if (gap_end < low_limit) + return -ENOMEM; + if (gap_start <= high_limit && gap_end - gap_start >= length) { + /* We found a suitable gap. Clip it with the original + * high_limit. */ + if (gap_end > info->high_limit) + gap_end = info->high_limit; + + if (align_and_check(&gap_end, gap_start, info, + is_shader_code, is_same_4gb_page)) + return gap_end; + } + + /* Visit left subtree if it looks promising */ + if (vma->vm_rb.rb_left) { + struct vm_area_struct *left = + rb_entry(vma->vm_rb.rb_left, + struct vm_area_struct, vm_rb); + if (left->rb_subtree_gap >= length) { + vma = left; + continue; + } + } + + /* Go back up the rbtree to find next candidate node */ + while (true) { + struct rb_node *prev = &vma->vm_rb; + + if (!rb_parent(prev)) + return -ENOMEM; + vma = rb_entry(rb_parent(prev), + struct vm_area_struct, vm_rb); + if (prev == vma->vm_rb.rb_right) { + gap_start = vma->vm_prev ? + vma->vm_prev->vm_end : 0; + goto check_current; + } + } + } + + return -ENOMEM; +} + + +/* This function is based on Linux kernel's arch_get_unmapped_area, but + * simplified slightly. Modifications come from the fact that some values + * about the memory area are known in advance. + */ +unsigned long kbase_context_get_unmapped_area(struct kbase_context *const kctx, + const unsigned long addr, const unsigned long len, + const unsigned long pgoff, const unsigned long flags) +{ + struct mm_struct *mm = current->mm; + struct vm_unmapped_area_info info; + unsigned long align_offset = 0; + unsigned long align_mask = 0; + unsigned long high_limit = mm->mmap_base; + unsigned long low_limit = PAGE_SIZE; + int cpu_va_bits = BITS_PER_LONG; + int gpu_pc_bits = + kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; + bool is_shader_code = false; + bool is_same_4gb_page = false; + unsigned long ret; + + /* err on fixed address */ + if ((flags & MAP_FIXED) || addr) + return -EINVAL; + +#ifdef CONFIG_64BIT + /* too big? */ + if (len > TASK_SIZE - SZ_2M) + return -ENOMEM; + + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + + high_limit = min_t(unsigned long, mm->mmap_base, + (kctx->same_va_end << PAGE_SHIFT)); + + /* If there's enough (> 33 bits) of GPU VA space, align + * to 2MB boundaries. + */ + if (kctx->kbdev->gpu_props.mmu.va_bits > 33) { + if (len >= SZ_2M) { + align_offset = SZ_2M; + align_mask = SZ_2M - 1; + } + } + + low_limit = SZ_2M; + } else { + cpu_va_bits = 32; + } +#endif /* CONFIG_64BIT */ + if ((PFN_DOWN(BASE_MEM_COOKIE_BASE) <= pgoff) && + (PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) > pgoff)) { + int cookie = pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); + struct kbase_va_region *reg; + + /* Need to hold gpu vm lock when using reg */ + kbase_gpu_vm_lock(kctx); + reg = kctx->pending_regions[cookie]; + if (!reg) { + kbase_gpu_vm_unlock(kctx); + return -EINVAL; + } + if (!(reg->flags & KBASE_REG_GPU_NX)) { + if (cpu_va_bits > gpu_pc_bits) { + align_offset = 1ULL << gpu_pc_bits; + align_mask = align_offset - 1; + is_shader_code = true; + } +#if !MALI_USE_CSF + } else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { + unsigned long extent_bytes = + (unsigned long)(reg->extent << PAGE_SHIFT); + /* kbase_check_alloc_sizes() already satisfies + * these checks, but they're here to avoid + * maintenance hazards due to the assumptions + * involved */ + WARN_ON(reg->extent > (ULONG_MAX >> PAGE_SHIFT)); + WARN_ON(reg->initial_commit > (ULONG_MAX >> PAGE_SHIFT)); + WARN_ON(!is_power_of_2(extent_bytes)); + align_mask = extent_bytes - 1; + align_offset = + extent_bytes - (reg->initial_commit << PAGE_SHIFT); +#endif /* !MALI_USE_CSF */ + } else if (reg->flags & KBASE_REG_GPU_VA_SAME_4GB_PAGE) { + is_same_4gb_page = true; + } + kbase_gpu_vm_unlock(kctx); +#ifndef CONFIG_64BIT + } else { + return current->mm->get_unmapped_area( + kctx->filp, addr, len, pgoff, flags); +#endif + } + + info.flags = 0; + info.length = len; + info.low_limit = low_limit; + info.high_limit = high_limit; + info.align_offset = align_offset; + info.align_mask = align_mask; + + ret = kbase_unmapped_area_topdown(&info, is_shader_code, + is_same_4gb_page); + + if (IS_ERR_VALUE(ret) && high_limit == mm->mmap_base && + high_limit < (kctx->same_va_end << PAGE_SHIFT)) { + /* Retry above mmap_base */ + info.low_limit = mm->mmap_base; + info.high_limit = min_t(u64, TASK_SIZE, + (kctx->same_va_end << PAGE_SHIFT)); + + ret = kbase_unmapped_area_topdown(&info, is_shader_code, + is_same_4gb_page); + } + + return ret; +} diff --git a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c new file mode 100755 index 000000000000..abaa6bb12b9d --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c @@ -0,0 +1,172 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "../mali_kbase_tracepoints.h" +#include "../mali_kbase_timeline.h" +#include "../mali_kbase_timeline_priv.h" + +#include + +void kbase_create_timeline_objects(struct kbase_device *kbdev) +{ + unsigned int as_nr; + unsigned int slot_i; + struct kbase_context *kctx; + struct kbase_timeline *timeline = kbdev->timeline; + struct kbase_tlstream *summary = + &kbdev->timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]; + + /* Summarize the Address Space objects. */ + for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) + __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], as_nr); + + /* Create Legacy GPU object to track in AOM for dumping */ + __kbase_tlstream_tl_new_gpu(summary, + kbdev, + kbdev->gpu_props.props.raw_props.gpu_id, + kbdev->gpu_props.num_cores); + + + for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) + __kbase_tlstream_tl_lifelink_as_gpu(summary, + &kbdev->as[as_nr], + kbdev); + + /* Trace the creation of a new kbase device and set its properties. */ + __kbase_tlstream_tl_kbase_new_device(summary, + kbdev->gpu_props.props.raw_props.gpu_id, + kbdev->gpu_props.num_cores, kbdev->csf.global_iface.group_num, + kbdev->nr_hw_address_spaces); + + /* Lock the context list, to ensure no changes to the list are made + * while we're summarizing the contexts and their contents. + */ + mutex_lock(&kbdev->kctx_list_lock); + + /* Hold the scheduler lock while we emit the current state + * We also need to continue holding the lock until after the first body + * stream tracepoints are emitted to ensure we don't change the + * scheduler until after then + */ + mutex_lock(&kbdev->csf.scheduler.lock); + + for (slot_i = 0; slot_i < kbdev->csf.global_iface.group_num; slot_i++) { + + struct kbase_queue_group *group = + kbdev->csf.scheduler.csg_slots[slot_i].resident_group; + + if (group) + __kbase_tlstream_tl_kbase_device_program_csg(summary, + kbdev->gpu_props.props.raw_props.gpu_id, + group->handle, slot_i); + } + + /* Reset body stream buffers while holding the kctx lock. + * As we are holding the lock, we can guarantee that no kctx creation or + * deletion tracepoints can be fired from outside of this function by + * some other thread. + */ + kbase_timeline_streams_body_reset(timeline); + + mutex_unlock(&kbdev->csf.scheduler.lock); + + /* For each context in the device... */ + list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { + size_t i; + struct kbase_tlstream *body = + &timeline->streams[TL_STREAM_TYPE_OBJ]; + + /* Lock the context's KCPU queues, to ensure no KCPU-queue + * related actions can occur in this context from now on. + */ + mutex_lock(&kctx->csf.kcpu_queues.lock); + + /* Acquire the MMU lock, to ensure we don't get a concurrent + * address space assignment while summarizing this context's + * address space. + */ + mutex_lock(&kbdev->mmu_hw_mutex); + + /* Trace the context itself into the body stream, not the + * summary stream. + * We place this in the body to ensure it is ordered after any + * other tracepoints related to the contents of the context that + * might have been fired before acquiring all of the per-context + * locks. + * This ensures that those tracepoints will not actually affect + * the object model state, as they reference a context that + * hasn't been traced yet. They may, however, cause benign + * errors to be emitted. + */ + __kbase_tlstream_tl_kbase_new_ctx(body, kctx->id, + kbdev->gpu_props.props.raw_props.gpu_id); + + /* Also trace with the legacy AOM tracepoint for dumping */ + __kbase_tlstream_tl_new_ctx(body, + kctx, + kctx->id, + (u32)(kctx->tgid)); + + /* Trace the currently assigned address space */ + if (kctx->as_nr != KBASEP_AS_NR_INVALID) + __kbase_tlstream_tl_kbase_ctx_assign_as(body, kctx->id, + kctx->as_nr); + + + /* Trace all KCPU queues in the context into the body stream. + * As we acquired the KCPU lock after resetting the body stream, + * it's possible that some KCPU-related events for this context + * occurred between that reset and now. + * These will cause errors to be emitted when parsing the + * timeline, but they will not affect the correctness of the + * object model. + */ + for (i = 0; i < KBASEP_MAX_KCPU_QUEUES; i++) { + const struct kbase_kcpu_command_queue *kcpu_queue = + kctx->csf.kcpu_queues.array[i]; + + if (kcpu_queue) + __kbase_tlstream_tl_kbase_new_kcpuqueue( + body, kcpu_queue, kcpu_queue->kctx->id, + kcpu_queue->num_pending_cmds); + } + + mutex_unlock(&kbdev->mmu_hw_mutex); + mutex_unlock(&kctx->csf.kcpu_queues.lock); + + /* Now that all per-context locks for this context have been + * released, any per-context tracepoints that are fired from + * any other threads will go into the body stream after + * everything that was just summarised into the body stream in + * this iteration of the loop, so will start to correctly update + * the object model state. + */ + }; + + mutex_unlock(&kbdev->kctx_list_lock); + + /* Static object are placed into summary packet that needs to be + * transmitted first. Flush all streams to make it available to + * user space. + */ + kbase_timeline_streams_flush(timeline); +} diff --git a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c new file mode 100755 index 000000000000..c368ac7288da --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c @@ -0,0 +1,97 @@ +/* + * + * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "../mali_kbase_tracepoints.h" +#include "../mali_kbase_timeline.h" +#include "../mali_kbase_timeline_priv.h" + +#include + +void kbase_create_timeline_objects(struct kbase_device *kbdev) +{ + unsigned int lpu_id; + unsigned int as_nr; + struct kbase_context *kctx; + struct kbase_timeline *timeline = kbdev->timeline; + struct kbase_tlstream *summary = + &timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]; + + /* Summarize the LPU objects. */ + for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { + u32 *lpu = + &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; + __kbase_tlstream_tl_new_lpu(summary, lpu, lpu_id, *lpu); + } + + /* Summarize the Address Space objects. */ + for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) + __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], as_nr); + + /* Create GPU object and make it retain all LPUs and address spaces. */ + __kbase_tlstream_tl_new_gpu(summary, + kbdev, + kbdev->gpu_props.props.raw_props.gpu_id, + kbdev->gpu_props.num_cores); + + for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { + void *lpu = + &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; + __kbase_tlstream_tl_lifelink_lpu_gpu(summary, lpu, kbdev); + } + + for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) + __kbase_tlstream_tl_lifelink_as_gpu(summary, + &kbdev->as[as_nr], + kbdev); + + /* Lock the context list, to ensure no changes to the list are made + * while we're summarizing the contexts and their contents. + */ + mutex_lock(&kbdev->kctx_list_lock); + + /* For each context in the device... */ + list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { + /* Summarize the context itself */ + __kbase_tlstream_tl_new_ctx(summary, + kctx, + kctx->id, + (u32)(kctx->tgid)); + }; + + /* Reset body stream buffers while holding the kctx lock. + * This ensures we can't fire both summary and normal tracepoints for + * the same objects. + * If we weren't holding the lock, it's possible that the summarized + * objects could have been created, destroyed, or used after we + * constructed the summary stream tracepoints, but before we reset + * the body stream, resulting in losing those object event tracepoints. + */ + kbase_timeline_streams_body_reset(timeline); + + mutex_unlock(&kbdev->kctx_list_lock); + + /* Static object are placed into summary packet that needs to be + * transmitted first. Flush all streams to make it available to + * user space. + */ + kbase_timeline_streams_flush(timeline); +} \ No newline at end of file diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c new file mode 100755 index 000000000000..8d8834fdcda6 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c @@ -0,0 +1,308 @@ +/* + * + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_timeline.h" +#include "mali_kbase_timeline_priv.h" +#include "mali_kbase_tracepoints.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/* The period of autoflush checker execution in milliseconds. */ +#define AUTOFLUSH_INTERVAL 1000 /* ms */ + +/*****************************************************************************/ + +/* These values are used in mali_kbase_tracepoints.h + * to retrieve the streams from a kbase_timeline instance. + */ +const size_t __obj_stream_offset = + offsetof(struct kbase_timeline, streams) + + sizeof(struct kbase_tlstream) * TL_STREAM_TYPE_OBJ; + +const size_t __aux_stream_offset = + offsetof(struct kbase_timeline, streams) + + sizeof(struct kbase_tlstream) * TL_STREAM_TYPE_AUX; + +/** + * kbasep_timeline_autoflush_timer_callback - autoflush timer callback + * @timer: Timer list + * + * Timer is executed periodically to check if any of the stream contains + * buffer ready to be submitted to user space. + */ +static void kbasep_timeline_autoflush_timer_callback(struct timer_list *timer) +{ + enum tl_stream_type stype; + int rcode; + struct kbase_timeline *timeline = + container_of(timer, struct kbase_timeline, autoflush_timer); + + CSTD_UNUSED(timer); + + for (stype = (enum tl_stream_type)0; stype < TL_STREAM_TYPE_COUNT; + stype++) { + struct kbase_tlstream *stream = &timeline->streams[stype]; + + int af_cnt = atomic_read(&stream->autoflush_counter); + + /* Check if stream contain unflushed data. */ + if (af_cnt < 0) + continue; + + /* Check if stream should be flushed now. */ + if (af_cnt != atomic_cmpxchg( + &stream->autoflush_counter, + af_cnt, + af_cnt + 1)) + continue; + if (!af_cnt) + continue; + + /* Autoflush this stream. */ + kbase_tlstream_flush_stream(stream); + } + + if (atomic_read(&timeline->autoflush_timer_active)) + rcode = mod_timer( + &timeline->autoflush_timer, + jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); + CSTD_UNUSED(rcode); +} + + + +/*****************************************************************************/ + +int kbase_timeline_init(struct kbase_timeline **timeline, + atomic_t *timeline_flags) +{ + enum tl_stream_type i; + struct kbase_timeline *result; +#if MALI_USE_CSF + struct kbase_tlstream *csffw_stream; +#endif + + if (!timeline || !timeline_flags) + return -EINVAL; + + result = kzalloc(sizeof(*result), GFP_KERNEL); + if (!result) + return -ENOMEM; + + mutex_init(&result->reader_lock); + init_waitqueue_head(&result->event_queue); + + /* Prepare stream structures. */ + for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) + kbase_tlstream_init(&result->streams[i], i, + &result->event_queue); + + /* Initialize autoflush timer. */ + atomic_set(&result->autoflush_timer_active, 0); + kbase_timer_setup(&result->autoflush_timer, + kbasep_timeline_autoflush_timer_callback); + result->timeline_flags = timeline_flags; + +#if MALI_USE_CSF + csffw_stream = &result->streams[TL_STREAM_TYPE_CSFFW]; + kbase_csf_tl_reader_init(&result->csf_tl_reader, csffw_stream); +#endif + + *timeline = result; + return 0; +} + +void kbase_timeline_term(struct kbase_timeline *timeline) +{ + enum tl_stream_type i; + + if (!timeline) + return; + +#if MALI_USE_CSF + kbase_csf_tl_reader_term(&timeline->csf_tl_reader); +#endif + + for (i = (enum tl_stream_type)0; i < TL_STREAM_TYPE_COUNT; i++) + kbase_tlstream_term(&timeline->streams[i]); + + kfree(timeline); +} + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ +static void kbase_tlstream_current_devfreq_target(struct kbase_device *kbdev) +{ + struct devfreq *devfreq = kbdev->devfreq; + + /* Devfreq initialization failure isn't a fatal error, so devfreq might + * be null. + */ + if (devfreq) { + unsigned long cur_freq = 0; + + mutex_lock(&devfreq->lock); +#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE + cur_freq = kbdev->current_nominal_freq; +#else + cur_freq = devfreq->last_status.current_frequency; +#endif + KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(kbdev, (u64)cur_freq); + mutex_unlock(&devfreq->lock); + } +} +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + +int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags) +{ + int ret; + u32 timeline_flags = TLSTREAM_ENABLED | flags; + struct kbase_timeline *timeline = kbdev->timeline; + + if (!atomic_cmpxchg(timeline->timeline_flags, 0, timeline_flags)) { + int rcode; + +#if MALI_USE_CSF + if (flags & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) { + ret = kbase_csf_tl_reader_start( + &timeline->csf_tl_reader, kbdev); + if (ret) + { + atomic_set(timeline->timeline_flags, 0); + return ret; + } + } +#endif + ret = anon_inode_getfd( + "[mali_tlstream]", + &kbasep_tlstream_fops, + timeline, + O_RDONLY | O_CLOEXEC); + if (ret < 0) { + atomic_set(timeline->timeline_flags, 0); +#if MALI_USE_CSF + kbase_csf_tl_reader_stop(&timeline->csf_tl_reader); +#endif + return ret; + } + + /* Reset and initialize header streams. */ + kbase_tlstream_reset( + &timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]); + + timeline->obj_header_btc = obj_desc_header_size; + timeline->aux_header_btc = aux_desc_header_size; + + /* Start autoflush timer. */ + atomic_set(&timeline->autoflush_timer_active, 1); + rcode = mod_timer( + &timeline->autoflush_timer, + jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); + CSTD_UNUSED(rcode); + +#if !MALI_USE_CSF + /* If job dumping is enabled, readjust the software event's + * timeout as the default value of 3 seconds is often + * insufficient. + */ + if (flags & BASE_TLSTREAM_JOB_DUMPING_ENABLED) { + dev_info(kbdev->dev, + "Job dumping is enabled, readjusting the software event's timeout\n"); + atomic_set(&kbdev->js_data.soft_job_timeout_ms, + 1800000); + } +#endif /* !MALI_USE_CSF */ + + /* Summary stream was cleared during acquire. + * Create static timeline objects that will be + * read by client. + */ + kbase_create_timeline_objects(kbdev); + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + /* Devfreq target tracepoints are only fired when the target + * changes, so we won't know the current target unless we + * send it now. + */ + kbase_tlstream_current_devfreq_target(kbdev); +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + + } else { + ret = -EBUSY; + } + + return ret; +} + +void kbase_timeline_streams_flush(struct kbase_timeline *timeline) +{ + enum tl_stream_type stype; + +#if MALI_USE_CSF + kbase_csf_tl_reader_flush_buffer(&timeline->csf_tl_reader); +#endif + + for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) + kbase_tlstream_flush_stream(&timeline->streams[stype]); +} + +void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline) +{ + kbase_tlstream_reset( + &timeline->streams[TL_STREAM_TYPE_OBJ]); + kbase_tlstream_reset( + &timeline->streams[TL_STREAM_TYPE_AUX]); +#if MALI_USE_CSF + kbase_tlstream_reset( + &timeline->streams[TL_STREAM_TYPE_CSFFW]); +#endif +} + +#if MALI_UNIT_TEST +void kbase_timeline_stats(struct kbase_timeline *timeline, + u32 *bytes_collected, u32 *bytes_generated) +{ + enum tl_stream_type stype; + + KBASE_DEBUG_ASSERT(bytes_collected); + + /* Accumulate bytes generated per stream */ + *bytes_generated = 0; + for (stype = (enum tl_stream_type)0; stype < TL_STREAM_TYPE_COUNT; + stype++) + *bytes_generated += atomic_read( + &timeline->streams[stype].bytes_generated); + + *bytes_collected = atomic_read(&timeline->bytes_collected); +} +#endif /* MALI_UNIT_TEST */ diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.h new file mode 100755 index 000000000000..cd48411b45cf --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.h @@ -0,0 +1,121 @@ +/* + * + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#if !defined(_KBASE_TIMELINE_H) +#define _KBASE_TIMELINE_H + +#include + +/*****************************************************************************/ + +struct kbase_timeline; + +/** + * kbase_timeline_init - initialize timeline infrastructure in kernel + * @timeline: Newly created instance of kbase_timeline will be stored in + * this pointer. + * @timeline_flags: Timeline status will be written to this variable when a + * client is attached/detached. The variable must be valid + * while timeline instance is valid. + * Return: zero on success, negative number on error + */ +int kbase_timeline_init(struct kbase_timeline **timeline, + atomic_t *timeline_flags); + +/** + * kbase_timeline_term - terminate timeline infrastructure in kernel + * + * @timeline: Timeline instance to be terminated. It must be previously created + * with kbase_timeline_init(). + */ +void kbase_timeline_term(struct kbase_timeline *timeline); + +/** + * kbase_timeline_io_acquire - acquire timeline stream file descriptor + * @kbdev: Kbase device + * @flags: Timeline stream flags + * + * This descriptor is meant to be used by userspace timeline to gain access to + * kernel timeline stream. This stream is later broadcasted by user space to the + * timeline client. + * Only one entity can own the descriptor at any given time. Descriptor shall be + * closed if unused. If descriptor cannot be obtained (i.e. when it is already + * being used) return will be a negative value. + * + * Return: file descriptor on success, negative number on error + */ +int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags); + +/** + * kbase_timeline_streams_flush - flush timeline streams. + * @timeline: Timeline instance + * + * Function will flush pending data in all timeline streams. + */ +void kbase_timeline_streams_flush(struct kbase_timeline *timeline); + +/** + * kbase_timeline_streams_body_reset - reset timeline body streams. + * + * Function will discard pending data in all timeline body streams. + * @timeline: Timeline instance + */ +void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline); + +#if MALI_UNIT_TEST +/** + * kbase_timeline_test - start timeline stream data generator + * @kbdev: Kernel common context + * @tpw_count: Number of trace point writers in each context + * @msg_delay: Time delay in milliseconds between trace points written by one + * writer + * @msg_count: Number of trace points written by one writer + * @aux_msg: If non-zero aux messages will be included + * + * This test starts a requested number of asynchronous writers in both IRQ and + * thread context. Each writer will generate required number of test + * tracepoints (tracepoints with embedded information about writer that + * should be verified by user space reader). Tracepoints will be emitted in + * all timeline body streams. If aux_msg is non-zero writer will also + * generate not testable tracepoints (tracepoints without information about + * writer). These tracepoints are used to check correctness of remaining + * timeline message generating functions. Writer will wait requested time + * between generating another set of messages. This call blocks until all + * writers finish. + */ +void kbase_timeline_test( + struct kbase_device *kbdev, + unsigned int tpw_count, + unsigned int msg_delay, + unsigned int msg_count, + int aux_msg); + +/** + * kbase_timeline_stats - read timeline stream statistics + * @timeline: Timeline instance + * @bytes_collected: Will hold number of bytes read by the user + * @bytes_generated: Will hold number of bytes generated by trace points + */ +void kbase_timeline_stats(struct kbase_timeline *timeline, u32 *bytes_collected, u32 *bytes_generated); +#endif /* MALI_UNIT_TEST */ + +#endif /* _KBASE_TIMELINE_H */ diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c new file mode 100755 index 000000000000..724f5fa23725 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c @@ -0,0 +1,362 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_timeline_priv.h" +#include "mali_kbase_tlstream.h" +#include "mali_kbase_tracepoints.h" + +#include + +/* The timeline stream file operations functions. */ +static ssize_t kbasep_timeline_io_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos); +static unsigned int kbasep_timeline_io_poll(struct file *filp, poll_table *wait); +static int kbasep_timeline_io_release(struct inode *inode, struct file *filp); + +/* The timeline stream file operations structure. */ +const struct file_operations kbasep_tlstream_fops = { + .owner = THIS_MODULE, + .release = kbasep_timeline_io_release, + .read = kbasep_timeline_io_read, + .poll = kbasep_timeline_io_poll, +}; + +/** + * kbasep_timeline_io_packet_pending - check timeline streams for pending packets + * @timeline: Timeline instance + * @ready_stream: Pointer to variable where stream will be placed + * @rb_idx_raw: Pointer to variable where read buffer index will be placed + * + * Function checks all streams for pending packets. It will stop as soon as + * packet ready to be submitted to user space is detected. Variables under + * pointers, passed as the parameters to this function will be updated with + * values pointing to right stream and buffer. + * + * Return: non-zero if any of timeline streams has at last one packet ready + */ +static int kbasep_timeline_io_packet_pending( + struct kbase_timeline *timeline, + struct kbase_tlstream **ready_stream, + unsigned int *rb_idx_raw) +{ + enum tl_stream_type i; + + KBASE_DEBUG_ASSERT(ready_stream); + KBASE_DEBUG_ASSERT(rb_idx_raw); + + for (i = (enum tl_stream_type)0; i < TL_STREAM_TYPE_COUNT; ++i) { + struct kbase_tlstream *stream = &timeline->streams[i]; + *rb_idx_raw = atomic_read(&stream->rbi); + /* Read buffer index may be updated by writer in case of + * overflow. Read and write buffer indexes must be + * loaded in correct order. + */ + smp_rmb(); + if (atomic_read(&stream->wbi) != *rb_idx_raw) { + *ready_stream = stream; + return 1; + } + + } + + return 0; +} + +/** + * kbasep_timeline_has_header_data() - + * check timeline headers for pending packets + * + * @timeline: Timeline instance + * + * Return: non-zero if any of timeline headers has at last one packet ready. + */ +static int kbasep_timeline_has_header_data( + struct kbase_timeline *timeline) +{ + return timeline->obj_header_btc + || timeline->aux_header_btc +#if MALI_USE_CSF + || timeline->csf_tl_reader.tl_header.btc +#endif + ; +} + +/** + * copy_stream_header() - copy timeline stream header. + * + * @buffer: Pointer to the buffer provided by user. + * @size: Maximum amount of data that can be stored in the buffer. + * @copy_len: Pointer to amount of bytes that has been copied already + * within the read system call. + * @hdr: Pointer to the stream header. + * @hdr_size: Header size. + * @hdr_btc: Pointer to the remaining number of bytes to copy. + * + * Returns: 0 if success, -1 otherwise. + */ +static inline int copy_stream_header( + char __user *buffer, size_t size, ssize_t *copy_len, + const char *hdr, + size_t hdr_size, + size_t *hdr_btc) +{ + const size_t offset = hdr_size - *hdr_btc; + const size_t copy_size = MIN(size - *copy_len, *hdr_btc); + + if (!*hdr_btc) + return 0; + + if (WARN_ON(*hdr_btc > hdr_size)) + return -1; + + if (copy_to_user(&buffer[*copy_len], &hdr[offset], copy_size)) + return -1; + + *hdr_btc -= copy_size; + *copy_len += copy_size; + + return 0; +} + +/** + * kbasep_timeline_copy_header - copy timeline headers to the user + * @timeline: Timeline instance + * @buffer: Pointer to the buffer provided by user + * @size: Maximum amount of data that can be stored in the buffer + * @copy_len: Pointer to amount of bytes that has been copied already + * within the read system call. + * + * This helper function checks if timeline headers have not been sent + * to the user, and if so, sends them. copy_len is respectively + * updated. + * + * Returns: 0 if success, -1 if copy_to_user has failed. + */ +static inline int kbasep_timeline_copy_headers( + struct kbase_timeline *timeline, + char __user *buffer, + size_t size, + ssize_t *copy_len) +{ + if (copy_stream_header(buffer, size, copy_len, + obj_desc_header, + obj_desc_header_size, + &timeline->obj_header_btc)) + return -1; + + if (copy_stream_header(buffer, size, copy_len, + aux_desc_header, + aux_desc_header_size, + &timeline->aux_header_btc)) + return -1; +#if MALI_USE_CSF + if (copy_stream_header(buffer, size, copy_len, + timeline->csf_tl_reader.tl_header.data, + timeline->csf_tl_reader.tl_header.size, + &timeline->csf_tl_reader.tl_header.btc)) + return -1; +#endif + return 0; +} + + +/** + * kbasep_timeline_io_read - copy data from streams to buffer provided by user + * @filp: Pointer to file structure + * @buffer: Pointer to the buffer provided by user + * @size: Maximum amount of data that can be stored in the buffer + * @f_pos: Pointer to file offset (unused) + * + * Return: number of bytes stored in the buffer + */ +static ssize_t kbasep_timeline_io_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos) +{ + ssize_t copy_len = 0; + struct kbase_timeline *timeline; + + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(f_pos); + + if (WARN_ON(!filp->private_data)) + return -EFAULT; + + timeline = (struct kbase_timeline *) filp->private_data; + + if (!buffer) + return -EINVAL; + + if ((*f_pos < 0) || (size < PACKET_SIZE)) + return -EINVAL; + + mutex_lock(&timeline->reader_lock); + + while (copy_len < size) { + struct kbase_tlstream *stream = NULL; + unsigned int rb_idx_raw = 0; + unsigned int wb_idx_raw; + unsigned int rb_idx; + size_t rb_size; + + if (kbasep_timeline_copy_headers( + timeline, buffer, size, ©_len)) { + copy_len = -EFAULT; + break; + } + + /* If we already read some packets and there is no + * packet pending then return back to user. + * If we don't have any data yet, wait for packet to be + * submitted. + */ + if (copy_len > 0) { + if (!kbasep_timeline_io_packet_pending( + timeline, + &stream, + &rb_idx_raw)) + break; + } else { + if (wait_event_interruptible( + timeline->event_queue, + kbasep_timeline_io_packet_pending( + timeline, + &stream, + &rb_idx_raw))) { + copy_len = -ERESTARTSYS; + break; + } + } + + if (WARN_ON(!stream)) { + copy_len = -EFAULT; + break; + } + + /* Check if this packet fits into the user buffer. + * If so copy its content. + */ + rb_idx = rb_idx_raw % PACKET_COUNT; + rb_size = atomic_read(&stream->buffer[rb_idx].size); + if (rb_size > size - copy_len) + break; + if (copy_to_user( + &buffer[copy_len], + stream->buffer[rb_idx].data, + rb_size)) { + copy_len = -EFAULT; + break; + } + + /* If the distance between read buffer index and write + * buffer index became more than PACKET_COUNT, then overflow + * happened and we need to ignore the last portion of bytes + * that we have just sent to user. + */ + smp_rmb(); + wb_idx_raw = atomic_read(&stream->wbi); + + if (wb_idx_raw - rb_idx_raw < PACKET_COUNT) { + copy_len += rb_size; + atomic_inc(&stream->rbi); +#if MALI_UNIT_TEST + atomic_add(rb_size, &timeline->bytes_collected); +#endif /* MALI_UNIT_TEST */ + + } else { + const unsigned int new_rb_idx_raw = + wb_idx_raw - PACKET_COUNT + 1; + /* Adjust read buffer index to the next valid buffer */ + atomic_set(&stream->rbi, new_rb_idx_raw); + } + } + + mutex_unlock(&timeline->reader_lock); + + return copy_len; +} + +/** + * kbasep_timeline_io_poll - poll timeline stream for packets + * @filp: Pointer to file structure + * @wait: Pointer to poll table + * Return: POLLIN if data can be read without blocking, otherwise zero + */ +static unsigned int kbasep_timeline_io_poll(struct file *filp, poll_table *wait) +{ + struct kbase_tlstream *stream; + unsigned int rb_idx; + struct kbase_timeline *timeline; + + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(wait); + + if (WARN_ON(!filp->private_data)) + return -EFAULT; + + timeline = (struct kbase_timeline *) filp->private_data; + + /* If there are header bytes to copy, read will not block */ + if (kbasep_timeline_has_header_data(timeline)) + return POLLIN; + + poll_wait(filp, &timeline->event_queue, wait); + if (kbasep_timeline_io_packet_pending(timeline, &stream, &rb_idx)) + return POLLIN; + return 0; +} + +/** + * kbasep_timeline_io_release - release timeline stream descriptor + * @inode: Pointer to inode structure + * @filp: Pointer to file structure + * + * Return always return zero + */ +static int kbasep_timeline_io_release(struct inode *inode, struct file *filp) +{ + struct kbase_timeline *timeline; + + KBASE_DEBUG_ASSERT(inode); + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(filp->private_data); + + CSTD_UNUSED(inode); + + timeline = (struct kbase_timeline *) filp->private_data; + +#if MALI_USE_CSF + kbase_csf_tl_reader_stop(&timeline->csf_tl_reader); +#endif + + /* Stop autoflush timer before releasing access to streams. */ + atomic_set(&timeline->autoflush_timer_active, 0); + del_timer_sync(&timeline->autoflush_timer); + + atomic_set(timeline->timeline_flags, 0); + return 0; +} diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_priv.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_priv.h new file mode 100755 index 000000000000..35eec467af90 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_priv.h @@ -0,0 +1,73 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#if !defined(_KBASE_TIMELINE_PRIV_H) +#define _KBASE_TIMELINE_PRIV_H + +#include +#include "mali_kbase_tlstream.h" + +#if MALI_USE_CSF +#include "csf/mali_kbase_csf_tl_reader.h" +#include "csf/mali_kbase_csf_trace_buffer.h" +#endif + +#include +#include +#include + +/** + * struct kbase_timeline - timeline state structure + * @streams: The timeline streams generated by kernel + * @autoflush_timer: Autoflush timer + * @autoflush_timer_active: If non-zero autoflush timer is active + * @reader_lock: Reader lock. Only one reader is allowed to + * have access to the timeline streams at any given time. + * @event_queue: Timeline stream event queue + * @bytes_collected: Number of bytes read by user + * @timeline_flags: Zero, if timeline is disabled. Timeline stream flags + * otherwise. See kbase_timeline_io_acquire(). + * @obj_header_btc: Remaining bytes to copy for the object stream header + * @aux_header_btc: Remaining bytes to copy for the aux stream header + */ +struct kbase_timeline { + struct kbase_tlstream streams[TL_STREAM_TYPE_COUNT]; + struct timer_list autoflush_timer; + atomic_t autoflush_timer_active; + struct mutex reader_lock; + wait_queue_head_t event_queue; +#if MALI_UNIT_TEST + atomic_t bytes_collected; +#endif /* MALI_UNIT_TEST */ + atomic_t *timeline_flags; + size_t obj_header_btc; + size_t aux_header_btc; +#if MALI_USE_CSF + struct kbase_csf_tl_reader csf_tl_reader; +#endif +}; + +extern const struct file_operations kbasep_tlstream_fops; + +void kbase_create_timeline_objects(struct kbase_device *kbdev); + +#endif /* _KBASE_TIMELINE_PRIV_H */ diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tl_serialize.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_tl_serialize.h new file mode 100755 index 000000000000..3e378279cf2c --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tl_serialize.h @@ -0,0 +1,125 @@ +/* + * + * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#if !defined(_KBASE_TL_SERIALIZE_H) +#define _KBASE_TL_SERIALIZE_H + +#include + +#include + +/* The number of nanoseconds in a second. */ +#define NSECS_IN_SEC 1000000000ull /* ns */ + +/** + * kbasep_serialize_bytes - serialize bytes to the message buffer + * + * Serialize bytes as is using memcpy() + * + * @buffer: Message buffer + * @pos: Message buffer offset + * @bytes: Bytes to serialize + * @len: Length of bytes array + * + * Return: updated position in the buffer + */ +static inline size_t kbasep_serialize_bytes( + char *buffer, + size_t pos, + const void *bytes, + size_t len) +{ + KBASE_DEBUG_ASSERT(buffer); + KBASE_DEBUG_ASSERT(bytes); + + memcpy(&buffer[pos], bytes, len); + + return pos + len; +} + +/** + * kbasep_serialize_string - serialize string to the message buffer + * + * String is serialized as 4 bytes for string size, + * then string content and then null terminator. + * + * @buffer: Message buffer + * @pos: Message buffer offset + * @string: String to serialize + * @max_write_size: Number of bytes that can be stored in buffer + * + * Return: updated position in the buffer + */ +static inline size_t kbasep_serialize_string( + char *buffer, + size_t pos, + const char *string, + size_t max_write_size) +{ + u32 string_len; + + KBASE_DEBUG_ASSERT(buffer); + KBASE_DEBUG_ASSERT(string); + /* Timeline string consists of at least string length and nul + * terminator. + */ + KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char)); + max_write_size -= sizeof(string_len); + + string_len = strlcpy( + &buffer[pos + sizeof(string_len)], + string, + max_write_size); + string_len += sizeof(char); + + /* Make sure that the source string fit into the buffer. */ + KBASE_DEBUG_ASSERT(string_len <= max_write_size); + + /* Update string length. */ + memcpy(&buffer[pos], &string_len, sizeof(string_len)); + + return pos + sizeof(string_len) + string_len; +} + +/** + * kbasep_serialize_timestamp - serialize timestamp to the message buffer + * + * Get current timestamp using kbasep_get_timestamp() + * and serialize it as 64 bit unsigned integer. + * + * @buffer: Message buffer + * @pos: Message buffer offset + * + * Return: updated position in the buffer + */ +static inline size_t kbasep_serialize_timestamp(void *buffer, size_t pos) +{ + u64 timestamp; + + timestamp = ktime_get_raw_ns(); + + return kbasep_serialize_bytes( + buffer, pos, + ×tamp, sizeof(timestamp)); +} +#endif /* _KBASE_TL_SERIALIZE_H */ + diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c new file mode 100755 index 000000000000..f4239cfafb9d --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c @@ -0,0 +1,306 @@ +/* + * + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include "mali_kbase_tlstream.h" +#include "mali_kbase_tl_serialize.h" +#include "mali_kbase_mipe_proto.h" + +/** + * kbasep_packet_header_setup - setup the packet header + * @buffer: pointer to the buffer + * @pkt_family: packet's family + * @pkt_type: packet's type + * @pkt_class: packet's class + * @stream_id: stream id + * @numbered: non-zero if this stream is numbered + * + * Function sets up immutable part of packet header in the given buffer. + */ +static void kbasep_packet_header_setup( + char *buffer, + enum tl_packet_family pkt_family, + enum tl_packet_class pkt_class, + enum tl_packet_type pkt_type, + unsigned int stream_id, + int numbered) +{ + u32 words[2] = { + MIPE_PACKET_HEADER_W0(pkt_family, pkt_class, pkt_type, stream_id), + MIPE_PACKET_HEADER_W1(0, !!numbered), + }; + memcpy(buffer, words, sizeof(words)); +} + +/** + * kbasep_packet_header_update - update the packet header + * @buffer: pointer to the buffer + * @data_size: amount of data carried in this packet + * @numbered: non-zero if the stream is numbered + * + * Function updates mutable part of packet header in the given buffer. + * Note that value of data_size must not including size of the header. + */ +static void kbasep_packet_header_update( + char *buffer, + size_t data_size, + int numbered) +{ + u32 word0; + u32 word1 = MIPE_PACKET_HEADER_W1((u32)data_size, !!numbered); + + KBASE_DEBUG_ASSERT(buffer); + CSTD_UNUSED(word0); + + memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); +} + +/** + * kbasep_packet_number_update - update the packet number + * @buffer: pointer to the buffer + * @counter: value of packet counter for this packet's stream + * + * Function updates packet number embedded within the packet placed in the + * given buffer. + */ +static void kbasep_packet_number_update(char *buffer, u32 counter) +{ + KBASE_DEBUG_ASSERT(buffer); + + memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter)); +} + +void kbase_tlstream_reset(struct kbase_tlstream *stream) +{ + unsigned int i; + + for (i = 0; i < PACKET_COUNT; i++) { + if (stream->numbered) + atomic_set( + &stream->buffer[i].size, + PACKET_HEADER_SIZE + + PACKET_NUMBER_SIZE); + else + atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE); + } + + atomic_set(&stream->wbi, 0); + atomic_set(&stream->rbi, 0); +} + +/* Configuration of timeline streams generated by kernel. */ +static const struct { + enum tl_packet_family pkt_family; + enum tl_packet_class pkt_class; + enum tl_packet_type pkt_type; + enum tl_stream_id stream_id; +} tl_stream_cfg[TL_STREAM_TYPE_COUNT] = { + { + TL_PACKET_FAMILY_TL, + TL_PACKET_CLASS_OBJ, + TL_PACKET_TYPE_SUMMARY, + TL_STREAM_ID_KERNEL, + }, + { + TL_PACKET_FAMILY_TL, + TL_PACKET_CLASS_OBJ, + TL_PACKET_TYPE_BODY, + TL_STREAM_ID_KERNEL, + }, + { + TL_PACKET_FAMILY_TL, + TL_PACKET_CLASS_AUX, + TL_PACKET_TYPE_BODY, + TL_STREAM_ID_KERNEL, + }, +#if MALI_USE_CSF + { + TL_PACKET_FAMILY_TL, + TL_PACKET_CLASS_OBJ, + TL_PACKET_TYPE_BODY, + TL_STREAM_ID_CSFFW, + }, +#endif +}; + +void kbase_tlstream_init( + struct kbase_tlstream *stream, + enum tl_stream_type stream_type, + wait_queue_head_t *ready_read) +{ + unsigned int i; + + KBASE_DEBUG_ASSERT(stream); + KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); + + spin_lock_init(&stream->lock); + + /* All packets carrying tracepoints shall be numbered. */ + if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type) + stream->numbered = 1; + else + stream->numbered = 0; + + for (i = 0; i < PACKET_COUNT; i++) + kbasep_packet_header_setup( + stream->buffer[i].data, + tl_stream_cfg[stream_type].pkt_family, + tl_stream_cfg[stream_type].pkt_class, + tl_stream_cfg[stream_type].pkt_type, + tl_stream_cfg[stream_type].stream_id, + stream->numbered); + +#if MALI_UNIT_TEST + atomic_set(&stream->bytes_generated, 0); +#endif + stream->ready_read = ready_read; + + kbase_tlstream_reset(stream); +} + +void kbase_tlstream_term(struct kbase_tlstream *stream) +{ + KBASE_DEBUG_ASSERT(stream); +} + +/** + * kbase_tlstream_msgbuf_submit - submit packet to user space + * @stream: Pointer to the stream structure + * @wb_idx_raw: Write buffer index + * @wb_size: Length of data stored in the current buffer + * + * Updates currently written buffer with the packet header. + * Then write index is incremented and the buffer is handed to user space. + * Parameters of the new buffer are returned using provided arguments. + * + * Return: length of data in the new buffer + * + * Warning: the user must update the stream structure with returned value. + */ +static size_t kbasep_tlstream_msgbuf_submit( + struct kbase_tlstream *stream, + unsigned int wb_idx_raw, + unsigned int wb_size) +{ + unsigned int wb_idx = wb_idx_raw % PACKET_COUNT; + + /* Set stream as flushed. */ + atomic_set(&stream->autoflush_counter, -1); + + kbasep_packet_header_update( + stream->buffer[wb_idx].data, + wb_size - PACKET_HEADER_SIZE, + stream->numbered); + + if (stream->numbered) + kbasep_packet_number_update( + stream->buffer[wb_idx].data, + wb_idx_raw); + + /* Increasing write buffer index will expose this packet to the reader. + * As stream->lock is not taken on reader side we must make sure memory + * is updated correctly before this will happen. */ + smp_wmb(); + atomic_inc(&stream->wbi); + + /* Inform user that packets are ready for reading. */ + wake_up_interruptible(stream->ready_read); + + wb_size = PACKET_HEADER_SIZE; + if (stream->numbered) + wb_size += PACKET_NUMBER_SIZE; + + return wb_size; +} + +char *kbase_tlstream_msgbuf_acquire( + struct kbase_tlstream *stream, + size_t msg_size, + unsigned long *flags) __acquires(&stream->lock) +{ + unsigned int wb_idx_raw; + unsigned int wb_idx; + size_t wb_size; + + KBASE_DEBUG_ASSERT( + PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >= + msg_size); + + spin_lock_irqsave(&stream->lock, *flags); + + wb_idx_raw = atomic_read(&stream->wbi); + wb_idx = wb_idx_raw % PACKET_COUNT; + wb_size = atomic_read(&stream->buffer[wb_idx].size); + + /* Select next buffer if data will not fit into current one. */ + if (PACKET_SIZE < wb_size + msg_size) { + wb_size = kbasep_tlstream_msgbuf_submit( + stream, wb_idx_raw, wb_size); + wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; + } + + /* Reserve space in selected buffer. */ + atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size); + +#if MALI_UNIT_TEST + atomic_add(msg_size, &stream->bytes_generated); +#endif /* MALI_UNIT_TEST */ + + return &stream->buffer[wb_idx].data[wb_size]; +} + +void kbase_tlstream_msgbuf_release( + struct kbase_tlstream *stream, + unsigned long flags) __releases(&stream->lock) +{ + /* Mark stream as containing unflushed data. */ + atomic_set(&stream->autoflush_counter, 0); + + spin_unlock_irqrestore(&stream->lock, flags); +} + +void kbase_tlstream_flush_stream( + struct kbase_tlstream *stream) +{ + unsigned long flags; + unsigned int wb_idx_raw; + unsigned int wb_idx; + size_t wb_size; + size_t min_size = PACKET_HEADER_SIZE; + + if (stream->numbered) + min_size += PACKET_NUMBER_SIZE; + + spin_lock_irqsave(&stream->lock, flags); + + wb_idx_raw = atomic_read(&stream->wbi); + wb_idx = wb_idx_raw % PACKET_COUNT; + wb_size = atomic_read(&stream->buffer[wb_idx].size); + + if (wb_size > min_size) { + wb_size = kbasep_tlstream_msgbuf_submit( + stream, wb_idx_raw, wb_size); + wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; + atomic_set(&stream->buffer[wb_idx].size, wb_size); + } + spin_unlock_irqrestore(&stream->lock, flags); +} + diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.h new file mode 100755 index 000000000000..faf88d676b5d --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.h @@ -0,0 +1,169 @@ +/* + * + * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#if !defined(_KBASE_TLSTREAM_H) +#define _KBASE_TLSTREAM_H + +#include +#include +#include + +/* The maximum size of a single packet used by timeline. */ +#define PACKET_SIZE 4096 /* bytes */ + +/* The number of packets used by one timeline stream. */ +#if defined(CONFIG_MALI_JOB_DUMP) || defined(CONFIG_MALI_VECTOR_DUMP) + #define PACKET_COUNT 64 +#else + #define PACKET_COUNT 32 +#endif + +/* The maximum expected length of string in tracepoint descriptor. */ +#define STRLEN_MAX 64 /* bytes */ + +/** + * struct kbase_tlstream - timeline stream structure + * @lock: Message order lock + * @buffer: Array of buffers + * @wbi: Write buffer index + * @rbi: Read buffer index + * @numbered: If non-zero stream's packets are sequentially numbered + * @autoflush_counter: Counter tracking stream's autoflush state + * @ready_read: Pointer to a wait queue, which is signaled when + * timeline messages are ready for collection. + * @bytes_generated: Number of bytes generated by tracepoint messages + * + * This structure holds information needed to construct proper packets in the + * timeline stream. + * + * Each message in the sequence must bear a timestamp that is + * greater than the previous message in the same stream. For this reason + * a lock is held throughout the process of message creation. + * + * Each stream contains a set of buffers. Each buffer will hold one MIPE + * packet. In case there is no free space required to store the incoming + * message the oldest buffer is discarded. Each packet in timeline body + * stream has a sequence number embedded, this value must increment + * monotonically and is used by the packets receiver to discover these + * buffer overflows. + * + * The autoflush counter is set to a negative number when there is no data + * pending for flush and it is set to zero on every update of the buffer. The + * autoflush timer will increment the counter by one on every expiry. If there + * is no activity on the buffer for two consecutive timer expiries, the stream + * buffer will be flushed. + */ +struct kbase_tlstream { + spinlock_t lock; + + struct { + atomic_t size; /* number of bytes in buffer */ + char data[PACKET_SIZE]; /* buffer's data */ + } buffer[PACKET_COUNT]; + + atomic_t wbi; + atomic_t rbi; + + int numbered; + atomic_t autoflush_counter; + wait_queue_head_t *ready_read; +#if MALI_UNIT_TEST + atomic_t bytes_generated; +#endif +}; + +/* Types of streams generated by timeline. */ +enum tl_stream_type { + TL_STREAM_TYPE_FIRST, + TL_STREAM_TYPE_OBJ_SUMMARY = TL_STREAM_TYPE_FIRST, + TL_STREAM_TYPE_OBJ, + TL_STREAM_TYPE_AUX, +#if MALI_USE_CSF + TL_STREAM_TYPE_CSFFW, +#endif + TL_STREAM_TYPE_COUNT +}; + +/** + * kbase_tlstream_init - initialize timeline stream + * @stream: Pointer to the stream structure + * @stream_type: Stream type + * @ready_read: Pointer to a wait queue to signal when + * timeline messages are ready for collection. + */ +void kbase_tlstream_init(struct kbase_tlstream *stream, + enum tl_stream_type stream_type, + wait_queue_head_t *ready_read); + +/** + * kbase_tlstream_term - terminate timeline stream + * @stream: Pointer to the stream structure + */ +void kbase_tlstream_term(struct kbase_tlstream *stream); + +/** + * kbase_tlstream_reset - reset stream + * @stream: Pointer to the stream structure + * + * Function discards all pending messages and resets packet counters. + */ +void kbase_tlstream_reset(struct kbase_tlstream *stream); + +/** + * kbase_tlstream_msgbuf_acquire - lock selected stream and reserve a buffer + * @stream: Pointer to the stream structure + * @msg_size: Message size + * @flags: Pointer to store flags passed back on stream release + * + * Lock the stream and reserve the number of bytes requested + * in msg_size for the user. + * + * Return: pointer to the buffer where a message can be stored + * + * Warning: The stream must be released with kbase_tlstream_msgbuf_release(). + * Only atomic operations are allowed while the stream is locked + * (i.e. do not use any operation that may sleep). + */ +char *kbase_tlstream_msgbuf_acquire(struct kbase_tlstream *stream, + size_t msg_size, unsigned long *flags) __acquires(&stream->lock); + +/** + * kbase_tlstream_msgbuf_release - unlock selected stream + * @stream: Pointer to the stream structure + * @flags: Value obtained during stream acquire + * + * Release the stream that has been previously + * locked with a call to kbase_tlstream_msgbuf_acquire(). + */ +void kbase_tlstream_msgbuf_release(struct kbase_tlstream *stream, + unsigned long flags) __releases(&stream->lock); + +/** + * kbase_tlstream_flush_stream - flush stream + * @stream: Pointer to the stream structure + * + * Flush pending data in the timeline stream. + */ +void kbase_tlstream_flush_stream(struct kbase_tlstream *stream); + +#endif /* _KBASE_TLSTREAM_H */ + diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c new file mode 100755 index 000000000000..de76fa57051e --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c @@ -0,0 +1,2974 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. + * DO NOT EDIT. + */ + +#include "mali_kbase_tracepoints.h" +#include "mali_kbase_tlstream.h" +#include "mali_kbase_tl_serialize.h" + +/* clang-format off */ + +/* Message ids of trace events that are recorded in the timeline stream. */ +enum tl_msg_id_obj { + KBASE_TL_NEW_CTX, + KBASE_TL_NEW_GPU, + KBASE_TL_NEW_LPU, + KBASE_TL_NEW_ATOM, + KBASE_TL_NEW_AS, + KBASE_TL_DEL_CTX, + KBASE_TL_DEL_ATOM, + KBASE_TL_LIFELINK_LPU_GPU, + KBASE_TL_LIFELINK_AS_GPU, + KBASE_TL_RET_CTX_LPU, + KBASE_TL_RET_ATOM_CTX, + KBASE_TL_RET_ATOM_LPU, + KBASE_TL_NRET_CTX_LPU, + KBASE_TL_NRET_ATOM_CTX, + KBASE_TL_NRET_ATOM_LPU, + KBASE_TL_RET_AS_CTX, + KBASE_TL_NRET_AS_CTX, + KBASE_TL_RET_ATOM_AS, + KBASE_TL_NRET_ATOM_AS, + KBASE_TL_ATTRIB_ATOM_CONFIG, + KBASE_TL_ATTRIB_ATOM_PRIORITY, + KBASE_TL_ATTRIB_ATOM_STATE, + KBASE_TL_ATTRIB_ATOM_PRIORITIZED, + KBASE_TL_ATTRIB_ATOM_JIT, + KBASE_TL_JIT_USEDPAGES, + KBASE_TL_ATTRIB_ATOM_JITALLOCINFO, + KBASE_TL_ATTRIB_ATOM_JITFREEINFO, + KBASE_TL_ATTRIB_AS_CONFIG, + KBASE_TL_EVENT_LPU_SOFTSTOP, + KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, + KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, + KBASE_TL_EVENT_ATOM_SOFTJOB_START, + KBASE_TL_EVENT_ATOM_SOFTJOB_END, + KBASE_TL_ARBITER_GRANTED, + KBASE_TL_ARBITER_STARTED, + KBASE_TL_ARBITER_STOP_REQUESTED, + KBASE_TL_ARBITER_STOPPED, + KBASE_JD_GPU_SOFT_RESET, + KBASE_TL_KBASE_NEW_DEVICE, + KBASE_TL_KBASE_DEVICE_PROGRAM_CSG, + KBASE_TL_KBASE_DEVICE_DEPROGRAM_CSG, + KBASE_TL_KBASE_NEW_CTX, + KBASE_TL_KBASE_DEL_CTX, + KBASE_TL_KBASE_CTX_ASSIGN_AS, + KBASE_TL_KBASE_CTX_UNASSIGN_AS, + KBASE_TL_KBASE_NEW_KCPUQUEUE, + KBASE_TL_KBASE_DEL_KCPUQUEUE, + KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL, + KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT, + KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT, + KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET, + KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT, + KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT, + KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE, + KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC, + KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC, + KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC, + KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE, + KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE, + KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START, + KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, + KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, + KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START, + KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END, + KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END, + KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END, + KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER, + KBASE_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW, + KBASE_TL_KBASE_CSFFW_RESET, + KBASE_OBJ_MSG_COUNT, +}; + +/* Message ids of trace events that are recorded in the auxiliary stream. */ +enum tl_msg_id_aux { + KBASE_AUX_PM_STATE, + KBASE_AUX_PAGEFAULT, + KBASE_AUX_PAGESALLOC, + KBASE_AUX_DEVFREQ_TARGET, + KBASE_AUX_PROTECTED_ENTER_START, + KBASE_AUX_PROTECTED_ENTER_END, + KBASE_AUX_PROTECTED_LEAVE_START, + KBASE_AUX_PROTECTED_LEAVE_END, + KBASE_AUX_JIT_STATS, + KBASE_AUX_EVENT_JOB_SLOT, + KBASE_AUX_MSG_COUNT, +}; + +#define OBJ_TP_LIST \ + TRACEPOINT_DESC(KBASE_TL_NEW_CTX, \ + "object ctx is created", \ + "@pII", \ + "ctx,ctx_nr,tgid") \ + TRACEPOINT_DESC(KBASE_TL_NEW_GPU, \ + "object gpu is created", \ + "@pII", \ + "gpu,gpu_id,core_count") \ + TRACEPOINT_DESC(KBASE_TL_NEW_LPU, \ + "object lpu is created", \ + "@pII", \ + "lpu,lpu_nr,lpu_fn") \ + TRACEPOINT_DESC(KBASE_TL_NEW_ATOM, \ + "object atom is created", \ + "@pI", \ + "atom,atom_nr") \ + TRACEPOINT_DESC(KBASE_TL_NEW_AS, \ + "address space object is created", \ + "@pI", \ + "address_space,as_nr") \ + TRACEPOINT_DESC(KBASE_TL_DEL_CTX, \ + "context is destroyed", \ + "@p", \ + "ctx") \ + TRACEPOINT_DESC(KBASE_TL_DEL_ATOM, \ + "atom is destroyed", \ + "@p", \ + "atom") \ + TRACEPOINT_DESC(KBASE_TL_LIFELINK_LPU_GPU, \ + "lpu is deleted with gpu", \ + "@pp", \ + "lpu,gpu") \ + TRACEPOINT_DESC(KBASE_TL_LIFELINK_AS_GPU, \ + "address space is deleted with gpu", \ + "@pp", \ + "address_space,gpu") \ + TRACEPOINT_DESC(KBASE_TL_RET_CTX_LPU, \ + "context is retained by lpu", \ + "@pp", \ + "ctx,lpu") \ + TRACEPOINT_DESC(KBASE_TL_RET_ATOM_CTX, \ + "atom is retained by context", \ + "@pp", \ + "atom,ctx") \ + TRACEPOINT_DESC(KBASE_TL_RET_ATOM_LPU, \ + "atom is retained by lpu", \ + "@pps", \ + "atom,lpu,attrib_match_list") \ + TRACEPOINT_DESC(KBASE_TL_NRET_CTX_LPU, \ + "context is released by lpu", \ + "@pp", \ + "ctx,lpu") \ + TRACEPOINT_DESC(KBASE_TL_NRET_ATOM_CTX, \ + "atom is released by context", \ + "@pp", \ + "atom,ctx") \ + TRACEPOINT_DESC(KBASE_TL_NRET_ATOM_LPU, \ + "atom is released by lpu", \ + "@pp", \ + "atom,lpu") \ + TRACEPOINT_DESC(KBASE_TL_RET_AS_CTX, \ + "address space is retained by context", \ + "@pp", \ + "address_space,ctx") \ + TRACEPOINT_DESC(KBASE_TL_NRET_AS_CTX, \ + "address space is released by context", \ + "@pp", \ + "address_space,ctx") \ + TRACEPOINT_DESC(KBASE_TL_RET_ATOM_AS, \ + "atom is retained by address space", \ + "@pp", \ + "atom,address_space") \ + TRACEPOINT_DESC(KBASE_TL_NRET_ATOM_AS, \ + "atom is released by address space", \ + "@pp", \ + "atom,address_space") \ + TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_CONFIG, \ + "atom job slot attributes", \ + "@pLLI", \ + "atom,descriptor,affinity,config") \ + TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_PRIORITY, \ + "atom priority", \ + "@pI", \ + "atom,prio") \ + TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_STATE, \ + "atom state", \ + "@pI", \ + "atom,state") \ + TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_PRIORITIZED, \ + "atom caused priority change", \ + "@p", \ + "atom") \ + TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_JIT, \ + "jit done for atom", \ + "@pLLILILLL", \ + "atom,edit_addr,new_addr,jit_flags,mem_flags,j_id,com_pgs,extent,va_pgs") \ + TRACEPOINT_DESC(KBASE_TL_JIT_USEDPAGES, \ + "used pages for jit", \ + "@LI", \ + "used_pages,j_id") \ + TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_JITALLOCINFO, \ + "Information about JIT allocations", \ + "@pLLLIIIII", \ + "atom,va_pgs,com_pgs,extent,j_id,bin_id,max_allocs,jit_flags,usg_id") \ + TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_JITFREEINFO, \ + "Information about JIT frees", \ + "@pI", \ + "atom,j_id") \ + TRACEPOINT_DESC(KBASE_TL_ATTRIB_AS_CONFIG, \ + "address space attributes", \ + "@pLLL", \ + "address_space,transtab,memattr,transcfg") \ + TRACEPOINT_DESC(KBASE_TL_EVENT_LPU_SOFTSTOP, \ + "softstop event on given lpu", \ + "@p", \ + "lpu") \ + TRACEPOINT_DESC(KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, \ + "atom softstopped", \ + "@p", \ + "atom") \ + TRACEPOINT_DESC(KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, \ + "atom softstop issued", \ + "@p", \ + "atom") \ + TRACEPOINT_DESC(KBASE_TL_EVENT_ATOM_SOFTJOB_START, \ + "atom soft job has started", \ + "@p", \ + "atom") \ + TRACEPOINT_DESC(KBASE_TL_EVENT_ATOM_SOFTJOB_END, \ + "atom soft job has completed", \ + "@p", \ + "atom") \ + TRACEPOINT_DESC(KBASE_TL_ARBITER_GRANTED, \ + "Arbiter has granted gpu access", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_TL_ARBITER_STARTED, \ + "Driver is running again and able to process jobs", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_TL_ARBITER_STOP_REQUESTED, \ + "Arbiter has requested driver to stop using gpu", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_TL_ARBITER_STOPPED, \ + "Driver has stopped using gpu", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_JD_GPU_SOFT_RESET, \ + "gpu soft reset", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_DEVICE, \ + "New KBase Device", \ + "@IIII", \ + "kbase_device_id,kbase_device_gpu_core_count,kbase_device_max_num_csgs,kbase_device_as_count") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_DEVICE_PROGRAM_CSG, \ + "CSG is programmed to a slot", \ + "@III", \ + "kbase_device_id,gpu_cmdq_grp_handle,kbase_device_csg_slot_index") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_DEVICE_DEPROGRAM_CSG, \ + "CSG is deprogrammed from a slot", \ + "@II", \ + "kbase_device_id,kbase_device_csg_slot_index") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_CTX, \ + "New KBase Context", \ + "@II", \ + "kernel_ctx_id,kbase_device_id") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_DEL_CTX, \ + "Delete KBase Context", \ + "@I", \ + "kernel_ctx_id") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_CTX_ASSIGN_AS, \ + "Address Space is assigned to a KBase context", \ + "@II", \ + "kernel_ctx_id,kbase_device_as_index") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_CTX_UNASSIGN_AS, \ + "Address Space is unassigned from a KBase context", \ + "@I", \ + "kernel_ctx_id") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_KCPUQUEUE, \ + "New KCPU Queue", \ + "@pII", \ + "kcpu_queue,kernel_ctx_id,kcpuq_num_pending_cmds") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_DEL_KCPUQUEUE, \ + "Delete KCPU Queue", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL, \ + "KCPU Queue enqueues Signal on Fence", \ + "@pp", \ + "kcpu_queue,fence") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT, \ + "KCPU Queue enqueues Wait on Fence", \ + "@pp", \ + "kcpu_queue,fence") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT, \ + "KCPU Queue enqueues Wait on Cross Queue Sync Object", \ + "@pLI", \ + "kcpu_queue,cqs_obj_gpu_addr,cqs_obj_compare_value") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET, \ + "KCPU Queue enqueues Set on Cross Queue Sync Object", \ + "@pL", \ + "kcpu_queue,cqs_obj_gpu_addr") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT, \ + "KCPU Queue enqueues Map Import", \ + "@pL", \ + "kcpu_queue,map_import_buf_gpu_addr") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT, \ + "KCPU Queue enqueues Unmap Import", \ + "@pL", \ + "kcpu_queue,map_import_buf_gpu_addr") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE, \ + "KCPU Queue enqueues Unmap Import ignoring reference count", \ + "@pL", \ + "kcpu_queue,map_import_buf_gpu_addr") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC, \ + "Begin array of KCPU Queue enqueues JIT Alloc", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC, \ + "Array item of KCPU Queue enqueues JIT Alloc", \ + "@pLLLLIIIII", \ + "kcpu_queue,jit_alloc_gpu_alloc_addr_dest,jit_alloc_va_pages,jit_alloc_commit_pages,jit_alloc_extent,jit_alloc_jit_id,jit_alloc_bin_id,jit_alloc_max_allocations,jit_alloc_flags,jit_alloc_usage_id") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC, \ + "End array of KCPU Queue enqueues JIT Alloc", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE, \ + "Begin array of KCPU Queue enqueues JIT Free", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE, \ + "Array item of KCPU Queue enqueues JIT Free", \ + "@pI", \ + "kcpu_queue,jit_alloc_jit_id") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE, \ + "End array of KCPU Queue enqueues JIT Free", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START, \ + "KCPU Queue starts a Signal on Fence", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END, \ + "KCPU Queue ends a Signal on Fence", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START, \ + "KCPU Queue starts a Wait on Fence", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END, \ + "KCPU Queue ends a Wait on Fence", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START, \ + "KCPU Queue starts a Wait on an array of Cross Queue Sync Objects", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END, \ + "KCPU Queue ends a Wait on an array of Cross Queue Sync Objects", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET, \ + "KCPU Queue executes a Set on an array of Cross Queue Sync Objects", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START, \ + "KCPU Queue starts a Map Import", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END, \ + "KCPU Queue ends a Map Import", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START, \ + "KCPU Queue starts an Unmap Import", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END, \ + "KCPU Queue ends an Unmap Import", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START, \ + "KCPU Queue starts an Unmap Import ignoring reference count", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END, \ + "KCPU Queue ends an Unmap Import ignoring reference count", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START, \ + "KCPU Queue starts an array of JIT Allocs", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, \ + "Begin array of KCPU Queue ends an array of JIT Allocs", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, \ + "Array item of KCPU Queue ends an array of JIT Allocs", \ + "@pLL", \ + "kcpu_queue,jit_alloc_gpu_alloc_addr,jit_alloc_mmu_flags") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, \ + "End array of KCPU Queue ends an array of JIT Allocs", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START, \ + "KCPU Queue starts an array of JIT Frees", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END, \ + "Begin array of KCPU Queue ends an array of JIT Frees", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END, \ + "Array item of KCPU Queue ends an array of JIT Frees", \ + "@pL", \ + "kcpu_queue,jit_free_pages_used") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END, \ + "End array of KCPU Queue ends an array of JIT Frees", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER, \ + "KCPU Queue executes an Error Barrier", \ + "@p", \ + "kcpu_queue") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW, \ + "An overflow has happened with the CSFFW Timeline stream", \ + "@LL", \ + "csffw_timestamp,csffw_cycle") \ + TRACEPOINT_DESC(KBASE_TL_KBASE_CSFFW_RESET, \ + "A reset has happened with the CSFFW", \ + "@L", \ + "csffw_cycle") \ + +#define MIPE_HEADER_BLOB_VAR_NAME __obj_desc_header +#define MIPE_HEADER_STREAM_ID TL_STREAM_ID_KERNEL +#define MIPE_HEADER_PKT_CLASS TL_PACKET_CLASS_OBJ +#define MIPE_HEADER_TRACEPOINT_LIST OBJ_TP_LIST +#define MIPE_HEADER_TRACEPOINT_LIST_SIZE KBASE_OBJ_MSG_COUNT + +#include "mali_kbase_mipe_gen_header.h" + +const char *obj_desc_header = (const char *) &__obj_desc_header; +const size_t obj_desc_header_size = sizeof(__obj_desc_header); + +#define AUX_TP_LIST \ + TRACEPOINT_DESC(KBASE_AUX_PM_STATE, \ + "PM state", \ + "@IL", \ + "core_type,core_state_bitset") \ + TRACEPOINT_DESC(KBASE_AUX_PAGEFAULT, \ + "Page fault", \ + "@IIL", \ + "ctx_nr,as_nr,page_cnt_change") \ + TRACEPOINT_DESC(KBASE_AUX_PAGESALLOC, \ + "Total alloc pages change", \ + "@IL", \ + "ctx_nr,page_cnt") \ + TRACEPOINT_DESC(KBASE_AUX_DEVFREQ_TARGET, \ + "New device frequency target", \ + "@L", \ + "target_freq") \ + TRACEPOINT_DESC(KBASE_AUX_PROTECTED_ENTER_START, \ + "enter protected mode start", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_AUX_PROTECTED_ENTER_END, \ + "enter protected mode end", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_AUX_PROTECTED_LEAVE_START, \ + "leave protected mode start", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_AUX_PROTECTED_LEAVE_END, \ + "leave protected mode end", \ + "@p", \ + "gpu") \ + TRACEPOINT_DESC(KBASE_AUX_JIT_STATS, \ + "per-bin JIT statistics", \ + "@IIIIII", \ + "ctx_nr,bid,max_allocs,allocs,va_pages,ph_pages") \ + TRACEPOINT_DESC(KBASE_AUX_EVENT_JOB_SLOT, \ + "event on a given job slot", \ + "@pIII", \ + "ctx,slot_nr,atom_nr,event") \ + +#define MIPE_HEADER_BLOB_VAR_NAME __aux_desc_header +#define MIPE_HEADER_STREAM_ID TL_STREAM_ID_KERNEL +#define MIPE_HEADER_PKT_CLASS TL_PACKET_CLASS_AUX +#define MIPE_HEADER_TRACEPOINT_LIST AUX_TP_LIST +#define MIPE_HEADER_TRACEPOINT_LIST_SIZE KBASE_AUX_MSG_COUNT + +#include "mali_kbase_mipe_gen_header.h" + +const char *aux_desc_header = (const char *) &__aux_desc_header; +const size_t aux_desc_header_size = sizeof(__aux_desc_header); + +void __kbase_tlstream_tl_new_ctx( + struct kbase_tlstream *stream, + const void *ctx, + u32 ctx_nr, + u32 tgid) +{ + const u32 msg_id = KBASE_TL_NEW_CTX; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(ctx) + + sizeof(ctx_nr) + + sizeof(tgid) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx, sizeof(ctx)); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx_nr, sizeof(ctx_nr)); + pos = kbasep_serialize_bytes(buffer, + pos, &tgid, sizeof(tgid)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_new_gpu( + struct kbase_tlstream *stream, + const void *gpu, + u32 gpu_id, + u32 core_count) +{ + const u32 msg_id = KBASE_TL_NEW_GPU; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + + sizeof(gpu_id) + + sizeof(core_count) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu_id, sizeof(gpu_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &core_count, sizeof(core_count)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_new_lpu( + struct kbase_tlstream *stream, + const void *lpu, + u32 lpu_nr, + u32 lpu_fn) +{ + const u32 msg_id = KBASE_TL_NEW_LPU; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(lpu) + + sizeof(lpu_nr) + + sizeof(lpu_fn) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &lpu, sizeof(lpu)); + pos = kbasep_serialize_bytes(buffer, + pos, &lpu_nr, sizeof(lpu_nr)); + pos = kbasep_serialize_bytes(buffer, + pos, &lpu_fn, sizeof(lpu_fn)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_new_atom( + struct kbase_tlstream *stream, + const void *atom, + u32 atom_nr) +{ + const u32 msg_id = KBASE_TL_NEW_ATOM; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(atom_nr) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &atom_nr, sizeof(atom_nr)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_new_as( + struct kbase_tlstream *stream, + const void *address_space, + u32 as_nr) +{ + const u32 msg_id = KBASE_TL_NEW_AS; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(address_space) + + sizeof(as_nr) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &address_space, sizeof(address_space)); + pos = kbasep_serialize_bytes(buffer, + pos, &as_nr, sizeof(as_nr)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_del_ctx( + struct kbase_tlstream *stream, + const void *ctx) +{ + const u32 msg_id = KBASE_TL_DEL_CTX; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(ctx) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx, sizeof(ctx)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_del_atom( + struct kbase_tlstream *stream, + const void *atom) +{ + const u32 msg_id = KBASE_TL_DEL_ATOM; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_lifelink_lpu_gpu( + struct kbase_tlstream *stream, + const void *lpu, + const void *gpu) +{ + const u32 msg_id = KBASE_TL_LIFELINK_LPU_GPU; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(lpu) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &lpu, sizeof(lpu)); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_lifelink_as_gpu( + struct kbase_tlstream *stream, + const void *address_space, + const void *gpu) +{ + const u32 msg_id = KBASE_TL_LIFELINK_AS_GPU; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(address_space) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &address_space, sizeof(address_space)); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_ret_ctx_lpu( + struct kbase_tlstream *stream, + const void *ctx, + const void *lpu) +{ + const u32 msg_id = KBASE_TL_RET_CTX_LPU; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(ctx) + + sizeof(lpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx, sizeof(ctx)); + pos = kbasep_serialize_bytes(buffer, + pos, &lpu, sizeof(lpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_ret_atom_ctx( + struct kbase_tlstream *stream, + const void *atom, + const void *ctx) +{ + const u32 msg_id = KBASE_TL_RET_ATOM_CTX; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(ctx) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx, sizeof(ctx)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_ret_atom_lpu( + struct kbase_tlstream *stream, + const void *atom, + const void *lpu, + const char *attrib_match_list) +{ + const u32 msg_id = KBASE_TL_RET_ATOM_LPU; + const size_t s0 = sizeof(u32) + sizeof(char) + + strnlen(attrib_match_list, STRLEN_MAX); + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(lpu) + + s0 + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &lpu, sizeof(lpu)); + pos = kbasep_serialize_string(buffer, + pos, attrib_match_list, s0); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_nret_ctx_lpu( + struct kbase_tlstream *stream, + const void *ctx, + const void *lpu) +{ + const u32 msg_id = KBASE_TL_NRET_CTX_LPU; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(ctx) + + sizeof(lpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx, sizeof(ctx)); + pos = kbasep_serialize_bytes(buffer, + pos, &lpu, sizeof(lpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_nret_atom_ctx( + struct kbase_tlstream *stream, + const void *atom, + const void *ctx) +{ + const u32 msg_id = KBASE_TL_NRET_ATOM_CTX; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(ctx) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx, sizeof(ctx)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_nret_atom_lpu( + struct kbase_tlstream *stream, + const void *atom, + const void *lpu) +{ + const u32 msg_id = KBASE_TL_NRET_ATOM_LPU; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(lpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &lpu, sizeof(lpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_ret_as_ctx( + struct kbase_tlstream *stream, + const void *address_space, + const void *ctx) +{ + const u32 msg_id = KBASE_TL_RET_AS_CTX; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(address_space) + + sizeof(ctx) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &address_space, sizeof(address_space)); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx, sizeof(ctx)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_nret_as_ctx( + struct kbase_tlstream *stream, + const void *address_space, + const void *ctx) +{ + const u32 msg_id = KBASE_TL_NRET_AS_CTX; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(address_space) + + sizeof(ctx) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &address_space, sizeof(address_space)); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx, sizeof(ctx)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_ret_atom_as( + struct kbase_tlstream *stream, + const void *atom, + const void *address_space) +{ + const u32 msg_id = KBASE_TL_RET_ATOM_AS; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(address_space) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &address_space, sizeof(address_space)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_nret_atom_as( + struct kbase_tlstream *stream, + const void *atom, + const void *address_space) +{ + const u32 msg_id = KBASE_TL_NRET_ATOM_AS; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(address_space) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &address_space, sizeof(address_space)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_attrib_atom_config( + struct kbase_tlstream *stream, + const void *atom, + u64 descriptor, + u64 affinity, + u32 config) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_CONFIG; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(descriptor) + + sizeof(affinity) + + sizeof(config) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &descriptor, sizeof(descriptor)); + pos = kbasep_serialize_bytes(buffer, + pos, &affinity, sizeof(affinity)); + pos = kbasep_serialize_bytes(buffer, + pos, &config, sizeof(config)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_attrib_atom_priority( + struct kbase_tlstream *stream, + const void *atom, + u32 prio) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(prio) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &prio, sizeof(prio)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_attrib_atom_state( + struct kbase_tlstream *stream, + const void *atom, + u32 state) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_STATE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(state) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &state, sizeof(state)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_attrib_atom_prioritized( + struct kbase_tlstream *stream, + const void *atom) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITIZED; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_attrib_atom_jit( + struct kbase_tlstream *stream, + const void *atom, + u64 edit_addr, + u64 new_addr, + u32 jit_flags, + u64 mem_flags, + u32 j_id, + u64 com_pgs, + u64 extent, + u64 va_pgs) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(edit_addr) + + sizeof(new_addr) + + sizeof(jit_flags) + + sizeof(mem_flags) + + sizeof(j_id) + + sizeof(com_pgs) + + sizeof(extent) + + sizeof(va_pgs) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &edit_addr, sizeof(edit_addr)); + pos = kbasep_serialize_bytes(buffer, + pos, &new_addr, sizeof(new_addr)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_flags, sizeof(jit_flags)); + pos = kbasep_serialize_bytes(buffer, + pos, &mem_flags, sizeof(mem_flags)); + pos = kbasep_serialize_bytes(buffer, + pos, &j_id, sizeof(j_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &com_pgs, sizeof(com_pgs)); + pos = kbasep_serialize_bytes(buffer, + pos, &extent, sizeof(extent)); + pos = kbasep_serialize_bytes(buffer, + pos, &va_pgs, sizeof(va_pgs)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_jit_usedpages( + struct kbase_tlstream *stream, + u64 used_pages, + u32 j_id) +{ + const u32 msg_id = KBASE_TL_JIT_USEDPAGES; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(used_pages) + + sizeof(j_id) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &used_pages, sizeof(used_pages)); + pos = kbasep_serialize_bytes(buffer, + pos, &j_id, sizeof(j_id)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_attrib_atom_jitallocinfo( + struct kbase_tlstream *stream, + const void *atom, + u64 va_pgs, + u64 com_pgs, + u64 extent, + u32 j_id, + u32 bin_id, + u32 max_allocs, + u32 jit_flags, + u32 usg_id) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JITALLOCINFO; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(va_pgs) + + sizeof(com_pgs) + + sizeof(extent) + + sizeof(j_id) + + sizeof(bin_id) + + sizeof(max_allocs) + + sizeof(jit_flags) + + sizeof(usg_id) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &va_pgs, sizeof(va_pgs)); + pos = kbasep_serialize_bytes(buffer, + pos, &com_pgs, sizeof(com_pgs)); + pos = kbasep_serialize_bytes(buffer, + pos, &extent, sizeof(extent)); + pos = kbasep_serialize_bytes(buffer, + pos, &j_id, sizeof(j_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &bin_id, sizeof(bin_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &max_allocs, sizeof(max_allocs)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_flags, sizeof(jit_flags)); + pos = kbasep_serialize_bytes(buffer, + pos, &usg_id, sizeof(usg_id)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_attrib_atom_jitfreeinfo( + struct kbase_tlstream *stream, + const void *atom, + u32 j_id) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JITFREEINFO; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + + sizeof(j_id) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + pos = kbasep_serialize_bytes(buffer, + pos, &j_id, sizeof(j_id)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_attrib_as_config( + struct kbase_tlstream *stream, + const void *address_space, + u64 transtab, + u64 memattr, + u64 transcfg) +{ + const u32 msg_id = KBASE_TL_ATTRIB_AS_CONFIG; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(address_space) + + sizeof(transtab) + + sizeof(memattr) + + sizeof(transcfg) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &address_space, sizeof(address_space)); + pos = kbasep_serialize_bytes(buffer, + pos, &transtab, sizeof(transtab)); + pos = kbasep_serialize_bytes(buffer, + pos, &memattr, sizeof(memattr)); + pos = kbasep_serialize_bytes(buffer, + pos, &transcfg, sizeof(transcfg)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_event_lpu_softstop( + struct kbase_tlstream *stream, + const void *lpu) +{ + const u32 msg_id = KBASE_TL_EVENT_LPU_SOFTSTOP; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(lpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &lpu, sizeof(lpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_event_atom_softstop_ex( + struct kbase_tlstream *stream, + const void *atom) +{ + const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_EX; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_event_atom_softstop_issue( + struct kbase_tlstream *stream, + const void *atom) +{ + const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_event_atom_softjob_start( + struct kbase_tlstream *stream, + const void *atom) +{ + const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTJOB_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_event_atom_softjob_end( + struct kbase_tlstream *stream, + const void *atom) +{ + const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTJOB_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &atom, sizeof(atom)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_arbiter_granted( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_TL_ARBITER_GRANTED; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_arbiter_started( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_TL_ARBITER_STARTED; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_arbiter_stop_requested( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_TL_ARBITER_STOP_REQUESTED; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_arbiter_stopped( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_TL_ARBITER_STOPPED; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_jd_gpu_soft_reset( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_JD_GPU_SOFT_RESET; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_aux_pm_state( + struct kbase_tlstream *stream, + u32 core_type, + u64 core_state_bitset) +{ + const u32 msg_id = KBASE_AUX_PM_STATE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(core_type) + + sizeof(core_state_bitset) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &core_type, sizeof(core_type)); + pos = kbasep_serialize_bytes(buffer, + pos, &core_state_bitset, sizeof(core_state_bitset)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_aux_pagefault( + struct kbase_tlstream *stream, + u32 ctx_nr, + u32 as_nr, + u64 page_cnt_change) +{ + const u32 msg_id = KBASE_AUX_PAGEFAULT; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(ctx_nr) + + sizeof(as_nr) + + sizeof(page_cnt_change) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx_nr, sizeof(ctx_nr)); + pos = kbasep_serialize_bytes(buffer, + pos, &as_nr, sizeof(as_nr)); + pos = kbasep_serialize_bytes(buffer, + pos, &page_cnt_change, sizeof(page_cnt_change)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_aux_pagesalloc( + struct kbase_tlstream *stream, + u32 ctx_nr, + u64 page_cnt) +{ + const u32 msg_id = KBASE_AUX_PAGESALLOC; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(ctx_nr) + + sizeof(page_cnt) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx_nr, sizeof(ctx_nr)); + pos = kbasep_serialize_bytes(buffer, + pos, &page_cnt, sizeof(page_cnt)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_aux_devfreq_target( + struct kbase_tlstream *stream, + u64 target_freq) +{ + const u32 msg_id = KBASE_AUX_DEVFREQ_TARGET; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(target_freq) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &target_freq, sizeof(target_freq)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_aux_protected_enter_start( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_aux_protected_enter_end( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_aux_protected_leave_start( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_aux_protected_leave_end( + struct kbase_tlstream *stream, + const void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_aux_jit_stats( + struct kbase_tlstream *stream, + u32 ctx_nr, + u32 bid, + u32 max_allocs, + u32 allocs, + u32 va_pages, + u32 ph_pages) +{ + const u32 msg_id = KBASE_AUX_JIT_STATS; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(ctx_nr) + + sizeof(bid) + + sizeof(max_allocs) + + sizeof(allocs) + + sizeof(va_pages) + + sizeof(ph_pages) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx_nr, sizeof(ctx_nr)); + pos = kbasep_serialize_bytes(buffer, + pos, &bid, sizeof(bid)); + pos = kbasep_serialize_bytes(buffer, + pos, &max_allocs, sizeof(max_allocs)); + pos = kbasep_serialize_bytes(buffer, + pos, &allocs, sizeof(allocs)); + pos = kbasep_serialize_bytes(buffer, + pos, &va_pages, sizeof(va_pages)); + pos = kbasep_serialize_bytes(buffer, + pos, &ph_pages, sizeof(ph_pages)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_aux_event_job_slot( + struct kbase_tlstream *stream, + const void *ctx, + u32 slot_nr, + u32 atom_nr, + u32 event) +{ + const u32 msg_id = KBASE_AUX_EVENT_JOB_SLOT; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(ctx) + + sizeof(slot_nr) + + sizeof(atom_nr) + + sizeof(event) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &ctx, sizeof(ctx)); + pos = kbasep_serialize_bytes(buffer, + pos, &slot_nr, sizeof(slot_nr)); + pos = kbasep_serialize_bytes(buffer, + pos, &atom_nr, sizeof(atom_nr)); + pos = kbasep_serialize_bytes(buffer, + pos, &event, sizeof(event)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_new_device( + struct kbase_tlstream *stream, + u32 kbase_device_id, + u32 kbase_device_gpu_core_count, + u32 kbase_device_max_num_csgs, + u32 kbase_device_as_count) +{ + const u32 msg_id = KBASE_TL_KBASE_NEW_DEVICE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kbase_device_id) + + sizeof(kbase_device_gpu_core_count) + + sizeof(kbase_device_max_num_csgs) + + sizeof(kbase_device_as_count) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_id, sizeof(kbase_device_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_gpu_core_count, sizeof(kbase_device_gpu_core_count)); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_max_num_csgs, sizeof(kbase_device_max_num_csgs)); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_as_count, sizeof(kbase_device_as_count)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_device_program_csg( + struct kbase_tlstream *stream, + u32 kbase_device_id, + u32 gpu_cmdq_grp_handle, + u32 kbase_device_csg_slot_index) +{ + const u32 msg_id = KBASE_TL_KBASE_DEVICE_PROGRAM_CSG; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kbase_device_id) + + sizeof(gpu_cmdq_grp_handle) + + sizeof(kbase_device_csg_slot_index) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_id, sizeof(kbase_device_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu_cmdq_grp_handle, sizeof(gpu_cmdq_grp_handle)); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_csg_slot_index, sizeof(kbase_device_csg_slot_index)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_device_deprogram_csg( + struct kbase_tlstream *stream, + u32 kbase_device_id, + u32 kbase_device_csg_slot_index) +{ + const u32 msg_id = KBASE_TL_KBASE_DEVICE_DEPROGRAM_CSG; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kbase_device_id) + + sizeof(kbase_device_csg_slot_index) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_id, sizeof(kbase_device_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_csg_slot_index, sizeof(kbase_device_csg_slot_index)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_new_ctx( + struct kbase_tlstream *stream, + u32 kernel_ctx_id, + u32 kbase_device_id) +{ + const u32 msg_id = KBASE_TL_KBASE_NEW_CTX; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kernel_ctx_id) + + sizeof(kbase_device_id) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_id, sizeof(kbase_device_id)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_del_ctx( + struct kbase_tlstream *stream, + u32 kernel_ctx_id) +{ + const u32 msg_id = KBASE_TL_KBASE_DEL_CTX; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kernel_ctx_id) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_ctx_assign_as( + struct kbase_tlstream *stream, + u32 kernel_ctx_id, + u32 kbase_device_as_index) +{ + const u32 msg_id = KBASE_TL_KBASE_CTX_ASSIGN_AS; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kernel_ctx_id) + + sizeof(kbase_device_as_index) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &kbase_device_as_index, sizeof(kbase_device_as_index)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_ctx_unassign_as( + struct kbase_tlstream *stream, + u32 kernel_ctx_id) +{ + const u32 msg_id = KBASE_TL_KBASE_CTX_UNASSIGN_AS; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kernel_ctx_id) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_new_kcpuqueue( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u32 kernel_ctx_id, + u32 kcpuq_num_pending_cmds) +{ + const u32 msg_id = KBASE_TL_KBASE_NEW_KCPUQUEUE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(kernel_ctx_id) + + sizeof(kcpuq_num_pending_cmds) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &kernel_ctx_id, sizeof(kernel_ctx_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpuq_num_pending_cmds, sizeof(kcpuq_num_pending_cmds)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_del_kcpuqueue( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_DEL_KCPUQUEUE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_signal( + struct kbase_tlstream *stream, + const void *kcpu_queue, + const void *fence) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(fence) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &fence, sizeof(fence)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( + struct kbase_tlstream *stream, + const void *kcpu_queue, + const void *fence) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(fence) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &fence, sizeof(fence)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 cqs_obj_gpu_addr, + u32 cqs_obj_compare_value) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(cqs_obj_gpu_addr) + + sizeof(cqs_obj_compare_value) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &cqs_obj_gpu_addr, sizeof(cqs_obj_gpu_addr)); + pos = kbasep_serialize_bytes(buffer, + pos, &cqs_obj_compare_value, sizeof(cqs_obj_compare_value)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 cqs_obj_gpu_addr) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(cqs_obj_gpu_addr) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &cqs_obj_gpu_addr, sizeof(cqs_obj_gpu_addr)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 map_import_buf_gpu_addr) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(map_import_buf_gpu_addr) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 map_import_buf_gpu_addr) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(map_import_buf_gpu_addr) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 map_import_buf_gpu_addr) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(map_import_buf_gpu_addr) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_alloc( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 jit_alloc_gpu_alloc_addr_dest, + u64 jit_alloc_va_pages, + u64 jit_alloc_commit_pages, + u64 jit_alloc_extent, + u32 jit_alloc_jit_id, + u32 jit_alloc_bin_id, + u32 jit_alloc_max_allocations, + u32 jit_alloc_flags, + u32 jit_alloc_usage_id) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(jit_alloc_gpu_alloc_addr_dest) + + sizeof(jit_alloc_va_pages) + + sizeof(jit_alloc_commit_pages) + + sizeof(jit_alloc_extent) + + sizeof(jit_alloc_jit_id) + + sizeof(jit_alloc_bin_id) + + sizeof(jit_alloc_max_allocations) + + sizeof(jit_alloc_flags) + + sizeof(jit_alloc_usage_id) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_gpu_alloc_addr_dest, sizeof(jit_alloc_gpu_alloc_addr_dest)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_va_pages, sizeof(jit_alloc_va_pages)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_commit_pages, sizeof(jit_alloc_commit_pages)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_extent, sizeof(jit_alloc_extent)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_jit_id, sizeof(jit_alloc_jit_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_bin_id, sizeof(jit_alloc_bin_id)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_max_allocations, sizeof(jit_alloc_max_allocations)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_flags, sizeof(jit_alloc_flags)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_usage_id, sizeof(jit_alloc_usage_id)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_alloc( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_free( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_free( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u32 jit_alloc_jit_id) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(jit_alloc_jit_id) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_jit_id, sizeof(jit_alloc_jit_id)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_free( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 jit_alloc_gpu_alloc_addr, + u64 jit_alloc_mmu_flags) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(jit_alloc_gpu_alloc_addr) + + sizeof(jit_alloc_mmu_flags) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_gpu_alloc_addr, sizeof(jit_alloc_gpu_alloc_addr)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_alloc_mmu_flags, sizeof(jit_alloc_mmu_flags)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_free_start( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 jit_free_pages_used) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(jit_free_pages_used) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &jit_free_pages_used, sizeof(jit_free_pages_used)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( + struct kbase_tlstream *stream, + u64 csffw_timestamp, + u64 csffw_cycle) +{ + const u32 msg_id = KBASE_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(csffw_timestamp) + + sizeof(csffw_cycle) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &csffw_timestamp, sizeof(csffw_timestamp)); + pos = kbasep_serialize_bytes(buffer, + pos, &csffw_cycle, sizeof(csffw_cycle)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_kbase_csffw_reset( + struct kbase_tlstream *stream, + u64 csffw_cycle) +{ + const u32 msg_id = KBASE_TL_KBASE_CSFFW_RESET; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(csffw_cycle) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &csffw_cycle, sizeof(csffw_cycle)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +/* clang-format on */ diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h new file mode 100755 index 000000000000..5651f0a0fc57 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h @@ -0,0 +1,2926 @@ +/* + * + * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +/* + * THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. + * DO NOT EDIT. + */ + +#if !defined(_KBASE_TRACEPOINTS_H) +#define _KBASE_TRACEPOINTS_H + +/* Tracepoints are abstract callbacks notifying that some important + * software or hardware event has happened. + * + * In this particular implementation, it results into a MIPE + * timeline event and, in some cases, it also fires an ftrace event + * (a.k.a. Gator events, see details below). + */ + +#include "mali_kbase.h" +#include "mali_kbase_gator.h" + +#include +#include + +/* clang-format off */ + +struct kbase_tlstream; + +extern const size_t __obj_stream_offset; +extern const size_t __aux_stream_offset; + +/* This macro dispatches a kbase_tlstream from + * a kbase_device instance. Only AUX or OBJ + * streams can be dispatched. It is aware of + * kbase_timeline binary representation and + * relies on offset variables: + * __obj_stream_offset and __aux_stream_offset. + */ +#define __TL_DISPATCH_STREAM(kbdev, stype) \ + ((struct kbase_tlstream *) \ + ((u8 *)kbdev->timeline + __ ## stype ## _stream_offset)) + +struct tp_desc; + +/* Descriptors of timeline messages transmitted in object events stream. */ +extern const char *obj_desc_header; +extern const size_t obj_desc_header_size; +/* Descriptors of timeline messages transmitted in auxiliary events stream. */ +extern const char *aux_desc_header; +extern const size_t aux_desc_header_size; + +#define TL_ATOM_STATE_IDLE 0 +#define TL_ATOM_STATE_READY 1 +#define TL_ATOM_STATE_DONE 2 +#define TL_ATOM_STATE_POSTED 3 + +#define TL_JS_EVENT_START GATOR_JOB_SLOT_START +#define TL_JS_EVENT_STOP GATOR_JOB_SLOT_STOP +#define TL_JS_EVENT_SOFT_STOP GATOR_JOB_SLOT_SOFT_STOPPED + +#define TLSTREAM_ENABLED (1 << 31) + +void __kbase_tlstream_tl_new_ctx( + struct kbase_tlstream *stream, + const void *ctx, + u32 ctx_nr, + u32 tgid); +void __kbase_tlstream_tl_new_gpu( + struct kbase_tlstream *stream, + const void *gpu, + u32 gpu_id, + u32 core_count); +void __kbase_tlstream_tl_new_lpu( + struct kbase_tlstream *stream, + const void *lpu, + u32 lpu_nr, + u32 lpu_fn); +void __kbase_tlstream_tl_new_atom( + struct kbase_tlstream *stream, + const void *atom, + u32 atom_nr); +void __kbase_tlstream_tl_new_as( + struct kbase_tlstream *stream, + const void *address_space, + u32 as_nr); +void __kbase_tlstream_tl_del_ctx( + struct kbase_tlstream *stream, + const void *ctx); +void __kbase_tlstream_tl_del_atom( + struct kbase_tlstream *stream, + const void *atom); +void __kbase_tlstream_tl_lifelink_lpu_gpu( + struct kbase_tlstream *stream, + const void *lpu, + const void *gpu); +void __kbase_tlstream_tl_lifelink_as_gpu( + struct kbase_tlstream *stream, + const void *address_space, + const void *gpu); +void __kbase_tlstream_tl_ret_ctx_lpu( + struct kbase_tlstream *stream, + const void *ctx, + const void *lpu); +void __kbase_tlstream_tl_ret_atom_ctx( + struct kbase_tlstream *stream, + const void *atom, + const void *ctx); +void __kbase_tlstream_tl_ret_atom_lpu( + struct kbase_tlstream *stream, + const void *atom, + const void *lpu, + const char *attrib_match_list); +void __kbase_tlstream_tl_nret_ctx_lpu( + struct kbase_tlstream *stream, + const void *ctx, + const void *lpu); +void __kbase_tlstream_tl_nret_atom_ctx( + struct kbase_tlstream *stream, + const void *atom, + const void *ctx); +void __kbase_tlstream_tl_nret_atom_lpu( + struct kbase_tlstream *stream, + const void *atom, + const void *lpu); +void __kbase_tlstream_tl_ret_as_ctx( + struct kbase_tlstream *stream, + const void *address_space, + const void *ctx); +void __kbase_tlstream_tl_nret_as_ctx( + struct kbase_tlstream *stream, + const void *address_space, + const void *ctx); +void __kbase_tlstream_tl_ret_atom_as( + struct kbase_tlstream *stream, + const void *atom, + const void *address_space); +void __kbase_tlstream_tl_nret_atom_as( + struct kbase_tlstream *stream, + const void *atom, + const void *address_space); +void __kbase_tlstream_tl_attrib_atom_config( + struct kbase_tlstream *stream, + const void *atom, + u64 descriptor, + u64 affinity, + u32 config); +void __kbase_tlstream_tl_attrib_atom_priority( + struct kbase_tlstream *stream, + const void *atom, + u32 prio); +void __kbase_tlstream_tl_attrib_atom_state( + struct kbase_tlstream *stream, + const void *atom, + u32 state); +void __kbase_tlstream_tl_attrib_atom_prioritized( + struct kbase_tlstream *stream, + const void *atom); +void __kbase_tlstream_tl_attrib_atom_jit( + struct kbase_tlstream *stream, + const void *atom, + u64 edit_addr, + u64 new_addr, + u32 jit_flags, + u64 mem_flags, + u32 j_id, + u64 com_pgs, + u64 extent, + u64 va_pgs); +void __kbase_tlstream_tl_jit_usedpages( + struct kbase_tlstream *stream, + u64 used_pages, + u32 j_id); +void __kbase_tlstream_tl_attrib_atom_jitallocinfo( + struct kbase_tlstream *stream, + const void *atom, + u64 va_pgs, + u64 com_pgs, + u64 extent, + u32 j_id, + u32 bin_id, + u32 max_allocs, + u32 jit_flags, + u32 usg_id); +void __kbase_tlstream_tl_attrib_atom_jitfreeinfo( + struct kbase_tlstream *stream, + const void *atom, + u32 j_id); +void __kbase_tlstream_tl_attrib_as_config( + struct kbase_tlstream *stream, + const void *address_space, + u64 transtab, + u64 memattr, + u64 transcfg); +void __kbase_tlstream_tl_event_lpu_softstop( + struct kbase_tlstream *stream, + const void *lpu); +void __kbase_tlstream_tl_event_atom_softstop_ex( + struct kbase_tlstream *stream, + const void *atom); +void __kbase_tlstream_tl_event_atom_softstop_issue( + struct kbase_tlstream *stream, + const void *atom); +void __kbase_tlstream_tl_event_atom_softjob_start( + struct kbase_tlstream *stream, + const void *atom); +void __kbase_tlstream_tl_event_atom_softjob_end( + struct kbase_tlstream *stream, + const void *atom); +void __kbase_tlstream_tl_arbiter_granted( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_tl_arbiter_started( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_tl_arbiter_stop_requested( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_tl_arbiter_stopped( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_jd_gpu_soft_reset( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_aux_pm_state( + struct kbase_tlstream *stream, + u32 core_type, + u64 core_state_bitset); +void __kbase_tlstream_aux_pagefault( + struct kbase_tlstream *stream, + u32 ctx_nr, + u32 as_nr, + u64 page_cnt_change); +void __kbase_tlstream_aux_pagesalloc( + struct kbase_tlstream *stream, + u32 ctx_nr, + u64 page_cnt); +void __kbase_tlstream_aux_devfreq_target( + struct kbase_tlstream *stream, + u64 target_freq); +void __kbase_tlstream_aux_protected_enter_start( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_aux_protected_enter_end( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_aux_protected_leave_start( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_aux_protected_leave_end( + struct kbase_tlstream *stream, + const void *gpu); +void __kbase_tlstream_aux_jit_stats( + struct kbase_tlstream *stream, + u32 ctx_nr, + u32 bid, + u32 max_allocs, + u32 allocs, + u32 va_pages, + u32 ph_pages); +void __kbase_tlstream_aux_event_job_slot( + struct kbase_tlstream *stream, + const void *ctx, + u32 slot_nr, + u32 atom_nr, + u32 event); +void __kbase_tlstream_tl_kbase_new_device( + struct kbase_tlstream *stream, + u32 kbase_device_id, + u32 kbase_device_gpu_core_count, + u32 kbase_device_max_num_csgs, + u32 kbase_device_as_count); +void __kbase_tlstream_tl_kbase_device_program_csg( + struct kbase_tlstream *stream, + u32 kbase_device_id, + u32 gpu_cmdq_grp_handle, + u32 kbase_device_csg_slot_index); +void __kbase_tlstream_tl_kbase_device_deprogram_csg( + struct kbase_tlstream *stream, + u32 kbase_device_id, + u32 kbase_device_csg_slot_index); +void __kbase_tlstream_tl_kbase_new_ctx( + struct kbase_tlstream *stream, + u32 kernel_ctx_id, + u32 kbase_device_id); +void __kbase_tlstream_tl_kbase_del_ctx( + struct kbase_tlstream *stream, + u32 kernel_ctx_id); +void __kbase_tlstream_tl_kbase_ctx_assign_as( + struct kbase_tlstream *stream, + u32 kernel_ctx_id, + u32 kbase_device_as_index); +void __kbase_tlstream_tl_kbase_ctx_unassign_as( + struct kbase_tlstream *stream, + u32 kernel_ctx_id); +void __kbase_tlstream_tl_kbase_new_kcpuqueue( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u32 kernel_ctx_id, + u32 kcpuq_num_pending_cmds); +void __kbase_tlstream_tl_kbase_del_kcpuqueue( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_signal( + struct kbase_tlstream *stream, + const void *kcpu_queue, + const void *fence); +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( + struct kbase_tlstream *stream, + const void *kcpu_queue, + const void *fence); +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 cqs_obj_gpu_addr, + u32 cqs_obj_compare_value); +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 cqs_obj_gpu_addr); +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 map_import_buf_gpu_addr); +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 map_import_buf_gpu_addr); +void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 map_import_buf_gpu_addr); +void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_alloc( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 jit_alloc_gpu_alloc_addr_dest, + u64 jit_alloc_va_pages, + u64 jit_alloc_commit_pages, + u64 jit_alloc_extent, + u32 jit_alloc_jit_id, + u32 jit_alloc_bin_id, + u32 jit_alloc_max_allocations, + u32 jit_alloc_flags, + u32 jit_alloc_usage_id); +void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_alloc( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_free( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_free( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u32 jit_alloc_jit_id); +void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_free( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 jit_alloc_gpu_alloc_addr, + u64 jit_alloc_mmu_flags); +void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_free_start( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 jit_free_pages_used); +void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( + struct kbase_tlstream *stream, + u64 csffw_timestamp, + u64 csffw_cycle); +void __kbase_tlstream_tl_kbase_csffw_reset( + struct kbase_tlstream *stream, + u64 csffw_cycle); + +struct kbase_tlstream; + +/** + * KBASE_TLSTREAM_TL_NEW_CTX - + * object ctx is created + * + * @kbdev: Kbase device + * @ctx: Name of the context object + * @ctx_nr: Kernel context number + * @tgid: Thread Group Id + */ +#define KBASE_TLSTREAM_TL_NEW_CTX( \ + kbdev, \ + ctx, \ + ctx_nr, \ + tgid \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_new_ctx( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + ctx, ctx_nr, tgid); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_NEW_GPU - + * object gpu is created + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + * @gpu_id: Name of the GPU object + * @core_count: Number of cores this GPU hosts + */ +#define KBASE_TLSTREAM_TL_NEW_GPU( \ + kbdev, \ + gpu, \ + gpu_id, \ + core_count \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_new_gpu( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu, gpu_id, core_count); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_NEW_LPU - + * object lpu is created + * + * @kbdev: Kbase device + * @lpu: Name of the Logical Processing Unit object + * @lpu_nr: Sequential number assigned to the newly created LPU + * @lpu_fn: Property describing functional abilities of this LPU + */ +#define KBASE_TLSTREAM_TL_NEW_LPU( \ + kbdev, \ + lpu, \ + lpu_nr, \ + lpu_fn \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_new_lpu( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + lpu, lpu_nr, lpu_fn); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_NEW_ATOM - + * object atom is created + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @atom_nr: Sequential number of an atom + */ +#define KBASE_TLSTREAM_TL_NEW_ATOM( \ + kbdev, \ + atom, \ + atom_nr \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_new_atom( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, atom_nr); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_NEW_AS - + * address space object is created + * + * @kbdev: Kbase device + * @address_space: Name of the address space object + * @as_nr: Address space number + */ +#define KBASE_TLSTREAM_TL_NEW_AS( \ + kbdev, \ + address_space, \ + as_nr \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_new_as( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + address_space, as_nr); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_DEL_CTX - + * context is destroyed + * + * @kbdev: Kbase device + * @ctx: Name of the context object + */ +#define KBASE_TLSTREAM_TL_DEL_CTX( \ + kbdev, \ + ctx \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_del_ctx( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + ctx); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_DEL_ATOM - + * atom is destroyed + * + * @kbdev: Kbase device + * @atom: Atom identifier + */ +#define KBASE_TLSTREAM_TL_DEL_ATOM( \ + kbdev, \ + atom \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_del_atom( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_LIFELINK_LPU_GPU - + * lpu is deleted with gpu + * + * @kbdev: Kbase device + * @lpu: Name of the Logical Processing Unit object + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_LIFELINK_LPU_GPU( \ + kbdev, \ + lpu, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_lifelink_lpu_gpu( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + lpu, gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_LIFELINK_AS_GPU - + * address space is deleted with gpu + * + * @kbdev: Kbase device + * @address_space: Name of the address space object + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_LIFELINK_AS_GPU( \ + kbdev, \ + address_space, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_lifelink_as_gpu( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + address_space, gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_RET_CTX_LPU - + * context is retained by lpu + * + * @kbdev: Kbase device + * @ctx: Name of the context object + * @lpu: Name of the Logical Processing Unit object + */ +#define KBASE_TLSTREAM_TL_RET_CTX_LPU( \ + kbdev, \ + ctx, \ + lpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_ret_ctx_lpu( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + ctx, lpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_RET_ATOM_CTX - + * atom is retained by context + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @ctx: Name of the context object + */ +#define KBASE_TLSTREAM_TL_RET_ATOM_CTX( \ + kbdev, \ + atom, \ + ctx \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_ret_atom_ctx( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, ctx); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_RET_ATOM_LPU - + * atom is retained by lpu + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @lpu: Name of the Logical Processing Unit object + * @attrib_match_list: List containing match operator attributes + */ +#define KBASE_TLSTREAM_TL_RET_ATOM_LPU( \ + kbdev, \ + atom, \ + lpu, \ + attrib_match_list \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_ret_atom_lpu( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, lpu, attrib_match_list); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_NRET_CTX_LPU - + * context is released by lpu + * + * @kbdev: Kbase device + * @ctx: Name of the context object + * @lpu: Name of the Logical Processing Unit object + */ +#define KBASE_TLSTREAM_TL_NRET_CTX_LPU( \ + kbdev, \ + ctx, \ + lpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_nret_ctx_lpu( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + ctx, lpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_NRET_ATOM_CTX - + * atom is released by context + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @ctx: Name of the context object + */ +#define KBASE_TLSTREAM_TL_NRET_ATOM_CTX( \ + kbdev, \ + atom, \ + ctx \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_nret_atom_ctx( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, ctx); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_NRET_ATOM_LPU - + * atom is released by lpu + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @lpu: Name of the Logical Processing Unit object + */ +#define KBASE_TLSTREAM_TL_NRET_ATOM_LPU( \ + kbdev, \ + atom, \ + lpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_nret_atom_lpu( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, lpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_RET_AS_CTX - + * address space is retained by context + * + * @kbdev: Kbase device + * @address_space: Name of the address space object + * @ctx: Name of the context object + */ +#define KBASE_TLSTREAM_TL_RET_AS_CTX( \ + kbdev, \ + address_space, \ + ctx \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_ret_as_ctx( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + address_space, ctx); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_NRET_AS_CTX - + * address space is released by context + * + * @kbdev: Kbase device + * @address_space: Name of the address space object + * @ctx: Name of the context object + */ +#define KBASE_TLSTREAM_TL_NRET_AS_CTX( \ + kbdev, \ + address_space, \ + ctx \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_nret_as_ctx( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + address_space, ctx); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_RET_ATOM_AS - + * atom is retained by address space + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @address_space: Name of the address space object + */ +#define KBASE_TLSTREAM_TL_RET_ATOM_AS( \ + kbdev, \ + atom, \ + address_space \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_ret_atom_as( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, address_space); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_NRET_ATOM_AS - + * atom is released by address space + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @address_space: Name of the address space object + */ +#define KBASE_TLSTREAM_TL_NRET_ATOM_AS( \ + kbdev, \ + atom, \ + address_space \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_nret_atom_as( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, address_space); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG - + * atom job slot attributes + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @descriptor: Job descriptor address + * @affinity: Job affinity + * @config: Job config + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG( \ + kbdev, \ + atom, \ + descriptor, \ + affinity, \ + config \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_attrib_atom_config( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, descriptor, affinity, config); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY - + * atom priority + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @prio: Atom priority + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY( \ + kbdev, \ + atom, \ + prio \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ + __kbase_tlstream_tl_attrib_atom_priority( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, prio); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE - + * atom state + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @state: Atom state + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE( \ + kbdev, \ + atom, \ + state \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ + __kbase_tlstream_tl_attrib_atom_state( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, state); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED - + * atom caused priority change + * + * @kbdev: Kbase device + * @atom: Atom identifier + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED( \ + kbdev, \ + atom \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ + __kbase_tlstream_tl_attrib_atom_prioritized( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - + * jit done for atom + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @edit_addr: Address edited by jit + * @new_addr: Address placed into the edited location + * @jit_flags: Flags specifying the special requirements for + * the JIT allocation. + * @mem_flags: Flags defining the properties of a memory region + * @j_id: Unique ID provided by the caller, this is used + * to pair allocation and free requests. + * @com_pgs: The minimum number of physical pages which + * should back the allocation. + * @extent: Granularity of physical pages to grow the + * allocation by during a fault. + * @va_pgs: The minimum number of virtual pages required + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( \ + kbdev, \ + atom, \ + edit_addr, \ + new_addr, \ + jit_flags, \ + mem_flags, \ + j_id, \ + com_pgs, \ + extent, \ + va_pgs \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ + __kbase_tlstream_tl_attrib_atom_jit( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, edit_addr, new_addr, jit_flags, mem_flags, j_id, com_pgs, extent, va_pgs); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_JIT_USEDPAGES - + * used pages for jit + * + * @kbdev: Kbase device + * @used_pages: Number of pages used for jit + * @j_id: Unique ID provided by the caller, this is used + * to pair allocation and free requests. + */ +#define KBASE_TLSTREAM_TL_JIT_USEDPAGES( \ + kbdev, \ + used_pages, \ + j_id \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_jit_usedpages( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + used_pages, j_id); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO - + * Information about JIT allocations + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @va_pgs: The minimum number of virtual pages required + * @com_pgs: The minimum number of physical pages which + * should back the allocation. + * @extent: Granularity of physical pages to grow the + * allocation by during a fault. + * @j_id: Unique ID provided by the caller, this is used + * to pair allocation and free requests. + * @bin_id: The JIT allocation bin, used in conjunction with + * max_allocations to limit the number of each + * type of JIT allocation. + * @max_allocs: Maximum allocations allowed in this bin. + * @jit_flags: Flags specifying the special requirements for + * the JIT allocation. + * @usg_id: A hint about which allocation should be reused. + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO( \ + kbdev, \ + atom, \ + va_pgs, \ + com_pgs, \ + extent, \ + j_id, \ + bin_id, \ + max_allocs, \ + jit_flags, \ + usg_id \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_attrib_atom_jitallocinfo( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, va_pgs, com_pgs, extent, j_id, bin_id, max_allocs, jit_flags, usg_id); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO - + * Information about JIT frees + * + * @kbdev: Kbase device + * @atom: Atom identifier + * @j_id: Unique ID provided by the caller, this is used + * to pair allocation and free requests. + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO( \ + kbdev, \ + atom, \ + j_id \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_attrib_atom_jitfreeinfo( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom, j_id); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG - + * address space attributes + * + * @kbdev: Kbase device + * @address_space: Name of the address space object + * @transtab: Configuration of the TRANSTAB register + * @memattr: Configuration of the MEMATTR register + * @transcfg: Configuration of the TRANSCFG register (or zero if not present) + */ +#define KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG( \ + kbdev, \ + address_space, \ + transtab, \ + memattr, \ + transcfg \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_attrib_as_config( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + address_space, transtab, memattr, transcfg); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP - + * softstop event on given lpu + * + * @kbdev: Kbase device + * @lpu: Name of the Logical Processing Unit object + */ +#define KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( \ + kbdev, \ + lpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_lpu_softstop( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + lpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX - + * atom softstopped + * + * @kbdev: Kbase device + * @atom: Atom identifier + */ +#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX( \ + kbdev, \ + atom \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_atom_softstop_ex( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE - + * atom softstop issued + * + * @kbdev: Kbase device + * @atom: Atom identifier + */ +#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE( \ + kbdev, \ + atom \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_atom_softstop_issue( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START - + * atom soft job has started + * + * @kbdev: Kbase device + * @atom: Atom identifier + */ +#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START( \ + kbdev, \ + atom \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_atom_softjob_start( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END - + * atom soft job has completed + * + * @kbdev: Kbase device + * @atom: Atom identifier + */ +#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END( \ + kbdev, \ + atom \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_atom_softjob_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + atom); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ARBITER_GRANTED - + * Arbiter has granted gpu access + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_ARBITER_GRANTED( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_arbiter_granted( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ARBITER_STARTED - + * Driver is running again and able to process jobs + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_ARBITER_STARTED( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_arbiter_started( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED - + * Arbiter has requested driver to stop using gpu + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_arbiter_stop_requested( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_ARBITER_STOPPED - + * Driver has stopped using gpu + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_ARBITER_STOPPED( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_arbiter_stopped( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_JD_GPU_SOFT_RESET - + * gpu soft reset + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_JD_GPU_SOFT_RESET( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_jd_gpu_soft_reset( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_AUX_PM_STATE - + * PM state + * + * @kbdev: Kbase device + * @core_type: Core type (shader, tiler, l2 cache, l3 cache) + * @core_state_bitset: 64bits bitmask reporting power state of the cores + * (1-ON, 0-OFF) + */ +#define KBASE_TLSTREAM_AUX_PM_STATE( \ + kbdev, \ + core_type, \ + core_state_bitset \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_pm_state( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + core_type, core_state_bitset); \ + } while (0) + +/** + * KBASE_TLSTREAM_AUX_PAGEFAULT - + * Page fault + * + * @kbdev: Kbase device + * @ctx_nr: Kernel context number + * @as_nr: Address space number + * @page_cnt_change: Number of pages to be added + */ +#define KBASE_TLSTREAM_AUX_PAGEFAULT( \ + kbdev, \ + ctx_nr, \ + as_nr, \ + page_cnt_change \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_pagefault( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + ctx_nr, as_nr, page_cnt_change); \ + } while (0) + +/** + * KBASE_TLSTREAM_AUX_PAGESALLOC - + * Total alloc pages change + * + * @kbdev: Kbase device + * @ctx_nr: Kernel context number + * @page_cnt: Number of pages used by the context + */ +#define KBASE_TLSTREAM_AUX_PAGESALLOC( \ + kbdev, \ + ctx_nr, \ + page_cnt \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_pagesalloc( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + ctx_nr, page_cnt); \ + } while (0) + +/** + * KBASE_TLSTREAM_AUX_DEVFREQ_TARGET - + * New device frequency target + * + * @kbdev: Kbase device + * @target_freq: New target frequency + */ +#define KBASE_TLSTREAM_AUX_DEVFREQ_TARGET( \ + kbdev, \ + target_freq \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_devfreq_target( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + target_freq); \ + } while (0) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START - + * enter protected mode start + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ + __kbase_tlstream_aux_protected_enter_start( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END - + * enter protected mode end + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ + __kbase_tlstream_aux_protected_enter_end( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START - + * leave protected mode start + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ + __kbase_tlstream_aux_protected_leave_start( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END - + * leave protected mode end + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END( \ + kbdev, \ + gpu \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ + __kbase_tlstream_aux_protected_leave_end( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + gpu); \ + } while (0) + +/** + * KBASE_TLSTREAM_AUX_JIT_STATS - + * per-bin JIT statistics + * + * @kbdev: Kbase device + * @ctx_nr: Kernel context number + * @bid: JIT bin id + * @max_allocs: Maximum allocations allowed in this bin. + * @allocs: Number of active allocations in this bin + * @va_pages: Number of virtual pages allocated in this bin + * @ph_pages: Number of physical pages allocated in this bin + */ +#define KBASE_TLSTREAM_AUX_JIT_STATS( \ + kbdev, \ + ctx_nr, \ + bid, \ + max_allocs, \ + allocs, \ + va_pages, \ + ph_pages \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_jit_stats( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + ctx_nr, bid, max_allocs, allocs, va_pages, ph_pages); \ + } while (0) + +/** + * KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT - + * event on a given job slot + * + * @kbdev: Kbase device + * @ctx: Name of the context object + * @slot_nr: Job slot number + * @atom_nr: Sequential number of an atom + * @event: Event type. One of TL_JS_EVENT values + */ +#define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT( \ + kbdev, \ + ctx, \ + slot_nr, \ + atom_nr, \ + event \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_event_job_slot( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + ctx, slot_nr, atom_nr, event); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE - + * New KBase Device + * + * @kbdev: Kbase device + * @kbase_device_id: The id of the physical hardware + * @kbase_device_gpu_core_count: The number of gpu cores in the physical hardware + * @kbase_device_max_num_csgs: The max number of CSGs the physical hardware supports + * @kbase_device_as_count: The number of address spaces the physical hardware has available + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ + kbdev, \ + kbase_device_id, \ + kbase_device_gpu_core_count, \ + kbase_device_max_num_csgs, \ + kbase_device_as_count \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_new_device( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kbase_device_id, kbase_device_gpu_core_count, kbase_device_max_num_csgs, kbase_device_as_count); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ + kbdev, \ + kbase_device_id, \ + kbase_device_gpu_core_count, \ + kbase_device_max_num_csgs, \ + kbase_device_as_count \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG - + * CSG is programmed to a slot + * + * @kbdev: Kbase device + * @kbase_device_id: The id of the physical hardware + * @gpu_cmdq_grp_handle: GPU Command Queue Group handle which will match userspace + * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ + kbdev, \ + kbase_device_id, \ + gpu_cmdq_grp_handle, \ + kbase_device_csg_slot_index \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_device_program_csg( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kbase_device_id, gpu_cmdq_grp_handle, kbase_device_csg_slot_index); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ + kbdev, \ + kbase_device_id, \ + gpu_cmdq_grp_handle, \ + kbase_device_csg_slot_index \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG - + * CSG is deprogrammed from a slot + * + * @kbdev: Kbase device + * @kbase_device_id: The id of the physical hardware + * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ + kbdev, \ + kbase_device_id, \ + kbase_device_csg_slot_index \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_device_deprogram_csg( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kbase_device_id, kbase_device_csg_slot_index); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ + kbdev, \ + kbase_device_id, \ + kbase_device_csg_slot_index \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_NEW_CTX - + * New KBase Context + * + * @kbdev: Kbase device + * @kernel_ctx_id: Unique ID for the KBase Context + * @kbase_device_id: The id of the physical hardware + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ + kbdev, \ + kernel_ctx_id, \ + kbase_device_id \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_new_ctx( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kernel_ctx_id, kbase_device_id); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ + kbdev, \ + kernel_ctx_id, \ + kbase_device_id \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_DEL_CTX - + * Delete KBase Context + * + * @kbdev: Kbase device + * @kernel_ctx_id: Unique ID for the KBase Context + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ + kbdev, \ + kernel_ctx_id \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_del_ctx( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kernel_ctx_id); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ + kbdev, \ + kernel_ctx_id \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS - + * Address Space is assigned to a KBase context + * + * @kbdev: Kbase device + * @kernel_ctx_id: Unique ID for the KBase Context + * @kbase_device_as_index: The index of the device address space being assigned + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( \ + kbdev, \ + kernel_ctx_id, \ + kbase_device_as_index \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_ctx_assign_as( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kernel_ctx_id, kbase_device_as_index); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( \ + kbdev, \ + kernel_ctx_id, \ + kbase_device_as_index \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS - + * Address Space is unassigned from a KBase context + * + * @kbdev: Kbase device + * @kernel_ctx_id: Unique ID for the KBase Context + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( \ + kbdev, \ + kernel_ctx_id \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_ctx_unassign_as( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kernel_ctx_id); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( \ + kbdev, \ + kernel_ctx_id \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE - + * New KCPU Queue + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @kernel_ctx_id: Unique ID for the KBase Context + * @kcpuq_num_pending_cmds: Number of commands already enqueued + * in the KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ + kbdev, \ + kcpu_queue, \ + kernel_ctx_id, \ + kcpuq_num_pending_cmds \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_new_kcpuqueue( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, kernel_ctx_id, kcpuq_num_pending_cmds); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ + kbdev, \ + kcpu_queue, \ + kernel_ctx_id, \ + kcpuq_num_pending_cmds \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE - + * Delete KCPU Queue + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_del_kcpuqueue( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL - + * KCPU Queue enqueues Signal on Fence + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @fence: Fence object handle + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ + kbdev, \ + kcpu_queue, \ + fence \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_signal( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, fence); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ + kbdev, \ + kcpu_queue, \ + fence \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT - + * KCPU Queue enqueues Wait on Fence + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @fence: Fence object handle + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ + kbdev, \ + kcpu_queue, \ + fence \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, fence); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ + kbdev, \ + kcpu_queue, \ + fence \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT - + * KCPU Queue enqueues Wait on Cross Queue Sync Object + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @cqs_obj_gpu_addr: CQS Object GPU ptr + * @cqs_obj_compare_value: Semaphore value that should be exceeded + * for the WAIT to pass + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ + kbdev, \ + kcpu_queue, \ + cqs_obj_gpu_addr, \ + cqs_obj_compare_value \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, cqs_obj_gpu_addr, cqs_obj_compare_value); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ + kbdev, \ + kcpu_queue, \ + cqs_obj_gpu_addr, \ + cqs_obj_compare_value \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET - + * KCPU Queue enqueues Set on Cross Queue Sync Object + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @cqs_obj_gpu_addr: CQS Object GPU ptr + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( \ + kbdev, \ + kcpu_queue, \ + cqs_obj_gpu_addr \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, cqs_obj_gpu_addr); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( \ + kbdev, \ + kcpu_queue, \ + cqs_obj_gpu_addr \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT - + * KCPU Queue enqueues Map Import + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @map_import_buf_gpu_addr: Map import buffer GPU ptr + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ + kbdev, \ + kcpu_queue, \ + map_import_buf_gpu_addr \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, map_import_buf_gpu_addr); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ + kbdev, \ + kcpu_queue, \ + map_import_buf_gpu_addr \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT - + * KCPU Queue enqueues Unmap Import + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @map_import_buf_gpu_addr: Map import buffer GPU ptr + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ + kbdev, \ + kcpu_queue, \ + map_import_buf_gpu_addr \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, map_import_buf_gpu_addr); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ + kbdev, \ + kcpu_queue, \ + map_import_buf_gpu_addr \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE - + * KCPU Queue enqueues Unmap Import ignoring reference count + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @map_import_buf_gpu_addr: Map import buffer GPU ptr + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ + kbdev, \ + kcpu_queue, \ + map_import_buf_gpu_addr \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, map_import_buf_gpu_addr); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ + kbdev, \ + kcpu_queue, \ + map_import_buf_gpu_addr \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC - + * Begin array of KCPU Queue enqueues JIT Alloc + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC - + * Array item of KCPU Queue enqueues JIT Alloc + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @jit_alloc_gpu_alloc_addr_dest: The GPU virtual address to write + * the JIT allocated GPU virtual address to + * @jit_alloc_va_pages: The minimum number of virtual pages required + * @jit_alloc_commit_pages: The minimum number of physical pages which + * should back the allocation + * @jit_alloc_extent: Granularity of physical pages to grow the allocation + * by during a fault + * @jit_alloc_jit_id: Unique ID provided by the caller, this is used + * to pair allocation and free requests. Zero is not a valid value + * @jit_alloc_bin_id: The JIT allocation bin, used in conjunction with + * max_allocations to limit the number of each type of JIT allocation + * @jit_alloc_max_allocations: The maximum number of allocations + * allowed within the bin specified by bin_id. Should be the same for all + * JIT allocations within the same bin. + * @jit_alloc_flags: Flags specifying the special requirements for the + * JIT allocation + * @jit_alloc_usage_id: A hint about which allocation should be + * reused. The kernel should attempt to use a previous allocation with the same + * usage_id + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ + kbdev, \ + kcpu_queue, \ + jit_alloc_gpu_alloc_addr_dest, \ + jit_alloc_va_pages, \ + jit_alloc_commit_pages, \ + jit_alloc_extent, \ + jit_alloc_jit_id, \ + jit_alloc_bin_id, \ + jit_alloc_max_allocations, \ + jit_alloc_flags, \ + jit_alloc_usage_id \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_alloc( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, jit_alloc_gpu_alloc_addr_dest, jit_alloc_va_pages, jit_alloc_commit_pages, jit_alloc_extent, jit_alloc_jit_id, jit_alloc_bin_id, jit_alloc_max_allocations, jit_alloc_flags, jit_alloc_usage_id); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ + kbdev, \ + kcpu_queue, \ + jit_alloc_gpu_alloc_addr_dest, \ + jit_alloc_va_pages, \ + jit_alloc_commit_pages, \ + jit_alloc_extent, \ + jit_alloc_jit_id, \ + jit_alloc_bin_id, \ + jit_alloc_max_allocations, \ + jit_alloc_flags, \ + jit_alloc_usage_id \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC - + * End array of KCPU Queue enqueues JIT Alloc + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_alloc( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE - + * Begin array of KCPU Queue enqueues JIT Free + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_free( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE - + * Array item of KCPU Queue enqueues JIT Free + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @jit_alloc_jit_id: Unique ID provided by the caller, this is used + * to pair allocation and free requests. Zero is not a valid value + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ + kbdev, \ + kcpu_queue, \ + jit_alloc_jit_id \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_free( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, jit_alloc_jit_id); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ + kbdev, \ + kcpu_queue, \ + jit_alloc_jit_id \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE - + * End array of KCPU Queue enqueues JIT Free + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_free( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START - + * KCPU Queue starts a Signal on Fence + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END - + * KCPU Queue ends a Signal on Fence + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START - + * KCPU Queue starts a Wait on Fence + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END - + * KCPU Queue ends a Wait on Fence + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START - + * KCPU Queue starts a Wait on an array of Cross Queue Sync Objects + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END - + * KCPU Queue ends a Wait on an array of Cross Queue Sync Objects + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET - + * KCPU Queue executes a Set on an array of Cross Queue Sync Objects + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START - + * KCPU Queue starts a Map Import + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END - + * KCPU Queue ends a Map Import + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START - + * KCPU Queue starts an Unmap Import + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END - + * KCPU Queue ends an Unmap Import + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START - + * KCPU Queue starts an Unmap Import ignoring reference count + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END - + * KCPU Queue ends an Unmap Import ignoring reference count + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START - + * KCPU Queue starts an array of JIT Allocs + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - + * Begin array of KCPU Queue ends an array of JIT Allocs + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - + * Array item of KCPU Queue ends an array of JIT Allocs + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @jit_alloc_gpu_alloc_addr: The JIT allocated GPU virtual address + * @jit_alloc_mmu_flags: The MMU flags for the JIT allocation + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ + kbdev, \ + kcpu_queue, \ + jit_alloc_gpu_alloc_addr, \ + jit_alloc_mmu_flags \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, jit_alloc_gpu_alloc_addr, jit_alloc_mmu_flags); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ + kbdev, \ + kcpu_queue, \ + jit_alloc_gpu_alloc_addr, \ + jit_alloc_mmu_flags \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - + * End array of KCPU Queue ends an array of JIT Allocs + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START - + * KCPU Queue starts an array of JIT Frees + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_free_start( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END - + * Begin array of KCPU Queue ends an array of JIT Frees + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END - + * Array item of KCPU Queue ends an array of JIT Frees + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @jit_free_pages_used: The actual number of pages used by the JIT + * allocation + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ + kbdev, \ + kcpu_queue, \ + jit_free_pages_used \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, jit_free_pages_used); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ + kbdev, \ + kcpu_queue, \ + jit_free_pages_used \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END - + * End array of KCPU Queue ends an array of JIT Frees + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER - + * KCPU Queue executes an Error Barrier + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER( \ + kbdev, \ + kcpu_queue \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW - + * An overflow has happened with the CSFFW Timeline stream + * + * @kbdev: Kbase device + * @csffw_timestamp: Timestamp of a CSFFW event + * @csffw_cycle: Cycle number of a CSFFW event + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ + kbdev, \ + csffw_timestamp, \ + csffw_cycle \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + csffw_timestamp, csffw_cycle); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ + kbdev, \ + csffw_timestamp, \ + csffw_cycle \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + +/** + * KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET - + * A reset has happened with the CSFFW + * + * @kbdev: Kbase device + * @csffw_cycle: Cycle number of a CSFFW event + */ +#if MALI_USE_CSF +#define KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET( \ + kbdev, \ + csffw_cycle \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ + __kbase_tlstream_tl_kbase_csffw_reset( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + csffw_cycle); \ + } while (0) +#else +#define KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET( \ + kbdev, \ + csffw_cycle \ + ) \ + do { } while (0) +#endif /* MALI_USE_CSF */ + + +/* Gator tracepoints are hooked into TLSTREAM interface. + * When the following tracepoints are called, corresponding + * Gator tracepoint will be called as well. + */ + +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) +/* `event` is one of TL_JS_EVENT values here. + * The values of TL_JS_EVENT are guaranteed to match + * with corresponding GATOR_JOB_SLOT values. + */ +#undef KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT +#define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, \ + context, slot_nr, atom_nr, event) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + kbase_trace_mali_job_slots_event(kbdev->id, \ + GATOR_MAKE_EVENT(event, slot_nr), \ + context, (u8) atom_nr); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_event_job_slot( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + context, slot_nr, atom_nr, event); \ + } while (0) + +#undef KBASE_TLSTREAM_AUX_PM_STATE +#define KBASE_TLSTREAM_AUX_PM_STATE(kbdev, core_type, state) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + kbase_trace_mali_pm_status(kbdev->id, \ + core_type, state); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_pm_state( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + core_type, state); \ + } while (0) + +#undef KBASE_TLSTREAM_AUX_PAGEFAULT +#define KBASE_TLSTREAM_AUX_PAGEFAULT(kbdev, \ + ctx_nr, as_nr, page_cnt_change) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + kbase_trace_mali_page_fault_insert_pages(kbdev->id, \ + as_nr, \ + page_cnt_change); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_pagefault( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + ctx_nr, as_nr, page_cnt_change); \ + } while (0) + +/* kbase_trace_mali_total_alloc_pages_change is handled differently here. + * We stream the total amount of pages allocated for `kbdev` rather + * than `page_count`, which is per-context. + */ +#undef KBASE_TLSTREAM_AUX_PAGESALLOC +#define KBASE_TLSTREAM_AUX_PAGESALLOC(kbdev, ctx_nr, page_cnt) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 global_pages_count = \ + atomic_read(&kbdev->memdev.used_pages); \ + \ + kbase_trace_mali_total_alloc_pages_change(kbdev->id, \ + global_pages_count); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_aux_pagesalloc( \ + __TL_DISPATCH_STREAM(kbdev, aux), \ + ctx_nr, page_cnt); \ + } while (0) +#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ + +/* clang-format on */ +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/Kbuild b/drivers/gpu/arm/bifrost_for_linux/Kbuild new file mode 100755 index 000000000000..531b92126b98 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/Kbuild @@ -0,0 +1,172 @@ +# +# (C) COPYRIGHT 2012-2016, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + +# Driver version string which is returned to userspace via an ioctl +MALI_RELEASE_NAME ?= "r8p0-01rel0" + +# Paths required for build +KBASE_PATH = $(src) +KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy +UMP_PATH = $(src)/../../../base + +ifeq ($(CONFIG_MALI_BIFROST_ERROR_INJECT),y) +MALI_ERROR_INJECT_ON = 1 +endif + +# Set up defaults if not defined by build system +MALI_CUSTOMER_RELEASE ?= 1 +MALI_UNIT_TEST ?= 0 +MALI_KERNEL_TEST_API ?= 0 +MALI_ERROR_INJECT_ON ?= 0 +MALI_MOCK_TEST ?= 0 +MALI_COVERAGE ?= 0 +MALI_INSTRUMENTATION_LEVEL ?= 0 +CONFIG_MALI_PLATFORM_NAME ?= "devicetree" +# This workaround is for what seems to be a compiler bug we observed in +# GCC 4.7 on AOSP 4.3. The bug caused an intermittent failure compiling +# the "_Pragma" syntax, where an error message is returned: +# +# "internal compiler error: unspellable token PRAGMA" +# +# This regression has thus far only been seen on the GCC 4.7 compiler bundled +# with AOSP 4.3.0. So this makefile, intended for in-tree kernel builds +# which are not known to be used with AOSP, is hardcoded to disable the +# workaround, i.e. set the define to 0. +MALI_GCC_WORKAROUND_MIDCOM_4598 ?= 0 + +# Set up our defines, which will be passed to gcc +DEFINES = \ + -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ + -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ + -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ + -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ + -DMALI_MOCK_TEST=$(MALI_MOCK_TEST) \ + -DMALI_COVERAGE=$(MALI_COVERAGE) \ + -DMALI_INSTRUMENTATION_LEVEL=$(MALI_INSTRUMENTATION_LEVEL) \ + -DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \ + -DMALI_GCC_WORKAROUND_MIDCOM_4598=$(MALI_GCC_WORKAROUND_MIDCOM_4598) + +ifeq ($(KBUILD_EXTMOD),) +# in-tree +DEFINES +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) +else +# out-of-tree +DEFINES +=-DMALI_KBASE_PLATFORM_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) +endif + +DEFINES += -I$(srctree)/drivers/staging/android + +# Use our defines when compiling +ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux +subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(OSK_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux + +SRC := \ + mali_kbase_device.c \ + mali_kbase_cache_policy.c \ + mali_kbase_mem.c \ + mali_kbase_mmu.c \ + mali_kbase_ctx_sched.c \ + mali_kbase_jd.c \ + mali_kbase_jd_debugfs.c \ + mali_kbase_jm.c \ + mali_kbase_gpuprops.c \ + mali_kbase_js.c \ + mali_kbase_js_ctx_attr.c \ + mali_kbase_event.c \ + mali_kbase_context.c \ + mali_kbase_pm.c \ + mali_kbase_config.c \ + mali_kbase_vinstr.c \ + mali_kbase_softjobs.c \ + mali_kbase_10969_workaround.c \ + mali_kbase_hw.c \ + mali_kbase_utility.c \ + mali_kbase_debug.c \ + mali_kbase_trace_timeline.c \ + mali_kbase_gpu_memory_debugfs.c \ + mali_kbase_mem_linux.c \ + mali_kbase_core_linux.c \ + mali_kbase_replay.c \ + mali_kbase_mem_profile_debugfs.c \ + mali_kbase_mmu_mode_lpae.c \ + mali_kbase_mmu_mode_aarch64.c \ + mali_kbase_disjoint_events.c \ + mali_kbase_gator_api.c \ + mali_kbase_debug_mem_view.c \ + mali_kbase_debug_job_fault.c \ + mali_kbase_smc.c \ + mali_kbase_mem_pool.c \ + mali_kbase_mem_pool_debugfs.c \ + mali_kbase_tlstream.c \ + mali_kbase_strings.c \ + mali_kbase_as_fault_debugfs.c \ + mali_kbase_regs_history_debugfs.c + + + + +ifeq ($(MALI_UNIT_TEST),1) + SRC += mali_kbase_tlstream_test.c +endif + +ifeq ($(MALI_CUSTOMER_RELEASE),0) + SRC += mali_kbase_regs_dump_debugfs.c +endif + + +ccflags-y += -I$(KBASE_PATH) + +# Tell the Linux build system from which .o file to create the kernel module +obj-$(CONFIG_MALI_BIFROST) += bifrost_kbase.o + +# Tell the Linux build system to enable building of our .c files +bifrost_kbase-y := $(SRC:.c=.o) + +# Kconfig passes in the name with quotes for in-tree builds - remove them. +platform_name := $(shell echo $(CONFIG_MALI_PLATFORM_NAME)) +MALI_PLATFORM_DIR := platform/$(platform_name) +ccflags-y += -I$(src)/$(MALI_PLATFORM_DIR) +include $(src)/$(MALI_PLATFORM_DIR)/Kbuild + +ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y) + ifeq ($(CONFIG_DEVFREQ_THERMAL),y) + include $(src)/ipa/Kbuild + endif +endif + +bifrost_kbase-$(CONFIG_MALI_BIFROST_DMA_FENCE) += \ + mali_kbase_dma_fence.o \ + mali_kbase_fence.o +bifrost_kbase-$(CONFIG_SYNC) += \ + mali_kbase_sync_android.o \ + mali_kbase_sync_common.o +bifrost_kbase-$(CONFIG_SYNC_FILE) += \ + mali_kbase_sync_file.o \ + mali_kbase_sync_common.o \ + mali_kbase_fence.o + +ifeq ($(MALI_MOCK_TEST),1) +# Test functionality +bifrost_kbase-y += tests/internal/src/mock/mali_kbase_pm_driver_mock.o +endif + +include $(src)/backend/gpu/Kbuild +bifrost_kbase-y += $(BACKEND:.c=.o) + + +ccflags-y += -I$(src)/backend/gpu +subdir-ccflags-y += -I$(src)/backend/gpu + +# For kutf and mali_kutf_irq_latency_test +obj-$(CONFIG_MALI_KUTF) += tests/ diff --git a/drivers/gpu/arm/bifrost_for_linux/Kconfig b/drivers/gpu/arm/bifrost_for_linux/Kconfig new file mode 100755 index 000000000000..e78b634031ee --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/Kconfig @@ -0,0 +1,196 @@ +# +# (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + +menuconfig MALI_BIFROST + tristate "Mali Bifrost series support (Linux only)" + default n + help + Enable this option to build support for a ARM Mali Bifrost GPU. + + To compile this driver as a module, choose M here: + this will generate a single module, called mali_kbase. + +config MALI_BIFROST_GATOR_SUPPORT + bool "Streamline support via Gator" + depends on MALI_BIFROST + default n + help + Adds diagnostic support for use with the ARM Streamline Performance Analyzer. + You will need the Gator device driver already loaded before loading this driver when enabling + Streamline debug support. + This is a legacy interface required by older versions of Streamline. + +config MALI_BIFROST_DVFS + bool "Enable legacy DVFS" + depends on MALI_BIFROST && !MALI_BIFROST_DEVFREQ + default n + help + Choose this option to enable legacy DVFS in the Mali Midgard DDK. + +config MALI_BIFROST_ENABLE_TRACE + bool "Enable kbase tracing" + depends on MALI_BIFROST + default n + help + Enables tracing in kbase. Trace log available through + the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled + +config MALI_BIFROST_DEVFREQ + bool "devfreq support for Mali" + depends on MALI_BIFROST && PM_DEVFREQ + help + Support devfreq for Mali. + + Using the devfreq framework and, by default, the simpleondemand + governor, the frequency of Mali will be dynamically selected from the + available OPPs. + +config MALI_BIFROST_DMA_FENCE + bool "DMA_BUF fence support for Mali" + depends on MALI_BIFROST && !KDS + default n + help + Support DMA_BUF fences for Mali. + + This option should only be enabled if KDS is not present and + the Linux Kernel has built in support for DMA_BUF fences. + +config MALI_PLATFORM_NAME + depends on MALI_BIFROST + string "Platform name" + default "devicetree" + help + Enter the name of the desired platform configuration directory to + include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must + exist. + +# MALI_BIFROST_EXPERT configuration options + +menuconfig MALI_BIFROST_EXPERT + depends on MALI_BIFROST + bool "Enable Expert Settings" + default n + help + Enabling this option and modifying the default settings may produce a driver with performance or + other limitations. + +config MALI_CORESTACK + bool "Support controlling power to the GPU core stack" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Enabling this feature on supported GPUs will let the driver powering + on/off the GPU core stack independently without involving the Power + Domain Controller. This should only be enabled on platforms which + integration of the PDC to the Mali GPU is known to be problematic. + This feature is currently only supported on t-Six and t-HEx GPUs. + + If unsure, say N. + +config MALI_BIFROST_PRFCNT_SET_SECONDARY + bool "Use secondary set of performance counters" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Select this option to use secondary set of performance counters. Kernel + features that depend on an access to the primary set of counters may + become unavailable. Enabling this option will prevent power management + from working optimally and may cause instrumentation tools to return + bogus results. + + If unsure, say N. + +config MALI_BIFROST_DEBUG + bool "Debug build" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Select this option for increased checking and reporting of errors. + +config MALI_BIFROST_FENCE_DEBUG + bool "Debug sync fence usage" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT && (SYNC || SYNC_FILE) + default y if MALI_BIFROST_DEBUG + help + Select this option to enable additional checking and reporting on the + use of sync fences in the Mali driver. + + This will add a 3s timeout to all sync fence waits in the Mali + driver, so that when work for Mali has been waiting on a sync fence + for a long time a debug message will be printed, detailing what fence + is causing the block, and which dependent Mali atoms are blocked as a + result of this. + + The timeout can be changed at runtime through the js_soft_timeout + device attribute, where the timeout is specified in milliseconds. + +config MALI_BIFROST_NO_MALI + bool "No Mali" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + This can be used to test the driver in a simulated environment + whereby the hardware is not physically present. If the hardware is physically + present it will not be used. This can be used to test the majority of the + driver without needing actual hardware or for software benchmarking. + All calls to the simulated hardware will complete immediately as if the hardware + completed the task. + +config MALI_BIFROST_ERROR_INJECT + bool "Error injection" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT && MALI_BIFROST_NO_MALI + default n + help + Enables insertion of errors to test module failure and recovery mechanisms. + +config MALI_BIFROST_TRACE_TIMELINE + bool "Timeline tracing" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Enables timeline tracing through the kernel tracepoint system. + +config MALI_BIFROST_SYSTEM_TRACE + bool "Enable system event tracing support" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Choose this option to enable system trace events for each + kbase event. This is typically used for debugging but has + minimal overhead when not in use. Enable only if you know what + you are doing. + +config MALI_2MB_ALLOC + bool "Attempt to allocate 2MB pages" + depends on MALI_BIFROST && MALI_BIFROST_EXPERT + default n + help + Rather than allocating all GPU memory page-by-page, attempt to + allocate 2MB pages from the kernel. This reduces TLB pressure and + helps to prevent memory fragmentation. + + If in doubt, say N + +config MALI_PWRSOFT_765 + bool "PWRSOFT-765 ticket" + default n + help + PWRSOFT-765 fixes devfreq cooling devices issues. However, they are + not merged in mainline kernel yet. So this define helps to guard those + parts of the code. + +source "drivers/gpu/arm/bifrost/platform/Kconfig" +# source "drivers/gpu/arm/bifrost/tests/Kconfig" diff --git a/drivers/gpu/arm/bifrost_for_linux/Makefile b/drivers/gpu/arm/bifrost_for_linux/Makefile new file mode 100755 index 000000000000..26522d566dd0 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/Makefile @@ -0,0 +1,42 @@ +# +# (C) COPYRIGHT 2010-2016, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + +KDIR ?= /lib/modules/$(shell uname -r)/build + +BUSLOG_PATH_RELATIVE = $(CURDIR)/../../../.. +UMP_PATH_RELATIVE = $(CURDIR)/../../../base/ump +KBASE_PATH_RELATIVE = $(CURDIR) +KDS_PATH_RELATIVE = $(CURDIR)/../../../.. +EXTRA_SYMBOLS = $(UMP_PATH_RELATIVE)/src/Module.symvers + +ifeq ($(MALI_UNIT_TEST), 1) + EXTRA_SYMBOLS += $(KBASE_PATH_RELATIVE)/tests/internal/src/kernel_assert_module/linux/Module.symvers +endif + +ifeq ($(CONFIG_MALI_FPGA_BUS_LOGGER),y) +#Add bus logger symbols +EXTRA_SYMBOLS += $(BUSLOG_PATH_RELATIVE)/drivers/base/bus_logger/Module.symvers +endif + +# GPL driver supports KDS +EXTRA_SYMBOLS += $(KDS_PATH_RELATIVE)/drivers/base/kds/Module.symvers + +# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions +all: + $(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules + +clean: + $(MAKE) -C $(KDIR) M=$(CURDIR) clean diff --git a/drivers/gpu/arm/bifrost_for_linux/Makefile.kbase b/drivers/gpu/arm/bifrost_for_linux/Makefile.kbase new file mode 100755 index 000000000000..2bef9c25eaeb --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/Makefile.kbase @@ -0,0 +1,17 @@ +# +# (C) COPYRIGHT 2010 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +EXTRA_CFLAGS += -I$(ROOT) -I$(KBASE_PATH) -I$(OSK_PATH)/src/linux/include -I$(KBASE_PATH)/platform_$(PLATFORM) + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/Kbuild b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/Kbuild new file mode 100755 index 000000000000..5eeba1b14710 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/Kbuild @@ -0,0 +1,60 @@ +# +# (C) COPYRIGHT 2014,2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +BACKEND += \ + backend/gpu/mali_kbase_cache_policy_backend.c \ + backend/gpu/mali_kbase_device_hw.c \ + backend/gpu/mali_kbase_gpu.c \ + backend/gpu/mali_kbase_gpuprops_backend.c \ + backend/gpu/mali_kbase_debug_job_fault_backend.c \ + backend/gpu/mali_kbase_irq_linux.c \ + backend/gpu/mali_kbase_instr_backend.c \ + backend/gpu/mali_kbase_jm_as.c \ + backend/gpu/mali_kbase_jm_hw.c \ + backend/gpu/mali_kbase_jm_rb.c \ + backend/gpu/mali_kbase_js_affinity.c \ + backend/gpu/mali_kbase_js_backend.c \ + backend/gpu/mali_kbase_mmu_hw_direct.c \ + backend/gpu/mali_kbase_pm_backend.c \ + backend/gpu/mali_kbase_pm_driver.c \ + backend/gpu/mali_kbase_pm_metrics.c \ + backend/gpu/mali_kbase_pm_ca.c \ + backend/gpu/mali_kbase_pm_ca_fixed.c \ + backend/gpu/mali_kbase_pm_always_on.c \ + backend/gpu/mali_kbase_pm_coarse_demand.c \ + backend/gpu/mali_kbase_pm_demand.c \ + backend/gpu/mali_kbase_pm_policy.c \ + backend/gpu/mali_kbase_time.c + +ifeq ($(MALI_CUSTOMER_RELEASE),0) +BACKEND += \ + backend/gpu/mali_kbase_pm_ca_random.c \ + backend/gpu/mali_kbase_pm_demand_always_powered.c \ + backend/gpu/mali_kbase_pm_fast_start.c +endif + +ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y) +BACKEND += \ + backend/gpu/mali_kbase_devfreq.c \ + backend/gpu/mali_kbase_pm_ca_devfreq.c +endif + +ifeq ($(CONFIG_MALI_BIFROST_NO_MALI),y) + # Dummy model + BACKEND += backend/gpu/mali_kbase_model_dummy.c + BACKEND += backend/gpu/mali_kbase_model_linux.c + # HW error simulation + BACKEND += backend/gpu/mali_kbase_model_error_generator.c +endif diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_backend_config.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_backend_config.h new file mode 100755 index 000000000000..c8ae87eb84a2 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_backend_config.h @@ -0,0 +1,29 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Backend specific configuration + */ + +#ifndef _KBASE_BACKEND_CONFIG_H_ +#define _KBASE_BACKEND_CONFIG_H_ + +/* Enable GPU reset API */ +#define KBASE_GPU_RESET_EN 1 + +#endif /* _KBASE_BACKEND_CONFIG_H_ */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.c new file mode 100755 index 000000000000..fef9a2cb743e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.c @@ -0,0 +1,29 @@ +/* + * + * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "backend/gpu/mali_kbase_cache_policy_backend.h" +#include + +void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, + u32 mode) +{ + kbdev->current_gpu_coherency_mode = mode; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) + kbase_reg_write(kbdev, COHERENCY_ENABLE, mode, NULL); +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.h new file mode 100755 index 000000000000..fe9869109a82 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_cache_policy_backend.h @@ -0,0 +1,34 @@ +/* + * + * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +#ifndef _KBASE_CACHE_POLICY_BACKEND_H_ +#define _KBASE_CACHE_POLICY_BACKEND_H_ + +#include "mali_kbase.h" +#include "mali_base_kernel.h" + +/** + * kbase_cache_set_coherency_mode() - Sets the system coherency mode + * in the GPU. + * @kbdev: Device pointer + * @mode: Coherency mode. COHERENCY_ACE/ACE_LITE + */ +void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, + u32 mode); + +#endif /* _KBASE_CACHE_POLICY_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_debug_job_fault_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_debug_job_fault_backend.c new file mode 100755 index 000000000000..7851ea6466c7 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_debug_job_fault_backend.c @@ -0,0 +1,157 @@ +/* + * + * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include "mali_kbase_debug_job_fault.h" + +#ifdef CONFIG_DEBUG_FS + +/*GPU_CONTROL_REG(r)*/ +static int gpu_control_reg_snapshot[] = { + GPU_ID, + SHADER_READY_LO, + SHADER_READY_HI, + TILER_READY_LO, + TILER_READY_HI, + L2_READY_LO, + L2_READY_HI +}; + +/* JOB_CONTROL_REG(r) */ +static int job_control_reg_snapshot[] = { + JOB_IRQ_MASK, + JOB_IRQ_STATUS +}; + +/* JOB_SLOT_REG(n,r) */ +static int job_slot_reg_snapshot[] = { + JS_HEAD_LO, + JS_HEAD_HI, + JS_TAIL_LO, + JS_TAIL_HI, + JS_AFFINITY_LO, + JS_AFFINITY_HI, + JS_CONFIG, + JS_STATUS, + JS_HEAD_NEXT_LO, + JS_HEAD_NEXT_HI, + JS_AFFINITY_NEXT_LO, + JS_AFFINITY_NEXT_HI, + JS_CONFIG_NEXT +}; + +/*MMU_REG(r)*/ +static int mmu_reg_snapshot[] = { + MMU_IRQ_MASK, + MMU_IRQ_STATUS +}; + +/* MMU_AS_REG(n,r) */ +static int as_reg_snapshot[] = { + AS_TRANSTAB_LO, + AS_TRANSTAB_HI, + AS_MEMATTR_LO, + AS_MEMATTR_HI, + AS_FAULTSTATUS, + AS_FAULTADDRESS_LO, + AS_FAULTADDRESS_HI, + AS_STATUS +}; + +bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, + int reg_range) +{ + int i, j; + int offset = 0; + int slot_number; + int as_number; + + if (kctx->reg_dump == NULL) + return false; + + slot_number = kctx->kbdev->gpu_props.num_job_slots; + as_number = kctx->kbdev->gpu_props.num_address_spaces; + + /* get the GPU control registers*/ + for (i = 0; i < sizeof(gpu_control_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + GPU_CONTROL_REG(gpu_control_reg_snapshot[i]); + offset += 2; + } + + /* get the Job control registers*/ + for (i = 0; i < sizeof(job_control_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + JOB_CONTROL_REG(job_control_reg_snapshot[i]); + offset += 2; + } + + /* get the Job Slot registers*/ + for (j = 0; j < slot_number; j++) { + for (i = 0; i < sizeof(job_slot_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + JOB_SLOT_REG(j, job_slot_reg_snapshot[i]); + offset += 2; + } + } + + /* get the MMU registers*/ + for (i = 0; i < sizeof(mmu_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = MMU_REG(mmu_reg_snapshot[i]); + offset += 2; + } + + /* get the Address space registers*/ + for (j = 0; j < as_number; j++) { + for (i = 0; i < sizeof(as_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + MMU_AS_REG(j, as_reg_snapshot[i]); + offset += 2; + } + } + + WARN_ON(offset >= (reg_range*2/4)); + + /* set the termination flag*/ + kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG; + kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG; + + dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n", + offset); + + return true; +} + +bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx) +{ + int offset = 0; + + if (kctx->reg_dump == NULL) + return false; + + while (kctx->reg_dump[offset] != REGISTER_DUMP_TERMINATION_FLAG) { + kctx->reg_dump[offset+1] = + kbase_reg_read(kctx->kbdev, + kctx->reg_dump[offset], NULL); + offset += 2; + } + return true; +} + + +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.c new file mode 100755 index 000000000000..003af36d8fe2 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.c @@ -0,0 +1,495 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include + +#include +#include +#include +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif + +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) +#include +#else /* Linux >= 3.13 */ +/* In 3.13 the OPP include header file, types, and functions were all + * renamed. Use the old filename for the include, and define the new names to + * the old, when an old kernel is detected. + */ +#include +#define dev_pm_opp opp +#define dev_pm_opp_get_voltage opp_get_voltage +#define dev_pm_opp_get_opp_count opp_get_opp_count +#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil +#define dev_pm_opp_find_freq_floor opp_find_freq_floor +#endif /* Linux >= 3.13 */ + +#include +#include +#include + +static struct devfreq_simple_ondemand_data ondemand_data; + +static struct monitor_dev_profile mali_mdevp = { + .type = MONITOR_TPYE_DEV, + .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, + .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, +}; + +/** + * opp_translate - Translate nominal OPP frequency from devicetree into real + * frequency and core mask + * @kbdev: Device pointer + * @freq: Nominal frequency + * @core_mask: Pointer to u64 to store core mask to + * + * Return: Real target frequency + * + * This function will only perform translation if an operating-points-v2-mali + * table is present in devicetree. If one is not present then it will return an + * untranslated frequency and all cores enabled. + */ +static unsigned long opp_translate(struct kbase_device *kbdev, + unsigned long freq, u64 *core_mask) +{ + int i; + + for (i = 0; i < kbdev->num_opps; i++) { + if (kbdev->opp_table[i].opp_freq == freq) { + *core_mask = kbdev->opp_table[i].core_mask; + return kbdev->opp_table[i].real_freq; + } + } + + /* Failed to find OPP - return all cores enabled & nominal frequency */ + *core_mask = kbdev->gpu_props.props.raw_props.shader_present; + + return freq; +} + +static int +kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct dev_pm_opp *opp; + unsigned long nominal_freq; + unsigned long freq = 0; + unsigned long voltage; + int err; + u64 core_mask; + + freq = *target_freq; + + opp = devfreq_recommended_opp(dev, &freq, flags); + if (IS_ERR(opp)) { + dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); + return PTR_ERR(opp); + } + voltage = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + + nominal_freq = freq; + /* + * Only update if there is a change of frequency + */ + if (kbdev->current_nominal_freq == nominal_freq) { + *target_freq = nominal_freq; +#ifdef CONFIG_REGULATOR + if (kbdev->current_voltage == voltage) + return 0; + err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); + if (err) { + dev_err(dev, "Failed to set voltage (%d)\n", err); + return err; + } + kbdev->current_voltage = voltage; +#endif + return 0; + } + + freq = opp_translate(kbdev, nominal_freq, &core_mask); +#ifdef CONFIG_REGULATOR + if (kbdev->regulator && kbdev->current_voltage != voltage + && kbdev->current_freq < freq) { + err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); + if (err) { + dev_err(dev, "Failed to increase voltage (%d)\n", err); + return err; + } + } +#endif + + err = clk_set_rate(kbdev->clock, freq); + if (err) { + dev_err(dev, "Failed to set clock %lu (target %lu)\n", + freq, *target_freq); + return err; + } + +#ifdef CONFIG_REGULATOR + if (kbdev->regulator && kbdev->current_voltage != voltage + && kbdev->current_freq > freq) { + err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); + if (err) { + dev_err(dev, "Failed to decrease voltage (%d)\n", err); + return err; + } + } +#endif + + if (kbdev->pm.backend.ca_current_policy->id == + KBASE_PM_CA_POLICY_ID_DEVFREQ) + kbase_devfreq_set_core_mask(kbdev, core_mask); + + *target_freq = nominal_freq; + kbdev->current_voltage = voltage; + kbdev->current_nominal_freq = nominal_freq; + kbdev->current_freq = freq; + kbdev->current_core_mask = core_mask; + if (kbdev->devfreq) + kbdev->devfreq->last_status.current_frequency = nominal_freq; + + KBASE_TLSTREAM_AUX_DEVFREQ_TARGET((u64)nominal_freq); + + kbase_pm_reset_dvfs_utilisation(kbdev); + + return err; +} + +static int +kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + *freq = kbdev->current_nominal_freq; + + return 0; +} + +static int +kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + kbase_pm_get_dvfs_utilisation(kbdev, + &stat->total_time, &stat->busy_time); + + stat->private_data = NULL; + + return 0; +} + +static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, + struct devfreq_dev_profile *dp) +{ + int count; + int i = 0; + unsigned long freq; + struct dev_pm_opp *opp; + + count = dev_pm_opp_get_opp_count(kbdev->dev); + if (count < 0) { + return count; + } + + dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), + GFP_KERNEL); + if (!dp->freq_table) + return -ENOMEM; + + for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) { + opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq); + if (IS_ERR(opp)) + break; + dev_pm_opp_put(opp); + + dp->freq_table[i] = freq; + } + + if (count != i) + dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n", + count, i); + + dp->max_state = i; + + return 0; +} + +static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev) +{ + struct devfreq_dev_profile *dp = kbdev->devfreq->profile; + + kfree(dp->freq_table); +} + +static void kbase_devfreq_exit(struct device *dev) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + kbase_devfreq_term_freq_table(kbdev); +} + +static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) +{ + struct device_node *opp_node = of_parse_phandle(kbdev->dev->of_node, + "operating-points-v2", 0); + struct device_node *node; + int i = 0; + int count; + + if (!opp_node) + return 0; + if (!of_device_is_compatible(opp_node, "operating-points-v2-mali")) + return 0; + + count = dev_pm_opp_get_opp_count(kbdev->dev); + kbdev->opp_table = kmalloc_array(count, + sizeof(struct kbase_devfreq_opp), GFP_KERNEL); + if (!kbdev->opp_table) + return -ENOMEM; + + for_each_available_child_of_node(opp_node, node) { + u64 core_mask; + u64 opp_freq, real_freq; + const void *core_count_p; + + if (of_property_read_u64(node, "opp-hz", &opp_freq)) { + dev_warn(kbdev->dev, "OPP is missing required opp-hz property\n"); + continue; + } + if (of_property_read_u64(node, "opp-hz-real", &real_freq)) + real_freq = opp_freq; + if (of_property_read_u64(node, "opp-core-mask", &core_mask)) + core_mask = + kbdev->gpu_props.props.raw_props.shader_present; + core_count_p = of_get_property(node, "opp-core-count", NULL); + if (core_count_p) { + u64 remaining_core_mask = + kbdev->gpu_props.props.raw_props.shader_present; + int core_count = be32_to_cpup(core_count_p); + + core_mask = 0; + + for (; core_count > 0; core_count--) { + int core = ffs(remaining_core_mask); + + if (!core) { + dev_err(kbdev->dev, "OPP has more cores than GPU\n"); + return -ENODEV; + } + + core_mask |= (1ull << (core-1)); + remaining_core_mask &= ~(1ull << (core-1)); + } + } + + if (!core_mask) { + dev_err(kbdev->dev, "OPP has invalid core mask of 0\n"); + return -ENODEV; + } + + kbdev->opp_table[i].opp_freq = opp_freq; + kbdev->opp_table[i].real_freq = real_freq; + kbdev->opp_table[i].core_mask = core_mask; + + dev_info(kbdev->dev, "OPP %d : opp_freq=%llu real_freq=%llu core_mask=%llx\n", + i, opp_freq, real_freq, core_mask); + + i++; + } + + kbdev->num_opps = i; + + return 0; +} + +static unsigned long kbase_devfreq_get_static_power(struct devfreq *devfreq, + unsigned long voltage) +{ + struct device *dev = devfreq->dev.parent; + struct kbase_device *kbdev = dev_get_drvdata(dev); + + return rockchip_ipa_get_static_power(kbdev->model_data, voltage); +} + +static struct devfreq_cooling_power kbase_cooling_power = { + .get_static_power = &kbase_devfreq_get_static_power, +}; + +int kbase_devfreq_init(struct kbase_device *kbdev) +{ + struct device_node *np = kbdev->dev->of_node; + struct devfreq_dev_profile *dp; + struct dev_pm_opp *opp; + unsigned long opp_rate; + int err; + + if (!kbdev->clock) { + dev_err(kbdev->dev, "Clock not available for devfreq\n"); + return -ENODEV; + } + + kbdev->current_freq = clk_get_rate(kbdev->clock); + kbdev->current_nominal_freq = kbdev->current_freq; + + dp = &kbdev->devfreq_profile; + + dp->initial_freq = kbdev->current_freq; + dp->polling_ms = 100; + dp->target = kbase_devfreq_target; + dp->get_dev_status = kbase_devfreq_status; + dp->get_cur_freq = kbase_devfreq_cur_freq; + dp->exit = kbase_devfreq_exit; + + if (kbase_devfreq_init_freq_table(kbdev, dp)) + return -EFAULT; + + err = kbase_devfreq_init_core_mask_table(kbdev); + if (err) + return err; + of_property_read_u32(np, "upthreshold", + &ondemand_data.upthreshold); + of_property_read_u32(np, "downdifferential", + &ondemand_data.downdifferential); + kbdev->devfreq = devfreq_add_device(kbdev->dev, dp, + "simple_ondemand", &ondemand_data); + if (IS_ERR(kbdev->devfreq)) { + kbase_devfreq_term_freq_table(kbdev); + return PTR_ERR(kbdev->devfreq); + } + + /* devfreq_add_device only copies a few of kbdev->dev's fields, so + * set drvdata explicitly so IPA models can access kbdev. */ + dev_set_drvdata(&kbdev->devfreq->dev, kbdev); + + err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq); + if (err) { + dev_err(kbdev->dev, + "Failed to register OPP notifier (%d)\n", err); + goto opp_notifier_failed; + } + + opp_rate = kbdev->current_freq; + opp = devfreq_recommended_opp(kbdev->dev, &opp_rate, 0); + if (!IS_ERR(opp)) + dev_pm_opp_put(opp); + kbdev->devfreq->last_status.current_frequency = opp_rate; + + mali_mdevp.data = kbdev->devfreq; + kbdev->mdev_info = rockchip_system_monitor_register(kbdev->dev, + &mali_mdevp); + if (IS_ERR(kbdev->mdev_info)) { + dev_dbg(kbdev->dev, "without system monitor\n"); + kbdev->mdev_info = NULL; + } +#ifdef CONFIG_DEVFREQ_THERMAL + if (of_find_compatible_node(kbdev->dev->of_node, NULL, + "simple-power-model")) { + of_property_read_u32(kbdev->dev->of_node, + "dynamic-power-coefficient", + (u32 *)&kbase_dcp->dyn_power_coeff); + kbdev->model_data = rockchip_ipa_power_model_init(kbdev->dev, + "gpu_leakage"); + if (IS_ERR_OR_NULL(kbdev->model_data)) { + kbdev->model_data = NULL; + dev_err(kbdev->dev, "failed to initialize power model\n"); + } else if (kbdev->model_data->dynamic_coefficient) { + kbase_dcp->dyn_power_coeff = + kbdev->model_data->dynamic_coefficient; + } + if (!kbase_dcp->dyn_power_coeff) { + dev_err(kbdev->dev, "failed to get dynamic-coefficient\n"); + err = -EINVAL; + goto cooling_failed; + } + + kbdev->devfreq_cooling = + of_devfreq_cooling_register_power(kbdev->dev->of_node, + kbdev->devfreq, + kbase_dcp); + if (IS_ERR(kbdev->devfreq_cooling)) { + dev_err(kbdev->dev, "failed to register cooling device\n"); + err = PTR_ERR(kbdev->devfreq_cooling); + goto cooling_failed; + } + } else { + err = kbase_ipa_init(kbdev); + if (err) { + dev_err(kbdev->dev, "IPA initialization failed\n"); + goto cooling_failed; + } + + kbdev->devfreq_cooling = of_devfreq_cooling_register_power( + kbdev->dev->of_node, + kbdev->devfreq, + &kbase_ipa_power_model_ops); + if (IS_ERR(kbdev->devfreq_cooling)) { + err = PTR_ERR(kbdev->devfreq_cooling); + dev_err(kbdev->dev, + "Failed to register cooling device (%d)\n", + err); + goto cooling_failed; + } + } +#endif + + return 0; + +#ifdef CONFIG_DEVFREQ_THERMAL +cooling_failed: + devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); +#endif /* CONFIG_DEVFREQ_THERMAL */ +opp_notifier_failed: + if (devfreq_remove_device(kbdev->devfreq)) + dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); + else + kbdev->devfreq = NULL; + + return err; +} + +void kbase_devfreq_term(struct kbase_device *kbdev) +{ + int err; + + dev_dbg(kbdev->dev, "Term Mali devfreq\n"); + + rockchip_system_monitor_unregister(kbdev->mdev_info); +#ifdef CONFIG_DEVFREQ_THERMAL + if (kbdev->devfreq_cooling) + devfreq_cooling_unregister(kbdev->devfreq_cooling); + + if (!kbdev->model_data) + kbase_ipa_term(kbdev); + kfree(kbdev->model_data); +#endif + + devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); + + err = devfreq_remove_device(kbdev->devfreq); + if (err) + dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); + else + kbdev->devfreq = NULL; + + kfree(kbdev->opp_table); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.h new file mode 100755 index 000000000000..7bcc350f3006 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_devfreq.h @@ -0,0 +1,25 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _BASE_DEVFREQ_H_ +#define _BASE_DEVFREQ_H_ + +int kbase_devfreq_init(struct kbase_device *kbdev); +void kbase_devfreq_term(struct kbase_device *kbdev); +int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev); + +#endif /* _BASE_DEVFREQ_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_hw.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_hw.c new file mode 100755 index 000000000000..17f253308ffc --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_hw.c @@ -0,0 +1,255 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * + */ +#include +#include +#include + +#include + +#if !defined(CONFIG_MALI_BIFROST_NO_MALI) + + +#ifdef CONFIG_DEBUG_FS + + +int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size) +{ + struct kbase_io_access *old_buf; + struct kbase_io_access *new_buf; + unsigned long flags; + + if (!new_size) + goto out_err; /* The new size must not be 0 */ + + new_buf = vmalloc(new_size * sizeof(*h->buf)); + if (!new_buf) + goto out_err; + + spin_lock_irqsave(&h->lock, flags); + + old_buf = h->buf; + + /* Note: we won't bother with copying the old data over. The dumping + * logic wouldn't work properly as it relies on 'count' both as a + * counter and as an index to the buffer which would have changed with + * the new array. This is a corner case that we don't need to support. + */ + h->count = 0; + h->size = new_size; + h->buf = new_buf; + + spin_unlock_irqrestore(&h->lock, flags); + + vfree(old_buf); + + return 0; + +out_err: + return -1; +} + + +int kbase_io_history_init(struct kbase_io_history *h, u16 n) +{ + h->enabled = false; + spin_lock_init(&h->lock); + h->count = 0; + h->size = 0; + h->buf = NULL; + if (kbase_io_history_resize(h, n)) + return -1; + + return 0; +} + + +void kbase_io_history_term(struct kbase_io_history *h) +{ + vfree(h->buf); + h->buf = NULL; +} + + +/* kbase_io_history_add - add new entry to the register access history + * + * @h: Pointer to the history data structure + * @addr: Register address + * @value: The value that is either read from or written to the register + * @write: 1 if it's a register write, 0 if it's a read + */ +static void kbase_io_history_add(struct kbase_io_history *h, + void __iomem const *addr, u32 value, u8 write) +{ + struct kbase_io_access *io; + unsigned long flags; + + spin_lock_irqsave(&h->lock, flags); + + io = &h->buf[h->count % h->size]; + io->addr = (uintptr_t)addr | write; + io->value = value; + ++h->count; + /* If count overflows, move the index by the buffer size so the entire + * buffer will still be dumped later */ + if (unlikely(!h->count)) + h->count = h->size; + + spin_unlock_irqrestore(&h->lock, flags); +} + + +void kbase_io_history_dump(struct kbase_device *kbdev) +{ + struct kbase_io_history *const h = &kbdev->io_history; + u16 i; + size_t iters; + unsigned long flags; + + if (!unlikely(h->enabled)) + return; + + spin_lock_irqsave(&h->lock, flags); + + dev_err(kbdev->dev, "Register IO History:"); + iters = (h->size > h->count) ? h->count : h->size; + dev_err(kbdev->dev, "Last %zu register accesses of %zu total:\n", iters, + h->count); + for (i = 0; i < iters; ++i) { + struct kbase_io_access *io = + &h->buf[(h->count - iters + i) % h->size]; + char const access = (io->addr & 1) ? 'w' : 'r'; + + dev_err(kbdev->dev, "%6i: %c: reg 0x%p val %08x\n", i, access, + (void *)(io->addr & ~0x1), io->value); + } + + spin_unlock_irqrestore(&h->lock, flags); +} + + +#endif /* CONFIG_DEBUG_FS */ + + +void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, + struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbdev->dev != NULL); + + writel(value, kbdev->reg + offset); + +#ifdef CONFIG_DEBUG_FS + if (unlikely(kbdev->io_history.enabled)) + kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, + value, 1); +#endif /* CONFIG_DEBUG_FS */ + dev_dbg(kbdev->dev, "w: reg %04x val %08x", offset, value); + + if (kctx && kctx->jctx.tb) + kbase_device_trace_register_access(kctx, REG_WRITE, offset, + value); +} + +KBASE_EXPORT_TEST_API(kbase_reg_write); + +u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, + struct kbase_context *kctx) +{ + u32 val; + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbdev->dev != NULL); + + val = readl(kbdev->reg + offset); + +#ifdef CONFIG_DEBUG_FS + if (unlikely(kbdev->io_history.enabled)) + kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, + val, 0); +#endif /* CONFIG_DEBUG_FS */ + dev_dbg(kbdev->dev, "r: reg %04x val %08x", offset, val); + + if (kctx && kctx->jctx.tb) + kbase_device_trace_register_access(kctx, REG_READ, offset, val); + return val; +} + +KBASE_EXPORT_TEST_API(kbase_reg_read); +#endif /* !defined(CONFIG_MALI_BIFROST_NO_MALI) */ + +/** + * kbase_report_gpu_fault - Report a GPU fault. + * @kbdev: Kbase device pointer + * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS + * was also set + * + * This function is called from the interrupt handler when a GPU fault occurs. + * It reports the details of the fault using dev_warn(). + */ +static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) +{ + u32 status; + u64 address; + + status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL); + address = (u64) kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTADDRESS_HI), NULL) << 32; + address |= kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTADDRESS_LO), NULL); + + dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx", + status & 0xFF, + kbase_exception_name(kbdev, status), + address); + if (multiple) + dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n"); +} + +void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) +{ + KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, NULL, 0u, val); + if (val & GPU_FAULT) + kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS); + + if (val & RESET_COMPLETED) + kbase_pm_reset_done(kbdev); + + if (val & PRFCNT_SAMPLE_COMPLETED) + kbase_instr_hwcnt_sample_done(kbdev); + + if (val & CLEAN_CACHES_COMPLETED) + kbase_clean_caches_done(kbdev); + + KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, val); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, NULL); + + /* kbase_pm_check_transitions must be called after the IRQ has been + * cleared. This is because it might trigger further power transitions + * and we don't want to miss the interrupt raised to notify us that + * these further transitions have finished. + */ + if (val & POWER_CHANGED_ALL) + kbase_pm_power_changed(kbdev); + + KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, NULL, 0u, val); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_internal.h new file mode 100755 index 000000000000..5b20445932fb --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_device_internal.h @@ -0,0 +1,67 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Backend-specific HW access device APIs + */ + +#ifndef _KBASE_DEVICE_INTERNAL_H_ +#define _KBASE_DEVICE_INTERNAL_H_ + +/** + * kbase_reg_write - write to GPU register + * @kbdev: Kbase device pointer + * @offset: Offset of register + * @value: Value to write + * @kctx: Kbase context pointer. May be NULL + * + * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If + * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr + * != KBASEP_AS_NR_INVALID). + */ +void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, + struct kbase_context *kctx); + +/** + * kbase_reg_read - read from GPU register + * @kbdev: Kbase device pointer + * @offset: Offset of register + * @kctx: Kbase context pointer. May be NULL + * + * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If + * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr + * != KBASEP_AS_NR_INVALID). + * + * Return: Value in desired register + */ +u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, + struct kbase_context *kctx); + + +/** + * kbase_gpu_interrupt - GPU interrupt handler + * @kbdev: Kbase device pointer + * @val: The value of the GPU IRQ status register which triggered the call + * + * This function is called from the interrupt handler when a GPU irq is to be + * handled. + */ +void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); + +#endif /* _KBASE_DEVICE_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpu.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpu.c new file mode 100755 index 000000000000..a7c3a77d8ac8 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpu.c @@ -0,0 +1,123 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend APIs + */ +#include +#include +#include +#include +#include +#include + +int kbase_backend_early_init(struct kbase_device *kbdev) +{ + int err; + + err = kbasep_platform_device_init(kbdev); + if (err) + return err; + + /* Ensure we can access the GPU registers */ + kbase_pm_register_access_enable(kbdev); + + /* Find out GPU properties based on the GPU feature registers */ + kbase_gpuprops_set(kbdev); + + /* We're done accessing the GPU registers for now. */ + kbase_pm_register_access_disable(kbdev); + + err = kbase_install_interrupts(kbdev); + if (err) + goto fail_interrupts; + + err = kbase_hwaccess_pm_init(kbdev); + if (err) + goto fail_pm; + + return 0; + +fail_pm: + kbase_release_interrupts(kbdev); +fail_interrupts: + kbasep_platform_device_term(kbdev); + + return err; +} + +void kbase_backend_early_term(struct kbase_device *kbdev) +{ + kbase_hwaccess_pm_term(kbdev); + kbase_release_interrupts(kbdev); + kbasep_platform_device_term(kbdev); +} + +int kbase_backend_late_init(struct kbase_device *kbdev) +{ + int err; + + err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); + if (err) + return err; + + err = kbase_backend_timer_init(kbdev); + if (err) + goto fail_timer; + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#ifndef CONFIG_MALI_BIFROST_NO_MALI + if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { + dev_err(kbdev->dev, "Interrupt assigment check failed.\n"); + err = -EINVAL; + goto fail_interrupt_test; + } +#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + err = kbase_job_slot_init(kbdev); + if (err) + goto fail_job_slot; + + init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); + + return 0; + +fail_job_slot: + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#ifndef CONFIG_MALI_BIFROST_NO_MALI +fail_interrupt_test: +#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + kbase_backend_timer_term(kbdev); +fail_timer: + kbase_hwaccess_pm_halt(kbdev); + + return err; +} + +void kbase_backend_late_term(struct kbase_device *kbdev) +{ + kbase_job_slot_halt(kbdev); + kbase_job_slot_term(kbdev); + kbase_backend_timer_term(kbdev); + kbase_hwaccess_pm_halt(kbdev); +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpuprops_backend.c new file mode 100755 index 000000000000..b395325b556b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_gpuprops_backend.c @@ -0,0 +1,110 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel property query backend APIs + */ + +#include +#include +#include +#include + +void kbase_backend_gpuprops_get(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump) +{ + int i; + + /* Fill regdump with the content of the relevant registers */ + regdump->gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID), NULL); + + regdump->l2_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_FEATURES), NULL); + regdump->suspend_size = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SUSPEND_SIZE), NULL); + regdump->tiler_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_FEATURES), NULL); + regdump->mem_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(MEM_FEATURES), NULL); + regdump->mmu_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(MMU_FEATURES), NULL); + regdump->as_present = kbase_reg_read(kbdev, + GPU_CONTROL_REG(AS_PRESENT), NULL); + regdump->js_present = kbase_reg_read(kbdev, + GPU_CONTROL_REG(JS_PRESENT), NULL); + + for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) + regdump->js_features[i] = kbase_reg_read(kbdev, + GPU_CONTROL_REG(JS_FEATURES_REG(i)), NULL); + + for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) + regdump->texture_features[i] = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)), NULL); + + regdump->thread_max_threads = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_MAX_THREADS), NULL); + regdump->thread_max_workgroup_size = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE), + NULL); + regdump->thread_max_barrier_size = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE), NULL); + regdump->thread_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_FEATURES), NULL); + + regdump->shader_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_PRESENT_LO), NULL); + regdump->shader_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_PRESENT_HI), NULL); + + regdump->tiler_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_PRESENT_LO), NULL); + regdump->tiler_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_PRESENT_HI), NULL); + + regdump->l2_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_PRESENT_LO), NULL); + regdump->l2_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_PRESENT_HI), NULL); + + regdump->stack_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(STACK_PRESENT_LO), NULL); + regdump->stack_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(STACK_PRESENT_HI), NULL); +} + +void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump) +{ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) { + /* Ensure we can access the GPU registers */ + kbase_pm_register_access_enable(kbdev); + + regdump->coherency_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(COHERENCY_FEATURES), NULL); + + /* We're done accessing the GPU registers for now. */ + kbase_pm_register_access_disable(kbdev); + } else { + /* Pre COHERENCY_FEATURES we only supported ACE_LITE */ + regdump->coherency_features = + COHERENCY_FEATURE_BIT(COHERENCY_NONE) | + COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); + } +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_backend.c new file mode 100755 index 000000000000..8084d054cc5b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_backend.c @@ -0,0 +1,492 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * GPU backend instrumentation APIs. + */ + +#include +#include +#include +#include +#include +#include + +/** + * kbasep_instr_hwcnt_cacheclean - Issue Cache Clean & Invalidate command to + * hardware + * + * @kbdev: Kbase device + */ +static void kbasep_instr_hwcnt_cacheclean(struct kbase_device *kbdev) +{ + unsigned long flags; + unsigned long pm_flags; + u32 irq_mask; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_REQUEST_CLEAN); + + /* Enable interrupt */ + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + irq_mask | CLEAN_CACHES_COMPLETED, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + + /* clean&invalidate the caches so we're sure the mmu tables for the dump + * buffer is valid */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAN_INV_CACHES, NULL); + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANING; + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); +} + +int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_uk_hwcnt_setup *setup) +{ + unsigned long flags, pm_flags; + int err = -EINVAL; + u32 irq_mask; + int ret; + u64 shader_cores_needed; + u32 prfcnt_config; + + shader_cores_needed = kbase_pm_get_present_cores(kbdev, + KBASE_PM_CORE_SHADER); + + /* alignment failure */ + if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1))) + goto out_err; + + /* Override core availability policy to ensure all cores are available + */ + kbase_pm_ca_instr_enable(kbdev); + + /* Request the cores early on synchronously - we'll release them on any + * errors (e.g. instrumentation already active) */ + kbase_pm_request_cores_sync(kbdev, true, shader_cores_needed); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { + /* Instrumentation is already enabled */ + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + goto out_unrequest_cores; + } + + /* Enable interrupt */ + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | + PRFCNT_SAMPLE_COMPLETED, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + + /* In use, this context is the owner */ + kbdev->hwcnt.kctx = kctx; + /* Remember the dump address so we can reprogram it later */ + kbdev->hwcnt.addr = setup->dump_buffer; + + /* Request the clean */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; + kbdev->hwcnt.backend.triggered = 0; + /* Clean&invalidate the caches so we're sure the mmu tables for the dump + * buffer is valid */ + ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, + &kbdev->hwcnt.backend.cache_clean_work); + KBASE_DEBUG_ASSERT(ret); + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + /* Wait for cacheclean to complete */ + wait_event(kbdev->hwcnt.backend.wait, + kbdev->hwcnt.backend.triggered != 0); + + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_IDLE); + + kbase_pm_request_l2_caches(kbdev); + + /* Configure */ + prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; +#ifdef CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY + { + u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + u32 product_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) + >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; + int arch_v6 = GPU_ID_IS_NEW_FORMAT(product_id); + + if (arch_v6) + prfcnt_config |= 1 << PRFCNT_CONFIG_SETSELECT_SHIFT; + } +#endif + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), + prfcnt_config | PRFCNT_CONFIG_MODE_OFF, kctx); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), + setup->dump_buffer & 0xFFFFFFFF, kctx); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), + setup->dump_buffer >> 32, kctx); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), + setup->jm_bm, kctx); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), + setup->shader_bm, kctx); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), + setup->mmu_l2_bm, kctx); + /* Due to PRLAM-8186 we need to disable the Tiler before we enable the + * HW counter dump. */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, + kctx); + else + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), + setup->tiler_bm, kctx); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), + prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL, kctx); + + /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump + */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), + setup->tiler_bm, kctx); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + err = 0; + + dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx); + return err; + out_unrequest_cores: + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_unrequest_cores(kbdev, true, shader_cores_needed); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + out_err: + return err; +} + +int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) +{ + unsigned long flags, pm_flags; + int err = -EINVAL; + u32 irq_mask; + struct kbase_device *kbdev = kctx->kbdev; + + while (1) { + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) { + /* Instrumentation is not enabled */ + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + goto out; + } + + if (kbdev->hwcnt.kctx != kctx) { + /* Instrumentation has been setup for another context */ + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + goto out; + } + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) + break; + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + /* Ongoing dump/setup - wait for its completion */ + wait_event(kbdev->hwcnt.backend.wait, + kbdev->hwcnt.backend.triggered != 0); + } + + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; + kbdev->hwcnt.backend.triggered = 0; + + /* Disable interrupt */ + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL); + + /* Disable the counters */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx); + + kbdev->hwcnt.kctx = NULL; + kbdev->hwcnt.addr = 0ULL; + + kbase_pm_ca_instr_disable(kbdev); + + kbase_pm_unrequest_cores(kbdev, true, + kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); + + kbase_pm_release_l2_caches(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", + kctx); + + err = 0; + + out: + return err; +} + +int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) +{ + unsigned long flags; + int err = -EINVAL; + struct kbase_device *kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.kctx != kctx) { + /* The instrumentation has been setup for another context */ + goto unlock; + } + + if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { + /* HW counters are disabled or another dump is ongoing, or we're + * resetting */ + goto unlock; + } + + kbdev->hwcnt.backend.triggered = 0; + + /* Mark that we're dumping - the PF handler can signal that we faulted + */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; + + /* Reconfigure the dump address */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), + kbdev->hwcnt.addr & 0xFFFFFFFF, NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), + kbdev->hwcnt.addr >> 32, NULL); + + /* Start dumping */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL, + kbdev->hwcnt.addr, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_PRFCNT_SAMPLE, kctx); + + dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx); + + err = 0; + + unlock: + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + return err; +} +KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); + +bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, + bool * const success) +{ + unsigned long flags; + bool complete = false; + struct kbase_device *kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) { + *success = true; + complete = true; + } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { + *success = false; + complete = true; + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + return complete; +} +KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); + +void kbasep_cache_clean_worker(struct work_struct *data) +{ + struct kbase_device *kbdev; + unsigned long flags; + + kbdev = container_of(data, struct kbase_device, + hwcnt.backend.cache_clean_work); + + mutex_lock(&kbdev->cacheclean_lock); + kbasep_instr_hwcnt_cacheclean(kbdev); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + /* Wait for our condition, and any reset to complete */ + while (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) { + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + wait_event(kbdev->hwcnt.backend.cache_clean_wait, + kbdev->hwcnt.backend.state != + KBASE_INSTR_STATE_CLEANING); + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + } + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_CLEANED); + + /* All finished and idle */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + mutex_unlock(&kbdev->cacheclean_lock); +} + +void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) { + int ret; + /* Always clean and invalidate the cache after a successful dump + */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; + ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, + &kbdev->hwcnt.backend.cache_clean_work); + KBASE_DEBUG_ASSERT(ret); + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); +} + +void kbase_clean_caches_done(struct kbase_device *kbdev) +{ + u32 irq_mask; + + if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { + unsigned long flags; + unsigned long pm_flags; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + /* Disable interrupt */ + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + irq_mask & ~CLEAN_CACHES_COMPLETED, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + + /* Wakeup... */ + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) { + /* Only wake if we weren't resetting */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANED; + wake_up(&kbdev->hwcnt.backend.cache_clean_wait); + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + } +} + +int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + unsigned long flags; + int err; + + /* Wait for dump & cacheclean to complete */ + wait_event(kbdev->hwcnt.backend.wait, + kbdev->hwcnt.backend.triggered != 0); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { + err = -EINVAL; + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + } else { + /* Dump done */ + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_IDLE); + err = 0; + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + return err; +} + +int kbase_instr_hwcnt_clear(struct kbase_context *kctx) +{ + unsigned long flags; + int err = -EINVAL; + struct kbase_device *kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + /* Check it's the context previously set up and we're not already + * dumping */ + if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != + KBASE_INSTR_STATE_IDLE) + goto out; + + /* Clear the counters */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_PRFCNT_CLEAR, kctx); + + err = 0; + +out: + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + return err; +} +KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear); + +int kbase_instr_backend_init(struct kbase_device *kbdev) +{ + int ret = 0; + + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; + + init_waitqueue_head(&kbdev->hwcnt.backend.wait); + init_waitqueue_head(&kbdev->hwcnt.backend.cache_clean_wait); + INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work, + kbasep_cache_clean_worker); + kbdev->hwcnt.backend.triggered = 0; + + kbdev->hwcnt.backend.cache_clean_wq = + alloc_workqueue("Mali cache cleaning workqueue", 0, 1); + if (NULL == kbdev->hwcnt.backend.cache_clean_wq) + ret = -EINVAL; + + return ret; +} + +void kbase_instr_backend_term(struct kbase_device *kbdev) +{ + destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq); +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_defs.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_defs.h new file mode 100755 index 000000000000..4794672da8f0 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_defs.h @@ -0,0 +1,58 @@ +/* + * + * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Backend-specific instrumentation definitions + */ + +#ifndef _KBASE_INSTR_DEFS_H_ +#define _KBASE_INSTR_DEFS_H_ + +/* + * Instrumentation State Machine States + */ +enum kbase_instr_state { + /* State where instrumentation is not active */ + KBASE_INSTR_STATE_DISABLED = 0, + /* State machine is active and ready for a command. */ + KBASE_INSTR_STATE_IDLE, + /* Hardware is currently dumping a frame. */ + KBASE_INSTR_STATE_DUMPING, + /* We've requested a clean to occur on a workqueue */ + KBASE_INSTR_STATE_REQUEST_CLEAN, + /* Hardware is currently cleaning and invalidating caches. */ + KBASE_INSTR_STATE_CLEANING, + /* Cache clean completed, and either a) a dump is complete, or + * b) instrumentation can now be setup. */ + KBASE_INSTR_STATE_CLEANED, + /* An error has occured during DUMPING (page fault). */ + KBASE_INSTR_STATE_FAULT +}; + +/* Structure used for instrumentation and HW counters dumping */ +struct kbase_instr_backend { + wait_queue_head_t wait; + int triggered; + + enum kbase_instr_state state; + wait_queue_head_t cache_clean_wait; + struct workqueue_struct *cache_clean_wq; + struct work_struct cache_clean_work; +}; + +#endif /* _KBASE_INSTR_DEFS_H_ */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_internal.h new file mode 100755 index 000000000000..e96aeae786e1 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_instr_internal.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Backend-specific HW access instrumentation APIs + */ + +#ifndef _KBASE_INSTR_INTERNAL_H_ +#define _KBASE_INSTR_INTERNAL_H_ + +/** + * kbasep_cache_clean_worker() - Workqueue for handling cache cleaning + * @data: a &struct work_struct + */ +void kbasep_cache_clean_worker(struct work_struct *data); + +/** + * kbase_clean_caches_done() - Cache clean interrupt received + * @kbdev: Kbase device + */ +void kbase_clean_caches_done(struct kbase_device *kbdev); + +/** + * kbase_instr_hwcnt_sample_done() - Dump complete interrupt received + * @kbdev: Kbase device + */ +void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev); + +#endif /* _KBASE_INSTR_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_internal.h new file mode 100755 index 000000000000..8781561e73d0 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_internal.h @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Backend specific IRQ APIs + */ + +#ifndef _KBASE_IRQ_INTERNAL_H_ +#define _KBASE_IRQ_INTERNAL_H_ + +int kbase_install_interrupts(struct kbase_device *kbdev); + +void kbase_release_interrupts(struct kbase_device *kbdev); + +/** + * kbase_synchronize_irqs - Ensure that all IRQ handlers have completed + * execution + * @kbdev: The kbase device + */ +void kbase_synchronize_irqs(struct kbase_device *kbdev); + +int kbasep_common_test_interrupt_handlers( + struct kbase_device * const kbdev); + +#endif /* _KBASE_IRQ_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_linux.c new file mode 100755 index 000000000000..d0666c86cf59 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_irq_linux.c @@ -0,0 +1,469 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include + +#include + +#if !defined(CONFIG_MALI_BIFROST_NO_MALI) + +/* GPU IRQ Tags */ +#define JOB_IRQ_TAG 0 +#define MMU_IRQ_TAG 1 +#define GPU_IRQ_TAG 2 + +static void *kbase_tag(void *ptr, u32 tag) +{ + return (void *)(((uintptr_t) ptr) | tag); +} + +static void *kbase_untag(void *ptr) +{ + return (void *)(((uintptr_t) ptr) & ~3); +} + +static irqreturn_t kbase_job_irq_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (!kbdev->pm.backend.driver_ready_for_irqs) + dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", + __func__, irq, val); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbase_job_done(kbdev, val); + + return IRQ_HANDLED; +} + +KBASE_EXPORT_TEST_API(kbase_job_irq_handler); + +static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return IRQ_NONE; + } + + atomic_inc(&kbdev->faults_pending); + + val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (!kbdev->pm.backend.driver_ready_for_irqs) + dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", + __func__, irq, val); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!val) { + atomic_dec(&kbdev->faults_pending); + return IRQ_NONE; + } + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbase_mmu_interrupt(kbdev, val); + + atomic_dec(&kbdev->faults_pending); + + return IRQ_HANDLED; +} + +static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (!kbdev->pm.backend.driver_ready_for_irqs) + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", + __func__, irq, val); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbase_gpu_interrupt(kbdev, val); + + return IRQ_HANDLED; +} + +KBASE_EXPORT_TEST_API(kbase_gpu_irq_handler); + +static irq_handler_t kbase_handler_table[] = { + [JOB_IRQ_TAG] = kbase_job_irq_handler, + [MMU_IRQ_TAG] = kbase_mmu_irq_handler, + [GPU_IRQ_TAG] = kbase_gpu_irq_handler, +}; + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define JOB_IRQ_HANDLER JOB_IRQ_TAG +#define MMU_IRQ_HANDLER MMU_IRQ_TAG +#define GPU_IRQ_HANDLER GPU_IRQ_TAG + +/** + * kbase_set_custom_irq_handler - Set a custom IRQ handler + * @kbdev: Device for which the handler is to be registered + * @custom_handler: Handler to be registered + * @irq_type: Interrupt type + * + * Registers given interrupt handler for requested interrupt type + * In the case where irq handler is not specified, the default handler shall be + * registered + * + * Return: 0 case success, error code otherwise + */ +int kbase_set_custom_irq_handler(struct kbase_device *kbdev, + irq_handler_t custom_handler, + int irq_type) +{ + int result = 0; + irq_handler_t requested_irq_handler = NULL; + + KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) && + (GPU_IRQ_HANDLER >= irq_type)); + + /* Release previous handler */ + if (kbdev->irqs[irq_type].irq) + free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type)); + + requested_irq_handler = (NULL != custom_handler) ? custom_handler : + kbase_handler_table[irq_type]; + + if (0 != request_irq(kbdev->irqs[irq_type].irq, + requested_irq_handler, + kbdev->irqs[irq_type].flags | IRQF_SHARED, + dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) { + result = -EINVAL; + dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", + kbdev->irqs[irq_type].irq, irq_type); +#ifdef CONFIG_SPARSE_IRQ + dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); +#endif /* CONFIG_SPARSE_IRQ */ + } + + return result; +} + +KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler); + +/* test correct interrupt assigment and reception by cpu */ +struct kbasep_irq_test { + struct hrtimer timer; + wait_queue_head_t wait; + int triggered; + u32 timeout; +}; + +static struct kbasep_irq_test kbasep_irq_test_data; + +#define IRQ_TEST_TIMEOUT 500 + +static irqreturn_t kbase_job_irq_test_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL); + + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbasep_irq_test_data.triggered = 1; + wake_up(&kbasep_irq_test_data.wait); + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val, NULL); + + return IRQ_HANDLED; +} + +static irqreturn_t kbase_mmu_irq_test_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL); + + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbasep_irq_test_data.triggered = 1; + wake_up(&kbasep_irq_test_data.wait); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), val, NULL); + + return IRQ_HANDLED; +} + +static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) +{ + struct kbasep_irq_test *test_data = container_of(timer, + struct kbasep_irq_test, timer); + + test_data->timeout = 1; + test_data->triggered = 1; + wake_up(&test_data->wait); + return HRTIMER_NORESTART; +} + +static int kbasep_common_test_interrupt( + struct kbase_device * const kbdev, u32 tag) +{ + int err = 0; + irq_handler_t test_handler; + + u32 old_mask_val; + u16 mask_offset; + u16 rawstat_offset; + + switch (tag) { + case JOB_IRQ_TAG: + test_handler = kbase_job_irq_test_handler; + rawstat_offset = JOB_CONTROL_REG(JOB_IRQ_RAWSTAT); + mask_offset = JOB_CONTROL_REG(JOB_IRQ_MASK); + break; + case MMU_IRQ_TAG: + test_handler = kbase_mmu_irq_test_handler; + rawstat_offset = MMU_REG(MMU_IRQ_RAWSTAT); + mask_offset = MMU_REG(MMU_IRQ_MASK); + break; + case GPU_IRQ_TAG: + /* already tested by pm_driver - bail out */ + default: + return 0; + } + + /* store old mask */ + old_mask_val = kbase_reg_read(kbdev, mask_offset, NULL); + /* mask interrupts */ + kbase_reg_write(kbdev, mask_offset, 0x0, NULL); + + if (kbdev->irqs[tag].irq) { + /* release original handler and install test handler */ + if (kbase_set_custom_irq_handler(kbdev, test_handler, tag) != 0) { + err = -EINVAL; + } else { + kbasep_irq_test_data.timeout = 0; + hrtimer_init(&kbasep_irq_test_data.timer, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + kbasep_irq_test_data.timer.function = + kbasep_test_interrupt_timeout; + + /* trigger interrupt */ + kbase_reg_write(kbdev, mask_offset, 0x1, NULL); + kbase_reg_write(kbdev, rawstat_offset, 0x1, NULL); + + hrtimer_start(&kbasep_irq_test_data.timer, + HR_TIMER_DELAY_MSEC(IRQ_TEST_TIMEOUT), + HRTIMER_MODE_REL); + + wait_event(kbasep_irq_test_data.wait, + kbasep_irq_test_data.triggered != 0); + + if (kbasep_irq_test_data.timeout != 0) { + dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n", + kbdev->irqs[tag].irq, tag); + err = -EINVAL; + } else { + dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n", + kbdev->irqs[tag].irq, tag); + } + + hrtimer_cancel(&kbasep_irq_test_data.timer); + kbasep_irq_test_data.triggered = 0; + + /* mask interrupts */ + kbase_reg_write(kbdev, mask_offset, 0x0, NULL); + + /* release test handler */ + free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag)); + } + + /* restore original interrupt */ + if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag], + kbdev->irqs[tag].flags | IRQF_SHARED, + dev_name(kbdev->dev), kbase_tag(kbdev, tag))) { + dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n", + kbdev->irqs[tag].irq, tag); + err = -EINVAL; + } + } + /* restore old mask */ + kbase_reg_write(kbdev, mask_offset, old_mask_val, NULL); + + return err; +} + +int kbasep_common_test_interrupt_handlers( + struct kbase_device * const kbdev) +{ + int err; + + init_waitqueue_head(&kbasep_irq_test_data.wait); + kbasep_irq_test_data.triggered = 0; + + /* A suspend won't happen during startup/insmod */ + kbase_pm_context_active(kbdev); + + err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG); + if (err) { + dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n"); + goto out; + } + + err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG); + if (err) { + dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n"); + goto out; + } + + dev_dbg(kbdev->dev, "Interrupts are correctly assigned.\n"); + + out: + kbase_pm_context_idle(kbdev); + + return err; +} +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + +int kbase_install_interrupts(struct kbase_device *kbdev) +{ + u32 nr = ARRAY_SIZE(kbase_handler_table); + int err; + u32 i; + + for (i = 0; i < nr; i++) { + err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i], + kbdev->irqs[i].flags | IRQF_SHARED, + dev_name(kbdev->dev), + kbase_tag(kbdev, i)); + if (err) { + dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", + kbdev->irqs[i].irq, i); +#ifdef CONFIG_SPARSE_IRQ + dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); +#endif /* CONFIG_SPARSE_IRQ */ + goto release; + } + } + + return 0; + + release: + while (i-- > 0) + free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); + + return err; +} + +void kbase_release_interrupts(struct kbase_device *kbdev) +{ + u32 nr = ARRAY_SIZE(kbase_handler_table); + u32 i; + + for (i = 0; i < nr; i++) { + if (kbdev->irqs[i].irq) + free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); + } +} + +void kbase_synchronize_irqs(struct kbase_device *kbdev) +{ + u32 nr = ARRAY_SIZE(kbase_handler_table); + u32 i; + + for (i = 0; i < nr; i++) { + if (kbdev->irqs[i].irq) + synchronize_irq(kbdev->irqs[i].irq); + } +} + +#endif /* !defined(CONFIG_MALI_BIFROST_NO_MALI) */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_as.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_as.c new file mode 100755 index 000000000000..c660c80341f4 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_as.c @@ -0,0 +1,235 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register backend context / address space management + */ + +#include +#include +#include + +/** + * assign_and_activate_kctx_addr_space - Assign an AS to a context + * @kbdev: Kbase device + * @kctx: Kbase context + * @current_as: Address Space to assign + * + * Assign an Address Space (AS) to a context, and add the context to the Policy. + * + * This includes + * setting up the global runpool_irq structure and the context on the AS, + * Activating the MMU on the AS, + * Allowing jobs to be submitted on the AS. + * + * Context: + * kbasep_js_kctx_info.jsctx_mutex held, + * kbasep_js_device_data.runpool_mutex held, + * AS transaction mutex held, + * Runpool IRQ lock held + */ +static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_as *current_as) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* Attribute handling */ + kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); + + /* Allow it to run jobs */ + kbasep_js_set_submit_allowed(js_devdata, kctx); + + kbase_js_runpool_inc_context_count(kbdev, kctx); +} + +bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + int i; + + if (kbdev->hwaccess.active_kctx == kctx) { + /* Context is already active */ + return true; + } + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + if (kbdev->as_to_kctx[i] == kctx) { + /* Context already has ASID - mark as active */ + return true; + } + } + + /* Context does not have address space assigned */ + return false; +} + +void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + int as_nr = kctx->as_nr; + + if (as_nr == KBASEP_AS_NR_INVALID) { + WARN(1, "Attempting to release context without ASID\n"); + return; + } + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (atomic_read(&kctx->refcount) != 1) { + WARN(1, "Attempting to release active ASID\n"); + return; + } + + kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx); + + kbase_ctx_sched_release_ctx(kctx); + kbase_js_runpool_dec_context_count(kbdev, kctx); +} + +void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ +} + +int kbase_backend_find_and_release_free_address_space( + struct kbase_device *kbdev, struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + unsigned long flags; + int i; + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + struct kbasep_js_kctx_info *as_js_kctx_info; + struct kbase_context *as_kctx; + + as_kctx = kbdev->as_to_kctx[i]; + as_js_kctx_info = &as_kctx->jctx.sched_info; + + /* Don't release privileged or active contexts, or contexts with + * jobs running. + * Note that a context will have at least 1 reference (which + * was previously taken by kbasep_js_schedule_ctx()) until + * descheduled. + */ + if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && + atomic_read(&as_kctx->refcount) == 1) { + if (!kbasep_js_runpool_retain_ctx_nolock(kbdev, + as_kctx)) { + WARN(1, "Failed to retain active context\n"); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return KBASEP_AS_NR_INVALID; + } + + kbasep_js_clear_submit_allowed(js_devdata, as_kctx); + + /* Drop and retake locks to take the jsctx_mutex on the + * context we're about to release without violating lock + * ordering + */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + + /* Release context from address space */ + mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx); + + if (!kbase_ctx_flag(as_kctx, KCTX_SCHEDULED)) { + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, + as_kctx, + true); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); + + return i; + } + + /* Context was retained while locks were dropped, + * continue looking for free AS */ + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + } + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return KBASEP_AS_NR_INVALID; +} + +bool kbase_backend_use_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, + int as_nr) +{ + struct kbasep_js_device_data *js_devdata; + struct kbase_as *new_address_space = NULL; + + js_devdata = &kbdev->js_data; + + if (kbdev->hwaccess.active_kctx == kctx) { + WARN(1, "Context is already scheduled in\n"); + return false; + } + + new_address_space = &kbdev->as[as_nr]; + + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space); + + if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { + /* We need to retain it to keep the corresponding address space + */ + kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + } + + return true; +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_defs.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_defs.h new file mode 100755 index 000000000000..08a7400e66d5 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_defs.h @@ -0,0 +1,123 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend specific definitions + */ + +#ifndef _KBASE_HWACCESS_GPU_DEFS_H_ +#define _KBASE_HWACCESS_GPU_DEFS_H_ + +/* SLOT_RB_SIZE must be < 256 */ +#define SLOT_RB_SIZE 2 +#define SLOT_RB_MASK (SLOT_RB_SIZE - 1) + +/** + * struct rb_entry - Ringbuffer entry + * @katom: Atom associated with this entry + */ +struct rb_entry { + struct kbase_jd_atom *katom; +}; + +/** + * struct slot_rb - Slot ringbuffer + * @entries: Ringbuffer entries + * @last_context: The last context to submit a job on this slot + * @read_idx: Current read index of buffer + * @write_idx: Current write index of buffer + * @job_chain_flag: Flag used to implement jobchain disambiguation + */ +struct slot_rb { + struct rb_entry entries[SLOT_RB_SIZE]; + + struct kbase_context *last_context; + + u8 read_idx; + u8 write_idx; + + u8 job_chain_flag; +}; + +/** + * struct kbase_backend_data - GPU backend specific data for HW access layer + * @slot_rb: Slot ringbuffers + * @rmu_workaround_flag: When PRLAM-8987 is present, this flag determines + * whether slots 0/1 or slot 2 are currently being + * pulled from + * @scheduling_timer: The timer tick used for rescheduling jobs + * @timer_running: Is the timer running? The runpool_mutex must be + * held whilst modifying this. + * @suspend_timer: Is the timer suspended? Set when a suspend + * occurs and cleared on resume. The runpool_mutex + * must be held whilst modifying this. + * @reset_gpu: Set to a KBASE_RESET_xxx value (see comments) + * @reset_workq: Work queue for performing the reset + * @reset_work: Work item for performing the reset + * @reset_wait: Wait event signalled when the reset is complete + * @reset_timer: Timeout for soft-stops before the reset + * @timeouts_updated: Have timeout values just been updated? + * + * The hwaccess_lock (a spinlock) must be held when accessing this structure + */ +struct kbase_backend_data { + struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS]; + + bool rmu_workaround_flag; + + struct hrtimer scheduling_timer; + + bool timer_running; + bool suspend_timer; + + atomic_t reset_gpu; + +/* The GPU reset isn't pending */ +#define KBASE_RESET_GPU_NOT_PENDING 0 +/* kbase_prepare_to_reset_gpu has been called */ +#define KBASE_RESET_GPU_PREPARED 1 +/* kbase_reset_gpu has been called - the reset will now definitely happen + * within the timeout period */ +#define KBASE_RESET_GPU_COMMITTED 2 +/* The GPU reset process is currently occuring (timeout has expired or + * kbasep_try_reset_gpu_early was called) */ +#define KBASE_RESET_GPU_HAPPENING 3 +/* Reset the GPU silently, used when resetting the GPU as part of normal + * behavior (e.g. when exiting protected mode). */ +#define KBASE_RESET_GPU_SILENT 4 + struct workqueue_struct *reset_workq; + struct work_struct reset_work; + wait_queue_head_t reset_wait; + struct hrtimer reset_timer; + + bool timeouts_updated; +}; + +/** + * struct kbase_jd_atom_backend - GPU backend specific katom data + */ +struct kbase_jd_atom_backend { +}; + +/** + * struct kbase_context_backend - GPU backend specific context data + */ +struct kbase_context_backend { +}; + +#endif /* _KBASE_HWACCESS_GPU_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_hw.c new file mode 100755 index 000000000000..cbca5eac82f1 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_hw.c @@ -0,0 +1,1512 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Base kernel job manager APIs + */ + +#include +#include +#include +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define beenthere(kctx, f, a...) \ + dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) + +#if KBASE_GPU_RESET_EN +static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev); +static void kbasep_reset_timeout_worker(struct work_struct *data); +static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer); +#endif /* KBASE_GPU_RESET_EN */ + +static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, + struct kbase_context *kctx) +{ + return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), kctx); +} + +void kbase_job_hw_submit(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + int js) +{ + struct kbase_context *kctx; + u32 cfg; + u64 jc_head = katom->jc; + + KBASE_DEBUG_ASSERT(kbdev); + KBASE_DEBUG_ASSERT(katom); + + kctx = katom->kctx; + + /* Command register must be available */ + KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx)); + /* Affinity is not violating */ + kbase_js_debug_log_current_affinities(kbdev); + KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js, + katom->affinity)); + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), + jc_head & 0xFFFFFFFF, kctx); + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), + jc_head >> 32, kctx); + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO), + katom->affinity & 0xFFFFFFFF, kctx); + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI), + katom->affinity >> 32, kctx); + + /* start MMU, medium priority, cache clean/flush on end, clean/flush on + * start */ + cfg = kctx->as_nr; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION) && + !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) + cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; + + if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) + cfg |= JS_CONFIG_START_FLUSH_NO_ACTION; + else + cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; + + if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && + !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) + cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; + else + cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10649)) + cfg |= JS_CONFIG_START_MMU; + + cfg |= JS_CONFIG_THREAD_PRI(8); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE) && + (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED)) + cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; + + if (kbase_hw_has_feature(kbdev, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { + if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) { + cfg |= JS_CONFIG_JOB_CHAIN_FLAG; + katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN; + kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = + true; + } else { + katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN; + kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = + false; + } + } + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg, kctx); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_FLUSH_ID_NEXT), + katom->flush_id, kctx); + + /* Write an approximate start timestamp. + * It's approximate because there might be a job in the HEAD register. + */ + katom->start_timestamp = ktime_get(); + + /* GO ! */ + dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx", + katom, kctx, js, jc_head, katom->affinity); + + KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, + (u32) katom->affinity); + +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) + kbase_trace_mali_job_slots_event( + GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js), + kctx, kbase_jd_atom_id(kctx, katom)); +#endif + KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(katom, jc_head, + katom->affinity, cfg); + KBASE_TLSTREAM_TL_RET_CTX_LPU( + kctx, + &kbdev->gpu_props.props.raw_props.js_features[ + katom->slot_nr]); + KBASE_TLSTREAM_TL_RET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); + KBASE_TLSTREAM_TL_RET_ATOM_LPU( + katom, + &kbdev->gpu_props.props.raw_props.js_features[js], + "ctx_nr,atom_nr"); +#ifdef CONFIG_GPU_TRACEPOINTS + if (!kbase_backend_nr_atoms_submitted(kbdev, js)) { + /* If this is the only job on the slot, trace it as starting */ + char js_string[16]; + + trace_gpu_sched_switch( + kbasep_make_job_slot_string(js, js_string, + sizeof(js_string)), + ktime_to_ns(katom->start_timestamp), + (u32)katom->kctx->id, 0, katom->work_id); + kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx; + } +#endif + kbase_timeline_job_slot_submit(kbdev, kctx, katom, js); + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), + JS_COMMAND_START, katom->kctx); +} + +/** + * kbasep_job_slot_update_head_start_timestamp - Update timestamp + * @kbdev: kbase device + * @js: job slot + * @end_timestamp: timestamp + * + * Update the start_timestamp of the job currently in the HEAD, based on the + * fact that we got an IRQ for the previous set of completed jobs. + * + * The estimate also takes into account the time the job was submitted, to + * work out the best estimate (which might still result in an over-estimate to + * the calculated time spent) + */ +static void kbasep_job_slot_update_head_start_timestamp( + struct kbase_device *kbdev, + int js, + ktime_t end_timestamp) +{ + if (kbase_backend_nr_atoms_on_slot(kbdev, js) > 0) { + struct kbase_jd_atom *katom; + ktime_t timestamp_diff; + /* The atom in the HEAD */ + katom = kbase_gpu_inspect(kbdev, js, 0); + + KBASE_DEBUG_ASSERT(katom != NULL); + + timestamp_diff = ktime_sub(end_timestamp, + katom->start_timestamp); + if (ktime_to_ns(timestamp_diff) >= 0) { + /* Only update the timestamp if it's a better estimate + * than what's currently stored. This is because our + * estimate that accounts for the throttle time may be + * too much of an overestimate */ + katom->start_timestamp = end_timestamp; + } + } +} + +/** + * kbasep_trace_tl_event_lpu_softstop - Call event_lpu_softstop timeline + * tracepoint + * @kbdev: kbase device + * @js: job slot + * + * Make a tracepoint call to the instrumentation module informing that + * softstop happened on given lpu (job slot). + */ +static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, + int js) +{ + KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( + &kbdev->gpu_props.props.raw_props.js_features[js]); +} + +void kbase_job_done(struct kbase_device *kbdev, u32 done) +{ + unsigned long flags; + int i; + u32 count = 0; + ktime_t end_timestamp = ktime_get(); + struct kbasep_js_device_data *js_devdata; + + KBASE_DEBUG_ASSERT(kbdev); + js_devdata = &kbdev->js_data; + + KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done); + + memset(&kbdev->slot_submit_count_irq[0], 0, + sizeof(kbdev->slot_submit_count_irq)); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + while (done) { + u32 failed = done >> 16; + + /* treat failed slots as finished slots */ + u32 finished = (done & 0xFFFF) | failed; + + /* Note: This is inherently unfair, as we always check + * for lower numbered interrupts before the higher + * numbered ones.*/ + i = ffs(finished) - 1; + KBASE_DEBUG_ASSERT(i >= 0); + + do { + int nr_done; + u32 active; + u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */ + u64 job_tail = 0; + + if (failed & (1u << i)) { + /* read out the job slot status code if the job + * slot reported failure */ + completion_code = kbase_reg_read(kbdev, + JOB_SLOT_REG(i, JS_STATUS), NULL); + + switch (completion_code) { + case BASE_JD_EVENT_STOPPED: +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) + kbase_trace_mali_job_slots_event( + GATOR_MAKE_EVENT( + GATOR_JOB_SLOT_SOFT_STOPPED, i), + NULL, 0); +#endif + + kbasep_trace_tl_event_lpu_softstop( + kbdev, i); + + /* Soft-stopped job - read the value of + * JS_TAIL so that the job chain can + * be resumed */ + job_tail = (u64)kbase_reg_read(kbdev, + JOB_SLOT_REG(i, JS_TAIL_LO), + NULL) | + ((u64)kbase_reg_read(kbdev, + JOB_SLOT_REG(i, JS_TAIL_HI), + NULL) << 32); + break; + case BASE_JD_EVENT_NOT_STARTED: + /* PRLAM-10673 can cause a TERMINATED + * job to come back as NOT_STARTED, but + * the error interrupt helps us detect + * it */ + completion_code = + BASE_JD_EVENT_TERMINATED; + /* fall through */ + default: + dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", + i, completion_code, + kbase_exception_name + (kbdev, + completion_code)); + } + + kbase_gpu_irq_evict(kbdev, i); + } + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), + done & ((1 << i) | (1 << (i + 16))), + NULL); + active = kbase_reg_read(kbdev, + JOB_CONTROL_REG(JOB_IRQ_JS_STATE), + NULL); + + if (((active >> i) & 1) == 0 && + (((done >> (i + 16)) & 1) == 0)) { + /* There is a potential race we must work + * around: + * + * 1. A job slot has a job in both current and + * next registers + * 2. The job in current completes + * successfully, the IRQ handler reads + * RAWSTAT and calls this function with the + * relevant bit set in "done" + * 3. The job in the next registers becomes the + * current job on the GPU + * 4. Sometime before the JOB_IRQ_CLEAR line + * above the job on the GPU _fails_ + * 5. The IRQ_CLEAR clears the done bit but not + * the failed bit. This atomically sets + * JOB_IRQ_JS_STATE. However since both jobs + * have now completed the relevant bits for + * the slot are set to 0. + * + * If we now did nothing then we'd incorrectly + * assume that _both_ jobs had completed + * successfully (since we haven't yet observed + * the fail bit being set in RAWSTAT). + * + * So at this point if there are no active jobs + * left we check to see if RAWSTAT has a failure + * bit set for the job slot. If it does we know + * that there has been a new failure that we + * didn't previously know about, so we make sure + * that we record this in active (but we wait + * for the next loop to deal with it). + * + * If we were handling a job failure (i.e. done + * has the relevant high bit set) then we know + * that the value read back from + * JOB_IRQ_JS_STATE is the correct number of + * remaining jobs because the failed job will + * have prevented any futher jobs from starting + * execution. + */ + u32 rawstat = kbase_reg_read(kbdev, + JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL); + + if ((rawstat >> (i + 16)) & 1) { + /* There is a failed job that we've + * missed - add it back to active */ + active |= (1u << i); + } + } + + dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", + completion_code); + + nr_done = kbase_backend_nr_atoms_submitted(kbdev, i); + nr_done -= (active >> i) & 1; + nr_done -= (active >> (i + 16)) & 1; + + if (nr_done <= 0) { + dev_warn(kbdev->dev, "Spurious interrupt on slot %d", + i); + + goto spurious; + } + + count += nr_done; + + while (nr_done) { + if (nr_done == 1) { + kbase_gpu_complete_hw(kbdev, i, + completion_code, + job_tail, + &end_timestamp); + kbase_jm_try_kick_all(kbdev); + } else { + /* More than one job has completed. + * Since this is not the last job being + * reported this time it must have + * passed. This is because the hardware + * will not allow further jobs in a job + * slot to complete until the failed job + * is cleared from the IRQ status. + */ + kbase_gpu_complete_hw(kbdev, i, + BASE_JD_EVENT_DONE, + 0, + &end_timestamp); + } + nr_done--; + } + spurious: + done = kbase_reg_read(kbdev, + JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) { + /* Workaround for missing interrupt caused by + * PRLAM-10883 */ + if (((active >> i) & 1) && (0 == + kbase_reg_read(kbdev, + JOB_SLOT_REG(i, + JS_STATUS), NULL))) { + /* Force job slot to be processed again + */ + done |= (1u << i); + } + } + + failed = done >> 16; + finished = (done & 0xFFFF) | failed; + if (done) + end_timestamp = ktime_get(); + } while (finished & (1 << i)); + + kbasep_job_slot_update_head_start_timestamp(kbdev, i, + end_timestamp); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#if KBASE_GPU_RESET_EN + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_COMMITTED) { + /* If we're trying to reset the GPU then we might be able to do + * it early (without waiting for a timeout) because some jobs + * have completed + */ + kbasep_try_reset_gpu_early(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count); +} +KBASE_EXPORT_TEST_API(kbase_job_done); + +static bool kbasep_soft_stop_allowed(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + bool soft_stops_allowed = true; + + if (kbase_jd_katom_is_protected(katom)) { + soft_stops_allowed = false; + } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) { + if ((katom->core_req & BASE_JD_REQ_T) != 0) + soft_stops_allowed = false; + } + return soft_stops_allowed; +} + +static bool kbasep_hard_stop_allowed(struct kbase_device *kbdev, + base_jd_core_req core_reqs) +{ + bool hard_stops_allowed = true; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) { + if ((core_reqs & BASE_JD_REQ_T) != 0) + hard_stops_allowed = false; + } + return hard_stops_allowed; +} + +void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, + int js, + u32 action, + base_jd_core_req core_reqs, + struct kbase_jd_atom *target_katom) +{ + struct kbase_context *kctx = target_katom->kctx; +#if KBASE_TRACE_ENABLE + u32 status_reg_before; + u64 job_in_head_before; + u32 status_reg_after; + + KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK))); + + /* Check the head pointer */ + job_in_head_before = ((u64) kbase_reg_read(kbdev, + JOB_SLOT_REG(js, JS_HEAD_LO), NULL)) + | (((u64) kbase_reg_read(kbdev, + JOB_SLOT_REG(js, JS_HEAD_HI), NULL)) + << 32); + status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), + NULL); +#endif + + if (action == JS_COMMAND_SOFT_STOP) { + bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev, + target_katom); + + if (!soft_stop_allowed) { +#ifdef CONFIG_MALI_BIFROST_DEBUG + dev_dbg(kbdev->dev, + "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X", + (unsigned int)core_reqs); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + return; + } + + /* We are about to issue a soft stop, so mark the atom as having + * been soft stopped */ + target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED; + + /* Mark the point where we issue the soft-stop command */ + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(target_katom); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { + int i; + + for (i = 0; + i < kbase_backend_nr_atoms_submitted(kbdev, js); + i++) { + struct kbase_jd_atom *katom; + + katom = kbase_gpu_inspect(kbdev, js, i); + + KBASE_DEBUG_ASSERT(katom); + + /* For HW_ISSUE_8316, only 'bad' jobs attacking + * the system can cause this issue: normally, + * all memory should be allocated in multiples + * of 4 pages, and growable memory should be + * changed size in multiples of 4 pages. + * + * Whilst such 'bad' jobs can be cleared by a + * GPU reset, the locking up of a uTLB entry + * caused by the bad job could also stall other + * ASs, meaning that other ASs' jobs don't + * complete in the 'grace' period before the + * reset. We don't want to lose other ASs' jobs + * when they would normally complete fine, so we + * must 'poke' the MMU regularly to help other + * ASs complete */ + kbase_as_poking_timer_retain_atom( + kbdev, katom->kctx, katom); + } + } + + if (kbase_hw_has_feature( + kbdev, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { + action = (target_katom->atom_flags & + KBASE_KATOM_FLAGS_JOBCHAIN) ? + JS_COMMAND_SOFT_STOP_1 : + JS_COMMAND_SOFT_STOP_0; + } + } else if (action == JS_COMMAND_HARD_STOP) { + bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev, + core_reqs); + + if (!hard_stop_allowed) { + /* Jobs can be hard-stopped for the following reasons: + * * CFS decides the job has been running too long (and + * soft-stop has not occurred). In this case the GPU + * will be reset by CFS if the job remains on the + * GPU. + * + * * The context is destroyed, kbase_jd_zap_context + * will attempt to hard-stop the job. However it also + * has a watchdog which will cause the GPU to be + * reset if the job remains on the GPU. + * + * * An (unhandled) MMU fault occurred. As long as + * BASE_HW_ISSUE_8245 is defined then the GPU will be + * reset. + * + * All three cases result in the GPU being reset if the + * hard-stop fails, so it is safe to just return and + * ignore the hard-stop request. + */ + dev_warn(kbdev->dev, + "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X", + (unsigned int)core_reqs); + return; + } + target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED; + + if (kbase_hw_has_feature( + kbdev, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { + action = (target_katom->atom_flags & + KBASE_KATOM_FLAGS_JOBCHAIN) ? + JS_COMMAND_HARD_STOP_1 : + JS_COMMAND_HARD_STOP_0; + } + } + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action, kctx); + +#if KBASE_TRACE_ENABLE + status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), + NULL); + if (status_reg_after == BASE_JD_EVENT_ACTIVE) { + struct kbase_jd_atom *head; + struct kbase_context *head_kctx; + + head = kbase_gpu_inspect(kbdev, js, 0); + head_kctx = head->kctx; + + if (status_reg_before == BASE_JD_EVENT_ACTIVE) + KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, + head, job_in_head_before, js); + else + KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, + 0, js); + + switch (action) { + case JS_COMMAND_SOFT_STOP: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx, + head, head->jc, js); + break; + case JS_COMMAND_SOFT_STOP_0: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, + head, head->jc, js); + break; + case JS_COMMAND_SOFT_STOP_1: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, + head, head->jc, js); + break; + case JS_COMMAND_HARD_STOP: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx, + head, head->jc, js); + break; + case JS_COMMAND_HARD_STOP_0: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, + head, head->jc, js); + break; + case JS_COMMAND_HARD_STOP_1: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, + head, head->jc, js); + break; + default: + BUG(); + break; + } + } else { + if (status_reg_before == BASE_JD_EVENT_ACTIVE) + KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, + job_in_head_before, js); + else + KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, + 0, js); + + switch (action) { + case JS_COMMAND_SOFT_STOP: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, + js); + break; + case JS_COMMAND_SOFT_STOP_0: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, + 0, js); + break; + case JS_COMMAND_SOFT_STOP_1: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, + 0, js); + break; + case JS_COMMAND_HARD_STOP: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, + js); + break; + case JS_COMMAND_HARD_STOP_0: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, + 0, js); + break; + case JS_COMMAND_HARD_STOP_1: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, + 0, js); + break; + default: + BUG(); + break; + } + } +#endif +} + +void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx) +{ + unsigned long flags; + struct kbase_device *kbdev; + int i; + + KBASE_DEBUG_ASSERT(kctx != NULL); + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + /* Cancel any remaining running jobs for this kctx */ + mutex_lock(&kctx->jctx.lock); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Invalidate all jobs in context, to prevent re-submitting */ + for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { + if (!work_pending(&kctx->jctx.atoms[i].work)) + kctx->jctx.atoms[i].event_code = + BASE_JD_EVENT_JOB_CANCELLED; + } + + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + kbase_job_slot_hardstop(kctx, i, NULL); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kctx->jctx.lock); +} + +void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, + struct kbase_jd_atom *target_katom) +{ + struct kbase_device *kbdev; + int js = target_katom->slot_nr; + int priority = target_katom->sched_priority; + int i; + bool stop_sent = false; + + KBASE_DEBUG_ASSERT(kctx != NULL); + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { + struct kbase_jd_atom *katom; + + katom = kbase_gpu_inspect(kbdev, js, i); + if (!katom) + continue; + + if (katom->kctx != kctx) + continue; + + if (katom->sched_priority > priority) { + if (!stop_sent) + KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE( + target_katom); + + kbase_job_slot_softstop(kbdev, js, katom); + stop_sent = true; + } + } +} + +struct zap_reset_data { + /* The stages are: + * 1. The timer has never been called + * 2. The zap has timed out, all slots are soft-stopped - the GPU reset + * will happen. The GPU has been reset when + * kbdev->hwaccess.backend.reset_waitq is signalled + * + * (-1 - The timer has been cancelled) + */ + int stage; + struct kbase_device *kbdev; + struct hrtimer timer; + spinlock_t lock; /* protects updates to stage member */ +}; + +static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer) +{ + struct zap_reset_data *reset_data = container_of(timer, + struct zap_reset_data, timer); + struct kbase_device *kbdev = reset_data->kbdev; + unsigned long flags; + + spin_lock_irqsave(&reset_data->lock, flags); + + if (reset_data->stage == -1) + goto out; + +#if KBASE_GPU_RESET_EN + if (kbase_prepare_to_reset_gpu(kbdev)) { + dev_err(kbdev->dev, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", + ZAP_TIMEOUT); + kbase_reset_gpu(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + reset_data->stage = 2; + + out: + spin_unlock_irqrestore(&reset_data->lock, flags); + + return HRTIMER_NORESTART; +} + +void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + struct zap_reset_data reset_data; + unsigned long flags; + + hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + reset_data.timer.function = zap_timeout_callback; + + spin_lock_init(&reset_data.lock); + + reset_data.kbdev = kbdev; + reset_data.stage = 1; + + hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT), + HRTIMER_MODE_REL); + + /* Wait for all jobs to finish, and for the context to be not-scheduled + * (due to kbase_job_zap_context(), we also guarentee it's not in the JS + * policy queue either */ + wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0); + wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, + !kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + spin_lock_irqsave(&reset_data.lock, flags); + if (reset_data.stage == 1) { + /* The timer hasn't run yet - so cancel it */ + reset_data.stage = -1; + } + spin_unlock_irqrestore(&reset_data.lock, flags); + + hrtimer_cancel(&reset_data.timer); + + if (reset_data.stage == 2) { + /* The reset has already started. + * Wait for the reset to complete + */ + wait_event(kbdev->hwaccess.backend.reset_wait, + atomic_read(&kbdev->hwaccess.backend.reset_gpu) + == KBASE_RESET_GPU_NOT_PENDING); + } + destroy_hrtimer_on_stack(&reset_data.timer); + + dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx); + + /* Ensure that the signallers of the waitqs have finished */ + mutex_lock(&kctx->jctx.lock); + mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&kctx->jctx.lock); +} + +u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) +{ + u32 flush_id = 0; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) { + mutex_lock(&kbdev->pm.lock); + if (kbdev->pm.backend.gpu_powered) + flush_id = kbase_reg_read(kbdev, + GPU_CONTROL_REG(LATEST_FLUSH), NULL); + mutex_unlock(&kbdev->pm.lock); + } + + return flush_id; +} + +int kbase_job_slot_init(struct kbase_device *kbdev) +{ +#if KBASE_GPU_RESET_EN + kbdev->hwaccess.backend.reset_workq = alloc_workqueue( + "Mali reset workqueue", 0, 1); + if (NULL == kbdev->hwaccess.backend.reset_workq) + return -EINVAL; + + KBASE_DEBUG_ASSERT(0 == + object_is_on_stack(&kbdev->hwaccess.backend.reset_work)); + INIT_WORK(&kbdev->hwaccess.backend.reset_work, + kbasep_reset_timeout_worker); + + hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + kbdev->hwaccess.backend.reset_timer.function = + kbasep_reset_timer_callback; +#endif + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_job_slot_init); + +void kbase_job_slot_halt(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +void kbase_job_slot_term(struct kbase_device *kbdev) +{ +#if KBASE_GPU_RESET_EN + destroy_workqueue(kbdev->hwaccess.backend.reset_workq); +#endif +} +KBASE_EXPORT_TEST_API(kbase_job_slot_term); + +#if KBASE_GPU_RESET_EN +/** + * kbasep_check_for_afbc_on_slot() - Check whether AFBC is in use on this slot + * @kbdev: kbase device pointer + * @kctx: context to check against + * @js: slot to check + * @target_katom: An atom to check, or NULL if all atoms from @kctx on + * slot @js should be checked + * + * This checks are based upon parameters that would normally be passed to + * kbase_job_slot_hardstop(). + * + * In the event of @target_katom being NULL, this will check the last jobs that + * are likely to be running on the slot to see if a) they belong to kctx, and + * so would be stopped, and b) whether they have AFBC + * + * In that case, It's guaranteed that a job currently executing on the HW with + * AFBC will be detected. However, this is a conservative check because it also + * detects jobs that have just completed too. + * + * Return: true when hard-stop _might_ stop an afbc atom, else false. + */ +static bool kbasep_check_for_afbc_on_slot(struct kbase_device *kbdev, + struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom) +{ + bool ret = false; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* When we have an atom the decision can be made straight away. */ + if (target_katom) + return !!(target_katom->core_req & BASE_JD_REQ_FS_AFBC); + + /* Otherwise, we must chweck the hardware to see if it has atoms from + * this context with AFBC. */ + for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { + struct kbase_jd_atom *katom; + + katom = kbase_gpu_inspect(kbdev, js, i); + if (!katom) + continue; + + /* Ignore atoms from other contexts, they won't be stopped when + * we use this for checking if we should hard-stop them */ + if (katom->kctx != kctx) + continue; + + /* An atom on this slot and this context: check for AFBC */ + if (katom->core_req & BASE_JD_REQ_FS_AFBC) { + ret = true; + break; + } + } + + return ret; +} +#endif /* KBASE_GPU_RESET_EN */ + +/** + * kbase_job_slot_softstop_swflags - Soft-stop a job with flags + * @kbdev: The kbase device + * @js: The job slot to soft-stop + * @target_katom: The job that should be soft-stopped (or NULL for any job) + * @sw_flags: Flags to pass in about the soft-stop + * + * Context: + * The job slot lock must be held when calling this function. + * The job slot must not already be in the process of being soft-stopped. + * + * Soft-stop the specified job slot, with extra information about the stop + * + * Where possible any job in the next register is evicted before the soft-stop. + */ +void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom, u32 sw_flags) +{ + KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK)); + kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom, + JS_COMMAND_SOFT_STOP | sw_flags); +} + +/** + * kbase_job_slot_softstop - Soft-stop the specified job slot + * @kbdev: The kbase device + * @js: The job slot to soft-stop + * @target_katom: The job that should be soft-stopped (or NULL for any job) + * Context: + * The job slot lock must be held when calling this function. + * The job slot must not already be in the process of being soft-stopped. + * + * Where possible any job in the next register is evicted before the soft-stop. + */ +void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom) +{ + kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u); +} + +/** + * kbase_job_slot_hardstop - Hard-stop the specified job slot + * @kctx: The kbase context that contains the job(s) that should + * be hard-stopped + * @js: The job slot to hard-stop + * @target_katom: The job that should be hard-stopped (or NULL for all + * jobs from the context) + * Context: + * The job slot lock must be held when calling this function. + */ +void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom) +{ + struct kbase_device *kbdev = kctx->kbdev; + bool stopped; +#if KBASE_GPU_RESET_EN + /* We make the check for AFBC before evicting/stopping atoms. Note + * that no other thread can modify the slots whilst we have the + * hwaccess_lock. */ + int needs_workaround_for_afbc = + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3542) + && kbasep_check_for_afbc_on_slot(kbdev, kctx, js, + target_katom); +#endif + + stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js, + target_katom, + JS_COMMAND_HARD_STOP); +#if KBASE_GPU_RESET_EN + if (stopped && (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) || + kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) || + needs_workaround_for_afbc)) { + /* MIDBASE-2916 if a fragment job with AFBC encoding is + * hardstopped, ensure to do a soft reset also in order to + * clear the GPU status. + * Workaround for HW issue 8401 has an issue,so after + * hard-stopping just reset the GPU. This will ensure that the + * jobs leave the GPU.*/ + if (kbase_prepare_to_reset_gpu_locked(kbdev)) { + dev_err(kbdev->dev, "Issueing GPU soft-reset after hard stopping due to hardware issue"); + kbase_reset_gpu_locked(kbdev); + } + } +#endif +} + +/** + * kbase_job_check_enter_disjoint - potentiall enter disjoint mode + * @kbdev: kbase device + * @action: the event which has occurred + * @core_reqs: core requirements of the atom + * @target_katom: the atom which is being affected + * + * For a certain soft/hard-stop action, work out whether to enter disjoint + * state. + * + * This does not register multiple disjoint events if the atom has already + * started a disjoint period + * + * @core_reqs can be supplied as 0 if the atom had not started on the hardware + * (and so a 'real' soft/hard-stop was not required, but it still interrupted + * flow, perhaps on another context) + * + * kbase_job_check_leave_disjoint() should be used to end the disjoint + * state when the soft/hard-stop action is complete + */ +void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, + base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom) +{ + u32 hw_action = action & JS_COMMAND_MASK; + + /* For hard-stop, don't enter if hard-stop not allowed */ + if (hw_action == JS_COMMAND_HARD_STOP && + !kbasep_hard_stop_allowed(kbdev, core_reqs)) + return; + + /* For soft-stop, don't enter if soft-stop not allowed, or isn't + * causing disjoint */ + if (hw_action == JS_COMMAND_SOFT_STOP && + !(kbasep_soft_stop_allowed(kbdev, target_katom) && + (action & JS_COMMAND_SW_CAUSES_DISJOINT))) + return; + + /* Nothing to do if already logged disjoint state on this atom */ + if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) + return; + + target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT; + kbase_disjoint_state_up(kbdev); +} + +/** + * kbase_job_check_enter_disjoint - potentially leave disjoint state + * @kbdev: kbase device + * @target_katom: atom which is finishing + * + * Work out whether to leave disjoint state when finishing an atom that was + * originated by kbase_job_check_enter_disjoint(). + */ +void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, + struct kbase_jd_atom *target_katom) +{ + if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) { + target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT; + kbase_disjoint_state_down(kbdev); + } +} + + +#if KBASE_GPU_RESET_EN +static void kbase_debug_dump_registers(struct kbase_device *kbdev) +{ + int i; + + kbase_io_history_dump(kbdev); + + dev_err(kbdev->dev, "Register state:"); + dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL)); + dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x", + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL), + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL)); + for (i = 0; i < 3; i++) { + dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x", + i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS), + NULL), + i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO), + NULL)); + } + dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", + kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL)); + dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL), + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL), + kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL)); + dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL)); + dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL)); + dev_err(kbdev->dev, " TILER_CONFIG=0x%08x JM_CONFIG=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG), NULL)); +} + +static void kbasep_reset_timeout_worker(struct work_struct *data) +{ + unsigned long flags; + struct kbase_device *kbdev; + ktime_t end_timestamp = ktime_get(); + struct kbasep_js_device_data *js_devdata; + bool try_schedule = false; + bool silent = false; + u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; + + KBASE_DEBUG_ASSERT(data); + + kbdev = container_of(data, struct kbase_device, + hwaccess.backend.reset_work); + + KBASE_DEBUG_ASSERT(kbdev); + js_devdata = &kbdev->js_data; + + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_SILENT) + silent = true; + + KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); + + /* Suspend vinstr. + * This call will block until vinstr is suspended. */ + kbase_vinstr_suspend(kbdev->vinstr_ctx); + + /* Make sure the timer has completed - this cannot be done from + * interrupt context, so this cannot be done within + * kbasep_try_reset_gpu_early. */ + hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); + + if (kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + /* This would re-activate the GPU. Since it's already idle, + * there's no need to reset it */ + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING); + kbase_disjoint_state_down(kbdev); + wake_up(&kbdev->hwaccess.backend.reset_wait); + kbase_vinstr_resume(kbdev->vinstr_ctx); + return; + } + + KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + spin_lock(&kbdev->hwaccess_lock); + spin_lock(&kbdev->mmu_mask_change); + /* We're about to flush out the IRQs and their bottom half's */ + kbdev->irq_reset_flush = true; + + /* Disable IRQ to avoid IRQ handlers to kick in after releasing the + * spinlock; this also clears any outstanding interrupts */ + kbase_pm_disable_interrupts_nolock(kbdev); + + spin_unlock(&kbdev->mmu_mask_change); + spin_unlock(&kbdev->hwaccess_lock); + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + /* Ensure that any IRQ handlers have finished + * Must be done without any locks IRQ handlers will take */ + kbase_synchronize_irqs(kbdev); + + /* Flush out any in-flight work items */ + kbase_flush_mmu_wqs(kbdev); + + /* The flush has completed so reset the active indicator */ + kbdev->irq_reset_flush = false; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { + /* Ensure that L2 is not transitioning when we send the reset + * command */ + while (--max_loops && kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_L2)) + ; + + WARN(!max_loops, "L2 power transition timed out while trying to reset\n"); + } + + mutex_lock(&kbdev->pm.lock); + /* We hold the pm lock, so there ought to be a current policy */ + KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy); + + /* All slot have been soft-stopped and we've waited + * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we + * assume that anything that is still left on the GPU is stuck there and + * we'll kill it when we reset the GPU */ + + if (!silent) + dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", + RESET_TIMEOUT); + + /* Output the state of some interesting registers to help in the + * debugging of GPU resets */ + if (!silent) + kbase_debug_dump_registers(kbdev); + + /* Complete any jobs that were still on the GPU */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->protected_mode = false; + kbase_backend_reset(kbdev, &end_timestamp); + kbase_pm_metrics_update(kbdev, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Reset the GPU */ + kbase_pm_init_hw(kbdev, 0); + + mutex_unlock(&kbdev->pm.lock); + + mutex_lock(&js_devdata->runpool_mutex); + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_restore_all_as(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_pm_enable_interrupts(kbdev); + + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING); + + kbase_disjoint_state_down(kbdev); + + wake_up(&kbdev->hwaccess.backend.reset_wait); + if (!silent) + dev_err(kbdev->dev, "Reset complete"); + + if (js_devdata->nr_contexts_pullable > 0 && !kbdev->poweroff_pending) + try_schedule = true; + + mutex_unlock(&js_devdata->runpool_mutex); + + mutex_lock(&kbdev->pm.lock); + + /* Find out what cores are required now */ + kbase_pm_update_cores_state(kbdev); + + /* Synchronously request and wait for those cores, because if + * instrumentation is enabled it would need them immediately. */ + kbase_pm_check_transitions_sync(kbdev); + + mutex_unlock(&kbdev->pm.lock); + + /* Try submitting some jobs to restart processing */ + if (try_schedule) { + KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, + 0); + kbase_js_sched_all(kbdev); + } + + /* Process any pending slot updates */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_backend_slot_update(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_pm_context_idle(kbdev); + + /* Release vinstr */ + kbase_vinstr_resume(kbdev->vinstr_ctx); + + KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); +} + +static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) +{ + struct kbase_device *kbdev = container_of(timer, struct kbase_device, + hwaccess.backend.reset_timer); + + KBASE_DEBUG_ASSERT(kbdev); + + /* Reset still pending? */ + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) == + KBASE_RESET_GPU_COMMITTED) + queue_work(kbdev->hwaccess.backend.reset_workq, + &kbdev->hwaccess.backend.reset_work); + + return HRTIMER_NORESTART; +} + +/* + * If all jobs are evicted from the GPU then we can reset the GPU + * immediately instead of waiting for the timeout to elapse + */ + +static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) +{ + int i; + int pending_jobs = 0; + + KBASE_DEBUG_ASSERT(kbdev); + + /* Count the number of jobs */ + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i); + + if (pending_jobs > 0) { + /* There are still jobs on the GPU - wait */ + return; + } + + /* To prevent getting incorrect registers when dumping failed job, + * skip early reset. + */ + if (kbdev->job_fault_debug != false) + return; + + /* Check that the reset has been committed to (i.e. kbase_reset_gpu has + * been called), and that no other thread beat this thread to starting + * the reset */ + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != + KBASE_RESET_GPU_COMMITTED) { + /* Reset has already occurred */ + return; + } + + queue_work(kbdev->hwaccess.backend.reset_workq, + &kbdev->hwaccess.backend.reset_work); +} + +static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbasep_try_reset_gpu_early_locked(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +/** + * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU + * @kbdev: kbase device + * + * This function just soft-stops all the slots to ensure that as many jobs as + * possible are saved. + * + * Return: + * The function returns a boolean which should be interpreted as follows: + * true - Prepared for reset, kbase_reset_gpu_locked should be called. + * false - Another thread is performing a reset, kbase_reset_gpu should + * not be called. + */ +bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) +{ + int i; + + KBASE_DEBUG_ASSERT(kbdev); + + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING, + KBASE_RESET_GPU_PREPARED) != + KBASE_RESET_GPU_NOT_PENDING) { + /* Some other thread is already resetting the GPU */ + return false; + } + + kbase_disjoint_state_up(kbdev); + + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + kbase_job_slot_softstop(kbdev, i, NULL); + + return true; +} + +bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) +{ + unsigned long flags; + bool ret; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + ret = kbase_prepare_to_reset_gpu_locked(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return ret; +} +KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); + +/* + * This function should be called after kbase_prepare_to_reset_gpu if it + * returns true. It should never be called without a corresponding call to + * kbase_prepare_to_reset_gpu. + * + * After this function is called (or not called if kbase_prepare_to_reset_gpu + * returned false), the caller should wait for + * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset + * has completed. + */ +void kbase_reset_gpu(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + + /* Note this is an assert/atomic_set because it is a software issue for + * a race to be occuring here */ + KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_PREPARED); + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED); + + dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", + kbdev->reset_timeout_ms); + + hrtimer_start(&kbdev->hwaccess.backend.reset_timer, + HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), + HRTIMER_MODE_REL); + + /* Try resetting early */ + kbasep_try_reset_gpu_early(kbdev); +} +KBASE_EXPORT_TEST_API(kbase_reset_gpu); + +void kbase_reset_gpu_locked(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + + /* Note this is an assert/atomic_set because it is a software issue for + * a race to be occuring here */ + KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_PREPARED); + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED); + + dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", + kbdev->reset_timeout_ms); + hrtimer_start(&kbdev->hwaccess.backend.reset_timer, + HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), + HRTIMER_MODE_REL); + + /* Try resetting early */ + kbasep_try_reset_gpu_early_locked(kbdev); +} + +void kbase_reset_gpu_silent(struct kbase_device *kbdev) +{ + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING, + KBASE_RESET_GPU_SILENT) != + KBASE_RESET_GPU_NOT_PENDING) { + /* Some other thread is already resetting the GPU */ + return; + } + + kbase_disjoint_state_up(kbdev); + + queue_work(kbdev->hwaccess.backend.reset_workq, + &kbdev->hwaccess.backend.reset_work); +} + +bool kbase_reset_gpu_active(struct kbase_device *kbdev) +{ + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_NOT_PENDING) + return false; + + return true; +} +#endif /* KBASE_GPU_RESET_EN */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_internal.h new file mode 100755 index 000000000000..1f382b3c1af4 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_internal.h @@ -0,0 +1,164 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Job Manager backend-specific low-level APIs. + */ + +#ifndef _KBASE_JM_HWACCESS_H_ +#define _KBASE_JM_HWACCESS_H_ + +#include +#include +#include + +#include + +/** + * kbase_job_submit_nolock() - Submit a job to a certain job-slot + * @kbdev: Device pointer + * @katom: Atom to submit + * @js: Job slot to submit on + * + * The caller must check kbasep_jm_is_submit_slots_free() != false before + * calling this. + * + * The following locking conditions are made on the caller: + * - it must hold the hwaccess_lock + */ +void kbase_job_submit_nolock(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, int js); + +/** + * kbase_job_done_slot() - Complete the head job on a particular job-slot + * @kbdev: Device pointer + * @s: Job slot + * @completion_code: Completion code of job reported by GPU + * @job_tail: Job tail address reported by GPU + * @end_timestamp: Timestamp of job completion + */ +void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, + u64 job_tail, ktime_t *end_timestamp); + +#ifdef CONFIG_GPU_TRACEPOINTS +static inline char *kbasep_make_job_slot_string(int js, char *js_string, + size_t js_size) +{ + snprintf(js_string, js_size, "job_slot_%i", js); + return js_string; +} +#endif + +/** + * kbase_job_hw_submit() - Submit a job to the GPU + * @kbdev: Device pointer + * @katom: Atom to submit + * @js: Job slot to submit on + * + * The caller must check kbasep_jm_is_submit_slots_free() != false before + * calling this. + * + * The following locking conditions are made on the caller: + * - it must hold the hwaccess_lock + */ +void kbase_job_hw_submit(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + int js); + +/** + * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop + * on the specified atom + * @kbdev: Device pointer + * @js: Job slot to stop on + * @action: The action to perform, either JSn_COMMAND_HARD_STOP or + * JSn_COMMAND_SOFT_STOP + * @core_reqs: Core requirements of atom to stop + * @target_katom: Atom to stop + * + * The following locking conditions are made on the caller: + * - it must hold the hwaccess_lock + */ +void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, + int js, + u32 action, + base_jd_core_req core_reqs, + struct kbase_jd_atom *target_katom); + +/** + * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job + * slot belonging to a given context. + * @kbdev: Device pointer + * @kctx: Context pointer. May be NULL + * @katom: Specific atom to stop. May be NULL + * @js: Job slot to hard stop + * @action: The action to perform, either JSn_COMMAND_HARD_STOP or + * JSn_COMMAND_SOFT_STOP + * + * If no context is provided then all jobs on the slot will be soft or hard + * stopped. + * + * If a katom is provided then only that specific atom will be stopped. In this + * case the kctx parameter is ignored. + * + * Jobs that are on the slot but are not yet on the GPU will be unpulled and + * returned to the job scheduler. + * + * Return: true if an atom was stopped, false otherwise + */ +bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js, + struct kbase_jd_atom *katom, + u32 action); + +/** + * kbase_job_slot_init - Initialise job slot framework + * @kbdev: Device pointer + * + * Called on driver initialisation + * + * Return: 0 on success + */ +int kbase_job_slot_init(struct kbase_device *kbdev); + +/** + * kbase_job_slot_halt - Halt the job slot framework + * @kbdev: Device pointer + * + * Should prevent any further job slot processing + */ +void kbase_job_slot_halt(struct kbase_device *kbdev); + +/** + * kbase_job_slot_term - Terminate job slot framework + * @kbdev: Device pointer + * + * Called on driver termination + */ +void kbase_job_slot_term(struct kbase_device *kbdev); + +/** + * kbase_gpu_cacheclean - Cause a GPU cache clean & flush + * @kbdev: Device pointer + * + * Caller must not be in IRQ context + */ +void kbase_gpu_cacheclean(struct kbase_device *kbdev); + +#endif /* _KBASE_JM_HWACCESS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.c new file mode 100755 index 000000000000..a41e7b5b7afb --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.c @@ -0,0 +1,1947 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend specific APIs + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Return whether the specified ringbuffer is empty. HW access lock must be + * held */ +#define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) +/* Return number of atoms currently in the specified ringbuffer. HW access lock + * must be held */ +#define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) + +static void kbase_gpu_release_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + ktime_t *end_timestamp); + +/** + * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer + * @kbdev: Device pointer + * @katom: Atom to enqueue + * + * Context: Caller must hold the HW access lock + */ +static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; + + WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; + rb->write_idx++; + + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; +} + +/** + * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once + * it has been completed + * @kbdev: Device pointer + * @js: Job slot to remove atom from + * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in + * which case current time will be used. + * + * Context: Caller must hold the HW access lock + * + * Return: Atom removed from ringbuffer + */ +static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, + int js, + ktime_t *end_timestamp) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; + struct kbase_jd_atom *katom; + + if (SLOT_RB_EMPTY(rb)) { + WARN(1, "GPU ringbuffer unexpectedly empty\n"); + return NULL; + } + + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; + + kbase_gpu_release_atom(kbdev, katom, end_timestamp); + + rb->read_idx++; + + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; + + kbase_js_debug_log_current_affinities(kbdev); + + return katom; +} + +struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, + int idx) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if ((SLOT_RB_ENTRIES(rb) - 1) < idx) + return NULL; /* idx out of range */ + + return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; +} + +struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, + int js) +{ + return kbase_gpu_inspect(kbdev, js, 0); +} + +struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, + int js) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; + + if (SLOT_RB_EMPTY(rb)) + return NULL; + + return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; +} + +/** + * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently + * on the GPU + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return: true if there are atoms on the GPU for slot js, + * false otherwise + */ +static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js) +{ + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (!katom) + return false; + if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED || + katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY) + return true; + } + + return false; +} + +/** + * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms + * currently on the GPU + * @kbdev: Device pointer + * + * Return: true if there are any atoms on the GPU, false otherwise + */ +static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) +{ + int js; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) + return true; + } + } + return false; +} + +int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js) +{ + int nr = 0; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (katom && (katom->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED)) + nr++; + } + + return nr; +} + +int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js) +{ + int nr = 0; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + if (kbase_gpu_inspect(kbdev, js, i)) + nr++; + } + + return nr; +} + +static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js, + enum kbase_atom_gpu_rb_state min_rb_state) +{ + int nr = 0; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (katom && (katom->gpu_rb_state >= min_rb_state)) + nr++; + } + + return nr; +} + +/** + * check_secure_atom - Check if the given atom is in the given secure state and + * has a ringbuffer state of at least + * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION + * @katom: Atom pointer + * @secure: Desired secure state + * + * Return: true if atom is in the given state, false otherwise + */ +static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) +{ + if (katom->gpu_rb_state >= + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && + ((kbase_jd_katom_is_protected(katom) && secure) || + (!kbase_jd_katom_is_protected(katom) && !secure))) + return true; + + return false; +} + +/** + * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given + * secure state in the ringbuffers of at least + * state + * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE + * @kbdev: Device pointer + * @secure: Desired secure state + * + * Return: true if any atoms are in the given state, false otherwise + */ +static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, + bool secure) +{ + int js, i; + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, + js, i); + + if (katom) { + if (check_secure_atom(katom, secure)) + return true; + } + } + } + + return false; +} + +int kbase_backend_slot_free(struct kbase_device *kbdev, int js) +{ + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != + KBASE_RESET_GPU_NOT_PENDING) { + /* The GPU is being reset - so prevent submission */ + return 0; + } + + return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); +} + + +static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev, + int js, + struct kbase_jd_atom *katom) +{ + /* The most recently checked affinity. Having this at this scope allows + * us to guarantee that we've checked the affinity in this function + * call. + */ + u64 recently_chosen_affinity = 0; + bool chosen_affinity = false; + bool retry; + + do { + retry = false; + + /* NOTE: The following uses a number of FALLTHROUGHs to optimize + * the calls to this function. Ending of the function is + * indicated by BREAK OUT */ + switch (katom->coreref_state) { + /* State when job is first attempted to be run */ + case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: + KBASE_DEBUG_ASSERT(katom->affinity == 0); + + /* Compute affinity */ + if (false == kbase_js_choose_affinity( + &recently_chosen_affinity, kbdev, katom, + js)) { + /* No cores are currently available */ + /* *** BREAK OUT: No state transition *** */ + break; + } + + chosen_affinity = true; + + /* Request the cores */ + kbase_pm_request_cores(kbdev, + katom->core_req & BASE_JD_REQ_T, + recently_chosen_affinity); + + katom->affinity = recently_chosen_affinity; + + /* Proceed to next state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: + { + enum kbase_pm_cores_ready cores_ready; + + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + + cores_ready = kbase_pm_register_inuse_cores( + kbdev, + katom->core_req & BASE_JD_REQ_T, + katom->affinity); + if (cores_ready == KBASE_NEW_AFFINITY) { + /* Affinity no longer valid - return to + * previous state */ + kbasep_js_job_check_deref_cores(kbdev, + katom); + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_REGISTER_INUSE_FAILED, + katom->kctx, katom, + katom->jc, js, + (u32) katom->affinity); + /* *** BREAK OUT: Return to previous + * state, retry *** */ + retry = true; + break; + } + if (cores_ready == KBASE_CORES_NOT_READY) { + /* Stay in this state and return, to + * retry at this state later */ + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_REGISTER_INUSE_FAILED, + katom->kctx, katom, + katom->jc, js, + (u32) katom->affinity); + /* *** BREAK OUT: No state transition + * *** */ + break; + } + /* Proceed to next state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; + } + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + + /* Optimize out choosing the affinity twice in the same + * function call */ + if (chosen_affinity == false) { + /* See if the affinity changed since a previous + * call. */ + if (false == kbase_js_choose_affinity( + &recently_chosen_affinity, + kbdev, katom, js)) { + /* No cores are currently available */ + kbasep_js_job_check_deref_cores(kbdev, + katom); + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_REQUEST_ON_RECHECK_FAILED, + katom->kctx, katom, + katom->jc, js, + (u32) recently_chosen_affinity); + /* *** BREAK OUT: Transition to lower + * state *** */ + break; + } + chosen_affinity = true; + } + + /* Now see if this requires a different set of cores */ + if (recently_chosen_affinity != katom->affinity) { + enum kbase_pm_cores_ready cores_ready; + + kbase_pm_request_cores(kbdev, + katom->core_req & BASE_JD_REQ_T, + recently_chosen_affinity); + + /* Register new cores whilst we still hold the + * old ones, to minimize power transitions */ + cores_ready = + kbase_pm_register_inuse_cores(kbdev, + katom->core_req & BASE_JD_REQ_T, + recently_chosen_affinity); + kbasep_js_job_check_deref_cores(kbdev, katom); + + /* Fixup the state that was reduced by + * deref_cores: */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; + katom->affinity = recently_chosen_affinity; + if (cores_ready == KBASE_NEW_AFFINITY) { + /* Affinity no longer valid - return to + * previous state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; + + kbasep_js_job_check_deref_cores(kbdev, + katom); + + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_REGISTER_INUSE_FAILED, + katom->kctx, katom, + katom->jc, js, + (u32) katom->affinity); + /* *** BREAK OUT: Return to previous + * state, retry *** */ + retry = true; + break; + } + /* Now might be waiting for powerup again, with + * a new affinity */ + if (cores_ready == KBASE_CORES_NOT_READY) { + /* Return to previous state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_REGISTER_ON_RECHECK_FAILED, + katom->kctx, katom, + katom->jc, js, + (u32) katom->affinity); + /* *** BREAK OUT: Transition to lower + * state *** */ + break; + } + } + /* Proceed to next state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS: + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + KBASE_DEBUG_ASSERT(katom->affinity == + recently_chosen_affinity); + + /* Note: this is where the caller must've taken the + * hwaccess_lock */ + + /* Check for affinity violations - if there are any, + * then we just ask the caller to requeue and try again + * later */ + if (kbase_js_affinity_would_violate(kbdev, js, + katom->affinity) != false) { + /* Return to previous state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; + /* *** BREAK OUT: Transition to lower state *** + */ + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_AFFINITY_WOULD_VIOLATE, + katom->kctx, katom, katom->jc, js, + (u32) katom->affinity); + break; + } + + /* No affinity violations would result, so the cores are + * ready */ + katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY; + /* *** BREAK OUT: Cores Ready *** */ + break; + + default: + KBASE_DEBUG_ASSERT_MSG(false, + "Unhandled kbase_atom_coreref_state %d", + katom->coreref_state); + break; + } + } while (retry != false); + + return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY); +} + +static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(katom != NULL); + + switch (katom->coreref_state) { + case KBASE_ATOM_COREREF_STATE_READY: + /* State where atom was submitted to the HW - just proceed to + * power-down */ + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + + /* *** FALLTHROUGH *** */ + + case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: + /* State where cores were registered */ + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T, + katom->affinity); + + break; + + case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: + /* State where cores were requested, but not registered */ + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T, + katom->affinity); + break; + + case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: + /* Initial state - nothing required */ + KBASE_DEBUG_ASSERT(katom->affinity == 0); + break; + + default: + KBASE_DEBUG_ASSERT_MSG(false, + "Unhandled coreref_state: %d", + katom->coreref_state); + break; + } + + katom->affinity = 0; + katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; +} + +static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev, + base_jd_core_req core_req, u64 affinity, + enum kbase_atom_coreref_state coreref_state) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + switch (coreref_state) { + case KBASE_ATOM_COREREF_STATE_READY: + /* State where atom was submitted to the HW - just proceed to + * power-down */ + KBASE_DEBUG_ASSERT(affinity != 0 || + (core_req & BASE_JD_REQ_T)); + + /* *** FALLTHROUGH *** */ + + case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: + /* State where cores were registered */ + KBASE_DEBUG_ASSERT(affinity != 0 || + (core_req & BASE_JD_REQ_T)); + kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T, + affinity); + + break; + + case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: + /* State where cores were requested, but not registered */ + KBASE_DEBUG_ASSERT(affinity != 0 || + (core_req & BASE_JD_REQ_T)); + kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T, + affinity); + break; + + case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: + /* Initial state - nothing required */ + KBASE_DEBUG_ASSERT(affinity == 0); + break; + + default: + KBASE_DEBUG_ASSERT_MSG(false, + "Unhandled coreref_state: %d", + coreref_state); + break; + } +} + +static void kbase_gpu_release_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + ktime_t *end_timestamp) +{ + struct kbase_context *kctx = katom->kctx; + + switch (katom->gpu_rb_state) { + case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: + /* Should be impossible */ + WARN(1, "Attempting to release atom not in ringbuffer\n"); + break; + + case KBASE_ATOM_GPU_RB_SUBMITTED: + /* Inform power management at start/finish of atom so it can + * update its GPU utilisation metrics. Mark atom as not + * submitted beforehand. */ + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; + kbase_pm_metrics_update(kbdev, end_timestamp); + + if (katom->core_req & BASE_JD_REQ_PERMON) + kbase_pm_release_gpu_cycle_counter_nolock(kbdev); + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, + &kbdev->gpu_props.props.raw_props.js_features + [katom->slot_nr]); + KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); + KBASE_TLSTREAM_TL_NRET_CTX_LPU(kctx, + &kbdev->gpu_props.props.raw_props.js_features + [katom->slot_nr]); + + case KBASE_ATOM_GPU_RB_READY: + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: + kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr, + katom->affinity); + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: + break; + + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: + if (katom->protected_state.enter != + KBASE_ATOM_ENTER_PROTECTED_CHECK || + katom->protected_state.exit != + KBASE_ATOM_EXIT_PROTECTED_CHECK) + kbdev->protected_mode_transition = false; + + if (kbase_jd_katom_is_protected(katom) && + (katom->protected_state.enter == + KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) { + kbase_vinstr_resume(kbdev->vinstr_ctx); + + /* Go back to configured model for IPA */ + kbase_ipa_model_use_configured_locked(kbdev); + } + + + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + case KBASE_ATOM_GPU_RB_RETURN_TO_JS: + break; + } + + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; + katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; +} + +static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + kbase_gpu_release_atom(kbdev, katom, NULL); + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; +} + +static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + bool slot_busy[3]; + + if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) + return true; + slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0, + KBASE_ATOM_GPU_RB_WAITING_AFFINITY); + slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1, + KBASE_ATOM_GPU_RB_WAITING_AFFINITY); + slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2, + KBASE_ATOM_GPU_RB_WAITING_AFFINITY); + + if ((js == 2 && !(slot_busy[0] || slot_busy[1])) || + (js != 2 && !slot_busy[2])) + return true; + + /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */ + if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) || + kbase_gpu_atoms_submitted(kbdev, 1) || + backend->rmu_workaround_flag)) + return false; + + /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */ + if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) || + !backend->rmu_workaround_flag)) + return false; + + backend->rmu_workaround_flag = !backend->rmu_workaround_flag; + + return true; +} + +/** + * other_slots_busy - Determine if any job slots other than @js are currently + * running atoms + * @kbdev: Device pointer + * @js: Job slot + * + * Return: true if any slots other than @js are busy, false otherwise + */ +static inline bool other_slots_busy(struct kbase_device *kbdev, int js) +{ + int slot; + + for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) { + if (slot == js) + continue; + + if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot, + KBASE_ATOM_GPU_RB_SUBMITTED)) + return true; + } + + return false; +} + +static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) +{ + return kbdev->protected_mode; +} + +static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) +{ + int err = -EINVAL; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ONCE(!kbdev->protected_ops, + "Cannot enter protected mode: protected callbacks not specified.\n"); + + /* + * When entering into protected mode, we must ensure that the + * GPU is not operating in coherent mode as well. This is to + * ensure that no protected memory can be leaked. + */ + if (kbdev->system_coherency == COHERENCY_ACE) + kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); + + if (kbdev->protected_ops) { + /* Switch GPU to protected mode */ + err = kbdev->protected_ops->protected_mode_enable( + kbdev->protected_dev); + + if (err) + dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", + err); + else + kbdev->protected_mode = true; + } + + return err; +} + +static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ONCE(!kbdev->protected_ops, + "Cannot exit protected mode: protected callbacks not specified.\n"); + + if (!kbdev->protected_ops) + return -EINVAL; + + /* The protected mode disable callback will be called as part of reset + */ + kbase_reset_gpu_silent(kbdev); + + return 0; +} + +static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, + struct kbase_jd_atom **katom, int idx, int js) +{ + int err = 0; + + switch (katom[idx]->protected_state.enter) { + case KBASE_ATOM_ENTER_PROTECTED_CHECK: + KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev); + /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV + * should ensure that we are not already transitiong, and that + * there are no atoms currently on the GPU. */ + WARN_ON(kbdev->protected_mode_transition); + WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); + + kbdev->protected_mode_transition = true; + katom[idx]->protected_state.enter = + KBASE_ATOM_ENTER_PROTECTED_VINSTR; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_ENTER_PROTECTED_VINSTR: + if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) { + /* + * We can't switch now because + * the vinstr core state switch + * is not done yet. + */ + return -EAGAIN; + } + + /* Use generic model for IPA in protected mode */ + kbase_ipa_model_use_fallback_locked(kbdev); + + /* Once reaching this point GPU must be + * switched to protected mode or vinstr + * re-enabled. */ + + /* + * Not in correct mode, begin protected mode switch. + * Entering protected mode requires us to power down the L2, + * and drop out of fully coherent mode. + */ + katom[idx]->protected_state.enter = + KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; + + kbase_pm_update_cores_state_nolock(kbdev); + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: + /* Avoid unnecessary waiting on non-ACE platforms. */ + if (kbdev->current_gpu_coherency_mode == COHERENCY_ACE) { + if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || + kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { + /* + * The L2 is still powered, wait for all the users to + * finish with it before doing the actual reset. + */ + return -EAGAIN; + } + } + + katom[idx]->protected_state.enter = + KBASE_ATOM_ENTER_PROTECTED_FINISHED; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_ENTER_PROTECTED_FINISHED: + + /* No jobs running, so we can switch GPU mode right now. */ + err = kbase_gpu_protected_mode_enter(kbdev); + + /* + * Regardless of result, we are no longer transitioning + * the GPU. + */ + kbdev->protected_mode_transition = false; + KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev); + if (err) { + /* + * Failed to switch into protected mode, resume + * vinstr core and fail atom. + */ + kbase_vinstr_resume(kbdev->vinstr_ctx); + katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; + kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); + /* Only return if head atom or previous atom + * already removed - as atoms must be returned + * in order. */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, katom[idx]); + } + + /* Go back to configured model for IPA */ + kbase_ipa_model_use_configured_locked(kbdev); + + return -EINVAL; + } + + /* Protected mode sanity checks. */ + KBASE_DEBUG_ASSERT_MSG( + kbase_jd_katom_is_protected(katom[idx]) == + kbase_gpu_in_protected_mode(kbdev), + "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", + kbase_jd_katom_is_protected(katom[idx]), + kbase_gpu_in_protected_mode(kbdev)); + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_READY; + } + + return 0; +} + +static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, + struct kbase_jd_atom **katom, int idx, int js) +{ + int err = 0; + + + switch (katom[idx]->protected_state.exit) { + case KBASE_ATOM_EXIT_PROTECTED_CHECK: + KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev); + /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV + * should ensure that we are not already transitiong, and that + * there are no atoms currently on the GPU. */ + WARN_ON(kbdev->protected_mode_transition); + WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); + + /* + * Exiting protected mode requires a reset, but first the L2 + * needs to be powered down to ensure it's not active when the + * reset is issued. + */ + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; + + kbdev->protected_mode_transition = true; + kbase_pm_update_cores_state_nolock(kbdev); + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: + if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || + kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { + /* + * The L2 is still powered, wait for all the users to + * finish with it before doing the actual reset. + */ + return -EAGAIN; + } + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_RESET; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_EXIT_PROTECTED_RESET: + /* Issue the reset to the GPU */ + err = kbase_gpu_protected_mode_reset(kbdev); + + if (err) { + kbdev->protected_mode_transition = false; + + /* Failed to exit protected mode, fail atom */ + katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; + kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); + /* Only return if head atom or previous atom + * already removed - as atoms must be returned + * in order */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, katom[idx]); + } + + kbase_vinstr_resume(kbdev->vinstr_ctx); + + /* Use generic model for IPA in protected mode */ + kbase_ipa_model_use_fallback_locked(kbdev); + + return -EINVAL; + } + + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: + /* A GPU reset is issued when exiting protected mode. Once the + * reset is done all atoms' state will also be reset. For this + * reason, if the atom is still in this state we can safely + * say that the reset has not completed i.e., we have not + * finished exiting protected mode yet. + */ + return -EAGAIN; + } + + return 0; +} + +void kbase_backend_slot_update(struct kbase_device *kbdev) +{ + int js; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + struct kbase_jd_atom *katom[2]; + int idx; + + katom[0] = kbase_gpu_inspect(kbdev, js, 0); + katom[1] = kbase_gpu_inspect(kbdev, js, 1); + WARN_ON(katom[1] && !katom[0]); + + for (idx = 0; idx < SLOT_RB_SIZE; idx++) { + bool cores_ready; + int ret; + + if (!katom[idx]) + continue; + + switch (katom[idx]->gpu_rb_state) { + case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: + /* Should be impossible */ + WARN(1, "Attempting to update atom not in ringbuffer\n"); + break; + + case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: + if (katom[idx]->atom_flags & + KBASE_KATOM_FLAG_X_DEP_BLOCKED) + break; + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: + if (kbase_gpu_check_secure_atoms(kbdev, + !kbase_jd_katom_is_protected( + katom[idx]))) + break; + + if ((idx == 1) && (kbase_jd_katom_is_protected( + katom[0]) != + kbase_jd_katom_is_protected( + katom[1]))) + break; + + if (kbdev->protected_mode_transition) + break; + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: + + /* + * Exiting protected mode must be done before + * the references on the cores are taken as + * a power down the L2 is required which + * can't happen after the references for this + * atom are taken. + */ + + if (!kbase_gpu_in_protected_mode(kbdev) && + kbase_jd_katom_is_protected(katom[idx])) { + /* Atom needs to transition into protected mode. */ + ret = kbase_jm_enter_protected_mode(kbdev, + katom, idx, js); + if (ret) + break; + } else if (kbase_gpu_in_protected_mode(kbdev) && + !kbase_jd_katom_is_protected(katom[idx])) { + /* Atom needs to transition out of protected mode. */ + ret = kbase_jm_exit_protected_mode(kbdev, + katom, idx, js); + if (ret) + break; + } + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_CHECK; + + /* Atom needs no protected mode transition. */ + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: + if (katom[idx]->will_fail_event_code) { + kbase_gpu_mark_atom_for_return(kbdev, + katom[idx]); + /* Set EVENT_DONE so this atom will be + completed, not unpulled. */ + katom[idx]->event_code = + BASE_JD_EVENT_DONE; + /* Only return if head atom or previous + * atom already removed - as atoms must + * be returned in order. */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, katom[idx]); + } + break; + } + + cores_ready = + kbasep_js_job_check_ref_cores(kbdev, js, + katom[idx]); + + if (katom[idx]->event_code == + BASE_JD_EVENT_PM_EVENT) { + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_RETURN_TO_JS; + break; + } + + if (!cores_ready) + break; + + kbase_js_affinity_retain_slot_cores(kbdev, js, + katom[idx]->affinity); + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_AFFINITY; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: + if (!kbase_gpu_rmu_workaround(kbdev, js)) + break; + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_READY; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_GPU_RB_READY: + + if (idx == 1) { + /* Only submit if head atom or previous + * atom already submitted */ + if ((katom[0]->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED && + katom[0]->gpu_rb_state != + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) + break; + + /* If intra-slot serialization in use + * then don't submit atom to NEXT slot + */ + if (kbdev->serialize_jobs & + KBASE_SERIALIZE_INTRA_SLOT) + break; + } + + /* If inter-slot serialization in use then don't + * submit atom if any other slots are in use */ + if ((kbdev->serialize_jobs & + KBASE_SERIALIZE_INTER_SLOT) && + other_slots_busy(kbdev, js)) + break; + + if ((kbdev->serialize_jobs & + KBASE_SERIALIZE_RESET) && + kbase_reset_gpu_active(kbdev)) + break; + + /* Check if this job needs the cycle counter + * enabled before submission */ + if (katom[idx]->core_req & BASE_JD_REQ_PERMON) + kbase_pm_request_gpu_cycle_counter_l2_is_on( + kbdev); + + kbase_job_hw_submit(kbdev, katom[idx], js); + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_SUBMITTED; + + /* Inform power management at start/finish of + * atom so it can update its GPU utilisation + * metrics. */ + kbase_pm_metrics_update(kbdev, + &katom[idx]->start_timestamp); + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + + case KBASE_ATOM_GPU_RB_SUBMITTED: + /* Atom submitted to HW, nothing else to do */ + break; + + case KBASE_ATOM_GPU_RB_RETURN_TO_JS: + /* Only return if head atom or previous atom + * already removed - as atoms must be returned + * in order */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, + katom[idx]); + } + break; + } + } + } + + /* Warn if PRLAM-8987 affinity restrictions are violated */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) + WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) || + kbase_gpu_atoms_submitted(kbdev, 1)) && + kbase_gpu_atoms_submitted(kbdev, 2)); +} + + +void kbase_backend_run_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + kbase_gpu_enqueue_atom(kbdev, katom); + kbase_backend_slot_update(kbdev); +} + +#define HAS_DEP(katom) (katom->pre_dep || katom->atom_flags & \ + (KBASE_KATOM_FLAG_X_DEP_BLOCKED | KBASE_KATOM_FLAG_FAIL_BLOCKER)) + +bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js) +{ + struct kbase_jd_atom *katom; + struct kbase_jd_atom *next_katom; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom = kbase_gpu_inspect(kbdev, js, 0); + next_katom = kbase_gpu_inspect(kbdev, js, 1); + + if (next_katom && katom->kctx == next_katom->kctx && + next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && + HAS_DEP(next_katom) && + (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) + != 0 || + kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) + != 0)) { + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), + JS_COMMAND_NOP, NULL); + next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; + + KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, + &kbdev->gpu_props.props.raw_props.js_features + [katom->slot_nr]); + KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as + [katom->kctx->as_nr]); + KBASE_TLSTREAM_TL_NRET_CTX_LPU(katom->kctx, + &kbdev->gpu_props.props.raw_props.js_features + [katom->slot_nr]); + + return true; + } + + return false; +} + +void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, + u32 completion_code, + u64 job_tail, + ktime_t *end_timestamp) +{ + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); + struct kbase_context *kctx = katom->kctx; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* + * When a hard-stop is followed close after a soft-stop, the completion + * code may be set to STOPPED, even though the job is terminated + */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) { + if (completion_code == BASE_JD_EVENT_STOPPED && + (katom->atom_flags & + KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { + completion_code = BASE_JD_EVENT_TERMINATED; + } + } + + if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) || (katom->core_req & + BASE_JD_REQ_SKIP_CACHE_END)) && + completion_code != BASE_JD_EVENT_DONE && + !(completion_code & BASE_JD_SW_EVENT)) { + /* When a job chain fails, on a T60x or when + * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not + * flushed. To prevent future evictions causing possible memory + * corruption we need to flush the cache manually before any + * affected memory gets reused. */ + katom->need_cache_flush_cores_retained = katom->affinity; + kbase_pm_request_cores(kbdev, false, katom->affinity); + } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) { + if (kbdev->gpu_props.num_core_groups > 1 && + !(katom->affinity & + kbdev->gpu_props.props.coherency_info.group[0].core_mask + ) && + (katom->affinity & + kbdev->gpu_props.props.coherency_info.group[1].core_mask + )) { + dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n"); + katom->need_cache_flush_cores_retained = + katom->affinity; + kbase_pm_request_cores(kbdev, false, + katom->affinity); + } + } + + katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); + kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0); + + if (completion_code == BASE_JD_EVENT_STOPPED) { + struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, + 0); + + /* + * Dequeue next atom from ringbuffers on same slot if required. + * This atom will already have been removed from the NEXT + * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that + * the atoms on this slot are returned in the correct order. + */ + if (next_katom && katom->kctx == next_katom->kctx && + next_katom->sched_priority == + katom->sched_priority) { + kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); + kbase_jm_return_atom_to_js(kbdev, next_katom); + } + } else if (completion_code != BASE_JD_EVENT_DONE) { + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + int i; + +#if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0 + KBASE_TRACE_DUMP(kbdev); +#endif + kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); + + /* + * Remove all atoms on the same context from ringbuffers. This + * will not remove atoms that are already on the GPU, as these + * are guaranteed not to have fail dependencies on the failed + * atom. + */ + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { + struct kbase_jd_atom *katom_idx0 = + kbase_gpu_inspect(kbdev, i, 0); + struct kbase_jd_atom *katom_idx1 = + kbase_gpu_inspect(kbdev, i, 1); + + if (katom_idx0 && katom_idx0->kctx == katom->kctx && + HAS_DEP(katom_idx0) && + katom_idx0->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Dequeue katom_idx0 from ringbuffer */ + kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); + + if (katom_idx1 && + katom_idx1->kctx == katom->kctx + && HAS_DEP(katom_idx1) && + katom_idx0->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Dequeue katom_idx1 from ringbuffer */ + kbase_gpu_dequeue_atom(kbdev, i, + end_timestamp); + + katom_idx1->event_code = + BASE_JD_EVENT_STOPPED; + kbase_jm_return_atom_to_js(kbdev, + katom_idx1); + } + katom_idx0->event_code = BASE_JD_EVENT_STOPPED; + kbase_jm_return_atom_to_js(kbdev, katom_idx0); + + } else if (katom_idx1 && + katom_idx1->kctx == katom->kctx && + HAS_DEP(katom_idx1) && + katom_idx1->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Can not dequeue this atom yet - will be + * dequeued when atom at idx0 completes */ + katom_idx1->event_code = BASE_JD_EVENT_STOPPED; + kbase_gpu_mark_atom_for_return(kbdev, + katom_idx1); + } + } + } + + KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, + js, completion_code); + + if (job_tail != 0 && job_tail != katom->jc) { + bool was_updated = (job_tail != katom->jc); + + /* Some of the job has been executed, so we update the job chain + * address to where we should resume from */ + katom->jc = job_tail; + if (was_updated) + KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, + katom, job_tail, js); + } + + /* Only update the event code for jobs that weren't cancelled */ + if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) + katom->event_code = (base_jd_event_code)completion_code; + + kbase_device_trace_register_access(kctx, REG_WRITE, + JOB_CONTROL_REG(JOB_IRQ_CLEAR), + 1 << js); + + /* Complete the job, and start new ones + * + * Also defer remaining work onto the workqueue: + * - Re-queue Soft-stopped jobs + * - For any other jobs, queue the job back into the dependency system + * - Schedule out the parent context if necessary, and schedule a new + * one in. + */ +#ifdef CONFIG_GPU_TRACEPOINTS + { + /* The atom in the HEAD */ + struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, + 0); + + if (next_katom && next_katom->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED) { + char js_string[16]; + + trace_gpu_sched_switch(kbasep_make_job_slot_string(js, + js_string, + sizeof(js_string)), + ktime_to_ns(*end_timestamp), + (u32)next_katom->kctx->id, 0, + next_katom->work_id); + kbdev->hwaccess.backend.slot_rb[js].last_context = + next_katom->kctx; + } else { + char js_string[16]; + + trace_gpu_sched_switch(kbasep_make_job_slot_string(js, + js_string, + sizeof(js_string)), + ktime_to_ns(ktime_get()), 0, 0, + 0); + kbdev->hwaccess.backend.slot_rb[js].last_context = 0; + } + } +#endif + + if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) + kbase_reset_gpu_silent(kbdev); + + if (completion_code == BASE_JD_EVENT_STOPPED) + katom = kbase_jm_return_atom_to_js(kbdev, katom); + else + katom = kbase_jm_complete(kbdev, katom, end_timestamp); + + if (katom) { + /* Cross-slot dependency has now become runnable. Try to submit + * it. */ + + /* Check if there are lower priority jobs to soft stop */ + kbase_job_slot_ctx_priority_check_locked(kctx, katom); + + kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); + } + + /* Job completion may have unblocked other atoms. Try to update all job + * slots */ + kbase_backend_slot_update(kbdev); +} + +void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) +{ + int js; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* Reset should always take the GPU out of protected mode */ + WARN_ON(kbase_gpu_in_protected_mode(kbdev)); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + int atom_idx = 0; + int idx; + + for (idx = 0; idx < SLOT_RB_SIZE; idx++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, + js, atom_idx); + bool keep_in_jm_rb = false; + + if (!katom) + break; + if (katom->protected_state.exit == + KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) + { + KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev); + + kbase_vinstr_resume(kbdev->vinstr_ctx); + + /* protected mode sanity checks */ + KBASE_DEBUG_ASSERT_MSG( + kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev), + "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", + kbase_jd_katom_is_protected(katom), kbase_gpu_in_protected_mode(kbdev)); + KBASE_DEBUG_ASSERT_MSG( + (kbase_jd_katom_is_protected(katom) && js == 0) || + !kbase_jd_katom_is_protected(katom), + "Protected atom on JS%d not supported", js); + } + if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) + keep_in_jm_rb = true; + + kbase_gpu_release_atom(kbdev, katom, NULL); + + /* + * If the atom wasn't on HW when the reset was issued + * then leave it in the RB and next time we're kicked + * it will be processed again from the starting state. + */ + if (keep_in_jm_rb) { + kbasep_js_job_check_deref_cores(kbdev, katom); + katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; + katom->affinity = 0; + katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; + /* As the atom was not removed, increment the + * index so that we read the correct atom in the + * next iteration. */ + atom_idx++; + continue; + } + + /* + * The atom was on the HW when the reset was issued + * all we can do is fail the atom. + */ + kbase_gpu_dequeue_atom(kbdev, js, NULL); + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + kbase_jm_complete(kbdev, katom, end_timestamp); + } + } + + kbdev->protected_mode_transition = false; +} + +static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, + int js, + struct kbase_jd_atom *katom, + u32 action) +{ + u32 hw_action = action & JS_COMMAND_MASK; + + kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); + kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, + katom->core_req, katom); + katom->kctx->blocked_js[js][katom->sched_priority] = true; +} + +static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + u32 action, + bool disjoint) +{ + katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; + kbase_gpu_mark_atom_for_return(kbdev, katom); + katom->kctx->blocked_js[katom->slot_nr][katom->sched_priority] = true; + + if (disjoint) + kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, + katom); +} + +static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) +{ + if (katom->x_post_dep) { + struct kbase_jd_atom *dep_atom = katom->x_post_dep; + + if (dep_atom->gpu_rb_state != + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && + dep_atom->gpu_rb_state != + KBASE_ATOM_GPU_RB_RETURN_TO_JS) + return dep_atom->slot_nr; + } + return -1; +} + +static void kbase_job_evicted(struct kbase_jd_atom *katom) +{ + kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom, + katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT); +} + +bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js, + struct kbase_jd_atom *katom, + u32 action) +{ + struct kbase_jd_atom *katom_idx0; + struct kbase_jd_atom *katom_idx1; + + bool katom_idx0_valid, katom_idx1_valid; + + bool ret = false; + + int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; + int prio_idx0 = 0, prio_idx1 = 0; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); + katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); + + if (katom_idx0) + prio_idx0 = katom_idx0->sched_priority; + if (katom_idx1) + prio_idx1 = katom_idx1->sched_priority; + + if (katom) { + katom_idx0_valid = (katom_idx0 == katom); + /* If idx0 is to be removed and idx1 is on the same context, + * then idx1 must also be removed otherwise the atoms might be + * returned out of order */ + if (katom_idx1) + katom_idx1_valid = (katom_idx1 == katom) || + (katom_idx0_valid && + (katom_idx0->kctx == + katom_idx1->kctx)); + else + katom_idx1_valid = false; + } else { + katom_idx0_valid = (katom_idx0 && + (!kctx || katom_idx0->kctx == kctx)); + katom_idx1_valid = (katom_idx1 && + (!kctx || katom_idx1->kctx == kctx) && + prio_idx0 == prio_idx1); + } + + if (katom_idx0_valid) + stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); + if (katom_idx1_valid) + stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); + + if (katom_idx0_valid) { + if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Simple case - just dequeue and return */ + kbase_gpu_dequeue_atom(kbdev, js, NULL); + if (katom_idx1_valid) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + katom_idx1->event_code = + BASE_JD_EVENT_REMOVED_FROM_NEXT; + kbase_jm_return_atom_to_js(kbdev, katom_idx1); + katom_idx1->kctx->blocked_js[js][prio_idx1] = + true; + } + + katom_idx0->event_code = + BASE_JD_EVENT_REMOVED_FROM_NEXT; + kbase_jm_return_atom_to_js(kbdev, katom_idx0); + katom_idx0->kctx->blocked_js[js][prio_idx0] = true; + } else { + /* katom_idx0 is on GPU */ + if (katom_idx1 && katom_idx1->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* katom_idx0 and katom_idx1 are on GPU */ + + if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_COMMAND_NEXT), NULL) == 0) { + /* idx0 has already completed - stop + * idx1 if needed*/ + if (katom_idx1_valid) { + kbase_gpu_stop_atom(kbdev, js, + katom_idx1, + action); + ret = true; + } + } else { + /* idx1 is in NEXT registers - attempt + * to remove */ + kbase_reg_write(kbdev, + JOB_SLOT_REG(js, + JS_COMMAND_NEXT), + JS_COMMAND_NOP, NULL); + + if (kbase_reg_read(kbdev, + JOB_SLOT_REG(js, + JS_HEAD_NEXT_LO), NULL) + != 0 || + kbase_reg_read(kbdev, + JOB_SLOT_REG(js, + JS_HEAD_NEXT_HI), NULL) + != 0) { + /* idx1 removed successfully, + * will be handled in IRQ */ + kbase_job_evicted(katom_idx1); + kbase_gpu_remove_atom(kbdev, + katom_idx1, + action, true); + stop_x_dep_idx1 = + should_stop_x_dep_slot(katom_idx1); + + /* stop idx0 if still on GPU */ + kbase_gpu_stop_atom(kbdev, js, + katom_idx0, + action); + ret = true; + } else if (katom_idx1_valid) { + /* idx0 has already completed, + * stop idx1 if needed */ + kbase_gpu_stop_atom(kbdev, js, + katom_idx1, + action); + ret = true; + } + } + } else if (katom_idx1_valid) { + /* idx1 not on GPU but must be dequeued*/ + + /* idx1 will be handled in IRQ */ + kbase_gpu_remove_atom(kbdev, katom_idx1, action, + false); + /* stop idx0 */ + /* This will be repeated for anything removed + * from the next registers, since their normal + * flow was also interrupted, and this function + * might not enter disjoint state e.g. if we + * don't actually do a hard stop on the head + * atom */ + kbase_gpu_stop_atom(kbdev, js, katom_idx0, + action); + ret = true; + } else { + /* no atom in idx1 */ + /* just stop idx0 */ + kbase_gpu_stop_atom(kbdev, js, katom_idx0, + action); + ret = true; + } + } + } else if (katom_idx1_valid) { + if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Mark for return */ + /* idx1 will be returned once idx0 completes */ + kbase_gpu_remove_atom(kbdev, katom_idx1, action, + false); + } else { + /* idx1 is on GPU */ + if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_COMMAND_NEXT), NULL) == 0) { + /* idx0 has already completed - stop idx1 */ + kbase_gpu_stop_atom(kbdev, js, katom_idx1, + action); + ret = true; + } else { + /* idx1 is in NEXT registers - attempt to + * remove */ + kbase_reg_write(kbdev, JOB_SLOT_REG(js, + JS_COMMAND_NEXT), + JS_COMMAND_NOP, NULL); + + if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_HEAD_NEXT_LO), NULL) != 0 || + kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_HEAD_NEXT_HI), NULL) != 0) { + /* idx1 removed successfully, will be + * handled in IRQ once idx0 completes */ + kbase_job_evicted(katom_idx1); + kbase_gpu_remove_atom(kbdev, katom_idx1, + action, + false); + } else { + /* idx0 has already completed - stop + * idx1 */ + kbase_gpu_stop_atom(kbdev, js, + katom_idx1, + action); + ret = true; + } + } + } + } + + + if (stop_x_dep_idx0 != -1) + kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0, + NULL, action); + + if (stop_x_dep_idx1 != -1) + kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1, + NULL, action); + + return ret; +} + +void kbase_gpu_cacheclean(struct kbase_device *kbdev) +{ + /* Limit the number of loops to avoid a hang if the interrupt is missed + */ + u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; + + mutex_lock(&kbdev->cacheclean_lock); + + /* use GPU_COMMAND completion solution */ + /* clean & invalidate the caches */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAN_INV_CACHES, NULL); + + /* wait for cache flush to complete before continuing */ + while (--max_loops && + (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & + CLEAN_CACHES_COMPLETED) == 0) + ; + + /* clear the CLEAN_CACHES_COMPLETED irq */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, + CLEAN_CACHES_COMPLETED); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), + CLEAN_CACHES_COMPLETED, NULL); + KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state != + KBASE_INSTR_STATE_CLEANING, + "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang."); + + mutex_unlock(&kbdev->cacheclean_lock); +} + +void kbase_backend_cacheclean(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + if (katom->need_cache_flush_cores_retained) { + unsigned long flags; + + kbase_gpu_cacheclean(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_unrequest_cores(kbdev, false, + katom->need_cache_flush_cores_retained); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + katom->need_cache_flush_cores_retained = 0; + } +} + +void kbase_backend_complete_wq(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + /* + * If cache flush required due to HW workaround then perform the flush + * now + */ + kbase_backend_cacheclean(kbdev, katom); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) && + (katom->core_req & BASE_JD_REQ_FS) && + katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT && + (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) && + !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) { + dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n"); + if (kbasep_10969_workaround_clamp_coordinates(katom)) { + /* The job had a TILE_RANGE_FAULT after was soft-stopped + * Due to an HW issue we try to execute the job again. + */ + dev_dbg(kbdev->dev, + "Clamping has been executed, try to rerun the job\n" + ); + katom->event_code = BASE_JD_EVENT_STOPPED; + katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN; + } + } + + /* Clear the coreref_state now - while check_deref_cores() may not have + * been called yet, the caller will have taken a copy of this field. If + * this is not done, then if the atom is re-scheduled (following a soft + * stop) then the core reference would not be retaken. */ + katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; + katom->affinity = 0; +} + +void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, + base_jd_core_req core_req, u64 affinity, + enum kbase_atom_coreref_state coreref_state) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity, + coreref_state); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (!kbdev->pm.active_count) { + mutex_lock(&kbdev->js_data.runpool_mutex); + mutex_lock(&kbdev->pm.lock); + kbase_pm_update_active(kbdev); + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&kbdev->js_data.runpool_mutex); + } +} + +void kbase_gpu_dump_slots(struct kbase_device *kbdev) +{ + unsigned long flags; + int js; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n"); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + int idx; + + for (idx = 0; idx < SLOT_RB_SIZE; idx++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, + js, + idx); + + if (katom) + dev_info(kbdev->dev, + " js%d idx%d : katom=%p gpu_rb_state=%d\n", + js, idx, katom, katom->gpu_rb_state); + else + dev_info(kbdev->dev, " js%d idx%d : empty\n", + js, idx); + } + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + + + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.h new file mode 100755 index 000000000000..1e0e05ad3ea4 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_jm_rb.h @@ -0,0 +1,76 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend specific APIs + */ + +#ifndef _KBASE_HWACCESS_GPU_H_ +#define _KBASE_HWACCESS_GPU_H_ + +#include + +/** + * kbase_gpu_irq_evict - Evict an atom from a NEXT slot + * + * @kbdev: Device pointer + * @js: Job slot to evict from + * + * Evict the atom in the NEXT slot for the specified job slot. This function is + * called from the job complete IRQ handler when the previous job has failed. + * + * Return: true if job evicted from NEXT registers, false otherwise + */ +bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js); + +/** + * kbase_gpu_complete_hw - Complete an atom on job slot js + * + * @kbdev: Device pointer + * @js: Job slot that has completed + * @completion_code: Event code from job that has completed + * @job_tail: The tail address from the hardware if the job has partially + * completed + * @end_timestamp: Time of completion + */ +void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, + u32 completion_code, + u64 job_tail, + ktime_t *end_timestamp); + +/** + * kbase_gpu_inspect - Inspect the contents of the HW access ringbuffer + * + * @kbdev: Device pointer + * @js: Job slot to inspect + * @idx: Index into ringbuffer. 0 is the job currently running on + * the slot, 1 is the job waiting, all other values are invalid. + * Return: The atom at that position in the ringbuffer + * or NULL if no atom present + */ +struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, + int idx); + +/** + * kbase_gpu_dump_slots - Print the contents of the slot ringbuffers + * + * @kbdev: Device pointer + */ +void kbase_gpu_dump_slots(struct kbase_device *kbdev); + +#endif /* _KBASE_HWACCESS_GPU_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.c new file mode 100755 index 000000000000..54d8ddd80097 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.c @@ -0,0 +1,303 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel affinity manager APIs + */ + +#include +#include "mali_kbase_js_affinity.h" +#include "mali_kbase_hw.h" + +#include + + +bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, + int js) +{ + /* + * Here are the reasons for using job slot 2: + * - BASE_HW_ISSUE_8987 (which is entirely used for that purpose) + * - In absence of the above, then: + * - Atoms with BASE_JD_REQ_COHERENT_GROUP + * - But, only when there aren't contexts with + * KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, because the atoms that run on + * all cores on slot 1 could be blocked by those using a coherent group + * on slot 2 + * - And, only when you actually have 2 or more coregroups - if you + * only have 1 coregroup, then having jobs for slot 2 implies they'd + * also be for slot 1, meaning you'll get interference from them. Jobs + * able to run on slot 2 could also block jobs that can only run on + * slot 1 (tiler jobs) + */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) + return true; + + if (js != 2) + return true; + + /* Only deal with js==2 now: */ + if (kbdev->gpu_props.num_core_groups > 1) { + /* Only use slot 2 in the 2+ coregroup case */ + if (kbasep_js_ctx_attr_is_attr_on_runpool(kbdev, + KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) == + false) { + /* ...But only when we *don't* have atoms that run on + * all cores */ + + /* No specific check for BASE_JD_REQ_COHERENT_GROUP + * atoms - the policy will sort that out */ + return true; + } + } + + /* Above checks failed mean we shouldn't use slot 2 */ + return false; +} + +/* + * As long as it has been decided to have a deeper modification of + * what job scheduler, power manager and affinity manager will + * implement, this function is just an intermediate step that + * assumes: + * - all working cores will be powered on when this is called. + * - largest current configuration is 2 core groups. + * - It has been decided not to have hardcoded values so the low + * and high cores in a core split will be evently distributed. + * - Odd combinations of core requirements have been filtered out + * and do not get to this function (e.g. CS+T+NSS is not + * supported here). + * - This function is frequently called and can be optimized, + * (see notes in loops), but as the functionallity will likely + * be modified, optimization has not been addressed. +*/ +bool kbase_js_choose_affinity(u64 * const affinity, + struct kbase_device *kbdev, + struct kbase_jd_atom *katom, int js) +{ + base_jd_core_req core_req = katom->core_req; + unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; + u64 core_availability_mask; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + core_availability_mask = kbase_pm_ca_get_core_mask(kbdev); + + /* + * If no cores are currently available (core availability policy is + * transitioning) then fail. + */ + if (0 == core_availability_mask) { + *affinity = 0; + return false; + } + + KBASE_DEBUG_ASSERT(js >= 0); + + if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) == + BASE_JD_REQ_T) { + /* If the hardware supports XAFFINITY then we'll only enable + * the tiler (which is the default so this is a no-op), + * otherwise enable shader core 0. */ + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) + *affinity = 1; + else + *affinity = 0; + + return true; + } + + if (1 == kbdev->gpu_props.num_cores) { + /* trivial case only one core, nothing to do */ + *affinity = core_availability_mask & + kbdev->pm.debug_core_mask[js]; + } else { + if ((core_req & (BASE_JD_REQ_COHERENT_GROUP | + BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) { + if (js == 0 || num_core_groups == 1) { + /* js[0] and single-core-group systems just get + * the first core group */ + *affinity = + kbdev->gpu_props.props.coherency_info.group[0].core_mask + & core_availability_mask & + kbdev->pm.debug_core_mask[js]; + } else { + /* js[1], js[2] use core groups 0, 1 for + * dual-core-group systems */ + u32 core_group_idx = ((u32) js) - 1; + + KBASE_DEBUG_ASSERT(core_group_idx < + num_core_groups); + *affinity = + kbdev->gpu_props.props.coherency_info.group[core_group_idx].core_mask + & core_availability_mask & + kbdev->pm.debug_core_mask[js]; + + /* If the job is specifically targeting core + * group 1 and the core availability policy is + * keeping that core group off, then fail */ + if (*affinity == 0 && core_group_idx == 1 && + kbdev->pm.backend.cg1_disabled + == true) + katom->event_code = + BASE_JD_EVENT_PM_EVENT; + } + } else { + /* All cores are available when no core split is + * required */ + *affinity = core_availability_mask & + kbdev->pm.debug_core_mask[js]; + } + } + + /* + * If no cores are currently available in the desired core group(s) + * (core availability policy is transitioning) then fail. + */ + if (*affinity == 0) + return false; + + /* Enable core 0 if tiler required for hardware without XAFFINITY + * support (notes above) */ + if (core_req & BASE_JD_REQ_T) { + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) + *affinity = *affinity | 1; + } + + return true; +} + +static inline bool kbase_js_affinity_is_violating( + struct kbase_device *kbdev, + u64 *affinities) +{ + /* This implementation checks whether the two slots involved in Generic + * thread creation have intersecting affinity. This is due to micro- + * architectural issues where a job in slot A targetting cores used by + * slot B could prevent the job in slot B from making progress until the + * job in slot A has completed. + */ + u64 affinity_set_left; + u64 affinity_set_right; + u64 intersection; + + KBASE_DEBUG_ASSERT(affinities != NULL); + + affinity_set_left = affinities[1]; + + affinity_set_right = affinities[2]; + + /* A violation occurs when any bit in the left_set is also in the + * right_set */ + intersection = affinity_set_left & affinity_set_right; + + return (bool) (intersection != (u64) 0u); +} + +bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, + u64 affinity) +{ + struct kbasep_js_device_data *js_devdata; + u64 new_affinities[BASE_JM_MAX_NR_SLOTS]; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); + js_devdata = &kbdev->js_data; + + memcpy(new_affinities, js_devdata->runpool_irq.slot_affinities, + sizeof(js_devdata->runpool_irq.slot_affinities)); + + new_affinities[js] |= affinity; + + return kbase_js_affinity_is_violating(kbdev, new_affinities); +} + +void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, + u64 affinity) +{ + struct kbasep_js_device_data *js_devdata; + u64 cores; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); + js_devdata = &kbdev->js_data; + + KBASE_DEBUG_ASSERT(kbase_js_affinity_would_violate(kbdev, js, affinity) + == false); + + cores = affinity; + while (cores) { + int bitnum = fls64(cores) - 1; + u64 bit = 1ULL << bitnum; + s8 cnt; + + cnt = + ++(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]); + + if (cnt == 1) + js_devdata->runpool_irq.slot_affinities[js] |= bit; + + cores &= ~bit; + } +} + +void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, + u64 affinity) +{ + struct kbasep_js_device_data *js_devdata; + u64 cores; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); + js_devdata = &kbdev->js_data; + + cores = affinity; + while (cores) { + int bitnum = fls64(cores) - 1; + u64 bit = 1ULL << bitnum; + s8 cnt; + + KBASE_DEBUG_ASSERT( + js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum] > 0); + + cnt = + --(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]); + + if (0 == cnt) + js_devdata->runpool_irq.slot_affinities[js] &= ~bit; + + cores &= ~bit; + } +} + +#if KBASE_TRACE_ENABLE +void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata; + int slot_nr; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + js_devdata = &kbdev->js_data; + + for (slot_nr = 0; slot_nr < 3; ++slot_nr) + KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_AFFINITY_CURRENT, NULL, + NULL, 0u, slot_nr, + (u32) js_devdata->runpool_irq.slot_affinities[slot_nr]); +} +#endif /* KBASE_TRACE_ENABLE */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.h new file mode 100755 index 000000000000..35d9781ae092 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_affinity.h @@ -0,0 +1,129 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Affinity Manager internal APIs. + */ + +#ifndef _KBASE_JS_AFFINITY_H_ +#define _KBASE_JS_AFFINITY_H_ + +/** + * kbase_js_can_run_job_on_slot_no_lock - Decide whether it is possible to + * submit a job to a particular job slot in the current status + * + * @kbdev: The kbase device structure of the device + * @js: Job slot number to check for allowance + * + * Will check if submitting to the given job slot is allowed in the current + * status. For example using job slot 2 while in soft-stoppable state and only + * having 1 coregroup is not allowed by the policy. This function should be + * called prior to submitting a job to a slot to make sure policy rules are not + * violated. + * + * The following locking conditions are made on the caller + * - it must hold hwaccess_lock + */ +bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, int js); + +/** + * kbase_js_choose_affinity - Compute affinity for a given job. + * + * @affinity: Affinity bitmap computed + * @kbdev: The kbase device structure of the device + * @katom: Job chain of which affinity is going to be found + * @js: Slot the job chain is being submitted + * + * Currently assumes an all-on/all-off power management policy. + * Also assumes there is at least one core with tiler available. + * + * Returns true if a valid affinity was chosen, false if + * no cores were available. + */ +bool kbase_js_choose_affinity(u64 * const affinity, + struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + int js); + +/** + * kbase_js_affinity_would_violate - Determine whether a proposed affinity on + * job slot @js would cause a violation of affinity restrictions. + * + * @kbdev: Kbase device structure + * @js: The job slot to test + * @affinity: The affinity mask to test + * + * The following locks must be held by the caller + * - hwaccess_lock + * + * Return: true if the affinity would violate the restrictions + */ +bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, + u64 affinity); + +/** + * kbase_js_affinity_retain_slot_cores - Affinity tracking: retain cores used by + * a slot + * + * @kbdev: Kbase device structure + * @js: The job slot retaining the cores + * @affinity: The cores to retain + * + * The following locks must be held by the caller + * - hwaccess_lock + */ +void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, + u64 affinity); + +/** + * kbase_js_affinity_release_slot_cores - Affinity tracking: release cores used + * by a slot + * + * @kbdev: Kbase device structure + * @js: Job slot + * @affinity: Bit mask of core to be released + * + * Cores must be released as soon as a job is dequeued from a slot's 'submit + * slots', and before another job is submitted to those slots. Otherwise, the + * refcount could exceed the maximum number submittable to a slot, + * %BASE_JM_SUBMIT_SLOTS. + * + * The following locks must be held by the caller + * - hwaccess_lock + */ +void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, + u64 affinity); + +/** + * kbase_js_debug_log_current_affinities - log the current affinities + * + * @kbdev: Kbase device structure + * + * Output to the Trace log the current tracked affinities on all slots + */ +#if KBASE_TRACE_ENABLE +void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev); +#else /* KBASE_TRACE_ENABLE */ +static inline void +kbase_js_debug_log_current_affinities(struct kbase_device *kbdev) +{ +} +#endif /* KBASE_TRACE_ENABLE */ + +#endif /* _KBASE_JS_AFFINITY_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_backend.c new file mode 100755 index 000000000000..d392fa2a85d9 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_backend.c @@ -0,0 +1,356 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend specific job scheduler APIs + */ + +#include +#include +#include +#include + +/* + * Define for when dumping is enabled. + * This should not be based on the instrumentation level as whether dumping is + * enabled for a particular level is down to the integrator. However this is + * being used for now as otherwise the cinstr headers would be needed. + */ +#define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL) + +/* + * Hold the runpool_mutex for this + */ +static inline bool timer_callback_should_run(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + s8 nr_running_ctxs; + + lockdep_assert_held(&kbdev->js_data.runpool_mutex); + + /* Timer must stop if we are suspending */ + if (backend->suspend_timer) + return false; + + /* nr_contexts_pullable is updated with the runpool_mutex. However, the + * locking in the caller gives us a barrier that ensures + * nr_contexts_pullable is up-to-date for reading */ + nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (kbdev->js_data.softstop_always) { + /* Debug support for allowing soft-stop on a single context */ + return true; + } +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) { + /* Timeouts would have to be 4x longer (due to micro- + * architectural design) to support OpenCL conformance tests, so + * only run the timer when there's: + * - 2 or more CL contexts + * - 1 or more GLES contexts + * + * NOTE: We will treat a context that has both Compute and Non- + * Compute jobs will be treated as an OpenCL context (hence, we + * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). + */ + { + s8 nr_compute_ctxs = + kbasep_js_ctx_attr_count_on_runpool(kbdev, + KBASEP_JS_CTX_ATTR_COMPUTE); + s8 nr_noncompute_ctxs = nr_running_ctxs - + nr_compute_ctxs; + + return (bool) (nr_compute_ctxs >= 2 || + nr_noncompute_ctxs > 0); + } + } else { + /* Run the timer callback whenever you have at least 1 context + */ + return (bool) (nr_running_ctxs > 0); + } +} + +static enum hrtimer_restart timer_callback(struct hrtimer *timer) +{ + unsigned long flags; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + struct kbase_backend_data *backend; + int s; + bool reset_needed = false; + + KBASE_DEBUG_ASSERT(timer != NULL); + + backend = container_of(timer, struct kbase_backend_data, + scheduling_timer); + kbdev = container_of(backend, struct kbase_device, hwaccess.backend); + js_devdata = &kbdev->js_data; + + /* Loop through the slots */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) { + struct kbase_jd_atom *atom = NULL; + + if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) { + atom = kbase_gpu_inspect(kbdev, s, 0); + KBASE_DEBUG_ASSERT(atom != NULL); + } + + if (atom != NULL) { + /* The current version of the model doesn't support + * Soft-Stop */ + if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { + u32 ticks = atom->ticks++; + +#if !CINSTR_DUMPING_ENABLED + u32 soft_stop_ticks, hard_stop_ticks, + gpu_reset_ticks; + if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { + soft_stop_ticks = + js_devdata->soft_stop_ticks_cl; + hard_stop_ticks = + js_devdata->hard_stop_ticks_cl; + gpu_reset_ticks = + js_devdata->gpu_reset_ticks_cl; + } else { + soft_stop_ticks = + js_devdata->soft_stop_ticks; + hard_stop_ticks = + js_devdata->hard_stop_ticks_ss; + gpu_reset_ticks = + js_devdata->gpu_reset_ticks_ss; + } + + /* If timeouts have been changed then ensure + * that atom tick count is not greater than the + * new soft_stop timeout. This ensures that + * atoms do not miss any of the timeouts due to + * races between this worker and the thread + * changing the timeouts. */ + if (backend->timeouts_updated && + ticks > soft_stop_ticks) + ticks = atom->ticks = soft_stop_ticks; + + /* Job is Soft-Stoppable */ + if (ticks == soft_stop_ticks) { + int disjoint_threshold = + KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD; + u32 softstop_flags = 0u; + /* Job has been scheduled for at least + * js_devdata->soft_stop_ticks ticks. + * Soft stop the slot so we can run + * other jobs. + */ + dev_dbg(kbdev->dev, "Soft-stop"); +#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS + /* nr_user_contexts_running is updated + * with the runpool_mutex, but we can't + * take that here. + * + * However, if it's about to be + * increased then the new context can't + * run any jobs until they take the + * hwaccess_lock, so it's OK to observe + * the older value. + * + * Similarly, if it's about to be + * decreased, the last job from another + * context has already finished, so it's + * not too bad that we observe the older + * value and register a disjoint event + * when we try soft-stopping */ + if (js_devdata->nr_user_contexts_running + >= disjoint_threshold) + softstop_flags |= + JS_COMMAND_SW_CAUSES_DISJOINT; + + kbase_job_slot_softstop_swflags(kbdev, + s, atom, softstop_flags); +#endif + } else if (ticks == hard_stop_ticks) { + /* Job has been scheduled for at least + * js_devdata->hard_stop_ticks_ss ticks. + * It should have been soft-stopped by + * now. Hard stop the slot. + */ +#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS + int ms = + js_devdata->scheduling_period_ns + / 1000000u; + dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", + (unsigned long)ticks, + (unsigned long)ms); + kbase_job_slot_hardstop(atom->kctx, s, + atom); +#endif + } else if (ticks == gpu_reset_ticks) { + /* Job has been scheduled for at least + * js_devdata->gpu_reset_ticks_ss ticks. + * It should have left the GPU by now. + * Signal that the GPU needs to be + * reset. + */ + reset_needed = true; + } +#else /* !CINSTR_DUMPING_ENABLED */ + /* NOTE: During CINSTR_DUMPING_ENABLED, we use + * the alternate timeouts, which makes the hard- + * stop and GPU reset timeout much longer. We + * also ensure that we don't soft-stop at all. + */ + if (ticks == js_devdata->soft_stop_ticks) { + /* Job has been scheduled for at least + * js_devdata->soft_stop_ticks. We do + * not soft-stop during + * CINSTR_DUMPING_ENABLED, however. + */ + dev_dbg(kbdev->dev, "Soft-stop"); + } else if (ticks == + js_devdata->hard_stop_ticks_dumping) { + /* Job has been scheduled for at least + * js_devdata->hard_stop_ticks_dumping + * ticks. Hard stop the slot. + */ +#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS + int ms = + js_devdata->scheduling_period_ns + / 1000000u; + dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", + (unsigned long)ticks, + (unsigned long)ms); + kbase_job_slot_hardstop(atom->kctx, s, + atom); +#endif + } else if (ticks == + js_devdata->gpu_reset_ticks_dumping) { + /* Job has been scheduled for at least + * js_devdata->gpu_reset_ticks_dumping + * ticks. It should have left the GPU by + * now. Signal that the GPU needs to be + * reset. + */ + reset_needed = true; + } +#endif /* !CINSTR_DUMPING_ENABLED */ + } + } + } +#if KBASE_GPU_RESET_EN + if (reset_needed) { + dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve."); + + if (kbase_prepare_to_reset_gpu_locked(kbdev)) + kbase_reset_gpu_locked(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + /* the timer is re-issued if there is contexts in the run-pool */ + + if (backend->timer_running) + hrtimer_start(&backend->scheduling_timer, + HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), + HRTIMER_MODE_REL); + + backend->timeouts_updated = false; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return HRTIMER_NORESTART; +} + +void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + unsigned long flags; + + lockdep_assert_held(&js_devdata->runpool_mutex); + + if (!timer_callback_should_run(kbdev)) { + /* Take spinlock to force synchronisation with timer */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + backend->timer_running = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + /* From now on, return value of timer_callback_should_run() will + * also cause the timer to not requeue itself. Its return value + * cannot change, because it depends on variables updated with + * the runpool_mutex held, which the caller of this must also + * hold */ + hrtimer_cancel(&backend->scheduling_timer); + } + + if (timer_callback_should_run(kbdev) && !backend->timer_running) { + /* Take spinlock to force synchronisation with timer */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + backend->timer_running = true; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + hrtimer_start(&backend->scheduling_timer, + HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), + HRTIMER_MODE_REL); + + KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, + 0u); + } +} + +int kbase_backend_timer_init(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + backend->scheduling_timer.function = timer_callback; + + backend->timer_running = false; + + return 0; +} + +void kbase_backend_timer_term(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + hrtimer_cancel(&backend->scheduling_timer); +} + +void kbase_backend_timer_suspend(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + backend->suspend_timer = true; + + kbase_backend_ctx_count_changed(kbdev); +} + +void kbase_backend_timer_resume(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + backend->suspend_timer = false; + + kbase_backend_ctx_count_changed(kbdev); +} + +void kbase_backend_timeouts_changed(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + backend->timeouts_updated = true; +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_internal.h new file mode 100755 index 000000000000..3f53779c6747 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_js_internal.h @@ -0,0 +1,69 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend specific job scheduler APIs + */ + +#ifndef _KBASE_JS_BACKEND_H_ +#define _KBASE_JS_BACKEND_H_ + +/** + * kbase_backend_timer_init() - Initialise the JS scheduling timer + * @kbdev: Device pointer + * + * This function should be called at driver initialisation + * + * Return: 0 on success + */ +int kbase_backend_timer_init(struct kbase_device *kbdev); + +/** + * kbase_backend_timer_term() - Terminate the JS scheduling timer + * @kbdev: Device pointer + * + * This function should be called at driver termination + */ +void kbase_backend_timer_term(struct kbase_device *kbdev); + +/** + * kbase_backend_timer_suspend - Suspend is happening, stop the JS scheduling + * timer + * @kbdev: Device pointer + * + * This function should be called on suspend, after the active count has reached + * zero. This is required as the timer may have been started on job submission + * to the job scheduler, but before jobs are submitted to the GPU. + * + * Caller must hold runpool_mutex. + */ +void kbase_backend_timer_suspend(struct kbase_device *kbdev); + +/** + * kbase_backend_timer_resume - Resume is happening, re-evaluate the JS + * scheduling timer + * @kbdev: Device pointer + * + * This function should be called on resume. Note that is is not guaranteed to + * re-start the timer, only evalute whether it should be re-started. + * + * Caller must hold runpool_mutex. + */ +void kbase_backend_timer_resume(struct kbase_device *kbdev); + +#endif /* _KBASE_JS_BACKEND_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.c new file mode 100755 index 000000000000..aa1817c8bca9 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.c @@ -0,0 +1,401 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include + +#include +#include +#include +#include +#include +#include + +static inline u64 lock_region(struct kbase_device *kbdev, u64 pfn, + u32 num_pages) +{ + u64 region; + + /* can't lock a zero sized range */ + KBASE_DEBUG_ASSERT(num_pages); + + region = pfn << PAGE_SHIFT; + /* + * fls returns (given the ASSERT above): + * 1 .. 32 + * + * 10 + fls(num_pages) + * results in the range (11 .. 42) + */ + + /* gracefully handle num_pages being zero */ + if (0 == num_pages) { + region |= 11; + } else { + u8 region_width; + + region_width = 10 + fls(num_pages); + if (num_pages != (1ul << (region_width - 11))) { + /* not pow2, so must go up to the next pow2 */ + region_width += 1; + } + KBASE_DEBUG_ASSERT(region_width <= KBASE_LOCK_REGION_MAX_SIZE); + KBASE_DEBUG_ASSERT(region_width >= KBASE_LOCK_REGION_MIN_SIZE); + region |= region_width; + } + + return region; +} + +static int wait_ready(struct kbase_device *kbdev, + unsigned int as_nr, struct kbase_context *kctx) +{ + unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; + u32 val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), kctx); + + /* Wait for the MMU status to indicate there is no active command, in + * case one is pending. Do not log remaining register accesses. */ + while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) + val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), NULL); + + if (max_loops == 0) { + dev_err(kbdev->dev, "AS_ACTIVE bit stuck\n"); + return -1; + } + + /* If waiting in loop was performed, log last read value. */ + if (KBASE_AS_INACTIVE_MAX_LOOPS - 1 > max_loops) + kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), kctx); + + return 0; +} + +static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd, + struct kbase_context *kctx) +{ + int status; + + /* write AS_COMMAND when MMU is ready to accept another command */ + status = wait_ready(kbdev, as_nr, kctx); + if (status == 0) + kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd, + kctx); + + return status; +} + +static void validate_protected_page_fault(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + /* GPUs which support (native) protected mode shall not report page + * fault addresses unless it has protected debug mode and protected + * debug mode is turned on */ + u32 protected_debug_mode = 0; + + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) + return; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { + protected_debug_mode = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_STATUS), + kctx) & GPU_DBGEN; + } + + if (!protected_debug_mode) { + /* fault_addr should never be reported in protected mode. + * However, we just continue by printing an error message */ + dev_err(kbdev->dev, "Fault address reported in protected mode\n"); + } +} + +void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) +{ + const int num_as = 16; + const int busfault_shift = MMU_PAGE_FAULT_FLAGS; + const int pf_shift = 0; + const unsigned long as_bit_mask = (1UL << num_as) - 1; + unsigned long flags; + u32 new_mask; + u32 tmp; + + /* bus faults */ + u32 bf_bits = (irq_stat >> busfault_shift) & as_bit_mask; + /* page faults (note: Ignore ASes with both pf and bf) */ + u32 pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + + /* remember current mask */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL); + /* mask interrupts for now */ + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); + + while (bf_bits | pf_bits) { + struct kbase_as *as; + int as_no; + struct kbase_context *kctx; + + /* + * the while logic ensures we have a bit set, no need to check + * for not-found here + */ + as_no = ffs(bf_bits | pf_bits) - 1; + as = &kbdev->as[as_no]; + + /* + * Refcount the kctx ASAP - it shouldn't disappear anyway, since + * Bus/Page faults _should_ only occur whilst jobs are running, + * and a job causing the Bus/Page fault shouldn't complete until + * the MMU is updated + */ + kctx = kbasep_js_runpool_lookup_ctx(kbdev, as_no); + + + /* find faulting address */ + as->fault_addr = kbase_reg_read(kbdev, + MMU_AS_REG(as_no, + AS_FAULTADDRESS_HI), + kctx); + as->fault_addr <<= 32; + as->fault_addr |= kbase_reg_read(kbdev, + MMU_AS_REG(as_no, + AS_FAULTADDRESS_LO), + kctx); + + /* Mark the fault protected or not */ + as->protected_mode = kbdev->protected_mode; + + if (kbdev->protected_mode && as->fault_addr) + { + /* check if address reporting is allowed */ + validate_protected_page_fault(kbdev, kctx); + } + + /* report the fault to debugfs */ + kbase_as_fault_debugfs_new(kbdev, as_no); + + /* record the fault status */ + as->fault_status = kbase_reg_read(kbdev, + MMU_AS_REG(as_no, + AS_FAULTSTATUS), + kctx); + + /* find the fault type */ + as->fault_type = (bf_bits & (1 << as_no)) ? + KBASE_MMU_FAULT_TYPE_BUS : + KBASE_MMU_FAULT_TYPE_PAGE; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { + as->fault_extra_addr = kbase_reg_read(kbdev, + MMU_AS_REG(as_no, AS_FAULTEXTRA_HI), + kctx); + as->fault_extra_addr <<= 32; + as->fault_extra_addr |= kbase_reg_read(kbdev, + MMU_AS_REG(as_no, AS_FAULTEXTRA_LO), + kctx); + } + + if (kbase_as_has_bus_fault(as)) { + /* Mark bus fault as handled. + * Note that a bus fault is processed first in case + * where both a bus fault and page fault occur. + */ + bf_bits &= ~(1UL << as_no); + + /* remove the queued BF (and PF) from the mask */ + new_mask &= ~(MMU_BUS_ERROR(as_no) | + MMU_PAGE_FAULT(as_no)); + } else { + /* Mark page fault as handled */ + pf_bits &= ~(1UL << as_no); + + /* remove the queued PF from the mask */ + new_mask &= ~MMU_PAGE_FAULT(as_no); + } + + /* Process the interrupt for this address space */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_interrupt_process(kbdev, kctx, as); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + /* reenable interrupts */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL); + new_mask |= tmp; + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask, NULL); + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); +} + +void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx) +{ + struct kbase_mmu_setup *current_setup = &as->current_setup; + u32 transcfg = 0; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { + transcfg = current_setup->transcfg & 0xFFFFFFFFUL; + + /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK */ + /* Clear PTW_MEMATTR bits */ + transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; + /* Enable correct PTW_MEMATTR bits */ + transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; + + if (kbdev->system_coherency == COHERENCY_ACE) { + /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) */ + /* Clear PTW_SH bits */ + transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); + /* Enable correct PTW_SH bits */ + transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); + } + + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), + transcfg, kctx); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), + (current_setup->transcfg >> 32) & 0xFFFFFFFFUL, + kctx); + } else { + if (kbdev->system_coherency == COHERENCY_ACE) + current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER; + } + + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO), + current_setup->transtab & 0xFFFFFFFFUL, kctx); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI), + (current_setup->transtab >> 32) & 0xFFFFFFFFUL, kctx); + + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO), + current_setup->memattr & 0xFFFFFFFFUL, kctx); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI), + (current_setup->memattr >> 32) & 0xFFFFFFFFUL, kctx); + + KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(as, + current_setup->transtab, + current_setup->memattr, + transcfg); + + write_cmd(kbdev, as->number, AS_COMMAND_UPDATE, kctx); +} + +int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, u64 vpfn, u32 nr, u32 op, + unsigned int handling_irq) +{ + int ret; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + + if (op == AS_COMMAND_UNLOCK) { + /* Unlock doesn't require a lock first */ + ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); + } else { + u64 lock_addr = lock_region(kbdev, vpfn, nr); + + /* Lock the region that needs to be updated */ + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_LO), + lock_addr & 0xFFFFFFFFUL, kctx); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_HI), + (lock_addr >> 32) & 0xFFFFFFFFUL, kctx); + write_cmd(kbdev, as->number, AS_COMMAND_LOCK, kctx); + + /* Run the MMU operation */ + write_cmd(kbdev, as->number, op, kctx); + + /* Wait for the flush to complete */ + ret = wait_ready(kbdev, as->number, kctx); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9630)) { + /* Issue an UNLOCK command to ensure that valid page + tables are re-read by the GPU after an update. + Note that, the FLUSH command should perform all the + actions necessary, however the bus logs show that if + multiple page faults occur within an 8 page region + the MMU does not always re-read the updated page + table entries for later faults or is only partially + read, it subsequently raises the page fault IRQ for + the same addresses, the unlock ensures that the MMU + cache is flushed, so updates can be re-read. As the + region is now unlocked we need to issue 2 UNLOCK + commands in order to flush the MMU/uTLB, + see PRLAM-8812. + */ + write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); + write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); + } + } + + return ret; +} + +void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, enum kbase_mmu_fault_type type) +{ + unsigned long flags; + u32 pf_bf_mask; + + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + + /* + * A reset is in-flight and we're flushing the IRQ + bottom half + * so don't update anything as it could race with the reset code. + */ + if (kbdev->irq_reset_flush) + goto unlock; + + /* Clear the page (and bus fault IRQ as well in case one occurred) */ + pf_bf_mask = MMU_PAGE_FAULT(as->number); + if (type == KBASE_MMU_FAULT_TYPE_BUS || + type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) + pf_bf_mask |= MMU_BUS_ERROR(as->number); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask, kctx); + +unlock: + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); +} + +void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, enum kbase_mmu_fault_type type) +{ + unsigned long flags; + u32 irq_mask; + + /* Enable the page fault IRQ (and bus fault IRQ as well in case one + * occurred) */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + + /* + * A reset is in-flight and we're flushing the IRQ + bottom half + * so don't update anything as it could race with the reset code. + */ + if (kbdev->irq_reset_flush) + goto unlock; + + irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), kctx) | + MMU_PAGE_FAULT(as->number); + + if (type == KBASE_MMU_FAULT_TYPE_BUS || + type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) + irq_mask |= MMU_BUS_ERROR(as->number); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask, kctx); + +unlock: + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.h new file mode 100755 index 000000000000..c02253c6acc3 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_mmu_hw_direct.h @@ -0,0 +1,42 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Interface file for the direct implementation for MMU hardware access + * + * Direct MMU hardware interface + * + * This module provides the interface(s) that are required by the direct + * register access implementation of the MMU hardware interface + */ + +#ifndef _MALI_KBASE_MMU_HW_DIRECT_H_ +#define _MALI_KBASE_MMU_HW_DIRECT_H_ + +#include + +/** + * kbase_mmu_interrupt - Process an MMU interrupt. + * + * Process the MMU interrupt that was reported by the &kbase_device. + * + * @kbdev: kbase context to clear the fault from. + * @irq_stat: Value of the MMU_IRQ_STATUS register + */ +void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); + +#endif /* _MALI_KBASE_MMU_HW_DIRECT_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.c new file mode 100755 index 000000000000..0614348e935a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.c @@ -0,0 +1,63 @@ +/* + * + * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * "Always on" power management policy + */ + +#include +#include + +static u64 always_on_get_core_mask(struct kbase_device *kbdev) +{ + return kbdev->gpu_props.props.raw_props.shader_present; +} + +static bool always_on_get_core_active(struct kbase_device *kbdev) +{ + return true; +} + +static void always_on_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static void always_on_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +/* + * The struct kbase_pm_policy structure for the demand power policy. + * + * This is the static structure that defines the demand power policy's callback + * and name. + */ +const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { + "always_on", /* name */ + always_on_init, /* init */ + always_on_term, /* term */ + always_on_get_core_mask, /* get_core_mask */ + always_on_get_core_active, /* get_core_active */ + 0u, /* flags */ + KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ +}; + +KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.h new file mode 100755 index 000000000000..f9d244b01bc2 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_always_on.h @@ -0,0 +1,77 @@ + +/* + * + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * "Always on" power management policy + */ + +#ifndef MALI_KBASE_PM_ALWAYS_ON_H +#define MALI_KBASE_PM_ALWAYS_ON_H + +/** + * DOC: + * The "Always on" power management policy has the following + * characteristics: + * + * - When KBase indicates that the GPU will be powered up, but we don't yet + * know which Job Chains are to be run: + * All Shader Cores are powered up, regardless of whether or not they will + * be needed later. + * + * - When KBase indicates that a set of Shader Cores are needed to submit the + * currently queued Job Chains: + * All Shader Cores are kept powered, regardless of whether or not they will + * be needed + * + * - When KBase indicates that the GPU need not be powered: + * The Shader Cores are kept powered, regardless of whether or not they will + * be needed. The GPU itself is also kept powered, even though it is not + * needed. + * + * This policy is automatically overridden during system suspend: the desired + * core state is ignored, and the cores are forced off regardless of what the + * policy requests. After resuming from suspend, new changes to the desired + * core state made by the policy are honored. + * + * Note: + * + * - KBase indicates the GPU will be powered up when it has a User Process that + * has just started to submit Job Chains. + * + * - KBase indicates the GPU need not be powered when all the Job Chains from + * User Processes have finished, and it is waiting for a User Process to + * submit some more Job Chains. + */ + +/** + * struct kbasep_pm_policy_always_on - Private struct for policy instance data + * @dummy: unused dummy variable + * + * This contains data that is private to the particular power policy that is + * active. + */ +struct kbasep_pm_policy_always_on { + int dummy; +}; + +extern const struct kbase_pm_policy kbase_pm_always_on_policy_ops; + +#endif /* MALI_KBASE_PM_ALWAYS_ON_H */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_backend.c new file mode 100755 index 000000000000..cd8932650ed5 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_backend.c @@ -0,0 +1,478 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * GPU backend implementation of base kernel power management APIs + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data); + +void kbase_pm_register_access_enable(struct kbase_device *kbdev) +{ + struct kbase_pm_callback_conf *callbacks; + + callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; + + if (callbacks) + callbacks->power_on_callback(kbdev); + + kbdev->pm.backend.gpu_powered = true; +} + +void kbase_pm_register_access_disable(struct kbase_device *kbdev) +{ + struct kbase_pm_callback_conf *callbacks; + + callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; + + if (callbacks) + callbacks->power_off_callback(kbdev); + + kbdev->pm.backend.gpu_powered = false; +} + +int kbase_hwaccess_pm_init(struct kbase_device *kbdev) +{ + int ret = 0; + struct kbase_pm_callback_conf *callbacks; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + mutex_init(&kbdev->pm.lock); + + kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait", + WQ_HIGHPRI | WQ_UNBOUND, 1); + if (!kbdev->pm.backend.gpu_poweroff_wait_wq) + return -ENOMEM; + + INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, + kbase_pm_gpu_poweroff_wait_wq); + + kbdev->pm.backend.gpu_powered = false; + kbdev->pm.suspending = false; +#ifdef CONFIG_MALI_BIFROST_DEBUG + kbdev->pm.backend.driver_ready_for_irqs = false; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + kbdev->pm.backend.gpu_in_desired_state = true; + init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); + + callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; + if (callbacks) { + kbdev->pm.backend.callback_power_on = + callbacks->power_on_callback; + kbdev->pm.backend.callback_power_off = + callbacks->power_off_callback; + kbdev->pm.backend.callback_power_suspend = + callbacks->power_suspend_callback; + kbdev->pm.backend.callback_power_resume = + callbacks->power_resume_callback; + kbdev->pm.callback_power_runtime_init = + callbacks->power_runtime_init_callback; + kbdev->pm.callback_power_runtime_term = + callbacks->power_runtime_term_callback; + kbdev->pm.backend.callback_power_runtime_on = + callbacks->power_runtime_on_callback; + kbdev->pm.backend.callback_power_runtime_off = + callbacks->power_runtime_off_callback; + kbdev->pm.backend.callback_power_runtime_idle = + callbacks->power_runtime_idle_callback; + } else { + kbdev->pm.backend.callback_power_on = NULL; + kbdev->pm.backend.callback_power_off = NULL; + kbdev->pm.backend.callback_power_suspend = NULL; + kbdev->pm.backend.callback_power_resume = NULL; + kbdev->pm.callback_power_runtime_init = NULL; + kbdev->pm.callback_power_runtime_term = NULL; + kbdev->pm.backend.callback_power_runtime_on = NULL; + kbdev->pm.backend.callback_power_runtime_off = NULL; + kbdev->pm.backend.callback_power_runtime_idle = NULL; + } + + /* Initialise the metrics subsystem */ + ret = kbasep_pm_metrics_init(kbdev); + if (ret) + return ret; + + init_waitqueue_head(&kbdev->pm.backend.l2_powered_wait); + kbdev->pm.backend.l2_powered = 0; + + init_waitqueue_head(&kbdev->pm.backend.reset_done_wait); + kbdev->pm.backend.reset_done = false; + + init_waitqueue_head(&kbdev->pm.zero_active_count_wait); + kbdev->pm.active_count = 0; + + spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock); + spin_lock_init(&kbdev->pm.backend.gpu_powered_lock); + + init_waitqueue_head(&kbdev->pm.backend.poweroff_wait); + + if (kbase_pm_ca_init(kbdev) != 0) + goto workq_fail; + + if (kbase_pm_policy_init(kbdev) != 0) + goto pm_policy_fail; + + return 0; + +pm_policy_fail: + kbase_pm_ca_term(kbdev); +workq_fail: + kbasep_pm_metrics_term(kbdev); + return -EINVAL; +} + +void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) +{ + lockdep_assert_held(&kbdev->pm.lock); + + /* Turn clocks and interrupts on - no-op if we haven't done a previous + * kbase_pm_clock_off() */ + kbase_pm_clock_on(kbdev, is_resume); + + /* Update core status as required by the policy */ + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START); + kbase_pm_update_cores_state(kbdev); + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END); + + /* NOTE: We don't wait to reach the desired state, since running atoms + * will wait for that state to be reached anyway */ +} + +static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) +{ + struct kbase_device *kbdev = container_of(data, struct kbase_device, + pm.backend.gpu_poweroff_wait_work); + struct kbase_pm_device_data *pm = &kbdev->pm; + struct kbase_pm_backend_data *backend = &pm->backend; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + unsigned long flags; + +#if !PLATFORM_POWER_DOWN_ONLY + /* Wait for power transitions to complete. We do this with no locks held + * so that we don't deadlock with any pending workqueues */ + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START); + kbase_pm_check_transitions_sync(kbdev); + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END); +#endif /* !PLATFORM_POWER_DOWN_ONLY */ + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + +#if PLATFORM_POWER_DOWN_ONLY + if (kbdev->pm.backend.gpu_powered) { + if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2)) { + /* If L2 cache is powered then we must flush it before + * we power off the GPU. Normally this would have been + * handled when the L2 was powered off. */ + kbase_gpu_cacheclean(kbdev); + } + } +#endif /* PLATFORM_POWER_DOWN_ONLY */ + + if (!backend->poweron_required) { +#if !PLATFORM_POWER_DOWN_ONLY + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + WARN_ON(kbdev->l2_available_bitmap || + kbdev->shader_available_bitmap || + kbdev->tiler_available_bitmap); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#endif /* !PLATFORM_POWER_DOWN_ONLY */ + + /* Consume any change-state events */ + kbase_timeline_pm_check_handle_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + + /* Disable interrupts and turn the clock off */ + if (!kbase_pm_clock_off(kbdev, backend->poweroff_is_suspend)) { + /* + * Page/bus faults are pending, must drop locks to + * process. Interrupts are disabled so no more faults + * should be generated at this point. + */ + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + kbase_flush_mmu_wqs(kbdev); + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + /* Turn off clock now that fault have been handled. We + * dropped locks so poweron_required may have changed - + * power back on if this is the case.*/ + if (backend->poweron_required) + kbase_pm_clock_on(kbdev, false); + else + WARN_ON(!kbase_pm_clock_off(kbdev, + backend->poweroff_is_suspend)); + } + } + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + backend->poweroff_wait_in_progress = false; + if (backend->poweron_required) { + backend->poweron_required = false; + kbase_pm_update_cores_state_nolock(kbdev); + kbase_backend_slot_update(kbdev); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + + wake_up(&kbdev->pm.backend.poweroff_wait); +} + +void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend) +{ + unsigned long flags; + + lockdep_assert_held(&kbdev->pm.lock); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (!kbdev->pm.backend.poweroff_wait_in_progress) { + /* Force all cores off */ + kbdev->pm.backend.desired_shader_state = 0; + kbdev->pm.backend.desired_tiler_state = 0; + + /* Force all cores to be unavailable, in the situation where + * transitions are in progress for some cores but not others, + * and kbase_pm_check_transitions_nolock can not immediately + * power off the cores */ + kbdev->shader_available_bitmap = 0; + kbdev->tiler_available_bitmap = 0; + kbdev->l2_available_bitmap = 0; + + kbdev->pm.backend.poweroff_wait_in_progress = true; + kbdev->pm.backend.poweroff_is_suspend = is_suspend; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + /*Kick off wq here. Callers will have to wait*/ + queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq, + &kbdev->pm.backend.gpu_poweroff_wait_work); + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } +} + +static bool is_poweroff_in_progress(struct kbase_device *kbdev) +{ + bool ret; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + ret = (kbdev->pm.backend.poweroff_wait_in_progress == false); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return ret; +} + +void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev) +{ + wait_event_killable(kbdev->pm.backend.poweroff_wait, + is_poweroff_in_progress(kbdev)); +} + +int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, + unsigned int flags) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + unsigned long irq_flags; + int ret; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + /* A suspend won't happen during startup/insmod */ + KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); + + /* Power up the GPU, don't enable IRQs as we are not ready to receive + * them. */ + ret = kbase_pm_init_hw(kbdev, flags); + if (ret) { + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + return ret; + } + + kbasep_pm_init_core_use_bitmaps(kbdev); + + kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] = + kbdev->pm.debug_core_mask[1] = + kbdev->pm.debug_core_mask[2] = + kbdev->gpu_props.props.raw_props.shader_present; + + /* Pretend the GPU is active to prevent a power policy turning the GPU + * cores off */ + kbdev->pm.active_count = 1; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + /* Ensure cycle counter is off */ + kbdev->pm.backend.gpu_cycle_counter_requests = 0; + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + + /* We are ready to receive IRQ's now as power policy is set up, so + * enable them now. */ +#ifdef CONFIG_MALI_BIFROST_DEBUG + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, irq_flags); + kbdev->pm.backend.driver_ready_for_irqs = true; + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, irq_flags); +#endif + kbase_pm_enable_interrupts(kbdev); + + /* Turn on the GPU and any cores needed by the policy */ + kbase_pm_do_poweron(kbdev, false); + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + + /* Idle the GPU and/or cores, if the policy wants it to */ + kbase_pm_context_idle(kbdev); + + return 0; +} + +void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + mutex_lock(&kbdev->pm.lock); + kbase_pm_cancel_deferred_poweroff(kbdev); + kbase_pm_do_poweroff(kbdev, false); + mutex_unlock(&kbdev->pm.lock); +} + +KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); + +void kbase_hwaccess_pm_term(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0); + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0); + + /* Free any resources the policy allocated */ + kbase_pm_policy_term(kbdev); + kbase_pm_ca_term(kbdev); + + /* Shut down the metrics subsystem */ + kbasep_pm_metrics_term(kbdev); + + destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); +} + +void kbase_pm_power_changed(struct kbase_device *kbdev) +{ + bool cores_are_available; + unsigned long flags; + + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + cores_are_available = kbase_pm_check_transitions_nolock(kbdev); + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END); + + if (cores_are_available) { + /* Log timelining information that a change in state has + * completed */ + kbase_timeline_pm_handle_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + + kbase_backend_slot_update(kbdev); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, + u64 new_core_mask_js0, u64 new_core_mask_js1, + u64 new_core_mask_js2) +{ + kbdev->pm.debug_core_mask[0] = new_core_mask_js0; + kbdev->pm.debug_core_mask[1] = new_core_mask_js1; + kbdev->pm.debug_core_mask[2] = new_core_mask_js2; + kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 | + new_core_mask_js2; + + kbase_pm_update_cores_state_nolock(kbdev); +} + +void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) +{ + kbase_pm_update_active(kbdev); +} + +void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev) +{ + kbase_pm_update_active(kbdev); +} + +void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + /* Force power off the GPU and all cores (regardless of policy), only + * after the PM active count reaches zero (otherwise, we risk turning it + * off prematurely) */ + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + kbase_pm_cancel_deferred_poweroff(kbdev); + kbase_pm_do_poweroff(kbdev, true); + + kbase_backend_timer_suspend(kbdev); + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + + kbase_pm_wait_for_poweroff_complete(kbdev); +} + +void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + kbdev->pm.suspending = false; + kbase_pm_do_poweron(kbdev, true); + + kbase_backend_timer_resume(kbdev); + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.c new file mode 100755 index 000000000000..c17db8be8877 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.c @@ -0,0 +1,182 @@ +/* + * + * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Base kernel core availability APIs + */ + +#include +#include +#include + +static const struct kbase_pm_ca_policy *const policy_list[] = { + &kbase_pm_ca_fixed_policy_ops, +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + &kbase_pm_ca_devfreq_policy_ops, +#endif +#if !MALI_CUSTOMER_RELEASE + &kbase_pm_ca_random_policy_ops +#endif +}; + +/** + * POLICY_COUNT - The number of policies available in the system. + * + * This is derived from the number of functions listed in policy_list. + */ +#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list)) + +int kbase_pm_ca_init(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + kbdev->pm.backend.ca_current_policy = policy_list[0]; + + kbdev->pm.backend.ca_current_policy->init(kbdev); + + return 0; +} + +void kbase_pm_ca_term(struct kbase_device *kbdev) +{ + kbdev->pm.backend.ca_current_policy->term(kbdev); +} + +int kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **list) +{ + if (!list) + return POLICY_COUNT; + + *list = policy_list; + + return POLICY_COUNT; +} + +KBASE_EXPORT_TEST_API(kbase_pm_ca_list_policies); + +const struct kbase_pm_ca_policy +*kbase_pm_ca_get_policy(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + return kbdev->pm.backend.ca_current_policy; +} + +KBASE_EXPORT_TEST_API(kbase_pm_ca_get_policy); + +void kbase_pm_ca_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_ca_policy *new_policy) +{ + const struct kbase_pm_ca_policy *old_policy; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(new_policy != NULL); + + KBASE_TRACE_ADD(kbdev, PM_CA_SET_POLICY, NULL, NULL, 0u, + new_policy->id); + + /* During a policy change we pretend the GPU is active */ + /* A suspend won't happen here, because we're in a syscall from a + * userspace thread */ + kbase_pm_context_active(kbdev); + + mutex_lock(&kbdev->pm.lock); + + /* Remove the policy to prevent IRQ handlers from working on it */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + old_policy = kbdev->pm.backend.ca_current_policy; + kbdev->pm.backend.ca_current_policy = NULL; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (old_policy->term) + old_policy->term(kbdev); + + if (new_policy->init) + new_policy->init(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.ca_current_policy = new_policy; + + /* If any core power state changes were previously attempted, but + * couldn't be made because the policy was changing (current_policy was + * NULL), then re-try them here. */ + kbase_pm_update_cores_state_nolock(kbdev); + + kbdev->pm.backend.ca_current_policy->update_core_status(kbdev, + kbdev->shader_ready_bitmap, + kbdev->shader_transitioning_bitmap); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->pm.lock); + + /* Now the policy change is finished, we release our fake context active + * reference */ + kbase_pm_context_idle(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_ca_set_policy); + +u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* All cores must be enabled when instrumentation is in use */ + if (kbdev->pm.backend.instr_enabled) + return kbdev->gpu_props.props.raw_props.shader_present & + kbdev->pm.debug_core_mask_all; + + if (kbdev->pm.backend.ca_current_policy == NULL) + return kbdev->gpu_props.props.raw_props.shader_present & + kbdev->pm.debug_core_mask_all; + + return kbdev->pm.backend.ca_current_policy->get_core_mask(kbdev) & + kbdev->pm.debug_core_mask_all; +} + +KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask); + +void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, + u64 cores_transitioning) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->pm.backend.ca_current_policy != NULL) + kbdev->pm.backend.ca_current_policy->update_core_status(kbdev, + cores_ready, + cores_transitioning); +} + +void kbase_pm_ca_instr_enable(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.instr_enabled = true; + + kbase_pm_update_cores_state_nolock(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +void kbase_pm_ca_instr_disable(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + kbdev->pm.backend.instr_enabled = false; + + kbase_pm_update_cores_state_nolock(kbdev); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.h new file mode 100755 index 000000000000..ee9e751f2d79 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca.h @@ -0,0 +1,92 @@ +/* + * + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Base kernel core availability APIs + */ + +#ifndef _KBASE_PM_CA_H_ +#define _KBASE_PM_CA_H_ + +/** + * kbase_pm_ca_init - Initialize core availability framework + * + * Must be called before calling any other core availability function + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 if the core availability framework was successfully initialized, + * -errno otherwise + */ +int kbase_pm_ca_init(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_term - Terminate core availability framework + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_ca_term(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_get_core_mask - Get currently available shaders core mask + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Returns a mask of the currently available shader cores. + * Calls into the core availability policy + * + * Return: The bit mask of available cores + */ +u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_update_core_status - Update core status + * + * @kbdev: The kbase device structure for the device (must be + * a valid pointer) + * @cores_ready: The bit mask of cores ready for job submission + * @cores_transitioning: The bit mask of cores that are transitioning power + * state + * + * Update core availability policy with current core power status + * + * Calls into the core availability policy + */ +void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, + u64 cores_transitioning); + +/** + * kbase_pm_ca_instr_enable - Enable override for instrumentation + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This overrides the output of the core availability policy, ensuring that all + * cores are available + */ +void kbase_pm_ca_instr_enable(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_instr_disable - Disable override for instrumentation + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This disables any previously enabled override, and resumes normal policy + * functionality + */ +void kbase_pm_ca_instr_disable(struct kbase_device *kbdev); + +#endif /* _KBASE_PM_CA_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.c new file mode 100755 index 000000000000..66bf660cffb6 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.c @@ -0,0 +1,129 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * A core availability policy implementing core mask selection from devfreq OPPs + * + */ + +#include +#include +#include +#include + +void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) +{ + struct kbasep_pm_ca_policy_devfreq *data = + &kbdev->pm.backend.ca_policy_data.devfreq; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + data->cores_desired = core_mask; + + /* Disable any cores that are now unwanted */ + data->cores_enabled &= data->cores_desired; + + kbdev->pm.backend.ca_in_transition = true; + + /* If there are no cores to be powered off then power on desired cores + */ + if (!(data->cores_used & ~data->cores_desired)) { + data->cores_enabled = data->cores_desired; + kbdev->pm.backend.ca_in_transition = false; + } + + kbase_pm_update_cores_state_nolock(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX %llX\n", + data->cores_desired, data->cores_enabled); +} + +static void devfreq_init(struct kbase_device *kbdev) +{ + struct kbasep_pm_ca_policy_devfreq *data = + &kbdev->pm.backend.ca_policy_data.devfreq; + + if (kbdev->current_core_mask) { + data->cores_enabled = kbdev->current_core_mask; + data->cores_desired = kbdev->current_core_mask; + } else { + data->cores_enabled = + kbdev->gpu_props.props.raw_props.shader_present; + data->cores_desired = + kbdev->gpu_props.props.raw_props.shader_present; + } + data->cores_used = 0; + kbdev->pm.backend.ca_in_transition = false; +} + +static void devfreq_term(struct kbase_device *kbdev) +{ +} + +static u64 devfreq_get_core_mask(struct kbase_device *kbdev) +{ + return kbdev->pm.backend.ca_policy_data.devfreq.cores_enabled; +} + +static void devfreq_update_core_status(struct kbase_device *kbdev, + u64 cores_ready, + u64 cores_transitioning) +{ + struct kbasep_pm_ca_policy_devfreq *data = + &kbdev->pm.backend.ca_policy_data.devfreq; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + data->cores_used = cores_ready | cores_transitioning; + + /* If in desired state then clear transition flag */ + if (data->cores_enabled == data->cores_desired) + kbdev->pm.backend.ca_in_transition = false; + + /* If all undesired cores are now off then power on desired cores. + * The direct comparison against cores_enabled limits potential + * recursion to one level */ + if (!(data->cores_used & ~data->cores_desired) && + data->cores_enabled != data->cores_desired) { + data->cores_enabled = data->cores_desired; + + kbase_pm_update_cores_state_nolock(kbdev); + + kbdev->pm.backend.ca_in_transition = false; + } +} + +/* + * The struct kbase_pm_ca_policy structure for the devfreq core availability + * policy. + * + * This is the static structure that defines the devfreq core availability power + * policy's callback and name. + */ +const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops = { + "devfreq", /* name */ + devfreq_init, /* init */ + devfreq_term, /* term */ + devfreq_get_core_mask, /* get_core_mask */ + devfreq_update_core_status, /* update_core_status */ + 0u, /* flags */ + KBASE_PM_CA_POLICY_ID_DEVFREQ, /* id */ +}; + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.h new file mode 100755 index 000000000000..7ab3cd4d8460 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_devfreq.h @@ -0,0 +1,55 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * A core availability policy for use with devfreq, where core masks are + * associated with OPPs. + */ + +#ifndef MALI_KBASE_PM_CA_DEVFREQ_H +#define MALI_KBASE_PM_CA_DEVFREQ_H + +/** + * struct kbasep_pm_ca_policy_devfreq - Private structure for devfreq ca policy + * + * This contains data that is private to the devfreq core availability + * policy. + * + * @cores_desired: Cores that the policy wants to be available + * @cores_enabled: Cores that the policy is currently returning as available + * @cores_used: Cores currently powered or transitioning + */ +struct kbasep_pm_ca_policy_devfreq { + u64 cores_desired; + u64 cores_enabled; + u64 cores_used; +}; + +extern const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops; + +/** + * kbase_devfreq_set_core_mask - Set core mask for policy to use + * @kbdev: Device pointer + * @core_mask: New core mask + * + * The new core mask will have immediate effect if the GPU is powered, or will + * take effect when it is next powered on. + */ +void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask); + +#endif /* MALI_KBASE_PM_CA_DEVFREQ_H */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.c new file mode 100755 index 000000000000..864612d31f9b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.c @@ -0,0 +1,65 @@ +/* + * + * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * A power policy implementing fixed core availability + */ + +#include +#include + +static void fixed_init(struct kbase_device *kbdev) +{ + kbdev->pm.backend.ca_in_transition = false; +} + +static void fixed_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static u64 fixed_get_core_mask(struct kbase_device *kbdev) +{ + return kbdev->gpu_props.props.raw_props.shader_present; +} + +static void fixed_update_core_status(struct kbase_device *kbdev, + u64 cores_ready, + u64 cores_transitioning) +{ + CSTD_UNUSED(kbdev); + CSTD_UNUSED(cores_ready); + CSTD_UNUSED(cores_transitioning); +} + +/* + * The struct kbase_pm_policy structure for the fixed power policy. + * + * This is the static structure that defines the fixed power policy's callback + * and name. + */ +const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops = { + "fixed", /* name */ + fixed_init, /* init */ + fixed_term, /* term */ + fixed_get_core_mask, /* get_core_mask */ + fixed_update_core_status, /* update_core_status */ + 0u, /* flags */ + KBASE_PM_CA_POLICY_ID_FIXED, /* id */ +}; + +KBASE_EXPORT_TEST_API(kbase_pm_ca_fixed_policy_ops); diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.h new file mode 100755 index 000000000000..a763155cb703 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_ca_fixed.h @@ -0,0 +1,40 @@ +/* + * + * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * A power policy implementing fixed core availability + */ + +#ifndef MALI_KBASE_PM_CA_FIXED_H +#define MALI_KBASE_PM_CA_FIXED_H + +/** + * struct kbasep_pm_ca_policy_fixed - Private structure for policy instance data + * + * @dummy: Dummy member - no state is needed + * + * This contains data that is private to the particular power policy that is + * active. + */ +struct kbasep_pm_ca_policy_fixed { + int dummy; +}; + +extern const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops; + +#endif /* MALI_KBASE_PM_CA_FIXED_H */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.c new file mode 100755 index 000000000000..f891fa225a89 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.c @@ -0,0 +1,70 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * "Coarse Demand" power management policy + */ + +#include +#include + +static u64 coarse_demand_get_core_mask(struct kbase_device *kbdev) +{ + if (kbdev->pm.active_count == 0) + return 0; + + return kbdev->gpu_props.props.raw_props.shader_present; +} + +static bool coarse_demand_get_core_active(struct kbase_device *kbdev) +{ + if (0 == kbdev->pm.active_count && !(kbdev->shader_needed_bitmap | + kbdev->shader_inuse_bitmap) && !kbdev->tiler_needed_cnt + && !kbdev->tiler_inuse_cnt) + return false; + + return true; +} + +static void coarse_demand_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static void coarse_demand_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +/* The struct kbase_pm_policy structure for the demand power policy. + * + * This is the static structure that defines the demand power policy's callback + * and name. + */ +const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { + "coarse_demand", /* name */ + coarse_demand_init, /* init */ + coarse_demand_term, /* term */ + coarse_demand_get_core_mask, /* get_core_mask */ + coarse_demand_get_core_active, /* get_core_active */ + 0u, /* flags */ + KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ +}; + +KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.h new file mode 100755 index 000000000000..749d305eee9a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_coarse_demand.h @@ -0,0 +1,64 @@ +/* + * + * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * "Coarse Demand" power management policy + */ + +#ifndef MALI_KBASE_PM_COARSE_DEMAND_H +#define MALI_KBASE_PM_COARSE_DEMAND_H + +/** + * DOC: + * The "Coarse" demand power management policy has the following + * characteristics: + * - When KBase indicates that the GPU will be powered up, but we don't yet + * know which Job Chains are to be run: + * - All Shader Cores are powered up, regardless of whether or not they will + * be needed later. + * - When KBase indicates that a set of Shader Cores are needed to submit the + * currently queued Job Chains: + * - All Shader Cores are kept powered, regardless of whether or not they will + * be needed + * - When KBase indicates that the GPU need not be powered: + * - The Shader Cores are powered off, and the GPU itself is powered off too. + * + * @note: + * - KBase indicates the GPU will be powered up when it has a User Process that + * has just started to submit Job Chains. + * - KBase indicates the GPU need not be powered when all the Job Chains from + * User Processes have finished, and it is waiting for a User Process to + * submit some more Job Chains. + */ + +/** + * struct kbasep_pm_policy_coarse_demand - Private structure for coarse demand + * policy + * + * This contains data that is private to the coarse demand power policy. + * + * @dummy: Dummy member - no state needed + */ +struct kbasep_pm_policy_coarse_demand { + int dummy; +}; + +extern const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops; + +#endif /* MALI_KBASE_PM_COARSE_DEMAND_H */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_defs.h new file mode 100755 index 000000000000..564fbda1116a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_defs.h @@ -0,0 +1,519 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Backend-specific Power Manager definitions + */ + +#ifndef _KBASE_PM_HWACCESS_DEFS_H_ +#define _KBASE_PM_HWACCESS_DEFS_H_ + +#include "mali_kbase_pm_ca_fixed.h" +#include "mali_kbase_pm_ca_devfreq.h" +#if !MALI_CUSTOMER_RELEASE +#include "mali_kbase_pm_ca_random.h" +#endif + +#include "mali_kbase_pm_always_on.h" +#include "mali_kbase_pm_coarse_demand.h" +#include "mali_kbase_pm_demand.h" +#if !MALI_CUSTOMER_RELEASE +#include "mali_kbase_pm_demand_always_powered.h" +#include "mali_kbase_pm_fast_start.h" +#endif + +/* Forward definition - see mali_kbase.h */ +struct kbase_device; +struct kbase_jd_atom; + +/** + * enum kbase_pm_core_type - The types of core in a GPU. + * + * These enumerated values are used in calls to + * - kbase_pm_get_present_cores() + * - kbase_pm_get_active_cores() + * - kbase_pm_get_trans_cores() + * - kbase_pm_get_ready_cores(). + * + * They specify which type of core should be acted on. These values are set in + * a manner that allows core_type_to_reg() function to be simpler and more + * efficient. + * + * @KBASE_PM_CORE_L2: The L2 cache + * @KBASE_PM_CORE_SHADER: Shader cores + * @KBASE_PM_CORE_TILER: Tiler cores + * @KBASE_PM_CORE_STACK: Core stacks + */ +enum kbase_pm_core_type { + KBASE_PM_CORE_L2 = L2_PRESENT_LO, + KBASE_PM_CORE_SHADER = SHADER_PRESENT_LO, + KBASE_PM_CORE_TILER = TILER_PRESENT_LO, + KBASE_PM_CORE_STACK = STACK_PRESENT_LO +}; + +/** + * struct kbasep_pm_metrics_data - Metrics data collected for use by the power + * management framework. + * + * @time_period_start: time at which busy/idle measurements started + * @time_busy: number of ns the GPU was busy executing jobs since the + * @time_period_start timestamp. + * @time_idle: number of ns since time_period_start the GPU was not executing + * jobs since the @time_period_start timestamp. + * @prev_busy: busy time in ns of previous time period. + * Updated when metrics are reset. + * @prev_idle: idle time in ns of previous time period + * Updated when metrics are reset. + * @gpu_active: true when the GPU is executing jobs. false when + * not. Updated when the job scheduler informs us a job in submitted + * or removed from a GPU slot. + * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that + * if two CL jobs were active for 400ns, this value would be updated + * with 800. + * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that + * if two GL jobs were active for 400ns, this value would be updated + * with 800. + * @active_cl_ctx: number of CL jobs active on the GPU. Array is per-device. + * @active_gl_ctx: number of GL jobs active on the GPU. Array is per-slot. As + * GL jobs never run on slot 2 this slot is not recorded. + * @lock: spinlock protecting the kbasep_pm_metrics_data structure + * @timer: timer to regularly make DVFS decisions based on the power + * management metrics. + * @timer_active: boolean indicating @timer is running + * @platform_data: pointer to data controlled by platform specific code + * @kbdev: pointer to kbase device for which metrics are collected + * + */ +struct kbasep_pm_metrics_data { + ktime_t time_period_start; + u32 time_busy; + u32 time_idle; + u32 prev_busy; + u32 prev_idle; + bool gpu_active; + u32 busy_cl[2]; + u32 busy_gl; + u32 active_cl_ctx[2]; + u32 active_gl_ctx[2]; /* GL jobs can only run on 2 of the 3 job slots */ + spinlock_t lock; + +#ifdef CONFIG_MALI_BIFROST_DVFS + struct hrtimer timer; + bool timer_active; +#endif + + void *platform_data; + struct kbase_device *kbdev; +}; + +union kbase_pm_policy_data { + struct kbasep_pm_policy_always_on always_on; + struct kbasep_pm_policy_coarse_demand coarse_demand; + struct kbasep_pm_policy_demand demand; +#if !MALI_CUSTOMER_RELEASE + struct kbasep_pm_policy_demand_always_powered demand_always_powered; + struct kbasep_pm_policy_fast_start fast_start; +#endif +}; + +union kbase_pm_ca_policy_data { + struct kbasep_pm_ca_policy_fixed fixed; + struct kbasep_pm_ca_policy_devfreq devfreq; +#if !MALI_CUSTOMER_RELEASE + struct kbasep_pm_ca_policy_random random; +#endif +}; + +/** + * struct kbase_pm_backend_data - Data stored per device for power management. + * + * This structure contains data for the power management framework. There is one + * instance of this structure per device in the system. + * + * @ca_current_policy: The policy that is currently actively controlling core + * availability. + * @pm_current_policy: The policy that is currently actively controlling the + * power state. + * @ca_policy_data: Private data for current CA policy + * @pm_policy_data: Private data for current PM policy + * @ca_in_transition: Flag indicating when core availability policy is + * transitioning cores. The core availability policy must + * set this when a change in core availability is occurring. + * power_change_lock must be held when accessing this. + * @reset_done: Flag when a reset is complete + * @reset_done_wait: Wait queue to wait for changes to @reset_done + * @l2_powered_wait: Wait queue for whether the l2 cache has been powered as + * requested + * @l2_powered: State indicating whether all the l2 caches are powered. + * Non-zero indicates they're *all* powered + * Zero indicates that some (or all) are not powered + * @gpu_cycle_counter_requests: The reference count of active gpu cycle counter + * users + * @gpu_cycle_counter_requests_lock: Lock to protect @gpu_cycle_counter_requests + * @desired_shader_state: A bit mask identifying the shader cores that the + * power policy would like to be on. The current state + * of the cores may be different, but there should be + * transitions in progress that will eventually achieve + * this state (assuming that the policy doesn't change + * its mind in the mean time). + * @powering_on_shader_state: A bit mask indicating which shader cores are + * currently in a power-on transition + * @desired_tiler_state: A bit mask identifying the tiler cores that the power + * policy would like to be on. See @desired_shader_state + * @powering_on_tiler_state: A bit mask indicating which tiler core are + * currently in a power-on transition + * @powering_on_l2_state: A bit mask indicating which l2-caches are currently + * in a power-on transition + * @powering_on_stack_state: A bit mask indicating which core stacks are + * currently in a power-on transition + * @gpu_in_desired_state: This flag is set if the GPU is powered as requested + * by the desired_xxx_state variables + * @gpu_in_desired_state_wait: Wait queue set when @gpu_in_desired_state != 0 + * @gpu_powered: Set to true when the GPU is powered and register + * accesses are possible, false otherwise + * @instr_enabled: Set to true when instrumentation is enabled, + * false otherwise + * @cg1_disabled: Set if the policy wants to keep the second core group + * powered off + * @driver_ready_for_irqs: Debug state indicating whether sufficient + * initialization of the driver has occurred to handle + * IRQs + * @gpu_powered_lock: Spinlock that must be held when writing @gpu_powered or + * accessing @driver_ready_for_irqs + * @metrics: Structure to hold metrics for the GPU + * @gpu_poweroff_pending: number of poweroff timer ticks until the GPU is + * powered off + * @shader_poweroff_pending_time: number of poweroff timer ticks until shaders + * and/or timers are powered off + * @gpu_poweroff_timer: Timer for powering off GPU + * @gpu_poweroff_wq: Workqueue to power off GPU on when timer fires + * @gpu_poweroff_work: Workitem used on @gpu_poweroff_wq + * @shader_poweroff_pending: Bit mask of shaders to be powered off on next + * timer callback + * @tiler_poweroff_pending: Bit mask of tilers to be powered off on next timer + * callback + * @poweroff_timer_needed: true if the poweroff timer is currently required, + * false otherwise + * @poweroff_timer_running: true if the poweroff timer is currently running, + * false otherwise + * power_change_lock should be held when accessing, + * unless there is no way the timer can be running (eg + * hrtimer_cancel() was called immediately before) + * @poweroff_wait_in_progress: true if a wait for GPU power off is in progress. + * hwaccess_lock must be held when accessing + * @poweron_required: true if a GPU power on is required. Should only be set + * when poweroff_wait_in_progress is true, and therefore the + * GPU can not immediately be powered on. pm.lock must be + * held when accessing + * @poweroff_is_suspend: true if the GPU is being powered off due to a suspend + * request. pm.lock must be held when accessing + * @gpu_poweroff_wait_wq: workqueue for waiting for GPU to power off + * @gpu_poweroff_wait_work: work item for use with @gpu_poweroff_wait_wq + * @poweroff_wait: waitqueue for waiting for @gpu_poweroff_wait_work to complete + * @callback_power_on: Callback when the GPU needs to be turned on. See + * &struct kbase_pm_callback_conf + * @callback_power_off: Callback when the GPU may be turned off. See + * &struct kbase_pm_callback_conf + * @callback_power_suspend: Callback when a suspend occurs and the GPU needs to + * be turned off. See &struct kbase_pm_callback_conf + * @callback_power_resume: Callback when a resume occurs and the GPU needs to + * be turned on. See &struct kbase_pm_callback_conf + * @callback_power_runtime_on: Callback when the GPU needs to be turned on. See + * &struct kbase_pm_callback_conf + * @callback_power_runtime_off: Callback when the GPU may be turned off. See + * &struct kbase_pm_callback_conf + * @callback_power_runtime_idle: Optional callback when the GPU may be idle. See + * &struct kbase_pm_callback_conf + * + * Note: + * During an IRQ, @ca_current_policy or @pm_current_policy can be NULL when the + * policy is being changed with kbase_pm_ca_set_policy() or + * kbase_pm_set_policy(). The change is protected under + * kbase_device.pm.power_change_lock. Direct access to this + * from IRQ context must therefore check for NULL. If NULL, then + * kbase_pm_ca_set_policy() or kbase_pm_set_policy() will re-issue the policy + * functions that would have been done under IRQ. + */ +struct kbase_pm_backend_data { + const struct kbase_pm_ca_policy *ca_current_policy; + const struct kbase_pm_policy *pm_current_policy; + union kbase_pm_ca_policy_data ca_policy_data; + union kbase_pm_policy_data pm_policy_data; + bool ca_in_transition; + bool reset_done; + wait_queue_head_t reset_done_wait; + wait_queue_head_t l2_powered_wait; + int l2_powered; + int gpu_cycle_counter_requests; + spinlock_t gpu_cycle_counter_requests_lock; + + u64 desired_shader_state; + u64 powering_on_shader_state; + u64 desired_tiler_state; + u64 powering_on_tiler_state; + u64 powering_on_l2_state; +#ifdef CONFIG_MALI_CORESTACK + u64 powering_on_stack_state; +#endif /* CONFIG_MALI_CORESTACK */ + + bool gpu_in_desired_state; + wait_queue_head_t gpu_in_desired_state_wait; + + bool gpu_powered; + + bool instr_enabled; + + bool cg1_disabled; + +#ifdef CONFIG_MALI_BIFROST_DEBUG + bool driver_ready_for_irqs; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + spinlock_t gpu_powered_lock; + + + struct kbasep_pm_metrics_data metrics; + + int gpu_poweroff_pending; + int shader_poweroff_pending_time; + + struct hrtimer gpu_poweroff_timer; + struct workqueue_struct *gpu_poweroff_wq; + struct work_struct gpu_poweroff_work; + + u64 shader_poweroff_pending; + u64 tiler_poweroff_pending; + + bool poweroff_timer_needed; + bool poweroff_timer_running; + + bool poweroff_wait_in_progress; + bool poweron_required; + bool poweroff_is_suspend; + + struct workqueue_struct *gpu_poweroff_wait_wq; + struct work_struct gpu_poweroff_wait_work; + + wait_queue_head_t poweroff_wait; + + int (*callback_power_on)(struct kbase_device *kbdev); + void (*callback_power_off)(struct kbase_device *kbdev); + void (*callback_power_suspend)(struct kbase_device *kbdev); + void (*callback_power_resume)(struct kbase_device *kbdev); + int (*callback_power_runtime_on)(struct kbase_device *kbdev); + void (*callback_power_runtime_off)(struct kbase_device *kbdev); + int (*callback_power_runtime_idle)(struct kbase_device *kbdev); +}; + + +/* List of policy IDs */ +enum kbase_pm_policy_id { + KBASE_PM_POLICY_ID_DEMAND = 1, + KBASE_PM_POLICY_ID_ALWAYS_ON, + KBASE_PM_POLICY_ID_COARSE_DEMAND, +#if !MALI_CUSTOMER_RELEASE + KBASE_PM_POLICY_ID_DEMAND_ALWAYS_POWERED, + KBASE_PM_POLICY_ID_FAST_START +#endif +}; + +typedef u32 kbase_pm_policy_flags; + +/** + * struct kbase_pm_policy - Power policy structure. + * + * Each power policy exposes a (static) instance of this structure which + * contains function pointers to the policy's methods. + * + * @name: The name of this policy + * @init: Function called when the policy is selected + * @term: Function called when the policy is unselected + * @get_core_mask: Function called to get the current shader core mask + * @get_core_active: Function called to get the current overall GPU power + * state + * @flags: Field indicating flags for this policy + * @id: Field indicating an ID for this policy. This is not + * necessarily the same as its index in the list returned + * by kbase_pm_list_policies(). + * It is used purely for debugging. + */ +struct kbase_pm_policy { + char *name; + + /** + * Function called when the policy is selected + * + * This should initialize the kbdev->pm.pm_policy_data structure. It + * should not attempt to make any changes to hardware state. + * + * It is undefined what state the cores are in when the function is + * called. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + */ + void (*init)(struct kbase_device *kbdev); + + /** + * Function called when the policy is unselected. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + */ + void (*term)(struct kbase_device *kbdev); + + /** + * Function called to get the current shader core mask + * + * The returned mask should meet or exceed (kbdev->shader_needed_bitmap + * | kbdev->shader_inuse_bitmap). + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * + * Return: The mask of shader cores to be powered + */ + u64 (*get_core_mask)(struct kbase_device *kbdev); + + /** + * Function called to get the current overall GPU power state + * + * This function should consider the state of kbdev->pm.active_count. If + * this count is greater than 0 then there is at least one active + * context on the device and the GPU should be powered. If it is equal + * to 0 then there are no active contexts and the GPU could be powered + * off if desired. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * + * Return: true if the GPU should be powered, false otherwise + */ + bool (*get_core_active)(struct kbase_device *kbdev); + + kbase_pm_policy_flags flags; + enum kbase_pm_policy_id id; +}; + + +enum kbase_pm_ca_policy_id { + KBASE_PM_CA_POLICY_ID_FIXED = 1, + KBASE_PM_CA_POLICY_ID_DEVFREQ, + KBASE_PM_CA_POLICY_ID_RANDOM +}; + +typedef u32 kbase_pm_ca_policy_flags; + +/** + * Maximum length of a CA policy names + */ +#define KBASE_PM_CA_MAX_POLICY_NAME_LEN 15 + +/** + * struct kbase_pm_ca_policy - Core availability policy structure. + * + * Each core availability policy exposes a (static) instance of this structure + * which contains function pointers to the policy's methods. + * + * @name: The name of this policy + * @init: Function called when the policy is selected + * @term: Function called when the policy is unselected + * @get_core_mask: Function called to get the current shader core + * availability mask + * @update_core_status: Function called to update the current core status + * @flags: Field indicating flags for this policy + * @id: Field indicating an ID for this policy. This is not + * necessarily the same as its index in the list returned + * by kbase_pm_list_policies(). + * It is used purely for debugging. + */ +struct kbase_pm_ca_policy { + char name[KBASE_PM_CA_MAX_POLICY_NAME_LEN + 1]; + + /** + * Function called when the policy is selected + * + * This should initialize the kbdev->pm.ca_policy_data structure. It + * should not attempt to make any changes to hardware state. + * + * It is undefined what state the cores are in when the function is + * called. + * + * @kbdev The kbase device structure for the device (must be a + * valid pointer) + */ + void (*init)(struct kbase_device *kbdev); + + /** + * Function called when the policy is unselected. + * + * @kbdev The kbase device structure for the device (must be a + * valid pointer) + */ + void (*term)(struct kbase_device *kbdev); + + /** + * Function called to get the current shader core availability mask + * + * When a change in core availability is occurring, the policy must set + * kbdev->pm.ca_in_transition to true. This is to indicate that + * reporting changes in power state cannot be optimized out, even if + * kbdev->pm.desired_shader_state remains unchanged. This must be done + * by any functions internal to the Core Availability Policy that change + * the return value of kbase_pm_ca_policy::get_core_mask. + * + * @kbdev The kbase device structure for the device (must be a + * valid pointer) + * + * Return: The current core availability mask + */ + u64 (*get_core_mask)(struct kbase_device *kbdev); + + /** + * Function called to update the current core status + * + * If none of the cores in core group 0 are ready or transitioning, then + * the policy must ensure that the next call to get_core_mask does not + * return 0 for all cores in core group 0. It is an error to disable + * core group 0 through the core availability policy. + * + * When a change in core availability has finished, the policy must set + * kbdev->pm.ca_in_transition to false. This is to indicate that + * changes in power state can once again be optimized out when + * kbdev->pm.desired_shader_state is unchanged. + * + * @kbdev: The kbase device structure for the device + * (must be a valid pointer) + * @cores_ready: The mask of cores currently powered and + * ready to run jobs + * @cores_transitioning: The mask of cores currently transitioning + * power state + */ + void (*update_core_status)(struct kbase_device *kbdev, u64 cores_ready, + u64 cores_transitioning); + + kbase_pm_ca_policy_flags flags; + + /** + * Field indicating an ID for this policy. This is not necessarily the + * same as its index in the list returned by kbase_pm_list_policies(). + * It is used purely for debugging. + */ + enum kbase_pm_ca_policy_id id; +}; + +#endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.c new file mode 100755 index 000000000000..81322fd0dd17 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.c @@ -0,0 +1,73 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * A simple demand based power management policy + */ + +#include +#include + +static u64 demand_get_core_mask(struct kbase_device *kbdev) +{ + u64 desired = kbdev->shader_needed_bitmap | kbdev->shader_inuse_bitmap; + + if (0 == kbdev->pm.active_count) + return 0; + + return desired; +} + +static bool demand_get_core_active(struct kbase_device *kbdev) +{ + if (0 == kbdev->pm.active_count && !(kbdev->shader_needed_bitmap | + kbdev->shader_inuse_bitmap) && !kbdev->tiler_needed_cnt + && !kbdev->tiler_inuse_cnt) + return false; + + return true; +} + +static void demand_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static void demand_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +/* + * The struct kbase_pm_policy structure for the demand power policy. + * + * This is the static structure that defines the demand power policy's callback + * and name. + */ +const struct kbase_pm_policy kbase_pm_demand_policy_ops = { + "demand", /* name */ + demand_init, /* init */ + demand_term, /* term */ + demand_get_core_mask, /* get_core_mask */ + demand_get_core_active, /* get_core_active */ + 0u, /* flags */ + KBASE_PM_POLICY_ID_DEMAND, /* id */ +}; + +KBASE_EXPORT_TEST_API(kbase_pm_demand_policy_ops); diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.h new file mode 100755 index 000000000000..c0c84b6e9189 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_demand.h @@ -0,0 +1,64 @@ +/* + * + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * A simple demand based power management policy + */ + +#ifndef MALI_KBASE_PM_DEMAND_H +#define MALI_KBASE_PM_DEMAND_H + +/** + * DOC: Demand power management policy + * + * The demand power management policy has the following characteristics: + * - When KBase indicates that the GPU will be powered up, but we don't yet + * know which Job Chains are to be run: + * - The Shader Cores are not powered up + * + * - When KBase indicates that a set of Shader Cores are needed to submit the + * currently queued Job Chains: + * - Only those Shader Cores are powered up + * + * - When KBase indicates that the GPU need not be powered: + * - The Shader Cores are powered off, and the GPU itself is powered off too. + * + * Note: + * - KBase indicates the GPU will be powered up when it has a User Process that + * has just started to submit Job Chains. + * + * - KBase indicates the GPU need not be powered when all the Job Chains from + * User Processes have finished, and it is waiting for a User Process to + * submit some more Job Chains. + */ + +/** + * struct kbasep_pm_policy_demand - Private structure for policy instance data + * + * @dummy: No state is needed, a dummy variable + * + * This contains data that is private to the demand power policy. + */ +struct kbasep_pm_policy_demand { + int dummy; +}; + +extern const struct kbase_pm_policy kbase_pm_demand_policy_ops; + +#endif /* MALI_KBASE_PM_DEMAND_H */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_driver.c new file mode 100755 index 000000000000..707f71a79a77 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_driver.c @@ -0,0 +1,1672 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel Power Management hardware control + */ + +#include +#include +#include +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if MALI_MOCK_TEST +#define MOCKABLE(function) function##_original +#else +#define MOCKABLE(function) function +#endif /* MALI_MOCK_TEST */ + +/** + * enum kbasep_pm_action - Actions that can be performed on a core. + * + * This enumeration is private to the file. Its values are set to allow + * core_type_to_reg() function, which decodes this enumeration, to be simpler + * and more efficient. + * + * @ACTION_PRESENT: The cores that are present + * @ACTION_READY: The cores that are ready + * @ACTION_PWRON: Power on the cores specified + * @ACTION_PWROFF: Power off the cores specified + * @ACTION_PWRTRANS: The cores that are transitioning + * @ACTION_PWRACTIVE: The cores that are active + */ +enum kbasep_pm_action { + ACTION_PRESENT = 0, + ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO), + ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO), + ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO), + ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO), + ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO) +}; + +static u64 kbase_pm_get_state( + struct kbase_device *kbdev, + enum kbase_pm_core_type core_type, + enum kbasep_pm_action action); + +/** + * core_type_to_reg - Decode a core type and action to a register. + * + * Given a core type (defined by kbase_pm_core_type) and an action (defined + * by kbasep_pm_action) this function will return the register offset that + * will perform the action on the core type. The register returned is the _LO + * register and an offset must be applied to use the _HI register. + * + * @core_type: The type of core + * @action: The type of action + * + * Return: The register offset of the _LO register that performs an action of + * type @action on a core of type @core_type. + */ +static u32 core_type_to_reg(enum kbase_pm_core_type core_type, + enum kbasep_pm_action action) +{ +#ifdef CONFIG_MALI_CORESTACK + if (core_type == KBASE_PM_CORE_STACK) { + switch (action) { + case ACTION_PRESENT: + return STACK_PRESENT_LO; + case ACTION_READY: + return STACK_READY_LO; + case ACTION_PWRON: + return STACK_PWRON_LO; + case ACTION_PWROFF: + return STACK_PWROFF_LO; + case ACTION_PWRTRANS: + return STACK_PWRTRANS_LO; + default: + BUG(); + } + } +#endif /* CONFIG_MALI_CORESTACK */ + + return (u32)core_type + (u32)action; +} + +#ifdef CONFIG_ARM64 +static void mali_cci_flush_l2(struct kbase_device *kbdev) +{ + const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; + u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS; + u32 raw; + + /* + * Note that we don't take the cache flush mutex here since + * we expect to be the last user of the L2, all other L2 users + * would have dropped their references, to initiate L2 power + * down, L2 power down being the only valid place for this + * to be called from. + */ + + kbase_reg_write(kbdev, + GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAN_INV_CACHES, + NULL); + + raw = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), + NULL); + + /* Wait for cache flush to complete before continuing, exit on + * gpu resets or loop expiry. */ + while (((raw & mask) == 0) && --loops) { + raw = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), + NULL); + } +} +#endif + +/** + * kbase_pm_invoke - Invokes an action on a core set + * + * This function performs the action given by @action on a set of cores of a + * type given by @core_type. It is a static function used by + * kbase_pm_transition_core_type() + * + * @kbdev: The kbase device structure of the device + * @core_type: The type of core that the action should be performed on + * @cores: A bit mask of cores to perform the action on (low 32 bits) + * @action: The action to perform on the cores + */ +static void kbase_pm_invoke(struct kbase_device *kbdev, + enum kbase_pm_core_type core_type, + u64 cores, + enum kbasep_pm_action action) +{ + u32 reg; + u32 lo = cores & 0xFFFFFFFF; + u32 hi = (cores >> 32) & 0xFFFFFFFF; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + reg = core_type_to_reg(core_type, action); + + KBASE_DEBUG_ASSERT(reg); +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) + if (cores) { + if (action == ACTION_PWRON) + kbase_trace_mali_pm_power_on(core_type, cores); + else if (action == ACTION_PWROFF) + kbase_trace_mali_pm_power_off(core_type, cores); + } +#endif + + if (cores) { + u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY); + + if (action == ACTION_PWRON) + state |= cores; + else if (action == ACTION_PWROFF) + state &= ~cores; + KBASE_TLSTREAM_AUX_PM_STATE(core_type, state); + } + + /* Tracing */ + if (cores) { + if (action == ACTION_PWRON) + switch (core_type) { + case KBASE_PM_CORE_SHADER: + KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u, + lo); + break; + case KBASE_PM_CORE_TILER: + KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL, + NULL, 0u, lo); + break; + case KBASE_PM_CORE_L2: + KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL, + 0u, lo); + break; + default: + break; + } + else if (action == ACTION_PWROFF) + switch (core_type) { + case KBASE_PM_CORE_SHADER: + KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL, + 0u, lo); + break; + case KBASE_PM_CORE_TILER: + KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, + NULL, 0u, lo); + break; + case KBASE_PM_CORE_L2: + KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL, + 0u, lo); + /* disable snoops before L2 is turned off */ + kbase_pm_cache_snoop_disable(kbdev); + break; + default: + break; + } + } + + if (lo != 0) + kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL); + + if (hi != 0) + kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL); +} + +/** + * kbase_pm_get_state - Get information about a core set + * + * This function gets information (chosen by @action) about a set of cores of + * a type given by @core_type. It is a static function used by + * kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and + * kbase_pm_get_ready_cores(). + * + * @kbdev: The kbase device structure of the device + * @core_type: The type of core that the should be queried + * @action: The property of the cores to query + * + * Return: A bit mask specifying the state of the cores + */ +static u64 kbase_pm_get_state(struct kbase_device *kbdev, + enum kbase_pm_core_type core_type, + enum kbasep_pm_action action) +{ + u32 reg; + u32 lo, hi; + + reg = core_type_to_reg(core_type, action); + + KBASE_DEBUG_ASSERT(reg); + + lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL); + hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL); + + return (((u64) hi) << 32) | ((u64) lo); +} + +void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev) +{ + kbdev->shader_inuse_bitmap = 0; + kbdev->shader_needed_bitmap = 0; + kbdev->shader_available_bitmap = 0; + kbdev->tiler_available_bitmap = 0; + kbdev->l2_users_count = 0; + kbdev->l2_available_bitmap = 0; + kbdev->tiler_needed_cnt = 0; + kbdev->tiler_inuse_cnt = 0; + + memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt)); +} + +/** + * kbase_pm_get_present_cores - Get the cores that are present + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of the cores that are present + */ +u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + switch (type) { + case KBASE_PM_CORE_L2: + return kbdev->gpu_props.props.raw_props.l2_present; + case KBASE_PM_CORE_SHADER: + return kbdev->gpu_props.props.raw_props.shader_present; + case KBASE_PM_CORE_TILER: + return kbdev->gpu_props.props.raw_props.tiler_present; +#ifdef CONFIG_MALI_CORESTACK + case KBASE_PM_CORE_STACK: + return kbdev->gpu_props.props.raw_props.stack_present; +#endif /* CONFIG_MALI_CORESTACK */ + default: + break; + } + KBASE_DEBUG_ASSERT(0); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores); + +/** + * kbase_pm_get_active_cores - Get the cores that are "active" + * (busy processing work) + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of cores that are active + */ +u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE); +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores); + +/** + * kbase_pm_get_trans_cores - Get the cores that are transitioning between + * power states + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of cores that are transitioning + */ +u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS); +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores); + +/** + * kbase_pm_get_ready_cores - Get the cores that are powered on + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of cores that are ready (powered on) + */ +u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + u64 result; + + result = kbase_pm_get_state(kbdev, type, ACTION_READY); + + switch (type) { + case KBASE_PM_CORE_SHADER: + KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u, + (u32) result); + break; + case KBASE_PM_CORE_TILER: + KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u, + (u32) result); + break; + case KBASE_PM_CORE_L2: + KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u, + (u32) result); + break; + default: + break; + } + + return result; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores); + +/** + * kbase_pm_transition_core_type - Perform power transitions for a particular + * core type. + * + * This function will perform any available power transitions to make the actual + * hardware state closer to the desired state. If a core is currently + * transitioning then changes to the power state of that call cannot be made + * until the transition has finished. Cores which are not present in the + * hardware are ignored if they are specified in the desired_state bitmask, + * however the return value will always be 0 in this case. + * + * @kbdev: The kbase device + * @type: The core type to perform transitions for + * @desired_state: A bit mask of the desired state of the cores + * @in_use: A bit mask of the cores that are currently running + * jobs. These cores have to be kept powered up because + * there are jobs running (or about to run) on them. + * @available: Receives a bit mask of the cores that the job + * scheduler can use to submit jobs to. May be NULL if + * this is not needed. + * @powering_on: Bit mask to update with cores that are + * transitioning to a power-on state. + * + * Return: true if the desired state has been reached, false otherwise + */ +static bool kbase_pm_transition_core_type(struct kbase_device *kbdev, + enum kbase_pm_core_type type, + u64 desired_state, + u64 in_use, + u64 * const available, + u64 *powering_on) +{ + u64 present; + u64 ready; + u64 trans; + u64 powerup; + u64 powerdown; + u64 powering_on_trans; + u64 desired_state_in_use; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* Get current state */ + present = kbase_pm_get_present_cores(kbdev, type); + trans = kbase_pm_get_trans_cores(kbdev, type); + ready = kbase_pm_get_ready_cores(kbdev, type); + /* mask off ready from trans in case transitions finished between the + * register reads */ + trans &= ~ready; + + if (trans) /* Do not progress if any cores are transitioning */ + return false; + + powering_on_trans = trans & *powering_on; + *powering_on = powering_on_trans; + + if (available != NULL) + *available = (ready | powering_on_trans) & desired_state; + + /* Update desired state to include the in-use cores. These have to be + * kept powered up because there are jobs running or about to run on + * these cores + */ + desired_state_in_use = desired_state | in_use; + + /* Update state of whether l2 caches are powered */ + if (type == KBASE_PM_CORE_L2) { + if ((ready == present) && (desired_state_in_use == ready) && + (trans == 0)) { + /* All are ready, none will be turned off, and none are + * transitioning */ + kbdev->pm.backend.l2_powered = 1; + /* + * Ensure snoops are enabled after L2 is powered up, + * note that kbase keeps track of the snoop state, so + * safe to repeatedly call. + */ + kbase_pm_cache_snoop_enable(kbdev); + if (kbdev->l2_users_count > 0) { + /* Notify any registered l2 cache users + * (optimized out when no users waiting) */ + wake_up(&kbdev->pm.backend.l2_powered_wait); + } + } else + kbdev->pm.backend.l2_powered = 0; + } + + if (desired_state == ready && (trans == 0)) + return true; + + /* Restrict the cores to those that are actually present */ + powerup = desired_state_in_use & present; + powerdown = (~desired_state_in_use) & present; + + /* Restrict to cores that are not already in the desired state */ + powerup &= ~ready; + powerdown &= ready; + + /* Don't transition any cores that are already transitioning, except for + * Mali cores that support the following case: + * + * If the SHADER_PWRON or TILER_PWRON registers are written to turn on + * a core that is currently transitioning to power off, then this is + * remembered and the shader core is automatically powered up again once + * the original transition completes. Once the automatic power on is + * complete any job scheduled on the shader core should start. + */ + powerdown &= ~trans; + + if (kbase_hw_has_feature(kbdev, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS)) + if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type) + trans = powering_on_trans; /* for exception cases, only + * mask off cores in power on + * transitions */ + + powerup &= ~trans; + + /* Perform transitions if any */ + kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON); +#if !PLATFORM_POWER_DOWN_ONLY + kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF); +#endif + + /* Recalculate cores transitioning on, and re-evaluate our state */ + powering_on_trans |= powerup; + *powering_on = powering_on_trans; + if (available != NULL) + *available = (ready | powering_on_trans) & desired_state; + + return false; +} + +KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type); + +/** + * get_desired_cache_status - Determine which caches should be on for a + * particular core state + * + * This function takes a bit mask of the present caches and the cores (or + * caches) that are attached to the caches that will be powered. It then + * computes which caches should be turned on to allow the cores requested to be + * powered up. + * + * @present: The bit mask of present caches + * @cores_powered: A bit mask of cores (or L2 caches) that are desired to + * be powered + * @tilers_powered: The bit mask of tilers that are desired to be powered + * + * Return: A bit mask of the caches that should be turned on + */ +static u64 get_desired_cache_status(u64 present, u64 cores_powered, + u64 tilers_powered) +{ + u64 desired = 0; + + while (present) { + /* Find out which is the highest set bit */ + u64 bit = fls64(present) - 1; + u64 bit_mask = 1ull << bit; + /* Create a mask which has all bits from 'bit' upwards set */ + + u64 mask = ~(bit_mask - 1); + + /* If there are any cores powered at this bit or above (that + * haven't previously been processed) then we need this core on + */ + if (cores_powered & mask) + desired |= bit_mask; + + /* Remove bits from cores_powered and present */ + cores_powered &= ~mask; + present &= ~bit_mask; + } + + /* Power up the required L2(s) for the tiler */ + if (tilers_powered) + desired |= 1; + + return desired; +} + +KBASE_EXPORT_TEST_API(get_desired_cache_status); + +#ifdef CONFIG_MALI_CORESTACK +u64 kbase_pm_core_stack_mask(u64 cores) +{ + u64 stack_mask = 0; + size_t const MAX_CORE_ID = 31; + size_t const NUM_CORES_PER_STACK = 4; + size_t i; + + for (i = 0; i <= MAX_CORE_ID; ++i) { + if (test_bit(i, (unsigned long *)&cores)) { + /* Every core which ID >= 16 is filled to stacks 4-7 + * instead of 0-3 */ + size_t const stack_num = (i > 16) ? + (i % NUM_CORES_PER_STACK) + 4 : + (i % NUM_CORES_PER_STACK); + set_bit(stack_num, (unsigned long *)&stack_mask); + } + } + + return stack_mask; +} +#endif /* CONFIG_MALI_CORESTACK */ + +bool +MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev) +{ + bool cores_are_available = false; + bool in_desired_state = true; + u64 desired_l2_state; +#ifdef CONFIG_MALI_CORESTACK + u64 desired_stack_state; + u64 stacks_powered; +#endif /* CONFIG_MALI_CORESTACK */ + u64 cores_powered; + u64 tilers_powered; + u64 tiler_available_bitmap; + u64 tiler_transitioning_bitmap; + u64 shader_available_bitmap; + u64 shader_ready_bitmap; + u64 shader_transitioning_bitmap; + u64 l2_available_bitmap; + u64 prev_l2_available_bitmap; + u64 l2_inuse_bitmap; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->hwaccess_lock); + + spin_lock(&kbdev->pm.backend.gpu_powered_lock); + if (kbdev->pm.backend.gpu_powered == false) { + spin_unlock(&kbdev->pm.backend.gpu_powered_lock); + if (kbdev->pm.backend.desired_shader_state == 0 && + kbdev->pm.backend.desired_tiler_state == 0) + return true; + return false; + } + + /* Trace that a change-state is being requested, and that it took + * (effectively) no time to start it. This is useful for counting how + * many state changes occurred, in a way that's backwards-compatible + * with processing the trace data */ + kbase_timeline_pm_send_event(kbdev, + KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); + kbase_timeline_pm_handle_event(kbdev, + KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); + + /* If any cores are already powered then, we must keep the caches on */ + shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_SHADER); + cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); + cores_powered |= kbdev->pm.backend.desired_shader_state; + +#ifdef CONFIG_MALI_CORESTACK + /* Work out which core stacks want to be powered */ + desired_stack_state = kbase_pm_core_stack_mask(cores_powered); + stacks_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK) | + desired_stack_state; +#endif /* CONFIG_MALI_CORESTACK */ + + /* Work out which tilers want to be powered */ + tiler_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_TILER); + tilers_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER); + tilers_powered |= kbdev->pm.backend.desired_tiler_state; + + /* If there are l2 cache users registered, keep all l2s powered even if + * all other cores are off. */ + if (kbdev->l2_users_count > 0) + cores_powered |= kbdev->gpu_props.props.raw_props.l2_present; + + desired_l2_state = get_desired_cache_status( + kbdev->gpu_props.props.raw_props.l2_present, + cores_powered, tilers_powered); + + l2_inuse_bitmap = get_desired_cache_status( + kbdev->gpu_props.props.raw_props.l2_present, + cores_powered | shader_transitioning_bitmap, + tilers_powered | tiler_transitioning_bitmap); + +#ifdef CONFIG_MALI_CORESTACK + if (stacks_powered) + desired_l2_state |= 1; +#endif /* CONFIG_MALI_CORESTACK */ + + /* If any l2 cache is on, then enable l2 #0, for use by job manager */ + if (0 != desired_l2_state) + desired_l2_state |= 1; + + prev_l2_available_bitmap = kbdev->l2_available_bitmap; + in_desired_state &= kbase_pm_transition_core_type(kbdev, + KBASE_PM_CORE_L2, desired_l2_state, l2_inuse_bitmap, + &l2_available_bitmap, + &kbdev->pm.backend.powering_on_l2_state); + + if (kbdev->l2_available_bitmap != l2_available_bitmap) + KBASE_TIMELINE_POWER_L2(kbdev, l2_available_bitmap); + + kbdev->l2_available_bitmap = l2_available_bitmap; + + +#ifdef CONFIG_MALI_CORESTACK + if (in_desired_state) { + in_desired_state &= kbase_pm_transition_core_type(kbdev, + KBASE_PM_CORE_STACK, desired_stack_state, 0, + &kbdev->stack_available_bitmap, + &kbdev->pm.backend.powering_on_stack_state); + } +#endif /* CONFIG_MALI_CORESTACK */ + + if (in_desired_state) { + in_desired_state &= kbase_pm_transition_core_type(kbdev, + KBASE_PM_CORE_TILER, + kbdev->pm.backend.desired_tiler_state, + 0, &tiler_available_bitmap, + &kbdev->pm.backend.powering_on_tiler_state); + in_desired_state &= kbase_pm_transition_core_type(kbdev, + KBASE_PM_CORE_SHADER, + kbdev->pm.backend.desired_shader_state, + kbdev->shader_inuse_bitmap, + &shader_available_bitmap, + &kbdev->pm.backend.powering_on_shader_state); + + if (kbdev->shader_available_bitmap != shader_available_bitmap) { + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, + NULL, 0u, + (u32) shader_available_bitmap); + KBASE_TIMELINE_POWER_SHADER(kbdev, + shader_available_bitmap); + } + + kbdev->shader_available_bitmap = shader_available_bitmap; + + if (kbdev->tiler_available_bitmap != tiler_available_bitmap) { + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, + NULL, NULL, 0u, + (u32) tiler_available_bitmap); + KBASE_TIMELINE_POWER_TILER(kbdev, + tiler_available_bitmap); + } + + kbdev->tiler_available_bitmap = tiler_available_bitmap; + + } else if ((l2_available_bitmap & + kbdev->gpu_props.props.raw_props.tiler_present) != + kbdev->gpu_props.props.raw_props.tiler_present) { + tiler_available_bitmap = 0; + + if (kbdev->tiler_available_bitmap != tiler_available_bitmap) + KBASE_TIMELINE_POWER_TILER(kbdev, + tiler_available_bitmap); + + kbdev->tiler_available_bitmap = tiler_available_bitmap; + } + + /* State updated for slow-path waiters */ + kbdev->pm.backend.gpu_in_desired_state = in_desired_state; + + shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_SHADER); + shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_SHADER); + + /* Determine whether the cores are now available (even if the set of + * available cores is empty). Note that they can be available even if + * we've not finished transitioning to the desired state */ + if ((kbdev->shader_available_bitmap & + kbdev->pm.backend.desired_shader_state) + == kbdev->pm.backend.desired_shader_state && + (kbdev->tiler_available_bitmap & + kbdev->pm.backend.desired_tiler_state) + == kbdev->pm.backend.desired_tiler_state) { + cores_are_available = true; + + KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u, + (u32)(kbdev->shader_available_bitmap & + kbdev->pm.backend.desired_shader_state)); + KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u, + (u32)(kbdev->tiler_available_bitmap & + kbdev->pm.backend.desired_tiler_state)); + + /* Log timelining information about handling events that power + * up cores, to match up either with immediate submission either + * because cores already available, or from PM IRQ */ + if (!in_desired_state) + kbase_timeline_pm_send_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + } + + if (in_desired_state) { + KBASE_DEBUG_ASSERT(cores_are_available); + +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) + kbase_trace_mali_pm_status(KBASE_PM_CORE_L2, + kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_L2)); + kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER, + kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_SHADER)); + kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER, + kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_TILER)); +#ifdef CONFIG_MALI_CORESTACK + kbase_trace_mali_pm_status(KBASE_PM_CORE_STACK, + kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_STACK)); +#endif /* CONFIG_MALI_CORESTACK */ +#endif + + KBASE_TLSTREAM_AUX_PM_STATE( + KBASE_PM_CORE_L2, + kbase_pm_get_ready_cores( + kbdev, KBASE_PM_CORE_L2)); + KBASE_TLSTREAM_AUX_PM_STATE( + KBASE_PM_CORE_SHADER, + kbase_pm_get_ready_cores( + kbdev, KBASE_PM_CORE_SHADER)); + KBASE_TLSTREAM_AUX_PM_STATE( + KBASE_PM_CORE_TILER, + kbase_pm_get_ready_cores( + kbdev, + KBASE_PM_CORE_TILER)); +#ifdef CONFIG_MALI_CORESTACK + KBASE_TLSTREAM_AUX_PM_STATE( + KBASE_PM_CORE_STACK, + kbase_pm_get_ready_cores( + kbdev, + KBASE_PM_CORE_STACK)); +#endif /* CONFIG_MALI_CORESTACK */ + + KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL, + kbdev->pm.backend.gpu_in_desired_state, + (u32)kbdev->pm.backend.desired_shader_state); + KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u, + (u32)kbdev->pm.backend.desired_tiler_state); + + /* Log timelining information for synchronous waiters */ + kbase_timeline_pm_send_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + /* Wake slow-path waiters. Job scheduler does not use this. */ + KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0); + + wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); + } + + spin_unlock(&kbdev->pm.backend.gpu_powered_lock); + + /* kbase_pm_ca_update_core_status can cause one-level recursion into + * this function, so it must only be called once all changes to kbdev + * have been committed, and after the gpu_powered_lock has been + * dropped. */ + if (kbdev->shader_ready_bitmap != shader_ready_bitmap || + kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) { + kbdev->shader_ready_bitmap = shader_ready_bitmap; + kbdev->shader_transitioning_bitmap = + shader_transitioning_bitmap; + + kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap, + shader_transitioning_bitmap); + } + + /* The core availability policy is not allowed to keep core group 0 + * turned off (unless it was changing the l2 power state) */ + if (!((shader_ready_bitmap | shader_transitioning_bitmap) & + kbdev->gpu_props.props.coherency_info.group[0].core_mask) && + (prev_l2_available_bitmap == desired_l2_state) && + !(kbase_pm_ca_get_core_mask(kbdev) & + kbdev->gpu_props.props.coherency_info.group[0].core_mask)) + BUG(); + + /* The core availability policy is allowed to keep core group 1 off, + * but all jobs specifically targeting CG1 must fail */ + if (!((shader_ready_bitmap | shader_transitioning_bitmap) & + kbdev->gpu_props.props.coherency_info.group[1].core_mask) && + !(kbase_pm_ca_get_core_mask(kbdev) & + kbdev->gpu_props.props.coherency_info.group[1].core_mask)) + kbdev->pm.backend.cg1_disabled = true; + else + kbdev->pm.backend.cg1_disabled = false; + + return cores_are_available; +} +KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock); + +/* Timeout for kbase_pm_check_transitions_sync when wait_event_killable has + * aborted due to a fatal signal. If the time spent waiting has exceeded this + * threshold then there is most likely a hardware issue. */ +#define PM_TIMEOUT (5*HZ) /* 5s */ + +void kbase_pm_check_transitions_sync(struct kbase_device *kbdev) +{ + unsigned long flags; + unsigned long timeout; + bool cores_are_available; + int ret; + + /* Force the transition to be checked and reported - the cores may be + * 'available' (for job submission) but not fully powered up. */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + cores_are_available = kbase_pm_check_transitions_nolock(kbdev); + + /* Don't need 'cores_are_available', because we don't return anything */ + CSTD_UNUSED(cores_are_available); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + timeout = jiffies + PM_TIMEOUT; + + /* Wait for cores */ + ret = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait, + kbdev->pm.backend.gpu_in_desired_state); + + if (ret < 0 && time_after(jiffies, timeout)) { + dev_err(kbdev->dev, "Power transition timed out unexpectedly\n"); + dev_err(kbdev->dev, "Desired state :\n"); + dev_err(kbdev->dev, "\tShader=%016llx\n", + kbdev->pm.backend.desired_shader_state); + dev_err(kbdev->dev, "\tTiler =%016llx\n", + kbdev->pm.backend.desired_tiler_state); + dev_err(kbdev->dev, "Current state :\n"); + dev_err(kbdev->dev, "\tShader=%08x%08x\n", + kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_READY_HI), NULL), + kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_READY_LO), + NULL)); + dev_err(kbdev->dev, "\tTiler =%08x%08x\n", + kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_READY_HI), NULL), + kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_READY_LO), NULL)); + dev_err(kbdev->dev, "\tL2 =%08x%08x\n", + kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_READY_HI), NULL), + kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_READY_LO), NULL)); + dev_err(kbdev->dev, "Cores transitioning :\n"); + dev_err(kbdev->dev, "\tShader=%08x%08x\n", + kbase_reg_read(kbdev, GPU_CONTROL_REG( + SHADER_PWRTRANS_HI), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG( + SHADER_PWRTRANS_LO), NULL)); + dev_err(kbdev->dev, "\tTiler =%08x%08x\n", + kbase_reg_read(kbdev, GPU_CONTROL_REG( + TILER_PWRTRANS_HI), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG( + TILER_PWRTRANS_LO), NULL)); + dev_err(kbdev->dev, "\tL2 =%08x%08x\n", + kbase_reg_read(kbdev, GPU_CONTROL_REG( + L2_PWRTRANS_HI), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG( + L2_PWRTRANS_LO), NULL)); +#if KBASE_GPU_RESET_EN + dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n"); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); +#endif /* KBASE_GPU_RESET_EN */ + } else { + /* Log timelining information that a change in state has + * completed */ + kbase_timeline_pm_handle_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + } +} +KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync); + +void kbase_pm_enable_interrupts(struct kbase_device *kbdev) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + /* + * Clear all interrupts, + * and unmask them all. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, + NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL, + NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, + NULL); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL); +} + +KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); + +void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(NULL != kbdev); + /* + * Mask all interrupts, + * and clear them all. + */ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, + NULL); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, + NULL); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); +} + +void kbase_pm_disable_interrupts(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_disable_interrupts_nolock(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts); + + +/* + * pmu layout: + * 0x0000: PMU TAG (RO) (0xCAFECAFE) + * 0x0004: PMU VERSION ID (RO) (0x00000000) + * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE) + */ +void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) +{ + bool reset_required = is_resume; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->pm.lock); + + if (kbdev->pm.backend.gpu_powered) { + /* Already turned on */ + if (kbdev->poweroff_pending) + kbase_pm_enable_interrupts(kbdev); + kbdev->poweroff_pending = false; + KBASE_DEBUG_ASSERT(!is_resume); + return; + } + + kbdev->poweroff_pending = false; + + KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u); + + if (is_resume && kbdev->pm.backend.callback_power_resume) { + kbdev->pm.backend.callback_power_resume(kbdev); + return; + } else if (kbdev->pm.backend.callback_power_on) { + kbdev->pm.backend.callback_power_on(kbdev); + /* If your platform properly keeps the GPU state you may use the + * return value of the callback_power_on function to + * conditionally reset the GPU on power up. Currently we are + * conservative and always reset the GPU. */ + reset_required = true; + } + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + kbdev->pm.backend.gpu_powered = true; + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (reset_required) { + /* GPU state was lost, reset GPU to ensure it is in a + * consistent state */ + kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); + } + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_restore_all_as(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + /* Lastly, enable the interrupts */ + kbase_pm_enable_interrupts(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_clock_on); + +bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->pm.lock); + + /* ASSERT that the cores should now be unavailable. No lock needed. */ + KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u); + + kbdev->poweroff_pending = true; + + if (!kbdev->pm.backend.gpu_powered) { + /* Already turned off */ + if (is_suspend && kbdev->pm.backend.callback_power_suspend) + kbdev->pm.backend.callback_power_suspend(kbdev); + return true; + } + + KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u); + + /* Disable interrupts. This also clears any outstanding interrupts */ + kbase_pm_disable_interrupts(kbdev); + /* Ensure that any IRQ handlers have finished */ + kbase_synchronize_irqs(kbdev); + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (atomic_read(&kbdev->faults_pending)) { + /* Page/bus faults are still being processed. The GPU can not + * be powered off until they have completed */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return false; + } + + kbase_pm_cache_snoop_disable(kbdev); + + /* The GPU power may be turned off from this point */ + kbdev->pm.backend.gpu_powered = false; + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (is_suspend && kbdev->pm.backend.callback_power_suspend) + kbdev->pm.backend.callback_power_suspend(kbdev); + else if (kbdev->pm.backend.callback_power_off) + kbdev->pm.backend.callback_power_off(kbdev); + return true; +} + +KBASE_EXPORT_TEST_API(kbase_pm_clock_off); + +struct kbasep_reset_timeout_data { + struct hrtimer timer; + bool timed_out; + struct kbase_device *kbdev; +}; + +void kbase_pm_reset_done(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + kbdev->pm.backend.reset_done = true; + wake_up(&kbdev->pm.backend.reset_done_wait); +} + +/** + * kbase_pm_wait_for_reset - Wait for a reset to happen + * + * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state. + * + * @kbdev: Kbase device + */ +static void kbase_pm_wait_for_reset(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->pm.lock); + + wait_event(kbdev->pm.backend.reset_done_wait, + (kbdev->pm.backend.reset_done)); + kbdev->pm.backend.reset_done = false; +} + +KBASE_EXPORT_TEST_API(kbase_pm_reset_done); + +static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) +{ + struct kbasep_reset_timeout_data *rtdata = + container_of(timer, struct kbasep_reset_timeout_data, timer); + + rtdata->timed_out = 1; + + /* Set the wait queue to wake up kbase_pm_init_hw even though the reset + * hasn't completed */ + kbase_pm_reset_done(rtdata->kbdev); + + return HRTIMER_NORESTART; +} + +static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev) +{ + struct device_node *np = kbdev->dev->of_node; + u32 jm_values[4]; + const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT; + const u32 major = (gpu_id & GPU_ID_VERSION_MAJOR) >> + GPU_ID_VERSION_MAJOR_SHIFT; + + kbdev->hw_quirks_sc = 0; + + /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443. + * and needed due to MIDGLES-3539. See PRLAM-11035 */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) || + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035)) + kbdev->hw_quirks_sc |= SC_LS_PAUSEBUFFER_DISABLE; + + /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327. + */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327)) + kbdev->hw_quirks_sc |= SC_SDC_DISABLE_OQ_DISCARD; + +#ifdef CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY + /* Enable alternative hardware counter selection if configured. */ + if (!GPU_ID_IS_NEW_FORMAT(prod_id)) + kbdev->hw_quirks_sc |= SC_ALT_COUNTERS; +#endif + + /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797)) + kbdev->hw_quirks_sc |= SC_ENABLE_TEXGRD_FLAGS; + + if (!kbase_hw_has_issue(kbdev, GPUCORE_1619)) { + if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ + kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; + else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */ + kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; + } + + if (!kbdev->hw_quirks_sc) + kbdev->hw_quirks_sc = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_CONFIG), NULL); + + kbdev->hw_quirks_tiler = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_CONFIG), NULL); + + /* Set tiler clock gate override if required */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) + kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; + + /* Limit the GPU bus bandwidth if the platform needs this. */ + kbdev->hw_quirks_mmu = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_MMU_CONFIG), NULL); + + /* Limit read ID width for AXI */ + kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS); + kbdev->hw_quirks_mmu |= (DEFAULT_ARID_LIMIT & 0x3) << + L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT; + + /* Limit write ID width for AXI */ + kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES); + kbdev->hw_quirks_mmu |= (DEFAULT_AWID_LIMIT & 0x3) << + L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT; + + if (kbdev->system_coherency == COHERENCY_ACE) { + /* Allow memory configuration disparity to be ignored, we + * optimize the use of shared memory and thus we expect + * some disparity in the memory configuration */ + kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY; + } + + kbdev->hw_quirks_jm = 0; + /* Only for T86x/T88x-based products after r2p0 */ + if (prod_id >= 0x860 && prod_id <= 0x880 && major >= 2) { + + if (of_property_read_u32_array(np, + "jm_config", + &jm_values[0], + ARRAY_SIZE(jm_values))) { + /* Entry not in device tree, use defaults */ + jm_values[0] = 0; + jm_values[1] = 0; + jm_values[2] = 0; + jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT; + } + + /* Limit throttle limit to 6 bits*/ + if (jm_values[3] > JM_MAX_JOB_THROTTLE_LIMIT) { + dev_dbg(kbdev->dev, "JOB_THROTTLE_LIMIT supplied in device tree is too large. Limiting to MAX (63)."); + jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT; + } + + /* Aggregate to one integer. */ + kbdev->hw_quirks_jm |= (jm_values[0] ? + JM_TIMESTAMP_OVERRIDE : 0); + kbdev->hw_quirks_jm |= (jm_values[1] ? + JM_CLOCK_GATE_OVERRIDE : 0); + kbdev->hw_quirks_jm |= (jm_values[2] ? + JM_JOB_THROTTLE_ENABLE : 0); + kbdev->hw_quirks_jm |= (jm_values[3] << + JM_JOB_THROTTLE_LIMIT_SHIFT); + + } else if (GPU_ID_IS_NEW_FORMAT(prod_id) && + (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == + GPU_ID2_PRODUCT_TMIX)) { + /* Only for tMIx */ + u32 coherency_features; + + coherency_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(COHERENCY_FEATURES), NULL); + + /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly + * documented for tMIx so force correct value here. + */ + if (coherency_features == + COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { + kbdev->hw_quirks_jm |= + (COHERENCY_ACE_LITE | COHERENCY_ACE) << + JM_FORCE_COHERENCY_FEATURES_SHIFT; + } + } + + + if (!kbdev->hw_quirks_jm) + kbdev->hw_quirks_jm = kbase_reg_read(kbdev, + GPU_CONTROL_REG(JM_CONFIG), NULL); + +#ifdef CONFIG_MALI_CORESTACK +#define MANUAL_POWER_CONTROL ((u32)(1 << 8)) + kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL; +#endif /* CONFIG_MALI_CORESTACK */ +} + +static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) +{ + kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), + kbdev->hw_quirks_sc, NULL); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG), + kbdev->hw_quirks_tiler, NULL); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), + kbdev->hw_quirks_mmu, NULL); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG), + kbdev->hw_quirks_jm, NULL); + +} + +void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) +{ + if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && + !kbdev->cci_snoop_enabled) { +#ifdef CONFIG_ARM64 + if (kbdev->snoop_enable_smc != 0) + kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0); +#endif /* CONFIG_ARM64 */ + dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n"); + kbdev->cci_snoop_enabled = true; + } +} + +void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) +{ + if (kbdev->cci_snoop_enabled) { +#ifdef CONFIG_ARM64 + if (kbdev->snoop_disable_smc != 0) { + mali_cci_flush_l2(kbdev); + kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0); + } +#endif /* CONFIG_ARM64 */ + dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n"); + kbdev->cci_snoop_enabled = false; + } +} + +static int kbase_pm_do_reset(struct kbase_device *kbdev) +{ + struct kbasep_reset_timeout_data rtdata; + + KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0); + + KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_SOFT_RESET, NULL); + + /* Unmask the reset complete interrupt only */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED, + NULL); + + /* Initialize a structure for tracking the status of the reset */ + rtdata.kbdev = kbdev; + rtdata.timed_out = 0; + + /* Create a timer to use as a timeout on the reset */ + hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rtdata.timer.function = kbasep_reset_timeout; + + hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), + HRTIMER_MODE_REL); + + /* Wait for the RESET_COMPLETED interrupt to be raised */ + kbase_pm_wait_for_reset(kbdev); + + if (rtdata.timed_out == 0) { + /* GPU has been reset */ + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + return 0; + } + + /* No interrupt has been received - check if the RAWSTAT register says + * the reset has completed */ + if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & + RESET_COMPLETED) { + /* The interrupt is set in the RAWSTAT; this suggests that the + * interrupts are not getting to the CPU */ + dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); + /* If interrupts aren't working we can't continue. */ + destroy_hrtimer_on_stack(&rtdata.timer); + return -EINVAL; + } + + /* The GPU doesn't seem to be responding to the reset so try a hard + * reset */ + dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", + RESET_TIMEOUT); + KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_HARD_RESET, NULL); + + /* Restart the timer to wait for the hard reset to complete */ + rtdata.timed_out = 0; + + hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), + HRTIMER_MODE_REL); + + /* Wait for the RESET_COMPLETED interrupt to be raised */ + kbase_pm_wait_for_reset(kbdev); + + if (rtdata.timed_out == 0) { + /* GPU has been reset */ + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + return 0; + } + + destroy_hrtimer_on_stack(&rtdata.timer); + + dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", + RESET_TIMEOUT); + + return -EINVAL; +} + +static int kbasep_protected_mode_enable(struct protected_mode_device *pdev) +{ + struct kbase_device *kbdev = pdev->data; + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_SET_PROTECTED_MODE, NULL); + return 0; +} + +static int kbasep_protected_mode_disable(struct protected_mode_device *pdev) +{ + struct kbase_device *kbdev = pdev->data; + + lockdep_assert_held(&kbdev->pm.lock); + + return kbase_pm_do_reset(kbdev); +} + +struct protected_mode_ops kbase_native_protected_ops = { + .protected_mode_enable = kbasep_protected_mode_enable, + .protected_mode_disable = kbasep_protected_mode_disable +}; + +int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) +{ + unsigned long irq_flags; + int err; + bool resume_vinstr = false; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->pm.lock); + + /* Ensure the clock is on before attempting to access the hardware */ + if (!kbdev->pm.backend.gpu_powered) { + if (kbdev->pm.backend.callback_power_on) + kbdev->pm.backend.callback_power_on(kbdev); + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, + irq_flags); + kbdev->pm.backend.gpu_powered = true; + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + irq_flags); + } + + /* Ensure interrupts are off to begin with, this also clears any + * outstanding interrupts */ + kbase_pm_disable_interrupts(kbdev); + /* Ensure cache snoops are disabled before reset. */ + kbase_pm_cache_snoop_disable(kbdev); + /* Prepare for the soft-reset */ + kbdev->pm.backend.reset_done = false; + + /* The cores should be made unavailable due to the reset */ + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + if (kbdev->shader_available_bitmap != 0u) + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, + NULL, 0u, (u32)0u); + if (kbdev->tiler_available_bitmap != 0u) + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, + NULL, NULL, 0u, (u32)0u); + kbdev->shader_available_bitmap = 0u; + kbdev->tiler_available_bitmap = 0u; + kbdev->l2_available_bitmap = 0u; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); + + /* Soft reset the GPU */ + if (kbdev->protected_mode_support) + err = kbdev->protected_ops->protected_mode_disable( + kbdev->protected_dev); + else + err = kbase_pm_do_reset(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + if (kbdev->protected_mode) + resume_vinstr = true; + kbdev->protected_mode = false; + kbase_ipa_model_use_configured_locked(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); + + if (err) + goto exit; + + if (flags & PM_HW_ISSUES_DETECT) + kbase_pm_hw_issues_detect(kbdev); + + kbase_pm_hw_issues_apply(kbdev); + kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); + + /* Sanity check protected mode was left after reset */ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { + u32 gpu_status = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_STATUS), NULL); + + WARN_ON(gpu_status & GPU_STATUS_PROTECTED_MODE_ACTIVE); + } + + /* If cycle counter was in use re-enable it, enable_irqs will only be + * false when called from kbase_pm_powerup */ + if (kbdev->pm.backend.gpu_cycle_counter_requests && + (flags & PM_ENABLE_IRQS)) { + /* enable interrupts as the L2 may have to be powered on */ + kbase_pm_enable_interrupts(kbdev); + kbase_pm_request_l2_caches(kbdev); + + /* Re-enable the counters if we need to */ + spin_lock_irqsave( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + if (kbdev->pm.backend.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CYCLE_COUNT_START, NULL); + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + kbase_pm_release_l2_caches(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); + + kbase_pm_disable_interrupts(kbdev); + } + + if (flags & PM_ENABLE_IRQS) + kbase_pm_enable_interrupts(kbdev); + +exit: + /* If GPU is leaving protected mode resume vinstr operation. */ + if (kbdev->vinstr_ctx && resume_vinstr) + kbase_vinstr_resume(kbdev->vinstr_ctx); + + return err; +} + +/** + * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters + * + * Increase the count of cycle counter users and turn the cycle counters on if + * they were previously off + * + * This function is designed to be called by + * kbase_pm_request_gpu_cycle_counter() or + * kbase_pm_request_gpu_cycle_counter_l2_is_on() only + * + * When this function is called the l2 cache must be on and the l2 cache users + * count must have been incremented by a call to ( + * kbase_pm_request_l2_caches() or kbase_pm_request_l2_caches_l2_on() ) + * + * @kbdev: The kbase device structure of the device + */ +static void +kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); + + ++kbdev->pm.backend.gpu_cycle_counter_requests; + + if (1 == kbdev->pm.backend.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CYCLE_COUNT_START, NULL); + + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); +} + +void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < + INT_MAX); + + kbase_pm_request_l2_caches(kbdev); + + kbase_pm_request_gpu_cycle_counter_do_request(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter); + +void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < + INT_MAX); + + kbase_pm_request_l2_caches_l2_is_on(kbdev); + + kbase_pm_request_gpu_cycle_counter_do_request(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on); + +void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0); + + --kbdev->pm.backend.gpu_cycle_counter_requests; + + if (0 == kbdev->pm.backend.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CYCLE_COUNT_STOP, NULL); + + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); + + kbase_pm_release_l2_caches(kbdev); +} + +void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_pm_release_gpu_cycle_counter_nolock(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter); diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_internal.h new file mode 100755 index 000000000000..9fbe094541c5 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_internal.h @@ -0,0 +1,548 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Power management API definitions used internally by GPU backend + */ + +#ifndef _KBASE_BACKEND_PM_INTERNAL_H_ +#define _KBASE_BACKEND_PM_INTERNAL_H_ + +#include + +#include "mali_kbase_pm_ca.h" +#include "mali_kbase_pm_policy.h" + + +/** + * kbase_pm_dev_idle - The GPU is idle. + * + * The OS may choose to turn off idle devices + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_dev_idle(struct kbase_device *kbdev); + +/** + * kbase_pm_dev_activate - The GPU is active. + * + * The OS should avoid opportunistically turning off the GPU while it is active + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_dev_activate(struct kbase_device *kbdev); + +/** + * kbase_pm_get_present_cores - Get details of the cores that are present in + * the device. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) present in the GPU device and also a count of + * the number of cores. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of cores present + */ +u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_get_active_cores - Get details of the cores that are currently + * active in the device. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) that are actively processing work (i.e. + * turned on *and* busy). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of active cores + */ +u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_get_trans_cores - Get details of the cores that are currently + * transitioning between power states. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) that are currently transitioning between + * power states. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of transitioning cores + */ +u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_get_ready_cores - Get details of the cores that are currently + * powered and ready for jobs. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) that are powered and ready for jobs (they may + * or may not be currently executing jobs). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of ready cores + */ +u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_clock_on - Turn the clock for the device on, and enable device + * interrupts. + * + * This function can be used by a power policy to turn the clock for the GPU on. + * It should be modified during integration to perform the necessary actions to + * ensure that the GPU is fully powered and clocked. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @is_resume: true if clock on due to resume after suspend, false otherwise + */ +void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume); + +/** + * kbase_pm_clock_off - Disable device interrupts, and turn the clock for the + * device off. + * + * This function can be used by a power policy to turn the clock for the GPU + * off. It should be modified during integration to perform the necessary + * actions to turn the clock off (if this is possible in the integration). + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @is_suspend: true if clock off due to suspend, false otherwise + * + * Return: true if clock was turned off, or + * false if clock can not be turned off due to pending page/bus fault + * workers. Caller must flush MMU workqueues and retry + */ +bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend); + +/** + * kbase_pm_enable_interrupts - Enable interrupts on the device. + * + * Interrupts are also enabled after a call to kbase_pm_clock_on(). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_enable_interrupts(struct kbase_device *kbdev); + +/** + * kbase_pm_disable_interrupts - Disable interrupts on the device. + * + * This prevents delivery of Power Management interrupts to the CPU so that + * kbase_pm_check_transitions_nolock() will not be called from the IRQ handler + * until kbase_pm_enable_interrupts() or kbase_pm_clock_on() is called. + * + * Interrupts are also disabled after a call to kbase_pm_clock_off(). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_disable_interrupts(struct kbase_device *kbdev); + +/** + * kbase_pm_disable_interrupts_nolock - Version of kbase_pm_disable_interrupts() + * that does not take the hwaccess_lock + * + * Caller must hold the hwaccess_lock. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_init_hw - Initialize the hardware. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @flags: Flags specifying the type of PM init + * + * This function checks the GPU ID register to ensure that the GPU is supported + * by the driver and performs a reset on the device so that it is in a known + * state before the device is used. + * + * Return: 0 if the device is supported and successfully reset. + */ +int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags); + +/** + * kbase_pm_reset_done - The GPU has been reset successfully. + * + * This function must be called by the GPU interrupt handler when the + * RESET_COMPLETED bit is set. It signals to the power management initialization + * code that the GPU has been successfully reset. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_reset_done(struct kbase_device *kbdev); + + +/** + * kbase_pm_check_transitions_nolock - Check if there are any power transitions + * to make, and if so start them. + * + * This function will check the desired_xx_state members of + * struct kbase_pm_device_data and the actual status of the hardware to see if + * any power transitions can be made at this time to make the hardware state + * closer to the state desired by the power policy. + * + * The return value can be used to check whether all the desired cores are + * available, and so whether it's worth submitting a job (e.g. from a Power + * Management IRQ). + * + * Note that this still returns true when desired_xx_state has no + * cores. That is: of the no cores desired, none were *un*available. In + * this case, the caller may still need to try submitting jobs. This is because + * the Core Availability Policy might have taken us to an intermediate state + * where no cores are powered, before powering on more cores (e.g. for core + * rotation) + * + * The caller must hold kbase_device.pm.power_change_lock + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: non-zero when all desired cores are available. That is, + * it's worthwhile for the caller to submit a job. + * false otherwise + */ +bool kbase_pm_check_transitions_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_check_transitions_sync - Synchronous and locking variant of + * kbase_pm_check_transitions_nolock() + * + * On returning, the desired state at the time of the call will have been met. + * + * There is nothing to stop the core being switched off by calls to + * kbase_pm_release_cores() or kbase_pm_unrequest_cores(). Therefore, the + * caller must have already made a call to + * kbase_pm_request_cores()/kbase_pm_request_cores_sync() previously. + * + * The usual use-case for this is to ensure cores are 'READY' after performing + * a GPU Reset. + * + * Unlike kbase_pm_check_transitions_nolock(), the caller must not hold + * kbase_device.pm.power_change_lock, because this function will take that + * lock itself. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_check_transitions_sync(struct kbase_device *kbdev); + +/** + * kbase_pm_update_cores_state_nolock - Variant of kbase_pm_update_cores_state() + * where the caller must hold + * kbase_device.pm.power_change_lock + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_update_cores_state - Update the desired state of shader cores from + * the Power Policy, and begin any power + * transitions. + * + * This function will update the desired_xx_state members of + * struct kbase_pm_device_data by calling into the current Power Policy. It will + * then begin power transitions to make the hardware acheive the desired shader + * core state. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_update_cores_state(struct kbase_device *kbdev); + +/** + * kbase_pm_cancel_deferred_poweroff - Cancel any pending requests to power off + * the GPU and/or shader cores. + * + * This should be called by any functions which directly power off the GPU. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev); + +/** + * kbasep_pm_init_core_use_bitmaps - Initialise data tracking the required + * and used cores. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev); + +/** + * kbasep_pm_metrics_init - Initialize the metrics gathering framework. + * + * This must be called before other metric gathering APIs are called. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 on success, error code on error + */ +int kbasep_pm_metrics_init(struct kbase_device *kbdev); + +/** + * kbasep_pm_metrics_term - Terminate the metrics gathering framework. + * + * This must be called when metric gathering is no longer required. It is an + * error to call any metrics gathering function (other than + * kbasep_pm_metrics_init()) after calling this function. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbasep_pm_metrics_term(struct kbase_device *kbdev); + +/** + * kbase_pm_report_vsync - Function to be called by the frame buffer driver to + * update the vsync metric. + * + * This function should be called by the frame buffer driver to update whether + * the system is hitting the vsync target or not. buffer_updated should be true + * if the vsync corresponded with a new frame being displayed, otherwise it + * should be false. This function does not need to be called every vsync, but + * only when the value of @buffer_updated differs from a previous call. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * @buffer_updated: True if the buffer has been updated on this VSync, + * false otherwise + */ +void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated); + +/** + * kbase_pm_get_dvfs_action - Determine whether the DVFS system should change + * the clock speed of the GPU. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This function should be called regularly by the DVFS system to check whether + * the clock speed of the GPU needs updating. + */ +void kbase_pm_get_dvfs_action(struct kbase_device *kbdev); + +/** + * kbase_pm_request_gpu_cycle_counter - Mark that the GPU cycle counter is + * needed + * + * If the caller is the first caller then the GPU cycle counters will be enabled + * along with the l2 cache + * + * The GPU must be powered when calling this function (i.e. + * kbase_pm_context_active() must have been called). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev); + +/** + * kbase_pm_request_gpu_cycle_counter_l2_is_on - Mark GPU cycle counter is + * needed (l2 cache already on) + * + * This is a version of the above function + * (kbase_pm_request_gpu_cycle_counter()) suitable for being called when the + * l2 cache is known to be on and assured to be on until the subsequent call of + * kbase_pm_release_gpu_cycle_counter() such as when a job is submitted. It does + * not sleep and can be called from atomic functions. + * + * The GPU must be powered when calling this function (i.e. + * kbase_pm_context_active() must have been called) and the l2 cache must be + * powered on. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev); + +/** + * kbase_pm_release_gpu_cycle_counter - Mark that the GPU cycle counter is no + * longer in use + * + * If the caller is the last caller then the GPU cycle counters will be + * disabled. A request must have been made before a call to this. + * + * Caller must not hold the hwaccess_lock, as it will be taken in this function. + * If the caller is already holding this lock then + * kbase_pm_release_gpu_cycle_counter_nolock() must be used instead. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev); + +/** + * kbase_pm_release_gpu_cycle_counter_nolock - Version of kbase_pm_release_gpu_cycle_counter() + * that does not take hwaccess_lock + * + * Caller must hold the hwaccess_lock. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_wait_for_poweroff_complete - Wait for the poweroff workqueue to + * complete + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev); + +/** + * kbase_pm_register_access_enable - Enable access to GPU registers + * + * Enables access to the GPU registers before power management has powered up + * the GPU with kbase_pm_powerup(). + * + * Access to registers should be done using kbase_os_reg_read()/write() at this + * stage, not kbase_reg_read()/write(). + * + * This results in the power management callbacks provided in the driver + * configuration to get called to turn on power and/or clocks to the GPU. See + * kbase_pm_callback_conf. + * + * This should only be used before power management is powered up with + * kbase_pm_powerup() + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_register_access_enable(struct kbase_device *kbdev); + +/** + * kbase_pm_register_access_disable - Disable early register access + * + * Disables access to the GPU registers enabled earlier by a call to + * kbase_pm_register_access_enable(). + * + * This results in the power management callbacks provided in the driver + * configuration to get called to turn off power and/or clocks to the GPU. See + * kbase_pm_callback_conf + * + * This should only be used before power management is powered up with + * kbase_pm_powerup() + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_register_access_disable(struct kbase_device *kbdev); + +/* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline + * function */ + +/** + * kbase_pm_metrics_is_active - Check if the power management metrics + * collection is active. + * + * Note that this returns if the power management metrics collection was + * active at the time of calling, it is possible that after the call the metrics + * collection enable may have changed state. + * + * The caller must handle the consequence that the state may have changed. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * Return: true if metrics collection was active else false. + */ +bool kbase_pm_metrics_is_active(struct kbase_device *kbdev); + +/** + * kbase_pm_do_poweron - Power on the GPU, and any cores that are requested. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @is_resume: true if power on due to resume after suspend, + * false otherwise + */ +void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume); + +/** + * kbase_pm_do_poweroff - Power off the GPU, and any cores that have been + * requested. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @is_suspend: true if power off due to suspend, + * false otherwise + */ +void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend); + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) +void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, + unsigned long *total, unsigned long *busy); +void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev); +#endif /* defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) */ + +#ifdef CONFIG_MALI_BIFROST_DVFS + +/** + * kbase_platform_dvfs_event - Report utilisation to DVFS code + * + * Function provided by platform specific code when DVFS is enabled to allow + * the power management metrics system to report utilisation. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * @utilisation: The current calculated utilisation by the metrics system. + * @util_gl_share: The current calculated gl share of utilisation. + * @util_cl_share: The current calculated cl share of utilisation per core + * group. + * Return: Returns 0 on failure and non zero on success. + */ + +int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, + u32 util_gl_share, u32 util_cl_share[2]); +#endif + +void kbase_pm_power_changed(struct kbase_device *kbdev); + +/** + * kbase_pm_metrics_update - Inform the metrics system that an atom is either + * about to be run or has just completed. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @now: Pointer to the timestamp of the change, or NULL to use current time + * + * Caller must hold hwaccess_lock + */ +void kbase_pm_metrics_update(struct kbase_device *kbdev, + ktime_t *now); + +/** + * kbase_pm_cache_snoop_enable - Allow CPU snoops on the GPU + * If the GPU does not have coherency this is a no-op + * @kbdev: Device pointer + * + * This function should be called after L2 power up. + */ + +void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev); + +/** + * kbase_pm_cache_snoop_disable - Prevent CPU snoops on the GPU + * If the GPU does not have coherency this is a no-op + * @kbdev: Device pointer + * + * This function should be called before L2 power off. + */ +void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev); + +#endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_metrics.c new file mode 100755 index 000000000000..ba13bcd8b291 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_metrics.c @@ -0,0 +1,401 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Metrics for power management + */ + +#include +#include +#include +#include + +/* When VSync is being hit aim for utilisation between 70-90% */ +#define KBASE_PM_VSYNC_MIN_UTILISATION 70 +#define KBASE_PM_VSYNC_MAX_UTILISATION 90 +/* Otherwise aim for 10-40% */ +#define KBASE_PM_NO_VSYNC_MIN_UTILISATION 10 +#define KBASE_PM_NO_VSYNC_MAX_UTILISATION 40 + +/* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns + * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly + * under 11s. Exceeding this will cause overflow */ +#define KBASE_PM_TIME_SHIFT 8 + +/* Maximum time between sampling of utilization data, without resetting the + * counters. */ +#define MALI_UTILIZATION_MAX_PERIOD 100000 /* ns = 100ms */ + +#ifdef CONFIG_MALI_BIFROST_DVFS +static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) +{ + unsigned long flags; + struct kbasep_pm_metrics_data *metrics; + + KBASE_DEBUG_ASSERT(timer != NULL); + + metrics = container_of(timer, struct kbasep_pm_metrics_data, timer); + kbase_pm_get_dvfs_action(metrics->kbdev); + + spin_lock_irqsave(&metrics->lock, flags); + + if (metrics->timer_active) + hrtimer_start(timer, + HR_TIMER_DELAY_MSEC(metrics->kbdev->pm.dvfs_period), + HRTIMER_MODE_REL); + + spin_unlock_irqrestore(&metrics->lock, flags); + + return HRTIMER_NORESTART; +} +#endif /* CONFIG_MALI_BIFROST_DVFS */ + +int kbasep_pm_metrics_init(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + kbdev->pm.backend.metrics.kbdev = kbdev; + + kbdev->pm.backend.metrics.time_period_start = ktime_get(); + kbdev->pm.backend.metrics.time_busy = 0; + kbdev->pm.backend.metrics.time_idle = 0; + kbdev->pm.backend.metrics.prev_busy = 0; + kbdev->pm.backend.metrics.prev_idle = 0; + kbdev->pm.backend.metrics.gpu_active = false; + kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; + kbdev->pm.backend.metrics.busy_cl[0] = 0; + kbdev->pm.backend.metrics.busy_cl[1] = 0; + kbdev->pm.backend.metrics.busy_gl = 0; + + spin_lock_init(&kbdev->pm.backend.metrics.lock); + +#ifdef CONFIG_MALI_BIFROST_DVFS + kbdev->pm.backend.metrics.timer_active = true; + hrtimer_init(&kbdev->pm.backend.metrics.timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + kbdev->pm.backend.metrics.timer.function = dvfs_callback; + + hrtimer_start(&kbdev->pm.backend.metrics.timer, + HR_TIMER_DELAY_MSEC(kbdev->pm.dvfs_period), + HRTIMER_MODE_REL); +#endif /* CONFIG_MALI_BIFROST_DVFS */ + + return 0; +} + +KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init); + +void kbasep_pm_metrics_term(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_BIFROST_DVFS + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + kbdev->pm.backend.metrics.timer_active = false; + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); + + hrtimer_cancel(&kbdev->pm.backend.metrics.timer); +#endif /* CONFIG_MALI_BIFROST_DVFS */ +} + +KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); + +/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this + * function + */ +static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, + ktime_t now) +{ + ktime_t diff; + + lockdep_assert_held(&kbdev->pm.backend.metrics.lock); + + diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start); + if (ktime_to_ns(diff) < 0) + return; + + if (kbdev->pm.backend.metrics.gpu_active) { + u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); + + kbdev->pm.backend.metrics.time_busy += ns_time; + if (kbdev->pm.backend.metrics.active_cl_ctx[0]) + kbdev->pm.backend.metrics.busy_cl[0] += ns_time; + if (kbdev->pm.backend.metrics.active_cl_ctx[1]) + kbdev->pm.backend.metrics.busy_cl[1] += ns_time; + if (kbdev->pm.backend.metrics.active_gl_ctx[0]) + kbdev->pm.backend.metrics.busy_gl += ns_time; + if (kbdev->pm.backend.metrics.active_gl_ctx[1]) + kbdev->pm.backend.metrics.busy_gl += ns_time; + } else { + kbdev->pm.backend.metrics.time_idle += (u32) (ktime_to_ns(diff) + >> KBASE_PM_TIME_SHIFT); + } + + kbdev->pm.backend.metrics.time_period_start = now; +} + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) +/* Caller needs to hold kbdev->pm.backend.metrics.lock before calling this + * function. + */ +static void kbase_pm_reset_dvfs_utilisation_unlocked(struct kbase_device *kbdev, + ktime_t now) +{ + /* Store previous value */ + kbdev->pm.backend.metrics.prev_idle = + kbdev->pm.backend.metrics.time_idle; + kbdev->pm.backend.metrics.prev_busy = + kbdev->pm.backend.metrics.time_busy; + + /* Reset current values */ + kbdev->pm.backend.metrics.time_period_start = now; + kbdev->pm.backend.metrics.time_idle = 0; + kbdev->pm.backend.metrics.time_busy = 0; + kbdev->pm.backend.metrics.busy_cl[0] = 0; + kbdev->pm.backend.metrics.busy_cl[1] = 0; + kbdev->pm.backend.metrics.busy_gl = 0; +} + +void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, ktime_get()); + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); +} + +void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, + unsigned long *total_out, unsigned long *busy_out) +{ + ktime_t now = ktime_get(); + unsigned long flags, busy, total; + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + kbase_pm_get_dvfs_utilisation_calc(kbdev, now); + + busy = kbdev->pm.backend.metrics.time_busy; + total = busy + kbdev->pm.backend.metrics.time_idle; + + /* Reset stats if older than MALI_UTILIZATION_MAX_PERIOD (default + * 100ms) */ + if (total >= MALI_UTILIZATION_MAX_PERIOD) { + kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); + } else if (total < (MALI_UTILIZATION_MAX_PERIOD / 2)) { + total += kbdev->pm.backend.metrics.prev_idle + + kbdev->pm.backend.metrics.prev_busy; + busy += kbdev->pm.backend.metrics.prev_busy; + } + + *total_out = total; + *busy_out = busy; + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); +} +#endif + +#ifdef CONFIG_MALI_BIFROST_DVFS + +/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this + * function + */ +int kbase_pm_get_dvfs_utilisation_old(struct kbase_device *kbdev, + int *util_gl_share, + int util_cl_share[2], + ktime_t now) +{ + int utilisation; + int busy; + + kbase_pm_get_dvfs_utilisation_calc(kbdev, now); + + if (kbdev->pm.backend.metrics.time_idle + + kbdev->pm.backend.metrics.time_busy == 0) { + /* No data - so we return NOP */ + utilisation = -1; + if (util_gl_share) + *util_gl_share = -1; + if (util_cl_share) { + util_cl_share[0] = -1; + util_cl_share[1] = -1; + } + goto out; + } + + utilisation = (100 * kbdev->pm.backend.metrics.time_busy) / + (kbdev->pm.backend.metrics.time_idle + + kbdev->pm.backend.metrics.time_busy); + + busy = kbdev->pm.backend.metrics.busy_gl + + kbdev->pm.backend.metrics.busy_cl[0] + + kbdev->pm.backend.metrics.busy_cl[1]; + + if (busy != 0) { + if (util_gl_share) + *util_gl_share = + (100 * kbdev->pm.backend.metrics.busy_gl) / + busy; + if (util_cl_share) { + util_cl_share[0] = + (100 * kbdev->pm.backend.metrics.busy_cl[0]) / + busy; + util_cl_share[1] = + (100 * kbdev->pm.backend.metrics.busy_cl[1]) / + busy; + } + } else { + if (util_gl_share) + *util_gl_share = -1; + if (util_cl_share) { + util_cl_share[0] = -1; + util_cl_share[1] = -1; + } + } + +out: + return utilisation; +} + +void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) +{ + unsigned long flags; + int utilisation, util_gl_share; + int util_cl_share[2]; + ktime_t now; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + + now = ktime_get(); + + utilisation = kbase_pm_get_dvfs_utilisation_old(kbdev, &util_gl_share, + util_cl_share, now); + + if (utilisation < 0 || util_gl_share < 0 || util_cl_share[0] < 0 || + util_cl_share[1] < 0) { + utilisation = 0; + util_gl_share = 0; + util_cl_share[0] = 0; + util_cl_share[1] = 0; + goto out; + } + +out: +#ifdef CONFIG_MALI_BIFROST_DVFS + kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, + util_cl_share); +#endif /*CONFIG_MALI_BIFROST_DVFS */ + + kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); + + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); +} + +bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) +{ + bool isactive; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + isactive = kbdev->pm.backend.metrics.timer_active; + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); + + return isactive; +} +KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active); + +#endif /* CONFIG_MALI_BIFROST_DVFS */ + +/** + * kbase_pm_metrics_active_calc - Update PM active counts based on currently + * running atoms + * @kbdev: Device pointer + * + * The caller must hold kbdev->pm.backend.metrics.lock + */ +static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) +{ + int js; + + lockdep_assert_held(&kbdev->pm.backend.metrics.lock); + + kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; + kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; + kbdev->pm.backend.metrics.gpu_active = false; + + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); + + /* Head atom may have just completed, so if it isn't running + * then try the next atom */ + if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) + katom = kbase_gpu_inspect(kbdev, js, 1); + + if (katom && katom->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED) { + if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { + int device_nr = (katom->core_req & + BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) + ? katom->device_nr : 0; + if (!WARN_ON(device_nr >= 2)) + kbdev->pm.backend.metrics. + active_cl_ctx[device_nr] = 1; + } else { + /* Slot 2 should not be running non-compute + * atoms */ + if (!WARN_ON(js >= 2)) + kbdev->pm.backend.metrics. + active_gl_ctx[js] = 1; + } + kbdev->pm.backend.metrics.gpu_active = true; + } + } +} + +/* called when job is submitted to or removed from a GPU slot */ +void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) +{ + unsigned long flags; + ktime_t now; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + + if (!timestamp) { + now = ktime_get(); + timestamp = &now; + } + + /* Track how long CL and/or GL jobs have been busy for */ + kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); + + kbase_pm_metrics_active_calc(kbdev); + + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.c new file mode 100755 index 000000000000..b98c68d9a42a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.c @@ -0,0 +1,973 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Power policy API implementations + */ + +#include +#include +#include +#include +#include + +static const struct kbase_pm_policy *const policy_list[] = { +#ifdef CONFIG_MALI_BIFROST_NO_MALI + &kbase_pm_always_on_policy_ops, + &kbase_pm_demand_policy_ops, + &kbase_pm_coarse_demand_policy_ops, +#if !MALI_CUSTOMER_RELEASE + &kbase_pm_demand_always_powered_policy_ops, + &kbase_pm_fast_start_policy_ops, +#endif +#else /* CONFIG_MALI_BIFROST_NO_MALI */ +#if !PLATFORM_POWER_DOWN_ONLY + &kbase_pm_demand_policy_ops, +#endif /* !PLATFORM_POWER_DOWN_ONLY */ + &kbase_pm_coarse_demand_policy_ops, + &kbase_pm_always_on_policy_ops, +#if !MALI_CUSTOMER_RELEASE +#if !PLATFORM_POWER_DOWN_ONLY + &kbase_pm_demand_always_powered_policy_ops, + &kbase_pm_fast_start_policy_ops, +#endif /* !PLATFORM_POWER_DOWN_ONLY */ +#endif +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ +}; + +/* The number of policies available in the system. + * This is derived from the number of functions listed in policy_get_functions. + */ +#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list)) + + +/* Function IDs for looking up Timeline Trace codes in + * kbase_pm_change_state_trace_code */ +enum kbase_pm_func_id { + KBASE_PM_FUNC_ID_REQUEST_CORES_START, + KBASE_PM_FUNC_ID_REQUEST_CORES_END, + KBASE_PM_FUNC_ID_RELEASE_CORES_START, + KBASE_PM_FUNC_ID_RELEASE_CORES_END, + /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither + * expect to hit it nor tend to hit it very much anyway. We can detect + * whether we need more instrumentation by a difference between + * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */ + + /* Must be the last */ + KBASE_PM_FUNC_ID_COUNT +}; + + +/* State changes during request/unrequest/release-ing cores */ +enum { + KBASE_PM_CHANGE_STATE_SHADER = (1u << 0), + KBASE_PM_CHANGE_STATE_TILER = (1u << 1), + + /* These two must be last */ + KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER | + KBASE_PM_CHANGE_STATE_SHADER), + KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1 +}; +typedef u32 kbase_pm_change_state; + + +#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE +/* Timeline Trace code lookups for each function */ +static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT] + [KBASE_PM_CHANGE_STATE_COUNT] = { + /* kbase_pm_request_cores */ + [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0, + [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START, + [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START, + [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER | + KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START, + + [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0, + [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END, + [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END, + [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER | + KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END, + + /* kbase_pm_release_cores */ + [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0, + [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START, + [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START, + [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER | + KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START, + + [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0, + [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END, + [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END, + [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER | + KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END +}; + +static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev, + enum kbase_pm_func_id func_id, + kbase_pm_change_state state) +{ + int trace_code; + + KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT); + KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) == + state); + + trace_code = kbase_pm_change_state_trace_code[func_id][state]; + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code); +} + +#else /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ +static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev, + enum kbase_pm_func_id func_id, kbase_pm_change_state state) +{ +} + +#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ + +/** + * kbasep_pm_do_poweroff_cores - Process a poweroff request and power down any + * requested shader cores + * @kbdev: Device pointer + */ +static void kbasep_pm_do_poweroff_cores(struct kbase_device *kbdev) +{ + u64 prev_shader_state = kbdev->pm.backend.desired_shader_state; + u64 prev_tiler_state = kbdev->pm.backend.desired_tiler_state; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->pm.backend.desired_shader_state &= + ~kbdev->pm.backend.shader_poweroff_pending; + kbdev->pm.backend.desired_tiler_state &= + ~kbdev->pm.backend.tiler_poweroff_pending; + + kbdev->pm.backend.shader_poweroff_pending = 0; + kbdev->pm.backend.tiler_poweroff_pending = 0; + + if (prev_shader_state != kbdev->pm.backend.desired_shader_state || + prev_tiler_state != + kbdev->pm.backend.desired_tiler_state || + kbdev->pm.backend.ca_in_transition) { + bool cores_are_available; + + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START); + cores_are_available = kbase_pm_check_transitions_nolock(kbdev); + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END); + + /* Don't need 'cores_are_available', + * because we don't return anything */ + CSTD_UNUSED(cores_are_available); + } +} + +static enum hrtimer_restart +kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer) +{ + struct kbase_device *kbdev; + unsigned long flags; + + kbdev = container_of(timer, struct kbase_device, + pm.backend.gpu_poweroff_timer); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* It is safe for this call to do nothing if the work item is already + * queued. The worker function will read the must up-to-date state of + * kbdev->pm.backend.gpu_poweroff_pending under lock. + * + * If a state change occurs while the worker function is processing, + * this call will succeed as a work item can be requeued once it has + * started processing. + */ + if (kbdev->pm.backend.gpu_poweroff_pending) + queue_work(kbdev->pm.backend.gpu_poweroff_wq, + &kbdev->pm.backend.gpu_poweroff_work); + + if (kbdev->pm.backend.shader_poweroff_pending || + kbdev->pm.backend.tiler_poweroff_pending) { + kbdev->pm.backend.shader_poweroff_pending_time--; + + KBASE_DEBUG_ASSERT( + kbdev->pm.backend.shader_poweroff_pending_time + >= 0); + + if (!kbdev->pm.backend.shader_poweroff_pending_time) + kbasep_pm_do_poweroff_cores(kbdev); + } + + if (kbdev->pm.backend.poweroff_timer_needed) { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time); + + return HRTIMER_RESTART; + } + + kbdev->pm.backend.poweroff_timer_running = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return HRTIMER_NORESTART; +} + +static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data) +{ + unsigned long flags; + struct kbase_device *kbdev; + bool do_poweroff = false; + + kbdev = container_of(data, struct kbase_device, + pm.backend.gpu_poweroff_work); + + mutex_lock(&kbdev->pm.lock); + + if (kbdev->pm.backend.gpu_poweroff_pending == 0) { + mutex_unlock(&kbdev->pm.lock); + return; + } + + kbdev->pm.backend.gpu_poweroff_pending--; + + if (kbdev->pm.backend.gpu_poweroff_pending > 0) { + mutex_unlock(&kbdev->pm.lock); + return; + } + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Only power off the GPU if a request is still pending */ + if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev)) + do_poweroff = true; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (do_poweroff) { + kbdev->pm.backend.poweroff_timer_needed = false; + hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer); + kbdev->pm.backend.poweroff_timer_running = false; + + /* Power off the GPU */ + kbase_pm_do_poweroff(kbdev, false); + } + + mutex_unlock(&kbdev->pm.lock); +} + +int kbase_pm_policy_init(struct kbase_device *kbdev) +{ + struct workqueue_struct *wq; + + wq = alloc_workqueue("kbase_pm_do_poweroff", + WQ_HIGHPRI | WQ_UNBOUND, 1); + if (!wq) + return -ENOMEM; + + kbdev->pm.backend.gpu_poweroff_wq = wq; + INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work, + kbasep_pm_do_gpu_poweroff_wq); + hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + kbdev->pm.backend.gpu_poweroff_timer.function = + kbasep_pm_do_gpu_poweroff_callback; + kbdev->pm.backend.pm_current_policy = policy_list[0]; + kbdev->pm.backend.pm_current_policy->init(kbdev); + kbdev->pm.gpu_poweroff_time = + HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS); + kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; + kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU; + + return 0; +} + +void kbase_pm_policy_term(struct kbase_device *kbdev) +{ + kbdev->pm.backend.pm_current_policy->term(kbdev); + destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq); +} + +void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev) +{ + unsigned long flags; + + lockdep_assert_held(&kbdev->pm.lock); + + kbdev->pm.backend.poweroff_timer_needed = false; + hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.poweroff_timer_running = false; + + /* If wq is already running but is held off by pm.lock, make sure it has + * no effect */ + kbdev->pm.backend.gpu_poweroff_pending = 0; + + kbdev->pm.backend.shader_poweroff_pending = 0; + kbdev->pm.backend.tiler_poweroff_pending = 0; + kbdev->pm.backend.shader_poweroff_pending_time = 0; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +void kbase_pm_update_active(struct kbase_device *kbdev) +{ + struct kbase_pm_device_data *pm = &kbdev->pm; + struct kbase_pm_backend_data *backend = &pm->backend; + unsigned long flags; + bool active; + + lockdep_assert_held(&pm->lock); + + /* pm_current_policy will never be NULL while pm.lock is held */ + KBASE_DEBUG_ASSERT(backend->pm_current_policy); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + active = backend->pm_current_policy->get_core_active(kbdev); + + if (active) { + if (backend->gpu_poweroff_pending) { + /* Cancel any pending power off request */ + backend->gpu_poweroff_pending = 0; + + /* If a request was pending then the GPU was still + * powered, so no need to continue */ + if (!kbdev->poweroff_pending) { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + return; + } + } + + if (!backend->poweroff_timer_running && !backend->gpu_powered && + (pm->poweroff_gpu_ticks || + pm->poweroff_shader_ticks)) { + backend->poweroff_timer_needed = true; + backend->poweroff_timer_running = true; + hrtimer_start(&backend->gpu_poweroff_timer, + pm->gpu_poweroff_time, + HRTIMER_MODE_REL); + } + + /* Power on the GPU and any cores requested by the policy */ + if (pm->backend.poweroff_wait_in_progress) { + pm->backend.poweron_required = true; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_pm_do_poweron(kbdev, false); + } + } else { + /* It is an error for the power policy to power off the GPU + * when there are contexts active */ + KBASE_DEBUG_ASSERT(pm->active_count == 0); + + if (backend->shader_poweroff_pending || + backend->tiler_poweroff_pending) { + backend->shader_poweroff_pending = 0; + backend->tiler_poweroff_pending = 0; + backend->shader_poweroff_pending_time = 0; + } + + /* Request power off */ + if (pm->backend.gpu_powered) { + if (pm->poweroff_gpu_ticks) { + backend->gpu_poweroff_pending = + pm->poweroff_gpu_ticks; + backend->poweroff_timer_needed = true; + if (!backend->poweroff_timer_running) { + /* Start timer if not running (eg if + * power policy has been changed from + * always_on to something else). This + * will ensure the GPU is actually + * powered off */ + backend->poweroff_timer_running + = true; + hrtimer_start( + &backend->gpu_poweroff_timer, + pm->gpu_poweroff_time, + HRTIMER_MODE_REL); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + + /* Power off the GPU immediately */ + kbase_pm_do_poweroff(kbdev, false); + } + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + } +} + +void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) +{ + u64 desired_bitmap; + u64 desired_tiler_bitmap; + bool cores_are_available; + bool do_poweroff = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->pm.backend.pm_current_policy == NULL) + return; + if (kbdev->pm.backend.poweroff_wait_in_progress) + return; + + if (kbdev->protected_mode_transition && !kbdev->shader_needed_bitmap && + !kbdev->shader_inuse_bitmap && !kbdev->tiler_needed_cnt + && !kbdev->tiler_inuse_cnt) { + /* We are trying to change in/out of protected mode - force all + * cores off so that the L2 powers down */ + desired_bitmap = 0; + desired_tiler_bitmap = 0; + } else { + desired_bitmap = + kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev); + desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev); + + if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0) + desired_tiler_bitmap = 1; + else + desired_tiler_bitmap = 0; + + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) { + /* Unless XAFFINITY is supported, enable core 0 if tiler + * required, regardless of core availability */ + if (kbdev->tiler_needed_cnt > 0 || + kbdev->tiler_inuse_cnt > 0) + desired_bitmap |= 1; + } + } + + if (kbdev->pm.backend.desired_shader_state != desired_bitmap) + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u, + (u32)desired_bitmap); + /* Are any cores being powered on? */ + if (~kbdev->pm.backend.desired_shader_state & desired_bitmap || + ~kbdev->pm.backend.desired_tiler_state & desired_tiler_bitmap || + kbdev->pm.backend.ca_in_transition) { + /* Check if we are powering off any cores before updating shader + * state */ + if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap || + kbdev->pm.backend.desired_tiler_state & + ~desired_tiler_bitmap) { + /* Start timer to power off cores */ + kbdev->pm.backend.shader_poweroff_pending |= + (kbdev->pm.backend.desired_shader_state & + ~desired_bitmap); + kbdev->pm.backend.tiler_poweroff_pending |= + (kbdev->pm.backend.desired_tiler_state & + ~desired_tiler_bitmap); + + if (kbdev->pm.poweroff_shader_ticks && + !kbdev->protected_mode_transition) + kbdev->pm.backend.shader_poweroff_pending_time = + kbdev->pm.poweroff_shader_ticks; + else + do_poweroff = true; + } + + kbdev->pm.backend.desired_shader_state = desired_bitmap; + kbdev->pm.backend.desired_tiler_state = desired_tiler_bitmap; + + /* If any cores are being powered on, transition immediately */ + cores_are_available = kbase_pm_check_transitions_nolock(kbdev); + } else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap || + kbdev->pm.backend.desired_tiler_state & + ~desired_tiler_bitmap) { + /* Start timer to power off cores */ + kbdev->pm.backend.shader_poweroff_pending |= + (kbdev->pm.backend.desired_shader_state & + ~desired_bitmap); + kbdev->pm.backend.tiler_poweroff_pending |= + (kbdev->pm.backend.desired_tiler_state & + ~desired_tiler_bitmap); + if (kbdev->pm.poweroff_shader_ticks && + !kbdev->protected_mode_transition) + kbdev->pm.backend.shader_poweroff_pending_time = + kbdev->pm.poweroff_shader_ticks; + else + kbasep_pm_do_poweroff_cores(kbdev); + } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 && + desired_tiler_bitmap != 0 && + kbdev->pm.backend.poweroff_timer_needed) { + /* If power policy is keeping cores on despite there being no + * active contexts then disable poweroff timer as it isn't + * required. + * Only reset poweroff_timer_needed if we're not in the middle + * of the power off callback */ + kbdev->pm.backend.poweroff_timer_needed = false; + } + + /* Ensure timer does not power off wanted cores and make sure to power + * off unwanted cores */ + if (kbdev->pm.backend.shader_poweroff_pending || + kbdev->pm.backend.tiler_poweroff_pending) { + kbdev->pm.backend.shader_poweroff_pending &= + ~(kbdev->pm.backend.desired_shader_state & + desired_bitmap); + kbdev->pm.backend.tiler_poweroff_pending &= + ~(kbdev->pm.backend.desired_tiler_state & + desired_tiler_bitmap); + + if (!kbdev->pm.backend.shader_poweroff_pending && + !kbdev->pm.backend.tiler_poweroff_pending) + kbdev->pm.backend.shader_poweroff_pending_time = 0; + } + + /* Shader poweroff is deferred to the end of the function, to eliminate + * issues caused by the core availability policy recursing into this + * function */ + if (do_poweroff) + kbasep_pm_do_poweroff_cores(kbdev); + + /* Don't need 'cores_are_available', because we don't return anything */ + CSTD_UNUSED(cores_are_available); +} + +void kbase_pm_update_cores_state(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_pm_update_cores_state_nolock(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +int kbase_pm_list_policies(const struct kbase_pm_policy * const **list) +{ + if (!list) + return POLICY_COUNT; + + *list = policy_list; + + return POLICY_COUNT; +} + +KBASE_EXPORT_TEST_API(kbase_pm_list_policies); + +const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + return kbdev->pm.backend.pm_current_policy; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_policy); + +void kbase_pm_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_policy *new_policy) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + const struct kbase_pm_policy *old_policy; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(new_policy != NULL); + + KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id); + + /* During a policy change we pretend the GPU is active */ + /* A suspend won't happen here, because we're in a syscall from a + * userspace thread */ + kbase_pm_context_active(kbdev); + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + /* Remove the policy to prevent IRQ handlers from working on it */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + old_policy = kbdev->pm.backend.pm_current_policy; + kbdev->pm.backend.pm_current_policy = NULL; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, + old_policy->id); + if (old_policy->term) + old_policy->term(kbdev); + + KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, + new_policy->id); + if (new_policy->init) + new_policy->init(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.pm_current_policy = new_policy; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* If any core power state changes were previously attempted, but + * couldn't be made because the policy was changing (current_policy was + * NULL), then re-try them here. */ + kbase_pm_update_active(kbdev); + kbase_pm_update_cores_state(kbdev); + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + + /* Now the policy change is finished, we release our fake context active + * reference */ + kbase_pm_context_idle(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_set_policy); + +/* Check whether a state change has finished, and trace it as completed */ +static void +kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev) +{ + if ((kbdev->shader_available_bitmap & + kbdev->pm.backend.desired_shader_state) + == kbdev->pm.backend.desired_shader_state && + (kbdev->tiler_available_bitmap & + kbdev->pm.backend.desired_tiler_state) + == kbdev->pm.backend.desired_tiler_state) + kbase_timeline_pm_check_handle_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); +} + +void kbase_pm_request_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores) +{ + u64 cores; + + kbase_pm_change_state change_gpu_state = 0u; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + cores = shader_cores; + while (cores) { + int bitnum = fls64(cores) - 1; + u64 bit = 1ULL << bitnum; + + /* It should be almost impossible for this to overflow. It would + * require 2^32 atoms to request a particular core, which would + * require 2^24 contexts to submit. This would require an amount + * of memory that is impossible on a 32-bit system and extremely + * unlikely on a 64-bit system. */ + int cnt = ++kbdev->shader_needed_cnt[bitnum]; + + if (1 == cnt) { + kbdev->shader_needed_bitmap |= bit; + change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; + } + + cores &= ~bit; + } + + if (tiler_required) { + int cnt = ++kbdev->tiler_needed_cnt; + + if (1 == cnt) + change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; + + KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0); + } + + if (change_gpu_state) { + KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL, + NULL, 0u, (u32) kbdev->shader_needed_bitmap); + + kbase_timeline_pm_cores_func(kbdev, + KBASE_PM_FUNC_ID_REQUEST_CORES_START, + change_gpu_state); + kbase_pm_update_cores_state_nolock(kbdev); + kbase_timeline_pm_cores_func(kbdev, + KBASE_PM_FUNC_ID_REQUEST_CORES_END, + change_gpu_state); + } +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_cores); + +void kbase_pm_unrequest_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores) +{ + kbase_pm_change_state change_gpu_state = 0u; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + while (shader_cores) { + int bitnum = fls64(shader_cores) - 1; + u64 bit = 1ULL << bitnum; + int cnt; + + KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0); + + cnt = --kbdev->shader_needed_cnt[bitnum]; + + if (0 == cnt) { + kbdev->shader_needed_bitmap &= ~bit; + + change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; + } + + shader_cores &= ~bit; + } + + if (tiler_required) { + int cnt; + + KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0); + + cnt = --kbdev->tiler_needed_cnt; + + if (0 == cnt) + change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; + } + + if (change_gpu_state) { + KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL, + NULL, 0u, (u32) kbdev->shader_needed_bitmap); + + kbase_pm_update_cores_state_nolock(kbdev); + + /* Trace that any state change effectively completes immediately + * - no-one will wait on the state change */ + kbase_pm_trace_check_and_finish_state_change(kbdev); + } +} + +KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores); + +enum kbase_pm_cores_ready +kbase_pm_register_inuse_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores) +{ + u64 prev_shader_needed; /* Just for tracing */ + u64 prev_shader_inuse; /* Just for tracing */ + + lockdep_assert_held(&kbdev->hwaccess_lock); + + prev_shader_needed = kbdev->shader_needed_bitmap; + prev_shader_inuse = kbdev->shader_inuse_bitmap; + + /* If desired_shader_state does not contain the requested cores, then + * power management is not attempting to powering those cores (most + * likely due to core availability policy) and a new job affinity must + * be chosen */ + if ((kbdev->pm.backend.desired_shader_state & shader_cores) != + shader_cores) { + return (kbdev->pm.backend.poweroff_wait_in_progress || + kbdev->pm.backend.pm_current_policy == NULL) ? + KBASE_CORES_NOT_READY : KBASE_NEW_AFFINITY; + } + + if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores || + (tiler_required && !kbdev->tiler_available_bitmap)) { + /* Trace ongoing core transition */ + kbase_timeline_pm_l2_transition_start(kbdev); + return KBASE_CORES_NOT_READY; + } + + /* If we started to trace a state change, then trace it has being + * finished by now, at the very latest */ + kbase_pm_trace_check_and_finish_state_change(kbdev); + /* Trace core transition done */ + kbase_timeline_pm_l2_transition_done(kbdev); + + while (shader_cores) { + int bitnum = fls64(shader_cores) - 1; + u64 bit = 1ULL << bitnum; + int cnt; + + KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0); + + cnt = --kbdev->shader_needed_cnt[bitnum]; + + if (0 == cnt) + kbdev->shader_needed_bitmap &= ~bit; + + /* shader_inuse_cnt should not overflow because there can only + * be a very limited number of jobs on the h/w at one time */ + + kbdev->shader_inuse_cnt[bitnum]++; + kbdev->shader_inuse_bitmap |= bit; + + shader_cores &= ~bit; + } + + if (tiler_required) { + KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0); + + --kbdev->tiler_needed_cnt; + + kbdev->tiler_inuse_cnt++; + + KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0); + } + + if (prev_shader_needed != kbdev->shader_needed_bitmap) + KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL, + NULL, 0u, (u32) kbdev->shader_needed_bitmap); + + if (prev_shader_inuse != kbdev->shader_inuse_bitmap) + KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL, + NULL, 0u, (u32) kbdev->shader_inuse_bitmap); + + return KBASE_CORES_READY; +} + +KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores); + +void kbase_pm_release_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores) +{ + kbase_pm_change_state change_gpu_state = 0u; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + while (shader_cores) { + int bitnum = fls64(shader_cores) - 1; + u64 bit = 1ULL << bitnum; + int cnt; + + KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0); + + cnt = --kbdev->shader_inuse_cnt[bitnum]; + + if (0 == cnt) { + kbdev->shader_inuse_bitmap &= ~bit; + change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; + } + + shader_cores &= ~bit; + } + + if (tiler_required) { + int cnt; + + KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0); + + cnt = --kbdev->tiler_inuse_cnt; + + if (0 == cnt) + change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; + } + + if (change_gpu_state) { + KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL, + NULL, 0u, (u32) kbdev->shader_inuse_bitmap); + + kbase_timeline_pm_cores_func(kbdev, + KBASE_PM_FUNC_ID_RELEASE_CORES_START, + change_gpu_state); + kbase_pm_update_cores_state_nolock(kbdev); + kbase_timeline_pm_cores_func(kbdev, + KBASE_PM_FUNC_ID_RELEASE_CORES_END, + change_gpu_state); + + /* Trace that any state change completed immediately */ + kbase_pm_trace_check_and_finish_state_change(kbdev); + } +} + +KBASE_EXPORT_TEST_API(kbase_pm_release_cores); + +void kbase_pm_request_cores_sync(struct kbase_device *kbdev, + bool tiler_required, + u64 shader_cores) +{ + unsigned long flags; + + kbase_pm_wait_for_poweroff_complete(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_request_cores(kbdev, tiler_required, shader_cores); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_pm_check_transitions_sync(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync); + +void kbase_pm_request_l2_caches(struct kbase_device *kbdev) +{ + unsigned long flags; + u32 prior_l2_users_count; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + prior_l2_users_count = kbdev->l2_users_count++; + + KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0); + + /* if the GPU is reset while the l2 is on, l2 will be off but + * prior_l2_users_count will be > 0. l2_available_bitmap will have been + * set to 0 though by kbase_pm_init_hw */ + if (!prior_l2_users_count || !kbdev->l2_available_bitmap) + kbase_pm_check_transitions_nolock(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + wait_event(kbdev->pm.backend.l2_powered_wait, + kbdev->pm.backend.l2_powered == 1); + + /* Trace that any state change completed immediately */ + kbase_pm_trace_check_and_finish_state_change(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches); + +void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->l2_users_count++; +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on); + +void kbase_pm_release_l2_caches(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0); + + --kbdev->l2_users_count; + + if (!kbdev->l2_users_count) { + kbase_pm_check_transitions_nolock(kbdev); + /* Trace that any state change completed immediately */ + kbase_pm_trace_check_and_finish_state_change(kbdev); + } +} + +KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches); diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.h new file mode 100755 index 000000000000..611a90e66e65 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_pm_policy.h @@ -0,0 +1,227 @@ +/* + * + * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Power policy API definitions + */ + +#ifndef _KBASE_PM_POLICY_H_ +#define _KBASE_PM_POLICY_H_ + +/** + * kbase_pm_policy_init - Initialize power policy framework + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Must be called before calling any other policy function + * + * Return: 0 if the power policy framework was successfully + * initialized, -errno otherwise. + */ +int kbase_pm_policy_init(struct kbase_device *kbdev); + +/** + * kbase_pm_policy_term - Terminate power policy framework + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_policy_term(struct kbase_device *kbdev); + +/** + * kbase_pm_update_active - Update the active power state of the GPU + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Calls into the current power policy + */ +void kbase_pm_update_active(struct kbase_device *kbdev); + +/** + * kbase_pm_update_cores - Update the desired core state of the GPU + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Calls into the current power policy + */ +void kbase_pm_update_cores(struct kbase_device *kbdev); + + +enum kbase_pm_cores_ready { + KBASE_CORES_NOT_READY = 0, + KBASE_NEW_AFFINITY = 1, + KBASE_CORES_READY = 2 +}; + + +/** + * kbase_pm_request_cores_sync - Synchronous variant of kbase_pm_request_cores() + * + * @kbdev: The kbase device structure for the device + * @tiler_required: true if the tiler is required, false otherwise + * @shader_cores: A bitmask of shader cores which are necessary for the job + * + * When this function returns, the @shader_cores will be in the READY state. + * + * This is safe variant of kbase_pm_check_transitions_sync(): it handles the + * work of ensuring the requested cores will remain powered until a matching + * call to kbase_pm_unrequest_cores()/kbase_pm_release_cores() (as appropriate) + * is made. + */ +void kbase_pm_request_cores_sync(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores); + +/** + * kbase_pm_request_cores - Mark one or more cores as being required + * for jobs to be submitted + * + * @kbdev: The kbase device structure for the device + * @tiler_required: true if the tiler is required, false otherwise + * @shader_cores: A bitmask of shader cores which are necessary for the job + * + * This function is called by the job scheduler to mark one or more cores as + * being required to submit jobs that are ready to run. + * + * The cores requested are reference counted and a subsequent call to + * kbase_pm_register_inuse_cores() or kbase_pm_unrequest_cores() should be + * made to dereference the cores as being 'needed'. + * + * The active power policy will meet or exceed the requirements of the + * requested cores in the system. Any core transitions needed will be begun + * immediately, but they might not complete/the cores might not be available + * until a Power Management IRQ. + * + * Return: 0 if the cores were successfully requested, or -errno otherwise. + */ +void kbase_pm_request_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores); + +/** + * kbase_pm_unrequest_cores - Unmark one or more cores as being required for + * jobs to be submitted. + * + * @kbdev: The kbase device structure for the device + * @tiler_required: true if the tiler is required, false otherwise + * @shader_cores: A bitmask of shader cores (as given to + * kbase_pm_request_cores() ) + * + * This function undoes the effect of kbase_pm_request_cores(). It should be + * used when a job is not going to be submitted to the hardware (e.g. the job is + * cancelled before it is enqueued). + * + * The active power policy will meet or exceed the requirements of the + * requested cores in the system. Any core transitions needed will be begun + * immediately, but they might not complete until a Power Management IRQ. + * + * The policy may use this as an indication that it can power down cores. + */ +void kbase_pm_unrequest_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores); + +/** + * kbase_pm_register_inuse_cores - Register a set of cores as in use by a job + * + * @kbdev: The kbase device structure for the device + * @tiler_required: true if the tiler is required, false otherwise + * @shader_cores: A bitmask of shader cores (as given to + * kbase_pm_request_cores() ) + * + * This function should be called after kbase_pm_request_cores() when the job + * is about to be submitted to the hardware. It will check that the necessary + * cores are available and if so update the 'needed' and 'inuse' bitmasks to + * reflect that the job is now committed to being run. + * + * If the necessary cores are not currently available then the function will + * return %KBASE_CORES_NOT_READY and have no effect. + * + * Return: %KBASE_CORES_NOT_READY if the cores are not immediately ready, + * + * %KBASE_NEW_AFFINITY if the affinity requested is not allowed, + * + * %KBASE_CORES_READY if the cores requested are already available + */ +enum kbase_pm_cores_ready kbase_pm_register_inuse_cores( + struct kbase_device *kbdev, + bool tiler_required, + u64 shader_cores); + +/** + * kbase_pm_release_cores - Release cores after a job has run + * + * @kbdev: The kbase device structure for the device + * @tiler_required: true if the tiler is required, false otherwise + * @shader_cores: A bitmask of shader cores (as given to + * kbase_pm_register_inuse_cores() ) + * + * This function should be called when a job has finished running on the + * hardware. A call to kbase_pm_register_inuse_cores() must have previously + * occurred. The reference counts of the specified cores will be decremented + * which may cause the bitmask of 'inuse' cores to be reduced. The power policy + * may then turn off any cores which are no longer 'inuse'. + */ +void kbase_pm_release_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores); + +/** + * kbase_pm_request_l2_caches - Request l2 caches + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Request the use of l2 caches for all core groups, power up, wait and prevent + * the power manager from powering down the l2 caches. + * + * This tells the power management that the caches should be powered up, and + * they should remain powered, irrespective of the usage of shader cores. This + * does not return until the l2 caches are powered up. + * + * The caller must call kbase_pm_release_l2_caches() when they are finished + * to allow normal power management of the l2 caches to resume. + * + * This should only be used when power management is active. + */ +void kbase_pm_request_l2_caches(struct kbase_device *kbdev); + +/** + * kbase_pm_request_l2_caches_l2_is_on - Request l2 caches but don't power on + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Increment the count of l2 users but do not attempt to power on the l2 + * + * It is the callers responsibility to ensure that the l2 is already powered up + * and to eventually call kbase_pm_release_l2_caches() + */ +void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev); + +/** + * kbase_pm_request_l2_caches - Release l2 caches + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Release the use of l2 caches for all core groups and allow the power manager + * to power them down when necessary. + * + * This tells the power management that the caches can be powered down if + * necessary, with respect to the usage of shader cores. + * + * The caller must have called kbase_pm_request_l2_caches() prior to a call + * to this. + * + * This should only be used when power management is active. + */ +void kbase_pm_release_l2_caches(struct kbase_device *kbdev); + +#endif /* _KBASE_PM_POLICY_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.c new file mode 100755 index 000000000000..0068e1091f4c --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.c @@ -0,0 +1,103 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include + +void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, + u64 *system_time, struct timespec *ts) +{ + u32 hi1, hi2; + + kbase_pm_request_gpu_cycle_counter(kbdev); + + /* Read hi, lo, hi to ensure that overflow from lo to hi is handled + * correctly */ + do { + hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), + NULL); + *cycle_counter = kbase_reg_read(kbdev, + GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL); + hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), + NULL); + *cycle_counter |= (((u64) hi1) << 32); + } while (hi1 != hi2); + + /* Read hi, lo, hi to ensure that overflow from lo to hi is handled + * correctly */ + do { + hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), + NULL); + *system_time = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TIMESTAMP_LO), NULL); + hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), + NULL); + *system_time |= (((u64) hi1) << 32); + } while (hi1 != hi2); + + /* Record the CPU's idea of current time */ + getrawmonotonic(ts); + + kbase_pm_release_gpu_cycle_counter(kbdev); +} + +/** + * kbase_wait_write_flush - Wait for GPU write flush + * @kctx: Context pointer + * + * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush + * its write buffer. + * + * Only in use for BASE_HW_ISSUE_6367 + * + * Note : If GPU resets occur then the counters are reset to zero, the delay may + * not be as expected. + */ +#ifndef CONFIG_MALI_BIFROST_NO_MALI +void kbase_wait_write_flush(struct kbase_context *kctx) +{ + u32 base_count = 0; + + /* + * The caller must be holding onto the kctx or the call is from + * userspace. + */ + kbase_pm_context_active(kctx->kbdev); + kbase_pm_request_gpu_cycle_counter(kctx->kbdev); + + while (true) { + u32 new_count; + + new_count = kbase_reg_read(kctx->kbdev, + GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL); + /* First time around, just store the count. */ + if (base_count == 0) { + base_count = new_count; + continue; + } + + /* No need to handle wrapping, unsigned maths works for this. */ + if ((new_count - base_count) > 1000) + break; + } + + kbase_pm_release_gpu_cycle_counter(kctx->kbdev); + kbase_pm_context_idle(kctx->kbdev); +} +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ diff --git a/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.h b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.h new file mode 100755 index 000000000000..0559b2f7097d --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/backend/gpu/mali_kbase_time.h @@ -0,0 +1,52 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_BACKEND_TIME_H_ +#define _KBASE_BACKEND_TIME_H_ + +/** + * kbase_backend_get_gpu_time() - Get current GPU time + * @kbdev: Device pointer + * @cycle_counter: Pointer to u64 to store cycle counter in + * @system_time: Pointer to u64 to store system time in + * @ts: Pointer to struct timespec to store current monotonic + * time in + */ +void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, + u64 *system_time, struct timespec *ts); + +/** + * kbase_wait_write_flush() - Wait for GPU write flush + * @kctx: Context pointer + * + * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush + * its write buffer. + * + * If GPU resets occur then the counters are reset to zero, the delay may not be + * as expected. + * + * This function is only in use for BASE_HW_ISSUE_6367 + */ +#ifdef CONFIG_MALI_BIFROST_NO_MALI +static inline void kbase_wait_write_flush(struct kbase_context *kctx) +{ +} +#else +void kbase_wait_write_flush(struct kbase_context *kctx); +#endif + +#endif /* _KBASE_BACKEND_TIME_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/docs/Doxyfile b/drivers/gpu/arm/bifrost_for_linux/docs/Doxyfile new file mode 100755 index 000000000000..35ff2f1ce4a0 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/docs/Doxyfile @@ -0,0 +1,126 @@ +# +# (C) COPYRIGHT 2011-2013, 2015 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + +############################################################################## + +# This file contains per-module Doxygen configuration. Please do not add +# extra settings to this file without consulting all stakeholders, as they +# may cause override project-wide settings. +# +# Additionally, when defining aliases, macros, sections etc, use the module +# name as a prefix e.g. gles_my_alias. + +############################################################################## + +@INCLUDE = ../../bldsys/Doxyfile_common + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT += ../../kernel/drivers/gpu/arm/midgard/ + +############################################################################## +# Everything below here is optional, and in most cases not required +############################################################################## + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES += + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS += + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS += + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +EXCLUDE += ../../kernel/drivers/gpu/arm/midgard/platform ../../kernel/drivers/gpu/arm/midgard/platform_dummy ../../kernel/drivers/gpu/arm/midgard/scripts ../../kernel/drivers/gpu/arm/midgard/tests ../../kernel/drivers/gpu/arm/midgard/Makefile ../../kernel/drivers/gpu/arm/midgard/Makefile.kbase ../../kernel/drivers/gpu/arm/midgard/Kbuild ../../kernel/drivers/gpu/arm/midgard/Kconfig ../../kernel/drivers/gpu/arm/midgard/sconscript ../../kernel/drivers/gpu/arm/midgard/docs ../../kernel/drivers/gpu/arm/midgard/pm_test_script.sh ../../kernel/drivers/gpu/arm/midgard/mali_uk.h ../../kernel/drivers/gpu/arm/midgard/Makefile + + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS += + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS += + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH += + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH += + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH += + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED += + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED += + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS += ../../kernel/drivers/gpu/arm/midgard/docs + diff --git a/drivers/gpu/arm/bifrost_for_linux/docs/policy_operation_diagram.dot b/drivers/gpu/arm/bifrost_for_linux/docs/policy_operation_diagram.dot new file mode 100755 index 000000000000..7ae05c2f8ded --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/docs/policy_operation_diagram.dot @@ -0,0 +1,112 @@ +/* + * + * (C) COPYRIGHT 2010 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +digraph policy_objects_diagram { + rankdir=LR; + size="12,8"; + compound=true; + + node [ shape = box ]; + + subgraph cluster_policy_queues { + low_queue [ shape=record label = "LowP | {ctx_lo | ... | ctx_i | ... | ctx_hi}" ]; + queues_middle_sep [ label="" shape=plaintext width=0 height=0 ]; + + rt_queue [ shape=record label = "RT | {ctx_lo | ... | ctx_j | ... | ctx_hi}" ]; + + label = "Policy's Queue(s)"; + } + + call_enqueue [ shape=plaintext label="enqueue_ctx()" ]; + + { + rank=same; + ordering=out; + call_dequeue [ shape=plaintext label="dequeue_head_ctx()\n+ runpool_add_ctx()" ]; + call_ctxfinish [ shape=plaintext label="runpool_remove_ctx()" ]; + + call_ctxdone [ shape=plaintext label="don't requeue;\n/* ctx has no more jobs */" ]; + } + + subgraph cluster_runpool { + + as0 [ width=2 height = 0.25 label="AS0: Job_1, ..., Job_n" ]; + as1 [ width=2 height = 0.25 label="AS1: Job_1, ..., Job_m" ]; + as2 [ width=2 height = 0.25 label="AS2: Job_1, ..., Job_p" ]; + as3 [ width=2 height = 0.25 label="AS3: Job_1, ..., Job_q" ]; + + label = "Policy's Run Pool"; + } + + { + rank=same; + call_jdequeue [ shape=plaintext label="dequeue_job()" ]; + sstop_dotfixup [ shape=plaintext label="" width=0 height=0 ]; + } + + { + rank=same; + ordering=out; + sstop [ shape=ellipse label="SS-Timer expires" ] + jobslots [ shape=record label="Jobslots: | <0>js[0] | <1>js[1] | <2>js[2]" ]; + + irq [ label="IRQ" shape=ellipse ]; + + job_finish [ shape=plaintext label="don't requeue;\n/* job done */" ]; + } + + hstop [ shape=ellipse label="HS-Timer expires" ] + + /* + * Edges + */ + + call_enqueue -> queues_middle_sep [ lhead=cluster_policy_queues ]; + + low_queue:qr -> call_dequeue:w; + rt_queue:qr -> call_dequeue:w; + + call_dequeue -> as1 [lhead=cluster_runpool]; + + as1->call_jdequeue [ltail=cluster_runpool]; + call_jdequeue->jobslots:0; + call_jdequeue->sstop_dotfixup [ arrowhead=none]; + sstop_dotfixup->sstop [label="Spawn SS-Timer"]; + sstop->jobslots [label="SoftStop"]; + sstop->hstop [label="Spawn HS-Timer"]; + hstop->jobslots:ne [label="HardStop"]; + + + as3->call_ctxfinish:ne [ ltail=cluster_runpool ]; + call_ctxfinish:sw->rt_queue:qm [ lhead=cluster_policy_queues label="enqueue_ctx()\n/* ctx still has jobs */" ]; + + call_ctxfinish->call_ctxdone [constraint=false]; + + call_ctxdone->call_enqueue [weight=0.1 labeldistance=20.0 labelangle=0.0 taillabel="Job submitted to the ctx" style=dotted constraint=false]; + + + { + jobslots->irq [constraint=false]; + + irq->job_finish [constraint=false]; + } + + irq->as2 [lhead=cluster_runpool label="requeue_job()\n/* timeslice expired */" ]; + +} diff --git a/drivers/gpu/arm/bifrost_for_linux/docs/policy_overview.dot b/drivers/gpu/arm/bifrost_for_linux/docs/policy_overview.dot new file mode 100755 index 000000000000..159b993b7d61 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/docs/policy_overview.dot @@ -0,0 +1,63 @@ +/* + * + * (C) COPYRIGHT 2010 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +digraph policy_objects_diagram { + rankdir=LR + size="6,6" + compound=true; + + node [ shape = box ]; + + call_enqueue [ shape=plaintext label="enqueue ctx" ]; + + + policy_queue [ label="Policy's Queue" ]; + + { + rank=same; + runpool [ label="Policy's Run Pool" ]; + + ctx_finish [ label="ctx finished" ]; + } + + { + rank=same; + jobslots [ shape=record label="Jobslots: | <0>js[0] | <1>js[1] | <2>js[2]" ]; + + job_finish [ label="Job finished" ]; + } + + + + /* + * Edges + */ + + call_enqueue -> policy_queue; + + policy_queue->runpool [label="dequeue ctx" weight=0.1]; + runpool->policy_queue [label="requeue ctx" weight=0.1]; + + runpool->ctx_finish [ style=dotted ]; + + runpool->jobslots [label="dequeue job" weight=0.1]; + jobslots->runpool [label="requeue job" weight=0.1]; + + jobslots->job_finish [ style=dotted ]; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/Kbuild b/drivers/gpu/arm/bifrost_for_linux/ipa/Kbuild new file mode 100755 index 000000000000..0776428fce4f --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/ipa/Kbuild @@ -0,0 +1,27 @@ +# +# (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +bifrost_kbase-y += \ + ipa/mali_kbase_ipa_simple.o \ + ipa/mali_kbase_ipa.o + +bifrost_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o + +ifneq ($(wildcard $(srctree)/$(src)/ipa/mali_kbase_ipa_vinstr_g71.c),) + bifrost_kbase-y += \ + ipa/mali_kbase_ipa_vinstr_g71.o \ + ipa/mali_kbase_ipa_vinstr_common.o + +endif diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.c b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.c new file mode 100755 index 000000000000..d6332b55e970 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.c @@ -0,0 +1,580 @@ +/* + * + * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +#include +#include +#include +#include "mali_kbase.h" +#include "mali_kbase_ipa.h" +#include "mali_kbase_ipa_debugfs.h" +#include "mali_kbase_ipa_simple.h" + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) +#include +#else +#include +#define dev_pm_opp_find_freq_exact opp_find_freq_exact +#define dev_pm_opp_get_voltage opp_get_voltage +#define dev_pm_opp opp +#endif + +#define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model" +#define KBASE_IPA_G71_MODEL_NAME "mali-g71-power-model" + +static struct kbase_ipa_model_ops *kbase_ipa_all_model_ops[] = { + &kbase_simple_ipa_model_ops, + &kbase_g71_ipa_model_ops +}; + +int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) +{ + int err = 0; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (model->ops->recalculate) { + err = model->ops->recalculate(model); + if (err) { + dev_err(model->kbdev->dev, + "recalculation of power model %s returned error %d\n", + model->ops->name, err); + } + } + + return err; +} + +static struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, + const char *name) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(kbase_ipa_all_model_ops); ++i) { + struct kbase_ipa_model_ops *ops = kbase_ipa_all_model_ops[i]; + + if (!strcmp(ops->name, name)) + return ops; + } + + dev_err(kbdev->dev, "power model \'%s\' not found\n", name); + + return NULL; +} + +void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) +{ + atomic_set(&kbdev->ipa_use_configured_model, false); +} + +void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) +{ + atomic_set(&kbdev->ipa_use_configured_model, true); +} + +const char *kbase_ipa_model_name_from_id(u32 gpu_id) +{ + const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + if (GPU_ID_IS_NEW_FORMAT(prod_id)) { + switch (GPU_ID2_MODEL_MATCH_VALUE(prod_id)) { + case GPU_ID2_PRODUCT_TMIX: + return KBASE_IPA_G71_MODEL_NAME; + default: + return KBASE_IPA_FALLBACK_MODEL_NAME; + } + } + + return KBASE_IPA_FALLBACK_MODEL_NAME; +} + +static struct device_node *get_model_dt_node(struct kbase_ipa_model *model) +{ + struct device_node *model_dt_node; + char compat_string[64]; + + snprintf(compat_string, sizeof(compat_string), "arm,%s", + model->ops->name); + + model_dt_node = of_find_compatible_node(model->kbdev->dev->of_node, + NULL, compat_string); + if (!model_dt_node && !model->missing_dt_node_warning) { + dev_warn(model->kbdev->dev, + "Couldn't find power_model DT node matching \'%s\'\n", + compat_string); + model->missing_dt_node_warning = true; + } + + return model_dt_node; +} + +int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, + const char *name, s32 *addr, + size_t num_elems, bool dt_required) +{ + int err, i; + struct device_node *model_dt_node = get_model_dt_node(model); + char *origin; + + err = of_property_read_u32_array(model_dt_node, name, addr, num_elems); + + if (err && dt_required) { + memset(addr, 0, sizeof(s32) * num_elems); + dev_warn(model->kbdev->dev, + "Error %d, no DT entry: %s.%s = %zu*[0]\n", + err, model->ops->name, name, num_elems); + origin = "zero"; + } else if (err && !dt_required) { + origin = "default"; + } else /* !err */ { + origin = "DT"; + } + + /* Create a unique debugfs entry for each element */ + for (i = 0; i < num_elems; ++i) { + char elem_name[32]; + + if (num_elems == 1) + snprintf(elem_name, sizeof(elem_name), "%s", name); + else + snprintf(elem_name, sizeof(elem_name), "%s.%d", + name, i); + + dev_dbg(model->kbdev->dev, "%s.%s = %d (%s)\n", + model->ops->name, elem_name, addr[i], origin); + + err = kbase_ipa_model_param_add(model, elem_name, + &addr[i], sizeof(s32), + PARAM_TYPE_S32); + if (err) + goto exit; + } +exit: + return err; +} + +int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, + const char *name, char *addr, + size_t size, bool dt_required) +{ + int err; + struct device_node *model_dt_node = get_model_dt_node(model); + const char *string_prop_value; + char *origin; + + err = of_property_read_string(model_dt_node, name, + &string_prop_value); + if (err && dt_required) { + strncpy(addr, "", size - 1); + dev_warn(model->kbdev->dev, + "Error %d, no DT entry: %s.%s = \'%s\'\n", + err, model->ops->name, name, addr); + err = 0; + origin = "zero"; + } else if (err && !dt_required) { + origin = "default"; + } else /* !err */ { + strncpy(addr, string_prop_value, size - 1); + origin = "DT"; + } + + addr[size - 1] = '\0'; + + dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (%s)\n", + model->ops->name, name, string_prop_value, origin); + + err = kbase_ipa_model_param_add(model, name, addr, size, + PARAM_TYPE_STRING); + + return err; +} + +void kbase_ipa_term_model(struct kbase_ipa_model *model) +{ + if (!model) + return; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (model->ops->term) + model->ops->term(model); + + kbase_ipa_model_param_free_all(model); + + kfree(model); +} +KBASE_EXPORT_TEST_API(kbase_ipa_term_model); + +struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, + struct kbase_ipa_model_ops *ops) +{ + struct kbase_ipa_model *model; + int err; + + lockdep_assert_held(&kbdev->ipa.lock); + + if (!ops || !ops->name) + return NULL; + + model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL); + if (!model) + return NULL; + + model->kbdev = kbdev; + model->ops = ops; + INIT_LIST_HEAD(&model->params); + + err = model->ops->init(model); + if (err) { + dev_err(kbdev->dev, + "init of power model \'%s\' returned error %d\n", + ops->name, err); + kfree(model); + return NULL; + } + + err = kbase_ipa_model_recalculate(model); + if (err) { + kbase_ipa_term_model(model); + return NULL; + } + + return model; +} +KBASE_EXPORT_TEST_API(kbase_ipa_init_model); + +static void kbase_ipa_term_locked(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->ipa.lock); + + /* Clean up the models */ + if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) + kbase_ipa_term_model(kbdev->ipa.configured_model); + kbase_ipa_term_model(kbdev->ipa.fallback_model); + + kbdev->ipa.configured_model = NULL; + kbdev->ipa.fallback_model = NULL; +} + +int kbase_ipa_init(struct kbase_device *kbdev) +{ + + const char *model_name; + struct kbase_ipa_model_ops *ops; + struct kbase_ipa_model *default_model = NULL; + int err; + + mutex_init(&kbdev->ipa.lock); + /* + * Lock during init to avoid warnings from lockdep_assert_held (there + * shouldn't be any concurrent access yet). + */ + mutex_lock(&kbdev->ipa.lock); + + /* The simple IPA model must *always* be present.*/ + ops = kbase_ipa_model_ops_find(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME); + + if (!ops->do_utilization_scaling_in_framework) { + dev_err(kbdev->dev, + "Fallback IPA model %s should not account for utilization\n", + ops->name); + err = -EINVAL; + goto end; + } + + default_model = kbase_ipa_init_model(kbdev, ops); + if (!default_model) { + err = -EINVAL; + goto end; + } + + kbdev->ipa.fallback_model = default_model; + err = of_property_read_string(kbdev->dev->of_node, + "ipa-model", + &model_name); + if (err) { + /* Attempt to load a match from GPU-ID */ + u32 gpu_id; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + model_name = kbase_ipa_model_name_from_id(gpu_id); + dev_dbg(kbdev->dev, + "Inferring model from GPU ID 0x%x: \'%s\'\n", + gpu_id, model_name); + err = 0; + } else { + dev_dbg(kbdev->dev, + "Using ipa-model parameter from DT: \'%s\'\n", + model_name); + } + + if (strcmp(KBASE_IPA_FALLBACK_MODEL_NAME, model_name) != 0) { + ops = kbase_ipa_model_ops_find(kbdev, model_name); + kbdev->ipa.configured_model = kbase_ipa_init_model(kbdev, ops); + if (!kbdev->ipa.configured_model) { + err = -EINVAL; + goto end; + } + } else { + kbdev->ipa.configured_model = default_model; + } + + kbase_ipa_model_use_configured_locked(kbdev); + +end: + if (err) + kbase_ipa_term_locked(kbdev); + else + dev_info(kbdev->dev, + "Using configured power model %s, and fallback %s\n", + kbdev->ipa.configured_model->ops->name, + kbdev->ipa.fallback_model->ops->name); + + mutex_unlock(&kbdev->ipa.lock); + return err; +} +KBASE_EXPORT_TEST_API(kbase_ipa_init); + +void kbase_ipa_term(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->ipa.lock); + kbase_ipa_term_locked(kbdev); + mutex_unlock(&kbdev->ipa.lock); +} +KBASE_EXPORT_TEST_API(kbase_ipa_term); + +/** + * kbase_scale_dynamic_power() - Scale a dynamic power coefficient to an OPP + * @c: Dynamic model coefficient, in pW/(Hz V^2). Should be in range + * 0 < c < 2^26 to prevent overflow. + * @freq: Frequency, in Hz. Range: 2^23 < freq < 2^30 (~8MHz to ~1GHz) + * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) + * + * Keep a record of the approximate range of each value at every stage of the + * calculation, to ensure we don't overflow. This makes heavy use of the + * approximations 1000 = 2^10 and 1000000 = 2^20, but does the actual + * calculations in decimal for increased accuracy. + * + * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) + */ +static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq, + const u32 voltage) +{ + /* Range: 2^8 < v2 < 2^16 m(V^2) */ + const u32 v2 = (voltage * voltage) / 1000; + + /* Range: 2^3 < f_MHz < 2^10 MHz */ + const u32 f_MHz = freq / 1000000; + + /* Range: 2^11 < v2f_big < 2^26 kHz V^2 */ + const u32 v2f_big = v2 * f_MHz; + + /* Range: 2^1 < v2f < 2^16 MHz V^2 */ + const u32 v2f = v2f_big / 1000; + + /* Range (working backwards from next line): 0 < v2fc < 2^23 uW. + * Must be < 2^42 to avoid overflowing the return value. */ + const u64 v2fc = (u64) c * (u64) v2f; + + /* Range: 0 < v2fc / 1000 < 2^13 mW */ + return v2fc / 1000; +} + +/** + * kbase_scale_static_power() - Scale a static power coefficient to an OPP + * @c: Static model coefficient, in uW/V^3. Should be in range + * 0 < c < 2^32 to prevent overflow. + * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) + * + * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) + */ +u32 kbase_scale_static_power(const u32 c, const u32 voltage) +{ + /* Range: 2^8 < v2 < 2^16 m(V^2) */ + const u32 v2 = (voltage * voltage) / 1000; + + /* Range: 2^17 < v3_big < 2^29 m(V^2) mV */ + const u32 v3_big = v2 * voltage; + + /* Range: 2^7 < v3 < 2^19 m(V^3) */ + const u32 v3 = v3_big / 1000; + + /* + * Range (working backwards from next line): 0 < v3c_big < 2^33 nW. + * The result should be < 2^52 to avoid overflowing the return value. + */ + const u64 v3c_big = (u64) c * (u64) v3; + + /* Range: 0 < v3c_big / 1000000 < 2^13 mW */ + return v3c_big / 1000000; +} + +static struct kbase_ipa_model *get_current_model(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->ipa.lock); + + if (atomic_read(&kbdev->ipa_use_configured_model)) + return kbdev->ipa.configured_model; + else + return kbdev->ipa.fallback_model; +} + +static u32 get_static_power_locked(struct kbase_device *kbdev, + struct kbase_ipa_model *model, + unsigned long voltage) +{ + u32 power = 0; + int err; + u32 power_coeff; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (!model->ops->get_static_coeff) + model = kbdev->ipa.fallback_model; + + if (model->ops->get_static_coeff) { + err = model->ops->get_static_coeff(model, &power_coeff); + if (!err) + power = kbase_scale_static_power(power_coeff, + (u32) voltage); + } + + return power; +} + +#ifdef CONFIG_MALI_PWRSOFT_765 +static unsigned long kbase_get_static_power(struct devfreq *df, + unsigned long voltage) +#else +static unsigned long kbase_get_static_power(unsigned long voltage) +#endif +{ + struct kbase_ipa_model *model; + u32 power = 0; +#ifdef CONFIG_MALI_PWRSOFT_765 + struct kbase_device *kbdev = dev_get_drvdata(&df->dev); +#else + struct kbase_device *kbdev = kbase_find_device(-1); +#endif + + mutex_lock(&kbdev->ipa.lock); + + model = get_current_model(kbdev); + power = get_static_power_locked(kbdev, model, voltage); + + mutex_unlock(&kbdev->ipa.lock); + +#ifndef CONFIG_MALI_PWRSOFT_765 + kbase_release_device(kbdev); +#endif + + return power; +} + +#ifdef CONFIG_MALI_PWRSOFT_765 +static unsigned long kbase_get_dynamic_power(struct devfreq *df, + unsigned long freq, + unsigned long voltage) +#else +static unsigned long kbase_get_dynamic_power(unsigned long freq, + unsigned long voltage) +#endif +{ + struct kbase_ipa_model *model; + u32 power_coeff = 0, power = 0; + int err = 0; +#ifdef CONFIG_MALI_PWRSOFT_765 + struct kbase_device *kbdev = dev_get_drvdata(&df->dev); +#else + struct kbase_device *kbdev = kbase_find_device(-1); +#endif + + mutex_lock(&kbdev->ipa.lock); + + model = kbdev->ipa.fallback_model; + + err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); + + if (!err) + power = kbase_scale_dynamic_power(power_coeff, freq, voltage); + else + dev_err_ratelimited(kbdev->dev, + "Model %s returned error code %d\n", + model->ops->name, err); + + mutex_unlock(&kbdev->ipa.lock); + +#ifndef CONFIG_MALI_PWRSOFT_765 + kbase_release_device(kbdev); +#endif + + return power; +} + +int kbase_get_real_power(struct devfreq *df, u32 *power, + unsigned long freq, + unsigned long voltage) +{ + struct kbase_ipa_model *model; + u32 power_coeff = 0; + int err = 0; + struct kbase_device *kbdev = dev_get_drvdata(&df->dev); + + mutex_lock(&kbdev->ipa.lock); + + model = get_current_model(kbdev); + + err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); + + /* If we switch to protected model between get_current_model() and + * get_dynamic_coeff(), counter reading could fail. If that happens + * (unlikely, but possible), revert to the fallback model. */ + if (err && model != kbdev->ipa.fallback_model) { + model = kbdev->ipa.fallback_model; + err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); + } + + if (err) + goto exit_unlock; + + *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); + + if (model->ops->do_utilization_scaling_in_framework) { + struct devfreq_dev_status *status = &df->last_status; + unsigned long total_time = max(status->total_time, 1ul); + u64 busy_time = min(status->busy_time, total_time); + + *power = ((u64) *power * (u64) busy_time) / total_time; + } + + *power += get_static_power_locked(kbdev, model, voltage); + +exit_unlock: + mutex_unlock(&kbdev->ipa.lock); + + return err; +} +KBASE_EXPORT_TEST_API(kbase_get_real_power); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +struct devfreq_cooling_ops kbase_ipa_power_model_ops = { +#else +struct devfreq_cooling_power kbase_ipa_power_model_ops = { +#endif + .get_static_power = &kbase_get_static_power, + .get_dynamic_power = &kbase_get_dynamic_power, +}; +KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops); diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.h b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.h new file mode 100755 index 000000000000..67478fe911ea --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa.h @@ -0,0 +1,165 @@ +/* + * + * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_IPA_H_ +#define _KBASE_IPA_H_ + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + +struct devfreq; + +struct kbase_ipa_model { + struct list_head link; + struct kbase_device *kbdev; + void *model_data; + struct kbase_ipa_model_ops *ops; + struct list_head params; + bool missing_dt_node_warning; +}; + +/** + * kbase_ipa_model_add_param_s32 - Add an integer model parameter + * @model: pointer to IPA model + * @name: name of corresponding debugfs entry + * @addr: address where the value is stored + * @num_elems: number of elements (1 if not an array) + * @dt_required: if false, a corresponding devicetree entry is not required, + * and the current value will be used. If true, a warning is + * output and the data is zeroed + * + * Return: 0 on success, or an error code + */ +int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, + const char *name, s32 *addr, + size_t num_elems, bool dt_required); + +/** + * kbase_ipa_model_add_param_string - Add a string model parameter + * @model: pointer to IPA model + * @name: name of corresponding debugfs entry + * @addr: address where the value is stored + * @size: size, in bytes, of the value storage (so the maximum string + * length is size - 1) + * @dt_required: if false, a corresponding devicetree entry is not required, + * and the current value will be used. If true, a warning is + * output and the data is zeroed + * + * Return: 0 on success, or an error code + */ +int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, + const char *name, char *addr, + size_t size, bool dt_required); + +struct kbase_ipa_model_ops { + char *name; + /* The init, recalculate and term ops on the default model are always + * called. However, all the other models are only invoked if the model + * is selected in the device tree. Otherwise they are never + * initialized. Additional resources can be acquired by models in + * init(), however they must be terminated in the term(). + */ + int (*init)(struct kbase_ipa_model *model); + /* Called immediately after init(), or when a parameter is changed, so + * that any coefficients derived from model parameters can be + * recalculated. */ + int (*recalculate)(struct kbase_ipa_model *model); + void (*term)(struct kbase_ipa_model *model); + /* + * get_dynamic_coeff() - calculate dynamic power coefficient + * @model: pointer to model + * @coeffp: pointer to return value location + * @current_freq: frequency the GPU has been running at for the + * previous sampling period. + * + * Calculate a dynamic power coefficient, with units pW/(Hz V^2), which + * is then scaled by the IPA framework according to the current OPP's + * frequency and voltage. + * + * Return: 0 on success, or an error code. + */ + int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp, + u32 current_freq); + /* + * get_static_coeff() - calculate static power coefficient + * @model: pointer to model + * @coeffp: pointer to return value location + * + * Calculate a static power coefficient, with units uW/(V^3), which is + * scaled by the IPA framework according to the current OPP's voltage. + * + * Return: 0 on success, or an error code. + */ + int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); + /* If false, the model's get_dynamic_coeff() method accounts for how + * long the GPU was active over the sample period. If true, the + * framework will scale the calculated power according to the + * utilization stats recorded by devfreq in get_real_power(). */ + bool do_utilization_scaling_in_framework; +}; + +/* Models can be registered only in the platform's platform_init_func call */ +int kbase_ipa_model_ops_register(struct kbase_device *kbdev, + struct kbase_ipa_model_ops *new_model_ops); +struct kbase_ipa_model *kbase_ipa_get_model(struct kbase_device *kbdev, + const char *name); + +int kbase_ipa_init(struct kbase_device *kbdev); +void kbase_ipa_term(struct kbase_device *kbdev); +void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev); +void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev); +int kbase_ipa_model_recalculate(struct kbase_ipa_model *model); +struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, + struct kbase_ipa_model_ops *ops); +void kbase_ipa_term_model(struct kbase_ipa_model *model); + +extern struct kbase_ipa_model_ops kbase_g71_ipa_model_ops; + +#if MALI_UNIT_TEST +/** + * kbase_get_real_power() - get the real power consumption of the GPU + * @df: dynamic voltage and frequency scaling information for the GPU. + * @power: where to store the power consumption, in mW. + * @freq: a frequency, in HZ. + * @voltage: a voltage, in mV. + * + * This function is only exposed for use by unit tests. The returned value + * incorporates both static and dynamic power consumption. + * + * Return: 0 on success, or an error code. + */ +int kbase_get_real_power(struct devfreq *df, u32 *power, + unsigned long freq, + unsigned long voltage); +#endif /* MALI_UNIT_TEST */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +extern struct devfreq_cooling_ops kbase_ipa_power_model_ops; +#else +extern struct devfreq_cooling_power kbase_ipa_power_model_ops; +#endif + +#else /* !(defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ + +static inline void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) +{ } + +static inline void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) +{ } + +#endif /* (defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ + +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.c new file mode 100755 index 000000000000..eafc14009ddc --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.c @@ -0,0 +1,219 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include + +#include "mali_kbase.h" +#include "mali_kbase_ipa.h" +#include "mali_kbase_ipa_debugfs.h" + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)) +#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE +#endif + +struct kbase_ipa_model_param { + char *name; + union { + void *voidp; + s32 *s32p; + char *str; + } addr; + size_t size; + enum kbase_ipa_model_param_type type; + struct kbase_ipa_model *model; + struct list_head link; +}; + +static int param_int_get(void *data, u64 *val) +{ + struct kbase_ipa_model_param *param = data; + + mutex_lock(¶m->model->kbdev->ipa.lock); + *(s64 *) val = *param->addr.s32p; + mutex_unlock(¶m->model->kbdev->ipa.lock); + + return 0; +} + +static int param_int_set(void *data, u64 val) +{ + struct kbase_ipa_model_param *param = data; + struct kbase_ipa_model *model = param->model; + s64 sval = (s64) val; + int err = 0; + + if (sval < S32_MIN || sval > S32_MAX) + return -ERANGE; + + mutex_lock(¶m->model->kbdev->ipa.lock); + *param->addr.s32p = val; + err = kbase_ipa_model_recalculate(model); + mutex_unlock(¶m->model->kbdev->ipa.lock); + + return err; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_s32, param_int_get, param_int_set, "%lld\n"); + +static ssize_t param_string_get(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct kbase_ipa_model_param *param = file->private_data; + ssize_t ret; + size_t len; + + mutex_lock(¶m->model->kbdev->ipa.lock); + len = strnlen(param->addr.str, param->size - 1) + 1; + ret = simple_read_from_buffer(user_buf, count, ppos, + param->addr.str, len); + mutex_unlock(¶m->model->kbdev->ipa.lock); + + return ret; +} + +static ssize_t param_string_set(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct kbase_ipa_model_param *param = file->private_data; + struct kbase_ipa_model *model = param->model; + ssize_t ret = count; + size_t buf_size; + int err; + + mutex_lock(&model->kbdev->ipa.lock); + + if (count > param->size) { + ret = -EINVAL; + goto end; + } + + buf_size = min(param->size - 1, count); + if (copy_from_user(param->addr.str, user_buf, buf_size)) { + ret = -EFAULT; + goto end; + } + + param->addr.str[buf_size] = '\0'; + + err = kbase_ipa_model_recalculate(model); + if (err < 0) + ret = err; + +end: + mutex_unlock(&model->kbdev->ipa.lock); + + return ret; +} + +static const struct file_operations fops_string = { + .read = param_string_get, + .write = param_string_set, + .open = simple_open, + .llseek = default_llseek, +}; + +int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, + void *addr, size_t size, + enum kbase_ipa_model_param_type type) +{ + struct kbase_ipa_model_param *param; + + param = kzalloc(sizeof(*param), GFP_KERNEL); + + if (!param) + return -ENOMEM; + + /* 'name' is stack-allocated for array elements, so copy it into + * heap-allocated storage */ + param->name = kstrdup(name, GFP_KERNEL); + param->addr.voidp = addr; + param->size = size; + param->type = type; + param->model = model; + + list_add(¶m->link, &model->params); + + return 0; +} + +void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_param *param_p, *param_n; + + list_for_each_entry_safe(param_p, param_n, &model->params, link) { + list_del(¶m_p->link); + kfree(param_p->name); + kfree(param_p); + } +} + +static void kbase_ipa_model_debugfs_init(struct kbase_ipa_model *model) +{ + struct list_head *it; + struct dentry *dir; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + dir = debugfs_create_dir(model->ops->name, + model->kbdev->mali_debugfs_directory); + + if (!dir) { + dev_err(model->kbdev->dev, + "Couldn't create mali debugfs %s directory", + model->ops->name); + return; + } + + list_for_each(it, &model->params) { + struct kbase_ipa_model_param *param = + list_entry(it, + struct kbase_ipa_model_param, + link); + const struct file_operations *fops = NULL; + + switch (param->type) { + case PARAM_TYPE_S32: + fops = &fops_s32; + break; + case PARAM_TYPE_STRING: + fops = &fops_string; + break; + } + + if (unlikely(!fops)) { + dev_err(model->kbdev->dev, + "Type not set for %s parameter %s\n", + model->ops->name, param->name); + } else { + debugfs_create_file(param->name, S_IRUGO | S_IWUSR, + dir, param, fops); + } + } +} + +void kbase_ipa_debugfs_init(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->ipa.lock); + + if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) + kbase_ipa_model_debugfs_init(kbdev->ipa.configured_model); + kbase_ipa_model_debugfs_init(kbdev->ipa.fallback_model); + + mutex_unlock(&kbdev->ipa.lock); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.h new file mode 100755 index 000000000000..ec06e2096f94 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_debugfs.h @@ -0,0 +1,49 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_IPA_DEBUGFS_H_ +#define _KBASE_IPA_DEBUGFS_H_ + +enum kbase_ipa_model_param_type { + PARAM_TYPE_S32 = 1, + PARAM_TYPE_STRING, +}; + +#ifdef CONFIG_DEBUG_FS + +void kbase_ipa_debugfs_init(struct kbase_device *kbdev); +int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, + void *addr, size_t size, + enum kbase_ipa_model_param_type type); +void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model); + +#else /* CONFIG_DEBUG_FS */ + +static inline int kbase_ipa_model_param_add(struct kbase_ipa_model *model, + const char *name, void *addr, + size_t size, + enum kbase_ipa_model_param_type type) +{ + return 0; +} + +static inline void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) +{ } + +#endif /* CONFIG_DEBUG_FS */ + +#endif /* _KBASE_IPA_DEBUGFS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.c new file mode 100755 index 000000000000..b35cea451765 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.c @@ -0,0 +1,327 @@ +/* + * + * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif +#include +#include +#include + +#include "mali_kbase.h" +#include "mali_kbase_defs.h" +#include "mali_kbase_ipa_simple.h" +#include "mali_kbase_ipa_debugfs.h" + +#if MALI_UNIT_TEST + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +static unsigned long dummy_temp; + +static int kbase_simple_power_model_get_dummy_temp( + struct thermal_zone_device *tz, + unsigned long *temp) +{ + *temp = READ_ONCE(dummy_temp); + return 0; +} + +#else +static int dummy_temp; + +static int kbase_simple_power_model_get_dummy_temp( + struct thermal_zone_device *tz, + int *dummy_temp) +{ + *temp = READ_ONCE(dummy_temp); + return 0; +} +#endif + +/* Intercept calls to the kernel function using a macro */ +#ifdef thermal_zone_get_temp +#undef thermal_zone_get_temp +#endif +#define thermal_zone_get_temp(tz, temp) \ + kbase_simple_power_model_get_dummy_temp(tz, temp) + +void kbase_simple_power_model_set_dummy_temp(int temp) +{ + WRITE_ONCE(dummy_temp, temp); +} +KBASE_EXPORT_TEST_API(kbase_simple_power_model_set_dummy_temp); + +#endif /* MALI_UNIT_TEST */ + +/* + * This model is primarily designed for the Juno platform. It may not be + * suitable for other platforms. The additional resources in this model + * should preferably be minimal, as this model is rarely used when a dynamic + * model is available. + */ + +/** + * struct kbase_ipa_model_simple_data - IPA context per device + * @dynamic_coefficient: dynamic coefficient of the model + * @static_coefficient: static coefficient of the model + * @ts: Thermal scaling coefficients of the model + * @tz_name: Thermal zone name + * @gpu_tz: thermal zone device + * @poll_temperature_thread: Handle for temperature polling thread + * @current_temperature: Most recent value of polled temperature + * @temperature_poll_interval_ms: How often temperature should be checked, in ms + */ + +struct kbase_ipa_model_simple_data { + u32 dynamic_coefficient; + u32 static_coefficient; + s32 ts[4]; + char tz_name[16]; + struct thermal_zone_device *gpu_tz; + struct task_struct *poll_temperature_thread; + int current_temperature; + int temperature_poll_interval_ms; +}; +#define FALLBACK_STATIC_TEMPERATURE 55000 + +/** + * calculate_temp_scaling_factor() - Calculate temperature scaling coefficient + * @ts: Signed coefficients, in order t^0 to t^3, with units Deg^-N + * @t: Temperature, in mDeg C. Range: -2^17 < t < 2^17 + * + * Scale the temperature according to a cubic polynomial whose coefficients are + * provided in the device tree. The result is used to scale the static power + * coefficient, where 1000000 means no change. + * + * Return: Temperature scaling factor. Range 0 <= ret <= 10,000,000. + */ +static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t) +{ + /* Range: -2^24 < t2 < 2^24 m(Deg^2) */ + const s64 t2 = (t * t) / 1000; + + /* Range: -2^31 < t3 < 2^31 m(Deg^3) */ + const s64 t3 = (t * t2) / 1000; + + /* + * Sum the parts. t^[1-3] are in m(Deg^N), but the coefficients are in + * Deg^-N, so we need to multiply the last coefficient by 1000. + * Range: -2^63 < res_big < 2^63 + */ + const s64 res_big = ts[3] * t3 /* +/- 2^62 */ + + ts[2] * t2 /* +/- 2^55 */ + + ts[1] * t /* +/- 2^48 */ + + ts[0] * 1000; /* +/- 2^41 */ + + /* Range: -2^60 < res_unclamped < 2^60 */ + s64 res_unclamped = res_big / 1000; + + /* Clamp to range of 0x to 10x the static power */ + return clamp(res_unclamped, (s64) 0, (s64) 10000000); +} + +/* We can't call thermal_zone_get_temp() directly in model_static_coeff(), + * because we don't know if tz->lock is held in the same thread. So poll it in + * a separate thread to get around this. */ +static int poll_temperature(void *data) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *) data; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) + unsigned long temp; +#else + int temp; +#endif + + set_freezable(); + + while (!kthread_should_stop()) { + struct thermal_zone_device *tz = READ_ONCE(model_data->gpu_tz); + + if (tz) { + int ret; + + ret = thermal_zone_get_temp(tz, &temp); + if (ret) { + pr_warn_ratelimited("Error reading temperature for gpu thermal zone: %d\n", + ret); + temp = FALLBACK_STATIC_TEMPERATURE; + } + } else { + temp = FALLBACK_STATIC_TEMPERATURE; + } + + WRITE_ONCE(model_data->current_temperature, temp); + + msleep_interruptible(READ_ONCE(model_data->temperature_poll_interval_ms)); + + try_to_freeze(); + } + + return 0; +} + +static int model_static_coeff(struct kbase_ipa_model *model, u32 *coeffp) +{ + u32 temp_scaling_factor; + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *) model->model_data; + u64 coeff_big; + int temp; + + temp = READ_ONCE(model_data->current_temperature); + + /* Range: 0 <= temp_scaling_factor < 2^24 */ + temp_scaling_factor = calculate_temp_scaling_factor(model_data->ts, + temp); + + /* + * Range: 0 <= coeff_big < 2^52 to avoid overflowing *coeffp. This + * means static_coefficient must be in range + * 0 <= static_coefficient < 2^28. + */ + coeff_big = (u64) model_data->static_coefficient * (u64) temp_scaling_factor; + *coeffp = coeff_big / 1000000; + + return 0; +} + +static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, + u32 current_freq) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *) model->model_data; + + *coeffp = model_data->dynamic_coefficient; + + return 0; +} + +static int add_params(struct kbase_ipa_model *model) +{ + int err = 0; + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *)model->model_data; + + err = kbase_ipa_model_add_param_s32(model, "static-coefficient", + &model_data->static_coefficient, + 1, true); + if (err) + goto end; + + err = kbase_ipa_model_add_param_s32(model, "dynamic-coefficient", + &model_data->dynamic_coefficient, + 1, true); + if (err) + goto end; + + err = kbase_ipa_model_add_param_s32(model, "ts", + model_data->ts, 4, true); + if (err) + goto end; + + err = kbase_ipa_model_add_param_string(model, "thermal-zone", + model_data->tz_name, + sizeof(model_data->tz_name), true); + if (err) + goto end; + + model_data->temperature_poll_interval_ms = 200; + err = kbase_ipa_model_add_param_s32(model, "temp-poll-interval-ms", + &model_data->temperature_poll_interval_ms, + 1, false); + +end: + return err; +} + +static int kbase_simple_power_model_init(struct kbase_ipa_model *model) +{ + int err; + struct kbase_ipa_model_simple_data *model_data; + + model_data = kzalloc(sizeof(struct kbase_ipa_model_simple_data), + GFP_KERNEL); + if (!model_data) + return -ENOMEM; + + model->model_data = (void *) model_data; + + model_data->current_temperature = FALLBACK_STATIC_TEMPERATURE; + model_data->poll_temperature_thread = kthread_run(poll_temperature, + (void *) model_data, + "mali-simple-power-model-temp-poll"); + if (IS_ERR(model_data->poll_temperature_thread)) { + kfree(model_data); + return PTR_ERR(model_data->poll_temperature_thread); + } + + err = add_params(model); + if (err) { + kbase_ipa_model_param_free_all(model); + kthread_stop(model_data->poll_temperature_thread); + kfree(model_data); + } + + return err; +} + +static int kbase_simple_power_model_recalculate(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *)model->model_data; + struct thermal_zone_device *tz; + + if (!strnlen(model_data->tz_name, sizeof(model_data->tz_name))) { + tz = NULL; + } else { + tz = thermal_zone_get_zone_by_name(model_data->tz_name); + + if (IS_ERR_OR_NULL(tz)) { + pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n", + PTR_ERR(tz), model_data->tz_name); + tz = NULL; + return -EPROBE_DEFER; + } + } + + WRITE_ONCE(model_data->gpu_tz, tz); + + return 0; +} + +static void kbase_simple_power_model_term(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *)model->model_data; + + kthread_stop(model_data->poll_temperature_thread); + + kfree(model_data); +} + +struct kbase_ipa_model_ops kbase_simple_ipa_model_ops = { + .name = "mali-simple-power-model", + .init = &kbase_simple_power_model_init, + .recalculate = &kbase_simple_power_model_recalculate, + .term = &kbase_simple_power_model_term, + .get_dynamic_coeff = &model_dynamic_coeff, + .get_static_coeff = &model_static_coeff, + .do_utilization_scaling_in_framework = true, +}; diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.h b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.h new file mode 100755 index 000000000000..23cd55f5867d --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_simple.h @@ -0,0 +1,40 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_IPA_SIMPLE_H_ +#define _KBASE_IPA_SIMPLE_H_ + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + +extern struct kbase_ipa_model_ops kbase_simple_ipa_model_ops; + +#if MALI_UNIT_TEST +/** + * kbase_simple_power_model_set_dummy_temp() - set a dummy temperature value + * @temp: Temperature of the thermal zone, in millidegrees celsius. + * + * This is only intended for use in unit tests, to ensure that the temperature + * values used by the simple power model are predictable. Deterministic + * behavior is necessary to allow validation of the static power values + * computed by this model. + */ +void kbase_simple_power_model_set_dummy_temp(int temp); +#endif /* MALI_UNIT_TEST */ + +#endif /* (defined(CONFIG_MALI_BIFROST_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ + +#endif /* _KBASE_IPA_SIMPLE_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.c b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.c new file mode 100755 index 000000000000..b3d480030c2b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.c @@ -0,0 +1,217 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "mali_kbase_ipa_vinstr_common.h" + +#if MALI_UNIT_TEST +static ktime_t dummy_time; + +/* Intercept calls to the kernel function using a macro */ +#ifdef ktime_get +#undef ktime_get +#endif +#define ktime_get() (READ_ONCE(dummy_time)) + +void kbase_ipa_set_dummy_time(ktime_t t) +{ + WRITE_ONCE(dummy_time, t); +} +KBASE_EXPORT_TEST_API(kbase_ipa_set_dummy_time); + +#endif /* MALI_UNIT_TEST */ + +/** + * read_hwcnt() - read a counter value + * @model_data: pointer to model data + * @offset: offset, in bytes, into vinstr buffer + * + * Return: A 32-bit counter value. Range: 0 < value < 2^27 (worst case would be + * incrementing every cycle over a ~100ms sample period at a high frequency, + * e.g. 1 GHz: 2^30 * 0.1seconds ~= 2^27. + */ +static inline u32 kbase_ipa_read_hwcnt( + struct kbase_ipa_model_vinstr_data *model_data, + u32 offset) +{ + u8 *p = model_data->vinstr_buffer; + + return *(u32 *)&p[offset]; +} + +static inline s64 kbase_ipa_add_saturate(s64 a, s64 b) +{ + if (S64_MAX - a < b) + return S64_MAX; + return a + b; +} + +s64 kbase_ipa_sum_all_shader_cores( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, u32 counter) +{ + struct kbase_device *kbdev = model_data->kbdev; + u64 core_mask; + u32 base = 0; + s64 ret = 0; + + core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; + while (core_mask != 0ull) { + if ((core_mask & 1ull) != 0ull) { + /* 0 < counter_value < 2^27 */ + u32 counter_value = kbase_ipa_read_hwcnt(model_data, + base + counter); + + /* 0 < ret < 2^27 * max_num_cores = 2^32 */ + ret = kbase_ipa_add_saturate(ret, counter_value); + } + base += KBASE_IPA_NR_BYTES_PER_BLOCK; + core_mask >>= 1; + } + + /* Range: -2^54 < ret < 2^54 */ + ret *= coeff; + + return ret / 1000000; +} + +s64 kbase_ipa_single_counter( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, u32 counter) +{ + /* Range: 0 < counter_value < 2^27 */ + const u32 counter_value = kbase_ipa_read_hwcnt(model_data, counter); + + /* Range: -2^49 < ret < 2^49 */ + const s64 multiplied = (s64) counter_value * (s64) coeff; + + /* Range: -2^29 < return < 2^29 */ + return multiplied / 1000000; +} + +int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) +{ + struct kbase_device *kbdev = model_data->kbdev; + struct kbase_uk_hwcnt_reader_setup setup; + size_t dump_size; + + dump_size = kbase_vinstr_dump_size(kbdev); + model_data->vinstr_buffer = kzalloc(dump_size, GFP_KERNEL); + if (!model_data->vinstr_buffer) { + dev_err(kbdev->dev, "Failed to allocate IPA dump buffer"); + return -1; + } + + setup.jm_bm = ~0u; + setup.shader_bm = ~0u; + setup.tiler_bm = ~0u; + setup.mmu_l2_bm = ~0u; + model_data->vinstr_cli = kbase_vinstr_hwcnt_kernel_setup(kbdev->vinstr_ctx, + &setup, model_data->vinstr_buffer); + if (!model_data->vinstr_cli) { + dev_err(kbdev->dev, "Failed to register IPA with vinstr core"); + kfree(model_data->vinstr_buffer); + model_data->vinstr_buffer = NULL; + return -1; + } + + model_data->last_sample_read_time = ktime_get(); + kbase_vinstr_hwc_clear(model_data->vinstr_cli); + + return 0; +} + +void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) +{ + if (model_data->vinstr_cli) + kbase_vinstr_detach_client(model_data->vinstr_cli); + model_data->vinstr_cli = NULL; + kfree(model_data->vinstr_buffer); + model_data->vinstr_buffer = NULL; +} + +int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, + u32 current_freq) +{ + struct kbase_ipa_model_vinstr_data *model_data = + (struct kbase_ipa_model_vinstr_data *)model->model_data; + s64 energy = 0; + size_t i; + ktime_t now = ktime_get(); + ktime_t time_since_last_sample = + ktime_sub(now, model_data->last_sample_read_time); + /* Range: 2^0 < time_since_last_sample_ms < 2^10 (1-1000ms) */ + s64 time_since_last_sample_ms = ktime_to_ms(time_since_last_sample); + u64 coeff = 0; + u64 num_cycles; + int err = 0; + + err = kbase_vinstr_hwc_dump(model_data->vinstr_cli, + BASE_HWCNT_READER_EVENT_MANUAL); + if (err) + goto err0; + + model_data->last_sample_read_time = now; + + /* Range of 'energy' is +/- 2^34 * number of IPA groups, so around + * -2^38 < energy < 2^38 */ + for (i = 0; i < model_data->groups_def_num; i++) { + const struct kbase_ipa_group *group = &model_data->groups_def[i]; + s32 coeff, group_energy; + + coeff = model_data->group_values[i]; + group_energy = group->op(model_data, coeff, group->counter); + + energy = kbase_ipa_add_saturate(energy, group_energy); + } + + /* Range: 0 <= coeff < 2^38 */ + if (energy > 0) + coeff = energy; + + /* Scale by user-specified factor and divide by 1000. But actually + * cancel the division out, because we want the num_cycles in KHz and + * don't want to lose precision. */ + + /* Range: 0 < coeff < 2^53 */ + coeff = coeff * model_data->scaling_factor; + + if (time_since_last_sample_ms == 0) { + time_since_last_sample_ms = 1; + } else if (time_since_last_sample_ms < 0) { + err = -ERANGE; + goto err0; + } + + /* Range: 2^20 < num_cycles < 2^40 mCycles */ + num_cycles = (u64) current_freq * (u64) time_since_last_sample_ms; + /* Range: 2^10 < num_cycles < 2^30 Cycles */ + num_cycles /= 1000000; + + /* num_cycles should never be 0 in _normal_ usage (because we expect + * frequencies on the order of MHz and >10ms polling intervals), but + * protect against divide-by-zero anyway. */ + if (num_cycles == 0) + num_cycles = 1; + + /* Range: 0 < coeff < 2^43 */ + coeff = div_u64(coeff, num_cycles); + +err0: + /* Clamp to a sensible range - 2^16 gives about 14W at 400MHz/750mV */ + *coeffp = clamp(coeff, (u64) 0, (u64) 1 << 16); + return err; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.h b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.h new file mode 100755 index 000000000000..25b36c8e3089 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_common.h @@ -0,0 +1,161 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_IPA_VINSTR_COMMON_H_ +#define _KBASE_IPA_VINSTR_COMMON_H_ + +#include "mali_kbase.h" + +/* Maximum length for the name of an IPA group. */ +#define KBASE_IPA_MAX_GROUP_NAME_LEN 15 + +/* Maximum number of IPA groups for an IPA model. */ +#define KBASE_IPA_MAX_GROUP_DEF_NUM 16 + +/* Number of bytes per hardware counter in a vinstr_buffer. */ +#define KBASE_IPA_NR_BYTES_PER_CNT 4 + +/* Number of hardware counters per block in a vinstr_buffer. */ +#define KBASE_IPA_NR_CNT_PER_BLOCK 64 + +/* Number of bytes per block in a vinstr_buffer. */ +#define KBASE_IPA_NR_BYTES_PER_BLOCK \ + (KBASE_IPA_NR_CNT_PER_BLOCK * KBASE_IPA_NR_BYTES_PER_CNT) + + + +/** + * struct kbase_ipa_model_vinstr_data - IPA context per device + * @kbdev: pointer to kbase device + * @groups_def: Array of IPA groups. + * @groups_def_num: Number of elements in the array of IPA groups. + * @vinstr_cli: vinstr client handle + * @vinstr_buffer: buffer to dump hardware counters onto + * @last_sample_read_time: timestamp of last vinstr buffer read + * @scaling_factor: user-specified power scaling factor. This is + * interpreted as a fraction where the denominator is + * 1000. Range approx 0.0-32.0: + * 0 < scaling_factor < 2^15 + */ +struct kbase_ipa_model_vinstr_data { + struct kbase_device *kbdev; + s32 group_values[KBASE_IPA_MAX_GROUP_DEF_NUM]; + const struct kbase_ipa_group *groups_def; + size_t groups_def_num; + struct kbase_vinstr_client *vinstr_cli; + void *vinstr_buffer; + ktime_t last_sample_read_time; + s32 scaling_factor; +}; + +/** + * struct ipa_group - represents a single IPA group + * @name: name of the IPA group + * @default_value: default value of coefficient for IPA group. + * Coefficients are interpreted as fractions where the + * denominator is 1000000. + * @op: which operation to be performed on the counter values + * @counter: counter used to calculate energy for IPA group + */ +struct kbase_ipa_group { + char name[KBASE_IPA_MAX_GROUP_NAME_LEN + 1]; + s32 default_value; + s64 (*op)(struct kbase_ipa_model_vinstr_data *, s32, u32); + u32 counter; +}; + +/* + * sum_all_shader_cores() - sum a counter over all cores + * @model_data pointer to model data + * @coeff model coefficient. Unity is ~2^20, so range approx + * +/- 4.0: -2^22 < coeff < 2^22 + + * Calculate energy estimation based on hardware counter `counter' + * across all shader cores. + * + * Return: Sum of counter values. Range: -2^34 < ret < 2^34 + */ +s64 kbase_ipa_sum_all_shader_cores( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, u32 counter); + +/* + * sum_single_counter() - sum a single counter + * @model_data pointer to model data + * @coeff model coefficient. Unity is ~2^20, so range approx + * +/- 4.0: -2^22 < coeff < 2^22 + + * Calculate energy estimation based on hardware counter `counter'. + * + * Return: Counter value. Range: -2^34 < ret < 2^34 + */ +s64 kbase_ipa_single_counter( + struct kbase_ipa_model_vinstr_data *model_data, + s32 coeff, u32 counter); + +/* + * attach_vinstr() - attach a vinstr_buffer to an IPA model. + * @model_data pointer to model data + * + * Attach a vinstr_buffer to an IPA model. The vinstr_buffer + * allows access to the hardware counters used to calculate + * energy consumption. + * + * Return: 0 on success, or an error code. + */ +int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); + +/* + * detach_vinstr() - detach a vinstr_buffer from an IPA model. + * @model_data pointer to model data + * + * Detach a vinstr_buffer from an IPA model. + */ +void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); + +/** + * kbase_ipa_vinstr_dynamic_coeff() - calculate dynamic power based on HW counters + * @model: pointer to instantiated model + * @coeffp: pointer to location where calculated power, in + * pW/(Hz V^2), is stored. + * @current_freq: frequency the GPU has been running at over the sample + * period. In Hz. Range: 10 MHz < 1GHz, + * 2^20 < current_freq < 2^30 + * + * This is a GPU-agnostic implementation of the get_dynamic_coeff() + * function of an IPA model. It relies on the model being populated + * with GPU-specific attributes at initialization time. + * + * Return: 0 on success, or an error code. + */ +int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, + u32 current_freq); + +#if MALI_UNIT_TEST +/** + * kbase_ipa_set_dummy_time() - set a dummy monotonic time value + * @t: a monotonic time value + * + * This is only intended for use in unit tests, to ensure that the kernel time + * values used by a power model are predictable. Deterministic behavior is + * necessary to allow validation of the dynamic power values computed by the + * model. + */ +void kbase_ipa_set_dummy_time(ktime_t t); +#endif /* MALI_UNIT_TEST */ + +#endif /* _KBASE_IPA_VINSTR_COMMON_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_g71.c b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_g71.c new file mode 100755 index 000000000000..81f6fddbd79b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/ipa/mali_kbase_ipa_vinstr_g71.c @@ -0,0 +1,136 @@ +/* + * + * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +#include + +#include "mali_kbase_ipa_vinstr_common.h" +#include "mali_kbase.h" +#include "mali_kbase_ipa_debugfs.h" + + +#define JM_BASE (0 * KBASE_IPA_NR_BYTES_PER_BLOCK) +#define TILER_BASE (1 * KBASE_IPA_NR_BYTES_PER_BLOCK) +#define MMU_BASE (2 * KBASE_IPA_NR_BYTES_PER_BLOCK) +#define SC0_BASE (3 * KBASE_IPA_NR_BYTES_PER_BLOCK) + +#define GPU_ACTIVE (JM_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 6) +#define TILER_ACTIVE (TILER_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 45) +#define L2_ANY_LOOKUP (MMU_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 25) +#define FRAG_ACTIVE (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 4) +#define EXEC_CORE_ACTIVE (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 26) +#define EXEC_INSTR_COUNT (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 28) +#define TEX_COORD_ISSUE (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 40) +#define VARY_SLOT_32 (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 50) +#define VARY_SLOT_16 (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 51) +#define BEATS_RD_LSC (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 56) +#define BEATS_WR_LSC (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 61) +#define BEATS_WR_TIB (SC0_BASE + KBASE_IPA_NR_BYTES_PER_CNT * 62) + +static const struct kbase_ipa_group ipa_groups_def[] = { + { + .name = "l2_access", + .default_value = 526300, + .op = kbase_ipa_single_counter, + .counter = L2_ANY_LOOKUP, + }, + { + .name = "exec_instr_count", + .default_value = 301100, + .op = kbase_ipa_sum_all_shader_cores, + .counter = EXEC_INSTR_COUNT, + }, + { + .name = "tex_issue", + .default_value = 197400, + .op = kbase_ipa_sum_all_shader_cores, + .counter = TEX_COORD_ISSUE, + }, + { + .name = "tile_wb", + .default_value = -156400, + .op = kbase_ipa_sum_all_shader_cores, + .counter = BEATS_WR_TIB, + }, + { + .name = "gpu_active", + .default_value = 115800, + .op = kbase_ipa_single_counter, + .counter = GPU_ACTIVE, + }, +}; + +static int kbase_g71_power_model_init(struct kbase_ipa_model *model) +{ + int i, err = 0; + struct kbase_ipa_model_vinstr_data *model_data; + + model_data = kzalloc(sizeof(*model_data), GFP_KERNEL); + if (!model_data) + return -ENOMEM; + + model_data->kbdev = model->kbdev; + model_data->groups_def = ipa_groups_def; + BUILD_BUG_ON(ARRAY_SIZE(ipa_groups_def) > KBASE_IPA_MAX_GROUP_DEF_NUM); + model_data->groups_def_num = ARRAY_SIZE(ipa_groups_def); + + model->model_data = (void *) model_data; + + for (i = 0; i < ARRAY_SIZE(ipa_groups_def); ++i) { + const struct kbase_ipa_group *group = &ipa_groups_def[i]; + + model_data->group_values[i] = group->default_value; + err = kbase_ipa_model_add_param_s32(model, group->name, + &model_data->group_values[i], + 1, false); + if (err) + goto exit; + } + + model_data->scaling_factor = 15000; + err = kbase_ipa_model_add_param_s32(model, "scale", + &model_data->scaling_factor, + 1, false); + if (err) + goto exit; + + err = kbase_ipa_attach_vinstr(model_data); + +exit: + if (err) { + kbase_ipa_model_param_free_all(model); + kfree(model_data); + } + return err; +} + +static void kbase_g71_power_model_term(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_vinstr_data *model_data = + (struct kbase_ipa_model_vinstr_data *)model->model_data; + + kbase_ipa_detach_vinstr(model_data); + kfree(model_data); +} + + +struct kbase_ipa_model_ops kbase_g71_ipa_model_ops = { + .name = "mali-g71-power-model", + .init = kbase_g71_power_model_init, + .term = kbase_g71_power_model_term, + .get_dynamic_coeff = kbase_ipa_vinstr_dynamic_coeff, + .do_utilization_scaling_in_framework = false, +}; +KBASE_EXPORT_TEST_API(kbase_g71_ipa_model_ops); diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_features.h b/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_features.h new file mode 100755 index 000000000000..219586d4d2da --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_features.h @@ -0,0 +1,282 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, + * please update base/tools/hwconfig_generator/hwc_{issues,features}.py + * For more information see base/tools/hwconfig_generator/README + */ + +#ifndef _BASE_HWCONFIG_FEATURES_H_ +#define _BASE_HWCONFIG_FEATURES_H_ + +enum base_hw_feature { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, + BASE_HW_FEATURE_IMAGES_IN_FRAGMENT_SHADERS, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_V4, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_TLS_HASHING, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_generic[] = { + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t60x[] = { + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_V4, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t62x[] = { + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_V4, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t72x[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_V4, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t76x[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tFxx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t83x[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t82x[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tMIx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tHEx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tSIx[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tDVx[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + + + + + +#endif /* _BASE_HWCONFIG_FEATURES_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_issues.h b/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_issues.h new file mode 100755 index 000000000000..1c5ee496ac85 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_base_hwconfig_issues.h @@ -0,0 +1,1126 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, + * please update base/tools/hwconfig_generator/hwc_{issues,features}.py + * For more information see base/tools/hwconfig_generator/README + */ + +#ifndef _BASE_HWCONFIG_ISSUES_H_ +#define _BASE_HWCONFIG_ISSUES_H_ + +enum base_hw_issue { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_6367, + BASE_HW_ISSUE_6398, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_6787, + BASE_HW_ISSUE_7027, + BASE_HW_ISSUE_7144, + BASE_HW_ISSUE_7304, + BASE_HW_ISSUE_8073, + BASE_HW_ISSUE_8186, + BASE_HW_ISSUE_8215, + BASE_HW_ISSUE_8245, + BASE_HW_ISSUE_8250, + BASE_HW_ISSUE_8260, + BASE_HW_ISSUE_8280, + BASE_HW_ISSUE_8316, + BASE_HW_ISSUE_8381, + BASE_HW_ISSUE_8394, + BASE_HW_ISSUE_8401, + BASE_HW_ISSUE_8408, + BASE_HW_ISSUE_8443, + BASE_HW_ISSUE_8456, + BASE_HW_ISSUE_8564, + BASE_HW_ISSUE_8634, + BASE_HW_ISSUE_8778, + BASE_HW_ISSUE_8791, + BASE_HW_ISSUE_8833, + BASE_HW_ISSUE_8879, + BASE_HW_ISSUE_8896, + BASE_HW_ISSUE_8975, + BASE_HW_ISSUE_8986, + BASE_HW_ISSUE_8987, + BASE_HW_ISSUE_9010, + BASE_HW_ISSUE_9418, + BASE_HW_ISSUE_9423, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_9510, + BASE_HW_ISSUE_9566, + BASE_HW_ISSUE_9630, + BASE_HW_ISSUE_10127, + BASE_HW_ISSUE_10327, + BASE_HW_ISSUE_10410, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10487, + BASE_HW_ISSUE_10607, + BASE_HW_ISSUE_10632, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10676, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10797, + BASE_HW_ISSUE_10817, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10959, + BASE_HW_ISSUE_10969, + BASE_HW_ISSUE_10984, + BASE_HW_ISSUE_10995, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11035, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + GPUCORE_1619, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_generic[] = { + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t60x_r0p0_15dev0[] = { + BASE_HW_ISSUE_6367, + BASE_HW_ISSUE_6398, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_6787, + BASE_HW_ISSUE_7027, + BASE_HW_ISSUE_7144, + BASE_HW_ISSUE_7304, + BASE_HW_ISSUE_8073, + BASE_HW_ISSUE_8186, + BASE_HW_ISSUE_8215, + BASE_HW_ISSUE_8245, + BASE_HW_ISSUE_8250, + BASE_HW_ISSUE_8260, + BASE_HW_ISSUE_8280, + BASE_HW_ISSUE_8316, + BASE_HW_ISSUE_8381, + BASE_HW_ISSUE_8394, + BASE_HW_ISSUE_8401, + BASE_HW_ISSUE_8408, + BASE_HW_ISSUE_8443, + BASE_HW_ISSUE_8456, + BASE_HW_ISSUE_8564, + BASE_HW_ISSUE_8634, + BASE_HW_ISSUE_8778, + BASE_HW_ISSUE_8791, + BASE_HW_ISSUE_8833, + BASE_HW_ISSUE_8896, + BASE_HW_ISSUE_8975, + BASE_HW_ISSUE_8986, + BASE_HW_ISSUE_8987, + BASE_HW_ISSUE_9010, + BASE_HW_ISSUE_9418, + BASE_HW_ISSUE_9423, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_9510, + BASE_HW_ISSUE_9566, + BASE_HW_ISSUE_9630, + BASE_HW_ISSUE_10410, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10487, + BASE_HW_ISSUE_10607, + BASE_HW_ISSUE_10632, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10676, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10969, + BASE_HW_ISSUE_10984, + BASE_HW_ISSUE_10995, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11035, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_3964, + GPUCORE_1619, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t60x_r0p0_eac[] = { + BASE_HW_ISSUE_6367, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_6787, + BASE_HW_ISSUE_7027, + BASE_HW_ISSUE_7304, + BASE_HW_ISSUE_8408, + BASE_HW_ISSUE_8564, + BASE_HW_ISSUE_8778, + BASE_HW_ISSUE_8975, + BASE_HW_ISSUE_9010, + BASE_HW_ISSUE_9418, + BASE_HW_ISSUE_9423, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_9510, + BASE_HW_ISSUE_10410, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10487, + BASE_HW_ISSUE_10607, + BASE_HW_ISSUE_10632, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10676, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10969, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11035, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t60x_r0p1[] = { + BASE_HW_ISSUE_6367, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_6787, + BASE_HW_ISSUE_7027, + BASE_HW_ISSUE_7304, + BASE_HW_ISSUE_8408, + BASE_HW_ISSUE_8564, + BASE_HW_ISSUE_8778, + BASE_HW_ISSUE_8975, + BASE_HW_ISSUE_9010, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_9510, + BASE_HW_ISSUE_10410, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10487, + BASE_HW_ISSUE_10607, + BASE_HW_ISSUE_10632, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10676, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11035, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t62x_r0p1[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10127, + BASE_HW_ISSUE_10327, + BASE_HW_ISSUE_10410, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10487, + BASE_HW_ISSUE_10607, + BASE_HW_ISSUE_10632, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10676, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10817, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10959, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11035, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t62x_r1p0[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10959, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t62x_r1p1[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10959, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r0p1_50rel0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r0p2[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r0p3[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t72x_r0p0[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10797, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t72x_r1p0[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10797, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t72x_r1p1[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10797, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t72x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10797, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t76x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t60x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_8778, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t62x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tFRx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tFRx_r0p2[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tFRx_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tFRx_r2p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tFRx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t86x_r0p2[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t86x_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t86x_r2p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t86x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t83x_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t83x_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t83x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + GPUCORE_1619, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t82x_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t82x_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t82x_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t82x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tMIx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tHEx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tSIx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tDVx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + + + + + + + + + +#endif /* _BASE_HWCONFIG_ISSUES_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_base_kernel.h b/drivers/gpu/arm/bifrost_for_linux/mali_base_kernel.h new file mode 100755 index 000000000000..6f5c68e288cd --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_base_kernel.h @@ -0,0 +1,1822 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base structures shared with the kernel. + */ + +#ifndef _BASE_KERNEL_H_ +#define _BASE_KERNEL_H_ + +/* Support UK10_2 IOCTLS */ +#define BASE_LEGACY_UK10_2_SUPPORT 1 + +/* Support UK10_4 IOCTLS */ +#define BASE_LEGACY_UK10_4_SUPPORT 1 + +typedef struct base_mem_handle { + struct { + u64 handle; + } basep; +} base_mem_handle; + +#include "mali_base_mem_priv.h" +#include "mali_kbase_profiling_gator_api.h" +#include "mali_midg_coherency.h" +#include "mali_kbase_gpu_id.h" + +/* + * Dependency stuff, keep it private for now. May want to expose it if + * we decide to make the number of semaphores a configurable + * option. + */ +#define BASE_JD_ATOM_COUNT 256 + +/* Set/reset values for a software event */ +#define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) +#define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) + +#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3 + +#define BASE_MAX_COHERENT_GROUPS 16 + +#if defined CDBG_ASSERT +#define LOCAL_ASSERT CDBG_ASSERT +#elif defined KBASE_DEBUG_ASSERT +#define LOCAL_ASSERT KBASE_DEBUG_ASSERT +#else +#error assert macro not defined! +#endif + +#if defined PAGE_MASK +#define LOCAL_PAGE_LSB ~PAGE_MASK +#else +#include + +#if defined OSU_CONFIG_CPU_PAGE_SIZE_LOG2 +#define LOCAL_PAGE_LSB ((1ul << OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - 1) +#else +#error Failed to find page size +#endif +#endif + +/** + * @addtogroup base_user_api User-side Base APIs + * @{ + */ + +/** + * @addtogroup base_user_api_memory User-side Base Memory APIs + * @{ + */ + +/** + * typedef base_mem_alloc_flags - Memory allocation, access/hint flags. + * + * A combination of MEM_PROT/MEM_HINT flags must be passed to each allocator + * in order to determine the best cache policy. Some combinations are + * of course invalid (e.g. MEM_PROT_CPU_WR | MEM_HINT_CPU_RD), + * which defines a write-only region on the CPU side, which is + * heavily read by the CPU... + * Other flags are only meaningful to a particular allocator. + * More flags can be added to this list, as long as they don't clash + * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). + */ +typedef u32 base_mem_alloc_flags; + +/* Memory allocation, access/hint flags. + * + * See base_mem_alloc_flags. + */ + +/* IN */ +/* Read access CPU side + */ +#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) + +/* Write access CPU side + */ +#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) + +/* Read access GPU side + */ +#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) + +/* Write access GPU side + */ +#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) + +/* Execute allowed on the GPU side + */ +#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) + + /* BASE_MEM_HINT flags have been removed, but their values are reserved + * for backwards compatibility with older user-space drivers. The values + * can be re-used once support for r5p0 user-space drivers is removed, + * presumably in r7p0. + * + * RESERVED: (1U << 5) + * RESERVED: (1U << 6) + * RESERVED: (1U << 7) + * RESERVED: (1U << 8) + */ + +/* Grow backing store on GPU Page Fault + */ +#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) + +/* Page coherence Outer shareable, if available + */ +#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) + +/* Page coherence Inner shareable + */ +#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) + +/* Should be cached on the CPU + */ +#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) + +/* IN/OUT */ +/* Must have same VA on both the GPU and the CPU + */ +#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) + +/* OUT */ +/* Must call mmap to acquire a GPU address for the alloc + */ +#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) + +/* IN */ +/* Page coherence Outer shareable, required. + */ +#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) + +/* Secure memory + */ +#define BASE_MEM_SECURE ((base_mem_alloc_flags)1 << 16) + +/* Not needed physical memory + */ +#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) + +/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the + * addresses to be the same + */ +#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) + +/** + * Bit 19 is reserved. + * + * Do not remove, use the next unreserved bit for new flags + **/ +#define BASE_MEM_RESERVED_BIT_19 ((base_mem_alloc_flags)1 << 19) + +/* Number of bits used as flags for base memory management + * + * Must be kept in sync with the base_mem_alloc_flags flags + */ +#define BASE_MEM_FLAGS_NR_BITS 20 + +/* A mask for all output bits, excluding IN/OUT bits. + */ +#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP + +/* A mask for all input bits, including IN/OUT bits. + */ +#define BASE_MEM_FLAGS_INPUT_MASK \ + (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) + +/* A mask for all the flags which are modifiable via the base_mem_set_flags + * interface. + */ +#define BASE_MEM_FLAGS_MODIFIABLE \ + (BASE_MEM_DONT_NEED | BASE_MEM_COHERENT_SYSTEM | \ + BASE_MEM_COHERENT_LOCAL) + +/** + * enum base_mem_import_type - Memory types supported by @a base_mem_import + * + * @BASE_MEM_IMPORT_TYPE_INVALID: Invalid type + * @BASE_MEM_IMPORT_TYPE_UMP: UMP import. Handle type is ump_secure_id. + * @BASE_MEM_IMPORT_TYPE_UMM: UMM import. Handle type is a file descriptor (int) + * @BASE_MEM_IMPORT_TYPE_USER_BUFFER: User buffer import. Handle is a + * base_mem_import_user_buffer + * + * Each type defines what the supported handle type is. + * + * If any new type is added here ARM must be contacted + * to allocate a numeric value for it. + * Do not just add a new type without synchronizing with ARM + * as future releases from ARM might include other new types + * which could clash with your custom types. + */ +typedef enum base_mem_import_type { + BASE_MEM_IMPORT_TYPE_INVALID = 0, + BASE_MEM_IMPORT_TYPE_UMP = 1, + BASE_MEM_IMPORT_TYPE_UMM = 2, + BASE_MEM_IMPORT_TYPE_USER_BUFFER = 3 +} base_mem_import_type; + +/** + * struct base_mem_import_user_buffer - Handle of an imported user buffer + * + * @ptr: address of imported user buffer + * @length: length of imported user buffer in bytes + * + * This structure is used to represent a handle of an imported user buffer. + */ + +struct base_mem_import_user_buffer { + u64 ptr; + u64 length; +}; + +/** + * @brief Invalid memory handle. + * + * Return value from functions returning @ref base_mem_handle on error. + * + * @warning @ref base_mem_handle_new_invalid must be used instead of this macro + * in C++ code or other situations where compound literals cannot be used. + */ +#define BASE_MEM_INVALID_HANDLE ((base_mem_handle) { {BASEP_MEM_INVALID_HANDLE} }) + +/** + * @brief Special write-alloc memory handle. + * + * A special handle is used to represent a region where a special page is mapped + * with a write-alloc cache setup, typically used when the write result of the + * GPU isn't needed, but the GPU must write anyway. + * + * @warning @ref base_mem_handle_new_write_alloc must be used instead of this macro + * in C++ code or other situations where compound literals cannot be used. + */ +#define BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ((base_mem_handle) { {BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE} }) + +#define BASEP_MEM_INVALID_HANDLE (0ull << 12) +#define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12) +#define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12) +#define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12) +#define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12) +/* reserved handles ..-64< for future special handles */ +#define BASE_MEM_COOKIE_BASE (64ul << 12) +#define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ + BASE_MEM_COOKIE_BASE) + +/* Mask to detect 4GB boundary alignment */ +#define BASE_MEM_MASK_4GB 0xfffff000UL + + +/* Bit mask of cookies used for for memory allocation setup */ +#define KBASE_COOKIE_MASK ~1UL /* bit 0 is reserved */ + + +/** + * @brief Result codes of changing the size of the backing store allocated to a tmem region + */ +typedef enum base_backing_threshold_status { + BASE_BACKING_THRESHOLD_OK = 0, /**< Resize successful */ + BASE_BACKING_THRESHOLD_ERROR_OOM = -2, /**< Increase failed due to an out-of-memory condition */ + BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS = -4 /**< Invalid arguments (not tmem, illegal size request, etc.) */ +} base_backing_threshold_status; + +/** + * @addtogroup base_user_api_memory_defered User-side Base Defered Memory Coherency APIs + * @{ + */ + +/** + * @brief a basic memory operation (sync-set). + * + * The content of this structure is private, and should only be used + * by the accessors. + */ +typedef struct base_syncset { + struct basep_syncset basep_sset; +} base_syncset; + +/** @} end group base_user_api_memory_defered */ + +/** + * Handle to represent imported memory object. + * Simple opague handle to imported memory, can't be used + * with anything but base_external_resource_init to bind to an atom. + */ +typedef struct base_import_handle { + struct { + u64 handle; + } basep; +} base_import_handle; + +/** @} end group base_user_api_memory */ + +/** + * @addtogroup base_user_api_job_dispatch User-side Base Job Dispatcher APIs + * @{ + */ + +typedef int platform_fence_type; +#define INVALID_PLATFORM_FENCE ((platform_fence_type)-1) + +/** + * Base stream handle. + * + * References an underlying base stream object. + */ +typedef struct base_stream { + struct { + int fd; + } basep; +} base_stream; + +/** + * Base fence handle. + * + * References an underlying base fence object. + */ +typedef struct base_fence { + struct { + int fd; + int stream_fd; + } basep; +} base_fence; + +/** + * @brief Per-job data + * + * This structure is used to store per-job data, and is completely unused + * by the Base driver. It can be used to store things such as callback + * function pointer, data to handle job completion. It is guaranteed to be + * untouched by the Base driver. + */ +typedef struct base_jd_udata { + u64 blob[2]; /**< per-job data array */ +} base_jd_udata; + +/** + * @brief Memory aliasing info + * + * Describes a memory handle to be aliased. + * A subset of the handle can be chosen for aliasing, given an offset and a + * length. + * A special handle BASE_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a + * region where a special page is mapped with a write-alloc cache setup, + * typically used when the write result of the GPU isn't needed, but the GPU + * must write anyway. + * + * Offset and length are specified in pages. + * Offset must be within the size of the handle. + * Offset+length must not overrun the size of the handle. + * + * @handle Handle to alias, can be BASE_MEM_WRITE_ALLOC_PAGES_HANDLE + * @offset Offset within the handle to start aliasing from, in pages. + * Not used with BASE_MEM_WRITE_ALLOC_PAGES_HANDLE. + * @length Length to alias, in pages. For BASE_MEM_WRITE_ALLOC_PAGES_HANDLE + * specifies the number of times the special page is needed. + */ +struct base_mem_aliasing_info { + base_mem_handle handle; + u64 offset; + u64 length; +}; + +/** + * struct base_jit_alloc_info - Structure which describes a JIT allocation + * request. + * @gpu_alloc_addr: The GPU virtual address to write the JIT + * allocated GPU virtual address to. + * @va_pages: The minimum number of virtual pages required. + * @commit_pages: The minimum number of physical pages which + * should back the allocation. + * @extent: Granularity of physical pages to grow the + * allocation by during a fault. + * @id: Unique ID provided by the caller, this is used + * to pair allocation and free requests. + * Zero is not a valid value. + */ +struct base_jit_alloc_info { + u64 gpu_alloc_addr; + u64 va_pages; + u64 commit_pages; + u64 extent; + u8 id; +}; + +/** + * @brief Job dependency type. + * + * A flags field will be inserted into the atom structure to specify whether a dependency is a data or + * ordering dependency (by putting it before/after 'core_req' in the structure it should be possible to add without + * changing the structure size). + * When the flag is set for a particular dependency to signal that it is an ordering only dependency then + * errors will not be propagated. + */ +typedef u8 base_jd_dep_type; + + +#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */ +#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */ +#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */ + +/** + * @brief Job chain hardware requirements. + * + * A job chain must specify what GPU features it needs to allow the + * driver to schedule the job correctly. By not specifying the + * correct settings can/will cause an early job termination. Multiple + * values can be ORed together to specify multiple requirements. + * Special case is ::BASE_JD_REQ_DEP, which is used to express complex + * dependencies, and that doesn't execute anything on the hardware. + */ +typedef u32 base_jd_core_req; + +/* Requirements that come from the HW */ + +/** + * No requirement, dependency only + */ +#define BASE_JD_REQ_DEP ((base_jd_core_req)0) + +/** + * Requires fragment shaders + */ +#define BASE_JD_REQ_FS ((base_jd_core_req)1 << 0) + +/** + * Requires compute shaders + * This covers any of the following Midgard Job types: + * - Vertex Shader Job + * - Geometry Shader Job + * - An actual Compute Shader Job + * + * Compare this with @ref BASE_JD_REQ_ONLY_COMPUTE, which specifies that the + * job is specifically just the "Compute Shader" job type, and not the "Vertex + * Shader" nor the "Geometry Shader" job type. + */ +#define BASE_JD_REQ_CS ((base_jd_core_req)1 << 1) +#define BASE_JD_REQ_T ((base_jd_core_req)1 << 2) /**< Requires tiling */ +#define BASE_JD_REQ_CF ((base_jd_core_req)1 << 3) /**< Requires cache flushes */ +#define BASE_JD_REQ_V ((base_jd_core_req)1 << 4) /**< Requires value writeback */ + +/* SW-only requirements - the HW does not expose these as part of the job slot capabilities */ + +/* Requires fragment job with AFBC encoding */ +#define BASE_JD_REQ_FS_AFBC ((base_jd_core_req)1 << 13) + +/** + * SW-only requirement: coalesce completion events. + * If this bit is set then completion of this atom will not cause an event to + * be sent to userspace, whether successful or not; completion events will be + * deferred until an atom completes which does not have this bit set. + * + * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES. + */ +#define BASE_JD_REQ_EVENT_COALESCE ((base_jd_core_req)1 << 5) + +/** + * SW Only requirement: the job chain requires a coherent core group. We don't + * mind which coherent core group is used. + */ +#define BASE_JD_REQ_COHERENT_GROUP ((base_jd_core_req)1 << 6) + +/** + * SW Only requirement: The performance counters should be enabled only when + * they are needed, to reduce power consumption. + */ + +#define BASE_JD_REQ_PERMON ((base_jd_core_req)1 << 7) + +/** + * SW Only requirement: External resources are referenced by this atom. + * When external resources are referenced no syncsets can be bundled with the atom + * but should instead be part of a NULL jobs inserted into the dependency tree. + * The first pre_dep object must be configured for the external resouces to use, + * the second pre_dep object can be used to create other dependencies. + * + * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE. + */ +#define BASE_JD_REQ_EXTERNAL_RESOURCES ((base_jd_core_req)1 << 8) + +/** + * SW Only requirement: Software defined job. Jobs with this bit set will not be submitted + * to the hardware but will cause some action to happen within the driver + */ +#define BASE_JD_REQ_SOFT_JOB ((base_jd_core_req)1 << 9) + +#define BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME (BASE_JD_REQ_SOFT_JOB | 0x1) +#define BASE_JD_REQ_SOFT_FENCE_TRIGGER (BASE_JD_REQ_SOFT_JOB | 0x2) +#define BASE_JD_REQ_SOFT_FENCE_WAIT (BASE_JD_REQ_SOFT_JOB | 0x3) + +/** + * SW Only requirement : Replay job. + * + * If the preceding job fails, the replay job will cause the jobs specified in + * the list of base_jd_replay_payload pointed to by the jc pointer to be + * replayed. + * + * A replay job will only cause jobs to be replayed up to BASEP_JD_REPLAY_LIMIT + * times. If a job fails more than BASEP_JD_REPLAY_LIMIT times then the replay + * job is failed, as well as any following dependencies. + * + * The replayed jobs will require a number of atom IDs. If there are not enough + * free atom IDs then the replay job will fail. + * + * If the preceding job does not fail, then the replay job is returned as + * completed. + * + * The replayed jobs will never be returned to userspace. The preceding failed + * job will be returned to userspace as failed; the status of this job should + * be ignored. Completion should be determined by the status of the replay soft + * job. + * + * In order for the jobs to be replayed, the job headers will have to be + * modified. The Status field will be reset to NOT_STARTED. If the Job Type + * field indicates a Vertex Shader Job then it will be changed to Null Job. + * + * The replayed jobs have the following assumptions : + * + * - No external resources. Any required external resources will be held by the + * replay atom. + * - Pre-dependencies are created based on job order. + * - Atom numbers are automatically assigned. + * - device_nr is set to 0. This is not relevant as + * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set. + * - Priority is inherited from the replay job. + */ +#define BASE_JD_REQ_SOFT_REPLAY (BASE_JD_REQ_SOFT_JOB | 0x4) +/** + * SW only requirement: event wait/trigger job. + * + * - BASE_JD_REQ_SOFT_EVENT_WAIT: this job will block until the event is set. + * - BASE_JD_REQ_SOFT_EVENT_SET: this job sets the event, thus unblocks the + * other waiting jobs. It completes immediately. + * - BASE_JD_REQ_SOFT_EVENT_RESET: this job resets the event, making it + * possible for other jobs to wait upon. It completes immediately. + */ +#define BASE_JD_REQ_SOFT_EVENT_WAIT (BASE_JD_REQ_SOFT_JOB | 0x5) +#define BASE_JD_REQ_SOFT_EVENT_SET (BASE_JD_REQ_SOFT_JOB | 0x6) +#define BASE_JD_REQ_SOFT_EVENT_RESET (BASE_JD_REQ_SOFT_JOB | 0x7) + +#define BASE_JD_REQ_SOFT_DEBUG_COPY (BASE_JD_REQ_SOFT_JOB | 0x8) + +/** + * SW only requirement: Just In Time allocation + * + * This job requests a JIT allocation based on the request in the + * @base_jit_alloc_info structure which is passed via the jc element of + * the atom. + * + * It should be noted that the id entry in @base_jit_alloc_info must not + * be reused until it has been released via @BASE_JD_REQ_SOFT_JIT_FREE. + * + * Should this soft job fail it is expected that a @BASE_JD_REQ_SOFT_JIT_FREE + * soft job to free the JIT allocation is still made. + * + * The job will complete immediately. + */ +#define BASE_JD_REQ_SOFT_JIT_ALLOC (BASE_JD_REQ_SOFT_JOB | 0x9) +/** + * SW only requirement: Just In Time free + * + * This job requests a JIT allocation created by @BASE_JD_REQ_SOFT_JIT_ALLOC + * to be freed. The ID of the JIT allocation is passed via the jc element of + * the atom. + * + * The job will complete immediately. + */ +#define BASE_JD_REQ_SOFT_JIT_FREE (BASE_JD_REQ_SOFT_JOB | 0xa) + +/** + * SW only requirement: Map external resource + * + * This job requests external resource(s) are mapped once the dependencies + * of the job have been satisfied. The list of external resources are + * passed via the jc element of the atom which is a pointer to a + * @base_external_resource_list. + */ +#define BASE_JD_REQ_SOFT_EXT_RES_MAP (BASE_JD_REQ_SOFT_JOB | 0xb) +/** + * SW only requirement: Unmap external resource + * + * This job requests external resource(s) are unmapped once the dependencies + * of the job has been satisfied. The list of external resources are + * passed via the jc element of the atom which is a pointer to a + * @base_external_resource_list. + */ +#define BASE_JD_REQ_SOFT_EXT_RES_UNMAP (BASE_JD_REQ_SOFT_JOB | 0xc) + +/** + * HW Requirement: Requires Compute shaders (but not Vertex or Geometry Shaders) + * + * This indicates that the Job Chain contains Midgard Jobs of the 'Compute Shaders' type. + * + * In contrast to @ref BASE_JD_REQ_CS, this does \b not indicate that the Job + * Chain contains 'Geometry Shader' or 'Vertex Shader' jobs. + */ +#define BASE_JD_REQ_ONLY_COMPUTE ((base_jd_core_req)1 << 10) + +/** + * HW Requirement: Use the base_jd_atom::device_nr field to specify a + * particular core group + * + * If both @ref BASE_JD_REQ_COHERENT_GROUP and this flag are set, this flag takes priority + * + * This is only guaranteed to work for @ref BASE_JD_REQ_ONLY_COMPUTE atoms. + * + * If the core availability policy is keeping the required core group turned off, then + * the job will fail with a @ref BASE_JD_EVENT_PM_EVENT error code. + */ +#define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ((base_jd_core_req)1 << 11) + +/** + * SW Flag: If this bit is set then the successful completion of this atom + * will not cause an event to be sent to userspace + */ +#define BASE_JD_REQ_EVENT_ONLY_ON_FAILURE ((base_jd_core_req)1 << 12) + +/** + * SW Flag: If this bit is set then completion of this atom will not cause an + * event to be sent to userspace, whether successful or not. + */ +#define BASEP_JD_REQ_EVENT_NEVER ((base_jd_core_req)1 << 14) + +/** + * SW Flag: Skip GPU cache clean and invalidation before starting a GPU job. + * + * If this bit is set then the GPU's cache will not be cleaned and invalidated + * until a GPU job starts which does not have this bit set or a job completes + * which does not have the @ref BASE_JD_REQ_SKIP_CACHE_END bit set. Do not use if + * the CPU may have written to memory addressed by the job since the last job + * without this bit set was submitted. + */ +#define BASE_JD_REQ_SKIP_CACHE_START ((base_jd_core_req)1 << 15) + +/** + * SW Flag: Skip GPU cache clean and invalidation after a GPU job completes. + * + * If this bit is set then the GPU's cache will not be cleaned and invalidated + * until a GPU job completes which does not have this bit set or a job starts + * which does not have the @ref BASE_JD_REQ_SKIP_CACHE_START bti set. Do not use if + * the CPU may read from or partially overwrite memory addressed by the job + * before the next job without this bit set completes. + */ +#define BASE_JD_REQ_SKIP_CACHE_END ((base_jd_core_req)1 << 16) + +/** + * These requirement bits are currently unused in base_jd_core_req + */ +#define BASEP_JD_REQ_RESERVED \ + (~(BASE_JD_REQ_ATOM_TYPE | BASE_JD_REQ_EXTERNAL_RESOURCES | \ + BASE_JD_REQ_EVENT_ONLY_ON_FAILURE | BASEP_JD_REQ_EVENT_NEVER | \ + BASE_JD_REQ_EVENT_COALESCE | \ + BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | \ + BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ + BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END)) + +/** + * Mask of all bits in base_jd_core_req that control the type of the atom. + * + * This allows dependency only atoms to have flags set + */ +#define BASE_JD_REQ_ATOM_TYPE \ + (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T | BASE_JD_REQ_CF | \ + BASE_JD_REQ_V | BASE_JD_REQ_SOFT_JOB | BASE_JD_REQ_ONLY_COMPUTE) + +/** + * Mask of all bits in base_jd_core_req that control the type of a soft job. + */ +#define BASE_JD_REQ_SOFT_JOB_TYPE (BASE_JD_REQ_SOFT_JOB | 0x1f) + +/* + * Returns non-zero value if core requirements passed define a soft job or + * a dependency only job. + */ +#define BASE_JD_REQ_SOFT_JOB_OR_DEP(core_req) \ + ((core_req & BASE_JD_REQ_SOFT_JOB) || \ + (core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) + +/** + * @brief States to model state machine processed by kbasep_js_job_check_ref_cores(), which + * handles retaining cores for power management and affinity management. + * + * The state @ref KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY prevents an attack + * where lots of atoms could be submitted before powerup, and each has an + * affinity chosen that causes other atoms to have an affinity + * violation. Whilst the affinity was not causing violations at the time it + * was chosen, it could cause violations thereafter. For example, 1000 jobs + * could have had their affinity chosen during the powerup time, so any of + * those 1000 jobs could cause an affinity violation later on. + * + * The attack would otherwise occur because other atoms/contexts have to wait for: + * -# the currently running atoms (which are causing the violation) to + * finish + * -# and, the atoms that had their affinity chosen during powerup to + * finish. These are run preferentially because they don't cause a + * violation, but instead continue to cause the violation in others. + * -# or, the attacker is scheduled out (which might not happen for just 2 + * contexts) + * + * By re-choosing the affinity (which is designed to avoid violations at the + * time it's chosen), we break condition (2) of the wait, which minimizes the + * problem to just waiting for current jobs to finish (which can be bounded if + * the Job Scheduling Policy has a timer). + */ +enum kbase_atom_coreref_state { + /** Starting state: No affinity chosen, and cores must be requested. kbase_jd_atom::affinity==0 */ + KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED, + /** Cores requested, but waiting for them to be powered. Requested cores given by kbase_jd_atom::affinity */ + KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES, + /** Cores given by kbase_jd_atom::affinity are powered, but affinity might be out-of-date, so must recheck */ + KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY, + /** Cores given by kbase_jd_atom::affinity are powered, and affinity is up-to-date, but must check for violations */ + KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS, + /** Cores are powered, kbase_jd_atom::affinity up-to-date, no affinity violations: atom can be submitted to HW */ + KBASE_ATOM_COREREF_STATE_READY +}; + +/* + * Base Atom priority + * + * Only certain priority levels are actually implemented, as specified by the + * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority + * level that is not one of those defined below. + * + * Priority levels only affect scheduling between atoms of the same type within + * a base context, and only after the atoms have had dependencies resolved. + * Fragment atoms does not affect non-frament atoms with lower priorities, and + * the other way around. For example, a low priority atom that has had its + * dependencies resolved might run before a higher priority atom that has not + * had its dependencies resolved. + * + * The scheduling between base contexts/processes and between atoms from + * different base contexts/processes is unaffected by atom priority. + * + * The atoms are scheduled as follows with respect to their priorities: + * - Let atoms 'X' and 'Y' be for the same job slot who have dependencies + * resolved, and atom 'X' has a higher priority than atom 'Y' + * - If atom 'Y' is currently running on the HW, then it is interrupted to + * allow atom 'X' to run soon after + * - If instead neither atom 'Y' nor atom 'X' are running, then when choosing + * the next atom to run, atom 'X' will always be chosen instead of atom 'Y' + * - Any two atoms that have the same priority could run in any order with + * respect to each other. That is, there is no ordering constraint between + * atoms of the same priority. + */ +typedef u8 base_jd_prio; + +/* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */ +#define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0) +/* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and + * BASE_JD_PRIO_LOW */ +#define BASE_JD_PRIO_HIGH ((base_jd_prio)1) +/* Low atom priority. */ +#define BASE_JD_PRIO_LOW ((base_jd_prio)2) + +/* Count of the number of priority levels. This itself is not a valid + * base_jd_prio setting */ +#define BASE_JD_NR_PRIO_LEVELS 3 + +enum kbase_jd_atom_state { + /** Atom is not used */ + KBASE_JD_ATOM_STATE_UNUSED, + /** Atom is queued in JD */ + KBASE_JD_ATOM_STATE_QUEUED, + /** Atom has been given to JS (is runnable/running) */ + KBASE_JD_ATOM_STATE_IN_JS, + /** Atom has been completed, but not yet handed back to job dispatcher + * for dependency resolution */ + KBASE_JD_ATOM_STATE_HW_COMPLETED, + /** Atom has been completed, but not yet handed back to userspace */ + KBASE_JD_ATOM_STATE_COMPLETED +}; + +typedef u8 base_atom_id; /**< Type big enough to store an atom number in */ + +struct base_dependency { + base_atom_id atom_id; /**< An atom number */ + base_jd_dep_type dependency_type; /**< Dependency type */ +}; + +/* This structure has changed since UK 10.2 for which base_jd_core_req was a u16 value. + * In order to keep the size of the structure same, padding field has been adjusted + * accordingly and core_req field of a u32 type (to which UK 10.3 base_jd_core_req defines) + * is added at the end of the structure. Place in the structure previously occupied by u16 core_req + * is kept but renamed to compat_core_req and as such it can be used in ioctl call for job submission + * as long as UK 10.2 legacy is supported. Once when this support ends, this field can be left + * for possible future use. */ +typedef struct base_jd_atom_v2 { + u64 jc; /**< job-chain GPU address */ + struct base_jd_udata udata; /**< user data */ + u64 extres_list; /**< list of external resources */ + u16 nr_extres; /**< nr of external resources */ + u16 compat_core_req; /**< core requirements which correspond to the legacy support for UK 10.2 */ + struct base_dependency pre_dep[2]; /**< pre-dependencies, one need to use SETTER function to assign this field, + this is done in order to reduce possibility of improper assigment of a dependency field */ + base_atom_id atom_number; /**< unique number to identify the atom */ + base_jd_prio prio; /**< Atom priority. Refer to @ref base_jd_prio for more details */ + u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */ + u8 padding[1]; + base_jd_core_req core_req; /**< core requirements */ +} base_jd_atom_v2; + +typedef enum base_external_resource_access { + BASE_EXT_RES_ACCESS_SHARED, + BASE_EXT_RES_ACCESS_EXCLUSIVE +} base_external_resource_access; + +typedef struct base_external_resource { + u64 ext_resource; +} base_external_resource; + + +/** + * The maximum number of external resources which can be mapped/unmapped + * in a single request. + */ +#define BASE_EXT_RES_COUNT_MAX 10 + +/** + * struct base_external_resource_list - Structure which describes a list of + * external resources. + * @count: The number of resources. + * @ext_res: Array of external resources which is + * sized at allocation time. + */ +struct base_external_resource_list { + u64 count; + struct base_external_resource ext_res[1]; +}; + +struct base_jd_debug_copy_buffer { + u64 address; + u64 size; + struct base_external_resource extres; +}; + +/** + * @brief Setter for a dependency structure + * + * @param[in] dep The kbase jd atom dependency to be initialized. + * @param id The atom_id to be assigned. + * @param dep_type The dep_type to be assigned. + * + */ +static inline void base_jd_atom_dep_set(struct base_dependency *dep, + base_atom_id id, base_jd_dep_type dep_type) +{ + LOCAL_ASSERT(dep != NULL); + + /* + * make sure we don't set not allowed combinations + * of atom_id/dependency_type. + */ + LOCAL_ASSERT((id == 0 && dep_type == BASE_JD_DEP_TYPE_INVALID) || + (id > 0 && dep_type != BASE_JD_DEP_TYPE_INVALID)); + + dep->atom_id = id; + dep->dependency_type = dep_type; +} + +/** + * @brief Make a copy of a dependency structure + * + * @param[in,out] dep The kbase jd atom dependency to be written. + * @param[in] from The dependency to make a copy from. + * + */ +static inline void base_jd_atom_dep_copy(struct base_dependency *dep, + const struct base_dependency *from) +{ + LOCAL_ASSERT(dep != NULL); + + base_jd_atom_dep_set(dep, from->atom_id, from->dependency_type); +} + +/** + * @brief Soft-atom fence trigger setup. + * + * Sets up an atom to be a SW-only atom signaling a fence + * when it reaches the run state. + * + * Using the existing base dependency system the fence can + * be set to trigger when a GPU job has finished. + * + * The base fence object must not be terminated until the atom + * has been submitted to @ref base_jd_submit and @ref base_jd_submit + * has returned. + * + * @a fence must be a valid fence set up with @a base_fence_init. + * Calling this function with a uninitialized fence results in undefined behavior. + * + * @param[out] atom A pre-allocated atom to configure as a fence trigger SW atom + * @param[in] fence The base fence object to trigger. + * + * @pre @p fence must reference a @ref base_fence successfully initialized by + * calling @ref base_fence_init. + * @pre @p fence was @e not initialized by calling @ref base_fence_import, nor + * is it associated with a fence-trigger job that was already submitted + * by calling @ref base_jd_submit. + * @post @p atom can be submitted by calling @ref base_jd_submit. + */ +static inline void base_jd_fence_trigger_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence) +{ + LOCAL_ASSERT(atom); + LOCAL_ASSERT(fence); + LOCAL_ASSERT(fence->basep.fd == INVALID_PLATFORM_FENCE); + LOCAL_ASSERT(fence->basep.stream_fd >= 0); + atom->jc = (uintptr_t) fence; + atom->core_req = BASE_JD_REQ_SOFT_FENCE_TRIGGER; +} + +/** + * @brief Soft-atom fence wait setup. + * + * Sets up an atom to be a SW-only atom waiting on a fence. + * When the fence becomes triggered the atom becomes runnable + * and completes immediately. + * + * Using the existing base dependency system the fence can + * be set to block a GPU job until it has been triggered. + * + * The base fence object must not be terminated until the atom + * has been submitted to @ref base_jd_submit and + * @ref base_jd_submit has returned. + * + * @param[out] atom A pre-allocated atom to configure as a fence wait SW atom + * @param[in] fence The base fence object to wait on + * + * @pre @p fence must reference a @ref base_fence successfully initialized by + * calling @ref base_fence_import, or it must be associated with a + * fence-trigger job that was already submitted by calling + * @ref base_jd_submit. + * @post @p atom can be submitted by calling @ref base_jd_submit. + */ +static inline void base_jd_fence_wait_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence) +{ + LOCAL_ASSERT(atom); + LOCAL_ASSERT(fence); + LOCAL_ASSERT(fence->basep.fd >= 0); + atom->jc = (uintptr_t) fence; + atom->core_req = BASE_JD_REQ_SOFT_FENCE_WAIT; +} + +/** + * @brief External resource info initialization. + * + * Sets up an external resource object to reference + * a memory allocation and the type of access requested. + * + * @param[in] res The resource object to initialize + * @param handle The handle to the imported memory object, must be + * obtained by calling @ref base_mem_as_import_handle(). + * @param access The type of access requested + */ +static inline void base_external_resource_init(struct base_external_resource *res, struct base_import_handle handle, base_external_resource_access access) +{ + u64 address; + + address = handle.basep.handle; + + LOCAL_ASSERT(res != NULL); + LOCAL_ASSERT(0 == (address & LOCAL_PAGE_LSB)); + LOCAL_ASSERT(access == BASE_EXT_RES_ACCESS_SHARED || access == BASE_EXT_RES_ACCESS_EXCLUSIVE); + + res->ext_resource = address | (access & LOCAL_PAGE_LSB); +} + +/** + * @brief Job chain event code bits + * Defines the bits used to create ::base_jd_event_code + */ +enum { + BASE_JD_SW_EVENT_KERNEL = (1u << 15), /**< Kernel side event */ + BASE_JD_SW_EVENT = (1u << 14), /**< SW defined event */ + BASE_JD_SW_EVENT_SUCCESS = (1u << 13), /**< Event idicates success (SW events only) */ + BASE_JD_SW_EVENT_JOB = (0u << 11), /**< Job related event */ + BASE_JD_SW_EVENT_BAG = (1u << 11), /**< Bag related event */ + BASE_JD_SW_EVENT_INFO = (2u << 11), /**< Misc/info event */ + BASE_JD_SW_EVENT_RESERVED = (3u << 11), /**< Reserved event type */ + BASE_JD_SW_EVENT_TYPE_MASK = (3u << 11) /**< Mask to extract the type from an event code */ +}; + +/** + * @brief Job chain event codes + * + * HW and low-level SW events are represented by event codes. + * The status of jobs which succeeded are also represented by + * an event code (see ::BASE_JD_EVENT_DONE). + * Events are usually reported as part of a ::base_jd_event. + * + * The event codes are encoded in the following way: + * @li 10:0 - subtype + * @li 12:11 - type + * @li 13 - SW success (only valid if the SW bit is set) + * @li 14 - SW event (HW event if not set) + * @li 15 - Kernel event (should never be seen in userspace) + * + * Events are split up into ranges as follows: + * - BASE_JD_EVENT_RANGE_\_START + * - BASE_JD_EVENT_RANGE_\_END + * + * \a code is in \'s range when: + * - BASE_JD_EVENT_RANGE_\_START <= code < BASE_JD_EVENT_RANGE_\_END + * + * Ranges can be asserted for adjacency by testing that the END of the previous + * is equal to the START of the next. This is useful for optimizing some tests + * for range. + * + * A limitation is that the last member of this enum must explicitly be handled + * (with an assert-unreachable statement) in switch statements that use + * variables of this type. Otherwise, the compiler warns that we have not + * handled that enum value. + */ +typedef enum base_jd_event_code { + /* HW defined exceptions */ + + /** Start of HW Non-fault status codes + * + * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault, + * because the job was hard-stopped + */ + BASE_JD_EVENT_RANGE_HW_NONFAULT_START = 0, + + /* non-fatal exceptions */ + BASE_JD_EVENT_NOT_STARTED = 0x00, /**< Can't be seen by userspace, treated as 'previous job done' */ + BASE_JD_EVENT_DONE = 0x01, + BASE_JD_EVENT_STOPPED = 0x03, /**< Can't be seen by userspace, becomes TERMINATED, DONE or JOB_CANCELLED */ + BASE_JD_EVENT_TERMINATED = 0x04, /**< This is actually a fault status code - the job was hard stopped */ + BASE_JD_EVENT_ACTIVE = 0x08, /**< Can't be seen by userspace, jobs only returned on complete/fail/cancel */ + + /** End of HW Non-fault status codes + * + * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault, + * because the job was hard-stopped + */ + BASE_JD_EVENT_RANGE_HW_NONFAULT_END = 0x40, + + /** Start of HW fault and SW Error status codes */ + BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START = 0x40, + + /* job exceptions */ + BASE_JD_EVENT_JOB_CONFIG_FAULT = 0x40, + BASE_JD_EVENT_JOB_POWER_FAULT = 0x41, + BASE_JD_EVENT_JOB_READ_FAULT = 0x42, + BASE_JD_EVENT_JOB_WRITE_FAULT = 0x43, + BASE_JD_EVENT_JOB_AFFINITY_FAULT = 0x44, + BASE_JD_EVENT_JOB_BUS_FAULT = 0x48, + BASE_JD_EVENT_INSTR_INVALID_PC = 0x50, + BASE_JD_EVENT_INSTR_INVALID_ENC = 0x51, + BASE_JD_EVENT_INSTR_TYPE_MISMATCH = 0x52, + BASE_JD_EVENT_INSTR_OPERAND_FAULT = 0x53, + BASE_JD_EVENT_INSTR_TLS_FAULT = 0x54, + BASE_JD_EVENT_INSTR_BARRIER_FAULT = 0x55, + BASE_JD_EVENT_INSTR_ALIGN_FAULT = 0x56, + BASE_JD_EVENT_DATA_INVALID_FAULT = 0x58, + BASE_JD_EVENT_TILE_RANGE_FAULT = 0x59, + BASE_JD_EVENT_STATE_FAULT = 0x5A, + BASE_JD_EVENT_OUT_OF_MEMORY = 0x60, + BASE_JD_EVENT_UNKNOWN = 0x7F, + + /* GPU exceptions */ + BASE_JD_EVENT_DELAYED_BUS_FAULT = 0x80, + BASE_JD_EVENT_SHAREABILITY_FAULT = 0x88, + + /* MMU exceptions */ + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1 = 0xC1, + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2 = 0xC2, + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3 = 0xC3, + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4 = 0xC4, + BASE_JD_EVENT_PERMISSION_FAULT = 0xC8, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1 = 0xD1, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2 = 0xD2, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3 = 0xD3, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4 = 0xD4, + BASE_JD_EVENT_ACCESS_FLAG = 0xD8, + + /* SW defined exceptions */ + BASE_JD_EVENT_MEM_GROWTH_FAILED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x000, + BASE_JD_EVENT_TIMED_OUT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x001, + BASE_JD_EVENT_JOB_CANCELLED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x002, + BASE_JD_EVENT_JOB_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x003, + BASE_JD_EVENT_PM_EVENT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x004, + BASE_JD_EVENT_FORCE_REPLAY = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x005, + + BASE_JD_EVENT_BAG_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_BAG | 0x003, + + /** End of HW fault and SW Error status codes */ + BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_RESERVED | 0x3FF, + + /** Start of SW Success status codes */ + BASE_JD_EVENT_RANGE_SW_SUCCESS_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | 0x000, + + BASE_JD_EVENT_PROGRESS_REPORT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_JOB | 0x000, + BASE_JD_EVENT_BAG_DONE = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_BAG | 0x000, + BASE_JD_EVENT_DRV_TERMINATED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_INFO | 0x000, + + /** End of SW Success status codes */ + BASE_JD_EVENT_RANGE_SW_SUCCESS_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_RESERVED | 0x3FF, + + /** Start of Kernel-only status codes. Such codes are never returned to user-space */ + BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | 0x000, + BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x000, + + /** End of Kernel-only status codes. */ + BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_RESERVED | 0x3FF +} base_jd_event_code; + +/** + * @brief Event reporting structure + * + * This structure is used by the kernel driver to report information + * about GPU events. The can either be HW-specific events or low-level + * SW events, such as job-chain completion. + * + * The event code contains an event type field which can be extracted + * by ANDing with ::BASE_JD_SW_EVENT_TYPE_MASK. + * + * Based on the event type base_jd_event::data holds: + * @li ::BASE_JD_SW_EVENT_JOB : the offset in the ring-buffer for the completed + * job-chain + * @li ::BASE_JD_SW_EVENT_BAG : The address of the ::base_jd_bag that has + * been completed (ie all contained job-chains have been completed). + * @li ::BASE_JD_SW_EVENT_INFO : base_jd_event::data not used + */ +typedef struct base_jd_event_v2 { + base_jd_event_code event_code; /**< event code */ + base_atom_id atom_number; /**< the atom number that has completed */ + struct base_jd_udata udata; /**< user data */ +} base_jd_event_v2; + +/** + * @brief Structure for BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS jobs. + * + * This structure is stored into the memory pointed to by the @c jc field + * of @ref base_jd_atom. + * + * It must not occupy the same CPU cache line(s) as any neighboring data. + * This is to avoid cases where access to pages containing the structure + * is shared between cached and un-cached memory regions, which would + * cause memory corruption. + */ + +typedef struct base_dump_cpu_gpu_counters { + u64 system_time; + u64 cycle_counter; + u64 sec; + u32 usec; + u8 padding[36]; +} base_dump_cpu_gpu_counters; + +/** @} end group base_user_api_job_dispatch */ + +#define GPU_MAX_JOB_SLOTS 16 + +/** + * @page page_base_user_api_gpuprops User-side Base GPU Property Query API + * + * The User-side Base GPU Property Query API encapsulates two + * sub-modules: + * + * - @ref base_user_api_gpuprops_dyn "Dynamic GPU Properties" + * - @ref base_plat_config_gpuprops "Base Platform Config GPU Properties" + * + * There is a related third module outside of Base, which is owned by the MIDG + * module: + * - @ref gpu_props_static "Midgard Compile-time GPU Properties" + * + * Base only deals with properties that vary between different Midgard + * implementations - the Dynamic GPU properties and the Platform Config + * properties. + * + * For properties that are constant for the Midgard Architecture, refer to the + * MIDG module. However, we will discuss their relevance here just to + * provide background information. + * + * @section sec_base_user_api_gpuprops_about About the GPU Properties in Base and MIDG modules + * + * The compile-time properties (Platform Config, Midgard Compile-time + * properties) are exposed as pre-processor macros. + * + * Complementing the compile-time properties are the Dynamic GPU + * Properties, which act as a conduit for the Midgard Configuration + * Discovery. + * + * In general, the dynamic properties are present to verify that the platform + * has been configured correctly with the right set of Platform Config + * Compile-time Properties. + * + * As a consistent guide across the entire DDK, the choice for dynamic or + * compile-time should consider the following, in order: + * -# Can the code be written so that it doesn't need to know the + * implementation limits at all? + * -# If you need the limits, get the information from the Dynamic Property + * lookup. This should be done once as you fetch the context, and then cached + * as part of the context data structure, so it's cheap to access. + * -# If there's a clear and arguable inefficiency in using Dynamic Properties, + * then use a Compile-Time Property (Platform Config, or Midgard Compile-time + * property). Examples of where this might be sensible follow: + * - Part of a critical inner-loop + * - Frequent re-use throughout the driver, causing significant extra load + * instructions or control flow that would be worthwhile optimizing out. + * + * We cannot provide an exhaustive set of examples, neither can we provide a + * rule for every possible situation. Use common sense, and think about: what + * the rest of the driver will be doing; how the compiler might represent the + * value if it is a compile-time constant; whether an OEM shipping multiple + * devices would benefit much more from a single DDK binary, instead of + * insignificant micro-optimizations. + * + * @section sec_base_user_api_gpuprops_dyn Dynamic GPU Properties + * + * Dynamic GPU properties are presented in two sets: + * -# the commonly used properties in @ref base_gpu_props, which have been + * unpacked from GPU register bitfields. + * -# The full set of raw, unprocessed properties in @ref gpu_raw_gpu_props + * (also a member of @ref base_gpu_props). All of these are presented in + * the packed form, as presented by the GPU registers themselves. + * + * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to + * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device + * behaving differently?". In this case, all information about the + * configuration is potentially useful, but it does not need to be processed + * by the driver. Instead, the raw registers can be processed by the Mali + * Tools software on the host PC. + * + * The properties returned extend the Midgard Configuration Discovery + * registers. For example, GPU clock speed is not specified in the Midgard + * Architecture, but is necessary for OpenCL's clGetDeviceInfo() function. + * + * The GPU properties are obtained by a call to + * _mali_base_get_gpu_props(). This simply returns a pointer to a const + * base_gpu_props structure. It is constant for the life of a base + * context. Multiple calls to _mali_base_get_gpu_props() to a base context + * return the same pointer to a constant structure. This avoids cache pollution + * of the common data. + * + * This pointer must not be freed, because it does not point to the start of a + * region allocated by the memory allocator; instead, just close the @ref + * base_context. + * + * + * @section sec_base_user_api_gpuprops_config Platform Config Compile-time Properties + * + * The Platform Config File sets up gpu properties that are specific to a + * certain platform. Properties that are 'Implementation Defined' in the + * Midgard Architecture spec are placed here. + * + * @note Reference configurations are provided for Midgard Implementations, such as + * the Mali-T600 family. The customer need not repeat this information, and can select one of + * these reference configurations. For example, VA_BITS, PA_BITS and the + * maximum number of samples per pixel might vary between Midgard Implementations, but + * \b not for platforms using the Mali-T604. This information is placed in + * the reference configuration files. + * + * The System Integrator creates the following structure: + * - platform_XYZ + * - platform_XYZ/plat + * - platform_XYZ/plat/plat_config.h + * + * They then edit plat_config.h, using the example plat_config.h files as a + * guide. + * + * At the very least, the customer must set @ref CONFIG_GPU_CORE_TYPE, and will + * receive a helpful \#error message if they do not do this correctly. This + * selects the Reference Configuration for the Midgard Implementation. The rationale + * behind this decision (against asking the customer to write \#include + * in their plat_config.h) is as follows: + * - This mechanism 'looks' like a regular config file (such as Linux's + * .config) + * - It is difficult to get wrong in a way that will produce strange build + * errors: + * - They need not know where the mali_t600.h, other_midg_gpu.h etc. files are stored - and + * so they won't accidentally pick another file with 'mali_t600' in its name + * - When the build doesn't work, the System Integrator may think the DDK is + * doesn't work, and attempt to fix it themselves: + * - For the @ref CONFIG_GPU_CORE_TYPE mechanism, the only way to get past the + * error is to set @ref CONFIG_GPU_CORE_TYPE, and this is what the \#error tells + * you. + * - For a \#include mechanism, checks must still be made elsewhere, which the + * System Integrator may try working around by setting \#defines (such as + * VA_BITS) themselves in their plat_config.h. In the worst case, they may + * set the prevention-mechanism \#define of + * "A_CORRECT_MIDGARD_CORE_WAS_CHOSEN". + * - In this case, they would believe they are on the right track, because + * the build progresses with their fix, but with errors elsewhere. + * + * However, there is nothing to prevent the customer using \#include to organize + * their own configurations files hierarchically. + * + * The mechanism for the header file processing is as follows: + * + * @dot + digraph plat_config_mechanism { + rankdir=BT + size="6,6" + + "mali_base.h"; + "gpu/mali_gpu.h"; + + node [ shape=box ]; + { + rank = same; ordering = out; + + "gpu/mali_gpu_props.h"; + "base/midg_gpus/mali_t600.h"; + "base/midg_gpus/other_midg_gpu.h"; + } + { rank = same; "plat/plat_config.h"; } + { + rank = same; + "gpu/mali_gpu.h" [ shape=box ]; + gpu_chooser [ label="" style="invisible" width=0 height=0 fixedsize=true ]; + select_gpu [ label="Mali-T600 | Other\n(select_gpu.h)" shape=polygon,sides=4,distortion=0.25 width=3.3 height=0.99 fixedsize=true ] ; + } + node [ shape=box ]; + { rank = same; "plat/plat_config.h"; } + { rank = same; "mali_base.h"; } + + "mali_base.h" -> "gpu/mali_gpu.h" -> "gpu/mali_gpu_props.h"; + "mali_base.h" -> "plat/plat_config.h" ; + "mali_base.h" -> select_gpu ; + + "plat/plat_config.h" -> gpu_chooser [style="dotted,bold" dir=none weight=4] ; + gpu_chooser -> select_gpu [style="dotted,bold"] ; + + select_gpu -> "base/midg_gpus/mali_t600.h" ; + select_gpu -> "base/midg_gpus/other_midg_gpu.h" ; + } + @enddot + * + * + * @section sec_base_user_api_gpuprops_kernel Kernel Operation + * + * During Base Context Create time, user-side makes a single kernel call: + * - A call to fill user memory with GPU information structures + * + * The kernel-side will fill the provided the entire processed @ref base_gpu_props + * structure, because this information is required in both + * user and kernel side; it does not make sense to decode it twice. + * + * Coherency groups must be derived from the bitmasks, but this can be done + * kernel side, and just once at kernel startup: Coherency groups must already + * be known kernel-side, to support chains that specify a 'Only Coherent Group' + * SW requirement, or 'Only Coherent Group with Tiler' SW requirement. + * + * @section sec_base_user_api_gpuprops_cocalc Coherency Group calculation + * Creation of the coherent group data is done at device-driver startup, and so + * is one-time. This will most likely involve a loop with CLZ, shifting, and + * bit clearing on the L2_PRESENT mask, depending on whether the + * system is L2 Coherent. The number of shader cores is done by a + * population count, since faulty cores may be disabled during production, + * producing a non-contiguous mask. + * + * The memory requirements for this algorithm can be determined either by a u64 + * population count on the L2_PRESENT mask (a LUT helper already is + * required for the above), or simple assumption that there can be no more than + * 16 coherent groups, since core groups are typically 4 cores. + */ + +/** + * @addtogroup base_user_api_gpuprops User-side Base GPU Property Query APIs + * @{ + */ + +/** + * @addtogroup base_user_api_gpuprops_dyn Dynamic HW Properties + * @{ + */ + +#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3 + +#define BASE_MAX_COHERENT_GROUPS 16 + +struct mali_base_gpu_core_props { + /** + * Product specific value. + */ + u32 product_id; + + /** + * Status of the GPU release. + * No defined values, but starts at 0 and increases by one for each + * release status (alpha, beta, EAC, etc.). + * 4 bit values (0-15). + */ + u16 version_status; + + /** + * Minor release number of the GPU. "P" part of an "RnPn" release number. + * 8 bit values (0-255). + */ + u16 minor_revision; + + /** + * Major release number of the GPU. "R" part of an "RnPn" release number. + * 4 bit values (0-15). + */ + u16 major_revision; + + u16 padding; + + /** + * This property is deprecated since it has not contained the real current + * value of GPU clock speed. It is kept here only for backwards compatibility. + * For the new ioctl interface, it is ignored and is treated as a padding + * to keep the structure of the same size and retain the placement of its + * members. + */ + u32 gpu_speed_mhz; + + /** + * @usecase GPU clock max/min speed is required for computing best/worst case + * in tasks as job scheduling ant irq_throttling. (It is not specified in the + * Midgard Architecture). + * Also, GPU clock max speed is used for OpenCL's clGetDeviceInfo() function. + */ + u32 gpu_freq_khz_max; + u32 gpu_freq_khz_min; + + /** + * Size of the shader program counter, in bits. + */ + u32 log2_program_counter_size; + + /** + * TEXTURE_FEATURES_x registers, as exposed by the GPU. This is a + * bitpattern where a set bit indicates that the format is supported. + * + * Before using a texture format, it is recommended that the corresponding + * bit be checked. + */ + u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; + + /** + * Theoretical maximum memory available to the GPU. It is unlikely that a + * client will be able to allocate all of this memory for their own + * purposes, but this at least provides an upper bound on the memory + * available to the GPU. + * + * This is required for OpenCL's clGetDeviceInfo() call when + * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The + * client will not be expecting to allocate anywhere near this value. + */ + u64 gpu_available_memory_size; +}; + +/** + * + * More information is possible - but associativity and bus width are not + * required by upper-level apis. + */ +struct mali_base_gpu_l2_cache_props { + u8 log2_line_size; + u8 log2_cache_size; + u8 num_l2_slices; /* Number of L2C slices. 1 or higher */ + u8 padding[5]; +}; + +struct mali_base_gpu_tiler_props { + u32 bin_size_bytes; /* Max is 4*2^15 */ + u32 max_active_levels; /* Max is 2^15 */ +}; + +/** + * GPU threading system details. + */ +struct mali_base_gpu_thread_props { + u32 max_threads; /* Max. number of threads per core */ + u32 max_workgroup_size; /* Max. number of threads per workgroup */ + u32 max_barrier_size; /* Max. number of threads that can synchronize on a simple barrier */ + u16 max_registers; /* Total size [1..65535] of the register file available per core. */ + u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */ + u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */ + u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */ + u8 padding[7]; +}; + +/** + * @brief descriptor for a coherent group + * + * \c core_mask exposes all cores in that coherent group, and \c num_cores + * provides a cached population-count for that mask. + * + * @note Whilst all cores are exposed in the mask, not all may be available to + * the application, depending on the Kernel Power policy. + * + * @note if u64s must be 8-byte aligned, then this structure has 32-bits of wastage. + */ +struct mali_base_gpu_coherent_group { + u64 core_mask; /**< Core restriction mask required for the group */ + u16 num_cores; /**< Number of cores in the group */ + u16 padding[3]; +}; + +/** + * @brief Coherency group information + * + * Note that the sizes of the members could be reduced. However, the \c group + * member might be 8-byte aligned to ensure the u64 core_mask is 8-byte + * aligned, thus leading to wastage if the other members sizes were reduced. + * + * The groups are sorted by core mask. The core masks are non-repeating and do + * not intersect. + */ +struct mali_base_gpu_coherent_group_info { + u32 num_groups; + + /** + * Number of core groups (coherent or not) in the GPU. Equivalent to the number of L2 Caches. + * + * The GPU Counter dumping writes 2048 bytes per core group, regardless of + * whether the core groups are coherent or not. Hence this member is needed + * to calculate how much memory is required for dumping. + * + * @note Do not use it to work out how many valid elements are in the + * group[] member. Use num_groups instead. + */ + u32 num_core_groups; + + /** + * Coherency features of the memory, accessed by @ref gpu_mem_features + * methods + */ + u32 coherency; + + u32 padding; + + /** + * Descriptors of coherent groups + */ + struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS]; +}; + +/** + * A complete description of the GPU's Hardware Configuration Discovery + * registers. + * + * The information is presented inefficiently for access. For frequent access, + * the values should be better expressed in an unpacked form in the + * base_gpu_props structure. + * + * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to + * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device + * behaving differently?". In this case, all information about the + * configuration is potentially useful, but it does not need to be processed + * by the driver. Instead, the raw registers can be processed by the Mali + * Tools software on the host PC. + * + */ +struct gpu_raw_gpu_props { + u64 shader_present; + u64 tiler_present; + u64 l2_present; + u64 stack_present; + + u32 l2_features; + u32 suspend_size; /* API 8.2+ */ + u32 mem_features; + u32 mmu_features; + + u32 as_present; + + u32 js_present; + u32 js_features[GPU_MAX_JOB_SLOTS]; + u32 tiler_features; + u32 texture_features[3]; + + u32 gpu_id; + + u32 thread_max_threads; + u32 thread_max_workgroup_size; + u32 thread_max_barrier_size; + u32 thread_features; + + /* + * Note: This is the _selected_ coherency mode rather than the + * available modes as exposed in the coherency_features register. + */ + u32 coherency_mode; +}; + +/** + * Return structure for _mali_base_get_gpu_props(). + * + * NOTE: the raw_props member in this data structure contains the register + * values from which the value of the other members are derived. The derived + * members exist to allow for efficient access and/or shielding the details + * of the layout of the registers. + * + */ +typedef struct mali_base_gpu_props { + struct mali_base_gpu_core_props core_props; + struct mali_base_gpu_l2_cache_props l2_props; + u64 unused_1; /* keep for backwards compatibility */ + struct mali_base_gpu_tiler_props tiler_props; + struct mali_base_gpu_thread_props thread_props; + + /** This member is large, likely to be 128 bytes */ + struct gpu_raw_gpu_props raw_props; + + /** This must be last member of the structure */ + struct mali_base_gpu_coherent_group_info coherency_info; +} base_gpu_props; + +/** @} end group base_user_api_gpuprops_dyn */ + +/** @} end group base_user_api_gpuprops */ + +/** + * @addtogroup base_user_api_core User-side Base core APIs + * @{ + */ + +/** + * \enum base_context_create_flags + * + * Flags to pass to ::base_context_init. + * Flags can be ORed together to enable multiple things. + * + * These share the same space as BASEP_CONTEXT_FLAG_*, and so must + * not collide with them. + */ +enum base_context_create_flags { + /** No flags set */ + BASE_CONTEXT_CREATE_FLAG_NONE = 0, + + /** Base context is embedded in a cctx object (flag used for CINSTR software counter macros) */ + BASE_CONTEXT_CCTX_EMBEDDED = (1u << 0), + + /** Base context is a 'System Monitor' context for Hardware counters. + * + * One important side effect of this is that job submission is disabled. */ + BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED = (1u << 1) +}; + +/** + * Bitpattern describing the ::base_context_create_flags that can be passed to base_context_init() + */ +#define BASE_CONTEXT_CREATE_ALLOWED_FLAGS \ + (((u32)BASE_CONTEXT_CCTX_EMBEDDED) | \ + ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED)) + +/** + * Bitpattern describing the ::base_context_create_flags that can be passed to the kernel + */ +#define BASE_CONTEXT_CREATE_KERNEL_FLAGS \ + ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) + +/* + * Private flags used on the base context + * + * These start at bit 31, and run down to zero. + * + * They share the same space as @ref base_context_create_flags, and so must + * not collide with them. + */ +/** Private flag tracking whether job descriptor dumping is disabled */ +#define BASEP_CONTEXT_FLAG_JOB_DUMP_DISABLED ((u32)(1 << 31)) + +/** @} end group base_user_api_core */ + +/** @} end group base_user_api */ + +/** + * @addtogroup base_plat_config_gpuprops Base Platform Config GPU Properties + * @{ + * + * C Pre-processor macros are exposed here to do with Platform + * Config. + * + * These include: + * - GPU Properties that are constant on a particular Midgard Family + * Implementation e.g. Maximum samples per pixel on Mali-T600. + * - General platform config for the GPU, such as the GPU major and minor + * revison. + */ + +/** @} end group base_plat_config_gpuprops */ + +/** + * @addtogroup base_api Base APIs + * @{ + */ + +/** + * @brief The payload for a replay job. This must be in GPU memory. + */ +typedef struct base_jd_replay_payload { + /** + * Pointer to the first entry in the base_jd_replay_jc list. These + * will be replayed in @b reverse order (so that extra ones can be added + * to the head in future soft jobs without affecting this soft job) + */ + u64 tiler_jc_list; + + /** + * Pointer to the fragment job chain. + */ + u64 fragment_jc; + + /** + * Pointer to the tiler heap free FBD field to be modified. + */ + u64 tiler_heap_free; + + /** + * Hierarchy mask for the replayed fragment jobs. May be zero. + */ + u16 fragment_hierarchy_mask; + + /** + * Hierarchy mask for the replayed tiler jobs. May be zero. + */ + u16 tiler_hierarchy_mask; + + /** + * Default weight to be used for hierarchy levels not in the original + * mask. + */ + u32 hierarchy_default_weight; + + /** + * Core requirements for the tiler job chain + */ + base_jd_core_req tiler_core_req; + + /** + * Core requirements for the fragment job chain + */ + base_jd_core_req fragment_core_req; +} base_jd_replay_payload; + +#ifdef BASE_LEGACY_UK10_2_SUPPORT +typedef struct base_jd_replay_payload_uk10_2 { + u64 tiler_jc_list; + u64 fragment_jc; + u64 tiler_heap_free; + u16 fragment_hierarchy_mask; + u16 tiler_hierarchy_mask; + u32 hierarchy_default_weight; + u16 tiler_core_req; + u16 fragment_core_req; + u8 padding[4]; +} base_jd_replay_payload_uk10_2; +#endif /* BASE_LEGACY_UK10_2_SUPPORT */ + +/** + * @brief An entry in the linked list of job chains to be replayed. This must + * be in GPU memory. + */ +typedef struct base_jd_replay_jc { + /** + * Pointer to next entry in the list. A setting of NULL indicates the + * end of the list. + */ + u64 next; + + /** + * Pointer to the job chain. + */ + u64 jc; + +} base_jd_replay_jc; + +/* Maximum number of jobs allowed in a fragment chain in the payload of a + * replay job */ +#define BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT 256 + +/** @} end group base_api */ + +typedef struct base_profiling_controls { + u32 profiling_controls[FBDUMP_CONTROL_MAX]; +} base_profiling_controls; + +/* Enable additional tracepoints for latency measurements (TL_ATOM_READY, + * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) */ +#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) + +/* Indicate that job dumping is enabled. This could affect certain timers + * to account for the performance impact. */ +#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) + +#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ + BASE_TLSTREAM_JOB_DUMPING_ENABLED) + +#endif /* _BASE_KERNEL_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_base_mem_priv.h b/drivers/gpu/arm/bifrost_for_linux/mali_base_mem_priv.h new file mode 100755 index 000000000000..4a98a72cc37a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_base_mem_priv.h @@ -0,0 +1,52 @@ +/* + * + * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _BASE_MEM_PRIV_H_ +#define _BASE_MEM_PRIV_H_ + +#define BASE_SYNCSET_OP_MSYNC (1U << 0) +#define BASE_SYNCSET_OP_CSYNC (1U << 1) + +/* + * This structure describe a basic memory coherency operation. + * It can either be: + * @li a sync from CPU to Memory: + * - type = ::BASE_SYNCSET_OP_MSYNC + * - mem_handle = a handle to the memory object on which the operation + * is taking place + * - user_addr = the address of the range to be synced + * - size = the amount of data to be synced, in bytes + * - offset is ignored. + * @li a sync from Memory to CPU: + * - type = ::BASE_SYNCSET_OP_CSYNC + * - mem_handle = a handle to the memory object on which the operation + * is taking place + * - user_addr = the address of the range to be synced + * - size = the amount of data to be synced, in bytes. + * - offset is ignored. + */ +struct basep_syncset { + base_mem_handle mem_handle; + u64 user_addr; + u64 size; + u8 type; + u8 padding[7]; +}; + +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_base_vendor_specific_func.h b/drivers/gpu/arm/bifrost_for_linux/mali_base_vendor_specific_func.h new file mode 100755 index 000000000000..be454a216a39 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_base_vendor_specific_func.h @@ -0,0 +1,24 @@ +/* + * + * (C) COPYRIGHT 2010, 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +#ifndef _BASE_VENDOR_SPEC_FUNC_H_ +#define _BASE_VENDOR_SPEC_FUNC_H_ + +int kbase_get_vendor_specific_cpu_clock_speed(u32 * const); + +#endif /*_BASE_VENDOR_SPEC_FUNC_H_*/ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase.h new file mode 100755 index 000000000000..1fe936ea6012 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase.h @@ -0,0 +1,616 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_H_ +#define _KBASE_H_ + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) +#include +#endif +#include +#include +#include +#include +#include + +#include "mali_base_kernel.h" +#include +#include + +/* + * Include mali_kbase_defs.h first as this provides types needed by other local + * header files. + */ +#include "mali_kbase_defs.h" + +#include "mali_kbase_context.h" +#include "mali_kbase_strings.h" +#include "mali_kbase_mem_lowlevel.h" +#include "mali_kbase_trace_timeline.h" +#include "mali_kbase_js.h" +#include "mali_kbase_mem.h" +#include "mali_kbase_utility.h" +#include "mali_kbase_gpu_memory_debugfs.h" +#include "mali_kbase_mem_profile_debugfs.h" +#include "mali_kbase_debug_job_fault.h" +#include "mali_kbase_jd_debugfs.h" +#include "mali_kbase_gpuprops.h" +#include "mali_kbase_jm.h" +#include "mali_kbase_vinstr.h" + +#include "ipa/mali_kbase_ipa.h" + +#ifdef CONFIG_GPU_TRACEPOINTS +#include +#endif + +#ifndef u64_to_user_ptr +/* Introduced in Linux v4.6 */ +#define u64_to_user_ptr(x) ((void __user *)(uintptr_t)x) +#endif + +/* + * Kernel-side Base (KBase) APIs + */ + +struct kbase_device *kbase_device_alloc(void); +/* +* note: configuration attributes member of kbdev needs to have +* been setup before calling kbase_device_init +*/ + +/* +* API to acquire device list semaphore and return pointer +* to the device list head +*/ +const struct list_head *kbase_dev_list_get(void); +/* API to release the device list semaphore */ +void kbase_dev_list_put(const struct list_head *dev_list); + +int kbase_device_init(struct kbase_device * const kbdev); +void kbase_device_term(struct kbase_device *kbdev); +void kbase_device_free(struct kbase_device *kbdev); +int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature); + +/* Needed for gator integration and for reporting vsync information */ +struct kbase_device *kbase_find_device(int minor); +void kbase_release_device(struct kbase_device *kbdev); + +void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value); + +struct kbase_context * +kbase_create_context(struct kbase_device *kbdev, bool is_compat); +void kbase_destroy_context(struct kbase_context *kctx); + +int kbase_jd_init(struct kbase_context *kctx); +void kbase_jd_exit(struct kbase_context *kctx); + +/** + * kbase_jd_submit - Submit atoms to the job dispatcher + * + * @kctx: The kbase context to submit to + * @user_addr: The address in user space of the struct base_jd_atom_v2 array + * @nr_atoms: The number of atoms in the array + * @stride: sizeof(struct base_jd_atom_v2) + * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) + * + * Return: 0 on success or error code + */ +int kbase_jd_submit(struct kbase_context *kctx, + void __user *user_addr, u32 nr_atoms, u32 stride, + bool uk6_atom); + +/** + * kbase_jd_done_worker - Handle a job completion + * @data: a &struct work_struct + * + * This function requeues the job from the runpool (if it was soft-stopped or + * removed from NEXT registers). + * + * Removes it from the system if it finished/failed/was cancelled. + * + * Resolves dependencies to add dependent jobs to the context, potentially + * starting them if necessary (which may add more references to the context) + * + * Releases the reference to the context from the no-longer-running job. + * + * Handles retrying submission outside of IRQ context if it failed from within + * IRQ context. + */ +void kbase_jd_done_worker(struct work_struct *data); + +void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp, + kbasep_js_atom_done_code done_code); +void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom); +void kbase_jd_zap_context(struct kbase_context *kctx); +bool jd_done_nolock(struct kbase_jd_atom *katom, + struct list_head *completed_jobs_ctx); +void kbase_jd_free_external_resources(struct kbase_jd_atom *katom); +bool jd_submit_atom(struct kbase_context *kctx, + const struct base_jd_atom_v2 *user_atom, + struct kbase_jd_atom *katom); +void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom); + +void kbase_job_done(struct kbase_device *kbdev, u32 done); + +/** + * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms + * and soft stop them + * @kctx: Pointer to context to check. + * @katom: Pointer to priority atom. + * + * Atoms from @kctx on the same job slot as @katom, which have lower priority + * than @katom will be soft stopped and put back in the queue, so that atoms + * with higher priority can run. + * + * The hwaccess_lock must be held when calling this function. + */ +void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom); +void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom, u32 sw_flags); +void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom); +void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, + base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom); +void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, + struct kbase_jd_atom *target_katom); + +void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event); +int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent); +int kbase_event_pending(struct kbase_context *ctx); +int kbase_event_init(struct kbase_context *kctx); +void kbase_event_close(struct kbase_context *kctx); +void kbase_event_cleanup(struct kbase_context *kctx); +void kbase_event_wakeup(struct kbase_context *kctx); + +int kbase_process_soft_job(struct kbase_jd_atom *katom); +int kbase_prepare_soft_job(struct kbase_jd_atom *katom); +void kbase_finish_soft_job(struct kbase_jd_atom *katom); +void kbase_cancel_soft_job(struct kbase_jd_atom *katom); +void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev); +void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom); +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom); +#endif +int kbase_soft_event_update(struct kbase_context *kctx, + u64 event, + unsigned char new_status); + +bool kbase_replay_process(struct kbase_jd_atom *katom); + +void kbasep_soft_job_timeout_worker(struct timer_list *t); +void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt); + +/* api used internally for register access. Contains validation and tracing */ +void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value); +int kbase_device_trace_buffer_install( + struct kbase_context *kctx, u32 *tb, size_t size); +void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx); + +/* api to be ported per OS, only need to do the raw register access */ +void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value); +u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset); + +void kbasep_as_do_poke(struct work_struct *work); + +/** Returns the name associated with a Mali exception code + * + * This function is called from the interrupt handler when a GPU fault occurs. + * It reports the details of the fault using KBASE_DEBUG_PRINT_WARN. + * + * @param[in] kbdev The kbase device that the GPU fault occurred from. + * @param[in] exception_code exception code + * @return name associated with the exception code + */ +const char *kbase_exception_name(struct kbase_device *kbdev, + u32 exception_code); + +/** + * Check whether a system suspend is in progress, or has already been suspended + * + * The caller should ensure that either kbdev->pm.active_count_lock is held, or + * a dmb was executed recently (to ensure the value is most + * up-to-date). However, without a lock the value could change afterwards. + * + * @return false if a suspend is not in progress + * @return !=false otherwise + */ +static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) +{ + return kbdev->pm.suspending; +} + +/** + * Return the atom's ID, as was originally supplied by userspace in + * base_jd_atom_v2::atom_number + */ +static inline int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + int result; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(katom->kctx == kctx); + + result = katom - &kctx->jctx.atoms[0]; + KBASE_DEBUG_ASSERT(result >= 0 && result <= BASE_JD_ATOM_COUNT); + return result; +} + +/** + * kbase_jd_atom_from_id - Return the atom structure for the given atom ID + * @kctx: Context pointer + * @id: ID of atom to retrieve + * + * Return: Pointer to struct kbase_jd_atom associated with the supplied ID + */ +static inline struct kbase_jd_atom *kbase_jd_atom_from_id( + struct kbase_context *kctx, int id) +{ + return &kctx->jctx.atoms[id]; +} + +/** + * Initialize the disjoint state + * + * The disjoint event count and state are both set to zero. + * + * Disjoint functions usage: + * + * The disjoint event count should be incremented whenever a disjoint event occurs. + * + * There are several cases which are regarded as disjoint behavior. Rather than just increment + * the counter during disjoint events we also increment the counter when jobs may be affected + * by what the GPU is currently doing. To facilitate this we have the concept of disjoint state. + * + * Disjoint state is entered during GPU reset and for the entire time that an atom is replaying + * (as part of the replay workaround). Increasing the disjoint state also increases the count of + * disjoint events. + * + * The disjoint state is then used to increase the count of disjoint events during job submission + * and job completion. Any atom submitted or completed while the disjoint state is greater than + * zero is regarded as a disjoint event. + * + * The disjoint event counter is also incremented immediately whenever a job is soft stopped + * and during context creation. + * + * @param kbdev The kbase device + */ +void kbase_disjoint_init(struct kbase_device *kbdev); + +/** + * Increase the count of disjoint events + * called when a disjoint event has happened + * + * @param kbdev The kbase device + */ +void kbase_disjoint_event(struct kbase_device *kbdev); + +/** + * Increase the count of disjoint events only if the GPU is in a disjoint state + * + * This should be called when something happens which could be disjoint if the GPU + * is in a disjoint state. The state refcount keeps track of this. + * + * @param kbdev The kbase device + */ +void kbase_disjoint_event_potential(struct kbase_device *kbdev); + +/** + * Returns the count of disjoint events + * + * @param kbdev The kbase device + * @return the count of disjoint events + */ +u32 kbase_disjoint_event_get(struct kbase_device *kbdev); + +/** + * Increment the refcount state indicating that the GPU is in a disjoint state. + * + * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) + * eventually after the disjoint state has completed @ref kbase_disjoint_state_down + * should be called + * + * @param kbdev The kbase device + */ +void kbase_disjoint_state_up(struct kbase_device *kbdev); + +/** + * Decrement the refcount state + * + * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) + * + * Called after @ref kbase_disjoint_state_up once the disjoint state is over + * + * @param kbdev The kbase device + */ +void kbase_disjoint_state_down(struct kbase_device *kbdev); + +/** + * If a job is soft stopped and the number of contexts is >= this value + * it is reported as a disjoint event + */ +#define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2 + +#if !defined(UINT64_MAX) + #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL) +#endif + +#if KBASE_TRACE_ENABLE +void kbasep_trace_debugfs_init(struct kbase_device *kbdev); + +#ifndef CONFIG_MALI_BIFROST_SYSTEM_TRACE +/** Add trace values about a job-slot + * + * @note Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any + * functions called to get the parameters supplied to this macro must: + * - be static or static inline + * - must just return 0 and have no other statements present in the body. + */ +#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot) \ + kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ + KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, 0) + +/** Add trace values about a job-slot, with info + * + * @note Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any + * functions called to get the parameters supplied to this macro must: + * - be static or static inline + * - must just return 0 and have no other statements present in the body. + */ +#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val) \ + kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ + KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, info_val) + +/** Add trace values about a ctx refcount + * + * @note Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any + * functions called to get the parameters supplied to this macro must: + * - be static or static inline + * - must just return 0 and have no other statements present in the body. + */ +#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount) \ + kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ + KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, 0) +/** Add trace values about a ctx refcount, and info + * + * @note Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any + * functions called to get the parameters supplied to this macro must: + * - be static or static inline + * - must just return 0 and have no other statements present in the body. + */ +#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val) \ + kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ + KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, info_val) + +/** Add trace values (no slot or refcount) + * + * @note Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any + * functions called to get the parameters supplied to this macro must: + * - be static or static inline + * - must just return 0 and have no other statements present in the body. + */ +#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val) \ + kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ + 0, 0, 0, info_val) + +/** Clear the trace */ +#define KBASE_TRACE_CLEAR(kbdev) \ + kbasep_trace_clear(kbdev) + +/** Dump the slot trace */ +#define KBASE_TRACE_DUMP(kbdev) \ + kbasep_trace_dump(kbdev) + +/** PRIVATE - do not use directly. Use KBASE_TRACE_ADD() instead */ +void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val); +/** PRIVATE - do not use directly. Use KBASE_TRACE_CLEAR() instead */ +void kbasep_trace_clear(struct kbase_device *kbdev); +#else /* #ifndef CONFIG_MALI_BIFROST_SYSTEM_TRACE */ +/* Dispatch kbase trace events as system trace events */ +#include +#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ + trace_mali_##code(jobslot, 0) + +#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ + trace_mali_##code(jobslot, info_val) + +#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ + trace_mali_##code(refcount, 0) + +#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ + trace_mali_##code(refcount, info_val) + +#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val)\ + trace_mali_##code(gpu_addr, info_val) + +#define KBASE_TRACE_CLEAR(kbdev)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(0);\ + } while (0) +#define KBASE_TRACE_DUMP(kbdev)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(0);\ + } while (0) + +#endif /* #ifndef CONFIG_MALI_BIFROST_SYSTEM_TRACE */ +#else +#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(ctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(jobslot);\ + } while (0) + +#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(ctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(jobslot);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(ctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(refcount);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(ctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_TRACE_ADD(kbdev, code, subcode, ctx, katom, val)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(subcode);\ + CSTD_UNUSED(ctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_TRACE_CLEAR(kbdev)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(0);\ + } while (0) +#define KBASE_TRACE_DUMP(kbdev)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(0);\ + } while (0) +#endif /* KBASE_TRACE_ENABLE */ +/** PRIVATE - do not use directly. Use KBASE_TRACE_DUMP() instead */ +void kbasep_trace_dump(struct kbase_device *kbdev); + +#ifdef CONFIG_MALI_BIFROST_DEBUG +/** + * kbase_set_driver_inactive - Force driver to go inactive + * @kbdev: Device pointer + * @inactive: true if driver should go inactive, false otherwise + * + * Forcing the driver inactive will cause all future IOCTLs to wait until the + * driver is made active again. This is intended solely for the use of tests + * which require that no jobs are running while the test executes. + */ +void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) + +/* kbase_io_history_init - initialize data struct for register access history + * + * @kbdev The register history to initialize + * @n The number of register accesses that the buffer could hold + * + * @return 0 if successfully initialized, failure otherwise + */ +int kbase_io_history_init(struct kbase_io_history *h, u16 n); + +/* kbase_io_history_term - uninit all resources for the register access history + * + * @h The register history to terminate + */ +void kbase_io_history_term(struct kbase_io_history *h); + +/* kbase_io_history_dump - print the register history to the kernel ring buffer + * + * @kbdev Pointer to kbase_device containing the register history to dump + */ +void kbase_io_history_dump(struct kbase_device *kbdev); + +/** + * kbase_io_history_resize - resize the register access history buffer. + * + * @h: Pointer to a valid register history to resize + * @new_size: Number of accesses the buffer could hold + * + * A successful resize will clear all recent register accesses. + * If resizing fails for any reason (e.g., could not allocate memory, invalid + * buffer size) then the original buffer will be kept intact. + * + * @return 0 if the buffer was resized, failure otherwise + */ +int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size); + +#else /* CONFIG_DEBUG_FS */ + +#define kbase_io_history_init(...) ((int)0) + +#define kbase_io_history_term CSTD_NOP + +#define kbase_io_history_dump CSTD_NOP + +#define kbase_io_history_resize CSTD_NOP + +#endif /* CONFIG_DEBUG_FS */ + + +#endif + + + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.c new file mode 100755 index 000000000000..6b3559d93351 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.c @@ -0,0 +1,210 @@ +/* + * + * (C) COPYRIGHT 2013-2015,2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +#include +#include +#include + +/* This function is used to solve an HW issue with single iterator GPUs. + * If a fragment job is soft-stopped on the edge of its bounding box, can happen that the + * restart index is out of bounds and the rerun causes a tile range fault. If this happens + * we try to clamp the restart index to a correct value and rerun the job. + */ +/* Mask of X and Y coordinates for the coordinates words in the descriptors*/ +#define X_COORDINATE_MASK 0x00000FFF +#define Y_COORDINATE_MASK 0x0FFF0000 +/* Max number of words needed from the fragment shader job descriptor */ +#define JOB_HEADER_SIZE_IN_WORDS 10 +#define JOB_HEADER_SIZE (JOB_HEADER_SIZE_IN_WORDS*sizeof(u32)) + +/* Word 0: Status Word */ +#define JOB_DESC_STATUS_WORD 0 +/* Word 1: Restart Index */ +#define JOB_DESC_RESTART_INDEX_WORD 1 +/* Word 2: Fault address low word */ +#define JOB_DESC_FAULT_ADDR_LOW_WORD 2 +/* Word 8: Minimum Tile Coordinates */ +#define FRAG_JOB_DESC_MIN_TILE_COORD_WORD 8 +/* Word 9: Maximum Tile Coordinates */ +#define FRAG_JOB_DESC_MAX_TILE_COORD_WORD 9 + +int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom) +{ + struct device *dev = katom->kctx->kbdev->dev; + u32 clamped = 0; + struct kbase_va_region *region; + struct tagged_addr *page_array; + u64 page_index; + u32 offset = katom->jc & (~PAGE_MASK); + u32 *page_1 = NULL; + u32 *page_2 = NULL; + u32 job_header[JOB_HEADER_SIZE_IN_WORDS]; + void *dst = job_header; + u32 minX, minY, maxX, maxY; + u32 restartX, restartY; + struct page *p; + u32 copy_size; + + dev_warn(dev, "Called TILE_RANGE_FAULT workaround clamping function.\n"); + if (!(katom->core_req & BASE_JD_REQ_FS)) + return 0; + + kbase_gpu_vm_lock(katom->kctx); + region = kbase_region_tracker_find_region_enclosing_address(katom->kctx, + katom->jc); + if (!region || (region->flags & KBASE_REG_FREE)) + goto out_unlock; + + page_array = kbase_get_cpu_phy_pages(region); + if (!page_array) + goto out_unlock; + + page_index = (katom->jc >> PAGE_SHIFT) - region->start_pfn; + + p = phys_to_page(as_phys_addr_t(page_array[page_index])); + + /* we need the first 10 words of the fragment shader job descriptor. + * We need to check that the offset + 10 words is less that the page + * size otherwise we need to load the next page. + * page_size_overflow will be equal to 0 in case the whole descriptor + * is within the page > 0 otherwise. + */ + copy_size = MIN(PAGE_SIZE - offset, JOB_HEADER_SIZE); + + page_1 = kmap_atomic(p); + + /* page_1 is a u32 pointer, offset is expressed in bytes */ + page_1 += offset>>2; + + kbase_sync_single_for_cpu(katom->kctx->kbdev, + kbase_dma_addr(p) + offset, + copy_size, DMA_BIDIRECTIONAL); + + memcpy(dst, page_1, copy_size); + + /* The data needed overflows page the dimension, + * need to map the subsequent page */ + if (copy_size < JOB_HEADER_SIZE) { + p = phys_to_page(as_phys_addr_t(page_array[page_index + 1])); + page_2 = kmap_atomic(p); + + kbase_sync_single_for_cpu(katom->kctx->kbdev, + kbase_dma_addr(p), + JOB_HEADER_SIZE - copy_size, DMA_BIDIRECTIONAL); + + memcpy(dst + copy_size, page_2, JOB_HEADER_SIZE - copy_size); + } + + /* We managed to correctly map one or two pages (in case of overflow) */ + /* Get Bounding Box data and restart index from fault address low word */ + minX = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & X_COORDINATE_MASK; + minY = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & Y_COORDINATE_MASK; + maxX = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & X_COORDINATE_MASK; + maxY = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & Y_COORDINATE_MASK; + restartX = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & X_COORDINATE_MASK; + restartY = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & Y_COORDINATE_MASK; + + dev_warn(dev, "Before Clamping:\n" + "Jobstatus: %08x\n" + "restartIdx: %08x\n" + "Fault_addr_low: %08x\n" + "minCoordsX: %08x minCoordsY: %08x\n" + "maxCoordsX: %08x maxCoordsY: %08x\n", + job_header[JOB_DESC_STATUS_WORD], + job_header[JOB_DESC_RESTART_INDEX_WORD], + job_header[JOB_DESC_FAULT_ADDR_LOW_WORD], + minX, minY, + maxX, maxY); + + /* Set the restart index to the one which generated the fault*/ + job_header[JOB_DESC_RESTART_INDEX_WORD] = + job_header[JOB_DESC_FAULT_ADDR_LOW_WORD]; + + if (restartX < minX) { + job_header[JOB_DESC_RESTART_INDEX_WORD] = (minX) | restartY; + dev_warn(dev, + "Clamping restart X index to minimum. %08x clamped to %08x\n", + restartX, minX); + clamped = 1; + } + if (restartY < minY) { + job_header[JOB_DESC_RESTART_INDEX_WORD] = (minY) | restartX; + dev_warn(dev, + "Clamping restart Y index to minimum. %08x clamped to %08x\n", + restartY, minY); + clamped = 1; + } + if (restartX > maxX) { + job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxX) | restartY; + dev_warn(dev, + "Clamping restart X index to maximum. %08x clamped to %08x\n", + restartX, maxX); + clamped = 1; + } + if (restartY > maxY) { + job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxY) | restartX; + dev_warn(dev, + "Clamping restart Y index to maximum. %08x clamped to %08x\n", + restartY, maxY); + clamped = 1; + } + + if (clamped) { + /* Reset the fault address low word + * and set the job status to STOPPED */ + job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] = 0x0; + job_header[JOB_DESC_STATUS_WORD] = BASE_JD_EVENT_STOPPED; + dev_warn(dev, "After Clamping:\n" + "Jobstatus: %08x\n" + "restartIdx: %08x\n" + "Fault_addr_low: %08x\n" + "minCoordsX: %08x minCoordsY: %08x\n" + "maxCoordsX: %08x maxCoordsY: %08x\n", + job_header[JOB_DESC_STATUS_WORD], + job_header[JOB_DESC_RESTART_INDEX_WORD], + job_header[JOB_DESC_FAULT_ADDR_LOW_WORD], + minX, minY, + maxX, maxY); + + /* Flush CPU cache to update memory for future GPU reads*/ + memcpy(page_1, dst, copy_size); + p = phys_to_page(as_phys_addr_t(page_array[page_index])); + + kbase_sync_single_for_device(katom->kctx->kbdev, + kbase_dma_addr(p) + offset, + copy_size, DMA_TO_DEVICE); + + if (copy_size < JOB_HEADER_SIZE) { + memcpy(page_2, dst + copy_size, + JOB_HEADER_SIZE - copy_size); + p = phys_to_page(as_phys_addr_t(page_array[page_index + + 1])); + + kbase_sync_single_for_device(katom->kctx->kbdev, + kbase_dma_addr(p), + JOB_HEADER_SIZE - copy_size, + DMA_TO_DEVICE); + } + } + if (copy_size < JOB_HEADER_SIZE) + kunmap_atomic(page_2); + + kunmap_atomic(page_1); + +out_unlock: + kbase_gpu_vm_unlock(katom->kctx); + return clamped; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.h new file mode 100755 index 000000000000..099a29861672 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_10969_workaround.h @@ -0,0 +1,23 @@ +/* + * + * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_10969_WORKAROUND_ +#define _KBASE_10969_WORKAROUND_ + +int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom); + +#endif /* _KBASE_10969_WORKAROUND_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.c new file mode 100755 index 000000000000..cc729d416858 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.c @@ -0,0 +1,102 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include + +#include +#include + +#ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_MALI_BIFROST_DEBUG + +static int kbase_as_fault_read(struct seq_file *sfile, void *data) +{ + uintptr_t as_no = (uintptr_t) sfile->private; + + struct list_head *entry; + const struct list_head *kbdev_list; + struct kbase_device *kbdev = NULL; + + kbdev_list = kbase_dev_list_get(); + + list_for_each(entry, kbdev_list) { + kbdev = list_entry(entry, struct kbase_device, entry); + + if(kbdev->debugfs_as_read_bitmap & (1ULL << as_no)) { + + /* don't show this one again until another fault occors */ + kbdev->debugfs_as_read_bitmap &= ~(1ULL << as_no); + + /* output the last page fault addr */ + seq_printf(sfile, "%llu\n", (u64) kbdev->as[as_no].fault_addr); + } + + } + + kbase_dev_list_put(kbdev_list); + + return 0; +} + +static int kbase_as_fault_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbase_as_fault_read , in->i_private); +} + +static const struct file_operations as_fault_fops = { + .open = kbase_as_fault_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +#endif /* CONFIG_MALI_BIFROST_DEBUG */ +#endif /* CONFIG_DEBUG_FS */ + +/* + * Initialize debugfs entry for each address space + */ +void kbase_as_fault_debugfs_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_MALI_BIFROST_DEBUG + uint i; + char as_name[64]; + struct dentry *debugfs_directory; + + kbdev->debugfs_as_read_bitmap = 0ULL; + + KBASE_DEBUG_ASSERT(kbdev->nr_hw_address_spaces); + KBASE_DEBUG_ASSERT(sizeof(kbdev->as[0].fault_addr) == sizeof(u64)); + + debugfs_directory = debugfs_create_dir("address_spaces", + kbdev->mali_debugfs_directory); + + if(debugfs_directory) { + for(i = 0; i < kbdev->nr_hw_address_spaces; i++) { + snprintf(as_name, ARRAY_SIZE(as_name), "as%u", i); + debugfs_create_file(as_name, S_IRUGO, + debugfs_directory, (void*) ((uintptr_t) i), &as_fault_fops); + } + } + else + dev_warn(kbdev->dev, "unable to create address_spaces debugfs directory"); + +#endif /* CONFIG_MALI_BIFROST_DEBUG */ +#endif /* CONFIG_DEBUG_FS */ + return; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.h new file mode 100755 index 000000000000..66387e1c3f6a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_as_fault_debugfs.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_AS_FAULT_DEBUG_FS_H +#define _KBASE_AS_FAULT_DEBUG_FS_H + +/** + * kbase_as_fault_debugfs_init() - Add debugfs files for reporting page faults + * + * @kbdev: Pointer to kbase_device + */ +void kbase_as_fault_debugfs_init(struct kbase_device *kbdev); + +/** + * kbase_as_fault_debugfs_new() - make the last fault available on debugfs + * + * @kbdev: Pointer to kbase_device + * @as_no: The address space the fault occurred on + */ +static inline void +kbase_as_fault_debugfs_new(struct kbase_device *kbdev, int as_no) +{ +#ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_MALI_BIFROST_DEBUG + kbdev->debugfs_as_read_bitmap |= (1ULL << as_no); +#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + return; +} + +#endif /*_KBASE_AS_FAULT_DEBUG_FS_H*/ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.c new file mode 100755 index 000000000000..1d11de67aa80 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.c @@ -0,0 +1,54 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Cache Policy API. + */ + +#include "mali_kbase_cache_policy.h" + +/* + * The output flags should be a combination of the following values: + * KBASE_REG_CPU_CACHED: CPU cache should be enabled. + */ +u32 kbase_cache_enabled(u32 flags, u32 nr_pages) +{ + u32 cache_flags = 0; + + CSTD_UNUSED(nr_pages); + + if (flags & BASE_MEM_CACHED_CPU) + cache_flags |= KBASE_REG_CPU_CACHED; + + return cache_flags; +} + + +void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir) +{ + dma_sync_single_for_device(kbdev->dev, handle, size, dir); +} + + +void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir) +{ + dma_sync_single_for_cpu(kbdev->dev, handle, size, dir); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.h new file mode 100755 index 000000000000..0c18bdb357b0 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_cache_policy.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Cache Policy API. + */ + +#ifndef _KBASE_CACHE_POLICY_H_ +#define _KBASE_CACHE_POLICY_H_ + +#include "mali_kbase.h" +#include "mali_base_kernel.h" + +/** + * kbase_cache_enabled - Choose the cache policy for a specific region + * @flags: flags describing attributes of the region + * @nr_pages: total number of pages (backed or not) for the region + * + * Tells whether the CPU and GPU caches should be enabled or not for a specific + * region. + * This function can be modified to customize the cache policy depending on the + * flags and size of the region. + * + * Return: a combination of %KBASE_REG_CPU_CACHED and %KBASE_REG_GPU_CACHED + * depending on the cache policy + */ +u32 kbase_cache_enabled(u32 flags, u32 nr_pages); + +#endif /* _KBASE_CACHE_POLICY_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.c new file mode 100755 index 000000000000..fb615ae02ead --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.c @@ -0,0 +1,51 @@ +/* + * + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include + +int kbasep_platform_device_init(struct kbase_device *kbdev) +{ + struct kbase_platform_funcs_conf *platform_funcs_p; + + platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; + if (platform_funcs_p && platform_funcs_p->platform_init_func) + return platform_funcs_p->platform_init_func(kbdev); + + return 0; +} + +void kbasep_platform_device_term(struct kbase_device *kbdev) +{ + struct kbase_platform_funcs_conf *platform_funcs_p; + + platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; + if (platform_funcs_p && platform_funcs_p->platform_term_func) + platform_funcs_p->platform_term_func(kbdev); +} + +int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed) +{ + KBASE_DEBUG_ASSERT(NULL != clock_speed); + + *clock_speed = 100; + return 0; +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.h new file mode 100755 index 000000000000..212e3b14d96c --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config.h @@ -0,0 +1,343 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_config.h + * Configuration API and Attributes for KBase + */ + +#ifndef _KBASE_CONFIG_H_ +#define _KBASE_CONFIG_H_ + +#include + +#include +#include + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_config Configuration API and Attributes + * @{ + */ + +#include + +/* Forward declaration of struct kbase_device */ +struct kbase_device; + +/** + * kbase_platform_funcs_conf - Specifies platform init/term function pointers + * + * Specifies the functions pointers for platform specific initialization and + * termination. By default no functions are required. No additional platform + * specific control is necessary. + */ +struct kbase_platform_funcs_conf { + /** + * platform_init_func - platform specific init function pointer + * @kbdev - kbase_device pointer + * + * Returns 0 on success, negative error code otherwise. + * + * Function pointer for platform specific initialization or NULL if no + * initialization function is required. At the point this the GPU is + * not active and its power and clocks are in unknown (platform specific + * state) as kbase doesn't yet have control of power and clocks. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed (and possibly initialized) in here. + */ + int (*platform_init_func)(struct kbase_device *kbdev); + /** + * platform_term_func - platform specific termination function pointer + * @kbdev - kbase_device pointer + * + * Function pointer for platform specific termination or NULL if no + * termination function is required. At the point this the GPU will be + * idle but still powered and clocked. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed (and possibly terminated) in here. + */ + void (*platform_term_func)(struct kbase_device *kbdev); +}; + +/* + * @brief Specifies the callbacks for power management + * + * By default no callbacks will be made and the GPU must not be powered off. + */ +struct kbase_pm_callback_conf { + /** Callback for when the GPU is idle and the power to it can be switched off. + * + * The system integrator can decide whether to either do nothing, just switch off + * the clocks to the GPU, or to completely power down the GPU. + * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the + * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). + */ + void (*power_off_callback)(struct kbase_device *kbdev); + + /** Callback for when the GPU is about to become active and power must be supplied. + * + * This function must not return until the GPU is powered and clocked sufficiently for register access to + * succeed. The return value specifies whether the GPU was powered down since the call to power_off_callback. + * If the GPU state has been lost then this function must return 1, otherwise it should return 0. + * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the + * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). + * + * The return value of the first call to this function is ignored. + * + * @return 1 if the GPU state may have been lost, 0 otherwise. + */ + int (*power_on_callback)(struct kbase_device *kbdev); + + /** Callback for when the system is requesting a suspend and GPU power + * must be switched off. + * + * Note that if this callback is present, then this may be called + * without a preceding call to power_off_callback. Therefore this + * callback must be able to take any action that might otherwise happen + * in power_off_callback. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed and modified in here. It is the platform \em + * callbacks responsibility to initialize and terminate this pointer if + * used (see @ref kbase_platform_funcs_conf). + */ + void (*power_suspend_callback)(struct kbase_device *kbdev); + + /** Callback for when the system is resuming from a suspend and GPU + * power must be switched on. + * + * Note that if this callback is present, then this may be called + * without a following call to power_on_callback. Therefore this + * callback must be able to take any action that might otherwise happen + * in power_on_callback. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed and modified in here. It is the platform \em + * callbacks responsibility to initialize and terminate this pointer if + * used (see @ref kbase_platform_funcs_conf). + */ + void (*power_resume_callback)(struct kbase_device *kbdev); + + /** Callback for handling runtime power management initialization. + * + * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback + * will become active from calls made to the OS from within this function. + * The runtime calls can be triggered by calls from @ref power_off_callback and @ref power_on_callback. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * + * @return 0 on success, else int error code. + */ + int (*power_runtime_init_callback)(struct kbase_device *kbdev); + + /** Callback for handling runtime power management termination. + * + * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback + * should no longer be called by the OS on completion of this function. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + */ + void (*power_runtime_term_callback)(struct kbase_device *kbdev); + + /** Callback for runtime power-off power management callback + * + * For linux this callback will be called by the kernel runtime_suspend callback. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * + * @return 0 on success, else OS error code. + */ + void (*power_runtime_off_callback)(struct kbase_device *kbdev); + + /** Callback for runtime power-on power management callback + * + * For linux this callback will be called by the kernel runtime_resume callback. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + */ + int (*power_runtime_on_callback)(struct kbase_device *kbdev); + + /* + * Optional callback for checking if GPU can be suspended when idle + * + * This callback will be called by the runtime power management core + * when the reference count goes to 0 to provide notification that the + * GPU now seems idle. + * + * If this callback finds that the GPU can't be powered off, or handles + * suspend by powering off directly or queueing up a power off, a + * non-zero value must be returned to prevent the runtime PM core from + * also triggering a suspend. + * + * Returning 0 will cause the runtime PM core to conduct a regular + * autosuspend. + * + * This callback is optional and if not provided regular autosuspend + * will be triggered. + * + * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use + * this feature. + * + * Return 0 if GPU can be suspended, positive value if it can not be + * suspeneded by runtime PM, else OS error code + */ + int (*power_runtime_idle_callback)(struct kbase_device *kbdev); +}; + +/** + * kbase_cpuprops_get_default_clock_speed - default for CPU_SPEED_FUNC + * @clock_speed - see kbase_cpu_clk_speed_func for details on the parameters + * + * Returns 0 on success, negative error code otherwise. + * + * Default implementation of CPU_SPEED_FUNC. This function sets clock_speed + * to 100, so will be an underestimate for any real system. + */ +int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed); + +/** + * kbase_cpu_clk_speed_func - Type of the function pointer for CPU_SPEED_FUNC + * @param clock_speed - pointer to store the current CPU clock speed in MHz + * + * Returns 0 on success, otherwise negative error code. + * + * This is mainly used to implement OpenCL's clGetDeviceInfo(). + */ +typedef int (*kbase_cpu_clk_speed_func) (u32 *clock_speed); + +/** + * kbase_gpu_clk_speed_func - Type of the function pointer for GPU_SPEED_FUNC + * @param clock_speed - pointer to store the current GPU clock speed in MHz + * + * Returns 0 on success, otherwise negative error code. + * When an error is returned the caller assumes maximum GPU speed stored in + * gpu_freq_khz_max. + * + * If the system timer is not available then this function is required + * for the OpenCL queue profiling to return correct timing information. + * + */ +typedef int (*kbase_gpu_clk_speed_func) (u32 *clock_speed); + +#ifdef CONFIG_OF +struct kbase_platform_config { +}; +#else + +/* + * @brief Specifies start and end of I/O memory region. + */ +struct kbase_io_memory_region { + u64 start; + u64 end; +}; + +/* + * @brief Specifies I/O related resources like IRQs and memory region for I/O operations. + */ +struct kbase_io_resources { + u32 job_irq_number; + u32 mmu_irq_number; + u32 gpu_irq_number; + struct kbase_io_memory_region io_memory_region; +}; + +struct kbase_platform_config { + const struct kbase_io_resources *io_resources; +}; + +#endif /* CONFIG_OF */ + +/** + * @brief Gets the pointer to platform config. + * + * @return Pointer to the platform config + */ +struct kbase_platform_config *kbase_get_platform_config(void); + +/** + * kbasep_platform_device_init: - Platform specific call to initialize hardware + * @kbdev: kbase device pointer + * + * Function calls a platform defined routine if specified in the configuration + * attributes. The routine can initialize any hardware and context state that + * is required for the GPU block to function. + * + * Return: 0 if no errors have been found in the config. + * Negative error code otherwise. + */ +int kbasep_platform_device_init(struct kbase_device *kbdev); + +/** + * kbasep_platform_device_term - Platform specific call to terminate hardware + * @kbdev: Kbase device pointer + * + * Function calls a platform defined routine if specified in the configuration + * attributes. The routine can destroy any platform specific context state and + * shut down any hardware functionality that are outside of the Power Management + * callbacks. + * + */ +void kbasep_platform_device_term(struct kbase_device *kbdev); + + +/** + * kbase_platform_early_init - Early initialisation of the platform code + * + * This function will be called when the module is loaded to perform any + * early initialisation required by the platform code. Such as reading + * platform specific device tree entries for the GPU. + * + * Return: 0 for success, any other fail causes module initialisation to fail + */ +int kbase_platform_early_init(void); + +#ifndef CONFIG_OF +/** + * kbase_platform_register - Register a platform device for the GPU + * + * This can be used to register a platform device on systems where device tree + * is not enabled and the platform initialisation code in the kernel doesn't + * create the GPU device. Where possible device tree should be used instead. + * + * Return: 0 for success, any other fail causes module initialisation to fail + */ +int kbase_platform_register(void); + +/** + * kbase_platform_unregister - Unregister a fake platform device + * + * Unregister the platform device created with kbase_platform_register() + */ +void kbase_platform_unregister(void); +#endif + + /** @} *//* end group kbase_config */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_CONFIG_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config_defaults.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config_defaults.h new file mode 100755 index 000000000000..69079e7d9680 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_config_defaults.h @@ -0,0 +1,226 @@ +/* + * + * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_config_defaults.h + * + * Default values for configuration settings + * + */ + +#ifndef _KBASE_CONFIG_DEFAULTS_H_ +#define _KBASE_CONFIG_DEFAULTS_H_ + +/* Include mandatory definitions per platform */ +#include + +/** +* Boolean indicating whether the driver is configured to be secure at +* a potential loss of performance. +* +* This currently affects only r0p0-15dev0 HW and earlier. +* +* On r0p0-15dev0 HW and earlier, there are tradeoffs between security and +* performance: +* +* - When this is set to true, the driver remains fully secure, +* but potentially loses performance compared with setting this to +* false. +* - When set to false, the driver is open to certain security +* attacks. +* +* From r0p0-00rel0 and onwards, there is no security loss by setting +* this to false, and no performance loss by setting it to +* true. +*/ +#define DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE false + +enum { + /** + * Use unrestricted Address ID width on the AXI bus. + */ + KBASE_AID_32 = 0x0, + + /** + * Restrict GPU to a half of maximum Address ID count. + * This will reduce performance, but reduce bus load due to GPU. + */ + KBASE_AID_16 = 0x3, + + /** + * Restrict GPU to a quarter of maximum Address ID count. + * This will reduce performance, but reduce bus load due to GPU. + */ + KBASE_AID_8 = 0x2, + + /** + * Restrict GPU to an eighth of maximum Address ID count. + * This will reduce performance, but reduce bus load due to GPU. + */ + KBASE_AID_4 = 0x1 +}; + +/** + * Default setting for read Address ID limiting on AXI bus. + * + * Attached value: u32 register value + * KBASE_AID_32 - use the full 32 IDs (5 ID bits) + * KBASE_AID_16 - use 16 IDs (4 ID bits) + * KBASE_AID_8 - use 8 IDs (3 ID bits) + * KBASE_AID_4 - use 4 IDs (2 ID bits) + * Default value: KBASE_AID_32 (no limit). Note hardware implementation + * may limit to a lower value. + */ +#define DEFAULT_ARID_LIMIT KBASE_AID_32 + +/** + * Default setting for write Address ID limiting on AXI. + * + * Attached value: u32 register value + * KBASE_AID_32 - use the full 32 IDs (5 ID bits) + * KBASE_AID_16 - use 16 IDs (4 ID bits) + * KBASE_AID_8 - use 8 IDs (3 ID bits) + * KBASE_AID_4 - use 4 IDs (2 ID bits) + * Default value: KBASE_AID_32 (no limit). Note hardware implementation + * may limit to a lower value. + */ +#define DEFAULT_AWID_LIMIT KBASE_AID_32 + +/** + * Default UMP device mapping. A UMP_DEVICE__SHIFT value which + * defines which UMP device this GPU should be mapped to. + */ +#define DEFAULT_UMP_GPU_DEVICE_SHIFT UMP_DEVICE_Z_SHIFT + +/* + * Default period for DVFS sampling + */ +#define DEFAULT_PM_DVFS_PERIOD 100 /* 100ms */ + +/* + * Power Management poweroff tick granuality. This is in nanoseconds to + * allow HR timer support. + * + * On each scheduling tick, the power manager core may decide to: + * -# Power off one or more shader cores + * -# Power off the entire GPU + */ +#define DEFAULT_PM_GPU_POWEROFF_TICK_NS (400000) /* 400us */ + +/* + * Power Manager number of ticks before shader cores are powered off + */ +#define DEFAULT_PM_POWEROFF_TICK_SHADER (2) /* 400-800us */ + +/* + * Power Manager number of ticks before GPU is powered off + */ +#define DEFAULT_PM_POWEROFF_TICK_GPU (2) /* 400-800us */ + +/* + * Default scheduling tick granuality + */ +#define DEFAULT_JS_SCHEDULING_PERIOD_NS (100000000u) /* 100ms */ + +/* + * Default minimum number of scheduling ticks before jobs are soft-stopped. + * + * This defines the time-slice for a job (which may be different from that of a + * context) + */ +#define DEFAULT_JS_SOFT_STOP_TICKS (1) /* 100ms-200ms */ + +/* + * Default minimum number of scheduling ticks before CL jobs are soft-stopped. + */ +#define DEFAULT_JS_SOFT_STOP_TICKS_CL (1) /* 100ms-200ms */ + +/* + * Default minimum number of scheduling ticks before jobs are hard-stopped + */ +#define DEFAULT_JS_HARD_STOP_TICKS_SS (50) /* 5s */ +#define DEFAULT_JS_HARD_STOP_TICKS_SS_8408 (300) /* 30s */ + +/* + * Default minimum number of scheduling ticks before CL jobs are hard-stopped. + */ +#define DEFAULT_JS_HARD_STOP_TICKS_CL (50) /* 5s */ + +/* + * Default minimum number of scheduling ticks before jobs are hard-stopped + * during dumping + */ +#define DEFAULT_JS_HARD_STOP_TICKS_DUMPING (15000) /* 1500s */ + +/* + * Default timeout for some software jobs, after which the software event wait + * jobs will be cancelled. + */ +#define DEFAULT_JS_SOFT_JOB_TIMEOUT (3000) /* 3s */ + +/* + * Default minimum number of scheduling ticks before the GPU is reset to clear a + * "stuck" job + */ +#define DEFAULT_JS_RESET_TICKS_SS (55) /* 5.5s */ +#define DEFAULT_JS_RESET_TICKS_SS_8408 (450) /* 45s */ + +/* + * Default minimum number of scheduling ticks before the GPU is reset to clear a + * "stuck" CL job. + */ +#define DEFAULT_JS_RESET_TICKS_CL (55) /* 5.5s */ + +/* + * Default minimum number of scheduling ticks before the GPU is reset to clear a + * "stuck" job during dumping. + */ +#define DEFAULT_JS_RESET_TICKS_DUMPING (15020) /* 1502s */ + +/* + * Default number of milliseconds given for other jobs on the GPU to be + * soft-stopped when the GPU needs to be reset. + */ +#define DEFAULT_RESET_TIMEOUT_MS (3000) /* 3s */ + +/* + * Default timeslice that a context is scheduled in for, in nanoseconds. + * + * When a context has used up this amount of time across its jobs, it is + * scheduled out to let another run. + * + * @note the resolution is nanoseconds (ns) here, because that's the format + * often used by the OS. + */ +#define DEFAULT_JS_CTX_TIMESLICE_NS (50000000) /* 50ms */ + +/* + * Perform GPU power down using only platform specific code, skipping DDK power + * management. + * + * If this is non-zero then kbase will avoid powering down shader cores, the + * tiler, and the L2 cache, instead just powering down the entire GPU through + * platform specific code. This may be required for certain platform + * integrations. + * + * Note that as this prevents kbase from powering down shader cores, this limits + * the available power policies to coarse_demand and always_on. + */ +#define PLATFORM_POWER_DOWN_ONLY (0) + +#endif /* _KBASE_CONFIG_DEFAULTS_H_ */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.c new file mode 100755 index 000000000000..7a09aa26128e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.c @@ -0,0 +1,362 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel context APIs + */ + +#include +#include +#include +#include +#include + +/** + * kbase_create_context() - Create a kernel base context. + * @kbdev: Kbase device + * @is_compat: Force creation of a 32-bit context + * + * Allocate and init a kernel base context. + * + * Return: new kbase context + */ +struct kbase_context * +kbase_create_context(struct kbase_device *kbdev, bool is_compat) +{ + struct kbase_context *kctx; + int err; + struct page *p; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + /* zero-inited as lot of code assume it's zero'ed out on create */ + kctx = vzalloc(sizeof(*kctx)); + + if (!kctx) + goto out; + + /* creating a context is considered a disjoint event */ + kbase_disjoint_event(kbdev); + + kctx->kbdev = kbdev; + kctx->as_nr = KBASEP_AS_NR_INVALID; + atomic_set(&kctx->refcount, 0); + if (is_compat) + kbase_ctx_flag_set(kctx, KCTX_COMPAT); +#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE + kctx->timeline.owner_tgid = task_tgid_nr(current); +#endif + atomic_set(&kctx->setup_complete, 0); + atomic_set(&kctx->setup_in_progress, 0); + spin_lock_init(&kctx->mm_update_lock); + kctx->process_mm = NULL; + atomic_set(&kctx->nonmapped_pages, 0); + kctx->slots_pullable = 0; + kctx->tgid = current->tgid; + kctx->pid = current->pid; + + err = kbase_mem_pool_init(&kctx->mem_pool, + kbdev->mem_pool_max_size_default, + KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER, + kctx->kbdev, + &kbdev->mem_pool); + if (err) + goto free_kctx; + + err = kbase_mem_pool_init(&kctx->lp_mem_pool, + (kbdev->mem_pool_max_size_default >> 9), + KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, + kctx->kbdev, + &kbdev->lp_mem_pool); + if (err) + goto free_mem_pool; + + err = kbase_mem_evictable_init(kctx); + if (err) + goto free_both_pools; + + atomic_set(&kctx->used_pages, 0); + + err = kbase_jd_init(kctx); + if (err) + goto deinit_evictable; + + err = kbasep_js_kctx_init(kctx); + if (err) + goto free_jd; /* safe to call kbasep_js_kctx_term in this case */ + + err = kbase_event_init(kctx); + if (err) + goto free_jd; + + atomic_set(&kctx->drain_pending, 0); + + mutex_init(&kctx->reg_lock); + + mutex_init(&kctx->mem_partials_lock); + INIT_LIST_HEAD(&kctx->mem_partials); + + INIT_LIST_HEAD(&kctx->waiting_soft_jobs); + spin_lock_init(&kctx->waiting_soft_jobs_lock); +#ifdef CONFIG_KDS + INIT_LIST_HEAD(&kctx->waiting_kds_resource); +#endif + err = kbase_dma_fence_init(kctx); + if (err) + goto free_event; + + err = kbase_mmu_init(kctx); + if (err) + goto term_dma_fence; + + do { + err = kbase_mem_pool_grow(&kctx->mem_pool, + MIDGARD_MMU_BOTTOMLEVEL); + if (err) + goto pgd_no_mem; + + mutex_lock(&kctx->mmu_lock); + kctx->pgd = kbase_mmu_alloc_pgd(kctx); + mutex_unlock(&kctx->mmu_lock); + } while (!kctx->pgd); + + p = kbase_mem_alloc_page(&kctx->mem_pool); + if (!p) + goto no_sink_page; + kctx->aliasing_sink_page = as_tagged(page_to_phys(p)); + + init_waitqueue_head(&kctx->event_queue); + + kctx->cookies = KBASE_COOKIE_MASK; + + /* Make sure page 0 is not used... */ + err = kbase_region_tracker_init(kctx); + if (err) + goto no_region_tracker; + + err = kbase_sticky_resource_init(kctx); + if (err) + goto no_sticky; + + err = kbase_jit_init(kctx); + if (err) + goto no_jit; +#ifdef CONFIG_GPU_TRACEPOINTS + atomic_set(&kctx->jctx.work_id, 0); +#endif +#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE + atomic_set(&kctx->timeline.jd_atoms_in_flight, 0); +#endif + + kctx->id = atomic_add_return(1, &(kbdev->ctx_num)) - 1; + + mutex_init(&kctx->vinstr_cli_lock); + + timer_setup(&kctx->soft_job_timeout, + kbasep_soft_job_timeout_worker, + 0); + + return kctx; + +no_jit: + kbase_gpu_vm_lock(kctx); + kbase_sticky_resource_term(kctx); + kbase_gpu_vm_unlock(kctx); +no_sticky: + kbase_region_tracker_term(kctx); +no_region_tracker: + kbase_mem_pool_free(&kctx->mem_pool, p, false); +no_sink_page: + /* VM lock needed for the call to kbase_mmu_free_pgd */ + kbase_gpu_vm_lock(kctx); + kbase_mmu_free_pgd(kctx); + kbase_gpu_vm_unlock(kctx); +pgd_no_mem: + kbase_mmu_term(kctx); +term_dma_fence: + kbase_dma_fence_term(kctx); +free_event: + kbase_event_cleanup(kctx); +free_jd: + /* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */ + kbasep_js_kctx_term(kctx); + kbase_jd_exit(kctx); +deinit_evictable: + kbase_mem_evictable_deinit(kctx); +free_both_pools: + kbase_mem_pool_term(&kctx->lp_mem_pool); +free_mem_pool: + kbase_mem_pool_term(&kctx->mem_pool); +free_kctx: + vfree(kctx); +out: + return NULL; +} +KBASE_EXPORT_SYMBOL(kbase_create_context); + +static void kbase_reg_pending_dtor(struct kbase_va_region *reg) +{ + dev_dbg(reg->kctx->kbdev->dev, "Freeing pending unmapped region\n"); + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); + kfree(reg); +} + +/** + * kbase_destroy_context - Destroy a kernel base context. + * @kctx: Context to destroy + * + * Calls kbase_destroy_os_context() to free OS specific structures. + * Will release all outstanding regions. + */ +void kbase_destroy_context(struct kbase_context *kctx) +{ + struct kbase_device *kbdev; + int pages; + unsigned long pending_regions_to_clean; + unsigned long flags; + struct page *p; + + KBASE_DEBUG_ASSERT(NULL != kctx); + + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(NULL != kbdev); + + KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u); + + /* Ensure the core is powered up for the destroy process */ + /* A suspend won't happen here, because we're in a syscall from a userspace + * thread. */ + kbase_pm_context_active(kbdev); + + kbase_jd_zap_context(kctx); + +#ifdef CONFIG_DEBUG_FS + /* Removing the rest of the debugfs entries here as we want to keep the + * atom debugfs interface alive until all atoms have completed. This + * is useful for debugging hung contexts. */ + debugfs_remove_recursive(kctx->kctx_dentry); +#endif + + kbase_event_cleanup(kctx); + + /* + * JIT must be terminated before the code below as it must be called + * without the region lock being held. + * The code above ensures no new JIT allocations can be made by + * by the time we get to this point of context tear down. + */ + kbase_jit_term(kctx); + + kbase_gpu_vm_lock(kctx); + + kbase_sticky_resource_term(kctx); + + /* MMU is disabled as part of scheduling out the context */ + kbase_mmu_free_pgd(kctx); + + /* drop the aliasing sink page now that it can't be mapped anymore */ + p = phys_to_page(as_phys_addr_t(kctx->aliasing_sink_page)); + kbase_mem_pool_free(&kctx->mem_pool, p, false); + + /* free pending region setups */ + pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK; + while (pending_regions_to_clean) { + unsigned int cookie = __ffs(pending_regions_to_clean); + + BUG_ON(!kctx->pending_regions[cookie]); + + kbase_reg_pending_dtor(kctx->pending_regions[cookie]); + + kctx->pending_regions[cookie] = NULL; + pending_regions_to_clean &= ~(1UL << cookie); + } + + kbase_region_tracker_term(kctx); + kbase_gpu_vm_unlock(kctx); + + /* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */ + kbasep_js_kctx_term(kctx); + + kbase_jd_exit(kctx); + + kbase_pm_context_idle(kbdev); + + kbase_dma_fence_term(kctx); + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + kbase_ctx_sched_remove_ctx(kctx); + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_mmu_term(kctx); + + pages = atomic_read(&kctx->used_pages); + if (pages != 0) + dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); + + kbase_mem_evictable_deinit(kctx); + kbase_mem_pool_term(&kctx->mem_pool); + kbase_mem_pool_term(&kctx->lp_mem_pool); + WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0); + + vfree(kctx); +} +KBASE_EXPORT_SYMBOL(kbase_destroy_context); + +/** + * kbase_context_set_create_flags - Set creation flags on a context + * @kctx: Kbase context + * @flags: Flags to set + * + * Return: 0 on success + */ +int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags) +{ + int err = 0; + struct kbasep_js_kctx_info *js_kctx_info; + unsigned long irq_flags; + + KBASE_DEBUG_ASSERT(NULL != kctx); + + js_kctx_info = &kctx->jctx.sched_info; + + /* Validate flags */ + if (flags != (flags & BASE_CONTEXT_CREATE_KERNEL_FLAGS)) { + err = -EINVAL; + goto out; + } + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); + + /* Translate the flags */ + if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) + kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); + + /* Latch the initial attributes into the Job Scheduler */ + kbasep_js_ctx_attr_set_initial_attrs(kctx->kbdev, kctx); + + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + out: + return err; +} +KBASE_EXPORT_SYMBOL(kbase_context_set_create_flags); diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.h new file mode 100755 index 000000000000..a3f5bb0ce0da --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_context.h @@ -0,0 +1,90 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_CONTEXT_H_ +#define _KBASE_CONTEXT_H_ + +#include + + +int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags); + +/** + * kbase_ctx_flag - Check if @flag is set on @kctx + * @kctx: Pointer to kbase context to check + * @flag: Flag to check + * + * Return: true if @flag is set on @kctx, false if not. + */ +static inline bool kbase_ctx_flag(struct kbase_context *kctx, + enum kbase_context_flags flag) +{ + return atomic_read(&kctx->flags) & flag; +} + +/** + * kbase_ctx_flag_clear - Clear @flag on @kctx + * @kctx: Pointer to kbase context + * @flag: Flag to clear + * + * Clear the @flag on @kctx. This is done atomically, so other flags being + * cleared or set at the same time will be safe. + * + * Some flags have locking requirements, check the documentation for the + * respective flags. + */ +static inline void kbase_ctx_flag_clear(struct kbase_context *kctx, + enum kbase_context_flags flag) +{ +#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE + /* + * Earlier kernel versions doesn't have atomic_andnot() or + * atomic_and(). atomic_clear_mask() was only available on some + * architectures and removed on arm in v3.13 on arm and arm64. + * + * Use a compare-exchange loop to clear the flag on pre 4.3 kernels, + * when atomic_andnot() becomes available. + */ + int old, new; + + do { + old = atomic_read(&kctx->flags); + new = old & ~flag; + + } while (atomic_cmpxchg(&kctx->flags, old, new) != old); +#else + atomic_andnot(flag, &kctx->flags); +#endif +} + +/** + * kbase_ctx_flag_set - Set @flag on @kctx + * @kctx: Pointer to kbase context + * @flag: Flag to clear + * + * Set the @flag on @kctx. This is done atomically, so other flags being + * cleared or set at the same time will be safe. + * + * Some flags have locking requirements, check the documentation for the + * respective flags. + */ +static inline void kbase_ctx_flag_set(struct kbase_context *kctx, + enum kbase_context_flags flag) +{ + atomic_or(flag, &kctx->flags); +} +#endif /* _KBASE_CONTEXT_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_core_linux.c new file mode 100755 index 000000000000..347fee2643bb --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_core_linux.c @@ -0,0 +1,4971 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_MALI_BIFROST_DEVFREQ +#include +#include +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif /* CONFIG_DEVFREQ_THERMAL */ +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ +#ifdef CONFIG_MALI_BIFROST_NO_MALI +#include "mali_kbase_model_linux.h" +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ +#include "mali_kbase_mem_profile_debugfs_buf_size.h" +#include "mali_kbase_debug_mem_view.h" +#include "mali_kbase_mem.h" +#include "mali_kbase_mem_pool_debugfs.h" +#if !MALI_CUSTOMER_RELEASE +#include "mali_kbase_regs_dump_debugfs.h" +#endif /* !MALI_CUSTOMER_RELEASE */ +#include "mali_kbase_regs_history_debugfs.h" +#include +#include +#include +#include +#include "mali_kbase_ioctl.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* is_compat_task */ +#include +#include +#include +#include +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ +#include +#include + +#include + + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) +#include +#else +#include +#endif + +#include + +#include + +/* GPU IRQ Tags */ +#define JOB_IRQ_TAG 0 +#define MMU_IRQ_TAG 1 +#define GPU_IRQ_TAG 2 + +#if MALI_UNIT_TEST +static struct kbase_exported_test_data shared_kernel_test_data; +EXPORT_SYMBOL(shared_kernel_test_data); +#endif /* MALI_UNIT_TEST */ + +static int kbase_dev_nr; + +static DEFINE_MUTEX(kbase_dev_list_lock); +static LIST_HEAD(kbase_dev_list); + +#define KERNEL_SIDE_DDK_VERSION_STRING "K:" MALI_RELEASE_NAME "(GPL)" +static inline void __compile_time_asserts(void) +{ + CSTD_COMPILE_TIME_ASSERT(sizeof(KERNEL_SIDE_DDK_VERSION_STRING) <= KBASE_GET_VERSION_BUFFER_SIZE); +} + +static int kbase_api_handshake(struct kbase_context *kctx, + struct kbase_ioctl_version_check *version) +{ + switch (version->major) { + case BASE_UK_VERSION_MAJOR: + /* set minor to be the lowest common */ + version->minor = min_t(int, BASE_UK_VERSION_MINOR, + (int)version->minor); + break; + default: + /* We return our actual version regardless if it + * matches the version returned by userspace - + * userspace can bail if it can't handle this + * version */ + version->major = BASE_UK_VERSION_MAJOR; + version->minor = BASE_UK_VERSION_MINOR; + break; + } + + /* save the proposed version number for later use */ + kctx->api_version = KBASE_API_VERSION(version->major, version->minor); + + return 0; +} + +/** + * enum mali_error - Mali error codes shared with userspace + * + * This is subset of those common Mali errors that can be returned to userspace. + * Values of matching user and kernel space enumerators MUST be the same. + * MALI_ERROR_NONE is guaranteed to be 0. + * + * @MALI_ERROR_NONE: Success + * @MALI_ERROR_OUT_OF_GPU_MEMORY: Not used in the kernel driver + * @MALI_ERROR_OUT_OF_MEMORY: Memory allocation failure + * @MALI_ERROR_FUNCTION_FAILED: Generic error code + */ +enum mali_error { + MALI_ERROR_NONE = 0, + MALI_ERROR_OUT_OF_GPU_MEMORY, + MALI_ERROR_OUT_OF_MEMORY, + MALI_ERROR_FUNCTION_FAILED, +}; + +enum { + inited_mem = (1u << 0), + inited_js = (1u << 1), + inited_pm_runtime_init = (1u << 2), +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + inited_devfreq = (1u << 3), +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + inited_tlstream = (1u << 4), + inited_backend_early = (1u << 5), + inited_backend_late = (1u << 6), + inited_device = (1u << 7), + inited_vinstr = (1u << 8), + + inited_job_fault = (1u << 10), + inited_sysfs_group = (1u << 11), + inited_misc_register = (1u << 12), + inited_get_device = (1u << 13), + inited_dev_list = (1u << 14), + inited_debugfs = (1u << 15), + inited_gpu_device = (1u << 16), + inited_registers_map = (1u << 17), + inited_io_history = (1u << 18), + inited_power_control = (1u << 19), + inited_buslogger = (1u << 20), + inited_protected = (1u << 21), + inited_ctx_sched = (1u << 22) +}; + + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define INACTIVE_WAIT_MS (5000) + +void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive) +{ + kbdev->driver_inactive = inactive; + wake_up(&kbdev->driver_inactive_wait); + + /* Wait for any running IOCTLs to complete */ + if (inactive) + msleep(INACTIVE_WAIT_MS); +} +KBASE_EXPORT_TEST_API(kbase_set_driver_inactive); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + +/** + * kbase_legacy_dispatch - UKK dispatch function + * + * This is the dispatch function for the legacy UKK ioctl interface. No new + * ioctls should be added to this function, see kbase_ioctl instead. + * + * @kctx: The kernel context structure + * @args: Pointer to the data structure passed from/to user space + * @args_size: Size of the data structure + */ +static int kbase_legacy_dispatch(struct kbase_context *kctx, + void * const args, u32 args_size) +{ + struct kbase_device *kbdev; + union uk_header *ukh = args; + u32 id; + int ret = 0; + + KBASE_DEBUG_ASSERT(ukh != NULL); + + kbdev = kctx->kbdev; + id = ukh->id; + ukh->ret = MALI_ERROR_NONE; /* Be optimistic */ + +#ifdef CONFIG_MALI_BIFROST_DEBUG + wait_event(kbdev->driver_inactive_wait, + kbdev->driver_inactive == false); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + if (UKP_FUNC_ID_CHECK_VERSION == id) { + struct uku_version_check_args *version_check; + struct kbase_ioctl_version_check version; + + if (args_size != sizeof(struct uku_version_check_args)) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + return 0; + } + version_check = (struct uku_version_check_args *)args; + version.minor = version_check->minor; + version.major = version_check->major; + + kbase_api_handshake(kctx, &version); + + version_check->minor = version.minor; + version_check->major = version.major; + ukh->ret = MALI_ERROR_NONE; + return 0; + } + + /* block calls until version handshake */ + if (kctx->api_version == 0) + return -EINVAL; + + if (!atomic_read(&kctx->setup_complete)) { + struct kbase_uk_set_flags *kbase_set_flags; + + /* setup pending, try to signal that we'll do the setup, + * if setup was already in progress, err this call + */ + if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) + return -EINVAL; + + /* if unexpected call, will stay stuck in setup mode + * (is it the only call we accept?) + */ + if (id != KBASE_FUNC_SET_FLAGS) + return -EINVAL; + + kbase_set_flags = (struct kbase_uk_set_flags *)args; + + /* if not matching the expected call, stay in setup mode */ + if (sizeof(*kbase_set_flags) != args_size) + goto bad_size; + + /* if bad flags, will stay stuck in setup mode */ + if (kbase_context_set_create_flags(kctx, + kbase_set_flags->create_flags) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + + atomic_set(&kctx->setup_complete, 1); + return 0; + } + + /* setup complete, perform normal operation */ + switch (id) { + case KBASE_FUNC_MEM_JIT_INIT: + { + struct kbase_uk_mem_jit_init *jit_init = args; + + if (sizeof(*jit_init) != args_size) + goto bad_size; + + if (kbase_region_tracker_init_jit(kctx, + jit_init->va_pages)) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + case KBASE_FUNC_MEM_ALLOC: + { + struct kbase_uk_mem_alloc *mem = args; + struct kbase_va_region *reg; + + if (sizeof(*mem) != args_size) + goto bad_size; + +#if defined(CONFIG_64BIT) + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* force SAME_VA if a 64-bit client */ + mem->flags |= BASE_MEM_SAME_VA; + } +#endif + + reg = kbase_mem_alloc(kctx, mem->va_pages, + mem->commit_pages, mem->extent, + &mem->flags, &mem->gpu_va); + mem->va_alignment = 0; + + if (!reg) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + case KBASE_FUNC_MEM_IMPORT: { + struct kbase_uk_mem_import *mem_import = args; + void __user *phandle; + + if (sizeof(*mem_import) != args_size) + goto bad_size; +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + phandle = compat_ptr(mem_import->phandle); + else +#endif + phandle = u64_to_user_ptr(mem_import->phandle); + + if (mem_import->type == BASE_MEM_IMPORT_TYPE_INVALID) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + + if (kbase_mem_import(kctx, + (enum base_mem_import_type) + mem_import->type, + phandle, + 0, + &mem_import->gpu_va, + &mem_import->va_pages, + &mem_import->flags)) { + mem_import->type = BASE_MEM_IMPORT_TYPE_INVALID; + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + } + break; + } + case KBASE_FUNC_MEM_ALIAS: { + struct kbase_uk_mem_alias *alias = args; + struct base_mem_aliasing_info __user *user_ai; + struct base_mem_aliasing_info *ai; + + if (sizeof(*alias) != args_size) + goto bad_size; + + if (alias->nents > 2048) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + if (!alias->nents) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + user_ai = compat_ptr(alias->ai); + else +#endif + user_ai = u64_to_user_ptr(alias->ai); + + ai = vmalloc(sizeof(*ai) * alias->nents); + + if (!ai) { + ukh->ret = MALI_ERROR_OUT_OF_MEMORY; + break; + } + + if (copy_from_user(ai, user_ai, + sizeof(*ai) * alias->nents)) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + goto copy_failed; + } + + alias->gpu_va = kbase_mem_alias(kctx, &alias->flags, + alias->stride, + alias->nents, ai, + &alias->va_pages); + if (!alias->gpu_va) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + goto no_alias; + } +no_alias: +copy_failed: + vfree(ai); + break; + } + case KBASE_FUNC_MEM_COMMIT: + { + struct kbase_uk_mem_commit *commit = args; + int ret; + + if (sizeof(*commit) != args_size) + goto bad_size; + + ret = kbase_mem_commit(kctx, commit->gpu_addr, + commit->pages); + + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + commit->result_subcode = + BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS; + + if (ret == 0) { + ukh->ret = MALI_ERROR_NONE; + commit->result_subcode = + BASE_BACKING_THRESHOLD_OK; + } else if (ret == -ENOMEM) { + commit->result_subcode = + BASE_BACKING_THRESHOLD_ERROR_OOM; + } + + break; + } + + case KBASE_FUNC_MEM_QUERY: + { + struct kbase_uk_mem_query *query = args; + + if (sizeof(*query) != args_size) + goto bad_size; + + if (kbase_mem_query(kctx, query->gpu_addr, + query->query, &query->value) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + ukh->ret = MALI_ERROR_NONE; + break; + } + break; + + case KBASE_FUNC_MEM_FLAGS_CHANGE: + { + struct kbase_uk_mem_flags_change *fc = args; + + if (sizeof(*fc) != args_size) + goto bad_size; + + if (kbase_mem_flags_change(kctx, fc->gpu_va, + fc->flags, fc->mask) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + + break; + } + case KBASE_FUNC_MEM_FREE: + { + struct kbase_uk_mem_free *mem = args; + + if (sizeof(*mem) != args_size) + goto bad_size; + + if (kbase_mem_free(kctx, mem->gpu_addr) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + + case KBASE_FUNC_JOB_SUBMIT: + { + struct kbase_uk_job_submit *job = args; + char __user *user_buf; + + if (sizeof(*job) != args_size) + goto bad_size; + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + user_buf = compat_ptr(job->addr); + else +#endif + user_buf = u64_to_user_ptr(job->addr); + + if (kbase_jd_submit(kctx, user_buf, + job->nr_atoms, + job->stride, + false) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + + case KBASE_FUNC_SYNC: + { + struct kbase_uk_sync_now *sn = args; + + if (sizeof(*sn) != args_size) + goto bad_size; + + if (kbase_sync_now(kctx, &sn->sset.basep_sset) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + + case KBASE_FUNC_DISJOINT_QUERY: + { + struct kbase_uk_disjoint_query *dquery = args; + + if (sizeof(*dquery) != args_size) + goto bad_size; + + /* Get the disjointness counter value. */ + dquery->counter = kbase_disjoint_event_get(kctx->kbdev); + break; + } + + case KBASE_FUNC_POST_TERM: + { + kbase_event_close(kctx); + break; + } + + case KBASE_FUNC_HWCNT_SETUP: + { + struct kbase_uk_hwcnt_setup *setup = args; + + if (sizeof(*setup) != args_size) + goto bad_size; + + mutex_lock(&kctx->vinstr_cli_lock); + if (kbase_vinstr_legacy_hwc_setup(kbdev->vinstr_ctx, + &kctx->vinstr_cli, setup) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + mutex_unlock(&kctx->vinstr_cli_lock); + break; + } + + case KBASE_FUNC_HWCNT_DUMP: + { + /* args ignored */ + mutex_lock(&kctx->vinstr_cli_lock); + if (kbase_vinstr_hwc_dump(kctx->vinstr_cli, + BASE_HWCNT_READER_EVENT_MANUAL) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + mutex_unlock(&kctx->vinstr_cli_lock); + break; + } + + case KBASE_FUNC_HWCNT_CLEAR: + { + /* args ignored */ + mutex_lock(&kctx->vinstr_cli_lock); + if (kbase_vinstr_hwc_clear(kctx->vinstr_cli) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + mutex_unlock(&kctx->vinstr_cli_lock); + break; + } + + case KBASE_FUNC_HWCNT_READER_SETUP: + { + struct kbase_uk_hwcnt_reader_setup *setup = args; + + if (sizeof(*setup) != args_size) + goto bad_size; + + mutex_lock(&kctx->vinstr_cli_lock); + if (kbase_vinstr_hwcnt_reader_setup(kbdev->vinstr_ctx, + setup) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + mutex_unlock(&kctx->vinstr_cli_lock); + break; + } + + case KBASE_FUNC_GPU_PROPS_REG_DUMP: + { + struct kbase_uk_gpuprops *setup = args; + + if (sizeof(*setup) != args_size) + goto bad_size; + + if (kbase_gpuprops_uk_get_props(kctx, setup) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + case KBASE_FUNC_FIND_CPU_OFFSET: + { + struct kbase_uk_find_cpu_offset *find = args; + + if (sizeof(*find) != args_size) + goto bad_size; + + if (find->gpu_addr & ~PAGE_MASK) { + dev_warn(kbdev->dev, "kbase_legacy_dispatch case KBASE_FUNC_FIND_CPU_OFFSET: find->gpu_addr: passed parameter is invalid"); + goto out_bad; + } + + if (find->size > SIZE_MAX || find->cpu_addr > ULONG_MAX) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + } else { + int err; + + err = kbasep_find_enclosing_cpu_mapping_offset( + kctx, + find->cpu_addr, + find->size, + &find->offset); + + if (err) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + } + break; + } + case KBASE_FUNC_GET_VERSION: + { + struct kbase_uk_get_ddk_version *get_version = (struct kbase_uk_get_ddk_version *)args; + + if (sizeof(*get_version) != args_size) + goto bad_size; + + /* version buffer size check is made in compile time assert */ + memcpy(get_version->version_buffer, KERNEL_SIDE_DDK_VERSION_STRING, sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); + get_version->version_string_size = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); + break; + } + + case KBASE_FUNC_STREAM_CREATE: + { +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + struct kbase_uk_stream_create *screate = (struct kbase_uk_stream_create *)args; + + if (sizeof(*screate) != args_size) + goto bad_size; + + if (strnlen(screate->name, sizeof(screate->name)) >= sizeof(screate->name)) { + /* not NULL terminated */ + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + + if (kbase_sync_fence_stream_create(screate->name, + &screate->fd) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + ukh->ret = MALI_ERROR_NONE; +#else /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + ukh->ret = MALI_ERROR_FUNCTION_FAILED; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + break; + } + case KBASE_FUNC_FENCE_VALIDATE: + { +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + struct kbase_uk_fence_validate *fence_validate = (struct kbase_uk_fence_validate *)args; + + if (sizeof(*fence_validate) != args_size) + goto bad_size; + + if (kbase_sync_fence_validate(fence_validate->fd) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + ukh->ret = MALI_ERROR_NONE; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + break; + } + + case KBASE_FUNC_SET_TEST_DATA: + { +#if MALI_UNIT_TEST + struct kbase_uk_set_test_data *set_data = args; + + shared_kernel_test_data = set_data->test_data; + shared_kernel_test_data.kctx = (uintptr_t)kctx; + shared_kernel_test_data.mm = (uintptr_t)current->mm; + ukh->ret = MALI_ERROR_NONE; +#endif /* MALI_UNIT_TEST */ + break; + } + + case KBASE_FUNC_INJECT_ERROR: + { +#ifdef CONFIG_MALI_BIFROST_ERROR_INJECT + unsigned long flags; + struct kbase_error_params params = ((struct kbase_uk_error_params *)args)->params; + + /*mutex lock */ + spin_lock_irqsave(&kbdev->reg_op_lock, flags); + if (job_atom_inject_error(¶ms) != 0) + ukh->ret = MALI_ERROR_OUT_OF_MEMORY; + else + ukh->ret = MALI_ERROR_NONE; + spin_unlock_irqrestore(&kbdev->reg_op_lock, flags); + /*mutex unlock */ +#endif /* CONFIG_MALI_BIFROST_ERROR_INJECT */ + break; + } + + case KBASE_FUNC_MODEL_CONTROL: + { +#ifdef CONFIG_MALI_BIFROST_NO_MALI + unsigned long flags; + struct kbase_model_control_params params = + ((struct kbase_uk_model_control_params *)args)->params; + + /*mutex lock */ + spin_lock_irqsave(&kbdev->reg_op_lock, flags); + if (gpu_model_control(kbdev->model, ¶ms) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + ukh->ret = MALI_ERROR_NONE; + spin_unlock_irqrestore(&kbdev->reg_op_lock, flags); + /*mutex unlock */ +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ + break; + } + + case KBASE_FUNC_GET_PROFILING_CONTROLS: + { + struct kbase_uk_profiling_controls *controls = + (struct kbase_uk_profiling_controls *)args; + u32 i; + + if (sizeof(*controls) != args_size) + goto bad_size; + + for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) + controls->profiling_controls[i] = + kbdev->kbase_profiling_controls[i]; + + break; + } + + /* used only for testing purposes; these controls are to be set by gator through gator API */ + case KBASE_FUNC_SET_PROFILING_CONTROLS: + { + struct kbase_uk_profiling_controls *controls = + (struct kbase_uk_profiling_controls *)args; + u32 i; + + if (sizeof(*controls) != args_size) + goto bad_size; + + for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) + _mali_profiling_control(i, controls->profiling_controls[i]); + + break; + } + + case KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD: + { + struct kbase_uk_debugfs_mem_profile_add *add_data = + (struct kbase_uk_debugfs_mem_profile_add *)args; + char *buf; + char __user *user_buf; + + if (sizeof(*add_data) != args_size) + goto bad_size; + + if (add_data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { + dev_err(kbdev->dev, "buffer too big\n"); + goto out_bad; + } + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + user_buf = compat_ptr(add_data->buf); + else +#endif + user_buf = u64_to_user_ptr(add_data->buf); + + buf = kmalloc(add_data->len, GFP_KERNEL); + if (ZERO_OR_NULL_PTR(buf)) + goto out_bad; + + if (0 != copy_from_user(buf, user_buf, add_data->len)) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + kfree(buf); + goto out_bad; + } + + if (kbasep_mem_profile_debugfs_insert(kctx, buf, + add_data->len)) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + goto out_bad; + } + + break; + } + +#ifdef CONFIG_MALI_BIFROST_NO_MALI + case KBASE_FUNC_SET_PRFCNT_VALUES: + { + + struct kbase_uk_prfcnt_values *params = + ((struct kbase_uk_prfcnt_values *)args); + gpu_model_set_dummy_prfcnt_sample(params->data, + params->size); + + break; + } +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ +#ifdef BASE_LEGACY_UK10_4_SUPPORT + case KBASE_FUNC_TLSTREAM_ACQUIRE_V10_4: + { + struct kbase_uk_tlstream_acquire_v10_4 *tlstream_acquire + = args; + int ret; + + if (sizeof(*tlstream_acquire) != args_size) + goto bad_size; + + ret = kbase_tlstream_acquire( + kctx, 0); + if (ret < 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + tlstream_acquire->fd = ret; + break; + } +#endif /* BASE_LEGACY_UK10_4_SUPPORT */ + case KBASE_FUNC_TLSTREAM_ACQUIRE: + { + struct kbase_uk_tlstream_acquire *tlstream_acquire = + args; + int ret; + + if (sizeof(*tlstream_acquire) != args_size) + goto bad_size; + + if (tlstream_acquire->flags & ~BASE_TLSTREAM_FLAGS_MASK) + goto out_bad; + + ret = kbase_tlstream_acquire( + kctx, tlstream_acquire->flags); + if (ret < 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + tlstream_acquire->fd = ret; + break; + } + case KBASE_FUNC_TLSTREAM_FLUSH: + { + struct kbase_uk_tlstream_flush *tlstream_flush = + args; + + if (sizeof(*tlstream_flush) != args_size) + goto bad_size; + + kbase_tlstream_flush_streams(); + break; + } +#if MALI_UNIT_TEST + case KBASE_FUNC_TLSTREAM_TEST: + { + struct kbase_uk_tlstream_test *tlstream_test = args; + + if (sizeof(*tlstream_test) != args_size) + goto bad_size; + + kbase_tlstream_test( + tlstream_test->tpw_count, + tlstream_test->msg_delay, + tlstream_test->msg_count, + tlstream_test->aux_msg); + break; + } + case KBASE_FUNC_TLSTREAM_STATS: + { + struct kbase_uk_tlstream_stats *tlstream_stats = args; + + if (sizeof(*tlstream_stats) != args_size) + goto bad_size; + + kbase_tlstream_stats( + &tlstream_stats->bytes_collected, + &tlstream_stats->bytes_generated); + break; + } +#endif /* MALI_UNIT_TEST */ + + case KBASE_FUNC_GET_CONTEXT_ID: + { + struct kbase_uk_context_id *info = args; + + info->id = kctx->id; + break; + } + + case KBASE_FUNC_SOFT_EVENT_UPDATE: + { + struct kbase_uk_soft_event_update *update = args; + + if (sizeof(*update) != args_size) + goto bad_size; + + if (((update->new_status != BASE_JD_SOFT_EVENT_SET) && + (update->new_status != BASE_JD_SOFT_EVENT_RESET)) || + (update->flags != 0)) + goto out_bad; + + if (kbase_soft_event_update(kctx, update->evt, + update->new_status)) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + + break; + } + + default: + dev_err(kbdev->dev, "unknown ioctl %u\n", id); + goto out_bad; + } + + return ret; + + bad_size: + dev_err(kbdev->dev, "Wrong syscall size (%d) for %08x\n", args_size, id); + out_bad: + return -EINVAL; +} + +static struct kbase_device *to_kbase_device(struct device *dev) +{ + return dev_get_drvdata(dev); +} + +static int assign_irqs(struct platform_device *pdev) +{ + struct kbase_device *kbdev = to_kbase_device(&pdev->dev); + int i; + + if (!kbdev) + return -ENODEV; + + /* 3 IRQ resources */ + for (i = 0; i < 3; i++) { + struct resource *irq_res; + int irqtag; + + irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, i); + if (!irq_res) { + dev_err(kbdev->dev, "No IRQ resource at index %d\n", i); + return -ENOENT; + } + +#ifdef CONFIG_OF + if (!strncmp(irq_res->name, "JOB", 4)) { + irqtag = JOB_IRQ_TAG; + } else if (!strncmp(irq_res->name, "MMU", 4)) { + irqtag = MMU_IRQ_TAG; + } else if (!strncmp(irq_res->name, "GPU", 4)) { + irqtag = GPU_IRQ_TAG; + } else { + dev_err(&pdev->dev, "Invalid irq res name: '%s'\n", + irq_res->name); + return -EINVAL; + } +#else + irqtag = i; +#endif /* CONFIG_OF */ + kbdev->irqs[irqtag].irq = irq_res->start; + kbdev->irqs[irqtag].flags = irq_res->flags & IRQF_TRIGGER_MASK; + } + + return 0; +} + +/* + * API to acquire device list mutex and + * return pointer to the device list head + */ +const struct list_head *kbase_dev_list_get(void) +{ + mutex_lock(&kbase_dev_list_lock); + return &kbase_dev_list; +} +KBASE_EXPORT_TEST_API(kbase_dev_list_get); + +/* API to release the device list mutex */ +void kbase_dev_list_put(const struct list_head *dev_list) +{ + mutex_unlock(&kbase_dev_list_lock); +} +KBASE_EXPORT_TEST_API(kbase_dev_list_put); + +/* Find a particular kbase device (as specified by minor number), or find the "first" device if -1 is specified */ +struct kbase_device *kbase_find_device(int minor) +{ + struct kbase_device *kbdev = NULL; + struct list_head *entry; + const struct list_head *dev_list = kbase_dev_list_get(); + + list_for_each(entry, dev_list) { + struct kbase_device *tmp; + + tmp = list_entry(entry, struct kbase_device, entry); + if (tmp->mdev.minor == minor || minor == -1) { + kbdev = tmp; + get_device(kbdev->dev); + break; + } + } + kbase_dev_list_put(dev_list); + + return kbdev; +} +EXPORT_SYMBOL(kbase_find_device); + +void kbase_release_device(struct kbase_device *kbdev) +{ + put_device(kbdev->dev); +} +EXPORT_SYMBOL(kbase_release_device); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && \ + !(LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 28) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)) +/* + * Older versions, before v4.6, of the kernel doesn't have + * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28 + */ +static int kstrtobool_from_user(const char __user *s, size_t count, bool *res) +{ + char buf[32]; + + count = min(sizeof(buf), count); + + if (copy_from_user(buf, s, count)) + return -EFAULT; + buf[count] = '\0'; + + return strtobool(buf, res); +} +#endif + +static ssize_t write_ctx_infinite_cache(struct file *f, const char __user *ubuf, size_t size, loff_t *off) +{ + struct kbase_context *kctx = f->private_data; + int err; + bool value; + + err = kstrtobool_from_user(ubuf, size, &value); + if (err) + return err; + + if (value) + kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); + else + kbase_ctx_flag_clear(kctx, KCTX_INFINITE_CACHE); + + return size; +} + +static ssize_t read_ctx_infinite_cache(struct file *f, char __user *ubuf, size_t size, loff_t *off) +{ + struct kbase_context *kctx = f->private_data; + char buf[32]; + int count; + bool value; + + value = kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE); + + count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); + + return simple_read_from_buffer(ubuf, size, off, buf, count); +} + +static const struct file_operations kbase_infinite_cache_fops = { + .open = simple_open, + .write = write_ctx_infinite_cache, + .read = read_ctx_infinite_cache, +}; + +static int kbase_open(struct inode *inode, struct file *filp) +{ + struct kbase_device *kbdev = NULL; + struct kbase_context *kctx; + int ret = 0; +#ifdef CONFIG_DEBUG_FS + char kctx_name[64]; +#endif + + kbdev = kbase_find_device(iminor(inode)); + + if (!kbdev) + return -ENODEV; + + kctx = kbase_create_context(kbdev, is_compat_task()); + if (!kctx) { + ret = -ENOMEM; + goto out; + } + + init_waitqueue_head(&kctx->event_queue); + filp->private_data = kctx; + kctx->filp = filp; + + if (kbdev->infinite_cache_active_default) + kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); + +#ifdef CONFIG_DEBUG_FS + snprintf(kctx_name, 64, "%d_%d", kctx->tgid, kctx->id); + + kctx->kctx_dentry = debugfs_create_dir(kctx_name, + kbdev->debugfs_ctx_directory); + + if (IS_ERR_OR_NULL(kctx->kctx_dentry)) { + ret = -ENOMEM; + goto out; + } + + debugfs_create_file("infinite_cache", 0644, kctx->kctx_dentry, + kctx, &kbase_infinite_cache_fops); + + mutex_init(&kctx->mem_profile_lock); + + kbasep_jd_debugfs_ctx_init(kctx); + kbase_debug_mem_view_init(filp); + + kbase_debug_job_fault_context_init(kctx); + + kbase_mem_pool_debugfs_init(kctx->kctx_dentry, &kctx->mem_pool, &kctx->lp_mem_pool); + + kbase_jit_debugfs_init(kctx); +#endif /* CONFIG_DEBUG_FS */ + + dev_dbg(kbdev->dev, "created base context\n"); + + { + struct kbasep_kctx_list_element *element; + + element = kzalloc(sizeof(*element), GFP_KERNEL); + if (element) { + mutex_lock(&kbdev->kctx_list_lock); + element->kctx = kctx; + list_add(&element->link, &kbdev->kctx_list); + KBASE_TLSTREAM_TL_NEW_CTX( + element->kctx, + element->kctx->id, + (u32)(element->kctx->tgid)); + mutex_unlock(&kbdev->kctx_list_lock); + } else { + /* we don't treat this as a fail - just warn about it */ + dev_warn(kbdev->dev, "couldn't add kctx to kctx_list\n"); + } + } + return 0; + + out: + kbase_release_device(kbdev); + return ret; +} + +static int kbase_release(struct inode *inode, struct file *filp) +{ + struct kbase_context *kctx = filp->private_data; + struct kbase_device *kbdev = kctx->kbdev; + struct kbasep_kctx_list_element *element, *tmp; + bool found_element = false; + + KBASE_TLSTREAM_TL_DEL_CTX(kctx); + +#ifdef CONFIG_DEBUG_FS + kbasep_mem_profile_debugfs_remove(kctx); + kbase_debug_job_fault_context_term(kctx); +#endif + + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) { + if (element->kctx == kctx) { + list_del(&element->link); + kfree(element); + found_element = true; + } + } + mutex_unlock(&kbdev->kctx_list_lock); + if (!found_element) + dev_warn(kbdev->dev, "kctx not in kctx_list\n"); + + filp->private_data = NULL; + + mutex_lock(&kctx->vinstr_cli_lock); + /* If this client was performing hwcnt dumping and did not explicitly + * detach itself, remove it from the vinstr core now */ + if (kctx->vinstr_cli) { + struct kbase_uk_hwcnt_setup setup; + + setup.dump_buffer = 0llu; + kbase_vinstr_legacy_hwc_setup( + kbdev->vinstr_ctx, &kctx->vinstr_cli, &setup); + } + mutex_unlock(&kctx->vinstr_cli_lock); + + kbase_destroy_context(kctx); + + dev_dbg(kbdev->dev, "deleted base context\n"); + kbase_release_device(kbdev); + return 0; +} + +#define CALL_MAX_SIZE 536 + +static long kbase_legacy_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + u64 msg[(CALL_MAX_SIZE + 7) >> 3] = { 0xdeadbeefdeadbeefull }; /* alignment fixup */ + u32 size = _IOC_SIZE(cmd); + struct kbase_context *kctx = filp->private_data; + + if (size > CALL_MAX_SIZE) + return -ENOTTY; + + if (0 != copy_from_user(&msg, (void __user *)arg, size)) { + dev_err(kctx->kbdev->dev, "failed to copy ioctl argument into kernel space\n"); + return -EFAULT; + } + + if (kbase_legacy_dispatch(kctx, &msg, size) != 0) + return -EFAULT; + + if (0 != copy_to_user((void __user *)arg, &msg, size)) { + dev_err(kctx->kbdev->dev, "failed to copy results of UK call back to user space\n"); + return -EFAULT; + } + return 0; +} + +static int kbase_api_set_flags(struct kbase_context *kctx, + struct kbase_ioctl_set_flags *flags) +{ + int err; + + /* setup pending, try to signal that we'll do the setup, + * if setup was already in progress, err this call + */ + if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) + return -EINVAL; + + err = kbase_context_set_create_flags(kctx, flags->create_flags); + /* if bad flags, will stay stuck in setup mode */ + if (err) + return err; + + atomic_set(&kctx->setup_complete, 1); + return 0; +} + +static int kbase_api_job_submit(struct kbase_context *kctx, + struct kbase_ioctl_job_submit *submit) +{ + return kbase_jd_submit(kctx, u64_to_user_ptr(submit->addr), + submit->nr_atoms, + submit->stride, false); +} + +static int kbase_api_get_gpuprops(struct kbase_context *kctx, + struct kbase_ioctl_get_gpuprops *get_props) +{ + struct kbase_gpu_props *kprops = &kctx->kbdev->gpu_props; + int err; + + if (get_props->flags != 0) { + dev_err(kctx->kbdev->dev, "Unsupported flags to get_gpuprops"); + return -EINVAL; + } + + if (get_props->size == 0) + return kprops->prop_buffer_size; + if (get_props->size < kprops->prop_buffer_size) + return -EINVAL; + + err = copy_to_user(u64_to_user_ptr(get_props->buffer), + kprops->prop_buffer, + kprops->prop_buffer_size); + if (err) + return -EFAULT; + return kprops->prop_buffer_size; +} + +static int kbase_api_post_term(struct kbase_context *kctx) +{ + kbase_event_close(kctx); + return 0; +} + +static int kbase_api_mem_alloc(struct kbase_context *kctx, + union kbase_ioctl_mem_alloc *alloc) +{ + struct kbase_va_region *reg; + u64 flags = alloc->in.flags; + u64 gpu_va; + +#if defined(CONFIG_64BIT) + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* force SAME_VA if a 64-bit client */ + flags |= BASE_MEM_SAME_VA; + } +#endif + + reg = kbase_mem_alloc(kctx, alloc->in.va_pages, + alloc->in.commit_pages, + alloc->in.extent, + &flags, &gpu_va); + + if (!reg) + return -ENOMEM; + + alloc->out.flags = flags; + alloc->out.gpu_va = gpu_va; + + return 0; +} + +static int kbase_api_mem_query(struct kbase_context *kctx, + union kbase_ioctl_mem_query *query) +{ + return kbase_mem_query(kctx, query->in.gpu_addr, + query->in.query, &query->out.value); +} + +static int kbase_api_mem_free(struct kbase_context *kctx, + struct kbase_ioctl_mem_free *free) +{ + return kbase_mem_free(kctx, free->gpu_addr); +} + +static int kbase_api_hwcnt_reader_setup(struct kbase_context *kctx, + struct kbase_ioctl_hwcnt_reader_setup *setup) +{ + int ret; + struct kbase_uk_hwcnt_reader_setup args = { + .buffer_count = setup->buffer_count, + .jm_bm = setup->jm_bm, + .shader_bm = setup->shader_bm, + .tiler_bm = setup->tiler_bm, + .mmu_l2_bm = setup->mmu_l2_bm + }; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_hwcnt_reader_setup(kctx->kbdev->vinstr_ctx, &args); + mutex_unlock(&kctx->vinstr_cli_lock); + + if (ret) + return ret; + return args.fd; +} + +static int kbase_api_hwcnt_enable(struct kbase_context *kctx, + struct kbase_ioctl_hwcnt_enable *enable) +{ + int ret; + struct kbase_uk_hwcnt_setup args = { + .dump_buffer = enable->dump_buffer, + .jm_bm = enable->jm_bm, + .shader_bm = enable->shader_bm, + .tiler_bm = enable->tiler_bm, + .mmu_l2_bm = enable->mmu_l2_bm + }; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_legacy_hwc_setup(kctx->kbdev->vinstr_ctx, + &kctx->vinstr_cli, &args); + mutex_unlock(&kctx->vinstr_cli_lock); + + return ret; +} + +static int kbase_api_hwcnt_dump(struct kbase_context *kctx) +{ + int ret; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_hwc_dump(kctx->vinstr_cli, + BASE_HWCNT_READER_EVENT_MANUAL); + mutex_unlock(&kctx->vinstr_cli_lock); + + return ret; +} + +static int kbase_api_hwcnt_clear(struct kbase_context *kctx) +{ + int ret; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_hwc_clear(kctx->vinstr_cli); + mutex_unlock(&kctx->vinstr_cli_lock); + + return ret; +} + +static int kbase_api_disjoint_query(struct kbase_context *kctx, + struct kbase_ioctl_disjoint_query *query) +{ + query->counter = kbase_disjoint_event_get(kctx->kbdev); + + return 0; +} + +static int kbase_api_get_ddk_version(struct kbase_context *kctx, + struct kbase_ioctl_get_ddk_version *version) +{ + int ret; + int len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); + + if (version->version_buffer == 0) + return len; + + if (version->size < len) + return -EOVERFLOW; + + ret = copy_to_user(u64_to_user_ptr(version->version_buffer), + KERNEL_SIDE_DDK_VERSION_STRING, + sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); + + if (ret) + return -EFAULT; + + return len; +} + +static int kbase_api_mem_jit_init(struct kbase_context *kctx, + struct kbase_ioctl_mem_jit_init *jit_init) +{ + return kbase_region_tracker_init_jit(kctx, jit_init->va_pages); +} + +static int kbase_api_mem_sync(struct kbase_context *kctx, + struct kbase_ioctl_mem_sync *sync) +{ + struct basep_syncset sset = { + .mem_handle.basep.handle = sync->handle, + .user_addr = sync->user_addr, + .size = sync->size, + .type = sync->type + }; + + return kbase_sync_now(kctx, &sset); +} + +static int kbase_api_mem_find_cpu_offset(struct kbase_context *kctx, + union kbase_ioctl_mem_find_cpu_offset *find) +{ + return kbasep_find_enclosing_cpu_mapping_offset( + kctx, + find->in.cpu_addr, + find->in.size, + &find->out.offset); +} + +static int kbase_api_get_context_id(struct kbase_context *kctx, + struct kbase_ioctl_get_context_id *info) +{ + info->id = kctx->id; + + return 0; +} + +static int kbase_api_tlstream_acquire(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_acquire *acquire) +{ + return kbase_tlstream_acquire(kctx, acquire->flags); +} + +static int kbase_api_tlstream_flush(struct kbase_context *kctx) +{ + kbase_tlstream_flush_streams(); + + return 0; +} + +static int kbase_api_mem_commit(struct kbase_context *kctx, + struct kbase_ioctl_mem_commit *commit) +{ + return kbase_mem_commit(kctx, commit->gpu_addr, commit->pages); +} + +static int kbase_api_mem_alias(struct kbase_context *kctx, + union kbase_ioctl_mem_alias *alias) +{ + struct base_mem_aliasing_info *ai; + u64 flags; + int err; + + if (alias->in.nents == 0 || alias->in.nents > 2048) + return -EINVAL; + + ai = vmalloc(sizeof(*ai) * alias->in.nents); + if (!ai) + return -ENOMEM; + + err = copy_from_user(ai, + u64_to_user_ptr(alias->in.aliasing_info), + sizeof(*ai) * alias->in.nents); + if (err) { + vfree(ai); + return -EFAULT; + } + + flags = alias->in.flags; + + alias->out.gpu_va = kbase_mem_alias(kctx, &flags, + alias->in.stride, alias->in.nents, + ai, &alias->out.va_pages); + + alias->out.flags = flags; + + vfree(ai); + + if (alias->out.gpu_va == 0) + return -ENOMEM; + + return 0; +} + +static int kbase_api_mem_import(struct kbase_context *kctx, + union kbase_ioctl_mem_import *import) +{ + int ret; + u64 flags = import->in.flags; + + ret = kbase_mem_import(kctx, + import->in.type, + u64_to_user_ptr(import->in.phandle), + import->in.padding, + &import->out.gpu_va, + &import->out.va_pages, + &flags); + + import->out.flags = flags; + + return ret; +} + +static int kbase_api_mem_flags_change(struct kbase_context *kctx, + struct kbase_ioctl_mem_flags_change *change) +{ + return kbase_mem_flags_change(kctx, change->gpu_va, + change->flags, change->mask); +} + +static int kbase_api_stream_create(struct kbase_context *kctx, + struct kbase_ioctl_stream_create *stream) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + int fd, ret; + + /* Name must be NULL-terminated and padded with NULLs, so check last + * character is NULL + */ + if (stream->name[sizeof(stream->name)-1] != 0) + return -EINVAL; + + ret = kbase_sync_fence_stream_create(stream->name, &fd); + + if (ret) + return ret; + return fd; +#else + return -ENOENT; +#endif +} + +static int kbase_api_fence_validate(struct kbase_context *kctx, + struct kbase_ioctl_fence_validate *validate) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + return kbase_sync_fence_validate(validate->fd); +#else + return -ENOENT; +#endif +} + +static int kbase_api_get_profiling_controls(struct kbase_context *kctx, + struct kbase_ioctl_get_profiling_controls *controls) +{ + int ret; + + if (controls->count > (FBDUMP_CONTROL_MAX - FBDUMP_CONTROL_MIN)) + return -EINVAL; + + ret = copy_to_user(u64_to_user_ptr(controls->buffer), + &kctx->kbdev->kbase_profiling_controls[ + FBDUMP_CONTROL_MIN], + controls->count * sizeof(u32)); + + if (ret) + return -EFAULT; + return 0; +} + +static int kbase_api_mem_profile_add(struct kbase_context *kctx, + struct kbase_ioctl_mem_profile_add *data) +{ + char *buf; + int err; + + if (data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { + dev_err(kctx->kbdev->dev, "mem_profile_add: buffer too big\n"); + return -EINVAL; + } + + buf = kmalloc(data->len, GFP_KERNEL); + if (ZERO_OR_NULL_PTR(buf)) + return -ENOMEM; + + err = copy_from_user(buf, u64_to_user_ptr(data->buffer), + data->len); + if (err) { + kfree(buf); + return -EFAULT; + } + + return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); +} + +static int kbase_api_soft_event_update(struct kbase_context *kctx, + struct kbase_ioctl_soft_event_update *update) +{ + if (update->flags != 0) + return -EINVAL; + + return kbase_soft_event_update(kctx, update->event, update->new_status); +} + +#if MALI_UNIT_TEST +static int kbase_api_tlstream_test(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_test *test) +{ + kbase_tlstream_test( + test->tpw_count, + test->msg_delay, + test->msg_count, + test->aux_msg); + + return 0; +} + +static int kbase_api_tlstream_stats(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_stats *stats) +{ + kbase_tlstream_stats( + &stats->bytes_collected, + &stats->bytes_generated); + + return 0; +} +#endif /* MALI_UNIT_TEST */ + +#define KBASE_HANDLE_IOCTL(cmd, function) \ + case cmd: \ + do { \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ + return function(kctx); \ + } while (0) + +#define KBASE_HANDLE_IOCTL_IN(cmd, function, type) \ + case cmd: \ + do { \ + type param; \ + int err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return function(kctx, ¶m); \ + } while (0) + +#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type) \ + case cmd: \ + do { \ + type param; \ + int ret, err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + ret = function(kctx, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return ret; \ + } while (0) + +#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type) \ + case cmd: \ + do { \ + type param; \ + int ret, err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + ret = function(kctx, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return ret; \ + } while (0) + +static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct kbase_context *kctx = filp->private_data; + struct kbase_device *kbdev = kctx->kbdev; + void __user *uarg = (void __user *)arg; + + /* The UK ioctl values overflow the cmd field causing the type to be + * incremented + */ + if (_IOC_TYPE(cmd) == LINUX_UK_BASE_MAGIC+2) + return kbase_legacy_ioctl(filp, cmd, arg); + + /* The UK version check IOCTL doesn't overflow the cmd field, so is + * handled separately here + */ + if (cmd == _IOC(_IOC_READ|_IOC_WRITE, LINUX_UK_BASE_MAGIC, + UKP_FUNC_ID_CHECK_VERSION, + sizeof(struct uku_version_check_args))) + return kbase_legacy_ioctl(filp, cmd, arg); + + /* Only these ioctls are available until setup is complete */ + switch (cmd) { + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK, + kbase_api_handshake, + struct kbase_ioctl_version_check); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_FLAGS, + kbase_api_set_flags, + struct kbase_ioctl_set_flags); + } + + /* Block call until version handshake and setup is complete */ + if (kctx->api_version == 0 || !atomic_read(&kctx->setup_complete)) + return -EINVAL; + + /* Normal ioctls */ + switch (cmd) { + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, + kbase_api_job_submit, + struct kbase_ioctl_job_submit); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_GPUPROPS, + kbase_api_get_gpuprops, + struct kbase_ioctl_get_gpuprops); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, + kbase_api_post_term); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, + kbase_api_mem_alloc, + union kbase_ioctl_mem_alloc); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY, + kbase_api_mem_query, + union kbase_ioctl_mem_query); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FREE, + kbase_api_mem_free, + struct kbase_ioctl_mem_free); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_READER_SETUP, + kbase_api_hwcnt_reader_setup, + struct kbase_ioctl_hwcnt_reader_setup); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_ENABLE, + kbase_api_hwcnt_enable, + struct kbase_ioctl_hwcnt_enable); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_DUMP, + kbase_api_hwcnt_dump); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_CLEAR, + kbase_api_hwcnt_clear); + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_DISJOINT_QUERY, + kbase_api_disjoint_query, + struct kbase_ioctl_disjoint_query); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_DDK_VERSION, + kbase_api_get_ddk_version, + struct kbase_ioctl_get_ddk_version); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT, + kbase_api_mem_jit_init, + struct kbase_ioctl_mem_jit_init); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_SYNC, + kbase_api_mem_sync, + struct kbase_ioctl_mem_sync); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_CPU_OFFSET, + kbase_api_mem_find_cpu_offset, + union kbase_ioctl_mem_find_cpu_offset); + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_GET_CONTEXT_ID, + kbase_api_get_context_id, + struct kbase_ioctl_get_context_id); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_ACQUIRE, + kbase_api_tlstream_acquire, + struct kbase_ioctl_tlstream_acquire); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_TLSTREAM_FLUSH, + kbase_api_tlstream_flush); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_COMMIT, + kbase_api_mem_commit, + struct kbase_ioctl_mem_commit); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALIAS, + kbase_api_mem_alias, + union kbase_ioctl_mem_alias); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_IMPORT, + kbase_api_mem_import, + union kbase_ioctl_mem_import); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FLAGS_CHANGE, + kbase_api_mem_flags_change, + struct kbase_ioctl_mem_flags_change); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STREAM_CREATE, + kbase_api_stream_create, + struct kbase_ioctl_stream_create); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_FENCE_VALIDATE, + kbase_api_fence_validate, + struct kbase_ioctl_fence_validate); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_PROFILING_CONTROLS, + kbase_api_get_profiling_controls, + struct kbase_ioctl_get_profiling_controls); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_PROFILE_ADD, + kbase_api_mem_profile_add, + struct kbase_ioctl_mem_profile_add); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, + kbase_api_soft_event_update, + struct kbase_ioctl_soft_event_update); + +#if MALI_UNIT_TEST + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_TEST, + kbase_api_tlstream_test, + struct kbase_ioctl_tlstream_test); + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, + kbase_api_tlstream_stats, + struct kbase_ioctl_tlstream_stats); +#endif + } + + dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); + + return -ENOIOCTLCMD; +} + +static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) +{ + struct kbase_context *kctx = filp->private_data; + struct base_jd_event_v2 uevent; + int out_count = 0; + + if (count < sizeof(uevent)) + return -ENOBUFS; + + do { + while (kbase_event_dequeue(kctx, &uevent)) { + if (out_count > 0) + goto out; + + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; + + if (wait_event_interruptible(kctx->event_queue, + kbase_event_pending(kctx)) != 0) + return -ERESTARTSYS; + } + if (uevent.event_code == BASE_JD_EVENT_DRV_TERMINATED) { + if (out_count == 0) + return -EPIPE; + goto out; + } + + if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) + return -EFAULT; + + buf += sizeof(uevent); + out_count++; + count -= sizeof(uevent); + } while (count >= sizeof(uevent)); + + out: + return out_count * sizeof(uevent); +} + +static unsigned int kbase_poll(struct file *filp, poll_table *wait) +{ + struct kbase_context *kctx = filp->private_data; + + poll_wait(filp, &kctx->event_queue, wait); + if (kbase_event_pending(kctx)) + return POLLIN | POLLRDNORM; + + return 0; +} + +void kbase_event_wakeup(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx); + + wake_up_interruptible(&kctx->event_queue); +} + +KBASE_EXPORT_TEST_API(kbase_event_wakeup); + +static int kbase_check_flags(int flags) +{ + /* Enforce that the driver keeps the O_CLOEXEC flag so that execve() always + * closes the file descriptor in a child process. + */ + if (0 == (flags & O_CLOEXEC)) + return -EINVAL; + + return 0; +} + + +/** + * align_and_check - Align the specified pointer to the provided alignment and + * check that it is still in range. + * @gap_end: Highest possible start address for allocation (end of gap in + * address space) + * @gap_start: Start address of current memory area / gap in address space + * @info: vm_unmapped_area_info structure passed to caller, containing + * alignment, length and limits for the allocation + * @is_shader_code: True if the allocation is for shader code (which has + * additional alignment requirements) + * + * Return: true if gap_end is now aligned correctly and is still in range, + * false otherwise + */ +static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, + struct vm_unmapped_area_info *info, bool is_shader_code) +{ + /* Compute highest gap address at the desired alignment */ + (*gap_end) -= info->length; + (*gap_end) -= (*gap_end - info->align_offset) & info->align_mask; + + if (is_shader_code) { + /* Check for 4GB boundary */ + if (0 == (*gap_end & BASE_MEM_MASK_4GB)) + (*gap_end) -= (info->align_offset ? info->align_offset : + info->length); + if (0 == ((*gap_end + info->length) & BASE_MEM_MASK_4GB)) + (*gap_end) -= (info->align_offset ? info->align_offset : + info->length); + + if (!(*gap_end & BASE_MEM_MASK_4GB) || !((*gap_end + + info->length) & BASE_MEM_MASK_4GB)) + return false; + } + + + if ((*gap_end < info->low_limit) || (*gap_end < gap_start)) + return false; + + + return true; +} + +/* The following function is taken from the kernel and just + * renamed. As it's not exported to modules we must copy-paste it here. + */ + +static unsigned long kbase_unmapped_area_topdown(struct vm_unmapped_area_info + *info, bool is_shader_code) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + unsigned long length, low_limit, high_limit, gap_start, gap_end; + + /* Adjust search length to account for worst case alignment overhead */ + length = info->length + info->align_mask; + if (length < info->length) + return -ENOMEM; + + /* + * Adjust search limits by the desired length. + * See implementation comment at top of unmapped_area(). + */ + gap_end = info->high_limit; + if (gap_end < length) + return -ENOMEM; + high_limit = gap_end - length; + + if (info->low_limit > high_limit) + return -ENOMEM; + low_limit = info->low_limit + length; + + /* Check highest gap, which does not precede any rbtree node */ + gap_start = mm->highest_vm_end; + if (gap_start <= high_limit) { + if (align_and_check(&gap_end, gap_start, info, is_shader_code)) + return gap_end; + } + + /* Check if rbtree root looks promising */ + if (RB_EMPTY_ROOT(&mm->mm_rb)) + return -ENOMEM; + vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); + if (vma->rb_subtree_gap < length) + return -ENOMEM; + + while (true) { + /* Visit right subtree if it looks promising */ + gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0; + if (gap_start <= high_limit && vma->vm_rb.rb_right) { + struct vm_area_struct *right = + rb_entry(vma->vm_rb.rb_right, + struct vm_area_struct, vm_rb); + if (right->rb_subtree_gap >= length) { + vma = right; + continue; + } + } + +check_current: + /* Check if current node has a suitable gap */ + gap_end = vma->vm_start; + if (gap_end < low_limit) + return -ENOMEM; + if (gap_start <= high_limit && gap_end - gap_start >= length) { + /* We found a suitable gap. Clip it with the original + * high_limit. */ + if (gap_end > info->high_limit) + gap_end = info->high_limit; + + if (align_and_check(&gap_end, gap_start, info, + is_shader_code)) + return gap_end; + } + + /* Visit left subtree if it looks promising */ + if (vma->vm_rb.rb_left) { + struct vm_area_struct *left = + rb_entry(vma->vm_rb.rb_left, + struct vm_area_struct, vm_rb); + if (left->rb_subtree_gap >= length) { + vma = left; + continue; + } + } + + /* Go back up the rbtree to find next candidate node */ + while (true) { + struct rb_node *prev = &vma->vm_rb; + if (!rb_parent(prev)) + return -ENOMEM; + vma = rb_entry(rb_parent(prev), + struct vm_area_struct, vm_rb); + if (prev == vma->vm_rb.rb_right) { + gap_start = vma->vm_prev ? + vma->vm_prev->vm_end : 0; + goto check_current; + } + } + } + + return -ENOMEM; +} + +static unsigned long kbase_get_unmapped_area(struct file *filp, + const unsigned long addr, const unsigned long len, + const unsigned long pgoff, const unsigned long flags) +{ + /* based on get_unmapped_area, but simplified slightly due to that some + * values are known in advance */ + struct kbase_context *kctx = filp->private_data; + struct mm_struct *mm = current->mm; + struct vm_unmapped_area_info info; + unsigned long align_offset = 0; + unsigned long align_mask = 0; + unsigned long high_limit = mm->mmap_base; + unsigned long low_limit = PAGE_SIZE; + int cpu_va_bits = BITS_PER_LONG; + int gpu_pc_bits = + kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; + bool is_shader_code = false; + unsigned long ret; + + /* err on fixed address */ + if ((flags & MAP_FIXED) || addr) + return -EINVAL; + +#ifdef CONFIG_64BIT + /* too big? */ + if (len > TASK_SIZE - SZ_2M) + return -ENOMEM; + + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + + if (kbase_hw_has_feature(kctx->kbdev, + BASE_HW_FEATURE_33BIT_VA)) { + high_limit = kctx->same_va_end << PAGE_SHIFT; + } else { + high_limit = min_t(unsigned long, mm->mmap_base, + (kctx->same_va_end << PAGE_SHIFT)); + if (len >= SZ_2M) { + align_offset = SZ_2M; + align_mask = SZ_2M - 1; + } + } + + low_limit = SZ_2M; + } else { + cpu_va_bits = 32; + } +#endif /* CONFIG_64BIT */ + if ((PFN_DOWN(BASE_MEM_COOKIE_BASE) <= pgoff) && + (PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) > pgoff)) { + int cookie = pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); + + if (!kctx->pending_regions[cookie]) + return -EINVAL; + + if (!(kctx->pending_regions[cookie]->flags & + KBASE_REG_GPU_NX)) { + if (cpu_va_bits > gpu_pc_bits) { + align_offset = 1ULL << gpu_pc_bits; + align_mask = align_offset - 1; + is_shader_code = true; + } + } +#ifndef CONFIG_64BIT + } else { + return current->mm->get_unmapped_area(filp, addr, len, pgoff, + flags); +#endif + } + + info.flags = 0; + info.length = len; + info.low_limit = low_limit; + info.high_limit = high_limit; + info.align_offset = align_offset; + info.align_mask = align_mask; + + ret = kbase_unmapped_area_topdown(&info, is_shader_code); + + if (IS_ERR_VALUE(ret) && high_limit == mm->mmap_base && + high_limit < (kctx->same_va_end << PAGE_SHIFT)) { + /* Retry above mmap_base */ + info.low_limit = mm->mmap_base; + info.high_limit = min_t(u64, TASK_SIZE, + (kctx->same_va_end << PAGE_SHIFT)); + + ret = kbase_unmapped_area_topdown(&info, is_shader_code); + } + + return ret; +} + +static const struct file_operations kbase_fops = { + .owner = THIS_MODULE, + .open = kbase_open, + .release = kbase_release, + .read = kbase_read, + .poll = kbase_poll, + .unlocked_ioctl = kbase_ioctl, + .compat_ioctl = kbase_ioctl, + .mmap = kbase_mmap, + .check_flags = kbase_check_flags, + .get_unmapped_area = kbase_get_unmapped_area, +}; + +#ifndef CONFIG_MALI_BIFROST_NO_MALI +void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value) +{ + writel(value, kbdev->reg + offset); +} + +u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset) +{ + return readl(kbdev->reg + offset); +} +#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ + +/** + * show_policy - Show callback for the power_policy sysfs file. + * + * This function is called to get the contents of the power_policy sysfs + * file. This is a list of the available policies with the currently active one + * surrounded by square brackets. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf) +{ + struct kbase_device *kbdev; + const struct kbase_pm_policy *current_policy; + const struct kbase_pm_policy *const *policy_list; + int policy_count; + int i; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + current_policy = kbase_pm_get_policy(kbdev); + + policy_count = kbase_pm_list_policies(&policy_list); + + for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { + if (policy_list[i] == current_policy) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); + else + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); + } + + if (ret < PAGE_SIZE - 1) { + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); + } else { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/** + * set_policy - Store callback for the power_policy sysfs file. + * + * This function is called when the power_policy sysfs file is written to. + * It matches the requested policy against the available policies and if a + * matching policy is found calls kbase_pm_set_policy() to change the + * policy. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + const struct kbase_pm_policy *new_policy = NULL; + const struct kbase_pm_policy *const *policy_list; + int policy_count; + int i; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + policy_count = kbase_pm_list_policies(&policy_list); + + for (i = 0; i < policy_count; i++) { + if (sysfs_streq(policy_list[i]->name, buf)) { + new_policy = policy_list[i]; + break; + } + } + + if (!new_policy) { + dev_err(dev, "power_policy: policy not found\n"); + return -EINVAL; + } + + kbase_pm_set_policy(kbdev, new_policy); + + return count; +} + +/* + * The sysfs file power_policy. + * + * This is used for obtaining information about the available policies, + * determining which policy is currently active, and changing the active + * policy. + */ +static DEVICE_ATTR(power_policy, S_IRUGO | S_IWUSR, show_policy, set_policy); + +/** + * show_ca_policy - Show callback for the core_availability_policy sysfs file. + * + * This function is called to get the contents of the core_availability_policy + * sysfs file. This is a list of the available policies with the currently + * active one surrounded by square brackets. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_ca_policy(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + const struct kbase_pm_ca_policy *current_policy; + const struct kbase_pm_ca_policy *const *policy_list; + int policy_count; + int i; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + current_policy = kbase_pm_ca_get_policy(kbdev); + + policy_count = kbase_pm_ca_list_policies(&policy_list); + + for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { + if (policy_list[i] == current_policy) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); + else + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); + } + + if (ret < PAGE_SIZE - 1) { + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); + } else { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/** + * set_ca_policy - Store callback for the core_availability_policy sysfs file. + * + * This function is called when the core_availability_policy sysfs file is + * written to. It matches the requested policy against the available policies + * and if a matching policy is found calls kbase_pm_set_policy() to change + * the policy. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_ca_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + const struct kbase_pm_ca_policy *new_policy = NULL; + const struct kbase_pm_ca_policy *const *policy_list; + int policy_count; + int i; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + policy_count = kbase_pm_ca_list_policies(&policy_list); + + for (i = 0; i < policy_count; i++) { + if (sysfs_streq(policy_list[i]->name, buf)) { + new_policy = policy_list[i]; + break; + } + } + + if (!new_policy) { + dev_err(dev, "core_availability_policy: policy not found\n"); + return -EINVAL; + } + + kbase_pm_ca_set_policy(kbdev, new_policy); + + return count; +} + +/* + * The sysfs file core_availability_policy + * + * This is used for obtaining information about the available policies, + * determining which policy is currently active, and changing the active + * policy. + */ +static DEVICE_ATTR(core_availability_policy, S_IRUGO | S_IWUSR, show_ca_policy, set_ca_policy); + +/* + * show_core_mask - Show callback for the core_mask sysfs file. + * + * This function is called to get the contents of the core_mask sysfs file. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Current core mask (JS0) : 0x%llX\n", + kbdev->pm.debug_core_mask[0]); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Current core mask (JS1) : 0x%llX\n", + kbdev->pm.debug_core_mask[1]); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Current core mask (JS2) : 0x%llX\n", + kbdev->pm.debug_core_mask[2]); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Available core mask : 0x%llX\n", + kbdev->gpu_props.props.raw_props.shader_present); + + return ret; +} + +/** + * set_core_mask - Store callback for the core_mask sysfs file. + * + * This function is called when the core_mask sysfs file is written to. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + u64 new_core_mask[3]; + int items; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + items = sscanf(buf, "%llx %llx %llx", + &new_core_mask[0], &new_core_mask[1], + &new_core_mask[2]); + + if (items == 1) + new_core_mask[1] = new_core_mask[2] = new_core_mask[0]; + + if (items == 1 || items == 3) { + u64 shader_present = + kbdev->gpu_props.props.raw_props.shader_present; + u64 group0_core_mask = + kbdev->gpu_props.props.coherency_info.group[0]. + core_mask; + + if ((new_core_mask[0] & shader_present) != new_core_mask[0] || + !(new_core_mask[0] & group0_core_mask) || + (new_core_mask[1] & shader_present) != + new_core_mask[1] || + !(new_core_mask[1] & group0_core_mask) || + (new_core_mask[2] & shader_present) != + new_core_mask[2] || + !(new_core_mask[2] & group0_core_mask)) { + dev_err(dev, "power_policy: invalid core specification\n"); + return -EINVAL; + } + + if (kbdev->pm.debug_core_mask[0] != new_core_mask[0] || + kbdev->pm.debug_core_mask[1] != + new_core_mask[1] || + kbdev->pm.debug_core_mask[2] != + new_core_mask[2]) { + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_pm_set_debug_core_mask(kbdev, new_core_mask[0], + new_core_mask[1], new_core_mask[2]); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + return count; + } + + dev_err(kbdev->dev, "Couldn't process set_core_mask write operation.\n" + "Use format \n" + "or \n"); + return -EINVAL; +} + +/* + * The sysfs file core_mask. + * + * This is used to restrict shader core availability for debugging purposes. + * Reading it will show the current core mask and the mask of cores available. + * Writing to it will set the current core mask. + */ +static DEVICE_ATTR(core_mask, S_IRUGO | S_IWUSR, show_core_mask, set_core_mask); + +/** + * set_soft_job_timeout - Store callback for the soft_job_timeout sysfs + * file. + * + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The value written to the sysfs file. + * @count: The number of bytes written to the sysfs file. + * + * This allows setting the timeout for software jobs. Waiting soft event wait + * jobs will be cancelled after this period expires, while soft fence wait jobs + * will print debug information if the fence debug feature is enabled. + * + * This is expressed in milliseconds. + * + * Return: count if the function succeeded. An error code on failure. + */ +static ssize_t set_soft_job_timeout(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int soft_job_timeout_ms; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + if ((kstrtoint(buf, 0, &soft_job_timeout_ms) != 0) || + (soft_job_timeout_ms <= 0)) + return -EINVAL; + + atomic_set(&kbdev->js_data.soft_job_timeout_ms, + soft_job_timeout_ms); + + return count; +} + +/** + * show_soft_job_timeout - Show callback for the soft_job_timeout sysfs + * file. + * + * This will return the timeout for the software jobs. + * + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer for the sysfs file contents. + * + * Return: The number of bytes output to buf. + */ +static ssize_t show_soft_job_timeout(struct device *dev, + struct device_attribute *attr, + char * const buf) +{ + struct kbase_device *kbdev; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + return scnprintf(buf, PAGE_SIZE, "%i\n", + atomic_read(&kbdev->js_data.soft_job_timeout_ms)); +} + +static DEVICE_ATTR(soft_job_timeout, S_IRUGO | S_IWUSR, + show_soft_job_timeout, set_soft_job_timeout); + +static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, + int default_ticks, u32 old_ticks) +{ + if (timeout_ms > 0) { + u64 ticks = timeout_ms * 1000000ULL; + do_div(ticks, kbdev->js_data.scheduling_period_ns); + if (!ticks) + return 1; + return ticks; + } else if (timeout_ms < 0) { + return default_ticks; + } else { + return old_ticks; + } +} + +/** + * set_js_timeouts - Store callback for the js_timeouts sysfs file. + * + * This function is called to get the contents of the js_timeouts sysfs + * file. This file contains five values separated by whitespace. The values + * are basically the same as %JS_SOFT_STOP_TICKS, %JS_HARD_STOP_TICKS_SS, + * %JS_HARD_STOP_TICKS_DUMPING, %JS_RESET_TICKS_SS, %JS_RESET_TICKS_DUMPING + * configuration values (in that order), with the difference that the js_timeout + * values are expressed in MILLISECONDS. + * + * The js_timeouts sysfile file allows the current values in + * use by the job scheduler to get override. Note that a value needs to + * be other than 0 for it to override the current job scheduler value. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_js_timeouts(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int items; + long js_soft_stop_ms; + long js_soft_stop_ms_cl; + long js_hard_stop_ms_ss; + long js_hard_stop_ms_cl; + long js_hard_stop_ms_dumping; + long js_reset_ms_ss; + long js_reset_ms_cl; + long js_reset_ms_dumping; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + items = sscanf(buf, "%ld %ld %ld %ld %ld %ld %ld %ld", + &js_soft_stop_ms, &js_soft_stop_ms_cl, + &js_hard_stop_ms_ss, &js_hard_stop_ms_cl, + &js_hard_stop_ms_dumping, &js_reset_ms_ss, + &js_reset_ms_cl, &js_reset_ms_dumping); + + if (items == 8) { + struct kbasep_js_device_data *js_data = &kbdev->js_data; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + +#define UPDATE_TIMEOUT(ticks_name, ms_name, default) do {\ + js_data->ticks_name = timeout_ms_to_ticks(kbdev, ms_name, \ + default, js_data->ticks_name); \ + dev_dbg(kbdev->dev, "Overriding " #ticks_name \ + " with %lu ticks (%lu ms)\n", \ + (unsigned long)js_data->ticks_name, \ + ms_name); \ + } while (0) + + UPDATE_TIMEOUT(soft_stop_ticks, js_soft_stop_ms, + DEFAULT_JS_SOFT_STOP_TICKS); + UPDATE_TIMEOUT(soft_stop_ticks_cl, js_soft_stop_ms_cl, + DEFAULT_JS_SOFT_STOP_TICKS_CL); + UPDATE_TIMEOUT(hard_stop_ticks_ss, js_hard_stop_ms_ss, + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? + DEFAULT_JS_HARD_STOP_TICKS_SS_8408 : + DEFAULT_JS_HARD_STOP_TICKS_SS); + UPDATE_TIMEOUT(hard_stop_ticks_cl, js_hard_stop_ms_cl, + DEFAULT_JS_HARD_STOP_TICKS_CL); + UPDATE_TIMEOUT(hard_stop_ticks_dumping, + js_hard_stop_ms_dumping, + DEFAULT_JS_HARD_STOP_TICKS_DUMPING); + UPDATE_TIMEOUT(gpu_reset_ticks_ss, js_reset_ms_ss, + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? + DEFAULT_JS_RESET_TICKS_SS_8408 : + DEFAULT_JS_RESET_TICKS_SS); + UPDATE_TIMEOUT(gpu_reset_ticks_cl, js_reset_ms_cl, + DEFAULT_JS_RESET_TICKS_CL); + UPDATE_TIMEOUT(gpu_reset_ticks_dumping, js_reset_ms_dumping, + DEFAULT_JS_RESET_TICKS_DUMPING); + + kbase_js_set_timeouts(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return count; + } + + dev_err(kbdev->dev, "Couldn't process js_timeouts write operation.\n" + "Use format \n" + "Write 0 for no change, -1 to restore default timeout\n"); + return -EINVAL; +} + +static unsigned long get_js_timeout_in_ms( + u32 scheduling_period_ns, + u32 ticks) +{ + u64 ms = (u64)ticks * scheduling_period_ns; + + do_div(ms, 1000000UL); + return ms; +} + +/** + * show_js_timeouts - Show callback for the js_timeouts sysfs file. + * + * This function is called to get the contents of the js_timeouts sysfs + * file. It returns the last set values written to the js_timeouts sysfs file. + * If the file didn't get written yet, the values will be current setting in + * use. + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_js_timeouts(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + unsigned long js_soft_stop_ms; + unsigned long js_soft_stop_ms_cl; + unsigned long js_hard_stop_ms_ss; + unsigned long js_hard_stop_ms_cl; + unsigned long js_hard_stop_ms_dumping; + unsigned long js_reset_ms_ss; + unsigned long js_reset_ms_cl; + unsigned long js_reset_ms_dumping; + u32 scheduling_period_ns; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + scheduling_period_ns = kbdev->js_data.scheduling_period_ns; + +#define GET_TIMEOUT(name) get_js_timeout_in_ms(\ + scheduling_period_ns, \ + kbdev->js_data.name) + + js_soft_stop_ms = GET_TIMEOUT(soft_stop_ticks); + js_soft_stop_ms_cl = GET_TIMEOUT(soft_stop_ticks_cl); + js_hard_stop_ms_ss = GET_TIMEOUT(hard_stop_ticks_ss); + js_hard_stop_ms_cl = GET_TIMEOUT(hard_stop_ticks_cl); + js_hard_stop_ms_dumping = GET_TIMEOUT(hard_stop_ticks_dumping); + js_reset_ms_ss = GET_TIMEOUT(gpu_reset_ticks_ss); + js_reset_ms_cl = GET_TIMEOUT(gpu_reset_ticks_cl); + js_reset_ms_dumping = GET_TIMEOUT(gpu_reset_ticks_dumping); + +#undef GET_TIMEOUT + + ret = scnprintf(buf, PAGE_SIZE, "%lu %lu %lu %lu %lu %lu %lu %lu\n", + js_soft_stop_ms, js_soft_stop_ms_cl, + js_hard_stop_ms_ss, js_hard_stop_ms_cl, + js_hard_stop_ms_dumping, js_reset_ms_ss, + js_reset_ms_cl, js_reset_ms_dumping); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/* + * The sysfs file js_timeouts. + * + * This is used to override the current job scheduler values for + * JS_STOP_STOP_TICKS_SS + * JS_STOP_STOP_TICKS_CL + * JS_HARD_STOP_TICKS_SS + * JS_HARD_STOP_TICKS_CL + * JS_HARD_STOP_TICKS_DUMPING + * JS_RESET_TICKS_SS + * JS_RESET_TICKS_CL + * JS_RESET_TICKS_DUMPING. + */ +static DEVICE_ATTR(js_timeouts, S_IRUGO | S_IWUSR, show_js_timeouts, set_js_timeouts); + +static u32 get_new_js_timeout( + u32 old_period, + u32 old_ticks, + u32 new_scheduling_period_ns) +{ + u64 ticks = (u64)old_period * (u64)old_ticks; + do_div(ticks, new_scheduling_period_ns); + return ticks?ticks:1; +} + +/** + * set_js_scheduling_period - Store callback for the js_scheduling_period sysfs + * file + * @dev: The device the sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the js_scheduling_period sysfs file is written + * to. It checks the data written, and if valid updates the js_scheduling_period + * value + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_js_scheduling_period(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + unsigned int js_scheduling_period; + u32 new_scheduling_period_ns; + u32 old_period; + struct kbasep_js_device_data *js_data; + unsigned long flags; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + js_data = &kbdev->js_data; + + ret = kstrtouint(buf, 0, &js_scheduling_period); + if (ret || !js_scheduling_period) { + dev_err(kbdev->dev, "Couldn't process js_scheduling_period write operation.\n" + "Use format \n"); + return -EINVAL; + } + + new_scheduling_period_ns = js_scheduling_period * 1000000; + + /* Update scheduling timeouts */ + mutex_lock(&js_data->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* If no contexts have been scheduled since js_timeouts was last written + * to, the new timeouts might not have been latched yet. So check if an + * update is pending and use the new values if necessary. */ + + /* Use previous 'new' scheduling period as a base if present. */ + old_period = js_data->scheduling_period_ns; + +#define SET_TIMEOUT(name) \ + (js_data->name = get_new_js_timeout(\ + old_period, \ + kbdev->js_data.name, \ + new_scheduling_period_ns)) + + SET_TIMEOUT(soft_stop_ticks); + SET_TIMEOUT(soft_stop_ticks_cl); + SET_TIMEOUT(hard_stop_ticks_ss); + SET_TIMEOUT(hard_stop_ticks_cl); + SET_TIMEOUT(hard_stop_ticks_dumping); + SET_TIMEOUT(gpu_reset_ticks_ss); + SET_TIMEOUT(gpu_reset_ticks_cl); + SET_TIMEOUT(gpu_reset_ticks_dumping); + +#undef SET_TIMEOUT + + js_data->scheduling_period_ns = new_scheduling_period_ns; + + kbase_js_set_timeouts(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_data->runpool_mutex); + + dev_dbg(kbdev->dev, "JS scheduling period: %dms\n", + js_scheduling_period); + + return count; +} + +/** + * show_js_scheduling_period - Show callback for the js_scheduling_period sysfs + * entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current period used for the JS scheduling + * period. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_js_scheduling_period(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + u32 period; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + period = kbdev->js_data.scheduling_period_ns; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", + period / 1000000); + + return ret; +} + +static DEVICE_ATTR(js_scheduling_period, S_IRUGO | S_IWUSR, + show_js_scheduling_period, set_js_scheduling_period); + +#if !MALI_CUSTOMER_RELEASE +/** + * set_force_replay - Store callback for the force_replay sysfs file. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_force_replay(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + if (!strncmp("limit=", buf, MIN(6, count))) { + int force_replay_limit; + int items = sscanf(buf, "limit=%u", &force_replay_limit); + + if (items == 1) { + kbdev->force_replay_random = false; + kbdev->force_replay_limit = force_replay_limit; + kbdev->force_replay_count = 0; + + return count; + } + } else if (!strncmp("random_limit", buf, MIN(12, count))) { + kbdev->force_replay_random = true; + kbdev->force_replay_count = 0; + + return count; + } else if (!strncmp("norandom_limit", buf, MIN(14, count))) { + kbdev->force_replay_random = false; + kbdev->force_replay_limit = KBASEP_FORCE_REPLAY_DISABLED; + kbdev->force_replay_count = 0; + + return count; + } else if (!strncmp("core_req=", buf, MIN(9, count))) { + unsigned int core_req; + int items = sscanf(buf, "core_req=%x", &core_req); + + if (items == 1) { + kbdev->force_replay_core_req = (base_jd_core_req)core_req; + + return count; + } + } + dev_err(kbdev->dev, "Couldn't process force_replay write operation.\nPossible settings: limit=, random_limit, norandom_limit, core_req=\n"); + return -EINVAL; +} + +/** + * show_force_replay - Show callback for the force_replay sysfs file. + * + * This function is called to get the contents of the force_replay sysfs + * file. It returns the last set value written to the force_replay sysfs file. + * If the file didn't get written yet, the values will be 0. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_force_replay(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + if (kbdev->force_replay_random) + ret = scnprintf(buf, PAGE_SIZE, + "limit=0\nrandom_limit\ncore_req=%x\n", + kbdev->force_replay_core_req); + else + ret = scnprintf(buf, PAGE_SIZE, + "limit=%u\nnorandom_limit\ncore_req=%x\n", + kbdev->force_replay_limit, + kbdev->force_replay_core_req); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/* + * The sysfs file force_replay. + */ +static DEVICE_ATTR(force_replay, S_IRUGO | S_IWUSR, show_force_replay, + set_force_replay); +#endif /* !MALI_CUSTOMER_RELEASE */ + +#ifdef CONFIG_MALI_BIFROST_DEBUG +static ssize_t set_js_softstop_always(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + int softstop_always; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtoint(buf, 0, &softstop_always); + if (ret || ((softstop_always != 0) && (softstop_always != 1))) { + dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->js_data.softstop_always = (bool) softstop_always; + dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", + (kbdev->js_data.softstop_always) ? + "Enabled" : "Disabled"); + return count; +} + +static ssize_t show_js_softstop_always(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->js_data.softstop_always); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/* + * By default, soft-stops are disabled when only a single context is present. + * The ability to enable soft-stop when only a single context is present can be + * used for debug and unit-testing purposes. + * (see CL t6xx_stress_1 unit-test as an example whereby this feature is used.) + */ +static DEVICE_ATTR(js_softstop_always, S_IRUGO | S_IWUSR, show_js_softstop_always, set_js_softstop_always); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + +#ifdef CONFIG_MALI_BIFROST_DEBUG +typedef void (kbasep_debug_command_func) (struct kbase_device *); + +enum kbasep_debug_command_code { + KBASEP_DEBUG_COMMAND_DUMPTRACE, + + /* This must be the last enum */ + KBASEP_DEBUG_COMMAND_COUNT +}; + +struct kbasep_debug_command { + char *str; + kbasep_debug_command_func *func; +}; + +/* Debug commands supported by the driver */ +static const struct kbasep_debug_command debug_commands[] = { + { + .str = "dumptrace", + .func = &kbasep_trace_dump, + } +}; + +/** + * show_debug - Show callback for the debug_command sysfs file. + * + * This function is called to get the contents of the debug_command sysfs + * file. This is a list of the available debug commands, separated by newlines. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_debug(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + int i; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT && ret < PAGE_SIZE; i++) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s\n", debug_commands[i].str); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/** + * issue_debug - Store callback for the debug_command sysfs file. + * + * This function is called when the debug_command sysfs file is written to. + * It matches the requested command against the available commands, and if + * a matching command is found calls the associated function from + * @debug_commands to issue the command. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t issue_debug(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int i; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT; i++) { + if (sysfs_streq(debug_commands[i].str, buf)) { + debug_commands[i].func(kbdev); + return count; + } + } + + /* Debug Command not found */ + dev_err(dev, "debug_command: command not known\n"); + return -EINVAL; +} + +/* The sysfs file debug_command. + * + * This is used to issue general debug commands to the device driver. + * Reading it will produce a list of debug commands, separated by newlines. + * Writing to it with one of those commands will issue said command. + */ +static DEVICE_ATTR(debug_command, S_IRUGO | S_IWUSR, show_debug, issue_debug); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + +/** + * kbase_show_gpuinfo - Show callback for the gpuinfo sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get a description of the present Mali + * GPU via the gpuinfo sysfs entry. This includes the GPU family, the + * number of cores, the hardware version and the raw product id. For + * example + * + * Mali-T60x MP4 r0p0 0x6956 + * + * Return: The number of bytes output to @buf. + */ +static ssize_t kbase_show_gpuinfo(struct device *dev, + struct device_attribute *attr, char *buf) +{ + static const struct gpu_product_id_name { + unsigned id; + char *name; + } gpu_product_id_names[] = { + { .id = GPU_ID_PI_T60X, .name = "Mali-T60x" }, + { .id = GPU_ID_PI_T62X, .name = "Mali-T62x" }, + { .id = GPU_ID_PI_T72X, .name = "Mali-T72x" }, + { .id = GPU_ID_PI_T76X, .name = "Mali-T76x" }, + { .id = GPU_ID_PI_T82X, .name = "Mali-T82x" }, + { .id = GPU_ID_PI_T83X, .name = "Mali-T83x" }, + { .id = GPU_ID_PI_T86X, .name = "Mali-T86x" }, + { .id = GPU_ID_PI_TFRX, .name = "Mali-T88x" }, + { .id = GPU_ID2_PRODUCT_TMIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G71" }, + { .id = GPU_ID2_PRODUCT_THEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G72" }, + { .id = GPU_ID2_PRODUCT_TSIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G51" }, + { .id = GPU_ID2_PRODUCT_TDVX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G31" }, + }; + const char *product_name = "(Unknown Mali GPU)"; + struct kbase_device *kbdev; + u32 gpu_id; + unsigned product_id, product_id_mask; + unsigned i; + bool is_new_format; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + product_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; + is_new_format = GPU_ID_IS_NEW_FORMAT(product_id); + product_id_mask = + (is_new_format ? + GPU_ID2_PRODUCT_MODEL : + GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + for (i = 0; i < ARRAY_SIZE(gpu_product_id_names); ++i) { + const struct gpu_product_id_name *p = &gpu_product_id_names[i]; + + if ((GPU_ID_IS_NEW_FORMAT(p->id) == is_new_format) && + (p->id & product_id_mask) == + (product_id & product_id_mask)) { + product_name = p->name; + break; + } + } + + return scnprintf(buf, PAGE_SIZE, "%s %d cores 2EE r%dp%d 0x%04X\n", + product_name, kbdev->gpu_props.num_cores, + (gpu_id & GPU_ID_VERSION_MAJOR) >> GPU_ID_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID_VERSION_MINOR) >> GPU_ID_VERSION_MINOR_SHIFT, + product_id); +} +static DEVICE_ATTR(gpuinfo, S_IRUGO, kbase_show_gpuinfo, NULL); + +/** + * set_dvfs_period - Store callback for the dvfs_period sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the dvfs_period sysfs file is written to. It + * checks the data written, and if valid updates the DVFS period variable, + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_dvfs_period(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + int dvfs_period; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtoint(buf, 0, &dvfs_period); + if (ret || dvfs_period <= 0) { + dev_err(kbdev->dev, "Couldn't process dvfs_period write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->pm.dvfs_period = dvfs_period; + dev_dbg(kbdev->dev, "DVFS period: %dms\n", dvfs_period); + + return count; +} + +/** + * show_dvfs_period - Show callback for the dvfs_period sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current period used for the DVFS sample + * timer. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_dvfs_period(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->pm.dvfs_period); + + return ret; +} + +static DEVICE_ATTR(dvfs_period, S_IRUGO | S_IWUSR, show_dvfs_period, + set_dvfs_period); + +/** + * set_pm_poweroff - Store callback for the pm_poweroff sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the pm_poweroff sysfs file is written to. + * + * This file contains three values separated by whitespace. The values + * are gpu_poweroff_time (the period of the poweroff timer, in ns), + * poweroff_shader_ticks (the number of poweroff timer ticks before an idle + * shader is powered off), and poweroff_gpu_ticks (the number of poweroff timer + * ticks before the GPU is powered off), in that order. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_pm_poweroff(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int items; + s64 gpu_poweroff_time; + int poweroff_shader_ticks, poweroff_gpu_ticks; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + items = sscanf(buf, "%llu %u %u", &gpu_poweroff_time, + &poweroff_shader_ticks, + &poweroff_gpu_ticks); + if (items != 3) { + dev_err(kbdev->dev, "Couldn't process pm_poweroff write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->pm.gpu_poweroff_time = HR_TIMER_DELAY_NSEC(gpu_poweroff_time); + kbdev->pm.poweroff_shader_ticks = poweroff_shader_ticks; + kbdev->pm.poweroff_gpu_ticks = poweroff_gpu_ticks; + + return count; +} + +/** + * show_pm_poweroff - Show callback for the pm_poweroff sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current period used for the DVFS sample + * timer. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_pm_poweroff(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%llu %u %u\n", + ktime_to_ns(kbdev->pm.gpu_poweroff_time), + kbdev->pm.poweroff_shader_ticks, + kbdev->pm.poweroff_gpu_ticks); + + return ret; +} + +static DEVICE_ATTR(pm_poweroff, S_IRUGO | S_IWUSR, show_pm_poweroff, + set_pm_poweroff); + +/** + * set_reset_timeout - Store callback for the reset_timeout sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the reset_timeout sysfs file is written to. It + * checks the data written, and if valid updates the reset timeout. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_reset_timeout(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + int reset_timeout; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtoint(buf, 0, &reset_timeout); + if (ret || reset_timeout <= 0) { + dev_err(kbdev->dev, "Couldn't process reset_timeout write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->reset_timeout_ms = reset_timeout; + dev_dbg(kbdev->dev, "Reset timeout: %dms\n", reset_timeout); + + return count; +} + +/** + * show_reset_timeout - Show callback for the reset_timeout sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current reset timeout. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_reset_timeout(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->reset_timeout_ms); + + return ret; +} + +static DEVICE_ATTR(reset_timeout, S_IRUGO | S_IWUSR, show_reset_timeout, + set_reset_timeout); + + + +static ssize_t show_mem_pool_size(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%zu\n", + kbase_mem_pool_size(&kbdev->mem_pool)); + + return ret; +} + +static ssize_t set_mem_pool_size(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + size_t new_size; + int err; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + err = kstrtoul(buf, 0, (unsigned long *)&new_size); + if (err) + return err; + + kbase_mem_pool_trim(&kbdev->mem_pool, new_size); + + return count; +} + +static DEVICE_ATTR(mem_pool_size, S_IRUGO | S_IWUSR, show_mem_pool_size, + set_mem_pool_size); + +static ssize_t show_mem_pool_max_size(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%zu\n", + kbase_mem_pool_max_size(&kbdev->mem_pool)); + + return ret; +} + +static ssize_t set_mem_pool_max_size(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + size_t new_max_size; + int err; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + err = kstrtoul(buf, 0, (unsigned long *)&new_max_size); + if (err) + return -EINVAL; + + kbase_mem_pool_set_max_size(&kbdev->mem_pool, new_max_size); + + return count; +} + +static DEVICE_ATTR(mem_pool_max_size, S_IRUGO | S_IWUSR, show_mem_pool_max_size, + set_mem_pool_max_size); + +/** + * show_lp_mem_pool_size - Show size of the large memory pages pool. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the pool size. + * + * This function is called to get the number of large memory pages which currently populate the kbdev pool. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_lp_mem_pool_size(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + return scnprintf(buf, PAGE_SIZE, "%zu\n", kbase_mem_pool_size(&kbdev->lp_mem_pool)); +} + +/** + * set_lp_mem_pool_size - Set size of the large memory pages pool. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The value written to the sysfs file. + * @count: The number of bytes written to the sysfs file. + * + * This function is called to set the number of large memory pages which should populate the kbdev pool. + * This may cause existing pages to be removed from the pool, or new pages to be created and then added to the pool. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_lp_mem_pool_size(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + unsigned long new_size; + int err; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + err = kstrtoul(buf, 0, &new_size); + if (err) + return err; + + kbase_mem_pool_trim(&kbdev->lp_mem_pool, new_size); + + return count; +} + +static DEVICE_ATTR(lp_mem_pool_size, S_IRUGO | S_IWUSR, show_lp_mem_pool_size, + set_lp_mem_pool_size); + +/** + * show_lp_mem_pool_max_size - Show maximum size of the large memory pages pool. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the pool size. + * + * This function is called to get the maximum number of large memory pages that the kbdev pool can possibly contain. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_lp_mem_pool_max_size(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + return scnprintf(buf, PAGE_SIZE, "%zu\n", kbase_mem_pool_max_size(&kbdev->lp_mem_pool)); +} + +/** + * set_lp_mem_pool_max_size - Set maximum size of the large memory pages pool. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The value written to the sysfs file. + * @count: The number of bytes written to the sysfs file. + * + * This function is called to set the maximum number of large memory pages that the kbdev pool can possibly contain. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_lp_mem_pool_max_size(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + unsigned long new_max_size; + int err; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + err = kstrtoul(buf, 0, &new_max_size); + if (err) + return -EINVAL; + + kbase_mem_pool_set_max_size(&kbdev->lp_mem_pool, new_max_size); + + return count; +} + +static DEVICE_ATTR(lp_mem_pool_max_size, S_IRUGO | S_IWUSR, show_lp_mem_pool_max_size, + set_lp_mem_pool_max_size); + +#ifdef CONFIG_DEBUG_FS + +/* Number of entries in serialize_jobs_settings[] */ +#define NR_SERIALIZE_JOBS_SETTINGS 5 +/* Maximum string length in serialize_jobs_settings[].name */ +#define MAX_SERIALIZE_JOBS_NAME_LEN 16 + +static struct +{ + char *name; + u8 setting; +} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { + {"none", 0}, + {"intra-slot", KBASE_SERIALIZE_INTRA_SLOT}, + {"inter-slot", KBASE_SERIALIZE_INTER_SLOT}, + {"full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT}, + {"full-reset", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | + KBASE_SERIALIZE_RESET} +}; + +/** + * kbasep_serialize_jobs_seq_show - Show callback for the serialize_jobs debugfs + * file + * @sfile: seq_file pointer + * @data: Private callback data + * + * This function is called to get the contents of the serialize_jobs debugfs + * file. This is a list of the available settings with the currently active one + * surrounded by square brackets. + * + * Return: 0 on success, or an error code on error + */ +static int kbasep_serialize_jobs_seq_show(struct seq_file *sfile, void *data) +{ + struct kbase_device *kbdev = sfile->private; + int i; + + CSTD_UNUSED(data); + + for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { + if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) + seq_printf(sfile, "[%s] ", + serialize_jobs_settings[i].name); + else + seq_printf(sfile, "%s ", + serialize_jobs_settings[i].name); + } + + seq_puts(sfile, "\n"); + + return 0; +} + +/** + * kbasep_serialize_jobs_debugfs_write - Store callback for the serialize_jobs + * debugfs file. + * @file: File pointer + * @ubuf: User buffer containing data to store + * @count: Number of bytes in user buffer + * @ppos: File position + * + * This function is called when the serialize_jobs debugfs file is written to. + * It matches the requested setting against the available settings and if a + * matching setting is found updates kbdev->serialize_jobs. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct kbase_device *kbdev = s->private; + char buf[MAX_SERIALIZE_JOBS_NAME_LEN]; + int i; + bool valid = false; + + CSTD_UNUSED(ppos); + + count = min_t(size_t, sizeof(buf) - 1, count); + if (copy_from_user(buf, ubuf, count)) + return -EFAULT; + + buf[count] = 0; + + for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { + if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { + kbdev->serialize_jobs = + serialize_jobs_settings[i].setting; + valid = true; + break; + } + } + + if (!valid) { + dev_err(kbdev->dev, "serialize_jobs: invalid setting\n"); + return -EINVAL; + } + + return count; +} + +/** + * kbasep_serialize_jobs_debugfs_open - Open callback for the serialize_jobs + * debugfs file + * @in: inode pointer + * @file: file pointer + * + * Return: Zero on success, error code on failure + */ +static int kbasep_serialize_jobs_debugfs_open(struct inode *in, + struct file *file) +{ + return single_open(file, kbasep_serialize_jobs_seq_show, in->i_private); +} + +static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { + .open = kbasep_serialize_jobs_debugfs_open, + .read = seq_read, + .write = kbasep_serialize_jobs_debugfs_write, + .llseek = seq_lseek, + .release = single_release, +}; + +#endif /* CONFIG_DEBUG_FS */ + +static int kbasep_protected_mode_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_OF + struct device_node *protected_node; + struct platform_device *pdev; + struct protected_mode_device *protected_dev; +#endif + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { + /* Use native protected ops */ + kbdev->protected_dev = kzalloc(sizeof(*kbdev->protected_dev), + GFP_KERNEL); + if (!kbdev->protected_dev) + return -ENOMEM; + kbdev->protected_dev->data = kbdev; + kbdev->protected_ops = &kbase_native_protected_ops; + kbdev->protected_mode_support = true; + return 0; + } + + kbdev->protected_mode_support = false; + +#ifdef CONFIG_OF + protected_node = of_parse_phandle(kbdev->dev->of_node, + "protected-mode-switcher", 0); + + if (!protected_node) + protected_node = of_parse_phandle(kbdev->dev->of_node, + "secure-mode-switcher", 0); + + if (!protected_node) { + /* If protected_node cannot be looked up then we assume + * protected mode is not supported on this platform. */ + dev_info(kbdev->dev, "Protected mode not available\n"); + return 0; + } + + pdev = of_find_device_by_node(protected_node); + if (!pdev) + return -EINVAL; + + protected_dev = platform_get_drvdata(pdev); + if (!protected_dev) + return -EPROBE_DEFER; + + kbdev->protected_ops = &protected_dev->ops; + kbdev->protected_dev = protected_dev; + + if (kbdev->protected_ops) { + int err; + + /* Make sure protected mode is disabled on startup */ + mutex_lock(&kbdev->pm.lock); + err = kbdev->protected_ops->protected_mode_disable( + kbdev->protected_dev); + mutex_unlock(&kbdev->pm.lock); + + /* protected_mode_disable() returns -EINVAL if not supported */ + kbdev->protected_mode_support = (err != -EINVAL); + } +#endif + return 0; +} + +static void kbasep_protected_mode_term(struct kbase_device *kbdev) +{ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) + kfree(kbdev->protected_dev); +} + +#ifdef CONFIG_MALI_BIFROST_NO_MALI +static int kbase_common_reg_map(struct kbase_device *kbdev) +{ + return 0; +} +static void kbase_common_reg_unmap(struct kbase_device * const kbdev) +{ +} +#else /* CONFIG_MALI_BIFROST_NO_MALI */ +static int kbase_common_reg_map(struct kbase_device *kbdev) +{ + int err = 0; + + if (!request_mem_region(kbdev->reg_start, kbdev->reg_size, dev_name(kbdev->dev))) { + dev_err(kbdev->dev, "Register window unavailable\n"); + err = -EIO; + goto out_region; + } + + kbdev->reg = ioremap(kbdev->reg_start, kbdev->reg_size); + if (!kbdev->reg) { + dev_err(kbdev->dev, "Can't remap register window\n"); + err = -EINVAL; + goto out_ioremap; + } + + return err; + + out_ioremap: + release_mem_region(kbdev->reg_start, kbdev->reg_size); + out_region: + return err; +} + +static void kbase_common_reg_unmap(struct kbase_device * const kbdev) +{ + if (kbdev->reg) { + iounmap(kbdev->reg); + release_mem_region(kbdev->reg_start, kbdev->reg_size); + kbdev->reg = NULL; + kbdev->reg_start = 0; + kbdev->reg_size = 0; + } +} +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ + +static int registers_map(struct kbase_device * const kbdev) +{ + + /* the first memory resource is the physical address of the GPU + * registers */ + struct platform_device *pdev = to_platform_device(kbdev->dev); + struct resource *reg_res; + int err; + + reg_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!reg_res) { + dev_err(kbdev->dev, "Invalid register resource\n"); + return -ENOENT; + } + + kbdev->reg_start = reg_res->start; + kbdev->reg_size = resource_size(reg_res); + + err = kbase_common_reg_map(kbdev); + if (err) { + dev_err(kbdev->dev, "Failed to map registers\n"); + return err; + } + + return 0; +} + +static void registers_unmap(struct kbase_device *kbdev) +{ + kbase_common_reg_unmap(kbdev); +} + +static int power_control_init(struct platform_device *pdev) +{ + struct kbase_device *kbdev = to_kbase_device(&pdev->dev); + int err = 0; + + if (!kbdev) + return -ENODEV; + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ + && defined(CONFIG_REGULATOR) + kbdev->regulator = regulator_get_optional(kbdev->dev, "mali"); + if (IS_ERR_OR_NULL(kbdev->regulator)) { + err = PTR_ERR(kbdev->regulator); + kbdev->regulator = NULL; + if (err == -EPROBE_DEFER) { + dev_err(&pdev->dev, "Failed to get regulator\n"); + return err; + } + dev_info(kbdev->dev, + "Continuing without Mali regulator control\n"); + /* Allow probe to continue without regulator */ + } +#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ + + kbdev->clock = clk_get(kbdev->dev, "clk_mali"); + if (IS_ERR_OR_NULL(kbdev->clock)) { + err = PTR_ERR(kbdev->clock); + kbdev->clock = NULL; + if (err == -EPROBE_DEFER) { + dev_err(&pdev->dev, "Failed to get clock\n"); + goto fail; + } + dev_info(kbdev->dev, "Continuing without Mali clock control\n"); + /* Allow probe to continue without clock. */ + } else { + err = clk_prepare(kbdev->clock); + if (err) { + dev_err(kbdev->dev, + "Failed to prepare and enable clock (%d)\n", + err); + goto fail; + } + } + + err = kbase_platform_rk_init_opp_table(kbdev); + if (err) + dev_err(kbdev->dev, "Failed to init_opp_table (%d)\n", err); + + return 0; + +fail: + +if (kbdev->clock != NULL) { + clk_put(kbdev->clock); + kbdev->clock = NULL; +} + +#ifdef CONFIG_REGULATOR + if (NULL != kbdev->regulator) { + regulator_put(kbdev->regulator); + kbdev->regulator = NULL; + } +#endif + + return err; +} + +static void power_control_term(struct kbase_device *kbdev) +{ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) || \ + defined(LSK_OPPV2_BACKPORT) + dev_pm_opp_of_remove_table(kbdev->dev); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) + of_free_opp_table(kbdev->dev); +#endif + + if (kbdev->clock) { + clk_unprepare(kbdev->clock); + clk_put(kbdev->clock); + kbdev->clock = NULL; + } + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ + && defined(CONFIG_REGULATOR) + if (kbdev->regulator) { + regulator_put(kbdev->regulator); + kbdev->regulator = NULL; + } +#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ +} + +#ifdef CONFIG_DEBUG_FS + +#if KBASE_GPU_RESET_EN +#include + +static void trigger_quirks_reload(struct kbase_device *kbdev) +{ + kbase_pm_context_active(kbdev); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + kbase_pm_context_idle(kbdev); +} + +#define MAKE_QUIRK_ACCESSORS(type) \ +static int type##_quirks_set(void *data, u64 val) \ +{ \ + struct kbase_device *kbdev; \ + kbdev = (struct kbase_device *)data; \ + kbdev->hw_quirks_##type = (u32)val; \ + trigger_quirks_reload(kbdev); \ + return 0;\ +} \ +\ +static int type##_quirks_get(void *data, u64 *val) \ +{ \ + struct kbase_device *kbdev;\ + kbdev = (struct kbase_device *)data;\ + *val = kbdev->hw_quirks_##type;\ + return 0;\ +} \ +DEFINE_SIMPLE_ATTRIBUTE(fops_##type##_quirks, type##_quirks_get,\ + type##_quirks_set, "%llu\n") + +MAKE_QUIRK_ACCESSORS(sc); +MAKE_QUIRK_ACCESSORS(tiler); +MAKE_QUIRK_ACCESSORS(mmu); +MAKE_QUIRK_ACCESSORS(jm); + +#endif /* KBASE_GPU_RESET_EN */ + +/** + * debugfs_protected_debug_mode_read - "protected_debug_mode" debugfs read + * @file: File object to read is for + * @buf: User buffer to populate with data + * @len: Length of user buffer + * @ppos: Offset within file object + * + * Retrieves the current status of protected debug mode + * (0 = disabled, 1 = enabled) + * + * Return: Number of bytes added to user buffer + */ +static ssize_t debugfs_protected_debug_mode_read(struct file *file, + char __user *buf, size_t len, loff_t *ppos) +{ + struct kbase_device *kbdev = (struct kbase_device *)file->private_data; + u32 gpu_status; + ssize_t ret_val; + + kbase_pm_context_active(kbdev); + gpu_status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL); + kbase_pm_context_idle(kbdev); + + if (gpu_status & GPU_DBGEN) + ret_val = simple_read_from_buffer(buf, len, ppos, "1\n", 2); + else + ret_val = simple_read_from_buffer(buf, len, ppos, "0\n", 2); + + return ret_val; +} + +/* + * struct fops_protected_debug_mode - "protected_debug_mode" debugfs fops + * + * Contains the file operations for the "protected_debug_mode" debugfs file + */ +static const struct file_operations fops_protected_debug_mode = { + .open = simple_open, + .read = debugfs_protected_debug_mode_read, + .llseek = default_llseek, +}; + +static int kbase_device_debugfs_init(struct kbase_device *kbdev) +{ + struct dentry *debugfs_ctx_defaults_directory; + int err; + + kbdev->mali_debugfs_directory = debugfs_create_dir(kbdev->devname, + NULL); + if (!kbdev->mali_debugfs_directory) { + dev_err(kbdev->dev, "Couldn't create mali debugfs directory\n"); + err = -ENOMEM; + goto out; + } + + kbdev->debugfs_ctx_directory = debugfs_create_dir("ctx", + kbdev->mali_debugfs_directory); + if (!kbdev->debugfs_ctx_directory) { + dev_err(kbdev->dev, "Couldn't create mali debugfs ctx directory\n"); + err = -ENOMEM; + goto out; + } + + debugfs_ctx_defaults_directory = debugfs_create_dir("defaults", + kbdev->debugfs_ctx_directory); + if (!debugfs_ctx_defaults_directory) { + dev_err(kbdev->dev, "Couldn't create mali debugfs ctx defaults directory\n"); + err = -ENOMEM; + goto out; + } + +#if !MALI_CUSTOMER_RELEASE + kbasep_regs_dump_debugfs_init(kbdev); +#endif /* !MALI_CUSTOMER_RELEASE */ + kbasep_regs_history_debugfs_init(kbdev); + + kbase_debug_job_fault_debugfs_init(kbdev); + kbasep_gpu_memory_debugfs_init(kbdev); + kbase_as_fault_debugfs_init(kbdev); +#if KBASE_GPU_RESET_EN + /* fops_* variables created by invocations of macro + * MAKE_QUIRK_ACCESSORS() above. */ + debugfs_create_file("quirks_sc", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_sc_quirks); + debugfs_create_file("quirks_tiler", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_tiler_quirks); + debugfs_create_file("quirks_mmu", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_mmu_quirks); + debugfs_create_file("quirks_jm", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_jm_quirks); +#endif /* KBASE_GPU_RESET_EN */ + + debugfs_create_bool("infinite_cache", 0644, + debugfs_ctx_defaults_directory, + &kbdev->infinite_cache_active_default); + + debugfs_create_size_t("mem_pool_max_size", 0644, + debugfs_ctx_defaults_directory, + &kbdev->mem_pool_max_size_default); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { + debugfs_create_file("protected_debug_mode", S_IRUGO, + kbdev->mali_debugfs_directory, kbdev, + &fops_protected_debug_mode); + } + +#if KBASE_TRACE_ENABLE + kbasep_trace_debugfs_init(kbdev); +#endif /* KBASE_TRACE_ENABLE */ + +#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE + kbasep_trace_timeline_debugfs_init(kbdev); +#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ +#ifdef CONFIG_DEVFREQ_THERMAL + if ((kbdev->inited_subsys & inited_devfreq) && !kbdev->model_data) + kbase_ipa_debugfs_init(kbdev); +#endif /* CONFIG_DEVFREQ_THERMAL */ +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + +#ifdef CONFIG_DEBUG_FS + debugfs_create_file("serialize_jobs", S_IRUGO | S_IWUSR, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_serialize_jobs_debugfs_fops); +#endif /* CONFIG_DEBUG_FS */ + + return 0; + +out: + debugfs_remove_recursive(kbdev->mali_debugfs_directory); + return err; +} + +static void kbase_device_debugfs_term(struct kbase_device *kbdev) +{ + debugfs_remove_recursive(kbdev->mali_debugfs_directory); +} + +#else /* CONFIG_DEBUG_FS */ +static inline int kbase_device_debugfs_init(struct kbase_device *kbdev) +{ + return 0; +} + +static inline void kbase_device_debugfs_term(struct kbase_device *kbdev) { } +#endif /* CONFIG_DEBUG_FS */ + +static void kbase_device_coherency_init(struct kbase_device *kbdev, + unsigned prod_id) +{ +#ifdef CONFIG_OF + u32 supported_coherency_bitmap = + kbdev->gpu_props.props.raw_props.coherency_mode; + const void *coherency_override_dts; + u32 override_coherency; + + /* Only for tMIx : + * (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly + * documented for tMIx so force correct value here. + */ + if (GPU_ID_IS_NEW_FORMAT(prod_id) && + (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == + GPU_ID2_PRODUCT_TMIX)) + if (supported_coherency_bitmap == + COHERENCY_FEATURE_BIT(COHERENCY_ACE)) + supported_coherency_bitmap |= + COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); + +#endif /* CONFIG_OF */ + + kbdev->system_coherency = COHERENCY_NONE; + + /* device tree may override the coherency */ +#ifdef CONFIG_OF + coherency_override_dts = of_get_property(kbdev->dev->of_node, + "system-coherency", + NULL); + if (coherency_override_dts) { + + override_coherency = be32_to_cpup(coherency_override_dts); + + if ((override_coherency <= COHERENCY_NONE) && + (supported_coherency_bitmap & + COHERENCY_FEATURE_BIT(override_coherency))) { + + kbdev->system_coherency = override_coherency; + + dev_info(kbdev->dev, + "Using coherency mode %u set from dtb", + override_coherency); + } else + dev_warn(kbdev->dev, + "Ignoring unsupported coherency mode %u set from dtb", + override_coherency); + } + +#endif /* CONFIG_OF */ + + kbdev->gpu_props.props.raw_props.coherency_mode = + kbdev->system_coherency; +} + +#ifdef CONFIG_MALI_FPGA_BUS_LOGGER + +/* Callback used by the kbase bus logger client, to initiate a GPU reset + * when the bus log is restarted. GPU reset is used as reference point + * in HW bus log analyses. + */ +static void kbase_logging_started_cb(void *data) +{ + struct kbase_device *kbdev = (struct kbase_device *)data; + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + dev_info(kbdev->dev, "KBASE - Bus logger restarted\n"); +} +#endif + +static struct attribute *kbase_attrs[] = { +#ifdef CONFIG_MALI_BIFROST_DEBUG + &dev_attr_debug_command.attr, + &dev_attr_js_softstop_always.attr, +#endif +#if !MALI_CUSTOMER_RELEASE + &dev_attr_force_replay.attr, +#endif + &dev_attr_js_timeouts.attr, + &dev_attr_soft_job_timeout.attr, + &dev_attr_gpuinfo.attr, + &dev_attr_dvfs_period.attr, + &dev_attr_pm_poweroff.attr, + &dev_attr_reset_timeout.attr, + &dev_attr_js_scheduling_period.attr, + &dev_attr_power_policy.attr, + &dev_attr_core_availability_policy.attr, + &dev_attr_core_mask.attr, + &dev_attr_mem_pool_size.attr, + &dev_attr_mem_pool_max_size.attr, + &dev_attr_lp_mem_pool_size.attr, + &dev_attr_lp_mem_pool_max_size.attr, + NULL +}; + +static const struct attribute_group kbase_attr_group = { + .attrs = kbase_attrs, +}; + +static int kbase_platform_device_remove(struct platform_device *pdev) +{ + struct kbase_device *kbdev = to_kbase_device(&pdev->dev); + const struct list_head *dev_list; + + if (!kbdev) + return -ENODEV; + + kfree(kbdev->gpu_props.prop_buffer); + +#ifdef CONFIG_MALI_FPGA_BUS_LOGGER + if (kbdev->inited_subsys & inited_buslogger) { + bl_core_client_unregister(kbdev->buslogger); + kbdev->inited_subsys &= ~inited_buslogger; + } +#endif + + + if (kbdev->inited_subsys & inited_dev_list) { + dev_list = kbase_dev_list_get(); + list_del(&kbdev->entry); + kbase_dev_list_put(dev_list); + kbdev->inited_subsys &= ~inited_dev_list; + } + + if (kbdev->inited_subsys & inited_misc_register) { + misc_deregister(&kbdev->mdev); + kbdev->inited_subsys &= ~inited_misc_register; + } + + if (kbdev->inited_subsys & inited_sysfs_group) { + sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); + kbdev->inited_subsys &= ~inited_sysfs_group; + } + + if (kbdev->inited_subsys & inited_get_device) { + put_device(kbdev->dev); + kbdev->inited_subsys &= ~inited_get_device; + } + + if (kbdev->inited_subsys & inited_debugfs) { + kbase_device_debugfs_term(kbdev); + kbdev->inited_subsys &= ~inited_debugfs; + } + + if (kbdev->inited_subsys & inited_job_fault) { + kbase_debug_job_fault_dev_term(kbdev); + kbdev->inited_subsys &= ~inited_job_fault; + } + if (kbdev->inited_subsys & inited_vinstr) { + kbase_vinstr_term(kbdev->vinstr_ctx); + kbdev->inited_subsys &= ~inited_vinstr; + } + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + if (kbdev->inited_subsys & inited_devfreq) { + kbase_devfreq_term(kbdev); + kbdev->inited_subsys &= ~inited_devfreq; + } +#endif + + if (kbdev->inited_subsys & inited_backend_late) { + kbase_backend_late_term(kbdev); + kbdev->inited_subsys &= ~inited_backend_late; + } + + if (kbdev->inited_subsys & inited_tlstream) { + kbase_tlstream_term(); + kbdev->inited_subsys &= ~inited_tlstream; + } + + /* Bring job and mem sys to a halt before we continue termination */ + + if (kbdev->inited_subsys & inited_js) + kbasep_js_devdata_halt(kbdev); + + if (kbdev->inited_subsys & inited_mem) + kbase_mem_halt(kbdev); + + if (kbdev->inited_subsys & inited_protected) { + kbasep_protected_mode_term(kbdev); + kbdev->inited_subsys &= ~inited_protected; + } + + if (kbdev->inited_subsys & inited_js) { + kbasep_js_devdata_term(kbdev); + kbdev->inited_subsys &= ~inited_js; + } + + if (kbdev->inited_subsys & inited_mem) { + kbase_mem_term(kbdev); + kbdev->inited_subsys &= ~inited_mem; + } + + if (kbdev->inited_subsys & inited_pm_runtime_init) { + kbdev->pm.callback_power_runtime_term(kbdev); + kbdev->inited_subsys &= ~inited_pm_runtime_init; + } + + if (kbdev->inited_subsys & inited_ctx_sched) { + kbase_ctx_sched_term(kbdev); + kbdev->inited_subsys &= ~inited_ctx_sched; + } + + if (kbdev->inited_subsys & inited_device) { + kbase_device_term(kbdev); + kbdev->inited_subsys &= ~inited_device; + } + + if (kbdev->inited_subsys & inited_backend_early) { + kbase_backend_early_term(kbdev); + kbdev->inited_subsys &= ~inited_backend_early; + } + + if (kbdev->inited_subsys & inited_io_history) { + kbase_io_history_term(&kbdev->io_history); + kbdev->inited_subsys &= ~inited_io_history; + } + + if (kbdev->inited_subsys & inited_power_control) { + power_control_term(kbdev); + kbdev->inited_subsys &= ~inited_power_control; + } + + if (kbdev->inited_subsys & inited_registers_map) { + registers_unmap(kbdev); + kbdev->inited_subsys &= ~inited_registers_map; + } + +#ifdef CONFIG_MALI_BIFROST_NO_MALI + if (kbdev->inited_subsys & inited_gpu_device) { + gpu_device_destroy(kbdev); + kbdev->inited_subsys &= ~inited_gpu_device; + } +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ + + if (kbdev->inited_subsys != 0) + dev_err(kbdev->dev, "Missing sub system termination\n"); + + kbase_device_free(kbdev); + + return 0; +} + + +/* Number of register accesses for the buffer that we allocate during + * initialization time. The buffer size can be changed later via debugfs. */ +#define KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ((u16)512) + +static int kbase_platform_device_probe(struct platform_device *pdev) +{ + struct kbase_device *kbdev; + struct mali_base_gpu_core_props *core_props; + u32 gpu_id; + unsigned prod_id; + const struct list_head *dev_list; + int err = 0; + +#ifdef CONFIG_OF + err = kbase_platform_early_init(); + if (err) { + dev_err(&pdev->dev, "Early platform initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } +#endif + kbdev = kbase_device_alloc(); + if (!kbdev) { + dev_err(&pdev->dev, "Allocate device failed\n"); + kbase_platform_device_remove(pdev); + return -ENOMEM; + } + + kbdev->dev = &pdev->dev; + dev_set_drvdata(kbdev->dev, kbdev); + +#ifdef CONFIG_MALI_BIFROST_NO_MALI + err = gpu_device_create(kbdev); + if (err) { + dev_err(&pdev->dev, "Dummy model initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_gpu_device; +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ + + err = assign_irqs(pdev); + if (err) { + dev_err(&pdev->dev, "IRQ search failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + + err = registers_map(kbdev); + if (err) { + dev_err(&pdev->dev, "Register map failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_registers_map; + + err = power_control_init(pdev); + if (err) { + dev_err(&pdev->dev, "Power control initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_power_control; + + err = kbase_io_history_init(&kbdev->io_history, + KBASEP_DEFAULT_REGISTER_HISTORY_SIZE); + if (err) { + dev_err(&pdev->dev, "Register access history initialization failed\n"); + kbase_platform_device_remove(pdev); + return -ENOMEM; + } + kbdev->inited_subsys |= inited_io_history; + + err = kbase_backend_early_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Early backend initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_backend_early; + + scnprintf(kbdev->devname, DEVNAME_SIZE, "%s%d", kbase_drv_name, + kbase_dev_nr); + + kbase_disjoint_init(kbdev); + + /* obtain min/max configured gpu frequencies */ + core_props = &(kbdev->gpu_props.props.core_props); + core_props->gpu_freq_khz_min = GPU_FREQ_KHZ_MIN; + core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX; + + err = kbase_device_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Device initialization failed (%d)\n", err); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_device; + + err = kbase_ctx_sched_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Context scheduler initialization failed (%d)\n", + err); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_ctx_sched; + + if (kbdev->pm.callback_power_runtime_init) { + err = kbdev->pm.callback_power_runtime_init(kbdev); + if (err) { + dev_err(kbdev->dev, + "Runtime PM initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_pm_runtime_init; + } + + err = kbase_mem_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Memory subsystem initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_mem; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + gpu_id &= GPU_ID_VERSION_PRODUCT_ID; + prod_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + kbase_device_coherency_init(kbdev, prod_id); + + err = kbasep_protected_mode_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Protected mode subsystem initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_protected; + + dev_list = kbase_dev_list_get(); + list_add(&kbdev->entry, &kbase_dev_list); + kbase_dev_list_put(dev_list); + kbdev->inited_subsys |= inited_dev_list; + + err = kbasep_js_devdata_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Job JS devdata initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_js; + + err = kbase_tlstream_init(); + if (err) { + dev_err(kbdev->dev, "Timeline stream initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_tlstream; + + err = kbase_backend_late_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Late backend initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_backend_late; + + /* Initialize the kctx list. This is used by vinstr. */ + mutex_init(&kbdev->kctx_list_lock); + INIT_LIST_HEAD(&kbdev->kctx_list); + + kbdev->vinstr_ctx = kbase_vinstr_init(kbdev); + if (!kbdev->vinstr_ctx) { + dev_err(kbdev->dev, + "Virtual instrumentation initialization failed\n"); + kbase_platform_device_remove(pdev); + return -EINVAL; + } + kbdev->inited_subsys |= inited_vinstr; + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + /* Devfreq uses vinstr, so must be initialized after it. */ + err = kbase_devfreq_init(kbdev); + if (!err) + kbdev->inited_subsys |= inited_devfreq; + else + dev_err(kbdev->dev, "Continuing without devfreq\n"); +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + + err = kbase_debug_job_fault_dev_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Job fault debug initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_job_fault; + + err = kbase_device_debugfs_init(kbdev); + if (err) { + dev_err(kbdev->dev, "DebugFS initialization failed"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_debugfs; + + kbdev->mdev.minor = MISC_DYNAMIC_MINOR; + kbdev->mdev.name = kbdev->devname; + kbdev->mdev.fops = &kbase_fops; + kbdev->mdev.parent = get_device(kbdev->dev); + kbdev->inited_subsys |= inited_get_device; + + /* This needs to happen before registering the device with misc_register(), + * otherwise it causes a race condition between registering the device and a + * uevent event being generated for userspace, causing udev rules to run + * which might expect certain sysfs attributes present. As a result of the + * race condition we avoid, some Mali sysfs entries may have appeared to + * udev to not exist. + + * For more information, see + * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the + * paragraph that starts with "Word of warning", currently the second-last + * paragraph. + */ + err = sysfs_create_group(&kbdev->dev->kobj, &kbase_attr_group); + if (err) { + dev_err(&pdev->dev, "SysFS group creation failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_sysfs_group; + + err = misc_register(&kbdev->mdev); + if (err) { + dev_err(kbdev->dev, "Misc device registration failed for %s\n", + kbdev->devname); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_misc_register; + + +#ifdef CONFIG_MALI_FPGA_BUS_LOGGER + err = bl_core_client_register(kbdev->devname, + kbase_logging_started_cb, + kbdev, &kbdev->buslogger, + THIS_MODULE, NULL); + if (err == 0) { + kbdev->inited_subsys |= inited_buslogger; + bl_core_set_threshold(kbdev->buslogger, 1024*1024*1024); + } else { + dev_warn(kbdev->dev, "Bus log client registration failed\n"); + err = 0; + } +#endif + + err = kbase_gpuprops_populate_user_buffer(kbdev); + if (err) { + dev_err(&pdev->dev, "GPU property population failed"); + kbase_platform_device_remove(pdev); + return err; + } + + dev_info(kbdev->dev, + "Probed as %s\n", dev_name(kbdev->mdev.this_device)); + + kbase_dev_nr++; + + return err; +} + +#undef KBASEP_DEFAULT_REGISTER_HISTORY_SIZE + +/** + * kbase_device_suspend - Suspend callback from the OS. + * + * This is called by Linux when the device should suspend. + * + * @dev: The device to suspend + * + * Return: A standard Linux error code + */ +static int kbase_device_suspend(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + if (kbdev->inited_subsys & inited_devfreq) + devfreq_suspend_device(kbdev->devfreq); +#endif + + kbase_pm_suspend(kbdev); + return 0; +} + +/** + * kbase_device_resume - Resume callback from the OS. + * + * This is called by Linux when the device should resume from suspension. + * + * @dev: The device to resume + * + * Return: A standard Linux error code + */ +static int kbase_device_resume(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + kbase_pm_resume(kbdev); + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + if (kbdev->inited_subsys & inited_devfreq) + devfreq_resume_device(kbdev->devfreq); +#endif + return 0; +} + +/** + * kbase_device_runtime_suspend - Runtime suspend callback from the OS. + * + * This is called by Linux when the device should prepare for a condition in + * which it will not be able to communicate with the CPU(s) and RAM due to + * power management. + * + * @dev: The device to suspend + * + * Return: A standard Linux error code + */ +#ifdef KBASE_PM_RUNTIME +static int kbase_device_runtime_suspend(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + if (kbdev->inited_subsys & inited_devfreq) + devfreq_suspend_device(kbdev->devfreq); +#endif + + if (kbdev->pm.backend.callback_power_runtime_off) { + kbdev->pm.backend.callback_power_runtime_off(kbdev); + dev_dbg(dev, "runtime suspend\n"); + } + return 0; +} +#endif /* KBASE_PM_RUNTIME */ + +/** + * kbase_device_runtime_resume - Runtime resume callback from the OS. + * + * This is called by Linux when the device should go into a fully active state. + * + * @dev: The device to suspend + * + * Return: A standard Linux error code + */ + +#ifdef KBASE_PM_RUNTIME +static int kbase_device_runtime_resume(struct device *dev) +{ + int ret = 0; + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + if (kbdev->pm.backend.callback_power_runtime_on) { + ret = kbdev->pm.backend.callback_power_runtime_on(kbdev); + dev_dbg(dev, "runtime resume\n"); + } + +#if defined(CONFIG_MALI_BIFROST_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + if (kbdev->inited_subsys & inited_devfreq) + devfreq_resume_device(kbdev->devfreq); +#endif + + return ret; +} +#endif /* KBASE_PM_RUNTIME */ + + +#ifdef KBASE_PM_RUNTIME +/** + * kbase_device_runtime_idle - Runtime idle callback from the OS. + * @dev: The device to suspend + * + * This is called by Linux when the device appears to be inactive and it might + * be placed into a low power state. + * + * Return: 0 if device can be suspended, non-zero to avoid runtime autosuspend, + * otherwise a standard Linux error code + */ +static int kbase_device_runtime_idle(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + /* Use platform specific implementation if it exists. */ + if (kbdev->pm.backend.callback_power_runtime_idle) + return kbdev->pm.backend.callback_power_runtime_idle(kbdev); + + return 0; +} +#endif /* KBASE_PM_RUNTIME */ + +/* The power management operations for the platform driver. + */ +static const struct dev_pm_ops kbase_pm_ops = { + .suspend = kbase_device_suspend, + .resume = kbase_device_resume, +#ifdef KBASE_PM_RUNTIME + .runtime_suspend = kbase_device_runtime_suspend, + .runtime_resume = kbase_device_runtime_resume, + .runtime_idle = kbase_device_runtime_idle, +#endif /* KBASE_PM_RUNTIME */ +}; + +#ifdef CONFIG_OF +static const struct of_device_id kbase_dt_ids[] = { + { .compatible = "arm,malit6xx" }, + { .compatible = "arm,mali-midgard" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, kbase_dt_ids); +#endif + +static struct platform_driver kbase_platform_driver = { + .probe = kbase_platform_device_probe, + .remove = kbase_platform_device_remove, + .driver = { + .name = kbase_drv_name, + .owner = THIS_MODULE, + .pm = &kbase_pm_ops, + .of_match_table = of_match_ptr(kbase_dt_ids), + }, +}; + +/* + * The driver will not provide a shortcut to create the Mali platform device + * anymore when using Device Tree. + */ +#ifdef CONFIG_OF +module_platform_driver(kbase_platform_driver); +#else + +static int __init kbase_driver_init(void) +{ + int ret; + + ret = kbase_platform_early_init(); + if (ret) + return ret; + + ret = kbase_platform_register(); + if (ret) + return ret; + + ret = platform_driver_register(&kbase_platform_driver); + + if (ret) + kbase_platform_unregister(); + + return ret; +} + +static void __exit kbase_driver_exit(void) +{ + platform_driver_unregister(&kbase_platform_driver); + kbase_platform_unregister(); +} + +module_init(kbase_driver_init); +module_exit(kbase_driver_exit); + +#endif /* CONFIG_OF */ + +MODULE_LICENSE("GPL"); +MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \ + __stringify(BASE_UK_VERSION_MAJOR) "." \ + __stringify(BASE_UK_VERSION_MINOR) ")"); + +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) || defined(CONFIG_MALI_BIFROST_SYSTEM_TRACE) +#define CREATE_TRACE_POINTS +#endif + +#ifdef CONFIG_MALI_BIFROST_GATOR_SUPPORT +/* Create the trace points (otherwise we just get code to call a tracepoint) */ +#include "mali_linux_trace.h" + +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_job_slots_event); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_status); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_on); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_off); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_page_fault_insert_pages); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_in_use); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_released); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_total_alloc_pages_change); + +void kbase_trace_mali_pm_status(u32 event, u64 value) +{ + trace_mali_pm_status(event, value); +} + +void kbase_trace_mali_pm_power_off(u32 event, u64 value) +{ + trace_mali_pm_power_off(event, value); +} + +void kbase_trace_mali_pm_power_on(u32 event, u64 value) +{ + trace_mali_pm_power_on(event, value); +} + +void kbase_trace_mali_job_slots_event(u32 event, const struct kbase_context *kctx, u8 atom_id) +{ + trace_mali_job_slots_event(event, (kctx != NULL ? kctx->tgid : 0), (kctx != NULL ? kctx->pid : 0), atom_id); +} + +void kbase_trace_mali_page_fault_insert_pages(int event, u32 value) +{ + trace_mali_page_fault_insert_pages(event, value); +} + +void kbase_trace_mali_mmu_as_in_use(int event) +{ + trace_mali_mmu_as_in_use(event); +} + +void kbase_trace_mali_mmu_as_released(int event) +{ + trace_mali_mmu_as_released(event); +} + +void kbase_trace_mali_total_alloc_pages_change(long long int event) +{ + trace_mali_total_alloc_pages_change(event); +} +#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ +#ifdef CONFIG_MALI_BIFROST_SYSTEM_TRACE +#include "mali_linux_kbase_trace.h" +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.c new file mode 100755 index 000000000000..e2f7baabad43 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.c @@ -0,0 +1,203 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include + +#include "mali_kbase_ctx_sched.h" + +int kbase_ctx_sched_init(struct kbase_device *kbdev) +{ + int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; + + /* These two must be recalculated if nr_hw_address_spaces changes + * (e.g. for HW workarounds) */ + kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) { + bool use_workaround; + + use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE; + if (use_workaround) { + dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only"); + kbdev->nr_user_address_spaces = 1; + } + } + + kbdev->as_free = as_present; /* All ASs initially free */ + + memset(kbdev->as_to_kctx, 0, sizeof(kbdev->as_to_kctx)); + + return 0; +} + +void kbase_ctx_sched_term(struct kbase_device *kbdev) +{ + s8 i; + + /* Sanity checks */ + for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { + WARN_ON(kbdev->as_to_kctx[i] != NULL); + WARN_ON(!(kbdev->as_free & (1u << i))); + } +} + +/* kbasep_ctx_sched_find_as_for_ctx - Find a free address space + * + * @kbdev: The context for which to find a free address space + * + * Return: A valid AS if successful, otherwise KBASEP_AS_NR_INVALID + * + * This function returns an address space available for use. It would prefer + * returning an AS that has been previously assigned to the context to + * avoid having to reprogram the MMU. + */ +static int kbasep_ctx_sched_find_as_for_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + int free_as; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* First check if the previously assigned AS is available */ + if ((kctx->as_nr != KBASEP_AS_NR_INVALID) && + (kbdev->as_free & (1u << kctx->as_nr))) + return kctx->as_nr; + + /* The previously assigned AS was taken, we'll be returning any free + * AS at this point. + */ + free_as = ffs(kbdev->as_free) - 1; + if (free_as >= 0 && free_as < kbdev->nr_hw_address_spaces) + return free_as; + + return KBASEP_AS_NR_INVALID; +} + +int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(!kbdev->pm.backend.gpu_powered); + + if (atomic_inc_return(&kctx->refcount) == 1) { + int const free_as = kbasep_ctx_sched_find_as_for_ctx(kctx); + + if (free_as != KBASEP_AS_NR_INVALID) { + kbdev->as_free &= ~(1u << free_as); + /* Only program the MMU if the context has not been + * assigned the same address space before. + */ + if (free_as != kctx->as_nr) { + struct kbase_context *const prev_kctx = + kbdev->as_to_kctx[free_as]; + + if (prev_kctx) { + WARN_ON(atomic_read(&prev_kctx->refcount) != 0); + kbase_mmu_disable(prev_kctx); + prev_kctx->as_nr = KBASEP_AS_NR_INVALID; + } + + kctx->as_nr = free_as; + kbdev->as_to_kctx[free_as] = kctx; + kbase_mmu_update(kctx); + } + } else { + atomic_dec(&kctx->refcount); + + /* Failed to find an available address space, we must + * be returning an error at this point. + */ + WARN_ON(kctx->as_nr != KBASEP_AS_NR_INVALID); + } + } + + return kctx->as_nr; +} + +void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->hwaccess_lock); + WARN_ON(atomic_read(&kctx->refcount) == 0); + WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID); + WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx); + + atomic_inc(&kctx->refcount); +} + +void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (atomic_dec_return(&kctx->refcount) == 0) + kbdev->as_free |= (1u << kctx->as_nr); +} + +void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(atomic_read(&kctx->refcount) != 0); + + if (kctx->as_nr != KBASEP_AS_NR_INVALID) { + if (kbdev->pm.backend.gpu_powered) + kbase_mmu_disable(kctx); + + kbdev->as_to_kctx[kctx->as_nr] = NULL; + kctx->as_nr = KBASEP_AS_NR_INVALID; + } +} + +void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) +{ + s8 i; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(!kbdev->pm.backend.gpu_powered); + + for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { + struct kbase_context *kctx; + + kctx = kbdev->as_to_kctx[i]; + if (kctx) { + if (atomic_read(&kctx->refcount)) { + WARN_ON(kctx->as_nr != i); + + kbase_mmu_update(kctx); + } else { + /* This context might have been assigned an + * AS before, clear it. + */ + kbdev->as_to_kctx[kctx->as_nr] = NULL; + kctx->as_nr = KBASEP_AS_NR_INVALID; + } + } else { + kbase_mmu_disable_as(kbdev, i); + } + } +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.h new file mode 100755 index 000000000000..2330d48c8e51 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ctx_sched.h @@ -0,0 +1,131 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_CTX_SCHED_H_ +#define _KBASE_CTX_SCHED_H_ + +#include + +/* The Context Scheduler manages address space assignment and reference + * counting to kbase_context. The interface has been designed to minimise + * interactions between the Job Scheduler and Power Management/MMU to support + * the existing Job Scheduler interface. + * + * The initial implementation of the Context Scheduler does not schedule + * contexts. Instead it relies on the Job Scheduler to make decisions of + * when to schedule/evict contexts if address spaces are starved. In the + * future, once an interface between the CS and JS have been devised to + * provide enough information about how each context is consuming GPU resources, + * those decisions can be made in the CS itself, thereby reducing duplicated + * code. + */ + +/* base_ctx_sched_init - Initialise the context scheduler + * + * @kbdev: The device for which the context scheduler needs to be + * initialised + * + * Return: 0 for success, otherwise failure + * + * This must be called during device initilisation. The number of hardware + * address spaces must already be established before calling this function. + */ +int kbase_ctx_sched_init(struct kbase_device *kbdev); + +/* base_ctx_sched_term - Terminate the context scheduler + * + * @kbdev: The device for which the context scheduler needs to be + * terminated + * + * This must be called during device termination after all contexts have been + * destroyed. + */ +void kbase_ctx_sched_term(struct kbase_device *kbdev); + +/* kbase_ctx_sched_retain_ctx - Retain a reference to the @ref kbase_context + * + * @kctx: The context to which to retain a reference + * + * Return: The address space that the context has been assigned to or + * KBASEP_AS_NR_INVALID if no address space was available. + * + * This function should be called whenever an address space should be assigned + * to a context and programmed onto the MMU. It should typically be called + * when jobs are ready to be submitted to the GPU. + * + * It can be called as many times as necessary. The address space will be + * assigned to the context for as long as there is a reference to said context. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx); + +/* kbase_ctx_sched_retain_ctx_refcount + * + * @kctx: The context to which to retain a reference + * + * This function only retains a reference to the context. It must be called + * only when the context already has a reference. + * + * This is typically called inside an atomic session where we know the context + * is already scheduled in but want to take an extra reference to ensure that + * it doesn't get descheduled. + * + * The kbase_device::hwaccess_lock must be held whilst calling this function + */ +void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx); + +/* kbase_ctx_sched_release_ctx - Release a reference to the @ref kbase_context + * + * @kctx: The context from which to release a reference + * + * This function should be called whenever an address space could be unassigned + * from a context. When there are no more references to said context, the + * address space previously assigned to this context shall be reassigned to + * other contexts as needed. + * + * The kbase_device::hwaccess_lock must be held whilst calling this function + */ +void kbase_ctx_sched_release_ctx(struct kbase_context *kctx); + +/* kbase_ctx_sched_remove_ctx - Unassign previously assigned address space + * + * @kctx: The context to be removed + * + * This function should be called when a context is being destroyed. The + * context must no longer have any reference. If it has been assigned an + * address space before then the AS will be unprogrammed. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); + +/* kbase_ctx_sched_restore_all_as - Reprogram all address spaces + * + * @kbdev: The device for which address spaces to be reprogrammed + * + * This function shall reprogram all address spaces previously assigned to + * contexts. It can be used after the GPU is reset. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev); + +#endif /* _KBASE_CTX_SCHED_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.c new file mode 100755 index 000000000000..fb57ac2e31ad --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.c @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include + +static struct kbasep_debug_assert_cb kbasep_debug_assert_registered_cb = { + NULL, + NULL +}; + +void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param) +{ + kbasep_debug_assert_registered_cb.func = func; + kbasep_debug_assert_registered_cb.param = param; +} + +void kbasep_debug_assert_call_hook(void) +{ + if (kbasep_debug_assert_registered_cb.func != NULL) + kbasep_debug_assert_registered_cb.func(kbasep_debug_assert_registered_cb.param); +} +KBASE_EXPORT_SYMBOL(kbasep_debug_assert_call_hook); + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.h new file mode 100755 index 000000000000..31b754c5507b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug.h @@ -0,0 +1,164 @@ +/* + * + * (C) COPYRIGHT 2012-2015, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_DEBUG_H +#define _KBASE_DEBUG_H + +#include + +/** @brief If equals to 0, a trace containing the file, line, and function will be displayed before each message. */ +#define KBASE_DEBUG_SKIP_TRACE 0 + +/** @brief If different from 0, the trace will only contain the file and line. */ +#define KBASE_DEBUG_SKIP_FUNCTION_NAME 0 + +/** @brief Disable the asserts tests if set to 1. Default is to disable the asserts in release. */ +#ifndef KBASE_DEBUG_DISABLE_ASSERTS +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define KBASE_DEBUG_DISABLE_ASSERTS 0 +#else +#define KBASE_DEBUG_DISABLE_ASSERTS 1 +#endif +#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ + +/** Function type that is called on an KBASE_DEBUG_ASSERT() or KBASE_DEBUG_ASSERT_MSG() */ +typedef void (kbase_debug_assert_hook) (void *); + +struct kbasep_debug_assert_cb { + kbase_debug_assert_hook *func; + void *param; +}; + +/** + * @def KBASEP_DEBUG_PRINT_TRACE + * @brief Private macro containing the format of the trace to display before every message + * @sa KBASE_DEBUG_SKIP_TRACE, KBASE_DEBUG_SKIP_FUNCTION_NAME + */ +#if !KBASE_DEBUG_SKIP_TRACE +#define KBASEP_DEBUG_PRINT_TRACE \ + "In file: " __FILE__ " line: " CSTD_STR2(__LINE__) +#if !KBASE_DEBUG_SKIP_FUNCTION_NAME +#define KBASEP_DEBUG_PRINT_FUNCTION __func__ +#else +#define KBASEP_DEBUG_PRINT_FUNCTION "" +#endif +#else +#define KBASEP_DEBUG_PRINT_TRACE "" +#endif + +/** + * @def KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) + * @brief (Private) system printing function associated to the @ref KBASE_DEBUG_ASSERT_MSG event. + * @param trace location in the code from where the message is printed + * @param function function from where the message is printed + * @param ... Format string followed by format arguments. + * @note function parameter cannot be concatenated with other strings + */ +/* Select the correct system output function*/ +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...)\ + do { \ + pr_err("Mali: %s function:%s ", trace, function);\ + pr_err(__VA_ARGS__);\ + pr_err("\n");\ + } while (false) +#else +#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) CSTD_NOP() +#endif + +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define KBASE_CALL_ASSERT_HOOK() kbasep_debug_assert_call_hook() +#else +#define KBASE_CALL_ASSERT_HOOK() CSTD_NOP() +#endif + +/** + * @def KBASE_DEBUG_ASSERT(expr) + * @brief Calls @ref KBASE_PRINT_ASSERT and prints the expression @a expr if @a expr is false + * + * @note This macro does nothing if the flag @ref KBASE_DEBUG_DISABLE_ASSERTS is set to 1 + * + * @param expr Boolean expression + */ +#define KBASE_DEBUG_ASSERT(expr) \ + KBASE_DEBUG_ASSERT_MSG(expr, #expr) + +#if KBASE_DEBUG_DISABLE_ASSERTS +#define KBASE_DEBUG_ASSERT_MSG(expr, ...) CSTD_NOP() +#else + /** + * @def KBASE_DEBUG_ASSERT_MSG(expr, ...) + * @brief Calls @ref KBASEP_DEBUG_ASSERT_OUT and prints the given message if @a expr is false + * + * @note This macro does nothing if the flag @ref KBASE_DEBUG_DISABLE_ASSERTS is set to 1 + * + * @param expr Boolean expression + * @param ... Message to display when @a expr is false, as a format string followed by format arguments. + */ +#define KBASE_DEBUG_ASSERT_MSG(expr, ...) \ + do { \ + if (!(expr)) { \ + KBASEP_DEBUG_ASSERT_OUT(KBASEP_DEBUG_PRINT_TRACE, KBASEP_DEBUG_PRINT_FUNCTION, __VA_ARGS__);\ + KBASE_CALL_ASSERT_HOOK();\ + BUG();\ + } \ + } while (false) +#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ + +/** + * @def KBASE_DEBUG_CODE( X ) + * @brief Executes the code inside the macro only in debug mode + * + * @param X Code to compile only in debug mode. + */ +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define KBASE_DEBUG_CODE(X) X +#else +#define KBASE_DEBUG_CODE(X) CSTD_NOP() +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + +/** @} */ + +/** + * @brief Register a function to call on ASSERT + * + * Such functions will \b only be called during Debug mode, and for debugging + * features \b only. Do not rely on them to be called in general use. + * + * To disable the hook, supply NULL to \a func. + * + * @note This function is not thread-safe, and should only be used to + * register/deregister once in the module's lifetime. + * + * @param[in] func the function to call when an assert is triggered. + * @param[in] param the parameter to pass to \a func when calling it + */ +void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param); + +/** + * @brief Call a debug assert hook previously registered with kbase_debug_assert_register_hook() + * + * @note This function is not thread-safe with respect to multiple threads + * registering functions and parameters with + * kbase_debug_assert_register_hook(). Otherwise, thread safety is the + * responsibility of the registered hook. + */ +void kbasep_debug_assert_call_hook(void); + +#endif /* _KBASE_DEBUG_H */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.c new file mode 100755 index 000000000000..f29430ddf8f9 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.c @@ -0,0 +1,499 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include + +#ifdef CONFIG_DEBUG_FS + +static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev) +{ + struct list_head *event_list = &kbdev->job_fault_event_list; + unsigned long flags; + bool ret; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + ret = !list_empty(event_list); + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + return ret; +} + +static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + struct list_head *event_list = &kctx->kbdev->job_fault_event_list; + struct base_job_fault_event *event; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + if (list_empty(event_list)) { + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + return true; + } + list_for_each_entry(event, event_list, head) { + if (event->katom->kctx == kctx) { + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, + flags); + return false; + } + } + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + return true; +} + +/* wait until the fault happen and copy the event */ +static int kbase_job_fault_event_wait(struct kbase_device *kbdev, + struct base_job_fault_event *event) +{ + struct list_head *event_list = &kbdev->job_fault_event_list; + struct base_job_fault_event *event_in; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + if (list_empty(event_list)) { + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + if (wait_event_interruptible(kbdev->job_fault_wq, + kbase_is_job_fault_event_pending(kbdev))) + return -ERESTARTSYS; + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + } + + event_in = list_entry(event_list->next, + struct base_job_fault_event, head); + event->event_code = event_in->event_code; + event->katom = event_in->katom; + + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + return 0; + +} + +/* remove the event from the queue */ +static struct base_job_fault_event *kbase_job_fault_event_dequeue( + struct kbase_device *kbdev, struct list_head *event_list) +{ + struct base_job_fault_event *event; + + event = list_entry(event_list->next, + struct base_job_fault_event, head); + list_del(event_list->next); + + return event; + +} + +/* Remove all the following atoms after the failed atom in the same context + * Call the postponed bottom half of job done. + * Then, this context could be rescheduled. + */ +static void kbase_job_fault_resume_event_cleanup(struct kbase_context *kctx) +{ + struct list_head *event_list = &kctx->job_fault_resume_event_list; + + while (!list_empty(event_list)) { + struct base_job_fault_event *event; + + event = kbase_job_fault_event_dequeue(kctx->kbdev, + &kctx->job_fault_resume_event_list); + kbase_jd_done_worker(&event->katom->work); + } + +} + +/* Remove all the failed atoms that belong to different contexts + * Resume all the contexts that were suspend due to failed job + */ +static void kbase_job_fault_event_cleanup(struct kbase_device *kbdev) +{ + struct list_head *event_list = &kbdev->job_fault_event_list; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + while (!list_empty(event_list)) { + kbase_job_fault_event_dequeue(kbdev, event_list); + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + wake_up(&kbdev->job_fault_resume_wq); + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + } + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); +} + +static void kbase_job_fault_resume_worker(struct work_struct *data) +{ + struct base_job_fault_event *event = container_of(data, + struct base_job_fault_event, job_fault_work); + struct kbase_context *kctx; + struct kbase_jd_atom *katom; + + katom = event->katom; + kctx = katom->kctx; + + dev_info(kctx->kbdev->dev, "Job dumping wait\n"); + + /* When it was waked up, it need to check if queue is empty or the + * failed atom belongs to different context. If yes, wake up. Both + * of them mean the failed job has been dumped. Please note, it + * should never happen that the job_fault_event_list has the two + * atoms belong to the same context. + */ + wait_event(kctx->kbdev->job_fault_resume_wq, + kbase_ctx_has_no_event_pending(kctx)); + + atomic_set(&kctx->job_fault_count, 0); + kbase_jd_done_worker(&katom->work); + + /* In case the following atoms were scheduled during failed job dump + * the job_done_worker was held. We need to rerun it after the dump + * was finished + */ + kbase_job_fault_resume_event_cleanup(kctx); + + dev_info(kctx->kbdev->dev, "Job dumping finish, resume scheduler\n"); +} + +static struct base_job_fault_event *kbase_job_fault_event_queue( + struct list_head *event_list, + struct kbase_jd_atom *atom, + u32 completion_code) +{ + struct base_job_fault_event *event; + + event = &atom->fault_event; + + event->katom = atom; + event->event_code = completion_code; + + list_add_tail(&event->head, event_list); + + return event; + +} + +static void kbase_job_fault_event_post(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, u32 completion_code) +{ + struct base_job_fault_event *event; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list, + katom, completion_code); + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + wake_up_interruptible(&kbdev->job_fault_wq); + + INIT_WORK(&event->job_fault_work, kbase_job_fault_resume_worker); + queue_work(kbdev->job_fault_resume_workq, &event->job_fault_work); + + dev_info(katom->kctx->kbdev->dev, "Job fault happen, start dump: %d_%d", + katom->kctx->tgid, katom->kctx->id); + +} + +/* + * This function will process the job fault + * Get the register copy + * Send the failed job dump event + * Create a Wait queue to wait until the job dump finish + */ + +bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, + u32 completion_code) +{ + struct kbase_context *kctx = katom->kctx; + + /* Check if dumping is in the process + * only one atom of each context can be dumped at the same time + * If the atom belongs to different context, it can be dumped + */ + if (atomic_read(&kctx->job_fault_count) > 0) { + kbase_job_fault_event_queue( + &kctx->job_fault_resume_event_list, + katom, completion_code); + dev_info(kctx->kbdev->dev, "queue:%d\n", + kbase_jd_atom_id(kctx, katom)); + return true; + } + + if (kctx->kbdev->job_fault_debug == true) { + + if (completion_code != BASE_JD_EVENT_DONE) { + + if (kbase_job_fault_get_reg_snapshot(kctx) == false) { + dev_warn(kctx->kbdev->dev, "get reg dump failed\n"); + return false; + } + + kbase_job_fault_event_post(kctx->kbdev, katom, + completion_code); + atomic_inc(&kctx->job_fault_count); + dev_info(kctx->kbdev->dev, "post:%d\n", + kbase_jd_atom_id(kctx, katom)); + return true; + + } + } + return false; + +} + +static int debug_job_fault_show(struct seq_file *m, void *v) +{ + struct kbase_device *kbdev = m->private; + struct base_job_fault_event *event = (struct base_job_fault_event *)v; + struct kbase_context *kctx = event->katom->kctx; + int i; + + dev_info(kbdev->dev, "debug job fault seq show:%d_%d, %d", + kctx->tgid, kctx->id, event->reg_offset); + + if (kctx->reg_dump == NULL) { + dev_warn(kbdev->dev, "reg dump is NULL"); + return -1; + } + + if (kctx->reg_dump[event->reg_offset] == + REGISTER_DUMP_TERMINATION_FLAG) { + /* Return the error here to stop the read. And the + * following next() will not be called. The stop can + * get the real event resource and release it + */ + return -1; + } + + if (event->reg_offset == 0) + seq_printf(m, "%d_%d\n", kctx->tgid, kctx->id); + + for (i = 0; i < 50; i++) { + if (kctx->reg_dump[event->reg_offset] == + REGISTER_DUMP_TERMINATION_FLAG) { + break; + } + seq_printf(m, "%08x: %08x\n", + kctx->reg_dump[event->reg_offset], + kctx->reg_dump[1+event->reg_offset]); + event->reg_offset += 2; + + } + + + return 0; +} +static void *debug_job_fault_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct kbase_device *kbdev = m->private; + struct base_job_fault_event *event = (struct base_job_fault_event *)v; + + dev_info(kbdev->dev, "debug job fault seq next:%d, %d", + event->reg_offset, (int)*pos); + + return event; +} + +static void *debug_job_fault_start(struct seq_file *m, loff_t *pos) +{ + struct kbase_device *kbdev = m->private; + struct base_job_fault_event *event; + + dev_info(kbdev->dev, "fault job seq start:%d", (int)*pos); + + /* The condition is trick here. It needs make sure the + * fault hasn't happened and the dumping hasn't been started, + * or the dumping has finished + */ + if (*pos == 0) { + event = kmalloc(sizeof(*event), GFP_KERNEL); + if (!event) + return NULL; + event->reg_offset = 0; + if (kbase_job_fault_event_wait(kbdev, event)) { + kfree(event); + return NULL; + } + + /* The cache flush workaround is called in bottom half of + * job done but we delayed it. Now we should clean cache + * earlier. Then the GPU memory dump should be correct. + */ + kbase_backend_cacheclean(kbdev, event->katom); + } else + return NULL; + + return event; +} + +static void debug_job_fault_stop(struct seq_file *m, void *v) +{ + struct kbase_device *kbdev = m->private; + + /* here we wake up the kbase_jd_done_worker after stop, it needs + * get the memory dump before the register dump in debug daemon, + * otherwise, the memory dump may be incorrect. + */ + + if (v != NULL) { + kfree(v); + dev_info(kbdev->dev, "debug job fault seq stop stage 1"); + + } else { + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + if (!list_empty(&kbdev->job_fault_event_list)) { + kbase_job_fault_event_dequeue(kbdev, + &kbdev->job_fault_event_list); + wake_up(&kbdev->job_fault_resume_wq); + } + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + dev_info(kbdev->dev, "debug job fault seq stop stage 2"); + } + +} + +static const struct seq_operations ops = { + .start = debug_job_fault_start, + .next = debug_job_fault_next, + .stop = debug_job_fault_stop, + .show = debug_job_fault_show, +}; + +static int debug_job_fault_open(struct inode *in, struct file *file) +{ + struct kbase_device *kbdev = in->i_private; + + seq_open(file, &ops); + + ((struct seq_file *)file->private_data)->private = kbdev; + dev_info(kbdev->dev, "debug job fault seq open"); + + kbdev->job_fault_debug = true; + + return 0; + +} + +static int debug_job_fault_release(struct inode *in, struct file *file) +{ + struct kbase_device *kbdev = in->i_private; + + seq_release(in, file); + + kbdev->job_fault_debug = false; + + /* Clean the unprocessed job fault. After that, all the suspended + * contexts could be rescheduled. + */ + kbase_job_fault_event_cleanup(kbdev); + + dev_info(kbdev->dev, "debug job fault seq close"); + + return 0; +} + +static const struct file_operations kbasep_debug_job_fault_fops = { + .open = debug_job_fault_open, + .read = seq_read, + .llseek = seq_lseek, + .release = debug_job_fault_release, +}; + +/* + * Initialize debugfs entry for job fault dump + */ +void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("job_fault", S_IRUGO, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_debug_job_fault_fops); +} + + +int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) +{ + + INIT_LIST_HEAD(&kbdev->job_fault_event_list); + + init_waitqueue_head(&(kbdev->job_fault_wq)); + init_waitqueue_head(&(kbdev->job_fault_resume_wq)); + spin_lock_init(&kbdev->job_fault_event_lock); + + kbdev->job_fault_resume_workq = alloc_workqueue( + "kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1); + if (!kbdev->job_fault_resume_workq) + return -ENOMEM; + + kbdev->job_fault_debug = false; + + return 0; +} + +/* + * Release the relevant resource per device + */ +void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) +{ + destroy_workqueue(kbdev->job_fault_resume_workq); +} + + +/* + * Initialize the relevant data structure per context + */ +void kbase_debug_job_fault_context_init(struct kbase_context *kctx) +{ + + /* We need allocate double size register range + * Because this memory will keep the register address and value + */ + kctx->reg_dump = vmalloc(0x4000 * 2); + if (kctx->reg_dump == NULL) + return; + + if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { + vfree(kctx->reg_dump); + kctx->reg_dump = NULL; + } + INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); + atomic_set(&kctx->job_fault_count, 0); + +} + +/* + * release the relevant resource per context + */ +void kbase_debug_job_fault_context_term(struct kbase_context *kctx) +{ + vfree(kctx->reg_dump); +} + +#else /* CONFIG_DEBUG_FS */ + +int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) +{ + kbdev->job_fault_debug = false; + + return 0; +} + +void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) +{ +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.h new file mode 100755 index 000000000000..a2bf8983c37c --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_job_fault.h @@ -0,0 +1,96 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_DEBUG_JOB_FAULT_H +#define _KBASE_DEBUG_JOB_FAULT_H + +#include +#include + +#define REGISTER_DUMP_TERMINATION_FLAG 0xFFFFFFFF + +/** + * kbase_debug_job_fault_dev_init - Create the fault event wait queue + * per device and initialize the required lists. + * @kbdev: Device pointer + * + * Return: Zero on success or a negative error code. + */ +int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev); + +/** + * kbase_debug_job_fault_debugfs_init - Initialize job fault debug sysfs + * @kbdev: Device pointer + */ +void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev); + +/** + * kbase_debug_job_fault_dev_term - Clean up resources created in + * kbase_debug_job_fault_dev_init. + * @kbdev: Device pointer + */ +void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); + +/** + * kbase_debug_job_fault_context_init - Initialize the relevant + * data structure per context + * @kctx: KBase context pointer + */ +void kbase_debug_job_fault_context_init(struct kbase_context *kctx); + +/** + * kbase_debug_job_fault_context_term - Release the relevant + * resource per context + * @kctx: KBase context pointer + */ +void kbase_debug_job_fault_context_term(struct kbase_context *kctx); + +/** + * kbase_debug_job_fault_process - Process the failed job. + * It will send a event and wake up the job fault waiting queue + * Then create a work queue to wait for job dump finish + * This function should be called in the interrupt handler and before + * jd_done that make sure the jd_done_worker will be delayed until the + * job dump finish + * @katom: The failed atom pointer + * @completion_code: the job status + * @return true if dump is going on + */ +bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, + u32 completion_code); + + +/** + * kbase_debug_job_fault_reg_snapshot_init - Set the interested registers + * address during the job fault process, the relevant registers will + * be saved when a job fault happen + * @kctx: KBase context pointer + * @reg_range: Maximum register address space + * @return true if initializing successfully + */ +bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, + int reg_range); + +/** + * kbase_job_fault_get_reg_snapshot - Read the interested registers for + * failed job dump + * @kctx: KBase context pointer + * @return true if getting registers successfully + */ +bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx); + +#endif /*_KBASE_DEBUG_JOB_FAULT_H*/ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.c new file mode 100755 index 000000000000..aa271566e917 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.c @@ -0,0 +1,306 @@ +/* + * + * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Debugfs interface to dump the memory visible to the GPU + */ + +#include "mali_kbase_debug_mem_view.h" +#include "mali_kbase.h" + +#include +#include + +#ifdef CONFIG_DEBUG_FS + +struct debug_mem_mapping { + struct list_head node; + + struct kbase_mem_phy_alloc *alloc; + unsigned long flags; + + u64 start_pfn; + size_t nr_pages; +}; + +struct debug_mem_data { + struct list_head mapping_list; + struct kbase_context *kctx; +}; + +struct debug_mem_seq_off { + struct list_head *lh; + size_t offset; +}; + +static void *debug_mem_start(struct seq_file *m, loff_t *_pos) +{ + struct debug_mem_data *mem_data = m->private; + struct debug_mem_seq_off *data; + struct debug_mem_mapping *map; + loff_t pos = *_pos; + + list_for_each_entry(map, &mem_data->mapping_list, node) { + if (pos >= map->nr_pages) { + pos -= map->nr_pages; + } else { + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + data->lh = &map->node; + data->offset = pos; + return data; + } + } + + /* Beyond the end */ + return NULL; +} + +static void debug_mem_stop(struct seq_file *m, void *v) +{ + kfree(v); +} + +static void *debug_mem_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct debug_mem_data *mem_data = m->private; + struct debug_mem_seq_off *data = v; + struct debug_mem_mapping *map; + + map = list_entry(data->lh, struct debug_mem_mapping, node); + + if (data->offset < map->nr_pages - 1) { + data->offset++; + ++*pos; + return data; + } + + if (list_is_last(data->lh, &mem_data->mapping_list)) { + kfree(data); + return NULL; + } + + data->lh = data->lh->next; + data->offset = 0; + ++*pos; + + return data; +} + +static int debug_mem_show(struct seq_file *m, void *v) +{ + struct debug_mem_data *mem_data = m->private; + struct debug_mem_seq_off *data = v; + struct debug_mem_mapping *map; + int i, j; + struct page *page; + uint32_t *mapping; + pgprot_t prot = PAGE_KERNEL; + + map = list_entry(data->lh, struct debug_mem_mapping, node); + + kbase_gpu_vm_lock(mem_data->kctx); + + if (data->offset >= map->alloc->nents) { + seq_printf(m, "%016llx: Unbacked page\n\n", (map->start_pfn + + data->offset) << PAGE_SHIFT); + goto out; + } + + if (!(map->flags & KBASE_REG_CPU_CACHED)) + prot = pgprot_writecombine(prot); + + page = phys_to_page(as_phys_addr_t(map->alloc->pages[data->offset])); + mapping = vmap(&page, 1, VM_MAP, prot); + if (!mapping) + goto out; + + for (i = 0; i < PAGE_SIZE; i += 4*sizeof(*mapping)) { + seq_printf(m, "%016llx:", i + ((map->start_pfn + + data->offset) << PAGE_SHIFT)); + + for (j = 0; j < 4*sizeof(*mapping); j += sizeof(*mapping)) + seq_printf(m, " %08x", mapping[(i+j)/sizeof(*mapping)]); + seq_putc(m, '\n'); + } + + vunmap(mapping); + + seq_putc(m, '\n'); + +out: + kbase_gpu_vm_unlock(mem_data->kctx); + return 0; +} + +static const struct seq_operations ops = { + .start = debug_mem_start, + .next = debug_mem_next, + .stop = debug_mem_stop, + .show = debug_mem_show, +}; + +static int debug_mem_zone_open(struct rb_root *rbtree, + struct debug_mem_data *mem_data) +{ + int ret = 0; + struct rb_node *p; + struct kbase_va_region *reg; + struct debug_mem_mapping *mapping; + + for (p = rb_first(rbtree); p; p = rb_next(p)) { + reg = rb_entry(p, struct kbase_va_region, rblink); + + if (reg->gpu_alloc == NULL) + /* Empty region - ignore */ + continue; + + mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) { + ret = -ENOMEM; + goto out; + } + + mapping->alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + mapping->start_pfn = reg->start_pfn; + mapping->nr_pages = reg->nr_pages; + mapping->flags = reg->flags; + list_add_tail(&mapping->node, &mem_data->mapping_list); + } + +out: + return ret; +} + +static int debug_mem_open(struct inode *i, struct file *file) +{ + struct file *kctx_file = i->i_private; + struct kbase_context *kctx = kctx_file->private_data; + struct debug_mem_data *mem_data; + int ret; + + ret = seq_open(file, &ops); + if (ret) + return ret; + + mem_data = kmalloc(sizeof(*mem_data), GFP_KERNEL); + if (!mem_data) { + ret = -ENOMEM; + goto out; + } + + mem_data->kctx = kctx; + + INIT_LIST_HEAD(&mem_data->mapping_list); + + get_file(kctx_file); + + kbase_gpu_vm_lock(kctx); + + ret = debug_mem_zone_open(&kctx->reg_rbtree_same, mem_data); + if (0 != ret) { + kbase_gpu_vm_unlock(kctx); + goto out; + } + + ret = debug_mem_zone_open(&kctx->reg_rbtree_exec, mem_data); + if (0 != ret) { + kbase_gpu_vm_unlock(kctx); + goto out; + } + + ret = debug_mem_zone_open(&kctx->reg_rbtree_custom, mem_data); + if (0 != ret) { + kbase_gpu_vm_unlock(kctx); + goto out; + } + + kbase_gpu_vm_unlock(kctx); + + ((struct seq_file *)file->private_data)->private = mem_data; + + return 0; + +out: + if (mem_data) { + while (!list_empty(&mem_data->mapping_list)) { + struct debug_mem_mapping *mapping; + + mapping = list_first_entry(&mem_data->mapping_list, + struct debug_mem_mapping, node); + kbase_mem_phy_alloc_put(mapping->alloc); + list_del(&mapping->node); + kfree(mapping); + } + fput(kctx_file); + kfree(mem_data); + } + seq_release(i, file); + return ret; +} + +static int debug_mem_release(struct inode *inode, struct file *file) +{ + struct file *kctx_file = inode->i_private; + struct seq_file *sfile = file->private_data; + struct debug_mem_data *mem_data = sfile->private; + struct debug_mem_mapping *mapping; + + seq_release(inode, file); + + while (!list_empty(&mem_data->mapping_list)) { + mapping = list_first_entry(&mem_data->mapping_list, + struct debug_mem_mapping, node); + kbase_mem_phy_alloc_put(mapping->alloc); + list_del(&mapping->node); + kfree(mapping); + } + + kfree(mem_data); + + fput(kctx_file); + + return 0; +} + +static const struct file_operations kbase_debug_mem_view_fops = { + .open = debug_mem_open, + .release = debug_mem_release, + .read = seq_read, + .llseek = seq_lseek +}; + +/** + * kbase_debug_mem_view_init - Initialise the mem_view sysfs file + * @kctx_file: The /dev/mali0 file instance for the context + * + * This function creates a "mem_view" file which can be used to get a view of + * the context's memory as the GPU sees it (i.e. using the GPU's page tables). + * + * The file is cleaned up by a call to debugfs_remove_recursive() deleting the + * parent directory. + */ +void kbase_debug_mem_view_init(struct file *kctx_file) +{ + struct kbase_context *kctx = kctx_file->private_data; + + debugfs_create_file("mem_view", S_IRUGO, kctx->kctx_dentry, kctx_file, + &kbase_debug_mem_view_fops); +} + +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.h new file mode 100755 index 000000000000..20ab51a776c6 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_debug_mem_view.h @@ -0,0 +1,25 @@ +/* + * + * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_DEBUG_MEM_VIEW_H +#define _KBASE_DEBUG_MEM_VIEW_H + +#include + +void kbase_debug_mem_view_init(struct file *kctx_file); + +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_defs.h new file mode 100755 index 000000000000..73721f5da139 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_defs.h @@ -0,0 +1,1641 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_defs.h + * + * Defintions (types, defines, etcs) common to Kbase. They are placed here to + * allow the hierarchy of header files to work. + */ + +#ifndef _KBASE_DEFS_H_ +#define _KBASE_DEFS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifdef CONFIG_MALI_FPGA_BUS_LOGGER +#include +#endif + + +#ifdef CONFIG_KDS +#include +#endif /* CONFIG_KDS */ + +#if defined(CONFIG_SYNC) +#include +#else +#include "mali_kbase_fence_defs.h" +#endif + +#ifdef CONFIG_DEBUG_FS +#include +#endif /* CONFIG_DEBUG_FS */ + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ +#include +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + +#include +#include + +#if defined(CONFIG_PM_RUNTIME) || \ + (defined(CONFIG_PM) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) +#define KBASE_PM_RUNTIME 1 +#endif + +/** Enable SW tracing when set */ +#ifdef CONFIG_MALI_BIFROST_ENABLE_TRACE +#define KBASE_TRACE_ENABLE 1 +#endif + +#ifndef KBASE_TRACE_ENABLE +#ifdef CONFIG_MALI_BIFROST_DEBUG +#define KBASE_TRACE_ENABLE 1 +#else +#define KBASE_TRACE_ENABLE 0 +#endif /* CONFIG_MALI_BIFROST_DEBUG */ +#endif /* KBASE_TRACE_ENABLE */ + +/** Dump Job slot trace on error (only active if KBASE_TRACE_ENABLE != 0) */ +#define KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR 1 + +/** + * Number of milliseconds before resetting the GPU when a job cannot be "zapped" from the hardware. + * Note that the time is actually ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and the GPU + * actually being reset to give other contexts time for their jobs to be soft-stopped and removed from the hardware + * before resetting. + */ +#define ZAP_TIMEOUT 1000 + +/** Number of milliseconds before we time out on a GPU soft/hard reset */ +#define RESET_TIMEOUT 500 + +/** + * Prevent soft-stops from occuring in scheduling situations + * + * This is not due to HW issues, but when scheduling is desired to be more predictable. + * + * Therefore, soft stop may still be disabled due to HW issues. + * + * @note Soft stop will still be used for non-scheduling purposes e.g. when terminating a context. + * + * @note if not in use, define this value to 0 instead of \#undef'ing it + */ +#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0 + +/** + * Prevent hard-stops from occuring in scheduling situations + * + * This is not due to HW issues, but when scheduling is desired to be more predictable. + * + * @note Hard stop will still be used for non-scheduling purposes e.g. when terminating a context. + * + * @note if not in use, define this value to 0 instead of \#undef'ing it + */ +#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 + +/** + * The maximum number of Job Slots to support in the Hardware. + * + * You can optimize this down if your target devices will only ever support a + * small number of job slots. + */ +#define BASE_JM_MAX_NR_SLOTS 3 + +/** + * The maximum number of Address Spaces to support in the Hardware. + * + * You can optimize this down if your target devices will only ever support a + * small number of Address Spaces + */ +#define BASE_MAX_NR_AS 16 + +/* mmu */ +#define MIDGARD_MMU_VA_BITS 48 + +#define MIDGARD_MMU_LEVEL(x) (x) + +#if MIDGARD_MMU_VA_BITS > 39 +#define MIDGARD_MMU_TOPLEVEL MIDGARD_MMU_LEVEL(0) +#else +#define MIDGARD_MMU_TOPLEVEL MIDGARD_MMU_LEVEL(1) +#endif + +#define MIDGARD_MMU_BOTTOMLEVEL MIDGARD_MMU_LEVEL(3) + +#define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW | KBASE_REG_GPU_WR) + +/** setting in kbase_context::as_nr that indicates it's invalid */ +#define KBASEP_AS_NR_INVALID (-1) + +#define KBASE_LOCK_REGION_MAX_SIZE (63) +#define KBASE_LOCK_REGION_MIN_SIZE (11) + +#define KBASE_TRACE_SIZE_LOG2 8 /* 256 entries */ +#define KBASE_TRACE_SIZE (1 << KBASE_TRACE_SIZE_LOG2) +#define KBASE_TRACE_MASK ((1 << KBASE_TRACE_SIZE_LOG2)-1) + +#include "mali_kbase_js_defs.h" +#include "mali_kbase_hwaccess_defs.h" + +#define KBASEP_FORCE_REPLAY_DISABLED 0 + +/* Maximum force replay limit when randomization is enabled */ +#define KBASEP_FORCE_REPLAY_RANDOM_LIMIT 16 + +/** Atom has been previously soft-stoppped */ +#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED (1<<1) +/** Atom has been previously retried to execute */ +#define KBASE_KATOM_FLAGS_RERUN (1<<2) +#define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3) +/** Atom has been previously hard-stopped. */ +#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4) +/** Atom has caused us to enter disjoint state */ +#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5) +/* Atom blocked on cross-slot dependency */ +#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7) +/* Atom has fail dependency on cross-slot dependency */ +#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8) +/* Atom is currently in the list of atoms blocked on cross-slot dependencies */ +#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9) +/* Atom is currently holding a context reference */ +#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10) +/* Atom requires GPU to be in protected mode */ +#define KBASE_KATOM_FLAG_PROTECTED (1<<11) +/* Atom has been stored in runnable_tree */ +#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12) + +/* SW related flags about types of JS_COMMAND action + * NOTE: These must be masked off by JS_COMMAND_MASK */ + +/** This command causes a disjoint event */ +#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100 + +/** Bitmask of all SW related flags */ +#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT) + +#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK) +#error JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK. Must update JS_COMMAND_SW_<..> bitmasks +#endif + +/** Soft-stop command that causes a Disjoint event. This of course isn't + * entirely masked off by JS_COMMAND_MASK */ +#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \ + (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP) + +#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT + +/* Serialize atoms within a slot (ie only one atom per job slot) */ +#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0) +/* Serialize atoms between slots (ie only one job slot running at any time) */ +#define KBASE_SERIALIZE_INTER_SLOT (1 << 1) +/* Reset the GPU after each atom completion */ +#define KBASE_SERIALIZE_RESET (1 << 2) + +/* Forward declarations */ +struct kbase_context; +struct kbase_device; +struct kbase_as; +struct kbase_mmu_setup; + +#ifdef CONFIG_DEBUG_FS +struct base_job_fault_event { + + u32 event_code; + struct kbase_jd_atom *katom; + struct work_struct job_fault_work; + struct list_head head; + int reg_offset; +}; + +#endif + +struct kbase_jd_atom_dependency { + struct kbase_jd_atom *atom; + u8 dep_type; +}; + +/** + * struct kbase_io_access - holds information about 1 register access + * + * @addr: first bit indicates r/w (r=0, w=1) + * @value: value written or read + */ +struct kbase_io_access { + uintptr_t addr; + u32 value; +}; + +/** + * struct kbase_io_history - keeps track of all recent register accesses + * + * @enabled: true if register accesses are recorded, false otherwise + * @lock: spinlock protecting kbase_io_access array + * @count: number of registers read/written + * @size: number of elements in kbase_io_access array + * @buf: array of kbase_io_access + */ +struct kbase_io_history { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) + bool enabled; +#else + u32 enabled; +#endif + + spinlock_t lock; + size_t count; + u16 size; + struct kbase_io_access *buf; +}; + +/** + * @brief The function retrieves a read-only reference to the atom field from + * the kbase_jd_atom_dependency structure + * + * @param[in] dep kbase jd atom dependency. + * + * @return readonly reference to dependent ATOM. + */ +static inline const struct kbase_jd_atom * kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep) +{ + LOCAL_ASSERT(dep != NULL); + + return (const struct kbase_jd_atom *)(dep->atom); +} + +/** + * @brief The function retrieves a read-only reference to the dependency type field from + * the kbase_jd_atom_dependency structure + * + * @param[in] dep kbase jd atom dependency. + * + * @return A dependency type value. + */ +static inline u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency *dep) +{ + LOCAL_ASSERT(dep != NULL); + + return dep->dep_type; +} + +/** + * @brief Setter macro for dep_atom array entry in kbase_jd_atom + * + * @param[in] dep The kbase jd atom dependency. + * @param[in] a The ATOM to be set as a dependency. + * @param type The ATOM dependency type to be set. + * + */ +static inline void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency *const_dep, + struct kbase_jd_atom *a, u8 type) +{ + struct kbase_jd_atom_dependency *dep; + + LOCAL_ASSERT(const_dep != NULL); + + dep = (struct kbase_jd_atom_dependency *)const_dep; + + dep->atom = a; + dep->dep_type = type; +} + +/** + * @brief Setter macro for dep_atom array entry in kbase_jd_atom + * + * @param[in] dep The kbase jd atom dependency to be cleared. + * + */ +static inline void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency *const_dep) +{ + struct kbase_jd_atom_dependency *dep; + + LOCAL_ASSERT(const_dep != NULL); + + dep = (struct kbase_jd_atom_dependency *)const_dep; + + dep->atom = NULL; + dep->dep_type = BASE_JD_DEP_TYPE_INVALID; +} + +enum kbase_atom_gpu_rb_state { + /* Atom is not currently present in slot ringbuffer */ + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB, + /* Atom is in slot ringbuffer but is blocked on a previous atom */ + KBASE_ATOM_GPU_RB_WAITING_BLOCKED, + /* Atom is in slot ringbuffer but is waiting for a previous protected + * mode transition to complete */ + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV, + /* Atom is in slot ringbuffer but is waiting for proected mode + * transition */ + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION, + /* Atom is in slot ringbuffer but is waiting for cores to become + * available */ + KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE, + /* Atom is in slot ringbuffer but is blocked on affinity */ + KBASE_ATOM_GPU_RB_WAITING_AFFINITY, + /* Atom is in slot ringbuffer and ready to run */ + KBASE_ATOM_GPU_RB_READY, + /* Atom is in slot ringbuffer and has been submitted to the GPU */ + KBASE_ATOM_GPU_RB_SUBMITTED, + /* Atom must be returned to JS as soon as it reaches the head of the + * ringbuffer due to a previous failure */ + KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1 +}; + +enum kbase_atom_enter_protected_state { + /* + * Starting state: + * Check if a transition into protected mode is required. + * + * NOTE: The integer value of this must + * match KBASE_ATOM_EXIT_PROTECTED_CHECK. + */ + KBASE_ATOM_ENTER_PROTECTED_CHECK = 0, + /* Wait for vinstr to suspend. */ + KBASE_ATOM_ENTER_PROTECTED_VINSTR, + /* Wait for the L2 to become idle in preparation for + * the coherency change. */ + KBASE_ATOM_ENTER_PROTECTED_IDLE_L2, + /* End state; + * Prepare coherency change. */ + KBASE_ATOM_ENTER_PROTECTED_FINISHED, +}; + +enum kbase_atom_exit_protected_state { + /* + * Starting state: + * Check if a transition out of protected mode is required. + * + * NOTE: The integer value of this must + * match KBASE_ATOM_ENTER_PROTECTED_CHECK. + */ + KBASE_ATOM_EXIT_PROTECTED_CHECK = 0, + /* Wait for the L2 to become idle in preparation + * for the reset. */ + KBASE_ATOM_EXIT_PROTECTED_IDLE_L2, + /* Issue the protected reset. */ + KBASE_ATOM_EXIT_PROTECTED_RESET, + /* End state; + * Wait for the reset to complete. */ + KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT, +}; + +struct kbase_ext_res { + u64 gpu_address; + struct kbase_mem_phy_alloc *alloc; +}; + +struct kbase_jd_atom { + struct work_struct work; + ktime_t start_timestamp; + + struct base_jd_udata udata; + struct kbase_context *kctx; + + struct list_head dep_head[2]; + struct list_head dep_item[2]; + const struct kbase_jd_atom_dependency dep[2]; + /* List head used during job dispatch job_done processing - as + * dependencies may not be entirely resolved at this point, we need to + * use a separate list head. */ + struct list_head jd_item; + /* true if atom's jd_item is currently on a list. Prevents atom being + * processed twice. */ + bool in_jd_list; + + u16 nr_extres; + struct kbase_ext_res *extres; + + u32 device_nr; + u64 affinity; + u64 jc; + enum kbase_atom_coreref_state coreref_state; +#ifdef CONFIG_KDS + struct list_head node; + struct kds_resource_set *kds_rset; + bool kds_dep_satisfied; +#endif /* CONFIG_KDS */ +#if defined(CONFIG_SYNC) + /* Stores either an input or output fence, depending on soft-job type */ + struct sync_fence *fence; + struct sync_fence_waiter sync_waiter; +#endif /* CONFIG_SYNC */ +#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + struct { + /* Use the functions/API defined in mali_kbase_fence.h to + * when working with this sub struct */ +#if defined(CONFIG_SYNC_FILE) + /* Input fence */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence_in; +#else + struct dma_fence *fence_in; +#endif +#endif + /* This points to the dma-buf output fence for this atom. If + * this is NULL then there is no fence for this atom and the + * following fields related to dma_fence may have invalid data. + * + * The context and seqno fields contain the details for this + * fence. + * + * This fence is signaled when the katom is completed, + * regardless of the event_code of the katom (signal also on + * failure). + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + /* The dma-buf fence context number for this atom. A unique + * context number is allocated to each katom in the context on + * context creation. + */ + unsigned int context; + /* The dma-buf fence sequence number for this atom. This is + * increased every time this katom uses dma-buf fence. + */ + atomic_t seqno; + /* This contains a list of all callbacks set up to wait on + * other fences. This atom must be held back from JS until all + * these callbacks have been called and dep_count have reached + * 0. The initial value of dep_count must be equal to the + * number of callbacks on this list. + * + * This list is protected by jctx.lock. Callbacks are added to + * this list when the atom is built and the wait are set up. + * All the callbacks then stay on the list until all callbacks + * have been called and the atom is queued, or cancelled, and + * then all callbacks are taken off the list and freed. + */ + struct list_head callbacks; + /* Atomic counter of number of outstandind dma-buf fence + * dependencies for this atom. When dep_count reaches 0 the + * atom may be queued. + * + * The special value "-1" may only be set after the count + * reaches 0, while holding jctx.lock. This indicates that the + * atom has been handled, either queued in JS or cancelled. + * + * If anyone but the dma-fence worker sets this to -1 they must + * ensure that any potentially queued worker must have + * completed before allowing the atom to be marked as unused. + * This can be done by flushing the fence work queue: + * kctx->dma_fence.wq. + */ + atomic_t dep_count; + } dma_fence; +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || CONFIG_SYNC_FILE*/ + + /* Note: refer to kbasep_js_atom_retained_state, which will take a copy of some of the following members */ + enum base_jd_event_code event_code; + base_jd_core_req core_req; /**< core requirements */ + /** Job Slot to retry submitting to if submission from IRQ handler failed + * + * NOTE: see if this can be unified into the another member e.g. the event */ + int retry_submit_on_slot; + + u32 ticks; + /* JS atom priority with respect to other atoms on its kctx. */ + int sched_priority; + + int poking; /* BASE_HW_ISSUE_8316 */ + + wait_queue_head_t completed; + enum kbase_jd_atom_state status; +#ifdef CONFIG_GPU_TRACEPOINTS + int work_id; +#endif + /* Assigned after atom is completed. Used to check whether PRLAM-10676 workaround should be applied */ + int slot_nr; + + u32 atom_flags; + + /* Number of times this atom has been retried. Used by replay soft job. + */ + int retry_count; + + enum kbase_atom_gpu_rb_state gpu_rb_state; + + u64 need_cache_flush_cores_retained; + + atomic_t blocked; + + /* Pointer to atom that this atom has same-slot dependency on */ + struct kbase_jd_atom *pre_dep; + /* Pointer to atom that has same-slot dependency on this atom */ + struct kbase_jd_atom *post_dep; + + /* Pointer to atom that this atom has cross-slot dependency on */ + struct kbase_jd_atom *x_pre_dep; + /* Pointer to atom that has cross-slot dependency on this atom */ + struct kbase_jd_atom *x_post_dep; + + /* The GPU's flush count recorded at the time of submission, used for + * the cache flush optimisation */ + u32 flush_id; + + struct kbase_jd_atom_backend backend; +#ifdef CONFIG_DEBUG_FS + struct base_job_fault_event fault_event; +#endif + + /* List head used for three different purposes: + * 1. Overflow list for JS ring buffers. If an atom is ready to run, + * but there is no room in the JS ring buffer, then the atom is put + * on the ring buffer's overflow list using this list node. + * 2. List of waiting soft jobs. + */ + struct list_head queue; + + /* Used to keep track of all JIT free/alloc jobs in submission order + */ + struct list_head jit_node; + bool jit_blocked; + + /* If non-zero, this indicates that the atom will fail with the set + * event_code when the atom is processed. */ + enum base_jd_event_code will_fail_event_code; + + /* Atoms will only ever be transitioning into, or out of + * protected mode so we do not need two separate fields. + */ + union { + enum kbase_atom_enter_protected_state enter; + enum kbase_atom_exit_protected_state exit; + } protected_state; + + struct rb_node runnable_tree_node; + + /* 'Age' of atom relative to other atoms in the context. */ + u32 age; +}; + +static inline bool kbase_jd_katom_is_protected(const struct kbase_jd_atom *katom) +{ + return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED); +} + +/* + * Theory of operations: + * + * Atom objects are statically allocated within the context structure. + * + * Each atom is the head of two lists, one for the "left" set of dependencies, one for the "right" set. + */ + +#define KBASE_JD_DEP_QUEUE_SIZE 256 + +struct kbase_jd_context { + struct mutex lock; + struct kbasep_js_kctx_info sched_info; + struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; + + /** Tracks all job-dispatch jobs. This includes those not tracked by + * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ + u32 job_nr; + + /** Waitq that reflects whether there are no jobs (including SW-only + * dependency jobs). This is set when no jobs are present on the ctx, + * and clear when there are jobs. + * + * @note: Job Dispatcher knows about more jobs than the Job Scheduler: + * the Job Scheduler is unaware of jobs that are blocked on dependencies, + * and SW-only dependency jobs. + * + * This waitq can be waited upon to find out when the context jobs are all + * done/cancelled (including those that might've been blocked on + * dependencies) - and so, whether it can be terminated. However, it should + * only be terminated once it is not present in the run-pool (see + * kbasep_js_kctx_info::ctx::is_scheduled). + * + * Since the waitq is only set under kbase_jd_context::lock, + * the waiter should also briefly obtain and drop kbase_jd_context::lock to + * guarentee that the setter has completed its work on the kbase_context + * + * This must be updated atomically with: + * - kbase_jd_context::job_nr */ + wait_queue_head_t zero_jobs_wait; + + /** Job Done workqueue. */ + struct workqueue_struct *job_done_wq; + + spinlock_t tb_lock; + u32 *tb; + size_t tb_wrap_offset; + +#ifdef CONFIG_KDS + struct kds_callback kds_cb; +#endif /* CONFIG_KDS */ +#ifdef CONFIG_GPU_TRACEPOINTS + atomic_t work_id; +#endif +}; + +struct kbase_device_info { + u32 features; +}; + +/** Poking state for BASE_HW_ISSUE_8316 */ +enum { + KBASE_AS_POKE_STATE_IN_FLIGHT = 1<<0, + KBASE_AS_POKE_STATE_KILLING_POKE = 1<<1 +}; + +/** Poking state for BASE_HW_ISSUE_8316 */ +typedef u32 kbase_as_poke_state; + +struct kbase_mmu_setup { + u64 transtab; + u64 memattr; + u64 transcfg; +}; + +/** + * Important: Our code makes assumptions that a struct kbase_as structure is always at + * kbase_device->as[number]. This is used to recover the containing + * struct kbase_device from a struct kbase_as structure. + * + * Therefore, struct kbase_as structures must not be allocated anywhere else. + */ +struct kbase_as { + int number; + + struct workqueue_struct *pf_wq; + struct work_struct work_pagefault; + struct work_struct work_busfault; + enum kbase_mmu_fault_type fault_type; + bool protected_mode; + u32 fault_status; + u64 fault_addr; + u64 fault_extra_addr; + + struct kbase_mmu_setup current_setup; + + /* BASE_HW_ISSUE_8316 */ + struct workqueue_struct *poke_wq; + struct work_struct poke_work; + /** Protected by hwaccess_lock */ + int poke_refcount; + /** Protected by hwaccess_lock */ + kbase_as_poke_state poke_state; + struct hrtimer poke_timer; +}; + +static inline int kbase_as_has_bus_fault(struct kbase_as *as) +{ + return as->fault_type == KBASE_MMU_FAULT_TYPE_BUS; +} + +static inline int kbase_as_has_page_fault(struct kbase_as *as) +{ + return as->fault_type == KBASE_MMU_FAULT_TYPE_PAGE; +} + +struct kbasep_mem_device { + atomic_t used_pages; /* Tracks usage of OS shared memory. Updated + when OS memory is allocated/freed. */ + +}; + +#define KBASE_TRACE_CODE(X) KBASE_TRACE_CODE_ ## X + +enum kbase_trace_code { + /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE + * THIS MUST BE USED AT THE START OF THE ENUM */ +#define KBASE_TRACE_CODE_MAKE_CODE(X) KBASE_TRACE_CODE(X) +#include "mali_kbase_trace_defs.h" +#undef KBASE_TRACE_CODE_MAKE_CODE + /* Comma on its own, to extend the list */ + , + /* Must be the last in the enum */ + KBASE_TRACE_CODE_COUNT +}; + +#define KBASE_TRACE_FLAG_REFCOUNT (((u8)1) << 0) +#define KBASE_TRACE_FLAG_JOBSLOT (((u8)1) << 1) + +struct kbase_trace { + struct timespec timestamp; + u32 thread_id; + u32 cpu; + void *ctx; + bool katom; + int atom_number; + u64 atom_udata[2]; + u64 gpu_addr; + unsigned long info_val; + u8 code; + u8 jobslot; + u8 refcount; + u8 flags; +}; + +/** Event IDs for the power management framework. + * + * Any of these events might be missed, so they should not be relied upon to + * find the precise state of the GPU at a particular time in the + * trace. Overall, we should get a high percentage of these events for + * statisical purposes, and so a few missing should not be a problem */ +enum kbase_timeline_pm_event { + /* helper for tests */ + KBASEP_TIMELINE_PM_EVENT_FIRST, + + /** Event reserved for backwards compatibility with 'init' events */ + KBASE_TIMELINE_PM_EVENT_RESERVED_0 = KBASEP_TIMELINE_PM_EVENT_FIRST, + + /** The power state of the device has changed. + * + * Specifically, the device has reached a desired or available state. + */ + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED, + + /** The GPU is becoming active. + * + * This event is sent when the first context is about to use the GPU. + */ + KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE, + + /** The GPU is becoming idle. + * + * This event is sent when the last context has finished using the GPU. + */ + KBASE_TIMELINE_PM_EVENT_GPU_IDLE, + + /** Event reserved for backwards compatibility with 'policy_change' + * events */ + KBASE_TIMELINE_PM_EVENT_RESERVED_4, + + /** Event reserved for backwards compatibility with 'system_suspend' + * events */ + KBASE_TIMELINE_PM_EVENT_RESERVED_5, + + /** Event reserved for backwards compatibility with 'system_resume' + * events */ + KBASE_TIMELINE_PM_EVENT_RESERVED_6, + + /** The job scheduler is requesting to power up/down cores. + * + * This event is sent when: + * - powered down cores are needed to complete a job + * - powered up cores are not needed anymore + */ + KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE, + + KBASEP_TIMELINE_PM_EVENT_LAST = KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE, +}; + +#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE +struct kbase_trace_kctx_timeline { + atomic_t jd_atoms_in_flight; + u32 owner_tgid; +}; + +struct kbase_trace_kbdev_timeline { + /* Note: strictly speaking, not needed, because it's in sync with + * kbase_device::jm_slots[]::submitted_nr + * + * But it's kept as an example of how to add global timeline tracking + * information + * + * The caller must hold hwaccess_lock when accessing this */ + u8 slot_atoms_submitted[BASE_JM_MAX_NR_SLOTS]; + + /* Last UID for each PM event */ + atomic_t pm_event_uid[KBASEP_TIMELINE_PM_EVENT_LAST+1]; + /* Counter for generating PM event UIDs */ + atomic_t pm_event_uid_counter; + /* + * L2 transition state - true indicates that the transition is ongoing + * Expected to be protected by hwaccess_lock */ + bool l2_transitioning; +}; +#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ + + +struct kbasep_kctx_list_element { + struct list_head link; + struct kbase_context *kctx; +}; + +/** + * Data stored per device for power management. + * + * This structure contains data for the power management framework. There is one + * instance of this structure per device in the system. + */ +struct kbase_pm_device_data { + /** + * The lock protecting Power Management structures accessed outside of + * IRQ. + * + * This lock must also be held whenever the GPU is being powered on or + * off. + */ + struct mutex lock; + + /** The reference count of active contexts on this device. */ + int active_count; + /** Flag indicating suspending/suspended */ + bool suspending; + /* Wait queue set when active_count == 0 */ + wait_queue_head_t zero_active_count_wait; + + /** + * Bit masks identifying the available shader cores that are specified + * via sysfs. One mask per job slot. + */ + u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS]; + u64 debug_core_mask_all; + + /** + * Callback for initializing the runtime power management. + * + * @param kbdev The kbase device + * + * @return 0 on success, else error code + */ + int (*callback_power_runtime_init)(struct kbase_device *kbdev); + + /** + * Callback for terminating the runtime power management. + * + * @param kbdev The kbase device + */ + void (*callback_power_runtime_term)(struct kbase_device *kbdev); + + /* Time in milliseconds between each dvfs sample */ + u32 dvfs_period; + + /* Period of GPU poweroff timer */ + ktime_t gpu_poweroff_time; + + /* Number of ticks of GPU poweroff timer before shader is powered off */ + int poweroff_shader_ticks; + + /* Number of ticks of GPU poweroff timer before GPU is powered off */ + int poweroff_gpu_ticks; + + struct kbase_pm_backend_data backend; +}; + +/** + * struct kbase_mem_pool - Page based memory pool for kctx/kbdev + * @kbdev: Kbase device where memory is used + * @cur_size: Number of free pages currently in the pool (may exceed @max_size + * in some corner cases) + * @max_size: Maximum number of free pages in the pool + * @order: order = 0 refers to a pool of 4 KB pages + * order = 9 refers to a pool of 2 MB pages (2^9 * 4KB = 2 MB) + * @pool_lock: Lock protecting the pool - must be held when modifying @cur_size + * and @page_list + * @page_list: List of free pages in the pool + * @reclaim: Shrinker for kernel reclaim of free pages + * @next_pool: Pointer to next pool where pages can be allocated when this pool + * is empty. Pages will spill over to the next pool when this pool + * is full. Can be NULL if there is no next pool. + */ +struct kbase_mem_pool { + struct kbase_device *kbdev; + size_t cur_size; + size_t max_size; + size_t order; + spinlock_t pool_lock; + struct list_head page_list; + struct shrinker reclaim; + + struct kbase_mem_pool *next_pool; +}; + +/** + * struct kbase_devfreq_opp - Lookup table for converting between nominal OPP + * frequency, and real frequency and core mask + * @opp_freq: Nominal OPP frequency + * @real_freq: Real GPU frequency + * @core_mask: Shader core mask + */ +struct kbase_devfreq_opp { + u64 opp_freq; + u64 real_freq; + u64 core_mask; +}; + +struct kbase_mmu_mode { + void (*update)(struct kbase_context *kctx); + void (*get_as_setup)(struct kbase_context *kctx, + struct kbase_mmu_setup * const setup); + void (*disable_as)(struct kbase_device *kbdev, int as_nr); + phys_addr_t (*pte_to_phy_addr)(u64 entry); + int (*ate_is_valid)(u64 ate, unsigned int level); + int (*pte_is_valid)(u64 pte, unsigned int level); + void (*entry_set_ate)(u64 *entry, struct tagged_addr phy, + unsigned long flags, unsigned int level); + void (*entry_set_pte)(u64 *entry, phys_addr_t phy); + void (*entry_invalidate)(u64 *entry); +}; + +struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void); +struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void); + + +#define DEVNAME_SIZE 16 + +struct kbase_device { + s8 slot_submit_count_irq[BASE_JM_MAX_NR_SLOTS]; + + u32 hw_quirks_sc; + u32 hw_quirks_tiler; + u32 hw_quirks_mmu; + u32 hw_quirks_jm; + + struct list_head entry; + struct device *dev; + struct miscdevice mdev; + u64 reg_start; + size_t reg_size; + void __iomem *reg; + + struct { + int irq; + int flags; + } irqs[3]; + + struct clk *clock; +#ifdef CONFIG_REGULATOR + struct regulator *regulator; +#endif + char devname[DEVNAME_SIZE]; + +#ifdef CONFIG_MALI_BIFROST_NO_MALI + void *model; + struct kmem_cache *irq_slab; + struct workqueue_struct *irq_workq; + atomic_t serving_job_irq; + atomic_t serving_gpu_irq; + atomic_t serving_mmu_irq; + spinlock_t reg_op_lock; +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ + + struct kbase_pm_device_data pm; + struct kbasep_js_device_data js_data; + struct kbase_mem_pool mem_pool; + struct kbase_mem_pool lp_mem_pool; + struct kbasep_mem_device memdev; + struct kbase_mmu_mode const *mmu_mode; + + struct kbase_as as[BASE_MAX_NR_AS]; + /* The below variables (as_free and as_to_kctx) are managed by the + * Context Scheduler. The kbasep_js_device_data::runpool_irq::lock must + * be held whilst accessing these. + */ + u16 as_free; /* Bitpattern of free Address Spaces */ + /* Mapping from active Address Spaces to kbase_context */ + struct kbase_context *as_to_kctx[BASE_MAX_NR_AS]; + + + spinlock_t mmu_mask_change; + + struct kbase_gpu_props gpu_props; + + /** List of SW workarounds for HW issues */ + unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; + /** List of features available */ + unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; + + /* Bitmaps of cores that are currently in use (running jobs). + * These should be kept up to date by the job scheduler. + * + * pm.power_change_lock should be held when accessing these members. + * + * kbase_pm_check_transitions_nolock() should be called when bits are + * cleared to update the power management system and allow transitions to + * occur. */ + u64 shader_inuse_bitmap; + + /* Refcount for cores in use */ + u32 shader_inuse_cnt[64]; + + /* Bitmaps of cores the JS needs for jobs ready to run */ + u64 shader_needed_bitmap; + + /* Refcount for cores needed */ + u32 shader_needed_cnt[64]; + + u32 tiler_inuse_cnt; + + u32 tiler_needed_cnt; + + /* struct for keeping track of the disjoint information + * + * The state is > 0 if the GPU is in a disjoint state. Otherwise 0 + * The count is the number of disjoint events that have occurred on the GPU + */ + struct { + atomic_t count; + atomic_t state; + } disjoint_event; + + /* Refcount for tracking users of the l2 cache, e.g. when using hardware counter instrumentation. */ + u32 l2_users_count; + + /* Bitmaps of cores that are currently available (powered up and the power policy is happy for jobs to be + * submitted to these cores. These are updated by the power management code. The job scheduler should avoid + * submitting new jobs to any cores that are not marked as available. + * + * pm.power_change_lock should be held when accessing these members. + */ + u64 shader_available_bitmap; + u64 tiler_available_bitmap; + u64 l2_available_bitmap; + u64 stack_available_bitmap; + + u64 shader_ready_bitmap; + u64 shader_transitioning_bitmap; + + s8 nr_hw_address_spaces; /**< Number of address spaces in the GPU (constant after driver initialisation) */ + s8 nr_user_address_spaces; /**< Number of address spaces available to user contexts */ + + /* Structure used for instrumentation and HW counters dumping */ + struct kbase_hwcnt { + /* The lock should be used when accessing any of the following members */ + spinlock_t lock; + + struct kbase_context *kctx; + u64 addr; + + struct kbase_instr_backend backend; + } hwcnt; + + struct kbase_vinstr_context *vinstr_ctx; + +#if KBASE_TRACE_ENABLE + spinlock_t trace_lock; + u16 trace_first_out; + u16 trace_next_in; + struct kbase_trace *trace_rbuf; +#endif + + u32 reset_timeout_ms; + + struct mutex cacheclean_lock; + + /* Platform specific private data to be accessed by mali_kbase_config_xxx.c only */ + void *platform_context; + + /* List of kbase_contexts created */ + struct list_head kctx_list; + struct mutex kctx_list_lock; + +#ifdef CONFIG_MALI_BIFROST_DEVFREQ + struct devfreq_dev_profile devfreq_profile; + struct devfreq *devfreq; + unsigned long current_freq; + unsigned long current_nominal_freq; + unsigned long current_voltage; + u64 current_core_mask; + struct kbase_devfreq_opp *opp_table; + int num_opps; + struct monitor_dev_info *mdev_info; + struct ipa_power_model_data *model_data; +#ifdef CONFIG_DEVFREQ_THERMAL +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + struct devfreq_cooling_device *devfreq_cooling; +#else + struct thermal_cooling_device *devfreq_cooling; +#endif + /* Current IPA model - true for configured model, false for fallback */ + atomic_t ipa_use_configured_model; + struct { + /* Access to this struct must be with ipa.lock held */ + struct mutex lock; + struct kbase_ipa_model *configured_model; + struct kbase_ipa_model *fallback_model; + } ipa; +#endif /* CONFIG_DEVFREQ_THERMAL */ +#endif /* CONFIG_MALI_BIFROST_DEVFREQ */ + + +#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE + struct kbase_trace_kbdev_timeline timeline; +#endif + + /* + * Control for enabling job dump on failure, set when control debugfs + * is opened. + */ + bool job_fault_debug; + +#ifdef CONFIG_DEBUG_FS + /* directory for debugfs entries */ + struct dentry *mali_debugfs_directory; + /* Root directory for per context entry */ + struct dentry *debugfs_ctx_directory; + +#ifdef CONFIG_MALI_BIFROST_DEBUG + /* bit for each as, set if there is new data to report */ + u64 debugfs_as_read_bitmap; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + /* failed job dump, used for separate debug process */ + wait_queue_head_t job_fault_wq; + wait_queue_head_t job_fault_resume_wq; + struct workqueue_struct *job_fault_resume_workq; + struct list_head job_fault_event_list; + spinlock_t job_fault_event_lock; + struct kbase_context *kctx_fault; + +#if !MALI_CUSTOMER_RELEASE + /* Per-device data for register dumping interface */ + struct { + u16 reg_offset; /* Offset of a GPU_CONTROL register to be + dumped upon request */ + } regs_dump_debugfs_data; +#endif /* !MALI_CUSTOMER_RELEASE */ +#endif /* CONFIG_DEBUG_FS */ + + /* fbdump profiling controls set by gator */ + u32 kbase_profiling_controls[FBDUMP_CONTROL_MAX]; + + +#if MALI_CUSTOMER_RELEASE == 0 + /* Number of jobs that are run before a job is forced to fail and + * replay. May be KBASEP_FORCE_REPLAY_DISABLED, to disable forced + * failures. */ + int force_replay_limit; + /* Count of jobs between forced failures. Incremented on each job. A + * job is forced to fail once this is greater than or equal to + * force_replay_limit. */ + int force_replay_count; + /* Core requirement for jobs to be failed and replayed. May be zero. */ + base_jd_core_req force_replay_core_req; + /* true if force_replay_limit should be randomized. The random + * value will be in the range of 1 - KBASEP_FORCE_REPLAY_RANDOM_LIMIT. + */ + bool force_replay_random; +#endif + + /* Total number of created contexts */ + atomic_t ctx_num; + +#ifdef CONFIG_DEBUG_FS + /* Holds the most recent register accesses */ + struct kbase_io_history io_history; +#endif /* CONFIG_DEBUG_FS */ + + struct kbase_hwaccess_data hwaccess; + + /* Count of page/bus faults waiting for workqueues to process */ + atomic_t faults_pending; + + /* true if GPU is powered off or power off operation is in progress */ + bool poweroff_pending; + + + /* defaults for new context created for this device */ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) + bool infinite_cache_active_default; +#else + u32 infinite_cache_active_default; +#endif + size_t mem_pool_max_size_default; + + /* current gpu coherency mode */ + u32 current_gpu_coherency_mode; + /* system coherency mode */ + u32 system_coherency; + /* Flag to track when cci snoops have been enabled on the interface */ + bool cci_snoop_enabled; + + /* SMC function IDs to call into Trusted firmware to enable/disable + * cache snooping. Value of 0 indicates that they are not used + */ + u32 snoop_enable_smc; + u32 snoop_disable_smc; + + /* Protected mode operations */ + struct protected_mode_ops *protected_ops; + + /* Protected device attached to this kbase device */ + struct protected_mode_device *protected_dev; + + /* + * true when GPU is put into protected mode + */ + bool protected_mode; + + /* + * true when GPU is transitioning into or out of protected mode + */ + bool protected_mode_transition; + + /* + * true if protected mode is supported + */ + bool protected_mode_support; + + +#ifdef CONFIG_MALI_BIFROST_DEBUG + wait_queue_head_t driver_inactive_wait; + bool driver_inactive; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + +#ifdef CONFIG_MALI_FPGA_BUS_LOGGER + /* + * Bus logger integration. + */ + struct bus_logger_client *buslogger; +#endif + /* Boolean indicating if an IRQ flush during reset is in progress. */ + bool irq_reset_flush; + + /* list of inited sub systems. Used during terminate/error recovery */ + u32 inited_subsys; + + spinlock_t hwaccess_lock; + + /* Protects access to MMU operations */ + struct mutex mmu_hw_mutex; + + /* Current serialization mode. See KBASE_SERIALIZE_* for details */ + u8 serialize_jobs; +}; + +/** + * struct jsctx_queue - JS context atom queue + * @runnable_tree: Root of RB-tree containing currently runnable atoms on this + * job slot. + * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot + * dependencies. Atoms on this list will be moved to the + * runnable_tree when the blocking atom completes. + * + * hwaccess_lock must be held when accessing this structure. + */ +struct jsctx_queue { + struct rb_root runnable_tree; + struct list_head x_dep_head; +}; + + +#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \ + (((minor) & 0xFFF) << 8) | \ + ((0 & 0xFF) << 0)) + +/** + * enum kbase_context_flags - Flags for kbase contexts + * + * @KCTX_COMPAT: Set when the context process is a compat process, 32-bit + * process on a 64-bit kernel. + * + * @KCTX_RUNNABLE_REF: Set when context is counted in + * kbdev->js_data.nr_contexts_runnable. Must hold queue_mutex when accessing. + * + * @KCTX_ACTIVE: Set when the context is active. + * + * @KCTX_PULLED: Set when last kick() caused atoms to be pulled from this + * context. + * + * @KCTX_MEM_PROFILE_INITIALIZED: Set when the context's memory profile has been + * initialized. + * + * @KCTX_INFINITE_CACHE: Set when infinite cache is to be enabled for new + * allocations. Existing allocations will not change. + * + * @KCTX_SUBMIT_DISABLED: Set to prevent context from submitting any jobs. + * + * @KCTX_PRIVILEGED:Set if the context uses an address space and should be kept + * scheduled in. + * + * @KCTX_SCHEDULED: Set when the context is scheduled on the Run Pool. + * This is only ever updated whilst the jsctx_mutex is held. + * + * @KCTX_DYING: Set when the context process is in the process of being evicted. + * + * @KCTX_NO_IMPLICIT_SYNC: Set when explicit Android fences are in use on this + * context, to disable use of implicit dma-buf fences. This is used to avoid + * potential synchronization deadlocks. + * + * All members need to be separate bits. This enum is intended for use in a + * bitmask where multiple values get OR-ed together. + */ +enum kbase_context_flags { + KCTX_COMPAT = 1U << 0, + KCTX_RUNNABLE_REF = 1U << 1, + KCTX_ACTIVE = 1U << 2, + KCTX_PULLED = 1U << 3, + KCTX_MEM_PROFILE_INITIALIZED = 1U << 4, + KCTX_INFINITE_CACHE = 1U << 5, + KCTX_SUBMIT_DISABLED = 1U << 6, + KCTX_PRIVILEGED = 1U << 7, + KCTX_SCHEDULED = 1U << 8, + KCTX_DYING = 1U << 9, + KCTX_NO_IMPLICIT_SYNC = 1U << 10, +}; + +struct kbase_sub_alloc { + struct list_head link; + struct page *page; + DECLARE_BITMAP(sub_pages, SZ_2M / SZ_4K); +}; + +struct kbase_context { + struct file *filp; + struct kbase_device *kbdev; + u32 id; /* System wide unique id */ + unsigned long api_version; + phys_addr_t pgd; + struct list_head event_list; + struct list_head event_coalesce_list; + struct mutex event_mutex; + atomic_t event_closed; + struct workqueue_struct *event_workq; + atomic_t event_count; + int event_coalesce_count; + + atomic_t flags; + + atomic_t setup_complete; + atomic_t setup_in_progress; + + u64 *mmu_teardown_pages; + + struct tagged_addr aliasing_sink_page; + + struct mutex mem_partials_lock; + struct list_head mem_partials; + + struct mutex mmu_lock; + struct mutex reg_lock; /* To be converted to a rwlock? */ + struct rb_root reg_rbtree_same; /* RB tree of GPU (live) regions, + * SAME_VA zone */ + struct rb_root reg_rbtree_exec; /* RB tree of GPU (live) regions, + * EXEC zone */ + struct rb_root reg_rbtree_custom; /* RB tree of GPU (live) regions, + * CUSTOM_VA zone */ + + unsigned long cookies; + struct kbase_va_region *pending_regions[BITS_PER_LONG]; + + wait_queue_head_t event_queue; + pid_t tgid; + pid_t pid; + + struct kbase_jd_context jctx; + atomic_t used_pages; + atomic_t nonmapped_pages; + + struct kbase_mem_pool mem_pool; + struct kbase_mem_pool lp_mem_pool; + + struct shrinker reclaim; + struct list_head evict_list; + + struct list_head waiting_soft_jobs; + spinlock_t waiting_soft_jobs_lock; +#ifdef CONFIG_KDS + struct list_head waiting_kds_resource; +#endif +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + struct { + struct list_head waiting_resource; + struct workqueue_struct *wq; + } dma_fence; +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + /** This is effectively part of the Run Pool, because it only has a valid + * setting (!=KBASEP_AS_NR_INVALID) whilst the context is scheduled in + * + * The hwaccess_lock must be held whilst accessing this. + * + * If the context relating to this as_nr is required, you must use + * kbasep_js_runpool_retain_ctx() to ensure that the context doesn't disappear + * whilst you're using it. Alternatively, just hold the hwaccess_lock + * to ensure the context doesn't disappear (but this has restrictions on what other locks + * you can take whilst doing this) */ + int as_nr; + + /* Keeps track of the number of users of this context. A user can be a + * job that is available for execution, instrumentation needing to 'pin' + * a context for counter collection, etc. If the refcount reaches 0 then + * this context is considered inactive and the previously programmed + * AS might be cleared at any point. + */ + atomic_t refcount; + + /* NOTE: + * + * Flags are in jctx.sched_info.ctx.flags + * Mutable flags *must* be accessed under jctx.sched_info.ctx.jsctx_mutex + * + * All other flags must be added there */ + spinlock_t mm_update_lock; + struct mm_struct *process_mm; + /* End of the SAME_VA zone */ + u64 same_va_end; + +#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE + struct kbase_trace_kctx_timeline timeline; +#endif +#ifdef CONFIG_DEBUG_FS + /* Content of mem_profile file */ + char *mem_profile_data; + /* Size of @c mem_profile_data */ + size_t mem_profile_size; + /* Mutex guarding memory profile state */ + struct mutex mem_profile_lock; + /* Memory profile directory under debugfs */ + struct dentry *kctx_dentry; + + /* for job fault debug */ + unsigned int *reg_dump; + atomic_t job_fault_count; + /* This list will keep the following atoms during the dump + * in the same context + */ + struct list_head job_fault_resume_event_list; + +#endif /* CONFIG_DEBUG_FS */ + + struct jsctx_queue jsctx_queue + [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS]; + + /* Number of atoms currently pulled from this context */ + atomic_t atoms_pulled; + /* Number of atoms currently pulled from this context, per slot */ + atomic_t atoms_pulled_slot[BASE_JM_MAX_NR_SLOTS]; + /* Number of atoms currently pulled from this context, per slot and + * priority. Hold hwaccess_lock when accessing */ + int atoms_pulled_slot_pri[BASE_JM_MAX_NR_SLOTS][ + KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + + /* true if slot is blocked on the given priority. This will be set on a + * soft-stop */ + bool blocked_js[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + + /* Bitmask of slots that can be pulled from */ + u32 slots_pullable; + + /* Backend specific data */ + struct kbase_context_backend backend; + + /* Work structure used for deferred ASID assignment */ + struct work_struct work; + + /* Only one userspace vinstr client per kbase context */ + struct kbase_vinstr_client *vinstr_cli; + struct mutex vinstr_cli_lock; + + /* List of completed jobs waiting for events to be posted */ + struct list_head completed_jobs; + /* Number of work items currently pending on job_done_wq */ + atomic_t work_count; + + /* Waiting soft-jobs will fail when this timer expires */ + struct timer_list soft_job_timeout; + + /* JIT allocation management */ + struct kbase_va_region *jit_alloc[256]; + struct list_head jit_active_head; + struct list_head jit_pool_head; + struct list_head jit_destroy_head; + struct mutex jit_evict_lock; + struct work_struct jit_work; + + /* A list of the JIT soft-jobs in submission order + * (protected by kbase_jd_context.lock) + */ + struct list_head jit_atoms_head; + /* A list of pending JIT alloc soft-jobs (using the 'queue' list_head) + * (protected by kbase_jd_context.lock) + */ + struct list_head jit_pending_alloc; + + /* External sticky resource management */ + struct list_head ext_res_meta_head; + + /* Used to record that a drain was requested from atomic context */ + atomic_t drain_pending; + + /* Current age count, used to determine age for newly submitted atoms */ + u32 age_count; +}; + +/** + * struct kbase_ctx_ext_res_meta - Structure which binds an external resource + * to a @kbase_context. + * @ext_res_node: List head for adding the metadata to a + * @kbase_context. + * @alloc: The physical memory allocation structure + * which is mapped. + * @gpu_addr: The GPU virtual address the resource is + * mapped to. + * + * External resources can be mapped into multiple contexts as well as the same + * context multiple times. + * As kbase_va_region itself isn't refcounted we can't attach our extra + * information to it as it could be removed under our feet leaving external + * resources pinned. + * This metadata structure binds a single external resource to a single + * context, ensuring that per context mapping is tracked separately so it can + * be overridden when needed and abuses by the application (freeing the resource + * multiple times) don't effect the refcount of the physical allocation. + */ +struct kbase_ctx_ext_res_meta { + struct list_head ext_res_node; + struct kbase_mem_phy_alloc *alloc; + u64 gpu_addr; +}; + +enum kbase_reg_access_type { + REG_READ, + REG_WRITE +}; + +enum kbase_share_attr_bits { + /* (1ULL << 8) bit is reserved */ + SHARE_BOTH_BITS = (2ULL << 8), /* inner and outer shareable coherency */ + SHARE_INNER_BITS = (3ULL << 8) /* inner shareable coherency */ +}; + +/** + * kbase_device_is_cpu_coherent - Returns if the device is CPU coherent. + * @kbdev: kbase device + * + * Return: true if the device access are coherent, false if not. + */ +static inline bool kbase_device_is_cpu_coherent(struct kbase_device *kbdev) +{ + if ((kbdev->system_coherency == COHERENCY_ACE_LITE) || + (kbdev->system_coherency == COHERENCY_ACE)) + return true; + + return false; +} + +/* Conversion helpers for setting up high resolution timers */ +#define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x))*1000000U)) +#define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x)) + +/* Maximum number of loops polling the GPU for a cache flush before we assume it must have completed */ +#define KBASE_CLEAN_CACHE_MAX_LOOPS 100000 +/* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */ +#define KBASE_AS_INACTIVE_MAX_LOOPS 100000 + +/* Maximum number of times a job can be replayed */ +#define BASEP_JD_REPLAY_LIMIT 15 + +/* JobDescriptorHeader - taken from the architecture specifications, the layout + * is currently identical for all GPU archs. */ +struct job_descriptor_header { + u32 exception_status; + u32 first_incomplete_task; + u64 fault_pointer; + u8 job_descriptor_size : 1; + u8 job_type : 7; + u8 job_barrier : 1; + u8 _reserved_01 : 1; + u8 _reserved_1 : 1; + u8 _reserved_02 : 1; + u8 _reserved_03 : 1; + u8 _reserved_2 : 1; + u8 _reserved_04 : 1; + u8 _reserved_05 : 1; + u16 job_index; + u16 job_dependency_index_1; + u16 job_dependency_index_2; + union { + u64 _64; + u32 _32; + } next_job; +}; + +#endif /* _KBASE_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_device.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_device.c new file mode 100755 index 000000000000..2d11f11f3be0 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_device.c @@ -0,0 +1,674 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel device APIs + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +/* NOTE: Magic - 0x45435254 (TRCE in ASCII). + * Supports tracing feature provided in the base module. + * Please keep it in sync with the value of base module. + */ +#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254 + +#if KBASE_TRACE_ENABLE +static const char *kbasep_trace_code_string[] = { + /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE + * THIS MUST BE USED AT THE START OF THE ARRAY */ +#define KBASE_TRACE_CODE_MAKE_CODE(X) # X +#include "mali_kbase_trace_defs.h" +#undef KBASE_TRACE_CODE_MAKE_CODE +}; +#endif + +#define DEBUG_MESSAGE_SIZE 256 + +static int kbasep_trace_init(struct kbase_device *kbdev); +static void kbasep_trace_term(struct kbase_device *kbdev); +static void kbasep_trace_hook_wrapper(void *param); + +struct kbase_device *kbase_device_alloc(void) +{ + return kzalloc(sizeof(struct kbase_device), GFP_KERNEL); +} + +static int kbase_device_as_init(struct kbase_device *kbdev, int i) +{ + const char format[] = "mali_mmu%d"; + char name[sizeof(format)]; + const char poke_format[] = "mali_mmu%d_poker"; + char poke_name[sizeof(poke_format)]; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) + snprintf(poke_name, sizeof(poke_name), poke_format, i); + + snprintf(name, sizeof(name), format, i); + + kbdev->as[i].number = i; + kbdev->as[i].fault_addr = 0ULL; + + kbdev->as[i].pf_wq = alloc_workqueue(name, 0, 1); + if (!kbdev->as[i].pf_wq) + return -EINVAL; + + INIT_WORK(&kbdev->as[i].work_pagefault, page_fault_worker); + INIT_WORK(&kbdev->as[i].work_busfault, bus_fault_worker); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { + struct hrtimer *poke_timer = &kbdev->as[i].poke_timer; + struct work_struct *poke_work = &kbdev->as[i].poke_work; + + kbdev->as[i].poke_wq = alloc_workqueue(poke_name, 0, 1); + if (!kbdev->as[i].poke_wq) { + destroy_workqueue(kbdev->as[i].pf_wq); + return -EINVAL; + } + KBASE_DEBUG_ASSERT(!object_is_on_stack(poke_work)); + INIT_WORK(poke_work, kbasep_as_do_poke); + + hrtimer_init(poke_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + + poke_timer->function = kbasep_as_poke_timer_callback; + + kbdev->as[i].poke_refcount = 0; + kbdev->as[i].poke_state = 0u; + } + + return 0; +} + +static void kbase_device_as_term(struct kbase_device *kbdev, int i) +{ + destroy_workqueue(kbdev->as[i].pf_wq); + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) + destroy_workqueue(kbdev->as[i].poke_wq); +} + +static int kbase_device_all_as_init(struct kbase_device *kbdev) +{ + int i, err; + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + err = kbase_device_as_init(kbdev, i); + if (err) + goto free_workqs; + } + + return 0; + +free_workqs: + for (; i > 0; i--) + kbase_device_as_term(kbdev, i); + + return err; +} + +static void kbase_device_all_as_term(struct kbase_device *kbdev) +{ + int i; + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) + kbase_device_as_term(kbdev, i); +} + +int kbase_device_init(struct kbase_device * const kbdev) +{ + int i, err; +#ifdef CONFIG_ARM64 + struct device_node *np = NULL; +#endif /* CONFIG_ARM64 */ + + spin_lock_init(&kbdev->mmu_mask_change); + mutex_init(&kbdev->mmu_hw_mutex); +#ifdef CONFIG_ARM64 + kbdev->cci_snoop_enabled = false; + np = kbdev->dev->of_node; + if (np != NULL) { + if (of_property_read_u32(np, "snoop_enable_smc", + &kbdev->snoop_enable_smc)) + kbdev->snoop_enable_smc = 0; + if (of_property_read_u32(np, "snoop_disable_smc", + &kbdev->snoop_disable_smc)) + kbdev->snoop_disable_smc = 0; + /* Either both or none of the calls should be provided. */ + if (!((kbdev->snoop_disable_smc == 0 + && kbdev->snoop_enable_smc == 0) + || (kbdev->snoop_disable_smc != 0 + && kbdev->snoop_enable_smc != 0))) { + WARN_ON(1); + err = -EINVAL; + goto fail; + } + } +#endif /* CONFIG_ARM64 */ + /* Get the list of workarounds for issues on the current HW + * (identified by the GPU_ID register) + */ + err = kbase_hw_set_issues_mask(kbdev); + if (err) + goto fail; + + /* Set the list of features available on the current HW + * (identified by the GPU_ID register) + */ + kbase_hw_set_features_mask(kbdev); + + kbase_gpuprops_set_features(kbdev); + + /* On Linux 4.0+, dma coherency is determined from device tree */ +#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) + set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops); +#endif + + /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our + * device structure was created by device-tree + */ + if (!kbdev->dev->dma_mask) + kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask; + + err = dma_set_mask(kbdev->dev, + DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); + if (err) + goto dma_set_mask_failed; + + err = dma_set_coherent_mask(kbdev->dev, + DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); + if (err) + goto dma_set_mask_failed; + + kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces; + + err = kbase_device_all_as_init(kbdev); + if (err) + goto as_init_failed; + + spin_lock_init(&kbdev->hwcnt.lock); + + err = kbasep_trace_init(kbdev); + if (err) + goto term_as; + + mutex_init(&kbdev->cacheclean_lock); + +#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE + for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) + kbdev->timeline.slot_atoms_submitted[i] = 0; + + for (i = 0; i <= KBASEP_TIMELINE_PM_EVENT_LAST; ++i) + atomic_set(&kbdev->timeline.pm_event_uid[i], 0); +#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ + + /* fbdump profiling controls set to 0 - fbdump not enabled until changed by gator */ + for (i = 0; i < FBDUMP_CONTROL_MAX; i++) + kbdev->kbase_profiling_controls[i] = 0; + + kbase_debug_assert_register_hook(&kbasep_trace_hook_wrapper, kbdev); + + atomic_set(&kbdev->ctx_num, 0); + + err = kbase_instr_backend_init(kbdev); + if (err) + goto term_trace; + + kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; + + kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); + else + kbdev->mmu_mode = kbase_mmu_mode_get_lpae(); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + init_waitqueue_head(&kbdev->driver_inactive_wait); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + return 0; +term_trace: + kbasep_trace_term(kbdev); +term_as: + kbase_device_all_as_term(kbdev); +as_init_failed: +dma_set_mask_failed: +fail: + return err; +} + +void kbase_device_term(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + +#if KBASE_TRACE_ENABLE + kbase_debug_assert_register_hook(NULL, NULL); +#endif + + kbase_instr_backend_term(kbdev); + + kbasep_trace_term(kbdev); + + kbase_device_all_as_term(kbdev); +} + +void kbase_device_free(struct kbase_device *kbdev) +{ + kfree(kbdev); +} + +int kbase_device_trace_buffer_install( + struct kbase_context *kctx, u32 *tb, size_t size) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(tb); + + /* Interface uses 16-bit value to track last accessed entry. Each entry + * is composed of two 32-bit words. + * This limits the size that can be handled without an overflow. */ + if (0xFFFF * (2 * sizeof(u32)) < size) + return -EINVAL; + + /* set up the header */ + /* magic number in the first 4 bytes */ + tb[0] = TRACE_BUFFER_HEADER_SPECIAL; + /* Store (write offset = 0, wrap counter = 0, transaction active = no) + * write offset 0 means never written. + * Offsets 1 to (wrap_offset - 1) used to store values when trace started + */ + tb[1] = 0; + + /* install trace buffer */ + spin_lock_irqsave(&kctx->jctx.tb_lock, flags); + kctx->jctx.tb_wrap_offset = size / 8; + kctx->jctx.tb = tb; + spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); + + return 0; +} + +void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(kctx); + spin_lock_irqsave(&kctx->jctx.tb_lock, flags); + kctx->jctx.tb = NULL; + kctx->jctx.tb_wrap_offset = 0; + spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); +} + +void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value) +{ + unsigned long flags; + + spin_lock_irqsave(&kctx->jctx.tb_lock, flags); + if (kctx->jctx.tb) { + u16 wrap_count; + u16 write_offset; + u32 *tb = kctx->jctx.tb; + u32 header_word; + + header_word = tb[1]; + KBASE_DEBUG_ASSERT(0 == (header_word & 0x1)); + + wrap_count = (header_word >> 1) & 0x7FFF; + write_offset = (header_word >> 16) & 0xFFFF; + + /* mark as transaction in progress */ + tb[1] |= 0x1; + mb(); + + /* calculate new offset */ + write_offset++; + if (write_offset == kctx->jctx.tb_wrap_offset) { + /* wrap */ + write_offset = 1; + wrap_count++; + wrap_count &= 0x7FFF; /* 15bit wrap counter */ + } + + /* store the trace entry at the selected offset */ + tb[write_offset * 2 + 0] = (reg_offset & ~0x3) | ((type == REG_WRITE) ? 0x1 : 0x0); + tb[write_offset * 2 + 1] = reg_value; + mb(); + + /* new header word */ + header_word = (write_offset << 16) | (wrap_count << 1) | 0x0; /* transaction complete */ + tb[1] = header_word; + } + spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); +} + +/* + * Device trace functions + */ +#if KBASE_TRACE_ENABLE + +static int kbasep_trace_init(struct kbase_device *kbdev) +{ + struct kbase_trace *rbuf; + + rbuf = kmalloc_array(KBASE_TRACE_SIZE, sizeof(*rbuf), GFP_KERNEL); + + if (!rbuf) + return -EINVAL; + + kbdev->trace_rbuf = rbuf; + spin_lock_init(&kbdev->trace_lock); + return 0; +} + +static void kbasep_trace_term(struct kbase_device *kbdev) +{ + kfree(kbdev->trace_rbuf); +} + +static void kbasep_trace_format_msg(struct kbase_trace *trace_msg, char *buffer, int len) +{ + s32 written = 0; + + /* Initial part of message */ + written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d.%.6d,%d,%d,%s,%p,", (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, kbasep_trace_code_string[trace_msg->code], trace_msg->ctx), 0); + + if (trace_msg->katom) + written += MAX(snprintf(buffer + written, MAX(len - written, 0), "atom %d (ud: 0x%llx 0x%llx)", trace_msg->atom_number, trace_msg->atom_udata[0], trace_msg->atom_udata[1]), 0); + + written += MAX(snprintf(buffer + written, MAX(len - written, 0), ",%.8llx,", trace_msg->gpu_addr), 0); + + /* NOTE: Could add function callbacks to handle different message types */ + /* Jobslot present */ + if (trace_msg->flags & KBASE_TRACE_FLAG_JOBSLOT) + written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->jobslot), 0); + + written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); + + /* Refcount present */ + if (trace_msg->flags & KBASE_TRACE_FLAG_REFCOUNT) + written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->refcount), 0); + + written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); + + /* Rest of message */ + written += MAX(snprintf(buffer + written, MAX(len - written, 0), "0x%.8lx", trace_msg->info_val), 0); +} + +static void kbasep_trace_dump_msg(struct kbase_device *kbdev, struct kbase_trace *trace_msg) +{ + char buffer[DEBUG_MESSAGE_SIZE]; + + kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); + dev_dbg(kbdev->dev, "%s", buffer); +} + +void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val) +{ + unsigned long irqflags; + struct kbase_trace *trace_msg; + + spin_lock_irqsave(&kbdev->trace_lock, irqflags); + + trace_msg = &kbdev->trace_rbuf[kbdev->trace_next_in]; + + /* Fill the message */ + trace_msg->thread_id = task_pid_nr(current); + trace_msg->cpu = task_cpu(current); + + getnstimeofday(&trace_msg->timestamp); + + trace_msg->code = code; + trace_msg->ctx = ctx; + + if (NULL == katom) { + trace_msg->katom = false; + } else { + trace_msg->katom = true; + trace_msg->atom_number = kbase_jd_atom_id(katom->kctx, katom); + trace_msg->atom_udata[0] = katom->udata.blob[0]; + trace_msg->atom_udata[1] = katom->udata.blob[1]; + } + + trace_msg->gpu_addr = gpu_addr; + trace_msg->jobslot = jobslot; + trace_msg->refcount = MIN((unsigned int)refcount, 0xFF); + trace_msg->info_val = info_val; + trace_msg->flags = flags; + + /* Update the ringbuffer indices */ + kbdev->trace_next_in = (kbdev->trace_next_in + 1) & KBASE_TRACE_MASK; + if (kbdev->trace_next_in == kbdev->trace_first_out) + kbdev->trace_first_out = (kbdev->trace_first_out + 1) & KBASE_TRACE_MASK; + + /* Done */ + + spin_unlock_irqrestore(&kbdev->trace_lock, irqflags); +} + +void kbasep_trace_clear(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->trace_lock, flags); + kbdev->trace_first_out = kbdev->trace_next_in; + spin_unlock_irqrestore(&kbdev->trace_lock, flags); +} + +void kbasep_trace_dump(struct kbase_device *kbdev) +{ + unsigned long flags; + u32 start; + u32 end; + + dev_dbg(kbdev->dev, "Dumping trace:\nsecs,nthread,cpu,code,ctx,katom,gpu_addr,jobslot,refcount,info_val"); + spin_lock_irqsave(&kbdev->trace_lock, flags); + start = kbdev->trace_first_out; + end = kbdev->trace_next_in; + + while (start != end) { + struct kbase_trace *trace_msg = &kbdev->trace_rbuf[start]; + + kbasep_trace_dump_msg(kbdev, trace_msg); + + start = (start + 1) & KBASE_TRACE_MASK; + } + dev_dbg(kbdev->dev, "TRACE_END"); + + spin_unlock_irqrestore(&kbdev->trace_lock, flags); + + KBASE_TRACE_CLEAR(kbdev); +} + +static void kbasep_trace_hook_wrapper(void *param) +{ + struct kbase_device *kbdev = (struct kbase_device *)param; + + kbasep_trace_dump(kbdev); +} + +#ifdef CONFIG_DEBUG_FS +struct trace_seq_state { + struct kbase_trace trace_buf[KBASE_TRACE_SIZE]; + u32 start; + u32 end; +}; + +static void *kbasep_trace_seq_start(struct seq_file *s, loff_t *pos) +{ + struct trace_seq_state *state = s->private; + int i; + + if (*pos > KBASE_TRACE_SIZE) + return NULL; + i = state->start + *pos; + if ((state->end >= state->start && i >= state->end) || + i >= state->end + KBASE_TRACE_SIZE) + return NULL; + + i &= KBASE_TRACE_MASK; + + return &state->trace_buf[i]; +} + +static void kbasep_trace_seq_stop(struct seq_file *s, void *data) +{ +} + +static void *kbasep_trace_seq_next(struct seq_file *s, void *data, loff_t *pos) +{ + struct trace_seq_state *state = s->private; + int i; + + (*pos)++; + + i = (state->start + *pos) & KBASE_TRACE_MASK; + if (i == state->end) + return NULL; + + return &state->trace_buf[i]; +} + +static int kbasep_trace_seq_show(struct seq_file *s, void *data) +{ + struct kbase_trace *trace_msg = data; + char buffer[DEBUG_MESSAGE_SIZE]; + + kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); + seq_printf(s, "%s\n", buffer); + return 0; +} + +static const struct seq_operations kbasep_trace_seq_ops = { + .start = kbasep_trace_seq_start, + .next = kbasep_trace_seq_next, + .stop = kbasep_trace_seq_stop, + .show = kbasep_trace_seq_show, +}; + +static int kbasep_trace_debugfs_open(struct inode *inode, struct file *file) +{ + struct kbase_device *kbdev = inode->i_private; + unsigned long flags; + + struct trace_seq_state *state; + + state = __seq_open_private(file, &kbasep_trace_seq_ops, sizeof(*state)); + if (!state) + return -ENOMEM; + + spin_lock_irqsave(&kbdev->trace_lock, flags); + state->start = kbdev->trace_first_out; + state->end = kbdev->trace_next_in; + memcpy(state->trace_buf, kbdev->trace_rbuf, sizeof(state->trace_buf)); + spin_unlock_irqrestore(&kbdev->trace_lock, flags); + + return 0; +} + +static const struct file_operations kbasep_trace_debugfs_fops = { + .open = kbasep_trace_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + +void kbasep_trace_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("mali_trace", S_IRUGO, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_trace_debugfs_fops); +} + +#else +void kbasep_trace_debugfs_init(struct kbase_device *kbdev) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +#else /* KBASE_TRACE_ENABLE */ +static int kbasep_trace_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); + return 0; +} + +static void kbasep_trace_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static void kbasep_trace_hook_wrapper(void *param) +{ + CSTD_UNUSED(param); +} + +void kbasep_trace_dump(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} +#endif /* KBASE_TRACE_ENABLE */ + +void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value) +{ + switch (control) { + case FBDUMP_CONTROL_ENABLE: + /* fall through */ + case FBDUMP_CONTROL_RATE: + /* fall through */ + case SW_COUNTER_ENABLE: + /* fall through */ + case FBDUMP_CONTROL_RESIZE_FACTOR: + kbdev->kbase_profiling_controls[control] = value; + break; + default: + dev_err(kbdev->dev, "Profiling control %d not found\n", control); + break; + } +} + +/* + * Called by gator to control the production of + * profiling information at runtime + * */ + +void _mali_profiling_control(u32 action, u32 value) +{ + struct kbase_device *kbdev = NULL; + + /* find the first i.e. call with -1 */ + kbdev = kbase_find_device(-1); + + if (NULL != kbdev) + kbase_set_profiling_control(kbdev, action, value); +} +KBASE_EXPORT_SYMBOL(_mali_profiling_control); + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_disjoint_events.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_disjoint_events.c new file mode 100755 index 000000000000..f70bcccf4050 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_disjoint_events.c @@ -0,0 +1,76 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Base kernel disjoint events helper functions + */ + +#include + +void kbase_disjoint_init(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + atomic_set(&kbdev->disjoint_event.count, 0); + atomic_set(&kbdev->disjoint_event.state, 0); +} + +/* increment the disjoint event count */ +void kbase_disjoint_event(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + atomic_inc(&kbdev->disjoint_event.count); +} + +/* increment the state and the event counter */ +void kbase_disjoint_state_up(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + atomic_inc(&kbdev->disjoint_event.state); + + kbase_disjoint_event(kbdev); +} + +/* decrement the state */ +void kbase_disjoint_state_down(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(atomic_read(&kbdev->disjoint_event.state) > 0); + + kbase_disjoint_event(kbdev); + + atomic_dec(&kbdev->disjoint_event.state); +} + +/* increments the count only if the state is > 0 */ +void kbase_disjoint_event_potential(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + if (atomic_read(&kbdev->disjoint_event.state)) + kbase_disjoint_event(kbdev); +} + +u32 kbase_disjoint_event_get(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + return atomic_read(&kbdev->disjoint_event.count); +} +KBASE_EXPORT_TEST_API(kbase_disjoint_event_get); diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.c new file mode 100755 index 000000000000..8a571266534b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.c @@ -0,0 +1,449 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* Include mali_kbase_dma_fence.h before checking for CONFIG_MALI_BIFROST_DMA_FENCE as + * it will be set there. + */ +#include "mali_kbase_dma_fence.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static void +kbase_dma_fence_work(struct work_struct *pwork); + +static void +kbase_dma_fence_waiters_add(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + list_add_tail(&katom->queue, &kctx->dma_fence.waiting_resource); +} + +static void +kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom) +{ + list_del(&katom->queue); +} + +static int +kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info, + struct ww_acquire_ctx *ctx) +{ + struct reservation_object *content_res = NULL; + unsigned int content_res_idx = 0; + unsigned int r; + int err = 0; + + ww_acquire_init(ctx, &reservation_ww_class); + +retry: + for (r = 0; r < info->dma_fence_resv_count; r++) { + if (info->resv_objs[r] == content_res) { + content_res = NULL; + continue; + } + + err = ww_mutex_lock(&info->resv_objs[r]->lock, ctx); + if (err) + goto error; + } + + ww_acquire_done(ctx); + return err; + +error: + content_res_idx = r; + + /* Unlock the locked one ones */ + while (r--) + ww_mutex_unlock(&info->resv_objs[r]->lock); + + if (content_res) + ww_mutex_unlock(&content_res->lock); + + /* If we deadlock try with lock_slow and retry */ + if (err == -EDEADLK) { + content_res = info->resv_objs[content_res_idx]; + ww_mutex_lock_slow(&content_res->lock, ctx); + goto retry; + } + + /* If we are here the function failed */ + ww_acquire_fini(ctx); + return err; +} + +static void +kbase_dma_fence_unlock_reservations(struct kbase_dma_fence_resv_info *info, + struct ww_acquire_ctx *ctx) +{ + unsigned int r; + + for (r = 0; r < info->dma_fence_resv_count; r++) + ww_mutex_unlock(&info->resv_objs[r]->lock); + ww_acquire_fini(ctx); +} + +/** + * kbase_dma_fence_queue_work() - Queue work to handle @katom + * @katom: Pointer to atom for which to queue work + * + * Queue kbase_dma_fence_work() for @katom to clean up the fence callbacks and + * submit the atom. + */ +static void +kbase_dma_fence_queue_work(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + bool ret; + + INIT_WORK(&katom->work, kbase_dma_fence_work); + ret = queue_work(kctx->dma_fence.wq, &katom->work); + /* Warn if work was already queued, that should not happen. */ + WARN_ON(!ret); +} + +/** + * kbase_dma_fence_cancel_atom() - Cancels waiting on an atom + * @katom: Katom to cancel + * + * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. + */ +static void +kbase_dma_fence_cancel_atom(struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&katom->kctx->jctx.lock); + + /* Cancel callbacks and clean up. */ + kbase_fence_free_callbacks(katom); + + /* Mark the atom as handled in case all fences signaled just before + * canceling the callbacks and the worker was queued. + */ + kbase_fence_dep_count_set(katom, -1); + + /* Prevent job_done_nolock from being called twice on an atom when + * there is a race between job completion and cancellation. + */ + + if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { + /* Wait was cancelled - zap the atom */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); + } +} + +/** + * kbase_dma_fence_work() - Worker thread called when a fence is signaled + * @pwork: work_struct containing a pointer to a katom + * + * This function will clean and mark all dependencies as satisfied + */ +static void +kbase_dma_fence_work(struct work_struct *pwork) +{ + struct kbase_jd_atom *katom; + struct kbase_jd_context *ctx; + + katom = container_of(pwork, struct kbase_jd_atom, work); + ctx = &katom->kctx->jctx; + + mutex_lock(&ctx->lock); + if (kbase_fence_dep_count_read(katom) != 0) + goto out; + + kbase_fence_dep_count_set(katom, -1); + + /* Remove atom from list of dma-fence waiting atoms. */ + kbase_dma_fence_waiters_remove(katom); + /* Cleanup callbacks. */ + kbase_fence_free_callbacks(katom); + /* + * Queue atom on GPU, unless it has already completed due to a failing + * dependency. Run jd_done_nolock() on the katom if it is completed. + */ + if (unlikely(katom->status == KBASE_JD_ATOM_STATE_COMPLETED)) + jd_done_nolock(katom, NULL); + else + kbase_jd_dep_clear_locked(katom); + +out: + mutex_unlock(&ctx->lock); +} + +static void +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) +#else +kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) +#endif +{ + struct kbase_fence_cb *kcb = container_of(cb, + struct kbase_fence_cb, + fence_cb); + struct kbase_jd_atom *katom = kcb->katom; + + /* If the atom is zapped dep_count will be forced to a negative number + * preventing this callback from ever scheduling work. Which in turn + * would reschedule the atom. + */ + + if (kbase_fence_dep_count_dec_and_test(katom)) + kbase_dma_fence_queue_work(katom); +} + +static int +kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, + struct reservation_object *resv, + bool exclusive) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *excl_fence = NULL; + struct fence **shared_fences = NULL; +#else + struct dma_fence *excl_fence = NULL; + struct dma_fence **shared_fences = NULL; +#endif + unsigned int shared_count = 0; + int err, i; + + err = reservation_object_get_fences_rcu(resv, + &excl_fence, + &shared_count, + &shared_fences); + if (err) + return err; + + if (excl_fence) { + err = kbase_fence_add_callback(katom, + excl_fence, + kbase_dma_fence_cb); + + /* Release our reference, taken by reservation_object_get_fences_rcu(), + * to the fence. We have set up our callback (if that was possible), + * and it's the fence's owner is responsible for singling the fence + * before allowing it to disappear. + */ + dma_fence_put(excl_fence); + + if (err) + goto out; + } + + if (exclusive) { + for (i = 0; i < shared_count; i++) { + err = kbase_fence_add_callback(katom, + shared_fences[i], + kbase_dma_fence_cb); + if (err) + goto out; + } + } + + /* Release all our references to the shared fences, taken by + * reservation_object_get_fences_rcu(). We have set up our callback (if + * that was possible), and it's the fence's owner is responsible for + * signaling the fence before allowing it to disappear. + */ +out: + for (i = 0; i < shared_count; i++) + dma_fence_put(shared_fences[i]); + kfree(shared_fences); + + if (err) { + /* + * On error, cancel and clean up all callbacks that was set up + * before the error. + */ + kbase_fence_free_callbacks(katom); + } + + return err; +} + +void kbase_dma_fence_add_reservation(struct reservation_object *resv, + struct kbase_dma_fence_resv_info *info, + bool exclusive) +{ + unsigned int i; + + for (i = 0; i < info->dma_fence_resv_count; i++) { + /* Duplicate resource, ignore */ + if (info->resv_objs[i] == resv) + return; + } + + info->resv_objs[info->dma_fence_resv_count] = resv; + if (exclusive) + set_bit(info->dma_fence_resv_count, + info->dma_fence_excl_bitmap); + (info->dma_fence_resv_count)++; +} + +int kbase_dma_fence_wait(struct kbase_jd_atom *katom, + struct kbase_dma_fence_resv_info *info) +{ + int err, i; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + struct ww_acquire_ctx ww_ctx; + + lockdep_assert_held(&katom->kctx->jctx.lock); + + fence = kbase_fence_out_new(katom); + if (!fence) { + err = -ENOMEM; + dev_err(katom->kctx->kbdev->dev, + "Error %d creating fence.\n", err); + return err; + } + + kbase_fence_dep_count_set(katom, 1); + + err = kbase_dma_fence_lock_reservations(info, &ww_ctx); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d locking reservations.\n", err); + kbase_fence_dep_count_set(katom, -1); + kbase_fence_out_remove(katom); + return err; + } + + for (i = 0; i < info->dma_fence_resv_count; i++) { + struct reservation_object *obj = info->resv_objs[i]; + + if (!test_bit(i, info->dma_fence_excl_bitmap)) { + err = reservation_object_reserve_shared(obj); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d reserving space for shared fence.\n", err); + goto end; + } + + err = kbase_dma_fence_add_reservation_callback(katom, obj, false); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d adding reservation to callback.\n", err); + goto end; + } + + reservation_object_add_shared_fence(obj, fence); + } else { + err = kbase_dma_fence_add_reservation_callback(katom, obj, true); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d adding reservation to callback.\n", err); + goto end; + } + + reservation_object_add_excl_fence(obj, fence); + } + } + +end: + kbase_dma_fence_unlock_reservations(info, &ww_ctx); + + if (likely(!err)) { + /* Test if the callbacks are already triggered */ + if (kbase_fence_dep_count_dec_and_test(katom)) { + kbase_fence_dep_count_set(katom, -1); + kbase_fence_free_callbacks(katom); + } else { + /* Add katom to the list of dma-buf fence waiting atoms + * only if it is still waiting. + */ + kbase_dma_fence_waiters_add(katom); + } + } else { + /* There was an error, cancel callbacks, set dep_count to -1 to + * indicate that the atom has been handled (the caller will + * kill it for us), signal the fence, free callbacks and the + * fence. + */ + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + kbase_dma_fence_signal(katom); + } + + return err; +} + +void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx) +{ + struct list_head *list = &kctx->dma_fence.waiting_resource; + + while (!list_empty(list)) { + struct kbase_jd_atom *katom; + + katom = list_first_entry(list, struct kbase_jd_atom, queue); + kbase_dma_fence_waiters_remove(katom); + kbase_dma_fence_cancel_atom(katom); + } +} + +void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom) +{ + /* Cancel callbacks and clean up. */ + if (kbase_fence_free_callbacks(katom)) + kbase_dma_fence_queue_work(katom); +} + +void kbase_dma_fence_signal(struct kbase_jd_atom *katom) +{ + if (!katom->dma_fence.fence) + return; + + /* Signal the atom's fence. */ + dma_fence_signal(katom->dma_fence.fence); + + kbase_fence_out_remove(katom); + + kbase_fence_free_callbacks(katom); +} + +void kbase_dma_fence_term(struct kbase_context *kctx) +{ + destroy_workqueue(kctx->dma_fence.wq); + kctx->dma_fence.wq = NULL; +} + +int kbase_dma_fence_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->dma_fence.waiting_resource); + + kctx->dma_fence.wq = alloc_workqueue("mali-fence-%d", + WQ_UNBOUND, 1, kctx->pid); + if (!kctx->dma_fence.wq) + return -ENOMEM; + + return 0; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.h new file mode 100755 index 000000000000..b02ea9774c4f --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_dma_fence.h @@ -0,0 +1,131 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_DMA_FENCE_H_ +#define _KBASE_DMA_FENCE_H_ + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + +#include +#include +#include + + +/* Forward declaration from mali_kbase_defs.h */ +struct kbase_jd_atom; +struct kbase_context; + +/** + * struct kbase_dma_fence_resv_info - Structure with list of reservation objects + * @resv_objs: Array of reservation objects to attach the + * new fence to. + * @dma_fence_resv_count: Number of reservation objects in the array. + * @dma_fence_excl_bitmap: Specifies which resv_obj are exclusive. + * + * This is used by some functions to pass around a collection of data about + * reservation objects. + */ +struct kbase_dma_fence_resv_info { + struct reservation_object **resv_objs; + unsigned int dma_fence_resv_count; + unsigned long *dma_fence_excl_bitmap; +}; + +/** + * kbase_dma_fence_add_reservation() - Adds a resv to the array of resv_objs + * @resv: Reservation object to add to the array. + * @info: Pointer to struct with current reservation info + * @exclusive: Boolean indicating if exclusive access is needed + * + * The function adds a new reservation_object to an existing array of + * reservation_objects. At the same time keeps track of which objects require + * exclusive access in dma_fence_excl_bitmap. + */ +void kbase_dma_fence_add_reservation(struct reservation_object *resv, + struct kbase_dma_fence_resv_info *info, + bool exclusive); + +/** + * kbase_dma_fence_wait() - Creates a new fence and attaches it to the resv_objs + * @katom: Katom with the external dependency. + * @info: Pointer to struct with current reservation info + * + * Return: An error code or 0 if succeeds + */ +int kbase_dma_fence_wait(struct kbase_jd_atom *katom, + struct kbase_dma_fence_resv_info *info); + +/** + * kbase_dma_fence_cancel_ctx() - Cancel all dma-fences blocked atoms on kctx + * @kctx: Pointer to kbase context + * + * This function will cancel and clean up all katoms on @kctx that is waiting + * on dma-buf fences. + * + * Locking: jctx.lock needs to be held when calling this function. + */ +void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx); + +/** + * kbase_dma_fence_cancel_callbacks() - Cancel only callbacks on katom + * @katom: Pointer to katom whose callbacks are to be canceled + * + * This function cancels all dma-buf fence callbacks on @katom, but does not + * cancel the katom itself. + * + * The caller is responsible for ensuring that jd_done_nolock is called on + * @katom. + * + * Locking: jctx.lock must be held when calling this function. + */ +void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom); + +/** + * kbase_dma_fence_signal() - Signal katom's fence and clean up after wait + * @katom: Pointer to katom to signal and clean up + * + * This function will signal the @katom's fence, if it has one, and clean up + * the callback data from the katom's wait on earlier fences. + * + * Locking: jctx.lock must be held while calling this function. + */ +void kbase_dma_fence_signal(struct kbase_jd_atom *katom); + +/** + * kbase_dma_fence_term() - Terminate Mali dma-fence context + * @kctx: kbase context to terminate + */ +void kbase_dma_fence_term(struct kbase_context *kctx); + +/** + * kbase_dma_fence_init() - Initialize Mali dma-fence context + * @kctx: kbase context to initialize + */ +int kbase_dma_fence_init(struct kbase_context *kctx); + + +#else /* CONFIG_MALI_BIFROST_DMA_FENCE */ +/* Dummy functions for when dma-buf fence isn't enabled. */ + +static inline int kbase_dma_fence_init(struct kbase_context *kctx) +{ + return 0; +} + +static inline void kbase_dma_fence_term(struct kbase_context *kctx) {} +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_event.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_event.c new file mode 100755 index 000000000000..188148645f37 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_event.c @@ -0,0 +1,259 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include + +static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + struct base_jd_udata data; + + lockdep_assert_held(&kctx->jctx.lock); + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(katom != NULL); + KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); + + data = katom->udata; + + KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(1, &kctx->timeline.jd_atoms_in_flight)); + + KBASE_TLSTREAM_TL_NRET_ATOM_CTX(katom, kctx); + KBASE_TLSTREAM_TL_DEL_ATOM(katom); + + katom->status = KBASE_JD_ATOM_STATE_UNUSED; + + wake_up(&katom->completed); + + return data; +} + +int kbase_event_pending(struct kbase_context *ctx) +{ + KBASE_DEBUG_ASSERT(ctx); + + return (atomic_read(&ctx->event_count) != 0) || + (atomic_read(&ctx->event_closed) != 0); +} + +KBASE_EXPORT_TEST_API(kbase_event_pending); + +int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent) +{ + struct kbase_jd_atom *atom; + + KBASE_DEBUG_ASSERT(ctx); + + mutex_lock(&ctx->event_mutex); + + if (list_empty(&ctx->event_list)) { + if (!atomic_read(&ctx->event_closed)) { + mutex_unlock(&ctx->event_mutex); + return -1; + } + + /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */ + mutex_unlock(&ctx->event_mutex); + uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED; + memset(&uevent->udata, 0, sizeof(uevent->udata)); + dev_dbg(ctx->kbdev->dev, + "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n", + BASE_JD_EVENT_DRV_TERMINATED); + return 0; + } + + /* normal event processing */ + atomic_dec(&ctx->event_count); + atom = list_entry(ctx->event_list.next, struct kbase_jd_atom, dep_item[0]); + list_del(ctx->event_list.next); + + mutex_unlock(&ctx->event_mutex); + + dev_dbg(ctx->kbdev->dev, "event dequeuing %p\n", (void *)atom); + uevent->event_code = atom->event_code; + uevent->atom_number = (atom - ctx->jctx.atoms); + + if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) + kbase_jd_free_external_resources(atom); + + mutex_lock(&ctx->jctx.lock); + uevent->udata = kbase_event_process(ctx, atom); + mutex_unlock(&ctx->jctx.lock); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_event_dequeue); + +/** + * kbase_event_process_noreport_worker - Worker for processing atoms that do not + * return an event but do have external + * resources + * @data: Work structure + */ +static void kbase_event_process_noreport_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) + kbase_jd_free_external_resources(katom); + + mutex_lock(&kctx->jctx.lock); + kbase_event_process(kctx, katom); + mutex_unlock(&kctx->jctx.lock); +} + +/** + * kbase_event_process_noreport - Process atoms that do not return an event + * @kctx: Context pointer + * @katom: Atom to be processed + * + * Atoms that do not have external resources will be processed immediately. + * Atoms that do have external resources will be processed on a workqueue, in + * order to avoid locking issues. + */ +static void kbase_event_process_noreport(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + INIT_WORK(&katom->work, kbase_event_process_noreport_worker); + queue_work(kctx->event_workq, &katom->work); + } else { + kbase_event_process(kctx, katom); + } +} + +/** + * kbase_event_coalesce - Move pending events to the main event list + * @kctx: Context pointer + * + * kctx->event_list and kctx->event_coalesce_count must be protected + * by a lock unless this is the last thread using them + * (and we're about to terminate the lock). + * + * Return: The number of pending events moved to the main event list + */ +static int kbase_event_coalesce(struct kbase_context *kctx) +{ + const int event_count = kctx->event_coalesce_count; + + /* Join the list of pending events onto the tail of the main list + and reset it */ + list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); + kctx->event_coalesce_count = 0; + + /* Return the number of events moved */ + return event_count; +} + +void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom) +{ + if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) { + if (atom->event_code == BASE_JD_EVENT_DONE) { + /* Don't report the event */ + kbase_event_process_noreport(ctx, atom); + return; + } + } + + if (atom->core_req & BASEP_JD_REQ_EVENT_NEVER) { + /* Don't report the event */ + kbase_event_process_noreport(ctx, atom); + return; + } + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, TL_ATOM_STATE_POSTED); + if (atom->core_req & BASE_JD_REQ_EVENT_COALESCE) { + /* Don't report the event until other event(s) have completed */ + mutex_lock(&ctx->event_mutex); + list_add_tail(&atom->dep_item[0], &ctx->event_coalesce_list); + ++ctx->event_coalesce_count; + mutex_unlock(&ctx->event_mutex); + } else { + /* Report the event and any pending events now */ + int event_count = 1; + + mutex_lock(&ctx->event_mutex); + event_count += kbase_event_coalesce(ctx); + list_add_tail(&atom->dep_item[0], &ctx->event_list); + atomic_add(event_count, &ctx->event_count); + mutex_unlock(&ctx->event_mutex); + + kbase_event_wakeup(ctx); + } +} +KBASE_EXPORT_TEST_API(kbase_event_post); + +void kbase_event_close(struct kbase_context *kctx) +{ + mutex_lock(&kctx->event_mutex); + atomic_set(&kctx->event_closed, true); + mutex_unlock(&kctx->event_mutex); + kbase_event_wakeup(kctx); +} + +int kbase_event_init(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx); + + INIT_LIST_HEAD(&kctx->event_list); + INIT_LIST_HEAD(&kctx->event_coalesce_list); + mutex_init(&kctx->event_mutex); + atomic_set(&kctx->event_count, 0); + kctx->event_coalesce_count = 0; + atomic_set(&kctx->event_closed, false); + kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); + + if (NULL == kctx->event_workq) + return -EINVAL; + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_event_init); + +void kbase_event_cleanup(struct kbase_context *kctx) +{ + int event_count; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(kctx->event_workq); + + flush_workqueue(kctx->event_workq); + destroy_workqueue(kctx->event_workq); + + /* We use kbase_event_dequeue to remove the remaining events as that + * deals with all the cleanup needed for the atoms. + * + * Note: use of kctx->event_list without a lock is safe because this must be the last + * thread using it (because we're about to terminate the lock) + */ + event_count = kbase_event_coalesce(kctx); + atomic_add(event_count, &kctx->event_count); + + while (!list_empty(&kctx->event_list)) { + struct base_jd_event_v2 event; + + kbase_event_dequeue(kctx, &event); + } +} + +KBASE_EXPORT_TEST_API(kbase_event_cleanup); diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.c new file mode 100755 index 000000000000..fcb373372596 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.c @@ -0,0 +1,196 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include +#include +#include + +/* Spin lock protecting all Mali fences as fence->lock. */ +static DEFINE_SPINLOCK(kbase_fence_lock); + +static const char * +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_get_driver_name(struct fence *fence) +#else +kbase_fence_get_driver_name(struct dma_fence *fence) +#endif +{ + return kbase_drv_name; +} + +static const char * +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_get_timeline_name(struct fence *fence) +#else +kbase_fence_get_timeline_name(struct dma_fence *fence) +#endif +{ + return kbase_timeline_name; +} + +static bool +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_enable_signaling(struct fence *fence) +#else +kbase_fence_enable_signaling(struct dma_fence *fence) +#endif +{ + return true; +} + +static void +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_fence_value_str(struct fence *fence, char *str, int size) +#else +kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) +#endif +{ + snprintf(str, size, "%u", fence->seqno); +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +const struct fence_ops kbase_fence_ops = { + .wait = fence_default_wait, +#else +const struct dma_fence_ops kbase_fence_ops = { + .wait = dma_fence_default_wait, +#endif + .get_driver_name = kbase_fence_get_driver_name, + .get_timeline_name = kbase_fence_get_timeline_name, + .enable_signaling = kbase_fence_enable_signaling, + .fence_value_str = kbase_fence_fence_value_str +}; + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +struct fence * +kbase_fence_out_new(struct kbase_jd_atom *katom) +#else +struct dma_fence * +kbase_fence_out_new(struct kbase_jd_atom *katom) +#endif +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + WARN_ON(katom->dma_fence.fence); + + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return NULL; + + dma_fence_init(fence, + &kbase_fence_ops, + &kbase_fence_lock, + katom->dma_fence.context, + atomic_inc_return(&katom->dma_fence.seqno)); + + katom->dma_fence.fence = fence; + + return fence; +} + +bool +kbase_fence_free_callbacks(struct kbase_jd_atom *katom) +{ + struct kbase_fence_cb *cb, *tmp; + bool res = false; + + lockdep_assert_held(&katom->kctx->jctx.lock); + + /* Clean up and free callbacks. */ + list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) { + bool ret; + + /* Cancel callbacks that hasn't been called yet. */ + ret = dma_fence_remove_callback(cb->fence, &cb->fence_cb); + if (ret) { + int ret; + + /* Fence had not signaled, clean up after + * canceling. + */ + ret = atomic_dec_return(&katom->dma_fence.dep_count); + + if (unlikely(ret == 0)) + res = true; + } + + /* + * Release the reference taken in + * kbase_fence_add_callback(). + */ + dma_fence_put(cb->fence); + list_del(&cb->node); + kfree(cb); + } + + return res; +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +int +kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct fence *fence, + fence_func_t callback) +#else +int +kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct dma_fence *fence, + dma_fence_func_t callback) +#endif +{ + int err = 0; + struct kbase_fence_cb *kbase_fence_cb; + + if (!fence) + return -EINVAL; + + kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL); + if (!kbase_fence_cb) + return -ENOMEM; + + kbase_fence_cb->fence = fence; + kbase_fence_cb->katom = katom; + INIT_LIST_HEAD(&kbase_fence_cb->node); + + err = dma_fence_add_callback(fence, &kbase_fence_cb->fence_cb, + callback); + if (err == -ENOENT) { + /* Fence signaled, clear the error and return */ + err = 0; + kfree(kbase_fence_cb); + } else if (err) { + kfree(kbase_fence_cb); + } else { + /* + * Get reference to fence that will be kept until callback gets + * cleaned up in kbase_fence_free_callbacks(). + */ + dma_fence_get(fence); + atomic_inc(&katom->dma_fence.dep_count); + /* Add callback to katom's list of callbacks */ + list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks); + } + + return err; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.h new file mode 100755 index 000000000000..9f59d30a1e2e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence.h @@ -0,0 +1,270 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_FENCE_H_ +#define _KBASE_FENCE_H_ + +/* + * mali_kbase_fence.[hc] has common fence code used by both + * - CONFIG_MALI_BIFROST_DMA_FENCE - implicit DMA fences + * - CONFIG_SYNC_FILE - explicit fences beginning with 4.9 kernel + */ + +#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + +#include +#include "mali_kbase_fence_defs.h" +#include "mali_kbase.h" + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +extern const struct fence_ops kbase_fence_ops; +#else +extern const struct dma_fence_ops kbase_fence_ops; +#endif + +/** +* struct kbase_fence_cb - Mali dma-fence callback data struct +* @fence_cb: Callback function +* @katom: Pointer to katom that is waiting on this callback +* @fence: Pointer to the fence object on which this callback is waiting +* @node: List head for linking this callback to the katom +*/ +struct kbase_fence_cb { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence_cb fence_cb; + struct fence *fence; +#else + struct dma_fence_cb fence_cb; + struct dma_fence *fence; +#endif + struct kbase_jd_atom *katom; + struct list_head node; +}; + +/** + * kbase_fence_out_new() - Creates a new output fence and puts it on the atom + * @katom: Atom to create an output fence for + * + * return: A new fence object on success, NULL on failure. + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); +#else +struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); +#endif + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_fence_in_set() - Assign input fence to atom + * @katom: Atom to assign input fence to + * @fence: Input fence to assign to atom + * + * This function will take ownership of one fence reference! + */ +#define kbase_fence_fence_in_set(katom, fence) \ + do { \ + WARN_ON((katom)->dma_fence.fence_in); \ + (katom)->dma_fence.fence_in = fence; \ + } while (0) +#endif + +/** + * kbase_fence_out_remove() - Removes the output fence from atom + * @katom: Atom to remove output fence for + * + * This will also release the reference to this fence which the atom keeps + */ +static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom) +{ + if (katom->dma_fence.fence) { + dma_fence_put(katom->dma_fence.fence); + katom->dma_fence.fence = NULL; + } +} + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_out_remove() - Removes the input fence from atom + * @katom: Atom to remove input fence for + * + * This will also release the reference to this fence which the atom keeps + */ +static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom) +{ + if (katom->dma_fence.fence_in) { + dma_fence_put(katom->dma_fence.fence_in); + katom->dma_fence.fence_in = NULL; + } +} +#endif + +/** + * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us + * @katom: Atom to check output fence for + * + * Return: true if fence exists and is valid, otherwise false + */ +static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom) +{ + return katom->dma_fence.fence && + katom->dma_fence.fence->ops == &kbase_fence_ops; +} + +/** + * kbase_fence_out_signal() - Signal output fence of atom + * @katom: Atom to signal output fence for + * @status: Status to signal with (0 for success, < 0 for error) + * + * Return: 0 on success, < 0 on error + */ +static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, + int status) +{ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) + katom->dma_fence.fence->error = status; +#else + katom->dma_fence.fence->status = status; +#endif + return dma_fence_signal(katom->dma_fence.fence); +} + +/** + * kbase_fence_add_callback() - Add callback on @fence to block @katom + * @katom: Pointer to katom that will be blocked by @fence + * @fence: Pointer to fence on which to set up the callback + * @callback: Pointer to function to be called when fence is signaled + * + * Caller needs to hold a reference to @fence when calling this function, and + * the caller is responsible for releasing that reference. An additional + * reference to @fence will be taken when the callback was successfully set up + * and @fence needs to be kept valid until the callback has been called and + * cleanup have been done. + * + * Return: 0 on success: fence was either already signaled, or callback was + * set up. Negative error code is returned on error. + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +int kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct fence *fence, + fence_func_t callback); +#else +int kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct dma_fence *fence, + dma_fence_func_t callback); +#endif + +/** + * kbase_fence_dep_count_set() - Set dep_count value on atom to specified value + * @katom: Atom to set dep_count for + * @val: value to set dep_count to + * + * The dep_count is available to the users of this module so that they can + * synchronize completion of the wait with cancellation and adding of more + * callbacks. For instance, a user could do the following: + * + * dep_count set to 1 + * callback #1 added, dep_count is increased to 2 + * callback #1 happens, dep_count decremented to 1 + * since dep_count > 0, no completion is done + * callback #2 is added, dep_count is increased to 2 + * dep_count decremented to 1 + * callback #2 happens, dep_count decremented to 0 + * since dep_count now is zero, completion executes + * + * The dep_count can also be used to make sure that the completion only + * executes once. This is typically done by setting dep_count to -1 for the + * thread that takes on this responsibility. + */ +static inline void +kbase_fence_dep_count_set(struct kbase_jd_atom *katom, int val) +{ + atomic_set(&katom->dma_fence.dep_count, val); +} + +/** + * kbase_fence_dep_count_dec_and_test() - Decrements dep_count + * @katom: Atom to decrement dep_count for + * + * See @kbase_fence_dep_count_set for general description about dep_count + * + * Return: true if value was decremented to zero, otherwise false + */ +static inline bool +kbase_fence_dep_count_dec_and_test(struct kbase_jd_atom *katom) +{ + return atomic_dec_and_test(&katom->dma_fence.dep_count); +} + +/** + * kbase_fence_dep_count_read() - Returns the current dep_count value + * @katom: Pointer to katom + * + * See @kbase_fence_dep_count_set for general description about dep_count + * + * Return: The current dep_count value + */ +static inline int kbase_fence_dep_count_read(struct kbase_jd_atom *katom) +{ + return atomic_read(&katom->dma_fence.dep_count); +} + +/** + * kbase_fence_free_callbacks() - Free dma-fence callbacks on a katom + * @katom: Pointer to katom + * + * This function will free all fence callbacks on the katom's list of + * callbacks. Callbacks that have not yet been called, because their fence + * hasn't yet signaled, will first be removed from the fence. + * + * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. + * + * Return: true if dep_count reached 0, otherwise false. + */ +bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom); + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_in_get() - Retrieve input fence for atom. + * @katom: Atom to get input fence from + * + * A ref will be taken for the fence, so use @kbase_fence_put() to release it + * + * Return: The fence, or NULL if there is no input fence for atom + */ +#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in) +#endif + +/** + * kbase_fence_out_get() - Retrieve output fence for atom. + * @katom: Atom to get output fence from + * + * A ref will be taken for the fence, so use @kbase_fence_put() to release it + * + * Return: The fence, or NULL if there is no output fence for atom + */ +#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) + +/** + * kbase_fence_put() - Releases a reference to a fence + * @fence: Fence to release reference for. + */ +#define kbase_fence_put(fence) dma_fence_put(fence) + + +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || defined(CONFIG_SYNC_FILE */ + +#endif /* _KBASE_FENCE_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence_defs.h new file mode 100755 index 000000000000..d2d7c436918c --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_fence_defs.h @@ -0,0 +1,51 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_FENCE_DEFS_H_ +#define _KBASE_FENCE_DEFS_H_ + +/* + * There was a big rename in the 4.10 kernel (fence* -> dma_fence*) + * This file hides the compatibility issues with this for the rest the driver + */ + +#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + +#include + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + +#include + +#define dma_fence_context_alloc(a) fence_context_alloc(a) +#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e) +#define dma_fence_get(a) fence_get(a) +#define dma_fence_put(a) fence_put(a) +#define dma_fence_signal(a) fence_signal(a) +#define dma_fence_is_signaled(a) fence_is_signaled(a) +#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c) +#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b) + +#else + +#include + +#endif /* < 4.10.0 */ + +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || CONFIG_SYNC_FILE */ + +#endif /* _KBASE_FENCE_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator.h new file mode 100755 index 000000000000..87697b15d986 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* NB taken from gator */ +/* + * List of possible actions to be controlled by DS-5 Streamline. + * The following numbers are used by gator to control the frame buffer dumping + * and s/w counter reporting. We cannot use the enums in mali_uk_types.h because + * they are unknown inside gator. + */ +#ifndef _KBASE_GATOR_H_ +#define _KBASE_GATOR_H_ + +#ifdef CONFIG_MALI_BIFROST_GATOR_SUPPORT +#define GATOR_MAKE_EVENT(type, number) (((type) << 24) | ((number) << 16)) +#define GATOR_JOB_SLOT_START 1 +#define GATOR_JOB_SLOT_STOP 2 +#define GATOR_JOB_SLOT_SOFT_STOPPED 3 + +void kbase_trace_mali_job_slots_event(u32 event, const struct kbase_context *kctx, u8 atom_id); +void kbase_trace_mali_pm_status(u32 event, u64 value); +void kbase_trace_mali_pm_power_off(u32 event, u64 value); +void kbase_trace_mali_pm_power_on(u32 event, u64 value); +void kbase_trace_mali_page_fault_insert_pages(int event, u32 value); +void kbase_trace_mali_mmu_as_in_use(int event); +void kbase_trace_mali_mmu_as_released(int event); +void kbase_trace_mali_total_alloc_pages_change(long long int event); + +#endif /* CONFIG_MALI_BIFROST_GATOR_SUPPORT */ + +#endif /* _KBASE_GATOR_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.c new file mode 100755 index 000000000000..860e10159fb3 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.c @@ -0,0 +1,334 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "mali_kbase.h" +#include "mali_kbase_hw.h" +#include "mali_kbase_mem_linux.h" +#include "mali_kbase_gator_api.h" +#include "mali_kbase_gator_hwcnt_names.h" + +#define MALI_MAX_CORES_PER_GROUP 4 +#define MALI_MAX_NUM_BLOCKS_PER_GROUP 8 +#define MALI_COUNTERS_PER_BLOCK 64 +#define MALI_BYTES_PER_COUNTER 4 + +struct kbase_gator_hwcnt_handles { + struct kbase_device *kbdev; + struct kbase_vinstr_client *vinstr_cli; + void *vinstr_buffer; + struct work_struct dump_work; + int dump_complete; + spinlock_t dump_lock; +}; + +static void dump_worker(struct work_struct *work); + +const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters) +{ + const char * const *hardware_counters; + struct kbase_device *kbdev; + uint32_t product_id; + uint32_t count; + + if (!total_counters) + return NULL; + + /* Get the first device - it doesn't matter in this case */ + kbdev = kbase_find_device(-1); + if (!kbdev) + return NULL; + + product_id = kbdev->gpu_props.props.core_props.product_id; + + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + switch (GPU_ID2_MODEL_MATCH_VALUE(product_id)) { + case GPU_ID2_PRODUCT_TMIX: + hardware_counters = hardware_counters_mali_tMIx; + count = ARRAY_SIZE(hardware_counters_mali_tMIx); + break; + case GPU_ID2_PRODUCT_THEX: + hardware_counters = hardware_counters_mali_tHEx; + count = ARRAY_SIZE(hardware_counters_mali_tHEx); + break; + case GPU_ID2_PRODUCT_TSIX: + hardware_counters = hardware_counters_mali_tSIx; + count = ARRAY_SIZE(hardware_counters_mali_tSIx); + break; + default: + hardware_counters = NULL; + count = 0; + dev_err(kbdev->dev, "Unrecognized product ID: %u\n", + product_id); + break; + } + } else { + switch (product_id) { + /* If we are using a Mali-T60x device */ + case GPU_ID_PI_T60X: + hardware_counters = hardware_counters_mali_t60x; + count = ARRAY_SIZE(hardware_counters_mali_t60x); + break; + /* If we are using a Mali-T62x device */ + case GPU_ID_PI_T62X: + hardware_counters = hardware_counters_mali_t62x; + count = ARRAY_SIZE(hardware_counters_mali_t62x); + break; + /* If we are using a Mali-T72x device */ + case GPU_ID_PI_T72X: + hardware_counters = hardware_counters_mali_t72x; + count = ARRAY_SIZE(hardware_counters_mali_t72x); + break; + /* If we are using a Mali-T76x device */ + case GPU_ID_PI_T76X: + hardware_counters = hardware_counters_mali_t76x; + count = ARRAY_SIZE(hardware_counters_mali_t76x); + break; + /* If we are using a Mali-T82x device */ + case GPU_ID_PI_T82X: + hardware_counters = hardware_counters_mali_t82x; + count = ARRAY_SIZE(hardware_counters_mali_t82x); + break; + /* If we are using a Mali-T83x device */ + case GPU_ID_PI_T83X: + hardware_counters = hardware_counters_mali_t83x; + count = ARRAY_SIZE(hardware_counters_mali_t83x); + break; + /* If we are using a Mali-T86x device */ + case GPU_ID_PI_T86X: + hardware_counters = hardware_counters_mali_t86x; + count = ARRAY_SIZE(hardware_counters_mali_t86x); + break; + /* If we are using a Mali-T88x device */ + case GPU_ID_PI_TFRX: + hardware_counters = hardware_counters_mali_t88x; + count = ARRAY_SIZE(hardware_counters_mali_t88x); + break; + default: + hardware_counters = NULL; + count = 0; + dev_err(kbdev->dev, "Unrecognized product ID: %u\n", + product_id); + break; + } + } + + /* Release the kbdev reference. */ + kbase_release_device(kbdev); + + *total_counters = count; + + /* If we return a string array take a reference on the module (or fail). */ + if (hardware_counters && !try_module_get(THIS_MODULE)) + return NULL; + + return hardware_counters; +} +KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init_names); + +void kbase_gator_hwcnt_term_names(void) +{ + /* Release the module reference. */ + module_put(THIS_MODULE); +} +KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term_names); + +struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info) +{ + struct kbase_gator_hwcnt_handles *hand; + struct kbase_uk_hwcnt_reader_setup setup; + uint32_t dump_size = 0, i = 0; + + if (!in_out_info) + return NULL; + + hand = kzalloc(sizeof(*hand), GFP_KERNEL); + if (!hand) + return NULL; + + INIT_WORK(&hand->dump_work, dump_worker); + spin_lock_init(&hand->dump_lock); + + /* Get the first device */ + hand->kbdev = kbase_find_device(-1); + if (!hand->kbdev) + goto free_hand; + + dump_size = kbase_vinstr_dump_size(hand->kbdev); + hand->vinstr_buffer = kzalloc(dump_size, GFP_KERNEL); + if (!hand->vinstr_buffer) + goto release_device; + in_out_info->kernel_dump_buffer = hand->vinstr_buffer; + + in_out_info->nr_cores = hand->kbdev->gpu_props.num_cores; + in_out_info->nr_core_groups = hand->kbdev->gpu_props.num_core_groups; + in_out_info->gpu_id = hand->kbdev->gpu_props.props.core_props.product_id; + + /* If we are using a v4 device (Mali-T6xx or Mali-T72x) */ + if (kbase_hw_has_feature(hand->kbdev, BASE_HW_FEATURE_V4)) { + uint32_t cg, j; + uint64_t core_mask; + + /* There are 8 hardware counters blocks per core group */ + in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * + MALI_MAX_NUM_BLOCKS_PER_GROUP * + in_out_info->nr_core_groups, GFP_KERNEL); + + if (!in_out_info->hwc_layout) + goto free_vinstr_buffer; + + dump_size = in_out_info->nr_core_groups * + MALI_MAX_NUM_BLOCKS_PER_GROUP * + MALI_COUNTERS_PER_BLOCK * + MALI_BYTES_PER_COUNTER; + + for (cg = 0; cg < in_out_info->nr_core_groups; cg++) { + core_mask = hand->kbdev->gpu_props.props.coherency_info.group[cg].core_mask; + + for (j = 0; j < MALI_MAX_CORES_PER_GROUP; j++) { + if (core_mask & (1u << j)) + in_out_info->hwc_layout[i++] = SHADER_BLOCK; + else + in_out_info->hwc_layout[i++] = RESERVED_BLOCK; + } + + in_out_info->hwc_layout[i++] = TILER_BLOCK; + in_out_info->hwc_layout[i++] = MMU_L2_BLOCK; + + in_out_info->hwc_layout[i++] = RESERVED_BLOCK; + + if (0 == cg) + in_out_info->hwc_layout[i++] = JM_BLOCK; + else + in_out_info->hwc_layout[i++] = RESERVED_BLOCK; + } + /* If we are using any other device */ + } else { + uint32_t nr_l2, nr_sc_bits, j; + uint64_t core_mask; + + nr_l2 = hand->kbdev->gpu_props.props.l2_props.num_l2_slices; + + core_mask = hand->kbdev->gpu_props.props.coherency_info.group[0].core_mask; + + nr_sc_bits = fls64(core_mask); + + /* The job manager and tiler sets of counters + * are always present */ + in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * (2 + nr_sc_bits + nr_l2), GFP_KERNEL); + + if (!in_out_info->hwc_layout) + goto free_vinstr_buffer; + + dump_size = (2 + nr_sc_bits + nr_l2) * MALI_COUNTERS_PER_BLOCK * MALI_BYTES_PER_COUNTER; + + in_out_info->hwc_layout[i++] = JM_BLOCK; + in_out_info->hwc_layout[i++] = TILER_BLOCK; + + for (j = 0; j < nr_l2; j++) + in_out_info->hwc_layout[i++] = MMU_L2_BLOCK; + + while (core_mask != 0ull) { + if ((core_mask & 1ull) != 0ull) + in_out_info->hwc_layout[i++] = SHADER_BLOCK; + else + in_out_info->hwc_layout[i++] = RESERVED_BLOCK; + core_mask >>= 1; + } + } + + in_out_info->nr_hwc_blocks = i; + in_out_info->size = dump_size; + + setup.jm_bm = in_out_info->bitmask[0]; + setup.tiler_bm = in_out_info->bitmask[1]; + setup.shader_bm = in_out_info->bitmask[2]; + setup.mmu_l2_bm = in_out_info->bitmask[3]; + hand->vinstr_cli = kbase_vinstr_hwcnt_kernel_setup(hand->kbdev->vinstr_ctx, + &setup, hand->vinstr_buffer); + if (!hand->vinstr_cli) { + dev_err(hand->kbdev->dev, "Failed to register gator with vinstr core"); + goto free_layout; + } + + return hand; + +free_layout: + kfree(in_out_info->hwc_layout); + +free_vinstr_buffer: + kfree(hand->vinstr_buffer); + +release_device: + kbase_release_device(hand->kbdev); + +free_hand: + kfree(hand); + return NULL; +} +KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init); + +void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles) +{ + if (in_out_info) + kfree(in_out_info->hwc_layout); + + if (opaque_handles) { + cancel_work_sync(&opaque_handles->dump_work); + kbase_vinstr_detach_client(opaque_handles->vinstr_cli); + kfree(opaque_handles->vinstr_buffer); + kbase_release_device(opaque_handles->kbdev); + kfree(opaque_handles); + } +} +KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term); + +static void dump_worker(struct work_struct *work) +{ + struct kbase_gator_hwcnt_handles *hand; + + hand = container_of(work, struct kbase_gator_hwcnt_handles, dump_work); + if (!kbase_vinstr_hwc_dump(hand->vinstr_cli, + BASE_HWCNT_READER_EVENT_MANUAL)) { + spin_lock_bh(&hand->dump_lock); + hand->dump_complete = 1; + spin_unlock_bh(&hand->dump_lock); + } else { + schedule_work(&hand->dump_work); + } +} + +uint32_t kbase_gator_instr_hwcnt_dump_complete( + struct kbase_gator_hwcnt_handles *opaque_handles, + uint32_t * const success) +{ + + if (opaque_handles && success) { + *success = opaque_handles->dump_complete; + opaque_handles->dump_complete = 0; + return *success; + } + return 0; +} +KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_complete); + +uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles) +{ + if (opaque_handles) + schedule_work(&opaque_handles->dump_work); + return 0; +} +KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_irq); diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.h new file mode 100755 index 000000000000..ef9ac0f7b633 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_api.h @@ -0,0 +1,219 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_GATOR_API_H_ +#define _KBASE_GATOR_API_H_ + +/** + * @brief This file describes the API used by Gator to fetch hardware counters. + */ + +/* This define is used by the gator kernel module compile to select which DDK + * API calling convention to use. If not defined (legacy DDK) gator assumes + * version 1. The version to DDK release mapping is: + * Version 1 API: DDK versions r1px, r2px + * Version 2 API: DDK versions r3px, r4px + * Version 3 API: DDK version r5p0 and newer + * + * API Usage + * ========= + * + * 1] Call kbase_gator_hwcnt_init_names() to return the list of short counter + * names for the GPU present in this device. + * + * 2] Create a kbase_gator_hwcnt_info structure and set the counter enables for + * the counters you want enabled. The enables can all be set for simplicity in + * most use cases, but disabling some will let you minimize bandwidth impact. + * + * 3] Call kbase_gator_hwcnt_init() using the above structure, to create a + * counter context. On successful return the DDK will have populated the + * structure with a variety of useful information. + * + * 4] Call kbase_gator_hwcnt_dump_irq() to queue a non-blocking request for a + * counter dump. If this returns a non-zero value the request has been queued, + * otherwise the driver has been unable to do so (typically because of another + * user of the instrumentation exists concurrently). + * + * 5] Call kbase_gator_hwcnt_dump_complete() to test whether the previously + * requested dump has been succesful. If this returns non-zero the counter dump + * has resolved, but the value of *success must also be tested as the dump + * may have not been successful. If it returns zero the counter dump was + * abandoned due to the device being busy (typically because of another + * user of the instrumentation exists concurrently). + * + * 6] Process the counters stored in the buffer pointed to by ... + * + * kbase_gator_hwcnt_info->kernel_dump_buffer + * + * In pseudo code you can find all of the counters via this approach: + * + * + * hwcnt_info # pointer to kbase_gator_hwcnt_info structure + * hwcnt_name # pointer to name list + * + * u32 * hwcnt_data = (u32*)hwcnt_info->kernel_dump_buffer + * + * # Iterate over each 64-counter block in this GPU configuration + * for( i = 0; i < hwcnt_info->nr_hwc_blocks; i++) { + * hwc_type type = hwcnt_info->hwc_layout[i]; + * + * # Skip reserved type blocks - they contain no counters at all + * if( type == RESERVED_BLOCK ) { + * continue; + * } + * + * size_t name_offset = type * 64; + * size_t data_offset = i * 64; + * + * # Iterate over the names of the counters in this block type + * for( j = 0; j < 64; j++) { + * const char * name = hwcnt_name[name_offset+j]; + * + * # Skip empty name strings - there is no counter here + * if( name[0] == '\0' ) { + * continue; + * } + * + * u32 data = hwcnt_data[data_offset+j]; + * + * printk( "COUNTER: %s DATA: %u\n", name, data ); + * } + * } + * + * + * Note that in most implementations you typically want to either SUM or + * AVERAGE multiple instances of the same counter if, for example, you have + * multiple shader cores or multiple L2 caches. The most sensible view for + * analysis is to AVERAGE shader core counters, but SUM L2 cache and MMU + * counters. + * + * 7] Goto 4, repeating until you want to stop collecting counters. + * + * 8] Release the dump resources by calling kbase_gator_hwcnt_term(). + * + * 9] Release the name table resources by calling + * kbase_gator_hwcnt_term_names(). This function must only be called if + * init_names() returned a non-NULL value. + **/ + +#define MALI_DDK_GATOR_API_VERSION 3 + +enum hwc_type { + JM_BLOCK = 0, + TILER_BLOCK, + SHADER_BLOCK, + MMU_L2_BLOCK, + RESERVED_BLOCK +}; + +struct kbase_gator_hwcnt_info { + /* Passed from Gator to kbase */ + + /* the bitmask of enabled hardware counters for each counter block */ + uint16_t bitmask[4]; + + /* Passed from kbase to Gator */ + + /* ptr to counter dump memory */ + void *kernel_dump_buffer; + + /* size of counter dump memory */ + uint32_t size; + + /* the ID of the Mali device */ + uint32_t gpu_id; + + /* the number of shader cores in the GPU */ + uint32_t nr_cores; + + /* the number of core groups */ + uint32_t nr_core_groups; + + /* the memory layout of the performance counters */ + enum hwc_type *hwc_layout; + + /* the total number of hardware couter blocks */ + uint32_t nr_hwc_blocks; +}; + +/** + * @brief Opaque block of Mali data which Gator needs to return to the API later. + */ +struct kbase_gator_hwcnt_handles; + +/** + * @brief Initialize the resources Gator needs for performance profiling. + * + * @param in_out_info A pointer to a structure containing the enabled counters passed from Gator and all the Mali + * specific information that will be returned to Gator. On entry Gator must have populated the + * 'bitmask' field with the counters it wishes to enable for each class of counter block. + * Each entry in the array corresponds to a single counter class based on the "hwc_type" + * enumeration, and each bit corresponds to an enable for 4 sequential counters (LSB enables + * the first 4 counters in the block, and so on). See the GPU counter array as returned by + * kbase_gator_hwcnt_get_names() for the index values of each counter for the curernt GPU. + * + * @return Pointer to an opaque handle block on success, NULL on error. + */ +extern struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info); + +/** + * @brief Free all resources once Gator has finished using performance counters. + * + * @param in_out_info A pointer to a structure containing the enabled counters passed from Gator and all the + * Mali specific information that will be returned to Gator. + * @param opaque_handles A wrapper structure for kbase structures. + */ +extern void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles); + +/** + * @brief Poll whether a counter dump is successful. + * + * @param opaque_handles A wrapper structure for kbase structures. + * @param[out] success Non-zero on success, zero on failure. + * + * @return Zero if the dump is still pending, non-zero if the dump has completed. Note that a + * completed dump may not have dumped succesfully, so the caller must test for both + * a completed and successful dump before processing counters. + */ +extern uint32_t kbase_gator_instr_hwcnt_dump_complete(struct kbase_gator_hwcnt_handles *opaque_handles, uint32_t * const success); + +/** + * @brief Request the generation of a new counter dump. + * + * @param opaque_handles A wrapper structure for kbase structures. + * + * @return Zero if the hardware device is busy and cannot handle the request, non-zero otherwise. + */ +extern uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles); + +/** + * @brief This function is used to fetch the names table based on the Mali device in use. + * + * @param[out] total_counters The total number of counters short names in the Mali devices' list. + * + * @return Pointer to an array of strings of length *total_counters. + */ +extern const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters); + +/** + * @brief This function is used to terminate the use of the names table. + * + * This function must only be called if the initial call to kbase_gator_hwcnt_init_names returned a non-NULL value. + */ +extern void kbase_gator_hwcnt_term_names(void); + +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names.h new file mode 100755 index 000000000000..24103e292453 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names.h @@ -0,0 +1,2167 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_GATOR_HWCNT_NAMES_H_ +#define _KBASE_GATOR_HWCNT_NAMES_H_ + +/* + * "Short names" for hardware counters used by Streamline. Counters names are + * stored in accordance with their memory layout in the binary counter block + * emitted by the Mali GPU. Each "master" in the GPU emits a fixed-size block + * of 64 counters, and each GPU implements the same set of "masters" although + * the counters each master exposes within its block of 64 may vary. + * + * Counters which are an empty string are simply "holes" in the counter memory + * where no counter exists. + */ + +static const char * const hardware_counters_mali_t60x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T60x_MESSAGES_SENT", + "T60x_MESSAGES_RECEIVED", + "T60x_GPU_ACTIVE", + "T60x_IRQ_ACTIVE", + "T60x_JS0_JOBS", + "T60x_JS0_TASKS", + "T60x_JS0_ACTIVE", + "", + "T60x_JS0_WAIT_READ", + "T60x_JS0_WAIT_ISSUE", + "T60x_JS0_WAIT_DEPEND", + "T60x_JS0_WAIT_FINISH", + "T60x_JS1_JOBS", + "T60x_JS1_TASKS", + "T60x_JS1_ACTIVE", + "", + "T60x_JS1_WAIT_READ", + "T60x_JS1_WAIT_ISSUE", + "T60x_JS1_WAIT_DEPEND", + "T60x_JS1_WAIT_FINISH", + "T60x_JS2_JOBS", + "T60x_JS2_TASKS", + "T60x_JS2_ACTIVE", + "", + "T60x_JS2_WAIT_READ", + "T60x_JS2_WAIT_ISSUE", + "T60x_JS2_WAIT_DEPEND", + "T60x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T60x_TI_JOBS_PROCESSED", + "T60x_TI_TRIANGLES", + "T60x_TI_QUADS", + "T60x_TI_POLYGONS", + "T60x_TI_POINTS", + "T60x_TI_LINES", + "T60x_TI_VCACHE_HIT", + "T60x_TI_VCACHE_MISS", + "T60x_TI_FRONT_FACING", + "T60x_TI_BACK_FACING", + "T60x_TI_PRIM_VISIBLE", + "T60x_TI_PRIM_CULLED", + "T60x_TI_PRIM_CLIPPED", + "T60x_TI_LEVEL0", + "T60x_TI_LEVEL1", + "T60x_TI_LEVEL2", + "T60x_TI_LEVEL3", + "T60x_TI_LEVEL4", + "T60x_TI_LEVEL5", + "T60x_TI_LEVEL6", + "T60x_TI_LEVEL7", + "T60x_TI_COMMAND_1", + "T60x_TI_COMMAND_2", + "T60x_TI_COMMAND_3", + "T60x_TI_COMMAND_4", + "T60x_TI_COMMAND_4_7", + "T60x_TI_COMMAND_8_15", + "T60x_TI_COMMAND_16_63", + "T60x_TI_COMMAND_64", + "T60x_TI_COMPRESS_IN", + "T60x_TI_COMPRESS_OUT", + "T60x_TI_COMPRESS_FLUSH", + "T60x_TI_TIMESTAMPS", + "T60x_TI_PCACHE_HIT", + "T60x_TI_PCACHE_MISS", + "T60x_TI_PCACHE_LINE", + "T60x_TI_PCACHE_STALL", + "T60x_TI_WRBUF_HIT", + "T60x_TI_WRBUF_MISS", + "T60x_TI_WRBUF_LINE", + "T60x_TI_WRBUF_PARTIAL", + "T60x_TI_WRBUF_STALL", + "T60x_TI_ACTIVE", + "T60x_TI_LOADING_DESC", + "T60x_TI_INDEX_WAIT", + "T60x_TI_INDEX_RANGE_WAIT", + "T60x_TI_VERTEX_WAIT", + "T60x_TI_PCACHE_WAIT", + "T60x_TI_WRBUF_WAIT", + "T60x_TI_BUS_READ", + "T60x_TI_BUS_WRITE", + "", + "", + "", + "", + "", + "T60x_TI_UTLB_STALL", + "T60x_TI_UTLB_REPLAY_MISS", + "T60x_TI_UTLB_REPLAY_FULL", + "T60x_TI_UTLB_NEW_MISS", + "T60x_TI_UTLB_HIT", + + /* Shader Core */ + "", + "", + "", + "", + "T60x_FRAG_ACTIVE", + "T60x_FRAG_PRIMITIVES", + "T60x_FRAG_PRIMITIVES_DROPPED", + "T60x_FRAG_CYCLES_DESC", + "T60x_FRAG_CYCLES_PLR", + "T60x_FRAG_CYCLES_VERT", + "T60x_FRAG_CYCLES_TRISETUP", + "T60x_FRAG_CYCLES_RAST", + "T60x_FRAG_THREADS", + "T60x_FRAG_DUMMY_THREADS", + "T60x_FRAG_QUADS_RAST", + "T60x_FRAG_QUADS_EZS_TEST", + "T60x_FRAG_QUADS_EZS_KILLED", + "T60x_FRAG_THREADS_LZS_TEST", + "T60x_FRAG_THREADS_LZS_KILLED", + "T60x_FRAG_CYCLES_NO_TILE", + "T60x_FRAG_NUM_TILES", + "T60x_FRAG_TRANS_ELIM", + "T60x_COMPUTE_ACTIVE", + "T60x_COMPUTE_TASKS", + "T60x_COMPUTE_THREADS", + "T60x_COMPUTE_CYCLES_DESC", + "T60x_TRIPIPE_ACTIVE", + "T60x_ARITH_WORDS", + "T60x_ARITH_CYCLES_REG", + "T60x_ARITH_CYCLES_L0", + "T60x_ARITH_FRAG_DEPEND", + "T60x_LS_WORDS", + "T60x_LS_ISSUES", + "T60x_LS_RESTARTS", + "T60x_LS_REISSUES_MISS", + "T60x_LS_REISSUES_VD", + "T60x_LS_REISSUE_ATTRIB_MISS", + "T60x_LS_NO_WB", + "T60x_TEX_WORDS", + "T60x_TEX_BUBBLES", + "T60x_TEX_WORDS_L0", + "T60x_TEX_WORDS_DESC", + "T60x_TEX_ISSUES", + "T60x_TEX_RECIRC_FMISS", + "T60x_TEX_RECIRC_DESC", + "T60x_TEX_RECIRC_MULTI", + "T60x_TEX_RECIRC_PMISS", + "T60x_TEX_RECIRC_CONF", + "T60x_LSC_READ_HITS", + "T60x_LSC_READ_MISSES", + "T60x_LSC_WRITE_HITS", + "T60x_LSC_WRITE_MISSES", + "T60x_LSC_ATOMIC_HITS", + "T60x_LSC_ATOMIC_MISSES", + "T60x_LSC_LINE_FETCHES", + "T60x_LSC_DIRTY_LINE", + "T60x_LSC_SNOOPS", + "T60x_AXI_TLB_STALL", + "T60x_AXI_TLB_MISS", + "T60x_AXI_TLB_TRANSACTION", + "T60x_LS_TLB_MISS", + "T60x_LS_TLB_HIT", + "T60x_AXI_BEATS_READ", + "T60x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T60x_MMU_HIT", + "T60x_MMU_NEW_MISS", + "T60x_MMU_REPLAY_FULL", + "T60x_MMU_REPLAY_MISS", + "T60x_MMU_TABLE_WALK", + "", + "", + "", + "", + "", + "", + "", + "T60x_UTLB_HIT", + "T60x_UTLB_NEW_MISS", + "T60x_UTLB_REPLAY_FULL", + "T60x_UTLB_REPLAY_MISS", + "T60x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T60x_L2_EXT_WRITE_BEATS", + "T60x_L2_EXT_READ_BEATS", + "T60x_L2_ANY_LOOKUP", + "T60x_L2_READ_LOOKUP", + "T60x_L2_SREAD_LOOKUP", + "T60x_L2_READ_REPLAY", + "T60x_L2_READ_SNOOP", + "T60x_L2_READ_HIT", + "T60x_L2_CLEAN_MISS", + "T60x_L2_WRITE_LOOKUP", + "T60x_L2_SWRITE_LOOKUP", + "T60x_L2_WRITE_REPLAY", + "T60x_L2_WRITE_SNOOP", + "T60x_L2_WRITE_HIT", + "T60x_L2_EXT_READ_FULL", + "T60x_L2_EXT_READ_HALF", + "T60x_L2_EXT_WRITE_FULL", + "T60x_L2_EXT_WRITE_HALF", + "T60x_L2_EXT_READ", + "T60x_L2_EXT_READ_LINE", + "T60x_L2_EXT_WRITE", + "T60x_L2_EXT_WRITE_LINE", + "T60x_L2_EXT_WRITE_SMALL", + "T60x_L2_EXT_BARRIER", + "T60x_L2_EXT_AR_STALL", + "T60x_L2_EXT_R_BUF_FULL", + "T60x_L2_EXT_RD_BUF_FULL", + "T60x_L2_EXT_R_RAW", + "T60x_L2_EXT_W_STALL", + "T60x_L2_EXT_W_BUF_FULL", + "T60x_L2_EXT_R_W_HAZARD", + "T60x_L2_TAG_HAZARD", + "T60x_L2_SNOOP_FULL", + "T60x_L2_REPLAY_FULL" +}; +static const char * const hardware_counters_mali_t62x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T62x_MESSAGES_SENT", + "T62x_MESSAGES_RECEIVED", + "T62x_GPU_ACTIVE", + "T62x_IRQ_ACTIVE", + "T62x_JS0_JOBS", + "T62x_JS0_TASKS", + "T62x_JS0_ACTIVE", + "", + "T62x_JS0_WAIT_READ", + "T62x_JS0_WAIT_ISSUE", + "T62x_JS0_WAIT_DEPEND", + "T62x_JS0_WAIT_FINISH", + "T62x_JS1_JOBS", + "T62x_JS1_TASKS", + "T62x_JS1_ACTIVE", + "", + "T62x_JS1_WAIT_READ", + "T62x_JS1_WAIT_ISSUE", + "T62x_JS1_WAIT_DEPEND", + "T62x_JS1_WAIT_FINISH", + "T62x_JS2_JOBS", + "T62x_JS2_TASKS", + "T62x_JS2_ACTIVE", + "", + "T62x_JS2_WAIT_READ", + "T62x_JS2_WAIT_ISSUE", + "T62x_JS2_WAIT_DEPEND", + "T62x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T62x_TI_JOBS_PROCESSED", + "T62x_TI_TRIANGLES", + "T62x_TI_QUADS", + "T62x_TI_POLYGONS", + "T62x_TI_POINTS", + "T62x_TI_LINES", + "T62x_TI_VCACHE_HIT", + "T62x_TI_VCACHE_MISS", + "T62x_TI_FRONT_FACING", + "T62x_TI_BACK_FACING", + "T62x_TI_PRIM_VISIBLE", + "T62x_TI_PRIM_CULLED", + "T62x_TI_PRIM_CLIPPED", + "T62x_TI_LEVEL0", + "T62x_TI_LEVEL1", + "T62x_TI_LEVEL2", + "T62x_TI_LEVEL3", + "T62x_TI_LEVEL4", + "T62x_TI_LEVEL5", + "T62x_TI_LEVEL6", + "T62x_TI_LEVEL7", + "T62x_TI_COMMAND_1", + "T62x_TI_COMMAND_2", + "T62x_TI_COMMAND_3", + "T62x_TI_COMMAND_4", + "T62x_TI_COMMAND_5_7", + "T62x_TI_COMMAND_8_15", + "T62x_TI_COMMAND_16_63", + "T62x_TI_COMMAND_64", + "T62x_TI_COMPRESS_IN", + "T62x_TI_COMPRESS_OUT", + "T62x_TI_COMPRESS_FLUSH", + "T62x_TI_TIMESTAMPS", + "T62x_TI_PCACHE_HIT", + "T62x_TI_PCACHE_MISS", + "T62x_TI_PCACHE_LINE", + "T62x_TI_PCACHE_STALL", + "T62x_TI_WRBUF_HIT", + "T62x_TI_WRBUF_MISS", + "T62x_TI_WRBUF_LINE", + "T62x_TI_WRBUF_PARTIAL", + "T62x_TI_WRBUF_STALL", + "T62x_TI_ACTIVE", + "T62x_TI_LOADING_DESC", + "T62x_TI_INDEX_WAIT", + "T62x_TI_INDEX_RANGE_WAIT", + "T62x_TI_VERTEX_WAIT", + "T62x_TI_PCACHE_WAIT", + "T62x_TI_WRBUF_WAIT", + "T62x_TI_BUS_READ", + "T62x_TI_BUS_WRITE", + "", + "", + "", + "", + "", + "T62x_TI_UTLB_STALL", + "T62x_TI_UTLB_REPLAY_MISS", + "T62x_TI_UTLB_REPLAY_FULL", + "T62x_TI_UTLB_NEW_MISS", + "T62x_TI_UTLB_HIT", + + /* Shader Core */ + "", + "", + "", + "T62x_SHADER_CORE_ACTIVE", + "T62x_FRAG_ACTIVE", + "T62x_FRAG_PRIMITIVES", + "T62x_FRAG_PRIMITIVES_DROPPED", + "T62x_FRAG_CYCLES_DESC", + "T62x_FRAG_CYCLES_FPKQ_ACTIVE", + "T62x_FRAG_CYCLES_VERT", + "T62x_FRAG_CYCLES_TRISETUP", + "T62x_FRAG_CYCLES_EZS_ACTIVE", + "T62x_FRAG_THREADS", + "T62x_FRAG_DUMMY_THREADS", + "T62x_FRAG_QUADS_RAST", + "T62x_FRAG_QUADS_EZS_TEST", + "T62x_FRAG_QUADS_EZS_KILLED", + "T62x_FRAG_THREADS_LZS_TEST", + "T62x_FRAG_THREADS_LZS_KILLED", + "T62x_FRAG_CYCLES_NO_TILE", + "T62x_FRAG_NUM_TILES", + "T62x_FRAG_TRANS_ELIM", + "T62x_COMPUTE_ACTIVE", + "T62x_COMPUTE_TASKS", + "T62x_COMPUTE_THREADS", + "T62x_COMPUTE_CYCLES_DESC", + "T62x_TRIPIPE_ACTIVE", + "T62x_ARITH_WORDS", + "T62x_ARITH_CYCLES_REG", + "T62x_ARITH_CYCLES_L0", + "T62x_ARITH_FRAG_DEPEND", + "T62x_LS_WORDS", + "T62x_LS_ISSUES", + "T62x_LS_RESTARTS", + "T62x_LS_REISSUES_MISS", + "T62x_LS_REISSUES_VD", + "T62x_LS_REISSUE_ATTRIB_MISS", + "T62x_LS_NO_WB", + "T62x_TEX_WORDS", + "T62x_TEX_BUBBLES", + "T62x_TEX_WORDS_L0", + "T62x_TEX_WORDS_DESC", + "T62x_TEX_ISSUES", + "T62x_TEX_RECIRC_FMISS", + "T62x_TEX_RECIRC_DESC", + "T62x_TEX_RECIRC_MULTI", + "T62x_TEX_RECIRC_PMISS", + "T62x_TEX_RECIRC_CONF", + "T62x_LSC_READ_HITS", + "T62x_LSC_READ_MISSES", + "T62x_LSC_WRITE_HITS", + "T62x_LSC_WRITE_MISSES", + "T62x_LSC_ATOMIC_HITS", + "T62x_LSC_ATOMIC_MISSES", + "T62x_LSC_LINE_FETCHES", + "T62x_LSC_DIRTY_LINE", + "T62x_LSC_SNOOPS", + "T62x_AXI_TLB_STALL", + "T62x_AXI_TLB_MISS", + "T62x_AXI_TLB_TRANSACTION", + "T62x_LS_TLB_MISS", + "T62x_LS_TLB_HIT", + "T62x_AXI_BEATS_READ", + "T62x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T62x_MMU_HIT", + "T62x_MMU_NEW_MISS", + "T62x_MMU_REPLAY_FULL", + "T62x_MMU_REPLAY_MISS", + "T62x_MMU_TABLE_WALK", + "", + "", + "", + "", + "", + "", + "", + "T62x_UTLB_HIT", + "T62x_UTLB_NEW_MISS", + "T62x_UTLB_REPLAY_FULL", + "T62x_UTLB_REPLAY_MISS", + "T62x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T62x_L2_EXT_WRITE_BEATS", + "T62x_L2_EXT_READ_BEATS", + "T62x_L2_ANY_LOOKUP", + "T62x_L2_READ_LOOKUP", + "T62x_L2_SREAD_LOOKUP", + "T62x_L2_READ_REPLAY", + "T62x_L2_READ_SNOOP", + "T62x_L2_READ_HIT", + "T62x_L2_CLEAN_MISS", + "T62x_L2_WRITE_LOOKUP", + "T62x_L2_SWRITE_LOOKUP", + "T62x_L2_WRITE_REPLAY", + "T62x_L2_WRITE_SNOOP", + "T62x_L2_WRITE_HIT", + "T62x_L2_EXT_READ_FULL", + "T62x_L2_EXT_READ_HALF", + "T62x_L2_EXT_WRITE_FULL", + "T62x_L2_EXT_WRITE_HALF", + "T62x_L2_EXT_READ", + "T62x_L2_EXT_READ_LINE", + "T62x_L2_EXT_WRITE", + "T62x_L2_EXT_WRITE_LINE", + "T62x_L2_EXT_WRITE_SMALL", + "T62x_L2_EXT_BARRIER", + "T62x_L2_EXT_AR_STALL", + "T62x_L2_EXT_R_BUF_FULL", + "T62x_L2_EXT_RD_BUF_FULL", + "T62x_L2_EXT_R_RAW", + "T62x_L2_EXT_W_STALL", + "T62x_L2_EXT_W_BUF_FULL", + "T62x_L2_EXT_R_W_HAZARD", + "T62x_L2_TAG_HAZARD", + "T62x_L2_SNOOP_FULL", + "T62x_L2_REPLAY_FULL" +}; + +static const char * const hardware_counters_mali_t72x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T72x_GPU_ACTIVE", + "T72x_IRQ_ACTIVE", + "T72x_JS0_JOBS", + "T72x_JS0_TASKS", + "T72x_JS0_ACTIVE", + "T72x_JS1_JOBS", + "T72x_JS1_TASKS", + "T72x_JS1_ACTIVE", + "T72x_JS2_JOBS", + "T72x_JS2_TASKS", + "T72x_JS2_ACTIVE", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T72x_TI_JOBS_PROCESSED", + "T72x_TI_TRIANGLES", + "T72x_TI_QUADS", + "T72x_TI_POLYGONS", + "T72x_TI_POINTS", + "T72x_TI_LINES", + "T72x_TI_FRONT_FACING", + "T72x_TI_BACK_FACING", + "T72x_TI_PRIM_VISIBLE", + "T72x_TI_PRIM_CULLED", + "T72x_TI_PRIM_CLIPPED", + "", + "", + "", + "", + "", + "", + "", + "", + "T72x_TI_ACTIVE", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Shader Core */ + "", + "", + "", + "", + "T72x_FRAG_ACTIVE", + "T72x_FRAG_PRIMITIVES", + "T72x_FRAG_PRIMITIVES_DROPPED", + "T72x_FRAG_THREADS", + "T72x_FRAG_DUMMY_THREADS", + "T72x_FRAG_QUADS_RAST", + "T72x_FRAG_QUADS_EZS_TEST", + "T72x_FRAG_QUADS_EZS_KILLED", + "T72x_FRAG_THREADS_LZS_TEST", + "T72x_FRAG_THREADS_LZS_KILLED", + "T72x_FRAG_CYCLES_NO_TILE", + "T72x_FRAG_NUM_TILES", + "T72x_FRAG_TRANS_ELIM", + "T72x_COMPUTE_ACTIVE", + "T72x_COMPUTE_TASKS", + "T72x_COMPUTE_THREADS", + "T72x_TRIPIPE_ACTIVE", + "T72x_ARITH_WORDS", + "T72x_ARITH_CYCLES_REG", + "T72x_LS_WORDS", + "T72x_LS_ISSUES", + "T72x_LS_RESTARTS", + "T72x_LS_REISSUES_MISS", + "T72x_TEX_WORDS", + "T72x_TEX_BUBBLES", + "T72x_TEX_ISSUES", + "T72x_LSC_READ_HITS", + "T72x_LSC_READ_MISSES", + "T72x_LSC_WRITE_HITS", + "T72x_LSC_WRITE_MISSES", + "T72x_LSC_ATOMIC_HITS", + "T72x_LSC_ATOMIC_MISSES", + "T72x_LSC_LINE_FETCHES", + "T72x_LSC_DIRTY_LINE", + "T72x_LSC_SNOOPS", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*L2 and MMU */ + "", + "", + "", + "", + "T72x_L2_EXT_WRITE_BEAT", + "T72x_L2_EXT_READ_BEAT", + "T72x_L2_READ_SNOOP", + "T72x_L2_READ_HIT", + "T72x_L2_WRITE_SNOOP", + "T72x_L2_WRITE_HIT", + "T72x_L2_EXT_WRITE_SMALL", + "T72x_L2_EXT_BARRIER", + "T72x_L2_EXT_AR_STALL", + "T72x_L2_EXT_W_STALL", + "T72x_L2_SNOOP_FULL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" +}; + +static const char * const hardware_counters_mali_t76x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T76x_MESSAGES_SENT", + "T76x_MESSAGES_RECEIVED", + "T76x_GPU_ACTIVE", + "T76x_IRQ_ACTIVE", + "T76x_JS0_JOBS", + "T76x_JS0_TASKS", + "T76x_JS0_ACTIVE", + "", + "T76x_JS0_WAIT_READ", + "T76x_JS0_WAIT_ISSUE", + "T76x_JS0_WAIT_DEPEND", + "T76x_JS0_WAIT_FINISH", + "T76x_JS1_JOBS", + "T76x_JS1_TASKS", + "T76x_JS1_ACTIVE", + "", + "T76x_JS1_WAIT_READ", + "T76x_JS1_WAIT_ISSUE", + "T76x_JS1_WAIT_DEPEND", + "T76x_JS1_WAIT_FINISH", + "T76x_JS2_JOBS", + "T76x_JS2_TASKS", + "T76x_JS2_ACTIVE", + "", + "T76x_JS2_WAIT_READ", + "T76x_JS2_WAIT_ISSUE", + "T76x_JS2_WAIT_DEPEND", + "T76x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T76x_TI_JOBS_PROCESSED", + "T76x_TI_TRIANGLES", + "T76x_TI_QUADS", + "T76x_TI_POLYGONS", + "T76x_TI_POINTS", + "T76x_TI_LINES", + "T76x_TI_VCACHE_HIT", + "T76x_TI_VCACHE_MISS", + "T76x_TI_FRONT_FACING", + "T76x_TI_BACK_FACING", + "T76x_TI_PRIM_VISIBLE", + "T76x_TI_PRIM_CULLED", + "T76x_TI_PRIM_CLIPPED", + "T76x_TI_LEVEL0", + "T76x_TI_LEVEL1", + "T76x_TI_LEVEL2", + "T76x_TI_LEVEL3", + "T76x_TI_LEVEL4", + "T76x_TI_LEVEL5", + "T76x_TI_LEVEL6", + "T76x_TI_LEVEL7", + "T76x_TI_COMMAND_1", + "T76x_TI_COMMAND_2", + "T76x_TI_COMMAND_3", + "T76x_TI_COMMAND_4", + "T76x_TI_COMMAND_5_7", + "T76x_TI_COMMAND_8_15", + "T76x_TI_COMMAND_16_63", + "T76x_TI_COMMAND_64", + "T76x_TI_COMPRESS_IN", + "T76x_TI_COMPRESS_OUT", + "T76x_TI_COMPRESS_FLUSH", + "T76x_TI_TIMESTAMPS", + "T76x_TI_PCACHE_HIT", + "T76x_TI_PCACHE_MISS", + "T76x_TI_PCACHE_LINE", + "T76x_TI_PCACHE_STALL", + "T76x_TI_WRBUF_HIT", + "T76x_TI_WRBUF_MISS", + "T76x_TI_WRBUF_LINE", + "T76x_TI_WRBUF_PARTIAL", + "T76x_TI_WRBUF_STALL", + "T76x_TI_ACTIVE", + "T76x_TI_LOADING_DESC", + "T76x_TI_INDEX_WAIT", + "T76x_TI_INDEX_RANGE_WAIT", + "T76x_TI_VERTEX_WAIT", + "T76x_TI_PCACHE_WAIT", + "T76x_TI_WRBUF_WAIT", + "T76x_TI_BUS_READ", + "T76x_TI_BUS_WRITE", + "", + "", + "", + "", + "", + "T76x_TI_UTLB_HIT", + "T76x_TI_UTLB_NEW_MISS", + "T76x_TI_UTLB_REPLAY_FULL", + "T76x_TI_UTLB_REPLAY_MISS", + "T76x_TI_UTLB_STALL", + + /* Shader Core */ + "", + "", + "", + "", + "T76x_FRAG_ACTIVE", + "T76x_FRAG_PRIMITIVES", + "T76x_FRAG_PRIMITIVES_DROPPED", + "T76x_FRAG_CYCLES_DESC", + "T76x_FRAG_CYCLES_FPKQ_ACTIVE", + "T76x_FRAG_CYCLES_VERT", + "T76x_FRAG_CYCLES_TRISETUP", + "T76x_FRAG_CYCLES_EZS_ACTIVE", + "T76x_FRAG_THREADS", + "T76x_FRAG_DUMMY_THREADS", + "T76x_FRAG_QUADS_RAST", + "T76x_FRAG_QUADS_EZS_TEST", + "T76x_FRAG_QUADS_EZS_KILLED", + "T76x_FRAG_THREADS_LZS_TEST", + "T76x_FRAG_THREADS_LZS_KILLED", + "T76x_FRAG_CYCLES_NO_TILE", + "T76x_FRAG_NUM_TILES", + "T76x_FRAG_TRANS_ELIM", + "T76x_COMPUTE_ACTIVE", + "T76x_COMPUTE_TASKS", + "T76x_COMPUTE_THREADS", + "T76x_COMPUTE_CYCLES_DESC", + "T76x_TRIPIPE_ACTIVE", + "T76x_ARITH_WORDS", + "T76x_ARITH_CYCLES_REG", + "T76x_ARITH_CYCLES_L0", + "T76x_ARITH_FRAG_DEPEND", + "T76x_LS_WORDS", + "T76x_LS_ISSUES", + "T76x_LS_REISSUE_ATTR", + "T76x_LS_REISSUES_VARY", + "T76x_LS_VARY_RV_MISS", + "T76x_LS_VARY_RV_HIT", + "T76x_LS_NO_UNPARK", + "T76x_TEX_WORDS", + "T76x_TEX_BUBBLES", + "T76x_TEX_WORDS_L0", + "T76x_TEX_WORDS_DESC", + "T76x_TEX_ISSUES", + "T76x_TEX_RECIRC_FMISS", + "T76x_TEX_RECIRC_DESC", + "T76x_TEX_RECIRC_MULTI", + "T76x_TEX_RECIRC_PMISS", + "T76x_TEX_RECIRC_CONF", + "T76x_LSC_READ_HITS", + "T76x_LSC_READ_OP", + "T76x_LSC_WRITE_HITS", + "T76x_LSC_WRITE_OP", + "T76x_LSC_ATOMIC_HITS", + "T76x_LSC_ATOMIC_OP", + "T76x_LSC_LINE_FETCHES", + "T76x_LSC_DIRTY_LINE", + "T76x_LSC_SNOOPS", + "T76x_AXI_TLB_STALL", + "T76x_AXI_TLB_MISS", + "T76x_AXI_TLB_TRANSACTION", + "T76x_LS_TLB_MISS", + "T76x_LS_TLB_HIT", + "T76x_AXI_BEATS_READ", + "T76x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T76x_MMU_HIT", + "T76x_MMU_NEW_MISS", + "T76x_MMU_REPLAY_FULL", + "T76x_MMU_REPLAY_MISS", + "T76x_MMU_TABLE_WALK", + "T76x_MMU_REQUESTS", + "", + "", + "T76x_UTLB_HIT", + "T76x_UTLB_NEW_MISS", + "T76x_UTLB_REPLAY_FULL", + "T76x_UTLB_REPLAY_MISS", + "T76x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T76x_L2_EXT_WRITE_BEATS", + "T76x_L2_EXT_READ_BEATS", + "T76x_L2_ANY_LOOKUP", + "T76x_L2_READ_LOOKUP", + "T76x_L2_SREAD_LOOKUP", + "T76x_L2_READ_REPLAY", + "T76x_L2_READ_SNOOP", + "T76x_L2_READ_HIT", + "T76x_L2_CLEAN_MISS", + "T76x_L2_WRITE_LOOKUP", + "T76x_L2_SWRITE_LOOKUP", + "T76x_L2_WRITE_REPLAY", + "T76x_L2_WRITE_SNOOP", + "T76x_L2_WRITE_HIT", + "T76x_L2_EXT_READ_FULL", + "", + "T76x_L2_EXT_WRITE_FULL", + "T76x_L2_EXT_R_W_HAZARD", + "T76x_L2_EXT_READ", + "T76x_L2_EXT_READ_LINE", + "T76x_L2_EXT_WRITE", + "T76x_L2_EXT_WRITE_LINE", + "T76x_L2_EXT_WRITE_SMALL", + "T76x_L2_EXT_BARRIER", + "T76x_L2_EXT_AR_STALL", + "T76x_L2_EXT_R_BUF_FULL", + "T76x_L2_EXT_RD_BUF_FULL", + "T76x_L2_EXT_R_RAW", + "T76x_L2_EXT_W_STALL", + "T76x_L2_EXT_W_BUF_FULL", + "T76x_L2_EXT_R_BUF_FULL", + "T76x_L2_TAG_HAZARD", + "T76x_L2_SNOOP_FULL", + "T76x_L2_REPLAY_FULL" +}; + +static const char * const hardware_counters_mali_t82x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T82x_MESSAGES_SENT", + "T82x_MESSAGES_RECEIVED", + "T82x_GPU_ACTIVE", + "T82x_IRQ_ACTIVE", + "T82x_JS0_JOBS", + "T82x_JS0_TASKS", + "T82x_JS0_ACTIVE", + "", + "T82x_JS0_WAIT_READ", + "T82x_JS0_WAIT_ISSUE", + "T82x_JS0_WAIT_DEPEND", + "T82x_JS0_WAIT_FINISH", + "T82x_JS1_JOBS", + "T82x_JS1_TASKS", + "T82x_JS1_ACTIVE", + "", + "T82x_JS1_WAIT_READ", + "T82x_JS1_WAIT_ISSUE", + "T82x_JS1_WAIT_DEPEND", + "T82x_JS1_WAIT_FINISH", + "T82x_JS2_JOBS", + "T82x_JS2_TASKS", + "T82x_JS2_ACTIVE", + "", + "T82x_JS2_WAIT_READ", + "T82x_JS2_WAIT_ISSUE", + "T82x_JS2_WAIT_DEPEND", + "T82x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T82x_TI_JOBS_PROCESSED", + "T82x_TI_TRIANGLES", + "T82x_TI_QUADS", + "T82x_TI_POLYGONS", + "T82x_TI_POINTS", + "T82x_TI_LINES", + "T82x_TI_FRONT_FACING", + "T82x_TI_BACK_FACING", + "T82x_TI_PRIM_VISIBLE", + "T82x_TI_PRIM_CULLED", + "T82x_TI_PRIM_CLIPPED", + "", + "", + "", + "", + "", + "", + "", + "", + "T82x_TI_ACTIVE", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Shader Core */ + "", + "", + "", + "", + "T82x_FRAG_ACTIVE", + "T82x_FRAG_PRIMITIVES", + "T82x_FRAG_PRIMITIVES_DROPPED", + "T82x_FRAG_CYCLES_DESC", + "T82x_FRAG_CYCLES_FPKQ_ACTIVE", + "T82x_FRAG_CYCLES_VERT", + "T82x_FRAG_CYCLES_TRISETUP", + "T82x_FRAG_CYCLES_EZS_ACTIVE", + "T82x_FRAG_THREADS", + "T82x_FRAG_DUMMY_THREADS", + "T82x_FRAG_QUADS_RAST", + "T82x_FRAG_QUADS_EZS_TEST", + "T82x_FRAG_QUADS_EZS_KILLED", + "T82x_FRAG_THREADS_LZS_TEST", + "T82x_FRAG_THREADS_LZS_KILLED", + "T82x_FRAG_CYCLES_NO_TILE", + "T82x_FRAG_NUM_TILES", + "T82x_FRAG_TRANS_ELIM", + "T82x_COMPUTE_ACTIVE", + "T82x_COMPUTE_TASKS", + "T82x_COMPUTE_THREADS", + "T82x_COMPUTE_CYCLES_DESC", + "T82x_TRIPIPE_ACTIVE", + "T82x_ARITH_WORDS", + "T82x_ARITH_CYCLES_REG", + "T82x_ARITH_CYCLES_L0", + "T82x_ARITH_FRAG_DEPEND", + "T82x_LS_WORDS", + "T82x_LS_ISSUES", + "T82x_LS_REISSUE_ATTR", + "T82x_LS_REISSUES_VARY", + "T82x_LS_VARY_RV_MISS", + "T82x_LS_VARY_RV_HIT", + "T82x_LS_NO_UNPARK", + "T82x_TEX_WORDS", + "T82x_TEX_BUBBLES", + "T82x_TEX_WORDS_L0", + "T82x_TEX_WORDS_DESC", + "T82x_TEX_ISSUES", + "T82x_TEX_RECIRC_FMISS", + "T82x_TEX_RECIRC_DESC", + "T82x_TEX_RECIRC_MULTI", + "T82x_TEX_RECIRC_PMISS", + "T82x_TEX_RECIRC_CONF", + "T82x_LSC_READ_HITS", + "T82x_LSC_READ_OP", + "T82x_LSC_WRITE_HITS", + "T82x_LSC_WRITE_OP", + "T82x_LSC_ATOMIC_HITS", + "T82x_LSC_ATOMIC_OP", + "T82x_LSC_LINE_FETCHES", + "T82x_LSC_DIRTY_LINE", + "T82x_LSC_SNOOPS", + "T82x_AXI_TLB_STALL", + "T82x_AXI_TLB_MISS", + "T82x_AXI_TLB_TRANSACTION", + "T82x_LS_TLB_MISS", + "T82x_LS_TLB_HIT", + "T82x_AXI_BEATS_READ", + "T82x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T82x_MMU_HIT", + "T82x_MMU_NEW_MISS", + "T82x_MMU_REPLAY_FULL", + "T82x_MMU_REPLAY_MISS", + "T82x_MMU_TABLE_WALK", + "T82x_MMU_REQUESTS", + "", + "", + "T82x_UTLB_HIT", + "T82x_UTLB_NEW_MISS", + "T82x_UTLB_REPLAY_FULL", + "T82x_UTLB_REPLAY_MISS", + "T82x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T82x_L2_EXT_WRITE_BEATS", + "T82x_L2_EXT_READ_BEATS", + "T82x_L2_ANY_LOOKUP", + "T82x_L2_READ_LOOKUP", + "T82x_L2_SREAD_LOOKUP", + "T82x_L2_READ_REPLAY", + "T82x_L2_READ_SNOOP", + "T82x_L2_READ_HIT", + "T82x_L2_CLEAN_MISS", + "T82x_L2_WRITE_LOOKUP", + "T82x_L2_SWRITE_LOOKUP", + "T82x_L2_WRITE_REPLAY", + "T82x_L2_WRITE_SNOOP", + "T82x_L2_WRITE_HIT", + "T82x_L2_EXT_READ_FULL", + "", + "T82x_L2_EXT_WRITE_FULL", + "T82x_L2_EXT_R_W_HAZARD", + "T82x_L2_EXT_READ", + "T82x_L2_EXT_READ_LINE", + "T82x_L2_EXT_WRITE", + "T82x_L2_EXT_WRITE_LINE", + "T82x_L2_EXT_WRITE_SMALL", + "T82x_L2_EXT_BARRIER", + "T82x_L2_EXT_AR_STALL", + "T82x_L2_EXT_R_BUF_FULL", + "T82x_L2_EXT_RD_BUF_FULL", + "T82x_L2_EXT_R_RAW", + "T82x_L2_EXT_W_STALL", + "T82x_L2_EXT_W_BUF_FULL", + "T82x_L2_EXT_R_BUF_FULL", + "T82x_L2_TAG_HAZARD", + "T82x_L2_SNOOP_FULL", + "T82x_L2_REPLAY_FULL" +}; + +static const char * const hardware_counters_mali_t83x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T83x_MESSAGES_SENT", + "T83x_MESSAGES_RECEIVED", + "T83x_GPU_ACTIVE", + "T83x_IRQ_ACTIVE", + "T83x_JS0_JOBS", + "T83x_JS0_TASKS", + "T83x_JS0_ACTIVE", + "", + "T83x_JS0_WAIT_READ", + "T83x_JS0_WAIT_ISSUE", + "T83x_JS0_WAIT_DEPEND", + "T83x_JS0_WAIT_FINISH", + "T83x_JS1_JOBS", + "T83x_JS1_TASKS", + "T83x_JS1_ACTIVE", + "", + "T83x_JS1_WAIT_READ", + "T83x_JS1_WAIT_ISSUE", + "T83x_JS1_WAIT_DEPEND", + "T83x_JS1_WAIT_FINISH", + "T83x_JS2_JOBS", + "T83x_JS2_TASKS", + "T83x_JS2_ACTIVE", + "", + "T83x_JS2_WAIT_READ", + "T83x_JS2_WAIT_ISSUE", + "T83x_JS2_WAIT_DEPEND", + "T83x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T83x_TI_JOBS_PROCESSED", + "T83x_TI_TRIANGLES", + "T83x_TI_QUADS", + "T83x_TI_POLYGONS", + "T83x_TI_POINTS", + "T83x_TI_LINES", + "T83x_TI_FRONT_FACING", + "T83x_TI_BACK_FACING", + "T83x_TI_PRIM_VISIBLE", + "T83x_TI_PRIM_CULLED", + "T83x_TI_PRIM_CLIPPED", + "", + "", + "", + "", + "", + "", + "", + "", + "T83x_TI_ACTIVE", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Shader Core */ + "", + "", + "", + "", + "T83x_FRAG_ACTIVE", + "T83x_FRAG_PRIMITIVES", + "T83x_FRAG_PRIMITIVES_DROPPED", + "T83x_FRAG_CYCLES_DESC", + "T83x_FRAG_CYCLES_FPKQ_ACTIVE", + "T83x_FRAG_CYCLES_VERT", + "T83x_FRAG_CYCLES_TRISETUP", + "T83x_FRAG_CYCLES_EZS_ACTIVE", + "T83x_FRAG_THREADS", + "T83x_FRAG_DUMMY_THREADS", + "T83x_FRAG_QUADS_RAST", + "T83x_FRAG_QUADS_EZS_TEST", + "T83x_FRAG_QUADS_EZS_KILLED", + "T83x_FRAG_THREADS_LZS_TEST", + "T83x_FRAG_THREADS_LZS_KILLED", + "T83x_FRAG_CYCLES_NO_TILE", + "T83x_FRAG_NUM_TILES", + "T83x_FRAG_TRANS_ELIM", + "T83x_COMPUTE_ACTIVE", + "T83x_COMPUTE_TASKS", + "T83x_COMPUTE_THREADS", + "T83x_COMPUTE_CYCLES_DESC", + "T83x_TRIPIPE_ACTIVE", + "T83x_ARITH_WORDS", + "T83x_ARITH_CYCLES_REG", + "T83x_ARITH_CYCLES_L0", + "T83x_ARITH_FRAG_DEPEND", + "T83x_LS_WORDS", + "T83x_LS_ISSUES", + "T83x_LS_REISSUE_ATTR", + "T83x_LS_REISSUES_VARY", + "T83x_LS_VARY_RV_MISS", + "T83x_LS_VARY_RV_HIT", + "T83x_LS_NO_UNPARK", + "T83x_TEX_WORDS", + "T83x_TEX_BUBBLES", + "T83x_TEX_WORDS_L0", + "T83x_TEX_WORDS_DESC", + "T83x_TEX_ISSUES", + "T83x_TEX_RECIRC_FMISS", + "T83x_TEX_RECIRC_DESC", + "T83x_TEX_RECIRC_MULTI", + "T83x_TEX_RECIRC_PMISS", + "T83x_TEX_RECIRC_CONF", + "T83x_LSC_READ_HITS", + "T83x_LSC_READ_OP", + "T83x_LSC_WRITE_HITS", + "T83x_LSC_WRITE_OP", + "T83x_LSC_ATOMIC_HITS", + "T83x_LSC_ATOMIC_OP", + "T83x_LSC_LINE_FETCHES", + "T83x_LSC_DIRTY_LINE", + "T83x_LSC_SNOOPS", + "T83x_AXI_TLB_STALL", + "T83x_AXI_TLB_MISS", + "T83x_AXI_TLB_TRANSACTION", + "T83x_LS_TLB_MISS", + "T83x_LS_TLB_HIT", + "T83x_AXI_BEATS_READ", + "T83x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T83x_MMU_HIT", + "T83x_MMU_NEW_MISS", + "T83x_MMU_REPLAY_FULL", + "T83x_MMU_REPLAY_MISS", + "T83x_MMU_TABLE_WALK", + "T83x_MMU_REQUESTS", + "", + "", + "T83x_UTLB_HIT", + "T83x_UTLB_NEW_MISS", + "T83x_UTLB_REPLAY_FULL", + "T83x_UTLB_REPLAY_MISS", + "T83x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T83x_L2_EXT_WRITE_BEATS", + "T83x_L2_EXT_READ_BEATS", + "T83x_L2_ANY_LOOKUP", + "T83x_L2_READ_LOOKUP", + "T83x_L2_SREAD_LOOKUP", + "T83x_L2_READ_REPLAY", + "T83x_L2_READ_SNOOP", + "T83x_L2_READ_HIT", + "T83x_L2_CLEAN_MISS", + "T83x_L2_WRITE_LOOKUP", + "T83x_L2_SWRITE_LOOKUP", + "T83x_L2_WRITE_REPLAY", + "T83x_L2_WRITE_SNOOP", + "T83x_L2_WRITE_HIT", + "T83x_L2_EXT_READ_FULL", + "", + "T83x_L2_EXT_WRITE_FULL", + "T83x_L2_EXT_R_W_HAZARD", + "T83x_L2_EXT_READ", + "T83x_L2_EXT_READ_LINE", + "T83x_L2_EXT_WRITE", + "T83x_L2_EXT_WRITE_LINE", + "T83x_L2_EXT_WRITE_SMALL", + "T83x_L2_EXT_BARRIER", + "T83x_L2_EXT_AR_STALL", + "T83x_L2_EXT_R_BUF_FULL", + "T83x_L2_EXT_RD_BUF_FULL", + "T83x_L2_EXT_R_RAW", + "T83x_L2_EXT_W_STALL", + "T83x_L2_EXT_W_BUF_FULL", + "T83x_L2_EXT_R_BUF_FULL", + "T83x_L2_TAG_HAZARD", + "T83x_L2_SNOOP_FULL", + "T83x_L2_REPLAY_FULL" +}; + +static const char * const hardware_counters_mali_t86x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T86x_MESSAGES_SENT", + "T86x_MESSAGES_RECEIVED", + "T86x_GPU_ACTIVE", + "T86x_IRQ_ACTIVE", + "T86x_JS0_JOBS", + "T86x_JS0_TASKS", + "T86x_JS0_ACTIVE", + "", + "T86x_JS0_WAIT_READ", + "T86x_JS0_WAIT_ISSUE", + "T86x_JS0_WAIT_DEPEND", + "T86x_JS0_WAIT_FINISH", + "T86x_JS1_JOBS", + "T86x_JS1_TASKS", + "T86x_JS1_ACTIVE", + "", + "T86x_JS1_WAIT_READ", + "T86x_JS1_WAIT_ISSUE", + "T86x_JS1_WAIT_DEPEND", + "T86x_JS1_WAIT_FINISH", + "T86x_JS2_JOBS", + "T86x_JS2_TASKS", + "T86x_JS2_ACTIVE", + "", + "T86x_JS2_WAIT_READ", + "T86x_JS2_WAIT_ISSUE", + "T86x_JS2_WAIT_DEPEND", + "T86x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T86x_TI_JOBS_PROCESSED", + "T86x_TI_TRIANGLES", + "T86x_TI_QUADS", + "T86x_TI_POLYGONS", + "T86x_TI_POINTS", + "T86x_TI_LINES", + "T86x_TI_VCACHE_HIT", + "T86x_TI_VCACHE_MISS", + "T86x_TI_FRONT_FACING", + "T86x_TI_BACK_FACING", + "T86x_TI_PRIM_VISIBLE", + "T86x_TI_PRIM_CULLED", + "T86x_TI_PRIM_CLIPPED", + "T86x_TI_LEVEL0", + "T86x_TI_LEVEL1", + "T86x_TI_LEVEL2", + "T86x_TI_LEVEL3", + "T86x_TI_LEVEL4", + "T86x_TI_LEVEL5", + "T86x_TI_LEVEL6", + "T86x_TI_LEVEL7", + "T86x_TI_COMMAND_1", + "T86x_TI_COMMAND_2", + "T86x_TI_COMMAND_3", + "T86x_TI_COMMAND_4", + "T86x_TI_COMMAND_5_7", + "T86x_TI_COMMAND_8_15", + "T86x_TI_COMMAND_16_63", + "T86x_TI_COMMAND_64", + "T86x_TI_COMPRESS_IN", + "T86x_TI_COMPRESS_OUT", + "T86x_TI_COMPRESS_FLUSH", + "T86x_TI_TIMESTAMPS", + "T86x_TI_PCACHE_HIT", + "T86x_TI_PCACHE_MISS", + "T86x_TI_PCACHE_LINE", + "T86x_TI_PCACHE_STALL", + "T86x_TI_WRBUF_HIT", + "T86x_TI_WRBUF_MISS", + "T86x_TI_WRBUF_LINE", + "T86x_TI_WRBUF_PARTIAL", + "T86x_TI_WRBUF_STALL", + "T86x_TI_ACTIVE", + "T86x_TI_LOADING_DESC", + "T86x_TI_INDEX_WAIT", + "T86x_TI_INDEX_RANGE_WAIT", + "T86x_TI_VERTEX_WAIT", + "T86x_TI_PCACHE_WAIT", + "T86x_TI_WRBUF_WAIT", + "T86x_TI_BUS_READ", + "T86x_TI_BUS_WRITE", + "", + "", + "", + "", + "", + "T86x_TI_UTLB_HIT", + "T86x_TI_UTLB_NEW_MISS", + "T86x_TI_UTLB_REPLAY_FULL", + "T86x_TI_UTLB_REPLAY_MISS", + "T86x_TI_UTLB_STALL", + + /* Shader Core */ + "", + "", + "", + "", + "T86x_FRAG_ACTIVE", + "T86x_FRAG_PRIMITIVES", + "T86x_FRAG_PRIMITIVES_DROPPED", + "T86x_FRAG_CYCLES_DESC", + "T86x_FRAG_CYCLES_FPKQ_ACTIVE", + "T86x_FRAG_CYCLES_VERT", + "T86x_FRAG_CYCLES_TRISETUP", + "T86x_FRAG_CYCLES_EZS_ACTIVE", + "T86x_FRAG_THREADS", + "T86x_FRAG_DUMMY_THREADS", + "T86x_FRAG_QUADS_RAST", + "T86x_FRAG_QUADS_EZS_TEST", + "T86x_FRAG_QUADS_EZS_KILLED", + "T86x_FRAG_THREADS_LZS_TEST", + "T86x_FRAG_THREADS_LZS_KILLED", + "T86x_FRAG_CYCLES_NO_TILE", + "T86x_FRAG_NUM_TILES", + "T86x_FRAG_TRANS_ELIM", + "T86x_COMPUTE_ACTIVE", + "T86x_COMPUTE_TASKS", + "T86x_COMPUTE_THREADS", + "T86x_COMPUTE_CYCLES_DESC", + "T86x_TRIPIPE_ACTIVE", + "T86x_ARITH_WORDS", + "T86x_ARITH_CYCLES_REG", + "T86x_ARITH_CYCLES_L0", + "T86x_ARITH_FRAG_DEPEND", + "T86x_LS_WORDS", + "T86x_LS_ISSUES", + "T86x_LS_REISSUE_ATTR", + "T86x_LS_REISSUES_VARY", + "T86x_LS_VARY_RV_MISS", + "T86x_LS_VARY_RV_HIT", + "T86x_LS_NO_UNPARK", + "T86x_TEX_WORDS", + "T86x_TEX_BUBBLES", + "T86x_TEX_WORDS_L0", + "T86x_TEX_WORDS_DESC", + "T86x_TEX_ISSUES", + "T86x_TEX_RECIRC_FMISS", + "T86x_TEX_RECIRC_DESC", + "T86x_TEX_RECIRC_MULTI", + "T86x_TEX_RECIRC_PMISS", + "T86x_TEX_RECIRC_CONF", + "T86x_LSC_READ_HITS", + "T86x_LSC_READ_OP", + "T86x_LSC_WRITE_HITS", + "T86x_LSC_WRITE_OP", + "T86x_LSC_ATOMIC_HITS", + "T86x_LSC_ATOMIC_OP", + "T86x_LSC_LINE_FETCHES", + "T86x_LSC_DIRTY_LINE", + "T86x_LSC_SNOOPS", + "T86x_AXI_TLB_STALL", + "T86x_AXI_TLB_MISS", + "T86x_AXI_TLB_TRANSACTION", + "T86x_LS_TLB_MISS", + "T86x_LS_TLB_HIT", + "T86x_AXI_BEATS_READ", + "T86x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T86x_MMU_HIT", + "T86x_MMU_NEW_MISS", + "T86x_MMU_REPLAY_FULL", + "T86x_MMU_REPLAY_MISS", + "T86x_MMU_TABLE_WALK", + "T86x_MMU_REQUESTS", + "", + "", + "T86x_UTLB_HIT", + "T86x_UTLB_NEW_MISS", + "T86x_UTLB_REPLAY_FULL", + "T86x_UTLB_REPLAY_MISS", + "T86x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T86x_L2_EXT_WRITE_BEATS", + "T86x_L2_EXT_READ_BEATS", + "T86x_L2_ANY_LOOKUP", + "T86x_L2_READ_LOOKUP", + "T86x_L2_SREAD_LOOKUP", + "T86x_L2_READ_REPLAY", + "T86x_L2_READ_SNOOP", + "T86x_L2_READ_HIT", + "T86x_L2_CLEAN_MISS", + "T86x_L2_WRITE_LOOKUP", + "T86x_L2_SWRITE_LOOKUP", + "T86x_L2_WRITE_REPLAY", + "T86x_L2_WRITE_SNOOP", + "T86x_L2_WRITE_HIT", + "T86x_L2_EXT_READ_FULL", + "", + "T86x_L2_EXT_WRITE_FULL", + "T86x_L2_EXT_R_W_HAZARD", + "T86x_L2_EXT_READ", + "T86x_L2_EXT_READ_LINE", + "T86x_L2_EXT_WRITE", + "T86x_L2_EXT_WRITE_LINE", + "T86x_L2_EXT_WRITE_SMALL", + "T86x_L2_EXT_BARRIER", + "T86x_L2_EXT_AR_STALL", + "T86x_L2_EXT_R_BUF_FULL", + "T86x_L2_EXT_RD_BUF_FULL", + "T86x_L2_EXT_R_RAW", + "T86x_L2_EXT_W_STALL", + "T86x_L2_EXT_W_BUF_FULL", + "T86x_L2_EXT_R_BUF_FULL", + "T86x_L2_TAG_HAZARD", + "T86x_L2_SNOOP_FULL", + "T86x_L2_REPLAY_FULL" +}; + +static const char * const hardware_counters_mali_t88x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T88x_MESSAGES_SENT", + "T88x_MESSAGES_RECEIVED", + "T88x_GPU_ACTIVE", + "T88x_IRQ_ACTIVE", + "T88x_JS0_JOBS", + "T88x_JS0_TASKS", + "T88x_JS0_ACTIVE", + "", + "T88x_JS0_WAIT_READ", + "T88x_JS0_WAIT_ISSUE", + "T88x_JS0_WAIT_DEPEND", + "T88x_JS0_WAIT_FINISH", + "T88x_JS1_JOBS", + "T88x_JS1_TASKS", + "T88x_JS1_ACTIVE", + "", + "T88x_JS1_WAIT_READ", + "T88x_JS1_WAIT_ISSUE", + "T88x_JS1_WAIT_DEPEND", + "T88x_JS1_WAIT_FINISH", + "T88x_JS2_JOBS", + "T88x_JS2_TASKS", + "T88x_JS2_ACTIVE", + "", + "T88x_JS2_WAIT_READ", + "T88x_JS2_WAIT_ISSUE", + "T88x_JS2_WAIT_DEPEND", + "T88x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T88x_TI_JOBS_PROCESSED", + "T88x_TI_TRIANGLES", + "T88x_TI_QUADS", + "T88x_TI_POLYGONS", + "T88x_TI_POINTS", + "T88x_TI_LINES", + "T88x_TI_VCACHE_HIT", + "T88x_TI_VCACHE_MISS", + "T88x_TI_FRONT_FACING", + "T88x_TI_BACK_FACING", + "T88x_TI_PRIM_VISIBLE", + "T88x_TI_PRIM_CULLED", + "T88x_TI_PRIM_CLIPPED", + "T88x_TI_LEVEL0", + "T88x_TI_LEVEL1", + "T88x_TI_LEVEL2", + "T88x_TI_LEVEL3", + "T88x_TI_LEVEL4", + "T88x_TI_LEVEL5", + "T88x_TI_LEVEL6", + "T88x_TI_LEVEL7", + "T88x_TI_COMMAND_1", + "T88x_TI_COMMAND_2", + "T88x_TI_COMMAND_3", + "T88x_TI_COMMAND_4", + "T88x_TI_COMMAND_5_7", + "T88x_TI_COMMAND_8_15", + "T88x_TI_COMMAND_16_63", + "T88x_TI_COMMAND_64", + "T88x_TI_COMPRESS_IN", + "T88x_TI_COMPRESS_OUT", + "T88x_TI_COMPRESS_FLUSH", + "T88x_TI_TIMESTAMPS", + "T88x_TI_PCACHE_HIT", + "T88x_TI_PCACHE_MISS", + "T88x_TI_PCACHE_LINE", + "T88x_TI_PCACHE_STALL", + "T88x_TI_WRBUF_HIT", + "T88x_TI_WRBUF_MISS", + "T88x_TI_WRBUF_LINE", + "T88x_TI_WRBUF_PARTIAL", + "T88x_TI_WRBUF_STALL", + "T88x_TI_ACTIVE", + "T88x_TI_LOADING_DESC", + "T88x_TI_INDEX_WAIT", + "T88x_TI_INDEX_RANGE_WAIT", + "T88x_TI_VERTEX_WAIT", + "T88x_TI_PCACHE_WAIT", + "T88x_TI_WRBUF_WAIT", + "T88x_TI_BUS_READ", + "T88x_TI_BUS_WRITE", + "", + "", + "", + "", + "", + "T88x_TI_UTLB_HIT", + "T88x_TI_UTLB_NEW_MISS", + "T88x_TI_UTLB_REPLAY_FULL", + "T88x_TI_UTLB_REPLAY_MISS", + "T88x_TI_UTLB_STALL", + + /* Shader Core */ + "", + "", + "", + "", + "T88x_FRAG_ACTIVE", + "T88x_FRAG_PRIMITIVES", + "T88x_FRAG_PRIMITIVES_DROPPED", + "T88x_FRAG_CYCLES_DESC", + "T88x_FRAG_CYCLES_FPKQ_ACTIVE", + "T88x_FRAG_CYCLES_VERT", + "T88x_FRAG_CYCLES_TRISETUP", + "T88x_FRAG_CYCLES_EZS_ACTIVE", + "T88x_FRAG_THREADS", + "T88x_FRAG_DUMMY_THREADS", + "T88x_FRAG_QUADS_RAST", + "T88x_FRAG_QUADS_EZS_TEST", + "T88x_FRAG_QUADS_EZS_KILLED", + "T88x_FRAG_THREADS_LZS_TEST", + "T88x_FRAG_THREADS_LZS_KILLED", + "T88x_FRAG_CYCLES_NO_TILE", + "T88x_FRAG_NUM_TILES", + "T88x_FRAG_TRANS_ELIM", + "T88x_COMPUTE_ACTIVE", + "T88x_COMPUTE_TASKS", + "T88x_COMPUTE_THREADS", + "T88x_COMPUTE_CYCLES_DESC", + "T88x_TRIPIPE_ACTIVE", + "T88x_ARITH_WORDS", + "T88x_ARITH_CYCLES_REG", + "T88x_ARITH_CYCLES_L0", + "T88x_ARITH_FRAG_DEPEND", + "T88x_LS_WORDS", + "T88x_LS_ISSUES", + "T88x_LS_REISSUE_ATTR", + "T88x_LS_REISSUES_VARY", + "T88x_LS_VARY_RV_MISS", + "T88x_LS_VARY_RV_HIT", + "T88x_LS_NO_UNPARK", + "T88x_TEX_WORDS", + "T88x_TEX_BUBBLES", + "T88x_TEX_WORDS_L0", + "T88x_TEX_WORDS_DESC", + "T88x_TEX_ISSUES", + "T88x_TEX_RECIRC_FMISS", + "T88x_TEX_RECIRC_DESC", + "T88x_TEX_RECIRC_MULTI", + "T88x_TEX_RECIRC_PMISS", + "T88x_TEX_RECIRC_CONF", + "T88x_LSC_READ_HITS", + "T88x_LSC_READ_OP", + "T88x_LSC_WRITE_HITS", + "T88x_LSC_WRITE_OP", + "T88x_LSC_ATOMIC_HITS", + "T88x_LSC_ATOMIC_OP", + "T88x_LSC_LINE_FETCHES", + "T88x_LSC_DIRTY_LINE", + "T88x_LSC_SNOOPS", + "T88x_AXI_TLB_STALL", + "T88x_AXI_TLB_MISS", + "T88x_AXI_TLB_TRANSACTION", + "T88x_LS_TLB_MISS", + "T88x_LS_TLB_HIT", + "T88x_AXI_BEATS_READ", + "T88x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T88x_MMU_HIT", + "T88x_MMU_NEW_MISS", + "T88x_MMU_REPLAY_FULL", + "T88x_MMU_REPLAY_MISS", + "T88x_MMU_TABLE_WALK", + "T88x_MMU_REQUESTS", + "", + "", + "T88x_UTLB_HIT", + "T88x_UTLB_NEW_MISS", + "T88x_UTLB_REPLAY_FULL", + "T88x_UTLB_REPLAY_MISS", + "T88x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T88x_L2_EXT_WRITE_BEATS", + "T88x_L2_EXT_READ_BEATS", + "T88x_L2_ANY_LOOKUP", + "T88x_L2_READ_LOOKUP", + "T88x_L2_SREAD_LOOKUP", + "T88x_L2_READ_REPLAY", + "T88x_L2_READ_SNOOP", + "T88x_L2_READ_HIT", + "T88x_L2_CLEAN_MISS", + "T88x_L2_WRITE_LOOKUP", + "T88x_L2_SWRITE_LOOKUP", + "T88x_L2_WRITE_REPLAY", + "T88x_L2_WRITE_SNOOP", + "T88x_L2_WRITE_HIT", + "T88x_L2_EXT_READ_FULL", + "", + "T88x_L2_EXT_WRITE_FULL", + "T88x_L2_EXT_R_W_HAZARD", + "T88x_L2_EXT_READ", + "T88x_L2_EXT_READ_LINE", + "T88x_L2_EXT_WRITE", + "T88x_L2_EXT_WRITE_LINE", + "T88x_L2_EXT_WRITE_SMALL", + "T88x_L2_EXT_BARRIER", + "T88x_L2_EXT_AR_STALL", + "T88x_L2_EXT_R_BUF_FULL", + "T88x_L2_EXT_RD_BUF_FULL", + "T88x_L2_EXT_R_RAW", + "T88x_L2_EXT_W_STALL", + "T88x_L2_EXT_W_BUF_FULL", + "T88x_L2_EXT_R_BUF_FULL", + "T88x_L2_TAG_HAZARD", + "T88x_L2_SNOOP_FULL", + "T88x_L2_REPLAY_FULL" +}; + +#include "mali_kbase_gator_hwcnt_names_tmix.h" + +#include "mali_kbase_gator_hwcnt_names_thex.h" + +#include "mali_kbase_gator_hwcnt_names_tsix.h" + + + +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_thex.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_thex.h new file mode 100755 index 000000000000..15fd4efdc6ca --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_thex.h @@ -0,0 +1,291 @@ +/* + * + * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * This header was autogenerated, it should not be edited. + */ + +#ifndef _KBASE_GATOR_HWCNT_NAMES_THEX_H_ +#define _KBASE_GATOR_HWCNT_NAMES_THEX_H_ + +static const char * const hardware_counters_mali_tHEx[] = { + /* Performance counters for the Job Manager */ + "", + "", + "", + "", + "THEx_MESSAGES_SENT", + "THEx_MESSAGES_RECEIVED", + "THEx_GPU_ACTIVE", + "THEx_IRQ_ACTIVE", + "THEx_JS0_JOBS", + "THEx_JS0_TASKS", + "THEx_JS0_ACTIVE", + "", + "THEx_JS0_WAIT_READ", + "THEx_JS0_WAIT_ISSUE", + "THEx_JS0_WAIT_DEPEND", + "THEx_JS0_WAIT_FINISH", + "THEx_JS1_JOBS", + "THEx_JS1_TASKS", + "THEx_JS1_ACTIVE", + "", + "THEx_JS1_WAIT_READ", + "THEx_JS1_WAIT_ISSUE", + "THEx_JS1_WAIT_DEPEND", + "THEx_JS1_WAIT_FINISH", + "THEx_JS2_JOBS", + "THEx_JS2_TASKS", + "THEx_JS2_ACTIVE", + "", + "THEx_JS2_WAIT_READ", + "THEx_JS2_WAIT_ISSUE", + "THEx_JS2_WAIT_DEPEND", + "THEx_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Performance counters for the Tiler */ + "", + "", + "", + "", + "THEx_TILER_ACTIVE", + "THEx_JOBS_PROCESSED", + "THEx_TRIANGLES", + "THEx_LINES", + "THEx_POINTS", + "THEx_FRONT_FACING", + "THEx_BACK_FACING", + "THEx_PRIM_VISIBLE", + "THEx_PRIM_CULLED", + "THEx_PRIM_CLIPPED", + "THEx_PRIM_SAT_CULLED", + "THEx_BIN_ALLOC_INIT", + "THEx_BIN_ALLOC_OVERFLOW", + "THEx_BUS_READ", + "", + "THEx_BUS_WRITE", + "THEx_LOADING_DESC", + "THEx_IDVS_POS_SHAD_REQ", + "THEx_IDVS_POS_SHAD_WAIT", + "THEx_IDVS_POS_SHAD_STALL", + "THEx_IDVS_POS_FIFO_FULL", + "THEx_PREFETCH_STALL", + "THEx_VCACHE_HIT", + "THEx_VCACHE_MISS", + "THEx_VCACHE_LINE_WAIT", + "THEx_VFETCH_POS_READ_WAIT", + "THEx_VFETCH_VERTEX_WAIT", + "THEx_VFETCH_STALL", + "THEx_PRIMASSY_STALL", + "THEx_BBOX_GEN_STALL", + "THEx_IDVS_VBU_HIT", + "THEx_IDVS_VBU_MISS", + "THEx_IDVS_VBU_LINE_DEALLOCATE", + "THEx_IDVS_VAR_SHAD_REQ", + "THEx_IDVS_VAR_SHAD_STALL", + "THEx_BINNER_STALL", + "THEx_ITER_STALL", + "THEx_COMPRESS_MISS", + "THEx_COMPRESS_STALL", + "THEx_PCACHE_HIT", + "THEx_PCACHE_MISS", + "THEx_PCACHE_MISS_STALL", + "THEx_PCACHE_EVICT_STALL", + "THEx_PMGR_PTR_WR_STALL", + "THEx_PMGR_PTR_RD_STALL", + "THEx_PMGR_CMD_WR_STALL", + "THEx_WRBUF_ACTIVE", + "THEx_WRBUF_HIT", + "THEx_WRBUF_MISS", + "THEx_WRBUF_NO_FREE_LINE_STALL", + "THEx_WRBUF_NO_AXI_ID_STALL", + "THEx_WRBUF_AXI_STALL", + "", + "", + "", + "THEx_UTLB_TRANS", + "THEx_UTLB_TRANS_HIT", + "THEx_UTLB_TRANS_STALL", + "THEx_UTLB_TRANS_MISS_DELAY", + "THEx_UTLB_MMU_REQ", + + /* Performance counters for the Shader Core */ + "", + "", + "", + "", + "THEx_FRAG_ACTIVE", + "THEx_FRAG_PRIMITIVES", + "THEx_FRAG_PRIM_RAST", + "THEx_FRAG_FPK_ACTIVE", + "THEx_FRAG_STARVING", + "THEx_FRAG_WARPS", + "THEx_FRAG_PARTIAL_WARPS", + "THEx_FRAG_QUADS_RAST", + "THEx_FRAG_QUADS_EZS_TEST", + "THEx_FRAG_QUADS_EZS_UPDATE", + "THEx_FRAG_QUADS_EZS_KILL", + "THEx_FRAG_LZS_TEST", + "THEx_FRAG_LZS_KILL", + "", + "THEx_FRAG_PTILES", + "THEx_FRAG_TRANS_ELIM", + "THEx_QUAD_FPK_KILLER", + "", + "THEx_COMPUTE_ACTIVE", + "THEx_COMPUTE_TASKS", + "THEx_COMPUTE_WARPS", + "THEx_COMPUTE_STARVING", + "THEx_EXEC_CORE_ACTIVE", + "THEx_EXEC_ACTIVE", + "THEx_EXEC_INSTR_COUNT", + "THEx_EXEC_INSTR_DIVERGED", + "THEx_EXEC_INSTR_STARVING", + "THEx_ARITH_INSTR_SINGLE_FMA", + "THEx_ARITH_INSTR_DOUBLE", + "THEx_ARITH_INSTR_MSG", + "THEx_ARITH_INSTR_MSG_ONLY", + "THEx_TEX_INSTR", + "THEx_TEX_INSTR_MIPMAP", + "THEx_TEX_INSTR_COMPRESSED", + "THEx_TEX_INSTR_3D", + "THEx_TEX_INSTR_TRILINEAR", + "THEx_TEX_COORD_ISSUE", + "THEx_TEX_COORD_STALL", + "THEx_TEX_STARVE_CACHE", + "THEx_TEX_STARVE_FILTER", + "THEx_LS_MEM_READ_FULL", + "THEx_LS_MEM_READ_SHORT", + "THEx_LS_MEM_WRITE_FULL", + "THEx_LS_MEM_WRITE_SHORT", + "THEx_LS_MEM_ATOMIC", + "THEx_VARY_INSTR", + "THEx_VARY_SLOT_32", + "THEx_VARY_SLOT_16", + "THEx_ATTR_INSTR", + "THEx_ARITH_INSTR_FP_MUL", + "THEx_BEATS_RD_FTC", + "THEx_BEATS_RD_FTC_EXT", + "THEx_BEATS_RD_LSC", + "THEx_BEATS_RD_LSC_EXT", + "THEx_BEATS_RD_TEX", + "THEx_BEATS_RD_TEX_EXT", + "THEx_BEATS_RD_OTHER", + "THEx_BEATS_WR_LSC", + "THEx_BEATS_WR_TIB", + "", + + /* Performance counters for the Memory System */ + "", + "", + "", + "", + "THEx_MMU_REQUESTS", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "THEx_L2_RD_MSG_IN", + "THEx_L2_RD_MSG_IN_STALL", + "THEx_L2_WR_MSG_IN", + "THEx_L2_WR_MSG_IN_STALL", + "THEx_L2_SNP_MSG_IN", + "THEx_L2_SNP_MSG_IN_STALL", + "THEx_L2_RD_MSG_OUT", + "THEx_L2_RD_MSG_OUT_STALL", + "THEx_L2_WR_MSG_OUT", + "THEx_L2_ANY_LOOKUP", + "THEx_L2_READ_LOOKUP", + "THEx_L2_WRITE_LOOKUP", + "THEx_L2_EXT_SNOOP_LOOKUP", + "THEx_L2_EXT_READ", + "THEx_L2_EXT_READ_NOSNP", + "THEx_L2_EXT_READ_UNIQUE", + "THEx_L2_EXT_READ_BEATS", + "THEx_L2_EXT_AR_STALL", + "THEx_L2_EXT_AR_CNT_Q1", + "THEx_L2_EXT_AR_CNT_Q2", + "THEx_L2_EXT_AR_CNT_Q3", + "THEx_L2_EXT_RRESP_0_127", + "THEx_L2_EXT_RRESP_128_191", + "THEx_L2_EXT_RRESP_192_255", + "THEx_L2_EXT_RRESP_256_319", + "THEx_L2_EXT_RRESP_320_383", + "THEx_L2_EXT_WRITE", + "THEx_L2_EXT_WRITE_NOSNP_FULL", + "THEx_L2_EXT_WRITE_NOSNP_PTL", + "THEx_L2_EXT_WRITE_SNP_FULL", + "THEx_L2_EXT_WRITE_SNP_PTL", + "THEx_L2_EXT_WRITE_BEATS", + "THEx_L2_EXT_W_STALL", + "THEx_L2_EXT_AW_CNT_Q1", + "THEx_L2_EXT_AW_CNT_Q2", + "THEx_L2_EXT_AW_CNT_Q3", + "THEx_L2_EXT_SNOOP", + "THEx_L2_EXT_SNOOP_STALL", + "THEx_L2_EXT_SNOOP_RESP_CLEAN", + "THEx_L2_EXT_SNOOP_RESP_DATA", + "THEx_L2_EXT_SNOOP_INTERNAL", + "", + "", + "", + "", + "", + "", + "", +}; + +#endif /* _KBASE_GATOR_HWCNT_NAMES_THEX_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tmix.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tmix.h new file mode 100755 index 000000000000..8a215f723570 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tmix.h @@ -0,0 +1,291 @@ +/* + * + * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * This header was autogenerated, it should not be edited. + */ + +#ifndef _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ +#define _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ + +static const char * const hardware_counters_mali_tMIx[] = { + /* Performance counters for the Job Manager */ + "", + "", + "", + "", + "TMIx_MESSAGES_SENT", + "TMIx_MESSAGES_RECEIVED", + "TMIx_GPU_ACTIVE", + "TMIx_IRQ_ACTIVE", + "TMIx_JS0_JOBS", + "TMIx_JS0_TASKS", + "TMIx_JS0_ACTIVE", + "", + "TMIx_JS0_WAIT_READ", + "TMIx_JS0_WAIT_ISSUE", + "TMIx_JS0_WAIT_DEPEND", + "TMIx_JS0_WAIT_FINISH", + "TMIx_JS1_JOBS", + "TMIx_JS1_TASKS", + "TMIx_JS1_ACTIVE", + "", + "TMIx_JS1_WAIT_READ", + "TMIx_JS1_WAIT_ISSUE", + "TMIx_JS1_WAIT_DEPEND", + "TMIx_JS1_WAIT_FINISH", + "TMIx_JS2_JOBS", + "TMIx_JS2_TASKS", + "TMIx_JS2_ACTIVE", + "", + "TMIx_JS2_WAIT_READ", + "TMIx_JS2_WAIT_ISSUE", + "TMIx_JS2_WAIT_DEPEND", + "TMIx_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Performance counters for the Tiler */ + "", + "", + "", + "", + "TMIx_TILER_ACTIVE", + "TMIx_JOBS_PROCESSED", + "TMIx_TRIANGLES", + "TMIx_LINES", + "TMIx_POINTS", + "TMIx_FRONT_FACING", + "TMIx_BACK_FACING", + "TMIx_PRIM_VISIBLE", + "TMIx_PRIM_CULLED", + "TMIx_PRIM_CLIPPED", + "TMIx_PRIM_SAT_CULLED", + "TMIx_BIN_ALLOC_INIT", + "TMIx_BIN_ALLOC_OVERFLOW", + "TMIx_BUS_READ", + "", + "TMIx_BUS_WRITE", + "TMIx_LOADING_DESC", + "TMIx_IDVS_POS_SHAD_REQ", + "TMIx_IDVS_POS_SHAD_WAIT", + "TMIx_IDVS_POS_SHAD_STALL", + "TMIx_IDVS_POS_FIFO_FULL", + "TMIx_PREFETCH_STALL", + "TMIx_VCACHE_HIT", + "TMIx_VCACHE_MISS", + "TMIx_VCACHE_LINE_WAIT", + "TMIx_VFETCH_POS_READ_WAIT", + "TMIx_VFETCH_VERTEX_WAIT", + "TMIx_VFETCH_STALL", + "TMIx_PRIMASSY_STALL", + "TMIx_BBOX_GEN_STALL", + "TMIx_IDVS_VBU_HIT", + "TMIx_IDVS_VBU_MISS", + "TMIx_IDVS_VBU_LINE_DEALLOCATE", + "TMIx_IDVS_VAR_SHAD_REQ", + "TMIx_IDVS_VAR_SHAD_STALL", + "TMIx_BINNER_STALL", + "TMIx_ITER_STALL", + "TMIx_COMPRESS_MISS", + "TMIx_COMPRESS_STALL", + "TMIx_PCACHE_HIT", + "TMIx_PCACHE_MISS", + "TMIx_PCACHE_MISS_STALL", + "TMIx_PCACHE_EVICT_STALL", + "TMIx_PMGR_PTR_WR_STALL", + "TMIx_PMGR_PTR_RD_STALL", + "TMIx_PMGR_CMD_WR_STALL", + "TMIx_WRBUF_ACTIVE", + "TMIx_WRBUF_HIT", + "TMIx_WRBUF_MISS", + "TMIx_WRBUF_NO_FREE_LINE_STALL", + "TMIx_WRBUF_NO_AXI_ID_STALL", + "TMIx_WRBUF_AXI_STALL", + "", + "", + "", + "TMIx_UTLB_TRANS", + "TMIx_UTLB_TRANS_HIT", + "TMIx_UTLB_TRANS_STALL", + "TMIx_UTLB_TRANS_MISS_DELAY", + "TMIx_UTLB_MMU_REQ", + + /* Performance counters for the Shader Core */ + "", + "", + "", + "", + "TMIx_FRAG_ACTIVE", + "TMIx_FRAG_PRIMITIVES", + "TMIx_FRAG_PRIM_RAST", + "TMIx_FRAG_FPK_ACTIVE", + "TMIx_FRAG_STARVING", + "TMIx_FRAG_WARPS", + "TMIx_FRAG_PARTIAL_WARPS", + "TMIx_FRAG_QUADS_RAST", + "TMIx_FRAG_QUADS_EZS_TEST", + "TMIx_FRAG_QUADS_EZS_UPDATE", + "TMIx_FRAG_QUADS_EZS_KILL", + "TMIx_FRAG_LZS_TEST", + "TMIx_FRAG_LZS_KILL", + "", + "TMIx_FRAG_PTILES", + "TMIx_FRAG_TRANS_ELIM", + "TMIx_QUAD_FPK_KILLER", + "", + "TMIx_COMPUTE_ACTIVE", + "TMIx_COMPUTE_TASKS", + "TMIx_COMPUTE_WARPS", + "TMIx_COMPUTE_STARVING", + "TMIx_EXEC_CORE_ACTIVE", + "TMIx_EXEC_ACTIVE", + "TMIx_EXEC_INSTR_COUNT", + "TMIx_EXEC_INSTR_DIVERGED", + "TMIx_EXEC_INSTR_STARVING", + "TMIx_ARITH_INSTR_SINGLE_FMA", + "TMIx_ARITH_INSTR_DOUBLE", + "TMIx_ARITH_INSTR_MSG", + "TMIx_ARITH_INSTR_MSG_ONLY", + "TMIx_TEX_INSTR", + "TMIx_TEX_INSTR_MIPMAP", + "TMIx_TEX_INSTR_COMPRESSED", + "TMIx_TEX_INSTR_3D", + "TMIx_TEX_INSTR_TRILINEAR", + "TMIx_TEX_COORD_ISSUE", + "TMIx_TEX_COORD_STALL", + "TMIx_TEX_STARVE_CACHE", + "TMIx_TEX_STARVE_FILTER", + "TMIx_LS_MEM_READ_FULL", + "TMIx_LS_MEM_READ_SHORT", + "TMIx_LS_MEM_WRITE_FULL", + "TMIx_LS_MEM_WRITE_SHORT", + "TMIx_LS_MEM_ATOMIC", + "TMIx_VARY_INSTR", + "TMIx_VARY_SLOT_32", + "TMIx_VARY_SLOT_16", + "TMIx_ATTR_INSTR", + "TMIx_ARITH_INSTR_FP_MUL", + "TMIx_BEATS_RD_FTC", + "TMIx_BEATS_RD_FTC_EXT", + "TMIx_BEATS_RD_LSC", + "TMIx_BEATS_RD_LSC_EXT", + "TMIx_BEATS_RD_TEX", + "TMIx_BEATS_RD_TEX_EXT", + "TMIx_BEATS_RD_OTHER", + "TMIx_BEATS_WR_LSC", + "TMIx_BEATS_WR_TIB", + "", + + /* Performance counters for the Memory System */ + "", + "", + "", + "", + "TMIx_MMU_REQUESTS", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "TMIx_L2_RD_MSG_IN", + "TMIx_L2_RD_MSG_IN_STALL", + "TMIx_L2_WR_MSG_IN", + "TMIx_L2_WR_MSG_IN_STALL", + "TMIx_L2_SNP_MSG_IN", + "TMIx_L2_SNP_MSG_IN_STALL", + "TMIx_L2_RD_MSG_OUT", + "TMIx_L2_RD_MSG_OUT_STALL", + "TMIx_L2_WR_MSG_OUT", + "TMIx_L2_ANY_LOOKUP", + "TMIx_L2_READ_LOOKUP", + "TMIx_L2_WRITE_LOOKUP", + "TMIx_L2_EXT_SNOOP_LOOKUP", + "TMIx_L2_EXT_READ", + "TMIx_L2_EXT_READ_NOSNP", + "TMIx_L2_EXT_READ_UNIQUE", + "TMIx_L2_EXT_READ_BEATS", + "TMIx_L2_EXT_AR_STALL", + "TMIx_L2_EXT_AR_CNT_Q1", + "TMIx_L2_EXT_AR_CNT_Q2", + "TMIx_L2_EXT_AR_CNT_Q3", + "TMIx_L2_EXT_RRESP_0_127", + "TMIx_L2_EXT_RRESP_128_191", + "TMIx_L2_EXT_RRESP_192_255", + "TMIx_L2_EXT_RRESP_256_319", + "TMIx_L2_EXT_RRESP_320_383", + "TMIx_L2_EXT_WRITE", + "TMIx_L2_EXT_WRITE_NOSNP_FULL", + "TMIx_L2_EXT_WRITE_NOSNP_PTL", + "TMIx_L2_EXT_WRITE_SNP_FULL", + "TMIx_L2_EXT_WRITE_SNP_PTL", + "TMIx_L2_EXT_WRITE_BEATS", + "TMIx_L2_EXT_W_STALL", + "TMIx_L2_EXT_AW_CNT_Q1", + "TMIx_L2_EXT_AW_CNT_Q2", + "TMIx_L2_EXT_AW_CNT_Q3", + "TMIx_L2_EXT_SNOOP", + "TMIx_L2_EXT_SNOOP_STALL", + "TMIx_L2_EXT_SNOOP_RESP_CLEAN", + "TMIx_L2_EXT_SNOOP_RESP_DATA", + "TMIx_L2_EXT_SNOOP_INTERNAL", + "", + "", + "", + "", + "", + "", + "", +}; + +#endif /* _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tsix.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tsix.h new file mode 100755 index 000000000000..fb6a1437a1f6 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gator_hwcnt_names_tsix.h @@ -0,0 +1,291 @@ +/* + * + * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * This header was autogenerated, it should not be edited. + */ + +#ifndef _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ +#define _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ + +static const char * const hardware_counters_mali_tSIx[] = { + /* Performance counters for the Job Manager */ + "", + "", + "", + "", + "TSIx_MESSAGES_SENT", + "TSIx_MESSAGES_RECEIVED", + "TSIx_GPU_ACTIVE", + "TSIx_IRQ_ACTIVE", + "TSIx_JS0_JOBS", + "TSIx_JS0_TASKS", + "TSIx_JS0_ACTIVE", + "", + "TSIx_JS0_WAIT_READ", + "TSIx_JS0_WAIT_ISSUE", + "TSIx_JS0_WAIT_DEPEND", + "TSIx_JS0_WAIT_FINISH", + "TSIx_JS1_JOBS", + "TSIx_JS1_TASKS", + "TSIx_JS1_ACTIVE", + "", + "TSIx_JS1_WAIT_READ", + "TSIx_JS1_WAIT_ISSUE", + "TSIx_JS1_WAIT_DEPEND", + "TSIx_JS1_WAIT_FINISH", + "TSIx_JS2_JOBS", + "TSIx_JS2_TASKS", + "TSIx_JS2_ACTIVE", + "", + "TSIx_JS2_WAIT_READ", + "TSIx_JS2_WAIT_ISSUE", + "TSIx_JS2_WAIT_DEPEND", + "TSIx_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Performance counters for the Tiler */ + "", + "", + "", + "", + "TSIx_TILER_ACTIVE", + "TSIx_JOBS_PROCESSED", + "TSIx_TRIANGLES", + "TSIx_LINES", + "TSIx_POINTS", + "TSIx_FRONT_FACING", + "TSIx_BACK_FACING", + "TSIx_PRIM_VISIBLE", + "TSIx_PRIM_CULLED", + "TSIx_PRIM_CLIPPED", + "TSIx_PRIM_SAT_CULLED", + "TSIx_BIN_ALLOC_INIT", + "TSIx_BIN_ALLOC_OVERFLOW", + "TSIx_BUS_READ", + "", + "TSIx_BUS_WRITE", + "TSIx_LOADING_DESC", + "TSIx_IDVS_POS_SHAD_REQ", + "TSIx_IDVS_POS_SHAD_WAIT", + "TSIx_IDVS_POS_SHAD_STALL", + "TSIx_IDVS_POS_FIFO_FULL", + "TSIx_PREFETCH_STALL", + "TSIx_VCACHE_HIT", + "TSIx_VCACHE_MISS", + "TSIx_VCACHE_LINE_WAIT", + "TSIx_VFETCH_POS_READ_WAIT", + "TSIx_VFETCH_VERTEX_WAIT", + "TSIx_VFETCH_STALL", + "TSIx_PRIMASSY_STALL", + "TSIx_BBOX_GEN_STALL", + "TSIx_IDVS_VBU_HIT", + "TSIx_IDVS_VBU_MISS", + "TSIx_IDVS_VBU_LINE_DEALLOCATE", + "TSIx_IDVS_VAR_SHAD_REQ", + "TSIx_IDVS_VAR_SHAD_STALL", + "TSIx_BINNER_STALL", + "TSIx_ITER_STALL", + "TSIx_COMPRESS_MISS", + "TSIx_COMPRESS_STALL", + "TSIx_PCACHE_HIT", + "TSIx_PCACHE_MISS", + "TSIx_PCACHE_MISS_STALL", + "TSIx_PCACHE_EVICT_STALL", + "TSIx_PMGR_PTR_WR_STALL", + "TSIx_PMGR_PTR_RD_STALL", + "TSIx_PMGR_CMD_WR_STALL", + "TSIx_WRBUF_ACTIVE", + "TSIx_WRBUF_HIT", + "TSIx_WRBUF_MISS", + "TSIx_WRBUF_NO_FREE_LINE_STALL", + "TSIx_WRBUF_NO_AXI_ID_STALL", + "TSIx_WRBUF_AXI_STALL", + "", + "", + "", + "TSIx_UTLB_TRANS", + "TSIx_UTLB_TRANS_HIT", + "TSIx_UTLB_TRANS_STALL", + "TSIx_UTLB_TRANS_MISS_DELAY", + "TSIx_UTLB_MMU_REQ", + + /* Performance counters for the Shader Core */ + "", + "", + "", + "", + "TSIx_FRAG_ACTIVE", + "TSIx_FRAG_PRIMITIVES", + "TSIx_FRAG_PRIM_RAST", + "TSIx_FRAG_FPK_ACTIVE", + "TSIx_FRAG_STARVING", + "TSIx_FRAG_WARPS", + "TSIx_FRAG_PARTIAL_WARPS", + "TSIx_FRAG_QUADS_RAST", + "TSIx_FRAG_QUADS_EZS_TEST", + "TSIx_FRAG_QUADS_EZS_UPDATE", + "TSIx_FRAG_QUADS_EZS_KILL", + "TSIx_FRAG_LZS_TEST", + "TSIx_FRAG_LZS_KILL", + "", + "TSIx_FRAG_PTILES", + "TSIx_FRAG_TRANS_ELIM", + "TSIx_QUAD_FPK_KILLER", + "", + "TSIx_COMPUTE_ACTIVE", + "TSIx_COMPUTE_TASKS", + "TSIx_COMPUTE_WARPS", + "TSIx_COMPUTE_STARVING", + "TSIx_EXEC_CORE_ACTIVE", + "TSIx_EXEC_ACTIVE", + "TSIx_EXEC_INSTR_COUNT", + "TSIx_EXEC_INSTR_DIVERGED", + "TSIx_EXEC_INSTR_STARVING", + "TSIx_ARITH_INSTR_SINGLE_FMA", + "TSIx_ARITH_INSTR_DOUBLE", + "TSIx_ARITH_INSTR_MSG", + "TSIx_ARITH_INSTR_MSG_ONLY", + "TSIx_TEX_MSGI_NUM_QUADS", + "TSIx_TEX_DFCH_NUM_PASSES", + "TSIx_TEX_DFCH_NUM_PASSES_MISS", + "TSIx_TEX_DFCH_NUM_PASSES_MIP_MAP", + "TSIx_TEX_TIDX_NUM_SPLIT_MIP_MAP", + "TSIx_TEX_TFCH_NUM_LINES_FETCHED", + "TSIx_TEX_TFCH_NUM_LINES_FETCHED_BLOCK", + "TSIx_TEX_TFCH_NUM_OPERATIONS", + "TSIx_TEX_FILT_NUM_OPERATIONS", + "TSIx_LS_MEM_READ_FULL", + "TSIx_LS_MEM_READ_SHORT", + "TSIx_LS_MEM_WRITE_FULL", + "TSIx_LS_MEM_WRITE_SHORT", + "TSIx_LS_MEM_ATOMIC", + "TSIx_VARY_INSTR", + "TSIx_VARY_SLOT_32", + "TSIx_VARY_SLOT_16", + "TSIx_ATTR_INSTR", + "TSIx_ARITH_INSTR_FP_MUL", + "TSIx_BEATS_RD_FTC", + "TSIx_BEATS_RD_FTC_EXT", + "TSIx_BEATS_RD_LSC", + "TSIx_BEATS_RD_LSC_EXT", + "TSIx_BEATS_RD_TEX", + "TSIx_BEATS_RD_TEX_EXT", + "TSIx_BEATS_RD_OTHER", + "TSIx_BEATS_WR_LSC", + "TSIx_BEATS_WR_TIB", + "", + + /* Performance counters for the Memory System */ + "", + "", + "", + "", + "TSIx_MMU_REQUESTS", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "TSIx_L2_RD_MSG_IN", + "TSIx_L2_RD_MSG_IN_STALL", + "TSIx_L2_WR_MSG_IN", + "TSIx_L2_WR_MSG_IN_STALL", + "TSIx_L2_SNP_MSG_IN", + "TSIx_L2_SNP_MSG_IN_STALL", + "TSIx_L2_RD_MSG_OUT", + "TSIx_L2_RD_MSG_OUT_STALL", + "TSIx_L2_WR_MSG_OUT", + "TSIx_L2_ANY_LOOKUP", + "TSIx_L2_READ_LOOKUP", + "TSIx_L2_WRITE_LOOKUP", + "TSIx_L2_EXT_SNOOP_LOOKUP", + "TSIx_L2_EXT_READ", + "TSIx_L2_EXT_READ_NOSNP", + "TSIx_L2_EXT_READ_UNIQUE", + "TSIx_L2_EXT_READ_BEATS", + "TSIx_L2_EXT_AR_STALL", + "TSIx_L2_EXT_AR_CNT_Q1", + "TSIx_L2_EXT_AR_CNT_Q2", + "TSIx_L2_EXT_AR_CNT_Q3", + "TSIx_L2_EXT_RRESP_0_127", + "TSIx_L2_EXT_RRESP_128_191", + "TSIx_L2_EXT_RRESP_192_255", + "TSIx_L2_EXT_RRESP_256_319", + "TSIx_L2_EXT_RRESP_320_383", + "TSIx_L2_EXT_WRITE", + "TSIx_L2_EXT_WRITE_NOSNP_FULL", + "TSIx_L2_EXT_WRITE_NOSNP_PTL", + "TSIx_L2_EXT_WRITE_SNP_FULL", + "TSIx_L2_EXT_WRITE_SNP_PTL", + "TSIx_L2_EXT_WRITE_BEATS", + "TSIx_L2_EXT_W_STALL", + "TSIx_L2_EXT_AW_CNT_Q1", + "TSIx_L2_EXT_AW_CNT_Q2", + "TSIx_L2_EXT_AW_CNT_Q3", + "TSIx_L2_EXT_SNOOP", + "TSIx_L2_EXT_SNOOP_STALL", + "TSIx_L2_EXT_SNOOP_RESP_CLEAN", + "TSIx_L2_EXT_SNOOP_RESP_DATA", + "TSIx_L2_EXT_SNOOP_INTERNAL", + "", + "", + "", + "", + "", + "", + "", +}; + +#endif /* _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_id.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_id.h new file mode 100755 index 000000000000..2d368dfaf644 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_id.h @@ -0,0 +1,118 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +#ifndef _KBASE_GPU_ID_H_ +#define _KBASE_GPU_ID_H_ + +/* GPU_ID register */ +#define GPU_ID_VERSION_STATUS_SHIFT 0 +#define GPU_ID_VERSION_MINOR_SHIFT 4 +#define GPU_ID_VERSION_MAJOR_SHIFT 12 +#define GPU_ID_VERSION_PRODUCT_ID_SHIFT 16 +#define GPU_ID_VERSION_STATUS (0xF << GPU_ID_VERSION_STATUS_SHIFT) +#define GPU_ID_VERSION_MINOR (0xFF << GPU_ID_VERSION_MINOR_SHIFT) +#define GPU_ID_VERSION_MAJOR (0xF << GPU_ID_VERSION_MAJOR_SHIFT) +#define GPU_ID_VERSION_PRODUCT_ID (0xFFFF << GPU_ID_VERSION_PRODUCT_ID_SHIFT) + +/* Values for GPU_ID_VERSION_PRODUCT_ID bitfield */ +#define GPU_ID_PI_T60X 0x6956 +#define GPU_ID_PI_T62X 0x0620 +#define GPU_ID_PI_T76X 0x0750 +#define GPU_ID_PI_T72X 0x0720 +#define GPU_ID_PI_TFRX 0x0880 +#define GPU_ID_PI_T86X 0x0860 +#define GPU_ID_PI_T82X 0x0820 +#define GPU_ID_PI_T83X 0x0830 + +/* New GPU ID format when PRODUCT_ID is >= 0x1000 (and not 0x6956) */ +#define GPU_ID_PI_NEW_FORMAT_START 0x1000 +#define GPU_ID_IS_NEW_FORMAT(product_id) ((product_id) != GPU_ID_PI_T60X && \ + (product_id) >= \ + GPU_ID_PI_NEW_FORMAT_START) + +#define GPU_ID2_VERSION_STATUS_SHIFT 0 +#define GPU_ID2_VERSION_MINOR_SHIFT 4 +#define GPU_ID2_VERSION_MAJOR_SHIFT 12 +#define GPU_ID2_PRODUCT_MAJOR_SHIFT 16 +#define GPU_ID2_ARCH_REV_SHIFT 20 +#define GPU_ID2_ARCH_MINOR_SHIFT 24 +#define GPU_ID2_ARCH_MAJOR_SHIFT 28 +#define GPU_ID2_VERSION_STATUS (0xF << GPU_ID2_VERSION_STATUS_SHIFT) +#define GPU_ID2_VERSION_MINOR (0xFF << GPU_ID2_VERSION_MINOR_SHIFT) +#define GPU_ID2_VERSION_MAJOR (0xF << GPU_ID2_VERSION_MAJOR_SHIFT) +#define GPU_ID2_PRODUCT_MAJOR (0xF << GPU_ID2_PRODUCT_MAJOR_SHIFT) +#define GPU_ID2_ARCH_REV (0xF << GPU_ID2_ARCH_REV_SHIFT) +#define GPU_ID2_ARCH_MINOR (0xF << GPU_ID2_ARCH_MINOR_SHIFT) +#define GPU_ID2_ARCH_MAJOR (0xF << GPU_ID2_ARCH_MAJOR_SHIFT) +#define GPU_ID2_PRODUCT_MODEL (GPU_ID2_ARCH_MAJOR | GPU_ID2_PRODUCT_MAJOR) +#define GPU_ID2_VERSION (GPU_ID2_VERSION_MAJOR | \ + GPU_ID2_VERSION_MINOR | \ + GPU_ID2_VERSION_STATUS) + +/* Helper macro to create a partial GPU_ID (new format) that defines + a product ignoring its version. */ +#define GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, product_major) \ + (((arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ + ((arch_minor) << GPU_ID2_ARCH_MINOR_SHIFT) | \ + ((arch_rev) << GPU_ID2_ARCH_REV_SHIFT) | \ + ((product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) + +/* Helper macro to create a partial GPU_ID (new format) that specifies the + revision (major, minor, status) of a product */ +#define GPU_ID2_VERSION_MAKE(version_major, version_minor, version_status) \ + (((version_major) << GPU_ID2_VERSION_MAJOR_SHIFT) | \ + ((version_minor) << GPU_ID2_VERSION_MINOR_SHIFT) | \ + ((version_status) << GPU_ID2_VERSION_STATUS_SHIFT)) + +/* Helper macro to create a complete GPU_ID (new format) */ +#define GPU_ID2_MAKE(arch_major, arch_minor, arch_rev, product_major, \ + version_major, version_minor, version_status) \ + (GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, \ + product_major) | \ + GPU_ID2_VERSION_MAKE(version_major, version_minor, \ + version_status)) + +/* Helper macro to create a partial GPU_ID (new format) that identifies + a particular GPU model by its arch_major and product_major. */ +#define GPU_ID2_MODEL_MAKE(arch_major, product_major) \ + (((arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ + ((product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) + +/* Strip off the non-relevant bits from a product_id value and make it suitable + for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU + model. */ +#define GPU_ID2_MODEL_MATCH_VALUE(product_id) \ + (((product_id) << GPU_ID2_PRODUCT_MAJOR_SHIFT) & \ + GPU_ID2_PRODUCT_MODEL) + +#define GPU_ID2_PRODUCT_TMIX GPU_ID2_MODEL_MAKE(6u, 0) +#define GPU_ID2_PRODUCT_THEX GPU_ID2_MODEL_MAKE(6u, 1) +#define GPU_ID2_PRODUCT_TSIX GPU_ID2_MODEL_MAKE(7u, 0) +#define GPU_ID2_PRODUCT_TDVX GPU_ID2_MODEL_MAKE(7u, 3) + +/* Values for GPU_ID_VERSION_STATUS field for PRODUCT_ID GPU_ID_PI_T60X */ +#define GPU_ID_S_15DEV0 0x1 +#define GPU_ID_S_EAC 0x2 + +/* Helper macro to create a GPU_ID assuming valid values for id, major, + minor, status */ +#define GPU_ID_MAKE(id, major, minor, status) \ + (((id) << GPU_ID_VERSION_PRODUCT_ID_SHIFT) | \ + ((major) << GPU_ID_VERSION_MAJOR_SHIFT) | \ + ((minor) << GPU_ID_VERSION_MINOR_SHIFT) | \ + ((status) << GPU_ID_VERSION_STATUS_SHIFT)) + +#endif /* _KBASE_GPU_ID_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.c new file mode 100755 index 000000000000..6df0a1cb1264 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.c @@ -0,0 +1,97 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include + +#ifdef CONFIG_DEBUG_FS +/** Show callback for the @c gpu_memory debugfs file. + * + * This function is called to get the contents of the @c gpu_memory debugfs + * file. This is a report of current gpu memory usage. + * + * @param sfile The debugfs entry + * @param data Data associated with the entry + * + * @return 0 if successfully prints data in debugfs entry file + * -1 if it encountered an error + */ + +static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data) +{ + struct list_head *entry; + const struct list_head *kbdev_list; + + kbdev_list = kbase_dev_list_get(); + list_for_each(entry, kbdev_list) { + struct kbase_device *kbdev = NULL; + struct kbasep_kctx_list_element *element; + + kbdev = list_entry(entry, struct kbase_device, entry); + /* output the total memory usage and cap for this device */ + seq_printf(sfile, "%-16s %10u\n", + kbdev->devname, + atomic_read(&(kbdev->memdev.used_pages))); + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry(element, &kbdev->kctx_list, link) { + /* output the memory usage and cap for each kctx + * opened on this device */ + seq_printf(sfile, " %s-0x%p %10u\n", + "kctx", + element->kctx, + atomic_read(&(element->kctx->used_pages))); + } + mutex_unlock(&kbdev->kctx_list_lock); + } + kbase_dev_list_put(kbdev_list); + return 0; +} + +/* + * File operations related to debugfs entry for gpu_memory + */ +static int kbasep_gpu_memory_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_gpu_memory_seq_show , NULL); +} + +static const struct file_operations kbasep_gpu_memory_debugfs_fops = { + .open = kbasep_gpu_memory_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* + * Initialize debugfs entry for gpu_memory + */ +void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("gpu_memory", S_IRUGO, + kbdev->mali_debugfs_directory, NULL, + &kbasep_gpu_memory_debugfs_fops); + return; +} + +#else +/* + * Stub functions for when debugfs is disabled + */ +void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) +{ + return; +} +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.h new file mode 100755 index 000000000000..7045693eb910 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpu_memory_debugfs.h @@ -0,0 +1,37 @@ +/* + * + * (C) COPYRIGHT 2012-2014, 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_gpu_memory_debugfs.h + * Header file for gpu_memory entry in debugfs + * + */ + +#ifndef _KBASE_GPU_MEMORY_DEBUGFS_H +#define _KBASE_GPU_MEMORY_DEBUGFS_H + +#include +#include + +/** + * @brief Initialize gpu_memory debugfs entry + */ +void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev); + +#endif /*_KBASE_GPU_MEMORY_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.c new file mode 100755 index 000000000000..4130810f1038 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.c @@ -0,0 +1,514 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel property query APIs + */ + +#include +#include +#include +#include +#include +#include "mali_kbase_ioctl.h" +#include + +/** + * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield. + * @value: The value from which to extract bits. + * @offset: The first bit to extract (0 being the LSB). + * @size: The number of bits to extract. + * + * Context: @offset + @size <= 32. + * + * Return: Bits [@offset, @offset + @size) from @value. + */ +/* from mali_cdsb.h */ +#define KBASE_UBFX32(value, offset, size) \ + (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1)) + +int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props) +{ + kbase_gpu_clk_speed_func get_gpu_speed_mhz; + u32 gpu_speed_mhz; + int rc = 1; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != kbase_props); + + /* Current GPU speed is requested from the system integrator via the GPU_SPEED_FUNC function. + * If that function fails, or the function is not provided by the system integrator, we report the maximum + * GPU speed as specified by GPU_FREQ_KHZ_MAX. + */ + get_gpu_speed_mhz = (kbase_gpu_clk_speed_func) GPU_SPEED_FUNC; + if (get_gpu_speed_mhz != NULL) { + rc = get_gpu_speed_mhz(&gpu_speed_mhz); +#ifdef CONFIG_MALI_BIFROST_DEBUG + /* Issue a warning message when the reported GPU speed falls outside the min/max range */ + if (rc == 0) { + u32 gpu_speed_khz = gpu_speed_mhz * 1000; + + if (gpu_speed_khz < kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min || + gpu_speed_khz > kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max) + dev_warn(kctx->kbdev->dev, "GPU Speed is outside of min/max range (got %lu Khz, min %lu Khz, max %lu Khz)\n", + (unsigned long)gpu_speed_khz, + (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min, + (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max); + } +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + } + if (kctx->kbdev->clock) { + gpu_speed_mhz = clk_get_rate(kctx->kbdev->clock) / 1000000; + rc = 0; + } + if (rc != 0) + gpu_speed_mhz = kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max / 1000; + + kctx->kbdev->gpu_props.props.core_props.gpu_speed_mhz = gpu_speed_mhz; + + memcpy(&kbase_props->props, &kctx->kbdev->gpu_props.props, sizeof(kbase_props->props)); + + /* Before API 8.2 they expect L3 cache info here, which was always 0 */ + if (kctx->api_version < KBASE_API_VERSION(8, 2)) + kbase_props->props.raw_props.suspend_size = 0; + + return 0; +} + +static void kbase_gpuprops_construct_coherent_groups(base_gpu_props * const props) +{ + struct mali_base_gpu_coherent_group *current_group; + u64 group_present; + u64 group_mask; + u64 first_set, first_set_prev; + u32 num_groups = 0; + + KBASE_DEBUG_ASSERT(NULL != props); + + props->coherency_info.coherency = props->raw_props.mem_features; + props->coherency_info.num_core_groups = hweight64(props->raw_props.l2_present); + + if (props->coherency_info.coherency & GROUPS_L2_COHERENT) { + /* Group is l2 coherent */ + group_present = props->raw_props.l2_present; + } else { + /* Group is l1 coherent */ + group_present = props->raw_props.shader_present; + } + + /* + * The coherent group mask can be computed from the l2 present + * register. + * + * For the coherent group n: + * group_mask[n] = (first_set[n] - 1) & ~(first_set[n-1] - 1) + * where first_set is group_present with only its nth set-bit kept + * (i.e. the position from where a new group starts). + * + * For instance if the groups are l2 coherent and l2_present=0x0..01111: + * The first mask is: + * group_mask[1] = (first_set[1] - 1) & ~(first_set[0] - 1) + * = (0x0..010 - 1) & ~(0x0..01 - 1) + * = 0x0..00f + * The second mask is: + * group_mask[2] = (first_set[2] - 1) & ~(first_set[1] - 1) + * = (0x0..100 - 1) & ~(0x0..010 - 1) + * = 0x0..0f0 + * And so on until all the bits from group_present have been cleared + * (i.e. there is no group left). + */ + + current_group = props->coherency_info.group; + first_set = group_present & ~(group_present - 1); + + while (group_present != 0 && num_groups < BASE_MAX_COHERENT_GROUPS) { + group_present -= first_set; /* Clear the current group bit */ + first_set_prev = first_set; + + first_set = group_present & ~(group_present - 1); + group_mask = (first_set - 1) & ~(first_set_prev - 1); + + /* Populate the coherent_group structure for each group */ + current_group->core_mask = group_mask & props->raw_props.shader_present; + current_group->num_cores = hweight64(current_group->core_mask); + + num_groups++; + current_group++; + } + + if (group_present != 0) + pr_warn("Too many coherent groups (keeping only %d groups).\n", BASE_MAX_COHERENT_GROUPS); + + props->coherency_info.num_groups = num_groups; +} + +/** + * kbase_gpuprops_get_props - Get the GPU configuration + * @gpu_props: The &base_gpu_props structure + * @kbdev: The &struct kbase_device structure for the device + * + * Fill the &base_gpu_props structure with values from the GPU configuration + * registers. Only the raw properties are filled in this function + */ +static void kbase_gpuprops_get_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev) +{ + struct kbase_gpuprops_regdump regdump; + int i; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(NULL != gpu_props); + + /* Dump relevant registers */ + kbase_backend_gpuprops_get(kbdev, ®dump); + + gpu_props->raw_props.gpu_id = regdump.gpu_id; + gpu_props->raw_props.tiler_features = regdump.tiler_features; + gpu_props->raw_props.mem_features = regdump.mem_features; + gpu_props->raw_props.mmu_features = regdump.mmu_features; + gpu_props->raw_props.l2_features = regdump.l2_features; + gpu_props->raw_props.suspend_size = regdump.suspend_size; + + gpu_props->raw_props.as_present = regdump.as_present; + gpu_props->raw_props.js_present = regdump.js_present; + gpu_props->raw_props.shader_present = + ((u64) regdump.shader_present_hi << 32) + + regdump.shader_present_lo; + gpu_props->raw_props.tiler_present = + ((u64) regdump.tiler_present_hi << 32) + + regdump.tiler_present_lo; + gpu_props->raw_props.l2_present = + ((u64) regdump.l2_present_hi << 32) + + regdump.l2_present_lo; +#ifdef CONFIG_MALI_CORESTACK + gpu_props->raw_props.stack_present = + ((u64) regdump.stack_present_hi << 32) + + regdump.stack_present_lo; +#else /* CONFIG_MALI_CORESTACK */ + gpu_props->raw_props.stack_present = 0; +#endif /* CONFIG_MALI_CORESTACK */ + + for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) + gpu_props->raw_props.js_features[i] = regdump.js_features[i]; + + for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) + gpu_props->raw_props.texture_features[i] = regdump.texture_features[i]; + + gpu_props->raw_props.thread_max_barrier_size = regdump.thread_max_barrier_size; + gpu_props->raw_props.thread_max_threads = regdump.thread_max_threads; + gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size; + gpu_props->raw_props.thread_features = regdump.thread_features; +} + +void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props) +{ + gpu_props->core_props.version_status = + KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4); + gpu_props->core_props.minor_revision = + KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8); + gpu_props->core_props.major_revision = + KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4); + gpu_props->core_props.product_id = + KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16); +} + +/** + * kbase_gpuprops_calculate_props - Calculate the derived properties + * @gpu_props: The &base_gpu_props structure + * @kbdev: The &struct kbase_device structure for the device + * + * Fill the &base_gpu_props structure with values derived from the GPU + * configuration registers + */ +static void kbase_gpuprops_calculate_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev) +{ + int i; + + /* Populate the base_gpu_props structure */ + kbase_gpuprops_update_core_props_gpu_id(gpu_props); + gpu_props->core_props.log2_program_counter_size = KBASE_GPU_PC_SIZE_LOG2; + gpu_props->core_props.gpu_available_memory_size = totalram_pages << PAGE_SHIFT; + + for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) + gpu_props->core_props.texture_features[i] = gpu_props->raw_props.texture_features[i]; + + gpu_props->l2_props.log2_line_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 0U, 8); + gpu_props->l2_props.log2_cache_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); + + /* Field with number of l2 slices is added to MEM_FEATURES register + * since t76x. Below code assumes that for older GPU reserved bits will + * be read as zero. */ + gpu_props->l2_props.num_l2_slices = + KBASE_UBFX32(gpu_props->raw_props.mem_features, 8U, 4) + 1; + + gpu_props->tiler_props.bin_size_bytes = 1 << KBASE_UBFX32(gpu_props->raw_props.tiler_features, 0U, 6); + gpu_props->tiler_props.max_active_levels = KBASE_UBFX32(gpu_props->raw_props.tiler_features, 8U, 4); + + if (gpu_props->raw_props.thread_max_threads == 0) + gpu_props->thread_props.max_threads = THREAD_MT_DEFAULT; + else + gpu_props->thread_props.max_threads = gpu_props->raw_props.thread_max_threads; + + if (gpu_props->raw_props.thread_max_workgroup_size == 0) + gpu_props->thread_props.max_workgroup_size = THREAD_MWS_DEFAULT; + else + gpu_props->thread_props.max_workgroup_size = gpu_props->raw_props.thread_max_workgroup_size; + + if (gpu_props->raw_props.thread_max_barrier_size == 0) + gpu_props->thread_props.max_barrier_size = THREAD_MBS_DEFAULT; + else + gpu_props->thread_props.max_barrier_size = gpu_props->raw_props.thread_max_barrier_size; + + gpu_props->thread_props.max_registers = KBASE_UBFX32(gpu_props->raw_props.thread_features, 0U, 16); + gpu_props->thread_props.max_task_queue = KBASE_UBFX32(gpu_props->raw_props.thread_features, 16U, 8); + gpu_props->thread_props.max_thread_group_split = KBASE_UBFX32(gpu_props->raw_props.thread_features, 24U, 6); + gpu_props->thread_props.impl_tech = KBASE_UBFX32(gpu_props->raw_props.thread_features, 30U, 2); + + /* If values are not specified, then use defaults */ + if (gpu_props->thread_props.max_registers == 0) { + gpu_props->thread_props.max_registers = THREAD_MR_DEFAULT; + gpu_props->thread_props.max_task_queue = THREAD_MTQ_DEFAULT; + gpu_props->thread_props.max_thread_group_split = THREAD_MTGS_DEFAULT; + } + /* Initialize the coherent_group structure for each group */ + kbase_gpuprops_construct_coherent_groups(gpu_props); +} + +void kbase_gpuprops_set(struct kbase_device *kbdev) +{ + struct kbase_gpu_props *gpu_props; + struct gpu_raw_gpu_props *raw; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + gpu_props = &kbdev->gpu_props; + raw = &gpu_props->props.raw_props; + + /* Initialize the base_gpu_props structure from the hardware */ + kbase_gpuprops_get_props(&gpu_props->props, kbdev); + + /* Populate the derived properties */ + kbase_gpuprops_calculate_props(&gpu_props->props, kbdev); + + /* Populate kbase-only fields */ + gpu_props->l2_props.associativity = KBASE_UBFX32(raw->l2_features, 8U, 8); + gpu_props->l2_props.external_bus_width = KBASE_UBFX32(raw->l2_features, 24U, 8); + + gpu_props->mem.core_group = KBASE_UBFX32(raw->mem_features, 0U, 1); + + gpu_props->mmu.va_bits = KBASE_UBFX32(raw->mmu_features, 0U, 8); + gpu_props->mmu.pa_bits = KBASE_UBFX32(raw->mmu_features, 8U, 8); + + gpu_props->num_cores = hweight64(raw->shader_present); + gpu_props->num_core_groups = hweight64(raw->l2_present); + gpu_props->num_address_spaces = hweight32(raw->as_present); + gpu_props->num_job_slots = hweight32(raw->js_present); +} + +void kbase_gpuprops_set_features(struct kbase_device *kbdev) +{ + base_gpu_props *gpu_props; + struct kbase_gpuprops_regdump regdump; + + gpu_props = &kbdev->gpu_props.props; + + /* Dump relevant registers */ + kbase_backend_gpuprops_get_features(kbdev, ®dump); + + /* + * Copy the raw value from the register, later this will get turned + * into the selected coherency mode. + * Additionally, add non-coherent mode, as this is always supported. + */ + gpu_props->raw_props.coherency_mode = regdump.coherency_features | + COHERENCY_FEATURE_BIT(COHERENCY_NONE); +} + +static struct { + u32 type; + size_t offset; + int size; +} gpu_property_mapping[] = { +#define PROP(name, member) \ + {KBASE_GPUPROP_ ## name, offsetof(struct mali_base_gpu_props, member), \ + sizeof(((struct mali_base_gpu_props *)0)->member)} + PROP(PRODUCT_ID, core_props.product_id), + PROP(VERSION_STATUS, core_props.version_status), + PROP(MINOR_REVISION, core_props.minor_revision), + PROP(MAJOR_REVISION, core_props.major_revision), + PROP(GPU_SPEED_MHZ, core_props.gpu_speed_mhz), + PROP(GPU_FREQ_KHZ_MAX, core_props.gpu_freq_khz_max), + PROP(GPU_FREQ_KHZ_MIN, core_props.gpu_freq_khz_min), + PROP(LOG2_PROGRAM_COUNTER_SIZE, core_props.log2_program_counter_size), + PROP(TEXTURE_FEATURES_0, core_props.texture_features[0]), + PROP(TEXTURE_FEATURES_1, core_props.texture_features[1]), + PROP(TEXTURE_FEATURES_2, core_props.texture_features[2]), + PROP(GPU_AVAILABLE_MEMORY_SIZE, core_props.gpu_available_memory_size), + + PROP(L2_LOG2_LINE_SIZE, l2_props.log2_line_size), + PROP(L2_LOG2_CACHE_SIZE, l2_props.log2_cache_size), + PROP(L2_NUM_L2_SLICES, l2_props.num_l2_slices), + + PROP(TILER_BIN_SIZE_BYTES, tiler_props.bin_size_bytes), + PROP(TILER_MAX_ACTIVE_LEVELS, tiler_props.max_active_levels), + + PROP(MAX_THREADS, thread_props.max_threads), + PROP(MAX_WORKGROUP_SIZE, thread_props.max_workgroup_size), + PROP(MAX_BARRIER_SIZE, thread_props.max_barrier_size), + PROP(MAX_REGISTERS, thread_props.max_registers), + PROP(MAX_TASK_QUEUE, thread_props.max_task_queue), + PROP(MAX_THREAD_GROUP_SPLIT, thread_props.max_thread_group_split), + PROP(IMPL_TECH, thread_props.impl_tech), + + PROP(RAW_SHADER_PRESENT, raw_props.shader_present), + PROP(RAW_TILER_PRESENT, raw_props.tiler_present), + PROP(RAW_L2_PRESENT, raw_props.l2_present), + PROP(RAW_STACK_PRESENT, raw_props.stack_present), + PROP(RAW_L2_FEATURES, raw_props.l2_features), + PROP(RAW_SUSPEND_SIZE, raw_props.suspend_size), + PROP(RAW_MEM_FEATURES, raw_props.mem_features), + PROP(RAW_MMU_FEATURES, raw_props.mmu_features), + PROP(RAW_AS_PRESENT, raw_props.as_present), + PROP(RAW_JS_PRESENT, raw_props.js_present), + PROP(RAW_JS_FEATURES_0, raw_props.js_features[0]), + PROP(RAW_JS_FEATURES_1, raw_props.js_features[1]), + PROP(RAW_JS_FEATURES_2, raw_props.js_features[2]), + PROP(RAW_JS_FEATURES_3, raw_props.js_features[3]), + PROP(RAW_JS_FEATURES_4, raw_props.js_features[4]), + PROP(RAW_JS_FEATURES_5, raw_props.js_features[5]), + PROP(RAW_JS_FEATURES_6, raw_props.js_features[6]), + PROP(RAW_JS_FEATURES_7, raw_props.js_features[7]), + PROP(RAW_JS_FEATURES_8, raw_props.js_features[8]), + PROP(RAW_JS_FEATURES_9, raw_props.js_features[9]), + PROP(RAW_JS_FEATURES_10, raw_props.js_features[10]), + PROP(RAW_JS_FEATURES_11, raw_props.js_features[11]), + PROP(RAW_JS_FEATURES_12, raw_props.js_features[12]), + PROP(RAW_JS_FEATURES_13, raw_props.js_features[13]), + PROP(RAW_JS_FEATURES_14, raw_props.js_features[14]), + PROP(RAW_JS_FEATURES_15, raw_props.js_features[15]), + PROP(RAW_TILER_FEATURES, raw_props.tiler_features), + PROP(RAW_TEXTURE_FEATURES_0, raw_props.texture_features[0]), + PROP(RAW_TEXTURE_FEATURES_1, raw_props.texture_features[1]), + PROP(RAW_TEXTURE_FEATURES_2, raw_props.texture_features[2]), + PROP(RAW_GPU_ID, raw_props.gpu_id), + PROP(RAW_THREAD_MAX_THREADS, raw_props.thread_max_threads), + PROP(RAW_THREAD_MAX_WORKGROUP_SIZE, + raw_props.thread_max_workgroup_size), + PROP(RAW_THREAD_MAX_BARRIER_SIZE, raw_props.thread_max_barrier_size), + PROP(RAW_THREAD_FEATURES, raw_props.thread_features), + PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode), + + PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups), + PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups), + PROP(COHERENCY_COHERENCY, coherency_info.coherency), + PROP(COHERENCY_GROUP_0, coherency_info.group[0].core_mask), + PROP(COHERENCY_GROUP_1, coherency_info.group[1].core_mask), + PROP(COHERENCY_GROUP_2, coherency_info.group[2].core_mask), + PROP(COHERENCY_GROUP_3, coherency_info.group[3].core_mask), + PROP(COHERENCY_GROUP_4, coherency_info.group[4].core_mask), + PROP(COHERENCY_GROUP_5, coherency_info.group[5].core_mask), + PROP(COHERENCY_GROUP_6, coherency_info.group[6].core_mask), + PROP(COHERENCY_GROUP_7, coherency_info.group[7].core_mask), + PROP(COHERENCY_GROUP_8, coherency_info.group[8].core_mask), + PROP(COHERENCY_GROUP_9, coherency_info.group[9].core_mask), + PROP(COHERENCY_GROUP_10, coherency_info.group[10].core_mask), + PROP(COHERENCY_GROUP_11, coherency_info.group[11].core_mask), + PROP(COHERENCY_GROUP_12, coherency_info.group[12].core_mask), + PROP(COHERENCY_GROUP_13, coherency_info.group[13].core_mask), + PROP(COHERENCY_GROUP_14, coherency_info.group[14].core_mask), + PROP(COHERENCY_GROUP_15, coherency_info.group[15].core_mask), + +#undef PROP +}; + +int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev) +{ + struct kbase_gpu_props *kprops = &kbdev->gpu_props; + struct mali_base_gpu_props *props = &kprops->props; + u32 count = ARRAY_SIZE(gpu_property_mapping); + u32 i; + u32 size = 0; + u8 *p; + + for (i = 0; i < count; i++) { + /* 4 bytes for the ID, and the size of the property */ + size += 4 + gpu_property_mapping[i].size; + } + + kprops->prop_buffer_size = size; + kprops->prop_buffer = kmalloc(size, GFP_KERNEL); + + if (!kprops->prop_buffer) { + kprops->prop_buffer_size = 0; + return -ENOMEM; + } + + p = kprops->prop_buffer; + +#define WRITE_U8(v) (*p++ = (v) & 0xFF) +#define WRITE_U16(v) do { WRITE_U8(v); WRITE_U8((v) >> 8); } while (0) +#define WRITE_U32(v) do { WRITE_U16(v); WRITE_U16((v) >> 16); } while (0) +#define WRITE_U64(v) do { WRITE_U32(v); WRITE_U32((v) >> 32); } while (0) + + for (i = 0; i < count; i++) { + u32 type = gpu_property_mapping[i].type; + u8 type_size; + void *field = ((u8 *)props) + gpu_property_mapping[i].offset; + + switch (gpu_property_mapping[i].size) { + case 1: + type_size = KBASE_GPUPROP_VALUE_SIZE_U8; + break; + case 2: + type_size = KBASE_GPUPROP_VALUE_SIZE_U16; + break; + case 4: + type_size = KBASE_GPUPROP_VALUE_SIZE_U32; + break; + case 8: + type_size = KBASE_GPUPROP_VALUE_SIZE_U64; + break; + default: + dev_err(kbdev->dev, + "Invalid gpu_property_mapping type=%d size=%d", + type, gpu_property_mapping[i].size); + return -EINVAL; + } + + WRITE_U32((type<<2) | type_size); + + switch (type_size) { + case KBASE_GPUPROP_VALUE_SIZE_U8: + WRITE_U8(*((u8 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U16: + WRITE_U16(*((u16 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U32: + WRITE_U32(*((u32 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U64: + WRITE_U64(*((u64 *)field)); + break; + default: /* Cannot be reached */ + WARN_ON(1); + return -EINVAL; + } + } + + return 0; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.h new file mode 100755 index 000000000000..57b3eaf9cd53 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops.h @@ -0,0 +1,84 @@ +/* + * + * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_gpuprops.h + * Base kernel property query APIs + */ + +#ifndef _KBASE_GPUPROPS_H_ +#define _KBASE_GPUPROPS_H_ + +#include "mali_kbase_gpuprops_types.h" + +/* Forward definition - see mali_kbase.h */ +struct kbase_device; + +/** + * @brief Set up Kbase GPU properties. + * + * Set up Kbase GPU properties with information from the GPU registers + * + * @param kbdev The struct kbase_device structure for the device + */ +void kbase_gpuprops_set(struct kbase_device *kbdev); + +/** + * kbase_gpuprops_set_features - Set up Kbase GPU properties + * @kbdev: Device pointer + * + * This function sets up GPU properties that are dependent on the hardware + * features bitmask. This function must be preceeded by a call to + * kbase_hw_set_features_mask(). + */ +void kbase_gpuprops_set_features(struct kbase_device *kbdev); + +/** + * @brief Provide GPU properties to userside through UKU call. + * + * Fill the struct kbase_uk_gpuprops with values from GPU configuration registers. + * + * @param kctx The struct kbase_context structure + * @param kbase_props A copy of the struct kbase_uk_gpuprops structure from userspace + * + * @return 0 on success. Any other value indicates failure. + */ +int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props); + +/** + * kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer + * @kbdev: The kbase device + * + * Fills kbdev->gpu_props->prop_buffer with the GPU properties for user + * space to read. + */ +int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev); + +/** + * kbase_gpuprops_update_core_props_gpu_id - break down gpu id value + * @gpu_props: the &base_gpu_props structure + * + * Break down gpu_id value stored in base_gpu_props::raw_props.gpu_id into + * separate fields (version_status, minor_revision, major_revision, product_id) + * stored in base_gpu_props::core_props. + */ +void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props); + + +#endif /* _KBASE_GPUPROPS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops_types.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops_types.h new file mode 100755 index 000000000000..10794fc27318 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_gpuprops_types.h @@ -0,0 +1,92 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_gpuprops_types.h + * Base kernel property query APIs + */ + +#ifndef _KBASE_GPUPROPS_TYPES_H_ +#define _KBASE_GPUPROPS_TYPES_H_ + +#include "mali_base_kernel.h" + +#define KBASE_GPU_SPEED_MHZ 123 +#define KBASE_GPU_PC_SIZE_LOG2 24U + +struct kbase_gpuprops_regdump { + u32 gpu_id; + u32 l2_features; + u32 suspend_size; /* API 8.2+ */ + u32 tiler_features; + u32 mem_features; + u32 mmu_features; + u32 as_present; + u32 js_present; + u32 thread_max_threads; + u32 thread_max_workgroup_size; + u32 thread_max_barrier_size; + u32 thread_features; + u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; + u32 js_features[GPU_MAX_JOB_SLOTS]; + u32 shader_present_lo; + u32 shader_present_hi; + u32 tiler_present_lo; + u32 tiler_present_hi; + u32 l2_present_lo; + u32 l2_present_hi; + u32 stack_present_lo; + u32 stack_present_hi; + u32 coherency_features; +}; + +struct kbase_gpu_cache_props { + u8 associativity; + u8 external_bus_width; +}; + +struct kbase_gpu_mem_props { + u8 core_group; +}; + +struct kbase_gpu_mmu_props { + u8 va_bits; + u8 pa_bits; +}; + +struct kbase_gpu_props { + /* kernel-only properties */ + u8 num_cores; + u8 num_core_groups; + u8 num_address_spaces; + u8 num_job_slots; + + struct kbase_gpu_cache_props l2_props; + + struct kbase_gpu_mem_props mem; + struct kbase_gpu_mmu_props mmu; + + /* Properties shared with userspace */ + base_gpu_props props; + + u32 prop_buffer_size; + void *prop_buffer; +}; + +#endif /* _KBASE_GPUPROPS_TYPES_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.c new file mode 100755 index 000000000000..eb8368ccee5e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.c @@ -0,0 +1,446 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Run-time work-arounds helpers + */ + +#include +#include +#include +#include "mali_kbase.h" +#include "mali_kbase_hw.h" + +void kbase_hw_set_features_mask(struct kbase_device *kbdev) +{ + const enum base_hw_feature *features; + u32 gpu_id; + u32 product_id; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; + product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { + case GPU_ID2_PRODUCT_TMIX: + features = base_hw_features_tMIx; + break; + case GPU_ID2_PRODUCT_THEX: + features = base_hw_features_tHEx; + break; + case GPU_ID2_PRODUCT_TSIX: + features = base_hw_features_tSIx; + break; + case GPU_ID2_PRODUCT_TDVX: + features = base_hw_features_tDVx; + break; + default: + features = base_hw_features_generic; + break; + } + } else { + switch (product_id) { + case GPU_ID_PI_TFRX: + /* FALLTHROUGH */ + case GPU_ID_PI_T86X: + features = base_hw_features_tFxx; + break; + case GPU_ID_PI_T83X: + features = base_hw_features_t83x; + break; + case GPU_ID_PI_T82X: + features = base_hw_features_t82x; + break; + case GPU_ID_PI_T76X: + features = base_hw_features_t76x; + break; + case GPU_ID_PI_T72X: + features = base_hw_features_t72x; + break; + case GPU_ID_PI_T62X: + features = base_hw_features_t62x; + break; + case GPU_ID_PI_T60X: + features = base_hw_features_t60x; + break; + default: + features = base_hw_features_generic; + break; + } + } + + for (; *features != BASE_HW_FEATURE_END; features++) + set_bit(*features, &kbdev->hw_features_mask[0]); +} + +/** + * kbase_hw_get_issues_for_new_id - Get the hardware issues for a new GPU ID + * @kbdev: Device pointer + * + * Return: pointer to an array of hardware issues, terminated by + * BASE_HW_ISSUE_END. + * + * This function can only be used on new-format GPU IDs, i.e. those for which + * GPU_ID_IS_NEW_FORMAT evaluates as true. The GPU ID is read from the @kbdev. + * + * In debugging versions of the driver, unknown versions of a known GPU will + * be treated as the most recent known version not later than the actual + * version. In such circumstances, the GPU ID in @kbdev will also be replaced + * with the most recent known version. + * + * Note: The GPU configuration must have been read by kbase_gpuprops_get_props() + * before calling this function. + */ +static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( + struct kbase_device *kbdev) +{ + const enum base_hw_issue *issues = NULL; + + struct base_hw_product { + u32 product_model; + struct { + u32 version; + const enum base_hw_issue *issues; + } map[7]; + }; + + static const struct base_hw_product base_hw_products[] = { + {GPU_ID2_PRODUCT_TMIX, + {{GPU_ID2_VERSION_MAKE(0, 0, 1), + base_hw_issues_tMIx_r0p0_05dev0}, + {GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tMIx_r0p1}, + {U32_MAX /* sentinel value */, NULL} } }, + + {GPU_ID2_PRODUCT_THEX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1}, + {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tHEx_r0p1}, + {GPU_ID2_VERSION_MAKE(0, 2, 0), base_hw_issues_tHEx_r0p2}, + {GPU_ID2_VERSION_MAKE(0, 3, 0), base_hw_issues_tHEx_r0p3}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TSIX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1}, + {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0}, + {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_tSIx_r1p1}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TDVX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tDVx_r0p0}, + {U32_MAX, NULL} } }, + + + + + }; + + u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + const u32 product_model = gpu_id & GPU_ID2_PRODUCT_MODEL; + const struct base_hw_product *product = NULL; + size_t p; + + /* Stop when we reach the end of the products array. */ + for (p = 0; p < ARRAY_SIZE(base_hw_products); ++p) { + if (product_model == base_hw_products[p].product_model) { + product = &base_hw_products[p]; + break; + } + } + + if (product != NULL) { + /* Found a matching product. */ + const u32 version = gpu_id & GPU_ID2_VERSION; +#if !MALI_CUSTOMER_RELEASE + u32 fallback_version = 0; + const enum base_hw_issue *fallback_issues = NULL; +#endif + size_t v; + + /* Stop when we reach the end of the map. */ + for (v = 0; product->map[v].version != U32_MAX; ++v) { + + if (version == product->map[v].version) { + /* Exact match so stop. */ + issues = product->map[v].issues; + break; + } + +#if !MALI_CUSTOMER_RELEASE + /* Check whether this is a candidate for most recent + known version not later than the actual + version. */ + if ((version > product->map[v].version) && + (product->map[v].version >= fallback_version)) { + fallback_version = product->map[v].version; + fallback_issues = product->map[v].issues; + } +#endif + } + +#if !MALI_CUSTOMER_RELEASE + if ((issues == NULL) && (fallback_issues != NULL)) { + /* Fall back to the issue set of the most recent known + version not later than the actual version. */ + issues = fallback_issues; + + dev_info(kbdev->dev, + "r%dp%d status %d is unknown; treating as r%dp%d status %d", + (gpu_id & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT, + (fallback_version & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (fallback_version & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (fallback_version & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT); + + gpu_id &= ~GPU_ID2_VERSION; + gpu_id |= fallback_version; + kbdev->gpu_props.props.raw_props.gpu_id = gpu_id; + + kbase_gpuprops_update_core_props_gpu_id( + &kbdev->gpu_props.props); + } +#endif + } + return issues; +} + +int kbase_hw_set_issues_mask(struct kbase_device *kbdev) +{ + const enum base_hw_issue *issues; + u32 gpu_id; + u32 product_id; + u32 impl_tech; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; + product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; + impl_tech = kbdev->gpu_props.props.thread_props.impl_tech; + + if (impl_tech != IMPLEMENTATION_MODEL) { + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + issues = kbase_hw_get_issues_for_new_id(kbdev); + if (issues == NULL) { + dev_err(kbdev->dev, + "Unknown GPU ID %x", gpu_id); + return -EINVAL; + } + +#if !MALI_CUSTOMER_RELEASE + /* The GPU ID might have been replaced with the last + known version of the same GPU. */ + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; +#endif + + } else { + switch (gpu_id) { + case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_15DEV0): + issues = base_hw_issues_t60x_r0p0_15dev0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_EAC): + issues = base_hw_issues_t60x_r0p0_eac; + break; + case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 1, 0): + issues = base_hw_issues_t60x_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T62X, 0, 1, 0): + issues = base_hw_issues_t62x_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 1): + issues = base_hw_issues_t62x_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 1, 0): + issues = base_hw_issues_t62x_r1p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 0, 1): + issues = base_hw_issues_t76x_r0p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 1, 1): + issues = base_hw_issues_t76x_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 1, 9): + issues = base_hw_issues_t76x_r0p1_50rel0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 2, 1): + issues = base_hw_issues_t76x_r0p2; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 3, 1): + issues = base_hw_issues_t76x_r0p3; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 1, 0, 0): + issues = base_hw_issues_t76x_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 1): + case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 2): + issues = base_hw_issues_t72x_r0p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 0, 0): + issues = base_hw_issues_t72x_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 1, 0): + issues = base_hw_issues_t72x_r1p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 1, 2): + issues = base_hw_issues_tFRx_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 2, 0): + issues = base_hw_issues_tFRx_r0p2; + break; + case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 8): + issues = base_hw_issues_tFRx_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_TFRX, 2, 0, 0): + issues = base_hw_issues_tFRx_r2p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T86X, 0, 2, 0): + issues = base_hw_issues_t86x_r0p2; + break; + case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 8): + issues = base_hw_issues_t86x_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T86X, 2, 0, 0): + issues = base_hw_issues_t86x_r2p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T83X, 0, 1, 0): + issues = base_hw_issues_t83x_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 8): + issues = base_hw_issues_t83x_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 0, 0): + issues = base_hw_issues_t82x_r0p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 1, 0): + issues = base_hw_issues_t82x_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 8): + issues = base_hw_issues_t82x_r1p0; + break; + default: + dev_err(kbdev->dev, + "Unknown GPU ID %x", gpu_id); + return -EINVAL; + } + } + } else { + /* Software model */ + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { + case GPU_ID2_PRODUCT_TMIX: + issues = base_hw_issues_model_tMIx; + break; + case GPU_ID2_PRODUCT_THEX: + issues = base_hw_issues_model_tHEx; + break; + case GPU_ID2_PRODUCT_TSIX: + issues = base_hw_issues_model_tSIx; + break; + case GPU_ID2_PRODUCT_TDVX: + issues = base_hw_issues_model_tDVx; + break; + default: + dev_err(kbdev->dev, + "Unknown GPU ID %x", gpu_id); + return -EINVAL; + } + } else { + switch (product_id) { + case GPU_ID_PI_T60X: + issues = base_hw_issues_model_t60x; + break; + case GPU_ID_PI_T62X: + issues = base_hw_issues_model_t62x; + break; + case GPU_ID_PI_T72X: + issues = base_hw_issues_model_t72x; + break; + case GPU_ID_PI_T76X: + issues = base_hw_issues_model_t76x; + break; + case GPU_ID_PI_TFRX: + issues = base_hw_issues_model_tFRx; + break; + case GPU_ID_PI_T86X: + issues = base_hw_issues_model_t86x; + break; + case GPU_ID_PI_T83X: + issues = base_hw_issues_model_t83x; + break; + case GPU_ID_PI_T82X: + issues = base_hw_issues_model_t82x; + break; + default: + dev_err(kbdev->dev, "Unknown GPU ID %x", + gpu_id); + return -EINVAL; + } + } + } + + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + dev_info(kbdev->dev, + "GPU identified as 0x%x arch %d.%d.%d r%dp%d status %d", + (gpu_id & GPU_ID2_PRODUCT_MAJOR) >> + GPU_ID2_PRODUCT_MAJOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_MAJOR) >> + GPU_ID2_ARCH_MAJOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_MINOR) >> + GPU_ID2_ARCH_MINOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_REV) >> + GPU_ID2_ARCH_REV_SHIFT, + (gpu_id & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT); + } else { + dev_info(kbdev->dev, + "GPU identified as 0x%04x r%dp%d status %d", + (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT, + (gpu_id & GPU_ID_VERSION_MAJOR) >> + GPU_ID_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID_VERSION_MINOR) >> + GPU_ID_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID_VERSION_STATUS) >> + GPU_ID_VERSION_STATUS_SHIFT); + } + + for (; *issues != BASE_HW_ISSUE_END; issues++) + set_bit(*issues, &kbdev->hw_issues_mask[0]); + + return 0; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.h new file mode 100755 index 000000000000..754250ce968d --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hw.h @@ -0,0 +1,65 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file + * Run-time work-arounds helpers + */ + +#ifndef _KBASE_HW_H_ +#define _KBASE_HW_H_ + +#include "mali_kbase_defs.h" + +/** + * @brief Tell whether a work-around should be enabled + */ +#define kbase_hw_has_issue(kbdev, issue)\ + test_bit(issue, &(kbdev)->hw_issues_mask[0]) + +/** + * @brief Tell whether a feature is supported + */ +#define kbase_hw_has_feature(kbdev, feature)\ + test_bit(feature, &(kbdev)->hw_features_mask[0]) + +/** + * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID + * @kbdev: Device pointer + * + * Return: 0 if the GPU ID was recognized, otherwise -EINVAL. + * + * The GPU ID is read from the @kbdev. + * + * In debugging versions of the driver, unknown versions of a known GPU with a + * new-format ID will be treated as the most recent known version not later + * than the actual version. In such circumstances, the GPU ID in @kbdev will + * also be replaced with the most recent known version. + * + * Note: The GPU configuration must have been read by + * kbase_gpuprops_get_props() before calling this function. + */ +int kbase_hw_set_issues_mask(struct kbase_device *kbdev); + +/** + * @brief Set the features mask depending on the GPU ID + */ +void kbase_hw_set_features_mask(struct kbase_device *kbdev); + +#endif /* _KBASE_HW_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_backend.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_backend.h new file mode 100755 index 000000000000..b09be99e6b4e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_backend.h @@ -0,0 +1,54 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * HW access backend common APIs + */ + +#ifndef _KBASE_HWACCESS_BACKEND_H_ +#define _KBASE_HWACCESS_BACKEND_H_ + +/** + * kbase_backend_early_init - Perform any backend-specific initialization. + * @kbdev: Device pointer + * + * Return: 0 on success, or an error code on failure. + */ +int kbase_backend_early_init(struct kbase_device *kbdev); + +/** + * kbase_backend_late_init - Perform any backend-specific initialization. + * @kbdev: Device pointer + * + * Return: 0 on success, or an error code on failure. + */ +int kbase_backend_late_init(struct kbase_device *kbdev); + +/** + * kbase_backend_early_term - Perform any backend-specific termination. + * @kbdev: Device pointer + */ +void kbase_backend_early_term(struct kbase_device *kbdev); + +/** + * kbase_backend_late_term - Perform any backend-specific termination. + * @kbdev: Device pointer + */ +void kbase_backend_late_term(struct kbase_device *kbdev); + +#endif /* _KBASE_HWACCESS_BACKEND_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_defs.h new file mode 100755 index 000000000000..0acf297192fd --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_defs.h @@ -0,0 +1,36 @@ +/* + * + * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/** + * @file mali_kbase_hwaccess_gpu_defs.h + * HW access common definitions + */ + +#ifndef _KBASE_HWACCESS_DEFS_H_ +#define _KBASE_HWACCESS_DEFS_H_ + +#include + +/* The hwaccess_lock (a spinlock) must be held when accessing this structure */ +struct kbase_hwaccess_data { + struct kbase_context *active_kctx; + + struct kbase_backend_data backend; +}; + +#endif /* _KBASE_HWACCESS_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_gpuprops.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_gpuprops.h new file mode 100755 index 000000000000..cf8a8131c22e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_gpuprops.h @@ -0,0 +1,47 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/** + * Base kernel property query backend APIs + */ + +#ifndef _KBASE_HWACCESS_GPUPROPS_H_ +#define _KBASE_HWACCESS_GPUPROPS_H_ + +/** + * kbase_backend_gpuprops_get() - Fill @regdump with GPU properties read from + * GPU + * @kbdev: Device pointer + * @regdump: Pointer to struct kbase_gpuprops_regdump structure + */ +void kbase_backend_gpuprops_get(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump); + +/** + * kbase_backend_gpuprops_get - Fill @regdump with GPU properties read from GPU + * @kbdev: Device pointer + * @regdump: Pointer to struct kbase_gpuprops_regdump structure + * + * This function reads GPU properties that are dependent on the hardware + * features bitmask + */ +void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump); + + +#endif /* _KBASE_HWACCESS_GPUPROPS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_instr.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_instr.h new file mode 100755 index 000000000000..5de2b7535bb4 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_instr.h @@ -0,0 +1,116 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * HW Access instrumentation common APIs + */ + +#ifndef _KBASE_HWACCESS_INSTR_H_ +#define _KBASE_HWACCESS_INSTR_H_ + +#include + +/** + * kbase_instr_hwcnt_enable_internal - Enable HW counters collection + * @kbdev: Kbase device + * @kctx: Kbase context + * @setup: HW counter setup parameters + * + * Context: might sleep, waiting for reset to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_uk_hwcnt_setup *setup); + +/** + * kbase_instr_hwcnt_disable_internal - Disable HW counters collection + * @kctx: Kbase context + * + * Context: might sleep, waiting for an ongoing dump to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx); + +/** + * kbase_instr_hwcnt_request_dump() - Request HW counter dump from GPU + * @kctx: Kbase context + * + * Caller must either wait for kbase_instr_hwcnt_dump_complete() to return true, + * of call kbase_instr_hwcnt_wait_for_dump(). + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx); + +/** + * kbase_instr_hwcnt_wait_for_dump() - Wait until pending HW counter dump has + * completed. + * @kctx: Kbase context + * + * Context: will sleep, waiting for dump to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx); + +/** + * kbase_instr_hwcnt_dump_complete - Tell whether the HW counters dump has + * completed + * @kctx: Kbase context + * @success: Set to true if successful + * + * Context: does not sleep. + * + * Return: true if the dump is complete + */ +bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, + bool * const success); + +/** + * kbase_instr_hwcnt_clear() - Clear HW counters + * @kctx: Kbase context + * + * Context: might sleep, waiting for reset to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_clear(struct kbase_context *kctx); + +/** + * kbase_instr_backend_init() - Initialise the instrumentation backend + * @kbdev: Kbase device + * + * This function should be called during driver initialization. + * + * Return: 0 on success + */ +int kbase_instr_backend_init(struct kbase_device *kbdev); + +/** + * kbase_instr_backend_init() - Terminate the instrumentation backend + * @kbdev: Kbase device + * + * This function should be called during driver termination. + */ +void kbase_instr_backend_term(struct kbase_device *kbdev); + +#endif /* _KBASE_HWACCESS_INSTR_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_jm.h new file mode 100755 index 000000000000..750fda2cd81d --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_jm.h @@ -0,0 +1,381 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * HW access job manager common APIs + */ + +#ifndef _KBASE_HWACCESS_JM_H_ +#define _KBASE_HWACCESS_JM_H_ + +/** + * kbase_backend_run_atom() - Run an atom on the GPU + * @kbdev: Device pointer + * @atom: Atom to run + * + * Caller must hold the HW access lock + */ +void kbase_backend_run_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_backend_slot_update - Update state based on slot ringbuffers + * + * @kbdev: Device pointer + * + * Inspect the jobs in the slot ringbuffers and update state. + * + * This will cause jobs to be submitted to hardware if they are unblocked + */ +void kbase_backend_slot_update(struct kbase_device *kbdev); + +/** + * kbase_backend_find_and_release_free_address_space() - Release a free AS + * @kbdev: Device pointer + * @kctx: Context pointer + * + * This function can evict an idle context from the runpool, freeing up the + * address space it was using. + * + * The address space is marked as in use. The caller must either assign a + * context using kbase_gpu_use_ctx(), or release it using + * kbase_ctx_sched_release() + * + * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none + * available + */ +int kbase_backend_find_and_release_free_address_space( + struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the + * provided address space. + * @kbdev: Device pointer + * @kctx: Context pointer. May be NULL + * @as_nr: Free address space to use + * + * kbase_gpu_next_job() will pull atoms from the active context. + * + * Return: true if successful, false if ASID not assigned. + */ +bool kbase_backend_use_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, + int as_nr); + +/** + * kbase_backend_use_ctx_sched() - Activate a context. + * @kbdev: Device pointer + * @kctx: Context pointer + * + * kbase_gpu_next_job() will pull atoms from the active context. + * + * The context must already be scheduled and assigned to an address space. If + * the context is not scheduled, then kbase_gpu_use_ctx() should be used + * instead. + * + * Caller must hold hwaccess_lock + * + * Return: true if context is now active, false otherwise (ie if context does + * not have an address space assigned) + */ +bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbase_backend_release_ctx_irq - Release a context from the GPU. This will + * de-assign the assigned address space. + * @kbdev: Device pointer + * @kctx: Context pointer + * + * Caller must hold kbase_device->mmu_hw_mutex and hwaccess_lock + */ +void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbase_backend_release_ctx_noirq - Release a context from the GPU. This will + * de-assign the assigned address space. + * @kbdev: Device pointer + * @kctx: Context pointer + * + * Caller must hold kbase_device->mmu_hw_mutex + * + * This function must perform any operations that could not be performed in IRQ + * context by kbase_backend_release_ctx_irq(). + */ +void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbase_backend_cacheclean - Perform a cache clean if the given atom requires + * one + * @kbdev: Device pointer + * @katom: Pointer to the failed atom + * + * On some GPUs, the GPU cache must be cleaned following a failed atom. This + * function performs a clean if it is required by @katom. + */ +void kbase_backend_cacheclean(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + + +/** + * kbase_backend_complete_wq() - Perform backend-specific actions required on + * completing an atom. + * @kbdev: Device pointer + * @katom: Pointer to the atom to complete + * + * This function should only be called from kbase_jd_done_worker() or + * js_return_worker(). + * + * Return: true if atom has completed, false if atom should be re-submitted + */ +void kbase_backend_complete_wq(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_backend_complete_wq_post_sched - Perform backend-specific actions + * required on completing an atom, after + * any scheduling has taken place. + * @kbdev: Device pointer + * @core_req: Core requirements of atom + * @affinity: Affinity of atom + * @coreref_state: Coreref state of atom + * + * This function should only be called from kbase_jd_done_worker() or + * js_return_worker(). + */ +void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, + base_jd_core_req core_req, u64 affinity, + enum kbase_atom_coreref_state coreref_state); + +/** + * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU + * and remove any others from the ringbuffers. + * @kbdev: Device pointer + * @end_timestamp: Timestamp of reset + */ +void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp); + +/** + * kbase_backend_inspect_head() - Return the atom currently at the head of slot + * @js + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Atom currently at the head of slot @js, or NULL + */ +struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, + int js); + +/** + * kbase_backend_inspect_tail - Return the atom currently at the tail of slot + * @js + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Atom currently at the head of slot @js, or NULL + */ +struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, + int js); + +/** + * kbase_backend_nr_atoms_on_slot() - Return the number of atoms currently on a + * slot. + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Number of atoms currently on slot + */ +int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js); + +/** + * kbase_backend_nr_atoms_submitted() - Return the number of atoms on a slot + * that are currently on the GPU. + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Number of atoms currently on slot @js that are currently on the GPU. + */ +int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js); + +/** + * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs + * has changed. + * @kbdev: Device pointer + * + * Perform any required backend-specific actions (eg starting/stopping + * scheduling timers). + */ +void kbase_backend_ctx_count_changed(struct kbase_device *kbdev); + +/** + * kbase_backend_timeouts_changed() - Job Scheduler timeouts have changed. + * @kbdev: Device pointer + * + * Perform any required backend-specific actions (eg updating timeouts of + * currently running atoms). + */ +void kbase_backend_timeouts_changed(struct kbase_device *kbdev); + +/** + * kbase_backend_slot_free() - Return the number of jobs that can be currently + * submitted to slot @js. + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Number of jobs that can be submitted. + */ +int kbase_backend_slot_free(struct kbase_device *kbdev, int js); + +/** + * kbase_job_check_enter_disjoint - potentially leave disjoint state + * @kbdev: kbase device + * @target_katom: atom which is finishing + * + * Work out whether to leave disjoint state when finishing an atom that was + * originated by kbase_job_check_enter_disjoint(). + */ +void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, + struct kbase_jd_atom *target_katom); + +/** + * kbase_backend_jm_kill_jobs_from_kctx - Kill all jobs that are currently + * running from a context + * @kctx: Context pointer + * + * This is used in response to a page fault to remove all jobs from the faulting + * context from the hardware. + */ +void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx); + +/** + * kbase_jm_wait_for_zero_jobs - Wait for context to have zero jobs running, and + * to be descheduled. + * @kctx: Context pointer + * + * This should be called following kbase_js_zap_context(), to ensure the context + * can be safely destroyed. + */ +void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx); + +/** + * kbase_backend_get_current_flush_id - Return the current flush ID + * + * @kbdev: Device pointer + * + * Return: the current flush ID to be recorded for each job chain + */ +u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev); + +#if KBASE_GPU_RESET_EN +/** + * kbase_prepare_to_reset_gpu - Prepare for resetting the GPU. + * @kbdev: Device pointer + * + * This function just soft-stops all the slots to ensure that as many jobs as + * possible are saved. + * + * Return: a boolean which should be interpreted as follows: + * - true - Prepared for reset, kbase_reset_gpu should be called. + * - false - Another thread is performing a reset, kbase_reset_gpu should + * not be called. + */ +bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu - Reset the GPU + * @kbdev: Device pointer + * + * This function should be called after kbase_prepare_to_reset_gpu if it returns + * true. It should never be called without a corresponding call to + * kbase_prepare_to_reset_gpu. + * + * After this function is called (or not called if kbase_prepare_to_reset_gpu + * returned false), the caller should wait for kbdev->reset_waitq to be + * signalled to know when the reset has completed. + */ +void kbase_reset_gpu(struct kbase_device *kbdev); + +/** + * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU. + * @kbdev: Device pointer + * + * This function just soft-stops all the slots to ensure that as many jobs as + * possible are saved. + * + * Return: a boolean which should be interpreted as follows: + * - true - Prepared for reset, kbase_reset_gpu should be called. + * - false - Another thread is performing a reset, kbase_reset_gpu should + * not be called. + */ +bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_locked - Reset the GPU + * @kbdev: Device pointer + * + * This function should be called after kbase_prepare_to_reset_gpu if it + * returns true. It should never be called without a corresponding call to + * kbase_prepare_to_reset_gpu. + * + * After this function is called (or not called if kbase_prepare_to_reset_gpu + * returned false), the caller should wait for kbdev->reset_waitq to be + * signalled to know when the reset has completed. + */ +void kbase_reset_gpu_locked(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_silent - Reset the GPU silently + * @kbdev: Device pointer + * + * Reset the GPU without trying to cancel jobs and don't emit messages into + * the kernel log while doing the reset. + * + * This function should be used in cases where we are doing a controlled reset + * of the GPU as part of normal processing (e.g. exiting protected mode) where + * the driver will have ensured the scheduler has been idled and all other + * users of the GPU (e.g. instrumentation) have been suspended. + */ +void kbase_reset_gpu_silent(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_active - Reports if the GPU is being reset + * @kbdev: Device pointer + * + * Return: True if the GPU is in the process of being reset. + */ +bool kbase_reset_gpu_active(struct kbase_device *kbdev); +#endif + +/** + * kbase_job_slot_hardstop - Hard-stop the specified job slot + * @kctx: The kbase context that contains the job(s) that should + * be hard-stopped + * @js: The job slot to hard-stop + * @target_katom: The job that should be hard-stopped (or NULL for all + * jobs from the context) + * Context: + * The job slot lock must be held when calling this function. + */ +void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom); + +extern struct protected_mode_ops kbase_native_protected_ops; + +#endif /* _KBASE_HWACCESS_JM_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_pm.h new file mode 100755 index 000000000000..71c7d495c40a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_pm.h @@ -0,0 +1,209 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/** + * @file mali_kbase_hwaccess_pm.h + * HW access power manager common APIs + */ + +#ifndef _KBASE_HWACCESS_PM_H_ +#define _KBASE_HWACCESS_PM_H_ + +#include +#include + +#include + +/* Forward definition - see mali_kbase.h */ +struct kbase_device; + +/* Functions common to all HW access backends */ + +/** + * Initialize the power management framework. + * + * Must be called before any other power management function + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * + * @return 0 if the power management framework was successfully + * initialized. + */ +int kbase_hwaccess_pm_init(struct kbase_device *kbdev); + +/** + * Terminate the power management framework. + * + * No power management functions may be called after this (except + * @ref kbase_pm_init) + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_term(struct kbase_device *kbdev); + +/** + * kbase_hwaccess_pm_powerup - Power up the GPU. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @flags: Flags to pass on to kbase_pm_init_hw + * + * Power up GPU after all modules have been initialized and interrupt handlers + * installed. + * + * Return: 0 if powerup was successful. + */ +int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, + unsigned int flags); + +/** + * Halt the power management framework. + * + * Should ensure that no new interrupts are generated, but allow any currently + * running interrupt handlers to complete successfully. The GPU is forced off by + * the time this function returns, regardless of whether or not the active power + * policy asks for the GPU to be powered off. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_halt(struct kbase_device *kbdev); + +/** + * Perform any backend-specific actions to suspend the GPU + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev); + +/** + * Perform any backend-specific actions to resume the GPU from a suspend + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_resume(struct kbase_device *kbdev); + +/** + * Perform any required actions for activating the GPU. Called when the first + * context goes active. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); + +/** + * Perform any required actions for idling the GPU. Called when the last + * context goes idle. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); + + +/** + * Set the debug core mask. + * + * This determines which cores the power manager is allowed to use. + * + * @param kbdev The kbase device structure for the device (must be a + * valid pointer) + * @param new_core_mask_js0 The core mask to use for job slot 0 + * @param new_core_mask_js0 The core mask to use for job slot 1 + * @param new_core_mask_js0 The core mask to use for job slot 2 + */ +void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, + u64 new_core_mask_js0, u64 new_core_mask_js1, + u64 new_core_mask_js2); + + +/** + * Get the current policy. + * + * Returns the policy that is currently active. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * + * @return The current policy + */ +const struct kbase_pm_ca_policy +*kbase_pm_ca_get_policy(struct kbase_device *kbdev); + +/** + * Change the policy to the one specified. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * @param policy The policy to change to (valid pointer returned from + * @ref kbase_pm_ca_list_policies) + */ +void kbase_pm_ca_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_ca_policy *policy); + +/** + * Retrieve a static list of the available policies. + * + * @param[out] policies An array pointer to take the list of policies. This may + * be NULL. The contents of this array must not be + * modified. + * + * @return The number of policies + */ +int +kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **policies); + + +/** + * Get the current policy. + * + * Returns the policy that is currently active. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * + * @return The current policy + */ +const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev); + +/** + * Change the policy to the one specified. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * @param policy The policy to change to (valid pointer returned from + * @ref kbase_pm_list_policies) + */ +void kbase_pm_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_policy *policy); + +/** + * Retrieve a static list of the available policies. + * + * @param[out] policies An array pointer to take the list of policies. This may + * be NULL. The contents of this array must not be + * modified. + * + * @return The number of policies + */ +int kbase_pm_list_policies(const struct kbase_pm_policy * const **policies); + +#endif /* _KBASE_HWACCESS_PM_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_time.h new file mode 100755 index 000000000000..b9fe8e669c63 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwaccess_time.h @@ -0,0 +1,53 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/** + * + */ + +#ifndef _KBASE_BACKEND_TIME_H_ +#define _KBASE_BACKEND_TIME_H_ + +/** + * kbase_backend_get_gpu_time() - Get current GPU time + * @kbdev: Device pointer + * @cycle_counter: Pointer to u64 to store cycle counter in + * @system_time: Pointer to u64 to store system time in + * @ts: Pointer to struct timespec to store current monotonic + * time in + */ +void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, + u64 *system_time, struct timespec *ts); + +/** + * kbase_wait_write_flush() - Wait for GPU write flush + * @kctx: Context pointer + * + * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush + * its write buffer. + * + * If GPU resets occur then the counters are reset to zero, the delay may not be + * as expected. + * + * This function is only in use for BASE_HW_ISSUE_6367 + */ +#ifndef CONFIG_MALI_BIFROST_NO_MALI +void kbase_wait_write_flush(struct kbase_context *kctx); +#endif + +#endif /* _KBASE_BACKEND_TIME_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwcnt_reader.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwcnt_reader.h new file mode 100755 index 000000000000..cf7bf1b35dc5 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_hwcnt_reader.h @@ -0,0 +1,66 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_HWCNT_READER_H_ +#define _KBASE_HWCNT_READER_H_ + +/* The ids of ioctl commands. */ +#define KBASE_HWCNT_READER 0xBE +#define KBASE_HWCNT_READER_GET_HWVER _IOR(KBASE_HWCNT_READER, 0x00, u32) +#define KBASE_HWCNT_READER_GET_BUFFER_SIZE _IOR(KBASE_HWCNT_READER, 0x01, u32) +#define KBASE_HWCNT_READER_DUMP _IOW(KBASE_HWCNT_READER, 0x10, u32) +#define KBASE_HWCNT_READER_CLEAR _IOW(KBASE_HWCNT_READER, 0x11, u32) +#define KBASE_HWCNT_READER_GET_BUFFER _IOR(KBASE_HWCNT_READER, 0x20,\ + struct kbase_hwcnt_reader_metadata) +#define KBASE_HWCNT_READER_PUT_BUFFER _IOW(KBASE_HWCNT_READER, 0x21,\ + struct kbase_hwcnt_reader_metadata) +#define KBASE_HWCNT_READER_SET_INTERVAL _IOW(KBASE_HWCNT_READER, 0x30, u32) +#define KBASE_HWCNT_READER_ENABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x40, u32) +#define KBASE_HWCNT_READER_DISABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x41, u32) +#define KBASE_HWCNT_READER_GET_API_VERSION _IOW(KBASE_HWCNT_READER, 0xFF, u32) + +/** + * struct kbase_hwcnt_reader_metadata - hwcnt reader sample buffer metadata + * @timestamp: time when sample was collected + * @event_id: id of an event that triggered sample collection + * @buffer_idx: position in sampling area where sample buffer was stored + */ +struct kbase_hwcnt_reader_metadata { + u64 timestamp; + u32 event_id; + u32 buffer_idx; +}; + +/** + * enum base_hwcnt_reader_event - hwcnt dumping events + * @BASE_HWCNT_READER_EVENT_MANUAL: manual request for dump + * @BASE_HWCNT_READER_EVENT_PERIODIC: periodic dump + * @BASE_HWCNT_READER_EVENT_PREJOB: prejob dump request + * @BASE_HWCNT_READER_EVENT_POSTJOB: postjob dump request + * @BASE_HWCNT_READER_EVENT_COUNT: number of supported events + */ +enum base_hwcnt_reader_event { + BASE_HWCNT_READER_EVENT_MANUAL, + BASE_HWCNT_READER_EVENT_PERIODIC, + BASE_HWCNT_READER_EVENT_PREJOB, + BASE_HWCNT_READER_EVENT_POSTJOB, + + BASE_HWCNT_READER_EVENT_COUNT +}; + +#endif /* _KBASE_HWCNT_READER_H_ */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ioctl.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ioctl.h new file mode 100755 index 000000000000..e7c1daee470b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_ioctl.h @@ -0,0 +1,658 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_IOCTL_H_ +#define _KBASE_IOCTL_H_ + +#ifdef __cpluscplus +extern "C" { +#endif + +#include + +#define KBASE_IOCTL_TYPE 0x80 + +#ifdef ANDROID +/* Android's definition of ioctl is incorrect, specifying the type argument as + * 'int'. This creates a warning when using _IOWR (as the top bit is set). Work + * round this by redefining _IOC to include a case to 'int'. + */ +#undef _IOC +#define _IOC(dir, type, nr, size) \ + ((int)(((dir) << _IOC_DIRSHIFT) | ((type) << _IOC_TYPESHIFT) | \ + ((nr) << _IOC_NRSHIFT) | ((size) << _IOC_SIZESHIFT))) +#endif + +/** + * struct kbase_ioctl_version_check - Check version compatibility with kernel + * + * @major: Major version number + * @minor: Minor version number + */ +struct kbase_ioctl_version_check { + __u16 major; + __u16 minor; +}; + +#define KBASE_IOCTL_VERSION_CHECK \ + _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) + +/** + * struct kbase_ioctl_set_flags - Set kernel context creation flags + * + * @create_flags: Flags - see base_context_create_flags + */ +struct kbase_ioctl_set_flags { + __u32 create_flags; +}; + +#define KBASE_IOCTL_SET_FLAGS \ + _IOW(KBASE_IOCTL_TYPE, 1, struct kbase_ioctl_set_flags) + +/** + * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel + * + * @addr: Memory address of an array of struct base_jd_atom_v2 + * @nr_atoms: Number of entries in the array + * @stride: sizeof(struct base_jd_atom_v2) + */ +struct kbase_ioctl_job_submit { + __u64 addr; + __u32 nr_atoms; + __u32 stride; +}; + +#define KBASE_IOCTL_JOB_SUBMIT \ + _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit) + +/** + * struct kbase_ioctl_get_gpuprops - Read GPU properties from the kernel + * + * @buffer: Pointer to the buffer to store properties into + * @size: Size of the buffer + * @flags: Flags - must be zero for now + * + * The ioctl will return the number of bytes stored into @buffer or an error + * on failure (e.g. @size is too small). If @size is specified as 0 then no + * data will be written but the return value will be the number of bytes needed + * for all the properties. + * + * @flags may be used in the future to request a different format for the + * buffer. With @flags == 0 the following format is used. + * + * The buffer will be filled with pairs of values, a u32 key identifying the + * property followed by the value. The size of the value is identified using + * the bottom bits of the key. The value then immediately followed the key and + * is tightly packed (there is no padding). All keys and values are + * little-endian. + * + * 00 = u8 + * 01 = u16 + * 10 = u32 + * 11 = u64 + */ +struct kbase_ioctl_get_gpuprops { + __u64 buffer; + __u32 size; + __u32 flags; +}; + +#define KBASE_IOCTL_GET_GPUPROPS \ + _IOW(KBASE_IOCTL_TYPE, 3, struct kbase_ioctl_get_gpuprops) + +#define KBASE_IOCTL_POST_TERM \ + _IO(KBASE_IOCTL_TYPE, 4) + +/** + * union kbase_ioctl_mem_alloc - Allocate memory on the GPU + * + * @va_pages: The number of pages of virtual address space to reserve + * @commit_pages: The number of physical pages to allocate + * @extent: The number of extra pages to allocate on each GPU fault which grows + * the region + * @flags: Flags + * @gpu_va: The GPU virtual address which is allocated + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_alloc { + struct { + __u64 va_pages; + __u64 commit_pages; + __u64 extent; + __u64 flags; + } in; + struct { + __u64 flags; + __u64 gpu_va; + } out; +}; + +#define KBASE_IOCTL_MEM_ALLOC \ + _IOWR(KBASE_IOCTL_TYPE, 5, union kbase_ioctl_mem_alloc) + +/** + * struct kbase_ioctl_mem_query - Query properties of a GPU memory region + * @gpu_addr: A GPU address contained within the region + * @query: The type of query + * @value: The result of the query + * + * Use a %KBASE_MEM_QUERY_xxx flag as input for @query. + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_query { + struct { + __u64 gpu_addr; + __u64 query; + } in; + struct { + __u64 value; + } out; +}; + +#define KBASE_IOCTL_MEM_QUERY \ + _IOWR(KBASE_IOCTL_TYPE, 6, union kbase_ioctl_mem_query) + +#define KBASE_MEM_QUERY_COMMIT_SIZE 1 +#define KBASE_MEM_QUERY_VA_SIZE 2 +#define KBASE_MEM_QUERY_FLAGS 3 + +/** + * struct kbase_ioctl_mem_free - Free a memory region + * @gpu_addr: Handle to the region to free + */ +struct kbase_ioctl_mem_free { + __u64 gpu_addr; +}; + +#define KBASE_IOCTL_MEM_FREE \ + _IOW(KBASE_IOCTL_TYPE, 7, struct kbase_ioctl_mem_free) + +/** + * struct kbase_ioctl_hwcnt_reader_setup - Setup HWC dumper/reader + * @buffer_count: requested number of dumping buffers + * @jm_bm: counters selection bitmask (JM) + * @shader_bm: counters selection bitmask (Shader) + * @tiler_bm: counters selection bitmask (Tiler) + * @mmu_l2_bm: counters selection bitmask (MMU_L2) + * + * A fd is returned from the ioctl if successful, or a negative value on error + */ +struct kbase_ioctl_hwcnt_reader_setup { + __u32 buffer_count; + __u32 jm_bm; + __u32 shader_bm; + __u32 tiler_bm; + __u32 mmu_l2_bm; +}; + +#define KBASE_IOCTL_HWCNT_READER_SETUP \ + _IOW(KBASE_IOCTL_TYPE, 8, struct kbase_ioctl_hwcnt_reader_setup) + +/** + * struct kbase_ioctl_hwcnt_enable - Enable hardware counter collection + * @dump_buffer: GPU address to write counters to + * @jm_bm: counters selection bitmask (JM) + * @shader_bm: counters selection bitmask (Shader) + * @tiler_bm: counters selection bitmask (Tiler) + * @mmu_l2_bm: counters selection bitmask (MMU_L2) + */ +struct kbase_ioctl_hwcnt_enable { + __u64 dump_buffer; + __u32 jm_bm; + __u32 shader_bm; + __u32 tiler_bm; + __u32 mmu_l2_bm; +}; + +#define KBASE_IOCTL_HWCNT_ENABLE \ + _IOW(KBASE_IOCTL_TYPE, 9, struct kbase_ioctl_hwcnt_enable) + +#define KBASE_IOCTL_HWCNT_DUMP \ + _IO(KBASE_IOCTL_TYPE, 10) + +#define KBASE_IOCTL_HWCNT_CLEAR \ + _IO(KBASE_IOCTL_TYPE, 11) + +/** + * struct kbase_ioctl_disjoint_query - Query the disjoint counter + * @counter: A counter of disjoint events in the kernel + */ +struct kbase_ioctl_disjoint_query { + __u32 counter; +}; + +#define KBASE_IOCTL_DISJOINT_QUERY \ + _IOR(KBASE_IOCTL_TYPE, 12, struct kbase_ioctl_disjoint_query) + +/** + * struct kbase_ioctl_get_ddk_version - Query the kernel version + * @version_buffer: Buffer to receive the kernel version string + * @size: Size of the buffer + * + * The ioctl will return the number of bytes written into version_buffer + * (which includes a NULL byte) or a negative error code + */ +struct kbase_ioctl_get_ddk_version { + __u64 version_buffer; + __u32 size; +}; + +#define KBASE_IOCTL_GET_DDK_VERSION \ + _IOW(KBASE_IOCTL_TYPE, 13, struct kbase_ioctl_get_ddk_version) + +/** + * struct kbase_ioctl_mem_jit_init - Initialise the JIT memory allocator + * + * @va_pages: Number of VA pages to reserve for JIT + * + * Note that depending on the VA size of the application and GPU, the value + * specified in @va_pages may be ignored. + */ +struct kbase_ioctl_mem_jit_init { + __u64 va_pages; +}; + +#define KBASE_IOCTL_MEM_JIT_INIT \ + _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init) + +/** + * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory + * + * @handle: GPU memory handle (GPU VA) + * @user_addr: The address where it is mapped in user space + * @size: The number of bytes to synchronise + * @type: The direction to synchronise: 0 is sync to memory (clean), + * 1 is sync from memory (invalidate). Use the BASE_SYNCSET_OP_xxx constants. + * @padding: Padding to round up to a multiple of 8 bytes, must be zero + */ +struct kbase_ioctl_mem_sync { + __u64 handle; + __u64 user_addr; + __u64 size; + __u8 type; + __u8 padding[7]; +}; + +#define KBASE_IOCTL_MEM_SYNC \ + _IOW(KBASE_IOCTL_TYPE, 15, struct kbase_ioctl_mem_sync) + +/** + * union kbase_ioctl_mem_find_cpu_offset - Find the offset of a CPU pointer + * + * @gpu_addr: The GPU address of the memory region + * @cpu_addr: The CPU address to locate + * @size: A size in bytes to validate is contained within the region + * @offset: The offset from the start of the memory region to @cpu_addr + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_find_cpu_offset { + struct { + __u64 gpu_addr; + __u64 cpu_addr; + __u64 size; + } in; + struct { + __u64 offset; + } out; +}; + +#define KBASE_IOCTL_MEM_FIND_CPU_OFFSET \ + _IOWR(KBASE_IOCTL_TYPE, 16, union kbase_ioctl_mem_find_cpu_offset) + +/** + * struct kbase_ioctl_get_context_id - Get the kernel context ID + * + * @id: The kernel context ID + */ +struct kbase_ioctl_get_context_id { + __u32 id; +}; + +#define KBASE_IOCTL_GET_CONTEXT_ID \ + _IOR(KBASE_IOCTL_TYPE, 17, struct kbase_ioctl_get_context_id) + +/** + * struct kbase_ioctl_tlstream_acquire - Acquire a tlstream fd + * + * @flags: Flags + * + * The ioctl returns a file descriptor when successful + */ +struct kbase_ioctl_tlstream_acquire { + __u32 flags; +}; + +#define KBASE_IOCTL_TLSTREAM_ACQUIRE \ + _IOW(KBASE_IOCTL_TYPE, 18, struct kbase_ioctl_tlstream_acquire) + +#define KBASE_IOCTL_TLSTREAM_FLUSH \ + _IO(KBASE_IOCTL_TYPE, 19) + +/** + * struct kbase_ioctl_mem_commit - Change the amount of memory backing a region + * + * @gpu_addr: The memory region to modify + * @pages: The number of physical pages that should be present + * + * The ioctl may return on the following error codes or 0 for success: + * -ENOMEM: Out of memory + * -EINVAL: Invalid arguments + */ +struct kbase_ioctl_mem_commit { + __u64 gpu_addr; + __u64 pages; +}; + +#define KBASE_IOCTL_MEM_COMMIT \ + _IOW(KBASE_IOCTL_TYPE, 20, struct kbase_ioctl_mem_commit) + +/** + * union kbase_ioctl_mem_alias - Create an alias of memory regions + * @flags: Flags, see BASE_MEM_xxx + * @stride: Bytes between start of each memory region + * @nents: The number of regions to pack together into the alias + * @aliasing_info: Pointer to an array of struct base_mem_aliasing_info + * @gpu_va: Address of the new alias + * @va_pages: Size of the new alias + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_alias { + struct { + __u64 flags; + __u64 stride; + __u64 nents; + __u64 aliasing_info; + } in; + struct { + __u64 flags; + __u64 gpu_va; + __u64 va_pages; + } out; +}; + +#define KBASE_IOCTL_MEM_ALIAS \ + _IOWR(KBASE_IOCTL_TYPE, 21, union kbase_ioctl_mem_alias) + +/** + * union kbase_ioctl_mem_import - Import memory for use by the GPU + * @flags: Flags, see BASE_MEM_xxx + * @phandle: Handle to the external memory + * @type: Type of external memory, see base_mem_import_type + * @padding: Amount of extra VA pages to append to the imported buffer + * @gpu_va: Address of the new alias + * @va_pages: Size of the new alias + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_import { + struct { + __u64 flags; + __u64 phandle; + __u32 type; + __u32 padding; + } in; + struct { + __u64 flags; + __u64 gpu_va; + __u64 va_pages; + } out; +}; + +#define KBASE_IOCTL_MEM_IMPORT \ + _IOWR(KBASE_IOCTL_TYPE, 22, union kbase_ioctl_mem_import) + +/** + * struct kbase_ioctl_mem_flags_change - Change the flags for a memory region + * @gpu_va: The GPU region to modify + * @flags: The new flags to set + * @mask: Mask of the flags to modify + */ +struct kbase_ioctl_mem_flags_change { + __u64 gpu_va; + __u64 flags; + __u64 mask; +}; + +#define KBASE_IOCTL_MEM_FLAGS_CHANGE \ + _IOW(KBASE_IOCTL_TYPE, 23, struct kbase_ioctl_mem_flags_change) + +/** + * struct kbase_ioctl_stream_create - Create a synchronisation stream + * @name: A name to identify this stream. Must be NULL-terminated. + * + * Note that this is also called a "timeline", but is named stream to avoid + * confusion with other uses of the word. + * + * Unused bytes in @name (after the first NULL byte) must be also be NULL bytes. + * + * The ioctl returns a file descriptor. + */ +struct kbase_ioctl_stream_create { + char name[32]; +}; + +#define KBASE_IOCTL_STREAM_CREATE \ + _IOW(KBASE_IOCTL_TYPE, 24, struct kbase_ioctl_stream_create) + +/** + * struct kbase_ioctl_fence_validate - Validate a fd refers to a fence + * @fd: The file descriptor to validate + */ +struct kbase_ioctl_fence_validate { + int fd; +}; + +#define KBASE_IOCTL_FENCE_VALIDATE \ + _IOW(KBASE_IOCTL_TYPE, 25, struct kbase_ioctl_fence_validate) + +/** + * struct kbase_ioctl_get_profiling_controls - Get the profiling controls + * @count: The size of @buffer in u32 words + * @buffer: The buffer to receive the profiling controls + */ +struct kbase_ioctl_get_profiling_controls { + __u64 buffer; + __u32 count; +}; + +#define KBASE_IOCTL_GET_PROFILING_CONTROLS \ + _IOW(KBASE_IOCTL_TYPE, 26, struct kbase_ioctl_get_profiling_controls) + +/** + * struct kbase_ioctl_mem_profile_add - Provide profiling information to kernel + * @buffer: Pointer to the information + * @len: Length + * @padding: Padding + * + * The data provided is accessible through a debugfs file + */ +struct kbase_ioctl_mem_profile_add { + __u64 buffer; + __u32 len; + __u32 padding; +}; + +#define KBASE_IOCTL_MEM_PROFILE_ADD \ + _IOW(KBASE_IOCTL_TYPE, 27, struct kbase_ioctl_mem_profile_add) + +/** + * struct kbase_ioctl_soft_event_update - Update the status of a soft-event + * @event: GPU address of the event which has been updated + * @new_status: The new status to set + * @flags: Flags for future expansion + */ +struct kbase_ioctl_soft_event_update { + __u64 event; + __u32 new_status; + __u32 flags; +}; + +#define KBASE_IOCTL_SOFT_EVENT_UPDATE \ + _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) + +/* IOCTLs 29-32 are reserved */ + +/*************** + * test ioctls * + ***************/ +#if MALI_UNIT_TEST +/* These ioctls are purely for test purposes and are not used in the production + * driver, they therefore may change without notice + */ + +#define KBASE_IOCTL_TEST_TYPE (KBASE_IOCTL_TYPE + 1) + +/** + * struct kbase_ioctl_tlstream_test - Start a timeline stream test + * + * @tpw_count: number of trace point writers in each context + * @msg_delay: time delay between tracepoints from one writer in milliseconds + * @msg_count: number of trace points written by one writer + * @aux_msg: if non-zero aux messages will be included + */ +struct kbase_ioctl_tlstream_test { + __u32 tpw_count; + __u32 msg_delay; + __u32 msg_count; + __u32 aux_msg; +}; + +#define KBASE_IOCTL_TLSTREAM_TEST \ + _IOW(KBASE_IOCTL_TEST_TYPE, 1, struct kbase_ioctl_tlstream_test) + +/** + * struct kbase_ioctl_tlstream_stats - Read tlstream stats for test purposes + * @bytes_collected: number of bytes read by user + * @bytes_generated: number of bytes generated by tracepoints + */ +struct kbase_ioctl_tlstream_stats { + __u32 bytes_collected; + __u32 bytes_generated; +}; + +#define KBASE_IOCTL_TLSTREAM_STATS \ + _IOR(KBASE_IOCTL_TEST_TYPE, 2, struct kbase_ioctl_tlstream_stats) + +#endif + +/********************************** + * Definitions for GPU properties * + **********************************/ +#define KBASE_GPUPROP_VALUE_SIZE_U8 (0x0) +#define KBASE_GPUPROP_VALUE_SIZE_U16 (0x1) +#define KBASE_GPUPROP_VALUE_SIZE_U32 (0x2) +#define KBASE_GPUPROP_VALUE_SIZE_U64 (0x3) + +#define KBASE_GPUPROP_PRODUCT_ID 1 +#define KBASE_GPUPROP_VERSION_STATUS 2 +#define KBASE_GPUPROP_MINOR_REVISION 3 +#define KBASE_GPUPROP_MAJOR_REVISION 4 +#define KBASE_GPUPROP_GPU_SPEED_MHZ 5 +#define KBASE_GPUPROP_GPU_FREQ_KHZ_MAX 6 +#define KBASE_GPUPROP_GPU_FREQ_KHZ_MIN 7 +#define KBASE_GPUPROP_LOG2_PROGRAM_COUNTER_SIZE 8 +#define KBASE_GPUPROP_TEXTURE_FEATURES_0 9 +#define KBASE_GPUPROP_TEXTURE_FEATURES_1 10 +#define KBASE_GPUPROP_TEXTURE_FEATURES_2 11 +#define KBASE_GPUPROP_GPU_AVAILABLE_MEMORY_SIZE 12 + +#define KBASE_GPUPROP_L2_LOG2_LINE_SIZE 13 +#define KBASE_GPUPROP_L2_LOG2_CACHE_SIZE 14 +#define KBASE_GPUPROP_L2_NUM_L2_SLICES 15 + +#define KBASE_GPUPROP_TILER_BIN_SIZE_BYTES 16 +#define KBASE_GPUPROP_TILER_MAX_ACTIVE_LEVELS 17 + +#define KBASE_GPUPROP_MAX_THREADS 18 +#define KBASE_GPUPROP_MAX_WORKGROUP_SIZE 19 +#define KBASE_GPUPROP_MAX_BARRIER_SIZE 20 +#define KBASE_GPUPROP_MAX_REGISTERS 21 +#define KBASE_GPUPROP_MAX_TASK_QUEUE 22 +#define KBASE_GPUPROP_MAX_THREAD_GROUP_SPLIT 23 +#define KBASE_GPUPROP_IMPL_TECH 24 + +#define KBASE_GPUPROP_RAW_SHADER_PRESENT 25 +#define KBASE_GPUPROP_RAW_TILER_PRESENT 26 +#define KBASE_GPUPROP_RAW_L2_PRESENT 27 +#define KBASE_GPUPROP_RAW_STACK_PRESENT 28 +#define KBASE_GPUPROP_RAW_L2_FEATURES 29 +#define KBASE_GPUPROP_RAW_SUSPEND_SIZE 30 +#define KBASE_GPUPROP_RAW_MEM_FEATURES 31 +#define KBASE_GPUPROP_RAW_MMU_FEATURES 32 +#define KBASE_GPUPROP_RAW_AS_PRESENT 33 +#define KBASE_GPUPROP_RAW_JS_PRESENT 34 +#define KBASE_GPUPROP_RAW_JS_FEATURES_0 35 +#define KBASE_GPUPROP_RAW_JS_FEATURES_1 36 +#define KBASE_GPUPROP_RAW_JS_FEATURES_2 37 +#define KBASE_GPUPROP_RAW_JS_FEATURES_3 38 +#define KBASE_GPUPROP_RAW_JS_FEATURES_4 39 +#define KBASE_GPUPROP_RAW_JS_FEATURES_5 40 +#define KBASE_GPUPROP_RAW_JS_FEATURES_6 41 +#define KBASE_GPUPROP_RAW_JS_FEATURES_7 42 +#define KBASE_GPUPROP_RAW_JS_FEATURES_8 43 +#define KBASE_GPUPROP_RAW_JS_FEATURES_9 44 +#define KBASE_GPUPROP_RAW_JS_FEATURES_10 45 +#define KBASE_GPUPROP_RAW_JS_FEATURES_11 46 +#define KBASE_GPUPROP_RAW_JS_FEATURES_12 47 +#define KBASE_GPUPROP_RAW_JS_FEATURES_13 48 +#define KBASE_GPUPROP_RAW_JS_FEATURES_14 49 +#define KBASE_GPUPROP_RAW_JS_FEATURES_15 50 +#define KBASE_GPUPROP_RAW_TILER_FEATURES 51 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_0 52 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_1 53 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_2 54 +#define KBASE_GPUPROP_RAW_GPU_ID 55 +#define KBASE_GPUPROP_RAW_THREAD_MAX_THREADS 56 +#define KBASE_GPUPROP_RAW_THREAD_MAX_WORKGROUP_SIZE 57 +#define KBASE_GPUPROP_RAW_THREAD_MAX_BARRIER_SIZE 58 +#define KBASE_GPUPROP_RAW_THREAD_FEATURES 59 +#define KBASE_GPUPROP_RAW_COHERENCY_MODE 60 + +#define KBASE_GPUPROP_COHERENCY_NUM_GROUPS 61 +#define KBASE_GPUPROP_COHERENCY_NUM_CORE_GROUPS 62 +#define KBASE_GPUPROP_COHERENCY_COHERENCY 63 +#define KBASE_GPUPROP_COHERENCY_GROUP_0 64 +#define KBASE_GPUPROP_COHERENCY_GROUP_1 65 +#define KBASE_GPUPROP_COHERENCY_GROUP_2 66 +#define KBASE_GPUPROP_COHERENCY_GROUP_3 67 +#define KBASE_GPUPROP_COHERENCY_GROUP_4 68 +#define KBASE_GPUPROP_COHERENCY_GROUP_5 69 +#define KBASE_GPUPROP_COHERENCY_GROUP_6 70 +#define KBASE_GPUPROP_COHERENCY_GROUP_7 71 +#define KBASE_GPUPROP_COHERENCY_GROUP_8 72 +#define KBASE_GPUPROP_COHERENCY_GROUP_9 73 +#define KBASE_GPUPROP_COHERENCY_GROUP_10 74 +#define KBASE_GPUPROP_COHERENCY_GROUP_11 75 +#define KBASE_GPUPROP_COHERENCY_GROUP_12 76 +#define KBASE_GPUPROP_COHERENCY_GROUP_13 77 +#define KBASE_GPUPROP_COHERENCY_GROUP_14 78 +#define KBASE_GPUPROP_COHERENCY_GROUP_15 79 + +#ifdef __cpluscplus +} +#endif + +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd.c new file mode 100755 index 000000000000..144ebfcdfc59 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd.c @@ -0,0 +1,1847 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include +#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ +#ifdef CONFIG_COMPAT +#include +#endif +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "mali_kbase_dma_fence.h" + +#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) +/* random32 was renamed to prandom_u32 in 3.8 */ +#define prandom_u32 random32 +#endif + +/* Return whether katom will run on the GPU or not. Currently only soft jobs and + * dependency-only atoms do not run on the GPU */ +#define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ + ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \ + BASE_JD_REQ_DEP))) +/* + * This is the kernel side of the API. Only entry points are: + * - kbase_jd_submit(): Called from userspace to submit a single bag + * - kbase_jd_done(): Called from interrupt context to track the + * completion of a job. + * Callouts: + * - to the job manager (enqueue a job) + * - to the event subsystem (signals the completion/failure of bag/job-chains). + */ + +static void __user * +get_compat_pointer(struct kbase_context *kctx, const u64 p) +{ +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + return compat_ptr(p); +#endif + return u64_to_user_ptr(p); +} + +/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs + * + * Returns whether the JS needs a reschedule. + * + * Note that the caller must also check the atom status and + * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock + */ +static int jd_run_atom(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); + + if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { + /* Dependency only atom */ + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + return 0; + } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { + /* Soft-job */ + if (katom->will_fail_event_code) { + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + return 0; + } + if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) + == BASE_JD_REQ_SOFT_REPLAY) { + if (!kbase_replay_process(katom)) + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + } else if (kbase_process_soft_job(katom) == 0) { + kbase_finish_soft_job(katom); + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + } + return 0; + } + + katom->status = KBASE_JD_ATOM_STATE_IN_JS; + /* Queue an action about whether we should try scheduling a context */ + return kbasep_js_add_job(kctx, katom); +} + +#if defined(CONFIG_KDS) || defined(CONFIG_MALI_BIFROST_DMA_FENCE) +void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom) +{ + struct kbase_device *kbdev; + + KBASE_DEBUG_ASSERT(katom); + kbdev = katom->kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev); + + /* Check whether the atom's other dependencies were already met. If + * katom is a GPU atom then the job scheduler may be able to represent + * the dependencies, hence we may attempt to submit it before they are + * met. Other atoms must have had both dependencies resolved. + */ + if (IS_GPU_ATOM(katom) || + (!kbase_jd_katom_dep_atom(&katom->dep[0]) && + !kbase_jd_katom_dep_atom(&katom->dep[1]))) { + /* katom dep complete, attempt to run it */ + bool resched = false; + + resched = jd_run_atom(katom); + + if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) { + /* The atom has already finished */ + resched |= jd_done_nolock(katom, NULL); + } + + if (resched) + kbase_js_sched_all(kbdev); + } +} +#endif + +#ifdef CONFIG_KDS + +/* Add the katom to the kds waiting list. + * Atoms must be added to the waiting list after a successful call to kds_async_waitall. + * The caller must hold the kbase_jd_context.lock */ + +static void kbase_jd_kds_waiters_add(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx; + + KBASE_DEBUG_ASSERT(katom); + + kctx = katom->kctx; + + list_add_tail(&katom->node, &kctx->waiting_kds_resource); +} + +/* Remove the katom from the kds waiting list. + * Atoms must be removed from the waiting list before a call to kds_resource_set_release_sync. + * The supplied katom must first have been added to the list with a call to kbase_jd_kds_waiters_add. + * The caller must hold the kbase_jd_context.lock */ + +static void kbase_jd_kds_waiters_remove(struct kbase_jd_atom *katom) +{ + KBASE_DEBUG_ASSERT(katom); + list_del(&katom->node); +} + +static void kds_dep_clear(void *callback_parameter, void *callback_extra_parameter) +{ + struct kbase_jd_atom *katom; + struct kbase_jd_context *ctx; + + katom = (struct kbase_jd_atom *)callback_parameter; + KBASE_DEBUG_ASSERT(katom); + + ctx = &katom->kctx->jctx; + + /* If KDS resource has already been satisfied (e.g. due to zapping) + * do nothing. + */ + mutex_lock(&ctx->lock); + if (!katom->kds_dep_satisfied) { + katom->kds_dep_satisfied = true; + kbase_jd_dep_clear_locked(katom); + } + mutex_unlock(&ctx->lock); +} + +static void kbase_cancel_kds_wait_job(struct kbase_jd_atom *katom) +{ + KBASE_DEBUG_ASSERT(katom); + + /* Prevent job_done_nolock from being called twice on an atom when + * there is a race between job completion and cancellation */ + + if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { + /* Wait was cancelled - zap the atom */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); + } +} +#endif /* CONFIG_KDS */ + +void kbase_jd_free_external_resources(struct kbase_jd_atom *katom) +{ +#ifdef CONFIG_KDS + if (katom->kds_rset) { + struct kbase_jd_context *jctx = &katom->kctx->jctx; + + /* + * As the atom is no longer waiting, remove it from + * the waiting list. + */ + + mutex_lock(&jctx->lock); + kbase_jd_kds_waiters_remove(katom); + mutex_unlock(&jctx->lock); + + /* Release the kds resource or cancel if zapping */ + kds_resource_set_release_sync(&katom->kds_rset); + } +#endif /* CONFIG_KDS */ + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + /* Flush dma-fence workqueue to ensure that any callbacks that may have + * been queued are done before continuing. + * Any successfully completed atom would have had all it's callbacks + * completed before the atom was run, so only flush for failed atoms. + */ + if (katom->event_code != BASE_JD_EVENT_DONE) + flush_workqueue(katom->kctx->dma_fence.wq); +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ +} + +static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) +{ + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); + +#ifdef CONFIG_KDS + /* Prevent the KDS resource from triggering the atom in case of zapping */ + if (katom->kds_rset) + katom->kds_dep_satisfied = true; +#endif /* CONFIG_KDS */ + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + kbase_dma_fence_signal(katom); +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + kbase_gpu_vm_lock(katom->kctx); + /* only roll back if extres is non-NULL */ + if (katom->extres) { + u32 res_no; + + res_no = katom->nr_extres; + while (res_no-- > 0) { + struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; + struct kbase_va_region *reg; + + reg = kbase_region_tracker_find_region_base_address( + katom->kctx, + katom->extres[res_no].gpu_address); + kbase_unmap_external_resource(katom->kctx, reg, alloc); + } + kfree(katom->extres); + katom->extres = NULL; + } + kbase_gpu_vm_unlock(katom->kctx); +} + +/* + * Set up external resources needed by this job. + * + * jctx.lock must be held when this is called. + */ + +static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom) +{ + int err_ret_val = -EINVAL; + u32 res_no; +#ifdef CONFIG_KDS + u32 kds_res_count = 0; + struct kds_resource **kds_resources = NULL; + unsigned long *kds_access_bitmap = NULL; +#endif /* CONFIG_KDS */ +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + struct kbase_dma_fence_resv_info info = { + .dma_fence_resv_count = 0, + }; +#ifdef CONFIG_SYNC + /* + * When both dma-buf fence and Android native sync is enabled, we + * disable dma-buf fence for contexts that are using Android native + * fences. + */ + const bool implicit_sync = !kbase_ctx_flag(katom->kctx, + KCTX_NO_IMPLICIT_SYNC); +#else /* CONFIG_SYNC */ + const bool implicit_sync = true; +#endif /* CONFIG_SYNC */ +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + struct base_external_resource *input_extres; + + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); + + /* no resources encoded, early out */ + if (!katom->nr_extres) + return -EINVAL; + + katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL); + if (NULL == katom->extres) { + err_ret_val = -ENOMEM; + goto early_err_out; + } + + /* copy user buffer to the end of our real buffer. + * Make sure the struct sizes haven't changed in a way + * we don't support */ + BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres)); + input_extres = (struct base_external_resource *) + (((unsigned char *)katom->extres) + + (sizeof(*katom->extres) - sizeof(*input_extres)) * + katom->nr_extres); + + if (copy_from_user(input_extres, + get_compat_pointer(katom->kctx, user_atom->extres_list), + sizeof(*input_extres) * katom->nr_extres) != 0) { + err_ret_val = -EINVAL; + goto early_err_out; + } +#ifdef CONFIG_KDS + /* assume we have to wait for all */ + KBASE_DEBUG_ASSERT(0 != katom->nr_extres); + kds_resources = kmalloc_array(katom->nr_extres, sizeof(struct kds_resource *), GFP_KERNEL); + + if (!kds_resources) { + err_ret_val = -ENOMEM; + goto early_err_out; + } + + KBASE_DEBUG_ASSERT(0 != katom->nr_extres); + kds_access_bitmap = kcalloc(BITS_TO_LONGS(katom->nr_extres), + sizeof(unsigned long), + GFP_KERNEL); + if (!kds_access_bitmap) { + err_ret_val = -ENOMEM; + goto early_err_out; + } +#endif /* CONFIG_KDS */ + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (implicit_sync) { + info.resv_objs = kmalloc_array(katom->nr_extres, + sizeof(struct reservation_object *), + GFP_KERNEL); + if (!info.resv_objs) { + err_ret_val = -ENOMEM; + goto early_err_out; + } + + info.dma_fence_excl_bitmap = + kcalloc(BITS_TO_LONGS(katom->nr_extres), + sizeof(unsigned long), GFP_KERNEL); + if (!info.dma_fence_excl_bitmap) { + err_ret_val = -ENOMEM; + goto early_err_out; + } + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + /* Take the processes mmap lock */ + down_read(¤t->mm->mmap_sem); + + /* need to keep the GPU VM locked while we set up UMM buffers */ + kbase_gpu_vm_lock(katom->kctx); + for (res_no = 0; res_no < katom->nr_extres; res_no++) { + struct base_external_resource *res; + struct kbase_va_region *reg; + struct kbase_mem_phy_alloc *alloc; + bool exclusive; + + res = &input_extres[res_no]; + exclusive = (res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE) + ? true : false; + reg = kbase_region_tracker_find_region_enclosing_address( + katom->kctx, + res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE); + /* did we find a matching region object? */ + if (NULL == reg || (reg->flags & KBASE_REG_FREE)) { + /* roll back */ + goto failed_loop; + } + + if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) && + (reg->flags & KBASE_REG_SECURE)) { + katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED; + } + + alloc = kbase_map_external_resource(katom->kctx, reg, + current->mm +#ifdef CONFIG_KDS + , &kds_res_count, kds_resources, + kds_access_bitmap, exclusive +#endif + ); + if (!alloc) { + err_ret_val = -EINVAL; + goto failed_loop; + } + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (implicit_sync && + reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { + struct reservation_object *resv; + + resv = reg->gpu_alloc->imported.umm.dma_buf->resv; + if (resv) + kbase_dma_fence_add_reservation(resv, &info, + exclusive); + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + /* finish with updating out array with the data we found */ + /* NOTE: It is important that this is the last thing we do (or + * at least not before the first write) as we overwrite elements + * as we loop and could be overwriting ourself, so no writes + * until the last read for an element. + * */ + katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */ + katom->extres[res_no].alloc = alloc; + } + /* successfully parsed the extres array */ + /* drop the vm lock before we call into kds */ + kbase_gpu_vm_unlock(katom->kctx); + + /* Release the processes mmap lock */ + up_read(¤t->mm->mmap_sem); + +#ifdef CONFIG_KDS + if (kds_res_count) { + int wait_failed; + + /* We have resources to wait for with kds */ + katom->kds_dep_satisfied = false; + + wait_failed = kds_async_waitall(&katom->kds_rset, + &katom->kctx->jctx.kds_cb, katom, NULL, + kds_res_count, kds_access_bitmap, + kds_resources); + + if (wait_failed) + goto failed_kds_setup; + else + kbase_jd_kds_waiters_add(katom); + } else { + /* Nothing to wait for, so kds dep met */ + katom->kds_dep_satisfied = true; + } + kfree(kds_resources); + kfree(kds_access_bitmap); +#endif /* CONFIG_KDS */ + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (implicit_sync) { + if (info.dma_fence_resv_count) { + int ret; + + ret = kbase_dma_fence_wait(katom, &info); + if (ret < 0) + goto failed_dma_fence_setup; + } + + kfree(info.resv_objs); + kfree(info.dma_fence_excl_bitmap); + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + /* all done OK */ + return 0; + +/* error handling section */ + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE +failed_dma_fence_setup: +#ifdef CONFIG_KDS + /* If we are here, dma_fence setup failed but KDS didn't. + * Revert KDS setup if any. + */ + if (kds_res_count) { + mutex_unlock(&katom->kctx->jctx.lock); + kds_resource_set_release_sync(&katom->kds_rset); + mutex_lock(&katom->kctx->jctx.lock); + + kbase_jd_kds_waiters_remove(katom); + katom->kds_dep_satisfied = true; + } +#endif /* CONFIG_KDS */ +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ +#ifdef CONFIG_KDS +failed_kds_setup: +#endif +#if defined(CONFIG_KDS) || defined(CONFIG_MALI_BIFROST_DMA_FENCE) + /* Lock the processes mmap lock */ + down_read(¤t->mm->mmap_sem); + + /* lock before we unmap */ + kbase_gpu_vm_lock(katom->kctx); +#endif + + failed_loop: + /* undo the loop work */ + while (res_no-- > 0) { + struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; + + kbase_unmap_external_resource(katom->kctx, NULL, alloc); + } + kbase_gpu_vm_unlock(katom->kctx); + + /* Release the processes mmap lock */ + up_read(¤t->mm->mmap_sem); + + early_err_out: + kfree(katom->extres); + katom->extres = NULL; +#ifdef CONFIG_KDS + kfree(kds_resources); + kfree(kds_access_bitmap); +#endif /* CONFIG_KDS */ +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (implicit_sync) { + kfree(info.resv_objs); + kfree(info.dma_fence_excl_bitmap); + } +#endif + return err_ret_val; +} + +static inline void jd_resolve_dep(struct list_head *out_list, + struct kbase_jd_atom *katom, + u8 d, bool ctx_is_dying) +{ + u8 other_d = !d; + + while (!list_empty(&katom->dep_head[d])) { + struct kbase_jd_atom *dep_atom; + struct kbase_jd_atom *other_dep_atom; + u8 dep_type; + + dep_atom = list_entry(katom->dep_head[d].next, + struct kbase_jd_atom, dep_item[d]); + list_del(katom->dep_head[d].next); + + dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]); + kbase_jd_katom_dep_clear(&dep_atom->dep[d]); + + if (katom->event_code != BASE_JD_EVENT_DONE && + (dep_type != BASE_JD_DEP_TYPE_ORDER)) { +#ifdef CONFIG_KDS + if (!dep_atom->kds_dep_satisfied) { + /* Just set kds_dep_satisfied to true. If the callback happens after this then it will early out and + * do nothing. If the callback doesn't happen then kbase_jd_post_external_resources will clean up + */ + dep_atom->kds_dep_satisfied = true; + } +#endif + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + kbase_dma_fence_cancel_callbacks(dep_atom); +#endif + + dep_atom->event_code = katom->event_code; + KBASE_DEBUG_ASSERT(dep_atom->status != + KBASE_JD_ATOM_STATE_UNUSED); + + if ((dep_atom->core_req & BASE_JD_REQ_SOFT_REPLAY) + != BASE_JD_REQ_SOFT_REPLAY) { + dep_atom->will_fail_event_code = + dep_atom->event_code; + } else { + dep_atom->status = + KBASE_JD_ATOM_STATE_COMPLETED; + } + } + other_dep_atom = (struct kbase_jd_atom *) + kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]); + + if (!dep_atom->in_jd_list && (!other_dep_atom || + (IS_GPU_ATOM(dep_atom) && !ctx_is_dying && + !dep_atom->will_fail_event_code && + !other_dep_atom->will_fail_event_code))) { + bool dep_satisfied = true; +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + int dep_count; + + dep_count = kbase_fence_dep_count_read(dep_atom); + if (likely(dep_count == -1)) { + dep_satisfied = true; + } else { + /* + * There are either still active callbacks, or + * all fences for this @dep_atom has signaled, + * but the worker that will queue the atom has + * not yet run. + * + * Wait for the fences to signal and the fence + * worker to run and handle @dep_atom. If + * @dep_atom was completed due to error on + * @katom, then the fence worker will pick up + * the complete status and error code set on + * @dep_atom above. + */ + dep_satisfied = false; + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + +#ifdef CONFIG_KDS + dep_satisfied = dep_satisfied && dep_atom->kds_dep_satisfied; +#endif + + if (dep_satisfied) { + dep_atom->in_jd_list = true; + list_add_tail(&dep_atom->jd_item, out_list); + } + } + } +} + +KBASE_EXPORT_TEST_API(jd_resolve_dep); + +#if MALI_CUSTOMER_RELEASE == 0 +static void jd_force_failure(struct kbase_device *kbdev, struct kbase_jd_atom *katom) +{ + kbdev->force_replay_count++; + + if (kbdev->force_replay_count >= kbdev->force_replay_limit) { + kbdev->force_replay_count = 0; + katom->event_code = BASE_JD_EVENT_FORCE_REPLAY; + + if (kbdev->force_replay_random) + kbdev->force_replay_limit = + (prandom_u32() % KBASEP_FORCE_REPLAY_RANDOM_LIMIT) + 1; + + dev_info(kbdev->dev, "force_replay : promoting to error\n"); + } +} + +/** Test to see if atom should be forced to fail. + * + * This function will check if an atom has a replay job as a dependent. If so + * then it will be considered for forced failure. */ +static void jd_check_force_failure(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + int i; + + if ((kbdev->force_replay_limit == KBASEP_FORCE_REPLAY_DISABLED) || + (katom->core_req & BASEP_JD_REQ_EVENT_NEVER)) + return; + + for (i = 1; i < BASE_JD_ATOM_COUNT; i++) { + if (kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[0]) == katom || + kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[1]) == katom) { + struct kbase_jd_atom *dep_atom = &kctx->jctx.atoms[i]; + + if ((dep_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == + BASE_JD_REQ_SOFT_REPLAY && + (dep_atom->core_req & kbdev->force_replay_core_req) + == kbdev->force_replay_core_req) { + jd_force_failure(kbdev, katom); + return; + } + } + } +} +#endif + +/** + * is_dep_valid - Validate that a dependency is valid for early dependency + * submission + * @katom: Dependency atom to validate + * + * A dependency is valid if any of the following are true : + * - It does not exist (a non-existent dependency does not block submission) + * - It is in the job scheduler + * - It has completed, does not have a failure event code, and has not been + * marked to fail in the future + * + * Return: true if valid, false otherwise + */ +static bool is_dep_valid(struct kbase_jd_atom *katom) +{ + /* If there's no dependency then this is 'valid' from the perspective of + * early dependency submission */ + if (!katom) + return true; + + /* Dependency must have reached the job scheduler */ + if (katom->status < KBASE_JD_ATOM_STATE_IN_JS) + return false; + + /* If dependency has completed and has failed or will fail then it is + * not valid */ + if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && + (katom->event_code != BASE_JD_EVENT_DONE || + katom->will_fail_event_code)) + return false; + + return true; +} + +static void jd_try_submitting_deps(struct list_head *out_list, + struct kbase_jd_atom *node) +{ + int i; + + for (i = 0; i < 2; i++) { + struct list_head *pos; + + list_for_each(pos, &node->dep_head[i]) { + struct kbase_jd_atom *dep_atom = list_entry(pos, + struct kbase_jd_atom, dep_item[i]); + + if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) { + /*Check if atom deps look sane*/ + bool dep0_valid = is_dep_valid( + dep_atom->dep[0].atom); + bool dep1_valid = is_dep_valid( + dep_atom->dep[1].atom); + bool dep_satisfied = true; +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + int dep_count; + + dep_count = kbase_fence_dep_count_read( + dep_atom); + if (likely(dep_count == -1)) { + dep_satisfied = true; + } else { + /* + * There are either still active callbacks, or + * all fences for this @dep_atom has signaled, + * but the worker that will queue the atom has + * not yet run. + * + * Wait for the fences to signal and the fence + * worker to run and handle @dep_atom. If + * @dep_atom was completed due to error on + * @katom, then the fence worker will pick up + * the complete status and error code set on + * @dep_atom above. + */ + dep_satisfied = false; + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ +#ifdef CONFIG_KDS + dep_satisfied = dep_satisfied && + dep_atom->kds_dep_satisfied; +#endif + + if (dep0_valid && dep1_valid && dep_satisfied) { + dep_atom->in_jd_list = true; + list_add(&dep_atom->jd_item, out_list); + } + } + } + } +} + +/* + * Perform the necessary handling of an atom that has finished running + * on the GPU. + * + * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller + * is responsible for calling kbase_finish_soft_job *before* calling this function. + * + * The caller must hold the kbase_jd_context.lock. + */ +bool jd_done_nolock(struct kbase_jd_atom *katom, + struct list_head *completed_jobs_ctx) +{ + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct list_head completed_jobs; + struct list_head runnable_jobs; + bool need_to_try_schedule_context = false; + int i; + + INIT_LIST_HEAD(&completed_jobs); + INIT_LIST_HEAD(&runnable_jobs); + + KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); + +#if MALI_CUSTOMER_RELEASE == 0 + jd_check_force_failure(katom); +#endif + + /* This is needed in case an atom is failed due to being invalid, this + * can happen *before* the jobs that the atom depends on have completed */ + for (i = 0; i < 2; i++) { + if (kbase_jd_katom_dep_atom(&katom->dep[i])) { + list_del(&katom->dep_item[i]); + kbase_jd_katom_dep_clear(&katom->dep[i]); + } + } + + /* With PRLAM-10817 or PRLAM-10959 the last tile of a fragment job being soft-stopped can fail with + * BASE_JD_EVENT_TILE_RANGE_FAULT. + * + * So here if the fragment job failed with TILE_RANGE_FAULT and it has been soft-stopped, then we promote the + * error code to BASE_JD_EVENT_DONE + */ + + if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10817) || kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10959)) && + katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT) { + if ((katom->core_req & BASE_JD_REQ_FS) && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED)) { + /* Promote the failure to job done */ + katom->event_code = BASE_JD_EVENT_DONE; + katom->atom_flags = katom->atom_flags & (~KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED); + } + } + + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + list_add_tail(&katom->jd_item, &completed_jobs); + + while (!list_empty(&completed_jobs)) { + katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item); + list_del(completed_jobs.prev); + KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); + + for (i = 0; i < 2; i++) + jd_resolve_dep(&runnable_jobs, katom, i, + kbase_ctx_flag(kctx, KCTX_DYING)); + + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) + kbase_jd_post_external_resources(katom); + + while (!list_empty(&runnable_jobs)) { + struct kbase_jd_atom *node; + + node = list_entry(runnable_jobs.next, + struct kbase_jd_atom, jd_item); + list_del(runnable_jobs.next); + node->in_jd_list = false; + + KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); + + if (node->status != KBASE_JD_ATOM_STATE_COMPLETED && + !kbase_ctx_flag(kctx, KCTX_DYING)) { + need_to_try_schedule_context |= jd_run_atom(node); + } else { + node->event_code = katom->event_code; + + if ((node->core_req & + BASE_JD_REQ_SOFT_JOB_TYPE) == + BASE_JD_REQ_SOFT_REPLAY) { + if (kbase_replay_process(node)) + /* Don't complete this atom */ + continue; + } else if (node->core_req & + BASE_JD_REQ_SOFT_JOB) { + /* If this is a fence wait soft job + * then remove it from the list of sync + * waiters. + */ + if (BASE_JD_REQ_SOFT_FENCE_WAIT == node->core_req) + kbasep_remove_waiting_soft_job(node); + + kbase_finish_soft_job(node); + } + node->status = KBASE_JD_ATOM_STATE_COMPLETED; + } + + if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) { + list_add_tail(&node->jd_item, &completed_jobs); + } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS && + !node->will_fail_event_code) { + /* Node successfully submitted, try submitting + * dependencies as they may now be representable + * in JS */ + jd_try_submitting_deps(&runnable_jobs, node); + } + } + + /* Register a completed job as a disjoint event when the GPU + * is in a disjoint state (ie. being reset or replaying jobs). + */ + kbase_disjoint_event_potential(kctx->kbdev); + if (completed_jobs_ctx) + list_add_tail(&katom->jd_item, completed_jobs_ctx); + else + kbase_event_post(kctx, katom); + + /* Decrement and check the TOTAL number of jobs. This includes + * those not tracked by the scheduler: 'not ready to run' and + * 'dependency-only' jobs. */ + if (--kctx->jctx.job_nr == 0) + wake_up(&kctx->jctx.zero_jobs_wait); /* All events are safely queued now, and we can signal any waiter + * that we've got no more jobs (so we can be safely terminated) */ + } + + return need_to_try_schedule_context; +} + +KBASE_EXPORT_TEST_API(jd_done_nolock); + +#ifdef CONFIG_GPU_TRACEPOINTS +enum { + CORE_REQ_DEP_ONLY, + CORE_REQ_SOFT, + CORE_REQ_COMPUTE, + CORE_REQ_FRAGMENT, + CORE_REQ_VERTEX, + CORE_REQ_TILER, + CORE_REQ_FRAGMENT_VERTEX, + CORE_REQ_FRAGMENT_VERTEX_TILER, + CORE_REQ_FRAGMENT_TILER, + CORE_REQ_VERTEX_TILER, + CORE_REQ_UNKNOWN +}; +static const char * const core_req_strings[] = { + "Dependency Only Job", + "Soft Job", + "Compute Shader Job", + "Fragment Shader Job", + "Vertex/Geometry Shader Job", + "Tiler Job", + "Fragment Shader + Vertex/Geometry Shader Job", + "Fragment Shader + Vertex/Geometry Shader Job + Tiler Job", + "Fragment Shader + Tiler Job", + "Vertex/Geometry Shader Job + Tiler Job", + "Unknown Job" +}; +static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req) +{ + if (core_req & BASE_JD_REQ_SOFT_JOB) + return core_req_strings[CORE_REQ_SOFT]; + if (core_req & BASE_JD_REQ_ONLY_COMPUTE) + return core_req_strings[CORE_REQ_COMPUTE]; + switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) { + case BASE_JD_REQ_DEP: + return core_req_strings[CORE_REQ_DEP_ONLY]; + case BASE_JD_REQ_FS: + return core_req_strings[CORE_REQ_FRAGMENT]; + case BASE_JD_REQ_CS: + return core_req_strings[CORE_REQ_VERTEX]; + case BASE_JD_REQ_T: + return core_req_strings[CORE_REQ_TILER]; + case (BASE_JD_REQ_FS | BASE_JD_REQ_CS): + return core_req_strings[CORE_REQ_FRAGMENT_VERTEX]; + case (BASE_JD_REQ_FS | BASE_JD_REQ_T): + return core_req_strings[CORE_REQ_FRAGMENT_TILER]; + case (BASE_JD_REQ_CS | BASE_JD_REQ_T): + return core_req_strings[CORE_REQ_VERTEX_TILER]; + case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T): + return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER]; + } + return core_req_strings[CORE_REQ_UNKNOWN]; +} +#endif + +bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *user_atom, struct kbase_jd_atom *katom) +{ + struct kbase_jd_context *jctx = &kctx->jctx; + int queued = 0; + int i; + int sched_prio; + bool ret; + bool will_fail = false; + + /* Update the TOTAL number of jobs. This includes those not tracked by + * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ + jctx->job_nr++; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + katom->start_timestamp.tv64 = 0; +#else + katom->start_timestamp = 0; +#endif + katom->udata = user_atom->udata; + katom->kctx = kctx; + katom->nr_extres = user_atom->nr_extres; + katom->extres = NULL; + katom->device_nr = user_atom->device_nr; + katom->affinity = 0; + katom->jc = user_atom->jc; + katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; + katom->core_req = user_atom->core_req; + katom->atom_flags = 0; + katom->retry_count = 0; + katom->need_cache_flush_cores_retained = 0; + katom->pre_dep = NULL; + katom->post_dep = NULL; + katom->x_pre_dep = NULL; + katom->x_post_dep = NULL; + katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; + + /* Implicitly sets katom->protected_state.enter as well. */ + katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; + + katom->age = kctx->age_count++; + + INIT_LIST_HEAD(&katom->jd_item); +#ifdef CONFIG_KDS + /* Start by assuming that the KDS dependencies are satisfied, + * kbase_jd_pre_external_resources will correct this if there are dependencies */ + katom->kds_dep_satisfied = true; + katom->kds_rset = NULL; +#endif /* CONFIG_KDS */ +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + kbase_fence_dep_count_set(katom, -1); +#endif + + /* Don't do anything if there is a mess up with dependencies. + This is done in a separate cycle to check both the dependencies at ones, otherwise + it will be extra complexity to deal with 1st dependency ( just added to the list ) + if only the 2nd one has invalid config. + */ + for (i = 0; i < 2; i++) { + int dep_atom_number = user_atom->pre_dep[i].atom_id; + base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type; + + if (dep_atom_number) { + if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER && + dep_atom_type != BASE_JD_DEP_TYPE_DATA) { + katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + + /* Wrong dependency setup. Atom will be sent + * back to user space. Do not record any + * dependencies. */ + KBASE_TLSTREAM_TL_NEW_ATOM( + katom, + kbase_jd_atom_id(kctx, katom)); + KBASE_TLSTREAM_TL_RET_ATOM_CTX( + katom, kctx); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, + TL_ATOM_STATE_IDLE); + + ret = jd_done_nolock(katom, NULL); + goto out; + } + } + } + + /* Add dependencies */ + for (i = 0; i < 2; i++) { + int dep_atom_number = user_atom->pre_dep[i].atom_id; + base_jd_dep_type dep_atom_type; + struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; + + dep_atom_type = user_atom->pre_dep[i].dependency_type; + kbase_jd_katom_dep_clear(&katom->dep[i]); + + if (!dep_atom_number) + continue; + + if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || + dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) { + + if (dep_atom->event_code == BASE_JD_EVENT_DONE) + continue; + /* don't stop this atom if it has an order dependency + * only to the failed one, try to submit it through + * the normal path + */ + if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER && + dep_atom->event_code > BASE_JD_EVENT_ACTIVE) { + continue; + } + + /* Atom has completed, propagate the error code if any */ + katom->event_code = dep_atom->event_code; + katom->status = KBASE_JD_ATOM_STATE_QUEUED; + + /* This atom is going through soft replay or + * will be sent back to user space. Do not record any + * dependencies. */ + KBASE_TLSTREAM_TL_NEW_ATOM( + katom, + kbase_jd_atom_id(kctx, katom)); + KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, + TL_ATOM_STATE_IDLE); + + if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) + == BASE_JD_REQ_SOFT_REPLAY) { + if (kbase_replay_process(katom)) { + ret = false; + goto out; + } + } + will_fail = true; + + } else { + /* Atom is in progress, add this atom to the list */ + list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]); + kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type); + queued = 1; + } + } + + if (will_fail) { + if (!queued) { + ret = jd_done_nolock(katom, NULL); + + goto out; + } else { + katom->will_fail_event_code = katom->event_code; + ret = false; + + goto out; + } + } else { + /* These must occur after the above loop to ensure that an atom + * that depends on a previous atom with the same number behaves + * as expected */ + katom->event_code = BASE_JD_EVENT_DONE; + katom->status = KBASE_JD_ATOM_STATE_QUEUED; + } + + /* For invalid priority, be most lenient and choose the default */ + sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); + if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) + sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; + katom->sched_priority = sched_prio; + + /* Create a new atom recording all dependencies it was set up with. */ + KBASE_TLSTREAM_TL_NEW_ATOM( + katom, + kbase_jd_atom_id(kctx, katom)); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_IDLE); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(katom, katom->sched_priority); + KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); + for (i = 0; i < 2; i++) + if (BASE_JD_DEP_TYPE_INVALID != kbase_jd_katom_dep_type( + &katom->dep[i])) { + KBASE_TLSTREAM_TL_DEP_ATOM_ATOM( + (void *)kbase_jd_katom_dep_atom( + &katom->dep[i]), + (void *)katom); + } else if (BASE_JD_DEP_TYPE_INVALID != + user_atom->pre_dep[i].dependency_type) { + /* Resolved dependency. */ + int dep_atom_number = + user_atom->pre_dep[i].atom_id; + struct kbase_jd_atom *dep_atom = + &jctx->atoms[dep_atom_number]; + + KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM( + (void *)dep_atom, + (void *)katom); + } + + /* Reject atoms with job chain = NULL, as these cause issues with soft-stop */ + if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { + dev_warn(kctx->kbdev->dev, "Rejecting atom with jc = NULL"); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + ret = jd_done_nolock(katom, NULL); + goto out; + } + + /* Reject atoms with an invalid device_nr */ + if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) && + (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) { + dev_warn(kctx->kbdev->dev, + "Rejecting atom with invalid device_nr %d", + katom->device_nr); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + ret = jd_done_nolock(katom, NULL); + goto out; + } + + /* Reject atoms with invalid core requirements */ + if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && + (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) { + dev_warn(kctx->kbdev->dev, + "Rejecting atom with invalid core requirements"); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE; + ret = jd_done_nolock(katom, NULL); + goto out; + } + + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + /* handle what we need to do to access the external resources */ + if (kbase_jd_pre_external_resources(katom, user_atom) != 0) { + /* setup failed (no access, bad resource, unknown resource types, etc.) */ + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + ret = jd_done_nolock(katom, NULL); + goto out; + } + } + + /* Validate the atom. Function will return error if the atom is + * malformed. + * + * Soft-jobs never enter the job scheduler but have their own initialize method. + * + * If either fail then we immediately complete the atom with an error. + */ + if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) { + if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + ret = jd_done_nolock(katom, NULL); + goto out; + } + } else { + /* Soft-job */ + if (kbase_prepare_soft_job(katom) != 0) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + ret = jd_done_nolock(katom, NULL); + goto out; + } + } + +#ifdef CONFIG_GPU_TRACEPOINTS + katom->work_id = atomic_inc_return(&jctx->work_id); + trace_gpu_job_enqueue(kctx->id, katom->work_id, + kbasep_map_core_reqs_to_string(katom->core_req)); +#endif + + if (queued && !IS_GPU_ATOM(katom)) { + ret = false; + goto out; + } +#ifdef CONFIG_KDS + if (!katom->kds_dep_satisfied) { + /* Queue atom due to KDS dependency */ + ret = false; + goto out; + } +#endif /* CONFIG_KDS */ + + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (kbase_fence_dep_count_read(katom) != -1) { + ret = false; + goto out; + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + + if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) + == BASE_JD_REQ_SOFT_REPLAY) { + if (kbase_replay_process(katom)) + ret = false; + else + ret = jd_done_nolock(katom, NULL); + + goto out; + } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { + if (kbase_process_soft_job(katom) == 0) { + kbase_finish_soft_job(katom); + ret = jd_done_nolock(katom, NULL); + goto out; + } + + ret = false; + } else if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { + katom->status = KBASE_JD_ATOM_STATE_IN_JS; + ret = kbasep_js_add_job(kctx, katom); + /* If job was cancelled then resolve immediately */ + if (katom->event_code == BASE_JD_EVENT_JOB_CANCELLED) + ret = jd_done_nolock(katom, NULL); + } else { + /* This is a pure dependency. Resolve it immediately */ + ret = jd_done_nolock(katom, NULL); + } + + out: + return ret; +} + +int kbase_jd_submit(struct kbase_context *kctx, + void __user *user_addr, u32 nr_atoms, u32 stride, + bool uk6_atom) +{ + struct kbase_jd_context *jctx = &kctx->jctx; + int err = 0; + int i; + bool need_to_try_schedule_context = false; + struct kbase_device *kbdev; + u32 latest_flush; + + /* + * kbase_jd_submit isn't expected to fail and so all errors with the + * jobs are reported by immediately failing them (through event system) + */ + kbdev = kctx->kbdev; + + beenthere(kctx, "%s", "Enter"); + + if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it"); + return -EINVAL; + } + + if (stride != sizeof(base_jd_atom_v2)) { + dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel"); + return -EINVAL; + } + + KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(nr_atoms, + &kctx->timeline.jd_atoms_in_flight)); + + /* All atoms submitted in this call have the same flush ID */ + latest_flush = kbase_backend_get_current_flush_id(kbdev); + + for (i = 0; i < nr_atoms; i++) { + struct base_jd_atom_v2 user_atom; + struct kbase_jd_atom *katom; + + if (copy_from_user(&user_atom, user_addr, + sizeof(user_atom)) != 0) { + err = -EINVAL; + KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, + atomic_sub_return(nr_atoms - i, + &kctx->timeline.jd_atoms_in_flight)); + break; + } + +#ifdef BASE_LEGACY_UK10_2_SUPPORT + if (KBASE_API_VERSION(10, 3) > kctx->api_version) + user_atom.core_req = (u32)(user_atom.compat_core_req + & 0x7fff); +#endif /* BASE_LEGACY_UK10_2_SUPPORT */ + + user_addr = (void __user *)((uintptr_t) user_addr + stride); + + mutex_lock(&jctx->lock); +#ifndef compiletime_assert +#define compiletime_assert_defined +#define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \ +while (false) +#endif + compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) == + BASE_JD_ATOM_COUNT, + "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); + compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) == + sizeof(user_atom.atom_number), + "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); +#ifdef compiletime_assert_defined +#undef compiletime_assert +#undef compiletime_assert_defined +#endif + katom = &jctx->atoms[user_atom.atom_number]; + + /* Record the flush ID for the cache flush optimisation */ + katom->flush_id = latest_flush; + + while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) { + /* Atom number is already in use, wait for the atom to + * complete + */ + mutex_unlock(&jctx->lock); + + /* This thread will wait for the atom to complete. Due + * to thread scheduling we are not sure that the other + * thread that owns the atom will also schedule the + * context, so we force the scheduler to be active and + * hence eventually schedule this context at some point + * later. + */ + kbase_js_sched_all(kbdev); + + if (wait_event_killable(katom->completed, + katom->status == + KBASE_JD_ATOM_STATE_UNUSED) != 0) { + /* We're being killed so the result code + * doesn't really matter + */ + return 0; + } + mutex_lock(&jctx->lock); + } + + need_to_try_schedule_context |= + jd_submit_atom(kctx, &user_atom, katom); + + /* Register a completed job as a disjoint event when the GPU is in a disjoint state + * (ie. being reset or replaying jobs). + */ + kbase_disjoint_event_potential(kbdev); + + mutex_unlock(&jctx->lock); + } + + if (need_to_try_schedule_context) + kbase_js_sched_all(kbdev); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_jd_submit); + +void kbase_jd_done_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); + struct kbase_jd_context *jctx; + struct kbase_context *kctx; + struct kbasep_js_kctx_info *js_kctx_info; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + u64 cache_jc = katom->jc; + struct kbasep_js_atom_retained_state katom_retained_state; + bool context_idle; + base_jd_core_req core_req = katom->core_req; + u64 affinity = katom->affinity; + enum kbase_atom_coreref_state coreref_state = katom->coreref_state; + + /* Soft jobs should never reach this function */ + KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); + + kctx = katom->kctx; + jctx = &kctx->jctx; + kbdev = kctx->kbdev; + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); + + kbase_backend_complete_wq(kbdev, katom); + + /* + * Begin transaction on JD context and JS context + */ + mutex_lock(&jctx->lock); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_DONE); + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + /* This worker only gets called on contexts that are scheduled *in*. This is + * because it only happens in response to an IRQ from a job that was + * running. + */ + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (katom->event_code == BASE_JD_EVENT_STOPPED) { + /* Atom has been promoted to stopped */ + unsigned long flags; + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + katom->status = KBASE_JD_ATOM_STATE_IN_JS; + kbase_js_unpull(kctx, katom); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&jctx->lock); + + return; + } + + if (katom->event_code != BASE_JD_EVENT_DONE) + dev_err(kbdev->dev, + "t6xx: GPU fault 0x%02lx from job slot %d\n", + (unsigned long)katom->event_code, + katom->slot_nr); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) + kbase_as_poking_timer_release_atom(kbdev, kctx, katom); + + /* Retain state before the katom disappears */ + kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); + + context_idle = kbase_js_complete_atom_wq(kctx, katom); + + KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state)); + + kbasep_js_remove_job(kbdev, kctx, katom); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; + /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */ + jd_done_nolock(katom, &kctx->completed_jobs); + + /* katom may have been freed now, do not use! */ + + if (context_idle) { + unsigned long flags; + + context_idle = false; + mutex_lock(&js_devdata->queue_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* If kbase_sched() has scheduled this context back in then + * KCTX_ACTIVE will have been set after we marked it as + * inactive, and another pm reference will have been taken, so + * drop our reference. But do not call kbase_jm_idle_ctx(), as + * the context is active and fast-starting is allowed. + * + * If an atom has been fast-started then kctx->atoms_pulled will + * be non-zero but KCTX_ACTIVE will still be false (as the + * previous pm reference has been inherited). Do NOT drop our + * reference, as it has been re-used, and leave the context as + * active. + * + * If no new atoms have been started then KCTX_ACTIVE will still + * be false and atoms_pulled will be zero, so drop the reference + * and call kbase_jm_idle_ctx(). + * + * As the checks are done under both the queue_mutex and + * hwaccess_lock is should be impossible for this to race + * with the scheduler code. + */ + if (kbase_ctx_flag(kctx, KCTX_ACTIVE) || + !atomic_read(&kctx->atoms_pulled)) { + /* Calling kbase_jm_idle_ctx() here will ensure that + * atoms are not fast-started when we drop the + * hwaccess_lock. This is not performed if + * KCTX_ACTIVE is set as in that case another pm + * reference has been taken and a fast-start would be + * valid. + */ + if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) + kbase_jm_idle_ctx(kbdev, kctx); + context_idle = true; + } else { + kbase_ctx_flag_set(kctx, KCTX_ACTIVE); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->queue_mutex); + } + + /* + * Transaction complete + */ + mutex_unlock(&jctx->lock); + + /* Job is now no longer running, so can now safely release the context + * reference, and handle any actions that were logged against the atom's retained state */ + + kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); + + kbase_js_sched_all(kbdev); + + if (!atomic_dec_return(&kctx->work_count)) { + /* If worker now idle then post all events that jd_done_nolock() + * has queued */ + mutex_lock(&jctx->lock); + while (!list_empty(&kctx->completed_jobs)) { + struct kbase_jd_atom *atom = list_entry( + kctx->completed_jobs.next, + struct kbase_jd_atom, jd_item); + list_del(kctx->completed_jobs.next); + + kbase_event_post(kctx, atom); + } + mutex_unlock(&jctx->lock); + } + + kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, + coreref_state); + + if (context_idle) + kbase_pm_context_idle(kbdev); + + KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); +} + +/** + * jd_cancel_worker - Work queue job cancel function. + * @data: a &struct work_struct + * + * Only called as part of 'Zapping' a context (which occurs on termination). + * Operates serially with the kbase_jd_done_worker() on the work queue. + * + * This can only be called on contexts that aren't scheduled. + * + * We don't need to release most of the resources that would occur on + * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be + * running (by virtue of only being called on contexts that aren't + * scheduled). + */ +static void jd_cancel_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); + struct kbase_jd_context *jctx; + struct kbase_context *kctx; + struct kbasep_js_kctx_info *js_kctx_info; + bool need_to_try_schedule_context; + bool attr_state_changed; + struct kbase_device *kbdev; + + /* Soft jobs should never reach this function */ + KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); + + kctx = katom->kctx; + kbdev = kctx->kbdev; + jctx = &kctx->jctx; + js_kctx_info = &kctx->jctx.sched_info; + + KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); + + /* This only gets called on contexts that are scheduled out. Hence, we must + * make sure we don't de-ref the number of running jobs (there aren't + * any), nor must we try to schedule out the context (it's already + * scheduled out). + */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + /* Scheduler: Remove the job from the system */ + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + mutex_lock(&jctx->lock); + + need_to_try_schedule_context = jd_done_nolock(katom, NULL); + /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to + * schedule the context. There's also no need for the jsctx_mutex to have been taken + * around this too. */ + KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); + + /* katom may have been freed now, do not use! */ + mutex_unlock(&jctx->lock); + + if (attr_state_changed) + kbase_js_sched_all(kbdev); +} + +/** + * kbase_jd_done - Complete a job that has been removed from the Hardware + * @katom: atom which has been completed + * @slot_nr: slot the atom was on + * @end_timestamp: completion time + * @done_code: completion code + * + * This must be used whenever a job has been removed from the Hardware, e.g.: + * An IRQ indicates that the job finished (for both error and 'done' codes), or + * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop. + * + * Some work is carried out immediately, and the rest is deferred onto a + * workqueue + * + * Context: + * This can be called safely from atomic context. + * The caller must hold kbdev->hwaccess_lock + */ +void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, + ktime_t *end_timestamp, kbasep_js_atom_done_code done_code) +{ + struct kbase_context *kctx; + struct kbase_device *kbdev; + + KBASE_DEBUG_ASSERT(katom); + kctx = katom->kctx; + KBASE_DEBUG_ASSERT(kctx); + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev); + + if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) + katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; + + KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0); + + kbase_job_check_leave_disjoint(kbdev, katom); + + katom->slot_nr = slot_nr; + + atomic_inc(&kctx->work_count); + +#ifdef CONFIG_DEBUG_FS + /* a failed job happened and is waiting for dumping*/ + if (!katom->will_fail_event_code && + kbase_debug_job_fault_process(katom, katom->event_code)) + return; +#endif + + WARN_ON(work_pending(&katom->work)); + KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); + INIT_WORK(&katom->work, kbase_jd_done_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + +KBASE_EXPORT_TEST_API(kbase_jd_done); + +void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(NULL != katom); + kctx = katom->kctx; + KBASE_DEBUG_ASSERT(NULL != kctx); + + KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); + + /* This should only be done from a context that is not scheduled */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + WARN_ON(work_pending(&katom->work)); + + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); + INIT_WORK(&katom->work, jd_cancel_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + + +void kbase_jd_zap_context(struct kbase_context *kctx) +{ + struct kbase_jd_atom *katom; + struct list_head *entry, *tmp; + struct kbase_device *kbdev; + + KBASE_DEBUG_ASSERT(kctx); + + kbdev = kctx->kbdev; + + KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); + + kbase_js_zap_context(kctx); + + mutex_lock(&kctx->jctx.lock); + + /* + * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are + * queued outside the job scheduler. + */ + + del_timer_sync(&kctx->soft_job_timeout); + list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { + katom = list_entry(entry, struct kbase_jd_atom, queue); + kbase_cancel_soft_job(katom); + } + + +#ifdef CONFIG_KDS + + /* For each job waiting on a kds resource, cancel the wait and force the job to + * complete early, this is done so that we don't leave jobs outstanding waiting + * on kds resources which may never be released when contexts are zapped, resulting + * in a hang. + * + * Note that we can safely iterate over the list as the struct kbase_jd_context lock is held, + * this prevents items being removed when calling job_done_nolock in kbase_cancel_kds_wait_job. + */ + + list_for_each(entry, &kctx->waiting_kds_resource) { + katom = list_entry(entry, struct kbase_jd_atom, node); + + kbase_cancel_kds_wait_job(katom); + } +#endif + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + kbase_dma_fence_cancel_all_atoms(kctx); +#endif + + mutex_unlock(&kctx->jctx.lock); + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + /* Flush dma-fence workqueue to ensure that any callbacks that may have + * been queued are done before continuing. + */ + flush_workqueue(kctx->dma_fence.wq); +#endif + + kbase_jm_wait_for_zero_jobs(kctx); +} + +KBASE_EXPORT_TEST_API(kbase_jd_zap_context); + +int kbase_jd_init(struct kbase_context *kctx) +{ + int i; + int mali_err = 0; +#ifdef CONFIG_KDS + int err; +#endif /* CONFIG_KDS */ + + KBASE_DEBUG_ASSERT(kctx); + + kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", + WQ_HIGHPRI | WQ_UNBOUND, 1); + if (NULL == kctx->jctx.job_done_wq) { + mali_err = -ENOMEM; + goto out1; + } + + for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { + init_waitqueue_head(&kctx->jctx.atoms[i].completed); + + INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]); + INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]); + + /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */ + kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; + kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; + +#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + kctx->jctx.atoms[i].dma_fence.context = + dma_fence_context_alloc(1); + atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0); + INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks); +#endif + } + + mutex_init(&kctx->jctx.lock); + + init_waitqueue_head(&kctx->jctx.zero_jobs_wait); + + spin_lock_init(&kctx->jctx.tb_lock); + +#ifdef CONFIG_KDS + err = kds_callback_init(&kctx->jctx.kds_cb, 0, kds_dep_clear); + if (0 != err) { + mali_err = -EINVAL; + goto out2; + } +#endif /* CONFIG_KDS */ + + kctx->jctx.job_nr = 0; + INIT_LIST_HEAD(&kctx->completed_jobs); + atomic_set(&kctx->work_count, 0); + + return 0; + +#ifdef CONFIG_KDS + out2: + destroy_workqueue(kctx->jctx.job_done_wq); +#endif /* CONFIG_KDS */ + out1: + return mali_err; +} + +KBASE_EXPORT_TEST_API(kbase_jd_init); + +void kbase_jd_exit(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx); + +#ifdef CONFIG_KDS + kds_callback_term(&kctx->jctx.kds_cb); +#endif /* CONFIG_KDS */ + /* Work queue is emptied by this */ + destroy_workqueue(kctx->jctx.job_done_wq); +} + +KBASE_EXPORT_TEST_API(kbase_jd_exit); diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.c new file mode 100755 index 000000000000..fed4ad5816ab --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.c @@ -0,0 +1,235 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifdef CONFIG_DEBUG_FS + +#include +#include +#include +#include +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include +#endif + +struct kbase_jd_debugfs_depinfo { + u8 id; + char type; +}; + +static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, + struct seq_file *sfile) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + struct kbase_sync_fence_info info; + int res; + + switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + res = kbase_sync_fence_out_info_get(atom, &info); + if (0 == res) { + seq_printf(sfile, "Sa([%p]%d) ", + info.fence, info.status); + break; + } + case BASE_JD_REQ_SOFT_FENCE_WAIT: + res = kbase_sync_fence_in_info_get(atom, &info); + if (0 == res) { + seq_printf(sfile, "Wa([%p]%d) ", + info.fence, info.status); + break; + } + default: + break; + } +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + struct kbase_fence_cb *cb; + + if (atom->dma_fence.fence) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = atom->dma_fence.fence; +#else + struct dma_fence *fence = atom->dma_fence.fence; +#endif + + seq_printf(sfile, +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + "Sd(%u#%u: %s) ", +#else + "Sd(%llu#%u: %s) ", +#endif + fence->context, + fence->seqno, + dma_fence_is_signaled(fence) ? + "signaled" : "active"); + } + + list_for_each_entry(cb, &atom->dma_fence.callbacks, + node) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = cb->fence; +#else + struct dma_fence *fence = cb->fence; +#endif + + seq_printf(sfile, +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + "Wd(%u#%u: %s) ", +#else + "Wd(%llu#%u: %s) ", +#endif + fence->context, + fence->seqno, + dma_fence_is_signaled(fence) ? + "signaled" : "active"); + } + } +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + +} + +static void kbasep_jd_debugfs_atom_deps( + struct kbase_jd_debugfs_depinfo *deps, + struct kbase_jd_atom *atom) +{ + struct kbase_context *kctx = atom->kctx; + int i; + + for (i = 0; i < 2; i++) { + deps[i].id = (unsigned)(atom->dep[i].atom ? + kbase_jd_atom_id(kctx, atom->dep[i].atom) : 0); + + switch (atom->dep[i].dep_type) { + case BASE_JD_DEP_TYPE_INVALID: + deps[i].type = ' '; + break; + case BASE_JD_DEP_TYPE_DATA: + deps[i].type = 'D'; + break; + case BASE_JD_DEP_TYPE_ORDER: + deps[i].type = '>'; + break; + default: + deps[i].type = '?'; + break; + } + } +} +/** + * kbasep_jd_debugfs_atoms_show - Show callback for the JD atoms debugfs file. + * @sfile: The debugfs entry + * @data: Data associated with the entry + * + * This function is called to get the contents of the JD atoms debugfs file. + * This is a report of all atoms managed by kbase_jd_context.atoms + * + * Return: 0 if successfully prints data in debugfs entry file, failure + * otherwise + */ +static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) +{ + struct kbase_context *kctx = sfile->private; + struct kbase_jd_atom *atoms; + unsigned long irq_flags; + int i; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + /* Print version */ + seq_printf(sfile, "v%u\n", MALI_JD_DEBUGFS_VERSION); + + /* Print U/K API version */ + seq_printf(sfile, "ukv%u.%u\n", BASE_UK_VERSION_MAJOR, + BASE_UK_VERSION_MINOR); + + /* Print table heading */ + seq_puts(sfile, " ID, Core req, St, CR, Predeps, Start time, Additional info...\n"); + + atoms = kctx->jctx.atoms; + /* General atom states */ + mutex_lock(&kctx->jctx.lock); + /* JS-related states */ + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); + for (i = 0; i != BASE_JD_ATOM_COUNT; ++i) { + struct kbase_jd_atom *atom = &atoms[i]; + s64 start_timestamp = 0; + struct kbase_jd_debugfs_depinfo deps[2]; + + if (atom->status == KBASE_JD_ATOM_STATE_UNUSED) + continue; + + /* start_timestamp is cleared as soon as the atom leaves UNUSED state + * and set before a job is submitted to the h/w, a non-zero value means + * it is valid */ + if (ktime_to_ns(atom->start_timestamp)) + start_timestamp = ktime_to_ns( + ktime_sub(ktime_get(), atom->start_timestamp)); + + kbasep_jd_debugfs_atom_deps(deps, atom); + + seq_printf(sfile, + "%3u, %8x, %2u, %2u, %c%3u %c%3u, %20lld, ", + i, atom->core_req, atom->status, + atom->coreref_state, + deps[0].type, deps[0].id, + deps[1].type, deps[1].id, + start_timestamp); + + + kbase_jd_debugfs_fence_info(atom, sfile); + + seq_puts(sfile, "\n"); + } + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); + mutex_unlock(&kctx->jctx.lock); + + return 0; +} + + +/** + * kbasep_jd_debugfs_atoms_open - open operation for atom debugfs file + * @in: &struct inode pointer + * @file: &struct file pointer + * + * Return: file descriptor + */ +static int kbasep_jd_debugfs_atoms_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_jd_debugfs_atoms_show, in->i_private); +} + +static const struct file_operations kbasep_jd_debugfs_atoms_fops = { + .open = kbasep_jd_debugfs_atoms_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx != NULL); + + /* Expose all atoms */ + debugfs_create_file("atoms", S_IRUGO, kctx->kctx_dentry, kctx, + &kbasep_jd_debugfs_atoms_fops); + +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.h new file mode 100755 index 000000000000..fae32919b22f --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jd_debugfs.h @@ -0,0 +1,40 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_jd_debugfs.h + * Header file for job dispatcher-related entries in debugfs + */ + +#ifndef _KBASE_JD_DEBUGFS_H +#define _KBASE_JD_DEBUGFS_H + +#include + +#define MALI_JD_DEBUGFS_VERSION 2 + +/* Forward declarations */ +struct kbase_context; + +/** + * kbasep_jd_debugfs_ctx_init() - Add debugfs entries for JD system + * + * @kctx Pointer to kbase_context + */ +void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx); + +#endif /*_KBASE_JD_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.c new file mode 100755 index 000000000000..0c5c6a6f78cb --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.c @@ -0,0 +1,131 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * HW access job manager common APIs + */ + +#include +#include "mali_kbase_hwaccess_jm.h" +#include "mali_kbase_jm.h" + +/** + * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot + * @js on the active context. + * @kbdev: Device pointer + * @js: Job slot to run on + * @nr_jobs_to_submit: Number of jobs to attempt to submit + * + * Return: true if slot can still be submitted on, false if slot is now full. + */ +static bool kbase_jm_next_job(struct kbase_device *kbdev, int js, + int nr_jobs_to_submit) +{ + struct kbase_context *kctx; + int i; + + kctx = kbdev->hwaccess.active_kctx; + + if (!kctx) + return true; + + for (i = 0; i < nr_jobs_to_submit; i++) { + struct kbase_jd_atom *katom = kbase_js_pull(kctx, js); + + if (!katom) + return true; /* Context has no jobs on this slot */ + + kbase_backend_run_atom(kbdev, katom); + } + + return false; /* Slot ringbuffer should now be full */ +} + +u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask) +{ + u32 ret_mask = 0; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + while (js_mask) { + int js = ffs(js_mask) - 1; + int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js); + + if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit)) + ret_mask |= (1 << js); + + js_mask &= ~(1 << js); + } + + return ret_mask; +} + +void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!down_trylock(&js_devdata->schedule_sem)) { + kbase_jm_kick(kbdev, js_mask); + up(&js_devdata->schedule_sem); + } +} + +void kbase_jm_try_kick_all(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!down_trylock(&js_devdata->schedule_sem)) { + kbase_jm_kick_all(kbdev); + up(&js_devdata->schedule_sem); + } +} + +void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->hwaccess.active_kctx == kctx) + kbdev->hwaccess.active_kctx = NULL; +} + +struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (katom->event_code != BASE_JD_EVENT_STOPPED && + katom->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT) { + return kbase_js_complete_atom(katom, NULL); + } else { + kbase_js_unpull(katom->kctx, katom); + return NULL; + } +} + +struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, ktime_t *end_timestamp) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + return kbase_js_complete_atom(katom, end_timestamp); +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.h new file mode 100755 index 000000000000..a74ee24c8058 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_jm.h @@ -0,0 +1,110 @@ +/* + * + * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Job manager common APIs + */ + +#ifndef _KBASE_JM_H_ +#define _KBASE_JM_H_ + +/** + * kbase_jm_kick() - Indicate that there are jobs ready to run. + * @kbdev: Device pointer + * @js_mask: Mask of the job slots that can be pulled from. + * + * Caller must hold the hwaccess_lock and schedule_sem semaphore + * + * Return: Mask of the job slots that can still be submitted to. + */ +u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask); + +/** + * kbase_jm_kick_all() - Indicate that there are jobs ready to run on all job + * slots. + * @kbdev: Device pointer + * + * Caller must hold the hwaccess_lock and schedule_sem semaphore + * + * Return: Mask of the job slots that can still be submitted to. + */ +static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev) +{ + return kbase_jm_kick(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); +} + +/** + * kbase_jm_try_kick - Attempt to call kbase_jm_kick + * @kbdev: Device pointer + * @js_mask: Mask of the job slots that can be pulled from + * Context: Caller must hold hwaccess_lock + * + * If schedule_sem can be immediately obtained then this function will call + * kbase_jm_kick() otherwise it will do nothing. + */ +void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask); + +/** + * kbase_jm_try_kick_all() - Attempt to call kbase_jm_kick_all + * @kbdev: Device pointer + * Context: Caller must hold hwaccess_lock + * + * If schedule_sem can be immediately obtained then this function will call + * kbase_jm_kick_all() otherwise it will do nothing. + */ +void kbase_jm_try_kick_all(struct kbase_device *kbdev); + +/** + * kbase_jm_idle_ctx() - Mark a context as idle. + * @kbdev: Device pointer + * @kctx: Context to mark as idle + * + * No more atoms will be pulled from this context until it is marked as active + * by kbase_js_use_ctx(). + * + * The context should have no atoms currently pulled from it + * (kctx->atoms_pulled == 0). + * + * Caller must hold the hwaccess_lock + */ +void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * kbase_jm_return_atom_to_js() - Return an atom to the job scheduler that has + * been soft-stopped or will fail due to a + * dependency + * @kbdev: Device pointer + * @katom: Atom that has been stopped or will be failed + * + * Return: Atom that has now been unblocked and can now be run, or NULL if none + */ +struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_jm_complete() - Complete an atom + * @kbdev: Device pointer + * @katom: Atom that has completed + * @end_timestamp: Timestamp of atom completion + * + * Return: Atom that has now been unblocked and can now be run, or NULL if none + */ +struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, ktime_t *end_timestamp); + +#endif /* _KBASE_JM_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.c new file mode 100755 index 000000000000..677e438aedfa --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.c @@ -0,0 +1,2798 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Job Scheduler Implementation + */ +#include +#include +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) +#include +#endif +#include +#include +#include + +#include +#include + +#include "mali_kbase_jm.h" +#include "mali_kbase_hwaccess_jm.h" + +/* + * Private types + */ + +/* Bitpattern indicating the result of releasing a context */ +enum { + /* The context was descheduled - caller should try scheduling in a new + * one to keep the runpool full */ + KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), + /* Ctx attributes were changed - caller should try scheduling all + * contexts */ + KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) +}; + +typedef u32 kbasep_js_release_result; + +const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { + KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ + KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ + KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */ +}; + +const base_jd_prio +kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { + BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ + BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ + BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ +}; + + +/* + * Private function prototypes + */ +static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( + struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state); + +static int kbase_js_get_slot(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, + kbasep_js_ctx_job_cb callback); + +/* Helper for trace subcodes */ +#if KBASE_TRACE_ENABLE +static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + return atomic_read(&kctx->refcount); +} +#else /* KBASE_TRACE_ENABLE */ +static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + CSTD_UNUSED(kbdev); + CSTD_UNUSED(kctx); + return 0; +} +#endif /* KBASE_TRACE_ENABLE */ + +/* + * Private functions + */ + +/** + * core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements + * @features: JSn_FEATURE register value + * + * Given a JSn_FEATURE register value returns the core requirements that match + * + * Return: Core requirement bit mask + */ +static base_jd_core_req core_reqs_from_jsn_features(u16 features) +{ + base_jd_core_req core_req = 0u; + + if ((features & JS_FEATURE_SET_VALUE_JOB) != 0) + core_req |= BASE_JD_REQ_V; + + if ((features & JS_FEATURE_CACHE_FLUSH_JOB) != 0) + core_req |= BASE_JD_REQ_CF; + + if ((features & JS_FEATURE_COMPUTE_JOB) != 0) + core_req |= BASE_JD_REQ_CS; + + if ((features & JS_FEATURE_TILER_JOB) != 0) + core_req |= BASE_JD_REQ_T; + + if ((features & JS_FEATURE_FRAGMENT_JOB) != 0) + core_req |= BASE_JD_REQ_FS; + + return core_req; +} + +static void kbase_js_sync_timers(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->js_data.runpool_mutex); + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&kbdev->js_data.runpool_mutex); +} + +/* Hold the mmu_hw_mutex and hwaccess_lock for this */ +bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + bool result = false; + int as_nr; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + as_nr = kctx->as_nr; + if (atomic_read(&kctx->refcount) > 0) { + KBASE_DEBUG_ASSERT(as_nr >= 0); + + kbase_ctx_sched_retain_ctx_refcount(kctx); + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx, + NULL, 0u, atomic_read(&kctx->refcount)); + result = true; + } + + return result; +} + +/** + * jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * @prio: Priority to check. + * + * Return true if there are no atoms to pull. There may be running atoms in the + * ring buffer even if there are no atoms to pull. It is also possible for the + * ring buffer to be full (with running atoms) when this functions returns + * true. + * + * Return: true if there are no atoms to pull, false otherwise. + */ +static inline bool +jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio) +{ + struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + return RB_EMPTY_ROOT(&rb->runnable_tree); +} + +/** + * jsctx_rb_none_to_pull(): - Check if all priority ring buffers have no + * pullable atoms + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * + * Caller must hold hwaccess_lock + * + * Return: true if the ring buffers for all priorities have no pullable atoms, + * false otherwise. + */ +static inline bool +jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) +{ + int prio; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { + if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) + return false; + } + + return true; +} + +/** + * jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue. + * @kctx: Pointer to kbase context with the queue. + * @js: Job slot id to iterate. + * @prio: Priority id to iterate. + * @callback: Function pointer to callback. + * + * Iterate over a queue and invoke @callback for each entry in the queue, and + * remove the entry from the queue. + * + * If entries are added to the queue while this is running those entries may, or + * may not be covered. To ensure that all entries in the buffer have been + * enumerated when this function returns jsctx->lock must be held when calling + * this function. + * + * The HW access lock must always be held when calling this function. + */ +static void +jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, + kbasep_js_ctx_job_cb callback) +{ + struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + while (!RB_EMPTY_ROOT(&queue->runnable_tree)) { + struct rb_node *node = rb_first(&queue->runnable_tree); + struct kbase_jd_atom *entry = rb_entry(node, + struct kbase_jd_atom, runnable_tree_node); + + rb_erase(node, &queue->runnable_tree); + callback(kctx->kbdev, entry); + } + + while (!list_empty(&queue->x_dep_head)) { + struct kbase_jd_atom *entry = list_entry(queue->x_dep_head.next, + struct kbase_jd_atom, queue); + + list_del(queue->x_dep_head.next); + + callback(kctx->kbdev, entry); + } +} + +/** + * jsctx_queue_foreach(): - Execute callback for each entry in every queue + * @kctx: Pointer to kbase context with queue. + * @js: Job slot id to iterate. + * @callback: Function pointer to callback. + * + * Iterate over all the different priorities, and for each call + * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback + * for each entry, and remove the entry from the queue. + */ +static inline void +jsctx_queue_foreach(struct kbase_context *kctx, int js, + kbasep_js_ctx_job_cb callback) +{ + int prio; + + for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) + jsctx_queue_foreach_prio(kctx, js, prio, callback); +} + +/** + * jsctx_rb_peek_prio(): - Check buffer and get next atom + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * @prio: Priority id to check. + * + * Check the ring buffer for the specified @js and @prio and return a pointer to + * the next atom, unless the ring buffer is empty. + * + * Return: Pointer to next atom in buffer, or NULL if there is no atom. + */ +static inline struct kbase_jd_atom * +jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) +{ + struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + struct rb_node *node; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + node = rb_first(&rb->runnable_tree); + if (!node) + return NULL; + + return rb_entry(node, struct kbase_jd_atom, runnable_tree_node); +} + +/** + * jsctx_rb_peek(): - Check all priority buffers and get next atom + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * + * Check the ring buffers for all priorities, starting from + * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a + * pointer to the next atom, unless all the priority's ring buffers are empty. + * + * Caller must hold the hwaccess_lock. + * + * Return: Pointer to next atom in buffer, or NULL if there is no atom. + */ +static inline struct kbase_jd_atom * +jsctx_rb_peek(struct kbase_context *kctx, int js) +{ + int prio; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { + struct kbase_jd_atom *katom; + + katom = jsctx_rb_peek_prio(kctx, js, prio); + if (katom) + return katom; + } + + return NULL; +} + +/** + * jsctx_rb_pull(): - Mark atom in list as running + * @kctx: Pointer to kbase context with ring buffer. + * @katom: Pointer to katom to pull. + * + * Mark an atom previously obtained from jsctx_rb_peek() as running. + * + * @katom must currently be at the head of the ring buffer. + */ +static inline void +jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + /* Atoms must be pulled in the correct order. */ + WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio)); + + rb_erase(&katom->runnable_tree_node, &rb->runnable_tree); +} + +#define LESS_THAN_WRAP(a, b) ((s32)(a - b) < 0) + +static void +jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; + struct rb_node **new = &(queue->runnable_tree.rb_node), *parent = NULL; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + while (*new) { + struct kbase_jd_atom *entry = container_of(*new, + struct kbase_jd_atom, runnable_tree_node); + + parent = *new; + if (LESS_THAN_WRAP(katom->age, entry->age)) + new = &((*new)->rb_left); + else + new = &((*new)->rb_right); + } + + /* Add new node and rebalance tree. */ + rb_link_node(&katom->runnable_tree_node, parent, new); + rb_insert_color(&katom->runnable_tree_node, &queue->runnable_tree); +} + +/** + * jsctx_rb_unpull(): - Undo marking of atom in list as running + * @kctx: Pointer to kbase context with ring buffer. + * @katom: Pointer to katom to unpull. + * + * Undo jsctx_rb_pull() and put @katom back in the queue. + * + * jsctx_rb_unpull() must be called on atoms in the same order the atoms were + * pulled. + */ +static inline void +jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + jsctx_tree_add(kctx, katom); +} + +static bool kbase_js_ctx_pullable(struct kbase_context *kctx, + int js, + bool is_scheduled); +static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js); +static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js); + +/* + * Functions private to KBase ('Protected' functions) + */ +int kbasep_js_devdata_init(struct kbase_device * const kbdev) +{ + struct kbasep_js_device_data *jsdd; + int i; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + jsdd = &kbdev->js_data; + +#ifdef CONFIG_MALI_BIFROST_DEBUG + /* Soft-stop will be disabled on a single context by default unless + * softstop_always is set */ + jsdd->softstop_always = false; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + jsdd->nr_all_contexts_running = 0; + jsdd->nr_user_contexts_running = 0; + jsdd->nr_contexts_pullable = 0; + atomic_set(&jsdd->nr_contexts_runnable, 0); + /* No ctx allowed to submit */ + jsdd->runpool_irq.submit_allowed = 0u; + memset(jsdd->runpool_irq.ctx_attr_ref_count, 0, + sizeof(jsdd->runpool_irq.ctx_attr_ref_count)); + memset(jsdd->runpool_irq.slot_affinities, 0, + sizeof(jsdd->runpool_irq.slot_affinities)); + memset(jsdd->runpool_irq.slot_affinity_refcount, 0, + sizeof(jsdd->runpool_irq.slot_affinity_refcount)); + INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list); + + /* Config attributes */ + jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; + jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; + jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) + jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS_8408; + else + jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; + jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; + jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) + jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS_8408; + else + jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; + jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; + jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING; + jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS; + atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT); + + dev_dbg(kbdev->dev, "JS Config Attribs: "); + dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u", + jsdd->scheduling_period_ns); + dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u", + jsdd->soft_stop_ticks); + dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u", + jsdd->soft_stop_ticks_cl); + dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u", + jsdd->hard_stop_ticks_ss); + dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u", + jsdd->hard_stop_ticks_cl); + dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u", + jsdd->hard_stop_ticks_dumping); + dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u", + jsdd->gpu_reset_ticks_ss); + dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u", + jsdd->gpu_reset_ticks_cl); + dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u", + jsdd->gpu_reset_ticks_dumping); + dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u", + jsdd->ctx_timeslice_ns); + dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i", + atomic_read(&jsdd->soft_job_timeout_ms)); + + if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss && + jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss && + jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping && + jsdd->hard_stop_ticks_dumping < + jsdd->gpu_reset_ticks_dumping)) { + dev_err(kbdev->dev, "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n"); + return -EINVAL; + } + +#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS + dev_dbg(kbdev->dev, "Job Scheduling Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.", + jsdd->soft_stop_ticks, + jsdd->scheduling_period_ns); +#endif +#if KBASE_DISABLE_SCHEDULING_HARD_STOPS + dev_dbg(kbdev->dev, "Job Scheduling Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.", + jsdd->hard_stop_ticks_ss, + jsdd->hard_stop_ticks_dumping, + jsdd->scheduling_period_ns); +#endif +#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS + dev_dbg(kbdev->dev, "Note: The JS tick timer (if coded) will still be run, but do nothing."); +#endif + + for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) + jsdd->js_reqs[i] = core_reqs_from_jsn_features( + kbdev->gpu_props.props.raw_props.js_features[i]); + + /* On error, we could continue on: providing none of the below resources + * rely on the ones above */ + + mutex_init(&jsdd->runpool_mutex); + mutex_init(&jsdd->queue_mutex); + spin_lock_init(&kbdev->hwaccess_lock); + sema_init(&jsdd->schedule_sem, 1); + + for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { + INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i]); + INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i]); + } + + return 0; +} + +void kbasep_js_devdata_halt(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +void kbasep_js_devdata_term(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata; + s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { 0, }; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + js_devdata = &kbdev->js_data; + + /* The caller must de-register all contexts before calling this + */ + KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0); + KBASE_DEBUG_ASSERT(memcmp( + js_devdata->runpool_irq.ctx_attr_ref_count, + zero_ctx_attr_ref_count, + sizeof(zero_ctx_attr_ref_count)) == 0); + CSTD_UNUSED(zero_ctx_attr_ref_count); +} + +int kbasep_js_kctx_init(struct kbase_context * const kctx) +{ + struct kbase_device *kbdev; + struct kbasep_js_kctx_info *js_kctx_info; + int i, j; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) + INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]); + + js_kctx_info = &kctx->jctx.sched_info; + + js_kctx_info->ctx.nr_jobs = 0; + kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); + kbase_ctx_flag_clear(kctx, KCTX_DYING); + memset(js_kctx_info->ctx.ctx_attr_ref_count, 0, + sizeof(js_kctx_info->ctx.ctx_attr_ref_count)); + + /* Initially, the context is disabled from submission until the create + * flags are set */ + kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); + + /* On error, we could continue on: providing none of the below resources + * rely on the ones above */ + mutex_init(&js_kctx_info->ctx.jsctx_mutex); + + init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); + + for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { + for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { + INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); + kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; + } + } + + return 0; +} + +void kbasep_js_kctx_term(struct kbase_context *kctx) +{ + struct kbase_device *kbdev; + struct kbasep_js_kctx_info *js_kctx_info; + int js; + bool update_ctx_count = false; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + js_kctx_info = &kctx->jctx.sched_info; + + /* The caller must de-register all jobs before calling this */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0); + + mutex_lock(&kbdev->js_data.queue_mutex); + mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + if (kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)) { + WARN_ON(atomic_read(&kbdev->js_data.nr_contexts_runnable) <= 0); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + update_ctx_count = true; + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + } + + mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&kbdev->js_data.queue_mutex); + + if (update_ctx_count) { + mutex_lock(&kbdev->js_data.runpool_mutex); + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&kbdev->js_data.runpool_mutex); + } +} + +/** + * kbase_js_ctx_list_add_pullable_nolock - Variant of + * kbase_jd_ctx_list_add_pullable() + * where the caller must hold + * hwaccess_lock + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_pullable[js]); + + if (!kctx->slots_pullable) { + kbdev->js_data.nr_contexts_pullable++; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); + atomic_inc(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable |= (1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_add_pullable_head_nolock - Variant of + * kbase_js_ctx_list_add_pullable_head() + * where the caller must hold + * hwaccess_lock + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_pullable_head_nolock( + struct kbase_device *kbdev, struct kbase_context *kctx, int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_pullable[js]); + + if (!kctx->slots_pullable) { + kbdev->js_data.nr_contexts_pullable++; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); + atomic_inc(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable |= (1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_add_pullable_head - Add context to the head of the + * per-slot pullable context queue + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * If the context is on either the pullable or unpullable queues, then it is + * removed before being added to the head. + * + * This function should be used when a context has been scheduled, but no jobs + * can currently be pulled from it. + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + ret = kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return ret; +} + +/** + * kbase_js_ctx_list_add_unpullable_nolock - Add context to the tail of the + * per-slot unpullable context queue + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * The context must already be on the per-slot pullable queue. It will be + * removed from the pullable queue before being added to the unpullable queue. + * + * This function should be used when a context has been pulled from, and there + * are no jobs remaining on the specified slot. + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_unpullable[js]); + + if (kctx->slots_pullable == (1 << js)) { + kbdev->js_data.nr_contexts_pullable--; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable &= ~(1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_remove_nolock - Remove context from the per-slot pullable + * or unpullable context queues + * @kbdev: Device pointer + * @kctx: Context to remove from queue + * @js: Job slot to use + * + * The context must already be on one of the queues. + * + * This function should be used when a context has no jobs on the GPU, and no + * jobs remaining for the specified slot. + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])); + + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + if (kctx->slots_pullable == (1 << js)) { + kbdev->js_data.nr_contexts_pullable--; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable &= ~(1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_pop_head_nolock - Variant of kbase_js_ctx_list_pop_head() + * where the caller must hold + * hwaccess_lock + * @kbdev: Device pointer + * @js: Job slot to use + * + * Caller must hold hwaccess_lock + * + * Return: Context to use for specified slot. + * NULL if no contexts present for specified slot + */ +static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( + struct kbase_device *kbdev, + int js) +{ + struct kbase_context *kctx; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (list_empty(&kbdev->js_data.ctx_list_pullable[js])) + return NULL; + + kctx = list_entry(kbdev->js_data.ctx_list_pullable[js].next, + struct kbase_context, + jctx.sched_info.ctx.ctx_list_entry[js]); + + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + return kctx; +} + +/** + * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable + * queue. + * @kbdev: Device pointer + * @js: Job slot to use + * + * Return: Context to use for specified slot. + * NULL if no contexts present for specified slot + */ +static struct kbase_context *kbase_js_ctx_list_pop_head( + struct kbase_device *kbdev, int js) +{ + struct kbase_context *kctx; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kctx = kbase_js_ctx_list_pop_head_nolock(kbdev, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return kctx; +} + +/** + * kbase_js_ctx_pullable - Return if a context can be pulled from on the + * specified slot + * @kctx: Context pointer + * @js: Job slot to use + * @is_scheduled: true if the context is currently scheduled + * + * Caller must hold hwaccess_lock + * + * Return: true if context can be pulled from on specified slot + * false otherwise + */ +static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js, + bool is_scheduled) +{ + struct kbasep_js_device_data *js_devdata; + struct kbase_jd_atom *katom; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + js_devdata = &kctx->kbdev->js_data; + + if (is_scheduled) { + if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) + return false; + } + katom = jsctx_rb_peek(kctx, js); + if (!katom) + return false; /* No pullable atoms */ + if (kctx->blocked_js[js][katom->sched_priority]) + return false; + if (atomic_read(&katom->blocked)) + return false; /* next atom blocked */ + if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { + if (katom->x_pre_dep->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || + katom->x_pre_dep->will_fail_event_code) + return false; + if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && + kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) + return false; + } + + return true; +} + +static bool kbase_js_dep_validate(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + struct kbase_device *kbdev = kctx->kbdev; + bool ret = true; + bool has_dep = false, has_x_dep = false; + int js = kbase_js_get_slot(kbdev, katom); + int prio = katom->sched_priority; + int i; + + for (i = 0; i < 2; i++) { + struct kbase_jd_atom *dep_atom = katom->dep[i].atom; + + if (dep_atom) { + int dep_js = kbase_js_get_slot(kbdev, dep_atom); + int dep_prio = dep_atom->sched_priority; + + /* Dependent atom must already have been submitted */ + if (!(dep_atom->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { + ret = false; + break; + } + + /* Dependencies with different priorities can't + be represented in the ringbuffer */ + if (prio != dep_prio) { + ret = false; + break; + } + + if (js == dep_js) { + /* Only one same-slot dependency can be + * represented in the ringbuffer */ + if (has_dep) { + ret = false; + break; + } + /* Each dependee atom can only have one + * same-slot dependency */ + if (dep_atom->post_dep) { + ret = false; + break; + } + has_dep = true; + } else { + /* Only one cross-slot dependency can be + * represented in the ringbuffer */ + if (has_x_dep) { + ret = false; + break; + } + /* Each dependee atom can only have one + * cross-slot dependency */ + if (dep_atom->x_post_dep) { + ret = false; + break; + } + /* The dependee atom can not already be in the + * HW access ringbuffer */ + if (dep_atom->gpu_rb_state != + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + ret = false; + break; + } + /* The dependee atom can not already have + * completed */ + if (dep_atom->status != + KBASE_JD_ATOM_STATE_IN_JS) { + ret = false; + break; + } + /* Cross-slot dependencies must not violate + * PRLAM-8987 affinity restrictions */ + if (kbase_hw_has_issue(kbdev, + BASE_HW_ISSUE_8987) && + (js == 2 || dep_js == 2)) { + ret = false; + break; + } + has_x_dep = true; + } + + /* Dependency can be represented in ringbuffers */ + } + } + + /* If dependencies can be represented by ringbuffer then clear them from + * atom structure */ + if (ret) { + for (i = 0; i < 2; i++) { + struct kbase_jd_atom *dep_atom = katom->dep[i].atom; + + if (dep_atom) { + int dep_js = kbase_js_get_slot(kbdev, dep_atom); + + if ((js != dep_js) && + (dep_atom->status != + KBASE_JD_ATOM_STATE_COMPLETED) + && (dep_atom->status != + KBASE_JD_ATOM_STATE_HW_COMPLETED) + && (dep_atom->status != + KBASE_JD_ATOM_STATE_UNUSED)) { + + katom->atom_flags |= + KBASE_KATOM_FLAG_X_DEP_BLOCKED; + katom->x_pre_dep = dep_atom; + dep_atom->x_post_dep = katom; + if (kbase_jd_katom_dep_type( + &katom->dep[i]) == + BASE_JD_DEP_TYPE_DATA) + katom->atom_flags |= + KBASE_KATOM_FLAG_FAIL_BLOCKER; + } + if ((kbase_jd_katom_dep_type(&katom->dep[i]) + == BASE_JD_DEP_TYPE_DATA) && + (js == dep_js)) { + katom->pre_dep = dep_atom; + dep_atom->post_dep = katom; + } + + list_del(&katom->dep_item[i]); + kbase_jd_katom_dep_clear(&katom->dep[i]); + } + } + } + + return ret; +} + +bool kbasep_js_add_job(struct kbase_context *kctx, + struct kbase_jd_atom *atom) +{ + unsigned long flags; + struct kbasep_js_kctx_info *js_kctx_info; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + + bool enqueue_required = false; + bool timer_sync = false; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(atom != NULL); + lockdep_assert_held(&kctx->jctx.lock); + + kbdev = kctx->kbdev; + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + /* + * Begin Runpool transaction + */ + mutex_lock(&js_devdata->runpool_mutex); + + /* Refcount ctx.nr_jobs */ + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX); + ++(js_kctx_info->ctx.nr_jobs); + + /* Setup any scheduling information */ + kbasep_js_clear_job_retry_submit(atom); + + /* Lock for state available during IRQ */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbase_js_dep_validate(kctx, atom)) { + /* Dependencies could not be represented */ + --(js_kctx_info->ctx.nr_jobs); + + /* Setting atom status back to queued as it still has unresolved + * dependencies */ + atom->status = KBASE_JD_ATOM_STATE_QUEUED; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->runpool_mutex); + + goto out_unlock; + } + + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, TL_ATOM_STATE_READY); + KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, atom)); + + enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); + + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, + kbasep_js_trace_get_refcnt(kbdev, kctx)); + + /* Context Attribute Refcounting */ + kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); + + if (enqueue_required) { + if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false)) + timer_sync = kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, atom->slot_nr); + else + timer_sync = kbase_js_ctx_list_add_unpullable_nolock( + kbdev, kctx, atom->slot_nr); + } + /* If this context is active and the atom is the first on its slot, + * kick the job manager to attempt to fast-start the atom */ + if (enqueue_required && kctx == kbdev->hwaccess.active_kctx) + kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); + /* End runpool transaction */ + + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + /* A job got added while/after kbase_job_zap_context() + * was called on a non-scheduled context (e.g. KDS + * dependency resolved). Kill that job by killing the + * context. */ + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, + false); + } else if (js_kctx_info->ctx.nr_jobs == 1) { + /* Handle Refcount going from 0 to 1: schedule the + * context on the Queue */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx); + + /* Queue was updated - caller must try to + * schedule the head context */ + WARN_ON(!enqueue_required); + } + } +out_unlock: + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + mutex_unlock(&js_devdata->queue_mutex); + + return enqueue_required; +} + +void kbasep_js_remove_job(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_jd_atom *atom) +{ + struct kbasep_js_kctx_info *js_kctx_info; + struct kbasep_js_device_data *js_devdata; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(atom != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, + kbasep_js_trace_get_refcnt(kbdev, kctx)); + + /* De-refcount ctx.nr_jobs */ + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); + --(js_kctx_info->ctx.nr_jobs); +} + +bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + unsigned long flags; + struct kbasep_js_atom_retained_state katom_retained_state; + struct kbasep_js_device_data *js_devdata; + bool attr_state_changed; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(katom != NULL); + + js_devdata = &kbdev->js_data; + + kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); + kbasep_js_remove_job(kbdev, kctx, katom); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* The atom has 'finished' (will not be re-run), so no need to call + * kbasep_js_has_atom_finished(). + * + * This is because it returns false for soft-stopped atoms, but we + * want to override that, because we're cancelling an atom regardless of + * whether it was soft-stopped or not */ + attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, + &katom_retained_state); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return attr_state_changed; +} + +bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + unsigned long flags; + struct kbasep_js_device_data *js_devdata; + bool result; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + js_devdata = &kbdev->js_data; + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + return result; +} + +struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, + int as_nr) +{ + unsigned long flags; + struct kbase_context *found_kctx = NULL; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + found_kctx = kbdev->as_to_kctx[as_nr]; + + if (found_kctx != NULL) + kbase_ctx_sched_retain_ctx_refcount(found_kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return found_kctx; +} + +/** + * kbasep_js_release_result - Try running more jobs after releasing a context + * and/or atom + * + * @kbdev: The kbase_device to operate on + * @kctx: The kbase_context to operate on + * @katom_retained_state: Retained state from the atom + * @runpool_ctx_attr_change: True if the runpool context attributes have changed + * + * This collates a set of actions that must happen whilst hwaccess_lock is held. + * + * This includes running more jobs when: + * - The previously released kctx caused a ctx attribute change, + * - The released atom caused a ctx attribute change, + * - Slots were previously blocked due to affinity restrictions, + * - Submission during IRQ handling failed. + * + * Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were + * changed. The caller should try scheduling all contexts + */ +static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( + struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state, + bool runpool_ctx_attr_change) +{ + struct kbasep_js_device_data *js_devdata; + kbasep_js_release_result result = 0; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(katom_retained_state != NULL); + js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (js_devdata->nr_user_contexts_running != 0) { + bool retry_submit = false; + int retry_jobslot = 0; + + if (katom_retained_state) + retry_submit = kbasep_js_get_atom_retry_submit_slot( + katom_retained_state, &retry_jobslot); + + if (runpool_ctx_attr_change || retry_submit) { + /* A change in runpool ctx attributes might mean we can + * run more jobs than before */ + result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; + + KBASE_TRACE_ADD_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, + kctx, NULL, 0u, retry_jobslot); + } + } + return result; +} + +/* + * Internal function to release the reference on a ctx and an atom's "retained + * state", only taking the runpool and as transaction mutexes + * + * This also starts more jobs running in the case of an ctx-attribute state + * change + * + * This does none of the followup actions for scheduling: + * - It does not schedule in a new context + * - It does not requeue or handle dying contexts + * + * For those tasks, just call kbasep_js_runpool_release_ctx() instead + * + * Requires: + * - Context is scheduled in, and kctx->as_nr matches kctx_as_nr + * - Context has a non-zero refcount + * - Caller holds js_kctx_info->ctx.jsctx_mutex + * - Caller holds js_devdata->runpool_mutex + */ +static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( + struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state) +{ + unsigned long flags; + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + + kbasep_js_release_result release_result = 0u; + bool runpool_ctx_attr_change = false; + int kctx_as_nr; + int new_ref_count; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + kctx_as_nr = kctx->as_nr; + KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); + + /* + * Transaction begins on AS and runpool_irq + * + * Assert about out calling contract + */ + mutex_lock(&kbdev->pm.lock); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr); + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); + + /* Update refcount */ + kbase_ctx_sched_release_ctx(kctx); + new_ref_count = atomic_read(&kctx->refcount); + + /* Release the atom if it finished (i.e. wasn't soft-stopped) */ + if (kbasep_js_has_atom_finished(katom_retained_state)) + runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom( + kbdev, kctx, katom_retained_state); + + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u, + new_ref_count); + + if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && + !kbase_pm_is_suspending(kbdev)) { + /* Context is kept scheduled into an address space even when + * there are no jobs, in this case we have to handle the + * situation where all jobs have been evicted from the GPU and + * submission is disabled. + * + * At this point we re-enable submission to allow further jobs + * to be executed + */ + kbasep_js_set_submit_allowed(js_devdata, kctx); + } + + /* Make a set of checks to see if the context should be scheduled out. + * Note that there'll always be at least 1 reference to the context + * which was previously acquired by kbasep_js_schedule_ctx(). */ + if (new_ref_count == 1 && + (!kbasep_js_is_submit_allowed(js_devdata, kctx) || + kbdev->pm.suspending)) { + int num_slots = kbdev->gpu_props.num_job_slots; + int slot; + + /* Last reference, and we've been told to remove this context + * from the Run Pool */ + dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d", + kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, + kbasep_js_is_submit_allowed(js_devdata, kctx)); + +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) + kbase_trace_mali_mmu_as_released(kctx->as_nr); +#endif + KBASE_TLSTREAM_TL_NRET_AS_CTX(&kbdev->as[kctx->as_nr], kctx); + + kbase_backend_release_ctx_irq(kbdev, kctx); + + if (kbdev->hwaccess.active_kctx == kctx) + kbdev->hwaccess.active_kctx = NULL; + + /* Ctx Attribute handling + * + * Releasing atoms attributes must either happen before this, or + * after the KCTX_SHEDULED flag is changed, otherwise we + * double-decount the attributes + */ + runpool_ctx_attr_change |= + kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); + + /* Releasing the context and katom retained state can allow + * more jobs to run */ + release_result |= + kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, + kctx, katom_retained_state, + runpool_ctx_attr_change); + + /* + * Transaction ends on AS and runpool_irq: + * + * By this point, the AS-related data is now clear and ready + * for re-use. + * + * Since releases only occur once for each previous successful + * retain, and no more retains are allowed on this context, no + * other thread will be operating in this + * code whilst we are + */ + + /* Recalculate pullable status for all slots */ + for (slot = 0; slot < num_slots; slot++) { + if (kbase_js_ctx_pullable(kctx, slot, false)) + kbase_js_ctx_list_add_pullable_nolock(kbdev, + kctx, slot); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_backend_release_ctx_noirq(kbdev, kctx); + + mutex_unlock(&kbdev->pm.lock); + + /* Note: Don't reuse kctx_as_nr now */ + + /* Synchronize with any timers */ + kbase_backend_ctx_count_changed(kbdev); + + /* update book-keeping info */ + kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); + /* Signal any waiter that the context is not scheduled, so is + * safe for termination - once the jsctx_mutex is also dropped, + * and jobs have finished. */ + wake_up(&js_kctx_info->ctx.is_scheduled_wait); + + /* Queue an action to occur after we've dropped the lock */ + release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED | + KBASEP_JS_RELEASE_RESULT_SCHED_ALL; + } else { + kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, + katom_retained_state, runpool_ctx_attr_change); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->pm.lock); + } + + return release_result; +} + +void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_atom_retained_state katom_retained_state; + + /* Setup a dummy katom_retained_state */ + kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); + + kbasep_js_runpool_release_ctx_internal(kbdev, kctx, + &katom_retained_state); +} + +void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, bool has_pm_ref) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + /* This is called if and only if you've you've detached the context from + * the Runpool Queue, and not added it back to the Runpool + */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + /* Dying: don't requeue, but kill all jobs on the context. This + * happens asynchronously */ + dev_dbg(kbdev->dev, + "JS: ** Killing Context %p on RunPool Remove **", kctx); + kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); + } +} + +void kbasep_js_runpool_release_ctx_and_katom_retained_state( + struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + kbasep_js_release_result release_result; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, + katom_retained_state); + + /* Drop the runpool mutex to allow requeing kctx */ + mutex_unlock(&js_devdata->runpool_mutex); + + if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); + + /* Drop the jsctx_mutex to allow scheduling in a new context */ + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL) + kbase_js_sched_all(kbdev); +} + +void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_atom_retained_state katom_retained_state; + + kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); + + kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, + &katom_retained_state); +} + +/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into + * kbase_js_sched_all() */ +static void kbasep_js_runpool_release_ctx_no_schedule( + struct kbase_device *kbdev, struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + kbasep_js_release_result release_result; + struct kbasep_js_atom_retained_state katom_retained_state_struct; + struct kbasep_js_atom_retained_state *katom_retained_state = + &katom_retained_state_struct; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + kbasep_js_atom_retained_state_init_invalid(katom_retained_state); + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, + katom_retained_state); + + /* Drop the runpool mutex to allow requeing kctx */ + mutex_unlock(&js_devdata->runpool_mutex); + if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); + + /* Drop the jsctx_mutex to allow scheduling in a new context */ + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + /* NOTE: could return release_result if the caller would like to know + * whether it should schedule a new context, but currently no callers do + */ +} + +void kbase_js_set_timeouts(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbase_backend_timeouts_changed(kbdev); +} + +static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + unsigned long flags; + bool kctx_suspended = false; + int as_nr; + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + /* Pick available address space for this context */ + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + as_nr = kbase_ctx_sched_retain_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + if (as_nr == KBASEP_AS_NR_INVALID) { + as_nr = kbase_backend_find_and_release_free_address_space( + kbdev, kctx); + if (as_nr != KBASEP_AS_NR_INVALID) { + /* Attempt to retain the context again, this should + * succeed */ + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + as_nr = kbase_ctx_sched_retain_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + WARN_ON(as_nr == KBASEP_AS_NR_INVALID); + } + } + if (as_nr == KBASEP_AS_NR_INVALID) + return false; /* No address spaces currently available */ + + /* + * Atomic transaction on the Context and Run Pool begins + */ + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Check to see if context is dying due to kbase_job_zap_context() */ + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + /* Roll back the transaction so far and return */ + kbase_ctx_sched_release_ctx(kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return false; + } + + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, + 0u, + kbasep_js_trace_get_refcnt(kbdev, kctx)); + + kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); + + /* Assign context to previously chosen address space */ + if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { + /* Roll back the transaction so far and return */ + kbase_ctx_sched_release_ctx(kctx); + kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return false; + } + + kbdev->hwaccess.active_kctx = kctx; + +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) + kbase_trace_mali_mmu_as_in_use(kctx->as_nr); +#endif + KBASE_TLSTREAM_TL_RET_AS_CTX(&kbdev->as[kctx->as_nr], kctx); + + /* Cause any future waiter-on-termination to wait until the context is + * descheduled */ + wake_up(&js_kctx_info->ctx.is_scheduled_wait); + + /* Re-check for suspending: a suspend could've occurred, and all the + * contexts could've been removed from the runpool before we took this + * lock. In this case, we don't want to allow this context to run jobs, + * we just want it out immediately. + * + * The DMB required to read the suspend flag was issued recently as part + * of the hwaccess_lock locking. If a suspend occurs *after* that lock + * was taken (i.e. this condition doesn't execute), then the + * kbasep_js_suspend() code will cleanup this context instead (by virtue + * of it being called strictly after the suspend flag is set, and will + * wait for this lock to drop) */ + if (kbase_pm_is_suspending(kbdev)) { + /* Cause it to leave at some later point */ + bool retained; + + retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + KBASE_DEBUG_ASSERT(retained); + + kbasep_js_clear_submit_allowed(js_devdata, kctx); + kctx_suspended = true; + } + + /* Transaction complete */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + /* Synchronize with any timers */ + kbase_backend_ctx_count_changed(kbdev); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + /* Note: after this point, the context could potentially get scheduled + * out immediately */ + + if (kctx_suspended) { + /* Finishing forcing out the context due to a suspend. Use a + * variant of kbasep_js_runpool_release_ctx() that doesn't + * schedule a new context, to prevent a risk of recursion back + * into this function */ + kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); + return false; + } + return true; +} + +static bool kbase_js_use_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_backend_use_ctx_sched(kbdev, kctx)) { + /* Context already has ASID - mark as active */ + kbdev->hwaccess.active_kctx = kctx; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return true; /* Context already scheduled */ + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return kbasep_js_schedule_ctx(kbdev, kctx); +} + +void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_kctx_info *js_kctx_info; + struct kbasep_js_device_data *js_devdata; + bool is_scheduled; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + /* This must never be attempted whilst suspending - i.e. it should only + * happen in response to a syscall from a user-space thread */ + BUG_ON(kbase_pm_is_suspending(kbdev)); + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + /* Mark the context as privileged */ + kbase_ctx_flag_set(kctx, KCTX_PRIVILEGED); + + is_scheduled = kbase_ctx_flag(kctx, KCTX_SCHEDULED); + if (!is_scheduled) { + /* Add the context to the pullable list */ + if (kbase_js_ctx_list_add_pullable_head(kbdev, kctx, 0)) + kbase_js_sync_timers(kbdev); + + /* Fast-starting requires the jsctx_mutex to be dropped, + * because it works on multiple ctxs */ + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + /* Try to schedule the context in */ + kbase_js_sched_all(kbdev); + + /* Wait for the context to be scheduled in */ + wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, + kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + } else { + /* Already scheduled in - We need to retain it to keep the + * corresponding address space */ + kbasep_js_runpool_retain_ctx(kbdev, kctx); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + } +} +KBASE_EXPORT_TEST_API(kbasep_js_schedule_privileged_ctx); + +void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + + /* We don't need to use the address space anymore */ + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + kbase_ctx_flag_clear(kctx, KCTX_PRIVILEGED); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + /* Release the context - it will be scheduled out */ + kbasep_js_runpool_release_ctx(kbdev, kctx); + + kbase_js_sched_all(kbdev); +} +KBASE_EXPORT_TEST_API(kbasep_js_release_privileged_ctx); + +void kbasep_js_suspend(struct kbase_device *kbdev) +{ + unsigned long flags; + struct kbasep_js_device_data *js_devdata; + int i; + u16 retained = 0u; + int nr_privileged_ctx = 0; + + KBASE_DEBUG_ASSERT(kbdev); + KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev)); + js_devdata = &kbdev->js_data; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Prevent all contexts from submitting */ + js_devdata->runpool_irq.submit_allowed = 0; + + /* Retain each of the contexts, so we can cause it to leave even if it + * had no refcount to begin with */ + for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { + struct kbase_context *kctx = kbdev->as_to_kctx[i]; + + retained = retained << 1; + + if (kctx) { + kbase_ctx_sched_retain_ctx_refcount(kctx); + retained |= 1u; + /* We can only cope with up to 1 privileged context - + * the instrumented context. It'll be suspended by + * disabling instrumentation */ + if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { + ++nr_privileged_ctx; + WARN_ON(nr_privileged_ctx != 1); + } + } + } + CSTD_UNUSED(nr_privileged_ctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* De-ref the previous retain to ensure each context gets pulled out + * sometime later. */ + for (i = 0; + i < BASE_MAX_NR_AS; + ++i, retained = retained >> 1) { + struct kbase_context *kctx = kbdev->as_to_kctx[i]; + + if (retained & 1u) + kbasep_js_runpool_release_ctx(kbdev, kctx); + } + + /* Caller must wait for all Power Manager active references to be + * dropped */ +} + +void kbasep_js_resume(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata; + int js; + + KBASE_DEBUG_ASSERT(kbdev); + js_devdata = &kbdev->js_data; + KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); + + mutex_lock(&js_devdata->queue_mutex); + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + struct kbase_context *kctx, *n; + + list_for_each_entry_safe(kctx, n, + &kbdev->js_data.ctx_list_unpullable[js], + jctx.sched_info.ctx.ctx_list_entry[js]) { + struct kbasep_js_kctx_info *js_kctx_info; + unsigned long flags; + bool timer_sync = false; + + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_js_ctx_pullable(kctx, js, false)) + timer_sync = + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + } + } + mutex_unlock(&js_devdata->queue_mutex); + + /* Restart atom processing */ + kbase_js_sched_all(kbdev); + + /* JS Resume complete */ +} + +bool kbase_js_is_atom_valid(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + if ((katom->core_req & BASE_JD_REQ_FS) && + (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | + BASE_JD_REQ_T))) + return false; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987) && + (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) && + (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_T))) + return false; + + return true; +} + +static int kbase_js_get_slot(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + if (katom->core_req & BASE_JD_REQ_FS) + return 0; + + if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { + if (katom->device_nr == 1 && + kbdev->gpu_props.num_core_groups == 2) + return 2; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) + return 2; + } + + return 1; +} + +bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + bool enqueue_required; + + katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + lockdep_assert_held(&kctx->jctx.lock); + + /* If slot will transition from unpullable to pullable then add to + * pullable list */ + if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) { + enqueue_required = true; + } else { + enqueue_required = false; + } + if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || + (katom->pre_dep && (katom->pre_dep->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; + + list_add_tail(&katom->queue, &queue->x_dep_head); + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; + enqueue_required = false; + } else { + /* Check if there are lower priority jobs to soft stop */ + kbase_job_slot_ctx_priority_check_locked(kctx, katom); + + /* Add atom to ring buffer. */ + jsctx_tree_add(kctx, katom); + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; + } + + return enqueue_required; +} + +/** + * kbase_js_move_to_tree - Move atom (and any dependent atoms) to the + * runnable_tree, ready for execution + * @katom: Atom to submit + * + * It is assumed that @katom does not have KBASE_KATOM_FLAG_X_DEP_BLOCKED set, + * but is still present in the x_dep list. If @katom has a same-slot dependent + * atom then that atom (and any dependents) will also be moved. + */ +static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&katom->kctx->kbdev->hwaccess_lock); + + while (katom) { + WARN_ON(!(katom->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); + + if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { + list_del(&katom->queue); + katom->atom_flags &= + ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; + jsctx_tree_add(katom->kctx, katom); + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; + } else { + break; + } + + katom = katom->post_dep; + } +} + + +/** + * kbase_js_evict_deps - Evict dependencies of a failed atom. + * @kctx: Context pointer + * @katom: Pointer to the atom that has failed. + * @js: The job slot the katom was run on. + * @prio: Priority of the katom. + * + * Remove all post dependencies of an atom from the context ringbuffers. + * + * The original atom's event_code will be propogated to all dependent atoms. + * + * Context: Caller must hold the HW access lock + */ +static void kbase_js_evict_deps(struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js, int prio) +{ + struct kbase_jd_atom *x_dep = katom->x_post_dep; + struct kbase_jd_atom *next_katom = katom->post_dep; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + if (next_katom) { + KBASE_DEBUG_ASSERT(next_katom->status != + KBASE_JD_ATOM_STATE_HW_COMPLETED); + next_katom->will_fail_event_code = katom->event_code; + + } + + /* Has cross slot depenency. */ + if (x_dep && (x_dep->atom_flags & (KBASE_KATOM_FLAG_JSCTX_IN_TREE | + KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { + /* Remove dependency.*/ + x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; + + /* Fail if it had a data dependency. */ + if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) { + x_dep->will_fail_event_code = katom->event_code; + } + if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) + kbase_js_move_to_tree(x_dep); + } +} + +struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) +{ + struct kbase_jd_atom *katom; + struct kbasep_js_device_data *js_devdata; + struct kbase_device *kbdev; + int pulled; + + KBASE_DEBUG_ASSERT(kctx); + + kbdev = kctx->kbdev; + + js_devdata = &kbdev->js_data; + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) + return NULL; + if (kbase_pm_is_suspending(kbdev)) + return NULL; + + katom = jsctx_rb_peek(kctx, js); + if (!katom) + return NULL; + if (kctx->blocked_js[js][katom->sched_priority]) + return NULL; + if (atomic_read(&katom->blocked)) + return NULL; + + /* Due to ordering restrictions when unpulling atoms on failure, we do + * not allow multiple runs of fail-dep atoms from the same context to be + * present on the same slot */ + if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { + struct kbase_jd_atom *prev_atom = + kbase_backend_inspect_tail(kbdev, js); + + if (prev_atom && prev_atom->kctx != kctx) + return NULL; + } + + if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { + if (katom->x_pre_dep->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || + katom->x_pre_dep->will_fail_event_code) + return NULL; + if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && + kbase_backend_nr_atoms_on_slot(kbdev, js)) + return NULL; + } + + kbase_ctx_flag_set(kctx, KCTX_PULLED); + + pulled = atomic_inc_return(&kctx->atoms_pulled); + if (pulled == 1 && !kctx->slots_pullable) { + WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); + atomic_inc(&kbdev->js_data.nr_contexts_runnable); + } + atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]); + kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++; + jsctx_rb_pull(kctx, katom); + + kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + + katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF; + + katom->ticks = 0; + + return katom; +} + + +static void js_return_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; + struct kbasep_js_atom_retained_state retained_state; + int js = katom->slot_nr; + int prio = katom->sched_priority; + bool timer_sync = false; + bool context_idle = false; + unsigned long flags; + base_jd_core_req core_req = katom->core_req; + u64 affinity = katom->affinity; + enum kbase_atom_coreref_state coreref_state = katom->coreref_state; + + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(katom); + + kbase_backend_complete_wq(kbdev, katom); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) + kbase_as_poking_timer_release_atom(kbdev, kctx, katom); + + kbasep_js_atom_retained_state_copy(&retained_state, katom); + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + atomic_dec(&kctx->atoms_pulled); + atomic_dec(&kctx->atoms_pulled_slot[js]); + + atomic_dec(&katom->blocked); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kctx->atoms_pulled_slot_pri[js][katom->sched_priority]--; + + if (!atomic_read(&kctx->atoms_pulled_slot[js]) && + jsctx_rb_none_to_pull(kctx, js)) + timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); + + /* If this slot has been blocked due to soft-stopped atoms, and all + * atoms have now been processed, then unblock the slot */ + if (!kctx->atoms_pulled_slot_pri[js][prio] && + kctx->blocked_js[js][prio]) { + kctx->blocked_js[js][prio] = false; + + /* Only mark the slot as pullable if the context is not idle - + * that case is handled below */ + if (atomic_read(&kctx->atoms_pulled) && + kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + + if (!atomic_read(&kctx->atoms_pulled)) { + if (!kctx->slots_pullable) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + timer_sync = true; + } + + if (kctx->as_nr != KBASEP_AS_NR_INVALID && + !kbase_ctx_flag(kctx, KCTX_DYING)) { + int num_slots = kbdev->gpu_props.num_job_slots; + int slot; + + if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) + kbasep_js_set_submit_allowed(js_devdata, kctx); + + for (slot = 0; slot < num_slots; slot++) { + if (kbase_js_ctx_pullable(kctx, slot, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, slot); + } + } + + kbase_jm_idle_ctx(kbdev, kctx); + + context_idle = true; + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (context_idle) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + kbase_pm_context_idle(kbdev); + } + + if (timer_sync) + kbase_js_sync_timers(kbdev); + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; + kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, + &retained_state); + + kbase_js_sched_all(kbdev); + + kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, + coreref_state); +} + +void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + jsctx_rb_unpull(kctx, katom); + + WARN_ON(work_pending(&katom->work)); + + /* Block re-submission until workqueue has run */ + atomic_inc(&katom->blocked); + + kbase_job_check_leave_disjoint(kctx->kbdev, katom); + + KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); + INIT_WORK(&katom->work, js_return_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + +bool kbase_js_complete_atom_wq(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + struct kbasep_js_kctx_info *js_kctx_info; + struct kbasep_js_device_data *js_devdata; + struct kbase_device *kbdev; + unsigned long flags; + bool timer_sync = false; + int atom_slot; + bool context_idle = false; + int prio = katom->sched_priority; + + kbdev = kctx->kbdev; + atom_slot = katom->slot_nr; + + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { + context_idle = !atomic_dec_return(&kctx->atoms_pulled); + atomic_dec(&kctx->atoms_pulled_slot[atom_slot]); + kctx->atoms_pulled_slot_pri[atom_slot][prio]--; + + if (!atomic_read(&kctx->atoms_pulled) && + !kctx->slots_pullable) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + timer_sync = true; + } + + /* If this slot has been blocked due to soft-stopped atoms, and + * all atoms have now been processed, then unblock the slot */ + if (!kctx->atoms_pulled_slot_pri[atom_slot][prio] + && kctx->blocked_js[atom_slot][prio]) { + kctx->blocked_js[atom_slot][prio] = false; + if (kbase_js_ctx_pullable(kctx, atom_slot, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, atom_slot); + } + } + WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); + + if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) && + jsctx_rb_none_to_pull(kctx, atom_slot)) { + if (!list_empty( + &kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot])) + timer_sync |= kbase_js_ctx_list_remove_nolock( + kctx->kbdev, kctx, atom_slot); + } + + /* + * If submission is disabled on this context (most likely due to an + * atom failure) and there are now no atoms left in the system then + * re-enable submission so that context can be scheduled again. + */ + if (!kbasep_js_is_submit_allowed(js_devdata, kctx) && + !atomic_read(&kctx->atoms_pulled) && + !kbase_ctx_flag(kctx, KCTX_DYING)) { + int js; + + kbasep_js_set_submit_allowed(js_devdata, kctx); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + if (kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + } else if (katom->x_post_dep && + kbasep_js_is_submit_allowed(js_devdata, kctx)) { + int js; + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + if (kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + } + + /* Mark context as inactive. The pm reference will be dropped later in + * jd_done_worker(). + */ + if (context_idle) + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); + + return context_idle; +} + +struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, + ktime_t *end_timestamp) +{ + struct kbase_device *kbdev; + struct kbase_context *kctx = katom->kctx; + struct kbase_jd_atom *x_dep = katom->x_post_dep; + + kbdev = kctx->kbdev; + + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + if (katom->will_fail_event_code) + katom->event_code = katom->will_fail_event_code; + + katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; + + if (katom->event_code != BASE_JD_EVENT_DONE) { + kbase_js_evict_deps(kctx, katom, katom->slot_nr, + katom->sched_priority); + } + +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) + kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_STOP, + katom->slot_nr), NULL, 0); +#endif + + kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); + + /* Unblock cross dependency if present */ + if (x_dep && (katom->event_code == BASE_JD_EVENT_DONE || + !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) && + (x_dep->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { + bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, + false); + x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; + kbase_js_move_to_tree(x_dep); + if (!was_pullable && kbase_js_ctx_pullable(kctx, x_dep->slot_nr, + false)) + kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, + x_dep->slot_nr); + + if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) + return x_dep; + } + + return NULL; +} + +void kbase_js_sched(struct kbase_device *kbdev, int js_mask) +{ + struct kbasep_js_device_data *js_devdata; + struct kbase_context *last_active; + bool timer_sync = false; + bool ctx_waiting = false; + + js_devdata = &kbdev->js_data; + + down(&js_devdata->schedule_sem); + mutex_lock(&js_devdata->queue_mutex); + + last_active = kbdev->hwaccess.active_kctx; + + while (js_mask) { + int js; + + js = ffs(js_mask) - 1; + + while (1) { + struct kbase_context *kctx; + unsigned long flags; + bool context_idle = false; + + kctx = kbase_js_ctx_list_pop_head(kbdev, js); + + if (!kctx) { + js_mask &= ~(1 << js); + break; /* No contexts on pullable list */ + } + + if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) { + context_idle = true; + + if (kbase_pm_context_active_handle_suspend( + kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { + /* Suspend pending - return context to + * queue and stop scheduling */ + mutex_lock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + if (kbase_js_ctx_list_add_pullable_head( + kctx->kbdev, kctx, js)) + kbase_js_sync_timers(kbdev); + mutex_unlock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + up(&js_devdata->schedule_sem); + return; + } + kbase_ctx_flag_set(kctx, KCTX_ACTIVE); + } + + if (!kbase_js_use_ctx(kbdev, kctx)) { + mutex_lock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + /* Context can not be used at this time */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (kbase_js_ctx_pullable(kctx, js, false) + || kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) + timer_sync |= + kbase_js_ctx_list_add_pullable_head_nolock( + kctx->kbdev, kctx, js); + else + timer_sync |= + kbase_js_ctx_list_add_unpullable_nolock( + kctx->kbdev, kctx, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + mutex_unlock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + if (context_idle) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + kbase_pm_context_idle(kbdev); + } + + /* No more jobs can be submitted on this slot */ + js_mask &= ~(1 << js); + break; + } + mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_ctx_flag_clear(kctx, KCTX_PULLED); + + if (!kbase_jm_kick(kbdev, 1 << js)) + /* No more jobs can be submitted on this slot */ + js_mask &= ~(1 << js); + + if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { + bool pullable = kbase_js_ctx_pullable(kctx, js, + true); + + /* Failed to pull jobs - push to head of list. + * Unless this context is already 'active', in + * which case it's effectively already scheduled + * so push it to the back of the list. */ + if (pullable && kctx == last_active) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kctx->kbdev, + kctx, js); + else if (pullable) + timer_sync |= + kbase_js_ctx_list_add_pullable_head_nolock( + kctx->kbdev, + kctx, js); + else + timer_sync |= + kbase_js_ctx_list_add_unpullable_nolock( + kctx->kbdev, + kctx, js); + + /* If this context is not the active context, + * but the active context is pullable on this + * slot, then we need to remove the active + * marker to prevent it from submitting atoms in + * the IRQ handler, which would prevent this + * context from making progress. */ + if (last_active && kctx != last_active && + kbase_js_ctx_pullable( + last_active, js, true)) + ctx_waiting = true; + + if (context_idle) { + kbase_jm_idle_ctx(kbdev, kctx); + spin_unlock_irqrestore( + &kbdev->hwaccess_lock, + flags); + WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + kbase_pm_context_idle(kbdev); + } else { + spin_unlock_irqrestore( + &kbdev->hwaccess_lock, + flags); + } + mutex_unlock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + + js_mask &= ~(1 << js); + break; /* Could not run atoms on this slot */ + } + + /* Push to back of list */ + if (kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kctx->kbdev, kctx, js); + else + timer_sync |= + kbase_js_ctx_list_add_unpullable_nolock( + kctx->kbdev, kctx, js); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + } + } + + if (timer_sync) + kbase_js_sync_timers(kbdev); + + if (kbdev->hwaccess.active_kctx == last_active && ctx_waiting) + kbdev->hwaccess.active_kctx = NULL; + + mutex_unlock(&js_devdata->queue_mutex); + up(&js_devdata->schedule_sem); +} + +void kbase_js_zap_context(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; + int js; + + /* + * Critical assumption: No more submission is possible outside of the + * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs) + * whilst the struct kbase_context is terminating. + */ + + /* First, atomically do the following: + * - mark the context as dying + * - try to evict it from the queue */ + mutex_lock(&kctx->jctx.lock); + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + kbase_ctx_flag_set(kctx, KCTX_DYING); + + dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx); + + /* + * At this point we know: + * - If eviction succeeded, it was in the queue, but now no + * longer is + * - We must cancel the jobs here. No Power Manager active reference to + * release. + * - This happens asynchronously - kbase_jd_zap_context() will wait for + * those jobs to be killed. + * - If eviction failed, then it wasn't in the queue. It is one + * of the following: + * - a. it didn't have any jobs, and so is not in the Queue or + * the Run Pool (not scheduled) + * - Hence, no more work required to cancel jobs. No Power Manager + * active reference to release. + * - b. it was in the middle of a scheduling transaction (and thus must + * have at least 1 job). This can happen from a syscall or a + * kernel thread. We still hold the jsctx_mutex, and so the thread + * must be waiting inside kbasep_js_try_schedule_head_ctx(), + * before checking whether the runpool is full. That thread will + * continue after we drop the mutex, and will notice the context + * is dying. It will rollback the transaction, killing all jobs at + * the same time. kbase_jd_zap_context() will wait for those jobs + * to be killed. + * - Hence, no more work required to cancel jobs, or to release the + * Power Manager active reference. + * - c. it is scheduled, and may or may not be running jobs + * - We must cause it to leave the runpool by stopping it from + * submitting any more jobs. When it finally does leave, + * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs + * (because it is dying), release the Power Manager active reference, + * and will not requeue the context in the queue. + * kbase_jd_zap_context() will wait for those jobs to be killed. + * - Hence, work required just to make it leave the runpool. Cancelling + * jobs and releasing the Power manager active reference will be + * handled when it leaves the runpool. + */ + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + if (!list_empty( + &kctx->jctx.sched_info.ctx.ctx_list_entry[js])) + list_del_init( + &kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + } + + /* The following events require us to kill off remaining jobs + * and update PM book-keeping: + * - we evicted it correctly (it must have jobs to be in the + * Queue) + * + * These events need no action, but take this path anyway: + * - Case a: it didn't have any jobs, and was never in the Queue + * - Case b: scheduling transaction will be partially rolled- + * back (this already cancels the jobs) + */ + + KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, + kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx); + + /* Only cancel jobs when we evicted from the + * queue. No Power Manager active reference was held. + * + * Having is_dying set ensures that this kills, and + * doesn't requeue */ + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + mutex_unlock(&kctx->jctx.lock); + } else { + unsigned long flags; + bool was_retained; + + /* Case c: didn't evict, but it is scheduled - it's in the Run + * Pool */ + KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, + kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx); + + /* Disable the ctx from submitting any more jobs */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbasep_js_clear_submit_allowed(js_devdata, kctx); + + /* Retain and (later) release the context whilst it is is now + * disallowed from submitting jobs - ensures that someone + * somewhere will be removing the context later on */ + was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + + /* Since it's scheduled and we have the jsctx_mutex, it must be + * retained successfully */ + KBASE_DEBUG_ASSERT(was_retained); + + dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx); + + /* Cancel any remaining running jobs for this kctx - if any. + * Submit is disallowed which takes effect immediately, so no + * more new jobs will appear after we do this. */ + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) + kbase_job_slot_hardstop(kctx, js, NULL); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + mutex_unlock(&kctx->jctx.lock); + + dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", + kctx); + + kbasep_js_runpool_release_ctx(kbdev, kctx); + } + + KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); + + /* After this, you must wait on both the + * kbase_jd_context::zero_jobs_wait and the + * kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs + * to be destroyed, and the context to be de-scheduled (if it was on the + * runpool). + * + * kbase_jd_zap_context() will do this. */ +} + +static inline int trace_get_refcnt(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + return atomic_read(&kctx->refcount); +} + +/** + * kbase_js_foreach_ctx_job(): - Call a function on all jobs in context + * @kctx: Pointer to context. + * @callback: Pointer to function to call for each job. + * + * Call a function on all jobs belonging to a non-queued, non-running + * context, and detach the jobs from the context as it goes. + * + * Due to the locks that might be held at the time of the call, the callback + * may need to defer work on a workqueue to complete its actions (e.g. when + * cancelling jobs) + * + * Atoms will be removed from the queue, so this must only be called when + * cancelling jobs (which occurs as part of context destruction). + * + * The locking conditions on the caller are as follows: + * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex. + */ +static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, + kbasep_js_ctx_job_cb callback) +{ + struct kbase_device *kbdev; + unsigned long flags; + u32 js; + + kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, + 0u, trace_get_refcnt(kbdev, kctx)); + + /* Invoke callback on jobs on each slot in turn */ + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) + jsctx_queue_foreach(kctx, js, callback); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.h new file mode 100755 index 000000000000..ddada8e468a1 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js.h @@ -0,0 +1,925 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_js.h + * Job Scheduler APIs. + */ + +#ifndef _KBASE_JS_H_ +#define _KBASE_JS_H_ + +#include "mali_kbase_js_defs.h" +#include "mali_kbase_context.h" +#include "mali_kbase_defs.h" +#include "mali_kbase_debug.h" + +#include "mali_kbase_js_ctx_attr.h" + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_js Job Scheduler Internal APIs + * @{ + * + * These APIs are Internal to KBase. + */ + +/** + * @brief Initialize the Job Scheduler + * + * The struct kbasep_js_device_data sub-structure of \a kbdev must be zero + * initialized before passing to the kbasep_js_devdata_init() function. This is + * to give efficient error path code. + */ +int kbasep_js_devdata_init(struct kbase_device * const kbdev); + +/** + * @brief Halt the Job Scheduler. + * + * It is safe to call this on \a kbdev even if it the kbasep_js_device_data + * sub-structure was never initialized/failed initialization, to give efficient + * error-path code. + * + * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must + * be zero initialized before passing to the kbasep_js_devdata_init() + * function. This is to give efficient error path code. + * + * It is a Programming Error to call this whilst there are still kbase_context + * structures registered with this scheduler. + * + */ +void kbasep_js_devdata_halt(struct kbase_device *kbdev); + +/** + * @brief Terminate the Job Scheduler + * + * It is safe to call this on \a kbdev even if it the kbasep_js_device_data + * sub-structure was never initialized/failed initialization, to give efficient + * error-path code. + * + * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must + * be zero initialized before passing to the kbasep_js_devdata_init() + * function. This is to give efficient error path code. + * + * It is a Programming Error to call this whilst there are still kbase_context + * structures registered with this scheduler. + */ +void kbasep_js_devdata_term(struct kbase_device *kbdev); + +/** + * @brief Initialize the Scheduling Component of a struct kbase_context on the Job Scheduler. + * + * This effectively registers a struct kbase_context with a Job Scheduler. + * + * It does not register any jobs owned by the struct kbase_context with the scheduler. + * Those must be separately registered by kbasep_js_add_job(). + * + * The struct kbase_context must be zero intitialized before passing to the + * kbase_js_init() function. This is to give efficient error path code. + */ +int kbasep_js_kctx_init(struct kbase_context * const kctx); + +/** + * @brief Terminate the Scheduling Component of a struct kbase_context on the Job Scheduler + * + * This effectively de-registers a struct kbase_context from its Job Scheduler + * + * It is safe to call this on a struct kbase_context that has never had or failed + * initialization of its jctx.sched_info member, to give efficient error-path + * code. + * + * For this to work, the struct kbase_context must be zero intitialized before passing + * to the kbase_js_init() function. + * + * It is a Programming Error to call this whilst there are still jobs + * registered with this context. + */ +void kbasep_js_kctx_term(struct kbase_context *kctx); + +/** + * @brief Add a job chain to the Job Scheduler, and take necessary actions to + * schedule the context/run the job. + * + * This atomically does the following: + * - Update the numbers of jobs information + * - Add the job to the run pool if necessary (part of init_job) + * + * Once this is done, then an appropriate action is taken: + * - If the ctx is scheduled, it attempts to start the next job (which might be + * this added job) + * - Otherwise, and if this is the first job on the context, it enqueues it on + * the Policy Queue + * + * The Policy's Queue can be updated by this in the following ways: + * - In the above case that this is the first job on the context + * - If the context is high priority and the context is not scheduled, then it + * could cause the Policy to schedule out a low-priority context, allowing + * this context to be scheduled in. + * + * If the context is already scheduled on the RunPool, then adding a job to it + * is guarenteed not to update the Policy Queue. And so, the caller is + * guarenteed to not need to try scheduling a context from the Run Pool - it + * can safely assert that the result is false. + * + * It is a programming error to have more than U32_MAX jobs in flight at a time. + * + * The following locking conditions are made on the caller: + * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * - it must \em not hold hwaccess_lock (as this will be obtained internally) + * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally). + * + * @return true indicates that the Policy Queue was updated, and so the + * caller will need to try scheduling a context onto the Run Pool. + * @return false indicates that no updates were made to the Policy Queue, + * so no further action is required from the caller. This is \b always returned + * when the context is currently scheduled. + */ +bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom); + +/** + * @brief Remove a job chain from the Job Scheduler, except for its 'retained state'. + * + * Completely removing a job requires several calls: + * - kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of + * the atom + * - kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler + * - kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the + * remaining state held as part of the job having been run. + * + * In the common case of atoms completing normally, this set of actions is more optimal for spinlock purposes than having kbasep_js_remove_job() handle all of the actions. + * + * In the case of cancelling atoms, it is easier to call kbasep_js_remove_cancelled_job(), which handles all the necessary actions. + * + * It is a programming error to call this when: + * - \a atom is not a job belonging to kctx. + * - \a atom has already been removed from the Job Scheduler. + * - \a atom is still in the runpool + * + * Do not use this for removing jobs being killed by kbase_jd_cancel() - use + * kbasep_js_remove_cancelled_job() instead. + * + * The following locking conditions are made on the caller: + * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * + */ +void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *atom); + +/** + * @brief Completely remove a job chain from the Job Scheduler, in the case + * where the job chain was cancelled. + * + * This is a variant of kbasep_js_remove_job() that takes care of removing all + * of the retained state too. This is generally useful for cancelled atoms, + * which need not be handled in an optimal way. + * + * It is a programming error to call this when: + * - \a atom is not a job belonging to kctx. + * - \a atom has already been removed from the Job Scheduler. + * - \a atom is still in the runpool: + * - it is not being killed with kbasep_jd_cancel() + * + * The following locking conditions are made on the caller: + * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * - it must \em not hold the hwaccess_lock, (as this will be obtained + * internally) + * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this could be + * obtained internally) + * + * @return true indicates that ctx attributes have changed and the caller + * should call kbase_js_sched_all() to try to run more jobs + * @return false otherwise + */ +bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +/** + * @brief Refcount a context as being busy, preventing it from being scheduled + * out. + * + * @note This function can safely be called from IRQ context. + * + * The following locking conditions are made on the caller: + * - it must \em not hold mmu_hw_mutex and hwaccess_lock, because they will be + * used internally. + * + * @return value != false if the retain succeeded, and the context will not be scheduled out. + * @return false if the retain failed (because the context is being/has been scheduled out). + */ +bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * @brief Refcount a context as being busy, preventing it from being scheduled + * out. + * + * @note This function can safely be called from IRQ context. + * + * The following locks must be held by the caller: + * - mmu_hw_mutex, hwaccess_lock + * + * @return value != false if the retain succeeded, and the context will not be scheduled out. + * @return false if the retain failed (because the context is being/has been scheduled out). + */ +bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * @brief Lookup a context in the Run Pool based upon its current address space + * and ensure that is stays scheduled in. + * + * The context is refcounted as being busy to prevent it from scheduling + * out. It must be released with kbasep_js_runpool_release_ctx() when it is no + * longer required to stay scheduled in. + * + * @note This function can safely be called from IRQ context. + * + * The following locking conditions are made on the caller: + * - it must \em not hold the hwaccess_lock, because it will be used internally. + * If the hwaccess_lock is already held, then the caller should use + * kbasep_js_runpool_lookup_ctx_nolock() instead. + * + * @return a valid struct kbase_context on success, which has been refcounted as being busy. + * @return NULL on failure, indicating that no context was found in \a as_nr + */ +struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, int as_nr); + +/** + * @brief Handling the requeuing/killing of a context that was evicted from the + * policy queue or runpool. + * + * This should be used whenever handing off a context that has been evicted + * from the policy queue or the runpool: + * - If the context is not dying and has jobs, it gets re-added to the policy + * queue + * - Otherwise, it is not added + * + * In addition, if the context is dying the jobs are killed asynchronously. + * + * In all cases, the Power Manager active reference is released + * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. \a + * has_pm_ref must be set to false whenever the context was not previously in + * the runpool and does not hold a Power Manager active refcount. Note that + * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an + * active refcount even though they weren't in the runpool. + * + * The following locking conditions are made on the caller: + * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be + * obtained internally) + */ +void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, bool has_pm_ref); + +/** + * @brief Release a refcount of a context being busy, allowing it to be + * scheduled out. + * + * When the refcount reaches zero and the context \em might be scheduled out + * (depending on whether the Scheudling Policy has deemed it so, or if it has run + * out of jobs). + * + * If the context does get scheduled out, then The following actions will be + * taken as part of deschduling a context: + * - For the context being descheduled: + * - If the context is in the processing of dying (all the jobs are being + * removed from it), then descheduling also kills off any jobs remaining in the + * context. + * - If the context is not dying, and any jobs remain after descheduling the + * context then it is re-enqueued to the Policy's Queue. + * - Otherwise, the context is still known to the scheduler, but remains absent + * from the Policy Queue until a job is next added to it. + * - In all descheduling cases, the Power Manager active reference (obtained + * during kbasep_js_try_schedule_head_ctx()) is released (kbase_pm_context_idle()). + * + * Whilst the context is being descheduled, this also handles actions that + * cause more atoms to be run: + * - Attempt submitting atoms when the Context Attributes on the Runpool have + * changed. This is because the context being scheduled out could mean that + * there are more opportunities to run atoms. + * - Attempt submitting to a slot that was previously blocked due to affinity + * restrictions. This is usually only necessary when releasing a context + * happens as part of completing a previous job, but is harmless nonetheless. + * - Attempt scheduling in a new context (if one is available), and if necessary, + * running a job from that new context. + * + * Unlike retaining a context in the runpool, this function \b cannot be called + * from IRQ context. + * + * It is a programming error to call this on a \a kctx that is not currently + * scheduled, or that already has a zero refcount. + * + * The following locking conditions are made on the caller: + * - it must \em not hold the hwaccess_lock, because it will be used internally. + * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be + * obtained internally) + * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be + * obtained internally) + * + */ +void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * @brief Variant of kbasep_js_runpool_release_ctx() that handles additional + * actions from completing an atom. + * + * This is usually called as part of completing an atom and releasing the + * refcount on the context held by the atom. + * + * Therefore, the extra actions carried out are part of handling actions queued + * on a completed atom, namely: + * - Releasing the atom's context attributes + * - Retrying the submission on a particular slot, because we couldn't submit + * on that slot from an IRQ handler. + * + * The locking conditions of this function are the same as those for + * kbasep_js_runpool_release_ctx() + */ +void kbasep_js_runpool_release_ctx_and_katom_retained_state(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); + +/** + * @brief Variant of kbase_js_runpool_release_ctx() that assumes that + * kbasep_js_device_data::runpool_mutex and + * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not + * attempt to schedule new contexts. + */ +void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * @brief Schedule in a privileged context + * + * This schedules a context in regardless of the context priority. + * If the runpool is full, a context will be forced out of the runpool and the function will wait + * for the new context to be scheduled in. + * The context will be kept scheduled in (and the corresponding address space reserved) until + * kbasep_js_release_privileged_ctx is called). + * + * The following locking conditions are made on the caller: + * - it must \em not hold the hwaccess_lock, because it will be used internally. + * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be + * obtained internally) + * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally). + * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will + * be used internally. + * + */ +void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * @brief Release a privileged context, allowing it to be scheduled out. + * + * See kbasep_js_runpool_release_ctx for potential side effects. + * + * The following locking conditions are made on the caller: + * - it must \em not hold the hwaccess_lock, because it will be used internally. + * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be + * obtained internally) + * + */ +void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * @brief Try to submit the next job on each slot + * + * The following locks may be used: + * - kbasep_js_device_data::runpool_mutex + * - hwaccess_lock + */ +void kbase_js_try_run_jobs(struct kbase_device *kbdev); + +/** + * @brief Suspend the job scheduler during a Power Management Suspend event. + * + * Causes all contexts to be removed from the runpool, and prevents any + * contexts from (re)entering the runpool. + * + * This does not handle suspending the one privileged context: the caller must + * instead do this by by suspending the GPU HW Counter Instrumentation. + * + * This will eventually cause all Power Management active references held by + * contexts on the runpool to be released, without running any more atoms. + * + * The caller must then wait for all Power Mangement active refcount to become + * zero before completing the suspend. + * + * The emptying mechanism may take some time to complete, since it can wait for + * jobs to complete naturally instead of forcing them to end quickly. However, + * this is bounded by the Job Scheduler's Job Timeouts. Hence, this + * function is guaranteed to complete in a finite time. + */ +void kbasep_js_suspend(struct kbase_device *kbdev); + +/** + * @brief Resume the Job Scheduler after a Power Management Resume event. + * + * This restores the actions from kbasep_js_suspend(): + * - Schedules contexts back into the runpool + * - Resumes running atoms on the GPU + */ +void kbasep_js_resume(struct kbase_device *kbdev); + +/** + * @brief Submit an atom to the job scheduler. + * + * The atom is enqueued on the context's ringbuffer. The caller must have + * ensured that all dependencies can be represented in the ringbuffer. + * + * Caller must hold jctx->lock + * + * @param[in] kctx Context pointer + * @param[in] atom Pointer to the atom to submit + * + * @return Whether the context requires to be enqueued. */ +bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +/** + * jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer. + * @kctx: Context Pointer + * @prio: Priority (specifies the queue together with js). + * @js: Job slot (specifies the queue together with prio). + * + * Pushes all possible atoms from the linked list to the ringbuffer. + * Number of atoms are limited to free space in the ringbuffer and + * number of available atoms in the linked list. + * + */ +void jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js); +/** + * @brief Pull an atom from a context in the job scheduler for execution. + * + * The atom will not be removed from the ringbuffer at this stage. + * + * The HW access lock must be held when calling this function. + * + * @param[in] kctx Context to pull from + * @param[in] js Job slot to pull from + * @return Pointer to an atom, or NULL if there are no atoms for this + * slot that can be currently run. + */ +struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js); + +/** + * @brief Return an atom to the job scheduler ringbuffer. + * + * An atom is 'unpulled' if execution is stopped but intended to be returned to + * later. The most common reason for this is that the atom has been + * soft-stopped. + * + * Note that if multiple atoms are to be 'unpulled', they must be returned in + * the reverse order to which they were originally pulled. It is a programming + * error to return atoms in any other order. + * + * The HW access lock must be held when calling this function. + * + * @param[in] kctx Context pointer + * @param[in] atom Pointer to the atom to unpull + */ +void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom); + +/** + * @brief Complete an atom from jd_done_worker(), removing it from the job + * scheduler ringbuffer. + * + * If the atom failed then all dependee atoms marked for failure propagation + * will also fail. + * + * @param[in] kctx Context pointer + * @param[in] katom Pointer to the atom to complete + * @return true if the context is now idle (no jobs pulled) + * false otherwise + */ +bool kbase_js_complete_atom_wq(struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +/** + * @brief Complete an atom. + * + * Most of the work required to complete an atom will be performed by + * jd_done_worker(). + * + * The HW access lock must be held when calling this function. + * + * @param[in] katom Pointer to the atom to complete + * @param[in] end_timestamp The time that the atom completed (may be NULL) + * + * Return: Atom that has now been unblocked and can now be run, or NULL if none + */ +struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, + ktime_t *end_timestamp); + +/** + * @brief Submit atoms from all available contexts. + * + * This will attempt to submit as many jobs as possible to the provided job + * slots. It will exit when either all job slots are full, or all contexts have + * been used. + * + * @param[in] kbdev Device pointer + * @param[in] js_mask Mask of job slots to submit to + */ +void kbase_js_sched(struct kbase_device *kbdev, int js_mask); + +/** + * kbase_jd_zap_context - Attempt to deschedule a context that is being + * destroyed + * @kctx: Context pointer + * + * This will attempt to remove a context from any internal job scheduler queues + * and perform any other actions to ensure a context will not be submitted + * from. + * + * If the context is currently scheduled, then the caller must wait for all + * pending jobs to complete before taking any further action. + */ +void kbase_js_zap_context(struct kbase_context *kctx); + +/** + * @brief Validate an atom + * + * This will determine whether the atom can be scheduled onto the GPU. Atoms + * with invalid combinations of core requirements will be rejected. + * + * @param[in] kbdev Device pointer + * @param[in] katom Atom to validate + * @return true if atom is valid + * false otherwise + */ +bool kbase_js_is_atom_valid(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_js_set_timeouts - update all JS timeouts with user specified data + * @kbdev: Device pointer + * + * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is + * set to a positive number then that becomes the new value used, if a timeout + * is negative then the default is set. + */ +void kbase_js_set_timeouts(struct kbase_device *kbdev); + +/* + * Helpers follow + */ + +/** + * @brief Check that a context is allowed to submit jobs on this policy + * + * The purpose of this abstraction is to hide the underlying data size, and wrap up + * the long repeated line of code. + * + * As with any bool, never test the return value with true. + * + * The caller must hold hwaccess_lock. + */ +static inline bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) +{ + u16 test_bit; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + test_bit = (u16) (1u << kctx->as_nr); + + return (bool) (js_devdata->runpool_irq.submit_allowed & test_bit); +} + +/** + * @brief Allow a context to submit jobs on this policy + * + * The purpose of this abstraction is to hide the underlying data size, and wrap up + * the long repeated line of code. + * + * The caller must hold hwaccess_lock. + */ +static inline void kbasep_js_set_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) +{ + u16 set_bit; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + set_bit = (u16) (1u << kctx->as_nr); + + dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)", kctx, kctx->as_nr); + + js_devdata->runpool_irq.submit_allowed |= set_bit; +} + +/** + * @brief Prevent a context from submitting more jobs on this policy + * + * The purpose of this abstraction is to hide the underlying data size, and wrap up + * the long repeated line of code. + * + * The caller must hold hwaccess_lock. + */ +static inline void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) +{ + u16 clear_bit; + u16 clear_mask; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + clear_bit = (u16) (1u << kctx->as_nr); + clear_mask = ~clear_bit; + + dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)", kctx, kctx->as_nr); + + js_devdata->runpool_irq.submit_allowed &= clear_mask; +} + +/** + * @brief Manage the 'retry_submit_on_slot' part of a kbase_jd_atom + */ +static inline void kbasep_js_clear_job_retry_submit(struct kbase_jd_atom *atom) +{ + atom->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID; +} + +/** + * Mark a slot as requiring resubmission by carrying that information on a + * completing atom. + * + * @note This can ASSERT in debug builds if the submit slot has been set to + * something other than the current value for @a js. This is because you might + * be unintentionally stopping more jobs being submitted on the old submit + * slot, and that might cause a scheduling-hang. + * + * @note If you can guarantee that the atoms for the original slot will be + * submitted on some other slot, then call kbasep_js_clear_job_retry_submit() + * first to silence the ASSERT. + */ +static inline void kbasep_js_set_job_retry_submit_slot(struct kbase_jd_atom *atom, int js) +{ + KBASE_DEBUG_ASSERT(0 <= js && js <= BASE_JM_MAX_NR_SLOTS); + KBASE_DEBUG_ASSERT((atom->retry_submit_on_slot == + KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID) + || (atom->retry_submit_on_slot == js)); + + atom->retry_submit_on_slot = js; +} + +/** + * Create an initial 'invalid' atom retained state, that requires no + * atom-related work to be done on releasing with + * kbasep_js_runpool_release_ctx_and_katom_retained_state() + */ +static inline void kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state *retained_state) +{ + retained_state->event_code = BASE_JD_EVENT_NOT_STARTED; + retained_state->core_req = KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID; + retained_state->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID; +} + +/** + * Copy atom state that can be made available after jd_done_nolock() is called + * on that atom. + */ +static inline void kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state *retained_state, const struct kbase_jd_atom *katom) +{ + retained_state->event_code = katom->event_code; + retained_state->core_req = katom->core_req; + retained_state->retry_submit_on_slot = katom->retry_submit_on_slot; + retained_state->sched_priority = katom->sched_priority; + retained_state->device_nr = katom->device_nr; +} + +/** + * @brief Determine whether an atom has finished (given its retained state), + * and so should be given back to userspace/removed from the system. + * + * Reasons for an atom not finishing include: + * - Being soft-stopped (and so, the atom should be resubmitted sometime later) + * + * @param[in] katom_retained_state the retained state of the atom to check + * @return false if the atom has not finished + * @return !=false if the atom has finished + */ +static inline bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state) +{ + return (bool) (katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT); +} + +/** + * @brief Determine whether a struct kbasep_js_atom_retained_state is valid + * + * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates that the + * code should just ignore it. + * + * @param[in] katom_retained_state the atom's retained state to check + * @return false if the retained state is invalid, and can be ignored + * @return !=false if the retained state is valid + */ +static inline bool kbasep_js_atom_retained_state_is_valid(const struct kbasep_js_atom_retained_state *katom_retained_state) +{ + return (bool) (katom_retained_state->core_req != KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID); +} + +static inline bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_atom_retained_state *katom_retained_state, int *res) +{ + int js = katom_retained_state->retry_submit_on_slot; + + *res = js; + return (bool) (js >= 0); +} + +/** + * @brief Variant of kbasep_js_runpool_lookup_ctx() that can be used when the + * context is guaranteed to be already previously retained. + * + * It is a programming error to supply the \a as_nr of a context that has not + * been previously retained/has a busy refcount of zero. The only exception is + * when there is no ctx in \a as_nr (NULL returned). + * + * The following locking conditions are made on the caller: + * - it must \em not hold the hwaccess_lock, because it will be used internally. + * + * @return a valid struct kbase_context on success, with a refcount that is guaranteed + * to be non-zero and unmodified by this function. + * @return NULL on failure, indicating that no context was found in \a as_nr + */ +static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device *kbdev, int as_nr) +{ + struct kbase_context *found_kctx; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); + + found_kctx = kbdev->as_to_kctx[as_nr]; + KBASE_DEBUG_ASSERT(found_kctx == NULL || + atomic_read(&found_kctx->refcount) > 0); + + return found_kctx; +} + +/* + * The following locking conditions are made on the caller: + * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. + * - The caller must hold the kbasep_js_device_data::runpool_mutex + */ +static inline void kbase_js_runpool_inc_context_count( + struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + + /* Track total contexts */ + KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX); + ++(js_devdata->nr_all_contexts_running); + + if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + /* Track contexts that can submit jobs */ + KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running < + S8_MAX); + ++(js_devdata->nr_user_contexts_running); + } +} + +/* + * The following locking conditions are made on the caller: + * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. + * - The caller must hold the kbasep_js_device_data::runpool_mutex + */ +static inline void kbase_js_runpool_dec_context_count( + struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + + /* Track total contexts */ + --(js_devdata->nr_all_contexts_running); + KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0); + + if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + /* Track contexts that can submit jobs */ + --(js_devdata->nr_user_contexts_running); + KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0); + } +} + + +/** + * @brief Submit atoms from all available contexts to all job slots. + * + * This will attempt to submit as many jobs as possible. It will exit when + * either all job slots are full, or all contexts have been used. + * + * @param[in] kbdev Device pointer + */ +static inline void kbase_js_sched_all(struct kbase_device *kbdev) +{ + kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); +} + +extern const int +kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS]; + +extern const base_jd_prio +kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + +/** + * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio) + * to relative ordering + * @atom_prio: Priority ID to translate. + * + * Atom priority values for @ref base_jd_prio cannot be compared directly to + * find out which are higher or lower. + * + * This function will convert base_jd_prio values for successively lower + * priorities into a monotonically increasing sequence. That is, the lower the + * base_jd_prio priority, the higher the value produced by this function. This + * is in accordance with how the rest of the kernel treates priority. + * + * The mapping is 1:1 and the size of the valid input range is the same as the + * size of the valid output range, i.e. + * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS + * + * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions + * + * Return: On success: a value in the inclusive range + * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure: + * KBASE_JS_ATOM_SCHED_PRIO_INVALID + */ +static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio) +{ + if (atom_prio >= BASE_JD_NR_PRIO_LEVELS) + return KBASE_JS_ATOM_SCHED_PRIO_INVALID; + + return kbasep_js_atom_priority_to_relative[atom_prio]; +} + +static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio) +{ + unsigned int prio_idx; + + KBASE_DEBUG_ASSERT(0 <= sched_prio + && sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT); + + prio_idx = (unsigned int)sched_prio; + + return kbasep_js_relative_priority_to_atom[prio_idx]; +} + + /** @} *//* end group kbase_js */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_JS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.c new file mode 100755 index 000000000000..321506ada835 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.c @@ -0,0 +1,301 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +#include +#include + +/* + * Private functions follow + */ + +/** + * @brief Check whether a ctx has a certain attribute, and if so, retain that + * attribute on the runpool. + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx is scheduled on the runpool + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { + KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX); + ++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); + + if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) { + /* First refcount indicates a state change */ + runpool_state_changed = true; + KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute); + } + } + + return runpool_state_changed; +} + +/** + * @brief Check whether a ctx has a certain attribute, and if so, release that + * attribute on the runpool. + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx is scheduled on the runpool + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { + KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0); + --(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); + + if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) { + /* Last de-refcount indicates a state change */ + runpool_state_changed = true; + KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute); + } + } + + return runpool_state_changed; +} + +/** + * @brief Retain a certain attribute on a ctx, also retaining it on the runpool + * if the context is scheduled. + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + * + * @return true indicates a change in ctx attributes state of the runpool. + * This may allow the scheduler to submit more jobs than previously. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX); + + ++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); + + if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { + /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */ + KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); + runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute); + } + + return runpool_state_changed; +} + +/* + * @brief Release a certain attribute on a ctx, also releasing it from the runpool + * if the context is scheduled. + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + * + * @return true indicates a change in ctx attributes state of the runpool. + * This may allow the scheduler to submit more jobs than previously. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0); + + if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { + lockdep_assert_held(&kbdev->hwaccess_lock); + /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */ + runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute); + KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); + } + + /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */ + --(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); + + return runpool_state_changed; +} + +/* + * More commonly used public functions + */ + +void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + /* This context never submits, so don't track any scheduling attributes */ + return; + } + + /* Transfer attributes held in the context flags for contexts that have submit enabled */ + + /* ... More attributes can be added here ... */ + + /* The context should not have been scheduled yet, so ASSERT if this caused + * runpool state changes (note that other threads *can't* affect the value + * of runpool_state_changed, due to how it's calculated) */ + KBASE_DEBUG_ASSERT(runpool_state_changed == false); + CSTD_UNUSED(runpool_state_changed); +} + +void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + bool runpool_state_changed; + int i; + + /* Retain any existing attributes */ + for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { + /* The context is being scheduled in, so update the runpool with the new attributes */ + runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); + + /* We don't need to know about state changed, because retaining a + * context occurs on scheduling it, and that itself will also try + * to run new atoms */ + CSTD_UNUSED(runpool_state_changed); + } + } +} + +bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + bool runpool_state_changed = false; + int i; + + /* Release any existing attributes */ + for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { + /* The context is being scheduled out, so update the runpool on the removed attributes */ + runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); + } + } + + return runpool_state_changed; +} + +void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + bool runpool_state_changed = false; + base_jd_core_req core_req; + + KBASE_DEBUG_ASSERT(katom); + core_req = katom->core_req; + + if (core_req & BASE_JD_REQ_ONLY_COMPUTE) + runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); + else + runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); + + if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { + /* Atom that can run on slot1 or slot2, and can use all cores */ + runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); + } + + /* We don't need to know about state changed, because retaining an + * atom occurs on adding it, and that itself will also try to run + * new atoms */ + CSTD_UNUSED(runpool_state_changed); +} + +bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state) +{ + bool runpool_state_changed = false; + base_jd_core_req core_req; + + KBASE_DEBUG_ASSERT(katom_retained_state); + core_req = katom_retained_state->core_req; + + /* No-op for invalid atoms */ + if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false) + return false; + + if (core_req & BASE_JD_REQ_ONLY_COMPUTE) + runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); + else + runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); + + if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { + /* Atom that can run on slot1 or slot2, and can use all cores */ + runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); + } + + return runpool_state_changed; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.h new file mode 100755 index 000000000000..ce9183326a57 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_ctx_attr.h @@ -0,0 +1,158 @@ +/* + * + * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_js_ctx_attr.h + * Job Scheduler Context Attribute APIs + */ + +#ifndef _KBASE_JS_CTX_ATTR_H_ +#define _KBASE_JS_CTX_ATTR_H_ + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_js + * @{ + */ + +/** + * Set the initial attributes of a context (when context create flags are set) + * + * Requires: + * - Hold the jsctx_mutex + */ +void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * Retain all attributes of a context + * + * This occurs on scheduling in the context on the runpool (but after + * is_scheduled is set) + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx->is_scheduled is true + */ +void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * Release all attributes of a context + * + * This occurs on scheduling out the context from the runpool (but before + * is_scheduled is cleared) + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx->is_scheduled is true + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * Retain all attributes of an atom + * + * This occurs on adding an atom to a context + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + */ +void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); + +/** + * Release all attributes of an atom, given its retained state. + * + * This occurs after (permanently) removing an atom from a context + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + * + * This is a no-op when \a katom_retained_state is invalid. + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); + +/** + * Requires: + * - runpool_irq spinlock + */ +static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_device_data *js_devdata; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_devdata = &kbdev->js_data; + + return js_devdata->runpool_irq.ctx_attr_ref_count[attribute]; +} + +/** + * Requires: + * - runpool_irq spinlock + */ +static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) +{ + /* In general, attributes are 'on' when they have a non-zero refcount (note: the refcount will never be < 0) */ + return (bool) kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute); +} + +/** + * Requires: + * - jsctx mutex + */ +static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_kctx_info = &kctx->jctx.sched_info; + + /* In general, attributes are 'on' when they have a refcount (which should never be < 0) */ + return (bool) (js_kctx_info->ctx.ctx_attr_ref_count[attribute]); +} + + /** @} *//* end group kbase_js */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_JS_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_defs.h new file mode 100755 index 000000000000..0b4890d6b50e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_js_defs.h @@ -0,0 +1,386 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_js.h + * Job Scheduler Type Definitions + */ + +#ifndef _KBASE_JS_DEFS_H_ +#define _KBASE_JS_DEFS_H_ + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_js + * @{ + */ +/* Forward decls */ +struct kbase_device; +struct kbase_jd_atom; + + +typedef u32 kbase_context_flags; + +struct kbasep_atom_req { + base_jd_core_req core_req; + kbase_context_flags ctx_req; + u32 device_nr; +}; + +/** Callback function run on all of a context's jobs registered with the Job + * Scheduler */ +typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd_atom *katom); + +/** + * @brief Maximum number of jobs that can be submitted to a job slot whilst + * inside the IRQ handler. + * + * This is important because GPU NULL jobs can complete whilst the IRQ handler + * is running. Otherwise, it potentially allows an unlimited number of GPU NULL + * jobs to be submitted inside the IRQ handler, which increases IRQ latency. + */ +#define KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ 2 + +/** + * @brief Context attributes + * + * Each context attribute can be thought of as a boolean value that caches some + * state information about either the runpool, or the context: + * - In the case of the runpool, it is a cache of "Do any contexts owned by + * the runpool have attribute X?" + * - In the case of a context, it is a cache of "Do any atoms owned by the + * context have attribute X?" + * + * The boolean value of the context attributes often affect scheduling + * decisions, such as affinities to use and job slots to use. + * + * To accomodate changes of state in the context, each attribute is refcounted + * in the context, and in the runpool for all running contexts. Specifically: + * - The runpool holds a refcount of how many contexts in the runpool have this + * attribute. + * - The context holds a refcount of how many atoms have this attribute. + */ +enum kbasep_js_ctx_attr { + /** Attribute indicating a context that contains Compute jobs. That is, + * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE + * + * @note A context can be both 'Compute' and 'Non Compute' if it contains + * both types of jobs. + */ + KBASEP_JS_CTX_ATTR_COMPUTE, + + /** Attribute indicating a context that contains Non-Compute jobs. That is, + * the context has some jobs that are \b not of type @ref + * BASE_JD_REQ_ONLY_COMPUTE. + * + * @note A context can be both 'Compute' and 'Non Compute' if it contains + * both types of jobs. + */ + KBASEP_JS_CTX_ATTR_NON_COMPUTE, + + /** Attribute indicating that a context contains compute-job atoms that + * aren't restricted to a coherent group, and can run on all cores. + * + * Specifically, this is when the atom's \a core_req satisfy: + * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 + * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups + * + * Such atoms could be blocked from running if one of the coherent groups + * is being used by another job slot, so tracking this context attribute + * allows us to prevent such situations. + * + * @note This doesn't take into account the 1-coregroup case, where all + * compute atoms would effectively be able to run on 'all cores', but + * contexts will still not always get marked with this attribute. Instead, + * it is the caller's responsibility to take into account the number of + * coregroups when interpreting this attribute. + * + * @note Whilst Tiler atoms are normally combined with + * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without + * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy + * enough to handle anyway. + */ + KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, + + /** Must be the last in the enum */ + KBASEP_JS_CTX_ATTR_COUNT +}; + +enum { + /** Bit indicating that new atom should be started because this atom completed */ + KBASE_JS_ATOM_DONE_START_NEW_ATOMS = (1u << 0), + /** Bit indicating that the atom was evicted from the JS_NEXT registers */ + KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT = (1u << 1) +}; + +/** Combination of KBASE_JS_ATOM_DONE_<...> bits */ +typedef u32 kbasep_js_atom_done_code; + +/** + * @brief KBase Device Data Job Scheduler sub-structure + * + * This encapsulates the current context of the Job Scheduler on a particular + * device. This context is global to the device, and is not tied to any + * particular struct kbase_context running on the device. + * + * nr_contexts_running and as_free are optimized for packing together (by making + * them smaller types than u32). The operations on them should rarely involve + * masking. The use of signed types for arithmetic indicates to the compiler that + * the value will not rollover (which would be undefined behavior), and so under + * the Total License model, it is free to make optimizations based on that (i.e. + * to remove masking). + */ +struct kbasep_js_device_data { + /* Sub-structure to collect together Job Scheduling data used in IRQ + * context. The hwaccess_lock must be held when accessing. */ + struct runpool_irq { + /** Bitvector indicating whether a currently scheduled context is allowed to submit jobs. + * When bit 'N' is set in this, it indicates whether the context bound to address space + * 'N' is allowed to submit jobs. + */ + u16 submit_allowed; + + /** Context Attributes: + * Each is large enough to hold a refcount of the number of contexts + * that can fit into the runpool. This is currently BASE_MAX_NR_AS + * + * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store + * the refcount. Hence, it's not worthwhile reducing this to + * bit-manipulation on u32s to save space (where in contrast, 4 bit + * sub-fields would be easy to do and would save space). + * + * Whilst this must not become negative, the sign bit is used for: + * - error detection in debug builds + * - Optimization: it is undefined for a signed int to overflow, and so + * the compiler can optimize for that never happening (thus, no masking + * is required on updating the variable) */ + s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; + + /* + * Affinity management and tracking + */ + /** Bitvector to aid affinity checking. Element 'n' bit 'i' indicates + * that slot 'n' is using core i (i.e. slot_affinity_refcount[n][i] > 0) */ + u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; + /** Refcount for each core owned by each slot. Used to generate the + * slot_affinities array of bitvectors + * + * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, + * because it is refcounted only when a job is definitely about to be + * submitted to a slot, and is de-refcounted immediately after a job + * finishes */ + s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; + } runpool_irq; + + /** + * Run Pool mutex, for managing contexts within the runpool. + * Unless otherwise specified, you must hold this lock whilst accessing any + * members that follow + * + * In addition, this is used to access: + * - the kbasep_js_kctx_info::runpool substructure + */ + struct mutex runpool_mutex; + + /** + * Queue Lock, used to access the Policy's queue of contexts independently + * of the Run Pool. + * + * Of course, you don't need the Run Pool lock to access this. + */ + struct mutex queue_mutex; + + /** + * Scheduling semaphore. This must be held when calling + * kbase_jm_kick() + */ + struct semaphore schedule_sem; + + /** + * List of contexts that can currently be pulled from + */ + struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS]; + /** + * List of contexts that can not currently be pulled from, but have + * jobs currently running. + */ + struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS]; + + /** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */ + s8 nr_user_contexts_running; + /** Number of currently scheduled contexts (including ones that are not submitting jobs) */ + s8 nr_all_contexts_running; + + /** Core Requirements to match up with base_js_atom's core_req memeber + * @note This is a write-once member, and so no locking is required to read */ + base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; + + u32 scheduling_period_ns; /*< Value for JS_SCHEDULING_PERIOD_NS */ + u32 soft_stop_ticks; /*< Value for JS_SOFT_STOP_TICKS */ + u32 soft_stop_ticks_cl; /*< Value for JS_SOFT_STOP_TICKS_CL */ + u32 hard_stop_ticks_ss; /*< Value for JS_HARD_STOP_TICKS_SS */ + u32 hard_stop_ticks_cl; /*< Value for JS_HARD_STOP_TICKS_CL */ + u32 hard_stop_ticks_dumping; /*< Value for JS_HARD_STOP_TICKS_DUMPING */ + u32 gpu_reset_ticks_ss; /*< Value for JS_RESET_TICKS_SS */ + u32 gpu_reset_ticks_cl; /*< Value for JS_RESET_TICKS_CL */ + u32 gpu_reset_ticks_dumping; /*< Value for JS_RESET_TICKS_DUMPING */ + u32 ctx_timeslice_ns; /**< Value for JS_CTX_TIMESLICE_NS */ + + /**< Value for JS_SOFT_JOB_TIMEOUT */ + atomic_t soft_job_timeout_ms; + + /** List of suspended soft jobs */ + struct list_head suspended_soft_jobs_list; + +#ifdef CONFIG_MALI_BIFROST_DEBUG + /* Support soft-stop on a single context */ + bool softstop_always; +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + + /** The initalized-flag is placed at the end, to avoid cache-pollution (we should + * only be using this during init/term paths). + * @note This is a write-once member, and so no locking is required to read */ + int init_status; + + /* Number of contexts that can currently be pulled from */ + u32 nr_contexts_pullable; + + /* Number of contexts that can either be pulled from or are currently + * running */ + atomic_t nr_contexts_runnable; +}; + +/** + * @brief KBase Context Job Scheduling information structure + * + * This is a substructure in the struct kbase_context that encapsulates all the + * scheduling information. + */ +struct kbasep_js_kctx_info { + + /** + * Job Scheduler Context information sub-structure. These members are + * accessed regardless of whether the context is: + * - In the Policy's Run Pool + * - In the Policy's Queue + * - Not queued nor in the Run Pool. + * + * You must obtain the jsctx_mutex before accessing any other members of + * this substructure. + * + * You may not access any of these members from IRQ context. + */ + struct kbase_jsctx { + struct mutex jsctx_mutex; /**< Job Scheduler Context lock */ + + /** Number of jobs ready to run - does \em not include the jobs waiting in + * the dispatcher, and dependency-only jobs. See kbase_jd_context::job_nr + * for such jobs*/ + u32 nr_jobs; + + /** Context Attributes: + * Each is large enough to hold a refcount of the number of atoms on + * the context. **/ + u32 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; + + /** + * Wait queue to wait for KCTX_SHEDULED flag state changes. + * */ + wait_queue_head_t is_scheduled_wait; + + /** Link implementing JS queues. Context can be present on one + * list per job slot + */ + struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS]; + } ctx; + + /* The initalized-flag is placed at the end, to avoid cache-pollution (we should + * only be using this during init/term paths) */ + int init_status; +}; + +/** Subset of atom state that can be available after jd_done_nolock() is called + * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), + * because the original atom could disappear. */ +struct kbasep_js_atom_retained_state { + /** Event code - to determine whether the atom has finished */ + enum base_jd_event_code event_code; + /** core requirements */ + base_jd_core_req core_req; + /* priority */ + int sched_priority; + /** Job Slot to retry submitting to if submission from IRQ handler failed */ + int retry_submit_on_slot; + /* Core group atom was executed on */ + u32 device_nr; + +}; + +/** + * Value signifying 'no retry on a slot required' for: + * - kbase_js_atom_retained_state::retry_submit_on_slot + * - kbase_jd_atom::retry_submit_on_slot + */ +#define KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID (-1) + +/** + * base_jd_core_req value signifying 'invalid' for a kbase_jd_atom_retained_state. + * + * @see kbase_atom_retained_state_is_valid() + */ +#define KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID BASE_JD_REQ_DEP + +/** + * @brief The JS timer resolution, in microseconds + * + * Any non-zero difference in time will be at least this size. + */ +#define KBASEP_JS_TICK_RESOLUTION_US 1 + +/* + * Internal atom priority defines for kbase_jd_atom::sched_prio + */ +enum { + KBASE_JS_ATOM_SCHED_PRIO_HIGH = 0, + KBASE_JS_ATOM_SCHED_PRIO_MED, + KBASE_JS_ATOM_SCHED_PRIO_LOW, + KBASE_JS_ATOM_SCHED_PRIO_COUNT, +}; + +/* Invalid priority for kbase_jd_atom::sched_prio */ +#define KBASE_JS_ATOM_SCHED_PRIO_INVALID -1 + +/* Default priority in the case of contexts with no atoms, or being lenient + * about invalid priorities from userspace */ +#define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED + + /** @} *//* end group kbase_js */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_JS_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_linux.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_linux.h new file mode 100755 index 000000000000..6d1e61fd41e0 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_linux.h @@ -0,0 +1,43 @@ +/* + * + * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_linux.h + * Base kernel APIs, Linux implementation. + */ + +#ifndef _KBASE_LINUX_H_ +#define _KBASE_LINUX_H_ + +/* All things that are needed for the Linux port. */ +#include +#include +#include +#include +#include + +#if (defined(MALI_KERNEL_TEST_API) && (1 == MALI_KERNEL_TEST_API)) + #define KBASE_EXPORT_TEST_API(func) EXPORT_SYMBOL(func) +#else + #define KBASE_EXPORT_TEST_API(func) +#endif + +#define KBASE_EXPORT_SYMBOL(func) EXPORT_SYMBOL(func) + +#endif /* _KBASE_LINUX_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.c new file mode 100755 index 000000000000..a105b15d641c --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.c @@ -0,0 +1,2875 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mem.c + * Base kernel memory APIs + */ +#ifdef CONFIG_DMA_SHARED_BUFFER +#include +#endif /* CONFIG_DMA_SHARED_BUFFER */ +#ifdef CONFIG_UMP +#include +#endif /* CONFIG_UMP */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/* This function finds out which RB tree the given GPU VA region belongs to + * based on the region zone */ +static struct rb_root *kbase_reg_flags_to_rbtree(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + struct rb_root *rbtree = NULL; + + switch (reg->flags & KBASE_REG_ZONE_MASK) { + case KBASE_REG_ZONE_CUSTOM_VA: + rbtree = &kctx->reg_rbtree_custom; + break; + case KBASE_REG_ZONE_EXEC: + rbtree = &kctx->reg_rbtree_exec; + break; + case KBASE_REG_ZONE_SAME_VA: + rbtree = &kctx->reg_rbtree_same; + /* fall through */ + default: + rbtree = &kctx->reg_rbtree_same; + break; + } + + return rbtree; +} + +/* This function finds out which RB tree the given pfn from the GPU VA belongs + * to based on the memory zone the pfn refers to */ +static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, + u64 gpu_pfn) +{ + struct rb_root *rbtree = NULL; + +#ifdef CONFIG_64BIT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { +#endif /* CONFIG_64BIT */ + if (gpu_pfn >= KBASE_REG_ZONE_CUSTOM_VA_BASE) + rbtree = &kctx->reg_rbtree_custom; + else if (gpu_pfn >= KBASE_REG_ZONE_EXEC_BASE) + rbtree = &kctx->reg_rbtree_exec; + else + rbtree = &kctx->reg_rbtree_same; +#ifdef CONFIG_64BIT + } else { + if (gpu_pfn >= kctx->same_va_end) + rbtree = &kctx->reg_rbtree_custom; + else + rbtree = &kctx->reg_rbtree_same; + } +#endif /* CONFIG_64BIT */ + + return rbtree; +} + +/* This function inserts a region into the tree. */ +static void kbase_region_tracker_insert(struct kbase_context *kctx, + struct kbase_va_region *new_reg) +{ + u64 start_pfn = new_reg->start_pfn; + struct rb_node **link = NULL; + struct rb_node *parent = NULL; + struct rb_root *rbtree = NULL; + + rbtree = kbase_reg_flags_to_rbtree(kctx, new_reg); + + link = &(rbtree->rb_node); + /* Find the right place in the tree using tree search */ + while (*link) { + struct kbase_va_region *old_reg; + + parent = *link; + old_reg = rb_entry(parent, struct kbase_va_region, rblink); + + /* RBTree requires no duplicate entries. */ + KBASE_DEBUG_ASSERT(old_reg->start_pfn != start_pfn); + + if (old_reg->start_pfn > start_pfn) + link = &(*link)->rb_left; + else + link = &(*link)->rb_right; + } + + /* Put the new node there, and rebalance tree */ + rb_link_node(&(new_reg->rblink), parent, link); + + rb_insert_color(&(new_reg->rblink), rbtree); +} + +/* Find allocated region enclosing free range. */ +static struct kbase_va_region *kbase_region_tracker_find_region_enclosing_range_free( + struct kbase_context *kctx, u64 start_pfn, size_t nr_pages) +{ + struct rb_node *rbnode = NULL; + struct kbase_va_region *reg = NULL; + struct rb_root *rbtree = NULL; + + u64 end_pfn = start_pfn + nr_pages; + + rbtree = kbase_gpu_va_to_rbtree(kctx, start_pfn); + + rbnode = rbtree->rb_node; + + while (rbnode) { + u64 tmp_start_pfn, tmp_end_pfn; + + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + tmp_start_pfn = reg->start_pfn; + tmp_end_pfn = reg->start_pfn + reg->nr_pages; + + /* If start is lower than this, go left. */ + if (start_pfn < tmp_start_pfn) + rbnode = rbnode->rb_left; + /* If end is higher than this, then go right. */ + else if (end_pfn > tmp_end_pfn) + rbnode = rbnode->rb_right; + else /* Enclosing */ + return reg; + } + + return NULL; +} + +/* Find region enclosing given address. */ +struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr) +{ + struct rb_node *rbnode; + struct kbase_va_region *reg; + u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; + struct rb_root *rbtree = NULL; + + KBASE_DEBUG_ASSERT(NULL != kctx); + + lockdep_assert_held(&kctx->reg_lock); + + rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); + + rbnode = rbtree->rb_node; + + while (rbnode) { + u64 tmp_start_pfn, tmp_end_pfn; + + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + tmp_start_pfn = reg->start_pfn; + tmp_end_pfn = reg->start_pfn + reg->nr_pages; + + /* If start is lower than this, go left. */ + if (gpu_pfn < tmp_start_pfn) + rbnode = rbnode->rb_left; + /* If end is higher than this, then go right. */ + else if (gpu_pfn >= tmp_end_pfn) + rbnode = rbnode->rb_right; + else /* Enclosing */ + return reg; + } + + return NULL; +} + +KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address); + +/* Find region with given base address */ +struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr) +{ + u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; + struct rb_node *rbnode = NULL; + struct kbase_va_region *reg = NULL; + struct rb_root *rbtree = NULL; + + KBASE_DEBUG_ASSERT(NULL != kctx); + + lockdep_assert_held(&kctx->reg_lock); + + rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); + + rbnode = rbtree->rb_node; + + while (rbnode) { + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + if (reg->start_pfn > gpu_pfn) + rbnode = rbnode->rb_left; + else if (reg->start_pfn < gpu_pfn) + rbnode = rbnode->rb_right; + else + return reg; + + } + + return NULL; +} + +KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address); + +/* Find region meeting given requirements */ +static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs(struct kbase_context *kctx, struct kbase_va_region *reg_reqs, size_t nr_pages, size_t align) +{ + struct rb_node *rbnode = NULL; + struct kbase_va_region *reg = NULL; + struct rb_root *rbtree = NULL; + + /* Note that this search is a linear search, as we do not have a target + address in mind, so does not benefit from the rbtree search */ + + rbtree = kbase_reg_flags_to_rbtree(kctx, reg_reqs); + + rbnode = rb_first(rbtree); + + while (rbnode) { + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + if ((reg->nr_pages >= nr_pages) && + (reg->flags & KBASE_REG_FREE)) { + /* Check alignment */ + u64 start_pfn = (reg->start_pfn + align - 1) & ~(align - 1); + + if ((start_pfn >= reg->start_pfn) && + (start_pfn <= (reg->start_pfn + reg->nr_pages - 1)) && + ((start_pfn + nr_pages - 1) <= (reg->start_pfn + reg->nr_pages - 1))) + return reg; + } + rbnode = rb_next(rbnode); + } + + return NULL; +} + +/** + * @brief Remove a region object from the global list. + * + * The region reg is removed, possibly by merging with other free and + * compatible adjacent regions. It must be called with the context + * region lock held. The associated memory is not released (see + * kbase_free_alloced_region). Internal use only. + */ +static int kbase_remove_va_region(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + struct rb_node *rbprev; + struct kbase_va_region *prev = NULL; + struct rb_node *rbnext; + struct kbase_va_region *next = NULL; + struct rb_root *reg_rbtree = NULL; + + int merged_front = 0; + int merged_back = 0; + int err = 0; + + reg_rbtree = kbase_reg_flags_to_rbtree(kctx, reg); + + /* Try to merge with the previous block first */ + rbprev = rb_prev(&(reg->rblink)); + if (rbprev) { + prev = rb_entry(rbprev, struct kbase_va_region, rblink); + if (prev->flags & KBASE_REG_FREE) { + /* We're compatible with the previous VMA, + * merge with it */ + WARN_ON((prev->flags & KBASE_REG_ZONE_MASK) != + (reg->flags & KBASE_REG_ZONE_MASK)); + prev->nr_pages += reg->nr_pages; + rb_erase(&(reg->rblink), reg_rbtree); + reg = prev; + merged_front = 1; + } + } + + /* Try to merge with the next block second */ + /* Note we do the lookup here as the tree may have been rebalanced. */ + rbnext = rb_next(&(reg->rblink)); + if (rbnext) { + /* We're compatible with the next VMA, merge with it */ + next = rb_entry(rbnext, struct kbase_va_region, rblink); + if (next->flags & KBASE_REG_FREE) { + WARN_ON((next->flags & KBASE_REG_ZONE_MASK) != + (reg->flags & KBASE_REG_ZONE_MASK)); + next->start_pfn = reg->start_pfn; + next->nr_pages += reg->nr_pages; + rb_erase(&(reg->rblink), reg_rbtree); + merged_back = 1; + if (merged_front) { + /* We already merged with prev, free it */ + kbase_free_alloced_region(reg); + } + } + } + + /* If we failed to merge then we need to add a new block */ + if (!(merged_front || merged_back)) { + /* + * We didn't merge anything. Add a new free + * placeholder and remove the original one. + */ + struct kbase_va_region *free_reg; + + free_reg = kbase_alloc_free_region(kctx, reg->start_pfn, reg->nr_pages, reg->flags & KBASE_REG_ZONE_MASK); + if (!free_reg) { + err = -ENOMEM; + goto out; + } + rb_replace_node(&(reg->rblink), &(free_reg->rblink), reg_rbtree); + } + + out: + return err; +} + +KBASE_EXPORT_TEST_API(kbase_remove_va_region); + +/** + * @brief Insert a VA region to the list, replacing the current at_reg. + */ +static int kbase_insert_va_region_nolock(struct kbase_context *kctx, struct kbase_va_region *new_reg, struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages) +{ + struct rb_root *reg_rbtree = NULL; + int err = 0; + + reg_rbtree = kbase_reg_flags_to_rbtree(kctx, at_reg); + + /* Must be a free region */ + KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0); + /* start_pfn should be contained within at_reg */ + KBASE_DEBUG_ASSERT((start_pfn >= at_reg->start_pfn) && (start_pfn < at_reg->start_pfn + at_reg->nr_pages)); + /* at least nr_pages from start_pfn should be contained within at_reg */ + KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= at_reg->start_pfn + at_reg->nr_pages); + + new_reg->start_pfn = start_pfn; + new_reg->nr_pages = nr_pages; + + /* Regions are a whole use, so swap and delete old one. */ + if (at_reg->start_pfn == start_pfn && at_reg->nr_pages == nr_pages) { + rb_replace_node(&(at_reg->rblink), &(new_reg->rblink), + reg_rbtree); + kbase_free_alloced_region(at_reg); + } + /* New region replaces the start of the old one, so insert before. */ + else if (at_reg->start_pfn == start_pfn) { + at_reg->start_pfn += nr_pages; + KBASE_DEBUG_ASSERT(at_reg->nr_pages >= nr_pages); + at_reg->nr_pages -= nr_pages; + + kbase_region_tracker_insert(kctx, new_reg); + } + /* New region replaces the end of the old one, so insert after. */ + else if ((at_reg->start_pfn + at_reg->nr_pages) == (start_pfn + nr_pages)) { + at_reg->nr_pages -= nr_pages; + + kbase_region_tracker_insert(kctx, new_reg); + } + /* New region splits the old one, so insert and create new */ + else { + struct kbase_va_region *new_front_reg; + + new_front_reg = kbase_alloc_free_region(kctx, + at_reg->start_pfn, + start_pfn - at_reg->start_pfn, + at_reg->flags & KBASE_REG_ZONE_MASK); + + if (new_front_reg) { + at_reg->nr_pages -= nr_pages + new_front_reg->nr_pages; + at_reg->start_pfn = start_pfn + nr_pages; + + kbase_region_tracker_insert(kctx, new_front_reg); + kbase_region_tracker_insert(kctx, new_reg); + } else { + err = -ENOMEM; + } + } + + return err; +} + +/** + * @brief Add a VA region to the list. + */ +int kbase_add_va_region(struct kbase_context *kctx, + struct kbase_va_region *reg, u64 addr, + size_t nr_pages, size_t align) +{ + struct kbase_va_region *tmp; + u64 gpu_pfn = addr >> PAGE_SHIFT; + int err = 0; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != reg); + + lockdep_assert_held(&kctx->reg_lock); + + if (!align) + align = 1; + + /* must be a power of 2 */ + KBASE_DEBUG_ASSERT((align & (align - 1)) == 0); + KBASE_DEBUG_ASSERT(nr_pages > 0); + + /* Path 1: Map a specific address. Find the enclosing region, which *must* be free. */ + if (gpu_pfn) { + struct device *dev = kctx->kbdev->dev; + + KBASE_DEBUG_ASSERT(!(gpu_pfn & (align - 1))); + + tmp = kbase_region_tracker_find_region_enclosing_range_free(kctx, gpu_pfn, nr_pages); + if (!tmp) { + dev_warn(dev, "Enclosing region not found: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages); + err = -ENOMEM; + goto exit; + } + if (!(tmp->flags & KBASE_REG_FREE)) { + dev_warn(dev, "Zone mismatch: %lu != %lu", tmp->flags & KBASE_REG_ZONE_MASK, reg->flags & KBASE_REG_ZONE_MASK); + dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", tmp->start_pfn, tmp->flags, tmp->nr_pages, gpu_pfn, nr_pages); + dev_warn(dev, "in function %s (%p, %p, 0x%llx, 0x%zx, 0x%zx)\n", __func__, kctx, reg, addr, nr_pages, align); + err = -ENOMEM; + goto exit; + } + + err = kbase_insert_va_region_nolock(kctx, reg, tmp, gpu_pfn, nr_pages); + if (err) { + dev_warn(dev, "Failed to insert va region"); + err = -ENOMEM; + goto exit; + } + + goto exit; + } + + /* Path 2: Map any free address which meets the requirements. */ + { + u64 start_pfn; + + /* + * Depending on the zone the allocation request is for + * we might need to retry it. + */ + do { + tmp = kbase_region_tracker_find_region_meeting_reqs( + kctx, reg, nr_pages, align); + if (tmp) { + start_pfn = (tmp->start_pfn + align - 1) & + ~(align - 1); + err = kbase_insert_va_region_nolock(kctx, reg, + tmp, start_pfn, nr_pages); + break; + } + + /* + * If the allocation is not from the same zone as JIT + * then don't retry, we're out of VA and there is + * nothing which can be done about it. + */ + if ((reg->flags & KBASE_REG_ZONE_MASK) != + KBASE_REG_ZONE_CUSTOM_VA) + break; + } while (kbase_jit_evict(kctx)); + + if (!tmp) + err = -ENOMEM; + } + + exit: + return err; +} + +KBASE_EXPORT_TEST_API(kbase_add_va_region); + +/** + * @brief Initialize the internal region tracker data structure. + */ +static void kbase_region_tracker_ds_init(struct kbase_context *kctx, + struct kbase_va_region *same_va_reg, + struct kbase_va_region *exec_reg, + struct kbase_va_region *custom_va_reg) +{ + kctx->reg_rbtree_same = RB_ROOT; + kbase_region_tracker_insert(kctx, same_va_reg); + + /* Although exec and custom_va_reg don't always exist, + * initialize unconditionally because of the mem_view debugfs + * implementation which relies on these being empty */ + kctx->reg_rbtree_exec = RB_ROOT; + kctx->reg_rbtree_custom = RB_ROOT; + + if (exec_reg) + kbase_region_tracker_insert(kctx, exec_reg); + if (custom_va_reg) + kbase_region_tracker_insert(kctx, custom_va_reg); +} + +static void kbase_region_tracker_erase_rbtree(struct rb_root *rbtree) +{ + struct rb_node *rbnode; + struct kbase_va_region *reg; + + do { + rbnode = rb_first(rbtree); + if (rbnode) { + rb_erase(rbnode, rbtree); + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + kbase_free_alloced_region(reg); + } + } while (rbnode); +} + +void kbase_region_tracker_term(struct kbase_context *kctx) +{ + kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_same); + kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_exec); + kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_custom); +} + +/** + * Initialize the region tracker data structure. + */ +int kbase_region_tracker_init(struct kbase_context *kctx) +{ + struct kbase_va_region *same_va_reg; + struct kbase_va_region *exec_reg = NULL; + struct kbase_va_region *custom_va_reg = NULL; + size_t same_va_bits = sizeof(void *) * BITS_PER_BYTE; + u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE; + u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT; + u64 same_va_pages; + int err; + + /* Take the lock as kbase_free_alloced_region requires it */ + kbase_gpu_vm_lock(kctx); + +#if defined(CONFIG_ARM64) + same_va_bits = VA_BITS; +#elif defined(CONFIG_X86_64) + same_va_bits = 47; +#elif defined(CONFIG_64BIT) +#error Unsupported 64-bit architecture +#endif + +#ifdef CONFIG_64BIT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + same_va_bits = 32; + else if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA)) + same_va_bits = 33; +#endif + + if (kctx->kbdev->gpu_props.mmu.va_bits < same_va_bits) { + err = -EINVAL; + goto fail_unlock; + } + + same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; + /* all have SAME_VA */ + same_va_reg = kbase_alloc_free_region(kctx, 1, + same_va_pages, + KBASE_REG_ZONE_SAME_VA); + + if (!same_va_reg) { + err = -ENOMEM; + goto fail_unlock; + } + +#ifdef CONFIG_64BIT + /* 32-bit clients have exec and custom VA zones */ + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { +#endif + if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) { + err = -EINVAL; + goto fail_free_same_va; + } + /* If the current size of TMEM is out of range of the + * virtual address space addressable by the MMU then + * we should shrink it to fit + */ + if ((KBASE_REG_ZONE_CUSTOM_VA_BASE + KBASE_REG_ZONE_CUSTOM_VA_SIZE) >= gpu_va_limit) + custom_va_size = gpu_va_limit - KBASE_REG_ZONE_CUSTOM_VA_BASE; + + exec_reg = kbase_alloc_free_region(kctx, + KBASE_REG_ZONE_EXEC_BASE, + KBASE_REG_ZONE_EXEC_SIZE, + KBASE_REG_ZONE_EXEC); + + if (!exec_reg) { + err = -ENOMEM; + goto fail_free_same_va; + } + + custom_va_reg = kbase_alloc_free_region(kctx, + KBASE_REG_ZONE_CUSTOM_VA_BASE, + custom_va_size, KBASE_REG_ZONE_CUSTOM_VA); + + if (!custom_va_reg) { + err = -ENOMEM; + goto fail_free_exec; + } +#ifdef CONFIG_64BIT + } +#endif + + kbase_region_tracker_ds_init(kctx, same_va_reg, exec_reg, custom_va_reg); + + kctx->same_va_end = same_va_pages + 1; + + kbase_gpu_vm_unlock(kctx); + return 0; + +fail_free_exec: + kbase_free_alloced_region(exec_reg); +fail_free_same_va: + kbase_free_alloced_region(same_va_reg); +fail_unlock: + kbase_gpu_vm_unlock(kctx); + return err; +} + +int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages) +{ +#ifdef CONFIG_64BIT + struct kbase_va_region *same_va; + struct kbase_va_region *custom_va_reg; + u64 same_va_bits; + u64 total_va_size; + int err; + + /* + * Nothing to do for 32-bit clients, JIT uses the existing + * custom VA zone. + */ + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + return 0; + +#if defined(CONFIG_ARM64) + same_va_bits = VA_BITS; +#elif defined(CONFIG_X86_64) + same_va_bits = 47; +#elif defined(CONFIG_64BIT) +#error Unsupported 64-bit architecture +#endif + + if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA)) + same_va_bits = 33; + + total_va_size = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; + + kbase_gpu_vm_lock(kctx); + + /* + * Modify the same VA free region after creation. Be careful to ensure + * that allocations haven't been made as they could cause an overlap + * to happen with existing same VA allocations and the custom VA zone. + */ + same_va = kbase_region_tracker_find_region_base_address(kctx, + PAGE_SIZE); + if (!same_va) { + err = -ENOMEM; + goto fail_unlock; + } + + /* The region flag or region size has changed since creation so bail. */ + if ((!(same_va->flags & KBASE_REG_FREE)) || + (same_va->nr_pages != total_va_size)) { + err = -ENOMEM; + goto fail_unlock; + } + + if (same_va->nr_pages < jit_va_pages || + kctx->same_va_end < jit_va_pages) { + err = -ENOMEM; + goto fail_unlock; + } + + /* It's safe to adjust the same VA zone now */ + same_va->nr_pages -= jit_va_pages; + kctx->same_va_end -= jit_va_pages; + + /* + * Create a custom VA zone at the end of the VA for allocations which + * JIT can use so it doesn't have to allocate VA from the kernel. + */ + custom_va_reg = kbase_alloc_free_region(kctx, + kctx->same_va_end, + jit_va_pages, + KBASE_REG_ZONE_CUSTOM_VA); + + if (!custom_va_reg) { + /* + * The context will be destroyed if we fail here so no point + * reverting the change we made to same_va. + */ + err = -ENOMEM; + goto fail_unlock; + } + + kbase_region_tracker_insert(kctx, custom_va_reg); + + kbase_gpu_vm_unlock(kctx); + return 0; + +fail_unlock: + kbase_gpu_vm_unlock(kctx); + return err; +#else + return 0; +#endif +} + +int kbase_mem_init(struct kbase_device *kbdev) +{ + struct kbasep_mem_device *memdev; + int ret; + + KBASE_DEBUG_ASSERT(kbdev); + + memdev = &kbdev->memdev; + kbdev->mem_pool_max_size_default = KBASE_MEM_POOL_MAX_SIZE_KCTX; + + /* Initialize memory usage */ + atomic_set(&memdev->used_pages, 0); + + ret = kbase_mem_pool_init(&kbdev->mem_pool, + KBASE_MEM_POOL_MAX_SIZE_KBDEV, + KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER, + kbdev, + NULL); + if (ret) + return ret; + + ret = kbase_mem_pool_init(&kbdev->lp_mem_pool, + (KBASE_MEM_POOL_MAX_SIZE_KBDEV >> 9), + KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, + kbdev, + NULL); + if (ret) + kbase_mem_pool_term(&kbdev->mem_pool); + + return ret; +} + +void kbase_mem_halt(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +void kbase_mem_term(struct kbase_device *kbdev) +{ + struct kbasep_mem_device *memdev; + int pages; + + KBASE_DEBUG_ASSERT(kbdev); + + memdev = &kbdev->memdev; + + pages = atomic_read(&memdev->used_pages); + if (pages != 0) + dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); + + kbase_mem_pool_term(&kbdev->mem_pool); + kbase_mem_pool_term(&kbdev->lp_mem_pool); +} + +KBASE_EXPORT_TEST_API(kbase_mem_term); + + + + +/** + * @brief Allocate a free region object. + * + * The allocated object is not part of any list yet, and is flagged as + * KBASE_REG_FREE. No mapping is allocated yet. + * + * zone is KBASE_REG_ZONE_CUSTOM_VA, KBASE_REG_ZONE_SAME_VA, or KBASE_REG_ZONE_EXEC + * + */ +struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone) +{ + struct kbase_va_region *new_reg; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + /* zone argument should only contain zone related region flags */ + KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0); + KBASE_DEBUG_ASSERT(nr_pages > 0); + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE)); + + new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL); + + if (!new_reg) + return NULL; + + new_reg->cpu_alloc = NULL; /* no alloc bound yet */ + new_reg->gpu_alloc = NULL; /* no alloc bound yet */ + new_reg->kctx = kctx; + new_reg->flags = zone | KBASE_REG_FREE; + + new_reg->flags |= KBASE_REG_GROWABLE; + + new_reg->start_pfn = start_pfn; + new_reg->nr_pages = nr_pages; + + return new_reg; +} + +KBASE_EXPORT_TEST_API(kbase_alloc_free_region); + +/** + * @brief Free a region object. + * + * The described region must be freed of any mapping. + * + * If the region is not flagged as KBASE_REG_FREE, the region's + * alloc object will be released. + * It is a bug if no alloc object exists for non-free regions. + * + */ +void kbase_free_alloced_region(struct kbase_va_region *reg) +{ + if (!(reg->flags & KBASE_REG_FREE)) { + /* + * The physical allocation should have been removed from the + * eviction list before this function is called. However, in the + * case of abnormal process termination or the app leaking the + * memory kbase_mem_free_region is not called so it can still be + * on the list at termination time of the region tracker. + */ + if (!list_empty(®->gpu_alloc->evict_node)) { + /* + * Unlink the physical allocation before unmaking it + * evictable so that the allocation isn't grown back to + * its last backed size as we're going to unmap it + * anyway. + */ + reg->cpu_alloc->reg = NULL; + if (reg->cpu_alloc != reg->gpu_alloc) + reg->gpu_alloc->reg = NULL; + + /* + * If a region has been made evictable then we must + * unmake it before trying to free it. + * If the memory hasn't been reclaimed it will be + * unmapped and freed below, if it has been reclaimed + * then the operations below are no-ops. + */ + if (reg->flags & KBASE_REG_DONT_NEED) { + KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == + KBASE_MEM_TYPE_NATIVE); + kbase_mem_evictable_unmake(reg->gpu_alloc); + } + } + + /* + * Remove the region from the sticky resource metadata + * list should it be there. + */ + kbase_sticky_resource_release(reg->kctx, NULL, + reg->start_pfn << PAGE_SHIFT); + + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); + /* To detect use-after-free in debug builds */ + KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE); + } + kfree(reg); +} + +KBASE_EXPORT_TEST_API(kbase_free_alloced_region); + +int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align) +{ + int err; + size_t i = 0; + unsigned long attr; + unsigned long mask = ~KBASE_REG_MEMATTR_MASK; + + if ((kctx->kbdev->system_coherency == COHERENCY_ACE) && + (reg->flags & KBASE_REG_SHARE_BOTH)) + attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA); + else + attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC); + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != reg); + + err = kbase_add_va_region(kctx, reg, addr, nr_pages, align); + if (err) + return err; + + if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { + u64 stride; + struct kbase_mem_phy_alloc *alloc; + + alloc = reg->gpu_alloc; + stride = alloc->imported.alias.stride; + KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); + for (i = 0; i < alloc->imported.alias.nents; i++) { + if (alloc->imported.alias.aliased[i].alloc) { + err = kbase_mmu_insert_pages(kctx, + reg->start_pfn + (i * stride), + alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset, + alloc->imported.alias.aliased[i].length, + reg->flags); + if (err) + goto bad_insert; + + kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc); + } else { + err = kbase_mmu_insert_single_page(kctx, + reg->start_pfn + i * stride, + kctx->aliasing_sink_page, + alloc->imported.alias.aliased[i].length, + (reg->flags & mask) | attr); + + if (err) + goto bad_insert; + } + } + } else { + err = kbase_mmu_insert_pages(kctx, reg->start_pfn, + kbase_get_gpu_phy_pages(reg), + kbase_reg_current_backed_size(reg), + reg->flags); + if (err) + goto bad_insert; + kbase_mem_phy_alloc_gpu_mapped(reg->gpu_alloc); + } + + return err; + +bad_insert: + if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { + u64 stride; + + stride = reg->gpu_alloc->imported.alias.stride; + KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); + while (i--) + if (reg->gpu_alloc->imported.alias.aliased[i].alloc) { + kbase_mmu_teardown_pages(kctx, reg->start_pfn + (i * stride), reg->gpu_alloc->imported.alias.aliased[i].length); + kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); + } + } + + kbase_remove_va_region(kctx, reg); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_gpu_mmap); + +static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc, bool writeable); + +int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + int err; + + if (reg->start_pfn == 0) + return 0; + + if (reg->gpu_alloc && reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { + size_t i; + + err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, reg->nr_pages); + KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); + for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++) + if (reg->gpu_alloc->imported.alias.aliased[i].alloc) + kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); + } else { + err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, kbase_reg_current_backed_size(reg)); + kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc); + } + + if (reg->gpu_alloc && reg->gpu_alloc->type == + KBASE_MEM_TYPE_IMPORTED_USER_BUF) { + struct kbase_alloc_import_user_buf *user_buf = + ®->gpu_alloc->imported.user_buf; + + if (user_buf->current_mapping_usage_count & PINNED_ON_IMPORT) { + user_buf->current_mapping_usage_count &= + ~PINNED_ON_IMPORT; + + kbase_jd_user_buf_unmap(kctx, reg->gpu_alloc, + (reg->flags & KBASE_REG_GPU_WR)); + } + } + + if (err) + return err; + + err = kbase_remove_va_region(kctx, reg); + return err; +} + +static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( + struct kbase_context *kctx, + unsigned long uaddr, size_t size, u64 *offset) +{ + struct vm_area_struct *vma; + struct kbase_cpu_mapping *map; + unsigned long vm_pgoff_in_region; + unsigned long vm_off_in_region; + unsigned long map_start; + size_t map_size; + + lockdep_assert_held(¤t->mm->mmap_sem); + + if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */ + return NULL; + + vma = find_vma_intersection(current->mm, uaddr, uaddr+size); + + if (!vma || vma->vm_start > uaddr) + return NULL; + if (vma->vm_ops != &kbase_vm_ops) + /* Not ours! */ + return NULL; + + map = vma->vm_private_data; + + if (map->kctx != kctx) + /* Not from this context! */ + return NULL; + + vm_pgoff_in_region = vma->vm_pgoff - map->region->start_pfn; + vm_off_in_region = vm_pgoff_in_region << PAGE_SHIFT; + map_start = vma->vm_start - vm_off_in_region; + map_size = map->region->nr_pages << PAGE_SHIFT; + + if ((uaddr + size) > (map_start + map_size)) + /* Not within the CPU mapping */ + return NULL; + + *offset = (uaddr - vma->vm_start) + vm_off_in_region; + + return map; +} + +int kbasep_find_enclosing_cpu_mapping_offset( + struct kbase_context *kctx, + unsigned long uaddr, size_t size, u64 *offset) +{ + struct kbase_cpu_mapping *map; + + kbase_os_mem_map_lock(kctx); + + map = kbasep_find_enclosing_cpu_mapping(kctx, uaddr, size, offset); + + kbase_os_mem_map_unlock(kctx); + + if (!map) + return -EINVAL; + + return 0; +} + +KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset); + +void kbase_sync_single(struct kbase_context *kctx, + struct tagged_addr t_cpu_pa, struct tagged_addr t_gpu_pa, + off_t offset, size_t size, enum kbase_sync_type sync_fn) +{ + struct page *cpu_page; + phys_addr_t cpu_pa = as_phys_addr_t(t_cpu_pa); + phys_addr_t gpu_pa = as_phys_addr_t(t_gpu_pa); + + cpu_page = pfn_to_page(PFN_DOWN(cpu_pa)); + + if (likely(cpu_pa == gpu_pa)) { + dma_addr_t dma_addr; + + BUG_ON(!cpu_page); + BUG_ON(offset + size > PAGE_SIZE); + + dma_addr = kbase_dma_addr(cpu_page) + offset; + if (sync_fn == KBASE_SYNC_TO_CPU) + dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr, + size, DMA_BIDIRECTIONAL); + else if (sync_fn == KBASE_SYNC_TO_DEVICE) + dma_sync_single_for_device(kctx->kbdev->dev, dma_addr, + size, DMA_BIDIRECTIONAL); + } else { + void *src = NULL; + void *dst = NULL; + struct page *gpu_page; + + if (WARN(!gpu_pa, "No GPU PA found for infinite cache op")) + return; + + gpu_page = pfn_to_page(PFN_DOWN(gpu_pa)); + + if (sync_fn == KBASE_SYNC_TO_DEVICE) { + src = ((unsigned char *)kmap(cpu_page)) + offset; + dst = ((unsigned char *)kmap(gpu_page)) + offset; + } else if (sync_fn == KBASE_SYNC_TO_CPU) { + dma_sync_single_for_cpu(kctx->kbdev->dev, + kbase_dma_addr(gpu_page) + offset, + size, DMA_BIDIRECTIONAL); + src = ((unsigned char *)kmap(gpu_page)) + offset; + dst = ((unsigned char *)kmap(cpu_page)) + offset; + } + memcpy(dst, src, size); + kunmap(gpu_page); + kunmap(cpu_page); + if (sync_fn == KBASE_SYNC_TO_DEVICE) + dma_sync_single_for_device(kctx->kbdev->dev, + kbase_dma_addr(gpu_page) + offset, + size, DMA_BIDIRECTIONAL); + } +} + +static int kbase_do_syncset(struct kbase_context *kctx, + struct basep_syncset *sset, enum kbase_sync_type sync_fn) +{ + int err = 0; + struct kbase_va_region *reg; + struct kbase_cpu_mapping *map; + unsigned long start; + size_t size; + struct tagged_addr *cpu_pa; + struct tagged_addr *gpu_pa; + u64 page_off, page_count; + u64 i; + u64 offset; + + kbase_os_mem_map_lock(kctx); + kbase_gpu_vm_lock(kctx); + + /* find the region where the virtual address is contained */ + reg = kbase_region_tracker_find_region_enclosing_address(kctx, + sset->mem_handle.basep.handle); + if (!reg) { + dev_warn(kctx->kbdev->dev, "Can't find region at VA 0x%016llX", + sset->mem_handle.basep.handle); + err = -EINVAL; + goto out_unlock; + } + + if (!(reg->flags & KBASE_REG_CPU_CACHED) || + kbase_mem_is_imported(reg->gpu_alloc->type)) + goto out_unlock; + + start = (uintptr_t)sset->user_addr; + size = (size_t)sset->size; + + map = kbasep_find_enclosing_cpu_mapping(kctx, start, size, &offset); + if (!map) { + dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX", + start, sset->mem_handle.basep.handle); + err = -EINVAL; + goto out_unlock; + } + + page_off = offset >> PAGE_SHIFT; + offset &= ~PAGE_MASK; + page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + cpu_pa = kbase_get_cpu_phy_pages(reg); + gpu_pa = kbase_get_gpu_phy_pages(reg); + + if (page_off > reg->nr_pages || + page_off + page_count > reg->nr_pages) { + /* Sync overflows the region */ + err = -EINVAL; + goto out_unlock; + } + + /* Sync first page */ + if (as_phys_addr_t(cpu_pa[page_off])) { + size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); + + kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off], + offset, sz, sync_fn); + } + + /* Sync middle pages (if any) */ + for (i = 1; page_count > 2 && i < page_count - 1; i++) { + /* we grow upwards, so bail on first non-present page */ + if (!as_phys_addr_t(cpu_pa[page_off + i])) + break; + + kbase_sync_single(kctx, cpu_pa[page_off + i], + gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn); + } + + /* Sync last page (if any) */ + if (page_count > 1 && + as_phys_addr_t(cpu_pa[page_off + page_count - 1])) { + size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1; + + kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1], + gpu_pa[page_off + page_count - 1], 0, sz, + sync_fn); + } + +out_unlock: + kbase_gpu_vm_unlock(kctx); + kbase_os_mem_map_unlock(kctx); + return err; +} + +int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) +{ + int err = -EINVAL; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(sset != NULL); + + if (sset->mem_handle.basep.handle & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, + "mem_handle: passed parameter is invalid"); + return -EINVAL; + } + + switch (sset->type) { + case BASE_SYNCSET_OP_MSYNC: + err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_DEVICE); + break; + + case BASE_SYNCSET_OP_CSYNC: + err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_CPU); + break; + + default: + dev_warn(kctx->kbdev->dev, "Unknown msync op %d\n", sset->type); + break; + } + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_sync_now); + +/* vm lock must be held */ +int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + int err; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != reg); + lockdep_assert_held(&kctx->reg_lock); + + /* + * Unlink the physical allocation before unmaking it evictable so + * that the allocation isn't grown back to its last backed size + * as we're going to unmap it anyway. + */ + reg->cpu_alloc->reg = NULL; + if (reg->cpu_alloc != reg->gpu_alloc) + reg->gpu_alloc->reg = NULL; + + /* + * If a region has been made evictable then we must unmake it + * before trying to free it. + * If the memory hasn't been reclaimed it will be unmapped and freed + * below, if it has been reclaimed then the operations below are no-ops. + */ + if (reg->flags & KBASE_REG_DONT_NEED) { + KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == + KBASE_MEM_TYPE_NATIVE); + kbase_mem_evictable_unmake(reg->gpu_alloc); + } + + err = kbase_gpu_munmap(kctx, reg); + if (err) { + dev_warn(reg->kctx->kbdev->dev, "Could not unmap from the GPU...\n"); + goto out; + } + + /* This will also free the physical pages */ + kbase_free_alloced_region(reg); + + out: + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mem_free_region); + +/** + * @brief Free the region from the GPU and unregister it. + * + * This function implements the free operation on a memory segment. + * It will loudly fail if called with outstanding mappings. + */ +int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) +{ + int err = 0; + struct kbase_va_region *reg; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) { + dev_warn(kctx->kbdev->dev, "kbase_mem_free: gpu_addr parameter is invalid"); + return -EINVAL; + } + + if (0 == gpu_addr) { + dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); + return -EINVAL; + } + kbase_gpu_vm_lock(kctx); + + if (gpu_addr >= BASE_MEM_COOKIE_BASE && + gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) { + int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); + + reg = kctx->pending_regions[cookie]; + if (!reg) { + err = -EINVAL; + goto out_unlock; + } + + /* ask to unlink the cookie as we'll free it */ + + kctx->pending_regions[cookie] = NULL; + kctx->cookies |= (1UL << cookie); + + kbase_free_alloced_region(reg); + } else { + /* A real GPU va */ + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (!reg || (reg->flags & KBASE_REG_FREE)) { + dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX", + gpu_addr); + err = -EINVAL; + goto out_unlock; + } + + if ((reg->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA) { + /* SAME_VA must be freed through munmap */ + dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__, + gpu_addr); + err = -EINVAL; + goto out_unlock; + } + err = kbase_mem_free_region(kctx, reg); + } + + out_unlock: + kbase_gpu_vm_unlock(kctx); + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mem_free); + +int kbase_update_region_flags(struct kbase_context *kctx, + struct kbase_va_region *reg, unsigned long flags) +{ + KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0); + + reg->flags |= kbase_cache_enabled(flags, reg->nr_pages); + /* all memory is now growable */ + reg->flags |= KBASE_REG_GROWABLE; + + if (flags & BASE_MEM_GROW_ON_GPF) + reg->flags |= KBASE_REG_PF_GROW; + + if (flags & BASE_MEM_PROT_CPU_WR) + reg->flags |= KBASE_REG_CPU_WR; + + if (flags & BASE_MEM_PROT_CPU_RD) + reg->flags |= KBASE_REG_CPU_RD; + + if (flags & BASE_MEM_PROT_GPU_WR) + reg->flags |= KBASE_REG_GPU_WR; + + if (flags & BASE_MEM_PROT_GPU_RD) + reg->flags |= KBASE_REG_GPU_RD; + + if (0 == (flags & BASE_MEM_PROT_GPU_EX)) + reg->flags |= KBASE_REG_GPU_NX; + + if (!kbase_device_is_cpu_coherent(kctx->kbdev)) { + if (flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) + return -EINVAL; + } else if (flags & (BASE_MEM_COHERENT_SYSTEM | + BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { + reg->flags |= KBASE_REG_SHARE_BOTH; + } + + if (!(reg->flags & KBASE_REG_SHARE_BOTH) && + flags & BASE_MEM_COHERENT_LOCAL) { + reg->flags |= KBASE_REG_SHARE_IN; + } + + /* Set up default MEMATTR usage */ + if (kctx->kbdev->system_coherency == COHERENCY_ACE && + (reg->flags & KBASE_REG_SHARE_BOTH)) { + reg->flags |= + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); + } else { + reg->flags |= + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT); + } + + return 0; +} + +int kbase_alloc_phy_pages_helper( + struct kbase_mem_phy_alloc *alloc, + size_t nr_pages_requested) +{ + int new_page_count __maybe_unused; + size_t old_page_count = alloc->nents; + size_t nr_left = nr_pages_requested; + int res; + struct kbase_context *kctx; + struct tagged_addr *tp; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); + KBASE_DEBUG_ASSERT(alloc->imported.kctx); + + kctx = alloc->imported.kctx; + + if (nr_pages_requested == 0) + goto done; /*nothing to do*/ + + new_page_count = kbase_atomic_add_pages( + nr_pages_requested, &kctx->used_pages); + kbase_atomic_add_pages(nr_pages_requested, + &kctx->kbdev->memdev.used_pages); + + /* Increase mm counters before we allocate pages so that this + * allocation is visible to the OOM killer */ + kbase_process_page_usage_inc(kctx, nr_pages_requested); + + tp = alloc->pages + old_page_count; + +#ifdef CONFIG_MALI_2MB_ALLOC + /* Check if we have enough pages requested so we can allocate a large + * page (512 * 4KB = 2MB ) + */ + if (nr_left >= (SZ_2M / SZ_4K)) { + int nr_lp = nr_left / (SZ_2M / SZ_4K); + + res = kbase_mem_pool_alloc_pages(&kctx->lp_mem_pool, + nr_lp * (SZ_2M / SZ_4K), + tp, + true); + + if (res > 0) { + nr_left -= res; + tp += res; + } + + if (nr_left) { + struct kbase_sub_alloc *sa, *temp_sa; + + mutex_lock(&kctx->mem_partials_lock); + + list_for_each_entry_safe(sa, temp_sa, + &kctx->mem_partials, link) { + int pidx = 0; + + while (nr_left) { + pidx = find_next_zero_bit(sa->sub_pages, + SZ_2M / SZ_4K, + pidx); + bitmap_set(sa->sub_pages, pidx, 1); + *tp++ = as_tagged_tag(page_to_phys(sa->page + + pidx), + FROM_PARTIAL); + nr_left--; + + if (bitmap_full(sa->sub_pages, SZ_2M / SZ_4K)) { + /* unlink from partial list when full */ + list_del_init(&sa->link); + break; + } + } + } + mutex_unlock(&kctx->mem_partials_lock); + } + + /* only if we actually have a chunk left <512. If more it indicates + * that we couldn't allocate a 2MB above, so no point to retry here. + */ + if (nr_left > 0 && nr_left < (SZ_2M / SZ_4K)) { + /* create a new partial and suballocate the rest from it */ + struct page *np = NULL; + + do { + int err; + + np = kbase_mem_pool_alloc(&kctx->lp_mem_pool); + if (np) + break; + err = kbase_mem_pool_grow(&kctx->lp_mem_pool, 1); + if (err) + break; + } while (1); + + if (np) { + int i; + struct kbase_sub_alloc *sa; + struct page *p; + + sa = kmalloc(sizeof(*sa), GFP_KERNEL); + if (!sa) { + kbase_mem_pool_free(&kctx->lp_mem_pool, np, false); + goto no_new_partial; + } + + /* store pointers back to the control struct */ + np->lru.next = (void *)sa; + for (p = np; p < np + SZ_2M / SZ_4K; p++) + p->lru.prev = (void *)np; + INIT_LIST_HEAD(&sa->link); + bitmap_zero(sa->sub_pages, SZ_2M / SZ_4K); + sa->page = np; + + for (i = 0; i < nr_left; i++) + *tp++ = as_tagged_tag(page_to_phys(np + i), FROM_PARTIAL); + + bitmap_set(sa->sub_pages, 0, nr_left); + nr_left = 0; + + /* expose for later use */ + mutex_lock(&kctx->mem_partials_lock); + list_add(&sa->link, &kctx->mem_partials); + mutex_unlock(&kctx->mem_partials_lock); + } + } + } +no_new_partial: +#endif + + if (nr_left) { + res = kbase_mem_pool_alloc_pages(&kctx->mem_pool, + nr_left, + tp, + false); + if (res <= 0) + goto alloc_failed; + } + + /* + * Request a zone cache update, this scans only the new pages an + * appends their information to the zone cache. if the update + * fails then clear the cache so we fall-back to doing things + * page by page. + */ + if (kbase_zone_cache_update(alloc, old_page_count) != 0) + kbase_zone_cache_clear(alloc); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kctx->id, + (u64)new_page_count); + + alloc->nents += nr_pages_requested; +done: + return 0; + +alloc_failed: + /* rollback needed if got one or more 2MB but failed later */ + if (nr_left != nr_pages_requested) + kbase_mem_pool_free_pages(&kctx->lp_mem_pool, + nr_pages_requested - nr_left, + alloc->pages + old_page_count, + false, + false); + + kbase_process_page_usage_dec(kctx, nr_pages_requested); + kbase_atomic_sub_pages(nr_pages_requested, &kctx->used_pages); + kbase_atomic_sub_pages(nr_pages_requested, + &kctx->kbdev->memdev.used_pages); + + return -ENOMEM; +} + +static void free_partial(struct kbase_context *kctx, struct tagged_addr tp) +{ + struct page *p, *head_page; + struct kbase_sub_alloc *sa; + + p = phys_to_page(as_phys_addr_t(tp)); + head_page = (struct page *)p->lru.prev; + sa = (struct kbase_sub_alloc *)head_page->lru.next; + mutex_lock(&kctx->mem_partials_lock); + clear_bit(p - head_page, sa->sub_pages); + if (bitmap_empty(sa->sub_pages, SZ_2M / SZ_4K)) { + list_del(&sa->link); + kbase_mem_pool_free(&kctx->lp_mem_pool, head_page, true); + kfree(sa); + } else if (bitmap_weight(sa->sub_pages, SZ_2M / SZ_4K) == + SZ_2M / SZ_4K - 1) { + /* expose the partial again */ + list_add(&sa->link, &kctx->mem_partials); + } + mutex_unlock(&kctx->mem_partials_lock); +} + +int kbase_free_phy_pages_helper( + struct kbase_mem_phy_alloc *alloc, + size_t nr_pages_to_free) +{ + struct kbase_context *kctx = alloc->imported.kctx; + bool syncback; + bool reclaimed = (alloc->evicted != 0); + struct tagged_addr *start_free; + int new_page_count __maybe_unused; + size_t freed = 0; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); + KBASE_DEBUG_ASSERT(alloc->imported.kctx); + KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free); + + /* early out if nothing to do */ + if (0 == nr_pages_to_free) + return 0; + + start_free = alloc->pages + alloc->nents - nr_pages_to_free; + + syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; + + /* pad start_free to a valid start location */ + while (nr_pages_to_free && is_huge(*start_free) && + !is_huge_head(*start_free)) { + nr_pages_to_free--; + start_free++; + } + + /* + * Clear the zone cache, we don't expect JIT allocations to be + * shrunk in parts so there is no point trying to optimize for that + * by scanning for the changes caused by freeing this memory and + * updating the existing cache entries. + */ + kbase_zone_cache_clear(alloc); + + + while (nr_pages_to_free) { + if (is_huge_head(*start_free)) { + /* This is a 2MB entry, so free all the 512 pages that + * it points to + */ + kbase_mem_pool_free_pages(&kctx->lp_mem_pool, + 512, + start_free, + syncback, + reclaimed); + nr_pages_to_free -= 512; + start_free += 512; + freed += 512; + } else if (is_partial(*start_free)) { + free_partial(kctx, *start_free); + nr_pages_to_free--; + start_free++; + freed++; + } else { + struct tagged_addr *local_end_free; + + local_end_free = start_free; + while (nr_pages_to_free && + !is_huge(*local_end_free) && + !is_partial(*local_end_free)) { + local_end_free++; + nr_pages_to_free--; + } + kbase_mem_pool_free_pages(&kctx->mem_pool, + local_end_free - start_free, + start_free, + syncback, + reclaimed); + freed += local_end_free - start_free; + start_free += local_end_free - start_free; + } + } + + alloc->nents -= freed; + + /* + * If the allocation was not evicted (i.e. evicted == 0) then + * the page accounting needs to be done. + */ + if (!reclaimed) { + kbase_process_page_usage_dec(kctx, freed); + new_page_count = kbase_atomic_sub_pages(freed, + &kctx->used_pages); + kbase_atomic_sub_pages(freed, + &kctx->kbdev->memdev.used_pages); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kctx->id, + (u64)new_page_count); + } + + return 0; +} + +void kbase_mem_kref_free(struct kref *kref) +{ + struct kbase_mem_phy_alloc *alloc; + + alloc = container_of(kref, struct kbase_mem_phy_alloc, kref); + + switch (alloc->type) { + case KBASE_MEM_TYPE_NATIVE: { + WARN_ON(!alloc->imported.kctx); + /* + * The physical allocation must have been removed from the + * eviction list before trying to free it. + */ + WARN_ON(!list_empty(&alloc->evict_node)); + kbase_free_phy_pages_helper(alloc, alloc->nents); + break; + } + case KBASE_MEM_TYPE_ALIAS: { + /* just call put on the underlying phy allocs */ + size_t i; + struct kbase_aliased *aliased; + + aliased = alloc->imported.alias.aliased; + if (aliased) { + for (i = 0; i < alloc->imported.alias.nents; i++) + if (aliased[i].alloc) + kbase_mem_phy_alloc_put(aliased[i].alloc); + vfree(aliased); + } + break; + } + case KBASE_MEM_TYPE_RAW: + /* raw pages, external cleanup */ + break; + #ifdef CONFIG_UMP + case KBASE_MEM_TYPE_IMPORTED_UMP: + ump_dd_release(alloc->imported.ump_handle); + break; +#endif +#ifdef CONFIG_DMA_SHARED_BUFFER + case KBASE_MEM_TYPE_IMPORTED_UMM: + dma_buf_detach(alloc->imported.umm.dma_buf, + alloc->imported.umm.dma_attachment); + dma_buf_put(alloc->imported.umm.dma_buf); + break; +#endif + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + if (alloc->imported.user_buf.mm) + mmdrop(alloc->imported.user_buf.mm); + kfree(alloc->imported.user_buf.pages); + break; + case KBASE_MEM_TYPE_TB:{ + void *tb; + + tb = alloc->imported.kctx->jctx.tb; + kbase_device_trace_buffer_uninstall(alloc->imported.kctx); + vfree(tb); + break; + } + default: + WARN(1, "Unexecpted free of type %d\n", alloc->type); + break; + } + + /* Free based on allocation type */ + if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) + vfree(alloc); + else + kfree(alloc); +} + +KBASE_EXPORT_TEST_API(kbase_mem_kref_free); + +int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size) +{ + KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT(vsize > 0); + + /* validate user provided arguments */ + if (size > vsize || vsize > reg->nr_pages) + goto out_term; + + /* Prevent vsize*sizeof from wrapping around. + * For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail. + */ + if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages))) + goto out_term; + + KBASE_DEBUG_ASSERT(0 != vsize); + + if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0) + goto out_term; + + reg->cpu_alloc->reg = reg; + if (reg->cpu_alloc != reg->gpu_alloc) { + if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0) + goto out_rollback; + reg->gpu_alloc->reg = reg; + } + + return 0; + +out_rollback: + kbase_free_phy_pages_helper(reg->cpu_alloc, size); +out_term: + return -1; +} + +KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages); + +bool kbase_check_alloc_flags(unsigned long flags) +{ + /* Only known input flags should be set. */ + if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) + return false; + + /* At least one flag should be set */ + if (flags == 0) + return false; + + /* Either the GPU or CPU must be reading from the allocated memory */ + if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0) + return false; + + /* Either the GPU or CPU must be writing to the allocated memory */ + if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0) + return false; + + /* GPU cannot be writing to GPU executable memory and cannot grow the memory on page fault. */ + if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) + return false; + + /* GPU should have at least read or write access otherwise there is no + reason for allocating. */ + if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) + return false; + + /* BASE_MEM_IMPORT_SHARED is only valid for imported memory */ + if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED) + return false; + + return true; +} + +bool kbase_check_import_flags(unsigned long flags) +{ + /* Only known input flags should be set. */ + if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) + return false; + + /* At least one flag should be set */ + if (flags == 0) + return false; + + /* Imported memory cannot be GPU executable */ + if (flags & BASE_MEM_PROT_GPU_EX) + return false; + + /* Imported memory cannot grow on page fault */ + if (flags & BASE_MEM_GROW_ON_GPF) + return false; + + /* GPU should have at least read or write access otherwise there is no + reason for importing. */ + if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) + return false; + + /* Secure memory cannot be read by the CPU */ + if ((flags & BASE_MEM_SECURE) && (flags & BASE_MEM_PROT_CPU_RD)) + return false; + + return true; +} + +/** + * @brief Acquire the per-context region list lock + */ +void kbase_gpu_vm_lock(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx != NULL); + mutex_lock(&kctx->reg_lock); +} + +KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); + +/** + * @brief Release the per-context region list lock + */ +void kbase_gpu_vm_unlock(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx != NULL); + mutex_unlock(&kctx->reg_lock); +} + +KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); + +#ifdef CONFIG_DEBUG_FS +struct kbase_jit_debugfs_data { + int (*func)(struct kbase_jit_debugfs_data *); + struct mutex lock; + struct kbase_context *kctx; + u64 active_value; + u64 pool_value; + u64 destroy_value; + char buffer[50]; +}; + +static int kbase_jit_debugfs_common_open(struct inode *inode, + struct file *file, int (*func)(struct kbase_jit_debugfs_data *)) +{ + struct kbase_jit_debugfs_data *data; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->func = func; + mutex_init(&data->lock); + data->kctx = (struct kbase_context *) inode->i_private; + + file->private_data = data; + + return nonseekable_open(inode, file); +} + +static ssize_t kbase_jit_debugfs_common_read(struct file *file, + char __user *buf, size_t len, loff_t *ppos) +{ + struct kbase_jit_debugfs_data *data; + size_t size; + int ret; + + data = (struct kbase_jit_debugfs_data *) file->private_data; + mutex_lock(&data->lock); + + if (*ppos) { + size = strnlen(data->buffer, sizeof(data->buffer)); + } else { + if (!data->func) { + ret = -EACCES; + goto out_unlock; + } + + if (data->func(data)) { + ret = -EACCES; + goto out_unlock; + } + + size = scnprintf(data->buffer, sizeof(data->buffer), + "%llu,%llu,%llu", data->active_value, + data->pool_value, data->destroy_value); + } + + ret = simple_read_from_buffer(buf, len, ppos, data->buffer, size); + +out_unlock: + mutex_unlock(&data->lock); + return ret; +} + +static int kbase_jit_debugfs_common_release(struct inode *inode, + struct file *file) +{ + kfree(file->private_data); + return 0; +} + +#define KBASE_JIT_DEBUGFS_DECLARE(__fops, __func) \ +static int __fops ## _open(struct inode *inode, struct file *file) \ +{ \ + return kbase_jit_debugfs_common_open(inode, file, __func); \ +} \ +static const struct file_operations __fops = { \ + .owner = THIS_MODULE, \ + .open = __fops ## _open, \ + .release = kbase_jit_debugfs_common_release, \ + .read = kbase_jit_debugfs_common_read, \ + .write = NULL, \ + .llseek = generic_file_llseek, \ +} + +static int kbase_jit_debugfs_count_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct list_head *tmp; + + mutex_lock(&kctx->jit_evict_lock); + list_for_each(tmp, &kctx->jit_active_head) { + data->active_value++; + } + + list_for_each(tmp, &kctx->jit_pool_head) { + data->pool_value++; + } + + list_for_each(tmp, &kctx->jit_destroy_head) { + data->destroy_value++; + } + mutex_unlock(&kctx->jit_evict_lock); + + return 0; +} +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_count_fops, + kbase_jit_debugfs_count_get); + +static int kbase_jit_debugfs_vm_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct kbase_va_region *reg; + + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { + data->active_value += reg->nr_pages; + } + + list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { + data->pool_value += reg->nr_pages; + } + + list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { + data->destroy_value += reg->nr_pages; + } + mutex_unlock(&kctx->jit_evict_lock); + + return 0; +} +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_vm_fops, + kbase_jit_debugfs_vm_get); + +static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct kbase_va_region *reg; + + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { + data->active_value += reg->gpu_alloc->nents; + } + + list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { + data->pool_value += reg->gpu_alloc->nents; + } + + list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { + data->destroy_value += reg->gpu_alloc->nents; + } + mutex_unlock(&kctx->jit_evict_lock); + + return 0; +} +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops, + kbase_jit_debugfs_phys_get); + +void kbase_jit_debugfs_init(struct kbase_context *kctx) +{ + /* Debugfs entry for getting the number of JIT allocations. */ + debugfs_create_file("mem_jit_count", S_IRUGO, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_count_fops); + + /* + * Debugfs entry for getting the total number of virtual pages + * used by JIT allocations. + */ + debugfs_create_file("mem_jit_vm", S_IRUGO, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_vm_fops); + + /* + * Debugfs entry for getting the number of physical pages used + * by JIT allocations. + */ + debugfs_create_file("mem_jit_phys", S_IRUGO, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_phys_fops); +} +#endif /* CONFIG_DEBUG_FS */ + +/** + * kbase_jit_destroy_worker - Deferred worker which frees JIT allocations + * @work: Work item + * + * This function does the work of freeing JIT allocations whose physical + * backing has been released. + */ +static void kbase_jit_destroy_worker(struct work_struct *work) +{ + struct kbase_context *kctx; + struct kbase_va_region *reg; + + kctx = container_of(work, struct kbase_context, jit_work); + do { + mutex_lock(&kctx->jit_evict_lock); + if (list_empty(&kctx->jit_destroy_head)) { + mutex_unlock(&kctx->jit_evict_lock); + break; + } + + reg = list_first_entry(&kctx->jit_destroy_head, + struct kbase_va_region, jit_node); + + list_del(®->jit_node); + mutex_unlock(&kctx->jit_evict_lock); + + kbase_gpu_vm_lock(kctx); + kbase_mem_free_region(kctx, reg); + kbase_gpu_vm_unlock(kctx); + } while (1); +} + +int kbase_jit_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->jit_active_head); + INIT_LIST_HEAD(&kctx->jit_pool_head); + INIT_LIST_HEAD(&kctx->jit_destroy_head); + INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); + + INIT_LIST_HEAD(&kctx->jit_pending_alloc); + INIT_LIST_HEAD(&kctx->jit_atoms_head); + + return 0; +} + +struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, + struct base_jit_alloc_info *info) +{ + struct kbase_va_region *reg = NULL; + struct kbase_va_region *walker; + struct kbase_va_region *temp; + size_t current_diff = SIZE_MAX; + + int ret; + + mutex_lock(&kctx->jit_evict_lock); + /* + * Scan the pool for an existing allocation which meets our + * requirements and remove it. + */ + list_for_each_entry_safe(walker, temp, &kctx->jit_pool_head, jit_node) { + + if (walker->nr_pages >= info->va_pages) { + size_t min_size, max_size, diff; + + /* + * The JIT allocations VA requirements have been + * meet, it's suitable but other allocations + * might be a better fit. + */ + min_size = min_t(size_t, walker->gpu_alloc->nents, + info->commit_pages); + max_size = max_t(size_t, walker->gpu_alloc->nents, + info->commit_pages); + diff = max_size - min_size; + + if (current_diff > diff) { + current_diff = diff; + reg = walker; + } + + /* The allocation is an exact match, stop looking */ + if (current_diff == 0) + break; + } + } + + if (reg) { + /* + * Remove the found region from the pool and add it to the + * active list. + */ + list_move(®->jit_node, &kctx->jit_active_head); + + /* + * Remove the allocation from the eviction list as it's no + * longer eligible for eviction. This must be done before + * dropping the jit_evict_lock + */ + list_del_init(®->gpu_alloc->evict_node); + mutex_unlock(&kctx->jit_evict_lock); + + kbase_gpu_vm_lock(kctx); + + /* Make the physical backing no longer reclaimable */ + if (!kbase_mem_evictable_unmake(reg->gpu_alloc)) + goto update_failed; + + /* Grow the backing if required */ + if (reg->gpu_alloc->nents < info->commit_pages) { + size_t delta; + size_t old_size = reg->gpu_alloc->nents; + + /* Allocate some more pages */ + delta = info->commit_pages - reg->gpu_alloc->nents; + if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, delta) + != 0) + goto update_failed; + + if (reg->cpu_alloc != reg->gpu_alloc) { + if (kbase_alloc_phy_pages_helper( + reg->cpu_alloc, delta) != 0) { + kbase_free_phy_pages_helper( + reg->gpu_alloc, delta); + goto update_failed; + } + } + + ret = kbase_mem_grow_gpu_mapping(kctx, reg, + info->commit_pages, old_size); + /* + * The grow failed so put the allocation back in the + * pool and return failure. + */ + if (ret) + goto update_failed; + } + kbase_gpu_vm_unlock(kctx); + } else { + /* No suitable JIT allocation was found so create a new one */ + u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | + BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF | + BASE_MEM_COHERENT_LOCAL; + u64 gpu_addr; + + mutex_unlock(&kctx->jit_evict_lock); + + reg = kbase_mem_alloc(kctx, info->va_pages, info->commit_pages, + info->extent, &flags, &gpu_addr); + if (!reg) + goto out_unlocked; + + mutex_lock(&kctx->jit_evict_lock); + list_add(®->jit_node, &kctx->jit_active_head); + mutex_unlock(&kctx->jit_evict_lock); + } + + return reg; + +update_failed: + /* + * An update to an allocation from the pool failed, chances + * are slim a new allocation would fair any better so return + * the allocation to the pool and return the function with failure. + */ + kbase_gpu_vm_unlock(kctx); + mutex_lock(&kctx->jit_evict_lock); + list_move(®->jit_node, &kctx->jit_pool_head); + mutex_unlock(&kctx->jit_evict_lock); +out_unlocked: + return NULL; +} + +void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + /* The physical backing of memory in the pool is always reclaimable */ + kbase_gpu_vm_lock(kctx); + kbase_mem_evictable_make(reg->gpu_alloc); + kbase_gpu_vm_unlock(kctx); + + mutex_lock(&kctx->jit_evict_lock); + list_move(®->jit_node, &kctx->jit_pool_head); + mutex_unlock(&kctx->jit_evict_lock); +} + +void kbase_jit_backing_lost(struct kbase_va_region *reg) +{ + struct kbase_context *kctx = reg->kctx; + + lockdep_assert_held(&kctx->jit_evict_lock); + + /* + * JIT allocations will always be on a list, if the region + * is not on a list then it's not a JIT allocation. + */ + if (list_empty(®->jit_node)) + return; + + /* + * Freeing the allocation requires locks we might not be able + * to take now, so move the allocation to the free list and kick + * the worker which will do the freeing. + */ + list_move(®->jit_node, &kctx->jit_destroy_head); + + schedule_work(&kctx->jit_work); +} + +bool kbase_jit_evict(struct kbase_context *kctx) +{ + struct kbase_va_region *reg = NULL; + + lockdep_assert_held(&kctx->reg_lock); + + /* Free the oldest allocation from the pool */ + mutex_lock(&kctx->jit_evict_lock); + if (!list_empty(&kctx->jit_pool_head)) { + reg = list_entry(kctx->jit_pool_head.prev, + struct kbase_va_region, jit_node); + list_del(®->jit_node); + } + mutex_unlock(&kctx->jit_evict_lock); + + if (reg) + kbase_mem_free_region(kctx, reg); + + return (reg != NULL); +} + +void kbase_jit_term(struct kbase_context *kctx) +{ + struct kbase_va_region *walker; + + /* Free all allocations for this context */ + + /* + * Flush the freeing of allocations whose backing has been freed + * (i.e. everything in jit_destroy_head). + */ + cancel_work_sync(&kctx->jit_work); + + kbase_gpu_vm_lock(kctx); + mutex_lock(&kctx->jit_evict_lock); + /* Free all allocations from the pool */ + while (!list_empty(&kctx->jit_pool_head)) { + walker = list_first_entry(&kctx->jit_pool_head, + struct kbase_va_region, jit_node); + list_del(&walker->jit_node); + mutex_unlock(&kctx->jit_evict_lock); + kbase_mem_free_region(kctx, walker); + mutex_lock(&kctx->jit_evict_lock); + } + + /* Free all allocations from active list */ + while (!list_empty(&kctx->jit_active_head)) { + walker = list_first_entry(&kctx->jit_active_head, + struct kbase_va_region, jit_node); + list_del(&walker->jit_node); + mutex_unlock(&kctx->jit_evict_lock); + kbase_mem_free_region(kctx, walker); + mutex_lock(&kctx->jit_evict_lock); + } + mutex_unlock(&kctx->jit_evict_lock); + kbase_gpu_vm_unlock(kctx); +} + +static int kbase_jd_user_buf_map(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + long pinned_pages; + struct kbase_mem_phy_alloc *alloc; + struct page **pages; + struct tagged_addr *pa; + long i; + int err = -ENOMEM; + unsigned long address; + struct mm_struct *mm; + struct device *dev; + unsigned long offset; + unsigned long local_size; + + alloc = reg->gpu_alloc; + pa = kbase_get_gpu_phy_pages(reg); + address = alloc->imported.user_buf.address; + mm = alloc->imported.user_buf.mm; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); + + pages = alloc->imported.user_buf.pages; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + pinned_pages = get_user_pages(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR, + 0, pages, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + pinned_pages = get_user_pages_remote(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR, + 0, pages, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + pinned_pages = get_user_pages_remote(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL); +#else + pinned_pages = get_user_pages_remote(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL, NULL); +#endif + + if (pinned_pages <= 0) + return pinned_pages; + + if (pinned_pages != alloc->imported.user_buf.nr_pages) { + for (i = 0; i < pinned_pages; i++) + put_page(pages[i]); + return -ENOMEM; + } + + dev = kctx->kbdev->dev; + offset = address & ~PAGE_MASK; + local_size = alloc->imported.user_buf.size; + + for (i = 0; i < pinned_pages; i++) { + dma_addr_t dma_addr; + unsigned long min; + + min = MIN(PAGE_SIZE - offset, local_size); + dma_addr = dma_map_page(dev, pages[i], + offset, min, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) + goto unwind; + + alloc->imported.user_buf.dma_addrs[i] = dma_addr; + pa[i] = as_tagged(page_to_phys(pages[i])); + + local_size -= min; + offset = 0; + } + + alloc->nents = pinned_pages; + + err = kbase_mmu_insert_pages(kctx, reg->start_pfn, pa, + kbase_reg_current_backed_size(reg), + reg->flags); + if (err == 0) + return 0; + + alloc->nents = 0; + /* fall down */ +unwind: + while (i--) { + dma_unmap_page(kctx->kbdev->dev, + alloc->imported.user_buf.dma_addrs[i], + PAGE_SIZE, DMA_BIDIRECTIONAL); + } + + while (++i < pinned_pages) { + put_page(pages[i]); + pages[i] = NULL; + } + + return err; +} + +static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc, bool writeable) +{ + long i; + struct page **pages; + unsigned long size = alloc->imported.user_buf.size; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); + pages = alloc->imported.user_buf.pages; + for (i = 0; i < alloc->imported.user_buf.nr_pages; i++) { + unsigned long local_size; + dma_addr_t dma_addr = alloc->imported.user_buf.dma_addrs[i]; + + local_size = MIN(size, PAGE_SIZE - (dma_addr & ~PAGE_MASK)); + dma_unmap_page(kctx->kbdev->dev, dma_addr, local_size, + DMA_BIDIRECTIONAL); + if (writeable) + set_page_dirty_lock(pages[i]); + put_page(pages[i]); + pages[i] = NULL; + + size -= local_size; + } + alloc->nents = 0; +} + +/* to replace sg_dma_len. */ +#define MALI_SG_DMA_LEN(sg) ((sg)->length) + +#ifdef CONFIG_DMA_SHARED_BUFFER +static int kbase_jd_umm_map(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + struct sg_table *sgt; + struct scatterlist *s; + int i; + struct tagged_addr *pa; + int err; + size_t count = 0; + struct kbase_mem_phy_alloc *alloc; + + alloc = reg->gpu_alloc; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM); + KBASE_DEBUG_ASSERT(NULL == alloc->imported.umm.sgt); + sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, + DMA_BIDIRECTIONAL); + + if (IS_ERR_OR_NULL(sgt)) + return -EINVAL; + + /* save for later */ + alloc->imported.umm.sgt = sgt; + + pa = kbase_get_gpu_phy_pages(reg); + KBASE_DEBUG_ASSERT(pa); + + for_each_sg(sgt->sgl, s, sgt->nents, i) { + int j; + size_t pages = PFN_UP(MALI_SG_DMA_LEN(s)); + + WARN_ONCE(MALI_SG_DMA_LEN(s) & (PAGE_SIZE-1), + "MALI_SG_DMA_LEN(s)=%u is not a multiple of PAGE_SIZE\n", + MALI_SG_DMA_LEN(s)); + + WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE-1), + "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n", + (unsigned long long) sg_dma_address(s)); + + for (j = 0; (j < pages) && (count < reg->nr_pages); j++, + count++) + *pa++ = as_tagged(sg_dma_address(s) + + (j << PAGE_SHIFT)); + WARN_ONCE(j < pages, + "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n", + alloc->imported.umm.dma_buf->size); + } + + if (!(reg->flags & KBASE_REG_IMPORT_PAD) && + WARN_ONCE(count < reg->nr_pages, + "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n", + alloc->imported.umm.dma_buf->size)) { + err = -EINVAL; + goto err_unmap_attachment; + } + + /* Update nents as we now have pages to map */ + alloc->nents = reg->nr_pages; + + err = kbase_mmu_insert_pages(kctx, reg->start_pfn, + kbase_get_gpu_phy_pages(reg), + count, + reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD); + if (err) + goto err_unmap_attachment; + + if (reg->flags & KBASE_REG_IMPORT_PAD) { + err = kbase_mmu_insert_single_page(kctx, + reg->start_pfn + count, + kctx->aliasing_sink_page, + reg->nr_pages - count, + (reg->flags | KBASE_REG_GPU_RD) & + ~KBASE_REG_GPU_WR); + if (err) + goto err_teardown_orig_pages; + } + + return 0; + +err_teardown_orig_pages: + kbase_mmu_teardown_pages(kctx, reg->start_pfn, count); +err_unmap_attachment: + dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, + alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); + alloc->imported.umm.sgt = NULL; + + return err; +} + +static void kbase_jd_umm_unmap(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc) +{ + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(alloc); + KBASE_DEBUG_ASSERT(alloc->imported.umm.dma_attachment); + KBASE_DEBUG_ASSERT(alloc->imported.umm.sgt); + dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, + alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); + alloc->imported.umm.sgt = NULL; + alloc->nents = 0; +} +#endif /* CONFIG_DMA_SHARED_BUFFER */ + +#if (defined(CONFIG_KDS) && defined(CONFIG_UMP)) \ + || defined(CONFIG_DMA_SHARED_BUFFER_USES_KDS) +static void add_kds_resource(struct kds_resource *kds_res, + struct kds_resource **kds_resources, u32 *kds_res_count, + unsigned long *kds_access_bitmap, bool exclusive) +{ + u32 i; + + for (i = 0; i < *kds_res_count; i++) { + /* Duplicate resource, ignore */ + if (kds_resources[i] == kds_res) + return; + } + + kds_resources[*kds_res_count] = kds_res; + if (exclusive) + set_bit(*kds_res_count, kds_access_bitmap); + (*kds_res_count)++; +} +#endif + +struct kbase_mem_phy_alloc *kbase_map_external_resource( + struct kbase_context *kctx, struct kbase_va_region *reg, + struct mm_struct *locked_mm +#ifdef CONFIG_KDS + , u32 *kds_res_count, struct kds_resource **kds_resources, + unsigned long *kds_access_bitmap, bool exclusive +#endif + ) +{ + int err; + + /* decide what needs to happen for this resource */ + switch (reg->gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { + if (reg->gpu_alloc->imported.user_buf.mm != locked_mm) + goto exit; + + reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++; + if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) { + err = kbase_jd_user_buf_map(kctx, reg); + if (err) { + reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--; + goto exit; + } + } + } + break; + case KBASE_MEM_TYPE_IMPORTED_UMP: { +#if defined(CONFIG_KDS) && defined(CONFIG_UMP) + if (kds_res_count) { + struct kds_resource *kds_res; + + kds_res = ump_dd_kds_resource_get( + reg->gpu_alloc->imported.ump_handle); + if (kds_res) + add_kds_resource(kds_res, kds_resources, + kds_res_count, + kds_access_bitmap, exclusive); + } +#endif /*defined(CONFIG_KDS) && defined(CONFIG_UMP) */ + break; + } +#ifdef CONFIG_DMA_SHARED_BUFFER + case KBASE_MEM_TYPE_IMPORTED_UMM: { +#ifdef CONFIG_DMA_SHARED_BUFFER_USES_KDS + if (kds_res_count) { + struct kds_resource *kds_res; + + kds_res = get_dma_buf_kds_resource( + reg->gpu_alloc->imported.umm.dma_buf); + if (kds_res) + add_kds_resource(kds_res, kds_resources, + kds_res_count, + kds_access_bitmap, exclusive); + } +#endif + reg->gpu_alloc->imported.umm.current_mapping_usage_count++; + if (1 == reg->gpu_alloc->imported.umm.current_mapping_usage_count) { + err = kbase_jd_umm_map(kctx, reg); + if (err) { + reg->gpu_alloc->imported.umm.current_mapping_usage_count--; + goto exit; + } + } + break; + } +#endif + default: + goto exit; + } + + return kbase_mem_phy_alloc_get(reg->gpu_alloc); +exit: + return NULL; +} + +void kbase_unmap_external_resource(struct kbase_context *kctx, + struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc) +{ + switch (alloc->type) { +#ifdef CONFIG_DMA_SHARED_BUFFER + case KBASE_MEM_TYPE_IMPORTED_UMM: { + alloc->imported.umm.current_mapping_usage_count--; + + if (0 == alloc->imported.umm.current_mapping_usage_count) { + if (reg && reg->gpu_alloc == alloc) { + int err; + + err = kbase_mmu_teardown_pages( + kctx, + reg->start_pfn, + alloc->nents); + WARN_ON(err); + } + + kbase_jd_umm_unmap(kctx, alloc); + } + } + break; +#endif /* CONFIG_DMA_SHARED_BUFFER */ + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { + alloc->imported.user_buf.current_mapping_usage_count--; + + if (0 == alloc->imported.user_buf.current_mapping_usage_count) { + bool writeable = true; + + if (reg && reg->gpu_alloc == alloc) + kbase_mmu_teardown_pages( + kctx, + reg->start_pfn, + kbase_reg_current_backed_size(reg)); + + if (reg && ((reg->flags & KBASE_REG_GPU_WR) == 0)) + writeable = false; + + kbase_jd_user_buf_unmap(kctx, alloc, writeable); + } + } + break; + default: + break; + } + kbase_mem_phy_alloc_put(alloc); +} + +struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( + struct kbase_context *kctx, u64 gpu_addr) +{ + struct kbase_ctx_ext_res_meta *meta = NULL; + struct kbase_ctx_ext_res_meta *walker; + + lockdep_assert_held(&kctx->reg_lock); + + /* + * Walk the per context external resource metadata list for the + * metadata which matches the region which is being acquired. + */ + list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) { + if (walker->gpu_addr == gpu_addr) { + meta = walker; + break; + } + } + + /* No metadata exists so create one. */ + if (!meta) { + struct kbase_va_region *reg; + + /* Find the region */ + reg = kbase_region_tracker_find_region_enclosing_address( + kctx, gpu_addr); + if (NULL == reg || (reg->flags & KBASE_REG_FREE)) + goto failed; + + /* Allocate the metadata object */ + meta = kzalloc(sizeof(*meta), GFP_KERNEL); + if (!meta) + goto failed; + + /* + * Fill in the metadata object and acquire a reference + * for the physical resource. + */ + meta->alloc = kbase_map_external_resource(kctx, reg, NULL +#ifdef CONFIG_KDS + , NULL, NULL, + NULL, false +#endif + ); + + if (!meta->alloc) + goto fail_map; + + meta->gpu_addr = reg->start_pfn << PAGE_SHIFT; + + list_add(&meta->ext_res_node, &kctx->ext_res_meta_head); + } + + return meta; + +fail_map: + kfree(meta); +failed: + return NULL; +} + +bool kbase_sticky_resource_release(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) +{ + struct kbase_ctx_ext_res_meta *walker; + struct kbase_va_region *reg; + + lockdep_assert_held(&kctx->reg_lock); + + /* Search of the metadata if one isn't provided. */ + if (!meta) { + /* + * Walk the per context external resource metadata list for the + * metadata which matches the region which is being released. + */ + list_for_each_entry(walker, &kctx->ext_res_meta_head, + ext_res_node) { + if (walker->gpu_addr == gpu_addr) { + meta = walker; + break; + } + } + } + + /* No metadata so just return. */ + if (!meta) + return false; + + /* Drop the physical memory reference and free the metadata. */ + reg = kbase_region_tracker_find_region_enclosing_address( + kctx, + meta->gpu_addr); + + kbase_unmap_external_resource(kctx, reg, meta->alloc); + list_del(&meta->ext_res_node); + kfree(meta); + + return true; +} + +int kbase_sticky_resource_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->ext_res_meta_head); + + return 0; +} + +void kbase_sticky_resource_term(struct kbase_context *kctx) +{ + struct kbase_ctx_ext_res_meta *walker; + + lockdep_assert_held(&kctx->reg_lock); + + /* + * Free any sticky resources which haven't been unmapped. + * + * Note: + * We don't care about refcounts at this point as no future + * references to the meta data will be made. + * Region termination would find these if we didn't free them + * here, but it's more efficient if we do the clean up here. + */ + while (!list_empty(&kctx->ext_res_meta_head)) { + walker = list_first_entry(&kctx->ext_res_meta_head, + struct kbase_ctx_ext_res_meta, ext_res_node); + + kbase_sticky_resource_release(kctx, walker, 0); + } +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.h new file mode 100755 index 000000000000..f2fd75e2018b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem.h @@ -0,0 +1,1142 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mem.h + * Base kernel memory APIs + */ + +#ifndef _KBASE_MEM_H_ +#define _KBASE_MEM_H_ + +#ifndef _KBASE_H_ +#error "Don't include this file directly, use mali_kbase.h instead" +#endif + +#include +#ifdef CONFIG_KDS +#include +#endif /* CONFIG_KDS */ +#ifdef CONFIG_UMP +#include +#endif /* CONFIG_UMP */ +#include "mali_base_kernel.h" +#include +#include "mali_kbase_pm.h" +#include "mali_kbase_defs.h" +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) +#include "mali_kbase_gator.h" +#endif +/* Required for kbase_mem_evictable_unmake */ +#include "mali_kbase_mem_linux.h" + +/* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */ +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */ + +/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages. +The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and +page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table +updates and generates duplicate page faults as the page table information used by the MMU is not valid. */ +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */ + +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */ + +/* This must always be a power of 2 */ +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2) +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316) +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630) +/** + * A CPU mapping + */ +struct kbase_cpu_mapping { + struct list_head mappings_list; + struct kbase_mem_phy_alloc *alloc; + struct kbase_context *kctx; + struct kbase_va_region *region; + int count; + int free_on_close; +}; + +enum kbase_memory_type { + KBASE_MEM_TYPE_NATIVE, + KBASE_MEM_TYPE_IMPORTED_UMP, + KBASE_MEM_TYPE_IMPORTED_UMM, + KBASE_MEM_TYPE_IMPORTED_USER_BUF, + KBASE_MEM_TYPE_ALIAS, + KBASE_MEM_TYPE_TB, + KBASE_MEM_TYPE_RAW +}; + +/* internal structure, mirroring base_mem_aliasing_info, + * but with alloc instead of a gpu va (handle) */ +struct kbase_aliased { + struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */ + u64 offset; /* in pages */ + u64 length; /* in pages */ +}; + +/** + * @brief Physical pages tracking object properties + */ +#define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED (1ul << 0) +#define KBASE_MEM_PHY_ALLOC_LARGE (1ul << 1) + +/* physical pages tracking object. + * Set up to track N pages. + * N not stored here, the creator holds that info. + * This object only tracks how many elements are actually valid (present). + * Changing of nents or *pages should only happen if the kbase_mem_phy_alloc is not + * shared with another region or client. CPU mappings are OK to exist when changing, as + * long as the tracked mappings objects are updated as part of the change. + */ +struct kbase_mem_phy_alloc { + struct kref kref; /* number of users of this alloc */ + atomic_t gpu_mappings; + size_t nents; /* 0..N */ + struct tagged_addr *pages; /* N elements, only 0..nents are valid */ + + /* kbase_cpu_mappings */ + struct list_head mappings; + + /* Node used to store this allocation on the eviction list */ + struct list_head evict_node; + /* Physical backing size when the pages where evicted */ + size_t evicted; + /* + * Back reference to the region structure which created this + * allocation, or NULL if it has been freed. + */ + struct kbase_va_region *reg; + + /* type of buffer */ + enum kbase_memory_type type; + + unsigned long properties; + + struct list_head zone_cache; + + /* member in union valid based on @a type */ + union { +#ifdef CONFIG_UMP + ump_dd_handle ump_handle; +#endif /* CONFIG_UMP */ +#if defined(CONFIG_DMA_SHARED_BUFFER) + struct { + struct dma_buf *dma_buf; + struct dma_buf_attachment *dma_attachment; + unsigned int current_mapping_usage_count; + struct sg_table *sgt; + } umm; +#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ + struct { + u64 stride; + size_t nents; + struct kbase_aliased *aliased; + } alias; + /* Used by type = (KBASE_MEM_TYPE_NATIVE, KBASE_MEM_TYPE_TB) */ + struct kbase_context *kctx; + struct kbase_alloc_import_user_buf { + unsigned long address; + unsigned long size; + unsigned long nr_pages; + struct page **pages; + /* top bit (1<<31) of current_mapping_usage_count + * specifies that this import was pinned on import + * See PINNED_ON_IMPORT + */ + u32 current_mapping_usage_count; + struct mm_struct *mm; + dma_addr_t *dma_addrs; + } user_buf; + } imported; +}; + +/* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is + * used to signify that a buffer was pinned when it was imported. Since the + * reference count is limited by the number of atoms that can be submitted at + * once there should be no danger of overflowing into this bit. + * Stealing the top bit also has the benefit that + * current_mapping_usage_count != 0 if and only if the buffer is mapped. + */ +#define PINNED_ON_IMPORT (1<<31) + +static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) +{ + KBASE_DEBUG_ASSERT(alloc); + /* we only track mappings of NATIVE buffers */ + if (alloc->type == KBASE_MEM_TYPE_NATIVE) + atomic_inc(&alloc->gpu_mappings); +} + +static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc *alloc) +{ + KBASE_DEBUG_ASSERT(alloc); + /* we only track mappings of NATIVE buffers */ + if (alloc->type == KBASE_MEM_TYPE_NATIVE) + if (0 > atomic_dec_return(&alloc->gpu_mappings)) { + pr_err("Mismatched %s:\n", __func__); + dump_stack(); + } +} + +/** + * kbase_mem_is_imported - Indicate whether a memory type is imported + * + * @type: the memory type + * + * Return: true if the memory type is imported, false otherwise + */ +static inline bool kbase_mem_is_imported(enum kbase_memory_type type) +{ + return (type == KBASE_MEM_TYPE_IMPORTED_UMP) || + (type == KBASE_MEM_TYPE_IMPORTED_UMM) || + (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); +} + +void kbase_mem_kref_free(struct kref *kref); + +int kbase_mem_init(struct kbase_device *kbdev); +void kbase_mem_halt(struct kbase_device *kbdev); +void kbase_mem_term(struct kbase_device *kbdev); + +static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc) +{ + kref_get(&alloc->kref); + return alloc; +} + +static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc) +{ + kref_put(&alloc->kref, kbase_mem_kref_free); + return NULL; +} + +/** + * A GPU memory region, and attributes for CPU mappings. + */ +struct kbase_va_region { + struct rb_node rblink; + struct list_head link; + + struct kbase_context *kctx; /* Backlink to base context */ + + u64 start_pfn; /* The PFN in GPU space */ + size_t nr_pages; + +/* Free region */ +#define KBASE_REG_FREE (1ul << 0) +/* CPU write access */ +#define KBASE_REG_CPU_WR (1ul << 1) +/* GPU write access */ +#define KBASE_REG_GPU_WR (1ul << 2) +/* No eXecute flag */ +#define KBASE_REG_GPU_NX (1ul << 3) +/* Is CPU cached? */ +#define KBASE_REG_CPU_CACHED (1ul << 4) +/* Is GPU cached? */ +#define KBASE_REG_GPU_CACHED (1ul << 5) + +#define KBASE_REG_GROWABLE (1ul << 6) +/* Can grow on pf? */ +#define KBASE_REG_PF_GROW (1ul << 7) + +/* Bit 8 is unused */ + +/* inner shareable coherency */ +#define KBASE_REG_SHARE_IN (1ul << 9) +/* inner & outer shareable coherency */ +#define KBASE_REG_SHARE_BOTH (1ul << 10) + +/* Space for 4 different zones */ +#define KBASE_REG_ZONE_MASK (3ul << 11) +#define KBASE_REG_ZONE(x) (((x) & 3) << 11) + +/* GPU read access */ +#define KBASE_REG_GPU_RD (1ul<<13) +/* CPU read access */ +#define KBASE_REG_CPU_RD (1ul<<14) + +/* Index of chosen MEMATTR for this region (0..7) */ +#define KBASE_REG_MEMATTR_MASK (7ul << 16) +#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16) +#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16) + +#define KBASE_REG_SECURE (1ul << 19) + +#define KBASE_REG_DONT_NEED (1ul << 20) + +/* Imported buffer is padded? */ +#define KBASE_REG_IMPORT_PAD (1ul << 21) + +/* Bit 22 is reserved. + * + * Do not remove, use the next unreserved bit for new flags */ +#define KBASE_REG_RESERVED_BIT_22 (1ul << 22) + +#define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0) + +/* only used with 32-bit clients */ +/* + * On a 32bit platform, custom VA should be wired from (4GB + shader region) + * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface + * limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference). + * So we put the default limit to the maximum possible on Linux and shrink + * it down, if required by the GPU, during initialization. + */ + +/* + * Dedicated 16MB region for shader code: + * VA range 0x101000000-0x102000000 + */ +#define KBASE_REG_ZONE_EXEC KBASE_REG_ZONE(1) +#define KBASE_REG_ZONE_EXEC_BASE (0x101000000ULL >> PAGE_SHIFT) +#define KBASE_REG_ZONE_EXEC_SIZE ((16ULL * 1024 * 1024) >> PAGE_SHIFT) + +#define KBASE_REG_ZONE_CUSTOM_VA KBASE_REG_ZONE(2) +#define KBASE_REG_ZONE_CUSTOM_VA_BASE (KBASE_REG_ZONE_EXEC_BASE + KBASE_REG_ZONE_EXEC_SIZE) /* Starting after KBASE_REG_ZONE_EXEC */ +#define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) +/* end 32-bit clients only */ + + unsigned long flags; + + size_t extent; /* nr of pages alloc'd on PF */ + + struct kbase_mem_phy_alloc *cpu_alloc; /* the one alloc object we mmap to the CPU when mapping this region */ + struct kbase_mem_phy_alloc *gpu_alloc; /* the one alloc object we mmap to the GPU when mapping this region */ + + /* non-NULL if this memory object is a kds_resource */ + struct kds_resource *kds_res; + + /* List head used to store the region in the JIT allocation pool */ + struct list_head jit_node; +}; + +/* Common functions */ +static inline struct tagged_addr *kbase_get_cpu_phy_pages( + struct kbase_va_region *reg) +{ + KBASE_DEBUG_ASSERT(reg); + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); + + return reg->cpu_alloc->pages; +} + +static inline struct tagged_addr *kbase_get_gpu_phy_pages( + struct kbase_va_region *reg) +{ + KBASE_DEBUG_ASSERT(reg); + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); + + return reg->gpu_alloc->pages; +} + +static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg) +{ + KBASE_DEBUG_ASSERT(reg); + /* if no alloc object the backed size naturally is 0 */ + if (!reg->cpu_alloc) + return 0; + + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); + + return reg->cpu_alloc->nents; +} + +#define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */ + +static inline struct kbase_mem_phy_alloc *kbase_alloc_create(size_t nr_pages, enum kbase_memory_type type) +{ + struct kbase_mem_phy_alloc *alloc; + size_t alloc_size = sizeof(*alloc) + sizeof(*alloc->pages) * nr_pages; + size_t per_page_size = sizeof(*alloc->pages); + + /* Imported pages may have page private data already in use */ + if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { + alloc_size += nr_pages * + sizeof(*alloc->imported.user_buf.dma_addrs); + per_page_size += sizeof(*alloc->imported.user_buf.dma_addrs); + } + + /* + * Prevent nr_pages*per_page_size + sizeof(*alloc) from + * wrapping around. + */ + if (nr_pages > ((((size_t) -1) - sizeof(*alloc)) + / per_page_size)) + return ERR_PTR(-ENOMEM); + + /* Allocate based on the size to reduce internal fragmentation of vmem */ + if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) + alloc = vzalloc(alloc_size); + else + alloc = kzalloc(alloc_size, GFP_KERNEL); + + if (!alloc) + return ERR_PTR(-ENOMEM); + + /* Store allocation method */ + if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) + alloc->properties |= KBASE_MEM_PHY_ALLOC_LARGE; + + kref_init(&alloc->kref); + atomic_set(&alloc->gpu_mappings, 0); + alloc->nents = 0; + alloc->pages = (void *)(alloc + 1); + INIT_LIST_HEAD(&alloc->mappings); + alloc->type = type; + INIT_LIST_HEAD(&alloc->zone_cache); + + if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) + alloc->imported.user_buf.dma_addrs = + (void *) (alloc->pages + nr_pages); + + return alloc; +} + +static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, + struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(reg); + KBASE_DEBUG_ASSERT(!reg->cpu_alloc); + KBASE_DEBUG_ASSERT(!reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE); + + reg->cpu_alloc = kbase_alloc_create(reg->nr_pages, + KBASE_MEM_TYPE_NATIVE); + if (IS_ERR(reg->cpu_alloc)) + return PTR_ERR(reg->cpu_alloc); + else if (!reg->cpu_alloc) + return -ENOMEM; + reg->cpu_alloc->imported.kctx = kctx; + INIT_LIST_HEAD(®->cpu_alloc->evict_node); + if (kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE) + && (reg->flags & KBASE_REG_CPU_CACHED)) { + reg->gpu_alloc = kbase_alloc_create(reg->nr_pages, + KBASE_MEM_TYPE_NATIVE); + reg->gpu_alloc->imported.kctx = kctx; + INIT_LIST_HEAD(®->gpu_alloc->evict_node); + } else { + reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + } + + INIT_LIST_HEAD(®->jit_node); + reg->flags &= ~KBASE_REG_FREE; + return 0; +} + +static inline int kbase_atomic_add_pages(int num_pages, atomic_t *used_pages) +{ + int new_val = atomic_add_return(num_pages, used_pages); +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) + kbase_trace_mali_total_alloc_pages_change((long long int)new_val); +#endif + return new_val; +} + +static inline int kbase_atomic_sub_pages(int num_pages, atomic_t *used_pages) +{ + int new_val = atomic_sub_return(num_pages, used_pages); +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) + kbase_trace_mali_total_alloc_pages_change((long long int)new_val); +#endif + return new_val; +} + +/* + * Max size for kbdev memory pool (in pages) + */ +#define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT) + +/* + * Max size for kctx memory pool (in pages) + */ +#define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) + +/* + * The order required for a 2MB page allocation (2^order * 4KB = 2MB) + */ +#define KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER 9 + +/* + * The order required for a 4KB page allocation + */ +#define KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER 0 + +/** + * kbase_mem_pool_init - Create a memory pool for a kbase device + * @pool: Memory pool to initialize + * @max_size: Maximum number of free pages the pool can hold + * @order: Page order for physical page size (order=0=>4kB, order=9=>2MB) + * @kbdev: Kbase device where memory is used + * @next_pool: Pointer to the next pool or NULL. + * + * Allocations from @pool are in whole pages. Each @pool has a free list where + * pages can be quickly allocated from. The free list is initially empty and + * filled whenever pages are freed back to the pool. The number of free pages + * in the pool will in general not exceed @max_size, but the pool may in + * certain corner cases grow above @max_size. + * + * If @next_pool is not NULL, we will allocate from @next_pool before going to + * the kernel allocator. Similarily pages can spill over to @next_pool when + * @pool is full. Pages are zeroed before they spill over to another pool, to + * prevent leaking information between applications. + * + * A shrinker is registered so that Linux mm can reclaim pages from the pool as + * needed. + * + * Return: 0 on success, negative -errno on error + */ +int kbase_mem_pool_init(struct kbase_mem_pool *pool, + size_t max_size, + size_t order, + struct kbase_device *kbdev, + struct kbase_mem_pool *next_pool); + +/** + * kbase_mem_pool_term - Destroy a memory pool + * @pool: Memory pool to destroy + * + * Pages in the pool will spill over to @next_pool (if available) or freed to + * the kernel. + */ +void kbase_mem_pool_term(struct kbase_mem_pool *pool); + +/** + * kbase_mem_pool_alloc - Allocate a page from memory pool + * @pool: Memory pool to allocate from + * + * Allocations from the pool are made as follows: + * 1. If there are free pages in the pool, allocate a page from @pool. + * 2. Otherwise, if @next_pool is not NULL and has free pages, allocate a page + * from @next_pool. + * 3. Return NULL if no memory in the pool + * + * Return: Pointer to allocated page, or NULL if allocation failed. + */ +struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool); + +/** + * kbase_mem_pool_free - Free a page to memory pool + * @pool: Memory pool where page should be freed + * @page: Page to free to the pool + * @dirty: Whether some of the page may be dirty in the cache. + * + * Pages are freed to the pool as follows: + * 1. If @pool is not full, add @page to @pool. + * 2. Otherwise, if @next_pool is not NULL and not full, add @page to + * @next_pool. + * 3. Finally, free @page to the kernel. + */ +void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page, + bool dirty); + +/** + * kbase_mem_pool_alloc_pages - Allocate pages from memory pool + * @pool: Memory pool to allocate from + * @nr_pages: Number of pages to allocate + * @pages: Pointer to array where the physical address of the allocated + * pages will be stored. + * @partial_allowed: If fewer pages allocated is allowed + * + * Like kbase_mem_pool_alloc() but optimized for allocating many pages. + * + * Return: + * On success number of pages allocated (could be less than nr_pages if + * partial_allowed). + * On error an error code. + */ +int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages, + struct tagged_addr *pages, bool partial_allowed); + +/** + * kbase_mem_pool_free_pages - Free pages to memory pool + * @pool: Memory pool where pages should be freed + * @nr_pages: Number of pages to free + * @pages: Pointer to array holding the physical addresses of the pages to + * free. + * @dirty: Whether any pages may be dirty in the cache. + * @reclaimed: Whether the pages where reclaimable and thus should bypass + * the pool and go straight to the kernel. + * + * Like kbase_mem_pool_free() but optimized for freeing many pages. + */ +void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, + struct tagged_addr *pages, bool dirty, bool reclaimed); + +/** + * kbase_mem_pool_size - Get number of free pages in memory pool + * @pool: Memory pool to inspect + * + * Note: the size of the pool may in certain corner cases exceed @max_size! + * + * Return: Number of free pages in the pool + */ +static inline size_t kbase_mem_pool_size(struct kbase_mem_pool *pool) +{ + return READ_ONCE(pool->cur_size); +} + +/** + * kbase_mem_pool_max_size - Get maximum number of free pages in memory pool + * @pool: Memory pool to inspect + * + * Return: Maximum number of free pages in the pool + */ +static inline size_t kbase_mem_pool_max_size(struct kbase_mem_pool *pool) +{ + return pool->max_size; +} + + +/** + * kbase_mem_pool_set_max_size - Set maximum number of free pages in memory pool + * @pool: Memory pool to inspect + * @max_size: Maximum number of free pages the pool can hold + * + * If @max_size is reduced, the pool will be shrunk to adhere to the new limit. + * For details see kbase_mem_pool_shrink(). + */ +void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size); + +/** + * kbase_mem_pool_grow - Grow the pool + * @pool: Memory pool to grow + * @nr_to_grow: Number of pages to add to the pool + * + * Adds @nr_to_grow pages to the pool. Note that this may cause the pool to + * become larger than the maximum size specified. + * + * Returns: 0 on success, -ENOMEM if unable to allocate sufficent pages + */ +int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow); + +/** + * kbase_mem_pool_trim - Grow or shrink the pool to a new size + * @pool: Memory pool to trim + * @new_size: New number of pages in the pool + * + * If @new_size > @cur_size, fill the pool with new pages from the kernel, but + * not above the max_size for the pool. + * If @new_size < @cur_size, shrink the pool by freeing pages to the kernel. + */ +void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size); + +/** + * kbase_mem_alloc_page - Allocate a new page for a device + * @pool: Memory pool to allocate a page from + * + * Most uses should use kbase_mem_pool_alloc to allocate a page. However that + * function can fail in the event the pool is empty. + * + * Return: A new page or NULL if no memory + */ +struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool); + +int kbase_region_tracker_init(struct kbase_context *kctx); +int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages); +void kbase_region_tracker_term(struct kbase_context *kctx); + +struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr); + +/** + * @brief Check that a pointer is actually a valid region. + * + * Must be called with context lock held. + */ +struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr); + +struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone); +void kbase_free_alloced_region(struct kbase_va_region *reg); +int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); + +bool kbase_check_alloc_flags(unsigned long flags); +bool kbase_check_import_flags(unsigned long flags); + +/** + * kbase_update_region_flags - Convert user space flags to kernel region flags + * + * @kctx: kbase context + * @reg: The region to update the flags on + * @flags: The flags passed from user space + * + * The user space flag BASE_MEM_COHERENT_SYSTEM_REQUIRED will be rejected and + * this function will fail if the system does not support system coherency. + * + * Return: 0 if successful, -EINVAL if the flags are not supported + */ +int kbase_update_region_flags(struct kbase_context *kctx, + struct kbase_va_region *reg, unsigned long flags); + +void kbase_gpu_vm_lock(struct kbase_context *kctx); +void kbase_gpu_vm_unlock(struct kbase_context *kctx); + +int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size); + +int kbase_mmu_init(struct kbase_context *kctx); +void kbase_mmu_term(struct kbase_context *kctx); + +phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx); +void kbase_mmu_free_pgd(struct kbase_context *kctx); +int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags); +int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags); +int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr phys, size_t nr, + unsigned long flags); + +int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr); +int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags); + +/** + * @brief Register region and map it on the GPU. + * + * Call kbase_add_va_region() and map the region on the GPU. + */ +int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); + +/** + * @brief Remove the region from the GPU and unregister it. + * + * Must be called with context lock held. + */ +int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg); + +/** + * The caller has the following locking conditions: + * - It must hold kbase_device->mmu_hw_mutex + * - It must hold the hwaccess_lock + */ +void kbase_mmu_update(struct kbase_context *kctx); + +/** + * kbase_mmu_disable() - Disable the MMU for a previously active kbase context. + * @kctx: Kbase context + * + * Disable and perform the required cache maintenance to remove the all + * data from provided kbase context from the GPU caches. + * + * The caller has the following locking conditions: + * - It must hold kbase_device->mmu_hw_mutex + * - It must hold the hwaccess_lock + */ +void kbase_mmu_disable(struct kbase_context *kctx); + +/** + * kbase_mmu_disable_as() - Set the MMU to unmapped mode for the specified + * address space. + * @kbdev: Kbase device + * @as_nr: The address space number to set to unmapped. + * + * This function must only be called during reset/power-up and it used to + * ensure the registers are in a known state. + * + * The caller must hold kbdev->mmu_hw_mutex. + */ +void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr); + +void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); + +/** Dump the MMU tables to a buffer + * + * This function allocates a buffer (of @c nr_pages pages) to hold a dump of the MMU tables and fills it. If the + * buffer is too small then the return value will be NULL. + * + * The GPU vm lock must be held when calling this function. + * + * The buffer returned should be freed with @ref vfree when it is no longer required. + * + * @param[in] kctx The kbase context to dump + * @param[in] nr_pages The number of pages to allocate for the buffer. + * + * @return The address of the buffer containing the MMU dump or NULL on error (including if the @c nr_pages is too + * small) + */ +void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); + +/** + * kbase_sync_now - Perform cache maintenance on a memory region + * + * @kctx: The kbase context of the region + * @sset: A syncset structure describing the region and direction of the + * synchronisation required + * + * Return: 0 on success or error code + */ +int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset); +void kbase_sync_single(struct kbase_context *kctx, struct tagged_addr cpu_pa, + struct tagged_addr gpu_pa, off_t offset, size_t size, + enum kbase_sync_type sync_fn); +void kbase_pre_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); +void kbase_post_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); + +/* OS specific functions */ +int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr); +int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg); +void kbase_os_mem_map_lock(struct kbase_context *kctx); +void kbase_os_mem_map_unlock(struct kbase_context *kctx); + +/** + * @brief Update the memory allocation counters for the current process + * + * OS specific call to updates the current memory allocation counters for the current process with + * the supplied delta. + * + * @param[in] kctx The kbase context + * @param[in] pages The desired delta to apply to the memory usage counters. + */ + +void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages); + +/** + * @brief Add to the memory allocation counters for the current process + * + * OS specific call to add to the current memory allocation counters for the current process by + * the supplied amount. + * + * @param[in] kctx The kernel base context used for the allocation. + * @param[in] pages The desired delta to apply to the memory usage counters. + */ + +static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages) +{ + kbasep_os_process_page_usage_update(kctx, pages); +} + +/** + * @brief Subtract from the memory allocation counters for the current process + * + * OS specific call to subtract from the current memory allocation counters for the current process by + * the supplied amount. + * + * @param[in] kctx The kernel base context used for the allocation. + * @param[in] pages The desired delta to apply to the memory usage counters. + */ + +static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages) +{ + kbasep_os_process_page_usage_update(kctx, 0 - pages); +} + +/** + * kbasep_find_enclosing_cpu_mapping_offset() - Find the offset of the CPU + * mapping of a memory allocation containing a given address range + * + * Searches for a CPU mapping of any part of any region that fully encloses the + * CPU virtual address range specified by @uaddr and @size. Returns a failure + * indication if only part of the address range lies within a CPU mapping. + * + * @kctx: The kernel base context used for the allocation. + * @uaddr: Start of the CPU virtual address range. + * @size: Size of the CPU virtual address range (in bytes). + * @offset: The offset from the start of the allocation to the specified CPU + * virtual address. + * + * Return: 0 if offset was obtained successfully. Error code otherwise. + */ +int kbasep_find_enclosing_cpu_mapping_offset( + struct kbase_context *kctx, + unsigned long uaddr, size_t size, u64 *offset); + +enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer); +void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); +void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); + +/** +* @brief Allocates physical pages. +* +* Allocates \a nr_pages_requested and updates the alloc object. +* +* @param[in] alloc allocation object to add pages to +* @param[in] nr_pages_requested number of physical pages to allocate +* +* @return 0 if all pages have been successfully allocated. Error code otherwise +*/ +int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_requested); + +/** +* @brief Free physical pages. +* +* Frees \a nr_pages and updates the alloc object. +* +* @param[in] alloc allocation object to free pages from +* @param[in] nr_pages_to_free number of physical pages to free +*/ +int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free); + +static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) +{ + SetPagePrivate(p); + if (sizeof(dma_addr_t) > sizeof(p->private)) { + /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the + * private field stays the same. So we have to be clever and + * use the fact that we only store DMA addresses of whole pages, + * so the low bits should be zero */ + KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1))); + set_page_private(p, dma_addr >> PAGE_SHIFT); + } else { + set_page_private(p, dma_addr); + } +} + +static inline dma_addr_t kbase_dma_addr(struct page *p) +{ + if (sizeof(dma_addr_t) > sizeof(p->private)) + return ((dma_addr_t)page_private(p)) << PAGE_SHIFT; + + return (dma_addr_t)page_private(p); +} + +static inline void kbase_clear_dma_addr(struct page *p) +{ + ClearPagePrivate(p); +} + +/** +* @brief Process a bus or page fault. +* +* This function will process a fault on a specific address space +* +* @param[in] kbdev The @ref kbase_device the fault happened on +* @param[in] kctx The @ref kbase_context for the faulting address space if +* one was found. +* @param[in] as The address space that has the fault +*/ +void kbase_mmu_interrupt_process(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_as *as); + +/** + * @brief Process a page fault. + * + * @param[in] data work_struct passed by queue_work() + */ +void page_fault_worker(struct work_struct *data); + +/** + * @brief Process a bus fault. + * + * @param[in] data work_struct passed by queue_work() + */ +void bus_fault_worker(struct work_struct *data); + +/** + * @brief Flush MMU workqueues. + * + * This function will cause any outstanding page or bus faults to be processed. + * It should be called prior to powering off the GPU. + * + * @param[in] kbdev Device pointer + */ +void kbase_flush_mmu_wqs(struct kbase_device *kbdev); + +/** + * kbase_sync_single_for_device - update physical memory and give GPU ownership + * @kbdev: Device pointer + * @handle: DMA address of region + * @size: Size of region to sync + * @dir: DMA data direction + */ + +void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir); + +/** + * kbase_sync_single_for_cpu - update physical memory and give CPU ownership + * @kbdev: Device pointer + * @handle: DMA address of region + * @size: Size of region to sync + * @dir: DMA data direction + */ + +void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir); + +#ifdef CONFIG_DEBUG_FS +/** + * kbase_jit_debugfs_init - Add per context debugfs entry for JIT. + * @kctx: kbase context + */ +void kbase_jit_debugfs_init(struct kbase_context *kctx); +#endif /* CONFIG_DEBUG_FS */ + +/** + * kbase_jit_init - Initialize the JIT memory pool management + * @kctx: kbase context + * + * Returns zero on success or negative error number on failure. + */ +int kbase_jit_init(struct kbase_context *kctx); + +/** + * kbase_jit_allocate - Allocate JIT memory + * @kctx: kbase context + * @info: JIT allocation information + * + * Return: JIT allocation on success or NULL on failure. + */ +struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, + struct base_jit_alloc_info *info); + +/** + * kbase_jit_free - Free a JIT allocation + * @kctx: kbase context + * @reg: JIT allocation + * + * Frees a JIT allocation and places it into the free pool for later reuse. + */ +void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg); + +/** + * kbase_jit_backing_lost - Inform JIT that an allocation has lost backing + * @reg: JIT allocation + */ +void kbase_jit_backing_lost(struct kbase_va_region *reg); + +/** + * kbase_jit_evict - Evict a JIT allocation from the pool + * @kctx: kbase context + * + * Evict the least recently used JIT allocation from the pool. This can be + * required if normal VA allocations are failing due to VA exhaustion. + * + * Return: True if a JIT allocation was freed, false otherwise. + */ +bool kbase_jit_evict(struct kbase_context *kctx); + +/** + * kbase_jit_term - Terminate the JIT memory pool management + * @kctx: kbase context + */ +void kbase_jit_term(struct kbase_context *kctx); + +/** + * kbase_map_external_resource - Map an external resource to the GPU. + * @kctx: kbase context. + * @reg: The region to map. + * @locked_mm: The mm_struct which has been locked for this operation. + * @kds_res_count: The number of KDS resources. + * @kds_resources: Array of KDS resources. + * @kds_access_bitmap: Access bitmap for KDS. + * @exclusive: If the KDS resource requires exclusive access. + * + * Return: The physical allocation which backs the region on success or NULL + * on failure. + */ +struct kbase_mem_phy_alloc *kbase_map_external_resource( + struct kbase_context *kctx, struct kbase_va_region *reg, + struct mm_struct *locked_mm +#ifdef CONFIG_KDS + , u32 *kds_res_count, struct kds_resource **kds_resources, + unsigned long *kds_access_bitmap, bool exclusive +#endif + ); + +/** + * kbase_unmap_external_resource - Unmap an external resource from the GPU. + * @kctx: kbase context. + * @reg: The region to unmap or NULL if it has already been released. + * @alloc: The physical allocation being unmapped. + */ +void kbase_unmap_external_resource(struct kbase_context *kctx, + struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); + +/** + * kbase_sticky_resource_init - Initialize sticky resource management. + * @kctx: kbase context + * + * Returns zero on success or negative error number on failure. + */ +int kbase_sticky_resource_init(struct kbase_context *kctx); + +/** + * kbase_sticky_resource_acquire - Acquire a reference on a sticky resource. + * @kctx: kbase context. + * @gpu_addr: The GPU address of the external resource. + * + * Return: The metadata object which represents the binding between the + * external resource and the kbase context on success or NULL on failure. + */ +struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( + struct kbase_context *kctx, u64 gpu_addr); + +/** + * kbase_sticky_resource_release - Release a reference on a sticky resource. + * @kctx: kbase context. + * @meta: Binding metadata. + * @gpu_addr: GPU address of the external resource. + * + * If meta is NULL then gpu_addr will be used to scan the metadata list and + * find the matching metadata (if any), otherwise the provided meta will be + * used and gpu_addr will be ignored. + * + * Return: True if the release found the metadata and the reference was dropped. + */ +bool kbase_sticky_resource_release(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); + +/** + * kbase_sticky_resource_term - Terminate sticky resource management. + * @kctx: kbase context + */ +void kbase_sticky_resource_term(struct kbase_context *kctx); + +/** + * kbase_zone_cache_update - Update the memory zone cache after new pages have + * been added. + * @alloc: The physical memory allocation to build the cache for. + * @start_offset: Offset to where the new pages start. + * + * Updates an existing memory zone cache, updating the counters for the + * various zones. + * If the memory allocation doesn't already have a zone cache assume that + * one isn't created and thus don't do anything. + * + * Return: Zero cache was updated, negative error code on error. + */ +int kbase_zone_cache_update(struct kbase_mem_phy_alloc *alloc, + size_t start_offset); + +/** + * kbase_zone_cache_build - Build the memory zone cache. + * @alloc: The physical memory allocation to build the cache for. + * + * Create a new zone cache for the provided physical memory allocation if + * one doesn't already exist, if one does exist then just return. + * + * Return: Zero if the zone cache was created, negative error code on error. + */ +int kbase_zone_cache_build(struct kbase_mem_phy_alloc *alloc); + +/** + * kbase_zone_cache_clear - Clear the memory zone cache. + * @alloc: The physical memory allocation to clear the cache on. + */ +void kbase_zone_cache_clear(struct kbase_mem_phy_alloc *alloc); + +#endif /* _KBASE_MEM_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.c new file mode 100755 index 000000000000..842444c9b0bd --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.c @@ -0,0 +1,2678 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mem_linux.c + * Base kernel memory APIs, Linux implementation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) +#include +#endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */ +#ifdef CONFIG_DMA_SHARED_BUFFER +#include +#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ +#include +#include + +#include +#include +#include +#include + +static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma); + +/** + * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation + * @kctx: Context the region belongs to + * @reg: The GPU region + * @new_pages: The number of pages after the shrink + * @old_pages: The number of pages before the shrink + * + * Shrink (or completely remove) all CPU mappings which reference the shrunk + * part of the allocation. + * + * Note: Caller must be holding the processes mmap_sem lock. + */ +static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages); + +/** + * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation + * @kctx: Context the region belongs to + * @reg: The GPU region or NULL if there isn't one + * @new_pages: The number of pages after the shrink + * @old_pages: The number of pages before the shrink + * + * Return: 0 on success, negative -errno on error + * + * Unmap the shrunk pages from the GPU mapping. Note that the size of the region + * itself is unmodified as we still need to reserve the VA, only the page tables + * will be modified by this function. + */ +static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages); + +struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, + u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, + u64 *gpu_va) +{ + int zone; + int gpu_pc_bits; + struct kbase_va_region *reg; + struct device *dev; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(flags); + KBASE_DEBUG_ASSERT(gpu_va); + + dev = kctx->kbdev->dev; + *gpu_va = 0; /* return 0 on failure */ + + gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; + + if (0 == va_pages) { + dev_warn(dev, "kbase_mem_alloc called with 0 va_pages!"); + goto bad_size; + } + + if (va_pages > (U64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + if (!kbase_check_alloc_flags(*flags)) { + dev_warn(dev, + "kbase_mem_alloc called with bad flags (%llx)", + (unsigned long long)*flags); + goto bad_flags; + } + + if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable"); + goto bad_flags; + } + if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ + *flags &= ~BASE_MEM_COHERENT_SYSTEM; + } + + /* Limit GPU executable allocs to GPU PC size */ + if ((*flags & BASE_MEM_PROT_GPU_EX) && + (va_pages > (1ULL << gpu_pc_bits >> PAGE_SHIFT))) + goto bad_ex_size; + + /* find out which VA zone to use */ + if (*flags & BASE_MEM_SAME_VA) + zone = KBASE_REG_ZONE_SAME_VA; + else if (*flags & BASE_MEM_PROT_GPU_EX) + zone = KBASE_REG_ZONE_EXEC; + else + zone = KBASE_REG_ZONE_CUSTOM_VA; + + reg = kbase_alloc_free_region(kctx, 0, va_pages, zone); + if (!reg) { + dev_err(dev, "Failed to allocate free region"); + goto no_region; + } + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + if (kbase_reg_prepare_native(reg, kctx) != 0) { + dev_err(dev, "Failed to prepare region"); + goto prepare_failed; + } + + if (*flags & BASE_MEM_GROW_ON_GPF) + reg->extent = extent; + else + reg->extent = 0; + + if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) { + dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)", + (unsigned long long)commit_pages, + (unsigned long long)va_pages); + goto no_mem; + } + + kbase_gpu_vm_lock(kctx); + + /* mmap needed to setup VA? */ + if (*flags & BASE_MEM_SAME_VA) { + unsigned long prot = PROT_NONE; + unsigned long va_size = va_pages << PAGE_SHIFT; + unsigned long va_map = va_size; + unsigned long cookie, cookie_nr; + unsigned long cpu_addr; + + /* Bind to a cookie */ + if (!kctx->cookies) { + dev_err(dev, "No cookies available for allocation!"); + kbase_gpu_vm_unlock(kctx); + goto no_cookie; + } + /* return a cookie */ + cookie_nr = __ffs(kctx->cookies); + kctx->cookies &= ~(1UL << cookie_nr); + BUG_ON(kctx->pending_regions[cookie_nr]); + kctx->pending_regions[cookie_nr] = reg; + + kbase_gpu_vm_unlock(kctx); + + /* relocate to correct base */ + cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE); + cookie <<= PAGE_SHIFT; + + /* + * 10.1-10.4 UKU userland relies on the kernel to call mmap. + * For all other versions we can just return the cookie + */ + if (kctx->api_version < KBASE_API_VERSION(10, 1) || + kctx->api_version > KBASE_API_VERSION(10, 4)) { + *gpu_va = (u64) cookie; + return reg; + } + if (*flags & BASE_MEM_PROT_CPU_RD) + prot |= PROT_READ; + if (*flags & BASE_MEM_PROT_CPU_WR) + prot |= PROT_WRITE; + + cpu_addr = vm_mmap(kctx->filp, 0, va_map, prot, + MAP_SHARED, cookie); + + if (IS_ERR_VALUE(cpu_addr)) { + kbase_gpu_vm_lock(kctx); + kctx->pending_regions[cookie_nr] = NULL; + kctx->cookies |= (1UL << cookie_nr); + kbase_gpu_vm_unlock(kctx); + goto no_mmap; + } + + *gpu_va = (u64) cpu_addr; + } else /* we control the VA */ { + if (kbase_gpu_mmap(kctx, reg, 0, va_pages, 1) != 0) { + dev_warn(dev, "Failed to map memory on GPU"); + kbase_gpu_vm_unlock(kctx); + goto no_mmap; + } + /* return real GPU VA */ + *gpu_va = reg->start_pfn << PAGE_SHIFT; + + kbase_gpu_vm_unlock(kctx); + } + + return reg; + +no_mmap: +no_cookie: +no_mem: + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +invalid_flags: +prepare_failed: + kfree(reg); +no_region: +bad_ex_size: +bad_flags: +bad_size: + return NULL; +} +KBASE_EXPORT_TEST_API(kbase_mem_alloc); + +int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * const out) +{ + struct kbase_va_region *reg; + int ret = -EINVAL; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(out); + + if (gpu_addr & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, "mem_query: gpu_addr: passed parameter is invalid"); + return -EINVAL; + } + + kbase_gpu_vm_lock(kctx); + + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (!reg || (reg->flags & KBASE_REG_FREE)) + goto out_unlock; + + switch (query) { + case KBASE_MEM_QUERY_COMMIT_SIZE: + if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) { + *out = kbase_reg_current_backed_size(reg); + } else { + size_t i; + struct kbase_aliased *aliased; + *out = 0; + aliased = reg->cpu_alloc->imported.alias.aliased; + for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++) + *out += aliased[i].length; + } + break; + case KBASE_MEM_QUERY_VA_SIZE: + *out = reg->nr_pages; + break; + case KBASE_MEM_QUERY_FLAGS: + { + *out = 0; + if (KBASE_REG_CPU_WR & reg->flags) + *out |= BASE_MEM_PROT_CPU_WR; + if (KBASE_REG_CPU_RD & reg->flags) + *out |= BASE_MEM_PROT_CPU_RD; + if (KBASE_REG_CPU_CACHED & reg->flags) + *out |= BASE_MEM_CACHED_CPU; + if (KBASE_REG_GPU_WR & reg->flags) + *out |= BASE_MEM_PROT_GPU_WR; + if (KBASE_REG_GPU_RD & reg->flags) + *out |= BASE_MEM_PROT_GPU_RD; + if (!(KBASE_REG_GPU_NX & reg->flags)) + *out |= BASE_MEM_PROT_GPU_EX; + if (KBASE_REG_SHARE_BOTH & reg->flags) + *out |= BASE_MEM_COHERENT_SYSTEM; + if (KBASE_REG_SHARE_IN & reg->flags) + *out |= BASE_MEM_COHERENT_LOCAL; + break; + } + default: + *out = 0; + goto out_unlock; + } + + ret = 0; + +out_unlock: + kbase_gpu_vm_unlock(kctx); + return ret; +} + +/** + * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the + * Ephemeral memory eviction list. + * @s: Shrinker + * @sc: Shrinker control + * + * Return: Number of pages which can be freed. + */ +static +unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_context *kctx; + struct kbase_mem_phy_alloc *alloc; + unsigned long pages = 0; + + kctx = container_of(s, struct kbase_context, reclaim); + + mutex_lock(&kctx->jit_evict_lock); + + list_for_each_entry(alloc, &kctx->evict_list, evict_node) + pages += alloc->nents; + + mutex_unlock(&kctx->jit_evict_lock); + return pages; +} + +/** + * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction + * list for pages and try to reclaim them. + * @s: Shrinker + * @sc: Shrinker control + * + * Return: Number of pages freed (can be less then requested) or -1 if the + * shrinker failed to free pages in its pool. + * + * Note: + * This function accesses region structures without taking the region lock, + * this is required as the OOM killer can call the shrinker after the region + * lock has already been held. + * This is safe as we can guarantee that a region on the eviction list will + * not be freed (kbase_mem_free_region removes the allocation from the list + * before destroying it), or modified by other parts of the driver. + * The eviction list itself is guarded by the eviction lock and the MMU updates + * are protected by their own lock. + */ +static +unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_context *kctx; + struct kbase_mem_phy_alloc *alloc; + struct kbase_mem_phy_alloc *tmp; + unsigned long freed = 0; + + kctx = container_of(s, struct kbase_context, reclaim); + mutex_lock(&kctx->jit_evict_lock); + + list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { + int err; + + err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg, + 0, alloc->nents); + if (err != 0) { + /* + * Failed to remove GPU mapping, tell the shrinker + * to stop trying to shrink our slab even though we + * have pages in it. + */ + freed = -1; + goto out_unlock; + } + + /* + * Update alloc->evicted before freeing the backing so the + * helper can determine that it needs to bypass the accounting + * and memory pool. + */ + alloc->evicted = alloc->nents; + + kbase_free_phy_pages_helper(alloc, alloc->evicted); + freed += alloc->evicted; + list_del_init(&alloc->evict_node); + + /* + * Inform the JIT allocator this region has lost backing + * as it might need to free the allocation. + */ + kbase_jit_backing_lost(alloc->reg); + + /* Enough pages have been freed so stop now */ + if (freed > sc->nr_to_scan) + break; + } +out_unlock: + mutex_unlock(&kctx->jit_evict_lock); + + return freed; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s, + struct shrink_control *sc) +{ + if (sc->nr_to_scan == 0) + return kbase_mem_evictable_reclaim_count_objects(s, sc); + + return kbase_mem_evictable_reclaim_scan_objects(s, sc); +} +#endif + +int kbase_mem_evictable_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->evict_list); + mutex_init(&kctx->jit_evict_lock); + + /* Register shrinker */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) + kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink; +#else + kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; + kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; +#endif + kctx->reclaim.seeks = DEFAULT_SEEKS; + /* Kernel versions prior to 3.1 : + * struct shrinker does not define batch */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) + kctx->reclaim.batch = 0; +#endif + register_shrinker(&kctx->reclaim); + return 0; +} + +void kbase_mem_evictable_deinit(struct kbase_context *kctx) +{ + unregister_shrinker(&kctx->reclaim); +} + +struct kbase_mem_zone_cache_entry { + /* List head used to link the cache entry to the memory allocation. */ + struct list_head zone_node; + /* The zone the cacheline is for. */ + struct zone *zone; + /* The number of pages in the allocation which belong to this zone. */ + u64 count; +}; + +static bool kbase_zone_cache_builder(struct kbase_mem_phy_alloc *alloc, + size_t start_offset) +{ + struct kbase_mem_zone_cache_entry *cache = NULL; + size_t i; + int ret = 0; + + for (i = start_offset; i < alloc->nents; i++) { + struct page *p = phys_to_page(as_phys_addr_t(alloc->pages[i])); + struct zone *zone = page_zone(p); + bool create = true; + + if (cache && (cache->zone == zone)) { + /* + * Fast path check as most of the time adjacent + * pages come from the same zone. + */ + create = false; + } else { + /* + * Slow path check, walk all the cache entries to see + * if we already know about this zone. + */ + list_for_each_entry(cache, &alloc->zone_cache, zone_node) { + if (cache->zone == zone) { + create = false; + break; + } + } + } + + /* This zone wasn't found in the cache, create an entry for it */ + if (create) { + cache = kmalloc(sizeof(*cache), GFP_KERNEL); + if (!cache) { + ret = -ENOMEM; + goto bail; + } + cache->zone = zone; + cache->count = 0; + list_add(&cache->zone_node, &alloc->zone_cache); + } + + cache->count++; + } + return 0; + +bail: + return ret; +} + +int kbase_zone_cache_update(struct kbase_mem_phy_alloc *alloc, + size_t start_offset) +{ + /* + * Bail if the zone cache is empty, only update the cache if it + * existed in the first place. + */ + if (list_empty(&alloc->zone_cache)) + return 0; + + return kbase_zone_cache_builder(alloc, start_offset); +} + +int kbase_zone_cache_build(struct kbase_mem_phy_alloc *alloc) +{ + /* Bail if the zone cache already exists */ + if (!list_empty(&alloc->zone_cache)) + return 0; + + return kbase_zone_cache_builder(alloc, 0); +} + +void kbase_zone_cache_clear(struct kbase_mem_phy_alloc *alloc) +{ + struct kbase_mem_zone_cache_entry *walker; + + while(!list_empty(&alloc->zone_cache)){ + walker = list_first_entry(&alloc->zone_cache, + struct kbase_mem_zone_cache_entry, zone_node); + list_del(&walker->zone_node); + kfree(walker); + } +} + +/** + * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable. + * @alloc: The physical allocation + */ +static void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc) +{ + struct kbase_context *kctx = alloc->imported.kctx; + struct kbase_mem_zone_cache_entry *zone_cache; + int __maybe_unused new_page_count; + int err; + + /* Attempt to build a zone cache of tracking */ + err = kbase_zone_cache_build(alloc); + if (err == 0) { + /* Bulk update all the zones */ + list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) { + zone_page_state_add(zone_cache->count, + zone_cache->zone, NR_SLAB_RECLAIMABLE); + } + } else { + /* Fall-back to page by page updates */ + int i; + + for (i = 0; i < alloc->nents; i++) { + struct page *p; + struct zone *zone; + + p = phys_to_page(as_phys_addr_t(alloc->pages[i])); + zone = page_zone(p); + + zone_page_state_add(1, zone, NR_SLAB_RECLAIMABLE); + } + } + + kbase_process_page_usage_dec(kctx, alloc->nents); + new_page_count = kbase_atomic_sub_pages(alloc->nents, + &kctx->used_pages); + kbase_atomic_sub_pages(alloc->nents, &kctx->kbdev->memdev.used_pages); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kctx->id, + (u64)new_page_count); +} + +/** + * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable. + * @alloc: The physical allocation + */ +static +void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) +{ + struct kbase_context *kctx = alloc->imported.kctx; + struct kbase_mem_zone_cache_entry *zone_cache; + int __maybe_unused new_page_count; + int err; + + new_page_count = kbase_atomic_add_pages(alloc->nents, + &kctx->used_pages); + kbase_atomic_add_pages(alloc->nents, &kctx->kbdev->memdev.used_pages); + + /* Increase mm counters so that the allocation is accounted for + * against the process and thus is visible to the OOM killer, + * then remove it from the reclaimable accounting. */ + kbase_process_page_usage_inc(kctx, alloc->nents); + + /* Attempt to build a zone cache of tracking */ + err = kbase_zone_cache_build(alloc); + if (err == 0) { + /* Bulk update all the zones */ + list_for_each_entry(zone_cache, &alloc->zone_cache, zone_node) { + zone_page_state_add(-zone_cache->count, + zone_cache->zone, NR_SLAB_RECLAIMABLE); + } + } else { + /* Fall-back to page by page updates */ + int i; + + for (i = 0; i < alloc->nents; i++) { + struct page *p; + struct zone *zone; + + p = phys_to_page(as_phys_addr_t(alloc->pages[i])); + zone = page_zone(p); + zone_page_state_add(-1, zone, NR_SLAB_RECLAIMABLE); + } + } + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kctx->id, + (u64)new_page_count); +} + +int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) +{ + struct kbase_context *kctx = gpu_alloc->imported.kctx; + + lockdep_assert_held(&kctx->reg_lock); + + /* This alloction can't already be on a list. */ + WARN_ON(!list_empty(&gpu_alloc->evict_node)); + + kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg, + 0, gpu_alloc->nents); + + /* + * Add the allocation to the eviction list, after this point the shrink + * can reclaim it. + */ + mutex_lock(&kctx->jit_evict_lock); + list_add(&gpu_alloc->evict_node, &kctx->evict_list); + mutex_unlock(&kctx->jit_evict_lock); + kbase_mem_evictable_mark_reclaim(gpu_alloc); + + gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED; + return 0; +} + +bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) +{ + struct kbase_context *kctx = gpu_alloc->imported.kctx; + int err = 0; + + lockdep_assert_held(&kctx->reg_lock); + + /* + * First remove the allocation from the eviction list as it's no + * longer eligible for eviction. + */ + list_del_init(&gpu_alloc->evict_node); + + if (gpu_alloc->evicted == 0) { + /* + * The backing is still present, update the VM stats as it's + * in use again. + */ + kbase_mem_evictable_unmark_reclaim(gpu_alloc); + } else { + /* If the region is still alive ... */ + if (gpu_alloc->reg) { + /* ... allocate replacement backing ... */ + err = kbase_alloc_phy_pages_helper(gpu_alloc, + gpu_alloc->evicted); + + /* + * ... and grow the mapping back to its + * pre-eviction size. + */ + if (!err) + err = kbase_mem_grow_gpu_mapping(kctx, + gpu_alloc->reg, + gpu_alloc->evicted, 0); + + gpu_alloc->evicted = 0; + } + } + + /* If the region is still alive remove the DONT_NEED attribute. */ + if (gpu_alloc->reg) + gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED; + + return (err == 0); +} + +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask) +{ + struct kbase_va_region *reg; + int ret = -EINVAL; + unsigned int real_flags = 0; + unsigned int prev_flags = 0; + bool prev_needed, new_needed; + + KBASE_DEBUG_ASSERT(kctx); + + if (!gpu_addr) + return -EINVAL; + + if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) + return -EINVAL; + + /* nuke other bits */ + flags &= mask; + + /* check for only supported flags */ + if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE)) + goto out; + + /* mask covers bits we don't support? */ + if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE)) + goto out; + + /* convert flags */ + if (BASE_MEM_COHERENT_SYSTEM & flags) + real_flags |= KBASE_REG_SHARE_BOTH; + else if (BASE_MEM_COHERENT_LOCAL & flags) + real_flags |= KBASE_REG_SHARE_IN; + + /* now we can lock down the context, and find the region */ + down_write(¤t->mm->mmap_sem); + kbase_gpu_vm_lock(kctx); + + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (!reg || (reg->flags & KBASE_REG_FREE)) + goto out_unlock; + + /* Is the region being transitioning between not needed and needed? */ + prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED; + new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED; + if (prev_needed != new_needed) { + /* Aliased allocations can't be made ephemeral */ + if (atomic_read(®->cpu_alloc->gpu_mappings) > 1) + goto out_unlock; + + if (new_needed) { + /* Only native allocations can be marked not needed */ + if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) { + ret = -EINVAL; + goto out_unlock; + } + ret = kbase_mem_evictable_make(reg->gpu_alloc); + if (ret) + goto out_unlock; + } else { + kbase_mem_evictable_unmake(reg->gpu_alloc); + } + } + + /* limit to imported memory */ + if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) && + (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) + goto out_unlock; + + /* no change? */ + if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) { + ret = 0; + goto out_unlock; + } + + /* save for roll back */ + prev_flags = reg->flags; + reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); + reg->flags |= real_flags; + + /* Currently supporting only imported memory */ + switch (reg->gpu_alloc->type) { +#ifdef CONFIG_UMP + case KBASE_MEM_TYPE_IMPORTED_UMP: + ret = kbase_mmu_update_pages(kctx, reg->start_pfn, + kbase_get_gpu_phy_pages(reg), + reg->gpu_alloc->nents, reg->flags); + break; +#endif +#ifdef CONFIG_DMA_SHARED_BUFFER + case KBASE_MEM_TYPE_IMPORTED_UMM: + /* Future use will use the new flags, existing mapping will NOT be updated + * as memory should not be in use by the GPU when updating the flags. + */ + ret = 0; + WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count); + break; +#endif + default: + break; + } + + /* roll back on error, i.e. not UMP */ + if (ret) + reg->flags = prev_flags; + +out_unlock: + kbase_gpu_vm_unlock(kctx); + up_write(¤t->mm->mmap_sem); +out: + return ret; +} + +#define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS) + +#ifdef CONFIG_UMP +static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags) +{ + struct kbase_va_region *reg; + ump_dd_handle umph; + u64 block_count; + const ump_dd_physical_block_64 *block_array; + u64 i, j; + int page = 0; + ump_alloc_flags ump_flags; + ump_alloc_flags cpu_flags; + ump_alloc_flags gpu_flags; + + if (*flags & BASE_MEM_SECURE) + goto bad_flags; + + umph = ump_dd_from_secure_id(id); + if (UMP_DD_INVALID_MEMORY_HANDLE == umph) + goto bad_id; + + ump_flags = ump_dd_allocation_flags_get(umph); + cpu_flags = (ump_flags >> UMP_DEVICE_CPU_SHIFT) & UMP_DEVICE_MASK; + gpu_flags = (ump_flags >> DEFAULT_UMP_GPU_DEVICE_SHIFT) & + UMP_DEVICE_MASK; + + *va_pages = ump_dd_size_get_64(umph); + *va_pages >>= PAGE_SHIFT; + + if (!*va_pages) + goto bad_size; + + if (*va_pages > (U64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + if (*flags & BASE_MEM_SAME_VA) + reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); + else + reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); + + if (!reg) + goto no_region; + + /* we've got pages to map now, and support SAME_VA */ + *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; + + reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc_obj; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + reg->gpu_alloc->imported.ump_handle = umph; + + reg->flags &= ~KBASE_REG_FREE; + reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */ + reg->flags &= ~KBASE_REG_GROWABLE; /* UMP cannot be grown */ + + /* Override import flags based on UMP flags */ + *flags &= ~(BASE_MEM_CACHED_CPU); + *flags &= ~(BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR); + *flags &= ~(BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR); + + if ((cpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == + (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) { + reg->flags |= KBASE_REG_CPU_CACHED; + *flags |= BASE_MEM_CACHED_CPU; + } + + if (cpu_flags & UMP_PROT_CPU_WR) { + reg->flags |= KBASE_REG_CPU_WR; + *flags |= BASE_MEM_PROT_CPU_WR; + } + + if (cpu_flags & UMP_PROT_CPU_RD) { + reg->flags |= KBASE_REG_CPU_RD; + *flags |= BASE_MEM_PROT_CPU_RD; + } + + if ((gpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == + (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) + reg->flags |= KBASE_REG_GPU_CACHED; + + if (gpu_flags & UMP_PROT_DEVICE_WR) { + reg->flags |= KBASE_REG_GPU_WR; + *flags |= BASE_MEM_PROT_GPU_WR; + } + + if (gpu_flags & UMP_PROT_DEVICE_RD) { + reg->flags |= KBASE_REG_GPU_RD; + *flags |= BASE_MEM_PROT_GPU_RD; + } + + /* ump phys block query */ + ump_dd_phys_blocks_get_64(umph, &block_count, &block_array); + + for (i = 0; i < block_count; i++) { + for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) { + struct tagged_addr tagged; + + tagged = as_tagged(block_array[i].addr + + (j << PAGE_SHIFT)); + reg->gpu_alloc->pages[page] = tagged; + page++; + } + } + reg->gpu_alloc->nents = *va_pages; + reg->extent = 0; + + return reg; + +no_alloc_obj: + kfree(reg); +no_region: +bad_size: + ump_dd_release(umph); +bad_id: +bad_flags: + return NULL; +} +#endif /* CONFIG_UMP */ + +#ifdef CONFIG_DMA_SHARED_BUFFER +static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, + int fd, u64 *va_pages, u64 *flags, u32 padding) +{ + struct kbase_va_region *reg; + struct dma_buf *dma_buf; + struct dma_buf_attachment *dma_attachment; + bool shared_zone = false; + + dma_buf = dma_buf_get(fd); + if (IS_ERR_OR_NULL(dma_buf)) + goto no_buf; + + dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev); + if (!dma_attachment) + goto no_attachment; + + *va_pages = (PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT) + padding; + if (!*va_pages) + goto bad_size; + + if (*va_pages > (U64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + /* ignore SAME_VA */ + *flags &= ~BASE_MEM_SAME_VA; + + if (*flags & BASE_MEM_IMPORT_SHARED) + shared_zone = true; + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* + * 64-bit tasks require us to reserve VA on the CPU that we use + * on the GPU. + */ + shared_zone = true; + } +#endif + + if (shared_zone) { + *flags |= BASE_MEM_NEED_MMAP; + reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); + } else { + reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); + } + + if (!reg) + goto no_region; + + reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc_obj; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + /* No pages to map yet */ + reg->gpu_alloc->nents = 0; + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + reg->flags &= ~KBASE_REG_FREE; + reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */ + reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */ + reg->flags |= KBASE_REG_GPU_CACHED; + + if (*flags & BASE_MEM_SECURE) + reg->flags |= KBASE_REG_SECURE; + + if (padding) + reg->flags |= KBASE_REG_IMPORT_PAD; + + reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM; + reg->gpu_alloc->imported.umm.sgt = NULL; + reg->gpu_alloc->imported.umm.dma_buf = dma_buf; + reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment; + reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0; + reg->extent = 0; + + return reg; + +invalid_flags: + kbase_mem_phy_alloc_put(reg->gpu_alloc); +no_alloc_obj: + kfree(reg); +no_region: +bad_size: + dma_buf_detach(dma_buf, dma_attachment); +no_attachment: + dma_buf_put(dma_buf); +no_buf: + return NULL; +} +#endif /* CONFIG_DMA_SHARED_BUFFER */ + +static u32 kbase_get_cache_line_alignment(struct kbase_context *kctx) +{ + u32 cpu_cache_line_size = cache_line_size(); + u32 gpu_cache_line_size = + (1UL << kctx->kbdev->gpu_props.props.l2_props.log2_line_size); + + return ((cpu_cache_line_size > gpu_cache_line_size) ? + cpu_cache_line_size : + gpu_cache_line_size); +} + +static struct kbase_va_region *kbase_mem_from_user_buffer( + struct kbase_context *kctx, unsigned long address, + unsigned long size, u64 *va_pages, u64 *flags) +{ + long i; + struct kbase_va_region *reg; + long faulted_pages; + int zone = KBASE_REG_ZONE_CUSTOM_VA; + bool shared_zone = false; + u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx); + struct kbase_alloc_import_user_buf *user_buf; + struct page **pages = NULL; + + if ((address & (cache_line_alignment - 1)) != 0 || + (size & (cache_line_alignment - 1)) != 0) { + /* Coherency must be enabled to handle partial cache lines */ + if (*flags & (BASE_MEM_COHERENT_SYSTEM | + BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { + /* Force coherent system required flag, import will + * then fail if coherency isn't available + */ + *flags |= BASE_MEM_COHERENT_SYSTEM_REQUIRED; + } else { + dev_warn(kctx->kbdev->dev, + "User buffer is not cache line aligned and no coherency enabled\n"); + goto bad_size; + } + } + + *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) - + PFN_DOWN(address); + if (!*va_pages) + goto bad_size; + + if (*va_pages > (UINT64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + /* SAME_VA generally not supported with imported memory (no known use cases) */ + *flags &= ~BASE_MEM_SAME_VA; + + if (*flags & BASE_MEM_IMPORT_SHARED) + shared_zone = true; + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* + * 64-bit tasks require us to reserve VA on the CPU that we use + * on the GPU. + */ + shared_zone = true; + } +#endif + + if (shared_zone) { + *flags |= BASE_MEM_NEED_MMAP; + zone = KBASE_REG_ZONE_SAME_VA; + } + + reg = kbase_alloc_free_region(kctx, 0, *va_pages, zone); + + if (!reg) + goto no_region; + + reg->gpu_alloc = kbase_alloc_create(*va_pages, + KBASE_MEM_TYPE_IMPORTED_USER_BUF); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc_obj; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + reg->flags &= ~KBASE_REG_FREE; + reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */ + reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */ + + user_buf = ®->gpu_alloc->imported.user_buf; + + user_buf->size = size; + user_buf->address = address; + user_buf->nr_pages = *va_pages; + user_buf->mm = current->mm; + user_buf->pages = kmalloc_array(*va_pages, sizeof(struct page *), + GFP_KERNEL); + + if (!user_buf->pages) + goto no_page_array; + + /* If the region is coherent with the CPU then the memory is imported + * and mapped onto the GPU immediately. + * Otherwise get_user_pages is called as a sanity check, but with + * NULL as the pages argument which will fault the pages, but not + * pin them. The memory will then be pinned only around the jobs that + * specify the region as an external resource. + */ + if (reg->flags & KBASE_REG_SHARE_BOTH) { + pages = user_buf->pages; + *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; + } + + down_read(¤t->mm->mmap_sem); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + faulted_pages = get_user_pages(current, current->mm, address, *va_pages, + reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + faulted_pages = get_user_pages(address, *va_pages, + reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); +#else + faulted_pages = get_user_pages(address, *va_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL); +#endif + + up_read(¤t->mm->mmap_sem); + + if (faulted_pages != *va_pages) + goto fault_mismatch; + + atomic_inc(¤t->mm->mm_count); + + reg->gpu_alloc->nents = 0; + reg->extent = 0; + + if (pages) { + struct device *dev = kctx->kbdev->dev; + unsigned long local_size = user_buf->size; + unsigned long offset = user_buf->address & ~PAGE_MASK; + struct tagged_addr *pa = kbase_get_gpu_phy_pages(reg); + + /* Top bit signifies that this was pinned on import */ + user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT; + + for (i = 0; i < faulted_pages; i++) { + dma_addr_t dma_addr; + unsigned long min; + + min = MIN(PAGE_SIZE - offset, local_size); + dma_addr = dma_map_page(dev, pages[i], + offset, min, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) + goto unwind_dma_map; + + user_buf->dma_addrs[i] = dma_addr; + pa[i] = as_tagged(page_to_phys(pages[i])); + + local_size -= min; + offset = 0; + } + + reg->gpu_alloc->nents = faulted_pages; + } + + return reg; + +unwind_dma_map: + while (i--) { + dma_unmap_page(kctx->kbdev->dev, + user_buf->dma_addrs[i], + PAGE_SIZE, DMA_BIDIRECTIONAL); + } +fault_mismatch: + if (pages) { + for (i = 0; i < faulted_pages; i++) + put_page(pages[i]); + } + kfree(user_buf->pages); +no_page_array: +invalid_flags: + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +no_alloc_obj: + kfree(reg); +no_region: +bad_size: + return NULL; + +} + + +u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, + u64 nents, struct base_mem_aliasing_info *ai, + u64 *num_pages) +{ + struct kbase_va_region *reg; + u64 gpu_va; + size_t i; + bool coherent; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(flags); + KBASE_DEBUG_ASSERT(ai); + KBASE_DEBUG_ASSERT(num_pages); + + /* mask to only allowed flags */ + *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | + BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL | + BASE_MEM_COHERENT_SYSTEM_REQUIRED); + + if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) { + dev_warn(kctx->kbdev->dev, + "kbase_mem_alias called with bad flags (%llx)", + (unsigned long long)*flags); + goto bad_flags; + } + coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 || + (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0; + + if (!stride) + goto bad_stride; + + if (!nents) + goto bad_nents; + + if ((nents * stride) > (U64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + /* calculate the number of pages this alias will cover */ + *num_pages = nents * stride; + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* 64-bit tasks must MMAP anyway, but not expose this address to + * clients */ + *flags |= BASE_MEM_NEED_MMAP; + reg = kbase_alloc_free_region(kctx, 0, *num_pages, + KBASE_REG_ZONE_SAME_VA); + } else { +#else + if (1) { +#endif + reg = kbase_alloc_free_region(kctx, 0, *num_pages, + KBASE_REG_ZONE_CUSTOM_VA); + } + + if (!reg) + goto no_reg; + + /* zero-sized page array, as we don't need one/can support one */ + reg->gpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc_obj; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + reg->gpu_alloc->imported.alias.nents = nents; + reg->gpu_alloc->imported.alias.stride = stride; + reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents); + if (!reg->gpu_alloc->imported.alias.aliased) + goto no_aliased_array; + + kbase_gpu_vm_lock(kctx); + + /* validate and add src handles */ + for (i = 0; i < nents; i++) { + if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) { + if (ai[i].handle.basep.handle != + BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) + goto bad_handle; /* unsupported magic handle */ + if (!ai[i].length) + goto bad_handle; /* must be > 0 */ + if (ai[i].length > stride) + goto bad_handle; /* can't be larger than the + stride */ + reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; + } else { + struct kbase_va_region *aliasing_reg; + struct kbase_mem_phy_alloc *alloc; + + aliasing_reg = kbase_region_tracker_find_region_base_address( + kctx, + (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); + + /* validate found region */ + if (!aliasing_reg) + goto bad_handle; /* Not found */ + if (aliasing_reg->flags & KBASE_REG_FREE) + goto bad_handle; /* Free region */ + if (aliasing_reg->flags & KBASE_REG_DONT_NEED) + goto bad_handle; /* Ephemeral region */ + if (!aliasing_reg->gpu_alloc) + goto bad_handle; /* No alloc */ + if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) + goto bad_handle; /* Not a native alloc */ + if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0)) + goto bad_handle; + /* Non-coherent memory cannot alias + coherent memory, and vice versa.*/ + + /* check size against stride */ + if (!ai[i].length) + goto bad_handle; /* must be > 0 */ + if (ai[i].length > stride) + goto bad_handle; /* can't be larger than the + stride */ + + alloc = aliasing_reg->gpu_alloc; + + /* check against the alloc's size */ + if (ai[i].offset > alloc->nents) + goto bad_handle; /* beyond end */ + if (ai[i].offset + ai[i].length > alloc->nents) + goto bad_handle; /* beyond end */ + + reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc); + reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; + reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset; + } + } + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* Bind to a cookie */ + if (!kctx->cookies) { + dev_err(kctx->kbdev->dev, "No cookies available for allocation!"); + goto no_cookie; + } + /* return a cookie */ + gpu_va = __ffs(kctx->cookies); + kctx->cookies &= ~(1UL << gpu_va); + BUG_ON(kctx->pending_regions[gpu_va]); + kctx->pending_regions[gpu_va] = reg; + + /* relocate to correct base */ + gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); + gpu_va <<= PAGE_SHIFT; + } else /* we control the VA */ { +#else + if (1) { +#endif + if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) { + dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU"); + goto no_mmap; + } + /* return real GPU VA */ + gpu_va = reg->start_pfn << PAGE_SHIFT; + } + + reg->flags &= ~KBASE_REG_FREE; + reg->flags &= ~KBASE_REG_GROWABLE; + + kbase_gpu_vm_unlock(kctx); + + return gpu_va; + +#ifdef CONFIG_64BIT +no_cookie: +#endif +no_mmap: +bad_handle: + kbase_gpu_vm_unlock(kctx); +no_aliased_array: +invalid_flags: + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +no_alloc_obj: + kfree(reg); +no_reg: +bad_size: +bad_nents: +bad_stride: +bad_flags: + return 0; +} + +int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, + u64 *flags) +{ + struct kbase_va_region *reg; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(gpu_va); + KBASE_DEBUG_ASSERT(va_pages); + KBASE_DEBUG_ASSERT(flags); + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) + *flags |= BASE_MEM_SAME_VA; +#endif + + if (!kbase_check_import_flags(*flags)) { + dev_warn(kctx->kbdev->dev, + "kbase_mem_import called with bad flags (%llx)", + (unsigned long long)*flags); + goto bad_flags; + } + + if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + dev_warn(kctx->kbdev->dev, + "kbase_mem_import call required coherent mem when unavailable"); + goto bad_flags; + } + if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ + *flags &= ~BASE_MEM_COHERENT_SYSTEM; + } + + if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) { + dev_warn(kctx->kbdev->dev, + "padding is only supported for UMM"); + goto bad_flags; + } + + switch (type) { +#ifdef CONFIG_UMP + case BASE_MEM_IMPORT_TYPE_UMP: { + ump_secure_id id; + + if (get_user(id, (ump_secure_id __user *)phandle)) + reg = NULL; + else + reg = kbase_mem_from_ump(kctx, id, va_pages, flags); + } + break; +#endif /* CONFIG_UMP */ +#ifdef CONFIG_DMA_SHARED_BUFFER + case BASE_MEM_IMPORT_TYPE_UMM: { + int fd; + + if (get_user(fd, (int __user *)phandle)) + reg = NULL; + else + reg = kbase_mem_from_umm(kctx, fd, va_pages, flags, + padding); + } + break; +#endif /* CONFIG_DMA_SHARED_BUFFER */ + case BASE_MEM_IMPORT_TYPE_USER_BUFFER: { + struct base_mem_import_user_buffer user_buffer; + void __user *uptr; + + if (copy_from_user(&user_buffer, phandle, + sizeof(user_buffer))) { + reg = NULL; + } else { +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + uptr = compat_ptr(user_buffer.ptr); + else +#endif + uptr = u64_to_user_ptr(user_buffer.ptr); + + reg = kbase_mem_from_user_buffer(kctx, + (unsigned long)uptr, user_buffer.length, + va_pages, flags); + } + break; + } + default: { + reg = NULL; + break; + } + } + + if (!reg) + goto no_reg; + + kbase_gpu_vm_lock(kctx); + + /* mmap needed to setup VA? */ + if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) { + /* Bind to a cookie */ + if (!kctx->cookies) + goto no_cookie; + /* return a cookie */ + *gpu_va = __ffs(kctx->cookies); + kctx->cookies &= ~(1UL << *gpu_va); + BUG_ON(kctx->pending_regions[*gpu_va]); + kctx->pending_regions[*gpu_va] = reg; + + /* relocate to correct base */ + *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); + *gpu_va <<= PAGE_SHIFT; + + } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) { + /* we control the VA, mmap now to the GPU */ + if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0) + goto no_gpu_va; + /* return real GPU VA */ + *gpu_va = reg->start_pfn << PAGE_SHIFT; + } else { + /* we control the VA, but nothing to mmap yet */ + if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0) + goto no_gpu_va; + /* return real GPU VA */ + *gpu_va = reg->start_pfn << PAGE_SHIFT; + } + + /* clear out private flags */ + *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1); + + kbase_gpu_vm_unlock(kctx); + + return 0; + +no_gpu_va: +no_cookie: + kbase_gpu_vm_unlock(kctx); + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); + kfree(reg); +no_reg: +bad_flags: + *gpu_va = 0; + *va_pages = 0; + *flags = 0; + return -ENOMEM; +} + +int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages) +{ + struct tagged_addr *phy_pages; + u64 delta = new_pages - old_pages; + int ret = 0; + + lockdep_assert_held(&kctx->reg_lock); + + /* Map the new pages into the GPU */ + phy_pages = kbase_get_gpu_phy_pages(reg); + ret = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages, + phy_pages + old_pages, delta, reg->flags); + + return ret; +} + +static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages) +{ + u64 gpu_va_start = reg->start_pfn; + + if (new_pages == old_pages) + /* Nothing to do */ + return; + + unmap_mapping_range(kctx->filp->f_inode->i_mapping, + (gpu_va_start + new_pages)<start_pfn + new_pages, delta); + + return ret; +} + +int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) +{ + u64 old_pages; + u64 delta; + int res = -EINVAL; + struct kbase_va_region *reg; + bool read_locked = false; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(gpu_addr != 0); + + if (gpu_addr & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid"); + return -EINVAL; + } + + down_write(¤t->mm->mmap_sem); + kbase_gpu_vm_lock(kctx); + + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (!reg || (reg->flags & KBASE_REG_FREE)) + goto out_unlock; + + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) + goto out_unlock; + + if (0 == (reg->flags & KBASE_REG_GROWABLE)) + goto out_unlock; + + /* Would overflow the VA region */ + if (new_pages > reg->nr_pages) + goto out_unlock; + + /* can't be mapped more than once on the GPU */ + if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) + goto out_unlock; + /* can't grow regions which are ephemeral */ + if (reg->flags & KBASE_REG_DONT_NEED) + goto out_unlock; + + if (new_pages == reg->gpu_alloc->nents) { + /* no change */ + res = 0; + goto out_unlock; + } + + old_pages = kbase_reg_current_backed_size(reg); + if (new_pages > old_pages) { + delta = new_pages - old_pages; + + /* + * No update to the mm so downgrade the writer lock to a read + * lock so other readers aren't blocked after this point. + */ + downgrade_write(¤t->mm->mmap_sem); + read_locked = true; + + /* Allocate some more pages */ + if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) { + res = -ENOMEM; + goto out_unlock; + } + if (reg->cpu_alloc != reg->gpu_alloc) { + if (kbase_alloc_phy_pages_helper( + reg->gpu_alloc, delta) != 0) { + res = -ENOMEM; + kbase_free_phy_pages_helper(reg->cpu_alloc, + delta); + goto out_unlock; + } + } + + /* No update required for CPU mappings, that's done on fault. */ + + /* Update GPU mapping. */ + res = kbase_mem_grow_gpu_mapping(kctx, reg, + new_pages, old_pages); + + /* On error free the new pages */ + if (res) { + kbase_free_phy_pages_helper(reg->cpu_alloc, delta); + if (reg->cpu_alloc != reg->gpu_alloc) + kbase_free_phy_pages_helper(reg->gpu_alloc, + delta); + res = -ENOMEM; + goto out_unlock; + } + } else { + delta = old_pages - new_pages; + + /* Update all CPU mapping(s) */ + kbase_mem_shrink_cpu_mapping(kctx, reg, + new_pages, old_pages); + + /* Update the GPU mapping */ + res = kbase_mem_shrink_gpu_mapping(kctx, reg, + new_pages, old_pages); + if (res) { + res = -ENOMEM; + goto out_unlock; + } + + kbase_free_phy_pages_helper(reg->cpu_alloc, delta); + if (reg->cpu_alloc != reg->gpu_alloc) + kbase_free_phy_pages_helper(reg->gpu_alloc, delta); + } + +out_unlock: + kbase_gpu_vm_unlock(kctx); + if (read_locked) + up_read(¤t->mm->mmap_sem); + else + up_write(¤t->mm->mmap_sem); + + return res; +} + +static void kbase_cpu_vm_open(struct vm_area_struct *vma) +{ + struct kbase_cpu_mapping *map = vma->vm_private_data; + + KBASE_DEBUG_ASSERT(map); + KBASE_DEBUG_ASSERT(map->count > 0); + /* non-atomic as we're under Linux' mm lock */ + map->count++; +} + +static void kbase_cpu_vm_close(struct vm_area_struct *vma) +{ + struct kbase_cpu_mapping *map = vma->vm_private_data; + + KBASE_DEBUG_ASSERT(map); + KBASE_DEBUG_ASSERT(map->count > 0); + + /* non-atomic as we're under Linux' mm lock */ + if (--map->count) + return; + + KBASE_DEBUG_ASSERT(map->kctx); + KBASE_DEBUG_ASSERT(map->alloc); + + kbase_gpu_vm_lock(map->kctx); + + if (map->free_on_close) { + KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) == + KBASE_REG_ZONE_SAME_VA); + /* Avoid freeing memory on the process death which results in + * GPU Page Fault. Memory will be freed in kbase_destroy_context + */ + if (!(current->flags & PF_EXITING)) + kbase_mem_free_region(map->kctx, map->region); + } + + list_del(&map->mappings_list); + + kbase_gpu_vm_unlock(map->kctx); + + kbase_mem_phy_alloc_put(map->alloc); + kfree(map); +} + +KBASE_EXPORT_TEST_API(kbase_cpu_vm_close); + + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) +static int kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ +#else +static int kbase_cpu_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; +#endif + struct kbase_cpu_mapping *map = vma->vm_private_data; + pgoff_t rel_pgoff; + size_t i; + pgoff_t addr; + + KBASE_DEBUG_ASSERT(map); + KBASE_DEBUG_ASSERT(map->count > 0); + KBASE_DEBUG_ASSERT(map->kctx); + KBASE_DEBUG_ASSERT(map->alloc); + + rel_pgoff = vmf->pgoff - map->region->start_pfn; + + kbase_gpu_vm_lock(map->kctx); + if (rel_pgoff >= map->alloc->nents) + goto locked_bad_fault; + + /* Fault on access to DONT_NEED regions */ + if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED)) + goto locked_bad_fault; + + /* insert all valid pages from the fault location */ + i = rel_pgoff; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + addr = (pgoff_t)((uintptr_t)vmf->virtual_address >> PAGE_SHIFT); +#else + addr = (pgoff_t)(vmf->address >> PAGE_SHIFT); +#endif + while (i < map->alloc->nents && (addr < vma->vm_end >> PAGE_SHIFT)) { + int ret = vm_insert_pfn(vma, addr << PAGE_SHIFT, + PFN_DOWN(as_phys_addr_t(map->alloc->pages[i]))); + if (ret < 0 && ret != -EBUSY) + goto locked_bad_fault; + + i++; addr++; + } + + kbase_gpu_vm_unlock(map->kctx); + /* we resolved it, nothing for VM to do */ + return VM_FAULT_NOPAGE; + +locked_bad_fault: + kbase_gpu_vm_unlock(map->kctx); + return VM_FAULT_SIGBUS; +} + +const struct vm_operations_struct kbase_vm_ops = { + .open = kbase_cpu_vm_open, + .close = kbase_cpu_vm_close, + .fault = kbase_cpu_vm_fault +}; + +static int kbase_cpu_mmap(struct kbase_va_region *reg, struct vm_area_struct *vma, void *kaddr, size_t nr_pages, unsigned long aligned_offset, int free_on_close) +{ + struct kbase_cpu_mapping *map; + struct tagged_addr *page_array; + int err = 0; + int i; + + map = kzalloc(sizeof(*map), GFP_KERNEL); + + if (!map) { + WARN_ON(1); + err = -ENOMEM; + goto out; + } + + /* + * VM_DONTCOPY - don't make this mapping available in fork'ed processes + * VM_DONTEXPAND - disable mremap on this region + * VM_IO - disables paging + * VM_DONTDUMP - Don't include in core dumps (3.7 only) + * VM_MIXEDMAP - Support mixing struct page*s and raw pfns. + * This is needed to support using the dedicated and + * the OS based memory backends together. + */ + /* + * This will need updating to propagate coherency flags + * See MIDBASE-1057 + */ + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) + vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; +#else + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; +#endif + vma->vm_ops = &kbase_vm_ops; + vma->vm_private_data = map; + + page_array = kbase_get_cpu_phy_pages(reg); + + if (!(reg->flags & KBASE_REG_CPU_CACHED) && + (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) { + /* We can't map vmalloc'd memory uncached. + * Other memory will have been returned from + * kbase_mem_pool which would be + * suitable for mapping uncached. + */ + BUG_ON(kaddr); + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + } + + if (!kaddr) { + unsigned long addr = vma->vm_start + aligned_offset; + u64 start_off = vma->vm_pgoff - reg->start_pfn + + (aligned_offset>>PAGE_SHIFT); + + vma->vm_flags |= VM_PFNMAP; + for (i = 0; i < nr_pages; i++) { + phys_addr_t phys; + + phys = as_phys_addr_t(page_array[i + start_off]); + err = vm_insert_pfn(vma, addr, PFN_DOWN(phys)); + if (WARN_ON(err)) + break; + + addr += PAGE_SIZE; + } + } else { + WARN_ON(aligned_offset); + /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */ + vma->vm_flags |= VM_MIXEDMAP; + /* vmalloc remaping is easy... */ + err = remap_vmalloc_range(vma, kaddr, 0); + WARN_ON(err); + } + + if (err) { + kfree(map); + goto out; + } + + map->region = reg; + map->free_on_close = free_on_close; + map->kctx = reg->kctx; + map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + map->count = 1; /* start with one ref */ + + if (reg->flags & KBASE_REG_CPU_CACHED) + map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; + + list_add(&map->mappings_list, &map->alloc->mappings); + + out: + return err; +} + +static int kbase_trace_buffer_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kaddr) +{ + struct kbase_va_region *new_reg; + u32 nr_pages; + size_t size; + int err = 0; + u32 *tb; + int owns_tb = 1; + + dev_dbg(kctx->kbdev->dev, "in %s\n", __func__); + size = (vma->vm_end - vma->vm_start); + nr_pages = size >> PAGE_SHIFT; + + if (!kctx->jctx.tb) { + KBASE_DEBUG_ASSERT(0 != size); + tb = vmalloc_user(size); + + if (NULL == tb) { + err = -ENOMEM; + goto out; + } + + err = kbase_device_trace_buffer_install(kctx, tb, size); + if (err) { + vfree(tb); + goto out; + } + } else { + err = -EINVAL; + goto out; + } + + *kaddr = kctx->jctx.tb; + + new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA); + if (!new_reg) { + err = -ENOMEM; + WARN_ON(1); + goto out_no_region; + } + + new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB); + if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { + err = -ENOMEM; + new_reg->cpu_alloc = NULL; + WARN_ON(1); + goto out_no_alloc; + } + + new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); + + new_reg->cpu_alloc->imported.kctx = kctx; + new_reg->flags &= ~KBASE_REG_FREE; + new_reg->flags |= KBASE_REG_CPU_CACHED; + + /* alloc now owns the tb */ + owns_tb = 0; + + if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { + err = -ENOMEM; + WARN_ON(1); + goto out_no_va_region; + } + + *reg = new_reg; + + /* map read only, noexec */ + vma->vm_flags &= ~(VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); + /* the rest of the flags is added by the cpu_mmap handler */ + + dev_dbg(kctx->kbdev->dev, "%s done\n", __func__); + return 0; + +out_no_va_region: +out_no_alloc: + kbase_free_alloced_region(new_reg); +out_no_region: + if (owns_tb) { + kbase_device_trace_buffer_uninstall(kctx); + vfree(tb); + } +out: + return err; +} + +static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kmap_addr) +{ + struct kbase_va_region *new_reg; + void *kaddr; + u32 nr_pages; + size_t size; + int err = 0; + + dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n"); + size = (vma->vm_end - vma->vm_start); + nr_pages = size >> PAGE_SHIFT; + + kaddr = kbase_mmu_dump(kctx, nr_pages); + + if (!kaddr) { + err = -ENOMEM; + goto out; + } + + new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA); + if (!new_reg) { + err = -ENOMEM; + WARN_ON(1); + goto out; + } + + new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW); + if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { + err = -ENOMEM; + new_reg->cpu_alloc = NULL; + WARN_ON(1); + goto out_no_alloc; + } + + new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); + + new_reg->flags &= ~KBASE_REG_FREE; + new_reg->flags |= KBASE_REG_CPU_CACHED; + if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { + err = -ENOMEM; + WARN_ON(1); + goto out_va_region; + } + + *kmap_addr = kaddr; + *reg = new_reg; + + dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n"); + return 0; + +out_no_alloc: +out_va_region: + kbase_free_alloced_region(new_reg); +out: + return err; +} + + +void kbase_os_mem_map_lock(struct kbase_context *kctx) +{ + struct mm_struct *mm = current->mm; + (void)kctx; + down_read(&mm->mmap_sem); +} + +void kbase_os_mem_map_unlock(struct kbase_context *kctx) +{ + struct mm_struct *mm = current->mm; + (void)kctx; + up_read(&mm->mmap_sem); +} + +static int kbasep_reg_mmap(struct kbase_context *kctx, + struct vm_area_struct *vma, + struct kbase_va_region **regm, + size_t *nr_pages, size_t *aligned_offset) + +{ + int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); + struct kbase_va_region *reg; + int err = 0; + + *aligned_offset = 0; + + dev_dbg(kctx->kbdev->dev, "in kbasep_reg_mmap\n"); + + /* SAME_VA stuff, fetch the right region */ + reg = kctx->pending_regions[cookie]; + if (!reg) { + err = -ENOMEM; + goto out; + } + + if ((reg->flags & KBASE_REG_GPU_NX) && (reg->nr_pages != *nr_pages)) { + /* incorrect mmap size */ + /* leave the cookie for a potential later + * mapping, or to be reclaimed later when the + * context is freed */ + err = -ENOMEM; + goto out; + } + + if ((vma->vm_flags & VM_READ && !(reg->flags & KBASE_REG_CPU_RD)) || + (vma->vm_flags & VM_WRITE && !(reg->flags & KBASE_REG_CPU_WR))) { + /* VM flags inconsistent with region flags */ + err = -EPERM; + dev_err(kctx->kbdev->dev, "%s:%d inconsistent VM flags\n", + __FILE__, __LINE__); + goto out; + } + + /* adjust down nr_pages to what we have physically */ + *nr_pages = kbase_reg_current_backed_size(reg); + + if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset, + reg->nr_pages, 1) != 0) { + dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__); + /* Unable to map in GPU space. */ + WARN_ON(1); + err = -ENOMEM; + goto out; + } + /* no need for the cookie anymore */ + kctx->pending_regions[cookie] = NULL; + kctx->cookies |= (1UL << cookie); + + /* + * Overwrite the offset with the region start_pfn, so we effectively + * map from offset 0 in the region. However subtract the aligned + * offset so that when user space trims the mapping the beginning of + * the trimmed VMA has the correct vm_pgoff; + */ + vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT); +out: + *regm = reg; + dev_dbg(kctx->kbdev->dev, "kbasep_reg_mmap done\n"); + + return err; +} + +int kbase_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct kbase_context *kctx = file->private_data; + struct kbase_va_region *reg = NULL; + void *kaddr = NULL; + size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + int err = 0; + int free_on_close = 0; + struct device *dev = kctx->kbdev->dev; + size_t aligned_offset = 0; + + dev_dbg(dev, "kbase_mmap\n"); + + /* strip away corresponding VM_MAY% flags to the VM_% flags requested */ + vma->vm_flags &= ~((vma->vm_flags & (VM_READ | VM_WRITE)) << 4); + + if (0 == nr_pages) { + err = -EINVAL; + goto out; + } + + if (!(vma->vm_flags & VM_SHARED)) { + err = -EINVAL; + goto out; + } + + kbase_gpu_vm_lock(kctx); + + if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) { + /* The non-mapped tracking helper page */ + err = kbase_tracking_page_setup(kctx, vma); + goto out_unlock; + } + + /* if not the MTP, verify that the MTP has been mapped */ + rcu_read_lock(); + /* catches both when the special page isn't present or + * when we've forked */ + if (rcu_dereference(kctx->process_mm) != current->mm) { + err = -EINVAL; + rcu_read_unlock(); + goto out_unlock; + } + rcu_read_unlock(); + + switch (vma->vm_pgoff) { + case PFN_DOWN(BASEP_MEM_INVALID_HANDLE): + case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE): + /* Illegal handle for direct map */ + err = -EINVAL; + goto out_unlock; + case PFN_DOWN(BASE_MEM_TRACE_BUFFER_HANDLE): + err = kbase_trace_buffer_mmap(kctx, vma, ®, &kaddr); + if (0 != err) + goto out_unlock; + dev_dbg(dev, "kbase_trace_buffer_mmap ok\n"); + /* free the region on munmap */ + free_on_close = 1; + break; + case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE): + /* MMU dump */ + err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr); + if (0 != err) + goto out_unlock; + /* free the region on munmap */ + free_on_close = 1; + break; + case PFN_DOWN(BASE_MEM_COOKIE_BASE) ... + PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { + err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, + &aligned_offset); + if (0 != err) + goto out_unlock; + /* free the region on munmap */ + free_on_close = 1; + break; + } + default: { + reg = kbase_region_tracker_find_region_enclosing_address(kctx, + (u64)vma->vm_pgoff << PAGE_SHIFT); + + if (reg && !(reg->flags & KBASE_REG_FREE)) { + /* will this mapping overflow the size of the region? */ + if (nr_pages > (reg->nr_pages - + (vma->vm_pgoff - reg->start_pfn))) { + err = -ENOMEM; + goto out_unlock; + } + + if ((vma->vm_flags & VM_READ && + !(reg->flags & KBASE_REG_CPU_RD)) || + (vma->vm_flags & VM_WRITE && + !(reg->flags & KBASE_REG_CPU_WR))) { + /* VM flags inconsistent with region flags */ + err = -EPERM; + dev_err(dev, "%s:%d inconsistent VM flags\n", + __FILE__, __LINE__); + goto out_unlock; + } + +#ifdef CONFIG_DMA_SHARED_BUFFER + if (KBASE_MEM_TYPE_IMPORTED_UMM == + reg->cpu_alloc->type) { + err = dma_buf_mmap( + reg->cpu_alloc->imported.umm.dma_buf, + vma, vma->vm_pgoff - reg->start_pfn); + goto out_unlock; + } +#endif /* CONFIG_DMA_SHARED_BUFFER */ + + /* limit what we map to the amount currently backed */ + if (reg->cpu_alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) { + if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents) + nr_pages = 0; + else + nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn); + } + } else { + err = -ENOMEM; + goto out_unlock; + } + } /* default */ + } /* switch */ + + err = kbase_cpu_mmap(reg, vma, kaddr, nr_pages, aligned_offset, free_on_close); + + if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) { + /* MMU dump - userspace should now have a reference on + * the pages, so we can now free the kernel mapping */ + vfree(kaddr); + } + +out_unlock: + kbase_gpu_vm_unlock(kctx); +out: + if (err) + dev_err(dev, "mmap failed %d\n", err); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mmap); + +static void kbasep_sync_mem_regions(struct kbase_context *kctx, + struct kbase_vmap_struct *map, enum kbase_sync_type dest) +{ + size_t i; + off_t const offset = (uintptr_t)map->gpu_addr & ~PAGE_MASK; + size_t const page_count = PFN_UP(offset + map->size); + + /* Sync first page */ + size_t sz = MIN(((size_t) PAGE_SIZE - offset), map->size); + struct tagged_addr cpu_pa = map->cpu_pages[0]; + struct tagged_addr gpu_pa = map->gpu_pages[0]; + + kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz, dest); + + /* Sync middle pages (if any) */ + for (i = 1; page_count > 2 && i < page_count - 1; i++) { + cpu_pa = map->cpu_pages[i]; + gpu_pa = map->gpu_pages[i]; + kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE, dest); + } + + /* Sync last page (if any) */ + if (page_count > 1) { + cpu_pa = map->cpu_pages[page_count - 1]; + gpu_pa = map->gpu_pages[page_count - 1]; + sz = ((offset + map->size - 1) & ~PAGE_MASK) + 1; + kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, dest); + } +} + +void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, + unsigned long prot_request, struct kbase_vmap_struct *map) +{ + struct kbase_va_region *reg; + unsigned long page_index; + unsigned int offset = gpu_addr & ~PAGE_MASK; + size_t page_count = PFN_UP(offset + size); + struct tagged_addr *page_array; + struct page **pages; + void *cpu_addr = NULL; + pgprot_t prot; + size_t i; + + if (!size || !map) + return NULL; + + /* check if page_count calculation will wrap */ + if (size > ((size_t)-1 / PAGE_SIZE)) + return NULL; + + kbase_gpu_vm_lock(kctx); + + reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr); + if (!reg || (reg->flags & KBASE_REG_FREE)) + goto out_unlock; + + page_index = (gpu_addr >> PAGE_SHIFT) - reg->start_pfn; + + /* check if page_index + page_count will wrap */ + if (-1UL - page_count < page_index) + goto out_unlock; + + if (page_index + page_count > kbase_reg_current_backed_size(reg)) + goto out_unlock; + + if (reg->flags & KBASE_REG_DONT_NEED) + goto out_unlock; + + /* check access permissions can be satisfied + * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} */ + if ((reg->flags & prot_request) != prot_request) + goto out_unlock; + + page_array = kbase_get_cpu_phy_pages(reg); + if (!page_array) + goto out_unlock; + + pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); + if (!pages) + goto out_unlock; + + for (i = 0; i < page_count; i++) + pages[i] = phys_to_page(as_phys_addr_t(page_array[page_index + + i])); + + prot = PAGE_KERNEL; + if (!(reg->flags & KBASE_REG_CPU_CACHED)) { + /* Map uncached */ + prot = pgprot_writecombine(prot); + } + /* Note: enforcing a RO prot_request onto prot is not done, since: + * - CPU-arch-specific integration required + * - kbase_vmap() requires no access checks to be made/enforced */ + + cpu_addr = vmap(pages, page_count, VM_MAP, prot); + + kfree(pages); + + if (!cpu_addr) + goto out_unlock; + + map->gpu_addr = gpu_addr; + map->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index]; + map->gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index]; + map->addr = (void *)((uintptr_t)cpu_addr + offset); + map->size = size; + map->sync_needed = ((reg->flags & KBASE_REG_CPU_CACHED) != 0) && + !kbase_mem_is_imported(map->gpu_alloc->type); + + if (map->sync_needed) + kbasep_sync_mem_regions(kctx, map, KBASE_SYNC_TO_CPU); + kbase_gpu_vm_unlock(kctx); + + return map->addr; + +out_unlock: + kbase_gpu_vm_unlock(kctx); + return NULL; +} + +void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, + struct kbase_vmap_struct *map) +{ + /* 0 is specified for prot_request to indicate no access checks should + * be made. + * + * As mentioned in kbase_vmap_prot() this means that a kernel-side + * CPU-RO mapping is not enforced to allow this to work */ + return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map); +} +KBASE_EXPORT_TEST_API(kbase_vmap); + +void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map) +{ + void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK); + vunmap(addr); + + if (map->sync_needed) + kbasep_sync_mem_regions(kctx, map, KBASE_SYNC_TO_DEVICE); + map->gpu_addr = 0; + map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc); + map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc); + map->cpu_pages = NULL; + map->gpu_pages = NULL; + map->addr = NULL; + map->size = 0; + map->sync_needed = false; +} +KBASE_EXPORT_TEST_API(kbase_vunmap); + +void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages) +{ + struct mm_struct *mm; + + rcu_read_lock(); + mm = rcu_dereference(kctx->process_mm); + if (mm) { + atomic_add(pages, &kctx->nonmapped_pages); +#ifdef SPLIT_RSS_COUNTING + add_mm_counter(mm, MM_FILEPAGES, pages); +#else + spin_lock(&mm->page_table_lock); + add_mm_counter(mm, MM_FILEPAGES, pages); + spin_unlock(&mm->page_table_lock); +#endif + } + rcu_read_unlock(); +} + +static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx) +{ + int pages; + struct mm_struct *mm; + + spin_lock(&kctx->mm_update_lock); + mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock)); + if (!mm) { + spin_unlock(&kctx->mm_update_lock); + return; + } + + rcu_assign_pointer(kctx->process_mm, NULL); + spin_unlock(&kctx->mm_update_lock); + synchronize_rcu(); + + pages = atomic_xchg(&kctx->nonmapped_pages, 0); +#ifdef SPLIT_RSS_COUNTING + add_mm_counter(mm, MM_FILEPAGES, -pages); +#else + spin_lock(&mm->page_table_lock); + add_mm_counter(mm, MM_FILEPAGES, -pages); + spin_unlock(&mm->page_table_lock); +#endif +} + +static void kbase_special_vm_close(struct vm_area_struct *vma) +{ + struct kbase_context *kctx; + + kctx = vma->vm_private_data; + kbasep_os_process_page_usage_drain(kctx); +} + +static const struct vm_operations_struct kbase_vm_special_ops = { + .close = kbase_special_vm_close, +}; + +static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma) +{ + /* check that this is the only tracking page */ + spin_lock(&kctx->mm_update_lock); + if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) { + spin_unlock(&kctx->mm_update_lock); + return -EFAULT; + } + + rcu_assign_pointer(kctx->process_mm, current->mm); + + spin_unlock(&kctx->mm_update_lock); + + /* no real access */ + vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; +#else + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; +#endif + vma->vm_ops = &kbase_vm_special_ops; + vma->vm_private_data = kctx; + + return 0; +} +void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle) +{ + int res; + void *va; + dma_addr_t dma_pa; + struct kbase_va_region *reg; + struct tagged_addr *page_array; +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) + unsigned long attrs = DMA_ATTR_WRITE_COMBINE; +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) + DEFINE_DMA_ATTRS(attrs); +#endif + + u32 pages = ((size - 1) >> PAGE_SHIFT) + 1; + u32 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR | + BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR; + u32 i; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(0 != size); + KBASE_DEBUG_ASSERT(0 != pages); + + if (size == 0) + goto err; + + /* All the alloc calls return zeroed memory */ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) + va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, + attrs); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) + dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); + va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, + &attrs); +#else + va = dma_alloc_writecombine(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL); +#endif + if (!va) + goto err; + + /* Store the state so we can free it later. */ + handle->cpu_va = va; + handle->dma_pa = dma_pa; + handle->size = size; + + + reg = kbase_alloc_free_region(kctx, 0, pages, KBASE_REG_ZONE_SAME_VA); + if (!reg) + goto no_reg; + + reg->flags &= ~KBASE_REG_FREE; + if (kbase_update_region_flags(kctx, reg, flags) != 0) + goto invalid_flags; + + reg->cpu_alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW); + if (IS_ERR_OR_NULL(reg->cpu_alloc)) + goto no_alloc; + + reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + + page_array = kbase_get_cpu_phy_pages(reg); + + for (i = 0; i < pages; i++) + page_array[i] = as_tagged(dma_pa + (i << PAGE_SHIFT)); + + reg->cpu_alloc->nents = pages; + + kbase_gpu_vm_lock(kctx); + res = kbase_gpu_mmap(kctx, reg, (uintptr_t) va, pages, 1); + kbase_gpu_vm_unlock(kctx); + if (res) + goto no_mmap; + + return va; + +no_mmap: + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +no_alloc: +invalid_flags: + kfree(reg); +no_reg: +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) + dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, attrs); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) + dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, &attrs); +#else + dma_free_writecombine(kctx->kbdev->dev, size, va, dma_pa); +#endif +err: + return NULL; +} +KBASE_EXPORT_SYMBOL(kbase_va_alloc); + +void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle) +{ + struct kbase_va_region *reg; + int err; +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + DEFINE_DMA_ATTRS(attrs); +#endif + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(handle->cpu_va != NULL); + + kbase_gpu_vm_lock(kctx); + reg = kbase_region_tracker_find_region_base_address(kctx, (uintptr_t)handle->cpu_va); + KBASE_DEBUG_ASSERT(reg); + err = kbase_gpu_munmap(kctx, reg); + kbase_gpu_vm_unlock(kctx); + KBASE_DEBUG_ASSERT(!err); + + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); + kfree(reg); + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) + dma_free_attrs(kctx->kbdev->dev, handle->size, + handle->cpu_va, handle->dma_pa, DMA_ATTR_WRITE_COMBINE); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) + dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); + dma_free_attrs(kctx->kbdev->dev, handle->size, + handle->cpu_va, handle->dma_pa, &attrs); +#else + dma_free_writecombine(kctx->kbdev->dev, handle->size, + handle->cpu_va, handle->dma_pa); +#endif +} +KBASE_EXPORT_SYMBOL(kbase_va_free); + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.h new file mode 100755 index 000000000000..db35f62a7431 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_linux.h @@ -0,0 +1,240 @@ +/* + * + * (C) COPYRIGHT 2010, 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mem_linux.h + * Base kernel memory APIs, Linux implementation. + */ + +#ifndef _KBASE_MEM_LINUX_H_ +#define _KBASE_MEM_LINUX_H_ + +/** A HWC dump mapping */ +struct kbase_hwc_dma_mapping { + void *cpu_va; + dma_addr_t dma_pa; + size_t size; +}; + +struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, + u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, + u64 *gpu_va); +int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 *const pages); +int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, + u64 *flags); +u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages); +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask); + +/** + * kbase_mem_commit - Change the physical backing size of a region + * + * @kctx: The kernel context + * @gpu_addr: Handle to the memory region + * @new_pages: Number of physical pages to back the region with + * + * Return: 0 on success or error code + */ +int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages); + +int kbase_mmap(struct file *file, struct vm_area_struct *vma); + +/** + * kbase_mem_evictable_init - Initialize the Ephemeral memory the eviction + * mechanism. + * @kctx: The kbase context to initialize. + * + * Return: Zero on success or -errno on failure. + */ +int kbase_mem_evictable_init(struct kbase_context *kctx); + +/** + * kbase_mem_evictable_deinit - De-initialize the Ephemeral memory eviction + * mechanism. + * @kctx: The kbase context to de-initialize. + */ +void kbase_mem_evictable_deinit(struct kbase_context *kctx); + +/** + * kbase_mem_grow_gpu_mapping - Grow the GPU mapping of an allocation + * @kctx: Context the region belongs to + * @reg: The GPU region + * @new_pages: The number of pages after the grow + * @old_pages: The number of pages before the grow + * + * Return: 0 on success, -errno on error. + * + * Expand the GPU mapping to encompass the new psychical pages which have + * been added to the allocation. + * + * Note: Caller must be holding the region lock. + */ +int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages); + +/** + * kbase_mem_evictable_make - Make a physical allocation eligible for eviction + * @gpu_alloc: The physical allocation to make evictable + * + * Return: 0 on success, -errno on error. + * + * Take the provided region and make all the physical pages within it + * reclaimable by the kernel, updating the per-process VM stats as well. + * Remove any CPU mappings (as these can't be removed in the shrinker callback + * as mmap_sem might already be taken) but leave the GPU mapping intact as + * and until the shrinker reclaims the allocation. + * + * Note: Must be called with the region lock of the containing context. + */ +int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc); + +/** + * kbase_mem_evictable_unmake - Remove a physical allocations eligibility for + * eviction. + * @alloc: The physical allocation to remove eviction eligibility from. + * + * Return: True if the allocation had its backing restored and false if + * it hasn't. + * + * Make the physical pages in the region no longer reclaimable and update the + * per-process stats, if the shrinker has already evicted the memory then + * re-allocate it if the region is still alive. + * + * Note: Must be called with the region lock of the containing context. + */ +bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *alloc); + +struct kbase_vmap_struct { + u64 gpu_addr; + struct kbase_mem_phy_alloc *cpu_alloc; + struct kbase_mem_phy_alloc *gpu_alloc; + struct tagged_addr *cpu_pages; + struct tagged_addr *gpu_pages; + void *addr; + size_t size; + bool sync_needed; +}; + + +/** + * kbase_vmap_prot - Map a GPU VA range into the kernel safely, only if the + * requested access permissions are supported + * @kctx: Context the VA range belongs to + * @gpu_addr: Start address of VA range + * @size: Size of VA range + * @prot_request: Flags indicating how the caller will then access the memory + * @map: Structure to be given to kbase_vunmap() on freeing + * + * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error + * + * Map a GPU VA Range into the kernel. The VA range must be contained within a + * GPU memory region. Appropriate CPU cache-flushing operations are made as + * required, dependent on the CPU mapping for the memory region. + * + * This is safer than using kmap() on the pages directly, + * because the pages here are refcounted to prevent freeing (and hence reuse + * elsewhere in the system) until an kbase_vunmap() + * + * The flags in @prot_request should use KBASE_REG_{CPU,GPU}_{RD,WR}, to check + * whether the region should allow the intended access, and return an error if + * disallowed. This is essential for security of imported memory, particularly + * a user buf from SHM mapped into the process as RO. In that case, write + * access must be checked if the intention is for kernel to write to the + * memory. + * + * The checks are also there to help catch access errors on memory where + * security is not a concern: imported memory that is always RW, and memory + * that was allocated and owned by the process attached to @kctx. In this case, + * it helps to identify memory that was was mapped with the wrong access type. + * + * Note: KBASE_REG_GPU_{RD,WR} flags are currently supported for legacy cases + * where either the security of memory is solely dependent on those flags, or + * when userspace code was expecting only the GPU to access the memory (e.g. HW + * workarounds). + * + * All cache maintenance operations shall be ignored if the + * memory region has been imported. + * + */ +void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, + unsigned long prot_request, struct kbase_vmap_struct *map); + +/** + * kbase_vmap - Map a GPU VA range into the kernel safely + * @kctx: Context the VA range belongs to + * @gpu_addr: Start address of VA range + * @size: Size of VA range + * @map: Structure to be given to kbase_vunmap() on freeing + * + * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error + * + * Map a GPU VA Range into the kernel. The VA range must be contained within a + * GPU memory region. Appropriate CPU cache-flushing operations are made as + * required, dependent on the CPU mapping for the memory region. + * + * This is safer than using kmap() on the pages directly, + * because the pages here are refcounted to prevent freeing (and hence reuse + * elsewhere in the system) until an kbase_vunmap() + * + * kbase_vmap_prot() should be used in preference, since kbase_vmap() makes no + * checks to ensure the security of e.g. imported user bufs from RO SHM. + * + * Note: All cache maintenance operations shall be ignored if the memory region + * has been imported. + */ +void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, + struct kbase_vmap_struct *map); + +/** + * kbase_vunmap - Unmap a GPU VA range from the kernel + * @kctx: Context the VA range belongs to + * @map: Structure describing the mapping from the corresponding kbase_vmap() + * call + * + * Unmaps a GPU VA range from the kernel, given its @map structure obtained + * from kbase_vmap(). Appropriate CPU cache-flushing operations are made as + * required, dependent on the CPU mapping for the memory region. + * + * The reference taken on pages during kbase_vmap() is released. + * + * Note: All cache maintenance operations shall be ignored if the memory region + * has been imported. + */ +void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map); + +/** @brief Allocate memory from kernel space and map it onto the GPU + * + * @param kctx The context used for the allocation/mapping + * @param size The size of the allocation in bytes + * @param handle An opaque structure used to contain the state needed to free the memory + * @return the VA for kernel space and GPU MMU + */ +void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle); + +/** @brief Free/unmap memory allocated by kbase_va_alloc + * + * @param kctx The context used for the allocation/mapping + * @param handle An opaque structure returned by the kbase_va_alloc function. + */ +void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle); + +extern const struct vm_operations_struct kbase_vm_ops; + +#endif /* _KBASE_MEM_LINUX_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_lowlevel.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_lowlevel.h new file mode 100755 index 000000000000..f4e88491327e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_lowlevel.h @@ -0,0 +1,89 @@ +/* + * + * (C) COPYRIGHT 2012-2014,2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_MEM_LOWLEVEL_H +#define _KBASE_MEM_LOWLEVEL_H + +#ifndef _KBASE_H_ +#error "Don't include this file directly, use mali_kbase.h instead" +#endif + +#include + +/** + * @brief Flags for kbase_phy_allocator_pages_alloc + */ +#define KBASE_PHY_PAGES_FLAG_DEFAULT (0) /** Default allocation flag */ +#define KBASE_PHY_PAGES_FLAG_CLEAR (1 << 0) /** Clear the pages after allocation */ +#define KBASE_PHY_PAGES_FLAG_POISON (1 << 1) /** Fill the memory with a poison value */ + +#define KBASE_PHY_PAGES_SUPPORTED_FLAGS (KBASE_PHY_PAGES_FLAG_DEFAULT|KBASE_PHY_PAGES_FLAG_CLEAR|KBASE_PHY_PAGES_FLAG_POISON) + +#define KBASE_PHY_PAGES_POISON_VALUE 0xFD /** Value to fill the memory with when KBASE_PHY_PAGES_FLAG_POISON is set */ + +enum kbase_sync_type { + KBASE_SYNC_TO_CPU, + KBASE_SYNC_TO_DEVICE +}; + +struct tagged_addr { phys_addr_t tagged_addr; }; + +#define HUGE_PAGE (1u << 0) +#define HUGE_HEAD (1u << 1) +#define FROM_PARTIAL (1u << 2) + +static inline phys_addr_t as_phys_addr_t(struct tagged_addr t) +{ + return t.tagged_addr & PAGE_MASK; +} + +static inline struct tagged_addr as_tagged(phys_addr_t phys) +{ + struct tagged_addr t; + + t.tagged_addr = phys & PAGE_MASK; + return t; +} + +static inline struct tagged_addr as_tagged_tag(phys_addr_t phys, int tag) +{ + struct tagged_addr t; + + t.tagged_addr = (phys & PAGE_MASK) | (tag & ~PAGE_MASK); + return t; +} + +static inline bool is_huge(struct tagged_addr t) +{ + return t.tagged_addr & HUGE_PAGE; +} + +static inline bool is_huge_head(struct tagged_addr t) +{ + int mask = HUGE_HEAD | HUGE_PAGE; + + return mask == (t.tagged_addr & mask); +} + +static inline bool is_partial(struct tagged_addr t) +{ + return t.tagged_addr & FROM_PARTIAL; +} + +#endif /* _KBASE_LOWLEVEL_H */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool.c new file mode 100755 index 000000000000..696730ac5b2b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool.c @@ -0,0 +1,651 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include +#include +#include +#include +#include + +#define pool_dbg(pool, format, ...) \ + dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \ + (pool->next_pool) ? "kctx" : "kbdev", \ + kbase_mem_pool_size(pool), \ + kbase_mem_pool_max_size(pool), \ + ##__VA_ARGS__) + +#define NOT_DIRTY false +#define NOT_RECLAIMED false + +static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool) +{ + spin_lock(&pool->pool_lock); +} + +static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) +{ + spin_unlock(&pool->pool_lock); +} + +static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool) +{ + ssize_t max_size = kbase_mem_pool_max_size(pool); + ssize_t cur_size = kbase_mem_pool_size(pool); + + return max(max_size - cur_size, (ssize_t)0); +} + +static bool kbase_mem_pool_is_full(struct kbase_mem_pool *pool) +{ + return kbase_mem_pool_size(pool) >= kbase_mem_pool_max_size(pool); +} + +static bool kbase_mem_pool_is_empty(struct kbase_mem_pool *pool) +{ + return kbase_mem_pool_size(pool) == 0; +} + +static void kbase_mem_pool_add_locked(struct kbase_mem_pool *pool, + struct page *p) +{ + lockdep_assert_held(&pool->pool_lock); + + list_add(&p->lru, &pool->page_list); + pool->cur_size++; + + zone_page_state_add(1, page_zone(p), NR_SLAB_RECLAIMABLE); + + pool_dbg(pool, "added page\n"); +} + +static void kbase_mem_pool_add(struct kbase_mem_pool *pool, struct page *p) +{ + kbase_mem_pool_lock(pool); + kbase_mem_pool_add_locked(pool, p); + kbase_mem_pool_unlock(pool); +} + +static void kbase_mem_pool_add_list_locked(struct kbase_mem_pool *pool, + struct list_head *page_list, size_t nr_pages) +{ + struct page *p; + + lockdep_assert_held(&pool->pool_lock); + + list_for_each_entry(p, page_list, lru) { + zone_page_state_add(1, page_zone(p), NR_SLAB_RECLAIMABLE); + } + + list_splice(page_list, &pool->page_list); + pool->cur_size += nr_pages; + + pool_dbg(pool, "added %zu pages\n", nr_pages); +} + +static void kbase_mem_pool_add_list(struct kbase_mem_pool *pool, + struct list_head *page_list, size_t nr_pages) +{ + kbase_mem_pool_lock(pool); + kbase_mem_pool_add_list_locked(pool, page_list, nr_pages); + kbase_mem_pool_unlock(pool); +} + +static struct page *kbase_mem_pool_remove_locked(struct kbase_mem_pool *pool) +{ + struct page *p; + + lockdep_assert_held(&pool->pool_lock); + + if (kbase_mem_pool_is_empty(pool)) + return NULL; + + p = list_first_entry(&pool->page_list, struct page, lru); + list_del_init(&p->lru); + pool->cur_size--; + + zone_page_state_add(-1, page_zone(p), NR_SLAB_RECLAIMABLE); + + pool_dbg(pool, "removed page\n"); + + return p; +} + +static struct page *kbase_mem_pool_remove(struct kbase_mem_pool *pool) +{ + struct page *p; + + kbase_mem_pool_lock(pool); + p = kbase_mem_pool_remove_locked(pool); + kbase_mem_pool_unlock(pool); + + return p; +} + +static void kbase_mem_pool_sync_page(struct kbase_mem_pool *pool, + struct page *p) +{ + struct device *dev = pool->kbdev->dev; + dma_sync_single_for_device(dev, kbase_dma_addr(p), + (PAGE_SIZE << pool->order), DMA_BIDIRECTIONAL); +} + +static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool, + struct page *p) +{ + int i; + + for (i = 0; i < (1U << pool->order); i++) + clear_highpage(p+i); + + kbase_mem_pool_sync_page(pool, p); +} + +static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, + struct page *p) +{ + /* Zero page before spilling */ + kbase_mem_pool_zero_page(next_pool, p); + + kbase_mem_pool_add(next_pool, p); +} + +struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) +{ + struct page *p; + gfp_t gfp; + struct device *dev = pool->kbdev->dev; + dma_addr_t dma_addr; + int i; + +#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) + /* DMA cache sync fails for HIGHMEM before 3.5 on ARM */ + gfp = GFP_USER | __GFP_ZERO; +#else + gfp = GFP_HIGHUSER | __GFP_ZERO; +#endif + + if (current->flags & PF_KTHREAD) { + /* Don't trigger OOM killer from kernel threads, e.g. when + * growing memory on GPU page fault */ + gfp |= __GFP_NORETRY; + } + + /* don't warn on higer order failures */ + if (pool->order) + gfp |= __GFP_NOWARN; + + p = alloc_pages(gfp, pool->order); + if (!p) + return NULL; + + dma_addr = dma_map_page(dev, p, 0, (PAGE_SIZE << pool->order), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) { + __free_pages(p, pool->order); + return NULL; + } + + WARN_ON(dma_addr != page_to_phys(p)); + for (i = 0; i < (1u << pool->order); i++) + kbase_set_dma_addr(p+i, dma_addr + PAGE_SIZE * i); + + return p; +} + +static void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, + struct page *p) +{ + struct device *dev = pool->kbdev->dev; + dma_addr_t dma_addr = kbase_dma_addr(p); + int i; + + dma_unmap_page(dev, dma_addr, (PAGE_SIZE << pool->order), + DMA_BIDIRECTIONAL); + for (i = 0; i < (1u << pool->order); i++) + kbase_clear_dma_addr(p+i); + __free_pages(p, pool->order); + + pool_dbg(pool, "freed page to kernel\n"); +} + +static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool, + size_t nr_to_shrink) +{ + struct page *p; + size_t i; + + lockdep_assert_held(&pool->pool_lock); + + for (i = 0; i < nr_to_shrink && !kbase_mem_pool_is_empty(pool); i++) { + p = kbase_mem_pool_remove_locked(pool); + kbase_mem_pool_free_page(pool, p); + } + + return i; +} + +static size_t kbase_mem_pool_shrink(struct kbase_mem_pool *pool, + size_t nr_to_shrink) +{ + size_t nr_freed; + + kbase_mem_pool_lock(pool); + nr_freed = kbase_mem_pool_shrink_locked(pool, nr_to_shrink); + kbase_mem_pool_unlock(pool); + + return nr_freed; +} + +int kbase_mem_pool_grow(struct kbase_mem_pool *pool, + size_t nr_to_grow) +{ + struct page *p; + size_t i; + + for (i = 0; i < nr_to_grow; i++) { + p = kbase_mem_alloc_page(pool); + if (!p) + return -ENOMEM; + kbase_mem_pool_add(pool, p); + } + + return 0; +} + +void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size) +{ + size_t cur_size; + int err = 0; + + cur_size = kbase_mem_pool_size(pool); + + if (new_size > pool->max_size) + new_size = pool->max_size; + + if (new_size < cur_size) + kbase_mem_pool_shrink(pool, cur_size - new_size); + else if (new_size > cur_size) + err = kbase_mem_pool_grow(pool, new_size - cur_size); + + if (err) { + size_t grown_size = kbase_mem_pool_size(pool); + + dev_warn(pool->kbdev->dev, + "Mem pool not grown to the required size of %zu bytes, grown for additional %zu bytes instead!\n", + (new_size - cur_size), (grown_size - cur_size)); + } +} + +void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size) +{ + size_t cur_size; + size_t nr_to_shrink; + + kbase_mem_pool_lock(pool); + + pool->max_size = max_size; + + cur_size = kbase_mem_pool_size(pool); + if (max_size < cur_size) { + nr_to_shrink = cur_size - max_size; + kbase_mem_pool_shrink_locked(pool, nr_to_shrink); + } + + kbase_mem_pool_unlock(pool); +} + + +static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_mem_pool *pool; + + pool = container_of(s, struct kbase_mem_pool, reclaim); + pool_dbg(pool, "reclaim count: %zu\n", kbase_mem_pool_size(pool)); + return kbase_mem_pool_size(pool); +} + +static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_mem_pool *pool; + unsigned long freed; + + pool = container_of(s, struct kbase_mem_pool, reclaim); + + pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan); + + freed = kbase_mem_pool_shrink(pool, sc->nr_to_scan); + + pool_dbg(pool, "reclaim freed %ld pages\n", freed); + + return freed; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static int kbase_mem_pool_reclaim_shrink(struct shrinker *s, + struct shrink_control *sc) +{ + if (sc->nr_to_scan == 0) + return kbase_mem_pool_reclaim_count_objects(s, sc); + + return kbase_mem_pool_reclaim_scan_objects(s, sc); +} +#endif + +int kbase_mem_pool_init(struct kbase_mem_pool *pool, + size_t max_size, + size_t order, + struct kbase_device *kbdev, + struct kbase_mem_pool *next_pool) +{ + pool->cur_size = 0; + pool->max_size = max_size; + pool->order = order; + pool->kbdev = kbdev; + pool->next_pool = next_pool; + + spin_lock_init(&pool->pool_lock); + INIT_LIST_HEAD(&pool->page_list); + + /* Register shrinker */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) + pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink; +#else + pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects; + pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects; +#endif + pool->reclaim.seeks = DEFAULT_SEEKS; + /* Kernel versions prior to 3.1 : + * struct shrinker does not define batch */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) + pool->reclaim.batch = 0; +#endif + register_shrinker(&pool->reclaim); + + pool_dbg(pool, "initialized\n"); + + return 0; +} + +void kbase_mem_pool_term(struct kbase_mem_pool *pool) +{ + struct kbase_mem_pool *next_pool = pool->next_pool; + struct page *p; + size_t nr_to_spill = 0; + LIST_HEAD(spill_list); + int i; + + pool_dbg(pool, "terminate()\n"); + + unregister_shrinker(&pool->reclaim); + + kbase_mem_pool_lock(pool); + pool->max_size = 0; + + if (next_pool && !kbase_mem_pool_is_full(next_pool)) { + /* Spill to next pool (may overspill) */ + nr_to_spill = kbase_mem_pool_capacity(next_pool); + nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill); + + /* Zero pages first without holding the next_pool lock */ + for (i = 0; i < nr_to_spill; i++) { + p = kbase_mem_pool_remove_locked(pool); + kbase_mem_pool_zero_page(pool, p); + list_add(&p->lru, &spill_list); + } + } + + while (!kbase_mem_pool_is_empty(pool)) { + /* Free remaining pages to kernel */ + p = kbase_mem_pool_remove_locked(pool); + kbase_mem_pool_free_page(pool, p); + } + + kbase_mem_pool_unlock(pool); + + if (next_pool && nr_to_spill) { + /* Add new page list to next_pool */ + kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill); + + pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill); + } + + pool_dbg(pool, "terminated\n"); +} + +struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool) +{ + struct page *p; + + do { + pool_dbg(pool, "alloc()\n"); + p = kbase_mem_pool_remove(pool); + + if (p) + return p; + + pool = pool->next_pool; + } while (pool); + + return NULL; +} + +void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, + bool dirty) +{ + struct kbase_mem_pool *next_pool = pool->next_pool; + + pool_dbg(pool, "free()\n"); + + if (!kbase_mem_pool_is_full(pool)) { + /* Add to our own pool */ + if (dirty) + kbase_mem_pool_sync_page(pool, p); + + kbase_mem_pool_add(pool, p); + } else if (next_pool && !kbase_mem_pool_is_full(next_pool)) { + /* Spill to next pool */ + kbase_mem_pool_spill(next_pool, p); + } else { + /* Free page */ + kbase_mem_pool_free_page(pool, p); + } +} + +int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, + struct tagged_addr *pages, bool partial_allowed) +{ + struct page *p; + size_t nr_from_pool; + size_t i = 0; + int err = -ENOMEM; + size_t nr_pages_internal; + + nr_pages_internal = nr_4k_pages / (1u << (pool->order)); + + if (nr_pages_internal * (1u << pool->order) != nr_4k_pages) + return -EINVAL; + + pool_dbg(pool, "alloc_pages(4k=%zu):\n", nr_4k_pages); + pool_dbg(pool, "alloc_pages(internal=%zu):\n", nr_pages_internal); + + /* Get pages from this pool */ + kbase_mem_pool_lock(pool); + nr_from_pool = min(nr_pages_internal, kbase_mem_pool_size(pool)); + while (nr_from_pool--) { + int j; + p = kbase_mem_pool_remove_locked(pool); + if (pool->order) { + pages[i++] = as_tagged_tag(page_to_phys(p), + HUGE_HEAD | HUGE_PAGE); + for (j = 1; j < (1u << pool->order); j++) + pages[i++] = as_tagged_tag(page_to_phys(p) + + PAGE_SIZE * j, + HUGE_PAGE); + } else { + pages[i++] = as_tagged(page_to_phys(p)); + } + } + kbase_mem_pool_unlock(pool); + + if (i != nr_4k_pages && pool->next_pool) { + /* Allocate via next pool */ + err = kbase_mem_pool_alloc_pages(pool->next_pool, + nr_4k_pages - i, pages + i, partial_allowed); + + if (err < 0) + goto err_rollback; + + i += err; + } else { + /* Get any remaining pages from kernel */ + while (i != nr_4k_pages) { + p = kbase_mem_alloc_page(pool); + if (!p) { + if (partial_allowed) + goto done; + else + goto err_rollback; + } + + if (pool->order) { + int j; + + pages[i++] = as_tagged_tag(page_to_phys(p), + HUGE_PAGE | + HUGE_HEAD); + for (j = 1; j < (1u << pool->order); j++) { + phys_addr_t phys; + + phys = page_to_phys(p) + PAGE_SIZE * j; + pages[i++] = as_tagged_tag(phys, + HUGE_PAGE); + } + } else { + pages[i++] = as_tagged(page_to_phys(p)); + } + } + } + +done: + pool_dbg(pool, "alloc_pages(%zu) done\n", i); + + return i; + +err_rollback: + kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED); + return err; +} + +static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, + size_t nr_pages, struct tagged_addr *pages, + bool zero, bool sync) +{ + struct page *p; + size_t nr_to_pool = 0; + LIST_HEAD(new_page_list); + size_t i; + + if (!nr_pages) + return; + + pool_dbg(pool, "add_array(%zu, zero=%d, sync=%d):\n", + nr_pages, zero, sync); + + /* Zero/sync pages first without holding the pool lock */ + for (i = 0; i < nr_pages; i++) { + if (unlikely(!as_phys_addr_t(pages[i]))) + continue; + + if (is_huge_head(pages[i]) || !is_huge(pages[i])) { + p = phys_to_page(as_phys_addr_t(pages[i])); + if (zero) + kbase_mem_pool_zero_page(pool, p); + else if (sync) + kbase_mem_pool_sync_page(pool, p); + + list_add(&p->lru, &new_page_list); + nr_to_pool++; + } + pages[i] = as_tagged(0); + } + + /* Add new page list to pool */ + kbase_mem_pool_add_list(pool, &new_page_list, nr_to_pool); + + pool_dbg(pool, "add_array(%zu) added %zu pages\n", + nr_pages, nr_to_pool); +} + +void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, + struct tagged_addr *pages, bool dirty, bool reclaimed) +{ + struct kbase_mem_pool *next_pool = pool->next_pool; + struct page *p; + size_t nr_to_pool; + LIST_HEAD(to_pool_list); + size_t i = 0; + + pool_dbg(pool, "free_pages(%zu):\n", nr_pages); + + if (!reclaimed) { + /* Add to this pool */ + nr_to_pool = kbase_mem_pool_capacity(pool); + nr_to_pool = min(nr_pages, nr_to_pool); + + kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty); + + i += nr_to_pool; + + if (i != nr_pages && next_pool) { + /* Spill to next pool (may overspill) */ + nr_to_pool = kbase_mem_pool_capacity(next_pool); + nr_to_pool = min(nr_pages - i, nr_to_pool); + + kbase_mem_pool_add_array(next_pool, nr_to_pool, + pages + i, true, dirty); + i += nr_to_pool; + } + } + + /* Free any remaining pages to kernel */ + for (; i < nr_pages; i++) { + if (unlikely(!as_phys_addr_t(pages[i]))) + continue; + + if (is_huge(pages[i]) && !is_huge_head(pages[i])) { + pages[i] = as_tagged(0); + continue; + } + + p = phys_to_page(as_phys_addr_t(pages[i])); + + if (reclaimed) + zone_page_state_add(-1, page_zone(p), + NR_SLAB_RECLAIMABLE); + + kbase_mem_pool_free_page(pool, p); + pages[i] = as_tagged(0); + } + + pool_dbg(pool, "free_pages(%zu) done\n", nr_pages); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.c new file mode 100755 index 000000000000..319cf2568aba --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.c @@ -0,0 +1,88 @@ +/* + * + * (C) COPYRIGHT 2014-2015, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include + +#include + +#ifdef CONFIG_DEBUG_FS + +static int kbase_mem_pool_debugfs_size_get(void *data, u64 *val) +{ + struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; + + *val = kbase_mem_pool_size(pool); + + return 0; +} + +static int kbase_mem_pool_debugfs_size_set(void *data, u64 val) +{ + struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; + + kbase_mem_pool_trim(pool, val); + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(kbase_mem_pool_debugfs_size_fops, + kbase_mem_pool_debugfs_size_get, + kbase_mem_pool_debugfs_size_set, + "%llu\n"); + +static int kbase_mem_pool_debugfs_max_size_get(void *data, u64 *val) +{ + struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; + + *val = kbase_mem_pool_max_size(pool); + + return 0; +} + +static int kbase_mem_pool_debugfs_max_size_set(void *data, u64 val) +{ + struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; + + kbase_mem_pool_set_max_size(pool, val); + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(kbase_mem_pool_debugfs_max_size_fops, + kbase_mem_pool_debugfs_max_size_get, + kbase_mem_pool_debugfs_max_size_set, + "%llu\n"); + +void kbase_mem_pool_debugfs_init(struct dentry *parent, + struct kbase_mem_pool *pool, + struct kbase_mem_pool *lp_pool) +{ + debugfs_create_file("mem_pool_size", S_IRUGO | S_IWUSR, parent, + pool, &kbase_mem_pool_debugfs_size_fops); + + debugfs_create_file("mem_pool_max_size", S_IRUGO | S_IWUSR, parent, + pool, &kbase_mem_pool_debugfs_max_size_fops); + + debugfs_create_file("lp_mem_pool_size", S_IRUGO | S_IWUSR, parent, + lp_pool, &kbase_mem_pool_debugfs_size_fops); + + debugfs_create_file("lp_mem_pool_max_size", S_IRUGO | S_IWUSR, parent, + lp_pool, &kbase_mem_pool_debugfs_max_size_fops); +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.h new file mode 100755 index 000000000000..496eaf3f1e1a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_pool_debugfs.h @@ -0,0 +1,40 @@ +/* + * + * (C) COPYRIGHT 2014-2015, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_MEM_POOL_DEBUGFS_H +#define _KBASE_MEM_POOL_DEBUGFS_H + +#include + +/** + * kbase_mem_pool_debugfs_init - add debugfs knobs for @pool + * @parent: Parent debugfs dentry + * @pool: Memory pool of small pages to control + * @lp_pool: Memory pool of large pages to control + * + * Adds four debugfs files under @parent: + * - mem_pool_size: get/set the current size of @pool + * - mem_pool_max_size: get/set the max size of @pool + * - lp_mem_pool_size: get/set the current size of @lp_pool + * - lp_mem_pool_max_size: get/set the max size of @lp_pool + */ +void kbase_mem_pool_debugfs_init(struct dentry *parent, + struct kbase_mem_pool *pool, + struct kbase_mem_pool *lp_pool); + +#endif /*_KBASE_MEM_POOL_DEBUGFS_H*/ + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.c new file mode 100755 index 000000000000..d58fd8d62fde --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.c @@ -0,0 +1,121 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include + +#ifdef CONFIG_DEBUG_FS + +/** Show callback for the @c mem_profile debugfs file. + * + * This function is called to get the contents of the @c mem_profile debugfs + * file. This is a report of current memory usage and distribution in userspace. + * + * @param sfile The debugfs entry + * @param data Data associated with the entry + * + * @return 0 if it successfully prints data in debugfs entry file, non-zero otherwise + */ +static int kbasep_mem_profile_seq_show(struct seq_file *sfile, void *data) +{ + struct kbase_context *kctx = sfile->private; + + mutex_lock(&kctx->mem_profile_lock); + + seq_write(sfile, kctx->mem_profile_data, kctx->mem_profile_size); + + seq_putc(sfile, '\n'); + + mutex_unlock(&kctx->mem_profile_lock); + + return 0; +} + +/* + * File operations related to debugfs entry for mem_profile + */ +static int kbasep_mem_profile_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_mem_profile_seq_show, in->i_private); +} + +static const struct file_operations kbasep_mem_profile_debugfs_fops = { + .open = kbasep_mem_profile_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, + size_t size) +{ + int err = 0; + + mutex_lock(&kctx->mem_profile_lock); + + dev_dbg(kctx->kbdev->dev, "initialised: %d", + kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); + + if (!kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { + if (!debugfs_create_file("mem_profile", S_IRUGO, + kctx->kctx_dentry, kctx, + &kbasep_mem_profile_debugfs_fops)) { + err = -EAGAIN; + } else { + kbase_ctx_flag_set(kctx, + KCTX_MEM_PROFILE_INITIALIZED); + } + } + + if (kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { + kfree(kctx->mem_profile_data); + kctx->mem_profile_data = data; + kctx->mem_profile_size = size; + } else { + kfree(data); + } + + dev_dbg(kctx->kbdev->dev, "returning: %d, initialised: %d", + err, kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); + + mutex_unlock(&kctx->mem_profile_lock); + + return err; +} + +void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx) +{ + mutex_lock(&kctx->mem_profile_lock); + + dev_dbg(kctx->kbdev->dev, "initialised: %d", + kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); + + kfree(kctx->mem_profile_data); + kctx->mem_profile_data = NULL; + kctx->mem_profile_size = 0; + + mutex_unlock(&kctx->mem_profile_lock); +} + +#else /* CONFIG_DEBUG_FS */ + +int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, + size_t size) +{ + kfree(data); + return 0; +} +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.h new file mode 100755 index 000000000000..a1dc2e0b165b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs.h @@ -0,0 +1,59 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mem_profile_debugfs.h + * Header file for mem profiles entries in debugfs + * + */ + +#ifndef _KBASE_MEM_PROFILE_DEBUGFS_H +#define _KBASE_MEM_PROFILE_DEBUGFS_H + +#include +#include + +/** + * @brief Remove entry from Mali memory profile debugfs + */ +void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx); + +/** + * @brief Insert @p data to the debugfs file so it can be read by userspace + * + * The function takes ownership of @p data and frees it later when new data + * is inserted. + * + * If the debugfs entry corresponding to the @p kctx doesn't exist, + * an attempt will be made to create it. + * + * @param kctx The context whose debugfs file @p data should be inserted to + * @param data A NULL-terminated string to be inserted to the debugfs file, + * without the trailing new line character + * @param size The length of the @p data string + * @return 0 if @p data inserted correctly + * -EAGAIN in case of error + * @post @ref mem_profile_initialized will be set to @c true + * the first time this function succeeds. + */ +int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, + size_t size); + +#endif /*_KBASE_MEM_PROFILE_DEBUGFS_H*/ + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs_buf_size.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs_buf_size.h new file mode 100755 index 000000000000..82f0702974c2 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mem_profile_debugfs_buf_size.h @@ -0,0 +1,33 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_mem_profile_debugfs_buf_size.h + * Header file for the size of the buffer to accumulate the histogram report text in + */ + +#ifndef _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ +#define _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ + +/** + * The size of the buffer to accumulate the histogram report text in + * @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT + */ +#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t) (64 + ((80 + (56 * 64)) * 15) + 56)) + +#endif /*_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/ + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu.c new file mode 100755 index 000000000000..c63269aed53c --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu.c @@ -0,0 +1,2138 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mmu.c + * Base kernel MMU management. + */ + +/* #define DEBUG 1 */ +#include +#include +#include +#include +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) +#include +#endif +#include +#include +#include + +#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) + +#include +#include +#include +#include +#include +#include + +#define KBASE_MMU_PAGE_ENTRIES 512 + +/** + * kbase_mmu_flush_invalidate() - Flush and invalidate the GPU caches. + * @kctx: The KBase context. + * @vpfn: The virtual page frame number to start the flush on. + * @nr: The number of pages to flush. + * @sync: Set if the operation should be synchronous or not. + * + * Issue a cache flush + invalidate to the GPU caches and invalidate the TLBs. + * + * If sync is not set then transactions still in flight when the flush is issued + * may use the old page tables and the data they write will not be written out + * to memory, this function returns after the flush has been issued but + * before all accesses which might effect the flushed region have completed. + * + * If sync is set then accesses in the flushed region will be drained + * before data is flush and invalidated through L1, L2 and into memory, + * after which point this function will return. + */ +static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, + u64 vpfn, size_t nr, bool sync); + +/** + * kbase_mmu_sync_pgd - sync page directory to memory + * @kbdev: Device pointer. + * @handle: Address of DMA region. + * @size: Size of the region to sync. + * + * This should be called after each page directory update. + */ + +static void kbase_mmu_sync_pgd(struct kbase_device *kbdev, + dma_addr_t handle, size_t size) +{ + /* If page table is not coherent then ensure the gpu can read + * the pages from memory + */ + if (kbdev->system_coherency != COHERENCY_ACE) + dma_sync_single_for_device(kbdev->dev, handle, size, + DMA_TO_DEVICE); +} + +/* + * Definitions: + * - PGD: Page Directory. + * - PTE: Page Table Entry. A 64bit value pointing to the next + * level of translation + * - ATE: Address Transation Entry. A 64bit value pointing to + * a 4kB physical page. + */ + +static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, + struct kbase_as *as, const char *reason_str); + + +static size_t make_multiple(size_t minimum, size_t multiple) +{ + size_t remainder = minimum % multiple; + + if (remainder == 0) + return minimum; + + return minimum + multiple - remainder; +} + +void page_fault_worker(struct work_struct *data) +{ + u64 fault_pfn; + u32 fault_status; + size_t new_pages; + size_t fault_rel_pfn; + struct kbase_as *faulting_as; + int as_no; + struct kbase_context *kctx; + struct kbase_device *kbdev; + struct kbase_va_region *region; + int err; + bool grown = false; + + faulting_as = container_of(data, struct kbase_as, work_pagefault); + fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT; + as_no = faulting_as->number; + + kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); + + /* Grab the context that was already refcounted in kbase_mmu_interrupt(). + * Therefore, it cannot be scheduled out of this AS until we explicitly release it + */ + kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); + if (WARN_ON(!kctx)) { + atomic_dec(&kbdev->faults_pending); + return; + } + + KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev); + + if (unlikely(faulting_as->protected_mode)) + { + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Protected mode fault"); + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + + goto fault_done; + } + + fault_status = faulting_as->fault_status; + switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) { + + case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT: + /* need to check against the region to handle this one */ + break; + + case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Permission failure"); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Translation table bus fault"); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG: + /* nothing to do, but we don't expect this fault currently */ + dev_warn(kbdev->dev, "Access flag unexpectedly set"); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Address size fault"); + else + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code"); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Memory attributes fault"); + else + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code"); + goto fault_done; + + default: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code"); + goto fault_done; + } + + /* so we have a translation fault, let's see if it is for growable + * memory */ + kbase_gpu_vm_lock(kctx); + + region = kbase_region_tracker_find_region_enclosing_address(kctx, + faulting_as->fault_addr); + if (!region || region->flags & KBASE_REG_FREE) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Memory is not mapped on the GPU"); + goto fault_done; + } + + if (region->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "DMA-BUF is not mapped on the GPU"); + goto fault_done; + } + + if ((region->flags & GROWABLE_FLAGS_REQUIRED) + != GROWABLE_FLAGS_REQUIRED) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Memory is not growable"); + goto fault_done; + } + + if ((region->flags & KBASE_REG_DONT_NEED)) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Don't need memory can't be grown"); + goto fault_done; + } + + /* find the size we need to grow it by */ + /* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address + * validating the fault_adress to be within a size_t from the start_pfn */ + fault_rel_pfn = fault_pfn - region->start_pfn; + + if (fault_rel_pfn < kbase_reg_current_backed_size(region)) { + dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring", + faulting_as->fault_addr, region->start_pfn, + region->start_pfn + + kbase_reg_current_backed_size(region)); + + mutex_lock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + /* [1] in case another page fault occurred while we were + * handling the (duplicate) page fault we need to ensure we + * don't loose the other page fault as result of us clearing + * the MMU IRQ. Therefore, after we clear the MMU IRQ we send + * an UNLOCK command that will retry any stalled memory + * transaction (which should cause the other page fault to be + * raised again). + */ + kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0, + AS_COMMAND_UNLOCK, 1); + + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + kbase_gpu_vm_unlock(kctx); + + goto fault_done; + } + + new_pages = make_multiple(fault_rel_pfn - + kbase_reg_current_backed_size(region) + 1, + region->extent); + + /* cap to max vsize */ + if (new_pages + kbase_reg_current_backed_size(region) > + region->nr_pages) + new_pages = region->nr_pages - + kbase_reg_current_backed_size(region); + + if (0 == new_pages) { + mutex_lock(&kbdev->mmu_hw_mutex); + + /* Duplicate of a fault we've already handled, nothing to do */ + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + /* See comment [1] about UNLOCK usage */ + kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0, + AS_COMMAND_UNLOCK, 1); + + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + kbase_gpu_vm_unlock(kctx); + goto fault_done; + } + + if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) { + if (region->gpu_alloc != region->cpu_alloc) { + if (kbase_alloc_phy_pages_helper( + region->cpu_alloc, new_pages) == 0) { + grown = true; + } else { + kbase_free_phy_pages_helper(region->gpu_alloc, + new_pages); + } + } else { + grown = true; + } + } + + + if (grown) { + u64 pfn_offset; + u32 op; + + /* alloc success */ + KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages); + + /* set up the new pages */ + pfn_offset = kbase_reg_current_backed_size(region) - new_pages; + /* + * Note: + * Issuing an MMU operation will unlock the MMU and cause the + * translation to be replayed. If the page insertion fails then + * rather then trying to continue the context should be killed + * so the no_flush version of insert_pages is used which allows + * us to unlock the MMU as we see fit. + */ + err = kbase_mmu_insert_pages_no_flush(kctx, + region->start_pfn + pfn_offset, + &kbase_get_gpu_phy_pages(region)[pfn_offset], + new_pages, region->flags); + if (err) { + kbase_free_phy_pages_helper(region->gpu_alloc, new_pages); + if (region->gpu_alloc != region->cpu_alloc) + kbase_free_phy_pages_helper(region->cpu_alloc, + new_pages); + kbase_gpu_vm_unlock(kctx); + /* The locked VA region will be unlocked and the cache invalidated in here */ + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Page table update failure"); + goto fault_done; + } +#if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) + kbase_trace_mali_page_fault_insert_pages(as_no, new_pages); +#endif + KBASE_TLSTREAM_AUX_PAGEFAULT(kctx->id, (u64)new_pages); + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + + /* flush L2 and unlock the VA (resumes the MMU) */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367)) + op = AS_COMMAND_FLUSH; + else + op = AS_COMMAND_FLUSH_PT; + + /* clear MMU interrupt - this needs to be done after updating + * the page tables but before issuing a FLUSH command. The + * FLUSH cmd has a side effect that it restarts stalled memory + * transactions in other address spaces which may cause + * another fault to occur. If we didn't clear the interrupt at + * this stage a new IRQ might not be raised when the GPU finds + * a MMU IRQ is already pending. + */ + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + + kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx, + faulting_as->fault_addr >> PAGE_SHIFT, + new_pages, + op, 1); + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + + /* reenable this in the mask */ + kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + kbase_gpu_vm_unlock(kctx); + } else { + /* failed to extend, handle as a normal PF */ + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Page allocation failure"); + } + +fault_done: + /* + * By this point, the fault was handled in some way, + * so release the ctx refcount + */ + kbasep_js_runpool_release_ctx(kbdev, kctx); + + atomic_dec(&kbdev->faults_pending); +} + +phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx) +{ + u64 *page; + int i; + struct page *p; + int new_page_count __maybe_unused; + + KBASE_DEBUG_ASSERT(NULL != kctx); + new_page_count = kbase_atomic_add_pages(1, &kctx->used_pages); + kbase_atomic_add_pages(1, &kctx->kbdev->memdev.used_pages); + + p = kbase_mem_pool_alloc(&kctx->mem_pool); + if (!p) + goto sub_pages; + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kctx->id, + (u64)new_page_count); + + page = kmap(p); + if (NULL == page) + goto alloc_free; + + kbase_process_page_usage_inc(kctx, 1); + + for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) + kctx->kbdev->mmu_mode->entry_invalidate(&page[i]); + + kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); + + kunmap(p); + return page_to_phys(p); + +alloc_free: + kbase_mem_pool_free(&kctx->mem_pool, p, false); +sub_pages: + kbase_atomic_sub_pages(1, &kctx->used_pages); + kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd); + +/* Given PGD PFN for level N, return PGD PFN for level N+1, allocating the + * new table from the pool if needed and possible + */ +static int mmu_get_next_pgd(struct kbase_context *kctx, + phys_addr_t *pgd, u64 vpfn, int level) +{ + u64 *page; + phys_addr_t target_pgd; + struct page *p; + + KBASE_DEBUG_ASSERT(*pgd); + KBASE_DEBUG_ASSERT(NULL != kctx); + + lockdep_assert_held(&kctx->mmu_lock); + + /* + * Architecture spec defines level-0 as being the top-most. + * This is a bit unfortunate here, but we keep the same convention. + */ + vpfn >>= (3 - level) * 9; + vpfn &= 0x1FF; + + p = pfn_to_page(PFN_DOWN(*pgd)); + page = kmap(p); + if (NULL == page) { + dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kmap failure\n"); + return -EINVAL; + } + + target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]); + + if (!target_pgd) { + target_pgd = kbase_mmu_alloc_pgd(kctx); + if (!target_pgd) { + dev_dbg(kctx->kbdev->dev, "mmu_get_next_pgd: kbase_mmu_alloc_pgd failure\n"); + kunmap(p); + return -ENOMEM; + } + + kctx->kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd); + + kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); + /* Rely on the caller to update the address space flags. */ + } + + kunmap(p); + *pgd = target_pgd; + + return 0; +} + +/* + * Returns the PGD for the specified level of translation + */ +static int mmu_get_pgd_at_level(struct kbase_context *kctx, + u64 vpfn, + unsigned int level, + phys_addr_t *out_pgd) +{ + phys_addr_t pgd; + int l; + + lockdep_assert_held(&kctx->mmu_lock); + pgd = kctx->pgd; + + for (l = MIDGARD_MMU_TOPLEVEL; l < level; l++) { + int err = mmu_get_next_pgd(kctx, &pgd, vpfn, l); + /* Handle failure condition */ + if (err) { + dev_dbg(kctx->kbdev->dev, + "%s: mmu_get_next_pgd failure at level %d\n", + __func__, l); + return err; + } + } + + *out_pgd = pgd; + + return 0; +} + +#define mmu_get_bottom_pgd(kctx, vpfn, out_pgd) \ + mmu_get_pgd_at_level((kctx), (vpfn), MIDGARD_MMU_BOTTOMLEVEL, (out_pgd)) + + +static void mmu_insert_pages_failure_recovery(struct kbase_context *kctx, + u64 from_vpfn, u64 to_vpfn) +{ + phys_addr_t pgd; + u64 vpfn = from_vpfn; + struct kbase_mmu_mode const *mmu_mode; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(0 != vpfn); + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + KBASE_DEBUG_ASSERT(from_vpfn <= to_vpfn); + + lockdep_assert_held(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); + + mmu_mode = kctx->kbdev->mmu_mode; + + while (vpfn < to_vpfn) { + unsigned int i; + unsigned int idx = vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - idx; + unsigned int pcount = 0; + unsigned int left = to_vpfn - vpfn; + unsigned int level; + u64 *page; + + if (count > left) + count = left; + + /* need to check if this is a 2MB page or a 4kB */ + pgd = kctx->pgd; + + for (level = MIDGARD_MMU_TOPLEVEL; + level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { + idx = (vpfn >> ((3 - level) * 9)) & 0x1FF; + page = kmap(phys_to_page(pgd)); + if (mmu_mode->ate_is_valid(page[idx], level)) + break; /* keep the mapping */ + kunmap(phys_to_page(pgd)); + pgd = mmu_mode->pte_to_phy_addr(page[idx]); + } + + switch (level) { + case MIDGARD_MMU_LEVEL(2): + /* remap to single entry to update */ + pcount = 1; + break; + case MIDGARD_MMU_BOTTOMLEVEL: + /* page count is the same as the logical count */ + pcount = count; + break; + default: + dev_warn(kctx->kbdev->dev, "%sNo support for ATEs at level %d\n", + __func__, level); + goto next; + } + + /* Invalidate the entries we added */ + for (i = 0; i < pcount; i++) + mmu_mode->entry_invalidate(&page[idx + i]); + + kbase_mmu_sync_pgd(kctx->kbdev, + kbase_dma_addr(phys_to_page(pgd)) + 8 * idx, + 8 * pcount); + kunmap(phys_to_page(pgd)); + +next: + vpfn += count; + } +} + +/* + * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn' + */ +int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr phys, size_t nr, + unsigned long flags) +{ + phys_addr_t pgd; + u64 *pgd_page; + /* In case the insert_single_page only partially completes we need to be + * able to recover */ + bool recover_required = false; + u64 recover_vpfn = vpfn; + size_t recover_count = 0; + size_t remain = nr; + int err; + struct kbase_mmu_mode const *mmu_mode; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(0 != vpfn); + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + + mmu_mode = kctx->kbdev->mmu_mode; + + /* Early out if there is nothing to do */ + if (nr == 0) + return 0; + + mutex_lock(&kctx->mmu_lock); + + while (remain) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; + struct page *p; + + if (count > remain) + count = remain; + + /* + * Repeatedly calling mmu_get_bottom_pte() is clearly + * suboptimal. We don't have to re-parse the whole tree + * each time (just cache the l0-l2 sequence). + * On the other hand, it's only a gain when we map more than + * 256 pages at once (on average). Do we really care? + */ + do { + err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); + if (err != -ENOMEM) + break; + /* Fill the memory pool with enough pages for + * the page walk to succeed + */ + mutex_unlock(&kctx->mmu_lock); + err = kbase_mem_pool_grow(&kctx->mem_pool, + MIDGARD_MMU_BOTTOMLEVEL); + mutex_lock(&kctx->mmu_lock); + } while (!err); + if (err) { + dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n"); + if (recover_required) { + /* Invalidate the pages we have partially + * completed */ + mmu_insert_pages_failure_recovery(kctx, + recover_vpfn, + recover_vpfn + + recover_count + ); + } + goto fail_unlock; + } + + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap(p); + if (!pgd_page) { + dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n"); + if (recover_required) { + /* Invalidate the pages we have partially + * completed */ + mmu_insert_pages_failure_recovery(kctx, + recover_vpfn, + recover_vpfn + + recover_count + ); + } + err = -ENOMEM; + goto fail_unlock; + } + + for (i = 0; i < count; i++) { + unsigned int ofs = index + i; + + /* Fail if the current page is a valid ATE entry */ + KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); + + mmu_mode->entry_set_ate(&pgd_page[ofs], + phys, flags, + MIDGARD_MMU_BOTTOMLEVEL); + } + + vpfn += count; + remain -= count; + + kbase_mmu_sync_pgd(kctx->kbdev, + kbase_dma_addr(p) + (index * sizeof(u64)), + count * sizeof(u64)); + + kunmap(p); + /* We have started modifying the page table. + * If further pages need inserting and fail we need to undo what + * has already taken place */ + recover_required = true; + recover_count += count; + } + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); + return 0; + +fail_unlock: + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); + return err; +} + +static inline void cleanup_empty_pte(struct kbase_context *kctx, u64 *pte) +{ + phys_addr_t tmp_pgd; + struct page *tmp_p; + + tmp_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(*pte); + tmp_p = phys_to_page(tmp_pgd); + kbase_mem_pool_free(&kctx->mem_pool, tmp_p, false); + kbase_process_page_usage_dec(kctx, 1); + kbase_atomic_sub_pages(1, &kctx->used_pages); + kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); +} + +int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, + const u64 start_vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags) +{ + phys_addr_t pgd; + u64 *pgd_page; + u64 insert_vpfn = start_vpfn; + size_t remain = nr; + int err; + struct kbase_mmu_mode const *mmu_mode; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(start_vpfn); + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(start_vpfn <= (U64_MAX / PAGE_SIZE)); + + mmu_mode = kctx->kbdev->mmu_mode; + + /* Early out if there is nothing to do */ + if (nr == 0) + return 0; + + mutex_lock(&kctx->mmu_lock); + + while (remain) { + unsigned int i; + unsigned int vindex = insert_vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - vindex; + struct page *p; + unsigned int cur_level; + + if (count > remain) + count = remain; + + if (!vindex && is_huge_head(*phys)) + cur_level = MIDGARD_MMU_LEVEL(2); + else + cur_level = MIDGARD_MMU_BOTTOMLEVEL; + + /* + * Repeatedly calling mmu_get_pgd_at_level() is clearly + * suboptimal. We don't have to re-parse the whole tree + * each time (just cache the l0-l2 sequence). + * On the other hand, it's only a gain when we map more than + * 256 pages at once (on average). Do we really care? + */ + do { + err = mmu_get_pgd_at_level(kctx, insert_vpfn, cur_level, + &pgd); + if (err != -ENOMEM) + break; + /* Fill the memory pool with enough pages for + * the page walk to succeed + */ + mutex_unlock(&kctx->mmu_lock); + err = kbase_mem_pool_grow(&kctx->mem_pool, + cur_level); + mutex_lock(&kctx->mmu_lock); + } while (!err); + + if (err) { + dev_warn(kctx->kbdev->dev, + "%s: mmu_get_bottom_pgd failure\n", __func__); + if (insert_vpfn != start_vpfn) { + /* Invalidate the pages we have partially + * completed */ + mmu_insert_pages_failure_recovery(kctx, + start_vpfn, + insert_vpfn); + } + goto fail_unlock; + } + + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap(p); + if (!pgd_page) { + dev_warn(kctx->kbdev->dev, "%s: kmap failure\n", + __func__); + if (insert_vpfn != start_vpfn) { + /* Invalidate the pages we have partially + * completed */ + mmu_insert_pages_failure_recovery(kctx, + start_vpfn, + insert_vpfn); + } + err = -ENOMEM; + goto fail_unlock; + } + + if (cur_level == MIDGARD_MMU_LEVEL(2)) { + unsigned int level_index = (insert_vpfn >> 9) & 0x1FF; + u64 *target = &pgd_page[level_index]; + + if (mmu_mode->pte_is_valid(*target, cur_level)) + cleanup_empty_pte(kctx, target); + mmu_mode->entry_set_ate(target, *phys, flags, + cur_level); + } else { + for (i = 0; i < count; i++) { + unsigned int ofs = vindex + i; + u64 *target = &pgd_page[ofs]; + + /* Fail if the current page is a valid ATE entry + */ + KBASE_DEBUG_ASSERT(0 == (*target & 1UL)); + + kctx->kbdev->mmu_mode->entry_set_ate(target, + phys[i], flags, cur_level); + } + } + + phys += count; + insert_vpfn += count; + remain -= count; + + kbase_mmu_sync_pgd(kctx->kbdev, + kbase_dma_addr(p) + (vindex * sizeof(u64)), + count * sizeof(u64)); + + kunmap(p); + } + + mutex_unlock(&kctx->mmu_lock); + return 0; + +fail_unlock: + mutex_unlock(&kctx->mmu_lock); + return err; +} + +/* + * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn' + */ +int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags) +{ + int err; + + err = kbase_mmu_insert_pages_no_flush(kctx, vpfn, phys, nr, flags); + kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages); + +/** + * kbase_mmu_flush_invalidate_noretain() - Flush and invalidate the GPU caches + * without retaining the kbase context. + * @kctx: The KBase context. + * @vpfn: The virtual page frame number to start the flush on. + * @nr: The number of pages to flush. + * @sync: Set if the operation should be synchronous or not. + * + * As per kbase_mmu_flush_invalidate but doesn't retain the kctx or do any + * other locking. + */ +static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx, + u64 vpfn, size_t nr, bool sync) +{ + struct kbase_device *kbdev = kctx->kbdev; + int err; + u32 op; + + /* Early out if there is nothing to do */ + if (nr == 0) + return; + + if (sync) + op = AS_COMMAND_FLUSH_MEM; + else + op = AS_COMMAND_FLUSH_PT; + + err = kbase_mmu_hw_do_operation(kbdev, + &kbdev->as[kctx->as_nr], + kctx, vpfn, nr, op, 0); +#if KBASE_GPU_RESET_EN + if (err) { + /* Flush failed to complete, assume the + * GPU has hung and perform a reset to + * recover */ + dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n"); + + if (kbase_prepare_to_reset_gpu_locked(kbdev)) + kbase_reset_gpu_locked(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + +#ifndef CONFIG_MALI_BIFROST_NO_MALI + /* + * As this function could be called in interrupt context the sync + * request can't block. Instead log the request and the next flush + * request will pick it up. + */ + if ((!err) && sync && + kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367)) + atomic_set(&kctx->drain_pending, 1); +#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ +} + +static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, + u64 vpfn, size_t nr, bool sync) +{ + struct kbase_device *kbdev; + bool ctx_is_in_runpool; +#ifndef CONFIG_MALI_BIFROST_NO_MALI + bool drain_pending = false; + + if (atomic_xchg(&kctx->drain_pending, 0)) + drain_pending = true; +#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ + + /* Early out if there is nothing to do */ + if (nr == 0) + return; + + kbdev = kctx->kbdev; + mutex_lock(&kbdev->js_data.queue_mutex); + ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx); + mutex_unlock(&kbdev->js_data.queue_mutex); + + if (ctx_is_in_runpool) { + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + + if (!kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + int err; + u32 op; + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + + if (sync) + op = AS_COMMAND_FLUSH_MEM; + else + op = AS_COMMAND_FLUSH_PT; + + err = kbase_mmu_hw_do_operation(kbdev, + &kbdev->as[kctx->as_nr], + kctx, vpfn, nr, op, 0); + +#if KBASE_GPU_RESET_EN + if (err) { + /* Flush failed to complete, assume the + * GPU has hung and perform a reset to + * recover */ + dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + +#ifndef CONFIG_MALI_BIFROST_NO_MALI + /* + * The transaction lock must be dropped before here + * as kbase_wait_write_flush could take it if + * the GPU was powered down (static analysis doesn't + * know this can't happen). + */ + drain_pending |= (!err) && sync && + kbase_hw_has_issue(kctx->kbdev, + BASE_HW_ISSUE_6367); + if (drain_pending) { + /* Wait for GPU to flush write buffer */ + kbase_wait_write_flush(kctx); + } +#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ + + kbase_pm_context_idle(kbdev); + } + kbasep_js_runpool_release_ctx(kbdev, kctx); + } +} + +void kbase_mmu_update(struct kbase_context *kctx) +{ + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); + /* ASSERT that the context has a valid as_nr, which is only the case + * when it's scheduled in. + * + * as_nr won't change because the caller has the hwaccess_lock */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + + kctx->kbdev->mmu_mode->update(kctx); +} +KBASE_EXPORT_TEST_API(kbase_mmu_update); + +void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&kbdev->mmu_hw_mutex); + + kbdev->mmu_mode->disable_as(kbdev, as_nr); +} + +void kbase_mmu_disable(struct kbase_context *kctx) +{ + /* ASSERT that the context has a valid as_nr, which is only the case + * when it's scheduled in. + * + * as_nr won't change because the caller has the hwaccess_lock */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + /* + * The address space is being disabled, drain all knowledge of it out + * from the caches as pages and page tables might be freed after this. + * + * The job scheduler code will already be holding the locks and context + * so just do the flush. + */ + kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true); + + kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr); +} +KBASE_EXPORT_TEST_API(kbase_mmu_disable); + +/* + * We actually only discard the ATE, and not the page table + * pages. There is a potential DoS here, as we'll leak memory by + * having PTEs that are potentially unused. Will require physical + * page accounting, so MMU pages are part of the process allocation. + * + * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is + * currently scheduled into the runpool, and so potentially uses a lot of locks. + * These locks must be taken in the correct order with respect to others + * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more + * information. + */ +int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr) +{ + phys_addr_t pgd; + size_t requested_nr = nr; + struct kbase_mmu_mode const *mmu_mode; + int err = -EFAULT; + + KBASE_DEBUG_ASSERT(NULL != kctx); + beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr); + + if (0 == nr) { + /* early out if nothing to do */ + return 0; + } + + mutex_lock(&kctx->mmu_lock); + + mmu_mode = kctx->kbdev->mmu_mode; + + while (nr) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; + unsigned int pcount; + unsigned int level; + u64 *page; + + if (count > nr) + count = nr; + + /* need to check if this is a 2MB or a 4kB page */ + pgd = kctx->pgd; + + for (level = MIDGARD_MMU_TOPLEVEL; + level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { + phys_addr_t next_pgd; + + index = (vpfn >> ((3 - level) * 9)) & 0x1FF; + page = kmap(phys_to_page(pgd)); + if (mmu_mode->ate_is_valid(page[index], level)) + break; /* keep the mapping */ + else if (!mmu_mode->pte_is_valid(page[index], level)) { + /* nothing here, advance */ + switch (level) { + case MIDGARD_MMU_LEVEL(0): + count = 134217728; + break; + case MIDGARD_MMU_LEVEL(1): + count = 262144; + break; + case MIDGARD_MMU_LEVEL(2): + count = 512; + break; + case MIDGARD_MMU_LEVEL(3): + count = 1; + break; + } + if (count > nr) + count = nr; + goto next; + } + next_pgd = mmu_mode->pte_to_phy_addr(page[index]); + kunmap(phys_to_page(pgd)); + pgd = next_pgd; + } + + switch (level) { + case MIDGARD_MMU_LEVEL(0): + case MIDGARD_MMU_LEVEL(1): + dev_warn(kctx->kbdev->dev, + "%s: No support for ATEs at level %d\n", + __func__, level); + kunmap(phys_to_page(pgd)); + goto out; + case MIDGARD_MMU_LEVEL(2): + /* can only teardown if count >= 512 */ + if (count >= 512) { + pcount = 1; + } else { + dev_warn(kctx->kbdev->dev, + "%s: limiting teardown as it tries to do a partial 2MB teardown, need 512, but have %d to tear down\n", + __func__, count); + pcount = 0; + } + break; + case MIDGARD_MMU_BOTTOMLEVEL: + /* page count is the same as the logical count */ + pcount = count; + break; + default: + dev_err(kctx->kbdev->dev, + "%s: found non-mapped memory, early out\n", + __func__); + vpfn += count; + nr -= count; + continue; + } + + /* Invalidate the entries we added */ + for (i = 0; i < pcount; i++) + mmu_mode->entry_invalidate(&page[index + i]); + + kbase_mmu_sync_pgd(kctx->kbdev, + kbase_dma_addr(phys_to_page(pgd)) + + 8 * index, 8*pcount); + +next: + kunmap(phys_to_page(pgd)); + vpfn += count; + nr -= count; + } + err = 0; +out: + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages); + +/** + * Update the entries for specified number of pages pointed to by 'phys' at GPU PFN 'vpfn'. + * This call is being triggered as a response to the changes of the mem attributes + * + * @pre : The caller is responsible for validating the memory attributes + * + * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is + * currently scheduled into the runpool, and so potentially uses a lot of locks. + * These locks must be taken in the correct order with respect to others + * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more + * information. + */ +int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, + struct tagged_addr *phys, size_t nr, + unsigned long flags) +{ + phys_addr_t pgd; + u64 *pgd_page; + size_t requested_nr = nr; + struct kbase_mmu_mode const *mmu_mode; + int err; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(0 != vpfn); + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + + /* Early out if there is nothing to do */ + if (nr == 0) + return 0; + + mutex_lock(&kctx->mmu_lock); + + mmu_mode = kctx->kbdev->mmu_mode; + + dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags on GPU PFN 0x%llx from phys %p, %zu pages", + vpfn, phys, nr); + + while (nr) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + size_t count = KBASE_MMU_PAGE_ENTRIES - index; + struct page *p; + + if (count > nr) + count = nr; + + do { + err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); + if (err != -ENOMEM) + break; + /* Fill the memory pool with enough pages for + * the page walk to succeed + */ + mutex_unlock(&kctx->mmu_lock); + err = kbase_mem_pool_grow(&kctx->mem_pool, + MIDGARD_MMU_BOTTOMLEVEL); + mutex_lock(&kctx->mmu_lock); + } while (!err); + if (err) { + dev_warn(kctx->kbdev->dev, + "mmu_get_bottom_pgd failure\n"); + goto fail_unlock; + } + + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap(p); + if (!pgd_page) { + dev_warn(kctx->kbdev->dev, "kmap failure\n"); + err = -ENOMEM; + goto fail_unlock; + } + + for (i = 0; i < count; i++) + mmu_mode->entry_set_ate(&pgd_page[index + i], phys[i], + flags, MIDGARD_MMU_BOTTOMLEVEL); + + phys += count; + vpfn += count; + nr -= count; + + kbase_mmu_sync_pgd(kctx->kbdev, + kbase_dma_addr(p) + (index * sizeof(u64)), + count * sizeof(u64)); + + kunmap(pfn_to_page(PFN_DOWN(pgd))); + } + + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); + return 0; + +fail_unlock: + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); + return err; +} + +static void mmu_teardown_level(struct kbase_context *kctx, phys_addr_t pgd, + int level, u64 *pgd_page_buffer) +{ + phys_addr_t target_pgd; + struct page *p; + u64 *pgd_page; + int i; + struct kbase_mmu_mode const *mmu_mode; + + KBASE_DEBUG_ASSERT(NULL != kctx); + lockdep_assert_held(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); + + pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); + /* kmap_atomic should NEVER fail. */ + KBASE_DEBUG_ASSERT(NULL != pgd_page); + /* Copy the page to our preallocated buffer so that we can minimize + * kmap_atomic usage */ + memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE); + kunmap_atomic(pgd_page); + pgd_page = pgd_page_buffer; + + mmu_mode = kctx->kbdev->mmu_mode; + + for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { + target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]); + + if (target_pgd) { + if (mmu_mode->pte_is_valid(pgd_page[i], level)) { + mmu_teardown_level(kctx, + target_pgd, + level + 1, + pgd_page_buffer + + (PAGE_SIZE / sizeof(u64))); + } + } + } + + p = pfn_to_page(PFN_DOWN(pgd)); + kbase_mem_pool_free(&kctx->mem_pool, p, true); + kbase_process_page_usage_dec(kctx, 1); + kbase_atomic_sub_pages(1, &kctx->used_pages); + kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); +} + +int kbase_mmu_init(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages); + + mutex_init(&kctx->mmu_lock); + + /* Preallocate MMU depth of four pages for mmu_teardown_level to use */ + kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); + + if (NULL == kctx->mmu_teardown_pages) + return -ENOMEM; + + return 0; +} + +void kbase_mmu_term(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); + + kfree(kctx->mmu_teardown_pages); + kctx->mmu_teardown_pages = NULL; +} + +void kbase_mmu_free_pgd(struct kbase_context *kctx) +{ + int new_page_count = 0; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); + + mutex_lock(&kctx->mmu_lock); + mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, + kctx->mmu_teardown_pages); + mutex_unlock(&kctx->mmu_lock); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + kctx->id, + (u64)new_page_count); +} + +KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd); + +static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, int level, char ** const buffer, size_t *size_left) +{ + phys_addr_t target_pgd; + u64 *pgd_page; + int i; + size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64); + size_t dump_size; + struct kbase_mmu_mode const *mmu_mode; + + KBASE_DEBUG_ASSERT(NULL != kctx); + lockdep_assert_held(&kctx->mmu_lock); + + mmu_mode = kctx->kbdev->mmu_mode; + + pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd))); + if (!pgd_page) { + dev_warn(kctx->kbdev->dev, "kbasep_mmu_dump_level: kmap failure\n"); + return 0; + } + + if (*size_left >= size) { + /* A modified physical address that contains the page table level */ + u64 m_pgd = pgd | level; + + /* Put the modified physical address in the output buffer */ + memcpy(*buffer, &m_pgd, sizeof(m_pgd)); + *buffer += sizeof(m_pgd); + + /* Followed by the page table itself */ + memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES); + *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES; + + *size_left -= size; + } + + if (level < MIDGARD_MMU_BOTTOMLEVEL) { + for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { + if (mmu_mode->pte_is_valid(pgd_page[i], level)) { + target_pgd = mmu_mode->pte_to_phy_addr( + pgd_page[i]); + + dump_size = kbasep_mmu_dump_level(kctx, + target_pgd, level + 1, + buffer, size_left); + if (!dump_size) { + kunmap(pfn_to_page(PFN_DOWN(pgd))); + return 0; + } + size += dump_size; + } + } + } + + kunmap(pfn_to_page(PFN_DOWN(pgd))); + + return size; +} + +void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) +{ + void *kaddr; + size_t size_left; + + KBASE_DEBUG_ASSERT(kctx); + + if (0 == nr_pages) { + /* can't dump in a 0 sized buffer, early out */ + return NULL; + } + + size_left = nr_pages * PAGE_SIZE; + + KBASE_DEBUG_ASSERT(0 != size_left); + kaddr = vmalloc_user(size_left); + + mutex_lock(&kctx->mmu_lock); + + if (kaddr) { + u64 end_marker = 0xFFULL; + char *buffer; + char *mmu_dump_buffer; + u64 config[3]; + size_t dump_size, size = 0; + + buffer = (char *)kaddr; + mmu_dump_buffer = buffer; + + if (kctx->api_version >= KBASE_API_VERSION(8, 4)) { + struct kbase_mmu_setup as_setup; + + kctx->kbdev->mmu_mode->get_as_setup(kctx, &as_setup); + config[0] = as_setup.transtab; + config[1] = as_setup.memattr; + config[2] = as_setup.transcfg; + memcpy(buffer, &config, sizeof(config)); + mmu_dump_buffer += sizeof(config); + size_left -= sizeof(config); + size += sizeof(config); + } + + dump_size = kbasep_mmu_dump_level(kctx, + kctx->pgd, + MIDGARD_MMU_TOPLEVEL, + &mmu_dump_buffer, + &size_left); + + if (!dump_size) + goto fail_free; + + size += dump_size; + + /* Add on the size for the end marker */ + size += sizeof(u64); + + if (size > (nr_pages * PAGE_SIZE)) { + /* The buffer isn't big enough - free the memory and return failure */ + goto fail_free; + } + + /* Add the end marker */ + memcpy(mmu_dump_buffer, &end_marker, sizeof(u64)); + } + + mutex_unlock(&kctx->mmu_lock); + return kaddr; + +fail_free: + vfree(kaddr); + mutex_unlock(&kctx->mmu_lock); + return NULL; +} +KBASE_EXPORT_TEST_API(kbase_mmu_dump); + +void bus_fault_worker(struct work_struct *data) +{ + struct kbase_as *faulting_as; + int as_no; + struct kbase_context *kctx; + struct kbase_device *kbdev; +#if KBASE_GPU_RESET_EN + bool reset_status = false; +#endif /* KBASE_GPU_RESET_EN */ + + faulting_as = container_of(data, struct kbase_as, work_busfault); + + as_no = faulting_as->number; + + kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); + + /* Grab the context that was already refcounted in kbase_mmu_interrupt(). + * Therefore, it cannot be scheduled out of this AS until we explicitly release it + */ + kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); + if (WARN_ON(!kctx)) { + atomic_dec(&kbdev->faults_pending); + return; + } + + if (unlikely(faulting_as->protected_mode)) + { + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Permission failure"); + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + kbasep_js_runpool_release_ctx(kbdev, kctx); + atomic_dec(&kbdev->faults_pending); + return; + + } + +#if KBASE_GPU_RESET_EN + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { + /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. + * We start the reset before switching to UNMAPPED to ensure that unrelated jobs + * are evicted from the GPU before the switch. + */ + dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); + reset_status = kbase_prepare_to_reset_gpu(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */ + if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + unsigned long flags; + + /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + + /* Set the MMU into unmapped mode */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_disable(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + + kbase_pm_context_idle(kbdev); + } + +#if KBASE_GPU_RESET_EN + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) + kbase_reset_gpu(kbdev); +#endif /* KBASE_GPU_RESET_EN */ + + kbasep_js_runpool_release_ctx(kbdev, kctx); + + atomic_dec(&kbdev->faults_pending); +} + +const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) +{ + const char *e; + + switch (exception_code) { + /* Non-Fault Status code */ + case 0x00: + e = "NOT_STARTED/IDLE/OK"; + break; + case 0x01: + e = "DONE"; + break; + case 0x02: + e = "INTERRUPTED"; + break; + case 0x03: + e = "STOPPED"; + break; + case 0x04: + e = "TERMINATED"; + break; + case 0x08: + e = "ACTIVE"; + break; + /* Job exceptions */ + case 0x40: + e = "JOB_CONFIG_FAULT"; + break; + case 0x41: + e = "JOB_POWER_FAULT"; + break; + case 0x42: + e = "JOB_READ_FAULT"; + break; + case 0x43: + e = "JOB_WRITE_FAULT"; + break; + case 0x44: + e = "JOB_AFFINITY_FAULT"; + break; + case 0x48: + e = "JOB_BUS_FAULT"; + break; + case 0x50: + e = "INSTR_INVALID_PC"; + break; + case 0x51: + e = "INSTR_INVALID_ENC"; + break; + case 0x52: + e = "INSTR_TYPE_MISMATCH"; + break; + case 0x53: + e = "INSTR_OPERAND_FAULT"; + break; + case 0x54: + e = "INSTR_TLS_FAULT"; + break; + case 0x55: + e = "INSTR_BARRIER_FAULT"; + break; + case 0x56: + e = "INSTR_ALIGN_FAULT"; + break; + case 0x58: + e = "DATA_INVALID_FAULT"; + break; + case 0x59: + e = "TILE_RANGE_FAULT"; + break; + case 0x5A: + e = "ADDR_RANGE_FAULT"; + break; + case 0x60: + e = "OUT_OF_MEMORY"; + break; + /* GPU exceptions */ + case 0x80: + e = "DELAYED_BUS_FAULT"; + break; + case 0x88: + e = "SHAREABILITY_FAULT"; + break; + /* MMU exceptions */ + case 0xC0: + case 0xC1: + case 0xC2: + case 0xC3: + case 0xC4: + case 0xC5: + case 0xC6: + case 0xC7: + e = "TRANSLATION_FAULT"; + break; + case 0xC8: + e = "PERMISSION_FAULT"; + break; + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "PERMISSION_FAULT"; + else + e = "UNKNOWN"; + break; + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD5: + case 0xD6: + case 0xD7: + e = "TRANSTAB_BUS_FAULT"; + break; + case 0xD8: + e = "ACCESS_FLAG"; + break; + case 0xD9: + case 0xDA: + case 0xDB: + case 0xDC: + case 0xDD: + case 0xDE: + case 0xDF: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "ACCESS_FLAG"; + else + e = "UNKNOWN"; + break; + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "ADDRESS_SIZE_FAULT"; + else + e = "UNKNOWN"; + break; + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "MEMORY_ATTRIBUTES_FAULT"; + else + e = "UNKNOWN"; + break; + default: + e = "UNKNOWN"; + break; + }; + + return e; +} + +static const char *access_type_name(struct kbase_device *kbdev, + u32 fault_status) +{ + switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) { + case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + return "ATOMIC"; + else + return "UNKNOWN"; + case AS_FAULTSTATUS_ACCESS_TYPE_READ: + return "READ"; + case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: + return "WRITE"; + case AS_FAULTSTATUS_ACCESS_TYPE_EX: + return "EXECUTE"; + default: + WARN_ON(1); + return NULL; + } +} + +/** + * The caller must ensure it's retained the ctx to prevent it from being scheduled out whilst it's being worked on. + */ +static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, + struct kbase_as *as, const char *reason_str) +{ + unsigned long flags; + int exception_type; + int access_type; + int source_id; + int as_no; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + +#if KBASE_GPU_RESET_EN + bool reset_status = false; +#endif + + as_no = as->number; + kbdev = kctx->kbdev; + js_devdata = &kbdev->js_data; + + /* ASSERT that the context won't leave the runpool */ + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); + + /* decode the fault status */ + exception_type = as->fault_status & 0xFF; + access_type = (as->fault_status >> 8) & 0x3; + source_id = (as->fault_status >> 16); + + /* terminal fault, print info about the fault */ + dev_err(kbdev->dev, + "Unhandled Page fault in AS%d at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "decoded fault status: %s\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X\n" + "pid: %d\n", + as_no, as->fault_addr, + reason_str, + as->fault_status, + (as->fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"), + exception_type, kbase_exception_name(kbdev, exception_type), + access_type, access_type_name(kbdev, as->fault_status), + source_id, + kctx->pid); + + /* hardware counters dump fault handling */ + if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && + (kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_DUMPING)) { + unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; + + if ((as->fault_addr >= kbdev->hwcnt.addr) && + (as->fault_addr < (kbdev->hwcnt.addr + + (num_core_groups * 2048)))) + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; + } + + /* Stop the kctx from submitting more jobs and cause it to be scheduled + * out/rescheduled - this will occur on releasing the context's refcount */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbasep_js_clear_submit_allowed(js_devdata, kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Kill any running jobs from the context. Submit is disallowed, so no more jobs from this + * context can appear in the job slots from this point on */ + kbase_backend_jm_kill_jobs_from_kctx(kctx); + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); +#if KBASE_GPU_RESET_EN + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { + /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. + * We start the reset before switching to UNMAPPED to ensure that unrelated jobs + * are evicted from the GPU before the switch. + */ + dev_err(kbdev->dev, "Unhandled page fault. For this GPU version we now soft-reset the GPU as part of page fault recovery."); + reset_status = kbase_prepare_to_reset_gpu(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_disable(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + /* Clear down the fault */ + kbase_mmu_hw_clear_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + +#if KBASE_GPU_RESET_EN + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) + kbase_reset_gpu(kbdev); +#endif /* KBASE_GPU_RESET_EN */ +} + +void kbasep_as_do_poke(struct work_struct *work) +{ + struct kbase_as *as; + struct kbase_device *kbdev; + struct kbase_context *kctx; + unsigned long flags; + + KBASE_DEBUG_ASSERT(work); + as = container_of(work, struct kbase_as, poke_work); + kbdev = container_of(as, struct kbase_device, as[as->number]); + KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); + + /* GPU power will already be active by virtue of the caller holding a JS + * reference on the address space, and will not release it until this worker + * has finished */ + + /* Further to the comment above, we know that while this function is running + * the AS will not be released as before the atom is released this workqueue + * is flushed (in kbase_as_poking_timer_release_atom) + */ + kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as->number); + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + /* Force a uTLB invalidate */ + kbase_mmu_hw_do_operation(kbdev, as, kctx, 0, 0, + AS_COMMAND_UNLOCK, 0); + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (as->poke_refcount && + !(as->poke_state & KBASE_AS_POKE_STATE_KILLING_POKE)) { + /* Only queue up the timer if we need it, and we're not trying to kill it */ + hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer) +{ + struct kbase_as *as; + int queue_work_ret; + + KBASE_DEBUG_ASSERT(NULL != timer); + as = container_of(timer, struct kbase_as, poke_timer); + KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); + + queue_work_ret = queue_work(as->poke_wq, &as->poke_work); + KBASE_DEBUG_ASSERT(queue_work_ret); + return HRTIMER_NORESTART; +} + +/** + * Retain the poking timer on an atom's context (if the atom hasn't already + * done so), and start the timer (if it's not already started). + * + * This must only be called on a context that's scheduled in, and an atom + * that's running on the GPU. + * + * The caller must hold hwaccess_lock + * + * This can be called safely from atomic context + */ +void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + struct kbase_as *as; + + KBASE_DEBUG_ASSERT(kbdev); + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (katom->poking) + return; + + katom->poking = 1; + + /* It's safe to work on the as/as_nr without an explicit reference, + * because the caller holds the hwaccess_lock, and the atom itself + * was also running and had already taken a reference */ + as = &kbdev->as[kctx->as_nr]; + + if (++(as->poke_refcount) == 1) { + /* First refcount for poke needed: check if not already in flight */ + if (!as->poke_state) { + /* need to start poking */ + as->poke_state |= KBASE_AS_POKE_STATE_IN_FLIGHT; + queue_work(as->poke_wq, &as->poke_work); + } + } +} + +/** + * If an atom holds a poking timer, release it and wait for it to finish + * + * This must only be called on a context that's scheduled in, and an atom + * that still has a JS reference on the context + * + * This must \b not be called from atomic context, since it can sleep. + */ +void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + struct kbase_as *as; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev); + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + + if (!katom->poking) + return; + + as = &kbdev->as[kctx->as_nr]; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + KBASE_DEBUG_ASSERT(as->poke_refcount > 0); + KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); + + if (--(as->poke_refcount) == 0) { + as->poke_state |= KBASE_AS_POKE_STATE_KILLING_POKE; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + hrtimer_cancel(&as->poke_timer); + flush_workqueue(as->poke_wq); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Re-check whether it's still needed */ + if (as->poke_refcount) { + int queue_work_ret; + /* Poking still needed: + * - Another retain will not be starting the timer or queueing work, + * because it's still marked as in-flight + * - The hrtimer has finished, and has not started a new timer or + * queued work because it's been marked as killing + * + * So whatever happens now, just queue the work again */ + as->poke_state &= ~((kbase_as_poke_state)KBASE_AS_POKE_STATE_KILLING_POKE); + queue_work_ret = queue_work(as->poke_wq, &as->poke_work); + KBASE_DEBUG_ASSERT(queue_work_ret); + } else { + /* It isn't - so mark it as not in flight, and not killing */ + as->poke_state = 0u; + + /* The poke associated with the atom has now finished. If this is + * also the last atom on the context, then we can guarentee no more + * pokes (and thus no more poking register accesses) will occur on + * the context until new atoms are run */ + } + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + katom->poking = 0; +} + +void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!kctx) { + dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Suprious IRQ or SW Design Error?\n", + kbase_as_has_bus_fault(as) ? "Bus error" : "Page fault", + as->number, as->fault_addr); + + /* Since no ctx was found, the MMU must be disabled. */ + WARN_ON(as->current_setup.transtab); + + if (kbase_as_has_bus_fault(as)) { + kbase_mmu_hw_clear_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + } else if (kbase_as_has_page_fault(as)) { + kbase_mmu_hw_clear_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + } + +#if KBASE_GPU_RESET_EN + if (kbase_as_has_bus_fault(as) && + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { + bool reset_status; + /* + * Reset the GPU, like in bus_fault_worker, in case an + * earlier error hasn't been properly cleared by this + * point. + */ + dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); + reset_status = kbase_prepare_to_reset_gpu_locked(kbdev); + if (reset_status) + kbase_reset_gpu_locked(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + + return; + } + + if (kbase_as_has_bus_fault(as)) { + /* + * hw counters dumping in progress, signal the + * other thread that it failed + */ + if ((kbdev->hwcnt.kctx == kctx) && + (kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_DUMPING)) + kbdev->hwcnt.backend.state = + KBASE_INSTR_STATE_FAULT; + + /* + * Stop the kctx from submitting more jobs and cause it + * to be scheduled out/rescheduled when all references + * to it are released + */ + kbasep_js_clear_submit_allowed(js_devdata, kctx); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + dev_warn(kbdev->dev, + "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", + as->number, as->fault_addr, + as->fault_extra_addr); + else + dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", + as->number, as->fault_addr); + + /* + * We need to switch to UNMAPPED mode - but we do this in a + * worker so that we can sleep + */ + KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_busfault)); + WARN_ON(work_pending(&as->work_busfault)); + queue_work(as->pf_wq, &as->work_busfault); + atomic_inc(&kbdev->faults_pending); + } else { + KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_pagefault)); + WARN_ON(work_pending(&as->work_pagefault)); + queue_work(as->pf_wq, &as->work_pagefault); + atomic_inc(&kbdev->faults_pending); + } +} + +void kbase_flush_mmu_wqs(struct kbase_device *kbdev) +{ + int i; + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + struct kbase_as *as = &kbdev->as[i]; + + flush_workqueue(as->pf_wq); + } +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_hw.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_hw.h new file mode 100755 index 000000000000..986e959e9a0c --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_hw.h @@ -0,0 +1,123 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file + * Interface file for accessing MMU hardware functionality + */ + +/** + * @page mali_kbase_mmu_hw_page MMU hardware interface + * + * @section mali_kbase_mmu_hw_intro_sec Introduction + * This module provides an abstraction for accessing the functionality provided + * by the midgard MMU and thus allows all MMU HW access to be contained within + * one common place and allows for different backends (implementations) to + * be provided. + */ + +#ifndef _MALI_KBASE_MMU_HW_H_ +#define _MALI_KBASE_MMU_HW_H_ + +/* Forward declarations */ +struct kbase_device; +struct kbase_as; +struct kbase_context; + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup mali_kbase_mmu_hw MMU access APIs + * @{ + */ + +/** @brief MMU fault type descriptor. + */ +enum kbase_mmu_fault_type { + KBASE_MMU_FAULT_TYPE_UNKNOWN = 0, + KBASE_MMU_FAULT_TYPE_PAGE, + KBASE_MMU_FAULT_TYPE_BUS, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED +}; + +/** @brief Configure an address space for use. + * + * Configure the MMU using the address space details setup in the + * @ref kbase_context structure. + * + * @param[in] kbdev kbase device to configure. + * @param[in] as address space to configure. + * @param[in] kctx kbase context to configure. + */ +void kbase_mmu_hw_configure(struct kbase_device *kbdev, + struct kbase_as *as, struct kbase_context *kctx); + +/** @brief Issue an operation to the MMU. + * + * Issue an operation (MMU invalidate, MMU flush, etc) on the address space that + * is associated with the provided @ref kbase_context over the specified range + * + * @param[in] kbdev kbase device to issue the MMU operation on. + * @param[in] as address space to issue the MMU operation on. + * @param[in] kctx kbase context to issue the MMU operation on. + * @param[in] vpfn MMU Virtual Page Frame Number to start the + * operation on. + * @param[in] nr Number of pages to work on. + * @param[in] type Operation type (written to ASn_COMMAND). + * @param[in] handling_irq Is this operation being called during the handling + * of an interrupt? + * + * @return Zero if the operation was successful, non-zero otherwise. + */ +int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, u64 vpfn, u32 nr, u32 type, + unsigned int handling_irq); + +/** @brief Clear a fault that has been previously reported by the MMU. + * + * Clear a bus error or page fault that has been reported by the MMU. + * + * @param[in] kbdev kbase device to clear the fault from. + * @param[in] as address space to clear the fault from. + * @param[in] kctx kbase context to clear the fault from or NULL. + * @param[in] type The type of fault that needs to be cleared. + */ +void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, enum kbase_mmu_fault_type type); + +/** @brief Enable fault that has been previously reported by the MMU. + * + * After a page fault or bus error has been reported by the MMU these + * will be disabled. After these are handled this function needs to be + * called to enable the page fault or bus error fault again. + * + * @param[in] kbdev kbase device to again enable the fault from. + * @param[in] as address space to again enable the fault from. + * @param[in] kctx kbase context to again enable the fault from. + * @param[in] type The type of fault that needs to be enabled again. + */ +void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, enum kbase_mmu_fault_type type); + +/** @} *//* end group mali_kbase_mmu_hw */ +/** @} *//* end group base_kbase_api */ + +#endif /* _MALI_KBASE_MMU_HW_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_aarch64.c new file mode 100755 index 000000000000..0fb717b67af9 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_aarch64.c @@ -0,0 +1,214 @@ +/* + * + * (C) COPYRIGHT 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include "mali_kbase.h" +#include "mali_midg_regmap.h" +#include "mali_kbase_defs.h" + +#define ENTRY_TYPE_MASK 3ULL +/* For valid ATEs bit 1 = ((level == 3) ? 1 : 0). + * Valid ATE entries at level 3 are flagged with the value 3. + * Valid ATE entries at level 0-2 are flagged with the value 1. + */ +#define ENTRY_IS_ATE_L3 3ULL +#define ENTRY_IS_ATE_L02 1ULL +#define ENTRY_IS_INVAL 2ULL +#define ENTRY_IS_PTE 3ULL + +#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ +#define ENTRY_ACCESS_RW (1ULL << 6) /* bits 6:7 */ +#define ENTRY_ACCESS_RO (3ULL << 6) +#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ +#define ENTRY_ACCESS_BIT (1ULL << 10) +#define ENTRY_NX_BIT (1ULL << 54) + +/* Helper Function to perform assignment of page table entries, to + * ensure the use of strd, which is required on LPAE systems. + */ +static inline void page_table_entry_set(u64 *pte, u64 phy) +{ +#ifdef CONFIG_64BIT + *pte = phy; +#elif defined(CONFIG_ARM) + /* + * In order to prevent the compiler keeping cached copies of + * memory, we have to explicitly say that we have updated memory. + * + * Note: We could manually move the data ourselves into R0 and + * R1 by specifying register variables that are explicitly + * given registers assignments, the down side of this is that + * we have to assume cpu endianness. To avoid this we can use + * the ldrd to read the data from memory into R0 and R1 which + * will respect the cpu endianness, we then use strd to make + * the 64 bit assignment to the page table entry. + */ + asm volatile("ldrd r0, r1, [%[ptemp]]\n\t" + "strd r0, r1, [%[pte]]\n\t" + : "=m" (*pte) + : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy) + : "r0", "r1"); +#else +#error "64-bit atomic write must be implemented for your architecture" +#endif +} + +static void mmu_get_as_setup(struct kbase_context *kctx, + struct kbase_mmu_setup * const setup) +{ + /* Set up the required caching policies at the correct indices + * in the memattr register. + */ + setup->memattr = + (AS_MEMATTR_IMPL_DEF_CACHE_POLICY << + (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | + (AS_MEMATTR_FORCE_TO_CACHE_ALL << + (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | + (AS_MEMATTR_WRITE_ALLOC << + (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | + (AS_MEMATTR_AARCH64_OUTER_IMPL_DEF << + (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | + (AS_MEMATTR_AARCH64_OUTER_WA << + (AS_MEMATTR_INDEX_OUTER_WA * 8)); + + setup->transtab = (u64)kctx->pgd & AS_TRANSTAB_BASE_MASK; + setup->transcfg = AS_TRANSCFG_ADRMODE_AARCH64_4K; +} + +static void mmu_update(struct kbase_context *kctx) +{ + struct kbase_device * const kbdev = kctx->kbdev; + struct kbase_as * const as = &kbdev->as[kctx->as_nr]; + struct kbase_mmu_setup * const current_setup = &as->current_setup; + + mmu_get_as_setup(kctx, current_setup); + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as, kctx); +} + +static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) +{ + struct kbase_as * const as = &kbdev->as[as_nr]; + struct kbase_mmu_setup * const current_setup = &as->current_setup; + + current_setup->transtab = 0ULL; + current_setup->transcfg = AS_TRANSCFG_ADRMODE_UNMAPPED; + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as, NULL); +} + +static phys_addr_t pte_to_phy_addr(u64 entry) +{ + if (!(entry & 1)) + return 0; + + return entry & ~0xFFF; +} + +static int ate_is_valid(u64 ate, unsigned int level) +{ + if (level == MIDGARD_MMU_BOTTOMLEVEL) + return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE_L3); + else + return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE_L02); +} + +static int pte_is_valid(u64 pte, unsigned int level) +{ + /* PTEs cannot exist at the bottom level */ + if (level == MIDGARD_MMU_BOTTOMLEVEL) + return false; + return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); +} + +/* + * Map KBASE_REG flags to MMU flags + */ +static u64 get_mmu_flags(unsigned long flags) +{ + u64 mmu_flags; + + /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ + mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; + + /* Set access flags - note that AArch64 stage 1 does not support + * write-only access, so we use read/write instead + */ + if (flags & KBASE_REG_GPU_WR) + mmu_flags |= ENTRY_ACCESS_RW; + else if (flags & KBASE_REG_GPU_RD) + mmu_flags |= ENTRY_ACCESS_RO; + + /* nx if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; + + if (flags & KBASE_REG_SHARE_BOTH) { + /* inner and outer shareable */ + mmu_flags |= SHARE_BOTH_BITS; + } else if (flags & KBASE_REG_SHARE_IN) { + /* inner shareable coherency */ + mmu_flags |= SHARE_INNER_BITS; + } + + return mmu_flags; +} + +static void entry_set_ate(u64 *entry, + struct tagged_addr phy, + unsigned long flags, + unsigned int level) +{ + if (level == MIDGARD_MMU_BOTTOMLEVEL) + page_table_entry_set(entry, as_phys_addr_t(phy) | + get_mmu_flags(flags) | + ENTRY_ACCESS_BIT | ENTRY_IS_ATE_L3); + else + page_table_entry_set(entry, as_phys_addr_t(phy) | + get_mmu_flags(flags) | + ENTRY_ACCESS_BIT | ENTRY_IS_ATE_L02); +} + +static void entry_set_pte(u64 *entry, phys_addr_t phy) +{ + page_table_entry_set(entry, (phy & PAGE_MASK) | + ENTRY_ACCESS_BIT | ENTRY_IS_PTE); +} + +static void entry_invalidate(u64 *entry) +{ + page_table_entry_set(entry, ENTRY_IS_INVAL); +} + +static struct kbase_mmu_mode const aarch64_mode = { + .update = mmu_update, + .get_as_setup = mmu_get_as_setup, + .disable_as = mmu_disable_as, + .pte_to_phy_addr = pte_to_phy_addr, + .ate_is_valid = ate_is_valid, + .pte_is_valid = pte_is_valid, + .entry_set_ate = entry_set_ate, + .entry_set_pte = entry_set_pte, + .entry_invalidate = entry_invalidate +}; + +struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void) +{ + return &aarch64_mode; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_lpae.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_lpae.c new file mode 100755 index 000000000000..f080fdc0be88 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_mmu_mode_lpae.c @@ -0,0 +1,199 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include "mali_kbase.h" +#include "mali_midg_regmap.h" +#include "mali_kbase_defs.h" + +#define ENTRY_TYPE_MASK 3ULL +#define ENTRY_IS_ATE 1ULL +#define ENTRY_IS_INVAL 2ULL +#define ENTRY_IS_PTE 3ULL + +#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ +#define ENTRY_RD_BIT (1ULL << 6) +#define ENTRY_WR_BIT (1ULL << 7) +#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ +#define ENTRY_ACCESS_BIT (1ULL << 10) +#define ENTRY_NX_BIT (1ULL << 54) + +#define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | \ + ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT) + +/* Helper Function to perform assignment of page table entries, to + * ensure the use of strd, which is required on LPAE systems. + */ +static inline void page_table_entry_set(u64 *pte, u64 phy) +{ +#ifdef CONFIG_64BIT + *pte = phy; +#elif defined(CONFIG_ARM) + /* + * In order to prevent the compiler keeping cached copies of + * memory, we have to explicitly say that we have updated + * memory. + * + * Note: We could manually move the data ourselves into R0 and + * R1 by specifying register variables that are explicitly + * given registers assignments, the down side of this is that + * we have to assume cpu endianness. To avoid this we can use + * the ldrd to read the data from memory into R0 and R1 which + * will respect the cpu endianness, we then use strd to make + * the 64 bit assignment to the page table entry. + */ + asm volatile("ldrd r0, r1, [%[ptemp]]\n\t" + "strd r0, r1, [%[pte]]\n\t" + : "=m" (*pte) + : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy) + : "r0", "r1"); +#else +#error "64-bit atomic write must be implemented for your architecture" +#endif +} + +static void mmu_get_as_setup(struct kbase_context *kctx, + struct kbase_mmu_setup * const setup) +{ + /* Set up the required caching policies at the correct indices + * in the memattr register. */ + setup->memattr = + (AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY << + (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | + (AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL << + (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | + (AS_MEMATTR_LPAE_WRITE_ALLOC << + (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | + (AS_MEMATTR_LPAE_OUTER_IMPL_DEF << + (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | + (AS_MEMATTR_LPAE_OUTER_WA << + (AS_MEMATTR_INDEX_OUTER_WA * 8)) | + 0; /* The other indices are unused for now */ + + setup->transtab = ((u64)kctx->pgd & + ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_LPAE_ADDR_SPACE_MASK)) | + AS_TRANSTAB_LPAE_ADRMODE_TABLE | + AS_TRANSTAB_LPAE_READ_INNER; + + setup->transcfg = 0; +} + +static void mmu_update(struct kbase_context *kctx) +{ + struct kbase_device * const kbdev = kctx->kbdev; + struct kbase_as * const as = &kbdev->as[kctx->as_nr]; + struct kbase_mmu_setup * const current_setup = &as->current_setup; + + mmu_get_as_setup(kctx, current_setup); + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as, kctx); +} + +static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) +{ + struct kbase_as * const as = &kbdev->as[as_nr]; + struct kbase_mmu_setup * const current_setup = &as->current_setup; + + current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED; + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as, NULL); +} + +static phys_addr_t pte_to_phy_addr(u64 entry) +{ + if (!(entry & 1)) + return 0; + + return entry & ~0xFFF; +} + +static int ate_is_valid(u64 ate, unsigned int level) +{ + return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); +} + +static int pte_is_valid(u64 pte, unsigned int level) +{ + return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); +} + +/* + * Map KBASE_REG flags to MMU flags + */ +static u64 get_mmu_flags(unsigned long flags) +{ + u64 mmu_flags; + + /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ + mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; + + /* write perm if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0; + /* read perm if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0; + /* nx if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; + + if (flags & KBASE_REG_SHARE_BOTH) { + /* inner and outer shareable */ + mmu_flags |= SHARE_BOTH_BITS; + } else if (flags & KBASE_REG_SHARE_IN) { + /* inner shareable coherency */ + mmu_flags |= SHARE_INNER_BITS; + } + + return mmu_flags; +} + +static void entry_set_ate(u64 *entry, + struct tagged_addr phy, + unsigned long flags, + unsigned int level) +{ + page_table_entry_set(entry, as_phys_addr_t(phy) | get_mmu_flags(flags) | + ENTRY_IS_ATE); +} + +static void entry_set_pte(u64 *entry, phys_addr_t phy) +{ + page_table_entry_set(entry, (phy & ~0xFFF) | ENTRY_IS_PTE); +} + +static void entry_invalidate(u64 *entry) +{ + page_table_entry_set(entry, ENTRY_IS_INVAL); +} + +static struct kbase_mmu_mode const lpae_mode = { + .update = mmu_update, + .get_as_setup = mmu_get_as_setup, + .disable_as = mmu_disable_as, + .pte_to_phy_addr = pte_to_phy_addr, + .ate_is_valid = ate_is_valid, + .pte_is_valid = pte_is_valid, + .entry_set_ate = entry_set_ate, + .entry_set_pte = entry_set_pte, + .entry_invalidate = entry_invalidate +}; + +struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void) +{ + return &lpae_mode; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_platform_fake.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_platform_fake.c new file mode 100755 index 000000000000..0152b35f711b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_platform_fake.c @@ -0,0 +1,119 @@ +/* + * + * (C) COPYRIGHT 2011-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include +#include + + +/* + * This file is included only for type definitions and functions belonging to + * specific platform folders. Do not add dependencies with symbols that are + * defined somewhere else. + */ +#include + +#define PLATFORM_CONFIG_RESOURCE_COUNT 4 +#define PLATFORM_CONFIG_IRQ_RES_COUNT 3 + +static struct platform_device *mali_device; + +#ifndef CONFIG_OF +/** + * @brief Convert data in struct kbase_io_resources struct to Linux-specific resources + * + * Function converts data in struct kbase_io_resources struct to an array of Linux resource structures. Note that function + * assumes that size of linux_resource array is at least PLATFORM_CONFIG_RESOURCE_COUNT. + * Resources are put in fixed order: I/O memory region, job IRQ, MMU IRQ, GPU IRQ. + * + * @param[in] io_resource Input IO resource data + * @param[out] linux_resources Pointer to output array of Linux resource structures + */ +static void kbasep_config_parse_io_resources(const struct kbase_io_resources *io_resources, struct resource *const linux_resources) +{ + if (!io_resources || !linux_resources) { + pr_err("%s: couldn't find proper resources\n", __func__); + return; + } + + memset(linux_resources, 0, PLATFORM_CONFIG_RESOURCE_COUNT * sizeof(struct resource)); + + linux_resources[0].start = io_resources->io_memory_region.start; + linux_resources[0].end = io_resources->io_memory_region.end; + linux_resources[0].flags = IORESOURCE_MEM; + + linux_resources[1].start = io_resources->job_irq_number; + linux_resources[1].end = io_resources->job_irq_number; + linux_resources[1].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; + + linux_resources[2].start = io_resources->mmu_irq_number; + linux_resources[2].end = io_resources->mmu_irq_number; + linux_resources[2].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; + + linux_resources[3].start = io_resources->gpu_irq_number; + linux_resources[3].end = io_resources->gpu_irq_number; + linux_resources[3].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; +} +#endif /* CONFIG_OF */ + +int kbase_platform_register(void) +{ + struct kbase_platform_config *config; +#ifndef CONFIG_OF + struct resource resources[PLATFORM_CONFIG_RESOURCE_COUNT]; +#endif + int err; + + config = kbase_get_platform_config(); /* declared in midgard/mali_kbase_config.h but defined in platform folder */ + if (config == NULL) { + pr_err("%s: couldn't get platform config\n", __func__); + return -ENODEV; + } + + mali_device = platform_device_alloc("mali", 0); + if (mali_device == NULL) + return -ENOMEM; + +#ifndef CONFIG_OF + kbasep_config_parse_io_resources(config->io_resources, resources); + err = platform_device_add_resources(mali_device, resources, PLATFORM_CONFIG_RESOURCE_COUNT); + if (err) { + platform_device_put(mali_device); + mali_device = NULL; + return err; + } +#endif /* CONFIG_OF */ + + err = platform_device_add(mali_device); + if (err) { + platform_device_unregister(mali_device); + mali_device = NULL; + return err; + } + + return 0; +} +EXPORT_SYMBOL(kbase_platform_register); + +void kbase_platform_unregister(void) +{ + if (mali_device) + platform_device_unregister(mali_device); +} +EXPORT_SYMBOL(kbase_platform_unregister); diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.c new file mode 100755 index 000000000000..97d543464c28 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.c @@ -0,0 +1,205 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_pm.c + * Base kernel power management APIs + */ + +#include +#include +#include + +#include + +int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) +{ + return kbase_hwaccess_pm_powerup(kbdev, flags); +} + +void kbase_pm_halt(struct kbase_device *kbdev) +{ + kbase_hwaccess_pm_halt(kbdev); +} + +void kbase_pm_context_active(struct kbase_device *kbdev) +{ + (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); +} + +int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + int c; + int old_count; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + /* Trace timeline information about how long it took to handle the decision + * to powerup. Sometimes the event might be missed due to reading the count + * outside of mutex, but this is necessary to get the trace timing + * correct. */ + old_count = kbdev->pm.active_count; + if (old_count == 0) + kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + if (kbase_pm_is_suspending(kbdev)) { + switch (suspend_handler) { + case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: + if (kbdev->pm.active_count != 0) + break; + /* FALLTHROUGH */ + case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + if (old_count == 0) + kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); + return 1; + + case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE: + /* FALLTHROUGH */ + default: + KBASE_DEBUG_ASSERT_MSG(false, "unreachable"); + break; + } + } + c = ++kbdev->pm.active_count; + KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c); + KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_ACTIVE, NULL, NULL, 0u, c); + + /* Trace the event being handled */ + if (old_count == 0) + kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); + + if (c == 1) + /* First context active: Power on the GPU and any cores requested by + * the policy */ + kbase_hwaccess_pm_gpu_active(kbdev); + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_pm_context_active); + +void kbase_pm_context_idle(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + int c; + int old_count; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + /* Trace timeline information about how long it took to handle the decision + * to powerdown. Sometimes the event might be missed due to reading the + * count outside of mutex, but this is necessary to get the trace timing + * correct. */ + old_count = kbdev->pm.active_count; + if (old_count == 0) + kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE); + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + c = --kbdev->pm.active_count; + KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c); + KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_IDLE, NULL, NULL, 0u, c); + + KBASE_DEBUG_ASSERT(c >= 0); + + /* Trace the event being handled */ + if (old_count == 0) + kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE); + + if (c == 0) { + /* Last context has gone idle */ + kbase_hwaccess_pm_gpu_idle(kbdev); + + /* Wake up anyone waiting for this to become 0 (e.g. suspend). The + * waiters must synchronize with us by locking the pm.lock after + * waiting */ + wake_up(&kbdev->pm.zero_active_count_wait); + } + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); +} + +KBASE_EXPORT_TEST_API(kbase_pm_context_idle); + +void kbase_pm_suspend(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + + /* Suspend vinstr. + * This call will block until vinstr is suspended. */ + kbase_vinstr_suspend(kbdev->vinstr_ctx); + + mutex_lock(&kbdev->pm.lock); + KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); + kbdev->pm.suspending = true; + mutex_unlock(&kbdev->pm.lock); + + /* From now on, the active count will drop towards zero. Sometimes, it'll + * go up briefly before going down again. However, once it reaches zero it + * will stay there - guaranteeing that we've idled all pm references */ + + /* Suspend job scheduler and associated components, so that it releases all + * the PM active count references */ + kbasep_js_suspend(kbdev); + + /* Wait for the active count to reach zero. This is not the same as + * waiting for a power down, since not all policies power down when this + * reaches zero. */ + wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0); + + /* NOTE: We synchronize with anything that was just finishing a + * kbase_pm_context_idle() call by locking the pm.lock below */ + + kbase_hwaccess_pm_suspend(kbdev); +} + +void kbase_pm_resume(struct kbase_device *kbdev) +{ + /* MUST happen before any pm_context_active calls occur */ + kbase_hwaccess_pm_resume(kbdev); + + /* Initial active call, to power on the GPU/cores if needed */ + kbase_pm_context_active(kbdev); + + /* Resume any blocked atoms (which may cause contexts to be scheduled in + * and dependent atoms to run) */ + kbase_resume_suspended_soft_jobs(kbdev); + + /* Resume the Job Scheduler and associated components, and start running + * atoms */ + kbasep_js_resume(kbdev); + + /* Matching idle call, to power off the GPU/cores if we didn't actually + * need it and the policy doesn't want it on */ + kbase_pm_context_idle(kbdev); + + /* Resume vinstr operation */ + kbase_vinstr_resume(kbdev->vinstr_ctx); +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.h new file mode 100755 index 000000000000..37fa2479df74 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_pm.h @@ -0,0 +1,171 @@ +/* + * + * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_pm.h + * Power management API definitions + */ + +#ifndef _KBASE_PM_H_ +#define _KBASE_PM_H_ + +#include "mali_kbase_hwaccess_pm.h" + +#define PM_ENABLE_IRQS 0x01 +#define PM_HW_ISSUES_DETECT 0x02 + + +/** Initialize the power management framework. + * + * Must be called before any other power management function + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + * + * @return 0 if the power management framework was successfully initialized. + */ +int kbase_pm_init(struct kbase_device *kbdev); + +/** Power up GPU after all modules have been initialized and interrupt handlers installed. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + * + * @param flags Flags to pass on to kbase_pm_init_hw + * + * @return 0 if powerup was successful. + */ +int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags); + +/** + * Halt the power management framework. + * Should ensure that no new interrupts are generated, + * but allow any currently running interrupt handlers to complete successfully. + * The GPU is forced off by the time this function returns, regardless of + * whether or not the active power policy asks for the GPU to be powered off. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_halt(struct kbase_device *kbdev); + +/** Terminate the power management framework. + * + * No power management functions may be called after this + * (except @ref kbase_pm_init) + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_term(struct kbase_device *kbdev); + +/** Increment the count of active contexts. + * + * This function should be called when a context is about to submit a job. It informs the active power policy that the + * GPU is going to be in use shortly and the policy is expected to start turning on the GPU. + * + * This function will block until the GPU is available. + * + * This function ASSERTS if a suspend is occuring/has occurred whilst this is + * in use. Use kbase_pm_contect_active_unless_suspending() instead. + * + * @note a Suspend is only visible to Kernel threads; user-space threads in a + * syscall cannot witness a suspend, because they are frozen before the suspend + * begins. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_context_active(struct kbase_device *kbdev); + + +/** Handler codes for doing kbase_pm_context_active_handle_suspend() */ +enum kbase_pm_suspend_handler { + /** A suspend is not expected/not possible - this is the same as + * kbase_pm_context_active() */ + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, + /** If we're suspending, fail and don't increase the active count */ + KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE, + /** If we're suspending, succeed and allow the active count to increase iff + * it didn't go from 0->1 (i.e., we didn't re-activate the GPU). + * + * This should only be used when there is a bounded time on the activation + * (e.g. guarantee it's going to be idled very soon after) */ + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE +}; + +/** Suspend 'safe' variant of kbase_pm_context_active() + * + * If a suspend is in progress, this allows for various different ways of + * handling the suspend. Refer to @ref enum kbase_pm_suspend_handler for details. + * + * We returns a status code indicating whether we're allowed to keep the GPU + * active during the suspend, depending on the handler code. If the status code + * indicates a failure, the caller must abort whatever operation it was + * attempting, and potentially queue it up for after the OS has resumed. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + * @param suspend_handler The handler code for how to handle a suspend that might occur + * @return zero Indicates success + * @return non-zero Indicates failure due to the system being suspending/suspended. + */ +int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler); + +/** Decrement the reference count of active contexts. + * + * This function should be called when a context becomes idle. After this call the GPU may be turned off by the power + * policy so the calling code should ensure that it does not access the GPU's registers. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_context_idle(struct kbase_device *kbdev); + +/** + * Suspend the GPU and prevent any further register accesses to it from Kernel + * threads. + * + * This is called in response to an OS suspend event, and calls into the various + * kbase components to complete the suspend. + * + * @note the mechanisms used here rely on all user-space threads being frozen + * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up + * the GPU e.g. via atom submission. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_suspend(struct kbase_device *kbdev); + +/** + * Resume the GPU, allow register accesses to it, and resume running atoms on + * the GPU. + * + * This is called in response to an OS resume event, and calls into the various + * kbase components to complete the resume. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_resume(struct kbase_device *kbdev); + +/** + * kbase_pm_vsync_callback - vsync callback + * + * @buffer_updated: 1 if a new frame was displayed, 0 otherwise + * @data: Pointer to the kbase device as returned by kbase_find_device() + * + * Callback function used to notify the power management code that a vsync has + * occurred on the display. + */ +void kbase_pm_vsync_callback(int buffer_updated, void *data); + +#endif /* _KBASE_PM_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_profiling_gator_api.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_profiling_gator_api.h new file mode 100755 index 000000000000..7fb674eded37 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_profiling_gator_api.h @@ -0,0 +1,40 @@ +/* + * + * (C) COPYRIGHT 2010, 2013 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_profiling_gator_api.h + * Model interface + */ + +#ifndef _KBASE_PROFILING_GATOR_API_H_ +#define _KBASE_PROFILING_GATOR_API_H_ + +/* + * List of possible actions to be controlled by Streamline. + * The following numbers are used by gator to control + * the frame buffer dumping and s/w counter reporting. + */ +#define FBDUMP_CONTROL_ENABLE (1) +#define FBDUMP_CONTROL_RATE (2) +#define SW_COUNTER_ENABLE (3) +#define FBDUMP_CONTROL_RESIZE_FACTOR (4) +#define FBDUMP_CONTROL_MAX (5) +#define FBDUMP_CONTROL_MIN FBDUMP_CONTROL_ENABLE + +void _mali_profiling_control(u32 action, u32 value); + +#endif /* _KBASE_PROFILING_GATOR_API */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.c new file mode 100755 index 000000000000..9e73f9f4999e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.c @@ -0,0 +1,130 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "mali_kbase.h" + +#include "mali_kbase_regs_history_debugfs.h" + +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) + +#include + + +static int regs_history_size_get(void *data, u64 *val) +{ + struct kbase_io_history *const h = data; + + *val = h->size; + + return 0; +} + +static int regs_history_size_set(void *data, u64 val) +{ + struct kbase_io_history *const h = data; + + return kbase_io_history_resize(h, (u16)val); +} + + +DEFINE_SIMPLE_ATTRIBUTE(regs_history_size_fops, + regs_history_size_get, + regs_history_size_set, + "%llu\n"); + + +/** + * regs_history_show - show callback for the register access history file. + * + * @sfile: The debugfs entry + * @data: Data associated with the entry + * + * This function is called to dump all recent accesses to the GPU registers. + * + * @return 0 if successfully prints data in debugfs entry file, failure + * otherwise + */ +static int regs_history_show(struct seq_file *sfile, void *data) +{ + struct kbase_io_history *const h = sfile->private; + u16 i; + size_t iters; + unsigned long flags; + + if (!h->enabled) { + seq_puts(sfile, "The register access history is disabled\n"); + goto out; + } + + spin_lock_irqsave(&h->lock, flags); + + iters = (h->size > h->count) ? h->count : h->size; + seq_printf(sfile, "Last %zu register accesses of %zu total:\n", iters, + h->count); + for (i = 0; i < iters; ++i) { + struct kbase_io_access *io = + &h->buf[(h->count - iters + i) % h->size]; + char const access = (io->addr & 1) ? 'w' : 'r'; + + seq_printf(sfile, "%6i: %c: reg 0x%p val %08x\n", i, access, + (void *)(io->addr & ~0x1), io->value); + } + + spin_unlock_irqrestore(&h->lock, flags); + +out: + return 0; +} + + +/** + * regs_history_open - open operation for regs_history debugfs file + * + * @in: &struct inode pointer + * @file: &struct file pointer + * + * @return file descriptor + */ +static int regs_history_open(struct inode *in, struct file *file) +{ + return single_open(file, ®s_history_show, in->i_private); +} + + +static const struct file_operations regs_history_fops = { + .open = ®s_history_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + + +void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_bool("regs_history_enabled", S_IRUGO | S_IWUSR, + kbdev->mali_debugfs_directory, + &kbdev->io_history.enabled); + debugfs_create_file("regs_history_size", S_IRUGO | S_IWUSR, + kbdev->mali_debugfs_directory, + &kbdev->io_history, ®s_history_size_fops); + debugfs_create_file("regs_history", S_IRUGO, + kbdev->mali_debugfs_directory, &kbdev->io_history, + ®s_history_fops); +} + + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.h new file mode 100755 index 000000000000..fbb36b3f22e4 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_regs_history_debugfs.h @@ -0,0 +1,50 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * Header file for register access history support via debugfs + * + * This interface is made available via /sys/kernel/debug/mali#/regs_history*. + * + * Usage: + * - regs_history_enabled: whether recording of register accesses is enabled. + * Write 'y' to enable, 'n' to disable. + * - regs_history_size: size of the register history buffer, must be > 0 + * - regs_history: return the information about last accesses to the registers. + */ + +#ifndef _KBASE_REGS_HISTORY_DEBUGFS_H +#define _KBASE_REGS_HISTORY_DEBUGFS_H + +struct kbase_device; + +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI) + +/** + * kbasep_regs_history_debugfs_init - add debugfs entries for register history + * + * @kbdev: Pointer to kbase_device containing the register history + */ +void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev); + +#else /* CONFIG_DEBUG_FS */ + +#define kbasep_regs_history_debugfs_init CSTD_NOP + +#endif /* CONFIG_DEBUG_FS */ + +#endif /*_KBASE_REGS_HISTORY_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_replay.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_replay.c new file mode 100755 index 000000000000..9f4dc372770d --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_replay.c @@ -0,0 +1,1166 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_replay.c + * Replay soft job handlers + */ + +#include +#include +#include +#include +#include + +#define JOB_NOT_STARTED 0 +#define JOB_TYPE_NULL (1) +#define JOB_TYPE_VERTEX (5) +#define JOB_TYPE_TILER (7) +#define JOB_TYPE_FUSED (8) +#define JOB_TYPE_FRAGMENT (9) + +#define JOB_HEADER_32_FBD_OFFSET (31*4) +#define JOB_HEADER_64_FBD_OFFSET (44*4) + +#define FBD_POINTER_MASK (~0x3f) + +#define SFBD_TILER_OFFSET (48*4) + +#define MFBD_TILER_OFFSET (14*4) + +#define FBD_HIERARCHY_WEIGHTS 8 +#define FBD_HIERARCHY_MASK_MASK 0x1fff + +#define FBD_TYPE 1 + +#define HIERARCHY_WEIGHTS 13 + +#define JOB_HEADER_ID_MAX 0xffff + +#define JOB_SOURCE_ID(status) (((status) >> 16) & 0xFFFF) +#define JOB_POLYGON_LIST (0x03) + +struct fragment_job { + struct job_descriptor_header header; + + u32 x[2]; + union { + u64 _64; + u32 _32; + } fragment_fbd; +}; + +static void dump_job_head(struct kbase_context *kctx, char *head_str, + struct job_descriptor_header *job) +{ +#ifdef CONFIG_MALI_BIFROST_DEBUG + dev_dbg(kctx->kbdev->dev, "%s\n", head_str); + dev_dbg(kctx->kbdev->dev, + "addr = %p\n" + "exception_status = %x (Source ID: 0x%x Access: 0x%x Exception: 0x%x)\n" + "first_incomplete_task = %x\n" + "fault_pointer = %llx\n" + "job_descriptor_size = %x\n" + "job_type = %x\n" + "job_barrier = %x\n" + "_reserved_01 = %x\n" + "_reserved_02 = %x\n" + "_reserved_03 = %x\n" + "_reserved_04/05 = %x,%x\n" + "job_index = %x\n" + "dependencies = %x,%x\n", + job, job->exception_status, + JOB_SOURCE_ID(job->exception_status), + (job->exception_status >> 8) & 0x3, + job->exception_status & 0xFF, + job->first_incomplete_task, + job->fault_pointer, job->job_descriptor_size, + job->job_type, job->job_barrier, job->_reserved_01, + job->_reserved_02, job->_reserved_03, + job->_reserved_04, job->_reserved_05, + job->job_index, + job->job_dependency_index_1, + job->job_dependency_index_2); + + if (job->job_descriptor_size) + dev_dbg(kctx->kbdev->dev, "next = %llx\n", + job->next_job._64); + else + dev_dbg(kctx->kbdev->dev, "next = %x\n", + job->next_job._32); +#endif +} + +static int kbasep_replay_reset_sfbd(struct kbase_context *kctx, + u64 fbd_address, u64 tiler_heap_free, + u16 hierarchy_mask, u32 default_weight) +{ + struct { + u32 padding_1[1]; + u32 flags; + u64 padding_2[2]; + u64 heap_free_address; + u32 padding[8]; + u32 weights[FBD_HIERARCHY_WEIGHTS]; + } *fbd_tiler; + struct kbase_vmap_struct map; + + dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address); + + fbd_tiler = kbase_vmap(kctx, fbd_address + SFBD_TILER_OFFSET, + sizeof(*fbd_tiler), &map); + if (!fbd_tiler) { + dev_err(kctx->kbdev->dev, "kbasep_replay_reset_fbd: failed to map fbd\n"); + return -EINVAL; + } + +#ifdef CONFIG_MALI_BIFROST_DEBUG + dev_dbg(kctx->kbdev->dev, + "FBD tiler:\n" + "flags = %x\n" + "heap_free_address = %llx\n", + fbd_tiler->flags, fbd_tiler->heap_free_address); +#endif + if (hierarchy_mask) { + u32 weights[HIERARCHY_WEIGHTS]; + u16 old_hierarchy_mask = fbd_tiler->flags & + FBD_HIERARCHY_MASK_MASK; + int i, j = 0; + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) { + if (old_hierarchy_mask & (1 << i)) { + KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); + weights[i] = fbd_tiler->weights[j++]; + } else { + weights[i] = default_weight; + } + } + + + dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x New hierarchy mask=%x\n", + old_hierarchy_mask, hierarchy_mask); + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) + dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n", + i, weights[i]); + + j = 0; + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) { + if (hierarchy_mask & (1 << i)) { + KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); + + dev_dbg(kctx->kbdev->dev, " Writing hierarchy level %02d (%08x) to %d\n", + i, weights[i], j); + + fbd_tiler->weights[j++] = weights[i]; + } + } + + for (; j < FBD_HIERARCHY_WEIGHTS; j++) + fbd_tiler->weights[j] = 0; + + fbd_tiler->flags = hierarchy_mask | (1 << 16); + } + + fbd_tiler->heap_free_address = tiler_heap_free; + + dev_dbg(kctx->kbdev->dev, "heap_free_address=%llx flags=%x\n", + fbd_tiler->heap_free_address, fbd_tiler->flags); + + kbase_vunmap(kctx, &map); + + return 0; +} + +static int kbasep_replay_reset_mfbd(struct kbase_context *kctx, + u64 fbd_address, u64 tiler_heap_free, + u16 hierarchy_mask, u32 default_weight) +{ + struct kbase_vmap_struct map; + struct { + u32 padding_0; + u32 flags; + u64 padding_1[2]; + u64 heap_free_address; + u64 padding_2; + u32 weights[FBD_HIERARCHY_WEIGHTS]; + } *fbd_tiler; + + dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address); + + fbd_tiler = kbase_vmap(kctx, fbd_address + MFBD_TILER_OFFSET, + sizeof(*fbd_tiler), &map); + if (!fbd_tiler) { + dev_err(kctx->kbdev->dev, + "kbasep_replay_reset_fbd: failed to map fbd\n"); + return -EINVAL; + } + +#ifdef CONFIG_MALI_BIFROST_DEBUG + dev_dbg(kctx->kbdev->dev, "FBD tiler:\n" + "flags = %x\n" + "heap_free_address = %llx\n", + fbd_tiler->flags, + fbd_tiler->heap_free_address); +#endif + if (hierarchy_mask) { + u32 weights[HIERARCHY_WEIGHTS]; + u16 old_hierarchy_mask = (fbd_tiler->flags) & + FBD_HIERARCHY_MASK_MASK; + int i, j = 0; + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) { + if (old_hierarchy_mask & (1 << i)) { + KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); + weights[i] = fbd_tiler->weights[j++]; + } else { + weights[i] = default_weight; + } + } + + + dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x New hierarchy mask=%x\n", + old_hierarchy_mask, hierarchy_mask); + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) + dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n", + i, weights[i]); + + j = 0; + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) { + if (hierarchy_mask & (1 << i)) { + KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); + + dev_dbg(kctx->kbdev->dev, + " Writing hierarchy level %02d (%08x) to %d\n", + i, weights[i], j); + + fbd_tiler->weights[j++] = weights[i]; + } + } + + for (; j < FBD_HIERARCHY_WEIGHTS; j++) + fbd_tiler->weights[j] = 0; + + fbd_tiler->flags = hierarchy_mask | (1 << 16); + } + + fbd_tiler->heap_free_address = tiler_heap_free; + + kbase_vunmap(kctx, &map); + + return 0; +} + +/** + * @brief Reset the status of an FBD pointed to by a tiler job + * + * This performs two functions : + * - Set the hierarchy mask + * - Reset the tiler free heap address + * + * @param[in] kctx Context pointer + * @param[in] job_header Address of job header to reset. + * @param[in] tiler_heap_free The value to reset Tiler Heap Free to + * @param[in] hierarchy_mask The hierarchy mask to use + * @param[in] default_weight Default hierarchy weight to write when no other + * weight is given in the FBD + * @param[in] job_64 true if this job is using 64-bit + * descriptors + * + * @return 0 on success, error code on failure + */ +static int kbasep_replay_reset_tiler_job(struct kbase_context *kctx, + u64 job_header, u64 tiler_heap_free, + u16 hierarchy_mask, u32 default_weight, bool job_64) +{ + struct kbase_vmap_struct map; + u64 fbd_address; + + if (job_64) { + u64 *job_ext; + + job_ext = kbase_vmap(kctx, + job_header + JOB_HEADER_64_FBD_OFFSET, + sizeof(*job_ext), &map); + + if (!job_ext) { + dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n"); + return -EINVAL; + } + + fbd_address = *job_ext; + + kbase_vunmap(kctx, &map); + } else { + u32 *job_ext; + + job_ext = kbase_vmap(kctx, + job_header + JOB_HEADER_32_FBD_OFFSET, + sizeof(*job_ext), &map); + + if (!job_ext) { + dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n"); + return -EINVAL; + } + + fbd_address = *job_ext; + + kbase_vunmap(kctx, &map); + } + + if (fbd_address & FBD_TYPE) { + return kbasep_replay_reset_mfbd(kctx, + fbd_address & FBD_POINTER_MASK, + tiler_heap_free, + hierarchy_mask, + default_weight); + } else { + return kbasep_replay_reset_sfbd(kctx, + fbd_address & FBD_POINTER_MASK, + tiler_heap_free, + hierarchy_mask, + default_weight); + } +} + +/** + * @brief Reset the status of a job + * + * This performs the following functions : + * + * - Reset the Job Status field of each job to NOT_STARTED. + * - Set the Job Type field of any Vertex Jobs to Null Job. + * - For any jobs using an FBD, set the Tiler Heap Free field to the value of + * the tiler_heap_free parameter, and set the hierarchy level mask to the + * hier_mask parameter. + * - Offset HW dependencies by the hw_job_id_offset parameter + * - Set the Perform Job Barrier flag if this job is the first in the chain + * - Read the address of the next job header + * + * @param[in] kctx Context pointer + * @param[in,out] job_header Address of job header to reset. Set to address + * of next job header on exit. + * @param[in] prev_jc Previous job chain to link to, if this job is + * the last in the chain. + * @param[in] hw_job_id_offset Offset for HW job IDs + * @param[in] tiler_heap_free The value to reset Tiler Heap Free to + * @param[in] hierarchy_mask The hierarchy mask to use + * @param[in] default_weight Default hierarchy weight to write when no other + * weight is given in the FBD + * @param[in] first_in_chain true if this job is the first in the chain + * @param[in] fragment_chain true if this job is in the fragment chain + * + * @return 0 on success, error code on failure + */ +static int kbasep_replay_reset_job(struct kbase_context *kctx, + u64 *job_header, u64 prev_jc, + u64 tiler_heap_free, u16 hierarchy_mask, + u32 default_weight, u16 hw_job_id_offset, + bool first_in_chain, bool fragment_chain) +{ + struct fragment_job *frag_job; + struct job_descriptor_header *job; + u64 new_job_header; + struct kbase_vmap_struct map; + + frag_job = kbase_vmap(kctx, *job_header, sizeof(*frag_job), &map); + if (!frag_job) { + dev_err(kctx->kbdev->dev, + "kbasep_replay_parse_jc: failed to map jc\n"); + return -EINVAL; + } + job = &frag_job->header; + + dump_job_head(kctx, "Job header:", job); + + if (job->exception_status == JOB_NOT_STARTED && !fragment_chain) { + dev_err(kctx->kbdev->dev, "Job already not started\n"); + goto out_unmap; + } + job->exception_status = JOB_NOT_STARTED; + + if (job->job_type == JOB_TYPE_VERTEX) + job->job_type = JOB_TYPE_NULL; + + if (job->job_type == JOB_TYPE_FUSED) { + dev_err(kctx->kbdev->dev, "Fused jobs can not be replayed\n"); + goto out_unmap; + } + + if (first_in_chain) + job->job_barrier = 1; + + if ((job->job_dependency_index_1 + hw_job_id_offset) > + JOB_HEADER_ID_MAX || + (job->job_dependency_index_2 + hw_job_id_offset) > + JOB_HEADER_ID_MAX || + (job->job_index + hw_job_id_offset) > JOB_HEADER_ID_MAX) { + dev_err(kctx->kbdev->dev, + "Job indicies/dependencies out of valid range\n"); + goto out_unmap; + } + + if (job->job_dependency_index_1) + job->job_dependency_index_1 += hw_job_id_offset; + if (job->job_dependency_index_2) + job->job_dependency_index_2 += hw_job_id_offset; + + job->job_index += hw_job_id_offset; + + if (job->job_descriptor_size) { + new_job_header = job->next_job._64; + if (!job->next_job._64) + job->next_job._64 = prev_jc; + } else { + new_job_header = job->next_job._32; + if (!job->next_job._32) + job->next_job._32 = prev_jc; + } + dump_job_head(kctx, "Updated to:", job); + + if (job->job_type == JOB_TYPE_TILER) { + bool job_64 = job->job_descriptor_size != 0; + + if (kbasep_replay_reset_tiler_job(kctx, *job_header, + tiler_heap_free, hierarchy_mask, + default_weight, job_64) != 0) + goto out_unmap; + + } else if (job->job_type == JOB_TYPE_FRAGMENT) { + u64 fbd_address; + + if (job->job_descriptor_size) + fbd_address = frag_job->fragment_fbd._64; + else + fbd_address = (u64)frag_job->fragment_fbd._32; + + if (fbd_address & FBD_TYPE) { + if (kbasep_replay_reset_mfbd(kctx, + fbd_address & FBD_POINTER_MASK, + tiler_heap_free, + hierarchy_mask, + default_weight) != 0) + goto out_unmap; + } else { + if (kbasep_replay_reset_sfbd(kctx, + fbd_address & FBD_POINTER_MASK, + tiler_heap_free, + hierarchy_mask, + default_weight) != 0) + goto out_unmap; + } + } + + kbase_vunmap(kctx, &map); + + *job_header = new_job_header; + + return 0; + +out_unmap: + kbase_vunmap(kctx, &map); + return -EINVAL; +} + +/** + * @brief Find the highest job ID in a job chain + * + * @param[in] kctx Context pointer + * @param[in] jc Job chain start address + * @param[out] hw_job_id Highest job ID in chain + * + * @return 0 on success, error code on failure + */ +static int kbasep_replay_find_hw_job_id(struct kbase_context *kctx, + u64 jc, u16 *hw_job_id) +{ + while (jc) { + struct job_descriptor_header *job; + struct kbase_vmap_struct map; + + dev_dbg(kctx->kbdev->dev, + "kbasep_replay_find_hw_job_id: parsing jc=%llx\n", jc); + + job = kbase_vmap(kctx, jc, sizeof(*job), &map); + if (!job) { + dev_err(kctx->kbdev->dev, "failed to map jc\n"); + + return -EINVAL; + } + + if (job->job_index > *hw_job_id) + *hw_job_id = job->job_index; + + if (job->job_descriptor_size) + jc = job->next_job._64; + else + jc = job->next_job._32; + + kbase_vunmap(kctx, &map); + } + + return 0; +} + +/** + * @brief Reset the status of a number of jobs + * + * This function walks the provided job chain, and calls + * kbasep_replay_reset_job for each job. It also links the job chain to the + * provided previous job chain. + * + * The function will fail if any of the jobs passed already have status of + * NOT_STARTED. + * + * @param[in] kctx Context pointer + * @param[in] jc Job chain to be processed + * @param[in] prev_jc Job chain to be added to. May be NULL + * @param[in] tiler_heap_free The value to reset Tiler Heap Free to + * @param[in] hierarchy_mask The hierarchy mask to use + * @param[in] default_weight Default hierarchy weight to write when no other + * weight is given in the FBD + * @param[in] hw_job_id_offset Offset for HW job IDs + * @param[in] fragment_chain true if this chain is the fragment chain + * + * @return 0 on success, error code otherwise + */ +static int kbasep_replay_parse_jc(struct kbase_context *kctx, + u64 jc, u64 prev_jc, + u64 tiler_heap_free, u16 hierarchy_mask, + u32 default_weight, u16 hw_job_id_offset, + bool fragment_chain) +{ + bool first_in_chain = true; + int nr_jobs = 0; + + dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: jc=%llx hw_job_id=%x\n", + jc, hw_job_id_offset); + + while (jc) { + dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: parsing jc=%llx\n", jc); + + if (kbasep_replay_reset_job(kctx, &jc, prev_jc, + tiler_heap_free, hierarchy_mask, + default_weight, hw_job_id_offset, + first_in_chain, fragment_chain) != 0) + return -EINVAL; + + first_in_chain = false; + + nr_jobs++; + if (fragment_chain && + nr_jobs >= BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT) { + dev_err(kctx->kbdev->dev, + "Exceeded maximum number of jobs in fragment chain\n"); + return -EINVAL; + } + } + + return 0; +} + +/** + * @brief Reset the status of a replay job, and set up dependencies + * + * This performs the actions to allow the replay job to be re-run following + * completion of the passed dependency. + * + * @param[in] katom The atom to be reset + * @param[in] dep_atom The dependency to be attached to the atom + */ +static void kbasep_replay_reset_softjob(struct kbase_jd_atom *katom, + struct kbase_jd_atom *dep_atom) +{ + katom->status = KBASE_JD_ATOM_STATE_QUEUED; + kbase_jd_katom_dep_set(&katom->dep[0], dep_atom, BASE_JD_DEP_TYPE_DATA); + list_add_tail(&katom->dep_item[0], &dep_atom->dep_head[0]); +} + +/** + * @brief Allocate an unused katom + * + * This will search the provided context for an unused katom, and will mark it + * as KBASE_JD_ATOM_STATE_QUEUED. + * + * If no atoms are available then the function will fail. + * + * @param[in] kctx Context pointer + * @return An atom ID, or -1 on failure + */ +static int kbasep_allocate_katom(struct kbase_context *kctx) +{ + struct kbase_jd_context *jctx = &kctx->jctx; + int i; + + for (i = BASE_JD_ATOM_COUNT-1; i > 0; i--) { + if (jctx->atoms[i].status == KBASE_JD_ATOM_STATE_UNUSED) { + jctx->atoms[i].status = KBASE_JD_ATOM_STATE_QUEUED; + dev_dbg(kctx->kbdev->dev, + "kbasep_allocate_katom: Allocated atom %d\n", + i); + return i; + } + } + + return -1; +} + +/** + * @brief Release a katom + * + * This will mark the provided atom as available, and remove any dependencies. + * + * For use on error path. + * + * @param[in] kctx Context pointer + * @param[in] atom_id ID of atom to release + */ +static void kbasep_release_katom(struct kbase_context *kctx, int atom_id) +{ + struct kbase_jd_context *jctx = &kctx->jctx; + + dev_dbg(kctx->kbdev->dev, "kbasep_release_katom: Released atom %d\n", + atom_id); + + while (!list_empty(&jctx->atoms[atom_id].dep_head[0])) + list_del(jctx->atoms[atom_id].dep_head[0].next); + + while (!list_empty(&jctx->atoms[atom_id].dep_head[1])) + list_del(jctx->atoms[atom_id].dep_head[1].next); + + jctx->atoms[atom_id].status = KBASE_JD_ATOM_STATE_UNUSED; +} + +static void kbasep_replay_create_atom(struct kbase_context *kctx, + struct base_jd_atom_v2 *atom, + int atom_nr, + base_jd_prio prio) +{ + atom->nr_extres = 0; + atom->extres_list = 0; + atom->device_nr = 0; + atom->prio = prio; + atom->atom_number = atom_nr; + + base_jd_atom_dep_set(&atom->pre_dep[0], 0 , BASE_JD_DEP_TYPE_INVALID); + base_jd_atom_dep_set(&atom->pre_dep[1], 0 , BASE_JD_DEP_TYPE_INVALID); + + atom->udata.blob[0] = 0; + atom->udata.blob[1] = 0; +} + +/** + * @brief Create two atoms for the purpose of replaying jobs + * + * Two atoms are allocated and created. The jc pointer is not set at this + * stage. The second atom has a dependency on the first. The remaining fields + * are set up as follows : + * + * - No external resources. Any required external resources will be held by the + * replay atom. + * - device_nr is set to 0. This is not relevant as + * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set. + * - Priority is inherited from the replay job. + * + * @param[out] t_atom Atom to use for tiler jobs + * @param[out] f_atom Atom to use for fragment jobs + * @param[in] prio Priority of new atom (inherited from replay soft + * job) + * @return 0 on success, error code on failure + */ +static int kbasep_replay_create_atoms(struct kbase_context *kctx, + struct base_jd_atom_v2 *t_atom, + struct base_jd_atom_v2 *f_atom, + base_jd_prio prio) +{ + int t_atom_nr, f_atom_nr; + + t_atom_nr = kbasep_allocate_katom(kctx); + if (t_atom_nr < 0) { + dev_err(kctx->kbdev->dev, "Failed to allocate katom\n"); + return -EINVAL; + } + + f_atom_nr = kbasep_allocate_katom(kctx); + if (f_atom_nr < 0) { + dev_err(kctx->kbdev->dev, "Failed to allocate katom\n"); + kbasep_release_katom(kctx, t_atom_nr); + return -EINVAL; + } + + kbasep_replay_create_atom(kctx, t_atom, t_atom_nr, prio); + kbasep_replay_create_atom(kctx, f_atom, f_atom_nr, prio); + + base_jd_atom_dep_set(&f_atom->pre_dep[0], t_atom_nr , BASE_JD_DEP_TYPE_DATA); + + return 0; +} + +#ifdef CONFIG_MALI_BIFROST_DEBUG +static void payload_dump(struct kbase_context *kctx, base_jd_replay_payload *payload) +{ + u64 next; + + dev_dbg(kctx->kbdev->dev, "Tiler jc list :\n"); + next = payload->tiler_jc_list; + + while (next) { + struct kbase_vmap_struct map; + base_jd_replay_jc *jc_struct; + + jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &map); + + if (!jc_struct) + return; + + dev_dbg(kctx->kbdev->dev, "* jc_struct=%p jc=%llx next=%llx\n", + jc_struct, jc_struct->jc, jc_struct->next); + + next = jc_struct->next; + + kbase_vunmap(kctx, &map); + } +} +#endif + +/** + * @brief Parse a base_jd_replay_payload provided by userspace + * + * This will read the payload from userspace, and parse the job chains. + * + * @param[in] kctx Context pointer + * @param[in] replay_atom Replay soft job atom + * @param[in] t_atom Atom to use for tiler jobs + * @param[in] f_atom Atom to use for fragment jobs + * @return 0 on success, error code on failure + */ +static int kbasep_replay_parse_payload(struct kbase_context *kctx, + struct kbase_jd_atom *replay_atom, + struct base_jd_atom_v2 *t_atom, + struct base_jd_atom_v2 *f_atom) +{ + base_jd_replay_payload *payload = NULL; + u64 next; + u64 prev_jc = 0; + u16 hw_job_id_offset = 0; + int ret = -EINVAL; + struct kbase_vmap_struct map; + + dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: replay_atom->jc = %llx sizeof(payload) = %zu\n", + replay_atom->jc, sizeof(payload)); + + payload = kbase_vmap(kctx, replay_atom->jc, sizeof(*payload), &map); + if (!payload) { + dev_err(kctx->kbdev->dev, "kbasep_replay_parse_payload: failed to map payload into kernel space\n"); + return -EINVAL; + } + +#ifdef BASE_LEGACY_UK10_2_SUPPORT + if (KBASE_API_VERSION(10, 3) > replay_atom->kctx->api_version) { + base_jd_replay_payload_uk10_2 *payload_uk10_2; + u16 tiler_core_req; + u16 fragment_core_req; + + payload_uk10_2 = (base_jd_replay_payload_uk10_2 *) payload; + memcpy(&tiler_core_req, &payload_uk10_2->tiler_core_req, + sizeof(tiler_core_req)); + memcpy(&fragment_core_req, &payload_uk10_2->fragment_core_req, + sizeof(fragment_core_req)); + payload->tiler_core_req = (u32)(tiler_core_req & 0x7fff); + payload->fragment_core_req = (u32)(fragment_core_req & 0x7fff); + } +#endif /* BASE_LEGACY_UK10_2_SUPPORT */ + +#ifdef CONFIG_MALI_BIFROST_DEBUG + dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: payload=%p\n", payload); + dev_dbg(kctx->kbdev->dev, "Payload structure:\n" + "tiler_jc_list = %llx\n" + "fragment_jc = %llx\n" + "tiler_heap_free = %llx\n" + "fragment_hierarchy_mask = %x\n" + "tiler_hierarchy_mask = %x\n" + "hierarchy_default_weight = %x\n" + "tiler_core_req = %x\n" + "fragment_core_req = %x\n", + payload->tiler_jc_list, + payload->fragment_jc, + payload->tiler_heap_free, + payload->fragment_hierarchy_mask, + payload->tiler_hierarchy_mask, + payload->hierarchy_default_weight, + payload->tiler_core_req, + payload->fragment_core_req); + payload_dump(kctx, payload); +#endif + t_atom->core_req = payload->tiler_core_req | BASEP_JD_REQ_EVENT_NEVER; + f_atom->core_req = payload->fragment_core_req | BASEP_JD_REQ_EVENT_NEVER; + + /* Sanity check core requirements*/ + if ((t_atom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_T || + (f_atom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_FS || + t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES || + f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + + int t_atom_type = t_atom->core_req & BASE_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP; + int f_atom_type = f_atom->core_req & BASE_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP & ~BASE_JD_REQ_FS_AFBC; + int t_has_ex_res = t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES; + int f_has_ex_res = f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES; + + if (t_atom_type != BASE_JD_REQ_T) { + dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom not a tiler job. Was: 0x%x\n Expected: 0x%x", + t_atom_type, BASE_JD_REQ_T); + } + if (f_atom_type != BASE_JD_REQ_FS) { + dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom not a fragment shader. Was 0x%x Expected: 0x%x\n", + f_atom_type, BASE_JD_REQ_FS); + } + if (t_has_ex_res) { + dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom has external resources.\n"); + } + if (f_has_ex_res) { + dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom has external resources.\n"); + } + + goto out; + } + + /* Process tiler job chains */ + next = payload->tiler_jc_list; + if (!next) { + dev_err(kctx->kbdev->dev, "Invalid tiler JC list\n"); + goto out; + } + + while (next) { + base_jd_replay_jc *jc_struct; + struct kbase_vmap_struct jc_map; + u64 jc; + + jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &jc_map); + + if (!jc_struct) { + dev_err(kctx->kbdev->dev, "Failed to map jc struct\n"); + goto out; + } + + jc = jc_struct->jc; + next = jc_struct->next; + if (next) + jc_struct->jc = 0; + + kbase_vunmap(kctx, &jc_map); + + if (jc) { + u16 max_hw_job_id = 0; + + if (kbasep_replay_find_hw_job_id(kctx, jc, + &max_hw_job_id) != 0) + goto out; + + if (kbasep_replay_parse_jc(kctx, jc, prev_jc, + payload->tiler_heap_free, + payload->tiler_hierarchy_mask, + payload->hierarchy_default_weight, + hw_job_id_offset, false) != 0) { + goto out; + } + + hw_job_id_offset += max_hw_job_id; + + prev_jc = jc; + } + } + t_atom->jc = prev_jc; + + /* Process fragment job chain */ + f_atom->jc = payload->fragment_jc; + if (kbasep_replay_parse_jc(kctx, payload->fragment_jc, 0, + payload->tiler_heap_free, + payload->fragment_hierarchy_mask, + payload->hierarchy_default_weight, 0, + true) != 0) { + goto out; + } + + if (!t_atom->jc || !f_atom->jc) { + dev_err(kctx->kbdev->dev, "Invalid payload\n"); + goto out; + } + + dev_dbg(kctx->kbdev->dev, "t_atom->jc=%llx f_atom->jc=%llx\n", + t_atom->jc, f_atom->jc); + ret = 0; + +out: + kbase_vunmap(kctx, &map); + + return ret; +} + +static void kbase_replay_process_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom; + struct kbase_context *kctx; + struct kbase_jd_context *jctx; + bool need_to_try_schedule_context = false; + + struct base_jd_atom_v2 t_atom, f_atom; + struct kbase_jd_atom *t_katom, *f_katom; + base_jd_prio atom_prio; + + katom = container_of(data, struct kbase_jd_atom, work); + kctx = katom->kctx; + jctx = &kctx->jctx; + + mutex_lock(&jctx->lock); + + atom_prio = kbasep_js_sched_prio_to_atom_prio(katom->sched_priority); + + if (kbasep_replay_create_atoms( + kctx, &t_atom, &f_atom, atom_prio) != 0) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + goto out; + } + + t_katom = &jctx->atoms[t_atom.atom_number]; + f_katom = &jctx->atoms[f_atom.atom_number]; + + if (kbasep_replay_parse_payload(kctx, katom, &t_atom, &f_atom) != 0) { + kbasep_release_katom(kctx, t_atom.atom_number); + kbasep_release_katom(kctx, f_atom.atom_number); + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + goto out; + } + + kbasep_replay_reset_softjob(katom, f_katom); + + need_to_try_schedule_context |= jd_submit_atom(kctx, &t_atom, t_katom); + if (t_katom->event_code == BASE_JD_EVENT_JOB_INVALID) { + dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n"); + kbasep_release_katom(kctx, f_atom.atom_number); + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + goto out; + } + need_to_try_schedule_context |= jd_submit_atom(kctx, &f_atom, f_katom); + if (f_katom->event_code == BASE_JD_EVENT_JOB_INVALID) { + dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n"); + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + goto out; + } + + katom->event_code = BASE_JD_EVENT_DONE; + +out: + if (katom->event_code != BASE_JD_EVENT_DONE) { + kbase_disjoint_state_down(kctx->kbdev); + + need_to_try_schedule_context |= jd_done_nolock(katom, NULL); + } + + if (need_to_try_schedule_context) + kbase_js_sched_all(kctx->kbdev); + + mutex_unlock(&jctx->lock); +} + +/** + * @brief Check job replay fault + * + * This will read the job payload, checks fault type and source, then decides + * whether replay is required. + * + * @param[in] katom The atom to be processed + * @return true (success) if replay required or false on failure. + */ +static bool kbase_replay_fault_check(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct device *dev = kctx->kbdev->dev; + base_jd_replay_payload *payload; + u64 job_header; + u64 job_loop_detect; + struct job_descriptor_header *job; + struct kbase_vmap_struct job_map; + struct kbase_vmap_struct map; + bool err = false; + + /* Replay job if fault is of type BASE_JD_EVENT_JOB_WRITE_FAULT or + * if force_replay is enabled. + */ + if (BASE_JD_EVENT_TERMINATED == katom->event_code) { + return false; + } else if (BASE_JD_EVENT_JOB_WRITE_FAULT == katom->event_code) { + return true; + } else if (BASE_JD_EVENT_FORCE_REPLAY == katom->event_code) { + katom->event_code = BASE_JD_EVENT_DATA_INVALID_FAULT; + return true; + } else if (BASE_JD_EVENT_DATA_INVALID_FAULT != katom->event_code) { + /* No replay for faults of type other than + * BASE_JD_EVENT_DATA_INVALID_FAULT. + */ + return false; + } + + /* Job fault is BASE_JD_EVENT_DATA_INVALID_FAULT, now scan fragment jc + * to find out whether the source of exception is POLYGON_LIST. Replay + * is required if the source of fault is POLYGON_LIST. + */ + payload = kbase_vmap(kctx, katom->jc, sizeof(*payload), &map); + if (!payload) { + dev_err(dev, "kbase_replay_fault_check: failed to map payload.\n"); + return false; + } + +#ifdef CONFIG_MALI_BIFROST_DEBUG + dev_dbg(dev, "kbase_replay_fault_check: payload=%p\n", payload); + dev_dbg(dev, "\nPayload structure:\n" + "fragment_jc = 0x%llx\n" + "fragment_hierarchy_mask = 0x%x\n" + "fragment_core_req = 0x%x\n", + payload->fragment_jc, + payload->fragment_hierarchy_mask, + payload->fragment_core_req); +#endif + /* Process fragment job chain */ + job_header = (u64) payload->fragment_jc; + job_loop_detect = job_header; + while (job_header) { + job = kbase_vmap(kctx, job_header, sizeof(*job), &job_map); + if (!job) { + dev_err(dev, "failed to map jc\n"); + /* unmap payload*/ + kbase_vunmap(kctx, &map); + return false; + } + + + dump_job_head(kctx, "\njob_head structure:\n", job); + + /* Replay only when the polygon list reader caused the + * DATA_INVALID_FAULT */ + if ((BASE_JD_EVENT_DATA_INVALID_FAULT == katom->event_code) && + (JOB_POLYGON_LIST == JOB_SOURCE_ID(job->exception_status))) { + err = true; + kbase_vunmap(kctx, &job_map); + break; + } + + /* Move on to next fragment job in the list */ + if (job->job_descriptor_size) + job_header = job->next_job._64; + else + job_header = job->next_job._32; + + kbase_vunmap(kctx, &job_map); + + /* Job chain loop detected */ + if (job_header == job_loop_detect) + break; + } + + /* unmap payload*/ + kbase_vunmap(kctx, &map); + + return err; +} + + +/** + * @brief Process a replay job + * + * Called from kbase_process_soft_job. + * + * On exit, if the job has completed, katom->event_code will have been updated. + * If the job has not completed, and is replaying jobs, then the atom status + * will have been reset to KBASE_JD_ATOM_STATE_QUEUED. + * + * @param[in] katom The atom to be processed + * @return false if the atom has completed + * true if the atom is replaying jobs + */ +bool kbase_replay_process(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + + /* Don't replay this atom if these issues are not present in the + * hardware */ + if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11020) && + !kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11024)) { + dev_dbg(kbdev->dev, "Hardware does not need replay workaround"); + + /* Signal failure to userspace */ + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + + return false; + } + + if (katom->event_code == BASE_JD_EVENT_DONE) { + dev_dbg(kbdev->dev, "Previous job succeeded - not replaying\n"); + + if (katom->retry_count) + kbase_disjoint_state_down(kbdev); + + return false; + } + + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + dev_dbg(kbdev->dev, "Not replaying; context is dying\n"); + + if (katom->retry_count) + kbase_disjoint_state_down(kbdev); + + return false; + } + + /* Check job exception type and source before replaying. */ + if (!kbase_replay_fault_check(katom)) { + dev_dbg(kbdev->dev, + "Replay cancelled on event %x\n", katom->event_code); + /* katom->event_code is already set to the failure code of the + * previous job. + */ + return false; + } + + dev_warn(kbdev->dev, "Replaying jobs retry=%d\n", + katom->retry_count); + + katom->retry_count++; + + if (katom->retry_count > BASEP_JD_REPLAY_LIMIT) { + dev_err(kbdev->dev, "Replay exceeded limit - failing jobs\n"); + + kbase_disjoint_state_down(kbdev); + + /* katom->event_code is already set to the failure code of the + previous job */ + return false; + } + + /* only enter the disjoint state once for the whole time while the replay is ongoing */ + if (katom->retry_count == 1) + kbase_disjoint_state_up(kbdev); + + INIT_WORK(&katom->work, kbase_replay_process_worker); + queue_work(kctx->event_workq, &katom->work); + + return true; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.c new file mode 100755 index 000000000000..43175c85988f --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.c @@ -0,0 +1,74 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifdef CONFIG_ARM64 + +#include +#include + +#include + +static noinline u64 invoke_smc_fid(u64 function_id, + u64 arg0, u64 arg1, u64 arg2) +{ + register u64 x0 asm("x0") = function_id; + register u64 x1 asm("x1") = arg0; + register u64 x2 asm("x2") = arg1; + register u64 x3 asm("x3") = arg2; + + asm volatile( + __asmeq("%0", "x0") + __asmeq("%1", "x1") + __asmeq("%2", "x2") + __asmeq("%3", "x3") + "smc #0\n" + : "+r" (x0) + : "r" (x1), "r" (x2), "r" (x3)); + + return x0; +} + +u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2) +{ + /* Is fast call (bit 31 set) */ + KBASE_DEBUG_ASSERT(fid & ~SMC_FAST_CALL); + /* bits 16-23 must be zero for fast calls */ + KBASE_DEBUG_ASSERT((fid & (0xFF << 16)) == 0); + + return invoke_smc_fid(fid, arg0, arg1, arg2); +} + +u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, + u64 arg0, u64 arg1, u64 arg2) +{ + u32 fid = 0; + + /* Only the six bits allowed should be used. */ + KBASE_DEBUG_ASSERT((oen & ~SMC_OEN_MASK) == 0); + + fid |= SMC_FAST_CALL; /* Bit 31: Fast call */ + if (smc64) + fid |= SMC_64; /* Bit 30: 1=SMC64, 0=SMC32 */ + fid |= oen; /* Bit 29:24: OEN */ + /* Bit 23:16: Must be zero for fast calls */ + fid |= (function_number); /* Bit 15:0: function number */ + + return kbase_invoke_smc_fid(fid, arg0, arg1, arg2); +} + +#endif /* CONFIG_ARM64 */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.h new file mode 100755 index 000000000000..9bff3d2e8b4d --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_smc.h @@ -0,0 +1,67 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_SMC_H_ +#define _KBASE_SMC_H_ + +#ifdef CONFIG_ARM64 + +#include + +#define SMC_FAST_CALL (1 << 31) +#define SMC_64 (1 << 30) + +#define SMC_OEN_OFFSET 24 +#define SMC_OEN_MASK (0x3F << SMC_OEN_OFFSET) /* 6 bits */ +#define SMC_OEN_SIP (2 << SMC_OEN_OFFSET) +#define SMC_OEN_STD (4 << SMC_OEN_OFFSET) + + +/** + * kbase_invoke_smc_fid - Perform a secure monitor call + * @fid: The SMC function to call, see SMC Calling convention. + * @arg0: First argument to the SMC. + * @arg1: Second argument to the SMC. + * @arg2: Third argument to the SMC. + * + * See SMC Calling Convention for details. + * + * Return: the return value from the SMC. + */ +u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2); + +/** + * kbase_invoke_smc_fid - Perform a secure monitor call + * @oen: Owning Entity number (SIP, STD etc). + * @function_number: The function number within the OEN. + * @smc64: use SMC64 calling convention instead of SMC32. + * @arg0: First argument to the SMC. + * @arg1: Second argument to the SMC. + * @arg2: Third argument to the SMC. + * + * See SMC Calling Convention for details. + * + * Return: the return value from the SMC call. + */ +u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, + u64 arg0, u64 arg1, u64 arg2); + +#endif /* CONFIG_ARM64 */ + +#endif /* _KBASE_SMC_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_softjobs.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_softjobs.c new file mode 100755 index 000000000000..c24b94e0d6eb --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_softjobs.c @@ -0,0 +1,1513 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include + +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include +#include +#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @file mali_kbase_softjobs.c + * + * This file implements the logic behind software only jobs that are + * executed within the driver rather than being handed over to the GPU. + */ + +static void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + unsigned long lflags; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_add_tail(&katom->queue, &kctx->waiting_soft_jobs); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + unsigned long lflags; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_del(&katom->queue); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +static void kbasep_add_waiting_with_timeout(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + /* Record the start time of this atom so we could cancel it at + * the right time. + */ + katom->start_timestamp = ktime_get(); + + /* Add the atom to the waiting list before the timer is + * (re)started to make sure that it gets processed. + */ + kbasep_add_waiting_soft_job(katom); + + /* Schedule timeout of this atom after a period if it is not active */ + if (!timer_pending(&kctx->soft_job_timeout)) { + int timeout_ms = atomic_read( + &kctx->kbdev->js_data.soft_job_timeout_ms); + mod_timer(&kctx->soft_job_timeout, + jiffies + msecs_to_jiffies(timeout_ms)); + } +} + +static int kbasep_read_soft_event_status( + struct kbase_context *kctx, u64 evt, unsigned char *status) +{ + unsigned char *mapped_evt; + struct kbase_vmap_struct map; + + mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); + if (!mapped_evt) + return -EFAULT; + + *status = *mapped_evt; + + kbase_vunmap(kctx, &map); + + return 0; +} + +static int kbasep_write_soft_event_status( + struct kbase_context *kctx, u64 evt, unsigned char new_status) +{ + unsigned char *mapped_evt; + struct kbase_vmap_struct map; + + if ((new_status != BASE_JD_SOFT_EVENT_SET) && + (new_status != BASE_JD_SOFT_EVENT_RESET)) + return -EINVAL; + + mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); + if (!mapped_evt) + return -EFAULT; + + *mapped_evt = new_status; + + kbase_vunmap(kctx, &map); + + return 0; +} + +static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) +{ + struct kbase_vmap_struct map; + void *user_result; + struct timespec ts; + struct base_dump_cpu_gpu_counters data; + u64 system_time; + u64 cycle_counter; + u64 jc = katom->jc; + struct kbase_context *kctx = katom->kctx; + int pm_active_err; + + memset(&data, 0, sizeof(data)); + + /* Take the PM active reference as late as possible - otherwise, it could + * delay suspend until we process the atom (which may be at the end of a + * long chain of dependencies */ + pm_active_err = kbase_pm_context_active_handle_suspend(kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); + if (pm_active_err) { + struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; + + /* We're suspended - queue this on the list of suspended jobs + * Use dep_item[1], because dep_item[0] was previously in use + * for 'waiting_soft_jobs'. + */ + mutex_lock(&js_devdata->runpool_mutex); + list_add_tail(&katom->dep_item[1], &js_devdata->suspended_soft_jobs_list); + mutex_unlock(&js_devdata->runpool_mutex); + + /* Also adding this to the list of waiting soft job */ + kbasep_add_waiting_soft_job(katom); + + return pm_active_err; + } + + kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, + &ts); + + kbase_pm_context_idle(kctx->kbdev); + + data.sec = ts.tv_sec; + data.usec = ts.tv_nsec / 1000; + data.system_time = system_time; + data.cycle_counter = cycle_counter; + + /* Assume this atom will be cancelled until we know otherwise */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* GPU_WR access is checked on the range for returning the result to + * userspace for the following reasons: + * - security, this is currently how imported user bufs are checked. + * - userspace ddk guaranteed to assume region was mapped as GPU_WR */ + user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); + if (!user_result) + return 0; + + memcpy(user_result, &data, sizeof(data)); + + kbase_vunmap(kctx, &map); + + /* Atom was fine - mark it as done */ + katom->event_code = BASE_JD_EVENT_DONE; + + return 0; +} + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +/* Called by the explicit fence mechanism when a fence wait has completed */ +void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + mutex_lock(&kctx->jctx.lock); + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(kctx->kbdev); + mutex_unlock(&kctx->jctx.lock); +} +#endif + +static void kbasep_soft_event_complete_job(struct work_struct *work) +{ + struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + int resched; + + mutex_lock(&kctx->jctx.lock); + resched = jd_done_nolock(katom, NULL); + mutex_unlock(&kctx->jctx.lock); + + if (resched) + kbase_js_sched_all(kctx->kbdev); +} + +void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt) +{ + int cancel_timer = 1; + struct list_head *entry, *tmp; + unsigned long lflags; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { + struct kbase_jd_atom *katom = list_entry( + entry, struct kbase_jd_atom, queue); + + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_EVENT_WAIT: + if (katom->jc == evt) { + list_del(&katom->queue); + + katom->event_code = BASE_JD_EVENT_DONE; + INIT_WORK(&katom->work, + kbasep_soft_event_complete_job); + queue_work(kctx->jctx.job_done_wq, + &katom->work); + } else { + /* There are still other waiting jobs, we cannot + * cancel the timer yet. + */ + cancel_timer = 0; + } + break; +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG + case BASE_JD_REQ_SOFT_FENCE_WAIT: + /* Keep the timer running if fence debug is enabled and + * there are waiting fence jobs. + */ + cancel_timer = 0; + break; +#endif + } + } + + if (cancel_timer) + del_timer(&kctx->soft_job_timeout); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG +static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct device *dev = kctx->kbdev->dev; + int i; + + for (i = 0; i < 2; i++) { + struct kbase_jd_atom *dep; + + list_for_each_entry(dep, &katom->dep_head[i], dep_item[i]) { + if (dep->status == KBASE_JD_ATOM_STATE_UNUSED || + dep->status == KBASE_JD_ATOM_STATE_COMPLETED) + continue; + + if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) + == BASE_JD_REQ_SOFT_FENCE_TRIGGER) { + /* Found blocked trigger fence. */ + struct kbase_sync_fence_info info; + + if (!kbase_sync_fence_in_info_get(dep, &info)) { + dev_warn(dev, + "\tVictim trigger atom %d fence [%p] %s: %s\n", + kbase_jd_atom_id(kctx, dep), + info.fence, + info.name, + kbase_sync_status_string(info.status)); + } + } + + kbase_fence_debug_check_atom(dep); + } + } +} + +static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct device *dev = katom->kctx->kbdev->dev; + int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); + unsigned long lflags; + struct kbase_sync_fence_info info; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + + if (kbase_sync_fence_in_info_get(katom, &info)) { + /* Fence must have signaled just after timeout. */ + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); + return; + } + + dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%p] after %dms\n", + kctx->tgid, kctx->id, + kbase_jd_atom_id(kctx, katom), + info.fence, timeout_ms); + dev_warn(dev, "\tGuilty fence [%p] %s: %s\n", + info.fence, info.name, + kbase_sync_status_string(info.status)); + + /* Search for blocked trigger atoms */ + kbase_fence_debug_check_atom(katom); + + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); + + kbase_sync_fence_in_dump(katom); +} + +struct kbase_fence_debug_work { + struct kbase_jd_atom *katom; + struct work_struct work; +}; + +static void kbase_fence_debug_wait_timeout_worker(struct work_struct *work) +{ + struct kbase_fence_debug_work *w = container_of(work, + struct kbase_fence_debug_work, work); + struct kbase_jd_atom *katom = w->katom; + struct kbase_context *kctx = katom->kctx; + + mutex_lock(&kctx->jctx.lock); + kbase_fence_debug_wait_timeout(katom); + mutex_unlock(&kctx->jctx.lock); + + kfree(w); +} + +static void kbase_fence_debug_timeout(struct kbase_jd_atom *katom) +{ + struct kbase_fence_debug_work *work; + struct kbase_context *kctx = katom->kctx; + + /* Enqueue fence debug worker. Use job_done_wq to get + * debug print ordered with job completion. + */ + work = kzalloc(sizeof(struct kbase_fence_debug_work), GFP_ATOMIC); + /* Ignore allocation failure. */ + if (work) { + work->katom = katom; + INIT_WORK(&work->work, kbase_fence_debug_wait_timeout_worker); + queue_work(kctx->jctx.job_done_wq, &work->work); + } +} +#endif /* CONFIG_MALI_BIFROST_FENCE_DEBUG */ + +void kbasep_soft_job_timeout_worker(struct timer_list *t) +{ + struct kbase_context *kctx = from_timer(kctx, t, soft_job_timeout); + u32 timeout_ms = (u32)atomic_read( + &kctx->kbdev->js_data.soft_job_timeout_ms); + struct timer_list *timer = &kctx->soft_job_timeout; + ktime_t cur_time = ktime_get(); + bool restarting = false; + unsigned long lflags; + struct list_head *entry, *tmp; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { + struct kbase_jd_atom *katom = list_entry(entry, + struct kbase_jd_atom, queue); + s64 elapsed_time = ktime_to_ms(ktime_sub(cur_time, + katom->start_timestamp)); + + if (elapsed_time < (s64)timeout_ms) { + restarting = true; + continue; + } + + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_EVENT_WAIT: + /* Take it out of the list to ensure that it + * will be cancelled in all cases + */ + list_del(&katom->queue); + + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + INIT_WORK(&katom->work, kbasep_soft_event_complete_job); + queue_work(kctx->jctx.job_done_wq, &katom->work); + break; +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG + case BASE_JD_REQ_SOFT_FENCE_WAIT: + kbase_fence_debug_timeout(katom); + break; +#endif + } + } + + if (restarting) + mod_timer(timer, jiffies + msecs_to_jiffies(timeout_ms)); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +static int kbasep_soft_event_wait(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + unsigned char status; + + /* The status of this soft-job is stored in jc */ + if (kbasep_read_soft_event_status(kctx, katom->jc, &status)) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + return 0; + } + + if (status == BASE_JD_SOFT_EVENT_SET) + return 0; /* Event already set, nothing to do */ + + kbasep_add_waiting_with_timeout(katom); + + return 1; +} + +static void kbasep_soft_event_update_locked(struct kbase_jd_atom *katom, + unsigned char new_status) +{ + /* Complete jobs waiting on the same event */ + struct kbase_context *kctx = katom->kctx; + + if (kbasep_write_soft_event_status(kctx, katom->jc, new_status) != 0) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + return; + } + + if (new_status == BASE_JD_SOFT_EVENT_SET) + kbasep_complete_triggered_soft_events(kctx, katom->jc); +} + +/** + * kbase_soft_event_update() - Update soft event state + * @kctx: Pointer to context + * @event: Event to update + * @new_status: New status value of event + * + * Update the event, and wake up any atoms waiting for the event. + * + * Return: 0 on success, a negative error code on failure. + */ +int kbase_soft_event_update(struct kbase_context *kctx, + u64 event, + unsigned char new_status) +{ + int err = 0; + + mutex_lock(&kctx->jctx.lock); + + if (kbasep_write_soft_event_status(kctx, event, new_status)) { + err = -ENOENT; + goto out; + } + + if (new_status == BASE_JD_SOFT_EVENT_SET) + kbasep_complete_triggered_soft_events(kctx, event); + +out: + mutex_unlock(&kctx->jctx.lock); + + return err; +} + +static void kbasep_soft_event_cancel_job(struct kbase_jd_atom *katom) +{ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +struct kbase_debug_copy_buffer { + size_t size; + struct page **pages; + int nr_pages; + size_t offset; + struct kbase_mem_phy_alloc *gpu_alloc; + + struct page **extres_pages; + int nr_extres_pages; +}; + +static inline void free_user_buffer(struct kbase_debug_copy_buffer *buffer) +{ + struct page **pages = buffer->extres_pages; + int nr_pages = buffer->nr_extres_pages; + + if (pages) { + int i; + + for (i = 0; i < nr_pages; i++) { + struct page *pg = pages[i]; + + if (pg) + put_page(pg); + } + kfree(pages); + } +} + +static void kbase_debug_copy_finish(struct kbase_jd_atom *katom) +{ + struct kbase_debug_copy_buffer *buffers = + (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc; + unsigned int i; + unsigned int nr = katom->nr_extres; + + if (!buffers) + return; + + kbase_gpu_vm_lock(katom->kctx); + for (i = 0; i < nr; i++) { + int p; + struct kbase_mem_phy_alloc *gpu_alloc = buffers[i].gpu_alloc; + + if (!buffers[i].pages) + break; + for (p = 0; p < buffers[i].nr_pages; p++) { + struct page *pg = buffers[i].pages[p]; + + if (pg) + put_page(pg); + } + kfree(buffers[i].pages); + if (gpu_alloc) { + switch (gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + { + free_user_buffer(&buffers[i]); + break; + } + default: + /* Nothing to be done. */ + break; + } + kbase_mem_phy_alloc_put(gpu_alloc); + } + } + kbase_gpu_vm_unlock(katom->kctx); + kfree(buffers); + + katom->jc = 0; +} + +static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) +{ + struct kbase_debug_copy_buffer *buffers; + struct base_jd_debug_copy_buffer *user_buffers = NULL; + unsigned int i; + unsigned int nr = katom->nr_extres; + int ret = 0; + void __user *user_structs = (void __user *)(uintptr_t)katom->jc; + + if (!user_structs) + return -EINVAL; + + buffers = kcalloc(nr, sizeof(*buffers), GFP_KERNEL); + if (!buffers) { + ret = -ENOMEM; + katom->jc = 0; + goto out_cleanup; + } + katom->jc = (u64)(uintptr_t)buffers; + + user_buffers = kmalloc_array(nr, sizeof(*user_buffers), GFP_KERNEL); + + if (!user_buffers) { + ret = -ENOMEM; + goto out_cleanup; + } + + ret = copy_from_user(user_buffers, user_structs, + sizeof(*user_buffers)*nr); + if (ret) { + ret = -EFAULT; + goto out_cleanup; + } + + for (i = 0; i < nr; i++) { + u64 addr = user_buffers[i].address; + u64 page_addr = addr & PAGE_MASK; + u64 end_page_addr = addr + user_buffers[i].size - 1; + u64 last_page_addr = end_page_addr & PAGE_MASK; + int nr_pages = (last_page_addr-page_addr)/PAGE_SIZE+1; + int pinned_pages; + struct kbase_va_region *reg; + struct base_external_resource user_extres; + + if (!addr) + continue; + + buffers[i].nr_pages = nr_pages; + buffers[i].offset = addr & ~PAGE_MASK; + if (buffers[i].offset >= PAGE_SIZE) { + ret = -EINVAL; + goto out_cleanup; + } + buffers[i].size = user_buffers[i].size; + + buffers[i].pages = kcalloc(nr_pages, sizeof(struct page *), + GFP_KERNEL); + if (!buffers[i].pages) { + ret = -ENOMEM; + goto out_cleanup; + } + + pinned_pages = get_user_pages_fast(page_addr, + nr_pages, + 1, /* Write */ + buffers[i].pages); + if (pinned_pages < 0) { + ret = pinned_pages; + goto out_cleanup; + } + if (pinned_pages != nr_pages) { + ret = -EINVAL; + goto out_cleanup; + } + + user_extres = user_buffers[i].extres; + if (user_extres.ext_resource == 0ULL) { + ret = -EINVAL; + goto out_cleanup; + } + + kbase_gpu_vm_lock(katom->kctx); + reg = kbase_region_tracker_find_region_enclosing_address( + katom->kctx, user_extres.ext_resource & + ~BASE_EXT_RES_ACCESS_EXCLUSIVE); + + if (NULL == reg || NULL == reg->gpu_alloc || + (reg->flags & KBASE_REG_FREE)) { + ret = -EINVAL; + goto out_unlock; + } + + buffers[i].gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + buffers[i].nr_extres_pages = reg->nr_pages; + + if (reg->nr_pages*PAGE_SIZE != buffers[i].size) + dev_warn(katom->kctx->kbdev->dev, "Copy buffer is not of same size as the external resource to copy.\n"); + + switch (reg->gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + { + struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; + unsigned long nr_pages = + alloc->imported.user_buf.nr_pages; + + if (alloc->imported.user_buf.mm != current->mm) { + ret = -EINVAL; + goto out_unlock; + } + buffers[i].extres_pages = kcalloc(nr_pages, + sizeof(struct page *), GFP_KERNEL); + if (!buffers[i].extres_pages) { + ret = -ENOMEM; + goto out_unlock; + } + + ret = get_user_pages_fast( + alloc->imported.user_buf.address, + nr_pages, 0, + buffers[i].extres_pages); + if (ret != nr_pages) + goto out_unlock; + ret = 0; + break; + } + case KBASE_MEM_TYPE_IMPORTED_UMP: + { + dev_warn(katom->kctx->kbdev->dev, + "UMP is not supported for debug_copy jobs\n"); + ret = -EINVAL; + goto out_unlock; + } + default: + /* Nothing to be done. */ + break; + } + kbase_gpu_vm_unlock(katom->kctx); + } + kfree(user_buffers); + + return ret; + +out_unlock: + kbase_gpu_vm_unlock(katom->kctx); + +out_cleanup: + /* Frees allocated memory for kbase_debug_copy_job struct, including + * members, and sets jc to 0 */ + kbase_debug_copy_finish(katom); + kfree(user_buffers); + + return ret; +} + +static void kbase_mem_copy_from_extres_page(struct kbase_context *kctx, + void *extres_page, struct page **pages, unsigned int nr_pages, + unsigned int *target_page_nr, size_t offset, size_t *to_copy) +{ + void *target_page = kmap(pages[*target_page_nr]); + size_t chunk = PAGE_SIZE-offset; + + lockdep_assert_held(&kctx->reg_lock); + + if (!target_page) { + *target_page_nr += 1; + dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job."); + return; + } + + chunk = min(chunk, *to_copy); + + memcpy(target_page + offset, extres_page, chunk); + *to_copy -= chunk; + + kunmap(pages[*target_page_nr]); + + *target_page_nr += 1; + if (*target_page_nr >= nr_pages) + return; + + target_page = kmap(pages[*target_page_nr]); + if (!target_page) { + *target_page_nr += 1; + dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job."); + return; + } + + KBASE_DEBUG_ASSERT(target_page); + + chunk = min(offset, *to_copy); + memcpy(target_page, extres_page + PAGE_SIZE-offset, chunk); + *to_copy -= chunk; + + kunmap(pages[*target_page_nr]); +} + +static int kbase_mem_copy_from_extres(struct kbase_context *kctx, + struct kbase_debug_copy_buffer *buf_data) +{ + unsigned int i; + unsigned int target_page_nr = 0; + struct page **pages = buf_data->pages; + u64 offset = buf_data->offset; + size_t extres_size = buf_data->nr_extres_pages*PAGE_SIZE; + size_t to_copy = min(extres_size, buf_data->size); + size_t dma_to_copy; + struct kbase_mem_phy_alloc *gpu_alloc = buf_data->gpu_alloc; + int ret = 0; + + KBASE_DEBUG_ASSERT(pages != NULL); + + kbase_gpu_vm_lock(kctx); + if (!gpu_alloc) { + ret = -EINVAL; + goto out_unlock; + } + + switch (gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + { + for (i = 0; i < buf_data->nr_extres_pages; i++) { + struct page *pg = buf_data->extres_pages[i]; + void *extres_page = kmap(pg); + + if (extres_page) + kbase_mem_copy_from_extres_page(kctx, + extres_page, pages, + buf_data->nr_pages, + &target_page_nr, + offset, &to_copy); + + kunmap(pg); + if (target_page_nr >= buf_data->nr_pages) + break; + } + break; + } + break; +#ifdef CONFIG_DMA_SHARED_BUFFER + case KBASE_MEM_TYPE_IMPORTED_UMM: { + struct dma_buf *dma_buf = gpu_alloc->imported.umm.dma_buf; + + KBASE_DEBUG_ASSERT(dma_buf != NULL); + if (dma_buf->size > buf_data->nr_extres_pages * PAGE_SIZE) + dev_warn(kctx->kbdev->dev, "External resources buffer size mismatch"); + + dma_to_copy = min(dma_buf->size, + (size_t)(buf_data->nr_extres_pages * PAGE_SIZE)); + ret = dma_buf_begin_cpu_access(dma_buf, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) + 0, dma_to_copy, +#endif + DMA_FROM_DEVICE); + if (ret) + goto out_unlock; + + for (i = 0; i < dma_to_copy/PAGE_SIZE; i++) { + + void *extres_page = dma_buf_kmap(dma_buf, i); + + if (extres_page) + kbase_mem_copy_from_extres_page(kctx, + extres_page, pages, + buf_data->nr_pages, + &target_page_nr, + offset, &to_copy); + + dma_buf_kunmap(dma_buf, i, extres_page); + if (target_page_nr >= buf_data->nr_pages) + break; + } + dma_buf_end_cpu_access(dma_buf, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) + 0, dma_to_copy, +#endif + DMA_FROM_DEVICE); + break; + } +#endif + default: + ret = -EINVAL; + } +out_unlock: + kbase_gpu_vm_unlock(kctx); + return ret; + +} + +static int kbase_debug_copy(struct kbase_jd_atom *katom) +{ + struct kbase_debug_copy_buffer *buffers = + (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc; + unsigned int i; + + for (i = 0; i < katom->nr_extres; i++) { + int res = kbase_mem_copy_from_extres(katom->kctx, &buffers[i]); + + if (res) + return res; + } + + return 0; +} + +static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) +{ + __user void *data = (__user void *)(uintptr_t) katom->jc; + struct base_jit_alloc_info *info; + struct kbase_context *kctx = katom->kctx; + int ret; + + /* Fail the job if there is no info structure */ + if (!data) { + ret = -EINVAL; + goto fail; + } + + /* Copy the information for safe access and future storage */ + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) { + ret = -ENOMEM; + goto fail; + } + + if (copy_from_user(info, data, sizeof(*info)) != 0) { + ret = -EINVAL; + goto free_info; + } + + /* If the ID is zero then fail the job */ + if (info->id == 0) { + ret = -EINVAL; + goto free_info; + } + + /* Sanity check that the PA fits within the VA */ + if (info->va_pages < info->commit_pages) { + ret = -EINVAL; + goto free_info; + } + + /* Ensure the GPU address is correctly aligned */ + if ((info->gpu_alloc_addr & 0x7) != 0) { + ret = -EINVAL; + goto free_info; + } + + /* Replace the user pointer with our kernel allocated info structure */ + katom->jc = (u64)(uintptr_t) info; + katom->jit_blocked = false; + + lockdep_assert_held(&kctx->jctx.lock); + list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); + + /* + * Note: + * The provided info->gpu_alloc_addr isn't validated here as + * userland can cache allocations which means that even + * though the region is valid it doesn't represent the + * same thing it used to. + * + * Complete validation of va_pages, commit_pages and extent + * isn't done here as it will be done during the call to + * kbase_mem_alloc. + */ + return 0; + +free_info: + kfree(info); +fail: + katom->jc = 0; + return ret; +} + +static u8 kbase_jit_free_get_id(struct kbase_jd_atom *katom) +{ + if (WARN_ON(katom->core_req != BASE_JD_REQ_SOFT_JIT_FREE)) + return 0; + + return (u8) katom->jc; +} + +static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct base_jit_alloc_info *info; + struct kbase_va_region *reg; + struct kbase_vmap_struct mapping; + u64 *ptr, new_addr; + + if (katom->jit_blocked) { + list_del(&katom->queue); + katom->jit_blocked = false; + } + + info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc; + + /* The JIT ID is still in use so fail the allocation */ + if (kctx->jit_alloc[info->id]) { + katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; + return 0; + } + + /* Create a JIT allocation */ + reg = kbase_jit_allocate(kctx, info); + if (!reg) { + struct kbase_jd_atom *jit_atom; + bool can_block = false; + + lockdep_assert_held(&kctx->jctx.lock); + + jit_atom = list_first_entry(&kctx->jit_atoms_head, + struct kbase_jd_atom, jit_node); + + list_for_each_entry(jit_atom, &kctx->jit_atoms_head, jit_node) { + if (jit_atom == katom) + break; + if (jit_atom->core_req == BASE_JD_REQ_SOFT_JIT_FREE) { + u8 free_id = kbase_jit_free_get_id(jit_atom); + + if (free_id && kctx->jit_alloc[free_id]) { + /* A JIT free which is active and + * submitted before this atom + */ + can_block = true; + break; + } + } + } + + if (!can_block) { + /* Mark the allocation so we know it's in use even if + * the allocation itself fails. + */ + kctx->jit_alloc[info->id] = + (struct kbase_va_region *) -1; + + katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; + return 0; + } + + /* There are pending frees for an active allocation + * so we should wait to see whether they free the memory. + * Add to the beginning of the list to ensure that the atom is + * processed only once in kbase_jit_free_finish + */ + list_add(&katom->queue, &kctx->jit_pending_alloc); + katom->jit_blocked = true; + + return 1; + } + + /* + * Write the address of the JIT allocation to the user provided + * GPU allocation. + */ + ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr), + &mapping); + if (!ptr) { + /* + * Leave the allocation "live" as the JIT free jit will be + * submitted anyway. + */ + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return 0; + } + + new_addr = reg->start_pfn << PAGE_SHIFT; + *ptr = new_addr; + KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( + katom, info->gpu_alloc_addr, new_addr); + kbase_vunmap(kctx, &mapping); + + katom->event_code = BASE_JD_EVENT_DONE; + + /* + * Bind it to the user provided ID. Do this last so we can check for + * the JIT free racing this JIT alloc job. + */ + kctx->jit_alloc[info->id] = reg; + + return 0; +} + +static void kbase_jit_allocate_finish(struct kbase_jd_atom *katom) +{ + struct base_jit_alloc_info *info; + + lockdep_assert_held(&katom->kctx->jctx.lock); + + /* Remove atom from jit_atoms_head list */ + list_del(&katom->jit_node); + + if (katom->jit_blocked) { + list_del(&katom->queue); + katom->jit_blocked = false; + } + + info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc; + /* Free the info structure */ + kfree(info); +} + +static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + lockdep_assert_held(&kctx->jctx.lock); + list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); + + return 0; +} + +static void kbase_jit_free_process(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + u8 id = kbase_jit_free_get_id(katom); + + /* + * If the ID is zero or it is not in use yet then fail the job. + */ + if ((id == 0) || (kctx->jit_alloc[id] == NULL)) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return; + } + + /* + * If the ID is valid but the allocation request failed still succeed + * this soft job but don't try and free the allocation. + */ + if (kctx->jit_alloc[id] != (struct kbase_va_region *) -1) + kbase_jit_free(kctx, kctx->jit_alloc[id]); + + kctx->jit_alloc[id] = NULL; +} + +static void kbasep_jit_free_finish_worker(struct work_struct *work) +{ + struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + int resched; + + mutex_lock(&kctx->jctx.lock); + kbase_finish_soft_job(katom); + resched = jd_done_nolock(katom, NULL); + mutex_unlock(&kctx->jctx.lock); + + if (resched) + kbase_js_sched_all(kctx->kbdev); +} + +static void kbase_jit_free_finish(struct kbase_jd_atom *katom) +{ + struct list_head *i, *tmp; + struct kbase_context *kctx = katom->kctx; + + lockdep_assert_held(&kctx->jctx.lock); + /* Remove this atom from the kctx->jit_atoms_head list */ + list_del(&katom->jit_node); + + list_for_each_safe(i, tmp, &kctx->jit_pending_alloc) { + struct kbase_jd_atom *pending_atom = list_entry(i, + struct kbase_jd_atom, queue); + if (kbase_jit_allocate_process(pending_atom) == 0) { + /* Atom has completed */ + INIT_WORK(&pending_atom->work, + kbasep_jit_free_finish_worker); + queue_work(kctx->jctx.job_done_wq, &pending_atom->work); + } + } +} + +static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) +{ + __user struct base_external_resource_list *user_ext_res; + struct base_external_resource_list *ext_res; + u64 count = 0; + size_t copy_size; + int ret; + + user_ext_res = (__user struct base_external_resource_list *) + (uintptr_t) katom->jc; + + /* Fail the job if there is no info structure */ + if (!user_ext_res) { + ret = -EINVAL; + goto fail; + } + + if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) { + ret = -EINVAL; + goto fail; + } + + /* Is the number of external resources in range? */ + if (!count || count > BASE_EXT_RES_COUNT_MAX) { + ret = -EINVAL; + goto fail; + } + + /* Copy the information for safe access and future storage */ + copy_size = sizeof(*ext_res); + copy_size += sizeof(struct base_external_resource) * (count - 1); + ext_res = kzalloc(copy_size, GFP_KERNEL); + if (!ext_res) { + ret = -ENOMEM; + goto fail; + } + + if (copy_from_user(ext_res, user_ext_res, copy_size) != 0) { + ret = -EINVAL; + goto free_info; + } + + /* + * Overwrite the count with the first value incase it was changed + * after the fact. + */ + ext_res->count = count; + + /* + * Replace the user pointer with our kernel allocated + * ext_res structure. + */ + katom->jc = (u64)(uintptr_t) ext_res; + + return 0; + +free_info: + kfree(ext_res); +fail: + return ret; +} + +static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) +{ + struct base_external_resource_list *ext_res; + int i; + bool failed = false; + + ext_res = (struct base_external_resource_list *) (uintptr_t) katom->jc; + if (!ext_res) + goto failed_jc; + + kbase_gpu_vm_lock(katom->kctx); + + for (i = 0; i < ext_res->count; i++) { + u64 gpu_addr; + + gpu_addr = ext_res->ext_res[i].ext_resource & + ~BASE_EXT_RES_ACCESS_EXCLUSIVE; + if (map) { + if (!kbase_sticky_resource_acquire(katom->kctx, + gpu_addr)) + goto failed_loop; + } else + if (!kbase_sticky_resource_release(katom->kctx, NULL, + gpu_addr)) + failed = true; + } + + /* + * In the case of unmap we continue unmapping other resources in the + * case of failure but will always report failure if _any_ unmap + * request fails. + */ + if (failed) + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + else + katom->event_code = BASE_JD_EVENT_DONE; + + kbase_gpu_vm_unlock(katom->kctx); + + return; + +failed_loop: + while (--i > 0) { + u64 gpu_addr; + + gpu_addr = ext_res->ext_res[i].ext_resource & + ~BASE_EXT_RES_ACCESS_EXCLUSIVE; + + kbase_sticky_resource_release(katom->kctx, NULL, gpu_addr); + } + + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + kbase_gpu_vm_unlock(katom->kctx); + +failed_jc: + return; +} + +static void kbase_ext_res_finish(struct kbase_jd_atom *katom) +{ + struct base_external_resource_list *ext_res; + + ext_res = (struct base_external_resource_list *) (uintptr_t) katom->jc; + /* Free the info structure */ + kfree(ext_res); +} + +int kbase_process_soft_job(struct kbase_jd_atom *katom) +{ + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: + return kbase_dump_cpu_gpu_time(katom); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + katom->event_code = kbase_sync_fence_out_trigger(katom, + katom->event_code == BASE_JD_EVENT_DONE ? + 0 : -EFAULT); + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + { + int ret = kbase_sync_fence_in_wait(katom); + + if (ret == 1) { +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG + kbasep_add_waiting_with_timeout(katom); +#else + kbasep_add_waiting_soft_job(katom); +#endif + } + return ret; + } +#endif + + case BASE_JD_REQ_SOFT_REPLAY: + return kbase_replay_process(katom); + case BASE_JD_REQ_SOFT_EVENT_WAIT: + return kbasep_soft_event_wait(katom); + case BASE_JD_REQ_SOFT_EVENT_SET: + kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_SET); + break; + case BASE_JD_REQ_SOFT_EVENT_RESET: + kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_RESET); + break; + case BASE_JD_REQ_SOFT_DEBUG_COPY: + { + int res = kbase_debug_copy(katom); + + if (res) + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + break; + } + case BASE_JD_REQ_SOFT_JIT_ALLOC: + return kbase_jit_allocate_process(katom); + case BASE_JD_REQ_SOFT_JIT_FREE: + kbase_jit_free_process(katom); + break; + case BASE_JD_REQ_SOFT_EXT_RES_MAP: + kbase_ext_res_process(katom, true); + break; + case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: + kbase_ext_res_process(katom, false); + break; + } + + /* Atom is complete */ + return 0; +} + +void kbase_cancel_soft_job(struct kbase_jd_atom *katom) +{ + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_WAIT: + kbase_sync_fence_in_cancel_wait(katom); + break; +#endif + case BASE_JD_REQ_SOFT_EVENT_WAIT: + kbasep_soft_event_cancel_job(katom); + break; + default: + /* This soft-job doesn't support cancellation! */ + KBASE_DEBUG_ASSERT(0); + } +} + +int kbase_prepare_soft_job(struct kbase_jd_atom *katom) +{ + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: + { + if (!IS_ALIGNED(katom->jc, cache_line_size())) + return -EINVAL; + } + break; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + { + struct base_fence fence; + int fd; + + if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) + return -EINVAL; + + fd = kbase_sync_fence_out_create(katom, + fence.basep.stream_fd); + if (fd < 0) + return -EINVAL; + + fence.basep.fd = fd; + if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) { + kbase_sync_fence_out_remove(katom); + kbase_sync_fence_close_fd(fd); + fence.basep.fd = -EINVAL; + return -EINVAL; + } + } + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + { + struct base_fence fence; + int ret; + + if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) + return -EINVAL; + + /* Get a reference to the fence object */ + ret = kbase_sync_fence_in_from_fd(katom, + fence.basep.fd); + if (ret < 0) + return ret; + +#ifdef CONFIG_MALI_BIFROST_DMA_FENCE + /* + * Set KCTX_NO_IMPLICIT_FENCE in the context the first + * time a soft fence wait job is observed. This will + * prevent the implicit dma-buf fence to conflict with + * the Android native sync fences. + */ + if (!kbase_ctx_flag(katom->kctx, KCTX_NO_IMPLICIT_SYNC)) + kbase_ctx_flag_set(katom->kctx, KCTX_NO_IMPLICIT_SYNC); +#endif /* CONFIG_MALI_BIFROST_DMA_FENCE */ + } + break; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + case BASE_JD_REQ_SOFT_JIT_ALLOC: + return kbase_jit_allocate_prepare(katom); + case BASE_JD_REQ_SOFT_REPLAY: + break; + case BASE_JD_REQ_SOFT_JIT_FREE: + return kbase_jit_free_prepare(katom); + case BASE_JD_REQ_SOFT_EVENT_WAIT: + case BASE_JD_REQ_SOFT_EVENT_SET: + case BASE_JD_REQ_SOFT_EVENT_RESET: + if (katom->jc == 0) + return -EINVAL; + break; + case BASE_JD_REQ_SOFT_DEBUG_COPY: + return kbase_debug_copy_prepare(katom); + case BASE_JD_REQ_SOFT_EXT_RES_MAP: + return kbase_ext_res_prepare(katom); + case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: + return kbase_ext_res_prepare(katom); + default: + /* Unsupported soft-job */ + return -EINVAL; + } + return 0; +} + +void kbase_finish_soft_job(struct kbase_jd_atom *katom) +{ + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: + /* Nothing to do */ + break; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + /* If fence has not yet been signaled, do it now */ + kbase_sync_fence_out_trigger(katom, katom->event_code == + BASE_JD_EVENT_DONE ? 0 : -EFAULT); + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + /* Release katom's reference to fence object */ + kbase_sync_fence_in_remove(katom); + break; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + case BASE_JD_REQ_SOFT_DEBUG_COPY: + kbase_debug_copy_finish(katom); + break; + case BASE_JD_REQ_SOFT_JIT_ALLOC: + kbase_jit_allocate_finish(katom); + break; + case BASE_JD_REQ_SOFT_EXT_RES_MAP: + kbase_ext_res_finish(katom); + break; + case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: + kbase_ext_res_finish(katom); + break; + case BASE_JD_REQ_SOFT_JIT_FREE: + kbase_jit_free_finish(katom); + break; + } +} + +void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) +{ + LIST_HEAD(local_suspended_soft_jobs); + struct kbase_jd_atom *tmp_iter; + struct kbase_jd_atom *katom_iter; + struct kbasep_js_device_data *js_devdata; + bool resched = false; + + KBASE_DEBUG_ASSERT(kbdev); + + js_devdata = &kbdev->js_data; + + /* Move out the entire list */ + mutex_lock(&js_devdata->runpool_mutex); + list_splice_init(&js_devdata->suspended_soft_jobs_list, + &local_suspended_soft_jobs); + mutex_unlock(&js_devdata->runpool_mutex); + + /* + * Each atom must be detached from the list and ran separately - + * it could be re-added to the old list, but this is unlikely + */ + list_for_each_entry_safe(katom_iter, tmp_iter, + &local_suspended_soft_jobs, dep_item[1]) { + struct kbase_context *kctx = katom_iter->kctx; + + mutex_lock(&kctx->jctx.lock); + + /* Remove from the global list */ + list_del(&katom_iter->dep_item[1]); + /* Remove from the context's list of waiting soft jobs */ + kbasep_remove_waiting_soft_job(katom_iter); + + if (kbase_process_soft_job(katom_iter) == 0) { + kbase_finish_soft_job(katom_iter); + resched |= jd_done_nolock(katom_iter, NULL); + } else { + KBASE_DEBUG_ASSERT((katom_iter->core_req & + BASE_JD_REQ_SOFT_JOB_TYPE) + != BASE_JD_REQ_SOFT_REPLAY); + } + + mutex_unlock(&kctx->jctx.lock); + } + + if (resched) + kbase_js_sched_all(kbdev); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.c new file mode 100755 index 000000000000..c98762cec244 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.c @@ -0,0 +1,23 @@ + /* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +#include "mali_kbase_strings.h" + +#define KBASE_DRV_NAME "mali" +#define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline" + +const char kbase_drv_name[] = KBASE_DRV_NAME; +const char kbase_timeline_name[] = KBASE_TIMELINE_NAME; diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.h new file mode 100755 index 000000000000..41b8fdbec6a4 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_strings.h @@ -0,0 +1,19 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +extern const char kbase_drv_name[]; +extern const char kbase_timeline_name[]; diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync.h new file mode 100755 index 000000000000..54159262314a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync.h @@ -0,0 +1,203 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_sync.h + * + * This file contains our internal "API" for explicit fences. + * It hides the implementation details of the actual explicit fence mechanism + * used (Android fences or sync file with DMA fences). + */ + +#ifndef MALI_KBASE_SYNC_H +#define MALI_KBASE_SYNC_H + +#include +#ifdef CONFIG_SYNC +#include +#endif +#ifdef CONFIG_SYNC_FILE +#include "mali_kbase_fence_defs.h" +#include +#endif + +#include "mali_kbase.h" + +/** + * struct kbase_sync_fence_info - Information about a fence + * @fence: Pointer to fence (type is void*, as underlaying struct can differ) + * @name: The name given to this fence when it was created + * @status: < 0 means error, 0 means active, 1 means signaled + * + * Use kbase_sync_fence_in_info_get() or kbase_sync_fence_out_info_get() + * to get the information. + */ +struct kbase_sync_fence_info { + void *fence; + char name[32]; + int status; +}; + +/** + * kbase_sync_fence_stream_create() - Create a stream object + * @name: Name of stream (only used to ease debugging/visualization) + * @out_fd: A file descriptor representing the created stream object + * + * Can map down to a timeline implementation in some implementations. + * Exposed as a file descriptor. + * Life-time controlled via the file descriptor: + * - dup to add a ref + * - close to remove a ref + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_stream_create(const char *name, int *const out_fd); + +/** + * kbase_sync_fence_out_create Create an explicit output fence to specified atom + * @katom: Atom to assign the new explicit fence to + * @stream_fd: File descriptor for stream object to create fence on + * + * return: Valid file descriptor to fence or < 0 on error + */ +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); + +/** + * kbase_sync_fence_in_from_fd() Assigns an existing fence to specified atom + * @katom: Atom to assign the existing explicit fence to + * @fd: File descriptor to an existing fence + * + * Assigns an explicit input fence to atom. + * This can later be waited for by calling @kbase_sync_fence_in_wait + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); + +/** + * kbase_sync_fence_validate() - Validate a fd to be a valid fence + * @fd: File descriptor to check + * + * This function is only usable to catch unintentional user errors early, + * it does not stop malicious code changing the fd after this function returns. + * + * return 0: if fd is for a valid fence, < 0 if invalid + */ +int kbase_sync_fence_validate(int fd); + +/** + * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom + * @katom: Atom with an explicit fence to signal + * @result: < 0 means signal with error, 0 >= indicates success + * + * Signal output fence attached on katom and remove the fence from the atom. + * + * return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE + */ +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result); + +/** + * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled + * @katom: Atom with explicit fence to wait for + * + * If the fence is already signaled, then 0 is returned, and the caller must + * continue processing of the katom. + * + * If the fence isn't already signaled, then this kbase_sync framework will + * take responsibility to continue the processing once the fence is signaled. + * + * return: 0 if already signaled, otherwise 1 + */ +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits + * @katom: Atom to cancel wait for + * + * This function is fully responsible for continuing processing of this atom + * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all) + */ +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_in_remove() - Remove the input fence from the katom + * @katom: Atom to remove explicit input fence for + * + * This will also release the corresponding reference. + */ +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_out_remove() - Remove the output fence from the katom + * @katom: Atom to remove explicit output fence for + * + * This will also release the corresponding reference. + */ +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_close_fd() - Close a file descriptor representing a fence + * @fd: File descriptor to close + */ +static inline void kbase_sync_fence_close_fd(int fd) +{ + ksys_close(fd); +} + +/** + * kbase_sync_fence_in_info_get() - Retrieves information about input fence + * @katom: Atom to get fence information from + * @info: Struct to be filled with fence information + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info); + +/** + * kbase_sync_fence_out_info_get() - Retrieves information about output fence + * @katom: Atom to get fence information from + * @info: Struct to be filled with fence information + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info); + +/** + * kbase_sync_status_string() - Get string matching @status + * @status: Value of fence status. + * + * return: Pointer to string describing @status. + */ +const char *kbase_sync_status_string(int status); + +/* + * Internal worker used to continue processing of atom. + */ +void kbase_sync_fence_wait_worker(struct work_struct *data); + +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG +/** + * kbase_sync_fence_in_dump() Trigger a debug dump of atoms input fence state + * @katom: Atom to trigger fence debug dump for + */ +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); +#endif + +#endif /* MALI_KBASE_SYNC_H */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_android.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_android.c new file mode 100755 index 000000000000..e4528e2b9f25 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_android.c @@ -0,0 +1,537 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Code for supporting explicit Android fences (CONFIG_SYNC) + * Known to be good for kernels 4.5 and earlier. + * Replaced with CONFIG_SYNC_FILE for 4.9 and later kernels + * (see mali_kbase_sync_file.c) + */ + +#include +#include +#include +#include +#include +#include +#include +#include "sync.h" +#include +#include + +struct mali_sync_timeline { + struct sync_timeline timeline; + atomic_t counter; + atomic_t signaled; +}; + +struct mali_sync_pt { + struct sync_pt pt; + int order; + int result; +}; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +/* For backwards compatibility with kernels before 3.17. After 3.17 + * sync_pt_parent is included in the kernel. */ +static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) +{ + return pt->parent; +} +#endif + +static struct mali_sync_timeline *to_mali_sync_timeline( + struct sync_timeline *timeline) +{ + return container_of(timeline, struct mali_sync_timeline, timeline); +} + +static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) +{ + return container_of(pt, struct mali_sync_pt, pt); +} + +static struct sync_pt *timeline_dup(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_pt *new_mpt; + struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), + sizeof(struct mali_sync_pt)); + + if (!new_pt) + return NULL; + + new_mpt = to_mali_sync_pt(new_pt); + new_mpt->order = mpt->order; + new_mpt->result = mpt->result; + + return new_pt; +} + +static int timeline_has_signaled(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_timeline *mtl = to_mali_sync_timeline( + sync_pt_parent(pt)); + int result = mpt->result; + + int diff = atomic_read(&mtl->signaled) - mpt->order; + + if (diff >= 0) + return (result < 0) ? result : 1; + + return 0; +} + +static int timeline_compare(struct sync_pt *a, struct sync_pt *b) +{ + struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt); + struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt); + + int diff = ma->order - mb->order; + + if (diff == 0) + return 0; + + return (diff < 0) ? -1 : 1; +} + +static void timeline_value_str(struct sync_timeline *timeline, char *str, + int size) +{ + struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline); + + snprintf(str, size, "%d", atomic_read(&mtl->signaled)); +} + +static void pt_value_str(struct sync_pt *pt, char *str, int size) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + + snprintf(str, size, "%d(%d)", mpt->order, mpt->result); +} + +static struct sync_timeline_ops mali_timeline_ops = { + .driver_name = "Mali", + .dup = timeline_dup, + .has_signaled = timeline_has_signaled, + .compare = timeline_compare, + .timeline_value_str = timeline_value_str, + .pt_value_str = pt_value_str, +}; + +/* Allocates a timeline for Mali + * + * One timeline should be allocated per API context. + */ +static struct sync_timeline *mali_sync_timeline_alloc(const char *name) +{ + struct sync_timeline *tl; + struct mali_sync_timeline *mtl; + + tl = sync_timeline_create(&mali_timeline_ops, + sizeof(struct mali_sync_timeline), name); + if (!tl) + return NULL; + + /* Set the counter in our private struct */ + mtl = to_mali_sync_timeline(tl); + atomic_set(&mtl->counter, 0); + atomic_set(&mtl->signaled, 0); + + return tl; +} + +static int kbase_stream_close(struct inode *inode, struct file *file) +{ + struct sync_timeline *tl; + + tl = (struct sync_timeline *)file->private_data; + sync_timeline_destroy(tl); + return 0; +} + +static const struct file_operations stream_fops = { + .owner = THIS_MODULE, + .release = kbase_stream_close, +}; + +int kbase_sync_fence_stream_create(const char *name, int *const out_fd) +{ + struct sync_timeline *tl; + + if (!out_fd) + return -EINVAL; + + tl = mali_sync_timeline_alloc(name); + if (!tl) + return -EINVAL; + + *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY|O_CLOEXEC); + + if (*out_fd < 0) { + sync_timeline_destroy(tl); + return -EINVAL; + } + + return 0; +} + +/* Allocates a sync point within the timeline. + * + * The timeline must be the one allocated by kbase_sync_timeline_alloc + * + * Sync points must be triggered in *exactly* the same order as they are + * allocated. + */ +static struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent) +{ + struct sync_pt *pt = sync_pt_create(parent, + sizeof(struct mali_sync_pt)); + struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent); + struct mali_sync_pt *mpt; + + if (!pt) + return NULL; + + mpt = to_mali_sync_pt(pt); + mpt->order = atomic_inc_return(&mtl->counter); + mpt->result = 0; + + return pt; +} + +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) +{ + struct sync_timeline *tl; + struct sync_pt *pt; + struct sync_fence *fence; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) + struct files_struct *files; + struct fdtable *fdt; +#endif + int fd; + struct file *tl_file; + + tl_file = fget(tl_fd); + if (tl_file == NULL) + return -EBADF; + + if (tl_file->f_op != &stream_fops) { + fd = -EBADF; + goto out; + } + + tl = tl_file->private_data; + + pt = kbase_sync_pt_alloc(tl); + if (!pt) { + fd = -EFAULT; + goto out; + } + + fence = sync_fence_create("mali_fence", pt); + if (!fence) { + sync_pt_free(pt); + fd = -EFAULT; + goto out; + } + + /* from here the fence owns the sync_pt */ + + /* create a fd representing the fence */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) + fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); + if (fd < 0) { + sync_fence_put(fence); + goto out; + } +#else + fd = get_unused_fd(); + if (fd < 0) { + sync_fence_put(fence); + goto out; + } + + files = current->files; + spin_lock(&files->file_lock); + fdt = files_fdtable(files); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) + __set_close_on_exec(fd, fdt); +#else + FD_SET(fd, fdt->close_on_exec); +#endif + spin_unlock(&files->file_lock); +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ + + /* bind fence to the new fd */ + sync_fence_install(fence, fd); + + katom->fence = sync_fence_fdget(fd); + if (katom->fence == NULL) { + /* The only way the fence can be NULL is if userspace closed it + * for us, so we don't need to clear it up */ + fd = -EINVAL; + goto out; + } + +out: + fput(tl_file); + + return fd; +} + +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) +{ + katom->fence = sync_fence_fdget(fd); + return katom->fence ? 0 : -ENOENT; +} + +int kbase_sync_fence_validate(int fd) +{ + struct sync_fence *fence; + + fence = sync_fence_fdget(fd); + if (!fence) + return -EINVAL; + + sync_fence_put(fence); + return 0; +} + +/* Returns true if the specified timeline is allocated by Mali */ +static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) +{ + return timeline->ops == &mali_timeline_ops; +} + +/* Signals a particular sync point + * + * Sync points must be triggered in *exactly* the same order as they are + * allocated. + * + * If they are signaled in the wrong order then a message will be printed in + * debug builds and otherwise attempts to signal order sync_pts will be ignored. + * + * result can be negative to indicate error, any other value is interpreted as + * success. + */ +static void kbase_sync_signal_pt(struct sync_pt *pt, int result) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_timeline *mtl = to_mali_sync_timeline( + sync_pt_parent(pt)); + int signaled; + int diff; + + mpt->result = result; + + do { + signaled = atomic_read(&mtl->signaled); + + diff = signaled - mpt->order; + + if (diff > 0) { + /* The timeline is already at or ahead of this point. + * This should not happen unless userspace has been + * signaling fences out of order, so warn but don't + * violate the sync_pt API. + * The warning is only in debug builds to prevent + * a malicious user being able to spam dmesg. + */ +#ifdef CONFIG_MALI_BIFROST_DEBUG + pr_err("Fences were triggered in a different order to allocation!"); +#endif /* CONFIG_MALI_BIFROST_DEBUG */ + return; + } + } while (atomic_cmpxchg(&mtl->signaled, + signaled, mpt->order) != signaled); +} + +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) +{ + struct sync_pt *pt; + struct sync_timeline *timeline; + + if (!katom->fence) + return BASE_JD_EVENT_JOB_CANCELLED; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + if (!list_is_singular(&katom->fence->pt_list_head)) { +#else + if (katom->fence->num_fences != 1) { +#endif + /* Not exactly one item in the list - so it didn't (directly) + * come from us */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + pt = list_first_entry(&katom->fence->pt_list_head, + struct sync_pt, pt_list); +#else + pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); +#endif + timeline = sync_pt_parent(pt); + + if (!kbase_sync_timeline_is_ours(timeline)) { + /* Fence has a sync_pt which isn't ours! */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + + kbase_sync_signal_pt(pt, result); + + sync_timeline_signal(timeline); + + kbase_sync_fence_out_remove(katom); + + return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; +} + +static inline int kbase_fence_get_status(struct sync_fence *fence) +{ + if (!fence) + return -ENOENT; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + return fence->status; +#else + return atomic_read(&fence->status); +#endif +} + +static void kbase_fence_wait_callback(struct sync_fence *fence, + struct sync_fence_waiter *waiter) +{ + struct kbase_jd_atom *katom = container_of(waiter, + struct kbase_jd_atom, sync_waiter); + struct kbase_context *kctx = katom->kctx; + + /* Propagate the fence status to the atom. + * If negative then cancel this atom and its dependencies. + */ + if (kbase_fence_get_status(fence) < 0) + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* To prevent a potential deadlock we schedule the work onto the + * job_done_wq workqueue + * + * The issue is that we may signal the timeline while holding + * kctx->jctx.lock and the callbacks are run synchronously from + * sync_timeline_signal. So we simply defer the work. + */ + + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) +{ + int ret; + + sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback); + + ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter); + + if (ret == 1) { + /* Already signaled */ + return 0; + } + + if (ret < 0) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + /* We should cause the dependent jobs in the bag to be failed, + * to do this we schedule the work queue to complete this job */ + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(katom->kctx->jctx.job_done_wq, &katom->work); + } + + return 1; +} + +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) +{ + if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { + /* The wait wasn't cancelled - leave the cleanup for + * kbase_fence_wait_callback */ + return; + } + + /* Wait was cancelled - zap the atoms */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) +{ + if (katom->fence) { + sync_fence_put(katom->fence); + katom->fence = NULL; + } +} + +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) +{ + if (katom->fence) { + sync_fence_put(katom->fence); + katom->fence = NULL; + } +} + +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ + if (!katom->fence) + return -ENOENT; + + info->fence = katom->fence; + info->status = kbase_fence_get_status(katom->fence); + strlcpy(info->name, katom->fence->name, sizeof(info->name)); + + return 0; +} + +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ + if (!katom->fence) + return -ENOENT; + + info->fence = katom->fence; + info->status = kbase_fence_get_status(katom->fence); + strlcpy(info->name, katom->fence->name, sizeof(info->name)); + + return 0; +} + +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) +{ + /* Dump out the full state of all the Android sync fences. + * The function sync_dump() isn't exported to modules, so force + * sync_fence_wait() to time out to trigger sync_dump(). + */ + if (katom->fence) + sync_fence_wait(katom->fence, 1); +} +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_common.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_common.c new file mode 100755 index 000000000000..457def296684 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_common.c @@ -0,0 +1,43 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * @file mali_kbase_sync_common.c + * + * Common code for our explicit fence functionality + */ + +#include +#include "mali_kbase.h" + +void kbase_sync_fence_wait_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom; + + katom = container_of(data, struct kbase_jd_atom, work); + kbase_soft_event_wait_callback(katom); +} + +const char *kbase_sync_status_string(int status) +{ + if (status == 0) + return "signaled"; + else if (status > 0) + return "active"; + else + return "error"; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_file.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_file.c new file mode 100755 index 000000000000..509c0666f10f --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_sync_file.c @@ -0,0 +1,348 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Code for supporting explicit Linux fences (CONFIG_SYNC_FILE) + * Introduced in kernel 4.9. + * Android explicit fences (CONFIG_SYNC) can be used for older kernels + * (see mali_kbase_sync_android.c) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mali_kbase_fence_defs.h" +#include "mali_kbase_sync.h" +#include "mali_kbase_fence.h" +#include "mali_kbase.h" + +static const struct file_operations stream_fops = { + .owner = THIS_MODULE +}; + +int kbase_sync_fence_stream_create(const char *name, int *const out_fd) +{ + if (!out_fd) + return -EINVAL; + + *out_fd = anon_inode_getfd(name, &stream_fops, NULL, + O_RDONLY | O_CLOEXEC); + if (*out_fd < 0) + return -EINVAL; + + return 0; +} + +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + struct sync_file *sync_file; + int fd; + + fence = kbase_fence_out_new(katom); + if (!fence) + return -ENOMEM; + + /* Take an extra reference to the fence on behalf of the katom. + * This is needed because sync_file_create() will take ownership of + * one of these refs */ + dma_fence_get(fence); + + /* create a sync_file fd representing the fence */ + sync_file = sync_file_create(fence); + if (!sync_file) { + dma_fence_put(fence); + kbase_fence_out_remove(katom); + return -ENOMEM; + } + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + fput(sync_file->file); + kbase_fence_out_remove(katom); + return fd; + } + + fd_install(fd, sync_file->file); + + return fd; +} + +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = sync_file_get_fence(fd); +#else + struct dma_fence *fence = sync_file_get_fence(fd); +#endif + + if (!fence) + return -ENOENT; + + kbase_fence_fence_in_set(katom, fence); + + return 0; +} + +int kbase_sync_fence_validate(int fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = sync_file_get_fence(fd); +#else + struct dma_fence *fence = sync_file_get_fence(fd); +#endif + + if (!fence) + return -EINVAL; + + dma_fence_put(fence); + + return 0; /* valid */ +} + +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) +{ + int res; + + if (!kbase_fence_out_is_ours(katom)) { + /* Not our fence */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + + res = kbase_fence_out_signal(katom, result); + if (unlikely(res < 0)) { + dev_warn(katom->kctx->kbdev->dev, + "fence_signal() failed with %d\n", res); + } + + kbase_sync_fence_out_remove(katom); + + return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +static void kbase_fence_wait_callback(struct fence *fence, + struct fence_cb *cb) +#else +static void kbase_fence_wait_callback(struct dma_fence *fence, + struct dma_fence_cb *cb) +#endif +{ + struct kbase_fence_cb *kcb = container_of(cb, + struct kbase_fence_cb, + fence_cb); + struct kbase_jd_atom *katom = kcb->katom; + struct kbase_context *kctx = katom->kctx; + + /* Cancel atom if fence is erroneous */ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) + if (dma_fence_is_signaled(kcb->fence) && kcb->fence->error) +#else + if (dma_fence_is_signaled(kcb->fence) && kcb->fence->status < 0) +#endif + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + if (kbase_fence_dep_count_dec_and_test(katom)) { + /* We take responsibility of handling this */ + kbase_fence_dep_count_set(katom, -1); + + /* To prevent a potential deadlock we schedule the work onto the + * job_done_wq workqueue + * + * The issue is that we may signal the timeline while holding + * kctx->jctx.lock and the callbacks are run synchronously from + * sync_timeline_signal. So we simply defer the work. + */ + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); + } +} + +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) +{ + int err; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_in_get(katom); + if (!fence) + return 0; /* no input fence to wait for, good to go! */ + + kbase_fence_dep_count_set(katom, 1); + + err = kbase_fence_add_callback(katom, fence, kbase_fence_wait_callback); + + kbase_fence_put(fence); + + if (likely(!err)) { + /* Test if the callbacks are already triggered */ + if (kbase_fence_dep_count_dec_and_test(katom)) { + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + return 0; /* Already signaled, good to go right now */ + } + + /* Callback installed, so we just need to wait for it... */ + } else { + /* Failure */ + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* We should cause the dependent jobs in the bag to be failed, + * to do this we schedule the work queue to complete this job */ + + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(katom->kctx->jctx.job_done_wq, &katom->work); + } + + return 1; /* completion to be done later by callback/worker */ +} + +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) +{ + if (!kbase_fence_free_callbacks(katom)) { + /* The wait wasn't cancelled - + * leave the cleanup for kbase_fence_wait_callback */ + return; + } + + /* Take responsibility of completion */ + kbase_fence_dep_count_set(katom, -1); + + /* Wait was cancelled - zap the atoms */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) +{ + kbase_fence_out_remove(katom); +} + +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) +{ + kbase_fence_free_callbacks(katom); + kbase_fence_in_remove(katom); +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +static void kbase_sync_fence_info_get(struct fence *fence, + struct kbase_sync_fence_info *info) +#else +static void kbase_sync_fence_info_get(struct dma_fence *fence, + struct kbase_sync_fence_info *info) +#endif +{ + info->fence = fence; + + /* translate into CONFIG_SYNC status: + * < 0 : error + * 0 : active + * 1 : signaled + */ + if (dma_fence_is_signaled(fence)) { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) + int status = fence->error; +#else + int status = fence->status; +#endif + if (status < 0) + info->status = status; /* signaled with error */ + else + info->status = 1; /* signaled with success */ + } else { + info->status = 0; /* still active (unsignaled) */ + } + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + scnprintf(info->name, sizeof(info->name), "%u#%u", + fence->context, fence->seqno); +#else + scnprintf(info->name, sizeof(info->name), "%llu#%u", + fence->context, fence->seqno); +#endif +} + +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_in_get(katom); + if (!fence) + return -ENOENT; + + kbase_sync_fence_info_get(fence, info); + + kbase_fence_put(fence); + + return 0; +} + +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_out_get(katom); + if (!fence) + return -ENOENT; + + kbase_sync_fence_info_get(fence, info); + + kbase_fence_put(fence); + + return 0; +} + + +#ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) +{ + /* Not implemented */ +} +#endif diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.c new file mode 100755 index 000000000000..485565ebfe80 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.c @@ -0,0 +1,2569 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/*****************************************************************************/ + +/* The version of swtrace protocol used in timeline stream. */ +#define SWTRACE_VERSION 3 + +/* The maximum expected length of string in tracepoint descriptor. */ +#define STRLEN_MAX 64 /* bytes */ + +/* The number of nanoseconds in a second. */ +#define NSECS_IN_SEC 1000000000ull /* ns */ + +/* The period of autoflush checker execution in milliseconds. */ +#define AUTOFLUSH_INTERVAL 1000 /* ms */ + +/* The maximum size of a single packet used by timeline. */ +#define PACKET_SIZE 4096 /* bytes */ + +/* The number of packets used by one timeline stream. */ +#define PACKET_COUNT 16 + +/* The number of bytes reserved for packet header. + * These value must be defined according to MIPE documentation. */ +#define PACKET_HEADER_SIZE 8 /* bytes */ + +/* The number of bytes reserved for packet sequence number. + * These value must be defined according to MIPE documentation. */ +#define PACKET_NUMBER_SIZE 4 /* bytes */ + +/* Packet header - first word. + * These values must be defined according to MIPE documentation. */ +#define PACKET_STREAMID_POS 0 +#define PACKET_STREAMID_LEN 8 +#define PACKET_RSVD1_POS (PACKET_STREAMID_POS + PACKET_STREAMID_LEN) +#define PACKET_RSVD1_LEN 8 +#define PACKET_TYPE_POS (PACKET_RSVD1_POS + PACKET_RSVD1_LEN) +#define PACKET_TYPE_LEN 3 +#define PACKET_CLASS_POS (PACKET_TYPE_POS + PACKET_TYPE_LEN) +#define PACKET_CLASS_LEN 7 +#define PACKET_FAMILY_POS (PACKET_CLASS_POS + PACKET_CLASS_LEN) +#define PACKET_FAMILY_LEN 6 + +/* Packet header - second word + * These values must be defined according to MIPE documentation. */ +#define PACKET_LENGTH_POS 0 +#define PACKET_LENGTH_LEN 24 +#define PACKET_SEQBIT_POS (PACKET_LENGTH_POS + PACKET_LENGTH_LEN) +#define PACKET_SEQBIT_LEN 1 +#define PACKET_RSVD2_POS (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN) +#define PACKET_RSVD2_LEN 7 + +/* Types of streams generated by timeline. + * Order is significant! Header streams must precede respective body streams. */ +enum tl_stream_type { + TL_STREAM_TYPE_OBJ_HEADER, + TL_STREAM_TYPE_OBJ_SUMMARY, + TL_STREAM_TYPE_OBJ, + TL_STREAM_TYPE_AUX_HEADER, + TL_STREAM_TYPE_AUX, + + TL_STREAM_TYPE_COUNT +}; + +/* Timeline packet family ids. + * Values are significant! Check MIPE documentation. */ +enum tl_packet_family { + TL_PACKET_FAMILY_CTRL = 0, /* control packets */ + TL_PACKET_FAMILY_TL = 1, /* timeline packets */ + + TL_PACKET_FAMILY_COUNT +}; + +/* Packet classes used in timeline streams. + * Values are significant! Check MIPE documentation. */ +enum tl_packet_class { + TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */ + TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */ +}; + +/* Packet types used in timeline streams. + * Values are significant! Check MIPE documentation. */ +enum tl_packet_type { + TL_PACKET_TYPE_HEADER = 0, /* stream's header/directory */ + TL_PACKET_TYPE_BODY = 1, /* stream's body */ + TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */ +}; + +/* Message ids of trace events that are recorded in the timeline stream. */ +enum tl_msg_id_obj { + /* Timeline object events. */ + KBASE_TL_NEW_CTX, + KBASE_TL_NEW_GPU, + KBASE_TL_NEW_LPU, + KBASE_TL_NEW_ATOM, + KBASE_TL_NEW_AS, + KBASE_TL_DEL_CTX, + KBASE_TL_DEL_ATOM, + KBASE_TL_LIFELINK_LPU_GPU, + KBASE_TL_LIFELINK_AS_GPU, + KBASE_TL_RET_CTX_LPU, + KBASE_TL_RET_ATOM_CTX, + KBASE_TL_RET_ATOM_LPU, + KBASE_TL_NRET_CTX_LPU, + KBASE_TL_NRET_ATOM_CTX, + KBASE_TL_NRET_ATOM_LPU, + KBASE_TL_RET_AS_CTX, + KBASE_TL_NRET_AS_CTX, + KBASE_TL_RET_ATOM_AS, + KBASE_TL_NRET_ATOM_AS, + KBASE_TL_DEP_ATOM_ATOM, + KBASE_TL_NDEP_ATOM_ATOM, + KBASE_TL_RDEP_ATOM_ATOM, + KBASE_TL_ATTRIB_ATOM_CONFIG, + KBASE_TL_ATTRIB_ATOM_PRIORITY, + KBASE_TL_ATTRIB_ATOM_STATE, + KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, + KBASE_TL_ATTRIB_ATOM_JIT, + KBASE_TL_ATTRIB_AS_CONFIG, + KBASE_TL_EVENT_LPU_SOFTSTOP, + KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, + KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, + + /* Job dump specific events. */ + KBASE_JD_GPU_SOFT_RESET +}; + +/* Message ids of trace events that are recorded in the auxiliary stream. */ +enum tl_msg_id_aux { + KBASE_AUX_PM_STATE, + KBASE_AUX_PAGEFAULT, + KBASE_AUX_PAGESALLOC, + KBASE_AUX_DEVFREQ_TARGET, + KBASE_AUX_PROTECTED_ENTER_START, + KBASE_AUX_PROTECTED_ENTER_END, + KBASE_AUX_PROTECTED_LEAVE_START, + KBASE_AUX_PROTECTED_LEAVE_END +}; + +/*****************************************************************************/ + +/** + * struct tl_stream - timeline stream structure + * @lock: message order lock + * @buffer: array of buffers + * @wbi: write buffer index + * @rbi: read buffer index + * @numbered: if non-zero stream's packets are sequentially numbered + * @autoflush_counter: counter tracking stream's autoflush state + * + * This structure holds information needed to construct proper packets in the + * timeline stream. Each message in sequence must bear timestamp that is greater + * to one in previous message in the same stream. For this reason lock is held + * throughout the process of message creation. Each stream contains set of + * buffers. Each buffer will hold one MIPE packet. In case there is no free + * space required to store incoming message the oldest buffer is discarded. + * Each packet in timeline body stream has sequence number embedded (this value + * must increment monotonically and is used by packets receiver to discover + * buffer overflows. + * Autoflush counter is set to negative number when there is no data pending + * for flush and it is set to zero on every update of the buffer. Autoflush + * timer will increment the counter by one on every expiry. In case there will + * be no activity on the buffer during two consecutive timer expiries, stream + * buffer will be flushed. + */ +struct tl_stream { + spinlock_t lock; + + struct { + atomic_t size; /* number of bytes in buffer */ + char data[PACKET_SIZE]; /* buffer's data */ + } buffer[PACKET_COUNT]; + + atomic_t wbi; + atomic_t rbi; + + int numbered; + atomic_t autoflush_counter; +}; + +/** + * struct tp_desc - tracepoint message descriptor structure + * @id: tracepoint ID identifying message in stream + * @id_str: human readable version of tracepoint ID + * @name: tracepoint description + * @arg_types: tracepoint's arguments types declaration + * @arg_names: comma separated list of tracepoint's arguments names + */ +struct tp_desc { + u32 id; + const char *id_str; + const char *name; + const char *arg_types; + const char *arg_names; +}; + +/*****************************************************************************/ + +/* Configuration of timeline streams generated by kernel. + * Kernel emit only streams containing either timeline object events or + * auxiliary events. All streams have stream id value of 1 (as opposed to user + * space streams that have value of 0). */ +static const struct { + enum tl_packet_family pkt_family; + enum tl_packet_class pkt_class; + enum tl_packet_type pkt_type; + unsigned int stream_id; +} tl_stream_cfg[TL_STREAM_TYPE_COUNT] = { + {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_HEADER, 1}, + {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_SUMMARY, 1}, + {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY, 1}, + {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_HEADER, 1}, + {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_BODY, 1} +}; + +/* The timeline streams generated by kernel. */ +static struct tl_stream *tl_stream[TL_STREAM_TYPE_COUNT]; + +/* Autoflush timer. */ +static struct timer_list autoflush_timer; + +/* If non-zero autoflush timer is active. */ +static atomic_t autoflush_timer_active; + +/* Reader lock. Only one reader is allowed to have access to the timeline + * streams at any given time. */ +static DEFINE_MUTEX(tl_reader_lock); + +/* Timeline stream event queue. */ +static DECLARE_WAIT_QUEUE_HEAD(tl_event_queue); + +/* The timeline stream file operations functions. */ +static ssize_t kbasep_tlstream_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos); +static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait); +static int kbasep_tlstream_release(struct inode *inode, struct file *filp); + +/* The timeline stream file operations structure. */ +static const struct file_operations kbasep_tlstream_fops = { + .release = kbasep_tlstream_release, + .read = kbasep_tlstream_read, + .poll = kbasep_tlstream_poll, +}; + +/* Descriptors of timeline messages transmitted in object events stream. */ +static const struct tp_desc tp_desc_obj[] = { + { + KBASE_TL_NEW_CTX, + __stringify(KBASE_TL_NEW_CTX), + "object ctx is created", + "@pII", + "ctx,ctx_nr,tgid" + }, + { + KBASE_TL_NEW_GPU, + __stringify(KBASE_TL_NEW_GPU), + "object gpu is created", + "@pII", + "gpu,gpu_id,core_count" + }, + { + KBASE_TL_NEW_LPU, + __stringify(KBASE_TL_NEW_LPU), + "object lpu is created", + "@pII", + "lpu,lpu_nr,lpu_fn" + }, + { + KBASE_TL_NEW_ATOM, + __stringify(KBASE_TL_NEW_ATOM), + "object atom is created", + "@pI", + "atom,atom_nr" + }, + { + KBASE_TL_NEW_AS, + __stringify(KBASE_TL_NEW_AS), + "address space object is created", + "@pI", + "address_space,as_nr" + }, + { + KBASE_TL_DEL_CTX, + __stringify(KBASE_TL_DEL_CTX), + "context is destroyed", + "@p", + "ctx" + }, + { + KBASE_TL_DEL_ATOM, + __stringify(KBASE_TL_DEL_ATOM), + "atom is destroyed", + "@p", + "atom" + }, + { + KBASE_TL_LIFELINK_LPU_GPU, + __stringify(KBASE_TL_LIFELINK_LPU_GPU), + "lpu is deleted with gpu", + "@pp", + "lpu,gpu" + }, + { + KBASE_TL_LIFELINK_AS_GPU, + __stringify(KBASE_TL_LIFELINK_AS_GPU), + "address space is deleted with gpu", + "@pp", + "address_space,gpu" + }, + { + KBASE_TL_RET_CTX_LPU, + __stringify(KBASE_TL_RET_CTX_LPU), + "context is retained by lpu", + "@pp", + "ctx,lpu" + }, + { + KBASE_TL_RET_ATOM_CTX, + __stringify(KBASE_TL_RET_ATOM_CTX), + "atom is retained by context", + "@pp", + "atom,ctx" + }, + { + KBASE_TL_RET_ATOM_LPU, + __stringify(KBASE_TL_RET_ATOM_LPU), + "atom is retained by lpu", + "@pps", + "atom,lpu,attrib_match_list" + }, + { + KBASE_TL_NRET_CTX_LPU, + __stringify(KBASE_TL_NRET_CTX_LPU), + "context is released by lpu", + "@pp", + "ctx,lpu" + }, + { + KBASE_TL_NRET_ATOM_CTX, + __stringify(KBASE_TL_NRET_ATOM_CTX), + "atom is released by context", + "@pp", + "atom,ctx" + }, + { + KBASE_TL_NRET_ATOM_LPU, + __stringify(KBASE_TL_NRET_ATOM_LPU), + "atom is released by lpu", + "@pp", + "atom,lpu" + }, + { + KBASE_TL_RET_AS_CTX, + __stringify(KBASE_TL_RET_AS_CTX), + "address space is retained by context", + "@pp", + "address_space,ctx" + }, + { + KBASE_TL_NRET_AS_CTX, + __stringify(KBASE_TL_NRET_AS_CTX), + "address space is released by context", + "@pp", + "address_space,ctx" + }, + { + KBASE_TL_RET_ATOM_AS, + __stringify(KBASE_TL_RET_ATOM_AS), + "atom is retained by address space", + "@pp", + "atom,address_space" + }, + { + KBASE_TL_NRET_ATOM_AS, + __stringify(KBASE_TL_NRET_ATOM_AS), + "atom is released by address space", + "@pp", + "atom,address_space" + }, + { + KBASE_TL_DEP_ATOM_ATOM, + __stringify(KBASE_TL_DEP_ATOM_ATOM), + "atom2 depends on atom1", + "@pp", + "atom1,atom2" + }, + { + KBASE_TL_NDEP_ATOM_ATOM, + __stringify(KBASE_TL_NDEP_ATOM_ATOM), + "atom2 no longer depends on atom1", + "@pp", + "atom1,atom2" + }, + { + KBASE_TL_RDEP_ATOM_ATOM, + __stringify(KBASE_TL_RDEP_ATOM_ATOM), + "resolved dependecy of atom2 depending on atom1", + "@pp", + "atom1,atom2" + }, + { + KBASE_TL_ATTRIB_ATOM_CONFIG, + __stringify(KBASE_TL_ATTRIB_ATOM_CONFIG), + "atom job slot attributes", + "@pLLI", + "atom,descriptor,affinity,config" + }, + { + KBASE_TL_ATTRIB_ATOM_PRIORITY, + __stringify(KBASE_TL_ATTRIB_ATOM_PRIORITY), + "atom priority", + "@pI", + "atom,prio" + }, + { + KBASE_TL_ATTRIB_ATOM_STATE, + __stringify(KBASE_TL_ATTRIB_ATOM_STATE), + "atom state", + "@pI", + "atom,state" + }, + { + KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, + __stringify(KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE), + "atom caused priority change", + "@p", + "atom" + }, + { + KBASE_TL_ATTRIB_ATOM_JIT, + __stringify(KBASE_TL_ATTRIB_ATOM_JIT), + "jit done for atom", + "@pLL", + "atom,edit_addr,new_addr" + }, + { + KBASE_TL_ATTRIB_AS_CONFIG, + __stringify(KBASE_TL_ATTRIB_AS_CONFIG), + "address space attributes", + "@pLLL", + "address_space,transtab,memattr,transcfg" + }, + { + KBASE_TL_EVENT_LPU_SOFTSTOP, + __stringify(KBASE_TL_EVENT_LPU_SOFTSTOP), + "softstop event on given lpu", + "@p", + "lpu" + }, + { + KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, + __stringify(KBASE_TL_EVENT_ATOM_SOFTSTOP_EX), + "atom softstopped", + "@p", + "atom" + }, + { + KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, + __stringify(KBASE_TL_EVENT_SOFTSTOP_ISSUE), + "atom softstop issued", + "@p", + "atom" + }, + { + KBASE_JD_GPU_SOFT_RESET, + __stringify(KBASE_JD_GPU_SOFT_RESET), + "gpu soft reset", + "@p", + "gpu" + }, +}; + +/* Descriptors of timeline messages transmitted in auxiliary events stream. */ +static const struct tp_desc tp_desc_aux[] = { + { + KBASE_AUX_PM_STATE, + __stringify(KBASE_AUX_PM_STATE), + "PM state", + "@IL", + "core_type,core_state_bitset" + }, + { + KBASE_AUX_PAGEFAULT, + __stringify(KBASE_AUX_PAGEFAULT), + "Page fault", + "@IL", + "ctx_nr,page_cnt_change" + }, + { + KBASE_AUX_PAGESALLOC, + __stringify(KBASE_AUX_PAGESALLOC), + "Total alloc pages change", + "@IL", + "ctx_nr,page_cnt" + }, + { + KBASE_AUX_DEVFREQ_TARGET, + __stringify(KBASE_AUX_DEVFREQ_TARGET), + "New device frequency target", + "@L", + "target_freq" + }, + { + KBASE_AUX_PROTECTED_ENTER_START, + __stringify(KBASE_AUX_PROTECTED_ENTER_START), + "enter protected mode start", + "@p", + "gpu" + }, + { + KBASE_AUX_PROTECTED_ENTER_END, + __stringify(KBASE_AUX_PROTECTED_ENTER_END), + "enter protected mode end", + "@p", + "gpu" + }, + { + KBASE_AUX_PROTECTED_LEAVE_START, + __stringify(KBASE_AUX_PROTECTED_LEAVE_START), + "leave protected mode start", + "@p", + "gpu" + }, + { + KBASE_AUX_PROTECTED_LEAVE_END, + __stringify(KBASE_AUX_PROTECTED_LEAVE_END), + "leave protected mode end", + "@p", + "gpu" + } +}; + +#if MALI_UNIT_TEST +/* Number of bytes read by user. */ +static atomic_t tlstream_bytes_collected = {0}; + +/* Number of bytes generated by tracepoint messages. */ +static atomic_t tlstream_bytes_generated = {0}; +#endif /* MALI_UNIT_TEST */ + +/*****************************************************************************/ + +/* Indicator of whether the timeline stream file descriptor is used. */ +atomic_t kbase_tlstream_enabled = {0}; + +/*****************************************************************************/ + +/** + * kbasep_tlstream_get_timestamp - return timestamp + * + * Function returns timestamp value based on raw monotonic timer. Value will + * wrap around zero in case of overflow. + * Return: timestamp value + */ +static u64 kbasep_tlstream_get_timestamp(void) +{ + struct timespec ts; + u64 timestamp; + + getrawmonotonic(&ts); + timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; + return timestamp; +} + +/** + * kbasep_tlstream_write_bytes - write data to message buffer + * @buffer: buffer where data will be written + * @pos: position in the buffer where to place data + * @bytes: pointer to buffer holding data + * @len: length of data to be written + * + * Return: updated position in the buffer + */ +static size_t kbasep_tlstream_write_bytes( + char *buffer, + size_t pos, + const void *bytes, + size_t len) +{ + KBASE_DEBUG_ASSERT(buffer); + KBASE_DEBUG_ASSERT(bytes); + + memcpy(&buffer[pos], bytes, len); + + return pos + len; +} + +/** + * kbasep_tlstream_write_string - write string to message buffer + * @buffer: buffer where data will be written + * @pos: position in the buffer where to place data + * @string: pointer to buffer holding the source string + * @max_write_size: number of bytes that can be stored in buffer + * + * Return: updated position in the buffer + */ +static size_t kbasep_tlstream_write_string( + char *buffer, + size_t pos, + const char *string, + size_t max_write_size) +{ + u32 string_len; + + KBASE_DEBUG_ASSERT(buffer); + KBASE_DEBUG_ASSERT(string); + /* Timeline string consists of at least string length and nul + * terminator. */ + KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char)); + max_write_size -= sizeof(string_len); + + string_len = strlcpy( + &buffer[pos + sizeof(string_len)], + string, + max_write_size); + string_len += sizeof(char); + + /* Make sure that the source string fit into the buffer. */ + KBASE_DEBUG_ASSERT(string_len <= max_write_size); + + /* Update string length. */ + memcpy(&buffer[pos], &string_len, sizeof(string_len)); + + return pos + sizeof(string_len) + string_len; +} + +/** + * kbasep_tlstream_write_timestamp - write timestamp to message buffer + * @buffer: buffer where data will be written + * @pos: position in the buffer where to place data + * + * Return: updated position in the buffer + */ +static size_t kbasep_tlstream_write_timestamp(void *buffer, size_t pos) +{ + u64 timestamp = kbasep_tlstream_get_timestamp(); + + return kbasep_tlstream_write_bytes( + buffer, pos, + ×tamp, sizeof(timestamp)); +} + +/** + * kbasep_tlstream_put_bits - put bits in a word + * @word: pointer to the words being modified + * @value: value that shall be written to given position + * @bitpos: position where value shall be written (in bits) + * @bitlen: length of value (in bits) + */ +static void kbasep_tlstream_put_bits( + u32 *word, + u32 value, + unsigned int bitpos, + unsigned int bitlen) +{ + const u32 mask = ((1 << bitlen) - 1) << bitpos; + + KBASE_DEBUG_ASSERT(word); + KBASE_DEBUG_ASSERT((0 != bitlen) && (32 >= bitlen)); + KBASE_DEBUG_ASSERT((bitpos + bitlen) <= 32); + + *word &= ~mask; + *word |= ((value << bitpos) & mask); +} + +/** + * kbasep_tlstream_packet_header_setup - setup the packet header + * @buffer: pointer to the buffer + * @pkt_family: packet's family + * @pkt_type: packet's type + * @pkt_class: packet's class + * @stream_id: stream id + * @numbered: non-zero if this stream is numbered + * + * Function sets up immutable part of packet header in the given buffer. + */ +static void kbasep_tlstream_packet_header_setup( + char *buffer, + enum tl_packet_family pkt_family, + enum tl_packet_class pkt_class, + enum tl_packet_type pkt_type, + unsigned int stream_id, + int numbered) +{ + u32 word0 = 0; + u32 word1 = 0; + + KBASE_DEBUG_ASSERT(buffer); + KBASE_DEBUG_ASSERT(pkt_family == TL_PACKET_FAMILY_TL); + KBASE_DEBUG_ASSERT( + (pkt_type == TL_PACKET_TYPE_HEADER) || + (pkt_type == TL_PACKET_TYPE_SUMMARY) || + (pkt_type == TL_PACKET_TYPE_BODY)); + KBASE_DEBUG_ASSERT( + (pkt_class == TL_PACKET_CLASS_OBJ) || + (pkt_class == TL_PACKET_CLASS_AUX)); + + kbasep_tlstream_put_bits( + &word0, pkt_family, + PACKET_FAMILY_POS, PACKET_FAMILY_LEN); + kbasep_tlstream_put_bits( + &word0, pkt_class, + PACKET_CLASS_POS, PACKET_CLASS_LEN); + kbasep_tlstream_put_bits( + &word0, pkt_type, + PACKET_TYPE_POS, PACKET_TYPE_LEN); + kbasep_tlstream_put_bits( + &word0, stream_id, + PACKET_STREAMID_POS, PACKET_STREAMID_LEN); + + if (numbered) + kbasep_tlstream_put_bits( + &word1, 1, + PACKET_SEQBIT_POS, PACKET_SEQBIT_LEN); + + memcpy(&buffer[0], &word0, sizeof(word0)); + memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); +} + +/** + * kbasep_tlstream_packet_header_update - update the packet header + * @buffer: pointer to the buffer + * @data_size: amount of data carried in this packet + * + * Function updates mutable part of packet header in the given buffer. + * Note that value of data_size must not including size of the header. + */ +static void kbasep_tlstream_packet_header_update( + char *buffer, + size_t data_size) +{ + u32 word0; + u32 word1; + + KBASE_DEBUG_ASSERT(buffer); + CSTD_UNUSED(word0); + + memcpy(&word1, &buffer[sizeof(word0)], sizeof(word1)); + + kbasep_tlstream_put_bits( + &word1, data_size, + PACKET_LENGTH_POS, PACKET_LENGTH_LEN); + + memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); +} + +/** + * kbasep_tlstream_packet_number_update - update the packet number + * @buffer: pointer to the buffer + * @counter: value of packet counter for this packet's stream + * + * Function updates packet number embedded within the packet placed in the + * given buffer. + */ +static void kbasep_tlstream_packet_number_update(char *buffer, u32 counter) +{ + KBASE_DEBUG_ASSERT(buffer); + + memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter)); +} + +/** + * kbasep_timeline_stream_reset - reset stream + * @stream: pointer to the stream structure + * + * Function discards all pending messages and resets packet counters. + */ +static void kbasep_timeline_stream_reset(struct tl_stream *stream) +{ + unsigned int i; + + for (i = 0; i < PACKET_COUNT; i++) { + if (stream->numbered) + atomic_set( + &stream->buffer[i].size, + PACKET_HEADER_SIZE + + PACKET_NUMBER_SIZE); + else + atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE); + } + + atomic_set(&stream->wbi, 0); + atomic_set(&stream->rbi, 0); +} + +/** + * kbasep_timeline_stream_init - initialize timeline stream + * @stream: pointer to the stream structure + * @stream_type: stream type + */ +static void kbasep_timeline_stream_init( + struct tl_stream *stream, + enum tl_stream_type stream_type) +{ + unsigned int i; + + KBASE_DEBUG_ASSERT(stream); + KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); + + spin_lock_init(&stream->lock); + + /* All packets carrying tracepoints shall be numbered. */ + if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type) + stream->numbered = 1; + else + stream->numbered = 0; + + for (i = 0; i < PACKET_COUNT; i++) + kbasep_tlstream_packet_header_setup( + stream->buffer[i].data, + tl_stream_cfg[stream_type].pkt_family, + tl_stream_cfg[stream_type].pkt_class, + tl_stream_cfg[stream_type].pkt_type, + tl_stream_cfg[stream_type].stream_id, + stream->numbered); + + kbasep_timeline_stream_reset(tl_stream[stream_type]); +} + +/** + * kbasep_timeline_stream_term - terminate timeline stream + * @stream: pointer to the stream structure + */ +static void kbasep_timeline_stream_term(struct tl_stream *stream) +{ + KBASE_DEBUG_ASSERT(stream); +} + +/** + * kbasep_tlstream_msgbuf_submit - submit packet to the user space + * @stream: pointer to the stream structure + * @wb_idx_raw: write buffer index + * @wb_size: length of data stored in current buffer + * + * Function updates currently written buffer with packet header. Then write + * index is incremented and buffer is handled to user space. Parameters + * of new buffer are returned using provided arguments. + * + * Return: length of data in new buffer + * + * Warning: User must update the stream structure with returned value. + */ +static size_t kbasep_tlstream_msgbuf_submit( + struct tl_stream *stream, + unsigned int wb_idx_raw, + unsigned int wb_size) +{ + unsigned int rb_idx_raw = atomic_read(&stream->rbi); + unsigned int wb_idx = wb_idx_raw % PACKET_COUNT; + + /* Set stream as flushed. */ + atomic_set(&stream->autoflush_counter, -1); + + kbasep_tlstream_packet_header_update( + stream->buffer[wb_idx].data, + wb_size - PACKET_HEADER_SIZE); + + if (stream->numbered) + kbasep_tlstream_packet_number_update( + stream->buffer[wb_idx].data, + wb_idx_raw); + + /* Increasing write buffer index will expose this packet to the reader. + * As stream->lock is not taken on reader side we must make sure memory + * is updated correctly before this will happen. */ + smp_wmb(); + wb_idx_raw++; + atomic_set(&stream->wbi, wb_idx_raw); + + /* Inform user that packets are ready for reading. */ + wake_up_interruptible(&tl_event_queue); + + /* Detect and mark overflow in this stream. */ + if (PACKET_COUNT == wb_idx_raw - rb_idx_raw) { + /* Reader side depends on this increment to correctly handle + * overflows. The value shall be updated only if it was not + * modified by the reader. The data holding buffer will not be + * updated before stream->lock is released, however size of the + * buffer will. Make sure this increment is globally visible + * before information about selected write buffer size. */ + atomic_cmpxchg(&stream->rbi, rb_idx_raw, rb_idx_raw + 1); + } + + wb_size = PACKET_HEADER_SIZE; + if (stream->numbered) + wb_size += PACKET_NUMBER_SIZE; + + return wb_size; +} + +/** + * kbasep_tlstream_msgbuf_acquire - lock selected stream and reserves buffer + * @stream_type: type of the stream that shall be locked + * @msg_size: message size + * @flags: pointer to store flags passed back on stream release + * + * Function will lock the stream and reserve the number of bytes requested + * in msg_size for the user. + * + * Return: pointer to the buffer where message can be stored + * + * Warning: Stream must be released with kbasep_tlstream_msgbuf_release(). + * Only atomic operations are allowed while stream is locked + * (i.e. do not use any operation that may sleep). + */ +static char *kbasep_tlstream_msgbuf_acquire( + enum tl_stream_type stream_type, + size_t msg_size, + unsigned long *flags) __acquires(&stream->lock) +{ + struct tl_stream *stream; + unsigned int wb_idx_raw; + unsigned int wb_idx; + size_t wb_size; + + KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); + KBASE_DEBUG_ASSERT( + PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >= + msg_size); + + stream = tl_stream[stream_type]; + + spin_lock_irqsave(&stream->lock, *flags); + + wb_idx_raw = atomic_read(&stream->wbi); + wb_idx = wb_idx_raw % PACKET_COUNT; + wb_size = atomic_read(&stream->buffer[wb_idx].size); + + /* Select next buffer if data will not fit into current one. */ + if (PACKET_SIZE < wb_size + msg_size) { + wb_size = kbasep_tlstream_msgbuf_submit( + stream, wb_idx_raw, wb_size); + wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; + } + + /* Reserve space in selected buffer. */ + atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size); + +#if MALI_UNIT_TEST + atomic_add(msg_size, &tlstream_bytes_generated); +#endif /* MALI_UNIT_TEST */ + + return &stream->buffer[wb_idx].data[wb_size]; +} + +/** + * kbasep_tlstream_msgbuf_release - unlock selected stream + * @stream_type: type of the stream that shall be locked + * @flags: value obtained during stream acquire + * + * Function releases stream that has been previously locked with a call to + * kbasep_tlstream_msgbuf_acquire(). + */ +static void kbasep_tlstream_msgbuf_release( + enum tl_stream_type stream_type, + unsigned long flags) __releases(&stream->lock) +{ + struct tl_stream *stream; + + KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); + + stream = tl_stream[stream_type]; + + /* Mark stream as containing unflushed data. */ + atomic_set(&stream->autoflush_counter, 0); + + spin_unlock_irqrestore(&stream->lock, flags); +} + +/*****************************************************************************/ + +/** + * kbasep_tlstream_flush_stream - flush stream + * @stype: type of stream to be flushed + * + * Flush pending data in timeline stream. + */ +static void kbasep_tlstream_flush_stream(enum tl_stream_type stype) +{ + struct tl_stream *stream = tl_stream[stype]; + unsigned long flags; + unsigned int wb_idx_raw; + unsigned int wb_idx; + size_t wb_size; + size_t min_size = PACKET_HEADER_SIZE; + + if (stream->numbered) + min_size += PACKET_NUMBER_SIZE; + + spin_lock_irqsave(&stream->lock, flags); + + wb_idx_raw = atomic_read(&stream->wbi); + wb_idx = wb_idx_raw % PACKET_COUNT; + wb_size = atomic_read(&stream->buffer[wb_idx].size); + + if (wb_size > min_size) { + wb_size = kbasep_tlstream_msgbuf_submit( + stream, wb_idx_raw, wb_size); + wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; + atomic_set(&stream->buffer[wb_idx].size, wb_size); + } + spin_unlock_irqrestore(&stream->lock, flags); +} + +/** + * kbasep_tlstream_autoflush_timer_callback - autoflush timer callback + * @data: unused + * + * Timer is executed periodically to check if any of the stream contains + * buffer ready to be submitted to user space. + */ +static void kbasep_tlstream_autoflush_timer_callback(struct timer_list *t) +{ + enum tl_stream_type stype; + int rcode; + + for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) { + struct tl_stream *stream = tl_stream[stype]; + unsigned long flags; + unsigned int wb_idx_raw; + unsigned int wb_idx; + size_t wb_size; + size_t min_size = PACKET_HEADER_SIZE; + + int af_cnt = atomic_read(&stream->autoflush_counter); + + /* Check if stream contain unflushed data. */ + if (0 > af_cnt) + continue; + + /* Check if stream should be flushed now. */ + if (af_cnt != atomic_cmpxchg( + &stream->autoflush_counter, + af_cnt, + af_cnt + 1)) + continue; + if (!af_cnt) + continue; + + /* Autoflush this stream. */ + if (stream->numbered) + min_size += PACKET_NUMBER_SIZE; + + spin_lock_irqsave(&stream->lock, flags); + + wb_idx_raw = atomic_read(&stream->wbi); + wb_idx = wb_idx_raw % PACKET_COUNT; + wb_size = atomic_read(&stream->buffer[wb_idx].size); + + if (wb_size > min_size) { + wb_size = kbasep_tlstream_msgbuf_submit( + stream, wb_idx_raw, wb_size); + wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; + atomic_set(&stream->buffer[wb_idx].size, + wb_size); + } + spin_unlock_irqrestore(&stream->lock, flags); + } + + if (atomic_read(&autoflush_timer_active)) + rcode = mod_timer( + &autoflush_timer, + jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); + CSTD_UNUSED(rcode); +} + +/** + * kbasep_tlstream_packet_pending - check timeline streams for pending packets + * @stype: pointer to variable where stream type will be placed + * @rb_idx_raw: pointer to variable where read buffer index will be placed + * + * Function checks all streams for pending packets. It will stop as soon as + * packet ready to be submitted to user space is detected. Variables under + * pointers, passed as the parameters to this function will be updated with + * values pointing to right stream and buffer. + * + * Return: non-zero if any of timeline streams has at last one packet ready + */ +static int kbasep_tlstream_packet_pending( + enum tl_stream_type *stype, + unsigned int *rb_idx_raw) +{ + int pending = 0; + + KBASE_DEBUG_ASSERT(stype); + KBASE_DEBUG_ASSERT(rb_idx_raw); + + for ( + *stype = 0; + (*stype < TL_STREAM_TYPE_COUNT) && !pending; + (*stype)++) { + if (NULL != tl_stream[*stype]) { + *rb_idx_raw = atomic_read(&tl_stream[*stype]->rbi); + /* Read buffer index may be updated by writer in case of + * overflow. Read and write buffer indexes must be + * loaded in correct order. */ + smp_rmb(); + if (atomic_read(&tl_stream[*stype]->wbi) != *rb_idx_raw) + pending = 1; + } + } + (*stype)--; + + return pending; +} + +/** + * kbasep_tlstream_read - copy data from streams to buffer provided by user + * @filp: pointer to file structure (unused) + * @buffer: pointer to the buffer provided by user + * @size: maximum amount of data that can be stored in the buffer + * @f_pos: pointer to file offset (unused) + * + * Return: number of bytes stored in the buffer + */ +static ssize_t kbasep_tlstream_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos) +{ + ssize_t copy_len = 0; + + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(f_pos); + + if (!buffer) + return -EINVAL; + + if ((0 > *f_pos) || (PACKET_SIZE > size)) + return -EINVAL; + + mutex_lock(&tl_reader_lock); + + while (copy_len < size) { + enum tl_stream_type stype; + unsigned int rb_idx_raw = 0; + unsigned int rb_idx; + size_t rb_size; + + /* If we don't have any data yet, wait for packet to be + * submitted. If we already read some packets and there is no + * packet pending return back to user. */ + if (0 < copy_len) { + if (!kbasep_tlstream_packet_pending( + &stype, + &rb_idx_raw)) + break; + } else { + if (wait_event_interruptible( + tl_event_queue, + kbasep_tlstream_packet_pending( + &stype, + &rb_idx_raw))) { + copy_len = -ERESTARTSYS; + break; + } + } + + /* Check if this packet fits into the user buffer. + * If so copy its content. */ + rb_idx = rb_idx_raw % PACKET_COUNT; + rb_size = atomic_read(&tl_stream[stype]->buffer[rb_idx].size); + if (rb_size > size - copy_len) + break; + if (copy_to_user( + &buffer[copy_len], + tl_stream[stype]->buffer[rb_idx].data, + rb_size)) { + copy_len = -EFAULT; + break; + } + + /* If the rbi still points to the packet we just processed + * then there was no overflow so we add the copied size to + * copy_len and move rbi on to the next packet + */ + smp_rmb(); + if (atomic_read(&tl_stream[stype]->rbi) == rb_idx_raw) { + copy_len += rb_size; + atomic_inc(&tl_stream[stype]->rbi); + +#if MALI_UNIT_TEST + atomic_add(rb_size, &tlstream_bytes_collected); +#endif /* MALI_UNIT_TEST */ + } + } + + mutex_unlock(&tl_reader_lock); + + return copy_len; +} + +/** + * kbasep_tlstream_poll - poll timeline stream for packets + * @filp: pointer to file structure + * @wait: pointer to poll table + * Return: POLLIN if data can be read without blocking, otherwise zero + */ +static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait) +{ + enum tl_stream_type stream_type; + unsigned int rb_idx; + + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(wait); + + poll_wait(filp, &tl_event_queue, wait); + if (kbasep_tlstream_packet_pending(&stream_type, &rb_idx)) + return POLLIN; + return 0; +} + +/** + * kbasep_tlstream_release - release timeline stream descriptor + * @inode: pointer to inode structure + * @filp: pointer to file structure + * + * Return always return zero + */ +static int kbasep_tlstream_release(struct inode *inode, struct file *filp) +{ + KBASE_DEBUG_ASSERT(inode); + KBASE_DEBUG_ASSERT(filp); + CSTD_UNUSED(inode); + CSTD_UNUSED(filp); + + /* Stop autoflush timer before releasing access to streams. */ + atomic_set(&autoflush_timer_active, 0); + del_timer_sync(&autoflush_timer); + + atomic_set(&kbase_tlstream_enabled, 0); + return 0; +} + +/** + * kbasep_tlstream_timeline_header - prepare timeline header stream packet + * @stream_type: type of the stream that will carry header data + * @tp_desc: pointer to array with tracepoint descriptors + * @tp_count: number of descriptors in the given array + * + * Functions fills in information about tracepoints stored in body stream + * associated with this header stream. + */ +static void kbasep_tlstream_timeline_header( + enum tl_stream_type stream_type, + const struct tp_desc *tp_desc, + u32 tp_count) +{ + const u8 tv = SWTRACE_VERSION; /* protocol version */ + const u8 ps = sizeof(void *); /* pointer size */ + size_t msg_size = sizeof(tv) + sizeof(ps) + sizeof(tp_count); + char *buffer; + size_t pos = 0; + unsigned long flags; + unsigned int i; + + KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); + KBASE_DEBUG_ASSERT(tp_desc); + + /* Calculate the size of the timeline message. */ + for (i = 0; i < tp_count; i++) { + msg_size += sizeof(tp_desc[i].id); + msg_size += + strnlen(tp_desc[i].id_str, STRLEN_MAX) + + sizeof(char) + sizeof(u32); + msg_size += + strnlen(tp_desc[i].name, STRLEN_MAX) + + sizeof(char) + sizeof(u32); + msg_size += + strnlen(tp_desc[i].arg_types, STRLEN_MAX) + + sizeof(char) + sizeof(u32); + msg_size += + strnlen(tp_desc[i].arg_names, STRLEN_MAX) + + sizeof(char) + sizeof(u32); + } + + KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE >= msg_size); + + buffer = kbasep_tlstream_msgbuf_acquire(stream_type, msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &tv, sizeof(tv)); + pos = kbasep_tlstream_write_bytes(buffer, pos, &ps, sizeof(ps)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &tp_count, sizeof(tp_count)); + + for (i = 0; i < tp_count; i++) { + pos = kbasep_tlstream_write_bytes( + buffer, pos, + &tp_desc[i].id, sizeof(tp_desc[i].id)); + pos = kbasep_tlstream_write_string( + buffer, pos, + tp_desc[i].id_str, msg_size - pos); + pos = kbasep_tlstream_write_string( + buffer, pos, + tp_desc[i].name, msg_size - pos); + pos = kbasep_tlstream_write_string( + buffer, pos, + tp_desc[i].arg_types, msg_size - pos); + pos = kbasep_tlstream_write_string( + buffer, pos, + tp_desc[i].arg_names, msg_size - pos); + } + + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(stream_type, flags); + + /* We don't expect any more data to be read in this stream. + * As header stream must be read before its associated body stream, + * make this packet visible to the user straightaway. */ + kbasep_tlstream_flush_stream(stream_type); +} + +/*****************************************************************************/ + +int kbase_tlstream_init(void) +{ + enum tl_stream_type i; + + /* Prepare stream structures. */ + for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) { + tl_stream[i] = kmalloc(sizeof(**tl_stream), GFP_KERNEL); + if (!tl_stream[i]) + break; + kbasep_timeline_stream_init(tl_stream[i], i); + } + if (TL_STREAM_TYPE_COUNT > i) { + for (; i > 0; i--) { + kbasep_timeline_stream_term(tl_stream[i - 1]); + kfree(tl_stream[i - 1]); + } + return -ENOMEM; + } + + /* Initialize autoflush timer. */ + timer_setup(&autoflush_timer, + kbasep_tlstream_autoflush_timer_callback, + 0); + + return 0; +} + +void kbase_tlstream_term(void) +{ + enum tl_stream_type i; + + for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) { + kbasep_timeline_stream_term(tl_stream[i]); + kfree(tl_stream[i]); + } +} + +static void kbase_create_timeline_objects(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + unsigned int lpu_id; + unsigned int as_nr; + struct kbasep_kctx_list_element *element; + + /* Create LPU objects. */ + for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { + u32 *lpu = + &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; + KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, lpu_id, *lpu); + } + + /* Create Address Space objects. */ + for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) + KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(&kbdev->as[as_nr], as_nr); + + /* Create GPU object and make it retain all LPUs and address spaces. */ + KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU( + kbdev, + kbdev->gpu_props.props.raw_props.gpu_id, + kbdev->gpu_props.num_cores); + + for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { + void *lpu = + &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; + KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, kbdev); + } + for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) + KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU( + &kbdev->as[as_nr], + kbdev); + + /* Create object for each known context. */ + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry(element, &kbdev->kctx_list, link) { + KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX( + element->kctx, + element->kctx->id, + (u32)(element->kctx->tgid)); + } + /* Before releasing the lock, reset body stream buffers. + * This will prevent context creation message to be directed to both + * summary and body stream. + */ + kbase_tlstream_reset_body_streams(); + mutex_unlock(&kbdev->kctx_list_lock); + /* Static object are placed into summary packet that needs to be + * transmitted first. Flush all streams to make it available to + * user space. + */ + kbase_tlstream_flush_streams(); +} + +int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags) +{ + int ret; + u32 tlstream_enabled = TLSTREAM_ENABLED | flags; + + if (0 == atomic_cmpxchg(&kbase_tlstream_enabled, 0, tlstream_enabled)) { + int rcode; + + ret = anon_inode_getfd( + "[mali_tlstream]", + &kbasep_tlstream_fops, + kctx, + O_RDONLY | O_CLOEXEC); + if (ret < 0) { + atomic_set(&kbase_tlstream_enabled, 0); + return ret; + } + + /* Reset and initialize header streams. */ + kbasep_timeline_stream_reset( + tl_stream[TL_STREAM_TYPE_OBJ_HEADER]); + kbasep_timeline_stream_reset( + tl_stream[TL_STREAM_TYPE_OBJ_SUMMARY]); + kbasep_timeline_stream_reset( + tl_stream[TL_STREAM_TYPE_AUX_HEADER]); + kbasep_tlstream_timeline_header( + TL_STREAM_TYPE_OBJ_HEADER, + tp_desc_obj, + ARRAY_SIZE(tp_desc_obj)); + kbasep_tlstream_timeline_header( + TL_STREAM_TYPE_AUX_HEADER, + tp_desc_aux, + ARRAY_SIZE(tp_desc_aux)); + + /* Start autoflush timer. */ + atomic_set(&autoflush_timer_active, 1); + rcode = mod_timer( + &autoflush_timer, + jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); + CSTD_UNUSED(rcode); + + /* If job dumping is enabled, readjust the software event's + * timeout as the default value of 3 seconds is often + * insufficient. */ + if (flags & BASE_TLSTREAM_JOB_DUMPING_ENABLED) { + dev_info(kctx->kbdev->dev, + "Job dumping is enabled, readjusting the software event's timeout\n"); + atomic_set(&kctx->kbdev->js_data.soft_job_timeout_ms, + 1800000); + } + + /* Summary stream was cleared during acquire. + * Create static timeline objects that will be + * read by client. + */ + kbase_create_timeline_objects(kctx); + + } else { + ret = -EBUSY; + } + + return ret; +} + +void kbase_tlstream_flush_streams(void) +{ + enum tl_stream_type stype; + + for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) + kbasep_tlstream_flush_stream(stype); +} + +void kbase_tlstream_reset_body_streams(void) +{ + kbasep_timeline_stream_reset( + tl_stream[TL_STREAM_TYPE_OBJ]); + kbasep_timeline_stream_reset( + tl_stream[TL_STREAM_TYPE_AUX]); +} + +#if MALI_UNIT_TEST +void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated) +{ + KBASE_DEBUG_ASSERT(bytes_collected); + KBASE_DEBUG_ASSERT(bytes_generated); + *bytes_collected = atomic_read(&tlstream_bytes_collected); + *bytes_generated = atomic_read(&tlstream_bytes_generated); +} +#endif /* MALI_UNIT_TEST */ + +/*****************************************************************************/ + +void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid) +{ + const u32 msg_id = KBASE_TL_NEW_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) + + sizeof(tgid); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &nr, sizeof(nr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &tgid, sizeof(tgid)); + + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count) +{ + const u32 msg_id = KBASE_TL_NEW_GPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(id) + + sizeof(core_count); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &id, sizeof(id)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &core_count, sizeof(core_count)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn) +{ + const u32 msg_id = KBASE_TL_NEW_LPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(nr) + + sizeof(fn); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &nr, sizeof(nr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &fn, sizeof(fn)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu) +{ + const u32 msg_id = KBASE_TL_LIFELINK_LPU_GPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr) +{ + const u32 msg_id = KBASE_TL_NEW_AS; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(nr); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &nr, sizeof(nr)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu) +{ + const u32 msg_id = KBASE_TL_LIFELINK_AS_GPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +/*****************************************************************************/ + +void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid) +{ + const u32 msg_id = KBASE_TL_NEW_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) + + sizeof(tgid); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &nr, sizeof(nr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &tgid, sizeof(tgid)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_new_atom(void *atom, u32 nr) +{ + const u32 msg_id = KBASE_TL_NEW_ATOM; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + sizeof(atom) + + sizeof(nr); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &nr, sizeof(nr)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_del_ctx(void *context) +{ + const u32 msg_id = KBASE_TL_DEL_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(context); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_del_atom(void *atom) +{ + const u32 msg_id = KBASE_TL_DEL_ATOM; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu) +{ + const u32 msg_id = KBASE_TL_RET_CTX_LPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context) +{ + const u32 msg_id = KBASE_TL_RET_ATOM_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ret_atom_lpu( + void *atom, void *lpu, const char *attrib_match_list) +{ + const u32 msg_id = KBASE_TL_RET_ATOM_LPU; + const size_t msg_s0 = sizeof(u32) + sizeof(char) + + strnlen(attrib_match_list, STRLEN_MAX); + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + sizeof(lpu) + msg_s0; + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + pos = kbasep_tlstream_write_string( + buffer, pos, attrib_match_list, msg_s0); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu) +{ + const u32 msg_id = KBASE_TL_NRET_CTX_LPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context) +{ + const u32 msg_id = KBASE_TL_NRET_ATOM_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2) +{ + const u32 msg_id = KBASE_TL_DEP_ATOM_ATOM; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom1, sizeof(atom1)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom2, sizeof(atom2)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2) +{ + const u32 msg_id = KBASE_TL_NDEP_ATOM_ATOM; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom1, sizeof(atom1)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom2, sizeof(atom2)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2) +{ + const u32 msg_id = KBASE_TL_RDEP_ATOM_ATOM; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom1, sizeof(atom1)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom2, sizeof(atom2)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu) +{ + const u32 msg_id = KBASE_TL_NRET_ATOM_LPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx) +{ + const u32 msg_id = KBASE_TL_RET_AS_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &ctx, sizeof(ctx)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx) +{ + const u32 msg_id = KBASE_TL_NRET_AS_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &ctx, sizeof(ctx)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as) +{ + const u32 msg_id = KBASE_TL_RET_ATOM_AS; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as) +{ + const u32 msg_id = KBASE_TL_NRET_ATOM_AS; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_atom_config( + void *atom, u64 jd, u64 affinity, u32 config) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_CONFIG; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + + sizeof(jd) + sizeof(affinity) + sizeof(config); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &jd, sizeof(jd)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &affinity, sizeof(affinity)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &config, sizeof(config)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(prio); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &prio, sizeof(prio)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_STATE; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(state); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &state, sizeof(state)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_atom_jit( + void *atom, u64 edit_addr, u64 new_addr) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + + sizeof(edit_addr) + sizeof(new_addr); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &edit_addr, sizeof(edit_addr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &new_addr, sizeof(new_addr)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_as_config( + void *as, u64 transtab, u64 memattr, u64 transcfg) +{ + const u32 msg_id = KBASE_TL_ATTRIB_AS_CONFIG; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(as) + + sizeof(transtab) + sizeof(memattr) + sizeof(transcfg); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &transtab, sizeof(transtab)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &memattr, sizeof(memattr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &transcfg, sizeof(transcfg)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_event_lpu_softstop(void *lpu) +{ + const u32 msg_id = KBASE_TL_EVENT_LPU_SOFTSTOP; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(lpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom) +{ + const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_EX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_event_atom_softstop_issue(void *atom) +{ + const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_jd_gpu_soft_reset(void *gpu) +{ + const u32 msg_id = KBASE_JD_GPU_SOFT_RESET; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +/*****************************************************************************/ + +void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state) +{ + const u32 msg_id = KBASE_AUX_PM_STATE; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(core_type) + + sizeof(state); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &core_type, sizeof(core_type)); + pos = kbasep_tlstream_write_bytes(buffer, pos, &state, sizeof(state)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} + +void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change) +{ + const u32 msg_id = KBASE_AUX_PAGEFAULT; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) + + sizeof(page_count_change); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, + &page_count_change, sizeof(page_count_change)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} + +void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count) +{ + const u32 msg_id = KBASE_AUX_PAGESALLOC; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) + + sizeof(page_count); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &page_count, sizeof(page_count)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} + +void __kbase_tlstream_aux_devfreq_target(u64 target_freq) +{ + const u32 msg_id = KBASE_AUX_DEVFREQ_TARGET; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(target_freq); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &target_freq, sizeof(target_freq)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} + +void __kbase_tlstream_aux_protected_enter_start(void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_START; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} +void __kbase_tlstream_aux_protected_enter_end(void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_END; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} + +void __kbase_tlstream_aux_protected_leave_start(void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_START; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} +void __kbase_tlstream_aux_protected_leave_end(void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_END; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.h new file mode 100755 index 000000000000..c0a1117d5f25 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_tlstream.h @@ -0,0 +1,623 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#if !defined(_KBASE_TLSTREAM_H) +#define _KBASE_TLSTREAM_H + +#include + +/*****************************************************************************/ + +/** + * kbase_tlstream_init - initialize timeline infrastructure in kernel + * Return: zero on success, negative number on error + */ +int kbase_tlstream_init(void); + +/** + * kbase_tlstream_term - terminate timeline infrastructure in kernel + * + * Timeline need have to been previously enabled with kbase_tlstream_init(). + */ +void kbase_tlstream_term(void); + +/** + * kbase_tlstream_acquire - acquire timeline stream file descriptor + * @kctx: kernel common context + * @flags: timeline stream flags + * + * This descriptor is meant to be used by userspace timeline to gain access to + * kernel timeline stream. This stream is later broadcasted by user space to the + * timeline client. + * Only one entity can own the descriptor at any given time. Descriptor shall be + * closed if unused. If descriptor cannot be obtained (i.e. when it is already + * being used) return will be a negative value. + * + * Return: file descriptor on success, negative number on error + */ +int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags); + +/** + * kbase_tlstream_flush_streams - flush timeline streams. + * + * Function will flush pending data in all timeline streams. + */ +void kbase_tlstream_flush_streams(void); + +/** + * kbase_tlstream_reset_body_streams - reset timeline body streams. + * + * Function will discard pending data in all timeline body streams. + */ +void kbase_tlstream_reset_body_streams(void); + +#if MALI_UNIT_TEST +/** + * kbase_tlstream_test - start timeline stream data generator + * @tpw_count: number of trace point writers in each context + * @msg_delay: time delay in milliseconds between trace points written by one + * writer + * @msg_count: number of trace points written by one writer + * @aux_msg: if non-zero aux messages will be included + * + * This test starts a requested number of asynchronous writers in both IRQ and + * thread context. Each writer will generate required number of test + * tracepoints (tracepoints with embedded information about writer that + * should be verified by user space reader). Tracepoints will be emitted in + * all timeline body streams. If aux_msg is non-zero writer will also + * generate not testable tracepoints (tracepoints without information about + * writer). These tracepoints are used to check correctness of remaining + * timeline message generating functions. Writer will wait requested time + * between generating another set of messages. This call blocks until all + * writers finish. + */ +void kbase_tlstream_test( + unsigned int tpw_count, + unsigned int msg_delay, + unsigned int msg_count, + int aux_msg); + +/** + * kbase_tlstream_stats - read timeline stream statistics + * @bytes_collected: will hold number of bytes read by the user + * @bytes_generated: will hold number of bytes generated by trace points + */ +void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated); +#endif /* MALI_UNIT_TEST */ + +/*****************************************************************************/ + +#define TL_ATOM_STATE_IDLE 0 +#define TL_ATOM_STATE_READY 1 +#define TL_ATOM_STATE_DONE 2 +#define TL_ATOM_STATE_POSTED 3 + +void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid); +void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count); +void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn); +void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu); +void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr); +void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu); +void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid); +void __kbase_tlstream_tl_new_atom(void *atom, u32 nr); +void __kbase_tlstream_tl_del_ctx(void *context); +void __kbase_tlstream_tl_del_atom(void *atom); +void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu); +void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context); +void __kbase_tlstream_tl_ret_atom_lpu( + void *atom, void *lpu, const char *attrib_match_list); +void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu); +void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context); +void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu); +void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx); +void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx); +void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as); +void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as); +void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2); +void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2); +void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2); +void __kbase_tlstream_tl_attrib_atom_config( + void *atom, u64 jd, u64 affinity, u32 config); +void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio); +void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state); +void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom); +void __kbase_tlstream_tl_attrib_atom_jit( + void *atom, u64 edit_addr, u64 new_addr); +void __kbase_tlstream_tl_attrib_as_config( + void *as, u64 transtab, u64 memattr, u64 transcfg); +void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom); +void __kbase_tlstream_tl_event_lpu_softstop(void *lpu); +void __kbase_tlstream_tl_event_atom_softstop_issue(void *atom); +void __kbase_tlstream_jd_gpu_soft_reset(void *gpu); +void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state); +void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change); +void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count); +void __kbase_tlstream_aux_devfreq_target(u64 target_freq); +void __kbase_tlstream_aux_protected_enter_start(void *gpu); +void __kbase_tlstream_aux_protected_enter_end(void *gpu); +void __kbase_tlstream_aux_protected_leave_start(void *gpu); +void __kbase_tlstream_aux_protected_leave_end(void *gpu); + +#define TLSTREAM_ENABLED (1 << 31) + +extern atomic_t kbase_tlstream_enabled; + +#define __TRACE_IF_ENABLED(trace_name, ...) \ + do { \ + int enabled = atomic_read(&kbase_tlstream_enabled); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_##trace_name(__VA_ARGS__); \ + } while (0) + +#define __TRACE_IF_ENABLED_LATENCY(trace_name, ...) \ + do { \ + int enabled = atomic_read(&kbase_tlstream_enabled); \ + if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ + __kbase_tlstream_##trace_name(__VA_ARGS__); \ + } while (0) + +#define __TRACE_IF_ENABLED_JD(trace_name, ...) \ + do { \ + int enabled = atomic_read(&kbase_tlstream_enabled); \ + if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ + __kbase_tlstream_##trace_name(__VA_ARGS__); \ + } while (0) + +/*****************************************************************************/ + +/** + * KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX - create context object in timeline + * summary + * @context: name of the context object + * @nr: context number + * @tgid: thread Group Id + * + * Function emits a timeline message informing about context creation. Context + * is created with context number (its attribute), that can be used to link + * kbase context with userspace context. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX(context, nr, tgid) \ + __TRACE_IF_ENABLED(tl_summary_new_ctx, context, nr, tgid) + +/** + * KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU - create GPU object in timeline summary + * @gpu: name of the GPU object + * @id: id value of this GPU + * @core_count: number of cores this GPU hosts + * + * Function emits a timeline message informing about GPU creation. GPU is + * created with two attributes: id and core count. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU(gpu, id, core_count) \ + __TRACE_IF_ENABLED(tl_summary_new_gpu, gpu, id, core_count) + +/** + * KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU - create LPU object in timeline summary + * @lpu: name of the Logical Processing Unit object + * @nr: sequential number assigned to this LPU + * @fn: property describing this LPU's functional abilities + * + * Function emits a timeline message informing about LPU creation. LPU is + * created with two attributes: number linking this LPU with GPU's job slot + * and function bearing information about this LPU abilities. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, nr, fn) \ + __TRACE_IF_ENABLED(tl_summary_new_lpu, lpu, nr, fn) + +/** + * KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU - lifelink LPU object to GPU + * @lpu: name of the Logical Processing Unit object + * @gpu: name of the GPU object + * + * Function emits a timeline message informing that LPU object shall be deleted + * along with GPU object. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, gpu) \ + __TRACE_IF_ENABLED(tl_summary_lifelink_lpu_gpu, lpu, gpu) + +/** + * KBASE_TLSTREAM_TL_SUMMARY_NEW_AS - create address space object in timeline summary + * @as: name of the address space object + * @nr: sequential number assigned to this address space + * + * Function emits a timeline message informing about address space creation. + * Address space is created with one attribute: number identifying this + * address space. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(as, nr) \ + __TRACE_IF_ENABLED(tl_summary_new_as, as, nr) + +/** + * KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU - lifelink address space object to GPU + * @as: name of the address space object + * @gpu: name of the GPU object + * + * Function emits a timeline message informing that address space object + * shall be deleted along with GPU object. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU(as, gpu) \ + __TRACE_IF_ENABLED(tl_summary_lifelink_as_gpu, as, gpu) + +/** + * KBASE_TLSTREAM_TL_NEW_CTX - create context object in timeline + * @context: name of the context object + * @nr: context number + * @tgid: thread Group Id + * + * Function emits a timeline message informing about context creation. Context + * is created with context number (its attribute), that can be used to link + * kbase context with userspace context. + */ +#define KBASE_TLSTREAM_TL_NEW_CTX(context, nr, tgid) \ + __TRACE_IF_ENABLED(tl_new_ctx, context, nr, tgid) + +/** + * KBASE_TLSTREAM_TL_NEW_ATOM - create atom object in timeline + * @atom: name of the atom object + * @nr: sequential number assigned to this atom + * + * Function emits a timeline message informing about atom creation. Atom is + * created with atom number (its attribute) that links it with actual work + * bucket id understood by hardware. + */ +#define KBASE_TLSTREAM_TL_NEW_ATOM(atom, nr) \ + __TRACE_IF_ENABLED(tl_new_atom, atom, nr) + +/** + * KBASE_TLSTREAM_TL_DEL_CTX - destroy context object in timeline + * @context: name of the context object + * + * Function emits a timeline message informing that context object ceased to + * exist. + */ +#define KBASE_TLSTREAM_TL_DEL_CTX(context) \ + __TRACE_IF_ENABLED(tl_del_ctx, context) + +/** + * KBASE_TLSTREAM_TL_DEL_ATOM - destroy atom object in timeline + * @atom: name of the atom object + * + * Function emits a timeline message informing that atom object ceased to + * exist. + */ +#define KBASE_TLSTREAM_TL_DEL_ATOM(atom) \ + __TRACE_IF_ENABLED(tl_del_atom, atom) + +/** + * KBASE_TLSTREAM_TL_RET_CTX_LPU - retain context by LPU + * @context: name of the context object + * @lpu: name of the Logical Processing Unit object + * + * Function emits a timeline message informing that context is being held + * by LPU and must not be deleted unless it is released. + */ +#define KBASE_TLSTREAM_TL_RET_CTX_LPU(context, lpu) \ + __TRACE_IF_ENABLED(tl_ret_ctx_lpu, context, lpu) + +/** + * KBASE_TLSTREAM_TL_RET_ATOM_CTX - retain atom by context + * @atom: name of the atom object + * @context: name of the context object + * + * Function emits a timeline message informing that atom object is being held + * by context and must not be deleted unless it is released. + */ +#define KBASE_TLSTREAM_TL_RET_ATOM_CTX(atom, context) \ + __TRACE_IF_ENABLED(tl_ret_atom_ctx, atom, context) + +/** + * KBASE_TLSTREAM_TL_RET_ATOM_LPU - retain atom by LPU + * @atom: name of the atom object + * @lpu: name of the Logical Processing Unit object + * @attrib_match_list: list containing match operator attributes + * + * Function emits a timeline message informing that atom object is being held + * by LPU and must not be deleted unless it is released. + */ +#define KBASE_TLSTREAM_TL_RET_ATOM_LPU(atom, lpu, attrib_match_list) \ + __TRACE_IF_ENABLED(tl_ret_atom_lpu, atom, lpu, attrib_match_list) + +/** + * KBASE_TLSTREAM_TL_NRET_CTX_LPU - release context by LPU + * @context: name of the context object + * @lpu: name of the Logical Processing Unit object + * + * Function emits a timeline message informing that context is being released + * by LPU object. + */ +#define KBASE_TLSTREAM_TL_NRET_CTX_LPU(context, lpu) \ + __TRACE_IF_ENABLED(tl_nret_ctx_lpu, context, lpu) + +/** + * KBASE_TLSTREAM_TL_NRET_ATOM_CTX - release atom by context + * @atom: name of the atom object + * @context: name of the context object + * + * Function emits a timeline message informing that atom object is being + * released by context. + */ +#define KBASE_TLSTREAM_TL_NRET_ATOM_CTX(atom, context) \ + __TRACE_IF_ENABLED(tl_nret_atom_ctx, atom, context) + +/** + * KBASE_TLSTREAM_TL_NRET_ATOM_LPU - release atom by LPU + * @atom: name of the atom object + * @lpu: name of the Logical Processing Unit object + * + * Function emits a timeline message informing that atom object is being + * released by LPU. + */ +#define KBASE_TLSTREAM_TL_NRET_ATOM_LPU(atom, lpu) \ + __TRACE_IF_ENABLED(tl_nret_atom_lpu, atom, lpu) + +/** + * KBASE_TLSTREAM_TL_RET_AS_CTX - lifelink address space object to context + * @as: name of the address space object + * @ctx: name of the context object + * + * Function emits a timeline message informing that address space object + * is being held by the context object. + */ +#define KBASE_TLSTREAM_TL_RET_AS_CTX(as, ctx) \ + __TRACE_IF_ENABLED(tl_ret_as_ctx, as, ctx) + +/** + * KBASE_TLSTREAM_TL_NRET_AS_CTX - release address space by context + * @as: name of the address space object + * @ctx: name of the context object + * + * Function emits a timeline message informing that address space object + * is being released by atom. + */ +#define KBASE_TLSTREAM_TL_NRET_AS_CTX(as, ctx) \ + __TRACE_IF_ENABLED(tl_nret_as_ctx, as, ctx) + +/** + * KBASE_TLSTREAM_TL_RET_ATOM_AS - retain atom by address space + * @atom: name of the atom object + * @as: name of the address space object + * + * Function emits a timeline message informing that atom object is being held + * by address space and must not be deleted unless it is released. + */ +#define KBASE_TLSTREAM_TL_RET_ATOM_AS(atom, as) \ + __TRACE_IF_ENABLED(tl_ret_atom_as, atom, as) + +/** + * KBASE_TLSTREAM_TL_NRET_ATOM_AS - release atom by address space + * @atom: name of the atom object + * @as: name of the address space object + * + * Function emits a timeline message informing that atom object is being + * released by address space. + */ +#define KBASE_TLSTREAM_TL_NRET_ATOM_AS(atom, as) \ + __TRACE_IF_ENABLED(tl_nret_atom_as, atom, as) + +/** + * KBASE_TLSTREAM_TL_DEP_ATOM_ATOM - parent atom depends on child atom + * @atom1: name of the child atom object + * @atom2: name of the parent atom object that depends on child atom + * + * Function emits a timeline message informing that parent atom waits for + * child atom object to be completed before start its execution. + */ +#define KBASE_TLSTREAM_TL_DEP_ATOM_ATOM(atom1, atom2) \ + __TRACE_IF_ENABLED(tl_dep_atom_atom, atom1, atom2) + +/** + * KBASE_TLSTREAM_TL_NDEP_ATOM_ATOM - dependency between atoms resolved + * @atom1: name of the child atom object + * @atom2: name of the parent atom object that depended on child atom + * + * Function emits a timeline message informing that parent atom execution + * dependency on child atom has been resolved. + */ +#define KBASE_TLSTREAM_TL_NDEP_ATOM_ATOM(atom1, atom2) \ + __TRACE_IF_ENABLED(tl_ndep_atom_atom, atom1, atom2) + +/** + * KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM - information about already resolved dependency between atoms + * @atom1: name of the child atom object + * @atom2: name of the parent atom object that depended on child atom + * + * Function emits a timeline message informing that parent atom execution + * dependency on child atom has been resolved. + */ +#define KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM(atom1, atom2) \ + __TRACE_IF_ENABLED(tl_rdep_atom_atom, atom1, atom2) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG - atom job slot attributes + * @atom: name of the atom object + * @jd: job descriptor address + * @affinity: job affinity + * @config: job config + * + * Function emits a timeline message containing atom attributes. + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(atom, jd, affinity, config) \ + __TRACE_IF_ENABLED(tl_attrib_atom_config, atom, jd, affinity, config) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY - atom priority + * @atom: name of the atom object + * @prio: atom priority + * + * Function emits a timeline message containing atom priority. + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(atom, prio) \ + __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority, atom, prio) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE - atom state + * @atom: name of the atom object + * @state: atom state + * + * Function emits a timeline message containing atom state. + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, state) \ + __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_state, atom, state) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE - atom caused priority change + * @atom: name of the atom object + * + * Function emits a timeline message signalling priority change + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE(atom) \ + __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority_change, atom) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - jit happened on atom + * @atom: atom identifier + * @edit_addr: address edited by jit + * @new_addr: address placed into the edited location + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(atom, edit_addr, new_addr) \ + __TRACE_IF_ENABLED_JD(tl_attrib_atom_jit, atom, edit_addr, new_addr) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG - address space attributes + * @as: assigned address space + * @transtab: configuration of the TRANSTAB register + * @memattr: configuration of the MEMATTR register + * @transcfg: configuration of the TRANSCFG register (or zero if not present) + * + * Function emits a timeline message containing address space attributes. + */ +#define KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(as, transtab, memattr, transcfg) \ + __TRACE_IF_ENABLED(tl_attrib_as_config, as, transtab, memattr, transcfg) + +/** + * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ex + * @atom: atom identifier + */ +#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(atom) \ + __TRACE_IF_ENABLED(tl_event_atom_softstop_ex, atom) + +/** + * KBASE_TLSTREAM_TL_EVENT_LPU_softstop + * @lpu: name of the LPU object + */ +#define KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP(lpu) \ + __TRACE_IF_ENABLED(tl_event_lpu_softstop, lpu) + +/** + * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_issue + * @atom: atom identifier + */ +#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(atom) \ + __TRACE_IF_ENABLED(tl_event_atom_softstop_issue, atom) + +/** + * KBASE_TLSTREAM_JD_GPU_SOFT_RESET - The GPU is being soft reset + * @gpu: name of the GPU object + * + * This imperative tracepoint is specific to job dumping. + * Function emits a timeline message indicating GPU soft reset. + */ +#define KBASE_TLSTREAM_JD_GPU_SOFT_RESET(gpu) \ + __TRACE_IF_ENABLED(jd_gpu_soft_reset, gpu) + + +/** + * KBASE_TLSTREAM_AUX_PM_STATE - timeline message: power management state + * @core_type: core type (shader, tiler, l2 cache, l3 cache) + * @state: 64bits bitmask reporting power state of the cores (1-ON, 0-OFF) + */ +#define KBASE_TLSTREAM_AUX_PM_STATE(core_type, state) \ + __TRACE_IF_ENABLED(aux_pm_state, core_type, state) + +/** + * KBASE_TLSTREAM_AUX_PAGEFAULT - timeline message: MMU page fault event + * resulting in new pages being mapped + * @ctx_nr: kernel context number + * @page_count_change: number of pages to be added + */ +#define KBASE_TLSTREAM_AUX_PAGEFAULT(ctx_nr, page_count_change) \ + __TRACE_IF_ENABLED(aux_pagefault, ctx_nr, page_count_change) + +/** + * KBASE_TLSTREAM_AUX_PAGESALLOC - timeline message: total number of allocated + * pages is changed + * @ctx_nr: kernel context number + * @page_count: number of pages used by the context + */ +#define KBASE_TLSTREAM_AUX_PAGESALLOC(ctx_nr, page_count) \ + __TRACE_IF_ENABLED(aux_pagesalloc, ctx_nr, page_count) + +/** + * KBASE_TLSTREAM_AUX_DEVFREQ_TARGET - timeline message: new target DVFS + * frequency + * @target_freq: new target frequency + */ +#define KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(target_freq) \ + __TRACE_IF_ENABLED(aux_devfreq_target, target_freq) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START - The GPU has started transitioning + * to protected mode + * @gpu: name of the GPU object + * + * Function emits a timeline message indicating the GPU is starting to + * transition to protected mode. + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(gpu) \ + __TRACE_IF_ENABLED_LATENCY(aux_protected_enter_start, gpu) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END - The GPU has finished transitioning + * to protected mode + * @gpu: name of the GPU object + * + * Function emits a timeline message indicating the GPU has finished + * transitioning to protected mode. + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(gpu) \ + __TRACE_IF_ENABLED_LATENCY(aux_protected_enter_end, gpu) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START - The GPU has started transitioning + * to non-protected mode + * @gpu: name of the GPU object + * + * Function emits a timeline message indicating the GPU is starting to + * transition to non-protected mode. + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(gpu) \ + __TRACE_IF_ENABLED_LATENCY(aux_protected_leave_start, gpu) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END - The GPU has finished transitioning + * to non-protected mode + * @gpu: name of the GPU object + * + * Function emits a timeline message indicating the GPU has finished + * transitioning to non-protected mode. + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(gpu) \ + __TRACE_IF_ENABLED_LATENCY(aux_protected_leave_end, gpu) + +#endif /* _KBASE_TLSTREAM_H */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_defs.h new file mode 100755 index 000000000000..e2e0544208ce --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_defs.h @@ -0,0 +1,264 @@ +/* + * + * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** + * ***** DO NOT INCLUDE DIRECTLY ***** + * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ + +/* + * The purpose of this header file is just to contain a list of trace code idenitifers + * + * Each identifier is wrapped in a macro, so that its string form and enum form can be created + * + * Each macro is separated with a comma, to allow insertion into an array initializer or enum definition block. + * + * This allows automatic creation of an enum and a corresponding array of strings + * + * Before #including, the includer MUST #define KBASE_TRACE_CODE_MAKE_CODE. + * After #including, the includer MUST #under KBASE_TRACE_CODE_MAKE_CODE. + * + * e.g.: + * #define KBASE_TRACE_CODE( X ) KBASE_TRACE_CODE_ ## X + * typedef enum + * { + * #define KBASE_TRACE_CODE_MAKE_CODE( X ) KBASE_TRACE_CODE( X ) + * #include "mali_kbase_trace_defs.h" + * #undef KBASE_TRACE_CODE_MAKE_CODE + * } kbase_trace_code; + * + * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE + * + * + * The use of the macro here is: + * - KBASE_TRACE_CODE_MAKE_CODE( X ) + * + * Which produces: + * - For an enum, KBASE_TRACE_CODE_X + * - For a string, "X" + * + * + * For example: + * - KBASE_TRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to: + * - KBASE_TRACE_CODE_JM_JOB_COMPLETE for the enum + * - "JM_JOB_COMPLETE" for the string + * - To use it to trace an event, do: + * - KBASE_TRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val ); + */ + +#if 0 /* Dummy section to avoid breaking formatting */ +int dummy_array[] = { +#endif + +/* + * Core events + */ + /* no info_val, no gpu_addr, no atom */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), + /* no info_val, no gpu_addr, no atom */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM), + /* info_val == GPU_IRQ_STATUS register */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ), + /* info_val == bits cleared */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR), + /* info_val == GPU_IRQ_STATUS register */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE), + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET), + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET), + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR), + /* GPU addr==dump address */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE), + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES), +/* + * Job Slot management events + */ + /* info_val==irq rawstat at start */ + KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ), + /* info_val==jobs processed */ + KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ_END), +/* In the following: + * + * - ctx is set if a corresponding job found (NULL otherwise, e.g. some soft-stop cases) + * - uatom==kernel-side mapped uatom address (for correlation with user-side) + */ + /* info_val==exit code; gpu_addr==chain gpuaddr */ + KBASE_TRACE_CODE_MAKE_CODE(JM_JOB_DONE), + /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT), + /* gpu_addr is as follows: + * - If JS_STATUS active after soft-stop, val==gpu addr written to + * JS_HEAD on submit + * - otherwise gpu_addr==0 */ + KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP), + KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), + KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), + /* gpu_addr==JS_HEAD read */ + KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP), + /* gpu_addr==JS_HEAD read */ + KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), + /* gpu_addr==JS_HEAD read */ + KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), + /* gpu_addr==JS_TAIL read */ + KBASE_TRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), +/* gpu_addr is as follows: + * - If JS_STATUS active before soft-stop, val==JS_HEAD + * - otherwise gpu_addr==0 + */ + /* gpu_addr==JS_HEAD read */ + KBASE_TRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), + KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), + KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), + /* info_val == is_scheduled */ + KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), + /* info_val == is_scheduled */ + KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), + KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_DONE), + /* info_val == nr jobs submitted */ + KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), + /* gpu_addr==JS_HEAD_NEXT last written */ + KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), + KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), + KBASE_TRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), + KBASE_TRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), +/* + * Job dispatch events + */ + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_DONE), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), + /* gpu_addr==0, info_val==0, uatom==0 */ + KBASE_TRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), +/* + * Scheduler Core events + */ + KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX_NOLOCK), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JS_ADD_JOB), + /* gpu_addr==last value written/would be written to JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), + KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_RELEASE_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), + /* kctx is the one being evicted, info_val == kctx to put in */ + KBASE_TRACE_CODE_MAKE_CODE(JS_FAST_START_EVICTS_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), + /* info_val == lower 32 bits of affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), + /* info_val == lower 32 bits of affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), + /* info_val == lower 32 bits of affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), + /* info_val == lower 32 bits of rechecked affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), + /* info_val == lower 32 bits of rechecked affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), + /* info_val == lower 32 bits of affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), + /* info_val == the ctx attribute now on ctx */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), + /* info_val == the ctx attribute now on runpool */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), + /* info_val == the ctx attribute now off ctx */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), + /* info_val == the ctx attribute now off runpool */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), +/* + * Scheduler Policy events + */ + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), + /* info_val == whether it was evicted */ + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), + /* gpu_addr==JS_HEAD to write if the job were run */ + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), +/* + * Power Management Events + */ + KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP), + KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_L2), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_L2), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER), + /* PM_DESIRED_REACHED: gpu_addr == pm.gpu_in_desired_state */ + KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED), + KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_SHADER_INUSE), + KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_TILER_INUSE), + KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_SHADER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_TILER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_INUSE), + KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_INUSE), + KBASE_TRACE_CODE_MAKE_CODE(PM_UNREQUEST_CHANGE_SHADER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_UNREQUEST_CHANGE_TILER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS), + KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE), + KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE), + KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_ON), + KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_OFF), + /* info_val == policy number, or -1 for "Already changing" */ + KBASE_TRACE_CODE_MAKE_CODE(PM_SET_POLICY), + KBASE_TRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY), + /* info_val == policy number */ + KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT), + /* info_val == policy number */ + KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM), +/* Unused code just to make it easier to not have a comma at the end. + * All other codes MUST come before this */ + KBASE_TRACE_CODE_MAKE_CODE(DUMMY) + +#if 0 /* Dummy section to avoid breaking formatting */ +}; +#endif + +/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.c new file mode 100755 index 000000000000..d9854749f45b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.c @@ -0,0 +1,236 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include + +#define CREATE_TRACE_POINTS + +#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE +#include "mali_timeline.h" + +#include +#include + +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atoms_in_flight); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atom); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_active); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_action); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_power_active); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_l2_power_active); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_event); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_slot_atom); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_checktrans); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_context_active); + +struct kbase_trace_timeline_desc { + char *enum_str; + char *desc; + char *format; + char *format_desc; +}; + +static struct kbase_trace_timeline_desc kbase_trace_timeline_desc_table[] = { + #define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) { #enum_val, desc, format, format_desc } + #include "mali_kbase_trace_timeline_defs.h" + #undef KBASE_TIMELINE_TRACE_CODE +}; + +#define KBASE_NR_TRACE_CODES ARRAY_SIZE(kbase_trace_timeline_desc_table) + +static void *kbasep_trace_timeline_seq_start(struct seq_file *s, loff_t *pos) +{ + if (*pos >= KBASE_NR_TRACE_CODES) + return NULL; + + return &kbase_trace_timeline_desc_table[*pos]; +} + +static void kbasep_trace_timeline_seq_stop(struct seq_file *s, void *data) +{ +} + +static void *kbasep_trace_timeline_seq_next(struct seq_file *s, void *data, loff_t *pos) +{ + (*pos)++; + + if (*pos == KBASE_NR_TRACE_CODES) + return NULL; + + return &kbase_trace_timeline_desc_table[*pos]; +} + +static int kbasep_trace_timeline_seq_show(struct seq_file *s, void *data) +{ + struct kbase_trace_timeline_desc *trace_desc = data; + + seq_printf(s, "%s#%s#%s#%s\n", trace_desc->enum_str, trace_desc->desc, trace_desc->format, trace_desc->format_desc); + return 0; +} + + +static const struct seq_operations kbasep_trace_timeline_seq_ops = { + .start = kbasep_trace_timeline_seq_start, + .next = kbasep_trace_timeline_seq_next, + .stop = kbasep_trace_timeline_seq_stop, + .show = kbasep_trace_timeline_seq_show, +}; + +static int kbasep_trace_timeline_debugfs_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &kbasep_trace_timeline_seq_ops); +} + +static const struct file_operations kbasep_trace_timeline_debugfs_fops = { + .open = kbasep_trace_timeline_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +#ifdef CONFIG_DEBUG_FS + +void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("mali_timeline_defs", + S_IRUGO, kbdev->mali_debugfs_directory, NULL, + &kbasep_trace_timeline_debugfs_fops); +} + +#endif /* CONFIG_DEBUG_FS */ + +void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->timeline.slot_atoms_submitted[js] > 0) { + KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 1); + } else { + base_atom_id atom_number = kbase_jd_atom_id(kctx, katom); + + KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 1); + KBASE_TIMELINE_JOB_START(kctx, js, atom_number); + } + ++kbdev->timeline.slot_atoms_submitted[js]; + + KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, kbdev->timeline.slot_atoms_submitted[js]); +} + +void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js, + kbasep_js_atom_done_code done_code) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) { + KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 0); + } else { + /* Job finished in JS_HEAD */ + base_atom_id atom_number = kbase_jd_atom_id(kctx, katom); + + KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 0); + KBASE_TIMELINE_JOB_STOP(kctx, js, atom_number); + + /* see if we need to trace the job in JS_NEXT moving to JS_HEAD */ + if (kbase_backend_nr_atoms_submitted(kbdev, js)) { + struct kbase_jd_atom *next_katom; + struct kbase_context *next_kctx; + + /* Peek the next atom - note that the atom in JS_HEAD will already + * have been dequeued */ + next_katom = kbase_backend_inspect_head(kbdev, js); + WARN_ON(!next_katom); + next_kctx = next_katom->kctx; + KBASE_TIMELINE_JOB_START_NEXT(next_kctx, js, 0); + KBASE_TIMELINE_JOB_START_HEAD(next_kctx, js, 1); + KBASE_TIMELINE_JOB_START(next_kctx, js, kbase_jd_atom_id(next_kctx, next_katom)); + } + } + + --kbdev->timeline.slot_atoms_submitted[js]; + + KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, kbdev->timeline.slot_atoms_submitted[js]); +} + +void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent) +{ + int uid = 0; + int old_uid; + + /* If a producer already exists for the event, try to use their UID (multiple-producers) */ + uid = atomic_read(&kbdev->timeline.pm_event_uid[event_sent]); + old_uid = uid; + + /* Get a new non-zero UID if we don't have one yet */ + while (!uid) + uid = atomic_inc_return(&kbdev->timeline.pm_event_uid_counter); + + /* Try to use this UID */ + if (old_uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event_sent], old_uid, uid)) + /* If it changed, raced with another producer: we've lost this UID */ + uid = 0; + + KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_sent, uid); +} + +void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) +{ + int uid = atomic_read(&kbdev->timeline.pm_event_uid[event]); + + if (uid != 0) { + if (uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event], uid, 0)) + /* If it changed, raced with another consumer: we've lost this UID */ + uid = 0; + + KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event, uid); + } +} + +void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) +{ + int uid = atomic_read(&kbdev->timeline.pm_event_uid[event]); + + if (uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event], uid, 0)) + /* If it changed, raced with another consumer: we've lost this UID */ + uid = 0; + + KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event, uid); +} + +void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + /* Simply log the start of the transition */ + kbdev->timeline.l2_transitioning = true; + KBASE_TIMELINE_POWERING_L2(kbdev); +} + +void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + /* Simply log the end of the transition */ + if (kbdev->timeline.l2_transitioning) { + kbdev->timeline.l2_transitioning = false; + KBASE_TIMELINE_POWERED_L2(kbdev); + } +} + +#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.h new file mode 100755 index 000000000000..4b517f396f8c --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline.h @@ -0,0 +1,363 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#if !defined(_KBASE_TRACE_TIMELINE_H) +#define _KBASE_TRACE_TIMELINE_H + +#ifdef CONFIG_MALI_BIFROST_TRACE_TIMELINE + +enum kbase_trace_timeline_code { + #define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) enum_val + #include "mali_kbase_trace_timeline_defs.h" + #undef KBASE_TIMELINE_TRACE_CODE +}; + +#ifdef CONFIG_DEBUG_FS + +/** Initialize Timeline DebugFS entries */ +void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev); + +#else /* CONFIG_DEBUG_FS */ + +#define kbasep_trace_timeline_debugfs_init CSTD_NOP + +#endif /* CONFIG_DEBUG_FS */ + +/* mali_timeline.h defines kernel tracepoints used by the KBASE_TIMELINE + * functions. + * Output is timestamped by either sched_clock() (default), local_clock(), or + * cpu_clock(), depending on /sys/kernel/debug/tracing/trace_clock */ +#include "mali_timeline.h" + +/* Trace number of atoms in flight for kctx (atoms either not completed, or in + process of being returned to user */ +#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_atoms_in_flight(ts.tv_sec, ts.tv_nsec, \ + (int)kctx->timeline.owner_tgid, \ + count); \ + } while (0) + +/* Trace atom_id being Ready to Run */ +#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_atom(ts.tv_sec, ts.tv_nsec, \ + CTX_FLOW_ATOM_READY, \ + (int)kctx->timeline.owner_tgid, \ + atom_id); \ + } while (0) + +/* Trace number of atoms submitted to job slot js + * + * NOTE: This uses a different tracepoint to the head/next/soft-stop actions, + * so that those actions can be filtered out separately from this + * + * This is because this is more useful, as we can use it to calculate general + * utilization easily and accurately */ +#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_gpu_slot_active(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_SLOT_ACTIVE, \ + (int)kctx->timeline.owner_tgid, \ + js, count); \ + } while (0) + + +/* Trace atoms present in JS_NEXT */ +#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_SLOT_NEXT, \ + (int)kctx->timeline.owner_tgid, \ + js, count); \ + } while (0) + +/* Trace atoms present in JS_HEAD */ +#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_SLOT_HEAD, \ + (int)kctx->timeline.owner_tgid, \ + js, count); \ + } while (0) + +/* Trace that a soft stop/evict from next is being attempted on a slot */ +#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_SLOT_STOPPING, \ + (kctx) ? (int)kctx->timeline.owner_tgid : 0, \ + js, count); \ + } while (0) + + + +/* Trace state of overall GPU power */ +#define KBASE_TIMELINE_GPU_POWER(kbdev, active) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_POWER_ACTIVE, active); \ + } while (0) + +/* Trace state of tiler power */ +#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_POWER_TILER_ACTIVE, \ + hweight64(bitmap)); \ + } while (0) + +/* Trace number of shaders currently powered */ +#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_POWER_SHADER_ACTIVE, \ + hweight64(bitmap)); \ + } while (0) + +/* Trace state of L2 power */ +#define KBASE_TIMELINE_POWER_L2(kbdev, bitmap) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_POWER_L2_ACTIVE, \ + hweight64(bitmap)); \ + } while (0) + +/* Trace state of L2 cache*/ +#define KBASE_TIMELINE_POWERING_L2(kbdev) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_FLOW_GPU_POWER_L2_POWERING, \ + 1); \ + } while (0) + +#define KBASE_TIMELINE_POWERED_L2(kbdev) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_FLOW_GPU_POWER_L2_ACTIVE, \ + 1); \ + } while (0) + +/* Trace kbase_pm_send_event message send */ +#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \ + SW_FLOW_PM_SEND_EVENT, \ + event_type, pm_event_id); \ + } while (0) + +/* Trace kbase_pm_worker message receive */ +#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \ + SW_FLOW_PM_HANDLE_EVENT, \ + event_type, pm_event_id); \ + } while (0) + + +/* Trace atom_id starting in JS_HEAD */ +#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \ + HW_START_GPU_JOB_CHAIN_SW_APPROX, \ + (int)kctx->timeline.owner_tgid, \ + js, _consumerof_atom_number); \ + } while (0) + +/* Trace atom_id stopping on JS_HEAD */ +#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \ + HW_STOP_GPU_JOB_CHAIN_SW_APPROX, \ + (int)kctx->timeline.owner_tgid, \ + js, _producerof_atom_number_completed); \ + } while (0) + +/** Trace beginning/end of a call to kbase_pm_check_transitions_nolock from a + * certin caller */ +#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_pm_checktrans(ts.tv_sec, ts.tv_nsec, \ + trace_code, 1); \ + } while (0) + +/* Trace number of contexts active */ +#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) \ + do { \ + struct timespec ts; \ + getrawmonotonic(&ts); \ + trace_mali_timeline_context_active(ts.tv_sec, ts.tv_nsec, \ + count); \ + } while (0) + +/* NOTE: kbase_timeline_pm_cores_func() is in mali_kbase_pm_policy.c */ + +/** + * Trace that an atom is starting on a job slot + * + * The caller must be holding hwaccess_lock + */ +void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js); + +/** + * Trace that an atom has done on a job slot + * + * 'Done' in this sense can occur either because: + * - the atom in JS_HEAD finished + * - the atom in JS_NEXT was evicted + * + * Whether the atom finished or was evicted is passed in @a done_code + * + * It is assumed that the atom has already been removed from the submit slot, + * with either: + * - kbasep_jm_dequeue_submit_slot() + * - kbasep_jm_dequeue_tail_submit_slot() + * + * The caller must be holding hwaccess_lock + */ +void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js, + kbasep_js_atom_done_code done_code); + + +/** Trace a pm event starting */ +void kbase_timeline_pm_send_event(struct kbase_device *kbdev, + enum kbase_timeline_pm_event event_sent); + +/** Trace a pm event finishing */ +void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event); + +/** Check whether a pm event was present, and if so trace finishing it */ +void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event); + +/** Trace L2 power-up start */ +void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev); + +/** Trace L2 power-up done */ +void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev); + +#else + +#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) CSTD_NOP() + +#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) CSTD_NOP() + +#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) CSTD_NOP() + +#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) CSTD_NOP() + +#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) CSTD_NOP() + +#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) CSTD_NOP() + +#define KBASE_TIMELINE_GPU_POWER(kbdev, active) CSTD_NOP() + +#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) CSTD_NOP() + +#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) CSTD_NOP() + +#define KBASE_TIMELINE_POWER_L2(kbdev, active) CSTD_NOP() + +#define KBASE_TIMELINE_POWERING_L2(kbdev) CSTD_NOP() + +#define KBASE_TIMELINE_POWERED_L2(kbdev) CSTD_NOP() + +#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) CSTD_NOP() + +#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) CSTD_NOP() + +#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) CSTD_NOP() + +#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) CSTD_NOP() + +#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) CSTD_NOP() + +#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) CSTD_NOP() + +static inline void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); +} + +static inline void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js, + kbasep_js_atom_done_code done_code) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); +} + +static inline void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent) +{ +} + +static inline void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) +{ +} + +static inline void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) +{ +} + +static inline void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev) +{ +} + +static inline void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev) +{ +} +#endif /* CONFIG_MALI_BIFROST_TRACE_TIMELINE */ + +#endif /* _KBASE_TRACE_TIMELINE_H */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline_defs.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline_defs.h new file mode 100755 index 000000000000..156a95a67f4a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_trace_timeline_defs.h @@ -0,0 +1,140 @@ +/* + * + * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** + * ***** DO NOT INCLUDE DIRECTLY ***** + * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ + +/* + * Conventions on Event Names: + * + * - The prefix determines something about how the timeline should be + * displayed, and is split up into various parts, separated by underscores: + * - 'SW' and 'HW' as the first part will be used to determine whether a + * timeline is to do with Software or Hardware - effectively, separate + * 'channels' for Software and Hardware + * - 'START', 'STOP', 'ENTER', 'LEAVE' can be used in the second part, and + * signify related pairs of events - these are optional. + * - 'FLOW' indicates a generic event, which can use dependencies + * - This gives events such as: + * - 'SW_ENTER_FOO' + * - 'SW_LEAVE_FOO' + * - 'SW_FLOW_BAR_1' + * - 'SW_FLOW_BAR_2' + * - 'HW_START_BAZ' + * - 'HW_STOP_BAZ' + * - And an unadorned HW event: + * - 'HW_BAZ_FROZBOZ' + */ + +/* + * Conventions on parameter names: + * - anything with 'instance' in the name will have a separate timeline based + * on that instances. + * - underscored-prefixed parameters will by hidden by default on timelines + * + * Hence: + * - Different job slots have their own 'instance', based on the instance value + * - Per-context info (e.g. atoms on a context) have their own 'instance' + * (i.e. each context should be on a different timeline) + * + * Note that globally-shared resources can be tagged with a tgid, but we don't + * want an instance per context: + * - There's no point having separate Job Slot timelines for each context, that + * would be confusing - there's only really 3 job slots! + * - There's no point having separate Shader-powered timelines for each + * context, that would be confusing - all shader cores (whether it be 4, 8, + * etc) are shared in the system. + */ + + /* + * CTX events + */ + /* Separate timelines for each context 'instance'*/ + KBASE_TIMELINE_TRACE_CODE(CTX_SET_NR_ATOMS_IN_FLIGHT, "CTX: Atoms in flight", "%d,%d", "_instance_tgid,_value_number_of_atoms"), + KBASE_TIMELINE_TRACE_CODE(CTX_FLOW_ATOM_READY, "CTX: Atoms Ready to Run", "%d,%d,%d", "_instance_tgid,_consumerof_atom_number,_producerof_atom_number_ready"), + + /* + * SW Events + */ + /* Separate timelines for each slot 'instance' */ + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_ACTIVE, "SW: GPU slot active", "%d,%d,%d", "_tgid,_instance_slot,_value_number_of_atoms"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_NEXT, "SW: GPU atom in NEXT", "%d,%d,%d", "_tgid,_instance_slot,_value_is_an_atom_in_next"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_HEAD, "SW: GPU atom in HEAD", "%d,%d,%d", "_tgid,_instance_slot,_value_is_an_atom_in_head"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_STOPPING, "SW: Try Soft-Stop on GPU slot", "%d,%d,%d", "_tgid,_instance_slot,_value_is_slot_stopping"), + /* Shader and overall power is shared - can't have separate instances of + * it, just tagging with the context */ + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_ACTIVE, "SW: GPU power active", "%d,%d", "_tgid,_value_is_power_active"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_TILER_ACTIVE, "SW: GPU tiler powered", "%d,%d", "_tgid,_value_number_of_tilers"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_SHADER_ACTIVE, "SW: GPU shaders powered", "%d,%d", "_tgid,_value_number_of_shaders"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_L2_ACTIVE, "SW: GPU L2 powered", "%d,%d", "_tgid,_value_number_of_l2"), + + /* SW Power event messaging. _event_type is one from the kbase_pm_event enum */ + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_SEND_EVENT, "SW: PM Send Event", "%d,%d,%d", "_tgid,_event_type,_writerof_pm_event_id"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_HANDLE_EVENT, "SW: PM Handle Event", "%d,%d,%d", "_tgid,_event_type,_finalconsumerof_pm_event_id"), + /* SW L2 power events */ + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_GPU_POWER_L2_POWERING, "SW: GPU L2 powering", "%d,%d", "_tgid,_writerof_l2_transitioning"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_GPU_POWER_L2_ACTIVE, "SW: GPU L2 powering done", "%d,%d", "_tgid,_finalconsumerof_l2_transitioning"), + + KBASE_TIMELINE_TRACE_CODE(SW_SET_CONTEXT_ACTIVE, "SW: Context Active", "%d,%d", "_tgid,_value_active"), + + /* + * BEGIN: Significant SW Functions that call kbase_pm_check_transitions_nolock() + */ + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START, "SW: PM CheckTrans from kbase_pm_do_poweroff", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_poweroff"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END, "SW: PM CheckTrans from kbase_pm_do_poweroff", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_poweroff"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START, "SW: PM CheckTrans from kbase_pm_do_poweron", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_poweron"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END, "SW: PM CheckTrans from kbase_pm_do_poweron", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_poweron"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START, "SW: PM CheckTrans from kbase_gpu_interrupt", "%d,%d", "_tgid,_writerof_pm_checktrans_gpu_interrupt"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END, "SW: PM CheckTrans from kbase_gpu_interrupt", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_gpu_interrupt"), + + /* + * Significant Indirect callers of kbase_pm_check_transitions_nolock() + */ + /* kbase_pm_request_cores */ + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START, "SW: PM CheckTrans from kbase_pm_request_cores(shader)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_shader"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END, "SW: PM CheckTrans from kbase_pm_request_cores(shader)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_shader"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START, "SW: PM CheckTrans from kbase_pm_request_cores(tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END, "SW: PM CheckTrans from kbase_pm_request_cores(tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START, "SW: PM CheckTrans from kbase_pm_request_cores(shader+tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_shader_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END, "SW: PM CheckTrans from kbase_pm_request_cores(shader+tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_shader_tiler"), + /* kbase_pm_release_cores */ + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START, "SW: PM CheckTrans from kbase_pm_release_cores(shader)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_shader"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END, "SW: PM CheckTrans from kbase_pm_release_cores(shader)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_shader"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START, "SW: PM CheckTrans from kbase_pm_release_cores(tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END, "SW: PM CheckTrans from kbase_pm_release_cores(tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START, "SW: PM CheckTrans from kbase_pm_release_cores(shader+tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_shader_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END, "SW: PM CheckTrans from kbase_pm_release_cores(shader+tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_shader_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START, "SW: PM CheckTrans from kbasep_pm_do_shader_poweroff_callback", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_shader_poweroff_callback"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END, "SW: PM CheckTrans from kbasep_pm_do_shader_poweroff_callback", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_shader_poweroff_callback"), + /* + * END: SW Functions that call kbase_pm_check_transitions_nolock() + */ + + /* + * HW Events + */ + KBASE_TIMELINE_TRACE_CODE(HW_MMU_FAULT, +"HW: MMU Fault", "%d,%d,%d", "_tgid,fault_type,fault_stage,asid"), + KBASE_TIMELINE_TRACE_CODE(HW_START_GPU_JOB_CHAIN_SW_APPROX, +"HW: Job Chain start (SW approximated)", "%d,%d,%d", +"_tgid,job_slot,_consumerof_atom_number_ready"), + KBASE_TIMELINE_TRACE_CODE(HW_STOP_GPU_JOB_CHAIN_SW_APPROX, +"HW: Job Chain stop (SW approximated)", "%d,%d,%d", +"_tgid,job_slot,_producerof_atom_number_completed") diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_uku.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_uku.h new file mode 100755 index 000000000000..cf8ee0572dc5 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_uku.h @@ -0,0 +1,532 @@ +/* + * + * (C) COPYRIGHT 2008-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_UKU_H_ +#define _KBASE_UKU_H_ + +#include "mali_uk.h" +#include "mali_base_kernel.h" + +/* This file needs to support being included from kernel and userside (which use different defines) */ +#if defined(CONFIG_MALI_BIFROST_ERROR_INJECT) || MALI_ERROR_INJECT_ON +#define SUPPORT_MALI_ERROR_INJECT +#endif /* defined(CONFIG_MALI_BIFROST_ERROR_INJECT) || MALI_ERROR_INJECT_ON */ +#if defined(CONFIG_MALI_BIFROST_NO_MALI) +#define SUPPORT_MALI_NO_MALI +#elif defined(MALI_BIFROST_NO_MALI) +#if MALI_BIFROST_NO_MALI +#define SUPPORT_MALI_NO_MALI +#endif +#endif + +#if defined(SUPPORT_MALI_NO_MALI) || defined(SUPPORT_MALI_ERROR_INJECT) +#include "backend/gpu/mali_kbase_model_dummy.h" +#endif + +#include "mali_kbase_gpuprops_types.h" + +/* + * 10.1: + * - Do mmap in kernel for SAME_VA memory allocations rather then + * calling back into the kernel as a 2nd stage of the allocation request. + * + * 10.2: + * - Add KBASE_FUNC_MEM_JIT_INIT which allows clients to request a custom VA + * region for use with JIT (ignored on 32-bit platforms) + * + * 10.3: + * - base_jd_core_req typedef-ed to u32 (instead of to u16) + * - two flags added: BASE_JD_REQ_SKIP_CACHE_STAT / _END + * + * 10.4: + * - Removed KBASE_FUNC_EXT_BUFFER_LOCK used only in internal tests + * + * 10.5: + * - Reverted to performing mmap in user space so that tools like valgrind work. + * + * 10.6: + * - Add flags input variable to KBASE_FUNC_TLSTREAM_ACQUIRE + */ +#define BASE_UK_VERSION_MAJOR 10 +#define BASE_UK_VERSION_MINOR 6 + +#define LINUX_UK_BASE_MAGIC 0x80 + +struct kbase_uk_mem_alloc { + union uk_header header; + /* IN */ + u64 va_pages; + u64 commit_pages; + u64 extent; + /* IN/OUT */ + u64 flags; + /* OUT */ + u64 gpu_va; + u16 va_alignment; + u8 padding[6]; +}; + +struct kbase_uk_mem_free { + union uk_header header; + /* IN */ + u64 gpu_addr; + /* OUT */ +}; + +struct kbase_uk_mem_alias { + union uk_header header; + /* IN/OUT */ + u64 flags; + /* IN */ + u64 stride; + u64 nents; + u64 ai; + /* OUT */ + u64 gpu_va; + u64 va_pages; +}; + +struct kbase_uk_mem_import { + union uk_header header; + /* IN */ + u64 phandle; + u32 type; + u32 padding; + /* IN/OUT */ + u64 flags; + /* OUT */ + u64 gpu_va; + u64 va_pages; +}; + +struct kbase_uk_mem_flags_change { + union uk_header header; + /* IN */ + u64 gpu_va; + u64 flags; + u64 mask; +}; + +struct kbase_uk_job_submit { + union uk_header header; + /* IN */ + u64 addr; + u32 nr_atoms; + u32 stride; /* bytes between atoms, i.e. sizeof(base_jd_atom_v2) */ + /* OUT */ +}; + +struct kbase_uk_post_term { + union uk_header header; +}; + +struct kbase_uk_sync_now { + union uk_header header; + + /* IN */ + struct base_syncset sset; + + /* OUT */ +}; + +struct kbase_uk_hwcnt_setup { + union uk_header header; + + /* IN */ + u64 dump_buffer; + u32 jm_bm; + u32 shader_bm; + u32 tiler_bm; + u32 unused_1; /* keep for backwards compatibility */ + u32 mmu_l2_bm; + u32 padding; + /* OUT */ +}; + +/** + * struct kbase_uk_hwcnt_reader_setup - User/Kernel space data exchange structure + * @header: UK structure header + * @buffer_count: requested number of dumping buffers + * @jm_bm: counters selection bitmask (JM) + * @shader_bm: counters selection bitmask (Shader) + * @tiler_bm: counters selection bitmask (Tiler) + * @mmu_l2_bm: counters selection bitmask (MMU_L2) + * @fd: dumping notification file descriptor + * + * This structure sets up HWC dumper/reader for this context. + * Multiple instances can be created for single context. + */ +struct kbase_uk_hwcnt_reader_setup { + union uk_header header; + + /* IN */ + u32 buffer_count; + u32 jm_bm; + u32 shader_bm; + u32 tiler_bm; + u32 mmu_l2_bm; + + /* OUT */ + s32 fd; +}; + +struct kbase_uk_hwcnt_dump { + union uk_header header; +}; + +struct kbase_uk_hwcnt_clear { + union uk_header header; +}; + +struct kbase_uk_fence_validate { + union uk_header header; + /* IN */ + s32 fd; + u32 padding; + /* OUT */ +}; + +struct kbase_uk_stream_create { + union uk_header header; + /* IN */ + char name[32]; + /* OUT */ + s32 fd; + u32 padding; +}; + +struct kbase_uk_gpuprops { + union uk_header header; + + /* IN */ + struct mali_base_gpu_props props; + /* OUT */ +}; + +struct kbase_uk_mem_query { + union uk_header header; + /* IN */ + u64 gpu_addr; +#define KBASE_MEM_QUERY_COMMIT_SIZE 1 +#define KBASE_MEM_QUERY_VA_SIZE 2 +#define KBASE_MEM_QUERY_FLAGS 3 + u64 query; + /* OUT */ + u64 value; +}; + +struct kbase_uk_mem_commit { + union uk_header header; + /* IN */ + u64 gpu_addr; + u64 pages; + /* OUT */ + u32 result_subcode; + u32 padding; +}; + +struct kbase_uk_find_cpu_offset { + union uk_header header; + /* IN */ + u64 gpu_addr; + u64 cpu_addr; + u64 size; + /* OUT */ + u64 offset; +}; + +#define KBASE_GET_VERSION_BUFFER_SIZE 64 +struct kbase_uk_get_ddk_version { + union uk_header header; + /* OUT */ + char version_buffer[KBASE_GET_VERSION_BUFFER_SIZE]; + u32 version_string_size; + u32 padding; +}; + +struct kbase_uk_disjoint_query { + union uk_header header; + /* OUT */ + u32 counter; + u32 padding; +}; + +struct kbase_uk_set_flags { + union uk_header header; + /* IN */ + u32 create_flags; + u32 padding; +}; + +#if MALI_UNIT_TEST +#define TEST_ADDR_COUNT 4 +#define KBASE_TEST_BUFFER_SIZE 128 +struct kbase_exported_test_data { + u64 test_addr[TEST_ADDR_COUNT]; /**< memory address */ + u32 test_addr_pages[TEST_ADDR_COUNT]; /**< memory size in pages */ + u64 kctx; /**< base context created by process */ + u64 mm; /**< pointer to process address space */ + u8 buffer1[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */ + u8 buffer2[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */ +}; + +struct kbase_uk_set_test_data { + union uk_header header; + /* IN */ + struct kbase_exported_test_data test_data; +}; + +#endif /* MALI_UNIT_TEST */ + +#ifdef SUPPORT_MALI_ERROR_INJECT +struct kbase_uk_error_params { + union uk_header header; + /* IN */ + struct kbase_error_params params; +}; +#endif /* SUPPORT_MALI_ERROR_INJECT */ + +#ifdef SUPPORT_MALI_NO_MALI +struct kbase_uk_model_control_params { + union uk_header header; + /* IN */ + struct kbase_model_control_params params; +}; +#endif /* SUPPORT_MALI_NO_MALI */ + +struct kbase_uk_profiling_controls { + union uk_header header; + u32 profiling_controls[FBDUMP_CONTROL_MAX]; +}; + +struct kbase_uk_debugfs_mem_profile_add { + union uk_header header; + u32 len; + u32 padding; + u64 buf; +}; + +struct kbase_uk_context_id { + union uk_header header; + /* OUT */ + int id; +}; + +/** + * struct kbase_uk_tlstream_acquire - User/Kernel space data exchange structure + * @header: UK structure header + * @flags: timeline stream flags + * @fd: timeline stream file descriptor + * + * This structure is used when performing a call to acquire kernel side timeline + * stream file descriptor. + */ +struct kbase_uk_tlstream_acquire { + union uk_header header; + /* IN */ + u32 flags; + /* OUT */ + s32 fd; +}; + +/** + * struct kbase_uk_tlstream_acquire_v10_4 - User/Kernel space data exchange + * structure + * @header: UK structure header + * @fd: timeline stream file descriptor + * + * This structure is used when performing a call to acquire kernel side timeline + * stream file descriptor. + */ +struct kbase_uk_tlstream_acquire_v10_4 { + union uk_header header; + /* IN */ + /* OUT */ + s32 fd; +}; + +/** + * struct kbase_uk_tlstream_flush - User/Kernel space data exchange structure + * @header: UK structure header + * + * This structure is used when performing a call to flush kernel side + * timeline streams. + */ +struct kbase_uk_tlstream_flush { + union uk_header header; + /* IN */ + /* OUT */ +}; + +#if MALI_UNIT_TEST +/** + * struct kbase_uk_tlstream_test - User/Kernel space data exchange structure + * @header: UK structure header + * @tpw_count: number of trace point writers in each context + * @msg_delay: time delay between tracepoints from one writer in milliseconds + * @msg_count: number of trace points written by one writer + * @aux_msg: if non-zero aux messages will be included + * + * This structure is used when performing a call to start timeline stream test + * embedded in kernel. + */ +struct kbase_uk_tlstream_test { + union uk_header header; + /* IN */ + u32 tpw_count; + u32 msg_delay; + u32 msg_count; + u32 aux_msg; + /* OUT */ +}; + +/** + * struct kbase_uk_tlstream_stats - User/Kernel space data exchange structure + * @header: UK structure header + * @bytes_collected: number of bytes read by user + * @bytes_generated: number of bytes generated by tracepoints + * + * This structure is used when performing a call to obtain timeline stream + * statistics. + */ +struct kbase_uk_tlstream_stats { + union uk_header header; /**< UK structure header. */ + /* IN */ + /* OUT */ + u32 bytes_collected; + u32 bytes_generated; +}; +#endif /* MALI_UNIT_TEST */ + +/** + * struct struct kbase_uk_prfcnt_value for the KBASE_FUNC_SET_PRFCNT_VALUES ioctl + * @header: UK structure header + * @data: Counter samples for the dummy model + * @size:............Size of the counter sample data + */ +struct kbase_uk_prfcnt_values { + union uk_header header; + /* IN */ + u32 *data; + u32 size; +}; + +/** + * struct kbase_uk_soft_event_update - User/Kernel space data exchange structure + * @header: UK structure header + * @evt: the GPU address containing the event + * @new_status: the new event status, must be either BASE_JD_SOFT_EVENT_SET or + * BASE_JD_SOFT_EVENT_RESET + * @flags: reserved for future uses, must be set to 0 + * + * This structure is used to update the status of a software event. If the + * event's status is set to BASE_JD_SOFT_EVENT_SET, any job currently waiting + * on this event will complete. + */ +struct kbase_uk_soft_event_update { + union uk_header header; + /* IN */ + u64 evt; + u32 new_status; + u32 flags; +}; + +/** + * struct kbase_uk_mem_jit_init - User/Kernel space data exchange structure + * @header: UK structure header + * @va_pages: Number of virtual pages required for JIT + * + * This structure is used when requesting initialization of JIT. + */ +struct kbase_uk_mem_jit_init { + union uk_header header; + /* IN */ + u64 va_pages; +}; + +enum kbase_uk_function_id { + KBASE_FUNC_MEM_ALLOC = (UK_FUNC_ID + 0), + KBASE_FUNC_MEM_IMPORT = (UK_FUNC_ID + 1), + KBASE_FUNC_MEM_COMMIT = (UK_FUNC_ID + 2), + KBASE_FUNC_MEM_QUERY = (UK_FUNC_ID + 3), + KBASE_FUNC_MEM_FREE = (UK_FUNC_ID + 4), + KBASE_FUNC_MEM_FLAGS_CHANGE = (UK_FUNC_ID + 5), + KBASE_FUNC_MEM_ALIAS = (UK_FUNC_ID + 6), + + /* UK_FUNC_ID + 7 not in use since BASE_LEGACY_UK6_SUPPORT dropped */ + + KBASE_FUNC_SYNC = (UK_FUNC_ID + 8), + + KBASE_FUNC_POST_TERM = (UK_FUNC_ID + 9), + + KBASE_FUNC_HWCNT_SETUP = (UK_FUNC_ID + 10), + KBASE_FUNC_HWCNT_DUMP = (UK_FUNC_ID + 11), + KBASE_FUNC_HWCNT_CLEAR = (UK_FUNC_ID + 12), + + KBASE_FUNC_GPU_PROPS_REG_DUMP = (UK_FUNC_ID + 14), + + KBASE_FUNC_FIND_CPU_OFFSET = (UK_FUNC_ID + 15), + + KBASE_FUNC_GET_VERSION = (UK_FUNC_ID + 16), + KBASE_FUNC_SET_FLAGS = (UK_FUNC_ID + 18), + + KBASE_FUNC_SET_TEST_DATA = (UK_FUNC_ID + 19), + KBASE_FUNC_INJECT_ERROR = (UK_FUNC_ID + 20), + KBASE_FUNC_MODEL_CONTROL = (UK_FUNC_ID + 21), + + /* UK_FUNC_ID + 22 not in use since BASE_LEGACY_UK8_SUPPORT dropped */ + + KBASE_FUNC_FENCE_VALIDATE = (UK_FUNC_ID + 23), + KBASE_FUNC_STREAM_CREATE = (UK_FUNC_ID + 24), + KBASE_FUNC_GET_PROFILING_CONTROLS = (UK_FUNC_ID + 25), + KBASE_FUNC_SET_PROFILING_CONTROLS = (UK_FUNC_ID + 26), + /* to be used only for testing + * purposes, otherwise these controls + * are set through gator API */ + + KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD = (UK_FUNC_ID + 27), + KBASE_FUNC_JOB_SUBMIT = (UK_FUNC_ID + 28), + KBASE_FUNC_DISJOINT_QUERY = (UK_FUNC_ID + 29), + + KBASE_FUNC_GET_CONTEXT_ID = (UK_FUNC_ID + 31), + + KBASE_FUNC_TLSTREAM_ACQUIRE_V10_4 = (UK_FUNC_ID + 32), +#if MALI_UNIT_TEST + KBASE_FUNC_TLSTREAM_TEST = (UK_FUNC_ID + 33), + KBASE_FUNC_TLSTREAM_STATS = (UK_FUNC_ID + 34), +#endif /* MALI_UNIT_TEST */ + KBASE_FUNC_TLSTREAM_FLUSH = (UK_FUNC_ID + 35), + + KBASE_FUNC_HWCNT_READER_SETUP = (UK_FUNC_ID + 36), + +#ifdef SUPPORT_MALI_NO_MALI + KBASE_FUNC_SET_PRFCNT_VALUES = (UK_FUNC_ID + 37), +#endif + + KBASE_FUNC_SOFT_EVENT_UPDATE = (UK_FUNC_ID + 38), + + KBASE_FUNC_MEM_JIT_INIT = (UK_FUNC_ID + 39), + + KBASE_FUNC_TLSTREAM_ACQUIRE = (UK_FUNC_ID + 40), + + KBASE_FUNC_MAX +}; + +#endif /* _KBASE_UKU_H_ */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.c new file mode 100755 index 000000000000..be474ff87401 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.c @@ -0,0 +1,33 @@ +/* + * + * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include + +bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry) +{ + struct list_head *pos = base->next; + + while (pos != base) { + if (pos == entry) + return true; + + pos = pos->next; + } + return false; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.h new file mode 100755 index 000000000000..fd7252dab0de --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_utility.h @@ -0,0 +1,37 @@ +/* + * + * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_UTILITY_H +#define _KBASE_UTILITY_H + +#ifndef _KBASE_H_ +#error "Don't include this file directly, use mali_kbase.h instead" +#endif + +/** Test whether the given list entry is a member of the given list. + * + * @param base The head of the list to be tested + * @param entry The list entry to be tested + * + * @return true if entry is a member of base + * false otherwise + */ +bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry); + +#endif /* _KBASE_UTILITY_H */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.c b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.c new file mode 100755 index 000000000000..9c5b2e46c0e5 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.c @@ -0,0 +1,2072 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/*****************************************************************************/ + +/* Hwcnt reader API version */ +#define HWCNT_READER_API 1 + +/* The number of nanoseconds in a second. */ +#define NSECS_IN_SEC 1000000000ull /* ns */ + +/* The time resolution of dumping service. */ +#define DUMPING_RESOLUTION 500000ull /* ns */ + +/* The maximal supported number of dumping buffers. */ +#define MAX_BUFFER_COUNT 32 + +/* Size and number of hw counters blocks. */ +#define NR_CNT_BLOCKS_PER_GROUP 8 +#define NR_CNT_PER_BLOCK 64 +#define NR_BYTES_PER_CNT 4 +#define NR_BYTES_PER_HDR 16 +#define PRFCNT_EN_MASK_OFFSET 0x8 + +/*****************************************************************************/ + +enum { + SHADER_HWCNT_BM, + TILER_HWCNT_BM, + MMU_L2_HWCNT_BM, + JM_HWCNT_BM +}; + +enum vinstr_state { + VINSTR_IDLE, + VINSTR_DUMPING, + VINSTR_SUSPENDING, + VINSTR_SUSPENDED, + VINSTR_RESUMING +}; + +/** + * struct kbase_vinstr_context - vinstr context per device + * @lock: protects the entire vinstr context + * @kbdev: pointer to kbase device + * @kctx: pointer to kbase context + * @vmap: vinstr vmap for mapping hwcnt dump buffer + * @gpu_va: GPU hwcnt dump buffer address + * @cpu_va: the CPU side mapping of the hwcnt dump buffer + * @dump_size: size of the dump buffer in bytes + * @bitmap: current set of counters monitored, not always in sync + * with hardware + * @reprogram: when true, reprogram hwcnt block with the new set of + * counters + * @state: vinstr state + * @state_lock: protects information about vinstr state + * @suspend_waitq: notification queue to trigger state re-validation + * @suspend_cnt: reference counter of vinstr's suspend state + * @suspend_work: worker to execute on entering suspended state + * @resume_work: worker to execute on leaving suspended state + * @nclients: number of attached clients, pending or otherwise + * @waiting_clients: head of list of clients being periodically sampled + * @idle_clients: head of list of clients being idle + * @suspended_clients: head of list of clients being suspended + * @thread: periodic sampling thread + * @waitq: notification queue of sampling thread + * @request_pending: request for action for sampling thread + * @clients_present: when true, we have at least one client + * Note: this variable is in sync. with nclients and is + * present to preserve simplicity. Protected by state_lock. + */ +struct kbase_vinstr_context { + struct mutex lock; + struct kbase_device *kbdev; + struct kbase_context *kctx; + + struct kbase_vmap_struct vmap; + u64 gpu_va; + void *cpu_va; + size_t dump_size; + u32 bitmap[4]; + bool reprogram; + + enum vinstr_state state; + struct spinlock state_lock; + wait_queue_head_t suspend_waitq; + unsigned int suspend_cnt; + struct work_struct suspend_work; + struct work_struct resume_work; + + u32 nclients; + struct list_head waiting_clients; + struct list_head idle_clients; + struct list_head suspended_clients; + + struct task_struct *thread; + wait_queue_head_t waitq; + atomic_t request_pending; + + bool clients_present; +}; + +/** + * struct kbase_vinstr_client - a vinstr client attached to a vinstr context + * @vinstr_ctx: vinstr context client is attached to + * @list: node used to attach this client to list in vinstr context + * @buffer_count: number of buffers this client is using + * @event_mask: events this client reacts to + * @dump_size: size of one dump buffer in bytes + * @bitmap: bitmap request for JM, TILER, SHADER and MMU counters + * @legacy_buffer: userspace hwcnt dump buffer (legacy interface) + * @kernel_buffer: kernel hwcnt dump buffer (kernel client interface) + * @accum_buffer: temporary accumulation buffer for preserving counters + * @dump_time: next time this clients shall request hwcnt dump + * @dump_interval: interval between periodic hwcnt dumps + * @dump_buffers: kernel hwcnt dump buffers allocated by this client + * @dump_buffers_meta: metadata of dump buffers + * @meta_idx: index of metadata being accessed by userspace + * @read_idx: index of buffer read by userspace + * @write_idx: index of buffer being written by dumping service + * @waitq: client's notification queue + * @pending: when true, client has attached but hwcnt not yet updated + */ +struct kbase_vinstr_client { + struct kbase_vinstr_context *vinstr_ctx; + struct list_head list; + unsigned int buffer_count; + u32 event_mask; + size_t dump_size; + u32 bitmap[4]; + void __user *legacy_buffer; + void *kernel_buffer; + void *accum_buffer; + u64 dump_time; + u32 dump_interval; + char *dump_buffers; + struct kbase_hwcnt_reader_metadata *dump_buffers_meta; + atomic_t meta_idx; + atomic_t read_idx; + atomic_t write_idx; + wait_queue_head_t waitq; + bool pending; +}; + +/** + * struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer + * @hrtimer: high resolution timer + * @vinstr_ctx: vinstr context + */ +struct kbasep_vinstr_wake_up_timer { + struct hrtimer hrtimer; + struct kbase_vinstr_context *vinstr_ctx; +}; + +/*****************************************************************************/ + +static int kbasep_vinstr_service_task(void *data); + +static unsigned int kbasep_vinstr_hwcnt_reader_poll( + struct file *filp, + poll_table *wait); +static long kbasep_vinstr_hwcnt_reader_ioctl( + struct file *filp, + unsigned int cmd, + unsigned long arg); +static int kbasep_vinstr_hwcnt_reader_mmap( + struct file *filp, + struct vm_area_struct *vma); +static int kbasep_vinstr_hwcnt_reader_release( + struct inode *inode, + struct file *filp); + +/* The timeline stream file operations structure. */ +static const struct file_operations vinstr_client_fops = { + .poll = kbasep_vinstr_hwcnt_reader_poll, + .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, + .compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, + .mmap = kbasep_vinstr_hwcnt_reader_mmap, + .release = kbasep_vinstr_hwcnt_reader_release, +}; + +/*****************************************************************************/ + +static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_context *kctx = vinstr_ctx->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct kbase_uk_hwcnt_setup setup; + int err; + + setup.dump_buffer = vinstr_ctx->gpu_va; + setup.jm_bm = vinstr_ctx->bitmap[JM_HWCNT_BM]; + setup.tiler_bm = vinstr_ctx->bitmap[TILER_HWCNT_BM]; + setup.shader_bm = vinstr_ctx->bitmap[SHADER_HWCNT_BM]; + setup.mmu_l2_bm = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM]; + + /* Mark the context as active so the GPU is kept turned on */ + /* A suspend won't happen here, because we're in a syscall from a + * userspace thread. */ + kbase_pm_context_active(kbdev); + + /* Schedule the context in */ + kbasep_js_schedule_privileged_ctx(kbdev, kctx); + err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &setup); + if (err) { + /* Release the context. This had its own Power Manager Active + * reference */ + kbasep_js_release_privileged_ctx(kbdev, kctx); + + /* Also release our Power Manager Active reference */ + kbase_pm_context_idle(kbdev); + } + + return err; +} + +static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_context *kctx = vinstr_ctx->kctx; + struct kbase_device *kbdev = kctx->kbdev; + int err; + + err = kbase_instr_hwcnt_disable_internal(kctx); + if (err) { + dev_warn(kbdev->dev, "Failed to disable HW counters (ctx:%p)", + kctx); + return; + } + + /* Release the context. This had its own Power Manager Active reference. */ + kbasep_js_release_privileged_ctx(kbdev, kctx); + + /* Also release our Power Manager Active reference. */ + kbase_pm_context_idle(kbdev); + + dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx); +} + +static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx) +{ + disable_hwcnt(vinstr_ctx); + return enable_hwcnt(vinstr_ctx); +} + +static void hwcnt_bitmap_set(u32 dst[4], u32 src[4]) +{ + dst[JM_HWCNT_BM] = src[JM_HWCNT_BM]; + dst[TILER_HWCNT_BM] = src[TILER_HWCNT_BM]; + dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM]; + dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM]; +} + +static void hwcnt_bitmap_union(u32 dst[4], u32 src[4]) +{ + dst[JM_HWCNT_BM] |= src[JM_HWCNT_BM]; + dst[TILER_HWCNT_BM] |= src[TILER_HWCNT_BM]; + dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM]; + dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM]; +} + +size_t kbase_vinstr_dump_size(struct kbase_device *kbdev) +{ + size_t dump_size; + +#ifndef CONFIG_MALI_BIFROST_NO_MALI + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) { + u32 nr_cg; + + nr_cg = kbdev->gpu_props.num_core_groups; + dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP * + NR_CNT_PER_BLOCK * + NR_BYTES_PER_CNT; + } else +#endif /* CONFIG_MALI_BIFROST_NO_MALI */ + { + /* assume v5 for now */ + base_gpu_props *props = &kbdev->gpu_props.props; + u32 nr_l2 = props->l2_props.num_l2_slices; + u64 core_mask = props->coherency_info.group[0].core_mask; + u32 nr_blocks = fls64(core_mask); + + /* JM and tiler counter blocks are always present */ + dump_size = (2 + nr_l2 + nr_blocks) * + NR_CNT_PER_BLOCK * + NR_BYTES_PER_CNT; + } + return dump_size; +} +KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size); + +static size_t kbasep_vinstr_dump_size_ctx( + struct kbase_vinstr_context *vinstr_ctx) +{ + return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev); +} + +static int kbasep_vinstr_map_kernel_dump_buffer( + struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_va_region *reg; + struct kbase_context *kctx = vinstr_ctx->kctx; + u64 flags, nr_pages; + + flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR; + vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx); + nr_pages = PFN_UP(vinstr_ctx->dump_size); + + reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, + &vinstr_ctx->gpu_va); + if (!reg) + return -ENOMEM; + + vinstr_ctx->cpu_va = kbase_vmap( + kctx, + vinstr_ctx->gpu_va, + vinstr_ctx->dump_size, + &vinstr_ctx->vmap); + if (!vinstr_ctx->cpu_va) { + kbase_mem_free(kctx, vinstr_ctx->gpu_va); + return -ENOMEM; + } + + return 0; +} + +static void kbasep_vinstr_unmap_kernel_dump_buffer( + struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_context *kctx = vinstr_ctx->kctx; + + kbase_vunmap(kctx, &vinstr_ctx->vmap); + kbase_mem_free(kctx, vinstr_ctx->gpu_va); +} + +/** + * kbasep_vinstr_create_kctx - create kernel context for vinstr + * @vinstr_ctx: vinstr context + * Return: zero on success + */ +static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_device *kbdev = vinstr_ctx->kbdev; + struct kbasep_kctx_list_element *element = NULL; + unsigned long flags; + bool enable_backend = false; + int err; + + vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true); + if (!vinstr_ctx->kctx) + return -ENOMEM; + + /* Map the master kernel dump buffer. The HW dumps the counters + * into this memory region. */ + err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx); + if (err) + goto failed_map; + + /* Add kernel context to list of contexts associated with device. */ + element = kzalloc(sizeof(*element), GFP_KERNEL); + if (element) { + element->kctx = vinstr_ctx->kctx; + mutex_lock(&kbdev->kctx_list_lock); + list_add(&element->link, &kbdev->kctx_list); + + /* Inform timeline client about new context. + * Do this while holding the lock to avoid tracepoint + * being created in both body and summary stream. */ + KBASE_TLSTREAM_TL_NEW_CTX( + vinstr_ctx->kctx, + vinstr_ctx->kctx->id, + (u32)(vinstr_ctx->kctx->tgid)); + + mutex_unlock(&kbdev->kctx_list_lock); + } else { + /* Don't treat this as a fail - just warn about it. */ + dev_warn(kbdev->dev, + "couldn't add kctx to kctx_list\n"); + } + + /* Don't enable hardware counters if vinstr is suspended. + * Note that vinstr resume code is run under vinstr context lock, + * lower layer will be enabled as needed on resume. */ + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + if (VINSTR_IDLE == vinstr_ctx->state) + enable_backend = true; + vinstr_ctx->clients_present = true; + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + if (enable_backend) + err = enable_hwcnt(vinstr_ctx); + if (err) + goto failed_enable; + + vinstr_ctx->thread = kthread_run( + kbasep_vinstr_service_task, + vinstr_ctx, + "mali_vinstr_service"); + if (IS_ERR(vinstr_ctx->thread)) { + err = PTR_ERR(vinstr_ctx->thread); + goto failed_kthread; + } + + return 0; + +failed_kthread: + disable_hwcnt(vinstr_ctx); +failed_enable: + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + vinstr_ctx->clients_present = false; + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); + if (element) { + mutex_lock(&kbdev->kctx_list_lock); + list_del(&element->link); + kfree(element); + mutex_unlock(&kbdev->kctx_list_lock); + KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); + } +failed_map: + kbase_destroy_context(vinstr_ctx->kctx); + vinstr_ctx->kctx = NULL; + return err; +} + +/** + * kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context + * @vinstr_ctx: vinstr context + */ +static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_device *kbdev = vinstr_ctx->kbdev; + struct kbasep_kctx_list_element *element; + struct kbasep_kctx_list_element *tmp; + bool found = false; + unsigned long flags; + + /* Release hw counters dumping resources. */ + vinstr_ctx->thread = NULL; + disable_hwcnt(vinstr_ctx); + kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); + + /* Simplify state transitions by specifying that we have no clients. */ + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + vinstr_ctx->clients_present = false; + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + /* Remove kernel context from the device's contexts list. */ + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) { + if (element->kctx == vinstr_ctx->kctx) { + list_del(&element->link); + kfree(element); + found = true; + } + } + mutex_unlock(&kbdev->kctx_list_lock); + + if (!found) + dev_warn(kbdev->dev, "kctx not in kctx_list\n"); + + /* Destroy context. */ + kbase_destroy_context(vinstr_ctx->kctx); + + /* Inform timeline client about context destruction. */ + KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); + + vinstr_ctx->kctx = NULL; +} + +/** + * kbasep_vinstr_attach_client - Attach a client to the vinstr core + * @vinstr_ctx: vinstr context + * @buffer_count: requested number of dump buffers + * @bitmap: bitmaps describing which counters should be enabled + * @argp: pointer where notification descriptor shall be stored + * @kernel_buffer: pointer to kernel side buffer + * + * Return: vinstr opaque client handle or NULL on failure + */ +static struct kbase_vinstr_client *kbasep_vinstr_attach_client( + struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count, + u32 bitmap[4], void *argp, void *kernel_buffer) +{ + struct task_struct *thread = NULL; + struct kbase_vinstr_client *cli; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + if (buffer_count > MAX_BUFFER_COUNT + || (buffer_count & (buffer_count - 1))) + return NULL; + + cli = kzalloc(sizeof(*cli), GFP_KERNEL); + if (!cli) + return NULL; + + cli->vinstr_ctx = vinstr_ctx; + cli->buffer_count = buffer_count; + cli->event_mask = + (1 << BASE_HWCNT_READER_EVENT_MANUAL) | + (1 << BASE_HWCNT_READER_EVENT_PERIODIC); + cli->pending = true; + + hwcnt_bitmap_set(cli->bitmap, bitmap); + + mutex_lock(&vinstr_ctx->lock); + + hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap); + vinstr_ctx->reprogram = true; + + /* If this is the first client, create the vinstr kbase + * context. This context is permanently resident until the + * last client exits. */ + if (!vinstr_ctx->nclients) { + hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap); + if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0) + goto error; + + vinstr_ctx->reprogram = false; + cli->pending = false; + } + + /* The GPU resets the counter block every time there is a request + * to dump it. We need a per client kernel buffer for accumulating + * the counters. */ + cli->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx); + cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL); + if (!cli->accum_buffer) + goto error; + + /* Prepare buffers. */ + if (cli->buffer_count) { + int *fd = (int *)argp; + size_t tmp; + + /* Allocate area for buffers metadata storage. */ + tmp = sizeof(struct kbase_hwcnt_reader_metadata) * + cli->buffer_count; + cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL); + if (!cli->dump_buffers_meta) + goto error; + + /* Allocate required number of dumping buffers. */ + cli->dump_buffers = (char *)__get_free_pages( + GFP_KERNEL | __GFP_ZERO, + get_order(cli->dump_size * cli->buffer_count)); + if (!cli->dump_buffers) + goto error; + + /* Create descriptor for user-kernel data exchange. */ + *fd = anon_inode_getfd( + "[mali_vinstr_desc]", + &vinstr_client_fops, + cli, + O_RDONLY | O_CLOEXEC); + if (0 > *fd) + goto error; + } else if (kernel_buffer) { + cli->kernel_buffer = kernel_buffer; + } else { + cli->legacy_buffer = (void __user *)argp; + } + + atomic_set(&cli->read_idx, 0); + atomic_set(&cli->meta_idx, 0); + atomic_set(&cli->write_idx, 0); + init_waitqueue_head(&cli->waitq); + + vinstr_ctx->nclients++; + list_add(&cli->list, &vinstr_ctx->idle_clients); + + mutex_unlock(&vinstr_ctx->lock); + + return cli; + +error: + kfree(cli->dump_buffers_meta); + if (cli->dump_buffers) + free_pages( + (unsigned long)cli->dump_buffers, + get_order(cli->dump_size * cli->buffer_count)); + kfree(cli->accum_buffer); + if (!vinstr_ctx->nclients && vinstr_ctx->kctx) { + thread = vinstr_ctx->thread; + kbasep_vinstr_destroy_kctx(vinstr_ctx); + } + kfree(cli); + + mutex_unlock(&vinstr_ctx->lock); + + /* Thread must be stopped after lock is released. */ + if (thread) + kthread_stop(thread); + + return NULL; +} + +void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli) +{ + struct kbase_vinstr_context *vinstr_ctx; + struct kbase_vinstr_client *iter, *tmp; + struct task_struct *thread = NULL; + u32 zerobitmap[4] = { 0 }; + int cli_found = 0; + + KBASE_DEBUG_ASSERT(cli); + vinstr_ctx = cli->vinstr_ctx; + KBASE_DEBUG_ASSERT(vinstr_ctx); + + mutex_lock(&vinstr_ctx->lock); + + list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) { + if (iter == cli) { + vinstr_ctx->reprogram = true; + cli_found = 1; + list_del(&iter->list); + break; + } + } + if (!cli_found) { + list_for_each_entry_safe( + iter, tmp, &vinstr_ctx->waiting_clients, list) { + if (iter == cli) { + vinstr_ctx->reprogram = true; + cli_found = 1; + list_del(&iter->list); + break; + } + } + } + KBASE_DEBUG_ASSERT(cli_found); + + kfree(cli->dump_buffers_meta); + free_pages( + (unsigned long)cli->dump_buffers, + get_order(cli->dump_size * cli->buffer_count)); + kfree(cli->accum_buffer); + kfree(cli); + + vinstr_ctx->nclients--; + if (!vinstr_ctx->nclients) { + thread = vinstr_ctx->thread; + kbasep_vinstr_destroy_kctx(vinstr_ctx); + } + + /* Rebuild context bitmap now that the client has detached */ + hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap); + list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) + hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap); + list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) + hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap); + + mutex_unlock(&vinstr_ctx->lock); + + /* Thread must be stopped after lock is released. */ + if (thread) + kthread_stop(thread); +} +KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client); + +/* Accumulate counters in the dump buffer */ +static void accum_dump_buffer(void *dst, void *src, size_t dump_size) +{ + size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT; + u32 *d = dst; + u32 *s = src; + size_t i, j; + + for (i = 0; i < dump_size; i += block_size) { + /* skip over the header block */ + d += NR_BYTES_PER_HDR / sizeof(u32); + s += NR_BYTES_PER_HDR / sizeof(u32); + for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) { + /* saturate result if addition would result in wraparound */ + if (U32_MAX - *d < *s) + *d = U32_MAX; + else + *d += *s; + d++; + s++; + } + } +} + +/* This is the Midgard v4 patch function. It copies the headers for each + * of the defined blocks from the master kernel buffer and then patches up + * the performance counter enable mask for each of the blocks to exclude + * counters that were not requested by the client. */ +static void patch_dump_buffer_hdr_v4( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_vinstr_client *cli) +{ + u32 *mask; + u8 *dst = cli->accum_buffer; + u8 *src = vinstr_ctx->cpu_va; + u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups; + size_t i, group_size, group; + enum { + SC0_BASE = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + SC1_BASE = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + SC2_BASE = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + SC3_BASE = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + TILER_BASE = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + JM_BASE = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT + }; + + group_size = NR_CNT_BLOCKS_PER_GROUP * + NR_CNT_PER_BLOCK * + NR_BYTES_PER_CNT; + for (i = 0; i < nr_cg; i++) { + group = i * group_size; + /* copy shader core headers */ + memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE], + NR_BYTES_PER_HDR); + memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE], + NR_BYTES_PER_HDR); + memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE], + NR_BYTES_PER_HDR); + memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE], + NR_BYTES_PER_HDR); + + /* copy tiler header */ + memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE], + NR_BYTES_PER_HDR); + + /* copy mmu header */ + memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE], + NR_BYTES_PER_HDR); + + /* copy job manager header */ + memcpy(&dst[group + JM_BASE], &src[group + JM_BASE], + NR_BYTES_PER_HDR); + + /* patch the shader core enable mask */ + mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[SHADER_HWCNT_BM]; + mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[SHADER_HWCNT_BM]; + mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[SHADER_HWCNT_BM]; + mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[SHADER_HWCNT_BM]; + + /* patch the tiler core enable mask */ + mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[TILER_HWCNT_BM]; + + /* patch the mmu core enable mask */ + mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[MMU_L2_HWCNT_BM]; + + /* patch the job manager enable mask */ + mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[JM_HWCNT_BM]; + } +} + +/* This is the Midgard v5 patch function. It copies the headers for each + * of the defined blocks from the master kernel buffer and then patches up + * the performance counter enable mask for each of the blocks to exclude + * counters that were not requested by the client. */ +static void patch_dump_buffer_hdr_v5( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_vinstr_client *cli) +{ + struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev; + u32 i, nr_l2; + u64 core_mask; + u32 *mask; + u8 *dst = cli->accum_buffer; + u8 *src = vinstr_ctx->cpu_va; + size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT; + + /* copy and patch job manager header */ + memcpy(dst, src, NR_BYTES_PER_HDR); + mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[JM_HWCNT_BM]; + dst += block_size; + src += block_size; + + /* copy and patch tiler header */ + memcpy(dst, src, NR_BYTES_PER_HDR); + mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[TILER_HWCNT_BM]; + dst += block_size; + src += block_size; + + /* copy and patch MMU/L2C headers */ + nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices; + for (i = 0; i < nr_l2; i++) { + memcpy(dst, src, NR_BYTES_PER_HDR); + mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[MMU_L2_HWCNT_BM]; + dst += block_size; + src += block_size; + } + + /* copy and patch shader core headers */ + core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; + while (0ull != core_mask) { + memcpy(dst, src, NR_BYTES_PER_HDR); + if (0ull != (core_mask & 1ull)) { + /* if block is not reserved update header */ + mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[SHADER_HWCNT_BM]; + } + dst += block_size; + src += block_size; + + core_mask >>= 1; + } +} + +/** + * accum_clients - accumulate dumped hw counters for all known clients + * @vinstr_ctx: vinstr context + */ +static void accum_clients(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_vinstr_client *iter; + int v4 = 0; + +#ifndef CONFIG_MALI_BIFROST_NO_MALI + v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4); +#endif + + list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) { + /* Don't bother accumulating clients whose hwcnt requests + * have not yet been honoured. */ + if (iter->pending) + continue; + if (v4) + patch_dump_buffer_hdr_v4(vinstr_ctx, iter); + else + patch_dump_buffer_hdr_v5(vinstr_ctx, iter); + accum_dump_buffer( + iter->accum_buffer, + vinstr_ctx->cpu_va, + iter->dump_size); + } + list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) { + /* Don't bother accumulating clients whose hwcnt requests + * have not yet been honoured. */ + if (iter->pending) + continue; + if (v4) + patch_dump_buffer_hdr_v4(vinstr_ctx, iter); + else + patch_dump_buffer_hdr_v5(vinstr_ctx, iter); + accum_dump_buffer( + iter->accum_buffer, + vinstr_ctx->cpu_va, + iter->dump_size); + } +} + +/*****************************************************************************/ + +/** + * kbasep_vinstr_get_timestamp - return timestamp + * + * Function returns timestamp value based on raw monotonic timer. Value will + * wrap around zero in case of overflow. + * + * Return: timestamp value + */ +static u64 kbasep_vinstr_get_timestamp(void) +{ + struct timespec ts; + + getrawmonotonic(&ts); + return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; +} + +/** + * kbasep_vinstr_add_dump_request - register client's dumping request + * @cli: requesting client + * @waiting_clients: list of pending dumping requests + */ +static void kbasep_vinstr_add_dump_request( + struct kbase_vinstr_client *cli, + struct list_head *waiting_clients) +{ + struct kbase_vinstr_client *tmp; + + if (list_empty(waiting_clients)) { + list_add(&cli->list, waiting_clients); + return; + } + list_for_each_entry(tmp, waiting_clients, list) { + if (tmp->dump_time > cli->dump_time) { + list_add_tail(&cli->list, &tmp->list); + return; + } + } + list_add_tail(&cli->list, waiting_clients); +} + +/** + * kbasep_vinstr_collect_and_accumulate - collect hw counters via low level + * dump and accumulate them for known + * clients + * @vinstr_ctx: vinstr context + * @timestamp: pointer where collection timestamp will be recorded + * + * Return: zero on success + */ +static int kbasep_vinstr_collect_and_accumulate( + struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp) +{ + unsigned long flags; + int rcode; + +#ifdef CONFIG_MALI_BIFROST_NO_MALI + /* The dummy model needs the CPU mapping. */ + gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va); +#endif + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + if (VINSTR_IDLE != vinstr_ctx->state) { + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + return -EAGAIN; + } else { + vinstr_ctx->state = VINSTR_DUMPING; + } + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + /* Request HW counters dump. + * Disable preemption to make dump timestamp more accurate. */ + preempt_disable(); + *timestamp = kbasep_vinstr_get_timestamp(); + rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx); + preempt_enable(); + + if (!rcode) + rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx); + WARN_ON(rcode); + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + switch (vinstr_ctx->state) + { + case VINSTR_SUSPENDING: + schedule_work(&vinstr_ctx->suspend_work); + break; + case VINSTR_DUMPING: + vinstr_ctx->state = VINSTR_IDLE; + wake_up_all(&vinstr_ctx->suspend_waitq); + break; + default: + break; + } + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + /* Accumulate values of collected counters. */ + if (!rcode) + accum_clients(vinstr_ctx); + + return rcode; +} + +/** + * kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel + * buffer + * @cli: requesting client + * @timestamp: timestamp when counters were collected + * @event_id: id of event that caused triggered counters collection + * + * Return: zero on success + */ +static int kbasep_vinstr_fill_dump_buffer( + struct kbase_vinstr_client *cli, u64 timestamp, + enum base_hwcnt_reader_event event_id) +{ + unsigned int write_idx = atomic_read(&cli->write_idx); + unsigned int read_idx = atomic_read(&cli->read_idx); + + struct kbase_hwcnt_reader_metadata *meta; + void *buffer; + + /* Check if there is a place to copy HWC block into. */ + if (write_idx - read_idx == cli->buffer_count) + return -1; + write_idx %= cli->buffer_count; + + /* Fill in dump buffer and its metadata. */ + buffer = &cli->dump_buffers[write_idx * cli->dump_size]; + meta = &cli->dump_buffers_meta[write_idx]; + meta->timestamp = timestamp; + meta->event_id = event_id; + meta->buffer_idx = write_idx; + memcpy(buffer, cli->accum_buffer, cli->dump_size); + return 0; +} + +/** + * kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer + * allocated in userspace + * @cli: requesting client + * + * Return: zero on success + * + * This is part of legacy ioctl interface. + */ +static int kbasep_vinstr_fill_dump_buffer_legacy( + struct kbase_vinstr_client *cli) +{ + void __user *buffer = cli->legacy_buffer; + int rcode; + + /* Copy data to user buffer. */ + rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size); + if (rcode) { + pr_warn("error while copying buffer to user\n"); + return -EFAULT; + } + return 0; +} + +/** + * kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer + * allocated in kernel space + * @cli: requesting client + * + * Return: zero on success + * + * This is part of the kernel client interface. + */ +static int kbasep_vinstr_fill_dump_buffer_kernel( + struct kbase_vinstr_client *cli) +{ + memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size); + + return 0; +} + +/** + * kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst + * @vinstr_ctx: vinstr context + */ +static void kbasep_vinstr_reprogram( + struct kbase_vinstr_context *vinstr_ctx) +{ + unsigned long flags; + bool suspended = false; + + /* Don't enable hardware counters if vinstr is suspended. */ + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + if (VINSTR_IDLE != vinstr_ctx->state) + suspended = true; + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + if (suspended) + return; + + /* Change to suspended state is done while holding vinstr context + * lock. Below code will then no re-enable the instrumentation. */ + + if (vinstr_ctx->reprogram) { + struct kbase_vinstr_client *iter; + + if (!reprogram_hwcnt(vinstr_ctx)) { + vinstr_ctx->reprogram = false; + list_for_each_entry( + iter, + &vinstr_ctx->idle_clients, + list) + iter->pending = false; + list_for_each_entry( + iter, + &vinstr_ctx->waiting_clients, + list) + iter->pending = false; + } + } +} + +/** + * kbasep_vinstr_update_client - copy accumulated counters to user readable + * buffer and notify the user + * @cli: requesting client + * @timestamp: timestamp when counters were collected + * @event_id: id of event that caused triggered counters collection + * + * Return: zero on success + */ +static int kbasep_vinstr_update_client( + struct kbase_vinstr_client *cli, u64 timestamp, + enum base_hwcnt_reader_event event_id) +{ + int rcode = 0; + + /* Copy collected counters to user readable buffer. */ + if (cli->buffer_count) + rcode = kbasep_vinstr_fill_dump_buffer( + cli, timestamp, event_id); + else if (cli->kernel_buffer) + rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli); + else + rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli); + + if (rcode) + goto exit; + + + /* Notify client. Make sure all changes to memory are visible. */ + wmb(); + atomic_inc(&cli->write_idx); + wake_up_interruptible(&cli->waitq); + + /* Prepare for next request. */ + memset(cli->accum_buffer, 0, cli->dump_size); + +exit: + return rcode; +} + +/** + * kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function + * + * @hrtimer: high resolution timer + * + * Return: High resolution timer restart enum. + */ +static enum hrtimer_restart kbasep_vinstr_wake_up_callback( + struct hrtimer *hrtimer) +{ + struct kbasep_vinstr_wake_up_timer *timer = + container_of( + hrtimer, + struct kbasep_vinstr_wake_up_timer, + hrtimer); + + KBASE_DEBUG_ASSERT(timer); + + atomic_set(&timer->vinstr_ctx->request_pending, 1); + wake_up_all(&timer->vinstr_ctx->waitq); + + return HRTIMER_NORESTART; +} + +/** + * kbasep_vinstr_service_task - HWC dumping service thread + * + * @data: Pointer to vinstr context structure. + * + * Return: 0 on success; -ENOMEM if timer allocation fails + */ +static int kbasep_vinstr_service_task(void *data) +{ + struct kbase_vinstr_context *vinstr_ctx = data; + struct kbasep_vinstr_wake_up_timer *timer; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + timer = kmalloc(sizeof(*timer), GFP_KERNEL); + + if (!timer) { + dev_warn(vinstr_ctx->kbdev->dev, "Timer allocation failed!\n"); + return -ENOMEM; + } + + hrtimer_init(&timer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + + timer->hrtimer.function = kbasep_vinstr_wake_up_callback; + timer->vinstr_ctx = vinstr_ctx; + + while (!kthread_should_stop()) { + struct kbase_vinstr_client *cli = NULL; + struct kbase_vinstr_client *tmp; + int rcode; + + u64 timestamp = kbasep_vinstr_get_timestamp(); + u64 dump_time = 0; + struct list_head expired_requests; + + /* Hold lock while performing operations on lists of clients. */ + mutex_lock(&vinstr_ctx->lock); + + /* Closing thread must not interact with client requests. */ + if (current == vinstr_ctx->thread) { + atomic_set(&vinstr_ctx->request_pending, 0); + + if (!list_empty(&vinstr_ctx->waiting_clients)) { + cli = list_first_entry( + &vinstr_ctx->waiting_clients, + struct kbase_vinstr_client, + list); + dump_time = cli->dump_time; + } + } + + if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) { + mutex_unlock(&vinstr_ctx->lock); + + /* Sleep until next dumping event or service request. */ + if (cli) { + u64 diff = dump_time - timestamp; + + hrtimer_start( + &timer->hrtimer, + ns_to_ktime(diff), + HRTIMER_MODE_REL); + } + wait_event( + vinstr_ctx->waitq, + atomic_read( + &vinstr_ctx->request_pending) || + kthread_should_stop()); + hrtimer_cancel(&timer->hrtimer); + continue; + } + + rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, + ×tamp); + + INIT_LIST_HEAD(&expired_requests); + + /* Find all expired requests. */ + list_for_each_entry_safe( + cli, + tmp, + &vinstr_ctx->waiting_clients, + list) { + s64 tdiff = + (s64)(timestamp + DUMPING_RESOLUTION) - + (s64)cli->dump_time; + if (tdiff >= 0ll) { + list_del(&cli->list); + list_add(&cli->list, &expired_requests); + } else { + break; + } + } + + /* Fill data for each request found. */ + list_for_each_entry_safe(cli, tmp, &expired_requests, list) { + /* Ensure that legacy buffer will not be used from + * this kthread context. */ + BUG_ON(0 == cli->buffer_count); + /* Expect only periodically sampled clients. */ + BUG_ON(0 == cli->dump_interval); + + if (!rcode) + kbasep_vinstr_update_client( + cli, + timestamp, + BASE_HWCNT_READER_EVENT_PERIODIC); + + /* Set new dumping time. Drop missed probing times. */ + do { + cli->dump_time += cli->dump_interval; + } while (cli->dump_time < timestamp); + + list_del(&cli->list); + kbasep_vinstr_add_dump_request( + cli, + &vinstr_ctx->waiting_clients); + } + + /* Reprogram counters set if required. */ + kbasep_vinstr_reprogram(vinstr_ctx); + + mutex_unlock(&vinstr_ctx->lock); + } + + kfree(timer); + + return 0; +} + +/*****************************************************************************/ + +/** + * kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers + * @cli: pointer to vinstr client structure + * + * Return: non-zero if client has at least one dumping buffer filled that was + * not notified to user yet + */ +static int kbasep_vinstr_hwcnt_reader_buffer_ready( + struct kbase_vinstr_client *cli) +{ + KBASE_DEBUG_ASSERT(cli); + return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx); +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @buffer: pointer to userspace buffer + * @size: size of buffer + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( + struct kbase_vinstr_client *cli, void __user *buffer, + size_t size) +{ + unsigned int meta_idx = atomic_read(&cli->meta_idx); + unsigned int idx = meta_idx % cli->buffer_count; + + struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx]; + + /* Metadata sanity check. */ + KBASE_DEBUG_ASSERT(idx == meta->buffer_idx); + + if (sizeof(struct kbase_hwcnt_reader_metadata) != size) + return -EINVAL; + + /* Check if there is any buffer available. */ + if (atomic_read(&cli->write_idx) == meta_idx) + return -EAGAIN; + + /* Check if previously taken buffer was put back. */ + if (atomic_read(&cli->read_idx) != meta_idx) + return -EBUSY; + + /* Copy next available buffer's metadata to user. */ + if (copy_to_user(buffer, meta, size)) + return -EFAULT; + + atomic_inc(&cli->meta_idx); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @buffer: pointer to userspace buffer + * @size: size of buffer + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( + struct kbase_vinstr_client *cli, void __user *buffer, + size_t size) +{ + unsigned int read_idx = atomic_read(&cli->read_idx); + unsigned int idx = read_idx % cli->buffer_count; + + struct kbase_hwcnt_reader_metadata meta; + + if (sizeof(struct kbase_hwcnt_reader_metadata) != size) + return -EINVAL; + + /* Check if any buffer was taken. */ + if (atomic_read(&cli->meta_idx) == read_idx) + return -EPERM; + + /* Check if correct buffer is put back. */ + if (copy_from_user(&meta, buffer, size)) + return -EFAULT; + if (idx != meta.buffer_idx) + return -EINVAL; + + atomic_inc(&cli->read_idx); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @interval: periodic dumping interval (disable periodic dumping if zero) + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval( + struct kbase_vinstr_client *cli, u32 interval) +{ + struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + mutex_lock(&vinstr_ctx->lock); + + list_del(&cli->list); + + cli->dump_interval = interval; + + /* If interval is non-zero, enable periodic dumping for this client. */ + if (cli->dump_interval) { + if (DUMPING_RESOLUTION > cli->dump_interval) + cli->dump_interval = DUMPING_RESOLUTION; + cli->dump_time = + kbasep_vinstr_get_timestamp() + cli->dump_interval; + + kbasep_vinstr_add_dump_request( + cli, &vinstr_ctx->waiting_clients); + + atomic_set(&vinstr_ctx->request_pending, 1); + wake_up_all(&vinstr_ctx->waitq); + } else { + list_add(&cli->list, &vinstr_ctx->idle_clients); + } + + mutex_unlock(&vinstr_ctx->lock); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id + * @event_id: id of event + * Return: event_mask or zero if event is not supported or maskable + */ +static u32 kbasep_vinstr_hwcnt_reader_event_mask( + enum base_hwcnt_reader_event event_id) +{ + u32 event_mask = 0; + + switch (event_id) { + case BASE_HWCNT_READER_EVENT_PREJOB: + case BASE_HWCNT_READER_EVENT_POSTJOB: + /* These event are maskable. */ + event_mask = (1 << event_id); + break; + + case BASE_HWCNT_READER_EVENT_MANUAL: + case BASE_HWCNT_READER_EVENT_PERIODIC: + /* These event are non-maskable. */ + default: + /* These event are not supported. */ + break; + } + + return event_mask; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @event_id: id of event to enable + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event( + struct kbase_vinstr_client *cli, + enum base_hwcnt_reader_event event_id) +{ + struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; + u32 event_mask; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id); + if (!event_mask) + return -EINVAL; + + mutex_lock(&vinstr_ctx->lock); + cli->event_mask |= event_mask; + mutex_unlock(&vinstr_ctx->lock); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @event_id: id of event to disable + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event( + struct kbase_vinstr_client *cli, + enum base_hwcnt_reader_event event_id) +{ + struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; + u32 event_mask; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id); + if (!event_mask) + return -EINVAL; + + mutex_lock(&vinstr_ctx->lock); + cli->event_mask &= ~event_mask; + mutex_unlock(&vinstr_ctx->lock); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @hwver: pointer to user buffer where hw version will be stored + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( + struct kbase_vinstr_client *cli, u32 __user *hwver) +{ +#ifndef CONFIG_MALI_BIFROST_NO_MALI + struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; +#endif + + u32 ver = 5; + +#ifndef CONFIG_MALI_BIFROST_NO_MALI + KBASE_DEBUG_ASSERT(vinstr_ctx); + if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4)) + ver = 4; +#endif + + return put_user(ver, hwver); +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl + * @filp: pointer to file structure + * @cmd: user command + * @arg: command's argument + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg) +{ + long rcode = 0; + struct kbase_vinstr_client *cli; + + KBASE_DEBUG_ASSERT(filp); + + cli = filp->private_data; + KBASE_DEBUG_ASSERT(cli); + + if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd))) + return -EINVAL; + + switch (cmd) { + case KBASE_HWCNT_READER_GET_API_VERSION: + rcode = put_user(HWCNT_READER_API, (u32 __user *)arg); + break; + case KBASE_HWCNT_READER_GET_HWVER: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( + cli, (u32 __user *)arg); + break; + case KBASE_HWCNT_READER_GET_BUFFER_SIZE: + KBASE_DEBUG_ASSERT(cli->vinstr_ctx); + rcode = put_user( + (u32)cli->vinstr_ctx->dump_size, + (u32 __user *)arg); + break; + case KBASE_HWCNT_READER_DUMP: + rcode = kbase_vinstr_hwc_dump( + cli, BASE_HWCNT_READER_EVENT_MANUAL); + break; + case KBASE_HWCNT_READER_CLEAR: + rcode = kbase_vinstr_hwc_clear(cli); + break; + case KBASE_HWCNT_READER_GET_BUFFER: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( + cli, (void __user *)arg, _IOC_SIZE(cmd)); + break; + case KBASE_HWCNT_READER_PUT_BUFFER: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( + cli, (void __user *)arg, _IOC_SIZE(cmd)); + break; + case KBASE_HWCNT_READER_SET_INTERVAL: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval( + cli, (u32)arg); + break; + case KBASE_HWCNT_READER_ENABLE_EVENT: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event( + cli, (enum base_hwcnt_reader_event)arg); + break; + case KBASE_HWCNT_READER_DISABLE_EVENT: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event( + cli, (enum base_hwcnt_reader_event)arg); + break; + default: + rcode = -EINVAL; + break; + } + + return rcode; +} + +/** + * kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll + * @filp: pointer to file structure + * @wait: pointer to poll table + * Return: POLLIN if data can be read without blocking, otherwise zero + */ +static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp, + poll_table *wait) +{ + struct kbase_vinstr_client *cli; + + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(wait); + + cli = filp->private_data; + KBASE_DEBUG_ASSERT(cli); + + poll_wait(filp, &cli->waitq, wait); + if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli)) + return POLLIN; + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap + * @filp: pointer to file structure + * @vma: pointer to vma structure + * Return: zero on success + */ +static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp, + struct vm_area_struct *vma) +{ + struct kbase_vinstr_client *cli; + unsigned long size, addr, pfn, offset; + unsigned long vm_size = vma->vm_end - vma->vm_start; + + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(vma); + + cli = filp->private_data; + KBASE_DEBUG_ASSERT(cli); + + size = cli->buffer_count * cli->dump_size; + + if (vma->vm_pgoff > (size >> PAGE_SHIFT)) + return -EINVAL; + + offset = vma->vm_pgoff << PAGE_SHIFT; + if (vm_size > size - offset) + return -EINVAL; + + addr = __pa((unsigned long)cli->dump_buffers + offset); + pfn = addr >> PAGE_SHIFT; + + return remap_pfn_range( + vma, + vma->vm_start, + pfn, + vm_size, + vma->vm_page_prot); +} + +/** + * kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release + * @inode: pointer to inode structure + * @filp: pointer to file structure + * Return always return zero + */ +static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode, + struct file *filp) +{ + struct kbase_vinstr_client *cli; + + KBASE_DEBUG_ASSERT(inode); + KBASE_DEBUG_ASSERT(filp); + + cli = filp->private_data; + KBASE_DEBUG_ASSERT(cli); + + kbase_vinstr_detach_client(cli); + return 0; +} + +/*****************************************************************************/ + +/** + * kbasep_vinstr_kick_scheduler - trigger scheduler cycle + * @kbdev: pointer to kbase device structure + */ +static void kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + unsigned long flags; + + down(&js_devdata->schedule_sem); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_backend_slot_update(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + up(&js_devdata->schedule_sem); +} + +/** + * kbasep_vinstr_suspend_worker - worker suspending vinstr module + * @data: pointer to work structure + */ +static void kbasep_vinstr_suspend_worker(struct work_struct *data) +{ + struct kbase_vinstr_context *vinstr_ctx; + unsigned long flags; + + vinstr_ctx = container_of(data, struct kbase_vinstr_context, + suspend_work); + + mutex_lock(&vinstr_ctx->lock); + + if (vinstr_ctx->kctx) + disable_hwcnt(vinstr_ctx); + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + vinstr_ctx->state = VINSTR_SUSPENDED; + wake_up_all(&vinstr_ctx->suspend_waitq); + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + mutex_unlock(&vinstr_ctx->lock); + + /* Kick GPU scheduler to allow entering protected mode. + * This must happen after vinstr was suspended. */ + kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev); +} + +/** + * kbasep_vinstr_suspend_worker - worker resuming vinstr module + * @data: pointer to work structure + */ +static void kbasep_vinstr_resume_worker(struct work_struct *data) +{ + struct kbase_vinstr_context *vinstr_ctx; + unsigned long flags; + + vinstr_ctx = container_of(data, struct kbase_vinstr_context, + resume_work); + + mutex_lock(&vinstr_ctx->lock); + + if (vinstr_ctx->kctx) + enable_hwcnt(vinstr_ctx); + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + vinstr_ctx->state = VINSTR_IDLE; + wake_up_all(&vinstr_ctx->suspend_waitq); + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + mutex_unlock(&vinstr_ctx->lock); + + /* Kick GPU scheduler to allow entering protected mode. + * Note that scheduler state machine might requested re-entry to + * protected mode before vinstr was resumed. + * This must happen after vinstr was release. */ + kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev); +} + +/*****************************************************************************/ + +struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev) +{ + struct kbase_vinstr_context *vinstr_ctx; + + vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL); + if (!vinstr_ctx) + return NULL; + + INIT_LIST_HEAD(&vinstr_ctx->idle_clients); + INIT_LIST_HEAD(&vinstr_ctx->waiting_clients); + mutex_init(&vinstr_ctx->lock); + spin_lock_init(&vinstr_ctx->state_lock); + vinstr_ctx->kbdev = kbdev; + vinstr_ctx->thread = NULL; + vinstr_ctx->state = VINSTR_IDLE; + vinstr_ctx->suspend_cnt = 0; + INIT_WORK(&vinstr_ctx->suspend_work, kbasep_vinstr_suspend_worker); + INIT_WORK(&vinstr_ctx->resume_work, kbasep_vinstr_resume_worker); + init_waitqueue_head(&vinstr_ctx->suspend_waitq); + + atomic_set(&vinstr_ctx->request_pending, 0); + init_waitqueue_head(&vinstr_ctx->waitq); + + return vinstr_ctx; +} + +void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_vinstr_client *cli; + + /* Stop service thread first. */ + if (vinstr_ctx->thread) + kthread_stop(vinstr_ctx->thread); + + /* Wait for workers. */ + flush_work(&vinstr_ctx->suspend_work); + flush_work(&vinstr_ctx->resume_work); + + while (1) { + struct list_head *list = &vinstr_ctx->idle_clients; + + if (list_empty(list)) { + list = &vinstr_ctx->waiting_clients; + if (list_empty(list)) + break; + } + + cli = list_first_entry(list, struct kbase_vinstr_client, list); + list_del(&cli->list); + kfree(cli->accum_buffer); + kfree(cli); + vinstr_ctx->nclients--; + } + KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients); + if (vinstr_ctx->kctx) + kbasep_vinstr_destroy_kctx(vinstr_ctx); + kfree(vinstr_ctx); +} + +int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx, + struct kbase_uk_hwcnt_reader_setup *setup) +{ + struct kbase_vinstr_client *cli; + u32 bitmap[4]; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + KBASE_DEBUG_ASSERT(setup); + KBASE_DEBUG_ASSERT(setup->buffer_count); + + bitmap[SHADER_HWCNT_BM] = setup->shader_bm; + bitmap[TILER_HWCNT_BM] = setup->tiler_bm; + bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; + bitmap[JM_HWCNT_BM] = setup->jm_bm; + + cli = kbasep_vinstr_attach_client( + vinstr_ctx, + setup->buffer_count, + bitmap, + &setup->fd, + NULL); + + if (!cli) + return -ENOMEM; + + return 0; +} + +int kbase_vinstr_legacy_hwc_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_vinstr_client **cli, + struct kbase_uk_hwcnt_setup *setup) +{ + KBASE_DEBUG_ASSERT(vinstr_ctx); + KBASE_DEBUG_ASSERT(setup); + KBASE_DEBUG_ASSERT(cli); + + if (setup->dump_buffer) { + u32 bitmap[4]; + + bitmap[SHADER_HWCNT_BM] = setup->shader_bm; + bitmap[TILER_HWCNT_BM] = setup->tiler_bm; + bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; + bitmap[JM_HWCNT_BM] = setup->jm_bm; + + if (*cli) + return -EBUSY; + + *cli = kbasep_vinstr_attach_client( + vinstr_ctx, + 0, + bitmap, + (void *)(long)setup->dump_buffer, + NULL); + + if (!(*cli)) + return -ENOMEM; + } else { + if (!*cli) + return -EINVAL; + + kbase_vinstr_detach_client(*cli); + *cli = NULL; + } + + return 0; +} + +struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_uk_hwcnt_reader_setup *setup, + void *kernel_buffer) +{ + u32 bitmap[4]; + + if (!vinstr_ctx || !setup || !kernel_buffer) + return NULL; + + bitmap[SHADER_HWCNT_BM] = setup->shader_bm; + bitmap[TILER_HWCNT_BM] = setup->tiler_bm; + bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; + bitmap[JM_HWCNT_BM] = setup->jm_bm; + + return kbasep_vinstr_attach_client( + vinstr_ctx, + 0, + bitmap, + NULL, + kernel_buffer); +} +KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup); + +int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli, + enum base_hwcnt_reader_event event_id) +{ + int rcode = 0; + struct kbase_vinstr_context *vinstr_ctx; + u64 timestamp; + u32 event_mask; + + if (!cli) + return -EINVAL; + + vinstr_ctx = cli->vinstr_ctx; + KBASE_DEBUG_ASSERT(vinstr_ctx); + + KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT); + event_mask = 1 << event_id; + + mutex_lock(&vinstr_ctx->lock); + + if (event_mask & cli->event_mask) { + rcode = kbasep_vinstr_collect_and_accumulate( + vinstr_ctx, + ×tamp); + if (rcode) + goto exit; + + rcode = kbasep_vinstr_update_client(cli, timestamp, event_id); + if (rcode) + goto exit; + + kbasep_vinstr_reprogram(vinstr_ctx); + } + +exit: + mutex_unlock(&vinstr_ctx->lock); + + return rcode; +} +KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump); + +int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli) +{ + struct kbase_vinstr_context *vinstr_ctx; + int rcode; + u64 unused; + + if (!cli) + return -EINVAL; + + vinstr_ctx = cli->vinstr_ctx; + KBASE_DEBUG_ASSERT(vinstr_ctx); + + mutex_lock(&vinstr_ctx->lock); + + rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused); + if (rcode) + goto exit; + rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx); + if (rcode) + goto exit; + memset(cli->accum_buffer, 0, cli->dump_size); + + kbasep_vinstr_reprogram(vinstr_ctx); + +exit: + mutex_unlock(&vinstr_ctx->lock); + + return rcode; +} + +int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx) +{ + unsigned long flags; + int ret = -EAGAIN; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + switch (vinstr_ctx->state) { + case VINSTR_SUSPENDED: + vinstr_ctx->suspend_cnt++; + /* overflow shall not happen */ + BUG_ON(0 == vinstr_ctx->suspend_cnt); + ret = 0; + break; + + case VINSTR_IDLE: + if (vinstr_ctx->clients_present) { + vinstr_ctx->state = VINSTR_SUSPENDING; + schedule_work(&vinstr_ctx->suspend_work); + } else { + vinstr_ctx->state = VINSTR_SUSPENDED; + + vinstr_ctx->suspend_cnt++; + /* overflow shall not happen */ + WARN_ON(0 == vinstr_ctx->suspend_cnt); + ret = 0; + } + break; + + case VINSTR_DUMPING: + vinstr_ctx->state = VINSTR_SUSPENDING; + break; + + case VINSTR_SUSPENDING: + /* fall through */ + case VINSTR_RESUMING: + break; + + default: + BUG(); + break; + } + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + return ret; +} + +void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx) +{ + wait_event(vinstr_ctx->suspend_waitq, + (0 == kbase_vinstr_try_suspend(vinstr_ctx))); +} + +void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + BUG_ON(VINSTR_SUSPENDING == vinstr_ctx->state); + if (VINSTR_SUSPENDED == vinstr_ctx->state) { + BUG_ON(0 == vinstr_ctx->suspend_cnt); + vinstr_ctx->suspend_cnt--; + if (0 == vinstr_ctx->suspend_cnt) { + if (vinstr_ctx->clients_present) { + vinstr_ctx->state = VINSTR_RESUMING; + schedule_work(&vinstr_ctx->resume_work); + } else { + vinstr_ctx->state = VINSTR_IDLE; + } + } + } + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.h b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.h new file mode 100755 index 000000000000..6207d25aef06 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_kbase_vinstr.h @@ -0,0 +1,155 @@ +/* + * + * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_VINSTR_H_ +#define _KBASE_VINSTR_H_ + +#include +#include + +/*****************************************************************************/ + +struct kbase_vinstr_context; +struct kbase_vinstr_client; + +/*****************************************************************************/ + +/** + * kbase_vinstr_init() - initialize the vinstr core + * @kbdev: kbase device + * + * Return: pointer to the vinstr context on success or NULL on failure + */ +struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev); + +/** + * kbase_vinstr_term() - terminate the vinstr core + * @vinstr_ctx: vinstr context + */ +void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx); + +/** + * kbase_vinstr_hwcnt_reader_setup - configure hw counters reader + * @vinstr_ctx: vinstr context + * @setup: reader's configuration + * + * Return: zero on success + */ +int kbase_vinstr_hwcnt_reader_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_uk_hwcnt_reader_setup *setup); + +/** + * kbase_vinstr_legacy_hwc_setup - configure hw counters for dumping + * @vinstr_ctx: vinstr context + * @cli: pointer where to store pointer to new vinstr client structure + * @setup: hwc configuration + * + * Return: zero on success + */ +int kbase_vinstr_legacy_hwc_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_vinstr_client **cli, + struct kbase_uk_hwcnt_setup *setup); + +/** + * kbase_vinstr_hwcnt_kernel_setup - configure hw counters for kernel side + * client + * @vinstr_ctx: vinstr context + * @setup: reader's configuration + * @kernel_buffer: pointer to dump buffer + * + * setup->buffer_count and setup->fd are not used for kernel side clients. + * + * Return: pointer to client structure, or NULL on failure + */ +struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_uk_hwcnt_reader_setup *setup, + void *kernel_buffer); + +/** + * kbase_vinstr_hwc_dump - issue counter dump for vinstr client + * @cli: pointer to vinstr client + * @event_id: id of event that triggered hwcnt dump + * + * Return: zero on success + */ +int kbase_vinstr_hwc_dump( + struct kbase_vinstr_client *cli, + enum base_hwcnt_reader_event event_id); + +/** + * kbase_vinstr_hwc_clear - performs a reset of the hardware counters for + * a given kbase context + * @cli: pointer to vinstr client + * + * Return: zero on success + */ +int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli); + +/** + * kbase_vinstr_try_suspend - try suspending operation of a given vinstr context + * @vinstr_ctx: vinstr context + * + * Return: 0 on success, or negative if state change is in progress + * + * Warning: This API call is non-generic. It is meant to be used only by + * job scheduler state machine. + * + * Function initiates vinstr switch to suspended state. Once it was called + * vinstr enters suspending state. If function return non-zero value, it + * indicates that state switch is not complete and function must be called + * again. On state switch vinstr will trigger job scheduler state machine + * cycle. + */ +int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx); + +/** + * kbase_vinstr_suspend - suspends operation of a given vinstr context + * @vinstr_ctx: vinstr context + * + * Function initiates vinstr switch to suspended state. Then it blocks until + * operation is completed. + */ +void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx); + +/** + * kbase_vinstr_resume - resumes operation of a given vinstr context + * @vinstr_ctx: vinstr context + * + * Function can be called only if it was preceded by a successful call + * to kbase_vinstr_suspend. + */ +void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx); + +/** + * kbase_vinstr_dump_size - Return required size of dump buffer + * @kbdev: device pointer + * + * Return : buffer size in bytes + */ +size_t kbase_vinstr_dump_size(struct kbase_device *kbdev); + +/** + * kbase_vinstr_detach_client - Detach a client from the vinstr core + * @cli: pointer to vinstr client + */ +void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli); + +#endif /* _KBASE_VINSTR_H_ */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_linux_kbase_trace.h b/drivers/gpu/arm/bifrost_for_linux/mali_linux_kbase_trace.h new file mode 100755 index 000000000000..5d6b4021d626 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_linux_kbase_trace.h @@ -0,0 +1,201 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +#if !defined(_TRACE_MALI_KBASE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_MALI_KBASE_H + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mali + +#include + +DECLARE_EVENT_CLASS(mali_slot_template, + TP_PROTO(int jobslot, unsigned int info_val), + TP_ARGS(jobslot, info_val), + TP_STRUCT__entry( + __field(unsigned int, jobslot) + __field(unsigned int, info_val) + ), + TP_fast_assign( + __entry->jobslot = jobslot; + __entry->info_val = info_val; + ), + TP_printk("jobslot=%u info=%u", __entry->jobslot, __entry->info_val) +); + +#define DEFINE_MALI_SLOT_EVENT(name) \ +DEFINE_EVENT(mali_slot_template, mali_##name, \ + TP_PROTO(int jobslot, unsigned int info_val), \ + TP_ARGS(jobslot, info_val)) +DEFINE_MALI_SLOT_EVENT(JM_SUBMIT); +DEFINE_MALI_SLOT_EVENT(JM_JOB_DONE); +DEFINE_MALI_SLOT_EVENT(JM_UPDATE_HEAD); +DEFINE_MALI_SLOT_EVENT(JM_CHECK_HEAD); +DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP); +DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_0); +DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_1); +DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP); +DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_0); +DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_1); +DEFINE_MALI_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); +DEFINE_MALI_SLOT_EVENT(JM_SLOT_EVICT); +DEFINE_MALI_SLOT_EVENT(JM_BEGIN_RESET_WORKER); +DEFINE_MALI_SLOT_EVENT(JM_END_RESET_WORKER); +DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); +DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); +DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_CURRENT); +DEFINE_MALI_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); +DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); +DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); +DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); +DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); +DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); +DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); +DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); +DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); +#undef DEFINE_MALI_SLOT_EVENT + +DECLARE_EVENT_CLASS(mali_refcount_template, + TP_PROTO(int refcount, unsigned int info_val), + TP_ARGS(refcount, info_val), + TP_STRUCT__entry( + __field(unsigned int, refcount) + __field(unsigned int, info_val) + ), + TP_fast_assign( + __entry->refcount = refcount; + __entry->info_val = info_val; + ), + TP_printk("refcount=%u info=%u", __entry->refcount, __entry->info_val) +); + +#define DEFINE_MALI_REFCOUNT_EVENT(name) \ +DEFINE_EVENT(mali_refcount_template, mali_##name, \ + TP_PROTO(int refcount, unsigned int info_val), \ + TP_ARGS(refcount, info_val)) +DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX_NOLOCK); +DEFINE_MALI_REFCOUNT_EVENT(JS_ADD_JOB); +DEFINE_MALI_REFCOUNT_EVENT(JS_REMOVE_JOB); +DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_RELEASE_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); +DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_ACTIVE); +DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_IDLE); +#undef DEFINE_MALI_REFCOUNT_EVENT + +DECLARE_EVENT_CLASS(mali_add_template, + TP_PROTO(int gpu_addr, unsigned int info_val), + TP_ARGS(gpu_addr, info_val), + TP_STRUCT__entry( + __field(unsigned int, gpu_addr) + __field(unsigned int, info_val) + ), + TP_fast_assign( + __entry->gpu_addr = gpu_addr; + __entry->info_val = info_val; + ), + TP_printk("gpu_addr=%u info=%u", __entry->gpu_addr, __entry->info_val) +); + +#define DEFINE_MALI_ADD_EVENT(name) \ +DEFINE_EVENT(mali_add_template, mali_##name, \ + TP_PROTO(int gpu_addr, unsigned int info_val), \ + TP_ARGS(gpu_addr, info_val)) +DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY); +DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE); +DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET); +DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET); +DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE); +DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR); +DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES); +DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER); +DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER_END); +DEFINE_MALI_ADD_EVENT(JD_CANCEL_WORKER); +DEFINE_MALI_ADD_EVENT(JD_DONE); +DEFINE_MALI_ADD_EVENT(JD_CANCEL); +DEFINE_MALI_ADD_EVENT(JD_ZAP_CONTEXT); +DEFINE_MALI_ADD_EVENT(JM_IRQ); +DEFINE_MALI_ADD_EVENT(JM_IRQ_END); +DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS); +DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS_DONE); +DEFINE_MALI_ADD_EVENT(JM_ZAP_NON_SCHEDULED); +DEFINE_MALI_ADD_EVENT(JM_ZAP_SCHEDULED); +DEFINE_MALI_ADD_EVENT(JM_ZAP_DONE); +DEFINE_MALI_ADD_EVENT(JM_SUBMIT_AFTER_RESET); +DEFINE_MALI_ADD_EVENT(JM_JOB_COMPLETE); +DEFINE_MALI_ADD_EVENT(JS_FAST_START_EVICTS_CTX); +DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); +DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); +DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); +DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); +DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_END); +DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_START); +DEFINE_MALI_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED); +DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP); +DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP); +DEFINE_MALI_ADD_EVENT(PM_PWRON); +DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER); +DEFINE_MALI_ADD_EVENT(PM_PWRON_L2); +DEFINE_MALI_ADD_EVENT(PM_PWROFF); +DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER); +DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2); +DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED); +DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER); +DEFINE_MALI_ADD_EVENT(PM_UNREQUEST_CHANGE_SHADER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_REGISTER_CHANGE_SHADER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_REGISTER_CHANGE_SHADER_INUSE); +DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_INUSE); +DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE); +DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER); +DEFINE_MALI_ADD_EVENT(PM_GPU_ON); +DEFINE_MALI_ADD_EVENT(PM_GPU_OFF); +DEFINE_MALI_ADD_EVENT(PM_SET_POLICY); +DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT); +DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM); +DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY); +DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); +#undef DEFINE_MALI_ADD_EVENT + +#endif /* _TRACE_MALI_KBASE_H */ + +#undef TRACE_INCLUDE_PATH +#undef linux +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE mali_linux_kbase_trace + +/* This part must be outside protection */ +#include diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_linux_trace.h b/drivers/gpu/arm/bifrost_for_linux/mali_linux_trace.h new file mode 100755 index 000000000000..2be06a552768 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_linux_trace.h @@ -0,0 +1,189 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#if !defined(_TRACE_MALI_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_MALI_H + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mali +#define TRACE_INCLUDE_FILE mali_linux_trace + +#include + +#define MALI_JOB_SLOTS_EVENT_CHANGED + +/** + * mali_job_slots_event - called from mali_kbase_core_linux.c + * @event_id: ORed together bitfields representing a type of event, made with the GATOR_MAKE_EVENT() macro. + */ +TRACE_EVENT(mali_job_slots_event, + TP_PROTO(unsigned int event_id, unsigned int tgid, unsigned int pid, + unsigned char job_id), + TP_ARGS(event_id, tgid, pid, job_id), + TP_STRUCT__entry( + __field(unsigned int, event_id) + __field(unsigned int, tgid) + __field(unsigned int, pid) + __field(unsigned char, job_id) + ), + TP_fast_assign( + __entry->event_id = event_id; + __entry->tgid = tgid; + __entry->pid = pid; + __entry->job_id = job_id; + ), + TP_printk("event=%u tgid=%u pid=%u job_id=%u", + __entry->event_id, __entry->tgid, __entry->pid, __entry->job_id) +); + +/** + * mali_pm_status - Called by mali_kbase_pm_driver.c + * @event_id: core type (shader, tiler, l2 cache) + * @value: 64bits bitmask reporting either power status of the cores (1-ON, 0-OFF) + */ +TRACE_EVENT(mali_pm_status, + TP_PROTO(unsigned int event_id, unsigned long long value), + TP_ARGS(event_id, value), + TP_STRUCT__entry( + __field(unsigned int, event_id) + __field(unsigned long long, value) + ), + TP_fast_assign( + __entry->event_id = event_id; + __entry->value = value; + ), + TP_printk("event %u = %llu", __entry->event_id, __entry->value) +); + +/** + * mali_pm_power_on - Called by mali_kbase_pm_driver.c + * @event_id: core type (shader, tiler, l2 cache) + * @value: 64bits bitmask reporting the cores to power up + */ +TRACE_EVENT(mali_pm_power_on, + TP_PROTO(unsigned int event_id, unsigned long long value), + TP_ARGS(event_id, value), + TP_STRUCT__entry( + __field(unsigned int, event_id) + __field(unsigned long long, value) + ), + TP_fast_assign( + __entry->event_id = event_id; + __entry->value = value; + ), + TP_printk("event %u = %llu", __entry->event_id, __entry->value) +); + +/** + * mali_pm_power_off - Called by mali_kbase_pm_driver.c + * @event_id: core type (shader, tiler, l2 cache) + * @value: 64bits bitmask reporting the cores to power down + */ +TRACE_EVENT(mali_pm_power_off, + TP_PROTO(unsigned int event_id, unsigned long long value), + TP_ARGS(event_id, value), + TP_STRUCT__entry( + __field(unsigned int, event_id) + __field(unsigned long long, value) + ), + TP_fast_assign( + __entry->event_id = event_id; + __entry->value = value; + ), + TP_printk("event %u = %llu", __entry->event_id, __entry->value) +); + +/** + * mali_page_fault_insert_pages - Called by page_fault_worker() + * it reports an MMU page fault resulting in new pages being mapped. + * @event_id: MMU address space number. + * @value: number of newly allocated pages + */ +TRACE_EVENT(mali_page_fault_insert_pages, + TP_PROTO(int event_id, unsigned long value), + TP_ARGS(event_id, value), + TP_STRUCT__entry( + __field(int, event_id) + __field(unsigned long, value) + ), + TP_fast_assign( + __entry->event_id = event_id; + __entry->value = value; + ), + TP_printk("event %d = %lu", __entry->event_id, __entry->value) +); + +/** + * mali_mmu_as_in_use - Called by assign_and_activate_kctx_addr_space() + * it reports that a certain MMU address space is in use now. + * @event_id: MMU address space number. + */ +TRACE_EVENT(mali_mmu_as_in_use, + TP_PROTO(int event_id), + TP_ARGS(event_id), + TP_STRUCT__entry( + __field(int, event_id) + ), + TP_fast_assign( + __entry->event_id = event_id; + ), + TP_printk("event=%d", __entry->event_id) +); + +/** + * mali_mmu_as_released - Called by kbasep_js_runpool_release_ctx_internal() + * it reports that a certain MMU address space has been released now. + * @event_id: MMU address space number. + */ +TRACE_EVENT(mali_mmu_as_released, + TP_PROTO(int event_id), + TP_ARGS(event_id), + TP_STRUCT__entry( + __field(int, event_id) + ), + TP_fast_assign( + __entry->event_id = event_id; + ), + TP_printk("event=%d", __entry->event_id) +); + +/** + * mali_total_alloc_pages_change - Called by kbase_atomic_add_pages() + * and by kbase_atomic_sub_pages() + * it reports that the total number of allocated pages is changed. + * @event_id: number of pages to be added or subtracted (according to the sign). + */ +TRACE_EVENT(mali_total_alloc_pages_change, + TP_PROTO(long long int event_id), + TP_ARGS(event_id), + TP_STRUCT__entry( + __field(long long int, event_id) + ), + TP_fast_assign( + __entry->event_id = event_id; + ), + TP_printk("event=%lld", __entry->event_id) +); + +#endif /* _TRACE_MALI_H */ + +#undef TRACE_INCLUDE_PATH +#undef linux +#define TRACE_INCLUDE_PATH . + +/* This part must be outside protection */ +#include diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_malisw.h b/drivers/gpu/arm/bifrost_for_linux/mali_malisw.h new file mode 100755 index 000000000000..99452933eab4 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_malisw.h @@ -0,0 +1,131 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * Kernel-wide include for common macros and types. + */ + +#ifndef _MALISW_H_ +#define _MALISW_H_ + +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#define U8_MAX ((u8)~0U) +#define S8_MAX ((s8)(U8_MAX>>1)) +#define S8_MIN ((s8)(-S8_MAX - 1)) +#define U16_MAX ((u16)~0U) +#define S16_MAX ((s16)(U16_MAX>>1)) +#define S16_MIN ((s16)(-S16_MAX - 1)) +#define U32_MAX ((u32)~0U) +#define S32_MAX ((s32)(U32_MAX>>1)) +#define S32_MIN ((s32)(-S32_MAX - 1)) +#define U64_MAX ((u64)~0ULL) +#define S64_MAX ((s64)(U64_MAX>>1)) +#define S64_MIN ((s64)(-S64_MAX - 1)) +#endif /* LINUX_VERSION_CODE */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) +#define SIZE_MAX (~(size_t)0) +#endif /* LINUX_VERSION_CODE */ + +/** + * MIN - Return the lesser of two values. + * + * As a macro it may evaluate its arguments more than once. + * Refer to MAX macro for more details + */ +#define MIN(x, y) ((x) < (y) ? (x) : (y)) + +/** + * MAX - Return the greater of two values. + * + * As a macro it may evaluate its arguments more than once. + * If called on the same two arguments as MIN it is guaranteed to return + * the one that MIN didn't return. This is significant for types where not + * all values are comparable e.g. NaNs in floating-point types. But if you want + * to retrieve the min and max of two values, consider using a conditional swap + * instead. + */ +#define MAX(x, y) ((x) < (y) ? (y) : (x)) + +/** + * @hideinitializer + * Function-like macro for suppressing unused variable warnings. Where possible + * such variables should be removed; this macro is present for cases where we + * much support API backwards compatibility. + */ +#define CSTD_UNUSED(x) ((void)(x)) + +/** + * @hideinitializer + * Function-like macro for use where "no behavior" is desired. This is useful + * when compile time macros turn a function-like macro in to a no-op, but + * where having no statement is otherwise invalid. + */ +#define CSTD_NOP(...) ((void)#__VA_ARGS__) + +/** + * Function-like macro for converting a pointer in to a u64 for storing into + * an external data structure. This is commonly used when pairing a 32-bit + * CPU with a 64-bit peripheral, such as a Midgard GPU. C's type promotion + * is complex and a straight cast does not work reliably as pointers are + * often considered as signed. + */ +#define PTR_TO_U64(x) ((uint64_t)((uintptr_t)(x))) + +/** + * @hideinitializer + * Function-like macro for stringizing a single level macro. + * @code + * #define MY_MACRO 32 + * CSTD_STR1( MY_MACRO ) + * > "MY_MACRO" + * @endcode + */ +#define CSTD_STR1(x) #x + +/** + * @hideinitializer + * Function-like macro for stringizing a macro's value. This should not be used + * if the macro is defined in a way which may have no value; use the + * alternative @c CSTD_STR2N macro should be used instead. + * @code + * #define MY_MACRO 32 + * CSTD_STR2( MY_MACRO ) + * > "32" + * @endcode + */ +#define CSTD_STR2(x) CSTD_STR1(x) + +/** + * Specify an assertion value which is evaluated at compile time. Recommended + * usage is specification of a @c static @c INLINE function containing all of + * the assertions thus: + * + * @code + * static INLINE [module]_compile_time_assertions( void ) + * { + * COMPILE_TIME_ASSERT( sizeof(uintptr_t) == sizeof(intptr_t) ); + * } + * @endcode + * + * @note Use @c static not @c STATIC. We never want to turn off this @c static + * specification for testing purposes. + */ +#define CSTD_COMPILE_TIME_ASSERT(expr) \ + do { switch (0) { case 0: case (expr):; } } while (false) + +#endif /* _MALISW_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_midg_coherency.h b/drivers/gpu/arm/bifrost_for_linux/mali_midg_coherency.h new file mode 100755 index 000000000000..a509cbd5f175 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_midg_coherency.h @@ -0,0 +1,26 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _MIDG_COHERENCY_H_ +#define _MIDG_COHERENCY_H_ + +#define COHERENCY_ACE_LITE 0 +#define COHERENCY_ACE 1 +#define COHERENCY_NONE 31 +#define COHERENCY_FEATURE_BIT(x) (1 << (x)) + +#endif /* _MIDG_COHERENCY_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_midg_regmap.h b/drivers/gpu/arm/bifrost_for_linux/mali_midg_regmap.h new file mode 100755 index 000000000000..554ed8dcb3eb --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_midg_regmap.h @@ -0,0 +1,611 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _MIDGARD_REGMAP_H_ +#define _MIDGARD_REGMAP_H_ + +#include "mali_midg_coherency.h" +#include "mali_kbase_gpu_id.h" + +/* + * Begin Register Offsets + */ + +#define GPU_CONTROL_BASE 0x0000 +#define GPU_CONTROL_REG(r) (GPU_CONTROL_BASE + (r)) +#define GPU_ID 0x000 /* (RO) GPU and revision identifier */ +#define L2_FEATURES 0x004 /* (RO) Level 2 cache features */ +#define SUSPEND_SIZE 0x008 /* (RO) Fixed-function suspend buffer + size */ +#define TILER_FEATURES 0x00C /* (RO) Tiler Features */ +#define MEM_FEATURES 0x010 /* (RO) Memory system features */ +#define MMU_FEATURES 0x014 /* (RO) MMU features */ +#define AS_PRESENT 0x018 /* (RO) Address space slots present */ +#define JS_PRESENT 0x01C /* (RO) Job slots present */ +#define GPU_IRQ_RAWSTAT 0x020 /* (RW) */ +#define GPU_IRQ_CLEAR 0x024 /* (WO) */ +#define GPU_IRQ_MASK 0x028 /* (RW) */ +#define GPU_IRQ_STATUS 0x02C /* (RO) */ + +/* IRQ flags */ +#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ +#define MULTIPLE_GPU_FAULTS (1 << 7) /* More than one GPU Fault occurred. */ +#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. Intended to use with SOFT_RESET + commands which may take time. */ +#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ +#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down + and the power manager is idle. */ + +#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ +#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ + +#define GPU_IRQ_REG_ALL (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ + | POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED) + +#define GPU_COMMAND 0x030 /* (WO) */ +#define GPU_STATUS 0x034 /* (RO) */ +#define LATEST_FLUSH 0x038 /* (RO) */ + +#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */ +#define GPU_DBGEN (1 << 8) /* DBGEN wire status */ + +#define GPU_FAULTSTATUS 0x03C /* (RO) GPU exception type and fault status */ +#define GPU_FAULTADDRESS_LO 0x040 /* (RO) GPU exception fault address, low word */ +#define GPU_FAULTADDRESS_HI 0x044 /* (RO) GPU exception fault address, high word */ + +#define PWR_KEY 0x050 /* (WO) Power manager key register */ +#define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */ +#define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */ + +#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory region base address, low word */ +#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory region base address, high word */ +#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter configuration */ +#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable flags for Job Manager */ +#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable flags for shader cores */ +#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable flags for tiler */ +#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable flags for MMU/L2 cache */ + +#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */ +#define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */ +#define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */ +#define TIMESTAMP_HI 0x09C /* (RO) Global time stamp counter, high word */ + +#define THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */ +#define THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */ +#define THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */ +#define THREAD_FEATURES 0x0AC /* (RO) Thread features */ + +#define TEXTURE_FEATURES_0 0x0B0 /* (RO) Support flags for indexed texture formats 0..31 */ +#define TEXTURE_FEATURES_1 0x0B4 /* (RO) Support flags for indexed texture formats 32..63 */ +#define TEXTURE_FEATURES_2 0x0B8 /* (RO) Support flags for indexed texture formats 64..95 */ + +#define TEXTURE_FEATURES_REG(n) GPU_CONTROL_REG(TEXTURE_FEATURES_0 + ((n) << 2)) + +#define JS0_FEATURES 0x0C0 /* (RO) Features of job slot 0 */ +#define JS1_FEATURES 0x0C4 /* (RO) Features of job slot 1 */ +#define JS2_FEATURES 0x0C8 /* (RO) Features of job slot 2 */ +#define JS3_FEATURES 0x0CC /* (RO) Features of job slot 3 */ +#define JS4_FEATURES 0x0D0 /* (RO) Features of job slot 4 */ +#define JS5_FEATURES 0x0D4 /* (RO) Features of job slot 5 */ +#define JS6_FEATURES 0x0D8 /* (RO) Features of job slot 6 */ +#define JS7_FEATURES 0x0DC /* (RO) Features of job slot 7 */ +#define JS8_FEATURES 0x0E0 /* (RO) Features of job slot 8 */ +#define JS9_FEATURES 0x0E4 /* (RO) Features of job slot 9 */ +#define JS10_FEATURES 0x0E8 /* (RO) Features of job slot 10 */ +#define JS11_FEATURES 0x0EC /* (RO) Features of job slot 11 */ +#define JS12_FEATURES 0x0F0 /* (RO) Features of job slot 12 */ +#define JS13_FEATURES 0x0F4 /* (RO) Features of job slot 13 */ +#define JS14_FEATURES 0x0F8 /* (RO) Features of job slot 14 */ +#define JS15_FEATURES 0x0FC /* (RO) Features of job slot 15 */ + +#define JS_FEATURES_REG(n) GPU_CONTROL_REG(JS0_FEATURES + ((n) << 2)) + +#define SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */ +#define SHADER_PRESENT_HI 0x104 /* (RO) Shader core present bitmap, high word */ + +#define TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */ +#define TILER_PRESENT_HI 0x114 /* (RO) Tiler core present bitmap, high word */ + +#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */ +#define L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */ + +#define STACK_PRESENT_LO 0xE00 /* (RO) Core stack present bitmap, low word */ +#define STACK_PRESENT_HI 0xE04 /* (RO) Core stack present bitmap, high word */ + + +#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */ +#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */ + +#define TILER_READY_LO 0x150 /* (RO) Tiler core ready bitmap, low word */ +#define TILER_READY_HI 0x154 /* (RO) Tiler core ready bitmap, high word */ + +#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */ +#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */ + +#define STACK_READY_LO 0xE10 /* (RO) Core stack ready bitmap, low word */ +#define STACK_READY_HI 0xE14 /* (RO) Core stack ready bitmap, high word */ + + +#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */ +#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */ + +#define TILER_PWRON_LO 0x190 /* (WO) Tiler core power on bitmap, low word */ +#define TILER_PWRON_HI 0x194 /* (WO) Tiler core power on bitmap, high word */ + +#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */ +#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */ + +#define STACK_PWRON_LO 0xE20 /* (RO) Core stack power on bitmap, low word */ +#define STACK_PWRON_HI 0xE24 /* (RO) Core stack power on bitmap, high word */ + + +#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */ +#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */ + +#define TILER_PWROFF_LO 0x1D0 /* (WO) Tiler core power off bitmap, low word */ +#define TILER_PWROFF_HI 0x1D4 /* (WO) Tiler core power off bitmap, high word */ + +#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */ +#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */ + +#define STACK_PWROFF_LO 0xE30 /* (RO) Core stack power off bitmap, low word */ +#define STACK_PWROFF_HI 0xE34 /* (RO) Core stack power off bitmap, high word */ + + +#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */ +#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */ + +#define TILER_PWRTRANS_LO 0x210 /* (RO) Tiler core power transition bitmap, low word */ +#define TILER_PWRTRANS_HI 0x214 /* (RO) Tiler core power transition bitmap, high word */ + +#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */ +#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */ + +#define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */ +#define STACK_PWRTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */ + + +#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */ +#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */ + +#define TILER_PWRACTIVE_LO 0x250 /* (RO) Tiler core active bitmap, low word */ +#define TILER_PWRACTIVE_HI 0x254 /* (RO) Tiler core active bitmap, high word */ + +#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */ +#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */ + +#define COHERENCY_FEATURES 0x300 /* (RO) Coherency features present */ +#define COHERENCY_ENABLE 0x304 /* (RW) Coherency enable */ + +#define JM_CONFIG 0xF00 /* (RW) Job Manager configuration register (Implementation specific register) */ +#define SHADER_CONFIG 0xF04 /* (RW) Shader core configuration settings (Implementation specific register) */ +#define TILER_CONFIG 0xF08 /* (RW) Tiler core configuration settings (Implementation specific register) */ +#define L2_MMU_CONFIG 0xF0C /* (RW) Configuration of the L2 cache and MMU (Implementation specific register) */ + +#define JOB_CONTROL_BASE 0x1000 + +#define JOB_CONTROL_REG(r) (JOB_CONTROL_BASE + (r)) + +#define JOB_IRQ_RAWSTAT 0x000 /* Raw interrupt status register */ +#define JOB_IRQ_CLEAR 0x004 /* Interrupt clear register */ +#define JOB_IRQ_MASK 0x008 /* Interrupt mask register */ +#define JOB_IRQ_STATUS 0x00C /* Interrupt status register */ +#define JOB_IRQ_JS_STATE 0x010 /* status==active and _next == busy snapshot from last JOB_IRQ_CLEAR */ +#define JOB_IRQ_THROTTLE 0x014 /* cycles to delay delivering an interrupt externally. The JOB_IRQ_STATUS is NOT affected by this, just the delivery of the interrupt. */ + +#define JOB_SLOT0 0x800 /* Configuration registers for job slot 0 */ +#define JOB_SLOT1 0x880 /* Configuration registers for job slot 1 */ +#define JOB_SLOT2 0x900 /* Configuration registers for job slot 2 */ +#define JOB_SLOT3 0x980 /* Configuration registers for job slot 3 */ +#define JOB_SLOT4 0xA00 /* Configuration registers for job slot 4 */ +#define JOB_SLOT5 0xA80 /* Configuration registers for job slot 5 */ +#define JOB_SLOT6 0xB00 /* Configuration registers for job slot 6 */ +#define JOB_SLOT7 0xB80 /* Configuration registers for job slot 7 */ +#define JOB_SLOT8 0xC00 /* Configuration registers for job slot 8 */ +#define JOB_SLOT9 0xC80 /* Configuration registers for job slot 9 */ +#define JOB_SLOT10 0xD00 /* Configuration registers for job slot 10 */ +#define JOB_SLOT11 0xD80 /* Configuration registers for job slot 11 */ +#define JOB_SLOT12 0xE00 /* Configuration registers for job slot 12 */ +#define JOB_SLOT13 0xE80 /* Configuration registers for job slot 13 */ +#define JOB_SLOT14 0xF00 /* Configuration registers for job slot 14 */ +#define JOB_SLOT15 0xF80 /* Configuration registers for job slot 15 */ + +#define JOB_SLOT_REG(n, r) (JOB_CONTROL_REG(JOB_SLOT0 + ((n) << 7)) + (r)) + +#define JS_HEAD_LO 0x00 /* (RO) Job queue head pointer for job slot n, low word */ +#define JS_HEAD_HI 0x04 /* (RO) Job queue head pointer for job slot n, high word */ +#define JS_TAIL_LO 0x08 /* (RO) Job queue tail pointer for job slot n, low word */ +#define JS_TAIL_HI 0x0C /* (RO) Job queue tail pointer for job slot n, high word */ +#define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ +#define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ +#define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ +#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job + slot n */ + +#define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ +#define JS_STATUS 0x24 /* (RO) Status register for job slot n */ + +#define JS_HEAD_NEXT_LO 0x40 /* (RW) Next job queue head pointer for job slot n, low word */ +#define JS_HEAD_NEXT_HI 0x44 /* (RW) Next job queue head pointer for job slot n, high word */ + +#define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ +#define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ +#define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ +#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for + job slot n */ + +#define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ + +#define JS_FLUSH_ID_NEXT 0x70 /* (RW) Next job slot n cache flush ID */ + +#define MEMORY_MANAGEMENT_BASE 0x2000 +#define MMU_REG(r) (MEMORY_MANAGEMENT_BASE + (r)) + +#define MMU_IRQ_RAWSTAT 0x000 /* (RW) Raw interrupt status register */ +#define MMU_IRQ_CLEAR 0x004 /* (WO) Interrupt clear register */ +#define MMU_IRQ_MASK 0x008 /* (RW) Interrupt mask register */ +#define MMU_IRQ_STATUS 0x00C /* (RO) Interrupt status register */ + +#define MMU_AS0 0x400 /* Configuration registers for address space 0 */ +#define MMU_AS1 0x440 /* Configuration registers for address space 1 */ +#define MMU_AS2 0x480 /* Configuration registers for address space 2 */ +#define MMU_AS3 0x4C0 /* Configuration registers for address space 3 */ +#define MMU_AS4 0x500 /* Configuration registers for address space 4 */ +#define MMU_AS5 0x540 /* Configuration registers for address space 5 */ +#define MMU_AS6 0x580 /* Configuration registers for address space 6 */ +#define MMU_AS7 0x5C0 /* Configuration registers for address space 7 */ +#define MMU_AS8 0x600 /* Configuration registers for address space 8 */ +#define MMU_AS9 0x640 /* Configuration registers for address space 9 */ +#define MMU_AS10 0x680 /* Configuration registers for address space 10 */ +#define MMU_AS11 0x6C0 /* Configuration registers for address space 11 */ +#define MMU_AS12 0x700 /* Configuration registers for address space 12 */ +#define MMU_AS13 0x740 /* Configuration registers for address space 13 */ +#define MMU_AS14 0x780 /* Configuration registers for address space 14 */ +#define MMU_AS15 0x7C0 /* Configuration registers for address space 15 */ + +#define MMU_AS_REG(n, r) (MMU_REG(MMU_AS0 + ((n) << 6)) + (r)) + +#define AS_TRANSTAB_LO 0x00 /* (RW) Translation Table Base Address for address space n, low word */ +#define AS_TRANSTAB_HI 0x04 /* (RW) Translation Table Base Address for address space n, high word */ +#define AS_MEMATTR_LO 0x08 /* (RW) Memory attributes for address space n, low word. */ +#define AS_MEMATTR_HI 0x0C /* (RW) Memory attributes for address space n, high word. */ +#define AS_LOCKADDR_LO 0x10 /* (RW) Lock region address for address space n, low word */ +#define AS_LOCKADDR_HI 0x14 /* (RW) Lock region address for address space n, high word */ +#define AS_COMMAND 0x18 /* (WO) MMU command register for address space n */ +#define AS_FAULTSTATUS 0x1C /* (RO) MMU fault status register for address space n */ +#define AS_FAULTADDRESS_LO 0x20 /* (RO) Fault Address for address space n, low word */ +#define AS_FAULTADDRESS_HI 0x24 /* (RO) Fault Address for address space n, high word */ +#define AS_STATUS 0x28 /* (RO) Status flags for address space n */ + + +/* (RW) Translation table configuration for address space n, low word */ +#define AS_TRANSCFG_LO 0x30 +/* (RW) Translation table configuration for address space n, high word */ +#define AS_TRANSCFG_HI 0x34 +/* (RO) Secondary fault address for address space n, low word */ +#define AS_FAULTEXTRA_LO 0x38 +/* (RO) Secondary fault address for address space n, high word */ +#define AS_FAULTEXTRA_HI 0x3C + +/* End Register Offsets */ + +/* + * MMU_IRQ_RAWSTAT register values. Values are valid also for + MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers. + */ + +#define MMU_PAGE_FAULT_FLAGS 16 + +/* Macros returning a bitmask to retrieve page fault or bus error flags from + * MMU registers */ +#define MMU_PAGE_FAULT(n) (1UL << (n)) +#define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS)) + +/* + * Begin LPAE MMU TRANSTAB register values + */ +#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK 0xfffff000 +#define AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED (0u << 0) +#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY (1u << 1) +#define AS_TRANSTAB_LPAE_ADRMODE_TABLE (3u << 0) +#define AS_TRANSTAB_LPAE_READ_INNER (1u << 2) +#define AS_TRANSTAB_LPAE_SHARE_OUTER (1u << 4) + +#define AS_TRANSTAB_LPAE_ADRMODE_MASK 0x00000003 + +/* + * Begin AARCH64 MMU TRANSTAB register values + */ +#define MMU_HW_OUTA_BITS 40 +#define AS_TRANSTAB_BASE_MASK ((1ULL << MMU_HW_OUTA_BITS) - (1ULL << 4)) + +/* + * Begin MMU STATUS register values + */ +#define AS_STATUS_AS_ACTIVE 0x01 + +#define AS_FAULTSTATUS_EXCEPTION_CODE_MASK (0x7<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT (0x0<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT (0x1<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT (0x2<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG (0x3<<3) + +#define AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT (0x4<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT (0x5<<3) + +#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3<<8) +#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0<<8) +#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1<<8) +#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2<<8) +#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3<<8) + +/* + * Begin MMU TRANSCFG register values + */ + +#define AS_TRANSCFG_ADRMODE_LEGACY 0 +#define AS_TRANSCFG_ADRMODE_UNMAPPED 1 +#define AS_TRANSCFG_ADRMODE_IDENTITY 2 +#define AS_TRANSCFG_ADRMODE_AARCH64_4K 6 +#define AS_TRANSCFG_ADRMODE_AARCH64_64K 8 + +#define AS_TRANSCFG_ADRMODE_MASK 0xF + + +/* + * Begin TRANSCFG register values + */ +#define AS_TRANSCFG_PTW_MEMATTR_MASK (3 << 24) +#define AS_TRANSCFG_PTW_MEMATTR_NON_CACHEABLE (1 << 24) +#define AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK (2 << 24) + +#define AS_TRANSCFG_PTW_SH_MASK ((3 << 28)) +#define AS_TRANSCFG_PTW_SH_OS (2 << 28) +#define AS_TRANSCFG_PTW_SH_IS (3 << 28) + +/* + * Begin Command Values + */ + +/* JS_COMMAND register commands */ +#define JS_COMMAND_NOP 0x00 /* NOP Operation. Writing this value is ignored */ +#define JS_COMMAND_START 0x01 /* Start processing a job chain. Writing this value is ignored */ +#define JS_COMMAND_SOFT_STOP 0x02 /* Gently stop processing a job chain */ +#define JS_COMMAND_HARD_STOP 0x03 /* Rudely stop processing a job chain */ +#define JS_COMMAND_SOFT_STOP_0 0x04 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */ +#define JS_COMMAND_HARD_STOP_0 0x05 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */ +#define JS_COMMAND_SOFT_STOP_1 0x06 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */ +#define JS_COMMAND_HARD_STOP_1 0x07 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */ + +#define JS_COMMAND_MASK 0x07 /* Mask of bits currently in use by the HW */ + +/* AS_COMMAND register commands */ +#define AS_COMMAND_NOP 0x00 /* NOP Operation */ +#define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */ +#define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */ +#define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */ +#define AS_COMMAND_FLUSH 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs + (deprecated - only for use with T60x) */ +#define AS_COMMAND_FLUSH_PT 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs */ +#define AS_COMMAND_FLUSH_MEM 0x05 /* Wait for memory accesses to complete, flush all the L1s cache then + flush all L2 caches then issue a flush region command to all MMUs */ + +/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */ +#define JS_CONFIG_START_FLUSH_NO_ACTION (0u << 0) +#define JS_CONFIG_START_FLUSH_CLEAN (1u << 8) +#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE (3u << 8) +#define JS_CONFIG_START_MMU (1u << 10) +#define JS_CONFIG_JOB_CHAIN_FLAG (1u << 11) +#define JS_CONFIG_END_FLUSH_NO_ACTION JS_CONFIG_START_FLUSH_NO_ACTION +#define JS_CONFIG_END_FLUSH_CLEAN (1u << 12) +#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE (3u << 12) +#define JS_CONFIG_ENABLE_FLUSH_REDUCTION (1u << 14) +#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK (1u << 15) +#define JS_CONFIG_THREAD_PRI(n) ((n) << 16) + +/* JS_XAFFINITY register values */ +#define JS_XAFFINITY_XAFFINITY_ENABLE (1u << 0) +#define JS_XAFFINITY_TILER_ENABLE (1u << 8) +#define JS_XAFFINITY_CACHE_ENABLE (1u << 16) + +/* JS_STATUS register values */ + +/* NOTE: Please keep this values in sync with enum base_jd_event_code in mali_base_kernel.h. + * The values are separated to avoid dependency of userspace and kernel code. + */ + +/* Group of values representing the job status insead a particular fault */ +#define JS_STATUS_NO_EXCEPTION_BASE 0x00 +#define JS_STATUS_INTERRUPTED (JS_STATUS_NO_EXCEPTION_BASE + 0x02) /* 0x02 means INTERRUPTED */ +#define JS_STATUS_STOPPED (JS_STATUS_NO_EXCEPTION_BASE + 0x03) /* 0x03 means STOPPED */ +#define JS_STATUS_TERMINATED (JS_STATUS_NO_EXCEPTION_BASE + 0x04) /* 0x04 means TERMINATED */ + +/* General fault values */ +#define JS_STATUS_FAULT_BASE 0x40 +#define JS_STATUS_CONFIG_FAULT (JS_STATUS_FAULT_BASE) /* 0x40 means CONFIG FAULT */ +#define JS_STATUS_POWER_FAULT (JS_STATUS_FAULT_BASE + 0x01) /* 0x41 means POWER FAULT */ +#define JS_STATUS_READ_FAULT (JS_STATUS_FAULT_BASE + 0x02) /* 0x42 means READ FAULT */ +#define JS_STATUS_WRITE_FAULT (JS_STATUS_FAULT_BASE + 0x03) /* 0x43 means WRITE FAULT */ +#define JS_STATUS_AFFINITY_FAULT (JS_STATUS_FAULT_BASE + 0x04) /* 0x44 means AFFINITY FAULT */ +#define JS_STATUS_BUS_FAULT (JS_STATUS_FAULT_BASE + 0x08) /* 0x48 means BUS FAULT */ + +/* Instruction or data faults */ +#define JS_STATUS_INSTRUCTION_FAULT_BASE 0x50 +#define JS_STATUS_INSTR_INVALID_PC (JS_STATUS_INSTRUCTION_FAULT_BASE) /* 0x50 means INSTR INVALID PC */ +#define JS_STATUS_INSTR_INVALID_ENC (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x01) /* 0x51 means INSTR INVALID ENC */ +#define JS_STATUS_INSTR_TYPE_MISMATCH (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x02) /* 0x52 means INSTR TYPE MISMATCH */ +#define JS_STATUS_INSTR_OPERAND_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x03) /* 0x53 means INSTR OPERAND FAULT */ +#define JS_STATUS_INSTR_TLS_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x04) /* 0x54 means INSTR TLS FAULT */ +#define JS_STATUS_INSTR_BARRIER_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x05) /* 0x55 means INSTR BARRIER FAULT */ +#define JS_STATUS_INSTR_ALIGN_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x06) /* 0x56 means INSTR ALIGN FAULT */ +/* NOTE: No fault with 0x57 code defined in spec. */ +#define JS_STATUS_DATA_INVALID_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x08) /* 0x58 means DATA INVALID FAULT */ +#define JS_STATUS_TILE_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x09) /* 0x59 means TILE RANGE FAULT */ +#define JS_STATUS_ADDRESS_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x0A) /* 0x5A means ADDRESS RANGE FAULT */ + +/* Other faults */ +#define JS_STATUS_MEMORY_FAULT_BASE 0x60 +#define JS_STATUS_OUT_OF_MEMORY (JS_STATUS_MEMORY_FAULT_BASE) /* 0x60 means OUT OF MEMORY */ +#define JS_STATUS_UNKNOWN 0x7F /* 0x7F means UNKNOWN */ + +/* GPU_COMMAND values */ +#define GPU_COMMAND_NOP 0x00 /* No operation, nothing happens */ +#define GPU_COMMAND_SOFT_RESET 0x01 /* Stop all external bus interfaces, and then reset the entire GPU. */ +#define GPU_COMMAND_HARD_RESET 0x02 /* Immediately reset the entire GPU. */ +#define GPU_COMMAND_PRFCNT_CLEAR 0x03 /* Clear all performance counters, setting them all to zero. */ +#define GPU_COMMAND_PRFCNT_SAMPLE 0x04 /* Sample all performance counters, writing them out to memory */ +#define GPU_COMMAND_CYCLE_COUNT_START 0x05 /* Starts the cycle counter, and system timestamp propagation */ +#define GPU_COMMAND_CYCLE_COUNT_STOP 0x06 /* Stops the cycle counter, and system timestamp propagation */ +#define GPU_COMMAND_CLEAN_CACHES 0x07 /* Clean all caches */ +#define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */ +#define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */ + +/* End Command Values */ + +/* GPU_STATUS values */ +#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ +#define GPU_STATUS_PROTECTED_MODE_ACTIVE (1 << 7) /* Set if protected mode is active */ + +/* PRFCNT_CONFIG register values */ +#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ +#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ +#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ + +#define PRFCNT_CONFIG_MODE_OFF 0 /* The performance counters are disabled. */ +#define PRFCNT_CONFIG_MODE_MANUAL 1 /* The performance counters are enabled, but are only written out when a PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. */ +#define PRFCNT_CONFIG_MODE_TILE 2 /* The performance counters are enabled, and are written out each time a tile finishes rendering. */ + +/* AS_MEMATTR values: */ +/* Use GPU implementation-defined caching policy. */ +#define AS_MEMATTR_IMPL_DEF_CACHE_POLICY 0x88ull +/* The attribute set to force all resources to be cached. */ +#define AS_MEMATTR_FORCE_TO_CACHE_ALL 0x8Full +/* Inner write-alloc cache setup, no outer caching */ +#define AS_MEMATTR_WRITE_ALLOC 0x8Dull + +/* Set to implementation defined, outer caching */ +#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull +/* Set to write back memory, outer caching */ +#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull + +/* Use GPU implementation-defined caching policy. */ +#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull +/* The attribute set to force all resources to be cached. */ +#define AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL 0x4Full +/* Inner write-alloc cache setup, no outer caching */ +#define AS_MEMATTR_LPAE_WRITE_ALLOC 0x4Dull +/* Set to implementation defined, outer caching */ +#define AS_MEMATTR_LPAE_OUTER_IMPL_DEF 0x88ull +/* Set to write back memory, outer caching */ +#define AS_MEMATTR_LPAE_OUTER_WA 0x8Dull + +/* Symbols for default MEMATTR to use + * Default is - HW implementation defined caching */ +#define AS_MEMATTR_INDEX_DEFAULT 0 +#define AS_MEMATTR_INDEX_DEFAULT_ACE 3 + +/* HW implementation defined caching */ +#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0 +/* Force cache on */ +#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1 +/* Write-alloc */ +#define AS_MEMATTR_INDEX_WRITE_ALLOC 2 +/* Outer coherent, inner implementation defined policy */ +#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3 +/* Outer coherent, write alloc inner */ +#define AS_MEMATTR_INDEX_OUTER_WA 4 + +/* JS_FEATURES register */ + +#define JS_FEATURE_NULL_JOB (1u << 1) +#define JS_FEATURE_SET_VALUE_JOB (1u << 2) +#define JS_FEATURE_CACHE_FLUSH_JOB (1u << 3) +#define JS_FEATURE_COMPUTE_JOB (1u << 4) +#define JS_FEATURE_VERTEX_JOB (1u << 5) +#define JS_FEATURE_GEOMETRY_JOB (1u << 6) +#define JS_FEATURE_TILER_JOB (1u << 7) +#define JS_FEATURE_FUSED_JOB (1u << 8) +#define JS_FEATURE_FRAGMENT_JOB (1u << 9) + +/* End JS_FEATURES register */ + +/* L2_MMU_CONFIG register */ +#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT (23) +#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT (24) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) + +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT (26) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) +/* End L2_MMU_CONFIG register */ + +/* THREAD_* registers */ + +/* THREAD_FEATURES IMPLEMENTATION_TECHNOLOGY values */ +#define IMPLEMENTATION_UNSPECIFIED 0 +#define IMPLEMENTATION_SILICON 1 +#define IMPLEMENTATION_FPGA 2 +#define IMPLEMENTATION_MODEL 3 + +/* Default values when registers are not supported by the implemented hardware */ +#define THREAD_MT_DEFAULT 256 +#define THREAD_MWS_DEFAULT 256 +#define THREAD_MBS_DEFAULT 256 +#define THREAD_MR_DEFAULT 1024 +#define THREAD_MTQ_DEFAULT 4 +#define THREAD_MTGS_DEFAULT 10 + +/* End THREAD_* registers */ + +/* SHADER_CONFIG register */ + +#define SC_ALT_COUNTERS (1ul << 3) +#define SC_OVERRIDE_FWD_PIXEL_KILL (1ul << 4) +#define SC_SDC_DISABLE_OQ_DISCARD (1ul << 6) +#define SC_LS_ALLOW_ATTR_TYPES (1ul << 16) +#define SC_LS_PAUSEBUFFER_DISABLE (1ul << 16) +#define SC_TLS_HASH_ENABLE (1ul << 17) +#define SC_LS_ATTR_CHECK_DISABLE (1ul << 18) +#define SC_ENABLE_TEXGRD_FLAGS (1ul << 25) +/* End SHADER_CONFIG register */ + +/* TILER_CONFIG register */ + +#define TC_CLOCK_GATE_OVERRIDE (1ul << 0) + +/* End TILER_CONFIG register */ + +/* JM_CONFIG register */ + +#define JM_TIMESTAMP_OVERRIDE (1ul << 0) +#define JM_CLOCK_GATE_OVERRIDE (1ul << 1) +#define JM_JOB_THROTTLE_ENABLE (1ul << 2) +#define JM_JOB_THROTTLE_LIMIT_SHIFT (3) +#define JM_MAX_JOB_THROTTLE_LIMIT (0x3F) +#define JM_FORCE_COHERENCY_FEATURES_SHIFT (2) +#define JM_IDVS_GROUP_SIZE_SHIFT (16) +#define JM_MAX_IDVS_GROUP_SIZE (0x3F) +/* End JM_CONFIG register */ + + +#endif /* _MIDGARD_REGMAP_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_timeline.h b/drivers/gpu/arm/bifrost_for_linux/mali_timeline.h new file mode 100755 index 000000000000..bd5f6614b6bb --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_timeline.h @@ -0,0 +1,396 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mali_timeline + +#if !defined(_MALI_TIMELINE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _MALI_TIMELINE_H + +#include + +TRACE_EVENT(mali_timeline_atoms_in_flight, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int tgid, + int count), + + TP_ARGS(ts_sec, + ts_nsec, + tgid, + count), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, tgid) + __field(int, count) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->tgid = tgid; + __entry->count = count; + ), + + TP_printk("%i,%i.%.9i,%i,%i", CTX_SET_NR_ATOMS_IN_FLIGHT, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->tgid, + __entry->count) +); + + +TRACE_EVENT(mali_timeline_atom, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int tgid, + int atom_id), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + tgid, + atom_id), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, tgid) + __field(int, atom_id) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->tgid = tgid; + __entry->atom_id = atom_id; + ), + + TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->tgid, + __entry->atom_id, + __entry->atom_id) +); + +TRACE_EVENT(mali_timeline_gpu_slot_active, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int tgid, + int js, + int count), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + tgid, + js, + count), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, tgid) + __field(int, js) + __field(int, count) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->tgid = tgid; + __entry->js = js; + __entry->count = count; + ), + + TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->tgid, + __entry->js, + __entry->count) +); + +TRACE_EVENT(mali_timeline_gpu_slot_action, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int tgid, + int js, + int count), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + tgid, + js, + count), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, tgid) + __field(int, js) + __field(int, count) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->tgid = tgid; + __entry->js = js; + __entry->count = count; + ), + + TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->tgid, + __entry->js, + __entry->count) +); + +TRACE_EVENT(mali_timeline_gpu_power_active, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int active), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + active), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, active) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->active = active; + ), + + TP_printk("%i,%i.%.9i,0,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->active) + +); + +TRACE_EVENT(mali_timeline_l2_power_active, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int state), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + state), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, state) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->state = state; + ), + + TP_printk("%i,%i.%.9i,0,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->state) + +); +TRACE_EVENT(mali_timeline_pm_event, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int pm_event_type, + unsigned int pm_event_id), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + pm_event_type, + pm_event_id), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, pm_event_type) + __field(unsigned int, pm_event_id) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->pm_event_type = pm_event_type; + __entry->pm_event_id = pm_event_id; + ), + + TP_printk("%i,%i.%.9i,0,%i,%u", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->pm_event_type, __entry->pm_event_id) + +); + +TRACE_EVENT(mali_timeline_slot_atom, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int tgid, + int js, + int atom_id), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + tgid, + js, + atom_id), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, tgid) + __field(int, js) + __field(int, atom_id) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->tgid = tgid; + __entry->js = js; + __entry->atom_id = atom_id; + ), + + TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->tgid, + __entry->js, + __entry->atom_id) +); + +TRACE_EVENT(mali_timeline_pm_checktrans, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int trans_code, + int trans_id), + + TP_ARGS(ts_sec, + ts_nsec, + trans_code, + trans_id), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, trans_code) + __field(int, trans_id) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->trans_code = trans_code; + __entry->trans_id = trans_id; + ), + + TP_printk("%i,%i.%.9i,0,%i", __entry->trans_code, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->trans_id) + +); + +TRACE_EVENT(mali_timeline_context_active, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int count), + + TP_ARGS(ts_sec, + ts_nsec, + count), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, count) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->count = count; + ), + + TP_printk("%i,%i.%.9i,0,%i", SW_SET_CONTEXT_ACTIVE, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->count) +); + +#endif /* _MALI_TIMELINE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +/* This part must be outside protection */ +#include + diff --git a/drivers/gpu/arm/bifrost_for_linux/mali_uk.h b/drivers/gpu/arm/bifrost_for_linux/mali_uk.h new file mode 100755 index 000000000000..841d03fb5873 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/mali_uk.h @@ -0,0 +1,141 @@ +/* + * + * (C) COPYRIGHT 2010, 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_uk.h + * Types and definitions that are common across OSs for both the user + * and kernel side of the User-Kernel interface. + */ + +#ifndef _UK_H_ +#define _UK_H_ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @defgroup uk_api User-Kernel Interface API + * + * The User-Kernel Interface abstracts the communication mechanism between the user and kernel-side code of device + * drivers developed as part of the Midgard DDK. Currently that includes the Base driver and the UMP driver. + * + * It exposes an OS independent API to user-side code (UKU) which routes functions calls to an OS-independent + * kernel-side API (UKK) via an OS-specific communication mechanism. + * + * This API is internal to the Midgard DDK and is not exposed to any applications. + * + * @{ + */ + +/** + * These are identifiers for kernel-side drivers implementing a UK interface, aka UKK clients. The + * UK module maps this to an OS specific device name, e.g. "gpu_base" -> "GPU0:". Specify this + * identifier to select a UKK client to the uku_open() function. + * + * When a new UKK client driver is created a new identifier needs to be added to the uk_client_id + * enumeration and the uku_open() implemenation for the various OS ports need to be updated to + * provide a mapping of the identifier to the OS specific device name. + * + */ +enum uk_client_id { + /** + * Value used to identify the Base driver UK client. + */ + UK_CLIENT_MALI_T600_BASE, + + /** The number of uk clients supported. This must be the last member of the enum */ + UK_CLIENT_COUNT +}; + +/** + * Each function callable through the UK interface has a unique number. + * Functions provided by UK clients start from number UK_FUNC_ID. + * Numbers below UK_FUNC_ID are used for internal UK functions. + */ +enum uk_func { + UKP_FUNC_ID_CHECK_VERSION, /**< UKK Core internal function */ + /** + * Each UK client numbers the functions they provide starting from + * number UK_FUNC_ID. This number is then eventually assigned to the + * id field of the union uk_header structure when preparing to make a + * UK call. See your UK client for a list of their function numbers. + */ + UK_FUNC_ID = 512 +}; + +/** + * Arguments for a UK call are stored in a structure. This structure consists + * of a fixed size header and a payload. The header carries a 32-bit number + * identifying the UK function to be called (see uk_func). When the UKK client + * receives this header and executed the requested UK function, it will use + * the same header to store the result of the function in the form of a + * int return code. The size of this structure is such that the + * first member of the payload following the header can be accessed efficiently + * on a 32 and 64-bit kernel and the structure has the same size regardless + * of a 32 or 64-bit kernel. The uk_kernel_size_type type should be defined + * accordingly in the OS specific mali_uk_os.h header file. + */ +union uk_header { + /** + * 32-bit number identifying the UK function to be called. + * Also see uk_func. + */ + u32 id; + /** + * The int return code returned by the called UK function. + * See the specification of the particular UK function you are + * calling for the meaning of the error codes returned. All + * UK functions return 0 on success. + */ + u32 ret; + /* + * Used to ensure 64-bit alignment of this union. Do not remove. + * This field is used for padding and does not need to be initialized. + */ + u64 sizer; +}; + +/** + * This structure carries a 16-bit major and minor number and is sent along with an internal UK call + * used during uku_open to identify the versions of the UK module in use by the user-side and kernel-side. + */ +struct uku_version_check_args { + union uk_header header; + /**< UK call header */ + u16 major; + /**< This field carries the user-side major version on input and the kernel-side major version on output */ + u16 minor; + /**< This field carries the user-side minor version on input and the kernel-side minor version on output. */ + u8 padding[4]; +}; + +/** @} end group uk_api */ + +/** @} *//* end group base_api */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif /* _UK_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/Kconfig b/drivers/gpu/arm/bifrost_for_linux/platform/Kconfig new file mode 100755 index 000000000000..38835d3d1531 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/Kconfig @@ -0,0 +1,24 @@ +# +# (C) COPYRIGHT 2012-2013, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + + +# Add your platform specific Kconfig file here +# +# "drivers/gpu/arm/midgard/platform/xxx/Kconfig" +# +# Where xxx is the platform name is the name set in MALI_PLATFORM_NAME +# + diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/Kbuild b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/Kbuild new file mode 100755 index 000000000000..d40d7982ff04 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/Kbuild @@ -0,0 +1,18 @@ +# +# (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +mali_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_devicetree.o \ + $(MALI_PLATFORM_DIR)/mali_kbase_runtime_pm.o diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_devicetree.c b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_devicetree.c new file mode 100755 index 000000000000..29ccc29e4125 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_devicetree.c @@ -0,0 +1,40 @@ +/* + * + * (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} + +static struct kbase_platform_config dummy_platform_config; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &dummy_platform_config; +} + +int kbase_platform_register(void) +{ + return 0; +} + +void kbase_platform_unregister(void) +{ +} diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_platform.h new file mode 100755 index 000000000000..2ceca34945b9 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_config_platform.h @@ -0,0 +1,80 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * Maximum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX (5000) +/** + * Minimum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN (5000) + +/** + * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (NULL) + +/** + * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; + +/** + * Autosuspend delay + * + * The delay time (in milliseconds) to be used for autosuspend + */ +#define AUTO_SUSPEND_DELAY (100) diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_runtime_pm.c b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_runtime_pm.c new file mode 100755 index 000000000000..9fe37c8d835e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/devicetree/mali_kbase_runtime_pm.c @@ -0,0 +1,121 @@ +/* + * + * (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include "mali_kbase_config_platform.h" + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + int ret = 1; /* Assume GPU has been powered off */ + int error; + + dev_dbg(kbdev->dev, "pm_callback_power_on %p\n", + (void *)kbdev->dev->pm_domain); + + error = pm_runtime_get_sync(kbdev->dev); + if (error == 1) { + /* + * Let core know that the chip has not been + * powered off, so we can save on re-initialization. + */ + ret = 0; + } + + dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); + + return ret; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "pm_callback_power_off\n"); + + pm_runtime_mark_last_busy(kbdev->dev); + pm_runtime_put_autosuspend(kbdev->dev); +} + +int kbase_device_runtime_init(struct kbase_device *kbdev) +{ + int ret = 0; + + dev_dbg(kbdev->dev, "kbase_device_runtime_init\n"); + + pm_runtime_set_autosuspend_delay(kbdev->dev, AUTO_SUSPEND_DELAY); + pm_runtime_use_autosuspend(kbdev->dev); + + pm_runtime_set_active(kbdev->dev); + pm_runtime_enable(kbdev->dev); + + if (!pm_runtime_enabled(kbdev->dev)) { + dev_warn(kbdev->dev, "pm_runtime not enabled"); + ret = -ENOSYS; + } + + return ret; +} + +void kbase_device_runtime_disable(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "kbase_device_runtime_disable\n"); + pm_runtime_disable(kbdev->dev); +} + +static int pm_callback_runtime_on(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "pm_callback_runtime_on\n"); + + return 0; +} + +static void pm_callback_runtime_off(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "pm_callback_runtime_off\n"); +} + +static void pm_callback_resume(struct kbase_device *kbdev) +{ + int ret = pm_callback_runtime_on(kbdev); + + WARN_ON(ret); +} + +static void pm_callback_suspend(struct kbase_device *kbdev) +{ + pm_callback_runtime_off(kbdev); +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = pm_callback_suspend, + .power_resume_callback = pm_callback_resume, +#ifdef KBASE_PM_RUNTIME + .power_runtime_init_callback = kbase_device_runtime_init, + .power_runtime_term_callback = kbase_device_runtime_disable, + .power_runtime_on_callback = pm_callback_runtime_on, + .power_runtime_off_callback = pm_callback_runtime_off, +#else /* KBASE_PM_RUNTIME */ + .power_runtime_init_callback = NULL, + .power_runtime_term_callback = NULL, + .power_runtime_on_callback = NULL, + .power_runtime_off_callback = NULL, +#endif /* KBASE_PM_RUNTIME */ +}; + + diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/mali_kbase_platform_common.h b/drivers/gpu/arm/bifrost_for_linux/platform/mali_kbase_platform_common.h new file mode 100755 index 000000000000..7cb3be7f78ce --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/mali_kbase_platform_common.h @@ -0,0 +1,26 @@ +/* + * + * (C) COPYRIGHT 2010-2013 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @brief Entry point to transfer control to a platform for early initialization + * + * This function is called early on in the initialization during execution of + * @ref kbase_driver_init. + * + * @return Zero to indicate success non-zero for failure. + */ +int kbase_platform_early_init(void); diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/rk/Kbuild b/drivers/gpu/arm/bifrost_for_linux/platform/rk/Kbuild new file mode 100755 index 000000000000..7cc6c59d969f --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/rk/Kbuild @@ -0,0 +1,17 @@ +# +# (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + +bifrost_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_rk.o \ + diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/rk/custom_log.h b/drivers/gpu/arm/bifrost_for_linux/platform/rk/custom_log.h new file mode 100755 index 000000000000..5de70ee13d25 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/rk/custom_log.h @@ -0,0 +1,192 @@ +/* + * (C) COPYRIGHT RockChip Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +#ifndef __CUSTOM_LOG_H__ +#define __CUSTOM_LOG_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/* ----------------------------------------------------------------------------- + * Include Files + * ----------------------------------------------------------------------------- + */ +#include +#include + +/* ----------------------------------------------------------------------------- + * Macros Definition + * ----------------------------------------------------------------------------- + */ + +/** 若下列 macro 有被定义, æ‰ ä½¿èƒ½ log 输出. */ +/* #define ENABLE_DEBUG_LOG */ + +/*----------------------------------------------------------------------------*/ + +#ifdef ENABLE_VERBOSE_LOG +/** Verbose log. */ +#define V(fmt, args...) \ + pr_debug("V : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) +#else +#define V(...) ((void)0) +#endif + +#ifdef ENABLE_DEBUG_LOG +/** Debug log. */ +#define D(fmt, args...) \ + pr_info("D : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) +#else +#define D(...) ((void)0) +#endif + +#define I(fmt, args...) \ + pr_info("I : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +#define W(fmt, args...) \ + pr_warn("W : [File] : %s; [Line] : %d; [Func] : %s(); " \ + fmt "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +#define E(fmt, args...) \ + pr_err("E : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +/*-------------------------------------------------------*/ + +/** 使用 D(), 以åè¿›åˆ¶çš„å½¢å¼æ‰“å°å˜é‡ 'var' çš„ value. */ +#define D_DEC(var) D(#var " = %d.", var) + +#define E_DEC(var) E(#var " = %d.", var) + +/** 使用 D(), 以åå…­è¿›åˆ¶çš„å½¢å¼æ‰“å°å˜é‡ 'var' çš„ value. */ +#define D_HEX(var) D(#var " = 0x%x.", var) + +#define E_HEX(var) E(#var " = 0x%x.", var) + +/** + * 使用 D(), 以å六进制的形å¼, + * æ‰“å°æŒ‡é’ˆç±»åž‹å˜é‡ 'ptr' çš„ value. + */ +#define D_PTR(ptr) D(#ptr " = %p.", ptr) + +#define E_PTR(ptr) E(#ptr " = %p.", ptr) + +/** 使用 D(), æ‰“å° char 字串. */ +#define D_STR(p_str) \ +do { \ + if (!p_str) { \ + D(#p_str " = NULL."); \ + else \ + D(#p_str " = '%s'.", p_str); \ +} while (0) + +#define E_STR(p_str) \ +do { \ + if (!p_str) \ + E(#p_str " = NULL."); \ + else \ + E(#p_str " = '%s'.", p_str); \ +} while (0) + +#ifdef ENABLE_DEBUG_LOG +/** + * log 从 'p_start' 地å€å¼€å§‹çš„ 'len' 个字节的数æ®. + */ +#define D_MEM(p_start, len) \ +do { \ + int i = 0; \ + char *p = (char *)(p_start); \ + D("dump memory from addr of '" #p_start "', from %p, length %d' : ", \ + (p_start), \ + (len)); \ + pr_debug("\t\t"); \ + for (i = 0; i < (len); i++) \ + pr_debug("0x%02x, ", p[i]); \ + pr_debug("\n"); \ +} while (0) +#else +#define D_MEM(...) ((void)0) +#endif + +/*-------------------------------------------------------*/ + +/** + * 在特定æ¡ä»¶ä¸‹, 判定 error å‘生, + * å°†å˜é‡ 'ret_var' 设置 'err_code', + * log 输出对应的 Error Caution, + * ç„¶åŽè·³è½¬ 'label' 指定的代ç å¤„执行. + * @param msg + * 纯字串形å¼çš„æç¤ºä¿¡æ¯. + * @param ret_var + * æ ‡è¯†å‡½æ•°æ‰§è¡ŒçŠ¶æ€æˆ–者结果的å˜é‡, + * 将被设置具体的 Error Code. + * 通常是 'ret' or 'result'. + * @param err_code + * 表å¾ç‰¹å®š error 的常数标识, + * 通常是 å®çš„å½¢æ€. + * @param label + * 程åºå°†è¦è·³è½¬åˆ°çš„错误处ç†ä»£ç çš„æ ‡å·, + * 通常就是 'EXIT'. + * @param args... + * 对应 'msg_fmt' 实å‚中, + * '%s', '%d', ... 等转æ¢è¯´æ˜Žç¬¦çš„具体å¯å˜é•¿å®žå‚. + */ +#define SET_ERROR_AND_JUMP(msg_fmt, ret_var, err_code, label, args...) \ +do { \ + E("To set '" #ret_var "' to %d('" #err_code "'), because : " msg_fmt, \ + (err_code), \ + ## args); \ + (ret_var) = (err_code); \ + goto label; \ +} while (0) + +/* ----------------------------------------------------------------------------- + * Types and Structures Definition + * ----------------------------------------------------------------------------- + */ + +/* ----------------------------------------------------------------------------- + * Global Functions' Prototype + * ----------------------------------------------------------------------------- + */ + +/* ----------------------------------------------------------------------------- + * Inline Functions Implementation + * ----------------------------------------------------------------------------- + */ + +#ifdef __cplusplus +} +#endif + +#endif /* __CUSTOM_LOG_H__ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_platform.h new file mode 100755 index 000000000000..07c5b6f8a760 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_platform.h @@ -0,0 +1,88 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +/** + * @file mali_kbase_config_platform.h + * 声明 platform_config_of_rk (platform_rk çš„ platform_config). + */ + +/** + * Maximum frequency GPU will be clocked at. + * Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX (5000) + +/** + * Minimum frequency GPU will be clocked at. + * Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN (5000) + +/** + * CPU_SPEED_FUNC + * - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz + * - see kbase_cpu_clk_speed_func for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (NULL) + +/** + * GPU_SPEED_FUNC + * - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz + * - see kbase_gpu_clk_speed_func for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: + * pointer to @ref kbase_pm_callback_conf + * Default value: + * See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) +extern struct kbase_pm_callback_conf pm_callbacks; + +/** + * Platform specific configuration functions + * + * Attached value: + * pointer to @ref kbase_platform_funcs_conf + * Default value: + * See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (&platform_funcs) +extern struct kbase_platform_funcs_conf platform_funcs; + +/** + * Secure mode switch + * + * Attached value: pointer to @ref kbase_secure_ops + */ +#define SECURE_CALLBACKS (NULL) + diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_rk.c new file mode 100755 index 000000000000..926c2dd3f8c9 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_config_rk.c @@ -0,0 +1,459 @@ +/* + * (C) COPYRIGHT RockChip Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +/* #define ENABLE_DEBUG_LOG */ +#include "custom_log.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mali_kbase_rk.h" + +#define MAX_PROP_NAME_LEN 3 +#define LEAKAGE_TABLE_END ~1 +#define LEAKAGE_INVALID 0xff + +struct pvtm_config { + unsigned int freq; + unsigned int volt; + unsigned int ch[2]; + unsigned int sample_time; + unsigned int num; + unsigned int err; + unsigned int ref_temp; + int temp_prop[2]; + const char *tz_name; + struct thermal_zone_device *tz; +}; + +struct volt_sel_table { + int min; + int max; + int sel; +}; + +/** + * @file mali_kbase_config_rk.c + * 对 platform_config_of_rk 的具体实现. + * + * mali_device_driver 包å«ä¸¤éƒ¨åˆ† : + * .DP : platform_dependent_part_in_mdd : + * ä¾èµ– platform 部分, + * æºç åœ¨ /platform// + * 在 mali_device_driver 内部, + * 记为 platform_dependent_part, + * 也被记为 platform_specific_code. + * .DP : common_parts_in_mdd : + * arm 实现的通用的部分, + * æºç åœ¨ / 下. + * 在 mali_device_driver 内部, 记为 common_parts. + */ + +/*---------------------------------------------------------------------------*/ + +#ifdef CONFIG_REGULATOR +static int rk_pm_enable_regulator(struct kbase_device *kbdev); +static void rk_pm_disable_regulator(struct kbase_device *kbdev); +#else +static inline int rk_pm_enable_regulator(struct kbase_device *kbdev) +{ + return 0; +} + +static inline void rk_pm_disable_regulator(struct kbase_device *kbdev) +{ +} +#endif + +static int rk_pm_enable_clk(struct kbase_device *kbdev); + +static void rk_pm_disable_clk(struct kbase_device *kbdev); + +static int kbase_platform_rk_create_sysfs_files(struct device *dev); + +static void kbase_platform_rk_remove_sysfs_files(struct device *dev); + +/*---------------------------------------------------------------------------*/ + +static void rk_pm_power_off_delay_work(struct work_struct *work) +{ + struct rk_context *platform = + container_of(to_delayed_work(work), struct rk_context, work); + struct kbase_device *kbdev = platform->kbdev; + + if (!platform->is_powered) { + D("mali_dev is already powered off."); + return; + } + + if (pm_runtime_enabled(kbdev->dev)) { + D("to put_sync_suspend mali_dev."); + pm_runtime_put_sync_suspend(kbdev->dev); + } + + rk_pm_disable_regulator(kbdev); + + platform->is_powered = false; + KBASE_TIMELINE_GPU_POWER(kbdev, 0); + wake_unlock(&platform->wake_lock); +} + +static int kbase_platform_rk_init(struct kbase_device *kbdev) +{ + int ret = 0; + struct rk_context *platform; + + platform = kzalloc(sizeof(*platform), GFP_KERNEL); + if (!platform) { + E("err."); + return -ENOMEM; + } + + platform->is_powered = false; + platform->kbdev = kbdev; + + platform->delay_ms = 200; + if (of_property_read_u32(kbdev->dev->of_node, "power-off-delay-ms", + &platform->delay_ms)) + W("power-off-delay-ms not available."); + + platform->power_off_wq = create_freezable_workqueue("gpu_power_off_wq"); + if (!platform->power_off_wq) { + E("couldn't create workqueue"); + ret = -ENOMEM; + goto err_wq; + } + INIT_DEFERRABLE_WORK(&platform->work, rk_pm_power_off_delay_work); + + wake_lock_init(&platform->wake_lock, WAKE_LOCK_SUSPEND, "gpu"); + + platform->utilisation_period = DEFAULT_UTILISATION_PERIOD_IN_MS; + + ret = kbase_platform_rk_create_sysfs_files(kbdev->dev); + if (ret) { + E("fail to create sysfs_files. ret = %d.", ret); + goto err_sysfs_files; + } + + kbdev->platform_context = (void *)platform; + pm_runtime_enable(kbdev->dev); + + return 0; + +err_sysfs_files: + wake_lock_destroy(&platform->wake_lock); + destroy_workqueue(platform->power_off_wq); +err_wq: + return ret; +} + +static void kbase_platform_rk_term(struct kbase_device *kbdev) +{ + struct rk_context *platform = + (struct rk_context *)kbdev->platform_context; + + pm_runtime_disable(kbdev->dev); + kbdev->platform_context = NULL; + + if (platform) { + cancel_delayed_work_sync(&platform->work); + wake_lock_destroy(&platform->wake_lock); + destroy_workqueue(platform->power_off_wq); + platform->is_powered = false; + platform->kbdev = NULL; + kfree(platform); + } + kbase_platform_rk_remove_sysfs_files(kbdev->dev); +} + +struct kbase_platform_funcs_conf platform_funcs = { + .platform_init_func = &kbase_platform_rk_init, + .platform_term_func = &kbase_platform_rk_term, +}; + +/*---------------------------------------------------------------------------*/ + +static int rk_pm_callback_runtime_on(struct kbase_device *kbdev) +{ + return 0; +} + +static void rk_pm_callback_runtime_off(struct kbase_device *kbdev) +{ +} + +static int rk_pm_callback_power_on(struct kbase_device *kbdev) +{ + int ret = 1; /* Assume GPU has been powered off */ + int err = 0; + struct rk_context *platform = get_rk_context(kbdev); + + cancel_delayed_work_sync(&platform->work); + + err = rk_pm_enable_clk(kbdev); + if (err) { + E("failed to enable clk: %d", err); + return err; + } + + if (platform->is_powered) { + D("mali_device is already powered."); + return 0; + } + + /* we must enable vdd_gpu before pd_gpu_in_chip. */ + err = rk_pm_enable_regulator(kbdev); + if (err) { + E("fail to enable regulator, err : %d.", err); + return err; + } + + /* è‹¥ mali_dev çš„ runtime_pm 是 enabled çš„, 则... */ + if (pm_runtime_enabled(kbdev->dev)) { + D("to resume mali_dev syncly."); + /* 对 pd_in_chip çš„ on æ“作, + * 将在 pm_domain çš„ runtime_pm_callbacks 中完æˆ. + */ + err = pm_runtime_get_sync(kbdev->dev); + if (err < 0) { + E("failed to runtime resume device: %d.", err); + return err; + } else if (err == 1) { /* runtime_pm_status is still active */ + D("chip has NOT been powered off, no need to re-init."); + ret = 0; + } + } + + platform->is_powered = true; + KBASE_TIMELINE_GPU_POWER(kbdev, 1); + wake_lock(&platform->wake_lock); + + return ret; +} + +static void rk_pm_callback_power_off(struct kbase_device *kbdev) +{ + struct rk_context *platform = get_rk_context(kbdev); + + rk_pm_disable_clk(kbdev); + queue_delayed_work(platform->power_off_wq, &platform->work, + msecs_to_jiffies(platform->delay_ms)); +} + +int rk_kbase_device_runtime_init(struct kbase_device *kbdev) +{ + return 0; +} + +void rk_kbase_device_runtime_disable(struct kbase_device *kbdev) +{ +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = rk_pm_callback_power_on, + .power_off_callback = rk_pm_callback_power_off, +#ifdef CONFIG_PM + .power_runtime_init_callback = rk_kbase_device_runtime_init, + .power_runtime_term_callback = rk_kbase_device_runtime_disable, + .power_runtime_on_callback = rk_pm_callback_runtime_on, + .power_runtime_off_callback = rk_pm_callback_runtime_off, +#else /* CONFIG_PM */ + .power_runtime_init_callback = NULL, + .power_runtime_term_callback = NULL, + .power_runtime_on_callback = NULL, + .power_runtime_off_callback = NULL, +#endif /* CONFIG_PM */ +}; + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} + +/*---------------------------------------------------------------------------*/ + +void kbase_platform_rk_shutdown(struct kbase_device *kbdev) +{ + I("to make vdd_gpu enabled for turning off pd_gpu in pm_framework."); + rk_pm_enable_regulator(kbdev); +} + +/*---------------------------------------------------------------------------*/ + +#ifdef CONFIG_REGULATOR +static int rk_pm_enable_regulator(struct kbase_device *kbdev) +{ + int ret = 0; + + if (!kbdev->regulator) { + W("no mali regulator control, no need to enable."); + goto EXIT; + } + + D("to enable regulator."); + ret = regulator_enable(kbdev->regulator); + if (ret) { + E("fail to enable regulator, ret : %d.", ret); + goto EXIT; + } + +EXIT: + return ret; +} + +static void rk_pm_disable_regulator(struct kbase_device *kbdev) +{ + if (!(kbdev->regulator)) { + W("no mali regulator control, no need to disable."); + return; + } + + D("to disable regulator."); + regulator_disable(kbdev->regulator); +} +#endif + +static int rk_pm_enable_clk(struct kbase_device *kbdev) +{ + int err = 0; + + if (!(kbdev->clock)) { + W("no mali clock control, no need to enable."); + } else { + D("to enable clk."); + err = clk_enable(kbdev->clock); + if (err) + E("failed to enable clk: %d.", err); + } + + return err; +} + +static void rk_pm_disable_clk(struct kbase_device *kbdev) +{ + if (!(kbdev->clock)) { + W("no mali clock control, no need to disable."); + } else { + D("to disable clk."); + clk_disable(kbdev->clock); + } +} + +/*---------------------------------------------------------------------------*/ + +static ssize_t utilisation_period_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct rk_context *platform = get_rk_context(kbdev); + ssize_t ret = 0; + + ret += snprintf(buf, PAGE_SIZE, "%u\n", platform->utilisation_period); + + return ret; +} + +static ssize_t utilisation_period_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct rk_context *platform = get_rk_context(kbdev); + int ret = 0; + + ret = kstrtouint(buf, 0, &platform->utilisation_period); + if (ret) { + E("invalid input period : %s.", buf); + return ret; + } + D("set utilisation_period to '%d'.", platform->utilisation_period); + + return count; +} + +static ssize_t utilisation_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct rk_context *platform = get_rk_context(kbdev); + ssize_t ret = 0; + unsigned long period_in_us = platform->utilisation_period * 1000; + unsigned long total_time; + unsigned long busy_time; + unsigned long utilisation; + + kbase_pm_reset_dvfs_utilisation(kbdev); + usleep_range(period_in_us, period_in_us + 100); + kbase_pm_get_dvfs_utilisation(kbdev, &total_time, &busy_time); + /* 'devfreq_dev_profile' instance registered to devfreq + * also uses kbase_pm_reset_dvfs_utilisation + * and kbase_pm_get_dvfs_utilisation. + * it's better to cat this file when DVFS is disabled. + */ + D("total_time : %lu, busy_time : %lu.", total_time, busy_time); + + utilisation = busy_time * 100 / total_time; + ret += snprintf(buf, PAGE_SIZE, "%ld\n", utilisation); + + return ret; +} + +static DEVICE_ATTR_RW(utilisation_period); +static DEVICE_ATTR_RO(utilisation); + +static int kbase_platform_rk_create_sysfs_files(struct device *dev) +{ + int ret = 0; + + ret = device_create_file(dev, &dev_attr_utilisation_period); + if (ret) { + E("fail to create sysfs file 'utilisation_period'."); + goto out; + } + + ret = device_create_file(dev, &dev_attr_utilisation); + if (ret) { + E("fail to create sysfs file 'utilisation'."); + goto remove_utilisation_period; + } + + return 0; + +remove_utilisation_period: + device_remove_file(dev, &dev_attr_utilisation_period); +out: + return ret; +} + +static void kbase_platform_rk_remove_sysfs_files(struct device *dev) +{ + device_remove_file(dev, &dev_attr_utilisation_period); + device_remove_file(dev, &dev_attr_utilisation); +} + +int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev) +{ + return rockchip_init_opp_table(kbdev->dev, NULL, + "gpu_leakage", "mali"); +} diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_rk.h b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_rk.h new file mode 100755 index 000000000000..6eab25014d21 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/rk/mali_kbase_rk.h @@ -0,0 +1,62 @@ +/* drivers/gpu/t6xx/kbase/src/platform/rk/mali_kbase_platform.h + * Rockchip SoC Mali-Midgard platform-dependent codes + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software FoundatIon. + */ + +/** + * @file mali_kbase_rk.h + * + * defines work_context type of platform_dependent_part. + */ + +#ifndef _MALI_KBASE_RK_H_ +#define _MALI_KBASE_RK_H_ + +#include + +/*---------------------------------------------------------------------------*/ + +#define DEFAULT_UTILISATION_PERIOD_IN_MS (100) + +/*---------------------------------------------------------------------------*/ + +/* + * struct rk_context - work_context of platform_dependent_part_of_rk. + */ +struct rk_context { + /* + * record the status of common_parts calling 'power_on_callback' + * and 'power_off_callback'. + */ + bool is_powered; + + struct kbase_device *kbdev; + + struct workqueue_struct *power_off_wq; + /* delayed_work_to_power_off_gpu. */ + struct delayed_work work; + unsigned int delay_ms; + + /* + * WAKE_LOCK_SUSPEND for ensuring to run + * delayed_work_to_power_off_gpu before suspend. + */ + struct wake_lock wake_lock; + + /* debug only, the period in ms to count gpu_utilisation. */ + unsigned int utilisation_period; +}; + +/*---------------------------------------------------------------------------*/ + +static inline struct rk_context *get_rk_context( + const struct kbase_device *kbdev) +{ + return (struct rk_context *)(kbdev->platform_context); +} + +#endif /* _MALI_KBASE_RK_H_ */ + diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/Kbuild b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/Kbuild new file mode 100755 index 000000000000..d9d5e9085231 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/Kbuild @@ -0,0 +1,19 @@ +# +# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +mali_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ + $(MALI_PLATFORM_DIR)/mali_kbase_cpu_vexpress.o \ + mali_kbase_platform_fake.o diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_platform.h new file mode 100755 index 000000000000..02835f129aa3 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_platform.h @@ -0,0 +1,75 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "mali_kbase_cpu_vexpress.h" + +/** + * Maximum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX kbase_get_platform_max_freq() +/** + * Minimum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN kbase_get_platform_min_freq() + +/** + * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed) + +/** + * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_vexpress.c new file mode 100755 index 000000000000..15ce2bc5eea5 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_config_vexpress.c @@ -0,0 +1,85 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include +#include +#include "mali_kbase_cpu_vexpress.h" +#include "mali_kbase_config_platform.h" + +#define HARD_RESET_AT_POWER_OFF 0 + +#ifndef CONFIG_OF +static struct kbase_io_resources io_resources = { + .job_irq_number = 68, + .mmu_irq_number = 69, + .gpu_irq_number = 70, + .io_memory_region = { + .start = 0xFC010000, + .end = 0xFC010000 + (4096 * 4) - 1 + } +}; +#endif /* CONFIG_OF */ + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ + return 1; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ +#if HARD_RESET_AT_POWER_OFF + /* Cause a GPU hard reset to test whether we have actually idled the GPU + * and that we properly reconfigure the GPU on power up. + * Usually this would be dangerous, but if the GPU is working correctly it should + * be completely safe as the GPU should not be active at this point. + * However this is disabled normally because it will most likely interfere with + * bus logging etc. + */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); + kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); +#endif +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = NULL, + .power_resume_callback = NULL +}; + +static struct kbase_platform_config versatile_platform_config = { +#ifndef CONFIG_OF + .io_resources = &io_resources +#endif +}; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &versatile_platform_config; +} + + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.c b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.c new file mode 100755 index 000000000000..4665f98cbbe4 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.c @@ -0,0 +1,279 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include "mali_kbase_cpu_vexpress.h" + +#define HZ_IN_MHZ (1000000) + +#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000) +#define MOTHERBOARD_SYS_CFG_START (0x10000000) +#define SYS_CFGDATA_OFFSET (0x000000A0) +#define SYS_CFGCTRL_OFFSET (0x000000A4) +#define SYS_CFGSTAT_OFFSET (0x000000A8) + +#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31) +#define READ_REG_BIT_VALUE (0 << 30) +#define DCC_DEFAULT_BIT_VALUE (0 << 26) +#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20) +#define SITE_DEFAULT_BIT_VALUE (1 << 16) +#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12) +#define DEVICE_DEFAULT_BIT_VALUE (2 << 0) +#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0) +#define SYS_CFG_ERROR_BIT_VALUE (1 << 1) + +#define FEED_REG_BIT_MASK (0x0F) +#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03) +#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07) +#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B) +#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F) +#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13) + +/* the following three values used for reading + * HBI value of the LogicTile daughterboard */ +#define VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 (0x10000000) +#define VE_SYS_PROC_ID1_OFFSET (0x00000088) +#define VE_LOGIC_TILE_HBI_MASK (0x00000FFF) + +#define IS_SINGLE_BIT_SET(val, pos) (val&(1<> + FCLK_PA_DIVIDE_BIT_SHIFT); + /* CFGRW0[10:7] */ + pb_divide = ((reg_val & (FEED_REG_BIT_MASK << + FCLK_PB_DIVIDE_BIT_SHIFT)) >> + FCLK_PB_DIVIDE_BIT_SHIFT); + *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1); + } else if (IS_SINGLE_BIT_SET(reg_val, 1)) { + /* CFGRW0[1] - CLKOC */ + /* CFGRW0[6:3] */ + pa_divide = ((reg_val & (FEED_REG_BIT_MASK << + FCLK_PA_DIVIDE_BIT_SHIFT)) >> + FCLK_PA_DIVIDE_BIT_SHIFT); + /* CFGRW0[14:11] */ + pc_divide = ((reg_val & (FEED_REG_BIT_MASK << + FCLK_PC_DIVIDE_BIT_SHIFT)) >> + FCLK_PC_DIVIDE_BIT_SHIFT); + *cpu_clock = osc2_value * (pa_divide + 1) / (pc_divide + 1); + } else if (IS_SINGLE_BIT_SET(reg_val, 2)) { + /* CFGRW0[2] - FACLK */ + /* CFGRW0[18:15] */ + pa_divide = ((reg_val & (FEED_REG_BIT_MASK << + AXICLK_PA_DIVIDE_BIT_SHIFT)) >> + AXICLK_PA_DIVIDE_BIT_SHIFT); + /* CFGRW0[22:19] */ + pb_divide = ((reg_val & (FEED_REG_BIT_MASK << + AXICLK_PB_DIVIDE_BIT_SHIFT)) >> + AXICLK_PB_DIVIDE_BIT_SHIFT); + *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1); + } else { + err = -EIO; + } + +set_reg_error: +ongoing_request: + raw_spin_unlock(&syscfg_lock); + *cpu_clock /= HZ_IN_MHZ; + + if (!err) + cpu_clock_speed = *cpu_clock; + + iounmap(scc_reg); + +scc_reg_map_failed: + iounmap(syscfg_reg); + +syscfg_reg_map_failed: + + return err; +} + +/** + * kbase_get_platform_logic_tile_type - determines which LogicTile type + * is used by Versatile Express + * + * When platform_config build parameter is specified as vexpress, i.e., + * platform_config=vexpress, GPU frequency may vary dependent on the + * particular platform. The GPU frequency depends on the LogicTile type. + * + * This function determines which LogicTile type is used by the platform by + * reading the HBI value of the daughterboard which holds the LogicTile: + * + * 0x217 HBI0217 Virtex-6 + * 0x192 HBI0192 Virtex-5 + * 0x247 HBI0247 Virtex-7 + * + * Return: HBI value of the logic tile daughterboard, zero if not accessible + */ +static u32 kbase_get_platform_logic_tile_type(void) +{ + void __iomem *syscfg_reg = NULL; + u32 sys_procid1 = 0; + + syscfg_reg = ioremap(VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 + VE_SYS_PROC_ID1_OFFSET, 4); + if (NULL != syscfg_reg) { + sys_procid1 = readl(syscfg_reg); + iounmap(syscfg_reg); + } + + return sys_procid1 & VE_LOGIC_TILE_HBI_MASK; +} + +u32 kbase_get_platform_min_freq(void) +{ + u32 ve_logic_tile = kbase_get_platform_logic_tile_type(); + + switch (ve_logic_tile) { + case 0x217: + /* Virtex 6, HBI0217 */ + return VE_VIRTEX6_GPU_FREQ_MIN; + case 0x247: + /* Virtex 7, HBI0247 */ + return VE_VIRTEX7_GPU_FREQ_MIN; + default: + /* all other logic tiles, i.e., Virtex 5 HBI0192 + * or unsuccessful reading from the platform - + * fall back to some default value */ + return VE_DEFAULT_GPU_FREQ_MIN; + } +} + +u32 kbase_get_platform_max_freq(void) +{ + u32 ve_logic_tile = kbase_get_platform_logic_tile_type(); + + switch (ve_logic_tile) { + case 0x217: + /* Virtex 6, HBI0217 */ + return VE_VIRTEX6_GPU_FREQ_MAX; + case 0x247: + /* Virtex 7, HBI0247 */ + return VE_VIRTEX7_GPU_FREQ_MAX; + default: + /* all other logic tiles, i.e., Virtex 5 HBI0192 + * or unsuccessful reading from the platform - + * fall back to some default value */ + return VE_DEFAULT_GPU_FREQ_MAX; + } +} diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.h b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.h new file mode 100755 index 000000000000..da865698133a --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress/mali_kbase_cpu_vexpress.h @@ -0,0 +1,38 @@ +/* + * + * (C) COPYRIGHT 2012-2013, 2015-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_CPU_VEXPRESS_H_ +#define _KBASE_CPU_VEXPRESS_H_ + +/** + * Versatile Express implementation of @ref kbase_cpu_clk_speed_func. + */ +int kbase_get_vexpress_cpu_clock_speed(u32 *cpu_clock); + +/** + * Get the minimum GPU frequency for the attached logic tile + */ +u32 kbase_get_platform_min_freq(void); + +/** + * Get the maximum GPU frequency for the attached logic tile + */ +u32 kbase_get_platform_max_freq(void); + +#endif /* _KBASE_CPU_VEXPRESS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/Kbuild b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/Kbuild new file mode 100755 index 000000000000..df87c74f43ba --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/Kbuild @@ -0,0 +1,18 @@ +# +# (C) COPYRIGHT 2013-2014, 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +mali_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ + mali_kbase_platform_fake.o diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h new file mode 100755 index 000000000000..0efbf3962f98 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h @@ -0,0 +1,73 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * Maximum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX 5000 +/** + * Minimum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN 5000 + +/** + * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (&kbase_cpuprops_get_default_clock_speed) + +/** + * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c new file mode 100755 index 000000000000..3ff0930fb4a3 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c @@ -0,0 +1,79 @@ +/* + * + * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include + +#define HARD_RESET_AT_POWER_OFF 0 + +#ifndef CONFIG_OF +static struct kbase_io_resources io_resources = { + .job_irq_number = 68, + .mmu_irq_number = 69, + .gpu_irq_number = 70, + .io_memory_region = { + .start = 0x2f010000, + .end = 0x2f010000 + (4096 * 4) - 1} +}; +#endif + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ + return 1; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ +#if HARD_RESET_AT_POWER_OFF + /* Cause a GPU hard reset to test whether we have actually idled the GPU + * and that we properly reconfigure the GPU on power up. + * Usually this would be dangerous, but if the GPU is working correctly it should + * be completely safe as the GPU should not be active at this point. + * However this is disabled normally because it will most likely interfere with + * bus logging etc. + */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); + kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); +#endif +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = NULL, + .power_resume_callback = NULL +}; + +static struct kbase_platform_config versatile_platform_config = { +#ifndef CONFIG_OF + .io_resources = &io_resources +#endif +}; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &versatile_platform_config; +} + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/Kbuild b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/Kbuild new file mode 100755 index 000000000000..d9d5e9085231 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/Kbuild @@ -0,0 +1,19 @@ +# +# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +mali_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ + $(MALI_PLATFORM_DIR)/mali_kbase_cpu_vexpress.o \ + mali_kbase_platform_fake.o diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h new file mode 100755 index 000000000000..dbdf21e009f9 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h @@ -0,0 +1,75 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "mali_kbase_cpu_vexpress.h" + +/** + * Maximum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX 10000 +/** + * Minimum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN 10000 + +/** + * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed) + +/** + * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c new file mode 100755 index 000000000000..76ffe4a1e59e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c @@ -0,0 +1,83 @@ +/* + * + * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include +#include +#include "mali_kbase_cpu_vexpress.h" + +#define HARD_RESET_AT_POWER_OFF 0 + +#ifndef CONFIG_OF +static struct kbase_io_resources io_resources = { + .job_irq_number = 75, + .mmu_irq_number = 76, + .gpu_irq_number = 77, + .io_memory_region = { + .start = 0x2F000000, + .end = 0x2F000000 + (4096 * 4) - 1} +}; +#endif + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ + return 1; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ +#if HARD_RESET_AT_POWER_OFF + /* Cause a GPU hard reset to test whether we have actually idled the GPU + * and that we properly reconfigure the GPU on power up. + * Usually this would be dangerous, but if the GPU is working correctly it should + * be completely safe as the GPU should not be active at this point. + * However this is disabled normally because it will most likely interfere with + * bus logging etc. + */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); + kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); +#endif +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = NULL, + .power_resume_callback = NULL +}; + +static struct kbase_platform_config versatile_platform_config = { +#ifndef CONFIG_OF + .io_resources = &io_resources +#endif +}; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &versatile_platform_config; +} + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c new file mode 100755 index 000000000000..816dff49835f --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c @@ -0,0 +1,71 @@ +/* + * + * (C) COPYRIGHT 2011-2013 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include "mali_kbase_cpu_vexpress.h" + +#define HZ_IN_MHZ (1000000) + +#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000) +#define MOTHERBOARD_SYS_CFG_START (0x10000000) +#define SYS_CFGDATA_OFFSET (0x000000A0) +#define SYS_CFGCTRL_OFFSET (0x000000A4) +#define SYS_CFGSTAT_OFFSET (0x000000A8) + +#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31) +#define READ_REG_BIT_VALUE (0 << 30) +#define DCC_DEFAULT_BIT_VALUE (0 << 26) +#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20) +#define SITE_DEFAULT_BIT_VALUE (1 << 16) +#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12) +#define DEVICE_DEFAULT_BIT_VALUE (2 << 0) +#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0) +#define SYS_CFG_ERROR_BIT_VALUE (1 << 1) + +#define FEED_REG_BIT_MASK (0x0F) +#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03) +#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07) +#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B) +#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F) +#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13) + +#define IS_SINGLE_BIT_SET(val, pos) (val&(1< + +/** + * @addtogroup uk_api User-Kernel Interface API + * @{ + */ + +/** + * @addtogroup uk_api_kernel UKK (Kernel side) + * @{ + */ + +/** + * Internal OS specific data structure associated with each UKK session. Part + * of a ukk_session object. + */ +typedef struct ukkp_session { + int dummy; /**< No internal OS specific data at this time */ +} ukkp_session; + +/** @} end group uk_api_kernel */ + +/** @} end group uk_api */ + +#endif /* _UKK_OS_H__ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/protected_mode_switcher.h b/drivers/gpu/arm/bifrost_for_linux/protected_mode_switcher.h new file mode 100755 index 000000000000..5dc2f3ba8cf6 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/protected_mode_switcher.h @@ -0,0 +1,64 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _PROTECTED_MODE_SWITCH_H_ +#define _PROTECTED_MODE_SWITCH_H_ + +struct protected_mode_device; + +/** + * struct protected_mode_ops - Callbacks for protected mode switch operations + * + * @protected_mode_enable: Callback to enable protected mode for device + * @protected_mode_disable: Callback to disable protected mode for device + */ +struct protected_mode_ops { + /** + * protected_mode_enable() - Enable protected mode on device + * @dev: The struct device + * + * Return: 0 on success, non-zero on error + */ + int (*protected_mode_enable)( + struct protected_mode_device *protected_dev); + + /** + * protected_mode_disable() - Disable protected mode on device, and + * reset device + * @dev: The struct device + * + * Return: 0 on success, non-zero on error + */ + int (*protected_mode_disable)( + struct protected_mode_device *protected_dev); +}; + +/** + * struct protected_mode_device - Device structure for protected mode devices + * + * @ops - Callbacks associated with this device + * @data - Pointer to device private data + * + * This structure should be registered with the platform device using + * platform_set_drvdata(). + */ +struct protected_mode_device { + struct protected_mode_ops ops; + void *data; +}; + +#endif /* _PROTECTED_MODE_SWITCH_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/sconscript b/drivers/gpu/arm/bifrost_for_linux/sconscript new file mode 100755 index 000000000000..e738dd7a3869 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/sconscript @@ -0,0 +1,72 @@ +# +# (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +import sys +Import('env') + +SConscript( 'tests/sconscript' ) + +mock_test = 0 + +# Source files required for kbase. +kbase_src = [ + Glob('*.c'), + Glob('backend/*/*.c'), + Glob('internal/*/*.c'), + Glob('ipa/*.c'), + Glob('platform/%s/*.c' % env['platform_config']), +] + +if env['platform_config']=='juno_soc': + kbase_src += [Glob('platform/devicetree/*.c')] +else: + kbase_src += [Glob('platform/%s/*.c' % env['platform_config'])] + +if Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock') and env['unit'] == '1': + kbase_src += [Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock/*.c')] + mock_test = 1 + +make_args = env.kernel_get_config_defines(ret_list = True) + [ + 'PLATFORM=%s' % env['platform'], + 'MALI_ERROR_INJECT_ON=%s' % env['error_inject'], + 'MALI_KERNEL_TEST_API=%s' % env['debug'], + 'MALI_UNIT_TEST=%s' % env['unit'], + 'MALI_RELEASE_NAME=%s' % env['mali_release_name'], + 'MALI_MOCK_TEST=%s' % mock_test, + 'MALI_CUSTOMER_RELEASE=%s' % env['release'], + 'MALI_INSTRUMENTATION_LEVEL=%s' % env['instr'], + 'MALI_COVERAGE=%s' % env['coverage'], +] + +kbase = env.BuildKernelModule('$STATIC_LIB_PATH/mali_kbase.ko', kbase_src, + make_args = make_args) + +# Add a dependency on kds.ko. +# Only necessary when KDS is not built into the kernel. +# +if env['os'] != 'android': + if not env.KernelConfigEnabled("CONFIG_KDS"): + env.Depends(kbase, '$STATIC_LIB_PATH/kds.ko') + +# need Module.symvers from ump.ko build +if int(env['ump']) == 1: + env.Depends(kbase, '$STATIC_LIB_PATH/ump.ko') + +if 'smc_protected_mode_switcher' in env: + env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/smc_protected_mode_switcher.ko') + +env.KernelObjTarget('kbase', kbase) + +env.AppendUnique(BASE=['cutils_linked_list']) diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/Kbuild b/drivers/gpu/arm/bifrost_for_linux/tests/Kbuild new file mode 100755 index 000000000000..b4bed0473439 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/Kbuild @@ -0,0 +1,17 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +obj-$(CONFIG_MALI_KUTF) += kutf/ +obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test/ diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/Kconfig b/drivers/gpu/arm/bifrost_for_linux/tests/Kconfig new file mode 100755 index 000000000000..da0515c065de --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/Kconfig @@ -0,0 +1,17 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +source "drivers/gpu/arm/midgard/tests/kutf/Kconfig" +source "drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig" diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers.h new file mode 100755 index 000000000000..3f1dfc244d30 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers.h @@ -0,0 +1,216 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_HELPERS_H_ +#define _KERNEL_UTF_HELPERS_H_ + +/* kutf_helpers.h + * Test helper functions for the kernel UTF test infrastructure. + * + * This collection of helper functions are provided as 'stock' implementation + * helpers for certain features of kutf. Tests can implement common/boilerplate + * functionality using these, whilst still providing them the option of + * implementing completely custom functions themselves to use those kutf + * features. + */ + +#include +#include +#include + +/** + * enum kutf_helper_textbuf_flag - flags for textbufs + * @KUTF_HELPER_TEXTBUF_FLAG_DYING: Test is dying, textbuf should not allow + * writes, nor block on empty. + */ +enum kutf_helper_textbuf_flag { + KUTF_HELPER_TEXTBUF_FLAG_DYING = (1u << 0), +}; + +/** + * struct kutf_helper_textbuf_line - Structure representing a line of text + * + * The string itself is stored immediately after this. + * + * @node: List node for the textbuf's textbuf_list + * @str_size: Length of the string buffer, including the \0 terminator + * @str: 'Flexible array' for the string representing the line + */ +struct kutf_helper_textbuf_line { + struct list_head node; + int str_size; + char str[]; +}; + +/** + * struct kutf_helper_textbuf - Structure to representing sequential lines of + * text + * @lock: mutex to hold whilst accessing the structure + * @nr_user_clients: Number of userspace clients connected via an open() + * call + * @mempool: mempool for allocating lines + * @scratchpad: scratch area for receiving text of size max_line_size + * @used_bytes: number of valid bytes in the scratchpad + * @prev_pos: Previous position userspace has accessed + * @prev_line_pos: Previous start of line position userspace has accessed + * @textbuf_list: List head to store all the lines of text + * @max_line_size: Maximum size in memory allowed for a line of text + * @max_nr_lines: Maximum number of lines permitted in this textbuf + * @nr_lines: Number of entries in textbuf_list + * @flags: Flags indicating state of the textbuf, using values + * from enum kutf_helper_textbuf_flag + * @user_opened_wq: Waitq for when there's at least one userspace client + * connected to the textbuf via an open() call + * @not_full_wq: Waitq for when the textbuf can be enqueued into/can + * consume data from userspace + * @not_empty_wq: Waitq for when the textbuf can be dequeued from/can + * produce data for userspace + */ + +struct kutf_helper_textbuf { + struct mutex lock; + int nr_user_clients; + struct kutf_mempool *mempool; + char *scratchpad; + int used_bytes; + loff_t prev_pos; + loff_t prev_line_pos; + struct list_head textbuf_list; + int max_line_size; + int max_nr_lines; + int nr_lines; + unsigned long flags; + wait_queue_head_t user_opened_wq; + wait_queue_head_t not_full_wq; + wait_queue_head_t not_empty_wq; + +}; + +/* stock callbacks for userspace to read from/write to the 'data' file as a + * textbuf */ +extern struct kutf_userdata_ops kutf_helper_textbuf_userdata_ops; + +/** + * kutf_helper_textbuf_init() - init a textbuf for use as a 'data' file + * consumer/producer + * @textbuf: textbuf to initialize + * @mempool: mempool to allocate from + * @max_line_size: maximum line size expected to/from userspace + * @max_nr_lines: maximum number of lines to expect to/from userspace + * + * Initialize a textbuf so that it can consume writes made to the 'data' file, + * and produce reads for userspace on the 'data' file. Tests may then read the + * lines written by userspace, or fill the buffer so it may be read back by + * userspace. + * + * The caller should write the @textbuf pointer into the kutf_context's + * userdata_producer_priv or userdata_consumer_priv member during fixture + * creation. + * + * Usually a test will have separate textbufs for userspace to write to and + * read from. Using the same one for both will echo back to the user what they + * are writing. + * + * Lines are understood as being separated by the '\n' character, but no '\n' + * characters will be observed by the test + * + * @max_line_size puts an upper bound on the size of lines in a textbuf, + * including the \0 terminator. Lines exceeding this will be truncated, + * effectively ignoring incoming data until the next '\n' + * + * Combining this with @max_nr_lines puts an upper bound on the size of the + * file read in + * + * Return: 0 on success, or negative value on error. + */ +int kutf_helper_textbuf_init(struct kutf_helper_textbuf *textbuf, + struct kutf_mempool *mempool, int max_line_size, + int max_nr_lines); + +/** + * kutf_helper_textbuf_wait_for_user() - wait for userspace to open the 'data' + * file + * @textbuf: textbuf to wait on + * + * This can be used to synchronize with userspace so that subsequent calls to + * kutf_helper_textbuf_dequeue() and kutf_helper_textbuf_enqueue() should + * succeed. + * + * Waiting is done on a timeout. + * + * There is of course no guarantee that userspace will keep the file open after + * this, but any error in the dequeue/enqueue functions afterwards can be + * treated as such rather than "we're still waiting for userspace to begin" + * + * Return: 0 if waited successfully, -ETIMEDOUT if we exceeded the + * timeout, or some other negative value if there was an + * error during waiting. + */ + +int kutf_helper_textbuf_wait_for_user(struct kutf_helper_textbuf *textbuf); + + +/** + * kutf_helper_textbuf_dequeue() - dequeue a line from a textbuf + * @textbuf: textbuf dequeue a line as a string from + * @str_size: pointer to storage to receive the size of the string, + * which includes the '\0' terminator, or NULL if not + * required + * + * Dequeue (remove) a line from the start of the textbuf as a string, and + * return it. + * + * If no lines are available, then this will block until a line has been + * submitted. If a userspace client is not connected and there are no remaining + * lines, then this function returns NULL instead. + * + * The memory for the string comes from the kutf_mempool given during + * initialization of the textbuf, and shares the same lifetime as it. + * + * Return: pointer to the next line of the textbuf. NULL indicated + * all userspace clients disconnected. An error value to be + * checked with IS_ERR() family of functions if a signal or + * some other error occurred + */ +char *kutf_helper_textbuf_dequeue(struct kutf_helper_textbuf *textbuf, + int *str_size); + +/** + * kutf_helper_textbuf_enqueue() - enqueue a line to a textbuf + * @textbuf: textbuf to enqueue a line as a string to + * @enqueue_str: pointer to the string to enqueue to the textbuf + * @buf_max_size: maximum size of the buffer holding @enqueue_str + * + * Enqueue (add) a line to the end of a textbuf as a string. + * + * The caller should avoid placing '\n' characters in their strings, as these + * will not be split into multiple lines. + * + * A copy of the string will be made into the textbuf, so @enqueue_str can be + * freed immediately after if.the caller wishes to do so. + * + * If the maximum amount of lines has been reached, then this will block until + * a line has been removed to make space. If a userspace client is not + * connected and there is no space available, then this function returns + * -EBUSY. + * + * Return: 0 on success, or negative value on error + */ +int kutf_helper_textbuf_enqueue(struct kutf_helper_textbuf *textbuf, + char *enqueue_str, int buf_max_size); + +#endif /* _KERNEL_UTF_HELPERS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers_user.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers_user.h new file mode 100755 index 000000000000..759bf717c7cd --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_helpers_user.h @@ -0,0 +1,179 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_HELPERS_USER_H_ +#define _KERNEL_UTF_HELPERS_USER_H_ + +/* kutf_helpers.h + * Test helper functions for the kernel UTF test infrastructure, whose + * implementation mirrors that of similar functions for kutf-userside + */ + +#include +#include + + +#define KUTF_HELPER_MAX_VAL_NAME_LEN 255 + +enum kutf_helper_valtype { + KUTF_HELPER_VALTYPE_INVALID, + KUTF_HELPER_VALTYPE_U64, + KUTF_HELPER_VALTYPE_STR, + + KUTF_HELPER_VALTYPE_COUNT /* Must be last */ +}; + +struct kutf_helper_named_val { + enum kutf_helper_valtype type; + char *val_name; + union { + u64 val_u64; + char *val_str; + } u; +}; + +/* Extra error values for certain helpers when we want to distinguish between + * Linux's own error values too. + * + * These can only be used on certain functions returning an int type that are + * documented as returning one of these potential values, they cannot be used + * from functions return a ptr type, since we can't decode it with PTR_ERR + * + * No negative values are used - Linux error codes should be used instead, and + * indicate a problem in accessing the data file itself (are generally + * unrecoverable) + * + * Positive values indicate correct access but invalid parsing (can be + * recovered from assuming data in the future is correct) */ +enum kutf_helper_err { + /* No error - must be zero */ + KUTF_HELPER_ERR_NONE = 0, + /* Named value parsing encountered an invalid name */ + KUTF_HELPER_ERR_INVALID_NAME, + /* Named value parsing of string or u64 type encountered extra + * characters after the value (after the last digit for a u64 type or + * after the string end delimiter for string type) */ + KUTF_HELPER_ERR_CHARS_AFTER_VAL, + /* Named value parsing of string type couldn't find the string end + * delimiter. + * + * This cannot be encountered when the NAME="value" message exceeds the + * textbuf's maximum line length, because such messages are not checked + * for an end string delimiter */ + KUTF_HELPER_ERR_NO_END_DELIMITER, + /* Named value didn't parse as any of the known types */ + KUTF_HELPER_ERR_INVALID_VALUE, +}; + + +/* textbuf Send named NAME=value pair, u64 value + * + * NAME must match [A-Z0-9_]\+ and can be up to MAX_VAL_NAME_LEN characters long + * + * This is assuming the kernel-side test is using the 'textbuf' helpers + * + * Any failure will be logged on the suite's current test fixture + * + * Returns 0 on success, non-zero on failure + */ +int kutf_helper_textbuf_send_named_u64(struct kutf_context *context, + struct kutf_helper_textbuf *textbuf, char *val_name, u64 val); + +/* Get the maximum length of a string that can be represented as a particular + * NAME="value" pair without string-value truncation in the kernel's buffer + * + * Given val_name and the kernel buffer's size, this can be used to determine + * the maximum length of a string that can be sent as val_name="value" pair + * without having the string value truncated. Any string longer than this will + * be truncated at some point during communication to this size. + * + * The calculation is valid both for sending strings of val_str_len to kernel, + * and for receiving a string that was originally val_str_len from the kernel. + * + * It is assumed that valname is a valid name for + * kutf_test_helpers_textbuf_send_named_str(), and no checking will be made to + * ensure this. + * + * Returns the maximum string length that can be represented, or a negative + * value if the NAME="value" encoding itself wouldn't fit in kern_buf_sz + */ +int kutf_helper_textbuf_max_str_len_for_kern(char *val_name, int kern_buf_sz); + +/* textbuf Send named NAME="str" pair + * + * no escaping allowed in str. Any of the following characters will terminate + * the string: '"' '\\' '\n' + * + * NAME must match [A-Z0-9_]\+ and can be up to MAX_VAL_NAME_LEN characters long + * + * This is assuming the kernel-side test is using the 'textbuf' helpers + * + * Any failure will be logged on the suite's current test fixture + * + * Returns 0 on success, non-zero on failure */ +int kutf_helper_textbuf_send_named_str(struct kutf_context *context, + struct kutf_helper_textbuf *textbuf, char *val_name, + char *val_str); + +/* textbuf Receive named NAME=value pair + * + * This can receive u64 and string values - check named_val->type + * + * If you are not planning on dynamic handling of the named value's name and + * type, then kutf_test_helpers_textbuf_receive_check_val() is more useful as a + * convenience function. + * + * String members of named_val will come from memory allocated on the fixture's mempool + * + * Returns 0 on success. Negative value on failure to receive from the 'data' + * file, positive value indicates an enum kutf_helper_err value for correct + * reception of data but invalid parsing */ +int kutf_helper_textbuf_receive_named_val(struct kutf_helper_named_val *named_val, + struct kutf_helper_textbuf *textbuf); + +/* textbuf Receive and validate NAME=value pair + * + * As with kutf_test_helpers_textbuf_receive_named_val, but validate that the + * name and type are as expected, as a convenience for a common pattern found + * in tests. + * + * NOTE: this only returns an error value if there was actually a problem + * receiving data. + * + * NOTE: If the underlying data was received correctly, but: + * - isn't of the expected name + * - isn't the expected type + * - isn't correctly parsed for the type + * then the following happens: + * - failure result is recorded + * - named_val->type will be KUTF_HELPER_VALTYPE_INVALID + * - named_val->u will contain some default value that should be relatively + * harmless for the test, including being writable in the case of string + * values + * - return value will be 0 to indicate success + * + * The rationale behind this is that we'd prefer to continue the rest of the + * test with failures propagated, rather than hitting a timeout */ +int kutf_helper_textbuf_receive_check_val(struct kutf_helper_named_val *named_val, + struct kutf_context *context, struct kutf_helper_textbuf *textbuf, + char *expect_val_name, enum kutf_helper_valtype expect_val_type); + +/* Output a named value to kmsg */ +void kutf_helper_output_named_val(struct kutf_helper_named_val *named_val); + + +#endif /* _KERNEL_UTF_HELPERS_USER_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_mem.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_mem.h new file mode 100755 index 000000000000..584c9dd4bc13 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_mem.h @@ -0,0 +1,68 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_MEM_H_ +#define _KERNEL_UTF_MEM_H_ + +/* kutf_mem.h + * Functions for management of memory pools in the kernel. + * + * This module implements a memory pool allocator, allowing a test + * implementation to allocate linked allocations which can then be freed by a + * single free which releases all of the resources held by the entire pool. + * + * Note that it is not possible to free single resources within the pool once + * allocated. + */ + +#include +#include + +/** + * struct kutf_mempool - the memory pool context management structure + * @head: list head on which the allocations in this context are added to + * @lock: mutex for concurrent allocation from multiple threads + * + */ +struct kutf_mempool { + struct list_head head; + struct mutex lock; +}; + +/** + * kutf_mempool_init() - Initialize a memory pool. + * @pool: Memory pool structure to initialize, provided by the user + * + * Return: zero on success + */ +int kutf_mempool_init(struct kutf_mempool *pool); + +/** + * kutf_mempool_alloc() - Allocate memory from a pool + * @pool: Memory pool to allocate from + * @size: Size of memory wanted in number of bytes + * + * Return: Pointer to memory on success, NULL on failure. + */ +void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size); + +/** + * kutf_mempool_destroy() - Destroy a memory pool, freeing all memory within it. + * @pool: The memory pool to free + */ +void kutf_mempool_destroy(struct kutf_mempool *pool); +#endif /* _KERNEL_UTF_MEM_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_resultset.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_resultset.h new file mode 100755 index 000000000000..1cc85f1b7a46 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_resultset.h @@ -0,0 +1,121 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_RESULTSET_H_ +#define _KERNEL_UTF_RESULTSET_H_ + +/* kutf_resultset.h + * Functions and structures for handling test results and result sets. + * + * This section of the kernel UTF contains structures and functions used for the + * management of Results and Result Sets. + */ + +/** + * enum kutf_result_status - Status values for a single Test error. + * @KUTF_RESULT_BENCHMARK: Result is a meta-result containing benchmark + * results. + * @KUTF_RESULT_SKIP: The test was skipped. + * @KUTF_RESULT_UNKNOWN: The test has an unknown result. + * @KUTF_RESULT_PASS: The test result passed. + * @KUTF_RESULT_DEBUG: The test result passed, but raised a debug + * message. + * @KUTF_RESULT_INFO: The test result passed, but raised + * an informative message. + * @KUTF_RESULT_WARN: The test result passed, but raised a warning + * message. + * @KUTF_RESULT_FAIL: The test result failed with a non-fatal error. + * @KUTF_RESULT_FATAL: The test result failed with a fatal error. + * @KUTF_RESULT_ABORT: The test result failed due to a non-UTF + * assertion failure. + * @KUTF_RESULT_COUNT: The current number of possible status messages. + */ +enum kutf_result_status { + KUTF_RESULT_BENCHMARK = -3, + KUTF_RESULT_SKIP = -2, + KUTF_RESULT_UNKNOWN = -1, + + KUTF_RESULT_PASS = 0, + KUTF_RESULT_DEBUG = 1, + KUTF_RESULT_INFO = 2, + KUTF_RESULT_WARN = 3, + KUTF_RESULT_FAIL = 4, + KUTF_RESULT_FATAL = 5, + KUTF_RESULT_ABORT = 6, + + KUTF_RESULT_COUNT +}; + +/* The maximum size of a kutf_result_status result when + * converted to a string + */ +#define KUTF_ERROR_MAX_NAME_SIZE 21 + +#ifdef __KERNEL__ + +#include + +/** + * struct kutf_result - Represents a single test result. + * @node: Next result in the list of results. + * @status: The status summary (pass / warn / fail / etc). + * @message: A more verbose status message. + */ +struct kutf_result { + struct list_head node; + enum kutf_result_status status; + const char *message; +}; + +/** + * kutf_create_result_set() - Create a new result set + * to which results can be added. + * + * Return: The created resultset. + */ +struct kutf_result_set *kutf_create_result_set(void); + +/** + * kutf_add_result() - Add a result to the end of an existing resultset. + * + * @mempool: The memory pool to allocate the result storage from. + * @set: The resultset to add the result to. + * @status: The result status to add. + * @message: The result message to add. + */ +void kutf_add_result(struct kutf_mempool *mempool, struct kutf_result_set *set, + enum kutf_result_status status, const char *message); + +/** + * kutf_remove_result() - Remove a result from the head of a resultset. + * @set: The resultset. + * + * Return: result or NULL if there are no further results in the resultset. + */ +struct kutf_result *kutf_remove_result( + struct kutf_result_set *set); + +/** + * kutf_destroy_result_set() - Free a previously created resultset. + * + * @results: The result set whose resources to free. + */ +void kutf_destroy_result_set(struct kutf_result_set *results); + +#endif /* __KERNEL__ */ + +#endif /* _KERNEL_UTF_RESULTSET_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_suite.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_suite.h new file mode 100755 index 000000000000..cba2b2d84d62 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_suite.h @@ -0,0 +1,568 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_SUITE_H_ +#define _KERNEL_UTF_SUITE_H_ + +/* kutf_suite.h + * Functions for management of test suites. + * + * This collection of data structures, macros, and functions are used to + * create Test Suites, Tests within those Test Suites, and Fixture variants + * of each test. + */ + +#include + +#include +#include + +/** + * Pseudo-flag indicating an absence of any specified test class. Note that + * tests should not be annotated with this constant as it is simply a zero + * value; tests without a more specific class must be marked with the flag + * KUTF_F_TEST_GENERIC. + */ +#define KUTF_F_TEST_NONE ((unsigned int)(0)) + +/** + * Class indicating this test is a smoke test. + * A given set of smoke tests should be quick to run, enabling rapid turn-around + * of "regress-on-commit" test runs. + */ +#define KUTF_F_TEST_SMOKETEST ((unsigned int)(1 << 1)) + +/** + * Class indicating this test is a performance test. + * These tests typically produce a performance metric, such as "time to run" or + * "frames per second", + */ +#define KUTF_F_TEST_PERFORMANCE ((unsigned int)(1 << 2)) + +/** + * Class indicating that this test is a deprecated test. + * These tests have typically been replaced by an alternative test which is + * more efficient, or has better coverage. + */ +#define KUTF_F_TEST_DEPRECATED ((unsigned int)(1 << 3)) + +/** + * Class indicating that this test is a known failure. + * These tests have typically been run and failed, but marking them as a known + * failure means it is easier to triage results. + * + * It is typically more convenient to triage known failures using the + * results database and web UI, as this means there is no need to modify the + * test code. + */ +#define KUTF_F_TEST_EXPECTED_FAILURE ((unsigned int)(1 << 4)) + +/** + * Class indicating that this test is a generic test, which is not a member of + * a more specific test class. Tests which are not created with a specific set + * of filter flags by the user are assigned this test class by default. + */ +#define KUTF_F_TEST_GENERIC ((unsigned int)(1 << 5)) + +/** + * Class indicating this test is a resource allocation failure test. + * A resource allocation failure test will test that an error code is + * correctly propagated when an allocation fails. + */ +#define KUTF_F_TEST_RESFAIL ((unsigned int)(1 << 6)) + +/** + * Additional flag indicating that this test is an expected failure when + * run in resource failure mode. These tests are never run when running + * the low resource mode. + */ +#define KUTF_F_TEST_EXPECTED_FAILURE_RF ((unsigned int)(1 << 7)) + +/** + * Flag reserved for user-defined filter zero. + */ +#define KUTF_F_TEST_USER_0 ((unsigned int)(1 << 24)) + +/** + * Flag reserved for user-defined filter one. + */ +#define KUTF_F_TEST_USER_1 ((unsigned int)(1 << 25)) + +/** + * Flag reserved for user-defined filter two. + */ +#define KUTF_F_TEST_USER_2 ((unsigned int)(1 << 26)) + +/** + * Flag reserved for user-defined filter three. + */ +#define KUTF_F_TEST_USER_3 ((unsigned int)(1 << 27)) + +/** + * Flag reserved for user-defined filter four. + */ +#define KUTF_F_TEST_USER_4 ((unsigned int)(1 << 28)) + +/** + * Flag reserved for user-defined filter five. + */ +#define KUTF_F_TEST_USER_5 ((unsigned int)(1 << 29)) + +/** + * Flag reserved for user-defined filter six. + */ +#define KUTF_F_TEST_USER_6 ((unsigned int)(1 << 30)) + +/** + * Flag reserved for user-defined filter seven. + */ +#define KUTF_F_TEST_USER_7 ((unsigned int)(1 << 31)) + +/** + * Pseudo-flag indicating that all test classes should be executed. + */ +#define KUTF_F_TEST_ALL ((unsigned int)(0xFFFFFFFFU)) + +/** + * union kutf_callback_data - Union used to store test callback data + * @ptr_value: pointer to the location where test callback data + * are stored + * @u32_value: a number which represents test callback data + */ +union kutf_callback_data { + void *ptr_value; + u32 u32_value; +}; + +/** + * struct kutf_userdata_ops- Structure defining methods to exchange data + * with userspace via the 'data' file + * @open: Function used to notify when the 'data' file was opened + * @release: Function used to notify when the 'data' file was closed + * @notify_ended: Function used to notify when the test has ended. + * @consumer: Function used to consume writes from userspace + * @producer: Function used to produce data for userspace to read + * + * All ops can be NULL. + */ +struct kutf_userdata_ops { + int (*open)(void *priv); + void (*release)(void *priv); + void (*notify_ended)(void *priv); + ssize_t (*consumer)(void *priv, const char __user *userbuf, + size_t userbuf_len, loff_t *ppos); + ssize_t (*producer)(void *priv, char __user *userbuf, + size_t userbuf_len, loff_t *ppos); +}; + +/** + * struct kutf_context - Structure representing a kernel test context + * @kref: Refcount for number of users of this context + * @suite: Convenience pointer to the suite this context + * is running + * @test_fix: The fixture that is being run in this context + * @fixture_pool: The memory pool used for the duration of + * the fixture/text context. + * @fixture: The user provided fixture structure. + * @fixture_index: The index (id) of the current fixture. + * @fixture_name: The name of the current fixture (or NULL if unnamed). + * @test_data: Any user private data associated with this test + * @result_set: All the results logged by this test context + * @status: The status of the currently running fixture. + * @expected_status: The expected status on exist of the currently + * running fixture. + * @userdata_consumer_priv: Parameter to pass into kutf_userdata_ops + * consumer function. Must not be NULL if a + * consumer function was specified + * @userdata_producer_priv: Parameter to pass into kutf_userdata_ops + * producer function. Must not be NULL if a + * producer function was specified + * @userdata_dentry: The debugfs file for userdata exchange + */ +struct kutf_context { + struct kref kref; + struct kutf_suite *suite; + struct kutf_test_fixture *test_fix; + struct kutf_mempool fixture_pool; + void *fixture; + unsigned int fixture_index; + const char *fixture_name; + union kutf_callback_data test_data; + struct kutf_result_set *result_set; + enum kutf_result_status status; + enum kutf_result_status expected_status; + void *userdata_consumer_priv; + void *userdata_producer_priv; + struct dentry *userdata_dentry; +}; + +/** + * struct kutf_suite - Structure representing a kernel test suite + * @app: The application this suite belongs to. + * @name: The name of this suite. + * @suite_data: Any user private data associated with this + * suite. + * @create_fixture: Function used to create a new fixture instance + * @remove_fixture: Function used to destroy a new fixture instance + * @fixture_variants: The number of variants (must be at least 1). + * @suite_default_flags: Suite global filter flags which are set on + * all tests. + * @node: List node for suite_list + * @dir: The debugfs directory for this suite + * @test_list: List head to store all the tests which are + * part of this suite + */ +struct kutf_suite { + struct kutf_application *app; + const char *name; + union kutf_callback_data suite_data; + void *(*create_fixture)(struct kutf_context *context); + void (*remove_fixture)(struct kutf_context *context); + unsigned int fixture_variants; + unsigned int suite_default_flags; + struct list_head node; + struct dentry *dir; + struct list_head test_list; +}; + +/* ============================================================================ + Application functions +============================================================================ */ + +/** + * kutf_create_application() - Create an in kernel test application. + * @name: The name of the test application. + * + * Return: pointer to the kutf_application on success or NULL + * on failure + */ +struct kutf_application *kutf_create_application(const char *name); + +/** + * kutf_destroy_application() - Destroy an in kernel test application. + * + * @app: The test application to destroy. + */ +void kutf_destroy_application(struct kutf_application *app); + +/* ============================================================================ + Suite functions +============================================================================ */ + +/** + * kutf_create_suite() - Create a kernel test suite. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * + * Suite names must be unique. Should two suites with the same name be + * registered with the same application then this function will fail, if they + * are registered with different applications then the function will not detect + * this and the call will succeed. + * + * Return: pointer to the created kutf_suite on success or NULL + * on failure + */ +struct kutf_suite *kutf_create_suite( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context)); + +/** + * kutf_create_suite_with_filters() - Create a kernel test suite with user + * defined default filters. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * @filters: Filters to apply to a test if it doesn't provide its own + * + * Suite names must be unique. Should two suites with the same name be + * registered with the same application then this function will fail, if they + * are registered with different applications then the function will not detect + * this and the call will succeed. + * + * Return: pointer to the created kutf_suite on success or NULL on failure + */ +struct kutf_suite *kutf_create_suite_with_filters( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters); + +/** + * kutf_create_suite_with_filters_and_data() - Create a kernel test suite with + * user defined default filters. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * @filters: Filters to apply to a test if it doesn't provide its own + * @suite_data: Suite specific callback data, provided during the + * running of the test in the kutf_context + * + * Return: pointer to the created kutf_suite on success or NULL + * on failure + */ +struct kutf_suite *kutf_create_suite_with_filters_and_data( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data suite_data); + +/** + * kutf_add_test() - Add a test to a kernel test suite. + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * + * Note: As no filters are provided the test will use the suite filters instead + */ +void kutf_add_test(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context)); + +/** + * kutf_add_test_with_filters() - Add a test to a kernel test suite with filters + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * @filters: A set of filtering flags, assigning test categories. + */ +void kutf_add_test_with_filters(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters); + +/** + * kutf_add_test_with_filters_and_data() - Add a test to a kernel test suite + * with filters. + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * @filters: A set of filtering flags, assigning test categories. + * @test_data: Test specific callback data, provided during the + * running of the test in the kutf_context + */ +void kutf_add_test_with_filters_and_data( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data test_data); + +/** + * kutf_add_test_with_filters_data_and_userdata() - Add a test to a kernel test suite with filters and setup for + * receiving data from userside + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * @filters: A set of filtering flags, assigning test categories. + * @test_data: Test specific callback data, provided during the + * running of the test in the kutf_context + * @userdata_ops: Callbacks to use for sending and receiving data to + * userspace. A copy of the struct kutf_userdata_ops is + * taken. Each callback can be NULL. + * + */ +void kutf_add_test_with_filters_data_and_userdata( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data test_data, + struct kutf_userdata_ops *userdata_ops); + + +/* ============================================================================ + Test functions +============================================================================ */ +/** + * kutf_test_log_result_external() - Log a result which has been created + * externally into a in a standard form + * recognized by the log parser. + * @context: The test context the test is running in + * @message: The message for this result + * @new_status: The result status of this log message + */ +void kutf_test_log_result_external( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status); + +/** + * kutf_test_expect_abort() - Tell the kernel that you expect the current + * fixture to produce an abort. + * @context: The test context this test is running in. + */ +void kutf_test_expect_abort(struct kutf_context *context); + +/** + * kutf_test_expect_fatal() - Tell the kernel that you expect the current + * fixture to produce a fatal error. + * @context: The test context this test is running in. + */ +void kutf_test_expect_fatal(struct kutf_context *context); + +/** + * kutf_test_expect_fail() - Tell the kernel that you expect the current + * fixture to fail. + * @context: The test context this test is running in. + */ +void kutf_test_expect_fail(struct kutf_context *context); + +/** + * kutf_test_expect_warn() - Tell the kernel that you expect the current + * fixture to produce a warning. + * @context: The test context this test is running in. + */ +void kutf_test_expect_warn(struct kutf_context *context); + +/** + * kutf_test_expect_pass() - Tell the kernel that you expect the current + * fixture to pass. + * @context: The test context this test is running in. + */ +void kutf_test_expect_pass(struct kutf_context *context); + +/** + * kutf_test_skip() - Tell the kernel that the test should be skipped. + * @context: The test context this test is running in. + */ +void kutf_test_skip(struct kutf_context *context); + +/** + * kutf_test_skip_msg() - Tell the kernel that this test has been skipped, + * supplying a reason string. + * @context: The test context this test is running in. + * @message: A message string containing the reason for the skip. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a prebaked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_skip_msg(struct kutf_context *context, const char *message); + +/** + * kutf_test_pass() - Tell the kernel that this test has passed. + * @context: The test context this test is running in. + * @message: A message string containing the reason for the pass. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_pass(struct kutf_context *context, char const *message); + +/** + * kutf_test_debug() - Send a debug message + * @context: The test context this test is running in. + * @message: A message string containing the debug information. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_debug(struct kutf_context *context, char const *message); + +/** + * kutf_test_info() - Send an information message + * @context: The test context this test is running in. + * @message: A message string containing the information message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_info(struct kutf_context *context, char const *message); + +/** + * kutf_test_warn() - Send a warning message + * @context: The test context this test is running in. + * @message: A message string containing the warning message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_warn(struct kutf_context *context, char const *message); + +/** + * kutf_test_fail() - Tell the kernel that a test has failed + * @context: The test context this test is running in. + * @message: A message string containing the failure message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_fail(struct kutf_context *context, char const *message); + +/** + * kutf_test_fatal() - Tell the kernel that a test has triggered a fatal error + * @context: The test context this test is running in. + * @message: A message string containing the fatal error message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_fatal(struct kutf_context *context, char const *message); + +/** + * kutf_test_abort() - Tell the kernel that a test triggered an abort in the test + * + * @context: The test context this test is running in. + */ +void kutf_test_abort(struct kutf_context *context); + +#endif /* _KERNEL_UTF_SUITE_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_utils.h b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_utils.h new file mode 100755 index 000000000000..c458c1f73802 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/include/kutf/kutf_utils.h @@ -0,0 +1,55 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_UTILS_H_ +#define _KERNEL_UTF_UTILS_H_ + +/* kutf_utils.h + * Utilities for the kernel UTF test infrastructure. + * + * This collection of library functions are provided for use by kernel UTF + * and users of kernel UTF which don't directly fit within the other + * code modules. + */ + +#include + +/** + * Maximum size of the message strings within kernel UTF, messages longer then + * this will be truncated. + */ +#define KUTF_MAX_DSPRINTF_LEN 1024 + +/** + * kutf_dsprintf() - dynamic sprintf + * @pool: memory pool to allocate from + * @fmt: The format string describing the string to document. + * @... The parameters to feed in to the format string. + * + * This function implements sprintf which dynamically allocates memory to store + * the string. The library will free the memory containing the string when the + * result set is cleared or destroyed. + * + * Note The returned string may be truncated to fit an internal temporary + * buffer, which is KUTF_MAX_DSPRINTF_LEN bytes in length. + * + * Return: Returns pointer to allocated string, or NULL on error. + */ +const char *kutf_dsprintf(struct kutf_mempool *pool, + const char *fmt, ...); + +#endif /* _KERNEL_UTF_UTILS_H_ */ diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kbuild b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kbuild new file mode 100755 index 000000000000..97f80057224f --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kbuild @@ -0,0 +1,20 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +ccflags-y += -I$(src)/../include + +obj-$(CONFIG_MALI_KUTF) += kutf.o + +kutf-y := kutf_mem.o kutf_resultset.o kutf_suite.o kutf_utils.o kutf_helpers.o kutf_helpers_user.o diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kconfig b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kconfig new file mode 100755 index 000000000000..6a87bdbf746e --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Kconfig @@ -0,0 +1,22 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + +config MALI_KUTF + tristate "Mali Kernel Unit Test Framework" + default m + help + Enables MALI testing framework. To compile it as a module, + choose M here - this will generate a single module called kutf. diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Makefile b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Makefile new file mode 100755 index 000000000000..010c92ca39b9 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/Makefile @@ -0,0 +1,29 @@ +# +# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +# linux build system bootstrap for out-of-tree module + +# default to building for the host +ARCH ?= $(shell uname -m) + +ifeq ($(KDIR),) +$(error Must specify KDIR to point to the kernel to target)) +endif + +all: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS=-I$(CURDIR)/../include modules + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers.c new file mode 100755 index 000000000000..793d58c789ff --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers.c @@ -0,0 +1,768 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF test helpers */ +#include + +/* 10s timeout for user thread to open the 'data' file once the test is started */ +#define USERDATA_WAIT_TIMEOUT_MS 10000 +#include +#include +#include +#include +#include +#include + + +int kutf_helper_textbuf_init(struct kutf_helper_textbuf *textbuf, + struct kutf_mempool *mempool, int max_line_size, + int max_nr_lines) +{ + textbuf->scratchpad = kutf_mempool_alloc(mempool, max_line_size); + + if (!textbuf->scratchpad) + return -ENOMEM; + + mutex_init(&textbuf->lock); + textbuf->nr_user_clients = 0; + textbuf->mempool = mempool; + textbuf->used_bytes = 0; + textbuf->prev_pos = 0; + textbuf->prev_line_pos = 0; + INIT_LIST_HEAD(&textbuf->textbuf_list); + textbuf->max_line_size = max_line_size; + textbuf->max_nr_lines = max_nr_lines; + textbuf->nr_lines = 0; + textbuf->flags = 0ul; + init_waitqueue_head(&textbuf->user_opened_wq); + init_waitqueue_head(&textbuf->not_full_wq); + init_waitqueue_head(&textbuf->not_empty_wq); + + return 0; +} +EXPORT_SYMBOL(kutf_helper_textbuf_init); + +/** + * kutf_helper_textbuf_open() - Notify that userspace has opened the 'data' + * file for a textbuf + * + * @priv: private pointer from a kutf_userdata_exchange, which + * should be a pointer to a struct kutf_helper_textbuf + * + * Return: 0 on success, or negative value on error. + */ +static int kutf_helper_textbuf_open(void *priv) +{ + struct kutf_helper_textbuf *textbuf = priv; + int ret; + + ret = mutex_lock_interruptible(&textbuf->lock); + if (ret) + return -ERESTARTSYS; + + ++(textbuf->nr_user_clients); + wake_up(&textbuf->user_opened_wq); + + mutex_unlock(&textbuf->lock); + return ret; +} + +/** + * kutf_helper_textbuf_release() - Notify that userspace has closed the 'data' + * file for a textbuf + * + * @priv: private pointer from a kutf_userdata_exchange, which + * should be a pointer to a struct kutf_helper_textbuf + */ +static void kutf_helper_textbuf_release(void *priv) +{ + struct kutf_helper_textbuf *textbuf = priv; + + /* Shouldn't use interruptible variants here because if a signal is + * pending, we can't abort and restart the call */ + mutex_lock(&textbuf->lock); + + --(textbuf->nr_user_clients); + if (!textbuf->nr_user_clients) { + /* All clients disconnected, wakeup kernel-side waiters */ + wake_up(&textbuf->not_full_wq); + wake_up(&textbuf->not_empty_wq); + } + + mutex_unlock(&textbuf->lock); +} + +/** + * kutf_helper_textbuf_notify_test_ended() - Notify that the test has ended + * + * @priv: private pointer from a kutf_userdata_exchange, which + * should be a pointer to a struct kutf_helper_textbuf + * + * After this call, userspace should be allowed to finish remaining reads but + * not make new ones, and not be allowed to make new writes. + */ +static void kutf_helper_textbuf_notify_test_ended(void *priv) +{ + struct kutf_helper_textbuf *textbuf = priv; + + /* Shouldn't use interruptible variants here because if a signal is + * pending, we can't abort and restart the call */ + mutex_lock(&textbuf->lock); + + textbuf->flags |= KUTF_HELPER_TEXTBUF_FLAG_DYING; + + /* Consumers waiting due to being full should wake up and abort */ + wake_up(&textbuf->not_full_wq); + /* Producers waiting due to being empty should wake up and abort */ + wake_up(&textbuf->not_empty_wq); + + mutex_unlock(&textbuf->lock); +} + +/* Collect text in a textbuf scratchpad up to (but excluding) specified + * newline_off, and add it as a textbuf_line + * + * newline_off is permissible to be at the character after the end of the + * scratchpad (i.e. equal to textbuf->max_line_size), for handling when the + * line was longer than the size of the scratchpad. Nevertheless, the resulting + * size of the line is kept at textbuf->max_line_size, including the '\0' + * terminator. That is, the string length will be textbuf->max_line_size-1. + * + * Remaining characters strictly after newline_off are moved to the beginning + * of the scratchpad, to allow space for a longer line to be collected. This + * means the character specified at newline_off will be removed from/no longer + * be within the valid region of the scratchpad + * + * Returns number of bytes the scratchpad was shortened by, or an error + * otherwise + */ +static size_t collect_line(struct kutf_helper_textbuf *textbuf, int newline_off) +{ + /* '\n' terminator will be replaced as '\0' */ + int str_buf_size; + struct kutf_helper_textbuf_line *textbuf_line; + char *str_start; + int bytes_remain; + char *scratch = textbuf->scratchpad; + int nextline_off; + + str_buf_size = newline_off + 1; + if (str_buf_size > textbuf->max_line_size) + str_buf_size = textbuf->max_line_size; + + /* String is stored immediately after the line */ + textbuf_line = kutf_mempool_alloc(textbuf->mempool, str_buf_size + sizeof(struct kutf_helper_textbuf_line)); + if (!textbuf_line) + return -ENOMEM; + + str_start = &textbuf_line->str[0]; + + /* Copy in string, excluding the terminating '\n' character, replacing + * it with '\0' */ + strncpy(str_start, scratch, str_buf_size - 1); + str_start[str_buf_size-1] = '\0'; + textbuf_line->str_size = str_buf_size; + + /* Append to the textbuf */ + list_add_tail(&textbuf_line->node, &textbuf->textbuf_list); + ++(textbuf->nr_lines); + + /* Move the rest of the scratchpad to the start */ + nextline_off = newline_off + 1; + if (nextline_off > textbuf->used_bytes) + nextline_off = textbuf->used_bytes; + + bytes_remain = textbuf->used_bytes - nextline_off; + memmove(scratch, scratch + nextline_off, bytes_remain); + textbuf->used_bytes = bytes_remain; + + /* Wakeup anyone blocked on empty */ + wake_up(&textbuf->not_empty_wq); + + return nextline_off; +} + +/* Buffer size for truncating a string to its newline. + * Allocated on the stack, so keep it moderately small (within PAGE_SIZE) */ +#define TRUNCATE_BUF_SZ 512 + +/* Discard input from a userbuf up to a newline, then collect what was in the + * scratchpad into a new textbuf line */ +static ssize_t collect_longline_truncate(struct kutf_helper_textbuf *textbuf, + const char __user *userbuf, size_t userbuf_len) +{ + ssize_t bytes_processed = 0; + + while (userbuf_len > 0) { + int userbuf_copy_sz = userbuf_len; + size_t res; + char *newline_ptr; + char truncate_buf[TRUNCATE_BUF_SZ]; + + if (userbuf_len > TRUNCATE_BUF_SZ) + userbuf_copy_sz = TRUNCATE_BUF_SZ; + else + userbuf_copy_sz = (int)userbuf_len; + + /* copy what we can */ + res = copy_from_user(truncate_buf, userbuf, userbuf_copy_sz); + if (res == userbuf_copy_sz) + return -EFAULT; + userbuf_copy_sz -= res; + + /* Search for newline in what was copied */ + newline_ptr = strnchr(truncate_buf, userbuf_copy_sz, '\n'); + + if (newline_ptr) { + ssize_t sres; + /* Newline found: collect scratchpad and exit out */ + int newline_off = newline_ptr - truncate_buf; + + sres = collect_line(textbuf, textbuf->used_bytes); + if (sres < 0) + return sres; + + bytes_processed += newline_off + 1; + break; + } + + /* Newline not yet found: advance to the next part to copy */ + userbuf += userbuf_copy_sz; + userbuf_len -= userbuf_copy_sz; + bytes_processed += userbuf_copy_sz; + } + + return bytes_processed; +} + +/** + * kutf_helper_textbuf_consume() - 'data' file consumer function for writing to + * a textbuf + * @priv: private pointer from a kutf_userdata_exchange, which + * should be a pointer to a struct kutf_helper_textbuf to + * write into + * @userbuf: the userspace buffer to read from + * @userbuf_len: size of the userspace buffer + * @ppos: the current position in the buffer + * + * This consumer function is used as a write consumer for the 'data' file, + * receiving data that has been written to the 'data' file by userspace. It + * will read from the userspace buffer @userbuf and separates it into '\n' + * delimited lines for the textbuf pointed to by @priv . + * + * If there is insufficient space in textbuf, then it will block until there is + * space - for example, a kernel-side test calls + * kutf_helper_textbuf_dequeue(). Since this is expected to be called in the + * context of a syscall, the call can only be cancelled by sending an + * appropriate signal to the userspace process. + * + * The current position @ppos is advanced by the number of bytes successfully + * read. + * + * Return: the number of bytes read, or negative value on error. + */ +static ssize_t kutf_helper_textbuf_consume(void *priv, + const char __user *userbuf, size_t userbuf_len, loff_t *ppos) +{ + struct kutf_helper_textbuf *textbuf = priv; + int userbuf_copy_sz; + char *next_newline_ptr; + size_t bytes_processed = 0; + int newdata_off; + ssize_t ret; + + ret = mutex_lock_interruptible(&textbuf->lock); + if (ret) + return -ERESTARTSYS; + + /* Validate input */ + if (*ppos < 0) { + ret = -EINVAL; + goto out_unlock; + } + if (!userbuf_len) { + ret = 0; + goto out_unlock; + } + + while (textbuf->nr_lines >= textbuf->max_nr_lines && + !(textbuf->flags & KUTF_HELPER_TEXTBUF_FLAG_DYING)) { + /* Block on kernel-side dequeue making space available + * NOTE: should also handle O_NONBLOCK */ + mutex_unlock(&textbuf->lock); + ret = wait_event_interruptible(textbuf->not_full_wq, + (textbuf->nr_lines < textbuf->max_nr_lines || + (textbuf->flags & KUTF_HELPER_TEXTBUF_FLAG_DYING))); + if (ret) + return -ERESTARTSYS; + ret = mutex_lock_interruptible(&textbuf->lock); + if (ret) + return -ERESTARTSYS; + } + + if (textbuf->flags & KUTF_HELPER_TEXTBUF_FLAG_DYING) { + ret = -ENODEV; + goto out_unlock; + } + + if (textbuf->prev_pos != *ppos && textbuf->used_bytes) { + /* Seeking causes a new line to occur: + * Truncate what data was there into a textbuf-line, and reset + * the buffer */ + ret = collect_line(textbuf, textbuf->used_bytes); + if (ret < 0) + goto finish; + } else if (textbuf->used_bytes >= (textbuf->max_line_size - 1)) { + /* Line too long discard input until we find a '\n' */ + ret = collect_longline_truncate(textbuf, userbuf, userbuf_len); + + if (ret < 0) + goto finish; + + /* Update userbuf with how much was processed, which may be the + * entire buffer now */ + userbuf += ret; + userbuf_len -= ret; + bytes_processed += ret; + + /* If there's buffer remaining and we fault later (e.g. can't + * read or OOM) ensure ppos is updated */ + *ppos += ret; + + /* recheck in case entire buffer processed */ + if (!userbuf_len) + goto finish; + } + + /* An extra line may've been added, ensure we don't overfill */ + if (textbuf->nr_lines >= textbuf->max_nr_lines) + goto finish_noerr; + + userbuf_copy_sz = userbuf_len; + + /* Copy in as much as we can */ + if (userbuf_copy_sz > textbuf->max_line_size - textbuf->used_bytes) + userbuf_copy_sz = textbuf->max_line_size - textbuf->used_bytes; + + ret = copy_from_user(textbuf->scratchpad + textbuf->used_bytes, userbuf, userbuf_copy_sz); + if (ret == userbuf_copy_sz) { + ret = -EFAULT; + goto finish; + } + userbuf_copy_sz -= ret; + + newdata_off = textbuf->used_bytes; + textbuf->used_bytes += userbuf_copy_sz; + + while (textbuf->used_bytes && textbuf->nr_lines < textbuf->max_nr_lines) { + int new_bytes_remain = textbuf->used_bytes - newdata_off; + /* Find a new line - only the new part should be checked */ + next_newline_ptr = strnchr(textbuf->scratchpad + newdata_off, new_bytes_remain, '\n'); + + if (next_newline_ptr) { + int newline_off = next_newline_ptr - textbuf->scratchpad; + + /* if found, collect up to it, then memmove the rest */ + /* reset positions and see if we can fill any further */ + /* repeat until run out of data or line is filled */ + ret = collect_line(textbuf, newline_off); + + /* If filled up or OOM, rollback the remaining new + * data. Instead we'll try to grab it next time we're + * called */ + if (textbuf->nr_lines >= textbuf->max_nr_lines || ret < 0) + textbuf->used_bytes = newdata_off; + + if (ret < 0) + goto finish; + + /* Fix up ppos etc in case we'll be ending the loop */ + *ppos += ret - newdata_off; + bytes_processed += ret - newdata_off; + newdata_off = 0; + } else { + /* there's bytes left, but no new-line, so try to fill up next time */ + *ppos += new_bytes_remain; + bytes_processed += new_bytes_remain; + break; + } + } + +finish_noerr: + ret = bytes_processed; +finish: + textbuf->prev_pos = *ppos; +out_unlock: + mutex_unlock(&textbuf->lock); + + return ret; +} + +/** + * kutf_helper_textbuf_produce() - 'data' file producer function for reading + * from a textbuf + * @priv: private pointer from a kutf_userdata_exchange, which + * should be a pointer to a struct kutf_helper_textbuf to + * read from + * @userbuf: the userspace buffer to write to + * @userbuf_len: size of the userspace buffer + * @ppos: the current position in the buffer + * + * This producer function is used as a read producer for the 'data' file, + * allowing userspace to read from the 'data' file. It will write to the + * userspace buffer @userbuf, taking lines from the textbuf pointed to by + * @priv, separating each line with '\n'. + * + * If there is no data in the textbuf, then it will block until some appears - + * for example, a kernel-side test calls kutf_helper_textbuf_enqueue(). Since + * this is expected to be called in the context of a syscall, the call can only + * be cancelled by sending an appropriate signal to the userspace process. + * + * The current position @ppos is advanced by the number of bytes successfully + * written. + * + * Return: the number of bytes written, or negative value on error + */ +static ssize_t kutf_helper_textbuf_produce(void *priv, char __user *userbuf, + size_t userbuf_len, loff_t *ppos) +{ + struct kutf_helper_textbuf *textbuf = priv; + loff_t pos_offset; + struct kutf_helper_textbuf_line *line = NULL; + int line_start_pos; + size_t bytes_processed = 0; + ssize_t ret; + int copy_length; + + ret = mutex_lock_interruptible(&textbuf->lock); + if (ret) + return -ERESTARTSYS; + + /* Validate input */ + if (*ppos < 0) { + ret = -EINVAL; + goto finish; + } + if (!userbuf_len) { + ret = 0; + goto finish; + } + + /* Seeking to before the beginning of the line will have the effect of + * resetting the position to the start of the current data, since we've + * already discarded previous data */ + if (*ppos < textbuf->prev_line_pos) + textbuf->prev_line_pos = *ppos; + + while (!line) { + int needs_wake = 0; + + pos_offset = *ppos - textbuf->prev_line_pos; + line_start_pos = 0; + + /* Find the line for the offset, emptying the textbuf as we go */ + while (!list_empty(&textbuf->textbuf_list)) { + int line_end_pos; + + line = list_first_entry(&textbuf->textbuf_list, struct kutf_helper_textbuf_line, node); + + /* str_size used in line_end_pos because lines implicitly have + * a '\n', but we count the '\0' string terminator as that */ + line_end_pos = line_start_pos + line->str_size; + + if (pos_offset < line_end_pos) + break; + + line_start_pos += line->str_size; + /* Only discard a line when we're sure it's finished + * with, to avoid awkward rollback conditions if we've + * had to block */ + list_del(&line->node); + --(textbuf->nr_lines); + line = NULL; + needs_wake = 1; + } + + /* Update the start of the line pos for next time we're called */ + textbuf->prev_line_pos += line_start_pos; + + /* If space was freed up, wake waiters */ + if (needs_wake) + wake_up(&textbuf->not_full_wq); +; + if (!line) { + /* Only check before waiting, to ensure if the test + * does the last enqueue and immediately finishes, then + * we'll go back round the loop to receive the line + * instead of just dying straight away */ + if (textbuf->flags & KUTF_HELPER_TEXTBUF_FLAG_DYING) { + /* Indicate EOF rather than an error */ + ret = 0; + goto finish; + } + + /* No lines found, block for new ones + * NOTE: should also handle O_NONBLOCK */ + mutex_unlock(&textbuf->lock); + ret = wait_event_interruptible(textbuf->not_empty_wq, + (textbuf->nr_lines > 0 || + (textbuf->flags & KUTF_HELPER_TEXTBUF_FLAG_DYING))); + + /* signals here are not restartable */ + if (ret) + return ret; + ret = mutex_lock_interruptible(&textbuf->lock); + if (ret) + return ret; + } + + } + + + /* Find offset within the line, guaranteed to be within line->str_size */ + pos_offset -= line_start_pos; + + while (userbuf_len && line) { + /* Copy at most to the end of string, excluding terminator */ + copy_length = line->str_size - 1 - pos_offset; + if (copy_length > userbuf_len) + copy_length = userbuf_len; + + if (copy_length) { + ret = copy_to_user(userbuf, &line->str[pos_offset], copy_length); + if (ret == copy_length) { + ret = -EFAULT; + goto finish; + } + copy_length -= ret; + + userbuf += copy_length; + userbuf_len -= copy_length; + bytes_processed += copy_length; + *ppos += copy_length; + if (ret) + goto finish_noerr; + } + + /* Add terminator if one was needed */ + if (userbuf_len) { + copy_length = 1; + ret = copy_to_user(userbuf, "\n", copy_length); + if (ret == copy_length) { + ret = -EFAULT; + goto finish; + } + copy_length -= ret; + + userbuf += copy_length; + userbuf_len -= copy_length; + bytes_processed += copy_length; + *ppos += copy_length; + } else { + /* string wasn't completely copied this time - try to + * finish it next call */ + break; + } + + /* Line Completed - only now can safely delete it */ + textbuf->prev_line_pos += line->str_size; + list_del(&line->node); + --(textbuf->nr_lines); + line = NULL; + /* Space freed up, wake up waiters */ + wake_up(&textbuf->not_full_wq); + + /* Pick the next line */ + if (!list_empty(&textbuf->textbuf_list)) { + line = list_first_entry(&textbuf->textbuf_list, struct kutf_helper_textbuf_line, node); + pos_offset = 0; + } + /* if no more lines, we've copied at least some bytes, so only + * need to block on new lines the next time we're called */ + } + +finish_noerr: + ret = bytes_processed; +finish: + mutex_unlock(&textbuf->lock); + + return ret; +} + +int kutf_helper_textbuf_wait_for_user(struct kutf_helper_textbuf *textbuf) +{ + int err; + unsigned long now; + unsigned long timeout_jiffies = msecs_to_jiffies(USERDATA_WAIT_TIMEOUT_MS); + unsigned long time_end; + int ret = 0; + + /* Mutex locking using non-interruptible variants, since a signal to + * the user process will generally have to wait until we finish the + * test, because we can't restart the test. The exception is where + * we're blocked on a waitq */ + mutex_lock(&textbuf->lock); + + now = jiffies; + time_end = now + timeout_jiffies; + + while (!textbuf->nr_user_clients && time_before_eq(now, time_end)) { + unsigned long time_to_wait = time_end - now; + /* No users yet, block or timeout */ + mutex_unlock(&textbuf->lock); + /* Use interruptible here - in case we block for a long time + * and want to kill the user process */ + err = wait_event_interruptible_timeout(textbuf->user_opened_wq, + (textbuf->nr_user_clients > 0), time_to_wait); + /* Any error is not restartable due to how kutf runs tests */ + if (err < 0) + return -EINTR; + mutex_lock(&textbuf->lock); + + now = jiffies; + } + if (!textbuf->nr_user_clients) + ret = -ETIMEDOUT; + + mutex_unlock(&textbuf->lock); + + return ret; +} +EXPORT_SYMBOL(kutf_helper_textbuf_wait_for_user); + +char *kutf_helper_textbuf_dequeue(struct kutf_helper_textbuf *textbuf, + int *str_size) +{ + struct kutf_helper_textbuf_line *line; + char *ret = NULL; + + /* Mutex locking using non-interruptible variants, since a signal to + * the user process will generally have to wait until we finish the + * test, because we can't restart the test. The exception is where + * we're blocked on a waitq */ + mutex_lock(&textbuf->lock); + + while (list_empty(&textbuf->textbuf_list)) { + int err; + + if (!textbuf->nr_user_clients) { + /* No user-side clients - error */ + goto out; + } + + /* No lines found, block for new ones from user-side consumer */ + mutex_unlock(&textbuf->lock); + /* Use interruptible here - in case we block for a long time + * and want to kill the user process */ + err = wait_event_interruptible(textbuf->not_empty_wq, + (textbuf->nr_lines > 0 || !textbuf->nr_user_clients)); + /* Any error is not restartable due to how kutf runs tests */ + if (err) + return ERR_PTR(-EINTR); + mutex_lock(&textbuf->lock); + } + + line = list_first_entry(&textbuf->textbuf_list, struct kutf_helper_textbuf_line, node); + list_del(&line->node); + --(textbuf->nr_lines); + /* Space freed up, wake up waiters */ + wake_up(&textbuf->not_full_wq); + + if (str_size) + *str_size = line->str_size; + + ret = &line->str[0]; + +out: + mutex_unlock(&textbuf->lock); + return ret; +} +EXPORT_SYMBOL(kutf_helper_textbuf_dequeue); + +int kutf_helper_textbuf_enqueue(struct kutf_helper_textbuf *textbuf, + char *enqueue_str, int buf_max_size) +{ + struct kutf_helper_textbuf_line *textbuf_line; + int str_size = strnlen(enqueue_str, buf_max_size) + 1; + char *str_start; + int ret = 0; + + /* Mutex locking using non-interruptible variants, since a signal to + * the user process will generally have to wait until we finish the + * test, because we can't restart the test. The exception is where + * we're blocked on a waitq */ + mutex_lock(&textbuf->lock); + + if (str_size > textbuf->max_line_size) + str_size = textbuf->max_line_size; + + while (textbuf->nr_lines >= textbuf->max_nr_lines) { + if (!textbuf->nr_user_clients) { + /* No user-side clients - error */ + ret = -EBUSY; + goto out; + } + + /* Block on user-side producer making space available */ + mutex_unlock(&textbuf->lock); + /* Use interruptible here - in case we block for a long time + * and want to kill the user process */ + ret = wait_event_interruptible(textbuf->not_full_wq, + (textbuf->nr_lines < textbuf->max_nr_lines || !textbuf->nr_user_clients)); + /* Any error is not restartable due to how kutf runs tests */ + if (ret) + return -EINTR; + mutex_lock(&textbuf->lock); + } + + /* String is stored immediately after the line */ + textbuf_line = kutf_mempool_alloc(textbuf->mempool, str_size + sizeof(struct kutf_helper_textbuf_line)); + if (!textbuf_line) { + ret = -ENOMEM; + goto out; + } + + str_start = &textbuf_line->str[0]; + + /* Copy in string */ + strncpy(str_start, enqueue_str, str_size); + /* Enforce the '\0' termination */ + str_start[str_size-1] = '\0'; + textbuf_line->str_size = str_size; + + /* Append to the textbuf */ + list_add_tail(&textbuf_line->node, &textbuf->textbuf_list); + ++(textbuf->nr_lines); + + /* Wakeup anyone blocked on empty */ + wake_up(&textbuf->not_empty_wq); + +out: + mutex_unlock(&textbuf->lock); + return ret; +} +EXPORT_SYMBOL(kutf_helper_textbuf_enqueue); + + +struct kutf_userdata_ops kutf_helper_textbuf_userdata_ops = { + .open = kutf_helper_textbuf_open, + .release = kutf_helper_textbuf_release, + .notify_ended = kutf_helper_textbuf_notify_test_ended, + .consumer = kutf_helper_textbuf_consume, + .producer = kutf_helper_textbuf_produce, +}; +EXPORT_SYMBOL(kutf_helper_textbuf_userdata_ops); diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers_user.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers_user.c new file mode 100755 index 000000000000..cf3b00563c5f --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_helpers_user.c @@ -0,0 +1,460 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF test helpers that mirror those for kutf-userside */ +#include +#include + +#include +#include + +const char *valtype_names[] = { + "INVALID", + "U64", + "STR", +}; + +static const char *get_val_type_name(enum kutf_helper_valtype valtype) +{ + /* enums can be signed or unsigned (implementation dependant), so + * enforce it to prevent: + * a) "<0 comparison on unsigned type" warning - if we did both upper + * and lower bound check + * b) incorrect range checking if it was a signed type - if we did + * upper bound check only */ + unsigned int type_idx = (unsigned int)valtype; + + if (type_idx >= (unsigned int)KUTF_HELPER_VALTYPE_COUNT) + type_idx = (unsigned int)KUTF_HELPER_VALTYPE_INVALID; + + return valtype_names[type_idx]; +} + +/* Check up to str_len chars of val_str to see if it's a valid value name: + * + * - Has between 1 and KUTF_HELPER_MAX_VAL_NAME_LEN characters before the \0 terminator + * - And, each char is in the character set [A-Z0-9_] */ +static int validate_val_name(char *val_str, int str_len) +{ + int i = 0; + + for (i = 0; str_len && i <= KUTF_HELPER_MAX_VAL_NAME_LEN && val_str[i] != '\0'; ++i, --str_len) { + char val_chr = val_str[i]; + + if (val_chr >= 'A' && val_chr <= 'Z') + continue; + if (val_chr >= '0' && val_chr <= '9') + continue; + if (val_chr == '_') + continue; + + /* Character not in the set [A-Z0-9_] - report error */ + return 1; + } + + /* Names of 0 length are not valid */ + if (i == 0) + return 1; + /* Length greater than KUTF_HELPER_MAX_VAL_NAME_LEN not allowed */ + if (i > KUTF_HELPER_MAX_VAL_NAME_LEN || (i == KUTF_HELPER_MAX_VAL_NAME_LEN && val_str[i] != '\0')) + return 1; + + return 0; +} + +/* Find the length of the valid part of the string when it will be in quotes + * e.g. "str" + * + * That is, before any '\\', '\n' or '"' characters. This is so we don't have + * to escape the string */ +static int find_quoted_string_valid_len(char *str) +{ + char *ptr; + const char *check_chars = "\\\n\""; + + ptr = strpbrk(str, check_chars); + if (ptr) + return ptr-str; + + return strlen(str); +} + +#define MAX_U64_HEX_LEN 16 +/* (Name size) + ("=0x" size) + (64-bit hex value size) + (terminator) */ +#define NAMED_U64_VAL_BUF_SZ (KUTF_HELPER_MAX_VAL_NAME_LEN + 3 + MAX_U64_HEX_LEN + 1) + +int kutf_helper_textbuf_send_named_u64(struct kutf_context *context, + struct kutf_helper_textbuf *textbuf, char *val_name, u64 val) +{ + int ret = 1; + char msgbuf[NAMED_U64_VAL_BUF_SZ]; + const char *errmsg = NULL; + + if (validate_val_name(val_name, KUTF_HELPER_MAX_VAL_NAME_LEN + 1)) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send u64 value named '%s': Invalid value name", val_name); + goto out_err; + } + + ret = snprintf(msgbuf, NAMED_U64_VAL_BUF_SZ, "%s=0x%llx", val_name, val); + if (ret >= NAMED_U64_VAL_BUF_SZ || ret < 0) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send u64 value named '%s': snprintf() problem buffer size==%d ret=%d", + val_name, NAMED_U64_VAL_BUF_SZ, ret); + goto out_err; + } + msgbuf[NAMED_U64_VAL_BUF_SZ-1] = '\0'; + + ret = kutf_helper_textbuf_enqueue(textbuf, msgbuf, NAMED_U64_VAL_BUF_SZ); + if (ret) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send u64 value named '%s': send returned %d", + val_name, ret); + goto out_err; + } + + return ret; +out_err: + kutf_test_fail(context, errmsg); + return ret; +} +EXPORT_SYMBOL(kutf_helper_textbuf_send_named_u64); + +#define NAMED_VALUE_SEP "=" +#define NAMED_STR_START_DELIM NAMED_VALUE_SEP "\"" +#define NAMED_STR_END_DELIM "\"" + +int kutf_helper_textbuf_max_str_len_for_kern(char *val_name, + int kern_buf_sz) +{ + int val_name_len = strlen(val_name); + int start_delim_len = strlen(NAMED_STR_START_DELIM); + int max_msg_len = kern_buf_sz - 1; + int max_str_len; + + /* We do not include the end delimiter. Providing there is a line + * ending character when sending the message, the end delimiter can be + * truncated off safely to allow proper NAME="value" reception when + * value's length is too long */ + max_str_len = max_msg_len - val_name_len - start_delim_len; + + return max_str_len; +} +EXPORT_SYMBOL(kutf_helper_textbuf_max_str_len_for_kern); + +int kutf_helper_textbuf_send_named_str(struct kutf_context *context, + struct kutf_helper_textbuf *textbuf, char *val_name, + char *val_str) +{ + int val_str_len; + int str_buf_sz; + char *str_buf = NULL; + int ret = 1; + char *copy_ptr; + int val_name_len; + int start_delim_len = strlen(NAMED_STR_START_DELIM); + int end_delim_len = strlen(NAMED_STR_END_DELIM); + const char *errmsg = NULL; + + if (validate_val_name(val_name, KUTF_HELPER_MAX_VAL_NAME_LEN + 1)) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send u64 value named '%s': Invalid value name", val_name); + goto out_err; + } + val_name_len = strlen(val_name); + + val_str_len = find_quoted_string_valid_len(val_str); + + /* (name length) + ("=\"" length) + (val_str len) + ("\"" length) + terminator */ + str_buf_sz = val_name_len + start_delim_len + val_str_len + end_delim_len + 1; + + /* Using kmalloc() here instead of mempool since we know we need to free + * before we return */ + str_buf = kmalloc(str_buf_sz, GFP_KERNEL); + if (!str_buf) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send str value named '%s': kmalloc failed, str_buf_sz=%d", + val_name, str_buf_sz); + goto out_err; + } + copy_ptr = str_buf; + + /* Manually copy each string component instead of snprintf because + * val_str may need to end early, and less error path handling */ + + /* name */ + memcpy(copy_ptr, val_name, val_name_len); + copy_ptr += val_name_len; + + /* str start delimiter */ + memcpy(copy_ptr, NAMED_STR_START_DELIM, start_delim_len); + copy_ptr += start_delim_len; + + /* str value */ + memcpy(copy_ptr, val_str, val_str_len); + copy_ptr += val_str_len; + + /* str end delimiter */ + memcpy(copy_ptr, NAMED_STR_END_DELIM, end_delim_len); + copy_ptr += end_delim_len; + + /* Terminator */ + *copy_ptr = '\0'; + + ret = kutf_helper_textbuf_enqueue(textbuf, str_buf, str_buf_sz); + + if (ret) { + errmsg = kutf_dsprintf(&context->fixture_pool, + "Failed to send str value named '%s': send returned %d", + val_name, ret); + goto out_err; + } + + kfree(str_buf); + return ret; + +out_err: + kutf_test_fail(context, errmsg); + kfree(str_buf); + return ret; +} +EXPORT_SYMBOL(kutf_helper_textbuf_send_named_str); + +int kutf_helper_textbuf_receive_named_val(struct kutf_helper_named_val *named_val, + struct kutf_helper_textbuf *textbuf) +{ + int recv_sz; + char *recv_str; + char *search_ptr; + char *name_str = NULL; + int name_len; + int strval_len; + enum kutf_helper_valtype type = KUTF_HELPER_VALTYPE_INVALID; + char *strval = NULL; + u64 u64val = 0; + int orig_recv_sz; + int err = KUTF_HELPER_ERR_INVALID_VALUE; + + recv_str = kutf_helper_textbuf_dequeue(textbuf, &recv_sz); + if (!recv_str) + return -EBUSY; + else if (IS_ERR(recv_str)) + return PTR_ERR(recv_str); + orig_recv_sz = recv_sz; + + /* Find the '=', grab the name and validate it */ + search_ptr = strnchr(recv_str, recv_sz, NAMED_VALUE_SEP[0]); + if (search_ptr) { + name_len = search_ptr - recv_str; + if (!validate_val_name(recv_str, name_len)) { + /* no need to reallocate - just modify string in place */ + name_str = recv_str; + name_str[name_len] = '\0'; + + /* Move until after the '=' */ + recv_str += (name_len + 1); + recv_sz -= (name_len + 1); + } + } + if (!name_str) { + pr_err("Invalid name part for recevied string '%s'\n", recv_str); + return KUTF_HELPER_ERR_INVALID_NAME; + } + + /* detect value type */ + if (*recv_str == NAMED_STR_START_DELIM[1]) { + /* string delimiter start*/ + ++recv_str; + --recv_sz; + + /* Find end of string */ + search_ptr = strnchr(recv_str, recv_sz, NAMED_STR_END_DELIM[0]); + if (search_ptr) { + strval_len = search_ptr - recv_str; + /* Validate the string to ensure it contains no quotes */ + if (strval_len == find_quoted_string_valid_len(recv_str)) { + /* no need to reallocate - just modify string in place */ + strval = recv_str; + strval[strval_len] = '\0'; + + /* Move until after the end delimiter */ + recv_str += (strval_len + 1); + recv_sz -= (strval_len + 1); + type = KUTF_HELPER_VALTYPE_STR; + } else { + pr_err("String value contains invalid characters in rest of received string '%s'\n", recv_str); + err = KUTF_HELPER_ERR_CHARS_AFTER_VAL; + } + } else if (orig_recv_sz == textbuf->max_line_size) { + /* No end-delimiter found, but the line is at + * the max line size. Assume that before + * truncation the line had a closing delimiter + * anyway */ + strval_len = strlen(recv_str); + /* Validate the string to ensure it contains no quotes */ + if (strval_len == find_quoted_string_valid_len(recv_str)) { + strval = recv_str; + + /* Move to the end of the string */ + recv_str += strval_len; + recv_sz -= strval_len; + type = KUTF_HELPER_VALTYPE_STR; + } else { + pr_err("String value contains invalid characters in rest of received string '%s'\n", recv_str); + err = KUTF_HELPER_ERR_CHARS_AFTER_VAL; + } + } else { + pr_err("End of string delimiter not found in rest of received string '%s'\n", recv_str); + err = KUTF_HELPER_ERR_NO_END_DELIMITER; + } + } else { + /* possibly a number value - strtoull will parse it */ + err = kstrtoull(recv_str, 0, &u64val); + /* unlike userspace can't get an end ptr, but if kstrtoull() + * reads characters after the number it'll report -EINVAL */ + if (!err) { + int len_remain = strnlen(recv_str, recv_sz); + + type = KUTF_HELPER_VALTYPE_U64; + recv_str += len_remain; + recv_sz -= len_remain; + } else { + /* special case: not a number, report as such */ + pr_err("Rest of received string was not a numeric value or quoted string value: '%s'\n", recv_str); + } + } + + if (type == KUTF_HELPER_VALTYPE_INVALID) + return err; + + /* Any remaining characters - error */ + if (strnlen(recv_str, recv_sz) != 0) { + pr_err("Characters remain after value of type %s: '%s'\n", + get_val_type_name(type), recv_str); + return KUTF_HELPER_ERR_CHARS_AFTER_VAL; + } + + /* Success - write into the output structure */ + switch (type) { + case KUTF_HELPER_VALTYPE_U64: + named_val->u.val_u64 = u64val; + break; + case KUTF_HELPER_VALTYPE_STR: + named_val->u.val_str = strval; + break; + default: + pr_err("Unreachable, fix textbuf_receive_named_val\n"); + /* Coding error, report as though 'data' file failed */ + return -EINVAL; + } + + named_val->val_name = name_str; + named_val->type = type; + + return KUTF_HELPER_ERR_NONE; +} +EXPORT_SYMBOL(kutf_helper_textbuf_receive_named_val); + +#define DUMMY_MSG "" +int kutf_helper_textbuf_receive_check_val(struct kutf_helper_named_val *named_val, + struct kutf_context *context, struct kutf_helper_textbuf *textbuf, + char *expect_val_name, enum kutf_helper_valtype expect_val_type) +{ + int err; + + err = kutf_helper_textbuf_receive_named_val(named_val, textbuf); + if (err < 0) { + const char *msg = kutf_dsprintf(&context->fixture_pool, + "Failed to receive value named '%s'", + expect_val_name); + kutf_test_fail(context, msg); + return err; + } else if (err > 0) { + const char *msg = kutf_dsprintf(&context->fixture_pool, + "Named-value parse error when expecting value named '%s'", + expect_val_name); + kutf_test_fail(context, msg); + goto out_fail_and_fixup; + } + + if (strcmp(named_val->val_name, expect_val_name) != 0) { + const char *msg = kutf_dsprintf(&context->fixture_pool, + "Expecting to receive value named '%s' but got '%s'", + expect_val_name, named_val->val_name); + kutf_test_fail(context, msg); + goto out_fail_and_fixup; + } + + + if (named_val->type != expect_val_type) { + const char *msg = kutf_dsprintf(&context->fixture_pool, + "Expecting value named '%s' to be of type %s but got %s", + expect_val_name, get_val_type_name(expect_val_type), + get_val_type_name(named_val->type)); + kutf_test_fail(context, msg); + goto out_fail_and_fixup; + } + + return err; + +out_fail_and_fixup: + /* Produce a valid but incorrect value */ + switch (expect_val_type) { + case KUTF_HELPER_VALTYPE_U64: + named_val->u.val_u64 = 0ull; + break; + case KUTF_HELPER_VALTYPE_STR: + { + char *str = kutf_mempool_alloc(&context->fixture_pool, sizeof(DUMMY_MSG)); + + if (!str) + return -1; + + strcpy(str, DUMMY_MSG); + named_val->u.val_str = str; + break; + } + default: + break; + } + + /* Indicate that this is invalid */ + named_val->type = KUTF_HELPER_VALTYPE_INVALID; + + /* But at least allow the caller to continue in the test with failures */ + return 0; +} +EXPORT_SYMBOL(kutf_helper_textbuf_receive_check_val); + +void kutf_helper_output_named_val(struct kutf_helper_named_val *named_val) +{ + switch (named_val->type) { + case KUTF_HELPER_VALTYPE_U64: + pr_warn("%s=0x%llx\n", named_val->val_name, named_val->u.val_u64); + break; + case KUTF_HELPER_VALTYPE_STR: + pr_warn("%s=\"%s\"\n", named_val->val_name, named_val->u.val_str); + break; + case KUTF_HELPER_VALTYPE_INVALID: + pr_warn("%s is invalid\n", named_val->val_name); + break; + default: + pr_warn("%s has unknown type %d\n", named_val->val_name, named_val->type); + break; + } +} +EXPORT_SYMBOL(kutf_helper_output_named_val); diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_mem.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_mem.c new file mode 100755 index 000000000000..a75e15fde05f --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_mem.c @@ -0,0 +1,102 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF memory management functions */ + +#include +#include + +#include + + +/** + * struct kutf_alloc_entry - Structure representing an allocation. + * @node: List node for use with kutf_mempool. + * @data: Data area of the allocation + */ +struct kutf_alloc_entry { + struct list_head node; + u8 data[0]; +}; + +int kutf_mempool_init(struct kutf_mempool *pool) +{ + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + return -1; + } + + INIT_LIST_HEAD(&pool->head); + mutex_init(&pool->lock); + + return 0; +} +EXPORT_SYMBOL(kutf_mempool_init); + +void kutf_mempool_destroy(struct kutf_mempool *pool) +{ + struct list_head *remove; + struct list_head *tmp; + + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + return; + } + + mutex_lock(&pool->lock); + list_for_each_safe(remove, tmp, &pool->head) { + struct kutf_alloc_entry *remove_alloc; + + remove_alloc = list_entry(remove, struct kutf_alloc_entry, node); + list_del(&remove_alloc->node); + kfree(remove_alloc); + } + mutex_unlock(&pool->lock); + +} +EXPORT_SYMBOL(kutf_mempool_destroy); + +void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size) +{ + struct kutf_alloc_entry *ret; + + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + goto fail_pool; + } + + mutex_lock(&pool->lock); + + ret = kmalloc(sizeof(*ret) + size, GFP_KERNEL); + if (!ret) { + pr_err("Failed to allocate memory\n"); + goto fail_alloc; + } + + INIT_LIST_HEAD(&ret->node); + list_add(&ret->node, &pool->head); + + mutex_unlock(&pool->lock); + + return &ret->data[0]; + +fail_alloc: + mutex_unlock(&pool->lock); +fail_pool: + return NULL; +} +EXPORT_SYMBOL(kutf_mempool_alloc); diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_resultset.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_resultset.c new file mode 100755 index 000000000000..5bd04969fd55 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_resultset.c @@ -0,0 +1,95 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF result management functions */ + +#include +#include +#include + +#include + +/** + * struct kutf_result_set - Represents a set of results. + * @results: Pointer to the linked list where the results are stored. + */ +struct kutf_result_set { + struct list_head results; +}; + +struct kutf_result_set *kutf_create_result_set(void) +{ + struct kutf_result_set *set; + + set = kmalloc(sizeof(*set), GFP_KERNEL); + if (!set) { + pr_err("Failed to allocate resultset"); + goto fail_alloc; + } + + INIT_LIST_HEAD(&set->results); + + return set; + +fail_alloc: + return NULL; +} + +void kutf_add_result(struct kutf_mempool *mempool, + struct kutf_result_set *set, + enum kutf_result_status status, + const char *message) +{ + /* Create the new result */ + struct kutf_result *new_result; + + BUG_ON(set == NULL); + + new_result = kutf_mempool_alloc(mempool, sizeof(*new_result)); + if (!new_result) { + pr_err("Result allocation failed\n"); + return; + } + + INIT_LIST_HEAD(&new_result->node); + new_result->status = status; + new_result->message = message; + + list_add_tail(&new_result->node, &set->results); +} + +void kutf_destroy_result_set(struct kutf_result_set *set) +{ + if (!list_empty(&set->results)) + pr_err("kutf_destroy_result_set: Unread results from test\n"); + + kfree(set); +} + +struct kutf_result *kutf_remove_result(struct kutf_result_set *set) +{ + if (!list_empty(&set->results)) { + struct kutf_result *ret; + + ret = list_first_entry(&set->results, struct kutf_result, node); + list_del(&ret->node); + return ret; + } + + return NULL; +} + diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_suite.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_suite.c new file mode 100755 index 000000000000..ad30cc86a3b0 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_suite.c @@ -0,0 +1,1398 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF suite, test and fixture management including user to kernel + * interaction */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#if defined(CONFIG_DEBUG_FS) + +/** + * struct kutf_application - Structure which represents kutf application + * @name: The name of this test application. + * @dir: The debugfs directory for this test + * @suite_list: List head to store all the suites which are part of this + * application + */ +struct kutf_application { + const char *name; + struct dentry *dir; + struct list_head suite_list; +}; + +/** + * struct kutf_test_function - Structure which represents kutf test function + * @suite: Back reference to the suite this test function + * belongs to + * @filters: Filters that apply to this test function + * @test_id: Test ID + * @execute: Function to run for this test + * @test_data: Static data for this test + * @node: List node for test_list + * @variant_list: List head to store all the variants which can run on + * this function + * @dir: debugfs directory for this test function + * @userdata_ops: Callbacks to use for sending and receiving data to + * userspace. + */ +struct kutf_test_function { + struct kutf_suite *suite; + unsigned int filters; + unsigned int test_id; + void (*execute)(struct kutf_context *context); + union kutf_callback_data test_data; + struct list_head node; + struct list_head variant_list; + struct dentry *dir; + struct kutf_userdata_ops userdata_ops; +}; + +/** + * struct kutf_test_fixture - Structure which holds information on the kutf + * test fixture + * @test_func: Test function this fixture belongs to + * @fixture_index: Index of this fixture + * @node: List node for variant_list + * @dir: debugfs directory for this test fixture + * @nr_running: Current count of user-clients running this fixture + */ +struct kutf_test_fixture { + struct kutf_test_function *test_func; + unsigned int fixture_index; + struct list_head node; + struct dentry *dir; + atomic_t nr_running; +}; + +struct dentry *base_dir; + +/** + * struct kutf_convert_table - Structure which keeps test results + * @result_name: Status of the test result + * @result: Status value for a single test + */ +struct kutf_convert_table { + char result_name[50]; + enum kutf_result_status result; +}; + +struct kutf_convert_table kutf_convert[] = { +#define ADD_UTF_RESULT(_name) \ +{ \ + #_name, \ + _name, \ +}, +ADD_UTF_RESULT(KUTF_RESULT_BENCHMARK) +ADD_UTF_RESULT(KUTF_RESULT_SKIP) +ADD_UTF_RESULT(KUTF_RESULT_UNKNOWN) +ADD_UTF_RESULT(KUTF_RESULT_PASS) +ADD_UTF_RESULT(KUTF_RESULT_DEBUG) +ADD_UTF_RESULT(KUTF_RESULT_INFO) +ADD_UTF_RESULT(KUTF_RESULT_WARN) +ADD_UTF_RESULT(KUTF_RESULT_FAIL) +ADD_UTF_RESULT(KUTF_RESULT_FATAL) +ADD_UTF_RESULT(KUTF_RESULT_ABORT) +}; + +#define UTF_CONVERT_SIZE (ARRAY_SIZE(kutf_convert)) + +/** + * kutf_create_context() - Create a test context in which a specific fixture + * of an application will be run and its results + * reported back to the user + * @test_fix: Test fixture to be run. + * + * The context's refcount will be initialized to 1. + * + * Return: Returns the created test context on success or NULL on failure + */ +static struct kutf_context *kutf_create_context( + struct kutf_test_fixture *test_fix); + +/** + * kutf_destroy_context() - Destroy a previously created test context, only + * once its refcount has become zero + * @kref: pointer to kref member within the context + * + * This should only be used via a kref_put() call on the context's kref member + */ +static void kutf_destroy_context(struct kref *kref); + +/** + * kutf_context_get() - increment refcount on a context + * @context: the kutf context + * + * This must be used when the lifetime of the context might exceed that of the + * thread creating @context + */ +static void kutf_context_get(struct kutf_context *context); + +/** + * kutf_context_put() - decrement refcount on a context, destroying it when it + * reached zero + * @context: the kutf context + * + * This must be used only after a corresponding kutf_context_get() call on + * @context, and the caller no longer needs access to @context. + */ +static void kutf_context_put(struct kutf_context *context); + +/** + * kutf_set_result() - Set the test result against the specified test context + * @context: Test context + * @status: Result status + */ +static void kutf_set_result(struct kutf_context *context, + enum kutf_result_status status); + +/** + * kutf_set_expected_result() - Set the expected test result for the specified + * test context + * @context: Test context + * @expected_status: Expected result status + */ +static void kutf_set_expected_result(struct kutf_context *context, + enum kutf_result_status expected_status); + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)) +/* Pre 3.4.0 kernels don't have the simple_open helper */ + +/** + * simple_open() - Helper for file opening which stores the inode private data + * into the file private data + * @inode: File entry representation + * @file: A specific opening of the file + * + * Return: always 0; if inode private data do not exist, the file will not + * be assigned private data + */ +static int simple_open(struct inode *inode, struct file *file) +{ + if (inode->i_private) + file->private_data = inode->i_private; + return 0; +} +#endif + +/** + * kutf_result_to_string() - Converts a KUTF result into a string + * @result_str: Output result string + * @result: Result status to convert + * + * Return: 1 if test result was successfully converted to string, 0 otherwise + */ +static int kutf_result_to_string(char **result_str, + enum kutf_result_status result) +{ + int i; + int ret = 0; + + for (i = 0; i < UTF_CONVERT_SIZE; i++) { + if (result == kutf_convert[i].result) { + *result_str = kutf_convert[i].result_name; + ret = 1; + } + } + return ret; +} + +/** + * kutf_debugfs_const_string_read() - Simple debugfs read callback which + * returns a constant string + * @file: Opened file to read from + * @buf: User buffer to write the data into + * @len: Amount of data to read + * @ppos: Offset into file to read from + * + * Return: On success, the number of bytes read and offset @ppos advanced by + * this number; on error, negative value + */ +static ssize_t kutf_debugfs_const_string_read(struct file *file, + char __user *buf, size_t len, loff_t *ppos) +{ + char *str = file->private_data; + + return simple_read_from_buffer(buf, len, ppos, str, strlen(str)); +} + +static const struct file_operations kutf_debugfs_const_string_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = kutf_debugfs_const_string_read, + .llseek = default_llseek, +}; + +/** + * kutf_debugfs_data_open() Debugfs open callback for the "data" entry. + * @inode: inode of the opened file + * @file: Opened file to read from + * + * This function notifies the userdata callbacks that the userdata file has + * been opened, for tracking purposes. + * + * It is called on both the context's userdata_consumer_priv and + * userdata_producer_priv. + * + * This takes a refcount on the kutf_context + * + * Return: 0 on success + */ +static int kutf_debugfs_data_open(struct inode *inode, struct file *file) +{ + struct kutf_context *test_context = inode->i_private; + struct kutf_test_fixture *test_fix = test_context->test_fix; + struct kutf_test_function *test_func = test_fix->test_func; + int err; + + simple_open(inode, file); + + /* This is not an error */ + if (!test_func->userdata_ops.open) + goto out_no_ops; + + /* This is safe here - the 'data' file is only openable whilst the + * initial refcount is still present, and the initial refcount is only + * dropped strictly after the 'data' file is removed */ + kutf_context_get(test_context); + + if (test_context->userdata_consumer_priv) { + err = test_func->userdata_ops.open(test_context->userdata_consumer_priv); + if (err) + goto out_consumer_fail; + } + + if (test_context->userdata_producer_priv) { + err = test_func->userdata_ops.open(test_context->userdata_producer_priv); + if (err) + goto out_producer_fail; + } + +out_no_ops: + return 0; + +out_producer_fail: + if (test_func->userdata_ops.release && test_context->userdata_consumer_priv) + test_func->userdata_ops.release(test_context->userdata_consumer_priv); +out_consumer_fail: + kutf_context_put(test_context); + + return err; +} + + +/** + * kutf_debugfs_data_read() Debugfs read callback for the "data" entry. + * @file: Opened file to read from + * @buf: User buffer to write the data into + * @len: Amount of data to read + * @ppos: Offset into file to read from + * + * This function allows user and kernel to exchange extra data necessary for + * the test fixture. + * + * The data is read from the first struct kutf_context running the fixture + * + * Return: Number of bytes read + */ +static ssize_t kutf_debugfs_data_read(struct file *file, char __user *buf, + size_t len, loff_t *ppos) +{ + struct kutf_context *test_context = file->private_data; + struct kutf_test_fixture *test_fix = test_context->test_fix; + struct kutf_test_function *test_func = test_fix->test_func; + ssize_t (*producer)(void *private, char __user *userbuf, + size_t userbuf_len, loff_t *ppos); + ssize_t count; + + producer = test_func->userdata_ops.producer; + /* Can only read if there's a producer callback */ + if (!producer) + return -ENODEV; + + count = producer(test_context->userdata_producer_priv, buf, len, ppos); + + return count; +} + +/** + * kutf_debugfs_data_write() Debugfs write callback for the "data" entry. + * @file: Opened file to write to + * @buf: User buffer to read the data from + * @len: Amount of data to write + * @ppos: Offset into file to write to + * + * This function allows user and kernel to exchange extra data necessary for + * the test fixture. + * + * The data is added to the first struct kutf_context running the fixture + * + * Return: Number of bytes written + */ +static ssize_t kutf_debugfs_data_write(struct file *file, + const char __user *buf, size_t len, loff_t *ppos) +{ + struct kutf_context *test_context = file->private_data; + struct kutf_test_fixture *test_fix = test_context->test_fix; + struct kutf_test_function *test_func = test_fix->test_func; + ssize_t (*consumer)(void *private, const char __user *userbuf, + size_t userbuf_len, loff_t *ppos); + ssize_t count; + + consumer = test_func->userdata_ops.consumer; + /* Can only write if there's a consumer callback */ + if (!consumer) + return -ENODEV; + + count = consumer(test_context->userdata_consumer_priv, buf, len, ppos); + + return count; +} + + +/** + * kutf_debugfs_data_release() - Debugfs release callback for the "data" entry. + * @inode: File entry representation + * @file: A specific opening of the file + * + * This function notifies the userdata callbacks that the userdata file has + * been closed, for tracking purposes. + * + * It is called on both the context's userdata_consumer_priv and + * userdata_producer_priv. + * + * It also drops the refcount on the kutf_context that was taken during + * kutf_debugfs_data_open() + */ +static int kutf_debugfs_data_release(struct inode *inode, struct file *file) +{ + struct kutf_context *test_context = file->private_data; + struct kutf_test_fixture *test_fix = test_context->test_fix; + struct kutf_test_function *test_func = test_fix->test_func; + + if (!test_func->userdata_ops.release) + return 0; + + if (test_context->userdata_consumer_priv) + test_func->userdata_ops.release(test_context->userdata_consumer_priv); + if (test_context->userdata_producer_priv) + test_func->userdata_ops.release(test_context->userdata_producer_priv); + + kutf_context_put(test_context); + + return 0; +} + + +static const struct file_operations kutf_debugfs_data_ops = { + .owner = THIS_MODULE, + .open = kutf_debugfs_data_open, + .read = kutf_debugfs_data_read, + .write = kutf_debugfs_data_write, + .release = kutf_debugfs_data_release, + .llseek = default_llseek, +}; + +/** + * userdata_init() - Initialize userspace data exchange for a test, if + * specified by that test + * @test_context: Test context + * + * Note that this allows new refcounts to be made on test_context by userspace + * threads opening the 'data' file. + * + * Return: 0 on success, negative value corresponding to error code in failure + * and kutf result will be set appropriately to indicate the error + */ +static int userdata_init(struct kutf_context *test_context) +{ + struct kutf_test_fixture *test_fix = test_context->test_fix; + struct kutf_test_function *test_func = test_fix->test_func; + int err = 0; + struct dentry *userdata_dentry; + + /* Valid to have neither a producer or consumer, which is the case for + * tests not requiring usersdata */ + if ((!test_func->userdata_ops.consumer) && (!test_func->userdata_ops.producer)) + return err; + + if (test_func->userdata_ops.consumer && !test_context->userdata_consumer_priv) { + kutf_test_fatal(test_context, + "incorrect test setup - userdata consumer provided without private data"); + return -EFAULT; + } + + if (test_func->userdata_ops.producer && !test_context->userdata_producer_priv) { + kutf_test_fatal(test_context, + "incorrect test setup - userdata producer provided without private data"); + return -EFAULT; + } + + userdata_dentry = debugfs_create_file("data", S_IROTH, test_fix->dir, + test_context, &kutf_debugfs_data_ops); + + if (!userdata_dentry) { + pr_err("Failed to create debugfs file \"data\" when running fixture\n"); + /* Not using Fatal (which stops other tests running), + * nor Abort (which indicates teardown should not be done) */ + kutf_test_fail(test_context, + "failed to create 'data' file for userside data exchange"); + + /* Error code is discarded by caller, but consistent with other + * debugfs_create_file failures */ + err = -EEXIST; + } else { + test_context->userdata_dentry = userdata_dentry; + } + + + return err; +} + +/** + * userdata_term() - Terminate userspace data exchange for a test, if specified + * by that test + * @test_context: Test context + * + * Note This also prevents new refcounts being made on @test_context by userspace + * threads opening the 'data' file for this test. Any existing open file descriptors + * to the 'data' file will still be safe to use by userspace. + */ +static void userdata_term(struct kutf_context *test_context) +{ + struct kutf_test_fixture *test_fix = test_context->test_fix; + struct kutf_test_function *test_func = test_fix->test_func; + void (*notify_ended)(void *priv) = test_func->userdata_ops.notify_ended; + + /* debugfs_remove() is safe when parameter is error or NULL */ + debugfs_remove(test_context->userdata_dentry); + + /* debugfs_remove() doesn't kill any currently open file descriptors on + * this file, and such fds are still safe to use providing test_context + * is properly refcounted */ + + if (notify_ended) { + if (test_context->userdata_consumer_priv) + notify_ended(test_context->userdata_consumer_priv); + if (test_context->userdata_producer_priv) + notify_ended(test_context->userdata_producer_priv); + } + +} + +/** + * kutf_add_explicit_result() - Check if an explicit result needs to be added + * @context: KUTF test context + */ +static void kutf_add_explicit_result(struct kutf_context *context) +{ + switch (context->expected_status) { + case KUTF_RESULT_UNKNOWN: + if (context->status == KUTF_RESULT_UNKNOWN) + kutf_test_pass(context, "(implicit pass)"); + break; + + case KUTF_RESULT_WARN: + if (context->status == KUTF_RESULT_WARN) + kutf_test_pass(context, + "Pass (expected warn occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected warn missing)"); + break; + + case KUTF_RESULT_FAIL: + if (context->status == KUTF_RESULT_FAIL) + kutf_test_pass(context, + "Pass (expected fail occurred)"); + else if (context->status != KUTF_RESULT_SKIP) { + /* Force the expected status so the fail gets logged */ + context->expected_status = KUTF_RESULT_PASS; + kutf_test_fail(context, + "Fail (expected fail missing)"); + } + break; + + case KUTF_RESULT_FATAL: + if (context->status == KUTF_RESULT_FATAL) + kutf_test_pass(context, + "Pass (expected fatal occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected fatal missing)"); + break; + + case KUTF_RESULT_ABORT: + if (context->status == KUTF_RESULT_ABORT) + kutf_test_pass(context, + "Pass (expected abort occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected abort missing)"); + break; + default: + break; + } +} + +/** + * kutf_debugfs_run_open() Debugfs open callback for the "run" entry. + * @inode: inode of the opened file + * @file: Opened file to read from + * + * This function retrieves the test fixture data that is associated with the + * opened file and works back to get the test, suite and application so + * it can then run the test that is associated with the file entry. + * + * Return: 0 on success + */ +static int kutf_debugfs_run_open(struct inode *inode, struct file *file) +{ + struct kutf_test_fixture *test_fix = inode->i_private; + struct kutf_test_function *test_func = test_fix->test_func; + struct kutf_suite *suite = test_func->suite; + struct kutf_context *test_context; + int err = 0; + + /* For the moment, only one user-client should be attempting to run + * this at a time. This simplifies how we lookup the kutf_context when + * using the 'data' file. + * Removing this restriction would require a rewrite of the mechanism + * of the 'data' file to pass data in, perhaps 'data' created here and + * based upon userspace thread's pid */ + if (atomic_inc_return(&test_fix->nr_running) != 1) { + err = -EBUSY; + goto finish; + } + + test_context = kutf_create_context(test_fix); + if (!test_context) { + err = -ENODEV; + goto finish; + } + + file->private_data = test_context; + + /* + * Call the create fixture function if required before the + * fixture is run + */ + if (suite->create_fixture) + test_context->fixture = suite->create_fixture(test_context); + + /* Only run the test if the fixture was created (if required) */ + if ((suite->create_fixture && test_context->fixture) || + (!suite->create_fixture)) { + int late_err; + /* Setup any userdata exchange */ + late_err = userdata_init(test_context); + + if (!late_err) + /* Run this fixture */ + test_func->execute(test_context); + + userdata_term(test_context); + + if (suite->remove_fixture) + suite->remove_fixture(test_context); + + kutf_add_explicit_result(test_context); + } + +finish: + atomic_dec(&test_fix->nr_running); + return err; +} + +/** + * kutf_debugfs_run_read() - Debugfs read callback for the "run" entry. + * @file: Opened file to read from + * @buf: User buffer to write the data into + * @len: Amount of data to read + * @ppos: Offset into file to read from + * + * This function emits the results which where logged during the opening of + * the file kutf_debugfs_run_open. + * Results will be emitted one at a time, once all the results have been read + * 0 will be returned to indicate there is no more data. + * + * Return: Number of bytes read. + */ +static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, + size_t len, loff_t *ppos) +{ + struct kutf_context *test_context = file->private_data; + struct kutf_result *res; + unsigned long bytes_not_copied; + ssize_t bytes_copied = 0; + + /* Note: This code assumes a result is read completely */ + res = kutf_remove_result(test_context->result_set); + if (res) { + char *kutf_str_ptr = NULL; + unsigned int kutf_str_len = 0; + unsigned int message_len = 0; + char separator = ':'; + char terminator = '\n'; + + kutf_result_to_string(&kutf_str_ptr, res->status); + if (kutf_str_ptr) + kutf_str_len = strlen(kutf_str_ptr); + + if (res->message) + message_len = strlen(res->message); + + if ((kutf_str_len + 1 + message_len + 1) > len) { + pr_err("Not enough space in user buffer for a single result"); + return 0; + } + + /* First copy the result string */ + if (kutf_str_ptr) { + bytes_not_copied = copy_to_user(&buf[0], kutf_str_ptr, + kutf_str_len); + bytes_copied += kutf_str_len - bytes_not_copied; + if (bytes_not_copied) + goto exit; + } + + /* Then the separator */ + bytes_not_copied = copy_to_user(&buf[bytes_copied], + &separator, 1); + bytes_copied += 1 - bytes_not_copied; + if (bytes_not_copied) + goto exit; + + /* Finally Next copy the result string */ + if (res->message) { + bytes_not_copied = copy_to_user(&buf[bytes_copied], + res->message, message_len); + bytes_copied += message_len - bytes_not_copied; + if (bytes_not_copied) + goto exit; + } + + /* Finally the terminator */ + bytes_not_copied = copy_to_user(&buf[bytes_copied], + &terminator, 1); + bytes_copied += 1 - bytes_not_copied; + } +exit: + return bytes_copied; +} + +/** + * kutf_debugfs_run_release() - Debugfs release callback for the "run" entry. + * @inode: File entry representation + * @file: A specific opening of the file + * + * Release any resources that where created during the opening of the file + * + * Note that resources may not be released immediately, that might only happen + * later when other users of the kutf_context release their refcount. + * + * Return: 0 on success + */ +static int kutf_debugfs_run_release(struct inode *inode, struct file *file) +{ + struct kutf_context *test_context = file->private_data; + + kutf_context_put(test_context); + return 0; +} + +static const struct file_operations kutf_debugfs_run_ops = { + .owner = THIS_MODULE, + .open = kutf_debugfs_run_open, + .read = kutf_debugfs_run_read, + .release = kutf_debugfs_run_release, + .llseek = default_llseek, +}; + +/** + * create_fixture_variant() - Creates a fixture variant for the specified + * test function and index and the debugfs entries + * that represent it. + * @test_func: Test function + * @fixture_index: Fixture index + * + * Return: 0 on success, negative value corresponding to error code in failure + */ +static int create_fixture_variant(struct kutf_test_function *test_func, + unsigned int fixture_index) +{ + struct kutf_test_fixture *test_fix; + char name[11]; /* Enough to print the MAX_UINT32 + the null terminator */ + struct dentry *tmp; + int err; + + test_fix = kmalloc(sizeof(*test_fix), GFP_KERNEL); + if (!test_fix) { + pr_err("Failed to create debugfs directory when adding fixture\n"); + err = -ENOMEM; + goto fail_alloc; + } + + test_fix->test_func = test_func; + test_fix->fixture_index = fixture_index; + atomic_set(&test_fix->nr_running, 0); + + snprintf(name, sizeof(name), "%d", fixture_index); + test_fix->dir = debugfs_create_dir(name, test_func->dir); + if (!test_func->dir) { + pr_err("Failed to create debugfs directory when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_dir; + } + + tmp = debugfs_create_file("type", S_IROTH, test_fix->dir, "fixture\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_file; + } + + tmp = debugfs_create_file("run", S_IROTH, test_fix->dir, test_fix, + &kutf_debugfs_run_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"run\" when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_file; + } + + list_add(&test_fix->node, &test_func->variant_list); + return 0; + +fail_file: + debugfs_remove_recursive(test_fix->dir); +fail_dir: + kfree(test_fix); +fail_alloc: + return err; +} + +/** + * kutf_remove_test_variant() - Destroy a previously created fixture variant. + * @test_fix: Test fixture + */ +static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix) +{ + debugfs_remove_recursive(test_fix->dir); + kfree(test_fix); +} + +void kutf_add_test_with_filters_data_and_userdata( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data test_data, + struct kutf_userdata_ops *userdata_ops) +{ + struct kutf_test_function *test_func; + struct dentry *tmp; + unsigned int i; + + test_func = kmalloc(sizeof(*test_func), GFP_KERNEL); + if (!test_func) { + pr_err("Failed to allocate memory when adding test %s\n", name); + goto fail_alloc; + } + + INIT_LIST_HEAD(&test_func->variant_list); + + test_func->dir = debugfs_create_dir(name, suite->dir); + if (!test_func->dir) { + pr_err("Failed to create debugfs directory when adding test %s\n", name); + goto fail_dir; + } + + tmp = debugfs_create_file("type", S_IROTH, test_func->dir, "test\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); + goto fail_file; + } + + test_func->filters = filters; + tmp = debugfs_create_x32("filters", S_IROTH, test_func->dir, + &test_func->filters); + if (!tmp) { + pr_err("Failed to create debugfs file \"filters\" when adding test %s\n", name); + goto fail_file; + } + + test_func->test_id = id; + tmp = debugfs_create_u32("test_id", S_IROTH, test_func->dir, + &test_func->test_id); + if (!tmp) { + pr_err("Failed to create debugfs file \"test_id\" when adding test %s\n", name); + goto fail_file; + } + + for (i = 0; i < suite->fixture_variants; i++) { + if (create_fixture_variant(test_func, i)) { + pr_err("Failed to create fixture %d when adding test %s\n", i, name); + goto fail_file; + } + } + + test_func->suite = suite; + test_func->execute = execute; + test_func->test_data = test_data; + memcpy(&test_func->userdata_ops, userdata_ops, sizeof(*userdata_ops)); + + list_add(&test_func->node, &suite->test_list); + return; + +fail_file: + debugfs_remove_recursive(test_func->dir); +fail_dir: + kfree(test_func); +fail_alloc: + return; +} +EXPORT_SYMBOL(kutf_add_test_with_filters_data_and_userdata); + +void kutf_add_test_with_filters_and_data( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data test_data) +{ + struct kutf_userdata_ops userdata_ops = { + .open = NULL, + .release = NULL, + .consumer = NULL, + .producer = NULL, + }; + + kutf_add_test_with_filters_data_and_userdata(suite, id, name, execute, + filters, test_data, &userdata_ops); +} + +EXPORT_SYMBOL(kutf_add_test_with_filters_and_data); + +void kutf_add_test_with_filters( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + + kutf_add_test_with_filters_and_data(suite, + id, + name, + execute, + suite->suite_default_flags, + data); +} +EXPORT_SYMBOL(kutf_add_test_with_filters); + +void kutf_add_test(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context)) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + + kutf_add_test_with_filters_and_data(suite, + id, + name, + execute, + suite->suite_default_flags, + data); +} +EXPORT_SYMBOL(kutf_add_test); + +/** + * kutf_remove_test(): Remove a previously added test function. + * @test_func: Test function + */ +static void kutf_remove_test(struct kutf_test_function *test_func) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &test_func->variant_list) { + struct kutf_test_fixture *test_fix; + + test_fix = list_entry(pos, struct kutf_test_fixture, node); + kutf_remove_test_variant(test_fix); + } + + list_del(&test_func->node); + debugfs_remove_recursive(test_func->dir); + kfree(test_func); +} + +struct kutf_suite *kutf_create_suite_with_filters_and_data( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data suite_data) +{ + struct kutf_suite *suite; + struct dentry *tmp; + + suite = kmalloc(sizeof(*suite), GFP_KERNEL); + if (!suite) { + pr_err("Failed to allocate memory when creating suite %s\n", name); + goto fail_kmalloc; + } + + suite->dir = debugfs_create_dir(name, app->dir); + if (!suite->dir) { + pr_err("Failed to create debugfs directory when adding test %s\n", name); + goto fail_debugfs; + } + + tmp = debugfs_create_file("type", S_IROTH, suite->dir, "suite\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); + goto fail_file; + } + + INIT_LIST_HEAD(&suite->test_list); + suite->app = app; + suite->name = name; + suite->fixture_variants = fixture_count; + suite->create_fixture = create_fixture; + suite->remove_fixture = remove_fixture; + suite->suite_default_flags = filters; + suite->suite_data = suite_data; + + list_add(&suite->node, &app->suite_list); + + return suite; + +fail_file: + debugfs_remove_recursive(suite->dir); +fail_debugfs: + kfree(suite); +fail_kmalloc: + return NULL; +} +EXPORT_SYMBOL(kutf_create_suite_with_filters_and_data); + +struct kutf_suite *kutf_create_suite_with_filters( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + return kutf_create_suite_with_filters_and_data(app, + name, + fixture_count, + create_fixture, + remove_fixture, + filters, + data); +} +EXPORT_SYMBOL(kutf_create_suite_with_filters); + +struct kutf_suite *kutf_create_suite( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context)) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + return kutf_create_suite_with_filters_and_data(app, + name, + fixture_count, + create_fixture, + remove_fixture, + KUTF_F_TEST_GENERIC, + data); +} +EXPORT_SYMBOL(kutf_create_suite); + +/** + * kutf_destroy_suite() - Destroy a previously added test suite. + * @suite: Test suite + */ +static void kutf_destroy_suite(struct kutf_suite *suite) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &suite->test_list) { + struct kutf_test_function *test_func; + + test_func = list_entry(pos, struct kutf_test_function, node); + kutf_remove_test(test_func); + } + + list_del(&suite->node); + debugfs_remove_recursive(suite->dir); + kfree(suite); +} + +struct kutf_application *kutf_create_application(const char *name) +{ + struct kutf_application *app; + struct dentry *tmp; + + app = kmalloc(sizeof(*app), GFP_KERNEL); + if (!app) { + pr_err("Failed to create allocate memory when creating application %s\n", name); + goto fail_kmalloc; + } + + app->dir = debugfs_create_dir(name, base_dir); + if (!app->dir) { + pr_err("Failed to create debugfs direcotry when creating application %s\n", name); + goto fail_debugfs; + } + + tmp = debugfs_create_file("type", S_IROTH, app->dir, "application\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when creating application %s\n", name); + goto fail_file; + } + + INIT_LIST_HEAD(&app->suite_list); + app->name = name; + + return app; + +fail_file: + debugfs_remove_recursive(app->dir); +fail_debugfs: + kfree(app); +fail_kmalloc: + return NULL; +} +EXPORT_SYMBOL(kutf_create_application); + +void kutf_destroy_application(struct kutf_application *app) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &app->suite_list) { + struct kutf_suite *suite; + + suite = list_entry(pos, struct kutf_suite, node); + kutf_destroy_suite(suite); + } + + debugfs_remove_recursive(app->dir); + kfree(app); +} +EXPORT_SYMBOL(kutf_destroy_application); + +static struct kutf_context *kutf_create_context( + struct kutf_test_fixture *test_fix) +{ + struct kutf_context *new_context; + + new_context = kmalloc(sizeof(*new_context), GFP_KERNEL); + if (!new_context) { + pr_err("Failed to allocate test context"); + goto fail_alloc; + } + + new_context->result_set = kutf_create_result_set(); + if (!new_context->result_set) { + pr_err("Failed to create resultset"); + goto fail_result_set; + } + + new_context->test_fix = test_fix; + /* Save the pointer to the suite as the callbacks will require it */ + new_context->suite = test_fix->test_func->suite; + new_context->status = KUTF_RESULT_UNKNOWN; + new_context->expected_status = KUTF_RESULT_UNKNOWN; + + kutf_mempool_init(&new_context->fixture_pool); + new_context->fixture = NULL; + new_context->fixture_index = test_fix->fixture_index; + new_context->fixture_name = NULL; + new_context->test_data = test_fix->test_func->test_data; + new_context->userdata_consumer_priv = NULL; + new_context->userdata_producer_priv = NULL; + new_context->userdata_dentry = NULL; + + kref_init(&new_context->kref); + + return new_context; + +fail_result_set: + kfree(new_context); +fail_alloc: + return NULL; +} + +static void kutf_destroy_context(struct kref *kref) +{ + struct kutf_context *context; + + context = container_of(kref, struct kutf_context, kref); + kutf_destroy_result_set(context->result_set); + kutf_mempool_destroy(&context->fixture_pool); + kfree(context); +} + +static void kutf_context_get(struct kutf_context *context) +{ + kref_get(&context->kref); +} + +static void kutf_context_put(struct kutf_context *context) +{ + kref_put(&context->kref, kutf_destroy_context); +} + + +static void kutf_set_result(struct kutf_context *context, + enum kutf_result_status status) +{ + context->status = status; +} + +static void kutf_set_expected_result(struct kutf_context *context, + enum kutf_result_status expected_status) +{ + context->expected_status = expected_status; +} + +/** + * kutf_test_log_result() - Log a result for the specified test context + * @context: Test context + * @message: Result string + * @new_status: Result status + */ +static void kutf_test_log_result( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status) +{ + if (context->status < new_status) + context->status = new_status; + + if (context->expected_status != new_status) + kutf_add_result(&context->fixture_pool, context->result_set, + new_status, message); +} + +void kutf_test_log_result_external( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status) +{ + kutf_test_log_result(context, message, new_status); +} +EXPORT_SYMBOL(kutf_test_log_result_external); + +void kutf_test_expect_abort(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_ABORT); +} +EXPORT_SYMBOL(kutf_test_expect_abort); + +void kutf_test_expect_fatal(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_FATAL); +} +EXPORT_SYMBOL(kutf_test_expect_fatal); + +void kutf_test_expect_fail(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_FAIL); +} +EXPORT_SYMBOL(kutf_test_expect_fail); + +void kutf_test_expect_warn(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_WARN); +} +EXPORT_SYMBOL(kutf_test_expect_warn); + +void kutf_test_expect_pass(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_PASS); +} +EXPORT_SYMBOL(kutf_test_expect_pass); + +void kutf_test_skip(struct kutf_context *context) +{ + kutf_set_result(context, KUTF_RESULT_SKIP); + kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); + + kutf_test_log_result(context, "Test skipped", KUTF_RESULT_SKIP); +} +EXPORT_SYMBOL(kutf_test_skip); + +void kutf_test_skip_msg(struct kutf_context *context, const char *message) +{ + kutf_set_result(context, KUTF_RESULT_SKIP); + kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); + + kutf_test_log_result(context, kutf_dsprintf(&context->fixture_pool, + "Test skipped: %s", message), KUTF_RESULT_SKIP); + kutf_test_log_result(context, "!!!Test skipped!!!", KUTF_RESULT_SKIP); +} +EXPORT_SYMBOL(kutf_test_skip_msg); + +void kutf_test_debug(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_DEBUG); +} +EXPORT_SYMBOL(kutf_test_debug); + +void kutf_test_pass(struct kutf_context *context, char const *message) +{ + static const char explicit_message[] = "(explicit pass)"; + + if (!message) + message = explicit_message; + + kutf_test_log_result(context, message, KUTF_RESULT_PASS); +} +EXPORT_SYMBOL(kutf_test_pass); + +void kutf_test_info(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_INFO); +} +EXPORT_SYMBOL(kutf_test_info); + +void kutf_test_warn(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_WARN); +} +EXPORT_SYMBOL(kutf_test_warn); + +void kutf_test_fail(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_FAIL); +} +EXPORT_SYMBOL(kutf_test_fail); + +void kutf_test_fatal(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_FATAL); +} +EXPORT_SYMBOL(kutf_test_fatal); + +void kutf_test_abort(struct kutf_context *context) +{ + kutf_test_log_result(context, "", KUTF_RESULT_ABORT); +} +EXPORT_SYMBOL(kutf_test_abort); + +/** + * init_kutf_core() - Module entry point. + * + * Create the base entry point in debugfs. + */ +static int __init init_kutf_core(void) +{ + int ret; + + base_dir = debugfs_create_dir("kutf_tests", NULL); + if (!base_dir) { + ret = -ENODEV; + goto exit_dir; + } + + return 0; + +exit_dir: + return ret; +} + +/** + * exit_kutf_core() - Module exit point. + * + * Remove the base entry point in debugfs. + */ +static void __exit exit_kutf_core(void) +{ + debugfs_remove_recursive(base_dir); +} + +#else /* defined(CONFIG_DEBUG_FS) */ + +/** + * init_kutf_core() - Module entry point. + * + * Stub for when build against a kernel without debugfs support + */ +static int __init init_kutf_core(void) +{ + pr_debug("KUTF requires a kernel with debug fs support"); + + return -ENODEV; +} + +/** + * exit_kutf_core() - Module exit point. + * + * Stub for when build against a kernel without debugfs support + */ +static void __exit exit_kutf_core(void) +{ +} +#endif /* defined(CONFIG_DEBUG_FS) */ + +MODULE_LICENSE("GPL"); + +module_init(init_kutf_core); +module_exit(exit_kutf_core); diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_utils.c b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_utils.c new file mode 100755 index 000000000000..a429a2dbf788 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/kutf_utils.c @@ -0,0 +1,71 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF utility functions */ + +#include +#include +#include +#include + +#include +#include + +static char tmp_buffer[KUTF_MAX_DSPRINTF_LEN]; + +DEFINE_MUTEX(buffer_lock); + +const char *kutf_dsprintf(struct kutf_mempool *pool, + const char *fmt, ...) +{ + va_list args; + int len; + int size; + void *buffer; + + mutex_lock(&buffer_lock); + va_start(args, fmt); + len = vsnprintf(tmp_buffer, sizeof(tmp_buffer), fmt, args); + va_end(args); + + if (len < 0) { + pr_err("kutf_dsprintf: Bad format dsprintf format %s\n", fmt); + goto fail_format; + } + + if (len >= sizeof(tmp_buffer)) { + pr_warn("kutf_dsprintf: Truncated dsprintf message %s\n", fmt); + size = sizeof(tmp_buffer); + } else { + size = len + 1; + } + + buffer = kutf_mempool_alloc(pool, size); + if (!buffer) + goto fail_alloc; + + memcpy(buffer, tmp_buffer, size); + mutex_unlock(&buffer_lock); + + return buffer; + +fail_alloc: +fail_format: + mutex_unlock(&buffer_lock); + return NULL; +} +EXPORT_SYMBOL(kutf_dsprintf); diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/kutf/sconscript b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/sconscript new file mode 100755 index 000000000000..d7f112448e42 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/kutf/sconscript @@ -0,0 +1,21 @@ +# +# (C) COPYRIGHT 2014-2016, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +Import('kutf_env') + +make_args = kutf_env.kernel_get_config_defines(ret_list = True) + +mod = kutf_env.BuildKernelModule('$STATIC_LIB_PATH/kutf.ko', Glob('*.c'), make_args = make_args) +kutf_env.KernelObjTarget('kutf', mod) diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kbuild b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kbuild new file mode 100755 index 000000000000..0cd9cebe9d8b --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kbuild @@ -0,0 +1,20 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android + +obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test.o + +mali_kutf_irq_test-y := mali_kutf_irq_test_main.o diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kconfig b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kconfig new file mode 100755 index 000000000000..4caa8ec8a0e2 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Kconfig @@ -0,0 +1,23 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +config MALI_IRQ_LATENCY + tristate "Mali GPU IRQ latency measurement" + depends on MALI_BIFROST && MALI_BIFROST_DEBUG && MALI_KUTF + default m + help + This option will build a test module mali_kutf_irq_test that + can determine the latency of the Mali GPU IRQ on your system. + Choosing M here will generate a single module called mali_kutf_irq_test. diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Makefile b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Makefile new file mode 100755 index 000000000000..ced37b08e532 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/Makefile @@ -0,0 +1,47 @@ +# +# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +# linux build system bootstrap for out-of-tree module + +# default to building for the host +ARCH ?= $(shell uname -m) + +ifeq ($(KDIR),) +$(error Must specify KDIR to point to the kernel to target)) +endif + +TEST_CCFLAGS := \ + -DMALI_DEBUG=$(MALI_BIFROST_DEBUG) \ + -DMALI_BACKEND_KERNEL=$(MALI_BACKEND_KERNEL) \ + -DMALI_NO_MALI=$(MALI_BIFROST_NO_MALI) \ + -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ + -DMALI_USE_UMP=$(MALI_USE_UMP) \ + -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ + -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ + $(SCONS_CFLAGS) \ + -I$(CURDIR)/../include \ + -I$(CURDIR)/../../../../../../include \ + -I$(CURDIR)/../../../ \ + -I$(CURDIR)/../../ \ + -I$(CURDIR)/../../backend/gpu \ + -I$(CURDIR)/ \ + -I$(srctree)/drivers/staging/android \ + -I$(srctree)/include/linux + +all: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS="$(TEST_CCFLAGS)" KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../kutf/Module.symvers $(CURDIR)/../../Module.symvers" modules + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c new file mode 100755 index 000000000000..c9cc4447cf37 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c @@ -0,0 +1,269 @@ +/* + * + * (C) COPYRIGHT 2016, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include + +#include "mali_kbase.h" +#include + +#include +#include + +/* + * This file contains the code which is used for measuring interrupt latency + * of the Mali GPU IRQ. In particular, function mali_kutf_irq_latency() is + * used with this purpose and it is called within KUTF framework - a kernel + * unit test framework. The measured latency provided by this test should + * be representative for the latency of the Mali JOB/MMU IRQs as well. + */ + +/* KUTF test application pointer for this test */ +struct kutf_application *irq_app; + +/** + * struct kutf_irq_fixture data - test fixture used by the test functions. + * @kbdev: kbase device for the GPU. + * + */ +struct kutf_irq_fixture_data { + struct kbase_device *kbdev; +}; + +#define SEC_TO_NANO(s) ((s)*1000000000LL) + +/* ID for the GPU IRQ */ +#define GPU_IRQ_HANDLER 2 + +#define NR_TEST_IRQS 1000000 + +/* IRQ for the test to trigger. Currently MULTIPLE_GPU_FAULTS as we would not + * expect to see this in normal use (e.g., when Android is running). */ +#define TEST_IRQ MULTIPLE_GPU_FAULTS + +#define IRQ_TIMEOUT HZ + +/* Kernel API for setting irq throttle hook callback and irq time in us*/ +extern int kbase_set_custom_irq_handler(struct kbase_device *kbdev, + irq_handler_t custom_handler, + int irq_type); +extern irqreturn_t kbase_gpu_irq_handler(int irq, void *data); + +static DECLARE_WAIT_QUEUE_HEAD(wait); +static bool triggered; +static u64 irq_time; + +static void *kbase_untag(void *ptr) +{ + return (void *)(((uintptr_t) ptr) & ~3); +} + +/** + * kbase_gpu_irq_custom_handler - Custom IRQ throttle handler + * @irq: IRQ number + * @data: Data associated with this IRQ + * + * Return: state of the IRQ + */ +static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) +{ + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); + if (val & TEST_IRQ) { + struct timespec tval; + + getnstimeofday(&tval); + irq_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, + NULL); + + triggered = true; + wake_up(&wait); + + return IRQ_HANDLED; + } + + /* Trigger main irq handler */ + return kbase_gpu_irq_handler(irq, data); +} + +/** + * mali_kutf_irq_default_create_fixture() - Creates the fixture data required + * for all the tests in the irq suite. + * @context: KUTF context. + * + * Return: Fixture data created on success or NULL on failure + */ +static void *mali_kutf_irq_default_create_fixture( + struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data; + + data = kutf_mempool_alloc(&context->fixture_pool, + sizeof(struct kutf_irq_fixture_data)); + + if (!data) + goto fail; + + /* Acquire the kbase device */ + data->kbdev = kbase_find_device(-1); + if (data->kbdev == NULL) { + kutf_test_fail(context, "Failed to find kbase device"); + goto fail; + } + + return data; + +fail: + return NULL; +} + +/** + * mali_kutf_irq_default_remove_fixture() - Destroy fixture data previously + * created by mali_kutf_irq_default_create_fixture. + * + * @context: KUTF context. + */ +static void mali_kutf_irq_default_remove_fixture( + struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + + kbase_release_device(kbdev); +} + +/** + * mali_kutf_irq_latency() - measure GPU IRQ latency + * @context: kutf context within which to perform the test + * + * The test triggers IRQs manually, and measures the + * time between triggering the IRQ and the IRQ handler being executed. + * + * This is not a traditional test, in that the pass/fail status has little + * meaning (other than indicating that the IRQ handler executed at all). Instead + * the results are in the latencies provided with the test result. There is no + * meaningful pass/fail result that can be obtained here, instead the latencies + * are provided for manual analysis only. + */ +static void mali_kutf_irq_latency(struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + u64 min_time = U64_MAX, max_time = 0, average_time = 0; + int i; + bool test_failed = false; + + /* Force GPU to be powered */ + kbase_pm_context_active(kbdev); + + kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler, + GPU_IRQ_HANDLER); + + for (i = 0; i < NR_TEST_IRQS; i++) { + struct timespec tval; + u64 start_time; + int ret; + + triggered = false; + getnstimeofday(&tval); + start_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); + + /* Trigger fake IRQ */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), + TEST_IRQ, NULL); + + ret = wait_event_timeout(wait, triggered != false, IRQ_TIMEOUT); + + if (ret == 0) { + kutf_test_fail(context, "Timed out waiting for IRQ\n"); + test_failed = true; + break; + } + + if ((irq_time - start_time) < min_time) + min_time = irq_time - start_time; + if ((irq_time - start_time) > max_time) + max_time = irq_time - start_time; + average_time += irq_time - start_time; + + udelay(10); + } + + /* Go back to default handler */ + kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_HANDLER); + + kbase_pm_context_idle(kbdev); + + if (!test_failed) { + const char *results; + + do_div(average_time, NR_TEST_IRQS); + results = kutf_dsprintf(&context->fixture_pool, + "Min latency = %lldns, Max latency = %lldns, Average latency = %lldns\n", + min_time, max_time, average_time); + kutf_test_pass(context, results); + } +} + +/** + * Module entry point for this test. + */ +int mali_kutf_irq_test_main_init(void) +{ + struct kutf_suite *suite; + + irq_app = kutf_create_application("irq"); + + if (NULL == irq_app) { + pr_warn("Creation of test application failed!\n"); + return -ENOMEM; + } + + suite = kutf_create_suite(irq_app, "irq_default", + 1, mali_kutf_irq_default_create_fixture, + mali_kutf_irq_default_remove_fixture); + + if (NULL == suite) { + pr_warn("Creation of test suite failed!\n"); + kutf_destroy_application(irq_app); + return -ENOMEM; + } + + kutf_add_test(suite, 0x0, "irq_latency", + mali_kutf_irq_latency); + return 0; +} + +/** + * Module exit point for this test. + */ +void mali_kutf_irq_test_main_exit(void) +{ + kutf_destroy_application(irq_app); +} + +module_init(mali_kutf_irq_test_main_init); +module_exit(mali_kutf_irq_test_main_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("ARM Ltd."); +MODULE_VERSION("1.0"); diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/sconscript b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/sconscript new file mode 100755 index 000000000000..b06d9ea32924 --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/mali_kutf_irq_test/sconscript @@ -0,0 +1,30 @@ +# +# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +import os +Import('env') + +src = [Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/*.c'), Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile')] + +if env.GetOption('clean') : + env.Execute(Action("make clean", '[CLEAN] mali_kutf_irq_test')) + cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, []) + env.KernelObjTarget('mali_kutf_irq_test', cmd) +else: + makeAction=Action("cd ${SOURCE.dir} && make MALI_BIFROST_DEBUG=${debug} MALI_BACKEND_KERNEL=1 MALI_ERROR_INJECT_ON=${error_inject} MALI_BIFROST_NO_MALI=${no_mali} MALI_UNIT_TEST=${unit} MALI_USE_UMP=${ump} MALI_CUSTOMER_RELEASE=${release} %s && ( ( [ -f mali_kutf_irq_test.ko ] && cp mali_kutf_irq_test.ko $STATIC_LIB_PATH/ ) || touch $STATIC_LIB_PATH/mali_kutf_irq_test.ko)" % env.kernel_get_config_defines(), '$MAKECOMSTR') + cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, [makeAction]) + env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/kutf.ko') + env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/mali_kbase.ko') + env.KernelObjTarget('mali_kutf_irq_test', cmd) diff --git a/drivers/gpu/arm/bifrost_for_linux/tests/sconscript b/drivers/gpu/arm/bifrost_for_linux/tests/sconscript new file mode 100755 index 000000000000..04584117ccef --- /dev/null +++ b/drivers/gpu/arm/bifrost_for_linux/tests/sconscript @@ -0,0 +1,38 @@ +# +# (C) COPYRIGHT 2010-2011, 2013, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +Import ('env') + +kutf_env = env.Clone() +kutf_env.Append(CPPPATH = '#kernel/drivers/gpu/arm/midgard/tests/include') +Export('kutf_env') + +if Glob('internal/sconscript'): + SConscript('internal/sconscript') + +if kutf_env['debug'] == '1': + SConscript('kutf/sconscript') + SConscript('mali_kutf_irq_test/sconscript') + + if Glob('kutf_test/sconscript'): + SConscript('kutf_test/sconscript') + + if Glob('kutf_test_runner/sconscript'): + SConscript('kutf_test_runner/sconscript') + +if env['unit'] == '1': + SConscript('mali_kutf_ipa_test/sconscript') + SConscript('mali_kutf_ipa_unit_test/sconscript') + SConscript('mali_kutf_vinstr_test/sconscript') diff --git a/drivers/gpu/arm/mali400/.gitignore b/drivers/gpu/arm/mali400/.gitignore new file mode 100755 index 000000000000..d91c8078a009 --- /dev/null +++ b/drivers/gpu/arm/mali400/.gitignore @@ -0,0 +1 @@ +./mali/__malidrv_build_info.c diff --git a/drivers/gpu/arm/mali400/Kbuild b/drivers/gpu/arm/mali400/Kbuild new file mode 100755 index 000000000000..dbb7ad3e5d85 --- /dev/null +++ b/drivers/gpu/arm/mali400/Kbuild @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-y += mali/ diff --git a/drivers/gpu/arm/mali400/mali/.gitignore b/drivers/gpu/arm/mali400/mali/.gitignore new file mode 100755 index 000000000000..6b1a3ed27a7f --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/.gitignore @@ -0,0 +1 @@ +__malidrv_build_info.c diff --git a/drivers/gpu/arm/mali400/mali/Kbuild b/drivers/gpu/arm/mali400/mali/Kbuild new file mode 100755 index 000000000000..7390ab758f22 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/Kbuild @@ -0,0 +1,254 @@ +# +# Copyright (C) 2010-2011 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the GNU General Public License version 2 +# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained from Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +# This file is called by the Linux build system. + +# make $(src) as absolute path if it isn't already, by prefixing $(srctree) +src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src)) + +# set up defaults if not defined by the user +TIMESTAMP ?= default +OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB ?= 16 +USING_GPU_UTILIZATION ?= 1 +PROFILING_SKIP_PP_JOBS ?= 0 +PROFILING_SKIP_PP_AND_GP_JOBS ?= 0 +MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP ?= 0 +MALI_PP_SCHEDULER_KEEP_SUB_JOB_STARTS_ALIGNED ?= 0 +MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP_BETWEEN_APPS ?= 0 +MALI_UPPER_HALF_SCHEDULING ?= 1 +MALI_ENABLE_CPU_CYCLES ?= 0 +MALI_PLATFORM ?= rk + +# For customer releases the Linux Device Drivers will be provided as ARM proprietary and GPL releases: +# The ARM proprietary product will only include the license/proprietary directory +# The GPL product will only include the license/gpl directory +ccflags-y += -I$(src)/linux/license/gpl + + +ifeq ($(USING_GPU_UTILIZATION), 1) + ifeq ($(USING_DVFS), 1) + $(error USING_GPU_UTILIZATION conflict with USING_DVFS you can read the Integration Guide to choose which one do you need) + endif +endif + +ifneq ($(MALI_PLATFORM),) + EXTRA_DEFINES += -DMALI_FAKE_PLATFORM_DEVICE=1 + #MALI_PLATFORM_FILES = $(wildcard platform/$(MALI_PLATFORM)/*.c) + mali-y += \ + platform/$(MALI_PLATFORM)/rk.o +endif + +ifeq ($(MALI_PLATFORM_FILES),) +ifeq ($(CONFIG_ARCH_EXYNOS4),y) +EXTRA_DEFINES += -DMALI_FAKE_PLATFORM_DEVICE=1 +export MALI_PLATFORM=exynos4 +export MALI_PLATFORM_FILES_BUILDIN = $(notdir $(wildcard $(src)/platform/$(MALI_PLATFORM)/*.c)) +export MALI_PLATFORM_FILES_ADD_PREFIX = $(addprefix platform/$(MALI_PLATFORM)/,$(MALI_PLATFORM_FILES_BUILDIN)) +endif +endif + +mali-y += \ + linux/mali_osk_atomics.o \ + linux/mali_osk_irq.o \ + linux/mali_osk_wq.o \ + linux/mali_osk_locks.o \ + linux/mali_osk_wait_queue.o \ + linux/mali_osk_low_level_mem.o \ + linux/mali_osk_math.o \ + linux/mali_osk_memory.o \ + linux/mali_osk_misc.o \ + linux/mali_osk_mali.o \ + linux/mali_osk_notification.o \ + linux/mali_osk_time.o \ + linux/mali_osk_timers.o \ + linux/mali_osk_bitmap.o + +mali-y += linux/mali_memory.o linux/mali_memory_os_alloc.o +mali-y += linux/mali_memory_external.o +mali-y += linux/mali_memory_block_alloc.o +mali-y += linux/mali_memory_swap_alloc.o + +mali-y += \ + linux/mali_memory_manager.o \ + linux/mali_memory_virtual.o \ + linux/mali_memory_util.o \ + linux/mali_memory_cow.o \ + linux/mali_memory_defer_bind.o + +mali-y += \ + linux/mali_ukk_mem.o \ + linux/mali_ukk_gp.o \ + linux/mali_ukk_pp.o \ + linux/mali_ukk_core.o \ + linux/mali_ukk_soft_job.o \ + linux/mali_ukk_timeline.o + +mali-$(CONFIG_MALI_DEVFREQ) += \ + linux/mali_devfreq.o \ + common/mali_pm_metrics.o + +# Source files which always are included in a build +mali-y += \ + common/mali_kernel_core.o \ + linux/mali_kernel_linux.o \ + common/mali_session.o \ + linux/mali_device_pause_resume.o \ + common/mali_kernel_vsync.o \ + linux/mali_ukk_vsync.o \ + linux/mali_kernel_sysfs.o \ + common/mali_mmu.o \ + common/mali_mmu_page_directory.o \ + common/mali_mem_validation.o \ + common/mali_hw_core.o \ + common/mali_gp.o \ + common/mali_pp.o \ + common/mali_pp_job.o \ + common/mali_gp_job.o \ + common/mali_soft_job.o \ + common/mali_scheduler.o \ + common/mali_executor.o \ + common/mali_group.o \ + common/mali_dlbu.o \ + common/mali_broadcast.o \ + common/mali_pm.o \ + common/mali_pmu.o \ + common/mali_user_settings_db.o \ + common/mali_kernel_utilization.o \ + common/mali_control_timer.o \ + common/mali_l2_cache.o \ + common/mali_timeline.o \ + common/mali_timeline_fence_wait.o \ + common/mali_timeline_sync_fence.o \ + common/mali_spinlock_reentrant.o \ + common/mali_pm_domain.o \ + linux/mali_osk_pm.o \ + linux/mali_pmu_power_up_down.o \ + __malidrv_build_info.o + +ifneq ($(wildcard $(src)/linux/mali_slp_global_lock.c),) + mali-y += linux/mali_slp_global_lock.o +endif + +ifneq ($(MALI_PLATFORM_FILES),) + mali-y += $(MALI_PLATFORM_FILES:.c=.o) +endif + +ifneq ($(MALI_PLATFORM_FILES_ADD_PREFIX),) + mali-y += $(MALI_PLATFORM_FILES_ADD_PREFIX:.c=.o) +endif + +mali-$(CONFIG_MALI400_PROFILING) += linux/mali_ukk_profiling.o +mali-$(CONFIG_MALI400_PROFILING) += linux/mali_osk_profiling.o + +mali-$(CONFIG_MALI400_INTERNAL_PROFILING) += linux/mali_profiling_internal.o timestamp-$(TIMESTAMP)/mali_timestamp.o +ccflags-$(CONFIG_MALI400_INTERNAL_PROFILING) += -I$(src)/timestamp-$(TIMESTAMP) + +mali-$(CONFIG_DMA_SHARED_BUFFER) += linux/mali_memory_dma_buf.o +mali-$(CONFIG_DMA_SHARED_BUFFER) += linux/mali_memory_secure.o +mali-$(CONFIG_SYNC) += linux/mali_sync.o +mali-$(CONFIG_SYNC) += linux/mali_internal_sync.o +mali-$(CONFIG_SYNC_FILE) += linux/mali_sync.o +mali-$(CONFIG_SYNC_FILE) += linux/mali_internal_sync.o +mali-$(CONFIG_MALI_DMA_BUF_FENCE) += linux/mali_dma_fence.o +ccflags-$(CONFIG_SYNC) += -Idrivers/staging/android +ccflags-$(CONFIG_SYNC_FILE) += -Idrivers/staging/android + +mali-$(CONFIG_MALI400_UMP) += linux/mali_memory_ump.o + +mali-$(CONFIG_MALI_DVFS) += common/mali_dvfs_policy.o + +# Tell the Linux build system from which .o file to create the kernel module +obj-$(CONFIG_MALI400) := mali.o + +ccflags-y += $(EXTRA_DEFINES) + +# Set up our defines, which will be passed to gcc +ccflags-y += -DMALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP=$(MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP) +ccflags-y += -DMALI_PP_SCHEDULER_KEEP_SUB_JOB_STARTS_ALIGNED=$(MALI_PP_SCHEDULER_KEEP_SUB_JOB_STARTS_ALIGNED) +ccflags-y += -DMALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP_BETWEEN_APPS=$(MALI_PP_SCHEDULER_FORCE_NO_JOB_OVERLAP_BETWEEN_APPS) +ccflags-y += -DMALI_STATE_TRACKING=1 +ccflags-y += -DMALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB=$(OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB) +ccflags-y += -DUSING_GPU_UTILIZATION=$(USING_GPU_UTILIZATION) +ccflags-y += -DMALI_ENABLE_CPU_CYCLES=$(MALI_ENABLE_CPU_CYCLES) + +ifeq ($(MALI_UPPER_HALF_SCHEDULING),1) + ccflags-y += -DMALI_UPPER_HALF_SCHEDULING +endif + +#build-in include path is different +ifeq ($(MALI_PLATFORM_FILES),) +ccflags-$(CONFIG_MALI400_UMP) += -I$(src)/../ump/include/ +else +ccflags-$(CONFIG_MALI400_UMP) += -I$(src)/../../ump/include/ump +endif +ccflags-$(CONFIG_MALI400_DEBUG) += -DDEBUG + +# Use our defines when compiling +ccflags-y += -I$(src) -I$(src)/include -I$(src)/common -I$(src)/linux -I$(src)/platform -Wno-date-time + +# Get subversion revision number, fall back to only ${MALI_RELEASE_NAME} if no svn info is available +MALI_RELEASE_NAME=$(shell cat $(src)/.version 2> /dev/null) + +SVN_INFO = (cd $(src); svn info 2>/dev/null) + +ifneq ($(shell $(SVN_INFO) 2>/dev/null),) +# SVN detected +SVN_REV := $(shell $(SVN_INFO) | grep '^Revision: '| sed -e 's/^Revision: //' 2>/dev/null) +DRIVER_REV := $(MALI_RELEASE_NAME)-r$(SVN_REV) +CHANGE_DATE := $(shell $(SVN_INFO) | grep '^Last Changed Date: ' | cut -d: -f2- | cut -b2-) +CHANGED_REVISION := $(shell $(SVN_INFO) | grep '^Last Changed Rev: ' | cut -d: -f2- | cut -b2-) +REPO_URL := $(shell $(SVN_INFO) | grep '^URL: ' | cut -d: -f2- | cut -b2-) + +else # SVN +# GIT_REV := $(shell cd $(src); git describe --always 2>/dev/null) +ifneq ($(GIT_REV),) +# Git detected +DRIVER_REV := $(MALI_RELEASE_NAME)-$(GIT_REV) +CHANGE_DATE := $(shell cd $(src); git log -1 --format="%ci") +CHANGED_REVISION := $(GIT_REV) +REPO_URL := $(shell cd $(src); git describe --all --always 2>/dev/null) + +else # Git +# No Git or SVN detected +DRIVER_REV := $(MALI_RELEASE_NAME) +CHANGE_DATE := $(MALI_RELEASE_NAME) +CHANGED_REVISION := $(MALI_RELEASE_NAME) +endif +endif + +ccflags-y += -DSVN_REV_STRING=\"$(DRIVER_REV)\" + +VERSION_STRINGS := +VERSION_STRINGS += API_VERSION=$(shell cd $(src); grep "\#define _MALI_API_VERSION" $(FILES_PREFIX)include/linux/mali/mali_utgard_uk_types.h | cut -d' ' -f 3 ) +VERSION_STRINGS += REPO_URL=$(REPO_URL) +VERSION_STRINGS += REVISION=$(DRIVER_REV) +VERSION_STRINGS += CHANGED_REVISION=$(CHANGED_REVISION) +VERSION_STRINGS += CHANGE_DATE=$(CHANGE_DATE) +VERSION_STRINGS += BUILD_DATE=$(shell date) +ifdef CONFIG_MALI400_DEBUG +VERSION_STRINGS += BUILD=debug +else +VERSION_STRINGS += BUILD=release +endif +VERSION_STRINGS += TARGET_PLATFORM=$(TARGET_PLATFORM) +VERSION_STRINGS += MALI_PLATFORM=$(MALI_PLATFORM) +VERSION_STRINGS += KDIR=$(KDIR) +VERSION_STRINGS += OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB=$(OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB) +VERSION_STRINGS += USING_UMP=$(CONFIG_MALI400_UMP) +VERSION_STRINGS += USING_PROFILING=$(CONFIG_MALI400_PROFILING) +VERSION_STRINGS += USING_INTERNAL_PROFILING=$(CONFIG_MALI400_INTERNAL_PROFILING) +VERSION_STRINGS += USING_GPU_UTILIZATION=$(USING_GPU_UTILIZATION) +VERSION_STRINGS += USING_DVFS=$(CONFIG_MALI_DVFS) +VERSION_STRINGS += USING_DMA_BUF_FENCE = $(CONFIG_MALI_DMA_BUF_FENCE) +VERSION_STRINGS += MALI_UPPER_HALF_SCHEDULING=$(MALI_UPPER_HALF_SCHEDULING) + +# Create file with Mali driver configuration +$(src)/__malidrv_build_info.c: + @echo 'const char *__malidrv_build_info(void) { return "malidrv: $(VERSION_STRINGS)";}' > $(src)/__malidrv_build_info.c diff --git a/drivers/gpu/arm/mali400/mali/Kconfig b/drivers/gpu/arm/mali400/mali/Kconfig new file mode 100755 index 000000000000..34c5f72edcda --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/Kconfig @@ -0,0 +1,118 @@ +# SPDX-License-Identifier: GPL-2.0 +config MALI400 + tristate "Mali-300/400/450 support" + depends on ARM || ARM64 + select DMA_SHARED_BUFFER + help + This enables support for the ARM Mali-300, Mali-400, and Mali-450 + GPUs. + + To compile this driver as a module, choose M here: the module will be + called mali. + +config MALI450 + bool "Enable Mali-450 support" + depends on MALI400 + help + This enables support for Mali-450 specific features. + +config MALI470 + bool "Enable Mali-470 support" + depends on MALI400 + help + This enables support for Mali-470 specific features. + +config MALI400_DEBUG + bool "Enable debug in Mali driver" + depends on MALI400 + help + This enabled extra debug checks and messages in the Mali driver. + +config MALI400_PROFILING + bool "Enable Mali profiling" + depends on MALI400 + select TRACEPOINTS + default y + help + This enables gator profiling of Mali GPU events. + +config MALI400_INTERNAL_PROFILING + bool "Enable internal Mali profiling API" + depends on MALI400_PROFILING + default n + help + This enables the internal legacy Mali profiling API. + +config MALI400_UMP + bool "Enable UMP support" + depends on MALI400 + help + This enables support for the UMP memory sharing API in the Mali driver. + +config MALI_DVFS + bool "Enable Mali dynamically frequency change" + depends on MALI400 && !MALI_DEVFREQ + default y + help + This enables support for dynamic change frequency of Mali with the goal of lowering power consumption. + +config MALI_DMA_BUF_MAP_ON_ATTACH + bool "Map dma-buf attachments on attach" + depends on MALI400 && DMA_SHARED_BUFFER + default y + help + This makes the Mali driver map dma-buf attachments after doing + attach. If this is not set the dma-buf attachments will be mapped for + every time the GPU need to access the buffer. + + Mapping for each access can cause lower performance. + +config MALI_SHARED_INTERRUPTS + bool "Support for shared interrupts" + depends on MALI400 + default n + help + Adds functionality required to properly support shared interrupts. Without this support, + the device driver will fail during insmod if it detects shared interrupts. This also + works when the GPU is not using shared interrupts, but might have a slight performance + impact. + +config MALI_PMU_PARALLEL_POWER_UP + bool "Power up Mali PMU domains in parallel" + depends on MALI400 + default n + help + This makes the Mali driver power up all PMU power domains in parallel, instead of + powering up domains one by one, with a slight delay in between. Powering on all power + domains at the same time may cause peak currents higher than what some systems can handle. + These systems must not enable this option. + +config MALI_DT + bool "Using device tree to initialize module" + depends on MALI400 && OF + default n + help + This enable the Mali driver to choose the device tree path to get platform resoures + and disable the old config method. Mali driver could run on the platform which the + device tree is enabled in kernel and corresponding hardware description is implemented + properly in device DTS file. + +config MALI_DEVFREQ + bool "Using devfreq to tuning frequency" + depends on MALI400 && PM_DEVFREQ + default n + help + Support devfreq for Mali. + + Using the devfreq framework and, by default, the simpleondemand + governor, the frequency of Mali will be dynamically selected from the + available OPPs. + +config MALI_QUIET + bool "Make Mali driver very quiet" + depends on MALI400 && !MALI400_DEBUG + default n + help + This forces the Mali driver to never print any messages. + + If unsure, say N. diff --git a/drivers/gpu/arm/mali400/mali/Makefile b/drivers/gpu/arm/mali400/mali/Makefile new file mode 100755 index 000000000000..0b91321a5af1 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/Makefile @@ -0,0 +1,206 @@ +# +# Copyright (C) 2010-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the GNU General Public License version 2 +# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained from Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +USE_UMPV2=0 +USING_PROFILING ?= 1 +USING_INTERNAL_PROFILING ?= 0 +USING_DVFS ?= 1 +USING_DMA_BUF_FENCE ?= 0 +MALI_HEATMAPS_ENABLED ?= 0 +MALI_DMA_BUF_MAP_ON_ATTACH ?= 1 +MALI_PMU_PARALLEL_POWER_UP ?= 0 +USING_DT ?= 0 +MALI_MEM_SWAP_TRACKING ?= 0 +USING_DEVFREQ ?= 0 + +# The Makefile sets up "arch" based on the CONFIG, creates the version info +# string and the __malidrv_build_info.c file, and then call the Linux build +# system to actually build the driver. After that point the Kbuild file takes +# over. + +# set up defaults if not defined by the user +ARCH ?= arm + +OSKOS=linux +FILES_PREFIX= + +check_cc2 = \ + $(shell if $(1) -S -o /dev/null -xc /dev/null > /dev/null 2>&1; \ + then \ + echo "$(2)"; \ + else \ + echo "$(3)"; \ + fi ;) + +# This conditional makefile exports the global definition ARM_INTERNAL_BUILD. Customer releases will not include arm_internal.mak +-include ../../../arm_internal.mak + +# Give warning of old config parameters are used +ifneq ($(CONFIG),) +$(warning "You have specified the CONFIG variable which is no longer in used. Use TARGET_PLATFORM instead.") +endif + +ifneq ($(CPU),) +$(warning "You have specified the CPU variable which is no longer in used. Use TARGET_PLATFORM instead.") +endif + +# Include the mapping between TARGET_PLATFORM and KDIR + MALI_PLATFORM +-include MALI_CONFIGURATION +export KDIR ?= $(KDIR-$(TARGET_PLATFORM)) +export MALI_PLATFORM ?= $(MALI_PLATFORM-$(TARGET_PLATFORM)) + +ifneq ($(TARGET_PLATFORM),) +ifeq ($(MALI_PLATFORM),) +$(error "Invalid TARGET_PLATFORM: $(TARGET_PLATFORM)") +endif +endif + +# validate lookup result +ifeq ($(KDIR),) +$(error No KDIR found for platform $(TARGET_PLATFORM)) +endif + +ifeq ($(USING_GPU_UTILIZATION), 1) + ifeq ($(USING_DVFS), 1) + $(error USING_GPU_UTILIZATION conflict with USING_DVFS you can read the Integration Guide to choose which one do you need) + endif +endif + +ifeq ($(USING_UMP),1) +export CONFIG_MALI400_UMP=y +export EXTRA_DEFINES += -DCONFIG_MALI400_UMP=1 +ifeq ($(USE_UMPV2),1) +UMP_SYMVERS_FILE ?= ../umpv2/Module.symvers +else +UMP_SYMVERS_FILE ?= ../ump/Module.symvers +endif +KBUILD_EXTRA_SYMBOLS = $(realpath $(UMP_SYMVERS_FILE)) +$(warning $(KBUILD_EXTRA_SYMBOLS)) +endif + +# Define host system directory +KDIR-$(shell uname -m):=/lib/modules/$(shell uname -r)/build + +include $(KDIR)/.config + +ifeq ($(ARCH), arm) +# when compiling for ARM we're cross compiling +export CROSS_COMPILE ?= $(call check_cc2, arm-linux-gnueabi-gcc, arm-linux-gnueabi-, arm-none-linux-gnueabi-) +endif + +# report detected/selected settings +ifdef ARM_INTERNAL_BUILD +$(warning TARGET_PLATFORM $(TARGET_PLATFORM)) +$(warning KDIR $(KDIR)) +$(warning MALI_PLATFORM $(MALI_PLATFORM)) +endif + +# Set up build config +export CONFIG_MALI400=m +export CONFIG_MALI450=y +export CONFIG_MALI470=y + +export EXTRA_DEFINES += -DCONFIG_MALI400=1 +export EXTRA_DEFINES += -DCONFIG_MALI450=1 +export EXTRA_DEFINES += -DCONFIG_MALI470=1 + +ifneq ($(MALI_PLATFORM),) +export EXTRA_DEFINES += -DMALI_FAKE_PLATFORM_DEVICE=1 +export MALI_PLATFORM_FILES = $(wildcard platform/$(MALI_PLATFORM)/*.c) +endif + +ifeq ($(USING_PROFILING),1) +ifeq ($(CONFIG_TRACEPOINTS),) +$(warning CONFIG_TRACEPOINTS required for profiling) +else +export CONFIG_MALI400_PROFILING=y +export EXTRA_DEFINES += -DCONFIG_MALI400_PROFILING=1 +ifeq ($(USING_INTERNAL_PROFILING),1) +export CONFIG_MALI400_INTERNAL_PROFILING=y +export EXTRA_DEFINES += -DCONFIG_MALI400_INTERNAL_PROFILING=1 +endif +ifeq ($(MALI_HEATMAPS_ENABLED),1) +export MALI_HEATMAPS_ENABLED=y +export EXTRA_DEFINES += -DCONFIG_MALI400_HEATMAPS_ENABLED +endif +endif +endif + +ifeq ($(MALI_DMA_BUF_MAP_ON_ATTACH),1) +export CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH=y +export EXTRA_DEFINES += -DCONFIG_MALI_DMA_BUF_MAP_ON_ATTACH +endif + +ifeq ($(MALI_SHARED_INTERRUPTS),1) +export CONFIG_MALI_SHARED_INTERRUPTS=y +export EXTRA_DEFINES += -DCONFIG_MALI_SHARED_INTERRUPTS +endif + +ifeq ($(USING_DVFS),1) +export CONFIG_MALI_DVFS=y +export EXTRA_DEFINES += -DCONFIG_MALI_DVFS +endif + +ifeq ($(USING_DMA_BUF_FENCE),1) +export CONFIG_MALI_DMA_BUF_FENCE=y +export EXTRA_DEFINES += -DCONFIG_MALI_DMA_BUF_FENCE +endif + +ifeq ($(MALI_PMU_PARALLEL_POWER_UP),1) +export CONFIG_MALI_PMU_PARALLEL_POWER_UP=y +export EXTRA_DEFINES += -DCONFIG_MALI_PMU_PARALLEL_POWER_UP +endif + +ifdef CONFIG_OF +ifeq ($(USING_DT),1) +export CONFIG_MALI_DT=y +export EXTRA_DEFINES += -DCONFIG_MALI_DT +endif +endif + +ifeq ($(USING_DEVFREQ), 1) +ifdef CONFIG_PM_DEVFREQ +export CONFIG_MALI_DEVFREQ=y +export EXTRA_DEFINES += -DCONFIG_MALI_DEVFREQ=1 +else +$(warning "You want to support DEVFREQ but kernel didn't support DEVFREQ.") +endif +endif + +ifneq ($(BUILD),release) +# Debug +export CONFIG_MALI400_DEBUG=y +else +# Release +ifeq ($(MALI_QUIET),1) +export CONFIG_MALI_QUIET=y +export EXTRA_DEFINES += -DCONFIG_MALI_QUIET +endif +endif + +ifeq ($(MALI_SKIP_JOBS),1) +EXTRA_DEFINES += -DPROFILING_SKIP_PP_JOBS=1 -DPROFILING_SKIP_GP_JOBS=1 +endif + +ifeq ($(MALI_MEM_SWAP_TRACKING),1) +EXTRA_DEFINES += -DMALI_MEM_SWAP_TRACKING=1 +endif + +all: $(UMP_SYMVERS_FILE) + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) modules + @rm $(FILES_PREFIX)__malidrv_build_info.c $(FILES_PREFIX)__malidrv_build_info.o + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean + +kernelrelease: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) kernelrelease + +export CONFIG KBUILD_EXTRA_SYMBOLS diff --git a/drivers/gpu/arm/mali400/mali/common/mali_broadcast.c b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.c new file mode 100755 index 000000000000..79a418c36ccb --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.c @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_broadcast.h" +#include "mali_kernel_common.h" +#include "mali_osk.h" + +#define MALI_BROADCAST_REGISTER_SIZE 0x1000 +#define MALI_BROADCAST_REG_BROADCAST_MASK 0x0 +#define MALI_BROADCAST_REG_INTERRUPT_MASK 0x4 + +struct mali_bcast_unit { + struct mali_hw_core hw_core; + u32 current_mask; +}; + +struct mali_bcast_unit *mali_bcast_unit_create(const _mali_osk_resource_t *resource) +{ + struct mali_bcast_unit *bcast_unit = NULL; + + MALI_DEBUG_ASSERT_POINTER(resource); + MALI_DEBUG_PRINT(2, ("Broadcast: Creating Mali Broadcast unit: %s\n", + resource->description)); + + bcast_unit = _mali_osk_malloc(sizeof(struct mali_bcast_unit)); + if (NULL == bcast_unit) { + MALI_PRINT_ERROR(("Broadcast: Failed to allocate memory for Broadcast unit\n")); + return NULL; + } + + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&bcast_unit->hw_core, + resource, MALI_BROADCAST_REGISTER_SIZE)) { + bcast_unit->current_mask = 0; + mali_bcast_reset(bcast_unit); + + return bcast_unit; + } else { + MALI_PRINT_ERROR(("Broadcast: Failed map broadcast unit\n")); + } + + _mali_osk_free(bcast_unit); + + return NULL; +} + +void mali_bcast_unit_delete(struct mali_bcast_unit *bcast_unit) +{ + MALI_DEBUG_ASSERT_POINTER(bcast_unit); + mali_hw_core_delete(&bcast_unit->hw_core); + _mali_osk_free(bcast_unit); +} + +/* Call this function to add the @group's id into bcast mask + * Note: redundant calling this function with same @group + * doesn't make any difference as calling it once + */ +void mali_bcast_add_group(struct mali_bcast_unit *bcast_unit, + struct mali_group *group) +{ + u32 bcast_id; + u32 broadcast_mask; + + MALI_DEBUG_ASSERT_POINTER(bcast_unit); + MALI_DEBUG_ASSERT_POINTER(group); + + bcast_id = mali_pp_core_get_bcast_id(mali_group_get_pp_core(group)); + + broadcast_mask = bcast_unit->current_mask; + + broadcast_mask |= (bcast_id); /* add PP core to broadcast */ + broadcast_mask |= (bcast_id << 16); /* add MMU to broadcast */ + + /* store mask so we can restore on reset */ + bcast_unit->current_mask = broadcast_mask; +} + +/* Call this function to remove @group's id from bcast mask + * Note: redundant calling this function with same @group + * doesn't make any difference as calling it once + */ +void mali_bcast_remove_group(struct mali_bcast_unit *bcast_unit, + struct mali_group *group) +{ + u32 bcast_id; + u32 broadcast_mask; + + MALI_DEBUG_ASSERT_POINTER(bcast_unit); + MALI_DEBUG_ASSERT_POINTER(group); + + bcast_id = mali_pp_core_get_bcast_id(mali_group_get_pp_core(group)); + + broadcast_mask = bcast_unit->current_mask; + + broadcast_mask &= ~((bcast_id << 16) | bcast_id); + + /* store mask so we can restore on reset */ + bcast_unit->current_mask = broadcast_mask; +} + +void mali_bcast_reset(struct mali_bcast_unit *bcast_unit) +{ + MALI_DEBUG_ASSERT_POINTER(bcast_unit); + + MALI_DEBUG_PRINT(4, + ("Broadcast: setting mask 0x%08X + 0x%08X (reset)\n", + bcast_unit->current_mask, + bcast_unit->current_mask & 0xFF)); + + /* set broadcast mask */ + mali_hw_core_register_write(&bcast_unit->hw_core, + MALI_BROADCAST_REG_BROADCAST_MASK, + bcast_unit->current_mask); + + /* set IRQ override mask */ + mali_hw_core_register_write(&bcast_unit->hw_core, + MALI_BROADCAST_REG_INTERRUPT_MASK, + bcast_unit->current_mask & 0xFF); +} + +void mali_bcast_disable(struct mali_bcast_unit *bcast_unit) +{ + MALI_DEBUG_ASSERT_POINTER(bcast_unit); + + MALI_DEBUG_PRINT(4, ("Broadcast: setting mask 0x0 + 0x0 (disable)\n")); + + /* set broadcast mask */ + mali_hw_core_register_write(&bcast_unit->hw_core, + MALI_BROADCAST_REG_BROADCAST_MASK, + 0x0); + + /* set IRQ override mask */ + mali_hw_core_register_write(&bcast_unit->hw_core, + MALI_BROADCAST_REG_INTERRUPT_MASK, + 0x0); +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_broadcast.h b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.h new file mode 100755 index 000000000000..0475b7171d8d --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_broadcast.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_BROADCAST_H__ +#define __MALI_BROADCAST_H__ + +/* + * Interface for the broadcast unit on Mali-450. + * + * - Represents up to 8 × (MMU + PP) pairs. + * - Supports dynamically changing which (MMU + PP) pairs receive the broadcast by + * setting a mask. + */ + +#include "mali_hw_core.h" +#include "mali_group.h" + +struct mali_bcast_unit; + +struct mali_bcast_unit *mali_bcast_unit_create(const _mali_osk_resource_t *resource); +void mali_bcast_unit_delete(struct mali_bcast_unit *bcast_unit); + +/* Add a group to the list of (MMU + PP) pairs broadcasts go out to. */ +void mali_bcast_add_group(struct mali_bcast_unit *bcast_unit, struct mali_group *group); + +/* Remove a group to the list of (MMU + PP) pairs broadcasts go out to. */ +void mali_bcast_remove_group(struct mali_bcast_unit *bcast_unit, struct mali_group *group); + +/* Re-set cached mask. This needs to be called after having been suspended. */ +void mali_bcast_reset(struct mali_bcast_unit *bcast_unit); + +/** + * Disable broadcast unit + * + * mali_bcast_enable must be called to re-enable the unit. Cores may not be + * added or removed when the unit is disabled. + */ +void mali_bcast_disable(struct mali_bcast_unit *bcast_unit); + +/** + * Re-enable broadcast unit + * + * This resets the masks to include the cores present when mali_bcast_disable was called. + */ +MALI_STATIC_INLINE void mali_bcast_enable(struct mali_bcast_unit *bcast_unit) +{ + mali_bcast_reset(bcast_unit); +} + +#endif /* __MALI_BROADCAST_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_control_timer.c b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.c new file mode 100755 index 000000000000..5bed27a8c5c9 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.c @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2010-2012, 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_utilization.h" +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_dvfs_policy.h" +#include "mali_control_timer.h" + +static u64 period_start_time = 0; + +/** .KP : mali_control_timer */ +static _mali_osk_timer_t *mali_control_timer = NULL; +static mali_bool timer_running = MALI_FALSE; + +/** + * period_of_notifying_mali_utilization_to_platform_dependent_part, + * ms 为å•ä½. + */ +static u32 mali_control_timeout = 20; + +void mali_control_timer_add(u32 timeout)/* 'timeout' : 以 ms 为å•ä½. */ +{ + _mali_osk_timer_add(mali_control_timer, _mali_osk_time_mstoticks(timeout)); +} + +void mali_control_timer_mod(u32 timeout_in_ms) +{ + _mali_osk_timer_mod(mali_control_timer, _mali_osk_time_mstoticks(timeout_in_ms)); +} + +static void mali_control_timer_callback(void *arg) +{ + if (mali_utilization_enabled()) { + struct mali_gpu_utilization_data *util_data = NULL; + u64 time_period = 0; + mali_bool need_add_timer = MALI_TRUE; + + /* Calculate gpu utilization */ + util_data = mali_utilization_calculate(&period_start_time, &time_period, &need_add_timer); + + if (util_data) { +#if defined(CONFIG_MALI_DVFS) + mali_dvfs_policy_realize(util_data, time_period); +#else + mali_utilization_platform_realize(util_data); +#endif + + if (MALI_TRUE == timer_running) + if (MALI_TRUE == need_add_timer) { + mali_control_timer_mod(mali_control_timeout); + } + } + } +} + +/* Init a timer (for now it is used for GPU utilization and dvfs) */ +_mali_osk_errcode_t mali_control_timer_init(void) +{ + _mali_osk_device_data data; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + /* Use device specific settings (if defined) */ + if (0 != data.control_interval) { + mali_control_timeout = data.control_interval; + MALI_DEBUG_PRINT(2, ("Mali GPU Timer: %u\n", mali_control_timeout)); + } + } + + mali_control_timer = _mali_osk_timer_init(mali_control_timer_callback); + if (NULL == mali_control_timer) { + return _MALI_OSK_ERR_FAULT; + } + _mali_osk_timer_setcallback(mali_control_timer, mali_control_timer_callback, NULL); + + return _MALI_OSK_ERR_OK; +} + +void mali_control_timer_term(void) +{ + if (NULL != mali_control_timer) { + _mali_osk_timer_del(mali_control_timer); + timer_running = MALI_FALSE; + _mali_osk_timer_term(mali_control_timer); + mali_control_timer = NULL; + } +} + +mali_bool mali_control_timer_resume(u64 time_now) +{ + mali_utilization_data_assert_locked(); + + if (timer_running != MALI_TRUE) { + timer_running = MALI_TRUE; + + period_start_time = time_now; + + mali_utilization_reset(); + + return MALI_TRUE; + } + + return MALI_FALSE; +} + +void mali_control_timer_pause(void) +{ + mali_utilization_data_assert_locked(); + if (timer_running == MALI_TRUE) { + timer_running = MALI_FALSE; + } +} + +void mali_control_timer_suspend(mali_bool suspend) +{ + mali_utilization_data_lock(); + + if (timer_running == MALI_TRUE) { + timer_running = MALI_FALSE; + + mali_utilization_data_unlock(); + + if (suspend == MALI_TRUE) { + _mali_osk_timer_del(mali_control_timer); + mali_utilization_reset(); + } + } else { + mali_utilization_data_unlock(); + } +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_control_timer.h b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.h new file mode 100755 index 000000000000..c9e6e058ea8e --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_control_timer.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2010-2012, 2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_CONTROL_TIMER_H__ +#define __MALI_CONTROL_TIMER_H__ + +#include "mali_osk.h" + +_mali_osk_errcode_t mali_control_timer_init(void); + +void mali_control_timer_term(void); + +mali_bool mali_control_timer_resume(u64 time_now); + +void mali_control_timer_suspend(mali_bool suspend); +void mali_control_timer_pause(void); + +void mali_control_timer_add(u32 timeout); + +void mali_control_timer_mod(u32 timeout_in_ms); + +#endif /* __MALI_CONTROL_TIMER_H__ */ + diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dlbu.c b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.c new file mode 100755 index 000000000000..99b7f360768b --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.c @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_dlbu.h" +#include "mali_memory.h" +#include "mali_pp.h" +#include "mali_group.h" +#include "mali_osk.h" +#include "mali_hw_core.h" + +/** + * Size of DLBU registers in bytes + */ +#define MALI_DLBU_SIZE 0x400 + +mali_dma_addr mali_dlbu_phys_addr = 0; +static mali_io_address mali_dlbu_cpu_addr = NULL; + +/** + * DLBU register numbers + * Used in the register read/write routines. + * See the hardware documentation for more information about each register + */ +typedef enum mali_dlbu_register { + MALI_DLBU_REGISTER_MASTER_TLLIST_PHYS_ADDR = 0x0000, /**< Master tile list physical base address; + 31:12 Physical address to the page used for the DLBU + 0 DLBU enable - set this bit to 1 enables the AXI bus + between PPs and L2s, setting to 0 disables the router and + no further transactions are sent to DLBU */ + MALI_DLBU_REGISTER_MASTER_TLLIST_VADDR = 0x0004, /**< Master tile list virtual base address; + 31:12 Virtual address to the page used for the DLBU */ + MALI_DLBU_REGISTER_TLLIST_VBASEADDR = 0x0008, /**< Tile list virtual base address; + 31:12 Virtual address to the tile list. This address is used when + calculating the call address sent to PP.*/ + MALI_DLBU_REGISTER_FB_DIM = 0x000C, /**< Framebuffer dimension; + 23:16 Number of tiles in Y direction-1 + 7:0 Number of tiles in X direction-1 */ + MALI_DLBU_REGISTER_TLLIST_CONF = 0x0010, /**< Tile list configuration; + 29:28 select the size of each allocated block: 0=128 bytes, 1=256, 2=512, 3=1024 + 21:16 2^n number of tiles to be binned to one tile list in Y direction + 5:0 2^n number of tiles to be binned to one tile list in X direction */ + MALI_DLBU_REGISTER_START_TILE_POS = 0x0014, /**< Start tile positions; + 31:24 start position in Y direction for group 1 + 23:16 start position in X direction for group 1 + 15:8 start position in Y direction for group 0 + 7:0 start position in X direction for group 0 */ + MALI_DLBU_REGISTER_PP_ENABLE_MASK = 0x0018, /**< PP enable mask; + 7 enable PP7 for load balancing + 6 enable PP6 for load balancing + 5 enable PP5 for load balancing + 4 enable PP4 for load balancing + 3 enable PP3 for load balancing + 2 enable PP2 for load balancing + 1 enable PP1 for load balancing + 0 enable PP0 for load balancing */ +} mali_dlbu_register; + +typedef enum { + PP0ENABLE = 0, + PP1ENABLE, + PP2ENABLE, + PP3ENABLE, + PP4ENABLE, + PP5ENABLE, + PP6ENABLE, + PP7ENABLE +} mali_dlbu_pp_enable; + +struct mali_dlbu_core { + struct mali_hw_core hw_core; /**< Common for all HW cores */ + u32 pp_cores_mask; /**< This is a mask for the PP cores whose operation will be controlled by LBU + see MALI_DLBU_REGISTER_PP_ENABLE_MASK register */ +}; + +_mali_osk_errcode_t mali_dlbu_initialize(void) +{ + MALI_DEBUG_PRINT(2, ("Mali DLBU: Initializing\n")); + + if (_MALI_OSK_ERR_OK == + mali_mmu_get_table_page(&mali_dlbu_phys_addr, + &mali_dlbu_cpu_addr)) { + return _MALI_OSK_ERR_OK; + } + + return _MALI_OSK_ERR_FAULT; +} + +void mali_dlbu_terminate(void) +{ + MALI_DEBUG_PRINT(3, ("Mali DLBU: terminating\n")); + + if (0 != mali_dlbu_phys_addr && 0 != mali_dlbu_cpu_addr) { + mali_mmu_release_table_page(mali_dlbu_phys_addr, + mali_dlbu_cpu_addr); + mali_dlbu_phys_addr = 0; + mali_dlbu_cpu_addr = 0; + } +} + +struct mali_dlbu_core *mali_dlbu_create(const _mali_osk_resource_t *resource) +{ + struct mali_dlbu_core *core = NULL; + + MALI_DEBUG_PRINT(2, ("Mali DLBU: Creating Mali dynamic load balancing unit: %s\n", resource->description)); + + core = _mali_osk_malloc(sizeof(struct mali_dlbu_core)); + if (NULL != core) { + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&core->hw_core, resource, MALI_DLBU_SIZE)) { + core->pp_cores_mask = 0; + if (_MALI_OSK_ERR_OK == mali_dlbu_reset(core)) { + return core; + } + MALI_PRINT_ERROR(("Failed to reset DLBU %s\n", core->hw_core.description)); + mali_hw_core_delete(&core->hw_core); + } + + _mali_osk_free(core); + } else { + MALI_PRINT_ERROR(("Mali DLBU: Failed to allocate memory for DLBU core\n")); + } + + return NULL; +} + +void mali_dlbu_delete(struct mali_dlbu_core *dlbu) +{ + MALI_DEBUG_ASSERT_POINTER(dlbu); + mali_hw_core_delete(&dlbu->hw_core); + _mali_osk_free(dlbu); +} + +_mali_osk_errcode_t mali_dlbu_reset(struct mali_dlbu_core *dlbu) +{ + u32 dlbu_registers[7]; + _mali_osk_errcode_t err = _MALI_OSK_ERR_FAULT; + MALI_DEBUG_ASSERT_POINTER(dlbu); + + MALI_DEBUG_PRINT(4, ("Mali DLBU: mali_dlbu_reset: %s\n", dlbu->hw_core.description)); + + dlbu_registers[0] = mali_dlbu_phys_addr | 1; /* bit 0 enables the whole core */ + dlbu_registers[1] = MALI_DLBU_VIRT_ADDR; + dlbu_registers[2] = 0; + dlbu_registers[3] = 0; + dlbu_registers[4] = 0; + dlbu_registers[5] = 0; + dlbu_registers[6] = dlbu->pp_cores_mask; + + /* write reset values to core registers */ + mali_hw_core_register_write_array_relaxed(&dlbu->hw_core, MALI_DLBU_REGISTER_MASTER_TLLIST_PHYS_ADDR, dlbu_registers, 7); + + err = _MALI_OSK_ERR_OK; + + return err; +} + +void mali_dlbu_update_mask(struct mali_dlbu_core *dlbu) +{ + MALI_DEBUG_ASSERT_POINTER(dlbu); + + mali_hw_core_register_write(&dlbu->hw_core, MALI_DLBU_REGISTER_PP_ENABLE_MASK, dlbu->pp_cores_mask); +} + +void mali_dlbu_add_group(struct mali_dlbu_core *dlbu, struct mali_group *group) +{ + struct mali_pp_core *pp_core; + u32 bcast_id; + + MALI_DEBUG_ASSERT_POINTER(dlbu); + MALI_DEBUG_ASSERT_POINTER(group); + + pp_core = mali_group_get_pp_core(group); + bcast_id = mali_pp_core_get_bcast_id(pp_core); + + dlbu->pp_cores_mask |= bcast_id; + MALI_DEBUG_PRINT(3, ("Mali DLBU: Adding core[%d] New mask= 0x%02x\n", bcast_id , dlbu->pp_cores_mask)); +} + +/* Remove a group from the DLBU */ +void mali_dlbu_remove_group(struct mali_dlbu_core *dlbu, struct mali_group *group) +{ + struct mali_pp_core *pp_core; + u32 bcast_id; + + MALI_DEBUG_ASSERT_POINTER(dlbu); + MALI_DEBUG_ASSERT_POINTER(group); + + pp_core = mali_group_get_pp_core(group); + bcast_id = mali_pp_core_get_bcast_id(pp_core); + + dlbu->pp_cores_mask &= ~bcast_id; + MALI_DEBUG_PRINT(3, ("Mali DLBU: Removing core[%d] New mask= 0x%02x\n", bcast_id, dlbu->pp_cores_mask)); +} + +/* Configure the DLBU for \a job. This needs to be done before the job is started on the groups in the DLBU. */ +void mali_dlbu_config_job(struct mali_dlbu_core *dlbu, struct mali_pp_job *job) +{ + u32 *registers; + MALI_DEBUG_ASSERT(job); + registers = mali_pp_job_get_dlbu_registers(job); + MALI_DEBUG_PRINT(4, ("Mali DLBU: Starting job\n")); + + /* Writing 4 registers: + * DLBU registers except the first two (written once at DLBU initialisation / reset) and the PP_ENABLE_MASK register */ + mali_hw_core_register_write_array_relaxed(&dlbu->hw_core, MALI_DLBU_REGISTER_TLLIST_VBASEADDR, registers, 4); + +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dlbu.h b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.h new file mode 100755 index 000000000000..a7ecf41471d8 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_dlbu.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_DLBU_H__ +#define __MALI_DLBU_H__ + +#define MALI_DLBU_VIRT_ADDR 0xFFF00000 /* master tile virtual address fixed at this value and mapped into every session */ + +#include "mali_osk.h" + +struct mali_pp_job; +struct mali_group; +struct mali_dlbu_core; + +extern mali_dma_addr mali_dlbu_phys_addr; + +_mali_osk_errcode_t mali_dlbu_initialize(void); +void mali_dlbu_terminate(void); + +struct mali_dlbu_core *mali_dlbu_create(const _mali_osk_resource_t *resource); +void mali_dlbu_delete(struct mali_dlbu_core *dlbu); + +_mali_osk_errcode_t mali_dlbu_reset(struct mali_dlbu_core *dlbu); + +void mali_dlbu_add_group(struct mali_dlbu_core *dlbu, struct mali_group *group); +void mali_dlbu_remove_group(struct mali_dlbu_core *dlbu, struct mali_group *group); + +/** @brief Called to update HW after DLBU state changed + * + * This function must be called after \a mali_dlbu_add_group or \a + * mali_dlbu_remove_group to write the updated mask to hardware, unless the + * same is accomplished by calling \a mali_dlbu_reset. + */ +void mali_dlbu_update_mask(struct mali_dlbu_core *dlbu); + +void mali_dlbu_config_job(struct mali_dlbu_core *dlbu, struct mali_pp_job *job); + +#endif /* __MALI_DLBU_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c new file mode 100755 index 000000000000..55b21a410754 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.c @@ -0,0 +1,308 @@ +/* + * Copyright (C) 2010-2012, 2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include "mali_kernel_common.h" +#include "mali_scheduler.h" +#include "mali_dvfs_policy.h" +#include "mali_osk_mali.h" +#include "mali_osk_profiling.h" + +#define CLOCK_TUNING_TIME_DEBUG 0 + +#define MAX_PERFORMANCE_VALUE 256 +#define MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(percent) ((int) ((percent)*(MAX_PERFORMANCE_VALUE)/100.0 + 0.5)) + +/** The max fps the same as display vsync default 60, can set by module insert parameter */ +int mali_max_system_fps = 60; +/** A lower limit on their desired FPS default 58, can set by module insert parameter */ +int mali_desired_fps = 58; + +static int mali_fps_step1 = 0; +static int mali_fps_step2 = 0; + +static int clock_step = -1; +static int cur_clk_step = -1; +static struct mali_gpu_clock *gpu_clk = NULL; + +/*Function prototype */ +static int (*mali_gpu_set_freq)(int) = NULL; +static int (*mali_gpu_get_freq)(void) = NULL; + +static mali_bool mali_dvfs_enabled = MALI_FALSE; + +#define NUMBER_OF_NANOSECONDS_PER_SECOND 1000000000ULL +static u32 calculate_window_render_fps(u64 time_period) +{ + u32 max_window_number; + u64 tmp; + u64 max = time_period; + u32 leading_zeroes; + u32 shift_val; + u32 time_period_shift; + u32 max_window_number_shift; + u32 ret_val; + + max_window_number = mali_session_max_window_num(); + + /* To avoid float division, extend the dividend to ns unit */ + tmp = (u64)max_window_number * NUMBER_OF_NANOSECONDS_PER_SECOND; + if (tmp > time_period) { + max = tmp; + } + + /* + * We may have 64-bit values, a dividend or a divisor or both + * To avoid dependencies to a 64-bit divider, we shift down the two values + * equally first. + */ + leading_zeroes = _mali_osk_clz((u32)(max >> 32)); + shift_val = 32 - leading_zeroes; + + time_period_shift = (u32)(time_period >> shift_val); + max_window_number_shift = (u32)(tmp >> shift_val); + + ret_val = max_window_number_shift / time_period_shift; + + return ret_val; +} + +static bool mali_pickup_closest_avail_clock(int target_clock_mhz, mali_bool pick_clock_up) +{ + int i = 0; + bool clock_changed = false; + + /* Round up the closest available frequency step for target_clock_hz */ + for (i = 0; i < gpu_clk->num_of_steps; i++) { + /* Find the first item > target_clock_hz */ + if (((int)(gpu_clk->item[i].clock) - target_clock_mhz) > 0) { + break; + } + } + + /* If the target clock greater than the maximum clock just pick the maximum one*/ + if (i == gpu_clk->num_of_steps) { + i = gpu_clk->num_of_steps - 1; + } else { + if ((!pick_clock_up) && (i > 0)) { + i = i - 1; + } + } + + clock_step = i; + if (cur_clk_step != clock_step) { + clock_changed = true; + } + + return clock_changed; +} + +void mali_dvfs_policy_realize(struct mali_gpu_utilization_data *data, u64 time_period) +{ + int under_perform_boundary_value = 0; + int over_perform_boundary_value = 0; + int current_fps = 0; + int current_gpu_util = 0; + bool clock_changed = false; +#if CLOCK_TUNING_TIME_DEBUG + struct timeval start; + struct timeval stop; + unsigned int elapse_time; + do_gettimeofday(&start); +#endif + u32 window_render_fps; + + if (NULL == gpu_clk) { + MALI_DEBUG_PRINT(2, ("Enable DVFS but patform doesn't Support freq change. \n")); + return; + } + + window_render_fps = calculate_window_render_fps(time_period); + + current_fps = window_render_fps; + current_gpu_util = data->utilization_gpu; + + /* Get the specific under_perform_boundary_value and over_perform_boundary_value */ + if ((mali_desired_fps <= current_fps) && (current_fps < mali_max_system_fps)) { + under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(90); + over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(70); + } else if ((mali_fps_step1 <= current_fps) && (current_fps < mali_desired_fps)) { + under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(55); + over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(35); + } else if ((mali_fps_step2 <= current_fps) && (current_fps < mali_fps_step1)) { + under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(70); + over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(50); + } else { + under_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(55); + over_perform_boundary_value = MALI_PERCENTAGE_TO_UTILIZATION_FRACTION(35); + } + + MALI_DEBUG_PRINT(5, ("Using ARM power policy: gpu util = %d \n", current_gpu_util)); + MALI_DEBUG_PRINT(5, ("Using ARM power policy: under_perform = %d, over_perform = %d \n", under_perform_boundary_value, over_perform_boundary_value)); + MALI_DEBUG_PRINT(5, ("Using ARM power policy: render fps = %d, pressure render fps = %d \n", current_fps, window_render_fps)); + + /* Get current clock value */ + cur_clk_step = mali_gpu_get_freq(); + + /* Consider offscreen */ + if (0 == current_fps) { + /* GP or PP under perform, need to give full power */ + if (current_gpu_util > over_perform_boundary_value) { + if (cur_clk_step != gpu_clk->num_of_steps - 1) { + clock_changed = true; + clock_step = gpu_clk->num_of_steps - 1; + } + } + + /* If GPU is idle, use lowest power */ + if (0 == current_gpu_util) { + if (cur_clk_step != 0) { + clock_changed = true; + clock_step = 0; + } + } + + goto real_setting; + } + + /* 2. Calculate target clock if the GPU clock can be tuned */ + if (-1 != cur_clk_step) { + int target_clk_mhz = -1; + mali_bool pick_clock_up = MALI_TRUE; + + if (current_gpu_util > under_perform_boundary_value) { + /* when under perform, need to consider the fps part */ + target_clk_mhz = gpu_clk->item[cur_clk_step].clock * current_gpu_util * mali_desired_fps / under_perform_boundary_value / current_fps; + pick_clock_up = MALI_TRUE; + } else if (current_gpu_util < over_perform_boundary_value) { + /* when over perform, did't need to consider fps, system didn't want to reach desired fps */ + target_clk_mhz = gpu_clk->item[cur_clk_step].clock * current_gpu_util / under_perform_boundary_value; + pick_clock_up = MALI_FALSE; + } + + if (-1 != target_clk_mhz) { + clock_changed = mali_pickup_closest_avail_clock(target_clk_mhz, pick_clock_up); + } + } + +real_setting: + if (clock_changed) { + mali_gpu_set_freq(clock_step); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + gpu_clk->item[clock_step].clock, + gpu_clk->item[clock_step].vol / 1000, + 0, 0, 0); + } + +#if CLOCK_TUNING_TIME_DEBUG + do_gettimeofday(&stop); + + elapse_time = timeval_to_ns(&stop) - timeval_to_ns(&start); + MALI_DEBUG_PRINT(2, ("Using ARM power policy: eclapse time = %d\n", elapse_time)); +#endif +} + +_mali_osk_errcode_t mali_dvfs_policy_init(void) +{ + _mali_osk_device_data data; + _mali_osk_errcode_t err = _MALI_OSK_ERR_OK; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + if ((NULL != data.get_clock_info) && (NULL != data.set_freq) && (NULL != data.get_freq)) { + MALI_DEBUG_PRINT(2, ("Mali DVFS init: using arm dvfs policy \n")); + + + mali_fps_step1 = mali_max_system_fps / 3; + mali_fps_step2 = mali_max_system_fps / 5; + + data.get_clock_info(&gpu_clk); + + if (gpu_clk != NULL) { +#ifdef DEBUG + int i; + for (i = 0; i < gpu_clk->num_of_steps; i++) { + MALI_DEBUG_PRINT(5, ("mali gpu clock info: step%d clock(%d)Hz,vol(%d) \n", + i, gpu_clk->item[i].clock, gpu_clk->item[i].vol)); + } +#endif + } else { + MALI_DEBUG_PRINT(2, ("Mali DVFS init: platform didn't define enough info for ddk to do DVFS \n")); + } + + mali_gpu_get_freq = data.get_freq; + mali_gpu_set_freq = data.set_freq; + + if ((NULL != gpu_clk) && (gpu_clk->num_of_steps > 0) + && (NULL != mali_gpu_get_freq) && (NULL != mali_gpu_set_freq)) { + mali_dvfs_enabled = MALI_TRUE; + } + } else { + MALI_DEBUG_PRINT(2, ("Mali DVFS init: platform function callback incomplete, need check mali_gpu_device_data in platform .\n")); + } + } else { + err = _MALI_OSK_ERR_FAULT; + MALI_DEBUG_PRINT(2, ("Mali DVFS init: get platform data error .\n")); + } + + return err; +} + +/* + * Always give full power when start a new period, + * if mali dvfs enabled, for performance consideration + */ +void mali_dvfs_policy_new_period(void) +{ + /* Always give full power when start a new period */ + unsigned int cur_clk_step = 0; + + cur_clk_step = mali_gpu_get_freq(); + + if (cur_clk_step != (gpu_clk->num_of_steps - 1)) { + mali_gpu_set_freq(gpu_clk->num_of_steps - 1); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, gpu_clk->item[gpu_clk->num_of_steps - 1].clock, + gpu_clk->item[gpu_clk->num_of_steps - 1].vol / 1000, 0, 0, 0); + } +} + +mali_bool mali_dvfs_policy_enabled(void) +{ + return mali_dvfs_enabled; +} + +#if defined(CONFIG_MALI400_PROFILING) +void mali_get_current_gpu_clk_item(struct mali_gpu_clk_item *clk_item) +{ + if (mali_platform_device != NULL) { + + struct mali_gpu_device_data *device_data = NULL; + device_data = (struct mali_gpu_device_data *)mali_platform_device->dev.platform_data; + + if ((NULL != device_data->get_clock_info) && (NULL != device_data->get_freq)) { + + int cur_clk_step = device_data->get_freq(); + struct mali_gpu_clock *mali_gpu_clk = NULL; + + device_data->get_clock_info(&mali_gpu_clk); + clk_item->clock = mali_gpu_clk->item[cur_clk_step].clock; + clk_item->vol = mali_gpu_clk->item[cur_clk_step].vol; + } else { + MALI_DEBUG_PRINT(2, ("Mali GPU Utilization: platform function callback incomplete, need check mali_gpu_device_data in platform .\n")); + } + } +} +#endif + diff --git a/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h new file mode 100755 index 000000000000..662348c4e6ac --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_dvfs_policy.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2010-2012, 2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_DVFS_POLICY_H__ +#define __MALI_DVFS_POLICY_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +void mali_dvfs_policy_realize(struct mali_gpu_utilization_data *data, u64 time_period); + +_mali_osk_errcode_t mali_dvfs_policy_init(void); + +void mali_dvfs_policy_new_period(void); + +mali_bool mali_dvfs_policy_enabled(void); + +#if defined(CONFIG_MALI400_PROFILING) +void mali_get_current_gpu_clk_item(struct mali_gpu_clk_item *clk_item); +#endif + +#ifdef __cplusplus +} +#endif + +#endif/* __MALI_DVFS_POLICY_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_executor.c b/drivers/gpu/arm/mali400/mali/common/mali_executor.c new file mode 100755 index 000000000000..ee40520ed0ce --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_executor.c @@ -0,0 +1,2707 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_executor.h" +#include "mali_scheduler.h" +#include "mali_kernel_common.h" +#include "mali_kernel_core.h" +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_pp.h" +#include "mali_pp_job.h" +#include "mali_group.h" +#include "mali_pm.h" +#include "mali_timeline.h" +#include "mali_osk_profiling.h" +#include "mali_session.h" +#include "mali_osk_mali.h" + +/*Add for voltage scan function*/ +extern u32 mali_group_error; + +/* + * If dma_buf with map on demand is used, we defer job deletion and job queue + * if in atomic context, since both might sleep. + */ +#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +#define MALI_EXECUTOR_USE_DEFERRED_PP_JOB_DELETE 1 +#define MALI_EXECUTOR_USE_DEFERRED_PP_JOB_QUEUE 1 +#endif /* !defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) */ + +/* + * ---------- static type definitions (structs, enums, etc) ---------- + */ + +enum mali_executor_state_t { + EXEC_STATE_NOT_PRESENT, /* Virtual group on Mali-300/400 (do not use) */ + EXEC_STATE_DISABLED, /* Disabled by core scaling (do not use) */ + EXEC_STATE_EMPTY, /* No child groups for virtual group (do not use) */ + EXEC_STATE_INACTIVE, /* Can be used, but must be activate first */ + EXEC_STATE_IDLE, /* Active and ready to be used */ + EXEC_STATE_WORKING, /* Executing a job */ +}; + +/* + * ---------- global variables (exported due to inline functions) ---------- + */ + +/* Lock for this module (protecting all HW access except L2 caches) */ +_mali_osk_spinlock_irq_t *mali_executor_lock_obj = NULL; + +mali_bool mali_executor_hints[MALI_EXECUTOR_HINT_MAX]; + +/* + * ---------- static variables ---------- + */ + +/* Used to defer job scheduling */ +static _mali_osk_wq_work_t *executor_wq_high_pri = NULL; + +/* Store version from GP and PP (user space wants to know this) */ +static u32 pp_version = 0; +static u32 gp_version = 0; + +/* List of physical PP groups which are disabled by some external source */ +static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_disabled); +static u32 group_list_disabled_count = 0; + +/* List of groups which can be used, but activate first */ +static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_inactive); +static u32 group_list_inactive_count = 0; + +/* List of groups which are active and ready to be used */ +static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_idle); +static u32 group_list_idle_count = 0; + +/* List of groups which are executing a job */ +static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_working); +static u32 group_list_working_count = 0; + +/* Virtual group (if any) */ +static struct mali_group *virtual_group = NULL; + +/* Virtual group state is tracked with a state variable instead of 4 lists */ +static enum mali_executor_state_t virtual_group_state = EXEC_STATE_NOT_PRESENT; + +/* GP group */ +static struct mali_group *gp_group = NULL; + +/* GP group state is tracked with a state variable instead of 4 lists */ +static enum mali_executor_state_t gp_group_state = EXEC_STATE_NOT_PRESENT; + +static u32 gp_returned_cookie = 0; + +/* Total number of physical PP cores present */ +static u32 num_physical_pp_cores_total = 0; + +/* Number of physical cores which are enabled */ +static u32 num_physical_pp_cores_enabled = 0; + +/* Enable or disable core scaling */ +static mali_bool core_scaling_enabled = MALI_TRUE; + +/* Variables to allow safe pausing of the scheduler */ +static _mali_osk_wait_queue_t *executor_working_wait_queue = NULL; +static u32 pause_count = 0; + +/* PP cores haven't been enabled because of some pp cores haven't been disabled. */ +static int core_scaling_delay_up_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; + +/* Variables used to implement notify pp core changes to userspace when core scaling + * is finished in mali_executor_complete_group() function. */ +static _mali_osk_wq_work_t *executor_wq_notify_core_change = NULL; +static _mali_osk_wait_queue_t *executor_notify_core_change_wait_queue = NULL; + +/* + * ---------- Forward declaration of static functions ---------- + */ +static mali_bool mali_executor_is_suspended(void *data); +static mali_bool mali_executor_is_working(void); +static void mali_executor_disable_empty_virtual(void); +static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group); +static mali_bool mali_executor_has_virtual_group(void); +static mali_bool mali_executor_virtual_group_is_usable(void); +static void mali_executor_schedule(void); +static void mali_executor_wq_schedule(void *arg); +static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job); +static void mali_executor_complete_group(struct mali_group *group, + mali_bool success, + struct mali_gp_job **gp_job_done, + struct mali_pp_job **pp_job_done); +static void mali_executor_change_state_pp_physical(struct mali_group *group, + _mali_osk_list_t *old_list, + u32 *old_count, + _mali_osk_list_t *new_list, + u32 *new_count); +static mali_bool mali_executor_group_is_in_state(struct mali_group *group, + enum mali_executor_state_t state); + +static void mali_executor_group_enable_internal(struct mali_group *group); +static void mali_executor_group_disable_internal(struct mali_group *group); +static void mali_executor_core_scale(unsigned int target_core_nr); +static void mali_executor_core_scale_in_group_complete(struct mali_group *group); +static void mali_executor_notify_core_change(u32 num_cores); +static void mali_executor_wq_notify_core_change(void *arg); +static void mali_executor_change_group_status_disabled(struct mali_group *group); +static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_group); +static void mali_executor_set_state_pp_physical(struct mali_group *group, + _mali_osk_list_t *new_list, + u32 *new_count); + +/* + * ---------- Actual implementation ---------- + */ + +_mali_osk_errcode_t mali_executor_initialize(void) +{ + mali_executor_lock_obj = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_EXECUTOR); + if (NULL == mali_executor_lock_obj) { + mali_executor_terminate(); + return _MALI_OSK_ERR_NOMEM; + } + + executor_wq_high_pri = _mali_osk_wq_create_work_high_pri(mali_executor_wq_schedule, NULL); + if (NULL == executor_wq_high_pri) { + mali_executor_terminate(); + return _MALI_OSK_ERR_NOMEM; + } + + executor_working_wait_queue = _mali_osk_wait_queue_init(); + if (NULL == executor_working_wait_queue) { + mali_executor_terminate(); + return _MALI_OSK_ERR_NOMEM; + } + + executor_wq_notify_core_change = _mali_osk_wq_create_work(mali_executor_wq_notify_core_change, NULL); + if (NULL == executor_wq_notify_core_change) { + mali_executor_terminate(); + return _MALI_OSK_ERR_NOMEM; + } + + executor_notify_core_change_wait_queue = _mali_osk_wait_queue_init(); + if (NULL == executor_notify_core_change_wait_queue) { + mali_executor_terminate(); + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_executor_terminate(void) +{ + if (NULL != executor_notify_core_change_wait_queue) { + _mali_osk_wait_queue_term(executor_notify_core_change_wait_queue); + executor_notify_core_change_wait_queue = NULL; + } + + if (NULL != executor_wq_notify_core_change) { + _mali_osk_wq_delete_work(executor_wq_notify_core_change); + executor_wq_notify_core_change = NULL; + } + + if (NULL != executor_working_wait_queue) { + _mali_osk_wait_queue_term(executor_working_wait_queue); + executor_working_wait_queue = NULL; + } + + if (NULL != executor_wq_high_pri) { + _mali_osk_wq_delete_work(executor_wq_high_pri); + executor_wq_high_pri = NULL; + } + + if (NULL != mali_executor_lock_obj) { + _mali_osk_spinlock_irq_term(mali_executor_lock_obj); + mali_executor_lock_obj = NULL; + } +} + +void mali_executor_populate(void) +{ + u32 num_groups; + u32 i; + + num_groups = mali_group_get_glob_num_groups(); + + /* Do we have a virtual group? */ + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + if (mali_group_is_virtual(group)) { + virtual_group = group; + virtual_group_state = EXEC_STATE_INACTIVE; + break; + } + } + + /* Find all the available physical GP and PP cores */ + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + if (NULL != group) { + struct mali_pp_core *pp_core = mali_group_get_pp_core(group); + struct mali_gp_core *gp_core = mali_group_get_gp_core(group); + + if (!mali_group_is_virtual(group)) { + if (NULL != pp_core) { + if (0 == pp_version) { + /* Retrieve PP version from the first available PP core */ + pp_version = mali_pp_core_get_version(pp_core); + } + + if (NULL != virtual_group) { + mali_executor_lock(); + mali_group_add_group(virtual_group, group); + mali_executor_unlock(); + } else { + _mali_osk_list_add(&group->executor_list, &group_list_inactive); + group_list_inactive_count++; + } + + num_physical_pp_cores_total++; + } else { + MALI_DEBUG_ASSERT_POINTER(gp_core); + + if (0 == gp_version) { + /* Retrieve GP version */ + gp_version = mali_gp_core_get_version(gp_core); + } + + gp_group = group; + gp_group_state = EXEC_STATE_INACTIVE; + } + + } + } + } + + num_physical_pp_cores_enabled = num_physical_pp_cores_total; +} + +void mali_executor_depopulate(void) +{ + struct mali_group *group; + struct mali_group *temp; + + MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != gp_group_state); + + if (NULL != gp_group) { + mali_group_delete(gp_group); + gp_group = NULL; + } + + MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != virtual_group_state); + + if (NULL != virtual_group) { + mali_group_delete(virtual_group); + virtual_group = NULL; + } + + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&group_list_working)); + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) { + mali_group_delete(group); + } + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) { + mali_group_delete(group); + } + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) { + mali_group_delete(group); + } +} + +void mali_executor_suspend(void) +{ + mali_executor_lock(); + + /* Increment the pause_count so that no more jobs will be scheduled */ + pause_count++; + + mali_executor_unlock(); + + _mali_osk_wait_queue_wait_event(executor_working_wait_queue, + mali_executor_is_suspended, NULL); + + /* + * mali_executor_complete_XX() leaves jobs in idle state. + * deactivate option is used when we are going to power down + * the entire GPU (OS suspend) and want a consistent SW vs HW + * state. + */ + mali_executor_lock(); + + mali_executor_deactivate_list_idle(MALI_TRUE); + + /* + * The following steps are used to deactive all of activated + * (MALI_GROUP_STATE_ACTIVE) and activating (MALI_GROUP + * _STAET_ACTIVATION_PENDING) groups, to make sure the variable + * pd_mask_wanted is equal with 0. */ + if (MALI_GROUP_STATE_INACTIVE != mali_group_get_state(gp_group)) { + gp_group_state = EXEC_STATE_INACTIVE; + mali_group_deactivate(gp_group); + } + + if (mali_executor_has_virtual_group()) { + if (MALI_GROUP_STATE_INACTIVE + != mali_group_get_state(virtual_group)) { + virtual_group_state = EXEC_STATE_INACTIVE; + mali_group_deactivate(virtual_group); + } + } + + if (0 < group_list_inactive_count) { + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, + &group_list_inactive, + struct mali_group, executor_list) { + if (MALI_GROUP_STATE_ACTIVATION_PENDING + == mali_group_get_state(group)) { + mali_group_deactivate(group); + } + + /* + * On mali-450 platform, we may have physical group in the group inactive + * list, and its state is MALI_GROUP_STATE_ACTIVATION_PENDING, so we only + * deactivate it is not enough, we still also need add it back to virtual group. + * And now, virtual group must be in INACTIVE state, so it's safe to add + * physical group to virtual group at this point. + */ + if (NULL != virtual_group) { + _mali_osk_list_delinit(&group->executor_list); + group_list_inactive_count--; + + mali_group_add_group(virtual_group, group); + } + } + } + + mali_executor_unlock(); +} + +void mali_executor_resume(void) +{ + mali_executor_lock(); + + /* Decrement pause_count to allow scheduling again (if it reaches 0) */ + pause_count--; + if (0 == pause_count) { + mali_executor_schedule(); + } + + mali_executor_unlock(); +} + +u32 mali_executor_get_num_cores_total(void) +{ + return num_physical_pp_cores_total; +} + +u32 mali_executor_get_num_cores_enabled(void) +{ + return num_physical_pp_cores_enabled; +} + +struct mali_pp_core *mali_executor_get_virtual_pp(void) +{ + MALI_DEBUG_ASSERT_POINTER(virtual_group); + MALI_DEBUG_ASSERT_POINTER(virtual_group->pp_core); + return virtual_group->pp_core; +} + +struct mali_group *mali_executor_get_virtual_group(void) +{ + return virtual_group; +} + +void mali_executor_zap_all_active(struct mali_session_data *session) +{ + struct mali_group *group; + struct mali_group *temp; + mali_bool ret; + + mali_executor_lock(); + + /* + * This function is a bit complicated because + * mali_group_zap_session() can fail. This only happens because the + * group is in an unhandled page fault status. + * We need to make sure this page fault is handled before we return, + * so that we know every single outstanding MMU transactions have + * completed. This will allow caller to safely remove physical pages + * when we have returned. + */ + + MALI_DEBUG_ASSERT(NULL != gp_group); + ret = mali_group_zap_session(gp_group, session); + if (MALI_FALSE == ret) { + struct mali_gp_job *gp_job = NULL; + + mali_executor_complete_group(gp_group, MALI_FALSE, &gp_job, NULL); + + MALI_DEBUG_ASSERT_POINTER(gp_job); + + /* GP job completed, make sure it is freed */ + mali_scheduler_complete_gp_job(gp_job, MALI_FALSE, + MALI_TRUE, MALI_TRUE); + } + + if (mali_executor_has_virtual_group()) { + ret = mali_group_zap_session(virtual_group, session); + if (MALI_FALSE == ret) { + struct mali_pp_job *pp_job = NULL; + + mali_executor_complete_group(virtual_group, MALI_FALSE, NULL, &pp_job); + + if (NULL != pp_job) { + /* PP job completed, make sure it is freed */ + mali_scheduler_complete_pp_job(pp_job, 0, + MALI_FALSE, MALI_TRUE); + } + } + } + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, + struct mali_group, executor_list) { + ret = mali_group_zap_session(group, session); + if (MALI_FALSE == ret) { + ret = mali_group_zap_session(group, session); + if (MALI_FALSE == ret) { + struct mali_pp_job *pp_job = NULL; + + mali_executor_complete_group(group, MALI_FALSE, NULL, &pp_job); + + if (NULL != pp_job) { + /* PP job completed, free it */ + mali_scheduler_complete_pp_job(pp_job, + 0, MALI_FALSE, + MALI_TRUE); + } + } + } + } + + mali_executor_unlock(); +} + +void mali_executor_schedule_from_mask(mali_scheduler_mask mask, mali_bool deferred_schedule) +{ + if (MALI_SCHEDULER_MASK_EMPTY != mask) { + if (MALI_TRUE == deferred_schedule) { + _mali_osk_wq_schedule_work_high_pri(executor_wq_high_pri); + } else { + /* Schedule from this thread*/ + mali_executor_lock(); + mali_executor_schedule(); + mali_executor_unlock(); + } + } +} + +_mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group, + mali_bool in_upper_half) +{ + enum mali_interrupt_result int_result; + mali_bool time_out = MALI_FALSE; + + MALI_DEBUG_PRINT(4, ("Executor: GP interrupt from %s in %s half\n", + mali_group_core_description(group), + in_upper_half ? "upper" : "bottom")); + + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_working(group)); + + if (mali_group_has_timed_out(group)) { + int_result = MALI_INTERRUPT_RESULT_ERROR; + time_out = MALI_TRUE; + MALI_PRINT(("Executor GP: Job %d Timeout on %s\n", + mali_gp_job_get_id(group->gp_running_job), + mali_group_core_description(group))); + } else { + int_result = mali_group_get_interrupt_result_gp(group); + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + } + +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + /* No interrupts signalled, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#else + MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result); +#endif + + mali_group_mask_all_interrupts_gp(group); + + if (MALI_INTERRUPT_RESULT_SUCCESS_VS == int_result) { + if (mali_group_gp_is_active(group)) { + /* Only VS completed so far, while PLBU is still active */ + + /* Enable all but the current interrupt */ + mali_group_enable_interrupts_gp(group, int_result); + + mali_executor_unlock(); + return _MALI_OSK_ERR_OK; + } + } else if (MALI_INTERRUPT_RESULT_SUCCESS_PLBU == int_result) { + if (mali_group_gp_is_active(group)) { + /* Only PLBU completed so far, while VS is still active */ + + /* Enable all but the current interrupt */ + mali_group_enable_interrupts_gp(group, int_result); + + mali_executor_unlock(); + return _MALI_OSK_ERR_OK; + } + } else if (MALI_INTERRUPT_RESULT_OOM == int_result) { + struct mali_gp_job *job = mali_group_get_running_gp_job(group); + + /* PLBU out of mem */ + MALI_DEBUG_PRINT(3, ("Executor: PLBU needs more heap memory\n")); + +#if defined(CONFIG_MALI400_PROFILING) + /* Give group a chance to generate a SUSPEND event */ + mali_group_oom(group); +#endif + + /* + * no need to hold interrupt raised while + * waiting for more memory. + */ + mali_executor_send_gp_oom_to_user(job); + + mali_executor_unlock(); + + return _MALI_OSK_ERR_OK; + } + + /*Add for voltage scan function*/ + if (MALI_INTERRUPT_RESULT_ERROR == int_result) + mali_group_error++; + + /* We should now have a real interrupt to handle */ + + MALI_DEBUG_PRINT(4, ("Executor: Group %s completed with %s\n", + mali_group_core_description(group), + (MALI_INTERRUPT_RESULT_ERROR == int_result) ? + "ERROR" : "success")); + + if (in_upper_half && MALI_INTERRUPT_RESULT_ERROR == int_result) { + /* Don't bother to do processing of errors in upper half */ + mali_executor_unlock(); + + if (MALI_FALSE == time_out) { + mali_group_schedule_bottom_half_gp(group); + } + } else { + struct mali_gp_job *job; + mali_bool success; + + /* + if (MALI_TRUE == time_out) { + mali_group_dump_status(group); + } + */ + + success = (int_result != MALI_INTERRUPT_RESULT_ERROR) ? + MALI_TRUE : MALI_FALSE; + + mali_executor_complete_group(group, success, &job, NULL); + + mali_executor_unlock(); + + /* GP jobs always fully complete */ + MALI_DEBUG_ASSERT(NULL != job); + + /* This will notify user space and close the job object */ + mali_scheduler_complete_gp_job(job, success, + MALI_TRUE, MALI_TRUE); + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group, + mali_bool in_upper_half) +{ + enum mali_interrupt_result int_result; + mali_bool time_out = MALI_FALSE; + + MALI_DEBUG_PRINT(4, ("Executor: PP interrupt from %s in %s half\n", + mali_group_core_description(group), + in_upper_half ? "upper" : "bottom")); + + mali_executor_lock(); + + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + + if (in_upper_half) { + if (mali_group_is_in_virtual(group)) { + /* Child groups should never handle PP interrupts */ + MALI_DEBUG_ASSERT(!mali_group_has_timed_out(group)); + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + } + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_working(group)); + MALI_DEBUG_ASSERT(!mali_group_is_in_virtual(group)); + + if (mali_group_has_timed_out(group)) { + int_result = MALI_INTERRUPT_RESULT_ERROR; + time_out = MALI_TRUE; + MALI_PRINT(("Executor PP: Job %d Timeout on %s\n", + mali_pp_job_get_id(group->pp_running_job), + mali_group_core_description(group))); + } else { + int_result = mali_group_get_interrupt_result_pp(group); + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + } + +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + /* No interrupts signalled, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } else if (MALI_INTERRUPT_RESULT_SUCCESS == int_result) { + if (mali_group_is_virtual(group) && mali_group_pp_is_active(group)) { + /* Some child groups are still working, so nothing to do right now */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + } +#else + MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result); +#endif + + /*Add voltage scan function*/ + + if (MALI_INTERRUPT_RESULT_ERROR == int_result) + mali_group_error++; + + /* We should now have a real interrupt to handle */ + + MALI_DEBUG_PRINT(4, ("Executor: Group %s completed with %s\n", + mali_group_core_description(group), + (MALI_INTERRUPT_RESULT_ERROR == int_result) ? + "ERROR" : "success")); + + if (in_upper_half && MALI_INTERRUPT_RESULT_ERROR == int_result) { + /* Don't bother to do processing of errors in upper half */ + mali_group_mask_all_interrupts_pp(group); + mali_executor_unlock(); + + if (MALI_FALSE == time_out) { + mali_group_schedule_bottom_half_pp(group); + } + } else { + struct mali_pp_job *job = NULL; + mali_bool success; + + if (MALI_TRUE == time_out) { + mali_group_dump_status(group); + } + + success = (int_result == MALI_INTERRUPT_RESULT_SUCCESS) ? + MALI_TRUE : MALI_FALSE; + + mali_executor_complete_group(group, success, NULL, &job); + + mali_executor_unlock(); + + if (NULL != job) { + /* Notify user space and close the job object */ + mali_scheduler_complete_pp_job(job, + num_physical_pp_cores_total, + MALI_TRUE, MALI_TRUE); + } + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group, + mali_bool in_upper_half) +{ + enum mali_interrupt_result int_result; + + MALI_DEBUG_PRINT(4, ("Executor: MMU interrupt from %s in %s half\n", + mali_group_core_description(group), + in_upper_half ? "upper" : "bottom")); + + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_working(group)); + + int_result = mali_group_get_interrupt_result_mmu(group); + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + if (MALI_INTERRUPT_RESULT_NONE == int_result) { + /* No interrupts signalled, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#else + MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_ERROR == int_result); +#endif + + /* We should now have a real interrupt to handle */ + + if (in_upper_half) { + /* Don't bother to do processing of errors in upper half */ + + struct mali_group *parent = group->parent_group; + + mali_mmu_mask_all_interrupts(group->mmu); + + mali_executor_unlock(); + + if (NULL == parent) { + mali_group_schedule_bottom_half_mmu(group); + } else { + mali_group_schedule_bottom_half_mmu(parent); + } + + } else { + struct mali_gp_job *gp_job = NULL; + struct mali_pp_job *pp_job = NULL; + +#ifdef DEBUG + + u32 fault_address = mali_mmu_get_page_fault_addr(group->mmu); + u32 status = mali_mmu_get_status(group->mmu); + MALI_DEBUG_PRINT(2, ("Executor: Mali page fault detected at 0x%x from bus id %d of type %s on %s\n", + (void *)(uintptr_t)fault_address, + (status >> 6) & 0x1F, + (status & 32) ? "write" : "read", + group->mmu->hw_core.description)); + MALI_DEBUG_PRINT(3, ("Executor: MMU rawstat = 0x%08X, MMU status = 0x%08X\n", + mali_mmu_get_rawstat(group->mmu), status)); + mali_mmu_pagedir_diag(mali_session_get_page_directory(group->session), fault_address); +#endif + + mali_executor_complete_group(group, MALI_FALSE, &gp_job, &pp_job); + + mali_executor_unlock(); + + if (NULL != gp_job) { + MALI_DEBUG_ASSERT(NULL == pp_job); + + /* Notify user space and close the job object */ + mali_scheduler_complete_gp_job(gp_job, MALI_FALSE, + MALI_TRUE, MALI_TRUE); + } else if (NULL != pp_job) { + MALI_DEBUG_ASSERT(NULL == gp_job); + + /* Notify user space and close the job object */ + mali_scheduler_complete_pp_job(pp_job, + num_physical_pp_cores_total, + MALI_TRUE, MALI_TRUE); + } + } + + return _MALI_OSK_ERR_OK; +} + +void mali_executor_group_power_up(struct mali_group *groups[], u32 num_groups) +{ + u32 i; + mali_bool child_groups_activated = MALI_FALSE; + mali_bool do_schedule = MALI_FALSE; +#if defined(DEBUG) + u32 num_activated = 0; +#endif + + MALI_DEBUG_ASSERT_POINTER(groups); + MALI_DEBUG_ASSERT(0 < num_groups); + + mali_executor_lock(); + + MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups\n", num_groups)); + + for (i = 0; i < num_groups; i++) { + MALI_DEBUG_PRINT(3, ("Executor: powering up group %s\n", + mali_group_core_description(groups[i]))); + + mali_group_power_up(groups[i]); + + if ((MALI_GROUP_STATE_ACTIVATION_PENDING != mali_group_get_state(groups[i]) || + (MALI_TRUE != mali_executor_group_is_in_state(groups[i], EXEC_STATE_INACTIVE)))) { + /* nothing more to do for this group */ + continue; + } + + MALI_DEBUG_PRINT(3, ("Executor: activating group %s\n", + mali_group_core_description(groups[i]))); + +#if defined(DEBUG) + num_activated++; +#endif + + if (mali_group_is_in_virtual(groups[i])) { + /* + * At least one child group of virtual group is powered on. + */ + child_groups_activated = MALI_TRUE; + } else if (MALI_FALSE == mali_group_is_virtual(groups[i])) { + /* Set gp and pp not in virtual to active. */ + mali_group_set_active(groups[i]); + } + + /* Move group from inactive to idle list */ + if (groups[i] == gp_group) { + MALI_DEBUG_ASSERT(EXEC_STATE_INACTIVE == + gp_group_state); + gp_group_state = EXEC_STATE_IDLE; + } else if (MALI_FALSE == mali_group_is_in_virtual(groups[i]) + && MALI_FALSE == mali_group_is_virtual(groups[i])) { + MALI_DEBUG_ASSERT(MALI_TRUE == mali_executor_group_is_in_state(groups[i], + EXEC_STATE_INACTIVE)); + + mali_executor_change_state_pp_physical(groups[i], + &group_list_inactive, + &group_list_inactive_count, + &group_list_idle, + &group_list_idle_count); + } + + do_schedule = MALI_TRUE; + } + + if (mali_executor_has_virtual_group() && + MALI_TRUE == child_groups_activated && + MALI_GROUP_STATE_ACTIVATION_PENDING == + mali_group_get_state(virtual_group)) { + /* + * Try to active virtual group while it may be not sucessful every time, + * because there is one situation that not all of child groups are powered on + * in one time and virtual group is in activation pending state. + */ + if (mali_group_set_active(virtual_group)) { + /* Move group from inactive to idle */ + MALI_DEBUG_ASSERT(EXEC_STATE_INACTIVE == + virtual_group_state); + virtual_group_state = EXEC_STATE_IDLE; + + MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated, 1 virtual activated.\n", num_groups, num_activated)); + } else { + MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated\n", num_groups, num_activated)); + } + } else { + MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated\n", num_groups, num_activated)); + } + + if (MALI_TRUE == do_schedule) { + /* Trigger a schedule */ + mali_executor_schedule(); + } + + mali_executor_unlock(); +} + +void mali_executor_group_power_down(struct mali_group *groups[], + u32 num_groups) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(groups); + MALI_DEBUG_ASSERT(0 < num_groups); + + mali_executor_lock(); + + MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups\n", num_groups)); + + for (i = 0; i < num_groups; i++) { + /* Groups must be either disabled or inactive. while for virtual group, + * it maybe in empty state, because when we meet pm_runtime_suspend, + * virtual group could be powered off, and before we acquire mali_executor_lock, + * we must release mali_pm_state_lock, if there is a new physical job was queued, + * all of physical groups in virtual group could be pulled out, so we only can + * powered down an empty virtual group. Those physical groups will be powered + * up in following pm_runtime_resume callback function. + */ + MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(groups[i], + EXEC_STATE_DISABLED) || + mali_executor_group_is_in_state(groups[i], + EXEC_STATE_INACTIVE) || + mali_executor_group_is_in_state(groups[i], + EXEC_STATE_EMPTY)); + + MALI_DEBUG_PRINT(3, ("Executor: powering down group %s\n", + mali_group_core_description(groups[i]))); + + mali_group_power_down(groups[i]); + } + + MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups completed\n", num_groups)); + + mali_executor_unlock(); +} + +void mali_executor_abort_session(struct mali_session_data *session) +{ + struct mali_group *group; + struct mali_group *tmp_group; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT(session->is_aborting); + + MALI_DEBUG_PRINT(3, + ("Executor: Aborting all jobs from session 0x%08X.\n", + session)); + + mali_executor_lock(); + + if (mali_group_get_session(gp_group) == session) { + if (EXEC_STATE_WORKING == gp_group_state) { + struct mali_gp_job *gp_job = NULL; + + mali_executor_complete_group(gp_group, MALI_FALSE, &gp_job, NULL); + + MALI_DEBUG_ASSERT_POINTER(gp_job); + + /* GP job completed, make sure it is freed */ + mali_scheduler_complete_gp_job(gp_job, MALI_FALSE, + MALI_FALSE, MALI_TRUE); + } else { + /* Same session, but not working, so just clear it */ + mali_group_clear_session(gp_group); + } + } + + if (mali_executor_has_virtual_group()) { + if (EXEC_STATE_WORKING == virtual_group_state + && mali_group_get_session(virtual_group) == session) { + struct mali_pp_job *pp_job = NULL; + + mali_executor_complete_group(virtual_group, MALI_FALSE, NULL, &pp_job); + + if (NULL != pp_job) { + /* PP job completed, make sure it is freed */ + mali_scheduler_complete_pp_job(pp_job, 0, + MALI_FALSE, MALI_TRUE); + } + } + } + + _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_working, + struct mali_group, executor_list) { + if (mali_group_get_session(group) == session) { + struct mali_pp_job *pp_job = NULL; + + mali_executor_complete_group(group, MALI_FALSE, NULL, &pp_job); + + if (NULL != pp_job) { + /* PP job completed, make sure it is freed */ + mali_scheduler_complete_pp_job(pp_job, 0, + MALI_FALSE, MALI_TRUE); + } + } + } + + _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_idle, struct mali_group, executor_list) { + mali_group_clear_session(group); + } + + _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_inactive, struct mali_group, executor_list) { + mali_group_clear_session(group); + } + + _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_disabled, struct mali_group, executor_list) { + mali_group_clear_session(group); + } + + mali_executor_unlock(); +} + + +void mali_executor_core_scaling_enable(void) +{ + /* PS: Core scaling is by default enabled */ + core_scaling_enabled = MALI_TRUE; +} + +void mali_executor_core_scaling_disable(void) +{ + core_scaling_enabled = MALI_FALSE; +} + +mali_bool mali_executor_core_scaling_is_enabled(void) +{ + return core_scaling_enabled; +} + +void mali_executor_group_enable(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + + mali_executor_lock(); + + if ((NULL != mali_group_get_gp_core(group) || NULL != mali_group_get_pp_core(group)) + && (mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))) { + mali_executor_group_enable_internal(group); + } + + mali_executor_schedule(); + mali_executor_unlock(); + + _mali_osk_wq_schedule_work(executor_wq_notify_core_change); +} + +/* + * If a physical group is inactive or idle, we should disable it immediately, + * if group is in virtual, and virtual group is idle, disable given physical group in it. + */ +void mali_executor_group_disable(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + + mali_executor_lock(); + + if ((NULL != mali_group_get_gp_core(group) || NULL != mali_group_get_pp_core(group)) + && (!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))) { + mali_executor_group_disable_internal(group); + } + + mali_executor_schedule(); + mali_executor_unlock(); + + _mali_osk_wq_schedule_work(executor_wq_notify_core_change); +} + +mali_bool mali_executor_group_is_disabled(struct mali_group *group) +{ + /* NB: This function is not optimized for time critical usage */ + + mali_bool ret; + + MALI_DEBUG_ASSERT_POINTER(group); + + mali_executor_lock(); + ret = mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED); + mali_executor_unlock(); + + return ret; +} + +int mali_executor_set_perf_level(unsigned int target_core_nr, mali_bool override) +{ + if (target_core_nr == num_physical_pp_cores_enabled) return 0; + if (MALI_FALSE == core_scaling_enabled && MALI_FALSE == override) return -EPERM; + if (target_core_nr > num_physical_pp_cores_total) return -EINVAL; + if (0 == target_core_nr) return -EINVAL; + + mali_executor_core_scale(target_core_nr); + + _mali_osk_wq_schedule_work(executor_wq_notify_core_change); + + return 0; +} + +#if MALI_STATE_TRACKING +u32 mali_executor_dump_state(char *buf, u32 size) +{ + int n = 0; + struct mali_group *group; + struct mali_group *temp; + + mali_executor_lock(); + + switch (gp_group_state) { + case EXEC_STATE_INACTIVE: + n += _mali_osk_snprintf(buf + n, size - n, + "GP group is in state INACTIVE\n"); + break; + case EXEC_STATE_IDLE: + n += _mali_osk_snprintf(buf + n, size - n, + "GP group is in state IDLE\n"); + break; + case EXEC_STATE_WORKING: + n += _mali_osk_snprintf(buf + n, size - n, + "GP group is in state WORKING\n"); + break; + default: + n += _mali_osk_snprintf(buf + n, size - n, + "GP group is in unknown/illegal state %u\n", + gp_group_state); + break; + } + + n += mali_group_dump_state(gp_group, buf + n, size - n); + + n += _mali_osk_snprintf(buf + n, size - n, + "Physical PP groups in WORKING state (count = %u):\n", + group_list_working_count); + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, executor_list) { + n += mali_group_dump_state(group, buf + n, size - n); + } + + n += _mali_osk_snprintf(buf + n, size - n, + "Physical PP groups in IDLE state (count = %u):\n", + group_list_idle_count); + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) { + n += mali_group_dump_state(group, buf + n, size - n); + } + + n += _mali_osk_snprintf(buf + n, size - n, + "Physical PP groups in INACTIVE state (count = %u):\n", + group_list_inactive_count); + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) { + n += mali_group_dump_state(group, buf + n, size - n); + } + + n += _mali_osk_snprintf(buf + n, size - n, + "Physical PP groups in DISABLED state (count = %u):\n", + group_list_disabled_count); + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) { + n += mali_group_dump_state(group, buf + n, size - n); + } + + if (mali_executor_has_virtual_group()) { + switch (virtual_group_state) { + case EXEC_STATE_EMPTY: + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP group is in state EMPTY\n"); + break; + case EXEC_STATE_INACTIVE: + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP group is in state INACTIVE\n"); + break; + case EXEC_STATE_IDLE: + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP group is in state IDLE\n"); + break; + case EXEC_STATE_WORKING: + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP group is in state WORKING\n"); + break; + default: + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP group is in unknown/illegal state %u\n", + virtual_group_state); + break; + } + + n += mali_group_dump_state(virtual_group, buf + n, size - n); + } + + mali_executor_unlock(); + + n += _mali_osk_snprintf(buf + n, size - n, "\n"); + + return n; +} +#endif + +_mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + args->number_of_total_cores = num_physical_pp_cores_total; + args->number_of_enabled_cores = num_physical_pp_cores_enabled; + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + args->version = pp_version; + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + args->number_of_cores = 1; + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + args->version = gp_version; + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args) +{ + struct mali_session_data *session; + struct mali_gp_job *job; + + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + + if (_MALIGP_JOB_RESUME_WITH_NEW_HEAP == args->code) { + _mali_osk_notification_t *new_notification = NULL; + + new_notification = _mali_osk_notification_create( + _MALI_NOTIFICATION_GP_STALLED, + sizeof(_mali_uk_gp_job_suspended_s)); + + if (NULL != new_notification) { + MALI_DEBUG_PRINT(3, ("Executor: Resuming job %u with new heap; 0x%08X - 0x%08X\n", + args->cookie, args->arguments[0], args->arguments[1])); + + mali_executor_lock(); + + /* Resume the job in question if it is still running */ + job = mali_group_get_running_gp_job(gp_group); + if (NULL != job && + args->cookie == mali_gp_job_get_id(job) && + session == mali_gp_job_get_session(job)) { + /* + * Correct job is running, resume with new heap + */ + + mali_gp_job_set_oom_notification(job, + new_notification); + + /* This will also re-enable interrupts */ + mali_group_resume_gp_with_new_heap(gp_group, + args->cookie, + args->arguments[0], + args->arguments[1]); + + mali_executor_unlock(); + return _MALI_OSK_ERR_OK; + } else { + MALI_DEBUG_PRINT(2, ("Executor: Unable to resume gp job becasue gp time out or any other unexpected reason!\n")); + + _mali_osk_notification_delete(new_notification); + + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } + } else { + MALI_PRINT_ERROR(("Executor: Failed to allocate notification object. Will abort GP job.\n")); + } + } else { + MALI_DEBUG_PRINT(2, ("Executor: Aborting job %u, no new heap provided\n", args->cookie)); + } + + mali_executor_lock(); + + /* Abort the job in question if it is still running */ + job = mali_group_get_running_gp_job(gp_group); + if (NULL != job && + args->cookie == mali_gp_job_get_id(job) && + session == mali_gp_job_get_session(job)) { + /* Correct job is still running */ + struct mali_gp_job *job_done = NULL; + + mali_executor_complete_group(gp_group, MALI_FALSE, &job_done, NULL); + + /* The same job should have completed */ + MALI_DEBUG_ASSERT(job_done == job); + + /* GP job completed, make sure it is freed */ + mali_scheduler_complete_gp_job(job_done, MALI_FALSE, + MALI_TRUE, MALI_TRUE); + } + + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; +} + + +/* + * ---------- Implementation of static functions ---------- + */ + +void mali_executor_lock(void) +{ + _mali_osk_spinlock_irq_lock(mali_executor_lock_obj); + MALI_DEBUG_PRINT(5, ("Executor: lock taken\n")); +} + +void mali_executor_unlock(void) +{ + MALI_DEBUG_PRINT(5, ("Executor: Releasing lock\n")); + _mali_osk_spinlock_irq_unlock(mali_executor_lock_obj); +} + +static mali_bool mali_executor_is_suspended(void *data) +{ + mali_bool ret; + + /* This callback does not use the data pointer. */ + MALI_IGNORE(data); + + mali_executor_lock(); + + ret = pause_count > 0 && !mali_executor_is_working(); + + mali_executor_unlock(); + + return ret; +} + +static mali_bool mali_executor_is_working() +{ + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + return (0 != group_list_working_count || + EXEC_STATE_WORKING == gp_group_state || + EXEC_STATE_WORKING == virtual_group_state); +} + +static void mali_executor_disable_empty_virtual(void) +{ + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(virtual_group_state != EXEC_STATE_EMPTY); + MALI_DEBUG_ASSERT(virtual_group_state != EXEC_STATE_WORKING); + + if (mali_group_is_empty(virtual_group)) { + virtual_group_state = EXEC_STATE_EMPTY; + } +} + +static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group) +{ + mali_bool trigger_pm_update = MALI_FALSE; + + MALI_DEBUG_ASSERT_POINTER(group); + /* Only rejoining after job has completed (still active) */ + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == + mali_group_get_state(group)); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(MALI_TRUE == mali_executor_has_virtual_group()); + MALI_DEBUG_ASSERT(MALI_FALSE == mali_group_is_virtual(group)); + + /* Make sure group and virtual group have same status */ + + if (MALI_GROUP_STATE_INACTIVE == mali_group_get_state(virtual_group)) { + if (mali_group_deactivate(group)) { + trigger_pm_update = MALI_TRUE; + } + + if (virtual_group_state == EXEC_STATE_EMPTY) { + virtual_group_state = EXEC_STATE_INACTIVE; + } + } else if (MALI_GROUP_STATE_ACTIVATION_PENDING == + mali_group_get_state(virtual_group)) { + /* + * Activation is pending for virtual group, leave + * this child group as active. + */ + if (virtual_group_state == EXEC_STATE_EMPTY) { + virtual_group_state = EXEC_STATE_INACTIVE; + } + } else { + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == + mali_group_get_state(virtual_group)); + + if (virtual_group_state == EXEC_STATE_EMPTY) { + virtual_group_state = EXEC_STATE_IDLE; + } + } + + /* Remove group from idle list */ + MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(group, + EXEC_STATE_IDLE)); + _mali_osk_list_delinit(&group->executor_list); + group_list_idle_count--; + + /* + * And finally rejoin the virtual group + * group will start working on same job as virtual_group, + * if virtual_group is working on a job + */ + mali_group_add_group(virtual_group, group); + + return trigger_pm_update; +} + +static mali_bool mali_executor_has_virtual_group(void) +{ +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + return (NULL != virtual_group) ? MALI_TRUE : MALI_FALSE; +#else + return MALI_FALSE; +#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ +} + +static mali_bool mali_executor_virtual_group_is_usable(void) +{ +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return ((EXEC_STATE_INACTIVE == virtual_group_state || + EXEC_STATE_IDLE == virtual_group_state) && (virtual_group->state != MALI_GROUP_STATE_ACTIVATION_PENDING)) ? + MALI_TRUE : MALI_FALSE; +#else + return MALI_FALSE; +#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ +} + +static mali_bool mali_executor_tackle_gp_bound(void) +{ + struct mali_pp_job *job; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + job = mali_scheduler_job_pp_physical_peek(); + + if (NULL != job && MALI_TRUE == mali_is_mali400()) { + if (0 < group_list_working_count && + mali_pp_job_is_large_and_unstarted(job)) { + return MALI_TRUE; + } + } + + return MALI_FALSE; +} + +static mali_bool mali_executor_schedule_is_early_out(mali_bool *gpu_secure_mode_is_needed) +{ + struct mali_pp_job *next_pp_job_to_start = NULL; + struct mali_group *group; + struct mali_group *tmp_group; + struct mali_pp_job *physical_pp_job_working = NULL; + struct mali_pp_job *virtual_pp_job_working = NULL; + mali_bool gpu_working_in_protected_mode = MALI_FALSE; + mali_bool gpu_working_in_non_protected_mode = MALI_FALSE; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + + *gpu_secure_mode_is_needed = MALI_FALSE; + + /* Check if the gpu secure mode is supported, exit if not.*/ + if (MALI_FALSE == _mali_osk_gpu_secure_mode_is_supported()) { + return MALI_FALSE; + } + + /* Check if need to set gpu secure mode for the next pp job, + * get the next pp job that will be scheduled if exist. + */ + next_pp_job_to_start = mali_scheduler_job_pp_next(); + + /* Check current pp physical/virtual running job is protected job or not if exist.*/ + _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_working, + struct mali_group, executor_list) { + physical_pp_job_working = group->pp_running_job; + break; + } + + if (EXEC_STATE_WORKING == virtual_group_state) { + virtual_pp_job_working = virtual_group->pp_running_job; + } + + if (NULL != physical_pp_job_working) { + if (MALI_TRUE == mali_pp_job_is_protected_job(physical_pp_job_working)) { + gpu_working_in_protected_mode = MALI_TRUE; + } else { + gpu_working_in_non_protected_mode = MALI_TRUE; + } + } else if (NULL != virtual_pp_job_working) { + if (MALI_TRUE == mali_pp_job_is_protected_job(virtual_pp_job_working)) { + gpu_working_in_protected_mode = MALI_TRUE; + } else { + gpu_working_in_non_protected_mode = MALI_TRUE; + } + } else if (EXEC_STATE_WORKING == gp_group_state) { + gpu_working_in_non_protected_mode = MALI_TRUE; + } + + /* If the next pp job is the protected pp job.*/ + if ((NULL != next_pp_job_to_start) && MALI_TRUE == mali_pp_job_is_protected_job(next_pp_job_to_start)) { + /* if gp is working or any non-protected pp job is working now, unable to schedule protected pp job. */ + if (MALI_TRUE == gpu_working_in_non_protected_mode) + return MALI_TRUE; + + *gpu_secure_mode_is_needed = MALI_TRUE; + return MALI_FALSE; + + } + + if (MALI_TRUE == gpu_working_in_protected_mode) { + /* Unable to schedule non-protected pp job/gp job if exist protected pp running jobs*/ + return MALI_TRUE; + } + + return MALI_FALSE; +} +/* + * This is where jobs are actually started. + */ +static void mali_executor_schedule(void) +{ + u32 i; + u32 num_physical_needed = 0; + u32 num_physical_to_process = 0; + mali_bool trigger_pm_update = MALI_FALSE; + mali_bool deactivate_idle_group = MALI_TRUE; + mali_bool gpu_secure_mode_is_needed = MALI_FALSE; + mali_bool is_gpu_secure_mode = MALI_FALSE; + /* Physical groups + jobs to start in this function */ + struct mali_group *groups_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; + struct mali_pp_job *jobs_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; + u32 sub_jobs_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; + int num_jobs_to_start = 0; + + /* Virtual job to start in this function */ + struct mali_pp_job *virtual_job_to_start = NULL; + + /* GP job to start in this function */ + struct mali_gp_job *gp_job_to_start = NULL; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (pause_count > 0) { + /* Execution is suspended, don't schedule any jobs. */ + return; + } + + /* Lock needed in order to safely handle the job queues */ + mali_scheduler_lock(); + + /* 1. Check the schedule if need to early out. */ + if (MALI_TRUE == mali_executor_schedule_is_early_out(&gpu_secure_mode_is_needed)) { + mali_scheduler_unlock(); + return; + } + + /* 2. Activate gp firstly if have gp job queued. */ + if ((EXEC_STATE_INACTIVE == gp_group_state) + && (0 < mali_scheduler_job_gp_count()) + && (gpu_secure_mode_is_needed == MALI_FALSE)) { + + enum mali_group_state state = + mali_group_activate(gp_group); + if (MALI_GROUP_STATE_ACTIVE == state) { + /* Set GP group state to idle */ + gp_group_state = EXEC_STATE_IDLE; + } else { + trigger_pm_update = MALI_TRUE; + } + } + + /* 3. Prepare as many physical groups as needed/possible */ + + num_physical_needed = mali_scheduler_job_physical_head_count(gpu_secure_mode_is_needed); + + /* On mali-450 platform, we don't need to enter in this block frequently. */ + if (0 < num_physical_needed) { + + if (num_physical_needed <= group_list_idle_count) { + /* We have enough groups on idle list already */ + num_physical_to_process = num_physical_needed; + num_physical_needed = 0; + } else { + /* We need to get a hold of some more groups */ + num_physical_to_process = group_list_idle_count; + num_physical_needed -= group_list_idle_count; + } + + if (0 < num_physical_needed) { + + /* 3.1. Activate groups which are inactive */ + + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, + struct mali_group, executor_list) { + enum mali_group_state state = + mali_group_activate(group); + if (MALI_GROUP_STATE_ACTIVE == state) { + /* Move from inactive to idle */ + mali_executor_change_state_pp_physical(group, + &group_list_inactive, + &group_list_inactive_count, + &group_list_idle, + &group_list_idle_count); + num_physical_to_process++; + } else { + trigger_pm_update = MALI_TRUE; + } + + num_physical_needed--; + if (0 == num_physical_needed) { + /* We have activated all the groups we need */ + break; + } + } + } + + if (mali_executor_virtual_group_is_usable()) { + + /* + * 3.2. And finally, steal and activate groups + * from virtual group if we need even more + */ + while (0 < num_physical_needed) { + struct mali_group *group; + + group = mali_group_acquire_group(virtual_group); + if (NULL != group) { + enum mali_group_state state; + + mali_executor_disable_empty_virtual(); + + state = mali_group_activate(group); + if (MALI_GROUP_STATE_ACTIVE == state) { + /* Group is ready, add to idle list */ + _mali_osk_list_add( + &group->executor_list, + &group_list_idle); + group_list_idle_count++; + num_physical_to_process++; + } else { + /* + * Group is not ready yet, + * add to inactive list + */ + _mali_osk_list_add( + &group->executor_list, + &group_list_inactive); + group_list_inactive_count++; + + trigger_pm_update = MALI_TRUE; + } + num_physical_needed--; + } else { + /* + * We could not get enough groups + * from the virtual group. + */ + break; + } + } + } + + /* 3.3. Assign physical jobs to groups */ + + if (0 < num_physical_to_process) { + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, + struct mali_group, executor_list) { + struct mali_pp_job *job = NULL; + u32 sub_job = MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; + + MALI_DEBUG_ASSERT(num_jobs_to_start < + MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS); + + MALI_DEBUG_ASSERT(0 < + mali_scheduler_job_physical_head_count(gpu_secure_mode_is_needed)); + + /* If the next pp job is non-protected, check if gp bound now. */ + if ((MALI_FALSE == gpu_secure_mode_is_needed) + && (mali_executor_hint_is_enabled(MALI_EXECUTOR_HINT_GP_BOUND)) + && (MALI_TRUE == mali_executor_tackle_gp_bound())) { + /* + * We're gp bound, + * don't start this right now. + */ + deactivate_idle_group = MALI_FALSE; + num_physical_to_process = 0; + break; + } + + job = mali_scheduler_job_pp_physical_get( + &sub_job); + + if (MALI_FALSE == gpu_secure_mode_is_needed) { + MALI_DEBUG_ASSERT(MALI_FALSE == mali_pp_job_is_protected_job(job)); + } else { + MALI_DEBUG_ASSERT(MALI_TRUE == mali_pp_job_is_protected_job(job)); + } + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT(sub_job <= MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS); + + /* Put job + group on list of jobs to start later on */ + + groups_to_start[num_jobs_to_start] = group; + jobs_to_start[num_jobs_to_start] = job; + sub_jobs_to_start[num_jobs_to_start] = sub_job; + num_jobs_to_start++; + + /* Move group from idle to working */ + mali_executor_change_state_pp_physical(group, + &group_list_idle, + &group_list_idle_count, + &group_list_working, + &group_list_working_count); + + num_physical_to_process--; + if (0 == num_physical_to_process) { + /* Got all we needed */ + break; + } + } + } + } + + /* 4. Deactivate idle pp group , must put deactive here before active vitual group + * for cover case first only has physical job in normal queue but group inactive, + * so delay the job start go to active group, when group activated, + * call scheduler again, but now if we get high queue virtual job, + * we will do nothing in schedule cause executor schedule stop + */ + + if (MALI_TRUE == mali_executor_deactivate_list_idle(deactivate_idle_group + && (!mali_timeline_has_physical_pp_job()))) { + trigger_pm_update = MALI_TRUE; + } + + /* 5. Activate virtual group, if needed */ + if (EXEC_STATE_INACTIVE == virtual_group_state && + MALI_TRUE == mali_scheduler_job_next_is_virtual()) { + struct mali_pp_job *virtual_job = mali_scheduler_job_pp_virtual_peek(); + if ((MALI_FALSE == gpu_secure_mode_is_needed && MALI_FALSE == mali_pp_job_is_protected_job(virtual_job)) + || (MALI_TRUE == gpu_secure_mode_is_needed && MALI_TRUE == mali_pp_job_is_protected_job(virtual_job))) { + enum mali_group_state state = + mali_group_activate(virtual_group); + if (MALI_GROUP_STATE_ACTIVE == state) { + /* Set virtual group state to idle */ + virtual_group_state = EXEC_STATE_IDLE; + } else { + trigger_pm_update = MALI_TRUE; + } + } + } + + /* 6. To power up group asap, trigger pm update only when no need to swith the gpu mode. */ + + is_gpu_secure_mode = _mali_osk_gpu_secure_mode_is_enabled(); + + if ((MALI_FALSE == gpu_secure_mode_is_needed && MALI_FALSE == is_gpu_secure_mode) + || (MALI_TRUE == gpu_secure_mode_is_needed && MALI_TRUE == is_gpu_secure_mode)) { + if (MALI_TRUE == trigger_pm_update) { + trigger_pm_update = MALI_FALSE; + mali_pm_update_async(); + } + } + + /* 7. Assign jobs to idle virtual group (or deactivate if no job) */ + + if (EXEC_STATE_IDLE == virtual_group_state) { + if (MALI_TRUE == mali_scheduler_job_next_is_virtual()) { + struct mali_pp_job *virtual_job = mali_scheduler_job_pp_virtual_peek(); + if ((MALI_FALSE == gpu_secure_mode_is_needed && MALI_FALSE == mali_pp_job_is_protected_job(virtual_job)) + || (MALI_TRUE == gpu_secure_mode_is_needed && MALI_TRUE == mali_pp_job_is_protected_job(virtual_job))) { + virtual_job_to_start = + mali_scheduler_job_pp_virtual_get(); + virtual_group_state = EXEC_STATE_WORKING; + } + } else if (!mali_timeline_has_virtual_pp_job()) { + virtual_group_state = EXEC_STATE_INACTIVE; + + if (mali_group_deactivate(virtual_group)) { + trigger_pm_update = MALI_TRUE; + } + } + } + + /* 8. Assign job to idle GP group (or deactivate if no job) */ + + if (EXEC_STATE_IDLE == gp_group_state && MALI_FALSE == gpu_secure_mode_is_needed) { + if (0 < mali_scheduler_job_gp_count()) { + gp_job_to_start = mali_scheduler_job_gp_get(); + gp_group_state = EXEC_STATE_WORKING; + } else if (!mali_timeline_has_gp_job()) { + gp_group_state = EXEC_STATE_INACTIVE; + if (mali_group_deactivate(gp_group)) { + trigger_pm_update = MALI_TRUE; + } + } + } + + /* 9. We no longer need the schedule/queue lock */ + + mali_scheduler_unlock(); + + /* 10. start jobs */ + if (NULL != virtual_job_to_start) { + MALI_DEBUG_ASSERT(!mali_group_pp_is_active(virtual_group)); + mali_group_start_pp_job(virtual_group, + virtual_job_to_start, 0, is_gpu_secure_mode); + } + + for (i = 0; i < num_jobs_to_start; i++) { + MALI_DEBUG_ASSERT(!mali_group_pp_is_active( + groups_to_start[i])); + mali_group_start_pp_job(groups_to_start[i], + jobs_to_start[i], + sub_jobs_to_start[i], is_gpu_secure_mode); + } + + MALI_DEBUG_ASSERT_POINTER(gp_group); + + if (NULL != gp_job_to_start) { + MALI_DEBUG_ASSERT(!mali_group_gp_is_active(gp_group)); + mali_group_start_gp_job(gp_group, gp_job_to_start, is_gpu_secure_mode); + } + + /* 11. Trigger any pending PM updates */ + if (MALI_TRUE == trigger_pm_update) { + mali_pm_update_async(); + } +} + +/* Handler for deferred schedule requests */ +static void mali_executor_wq_schedule(void *arg) +{ + MALI_IGNORE(arg); + mali_executor_lock(); + mali_executor_schedule(); + mali_executor_unlock(); +} + +static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job) +{ + _mali_uk_gp_job_suspended_s *jobres; + _mali_osk_notification_t *notification; + + notification = mali_gp_job_get_oom_notification(job); + + /* + * Remember the id we send to user space, so we have something to + * verify when we get a response + */ + gp_returned_cookie = mali_gp_job_get_id(job); + + jobres = (_mali_uk_gp_job_suspended_s *)notification->result_buffer; + jobres->user_job_ptr = mali_gp_job_get_user_id(job); + jobres->cookie = gp_returned_cookie; + + mali_session_send_notification(mali_gp_job_get_session(job), + notification); +} +static struct mali_gp_job *mali_executor_complete_gp(struct mali_group *group, + mali_bool success) +{ + struct mali_gp_job *job; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + /* Extracts the needed HW status from core and reset */ + job = mali_group_complete_gp(group, success); + + MALI_DEBUG_ASSERT_POINTER(job); + + /* Core is now ready to go into idle list */ + gp_group_state = EXEC_STATE_IDLE; + + /* This will potentially queue more GP and PP jobs */ + mali_timeline_tracker_release(&job->tracker); + + /* Signal PP job */ + mali_gp_job_signal_pp_tracker(job, success); + + return job; +} + +static struct mali_pp_job *mali_executor_complete_pp(struct mali_group *group, + mali_bool success) +{ + struct mali_pp_job *job; + u32 sub_job; + mali_bool job_is_done; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + /* Extracts the needed HW status from core and reset */ + job = mali_group_complete_pp(group, success, &sub_job); + + MALI_DEBUG_ASSERT_POINTER(job); + + /* Core is now ready to go into idle list */ + if (mali_group_is_virtual(group)) { + virtual_group_state = EXEC_STATE_IDLE; + } else { + /* Move from working to idle state */ + mali_executor_change_state_pp_physical(group, + &group_list_working, + &group_list_working_count, + &group_list_idle, + &group_list_idle_count); + } + + /* It is the executor module which owns the jobs themselves by now */ + mali_pp_job_mark_sub_job_completed(job, success); + job_is_done = mali_pp_job_is_complete(job); + + if (job_is_done) { + /* This will potentially queue more GP and PP jobs */ + mali_timeline_tracker_release(&job->tracker); + } + + return job; +} + +static void mali_executor_complete_group(struct mali_group *group, + mali_bool success, + struct mali_gp_job **gp_job_done, + struct mali_pp_job **pp_job_done) +{ + struct mali_gp_core *gp_core = mali_group_get_gp_core(group); + struct mali_pp_core *pp_core = mali_group_get_pp_core(group); + struct mali_gp_job *gp_job = NULL; + struct mali_pp_job *pp_job = NULL; + mali_bool pp_job_is_done = MALI_TRUE; + + if (NULL != gp_core) { + gp_job = mali_executor_complete_gp(group, success); + } else { + MALI_DEBUG_ASSERT_POINTER(pp_core); + MALI_IGNORE(pp_core); + pp_job = mali_executor_complete_pp(group, success); + + pp_job_is_done = mali_pp_job_is_complete(pp_job); + } + + if (pause_count > 0) { + /* Execution has been suspended */ + + if (!mali_executor_is_working()) { + /* Last job completed, wake up sleepers */ + _mali_osk_wait_queue_wake_up( + executor_working_wait_queue); + } + } else if (MALI_TRUE == mali_group_disable_requested(group)) { + mali_executor_core_scale_in_group_complete(group); + + mali_executor_schedule(); + } else { + /* try to schedule new jobs */ + mali_executor_schedule(); + } + + if (NULL != gp_job) { + MALI_DEBUG_ASSERT_POINTER(gp_job_done); + *gp_job_done = gp_job; + } else if (pp_job_is_done) { + MALI_DEBUG_ASSERT_POINTER(pp_job); + MALI_DEBUG_ASSERT_POINTER(pp_job_done); + *pp_job_done = pp_job; + } +} + +static void mali_executor_change_state_pp_physical(struct mali_group *group, + _mali_osk_list_t *old_list, + u32 *old_count, + _mali_osk_list_t *new_list, + u32 *new_count) +{ + /* + * It's a bit more complicated to change the state for the physical PP + * groups since their state is determined by the list they are on. + */ +#if defined(DEBUG) + mali_bool found = MALI_FALSE; + struct mali_group *group_iter; + struct mali_group *temp; + u32 old_counted = 0; + u32 new_counted = 0; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(old_list); + MALI_DEBUG_ASSERT_POINTER(old_count); + MALI_DEBUG_ASSERT_POINTER(new_list); + MALI_DEBUG_ASSERT_POINTER(new_count); + + /* + * Verify that group is present on old list, + * and that the count is correct + */ + + _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, old_list, + struct mali_group, executor_list) { + old_counted++; + if (group == group_iter) { + found = MALI_TRUE; + } + } + + _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, new_list, + struct mali_group, executor_list) { + new_counted++; + } + + if (MALI_FALSE == found) { + if (old_list == &group_list_idle) { + MALI_DEBUG_PRINT(1, (" old Group list is idle,")); + } else if (old_list == &group_list_inactive) { + MALI_DEBUG_PRINT(1, (" old Group list is inactive,")); + } else if (old_list == &group_list_working) { + MALI_DEBUG_PRINT(1, (" old Group list is working,")); + } else if (old_list == &group_list_disabled) { + MALI_DEBUG_PRINT(1, (" old Group list is disable,")); + } + + if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_WORKING)) { + MALI_DEBUG_PRINT(1, (" group in working \n")); + } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_INACTIVE)) { + MALI_DEBUG_PRINT(1, (" group in inactive \n")); + } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_IDLE)) { + MALI_DEBUG_PRINT(1, (" group in idle \n")); + } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)) { + MALI_DEBUG_PRINT(1, (" but group in disabled \n")); + } + } + + MALI_DEBUG_ASSERT(MALI_TRUE == found); + MALI_DEBUG_ASSERT(0 < (*old_count)); + MALI_DEBUG_ASSERT((*old_count) == old_counted); + MALI_DEBUG_ASSERT((*new_count) == new_counted); +#endif + + _mali_osk_list_move(&group->executor_list, new_list); + (*old_count)--; + (*new_count)++; +} + +static void mali_executor_set_state_pp_physical(struct mali_group *group, + _mali_osk_list_t *new_list, + u32 *new_count) +{ + _mali_osk_list_add(&group->executor_list, new_list); + (*new_count)++; +} + +static mali_bool mali_executor_group_is_in_state(struct mali_group *group, + enum mali_executor_state_t state) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (gp_group == group) { + if (gp_group_state == state) { + return MALI_TRUE; + } + } else if (virtual_group == group || mali_group_is_in_virtual(group)) { + if (virtual_group_state == state) { + return MALI_TRUE; + } + } else { + /* Physical PP group */ + struct mali_group *group_iter; + struct mali_group *temp; + _mali_osk_list_t *list; + + if (EXEC_STATE_DISABLED == state) { + list = &group_list_disabled; + } else if (EXEC_STATE_INACTIVE == state) { + list = &group_list_inactive; + } else if (EXEC_STATE_IDLE == state) { + list = &group_list_idle; + } else { + MALI_DEBUG_ASSERT(EXEC_STATE_WORKING == state); + list = &group_list_working; + } + + _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, list, + struct mali_group, executor_list) { + if (group_iter == group) { + return MALI_TRUE; + } + } + } + + /* group not in correct state */ + return MALI_FALSE; +} + +static void mali_executor_group_enable_internal(struct mali_group *group) +{ + MALI_DEBUG_ASSERT(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)); + + /* Put into inactive state (== "lowest" enabled state) */ + if (group == gp_group) { + MALI_DEBUG_ASSERT(EXEC_STATE_DISABLED == gp_group_state); + gp_group_state = EXEC_STATE_INACTIVE; + } else { + mali_executor_change_state_pp_physical(group, + &group_list_disabled, + &group_list_disabled_count, + &group_list_inactive, + &group_list_inactive_count); + + ++num_physical_pp_cores_enabled; + MALI_DEBUG_PRINT(4, ("Enabling group id %d \n", group->pp_core->core_id)); + } + + if (MALI_GROUP_STATE_ACTIVE == mali_group_activate(group)) { + MALI_DEBUG_ASSERT(MALI_TRUE == mali_group_power_is_on(group)); + + /* Move from inactive to idle */ + if (group == gp_group) { + gp_group_state = EXEC_STATE_IDLE; + } else { + mali_executor_change_state_pp_physical(group, + &group_list_inactive, + &group_list_inactive_count, + &group_list_idle, + &group_list_idle_count); + + if (mali_executor_has_virtual_group()) { + if (mali_executor_physical_rejoin_virtual(group)) { + mali_pm_update_async(); + } + } + } + } else { + mali_pm_update_async(); + } +} + +static void mali_executor_group_disable_internal(struct mali_group *group) +{ + mali_bool working; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)); + + working = mali_executor_group_is_in_state(group, EXEC_STATE_WORKING); + if (MALI_TRUE == working) { + /** Group to be disabled once it completes current work, + * when virtual group completes, also check child groups for this flag */ + mali_group_set_disable_request(group, MALI_TRUE); + return; + } + + /* Put into disabled state */ + if (group == gp_group) { + /* GP group */ + MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != gp_group_state); + gp_group_state = EXEC_STATE_DISABLED; + } else { + if (mali_group_is_in_virtual(group)) { + /* A child group of virtual group. move the specific group from virtual group */ + MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != virtual_group_state); + + mali_executor_set_state_pp_physical(group, + &group_list_disabled, + &group_list_disabled_count); + + mali_group_remove_group(virtual_group, group); + mali_executor_disable_empty_virtual(); + } else { + mali_executor_change_group_status_disabled(group); + } + + --num_physical_pp_cores_enabled; + MALI_DEBUG_PRINT(4, ("Disabling group id %d \n", group->pp_core->core_id)); + } + + if (MALI_GROUP_STATE_INACTIVE != group->state) { + if (MALI_TRUE == mali_group_deactivate(group)) { + mali_pm_update_async(); + } + } +} + +static void mali_executor_notify_core_change(u32 num_cores) +{ + mali_bool done = MALI_FALSE; + + if (mali_is_mali450() || mali_is_mali470()) { + return; + } + + /* + * This function gets a bit complicated because we can't hold the session lock while + * allocating notification objects. + */ + while (!done) { + u32 i; + u32 num_sessions_alloc; + u32 num_sessions_with_lock; + u32 used_notification_objects = 0; + _mali_osk_notification_t **notobjs; + + /* Pre allocate the number of notifications objects we need right now (might change after lock has been taken) */ + num_sessions_alloc = mali_session_get_count(); + if (0 == num_sessions_alloc) { + /* No sessions to report to */ + return; + } + + notobjs = (_mali_osk_notification_t **)_mali_osk_malloc(sizeof(_mali_osk_notification_t *) * num_sessions_alloc); + if (NULL == notobjs) { + MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure)\n")); + /* there is probably no point in trying again, system must be really low on memory and probably unusable now anyway */ + return; + } + + for (i = 0; i < num_sessions_alloc; i++) { + notobjs[i] = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_NUM_CORE_CHANGE, sizeof(_mali_uk_pp_num_cores_changed_s)); + if (NULL != notobjs[i]) { + _mali_uk_pp_num_cores_changed_s *data = notobjs[i]->result_buffer; + data->number_of_enabled_cores = num_cores; + } else { + MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure %u)\n", i)); + } + } + + mali_session_lock(); + + /* number of sessions will not change while we hold the lock */ + num_sessions_with_lock = mali_session_get_count(); + + if (num_sessions_alloc >= num_sessions_with_lock) { + /* We have allocated enough notification objects for all the sessions atm */ + struct mali_session_data *session, *tmp; + MALI_SESSION_FOREACH(session, tmp, link) { + MALI_DEBUG_ASSERT(used_notification_objects < num_sessions_alloc); + if (NULL != notobjs[used_notification_objects]) { + mali_session_send_notification(session, notobjs[used_notification_objects]); + notobjs[used_notification_objects] = NULL; /* Don't track this notification object any more */ + } + used_notification_objects++; + } + done = MALI_TRUE; + } + + mali_session_unlock(); + + /* Delete any remaining/unused notification objects */ + for (; used_notification_objects < num_sessions_alloc; used_notification_objects++) { + if (NULL != notobjs[used_notification_objects]) { + _mali_osk_notification_delete(notobjs[used_notification_objects]); + } + } + + _mali_osk_free(notobjs); + } +} + +static mali_bool mali_executor_core_scaling_is_done(void *data) +{ + u32 i; + u32 num_groups; + mali_bool ret = MALI_TRUE; + + MALI_IGNORE(data); + + mali_executor_lock(); + + num_groups = mali_group_get_glob_num_groups(); + + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + if (NULL != group) { + if (MALI_TRUE == group->disable_requested && NULL != mali_group_get_pp_core(group)) { + ret = MALI_FALSE; + break; + } + } + } + mali_executor_unlock(); + + return ret; +} + +static void mali_executor_wq_notify_core_change(void *arg) +{ + MALI_IGNORE(arg); + + if (mali_is_mali450() || mali_is_mali470()) { + return; + } + + _mali_osk_wait_queue_wait_event(executor_notify_core_change_wait_queue, + mali_executor_core_scaling_is_done, NULL); + + mali_executor_notify_core_change(num_physical_pp_cores_enabled); +} + +/** + * Clear all disable request from the _last_ core scaling behavior. + */ +static void mali_executor_core_scaling_reset(void) +{ + u32 i; + u32 num_groups; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + num_groups = mali_group_get_glob_num_groups(); + + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + if (NULL != group) { + group->disable_requested = MALI_FALSE; + } + } + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + core_scaling_delay_up_mask[i] = 0; + } +} + +static void mali_executor_core_scale(unsigned int target_core_nr) +{ + int current_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; + int target_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; + int i; + + MALI_DEBUG_ASSERT(0 < target_core_nr); + MALI_DEBUG_ASSERT(num_physical_pp_cores_total >= target_core_nr); + + mali_executor_lock(); + + if (target_core_nr < num_physical_pp_cores_enabled) { + MALI_DEBUG_PRINT(2, ("Requesting %d cores: disabling %d cores\n", target_core_nr, num_physical_pp_cores_enabled - target_core_nr)); + } else { + MALI_DEBUG_PRINT(2, ("Requesting %d cores: enabling %d cores\n", target_core_nr, target_core_nr - num_physical_pp_cores_enabled)); + } + + /* When a new core scaling request is comming, we should remove the un-doing + * part of the last core scaling request. It's safe because we have only one + * lock(executor lock) protection. */ + mali_executor_core_scaling_reset(); + + mali_pm_get_best_power_cost_mask(num_physical_pp_cores_enabled, current_core_scaling_mask); + mali_pm_get_best_power_cost_mask(target_core_nr, target_core_scaling_mask); + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + target_core_scaling_mask[i] = target_core_scaling_mask[i] - current_core_scaling_mask[i]; + MALI_DEBUG_PRINT(5, ("target_core_scaling_mask[%d] = %d\n", i, target_core_scaling_mask[i])); + } + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (0 > target_core_scaling_mask[i]) { + struct mali_pm_domain *domain; + + domain = mali_pm_domain_get_from_index(i); + + /* Domain is valid and has pp cores */ + if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) { + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &domain->group_list, struct mali_group, pm_domain_list) { + if (NULL != mali_group_get_pp_core(group) && (!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)) + && (!mali_group_is_virtual(group))) { + mali_executor_group_disable_internal(group); + target_core_scaling_mask[i]++; + if ((0 == target_core_scaling_mask[i])) { + break; + } + + } + } + } + } + } + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + /** + * Target_core_scaling_mask[i] is bigger than 0, + * means we need to enable some pp cores in + * this domain whose domain index is i. + */ + if (0 < target_core_scaling_mask[i]) { + struct mali_pm_domain *domain; + + if (num_physical_pp_cores_enabled >= target_core_nr) { + break; + } + + domain = mali_pm_domain_get_from_index(i); + + /* Domain is valid and has pp cores */ + if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) { + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &domain->group_list, struct mali_group, pm_domain_list) { + if (NULL != mali_group_get_pp_core(group) && mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED) + && (!mali_group_is_virtual(group))) { + mali_executor_group_enable_internal(group); + target_core_scaling_mask[i]--; + + if ((0 == target_core_scaling_mask[i]) || num_physical_pp_cores_enabled == target_core_nr) { + break; + } + } + } + } + } + } + + /** + * Here, we may still have some pp cores not been enabled because of some + * pp cores need to be disabled are still in working state. + */ + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (0 < target_core_scaling_mask[i]) { + core_scaling_delay_up_mask[i] = target_core_scaling_mask[i]; + } + } + + mali_executor_schedule(); + mali_executor_unlock(); +} + +static void mali_executor_core_scale_in_group_complete(struct mali_group *group) +{ + int num_pp_cores_disabled = 0; + int num_pp_cores_to_enable = 0; + int i; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(MALI_TRUE == mali_group_disable_requested(group)); + + /* Disable child group of virtual group */ + if (mali_group_is_virtual(group)) { + struct mali_group *child; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + if (MALI_TRUE == mali_group_disable_requested(child)) { + mali_group_set_disable_request(child, MALI_FALSE); + mali_executor_group_disable_internal(child); + num_pp_cores_disabled++; + } + } + mali_group_set_disable_request(group, MALI_FALSE); + } else { + mali_executor_group_disable_internal(group); + mali_group_set_disable_request(group, MALI_FALSE); + if (NULL != mali_group_get_pp_core(group)) { + num_pp_cores_disabled++; + } + } + + num_pp_cores_to_enable = num_pp_cores_disabled; + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (0 < core_scaling_delay_up_mask[i]) { + struct mali_pm_domain *domain; + + if (0 == num_pp_cores_to_enable) { + break; + } + + domain = mali_pm_domain_get_from_index(i); + + /* Domain is valid and has pp cores */ + if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) { + struct mali_group *disabled_group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(disabled_group, temp, &domain->group_list, struct mali_group, pm_domain_list) { + if (NULL != mali_group_get_pp_core(disabled_group) && mali_executor_group_is_in_state(disabled_group, EXEC_STATE_DISABLED)) { + mali_executor_group_enable_internal(disabled_group); + core_scaling_delay_up_mask[i]--; + num_pp_cores_to_enable--; + + if ((0 == core_scaling_delay_up_mask[i]) || 0 == num_pp_cores_to_enable) { + break; + } + } + } + } + } + } + + _mali_osk_wait_queue_wake_up(executor_notify_core_change_wait_queue); +} + +static void mali_executor_change_group_status_disabled(struct mali_group *group) +{ + /* Physical PP group */ + mali_bool idle; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + idle = mali_executor_group_is_in_state(group, EXEC_STATE_IDLE); + if (MALI_TRUE == idle) { + mali_executor_change_state_pp_physical(group, + &group_list_idle, + &group_list_idle_count, + &group_list_disabled, + &group_list_disabled_count); + } else { + mali_executor_change_state_pp_physical(group, + &group_list_inactive, + &group_list_inactive_count, + &group_list_disabled, + &group_list_disabled_count); + } +} + +static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_group) +{ + mali_bool trigger_pm_update = MALI_FALSE; + + if (group_list_idle_count > 0) { + if (mali_executor_has_virtual_group()) { + + /* Rejoin virtual group on Mali-450 */ + + struct mali_group *group; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, + &group_list_idle, + struct mali_group, executor_list) { + if (mali_executor_physical_rejoin_virtual( + group)) { + trigger_pm_update = MALI_TRUE; + } + } + } else if (deactivate_idle_group) { + struct mali_group *group; + struct mali_group *temp; + + /* Deactivate group on Mali-300/400 */ + + _MALI_OSK_LIST_FOREACHENTRY(group, temp, + &group_list_idle, + struct mali_group, executor_list) { + if (mali_group_deactivate(group)) { + trigger_pm_update = MALI_TRUE; + } + + /* Move from idle to inactive */ + mali_executor_change_state_pp_physical(group, + &group_list_idle, + &group_list_idle_count, + &group_list_inactive, + &group_list_inactive_count); + } + } + } + + return trigger_pm_update; +} + +void mali_executor_running_status_print(void) +{ + struct mali_group *group = NULL; + struct mali_group *temp = NULL; + + MALI_PRINT(("GP running job: %p\n", gp_group->gp_running_job)); + if ((gp_group->gp_core) && (gp_group->is_working)) { + mali_group_dump_status(gp_group); + } + MALI_PRINT(("Physical PP groups in WORKING state (count = %u):\n", group_list_working_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, executor_list) { + MALI_PRINT(("PP running job: %p, subjob %d \n", group->pp_running_job, group->pp_running_sub_job)); + mali_group_dump_status(group); + } + MALI_PRINT(("Physical PP groups in INACTIVE state (count = %u):\n", group_list_inactive_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) { + MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); + MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); + } + MALI_PRINT(("Physical PP groups in IDLE state (count = %u):\n", group_list_idle_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) { + MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); + MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); + } + MALI_PRINT(("Physical PP groups in DISABLED state (count = %u):\n", group_list_disabled_count)); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) { + MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off")); + MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description)); + } + + if (mali_executor_has_virtual_group()) { + MALI_PRINT(("Virtual group running job: %p\n", virtual_group->pp_running_job)); + MALI_PRINT(("Virtual group status: %d\n", virtual_group_state)); + MALI_PRINT(("Virtual group->status: %d\n", virtual_group->state)); + MALI_PRINT(("\tSW power: %s\n", virtual_group->power_is_on ? "On" : "Off")); + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &virtual_group->group_list, + struct mali_group, group_list) { + int i = 0; + MALI_PRINT(("\tchild group(%s) running job: %p\n", group->pp_core->hw_core.description, group->pp_running_job)); + MALI_PRINT(("\tchild group(%s)->status: %d\n", group->pp_core->hw_core.description, group->state)); + MALI_PRINT(("\tchild group(%s) SW power: %s\n", group->pp_core->hw_core.description, group->power_is_on ? "On" : "Off")); + if (group->pm_domain) { + MALI_PRINT(("\tPower domain: id %u\n", mali_pm_domain_get_id(group->pm_domain))); + MALI_PRINT(("\tMask:0x%04x \n", mali_pm_domain_get_mask(group->pm_domain))); + MALI_PRINT(("\tUse-count:%u \n", mali_pm_domain_get_use_count(group->pm_domain))); + MALI_PRINT(("\tCurrent power status:%s \n", (mali_pm_domain_get_mask(group->pm_domain)& mali_pm_get_current_mask()) ? "On" : "Off")); + MALI_PRINT(("\tWanted power status:%s \n", (mali_pm_domain_get_mask(group->pm_domain)& mali_pm_get_wanted_mask()) ? "On" : "Off")); + } + + for (i = 0; i < 2; i++) { + if (NULL != group->l2_cache_core[i]) { + struct mali_pm_domain *domain; + domain = mali_l2_cache_get_pm_domain(group->l2_cache_core[i]); + MALI_PRINT(("\t L2(index %d) group SW power: %s\n", i, group->l2_cache_core[i]->power_is_on ? "On" : "Off")); + if (domain) { + MALI_PRINT(("\tL2 Power domain: id %u\n", mali_pm_domain_get_id(domain))); + MALI_PRINT(("\tL2 Mask:0x%04x \n", mali_pm_domain_get_mask(domain))); + MALI_PRINT(("\tL2 Use-count:%u \n", mali_pm_domain_get_use_count(domain))); + MALI_PRINT(("\tL2 Current power status:%s \n", (mali_pm_domain_get_mask(domain) & mali_pm_get_current_mask()) ? "On" : "Off")); + MALI_PRINT(("\tL2 Wanted power status:%s \n", (mali_pm_domain_get_mask(domain) & mali_pm_get_wanted_mask()) ? "On" : "Off")); + } + } + } + } + if (EXEC_STATE_WORKING == virtual_group_state) { + mali_group_dump_status(virtual_group); + } + } +} + +void mali_executor_status_dump(void) +{ + mali_executor_lock(); + mali_scheduler_lock(); + + /* print schedule queue status */ + mali_scheduler_gp_pp_job_queue_print(); + + mali_scheduler_unlock(); + mali_executor_unlock(); +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_executor.h b/drivers/gpu/arm/mali400/mali/common/mali_executor.h new file mode 100755 index 000000000000..4224d6a6cdc4 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_executor.h @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2012, 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_EXECUTOR_H__ +#define __MALI_EXECUTOR_H__ + +#include "mali_osk.h" +#include "mali_scheduler_types.h" +#include "mali_kernel_common.h" + +typedef enum { + MALI_EXECUTOR_HINT_GP_BOUND = 0 +#define MALI_EXECUTOR_HINT_MAX 1 +} mali_executor_hint; + +extern mali_bool mali_executor_hints[MALI_EXECUTOR_HINT_MAX]; + +/* forward declare struct instead of using include */ +struct mali_session_data; +struct mali_group; +struct mali_pp_core; + +extern _mali_osk_spinlock_irq_t *mali_executor_lock_obj; + +#define MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD() MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj); + +_mali_osk_errcode_t mali_executor_initialize(void); +void mali_executor_terminate(void); + +void mali_executor_populate(void); +void mali_executor_depopulate(void); + +void mali_executor_suspend(void); +void mali_executor_resume(void); + +u32 mali_executor_get_num_cores_total(void); +u32 mali_executor_get_num_cores_enabled(void); +struct mali_pp_core *mali_executor_get_virtual_pp(void); +struct mali_group *mali_executor_get_virtual_group(void); + +void mali_executor_zap_all_active(struct mali_session_data *session); + +/** + * Schedule GP and PP according to bitmask. + * + * @param mask A scheduling bitmask. + * @param deferred_schedule MALI_TRUE if schedule should be deferred, MALI_FALSE if not. + */ +void mali_executor_schedule_from_mask(mali_scheduler_mask mask, mali_bool deferred_schedule); + +_mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group, mali_bool in_upper_half); +_mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group, mali_bool in_upper_half); +_mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group, mali_bool in_upper_half); +void mali_executor_group_power_up(struct mali_group *groups[], u32 num_groups); +void mali_executor_group_power_down(struct mali_group *groups[], u32 num_groups); + +void mali_executor_abort_session(struct mali_session_data *session); + +void mali_executor_core_scaling_enable(void); +void mali_executor_core_scaling_disable(void); +mali_bool mali_executor_core_scaling_is_enabled(void); + +void mali_executor_group_enable(struct mali_group *group); +void mali_executor_group_disable(struct mali_group *group); +mali_bool mali_executor_group_is_disabled(struct mali_group *group); + +int mali_executor_set_perf_level(unsigned int target_core_nr, mali_bool override); + +#if MALI_STATE_TRACKING +u32 mali_executor_dump_state(char *buf, u32 size); +#endif + +MALI_STATIC_INLINE void mali_executor_hint_enable(mali_executor_hint hint) +{ + MALI_DEBUG_ASSERT(hint < MALI_EXECUTOR_HINT_MAX); + mali_executor_hints[hint] = MALI_TRUE; +} + +MALI_STATIC_INLINE void mali_executor_hint_disable(mali_executor_hint hint) +{ + MALI_DEBUG_ASSERT(hint < MALI_EXECUTOR_HINT_MAX); + mali_executor_hints[hint] = MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_executor_hint_is_enabled(mali_executor_hint hint) +{ + MALI_DEBUG_ASSERT(hint < MALI_EXECUTOR_HINT_MAX); + return mali_executor_hints[hint]; +} + +void mali_executor_running_status_print(void); +void mali_executor_status_dump(void); +void mali_executor_lock(void); +void mali_executor_unlock(void); +#endif /* __MALI_EXECUTOR_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp.c b/drivers/gpu/arm/mali400/mali/common/mali_gp.c new file mode 100755 index 000000000000..7d3d4aff7c3f --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_gp.c @@ -0,0 +1,357 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_gp.h" +#include "mali_hw_core.h" +#include "mali_group.h" +#include "mali_osk.h" +#include "regs/mali_gp_regs.h" +#include "mali_kernel_common.h" +#include "mali_kernel_core.h" +#if defined(CONFIG_MALI400_PROFILING) +#include "mali_osk_profiling.h" +#endif + +static struct mali_gp_core *mali_global_gp_core = NULL; + +/* Interrupt handlers */ +static void mali_gp_irq_probe_trigger(void *data); +static _mali_osk_errcode_t mali_gp_irq_probe_ack(void *data); + +struct mali_gp_core *mali_gp_create(const _mali_osk_resource_t *resource, struct mali_group *group) +{ + struct mali_gp_core *core = NULL; + + MALI_DEBUG_ASSERT(NULL == mali_global_gp_core); + MALI_DEBUG_PRINT(2, ("Mali GP: Creating Mali GP core: %s\n", resource->description)); + + core = _mali_osk_malloc(sizeof(struct mali_gp_core)); + if (NULL != core) { + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&core->hw_core, resource, MALIGP2_REGISTER_ADDRESS_SPACE_SIZE)) { + _mali_osk_errcode_t ret; + + ret = mali_gp_reset(core); + + if (_MALI_OSK_ERR_OK == ret) { + ret = mali_group_add_gp_core(group, core); + if (_MALI_OSK_ERR_OK == ret) { + /* Setup IRQ handlers (which will do IRQ probing if needed) */ + core->irq = _mali_osk_irq_init(resource->irq, + mali_group_upper_half_gp, + group, + mali_gp_irq_probe_trigger, + mali_gp_irq_probe_ack, + core, + resource->description); + if (NULL != core->irq) { + MALI_DEBUG_PRINT(4, ("Mali GP: set global gp core from 0x%08X to 0x%08X\n", mali_global_gp_core, core)); + mali_global_gp_core = core; + + return core; + } else { + MALI_PRINT_ERROR(("Mali GP: Failed to setup interrupt handlers for GP core %s\n", core->hw_core.description)); + } + mali_group_remove_gp_core(group); + } else { + MALI_PRINT_ERROR(("Mali GP: Failed to add core %s to group\n", core->hw_core.description)); + } + } + mali_hw_core_delete(&core->hw_core); + } + + _mali_osk_free(core); + } else { + MALI_PRINT_ERROR(("Failed to allocate memory for GP core\n")); + } + + return NULL; +} + +void mali_gp_delete(struct mali_gp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + + _mali_osk_irq_term(core->irq); + mali_hw_core_delete(&core->hw_core); + mali_global_gp_core = NULL; + _mali_osk_free(core); +} + +void mali_gp_stop_bus(struct mali_gp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_STOP_BUS); +} + +_mali_osk_errcode_t mali_gp_stop_bus_wait(struct mali_gp_core *core) +{ + int i; + + MALI_DEBUG_ASSERT_POINTER(core); + + /* Send the stop bus command. */ + mali_gp_stop_bus(core); + + /* Wait for bus to be stopped */ + for (i = 0; i < MALI_REG_POLL_COUNT_SLOW; i++) { + if (mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_STATUS) & MALIGP2_REG_VAL_STATUS_BUS_STOPPED) { + break; + } + } + + if (MALI_REG_POLL_COUNT_SLOW == i) { + MALI_PRINT_ERROR(("Mali GP: Failed to stop bus on %s\n", core->hw_core.description)); + return _MALI_OSK_ERR_FAULT; + } + return _MALI_OSK_ERR_OK; +} + +void mali_gp_hard_reset(struct mali_gp_core *core) +{ + const u32 reset_wait_target_register = MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_LIMIT; + const u32 reset_invalid_value = 0xC0FFE000; + const u32 reset_check_value = 0xC01A0000; + const u32 reset_default_value = 0; + int i; + + MALI_DEBUG_ASSERT_POINTER(core); + MALI_DEBUG_PRINT(4, ("Mali GP: Hard reset of core %s\n", core->hw_core.description)); + + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_invalid_value); + + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_RESET); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_check_value); + if (reset_check_value == mali_hw_core_register_read(&core->hw_core, reset_wait_target_register)) { + break; + } + } + + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_PRINT_ERROR(("Mali GP: The hard reset loop didn't work, unable to recover\n")); + } + + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_default_value); /* set it back to the default */ + /* Re-enable interrupts */ + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_MASK_ALL); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); + +} + +void mali_gp_reset_async(struct mali_gp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + + MALI_DEBUG_PRINT(4, ("Mali GP: Reset of core %s\n", core->hw_core.description)); + + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, 0); /* disable the IRQs */ + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALI400GP_REG_VAL_IRQ_RESET_COMPLETED); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALI400GP_REG_VAL_CMD_SOFT_RESET); + +} + +_mali_osk_errcode_t mali_gp_reset_wait(struct mali_gp_core *core) +{ + int i; + u32 rawstat = 0; + + MALI_DEBUG_ASSERT_POINTER(core); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { + rawstat = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT); + if (rawstat & MALI400GP_REG_VAL_IRQ_RESET_COMPLETED) { + break; + } + } + + if (i == MALI_REG_POLL_COUNT_FAST) { + MALI_PRINT_ERROR(("Mali GP: Failed to reset core %s, rawstat: 0x%08x\n", + core->hw_core.description, rawstat)); + return _MALI_OSK_ERR_FAULT; + } + + /* Re-enable interrupts */ + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_MASK_ALL); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_gp_reset(struct mali_gp_core *core) +{ + mali_gp_reset_async(core); + return mali_gp_reset_wait(core); +} + +void mali_gp_job_start(struct mali_gp_core *core, struct mali_gp_job *job) +{ + u32 startcmd = 0; + u32 *frame_registers = mali_gp_job_get_frame_registers(job); + u32 counter_src0 = mali_gp_job_get_perf_counter_src0(job); + u32 counter_src1 = mali_gp_job_get_perf_counter_src1(job); + + MALI_DEBUG_ASSERT_POINTER(core); + + if (mali_gp_job_has_vs_job(job)) { + startcmd |= (u32) MALIGP2_REG_VAL_CMD_START_VS; + } + + if (mali_gp_job_has_plbu_job(job)) { + startcmd |= (u32) MALIGP2_REG_VAL_CMD_START_PLBU; + } + + MALI_DEBUG_ASSERT(0 != startcmd); + + mali_hw_core_register_write_array_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR, frame_registers, MALIGP2_NUM_REGS_FRAME); + + if (MALI_HW_CORE_NO_COUNTER != counter_src0) { + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC, counter_src0); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_ENABLE, MALIGP2_REG_VAL_PERF_CNT_ENABLE); + } + if (MALI_HW_CORE_NO_COUNTER != counter_src1) { + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_SRC, counter_src1); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_ENABLE, MALIGP2_REG_VAL_PERF_CNT_ENABLE); + } + + MALI_DEBUG_PRINT(3, ("Mali GP: Starting job (0x%08x) on core %s with command 0x%08X\n", job, core->hw_core.description, startcmd)); + + mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_UPDATE_PLBU_ALLOC); + + /* Barrier to make sure the previous register write is finished */ + _mali_osk_write_mem_barrier(); + + /* This is the command that starts the core. + * + * Don't actually run the job if PROFILING_SKIP_PP_JOBS are set, just + * force core to assert the completion interrupt. + */ +#if !defined(PROFILING_SKIP_GP_JOBS) + mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, startcmd); +#else + { + u32 bits = 0; + + if (mali_gp_job_has_vs_job(job)) + bits = MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST; + if (mali_gp_job_has_plbu_job(job)) + bits |= MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST; + + mali_hw_core_register_write_relaxed(&core->hw_core, + MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT, bits); + } +#endif + + /* Barrier to make sure the previous register write is finished */ + _mali_osk_write_mem_barrier(); +} + +void mali_gp_resume_with_new_heap(struct mali_gp_core *core, u32 start_addr, u32 end_addr) +{ + u32 irq_readout; + + MALI_DEBUG_ASSERT_POINTER(core); + + irq_readout = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT); + + if (irq_readout & MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM) { + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, (MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | MALIGP2_REG_VAL_IRQ_HANG)); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); /* re-enable interrupts */ + mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_START_ADDR, start_addr); + mali_hw_core_register_write_relaxed(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_END_ADDR, end_addr); + + MALI_DEBUG_PRINT(3, ("Mali GP: Resuming job\n")); + + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_CMD, MALIGP2_REG_VAL_CMD_UPDATE_PLBU_ALLOC); + _mali_osk_write_mem_barrier(); + } + /* + * else: core has been reset between PLBU_OUT_OF_MEM interrupt and this new heap response. + * A timeout or a page fault on Mali-200 PP core can cause this behaviour. + */ +} + +u32 mali_gp_core_get_version(struct mali_gp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_VERSION); +} + +struct mali_gp_core *mali_gp_get_global_gp_core(void) +{ + return mali_global_gp_core; +} + +/* ------------- interrupt handling below ------------------ */ +static void mali_gp_irq_probe_trigger(void *data) +{ + struct mali_gp_core *core = (struct mali_gp_core *)data; + + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_USED); + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT, MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR); + _mali_osk_mem_barrier(); +} + +static _mali_osk_errcode_t mali_gp_irq_probe_ack(void *data) +{ + struct mali_gp_core *core = (struct mali_gp_core *)data; + u32 irq_readout; + + irq_readout = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_STAT); + if (MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR & irq_readout) { + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR); + _mali_osk_mem_barrier(); + return _MALI_OSK_ERR_OK; + } + + return _MALI_OSK_ERR_FAULT; +} + +/* ------ local helper functions below --------- */ +#if MALI_STATE_TRACKING +u32 mali_gp_dump_state(struct mali_gp_core *core, char *buf, u32 size) +{ + int n = 0; + + n += _mali_osk_snprintf(buf + n, size - n, "\tGP: %s\n", core->hw_core.description); + + return n; +} +#endif + +void mali_gp_update_performance_counters(struct mali_gp_core *core, struct mali_gp_job *job) +{ + u32 val0 = 0; + u32 val1 = 0; + u32 counter_src0 = mali_gp_job_get_perf_counter_src0(job); + u32 counter_src1 = mali_gp_job_get_perf_counter_src1(job); + + if (MALI_HW_CORE_NO_COUNTER != counter_src0) { + val0 = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_VALUE); + mali_gp_job_set_perf_counter_value0(job, val0); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_report_hw_counter(COUNTER_VP_0_C0, val0); + _mali_osk_profiling_record_global_counters(COUNTER_VP_0_C0, val0); +#endif + + } + + if (MALI_HW_CORE_NO_COUNTER != counter_src1) { + val1 = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_VALUE); + mali_gp_job_set_perf_counter_value1(job, val1); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_report_hw_counter(COUNTER_VP_0_C1, val1); + _mali_osk_profiling_record_global_counters(COUNTER_VP_0_C1, val1); +#endif + } +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp.h b/drivers/gpu/arm/mali400/mali/common/mali_gp.h new file mode 100755 index 000000000000..3156310f21c7 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_gp.h @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_GP_H__ +#define __MALI_GP_H__ + +#include "mali_osk.h" +#include "mali_gp_job.h" +#include "mali_hw_core.h" +#include "regs/mali_gp_regs.h" + +struct mali_group; + +/** + * Definition of the GP core struct + * Used to track a GP core in the system. + */ +struct mali_gp_core { + struct mali_hw_core hw_core; /**< Common for all HW cores */ + _mali_osk_irq_t *irq; /**< IRQ handler */ +}; + +_mali_osk_errcode_t mali_gp_initialize(void); +void mali_gp_terminate(void); + +struct mali_gp_core *mali_gp_create(const _mali_osk_resource_t *resource, struct mali_group *group); +void mali_gp_delete(struct mali_gp_core *core); + +void mali_gp_stop_bus(struct mali_gp_core *core); +_mali_osk_errcode_t mali_gp_stop_bus_wait(struct mali_gp_core *core); +void mali_gp_reset_async(struct mali_gp_core *core); +_mali_osk_errcode_t mali_gp_reset_wait(struct mali_gp_core *core); +void mali_gp_hard_reset(struct mali_gp_core *core); +_mali_osk_errcode_t mali_gp_reset(struct mali_gp_core *core); + +void mali_gp_job_start(struct mali_gp_core *core, struct mali_gp_job *job); +void mali_gp_resume_with_new_heap(struct mali_gp_core *core, u32 start_addr, u32 end_addr); + +u32 mali_gp_core_get_version(struct mali_gp_core *core); + +struct mali_gp_core *mali_gp_get_global_gp_core(void); + +#if MALI_STATE_TRACKING +u32 mali_gp_dump_state(struct mali_gp_core *core, char *buf, u32 size); +#endif + +void mali_gp_update_performance_counters(struct mali_gp_core *core, struct mali_gp_job *job); + +MALI_STATIC_INLINE const char *mali_gp_core_description(struct mali_gp_core *core) +{ + return core->hw_core.description; +} + +MALI_STATIC_INLINE enum mali_interrupt_result mali_gp_get_interrupt_result(struct mali_gp_core *core) +{ + u32 stat_used = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_STAT) & + MALIGP2_REG_VAL_IRQ_MASK_USED; + + if (0 == stat_used) { + return MALI_INTERRUPT_RESULT_NONE; + } else if ((MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | + MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST) == stat_used) { + return MALI_INTERRUPT_RESULT_SUCCESS; + } else if (MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST == stat_used) { + return MALI_INTERRUPT_RESULT_SUCCESS_VS; + } else if (MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST == stat_used) { + return MALI_INTERRUPT_RESULT_SUCCESS_PLBU; + } else if (MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM & stat_used) { + return MALI_INTERRUPT_RESULT_OOM; + } + + return MALI_INTERRUPT_RESULT_ERROR; +} + +MALI_STATIC_INLINE u32 mali_gp_get_rawstat(struct mali_gp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return mali_hw_core_register_read(&core->hw_core, + MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT); +} + +MALI_STATIC_INLINE u32 mali_gp_is_active(struct mali_gp_core *core) +{ + u32 status = mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_STATUS); + return (status & MALIGP2_REG_VAL_STATUS_MASK_ACTIVE) ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE void mali_gp_mask_all_interrupts(struct mali_gp_core *core) +{ + mali_hw_core_register_write(&core->hw_core, MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_VAL_IRQ_MASK_NONE); +} + +MALI_STATIC_INLINE void mali_gp_enable_interrupts(struct mali_gp_core *core, enum mali_interrupt_result exceptions) +{ + /* Enable all interrupts, except those specified in exceptions */ + u32 value; + + if (MALI_INTERRUPT_RESULT_SUCCESS_VS == exceptions) { + /* Enable all used except VS complete */ + value = MALIGP2_REG_VAL_IRQ_MASK_USED & + ~MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST; + } else { + MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_SUCCESS_PLBU == + exceptions); + /* Enable all used except PLBU complete */ + value = MALIGP2_REG_VAL_IRQ_MASK_USED & + ~MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST; + } + + mali_hw_core_register_write(&core->hw_core, + MALIGP2_REG_ADDR_MGMT_INT_MASK, + value); +} + +MALI_STATIC_INLINE u32 mali_gp_read_plbu_alloc_start_addr(struct mali_gp_core *core) +{ + return mali_hw_core_register_read(&core->hw_core, MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_START_ADDR); +} + +#endif /* __MALI_GP_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp_job.c b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.c new file mode 100755 index 000000000000..5d4d9f2530d3 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.c @@ -0,0 +1,306 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_gp_job.h" +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_uk_types.h" +#include "mali_memory_virtual.h" +#include "mali_memory_defer_bind.h" + +static u32 gp_counter_src0 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ +static u32 gp_counter_src1 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ +static void _mali_gp_del_varying_allocations(struct mali_gp_job *job); + + +static int _mali_gp_add_varying_allocations(struct mali_session_data *session, + struct mali_gp_job *job, + u32 *alloc, + u32 num) +{ + int i = 0; + struct mali_gp_allocation_node *alloc_node; + mali_mem_allocation *mali_alloc = NULL; + struct mali_vma_node *mali_vma_node = NULL; + + for (i = 0 ; i < num ; i++) { + MALI_DEBUG_ASSERT(alloc[i]); + alloc_node = _mali_osk_calloc(1, sizeof(struct mali_gp_allocation_node)); + if (alloc_node) { + INIT_LIST_HEAD(&alloc_node->node); + /* find mali allocation structure by vaddress*/ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, alloc[i], 0); + + if (likely(mali_vma_node)) { + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(alloc[i] == mali_vma_node->vm_node.start); + } else { + MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,can't find allocation %d by address =0x%x, num=%d\n", i, alloc[i], num)); + _mali_osk_free(alloc_node); + goto fail; + } + alloc_node->alloc = mali_alloc; + /* add to gp job varying alloc list*/ + list_move(&alloc_node->node, &job->varying_alloc); + } else + goto fail; + } + + return 0; +fail: + MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,failed to alloc memory!\n")); + _mali_gp_del_varying_allocations(job); + return -1; +} + + +static void _mali_gp_del_varying_allocations(struct mali_gp_job *job) +{ + struct mali_gp_allocation_node *alloc_node, *tmp_node; + + list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) { + list_del(&alloc_node->node); + kfree(alloc_node); + } + INIT_LIST_HEAD(&job->varying_alloc); +} + +struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker) +{ + struct mali_gp_job *job; + u32 perf_counter_flag; + u32 __user *memory_list = NULL; + struct mali_gp_allocation_node *alloc_node, *tmp_node; + _mali_uk_gp_start_job_s copy_of_uargs; + + job = _mali_osk_calloc(1, sizeof(struct mali_gp_job)); + if (NULL != job) { + job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_FINISHED, sizeof(_mali_uk_gp_job_finished_s)); + if (NULL == job->finished_notification) { + goto fail3; + } + + job->oom_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_STALLED, sizeof(_mali_uk_gp_job_suspended_s)); + if (NULL == job->oom_notification) { + goto fail2; + } + + if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) { + goto fail1; + } + + perf_counter_flag = mali_gp_job_get_perf_counter_flag(job); + + /* case when no counters came from user space + * so pass the debugfs / DS-5 provided global ones to the job object */ + if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) || + (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) { + mali_gp_job_set_perf_counter_src0(job, mali_gp_job_get_gp_counter_src0()); + mali_gp_job_set_perf_counter_src1(job, mali_gp_job_get_gp_counter_src1()); + } + + _mali_osk_list_init(&job->list); + job->session = session; + job->id = id; + job->heap_current_addr = job->uargs.frame_registers[4]; + job->perf_counter_value0 = 0; + job->perf_counter_value1 = 0; + job->pid = _mali_osk_get_pid(); + job->tid = _mali_osk_get_tid(); + + + INIT_LIST_HEAD(&job->varying_alloc); + INIT_LIST_HEAD(&job->vary_todo); + job->dmem = NULL; + + if (job->uargs.deferred_mem_num > session->allocation_mgr.mali_allocation_num) { + MALI_PRINT_ERROR(("Mali GP job: The number of varying buffer to defer bind is invalid !\n")); + goto fail1; + } + + /* add varying allocation list*/ + if (job->uargs.deferred_mem_num > 0) { + /* copy varying list from user space*/ + job->varying_list = _mali_osk_calloc(1, sizeof(u32) * job->uargs.deferred_mem_num); + if (!job->varying_list) { + MALI_PRINT_ERROR(("Mali GP job: allocate varying_list failed varying_alloc_num = %d !\n", job->uargs.deferred_mem_num)); + goto fail1; + } + + if (0 != _mali_osk_copy_from_user(©_of_uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) { + goto fail1; + } + memory_list = (u32 __user *)(uintptr_t)copy_of_uargs.deferred_mem_list; + + if (0 != _mali_osk_copy_from_user(job->varying_list, memory_list, sizeof(u32) * job->uargs.deferred_mem_num)) { + MALI_PRINT_ERROR(("Mali GP job: Failed to copy varying list from user space!\n")); + goto fail; + } + + if (unlikely(_mali_gp_add_varying_allocations(session, job, job->varying_list, + job->uargs.deferred_mem_num))) { + MALI_PRINT_ERROR(("Mali GP job: _mali_gp_add_varying_allocations failed!\n")); + goto fail; + } + + /* do preparetion for each allocation */ + list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) { + if (unlikely(_MALI_OSK_ERR_OK != mali_mem_defer_bind_allocation_prepare(alloc_node->alloc, &job->vary_todo, &job->required_varying_memsize))) { + MALI_PRINT_ERROR(("Mali GP job: mali_mem_defer_bind_allocation_prepare failed!\n")); + goto fail; + } + } + + _mali_gp_del_varying_allocations(job); + + /* bind varying here, to avoid memory latency issue. */ + { + struct mali_defer_mem_block dmem_block; + + INIT_LIST_HEAD(&dmem_block.free_pages); + atomic_set(&dmem_block.num_free_pages, 0); + + if (mali_mem_prepare_mem_for_job(job, &dmem_block)) { + MALI_PRINT_ERROR(("Mali GP job: mali_mem_prepare_mem_for_job failed!\n")); + goto fail; + } + if (_MALI_OSK_ERR_OK != mali_mem_defer_bind(job, &dmem_block)) { + MALI_PRINT_ERROR(("gp job create, mali_mem_defer_bind failed! GP %x fail!", job)); + goto fail; + } + } + + if (job->uargs.varying_memsize > MALI_UK_BIG_VARYING_SIZE) { + job->big_job = 1; + } + } + job->pp_tracker = pp_tracker; + if (NULL != job->pp_tracker) { + /* Take a reference on PP job's tracker that will be released when the GP + job is done. */ + mali_timeline_system_tracker_get(session->timeline_system, pp_tracker); + } + + mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_GP, NULL, job); + mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence)); + + return job; + } else { + MALI_PRINT_ERROR(("Mali GP job: _mali_osk_calloc failed!\n")); + return NULL; + } + + +fail: + _mali_osk_free(job->varying_list); + /* Handle allocate fail here, free all varying node */ + { + struct mali_backend_bind_list *bkn, *bkn_tmp; + list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) { + list_del(&bkn->node); + _mali_osk_free(bkn); + } + } +fail1: + _mali_osk_notification_delete(job->oom_notification); +fail2: + _mali_osk_notification_delete(job->finished_notification); +fail3: + _mali_osk_free(job); + return NULL; +} + +void mali_gp_job_delete(struct mali_gp_job *job) +{ + struct mali_backend_bind_list *bkn, *bkn_tmp; + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT(NULL == job->pp_tracker); + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list)); + _mali_osk_free(job->varying_list); + + /* Handle allocate fail here, free all varying node */ + list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) { + list_del(&bkn->node); + _mali_osk_free(bkn); + } + + mali_mem_defer_dmem_free(job); + + /* de-allocate the pre-allocated oom notifications */ + if (NULL != job->oom_notification) { + _mali_osk_notification_delete(job->oom_notification); + job->oom_notification = NULL; + } + if (NULL != job->finished_notification) { + _mali_osk_notification_delete(job->finished_notification); + job->finished_notification = NULL; + } + + _mali_osk_free(job); +} + +void mali_gp_job_list_add(struct mali_gp_job *job, _mali_osk_list_t *list) +{ + struct mali_gp_job *iter; + struct mali_gp_job *tmp; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + + /* Find position in list/queue where job should be added. */ + _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, list, + struct mali_gp_job, list) { + + /* A span is used to handle job ID wrapping. */ + bool job_is_after = (mali_gp_job_get_id(job) - + mali_gp_job_get_id(iter)) < + MALI_SCHEDULER_JOB_ID_SPAN; + + if (job_is_after) { + break; + } + } + + _mali_osk_list_add(&job->list, &iter->list); +} + +u32 mali_gp_job_get_gp_counter_src0(void) +{ + return gp_counter_src0; +} + +void mali_gp_job_set_gp_counter_src0(u32 counter) +{ + gp_counter_src0 = counter; +} + +u32 mali_gp_job_get_gp_counter_src1(void) +{ + return gp_counter_src1; +} + +void mali_gp_job_set_gp_counter_src1(u32 counter) +{ + gp_counter_src1 = counter; +} + +mali_scheduler_mask mali_gp_job_signal_pp_tracker(struct mali_gp_job *job, mali_bool success) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(job); + + if (NULL != job->pp_tracker) { + schedule_mask |= mali_timeline_system_tracker_put(job->session->timeline_system, job->pp_tracker, MALI_FALSE == success); + job->pp_tracker = NULL; + } + + return schedule_mask; +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_gp_job.h b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.h new file mode 100755 index 000000000000..b84333f9f810 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_gp_job.h @@ -0,0 +1,324 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_GP_JOB_H__ +#define __MALI_GP_JOB_H__ + +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_uk_types.h" +#include "mali_session.h" +#include "mali_timeline.h" +#include "mali_scheduler_types.h" +#include "mali_scheduler.h" +#include "mali_executor.h" +#include "mali_timeline.h" + +struct mali_defer_mem; +/** + * This structure represents a GP job + * + * The GP job object itself is not protected by any single lock, + * but relies on other locks instead (scheduler, executor and timeline lock). + * Think of the job object as moving between these sub systems through-out + * its lifetime. Different part of the GP job struct is used by different + * subsystems. Accessor functions ensure that correct lock is taken. + * Do NOT access any data members directly from outside this module! + */ +struct mali_gp_job { + /* + * These members are typically only set at creation, + * and only read later on. + * They do not require any lock protection. + */ + _mali_uk_gp_start_job_s uargs; /**< Arguments from user space */ + struct mali_session_data *session; /**< Session which submitted this job */ + u32 pid; /**< Process ID of submitting process */ + u32 tid; /**< Thread ID of submitting thread */ + u32 id; /**< Identifier for this job in kernel space (sequential numbering) */ + u32 cache_order; /**< Cache order used for L2 cache flushing (sequential numbering) */ + struct mali_timeline_tracker tracker; /**< Timeline tracker for this job */ + struct mali_timeline_tracker *pp_tracker; /**< Pointer to Timeline tracker for PP job that depends on this job. */ + _mali_osk_notification_t *finished_notification; /**< Notification sent back to userspace on job complete */ + + /* + * These members are used by the scheduler, + * protected by scheduler lock + */ + _mali_osk_list_t list; /**< Used to link jobs together in the scheduler queue */ + + /* + * These members are used by the executor and/or group, + * protected by executor lock + */ + _mali_osk_notification_t *oom_notification; /**< Notification sent back to userspace on OOM */ + + /* + * Set by executor/group on job completion, read by scheduler when + * returning job to user. Hold executor lock when setting, + * no lock needed when reading + */ + u32 heap_current_addr; /**< Holds the current HEAP address when the job has completed */ + u32 perf_counter_value0; /**< Value of performance counter 0 (to be returned to user space) */ + u32 perf_counter_value1; /**< Value of performance counter 1 (to be returned to user space) */ + struct mali_defer_mem *dmem; /** < used for defer bind to store dmem info */ + struct list_head varying_alloc; /**< hold the list of varying allocations */ + u32 bind_flag; /** < flag for deferbind*/ + u32 *varying_list; /**< varying memory list need to to defer bind*/ + struct list_head vary_todo; /**< list of backend list need to do defer bind*/ + u32 required_varying_memsize; /** < size of varying memory to reallocate*/ + u32 big_job; /** < if the gp job have large varying output and may take long time*/ +}; + +#define MALI_DEFER_BIND_MEMORY_PREPARED (0x1 << 0) +#define MALI_DEFER_BIND_MEMORY_BINDED (0x1 << 2) + +struct mali_gp_allocation_node { + struct list_head node; + mali_mem_allocation *alloc; +}; + +struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker); +void mali_gp_job_delete(struct mali_gp_job *job); + +u32 mali_gp_job_get_gp_counter_src0(void); +void mali_gp_job_set_gp_counter_src0(u32 counter); +u32 mali_gp_job_get_gp_counter_src1(void); +void mali_gp_job_set_gp_counter_src1(u32 counter); + +MALI_STATIC_INLINE u32 mali_gp_job_get_id(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (NULL == job) ? 0 : job->id; +} + +MALI_STATIC_INLINE void mali_gp_job_set_cache_order(struct mali_gp_job *job, + u32 cache_order) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + job->cache_order = cache_order; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_cache_order(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (NULL == job) ? 0 : job->cache_order; +} + +MALI_STATIC_INLINE u64 mali_gp_job_get_user_id(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.user_job_ptr; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_frame_builder_id(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.frame_builder_id; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_flush_id(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.flush_id; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_pid(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->pid; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_tid(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->tid; +} + +MALI_STATIC_INLINE u32 *mali_gp_job_get_frame_registers(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.frame_registers; +} + +MALI_STATIC_INLINE struct mali_session_data *mali_gp_job_get_session(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->session; +} + +MALI_STATIC_INLINE mali_bool mali_gp_job_has_vs_job(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->uargs.frame_registers[0] != job->uargs.frame_registers[1]) ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_gp_job_has_plbu_job(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->uargs.frame_registers[2] != job->uargs.frame_registers[3]) ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_current_heap_addr(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->heap_current_addr; +} + +MALI_STATIC_INLINE void mali_gp_job_set_current_heap_addr(struct mali_gp_job *job, u32 heap_addr) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + job->heap_current_addr = heap_addr; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_flag(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.perf_counter_flag; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_src0(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.perf_counter_src0; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_src1(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.perf_counter_src1; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_value0(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->perf_counter_value0; +} + +MALI_STATIC_INLINE u32 mali_gp_job_get_perf_counter_value1(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->perf_counter_value1; +} + +MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_src0(struct mali_gp_job *job, u32 src) +{ + MALI_DEBUG_ASSERT_POINTER(job); + job->uargs.perf_counter_src0 = src; +} + +MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_src1(struct mali_gp_job *job, u32 src) +{ + MALI_DEBUG_ASSERT_POINTER(job); + job->uargs.perf_counter_src1 = src; +} + +MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_value0(struct mali_gp_job *job, u32 value) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + job->perf_counter_value0 = value; +} + +MALI_STATIC_INLINE void mali_gp_job_set_perf_counter_value1(struct mali_gp_job *job, u32 value) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + job->perf_counter_value1 = value; +} + +void mali_gp_job_list_add(struct mali_gp_job *job, _mali_osk_list_t *list); + +MALI_STATIC_INLINE void mali_gp_job_list_move(struct mali_gp_job *job, + _mali_osk_list_t *list) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT(!_mali_osk_list_empty(&job->list)); + _mali_osk_list_move(&job->list, list); +} + +MALI_STATIC_INLINE void mali_gp_job_list_remove(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + _mali_osk_list_delinit(&job->list); +} + +MALI_STATIC_INLINE _mali_osk_notification_t * +mali_gp_job_get_finished_notification(struct mali_gp_job *job) +{ + _mali_osk_notification_t *notification; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(job->finished_notification); + + notification = job->finished_notification; + job->finished_notification = NULL; + + return notification; +} + +MALI_STATIC_INLINE _mali_osk_notification_t *mali_gp_job_get_oom_notification( + struct mali_gp_job *job) +{ + _mali_osk_notification_t *notification; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT_POINTER(job->oom_notification); + + notification = job->oom_notification; + job->oom_notification = NULL; + + return notification; +} + +MALI_STATIC_INLINE void mali_gp_job_set_oom_notification( + struct mali_gp_job *job, + _mali_osk_notification_t *notification) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(NULL == job->oom_notification); + job->oom_notification = notification; +} + +MALI_STATIC_INLINE struct mali_timeline_tracker *mali_gp_job_get_tracker( + struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return &(job->tracker); +} + + +MALI_STATIC_INLINE u32 *mali_gp_job_get_timeline_point_ptr( + struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (u32 __user *)(uintptr_t)job->uargs.timeline_point_ptr; +} + + +/** + * Release reference on tracker for PP job that depends on this GP job. + * + * @note If GP job has a reference on tracker, this function MUST be called before the GP job is + * deleted. + * + * @param job GP job that is done. + * @param success MALI_TRUE if job completed successfully, MALI_FALSE if not. + * @return A scheduling bitmask indicating whether scheduling needs to be done. + */ +mali_scheduler_mask mali_gp_job_signal_pp_tracker(struct mali_gp_job *job, mali_bool success); + +#endif /* __MALI_GP_JOB_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_group.c b/drivers/gpu/arm/mali400/mali/common/mali_group.c new file mode 100755 index 000000000000..47979a2f1a35 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_group.c @@ -0,0 +1,1875 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include "mali_kernel_common.h" +#include "mali_group.h" +#include "mali_osk.h" +#include "mali_l2_cache.h" +#include "mali_gp.h" +#include "mali_pp.h" +#include "mali_mmu.h" +#include "mali_dlbu.h" +#include "mali_broadcast.h" +#include "mali_scheduler.h" +#include "mali_osk_profiling.h" +#include "mali_osk_mali.h" +#include "mali_pm_domain.h" +#include "mali_pm.h" +#include "mali_executor.h" + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) +#include +#include +#endif + +#define MALI_MAX_NUM_DOMAIN_REFS (MALI_MAX_NUMBER_OF_GROUPS * 2) + +#if defined(CONFIG_MALI400_PROFILING) +static void mali_group_report_l2_cache_counters_per_core(struct mali_group *group, u32 core_num); +#endif /* #if defined(CONFIG_MALI400_PROFILING) */ + +static struct mali_group *mali_global_groups[MALI_MAX_NUMBER_OF_GROUPS] = { NULL, }; +static u32 mali_global_num_groups = 0; + +/* SW timer for job execution */ +int mali_max_job_runtime = MALI_MAX_JOB_RUNTIME_DEFAULT; + +/* local helper functions */ +static void mali_group_bottom_half_mmu(void *data); +static void mali_group_bottom_half_gp(void *data); +static void mali_group_bottom_half_pp(void *data); +static void mali_group_timeout(void *data); +static void mali_group_reset_pp(struct mali_group *group); +static void mali_group_reset_mmu(struct mali_group *group); + +static void mali_group_activate_page_directory(struct mali_group *group, struct mali_session_data *session, mali_bool is_reload); +static void mali_group_recovery_reset(struct mali_group *group); + +struct mali_group *mali_group_create(struct mali_l2_cache_core *core, + struct mali_dlbu_core *dlbu, + struct mali_bcast_unit *bcast, + u32 domain_index) +{ + struct mali_group *group = NULL; + + if (mali_global_num_groups >= MALI_MAX_NUMBER_OF_GROUPS) { + MALI_PRINT_ERROR(("Mali group: Too many group objects created\n")); + return NULL; + } + + group = _mali_osk_calloc(1, sizeof(struct mali_group)); + if (NULL != group) { + group->timeout_timer = _mali_osk_timer_init(mali_group_timeout); + if (NULL != group->timeout_timer) { + _mali_osk_timer_setcallback(group->timeout_timer, mali_group_timeout, (void *)group); + + group->l2_cache_core[0] = core; + _mali_osk_list_init(&group->group_list); + _mali_osk_list_init(&group->executor_list); + _mali_osk_list_init(&group->pm_domain_list); + group->bcast_core = bcast; + group->dlbu_core = dlbu; + + /* register this object as a part of the correct power domain */ + if ((NULL != core) || (NULL != dlbu) || (NULL != bcast)) + group->pm_domain = mali_pm_register_group(domain_index, group); + + mali_global_groups[mali_global_num_groups] = group; + mali_global_num_groups++; + + return group; + } + _mali_osk_free(group); + } + + return NULL; +} + +void mali_group_delete(struct mali_group *group) +{ + u32 i; + + MALI_DEBUG_PRINT(4, ("Deleting group %s\n", + mali_group_core_description(group))); + + MALI_DEBUG_ASSERT(NULL == group->parent_group); + MALI_DEBUG_ASSERT((MALI_GROUP_STATE_INACTIVE == group->state) || ((MALI_GROUP_STATE_ACTIVATION_PENDING == group->state))); + + /* Delete the resources that this group owns */ + if (NULL != group->gp_core) { + mali_gp_delete(group->gp_core); + } + + if (NULL != group->pp_core) { + mali_pp_delete(group->pp_core); + } + + if (NULL != group->mmu) { + mali_mmu_delete(group->mmu); + } + + if (mali_group_is_virtual(group)) { + /* Remove all groups from virtual group */ + struct mali_group *child; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + child->parent_group = NULL; + mali_group_delete(child); + } + + mali_dlbu_delete(group->dlbu_core); + + if (NULL != group->bcast_core) { + mali_bcast_unit_delete(group->bcast_core); + } + } + + for (i = 0; i < mali_global_num_groups; i++) { + if (mali_global_groups[i] == group) { + mali_global_groups[i] = NULL; + mali_global_num_groups--; + + if (i != mali_global_num_groups) { + /* We removed a group from the middle of the array -- move the last + * group to the current position to close the gap */ + mali_global_groups[i] = mali_global_groups[mali_global_num_groups]; + mali_global_groups[mali_global_num_groups] = NULL; + } + + break; + } + } + + if (NULL != group->timeout_timer) { + _mali_osk_timer_del(group->timeout_timer); + _mali_osk_timer_term(group->timeout_timer); + } + + if (NULL != group->bottom_half_work_mmu) { + _mali_osk_wq_delete_work(group->bottom_half_work_mmu); + } + + if (NULL != group->bottom_half_work_gp) { + _mali_osk_wq_delete_work(group->bottom_half_work_gp); + } + + if (NULL != group->bottom_half_work_pp) { + _mali_osk_wq_delete_work(group->bottom_half_work_pp); + } + + _mali_osk_free(group); +} + +_mali_osk_errcode_t mali_group_add_mmu_core(struct mali_group *group, struct mali_mmu_core *mmu_core) +{ + /* This group object now owns the MMU core object */ + group->mmu = mmu_core; + group->bottom_half_work_mmu = _mali_osk_wq_create_work(mali_group_bottom_half_mmu, group); + if (NULL == group->bottom_half_work_mmu) { + return _MALI_OSK_ERR_FAULT; + } + return _MALI_OSK_ERR_OK; +} + +void mali_group_remove_mmu_core(struct mali_group *group) +{ + /* This group object no longer owns the MMU core object */ + group->mmu = NULL; + if (NULL != group->bottom_half_work_mmu) { + _mali_osk_wq_delete_work(group->bottom_half_work_mmu); + } +} + +_mali_osk_errcode_t mali_group_add_gp_core(struct mali_group *group, struct mali_gp_core *gp_core) +{ + /* This group object now owns the GP core object */ + group->gp_core = gp_core; + group->bottom_half_work_gp = _mali_osk_wq_create_work(mali_group_bottom_half_gp, group); + if (NULL == group->bottom_half_work_gp) { + return _MALI_OSK_ERR_FAULT; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_group_remove_gp_core(struct mali_group *group) +{ + /* This group object no longer owns the GP core object */ + group->gp_core = NULL; + if (NULL != group->bottom_half_work_gp) { + _mali_osk_wq_delete_work(group->bottom_half_work_gp); + } +} + +_mali_osk_errcode_t mali_group_add_pp_core(struct mali_group *group, struct mali_pp_core *pp_core) +{ + /* This group object now owns the PP core object */ + group->pp_core = pp_core; + group->bottom_half_work_pp = _mali_osk_wq_create_work(mali_group_bottom_half_pp, group); + if (NULL == group->bottom_half_work_pp) { + return _MALI_OSK_ERR_FAULT; + } + return _MALI_OSK_ERR_OK; +} + +void mali_group_remove_pp_core(struct mali_group *group) +{ + /* This group object no longer owns the PP core object */ + group->pp_core = NULL; + if (NULL != group->bottom_half_work_pp) { + _mali_osk_wq_delete_work(group->bottom_half_work_pp); + } +} + +enum mali_group_state mali_group_activate(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_PRINT(4, ("Group: Activating group %s\n", + mali_group_core_description(group))); + + if (MALI_GROUP_STATE_INACTIVE == group->state) { + /* Group is inactive, get PM refs in order to power up */ + + /* + * We'll take a maximum of 2 power domain references pr group, + * one for the group itself, and one for it's L2 cache. + */ + struct mali_pm_domain *domains[MALI_MAX_NUM_DOMAIN_REFS]; + struct mali_group *groups[MALI_MAX_NUM_DOMAIN_REFS]; + u32 num_domains = 0; + mali_bool all_groups_on; + + /* Deal with child groups first */ + if (mali_group_is_virtual(group)) { + /* + * The virtual group might have 0, 1 or 2 L2s in + * its l2_cache_core array, but we ignore these and + * let the child groups take the needed L2 cache ref + * on behalf of the virtual group. + * In other words; The L2 refs are taken in pair with + * the physical group which the L2 is attached to. + */ + struct mali_group *child; + struct mali_group *temp; + + /* + * Child group is inactive, get PM + * refs in order to power up. + */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, + &group->group_list, + struct mali_group, group_list) { + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_INACTIVE + == child->state); + + child->state = MALI_GROUP_STATE_ACTIVATION_PENDING; + + MALI_DEBUG_ASSERT_POINTER( + child->pm_domain); + domains[num_domains] = child->pm_domain; + groups[num_domains] = child; + num_domains++; + + /* + * Take L2 domain ref for child group. + */ + MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS + > num_domains); + domains[num_domains] = mali_l2_cache_get_pm_domain( + child->l2_cache_core[0]); + groups[num_domains] = NULL; + MALI_DEBUG_ASSERT(NULL == + child->l2_cache_core[1]); + num_domains++; + } + } else { + /* Take L2 domain ref for physical groups. */ + MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS > + num_domains); + + domains[num_domains] = mali_l2_cache_get_pm_domain( + group->l2_cache_core[0]); + groups[num_domains] = NULL; + MALI_DEBUG_ASSERT(NULL == group->l2_cache_core[1]); + num_domains++; + } + + /* Do the group itself last (it's dependencies first) */ + + group->state = MALI_GROUP_STATE_ACTIVATION_PENDING; + + MALI_DEBUG_ASSERT_POINTER(group->pm_domain); + domains[num_domains] = group->pm_domain; + groups[num_domains] = group; + num_domains++; + + all_groups_on = mali_pm_get_domain_refs(domains, groups, + num_domains); + + /* + * Complete activation for group, include + * virtual group or physical group. + */ + if (MALI_TRUE == all_groups_on) { + + mali_group_set_active(group); + } + } else if (MALI_GROUP_STATE_ACTIVE == group->state) { + /* Already active */ + MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on); + } else { + /* + * Activation already pending, group->power_is_on could + * be both true or false. We need to wait for power up + * notification anyway. + */ + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVATION_PENDING + == group->state); + } + + MALI_DEBUG_PRINT(4, ("Group: group %s activation result: %s\n", + mali_group_core_description(group), + MALI_GROUP_STATE_ACTIVE == group->state ? + "ACTIVE" : "PENDING")); + + return group->state; +} + +mali_bool mali_group_set_active(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVATION_PENDING == group->state); + MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on); + + MALI_DEBUG_PRINT(4, ("Group: Activation completed for %s\n", + mali_group_core_description(group))); + + if (mali_group_is_virtual(group)) { + struct mali_group *child; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, + struct mali_group, group_list) { + if (MALI_TRUE != child->power_is_on) { + return MALI_FALSE; + } + + child->state = MALI_GROUP_STATE_ACTIVE; + } + + mali_group_reset(group); + } + + /* Go to ACTIVE state */ + group->state = MALI_GROUP_STATE_ACTIVE; + + return MALI_TRUE; +} + +mali_bool mali_group_deactivate(struct mali_group *group) +{ + struct mali_pm_domain *domains[MALI_MAX_NUM_DOMAIN_REFS]; + u32 num_domains = 0; + mali_bool power_down = MALI_FALSE; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_INACTIVE != group->state); + + MALI_DEBUG_PRINT(3, ("Group: Deactivating group %s\n", + mali_group_core_description(group))); + + group->state = MALI_GROUP_STATE_INACTIVE; + + MALI_DEBUG_ASSERT_POINTER(group->pm_domain); + domains[num_domains] = group->pm_domain; + num_domains++; + + if (mali_group_is_virtual(group)) { + /* Release refs for all child groups */ + struct mali_group *child; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, + &group->group_list, + struct mali_group, group_list) { + child->state = MALI_GROUP_STATE_INACTIVE; + + MALI_DEBUG_ASSERT_POINTER(child->pm_domain); + domains[num_domains] = child->pm_domain; + num_domains++; + + /* Release L2 cache domain for child groups */ + MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS > + num_domains); + domains[num_domains] = mali_l2_cache_get_pm_domain( + child->l2_cache_core[0]); + MALI_DEBUG_ASSERT(NULL == child->l2_cache_core[1]); + num_domains++; + } + + /* + * Must do mali_group_power_down() steps right here for + * virtual group, because virtual group itself is likely to + * stay powered on, however child groups are now very likely + * to be powered off (and thus lose their state). + */ + + mali_group_clear_session(group); + /* + * Disable the broadcast unit (clear it's mask). + * This is needed in case the GPU isn't actually + * powered down at this point and groups are + * removed from an inactive virtual group. + * If not, then the broadcast unit will intercept + * their interrupts! + */ + mali_bcast_disable(group->bcast_core); + } else { + /* Release L2 cache domain for physical groups */ + MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS > + num_domains); + domains[num_domains] = mali_l2_cache_get_pm_domain( + group->l2_cache_core[0]); + MALI_DEBUG_ASSERT(NULL == group->l2_cache_core[1]); + num_domains++; + } + + power_down = mali_pm_put_domain_refs(domains, num_domains); + + return power_down; +} + +void mali_group_power_up(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_PRINT(3, ("Group: Power up for %s\n", + mali_group_core_description(group))); + + group->power_is_on = MALI_TRUE; + + if (MALI_FALSE == mali_group_is_virtual(group) + && MALI_FALSE == mali_group_is_in_virtual(group)) { + mali_group_reset(group); + } + + /* + * When we just acquire only one physical group form virt group, + * we should remove the bcast&dlbu mask from virt group and + * reset bcast and dlbu core, although part of pp cores in virt + * group maybe not be powered on. + */ + if (MALI_TRUE == mali_group_is_virtual(group)) { + mali_bcast_reset(group->bcast_core); + mali_dlbu_update_mask(group->dlbu_core); + } +} + +void mali_group_power_down(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_PRINT(3, ("Group: Power down for %s\n", + mali_group_core_description(group))); + + group->power_is_on = MALI_FALSE; + + if (mali_group_is_virtual(group)) { + /* + * What we do for physical jobs in this function should + * already have been done in mali_group_deactivate() + * for virtual group. + */ + MALI_DEBUG_ASSERT(NULL == group->session); + } else { + mali_group_clear_session(group); + } +} + +MALI_DEBUG_CODE(static void mali_group_print_virtual(struct mali_group *vgroup) +{ + u32 i; + struct mali_group *group; + struct mali_group *temp; + + MALI_DEBUG_PRINT(4, ("Virtual group %s (%p)\n", + mali_group_core_description(vgroup), + vgroup)); + MALI_DEBUG_PRINT(4, ("l2_cache_core[0] = %p, ref = %d\n", vgroup->l2_cache_core[0], vgroup->l2_cache_core_ref_count[0])); + MALI_DEBUG_PRINT(4, ("l2_cache_core[1] = %p, ref = %d\n", vgroup->l2_cache_core[1], vgroup->l2_cache_core_ref_count[1])); + + i = 0; + _MALI_OSK_LIST_FOREACHENTRY(group, temp, &vgroup->group_list, struct mali_group, group_list) { + MALI_DEBUG_PRINT(4, ("[%d] %s (%p), l2_cache_core[0] = %p\n", + i, mali_group_core_description(group), + group, group->l2_cache_core[0])); + i++; + } +}) + +static void mali_group_dump_core_status(struct mali_group *group) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT(NULL != group->gp_core || (NULL != group->pp_core && !mali_group_is_virtual(group))); + + if (NULL != group->gp_core) { + MALI_PRINT(("Dump Group %s\n", group->gp_core->hw_core.description)); + + for (i = 0; i < 0xA8; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->gp_core->hw_core, i), + mali_hw_core_register_read(&group->gp_core->hw_core, i + 4), + mali_hw_core_register_read(&group->gp_core->hw_core, i + 8), + mali_hw_core_register_read(&group->gp_core->hw_core, i + 12))); + } + + + } else { + MALI_PRINT(("Dump Group %s\n", group->pp_core->hw_core.description)); + + for (i = 0; i < 0x5c; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->pp_core->hw_core, i), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 4), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 8), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 12))); + } + + /* Ignore some minor registers */ + for (i = 0x1000; i < 0x1068; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->pp_core->hw_core, i), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 4), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 8), + mali_hw_core_register_read(&group->pp_core->hw_core, i + 12))); + } + } + + MALI_PRINT(("Dump Group MMU\n")); + for (i = 0; i < 0x24; i += 0x10) { + MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->mmu->hw_core, i), + mali_hw_core_register_read(&group->mmu->hw_core, i + 4), + mali_hw_core_register_read(&group->mmu->hw_core, i + 8), + mali_hw_core_register_read(&group->mmu->hw_core, i + 12))); + } +} + + +/** + * @Dump group status + */ +void mali_group_dump_status(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + + if (mali_group_is_virtual(group)) { + struct mali_group *group_c; + struct mali_group *temp; + _MALI_OSK_LIST_FOREACHENTRY(group_c, temp, &group->group_list, struct mali_group, group_list) { + mali_group_dump_core_status(group_c); + } + } else { + mali_group_dump_core_status(group); + } +} + +/** + * @brief Add child group to virtual group parent + */ +void mali_group_add_group(struct mali_group *parent, struct mali_group *child) +{ + mali_bool found; + u32 i; + + MALI_DEBUG_PRINT(3, ("Adding group %s to virtual group %s\n", + mali_group_core_description(child), + mali_group_core_description(parent))); + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_virtual(parent)); + MALI_DEBUG_ASSERT(!mali_group_is_virtual(child)); + MALI_DEBUG_ASSERT(NULL == child->parent_group); + + _mali_osk_list_addtail(&child->group_list, &parent->group_list); + + child->parent_group = parent; + + MALI_DEBUG_ASSERT_POINTER(child->l2_cache_core[0]); + + MALI_DEBUG_PRINT(4, ("parent->l2_cache_core: [0] = %p, [1] = %p\n", parent->l2_cache_core[0], parent->l2_cache_core[1])); + MALI_DEBUG_PRINT(4, ("child->l2_cache_core: [0] = %p, [1] = %p\n", child->l2_cache_core[0], child->l2_cache_core[1])); + + /* Keep track of the L2 cache cores of child groups */ + found = MALI_FALSE; + for (i = 0; i < 2; i++) { + if (parent->l2_cache_core[i] == child->l2_cache_core[0]) { + MALI_DEBUG_ASSERT(parent->l2_cache_core_ref_count[i] > 0); + parent->l2_cache_core_ref_count[i]++; + found = MALI_TRUE; + } + } + + if (!found) { + /* First time we see this L2 cache, add it to our list */ + i = (NULL == parent->l2_cache_core[0]) ? 0 : 1; + + MALI_DEBUG_PRINT(4, ("First time we see l2_cache %p. Adding to [%d] = %p\n", child->l2_cache_core[0], i, parent->l2_cache_core[i])); + + MALI_DEBUG_ASSERT(NULL == parent->l2_cache_core[i]); + + parent->l2_cache_core[i] = child->l2_cache_core[0]; + parent->l2_cache_core_ref_count[i]++; + } + + /* Update Broadcast Unit and DLBU */ + mali_bcast_add_group(parent->bcast_core, child); + mali_dlbu_add_group(parent->dlbu_core, child); + + if (MALI_TRUE == parent->power_is_on) { + mali_bcast_reset(parent->bcast_core); + mali_dlbu_update_mask(parent->dlbu_core); + } + + if (MALI_TRUE == child->power_is_on) { + if (NULL == parent->session) { + if (NULL != child->session) { + /* + * Parent has no session, so clear + * child session as well. + */ + mali_mmu_activate_empty_page_directory(child->mmu); + } + } else { + if (parent->session == child->session) { + /* We already have same session as parent, + * so a simple zap should be enough. + */ + mali_mmu_zap_tlb(child->mmu); + } else { + /* + * Parent has a different session, so we must + * switch to that sessions page table + */ + mali_mmu_activate_page_directory(child->mmu, mali_session_get_page_directory(parent->session)); + } + + /* It is the parent which keeps the session from now on */ + child->session = NULL; + } + } else { + /* should have been cleared when child was powered down */ + MALI_DEBUG_ASSERT(NULL == child->session); + } + + /* Start job on child when parent is active */ + if (NULL != parent->pp_running_job) { + struct mali_pp_job *job = parent->pp_running_job; + + MALI_DEBUG_PRINT(3, ("Group %x joining running job %d on virtual group %x\n", + child, mali_pp_job_get_id(job), parent)); + + /* Only allowed to add active child to an active parent */ + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == parent->state); + MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == child->state); + + mali_pp_job_start(child->pp_core, job, mali_pp_core_get_id(child->pp_core), MALI_TRUE); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | + MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | + MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL, + mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0); +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + if (child->pp_core) { + trace_gpu_sched_switch( + mali_pp_core_description(child->pp_core), + sched_clock(), mali_pp_job_get_tid(job), + 0, mali_pp_job_get_id(job)); + } +#endif + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core), + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job)); +#endif + } + + MALI_DEBUG_CODE(mali_group_print_virtual(parent);) +} + +/** + * @brief Remove child group from virtual group parent + */ +void mali_group_remove_group(struct mali_group *parent, struct mali_group *child) +{ + u32 i; + + MALI_DEBUG_PRINT(3, ("Removing group %s from virtual group %s\n", + mali_group_core_description(child), + mali_group_core_description(parent))); + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_virtual(parent)); + MALI_DEBUG_ASSERT(!mali_group_is_virtual(child)); + MALI_DEBUG_ASSERT(parent == child->parent_group); + + /* Update Broadcast Unit and DLBU */ + mali_bcast_remove_group(parent->bcast_core, child); + mali_dlbu_remove_group(parent->dlbu_core, child); + + if (MALI_TRUE == parent->power_is_on) { + mali_bcast_reset(parent->bcast_core); + mali_dlbu_update_mask(parent->dlbu_core); + } + + child->session = parent->session; + child->parent_group = NULL; + + _mali_osk_list_delinit(&child->group_list); + if (_mali_osk_list_empty(&parent->group_list)) { + parent->session = NULL; + } + + /* Keep track of the L2 cache cores of child groups */ + i = (child->l2_cache_core[0] == parent->l2_cache_core[0]) ? 0 : 1; + + MALI_DEBUG_ASSERT(child->l2_cache_core[0] == parent->l2_cache_core[i]); + + parent->l2_cache_core_ref_count[i]--; + if (parent->l2_cache_core_ref_count[i] == 0) { + parent->l2_cache_core[i] = NULL; + } + + MALI_DEBUG_CODE(mali_group_print_virtual(parent)); +} + +struct mali_group *mali_group_acquire_group(struct mali_group *parent) +{ + struct mali_group *child = NULL; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(mali_group_is_virtual(parent)); + + if (!_mali_osk_list_empty(&parent->group_list)) { + child = _MALI_OSK_LIST_ENTRY(parent->group_list.prev, struct mali_group, group_list); + mali_group_remove_group(parent, child); + } + + if (NULL != child) { + if (MALI_GROUP_STATE_ACTIVE != parent->state + && MALI_TRUE == child->power_is_on) { + mali_group_reset(child); + } + } + + return child; +} + +void mali_group_reset(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT(NULL == group->gp_running_job); + MALI_DEBUG_ASSERT(NULL == group->pp_running_job); + + MALI_DEBUG_PRINT(3, ("Group: reset of %s\n", + mali_group_core_description(group))); + + if (NULL != group->dlbu_core) { + mali_dlbu_reset(group->dlbu_core); + } + + if (NULL != group->bcast_core) { + mali_bcast_reset(group->bcast_core); + } + + MALI_DEBUG_ASSERT(NULL != group->mmu); + mali_group_reset_mmu(group); + + if (NULL != group->gp_core) { + MALI_DEBUG_ASSERT(NULL == group->pp_core); + mali_gp_reset(group->gp_core); + } else { + MALI_DEBUG_ASSERT(NULL != group->pp_core); + mali_group_reset_pp(group); + } +} + +void mali_group_start_gp_job(struct mali_group *group, struct mali_gp_job *job, mali_bool gpu_secure_mode_pre_enabled) +{ + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_PRINT(3, ("Group: Starting GP job 0x%08X on group %s\n", + job, + mali_group_core_description(group))); + + session = mali_gp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(group->l2_cache_core[0]); + mali_l2_cache_invalidate_conditional(group->l2_cache_core[0], mali_gp_job_get_cache_order(job)); + + /* Reset GPU and disable gpu secure mode if needed. */ + if (MALI_TRUE == _mali_osk_gpu_secure_mode_is_enabled()) { + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + _mali_osk_gpu_reset_and_secure_mode_disable(); + /* Need to disable the pmu interrupt mask register */ + if (NULL != pmu) { + mali_pmu_reset(pmu); + } + } + + /* Reload mmu page table if needed */ + if (MALI_TRUE == gpu_secure_mode_pre_enabled) { + mali_group_reset(group); + mali_group_activate_page_directory(group, session, MALI_TRUE); + } else { + mali_group_activate_page_directory(group, session, MALI_FALSE); + } + + mali_gp_job_start(group->gp_core, job); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0) | + MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, + mali_gp_job_get_frame_builder_id(job), mali_gp_job_get_flush_id(job), 0, 0, 0); + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), + mali_gp_job_get_pid(job), mali_gp_job_get_tid(job), 0, 0, 0); + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_gp_job_get_pid(job), 1 /* active */, 1 /* GP */, 0 /* core */, + mali_gp_job_get_frame_builder_id(job), mali_gp_job_get_flush_id(job)); +#endif + +#if defined(CONFIG_MALI400_PROFILING) + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { + mali_group_report_l2_cache_counters_per_core(group, 0); + } +#endif /* #if defined(CONFIG_MALI400_PROFILING) */ + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + if (group->gp_core) { + trace_gpu_sched_switch(mali_gp_core_description(group->gp_core), + sched_clock(), mali_gp_job_get_tid(job), + 0, mali_gp_job_get_id(job)); + } +#endif + + group->gp_running_job = job; + group->is_working = MALI_TRUE; + + /* Setup SW timer and record start time */ + group->start_time = _mali_osk_time_tickcount(); + _mali_osk_timer_mod(group->timeout_timer, _mali_osk_time_mstoticks(mali_max_job_runtime)); + + MALI_DEBUG_PRINT(4, ("Group: Started GP job 0x%08X on group %s at %u\n", + job, + mali_group_core_description(group), + group->start_time)); +} + +/* Used to set all the registers except frame renderer list address and fragment shader stack address + * It means the caller must set these two registers properly before calling this function + */ +void mali_group_start_pp_job(struct mali_group *group, struct mali_pp_job *job, u32 sub_job, mali_bool gpu_secure_mode_pre_enabled) +{ + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_PRINT(3, ("Group: Starting PP job 0x%08X part %u/%u on group %s\n", + job, sub_job + 1, + mali_pp_job_get_sub_job_count(job), + mali_group_core_description(group))); + + session = mali_pp_job_get_session(job); + + if (NULL != group->l2_cache_core[0]) { + mali_l2_cache_invalidate_conditional(group->l2_cache_core[0], mali_pp_job_get_cache_order(job)); + } + + if (NULL != group->l2_cache_core[1]) { + mali_l2_cache_invalidate_conditional(group->l2_cache_core[1], mali_pp_job_get_cache_order(job)); + } + + /* Reset GPU and change gpu secure mode if needed. */ + if (MALI_TRUE == mali_pp_job_is_protected_job(job) && MALI_FALSE == _mali_osk_gpu_secure_mode_is_enabled()) { + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + _mali_osk_gpu_reset_and_secure_mode_enable(); + /* Need to disable the pmu interrupt mask register */ + if (NULL != pmu) { + mali_pmu_reset(pmu); + } + } else if (MALI_FALSE == mali_pp_job_is_protected_job(job) && MALI_TRUE == _mali_osk_gpu_secure_mode_is_enabled()) { + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + _mali_osk_gpu_reset_and_secure_mode_disable(); + /* Need to disable the pmu interrupt mask register */ + if (NULL != pmu) { + mali_pmu_reset(pmu); + } + } + + /* Reload the mmu page table if needed */ + if ((MALI_TRUE == mali_pp_job_is_protected_job(job) && MALI_FALSE == gpu_secure_mode_pre_enabled) + || (MALI_FALSE == mali_pp_job_is_protected_job(job) && MALI_TRUE == gpu_secure_mode_pre_enabled)) { + mali_group_reset(group); + mali_group_activate_page_directory(group, session, MALI_TRUE); + } else { + mali_group_activate_page_directory(group, session, MALI_FALSE); + } + + if (mali_group_is_virtual(group)) { + struct mali_group *child; + struct mali_group *temp; + u32 core_num = 0; + + MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job)); + + /* Configure DLBU for the job */ + mali_dlbu_config_job(group->dlbu_core, job); + + /* Write stack address for each child group */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + mali_pp_write_addr_stack(child->pp_core, job); + core_num++; + } + + mali_pp_job_start(group->pp_core, job, sub_job, MALI_FALSE); + } else { + mali_pp_job_start(group->pp_core, job, sub_job, MALI_FALSE); + } + + /* if the group is virtual, loop through physical groups which belong to this group + * and call profiling events for its cores as virtual */ + if (MALI_TRUE == mali_group_is_virtual(group)) { + struct mali_group *child; + struct mali_group *temp; + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | + MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | + MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL, + mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0); + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core), + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job)); +#endif + } + +#if defined(CONFIG_MALI400_PROFILING) + if (0 != group->l2_cache_core_ref_count[0]) { + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); + } + } + if (0 != group->l2_cache_core_ref_count[1]) { + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[1])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[1]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[1])); + } + } +#endif /* #if defined(CONFIG_MALI400_PROFILING) */ + + } else { /* group is physical - call profiling events for physical cores */ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) | + MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH, + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) | + MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL, + mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0); + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(group->pp_core), + mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job)); +#endif + +#if defined(CONFIG_MALI400_PROFILING) + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); + } +#endif /* #if defined(CONFIG_MALI400_PROFILING) */ + } + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + if (group->pp_core) { + trace_gpu_sched_switch(mali_pp_core_description(group->pp_core), + sched_clock(), mali_pp_job_get_tid(job), + 0, mali_pp_job_get_id(job)); + } +#endif + + group->pp_running_job = job; + group->pp_running_sub_job = sub_job; + group->is_working = MALI_TRUE; + + /* Setup SW timer and record start time */ + group->start_time = _mali_osk_time_tickcount(); + _mali_osk_timer_mod(group->timeout_timer, _mali_osk_time_mstoticks(mali_max_job_runtime)); + + MALI_DEBUG_PRINT(4, ("Group: Started PP job 0x%08X part %u/%u on group %s at %u\n", + job, sub_job + 1, + mali_pp_job_get_sub_job_count(job), + mali_group_core_description(group), + group->start_time)); + +} + +void mali_group_resume_gp_with_new_heap(struct mali_group *group, u32 job_id, u32 start_addr, u32 end_addr) +{ + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + MALI_DEBUG_ASSERT_POINTER(group->l2_cache_core[0]); + mali_l2_cache_invalidate(group->l2_cache_core[0]); + + mali_mmu_zap_tlb_without_stall(group->mmu); + + mali_gp_resume_with_new_heap(group->gp_core, start_addr, end_addr); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_RESUME | + MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), + 0, 0, 0, 0, 0); + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_gp_job_get_pid(group->gp_running_job), 1 /* active */, 1 /* GP */, 0 /* core */, + mali_gp_job_get_frame_builder_id(group->gp_running_job), mali_gp_job_get_flush_id(group->gp_running_job)); +#endif +} + +static void mali_group_reset_mmu(struct mali_group *group) +{ + struct mali_group *child; + struct mali_group *temp; + _mali_osk_errcode_t err; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (!mali_group_is_virtual(group)) { + /* This is a physical group or an idle virtual group -- simply wait for + * the reset to complete. */ + err = mali_mmu_reset(group->mmu); + MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err); + } else { /* virtual group */ + /* Loop through all members of this virtual group and wait + * until they are done resetting. + */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + err = mali_mmu_reset(child->mmu); + MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err); + } + } +} + +static void mali_group_reset_pp(struct mali_group *group) +{ + struct mali_group *child; + struct mali_group *temp; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + mali_pp_reset_async(group->pp_core); + + if (!mali_group_is_virtual(group) || NULL == group->pp_running_job) { + /* This is a physical group or an idle virtual group -- simply wait for + * the reset to complete. */ + mali_pp_reset_wait(group->pp_core); + } else { + /* Loop through all members of this virtual group and wait until they + * are done resetting. + */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + mali_pp_reset_wait(child->pp_core); + } + } +} + +struct mali_pp_job *mali_group_complete_pp(struct mali_group *group, mali_bool success, u32 *sub_job) +{ + struct mali_pp_job *pp_job_to_return; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_POINTER(group->pp_running_job); + MALI_DEBUG_ASSERT_POINTER(sub_job); + MALI_DEBUG_ASSERT(MALI_TRUE == group->is_working); + + /* Stop/clear the timeout timer. */ + _mali_osk_timer_del_async(group->timeout_timer); + + if (NULL != group->pp_running_job) { + + /* Deal with HW counters and profiling */ + + if (MALI_TRUE == mali_group_is_virtual(group)) { + struct mali_group *child; + struct mali_group *temp; + + /* update performance counters from each physical pp core within this virtual group */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + mali_pp_update_performance_counters(group->pp_core, child->pp_core, group->pp_running_job, mali_pp_core_get_id(child->pp_core)); + } + +#if defined(CONFIG_MALI400_PROFILING) + /* send profiling data per physical core */ + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) | + MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL, + mali_pp_job_get_perf_counter_value0(group->pp_running_job, mali_pp_core_get_id(child->pp_core)), + mali_pp_job_get_perf_counter_value1(group->pp_running_job, mali_pp_core_get_id(child->pp_core)), + mali_pp_job_get_perf_counter_src0(group->pp_running_job, group->pp_running_sub_job) | (mali_pp_job_get_perf_counter_src1(group->pp_running_job, group->pp_running_sub_job) << 8), + 0, 0); + + trace_mali_core_active(mali_pp_job_get_pid(group->pp_running_job), + 0 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core), + mali_pp_job_get_frame_builder_id(group->pp_running_job), + mali_pp_job_get_flush_id(group->pp_running_job)); + } + if (0 != group->l2_cache_core_ref_count[0]) { + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); + } + } + if (0 != group->l2_cache_core_ref_count[1]) { + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[1])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[1]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[1])); + } + } + +#endif + } else { + /* update performance counters for a physical group's pp core */ + mali_pp_update_performance_counters(group->pp_core, group->pp_core, group->pp_running_job, group->pp_running_sub_job); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) | + MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL, + mali_pp_job_get_perf_counter_value0(group->pp_running_job, group->pp_running_sub_job), + mali_pp_job_get_perf_counter_value1(group->pp_running_job, group->pp_running_sub_job), + mali_pp_job_get_perf_counter_src0(group->pp_running_job, group->pp_running_sub_job) | (mali_pp_job_get_perf_counter_src1(group->pp_running_job, group->pp_running_sub_job) << 8), + 0, 0); + + trace_mali_core_active(mali_pp_job_get_pid(group->pp_running_job), + 0 /* active */, 0 /* PP */, mali_pp_core_get_id(group->pp_core), + mali_pp_job_get_frame_builder_id(group->pp_running_job), + mali_pp_job_get_flush_id(group->pp_running_job)); + + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) { + mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0])); + } +#endif + } + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + if (group->gp_core) { + trace_gpu_sched_switch( + mali_gp_core_description(group->gp_core), + sched_clock(), 0, 0, 0); + } +#endif + + } + + if (success) { + /* Only do soft reset for successful jobs, a full recovery + * reset will be done for failed jobs. */ + mali_pp_reset_async(group->pp_core); + } + + pp_job_to_return = group->pp_running_job; + group->pp_running_job = NULL; + group->is_working = MALI_FALSE; + *sub_job = group->pp_running_sub_job; + + if (!success) { + MALI_DEBUG_PRINT(2, ("Mali group: Executing recovery reset due to job failure\n")); + mali_group_recovery_reset(group); + } else if (_MALI_OSK_ERR_OK != mali_pp_reset_wait(group->pp_core)) { + MALI_PRINT_ERROR(("Mali group: Executing recovery reset due to reset failure\n")); + mali_group_recovery_reset(group); + } + + return pp_job_to_return; +} + +struct mali_gp_job *mali_group_complete_gp(struct mali_group *group, mali_bool success) +{ + struct mali_gp_job *gp_job_to_return; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_POINTER(group->gp_running_job); + MALI_DEBUG_ASSERT(MALI_TRUE == group->is_working); + + /* Stop/clear the timeout timer. */ + _mali_osk_timer_del_async(group->timeout_timer); + + if (NULL != group->gp_running_job) { + mali_gp_update_performance_counters(group->gp_core, group->gp_running_job); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), + mali_gp_job_get_perf_counter_value0(group->gp_running_job), + mali_gp_job_get_perf_counter_value1(group->gp_running_job), + mali_gp_job_get_perf_counter_src0(group->gp_running_job) | (mali_gp_job_get_perf_counter_src1(group->gp_running_job) << 8), + 0, 0); + + if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) && + (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) + mali_group_report_l2_cache_counters_per_core(group, 0); +#endif + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + if (group->pp_core) { + trace_gpu_sched_switch( + mali_pp_core_description(group->pp_core), + sched_clock(), 0, 0, 0); + } +#endif + +#if defined(CONFIG_MALI400_PROFILING) + trace_mali_core_active(mali_gp_job_get_pid(group->gp_running_job), 0 /* active */, 1 /* GP */, 0 /* core */, + mali_gp_job_get_frame_builder_id(group->gp_running_job), mali_gp_job_get_flush_id(group->gp_running_job)); +#endif + + mali_gp_job_set_current_heap_addr(group->gp_running_job, + mali_gp_read_plbu_alloc_start_addr(group->gp_core)); + } + + if (success) { + /* Only do soft reset for successful jobs, a full recovery + * reset will be done for failed jobs. */ + mali_gp_reset_async(group->gp_core); + } + + gp_job_to_return = group->gp_running_job; + group->gp_running_job = NULL; + group->is_working = MALI_FALSE; + + if (!success) { + MALI_DEBUG_PRINT(2, ("Mali group: Executing recovery reset due to job failure\n")); + mali_group_recovery_reset(group); + } else if (_MALI_OSK_ERR_OK != mali_gp_reset_wait(group->gp_core)) { + MALI_PRINT_ERROR(("Mali group: Executing recovery reset due to reset failure\n")); + mali_group_recovery_reset(group); + } + + return gp_job_to_return; +} + +struct mali_group *mali_group_get_glob_group(u32 index) +{ + if (mali_global_num_groups > index) { + return mali_global_groups[index]; + } + + return NULL; +} + +u32 mali_group_get_glob_num_groups(void) +{ + return mali_global_num_groups; +} + +static void mali_group_activate_page_directory(struct mali_group *group, struct mali_session_data *session, mali_bool is_reload) +{ + MALI_DEBUG_PRINT(5, ("Mali group: Activating page directory 0x%08X from session 0x%08X on group %s\n", + mali_session_get_page_directory(session), session, + mali_group_core_description(group))); + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (group->session != session || MALI_TRUE == is_reload) { + /* Different session than last time, so we need to do some work */ + MALI_DEBUG_PRINT(5, ("Mali group: Activate session: %08x previous: %08x on group %s\n", + session, group->session, + mali_group_core_description(group))); + mali_mmu_activate_page_directory(group->mmu, mali_session_get_page_directory(session)); + group->session = session; + } else { + /* Same session as last time, so no work required */ + MALI_DEBUG_PRINT(4, ("Mali group: Activate existing session 0x%08X on group %s\n", + session->page_directory, + mali_group_core_description(group))); + mali_mmu_zap_tlb_without_stall(group->mmu); + } +} + +static void mali_group_recovery_reset(struct mali_group *group) +{ + _mali_osk_errcode_t err; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + /* Stop cores, bus stop */ + if (NULL != group->pp_core) { + mali_pp_stop_bus(group->pp_core); + } else { + mali_gp_stop_bus(group->gp_core); + } + + /* Flush MMU and clear page fault (if any) */ + mali_mmu_activate_fault_flush_page_directory(group->mmu); + mali_mmu_page_fault_done(group->mmu); + + /* Wait for cores to stop bus, then do a hard reset on them */ + if (NULL != group->pp_core) { + if (mali_group_is_virtual(group)) { + struct mali_group *child, *temp; + + /* Disable the broadcast unit while we do reset directly on the member cores. */ + mali_bcast_disable(group->bcast_core); + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) { + mali_pp_stop_bus_wait(child->pp_core); + mali_pp_hard_reset(child->pp_core); + } + + mali_bcast_enable(group->bcast_core); + } else { + mali_pp_stop_bus_wait(group->pp_core); + mali_pp_hard_reset(group->pp_core); + } + } else { + mali_gp_stop_bus_wait(group->gp_core); + mali_gp_hard_reset(group->gp_core); + } + + /* Reset MMU */ + err = mali_mmu_reset(group->mmu); + MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err); + MALI_IGNORE(err); + + group->session = NULL; +} + +#if MALI_STATE_TRACKING +u32 mali_group_dump_state(struct mali_group *group, char *buf, u32 size) +{ + int n = 0; + int i; + struct mali_group *child; + struct mali_group *temp; + + if (mali_group_is_virtual(group)) { + n += _mali_osk_snprintf(buf + n, size - n, + "Virtual PP Group: %p\n", group); + } else if (mali_group_is_in_virtual(group)) { + n += _mali_osk_snprintf(buf + n, size - n, + "Child PP Group: %p\n", group); + } else if (NULL != group->pp_core) { + n += _mali_osk_snprintf(buf + n, size - n, + "Physical PP Group: %p\n", group); + } else { + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + n += _mali_osk_snprintf(buf + n, size - n, + "GP Group: %p\n", group); + } + + switch (group->state) { + case MALI_GROUP_STATE_INACTIVE: + n += _mali_osk_snprintf(buf + n, size - n, + "\tstate: INACTIVE\n"); + break; + case MALI_GROUP_STATE_ACTIVATION_PENDING: + n += _mali_osk_snprintf(buf + n, size - n, + "\tstate: ACTIVATION_PENDING\n"); + break; + case MALI_GROUP_STATE_ACTIVE: + n += _mali_osk_snprintf(buf + n, size - n, + "\tstate: MALI_GROUP_STATE_ACTIVE\n"); + break; + default: + n += _mali_osk_snprintf(buf + n, size - n, + "\tstate: UNKNOWN (%d)\n", group->state); + MALI_DEBUG_ASSERT(0); + break; + } + + n += _mali_osk_snprintf(buf + n, size - n, + "\tSW power: %s\n", + group->power_is_on ? "On" : "Off"); + + n += mali_pm_dump_state_domain(group->pm_domain, buf + n, size - n); + + for (i = 0; i < 2; i++) { + if (NULL != group->l2_cache_core[i]) { + struct mali_pm_domain *domain; + domain = mali_l2_cache_get_pm_domain( + group->l2_cache_core[i]); + n += mali_pm_dump_state_domain(domain, + buf + n, size - n); + } + } + + if (group->gp_core) { + n += mali_gp_dump_state(group->gp_core, buf + n, size - n); + n += _mali_osk_snprintf(buf + n, size - n, + "\tGP running job: %p\n", group->gp_running_job); + } + + if (group->pp_core) { + n += mali_pp_dump_state(group->pp_core, buf + n, size - n); + n += _mali_osk_snprintf(buf + n, size - n, + "\tPP running job: %p, subjob %d \n", + group->pp_running_job, + group->pp_running_sub_job); + } + + _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, + struct mali_group, group_list) { + n += mali_group_dump_state(child, buf + n, size - n); + } + + return n; +} +#endif + +_mali_osk_errcode_t mali_group_upper_half_mmu(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + _mali_osk_errcode_t ret; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + if (NULL != group->gp_core) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), + mali_mmu_get_rawstat(group->mmu), 0); + } else { + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( + mali_pp_core_get_id(group->pp_core)), + mali_mmu_get_rawstat(group->mmu), 0); + } +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + + ret = mali_executor_interrupt_mmu(group, MALI_TRUE); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group) && (!mali_group_power_is_on(group))) { + /* group complete and on job shedule on it, it already power off */ + if (NULL != group->gp_core) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), + 0xFFFFFFFF, 0); + } else { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( + mali_pp_core_get_id(group->pp_core)), + 0xFFFFFFFF, 0); + } + + mali_executor_unlock(); + return ret; + } +#endif + + if (NULL != group->gp_core) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), + mali_mmu_get_rawstat(group->mmu), 0); + } else { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( + mali_pp_core_get_id(group->pp_core)), + mali_mmu_get_rawstat(group->mmu), 0); + } +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + + return ret; +} + +static void mali_group_bottom_half_mmu(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + + if (NULL != group->gp_core) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), + mali_mmu_get_rawstat(group->mmu), 0); + } else { + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( + mali_pp_core_get_id(group->pp_core)), + mali_mmu_get_rawstat(group->mmu), 0); + } + + mali_executor_interrupt_mmu(group, MALI_FALSE); + + if (NULL != group->gp_core) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0), + mali_mmu_get_rawstat(group->mmu), 0); + } else { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU( + mali_pp_core_get_id(group->pp_core)), + mali_mmu_get_rawstat(group->mmu), 0); + } +} + +_mali_osk_errcode_t mali_group_upper_half_gp(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + _mali_osk_errcode_t ret; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), + mali_gp_get_rawstat(group->gp_core), 0); + + MALI_DEBUG_PRINT(4, ("Group: Interrupt 0x%08X from %s\n", + mali_gp_get_rawstat(group->gp_core), + mali_group_core_description(group))); +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + ret = mali_executor_interrupt_gp(group, MALI_TRUE); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group) && (!mali_group_power_is_on(group))) { + /* group complete and on job shedule on it, it already power off */ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), + 0xFFFFFFFF, 0); + mali_executor_unlock(); + return ret; + } +#endif + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), + mali_gp_get_rawstat(group->gp_core), 0); +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + return ret; +} + +static void mali_group_bottom_half_gp(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), + mali_gp_get_rawstat(group->gp_core), 0); + + mali_executor_interrupt_gp(group, MALI_FALSE); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0), + mali_gp_get_rawstat(group->gp_core), 0); +} + +_mali_osk_errcode_t mali_group_upper_half_pp(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + _mali_osk_errcode_t ret; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group)) { + /* Not working, so nothing to do */ + mali_executor_unlock(); + return _MALI_OSK_ERR_FAULT; + } +#endif + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( + mali_pp_core_get_id(group->pp_core)), + mali_pp_get_rawstat(group->pp_core), 0); + + MALI_DEBUG_PRINT(4, ("Group: Interrupt 0x%08X from %s\n", + mali_pp_get_rawstat(group->pp_core), + mali_group_core_description(group))); +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + + ret = mali_executor_interrupt_pp(group, MALI_TRUE); + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_lock(); + if (!mali_group_is_working(group) && (!mali_group_power_is_on(group))) { + /* group complete and on job shedule on it, it already power off */ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( + mali_pp_core_get_id(group->pp_core)), + 0xFFFFFFFF, 0); + mali_executor_unlock(); + return ret; + } +#endif + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF, + 0, 0, /* No pid and tid for interrupt handler */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( + mali_pp_core_get_id(group->pp_core)), + mali_pp_get_rawstat(group->pp_core), 0); +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + mali_executor_unlock(); +#endif +#endif + return ret; +} + +static void mali_group_bottom_half_pp(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( + mali_pp_core_get_id(group->pp_core)), + mali_pp_get_rawstat(group->pp_core), 0); + + mali_executor_interrupt_pp(group, MALI_FALSE); + + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF, + 0, _mali_osk_get_tid(), /* pid and tid */ + MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP( + mali_pp_core_get_id(group->pp_core)), + mali_pp_get_rawstat(group->pp_core), 0); +} + +static void mali_group_timeout(void *data) +{ + struct mali_group *group = (struct mali_group *)data; + MALI_DEBUG_ASSERT_POINTER(group); + + MALI_DEBUG_PRINT(2, ("Group: timeout handler for %s at %u\n", + mali_group_core_description(group), + _mali_osk_time_tickcount())); + + if (NULL != group->gp_core) { + mali_group_schedule_bottom_half_gp(group); + } else { + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + mali_group_schedule_bottom_half_pp(group); + } +} + +mali_bool mali_group_zap_session(struct mali_group *group, + struct mali_session_data *session) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (group->session != session) { + /* not running from this session */ + return MALI_TRUE; /* success */ + } + + if (group->is_working) { + /* The Zap also does the stall and disable_stall */ + mali_bool zap_success = mali_mmu_zap_tlb(group->mmu); + return zap_success; + } else { + /* Just remove the session instead of zapping */ + mali_group_clear_session(group); + return MALI_TRUE; /* success */ + } +} + +#if defined(CONFIG_MALI400_PROFILING) +static void mali_group_report_l2_cache_counters_per_core(struct mali_group *group, u32 core_num) +{ + u32 source0 = 0; + u32 value0 = 0; + u32 source1 = 0; + u32 value1 = 0; + u32 profiling_channel = 0; + + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + switch (core_num) { + case 0: + profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS; + break; + case 1: + profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L21_COUNTERS; + break; + case 2: + profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L22_COUNTERS; + break; + default: + profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS; + break; + } + + if (0 == core_num) { + mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1); + } + if (1 == core_num) { + if (1 == mali_l2_cache_get_id(group->l2_cache_core[0])) { + mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1); + } else if (1 == mali_l2_cache_get_id(group->l2_cache_core[1])) { + mali_l2_cache_core_get_counter_values(group->l2_cache_core[1], &source0, &value0, &source1, &value1); + } + } + if (2 == core_num) { + if (2 == mali_l2_cache_get_id(group->l2_cache_core[0])) { + mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1); + } else if (2 == mali_l2_cache_get_id(group->l2_cache_core[1])) { + mali_l2_cache_core_get_counter_values(group->l2_cache_core[1], &source0, &value0, &source1, &value1); + } + } + + _mali_osk_profiling_add_event(profiling_channel, source1 << 8 | source0, value0, value1, 0, 0); +} +#endif /* #if defined(CONFIG_MALI400_PROFILING) */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_group.h b/drivers/gpu/arm/mali400/mali/common/mali_group.h new file mode 100755 index 000000000000..32481e4a6748 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_group.h @@ -0,0 +1,460 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_GROUP_H__ +#define __MALI_GROUP_H__ + +#include "mali_osk.h" +#include "mali_l2_cache.h" +#include "mali_mmu.h" +#include "mali_gp.h" +#include "mali_pp.h" +#include "mali_session.h" +#include "mali_osk_profiling.h" + +/** + * @brief Default max runtime [ms] for a core job - used by timeout timers + */ +#define MALI_MAX_JOB_RUNTIME_DEFAULT 5000 + +extern int mali_max_job_runtime; + +#define MALI_MAX_NUMBER_OF_GROUPS 10 +#define MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS 8 + +enum mali_group_state { + MALI_GROUP_STATE_INACTIVE, + MALI_GROUP_STATE_ACTIVATION_PENDING, + MALI_GROUP_STATE_ACTIVE, +}; + +/** + * The structure represents a render group + * A render group is defined by all the cores that share the same Mali MMU + */ + +struct mali_group { + struct mali_mmu_core *mmu; + struct mali_session_data *session; + + enum mali_group_state state; + mali_bool power_is_on; + + mali_bool is_working; + unsigned long start_time; /* in ticks */ + + struct mali_gp_core *gp_core; + struct mali_gp_job *gp_running_job; + + struct mali_pp_core *pp_core; + struct mali_pp_job *pp_running_job; + u32 pp_running_sub_job; + + struct mali_pm_domain *pm_domain; + + struct mali_l2_cache_core *l2_cache_core[2]; + u32 l2_cache_core_ref_count[2]; + + /* Parent virtual group (if any) */ + struct mali_group *parent_group; + + struct mali_dlbu_core *dlbu_core; + struct mali_bcast_unit *bcast_core; + + /* Used for working groups which needs to be disabled */ + mali_bool disable_requested; + + /* Used by group to link child groups (for virtual group) */ + _mali_osk_list_t group_list; + + /* Used by executor module in order to link groups of same state */ + _mali_osk_list_t executor_list; + + /* Used by PM domains to link groups of same domain */ + _mali_osk_list_t pm_domain_list; + + _mali_osk_wq_work_t *bottom_half_work_mmu; + _mali_osk_wq_work_t *bottom_half_work_gp; + _mali_osk_wq_work_t *bottom_half_work_pp; + + _mali_osk_timer_t *timeout_timer; +}; + +/** @brief Create a new Mali group object + * + * @return A pointer to a new group object + */ +struct mali_group *mali_group_create(struct mali_l2_cache_core *core, + struct mali_dlbu_core *dlbu, + struct mali_bcast_unit *bcast, + u32 domain_index); + +void mali_group_dump_status(struct mali_group *group); + +void mali_group_delete(struct mali_group *group); + +_mali_osk_errcode_t mali_group_add_mmu_core(struct mali_group *group, + struct mali_mmu_core *mmu_core); +void mali_group_remove_mmu_core(struct mali_group *group); + +_mali_osk_errcode_t mali_group_add_gp_core(struct mali_group *group, + struct mali_gp_core *gp_core); +void mali_group_remove_gp_core(struct mali_group *group); + +_mali_osk_errcode_t mali_group_add_pp_core(struct mali_group *group, + struct mali_pp_core *pp_core); +void mali_group_remove_pp_core(struct mali_group *group); + +MALI_STATIC_INLINE const char *mali_group_core_description( + struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + if (NULL != group->pp_core) { + return mali_pp_core_description(group->pp_core); + } else { + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + return mali_gp_core_description(group->gp_core); + } +} + +MALI_STATIC_INLINE mali_bool mali_group_is_virtual(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + return (NULL != group->dlbu_core); +#else + return MALI_FALSE; +#endif +} + +/** @brief Check if a group is a part of a virtual group or not + */ +MALI_STATIC_INLINE mali_bool mali_group_is_in_virtual(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + return (NULL != group->parent_group) ? MALI_TRUE : MALI_FALSE; +#else + return MALI_FALSE; +#endif +} + +/** @brief Reset group + * + * This function will reset the entire group, + * including all the cores present in the group. + * + * @param group Pointer to the group to reset + */ +void mali_group_reset(struct mali_group *group); + +MALI_STATIC_INLINE struct mali_session_data *mali_group_get_session( + struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + return group->session; +} + +MALI_STATIC_INLINE void mali_group_clear_session(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + if (NULL != group->session) { + mali_mmu_activate_empty_page_directory(group->mmu); + group->session = NULL; + } +} + +enum mali_group_state mali_group_activate(struct mali_group *group); + +/* + * Change state from ACTIVATION_PENDING to ACTIVE + * For virtual group, all childs need to be ACTIVE first + */ +mali_bool mali_group_set_active(struct mali_group *group); + +/* + * @return MALI_TRUE means one or more domains can now be powered off, + * and caller should call either mali_pm_update_async() or + * mali_pm_update_sync() in order to do so. + */ +mali_bool mali_group_deactivate(struct mali_group *group); + +MALI_STATIC_INLINE enum mali_group_state mali_group_get_state(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return group->state; +} + +MALI_STATIC_INLINE mali_bool mali_group_power_is_on(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + return group->power_is_on; +} + +void mali_group_power_up(struct mali_group *group); +void mali_group_power_down(struct mali_group *group); + +MALI_STATIC_INLINE void mali_group_set_disable_request( + struct mali_group *group, mali_bool disable) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + group->disable_requested = disable; + + /** + * When one of child group's disable_requeset is set TRUE, then + * the disable_request of parent group should also be set to TRUE. + * While, the disable_request of parent group should only be set to FALSE + * only when all of its child group's disable_request are set to FALSE. + */ + if (NULL != group->parent_group && MALI_TRUE == disable) { + group->parent_group->disable_requested = disable; + } +} + +MALI_STATIC_INLINE mali_bool mali_group_disable_requested( + struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return group->disable_requested; +} + +/** @brief Virtual groups */ +void mali_group_add_group(struct mali_group *parent, struct mali_group *child); +struct mali_group *mali_group_acquire_group(struct mali_group *parent); +void mali_group_remove_group(struct mali_group *parent, struct mali_group *child); + +/** @brief Checks if the group is working. + */ +MALI_STATIC_INLINE mali_bool mali_group_is_working(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + if (mali_group_is_in_virtual(group)) { + struct mali_group *tmp_group = mali_executor_get_virtual_group(); + return tmp_group->is_working; + } + return group->is_working; +} + +MALI_STATIC_INLINE struct mali_gp_job *mali_group_get_running_gp_job(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return group->gp_running_job; +} + +/** @brief Zap MMU TLB on all groups + * + * Zap TLB on group if \a session is active. + */ +mali_bool mali_group_zap_session(struct mali_group *group, + struct mali_session_data *session); + +/** @brief Get pointer to GP core object + */ +MALI_STATIC_INLINE struct mali_gp_core *mali_group_get_gp_core(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + return group->gp_core; +} + +/** @brief Get pointer to PP core object + */ +MALI_STATIC_INLINE struct mali_pp_core *mali_group_get_pp_core(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + return group->pp_core; +} + +/** @brief Start GP job + */ +void mali_group_start_gp_job(struct mali_group *group, struct mali_gp_job *job, mali_bool gpu_secure_mode_pre_enabled); + +void mali_group_start_pp_job(struct mali_group *group, struct mali_pp_job *job, u32 sub_job, mali_bool gpu_secure_mode_pre_enabled); + +/** @brief Start virtual group Job on a virtual group +*/ +void mali_group_start_job_on_virtual(struct mali_group *group, struct mali_pp_job *job, u32 first_subjob, u32 last_subjob); + + +/** @brief Start a subjob from a particular on a specific PP group +*/ +void mali_group_start_job_on_group(struct mali_group *group, struct mali_pp_job *job, u32 subjob); + + +/** @brief remove all the unused groups in tmp_unused group list, so that the group is in consistent status. + */ +void mali_group_non_dlbu_job_done_virtual(struct mali_group *group); + + +/** @brief Resume GP job that suspended waiting for more heap memory + */ +void mali_group_resume_gp_with_new_heap(struct mali_group *group, u32 job_id, u32 start_addr, u32 end_addr); + +MALI_STATIC_INLINE enum mali_interrupt_result mali_group_get_interrupt_result_gp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_gp_get_interrupt_result(group->gp_core); +} + +MALI_STATIC_INLINE enum mali_interrupt_result mali_group_get_interrupt_result_pp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_pp_get_interrupt_result(group->pp_core); +} + +MALI_STATIC_INLINE enum mali_interrupt_result mali_group_get_interrupt_result_mmu(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_mmu_get_interrupt_result(group->mmu); +} + +MALI_STATIC_INLINE mali_bool mali_group_gp_is_active(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_gp_is_active(group->gp_core); +} + +MALI_STATIC_INLINE mali_bool mali_group_pp_is_active(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_pp_is_active(group->pp_core); +} + +MALI_STATIC_INLINE mali_bool mali_group_has_timed_out(struct mali_group *group) +{ + unsigned long time_cost; + struct mali_group *tmp_group = group; + + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + + /* if the group is in virtual need to use virtual_group's start time */ + if (mali_group_is_in_virtual(group)) { + tmp_group = mali_executor_get_virtual_group(); + } + + time_cost = _mali_osk_time_tickcount() - tmp_group->start_time; + if (_mali_osk_time_mstoticks(mali_max_job_runtime) <= time_cost) { + /* + * current tick is at or after timeout end time, + * so this is a valid timeout + */ + return MALI_TRUE; + } else { + /* + * Not a valid timeout. A HW interrupt probably beat + * us to it, and the timer wasn't properly deleted + * (async deletion used due to atomic context). + */ + return MALI_FALSE; + } +} + +MALI_STATIC_INLINE void mali_group_mask_all_interrupts_gp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_gp_mask_all_interrupts(group->gp_core); +} + +MALI_STATIC_INLINE void mali_group_mask_all_interrupts_pp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return mali_pp_mask_all_interrupts(group->pp_core); +} + +MALI_STATIC_INLINE void mali_group_enable_interrupts_gp( + struct mali_group *group, + enum mali_interrupt_result exceptions) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + mali_gp_enable_interrupts(group->gp_core, exceptions); +} + +MALI_STATIC_INLINE void mali_group_schedule_bottom_half_gp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->gp_core); + _mali_osk_wq_schedule_work(group->bottom_half_work_gp); +} + + +MALI_STATIC_INLINE void mali_group_schedule_bottom_half_pp(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->pp_core); + _mali_osk_wq_schedule_work(group->bottom_half_work_pp); +} + +MALI_STATIC_INLINE void mali_group_schedule_bottom_half_mmu(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT_POINTER(group->mmu); + _mali_osk_wq_schedule_work(group->bottom_half_work_mmu); +} + +struct mali_pp_job *mali_group_complete_pp(struct mali_group *group, mali_bool success, u32 *sub_job); + +struct mali_gp_job *mali_group_complete_gp(struct mali_group *group, mali_bool success); + +#if defined(CONFIG_MALI400_PROFILING) +MALI_STATIC_INLINE void mali_group_oom(struct mali_group *group) +{ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SUSPEND | + MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0), + 0, 0, 0, 0, 0); +} +#endif + +struct mali_group *mali_group_get_glob_group(u32 index); +u32 mali_group_get_glob_num_groups(void); + +u32 mali_group_dump_state(struct mali_group *group, char *buf, u32 size); + + +_mali_osk_errcode_t mali_group_upper_half_mmu(void *data); +_mali_osk_errcode_t mali_group_upper_half_gp(void *data); +_mali_osk_errcode_t mali_group_upper_half_pp(void *data); + +MALI_STATIC_INLINE mali_bool mali_group_is_empty(struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(group); + MALI_DEBUG_ASSERT(mali_group_is_virtual(group)); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + return _mali_osk_list_empty(&group->group_list); +} + +#endif /* __MALI_GROUP_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_hw_core.c b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.c new file mode 100755 index 000000000000..a813816e998d --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.c @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_hw_core.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_osk_mali.h" + +_mali_osk_errcode_t mali_hw_core_create(struct mali_hw_core *core, const _mali_osk_resource_t *resource, u32 reg_size) +{ + core->phys_addr = resource->base; + core->phys_offset = resource->base - _mali_osk_resource_base_address(); + core->description = resource->description; + core->size = reg_size; + + MALI_DEBUG_ASSERT(core->phys_offset < core->phys_addr); + + if (_MALI_OSK_ERR_OK == _mali_osk_mem_reqregion(core->phys_addr, core->size, core->description)) { + core->mapped_registers = _mali_osk_mem_mapioregion(core->phys_addr, core->size, core->description); + if (NULL != core->mapped_registers) { + return _MALI_OSK_ERR_OK; + } else { + MALI_PRINT_ERROR(("Failed to map memory region for core %s at phys_addr 0x%08X\n", core->description, core->phys_addr)); + } + _mali_osk_mem_unreqregion(core->phys_addr, core->size); + } else { + MALI_PRINT_ERROR(("Failed to request memory region for core %s at phys_addr 0x%08X\n", core->description, core->phys_addr)); + } + + return _MALI_OSK_ERR_FAULT; +} + +void mali_hw_core_delete(struct mali_hw_core *core) +{ + if (NULL != core->mapped_registers) { + _mali_osk_mem_unmapioregion(core->phys_addr, core->size, core->mapped_registers); + core->mapped_registers = NULL; + } + _mali_osk_mem_unreqregion(core->phys_addr, core->size); +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_hw_core.h b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.h new file mode 100755 index 000000000000..38d96e240a20 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_hw_core.h @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_HW_CORE_H__ +#define __MALI_HW_CORE_H__ + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +/** + * The common parts for all Mali HW cores (GP, PP, MMU, L2 and PMU) + * This struct is embedded inside all core specific structs. + */ +struct mali_hw_core { + uintptr_t phys_addr; /**< Physical address of the registers */ + u32 phys_offset; /**< Offset from start of Mali to registers */ + u32 size; /**< Size of registers */ + mali_io_address mapped_registers; /**< Virtual mapping of the registers */ + const char *description; /**< Name of unit (as specified in device configuration) */ +}; + +#define MALI_REG_POLL_COUNT_FAST 1000000 +#define MALI_REG_POLL_COUNT_SLOW 1000000 + +/* + * GP and PP core translate their int_stat/rawstat into one of these + */ +enum mali_interrupt_result { + MALI_INTERRUPT_RESULT_NONE, + MALI_INTERRUPT_RESULT_SUCCESS, + MALI_INTERRUPT_RESULT_SUCCESS_VS, + MALI_INTERRUPT_RESULT_SUCCESS_PLBU, + MALI_INTERRUPT_RESULT_OOM, + MALI_INTERRUPT_RESULT_ERROR +}; + +_mali_osk_errcode_t mali_hw_core_create(struct mali_hw_core *core, const _mali_osk_resource_t *resource, u32 reg_size); +void mali_hw_core_delete(struct mali_hw_core *core); + +MALI_STATIC_INLINE u32 mali_hw_core_register_read(struct mali_hw_core *core, u32 relative_address) +{ + u32 read_val; + read_val = _mali_osk_mem_ioread32(core->mapped_registers, relative_address); + MALI_DEBUG_PRINT(6, ("register_read for core %s, relative addr=0x%04X, val=0x%08X\n", + core->description, relative_address, read_val)); + return read_val; +} + +MALI_STATIC_INLINE void mali_hw_core_register_write_relaxed(struct mali_hw_core *core, u32 relative_address, u32 new_val) +{ + MALI_DEBUG_PRINT(6, ("register_write_relaxed for core %s, relative addr=0x%04X, val=0x%08X\n", + core->description, relative_address, new_val)); + _mali_osk_mem_iowrite32_relaxed(core->mapped_registers, relative_address, new_val); +} + +/* Conditionally write a register. + * The register will only be written if the new value is different from the old_value. + * If the new value is different, the old value will also be updated */ +MALI_STATIC_INLINE void mali_hw_core_register_write_relaxed_conditional(struct mali_hw_core *core, u32 relative_address, u32 new_val, const u32 old_val) +{ + MALI_DEBUG_PRINT(6, ("register_write_relaxed for core %s, relative addr=0x%04X, val=0x%08X\n", + core->description, relative_address, new_val)); + if (old_val != new_val) { + _mali_osk_mem_iowrite32_relaxed(core->mapped_registers, relative_address, new_val); + } +} + +MALI_STATIC_INLINE void mali_hw_core_register_write(struct mali_hw_core *core, u32 relative_address, u32 new_val) +{ + MALI_DEBUG_PRINT(6, ("register_write for core %s, relative addr=0x%04X, val=0x%08X\n", + core->description, relative_address, new_val)); + _mali_osk_mem_iowrite32(core->mapped_registers, relative_address, new_val); +} + +MALI_STATIC_INLINE void mali_hw_core_register_write_array_relaxed(struct mali_hw_core *core, u32 relative_address, u32 *write_array, u32 nr_of_regs) +{ + u32 i; + MALI_DEBUG_PRINT(6, ("register_write_array: for core %s, relative addr=0x%04X, nr of regs=%u\n", + core->description, relative_address, nr_of_regs)); + + /* Do not use burst writes against the registers */ + for (i = 0; i < nr_of_regs; i++) { + mali_hw_core_register_write_relaxed(core, relative_address + i * 4, write_array[i]); + } +} + +/* Conditionally write a set of registers. + * The register will only be written if the new value is different from the old_value. + * If the new value is different, the old value will also be updated */ +MALI_STATIC_INLINE void mali_hw_core_register_write_array_relaxed_conditional(struct mali_hw_core *core, u32 relative_address, u32 *write_array, u32 nr_of_regs, const u32 *old_array) +{ + u32 i; + MALI_DEBUG_PRINT(6, ("register_write_array: for core %s, relative addr=0x%04X, nr of regs=%u\n", + core->description, relative_address, nr_of_regs)); + + /* Do not use burst writes against the registers */ + for (i = 0; i < nr_of_regs; i++) { + if (old_array[i] != write_array[i]) { + mali_hw_core_register_write_relaxed(core, relative_address + i * 4, write_array[i]); + } + } +} + +#endif /* __MALI_HW_CORE_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h b/drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h new file mode 100755 index 000000000000..6a8f0f0116a4 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_common.h @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_COMMON_H__ +#define __MALI_KERNEL_COMMON_H__ + +#include "mali_osk.h" + +/* Make sure debug is defined when it should be */ +#ifndef DEBUG +#if defined(_DEBUG) +#define DEBUG +#endif +#endif + +/* The file include several useful macros for error checking, debugging and printing. + * - MALI_PRINTF(...) Do not use this function: Will be included in Release builds. + * - MALI_DEBUG_PRINT(nr, (X) ) Prints the second argument if nr<=MALI_DEBUG_LEVEL. + * - MALI_DEBUG_ERROR( (X) ) Prints an errortext, a source trace, and the given error message. + * - MALI_DEBUG_ASSERT(exp,(X)) If the asserted expr is false, the program will exit. + * - MALI_DEBUG_ASSERT_POINTER(pointer) Triggers if the pointer is a zero pointer. + * - MALI_DEBUG_CODE( X ) The code inside the macro is only compiled in Debug builds. + * + * The (X) means that you must add an extra parenthesis around the argumentlist. + * + * The printf function: MALI_PRINTF(...) is routed to _mali_osk_debugmsg + * + * Suggested range for the DEBUG-LEVEL is [1:6] where + * [1:2] Is messages with highest priority, indicate possible errors. + * [3:4] Is messages with medium priority, output important variables. + * [5:6] Is messages with low priority, used during extensive debugging. + */ + +/** +* Fundamental error macro. Reports an error code. This is abstracted to allow us to +* easily switch to a different error reporting method if we want, and also to allow +* us to search for error returns easily. +* +* Note no closing semicolon - this is supplied in typical usage: +* +* MALI_ERROR(MALI_ERROR_OUT_OF_MEMORY); +*/ +#define MALI_ERROR(error_code) return (error_code) + +/** + * Basic error macro, to indicate success. + * Note no closing semicolon - this is supplied in typical usage: + * + * MALI_SUCCESS; + */ +#define MALI_SUCCESS MALI_ERROR(_MALI_OSK_ERR_OK) + +/** + * Basic error macro. This checks whether the given condition is true, and if not returns + * from this function with the supplied error code. This is a macro so that we can override it + * for stress testing. + * + * Note that this uses the do-while-0 wrapping to ensure that we don't get problems with dangling + * else clauses. Note also no closing semicolon - this is supplied in typical usage: + * + * MALI_CHECK((p!=NULL), ERROR_NO_OBJECT); + */ +#define MALI_CHECK(condition, error_code) do { if(!(condition)) MALI_ERROR(error_code); } while(0) + +/** + * Error propagation macro. If the expression given is anything other than + * _MALI_OSK_NO_ERROR, then the value is returned from the enclosing function + * as an error code. This effectively acts as a guard clause, and propagates + * error values up the call stack. This uses a temporary value to ensure that + * the error expression is not evaluated twice. + * If the counter for forcing a failure has been set using _mali_force_error, + * this error will be returned without evaluating the expression in + * MALI_CHECK_NO_ERROR + */ +#define MALI_CHECK_NO_ERROR(expression) \ + do { _mali_osk_errcode_t _check_no_error_result=(expression); \ + if(_check_no_error_result != _MALI_OSK_ERR_OK) \ + MALI_ERROR(_check_no_error_result); \ + } while(0) + +/** + * Pointer check macro. Checks non-null pointer. + */ +#define MALI_CHECK_NON_NULL(pointer, error_code) MALI_CHECK( ((pointer)!=NULL), (error_code) ) + +/** + * Error macro with goto. This checks whether the given condition is true, and if not jumps + * to the specified label using a goto. The label must therefore be local to the function in + * which this macro appears. This is most usually used to execute some clean-up code before + * exiting with a call to ERROR. + * + * Like the other macros, this is a macro to allow us to override the condition if we wish, + * e.g. to force an error during stress testing. + */ +#define MALI_CHECK_GOTO(condition, label) do { if(!(condition)) goto label; } while(0) + +/** + * Explicitly ignore a parameter passed into a function, to suppress compiler warnings. + * Should only be used with parameter names. + */ +#define MALI_IGNORE(x) x=x + +#if defined(CONFIG_MALI_QUIET) +#define MALI_PRINTF(args) +#else +#define MALI_PRINTF(args) _mali_osk_dbgmsg args; +#endif + +#define MALI_PRINT_ERROR(args) do{ \ + MALI_PRINTF(("Mali: ERR: %s\n" ,__FILE__)); \ + MALI_PRINTF((" %s()%4d\n ", __FUNCTION__, __LINE__)) ; \ + MALI_PRINTF(args); \ + MALI_PRINTF(("\n")); \ + } while(0) + +#define MALI_PRINT(args) do{ \ + MALI_PRINTF(("Mali: ")); \ + MALI_PRINTF(args); \ + } while (0) + +#ifdef DEBUG +#ifndef mali_debug_level +extern int mali_debug_level; +#endif + +#define MALI_DEBUG_CODE(code) code +#define MALI_DEBUG_PRINT(level, args) do { \ + if((level) <= mali_debug_level)\ + {MALI_PRINTF(("Mali<" #level ">: ")); MALI_PRINTF(args); } \ + } while (0) + +#define MALI_DEBUG_PRINT_ERROR(args) MALI_PRINT_ERROR(args) + +#define MALI_DEBUG_PRINT_IF(level,condition,args) \ + if((condition)&&((level) <= mali_debug_level))\ + {MALI_PRINTF(("Mali<" #level ">: ")); MALI_PRINTF(args); } + +#define MALI_DEBUG_PRINT_ELSE(level, args)\ + else if((level) <= mali_debug_level)\ + { MALI_PRINTF(("Mali<" #level ">: ")); MALI_PRINTF(args); } + +/** + * @note these variants of DEBUG ASSERTS will cause a debugger breakpoint + * to be entered (see _mali_osk_break() ). An alternative would be to call + * _mali_osk_abort(), on OSs that support it. + */ +#define MALI_DEBUG_PRINT_ASSERT(condition, args) do {if( !(condition)) { MALI_PRINT_ERROR(args); _mali_osk_break(); } } while(0) +#define MALI_DEBUG_ASSERT_POINTER(pointer) do {if( (pointer)== NULL) {MALI_PRINT_ERROR(("NULL pointer " #pointer)); _mali_osk_break();} } while(0) +#define MALI_DEBUG_ASSERT(condition) do {if( !(condition)) {MALI_PRINT_ERROR(("ASSERT failed: " #condition )); _mali_osk_break();} } while(0) + +#else /* DEBUG */ + +#define MALI_DEBUG_CODE(code) +#define MALI_DEBUG_PRINT(string,args) do {} while(0) +#define MALI_DEBUG_PRINT_ERROR(args) do {} while(0) +#define MALI_DEBUG_PRINT_IF(level,condition,args) do {} while(0) +#define MALI_DEBUG_PRINT_ELSE(level,condition,args) do {} while(0) +#define MALI_DEBUG_PRINT_ASSERT(condition,args) do {} while(0) +#define MALI_DEBUG_ASSERT_POINTER(pointer) do {} while(0) +#define MALI_DEBUG_ASSERT(condition) do {} while(0) + +#endif /* DEBUG */ + +/** + * variables from user space cannot be dereferenced from kernel space; tagging them + * with __user allows the GCC compiler to generate a warning. Other compilers may + * not support this so we define it here as an empty macro if the compiler doesn't + * define it. + */ +#ifndef __user +#define __user +#endif + +#endif /* __MALI_KERNEL_COMMON_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c new file mode 100755 index 000000000000..87f97b710257 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.c @@ -0,0 +1,1349 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_ukk.h" +#include "mali_kernel_core.h" +#include "mali_memory.h" +#include "mali_mem_validation.h" +#include "mali_mmu.h" +#include "mali_mmu_page_directory.h" +#include "mali_dlbu.h" +#include "mali_broadcast.h" +#include "mali_gp.h" +#include "mali_pp.h" +#include "mali_executor.h" +#include "mali_pp_job.h" +#include "mali_group.h" +#include "mali_pm.h" +#include "mali_pmu.h" +#include "mali_scheduler.h" +#include "mali_kernel_utilization.h" +#include "mali_l2_cache.h" +#include "mali_timeline.h" +#include "mali_soft_job.h" +#include "mali_pm_domain.h" +#if defined(CONFIG_MALI400_PROFILING) +#include "mali_osk_profiling.h" +#endif +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) +#include "mali_profiling_internal.h" +#endif +#include "mali_control_timer.h" +#include "mali_dvfs_policy.h" +#include +#include +#if defined(CONFIG_MALI_DMA_BUF_FENCE) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +#include +#else +#include +#endif +#endif + +#define MALI_SHARED_MEMORY_DEFAULT_SIZE 0xffffffff + +/* Mali GPU memory. Real values come from module parameter or from device specific data */ +unsigned int mali_dedicated_mem_start = 0; +unsigned int mali_dedicated_mem_size = 0; + +/* Default shared memory size is set to 4G. */ +unsigned int mali_shared_mem_size = MALI_SHARED_MEMORY_DEFAULT_SIZE; + +/* Frame buffer memory to be accessible by Mali GPU */ +int mali_fb_start = 0; +int mali_fb_size = 0; + +/* Mali max job runtime */ +extern int mali_max_job_runtime; + +/** Start profiling from module load? */ +int mali_boot_profiling = 0; + +/** Limits for the number of PP cores behind each L2 cache. */ +int mali_max_pp_cores_group_1 = 0xFF; +int mali_max_pp_cores_group_2 = 0xFF; + +int mali_inited_pp_cores_group_1 = 0; +int mali_inited_pp_cores_group_2 = 0; + +static _mali_product_id_t global_product_id = _MALI_PRODUCT_ID_UNKNOWN; +static uintptr_t global_gpu_base_address = 0; +static u32 global_gpu_major_version = 0; +static u32 global_gpu_minor_version = 0; + +mali_bool mali_gpu_class_is_mali450 = MALI_FALSE; +mali_bool mali_gpu_class_is_mali470 = MALI_FALSE; + +static _mali_osk_errcode_t mali_set_global_gpu_base_address(void) +{ + _mali_osk_errcode_t err = _MALI_OSK_ERR_OK; + + global_gpu_base_address = _mali_osk_resource_base_address(); + if (0 == global_gpu_base_address) { + err = _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + return err; +} + +static u32 mali_get_bcast_id(_mali_osk_resource_t *resource_pp) +{ + switch (resource_pp->base - global_gpu_base_address) { + case 0x08000: + case 0x20000: /* fall-through for aliased mapping */ + return 0x01; + case 0x0A000: + case 0x22000: /* fall-through for aliased mapping */ + return 0x02; + case 0x0C000: + case 0x24000: /* fall-through for aliased mapping */ + return 0x04; + case 0x0E000: + case 0x26000: /* fall-through for aliased mapping */ + return 0x08; + case 0x28000: + return 0x10; + case 0x2A000: + return 0x20; + case 0x2C000: + return 0x40; + case 0x2E000: + return 0x80; + default: + return 0; + } +} + +static _mali_osk_errcode_t mali_parse_product_info(void) +{ + _mali_osk_resource_t first_pp_resource; + + /* Find the first PP core resource (again) */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_PP0, &first_pp_resource)) { + /* Create a dummy PP object for this core so that we can read the version register */ + struct mali_group *group = mali_group_create(NULL, NULL, NULL, MALI_DOMAIN_INDEX_PP0); + if (NULL != group) { + struct mali_pp_core *pp_core = mali_pp_create(&first_pp_resource, group, MALI_FALSE, mali_get_bcast_id(&first_pp_resource)); + if (NULL != pp_core) { + u32 pp_version; + + pp_version = mali_pp_core_get_version(pp_core); + + mali_group_delete(group); + + global_gpu_major_version = (pp_version >> 8) & 0xFF; + global_gpu_minor_version = pp_version & 0xFF; + + switch (pp_version >> 16) { + case MALI200_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI200; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-200 r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + MALI_PRINT_ERROR(("Mali-200 is not supported by this driver.\n")); + _mali_osk_abort(); + break; + case MALI300_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI300; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-300 r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + break; + case MALI400_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI400; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-400 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + break; + case MALI450_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI450; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-450 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + break; + case MALI470_PP_PRODUCT_ID: + global_product_id = _MALI_PRODUCT_ID_MALI470; + MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-470 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version)); + break; + default: + MALI_DEBUG_PRINT(2, ("Found unknown Mali GPU (r%up%u)\n", global_gpu_major_version, global_gpu_minor_version)); + return _MALI_OSK_ERR_FAULT; + } + + return _MALI_OSK_ERR_OK; + } else { + MALI_PRINT_ERROR(("Failed to create initial PP object\n")); + } + } else { + MALI_PRINT_ERROR(("Failed to create initial group object\n")); + } + } else { + MALI_PRINT_ERROR(("First PP core not specified in config file\n")); + } + + return _MALI_OSK_ERR_FAULT; +} + +static void mali_delete_groups(void) +{ + struct mali_group *group; + + group = mali_group_get_glob_group(0); + while (NULL != group) { + mali_group_delete(group); + group = mali_group_get_glob_group(0); + } + + MALI_DEBUG_ASSERT(0 == mali_group_get_glob_num_groups()); +} + +static void mali_delete_l2_cache_cores(void) +{ + struct mali_l2_cache_core *l2; + + l2 = mali_l2_cache_core_get_glob_l2_core(0); + while (NULL != l2) { + mali_l2_cache_delete(l2); + l2 = mali_l2_cache_core_get_glob_l2_core(0); + } + + MALI_DEBUG_ASSERT(0 == mali_l2_cache_core_get_glob_num_l2_cores()); +} + +static struct mali_l2_cache_core *mali_create_l2_cache_core(_mali_osk_resource_t *resource, u32 domain_index) +{ + struct mali_l2_cache_core *l2_cache = NULL; + + if (NULL != resource) { + + MALI_DEBUG_PRINT(3, ("Found L2 cache %s\n", resource->description)); + + l2_cache = mali_l2_cache_create(resource, domain_index); + if (NULL == l2_cache) { + MALI_PRINT_ERROR(("Failed to create L2 cache object\n")); + return NULL; + } + } + MALI_DEBUG_PRINT(3, ("Created L2 cache core object\n")); + + return l2_cache; +} + +static _mali_osk_errcode_t mali_parse_config_l2_cache(void) +{ + struct mali_l2_cache_core *l2_cache = NULL; + + if (mali_is_mali400()) { + _mali_osk_resource_t l2_resource; + if (_MALI_OSK_ERR_OK != _mali_osk_resource_find(MALI400_OFFSET_L2_CACHE0, &l2_resource)) { + MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache in config file\n")); + return _MALI_OSK_ERR_FAULT; + } + + l2_cache = mali_create_l2_cache_core(&l2_resource, MALI_DOMAIN_INDEX_L20); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } else if (mali_is_mali450()) { + /* + * L2 for GP at 0x10000 + * L2 for PP0-3 at 0x01000 + * L2 for PP4-7 at 0x11000 (optional) + */ + + _mali_osk_resource_t l2_gp_resource; + _mali_osk_resource_t l2_pp_grp0_resource; + _mali_osk_resource_t l2_pp_grp1_resource; + + /* Make cluster for GP's L2 */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE0, &l2_gp_resource)) { + MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for GP\n")); + l2_cache = mali_create_l2_cache_core(&l2_gp_resource, MALI_DOMAIN_INDEX_L20); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } else { + MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for GP in config file\n")); + return _MALI_OSK_ERR_FAULT; + } + + /* Find corresponding l2 domain */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE1, &l2_pp_grp0_resource)) { + MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for PP group 0\n")); + l2_cache = mali_create_l2_cache_core(&l2_pp_grp0_resource, MALI_DOMAIN_INDEX_L21); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } else { + MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for PP group 0 in config file\n")); + return _MALI_OSK_ERR_FAULT; + } + + /* Second PP core group is optional, don't fail if we don't find it */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE2, &l2_pp_grp1_resource)) { + MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for PP group 1\n")); + l2_cache = mali_create_l2_cache_core(&l2_pp_grp1_resource, MALI_DOMAIN_INDEX_L22); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } + } else if (mali_is_mali470()) { + _mali_osk_resource_t l2c1_resource; + + /* Make cluster for L2C1 */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI470_OFFSET_L2_CACHE1, &l2c1_resource)) { + MALI_DEBUG_PRINT(3, ("Creating Mali-470 L2 cache 1\n")); + l2_cache = mali_create_l2_cache_core(&l2c1_resource, MALI_DOMAIN_INDEX_L21); + if (NULL == l2_cache) { + return _MALI_OSK_ERR_FAULT; + } + } else { + MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for L2C1\n")); + return _MALI_OSK_ERR_FAULT; + } + } + + return _MALI_OSK_ERR_OK; +} + +static struct mali_group *mali_create_group(struct mali_l2_cache_core *cache, + _mali_osk_resource_t *resource_mmu, + _mali_osk_resource_t *resource_gp, + _mali_osk_resource_t *resource_pp, + u32 domain_index) +{ + struct mali_mmu_core *mmu; + struct mali_group *group; + + MALI_DEBUG_PRINT(3, ("Starting new group for MMU %s\n", resource_mmu->description)); + + /* Create the group object */ + group = mali_group_create(cache, NULL, NULL, domain_index); + if (NULL == group) { + MALI_PRINT_ERROR(("Failed to create group object for MMU %s\n", resource_mmu->description)); + return NULL; + } + + /* Create the MMU object inside group */ + mmu = mali_mmu_create(resource_mmu, group, MALI_FALSE); + if (NULL == mmu) { + MALI_PRINT_ERROR(("Failed to create MMU object\n")); + mali_group_delete(group); + return NULL; + } + + if (NULL != resource_gp) { + /* Create the GP core object inside this group */ + struct mali_gp_core *gp_core = mali_gp_create(resource_gp, group); + if (NULL == gp_core) { + /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */ + MALI_PRINT_ERROR(("Failed to create GP object\n")); + mali_group_delete(group); + return NULL; + } + } + + if (NULL != resource_pp) { + struct mali_pp_core *pp_core; + + /* Create the PP core object inside this group */ + pp_core = mali_pp_create(resource_pp, group, MALI_FALSE, mali_get_bcast_id(resource_pp)); + if (NULL == pp_core) { + /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */ + MALI_PRINT_ERROR(("Failed to create PP object\n")); + mali_group_delete(group); + return NULL; + } + } + + return group; +} + +static _mali_osk_errcode_t mali_create_virtual_group(_mali_osk_resource_t *resource_mmu_pp_bcast, + _mali_osk_resource_t *resource_pp_bcast, + _mali_osk_resource_t *resource_dlbu, + _mali_osk_resource_t *resource_bcast) +{ + struct mali_mmu_core *mmu_pp_bcast_core; + struct mali_pp_core *pp_bcast_core; + struct mali_dlbu_core *dlbu_core; + struct mali_bcast_unit *bcast_core; + struct mali_group *group; + + MALI_DEBUG_PRINT(2, ("Starting new virtual group for MMU PP broadcast core %s\n", resource_mmu_pp_bcast->description)); + + /* Create the DLBU core object */ + dlbu_core = mali_dlbu_create(resource_dlbu); + if (NULL == dlbu_core) { + MALI_PRINT_ERROR(("Failed to create DLBU object \n")); + return _MALI_OSK_ERR_FAULT; + } + + /* Create the Broadcast unit core */ + bcast_core = mali_bcast_unit_create(resource_bcast); + if (NULL == bcast_core) { + MALI_PRINT_ERROR(("Failed to create Broadcast unit object!\n")); + mali_dlbu_delete(dlbu_core); + return _MALI_OSK_ERR_FAULT; + } + + /* Create the group object */ +#if defined(DEBUG) + /* Get a physical PP group to temporarily add to broadcast unit. IRQ + * verification needs a physical group in the broadcast unit to test + * the broadcast unit interrupt line. */ + { + struct mali_group *phys_group = NULL; + int i; + for (i = 0; i < mali_group_get_glob_num_groups(); i++) { + phys_group = mali_group_get_glob_group(i); + if (NULL != mali_group_get_pp_core(phys_group)) break; + } + MALI_DEBUG_ASSERT(NULL != mali_group_get_pp_core(phys_group)); + + /* Add the group temporarily to the broadcast, and update the + * broadcast HW. Since the HW is not updated when removing the + * group the IRQ check will work when the virtual PP is created + * later. + * + * When the virtual group gets populated, the actually used + * groups will be added to the broadcast unit and the HW will + * be updated. + */ + mali_bcast_add_group(bcast_core, phys_group); + mali_bcast_reset(bcast_core); + mali_bcast_remove_group(bcast_core, phys_group); + } +#endif /* DEBUG */ + group = mali_group_create(NULL, dlbu_core, bcast_core, MALI_DOMAIN_INDEX_DUMMY); + if (NULL == group) { + MALI_PRINT_ERROR(("Failed to create group object for MMU PP broadcast core %s\n", resource_mmu_pp_bcast->description)); + mali_bcast_unit_delete(bcast_core); + mali_dlbu_delete(dlbu_core); + return _MALI_OSK_ERR_FAULT; + } + + /* Create the MMU object inside group */ + mmu_pp_bcast_core = mali_mmu_create(resource_mmu_pp_bcast, group, MALI_TRUE); + if (NULL == mmu_pp_bcast_core) { + MALI_PRINT_ERROR(("Failed to create MMU PP broadcast object\n")); + mali_group_delete(group); + return _MALI_OSK_ERR_FAULT; + } + + /* Create the PP core object inside this group */ + pp_bcast_core = mali_pp_create(resource_pp_bcast, group, MALI_TRUE, 0); + if (NULL == pp_bcast_core) { + /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */ + MALI_PRINT_ERROR(("Failed to create PP object\n")); + mali_group_delete(group); + return _MALI_OSK_ERR_FAULT; + } + + return _MALI_OSK_ERR_OK; +} + +static _mali_osk_errcode_t mali_parse_config_groups(void) +{ + struct mali_group *group; + int cluster_id_gp = 0; + int cluster_id_pp_grp0 = 0; + int cluster_id_pp_grp1 = 0; + int i; + + _mali_osk_resource_t resource_gp; + _mali_osk_resource_t resource_gp_mmu; + _mali_osk_resource_t resource_pp[8]; + _mali_osk_resource_t resource_pp_mmu[8]; + _mali_osk_resource_t resource_pp_mmu_bcast; + _mali_osk_resource_t resource_pp_bcast; + _mali_osk_resource_t resource_dlbu; + _mali_osk_resource_t resource_bcast; + _mali_osk_errcode_t resource_gp_found; + _mali_osk_errcode_t resource_gp_mmu_found; + _mali_osk_errcode_t resource_pp_found[8]; + _mali_osk_errcode_t resource_pp_mmu_found[8]; + _mali_osk_errcode_t resource_pp_mmu_bcast_found; + _mali_osk_errcode_t resource_pp_bcast_found; + _mali_osk_errcode_t resource_dlbu_found; + _mali_osk_errcode_t resource_bcast_found; + + if (!(mali_is_mali400() || mali_is_mali450() || mali_is_mali470())) { + /* No known HW core */ + return _MALI_OSK_ERR_FAULT; + } + + if (MALI_MAX_JOB_RUNTIME_DEFAULT == mali_max_job_runtime) { + /* Group settings are not overridden by module parameters, so use device settings */ + _mali_osk_device_data data = { 0, }; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + /* Use device specific settings (if defined) */ + if (0 != data.max_job_runtime) { + mali_max_job_runtime = data.max_job_runtime; + } + } + } + + if (mali_is_mali450()) { + /* Mali-450 have separate L2s for GP, and PP core group(s) */ + cluster_id_pp_grp0 = 1; + cluster_id_pp_grp1 = 2; + } + + resource_gp_found = _mali_osk_resource_find(MALI_OFFSET_GP, &resource_gp); + resource_gp_mmu_found = _mali_osk_resource_find(MALI_OFFSET_GP_MMU, &resource_gp_mmu); + resource_pp_found[0] = _mali_osk_resource_find(MALI_OFFSET_PP0, &(resource_pp[0])); + resource_pp_found[1] = _mali_osk_resource_find(MALI_OFFSET_PP1, &(resource_pp[1])); + resource_pp_found[2] = _mali_osk_resource_find(MALI_OFFSET_PP2, &(resource_pp[2])); + resource_pp_found[3] = _mali_osk_resource_find(MALI_OFFSET_PP3, &(resource_pp[3])); + resource_pp_found[4] = _mali_osk_resource_find(MALI_OFFSET_PP4, &(resource_pp[4])); + resource_pp_found[5] = _mali_osk_resource_find(MALI_OFFSET_PP5, &(resource_pp[5])); + resource_pp_found[6] = _mali_osk_resource_find(MALI_OFFSET_PP6, &(resource_pp[6])); + resource_pp_found[7] = _mali_osk_resource_find(MALI_OFFSET_PP7, &(resource_pp[7])); + resource_pp_mmu_found[0] = _mali_osk_resource_find(MALI_OFFSET_PP0_MMU, &(resource_pp_mmu[0])); + resource_pp_mmu_found[1] = _mali_osk_resource_find(MALI_OFFSET_PP1_MMU, &(resource_pp_mmu[1])); + resource_pp_mmu_found[2] = _mali_osk_resource_find(MALI_OFFSET_PP2_MMU, &(resource_pp_mmu[2])); + resource_pp_mmu_found[3] = _mali_osk_resource_find(MALI_OFFSET_PP3_MMU, &(resource_pp_mmu[3])); + resource_pp_mmu_found[4] = _mali_osk_resource_find(MALI_OFFSET_PP4_MMU, &(resource_pp_mmu[4])); + resource_pp_mmu_found[5] = _mali_osk_resource_find(MALI_OFFSET_PP5_MMU, &(resource_pp_mmu[5])); + resource_pp_mmu_found[6] = _mali_osk_resource_find(MALI_OFFSET_PP6_MMU, &(resource_pp_mmu[6])); + resource_pp_mmu_found[7] = _mali_osk_resource_find(MALI_OFFSET_PP7_MMU, &(resource_pp_mmu[7])); + + + if (mali_is_mali450() || mali_is_mali470()) { + resource_bcast_found = _mali_osk_resource_find(MALI_OFFSET_BCAST, &resource_bcast); + resource_dlbu_found = _mali_osk_resource_find(MALI_OFFSET_DLBU, &resource_dlbu); + resource_pp_mmu_bcast_found = _mali_osk_resource_find(MALI_OFFSET_PP_BCAST_MMU, &resource_pp_mmu_bcast); + resource_pp_bcast_found = _mali_osk_resource_find(MALI_OFFSET_PP_BCAST, &resource_pp_bcast); + + if (_MALI_OSK_ERR_OK != resource_bcast_found || + _MALI_OSK_ERR_OK != resource_dlbu_found || + _MALI_OSK_ERR_OK != resource_pp_mmu_bcast_found || + _MALI_OSK_ERR_OK != resource_pp_bcast_found) { + /* Missing mandatory core(s) for Mali-450 or Mali-470 */ + MALI_DEBUG_PRINT(2, ("Missing mandatory resources, Mali-450 needs DLBU, Broadcast unit, virtual PP core and virtual MMU\n")); + return _MALI_OSK_ERR_FAULT; + } + } + + if (_MALI_OSK_ERR_OK != resource_gp_found || + _MALI_OSK_ERR_OK != resource_gp_mmu_found || + _MALI_OSK_ERR_OK != resource_pp_found[0] || + _MALI_OSK_ERR_OK != resource_pp_mmu_found[0]) { + /* Missing mandatory core(s) */ + MALI_DEBUG_PRINT(2, ("Missing mandatory resource, need at least one GP and one PP, both with a separate MMU\n")); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT(1 <= mali_l2_cache_core_get_glob_num_l2_cores()); + group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_gp), &resource_gp_mmu, &resource_gp, NULL, MALI_DOMAIN_INDEX_GP); + if (NULL == group) { + return _MALI_OSK_ERR_FAULT; + } + + /* Create group for first (and mandatory) PP core */ + MALI_DEBUG_ASSERT(mali_l2_cache_core_get_glob_num_l2_cores() >= (cluster_id_pp_grp0 + 1)); /* >= 1 on Mali-300 and Mali-400, >= 2 on Mali-450 */ + group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp0), &resource_pp_mmu[0], NULL, &resource_pp[0], MALI_DOMAIN_INDEX_PP0); + if (NULL == group) { + return _MALI_OSK_ERR_FAULT; + } + + mali_inited_pp_cores_group_1++; + + /* Create groups for rest of the cores in the first PP core group */ + for (i = 1; i < 4; i++) { /* First half of the PP cores belong to first core group */ + if (mali_inited_pp_cores_group_1 < mali_max_pp_cores_group_1) { + if (_MALI_OSK_ERR_OK == resource_pp_found[i] && _MALI_OSK_ERR_OK == resource_pp_mmu_found[i]) { + group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp0), &resource_pp_mmu[i], NULL, &resource_pp[i], MALI_DOMAIN_INDEX_PP0 + i); + if (NULL == group) { + return _MALI_OSK_ERR_FAULT; + } + + mali_inited_pp_cores_group_1++; + } + } + } + + /* Create groups for cores in the second PP core group */ + for (i = 4; i < 8; i++) { /* Second half of the PP cores belong to second core group */ + if (mali_inited_pp_cores_group_2 < mali_max_pp_cores_group_2) { + if (_MALI_OSK_ERR_OK == resource_pp_found[i] && _MALI_OSK_ERR_OK == resource_pp_mmu_found[i]) { + MALI_DEBUG_ASSERT(mali_l2_cache_core_get_glob_num_l2_cores() >= 2); /* Only Mali-450 have a second core group */ + group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp1), &resource_pp_mmu[i], NULL, &resource_pp[i], MALI_DOMAIN_INDEX_PP0 + i); + if (NULL == group) { + return _MALI_OSK_ERR_FAULT; + } + + mali_inited_pp_cores_group_2++; + } + } + } + + if (mali_is_mali450() || mali_is_mali470()) { + _mali_osk_errcode_t err = mali_create_virtual_group(&resource_pp_mmu_bcast, &resource_pp_bcast, &resource_dlbu, &resource_bcast); + if (_MALI_OSK_ERR_OK != err) { + return err; + } + } + + mali_max_pp_cores_group_1 = mali_inited_pp_cores_group_1; + mali_max_pp_cores_group_2 = mali_inited_pp_cores_group_2; + MALI_DEBUG_PRINT(2, ("%d+%d PP cores initialized\n", mali_inited_pp_cores_group_1, mali_inited_pp_cores_group_2)); + + return _MALI_OSK_ERR_OK; +} + +static _mali_osk_errcode_t mali_check_shared_interrupts(void) +{ +#if !defined(CONFIG_MALI_SHARED_INTERRUPTS) + if (MALI_TRUE == _mali_osk_shared_interrupts()) { + MALI_PRINT_ERROR(("Shared interrupts detected, but driver support is not enabled\n")); + return _MALI_OSK_ERR_FAULT; + } +#endif /* !defined(CONFIG_MALI_SHARED_INTERRUPTS) */ + + /* It is OK to compile support for shared interrupts even if Mali is not using it. */ + return _MALI_OSK_ERR_OK; +} + +static _mali_osk_errcode_t mali_parse_config_pmu(void) +{ + _mali_osk_resource_t resource_pmu; + + MALI_DEBUG_ASSERT(0 != global_gpu_base_address); + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_PMU, &resource_pmu)) { + struct mali_pmu_core *pmu; + + pmu = mali_pmu_create(&resource_pmu); + if (NULL == pmu) { + MALI_PRINT_ERROR(("Failed to create PMU\n")); + return _MALI_OSK_ERR_FAULT; + } + } + + /* It's ok if the PMU doesn't exist */ + return _MALI_OSK_ERR_OK; +} + +static _mali_osk_errcode_t mali_parse_config_memory(void) +{ + _mali_osk_device_data data = { 0, }; + _mali_osk_errcode_t ret; + + /* The priority of setting the value of mali_shared_mem_size, + * mali_dedicated_mem_start and mali_dedicated_mem_size: + * 1. module parameter; + * 2. platform data; + * 3. default value; + **/ + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + /* Memory settings are not overridden by module parameters, so use device settings */ + if (0 == mali_dedicated_mem_start && 0 == mali_dedicated_mem_size) { + /* Use device specific settings (if defined) */ + mali_dedicated_mem_start = data.dedicated_mem_start; + mali_dedicated_mem_size = data.dedicated_mem_size; + } + + if (MALI_SHARED_MEMORY_DEFAULT_SIZE == mali_shared_mem_size && + 0 != data.shared_mem_size) { + mali_shared_mem_size = data.shared_mem_size; + } + } + + if (0 < mali_dedicated_mem_size && 0 != mali_dedicated_mem_start) { + MALI_DEBUG_PRINT(2, ("Mali memory settings (dedicated: 0x%08X@0x%08X)\n", + mali_dedicated_mem_size, mali_dedicated_mem_start)); + + /* Dedicated memory */ + ret = mali_memory_core_resource_dedicated_memory(mali_dedicated_mem_start, mali_dedicated_mem_size); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to register dedicated memory\n")); + mali_memory_terminate(); + return ret; + } + } + + if (0 < mali_shared_mem_size) { + MALI_DEBUG_PRINT(2, ("Mali memory settings (shared: 0x%08X)\n", mali_shared_mem_size)); + + /* Shared OS memory */ + ret = mali_memory_core_resource_os_memory(mali_shared_mem_size); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to register shared OS memory\n")); + mali_memory_terminate(); + return ret; + } + } + + if (0 == mali_fb_start && 0 == mali_fb_size) { + /* Frame buffer settings are not overridden by module parameters, so use device settings */ + _mali_osk_device_data data = { 0, }; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + /* Use device specific settings (if defined) */ + mali_fb_start = data.fb_start; + mali_fb_size = data.fb_size; + } + + MALI_DEBUG_PRINT(2, ("Using device defined frame buffer settings (0x%08X@0x%08X)\n", + mali_fb_size, mali_fb_start)); + } else { + MALI_DEBUG_PRINT(2, ("Using module defined frame buffer settings (0x%08X@0x%08X)\n", + mali_fb_size, mali_fb_start)); + } + + if (0 != mali_fb_size) { + /* Register frame buffer */ + ret = mali_mem_validation_add_range(mali_fb_start, mali_fb_size); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to register frame buffer memory region\n")); + mali_memory_terminate(); + return ret; + } + } + + return _MALI_OSK_ERR_OK; +} + +static void mali_detect_gpu_class(void) +{ + if (_mali_osk_identify_gpu_resource() == 0x450) + mali_gpu_class_is_mali450 = MALI_TRUE; + + if (_mali_osk_identify_gpu_resource() == 0x470) + mali_gpu_class_is_mali470 = MALI_TRUE; +} + +static _mali_osk_errcode_t mali_init_hw_reset(void) +{ +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + _mali_osk_resource_t resource_bcast; + + /* Ensure broadcast unit is in a good state before we start creating + * groups and cores. + */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_BCAST, &resource_bcast)) { + struct mali_bcast_unit *bcast_core; + + bcast_core = mali_bcast_unit_create(&resource_bcast); + if (NULL == bcast_core) { + MALI_PRINT_ERROR(("Failed to create Broadcast unit object!\n")); + return _MALI_OSK_ERR_FAULT; + } + mali_bcast_unit_delete(bcast_core); + } +#endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */ + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_initialize_subsystems(void) +{ + _mali_osk_errcode_t err; + +#ifdef CONFIG_MALI_DT + err = _mali_osk_resource_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } +#endif + + mali_pp_job_initialize(); + + err = mali_timeline_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + err = mali_session_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /*Try to init gpu secure mode */ + _mali_osk_gpu_secure_mode_init(); + +#if defined(CONFIG_MALI400_PROFILING) + err = _mali_osk_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE); + if (_MALI_OSK_ERR_OK != err) { + /* No biggie if we weren't able to initialize the profiling */ + MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n")); + } +#endif + + err = mali_memory_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + err = mali_executor_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + err = mali_scheduler_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Configure memory early, needed by mali_mmu_initialize. */ + err = mali_parse_config_memory(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + err = mali_set_global_gpu_base_address(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Detect GPU class (uses L2 cache count) */ + mali_detect_gpu_class(); + + err = mali_check_shared_interrupts(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Initialize the MALI PMU (will not touch HW!) */ + err = mali_parse_config_pmu(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Initialize the power management module */ + err = mali_pm_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Make sure the entire GPU stays on for the rest of this function */ + mali_pm_init_begin(); + + /* Ensure HW is in a good state before starting to access cores. */ + err = mali_init_hw_reset(); + if (_MALI_OSK_ERR_OK != err) { + mali_terminate_subsystems(); + return err; + } + + /* Detect which Mali GPU we are dealing with */ + err = mali_parse_product_info(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + + /* The global_product_id is now populated with the correct Mali GPU */ + + /* Start configuring the actual Mali hardware. */ + + err = mali_mmu_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + + if (mali_is_mali450() || mali_is_mali470()) { + err = mali_dlbu_initialize(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + } + + err = mali_parse_config_l2_cache(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + + err = mali_parse_config_groups(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + + /* Move groups into executor */ + mali_executor_populate(); + + /* Need call after all group has assigned a domain */ + mali_pm_power_cost_setup(); + + /* Initialize the GPU timer */ + err = mali_control_timer_init(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + + /* Initialize the GPU utilization tracking */ + err = mali_utilization_init(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } + +#if defined(CONFIG_MALI_DVFS) + err = mali_dvfs_policy_init(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_init_end(); + mali_terminate_subsystems(); + return err; + } +#endif + + /* Allowing the system to be turned off */ + mali_pm_init_end(); + + return _MALI_OSK_ERR_OK; /* all ok */ +} + +void mali_terminate_subsystems(void) +{ + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + MALI_DEBUG_PRINT(2, ("terminate_subsystems() called\n")); + + mali_utilization_term(); + mali_control_timer_term(); + + mali_executor_depopulate(); + mali_delete_groups(); /* Delete groups not added to executor */ + mali_executor_terminate(); + + mali_scheduler_terminate(); + mali_pp_job_terminate(); + mali_delete_l2_cache_cores(); + mali_mmu_terminate(); + + if (mali_is_mali450() || mali_is_mali470()) { + mali_dlbu_terminate(); + } + + mali_pm_terminate(); + + if (NULL != pmu) { + mali_pmu_delete(pmu); + } + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_term(); +#endif + + _mali_osk_gpu_secure_mode_deinit(); + + mali_memory_terminate(); + + mali_session_terminate(); + + mali_timeline_terminate(); + + global_gpu_base_address = 0; +} + +_mali_product_id_t mali_kernel_core_get_product_id(void) +{ + return global_product_id; +} + +u32 mali_kernel_core_get_gpu_major_version(void) +{ + return global_gpu_major_version; +} + +u32 mali_kernel_core_get_gpu_minor_version(void) +{ + return global_gpu_minor_version; +} + +_mali_osk_errcode_t _mali_ukk_get_api_version(_mali_uk_get_api_version_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + /* check compatability */ + if (args->version == _MALI_UK_API_VERSION) { + args->compatible = 1; + } else { + args->compatible = 0; + } + + args->version = _MALI_UK_API_VERSION; /* report our version */ + + /* success regardless of being compatible or not */ + MALI_SUCCESS; +} + +_mali_osk_errcode_t _mali_ukk_get_api_version_v2(_mali_uk_get_api_version_v2_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + /* check compatability */ + if (args->version == _MALI_UK_API_VERSION) { + args->compatible = 1; + } else { + args->compatible = 0; + } + + args->version = _MALI_UK_API_VERSION; /* report our version */ + + /* success regardless of being compatible or not */ + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notification_s *args) +{ + _mali_osk_errcode_t err; + _mali_osk_notification_t *notification; + _mali_osk_notification_queue_t *queue; + struct mali_session_data *session; + + /* check input */ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + queue = session->ioctl_queue; + + /* if the queue does not exist we're currently shutting down */ + if (NULL == queue) { + MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n")); + args->type = _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS; + return _MALI_OSK_ERR_OK; + } + + /* receive a notification, might sleep */ + err = _mali_osk_notification_queue_receive(queue, ¬ification); + if (_MALI_OSK_ERR_OK != err) { + MALI_ERROR(err); /* errcode returned, pass on to caller */ + } + + /* copy the buffer to the user */ + args->type = (_mali_uk_notification_type)notification->notification_type; + _mali_osk_memcpy(&args->data, notification->result_buffer, notification->result_buffer_size); + + /* finished with the notification */ + _mali_osk_notification_delete(notification); + + return _MALI_OSK_ERR_OK; /* all ok */ +} + +_mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *args) +{ + _mali_osk_notification_t *notification; + _mali_osk_notification_queue_t *queue; + struct mali_session_data *session; + + /* check input */ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + queue = session->ioctl_queue; + + /* if the queue does not exist we're currently shutting down */ + if (NULL == queue) { + MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n")); + return _MALI_OSK_ERR_OK; + } + + notification = _mali_osk_notification_create(args->type, 0); + if (NULL == notification) { + MALI_PRINT_ERROR(("Failed to create notification object\n")); + return _MALI_OSK_ERR_NOMEM; + } + + _mali_osk_notification_queue_send(queue, notification); + + return _MALI_OSK_ERR_OK; /* all ok */ +} + +_mali_osk_errcode_t _mali_ukk_pending_submit(_mali_uk_pending_submit_s *args) +{ + wait_queue_head_t *queue; + + /* check input */ + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + queue = mali_session_get_wait_queue(); + + /* check pending big job number, might sleep if larger than MAX allowed number */ + if (wait_event_interruptible(*queue, MALI_MAX_PENDING_BIG_JOB > mali_scheduler_job_gp_big_job_count())) { + return _MALI_OSK_ERR_RESTARTSYSCALL; + } + + return _MALI_OSK_ERR_OK; /* all ok */ +} + + +_mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priority_s *args) +{ + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + + if (!session->use_high_priority_job_queue) { + session->use_high_priority_job_queue = MALI_TRUE; + MALI_DEBUG_PRINT(2, ("Session 0x%08X with pid %d was granted higher priority.\n", session, _mali_osk_get_pid())); + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_open(void **context) +{ + u32 i; + struct mali_session_data *session; + + /* allocated struct to track this session */ + session = (struct mali_session_data *)_mali_osk_calloc(1, sizeof(struct mali_session_data)); + MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_NOMEM); + + MALI_DEBUG_PRINT(3, ("Session starting\n")); + + /* create a response queue for this session */ + session->ioctl_queue = _mali_osk_notification_queue_init(); + if (NULL == session->ioctl_queue) { + goto err; + } + + /*create a wait queue for this session */ + session->wait_queue = _mali_osk_wait_queue_init(); + if (NULL == session->wait_queue) { + goto err_wait_queue; + } + + session->page_directory = mali_mmu_pagedir_alloc(); + if (NULL == session->page_directory) { + goto err_mmu; + } + + if (_MALI_OSK_ERR_OK != mali_mmu_pagedir_map(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE)) { + MALI_PRINT_ERROR(("Failed to map DLBU page into session\n")); + goto err_mmu; + } + + if (0 != mali_dlbu_phys_addr) { + mali_mmu_pagedir_update(session->page_directory, MALI_DLBU_VIRT_ADDR, mali_dlbu_phys_addr, + _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + } + + if (_MALI_OSK_ERR_OK != mali_memory_session_begin(session)) { + goto err_session; + } + + /* Create soft system. */ + session->soft_job_system = mali_soft_job_system_create(session); + if (NULL == session->soft_job_system) { + goto err_soft; + } + + /* Initialize the dma fence context.*/ +#if defined(CONFIG_MALI_DMA_BUF_FENCE) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + session->fence_context = dma_fence_context_alloc(1); +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) + session->fence_context = fence_context_alloc(1); + _mali_osk_atomic_init(&session->fence_seqno, 0); +#else + MALI_PRINT_ERROR(("The kernel version not support dma fence!\n")); + goto err_time_line; +#endif +#endif + + /* Create timeline system. */ + session->timeline_system = mali_timeline_system_create(session); + if (NULL == session->timeline_system) { + goto err_time_line; + } + +#if defined(CONFIG_MALI_DVFS) + _mali_osk_atomic_init(&session->number_of_window_jobs, 0); +#endif + + _mali_osk_atomic_init(&session->number_of_pp_jobs, 0); + + session->use_high_priority_job_queue = MALI_FALSE; + + /* Initialize list of PP jobs on this session. */ + _MALI_OSK_INIT_LIST_HEAD(&session->pp_job_list); + + /* Initialize the pp_job_fb_lookup_list array used to quickly lookup jobs from a given frame builder */ + for (i = 0; i < MALI_PP_JOB_FB_LOOKUP_LIST_SIZE; ++i) { + _MALI_OSK_INIT_LIST_HEAD(&session->pp_job_fb_lookup_list[i]); + } + + session->pid = _mali_osk_get_pid(); + session->comm = _mali_osk_get_comm(); + session->max_mali_mem_allocated_size = 0; + for (i = 0; i < MALI_MEM_TYPE_MAX; i ++) { + atomic_set(&session->mali_mem_array[i], 0); + } + atomic_set(&session->mali_mem_allocated_pages, 0); + *context = (void *)session; + + /* Add session to the list of all sessions. */ + mali_session_add(session); + + MALI_DEBUG_PRINT(3, ("Session started\n")); + return _MALI_OSK_ERR_OK; + +err_time_line: + mali_soft_job_system_destroy(session->soft_job_system); +err_soft: + mali_memory_session_end(session); +err_session: + mali_mmu_pagedir_free(session->page_directory); +err_mmu: + _mali_osk_wait_queue_term(session->wait_queue); +err_wait_queue: + _mali_osk_notification_queue_term(session->ioctl_queue); +err: + _mali_osk_free(session); + MALI_ERROR(_MALI_OSK_ERR_NOMEM); + +} + +#if defined(DEBUG) +/* parameter used for debug */ +extern u32 num_pm_runtime_resume; +extern u32 num_pm_updates; +extern u32 num_pm_updates_up; +extern u32 num_pm_updates_down; +#endif + +_mali_osk_errcode_t _mali_ukk_close(void **context) +{ + struct mali_session_data *session; + MALI_CHECK_NON_NULL(context, _MALI_OSK_ERR_INVALID_ARGS); + session = (struct mali_session_data *)*context; + + MALI_DEBUG_PRINT(3, ("Session ending\n")); + + MALI_DEBUG_ASSERT_POINTER(session->soft_job_system); + MALI_DEBUG_ASSERT_POINTER(session->timeline_system); + + /* Remove session from list of all sessions. */ + mali_session_remove(session); + + /* This flag is used to prevent queueing of jobs due to activation. */ + session->is_aborting = MALI_TRUE; + + /* Stop the soft job timer. */ + mali_timeline_system_stop_timer(session->timeline_system); + + /* Abort queued jobs */ + mali_scheduler_abort_session(session); + + /* Abort executing jobs */ + mali_executor_abort_session(session); + + /* Abort the soft job system. */ + mali_soft_job_system_abort(session->soft_job_system); + + /* Force execution of all pending bottom half processing for GP and PP. */ + _mali_osk_wq_flush(); + + /* The session PP list should now be empty. */ + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&session->pp_job_list)); + + /* At this point the GP and PP scheduler no longer has any jobs queued or running from this + * session, and all soft jobs in the soft job system has been destroyed. */ + + /* Any trackers left in the timeline system are directly or indirectly waiting on external + * sync fences. Cancel all sync fence waiters to trigger activation of all remaining + * trackers. This call will sleep until all timelines are empty. */ + mali_timeline_system_abort(session->timeline_system); + + /* Flush pending work. + * Needed to make sure all bottom half processing related to this + * session has been completed, before we free internal data structures. + */ + _mali_osk_wq_flush(); + + /* Destroy timeline system. */ + mali_timeline_system_destroy(session->timeline_system); + session->timeline_system = NULL; + + /* Destroy soft system. */ + mali_soft_job_system_destroy(session->soft_job_system); + session->soft_job_system = NULL; + + /*Wait for the session job lists become empty.*/ + _mali_osk_wait_queue_wait_event(session->wait_queue, mali_session_pp_job_is_empty, (void *) session); + + /* Free remaining memory allocated to this session */ + mali_memory_session_end(session); + +#if defined(CONFIG_MALI_DVFS) + _mali_osk_atomic_term(&session->number_of_window_jobs); +#endif + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_stop_sampling(session->pid); +#endif + + /* Free session data structures */ + mali_mmu_pagedir_unmap(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE); + mali_mmu_pagedir_free(session->page_directory); + _mali_osk_wait_queue_term(session->wait_queue); + _mali_osk_notification_queue_term(session->ioctl_queue); + _mali_osk_free(session); + + *context = NULL; + + MALI_DEBUG_PRINT(3, ("Session has ended\n")); + +#if defined(DEBUG) + MALI_DEBUG_PRINT(3, ("Stats: # runtime resumes: %u\n", num_pm_runtime_resume)); + MALI_DEBUG_PRINT(3, (" # PM updates: .... %u (up %u, down %u)\n", num_pm_updates, num_pm_updates_up, num_pm_updates_down)); + + num_pm_runtime_resume = 0; + num_pm_updates = 0; + num_pm_updates_up = 0; + num_pm_updates_down = 0; +#endif + + return _MALI_OSK_ERR_OK;; +} + +#if MALI_STATE_TRACKING +u32 _mali_kernel_core_dump_state(char *buf, u32 size) +{ + int n = 0; /* Number of bytes written to buf */ + + n += mali_scheduler_dump_state(buf + n, size - n); + n += mali_executor_dump_state(buf + n, size - n); + + return n; +} +#endif diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h new file mode 100755 index 000000000000..c471fc955107 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_core.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_CORE_H__ +#define __MALI_KERNEL_CORE_H__ + +#include "mali_osk.h" + +typedef enum { + _MALI_PRODUCT_ID_UNKNOWN, + _MALI_PRODUCT_ID_MALI200, + _MALI_PRODUCT_ID_MALI300, + _MALI_PRODUCT_ID_MALI400, + _MALI_PRODUCT_ID_MALI450, + _MALI_PRODUCT_ID_MALI470, +} _mali_product_id_t; + +extern mali_bool mali_gpu_class_is_mali450; +extern mali_bool mali_gpu_class_is_mali470; + +_mali_osk_errcode_t mali_initialize_subsystems(void); + +void mali_terminate_subsystems(void); + +_mali_product_id_t mali_kernel_core_get_product_id(void); + +u32 mali_kernel_core_get_gpu_major_version(void); + +u32 mali_kernel_core_get_gpu_minor_version(void); + +u32 _mali_kernel_core_dump_state(char *buf, u32 size); + +MALI_STATIC_INLINE mali_bool mali_is_mali470(void) +{ + return mali_gpu_class_is_mali470; +} + +MALI_STATIC_INLINE mali_bool mali_is_mali450(void) +{ + return mali_gpu_class_is_mali450; +} + +MALI_STATIC_INLINE mali_bool mali_is_mali400(void) +{ + if (mali_gpu_class_is_mali450 || mali_gpu_class_is_mali470) + return MALI_FALSE; + + return MALI_TRUE; +} +#endif /* __MALI_KERNEL_CORE_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c new file mode 100755 index 000000000000..d1b8dc3b0b0e --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.c @@ -0,0 +1,440 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_utilization.h" +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_scheduler.h" + +#include "mali_executor.h" +#include "mali_dvfs_policy.h" +#include "mali_control_timer.h" + +/* Thresholds for GP bound detection. */ +#define MALI_GP_BOUND_GP_UTILIZATION_THRESHOLD 240 +#define MALI_GP_BOUND_PP_UTILIZATION_THRESHOLD 250 + +static _mali_osk_spinlock_irq_t *utilization_data_lock; + +static u32 num_running_gp_cores = 0; +static u32 num_running_pp_cores = 0; + +static u64 work_start_time_gpu = 0; +static u64 work_start_time_gp = 0; +static u64 work_start_time_pp = 0; +static u64 accumulated_work_time_gpu = 0; +static u64 accumulated_work_time_gp = 0; +static u64 accumulated_work_time_pp = 0; + +static u32 last_utilization_gpu = 0 ; +static u32 last_utilization_gp = 0 ; +static u32 last_utilization_pp = 0 ; + +void (*mali_utilization_callback)(struct mali_gpu_utilization_data *data) = NULL; + +/* Define the first timer control timer timeout in milliseconds */ +static u32 mali_control_first_timeout = 100; +static struct mali_gpu_utilization_data mali_util_data = {0, }; + +struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u64 *time_period, mali_bool *need_add_timer) +{ + u64 time_now; + u32 leading_zeroes; + u32 shift_val; + u32 work_normalized_gpu; + u32 work_normalized_gp; + u32 work_normalized_pp; + u32 period_normalized; + u32 utilization_gpu; + u32 utilization_gp; + u32 utilization_pp; + + mali_utilization_data_lock(); + + time_now = _mali_osk_time_get_ns(); + + *time_period = time_now - *start_time; + + if (accumulated_work_time_gpu == 0 && work_start_time_gpu == 0) { + mali_control_timer_pause(); + /* + * No work done for this period + * - No need to reschedule timer + * - Report zero usage + */ + last_utilization_gpu = 0; + last_utilization_gp = 0; + last_utilization_pp = 0; + + mali_util_data.utilization_gpu = last_utilization_gpu; + mali_util_data.utilization_gp = last_utilization_gp; + mali_util_data.utilization_pp = last_utilization_pp; + + mali_utilization_data_unlock(); + + *need_add_timer = MALI_FALSE; + + mali_executor_hint_disable(MALI_EXECUTOR_HINT_GP_BOUND); + + MALI_DEBUG_PRINT(4, ("last_utilization_gpu = %d \n", last_utilization_gpu)); + MALI_DEBUG_PRINT(4, ("last_utilization_gp = %d \n", last_utilization_gp)); + MALI_DEBUG_PRINT(4, ("last_utilization_pp = %d \n", last_utilization_pp)); + + return &mali_util_data; + } + + /* If we are currently busy, update working period up to now */ + if (work_start_time_gpu != 0) { + accumulated_work_time_gpu += (time_now - work_start_time_gpu); + work_start_time_gpu = time_now; + + /* GP and/or PP will also be busy if the GPU is busy at this point */ + + if (work_start_time_gp != 0) { + accumulated_work_time_gp += (time_now - work_start_time_gp); + work_start_time_gp = time_now; + } + + if (work_start_time_pp != 0) { + accumulated_work_time_pp += (time_now - work_start_time_pp); + work_start_time_pp = time_now; + } + } + + /* + * We have two 64-bit values, a dividend and a divisor. + * To avoid dependencies to a 64-bit divider, we shift down the two values + * equally first. + * We shift the dividend up and possibly the divisor down, making the result X in 256. + */ + + /* Shift the 64-bit values down so they fit inside a 32-bit integer */ + leading_zeroes = _mali_osk_clz((u32)(*time_period >> 32)); + shift_val = 32 - leading_zeroes; + work_normalized_gpu = (u32)(accumulated_work_time_gpu >> shift_val); + work_normalized_gp = (u32)(accumulated_work_time_gp >> shift_val); + work_normalized_pp = (u32)(accumulated_work_time_pp >> shift_val); + period_normalized = (u32)(*time_period >> shift_val); + + /* + * Now, we should report the usage in parts of 256 + * this means we must shift up the dividend or down the divisor by 8 + * (we could do a combination, but we just use one for simplicity, + * but the end result should be good enough anyway) + */ + if (period_normalized > 0x00FFFFFF) { + /* The divisor is so big that it is safe to shift it down */ + period_normalized >>= 8; + } else { + /* + * The divisor is so small that we can shift up the dividend, without loosing any data. + * (dividend is always smaller than the divisor) + */ + work_normalized_gpu <<= 8; + work_normalized_gp <<= 8; + work_normalized_pp <<= 8; + } + + utilization_gpu = work_normalized_gpu / period_normalized; + utilization_gp = work_normalized_gp / period_normalized; + utilization_pp = work_normalized_pp / period_normalized; + + last_utilization_gpu = utilization_gpu; + last_utilization_gp = utilization_gp; + last_utilization_pp = utilization_pp; + + if ((MALI_GP_BOUND_GP_UTILIZATION_THRESHOLD < last_utilization_gp) && + (MALI_GP_BOUND_PP_UTILIZATION_THRESHOLD > last_utilization_pp)) { + mali_executor_hint_enable(MALI_EXECUTOR_HINT_GP_BOUND); + } else { + mali_executor_hint_disable(MALI_EXECUTOR_HINT_GP_BOUND); + } + + /* starting a new period */ + accumulated_work_time_gpu = 0; + accumulated_work_time_gp = 0; + accumulated_work_time_pp = 0; + + *start_time = time_now; + + mali_util_data.utilization_gp = last_utilization_gp; + mali_util_data.utilization_gpu = last_utilization_gpu; + mali_util_data.utilization_pp = last_utilization_pp; + + mali_utilization_data_unlock(); + + *need_add_timer = MALI_TRUE; + + MALI_DEBUG_PRINT(4, ("last_utilization_gpu = %d \n", last_utilization_gpu)); + MALI_DEBUG_PRINT(4, ("last_utilization_gp = %d \n", last_utilization_gp)); + MALI_DEBUG_PRINT(4, ("last_utilization_pp = %d \n", last_utilization_pp)); + + return &mali_util_data; +} + +_mali_osk_errcode_t mali_utilization_init(void) +{ +#if USING_GPU_UTILIZATION + _mali_osk_device_data data; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + if (NULL != data.utilization_callback) { + mali_utilization_callback = data.utilization_callback; + MALI_DEBUG_PRINT(2, ("Mali GPU Utilization: Utilization handler installed \n")); + } + } +#endif /* defined(USING_GPU_UTILIZATION) */ + + if (NULL == mali_utilization_callback) { + MALI_DEBUG_PRINT(2, ("Mali GPU Utilization: No platform utilization handler installed\n")); + } + + utilization_data_lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_UTILIZATION); + if (NULL == utilization_data_lock) { + return _MALI_OSK_ERR_FAULT; + } + + num_running_gp_cores = 0; + num_running_pp_cores = 0; + + return _MALI_OSK_ERR_OK; +} + +void mali_utilization_term(void) +{ + if (NULL != utilization_data_lock) { + _mali_osk_spinlock_irq_term(utilization_data_lock); + } +} + +void mali_utilization_gp_start(void) +{ + mali_utilization_data_lock(); + + ++num_running_gp_cores; + if (1 == num_running_gp_cores) { + u64 time_now = _mali_osk_time_get_ns(); + + /* First GP core started, consider GP busy from now and onwards */ + work_start_time_gp = time_now; + + if (0 == num_running_pp_cores) { + mali_bool is_resume = MALI_FALSE; + /* + * There are no PP cores running, so this is also the point + * at which we consider the GPU to be busy as well. + */ + work_start_time_gpu = time_now; + + is_resume = mali_control_timer_resume(time_now); + + mali_utilization_data_unlock(); + + if (is_resume) { + /* Do some policy in new period for performance consideration */ +#if defined(CONFIG_MALI_DVFS) + /* Clear session->number_of_window_jobs, prepare parameter for dvfs */ + mali_session_max_window_num(); + if (0 == last_utilization_gpu) { + /* + * for mali_dev_pause is called in set clock, + * so each time we change clock, we will set clock to + * highest step even if under down clock case, + * it is not nessesary, so we only set the clock under + * last time utilization equal 0, we stop the timer then + * start the GPU again case + */ + mali_dvfs_policy_new_period(); + } +#endif + /* + * First timeout using short interval for power consideration + * because we give full power in the new period, but if the + * job loading is light, finish in 10ms, the other time all keep + * in high freq it will wast time. + */ + mali_control_timer_add(mali_control_first_timeout); + } + } else { + mali_utilization_data_unlock(); + } + + } else { + /* Nothing to do */ + mali_utilization_data_unlock(); + } +} + +void mali_utilization_pp_start(void) +{ + mali_utilization_data_lock(); + + ++num_running_pp_cores; + if (1 == num_running_pp_cores) { + u64 time_now = _mali_osk_time_get_ns(); + + /* First PP core started, consider PP busy from now and onwards */ + work_start_time_pp = time_now; + + if (0 == num_running_gp_cores) { + mali_bool is_resume = MALI_FALSE; + /* + * There are no GP cores running, so this is also the point + * at which we consider the GPU to be busy as well. + */ + work_start_time_gpu = time_now; + + /* Start a new period if stoped */ + is_resume = mali_control_timer_resume(time_now); + + mali_utilization_data_unlock(); + + if (is_resume) { +#if defined(CONFIG_MALI_DVFS) + /* Clear session->number_of_window_jobs, prepare parameter for dvfs */ + mali_session_max_window_num(); + if (0 == last_utilization_gpu) { + /* + * for mali_dev_pause is called in set clock, + * so each time we change clock, we will set clock to + * highest step even if under down clock case, + * it is not nessesary, so we only set the clock under + * last time utilization equal 0, we stop the timer then + * start the GPU again case + */ + mali_dvfs_policy_new_period(); + } +#endif + + /* + * First timeout using short interval for power consideration + * because we give full power in the new period, but if the + * job loading is light, finish in 10ms, the other time all keep + * in high freq it will wast time. + */ + mali_control_timer_add(mali_control_first_timeout); + } + } else { + mali_utilization_data_unlock(); + } + } else { + /* Nothing to do */ + mali_utilization_data_unlock(); + } +} + +void mali_utilization_gp_end(void) +{ + mali_utilization_data_lock(); + + --num_running_gp_cores; + if (0 == num_running_gp_cores) { + u64 time_now = _mali_osk_time_get_ns(); + + /* Last GP core ended, consider GP idle from now and onwards */ + accumulated_work_time_gp += (time_now - work_start_time_gp); + work_start_time_gp = 0; + + if (0 == num_running_pp_cores) { + /* + * There are no PP cores running, so this is also the point + * at which we consider the GPU to be idle as well. + */ + accumulated_work_time_gpu += (time_now - work_start_time_gpu); + work_start_time_gpu = 0; + } + } + + mali_utilization_data_unlock(); +} + +void mali_utilization_pp_end(void) +{ + mali_utilization_data_lock(); + + --num_running_pp_cores; + if (0 == num_running_pp_cores) { + u64 time_now = _mali_osk_time_get_ns(); + + /* Last PP core ended, consider PP idle from now and onwards */ + accumulated_work_time_pp += (time_now - work_start_time_pp); + work_start_time_pp = 0; + + if (0 == num_running_gp_cores) { + /* + * There are no GP cores running, so this is also the point + * at which we consider the GPU to be idle as well. + */ + accumulated_work_time_gpu += (time_now - work_start_time_gpu); + work_start_time_gpu = 0; + } + } + + mali_utilization_data_unlock(); +} + +mali_bool mali_utilization_enabled(void) +{ +#if defined(CONFIG_MALI_DVFS) + return mali_dvfs_policy_enabled(); +#else + return (NULL != mali_utilization_callback); +#endif /* defined(CONFIG_MALI_DVFS) */ +} + +void mali_utilization_platform_realize(struct mali_gpu_utilization_data *util_data) +{ + MALI_DEBUG_ASSERT_POINTER(mali_utilization_callback); + + mali_utilization_callback(util_data); +} + +void mali_utilization_reset(void) +{ + accumulated_work_time_gpu = 0; + accumulated_work_time_gp = 0; + accumulated_work_time_pp = 0; + + last_utilization_gpu = 0; + last_utilization_gp = 0; + last_utilization_pp = 0; +} + +void mali_utilization_data_lock(void) +{ + _mali_osk_spinlock_irq_lock(utilization_data_lock); +} + +void mali_utilization_data_unlock(void) +{ + _mali_osk_spinlock_irq_unlock(utilization_data_lock); +} + +void mali_utilization_data_assert_locked(void) +{ + MALI_DEBUG_ASSERT_LOCK_HELD(utilization_data_lock); +} + +u32 _mali_ukk_utilization_gp_pp(void) +{ + return last_utilization_gpu; +} + +u32 _mali_ukk_utilization_gp(void) +{ + return last_utilization_gp; +} + +u32 _mali_ukk_utilization_pp(void) +{ + return last_utilization_pp; +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h new file mode 100755 index 000000000000..06f585dcb238 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_utilization.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_UTILIZATION_H__ +#define __MALI_KERNEL_UTILIZATION_H__ + +#include +#include "mali_osk.h" + +/** + * Initialize/start the Mali GPU utilization metrics reporting. + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t mali_utilization_init(void); + +/** + * Terminate the Mali GPU utilization metrics reporting + */ +void mali_utilization_term(void); + +/** + * Check if Mali utilization is enabled + */ +mali_bool mali_utilization_enabled(void); + +/** + * Should be called when a job is about to execute a GP job + */ +void mali_utilization_gp_start(void); + +/** + * Should be called when a job has completed executing a GP job + */ +void mali_utilization_gp_end(void); + +/** + * Should be called when a job is about to execute a PP job + */ +void mali_utilization_pp_start(void); + +/** + * Should be called when a job has completed executing a PP job + */ +void mali_utilization_pp_end(void); + +/** + * Should be called to calcution the GPU utilization + */ +struct mali_gpu_utilization_data *mali_utilization_calculate(u64 *start_time, u64 *time_period, mali_bool *need_add_timer); + +_mali_osk_spinlock_irq_t *mali_utilization_get_lock(void); + +void mali_utilization_platform_realize(struct mali_gpu_utilization_data *util_data); + +void mali_utilization_data_lock(void); + +void mali_utilization_data_unlock(void); + +void mali_utilization_data_assert_locked(void); + +void mali_utilization_reset(void); + + +#endif /* __MALI_KERNEL_UTILIZATION_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c b/drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c new file mode 100755 index 000000000000..dd44e5e7fa03 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_kernel_vsync.c @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" + +#include "mali_osk_profiling.h" + +_mali_osk_errcode_t _mali_ukk_vsync_event_report(_mali_uk_vsync_event_report_s *args) +{ + _mali_uk_vsync_event event = (_mali_uk_vsync_event)args->event; + MALI_IGNORE(event); /* event is not used for release code, and that is OK */ + + /* + * Manually generate user space events in kernel space. + * This saves user space from calling kernel space twice in this case. + * We just need to remember to add pid and tid manually. + */ + if (event == _MALI_UK_VSYNC_EVENT_BEGIN_WAIT) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SUSPEND | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VSYNC, + _mali_osk_get_pid(), _mali_osk_get_tid(), 0, 0, 0); + } + + if (event == _MALI_UK_VSYNC_EVENT_END_WAIT) { + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_RESUME | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VSYNC, + _mali_osk_get_pid(), _mali_osk_get_tid(), 0, 0, 0); + } + + + MALI_DEBUG_PRINT(4, ("Received VSYNC event: %d\n", event)); + MALI_SUCCESS; +} + diff --git a/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c new file mode 100755 index 000000000000..fe33f561b2aa --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.c @@ -0,0 +1,534 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_l2_cache.h" +#include "mali_hw_core.h" +#include "mali_scheduler.h" +#include "mali_pm.h" +#include "mali_pm_domain.h" + +/** + * Size of the Mali L2 cache registers in bytes + */ +#define MALI400_L2_CACHE_REGISTERS_SIZE 0x30 + +/** + * Mali L2 cache register numbers + * Used in the register read/write routines. + * See the hardware documentation for more information about each register + */ +typedef enum mali_l2_cache_register { + MALI400_L2_CACHE_REGISTER_SIZE = 0x0004, + MALI400_L2_CACHE_REGISTER_STATUS = 0x0008, + /*unused = 0x000C */ + MALI400_L2_CACHE_REGISTER_COMMAND = 0x0010, + MALI400_L2_CACHE_REGISTER_CLEAR_PAGE = 0x0014, + MALI400_L2_CACHE_REGISTER_MAX_READS = 0x0018, + MALI400_L2_CACHE_REGISTER_ENABLE = 0x001C, + MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0 = 0x0020, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0 = 0x0024, + MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1 = 0x0028, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1 = 0x002C, +} mali_l2_cache_register; + +/** + * Mali L2 cache commands + * These are the commands that can be sent to the Mali L2 cache unit + */ +typedef enum mali_l2_cache_command { + MALI400_L2_CACHE_COMMAND_CLEAR_ALL = 0x01, +} mali_l2_cache_command; + +/** + * Mali L2 cache commands + * These are the commands that can be sent to the Mali L2 cache unit + */ +typedef enum mali_l2_cache_enable { + MALI400_L2_CACHE_ENABLE_DEFAULT = 0x0, /* Default */ + MALI400_L2_CACHE_ENABLE_ACCESS = 0x01, + MALI400_L2_CACHE_ENABLE_READ_ALLOCATE = 0x02, +} mali_l2_cache_enable; + +/** + * Mali L2 cache status bits + */ +typedef enum mali_l2_cache_status { + MALI400_L2_CACHE_STATUS_COMMAND_BUSY = 0x01, + MALI400_L2_CACHE_STATUS_DATA_BUSY = 0x02, +} mali_l2_cache_status; + +#define MALI400_L2_MAX_READS_NOT_SET -1 + +static struct mali_l2_cache_core * + mali_global_l2s[MALI_MAX_NUMBER_OF_L2_CACHE_CORES] = { NULL, }; +static u32 mali_global_num_l2s = 0; + +int mali_l2_max_reads = MALI400_L2_MAX_READS_NOT_SET; + + +/* Local helper functions */ + +static void mali_l2_cache_reset(struct mali_l2_cache_core *cache); + +static _mali_osk_errcode_t mali_l2_cache_send_command( + struct mali_l2_cache_core *cache, u32 reg, u32 val); + +static void mali_l2_cache_lock(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + _mali_osk_spinlock_irq_lock(cache->lock); +} + +static void mali_l2_cache_unlock(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + _mali_osk_spinlock_irq_unlock(cache->lock); +} + +/* Implementation of the L2 cache interface */ + +struct mali_l2_cache_core *mali_l2_cache_create( + _mali_osk_resource_t *resource, u32 domain_index) +{ + struct mali_l2_cache_core *cache = NULL; +#if defined(DEBUG) + u32 cache_size; +#endif + + MALI_DEBUG_PRINT(4, ("Mali L2 cache: Creating Mali L2 cache: %s\n", + resource->description)); + + if (mali_global_num_l2s >= MALI_MAX_NUMBER_OF_L2_CACHE_CORES) { + MALI_PRINT_ERROR(("Mali L2 cache: Too many L2 caches\n")); + return NULL; + } + + cache = _mali_osk_malloc(sizeof(struct mali_l2_cache_core)); + if (NULL == cache) { + MALI_PRINT_ERROR(("Mali L2 cache: Failed to allocate memory for L2 cache core\n")); + return NULL; + } + + cache->core_id = mali_global_num_l2s; + cache->counter_src0 = MALI_HW_CORE_NO_COUNTER; + cache->counter_src1 = MALI_HW_CORE_NO_COUNTER; + cache->counter_value0_base = 0; + cache->counter_value1_base = 0; + cache->pm_domain = NULL; + cache->power_is_on = MALI_FALSE; + cache->last_invalidated_id = 0; + + if (_MALI_OSK_ERR_OK != mali_hw_core_create(&cache->hw_core, + resource, MALI400_L2_CACHE_REGISTERS_SIZE)) { + _mali_osk_free(cache); + return NULL; + } + +#if defined(DEBUG) + cache_size = mali_hw_core_register_read(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_SIZE); + MALI_DEBUG_PRINT(2, ("Mali L2 cache: Created %s: % 3uK, %u-way, % 2ubyte cache line, % 3ubit external bus\n", + resource->description, + 1 << (((cache_size >> 16) & 0xff) - 10), + 1 << ((cache_size >> 8) & 0xff), + 1 << (cache_size & 0xff), + 1 << ((cache_size >> 24) & 0xff))); +#endif + + cache->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_L2); + if (NULL == cache->lock) { + MALI_PRINT_ERROR(("Mali L2 cache: Failed to create counter lock for L2 cache core %s\n", + cache->hw_core.description)); + mali_hw_core_delete(&cache->hw_core); + _mali_osk_free(cache); + return NULL; + } + + /* register with correct power domain */ + cache->pm_domain = mali_pm_register_l2_cache( + domain_index, cache); + + mali_global_l2s[mali_global_num_l2s] = cache; + mali_global_num_l2s++; + + return cache; +} + +void mali_l2_cache_delete(struct mali_l2_cache_core *cache) +{ + u32 i; + for (i = 0; i < mali_global_num_l2s; i++) { + if (mali_global_l2s[i] != cache) { + continue; + } + + mali_global_l2s[i] = NULL; + mali_global_num_l2s--; + + if (i == mali_global_num_l2s) { + /* Removed last element, nothing more to do */ + break; + } + + /* + * We removed a l2 cache from the middle of the array, + * so move the last l2 cache to current position + */ + mali_global_l2s[i] = mali_global_l2s[mali_global_num_l2s]; + mali_global_l2s[mali_global_num_l2s] = NULL; + + /* All good */ + break; + } + + _mali_osk_spinlock_irq_term(cache->lock); + mali_hw_core_delete(&cache->hw_core); + _mali_osk_free(cache); +} + +void mali_l2_cache_power_up(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + + mali_l2_cache_lock(cache); + + mali_l2_cache_reset(cache); + + if ((1 << MALI_DOMAIN_INDEX_DUMMY) != cache->pm_domain->pmu_mask) + MALI_DEBUG_ASSERT(MALI_FALSE == cache->power_is_on); + cache->power_is_on = MALI_TRUE; + + mali_l2_cache_unlock(cache); +} + +void mali_l2_cache_power_down(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + + mali_l2_cache_lock(cache); + + MALI_DEBUG_ASSERT(MALI_TRUE == cache->power_is_on); + + /* + * The HW counters will start from zero again when we resume, + * but we should report counters as always increasing. + * Take a copy of the HW values now in order to add this to + * the values we report after being powered up. + * + * The physical power off of the L2 cache might be outside our + * own control (e.g. runtime PM). That is why we must manually + * set set the counter value to zero as well. + */ + + if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) { + cache->counter_value0_base += mali_hw_core_register_read( + &cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0); + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0, 0); + } + + if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) { + cache->counter_value1_base += mali_hw_core_register_read( + &cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1); + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1, 0); + } + + + cache->power_is_on = MALI_FALSE; + + mali_l2_cache_unlock(cache); +} + +void mali_l2_cache_core_set_counter_src( + struct mali_l2_cache_core *cache, u32 source_id, u32 counter) +{ + u32 reg_offset_src; + u32 reg_offset_val; + + MALI_DEBUG_ASSERT_POINTER(cache); + MALI_DEBUG_ASSERT(source_id >= 0 && source_id <= 1); + + mali_l2_cache_lock(cache); + + if (0 == source_id) { + /* start counting from 0 */ + cache->counter_value0_base = 0; + cache->counter_src0 = counter; + reg_offset_src = MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0; + reg_offset_val = MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0; + } else { + /* start counting from 0 */ + cache->counter_value1_base = 0; + cache->counter_src1 = counter; + reg_offset_src = MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1; + reg_offset_val = MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1; + } + + if (cache->power_is_on) { + u32 hw_src; + + if (MALI_HW_CORE_NO_COUNTER != counter) { + hw_src = counter; + } else { + hw_src = 0; /* disable value for HW */ + } + + /* Set counter src */ + mali_hw_core_register_write(&cache->hw_core, + reg_offset_src, hw_src); + + /* Make sure the HW starts counting from 0 again */ + mali_hw_core_register_write(&cache->hw_core, + reg_offset_val, 0); + } + + mali_l2_cache_unlock(cache); +} + +void mali_l2_cache_core_get_counter_values( + struct mali_l2_cache_core *cache, + u32 *src0, u32 *value0, u32 *src1, u32 *value1) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + MALI_DEBUG_ASSERT(NULL != src0); + MALI_DEBUG_ASSERT(NULL != value0); + MALI_DEBUG_ASSERT(NULL != src1); + MALI_DEBUG_ASSERT(NULL != value1); + + mali_l2_cache_lock(cache); + + *src0 = cache->counter_src0; + *src1 = cache->counter_src1; + + if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) { + if (MALI_TRUE == cache->power_is_on) { + *value0 = mali_hw_core_register_read(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL0); + } else { + *value0 = 0; + } + + /* Add base offset value (in case we have been power off) */ + *value0 += cache->counter_value0_base; + } + + if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) { + if (MALI_TRUE == cache->power_is_on) { + *value1 = mali_hw_core_register_read(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_VAL1); + } else { + *value1 = 0; + } + + /* Add base offset value (in case we have been power off) */ + *value1 += cache->counter_value1_base; + } + + mali_l2_cache_unlock(cache); +} + +struct mali_l2_cache_core *mali_l2_cache_core_get_glob_l2_core(u32 index) +{ + if (mali_global_num_l2s > index) { + return mali_global_l2s[index]; + } + + return NULL; +} + +u32 mali_l2_cache_core_get_glob_num_l2_cores(void) +{ + return mali_global_num_l2s; +} + +void mali_l2_cache_invalidate(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + + if (NULL == cache) { + return; + } + + mali_l2_cache_lock(cache); + + cache->last_invalidated_id = mali_scheduler_get_new_cache_order(); + mali_l2_cache_send_command(cache, MALI400_L2_CACHE_REGISTER_COMMAND, + MALI400_L2_CACHE_COMMAND_CLEAR_ALL); + + mali_l2_cache_unlock(cache); +} + +void mali_l2_cache_invalidate_conditional( + struct mali_l2_cache_core *cache, u32 id) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + + if (NULL == cache) { + return; + } + + /* + * If the last cache invalidation was done by a job with a higher id we + * don't have to flush. Since user space will store jobs w/ their + * corresponding memory in sequence (first job #0, then job #1, ...), + * we don't have to flush for job n-1 if job n has already invalidated + * the cache since we know for sure that job n-1's memory was already + * written when job n was started. + */ + + mali_l2_cache_lock(cache); + + if (((s32)id) > ((s32)cache->last_invalidated_id)) { + /* Set latest invalidated id to current "point in time" */ + cache->last_invalidated_id = + mali_scheduler_get_new_cache_order(); + mali_l2_cache_send_command(cache, + MALI400_L2_CACHE_REGISTER_COMMAND, + MALI400_L2_CACHE_COMMAND_CLEAR_ALL); + } + + mali_l2_cache_unlock(cache); +} + +void mali_l2_cache_invalidate_all(void) +{ + u32 i; + for (i = 0; i < mali_global_num_l2s; i++) { + struct mali_l2_cache_core *cache = mali_global_l2s[i]; + _mali_osk_errcode_t ret; + + MALI_DEBUG_ASSERT_POINTER(cache); + + mali_l2_cache_lock(cache); + + if (MALI_TRUE != cache->power_is_on) { + mali_l2_cache_unlock(cache); + continue; + } + + cache->last_invalidated_id = + mali_scheduler_get_new_cache_order(); + + ret = mali_l2_cache_send_command(cache, + MALI400_L2_CACHE_REGISTER_COMMAND, + MALI400_L2_CACHE_COMMAND_CLEAR_ALL); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to invalidate cache\n")); + } + + mali_l2_cache_unlock(cache); + } +} + +void mali_l2_cache_invalidate_all_pages(u32 *pages, u32 num_pages) +{ + u32 i; + for (i = 0; i < mali_global_num_l2s; i++) { + struct mali_l2_cache_core *cache = mali_global_l2s[i]; + u32 j; + + MALI_DEBUG_ASSERT_POINTER(cache); + + mali_l2_cache_lock(cache); + + if (MALI_TRUE != cache->power_is_on) { + mali_l2_cache_unlock(cache); + continue; + } + + for (j = 0; j < num_pages; j++) { + _mali_osk_errcode_t ret; + + ret = mali_l2_cache_send_command(cache, + MALI400_L2_CACHE_REGISTER_CLEAR_PAGE, + pages[j]); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to invalidate cache (page)\n")); + } + } + + mali_l2_cache_unlock(cache); + } +} + +/* -------- local helper functions below -------- */ + +static void mali_l2_cache_reset(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + MALI_DEBUG_ASSERT_LOCK_HELD(cache->lock); + + /* Invalidate cache (just to keep it in a known state at startup) */ + mali_l2_cache_send_command(cache, MALI400_L2_CACHE_REGISTER_COMMAND, + MALI400_L2_CACHE_COMMAND_CLEAR_ALL); + + /* Enable cache */ + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_ENABLE, + (u32)MALI400_L2_CACHE_ENABLE_ACCESS | + (u32)MALI400_L2_CACHE_ENABLE_READ_ALLOCATE); + + if (MALI400_L2_MAX_READS_NOT_SET != mali_l2_max_reads) { + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_MAX_READS, + (u32)mali_l2_max_reads); + } + + /* Restart any performance counters (if enabled) */ + if (cache->counter_src0 != MALI_HW_CORE_NO_COUNTER) { + + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_SRC0, + cache->counter_src0); + } + + if (cache->counter_src1 != MALI_HW_CORE_NO_COUNTER) { + mali_hw_core_register_write(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_PERFCNT_SRC1, + cache->counter_src1); + } +} + +static _mali_osk_errcode_t mali_l2_cache_send_command( + struct mali_l2_cache_core *cache, u32 reg, u32 val) +{ + int i = 0; + const int loop_count = 100000; + + MALI_DEBUG_ASSERT_POINTER(cache); + MALI_DEBUG_ASSERT_LOCK_HELD(cache->lock); + + /* + * First, wait for L2 cache command handler to go idle. + * (Commands received while processing another command will be ignored) + */ + for (i = 0; i < loop_count; i++) { + if (!(mali_hw_core_register_read(&cache->hw_core, + MALI400_L2_CACHE_REGISTER_STATUS) & + (u32)MALI400_L2_CACHE_STATUS_COMMAND_BUSY)) { + break; + } + } + + if (i == loop_count) { + MALI_DEBUG_PRINT(1, ("Mali L2 cache: aborting wait for command interface to go idle\n")); + return _MALI_OSK_ERR_FAULT; + } + + /* then issue the command */ + mali_hw_core_register_write(&cache->hw_core, reg, val); + + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h new file mode 100755 index 000000000000..c48a8844075f --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_l2_cache.h @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_L2_CACHE_H__ +#define __MALI_KERNEL_L2_CACHE_H__ + +#include "mali_osk.h" +#include "mali_hw_core.h" + +#define MALI_MAX_NUMBER_OF_L2_CACHE_CORES 3 +/* Maximum 1 GP and 4 PP for an L2 cache core (Mali-400 MP4) */ +#define MALI_MAX_NUMBER_OF_GROUPS_PER_L2_CACHE 5 + +/** + * Definition of the L2 cache core struct + * Used to track a L2 cache unit in the system. + * Contains information about the mapping of the registers + */ +struct mali_l2_cache_core { + /* Common HW core functionality */ + struct mali_hw_core hw_core; + + /* Synchronize L2 cache access */ + _mali_osk_spinlock_irq_t *lock; + + /* Unique core ID */ + u32 core_id; + + /* The power domain this L2 cache belongs to */ + struct mali_pm_domain *pm_domain; + + /* MALI_TRUE if power is on for this L2 cache */ + mali_bool power_is_on; + + /* A "timestamp" to avoid unnecessary flushes */ + u32 last_invalidated_id; + + /* Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ + u32 counter_src0; + + /* Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ + u32 counter_src1; + + /* + * Performance counter 0 value base/offset + * (allows accumulative reporting even after power off) + */ + u32 counter_value0_base; + + /* + * Performance counter 0 value base/offset + * (allows accumulative reporting even after power off) + */ + u32 counter_value1_base; + + /* Used by PM domains to link L2 caches of same domain */ + _mali_osk_list_t pm_domain_list; +}; + +_mali_osk_errcode_t mali_l2_cache_initialize(void); +void mali_l2_cache_terminate(void); + +struct mali_l2_cache_core *mali_l2_cache_create( + _mali_osk_resource_t *resource, u32 domain_index); +void mali_l2_cache_delete(struct mali_l2_cache_core *cache); + +MALI_STATIC_INLINE u32 mali_l2_cache_get_id(struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + return cache->core_id; +} + +MALI_STATIC_INLINE struct mali_pm_domain *mali_l2_cache_get_pm_domain( + struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + return cache->pm_domain; +} + +void mali_l2_cache_power_up(struct mali_l2_cache_core *cache); +void mali_l2_cache_power_down(struct mali_l2_cache_core *cache); + +void mali_l2_cache_core_set_counter_src( + struct mali_l2_cache_core *cache, u32 source_id, u32 counter); + +MALI_STATIC_INLINE u32 mali_l2_cache_core_get_counter_src0( + struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + return cache->counter_src0; +} + +MALI_STATIC_INLINE u32 mali_l2_cache_core_get_counter_src1( + struct mali_l2_cache_core *cache) +{ + MALI_DEBUG_ASSERT_POINTER(cache); + return cache->counter_src1; +} + +void mali_l2_cache_core_get_counter_values( + struct mali_l2_cache_core *cache, + u32 *src0, u32 *value0, u32 *src1, u32 *value1); + +struct mali_l2_cache_core *mali_l2_cache_core_get_glob_l2_core(u32 index); +u32 mali_l2_cache_core_get_glob_num_l2_cores(void); + +struct mali_group *mali_l2_cache_get_group( + struct mali_l2_cache_core *cache, u32 index); + +void mali_l2_cache_invalidate(struct mali_l2_cache_core *cache); +void mali_l2_cache_invalidate_conditional( + struct mali_l2_cache_core *cache, u32 id); + +void mali_l2_cache_invalidate_all(void); +void mali_l2_cache_invalidate_all_pages(u32 *pages, u32 num_pages); + +#endif /* __MALI_KERNEL_L2_CACHE_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c new file mode 100755 index 000000000000..eb95998f1469 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.c @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_mem_validation.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" + +#define MALI_INVALID_MEM_ADDR 0xFFFFFFFF + +typedef struct { + u32 phys_base; /**< Mali physical base of the memory, page aligned */ + u32 size; /**< size in bytes of the memory, multiple of page size */ +} _mali_mem_validation_t; + +static _mali_mem_validation_t mali_mem_validator = { MALI_INVALID_MEM_ADDR, MALI_INVALID_MEM_ADDR }; + +_mali_osk_errcode_t mali_mem_validation_add_range(u32 start, u32 size) +{ + /* Check that no other MEM_VALIDATION resources exist */ + if (MALI_INVALID_MEM_ADDR != mali_mem_validator.phys_base) { + MALI_PRINT_ERROR(("Failed to add frame buffer memory; another range is already specified\n")); + return _MALI_OSK_ERR_FAULT; + } + + /* Check restrictions on page alignment */ + if ((0 != (start & (~_MALI_OSK_CPU_PAGE_MASK))) || + (0 != (size & (~_MALI_OSK_CPU_PAGE_MASK)))) { + MALI_PRINT_ERROR(("Failed to add frame buffer memory; incorrect alignment\n")); + return _MALI_OSK_ERR_FAULT; + } + + mali_mem_validator.phys_base = start; + mali_mem_validator.size = size; + MALI_DEBUG_PRINT(2, ("Memory Validator installed for Mali physical address base=0x%08X, size=0x%08X\n", + mali_mem_validator.phys_base, mali_mem_validator.size)); + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_mem_validation_check(u32 phys_addr, u32 size) +{ +#if 0 + if (phys_addr < (phys_addr + size)) { /* Don't allow overflow (or zero size) */ + if ((0 == (phys_addr & (~_MALI_OSK_CPU_PAGE_MASK))) && + (0 == (size & (~_MALI_OSK_CPU_PAGE_MASK)))) { + if ((phys_addr >= mali_mem_validator.phys_base) && + ((phys_addr + (size - 1)) >= mali_mem_validator.phys_base) && + (phys_addr <= (mali_mem_validator.phys_base + (mali_mem_validator.size - 1))) && + ((phys_addr + (size - 1)) <= (mali_mem_validator.phys_base + (mali_mem_validator.size - 1)))) { + MALI_DEBUG_PRINT(3, ("Accepted range 0x%08X + size 0x%08X (= 0x%08X)\n", phys_addr, size, (phys_addr + size - 1))); + return _MALI_OSK_ERR_OK; + } + } + } + + MALI_PRINT_ERROR(("MALI PHYSICAL RANGE VALIDATION ERROR: The range supplied was: phys_base=0x%08X, size=0x%08X\n", phys_addr, size)); + + return _MALI_OSK_ERR_FAULT; +#endif + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h new file mode 100755 index 000000000000..05013f46f901 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_mem_validation.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2011-2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEM_VALIDATION_H__ +#define __MALI_MEM_VALIDATION_H__ + +#include "mali_osk.h" + +_mali_osk_errcode_t mali_mem_validation_add_range(u32 start, u32 size); +_mali_osk_errcode_t mali_mem_validation_check(u32 phys_addr, u32 size); + +#endif /* __MALI_MEM_VALIDATION_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu.c b/drivers/gpu/arm/mali400/mali/common/mali_mmu.c new file mode 100755 index 000000000000..b82486fa66c0 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu.c @@ -0,0 +1,433 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_ukk.h" + +#include "mali_mmu.h" +#include "mali_hw_core.h" +#include "mali_group.h" +#include "mali_mmu_page_directory.h" + +/** + * Size of the MMU registers in bytes + */ +#define MALI_MMU_REGISTERS_SIZE 0x24 + +/** + * MMU commands + * These are the commands that can be sent + * to the MMU unit. + */ +typedef enum mali_mmu_command { + MALI_MMU_COMMAND_ENABLE_PAGING = 0x00, /**< Enable paging (memory translation) */ + MALI_MMU_COMMAND_DISABLE_PAGING = 0x01, /**< Disable paging (memory translation) */ + MALI_MMU_COMMAND_ENABLE_STALL = 0x02, /**< Enable stall on page fault */ + MALI_MMU_COMMAND_DISABLE_STALL = 0x03, /**< Disable stall on page fault */ + MALI_MMU_COMMAND_ZAP_CACHE = 0x04, /**< Zap the entire page table cache */ + MALI_MMU_COMMAND_PAGE_FAULT_DONE = 0x05, /**< Page fault processed */ + MALI_MMU_COMMAND_HARD_RESET = 0x06 /**< Reset the MMU back to power-on settings */ +} mali_mmu_command; + +static void mali_mmu_probe_trigger(void *data); +static _mali_osk_errcode_t mali_mmu_probe_ack(void *data); + +MALI_STATIC_INLINE _mali_osk_errcode_t mali_mmu_raw_reset(struct mali_mmu_core *mmu); + +/* page fault queue flush helper pages + * note that the mapping pointers are currently unused outside of the initialization functions */ +static mali_dma_addr mali_page_fault_flush_page_directory = MALI_INVALID_PAGE; +static mali_io_address mali_page_fault_flush_page_directory_mapping = NULL; +static mali_dma_addr mali_page_fault_flush_page_table = MALI_INVALID_PAGE; +static mali_io_address mali_page_fault_flush_page_table_mapping = NULL; +static mali_dma_addr mali_page_fault_flush_data_page = MALI_INVALID_PAGE; +static mali_io_address mali_page_fault_flush_data_page_mapping = NULL; + +/* an empty page directory (no address valid) which is active on any MMU not currently marked as in use */ +static mali_dma_addr mali_empty_page_directory_phys = MALI_INVALID_PAGE; +static mali_io_address mali_empty_page_directory_virt = NULL; + + +_mali_osk_errcode_t mali_mmu_initialize(void) +{ + /* allocate the helper pages */ + mali_empty_page_directory_phys = mali_allocate_empty_page(&mali_empty_page_directory_virt); + if (0 == mali_empty_page_directory_phys) { + MALI_DEBUG_PRINT_ERROR(("Mali MMU: Could not allocate empty page directory.\n")); + mali_empty_page_directory_phys = MALI_INVALID_PAGE; + return _MALI_OSK_ERR_NOMEM; + } + + if (_MALI_OSK_ERR_OK != mali_create_fault_flush_pages(&mali_page_fault_flush_page_directory, + &mali_page_fault_flush_page_directory_mapping, + &mali_page_fault_flush_page_table, + &mali_page_fault_flush_page_table_mapping, + &mali_page_fault_flush_data_page, + &mali_page_fault_flush_data_page_mapping)) { + MALI_DEBUG_PRINT_ERROR(("Mali MMU: Could not allocate fault flush pages\n")); + mali_free_empty_page(mali_empty_page_directory_phys, mali_empty_page_directory_virt); + mali_empty_page_directory_phys = MALI_INVALID_PAGE; + mali_empty_page_directory_virt = NULL; + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_mmu_terminate(void) +{ + MALI_DEBUG_PRINT(3, ("Mali MMU: terminating\n")); + + /* Free global helper pages */ + mali_free_empty_page(mali_empty_page_directory_phys, mali_empty_page_directory_virt); + mali_empty_page_directory_phys = MALI_INVALID_PAGE; + mali_empty_page_directory_virt = NULL; + + /* Free the page fault flush pages */ + mali_destroy_fault_flush_pages(&mali_page_fault_flush_page_directory, + &mali_page_fault_flush_page_directory_mapping, + &mali_page_fault_flush_page_table, + &mali_page_fault_flush_page_table_mapping, + &mali_page_fault_flush_data_page, + &mali_page_fault_flush_data_page_mapping); +} + +struct mali_mmu_core *mali_mmu_create(_mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual) +{ + struct mali_mmu_core *mmu = NULL; + + MALI_DEBUG_ASSERT_POINTER(resource); + + MALI_DEBUG_PRINT(2, ("Mali MMU: Creating Mali MMU: %s\n", resource->description)); + + mmu = _mali_osk_calloc(1, sizeof(struct mali_mmu_core)); + if (NULL != mmu) { + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&mmu->hw_core, resource, MALI_MMU_REGISTERS_SIZE)) { + if (_MALI_OSK_ERR_OK == mali_group_add_mmu_core(group, mmu)) { + if (is_virtual) { + /* Skip reset and IRQ setup for virtual MMU */ + return mmu; + } + + if (_MALI_OSK_ERR_OK == mali_mmu_reset(mmu)) { + /* Setup IRQ handlers (which will do IRQ probing if needed) */ + mmu->irq = _mali_osk_irq_init(resource->irq, + mali_group_upper_half_mmu, + group, + mali_mmu_probe_trigger, + mali_mmu_probe_ack, + mmu, + resource->description); + if (NULL != mmu->irq) { + return mmu; + } else { + MALI_PRINT_ERROR(("Mali MMU: Failed to setup interrupt handlers for MMU %s\n", mmu->hw_core.description)); + } + } + mali_group_remove_mmu_core(group); + } else { + MALI_PRINT_ERROR(("Mali MMU: Failed to add core %s to group\n", mmu->hw_core.description)); + } + mali_hw_core_delete(&mmu->hw_core); + } + + _mali_osk_free(mmu); + } else { + MALI_PRINT_ERROR(("Failed to allocate memory for MMU\n")); + } + + return NULL; +} + +void mali_mmu_delete(struct mali_mmu_core *mmu) +{ + if (NULL != mmu->irq) { + _mali_osk_irq_term(mmu->irq); + } + + mali_hw_core_delete(&mmu->hw_core); + _mali_osk_free(mmu); +} + +static void mali_mmu_enable_paging(struct mali_mmu_core *mmu) +{ + int i; + + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_PAGING); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { + if (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS) & MALI_MMU_STATUS_BIT_PAGING_ENABLED) { + break; + } + } + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_PRINT_ERROR(("Enable paging request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); + } +} + +/** + * Issues the enable stall command to the MMU and waits for HW to complete the request + * @param mmu The MMU to enable paging for + * @return MALI_TRUE if HW stall was successfully engaged, otherwise MALI_FALSE (req timed out) + */ +static mali_bool mali_mmu_enable_stall(struct mali_mmu_core *mmu) +{ + int i; + u32 mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); + + if (0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED)) { + MALI_DEBUG_PRINT(4, ("MMU stall is implicit when Paging is not enabled.\n")); + return MALI_TRUE; + } + + if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { + MALI_DEBUG_PRINT(3, ("Aborting MMU stall request since it is in pagefault state.\n")); + return MALI_FALSE; + } + + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ENABLE_STALL); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { + mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); + if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { + break; + } + if ((mmu_status & MALI_MMU_STATUS_BIT_STALL_ACTIVE) && (0 == (mmu_status & MALI_MMU_STATUS_BIT_STALL_NOT_ACTIVE))) { + break; + } + if (0 == (mmu_status & (MALI_MMU_STATUS_BIT_PAGING_ENABLED))) { + break; + } + } + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_DEBUG_PRINT(2, ("Enable stall request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); + return MALI_FALSE; + } + + if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { + MALI_DEBUG_PRINT(2, ("Aborting MMU stall request since it has a pagefault.\n")); + return MALI_FALSE; + } + + return MALI_TRUE; +} + +/** + * Issues the disable stall command to the MMU and waits for HW to complete the request + * @param mmu The MMU to enable paging for + */ +static void mali_mmu_disable_stall(struct mali_mmu_core *mmu) +{ + int i; + u32 mmu_status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); + + if (0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED)) { + MALI_DEBUG_PRINT(3, ("MMU disable skipped since it was not enabled.\n")); + return; + } + if (mmu_status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { + MALI_DEBUG_PRINT(2, ("Aborting MMU disable stall request since it is in pagefault state.\n")); + return; + } + + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_DISABLE_STALL); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { + u32 status = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); + if (0 == (status & MALI_MMU_STATUS_BIT_STALL_ACTIVE)) { + break; + } + if (status & MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE) { + break; + } + if (0 == (mmu_status & MALI_MMU_STATUS_BIT_PAGING_ENABLED)) { + break; + } + } + if (MALI_REG_POLL_COUNT_FAST == i) MALI_DEBUG_PRINT(1, ("Disable stall request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); +} + +void mali_mmu_page_fault_done(struct mali_mmu_core *mmu) +{ + MALI_DEBUG_PRINT(4, ("Mali MMU: %s: Leaving page fault mode\n", mmu->hw_core.description)); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_PAGE_FAULT_DONE); +} + +MALI_STATIC_INLINE _mali_osk_errcode_t mali_mmu_raw_reset(struct mali_mmu_core *mmu) +{ + int i; + + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, 0xCAFEBABE); + MALI_DEBUG_ASSERT(0xCAFEB000 == mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR)); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_HARD_RESET); + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; ++i) { + if (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR) == 0) { + break; + } + } + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_PRINT_ERROR(("Reset request failed, MMU status is 0x%08X\n", mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); + return _MALI_OSK_ERR_FAULT; + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_mmu_reset(struct mali_mmu_core *mmu) +{ + _mali_osk_errcode_t err = _MALI_OSK_ERR_FAULT; + mali_bool stall_success; + MALI_DEBUG_ASSERT_POINTER(mmu); + + stall_success = mali_mmu_enable_stall(mmu); + if (!stall_success) { + err = _MALI_OSK_ERR_BUSY; + } + + MALI_DEBUG_PRINT(3, ("Mali MMU: mali_kernel_mmu_reset: %s\n", mmu->hw_core.description)); + + if (_MALI_OSK_ERR_OK == mali_mmu_raw_reset(mmu)) { + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_MASK, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR); + /* no session is active, so just activate the empty page directory */ + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, mali_empty_page_directory_phys); + mali_mmu_enable_paging(mmu); + err = _MALI_OSK_ERR_OK; + } + mali_mmu_disable_stall(mmu); + + return err; +} + +mali_bool mali_mmu_zap_tlb(struct mali_mmu_core *mmu) +{ + mali_bool stall_success = mali_mmu_enable_stall(mmu); + + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); + + if (MALI_FALSE == stall_success) { + /* False means that it is in Pagefault state. Not possible to disable_stall then */ + return MALI_FALSE; + } + + mali_mmu_disable_stall(mmu); + return MALI_TRUE; +} + +void mali_mmu_zap_tlb_without_stall(struct mali_mmu_core *mmu) +{ + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); +} + + +void mali_mmu_invalidate_page(struct mali_mmu_core *mmu, u32 mali_address) +{ + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_ZAP_ONE_LINE, MALI_MMU_PDE_ENTRY(mali_address)); +} + +static void mali_mmu_activate_address_space(struct mali_mmu_core *mmu, u32 page_directory) +{ + /* The MMU must be in stalled or page fault mode, for this writing to work */ + MALI_DEBUG_ASSERT(0 != (mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS) + & (MALI_MMU_STATUS_BIT_STALL_ACTIVE | MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE))); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_DTE_ADDR, page_directory); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_COMMAND, MALI_MMU_COMMAND_ZAP_CACHE); + +} + +void mali_mmu_activate_page_directory(struct mali_mmu_core *mmu, struct mali_page_directory *pagedir) +{ + mali_bool stall_success; + MALI_DEBUG_ASSERT_POINTER(mmu); + + MALI_DEBUG_PRINT(5, ("Asked to activate page directory 0x%x on MMU %s\n", pagedir, mmu->hw_core.description)); + + stall_success = mali_mmu_enable_stall(mmu); + MALI_DEBUG_ASSERT(stall_success); + MALI_IGNORE(stall_success); + mali_mmu_activate_address_space(mmu, pagedir->page_directory); + mali_mmu_disable_stall(mmu); +} + +void mali_mmu_activate_empty_page_directory(struct mali_mmu_core *mmu) +{ + mali_bool stall_success; + + MALI_DEBUG_ASSERT_POINTER(mmu); + MALI_DEBUG_PRINT(3, ("Activating the empty page directory on MMU %s\n", mmu->hw_core.description)); + + stall_success = mali_mmu_enable_stall(mmu); + + /* This function can only be called when the core is idle, so it could not fail. */ + MALI_DEBUG_ASSERT(stall_success); + MALI_IGNORE(stall_success); + + mali_mmu_activate_address_space(mmu, mali_empty_page_directory_phys); + mali_mmu_disable_stall(mmu); +} + +void mali_mmu_activate_fault_flush_page_directory(struct mali_mmu_core *mmu) +{ + mali_bool stall_success; + MALI_DEBUG_ASSERT_POINTER(mmu); + + MALI_DEBUG_PRINT(3, ("Activating the page fault flush page directory on MMU %s\n", mmu->hw_core.description)); + stall_success = mali_mmu_enable_stall(mmu); + /* This function is expect to fail the stalling, since it might be in PageFault mode when it is called */ + mali_mmu_activate_address_space(mmu, mali_page_fault_flush_page_directory); + if (MALI_TRUE == stall_success) mali_mmu_disable_stall(mmu); +} + +/* Is called when we want the mmu to give an interrupt */ +static void mali_mmu_probe_trigger(void *data) +{ + struct mali_mmu_core *mmu = (struct mali_mmu_core *)data; + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_RAWSTAT, MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR); +} + +/* Is called when the irq probe wants the mmu to acknowledge an interrupt from the hw */ +static _mali_osk_errcode_t mali_mmu_probe_ack(void *data) +{ + struct mali_mmu_core *mmu = (struct mali_mmu_core *)data; + u32 int_stat; + + int_stat = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_STATUS); + + MALI_DEBUG_PRINT(2, ("mali_mmu_probe_irq_acknowledge: intstat 0x%x\n", int_stat)); + if (int_stat & MALI_MMU_INTERRUPT_PAGE_FAULT) { + MALI_DEBUG_PRINT(2, ("Probe: Page fault detect: PASSED\n")); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_CLEAR, MALI_MMU_INTERRUPT_PAGE_FAULT); + } else { + MALI_DEBUG_PRINT(1, ("Probe: Page fault detect: FAILED\n")); + } + + if (int_stat & MALI_MMU_INTERRUPT_READ_BUS_ERROR) { + MALI_DEBUG_PRINT(2, ("Probe: Bus read error detect: PASSED\n")); + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_CLEAR, MALI_MMU_INTERRUPT_READ_BUS_ERROR); + } else { + MALI_DEBUG_PRINT(1, ("Probe: Bus read error detect: FAILED\n")); + } + + if ((int_stat & (MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR)) == + (MALI_MMU_INTERRUPT_PAGE_FAULT | MALI_MMU_INTERRUPT_READ_BUS_ERROR)) { + return _MALI_OSK_ERR_OK; + } + + return _MALI_OSK_ERR_FAULT; +} + +#if 0 +void mali_mmu_print_state(struct mali_mmu_core *mmu) +{ + MALI_DEBUG_PRINT(2, ("MMU: State of %s is 0x%08x\n", mmu->hw_core.description, mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS))); +} +#endif diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu.h b/drivers/gpu/arm/mali400/mali/common/mali_mmu.h new file mode 100755 index 000000000000..6ed48585f3d2 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu.h @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MMU_H__ +#define __MALI_MMU_H__ + +#include "mali_osk.h" +#include "mali_mmu_page_directory.h" +#include "mali_hw_core.h" + +/* Forward declaration from mali_group.h */ +struct mali_group; + +/** + * MMU register numbers + * Used in the register read/write routines. + * See the hardware documentation for more information about each register + */ +typedef enum mali_mmu_register { + MALI_MMU_REGISTER_DTE_ADDR = 0x0000, /**< Current Page Directory Pointer */ + MALI_MMU_REGISTER_STATUS = 0x0004, /**< Status of the MMU */ + MALI_MMU_REGISTER_COMMAND = 0x0008, /**< Command register, used to control the MMU */ + MALI_MMU_REGISTER_PAGE_FAULT_ADDR = 0x000C, /**< Logical address of the last page fault */ + MALI_MMU_REGISTER_ZAP_ONE_LINE = 0x010, /**< Used to invalidate the mapping of a single page from the MMU */ + MALI_MMU_REGISTER_INT_RAWSTAT = 0x0014, /**< Raw interrupt status, all interrupts visible */ + MALI_MMU_REGISTER_INT_CLEAR = 0x0018, /**< Indicate to the MMU that the interrupt has been received */ + MALI_MMU_REGISTER_INT_MASK = 0x001C, /**< Enable/disable types of interrupts */ + MALI_MMU_REGISTER_INT_STATUS = 0x0020 /**< Interrupt status based on the mask */ +} mali_mmu_register; + +/** + * MMU interrupt register bits + * Each cause of the interrupt is reported + * through the (raw) interrupt status registers. + * Multiple interrupts can be pending, so multiple bits + * can be set at once. + */ +typedef enum mali_mmu_interrupt { + MALI_MMU_INTERRUPT_PAGE_FAULT = 0x01, /**< A page fault occured */ + MALI_MMU_INTERRUPT_READ_BUS_ERROR = 0x02 /**< A bus read error occured */ +} mali_mmu_interrupt; + +typedef enum mali_mmu_status_bits { + MALI_MMU_STATUS_BIT_PAGING_ENABLED = 1 << 0, + MALI_MMU_STATUS_BIT_PAGE_FAULT_ACTIVE = 1 << 1, + MALI_MMU_STATUS_BIT_STALL_ACTIVE = 1 << 2, + MALI_MMU_STATUS_BIT_IDLE = 1 << 3, + MALI_MMU_STATUS_BIT_REPLAY_BUFFER_EMPTY = 1 << 4, + MALI_MMU_STATUS_BIT_PAGE_FAULT_IS_WRITE = 1 << 5, + MALI_MMU_STATUS_BIT_STALL_NOT_ACTIVE = 1 << 31, +} mali_mmu_status_bits; + +/** + * Definition of the MMU struct + * Used to track a MMU unit in the system. + * Contains information about the mapping of the registers + */ +struct mali_mmu_core { + struct mali_hw_core hw_core; /**< Common for all HW cores */ + _mali_osk_irq_t *irq; /**< IRQ handler */ +}; + +_mali_osk_errcode_t mali_mmu_initialize(void); + +void mali_mmu_terminate(void); + +struct mali_mmu_core *mali_mmu_create(_mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual); +void mali_mmu_delete(struct mali_mmu_core *mmu); + +_mali_osk_errcode_t mali_mmu_reset(struct mali_mmu_core *mmu); +mali_bool mali_mmu_zap_tlb(struct mali_mmu_core *mmu); +void mali_mmu_zap_tlb_without_stall(struct mali_mmu_core *mmu); +void mali_mmu_invalidate_page(struct mali_mmu_core *mmu, u32 mali_address); + +void mali_mmu_activate_page_directory(struct mali_mmu_core *mmu, struct mali_page_directory *pagedir); +void mali_mmu_activate_empty_page_directory(struct mali_mmu_core *mmu); +void mali_mmu_activate_fault_flush_page_directory(struct mali_mmu_core *mmu); + +void mali_mmu_page_fault_done(struct mali_mmu_core *mmu); + +MALI_STATIC_INLINE enum mali_interrupt_result mali_mmu_get_interrupt_result(struct mali_mmu_core *mmu) +{ + u32 rawstat_used = mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_RAWSTAT); + if (0 == rawstat_used) { + return MALI_INTERRUPT_RESULT_NONE; + } + + return MALI_INTERRUPT_RESULT_ERROR; +} + + +MALI_STATIC_INLINE u32 mali_mmu_get_int_status(struct mali_mmu_core *mmu) +{ + return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_STATUS); +} + +MALI_STATIC_INLINE u32 mali_mmu_get_rawstat(struct mali_mmu_core *mmu) +{ + return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_INT_RAWSTAT); +} + +MALI_STATIC_INLINE void mali_mmu_mask_all_interrupts(struct mali_mmu_core *mmu) +{ + mali_hw_core_register_write(&mmu->hw_core, MALI_MMU_REGISTER_INT_MASK, 0); +} + +MALI_STATIC_INLINE u32 mali_mmu_get_status(struct mali_mmu_core *mmu) +{ + return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_STATUS); +} + +MALI_STATIC_INLINE u32 mali_mmu_get_page_fault_addr(struct mali_mmu_core *mmu) +{ + return mali_hw_core_register_read(&mmu->hw_core, MALI_MMU_REGISTER_PAGE_FAULT_ADDR); +} + +#endif /* __MALI_MMU_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c new file mode 100755 index 000000000000..9ad3e8970b7d --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.c @@ -0,0 +1,495 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" +#include "mali_uk_types.h" +#include "mali_mmu_page_directory.h" +#include "mali_memory.h" +#include "mali_l2_cache.h" + +static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data); + +u32 mali_allocate_empty_page(mali_io_address *virt_addr) +{ + _mali_osk_errcode_t err; + mali_io_address mapping; + mali_dma_addr address; + + if (_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping)) { + /* Allocation failed */ + MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to get table page for empty pgdir\n")); + return 0; + } + + MALI_DEBUG_ASSERT_POINTER(mapping); + + err = fill_page(mapping, 0); + if (_MALI_OSK_ERR_OK != err) { + mali_mmu_release_table_page(address, mapping); + MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to zero page\n")); + return 0; + } + + *virt_addr = mapping; + return address; +} + +void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr) +{ + if (MALI_INVALID_PAGE != address) { + mali_mmu_release_table_page(address, virt_addr); + } +} + +_mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory, + mali_io_address *page_directory_mapping, + mali_dma_addr *page_table, mali_io_address *page_table_mapping, + mali_dma_addr *data_page, mali_io_address *data_page_mapping) +{ + _mali_osk_errcode_t err; + + err = mali_mmu_get_table_page(data_page, data_page_mapping); + if (_MALI_OSK_ERR_OK == err) { + err = mali_mmu_get_table_page(page_table, page_table_mapping); + if (_MALI_OSK_ERR_OK == err) { + err = mali_mmu_get_table_page(page_directory, page_directory_mapping); + if (_MALI_OSK_ERR_OK == err) { + fill_page(*data_page_mapping, 0); + fill_page(*page_table_mapping, *data_page | MALI_MMU_FLAGS_DEFAULT); + fill_page(*page_directory_mapping, *page_table | MALI_MMU_FLAGS_PRESENT); + MALI_SUCCESS; + } + mali_mmu_release_table_page(*page_table, *page_table_mapping); + *page_table = MALI_INVALID_PAGE; + } + mali_mmu_release_table_page(*data_page, *data_page_mapping); + *data_page = MALI_INVALID_PAGE; + } + return err; +} + +void mali_destroy_fault_flush_pages( + mali_dma_addr *page_directory, mali_io_address *page_directory_mapping, + mali_dma_addr *page_table, mali_io_address *page_table_mapping, + mali_dma_addr *data_page, mali_io_address *data_page_mapping) +{ + if (MALI_INVALID_PAGE != *page_directory) { + mali_mmu_release_table_page(*page_directory, *page_directory_mapping); + *page_directory = MALI_INVALID_PAGE; + *page_directory_mapping = NULL; + } + + if (MALI_INVALID_PAGE != *page_table) { + mali_mmu_release_table_page(*page_table, *page_table_mapping); + *page_table = MALI_INVALID_PAGE; + *page_table_mapping = NULL; + } + + if (MALI_INVALID_PAGE != *data_page) { + mali_mmu_release_table_page(*data_page, *data_page_mapping); + *data_page = MALI_INVALID_PAGE; + *data_page_mapping = NULL; + } +} + +static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data) +{ + int i; + MALI_DEBUG_ASSERT_POINTER(mapping); + + for (i = 0; i < MALI_MMU_PAGE_SIZE / 4; i++) { + _mali_osk_mem_iowrite32_relaxed(mapping, i * sizeof(u32), data); + } + _mali_osk_mem_barrier(); + MALI_SUCCESS; +} + +_mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size) +{ + const int first_pde = MALI_MMU_PDE_ENTRY(mali_address); + const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1); + _mali_osk_errcode_t err; + mali_io_address pde_mapping; + mali_dma_addr pde_phys; + int i, page_count; + u32 start_address; + if (last_pde < first_pde) + return _MALI_OSK_ERR_INVALID_ARGS; + + for (i = first_pde; i <= last_pde; i++) { + if (0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, + i * sizeof(u32)) & MALI_MMU_FLAGS_PRESENT)) { + /* Page table not present */ + MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]); + MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]); + + err = mali_mmu_get_table_page(&pde_phys, &pde_mapping); + if (_MALI_OSK_ERR_OK != err) { + MALI_PRINT_ERROR(("Failed to allocate page table page.\n")); + return err; + } + pagedir->page_entries_mapped[i] = pde_mapping; + + /* Update PDE, mark as present */ + _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), + pde_phys | MALI_MMU_FLAGS_PRESENT); + + MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]); + } + + if (first_pde == last_pde) { + pagedir->page_entries_usage_count[i] += size / MALI_MMU_PAGE_SIZE; + } else if (i == first_pde) { + start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE; + page_count = (start_address + MALI_MMU_VIRTUAL_PAGE_SIZE - mali_address) / MALI_MMU_PAGE_SIZE; + pagedir->page_entries_usage_count[i] += page_count; + } else if (i == last_pde) { + start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE; + page_count = (mali_address + size - start_address) / MALI_MMU_PAGE_SIZE; + pagedir->page_entries_usage_count[i] += page_count; + } else { + pagedir->page_entries_usage_count[i] = 1024; + } + } + _mali_osk_write_mem_barrier(); + + return _MALI_OSK_ERR_OK; +} + +MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size) +{ + int i; + const int first_pte = MALI_MMU_PTE_ENTRY(mali_address); + const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1); + + for (i = first_pte; i <= last_pte; i++) { + _mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0); + } +} + +static u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index) +{ + return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped, + index * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK); +} + + +_mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size) +{ + const int first_pde = MALI_MMU_PDE_ENTRY(mali_address); + const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1); + u32 left = size; + int i; + mali_bool pd_changed = MALI_FALSE; + u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */ + u32 num_pages_inv = 0; + mali_bool invalidate_all = MALI_FALSE; /* safety mechanism in case page_entries_usage_count is unreliable */ + + /* For all page directory entries in range. */ + for (i = first_pde; i <= last_pde; i++) { + u32 size_in_pde, offset; + + MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]); + MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]); + + /* Offset into page table, 0 if mali_address is 4MiB aligned */ + offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1)); + if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset) { + size_in_pde = left; + } else { + size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset; + } + + pagedir->page_entries_usage_count[i] -= size_in_pde / MALI_MMU_PAGE_SIZE; + + /* If entire page table is unused, free it */ + if (0 == pagedir->page_entries_usage_count[i]) { + u32 page_phys; + void *page_virt; + MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n")); + /* last reference removed, no need to zero out each PTE */ + + page_phys = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i * sizeof(u32))); + page_virt = pagedir->page_entries_mapped[i]; + pagedir->page_entries_mapped[i] = NULL; + _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0); + + mali_mmu_release_table_page(page_phys, page_virt); + pd_changed = MALI_TRUE; + } else { + MALI_DEBUG_ASSERT(num_pages_inv < 2); + if (num_pages_inv < 2) { + pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i); + num_pages_inv++; + } else { + invalidate_all = MALI_TRUE; + } + + /* If part of the page table is still in use, zero the relevant PTEs */ + mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde); + } + + left -= size_in_pde; + mali_address += size_in_pde; + } + _mali_osk_write_mem_barrier(); + + /* L2 pages invalidation */ + if (MALI_TRUE == pd_changed) { + MALI_DEBUG_ASSERT(num_pages_inv < 3); + if (num_pages_inv < 3) { + pages_to_invalidate[num_pages_inv] = pagedir->page_directory; + num_pages_inv++; + } else { + invalidate_all = MALI_TRUE; + } + } + + if (invalidate_all) { + mali_l2_cache_invalidate_all(); + } else { + mali_l2_cache_invalidate_all_pages(pages_to_invalidate, num_pages_inv); + } + + MALI_SUCCESS; +} + +struct mali_page_directory *mali_mmu_pagedir_alloc(void) +{ + struct mali_page_directory *pagedir; + _mali_osk_errcode_t err; + mali_dma_addr phys; + + pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory)); + if (NULL == pagedir) { + return NULL; + } + + err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped); + if (_MALI_OSK_ERR_OK != err) { + _mali_osk_free(pagedir); + return NULL; + } + + pagedir->page_directory = (u32)phys; + + /* Zero page directory */ + fill_page(pagedir->page_directory_mapped, 0); + + return pagedir; +} + +void mali_mmu_pagedir_free(struct mali_page_directory *pagedir) +{ + const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]); + int i; + + /* Free referenced page tables and zero PDEs. */ + for (i = 0; i < num_page_table_entries; i++) { + if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32( + pagedir->page_directory_mapped, + sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT)) { + mali_dma_addr phys = _mali_osk_mem_ioread32(pagedir->page_directory_mapped, + i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK; + _mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0); + mali_mmu_release_table_page(phys, pagedir->page_entries_mapped[i]); + } + } + _mali_osk_write_mem_barrier(); + + /* Free the page directory page. */ + mali_mmu_release_table_page(pagedir->page_directory, pagedir->page_directory_mapped); + + _mali_osk_free(pagedir); +} + + +void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address, + mali_dma_addr phys_address, u32 size, u32 permission_bits) +{ + u32 end_address = mali_address + size; + u32 mali_phys = (u32)phys_address; + + /* Map physical pages into MMU page tables */ + for (; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, mali_phys += MALI_MMU_PAGE_SIZE) { + MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]); + _mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)], + MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32), + mali_phys | permission_bits); + } +} + +void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr) +{ +#if defined(DEBUG) + u32 pde_index, pte_index; + u32 pde, pte; + + pde_index = MALI_MMU_PDE_ENTRY(fault_addr); + pte_index = MALI_MMU_PTE_ENTRY(fault_addr); + + + pde = _mali_osk_mem_ioread32(pagedir->page_directory_mapped, + pde_index * sizeof(u32)); + + + if (pde & MALI_MMU_FLAGS_PRESENT) { + u32 pte_addr = MALI_MMU_ENTRY_ADDRESS(pde); + + pte = _mali_osk_mem_ioread32(pagedir->page_entries_mapped[pde_index], + pte_index * sizeof(u32)); + + MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table present: %08x\n" + "\t\tPTE: %08x, page %08x is %s\n", + fault_addr, pte_addr, pte, + MALI_MMU_ENTRY_ADDRESS(pte), + pte & MALI_MMU_FLAGS_DEFAULT ? "rw" : "not present")); + } else { + MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table not present: %08x\n", + fault_addr, pde)); + } +#else + MALI_IGNORE(pagedir); + MALI_IGNORE(fault_addr); +#endif +} + +/* For instrumented */ +struct dump_info { + u32 buffer_left; + u32 register_writes_size; + u32 page_table_dump_size; + u32 *buffer; +}; + +static _mali_osk_errcode_t writereg(u32 where, u32 what, const char *comment, struct dump_info *info) +{ + if (NULL != info) { + info->register_writes_size += sizeof(u32) * 2; /* two 32-bit words */ + + if (NULL != info->buffer) { + /* check that we have enough space */ + if (info->buffer_left < sizeof(u32) * 2) MALI_ERROR(_MALI_OSK_ERR_NOMEM); + + *info->buffer = where; + info->buffer++; + + *info->buffer = what; + info->buffer++; + + info->buffer_left -= sizeof(u32) * 2; + } + } + + MALI_SUCCESS; +} + +static _mali_osk_errcode_t mali_mmu_dump_page(mali_io_address page, u32 phys_addr, struct dump_info *info) +{ + if (NULL != info) { + /* 4096 for the page and 4 bytes for the address */ + const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4; + const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE; + const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4; + + info->page_table_dump_size += dump_size_in_bytes; + + if (NULL != info->buffer) { + if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM); + + *info->buffer = phys_addr; + info->buffer++; + + _mali_osk_memcpy(info->buffer, page, page_size_in_bytes); + info->buffer += page_size_in_elements; + + info->buffer_left -= dump_size_in_bytes; + } + } + + MALI_SUCCESS; +} + +static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory *pagedir, struct dump_info *info) +{ + MALI_DEBUG_ASSERT_POINTER(pagedir); + MALI_DEBUG_ASSERT_POINTER(info); + + if (NULL != pagedir->page_directory_mapped) { + int i; + + MALI_CHECK_NO_ERROR( + mali_mmu_dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info) + ); + + for (i = 0; i < 1024; i++) { + if (NULL != pagedir->page_entries_mapped[i]) { + MALI_CHECK_NO_ERROR( + mali_mmu_dump_page(pagedir->page_entries_mapped[i], + _mali_osk_mem_ioread32(pagedir->page_directory_mapped, + i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info) + ); + } + } + } + + MALI_SUCCESS; +} + +static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory *pagedir, struct dump_info *info) +{ + MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory, + "set the page directory address", info)); + MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info)); + MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info)); + MALI_SUCCESS; +} + +_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args) +{ + struct dump_info info = { 0, 0, 0, NULL }; + struct mali_session_data *session_data; + + session_data = (struct mali_session_data *)(uintptr_t)(args->ctx); + MALI_DEBUG_ASSERT_POINTER(session_data); + MALI_DEBUG_ASSERT_POINTER(args); + + MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info)); + MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info)); + args->size = info.register_writes_size + info.page_table_dump_size; + MALI_SUCCESS; +} + +_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args) +{ + struct dump_info info = { 0, 0, 0, NULL }; + struct mali_session_data *session_data; + + MALI_DEBUG_ASSERT_POINTER(args); + + session_data = (struct mali_session_data *)(uintptr_t)(args->ctx); + MALI_DEBUG_ASSERT_POINTER(session_data); + + info.buffer_left = args->size; + info.buffer = (u32 *)(uintptr_t)args->buffer; + + args->register_writes = (uintptr_t)info.buffer; + MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info)); + + args->page_table_dump = (uintptr_t)info.buffer; + MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info)); + + args->register_writes_size = info.register_writes_size; + args->page_table_dump_size = info.page_table_dump_size; + + MALI_SUCCESS; +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h new file mode 100755 index 000000000000..3fdf07210259 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_mmu_page_directory.h @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MMU_PAGE_DIRECTORY_H__ +#define __MALI_MMU_PAGE_DIRECTORY_H__ + +#include "mali_osk.h" + +/** + * Size of an MMU page in bytes + */ +#define MALI_MMU_PAGE_SIZE 0x1000 + +/* + * Size of the address space referenced by a page table page + */ +#define MALI_MMU_VIRTUAL_PAGE_SIZE 0x400000 /* 4 MiB */ + +/** + * Page directory index from address + * Calculates the page directory index from the given address + */ +#define MALI_MMU_PDE_ENTRY(address) (((address)>>22) & 0x03FF) + +/** + * Page table index from address + * Calculates the page table index from the given address + */ +#define MALI_MMU_PTE_ENTRY(address) (((address)>>12) & 0x03FF) + +/** + * Extract the memory address from an PDE/PTE entry + */ +#define MALI_MMU_ENTRY_ADDRESS(value) ((value) & 0xFFFFFC00) + +#define MALI_INVALID_PAGE ((u32)(~0)) + +/** + * + */ +typedef enum mali_mmu_entry_flags { + MALI_MMU_FLAGS_PRESENT = 0x01, + MALI_MMU_FLAGS_READ_PERMISSION = 0x02, + MALI_MMU_FLAGS_WRITE_PERMISSION = 0x04, + MALI_MMU_FLAGS_OVERRIDE_CACHE = 0x8, + MALI_MMU_FLAGS_WRITE_CACHEABLE = 0x10, + MALI_MMU_FLAGS_WRITE_ALLOCATE = 0x20, + MALI_MMU_FLAGS_WRITE_BUFFERABLE = 0x40, + MALI_MMU_FLAGS_READ_CACHEABLE = 0x80, + MALI_MMU_FLAGS_READ_ALLOCATE = 0x100, + MALI_MMU_FLAGS_MASK = 0x1FF, +} mali_mmu_entry_flags; + + +#define MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE ( \ + MALI_MMU_FLAGS_PRESENT | \ + MALI_MMU_FLAGS_READ_PERMISSION | \ + MALI_MMU_FLAGS_WRITE_PERMISSION | \ + MALI_MMU_FLAGS_OVERRIDE_CACHE | \ + MALI_MMU_FLAGS_WRITE_CACHEABLE | \ + MALI_MMU_FLAGS_WRITE_BUFFERABLE | \ + MALI_MMU_FLAGS_READ_CACHEABLE | \ + MALI_MMU_FLAGS_READ_ALLOCATE ) + +#define MALI_MMU_FLAGS_DEFAULT ( \ + MALI_MMU_FLAGS_PRESENT | \ + MALI_MMU_FLAGS_READ_PERMISSION | \ + MALI_MMU_FLAGS_WRITE_PERMISSION ) + + +struct mali_page_directory { + u32 page_directory; /**< Physical address of the memory session's page directory */ + mali_io_address page_directory_mapped; /**< Pointer to the mapped version of the page directory into the kernel's address space */ + + mali_io_address page_entries_mapped[1024]; /**< Pointers to the page tables which exists in the page directory mapped into the kernel's address space */ + u32 page_entries_usage_count[1024]; /**< Tracks usage count of the page table pages, so they can be releases on the last reference */ +}; + +/* Map Mali virtual address space (i.e. ensure page tables exist for the virtual range) */ +_mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size); +_mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size); + +/* Back virtual address space with actual pages. Assumes input is contiguous and 4k aligned. */ +void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address, + mali_dma_addr phys_address, u32 size, u32 permission_bits); + +u32 mali_allocate_empty_page(mali_io_address *virtual); +void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr); +_mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory, + mali_io_address *page_directory_mapping, + mali_dma_addr *page_table, mali_io_address *page_table_mapping, + mali_dma_addr *data_page, mali_io_address *data_page_mapping); +void mali_destroy_fault_flush_pages( + mali_dma_addr *page_directory, mali_io_address *page_directory_mapping, + mali_dma_addr *page_table, mali_io_address *page_table_mapping, + mali_dma_addr *data_page, mali_io_address *data_page_mapping); + +struct mali_page_directory *mali_mmu_pagedir_alloc(void); +void mali_mmu_pagedir_free(struct mali_page_directory *pagedir); + +void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr); + +#endif /* __MALI_MMU_PAGE_DIRECTORY_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk.h b/drivers/gpu/arm/mali400/mali/common/mali_osk.h new file mode 100755 index 000000000000..9ade362d6b28 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk.h @@ -0,0 +1,1389 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk.h + * Defines the OS abstraction layer for the kernel device driver (OSK) + */ + +#ifndef __MALI_OSK_H__ +#define __MALI_OSK_H__ + +#include +#include "mali_osk_types.h" +#include "mali_osk_specific.h" /* include any per-os specifics */ +#include "mali_osk_locks.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @addtogroup uddapi Unified Device Driver (UDD) APIs + * + * @{ + */ + +/** + * @addtogroup oskapi UDD OS Abstraction for Kernel-side (OSK) APIs + * + * @{ + */ + +/** @addtogroup _mali_osk_lock OSK Mutual Exclusion Locks + * @{ */ + +#ifdef DEBUG +/** @brief Macro for asserting that the current thread holds a given lock + */ +#define MALI_DEBUG_ASSERT_LOCK_HELD(l) MALI_DEBUG_ASSERT(_mali_osk_lock_get_owner((_mali_osk_lock_debug_t *)l) == _mali_osk_get_tid()); + +/** @brief returns a lock's owner (thread id) if debugging is enabled + */ +#else +#define MALI_DEBUG_ASSERT_LOCK_HELD(l) do {} while(0) +#endif + +#define _mali_osk_ctxprintf seq_printf + +/** @} */ /* end group _mali_osk_lock */ + +/** @addtogroup _mali_osk_miscellaneous + * @{ */ + +/** @brief Find the containing structure of another structure + * + * This is the reverse of the operation 'offsetof'. This means that the + * following condition is satisfied: + * + * ptr == _MALI_OSK_CONTAINER_OF( &ptr->member, type, member ) + * + * When ptr is of type 'type'. + * + * Its purpose it to recover a larger structure that has wrapped a smaller one. + * + * @note no type or memory checking occurs to ensure that a wrapper structure + * does in fact exist, and that it is being recovered with respect to the + * correct member. + * + * @param ptr the pointer to the member that is contained within the larger + * structure + * @param type the type of the structure that contains the member + * @param member the name of the member in the structure that ptr points to. + * @return a pointer to a \a type object which contains \a member, as pointed + * to by \a ptr. + */ +#define _MALI_OSK_CONTAINER_OF(ptr, type, member) \ + ((type *)( ((char *)ptr) - offsetof(type,member) )) + +/** @addtogroup _mali_osk_wq + * @{ */ + +/** @brief Initialize work queues (for deferred work) + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_wq_init(void); + +/** @brief Terminate work queues (for deferred work) + */ +void _mali_osk_wq_term(void); + +/** @brief Create work in the work queue + * + * Creates a work object which can be scheduled in the work queue. When + * scheduled, \a handler will be called with \a data as the argument. + * + * Refer to \ref _mali_osk_wq_schedule_work() for details on how work + * is scheduled in the queue. + * + * The returned pointer must be freed with \ref _mali_osk_wq_delete_work() + * when no longer needed. + */ +_mali_osk_wq_work_t *_mali_osk_wq_create_work(_mali_osk_wq_work_handler_t handler, void *data); + +/** @brief A high priority version of \a _mali_osk_wq_create_work() + * + * Creates a work object which can be scheduled in the high priority work queue. + * + * This is unfortunately needed to get low latency scheduling of the Mali cores. Normally we would + * schedule the next job in hw_irq or tasklet, but often we can't since we need to synchronously map + * and unmap shared memory when a job is connected to external fences (timelines). And this requires + * taking a mutex. + * + * We do signal a lot of other (low priority) work also as part of the job being finished, and if we + * don't set this Mali scheduling thread as high priority, we see that the CPU scheduler often runs + * random things instead of starting the next GPU job when the GPU is idle. So setting the gpu + * scheduler to high priority does give a visually more responsive system. + * + * Start the high priority work with: \a _mali_osk_wq_schedule_work_high_pri() + */ +_mali_osk_wq_work_t *_mali_osk_wq_create_work_high_pri(_mali_osk_wq_work_handler_t handler, void *data); + +/** @brief Delete a work object + * + * This will flush the work queue to ensure that the work handler will not + * be called after deletion. + */ +void _mali_osk_wq_delete_work(_mali_osk_wq_work_t *work); + +/** @brief Delete a work object + * + * This will NOT flush the work queue, so only call this if you are sure that the work handler will + * not be called after deletion. + */ +void _mali_osk_wq_delete_work_nonflush(_mali_osk_wq_work_t *work); + +/** @brief Cause a queued, deferred call of the work handler + * + * _mali_osk_wq_schedule_work provides a mechanism for enqueuing deferred calls + * to the work handler. After calling \ref _mali_osk_wq_schedule_work(), the + * work handler will be scheduled to run at some point in the future. + * + * Typically this is called by the IRQ upper-half to defer further processing of + * IRQ-related work to the IRQ bottom-half handler. This is necessary for work + * that cannot be done in an IRQ context by the IRQ upper-half handler. Timer + * callbacks also use this mechanism, because they are treated as though they + * operate in an IRQ context. Refer to \ref _mali_osk_timer_t for more + * information. + * + * Code that operates in a kernel-process context (with no IRQ context + * restrictions) may also enqueue deferred calls to the IRQ bottom-half. The + * advantage over direct calling is that deferred calling allows the caller and + * IRQ bottom half to hold the same mutex, with a guarantee that they will not + * deadlock just by using this mechanism. + * + * _mali_osk_wq_schedule_work() places deferred call requests on a queue, to + * allow for more than one thread to make a deferred call. Therfore, if it is + * called 'K' times, then the IRQ bottom-half will be scheduled 'K' times too. + * 'K' is a number that is implementation-specific. + * + * _mali_osk_wq_schedule_work() is guaranteed to not block on: + * - enqueuing a deferred call request. + * - the completion of the work handler. + * + * This is to prevent deadlock. For example, if _mali_osk_wq_schedule_work() + * blocked, then it would cause a deadlock when the following two conditions + * hold: + * - The work handler callback (of type _mali_osk_wq_work_handler_t) locks + * a mutex + * - And, at the same time, the caller of _mali_osk_wq_schedule_work() also + * holds the same mutex + * + * @note care must be taken to not overflow the queue that + * _mali_osk_wq_schedule_work() operates on. Code must be structured to + * ensure that the number of requests made to the queue is bounded. Otherwise, + * work will be lost. + * + * The queue that _mali_osk_wq_schedule_work implements is a FIFO of N-writer, + * 1-reader type. The writers are the callers of _mali_osk_wq_schedule_work + * (all OSK-registered IRQ upper-half handlers in the system, watchdog timers, + * callers from a Kernel-process context). The reader is a single thread that + * handles all OSK-registered work. + * + * @param work a pointer to the _mali_osk_wq_work_t object corresponding to the + * work to begin processing. + */ +void _mali_osk_wq_schedule_work(_mali_osk_wq_work_t *work); + +/** @brief Cause a queued, deferred call of the high priority work handler + * + * Function is the same as \a _mali_osk_wq_schedule_work() with the only + * difference that it runs in a high (real time) priority on the system. + * + * Should only be used as a substitue for doing the same work in interrupts. + * + * This is allowed to sleep, but the work should be small since it will block + * all other applications. +*/ +void _mali_osk_wq_schedule_work_high_pri(_mali_osk_wq_work_t *work); + +/** @brief Flush the work queue + * + * This will flush the OSK work queue, ensuring all work in the queue has + * completed before returning. + * + * Since this blocks on the completion of work in the work-queue, the + * caller of this function \b must \b not hold any mutexes that are taken by + * any registered work handler. To do so may cause a deadlock. + * + */ +void _mali_osk_wq_flush(void); + +/** @brief Create work in the delayed work queue + * + * Creates a work object which can be scheduled in the work queue. When + * scheduled, a timer will be start and the \a handler will be called with + * \a data as the argument when timer out + * + * Refer to \ref _mali_osk_wq_delayed_schedule_work() for details on how work + * is scheduled in the queue. + * + * The returned pointer must be freed with \ref _mali_osk_wq_delayed_delete_work_nonflush() + * when no longer needed. + */ +_mali_osk_wq_delayed_work_t *_mali_osk_wq_delayed_create_work(_mali_osk_wq_work_handler_t handler, void *data); + +/** @brief Delete a work object + * + * This will NOT flush the work queue, so only call this if you are sure that the work handler will + * not be called after deletion. + */ +void _mali_osk_wq_delayed_delete_work_nonflush(_mali_osk_wq_delayed_work_t *work); + +/** @brief Cancel a delayed work without waiting for it to finish + * + * Note that the \a work callback function may still be running on return from + * _mali_osk_wq_delayed_cancel_work_async(). + * + * @param work The delayed work to be cancelled + */ +void _mali_osk_wq_delayed_cancel_work_async(_mali_osk_wq_delayed_work_t *work); + +/** @brief Cancel a delayed work and wait for it to finish + * + * When this function returns, the \a work was either cancelled or it finished running. + * + * @param work The delayed work to be cancelled + */ +void _mali_osk_wq_delayed_cancel_work_sync(_mali_osk_wq_delayed_work_t *work); + +/** @brief Put \a work task in global workqueue after delay + * + * After waiting for a given time this puts a job in the kernel-global + * workqueue. + * + * If \a work was already on a queue, this function will return without doing anything + * + * @param work job to be done + * @param delay number of jiffies to wait or 0 for immediate execution + */ +void _mali_osk_wq_delayed_schedule_work(_mali_osk_wq_delayed_work_t *work, u32 delay); + +/** @} */ /* end group _mali_osk_wq */ + + +/** @addtogroup _mali_osk_irq + * @{ */ + +/** @brief Initialize IRQ handling for a resource + * + * Registers an interrupt handler \a uhandler for the given IRQ number \a irqnum. + * \a data will be passed as argument to the handler when an interrupt occurs. + * + * If \a irqnum is -1, _mali_osk_irq_init will probe for the IRQ number using + * the supplied \a trigger_func and \a ack_func. These functions will also + * receive \a data as their argument. + * + * @param irqnum The IRQ number that the resource uses, as seen by the CPU. + * The value -1 has a special meaning which indicates the use of probing, and + * trigger_func and ack_func must be non-NULL. + * @param uhandler The interrupt handler, corresponding to a ISR handler for + * the resource + * @param int_data resource specific data, which will be passed to uhandler + * @param trigger_func Optional: a function to trigger the resource's irq, to + * probe for the interrupt. Use NULL if irqnum != -1. + * @param ack_func Optional: a function to acknowledge the resource's irq, to + * probe for the interrupt. Use NULL if irqnum != -1. + * @param probe_data resource-specific data, which will be passed to + * (if present) trigger_func and ack_func + * @param description textual description of the IRQ resource. + * @return on success, a pointer to a _mali_osk_irq_t object, which represents + * the IRQ handling on this resource. NULL on failure. + */ +_mali_osk_irq_t *_mali_osk_irq_init(u32 irqnum, _mali_osk_irq_uhandler_t uhandler, void *int_data, _mali_osk_irq_trigger_t trigger_func, _mali_osk_irq_ack_t ack_func, void *probe_data, const char *description); + +/** @brief Terminate IRQ handling on a resource. + * + * This will disable the interrupt from the device, and then waits for any + * currently executing IRQ handlers to complete. + * + * @note If work is deferred to an IRQ bottom-half handler through + * \ref _mali_osk_wq_schedule_work(), be sure to flush any remaining work + * with \ref _mali_osk_wq_flush() or (implicitly) with \ref _mali_osk_wq_delete_work() + * + * @param irq a pointer to the _mali_osk_irq_t object corresponding to the + * resource whose IRQ handling is to be terminated. + */ +void _mali_osk_irq_term(_mali_osk_irq_t *irq); + +/** @} */ /* end group _mali_osk_irq */ + + +/** @addtogroup _mali_osk_atomic + * @{ */ + +/** @brief Decrement an atomic counter + * + * @note It is an error to decrement the counter beyond -(1<<23) + * + * @param atom pointer to an atomic counter */ +void _mali_osk_atomic_dec(_mali_osk_atomic_t *atom); + +/** @brief Decrement an atomic counter, return new value + * + * @param atom pointer to an atomic counter + * @return The new value, after decrement */ +u32 _mali_osk_atomic_dec_return(_mali_osk_atomic_t *atom); + +/** @brief Increment an atomic counter + * + * @note It is an error to increment the counter beyond (1<<23)-1 + * + * @param atom pointer to an atomic counter */ +void _mali_osk_atomic_inc(_mali_osk_atomic_t *atom); + +/** @brief Increment an atomic counter, return new value + * + * @param atom pointer to an atomic counter */ +u32 _mali_osk_atomic_inc_return(_mali_osk_atomic_t *atom); + +/** @brief Initialize an atomic counter + * + * @note the parameter required is a u32, and so signed integers should be + * cast to u32. + * + * @param atom pointer to an atomic counter + * @param val the value to initialize the atomic counter. + */ +void _mali_osk_atomic_init(_mali_osk_atomic_t *atom, u32 val); + +/** @brief Read a value from an atomic counter + * + * This can only be safely used to determine the value of the counter when it + * is guaranteed that other threads will not be modifying the counter. This + * makes its usefulness limited. + * + * @param atom pointer to an atomic counter + */ +u32 _mali_osk_atomic_read(_mali_osk_atomic_t *atom); + +/** @brief Terminate an atomic counter + * + * @param atom pointer to an atomic counter + */ +void _mali_osk_atomic_term(_mali_osk_atomic_t *atom); + +/** @brief Assign a new val to atomic counter, and return the old atomic counter + * + * @param atom pointer to an atomic counter + * @param val the new value assign to the atomic counter + * @return the old value of the atomic counter + */ +u32 _mali_osk_atomic_xchg(_mali_osk_atomic_t *atom, u32 val); +/** @} */ /* end group _mali_osk_atomic */ + + +/** @defgroup _mali_osk_memory OSK Memory Allocation + * @{ */ + +/** @brief Allocate zero-initialized memory. + * + * Returns a buffer capable of containing at least \a n elements of \a size + * bytes each. The buffer is initialized to zero. + * + * If there is a need for a bigger block of memory (16KB or bigger), then + * consider to use _mali_osk_vmalloc() instead, as this function might + * map down to a OS function with size limitations. + * + * The buffer is suitably aligned for storage and subsequent access of every + * type that the compiler supports. Therefore, the pointer to the start of the + * buffer may be cast into any pointer type, and be subsequently accessed from + * such a pointer, without loss of information. + * + * When the buffer is no longer in use, it must be freed with _mali_osk_free(). + * Failure to do so will cause a memory leak. + * + * @note Most toolchains supply memory allocation functions that meet the + * compiler's alignment requirements. + * + * @param n Number of elements to allocate + * @param size Size of each element + * @return On success, the zero-initialized buffer allocated. NULL on failure + */ +void *_mali_osk_calloc(u32 n, u32 size); + +/** @brief Allocate memory. + * + * Returns a buffer capable of containing at least \a size bytes. The + * contents of the buffer are undefined. + * + * If there is a need for a bigger block of memory (16KB or bigger), then + * consider to use _mali_osk_vmalloc() instead, as this function might + * map down to a OS function with size limitations. + * + * The buffer is suitably aligned for storage and subsequent access of every + * type that the compiler supports. Therefore, the pointer to the start of the + * buffer may be cast into any pointer type, and be subsequently accessed from + * such a pointer, without loss of information. + * + * When the buffer is no longer in use, it must be freed with _mali_osk_free(). + * Failure to do so will cause a memory leak. + * + * @note Most toolchains supply memory allocation functions that meet the + * compiler's alignment requirements. + * + * Remember to free memory using _mali_osk_free(). + * @param size Number of bytes to allocate + * @return On success, the buffer allocated. NULL on failure. + */ +void *_mali_osk_malloc(u32 size); + +/** @brief Free memory. + * + * Reclaims the buffer pointed to by the parameter \a ptr for the system. + * All memory returned from _mali_osk_malloc() and _mali_osk_calloc() + * must be freed before the application exits. Otherwise, + * a memory leak will occur. + * + * Memory must be freed once. It is an error to free the same non-NULL pointer + * more than once. + * + * It is legal to free the NULL pointer. + * + * @param ptr Pointer to buffer to free + */ +void _mali_osk_free(void *ptr); + +/** @brief Allocate memory. + * + * Returns a buffer capable of containing at least \a size bytes. The + * contents of the buffer are undefined. + * + * This function is potentially slower than _mali_osk_malloc() and _mali_osk_calloc(), + * but do support bigger sizes. + * + * The buffer is suitably aligned for storage and subsequent access of every + * type that the compiler supports. Therefore, the pointer to the start of the + * buffer may be cast into any pointer type, and be subsequently accessed from + * such a pointer, without loss of information. + * + * When the buffer is no longer in use, it must be freed with _mali_osk_free(). + * Failure to do so will cause a memory leak. + * + * @note Most toolchains supply memory allocation functions that meet the + * compiler's alignment requirements. + * + * Remember to free memory using _mali_osk_free(). + * @param size Number of bytes to allocate + * @return On success, the buffer allocated. NULL on failure. + */ +void *_mali_osk_valloc(u32 size); + +/** @brief Free memory. + * + * Reclaims the buffer pointed to by the parameter \a ptr for the system. + * All memory returned from _mali_osk_valloc() must be freed before the + * application exits. Otherwise a memory leak will occur. + * + * Memory must be freed once. It is an error to free the same non-NULL pointer + * more than once. + * + * It is legal to free the NULL pointer. + * + * @param ptr Pointer to buffer to free + */ +void _mali_osk_vfree(void *ptr); + +/** @brief Copies memory. + * + * Copies the \a len bytes from the buffer pointed by the parameter \a src + * directly to the buffer pointed by \a dst. + * + * It is an error for \a src to overlap \a dst anywhere in \a len bytes. + * + * @param dst Pointer to the destination array where the content is to be + * copied. + * @param src Pointer to the source of data to be copied. + * @param len Number of bytes to copy. + * @return \a dst is always passed through unmodified. + */ +void *_mali_osk_memcpy(void *dst, const void *src, u32 len); + +/** @brief Fills memory. + * + * Sets the first \a n bytes of the block of memory pointed to by \a s to + * the specified value + * @param s Pointer to the block of memory to fill. + * @param c Value to be set, passed as u32. Only the 8 Least Significant Bits (LSB) + * are used. + * @param n Number of bytes to be set to the value. + * @return \a s is always passed through unmodified + */ +void *_mali_osk_memset(void *s, u32 c, u32 n); +/** @} */ /* end group _mali_osk_memory */ + + +/** @brief Checks the amount of memory allocated + * + * Checks that not more than \a max_allocated bytes are allocated. + * + * Some OS bring up an interactive out of memory dialogue when the + * system runs out of memory. This can stall non-interactive + * apps (e.g. automated test runs). This function can be used to + * not trigger the OOM dialogue by keeping allocations + * within a certain limit. + * + * @return MALI_TRUE when \a max_allocated bytes are not in use yet. MALI_FALSE + * when at least \a max_allocated bytes are in use. + */ +mali_bool _mali_osk_mem_check_allocated(u32 max_allocated); + + +/** @addtogroup _mali_osk_low_level_memory + * @{ */ + +/** @brief Issue a memory barrier + * + * This defines an arbitrary memory barrier operation, which forces an ordering constraint + * on memory read and write operations. + */ +void _mali_osk_mem_barrier(void); + +/** @brief Issue a write memory barrier + * + * This defines an write memory barrier operation which forces an ordering constraint + * on memory write operations. + */ +void _mali_osk_write_mem_barrier(void); + +/** @brief Map a physically contiguous region into kernel space + * + * This is primarily used for mapping in registers from resources, and Mali-MMU + * page tables. The mapping is only visable from kernel-space. + * + * Access has to go through _mali_osk_mem_ioread32 and _mali_osk_mem_iowrite32 + * + * @param phys CPU-physical base address of the memory to map in. This must + * be aligned to the system's page size, which is assumed to be 4K. + * @param size the number of bytes of physically contiguous address space to + * map in + * @param description A textual description of the memory being mapped in. + * @return On success, a Mali IO address through which the mapped-in + * memory/registers can be accessed. NULL on failure. + */ +mali_io_address _mali_osk_mem_mapioregion(uintptr_t phys, u32 size, const char *description); + +/** @brief Unmap a physically contiguous address range from kernel space. + * + * The address range should be one previously mapped in through + * _mali_osk_mem_mapioregion. + * + * It is a programming error to do (but not limited to) the following: + * - attempt an unmap twice + * - unmap only part of a range obtained through _mali_osk_mem_mapioregion + * - unmap more than the range obtained through _mali_osk_mem_mapioregion + * - unmap an address range that was not successfully mapped using + * _mali_osk_mem_mapioregion + * - provide a mapping that does not map to phys. + * + * @param phys CPU-physical base address of the memory that was originally + * mapped in. This must be aligned to the system's page size, which is assumed + * to be 4K + * @param size The number of bytes that were originally mapped in. + * @param mapping The Mali IO address through which the mapping is + * accessed. + */ +void _mali_osk_mem_unmapioregion(uintptr_t phys, u32 size, mali_io_address mapping); + +/** @brief Allocate and Map a physically contiguous region into kernel space + * + * This is used for allocating physically contiguous regions (such as Mali-MMU + * page tables) and mapping them into kernel space. The mapping is only + * visible from kernel-space. + * + * The alignment of the returned memory is guaranteed to be at least + * _MALI_OSK_CPU_PAGE_SIZE. + * + * Access must go through _mali_osk_mem_ioread32 and _mali_osk_mem_iowrite32 + * + * @note This function is primarily to provide support for OSs that are + * incapable of separating the tasks 'allocate physically contiguous memory' + * and 'map it into kernel space' + * + * @param[out] phys CPU-physical base address of memory that was allocated. + * (*phys) will be guaranteed to be aligned to at least + * _MALI_OSK_CPU_PAGE_SIZE on success. + * + * @param[in] size the number of bytes of physically contiguous memory to + * allocate. This must be a multiple of _MALI_OSK_CPU_PAGE_SIZE. + * + * @return On success, a Mali IO address through which the mapped-in + * memory/registers can be accessed. NULL on failure, and (*phys) is unmodified. + */ +mali_io_address _mali_osk_mem_allocioregion(u32 *phys, u32 size); + +/** @brief Free a physically contiguous address range from kernel space. + * + * The address range should be one previously mapped in through + * _mali_osk_mem_allocioregion. + * + * It is a programming error to do (but not limited to) the following: + * - attempt a free twice on the same ioregion + * - free only part of a range obtained through _mali_osk_mem_allocioregion + * - free more than the range obtained through _mali_osk_mem_allocioregion + * - free an address range that was not successfully mapped using + * _mali_osk_mem_allocioregion + * - provide a mapping that does not map to phys. + * + * @param phys CPU-physical base address of the memory that was originally + * mapped in, which was aligned to _MALI_OSK_CPU_PAGE_SIZE. + * @param size The number of bytes that were originally mapped in, which was + * a multiple of _MALI_OSK_CPU_PAGE_SIZE. + * @param mapping The Mali IO address through which the mapping is + * accessed. + */ +void _mali_osk_mem_freeioregion(u32 phys, u32 size, mali_io_address mapping); + +/** @brief Request a region of physically contiguous memory + * + * This is used to ensure exclusive access to a region of physically contigous + * memory. + * + * It is acceptable to implement this as a stub. However, it is then the job + * of the System Integrator to ensure that no other device driver will be using + * the physical address ranges used by Mali, while the Mali device driver is + * loaded. + * + * @param phys CPU-physical base address of the memory to request. This must + * be aligned to the system's page size, which is assumed to be 4K. + * @param size the number of bytes of physically contiguous address space to + * request. + * @param description A textual description of the memory being requested. + * @return _MALI_OSK_ERR_OK on success. Otherwise, a suitable + * _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_osk_mem_reqregion(uintptr_t phys, u32 size, const char *description); + +/** @brief Un-request a region of physically contiguous memory + * + * This is used to release a regious of physically contiguous memory previously + * requested through _mali_osk_mem_reqregion, so that other device drivers may + * use it. This will be called at time of Mali device driver termination. + * + * It is a programming error to attempt to: + * - unrequest a region twice + * - unrequest only part of a range obtained through _mali_osk_mem_reqregion + * - unrequest more than the range obtained through _mali_osk_mem_reqregion + * - unrequest an address range that was not successfully requested using + * _mali_osk_mem_reqregion + * + * @param phys CPU-physical base address of the memory to un-request. This must + * be aligned to the system's page size, which is assumed to be 4K + * @param size the number of bytes of physically contiguous address space to + * un-request. + */ +void _mali_osk_mem_unreqregion(uintptr_t phys, u32 size); + +/** @brief Read from a location currently mapped in through + * _mali_osk_mem_mapioregion + * + * This reads a 32-bit word from a 32-bit aligned location. It is a programming + * error to provide unaligned locations, or to read from memory that is not + * mapped in, or not mapped through either _mali_osk_mem_mapioregion() or + * _mali_osk_mem_allocioregion(). + * + * @param mapping Mali IO address to read from + * @param offset Byte offset from the given IO address to operate on, must be a multiple of 4 + * @return the 32-bit word from the specified location. + */ +u32 _mali_osk_mem_ioread32(volatile mali_io_address mapping, u32 offset); + +/** @brief Write to a location currently mapped in through + * _mali_osk_mem_mapioregion without memory barriers + * + * This write a 32-bit word to a 32-bit aligned location without using memory barrier. + * It is a programming error to provide unaligned locations, or to write to memory that is not + * mapped in, or not mapped through either _mali_osk_mem_mapioregion() or + * _mali_osk_mem_allocioregion(). + * + * @param mapping Mali IO address to write to + * @param offset Byte offset from the given IO address to operate on, must be a multiple of 4 + * @param val the 32-bit word to write. + */ +void _mali_osk_mem_iowrite32_relaxed(volatile mali_io_address addr, u32 offset, u32 val); + +/** @brief Write to a location currently mapped in through + * _mali_osk_mem_mapioregion with write memory barrier + * + * This write a 32-bit word to a 32-bit aligned location. It is a programming + * error to provide unaligned locations, or to write to memory that is not + * mapped in, or not mapped through either _mali_osk_mem_mapioregion() or + * _mali_osk_mem_allocioregion(). + * + * @param mapping Mali IO address to write to + * @param offset Byte offset from the given IO address to operate on, must be a multiple of 4 + * @param val the 32-bit word to write. + */ +void _mali_osk_mem_iowrite32(volatile mali_io_address mapping, u32 offset, u32 val); + +/** @brief Flush all CPU caches + * + * This should only be implemented if flushing of the cache is required for + * memory mapped in through _mali_osk_mem_mapregion. + */ +void _mali_osk_cache_flushall(void); + +/** @brief Flush any caches necessary for the CPU and MALI to have the same view of a range of uncached mapped memory + * + * This should only be implemented if your OS doesn't do a full cache flush (inner & outer) + * after allocating uncached mapped memory. + * + * Some OS do not perform a full cache flush (including all outer caches) for uncached mapped memory. + * They zero the memory through a cached mapping, then flush the inner caches but not the outer caches. + * This is required for MALI to have the correct view of the memory. + */ +void _mali_osk_cache_ensure_uncached_range_flushed(void *uncached_mapping, u32 offset, u32 size); + +/** @brief Safely copy as much data as possible from src to dest + * + * Do not crash if src or dest isn't available. + * + * @param dest Destination buffer (limited to user space mapped Mali memory) + * @param src Source buffer + * @param size Number of bytes to copy + * @return Number of bytes actually copied + */ +u32 _mali_osk_mem_write_safe(void *dest, const void *src, u32 size); + +/** @} */ /* end group _mali_osk_low_level_memory */ + + +/** @addtogroup _mali_osk_notification + * + * User space notification framework + * + * Communication with user space of asynchronous events is performed through a + * synchronous call to the \ref u_k_api. + * + * Since the events are asynchronous, the events have to be queued until a + * synchronous U/K API call can be made by user-space. A U/K API call might also + * be received before any event has happened. Therefore the notifications the + * different subsystems wants to send to user space has to be queued for later + * reception, or a U/K API call has to be blocked until an event has occured. + * + * Typical uses of notifications are after running of jobs on the hardware or + * when changes to the system is detected that needs to be relayed to user + * space. + * + * After an event has occured user space has to be notified using some kind of + * message. The notification framework supports sending messages to waiting + * threads or queueing of messages until a U/K API call is made. + * + * The notification queue is a FIFO. There are no restrictions on the numbers + * of readers or writers in the queue. + * + * A message contains what user space needs to identifiy how to handle an + * event. This includes a type field and a possible type specific payload. + * + * A notification to user space is represented by a + * \ref _mali_osk_notification_t object. A sender gets hold of such an object + * using _mali_osk_notification_create(). The buffer given by the + * _mali_osk_notification_t::result_buffer field in the object is used to store + * any type specific data. The other fields are internal to the queue system + * and should not be touched. + * + * @{ */ + +/** @brief Create a notification object + * + * Returns a notification object which can be added to the queue of + * notifications pending for user space transfer. + * + * The implementation will initialize all members of the + * \ref _mali_osk_notification_t object. In particular, the + * _mali_osk_notification_t::result_buffer member will be initialized to point + * to \a size bytes of storage, and that storage will be suitably aligned for + * storage of any structure. That is, the created buffer meets the same + * requirements as _mali_osk_malloc(). + * + * The notification object must be deleted when not in use. Use + * _mali_osk_notification_delete() for deleting it. + * + * @note You \b must \b not call _mali_osk_free() on a \ref _mali_osk_notification_t, + * object, or on a _mali_osk_notification_t::result_buffer. You must only use + * _mali_osk_notification_delete() to free the resources assocaited with a + * \ref _mali_osk_notification_t object. + * + * @param type The notification type + * @param size The size of the type specific buffer to send + * @return Pointer to a notification object with a suitable buffer, or NULL on error. + */ +_mali_osk_notification_t *_mali_osk_notification_create(u32 type, u32 size); + +/** @brief Delete a notification object + * + * This must be called to reclaim the resources of a notification object. This + * includes: + * - The _mali_osk_notification_t::result_buffer + * - The \ref _mali_osk_notification_t itself. + * + * A notification object \b must \b not be used after it has been deleted by + * _mali_osk_notification_delete(). + * + * In addition, the notification object may not be deleted while it is in a + * queue. That is, if it has been placed on a queue with + * _mali_osk_notification_queue_send(), then it must not be deleted until + * it has been received by a call to _mali_osk_notification_queue_receive(). + * Otherwise, the queue may be corrupted. + * + * @param object the notification object to delete. + */ +void _mali_osk_notification_delete(_mali_osk_notification_t *object); + +/** @brief Create a notification queue + * + * Creates a notification queue which can be used to queue messages for user + * delivery and get queued messages from + * + * The queue is a FIFO, and has no restrictions on the numbers of readers or + * writers. + * + * When the queue is no longer in use, it must be terminated with + * \ref _mali_osk_notification_queue_term(). Failure to do so will result in a + * memory leak. + * + * @return Pointer to a new notification queue or NULL on error. + */ +_mali_osk_notification_queue_t *_mali_osk_notification_queue_init(void); + +/** @brief Destroy a notification queue + * + * Destroys a notification queue and frees associated resources from the queue. + * + * A notification queue \b must \b not be destroyed in the following cases: + * - while there are \ref _mali_osk_notification_t objects in the queue. + * - while there are writers currently acting upon the queue. That is, while + * a thread is currently calling \ref _mali_osk_notification_queue_send() on + * the queue, or while a thread may call + * \ref _mali_osk_notification_queue_send() on the queue in the future. + * - while there are readers currently waiting upon the queue. That is, while + * a thread is currently calling \ref _mali_osk_notification_queue_receive() on + * the queue, or while a thread may call + * \ref _mali_osk_notification_queue_receive() on the queue in the future. + * + * Therefore, all \ref _mali_osk_notification_t objects must be flushed and + * deleted by the code that makes use of the notification queues, since only + * they know the structure of the _mali_osk_notification_t::result_buffer + * (even if it may only be a flat sturcture). + * + * @note Since the queue is a FIFO, the code using notification queues may + * create its own 'flush' type of notification, to assist in flushing the + * queue. + * + * Once the queue has been destroyed, it must not be used again. + * + * @param queue The queue to destroy + */ +void _mali_osk_notification_queue_term(_mali_osk_notification_queue_t *queue); + +/** @brief Schedule notification for delivery + * + * When a \ref _mali_osk_notification_t object has been created successfully + * and set up, it may be added to the queue of objects waiting for user space + * transfer. + * + * The sending will not block if the queue is full. + * + * A \ref _mali_osk_notification_t object \b must \b not be put on two different + * queues at the same time, or enqueued twice onto a single queue before + * reception. However, it is acceptable for it to be requeued \em after reception + * from a call to _mali_osk_notification_queue_receive(), even onto the same queue. + * + * Again, requeuing must also not enqueue onto two different queues at the same + * time, or enqueue onto the same queue twice before reception. + * + * @param queue The notification queue to add this notification to + * @param object The entry to add + */ +void _mali_osk_notification_queue_send(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t *object); + +/** @brief Receive a notification from a queue + * + * Receives a single notification from the given queue. + * + * If no notifciations are ready the thread will sleep until one becomes ready. + * Therefore, notifications may not be received into an + * IRQ or 'atomic' context (that is, a context where sleeping is disallowed). + * + * @param queue The queue to receive from + * @param result Pointer to storage of a pointer of type + * \ref _mali_osk_notification_t*. \a result will be written to such that the + * expression \a (*result) will evaluate to a pointer to a valid + * \ref _mali_osk_notification_t object, or NULL if none were received. + * @return _MALI_OSK_ERR_OK on success. _MALI_OSK_ERR_RESTARTSYSCALL if the sleep was interrupted. + */ +_mali_osk_errcode_t _mali_osk_notification_queue_receive(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result); + +/** @brief Dequeues a notification from a queue + * + * Receives a single notification from the given queue. + * + * If no notifciations are ready the function call will return an error code. + * + * @param queue The queue to receive from + * @param result Pointer to storage of a pointer of type + * \ref _mali_osk_notification_t*. \a result will be written to such that the + * expression \a (*result) will evaluate to a pointer to a valid + * \ref _mali_osk_notification_t object, or NULL if none were received. + * @return _MALI_OSK_ERR_OK on success, _MALI_OSK_ERR_ITEM_NOT_FOUND if queue was empty. + */ +_mali_osk_errcode_t _mali_osk_notification_queue_dequeue(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result); + +/** @} */ /* end group _mali_osk_notification */ + + +/** @addtogroup _mali_osk_timer + * + * Timers use the OS's representation of time, which are 'ticks'. This is to + * prevent aliasing problems between the internal timer time, and the time + * asked for. + * + * @{ */ + +/** @brief Initialize a timer + * + * Allocates resources for a new timer, and initializes them. This does not + * start the timer. + * + * @return a pointer to the allocated timer object, or NULL on failure. + */ +_mali_osk_timer_t *_mali_osk_timer_init(_mali_osk_timer_callback_t callback); + +/** @brief Start a timer + * + * It is an error to start a timer without setting the callback via + * _mali_osk_timer_setcallback(). + * + * It is an error to use this to start an already started timer. + * + * The timer will expire in \a ticks_to_expire ticks, at which point, the + * callback function will be invoked with the callback-specific data, + * as registered by _mali_osk_timer_setcallback(). + * + * @param tim the timer to start + * @param ticks_to_expire the amount of time in ticks for the timer to run + * before triggering. + */ +void _mali_osk_timer_add(_mali_osk_timer_t *tim, unsigned long ticks_to_expire); + +/** @brief Modify a timer + * + * Set the relative time at which a timer will expire, and start it if it is + * stopped. If \a ticks_to_expire 0 the timer fires immediately. + * + * It is an error to modify a timer without setting the callback via + * _mali_osk_timer_setcallback(). + * + * The timer will expire at \a ticks_to_expire from the time of the call, at + * which point, the callback function will be invoked with the + * callback-specific data, as set by _mali_osk_timer_setcallback(). + * + * @param tim the timer to modify, and start if necessary + * @param ticks_to_expire the \em absolute time in ticks at which this timer + * should trigger. + * + */ +void _mali_osk_timer_mod(_mali_osk_timer_t *tim, unsigned long ticks_to_expire); + +/** @brief Stop a timer, and block on its completion. + * + * Stop the timer. When the function returns, it is guaranteed that the timer's + * callback will not be running on any CPU core. + * + * Since stoping the timer blocks on compeletion of the callback, the callback + * may not obtain any mutexes that the caller holds. Otherwise, a deadlock will + * occur. + * + * @note While the callback itself is guaranteed to not be running, work + * enqueued on the work-queue by the timer (with + * \ref _mali_osk_wq_schedule_work()) may still run. The timer callback and + * work handler must take this into account. + * + * It is legal to stop an already stopped timer. + * + * @param tim the timer to stop. + * + */ +void _mali_osk_timer_del(_mali_osk_timer_t *tim); + +/** @brief Stop a timer. + * + * Stop the timer. When the function returns, the timer's callback may still be + * running on any CPU core. + * + * It is legal to stop an already stopped timer. + * + * @param tim the timer to stop. + */ +void _mali_osk_timer_del_async(_mali_osk_timer_t *tim); + +/** @brief Check if timer is pending. + * + * Check if timer is active. + * + * @param tim the timer to check + * @return MALI_TRUE if time is active, MALI_FALSE if it is not active + */ +mali_bool _mali_osk_timer_pending(_mali_osk_timer_t *tim); + +/** @brief Set a timer's callback parameters. + * + * This must be called at least once before a timer is started/modified. + * + * After a timer has been stopped or expires, the callback remains set. This + * means that restarting the timer will call the same function with the same + * parameters on expiry. + * + * @param tim the timer to set callback on. + * @param callback Function to call when timer expires + * @param data Function-specific data to supply to the function on expiry. + */ +void _mali_osk_timer_setcallback(_mali_osk_timer_t *tim, _mali_osk_timer_callback_t callback, void *data); + +/** @brief Terminate a timer, and deallocate resources. + * + * The timer must first be stopped by calling _mali_osk_timer_del(). + * + * It is a programming error for _mali_osk_timer_term() to be called on: + * - timer that is currently running + * - a timer that is currently executing its callback. + * + * @param tim the timer to deallocate. + */ +void _mali_osk_timer_term(_mali_osk_timer_t *tim); +/** @} */ /* end group _mali_osk_timer */ + + +/** @defgroup _mali_osk_time OSK Time functions + * + * \ref _mali_osk_time use the OS's representation of time, which are + * 'ticks'. This is to prevent aliasing problems between the internal timer + * time, and the time asked for. + * + * OS tick time is measured as a u32. The time stored in a u32 may either be + * an absolute time, or a time delta between two events. Whilst it is valid to + * use math opeartors to \em change the tick value represented as a u32, it + * is often only meaningful to do such operations on time deltas, rather than + * on absolute time. However, it is meaningful to add/subtract time deltas to + * absolute times. + * + * Conversion between tick time and milliseconds (ms) may not be loss-less, + * and are \em implementation \em depenedant. + * + * Code use OS time must take this into account, since: + * - a small OS time may (or may not) be rounded + * - a large time may (or may not) overflow + * + * @{ */ + +/** @brief Return whether ticka occurs after or at the same time as tickb + * + * Systems where ticks can wrap must handle that. + * + * @param ticka ticka + * @param tickb tickb + * @return MALI_TRUE if ticka represents a time that occurs at or after tickb. + */ +mali_bool _mali_osk_time_after_eq(unsigned long ticka, unsigned long tickb); + +/** @brief Convert milliseconds to OS 'ticks' + * + * @param ms time interval in milliseconds + * @return the corresponding time interval in OS ticks. + */ +unsigned long _mali_osk_time_mstoticks(u32 ms); + +/** @brief Convert OS 'ticks' to milliseconds + * + * @param ticks time interval in OS ticks. + * @return the corresponding time interval in milliseconds + */ +u32 _mali_osk_time_tickstoms(unsigned long ticks); + + +/** @brief Get the current time in OS 'ticks'. + * @return the current time in OS 'ticks'. + */ +unsigned long _mali_osk_time_tickcount(void); + +/** @brief Cause a microsecond delay + * + * The delay will have microsecond resolution, and is necessary for correct + * operation of the driver. At worst, the delay will be \b at least \a usecs + * microseconds, and so may be (significantly) more. + * + * This function may be implemented as a busy-wait, which is the most sensible + * implementation. On OSs where there are situations in which a thread must not + * sleep, this is definitely implemented as a busy-wait. + * + * @param usecs the number of microseconds to wait for. + */ +void _mali_osk_time_ubusydelay(u32 usecs); + +/** @brief Return time in nano seconds, since any given reference. + * + * @return Time in nano seconds + */ +u64 _mali_osk_time_get_ns(void); + +/** @brief Return time in nano seconds, since boot time. + * + * @return Time in nano seconds + */ +u64 _mali_osk_boot_time_get_ns(void); + +/** @} */ /* end group _mali_osk_time */ + +/** @defgroup _mali_osk_math OSK Math + * @{ */ + +/** @brief Count Leading Zeros (Little-endian) + * + * @note This function must be implemented to support the reference + * implementation of _mali_osk_find_first_zero_bit, as defined in + * mali_osk_bitops.h. + * + * @param val 32-bit words to count leading zeros on + * @return the number of leading zeros. + */ +u32 _mali_osk_clz(u32 val); + +/** @brief find last (most-significant) bit set + * + * @param val 32-bit words to count last bit set on + * @return last bit set. + */ +u32 _mali_osk_fls(u32 val); + +/** @} */ /* end group _mali_osk_math */ + +/** @addtogroup _mali_osk_wait_queue OSK Wait Queue functionality + * @{ */ + +/** @brief Initialize an empty Wait Queue */ +_mali_osk_wait_queue_t *_mali_osk_wait_queue_init(void); + +/** @brief Sleep if condition is false + * + * @param queue the queue to use + * @param condition function pointer to a boolean function + * @param data data parameter for condition function + * + * Put thread to sleep if the given \a condition function returns false. When + * being asked to wake up again, the condition will be re-checked and the + * thread only woken up if the condition is now true. + */ +void _mali_osk_wait_queue_wait_event(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data); + +/** @brief Sleep if condition is false + * + * @param queue the queue to use + * @param condition function pointer to a boolean function + * @param data data parameter for condition function + * @param timeout timeout in ms + * + * Put thread to sleep if the given \a condition function returns false. When + * being asked to wake up again, the condition will be re-checked and the + * thread only woken up if the condition is now true. Will return if time + * exceeds timeout. + */ +void _mali_osk_wait_queue_wait_event_timeout(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data, u32 timeout); + +/** @brief Wake up all threads in wait queue if their respective conditions are + * true + * + * @param queue the queue whose threads should be woken up + * + * Wake up all threads in wait queue \a queue whose condition is now true. + */ +void _mali_osk_wait_queue_wake_up(_mali_osk_wait_queue_t *queue); + +/** @brief terminate a wait queue + * + * @param queue the queue to terminate. + */ +void _mali_osk_wait_queue_term(_mali_osk_wait_queue_t *queue); +/** @} */ /* end group _mali_osk_wait_queue */ + + +/** @addtogroup _mali_osk_miscellaneous + * @{ */ + +/** @brief Output a device driver debug message. + * + * The interpretation of \a fmt is the same as the \c format parameter in + * _mali_osu_vsnprintf(). + * + * @param fmt a _mali_osu_vsnprintf() style format string + * @param ... a variable-number of parameters suitable for \a fmt + */ +void _mali_osk_dbgmsg(const char *fmt, ...); + +/** @brief Print fmt into buf. + * + * The interpretation of \a fmt is the same as the \c format parameter in + * _mali_osu_vsnprintf(). + * + * @param buf a pointer to the result buffer + * @param size the total number of bytes allowed to write to \a buf + * @param fmt a _mali_osu_vsnprintf() style format string + * @param ... a variable-number of parameters suitable for \a fmt + * @return The number of bytes written to \a buf + */ +u32 _mali_osk_snprintf(char *buf, u32 size, const char *fmt, ...); + +/** @brief Abnormal process abort. + * + * Terminates the caller-process if this function is called. + * + * This function will be called from Debug assert-macros in mali_kernel_common.h. + * + * This function will never return - because to continue from a Debug assert + * could cause even more problems, and hinder debugging of the initial problem. + * + * This function is only used in Debug builds, and is not used in Release builds. + */ +void _mali_osk_abort(void); + +/** @brief Sets breakpoint at point where function is called. + * + * This function will be called from Debug assert-macros in mali_kernel_common.h, + * to assist in debugging. If debugging at this level is not required, then this + * function may be implemented as a stub. + * + * This function is only used in Debug builds, and is not used in Release builds. + */ +void _mali_osk_break(void); + +/** @brief Return an identificator for calling process. + * + * @return Identificator for calling process. + */ +u32 _mali_osk_get_pid(void); + +/** @brief Return an name for calling process. + * + * @return name for calling process. + */ +char *_mali_osk_get_comm(void); + +/** @brief Return an identificator for calling thread. + * + * @return Identificator for calling thread. + */ +u32 _mali_osk_get_tid(void); + + +/** @brief Take a reference to the power manager system for the Mali device (synchronously). + * + * When function returns successfully, Mali is ON. + * + * @note Call \a _mali_osk_pm_dev_ref_put() to release this reference. + */ +_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_sync(void); + +/** @brief Take a reference to the external power manager system for the Mali device (asynchronously). + * + * Mali might not yet be on after this function as returned. + * Please use \a _mali_osk_pm_dev_barrier() or \a _mali_osk_pm_dev_ref_get_sync() + * to wait for Mali to be powered on. + * + * @note Call \a _mali_osk_pm_dev_ref_dec() to release this reference. + */ +_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_async(void); + +/** @brief Release the reference to the external power manger system for the Mali device. + * + * When reference count reach zero, the cores can be off. + * + * @note This must be used to release references taken with + * \a _mali_osk_pm_dev_ref_get_sync() or \a _mali_osk_pm_dev_ref_get_sync(). + */ +void _mali_osk_pm_dev_ref_put(void); + +/** @brief Block until pending PM operations are done + */ +void _mali_osk_pm_dev_barrier(void); + +/** @} */ /* end group _mali_osk_miscellaneous */ + +/** @defgroup _mali_osk_bitmap OSK Bitmap + * @{ */ + +/** @brief Allocate a unique number from the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @return An unique existence in the bitmap object. + */ +u32 _mali_osk_bitmap_alloc(struct _mali_osk_bitmap *bitmap); + +/** @brief Free a interger to the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @param obj An number allocated from bitmap object. + */ +void _mali_osk_bitmap_free(struct _mali_osk_bitmap *bitmap, u32 obj); + +/** @brief Allocate continuous number from the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @return start number of the continuous number block. + */ +u32 _mali_osk_bitmap_alloc_range(struct _mali_osk_bitmap *bitmap, int cnt); + +/** @brief Free a block of continuous number block to the bitmap object. + * + * @param bitmap Initialized bitmap object. + * @param obj Start number. + * @param cnt The size of the continuous number block. + */ +void _mali_osk_bitmap_free_range(struct _mali_osk_bitmap *bitmap, u32 obj, int cnt); + +/** @brief Available count could be used to allocate in the given bitmap object. + * + */ +u32 _mali_osk_bitmap_avail(struct _mali_osk_bitmap *bitmap); + +/** @brief Initialize an bitmap object.. + * + * @param bitmap An poiter of uninitialized bitmap object. + * @param num Size of thei bitmap object and decide the memory size allocated. + * @param reserve start number used to allocate. + */ +int _mali_osk_bitmap_init(struct _mali_osk_bitmap *bitmap, u32 num, u32 reserve); + +/** @brief Free the given bitmap object. + * + * @param bitmap Initialized bitmap object. + */ +void _mali_osk_bitmap_term(struct _mali_osk_bitmap *bitmap); +/** @} */ /* end group _mali_osk_bitmap */ + +/** @} */ /* end group osuapi */ + +/** @} */ /* end group uddapi */ + + + +#ifdef __cplusplus +} +#endif + +/* Check standard inlines */ +#ifndef MALI_STATIC_INLINE +#error MALI_STATIC_INLINE not defined on your OS +#endif + +#ifndef MALI_NON_STATIC_INLINE +#error MALI_NON_STATIC_INLINE not defined on your OS +#endif + +#endif /* __MALI_OSK_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h new file mode 100755 index 000000000000..bb1831753a40 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_bitops.h @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_bitops.h + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#ifndef __MALI_OSK_BITOPS_H__ +#define __MALI_OSK_BITOPS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +MALI_STATIC_INLINE void _mali_internal_clear_bit(u32 bit, u32 *addr) +{ + MALI_DEBUG_ASSERT(bit < 32); + MALI_DEBUG_ASSERT(NULL != addr); + + (*addr) &= ~(1 << bit); +} + +MALI_STATIC_INLINE void _mali_internal_set_bit(u32 bit, u32 *addr) +{ + MALI_DEBUG_ASSERT(bit < 32); + MALI_DEBUG_ASSERT(NULL != addr); + + (*addr) |= (1 << bit); +} + +MALI_STATIC_INLINE u32 _mali_internal_test_bit(u32 bit, u32 value) +{ + MALI_DEBUG_ASSERT(bit < 32); + return value & (1 << bit); +} + +MALI_STATIC_INLINE int _mali_internal_find_first_zero_bit(u32 value) +{ + u32 inverted; + u32 negated; + u32 isolated; + u32 leading_zeros; + + /* Begin with xxx...x0yyy...y, where ys are 1, number of ys is in range 0..31 */ + inverted = ~value; /* zzz...z1000...0 */ + /* Using count_trailing_zeros on inverted value - + * See ARM System Developers Guide for details of count_trailing_zeros */ + + /* Isolate the zero: it is preceeded by a run of 1s, so add 1 to it */ + negated = (u32) - inverted ; /* -a == ~a + 1 (mod 2^n) for n-bit numbers */ + /* negated = xxx...x1000...0 */ + + isolated = negated & inverted ; /* xxx...x1000...0 & zzz...z1000...0, zs are ~xs */ + /* And so the first zero bit is in the same position as the 1 == number of 1s that preceeded it + * Note that the output is zero if value was all 1s */ + + leading_zeros = _mali_osk_clz(isolated); + + return 31 - leading_zeros; +} + + +/** @defgroup _mali_osk_bitops OSK Non-atomic Bit-operations + * @{ */ + +/** + * These bit-operations do not work atomically, and so locks must be used if + * atomicity is required. + * + * Reference implementations for Little Endian are provided, and so it should + * not normally be necessary to re-implement these. Efficient bit-twiddling + * techniques are used where possible, implemented in portable C. + * + * Note that these reference implementations rely on _mali_osk_clz() being + * implemented. + */ + +/** @brief Clear a bit in a sequence of 32-bit words + * @param nr bit number to clear, starting from the (Little-endian) least + * significant bit + * @param addr starting point for counting. + */ +MALI_STATIC_INLINE void _mali_osk_clear_nonatomic_bit(u32 nr, u32 *addr) +{ + addr += nr >> 5; /* find the correct word */ + nr = nr & ((1 << 5) - 1); /* The bit number within the word */ + + _mali_internal_clear_bit(nr, addr); +} + +/** @brief Set a bit in a sequence of 32-bit words + * @param nr bit number to set, starting from the (Little-endian) least + * significant bit + * @param addr starting point for counting. + */ +MALI_STATIC_INLINE void _mali_osk_set_nonatomic_bit(u32 nr, u32 *addr) +{ + addr += nr >> 5; /* find the correct word */ + nr = nr & ((1 << 5) - 1); /* The bit number within the word */ + + _mali_internal_set_bit(nr, addr); +} + +/** @brief Test a bit in a sequence of 32-bit words + * @param nr bit number to test, starting from the (Little-endian) least + * significant bit + * @param addr starting point for counting. + * @return zero if bit was clear, non-zero if set. Do not rely on the return + * value being related to the actual word under test. + */ +MALI_STATIC_INLINE u32 _mali_osk_test_bit(u32 nr, u32 *addr) +{ + addr += nr >> 5; /* find the correct word */ + nr = nr & ((1 << 5) - 1); /* The bit number within the word */ + + return _mali_internal_test_bit(nr, *addr); +} + +/* Return maxbit if not found */ +/** @brief Find the first zero bit in a sequence of 32-bit words + * @param addr starting point for search. + * @param maxbit the maximum number of bits to search + * @return the number of the first zero bit found, or maxbit if none were found + * in the specified range. + */ +MALI_STATIC_INLINE u32 _mali_osk_find_first_zero_bit(const u32 *addr, u32 maxbit) +{ + u32 total; + + for (total = 0; total < maxbit; total += 32, ++addr) { + int result; + result = _mali_internal_find_first_zero_bit(*addr); + + /* non-negative signifies the bit was found */ + if (result >= 0) { + total += (u32)result; + break; + } + } + + /* Now check if we reached maxbit or above */ + if (total >= maxbit) { + total = maxbit; + } + + return total; /* either the found bit nr, or maxbit if not found */ +} +/** @} */ /* end group _mali_osk_bitops */ + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_OSK_BITOPS_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_list.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_list.h new file mode 100755 index 000000000000..9af2d7d4d621 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_list.h @@ -0,0 +1,273 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_list.h + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#ifndef __MALI_OSK_LIST_H__ +#define __MALI_OSK_LIST_H__ + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MALI_STATIC_INLINE void __mali_osk_list_add(_mali_osk_list_t *new_entry, _mali_osk_list_t *prev, _mali_osk_list_t *next) +{ + next->prev = new_entry; + new_entry->next = next; + new_entry->prev = prev; + prev->next = new_entry; +} + +MALI_STATIC_INLINE void __mali_osk_list_del(_mali_osk_list_t *prev, _mali_osk_list_t *next) +{ + next->prev = prev; + prev->next = next; +} + +/** @addtogroup _mali_osk_list OSK Doubly-Linked Circular Lists + * @{ */ + +/** Reference implementations of Doubly-linked Circular Lists are provided. + * There is often no need to re-implement these. + * + * @note The implementation may differ subtly from any lists the OS provides. + * For this reason, these lists should not be mixed with OS-specific lists + * inside the OSK/UKK implementation. */ + +/** @brief Initialize a list to be a head of an empty list + * @param exp the list to initialize. */ +#define _MALI_OSK_INIT_LIST_HEAD(exp) _mali_osk_list_init(exp) + +/** @brief Define a list variable, which is uninitialized. + * @param exp the name of the variable that the list will be defined as. */ +#define _MALI_OSK_LIST_HEAD(exp) _mali_osk_list_t exp + +/** @brief Define a list variable, which is initialized. + * @param exp the name of the variable that the list will be defined as. */ +#define _MALI_OSK_LIST_HEAD_STATIC_INIT(exp) _mali_osk_list_t exp = { &exp, &exp } + +/** @brief Initialize a list element. + * + * All list elements must be initialized before use. + * + * Do not use on any list element that is present in a list without using + * _mali_osk_list_del first, otherwise this will break the list. + * + * @param list the list element to initialize + */ +MALI_STATIC_INLINE void _mali_osk_list_init(_mali_osk_list_t *list) +{ + list->next = list; + list->prev = list; +} + +/** @brief Insert a single list element after an entry in a list + * + * As an example, if this is inserted to the head of a list, then this becomes + * the first element of the list. + * + * Do not use to move list elements from one list to another, as it will break + * the originating list. + * + * + * @param newlist the list element to insert + * @param list the list in which to insert. The new element will be the next + * entry in this list + */ +MALI_STATIC_INLINE void _mali_osk_list_add(_mali_osk_list_t *new_entry, _mali_osk_list_t *list) +{ + __mali_osk_list_add(new_entry, list, list->next); +} + +/** @brief Insert a single list element before an entry in a list + * + * As an example, if this is inserted to the head of a list, then this becomes + * the last element of the list. + * + * Do not use to move list elements from one list to another, as it will break + * the originating list. + * + * @param newlist the list element to insert + * @param list the list in which to insert. The new element will be the previous + * entry in this list + */ +MALI_STATIC_INLINE void _mali_osk_list_addtail(_mali_osk_list_t *new_entry, _mali_osk_list_t *list) +{ + __mali_osk_list_add(new_entry, list->prev, list); +} + +/** @brief Remove a single element from a list + * + * The element will no longer be present in the list. The removed list element + * will be uninitialized, and so should not be traversed. It must be + * initialized before further use. + * + * @param list the list element to remove. + */ +MALI_STATIC_INLINE void _mali_osk_list_del(_mali_osk_list_t *list) +{ + __mali_osk_list_del(list->prev, list->next); +} + +/** @brief Remove a single element from a list, and re-initialize it + * + * The element will no longer be present in the list. The removed list element + * will initialized, and so can be used as normal. + * + * @param list the list element to remove and initialize. + */ +MALI_STATIC_INLINE void _mali_osk_list_delinit(_mali_osk_list_t *list) +{ + __mali_osk_list_del(list->prev, list->next); + _mali_osk_list_init(list); +} + +/** @brief Determine whether a list is empty. + * + * An empty list is one that contains a single element that points to itself. + * + * @param list the list to check. + * @return non-zero if the list is empty, and zero otherwise. + */ +MALI_STATIC_INLINE mali_bool _mali_osk_list_empty(_mali_osk_list_t *list) +{ + return list->next == list; +} + +/** @brief Move a list element from one list to another. + * + * The list element must be initialized. + * + * As an example, moving a list item to the head of a new list causes this item + * to be the first element in the new list. + * + * @param move the list element to move + * @param list the new list into which the element will be inserted, as the next + * element in the list. + */ +MALI_STATIC_INLINE void _mali_osk_list_move(_mali_osk_list_t *move_entry, _mali_osk_list_t *list) +{ + __mali_osk_list_del(move_entry->prev, move_entry->next); + _mali_osk_list_add(move_entry, list); +} + +/** @brief Move an entire list + * + * The list element must be initialized. + * + * Allows you to move a list from one list head to another list head + * + * @param old_list The existing list head + * @param new_list The new list head (must be an empty list) + */ +MALI_STATIC_INLINE void _mali_osk_list_move_list(_mali_osk_list_t *old_list, _mali_osk_list_t *new_list) +{ + MALI_DEBUG_ASSERT(_mali_osk_list_empty(new_list)); + if (!_mali_osk_list_empty(old_list)) { + new_list->next = old_list->next; + new_list->prev = old_list->prev; + new_list->next->prev = new_list; + new_list->prev->next = new_list; + old_list->next = old_list; + old_list->prev = old_list; + } +} + +/** @brief Find the containing structure of a list + * + * When traversing a list, this is used to recover the containing structure, + * given that is contains a _mali_osk_list_t member. + * + * Each list must be of structures of one type, and must link the same members + * together, otherwise it will not be possible to correctly recover the + * sturctures that the lists link. + * + * @note no type or memory checking occurs to ensure that a structure does in + * fact exist for the list entry, and that it is being recovered with respect + * to the correct list member. + * + * @param ptr the pointer to the _mali_osk_list_t member in this structure + * @param type the type of the structure that contains the member + * @param member the member of the structure that ptr points to. + * @return a pointer to a \a type object which contains the _mali_osk_list_t + * \a member, as pointed to by the _mali_osk_list_t \a *ptr. + */ +#define _MALI_OSK_LIST_ENTRY(ptr, type, member) \ + _MALI_OSK_CONTAINER_OF(ptr, type, member) + +/** @brief Enumerate a list safely + * + * With this macro, lists can be enumerated in a 'safe' manner. That is, + * entries can be deleted from the list without causing an error during + * enumeration. To achieve this, a 'temporary' pointer is required, which must + * be provided to the macro. + * + * Use it like a 'for()', 'while()' or 'do()' construct, and so it must be + * followed by a statement or compound-statement which will be executed for + * each list entry. + * + * Upon loop completion, providing that an early out was not taken in the + * loop body, then it is guaranteed that ptr->member == list, even if the loop + * body never executed. + * + * @param ptr a pointer to an object of type 'type', which points to the + * structure that contains the currently enumerated list entry. + * @param tmp a pointer to an object of type 'type', which must not be used + * inside the list-execution statement. + * @param list a pointer to a _mali_osk_list_t, from which enumeration will + * begin + * @param type the type of the structure that contains the _mali_osk_list_t + * member that is part of the list to be enumerated. + * @param member the _mali_osk_list_t member of the structure that is part of + * the list to be enumerated. + */ +#define _MALI_OSK_LIST_FOREACHENTRY(ptr, tmp, list, type, member) \ + for (ptr = _MALI_OSK_LIST_ENTRY((list)->next, type, member), \ + tmp = _MALI_OSK_LIST_ENTRY(ptr->member.next, type, member); \ + &ptr->member != (list); \ + ptr = tmp, \ + tmp = _MALI_OSK_LIST_ENTRY(tmp->member.next, type, member)) + +/** @brief Enumerate a list in reverse order safely + * + * This macro is identical to @ref _MALI_OSK_LIST_FOREACHENTRY, except that + * entries are enumerated in reverse order. + * + * @param ptr a pointer to an object of type 'type', which points to the + * structure that contains the currently enumerated list entry. + * @param tmp a pointer to an object of type 'type', which must not be used + * inside the list-execution statement. + * @param list a pointer to a _mali_osk_list_t, from which enumeration will + * begin + * @param type the type of the structure that contains the _mali_osk_list_t + * member that is part of the list to be enumerated. + * @param member the _mali_osk_list_t member of the structure that is part of + * the list to be enumerated. + */ +#define _MALI_OSK_LIST_FOREACHENTRY_REVERSE(ptr, tmp, list, type, member) \ + for (ptr = _MALI_OSK_LIST_ENTRY((list)->prev, type, member), \ + tmp = _MALI_OSK_LIST_ENTRY(ptr->member.prev, type, member); \ + &ptr->member != (list); \ + ptr = tmp, \ + tmp = _MALI_OSK_LIST_ENTRY(tmp->member.prev, type, member)) + +/** @} */ /* end group _mali_osk_list */ + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_OSK_LIST_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h new file mode 100755 index 000000000000..bf69925a43a7 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_mali.h @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_mali.h + * Defines the OS abstraction layer which is specific for the Mali kernel device driver (OSK) + */ + +#ifndef __MALI_OSK_MALI_H__ +#define __MALI_OSK_MALI_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef CONFIG_MALI_DEVFREQ +struct mali_device { + struct device *dev; +#ifdef CONFIG_HAVE_CLK + struct clk *clock; +#endif +#ifdef CONFIG_REGULATOR + struct regulator *regulator; +#endif +#ifdef CONFIG_PM_DEVFREQ + struct devfreq_dev_profile devfreq_profile; + struct devfreq *devfreq; + unsigned long current_freq; + unsigned long current_voltage; + struct monitor_dev_info *mdev_info; +#ifdef CONFIG_DEVFREQ_THERMAL + struct thermal_cooling_device *devfreq_cooling; +#endif +#endif + struct mali_pm_metrics_data mali_metrics; +}; +#endif + +/** @addtogroup _mali_osk_miscellaneous + * @{ */ + +/** @brief Struct with device specific configuration data + */ +typedef struct mali_gpu_device_data _mali_osk_device_data; + +#ifdef CONFIG_MALI_DT +/** @brief Initialize those device resources when we use device tree + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_resource_initialize(void); +#endif + +/** @brief Find Mali GPU HW resource + * + * @param addr Address of Mali GPU resource to find + * @param res Storage for resource information if resource is found. + * @return _MALI_OSK_ERR_OK on success, _MALI_OSK_ERR_ITEM_NOT_FOUND if resource is not found + */ +_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res); + + +/** @brief Find Mali GPU HW base address + * + * @return 0 if resources are found, otherwise the Mali GPU component with lowest address. + */ +uintptr_t _mali_osk_resource_base_address(void); + +/** @brief Find the specific GPU resource. + * + * @return value + * 0x400 if Mali 400 specific GPU resource identified + * 0x450 if Mali 450 specific GPU resource identified + * 0x470 if Mali 470 specific GPU resource identified + * + */ +u32 _mali_osk_identify_gpu_resource(void); + +/** @brief Retrieve the Mali GPU specific data + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_device_data_get(_mali_osk_device_data *data); + +/** @brief Find the pmu domain config from device data. + * + * @param domain_config_array used to store pmu domain config found in device data. + * @param array_size is the size of array domain_config_array. + */ +void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size); + +/** @brief Get Mali PMU switch delay + * + *@return pmu switch delay if it is configured + */ +u32 _mali_osk_get_pmu_switch_delay(void); + +/** @brief Determines if Mali GPU has been configured with shared interrupts. + * + * @return MALI_TRUE if shared interrupts, MALI_FALSE if not. + */ +mali_bool _mali_osk_shared_interrupts(void); + +/** @brief Initialize the gpu secure mode. + * The gpu secure mode will initially be in a disabled state. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_gpu_secure_mode_init(void); + +/** @brief Deinitialize the gpu secure mode. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_gpu_secure_mode_deinit(void); + +/** @brief Reset GPU and enable the gpu secure mode. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_gpu_reset_and_secure_mode_enable(void); + +/** @brief Reset GPU and disable the gpu secure mode. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_gpu_reset_and_secure_mode_disable(void); + +/** @brief Check if the gpu secure mode has been enabled. + * @return MALI_TRUE if enabled, otherwise MALI_FALSE. + */ +mali_bool _mali_osk_gpu_secure_mode_is_enabled(void); + +/** @brief Check if the gpu secure mode is supported. + * @return MALI_TRUE if supported, otherwise MALI_FALSE. + */ +mali_bool _mali_osk_gpu_secure_mode_is_supported(void); + + +/** @} */ /* end group _mali_osk_miscellaneous */ + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_OSK_MALI_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h new file mode 100755 index 000000000000..6e4583db1c80 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_profiling.h @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_OSK_PROFILING_H__ +#define __MALI_OSK_PROFILING_H__ + +#if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS) + +#include "mali_linux_trace.h" +#include "mali_profiling_events.h" +#include "mali_profiling_gator_api.h" + +#define MALI_PROFILING_MAX_BUFFER_ENTRIES 1048576 + +#define MALI_PROFILING_NO_HW_COUNTER = ((u32)-1) + +/** @defgroup _mali_osk_profiling External profiling connectivity + * @{ */ + +/** + * Initialize the profiling module. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start); + +/* + * Terminate the profiling module. + */ +void _mali_osk_profiling_term(void); + +/** + * Stop the profile sampling operation. + */ +void _mali_osk_profiling_stop_sampling(u32 pid); + +/** + * Start recording profiling data + * + * The specified limit will determine how large the capture buffer is. + * MALI_PROFILING_MAX_BUFFER_ENTRIES determines the maximum size allowed by the device driver. + * + * @param limit The desired maximum number of events to record on input, the actual maximum on output. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_profiling_start(u32 *limit); + +/** + * Add an profiling event + * + * @param event_id The event identificator. + * @param data0 First data parameter, depending on event_id specified. + * @param data1 Second data parameter, depending on event_id specified. + * @param data2 Third data parameter, depending on event_id specified. + * @param data3 Fourth data parameter, depending on event_id specified. + * @param data4 Fifth data parameter, depending on event_id specified. + */ +void _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4); + +/** + * Report a hardware counter event. + * + * @param counter_id The ID of the counter. + * @param value The value of the counter. + */ + +/* Call Linux tracepoint directly */ +#define _mali_osk_profiling_report_hw_counter(counter_id, value) trace_mali_hw_counter(counter_id, value) + +/** + * Report SW counters + * + * @param counters array of counter values + */ +void _mali_osk_profiling_report_sw_counters(u32 *counters); + +void _mali_osk_profiling_record_global_counters(int counter_id, u32 value); + +/** + * Stop recording profiling data + * + * @param count Returns the number of recorded events. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_profiling_stop(u32 *count); + +/** + * Retrieves the number of events that can be retrieved + * + * @return The number of recorded events that can be retrieved. + */ +u32 _mali_osk_profiling_get_count(void); + +/** + * Retrieve an event + * + * @param index Event index (start with 0 and continue until this function fails to retrieve all events) + * @param timestamp The timestamp for the retrieved event will be stored here. + * @param event_id The event ID for the retrieved event will be stored here. + * @param data The 5 data values for the retrieved event will be stored here. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]); + +/** + * Clear the recorded buffer. + * + * This is needed in order to start another recording. + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t _mali_osk_profiling_clear(void); + +/** + * Checks if a recording of profiling data is in progress + * + * @return MALI_TRUE if recording of profiling data is in progress, MALI_FALSE if not + */ +mali_bool _mali_osk_profiling_is_recording(void); + +/** + * Checks if profiling data is available for retrival + * + * @return MALI_TRUE if profiling data is avaiable, MALI_FALSE if not + */ +mali_bool _mali_osk_profiling_have_recording(void); + +/** @} */ /* end group _mali_osk_profiling */ + +#else /* defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_TRACEPOINTS) */ + +/* Dummy add_event, for when profiling is disabled. */ + +#define _mali_osk_profiling_add_event(event_id, data0, data1, data2, data3, data4) + +#endif /* defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_TRACEPOINTS) */ + +#endif /* __MALI_OSK_PROFILING_H__ */ + + diff --git a/drivers/gpu/arm/mali400/mali/common/mali_osk_types.h b/drivers/gpu/arm/mali400/mali/common/mali_osk_types.h new file mode 100755 index 000000000000..b6fa94ce16b3 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_osk_types.h @@ -0,0 +1,471 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_types.h + * Defines types of the OS abstraction layer for the kernel device driver (OSK) + */ + +#ifndef __MALI_OSK_TYPES_H__ +#define __MALI_OSK_TYPES_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @addtogroup uddapi Unified Device Driver (UDD) APIs + * + * @{ + */ + +/** + * @addtogroup oskapi UDD OS Abstraction for Kernel-side (OSK) APIs + * + * @{ + */ + +/** @defgroup _mali_osk_miscellaneous OSK Miscellaneous functions, constants and types + * @{ */ + +/* Define integer types used by OSK. Note: these currently clash with Linux so we only define them if not defined already */ +#ifndef __KERNEL__ +typedef unsigned char u8; +typedef signed char s8; +typedef unsigned short u16; +typedef signed short s16; +typedef unsigned int u32; +typedef signed int s32; +typedef unsigned long long u64; +#define BITS_PER_LONG (sizeof(long)*8) +#else +/* Ensure Linux types u32, etc. are defined */ +#include +#endif + +/** @brief Mali Boolean type which uses MALI_TRUE and MALI_FALSE + */ +typedef unsigned long mali_bool; + +#ifndef MALI_TRUE +#define MALI_TRUE ((mali_bool)1) +#endif + +#ifndef MALI_FALSE +#define MALI_FALSE ((mali_bool)0) +#endif + +#define MALI_HW_CORE_NO_COUNTER ((u32)-1) + + +#define MALI_S32_MAX 0x7fffffff + +/** + * @brief OSK Error codes + * + * Each OS may use its own set of error codes, and may require that the + * User/Kernel interface take certain error code. This means that the common + * error codes need to be sufficiently rich to pass the correct error code + * thorugh from the OSK to U/K layer, across all OSs. + * + * The result is that some error codes will appear redundant on some OSs. + * Under all OSs, the OSK layer must translate native OS error codes to + * _mali_osk_errcode_t codes. Similarly, the U/K layer must translate from + * _mali_osk_errcode_t codes to native OS error codes. + */ +typedef enum { + _MALI_OSK_ERR_OK = 0, /**< Success. */ + _MALI_OSK_ERR_FAULT = -1, /**< General non-success */ + _MALI_OSK_ERR_INVALID_FUNC = -2, /**< Invalid function requested through User/Kernel interface (e.g. bad IOCTL number) */ + _MALI_OSK_ERR_INVALID_ARGS = -3, /**< Invalid arguments passed through User/Kernel interface */ + _MALI_OSK_ERR_NOMEM = -4, /**< Insufficient memory */ + _MALI_OSK_ERR_TIMEOUT = -5, /**< Timeout occurred */ + _MALI_OSK_ERR_RESTARTSYSCALL = -6, /**< Special: On certain OSs, must report when an interruptable mutex is interrupted. Ignore otherwise. */ + _MALI_OSK_ERR_ITEM_NOT_FOUND = -7, /**< Table Lookup failed */ + _MALI_OSK_ERR_BUSY = -8, /**< Device/operation is busy. Try again later */ + _MALI_OSK_ERR_UNSUPPORTED = -9, /**< Optional part of the interface used, and is unsupported */ +} _mali_osk_errcode_t; + +/** @} */ /* end group _mali_osk_miscellaneous */ + +/** @defgroup _mali_osk_wq OSK work queues + * @{ */ + +/** @brief Private type for work objects */ +typedef struct _mali_osk_wq_work_s _mali_osk_wq_work_t; +typedef struct _mali_osk_wq_delayed_work_s _mali_osk_wq_delayed_work_t; + +/** @brief Work queue handler function + * + * This function type is called when the work is scheduled by the work queue, + * e.g. as an IRQ bottom-half handler. + * + * Refer to \ref _mali_osk_wq_schedule_work() for more information on the + * work-queue and work handlers. + * + * @param arg resource-specific data + */ +typedef void (*_mali_osk_wq_work_handler_t)(void *arg); + +/* @} */ /* end group _mali_osk_wq */ + +/** @defgroup _mali_osk_irq OSK IRQ handling + * @{ */ + +/** @brief Private type for IRQ handling objects */ +typedef struct _mali_osk_irq_t_struct _mali_osk_irq_t; + +/** @brief Optional function to trigger an irq from a resource + * + * This function is implemented by the common layer to allow probing of a resource's IRQ. + * @param arg resource-specific data */ +typedef void (*_mali_osk_irq_trigger_t)(void *arg); + +/** @brief Optional function to acknowledge an irq from a resource + * + * This function is implemented by the common layer to allow probing of a resource's IRQ. + * @param arg resource-specific data + * @return _MALI_OSK_ERR_OK if the IRQ was successful, or a suitable _mali_osk_errcode_t on failure. */ +typedef _mali_osk_errcode_t (*_mali_osk_irq_ack_t)(void *arg); + +/** @brief IRQ 'upper-half' handler callback. + * + * This function is implemented by the common layer to do the initial handling of a + * resource's IRQ. This maps on to the concept of an ISR that does the minimum + * work necessary before handing off to an IST. + * + * The communication of the resource-specific data from the ISR to the IST is + * handled by the OSK implementation. + * + * On most systems, the IRQ upper-half handler executes in IRQ context. + * Therefore, the system may have restrictions about what can be done in this + * context + * + * If an IRQ upper-half handler requires more work to be done than can be + * acheived in an IRQ context, then it may defer the work with + * _mali_osk_wq_schedule_work(). Refer to \ref _mali_osk_wq_create_work() for + * more information. + * + * @param arg resource-specific data + * @return _MALI_OSK_ERR_OK if the IRQ was correctly handled, or a suitable + * _mali_osk_errcode_t otherwise. + */ +typedef _mali_osk_errcode_t (*_mali_osk_irq_uhandler_t)(void *arg); + + +/** @} */ /* end group _mali_osk_irq */ + + +/** @defgroup _mali_osk_atomic OSK Atomic counters + * @{ */ + +/** @brief Public type of atomic counters + * + * This is public for allocation on stack. On systems that support it, this is just a single 32-bit value. + * On others, it could be encapsulating an object stored elsewhere. + * + * Regardless of implementation, the \ref _mali_osk_atomic functions \b must be used + * for all accesses to the variable's value, even if atomicity is not required. + * Do not access u.val or u.obj directly. + */ +typedef struct { + union { + u32 val; + void *obj; + } u; +} _mali_osk_atomic_t; +/** @} */ /* end group _mali_osk_atomic */ + + +/** @defgroup _mali_osk_lock OSK Mutual Exclusion Locks + * @{ */ + + +/** @brief OSK Mutual Exclusion Lock ordered list + * + * This lists the various types of locks in the system and is used to check + * that locks are taken in the correct order. + * + * - Holding more than one lock of the same order at the same time is not + * allowed. + * - Taking a lock of a lower order than the highest-order lock currently held + * is not allowed. + * + */ +typedef enum { + /* || Locks || */ + /* || must be || */ + /* _||_ taken in _||_ */ + /* \ / this \ / */ + /* \/ order! \/ */ + + _MALI_OSK_LOCK_ORDER_FIRST = 0, + + _MALI_OSK_LOCK_ORDER_SESSIONS, + _MALI_OSK_LOCK_ORDER_MEM_SESSION, + _MALI_OSK_LOCK_ORDER_MEM_INFO, + _MALI_OSK_LOCK_ORDER_MEM_PT_CACHE, + _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP, + _MALI_OSK_LOCK_ORDER_PM_EXECUTION, + _MALI_OSK_LOCK_ORDER_EXECUTOR, + _MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM, + _MALI_OSK_LOCK_ORDER_SCHEDULER, + _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED, + _MALI_OSK_LOCK_ORDER_PROFILING, + _MALI_OSK_LOCK_ORDER_L2, + _MALI_OSK_LOCK_ORDER_L2_COMMAND, + _MALI_OSK_LOCK_ORDER_UTILIZATION, + _MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS, + _MALI_OSK_LOCK_ORDER_PM_STATE, + + _MALI_OSK_LOCK_ORDER_LAST, +} _mali_osk_lock_order_t; + + +/** @brief OSK Mutual Exclusion Lock flags type + * + * - Any lock can use the order parameter. + */ +typedef enum { + _MALI_OSK_LOCKFLAG_UNORDERED = 0x1, /**< Indicate that the order of this lock should not be checked */ + _MALI_OSK_LOCKFLAG_ORDERED = 0x2, + /** @enum _mali_osk_lock_flags_t + * + * Flags from 0x10000--0x80000000 are RESERVED for User-mode */ + +} _mali_osk_lock_flags_t; + +/** @brief Mutual Exclusion Lock Mode Optimization hint + * + * The lock mode is used to implement the read/write locking of locks when we call + * functions _mali_osk_mutex_rw_init/wait/signal/term/. In this case, the RO mode can + * be used to allow multiple concurrent readers, but no writers. The RW mode is used for + * writers, and so will wait for all readers to release the lock (if any present). + * Further readers and writers will wait until the writer releases the lock. + * + * The mode is purely an optimization hint: for example, it is permissible for + * all locks to behave in RW mode, regardless of that supplied. + * + * It is an error to attempt to use locks in anything other that RW mode when + * call functions _mali_osk_mutex_rw_wait/signal(). + * + */ +typedef enum { + _MALI_OSK_LOCKMODE_UNDEF = -1, /**< Undefined lock mode. For internal use only */ + _MALI_OSK_LOCKMODE_RW = 0x0, /**< Read-write mode, default. All readers and writers are mutually-exclusive */ + _MALI_OSK_LOCKMODE_RO, /**< Read-only mode, to support multiple concurrent readers, but mutual exclusion in the presence of writers. */ + /** @enum _mali_osk_lock_mode_t + * + * Lock modes 0x40--0x7F are RESERVED for User-mode */ +} _mali_osk_lock_mode_t; + +/** @brief Private types for Mutual Exclusion lock objects */ +typedef struct _mali_osk_lock_debug_s _mali_osk_lock_debug_t; +typedef struct _mali_osk_spinlock_s _mali_osk_spinlock_t; +typedef struct _mali_osk_spinlock_irq_s _mali_osk_spinlock_irq_t; +typedef struct _mali_osk_mutex_s _mali_osk_mutex_t; +typedef struct _mali_osk_mutex_rw_s _mali_osk_mutex_rw_t; + +/** @} */ /* end group _mali_osk_lock */ + +/** @defgroup _mali_osk_low_level_memory OSK Low-level Memory Operations + * @{ */ + +/** + * @brief Private data type for use in IO accesses to/from devices. + * + * This represents some range that is accessible from the device. Examples + * include: + * - Device Registers, which could be readable and/or writeable. + * - Memory that the device has access to, for storing configuration structures. + * + * Access to this range must be made through the _mali_osk_mem_ioread32() and + * _mali_osk_mem_iowrite32() functions. + */ +typedef struct _mali_io_address *mali_io_address; + +/** @defgroup _MALI_OSK_CPU_PAGE CPU Physical page size macros. + * + * The order of the page size is supplied for + * ease of use by algorithms that might require it, since it is easier to know + * it ahead of time rather than calculating it. + * + * The Mali Page Mask macro masks off the lower bits of a physical address to + * give the start address of the page for that physical address. + * + * @note The Mali device driver code is designed for systems with 4KB page size. + * Changing these macros will not make the entire Mali device driver work with + * page sizes other than 4KB. + * + * @note The CPU Physical Page Size has been assumed to be the same as the Mali + * Physical Page Size. + * + * @{ + */ + +/** CPU Page Order, as log to base 2 of the Page size. @see _MALI_OSK_CPU_PAGE_SIZE */ +#define _MALI_OSK_CPU_PAGE_ORDER ((u32)12) +/** CPU Page Size, in bytes. */ +#define _MALI_OSK_CPU_PAGE_SIZE (((u32)1) << (_MALI_OSK_CPU_PAGE_ORDER)) +/** CPU Page Mask, which masks off the offset within a page */ +#define _MALI_OSK_CPU_PAGE_MASK (~((((u32)1) << (_MALI_OSK_CPU_PAGE_ORDER)) - ((u32)1))) +/** @} */ /* end of group _MALI_OSK_CPU_PAGE */ + +/** @defgroup _MALI_OSK_MALI_PAGE Mali Physical Page size macros + * + * Mali Physical page size macros. The order of the page size is supplied for + * ease of use by algorithms that might require it, since it is easier to know + * it ahead of time rather than calculating it. + * + * The Mali Page Mask macro masks off the lower bits of a physical address to + * give the start address of the page for that physical address. + * + * @note The Mali device driver code is designed for systems with 4KB page size. + * Changing these macros will not make the entire Mali device driver work with + * page sizes other than 4KB. + * + * @note The Mali Physical Page Size has been assumed to be the same as the CPU + * Physical Page Size. + * + * @{ + */ + +/** Mali Page Order, as log to base 2 of the Page size. @see _MALI_OSK_MALI_PAGE_SIZE */ +#define _MALI_OSK_MALI_PAGE_ORDER PAGE_SHIFT +/** Mali Page Size, in bytes. */ +#define _MALI_OSK_MALI_PAGE_SIZE PAGE_SIZE +/** Mali Page Mask, which masks off the offset within a page */ +#define _MALI_OSK_MALI_PAGE_MASK PAGE_MASK +/** @} */ /* end of group _MALI_OSK_MALI_PAGE*/ + +/** @brief flags for mapping a user-accessible memory range + * + * Where a function with prefix '_mali_osk_mem_mapregion' accepts flags as one + * of the function parameters, it will use one of these. These allow per-page + * control over mappings. Compare with the mali_memory_allocation_flag type, + * which acts over an entire range + * + * These may be OR'd together with bitwise OR (|), but must be cast back into + * the type after OR'ing. + */ +typedef enum { + _MALI_OSK_MEM_MAPREGION_FLAG_OS_ALLOCATED_PHYSADDR = 0x1, /**< Physical address is OS Allocated */ +} _mali_osk_mem_mapregion_flags_t; +/** @} */ /* end group _mali_osk_low_level_memory */ + +/** @defgroup _mali_osk_notification OSK Notification Queues + * @{ */ + +/** @brief Private type for notification queue objects */ +typedef struct _mali_osk_notification_queue_t_struct _mali_osk_notification_queue_t; + +/** @brief Public notification data object type */ +typedef struct _mali_osk_notification_t_struct { + u32 notification_type; /**< The notification type */ + u32 result_buffer_size; /**< Size of the result buffer to copy to user space */ + void *result_buffer; /**< Buffer containing any type specific data */ +} _mali_osk_notification_t; + +/** @} */ /* end group _mali_osk_notification */ + + +/** @defgroup _mali_osk_timer OSK Timer Callbacks + * @{ */ + +/** @brief Function to call when a timer expires + * + * When a timer expires, this function is called. Note that on many systems, + * a timer callback will be executed in IRQ context. Therefore, restrictions + * may apply on what can be done inside the timer callback. + * + * If a timer requires more work to be done than can be acheived in an IRQ + * context, then it may defer the work with a work-queue. For example, it may + * use \ref _mali_osk_wq_schedule_work() to make use of a bottom-half handler + * to carry out the remaining work. + * + * Stopping the timer with \ref _mali_osk_timer_del() blocks on compeletion of + * the callback. Therefore, the callback may not obtain any mutexes also held + * by any callers of _mali_osk_timer_del(). Otherwise, a deadlock may occur. + * + * @param arg Function-specific data */ +typedef void (*_mali_osk_timer_callback_t)(void *arg); + +/** @brief Private type for Timer Callback Objects */ +typedef struct _mali_osk_timer_t_struct _mali_osk_timer_t; +/** @} */ /* end group _mali_osk_timer */ + + +/** @addtogroup _mali_osk_list OSK Doubly-Linked Circular Lists + * @{ */ + +/** @brief Public List objects. + * + * To use, add a _mali_osk_list_t member to the structure that may become part + * of a list. When traversing the _mali_osk_list_t objects, use the + * _MALI_OSK_CONTAINER_OF() macro to recover the structure from its + *_mali_osk_list_t member + * + * Each structure may have multiple _mali_osk_list_t members, so that the + * structure is part of multiple lists. When traversing lists, ensure that the + * correct _mali_osk_list_t member is used, because type-checking will be + * lost by the compiler. + */ +typedef struct _mali_osk_list_s { + struct _mali_osk_list_s *next; + struct _mali_osk_list_s *prev; +} _mali_osk_list_t; +/** @} */ /* end group _mali_osk_list */ + +/** @addtogroup _mali_osk_miscellaneous + * @{ */ + +/** @brief resource description struct + * + * Platform independent representation of a Mali HW resource + */ +typedef struct _mali_osk_resource { + const char *description; /**< short description of the resource */ + uintptr_t base; /**< Physical base address of the resource, as seen by Mali resources. */ + const char *irq_name; /**< Name of irq belong to this resource */ + u32 irq; /**< IRQ number delivered to the CPU, or -1 to tell the driver to probe for it (if possible) */ +} _mali_osk_resource_t; +/** @} */ /* end group _mali_osk_miscellaneous */ + +/** @defgroup _mali_osk_wait_queue OSK Wait Queue functionality + * @{ */ +/** @brief Private type for wait queue objects */ +typedef struct _mali_osk_wait_queue_t_struct _mali_osk_wait_queue_t; +/** @} */ /* end group _mali_osk_wait_queue */ + +/** @} */ /* end group osuapi */ + +/** @} */ /* end group uddapi */ + +/** @brief Mali print ctx type which uses seq_file + */ +typedef struct seq_file _mali_osk_print_ctx; + +#define _MALI_OSK_BITMAP_INVALIDATE_INDEX -1 + +typedef struct _mali_osk_bitmap { + u32 reserve; + u32 last; + u32 max; + u32 avail; + _mali_osk_spinlock_t *lock; + unsigned long *table; +} _mali_osk_bitmap_t; + + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_OSK_TYPES_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm.c b/drivers/gpu/arm/mali400/mali/common/mali_pm.c new file mode 100755 index 000000000000..3989a33aeaef --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pm.c @@ -0,0 +1,1362 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_pm.h" +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_scheduler.h" +#include "mali_group.h" +#include "mali_pm_domain.h" +#include "mali_pmu.h" + +#include "mali_executor.h" +#include "mali_control_timer.h" + +#if defined(DEBUG) +u32 num_pm_runtime_resume = 0; +u32 num_pm_updates = 0; +u32 num_pm_updates_up = 0; +u32 num_pm_updates_down = 0; +#endif + +#define MALI_PM_DOMAIN_DUMMY_MASK (1 << MALI_DOMAIN_INDEX_DUMMY) + +/* lock protecting power state (including pm_domains) */ +static _mali_osk_spinlock_irq_t *pm_lock_state = NULL; + +/* the wanted domain mask (protected by pm_lock_state) */ +static u32 pd_mask_wanted = 0; + +/* used to deferring the actual power changes */ +static _mali_osk_wq_work_t *pm_work = NULL; + +/* lock protecting power change execution */ +static _mali_osk_mutex_t *pm_lock_exec = NULL; + +/* PMU domains which are actually powered on (protected by pm_lock_exec) */ +static u32 pmu_mask_current = 0; + +/* + * domains which marked as powered on (protected by pm_lock_exec) + * This can be different from pmu_mask_current right after GPU power on + * if the PMU domains default to powered up. + */ +static u32 pd_mask_current = 0; + +static u16 domain_config[MALI_MAX_NUMBER_OF_DOMAINS] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1 << MALI_DOMAIN_INDEX_DUMMY +}; + +/* The relative core power cost */ +#define MALI_GP_COST 3 +#define MALI_PP_COST 6 +#define MALI_L2_COST 1 + +/* + *We have MALI_MAX_NUMBER_OF_PP_PHYSICAL_CORES + 1 rows in this matrix + *because we mush store the mask of different pp cores: 0, 1, 2, 3, 4, 5, 6, 7, 8. + */ +static int mali_pm_domain_power_cost_result[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS + 1][MALI_MAX_NUMBER_OF_DOMAINS]; +/* + * Keep track of runtime PM state, so that we know + * how to resume during OS resume. + */ +#ifdef CONFIG_PM_RUNTIME +static mali_bool mali_pm_runtime_active = MALI_FALSE; +#else +/* when kernel don't enable PM_RUNTIME, set the flag always true, + * for GPU will not power off by runtime */ +static mali_bool mali_pm_runtime_active = MALI_TRUE; +#endif + +static void mali_pm_state_lock(void); +static void mali_pm_state_unlock(void); +static _mali_osk_errcode_t mali_pm_create_pm_domains(void); +static void mali_pm_set_pmu_domain_config(void); +static u32 mali_pm_get_registered_cores_mask(void); +static void mali_pm_update_sync_internal(void); +static mali_bool mali_pm_common_suspend(void); +static void mali_pm_update_work(void *data); +#if defined(DEBUG) +const char *mali_pm_mask_to_string(u32 mask); +const char *mali_pm_group_stats_to_string(void); +#endif + +_mali_osk_errcode_t mali_pm_initialize(void) +{ + _mali_osk_errcode_t err; + struct mali_pmu_core *pmu; + + pm_lock_state = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_PM_STATE); + if (NULL == pm_lock_state) { + mali_pm_terminate(); + return _MALI_OSK_ERR_FAULT; + } + + pm_lock_exec = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_PM_STATE); + if (NULL == pm_lock_exec) { + mali_pm_terminate(); + return _MALI_OSK_ERR_FAULT; + } + + pm_work = _mali_osk_wq_create_work(mali_pm_update_work, NULL); + if (NULL == pm_work) { + mali_pm_terminate(); + return _MALI_OSK_ERR_FAULT; + } + + pmu = mali_pmu_get_global_pmu_core(); + if (NULL != pmu) { + /* + * We have a Mali PMU, set the correct domain + * configuration (default or custom) + */ + + u32 registered_cores_mask; + + mali_pm_set_pmu_domain_config(); + + registered_cores_mask = mali_pm_get_registered_cores_mask(); + mali_pmu_set_registered_cores_mask(pmu, registered_cores_mask); + + MALI_DEBUG_ASSERT(0 == pd_mask_wanted); + } + + /* Create all power domains needed (at least one dummy domain) */ + err = mali_pm_create_pm_domains(); + if (_MALI_OSK_ERR_OK != err) { + mali_pm_terminate(); + return err; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_pm_terminate(void) +{ + if (NULL != pm_work) { + _mali_osk_wq_delete_work(pm_work); + pm_work = NULL; + } + + mali_pm_domain_terminate(); + + if (NULL != pm_lock_exec) { + _mali_osk_mutex_term(pm_lock_exec); + pm_lock_exec = NULL; + } + + if (NULL != pm_lock_state) { + _mali_osk_spinlock_irq_term(pm_lock_state); + pm_lock_state = NULL; + } +} + +struct mali_pm_domain *mali_pm_register_l2_cache(u32 domain_index, + struct mali_l2_cache_core *l2_cache) +{ + struct mali_pm_domain *domain; + + domain = mali_pm_domain_get_from_mask(domain_config[domain_index]); + if (NULL == domain) { + MALI_DEBUG_ASSERT(0 == domain_config[domain_index]); + domain = mali_pm_domain_get_from_index( + MALI_DOMAIN_INDEX_DUMMY); + domain_config[domain_index] = MALI_PM_DOMAIN_DUMMY_MASK; + } else { + MALI_DEBUG_ASSERT(0 != domain_config[domain_index]); + } + + MALI_DEBUG_ASSERT(NULL != domain); + + mali_pm_domain_add_l2_cache(domain, l2_cache); + + return domain; /* return the actual domain this was registered in */ +} + +struct mali_pm_domain *mali_pm_register_group(u32 domain_index, + struct mali_group *group) +{ + struct mali_pm_domain *domain; + + domain = mali_pm_domain_get_from_mask(domain_config[domain_index]); + if (NULL == domain) { + MALI_DEBUG_ASSERT(0 == domain_config[domain_index]); + domain = mali_pm_domain_get_from_index( + MALI_DOMAIN_INDEX_DUMMY); + domain_config[domain_index] = MALI_PM_DOMAIN_DUMMY_MASK; + } else { + MALI_DEBUG_ASSERT(0 != domain_config[domain_index]); + } + + MALI_DEBUG_ASSERT(NULL != domain); + + mali_pm_domain_add_group(domain, group); + + return domain; /* return the actual domain this was registered in */ +} + +mali_bool mali_pm_get_domain_refs(struct mali_pm_domain **domains, + struct mali_group **groups, + u32 num_domains) +{ + mali_bool ret = MALI_TRUE; /* Assume all is powered on instantly */ + u32 i; + + mali_pm_state_lock(); + + for (i = 0; i < num_domains; i++) { + MALI_DEBUG_ASSERT_POINTER(domains[i]); + pd_mask_wanted |= mali_pm_domain_ref_get(domains[i]); + if (MALI_FALSE == mali_pm_domain_power_is_on(domains[i])) { + /* + * Tell caller that the corresponding group + * was not already powered on. + */ + ret = MALI_FALSE; + } else { + /* + * There is a time gap between we power on the domain and + * set the power state of the corresponding groups to be on. + */ + if (NULL != groups[i] && + MALI_FALSE == mali_group_power_is_on(groups[i])) { + ret = MALI_FALSE; + } + } + } + + MALI_DEBUG_PRINT(3, ("PM: wanted domain mask = 0x%08X (get refs)\n", pd_mask_wanted)); + + mali_pm_state_unlock(); + + return ret; +} + +mali_bool mali_pm_put_domain_refs(struct mali_pm_domain **domains, + u32 num_domains) +{ + u32 mask = 0; + mali_bool ret; + u32 i; + + mali_pm_state_lock(); + + for (i = 0; i < num_domains; i++) { + MALI_DEBUG_ASSERT_POINTER(domains[i]); + mask |= mali_pm_domain_ref_put(domains[i]); + } + + if (0 == mask) { + /* return false, all domains should still stay on */ + ret = MALI_FALSE; + } else { + /* Assert that we are dealing with a change */ + MALI_DEBUG_ASSERT((pd_mask_wanted & mask) == mask); + + /* Update our desired domain mask */ + pd_mask_wanted &= ~mask; + + /* return true; one or more domains can now be powered down */ + ret = MALI_TRUE; + } + + MALI_DEBUG_PRINT(3, ("PM: wanted domain mask = 0x%08X (put refs)\n", pd_mask_wanted)); + + mali_pm_state_unlock(); + + return ret; +} + +void mali_pm_init_begin(void) +{ + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + _mali_osk_pm_dev_ref_get_sync(); + + /* Ensure all PMU domains are on */ + if (NULL != pmu) { + mali_pmu_power_up_all(pmu); + } +} + +void mali_pm_init_end(void) +{ + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + /* Ensure all PMU domains are off */ + if (NULL != pmu) { + mali_pmu_power_down_all(pmu); + } + + _mali_osk_pm_dev_ref_put(); +} + +void mali_pm_update_sync(void) +{ + mali_pm_exec_lock(); + + if (MALI_TRUE == mali_pm_runtime_active) { + /* + * Only update if GPU is powered on. + * Deactivation of the last group will result in both a + * deferred runtime PM suspend operation and + * deferred execution of this function. + * mali_pm_runtime_active will be false if runtime PM + * executed first and thus the GPU is now fully powered off. + */ + mali_pm_update_sync_internal(); + } + + mali_pm_exec_unlock(); +} + +void mali_pm_update_async(void) +{ + _mali_osk_wq_schedule_work(pm_work); +} + +void mali_pm_os_suspend(mali_bool os_suspend) +{ + int ret; + + MALI_DEBUG_PRINT(3, ("Mali PM: OS suspend\n")); + + /* Suspend execution of all jobs, and go to inactive state */ + mali_executor_suspend(); + + if (os_suspend) { + mali_control_timer_suspend(MALI_TRUE); + } + + mali_pm_exec_lock(); + + ret = mali_pm_common_suspend(); + + MALI_DEBUG_ASSERT(MALI_TRUE == ret); + MALI_IGNORE(ret); + + mali_pm_exec_unlock(); +} + +void mali_pm_os_resume(void) +{ + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + MALI_DEBUG_PRINT(3, ("Mali PM: OS resume\n")); + + mali_pm_exec_lock(); + +#if defined(DEBUG) + mali_pm_state_lock(); + + /* Assert that things are as we left them in os_suspend(). */ + MALI_DEBUG_ASSERT(0 == pd_mask_wanted); + MALI_DEBUG_ASSERT(0 == pd_mask_current); + MALI_DEBUG_ASSERT(0 == pmu_mask_current); + + MALI_DEBUG_ASSERT(MALI_TRUE == mali_pm_domain_all_unused()); + + mali_pm_state_unlock(); +#endif + + if (MALI_TRUE == mali_pm_runtime_active) { + /* Runtime PM was active, so reset PMU */ + if (NULL != pmu) { + mali_pmu_reset(pmu); + pmu_mask_current = mali_pmu_get_mask(pmu); + + MALI_DEBUG_PRINT(3, ("Mali PM: OS resume 0x%x \n", pmu_mask_current)); + } + + mali_pm_update_sync_internal(); + } + + mali_pm_exec_unlock(); + + /* Start executing jobs again */ + mali_executor_resume(); +} + +mali_bool mali_pm_runtime_suspend(void) +{ + mali_bool ret; + + MALI_DEBUG_PRINT(3, ("Mali PM: Runtime suspend\n")); + + mali_pm_exec_lock(); + + /* + * Put SW state directly into "off" state, and do not bother to power + * down each power domain, because entire GPU will be powered off + * when we return. + * For runtime PM suspend, in contrast to OS suspend, there is a race + * between this function and the mali_pm_update_sync_internal(), which + * is fine... + */ + ret = mali_pm_common_suspend(); + if (MALI_TRUE == ret) { + mali_pm_runtime_active = MALI_FALSE; + } else { + /* + * Process the "power up" instead, + * which could have been "lost" + */ + mali_pm_update_sync_internal(); + } + + mali_pm_exec_unlock(); + + return ret; +} + +void mali_pm_runtime_resume(void) +{ + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + mali_pm_exec_lock(); + + mali_pm_runtime_active = MALI_TRUE; + +#if defined(DEBUG) + ++num_pm_runtime_resume; + + mali_pm_state_lock(); + + /* + * Assert that things are as we left them in runtime_suspend(), + * except for pd_mask_wanted which normally will be the reason we + * got here (job queued => domains wanted) + */ + MALI_DEBUG_ASSERT(0 == pd_mask_current); + MALI_DEBUG_ASSERT(0 == pmu_mask_current); + + mali_pm_state_unlock(); +#endif + + if (NULL != pmu) { + mali_pmu_reset(pmu); + pmu_mask_current = mali_pmu_get_mask(pmu); + MALI_DEBUG_PRINT(3, ("Mali PM: Runtime resume 0x%x \n", pmu_mask_current)); + } + + /* + * Normally we are resumed because a job has just been queued. + * pd_mask_wanted should thus be != 0. + * It is however possible for others to take a Mali Runtime PM ref + * without having a job queued. + * We should however always call mali_pm_update_sync_internal(), + * because this will take care of any potential mismatch between + * pmu_mask_current and pd_mask_current. + */ + mali_pm_update_sync_internal(); + + mali_pm_exec_unlock(); +} + +#if MALI_STATE_TRACKING +u32 mali_pm_dump_state_domain(struct mali_pm_domain *domain, + char *buf, u32 size) +{ + int n = 0; + + n += _mali_osk_snprintf(buf + n, size - n, + "\tPower domain: id %u\n", + mali_pm_domain_get_id(domain)); + + n += _mali_osk_snprintf(buf + n, size - n, + "\t\tMask: 0x%04x\n", + mali_pm_domain_get_mask(domain)); + + n += _mali_osk_snprintf(buf + n, size - n, + "\t\tUse count: %u\n", + mali_pm_domain_get_use_count(domain)); + + n += _mali_osk_snprintf(buf + n, size - n, + "\t\tCurrent power state: %s\n", + (mali_pm_domain_get_mask(domain) & pd_mask_current) ? + "On" : "Off"); + + n += _mali_osk_snprintf(buf + n, size - n, + "\t\tWanted power state: %s\n", + (mali_pm_domain_get_mask(domain) & pd_mask_wanted) ? + "On" : "Off"); + + return n; +} +#endif + +static void mali_pm_state_lock(void) +{ + _mali_osk_spinlock_irq_lock(pm_lock_state); +} + +static void mali_pm_state_unlock(void) +{ + _mali_osk_spinlock_irq_unlock(pm_lock_state); +} + +void mali_pm_exec_lock(void) +{ + _mali_osk_mutex_wait(pm_lock_exec); +} + +void mali_pm_exec_unlock(void) +{ + _mali_osk_mutex_signal(pm_lock_exec); +} + +static void mali_pm_domain_power_up(u32 power_up_mask, + struct mali_group *groups_up[MALI_MAX_NUMBER_OF_GROUPS], + u32 *num_groups_up, + struct mali_l2_cache_core *l2_up[MALI_MAX_NUMBER_OF_L2_CACHE_CORES], + u32 *num_l2_up) +{ + u32 domain_bit; + u32 notify_mask = power_up_mask; + + MALI_DEBUG_ASSERT(0 != power_up_mask); + MALI_DEBUG_ASSERT_POINTER(groups_up); + MALI_DEBUG_ASSERT_POINTER(num_groups_up); + MALI_DEBUG_ASSERT(0 == *num_groups_up); + MALI_DEBUG_ASSERT_POINTER(l2_up); + MALI_DEBUG_ASSERT_POINTER(num_l2_up); + MALI_DEBUG_ASSERT(0 == *num_l2_up); + + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_state); + + MALI_DEBUG_PRINT(5, + ("PM update: Powering up domains: . [%s]\n", + mali_pm_mask_to_string(power_up_mask))); + + pd_mask_current |= power_up_mask; + + domain_bit = _mali_osk_fls(notify_mask); + while (0 != domain_bit) { + u32 domain_id = domain_bit - 1; + struct mali_pm_domain *domain = + mali_pm_domain_get_from_index( + domain_id); + struct mali_l2_cache_core *l2_cache; + struct mali_l2_cache_core *l2_cache_tmp; + struct mali_group *group; + struct mali_group *group_tmp; + + /* Mark domain as powered up */ + mali_pm_domain_set_power_on(domain, MALI_TRUE); + + /* + * Make a note of the L2 and/or group(s) to notify + * (need to release the PM state lock before doing so) + */ + + _MALI_OSK_LIST_FOREACHENTRY(l2_cache, + l2_cache_tmp, + mali_pm_domain_get_l2_cache_list( + domain), + struct mali_l2_cache_core, + pm_domain_list) { + MALI_DEBUG_ASSERT(*num_l2_up < + MALI_MAX_NUMBER_OF_L2_CACHE_CORES); + l2_up[*num_l2_up] = l2_cache; + (*num_l2_up)++; + } + + _MALI_OSK_LIST_FOREACHENTRY(group, + group_tmp, + mali_pm_domain_get_group_list(domain), + struct mali_group, + pm_domain_list) { + MALI_DEBUG_ASSERT(*num_groups_up < + MALI_MAX_NUMBER_OF_GROUPS); + groups_up[*num_groups_up] = group; + + (*num_groups_up)++; + } + + /* Remove current bit and find next */ + notify_mask &= ~(1 << (domain_id)); + domain_bit = _mali_osk_fls(notify_mask); + } +} +static void mali_pm_domain_power_down(u32 power_down_mask, + struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS], + u32 *num_groups_down, + struct mali_l2_cache_core *l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES], + u32 *num_l2_down) +{ + u32 domain_bit; + u32 notify_mask = power_down_mask; + + MALI_DEBUG_ASSERT(0 != power_down_mask); + MALI_DEBUG_ASSERT_POINTER(groups_down); + MALI_DEBUG_ASSERT_POINTER(num_groups_down); + MALI_DEBUG_ASSERT(0 == *num_groups_down); + MALI_DEBUG_ASSERT_POINTER(l2_down); + MALI_DEBUG_ASSERT_POINTER(num_l2_down); + MALI_DEBUG_ASSERT(0 == *num_l2_down); + + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_state); + + MALI_DEBUG_PRINT(5, + ("PM update: Powering down domains: [%s]\n", + mali_pm_mask_to_string(power_down_mask))); + + pd_mask_current &= ~power_down_mask; + + domain_bit = _mali_osk_fls(notify_mask); + while (0 != domain_bit) { + u32 domain_id = domain_bit - 1; + struct mali_pm_domain *domain = + mali_pm_domain_get_from_index(domain_id); + struct mali_l2_cache_core *l2_cache; + struct mali_l2_cache_core *l2_cache_tmp; + struct mali_group *group; + struct mali_group *group_tmp; + + /* Mark domain as powered down */ + mali_pm_domain_set_power_on(domain, MALI_FALSE); + + /* + * Make a note of the L2s and/or groups to notify + * (need to release the PM state lock before doing so) + */ + + _MALI_OSK_LIST_FOREACHENTRY(l2_cache, + l2_cache_tmp, + mali_pm_domain_get_l2_cache_list(domain), + struct mali_l2_cache_core, + pm_domain_list) { + MALI_DEBUG_ASSERT(*num_l2_down < + MALI_MAX_NUMBER_OF_L2_CACHE_CORES); + l2_down[*num_l2_down] = l2_cache; + (*num_l2_down)++; + } + + _MALI_OSK_LIST_FOREACHENTRY(group, + group_tmp, + mali_pm_domain_get_group_list(domain), + struct mali_group, + pm_domain_list) { + MALI_DEBUG_ASSERT(*num_groups_down < + MALI_MAX_NUMBER_OF_GROUPS); + groups_down[*num_groups_down] = group; + (*num_groups_down)++; + } + + /* Remove current bit and find next */ + notify_mask &= ~(1 << (domain_id)); + domain_bit = _mali_osk_fls(notify_mask); + } +} + +/* + * Execute pending power domain changes + * pm_lock_exec lock must be taken by caller. + */ +static void mali_pm_update_sync_internal(void) +{ + /* + * This should only be called in non-atomic context + * (normally as deferred work) + * + * Look at the pending power domain changes, and execute these. + * Make sure group and schedulers are notified about changes. + */ + + struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core(); + + u32 power_down_mask; + u32 power_up_mask; + + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); + +#if defined(DEBUG) + ++num_pm_updates; +#endif + + /* Hold PM state lock while we look at (and obey) the wanted state */ + mali_pm_state_lock(); + + MALI_DEBUG_PRINT(5, ("PM update pre: Wanted domain mask: .. [%s]\n", + mali_pm_mask_to_string(pd_mask_wanted))); + MALI_DEBUG_PRINT(5, ("PM update pre: Current domain mask: . [%s]\n", + mali_pm_mask_to_string(pd_mask_current))); + MALI_DEBUG_PRINT(5, ("PM update pre: Current PMU mask: .... [%s]\n", + mali_pm_mask_to_string(pmu_mask_current))); + MALI_DEBUG_PRINT(5, ("PM update pre: Group power stats: ... <%s>\n", + mali_pm_group_stats_to_string())); + + /* Figure out which cores we need to power on */ + power_up_mask = pd_mask_wanted & + (pd_mask_wanted ^ pd_mask_current); + + if (0 != power_up_mask) { + u32 power_up_mask_pmu; + struct mali_group *groups_up[MALI_MAX_NUMBER_OF_GROUPS]; + u32 num_groups_up = 0; + struct mali_l2_cache_core * + l2_up[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; + u32 num_l2_up = 0; + u32 i; + +#if defined(DEBUG) + ++num_pm_updates_up; +#endif + + /* + * Make sure dummy/global domain is always included when + * powering up, since this is controlled by runtime PM, + * and device power is on at this stage. + */ + power_up_mask |= MALI_PM_DOMAIN_DUMMY_MASK; + + /* Power up only real PMU domains */ + power_up_mask_pmu = power_up_mask & ~MALI_PM_DOMAIN_DUMMY_MASK; + + /* But not those that happen to be powered on already */ + power_up_mask_pmu &= (power_up_mask ^ pmu_mask_current) & + power_up_mask; + + if (0 != power_up_mask_pmu) { + MALI_DEBUG_ASSERT(NULL != pmu); + pmu_mask_current |= power_up_mask_pmu; + mali_pmu_power_up(pmu, power_up_mask_pmu); + } + + /* + * Put the domains themselves in power up state. + * We get the groups and L2s to notify in return. + */ + mali_pm_domain_power_up(power_up_mask, + groups_up, &num_groups_up, + l2_up, &num_l2_up); + + /* Need to unlock PM state lock before notifying L2 + groups */ + mali_pm_state_unlock(); + + /* Notify each L2 cache that we have be powered up */ + for (i = 0; i < num_l2_up; i++) { + mali_l2_cache_power_up(l2_up[i]); + } + + /* + * Tell execution module about all the groups we have + * powered up. Groups will be notified as a result of this. + */ + mali_executor_group_power_up(groups_up, num_groups_up); + + /* Lock state again before checking for power down */ + mali_pm_state_lock(); + } + + /* Figure out which cores we need to power off */ + power_down_mask = pd_mask_current & + (pd_mask_wanted ^ pd_mask_current); + + /* + * Never power down the dummy/global domain here. This is to be done + * from a suspend request (since this domain is only physicall powered + * down at that point) + */ + power_down_mask &= ~MALI_PM_DOMAIN_DUMMY_MASK; + + if (0 != power_down_mask) { + u32 power_down_mask_pmu; + struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS]; + u32 num_groups_down = 0; + struct mali_l2_cache_core * + l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; + u32 num_l2_down = 0; + u32 i; + +#if defined(DEBUG) + ++num_pm_updates_down; +#endif + + /* + * Put the domains themselves in power down state. + * We get the groups and L2s to notify in return. + */ + mali_pm_domain_power_down(power_down_mask, + groups_down, &num_groups_down, + l2_down, &num_l2_down); + + /* Need to unlock PM state lock before notifying L2 + groups */ + mali_pm_state_unlock(); + + /* + * Tell execution module about all the groups we will be + * powering down. Groups will be notified as a result of this. + */ + if (0 < num_groups_down) { + mali_executor_group_power_down(groups_down, num_groups_down); + } + + /* Notify each L2 cache that we will be powering down */ + for (i = 0; i < num_l2_down; i++) { + mali_l2_cache_power_down(l2_down[i]); + } + + /* + * Power down only PMU domains which should not stay on + * Some domains might for instance currently be incorrectly + * powered up if default domain power state is all on. + */ + power_down_mask_pmu = pmu_mask_current & (~pd_mask_current); + + if (0 != power_down_mask_pmu) { + MALI_DEBUG_ASSERT(NULL != pmu); + pmu_mask_current &= ~power_down_mask_pmu; + mali_pmu_power_down(pmu, power_down_mask_pmu); + + } + } else { + /* + * Power down only PMU domains which should not stay on + * Some domains might for instance currently be incorrectly + * powered up if default domain power state is all on. + */ + u32 power_down_mask_pmu; + + /* No need for state lock since we'll only update PMU */ + mali_pm_state_unlock(); + + power_down_mask_pmu = pmu_mask_current & (~pd_mask_current); + + if (0 != power_down_mask_pmu) { + MALI_DEBUG_ASSERT(NULL != pmu); + pmu_mask_current &= ~power_down_mask_pmu; + mali_pmu_power_down(pmu, power_down_mask_pmu); + } + } + + MALI_DEBUG_PRINT(5, ("PM update post: Current domain mask: . [%s]\n", + mali_pm_mask_to_string(pd_mask_current))); + MALI_DEBUG_PRINT(5, ("PM update post: Current PMU mask: .... [%s]\n", + mali_pm_mask_to_string(pmu_mask_current))); + MALI_DEBUG_PRINT(5, ("PM update post: Group power stats: ... <%s>\n", + mali_pm_group_stats_to_string())); +} + +static mali_bool mali_pm_common_suspend(void) +{ + mali_pm_state_lock(); + + if (0 != pd_mask_wanted) { + MALI_DEBUG_PRINT(5, ("PM: Aborting suspend operation\n\n\n")); + mali_pm_state_unlock(); + return MALI_FALSE; + } + + MALI_DEBUG_PRINT(5, ("PM suspend pre: Wanted domain mask: .. [%s]\n", + mali_pm_mask_to_string(pd_mask_wanted))); + MALI_DEBUG_PRINT(5, ("PM suspend pre: Current domain mask: . [%s]\n", + mali_pm_mask_to_string(pd_mask_current))); + MALI_DEBUG_PRINT(5, ("PM suspend pre: Current PMU mask: .... [%s]\n", + mali_pm_mask_to_string(pmu_mask_current))); + MALI_DEBUG_PRINT(5, ("PM suspend pre: Group power stats: ... <%s>\n", + mali_pm_group_stats_to_string())); + + if (0 != pd_mask_current) { + /* + * We have still some domains powered on. + * It is for instance very normal that at least the + * dummy/global domain is marked as powered on at this point. + * (because it is physically powered on until this function + * returns) + */ + + struct mali_group *groups_down[MALI_MAX_NUMBER_OF_GROUPS]; + u32 num_groups_down = 0; + struct mali_l2_cache_core * + l2_down[MALI_MAX_NUMBER_OF_L2_CACHE_CORES]; + u32 num_l2_down = 0; + u32 i; + + /* + * Put the domains themselves in power down state. + * We get the groups and L2s to notify in return. + */ + mali_pm_domain_power_down(pd_mask_current, + groups_down, + &num_groups_down, + l2_down, + &num_l2_down); + + MALI_DEBUG_ASSERT(0 == pd_mask_current); + MALI_DEBUG_ASSERT(MALI_TRUE == mali_pm_domain_all_unused()); + + /* Need to unlock PM state lock before notifying L2 + groups */ + mali_pm_state_unlock(); + + /* + * Tell execution module about all the groups we will be + * powering down. Groups will be notified as a result of this. + */ + if (0 < num_groups_down) { + mali_executor_group_power_down(groups_down, num_groups_down); + } + + /* Notify each L2 cache that we will be powering down */ + for (i = 0; i < num_l2_down; i++) { + mali_l2_cache_power_down(l2_down[i]); + } + + pmu_mask_current = 0; + } else { + MALI_DEBUG_ASSERT(0 == pmu_mask_current); + + MALI_DEBUG_ASSERT(MALI_TRUE == mali_pm_domain_all_unused()); + + mali_pm_state_unlock(); + } + + MALI_DEBUG_PRINT(5, ("PM suspend post: Current domain mask: [%s]\n", + mali_pm_mask_to_string(pd_mask_current))); + MALI_DEBUG_PRINT(5, ("PM suspend post: Current PMU mask: ... [%s]\n", + mali_pm_mask_to_string(pmu_mask_current))); + MALI_DEBUG_PRINT(5, ("PM suspend post: Group power stats: .. <%s>\n", + mali_pm_group_stats_to_string())); + + return MALI_TRUE; +} + +static void mali_pm_update_work(void *data) +{ + MALI_IGNORE(data); + mali_pm_update_sync(); +} + +static _mali_osk_errcode_t mali_pm_create_pm_domains(void) +{ + int i; + + /* Create all domains (including dummy domain) */ + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (0x0 == domain_config[i]) continue; + + if (NULL == mali_pm_domain_create(domain_config[i])) { + return _MALI_OSK_ERR_NOMEM; + } + } + + return _MALI_OSK_ERR_OK; +} + +static void mali_pm_set_default_pm_domain_config(void) +{ + MALI_DEBUG_ASSERT(0 != _mali_osk_resource_base_address()); + + /* GP core */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_GP, NULL)) { + domain_config[MALI_DOMAIN_INDEX_GP] = 0x01; + } + + /* PP0 - PP3 core */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP0, NULL)) { + if (mali_is_mali400()) { + domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 2; + } else if (mali_is_mali450()) { + domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 1; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP0] = 0x01 << 0; + } + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP1, NULL)) { + if (mali_is_mali400()) { + domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 3; + } else if (mali_is_mali450()) { + domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 2; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP1] = 0x01 << 1; + } + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP2, NULL)) { + if (mali_is_mali400()) { + domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 4; + } else if (mali_is_mali450()) { + domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 2; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP2] = 0x01 << 1; + } + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP3, NULL)) { + if (mali_is_mali400()) { + domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 5; + } else if (mali_is_mali450()) { + domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 2; + } else if (mali_is_mali470()) { + domain_config[MALI_DOMAIN_INDEX_PP3] = 0x01 << 1; + } + } + + /* PP4 - PP7 */ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP4, NULL)) { + domain_config[MALI_DOMAIN_INDEX_PP4] = 0x01 << 3; + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP5, NULL)) { + domain_config[MALI_DOMAIN_INDEX_PP5] = 0x01 << 3; + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP6, NULL)) { + domain_config[MALI_DOMAIN_INDEX_PP6] = 0x01 << 3; + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI_OFFSET_PP7, NULL)) { + domain_config[MALI_DOMAIN_INDEX_PP7] = 0x01 << 3; + } + + /* L2gp/L2PP0/L2PP4 */ + if (mali_is_mali400()) { + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI400_OFFSET_L2_CACHE0, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L20] = 0x01 << 1; + } + } else if (mali_is_mali450()) { + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI450_OFFSET_L2_CACHE0, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L20] = 0x01 << 0; + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI450_OFFSET_L2_CACHE1, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L21] = 0x01 << 1; + } + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI450_OFFSET_L2_CACHE2, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L22] = 0x01 << 3; + } + } else if (mali_is_mali470()) { + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find( + MALI470_OFFSET_L2_CACHE1, NULL)) { + domain_config[MALI_DOMAIN_INDEX_L21] = 0x01 << 0; + } + } +} + +static u32 mali_pm_get_registered_cores_mask(void) +{ + int i = 0; + u32 mask = 0; + + for (i = 0; i < MALI_DOMAIN_INDEX_DUMMY; i++) { + mask |= domain_config[i]; + } + + return mask; +} + +static void mali_pm_set_pmu_domain_config(void) +{ + int i = 0; + + _mali_osk_device_data_pmu_config_get(domain_config, MALI_MAX_NUMBER_OF_DOMAINS - 1); + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS - 1; i++) { + if (0 != domain_config[i]) { + MALI_DEBUG_PRINT(2, ("Using customer pmu config:\n")); + break; + } + } + + if (MALI_MAX_NUMBER_OF_DOMAINS - 1 == i) { + MALI_DEBUG_PRINT(2, ("Using hw detect pmu config:\n")); + mali_pm_set_default_pm_domain_config(); + } + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS - 1; i++) { + if (domain_config[i]) { + MALI_DEBUG_PRINT(2, ("domain_config[%d] = 0x%x \n", i, domain_config[i])); + } + } + /* Can't override dummy domain mask */ + domain_config[MALI_DOMAIN_INDEX_DUMMY] = + 1 << MALI_DOMAIN_INDEX_DUMMY; +} + +#if defined(DEBUG) +const char *mali_pm_mask_to_string(u32 mask) +{ + static char bit_str[MALI_MAX_NUMBER_OF_DOMAINS + 1]; + int bit; + int str_pos = 0; + + /* Must be protected by lock since we use shared string buffer */ + if (NULL != pm_lock_exec) { + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); + } + + for (bit = MALI_MAX_NUMBER_OF_DOMAINS - 1; bit >= 0; bit--) { + if (mask & (1 << bit)) { + bit_str[str_pos] = 'X'; + } else { + bit_str[str_pos] = '-'; + } + str_pos++; + } + + bit_str[MALI_MAX_NUMBER_OF_DOMAINS] = '\0'; + + return bit_str; +} + +const char *mali_pm_group_stats_to_string(void) +{ + static char bit_str[MALI_MAX_NUMBER_OF_GROUPS + 1]; + u32 num_groups = mali_group_get_glob_num_groups(); + u32 i; + + /* Must be protected by lock since we use shared string buffer */ + if (NULL != pm_lock_exec) { + MALI_DEBUG_ASSERT_LOCK_HELD(pm_lock_exec); + } + + for (i = 0; i < num_groups && i < MALI_MAX_NUMBER_OF_GROUPS; i++) { + struct mali_group *group; + + group = mali_group_get_glob_group(i); + + if (MALI_TRUE == mali_group_power_is_on(group)) { + bit_str[i] = 'X'; + } else { + bit_str[i] = '-'; + } + } + + bit_str[i] = '\0'; + + return bit_str; +} +#endif + +/* + * num_pp is the number of PP cores which will be powered on given this mask + * cost is the total power cost of cores which will be powered on given this mask + */ +static void mali_pm_stat_from_mask(u32 mask, u32 *num_pp, u32 *cost) +{ + u32 i; + + /* loop through all cores */ + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (!(domain_config[i] & mask)) { + continue; + } + + switch (i) { + case MALI_DOMAIN_INDEX_GP: + *cost += MALI_GP_COST; + + break; + case MALI_DOMAIN_INDEX_PP0: /* Fall through */ + case MALI_DOMAIN_INDEX_PP1: /* Fall through */ + case MALI_DOMAIN_INDEX_PP2: /* Fall through */ + case MALI_DOMAIN_INDEX_PP3: + if (mali_is_mali400()) { + if ((domain_config[MALI_DOMAIN_INDEX_L20] & mask) + || (domain_config[MALI_DOMAIN_INDEX_DUMMY] + == domain_config[MALI_DOMAIN_INDEX_L20])) { + *num_pp += 1; + } + } else { + if ((domain_config[MALI_DOMAIN_INDEX_L21] & mask) + || (domain_config[MALI_DOMAIN_INDEX_DUMMY] + == domain_config[MALI_DOMAIN_INDEX_L21])) { + *num_pp += 1; + } + } + + *cost += MALI_PP_COST; + break; + case MALI_DOMAIN_INDEX_PP4: /* Fall through */ + case MALI_DOMAIN_INDEX_PP5: /* Fall through */ + case MALI_DOMAIN_INDEX_PP6: /* Fall through */ + case MALI_DOMAIN_INDEX_PP7: + MALI_DEBUG_ASSERT(mali_is_mali450()); + + if ((domain_config[MALI_DOMAIN_INDEX_L22] & mask) + || (domain_config[MALI_DOMAIN_INDEX_DUMMY] + == domain_config[MALI_DOMAIN_INDEX_L22])) { + *num_pp += 1; + } + + *cost += MALI_PP_COST; + break; + case MALI_DOMAIN_INDEX_L20: /* Fall through */ + case MALI_DOMAIN_INDEX_L21: /* Fall through */ + case MALI_DOMAIN_INDEX_L22: + *cost += MALI_L2_COST; + + break; + } + } +} + +void mali_pm_power_cost_setup(void) +{ + /* + * Two parallel arrays which store the best domain mask and its cost + * The index is the number of PP cores, E.g. Index 0 is for 1 PP option, + * might have mask 0x2 and with cost of 1, lower cost is better + */ + u32 best_mask[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS] = { 0 }; + u32 best_cost[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS] = { 0 }; + /* Array cores_in_domain is used to store the total pp cores in each pm domain. */ + u32 cores_in_domain[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 }; + /* Domain_count is used to represent the max domain we have.*/ + u32 max_domain_mask = 0; + u32 max_domain_id = 0; + u32 always_on_pp_cores = 0; + + u32 num_pp, cost, mask; + u32 i, j , k; + + /* Initialize statistics */ + for (i = 0; i < MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; i++) { + best_mask[i] = 0; + best_cost[i] = 0xFFFFFFFF; /* lower cost is better */ + } + + for (i = 0; i < MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS + 1; i++) { + for (j = 0; j < MALI_MAX_NUMBER_OF_DOMAINS; j++) { + mali_pm_domain_power_cost_result[i][j] = 0; + } + } + + /* Caculate number of pp cores of a given domain config. */ + for (i = MALI_DOMAIN_INDEX_PP0; i <= MALI_DOMAIN_INDEX_PP7; i++) { + if (0 < domain_config[i]) { + /* Get the max domain mask value used to caculate power cost + * and we don't count in always on pp cores. */ + if (MALI_PM_DOMAIN_DUMMY_MASK != domain_config[i] + && max_domain_mask < domain_config[i]) { + max_domain_mask = domain_config[i]; + } + + if (MALI_PM_DOMAIN_DUMMY_MASK == domain_config[i]) { + always_on_pp_cores++; + } + } + } + max_domain_id = _mali_osk_fls(max_domain_mask); + + /* + * Try all combinations of power domains and check how many PP cores + * they have and their power cost. + */ + for (mask = 0; mask < (1 << max_domain_id); mask++) { + num_pp = 0; + cost = 0; + + mali_pm_stat_from_mask(mask, &num_pp, &cost); + + /* This mask is usable for all MP1 up to num_pp PP cores, check statistics for all */ + for (i = 0; i < num_pp; i++) { + if (best_cost[i] >= cost) { + best_cost[i] = cost; + best_mask[i] = mask; + } + } + } + + /* + * If we want to enable x pp cores, if x is less than number of always_on pp cores, + * all of pp cores we will enable must be always_on pp cores. + */ + for (i = 0; i < mali_executor_get_num_cores_total(); i++) { + if (i < always_on_pp_cores) { + mali_pm_domain_power_cost_result[i + 1][MALI_MAX_NUMBER_OF_DOMAINS - 1] + = i + 1; + } else { + mali_pm_domain_power_cost_result[i + 1][MALI_MAX_NUMBER_OF_DOMAINS - 1] + = always_on_pp_cores; + } + } + + /* In this loop, variable i represent for the number of non-always on pp cores we want to enabled. */ + for (i = 0; i < (mali_executor_get_num_cores_total() - always_on_pp_cores); i++) { + if (best_mask[i] == 0) { + /* This MP variant is not available */ + continue; + } + + for (j = 0; j < MALI_MAX_NUMBER_OF_DOMAINS; j++) { + cores_in_domain[j] = 0; + } + + for (j = MALI_DOMAIN_INDEX_PP0; j <= MALI_DOMAIN_INDEX_PP7; j++) { + if (0 < domain_config[j] + && (MALI_PM_DOMAIN_DUMMY_MASK != domain_config[i])) { + cores_in_domain[_mali_osk_fls(domain_config[j]) - 1]++; + } + } + + /* In this loop, j represent for the number we have already enabled.*/ + for (j = 0; j <= i;) { + /* j used to visit all of domain to get the number of pp cores remained in it. */ + for (k = 0; k < max_domain_id; k++) { + /* If domain k in best_mask[i] is enabled and this domain has extra pp cores, + * we know we must pick at least one pp core from this domain. + * And then we move to next enabled pm domain. */ + if ((best_mask[i] & (0x1 << k)) && (0 < cores_in_domain[k])) { + cores_in_domain[k]--; + mali_pm_domain_power_cost_result[always_on_pp_cores + i + 1][k]++; + j++; + if (j > i) { + break; + } + } + } + } + } +} + +/* + * When we are doing core scaling, + * this function is called to return the best mask to + * achieve the best pp group power cost. + */ +void mali_pm_get_best_power_cost_mask(int num_requested, int *dst) +{ + MALI_DEBUG_ASSERT((mali_executor_get_num_cores_total() >= num_requested) && (0 <= num_requested)); + + _mali_osk_memcpy(dst, mali_pm_domain_power_cost_result[num_requested], MALI_MAX_NUMBER_OF_DOMAINS * sizeof(int)); +} + +u32 mali_pm_get_current_mask(void) +{ + return pd_mask_current; +} + +u32 mali_pm_get_wanted_mask(void) +{ + return pd_mask_wanted; +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm.h b/drivers/gpu/arm/mali400/mali/common/mali_pm.h new file mode 100755 index 000000000000..dac69958e034 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pm.h @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PM_H__ +#define __MALI_PM_H__ + +#include "mali_osk.h" +#include "mali_pm_domain.h" + +#define MALI_DOMAIN_INDEX_GP 0 +#define MALI_DOMAIN_INDEX_PP0 1 +#define MALI_DOMAIN_INDEX_PP1 2 +#define MALI_DOMAIN_INDEX_PP2 3 +#define MALI_DOMAIN_INDEX_PP3 4 +#define MALI_DOMAIN_INDEX_PP4 5 +#define MALI_DOMAIN_INDEX_PP5 6 +#define MALI_DOMAIN_INDEX_PP6 7 +#define MALI_DOMAIN_INDEX_PP7 8 +#define MALI_DOMAIN_INDEX_L20 9 +#define MALI_DOMAIN_INDEX_L21 10 +#define MALI_DOMAIN_INDEX_L22 11 +/* + * The dummy domain is used when there is no physical power domain + * (e.g. no PMU or always on cores) + */ +#define MALI_DOMAIN_INDEX_DUMMY 12 +#define MALI_MAX_NUMBER_OF_DOMAINS 13 + +/** + * Initialize the Mali PM module + * + * PM module covers Mali PM core, PM domains and Mali PMU + */ +_mali_osk_errcode_t mali_pm_initialize(void); + +/** + * Terminate the Mali PM module + */ +void mali_pm_terminate(void); + +void mali_pm_exec_lock(void); +void mali_pm_exec_unlock(void); + + +struct mali_pm_domain *mali_pm_register_l2_cache(u32 domain_index, + struct mali_l2_cache_core *l2_cache); +struct mali_pm_domain *mali_pm_register_group(u32 domain_index, + struct mali_group *group); + +mali_bool mali_pm_get_domain_refs(struct mali_pm_domain **domains, + struct mali_group **groups, + u32 num_domains); +mali_bool mali_pm_put_domain_refs(struct mali_pm_domain **domains, + u32 num_domains); + +void mali_pm_init_begin(void); +void mali_pm_init_end(void); + +void mali_pm_update_sync(void); +void mali_pm_update_async(void); + +/* Callback functions for system power management */ +void mali_pm_os_suspend(mali_bool os_suspend); +void mali_pm_os_resume(void); + +mali_bool mali_pm_runtime_suspend(void); +void mali_pm_runtime_resume(void); + +#if MALI_STATE_TRACKING +u32 mali_pm_dump_state_domain(struct mali_pm_domain *domain, + char *buf, u32 size); +#endif + +void mali_pm_power_cost_setup(void); + +void mali_pm_get_best_power_cost_mask(int num_requested, int *dst); + +#if defined(DEBUG) +const char *mali_pm_mask_to_string(u32 mask); +#endif + +u32 mali_pm_get_current_mask(void); +u32 mali_pm_get_wanted_mask(void); +#endif /* __MALI_PM_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c new file mode 100755 index 000000000000..8290f7d88f6a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.c @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_pm_domain.h" +#include "mali_pmu.h" +#include "mali_group.h" +#include "mali_pm.h" + +static struct mali_pm_domain *mali_pm_domains[MALI_MAX_NUMBER_OF_DOMAINS] = +{ NULL, }; + +void mali_pm_domain_initialize(void) +{ + /* Domains will be initialized/created on demand */ +} + +void mali_pm_domain_terminate(void) +{ + int i; + + /* Delete all domains that has been created */ + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + mali_pm_domain_delete(mali_pm_domains[i]); + mali_pm_domains[i] = NULL; + } +} + +struct mali_pm_domain *mali_pm_domain_create(u32 pmu_mask) +{ + struct mali_pm_domain *domain = NULL; + u32 domain_id = 0; + + domain = mali_pm_domain_get_from_mask(pmu_mask); + if (NULL != domain) return domain; + + MALI_DEBUG_PRINT(2, + ("Mali PM domain: Creating Mali PM domain (mask=0x%08X)\n", + pmu_mask)); + + domain = (struct mali_pm_domain *)_mali_osk_malloc( + sizeof(struct mali_pm_domain)); + if (NULL != domain) { + domain->power_is_on = MALI_FALSE; + domain->pmu_mask = pmu_mask; + domain->use_count = 0; + _mali_osk_list_init(&domain->group_list); + _mali_osk_list_init(&domain->l2_cache_list); + + domain_id = _mali_osk_fls(pmu_mask) - 1; + /* Verify the domain_id */ + MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > domain_id); + /* Verify that pmu_mask only one bit is set */ + MALI_DEBUG_ASSERT((1 << domain_id) == pmu_mask); + mali_pm_domains[domain_id] = domain; + + return domain; + } else { + MALI_DEBUG_PRINT_ERROR(("Unable to create PM domain\n")); + } + + return NULL; +} + +void mali_pm_domain_delete(struct mali_pm_domain *domain) +{ + if (NULL == domain) { + return; + } + + _mali_osk_list_delinit(&domain->group_list); + _mali_osk_list_delinit(&domain->l2_cache_list); + + _mali_osk_free(domain); +} + +void mali_pm_domain_add_group(struct mali_pm_domain *domain, + struct mali_group *group) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + MALI_DEBUG_ASSERT_POINTER(group); + + /* + * Use addtail because virtual group is created last and it needs + * to be at the end of the list (in order to be activated after + * all children. + */ + _mali_osk_list_addtail(&group->pm_domain_list, &domain->group_list); +} + +void mali_pm_domain_add_l2_cache(struct mali_pm_domain *domain, + struct mali_l2_cache_core *l2_cache) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + MALI_DEBUG_ASSERT_POINTER(l2_cache); + _mali_osk_list_add(&l2_cache->pm_domain_list, &domain->l2_cache_list); +} + +struct mali_pm_domain *mali_pm_domain_get_from_mask(u32 mask) +{ + u32 id = 0; + + if (0 == mask) { + return NULL; + } + + id = _mali_osk_fls(mask) - 1; + + MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > id); + /* Verify that pmu_mask only one bit is set */ + MALI_DEBUG_ASSERT((1 << id) == mask); + + return mali_pm_domains[id]; +} + +struct mali_pm_domain *mali_pm_domain_get_from_index(u32 id) +{ + MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > id); + + return mali_pm_domains[id]; +} + +u32 mali_pm_domain_ref_get(struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + + if (0 == domain->use_count) { + _mali_osk_pm_dev_ref_get_async(); + } + + ++domain->use_count; + MALI_DEBUG_PRINT(4, ("PM domain %p: ref_get, use_count => %u\n", domain, domain->use_count)); + + /* Return our mask so caller can check this against wanted mask */ + return domain->pmu_mask; +} + +u32 mali_pm_domain_ref_put(struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + + --domain->use_count; + MALI_DEBUG_PRINT(4, ("PM domain %p: ref_put, use_count => %u\n", domain, domain->use_count)); + + if (0 == domain->use_count) { + _mali_osk_pm_dev_ref_put(); + } + + /* + * Return the PMU mask which now could be be powered down + * (the bit for this domain). + * This is the responsibility of the caller (mali_pm) + */ + return (0 == domain->use_count ? domain->pmu_mask : 0); +} + +#if MALI_STATE_TRACKING +u32 mali_pm_domain_get_id(struct mali_pm_domain *domain) +{ + u32 id = 0; + + MALI_DEBUG_ASSERT_POINTER(domain); + MALI_DEBUG_ASSERT(0 != domain->pmu_mask); + + id = _mali_osk_fls(domain->pmu_mask) - 1; + + MALI_DEBUG_ASSERT(MALI_MAX_NUMBER_OF_DOMAINS > id); + /* Verify that pmu_mask only one bit is set */ + MALI_DEBUG_ASSERT((1 << id) == domain->pmu_mask); + /* Verify that we have stored the domain at right id/index */ + MALI_DEBUG_ASSERT(domain == mali_pm_domains[id]); + + return id; +} +#endif + +#if defined(DEBUG) +mali_bool mali_pm_domain_all_unused(void) +{ + int i; + + for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) { + if (NULL == mali_pm_domains[i]) { + /* Nothing to check */ + continue; + } + + if (MALI_TRUE == mali_pm_domains[i]->power_is_on) { + /* Not ready for suspend! */ + return MALI_FALSE; + } + + if (0 != mali_pm_domains[i]->use_count) { + /* Not ready for suspend! */ + return MALI_FALSE; + } + } + + return MALI_TRUE; +} +#endif diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h new file mode 100755 index 000000000000..5776abe39f3d --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pm_domain.h @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PM_DOMAIN_H__ +#define __MALI_PM_DOMAIN_H__ + +#include "mali_kernel_common.h" +#include "mali_osk.h" + +#include "mali_l2_cache.h" +#include "mali_group.h" +#include "mali_pmu.h" + +/* Instances are protected by PM state lock */ +struct mali_pm_domain { + mali_bool power_is_on; + s32 use_count; + u32 pmu_mask; + + /* Zero or more groups can belong to this domain */ + _mali_osk_list_t group_list; + + /* Zero or more L2 caches can belong to this domain */ + _mali_osk_list_t l2_cache_list; +}; + + +void mali_pm_domain_initialize(void); +void mali_pm_domain_terminate(void); + +struct mali_pm_domain *mali_pm_domain_create(u32 pmu_mask); +void mali_pm_domain_delete(struct mali_pm_domain *domain); + +void mali_pm_domain_add_l2_cache( + struct mali_pm_domain *domain, + struct mali_l2_cache_core *l2_cache); +void mali_pm_domain_add_group(struct mali_pm_domain *domain, + struct mali_group *group); + +struct mali_pm_domain *mali_pm_domain_get_from_mask(u32 mask); +struct mali_pm_domain *mali_pm_domain_get_from_index(u32 id); + +/* Ref counting */ +u32 mali_pm_domain_ref_get(struct mali_pm_domain *domain); +u32 mali_pm_domain_ref_put(struct mali_pm_domain *domain); + +MALI_STATIC_INLINE _mali_osk_list_t *mali_pm_domain_get_group_list( + struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + return &domain->group_list; +} + +MALI_STATIC_INLINE _mali_osk_list_t *mali_pm_domain_get_l2_cache_list( + struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + return &domain->l2_cache_list; +} + +MALI_STATIC_INLINE mali_bool mali_pm_domain_power_is_on( + struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + return domain->power_is_on; +} + +MALI_STATIC_INLINE void mali_pm_domain_set_power_on( + struct mali_pm_domain *domain, + mali_bool power_is_on) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + domain->power_is_on = power_is_on; +} + +MALI_STATIC_INLINE u32 mali_pm_domain_get_use_count( + struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + return domain->use_count; +} + +#if MALI_STATE_TRACKING +u32 mali_pm_domain_get_id(struct mali_pm_domain *domain); + +MALI_STATIC_INLINE u32 mali_pm_domain_get_mask(struct mali_pm_domain *domain) +{ + MALI_DEBUG_ASSERT_POINTER(domain); + return domain->pmu_mask; +} +#endif + +#if defined(DEBUG) +mali_bool mali_pm_domain_all_unused(void); +#endif + +#endif /* __MALI_PM_DOMAIN_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.c b/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.c new file mode 100755 index 000000000000..cf74823230f7 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.c @@ -0,0 +1,255 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include "mali_pm_metrics.h" +#include "mali_osk_locks.h" +#include "mali_osk_mali.h" +#include + +#define MALI_PM_TIME_SHIFT 0 +#define MALI_UTILIZATION_MAX_PERIOD 80000000/* ns = 100ms */ + +_mali_osk_errcode_t mali_pm_metrics_init(struct mali_device *mdev) +{ + int i = 0; + + MALI_DEBUG_ASSERT(mdev != NULL); + + mdev->mali_metrics.time_period_start = ktime_get(); + mdev->mali_metrics.time_period_start_gp = mdev->mali_metrics.time_period_start; + mdev->mali_metrics.time_period_start_pp = mdev->mali_metrics.time_period_start; + + mdev->mali_metrics.time_busy = 0; + mdev->mali_metrics.time_idle = 0; + mdev->mali_metrics.prev_busy = 0; + mdev->mali_metrics.prev_idle = 0; + mdev->mali_metrics.num_running_gp_cores = 0; + mdev->mali_metrics.num_running_pp_cores = 0; + mdev->mali_metrics.time_busy_gp = 0; + mdev->mali_metrics.time_idle_gp = 0; + + for (i = 0; i < MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; i++) { + mdev->mali_metrics.time_busy_pp[i] = 0; + mdev->mali_metrics.time_idle_pp[i] = 0; + } + mdev->mali_metrics.gpu_active = MALI_FALSE; + + mdev->mali_metrics.lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST); + if (NULL == mdev->mali_metrics.lock) { + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_pm_metrics_term(struct mali_device *mdev) +{ + _mali_osk_spinlock_irq_term(mdev->mali_metrics.lock); +} + +/*caller needs to hold mdev->mali_metrics.lock before calling this function*/ +void mali_pm_record_job_status(struct mali_device *mdev) +{ + ktime_t now; + ktime_t diff; + u64 ns_time; + + MALI_DEBUG_ASSERT(mdev != NULL); + + now = ktime_get(); + diff = ktime_sub(now, mdev->mali_metrics.time_period_start); + + ns_time = (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + mdev->mali_metrics.time_busy += ns_time; + mdev->mali_metrics.time_period_start = now; +} + +void mali_pm_record_gpu_idle(mali_bool is_gp) +{ + ktime_t now; + ktime_t diff; + u64 ns_time; + struct mali_device *mdev = dev_get_drvdata(&mali_platform_device->dev); + + MALI_DEBUG_ASSERT(mdev != NULL); + + _mali_osk_spinlock_irq_lock(mdev->mali_metrics.lock); + now = ktime_get(); + + if (MALI_TRUE == is_gp) { + --mdev->mali_metrics.num_running_gp_cores; + if (0 == mdev->mali_metrics.num_running_gp_cores) { + diff = ktime_sub(now, mdev->mali_metrics.time_period_start_gp); + ns_time = (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + mdev->mali_metrics.time_busy_gp += ns_time; + mdev->mali_metrics.time_period_start_gp = now; + + if (0 == mdev->mali_metrics.num_running_pp_cores) { + MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_TRUE); + diff = ktime_sub(now, mdev->mali_metrics.time_period_start); + ns_time = (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + mdev->mali_metrics.time_busy += ns_time; + mdev->mali_metrics.time_period_start = now; + mdev->mali_metrics.gpu_active = MALI_FALSE; + } + } + } else { + --mdev->mali_metrics.num_running_pp_cores; + if (0 == mdev->mali_metrics.num_running_pp_cores) { + diff = ktime_sub(now, mdev->mali_metrics.time_period_start_pp); + ns_time = (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + mdev->mali_metrics.time_busy_pp[0] += ns_time; + mdev->mali_metrics.time_period_start_pp = now; + + if (0 == mdev->mali_metrics.num_running_gp_cores) { + MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_TRUE); + diff = ktime_sub(now, mdev->mali_metrics.time_period_start); + ns_time = (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + mdev->mali_metrics.time_busy += ns_time; + mdev->mali_metrics.time_period_start = now; + mdev->mali_metrics.gpu_active = MALI_FALSE; + } + } + } + + _mali_osk_spinlock_irq_unlock(mdev->mali_metrics.lock); +} + +void mali_pm_record_gpu_active(mali_bool is_gp) +{ + ktime_t now; + ktime_t diff; + struct mali_device *mdev = dev_get_drvdata(&mali_platform_device->dev); + + MALI_DEBUG_ASSERT(mdev != NULL); + + _mali_osk_spinlock_irq_lock(mdev->mali_metrics.lock); + now = ktime_get(); + + if (MALI_TRUE == is_gp) { + mdev->mali_metrics.num_running_gp_cores++; + if (1 == mdev->mali_metrics.num_running_gp_cores) { + diff = ktime_sub(now, mdev->mali_metrics.time_period_start_gp); + mdev->mali_metrics.time_idle_gp += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + mdev->mali_metrics.time_period_start_gp = now; + if (0 == mdev->mali_metrics.num_running_pp_cores) { + MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_FALSE); + diff = ktime_sub(now, mdev->mali_metrics.time_period_start); + mdev->mali_metrics.time_idle += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + mdev->mali_metrics.time_period_start = now; + mdev->mali_metrics.gpu_active = MALI_TRUE; + } + } else { + MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_TRUE); + } + } else { + mdev->mali_metrics.num_running_pp_cores++; + if (1 == mdev->mali_metrics.num_running_pp_cores) { + diff = ktime_sub(now, mdev->mali_metrics.time_period_start_pp); + mdev->mali_metrics.time_idle_pp[0] += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + mdev->mali_metrics.time_period_start_pp = now; + if (0 == mdev->mali_metrics.num_running_gp_cores) { + MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_FALSE); + diff = ktime_sub(now, mdev->mali_metrics.time_period_start); + mdev->mali_metrics.time_idle += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + mdev->mali_metrics.time_period_start = now; + mdev->mali_metrics.gpu_active = MALI_TRUE; + } + } else { + MALI_DEBUG_ASSERT(mdev->mali_metrics.gpu_active == MALI_TRUE); + } + } + + _mali_osk_spinlock_irq_unlock(mdev->mali_metrics.lock); +} + + +/*caller needs to hold mdev->mali_metrics.lock before calling this function*/ +static void mali_pm_get_dvfs_utilisation_calc(struct mali_device *mdev, ktime_t now) +{ + ktime_t diff; + + MALI_DEBUG_ASSERT(mdev != NULL); + + diff = ktime_sub(now, mdev->mali_metrics.time_period_start); + + if (mdev->mali_metrics.gpu_active) { + mdev->mali_metrics.time_busy += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + } else { + mdev->mali_metrics.time_idle += (u64)(ktime_to_ns(diff) >> MALI_PM_TIME_SHIFT); + } +} + +/* Caller needs to hold mdev->mali_metrics.lock before calling this function. */ +static void mali_pm_reset_dvfs_utilisation_unlocked(struct mali_device *mdev, ktime_t now) +{ + /* Store previous value */ + mdev->mali_metrics.prev_idle = mdev->mali_metrics.time_idle; + mdev->mali_metrics.prev_busy = mdev->mali_metrics.time_busy; + + /* Reset current values */ + mdev->mali_metrics.time_period_start = now; + mdev->mali_metrics.time_period_start_gp = now; + mdev->mali_metrics.time_period_start_pp = now; + mdev->mali_metrics.time_idle = 0; + mdev->mali_metrics.time_busy = 0; + + mdev->mali_metrics.time_busy_gp = 0; + mdev->mali_metrics.time_idle_gp = 0; + mdev->mali_metrics.time_busy_pp[0] = 0; + mdev->mali_metrics.time_idle_pp[0] = 0; +} + +void mali_pm_reset_dvfs_utilisation(struct mali_device *mdev) +{ + _mali_osk_spinlock_irq_lock(mdev->mali_metrics.lock); + mali_pm_reset_dvfs_utilisation_unlocked(mdev, ktime_get()); + _mali_osk_spinlock_irq_unlock(mdev->mali_metrics.lock); +} + +void mali_pm_get_dvfs_utilisation(struct mali_device *mdev, + unsigned long *total_out, unsigned long *busy_out) +{ + ktime_t now = ktime_get(); + u64 busy = 0; + u64 total = 0; + + _mali_osk_spinlock_irq_lock(mdev->mali_metrics.lock); + + mali_pm_get_dvfs_utilisation_calc(mdev, now); + + busy = mdev->mali_metrics.time_busy; + total = busy + mdev->mali_metrics.time_idle; + + /* Reset stats if older than MALI_UTILIZATION_MAX_PERIOD (default + * 100ms) */ + if (total >= MALI_UTILIZATION_MAX_PERIOD) { + mali_pm_reset_dvfs_utilisation_unlocked(mdev, now); + } else if (total < (MALI_UTILIZATION_MAX_PERIOD / 2)) { + total += mdev->mali_metrics.prev_idle + + mdev->mali_metrics.prev_busy; + busy += mdev->mali_metrics.prev_busy; + } + + *total_out = (unsigned long)total; + *busy_out = (unsigned long)busy; + _mali_osk_spinlock_irq_unlock(mdev->mali_metrics.lock); +} + +void mali_pm_metrics_spin_lock(void) +{ + struct mali_device *mdev = dev_get_drvdata(&mali_platform_device->dev); + _mali_osk_spinlock_irq_lock(mdev->mali_metrics.lock); +} + +void mali_pm_metrics_spin_unlock(void) +{ + struct mali_device *mdev = dev_get_drvdata(&mali_platform_device->dev); + _mali_osk_spinlock_irq_unlock(mdev->mali_metrics.lock); +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.h b/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.h new file mode 100755 index 000000000000..2b136b0de4e3 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pm_metrics.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PM_METRICS_H__ +#define __MALI_PM_METRICS_H__ + +#ifdef CONFIG_MALI_DEVFREQ +#include "mali_osk_locks.h" +#include "mali_group.h" + +struct mali_device; + +/** + * Metrics data collected for use by the power management framework. + */ +struct mali_pm_metrics_data { + ktime_t time_period_start; + u64 time_busy; + u64 time_idle; + u64 prev_busy; + u64 prev_idle; + u32 num_running_gp_cores; + u32 num_running_pp_cores; + ktime_t time_period_start_gp; + u64 time_busy_gp; + u64 time_idle_gp; + ktime_t time_period_start_pp; + u64 time_busy_pp[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; + u64 time_idle_pp[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS]; + mali_bool gpu_active; + _mali_osk_spinlock_irq_t *lock; +}; + +/** + * Initialize/start the Mali GPU pm_metrics metrics reporting. + * + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t mali_pm_metrics_init(struct mali_device *mdev); + +/** + * Terminate the Mali GPU pm_metrics metrics reporting + */ +void mali_pm_metrics_term(struct mali_device *mdev); + +/** + * Should be called when a job is about to execute a GPU job + */ +void mali_pm_record_gpu_active(mali_bool is_gp); + +/** + * Should be called when a job is finished + */ +void mali_pm_record_gpu_idle(mali_bool is_gp); + +void mali_pm_reset_dvfs_utilisation(struct mali_device *mdev); + +void mali_pm_get_dvfs_utilisation(struct mali_device *mdev, unsigned long *total_out, unsigned long *busy_out); + +void mali_pm_metrics_spin_lock(void); + +void mali_pm_metrics_spin_unlock(void); +#else +void mali_pm_record_gpu_idle(mali_bool is_gp) {} +void mali_pm_record_gpu_active(mali_bool is_gp) {} +#endif +#endif /* __MALI_PM_METRICS_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pmu.c b/drivers/gpu/arm/mali400/mali/common/mali_pmu.c new file mode 100755 index 000000000000..6f0af59f6fd4 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pmu.c @@ -0,0 +1,270 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_pmu.c + * Mali driver functions for Mali 400 PMU hardware + */ +#include "mali_hw_core.h" +#include "mali_pmu.h" +#include "mali_pp.h" +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_pm.h" +#include "mali_osk_mali.h" + +struct mali_pmu_core *mali_global_pmu_core = NULL; + +static _mali_osk_errcode_t mali_pmu_wait_for_command_finish( + struct mali_pmu_core *pmu); + +struct mali_pmu_core *mali_pmu_create(_mali_osk_resource_t *resource) +{ + struct mali_pmu_core *pmu; + + MALI_DEBUG_ASSERT(NULL == mali_global_pmu_core); + MALI_DEBUG_PRINT(2, ("Mali PMU: Creating Mali PMU core\n")); + + pmu = (struct mali_pmu_core *)_mali_osk_malloc( + sizeof(struct mali_pmu_core)); + if (NULL != pmu) { + pmu->registered_cores_mask = 0; /* to be set later */ + + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&pmu->hw_core, + resource, PMU_REGISTER_ADDRESS_SPACE_SIZE)) { + + pmu->switch_delay = _mali_osk_get_pmu_switch_delay(); + + mali_global_pmu_core = pmu; + + return pmu; + } + _mali_osk_free(pmu); + } + + return NULL; +} + +void mali_pmu_delete(struct mali_pmu_core *pmu) +{ + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu == mali_global_pmu_core); + + MALI_DEBUG_PRINT(2, ("Mali PMU: Deleting Mali PMU core\n")); + + mali_global_pmu_core = NULL; + + mali_hw_core_delete(&pmu->hw_core); + _mali_osk_free(pmu); +} + +void mali_pmu_set_registered_cores_mask(struct mali_pmu_core *pmu, u32 mask) +{ + pmu->registered_cores_mask = mask; +} + +void mali_pmu_reset(struct mali_pmu_core *pmu) +{ + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); + + /* Setup the desired defaults */ + mali_hw_core_register_write_relaxed(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_MASK, 0); + mali_hw_core_register_write_relaxed(&pmu->hw_core, + PMU_REG_ADDR_MGMT_SW_DELAY, pmu->switch_delay); +} + +void mali_pmu_power_up_all(struct mali_pmu_core *pmu) +{ + u32 stat; + + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); + + mali_pm_exec_lock(); + + mali_pmu_reset(pmu); + + /* Now simply power up the domains which are marked as powered down */ + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + mali_pmu_power_up(pmu, stat); + + mali_pm_exec_unlock(); +} + +void mali_pmu_power_down_all(struct mali_pmu_core *pmu) +{ + u32 stat; + + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); + + mali_pm_exec_lock(); + + /* Now simply power down the domains which are marked as powered up */ + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + mali_pmu_power_down(pmu, (~stat) & pmu->registered_cores_mask); + + mali_pm_exec_unlock(); +} + +_mali_osk_errcode_t mali_pmu_power_down(struct mali_pmu_core *pmu, u32 mask) +{ + u32 stat; + _mali_osk_errcode_t err; + + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); + MALI_DEBUG_ASSERT(mask <= pmu->registered_cores_mask); + MALI_DEBUG_ASSERT(0 == (mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_RAWSTAT) & + PMU_REG_VAL_IRQ)); + + MALI_DEBUG_PRINT(3, + ("PMU power down: ...................... [%s]\n", + mali_pm_mask_to_string(mask))); + + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + + /* + * Assert that we are not powering down domains which are already + * powered down. + */ + MALI_DEBUG_ASSERT(0 == (stat & mask)); + + mask &= ~(0x1 << MALI_DOMAIN_INDEX_DUMMY); + + if (0 == mask || 0 == ((~stat) & mask)) return _MALI_OSK_ERR_OK; + + mali_hw_core_register_write(&pmu->hw_core, + PMU_REG_ADDR_MGMT_POWER_DOWN, mask); + + /* + * Do not wait for interrupt on Mali-300/400 if all domains are + * powered off by our power down command, because the HW will simply + * not generate an interrupt in this case. + */ + if (mali_is_mali450() || mali_is_mali470() || pmu->registered_cores_mask != (mask | stat)) { + err = mali_pmu_wait_for_command_finish(pmu); + if (_MALI_OSK_ERR_OK != err) { + return err; + } + } else { + mali_hw_core_register_write(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_CLEAR, PMU_REG_VAL_IRQ); + } + +#if defined(DEBUG) + /* Verify power status of domains after power down */ + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + MALI_DEBUG_ASSERT(mask == (stat & mask)); +#endif + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_pmu_power_up(struct mali_pmu_core *pmu, u32 mask) +{ + u32 stat; + _mali_osk_errcode_t err; +#if !defined(CONFIG_MALI_PMU_PARALLEL_POWER_UP) + u32 current_domain; +#endif + + MALI_DEBUG_ASSERT_POINTER(pmu); + MALI_DEBUG_ASSERT(pmu->registered_cores_mask != 0); + MALI_DEBUG_ASSERT(mask <= pmu->registered_cores_mask); + MALI_DEBUG_ASSERT(0 == (mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_RAWSTAT) & + PMU_REG_VAL_IRQ)); + + MALI_DEBUG_PRINT(3, + ("PMU power up: ........................ [%s]\n", + mali_pm_mask_to_string(mask))); + + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + stat &= pmu->registered_cores_mask; + + mask &= ~(0x1 << MALI_DOMAIN_INDEX_DUMMY); + if (0 == mask || 0 == (stat & mask)) return _MALI_OSK_ERR_OK; + + /* + * Assert that we are only powering up domains which are currently + * powered down. + */ + MALI_DEBUG_ASSERT(mask == (stat & mask)); + +#if defined(CONFIG_MALI_PMU_PARALLEL_POWER_UP) + mali_hw_core_register_write(&pmu->hw_core, + PMU_REG_ADDR_MGMT_POWER_UP, mask); + + err = mali_pmu_wait_for_command_finish(pmu); + if (_MALI_OSK_ERR_OK != err) { + return err; + } +#else + for (current_domain = 1; + current_domain <= pmu->registered_cores_mask; + current_domain <<= 1) { + if (current_domain & mask & stat) { + mali_hw_core_register_write(&pmu->hw_core, + PMU_REG_ADDR_MGMT_POWER_UP, + current_domain); + + err = mali_pmu_wait_for_command_finish(pmu); + if (_MALI_OSK_ERR_OK != err) { + return err; + } + } + } +#endif + +#if defined(DEBUG) + /* Verify power status of domains after power up */ + stat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_STATUS); + MALI_DEBUG_ASSERT(0 == (stat & mask)); +#endif /* defined(DEBUG) */ + + return _MALI_OSK_ERR_OK; +} + +static _mali_osk_errcode_t mali_pmu_wait_for_command_finish( + struct mali_pmu_core *pmu) +{ + u32 rawstat; + u32 timeout = MALI_REG_POLL_COUNT_SLOW; + + MALI_DEBUG_ASSERT(pmu); + + /* Wait for the command to complete */ + do { + rawstat = mali_hw_core_register_read(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_RAWSTAT); + --timeout; + } while (0 == (rawstat & PMU_REG_VAL_IRQ) && 0 < timeout); + + MALI_DEBUG_ASSERT(0 < timeout); + + if (0 == timeout) { + return _MALI_OSK_ERR_TIMEOUT; + } + + mali_hw_core_register_write(&pmu->hw_core, + PMU_REG_ADDR_MGMT_INT_CLEAR, PMU_REG_VAL_IRQ); + + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pmu.h b/drivers/gpu/arm/mali400/mali/common/mali_pmu.h new file mode 100755 index 000000000000..5b856240fdac --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pmu.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_platform.h + * Platform specific Mali driver functions + */ + +#ifndef __MALI_PMU_H__ +#define __MALI_PMU_H__ + +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_hw_core.h" + +/** @brief MALI inbuilt PMU hardware info and PMU hardware has knowledge of cores power mask + */ +struct mali_pmu_core { + struct mali_hw_core hw_core; + u32 registered_cores_mask; + u32 switch_delay; +}; + +/** @brief Register layout for hardware PMU + */ +typedef enum { + PMU_REG_ADDR_MGMT_POWER_UP = 0x00, /*< Power up register */ + PMU_REG_ADDR_MGMT_POWER_DOWN = 0x04, /*< Power down register */ + PMU_REG_ADDR_MGMT_STATUS = 0x08, /*< Core sleep status register */ + PMU_REG_ADDR_MGMT_INT_MASK = 0x0C, /*< Interrupt mask register */ + PMU_REG_ADDR_MGMT_INT_RAWSTAT = 0x10, /*< Interrupt raw status register */ + PMU_REG_ADDR_MGMT_INT_CLEAR = 0x18, /*< Interrupt clear register */ + PMU_REG_ADDR_MGMT_SW_DELAY = 0x1C, /*< Switch delay register */ + PMU_REGISTER_ADDRESS_SPACE_SIZE = 0x28, /*< Size of register space */ +} pmu_reg_addr_mgmt_addr; + +#define PMU_REG_VAL_IRQ 1 + +extern struct mali_pmu_core *mali_global_pmu_core; + +/** @brief Initialisation of MALI PMU + * + * This is called from entry point of the driver in order to create and intialize the PMU resource + * + * @param resource it will be a pointer to a PMU resource + * @param number_of_pp_cores Number of found PP resources in configuration + * @param number_of_l2_caches Number of found L2 cache resources in configuration + * @return The created PMU object, or NULL in case of failure. + */ +struct mali_pmu_core *mali_pmu_create(_mali_osk_resource_t *resource); + +/** @brief It deallocates the PMU resource + * + * This is called on the exit of the driver to terminate the PMU resource + * + * @param pmu Pointer to PMU core object to delete + */ +void mali_pmu_delete(struct mali_pmu_core *pmu); + +/** @brief Set registered cores mask + * + * @param pmu Pointer to PMU core object + * @param mask All available/valid domain bits + */ +void mali_pmu_set_registered_cores_mask(struct mali_pmu_core *pmu, u32 mask); + +/** @brief Retrieves the Mali PMU core object (if any) + * + * @return The Mali PMU object, or NULL if no PMU exists. + */ +MALI_STATIC_INLINE struct mali_pmu_core *mali_pmu_get_global_pmu_core(void) +{ + return mali_global_pmu_core; +} + +/** @brief Reset PMU core + * + * @param pmu Pointer to PMU core object to reset + */ +void mali_pmu_reset(struct mali_pmu_core *pmu); + +void mali_pmu_power_up_all(struct mali_pmu_core *pmu); + +void mali_pmu_power_down_all(struct mali_pmu_core *pmu); + +/** @brief Returns a mask of the currently powered up domains + * + * @param pmu Pointer to PMU core object + */ +MALI_STATIC_INLINE u32 mali_pmu_get_mask(struct mali_pmu_core *pmu) +{ + u32 stat = mali_hw_core_register_read(&pmu->hw_core, PMU_REG_ADDR_MGMT_STATUS); + return ((~stat) & pmu->registered_cores_mask); +} + +/** @brief MALI GPU power down using MALI in-built PMU + * + * Called to power down the specified cores. + * + * @param pmu Pointer to PMU core object to power down + * @param mask Mask specifying which power domains to power down + * @return _MALI_OSK_ERR_OK on success otherwise, a suitable _mali_osk_errcode_t error. + */ +_mali_osk_errcode_t mali_pmu_power_down(struct mali_pmu_core *pmu, u32 mask); + +/** @brief MALI GPU power up using MALI in-built PMU + * + * Called to power up the specified cores. + * + * @param pmu Pointer to PMU core object to power up + * @param mask Mask specifying which power domains to power up + * @return _MALI_OSK_ERR_OK on success otherwise, a suitable _mali_osk_errcode_t error. + */ +_mali_osk_errcode_t mali_pmu_power_up(struct mali_pmu_core *pmu, u32 mask); + +#endif /* __MALI_PMU_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp.c b/drivers/gpu/arm/mali400/mali/common/mali_pp.c new file mode 100755 index 000000000000..2dd8b8766f8e --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pp.c @@ -0,0 +1,502 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_pp_job.h" +#include "mali_pp.h" +#include "mali_hw_core.h" +#include "mali_group.h" +#include "regs/mali_200_regs.h" +#include "mali_kernel_common.h" +#include "mali_kernel_core.h" + +#if defined(CONFIG_MALI400_PROFILING) +#include "mali_osk_profiling.h" +#endif + +/* Number of frame registers on Mali-200 */ +#define MALI_PP_MALI200_NUM_FRAME_REGISTERS ((0x04C/4)+1) +/* Number of frame registers on Mali-300 and later */ +#define MALI_PP_MALI400_NUM_FRAME_REGISTERS ((0x058/4)+1) + +static struct mali_pp_core *mali_global_pp_cores[MALI_MAX_NUMBER_OF_PP_CORES] = { NULL }; +static u32 mali_global_num_pp_cores = 0; + +/* Interrupt handlers */ +static void mali_pp_irq_probe_trigger(void *data); +static _mali_osk_errcode_t mali_pp_irq_probe_ack(void *data); + +struct mali_pp_core *mali_pp_create(const _mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual, u32 bcast_id) +{ + struct mali_pp_core *core = NULL; + + MALI_DEBUG_PRINT(2, ("Mali PP: Creating Mali PP core: %s\n", resource->description)); + MALI_DEBUG_PRINT(2, ("Mali PP: Base address of PP core: 0x%x\n", resource->base)); + + if (mali_global_num_pp_cores >= MALI_MAX_NUMBER_OF_PP_CORES) { + MALI_PRINT_ERROR(("Mali PP: Too many PP core objects created\n")); + return NULL; + } + + core = _mali_osk_calloc(1, sizeof(struct mali_pp_core)); + if (NULL != core) { + core->core_id = mali_global_num_pp_cores; + core->bcast_id = bcast_id; + + if (_MALI_OSK_ERR_OK == mali_hw_core_create(&core->hw_core, resource, MALI200_REG_SIZEOF_REGISTER_BANK)) { + _mali_osk_errcode_t ret; + + if (!is_virtual) { + ret = mali_pp_reset(core); + } else { + ret = _MALI_OSK_ERR_OK; + } + + if (_MALI_OSK_ERR_OK == ret) { + ret = mali_group_add_pp_core(group, core); + if (_MALI_OSK_ERR_OK == ret) { + /* Setup IRQ handlers (which will do IRQ probing if needed) */ + MALI_DEBUG_ASSERT(!is_virtual || -1 != resource->irq); + + core->irq = _mali_osk_irq_init(resource->irq, + mali_group_upper_half_pp, + group, + mali_pp_irq_probe_trigger, + mali_pp_irq_probe_ack, + core, + resource->description); + if (NULL != core->irq) { + mali_global_pp_cores[mali_global_num_pp_cores] = core; + mali_global_num_pp_cores++; + + return core; + } else { + MALI_PRINT_ERROR(("Mali PP: Failed to setup interrupt handlers for PP core %s\n", core->hw_core.description)); + } + mali_group_remove_pp_core(group); + } else { + MALI_PRINT_ERROR(("Mali PP: Failed to add core %s to group\n", core->hw_core.description)); + } + } + mali_hw_core_delete(&core->hw_core); + } + + _mali_osk_free(core); + } else { + MALI_PRINT_ERROR(("Mali PP: Failed to allocate memory for PP core\n")); + } + + return NULL; +} + +void mali_pp_delete(struct mali_pp_core *core) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(core); + + _mali_osk_irq_term(core->irq); + mali_hw_core_delete(&core->hw_core); + + /* Remove core from global list */ + for (i = 0; i < mali_global_num_pp_cores; i++) { + if (mali_global_pp_cores[i] == core) { + mali_global_pp_cores[i] = NULL; + mali_global_num_pp_cores--; + + if (i != mali_global_num_pp_cores) { + /* We removed a PP core from the middle of the array -- move the last + * PP core to the current position to close the gap */ + mali_global_pp_cores[i] = mali_global_pp_cores[mali_global_num_pp_cores]; + mali_global_pp_cores[mali_global_num_pp_cores] = NULL; + } + + break; + } + } + + _mali_osk_free(core); +} + +void mali_pp_stop_bus(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + /* Will only send the stop bus command, and not wait for it to complete */ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI200_REG_VAL_CTRL_MGMT_STOP_BUS); +} + +_mali_osk_errcode_t mali_pp_stop_bus_wait(struct mali_pp_core *core) +{ + int i; + + MALI_DEBUG_ASSERT_POINTER(core); + + /* Send the stop bus command. */ + mali_pp_stop_bus(core); + + /* Wait for bus to be stopped */ + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { + if (mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS) & MALI200_REG_VAL_STATUS_BUS_STOPPED) + break; + } + + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_PRINT_ERROR(("Mali PP: Failed to stop bus on %s. Status: 0x%08x\n", core->hw_core.description, mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS))); + return _MALI_OSK_ERR_FAULT; + } + return _MALI_OSK_ERR_OK; +} + +/* Frame register reset values. + * Taken from the Mali400 TRM, 3.6. Pixel processor control register summary */ +static const u32 mali_frame_registers_reset_values[_MALI_PP_MAX_FRAME_REGISTERS] = { + 0x0, /* Renderer List Address Register */ + 0x0, /* Renderer State Word Base Address Register */ + 0x0, /* Renderer Vertex Base Register */ + 0x2, /* Feature Enable Register */ + 0x0, /* Z Clear Value Register */ + 0x0, /* Stencil Clear Value Register */ + 0x0, /* ABGR Clear Value 0 Register */ + 0x0, /* ABGR Clear Value 1 Register */ + 0x0, /* ABGR Clear Value 2 Register */ + 0x0, /* ABGR Clear Value 3 Register */ + 0x0, /* Bounding Box Left Right Register */ + 0x0, /* Bounding Box Bottom Register */ + 0x0, /* FS Stack Address Register */ + 0x0, /* FS Stack Size and Initial Value Register */ + 0x0, /* Reserved */ + 0x0, /* Reserved */ + 0x0, /* Origin Offset X Register */ + 0x0, /* Origin Offset Y Register */ + 0x75, /* Subpixel Specifier Register */ + 0x0, /* Tiebreak mode Register */ + 0x0, /* Polygon List Format Register */ + 0x0, /* Scaling Register */ + 0x0 /* Tilebuffer configuration Register */ +}; + +/* WBx register reset values */ +static const u32 mali_wb_registers_reset_values[_MALI_PP_MAX_WB_REGISTERS] = { + 0x0, /* WBx Source Select Register */ + 0x0, /* WBx Target Address Register */ + 0x0, /* WBx Target Pixel Format Register */ + 0x0, /* WBx Target AA Format Register */ + 0x0, /* WBx Target Layout */ + 0x0, /* WBx Target Scanline Length */ + 0x0, /* WBx Target Flags Register */ + 0x0, /* WBx MRT Enable Register */ + 0x0, /* WBx MRT Offset Register */ + 0x0, /* WBx Global Test Enable Register */ + 0x0, /* WBx Global Test Reference Value Register */ + 0x0 /* WBx Global Test Compare Function Register */ +}; + +/* Performance Counter 0 Enable Register reset value */ +static const u32 mali_perf_cnt_enable_reset_value = 0; + +_mali_osk_errcode_t mali_pp_hard_reset(struct mali_pp_core *core) +{ + /* Bus must be stopped before calling this function */ + const u32 reset_wait_target_register = MALI200_REG_ADDR_MGMT_PERF_CNT_0_LIMIT; + const u32 reset_invalid_value = 0xC0FFE000; + const u32 reset_check_value = 0xC01A0000; + int i; + + MALI_DEBUG_ASSERT_POINTER(core); + MALI_DEBUG_PRINT(2, ("Mali PP: Hard reset of core %s\n", core->hw_core.description)); + + /* Set register to a bogus value. The register will be used to detect when reset is complete */ + mali_hw_core_register_write_relaxed(&core->hw_core, reset_wait_target_register, reset_invalid_value); + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_NONE); + + /* Force core to reset */ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI200_REG_VAL_CTRL_MGMT_FORCE_RESET); + + /* Wait for reset to be complete */ + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, reset_check_value); + if (reset_check_value == mali_hw_core_register_read(&core->hw_core, reset_wait_target_register)) { + break; + } + } + + if (MALI_REG_POLL_COUNT_FAST == i) { + MALI_PRINT_ERROR(("Mali PP: The hard reset loop didn't work, unable to recover\n")); + } + + mali_hw_core_register_write(&core->hw_core, reset_wait_target_register, 0x00000000); /* set it back to the default */ + /* Re-enable interrupts */ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_MASK_ALL); + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); + + return _MALI_OSK_ERR_OK; +} + +void mali_pp_reset_async(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + + MALI_DEBUG_PRINT(4, ("Mali PP: Reset of core %s\n", core->hw_core.description)); + + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, 0); /* disable the IRQs */ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_MASK_ALL); + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI400PP_REG_VAL_CTRL_MGMT_SOFT_RESET); +} + +_mali_osk_errcode_t mali_pp_reset_wait(struct mali_pp_core *core) +{ + int i; + u32 rawstat = 0; + + for (i = 0; i < MALI_REG_POLL_COUNT_FAST; i++) { + u32 status = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS); + if (!(status & MALI200_REG_VAL_STATUS_RENDERING_ACTIVE)) { + rawstat = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT); + if (rawstat == MALI400PP_REG_VAL_IRQ_RESET_COMPLETED) { + break; + } + } + } + + if (i == MALI_REG_POLL_COUNT_FAST) { + MALI_PRINT_ERROR(("Mali PP: Failed to reset core %s, rawstat: 0x%08x\n", + core->hw_core.description, rawstat)); + return _MALI_OSK_ERR_FAULT; + } + + /* Re-enable interrupts */ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_MASK_ALL); + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_pp_reset(struct mali_pp_core *core) +{ + mali_pp_reset_async(core); + return mali_pp_reset_wait(core); +} + +void mali_pp_job_start(struct mali_pp_core *core, struct mali_pp_job *job, u32 sub_job, mali_bool restart_virtual) +{ + u32 relative_address; + u32 start_index; + u32 nr_of_regs; + u32 *frame_registers = mali_pp_job_get_frame_registers(job); + u32 *wb0_registers = mali_pp_job_get_wb0_registers(job); + u32 *wb1_registers = mali_pp_job_get_wb1_registers(job); + u32 *wb2_registers = mali_pp_job_get_wb2_registers(job); + u32 counter_src0 = mali_pp_job_get_perf_counter_src0(job, sub_job); + u32 counter_src1 = mali_pp_job_get_perf_counter_src1(job, sub_job); + + MALI_DEBUG_ASSERT_POINTER(core); + + /* Write frame registers */ + + /* + * There are two frame registers which are different for each sub job: + * 1. The Renderer List Address Register (MALI200_REG_ADDR_FRAME) + * 2. The FS Stack Address Register (MALI200_REG_ADDR_STACK) + */ + mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_FRAME, mali_pp_job_get_addr_frame(job, sub_job), mali_frame_registers_reset_values[MALI200_REG_ADDR_FRAME / sizeof(u32)]); + + /* For virtual jobs, the stack address shouldn't be broadcast but written individually */ + if (!mali_pp_job_is_virtual(job) || restart_virtual) { + mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_STACK, mali_pp_job_get_addr_stack(job, sub_job), mali_frame_registers_reset_values[MALI200_REG_ADDR_STACK / sizeof(u32)]); + } + + /* Write registers between MALI200_REG_ADDR_FRAME and MALI200_REG_ADDR_STACK */ + relative_address = MALI200_REG_ADDR_RSW; + start_index = MALI200_REG_ADDR_RSW / sizeof(u32); + nr_of_regs = (MALI200_REG_ADDR_STACK - MALI200_REG_ADDR_RSW) / sizeof(u32); + + mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, + relative_address, &frame_registers[start_index], + nr_of_regs, &mali_frame_registers_reset_values[start_index]); + + /* MALI200_REG_ADDR_STACK_SIZE */ + relative_address = MALI200_REG_ADDR_STACK_SIZE; + start_index = MALI200_REG_ADDR_STACK_SIZE / sizeof(u32); + + mali_hw_core_register_write_relaxed_conditional(&core->hw_core, + relative_address, frame_registers[start_index], + mali_frame_registers_reset_values[start_index]); + + /* Skip 2 reserved registers */ + + /* Write remaining registers */ + relative_address = MALI200_REG_ADDR_ORIGIN_OFFSET_X; + start_index = MALI200_REG_ADDR_ORIGIN_OFFSET_X / sizeof(u32); + nr_of_regs = MALI_PP_MALI400_NUM_FRAME_REGISTERS - MALI200_REG_ADDR_ORIGIN_OFFSET_X / sizeof(u32); + + mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, + relative_address, &frame_registers[start_index], + nr_of_regs, &mali_frame_registers_reset_values[start_index]); + + /* Write WBx registers */ + if (wb0_registers[0]) { /* M200_WB0_REG_SOURCE_SELECT register */ + mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_WB0, wb0_registers, _MALI_PP_MAX_WB_REGISTERS, mali_wb_registers_reset_values); + } + + if (wb1_registers[0]) { /* M200_WB1_REG_SOURCE_SELECT register */ + mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_WB1, wb1_registers, _MALI_PP_MAX_WB_REGISTERS, mali_wb_registers_reset_values); + } + + if (wb2_registers[0]) { /* M200_WB2_REG_SOURCE_SELECT register */ + mali_hw_core_register_write_array_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_WB2, wb2_registers, _MALI_PP_MAX_WB_REGISTERS, mali_wb_registers_reset_values); + } + + if (MALI_HW_CORE_NO_COUNTER != counter_src0) { + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC, counter_src0); + mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE, MALI200_REG_VAL_PERF_CNT_ENABLE, mali_perf_cnt_enable_reset_value); + } + if (MALI_HW_CORE_NO_COUNTER != counter_src1) { + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC, counter_src1); + mali_hw_core_register_write_relaxed_conditional(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE, MALI200_REG_VAL_PERF_CNT_ENABLE, mali_perf_cnt_enable_reset_value); + } + +#ifdef CONFIG_MALI400_HEATMAPS_ENABLED + if (job->uargs.perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_HEATMAP_ENABLE) { + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERFMON_CONTR, ((job->uargs.tilesx & 0x3FF) << 16) | 1); + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_PERFMON_BASE, job->uargs.heatmap_mem & 0xFFFFFFF8); + } +#endif /* CONFIG_MALI400_HEATMAPS_ENABLED */ + + MALI_DEBUG_PRINT(3, ("Mali PP: Starting job 0x%08X part %u/%u on PP core %s\n", job, sub_job + 1, mali_pp_job_get_sub_job_count(job), core->hw_core.description)); + + /* Adding barrier to make sure all rester writes are finished */ + _mali_osk_write_mem_barrier(); + + /* This is the command that starts the core. + * + * Don't actually run the job if PROFILING_SKIP_PP_JOBS are set, just + * force core to assert the completion interrupt. + */ +#if !defined(PROFILING_SKIP_PP_JOBS) + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_CTRL_MGMT, MALI200_REG_VAL_CTRL_MGMT_START_RENDERING); +#else + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_END_OF_FRAME); +#endif + + /* Adding barrier to make sure previous rester writes is finished */ + _mali_osk_write_mem_barrier(); +} + +u32 mali_pp_core_get_version(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_VERSION); +} + +struct mali_pp_core *mali_pp_get_global_pp_core(u32 index) +{ + if (mali_global_num_pp_cores > index) { + return mali_global_pp_cores[index]; + } + + return NULL; +} + +u32 mali_pp_get_glob_num_pp_cores(void) +{ + return mali_global_num_pp_cores; +} + +/* ------------- interrupt handling below ------------------ */ +static void mali_pp_irq_probe_trigger(void *data) +{ + struct mali_pp_core *core = (struct mali_pp_core *)data; + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT, MALI200_REG_VAL_IRQ_BUS_ERROR); + _mali_osk_mem_barrier(); +} + +static _mali_osk_errcode_t mali_pp_irq_probe_ack(void *data) +{ + struct mali_pp_core *core = (struct mali_pp_core *)data; + u32 irq_readout; + + irq_readout = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_STATUS); + if (MALI200_REG_VAL_IRQ_BUS_ERROR & irq_readout) { + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_CLEAR, MALI200_REG_VAL_IRQ_BUS_ERROR); + _mali_osk_mem_barrier(); + return _MALI_OSK_ERR_OK; + } + + return _MALI_OSK_ERR_FAULT; +} + + +#if 0 +static void mali_pp_print_registers(struct mali_pp_core *core) +{ + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_VERSION = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_VERSION))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_CURRENT_REND_LIST_ADDR = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_CURRENT_REND_LIST_ADDR))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_STATUS = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_INT_RAWSTAT = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_INT_MASK = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_INT_STATUS = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_STATUS))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC))); + MALI_DEBUG_PRINT(2, ("Mali PP: Register MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x%08X\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE))); +} +#endif + +#if 0 +void mali_pp_print_state(struct mali_pp_core *core) +{ + MALI_DEBUG_PRINT(2, ("Mali PP: State: 0x%08x\n", mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS))); +} +#endif + +void mali_pp_update_performance_counters(struct mali_pp_core *parent, struct mali_pp_core *child, struct mali_pp_job *job, u32 subjob) +{ + u32 val0 = 0; + u32 val1 = 0; + u32 counter_src0 = mali_pp_job_get_perf_counter_src0(job, subjob); + u32 counter_src1 = mali_pp_job_get_perf_counter_src1(job, subjob); +#if defined(CONFIG_MALI400_PROFILING) + int counter_index = COUNTER_FP_0_C0 + (2 * child->core_id); +#endif + + if (MALI_HW_CORE_NO_COUNTER != counter_src0) { + val0 = mali_hw_core_register_read(&child->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE); + mali_pp_job_set_perf_counter_value0(job, subjob, val0); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_report_hw_counter(counter_index, val0); + _mali_osk_profiling_record_global_counters(counter_index, val0); +#endif + } + + if (MALI_HW_CORE_NO_COUNTER != counter_src1) { + val1 = mali_hw_core_register_read(&child->hw_core, MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE); + mali_pp_job_set_perf_counter_value1(job, subjob, val1); + +#if defined(CONFIG_MALI400_PROFILING) + _mali_osk_profiling_report_hw_counter(counter_index + 1, val1); + _mali_osk_profiling_record_global_counters(counter_index + 1, val1); +#endif + } +} + +#if MALI_STATE_TRACKING +u32 mali_pp_dump_state(struct mali_pp_core *core, char *buf, u32 size) +{ + int n = 0; + + n += _mali_osk_snprintf(buf + n, size - n, "\tPP #%d: %s\n", core->core_id, core->hw_core.description); + + return n; +} +#endif diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp.h b/drivers/gpu/arm/mali400/mali/common/mali_pp.h new file mode 100755 index 000000000000..f98b29866ffa --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pp.h @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PP_H__ +#define __MALI_PP_H__ + +#include "mali_osk.h" +#include "mali_pp_job.h" +#include "mali_hw_core.h" + +struct mali_group; + +#define MALI_MAX_NUMBER_OF_PP_CORES 9 + +/** + * Definition of the PP core struct + * Used to track a PP core in the system. + */ +struct mali_pp_core { + struct mali_hw_core hw_core; /**< Common for all HW cores */ + _mali_osk_irq_t *irq; /**< IRQ handler */ + u32 core_id; /**< Unique core ID */ + u32 bcast_id; /**< The "flag" value used by the Mali-450 broadcast and DLBU unit */ +}; + +_mali_osk_errcode_t mali_pp_initialize(void); +void mali_pp_terminate(void); + +struct mali_pp_core *mali_pp_create(const _mali_osk_resource_t *resource, struct mali_group *group, mali_bool is_virtual, u32 bcast_id); +void mali_pp_delete(struct mali_pp_core *core); + +void mali_pp_stop_bus(struct mali_pp_core *core); +_mali_osk_errcode_t mali_pp_stop_bus_wait(struct mali_pp_core *core); +void mali_pp_reset_async(struct mali_pp_core *core); +_mali_osk_errcode_t mali_pp_reset_wait(struct mali_pp_core *core); +_mali_osk_errcode_t mali_pp_reset(struct mali_pp_core *core); +_mali_osk_errcode_t mali_pp_hard_reset(struct mali_pp_core *core); + +void mali_pp_job_start(struct mali_pp_core *core, struct mali_pp_job *job, u32 sub_job, mali_bool restart_virtual); + +u32 mali_pp_core_get_version(struct mali_pp_core *core); + +MALI_STATIC_INLINE u32 mali_pp_core_get_id(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return core->core_id; +} + +MALI_STATIC_INLINE u32 mali_pp_core_get_bcast_id(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return core->bcast_id; +} + +struct mali_pp_core *mali_pp_get_global_pp_core(u32 index); +u32 mali_pp_get_glob_num_pp_cores(void); + +/* Debug */ +u32 mali_pp_dump_state(struct mali_pp_core *core, char *buf, u32 size); + +/** + * Put instrumented HW counters from the core(s) to the job object (if enabled) + * + * parent and child is always the same, except for virtual jobs on Mali-450. + * In this case, the counters will be enabled on the virtual core (parent), + * but values need to be read from the child cores. + * + * @param parent The core used to see if the counters was enabled + * @param child The core to actually read the values from + * @job Job object to update with counter values (if enabled) + * @subjob Which subjob the counters are applicable for (core ID for virtual jobs) + */ +void mali_pp_update_performance_counters(struct mali_pp_core *parent, struct mali_pp_core *child, struct mali_pp_job *job, u32 subjob); + +MALI_STATIC_INLINE const char *mali_pp_core_description(struct mali_pp_core *core) +{ + return core->hw_core.description; +} + +MALI_STATIC_INLINE enum mali_interrupt_result mali_pp_get_interrupt_result(struct mali_pp_core *core) +{ + u32 rawstat_used = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_RAWSTAT) & + MALI200_REG_VAL_IRQ_MASK_USED; + if (0 == rawstat_used) { + return MALI_INTERRUPT_RESULT_NONE; + } else if (MALI200_REG_VAL_IRQ_END_OF_FRAME == rawstat_used) { + return MALI_INTERRUPT_RESULT_SUCCESS; + } + + return MALI_INTERRUPT_RESULT_ERROR; +} + +MALI_STATIC_INLINE u32 mali_pp_get_rawstat(struct mali_pp_core *core) +{ + MALI_DEBUG_ASSERT_POINTER(core); + return mali_hw_core_register_read(&core->hw_core, + MALI200_REG_ADDR_MGMT_INT_RAWSTAT); +} + + +MALI_STATIC_INLINE u32 mali_pp_is_active(struct mali_pp_core *core) +{ + u32 status = mali_hw_core_register_read(&core->hw_core, MALI200_REG_ADDR_MGMT_STATUS); + return (status & MALI200_REG_VAL_STATUS_RENDERING_ACTIVE) ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE void mali_pp_mask_all_interrupts(struct mali_pp_core *core) +{ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_NONE); +} + +MALI_STATIC_INLINE void mali_pp_enable_interrupts(struct mali_pp_core *core) +{ + mali_hw_core_register_write(&core->hw_core, MALI200_REG_ADDR_MGMT_INT_MASK, MALI200_REG_VAL_IRQ_MASK_USED); +} + +MALI_STATIC_INLINE void mali_pp_write_addr_renderer_list(struct mali_pp_core *core, + struct mali_pp_job *job, u32 subjob) +{ + u32 addr = mali_pp_job_get_addr_frame(job, subjob); + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_FRAME, addr); +} + + +MALI_STATIC_INLINE void mali_pp_write_addr_stack(struct mali_pp_core *core, struct mali_pp_job *job) +{ + u32 addr = mali_pp_job_get_addr_stack(job, core->core_id); + mali_hw_core_register_write_relaxed(&core->hw_core, MALI200_REG_ADDR_STACK, addr); +} + +#endif /* __MALI_PP_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp_job.c b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.c new file mode 100755 index 000000000000..b0216d4c1ac8 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.c @@ -0,0 +1,316 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_pp.h" +#include "mali_pp_job.h" +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_kernel_common.h" +#include "mali_uk_types.h" +#include "mali_executor.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +#include "linux/mali_memory_dma_buf.h" +#endif +#include "mali_memory_swap_alloc.h" +#include "mali_scheduler.h" + +static u32 pp_counter_src0 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */ +static u32 pp_counter_src1 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */ +static _mali_osk_atomic_t pp_counter_per_sub_job_count; /**< Number of values in the two arrays which is != MALI_HW_CORE_NO_COUNTER */ +static u32 pp_counter_per_sub_job_src0[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER }; +static u32 pp_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER }; + +void mali_pp_job_initialize(void) +{ + _mali_osk_atomic_init(&pp_counter_per_sub_job_count, 0); +} + +void mali_pp_job_terminate(void) +{ + _mali_osk_atomic_term(&pp_counter_per_sub_job_count); +} + +struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, + _mali_uk_pp_start_job_s __user *uargs, u32 id) +{ + struct mali_pp_job *job; + u32 perf_counter_flag; + + job = _mali_osk_calloc(1, sizeof(struct mali_pp_job)); + if (NULL != job) { + + _mali_osk_list_init(&job->list); + _mali_osk_list_init(&job->session_fb_lookup_list); + _mali_osk_atomic_inc(&session->number_of_pp_jobs); + + if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_pp_start_job_s))) { + goto fail; + } + + if (job->uargs.num_cores > _MALI_PP_MAX_SUB_JOBS) { + MALI_PRINT_ERROR(("Mali PP job: Too many sub jobs specified in job object\n")); + goto fail; + } + + if (!mali_pp_job_use_no_notification(job)) { + job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_FINISHED, sizeof(_mali_uk_pp_job_finished_s)); + if (NULL == job->finished_notification) goto fail; + } + + perf_counter_flag = mali_pp_job_get_perf_counter_flag(job); + + /* case when no counters came from user space + * so pass the debugfs / DS-5 provided global ones to the job object */ + if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) || + (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) { + u32 sub_job_count = _mali_osk_atomic_read(&pp_counter_per_sub_job_count); + + /* These counters apply for all virtual jobs, and where no per sub job counter is specified */ + job->uargs.perf_counter_src0 = pp_counter_src0; + job->uargs.perf_counter_src1 = pp_counter_src1; + + /* We only copy the per sub job array if it is enabled with at least one counter */ + if (0 < sub_job_count) { + job->perf_counter_per_sub_job_count = sub_job_count; + _mali_osk_memcpy(job->perf_counter_per_sub_job_src0, pp_counter_per_sub_job_src0, sizeof(pp_counter_per_sub_job_src0)); + _mali_osk_memcpy(job->perf_counter_per_sub_job_src1, pp_counter_per_sub_job_src1, sizeof(pp_counter_per_sub_job_src1)); + } + } + + job->session = session; + job->id = id; + + job->sub_jobs_num = job->uargs.num_cores ? job->uargs.num_cores : 1; + job->pid = _mali_osk_get_pid(); + job->tid = _mali_osk_get_tid(); + + _mali_osk_atomic_init(&job->sub_jobs_completed, 0); + _mali_osk_atomic_init(&job->sub_job_errors, 0); + job->swap_status = MALI_NO_SWAP_IN; + job->user_notification = MALI_FALSE; + job->num_pp_cores_in_virtual = 0; + + if (job->uargs.num_memory_cookies > session->allocation_mgr.mali_allocation_num) { + MALI_PRINT_ERROR(("Mali PP job: The number of memory cookies is invalid !\n")); + goto fail; + } + + if (job->uargs.num_memory_cookies > 0) { + u32 size; + u32 __user *memory_cookies = (u32 __user *)(uintptr_t)job->uargs.memory_cookies; + + size = sizeof(*memory_cookies) * (job->uargs.num_memory_cookies); + + job->memory_cookies = _mali_osk_malloc(size); + if (NULL == job->memory_cookies) { + MALI_PRINT_ERROR(("Mali PP job: Failed to allocate %d bytes of memory cookies!\n", size)); + goto fail; + } + + if (0 != _mali_osk_copy_from_user(job->memory_cookies, memory_cookies, size)) { + MALI_PRINT_ERROR(("Mali PP job: Failed to copy %d bytes of memory cookies from user!\n", size)); + goto fail; + } + } + + if (_MALI_OSK_ERR_OK != mali_pp_job_check(job)) { + /* Not a valid job. */ + goto fail; + } + + mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_PP, NULL, job); + mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence)); + + mali_mem_swap_in_pages(job); + + return job; + } + +fail: + if (NULL != job) { + mali_pp_job_delete(job); + } + + return NULL; +} + +void mali_pp_job_delete(struct mali_pp_job *job) +{ + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list)); + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->session_fb_lookup_list)); + + session = mali_pp_job_get_session(job); + MALI_DEBUG_ASSERT_POINTER(session); + + if (NULL != job->memory_cookies) { +#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) + /* Unmap buffers attached to job */ + mali_dma_buf_unmap_job(job); +#endif + if (MALI_NO_SWAP_IN != job->swap_status) { + mali_mem_swap_out_pages(job); + } + + _mali_osk_free(job->memory_cookies); + } + + if (job->user_notification) { + mali_scheduler_return_pp_job_to_user(job, + job->num_pp_cores_in_virtual); + } + + if (NULL != job->finished_notification) { + _mali_osk_notification_delete(job->finished_notification); + } + + _mali_osk_atomic_term(&job->sub_jobs_completed); + _mali_osk_atomic_term(&job->sub_job_errors); + _mali_osk_atomic_dec(&session->number_of_pp_jobs); + _mali_osk_free(job); + + _mali_osk_wait_queue_wake_up(session->wait_queue); +} + +void mali_pp_job_list_add(struct mali_pp_job *job, _mali_osk_list_t *list) +{ + struct mali_pp_job *iter; + struct mali_pp_job *tmp; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + + /* Find position in list/queue where job should be added. */ + _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, list, + struct mali_pp_job, list) { + /* job should be started after iter if iter is in progress. */ + if (0 < iter->sub_jobs_started) { + break; + } + + /* + * job should be started after iter if it has a higher + * job id. A span is used to handle job id wrapping. + */ + if ((mali_pp_job_get_id(job) - + mali_pp_job_get_id(iter)) < + MALI_SCHEDULER_JOB_ID_SPAN) { + break; + } + } + + _mali_osk_list_add(&job->list, &iter->list); +} + + +u32 mali_pp_job_get_perf_counter_src0(struct mali_pp_job *job, u32 sub_job) +{ + /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */ + if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) { + return job->uargs.perf_counter_src0; + } + + /* Use per sub job counter if enabled... */ + if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src0[sub_job]) { + return job->perf_counter_per_sub_job_src0[sub_job]; + } + + /* ...else default to global job counter */ + return job->uargs.perf_counter_src0; +} + +u32 mali_pp_job_get_perf_counter_src1(struct mali_pp_job *job, u32 sub_job) +{ + /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */ + if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) { + /* Virtual jobs always use the global job counter */ + return job->uargs.perf_counter_src1; + } + + /* Use per sub job counter if enabled... */ + if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src1[sub_job]) { + return job->perf_counter_per_sub_job_src1[sub_job]; + } + + /* ...else default to global job counter */ + return job->uargs.perf_counter_src1; +} + +void mali_pp_job_set_pp_counter_global_src0(u32 counter) +{ + pp_counter_src0 = counter; +} + +void mali_pp_job_set_pp_counter_global_src1(u32 counter) +{ + pp_counter_src1 = counter; +} + +void mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job, u32 counter) +{ + MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); + + if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src0[sub_job]) { + /* increment count since existing counter was disabled */ + _mali_osk_atomic_inc(&pp_counter_per_sub_job_count); + } + + if (MALI_HW_CORE_NO_COUNTER == counter) { + /* decrement count since new counter is disabled */ + _mali_osk_atomic_dec(&pp_counter_per_sub_job_count); + } + + /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */ + + pp_counter_per_sub_job_src0[sub_job] = counter; +} + +void mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job, u32 counter) +{ + MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); + + if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src1[sub_job]) { + /* increment count since existing counter was disabled */ + _mali_osk_atomic_inc(&pp_counter_per_sub_job_count); + } + + if (MALI_HW_CORE_NO_COUNTER == counter) { + /* decrement count since new counter is disabled */ + _mali_osk_atomic_dec(&pp_counter_per_sub_job_count); + } + + /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */ + + pp_counter_per_sub_job_src1[sub_job] = counter; +} + +u32 mali_pp_job_get_pp_counter_global_src0(void) +{ + return pp_counter_src0; +} + +u32 mali_pp_job_get_pp_counter_global_src1(void) +{ + return pp_counter_src1; +} + +u32 mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job) +{ + MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); + return pp_counter_per_sub_job_src0[sub_job]; +} + +u32 mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job) +{ + MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS); + return pp_counter_per_sub_job_src1[sub_job]; +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_pp_job.h b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.h new file mode 100755 index 000000000000..d0331f398ff9 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_pp_job.h @@ -0,0 +1,594 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PP_JOB_H__ +#define __MALI_PP_JOB_H__ + +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_uk_types.h" +#include "mali_session.h" +#include "mali_kernel_common.h" +#include "regs/mali_200_regs.h" +#include "mali_kernel_core.h" +#include "mali_dlbu.h" +#include "mali_timeline.h" +#include "mali_scheduler.h" +#include "mali_executor.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +#include "linux/mali_memory_dma_buf.h" +#endif +#if defined(CONFIG_MALI_DMA_BUF_FENCE) +#include "linux/mali_dma_fence.h" +#endif + +typedef enum pp_job_status { + MALI_NO_SWAP_IN, + MALI_SWAP_IN_FAIL, + MALI_SWAP_IN_SUCC, +} pp_job_status; + +/** + * This structure represents a PP job, including all sub jobs. + * + * The PP job object itself is not protected by any single lock, + * but relies on other locks instead (scheduler, executor and timeline lock). + * Think of the job object as moving between these sub systems through-out + * its lifetime. Different part of the PP job struct is used by different + * subsystems. Accessor functions ensure that correct lock is taken. + * Do NOT access any data members directly from outside this module! + */ +struct mali_pp_job { + /* + * These members are typically only set at creation, + * and only read later on. + * They do not require any lock protection. + */ + _mali_uk_pp_start_job_s uargs; /**< Arguments from user space */ + struct mali_session_data *session; /**< Session which submitted this job */ + u32 pid; /**< Process ID of submitting process */ + u32 tid; /**< Thread ID of submitting thread */ + u32 id; /**< Identifier for this job in kernel space (sequential numbering) */ + u32 cache_order; /**< Cache order used for L2 cache flushing (sequential numbering) */ + struct mali_timeline_tracker tracker; /**< Timeline tracker for this job */ + _mali_osk_notification_t *finished_notification; /**< Notification sent back to userspace on job complete */ + u32 perf_counter_per_sub_job_count; /**< Number of values in the two arrays which is != MALI_HW_CORE_NO_COUNTER */ + u32 perf_counter_per_sub_job_src0[_MALI_PP_MAX_SUB_JOBS]; /**< Per sub job counters src0 */ + u32 perf_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS]; /**< Per sub job counters src1 */ + u32 sub_jobs_num; /**< Number of subjobs; set to 1 for Mali-450 if DLBU is used, otherwise equals number of PP cores */ + + pp_job_status swap_status; /**< Used to track each PP job swap status, if fail, we need to drop them in scheduler part */ + mali_bool user_notification; /**< When we deferred delete PP job, we need to judge if we need to send job finish notification to user space */ + u32 num_pp_cores_in_virtual; /**< How many PP cores we have when job finished */ + + /* + * These members are used by both scheduler and executor. + * They are "protected" by atomic operations. + */ + _mali_osk_atomic_t sub_jobs_completed; /**< Number of completed sub-jobs in this superjob */ + _mali_osk_atomic_t sub_job_errors; /**< Bitfield with errors (errors for each single sub-job is or'ed together) */ + + /* + * These members are used by scheduler, but only when no one else + * knows about this job object but the working function. + * No lock is thus needed for these. + */ + u32 *memory_cookies; /**< Memory cookies attached to job */ + + /* + * These members are used by the scheduler, + * protected by scheduler lock + */ + _mali_osk_list_t list; /**< Used to link jobs together in the scheduler queue */ + _mali_osk_list_t session_fb_lookup_list; /**< Used to link jobs together from the same frame builder in the session */ + + u32 sub_jobs_started; /**< Total number of sub-jobs started (always started in ascending order) */ + + /* + * Set by executor/group on job completion, read by scheduler when + * returning job to user. Hold executor lock when setting, + * no lock needed when reading + */ + u32 perf_counter_value0[_MALI_PP_MAX_SUB_JOBS]; /**< Value of performance counter 0 (to be returned to user space), one for each sub job */ + u32 perf_counter_value1[_MALI_PP_MAX_SUB_JOBS]; /**< Value of performance counter 1 (to be returned to user space), one for each sub job */ + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) + struct mali_dma_fence_context dma_fence_context; /**< The mali dma fence context to record dma fence waiters that this job wait for */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + struct dma_fence *rendered_dma_fence; /**< the new dma fence link to this job */ +#else + struct fence *rendered_dma_fence; /**< the new dma fence link to this job */ +#endif +#endif +}; + +void mali_pp_job_initialize(void); +void mali_pp_job_terminate(void); + +struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, _mali_uk_pp_start_job_s *uargs, u32 id); +void mali_pp_job_delete(struct mali_pp_job *job); + +u32 mali_pp_job_get_perf_counter_src0(struct mali_pp_job *job, u32 sub_job); +u32 mali_pp_job_get_perf_counter_src1(struct mali_pp_job *job, u32 sub_job); + +void mali_pp_job_set_pp_counter_global_src0(u32 counter); +void mali_pp_job_set_pp_counter_global_src1(u32 counter); +void mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job, u32 counter); +void mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job, u32 counter); + +u32 mali_pp_job_get_pp_counter_global_src0(void); +u32 mali_pp_job_get_pp_counter_global_src1(void); +u32 mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job); +u32 mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job); + +MALI_STATIC_INLINE u32 mali_pp_job_get_id(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (NULL == job) ? 0 : job->id; +} + +MALI_STATIC_INLINE void mali_pp_job_set_cache_order(struct mali_pp_job *job, + u32 cache_order) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + job->cache_order = cache_order; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_cache_order(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (NULL == job) ? 0 : job->cache_order; +} + +MALI_STATIC_INLINE u64 mali_pp_job_get_user_id(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.user_job_ptr; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_frame_builder_id(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.frame_builder_id; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_flush_id(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.flush_id; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_pid(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->pid; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_tid(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->tid; +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_frame_registers(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.frame_registers; +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_dlbu_registers(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.dlbu_registers; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_is_virtual(struct mali_pp_job *job) +{ +#if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) + MALI_DEBUG_ASSERT_POINTER(job); + return (0 == job->uargs.num_cores) ? MALI_TRUE : MALI_FALSE; +#else + return MALI_FALSE; +#endif +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_addr_frame(struct mali_pp_job *job, u32 sub_job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + if (mali_pp_job_is_virtual(job)) { + return MALI_DLBU_VIRT_ADDR; + } else if (0 == sub_job) { + return job->uargs.frame_registers[MALI200_REG_ADDR_FRAME / sizeof(u32)]; + } else if (sub_job < _MALI_PP_MAX_SUB_JOBS) { + return job->uargs.frame_registers_addr_frame[sub_job - 1]; + } + + return 0; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_addr_stack(struct mali_pp_job *job, u32 sub_job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + if (0 == sub_job) { + return job->uargs.frame_registers[MALI200_REG_ADDR_STACK / sizeof(u32)]; + } else if (sub_job < _MALI_PP_MAX_SUB_JOBS) { + return job->uargs.frame_registers_addr_stack[sub_job - 1]; + } + + return 0; +} + +void mali_pp_job_list_add(struct mali_pp_job *job, _mali_osk_list_t *list); + +MALI_STATIC_INLINE void mali_pp_job_list_addtail(struct mali_pp_job *job, + _mali_osk_list_t *list) +{ + _mali_osk_list_addtail(&job->list, list); +} + +MALI_STATIC_INLINE void mali_pp_job_list_move(struct mali_pp_job *job, + _mali_osk_list_t *list) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT(!_mali_osk_list_empty(&job->list)); + _mali_osk_list_move(&job->list, list); +} + +MALI_STATIC_INLINE void mali_pp_job_list_remove(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + _mali_osk_list_delinit(&job->list); +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_wb0_registers(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb0_registers; +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_wb1_registers(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb1_registers; +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_wb2_registers(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb2_registers; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_wb0_source_addr(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)]; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_wb1_source_addr(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)]; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_wb2_source_addr(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)]; +} + +MALI_STATIC_INLINE void mali_pp_job_disable_wb0(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0; +} + +MALI_STATIC_INLINE void mali_pp_job_disable_wb1(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0; +} + +MALI_STATIC_INLINE void mali_pp_job_disable_wb2(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_all_writeback_unit_disabled(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + if (job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] || + job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] || + job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] + ) { + /* At least one output unit active */ + return MALI_FALSE; + } + + /* All outputs are disabled - we can abort the job */ + return MALI_TRUE; +} + +MALI_STATIC_INLINE void mali_pp_job_fb_lookup_add(struct mali_pp_job *job) +{ + u32 fb_lookup_id; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + + fb_lookup_id = MALI_PP_JOB_FB_LOOKUP_LIST_MASK & job->uargs.frame_builder_id; + + MALI_DEBUG_ASSERT(MALI_PP_JOB_FB_LOOKUP_LIST_SIZE > fb_lookup_id); + + _mali_osk_list_addtail(&job->session_fb_lookup_list, + &job->session->pp_job_fb_lookup_list[fb_lookup_id]); +} + +MALI_STATIC_INLINE void mali_pp_job_fb_lookup_remove(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + _mali_osk_list_delinit(&job->session_fb_lookup_list); +} + +MALI_STATIC_INLINE struct mali_session_data *mali_pp_job_get_session(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->session; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_has_started_sub_jobs(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + return (0 < job->sub_jobs_started) ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_has_unstarted_sub_jobs(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + return (job->sub_jobs_started < job->sub_jobs_num) ? MALI_TRUE : MALI_FALSE; +} + +/* Function used when we are terminating a session with jobs. Return TRUE if it has a rendering job. + Makes sure that no new subjobs are started. */ +MALI_STATIC_INLINE void mali_pp_job_mark_unstarted_failed(struct mali_pp_job *job) +{ + u32 jobs_remaining; + u32 i; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + + jobs_remaining = job->sub_jobs_num - job->sub_jobs_started; + job->sub_jobs_started += jobs_remaining; + + /* Not the most optimal way, but this is only used in error cases */ + for (i = 0; i < jobs_remaining; i++) { + _mali_osk_atomic_inc(&job->sub_jobs_completed); + _mali_osk_atomic_inc(&job->sub_job_errors); + } +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_is_complete(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->sub_jobs_num == + _mali_osk_atomic_read(&job->sub_jobs_completed)) ? + MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_first_unstarted_sub_job(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + return job->sub_jobs_started; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_sub_job_count(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->sub_jobs_num; +} + +MALI_STATIC_INLINE u32 mali_pp_job_unstarted_sub_job_count(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT(job->sub_jobs_num >= job->sub_jobs_started); + return (job->sub_jobs_num - job->sub_jobs_started); +} + +MALI_STATIC_INLINE u32 mali_pp_job_num_memory_cookies(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.num_memory_cookies; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_memory_cookie( + struct mali_pp_job *job, u32 index) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT(index < job->uargs.num_memory_cookies); + MALI_DEBUG_ASSERT_POINTER(job->memory_cookies); + return job->memory_cookies[index]; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_needs_dma_buf_mapping(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + if (0 < job->uargs.num_memory_cookies) { + return MALI_TRUE; + } + + return MALI_FALSE; +} + +MALI_STATIC_INLINE void mali_pp_job_mark_sub_job_started(struct mali_pp_job *job, u32 sub_job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + + /* Assert that we are marking the "first unstarted sub job" as started */ + MALI_DEBUG_ASSERT(job->sub_jobs_started == sub_job); + + job->sub_jobs_started++; +} + +MALI_STATIC_INLINE void mali_pp_job_mark_sub_job_completed(struct mali_pp_job *job, mali_bool success) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + _mali_osk_atomic_inc(&job->sub_jobs_completed); + if (MALI_FALSE == success) { + _mali_osk_atomic_inc(&job->sub_job_errors); + } +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_was_success(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + if (0 == _mali_osk_atomic_read(&job->sub_job_errors)) { + return MALI_TRUE; + } + return MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_use_no_notification( + struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->uargs.flags & _MALI_PP_JOB_FLAG_NO_NOTIFICATION) ? + MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_is_pilot_job(struct mali_pp_job *job) +{ + /* + * A pilot job is currently identified as jobs which + * require no callback notification. + */ + return mali_pp_job_use_no_notification(job); +} + +MALI_STATIC_INLINE _mali_osk_notification_t * +mali_pp_job_get_finished_notification(struct mali_pp_job *job) +{ + _mali_osk_notification_t *notification; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(job->finished_notification); + + notification = job->finished_notification; + job->finished_notification = NULL; + + return notification; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_is_window_surface( + struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->uargs.flags & _MALI_PP_JOB_FLAG_IS_WINDOW_SURFACE) + ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE mali_bool mali_pp_job_is_protected_job(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (job->uargs.flags & _MALI_PP_JOB_FLAG_PROTECTED) + ? MALI_TRUE : MALI_FALSE; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_flag(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->uargs.perf_counter_flag; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_value0(struct mali_pp_job *job, u32 sub_job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->perf_counter_value0[sub_job]; +} + +MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_value1(struct mali_pp_job *job, u32 sub_job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return job->perf_counter_value1[sub_job]; +} + +MALI_STATIC_INLINE void mali_pp_job_set_perf_counter_value0(struct mali_pp_job *job, u32 sub_job, u32 value) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + job->perf_counter_value0[sub_job] = value; +} + +MALI_STATIC_INLINE void mali_pp_job_set_perf_counter_value1(struct mali_pp_job *job, u32 sub_job, u32 value) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD(); + job->perf_counter_value1[sub_job] = value; +} + +MALI_STATIC_INLINE _mali_osk_errcode_t mali_pp_job_check(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + if (mali_pp_job_is_virtual(job) && job->sub_jobs_num != 1) { + return _MALI_OSK_ERR_FAULT; + } + return _MALI_OSK_ERR_OK; +} + +/** + * Returns MALI_TRUE if this job has more than two sub jobs and all sub jobs are unstarted. + * + * @param job Job to check. + * @return MALI_TRUE if job has more than two sub jobs and all sub jobs are unstarted, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_pp_job_is_large_and_unstarted(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT(!mali_pp_job_is_virtual(job)); + + return (0 == job->sub_jobs_started && 2 < job->sub_jobs_num); +} + +/** + * Get PP job's Timeline tracker. + * + * @param job PP job. + * @return Pointer to Timeline tracker for the job. + */ +MALI_STATIC_INLINE struct mali_timeline_tracker *mali_pp_job_get_tracker(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return &(job->tracker); +} + +MALI_STATIC_INLINE u32 *mali_pp_job_get_timeline_point_ptr( + struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + return (u32 __user *)(uintptr_t)job->uargs.timeline_point_ptr; +} + + +#endif /* __MALI_PP_JOB_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_scheduler.c b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.c new file mode 100755 index 000000000000..b5e6cfddbb0e --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.c @@ -0,0 +1,1548 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_scheduler.h" +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_osk_profiling.h" +#include "mali_kernel_utilization.h" +#include "mali_timeline.h" +#include "mali_gp_job.h" +#include "mali_pp_job.h" +#include "mali_executor.h" +#include "mali_group.h" +#include +#include +#include "mali_pm_metrics.h" + +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include "mali_memory_dma_buf.h" +#if defined(CONFIG_MALI_DMA_BUF_FENCE) +#include "mali_dma_fence.h" +#include +#endif +#endif + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) +#include +#include +#endif +/* + * ---------- static defines/constants ---------- + */ + +/* + * If dma_buf with map on demand is used, we defer job queue + * if in atomic context, since both might sleep. + */ +#if defined(CONFIG_DMA_SHARED_BUFFER) +#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +#define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE 1 +#endif +#endif + + +/* + * ---------- global variables (exported due to inline functions) ---------- + */ + +/* Lock protecting this module */ +_mali_osk_spinlock_irq_t *mali_scheduler_lock_obj = NULL; + +/* Queue of jobs to be executed on the GP group */ +struct mali_scheduler_job_queue job_queue_gp; + +/* Queue of PP jobs */ +struct mali_scheduler_job_queue job_queue_pp; + +_mali_osk_atomic_t mali_job_id_autonumber; +_mali_osk_atomic_t mali_job_cache_order_autonumber; +/* + * ---------- static variables ---------- + */ + +_mali_osk_wq_work_t *scheduler_wq_pp_job_delete = NULL; +_mali_osk_spinlock_irq_t *scheduler_pp_job_delete_lock = NULL; +static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_deletion_queue); + +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) +static _mali_osk_wq_work_t *scheduler_wq_pp_job_queue = NULL; +static _mali_osk_spinlock_irq_t *scheduler_pp_job_queue_lock = NULL; +static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_queue_list); +#endif + +/* + * ---------- Forward declaration of static functions ---------- + */ + +static mali_timeline_point mali_scheduler_submit_gp_job( + struct mali_session_data *session, struct mali_gp_job *job); +static _mali_osk_errcode_t mali_scheduler_submit_pp_job( + struct mali_session_data *session, struct mali_pp_job *job, mali_timeline_point *point); + +static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job); +static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job); + +static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job, + mali_bool success); + +static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job); +void mali_scheduler_do_pp_job_delete(void *arg); + +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) +static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job); +static void mali_scheduler_do_pp_job_queue(void *arg); +#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + +/* + * ---------- Actual implementation ---------- + */ + +_mali_osk_errcode_t mali_scheduler_initialize(void) +{ + _mali_osk_atomic_init(&mali_job_id_autonumber, 0); + _mali_osk_atomic_init(&mali_job_cache_order_autonumber, 0); + + _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.normal_pri); + _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.high_pri); + job_queue_gp.depth = 0; + job_queue_gp.big_job_num = 0; + + _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.normal_pri); + _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.high_pri); + job_queue_pp.depth = 0; + job_queue_pp.big_job_num = 0; + + mali_scheduler_lock_obj = _mali_osk_spinlock_irq_init( + _MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_SCHEDULER); + if (NULL == mali_scheduler_lock_obj) { + mali_scheduler_terminate(); + } + + scheduler_wq_pp_job_delete = _mali_osk_wq_create_work( + mali_scheduler_do_pp_job_delete, NULL); + if (NULL == scheduler_wq_pp_job_delete) { + mali_scheduler_terminate(); + return _MALI_OSK_ERR_FAULT; + } + + scheduler_pp_job_delete_lock = _mali_osk_spinlock_irq_init( + _MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED); + if (NULL == scheduler_pp_job_delete_lock) { + mali_scheduler_terminate(); + return _MALI_OSK_ERR_FAULT; + } + +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) + scheduler_wq_pp_job_queue = _mali_osk_wq_create_work( + mali_scheduler_do_pp_job_queue, NULL); + if (NULL == scheduler_wq_pp_job_queue) { + mali_scheduler_terminate(); + return _MALI_OSK_ERR_FAULT; + } + + scheduler_pp_job_queue_lock = _mali_osk_spinlock_irq_init( + _MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED); + if (NULL == scheduler_pp_job_queue_lock) { + mali_scheduler_terminate(); + return _MALI_OSK_ERR_FAULT; + } +#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + + return _MALI_OSK_ERR_OK; +} + +void mali_scheduler_terminate(void) +{ +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) + if (NULL != scheduler_pp_job_queue_lock) { + _mali_osk_spinlock_irq_term(scheduler_pp_job_queue_lock); + scheduler_pp_job_queue_lock = NULL; + } + + if (NULL != scheduler_wq_pp_job_queue) { + _mali_osk_wq_delete_work(scheduler_wq_pp_job_queue); + scheduler_wq_pp_job_queue = NULL; + } +#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + + if (NULL != scheduler_pp_job_delete_lock) { + _mali_osk_spinlock_irq_term(scheduler_pp_job_delete_lock); + scheduler_pp_job_delete_lock = NULL; + } + + if (NULL != scheduler_wq_pp_job_delete) { + _mali_osk_wq_delete_work(scheduler_wq_pp_job_delete); + scheduler_wq_pp_job_delete = NULL; + } + + if (NULL != mali_scheduler_lock_obj) { + _mali_osk_spinlock_irq_term(mali_scheduler_lock_obj); + mali_scheduler_lock_obj = NULL; + } + + _mali_osk_atomic_term(&mali_job_cache_order_autonumber); + _mali_osk_atomic_term(&mali_job_id_autonumber); +} + +u32 mali_scheduler_job_physical_head_count(mali_bool gpu_mode_is_secure) +{ + /* + * Count how many physical sub jobs are present from the head of queue + * until the first virtual job is present. + * Early out when we have reached maximum number of PP cores (8) + */ + u32 count = 0; + struct mali_pp_job *job; + struct mali_pp_job *temp; + + /* Check for partially started normal pri jobs */ + if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, + struct mali_pp_job, list); + + MALI_DEBUG_ASSERT_POINTER(job); + + if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) { + /* + * Remember; virtual jobs can't be queued and started + * at the same time, so this must be a physical job + */ + if ((MALI_FALSE == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job)) + || (MALI_TRUE == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job))) { + + count += mali_pp_job_unstarted_sub_job_count(job); + if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) { + return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; + } + } + } + } + + _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri, + struct mali_pp_job, list) { + if ((MALI_FALSE == mali_pp_job_is_virtual(job)) + && ((MALI_FALSE == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job)) + || (MALI_TRUE == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job)))) { + + count += mali_pp_job_unstarted_sub_job_count(job); + if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) { + return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; + } + } else { + /* Came across a virtual job, so stop counting */ + return count; + } + } + + _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri, + struct mali_pp_job, list) { + if ((MALI_FALSE == mali_pp_job_is_virtual(job)) + && (MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) + && ((MALI_FALSE == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job)) + || (MALI_TRUE == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job)))) { + + count += mali_pp_job_unstarted_sub_job_count(job); + if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) { + return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS; + } + } else { + /* Came across a virtual job, so stop counting */ + return count; + } + } + return count; +} + +struct mali_pp_job *mali_scheduler_job_pp_next(void) +{ + struct mali_pp_job *job; + struct mali_pp_job *temp; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + + /* Check for partially started normal pri jobs */ + if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, + struct mali_pp_job, list); + + MALI_DEBUG_ASSERT_POINTER(job); + + if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) { + return job; + } + } + + _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri, + struct mali_pp_job, list) { + return job; + } + + _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri, + struct mali_pp_job, list) { + return job; + } + + return NULL; +} + +mali_bool mali_scheduler_job_next_is_virtual(void) +{ + struct mali_pp_job *job; + + job = mali_scheduler_job_pp_virtual_peek(); + if (NULL != job) { + MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job)); + + return MALI_TRUE; + } + + return MALI_FALSE; +} + +struct mali_gp_job *mali_scheduler_job_gp_get(void) +{ + _mali_osk_list_t *queue; + struct mali_gp_job *job = NULL; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + MALI_DEBUG_ASSERT(0 < job_queue_gp.depth); + MALI_DEBUG_ASSERT(job_queue_gp.big_job_num <= job_queue_gp.depth); + + if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) { + queue = &job_queue_gp.high_pri; + } else { + queue = &job_queue_gp.normal_pri; + MALI_DEBUG_ASSERT(!_mali_osk_list_empty(queue)); + } + + job = _MALI_OSK_LIST_ENTRY(queue->next, struct mali_gp_job, list); + + MALI_DEBUG_ASSERT_POINTER(job); + + mali_gp_job_list_remove(job); + job_queue_gp.depth--; + if (job->big_job) { + job_queue_gp.big_job_num --; + if (job_queue_gp.big_job_num < MALI_MAX_PENDING_BIG_JOB) { + /* wake up process */ + wait_queue_head_t *queue = mali_session_get_wait_queue(); + wake_up(queue); + } + } + return job; +} + +struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void) +{ + struct mali_pp_job *job = NULL; + struct mali_pp_job *tmp_job = NULL; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + + /* + * For PP jobs we favour partially started jobs in normal + * priority queue over unstarted jobs in high priority queue + */ + + if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, + struct mali_pp_job, list); + MALI_DEBUG_ASSERT(NULL != tmp_job); + + if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) { + job = tmp_job; + } + } + + if (NULL == job || + MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) { + /* + * There isn't a partially started job in normal queue, so + * look in high priority queue. + */ + if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next, + struct mali_pp_job, list); + MALI_DEBUG_ASSERT(NULL != tmp_job); + + if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) { + job = tmp_job; + } + } + } + + return job; +} + +struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void) +{ + struct mali_pp_job *job = NULL; + struct mali_pp_job *tmp_job = NULL; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + + if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next, + struct mali_pp_job, list); + + if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) { + job = tmp_job; + } + } + + if (NULL == job) { + if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { + MALI_DEBUG_ASSERT(0 < job_queue_pp.depth); + + tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next, + struct mali_pp_job, list); + + if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) { + job = tmp_job; + } + } + } + + return job; +} + +struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job) +{ + struct mali_pp_job *job = mali_scheduler_job_pp_physical_peek(); + + MALI_DEBUG_ASSERT(MALI_FALSE == mali_pp_job_is_virtual(job)); + + if (NULL != job) { + *sub_job = mali_pp_job_get_first_unstarted_sub_job(job); + + mali_pp_job_mark_sub_job_started(job, *sub_job); + if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(job)) { + /* Remove from queue when last sub job has been retrieved */ + mali_pp_job_list_remove(job); + } + + job_queue_pp.depth--; + + /* + * Job about to start so it is no longer be + * possible to discard WB + */ + mali_pp_job_fb_lookup_remove(job); + } + + return job; +} + +struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void) +{ + struct mali_pp_job *job = mali_scheduler_job_pp_virtual_peek(); + + MALI_DEBUG_ASSERT(MALI_TRUE == mali_pp_job_is_virtual(job)); + + if (NULL != job) { + MALI_DEBUG_ASSERT(0 == + mali_pp_job_get_first_unstarted_sub_job(job)); + MALI_DEBUG_ASSERT(1 == + mali_pp_job_get_sub_job_count(job)); + + mali_pp_job_mark_sub_job_started(job, 0); + + mali_pp_job_list_remove(job); + + job_queue_pp.depth--; + + /* + * Job about to start so it is no longer be + * possible to discard WB + */ + mali_pp_job_fb_lookup_remove(job); + } + + return job; +} + +mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Timeline activation for job %u (0x%08X).\n", + mali_gp_job_get_id(job), job)); + + mali_scheduler_lock(); + + if (!mali_scheduler_queue_gp_job(job)) { + /* Failed to enqueue job, release job (with error) */ + + mali_scheduler_unlock(); + + mali_timeline_tracker_release(mali_gp_job_get_tracker(job)); + mali_gp_job_signal_pp_tracker(job, MALI_FALSE); + + /* This will notify user space and close the job object */ + mali_scheduler_complete_gp_job(job, MALI_FALSE, + MALI_TRUE, MALI_FALSE); + + return MALI_SCHEDULER_MASK_EMPTY; + } + + mali_scheduler_unlock(); + + return MALI_SCHEDULER_MASK_GP; +} + +mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Timeline activation for job %u (0x%08X).\n", + mali_pp_job_get_id(job), job)); + + if (MALI_TRUE == mali_timeline_tracker_activation_error( + mali_pp_job_get_tracker(job))) { + MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Job %u (0x%08X) activated with error, aborting.\n", + mali_pp_job_get_id(job), job)); + + mali_scheduler_lock(); + mali_pp_job_fb_lookup_remove(job); + mali_pp_job_mark_unstarted_failed(job); + mali_scheduler_unlock(); + + mali_timeline_tracker_release(mali_pp_job_get_tracker(job)); + + /* This will notify user space and close the job object */ + mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE); + + return MALI_SCHEDULER_MASK_EMPTY; + } + +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) + if (mali_pp_job_needs_dma_buf_mapping(job)) { + mali_scheduler_deferred_pp_job_queue(job); + return MALI_SCHEDULER_MASK_EMPTY; + } +#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + + mali_scheduler_lock(); + + if (!mali_scheduler_queue_pp_job(job)) { + /* Failed to enqueue job, release job (with error) */ + mali_pp_job_fb_lookup_remove(job); + mali_pp_job_mark_unstarted_failed(job); + mali_scheduler_unlock(); + + mali_timeline_tracker_release(mali_pp_job_get_tracker(job)); + + /* This will notify user space and close the job object */ + mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE); + + return MALI_SCHEDULER_MASK_EMPTY; + } + + mali_scheduler_unlock(); + return MALI_SCHEDULER_MASK_PP; +} + +void mali_scheduler_complete_gp_job(struct mali_gp_job *job, + mali_bool success, + mali_bool user_notification, + mali_bool dequeued) +{ + if (user_notification) { + mali_scheduler_return_gp_job_to_user(job, success); + } + + if (dequeued) { + _mali_osk_pm_dev_ref_put(); + + if (mali_utilization_enabled()) { + mali_utilization_gp_end(); + } + mali_pm_record_gpu_idle(MALI_TRUE); + } + + mali_gp_job_delete(job); +} + +void mali_scheduler_complete_pp_job(struct mali_pp_job *job, + u32 num_cores_in_virtual, + mali_bool user_notification, + mali_bool dequeued) +{ + job->user_notification = user_notification; + job->num_pp_cores_in_virtual = num_cores_in_virtual; + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) + if (NULL != job->rendered_dma_fence) + mali_dma_fence_signal_and_put(&job->rendered_dma_fence); +#endif + + if (dequeued) { +#if defined(CONFIG_MALI_DVFS) + if (mali_pp_job_is_window_surface(job)) { + struct mali_session_data *session; + session = mali_pp_job_get_session(job); + mali_session_inc_num_window_jobs(session); + } +#endif + _mali_osk_pm_dev_ref_put(); + + if (mali_utilization_enabled()) { + mali_utilization_pp_end(); + } + mali_pm_record_gpu_idle(MALI_FALSE); + } + + /* With ZRAM feature enabled, all pp jobs will be force to use deferred delete. */ + mali_scheduler_deferred_pp_job_delete(job); +} + +void mali_scheduler_abort_session(struct mali_session_data *session) +{ + struct mali_gp_job *gp_job; + struct mali_gp_job *gp_tmp; + struct mali_pp_job *pp_job; + struct mali_pp_job *pp_tmp; + _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_gp); + _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_pp); + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT(session->is_aborting); + + MALI_DEBUG_PRINT(3, ("Mali scheduler: Aborting all queued jobs from session 0x%08X.\n", + session)); + + mali_scheduler_lock(); + + /* Remove from GP normal priority queue */ + _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.normal_pri, + struct mali_gp_job, list) { + if (mali_gp_job_get_session(gp_job) == session) { + mali_gp_job_list_move(gp_job, &removed_jobs_gp); + job_queue_gp.depth--; + job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0; + } + } + + /* Remove from GP high priority queue */ + _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.high_pri, + struct mali_gp_job, list) { + if (mali_gp_job_get_session(gp_job) == session) { + mali_gp_job_list_move(gp_job, &removed_jobs_gp); + job_queue_gp.depth--; + job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0; + } + } + + /* Remove from PP normal priority queue */ + _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, + &job_queue_pp.normal_pri, + struct mali_pp_job, list) { + if (mali_pp_job_get_session(pp_job) == session) { + mali_pp_job_fb_lookup_remove(pp_job); + + job_queue_pp.depth -= + mali_pp_job_unstarted_sub_job_count( + pp_job); + mali_pp_job_mark_unstarted_failed(pp_job); + + if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) { + if (mali_pp_job_is_complete(pp_job)) { + mali_pp_job_list_move(pp_job, + &removed_jobs_pp); + } else { + mali_pp_job_list_remove(pp_job); + } + } + } + } + + /* Remove from PP high priority queue */ + _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, + &job_queue_pp.high_pri, + struct mali_pp_job, list) { + if (mali_pp_job_get_session(pp_job) == session) { + mali_pp_job_fb_lookup_remove(pp_job); + + job_queue_pp.depth -= + mali_pp_job_unstarted_sub_job_count( + pp_job); + mali_pp_job_mark_unstarted_failed(pp_job); + + if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) { + if (mali_pp_job_is_complete(pp_job)) { + mali_pp_job_list_move(pp_job, + &removed_jobs_pp); + } else { + mali_pp_job_list_remove(pp_job); + } + } + } + } + + /* + * Release scheduler lock so we can release trackers + * (which will potentially queue new jobs) + */ + mali_scheduler_unlock(); + + /* Release and complete all (non-running) found GP jobs */ + _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &removed_jobs_gp, + struct mali_gp_job, list) { + mali_timeline_tracker_release(mali_gp_job_get_tracker(gp_job)); + mali_gp_job_signal_pp_tracker(gp_job, MALI_FALSE); + _mali_osk_list_delinit(&gp_job->list); + mali_scheduler_complete_gp_job(gp_job, + MALI_FALSE, MALI_FALSE, MALI_TRUE); + } + + /* Release and complete non-running PP jobs */ + _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, &removed_jobs_pp, + struct mali_pp_job, list) { + mali_timeline_tracker_release(mali_pp_job_get_tracker(pp_job)); + _mali_osk_list_delinit(&pp_job->list); + mali_scheduler_complete_pp_job(pp_job, 0, + MALI_FALSE, MALI_TRUE); + } +} + +_mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx, + _mali_uk_gp_start_job_s *uargs) +{ + struct mali_session_data *session; + struct mali_gp_job *job; + mali_timeline_point point; + u32 __user *point_ptr = NULL; + + MALI_DEBUG_ASSERT_POINTER(uargs); + MALI_DEBUG_ASSERT_POINTER(ctx); + + session = (struct mali_session_data *)(uintptr_t)ctx; + + job = mali_gp_job_create(session, uargs, mali_scheduler_get_new_id(), + NULL); + if (NULL == job) { + MALI_PRINT_ERROR(("Failed to create GP job.\n")); + return _MALI_OSK_ERR_NOMEM; + } + + point_ptr = (u32 __user *)(uintptr_t)mali_gp_job_get_timeline_point_ptr(job); + + point = mali_scheduler_submit_gp_job(session, job); + + if (0 != _mali_osk_put_user(((u32) point), point_ptr)) { + /* + * Let user space know that something failed + * after the job was started. + */ + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx, + _mali_uk_pp_start_job_s *uargs) +{ + _mali_osk_errcode_t ret; + struct mali_session_data *session; + struct mali_pp_job *job; + mali_timeline_point point; + u32 __user *point_ptr = NULL; + + MALI_DEBUG_ASSERT_POINTER(uargs); + MALI_DEBUG_ASSERT_POINTER(ctx); + + session = (struct mali_session_data *)(uintptr_t)ctx; + + job = mali_pp_job_create(session, uargs, mali_scheduler_get_new_id()); + if (NULL == job) { + MALI_PRINT_ERROR(("Failed to create PP job.\n")); + return _MALI_OSK_ERR_NOMEM; + } + + point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(job); + + /* Submit PP job. */ + ret = mali_scheduler_submit_pp_job(session, job, &point); + job = NULL; + + if (_MALI_OSK_ERR_OK == ret) { + if (0 != _mali_osk_put_user(((u32) point), point_ptr)) { + /* + * Let user space know that something failed + * after the jobs were started. + */ + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + } + + return ret; +} + +_mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx, + _mali_uk_pp_and_gp_start_job_s *uargs) +{ + _mali_osk_errcode_t ret; + struct mali_session_data *session; + _mali_uk_pp_and_gp_start_job_s kargs; + struct mali_pp_job *pp_job; + struct mali_gp_job *gp_job; + u32 __user *point_ptr = NULL; + mali_timeline_point point; + _mali_uk_pp_start_job_s __user *pp_args; + _mali_uk_gp_start_job_s __user *gp_args; + + MALI_DEBUG_ASSERT_POINTER(ctx); + MALI_DEBUG_ASSERT_POINTER(uargs); + + session = (struct mali_session_data *) ctx; + + if (0 != _mali_osk_copy_from_user(&kargs, uargs, + sizeof(_mali_uk_pp_and_gp_start_job_s))) { + return _MALI_OSK_ERR_NOMEM; + } + + pp_args = (_mali_uk_pp_start_job_s __user *)(uintptr_t)kargs.pp_args; + gp_args = (_mali_uk_gp_start_job_s __user *)(uintptr_t)kargs.gp_args; + + pp_job = mali_pp_job_create(session, pp_args, + mali_scheduler_get_new_id()); + if (NULL == pp_job) { + MALI_PRINT_ERROR(("Failed to create PP job.\n")); + return _MALI_OSK_ERR_NOMEM; + } + + gp_job = mali_gp_job_create(session, gp_args, + mali_scheduler_get_new_id(), + mali_pp_job_get_tracker(pp_job)); + if (NULL == gp_job) { + MALI_PRINT_ERROR(("Failed to create GP job.\n")); + mali_pp_job_delete(pp_job); + return _MALI_OSK_ERR_NOMEM; + } + + point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(pp_job); + + /* Submit GP job. */ + mali_scheduler_submit_gp_job(session, gp_job); + gp_job = NULL; + + /* Submit PP job. */ + ret = mali_scheduler_submit_pp_job(session, pp_job, &point); + pp_job = NULL; + + if (_MALI_OSK_ERR_OK == ret) { + if (0 != _mali_osk_put_user(((u32) point), point_ptr)) { + /* + * Let user space know that something failed + * after the jobs were started. + */ + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + } + + return ret; +} + +void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args) +{ + struct mali_session_data *session; + struct mali_pp_job *job; + struct mali_pp_job *tmp; + u32 fb_lookup_id; + + MALI_DEBUG_ASSERT_POINTER(args); + MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + + fb_lookup_id = args->fb_id & MALI_PP_JOB_FB_LOOKUP_LIST_MASK; + + mali_scheduler_lock(); + + /* Iterate over all jobs for given frame builder_id. */ + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, + &session->pp_job_fb_lookup_list[fb_lookup_id], + struct mali_pp_job, session_fb_lookup_list) { + MALI_DEBUG_CODE(u32 disable_mask = 0); + + if (mali_pp_job_get_frame_builder_id(job) != + (u32) args->fb_id) { + MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Disable WB mismatching FB.\n")); + continue; + } + + MALI_DEBUG_CODE(disable_mask |= 0xD << (4 * 3)); + + if (mali_pp_job_get_wb0_source_addr(job) == args->wb0_memory) { + MALI_DEBUG_CODE(disable_mask |= 0x1 << (4 * 1)); + mali_pp_job_disable_wb0(job); + } + + if (mali_pp_job_get_wb1_source_addr(job) == args->wb1_memory) { + MALI_DEBUG_CODE(disable_mask |= 0x2 << (4 * 2)); + mali_pp_job_disable_wb1(job); + } + + if (mali_pp_job_get_wb2_source_addr(job) == args->wb2_memory) { + MALI_DEBUG_CODE(disable_mask |= 0x3 << (4 * 3)); + mali_pp_job_disable_wb2(job); + } + MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Disable WB: 0x%X.\n", + disable_mask)); + } + + mali_scheduler_unlock(); +} + +#if MALI_STATE_TRACKING +u32 mali_scheduler_dump_state(char *buf, u32 size) +{ + int n = 0; + + n += _mali_osk_snprintf(buf + n, size - n, "GP queues\n"); + n += _mali_osk_snprintf(buf + n, size - n, + "\tQueue depth: %u\n", job_queue_gp.depth); + n += _mali_osk_snprintf(buf + n, size - n, + "\tNormal priority queue is %s\n", + _mali_osk_list_empty(&job_queue_gp.normal_pri) ? + "empty" : "not empty"); + n += _mali_osk_snprintf(buf + n, size - n, + "\tHigh priority queue is %s\n", + _mali_osk_list_empty(&job_queue_gp.high_pri) ? + "empty" : "not empty"); + + n += _mali_osk_snprintf(buf + n, size - n, + "PP queues\n"); + n += _mali_osk_snprintf(buf + n, size - n, + "\tQueue depth: %u\n", job_queue_pp.depth); + n += _mali_osk_snprintf(buf + n, size - n, + "\tNormal priority queue is %s\n", + _mali_osk_list_empty(&job_queue_pp.normal_pri) + ? "empty" : "not empty"); + n += _mali_osk_snprintf(buf + n, size - n, + "\tHigh priority queue is %s\n", + _mali_osk_list_empty(&job_queue_pp.high_pri) + ? "empty" : "not empty"); + + n += _mali_osk_snprintf(buf + n, size - n, "\n"); + + return n; +} +#endif + +/* + * ---------- Implementation of static functions ---------- + */ + +static mali_timeline_point mali_scheduler_submit_gp_job( + struct mali_session_data *session, struct mali_gp_job *job) +{ + mali_timeline_point point; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(job); + + /* Add job to Timeline system. */ + point = mali_timeline_system_add_tracker(session->timeline_system, + mali_gp_job_get_tracker(job), MALI_TIMELINE_GP); + + return point; +} + +static _mali_osk_errcode_t mali_scheduler_submit_pp_job( + struct mali_session_data *session, struct mali_pp_job *job, mali_timeline_point *point) + +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) + struct ww_acquire_ctx ww_actx; + u32 i; + u32 num_memory_cookies = 0; + struct reservation_object **reservation_object_list = NULL; + unsigned int num_reservation_object = 0; +#endif + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(job); + + mali_scheduler_lock(); + /* + * Adding job to the lookup list used to quickly discard + * writeback units of queued jobs. + */ + mali_pp_job_fb_lookup_add(job); + mali_scheduler_unlock(); + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) + + /* Allocate the reservation_object_list to list the dma reservation object of dependent dma buffer */ + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + if (0 < num_memory_cookies) { + reservation_object_list = kzalloc(sizeof(struct reservation_object *) * num_memory_cookies, GFP_KERNEL); + if (NULL == reservation_object_list) { + MALI_PRINT_ERROR(("Failed to alloc the reservation object list.\n")); + ret = _MALI_OSK_ERR_NOMEM; + goto failed_to_alloc_reservation_object_list; + } + } + + /* Add the dma reservation object into reservation_object_list*/ + for (i = 0; i < num_memory_cookies; i++) { + mali_mem_backend *mem_backend = NULL; + struct reservation_object *tmp_reservation_object = NULL; + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + + mem_backend = mali_mem_backend_struct_search(session, mali_addr); + + MALI_DEBUG_ASSERT_POINTER(mem_backend); + + if (NULL == mem_backend) { + MALI_PRINT_ERROR(("Failed to find the memory backend for memory cookie[%d].\n", i)); + goto failed_to_find_mem_backend; + } + + if (MALI_MEM_DMA_BUF != mem_backend->type) + continue; + + tmp_reservation_object = mem_backend->dma_buf.attachment->buf->resv; + + if (NULL != tmp_reservation_object) { + mali_dma_fence_add_reservation_object_list(tmp_reservation_object, + reservation_object_list, &num_reservation_object); + } + } + + /* + * Add the mali dma fence callback to wait for all dependent dma buf, + * and extend the timeline system to support dma fence, + * then create the new internal dma fence to replace all last dma fence for dependent dma buf. + */ + if (0 < num_reservation_object) { + int error; + int num_dma_fence_waiter = 0; + /* Create one new dma fence.*/ + job->rendered_dma_fence = mali_dma_fence_new(job->session->fence_context, + _mali_osk_atomic_inc_return(&job->session->fence_seqno)); + + if (NULL == job->rendered_dma_fence) { + MALI_PRINT_ERROR(("Failed to creat one new dma fence.\n")); + ret = _MALI_OSK_ERR_FAULT; + goto failed_to_create_dma_fence; + } + + /* In order to avoid deadlock, wait/wound mutex lock to lock all dma buffers*/ + + error = mali_dma_fence_lock_reservation_object_list(reservation_object_list, + num_reservation_object, &ww_actx); + + if (0 != error) { + MALI_PRINT_ERROR(("Failed to lock all reservation objects.\n")); + ret = _MALI_OSK_ERR_FAULT; + goto failed_to_lock_reservation_object_list; + } + + mali_dma_fence_context_init(&job->dma_fence_context, + mali_timeline_dma_fence_callback, (void *)job); + + /* Add dma fence waiters and dma fence callback. */ + for (i = 0; i < num_reservation_object; i++) { + ret = mali_dma_fence_context_add_waiters(&job->dma_fence_context, reservation_object_list[i]); + if (_MALI_OSK_ERR_OK != ret) { + MALI_PRINT_ERROR(("Failed to add waiter into mali dma fence context.\n")); + goto failed_to_add_dma_fence_waiter; + } + } + + for (i = 0; i < num_reservation_object; i++) { + reservation_object_add_excl_fence(reservation_object_list[i], job->rendered_dma_fence); + } + + num_dma_fence_waiter = job->dma_fence_context.num_dma_fence_waiter; + + /* Add job to Timeline system. */ + (*point) = mali_timeline_system_add_tracker(session->timeline_system, + mali_pp_job_get_tracker(job), MALI_TIMELINE_PP); + + if (0 != num_dma_fence_waiter) { + mali_dma_fence_context_dec_count(&job->dma_fence_context); + } + + /* Unlock all wait/wound mutex lock. */ + mali_dma_fence_unlock_reservation_object_list(reservation_object_list, + num_reservation_object, &ww_actx); + } else { + /* Add job to Timeline system. */ + (*point) = mali_timeline_system_add_tracker(session->timeline_system, + mali_pp_job_get_tracker(job), MALI_TIMELINE_PP); + } + + kfree(reservation_object_list); + return ret; +#else + /* Add job to Timeline system. */ + (*point) = mali_timeline_system_add_tracker(session->timeline_system, + mali_pp_job_get_tracker(job), MALI_TIMELINE_PP); +#endif + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) +failed_to_add_dma_fence_waiter: + mali_dma_fence_context_term(&job->dma_fence_context); + mali_dma_fence_unlock_reservation_object_list(reservation_object_list, + num_reservation_object, &ww_actx); +failed_to_lock_reservation_object_list: + mali_dma_fence_signal_and_put(&job->rendered_dma_fence); +failed_to_create_dma_fence: +failed_to_find_mem_backend: + if (NULL != reservation_object_list) + kfree(reservation_object_list); +failed_to_alloc_reservation_object_list: + mali_pp_job_fb_lookup_remove(job); +#endif + return ret; +} + +static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job) +{ + struct mali_session_data *session; + _mali_osk_list_t *queue; + + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT_POINTER(job); + + session = mali_gp_job_get_session(job); + MALI_DEBUG_ASSERT_POINTER(session); + + if (unlikely(session->is_aborting)) { + MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Job %u (0x%08X) queued while session is aborting.\n", + mali_gp_job_get_id(job), job)); + return MALI_FALSE; /* job not queued */ + } + + mali_gp_job_set_cache_order(job, mali_scheduler_get_new_cache_order()); + + /* Determine which queue the job should be added to. */ + if (session->use_high_priority_job_queue) { + queue = &job_queue_gp.high_pri; + } else { + queue = &job_queue_gp.normal_pri; + } + + job_queue_gp.depth += 1; + job_queue_gp.big_job_num += (job->big_job) ? 1 : 0; + + /* Add job to queue (mali_gp_job_queue_add find correct place). */ + mali_gp_job_list_add(job, queue); + + /* + * We hold a PM reference for every job we hold queued (and running) + * It is important that we take this reference after job has been + * added the the queue so that any runtime resume could schedule this + * job right there and then. + */ + _mali_osk_pm_dev_ref_get_async(); + + if (mali_utilization_enabled()) { + /* + * We cheat a little bit by counting the GP as busy from the + * time a GP job is queued. This will be fine because we only + * loose the tiny idle gap between jobs, but we will instead + * get less utilization work to do (less locks taken) + */ + mali_utilization_gp_start(); + } + + mali_pm_record_gpu_active(MALI_TRUE); + + /* Add profiling events for job enqueued */ + _mali_osk_profiling_add_event( + MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE, + mali_gp_job_get_pid(job), + mali_gp_job_get_tid(job), + mali_gp_job_get_frame_builder_id(job), + mali_gp_job_get_flush_id(job), + 0); + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + trace_gpu_job_enqueue(mali_gp_job_get_tid(job), + mali_gp_job_get_id(job), "GP"); +#endif + + MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) queued\n", + mali_gp_job_get_id(job), job)); + + return MALI_TRUE; /* job queued */ +} + +static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job) +{ + struct mali_session_data *session; + _mali_osk_list_t *queue = NULL; + + MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD(); + MALI_DEBUG_ASSERT_POINTER(job); + + session = mali_pp_job_get_session(job); + MALI_DEBUG_ASSERT_POINTER(session); + + if (unlikely(session->is_aborting)) { + MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while session is aborting.\n", + mali_pp_job_get_id(job), job)); + return MALI_FALSE; /* job not queued */ + } else if (unlikely(MALI_SWAP_IN_FAIL == job->swap_status)) { + MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while swap in failed.\n", + mali_pp_job_get_id(job), job)); + return MALI_FALSE; + } + + mali_pp_job_set_cache_order(job, mali_scheduler_get_new_cache_order()); + + if (session->use_high_priority_job_queue) { + queue = &job_queue_pp.high_pri; + } else { + queue = &job_queue_pp.normal_pri; + } + + job_queue_pp.depth += + mali_pp_job_get_sub_job_count(job); + + /* Add job to queue (mali_gp_job_queue_add find correct place). */ + mali_pp_job_list_add(job, queue); + + /* + * We hold a PM reference for every job we hold queued (and running) + * It is important that we take this reference after job has been + * added the the queue so that any runtime resume could schedule this + * job right there and then. + */ + _mali_osk_pm_dev_ref_get_async(); + + if (mali_utilization_enabled()) { + /* + * We cheat a little bit by counting the PP as busy from the + * time a PP job is queued. This will be fine because we only + * loose the tiny idle gap between jobs, but we will instead + * get less utilization work to do (less locks taken) + */ + mali_utilization_pp_start(); + } + + mali_pm_record_gpu_active(MALI_FALSE); + + /* Add profiling events for job enqueued */ + _mali_osk_profiling_add_event( + MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | + MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE, + mali_pp_job_get_pid(job), + mali_pp_job_get_tid(job), + mali_pp_job_get_frame_builder_id(job), + mali_pp_job_get_flush_id(job), + 0); + +#if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS) + trace_gpu_job_enqueue(mali_pp_job_get_tid(job), + mali_pp_job_get_id(job), "PP"); +#endif + + MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) with %u parts queued.\n", + mali_pp_job_is_virtual(job) + ? "Virtual" : "Physical", + mali_pp_job_get_id(job), job, + mali_pp_job_get_sub_job_count(job))); + + return MALI_TRUE; /* job queued */ +} + +static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job, + mali_bool success) +{ + _mali_uk_gp_job_finished_s *jobres; + struct mali_session_data *session; + _mali_osk_notification_t *notification; + + MALI_DEBUG_ASSERT_POINTER(job); + + session = mali_gp_job_get_session(job); + MALI_DEBUG_ASSERT_POINTER(session); + + notification = mali_gp_job_get_finished_notification(job); + MALI_DEBUG_ASSERT_POINTER(notification); + + jobres = notification->result_buffer; + MALI_DEBUG_ASSERT_POINTER(jobres); + + jobres->pending_big_job_num = mali_scheduler_job_gp_big_job_count(); + + jobres->user_job_ptr = mali_gp_job_get_user_id(job); + if (MALI_TRUE == success) { + jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS; + } else { + jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR; + } + jobres->heap_current_addr = mali_gp_job_get_current_heap_addr(job); + jobres->perf_counter0 = mali_gp_job_get_perf_counter_value0(job); + jobres->perf_counter1 = mali_gp_job_get_perf_counter_value1(job); + + mali_session_send_notification(session, notification); +} + +void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, + u32 num_cores_in_virtual) +{ + u32 i; + u32 num_counters_to_copy; + _mali_uk_pp_job_finished_s *jobres; + struct mali_session_data *session; + _mali_osk_notification_t *notification; + + if (MALI_TRUE == mali_pp_job_use_no_notification(job)) { + return; + } + + MALI_DEBUG_ASSERT_POINTER(job); + + session = mali_pp_job_get_session(job); + MALI_DEBUG_ASSERT_POINTER(session); + + notification = mali_pp_job_get_finished_notification(job); + MALI_DEBUG_ASSERT_POINTER(notification); + + jobres = notification->result_buffer; + MALI_DEBUG_ASSERT_POINTER(jobres); + + jobres->user_job_ptr = mali_pp_job_get_user_id(job); + if (MALI_TRUE == mali_pp_job_was_success(job)) { + jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS; + } else { + jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR; + } + + if (mali_pp_job_is_virtual(job)) { + num_counters_to_copy = num_cores_in_virtual; + } else { + num_counters_to_copy = mali_pp_job_get_sub_job_count(job); + } + + for (i = 0; i < num_counters_to_copy; i++) { + jobres->perf_counter0[i] = + mali_pp_job_get_perf_counter_value0(job, i); + jobres->perf_counter1[i] = + mali_pp_job_get_perf_counter_value1(job, i); + jobres->perf_counter_src0 = + mali_pp_job_get_pp_counter_global_src0(); + jobres->perf_counter_src1 = + mali_pp_job_get_pp_counter_global_src1(); + } + + mali_session_send_notification(session, notification); +} + +static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock); + mali_pp_job_list_addtail(job, &scheduler_pp_job_deletion_queue); + _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock); + + _mali_osk_wq_schedule_work(scheduler_wq_pp_job_delete); +} + +void mali_scheduler_do_pp_job_delete(void *arg) +{ + _MALI_OSK_LIST_HEAD_STATIC_INIT(list); + struct mali_pp_job *job; + struct mali_pp_job *tmp; + + MALI_IGNORE(arg); + + /* + * Quickly "unhook" the jobs pending to be deleted, so we can release + * the lock before we start deleting the job objects + * (without any locks held) + */ + _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock); + _mali_osk_list_move_list(&scheduler_pp_job_deletion_queue, &list); + _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock); + + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, + struct mali_pp_job, list) { + _mali_osk_list_delinit(&job->list); + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) + mali_dma_fence_context_term(&job->dma_fence_context); +#endif + + mali_pp_job_delete(job); /* delete the job object itself */ + } +} + +#if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) + +static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + + _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock); + mali_pp_job_list_addtail(job, &scheduler_pp_job_queue_list); + _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock); + + _mali_osk_wq_schedule_work(scheduler_wq_pp_job_queue); +} + +static void mali_scheduler_do_pp_job_queue(void *arg) +{ + _MALI_OSK_LIST_HEAD_STATIC_INIT(list); + struct mali_pp_job *job; + struct mali_pp_job *tmp; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_IGNORE(arg); + + /* + * Quickly "unhook" the jobs pending to be queued, so we can release + * the lock before we start queueing the job objects + * (without any locks held) + */ + _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock); + _mali_osk_list_move_list(&scheduler_pp_job_queue_list, &list); + _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock); + + /* First loop through all jobs and do the pre-work (no locks needed) */ + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, + struct mali_pp_job, list) { + if (mali_pp_job_needs_dma_buf_mapping(job)) { + /* + * This operation could fail, but we continue anyway, + * because the worst that could happen is that this + * job will fail due to a Mali page fault. + */ + mali_dma_buf_map_job(job); + } + } + + mali_scheduler_lock(); + + /* Then loop through all jobs again to queue them (lock needed) */ + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, + struct mali_pp_job, list) { + + /* Remove from scheduler_pp_job_queue_list before queueing */ + mali_pp_job_list_remove(job); + + if (mali_scheduler_queue_pp_job(job)) { + /* Job queued successfully */ + schedule_mask |= MALI_SCHEDULER_MASK_PP; + } else { + /* Failed to enqueue job, release job (with error) */ + mali_pp_job_fb_lookup_remove(job); + mali_pp_job_mark_unstarted_failed(job); + + /* unlock scheduler in this uncommon case */ + mali_scheduler_unlock(); + + schedule_mask |= mali_timeline_tracker_release( + mali_pp_job_get_tracker(job)); + + /* Notify user space and close the job object */ + mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, + MALI_FALSE); + + mali_scheduler_lock(); + } + } + + mali_scheduler_unlock(); + + /* Trigger scheduling of jobs */ + mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); +} + +#endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */ + +void mali_scheduler_gp_pp_job_queue_print(void) +{ + struct mali_gp_job *gp_job = NULL; + struct mali_gp_job *tmp_gp_job = NULL; + struct mali_pp_job *pp_job = NULL; + struct mali_pp_job *tmp_pp_job = NULL; + + MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj); + + /* dump job queup status */ + if ((0 == job_queue_gp.depth) && (0 == job_queue_pp.depth)) { + MALI_PRINT(("No GP&PP job in the job queue.\n")); + return; + } + + MALI_PRINT(("Total (%d) GP job in the job queue.\n", job_queue_gp.depth)); + if (job_queue_gp.depth > 0) { + if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.high_pri, + struct mali_gp_job, list) { + MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job high_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid)); + } + } + + if (!_mali_osk_list_empty(&job_queue_gp.normal_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.normal_pri, + struct mali_gp_job, list) { + MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job normal_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid)); + } + } + } + + MALI_PRINT(("Total (%d) PP job in the job queue.\n", job_queue_pp.depth)); + if (job_queue_pp.depth > 0) { + if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.high_pri, + struct mali_pp_job, list) { + if (mali_pp_job_is_virtual(pp_job)) { + MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } else { + MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } + } + } + + if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) { + _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.normal_pri, + struct mali_pp_job, list) { + if (mali_pp_job_is_virtual(pp_job)) { + MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } else { + MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid)); + } + } + } + } + + /* dump group running job status */ + mali_executor_running_status_print(); +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_scheduler.h b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.h new file mode 100755 index 000000000000..de81a421ea9a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_scheduler.h @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_SCHEDULER_H__ +#define __MALI_SCHEDULER_H__ + +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_scheduler_types.h" +#include "mali_session.h" + +struct mali_scheduler_job_queue { + _MALI_OSK_LIST_HEAD(normal_pri); /* Queued jobs with normal priority */ + _MALI_OSK_LIST_HEAD(high_pri); /* Queued jobs with high priority */ + u32 depth; /* Depth of combined queues. */ + u32 big_job_num; +}; + +extern _mali_osk_spinlock_irq_t *mali_scheduler_lock_obj; + +/* Queue of jobs to be executed on the GP group */ +extern struct mali_scheduler_job_queue job_queue_gp; + +/* Queue of PP jobs */ +extern struct mali_scheduler_job_queue job_queue_pp; + +extern _mali_osk_atomic_t mali_job_id_autonumber; +extern _mali_osk_atomic_t mali_job_cache_order_autonumber; + +#define MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD() MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj); + +_mali_osk_errcode_t mali_scheduler_initialize(void); +void mali_scheduler_terminate(void); + +MALI_STATIC_INLINE void mali_scheduler_lock(void) +{ + _mali_osk_spinlock_irq_lock(mali_scheduler_lock_obj); + MALI_DEBUG_PRINT(5, ("Mali scheduler: scheduler lock taken.\n")); +} + +MALI_STATIC_INLINE void mali_scheduler_unlock(void) +{ + MALI_DEBUG_PRINT(5, ("Mali scheduler: Releasing scheduler lock.\n")); + _mali_osk_spinlock_irq_unlock(mali_scheduler_lock_obj); +} + +MALI_STATIC_INLINE u32 mali_scheduler_job_gp_count(void) +{ + return job_queue_gp.depth; +} +MALI_STATIC_INLINE u32 mali_scheduler_job_gp_big_job_count(void) +{ + return job_queue_gp.big_job_num; +} + +u32 mali_scheduler_job_physical_head_count(mali_bool gpu_mode_is_secure); + +mali_bool mali_scheduler_job_next_is_virtual(void); +struct mali_pp_job *mali_scheduler_job_pp_next(void); + +struct mali_gp_job *mali_scheduler_job_gp_get(void); +struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void); +struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void); +struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job); +struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void); + +MALI_STATIC_INLINE u32 mali_scheduler_get_new_id(void) +{ + return _mali_osk_atomic_inc_return(&mali_job_id_autonumber); +} + +MALI_STATIC_INLINE u32 mali_scheduler_get_new_cache_order(void) +{ + return _mali_osk_atomic_inc_return(&mali_job_cache_order_autonumber); +} + +/** + * @brief Used by the Timeline system to queue a GP job. + * + * @note @ref mali_executor_schedule_from_mask() should be called if this + * function returns non-zero. + * + * @param job The GP job that is being activated. + * + * @return A scheduling bitmask that can be used to decide if scheduling is + * necessary after this call. + */ +mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job); + +/** + * @brief Used by the Timeline system to queue a PP job. + * + * @note @ref mali_executor_schedule_from_mask() should be called if this + * function returns non-zero. + * + * @param job The PP job that is being activated. + * + * @return A scheduling bitmask that can be used to decide if scheduling is + * necessary after this call. + */ +mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job); + +void mali_scheduler_complete_gp_job(struct mali_gp_job *job, + mali_bool success, + mali_bool user_notification, + mali_bool dequeued); + +void mali_scheduler_complete_pp_job(struct mali_pp_job *job, + u32 num_cores_in_virtual, + mali_bool user_notification, + mali_bool dequeued); + +void mali_scheduler_abort_session(struct mali_session_data *session); + +void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job, + u32 num_cores_in_virtual); + +#if MALI_STATE_TRACKING +u32 mali_scheduler_dump_state(char *buf, u32 size); +#endif + +void mali_scheduler_gp_pp_job_queue_print(void); + +#endif /* __MALI_SCHEDULER_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h b/drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h new file mode 100755 index 000000000000..ba1d71d01d46 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_scheduler_types.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_SCHEDULER_TYPES_H__ +#define __MALI_SCHEDULER_TYPES_H__ + +#include "mali_osk.h" + +#define MALI_SCHEDULER_JOB_ID_SPAN 65535 + +/** + * Bitmask used for defered scheduling of subsystems. + */ +typedef u32 mali_scheduler_mask; + +#define MALI_SCHEDULER_MASK_GP (1<<0) +#define MALI_SCHEDULER_MASK_PP (1<<1) + +#define MALI_SCHEDULER_MASK_EMPTY 0 +#define MALI_SCHEDULER_MASK_ALL (MALI_SCHEDULER_MASK_GP | MALI_SCHEDULER_MASK_PP) + +#endif /* __MALI_SCHEDULER_TYPES_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_session.c b/drivers/gpu/arm/mali400/mali/common/mali_session.c new file mode 100755 index 000000000000..7504fb108779 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_session.c @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_session.h" +#include "mali_ukk.h" +#ifdef MALI_MEM_SWAP_TRACKING +#include "mali_memory_swap_alloc.h" +#endif + +_MALI_OSK_LIST_HEAD(mali_sessions); +static u32 mali_session_count = 0; + +_mali_osk_spinlock_irq_t *mali_sessions_lock = NULL; +wait_queue_head_t pending_queue; + +_mali_osk_errcode_t mali_session_initialize(void) +{ + _MALI_OSK_INIT_LIST_HEAD(&mali_sessions); + /* init wait queue for big varying job */ + init_waitqueue_head(&pending_queue); + + mali_sessions_lock = _mali_osk_spinlock_irq_init( + _MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_SESSIONS); + if (NULL == mali_sessions_lock) { + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_session_terminate(void) +{ + if (NULL != mali_sessions_lock) { + _mali_osk_spinlock_irq_term(mali_sessions_lock); + mali_sessions_lock = NULL; + } +} + +void mali_session_add(struct mali_session_data *session) +{ + mali_session_lock(); + _mali_osk_list_add(&session->link, &mali_sessions); + mali_session_count++; + mali_session_unlock(); +} + +void mali_session_remove(struct mali_session_data *session) +{ + mali_session_lock(); + _mali_osk_list_delinit(&session->link); + mali_session_count--; + mali_session_unlock(); +} + +u32 mali_session_get_count(void) +{ + return mali_session_count; +} + +mali_bool mali_session_pp_job_is_empty(void *data) +{ + struct mali_session_data *session = (struct mali_session_data *)data; + MALI_DEBUG_ASSERT_POINTER(session); + + if ( 0 == _mali_osk_atomic_read(&session->number_of_pp_jobs)) { + return MALI_TRUE; + } + return MALI_FALSE; +} + +wait_queue_head_t *mali_session_get_wait_queue(void) +{ + return &pending_queue; +} + +/* + * Get the max completed window jobs from all active session, + * which will be used in window render frame per sec calculate + */ +#if defined(CONFIG_MALI_DVFS) +u32 mali_session_max_window_num(void) +{ + struct mali_session_data *session, *tmp; + u32 max_window_num = 0; + u32 tmp_number = 0; + + mali_session_lock(); + + MALI_SESSION_FOREACH(session, tmp, link) { + tmp_number = _mali_osk_atomic_xchg( + &session->number_of_window_jobs, 0); + if (max_window_num < tmp_number) { + max_window_num = tmp_number; + } + } + + mali_session_unlock(); + + return max_window_num; +} +#endif + +void mali_session_memory_tracking(_mali_osk_print_ctx *print_ctx) +{ + struct mali_session_data *session, *tmp; + u32 mali_mem_usage; + u32 total_mali_mem_size; +#ifdef MALI_MEM_SWAP_TRACKING + u32 swap_pool_size; + u32 swap_unlock_size; +#endif + + MALI_DEBUG_ASSERT_POINTER(print_ctx); + mali_session_lock(); + MALI_SESSION_FOREACH(session, tmp, link) { +#ifdef MALI_MEM_SWAP_TRACKING + _mali_osk_ctxprintf(print_ctx, " %-25s %-10u %-10u %-15u %-15u %-10u %-10u %-10u\n", + session->comm, session->pid, + (atomic_read(&session->mali_mem_allocated_pages)) * _MALI_OSK_MALI_PAGE_SIZE, + (unsigned int)session->max_mali_mem_allocated_size, + (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_EXTERNAL])) * _MALI_OSK_MALI_PAGE_SIZE), + (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_UMP])) * _MALI_OSK_MALI_PAGE_SIZE), + (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_DMA_BUF])) * _MALI_OSK_MALI_PAGE_SIZE), + (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_SWAP])) * _MALI_OSK_MALI_PAGE_SIZE) + ); +#else + _mali_osk_ctxprintf(print_ctx, " %-25s %-10u %-10u %-15u %-15u %-10u %-10u \n", + session->comm, session->pid, + (unsigned int)((atomic_read(&session->mali_mem_allocated_pages)) * _MALI_OSK_MALI_PAGE_SIZE), + (unsigned int)session->max_mali_mem_allocated_size, + (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_EXTERNAL])) * _MALI_OSK_MALI_PAGE_SIZE), + (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_UMP])) * _MALI_OSK_MALI_PAGE_SIZE), + (unsigned int)((atomic_read(&session->mali_mem_array[MALI_MEM_DMA_BUF])) * _MALI_OSK_MALI_PAGE_SIZE) + ); +#endif + } + mali_session_unlock(); + mali_mem_usage = _mali_ukk_report_memory_usage(); + total_mali_mem_size = _mali_ukk_report_total_memory_size(); + _mali_osk_ctxprintf(print_ctx, "Mali mem usage: %u\nMali mem limit: %u\n", mali_mem_usage, total_mali_mem_size); +#ifdef MALI_MEM_SWAP_TRACKING + mali_mem_swap_tracking(&swap_pool_size, &swap_unlock_size); + _mali_osk_ctxprintf(print_ctx, "Mali swap mem pool : %u\nMali swap mem unlock: %u\n", swap_pool_size, swap_unlock_size); +#endif +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_session.h b/drivers/gpu/arm/mali400/mali/common/mali_session.h new file mode 100755 index 000000000000..da8b9927ee60 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_session.h @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_SESSION_H__ +#define __MALI_SESSION_H__ + +#include "mali_mmu_page_directory.h" +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "mali_memory_types.h" +#include "mali_memory_manager.h" + +struct mali_timeline_system; +struct mali_soft_system; + +/* Number of frame builder job lists per session. */ +#define MALI_PP_JOB_FB_LOOKUP_LIST_SIZE 16 +#define MALI_PP_JOB_FB_LOOKUP_LIST_MASK (MALI_PP_JOB_FB_LOOKUP_LIST_SIZE - 1) +/*Max pending big job allowed in kernel*/ +#define MALI_MAX_PENDING_BIG_JOB (2) + +struct mali_session_data { + _mali_osk_notification_queue_t *ioctl_queue; + + _mali_osk_wait_queue_t *wait_queue; /**The wait queue to wait for the number of pp job become 0.*/ + + _mali_osk_mutex_t *memory_lock; /**< Lock protecting the vm manipulation */ + _mali_osk_mutex_t *cow_lock; /** < Lock protecting the cow memory free manipulation */ +#if 0 + _mali_osk_list_t memory_head; /**< Track all the memory allocated in this session, for freeing on abnormal termination */ +#endif + struct mali_page_directory *page_directory; /**< MMU page directory for this session */ + + _MALI_OSK_LIST_HEAD(link); /**< Link for list of all sessions */ + _MALI_OSK_LIST_HEAD(pp_job_list); /**< List of all PP jobs on this session */ + +#if defined(CONFIG_MALI_DVFS) + _mali_osk_atomic_t number_of_window_jobs; /**< Record the window jobs completed on this session in a period */ +#endif + _mali_osk_atomic_t number_of_pp_jobs; /** < Record the pp jobs on this session */ + + _mali_osk_list_t pp_job_fb_lookup_list[MALI_PP_JOB_FB_LOOKUP_LIST_SIZE]; /**< List of PP job lists per frame builder id. Used to link jobs from same frame builder. */ + struct mali_soft_job_system *soft_job_system; /**< Soft job system for this session. */ + struct mali_timeline_system *timeline_system; /**< Timeline system for this session. */ + + mali_bool is_aborting; /**< MALI_TRUE if the session is aborting, MALI_FALSE if not. */ + mali_bool use_high_priority_job_queue; /**< If MALI_TRUE, jobs added from this session will use the high priority job queues. */ + u32 pid; + char *comm; + atomic_t mali_mem_array[MALI_MEM_TYPE_MAX]; /**< The array to record mem types' usage for this session. */ + atomic_t mali_mem_allocated_pages; /** The current allocated mali memory pages, which include mali os memory and mali dedicated memory.*/ + size_t max_mali_mem_allocated_size; /**< The past max mali memory allocated size, which include mali os memory and mali dedicated memory. */ + /* Added for new memroy system */ + struct mali_allocation_manager allocation_mgr; + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) + u32 fence_context; /** < The execution dma fence context this fence is run on. */ + _mali_osk_atomic_t fence_seqno; /** < Alinear increasing sequence number for this dma fence context. */ +#endif +}; + +_mali_osk_errcode_t mali_session_initialize(void); +void mali_session_terminate(void); + +/* List of all sessions. Actual list head in mali_kernel_core.c */ +extern _mali_osk_list_t mali_sessions; +/* Lock to protect modification and access to the mali_sessions list */ +extern _mali_osk_spinlock_irq_t *mali_sessions_lock; + +MALI_STATIC_INLINE void mali_session_lock(void) +{ + _mali_osk_spinlock_irq_lock(mali_sessions_lock); +} + +MALI_STATIC_INLINE void mali_session_unlock(void) +{ + _mali_osk_spinlock_irq_unlock(mali_sessions_lock); +} + +void mali_session_add(struct mali_session_data *session); +void mali_session_remove(struct mali_session_data *session); +u32 mali_session_get_count(void); +mali_bool mali_session_pp_job_is_empty(void *data); +wait_queue_head_t *mali_session_get_wait_queue(void); + +#define MALI_SESSION_FOREACH(session, tmp, link) \ + _MALI_OSK_LIST_FOREACHENTRY(session, tmp, &mali_sessions, struct mali_session_data, link) + +MALI_STATIC_INLINE struct mali_page_directory *mali_session_get_page_directory(struct mali_session_data *session) +{ + return session->page_directory; +} + +MALI_STATIC_INLINE void mali_session_memory_lock(struct mali_session_data *session) +{ + MALI_DEBUG_ASSERT_POINTER(session); + _mali_osk_mutex_wait(session->memory_lock); +} + +MALI_STATIC_INLINE void mali_session_memory_unlock(struct mali_session_data *session) +{ + MALI_DEBUG_ASSERT_POINTER(session); + _mali_osk_mutex_signal(session->memory_lock); +} + +MALI_STATIC_INLINE void mali_session_send_notification(struct mali_session_data *session, _mali_osk_notification_t *object) +{ + _mali_osk_notification_queue_send(session->ioctl_queue, object); +} + +#if defined(CONFIG_MALI_DVFS) + +MALI_STATIC_INLINE void mali_session_inc_num_window_jobs(struct mali_session_data *session) +{ + MALI_DEBUG_ASSERT_POINTER(session); + _mali_osk_atomic_inc(&session->number_of_window_jobs); +} + +/* + * Get the max completed window jobs from all active session, + * which will be used in window render frame per sec calculate + */ +u32 mali_session_max_window_num(void); + +#endif + +void mali_session_memory_tracking(_mali_osk_print_ctx *print_ctx); + +#endif /* __MALI_SESSION_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_soft_job.c b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.c new file mode 100755 index 000000000000..35cd830bc83a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.c @@ -0,0 +1,438 @@ +/* + * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_soft_job.h" +#include "mali_osk.h" +#include "mali_timeline.h" +#include "mali_session.h" +#include "mali_kernel_common.h" +#include "mali_uk_types.h" +#include "mali_scheduler.h" +#include "mali_executor.h" + +MALI_STATIC_INLINE void mali_soft_job_system_lock(struct mali_soft_job_system *system) +{ + MALI_DEBUG_ASSERT_POINTER(system); + _mali_osk_spinlock_irq_lock(system->lock); + MALI_DEBUG_PRINT(5, ("Mali Soft Job: soft system %p lock taken\n", system)); + MALI_DEBUG_ASSERT(0 == system->lock_owner); + MALI_DEBUG_CODE(system->lock_owner = _mali_osk_get_tid()); +} + +MALI_STATIC_INLINE void mali_soft_job_system_unlock(struct mali_soft_job_system *system) +{ + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_PRINT(5, ("Mali Soft Job: releasing soft system %p lock\n", system)); + MALI_DEBUG_ASSERT(_mali_osk_get_tid() == system->lock_owner); + MALI_DEBUG_CODE(system->lock_owner = 0); + _mali_osk_spinlock_irq_unlock(system->lock); +} + +#if defined(DEBUG) +MALI_STATIC_INLINE void mali_soft_job_system_assert_locked(struct mali_soft_job_system *system) +{ + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT(_mali_osk_get_tid() == system->lock_owner); +} +#define MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system) mali_soft_job_system_assert_locked(system) +#else +#define MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system) +#endif /* defined(DEBUG) */ + +struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session) +{ + struct mali_soft_job_system *system; + + MALI_DEBUG_ASSERT_POINTER(session); + + system = (struct mali_soft_job_system *) _mali_osk_calloc(1, sizeof(struct mali_soft_job_system)); + if (NULL == system) { + return NULL; + } + + system->session = session; + + system->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_SCHEDULER); + if (NULL == system->lock) { + mali_soft_job_system_destroy(system); + return NULL; + } + system->lock_owner = 0; + system->last_job_id = 0; + + _MALI_OSK_INIT_LIST_HEAD(&(system->jobs_used)); + + return system; +} + +void mali_soft_job_system_destroy(struct mali_soft_job_system *system) +{ + MALI_DEBUG_ASSERT_POINTER(system); + + /* All jobs should be free at this point. */ + MALI_DEBUG_ASSERT(_mali_osk_list_empty(&(system->jobs_used))); + + if (NULL != system) { + if (NULL != system->lock) { + _mali_osk_spinlock_irq_term(system->lock); + } + _mali_osk_free(system); + } +} + +static void mali_soft_job_system_free_job(struct mali_soft_job_system *system, struct mali_soft_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(system); + + mali_soft_job_system_lock(job->system); + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id); + MALI_DEBUG_ASSERT(system == job->system); + + _mali_osk_list_del(&(job->system_list)); + + mali_soft_job_system_unlock(job->system); + + _mali_osk_free(job); +} + +MALI_STATIC_INLINE struct mali_soft_job *mali_soft_job_system_lookup_job(struct mali_soft_job_system *system, u32 job_id) +{ + struct mali_soft_job *job, *tmp; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system); + + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &system->jobs_used, struct mali_soft_job, system_list) { + if (job->id == job_id) + return job; + } + + return NULL; +} + +void mali_soft_job_destroy(struct mali_soft_job *job) +{ + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(job->system); + + MALI_DEBUG_PRINT(4, ("Mali Soft Job: destroying soft job %u (0x%08X)\n", job->id, job)); + + if (NULL != job) { + if (0 < _mali_osk_atomic_dec_return(&job->refcount)) return; + + _mali_osk_atomic_term(&job->refcount); + + if (NULL != job->activated_notification) { + _mali_osk_notification_delete(job->activated_notification); + job->activated_notification = NULL; + } + + mali_soft_job_system_free_job(job->system, job); + } +} + +struct mali_soft_job *mali_soft_job_create(struct mali_soft_job_system *system, mali_soft_job_type type, u64 user_job) +{ + struct mali_soft_job *job; + _mali_osk_notification_t *notification = NULL; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT((MALI_SOFT_JOB_TYPE_USER_SIGNALED == type) || + (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == type)); + + notification = _mali_osk_notification_create(_MALI_NOTIFICATION_SOFT_ACTIVATED, sizeof(_mali_uk_soft_job_activated_s)); + if (unlikely(NULL == notification)) { + MALI_PRINT_ERROR(("Mali Soft Job: failed to allocate notification")); + return NULL; + } + + job = _mali_osk_malloc(sizeof(struct mali_soft_job)); + if (unlikely(NULL == job)) { + MALI_DEBUG_PRINT(2, ("Mali Soft Job: system alloc job failed. \n")); + return NULL; + } + + mali_soft_job_system_lock(system); + + job->system = system; + job->id = system->last_job_id++; + job->state = MALI_SOFT_JOB_STATE_ALLOCATED; + + _mali_osk_list_add(&(job->system_list), &(system->jobs_used)); + + job->type = type; + job->user_job = user_job; + job->activated = MALI_FALSE; + + job->activated_notification = notification; + + _mali_osk_atomic_init(&job->refcount, 1); + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state); + MALI_DEBUG_ASSERT(system == job->system); + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id); + + mali_soft_job_system_unlock(system); + + return job; +} + +mali_timeline_point mali_soft_job_start(struct mali_soft_job *job, struct mali_timeline_fence *fence) +{ + mali_timeline_point point; + struct mali_soft_job_system *system; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(fence); + + MALI_DEBUG_ASSERT_POINTER(job->system); + system = job->system; + + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT_POINTER(system->session->timeline_system); + + mali_soft_job_system_lock(system); + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state); + job->state = MALI_SOFT_JOB_STATE_STARTED; + + mali_soft_job_system_unlock(system); + + MALI_DEBUG_PRINT(4, ("Mali Soft Job: starting soft job %u (0x%08X)\n", job->id, job)); + + mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_SOFT, fence, job); + point = mali_timeline_system_add_tracker(system->session->timeline_system, &job->tracker, MALI_TIMELINE_SOFT); + + return point; +} + +static mali_bool mali_soft_job_is_activated(void *data) +{ + struct mali_soft_job *job; + + job = (struct mali_soft_job *) data; + MALI_DEBUG_ASSERT_POINTER(job); + + return job->activated; +} + +_mali_osk_errcode_t mali_soft_job_system_signal_job(struct mali_soft_job_system *system, u32 job_id) +{ + struct mali_soft_job *job; + struct mali_timeline_system *timeline_system; + mali_scheduler_mask schedule_mask; + + MALI_DEBUG_ASSERT_POINTER(system); + + mali_soft_job_system_lock(system); + + job = mali_soft_job_system_lookup_job(system, job_id); + + if ((NULL == job) || (MALI_SOFT_JOB_TYPE_USER_SIGNALED != job->type) + || !(MALI_SOFT_JOB_STATE_STARTED == job->state || MALI_SOFT_JOB_STATE_TIMED_OUT == job->state)) { + mali_soft_job_system_unlock(system); + MALI_PRINT_ERROR(("Mali Soft Job: invalid soft job id %u", job_id)); + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + if (MALI_SOFT_JOB_STATE_TIMED_OUT == job->state) { + job->state = MALI_SOFT_JOB_STATE_SIGNALED; + mali_soft_job_system_unlock(system); + + MALI_DEBUG_ASSERT(MALI_TRUE == job->activated); + MALI_DEBUG_PRINT(4, ("Mali Soft Job: soft job %u (0x%08X) was timed out\n", job->id, job)); + mali_soft_job_destroy(job); + + return _MALI_OSK_ERR_TIMEOUT; + } + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state); + + job->state = MALI_SOFT_JOB_STATE_SIGNALED; + mali_soft_job_system_unlock(system); + + /* Since the job now is in signaled state, timeouts from the timeline system will be + * ignored, and it is not possible to signal this job again. */ + + timeline_system = system->session->timeline_system; + MALI_DEBUG_ASSERT_POINTER(timeline_system); + + /* Wait until activated. */ + _mali_osk_wait_queue_wait_event(timeline_system->wait_queue, mali_soft_job_is_activated, (void *) job); + + MALI_DEBUG_PRINT(4, ("Mali Soft Job: signaling soft job %u (0x%08X)\n", job->id, job)); + + schedule_mask = mali_timeline_tracker_release(&job->tracker); + mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); + + mali_soft_job_destroy(job); + + return _MALI_OSK_ERR_OK; +} + +static void mali_soft_job_send_activated_notification(struct mali_soft_job *job) +{ + if (NULL != job->activated_notification) { + _mali_uk_soft_job_activated_s *res = job->activated_notification->result_buffer; + res->user_job = job->user_job; + mali_session_send_notification(job->system->session, job->activated_notification); + } + job->activated_notification = NULL; +} + +mali_scheduler_mask mali_soft_job_system_activate_job(struct mali_soft_job *job) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(job->system); + MALI_DEBUG_ASSERT_POINTER(job->system->session); + + MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeline activation for soft job %u (0x%08X).\n", job->id, job)); + + mali_soft_job_system_lock(job->system); + + if (unlikely(job->system->session->is_aborting)) { + MALI_DEBUG_PRINT(3, ("Mali Soft Job: Soft job %u (0x%08X) activated while session is aborting.\n", job->id, job)); + + mali_soft_job_system_unlock(job->system); + + /* Since we are in shutdown, we can ignore the scheduling bitmask. */ + mali_timeline_tracker_release(&job->tracker); + mali_soft_job_destroy(job); + return schedule_mask; + } + + /* Send activated notification. */ + mali_soft_job_send_activated_notification(job); + + /* Wake up sleeping signaler. */ + job->activated = MALI_TRUE; + + /* If job type is self signaled, release tracker, move soft job to free list, and scheduler at once */ + if (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == job->type) { + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state); + + job->state = MALI_SOFT_JOB_STATE_SIGNALED; + mali_soft_job_system_unlock(job->system); + + schedule_mask |= mali_timeline_tracker_release(&job->tracker); + + mali_soft_job_destroy(job); + } else { + _mali_osk_wait_queue_wake_up(job->tracker.system->wait_queue); + + mali_soft_job_system_unlock(job->system); + } + + return schedule_mask; +} + +mali_scheduler_mask mali_soft_job_system_timeout_job(struct mali_soft_job *job) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(job); + MALI_DEBUG_ASSERT_POINTER(job->system); + MALI_DEBUG_ASSERT_POINTER(job->system->session); + MALI_DEBUG_ASSERT(MALI_TRUE == job->activated); + + MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeline timeout for soft job %u (0x%08X).\n", job->id, job)); + + mali_soft_job_system_lock(job->system); + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state || + MALI_SOFT_JOB_STATE_SIGNALED == job->state); + + if (unlikely(job->system->session->is_aborting)) { + /* The session is aborting. This job will be released and destroyed by @ref + * mali_soft_job_system_abort(). */ + mali_soft_job_system_unlock(job->system); + + return MALI_SCHEDULER_MASK_EMPTY; + } + + if (MALI_SOFT_JOB_STATE_STARTED != job->state) { + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_SIGNALED == job->state); + + /* The job is about to be signaled, ignore timeout. */ + MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeout on soft job %u (0x%08X) in signaled state.\n", job->id, job)); + mali_soft_job_system_unlock(job->system); + return schedule_mask; + } + + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state); + + job->state = MALI_SOFT_JOB_STATE_TIMED_OUT; + _mali_osk_atomic_inc(&job->refcount); + + mali_soft_job_system_unlock(job->system); + + schedule_mask = mali_timeline_tracker_release(&job->tracker); + + mali_soft_job_destroy(job); + + return schedule_mask; +} + +void mali_soft_job_system_abort(struct mali_soft_job_system *system) +{ + struct mali_soft_job *job, *tmp; + _MALI_OSK_LIST_HEAD_STATIC_INIT(jobs); + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT(system->session->is_aborting); + + MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting soft job system for session 0x%08X.\n", system->session)); + + mali_soft_job_system_lock(system); + + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &system->jobs_used, struct mali_soft_job, system_list) { + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state || + MALI_SOFT_JOB_STATE_TIMED_OUT == job->state); + + if (MALI_SOFT_JOB_STATE_STARTED == job->state) { + /* If the job has been activated, we have to release the tracker and destroy + * the job. If not, the tracker will be released and the job destroyed when + * it is activated. */ + if (MALI_TRUE == job->activated) { + MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting unsignaled soft job %u (0x%08X).\n", job->id, job)); + + job->state = MALI_SOFT_JOB_STATE_SIGNALED; + _mali_osk_list_move(&job->system_list, &jobs); + } + } else if (MALI_SOFT_JOB_STATE_TIMED_OUT == job->state) { + MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting timed out soft job %u (0x%08X).\n", job->id, job)); + + /* We need to destroy this soft job. */ + _mali_osk_list_move(&job->system_list, &jobs); + } + } + + mali_soft_job_system_unlock(system); + + /* Release and destroy jobs. */ + _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &jobs, struct mali_soft_job, system_list) { + MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_SIGNALED == job->state || + MALI_SOFT_JOB_STATE_TIMED_OUT == job->state); + + if (MALI_SOFT_JOB_STATE_SIGNALED == job->state) { + mali_timeline_tracker_release(&job->tracker); + } + + /* Move job back to used list before destroying. */ + _mali_osk_list_move(&job->system_list, &system->jobs_used); + + mali_soft_job_destroy(job); + } +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_soft_job.h b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.h new file mode 100755 index 000000000000..018ef4c527d9 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_soft_job.h @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_SOFT_JOB_H__ +#define __MALI_SOFT_JOB_H__ + +#include "mali_osk.h" + +#include "mali_timeline.h" + +struct mali_timeline_fence; +struct mali_session_data; +struct mali_soft_job; +struct mali_soft_job_system; + +/** + * Soft job types. + * + * Soft jobs of type MALI_SOFT_JOB_TYPE_USER_SIGNALED will only complete after activation if either + * they are signaled by user-space (@ref mali_soft_job_system_signaled_job) or if they are timed out + * by the Timeline system. + * Soft jobs of type MALI_SOFT_JOB_TYPE_SELF_SIGNALED will release job resource automatically + * in kernel when the job is activated. + */ +typedef enum mali_soft_job_type { + MALI_SOFT_JOB_TYPE_SELF_SIGNALED, + MALI_SOFT_JOB_TYPE_USER_SIGNALED, +} mali_soft_job_type; + +/** + * Soft job state. + * + * mali_soft_job_system_start_job a job will first be allocated.The job's state set to MALI_SOFT_JOB_STATE_ALLOCATED. + * Once the job is added to the timeline system, the state changes to MALI_SOFT_JOB_STATE_STARTED. + * + * For soft jobs of type MALI_SOFT_JOB_TYPE_USER_SIGNALED the state is changed to + * MALI_SOFT_JOB_STATE_SIGNALED when @ref mali_soft_job_system_signal_job is called and the soft + * job's state is MALI_SOFT_JOB_STATE_STARTED or MALI_SOFT_JOB_STATE_TIMED_OUT. + * + * If a soft job of type MALI_SOFT_JOB_TYPE_USER_SIGNALED is timed out before being signaled, the + * state is changed to MALI_SOFT_JOB_STATE_TIMED_OUT. This can only happen to soft jobs in state + * MALI_SOFT_JOB_STATE_STARTED. + * + */ +typedef enum mali_soft_job_state { + MALI_SOFT_JOB_STATE_ALLOCATED, + MALI_SOFT_JOB_STATE_STARTED, + MALI_SOFT_JOB_STATE_SIGNALED, + MALI_SOFT_JOB_STATE_TIMED_OUT, +} mali_soft_job_state; + +#define MALI_SOFT_JOB_INVALID_ID ((u32) -1) + +/** + * Soft job struct. + * + * Soft job can be used to represent any kind of CPU work done in kernel-space. + */ +typedef struct mali_soft_job { + mali_soft_job_type type; /**< Soft job type. Must be one of MALI_SOFT_JOB_TYPE_*. */ + u64 user_job; /**< Identifier for soft job in user space. */ + _mali_osk_atomic_t refcount; /**< Soft jobs are reference counted to prevent premature deletion. */ + struct mali_timeline_tracker tracker; /**< Timeline tracker for soft job. */ + mali_bool activated; /**< MALI_TRUE if the job has been activated, MALI_FALSE if not. */ + _mali_osk_notification_t *activated_notification; /**< Pre-allocated notification object for ACTIVATED_NOTIFICATION. */ + + /* Protected by soft job system lock. */ + u32 id; /**< Used by user-space to find corresponding soft job in kernel-space. */ + mali_soft_job_state state; /**< State of soft job, must be one of MALI_SOFT_JOB_STATE_*. */ + struct mali_soft_job_system *system; /**< The soft job system this job is in. */ + _mali_osk_list_t system_list; /**< List element used by soft job system. */ +} mali_soft_job; + +/** + * Per-session soft job system. + * + * The soft job system is used to manage all soft jobs that belongs to a session. + */ +typedef struct mali_soft_job_system { + struct mali_session_data *session; /**< The session this soft job system belongs to. */ + _MALI_OSK_LIST_HEAD(jobs_used); /**< List of all allocated soft jobs. */ + + _mali_osk_spinlock_irq_t *lock; /**< Lock used to protect soft job system and its soft jobs. */ + u32 lock_owner; /**< Contains tid of thread that locked the system or 0, if not locked. */ + u32 last_job_id; /**< Recored the last job id protected by lock. */ +} mali_soft_job_system; + +/** + * Create a soft job system. + * + * @param session The session this soft job system will belong to. + * @return The new soft job system, or NULL if unsuccessful. + */ +struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session); + +/** + * Destroy a soft job system. + * + * @note The soft job must not have any started or activated jobs. Call @ref + * mali_soft_job_system_abort first. + * + * @param system The soft job system we are destroying. + */ +void mali_soft_job_system_destroy(struct mali_soft_job_system *system); + +/** + * Create a soft job. + * + * @param system Soft job system to create soft job from. + * @param type Type of the soft job. + * @param user_job Identifier for soft job in user space. + * @return New soft job if successful, NULL if not. + */ +struct mali_soft_job *mali_soft_job_create(struct mali_soft_job_system *system, mali_soft_job_type type, u64 user_job); + +/** + * Destroy soft job. + * + * @param job Soft job to destroy. + */ +void mali_soft_job_destroy(struct mali_soft_job *job); + +/** + * Start a soft job. + * + * The soft job will be added to the Timeline system which will then activate it after all + * dependencies have been resolved. + * + * Create soft jobs with @ref mali_soft_job_create before starting them. + * + * @param job Soft job to start. + * @param fence Fence representing dependencies for this soft job. + * @return Point on soft job timeline. + */ +mali_timeline_point mali_soft_job_start(struct mali_soft_job *job, struct mali_timeline_fence *fence); + +/** + * Use by user-space to signal that a soft job has completed. + * + * @note Only valid for soft jobs with type MALI_SOFT_JOB_TYPE_USER_SIGNALED. + * + * @note The soft job must be in state MALI_SOFT_JOB_STATE_STARTED for the signal to be successful. + * + * @note If the soft job was signaled successfully, or it received a time out, the soft job will be + * destroyed after this call and should no longer be used. + * + * @note This function will block until the soft job has been activated. + * + * @param system The soft job system the job was started in. + * @param job_id ID of soft job we are signaling. + * + * @return _MALI_OSK_ERR_ITEM_NOT_FOUND if the soft job ID was invalid, _MALI_OSK_ERR_TIMEOUT if the + * soft job was timed out or _MALI_OSK_ERR_OK if we successfully signaled the soft job. + */ +_mali_osk_errcode_t mali_soft_job_system_signal_job(struct mali_soft_job_system *system, u32 job_id); + +/** + * Used by the Timeline system to activate a soft job. + * + * @param job The soft job that is being activated. + * @return A scheduling bitmask. + */ +mali_scheduler_mask mali_soft_job_system_activate_job(struct mali_soft_job *job); + +/** + * Used by the Timeline system to timeout a soft job. + * + * A soft job is timed out if it completes or is signaled later than MALI_TIMELINE_TIMEOUT_HZ after + * activation. + * + * @param job The soft job that is being timed out. + * @return A scheduling bitmask. + */ +mali_scheduler_mask mali_soft_job_system_timeout_job(struct mali_soft_job *job); + +/** + * Used to cleanup activated soft jobs in the soft job system on session abort. + * + * @param system The soft job system that is being aborted. + */ +void mali_soft_job_system_abort(struct mali_soft_job_system *system); + +#endif /* __MALI_SOFT_JOB_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c new file mode 100755 index 000000000000..f829e99f02ab --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.c @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_spinlock_reentrant.h" + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +struct mali_spinlock_reentrant *mali_spinlock_reentrant_init(_mali_osk_lock_order_t lock_order) +{ + struct mali_spinlock_reentrant *spinlock; + + spinlock = _mali_osk_calloc(1, sizeof(struct mali_spinlock_reentrant)); + if (NULL == spinlock) { + return NULL; + } + + spinlock->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, lock_order); + if (NULL == spinlock->lock) { + mali_spinlock_reentrant_term(spinlock); + return NULL; + } + + return spinlock; +} + +void mali_spinlock_reentrant_term(struct mali_spinlock_reentrant *spinlock) +{ + MALI_DEBUG_ASSERT_POINTER(spinlock); + MALI_DEBUG_ASSERT(0 == spinlock->counter && 0 == spinlock->owner); + + if (NULL != spinlock->lock) { + _mali_osk_spinlock_irq_term(spinlock->lock); + } + + _mali_osk_free(spinlock); +} + +void mali_spinlock_reentrant_wait(struct mali_spinlock_reentrant *spinlock, u32 tid) +{ + MALI_DEBUG_ASSERT_POINTER(spinlock); + MALI_DEBUG_ASSERT_POINTER(spinlock->lock); + MALI_DEBUG_ASSERT(0 != tid); + + MALI_DEBUG_PRINT(5, ("%s ^\n", __FUNCTION__)); + + if (tid != spinlock->owner) { + _mali_osk_spinlock_irq_lock(spinlock->lock); + MALI_DEBUG_ASSERT(0 == spinlock->owner && 0 == spinlock->counter); + spinlock->owner = tid; + } + + MALI_DEBUG_PRINT(5, ("%s v\n", __FUNCTION__)); + + ++spinlock->counter; +} + +void mali_spinlock_reentrant_signal(struct mali_spinlock_reentrant *spinlock, u32 tid) +{ + MALI_DEBUG_ASSERT_POINTER(spinlock); + MALI_DEBUG_ASSERT_POINTER(spinlock->lock); + MALI_DEBUG_ASSERT(0 != tid && tid == spinlock->owner); + + --spinlock->counter; + if (0 == spinlock->counter) { + spinlock->owner = 0; + MALI_DEBUG_PRINT(5, ("%s release last\n", __FUNCTION__)); + _mali_osk_spinlock_irq_unlock(spinlock->lock); + } +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h new file mode 100755 index 000000000000..4d788ec1bbe4 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_spinlock_reentrant.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_SPINLOCK_REENTRANT_H__ +#define __MALI_SPINLOCK_REENTRANT_H__ + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +/** + * Reentrant spinlock. + */ +struct mali_spinlock_reentrant { + _mali_osk_spinlock_irq_t *lock; + u32 owner; + u32 counter; +}; + +/** + * Create a new reentrant spinlock. + * + * @param lock_order Lock order. + * @return New reentrant spinlock. + */ +struct mali_spinlock_reentrant *mali_spinlock_reentrant_init(_mali_osk_lock_order_t lock_order); + +/** + * Terminate reentrant spinlock and free any associated resources. + * + * @param spinlock Reentrant spinlock to terminate. + */ +void mali_spinlock_reentrant_term(struct mali_spinlock_reentrant *spinlock); + +/** + * Wait for reentrant spinlock to be signaled. + * + * @param spinlock Reentrant spinlock. + * @param tid Thread ID. + */ +void mali_spinlock_reentrant_wait(struct mali_spinlock_reentrant *spinlock, u32 tid); + +/** + * Signal reentrant spinlock. + * + * @param spinlock Reentrant spinlock. + * @param tid Thread ID. + */ +void mali_spinlock_reentrant_signal(struct mali_spinlock_reentrant *spinlock, u32 tid); + +/** + * Check if thread is holding reentrant spinlock. + * + * @param spinlock Reentrant spinlock. + * @param tid Thread ID. + * @return MALI_TRUE if thread is holding spinlock, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_spinlock_reentrant_is_held(struct mali_spinlock_reentrant *spinlock, u32 tid) +{ + MALI_DEBUG_ASSERT_POINTER(spinlock->lock); + return (tid == spinlock->owner && 0 < spinlock->counter); +} + +#endif /* __MALI_SPINLOCK_REENTRANT_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline.c new file mode 100755 index 000000000000..ffffee9306ce --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline.c @@ -0,0 +1,1964 @@ +/* + * Copyright (C) 2013-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include "mali_timeline.h" +#include "mali_kernel_common.h" +#include "mali_scheduler.h" +#include "mali_soft_job.h" +#include "mali_timeline_fence_wait.h" +#include "mali_timeline_sync_fence.h" +#include "mali_executor.h" +#include "mali_pp_job.h" + +#define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid())) + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +_mali_osk_wq_work_t *sync_fence_callback_work_t = NULL; +_mali_osk_spinlock_irq_t *sync_fence_callback_list_lock = NULL; +static _MALI_OSK_LIST_HEAD_STATIC_INIT(sync_fence_callback_queue); +#endif + +/* + * Following three elements are used to record how many + * gp, physical pp or virtual pp jobs are delayed in the whole + * timeline system, we can use these three value to decide + * if need to deactivate idle group. + */ +_mali_osk_atomic_t gp_tracker_count; +_mali_osk_atomic_t phy_pp_tracker_count; +_mali_osk_atomic_t virt_pp_tracker_count; + +static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system, + struct mali_timeline_waiter *waiter); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) +#include +#include +#include + +struct mali_deferred_fence_put_entry { + struct hlist_node list; + struct sync_fence *fence; +}; + +static HLIST_HEAD(mali_timeline_sync_fence_to_free_list); +static DEFINE_SPINLOCK(mali_timeline_sync_fence_to_free_lock); + +static void put_sync_fences(struct work_struct *ignore) +{ + struct hlist_head list; + struct hlist_node *tmp, *pos; + unsigned long flags; + struct mali_deferred_fence_put_entry *o; + + spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags); + hlist_move_list(&mali_timeline_sync_fence_to_free_list, &list); + spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags); + + hlist_for_each_entry_safe(o, pos, tmp, &list, list) { + sync_fence_put(o->fence); + kfree(o); + } +} + +static DECLARE_DELAYED_WORK(delayed_sync_fence_put, put_sync_fences); +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */ + +/* Callback that is called when a sync fence a tracker is waiting on is signaled. */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter) +#else +static void mali_timeline_sync_fence_callback(struct mali_internal_sync_fence *sync_fence, struct mali_internal_sync_fence_waiter *sync_fence_waiter) +#endif +{ + struct mali_timeline_tracker *tracker; + + MALI_IGNORE(sync_fence); + MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter); + + tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter); + MALI_DEBUG_ASSERT_POINTER(tracker); + + _mali_osk_spinlock_irq_lock(sync_fence_callback_list_lock); + _mali_osk_list_addtail(&tracker->sync_fence_signal_list, &sync_fence_callback_queue); + _mali_osk_spinlock_irq_unlock(sync_fence_callback_list_lock); + + _mali_osk_wq_schedule_work(sync_fence_callback_work_t); +} +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + +static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker) +{ + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type); + + return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job); +} + +static void mali_timeline_timer_callback(void *data) +{ + struct mali_timeline_system *system; + struct mali_timeline_tracker *tracker; + struct mali_timeline *timeline; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + u32 tid = _mali_osk_get_tid(); + + timeline = (struct mali_timeline *) data; + MALI_DEBUG_ASSERT_POINTER(timeline); + + system = timeline->system; + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + if (!system->timer_enabled) { + mali_spinlock_reentrant_signal(system->spinlock, tid); + return; + } + + tracker = timeline->tracker_tail; + timeline->timer_active = MALI_FALSE; + + if (NULL != tracker && MALI_TRUE == tracker->timer_active) { + /* This is likely the delayed work that has been schedule out before cancelled. */ + if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) { + mali_spinlock_reentrant_signal(system->spinlock, tid); + return; + } + + schedule_mask = mali_timeline_tracker_time_out(tracker); + tracker->timer_active = MALI_FALSE; + } else { + MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n")); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); +} + +void mali_timeline_system_stop_timer(struct mali_timeline_system *system) +{ + u32 i; + u32 tid = _mali_osk_get_tid(); + + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + system->timer_enabled = MALI_FALSE; + mali_spinlock_reentrant_signal(system->spinlock, tid); + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline = system->timelines[i]; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + if (NULL != timeline->delayed_work) { + _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work); + timeline->timer_active = MALI_FALSE; + } + } +} + +static void mali_timeline_destroy(struct mali_timeline *timeline) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + if (NULL != timeline) { + /* Assert that the timeline object has been properly cleaned up before destroying it. */ + MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next); + MALI_DEBUG_ASSERT(NULL == timeline->tracker_head); + MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail); + MALI_DEBUG_ASSERT(NULL == timeline->waiter_head); + MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail); + MALI_DEBUG_ASSERT(NULL != timeline->system); + MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id); + + if (NULL != timeline->delayed_work) { + _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work); + _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work); + } + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + if (NULL != timeline->sync_tl) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + sync_timeline_destroy(timeline->sync_tl); +#else + mali_internal_sync_timeline_destroy(timeline->sync_tl); +#endif + } +#else + _mali_osk_free(timeline); +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + } +} + +static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id) +{ + struct mali_timeline *timeline; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX); + + timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline)); + if (NULL == timeline) { + return NULL; + } + + /* Initially the timeline is empty. */ +#if defined(MALI_TIMELINE_DEBUG_START_POINT) + /* Start the timeline a bit before wrapping when debugging. */ + timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128; +#else + timeline->point_next = 1; +#endif + timeline->point_oldest = timeline->point_next; + + /* The tracker and waiter lists will initially be empty. */ + + timeline->system = system; + timeline->id = id; + + timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline); + if (NULL == timeline->delayed_work) { + mali_timeline_destroy(timeline); + return NULL; + } + + timeline->timer_active = MALI_FALSE; + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + { + char timeline_name[32]; + + switch (id) { + case MALI_TIMELINE_GP: + _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid()); + break; + case MALI_TIMELINE_PP: + _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid()); + break; + case MALI_TIMELINE_SOFT: + _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid()); + break; + default: + MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id)); + mali_timeline_destroy(timeline); + return NULL; + } + + timeline->destroyed = MALI_FALSE; + + timeline->sync_tl = mali_sync_timeline_create(timeline, timeline_name); + if (NULL == timeline->sync_tl) { + mali_timeline_destroy(timeline); + return NULL; + } + + timeline->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM); + if (NULL == timeline->spinlock) { + mali_timeline_destroy(timeline); + return NULL; + } + } +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + + return timeline; +} + +static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT_POINTER(tracker); + + if (mali_timeline_is_full(timeline)) { + /* Don't add tracker if timeline is full. */ + tracker->point = MALI_TIMELINE_NO_POINT; + return; + } + + tracker->timeline = timeline; + tracker->point = timeline->point_next; + + /* Find next available point. */ + timeline->point_next++; + if (MALI_TIMELINE_NO_POINT == timeline->point_next) { + timeline->point_next++; + } + + MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); + + if (MALI_TIMELINE_TRACKER_GP == tracker->type) { + _mali_osk_atomic_inc(&gp_tracker_count); + } else if (MALI_TIMELINE_TRACKER_PP == tracker->type) { + if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) { + _mali_osk_atomic_inc(&virt_pp_tracker_count); + } else { + _mali_osk_atomic_inc(&phy_pp_tracker_count); + } + } + + /* Add tracker as new head on timeline's tracker list. */ + if (NULL == timeline->tracker_head) { + /* Tracker list is empty. */ + MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail); + + timeline->tracker_tail = tracker; + + MALI_DEBUG_ASSERT(NULL == tracker->timeline_next); + MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev); + } else { + MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next); + + tracker->timeline_prev = timeline->tracker_head; + timeline->tracker_head->timeline_next = tracker; + + MALI_DEBUG_ASSERT(NULL == tracker->timeline_next); + } + timeline->tracker_head = tracker; + + MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next); + MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev); +} + +/* Inserting the waiter object into the given timeline */ +static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new) +{ + struct mali_timeline_waiter *waiter_prev; + struct mali_timeline_waiter *waiter_next; + + /* Waiter time must be between timeline head and tail, and there must + * be less than MALI_TIMELINE_MAX_POINT_SPAN elements between */ + MALI_DEBUG_ASSERT((waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN); + MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN); + + /* Finding out where to put this waiter, in the linked waiter list of the given timeline **/ + waiter_prev = timeline->waiter_head; /* Insert new after waiter_prev */ + waiter_next = NULL; /* Insert new before waiter_next */ + + /* Iterating backwards from head (newest) to tail (oldest) until we + * find the correct spot to insert the new waiter */ + while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) { + waiter_next = waiter_prev; + waiter_prev = waiter_prev->timeline_prev; + } + + if (NULL == waiter_prev && NULL == waiter_next) { + /* list is empty */ + timeline->waiter_head = waiter_new; + timeline->waiter_tail = waiter_new; + } else if (NULL == waiter_next) { + /* insert at head */ + waiter_new->timeline_prev = timeline->waiter_head; + timeline->waiter_head->timeline_next = waiter_new; + timeline->waiter_head = waiter_new; + } else if (NULL == waiter_prev) { + /* insert at tail */ + waiter_new->timeline_next = timeline->waiter_tail; + timeline->waiter_tail->timeline_prev = waiter_new; + timeline->waiter_tail = waiter_new; + } else { + /* insert between */ + waiter_new->timeline_next = waiter_next; + waiter_new->timeline_prev = waiter_prev; + waiter_next->timeline_prev = waiter_new; + waiter_prev->timeline_next = waiter_new; + } +} + +static void mali_timeline_update_delayed_work(struct mali_timeline *timeline) +{ + struct mali_timeline_system *system; + struct mali_timeline_tracker *oldest_tracker; + + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id); + + system = timeline->system; + MALI_DEBUG_ASSERT_POINTER(system); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + /* Timer is disabled, early out. */ + if (!system->timer_enabled) return; + + oldest_tracker = timeline->tracker_tail; + if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) { + if (MALI_FALSE == oldest_tracker->timer_active) { + if (MALI_TRUE == timeline->timer_active) { + _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work); + } + _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ); + oldest_tracker->timer_active = MALI_TRUE; + timeline->timer_active = MALI_TRUE; + } + } else if (MALI_TRUE == timeline->timer_active) { + _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work); + timeline->timer_active = MALI_FALSE; + } +} + +static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + MALI_DEBUG_CODE({ + struct mali_timeline_system *system = timeline->system; + MALI_DEBUG_ASSERT_POINTER(system); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + }); + + if (NULL != timeline->tracker_tail) { + /* Set oldest point to oldest tracker's point */ + timeline->point_oldest = timeline->tracker_tail->point; + } else { + /* No trackers, mark point list as empty */ + timeline->point_oldest = timeline->point_next; + } + + /* Release all waiters no longer on the timeline's point list. + * Releasing a waiter can trigger this function to be called again, so + * we do not store any pointers on stack. */ + while (NULL != timeline->waiter_tail) { + u32 waiter_time_relative; + u32 time_head_relative; + struct mali_timeline_waiter *waiter = timeline->waiter_tail; + + time_head_relative = timeline->point_next - timeline->point_oldest; + waiter_time_relative = waiter->point - timeline->point_oldest; + + if (waiter_time_relative < time_head_relative) { + /* This and all following waiters are on the point list, so we are done. */ + break; + } + + /* Remove waiter from timeline's waiter list. */ + if (NULL != waiter->timeline_next) { + waiter->timeline_next->timeline_prev = NULL; + } else { + /* This was the last waiter */ + timeline->waiter_head = NULL; + } + timeline->waiter_tail = waiter->timeline_next; + + /* Release waiter. This could activate a tracker, if this was + * the last waiter for the tracker. */ + schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter); + } + + return schedule_mask; +} + +static mali_scheduler_mask mali_timeline_release_with_depended_point(struct mali_timeline_tracker *tracker) +{ + struct mali_timeline *timeline; + struct mali_timeline_waiter *waiter; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + timeline = tracker->timeline; + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id); + + MALI_DEBUG_CODE({ + struct mali_timeline_system *system = timeline->system; + MALI_DEBUG_ASSERT_POINTER(system); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + }); + + /* Only release the waiter that wait for the tracker. */ + waiter = timeline->waiter_tail; + while (NULL != waiter) { + if (waiter->point == tracker->point) { + + struct mali_timeline_waiter *waiter_next; + struct mali_timeline_waiter *waiter_prev; + + waiter_next = waiter->timeline_next; + waiter_prev = waiter->timeline_prev; + waiter->timeline_next = NULL; + waiter->timeline_prev = NULL; + + if (NULL != waiter_prev) { + waiter_prev->timeline_next = waiter_next; + } + + if (NULL != waiter_next) { + waiter_next->timeline_prev = waiter_prev; + } + + if (waiter == timeline->waiter_tail) + timeline->waiter_tail = waiter_next; + + if (waiter == timeline->waiter_head) + timeline->waiter_head = NULL; + + schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter); + waiter = waiter_next; + }else { + + waiter = waiter->timeline_next; + } + } + + return schedule_mask; +} + +void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker, + mali_timeline_tracker_type type, + struct mali_timeline_fence *fence, + void *job) +{ + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT_POINTER(job); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type); + + /* Zero out all tracker members. */ + _mali_osk_memset(tracker, 0, sizeof(*tracker)); + + tracker->type = type; + tracker->job = job; + tracker->trigger_ref_count = 1; /* Prevents any callback from trigging while adding it */ + tracker->os_tick_create = _mali_osk_time_tickcount(); + MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC); + + tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE; + + /* Copy fence. */ + if (NULL != fence) { + _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence)); + } +} + +mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker) +{ + struct mali_timeline *timeline; + struct mali_timeline_system *system; + struct mali_timeline_tracker *tracker_next, *tracker_prev; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + u32 tid = _mali_osk_get_tid(); + + /* Upon entry a group lock will be held, but not a scheduler lock. */ + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); + + /* Tracker should have been triggered */ + MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count); + + /* All waiters should have been released at this point */ + MALI_DEBUG_ASSERT(NULL == tracker->waiter_head); + MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); + + MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job)); + + timeline = tracker->timeline; + if (NULL == timeline) { + /* Tracker was not on a timeline, there is nothing to release. */ + return MALI_SCHEDULER_MASK_EMPTY; + } + + system = timeline->system; + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + /* Tracker should still be on timeline */ + MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); + MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, tracker->point)); + + /* Tracker is no longer valid. */ + MALI_DEBUG_CODE(tracker->magic = 0); + + tracker_next = tracker->timeline_next; + tracker_prev = tracker->timeline_prev; + tracker->timeline_next = NULL; + tracker->timeline_prev = NULL; + + /* Removing tracker from timeline's tracker list */ + if (NULL == tracker_next) { + /* This tracker was the head */ + timeline->tracker_head = tracker_prev; + } else { + tracker_next->timeline_prev = tracker_prev; + } + + if (NULL == tracker_prev) { + /* This tracker was the tail */ + timeline->tracker_tail = tracker_next; + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + /* Update the timeline's oldest time and release any waiters */ + schedule_mask |= mali_timeline_update_oldest_point(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + } else { + tracker_prev->timeline_next = tracker_next; + if (MALI_TIMELINE_SOFT == tracker->timeline->id) { + /* Use the signaled soft tracker to release the depended soft waiter */ + schedule_mask |= mali_timeline_release_with_depended_point(tracker); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + } + } + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + /* Update delayed work only when it is the soft job timeline */ + if (MALI_TIMELINE_SOFT == tracker->timeline->id) { + mali_timeline_update_delayed_work(tracker->timeline); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + return schedule_mask; +} + +void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system, + struct mali_timeline_waiter *tail, + struct mali_timeline_waiter *head) +{ + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(head); + MALI_DEBUG_ASSERT_POINTER(tail); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + head->tracker_next = system->waiter_empty_list; + system->waiter_empty_list = tail; +} + +static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + struct mali_timeline_system *system; + struct mali_timeline *timeline; + u32 tid = _mali_osk_get_tid(); + + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); + + system = tracker->system; + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + tracker->os_tick_activate = _mali_osk_time_tickcount(); + + if (NULL != tracker->waiter_head) { + mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head); + tracker->waiter_head = NULL; + tracker->waiter_tail = NULL; + } + + switch (tracker->type) { + case MALI_TIMELINE_TRACKER_GP: + schedule_mask = mali_scheduler_activate_gp_job((struct mali_gp_job *) tracker->job); + + _mali_osk_atomic_dec(&gp_tracker_count); + break; + case MALI_TIMELINE_TRACKER_PP: + if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) { + _mali_osk_atomic_dec(&virt_pp_tracker_count); + } else { + _mali_osk_atomic_dec(&phy_pp_tracker_count); + } + schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job); + break; + case MALI_TIMELINE_TRACKER_SOFT: + timeline = tracker->timeline; + MALI_DEBUG_ASSERT_POINTER(timeline); + + schedule_mask |= mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job); + + /* Start a soft timer to make sure the soft job be released in a limited time */ + mali_spinlock_reentrant_wait(system->spinlock, tid); + mali_timeline_update_delayed_work(timeline); + mali_spinlock_reentrant_signal(system->spinlock, tid); + break; + case MALI_TIMELINE_TRACKER_WAIT: + mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job); + break; + case MALI_TIMELINE_TRACKER_SYNC: +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job); +#else + MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type)); +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + break; + default: + MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type)); + break; + } + + return schedule_mask; +} + +void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker) +{ + u32 tid = _mali_osk_get_tid(); + + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); + tracker->trigger_ref_count++; + + mali_spinlock_reentrant_signal(system->spinlock, tid); +} + +mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error) +{ + u32 tid = _mali_osk_get_tid(); + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(tracker); + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); + tracker->trigger_ref_count--; + + tracker->activation_error |= activation_error; + + if (0 == tracker->trigger_ref_count) { + schedule_mask |= mali_timeline_tracker_activate(tracker); + tracker = NULL; + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + return schedule_mask; +} + +void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(fence); + MALI_DEBUG_ASSERT_POINTER(uk_fence); + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + fence->points[i] = uk_fence->points[i]; + } + + fence->sync_fd = uk_fence->sync_fd; +} + +struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session) +{ + u32 i; + struct mali_timeline_system *system; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n")); + + system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system)); + if (NULL == system) { + return NULL; + } + + system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM); + if (NULL == system->spinlock) { + mali_timeline_system_destroy(system); + return NULL; + } + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i); + if (NULL == system->timelines[i]) { + mali_timeline_system_destroy(system); + return NULL; + } + } + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + system->signaled_sync_tl = mali_sync_timeline_create(NULL, "mali-always-signaled"); + if (NULL == system->signaled_sync_tl) { + mali_timeline_system_destroy(system); + return NULL; + } +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + + system->waiter_empty_list = NULL; + system->session = session; + system->timer_enabled = MALI_TRUE; + + system->wait_queue = _mali_osk_wait_queue_init(); + if (NULL == system->wait_queue) { + mali_timeline_system_destroy(system); + return NULL; + } + + return system; +} + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) ||defined(CONFIG_SYNC) ||defined(CONFIG_SYNC_FILE) +/** + * Check if there are any trackers left on timeline. + * + * Used as a wait queue conditional. + * + * @param data Timeline. + * @return MALI_TRUE if there are no trackers on timeline, MALI_FALSE if not. + */ +static mali_bool mali_timeline_has_no_trackers(void *data) +{ + struct mali_timeline *timeline = (struct mali_timeline *) data; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + return mali_timeline_is_empty(timeline); +} +#if defined(CONFIG_SYNC) ||defined(CONFIG_SYNC_FILE) +/** + * Cancel sync fence waiters waited upon by trackers on all timelines. + * + * Will return after all timelines have no trackers left. + * + * @param system Timeline system. + */ +static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system) +{ + u32 i; + u32 tid = _mali_osk_get_tid(); + struct mali_timeline_tracker *tracker, *tracker_next; + _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list); + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT(system->session->is_aborting); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + /* Cancel sync fence waiters. */ + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline = system->timelines[i]; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + tracker_next = timeline->tracker_tail; + while (NULL != tracker_next) { + tracker = tracker_next; + tracker_next = tracker->timeline_next; + + if (NULL == tracker->sync_fence) continue; + + MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker)); + + /* Cancel sync fence waiter. */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) { +#else + if (0 == mali_internal_sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) { +#endif + /* Callback was not called, move tracker to local list. */ + _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list); + } + } + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + /* Manually call sync fence callback in order to release waiter and trigger activation of tracker. */ + _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) { + mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter); + } + + /* Sleep until all sync fence callbacks are done and all timelines are empty. */ + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline = system->timelines[i]; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline); + } +} + +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) +static void mali_timeline_cancel_dma_fence_waiters(struct mali_timeline_system *system) +{ + u32 i, j; + u32 tid = _mali_osk_get_tid(); + struct mali_pp_job *pp_job = NULL; + struct mali_pp_job *next_pp_job = NULL; + struct mali_timeline *timeline = NULL; + struct mali_timeline_tracker *tracker, *tracker_next; + _MALI_OSK_LIST_HEAD_STATIC_INIT(pp_job_list); + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT(system->session->is_aborting); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + /* Cancel dma fence waiters. */ + timeline = system->timelines[MALI_TIMELINE_PP]; + MALI_DEBUG_ASSERT_POINTER(timeline); + + tracker_next = timeline->tracker_tail; + while (NULL != tracker_next) { + mali_bool fence_is_signaled = MALI_TRUE; + tracker = tracker_next; + tracker_next = tracker->timeline_next; + + if (NULL == tracker->waiter_dma_fence) continue; + pp_job = (struct mali_pp_job *)tracker->job; + MALI_DEBUG_ASSERT_POINTER(pp_job); + MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling dma fence waiter for tracker 0x%08X.\n", tracker)); + + for (j = 0; j < pp_job->dma_fence_context.num_dma_fence_waiter; j++) { + if (pp_job->dma_fence_context.mali_dma_fence_waiters[j]) { + /* Cancel a previously callback from the fence. + * This function returns true if the callback is successfully removed, + * or false if the fence has already been signaled. + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + bool ret = dma_fence_remove_callback(pp_job->dma_fence_context.mali_dma_fence_waiters[j]->fence, + &pp_job->dma_fence_context.mali_dma_fence_waiters[j]->base); + +#else + bool ret = fence_remove_callback(pp_job->dma_fence_context.mali_dma_fence_waiters[j]->fence, + &pp_job->dma_fence_context.mali_dma_fence_waiters[j]->base); +#endif + if (ret) { + fence_is_signaled = MALI_FALSE; + } + } + } + + /* Callbacks were not called, move pp job to local list. */ + if (MALI_FALSE == fence_is_signaled) + _mali_osk_list_add(&pp_job->list, &pp_job_list); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + /* Manually call dma fence callback in order to release waiter and trigger activation of tracker. */ + _MALI_OSK_LIST_FOREACHENTRY(pp_job, next_pp_job, &pp_job_list, struct mali_pp_job, list) { + mali_timeline_dma_fence_callback((void *)pp_job); + } + + /* Sleep until all dma fence callbacks are done and all timelines are empty. */ + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline = system->timelines[i]; + MALI_DEBUG_ASSERT_POINTER(timeline); + _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline); + } +} +#endif +#endif +void mali_timeline_system_abort(struct mali_timeline_system *system) +{ + MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid();); + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT(system->session->is_aborting); + + MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session)); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + mali_timeline_cancel_sync_fence_waiters(system); +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) + mali_timeline_cancel_dma_fence_waiters(system); +#endif + + /* Should not be any waiters or trackers left at this point. */ + MALI_DEBUG_CODE({ + u32 i; + mali_spinlock_reentrant_wait(system->spinlock, tid); + for (i = 0; i < MALI_TIMELINE_MAX; ++i) + { + struct mali_timeline *timeline = system->timelines[i]; + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next); + MALI_DEBUG_ASSERT(NULL == timeline->tracker_head); + MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail); + MALI_DEBUG_ASSERT(NULL == timeline->waiter_head); + MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail); + } + mali_spinlock_reentrant_signal(system->spinlock, tid); + }); +} + +void mali_timeline_system_destroy(struct mali_timeline_system *system) +{ + u32 i; + struct mali_timeline_waiter *waiter, *next; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + u32 tid = _mali_osk_get_tid(); +#endif + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + + MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n")); + + if (NULL != system) { + + /* There should be no waiters left on this queue. */ + if (NULL != system->wait_queue) { + _mali_osk_wait_queue_term(system->wait_queue); + system->wait_queue = NULL; + } + + /* Free all waiters in empty list */ + waiter = system->waiter_empty_list; + while (NULL != waiter) { + next = waiter->tracker_next; + _mali_osk_free(waiter); + waiter = next; + } + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + if (NULL != system->signaled_sync_tl) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + sync_timeline_destroy(system->signaled_sync_tl); +#else + mali_internal_sync_timeline_destroy(system->signaled_sync_tl); +#endif + } + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + if ((NULL != system->timelines[i]) && (NULL != system->timelines[i]->spinlock)) { + mali_spinlock_reentrant_wait(system->timelines[i]->spinlock, tid); + system->timelines[i]->destroyed = MALI_TRUE; + mali_spinlock_reentrant_signal(system->timelines[i]->spinlock, tid); + } + } +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + if (NULL != system->timelines[i]) { + mali_timeline_destroy(system->timelines[i]); + } + } + + if (NULL != system->spinlock) { + mali_spinlock_reentrant_term(system->spinlock); + } + + _mali_osk_free(system); + } +} + +/** + * Find how many waiters are needed for a given fence. + * + * @param fence The fence to check. + * @return Number of waiters needed for fence. + */ +static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence) +{ + u32 i, num_waiters = 0; + + MALI_DEBUG_ASSERT_POINTER(fence); + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + if (MALI_TIMELINE_NO_POINT != fence->points[i]) { + ++num_waiters; + } + } + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + if (-1 != fence->sync_fd) ++num_waiters; +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + + return num_waiters; +} + +static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system) +{ + struct mali_timeline_waiter *waiter; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + waiter = system->waiter_empty_list; + if (NULL != waiter) { + /* Remove waiter from empty list and zero it */ + system->waiter_empty_list = waiter->tracker_next; + _mali_osk_memset(waiter, 0, sizeof(*waiter)); + } + + /* Return NULL if list was empty. */ + return waiter; +} + +static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system, + struct mali_timeline_waiter **tail, + struct mali_timeline_waiter **head, + int max_num_waiters) +{ + u32 i, tid = _mali_osk_get_tid(); + mali_bool do_alloc; + struct mali_timeline_waiter *waiter; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(tail); + MALI_DEBUG_ASSERT_POINTER(head); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + *head = *tail = NULL; + do_alloc = MALI_FALSE; + i = 0; + while (i < max_num_waiters) { + if (MALI_FALSE == do_alloc) { + waiter = mali_timeline_system_get_zeroed_waiter(system); + if (NULL == waiter) { + do_alloc = MALI_TRUE; + mali_spinlock_reentrant_signal(system->spinlock, tid); + continue; + } + } else { + waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter)); + if (NULL == waiter) break; + } + ++i; + if (NULL == *tail) { + *tail = waiter; + *head = waiter; + } else { + (*head)->tracker_next = waiter; + *head = waiter; + } + } + if (MALI_TRUE == do_alloc) { + mali_spinlock_reentrant_wait(system->spinlock, tid); + } +} + +/** + * Create waiters for the given tracker. The tracker is activated when all waiters are release. + * + * @note Tracker can potentially be activated before this function returns. + * + * @param system Timeline system. + * @param tracker Tracker we will create waiters for. + * @param waiter_tail List of pre-allocated waiters. + * @param waiter_head List of pre-allocated waiters. + */ +static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system, + struct mali_timeline_tracker *tracker, + struct mali_timeline_waiter *waiter_tail, + struct mali_timeline_waiter *waiter_head) +{ + int i; + u32 tid = _mali_osk_get_tid(); + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_fence *sync_fence = NULL; +#else + struct mali_internal_sync_fence *sync_fence = NULL; +#endif +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(tracker); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + MALI_DEBUG_ASSERT(NULL == tracker->waiter_head); + MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); + MALI_DEBUG_ASSERT(NULL != tracker->job); + + /* Creating waiter object for all the timelines the fence is put on. Inserting this waiter + * into the timelines sorted list of waiters */ + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + mali_timeline_point point; + struct mali_timeline *timeline; + struct mali_timeline_waiter *waiter; + + /* Get point on current timeline from tracker's fence. */ + point = tracker->fence.points[i]; + + if (likely(MALI_TIMELINE_NO_POINT == point)) { + /* Fence contains no point on this timeline so we don't need a waiter. */ + continue; + } + + timeline = system->timelines[i]; + MALI_DEBUG_ASSERT_POINTER(timeline); + + if (unlikely(!mali_timeline_is_point_valid(timeline, point))) { + MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n", + point, timeline->point_oldest, timeline->point_next)); + continue; + } + + if (likely(mali_timeline_is_point_released(timeline, point))) { + /* Tracker representing the point has been released so we don't need a + * waiter. */ + continue; + } + + if ((MALI_TIMELINE_SOFT == timeline->id) && mali_timeline_is_tracker_released(timeline, point)) { + /* The tracker that the point related to has already been released, so no need to a waiter. */ + continue; + } + + /* The point is on timeline. */ + MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point)); + + /* Get a new zeroed waiter object. */ + if (likely(NULL != waiter_tail)) { + waiter = waiter_tail; + waiter_tail = waiter_tail->tracker_next; + } else { + MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); + continue; + } + + /* Yanking the trigger ref count of the tracker. */ + tracker->trigger_ref_count++; + + waiter->point = point; + waiter->tracker = tracker; + + /* Insert waiter on tracker's singly-linked waiter list. */ + if (NULL == tracker->waiter_head) { + /* list is empty */ + MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); + tracker->waiter_tail = waiter; + } else { + tracker->waiter_head->tracker_next = waiter; + } + tracker->waiter_head = waiter; + + /* Add waiter to timeline. */ + mali_timeline_insert_waiter(timeline, waiter); + } +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + if (-1 != tracker->fence.sync_fd) { + int ret; + struct mali_timeline_waiter *waiter; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + sync_fence = sync_fence_fdget(tracker->fence.sync_fd); +#else + sync_fence = mali_internal_sync_fence_fdget(tracker->fence.sync_fd); +#endif + if (unlikely(NULL == sync_fence)) { + MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd)); + goto exit; + } + + /* Check if we have a zeroed waiter object available. */ + if (unlikely(NULL == waiter_tail)) { + MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); + goto exit; + } + + /* Start asynchronous wait that will release waiter when the fence is signaled. */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback); + ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter); +#else + mali_internal_sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback); + ret = mali_internal_sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter); +#endif + if (1 == ret) { + /* Fence already signaled, no waiter needed. */ + tracker->fence.sync_fd = -1; + goto exit; + } else if (0 != ret) { + MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret)); + tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT; + goto exit; + } + + /* Grab new zeroed waiter object. */ + waiter = waiter_tail; + waiter_tail = waiter_tail->tracker_next; + + /* Increase the trigger ref count of the tracker. */ + tracker->trigger_ref_count++; + + waiter->point = MALI_TIMELINE_NO_POINT; + waiter->tracker = tracker; + + /* Insert waiter on tracker's singly-linked waiter list. */ + if (NULL == tracker->waiter_head) { + /* list is empty */ + MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); + tracker->waiter_tail = waiter; + } else { + tracker->waiter_head->tracker_next = waiter; + } + tracker->waiter_head = waiter; + + /* Also store waiter in separate field for easy access by sync callback. */ + tracker->waiter_sync = waiter; + + /* Store the sync fence in tracker so we can retrieve in abort session, if needed. */ + tracker->sync_fence = sync_fence; + + sync_fence = NULL; + } +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)*/ +#if defined(CONFIG_MALI_DMA_BUF_FENCE) + if ((NULL != tracker->timeline) && (MALI_TIMELINE_PP == tracker->timeline->id)) { + + struct mali_pp_job *job = (struct mali_pp_job *)tracker->job; + + if (0 < job->dma_fence_context.num_dma_fence_waiter) { + struct mali_timeline_waiter *waiter; + /* Check if we have a zeroed waiter object available. */ + if (unlikely(NULL == waiter_tail)) { + MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); + goto exit; + } + + /* Grab new zeroed waiter object. */ + waiter = waiter_tail; + waiter_tail = waiter_tail->tracker_next; + + /* Increase the trigger ref count of the tracker. */ + tracker->trigger_ref_count++; + + waiter->point = MALI_TIMELINE_NO_POINT; + waiter->tracker = tracker; + + /* Insert waiter on tracker's singly-linked waiter list. */ + if (NULL == tracker->waiter_head) { + /* list is empty */ + MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); + tracker->waiter_tail = waiter; + } else { + tracker->waiter_head->tracker_next = waiter; + } + tracker->waiter_head = waiter; + + /* Also store waiter in separate field for easy access by sync callback. */ + tracker->waiter_dma_fence = waiter; + } + } +#endif /* defined(CONFIG_MALI_DMA_BUF_FENCE)*/ + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) ||defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +exit: +#endif /* defined(CONFIG_MALI_DMA_BUF_FENCE) || defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + + if (NULL != waiter_tail) { + mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head); + } + + /* Release the initial trigger ref count. */ + tracker->trigger_ref_count--; + + /* If there were no waiters added to this tracker we activate immediately. */ + if (0 == tracker->trigger_ref_count) { + schedule_mask |= mali_timeline_tracker_activate(tracker); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + if (NULL != sync_fence) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + sync_fence_put(sync_fence); +#else + fput(sync_fence->file); +#endif + } +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + + mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); +} + +mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system, + struct mali_timeline_tracker *tracker, + enum mali_timeline_id timeline_id) +{ + int num_waiters = 0; + struct mali_timeline_waiter *waiter_tail, *waiter_head; + u32 tid = _mali_osk_get_tid(); + + mali_timeline_point point = MALI_TIMELINE_NO_POINT; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + MALI_DEBUG_ASSERT_POINTER(tracker); + + MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting); + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type); + MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); + + MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id)); + + MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); + tracker->system = system; + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + num_waiters = mali_timeline_fence_num_waiters(&tracker->fence); + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) + if (MALI_TIMELINE_PP == timeline_id) { + struct mali_pp_job *job = (struct mali_pp_job *)tracker->job; + if (0 < job->dma_fence_context.num_dma_fence_waiter) + num_waiters++; + } +#endif + + /* Allocate waiters. */ + mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters); + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + /* Add tracker to timeline. This will allocate a point for the tracker on the timeline. If + * timeline ID is MALI_TIMELINE_NONE the tracker will NOT be added to a timeline and the + * point will be MALI_TIMELINE_NO_POINT. + * + * NOTE: the tracker can fail to be added if the timeline is full. If this happens, the + * point will be MALI_TIMELINE_NO_POINT. */ + MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE); + if (likely(timeline_id < MALI_TIMELINE_MAX)) { + struct mali_timeline *timeline = system->timelines[timeline_id]; + mali_timeline_insert_tracker(timeline, tracker); + MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); + } + + point = tracker->point; + + /* Create waiters for tracker based on supplied fence. Each waiter will increase the + * trigger ref count. */ + mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head); + tracker = NULL; + + /* At this point the tracker object might have been freed so we should no longer + * access it. */ + + + /* The tracker will always be activated after calling add_tracker, even if NO_POINT is + * returned. */ + return point; +} + +static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system, + struct mali_timeline_waiter *waiter) +{ + struct mali_timeline_tracker *tracker; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(waiter); + + MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); + + tracker = waiter->tracker; + MALI_DEBUG_ASSERT_POINTER(tracker); + + /* At this point the waiter has been removed from the timeline's waiter list, but it is + * still on the tracker's waiter list. All of the tracker's waiters will be released when + * the tracker is activated. */ + + waiter->point = MALI_TIMELINE_NO_POINT; + waiter->tracker = NULL; + + tracker->trigger_ref_count--; + if (0 == tracker->trigger_ref_count) { + /* This was the last waiter; activate tracker */ + schedule_mask |= mali_timeline_tracker_activate(tracker); + tracker = NULL; + } + + return schedule_mask; +} + +mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system, + enum mali_timeline_id timeline_id) +{ + mali_timeline_point point; + struct mali_timeline *timeline; + u32 tid = _mali_osk_get_tid(); + + MALI_DEBUG_ASSERT_POINTER(system); + + if (MALI_TIMELINE_MAX <= timeline_id) { + return MALI_TIMELINE_NO_POINT; + } + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + timeline = system->timelines[timeline_id]; + MALI_DEBUG_ASSERT_POINTER(timeline); + + point = MALI_TIMELINE_NO_POINT; + if (timeline->point_oldest != timeline->point_next) { + point = timeline->point_next - 1; + if (MALI_TIMELINE_NO_POINT == point) point--; + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + return point; +} + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +static void mali_timeline_do_sync_fence_callback(void *arg) +{ + _MALI_OSK_LIST_HEAD_STATIC_INIT(list); + struct mali_timeline_tracker *tracker; + struct mali_timeline_tracker *tmp_tracker; + u32 tid = _mali_osk_get_tid(); + + MALI_IGNORE(arg); + + /* + * Quickly "unhook" the jobs pending to be deleted, so we can release + * the lock before we start deleting the job objects + * (without any locks held) + */ + _mali_osk_spinlock_irq_lock(sync_fence_callback_list_lock); + _mali_osk_list_move_list(&sync_fence_callback_queue, &list); + _mali_osk_spinlock_irq_unlock(sync_fence_callback_list_lock); + + _MALI_OSK_LIST_FOREACHENTRY(tracker, tmp_tracker, &list, + struct mali_timeline_tracker, sync_fence_signal_list) { + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + mali_bool is_aborting = MALI_FALSE; + int fence_status = 0; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_fence *sync_fence = NULL; +#else + struct mali_internal_sync_fence *sync_fence = NULL; +#endif + struct mali_timeline_system *system = NULL; + struct mali_timeline_waiter *waiter = NULL; + + _mali_osk_list_delinit(&tracker->sync_fence_signal_list); + + sync_fence = tracker->sync_fence; + MALI_DEBUG_ASSERT_POINTER(sync_fence); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + fence_status = sync_fence->status; +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + fence_status = atomic_read(&sync_fence->status); +#else + fence_status = sync_fence->fence->ops->signaled(sync_fence->fence); +#endif + + system = tracker->system; + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + is_aborting = system->session->is_aborting; + if (!is_aborting && (0 > fence_status)) { + MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status)); + tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT; + } + + waiter = tracker->waiter_sync; + MALI_DEBUG_ASSERT_POINTER(waiter); + + tracker->sync_fence = NULL; + tracker->fence.sync_fd = -1; + + schedule_mask |= mali_timeline_system_release_waiter(system, waiter); + + /* If aborting, wake up sleepers that are waiting for sync fence callbacks to complete. */ + if (is_aborting) { + _mali_osk_wait_queue_wake_up(system->wait_queue); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + /* + * Older versions of Linux, before 3.5, doesn't support fput() in interrupt + * context. For those older kernels, allocate a list object and put the + * fence object on that and defer the call to sync_fence_put() to a workqueue. + */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) + { + struct mali_deferred_fence_put_entry *obj; + + obj = kzalloc(sizeof(struct mali_deferred_fence_put_entry), GFP_ATOMIC); + if (obj) { + unsigned long flags; + mali_bool schedule = MALI_FALSE; + + obj->fence = sync_fence; + + spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags); + if (hlist_empty(&mali_timeline_sync_fence_to_free_list)) + schedule = MALI_TRUE; + hlist_add_head(&obj->list, &mali_timeline_sync_fence_to_free_list); + spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags); + + if (schedule) + schedule_delayed_work(&delayed_sync_fence_put, 0); + } + } +#else +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + sync_fence_put(sync_fence); +#else + fput(sync_fence->file); +#endif +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */ + + if (!is_aborting) { + mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE); + } + } +} +#endif +_mali_osk_errcode_t mali_timeline_initialize(void) +{ + _mali_osk_atomic_init(&gp_tracker_count, 0); + _mali_osk_atomic_init(&phy_pp_tracker_count, 0); + _mali_osk_atomic_init(&virt_pp_tracker_count, 0); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + sync_fence_callback_list_lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST); + if (NULL == sync_fence_callback_list_lock) { + return _MALI_OSK_ERR_NOMEM; + } + + sync_fence_callback_work_t = _mali_osk_wq_create_work( + mali_timeline_do_sync_fence_callback, NULL); + + if (NULL == sync_fence_callback_work_t) { + return _MALI_OSK_ERR_FAULT; + } +#endif + return _MALI_OSK_ERR_OK; +} + + +void mali_timeline_terminate(void) +{ + _mali_osk_atomic_term(&gp_tracker_count); + _mali_osk_atomic_term(&phy_pp_tracker_count); + _mali_osk_atomic_term(&virt_pp_tracker_count); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + if (NULL != sync_fence_callback_list_lock) { + _mali_osk_spinlock_irq_term(sync_fence_callback_list_lock); + sync_fence_callback_list_lock = NULL; + } + + if (NULL != sync_fence_callback_work_t) { + _mali_osk_wq_delete_work(sync_fence_callback_work_t); + sync_fence_callback_work_t = NULL; + } +#endif +} + +#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) + +static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id) +{ + struct mali_timeline *timeline; + struct mali_timeline_system *system; + + MALI_DEBUG_ASSERT_POINTER(tracker); + + MALI_DEBUG_ASSERT_POINTER(tracker->timeline); + timeline = tracker->timeline; + + MALI_DEBUG_ASSERT_POINTER(timeline->system); + system = timeline->system; + + if (MALI_TIMELINE_MAX > id) { + if (MALI_TIMELINE_NO_POINT != tracker->fence.points[id]) { + return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]); + } else { + return MALI_FALSE; + } + } else { + MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id); + return MALI_FALSE; + } +} + +static const char *timeline_id_to_string(enum mali_timeline_id id) +{ + switch (id) { + case MALI_TIMELINE_GP: + return "GP"; + case MALI_TIMELINE_PP: + return "PP"; + case MALI_TIMELINE_SOFT: + return "SOFT"; + default: + return "NONE"; + } +} + +static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type) +{ + switch (type) { + case MALI_TIMELINE_TRACKER_GP: + return "GP"; + case MALI_TIMELINE_TRACKER_PP: + return "PP"; + case MALI_TIMELINE_TRACKER_SOFT: + return "SOFT"; + case MALI_TIMELINE_TRACKER_WAIT: + return "WAIT"; + case MALI_TIMELINE_TRACKER_SYNC: + return "SYNC"; + default: + return "INVALID"; + } +} + +mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker) +{ + struct mali_timeline *timeline = NULL; + + MALI_DEBUG_ASSERT_POINTER(tracker); + timeline = tracker->timeline; + + if (0 != tracker->trigger_ref_count) { + return MALI_TIMELINE_TS_WAITING; + } + + if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) { + return MALI_TIMELINE_TS_ACTIVE; + } + + if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) { + return MALI_TIMELINE_TS_INIT; + } + + return MALI_TIMELINE_TS_FINISH; +} + +void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx) +{ + const char *tracker_state = "IWAF"; + char state_char = 'I'; + char tracker_type[32] = {0}; + + MALI_DEBUG_ASSERT_POINTER(tracker); + + state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)); + _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type)); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + if (0 != tracker->trigger_ref_count) { + if (print_ctx) + _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job)); + else + MALI_DEBUG_PRINT(2, ("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job))); + } else { + if (print_ctx) + _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job)); + else + MALI_DEBUG_PRINT(2, ("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job))); + + } +#else + if (0 != tracker->trigger_ref_count) { + if (print_ctx) + _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + (unsigned int)(uintptr_t)(tracker->job)); + else + MALI_DEBUG_PRINT(2, ("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + (unsigned int)(uintptr_t)(tracker->job))); + } else { + if (print_ctx) + _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + (unsigned int)(uintptr_t)(tracker->job)); + else + MALI_DEBUG_PRINT(2, ("TL: %s %u %c job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + (unsigned int)(uintptr_t)(tracker->job))); + + } +#endif +} + +void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx) +{ + struct mali_timeline_tracker *tracker = NULL; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + tracker = timeline->tracker_tail; + while (NULL != tracker) { + mali_timeline_debug_print_tracker(tracker, print_ctx); + tracker = tracker->timeline_next; + } +} + +#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)) +void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker) +{ + const char *tracker_state = "IWAF"; + char state_char = 'I'; + char tracker_type[32] = {0}; + + MALI_DEBUG_ASSERT_POINTER(tracker); + + state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)); + _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type)); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + if (0 != tracker->trigger_ref_count) { + MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + tracker->fence.sync_fd, tracker->sync_fence, tracker->job)); + } else { + MALI_PRINT(("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + tracker->fence.sync_fd, tracker->sync_fence, tracker->job)); + } +#else + if (0 != tracker->trigger_ref_count) { + MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", + tracker_type, tracker->point, state_char, tracker->trigger_ref_count, + is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], + is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], + is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], + tracker->job)); + } else { + MALI_PRINT(("TL: %s %u %c job:(0x%08X)\n", + tracker_type, tracker->point, state_char, + tracker->job)); + } +#endif +} + +void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline) +{ + struct mali_timeline_tracker *tracker = NULL; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + tracker = timeline->tracker_tail; + while (NULL != tracker) { + mali_timeline_debug_direct_print_tracker(tracker); + tracker = tracker->timeline_next; + } +} + +#endif + +void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx) +{ + int i; + int num_printed = 0; + u32 tid = _mali_osk_get_tid(); + + MALI_DEBUG_ASSERT_POINTER(system); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + /* Print all timelines */ + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline = system->timelines[i]; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + if (NULL == timeline->tracker_head) continue; + if (print_ctx) + _mali_osk_ctxprintf(print_ctx, "TL: Timeline %s:\n", + timeline_id_to_string((enum mali_timeline_id)i)); + else + MALI_DEBUG_PRINT(2, ("TL: Timeline %s: oldest (%u) next(%u)\n", + timeline_id_to_string((enum mali_timeline_id)i), timeline->point_oldest, timeline->point_next)); + + mali_timeline_debug_print_timeline(timeline, print_ctx); + num_printed++; + } + + if (0 == num_printed) { + if (print_ctx) + _mali_osk_ctxprintf(print_ctx, "TL: All timelines empty\n"); + else + MALI_DEBUG_PRINT(2, ("TL: All timelines empty\n")); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); +} + +#endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */ + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) +void mali_timeline_dma_fence_callback(void *pp_job_ptr) +{ + struct mali_timeline_system *system; + struct mali_timeline_waiter *waiter; + struct mali_timeline_tracker *tracker; + struct mali_pp_job *pp_job = (struct mali_pp_job *)pp_job_ptr; + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + u32 tid = _mali_osk_get_tid(); + mali_bool is_aborting = MALI_FALSE; + + MALI_DEBUG_ASSERT_POINTER(pp_job); + + tracker = &pp_job->tracker; + MALI_DEBUG_ASSERT_POINTER(tracker); + + system = tracker->system; + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(system->session); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + waiter = tracker->waiter_dma_fence; + MALI_DEBUG_ASSERT_POINTER(waiter); + + schedule_mask |= mali_timeline_system_release_waiter(system, waiter); + + is_aborting = system->session->is_aborting; + + /* If aborting, wake up sleepers that are waiting for dma fence callbacks to complete. */ + if (is_aborting) { + _mali_osk_wait_queue_wake_up(system->wait_queue); + } + + mali_spinlock_reentrant_signal(system->spinlock, tid); + + if (!is_aborting) { + mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE); + } +} +#endif diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline.h new file mode 100755 index 000000000000..3e8bfc8fb733 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline.h @@ -0,0 +1,587 @@ +/* + * Copyright (C) 2013-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_TIMELINE_H__ +#define __MALI_TIMELINE_H__ + +#include "mali_osk.h" +#include "mali_ukk.h" +#include "mali_session.h" +#include "mali_kernel_common.h" +#include "mali_spinlock_reentrant.h" +#include "mali_sync.h" +#include "mali_scheduler_types.h" +#include + +/** + * Soft job timeout. + * + * Soft jobs have to be signaled as complete after activation. Normally this is done by user space, + * but in order to guarantee that every soft job is completed, we also have a timer. + */ +#define MALI_TIMELINE_TIMEOUT_HZ ((unsigned long) (HZ * 3 / 2)) /* 1500 ms. */ + +/** + * Timeline type. + */ +typedef enum mali_timeline_id { + MALI_TIMELINE_GP = MALI_UK_TIMELINE_GP, /**< GP job timeline. */ + MALI_TIMELINE_PP = MALI_UK_TIMELINE_PP, /**< PP job timeline. */ + MALI_TIMELINE_SOFT = MALI_UK_TIMELINE_SOFT, /**< Soft job timeline. */ + MALI_TIMELINE_MAX = MALI_UK_TIMELINE_MAX +} mali_timeline_id; + +/** + * Used by trackers that should not be added to a timeline (@ref mali_timeline_system_add_tracker). + */ +#define MALI_TIMELINE_NONE MALI_TIMELINE_MAX + +/** + * Tracker type. + */ +typedef enum mali_timeline_tracker_type { + MALI_TIMELINE_TRACKER_GP = 0, /**< Tracker used by GP jobs. */ + MALI_TIMELINE_TRACKER_PP = 1, /**< Tracker used by PP jobs. */ + MALI_TIMELINE_TRACKER_SOFT = 2, /**< Tracker used by soft jobs. */ + MALI_TIMELINE_TRACKER_WAIT = 3, /**< Tracker used for fence wait. */ + MALI_TIMELINE_TRACKER_SYNC = 4, /**< Tracker used for sync fence. */ + MALI_TIMELINE_TRACKER_MAX = 5, +} mali_timeline_tracker_type; + +/** + * Tracker activation error. + */ +typedef u32 mali_timeline_activation_error; +#define MALI_TIMELINE_ACTIVATION_ERROR_NONE 0 +#define MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT (1<<1) +#define MALI_TIMELINE_ACTIVATION_ERROR_FATAL_BIT (1<<0) + +/** + * Type used to represent a point on a timeline. + */ +typedef u32 mali_timeline_point; + +/** + * Used to represent that no point on a timeline. + */ +#define MALI_TIMELINE_NO_POINT ((mali_timeline_point) 0) + +/** + * The maximum span of points on a timeline. A timeline will be considered full if the difference + * between the oldest and newest points is equal or larger to this value. + */ +#define MALI_TIMELINE_MAX_POINT_SPAN 65536 + +/** + * Magic value used to assert on validity of trackers. + */ +#define MALI_TIMELINE_TRACKER_MAGIC 0xabcdabcd + +struct mali_timeline; +struct mali_timeline_waiter; +struct mali_timeline_tracker; + +/** + * Timeline fence. + */ +struct mali_timeline_fence { + mali_timeline_point points[MALI_TIMELINE_MAX]; /**< For each timeline, a point or MALI_TIMELINE_NO_POINT. */ + s32 sync_fd; /**< A file descriptor representing a sync fence, or -1. */ +}; + +/** + * Timeline system. + * + * The Timeline system has a set of timelines associated with a session. + */ +struct mali_timeline_system { + struct mali_spinlock_reentrant *spinlock; /**< Spin lock protecting the timeline system */ + struct mali_timeline *timelines[MALI_TIMELINE_MAX]; /**< The timelines in this system */ + + /* Single-linked list of unused waiter objects. Uses the tracker_next field in tracker. */ + struct mali_timeline_waiter *waiter_empty_list; + + struct mali_session_data *session; /**< Session that owns this system. */ + + mali_bool timer_enabled; /**< Set to MALI_TRUE if soft job timer should be enabled, MALI_FALSE if not. */ + + _mali_osk_wait_queue_t *wait_queue; /**< Wait queue. */ + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_timeline *signaled_sync_tl; /**< Special sync timeline used to create pre-signaled sync fences */ +#else + struct mali_internal_sync_timeline *signaled_sync_tl; /**< Special sync timeline used to create pre-signaled sync fences */ +#endif +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ +}; + +/** + * Timeline. Each Timeline system will have MALI_TIMELINE_MAX timelines. + */ +struct mali_timeline { + mali_timeline_point point_next; /**< The next available point. */ + mali_timeline_point point_oldest; /**< The oldest point not released. */ + + /* Double-linked list of trackers. Sorted in ascending order by tracker->time_number with + * tail pointing to the tracker with the oldest time. */ + struct mali_timeline_tracker *tracker_head; + struct mali_timeline_tracker *tracker_tail; + + /* Double-linked list of waiters. Sorted in ascending order by waiter->time_number_wait + * with tail pointing to the waiter with oldest wait time. */ + struct mali_timeline_waiter *waiter_head; + struct mali_timeline_waiter *waiter_tail; + + struct mali_timeline_system *system; /**< Timeline system this timeline belongs to. */ + enum mali_timeline_id id; /**< Timeline type. */ + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_timeline *sync_tl; /**< Sync timeline that corresponds to this timeline. */ +#else + struct mali_internal_sync_timeline *sync_tl; +#endif + mali_bool destroyed; + struct mali_spinlock_reentrant *spinlock; /**< Spin lock protecting the timeline system */ +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + + /* The following fields are used to time out soft job trackers. */ + _mali_osk_wq_delayed_work_t *delayed_work; + mali_bool timer_active; +}; + +/** + * Timeline waiter. + */ +struct mali_timeline_waiter { + mali_timeline_point point; /**< Point on timeline we are waiting for to be released. */ + struct mali_timeline_tracker *tracker; /**< Tracker that is waiting. */ + + struct mali_timeline_waiter *timeline_next; /**< Next waiter on timeline's waiter list. */ + struct mali_timeline_waiter *timeline_prev; /**< Previous waiter on timeline's waiter list. */ + + struct mali_timeline_waiter *tracker_next; /**< Next waiter on tracker's waiter list. */ +}; + +/** + * Timeline tracker. + */ +struct mali_timeline_tracker { + MALI_DEBUG_CODE(u32 magic); /**< Should always be MALI_TIMELINE_TRACKER_MAGIC for a valid tracker. */ + + mali_timeline_point point; /**< Point on timeline for this tracker */ + + struct mali_timeline_tracker *timeline_next; /**< Next tracker on timeline's tracker list */ + struct mali_timeline_tracker *timeline_prev; /**< Previous tracker on timeline's tracker list */ + + u32 trigger_ref_count; /**< When zero tracker will be activated */ + mali_timeline_activation_error activation_error; /**< Activation error. */ + struct mali_timeline_fence fence; /**< Fence used to create this tracker */ + + /* Single-linked list of waiters. Sorted in order of insertions with + * tail pointing to first waiter. */ + struct mali_timeline_waiter *waiter_head; + struct mali_timeline_waiter *waiter_tail; + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + /* These are only used if the tracker is waiting on a sync fence. */ + struct mali_timeline_waiter *waiter_sync; /**< A direct pointer to timeline waiter representing sync fence. */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_fence_waiter sync_fence_waiter; /**< Used to connect sync fence and tracker in sync fence wait callback. */ + struct sync_fence *sync_fence; /**< The sync fence this tracker is waiting on. */ +#else + struct mali_internal_sync_fence_waiter sync_fence_waiter; /**< Used to connect sync fence and tracker in sync fence wait callback. */ + struct mali_internal_sync_fence *sync_fence; /**< The sync fence this tracker is waiting on. */ +#endif + _mali_osk_list_t sync_fence_cancel_list; /**< List node used to cancel sync fence waiters. */ + _mali_osk_list_t sync_fence_signal_list; /** < List node used to singal sync fence callback function. */ + +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) + struct mali_timeline_waiter *waiter_dma_fence; /**< A direct pointer to timeline waiter representing dma fence. */ +#endif + + struct mali_timeline_system *system; /**< Timeline system. */ + struct mali_timeline *timeline; /**< Timeline, or NULL if not on a timeline. */ + enum mali_timeline_tracker_type type; /**< Type of tracker. */ + void *job; /**< Owner of tracker. */ + + /* The following fields are used to time out soft job trackers. */ + unsigned long os_tick_create; + unsigned long os_tick_activate; + mali_bool timer_active; +}; + +extern _mali_osk_atomic_t gp_tracker_count; +extern _mali_osk_atomic_t phy_pp_tracker_count; +extern _mali_osk_atomic_t virt_pp_tracker_count; + +/** + * What follows is a set of functions to check the state of a timeline and to determine where on a + * timeline a given point is. Most of these checks will translate the timeline so the oldest point + * on the timeline is aligned with zero. Remember that all of these calculation are done on + * unsigned integers. + * + * The following example illustrates the three different states a point can be in. The timeline has + * been translated to put the oldest point at zero: + * + * + * + * [ point is in forbidden zone ] + * 64k wide + * MALI_TIMELINE_MAX_POINT_SPAN + * + * [ point is on timeline ) ( point is released ] + * + * 0--------------------------##############################--------------------2^32 - 1 + * ^ ^ + * \ | + * oldest point on timeline | + * \ + * next point on timeline + */ + +/** + * Compare two timeline points + * + * Returns true if a is after b, false if a is before or equal to b. + * + * This funcion ignores MALI_TIMELINE_MAX_POINT_SPAN. Wrapping is supported and + * the result will be correct if the points is less then UINT_MAX/2 apart. + * + * @param a Point on timeline + * @param b Point on timeline + * @return MALI_TRUE if a is after b + */ +MALI_STATIC_INLINE mali_bool mali_timeline_point_after(mali_timeline_point a, mali_timeline_point b) +{ + return 0 > ((s32)b) - ((s32)a); +} + +/** + * Check if a point is on timeline. A point is on a timeline if it is greater than, or equal to, + * the oldest point, and less than the next point. + * + * @param timeline Timeline. + * @param point Point on timeline. + * @return MALI_TRUE if point is on timeline, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_point_on(struct mali_timeline *timeline, mali_timeline_point point) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); + + return (point - timeline->point_oldest) < (timeline->point_next - timeline->point_oldest); +} + +/** + * Check if a point has been released. A point is released if it is older than the oldest point on + * the timeline, newer than the next point, and also not in the forbidden zone. + * + * @param timeline Timeline. + * @param point Point on timeline. + * @return MALI_TRUE if point has been release, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_point_released(struct mali_timeline *timeline, mali_timeline_point point) +{ + mali_timeline_point point_normalized; + mali_timeline_point next_normalized; + + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); + + point_normalized = point - timeline->point_oldest; + next_normalized = timeline->point_next - timeline->point_oldest; + + return point_normalized > (next_normalized + MALI_TIMELINE_MAX_POINT_SPAN); +} + +/** + * Check if the tracker that the point relate to has been released. A point is released if the tracker is not on the timeline. + * @param timeline Timeline. + * @param point Point on timeline. + * @return MALI_TRUE if the tracker has been release, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_tracker_released(struct mali_timeline *timeline, mali_timeline_point point) +{ + struct mali_timeline_tracker *tracker; + + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); + + tracker = timeline->tracker_tail; + + while (NULL != tracker) { + if (point == tracker->point) + return MALI_FALSE; + tracker = tracker->timeline_next; + } + + return MALI_TRUE; +} + +/** + * Check if a point is valid. A point is valid if is on the timeline or has been released. + * + * @param timeline Timeline. + * @param point Point on timeline. + * @return MALI_TRUE if point is valid, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_point_valid(struct mali_timeline *timeline, mali_timeline_point point) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + return mali_timeline_is_point_on(timeline, point) || mali_timeline_is_point_released(timeline, point); +} + +/** + * Check if timeline is empty (has no points on it). A timeline is empty if next == oldest. + * + * @param timeline Timeline. + * @return MALI_TRUE if timeline is empty, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_empty(struct mali_timeline *timeline) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + return timeline->point_next == timeline->point_oldest; +} + +/** + * Check if timeline is full. A valid timeline cannot span more than 64k points (@ref + * MALI_TIMELINE_MAX_POINT_SPAN). + * + * @param timeline Timeline. + * @return MALI_TRUE if timeline is full, MALI_FALSE if not. + */ +MALI_STATIC_INLINE mali_bool mali_timeline_is_full(struct mali_timeline *timeline) +{ + MALI_DEBUG_ASSERT_POINTER(timeline); + return MALI_TIMELINE_MAX_POINT_SPAN <= (timeline->point_next - timeline->point_oldest); +} + +/** + * Create a new timeline system. + * + * @param session The session this timeline system will belong to. + * @return New timeline system. + */ +struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session); + +/** + * Abort timeline system. + * + * This will release all pending waiters in the timeline system causing all trackers to be + * activated. + * + * @param system Timeline system to abort all jobs from. + */ +void mali_timeline_system_abort(struct mali_timeline_system *system); + +/** + * Destroy an empty timeline system. + * + * @note @ref mali_timeline_system_abort() should be called prior to this function. + * + * @param system Timeline system to destroy. + */ +void mali_timeline_system_destroy(struct mali_timeline_system *system); + +/** + * Stop the soft job timer. + * + * @param system Timeline system + */ +void mali_timeline_system_stop_timer(struct mali_timeline_system *system); + +/** + * Add a tracker to a timeline system and optionally also on a timeline. + * + * Once added to the timeline system, the tracker is guaranteed to be activated. The tracker can be + * activated before this function returns. Thus, it is also possible that the tracker is released + * before this function returns, depending on the tracker type. + * + * @note Tracker must be initialized (@ref mali_timeline_tracker_init) before being added to the + * timeline system. + * + * @param system Timeline system the tracker will be added to. + * @param tracker The tracker to be added. + * @param timeline_id Id of the timeline the tracker will be added to, or + * MALI_TIMELINE_NONE if it should not be added on a timeline. + * @return Point on timeline identifying this tracker, or MALI_TIMELINE_NO_POINT if not on timeline. + */ +mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system, + struct mali_timeline_tracker *tracker, + enum mali_timeline_id timeline_id); + +/** + * Get latest point on timeline. + * + * @param system Timeline system. + * @param timeline_id Id of timeline to get latest point from. + * @return Latest point on timeline, or MALI_TIMELINE_NO_POINT if the timeline is empty. + */ +mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system, + enum mali_timeline_id timeline_id); + +/** + * Initialize tracker. + * + * Must be called before tracker is added to timeline system (@ref mali_timeline_system_add_tracker). + * + * @param tracker Tracker to initialize. + * @param type Type of tracker. + * @param fence Fence used to set up dependencies for tracker. + * @param job Pointer to job struct this tracker is associated with. + */ +void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker, + mali_timeline_tracker_type type, + struct mali_timeline_fence *fence, + void *job); + +/** + * Grab trigger ref count on tracker. + * + * This will prevent tracker from being activated until the trigger ref count reaches zero. + * + * @note Tracker must have been initialized (@ref mali_timeline_tracker_init). + * + * @param system Timeline system. + * @param tracker Tracker. + */ +void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker); + +/** + * Release trigger ref count on tracker. + * + * If the trigger ref count reaches zero, the tracker will be activated. + * + * @param system Timeline system. + * @param tracker Tracker. + * @param activation_error Error bitmask if activated with error, or MALI_TIMELINE_ACTIVATION_ERROR_NONE if no error. + * @return Scheduling bitmask. + */ +mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error); + +/** + * Release a tracker from the timeline system. + * + * This is used to signal that the job being tracker is finished, either due to normal circumstances + * (job complete/abort) or due to a timeout. + * + * We may need to schedule some subsystems after a tracker has been released and the returned + * bitmask will tell us if it is necessary. If the return value is non-zero, this value needs to be + * sent as an input parameter to @ref mali_scheduler_schedule_from_mask() to do the scheduling. + * + * @note Tracker must have been activated before being released. + * @warning Not calling @ref mali_scheduler_schedule_from_mask() after releasing a tracker can lead + * to a deadlock. + * + * @param tracker Tracker being released. + * @return Scheduling bitmask. + */ +mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker); + +MALI_STATIC_INLINE mali_bool mali_timeline_tracker_activation_error( + struct mali_timeline_tracker *tracker) +{ + MALI_DEBUG_ASSERT_POINTER(tracker); + return (MALI_TIMELINE_ACTIVATION_ERROR_FATAL_BIT & + tracker->activation_error) ? MALI_TRUE : MALI_FALSE; +} + +/** + * Copy data from a UK fence to a Timeline fence. + * + * @param fence Timeline fence. + * @param uk_fence UK fence. + */ +void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence); + +_mali_osk_errcode_t mali_timeline_initialize(void); + +void mali_timeline_terminate(void); + +MALI_STATIC_INLINE mali_bool mali_timeline_has_gp_job(void) +{ + return 0 < _mali_osk_atomic_read(&gp_tracker_count); +} + +MALI_STATIC_INLINE mali_bool mali_timeline_has_physical_pp_job(void) +{ + return 0 < _mali_osk_atomic_read(&phy_pp_tracker_count); +} + +MALI_STATIC_INLINE mali_bool mali_timeline_has_virtual_pp_job(void) +{ + return 0 < _mali_osk_atomic_read(&virt_pp_tracker_count); +} + +#if defined(DEBUG) +#define MALI_TIMELINE_DEBUG_FUNCTIONS +#endif /* DEBUG */ +#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) + +/** + * Tracker state. Used for debug printing. + */ +typedef enum mali_timeline_tracker_state { + MALI_TIMELINE_TS_INIT = 0, + MALI_TIMELINE_TS_WAITING = 1, + MALI_TIMELINE_TS_ACTIVE = 2, + MALI_TIMELINE_TS_FINISH = 3, +} mali_timeline_tracker_state; + +/** + * Get tracker state. + * + * @param tracker Tracker to check. + * @return State of tracker. + */ +mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker); + +/** + * Print debug information about tracker. + * + * @param tracker Tracker to print. + */ +void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx); + +/** + * Print debug information about timeline. + * + * @param timeline Timeline to print. + */ +void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx); + +#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)) +void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker); +void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline); +#endif + +/** + * Print debug information about timeline system. + * + * @param system Timeline system to print. + */ +void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx); + +#endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */ + +#if defined(CONFIG_MALI_DMA_BUF_FENCE) +/** + * The timeline dma fence callback when dma fence signal. + * + * @param pp_job_ptr The pointer to pp job that link to the signaled dma fence. + */ +void mali_timeline_dma_fence_callback(void *pp_job_ptr); +#endif + +#endif /* __MALI_TIMELINE_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c new file mode 100755 index 000000000000..1ab13f50997f --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c @@ -0,0 +1,218 @@ +/* + * Copyright (C) 2013-2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include "mali_timeline_fence_wait.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_spinlock_reentrant.h" + +/** + * Allocate a fence waiter tracker. + * + * @return New fence waiter if successful, NULL if not. + */ +static struct mali_timeline_fence_wait_tracker *mali_timeline_fence_wait_tracker_alloc(void) +{ + return (struct mali_timeline_fence_wait_tracker *) _mali_osk_calloc(1, sizeof(struct mali_timeline_fence_wait_tracker)); +} + +/** + * Free fence waiter tracker. + * + * @param wait Fence wait tracker to free. + */ +static void mali_timeline_fence_wait_tracker_free(struct mali_timeline_fence_wait_tracker *wait) +{ + MALI_DEBUG_ASSERT_POINTER(wait); + _mali_osk_atomic_term(&wait->refcount); + _mali_osk_free(wait); +} + +/** + * Check if fence wait tracker has been activated. Used as a wait queue condition. + * + * @param data Fence waiter. + * @return MALI_TRUE if tracker has been activated, MALI_FALSE if not. + */ +static mali_bool mali_timeline_fence_wait_tracker_is_activated(void *data) +{ + struct mali_timeline_fence_wait_tracker *wait; + + wait = (struct mali_timeline_fence_wait_tracker *) data; + MALI_DEBUG_ASSERT_POINTER(wait); + + return wait->activated; +} + +/** + * Check if fence has been signaled. + * + * @param system Timeline system. + * @param fence Timeline fence. + * @return MALI_TRUE if fence is signaled, MALI_FALSE if not. + */ +static mali_bool mali_timeline_fence_wait_check_status(struct mali_timeline_system *system, struct mali_timeline_fence *fence) +{ + int i; + u32 tid = _mali_osk_get_tid(); + mali_bool ret = MALI_TRUE; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_fence *sync_fence = NULL; +#else + struct mali_internal_sync_fence *sync_fence = NULL; +#endif +#endif + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(fence); + + mali_spinlock_reentrant_wait(system->spinlock, tid); + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline; + mali_timeline_point point; + + point = fence->points[i]; + + if (likely(MALI_TIMELINE_NO_POINT == point)) { + /* Fence contains no point on this timeline. */ + continue; + } + + timeline = system->timelines[i]; + MALI_DEBUG_ASSERT_POINTER(timeline); + + if (unlikely(!mali_timeline_is_point_valid(timeline, point))) { + MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n", point, timeline->point_oldest, timeline->point_next)); + } + + if (!mali_timeline_is_point_released(timeline, point)) { + ret = MALI_FALSE; + goto exit; + } + } + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + if (-1 != fence->sync_fd) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + sync_fence = sync_fence_fdget(fence->sync_fd); +#else + sync_fence = mali_internal_sync_fence_fdget(fence->sync_fd); +#endif + if (likely(NULL != sync_fence)) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + if (0 == sync_fence->status) { +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + if (0 < atomic_read(&sync_fence->status)) { +#else + if (0 == sync_fence->fence->ops->signaled(sync_fence->fence)) { +#endif + ret = MALI_FALSE; + + } else { + ret = MALI_TRUE; + } + } else { + MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", fence->sync_fd)); + } + } +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + +exit: + mali_spinlock_reentrant_signal(system->spinlock, tid); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + if (NULL != sync_fence) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + sync_fence_put(sync_fence); +#else + fput(sync_fence->file); +#endif + } +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + + return ret; +} + +mali_bool mali_timeline_fence_wait(struct mali_timeline_system *system, struct mali_timeline_fence *fence, u32 timeout) +{ + struct mali_timeline_fence_wait_tracker *wait; + mali_timeline_point point; + mali_bool ret; + + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(fence); + + MALI_DEBUG_PRINT(4, ("Mali Timeline: wait on fence\n")); + + if (MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY == timeout) { + return mali_timeline_fence_wait_check_status(system, fence); + } + + wait = mali_timeline_fence_wait_tracker_alloc(); + if (unlikely(NULL == wait)) { + MALI_PRINT_ERROR(("Mali Timeline: failed to allocate data for fence wait\n")); + return MALI_FALSE; + } + + wait->activated = MALI_FALSE; + wait->system = system; + + /* Initialize refcount to two references. The reference first will be released by this + * function after the wait is over. The second reference will be released when the tracker + * is activated. */ + _mali_osk_atomic_init(&wait->refcount, 2); + + /* Add tracker to timeline system, but not to a timeline. */ + mali_timeline_tracker_init(&wait->tracker, MALI_TIMELINE_TRACKER_WAIT, fence, wait); + point = mali_timeline_system_add_tracker(system, &wait->tracker, MALI_TIMELINE_NONE); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point); + MALI_IGNORE(point); + + /* Wait for the tracker to be activated or time out. */ + if (MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER == timeout) { + _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_fence_wait_tracker_is_activated, (void *) wait); + } else { + _mali_osk_wait_queue_wait_event_timeout(system->wait_queue, mali_timeline_fence_wait_tracker_is_activated, (void *) wait, timeout); + } + + ret = wait->activated; + + if (0 == _mali_osk_atomic_dec_return(&wait->refcount)) { + mali_timeline_fence_wait_tracker_free(wait); + } + + return ret; +} + +void mali_timeline_fence_wait_activate(struct mali_timeline_fence_wait_tracker *wait) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(wait); + MALI_DEBUG_ASSERT_POINTER(wait->system); + + MALI_DEBUG_PRINT(4, ("Mali Timeline: activation for fence wait tracker\n")); + + MALI_DEBUG_ASSERT(MALI_FALSE == wait->activated); + wait->activated = MALI_TRUE; + + _mali_osk_wait_queue_wake_up(wait->system->wait_queue); + + /* Nothing can wait on this tracker, so nothing to schedule after release. */ + schedule_mask = mali_timeline_tracker_release(&wait->tracker); + MALI_DEBUG_ASSERT(MALI_SCHEDULER_MASK_EMPTY == schedule_mask); + MALI_IGNORE(schedule_mask); + + if (0 == _mali_osk_atomic_dec_return(&wait->refcount)) { + mali_timeline_fence_wait_tracker_free(wait); + } +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h new file mode 100755 index 000000000000..9da12baeef1a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2013, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_timeline_fence_wait.h + * + * This file contains functions used to wait until a Timeline fence is signaled. + */ + +#ifndef __MALI_TIMELINE_FENCE_WAIT_H__ +#define __MALI_TIMELINE_FENCE_WAIT_H__ + +#include "mali_osk.h" +#include "mali_timeline.h" + +/** + * If used as the timeout argument in @ref mali_timeline_fence_wait, a timer is not used and the + * function only returns when the fence is signaled. + */ +#define MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER ((u32) -1) + +/** + * If used as the timeout argument in @ref mali_timeline_fence_wait, the function will return + * immediately with the current state of the fence. + */ +#define MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY 0 + +/** + * Fence wait tracker. + * + * The fence wait tracker is added to the Timeline system with the fence we are waiting on as a + * dependency. We will then perform a blocking wait, possibly with a timeout, until the tracker is + * activated, which happens when the fence is signaled. + */ +struct mali_timeline_fence_wait_tracker { + mali_bool activated; /**< MALI_TRUE if the tracker has been activated, MALI_FALSE if not. */ + _mali_osk_atomic_t refcount; /**< Reference count. */ + struct mali_timeline_system *system; /**< Timeline system. */ + struct mali_timeline_tracker tracker; /**< Timeline tracker. */ +}; + +/** + * Wait for a fence to be signaled, or timeout is reached. + * + * @param system Timeline system. + * @param fence Fence to wait on. + * @param timeout Timeout in ms, or MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER or + * MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY. + * @return MALI_TRUE if signaled, MALI_FALSE if timed out. + */ +mali_bool mali_timeline_fence_wait(struct mali_timeline_system *system, struct mali_timeline_fence *fence, u32 timeout); + +/** + * Used by the Timeline system to activate a fence wait tracker. + * + * @param fence_wait_tracker Fence waiter tracker. + */ +void mali_timeline_fence_wait_activate(struct mali_timeline_fence_wait_tracker *fence_wait_tracker); + +#endif /* __MALI_TIMELINE_FENCE_WAIT_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c new file mode 100755 index 000000000000..bb7f6a04e8bd --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2013, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include "mali_timeline_sync_fence.h" + +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_sync.h" + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +/** + * Creates a sync fence tracker and a sync fence. Adds sync fence tracker to Timeline system and + * returns sync fence. The sync fence will be signaled when the sync fence tracker is activated. + * + * @param timeline Timeline. + * @param point Point on timeline. + * @return Sync fence that will be signaled when tracker is activated. + */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +static struct sync_fence *mali_timeline_sync_fence_create_and_add_tracker(struct mali_timeline *timeline, mali_timeline_point point) +#else +static struct mali_internal_sync_fence *mali_timeline_sync_fence_create_and_add_tracker(struct mali_timeline *timeline, mali_timeline_point point) +#endif +{ + struct mali_timeline_sync_fence_tracker *sync_fence_tracker; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_fence *sync_fence; +#else + struct mali_internal_sync_fence *sync_fence; +#endif + struct mali_timeline_fence fence; + + MALI_DEBUG_ASSERT_POINTER(timeline); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); + + /* Allocate sync fence tracker. */ + sync_fence_tracker = _mali_osk_calloc(1, sizeof(struct mali_timeline_sync_fence_tracker)); + if (NULL == sync_fence_tracker) { + MALI_PRINT_ERROR(("Mali Timeline: sync_fence_tracker allocation failed\n")); + return NULL; + } + + /* Create sync flag. */ + MALI_DEBUG_ASSERT_POINTER(timeline->sync_tl); + sync_fence_tracker->flag = mali_sync_flag_create(timeline->sync_tl, point); + if (NULL == sync_fence_tracker->flag) { + MALI_PRINT_ERROR(("Mali Timeline: sync_flag creation failed\n")); + _mali_osk_free(sync_fence_tracker); + return NULL; + } + + /* Create sync fence from sync flag. */ + sync_fence = mali_sync_flag_create_fence(sync_fence_tracker->flag); + if (NULL == sync_fence) { + MALI_PRINT_ERROR(("Mali Timeline: sync_fence creation failed\n")); + mali_sync_flag_put(sync_fence_tracker->flag); + _mali_osk_free(sync_fence_tracker); + return NULL; + } + + /* Setup fence for tracker. */ + _mali_osk_memset(&fence, 0, sizeof(struct mali_timeline_fence)); + fence.sync_fd = -1; + fence.points[timeline->id] = point; + + /* Finally, add the tracker to Timeline system. */ + mali_timeline_tracker_init(&sync_fence_tracker->tracker, MALI_TIMELINE_TRACKER_SYNC, &fence, sync_fence_tracker); + point = mali_timeline_system_add_tracker(timeline->system, &sync_fence_tracker->tracker, MALI_TIMELINE_NONE); + MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point); + + return sync_fence; +} + +s32 mali_timeline_sync_fence_create(struct mali_timeline_system *system, struct mali_timeline_fence *fence) +{ + u32 i; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_fence *sync_fence_acc = NULL; +#else + struct mali_internal_sync_fence *sync_fence_acc = NULL; +#endif + MALI_DEBUG_ASSERT_POINTER(system); + MALI_DEBUG_ASSERT_POINTER(fence); + + for (i = 0; i < MALI_TIMELINE_MAX; ++i) { + struct mali_timeline *timeline; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_fence *sync_fence; +#else + struct mali_internal_sync_fence *sync_fence; +#endif + if (MALI_TIMELINE_NO_POINT == fence->points[i]) continue; + + timeline = system->timelines[i]; + MALI_DEBUG_ASSERT_POINTER(timeline); + + sync_fence = mali_timeline_sync_fence_create_and_add_tracker(timeline, fence->points[i]); + if (NULL == sync_fence) goto error; + + if (NULL != sync_fence_acc) { + /* Merge sync fences. */ + sync_fence_acc = mali_sync_fence_merge(sync_fence_acc, sync_fence); + if (NULL == sync_fence_acc) goto error; + } else { + /* This was the first sync fence created. */ + sync_fence_acc = sync_fence; + } + } + + if (-1 != fence->sync_fd) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_fence *sync_fence; + sync_fence = sync_fence_fdget(fence->sync_fd); +#else + struct mali_internal_sync_fence *sync_fence; + sync_fence = mali_internal_sync_fence_fdget(fence->sync_fd); +#endif + + if (NULL == sync_fence) goto error; + + if (NULL != sync_fence_acc) { + sync_fence_acc = mali_sync_fence_merge(sync_fence_acc, sync_fence); + if (NULL == sync_fence_acc) goto error; + } else { + sync_fence_acc = sync_fence; + } + } + + if (NULL == sync_fence_acc) { + MALI_DEBUG_ASSERT_POINTER(system->signaled_sync_tl); + + /* There was nothing to wait on, so return an already signaled fence. */ + + sync_fence_acc = mali_sync_timeline_create_signaled_fence(system->signaled_sync_tl); + if (NULL == sync_fence_acc) goto error; + } + + /* Return file descriptor for the accumulated sync fence. */ + return mali_sync_fence_fd_alloc(sync_fence_acc); + +error: + if (NULL != sync_fence_acc) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + sync_fence_put(sync_fence_acc); +#else + fput(sync_fence_acc->file); +#endif + } + + return -1; +} + +void mali_timeline_sync_fence_activate(struct mali_timeline_sync_fence_tracker *sync_fence_tracker) +{ + mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; + + MALI_DEBUG_ASSERT_POINTER(sync_fence_tracker); + MALI_DEBUG_ASSERT_POINTER(sync_fence_tracker->flag); + + MALI_DEBUG_PRINT(4, ("Mali Timeline: activation for sync fence tracker\n")); + + /* Signal flag and release reference. */ + mali_sync_flag_signal(sync_fence_tracker->flag, 0); + mali_sync_flag_put(sync_fence_tracker->flag); + + /* Nothing can wait on this tracker, so nothing to schedule after release. */ + schedule_mask = mali_timeline_tracker_release(&sync_fence_tracker->tracker); + MALI_DEBUG_ASSERT(MALI_SCHEDULER_MASK_EMPTY == schedule_mask); + + _mali_osk_free(sync_fence_tracker); +} +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h new file mode 100755 index 000000000000..65e368ae7c9e --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2013, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_timeline_sync_fence.h + * + * This file contains code related to creating sync fences from timeline fences. + */ + +#ifndef __MALI_TIMELINE_SYNC_FENCE_H__ +#define __MALI_TIMELINE_SYNC_FENCE_H__ + +#include "mali_timeline.h" + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + +/** + * Sync fence tracker. + */ +struct mali_timeline_sync_fence_tracker { + struct mali_sync_flag *flag; /**< Sync flag used to connect tracker and sync fence. */ + struct mali_timeline_tracker tracker; /**< Timeline tracker. */ +}; + +/** + * Create a sync fence that will be signaled when @ref fence is signaled. + * + * @param system Timeline system. + * @param fence Fence to create sync fence from. + * @return File descriptor for new sync fence, or -1 on error. + */ +s32 mali_timeline_sync_fence_create(struct mali_timeline_system *system, struct mali_timeline_fence *fence); + +/** + * Used by the Timeline system to activate a sync fence tracker. + * + * @param sync_fence_tracker Sync fence tracker. + * + */ +void mali_timeline_sync_fence_activate(struct mali_timeline_sync_fence_tracker *sync_fence_tracker); + +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + +#endif /* __MALI_TIMELINE_SYNC_FENCE_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_ukk.h b/drivers/gpu/arm/mali400/mali/common/mali_ukk.h new file mode 100755 index 000000000000..55a05c50436a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_ukk.h @@ -0,0 +1,551 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_ukk.h + * Defines the kernel-side interface of the user-kernel interface + */ + +#ifndef __MALI_UKK_H__ +#define __MALI_UKK_H__ + +#include "mali_osk.h" +#include "mali_uk_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @addtogroup uddapi Unified Device Driver (UDD) APIs + * + * @{ + */ + +/** + * @addtogroup u_k_api UDD User/Kernel Interface (U/K) APIs + * + * - The _mali_uk functions are an abstraction of the interface to the device + * driver. On certain OSs, this would be implemented via the IOCTL interface. + * On other OSs, it could be via extension of some Device Driver Class, or + * direct function call for Bare metal/RTOSs. + * - It is important to note that: + * - The Device Driver has implemented the _mali_ukk set of functions + * - The Base Driver calls the corresponding set of _mali_uku functions. + * - What requires porting is solely the calling mechanism from User-side to + * Kernel-side, and propagating back the results. + * - Each U/K function is associated with a (group, number) pair from + * \ref _mali_uk_functions to make it possible for a common function in the + * Base Driver and Device Driver to route User/Kernel calls from/to the + * correct _mali_uk function. For example, in an IOCTL system, the IOCTL number + * would be formed based on the group and number assigned to the _mali_uk + * function, as listed in \ref _mali_uk_functions. On the user-side, each + * _mali_uku function would just make an IOCTL with the IOCTL-code being an + * encoded form of the (group, number) pair. On the kernel-side, the Device + * Driver's IOCTL handler decodes the IOCTL-code back into a (group, number) + * pair, and uses this to determine which corresponding _mali_ukk should be + * called. + * - Refer to \ref _mali_uk_functions for more information about this + * (group, number) pairing. + * - In a system where there is no distinction between user and kernel-side, + * the U/K interface may be implemented as:@code + * MALI_STATIC_INLINE _mali_osk_errcode_t _mali_uku_examplefunction( _mali_uk_examplefunction_s *args ) + * { + * return mali_ukk_examplefunction( args ); + * } + * @endcode + * - Therefore, all U/K calls behave \em as \em though they were direct + * function calls (but the \b implementation \em need \em not be a direct + * function calls) + * + * @note Naming the _mali_uk functions the same on both User and Kernel sides + * on non-RTOS systems causes debugging issues when setting breakpoints. In + * this case, it is not clear which function the breakpoint is put on. + * Therefore the _mali_uk functions in user space are prefixed with \c _mali_uku + * and in kernel space with \c _mali_ukk. The naming for the argument + * structures is unaffected. + * + * - The _mali_uk functions are synchronous. + * - Arguments to the _mali_uk functions are passed in a structure. The only + * parameter passed to the _mali_uk functions is a pointer to this structure. + * This first member of this structure, ctx, is a pointer to a context returned + * by _mali_uku_open(). For example:@code + * typedef struct + * { + * void *ctx; + * u32 number_of_cores; + * } _mali_uk_get_gp_number_of_cores_s; + * @endcode + * + * - Each _mali_uk function has its own argument structure named after the + * function. The argument is distinguished by the _s suffix. + * - The argument types are defined by the base driver and user-kernel + * interface. + * - All _mali_uk functions return a standard \ref _mali_osk_errcode_t. + * - Only arguments of type input or input/output need be initialized before + * calling a _mali_uk function. + * - Arguments of type output and input/output are only valid when the + * _mali_uk function returns \ref _MALI_OSK_ERR_OK. + * - The \c ctx member is always invalid after it has been used by a + * _mali_uk function, except for the context management functions + * + * + * \b Interface \b restrictions + * + * The requirements of the interface mean that an implementation of the + * User-kernel interface may do no 'real' work. For example, the following are + * illegal in the User-kernel implementation: + * - Calling functions necessary for operation on all systems, which would + * not otherwise get called on RTOS systems. + * - For example, a U/K interface that calls multiple _mali_ukk functions + * during one particular U/K call. This could not be achieved by the same code + * which uses direct function calls for the U/K interface. + * - Writing in values to the args members, when otherwise these members would + * not hold a useful value for a direct function call U/K interface. + * - For example, U/K interface implementation that take NULL members in + * their arguments structure from the user side, but those members are + * replaced with non-NULL values in the kernel-side of the U/K interface + * implementation. A scratch area for writing data is one such example. In this + * case, a direct function call U/K interface would segfault, because no code + * would be present to replace the NULL pointer with a meaningful pointer. + * - Note that we discourage the case where the U/K implementation changes + * a NULL argument member to non-NULL, and then the Device Driver code (outside + * of the U/K layer) re-checks this member for NULL, and corrects it when + * necessary. Whilst such code works even on direct function call U/K + * intefaces, it reduces the testing coverage of the Device Driver code. This + * is because we have no way of testing the NULL == value path on an OS + * implementation. + * + * A number of allowable examples exist where U/K interfaces do 'real' work: + * - The 'pointer switching' technique for \ref _mali_ukk_get_system_info + * - In this case, without the pointer switching on direct function call + * U/K interface, the Device Driver code still sees the same thing: a pointer + * to which it can write memory. This is because such a system has no + * distinction between a user and kernel pointer. + * - Writing an OS-specific value into the ukk_private member for + * _mali_ukk_mem_mmap(). + * - In this case, this value is passed around by Device Driver code, but + * its actual value is never checked. Device Driver code simply passes it from + * the U/K layer to the OSK layer, where it can be acted upon. In this case, + * \em some OS implementations of the U/K (_mali_ukk_mem_mmap()) and OSK + * (_mali_osk_mem_mapregion_init()) functions will collaborate on the + * meaning of ukk_private member. On other OSs, it may be unused by both + * U/K and OSK layers + * - Therefore, on error inside the U/K interface implementation itself, + * it will be as though the _mali_ukk function itself had failed, and cleaned + * up after itself. + * - Compare this to a direct function call U/K implementation, where all + * error cleanup is handled by the _mali_ukk function itself. The direct + * function call U/K interface implementation is automatically atomic. + * + * The last example highlights a consequence of all U/K interface + * implementations: they must be atomic with respect to the Device Driver code. + * And therefore, should Device Driver code succeed but the U/K implementation + * fail afterwards (but before return to user-space), then the U/K + * implementation must cause appropriate cleanup actions to preserve the + * atomicity of the interface. + * + * @{ + */ + + +/** @defgroup _mali_uk_context U/K Context management + * + * These functions allow for initialisation of the user-kernel interface once per process. + * + * Generally the context will store the OS specific object to communicate with the kernel device driver and further + * state information required by the specific implementation. The context is shareable among all threads in the caller process. + * + * On IOCTL systems, this is likely to be a file descriptor as a result of opening the kernel device driver. + * + * On a bare-metal/RTOS system with no distinction between kernel and + * user-space, the U/K interface simply calls the _mali_ukk variant of the + * function by direct function call. In this case, the context returned is the + * mali_session_data from _mali_ukk_open(). + * + * The kernel side implementations of the U/K interface expect the first member of the argument structure to + * be the context created by _mali_uku_open(). On some OS implementations, the meaning of this context + * will be different between user-side and kernel-side. In which case, the kernel-side will need to replace this context + * with the kernel-side equivalent, because user-side will not have access to kernel-side data. The context parameter + * in the argument structure therefore has to be of type input/output. + * + * It should be noted that the caller cannot reuse the \c ctx member of U/K + * argument structure after a U/K call, because it may be overwritten. Instead, + * the context handle must always be stored elsewhere, and copied into + * the appropriate U/K argument structure for each user-side call to + * the U/K interface. This is not usually a problem, since U/K argument + * structures are usually placed on the stack. + * + * @{ */ + +/** @brief Begin a new Mali Device Driver session + * + * This is used to obtain a per-process context handle for all future U/K calls. + * + * @param context pointer to storage to return a (void*)context handle. + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_open(void **context); + +/** @brief End a Mali Device Driver session + * + * This should be called when the process no longer requires use of the Mali Device Driver. + * + * The context handle must not be used after it has been closed. + * + * @param context pointer to a stored (void*)context handle. + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_close(void **context); + +/** @} */ /* end group _mali_uk_context */ + + +/** @addtogroup _mali_uk_core U/K Core + * + * The core functions provide the following functionality: + * - verify that the user and kernel API are compatible + * - retrieve information about the cores and memory banks in the system + * - wait for the result of jobs started on a core + * + * @{ */ + +/** @brief Waits for a job notification. + * + * Sleeps until notified or a timeout occurs. Returns information about the notification. + * + * @param args see _mali_uk_wait_for_notification_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notification_s *args); + +/** @brief Post a notification to the notification queue of this application. + * + * @param args see _mali_uk_post_notification_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *args); + +/** @brief Verifies if the user and kernel side of this API are compatible. + * + * This function is obsolete, but kept to allow old, incompatible user space + * clients to robustly detect the incompatibility. + * + * @param args see _mali_uk_get_api_version_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_api_version(_mali_uk_get_api_version_s *args); + +/** @brief Verifies if the user and kernel side of this API are compatible. + * + * @param args see _mali_uk_get_api_version_v2_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_api_version_v2(_mali_uk_get_api_version_v2_s *args); + +/** @brief Get the user space settings applicable for calling process. + * + * @param args see _mali_uk_get_user_settings_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_user_settings(_mali_uk_get_user_settings_s *args); + +/** @brief Get a user space setting applicable for calling process. + * + * @param args see _mali_uk_get_user_setting_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_user_setting(_mali_uk_get_user_setting_s *args); + +/* @brief Grant or deny high priority scheduling for this session. + * + * @param args see _mali_uk_request_high_priority_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priority_s *args); + +/** @brief Make process sleep if the pending big job in kernel >= MALI_MAX_PENDING_BIG_JOB + * + */ +_mali_osk_errcode_t _mali_ukk_pending_submit(_mali_uk_pending_submit_s *args); + +/** @} */ /* end group _mali_uk_core */ + + +/** @addtogroup _mali_uk_memory U/K Memory + * + * The memory functions provide functionality with and without a Mali-MMU present. + * + * For Mali-MMU based systems, the following functionality is provided: + * - Initialize and terminate MALI virtual address space + * - Allocate/deallocate physical memory to a MALI virtual address range and map into/unmap from the + * current process address space + * - Map/unmap external physical memory into the MALI virtual address range + * + * For Mali-nonMMU based systems: + * - Allocate/deallocate MALI memory + * + * @{ */ + +/** @brief Map Mali Memory into the current user process + * + * Maps Mali memory into the current user process in a generic way. + * + * This function is to be used for Mali-MMU mode. The function is available in both Mali-MMU and Mali-nonMMU modes, + * but should not be called by a user process in Mali-nonMMU mode. + * + * The implementation and operation of _mali_ukk_mem_mmap() is dependant on whether the driver is built for Mali-MMU + * or Mali-nonMMU: + * - In the nonMMU case, _mali_ukk_mem_mmap() requires a physical address to be specified. For this reason, an OS U/K + * implementation should not allow this to be called from user-space. In any case, nonMMU implementations are + * inherently insecure, and so the overall impact is minimal. Mali-MMU mode should be used if security is desired. + * - In the MMU case, _mali_ukk_mem_mmap() the _mali_uk_mem_mmap_s::phys_addr + * member is used for the \em Mali-virtual address desired for the mapping. The + * implementation of _mali_ukk_mem_mmap() will allocate both the CPU-virtual + * and CPU-physical addresses, and can cope with mapping a contiguous virtual + * address range to a sequence of non-contiguous physical pages. In this case, + * the CPU-physical addresses are not communicated back to the user-side, as + * they are unnecsessary; the \em Mali-virtual address range must be used for + * programming Mali structures. + * + * In the second (MMU) case, _mali_ukk_mem_mmap() handles management of + * CPU-virtual and CPU-physical ranges, but the \em caller must manage the + * \em Mali-virtual address range from the user-side. + * + * @note Mali-virtual address ranges are entirely separate between processes. + * It is not possible for a process to accidentally corrupt another process' + * \em Mali-virtual address space. + * + * @param args see _mali_uk_mem_mmap_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_mem_mmap(_mali_uk_mem_mmap_s *args); + +/** @brief Unmap Mali Memory from the current user process + * + * Unmaps Mali memory from the current user process in a generic way. This only operates on Mali memory supplied + * from _mali_ukk_mem_mmap(). + * + * @param args see _mali_uk_mem_munmap_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_mem_munmap(_mali_uk_mem_munmap_s *args); + +/** @brief Determine the buffer size necessary for an MMU page table dump. + * @param args see _mali_uk_query_mmu_page_table_dump_size_s in mali_utgard_uk_types.h + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args); +/** @brief Dump MMU Page tables. + * @param args see _mali_uk_dump_mmu_page_table_s in mali_utgard_uk_types.h + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args); + +/** @brief Write user data to specified Mali memory without causing segfaults. + * @param args see _mali_uk_mem_write_safe_s in mali_utgard_uk_types.h + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_mem_write_safe(_mali_uk_mem_write_safe_s *args); + +/** @} */ /* end group _mali_uk_memory */ + + +/** @addtogroup _mali_uk_pp U/K Fragment Processor + * + * The Fragment Processor (aka PP (Pixel Processor)) functions provide the following functionality: + * - retrieving version of the fragment processors + * - determine number of fragment processors + * - starting a job on a fragment processor + * + * @{ */ + +/** @brief Issue a request to start a new job on a Fragment Processor. + * + * If the request fails args->status is set to _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE and you can + * try to start the job again. + * + * An existing job could be returned for requeueing if the new job has a higher priority than a previously started job + * which the hardware hasn't actually started processing yet. In this case the new job will be started instead and the + * existing one returned, otherwise the new job is started and the status field args->status is set to + * _MALI_UK_START_JOB_STARTED. + * + * Job completion can be awaited with _mali_ukk_wait_for_notification(). + * + * @param ctx user-kernel context (mali_session) + * @param uargs see _mali_uk_pp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx, _mali_uk_pp_start_job_s *uargs); + +/** + * @brief Issue a request to start new jobs on both Vertex Processor and Fragment Processor. + * + * @note Will call into @ref _mali_ukk_pp_start_job and @ref _mali_ukk_gp_start_job. + * + * @param ctx user-kernel context (mali_session) + * @param uargs see _mali_uk_pp_and_gp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx, _mali_uk_pp_and_gp_start_job_s *uargs); + +/** @brief Returns the number of Fragment Processors in the system + * + * @param args see _mali_uk_get_pp_number_of_cores_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args); + +/** @brief Returns the version that all Fragment Processor cores are compatible with. + * + * This function may only be called when _mali_ukk_get_pp_number_of_cores() indicated at least one Fragment + * Processor core is available. + * + * @param args see _mali_uk_get_pp_core_version_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args); + +/** @brief Disable Write-back unit(s) on specified job + * + * @param args see _mali_uk_get_pp_core_version_s in "mali_utgard_uk_types.h" + */ +void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args); + + +/** @} */ /* end group _mali_uk_pp */ + + +/** @addtogroup _mali_uk_gp U/K Vertex Processor + * + * The Vertex Processor (aka GP (Geometry Processor)) functions provide the following functionality: + * - retrieving version of the Vertex Processors + * - determine number of Vertex Processors available + * - starting a job on a Vertex Processor + * + * @{ */ + +/** @brief Issue a request to start a new job on a Vertex Processor. + * + * If the request fails args->status is set to _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE and you can + * try to start the job again. + * + * An existing job could be returned for requeueing if the new job has a higher priority than a previously started job + * which the hardware hasn't actually started processing yet. In this case the new job will be started and the + * existing one returned, otherwise the new job is started and the status field args->status is set to + * _MALI_UK_START_JOB_STARTED. + * + * Job completion can be awaited with _mali_ukk_wait_for_notification(). + * + * @param ctx user-kernel context (mali_session) + * @param uargs see _mali_uk_gp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx, _mali_uk_gp_start_job_s *uargs); + +/** @brief Returns the number of Vertex Processors in the system. + * + * @param args see _mali_uk_get_gp_number_of_cores_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args); + +/** @brief Returns the version that all Vertex Processor cores are compatible with. + * + * This function may only be called when _mali_uk_get_gp_number_of_cores() indicated at least one Vertex + * Processor core is available. + * + * @param args see _mali_uk_get_gp_core_version_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args); + +/** @brief Resume or abort suspended Vertex Processor jobs. + * + * After receiving notification that a Vertex Processor job was suspended from + * _mali_ukk_wait_for_notification() you can use this function to resume or abort the job. + * + * @param args see _mali_uk_gp_suspend_response_s in "mali_utgard_uk_types.h" + * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. + */ +_mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args); + +/** @} */ /* end group _mali_uk_gp */ + +#if defined(CONFIG_MALI400_PROFILING) +/** @addtogroup _mali_uk_profiling U/K Timeline profiling module + * @{ */ + +/** @brief Add event to profiling buffer. + * + * @param args see _mali_uk_profiling_add_event_s in "mali_utgard_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args); + +/** @brief Get profiling stream fd. + * + * @param args see _mali_uk_profiling_stream_fd_get_s in "mali_utgard_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args); + +/** @brief Profiling control set. + * + * @param args see _mali_uk_profiling_control_set_s in "mali_utgard_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args); + +/** @} */ /* end group _mali_uk_profiling */ +#endif + +/** @addtogroup _mali_uk_vsync U/K VSYNC reporting module + * @{ */ + +/** @brief Report events related to vsync. + * + * @note Events should be reported when starting to wait for vsync and when the + * waiting is finished. This information can then be used in kernel space to + * complement the GPU utilization metric. + * + * @param args see _mali_uk_vsync_event_report_s in "mali_utgard_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_vsync_event_report(_mali_uk_vsync_event_report_s *args); + +/** @} */ /* end group _mali_uk_vsync */ + +/** @addtogroup _mali_sw_counters_report U/K Software counter reporting + * @{ */ + +/** @brief Report software counters. + * + * @param args see _mali_uk_sw_counters_report_s in "mali_uk_types.h" + */ +_mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s *args); + +/** @} */ /* end group _mali_sw_counters_report */ + +/** @} */ /* end group u_k_api */ + +/** @} */ /* end group uddapi */ + +u32 _mali_ukk_report_memory_usage(void); + +u32 _mali_ukk_report_total_memory_size(void); + +u32 _mali_ukk_utilization_gp_pp(void); + +u32 _mali_ukk_utilization_gp(void); + +u32 _mali_ukk_utilization_pp(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UKK_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c new file mode 100755 index 000000000000..1911eff87a72 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c @@ -0,0 +1,147 @@ +/** + * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" +#include "mali_uk_types.h" +#include "mali_user_settings_db.h" +#include "mali_session.h" + +static u32 mali_user_settings[_MALI_UK_USER_SETTING_MAX]; +const char *_mali_uk_user_setting_descriptions[] = _MALI_UK_USER_SETTING_DESCRIPTIONS; + +static void mali_user_settings_notify(_mali_uk_user_setting_t setting, u32 value) +{ + mali_bool done = MALI_FALSE; + + /* + * This function gets a bit complicated because we can't hold the session lock while + * allocating notification objects. + */ + + while (!done) { + u32 i; + u32 num_sessions_alloc; + u32 num_sessions_with_lock; + u32 used_notification_objects = 0; + _mali_osk_notification_t **notobjs; + + /* Pre allocate the number of notifications objects we need right now (might change after lock has been taken) */ + num_sessions_alloc = mali_session_get_count(); + if (0 == num_sessions_alloc) { + /* No sessions to report to */ + return; + } + + notobjs = (_mali_osk_notification_t **)_mali_osk_malloc(sizeof(_mali_osk_notification_t *) * num_sessions_alloc); + if (NULL == notobjs) { + MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure)\n")); + return; + } + + for (i = 0; i < num_sessions_alloc; i++) { + notobjs[i] = _mali_osk_notification_create(_MALI_NOTIFICATION_SETTINGS_CHANGED, + sizeof(_mali_uk_settings_changed_s)); + if (NULL != notobjs[i]) { + _mali_uk_settings_changed_s *data; + data = notobjs[i]->result_buffer; + + data->setting = setting; + data->value = value; + } else { + MALI_PRINT_ERROR(("Failed to notify user space session about setting change (alloc failure %u)\n", i)); + } + } + + mali_session_lock(); + + /* number of sessions will not change while we hold the lock */ + num_sessions_with_lock = mali_session_get_count(); + + if (num_sessions_alloc >= num_sessions_with_lock) { + /* We have allocated enough notification objects for all the sessions atm */ + struct mali_session_data *session, *tmp; + MALI_SESSION_FOREACH(session, tmp, link) { + MALI_DEBUG_ASSERT(used_notification_objects < num_sessions_alloc); + if (NULL != notobjs[used_notification_objects]) { + mali_session_send_notification(session, notobjs[used_notification_objects]); + notobjs[used_notification_objects] = NULL; /* Don't track this notification object any more */ + } + used_notification_objects++; + } + done = MALI_TRUE; + } + + mali_session_unlock(); + + /* Delete any remaining/unused notification objects */ + for (; used_notification_objects < num_sessions_alloc; used_notification_objects++) { + if (NULL != notobjs[used_notification_objects]) { + _mali_osk_notification_delete(notobjs[used_notification_objects]); + } + } + + _mali_osk_free(notobjs); + } +} + +void mali_set_user_setting(_mali_uk_user_setting_t setting, u32 value) +{ + mali_bool notify = MALI_FALSE; + + if (setting >= _MALI_UK_USER_SETTING_MAX) { + MALI_DEBUG_PRINT_ERROR(("Invalid user setting %ud\n")); + return; + } + + if (mali_user_settings[setting] != value) { + notify = MALI_TRUE; + } + + mali_user_settings[setting] = value; + + if (notify) { + mali_user_settings_notify(setting, value); + } +} + +u32 mali_get_user_setting(_mali_uk_user_setting_t setting) +{ + if (setting >= _MALI_UK_USER_SETTING_MAX) { + return 0; + } + + return mali_user_settings[setting]; +} + +_mali_osk_errcode_t _mali_ukk_get_user_setting(_mali_uk_get_user_setting_s *args) +{ + _mali_uk_user_setting_t setting; + MALI_DEBUG_ASSERT_POINTER(args); + + setting = args->setting; + + if (_MALI_UK_USER_SETTING_MAX > setting) { + args->value = mali_user_settings[setting]; + return _MALI_OSK_ERR_OK; + } else { + return _MALI_OSK_ERR_INVALID_ARGS; + } +} + +_mali_osk_errcode_t _mali_ukk_get_user_settings(_mali_uk_get_user_settings_s *args) +{ + MALI_DEBUG_ASSERT_POINTER(args); + + _mali_osk_memcpy(args->settings, mali_user_settings, sizeof(mali_user_settings)); + + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h new file mode 100755 index 000000000000..da9c0630e371 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h @@ -0,0 +1,39 @@ +/** + * Copyright (C) 2012-2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_USER_SETTINGS_DB_H__ +#define __MALI_USER_SETTINGS_DB_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "mali_uk_types.h" + +/** @brief Set Mali user setting in DB + * + * Update the DB with a new value for \a setting. If the value is different from theprevious set value running sessions will be notified of the change. + * + * @param setting the setting to be changed + * @param value the new value to set + */ +void mali_set_user_setting(_mali_uk_user_setting_t setting, u32 value); + +/** @brief Get current Mali user setting value from DB + * + * @param setting the setting to extract + * @return the value of the selected setting + */ +u32 mali_get_user_setting(_mali_uk_user_setting_t setting); + +#ifdef __cplusplus +} +#endif +#endif /* __MALI_KERNEL_USER_SETTING__ */ diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h new file mode 100755 index 000000000000..7df55c951d6f --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h @@ -0,0 +1,526 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_utgard.h + * Defines types and interface exposed by the Mali Utgard device driver + */ + +#ifndef __MALI_UTGARD_H__ +#define __MALI_UTGARD_H__ + +#include "mali_osk_types.h" +#ifdef CONFIG_MALI_DEVFREQ +#include +#include "mali_pm_metrics.h" +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif +#endif + +#define MALI_GPU_NAME_UTGARD "mali-utgard" + + +#define MALI_OFFSET_GP 0x00000 +#define MALI_OFFSET_GP_MMU 0x03000 + +#define MALI_OFFSET_PP0 0x08000 +#define MALI_OFFSET_PP0_MMU 0x04000 +#define MALI_OFFSET_PP1 0x0A000 +#define MALI_OFFSET_PP1_MMU 0x05000 +#define MALI_OFFSET_PP2 0x0C000 +#define MALI_OFFSET_PP2_MMU 0x06000 +#define MALI_OFFSET_PP3 0x0E000 +#define MALI_OFFSET_PP3_MMU 0x07000 + +#define MALI_OFFSET_PP4 0x28000 +#define MALI_OFFSET_PP4_MMU 0x1C000 +#define MALI_OFFSET_PP5 0x2A000 +#define MALI_OFFSET_PP5_MMU 0x1D000 +#define MALI_OFFSET_PP6 0x2C000 +#define MALI_OFFSET_PP6_MMU 0x1E000 +#define MALI_OFFSET_PP7 0x2E000 +#define MALI_OFFSET_PP7_MMU 0x1F000 + +#define MALI_OFFSET_L2_RESOURCE0 0x01000 +#define MALI_OFFSET_L2_RESOURCE1 0x10000 +#define MALI_OFFSET_L2_RESOURCE2 0x11000 + +#define MALI400_OFFSET_L2_CACHE0 MALI_OFFSET_L2_RESOURCE0 +#define MALI450_OFFSET_L2_CACHE0 MALI_OFFSET_L2_RESOURCE1 +#define MALI450_OFFSET_L2_CACHE1 MALI_OFFSET_L2_RESOURCE0 +#define MALI450_OFFSET_L2_CACHE2 MALI_OFFSET_L2_RESOURCE2 +#define MALI470_OFFSET_L2_CACHE1 MALI_OFFSET_L2_RESOURCE0 + +#define MALI_OFFSET_BCAST 0x13000 +#define MALI_OFFSET_DLBU 0x14000 + +#define MALI_OFFSET_PP_BCAST 0x16000 +#define MALI_OFFSET_PP_BCAST_MMU 0x15000 + +#define MALI_OFFSET_PMU 0x02000 +#define MALI_OFFSET_DMA 0x12000 + +/* Mali-300 */ + +#define MALI_GPU_RESOURCES_MALI300(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) + +#define MALI_GPU_RESOURCES_MALI300_PMU(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) + +/* Mali-400 */ + +#define MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) + +#define MALI_GPU_RESOURCES_MALI400_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) + +#define MALI_GPU_RESOURCES_MALI400_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) + +#define MALI_GPU_RESOURCES_MALI400_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) + +#define MALI_GPU_RESOURCES_MALI400_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) + +#define MALI_GPU_RESOURCES_MALI400_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) + +#define MALI_GPU_RESOURCES_MALI400_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) + +#define MALI_GPU_RESOURCES_MALI400_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ + MALI_GPU_RESOURCES_MALI400_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + + /* Mali-450 */ +#define MALI_GPU_RESOURCES_MALI450_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ + MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) + +#define MALI_GPU_RESOURCES_MALI450_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI450_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI450_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI450_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI450_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI450_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ + MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) + +#define MALI_GPU_RESOURCES_MALI450_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI450_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI450_MP6(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE2) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP4, pp3_irq, base_addr + MALI_OFFSET_PP4_MMU, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(4, base_addr + MALI_OFFSET_PP5, pp4_irq, base_addr + MALI_OFFSET_PP5_MMU, pp4_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(5, base_addr + MALI_OFFSET_PP6, pp5_irq, base_addr + MALI_OFFSET_PP6_MMU, pp5_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ + MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) + +#define MALI_GPU_RESOURCES_MALI450_MP6_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI450_MP6(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI450_MP8(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE2) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(4, base_addr + MALI_OFFSET_PP4, pp4_irq, base_addr + MALI_OFFSET_PP4_MMU, pp4_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(5, base_addr + MALI_OFFSET_PP5, pp5_irq, base_addr + MALI_OFFSET_PP5_MMU, pp5_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(6, base_addr + MALI_OFFSET_PP6, pp6_irq, base_addr + MALI_OFFSET_PP6_MMU, pp6_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(7, base_addr + MALI_OFFSET_PP7, pp7_irq, base_addr + MALI_OFFSET_PP7_MMU, pp7_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ + MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) + +#define MALI_GPU_RESOURCES_MALI450_MP8_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI450_MP8(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + + /* Mali - 470 */ +#define MALI_GPU_RESOURCES_MALI470_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI470_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI470_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCES_MALI470_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ + MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ + MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ + MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ + MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ + MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) + +#define MALI_GPU_RESOURCES_MALI470_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCES_MALI470_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ + MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ + +#define MALI_GPU_RESOURCE_L2(addr) \ + { \ + .name = "Mali_L2", \ + .flags = IORESOURCE_MEM, \ + .start = addr, \ + .end = addr + 0x200, \ + }, + +#define MALI_GPU_RESOURCE_GP(gp_addr, gp_irq) \ + { \ + .name = "Mali_GP", \ + .flags = IORESOURCE_MEM, \ + .start = gp_addr, \ + .end = gp_addr + 0x100, \ + }, \ + { \ + .name = "Mali_GP_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = gp_irq, \ + .end = gp_irq, \ + }, \ + +#define MALI_GPU_RESOURCE_GP_WITH_MMU(gp_addr, gp_irq, gp_mmu_addr, gp_mmu_irq) \ + { \ + .name = "Mali_GP", \ + .flags = IORESOURCE_MEM, \ + .start = gp_addr, \ + .end = gp_addr + 0x100, \ + }, \ + { \ + .name = "Mali_GP_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = gp_irq, \ + .end = gp_irq, \ + }, \ + { \ + .name = "Mali_GP_MMU", \ + .flags = IORESOURCE_MEM, \ + .start = gp_mmu_addr, \ + .end = gp_mmu_addr + 0x100, \ + }, \ + { \ + .name = "Mali_GP_MMU_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = gp_mmu_irq, \ + .end = gp_mmu_irq, \ + }, + +#define MALI_GPU_RESOURCE_PP(pp_addr, pp_irq) \ + { \ + .name = "Mali_PP", \ + .flags = IORESOURCE_MEM, \ + .start = pp_addr, \ + .end = pp_addr + 0x1100, \ + }, \ + { \ + .name = "Mali_PP_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = pp_irq, \ + .end = pp_irq, \ + }, \ + +#define MALI_GPU_RESOURCE_PP_WITH_MMU(id, pp_addr, pp_irq, pp_mmu_addr, pp_mmu_irq) \ + { \ + .name = "Mali_PP" #id, \ + .flags = IORESOURCE_MEM, \ + .start = pp_addr, \ + .end = pp_addr + 0x1100, \ + }, \ + { \ + .name = "Mali_PP" #id "_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = pp_irq, \ + .end = pp_irq, \ + }, \ + { \ + .name = "Mali_PP" #id "_MMU", \ + .flags = IORESOURCE_MEM, \ + .start = pp_mmu_addr, \ + .end = pp_mmu_addr + 0x100, \ + }, \ + { \ + .name = "Mali_PP" #id "_MMU_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = pp_mmu_irq, \ + .end = pp_mmu_irq, \ + }, + +#define MALI_GPU_RESOURCE_MMU(mmu_addr, mmu_irq) \ + { \ + .name = "Mali_MMU", \ + .flags = IORESOURCE_MEM, \ + .start = mmu_addr, \ + .end = mmu_addr + 0x100, \ + }, \ + { \ + .name = "Mali_MMU_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = mmu_irq, \ + .end = mmu_irq, \ + }, + +#define MALI_GPU_RESOURCE_PMU(pmu_addr) \ + { \ + .name = "Mali_PMU", \ + .flags = IORESOURCE_MEM, \ + .start = pmu_addr, \ + .end = pmu_addr + 0x100, \ + }, + +#define MALI_GPU_RESOURCE_DMA(dma_addr) \ + { \ + .name = "Mali_DMA", \ + .flags = IORESOURCE_MEM, \ + .start = dma_addr, \ + .end = dma_addr + 0x100, \ + }, + +#define MALI_GPU_RESOURCE_DLBU(dlbu_addr) \ + { \ + .name = "Mali_DLBU", \ + .flags = IORESOURCE_MEM, \ + .start = dlbu_addr, \ + .end = dlbu_addr + 0x100, \ + }, + +#define MALI_GPU_RESOURCE_BCAST(bcast_addr) \ + { \ + .name = "Mali_Broadcast", \ + .flags = IORESOURCE_MEM, \ + .start = bcast_addr, \ + .end = bcast_addr + 0x100, \ + }, + +#define MALI_GPU_RESOURCE_PP_BCAST(pp_addr, pp_irq) \ + { \ + .name = "Mali_PP_Broadcast", \ + .flags = IORESOURCE_MEM, \ + .start = pp_addr, \ + .end = pp_addr + 0x1100, \ + }, \ + { \ + .name = "Mali_PP_Broadcast_IRQ", \ + .flags = IORESOURCE_IRQ, \ + .start = pp_irq, \ + .end = pp_irq, \ + }, \ + +#define MALI_GPU_RESOURCE_PP_MMU_BCAST(pp_mmu_bcast_addr) \ + { \ + .name = "Mali_PP_MMU_Broadcast", \ + .flags = IORESOURCE_MEM, \ + .start = pp_mmu_bcast_addr, \ + .end = pp_mmu_bcast_addr + 0x100, \ + }, + + struct mali_gpu_utilization_data { + unsigned int utilization_gpu; /* Utilization for GP and all PP cores combined, 0 = no utilization, 256 = full utilization */ + unsigned int utilization_gp; /* Utilization for GP core only, 0 = no utilization, 256 = full utilization */ + unsigned int utilization_pp; /* Utilization for all PP cores combined, 0 = no utilization, 256 = full utilization */ + }; + + struct mali_gpu_clk_item { + unsigned int clock; /* unit(MHz) */ + unsigned int vol; + }; + + struct mali_gpu_clock { + struct mali_gpu_clk_item *item; + unsigned int num_of_steps; + }; + + struct mali_gpu_device_data { + /* Shared GPU memory */ + unsigned long shared_mem_size; + + /* + * Mali PMU switch delay. + * Only needed if the power gates are connected to the PMU in a high fanout + * network. This value is the number of Mali clock cycles it takes to + * enable the power gates and turn on the power mesh. + * This value will have no effect if a daisy chain implementation is used. + */ + u32 pmu_switch_delay; + + /* Mali Dynamic power domain configuration in sequence from 0-11 + * GP PP0 PP1 PP2 PP3 PP4 PP5 PP6 PP7, L2$0 L2$1 L2$2 + */ + u16 pmu_domain_config[12]; + + /* Dedicated GPU memory range (physical). */ + unsigned long dedicated_mem_start; + unsigned long dedicated_mem_size; + + /* Frame buffer memory to be accessible by Mali GPU (physical) */ + unsigned long fb_start; + unsigned long fb_size; + + /* Max runtime [ms] for jobs */ + int max_job_runtime; + + /* Report GPU utilization and related control in this interval (specified in ms) */ + unsigned long control_interval; + + /* Function that will receive periodic GPU utilization numbers */ + void (*utilization_callback)(struct mali_gpu_utilization_data *data); + + /* Fuction that platform callback for freq setting, needed when CONFIG_MALI_DVFS enabled */ + int (*set_freq)(int setting_clock_step); + /* Function that platfrom report it's clock info which driver can set, needed when CONFIG_MALI_DVFS enabled */ + void (*get_clock_info)(struct mali_gpu_clock **data); + /* Function that get the current clock info, needed when CONFIG_MALI_DVFS enabled */ + int (*get_freq)(void); + /* Function that init the mali gpu secure mode */ + int (*secure_mode_init)(void); + /* Function that deinit the mali gpu secure mode */ + void (*secure_mode_deinit)(void); + /* Function that reset GPU and enable gpu secure mode */ + int (*gpu_reset_and_secure_mode_enable)(void); + /* Function that Reset GPU and disable gpu secure mode */ + int (*gpu_reset_and_secure_mode_disable)(void); + /* ipa related interface customer need register */ +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + struct devfreq_cooling_power *gpu_cooling_ops; +#endif + }; + + /** + * Pause the scheduling and power state changes of Mali device driver. + * mali_dev_resume() must always be called as soon as possible after this function + * in order to resume normal operation of the Mali driver. + */ + void mali_dev_pause(void); + + /** + * Resume scheduling and allow power changes in Mali device driver. + * This must always be called after mali_dev_pause(). + */ + void mali_dev_resume(void); + + /** @brief Set the desired number of PP cores to use. + * + * The internal Mali PMU will be used, if present, to physically power off the PP cores. + * + * @param num_cores The number of desired cores + * @return 0 on success, otherwise error. -EINVAL means an invalid number of cores was specified. + */ + int mali_perf_set_num_pp_cores(unsigned int num_cores); + +#endif diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h new file mode 100755 index 000000000000..686708eaef75 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_UTGARD_IOCTL_H__ +#define __MALI_UTGARD_IOCTL_H__ + +#include +#include +#include /* file system operations */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file mali_kernel_ioctl.h + * Interface to the Linux device driver. + * This file describes the interface needed to use the Linux device driver. + * Its interface is designed to used by the HAL implementation through a thin arch layer. + */ + +/** + * ioctl commands + */ + +#define MALI_IOC_BASE 0x82 +#define MALI_IOC_CORE_BASE (_MALI_UK_CORE_SUBSYSTEM + MALI_IOC_BASE) +#define MALI_IOC_MEMORY_BASE (_MALI_UK_MEMORY_SUBSYSTEM + MALI_IOC_BASE) +#define MALI_IOC_PP_BASE (_MALI_UK_PP_SUBSYSTEM + MALI_IOC_BASE) +#define MALI_IOC_GP_BASE (_MALI_UK_GP_SUBSYSTEM + MALI_IOC_BASE) +#define MALI_IOC_PROFILING_BASE (_MALI_UK_PROFILING_SUBSYSTEM + MALI_IOC_BASE) +#define MALI_IOC_VSYNC_BASE (_MALI_UK_VSYNC_SUBSYSTEM + MALI_IOC_BASE) + +#define MALI_IOC_WAIT_FOR_NOTIFICATION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_WAIT_FOR_NOTIFICATION, _mali_uk_wait_for_notification_s) +#define MALI_IOC_GET_API_VERSION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_API_VERSION, u32) +#define MALI_IOC_GET_API_VERSION_V2 _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_API_VERSION, _mali_uk_get_api_version_v2_s) +/* rk_ext. */ +#define MALI_IOC_GET_RK_KO_VERSION _IOWR(MALI_IOC_CORE_BASE, _MALI_GET_RK_KO_VERSION, _mali_rk_ko_version_s) +#define MALI_IOC_POST_NOTIFICATION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_POST_NOTIFICATION, _mali_uk_post_notification_s) +#define MALI_IOC_GET_USER_SETTING _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_USER_SETTING, _mali_uk_get_user_setting_s) +#define MALI_IOC_GET_USER_SETTINGS _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_USER_SETTINGS, _mali_uk_get_user_settings_s) +#define MALI_IOC_REQUEST_HIGH_PRIORITY _IOW (MALI_IOC_CORE_BASE, _MALI_UK_REQUEST_HIGH_PRIORITY, _mali_uk_request_high_priority_s) +#define MALI_IOC_TIMELINE_GET_LATEST_POINT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_GET_LATEST_POINT, _mali_uk_timeline_get_latest_point_s) +#define MALI_IOC_TIMELINE_WAIT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_WAIT, _mali_uk_timeline_wait_s) +#define MALI_IOC_TIMELINE_CREATE_SYNC_FENCE _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_CREATE_SYNC_FENCE, _mali_uk_timeline_create_sync_fence_s) +#define MALI_IOC_SOFT_JOB_START _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_SOFT_JOB_START, _mali_uk_soft_job_start_s) +#define MALI_IOC_SOFT_JOB_SIGNAL _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_SOFT_JOB_SIGNAL, _mali_uk_soft_job_signal_s) +#define MALI_IOC_PENDING_SUBMIT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_PENDING_SUBMIT, _mali_uk_pending_submit_s) + +#define MALI_IOC_MEM_ALLOC _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_ALLOC_MEM, _mali_uk_alloc_mem_s) +#define MALI_IOC_MEM_FREE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_FREE_MEM, _mali_uk_free_mem_s) +#define MALI_IOC_MEM_BIND _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_BIND_MEM, _mali_uk_bind_mem_s) +#define MALI_IOC_MEM_UNBIND _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_UNBIND_MEM, _mali_uk_unbind_mem_s) +#define MALI_IOC_MEM_COW _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_COW_MEM, _mali_uk_cow_mem_s) +#define MALI_IOC_MEM_COW_MODIFY_RANGE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_COW_MODIFY_RANGE, _mali_uk_cow_modify_range_s) +#define MALI_IOC_MEM_RESIZE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_RESIZE_MEM, _mali_uk_mem_resize_s) +#define MALI_IOC_MEM_DMA_BUF_GET_SIZE _IOR(MALI_IOC_MEMORY_BASE, _MALI_UK_DMA_BUF_GET_SIZE, _mali_uk_dma_buf_get_size_s) +#define MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE _IOR (MALI_IOC_MEMORY_BASE, _MALI_UK_QUERY_MMU_PAGE_TABLE_DUMP_SIZE, _mali_uk_query_mmu_page_table_dump_size_s) +#define MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_DUMP_MMU_PAGE_TABLE, _mali_uk_dump_mmu_page_table_s) +#define MALI_IOC_MEM_WRITE_SAFE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_MEM_WRITE_SAFE, _mali_uk_mem_write_safe_s) + +#define MALI_IOC_PP_START_JOB _IOWR(MALI_IOC_PP_BASE, _MALI_UK_PP_START_JOB, _mali_uk_pp_start_job_s) +#define MALI_IOC_PP_AND_GP_START_JOB _IOWR(MALI_IOC_PP_BASE, _MALI_UK_PP_AND_GP_START_JOB, _mali_uk_pp_and_gp_start_job_s) +#define MALI_IOC_PP_NUMBER_OF_CORES_GET _IOR (MALI_IOC_PP_BASE, _MALI_UK_GET_PP_NUMBER_OF_CORES, _mali_uk_get_pp_number_of_cores_s) +#define MALI_IOC_PP_CORE_VERSION_GET _IOR (MALI_IOC_PP_BASE, _MALI_UK_GET_PP_CORE_VERSION, _mali_uk_get_pp_core_version_s) +#define MALI_IOC_PP_DISABLE_WB _IOW (MALI_IOC_PP_BASE, _MALI_UK_PP_DISABLE_WB, _mali_uk_pp_disable_wb_s) + +#define MALI_IOC_GP2_START_JOB _IOWR(MALI_IOC_GP_BASE, _MALI_UK_GP_START_JOB, _mali_uk_gp_start_job_s) +#define MALI_IOC_GP2_NUMBER_OF_CORES_GET _IOR (MALI_IOC_GP_BASE, _MALI_UK_GET_GP_NUMBER_OF_CORES, _mali_uk_get_gp_number_of_cores_s) +#define MALI_IOC_GP2_CORE_VERSION_GET _IOR (MALI_IOC_GP_BASE, _MALI_UK_GET_GP_CORE_VERSION, _mali_uk_get_gp_core_version_s) +#define MALI_IOC_GP2_SUSPEND_RESPONSE _IOW (MALI_IOC_GP_BASE, _MALI_UK_GP_SUSPEND_RESPONSE,_mali_uk_gp_suspend_response_s) + +#define MALI_IOC_PROFILING_ADD_EVENT _IOWR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_ADD_EVENT, _mali_uk_profiling_add_event_s) +#define MALI_IOC_PROFILING_REPORT_SW_COUNTERS _IOW (MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_REPORT_SW_COUNTERS, _mali_uk_sw_counters_report_s) +#define MALI_IOC_PROFILING_MEMORY_USAGE_GET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_MEMORY_USAGE_GET, _mali_uk_profiling_memory_usage_get_s) +#define MALI_IOC_PROFILING_STREAM_FD_GET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_STREAM_FD_GET, _mali_uk_profiling_stream_fd_get_s) +#define MALI_IOC_PROILING_CONTROL_SET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_CONTROL_SET, _mali_uk_profiling_control_set_s) + +#define MALI_IOC_VSYNC_EVENT_REPORT _IOW (MALI_IOC_VSYNC_BASE, _MALI_UK_VSYNC_EVENT_REPORT, _mali_uk_vsync_event_report_s) + +/* rk_ext : 对 r5p0 集æˆä¹‹åŽ, mali_so ä¸å†ä½¿ç”¨ä¸‹é¢çš„ ioctl, 而使用 MALI_IOC_GET_RK_KO_VERSION. */ +#if 0 +#define MALI_IOC_GET_MALI_VERSION_IN_RK30 _IOWR(MALI_IOC_CORE_BASE,_MALI_UK_GET_MALI_VERSION_IN_RK30,_mali_uk_get_mali_version_in_rk30_s *) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UTGARD_IOCTL_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h new file mode 100755 index 000000000000..17d31de931d0 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef _MALI_UTGARD_PROFILING_EVENTS_H_ +#define _MALI_UTGARD_PROFILING_EVENTS_H_ + +/* + * The event ID is a 32 bit value consisting of different fields + * reserved, 4 bits, for future use + * event type, 4 bits, cinstr_profiling_event_type_t + * event channel, 8 bits, the source of the event. + * event data, 16 bit field, data depending on event type + */ + +/** + * Specifies what kind of event this is + */ +typedef enum { + MALI_PROFILING_EVENT_TYPE_SINGLE = 0 << 24, + MALI_PROFILING_EVENT_TYPE_START = 1 << 24, + MALI_PROFILING_EVENT_TYPE_STOP = 2 << 24, + MALI_PROFILING_EVENT_TYPE_SUSPEND = 3 << 24, + MALI_PROFILING_EVENT_TYPE_RESUME = 4 << 24, +} cinstr_profiling_event_type_t; + + +/** + * Secifies the channel/source of the event + */ +typedef enum { + MALI_PROFILING_EVENT_CHANNEL_SOFTWARE = 0 << 16, + MALI_PROFILING_EVENT_CHANNEL_GP0 = 1 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP0 = 5 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP1 = 6 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP2 = 7 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP3 = 8 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP4 = 9 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP5 = 10 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP6 = 11 << 16, + MALI_PROFILING_EVENT_CHANNEL_PP7 = 12 << 16, + MALI_PROFILING_EVENT_CHANNEL_GPU = 21 << 16, +} cinstr_profiling_event_channel_t; + + +#define MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(num) (((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) + (num)) << 16) +#define MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(num) (((MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) + (num)) << 16) + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from software channel + */ +typedef enum { + MALI_PROFILING_EVENT_REASON_SINGLE_SW_NONE = 0, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_NEW_FRAME = 1, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_FLUSH = 2, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_SWAP_BUFFERS = 3, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_FB_EVENT = 4, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE = 5, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE = 6, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_READBACK = 7, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_WRITEBACK = 8, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_ENTER_API_FUNC = 10, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_LEAVE_API_FUNC = 11, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_DISCARD_ATTACHMENTS = 13, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_TRY_LOCK = 53, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_LOCK = 54, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_UNLOCK = 55, + MALI_PROFILING_EVENT_REASON_SINGLE_LOCK_CONTENDED = 56, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_MALI_FENCE_DUP = 57, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_SET_PP_JOB_FENCE = 58, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_WAIT_SYNC = 59, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_CREATE_FENCE_SYNC = 60, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_CREATE_NATIVE_FENCE_SYNC = 61, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_FENCE_FLUSH = 62, + MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_FLUSH_SERVER_WAITS = 63, +} cinstr_profiling_event_reason_single_sw_t; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_START/STOP is used from software channel + * to inform whether the core is physical or virtual + */ +typedef enum { + MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL = 0, + MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL = 1, +} cinstr_profiling_event_reason_start_stop_hw_t; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_START/STOP is used from software channel + */ +typedef enum { + /*MALI_PROFILING_EVENT_REASON_START_STOP_SW_NONE = 0,*/ + MALI_PROFILING_EVENT_REASON_START_STOP_SW_MALI = 1, + MALI_PROFILING_EVENT_REASON_START_STOP_SW_CALLBACK_THREAD = 2, + MALI_PROFILING_EVENT_REASON_START_STOP_SW_WORKER_THREAD = 3, + MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF = 4, + MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF = 5, +} cinstr_profiling_event_reason_start_stop_sw_t; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SUSPEND/RESUME is used from software channel + */ +typedef enum { + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_NONE = 0, /* used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_PIPELINE_FULL = 1, /* NOT used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VSYNC = 26, /* used in some build configurations */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_IFRAME_WAIT = 27, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_IFRAME_SYNC = 28, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VG_WAIT_FILTER_CLEANUP = 29, /* used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VG_WAIT_TEXTURE = 30, /* used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GLES_WAIT_MIPLEVEL = 31, /* used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GLES_WAIT_READPIXELS = 32, /* used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_WAIT_SWAP_IMMEDIATE = 33, /* NOT used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_ICS_QUEUE_BUFFER = 34, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_ICS_DEQUEUE_BUFFER = 35, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_UMP_LOCK = 36, /* Not currently used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_X11_GLOBAL_LOCK = 37, /* Not currently used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_X11_SWAP = 38, /* Not currently used */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_MALI_EGL_IMAGE_SYNC_WAIT = 39, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GP_JOB_HANDLING = 40, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_PP_JOB_HANDLING = 41, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_MALI_FENCE_MERGE = 42, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_MALI_FENCE_DUP = 43, + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_FLUSH_SERVER_WAITS = 44, + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_WAIT_SYNC = 45, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_JOBS_WAIT = 46, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_NOFRAMES_WAIT = 47, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_NOJOBS_WAIT = 48, /* USED */ + MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_SUBMIT_LIMITER_WAIT = 49, /* USED */ +} cinstr_profiling_event_reason_suspend_resume_sw_t; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from a HW channel (GPx+PPx) + */ +typedef enum { + MALI_PROFILING_EVENT_REASON_SINGLE_HW_NONE = 0, + MALI_PROFILING_EVENT_REASON_SINGLE_HW_INTERRUPT = 1, + MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH = 2, +} cinstr_profiling_event_reason_single_hw_t; + +/** + * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from the GPU channel + */ +typedef enum { + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_NONE = 0, + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE = 1, + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS = 2, + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L21_COUNTERS = 3, + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L22_COUNTERS = 4, +} cinstr_profiling_event_reason_single_gpu_t; + +/** + * These values are applicable for the 3rd data parameter when + * the type MALI_PROFILING_EVENT_TYPE_START is used from the software channel + * with the MALI_PROFILING_EVENT_REASON_START_STOP_BOTTOM_HALF reason. + */ +typedef enum { + MALI_PROFILING_EVENT_DATA_CORE_GP0 = 1, + MALI_PROFILING_EVENT_DATA_CORE_PP0 = 5, + MALI_PROFILING_EVENT_DATA_CORE_PP1 = 6, + MALI_PROFILING_EVENT_DATA_CORE_PP2 = 7, + MALI_PROFILING_EVENT_DATA_CORE_PP3 = 8, + MALI_PROFILING_EVENT_DATA_CORE_PP4 = 9, + MALI_PROFILING_EVENT_DATA_CORE_PP5 = 10, + MALI_PROFILING_EVENT_DATA_CORE_PP6 = 11, + MALI_PROFILING_EVENT_DATA_CORE_PP7 = 12, + MALI_PROFILING_EVENT_DATA_CORE_GP0_MMU = 22, /* GP0 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP0_MMU = 26, /* PP0 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP1_MMU = 27, /* PP1 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP2_MMU = 28, /* PP2 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP3_MMU = 29, /* PP3 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP4_MMU = 30, /* PP4 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP5_MMU = 31, /* PP5 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP6_MMU = 32, /* PP6 + 21 */ + MALI_PROFILING_EVENT_DATA_CORE_PP7_MMU = 33, /* PP7 + 21 */ + +} cinstr_profiling_event_data_core_t; + +#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(num) (MALI_PROFILING_EVENT_DATA_CORE_GP0 + (num)) +#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(num) (MALI_PROFILING_EVENT_DATA_CORE_GP0_MMU + (num)) +#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(num) (MALI_PROFILING_EVENT_DATA_CORE_PP0 + (num)) +#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU(num) (MALI_PROFILING_EVENT_DATA_CORE_PP0_MMU + (num)) + + +#endif /*_MALI_UTGARD_PROFILING_EVENTS_H_*/ diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h new file mode 100755 index 000000000000..c1927d1450dc --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h @@ -0,0 +1,305 @@ +/* + * Copyright (C) 2013, 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_UTGARD_PROFILING_GATOR_API_H__ +#define __MALI_UTGARD_PROFILING_GATOR_API_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define MALI_PROFILING_API_VERSION 4 + +#define MAX_NUM_L2_CACHE_CORES 3 +#define MAX_NUM_FP_CORES 8 +#define MAX_NUM_VP_CORES 1 + +#define _MALI_SPCIAL_COUNTER_DESCRIPTIONS \ + { \ + "Filmstrip_cnt0", \ + "Frequency", \ + "Voltage", \ + "vertex", \ + "fragment", \ + "Total_alloc_pages", \ + }; + +#define _MALI_MEM_COUTNER_DESCRIPTIONS \ + { \ + "untyped_memory", \ + "vertex_index_buffer", \ + "texture_buffer", \ + "varying_buffer", \ + "render_target", \ + "pbuffer_buffer", \ + "plbu_heap", \ + "pointer_array_buffer", \ + "slave_tilelist", \ + "untyped_gp_cmdlist", \ + "polygon_cmdlist", \ + "texture_descriptor", \ + "render_state_word", \ + "shader", \ + "stream_buffer", \ + "fragment_stack", \ + "uniform", \ + "untyped_frame_pool", \ + "untyped_surface", \ + }; + +/** The list of events supported by the Mali DDK. */ +typedef enum { + /* Vertex processor activity */ + ACTIVITY_VP_0 = 0, + + /* Fragment processor activity */ + ACTIVITY_FP_0, + ACTIVITY_FP_1, + ACTIVITY_FP_2, + ACTIVITY_FP_3, + ACTIVITY_FP_4, + ACTIVITY_FP_5, + ACTIVITY_FP_6, + ACTIVITY_FP_7, + + /* L2 cache counters */ + COUNTER_L2_0_C0, + COUNTER_L2_0_C1, + COUNTER_L2_1_C0, + COUNTER_L2_1_C1, + COUNTER_L2_2_C0, + COUNTER_L2_2_C1, + + /* Vertex processor counters */ + COUNTER_VP_0_C0, + COUNTER_VP_0_C1, + + /* Fragment processor counters */ + COUNTER_FP_0_C0, + COUNTER_FP_0_C1, + COUNTER_FP_1_C0, + COUNTER_FP_1_C1, + COUNTER_FP_2_C0, + COUNTER_FP_2_C1, + COUNTER_FP_3_C0, + COUNTER_FP_3_C1, + COUNTER_FP_4_C0, + COUNTER_FP_4_C1, + COUNTER_FP_5_C0, + COUNTER_FP_5_C1, + COUNTER_FP_6_C0, + COUNTER_FP_6_C1, + COUNTER_FP_7_C0, + COUNTER_FP_7_C1, + + /* + * If more hardware counters are added, the _mali_osk_hw_counter_table + * below should also be updated. + */ + + /* EGL software counters */ + COUNTER_EGL_BLIT_TIME, + + /* GLES software counters */ + COUNTER_GLES_DRAW_ELEMENTS_CALLS, + COUNTER_GLES_DRAW_ELEMENTS_NUM_INDICES, + COUNTER_GLES_DRAW_ELEMENTS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_ARRAYS_CALLS, + COUNTER_GLES_DRAW_ARRAYS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_POINTS, + COUNTER_GLES_DRAW_LINES, + COUNTER_GLES_DRAW_LINE_LOOP, + COUNTER_GLES_DRAW_LINE_STRIP, + COUNTER_GLES_DRAW_TRIANGLES, + COUNTER_GLES_DRAW_TRIANGLE_STRIP, + COUNTER_GLES_DRAW_TRIANGLE_FAN, + COUNTER_GLES_NON_VBO_DATA_COPY_TIME, + COUNTER_GLES_UNIFORM_BYTES_COPIED_TO_MALI, + COUNTER_GLES_UPLOAD_TEXTURE_TIME, + COUNTER_GLES_UPLOAD_VBO_TIME, + COUNTER_GLES_NUM_FLUSHES, + COUNTER_GLES_NUM_VSHADERS_GENERATED, + COUNTER_GLES_NUM_FSHADERS_GENERATED, + COUNTER_GLES_VSHADER_GEN_TIME, + COUNTER_GLES_FSHADER_GEN_TIME, + COUNTER_GLES_INPUT_TRIANGLES, + COUNTER_GLES_VXCACHE_HIT, + COUNTER_GLES_VXCACHE_MISS, + COUNTER_GLES_VXCACHE_COLLISION, + COUNTER_GLES_CULLED_TRIANGLES, + COUNTER_GLES_CULLED_LINES, + COUNTER_GLES_BACKFACE_TRIANGLES, + COUNTER_GLES_GBCLIP_TRIANGLES, + COUNTER_GLES_GBCLIP_LINES, + COUNTER_GLES_TRIANGLES_DRAWN, + COUNTER_GLES_DRAWCALL_TIME, + COUNTER_GLES_TRIANGLES_COUNT, + COUNTER_GLES_INDEPENDENT_TRIANGLES_COUNT, + COUNTER_GLES_STRIP_TRIANGLES_COUNT, + COUNTER_GLES_FAN_TRIANGLES_COUNT, + COUNTER_GLES_LINES_COUNT, + COUNTER_GLES_INDEPENDENT_LINES_COUNT, + COUNTER_GLES_STRIP_LINES_COUNT, + COUNTER_GLES_LOOP_LINES_COUNT, + + /* Special counter */ + + /* Framebuffer capture pseudo-counter */ + COUNTER_FILMSTRIP, + COUNTER_FREQUENCY, + COUNTER_VOLTAGE, + COUNTER_VP_ACTIVITY, + COUNTER_FP_ACTIVITY, + COUNTER_TOTAL_ALLOC_PAGES, + + /* Memory usage counter */ + COUNTER_MEM_UNTYPED, + COUNTER_MEM_VB_IB, + COUNTER_MEM_TEXTURE, + COUNTER_MEM_VARYING, + COUNTER_MEM_RT, + COUNTER_MEM_PBUFFER, + /* memory usages for gp command */ + COUNTER_MEM_PLBU_HEAP, + COUNTER_MEM_POINTER_ARRAY, + COUNTER_MEM_SLAVE_TILELIST, + COUNTER_MEM_UNTYPE_GP_CMDLIST, + /* memory usages for polygon list command */ + COUNTER_MEM_POLYGON_CMDLIST, + /* memory usages for pp command */ + COUNTER_MEM_TD, + COUNTER_MEM_RSW, + /* other memory usages */ + COUNTER_MEM_SHADER, + COUNTER_MEM_STREAMS, + COUNTER_MEM_FRAGMENT_STACK, + COUNTER_MEM_UNIFORM, + /* Special mem usage, which is used for mem pool allocation */ + COUNTER_MEM_UNTYPE_MEM_POOL, + COUNTER_MEM_UNTYPE_SURFACE, + + NUMBER_OF_EVENTS +} _mali_osk_counter_id; + +#define FIRST_ACTIVITY_EVENT ACTIVITY_VP_0 +#define LAST_ACTIVITY_EVENT ACTIVITY_FP_7 + +#define FIRST_HW_COUNTER COUNTER_L2_0_C0 +#define LAST_HW_COUNTER COUNTER_FP_7_C1 + +#define FIRST_SW_COUNTER COUNTER_EGL_BLIT_TIME +#define LAST_SW_COUNTER COUNTER_GLES_LOOP_LINES_COUNT + +#define FIRST_SPECIAL_COUNTER COUNTER_FILMSTRIP +#define LAST_SPECIAL_COUNTER COUNTER_TOTAL_ALLOC_PAGES + +#define FIRST_MEM_COUNTER COUNTER_MEM_UNTYPED +#define LAST_MEM_COUNTER COUNTER_MEM_UNTYPE_SURFACE + +#define MALI_PROFILING_MEM_COUNTERS_NUM (LAST_MEM_COUNTER - FIRST_MEM_COUNTER + 1) +#define MALI_PROFILING_SPECIAL_COUNTERS_NUM (LAST_SPECIAL_COUNTER - FIRST_SPECIAL_COUNTER + 1) +#define MALI_PROFILING_SW_COUNTERS_NUM (LAST_SW_COUNTER - FIRST_SW_COUNTER + 1) + +/** + * Define the stream header type for porfiling stream. + */ +#define STREAM_HEADER_FRAMEBUFFER 0x05 /* The stream packet header type for framebuffer dumping. */ +#define STREAM_HEADER_COUNTER_VALUE 0x09 /* The stream packet header type for hw/sw/memory counter sampling. */ +#define STREAM_HEADER_CORE_ACTIVITY 0x0a /* The stream packet header type for activity counter sampling. */ +#define STREAM_HEADER_SIZE 5 + +/** + * Define the packet header type of profiling control packet. + */ +#define PACKET_HEADER_ERROR 0x80 /* The response packet header type if error. */ +#define PACKET_HEADER_ACK 0x81 /* The response packet header type if OK. */ +#define PACKET_HEADER_COUNTERS_REQUEST 0x82 /* The control packet header type to request counter information from ddk. */ +#define PACKET_HEADER_COUNTERS_ACK 0x83 /* The response packet header type to send out counter information. */ +#define PACKET_HEADER_COUNTERS_ENABLE 0x84 /* The control packet header type to enable counters. */ +#define PACKET_HEADER_START_CAPTURE_VALUE 0x85 /* The control packet header type to start capture values. */ + +#define PACKET_HEADER_SIZE 5 + +/** + * Structure to pass performance counter data of a Mali core + */ +typedef struct _mali_profiling_core_counters { + u32 source0; + u32 value0; + u32 source1; + u32 value1; +} _mali_profiling_core_counters; + +/** + * Structure to pass performance counter data of Mali L2 cache cores + */ +typedef struct _mali_profiling_l2_counter_values { + struct _mali_profiling_core_counters cores[MAX_NUM_L2_CACHE_CORES]; +} _mali_profiling_l2_counter_values; + +/** + * Structure to pass data defining Mali instance in use: + * + * mali_product_id - Mali product id + * mali_version_major - Mali version major number + * mali_version_minor - Mali version minor number + * num_of_l2_cores - number of L2 cache cores + * num_of_fp_cores - number of fragment processor cores + * num_of_vp_cores - number of vertex processor cores + */ +typedef struct _mali_profiling_mali_version { + u32 mali_product_id; + u32 mali_version_major; + u32 mali_version_minor; + u32 num_of_l2_cores; + u32 num_of_fp_cores; + u32 num_of_vp_cores; +} _mali_profiling_mali_version; + +/** + * Structure to define the mali profiling counter struct. + */ +typedef struct mali_profiling_counter { + char counter_name[40]; + u32 counter_id; + u32 counter_event; + u32 prev_counter_value; + u32 current_counter_value; + u32 key; + int enabled; +} mali_profiling_counter; + +/* + * List of possible actions to be controlled by Streamline. + * The following numbers are used by gator to control the frame buffer dumping and s/w counter reporting. + * We cannot use the enums in mali_uk_types.h because they are unknown inside gator. + */ +#define FBDUMP_CONTROL_ENABLE (1) +#define FBDUMP_CONTROL_RATE (2) +#define SW_COUNTER_ENABLE (3) +#define FBDUMP_CONTROL_RESIZE_FACTOR (4) +#define MEM_COUNTER_ENABLE (5) +#define ANNOTATE_PROFILING_ENABLE (6) + +void _mali_profiling_control(u32 action, u32 value); + +u32 _mali_profiling_get_l2_counters(_mali_profiling_l2_counter_values *values); + +int _mali_profiling_set_event(u32 counter_id, s32 event_id); + +u32 _mali_profiling_get_api_version(void); + +void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UTGARD_PROFILING_GATOR_API_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h new file mode 100755 index 000000000000..34656f09b2ab --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h @@ -0,0 +1,1108 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_uk_types.h + * Defines the types and constants used in the user-kernel interface + */ + +#ifndef __MALI_UTGARD_UK_TYPES_H__ +#define __MALI_UTGARD_UK_TYPES_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Iteration functions depend on these values being consecutive. */ +#define MALI_UK_TIMELINE_GP 0 +#define MALI_UK_TIMELINE_PP 1 +#define MALI_UK_TIMELINE_SOFT 2 +#define MALI_UK_TIMELINE_MAX 3 + +#define MALI_UK_BIG_VARYING_SIZE (1024*1024*2) + +typedef struct { + u32 points[MALI_UK_TIMELINE_MAX]; + s32 sync_fd; +} _mali_uk_fence_t; + +/** + * @addtogroup uddapi Unified Device Driver (UDD) APIs + * + * @{ + */ + +/** + * @addtogroup u_k_api UDD User/Kernel Interface (U/K) APIs + * + * @{ + */ + +/** @defgroup _mali_uk_core U/K Core + * @{ */ + +/** Definition of subsystem numbers, to assist in creating a unique identifier + * for each U/K call. + * + * @see _mali_uk_functions */ +typedef enum { + _MALI_UK_CORE_SUBSYSTEM, /**< Core Group of U/K calls */ + _MALI_UK_MEMORY_SUBSYSTEM, /**< Memory Group of U/K calls */ + _MALI_UK_PP_SUBSYSTEM, /**< Fragment Processor Group of U/K calls */ + _MALI_UK_GP_SUBSYSTEM, /**< Vertex Processor Group of U/K calls */ + _MALI_UK_PROFILING_SUBSYSTEM, /**< Profiling Group of U/K calls */ + _MALI_UK_VSYNC_SUBSYSTEM, /**< VSYNC Group of U/K calls */ +} _mali_uk_subsystem_t; + +/** Within a function group each function has its unique sequence number + * to assist in creating a unique identifier for each U/K call. + * + * An ordered pair of numbers selected from + * ( \ref _mali_uk_subsystem_t,\ref _mali_uk_functions) will uniquely identify the + * U/K call across all groups of functions, and all functions. */ +typedef enum { + /** Core functions */ + + _MALI_UK_OPEN = 0, /**< _mali_ukk_open() */ + _MALI_UK_CLOSE, /**< _mali_ukk_close() */ + _MALI_UK_WAIT_FOR_NOTIFICATION, /**< _mali_ukk_wait_for_notification() */ + _MALI_UK_GET_API_VERSION, /**< _mali_ukk_get_api_version() */ + _MALI_UK_POST_NOTIFICATION, /**< _mali_ukk_post_notification() */ + _MALI_UK_GET_USER_SETTING, /**< _mali_ukk_get_user_setting() *//**< [out] */ + _MALI_UK_GET_USER_SETTINGS, /**< _mali_ukk_get_user_settings() *//**< [out] */ + _MALI_UK_REQUEST_HIGH_PRIORITY, /**< _mali_ukk_request_high_priority() */ + _MALI_UK_TIMELINE_GET_LATEST_POINT, /**< _mali_ukk_timeline_get_latest_point() */ + _MALI_UK_TIMELINE_WAIT, /**< _mali_ukk_timeline_wait() */ + _MALI_UK_TIMELINE_CREATE_SYNC_FENCE, /**< _mali_ukk_timeline_create_sync_fence() */ + _MALI_UK_SOFT_JOB_START, /**< _mali_ukk_soft_job_start() */ + _MALI_UK_SOFT_JOB_SIGNAL, /**< _mali_ukk_soft_job_signal() */ + _MALI_UK_PENDING_SUBMIT, /**< _mali_ukk_pending_submit() */ + + _MALI_GET_RK_KO_VERSION, /* rk_ext */ + _MALI_UK_GET_MALI_VERSION_IN_RK30, + + /** Memory functions */ + + _MALI_UK_ALLOC_MEM = 0, /**< _mali_ukk_alloc_mem() */ + _MALI_UK_FREE_MEM, /**< _mali_ukk_free_mem() */ + _MALI_UK_BIND_MEM, /**< _mali_ukk_mem_bind() */ + _MALI_UK_UNBIND_MEM, /**< _mali_ukk_mem_unbind() */ + _MALI_UK_COW_MEM, /**< _mali_ukk_mem_cow() */ + _MALI_UK_COW_MODIFY_RANGE, /**< _mali_ukk_mem_cow_modify_range() */ + _MALI_UK_RESIZE_MEM, /**<._mali_ukk_mem_resize() */ + _MALI_UK_QUERY_MMU_PAGE_TABLE_DUMP_SIZE, /**< _mali_ukk_mem_get_mmu_page_table_dump_size() */ + _MALI_UK_DUMP_MMU_PAGE_TABLE, /**< _mali_ukk_mem_dump_mmu_page_table() */ + _MALI_UK_DMA_BUF_GET_SIZE, /**< _mali_ukk_dma_buf_get_size() */ + _MALI_UK_MEM_WRITE_SAFE, /**< _mali_uku_mem_write_safe() */ + + /** Common functions for each core */ + + _MALI_UK_START_JOB = 0, /**< Start a Fragment/Vertex Processor Job on a core */ + _MALI_UK_GET_NUMBER_OF_CORES, /**< Get the number of Fragment/Vertex Processor cores */ + _MALI_UK_GET_CORE_VERSION, /**< Get the Fragment/Vertex Processor version compatible with all cores */ + + /** Fragment Processor Functions */ + + _MALI_UK_PP_START_JOB = _MALI_UK_START_JOB, /**< _mali_ukk_pp_start_job() */ + _MALI_UK_GET_PP_NUMBER_OF_CORES = _MALI_UK_GET_NUMBER_OF_CORES, /**< _mali_ukk_get_pp_number_of_cores() */ + _MALI_UK_GET_PP_CORE_VERSION = _MALI_UK_GET_CORE_VERSION, /**< _mali_ukk_get_pp_core_version() */ + _MALI_UK_PP_DISABLE_WB, /**< _mali_ukk_pp_job_disable_wb() */ + _MALI_UK_PP_AND_GP_START_JOB, /**< _mali_ukk_pp_and_gp_start_job() */ + + /** Vertex Processor Functions */ + + _MALI_UK_GP_START_JOB = _MALI_UK_START_JOB, /**< _mali_ukk_gp_start_job() */ + _MALI_UK_GET_GP_NUMBER_OF_CORES = _MALI_UK_GET_NUMBER_OF_CORES, /**< _mali_ukk_get_gp_number_of_cores() */ + _MALI_UK_GET_GP_CORE_VERSION = _MALI_UK_GET_CORE_VERSION, /**< _mali_ukk_get_gp_core_version() */ + _MALI_UK_GP_SUSPEND_RESPONSE, /**< _mali_ukk_gp_suspend_response() */ + + /** Profiling functions */ + + _MALI_UK_PROFILING_ADD_EVENT = 0, /**< __mali_uku_profiling_add_event() */ + _MALI_UK_PROFILING_REPORT_SW_COUNTERS,/**< __mali_uku_profiling_report_sw_counters() */ + _MALI_UK_PROFILING_MEMORY_USAGE_GET, /**< __mali_uku_profiling_memory_usage_get() */ + _MALI_UK_PROFILING_STREAM_FD_GET, /** < __mali_uku_profiling_stream_fd_get() */ + _MALI_UK_PROFILING_CONTROL_SET, /** < __mali_uku_profiling_control_set() */ + + /** VSYNC reporting fuctions */ + _MALI_UK_VSYNC_EVENT_REPORT = 0, /**< _mali_ukk_vsync_event_report() */ +} _mali_uk_functions; + +/** @defgroup _mali_uk_getsysteminfo U/K Get System Info + * @{ */ + +/** + * Type definition for the core version number. + * Used when returning the version number read from a core + * + * Its format is that of the 32-bit Version register for a particular core. + * Refer to the "Mali200 and MaliGP2 3D Graphics Processor Technical Reference + * Manual", ARM DDI 0415C, for more information. + */ +typedef u32 _mali_core_version; + +/** @} */ /* end group _mali_uk_core */ + + +/** @defgroup _mali_uk_gp U/K Vertex Processor + * @{ */ + +/** @defgroup _mali_uk_gp_suspend_response_s Vertex Processor Suspend Response + * @{ */ + +/** @brief Arguments for _mali_ukk_gp_suspend_response() + * + * When _mali_wait_for_notification() receives notification that a + * Vertex Processor job was suspended, you need to send a response to indicate + * what needs to happen with this job. You can either abort or resume the job. + * + * - set @c code to indicate response code. This is either @c _MALIGP_JOB_ABORT or + * @c _MALIGP_JOB_RESUME_WITH_NEW_HEAP to indicate you will provide a new heap + * for the job that will resolve the out of memory condition for the job. + * - copy the @c cookie value from the @c _mali_uk_gp_job_suspended_s notification; + * this is an identifier for the suspended job + * - set @c arguments[0] and @c arguments[1] to zero if you abort the job. If + * you resume it, @c argument[0] should specify the Mali start address for the new + * heap and @c argument[1] the Mali end address of the heap. + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * + */ +typedef enum _maligp_job_suspended_response_code { + _MALIGP_JOB_ABORT, /**< Abort the Vertex Processor job */ + _MALIGP_JOB_RESUME_WITH_NEW_HEAP /**< Resume the Vertex Processor job with a new heap */ +} _maligp_job_suspended_response_code; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 cookie; /**< [in] cookie from the _mali_uk_gp_job_suspended_s notification */ + _maligp_job_suspended_response_code code; /**< [in] abort or resume response code, see \ref _maligp_job_suspended_response_code */ + u32 arguments[2]; /**< [in] 0 when aborting a job. When resuming a job, the Mali start and end address for a new heap to resume the job with */ +} _mali_uk_gp_suspend_response_s; + +/** @} */ /* end group _mali_uk_gp_suspend_response_s */ + +/** @defgroup _mali_uk_gpstartjob_s Vertex Processor Start Job + * @{ */ + +/** @brief Status indicating the result of the execution of a Vertex or Fragment processor job */ +typedef enum { + _MALI_UK_JOB_STATUS_END_SUCCESS = 1 << (16 + 0), + _MALI_UK_JOB_STATUS_END_OOM = 1 << (16 + 1), + _MALI_UK_JOB_STATUS_END_ABORT = 1 << (16 + 2), + _MALI_UK_JOB_STATUS_END_TIMEOUT_SW = 1 << (16 + 3), + _MALI_UK_JOB_STATUS_END_HANG = 1 << (16 + 4), + _MALI_UK_JOB_STATUS_END_SEG_FAULT = 1 << (16 + 5), + _MALI_UK_JOB_STATUS_END_ILLEGAL_JOB = 1 << (16 + 6), + _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR = 1 << (16 + 7), + _MALI_UK_JOB_STATUS_END_SHUTDOWN = 1 << (16 + 8), + _MALI_UK_JOB_STATUS_END_SYSTEM_UNUSABLE = 1 << (16 + 9) +} _mali_uk_job_status; + +#define MALIGP2_NUM_REGS_FRAME (6) + +/** @brief Arguments for _mali_ukk_gp_start_job() + * + * To start a Vertex Processor job + * - associate the request with a reference to a @c mali_gp_job_info by setting + * user_job_ptr to the address of the @c mali_gp_job_info of the job. + * - set @c priority to the priority of the @c mali_gp_job_info + * - specify a timeout for the job by setting @c watchdog_msecs to the number of + * milliseconds the job is allowed to run. Specifying a value of 0 selects the + * default timeout in use by the device driver. + * - copy the frame registers from the @c mali_gp_job_info into @c frame_registers. + * - set the @c perf_counter_flag, @c perf_counter_src0 and @c perf_counter_src1 to zero + * for a non-instrumented build. For an instrumented build you can use up + * to two performance counters. Set the corresponding bit in @c perf_counter_flag + * to enable them. @c perf_counter_src0 and @c perf_counter_src1 specify + * the source of what needs to get counted (e.g. number of vertex loader + * cache hits). For source id values, see ARM DDI0415A, Table 3-60. + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * + * When @c _mali_ukk_gp_start_job() returns @c _MALI_OSK_ERR_OK, status contains the + * result of the request (see \ref _mali_uk_start_job_status). If the job could + * not get started (@c _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE) it should be + * tried again. + * + * After the job has started, @c _mali_wait_for_notification() will be notified + * that the job finished or got suspended. It may get suspended due to + * resource shortage. If it finished (see _mali_ukk_wait_for_notification()) + * the notification will contain a @c _mali_uk_gp_job_finished_s result. If + * it got suspended the notification will contain a @c _mali_uk_gp_job_suspended_s + * result. + * + * The @c _mali_uk_gp_job_finished_s contains the job status (see \ref _mali_uk_job_status), + * the number of milliseconds the job took to render, and values of core registers + * when the job finished (irq status, performance counters, renderer list + * address). A job has finished succesfully when its status is + * @c _MALI_UK_JOB_STATUS_FINISHED. If the hardware detected a timeout while rendering + * the job, or software detected the job is taking more than watchdog_msecs to + * complete, the status will indicate @c _MALI_UK_JOB_STATUS_HANG. + * If the hardware detected a bus error while accessing memory associated with the + * job, status will indicate @c _MALI_UK_JOB_STATUS_SEG_FAULT. + * status will indicate @c _MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to + * stop the job but the job didn't start on the hardware yet, e.g. when the + * driver shutdown. + * + * In case the job got suspended, @c _mali_uk_gp_job_suspended_s contains + * the @c user_job_ptr identifier used to start the job with, the @c reason + * why the job stalled (see \ref _maligp_job_suspended_reason) and a @c cookie + * to identify the core on which the job stalled. This @c cookie will be needed + * when responding to this nofication by means of _mali_ukk_gp_suspend_response(). + * (see _mali_ukk_gp_suspend_response()). The response is either to abort or + * resume the job. If the job got suspended due to an out of memory condition + * you may be able to resolve this by providing more memory and resuming the job. + * + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 user_job_ptr; /**< [in] identifier for the job in user space, a @c mali_gp_job_info* */ + u32 priority; /**< [in] job priority. A lower number means higher priority */ + u32 frame_registers[MALIGP2_NUM_REGS_FRAME]; /**< [in] core specific registers associated with this job */ + u32 perf_counter_flag; /**< [in] bitmask indicating which performance counters to enable, see \ref _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE and related macro definitions */ + u32 perf_counter_src0; /**< [in] source id for performance counter 0 (see ARM DDI0415A, Table 3-60) */ + u32 perf_counter_src1; /**< [in] source id for performance counter 1 (see ARM DDI0415A, Table 3-60) */ + u32 frame_builder_id; /**< [in] id of the originating frame builder */ + u32 flush_id; /**< [in] flush id within the originating frame builder */ + _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ + u64 timeline_point_ptr; /**< [in,out] pointer to u32: location where point on gp timeline for this job will be written */ + u32 varying_memsize; /** < [in] size of varying memory to use deffer bind*/ + u32 deferred_mem_num; + u64 deferred_mem_list; /** < [in] memory hanlde list of varying buffer to use deffer bind */ +} _mali_uk_gp_start_job_s; + +#define _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE (1<<0) /**< Enable performance counter SRC0 for a job */ +#define _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE (1<<1) /**< Enable performance counter SRC1 for a job */ +#define _MALI_PERFORMANCE_COUNTER_FLAG_HEATMAP_ENABLE (1<<2) /**< Enable per tile (aka heatmap) generation with for a job (using the enabled counter sources) */ + +/** @} */ /* end group _mali_uk_gpstartjob_s */ + +typedef struct { + u64 user_job_ptr; /**< [out] identifier for the job in user space */ + _mali_uk_job_status status; /**< [out] status of finished job */ + u32 heap_current_addr; /**< [out] value of the GP PLB PL heap start address register */ + u32 perf_counter0; /**< [out] value of performance counter 0 (see ARM DDI0415A) */ + u32 perf_counter1; /**< [out] value of performance counter 1 (see ARM DDI0415A) */ + u32 pending_big_job_num; +} _mali_uk_gp_job_finished_s; + +typedef struct { + u64 user_job_ptr; /**< [out] identifier for the job in user space */ + u32 cookie; /**< [out] identifier for the core in kernel space on which the job stalled */ +} _mali_uk_gp_job_suspended_s; + +/** @} */ /* end group _mali_uk_gp */ + + +/** @defgroup _mali_uk_pp U/K Fragment Processor + * @{ */ + +#define _MALI_PP_MAX_SUB_JOBS 8 + +#define _MALI_PP_MAX_FRAME_REGISTERS ((0x058/4)+1) + +#define _MALI_PP_MAX_WB_REGISTERS ((0x02C/4)+1) + +#define _MALI_DLBU_MAX_REGISTERS 4 + +/** Flag for _mali_uk_pp_start_job_s */ +#define _MALI_PP_JOB_FLAG_NO_NOTIFICATION (1<<0) +#define _MALI_PP_JOB_FLAG_IS_WINDOW_SURFACE (1<<1) +#define _MALI_PP_JOB_FLAG_PROTECTED (1<<2) + +/** @defgroup _mali_uk_ppstartjob_s Fragment Processor Start Job + * @{ */ + +/** @brief Arguments for _mali_ukk_pp_start_job() + * + * To start a Fragment Processor job + * - associate the request with a reference to a mali_pp_job by setting + * @c user_job_ptr to the address of the @c mali_pp_job of the job. + * - set @c priority to the priority of the mali_pp_job + * - specify a timeout for the job by setting @c watchdog_msecs to the number of + * milliseconds the job is allowed to run. Specifying a value of 0 selects the + * default timeout in use by the device driver. + * - copy the frame registers from the @c mali_pp_job into @c frame_registers. + * For MALI200 you also need to copy the write back 0,1 and 2 registers. + * - set the @c perf_counter_flag, @c perf_counter_src0 and @c perf_counter_src1 to zero + * for a non-instrumented build. For an instrumented build you can use up + * to two performance counters. Set the corresponding bit in @c perf_counter_flag + * to enable them. @c perf_counter_src0 and @c perf_counter_src1 specify + * the source of what needs to get counted (e.g. number of vertex loader + * cache hits). For source id values, see ARM DDI0415A, Table 3-60. + * - pass in the user-kernel context in @c ctx that was returned from _mali_ukk_open() + * + * When _mali_ukk_pp_start_job() returns @c _MALI_OSK_ERR_OK, @c status contains the + * result of the request (see \ref _mali_uk_start_job_status). If the job could + * not get started (@c _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE) it should be + * tried again. + * + * After the job has started, _mali_wait_for_notification() will be notified + * when the job finished. The notification will contain a + * @c _mali_uk_pp_job_finished_s result. It contains the @c user_job_ptr + * identifier used to start the job with, the job @c status (see \ref _mali_uk_job_status), + * the number of milliseconds the job took to render, and values of core registers + * when the job finished (irq status, performance counters, renderer list + * address). A job has finished succesfully when its status is + * @c _MALI_UK_JOB_STATUS_FINISHED. If the hardware detected a timeout while rendering + * the job, or software detected the job is taking more than @c watchdog_msecs to + * complete, the status will indicate @c _MALI_UK_JOB_STATUS_HANG. + * If the hardware detected a bus error while accessing memory associated with the + * job, status will indicate @c _MALI_UK_JOB_STATUS_SEG_FAULT. + * status will indicate @c _MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to + * stop the job but the job didn't start on the hardware yet, e.g. when the + * driver shutdown. + * + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 user_job_ptr; /**< [in] identifier for the job in user space */ + u32 priority; /**< [in] job priority. A lower number means higher priority */ + u32 frame_registers[_MALI_PP_MAX_FRAME_REGISTERS]; /**< [in] core specific registers associated with first sub job, see ARM DDI0415A */ + u32 frame_registers_addr_frame[_MALI_PP_MAX_SUB_JOBS - 1]; /**< [in] ADDR_FRAME registers for sub job 1-7 */ + u32 frame_registers_addr_stack[_MALI_PP_MAX_SUB_JOBS - 1]; /**< [in] ADDR_STACK registers for sub job 1-7 */ + u32 wb0_registers[_MALI_PP_MAX_WB_REGISTERS]; + u32 wb1_registers[_MALI_PP_MAX_WB_REGISTERS]; + u32 wb2_registers[_MALI_PP_MAX_WB_REGISTERS]; + u32 dlbu_registers[_MALI_DLBU_MAX_REGISTERS]; /**< [in] Dynamic load balancing unit registers */ + u32 num_cores; /**< [in] Number of cores to set up (valid range: 1-8(M450) or 4(M400)) */ + u32 perf_counter_flag; /**< [in] bitmask indicating which performance counters to enable, see \ref _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE and related macro definitions */ + u32 perf_counter_src0; /**< [in] source id for performance counter 0 (see ARM DDI0415A, Table 3-60) */ + u32 perf_counter_src1; /**< [in] source id for performance counter 1 (see ARM DDI0415A, Table 3-60) */ + u32 frame_builder_id; /**< [in] id of the originating frame builder */ + u32 flush_id; /**< [in] flush id within the originating frame builder */ + u32 flags; /**< [in] See _MALI_PP_JOB_FLAG_* for a list of avaiable flags */ + u32 tilesx; /**< [in] number of tiles in the x direction (needed for heatmap generation */ + u32 tilesy; /**< [in] number of tiles in y direction (needed for reading the heatmap memory) */ + u32 heatmap_mem; /**< [in] memory address to store counter values per tile (aka heatmap) */ + u32 num_memory_cookies; /**< [in] number of memory cookies attached to job */ + u64 memory_cookies; /**< [in] pointer to array of u32 memory cookies attached to job */ + _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ + u64 timeline_point_ptr; /**< [in,out] pointer to location of u32 where point on pp timeline for this job will be written */ +} _mali_uk_pp_start_job_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 gp_args; /**< [in,out] GP uk arguments (see _mali_uk_gp_start_job_s) */ + u64 pp_args; /**< [in,out] PP uk arguments (see _mali_uk_pp_start_job_s) */ +} _mali_uk_pp_and_gp_start_job_s; + +/** @} */ /* end group _mali_uk_ppstartjob_s */ + +typedef struct { + u64 user_job_ptr; /**< [out] identifier for the job in user space */ + _mali_uk_job_status status; /**< [out] status of finished job */ + u32 perf_counter0[_MALI_PP_MAX_SUB_JOBS]; /**< [out] value of perfomance counter 0 (see ARM DDI0415A), one for each sub job */ + u32 perf_counter1[_MALI_PP_MAX_SUB_JOBS]; /**< [out] value of perfomance counter 1 (see ARM DDI0415A), one for each sub job */ + u32 perf_counter_src0; + u32 perf_counter_src1; +} _mali_uk_pp_job_finished_s; + +typedef struct { + u32 number_of_enabled_cores; /**< [out] the new number of enabled cores */ +} _mali_uk_pp_num_cores_changed_s; + + + +/** + * Flags to indicate write-back units + */ +typedef enum { + _MALI_UK_PP_JOB_WB0 = 1, + _MALI_UK_PP_JOB_WB1 = 2, + _MALI_UK_PP_JOB_WB2 = 4, +} _mali_uk_pp_job_wbx_flag; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 fb_id; /**< [in] Frame builder ID of job to disable WB units for */ + u32 wb0_memory; + u32 wb1_memory; + u32 wb2_memory; +} _mali_uk_pp_disable_wb_s; + + +/** @} */ /* end group _mali_uk_pp */ + +/** @defgroup _mali_uk_soft_job U/K Soft Job + * @{ */ + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 user_job; /**< [in] identifier for the job in user space */ + u64 job_id_ptr; /**< [in,out] pointer to location of u32 where job id will be written */ + _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ + u32 point; /**< [out] point on soft timeline for this job */ + u32 type; /**< [in] type of soft job */ +} _mali_uk_soft_job_start_s; + +typedef struct { + u64 user_job; /**< [out] identifier for the job in user space */ +} _mali_uk_soft_job_activated_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 job_id; /**< [in] id for soft job */ +} _mali_uk_soft_job_signal_s; + +/** @} */ /* end group _mali_uk_soft_job */ + +typedef struct { + u32 counter_id; + u32 key; + int enable; +} _mali_uk_annotate_profiling_mem_counter_s; + +typedef struct { + u32 sampling_rate; + int enable; +} _mali_uk_annotate_profiling_enable_s; + + +/** @addtogroup _mali_uk_core U/K Core + * @{ */ + +/** @defgroup _mali_uk_waitfornotification_s Wait For Notification + * @{ */ + +/** @brief Notification type encodings + * + * Each Notification type is an ordered pair of (subsystem,id), and is unique. + * + * The encoding of subsystem,id into a 32-bit word is: + * encoding = (( subsystem << _MALI_NOTIFICATION_SUBSYSTEM_SHIFT ) & _MALI_NOTIFICATION_SUBSYSTEM_MASK) + * | (( id << _MALI_NOTIFICATION_ID_SHIFT ) & _MALI_NOTIFICATION_ID_MASK) + * + * @see _mali_uk_wait_for_notification_s + */ +typedef enum { + /** core notifications */ + + _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x20, + _MALI_NOTIFICATION_APPLICATION_QUIT = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x40, + _MALI_NOTIFICATION_SETTINGS_CHANGED = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x80, + _MALI_NOTIFICATION_SOFT_ACTIVATED = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x100, + + /** Fragment Processor notifications */ + + _MALI_NOTIFICATION_PP_FINISHED = (_MALI_UK_PP_SUBSYSTEM << 16) | 0x10, + _MALI_NOTIFICATION_PP_NUM_CORE_CHANGE = (_MALI_UK_PP_SUBSYSTEM << 16) | 0x20, + + /** Vertex Processor notifications */ + + _MALI_NOTIFICATION_GP_FINISHED = (_MALI_UK_GP_SUBSYSTEM << 16) | 0x10, + _MALI_NOTIFICATION_GP_STALLED = (_MALI_UK_GP_SUBSYSTEM << 16) | 0x20, + + /** Profiling notifications */ + _MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER = (_MALI_UK_PROFILING_SUBSYSTEM << 16) | 0x10, + _MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE = (_MALI_UK_PROFILING_SUBSYSTEM << 16) | 0x20, +} _mali_uk_notification_type; + +/** to assist in splitting up 32-bit notification value in subsystem and id value */ +#define _MALI_NOTIFICATION_SUBSYSTEM_MASK 0xFFFF0000 +#define _MALI_NOTIFICATION_SUBSYSTEM_SHIFT 16 +#define _MALI_NOTIFICATION_ID_MASK 0x0000FFFF +#define _MALI_NOTIFICATION_ID_SHIFT 0 + + +/** @brief Enumeration of possible settings which match mali_setting_t in user space + * + * + */ +typedef enum { + _MALI_UK_USER_SETTING_SW_EVENTS_ENABLE = 0, + _MALI_UK_USER_SETTING_COLORBUFFER_CAPTURE_ENABLED, + _MALI_UK_USER_SETTING_DEPTHBUFFER_CAPTURE_ENABLED, + _MALI_UK_USER_SETTING_STENCILBUFFER_CAPTURE_ENABLED, + _MALI_UK_USER_SETTING_PER_TILE_COUNTERS_CAPTURE_ENABLED, + _MALI_UK_USER_SETTING_BUFFER_CAPTURE_COMPOSITOR, + _MALI_UK_USER_SETTING_BUFFER_CAPTURE_WINDOW, + _MALI_UK_USER_SETTING_BUFFER_CAPTURE_OTHER, + _MALI_UK_USER_SETTING_BUFFER_CAPTURE_N_FRAMES, + _MALI_UK_USER_SETTING_BUFFER_CAPTURE_RESIZE_FACTOR, + _MALI_UK_USER_SETTING_SW_COUNTER_ENABLED, + _MALI_UK_USER_SETTING_MAX, +} _mali_uk_user_setting_t; + +/* See mali_user_settings_db.c */ +extern const char *_mali_uk_user_setting_descriptions[]; +#define _MALI_UK_USER_SETTING_DESCRIPTIONS \ + { \ + "sw_events_enable", \ + "colorbuffer_capture_enable", \ + "depthbuffer_capture_enable", \ + "stencilbuffer_capture_enable", \ + "per_tile_counters_enable", \ + "buffer_capture_compositor", \ + "buffer_capture_window", \ + "buffer_capture_other", \ + "buffer_capture_n_frames", \ + "buffer_capture_resize_factor", \ + "sw_counters_enable", \ + }; + +/** @brief struct to hold the value to a particular setting as seen in the kernel space + */ +typedef struct { + _mali_uk_user_setting_t setting; + u32 value; +} _mali_uk_settings_changed_s; + +/** @brief Arguments for _mali_ukk_wait_for_notification() + * + * On successful return from _mali_ukk_wait_for_notification(), the members of + * this structure will indicate the reason for notification. + * + * Specifically, the source of the notification can be identified by the + * subsystem and id fields of the mali_uk_notification_type in the code.type + * member. The type member is encoded in a way to divide up the types into a + * subsystem field, and a per-subsystem ID field. See + * _mali_uk_notification_type for more information. + * + * Interpreting the data union member depends on the notification type: + * + * - type == _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS + * - The kernel side is shutting down. No further + * _mali_uk_wait_for_notification() calls should be made. + * - In this case, the value of the data union member is undefined. + * - This is used to indicate to the user space client that it should close + * the connection to the Mali Device Driver. + * - type == _MALI_NOTIFICATION_PP_FINISHED + * - The notification data is of type _mali_uk_pp_job_finished_s. It contains the user_job_ptr + * identifier used to start the job with, the job status, the number of milliseconds the job took to render, + * and values of core registers when the job finished (irq status, performance counters, renderer list + * address). + * - A job has finished succesfully when its status member is _MALI_UK_JOB_STATUS_FINISHED. + * - If the hardware detected a timeout while rendering the job, or software detected the job is + * taking more than watchdog_msecs (see _mali_ukk_pp_start_job()) to complete, the status member will + * indicate _MALI_UK_JOB_STATUS_HANG. + * - If the hardware detected a bus error while accessing memory associated with the job, status will + * indicate _MALI_UK_JOB_STATUS_SEG_FAULT. + * - Status will indicate MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to stop the job but the job + * didn't start the hardware yet, e.g. when the driver closes. + * - type == _MALI_NOTIFICATION_GP_FINISHED + * - The notification data is of type _mali_uk_gp_job_finished_s. The notification is similar to that of + * type == _MALI_NOTIFICATION_PP_FINISHED, except that several other GP core register values are returned. + * The status values have the same meaning for type == _MALI_NOTIFICATION_PP_FINISHED. + * - type == _MALI_NOTIFICATION_GP_STALLED + * - The nofication data is of type _mali_uk_gp_job_suspended_s. It contains the user_job_ptr + * identifier used to start the job with, the reason why the job stalled and a cookie to identify the core on + * which the job stalled. + * - The reason member of gp_job_suspended is set to _MALIGP_JOB_SUSPENDED_OUT_OF_MEMORY + * when the polygon list builder unit has run out of memory. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_notification_type type; /**< [out] Type of notification available */ + union { + _mali_uk_gp_job_suspended_s gp_job_suspended;/**< [out] Notification data for _MALI_NOTIFICATION_GP_STALLED notification type */ + _mali_uk_gp_job_finished_s gp_job_finished; /**< [out] Notification data for _MALI_NOTIFICATION_GP_FINISHED notification type */ + _mali_uk_pp_job_finished_s pp_job_finished; /**< [out] Notification data for _MALI_NOTIFICATION_PP_FINISHED notification type */ + _mali_uk_settings_changed_s setting_changed;/**< [out] Notification data for _MALI_NOTIFICAATION_SETTINGS_CHANGED notification type */ + _mali_uk_soft_job_activated_s soft_job_activated; /**< [out] Notification data for _MALI_NOTIFICATION_SOFT_ACTIVATED notification type */ + _mali_uk_annotate_profiling_mem_counter_s profiling_mem_counter; + _mali_uk_annotate_profiling_enable_s profiling_enable; + } data; +} _mali_uk_wait_for_notification_s; + +/** @brief Arguments for _mali_ukk_post_notification() + * + * Posts the specified notification to the notification queue for this application. + * This is used to send a quit message to the callback thread. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_notification_type type; /**< [in] Type of notification to post */ +} _mali_uk_post_notification_s; + +/** @} */ /* end group _mali_uk_waitfornotification_s */ + +/** @defgroup _mali_uk_getapiversion_s Get API Version + * @{ */ + +/** helpers for Device Driver API version handling */ + +/** @brief Encode a version ID from a 16-bit input + * + * @note the input is assumed to be 16 bits. It must not exceed 16 bits. */ +#define _MAKE_VERSION_ID(x) (((x) << 16UL) | (x)) + +/** @brief Check whether a 32-bit value is likely to be Device Driver API + * version ID. */ +#define _IS_VERSION_ID(x) (((x) & 0xFFFF) == (((x) >> 16UL) & 0xFFFF)) + +/** @brief Decode a 16-bit version number from a 32-bit Device Driver API version + * ID */ +#define _GET_VERSION(x) (((x) >> 16UL) & 0xFFFF) + +/** @brief Determine whether two 32-bit encoded version IDs match */ +#define _IS_API_MATCH(x, y) (IS_VERSION_ID((x)) && IS_VERSION_ID((y)) && (GET_VERSION((x)) == GET_VERSION((y)))) + /** + * RK MALI version code + */ +#define _MALI_RK_LIBS_VERSION 1 + +/** + * API version define. + * Indicates the version of the kernel API + * The version is a 16bit integer incremented on each API change. + * The 16bit integer is stored twice in a 32bit integer + * For example, for version 1 the value would be 0x00010001 + */ +#define _MALI_API_VERSION 900 +#define _MALI_UK_API_VERSION _MAKE_VERSION_ID(_MALI_API_VERSION) + +/** + * The API version is a 16-bit integer stored in both the lower and upper 16-bits + * of a 32-bit value. The 16-bit API version value is incremented on each API + * change. Version 1 would be 0x00010001. Used in _mali_uk_get_api_version_s. + */ +typedef u32 _mali_uk_api_version; + +/** @brief Arguments for _mali_uk_get_api_version() + * + * The user-side interface version must be written into the version member, + * encoded using _MAKE_VERSION_ID(). It will be compared to the API version of + * the kernel-side interface. + * + * On successful return, the version member will be the API version of the + * kernel-side interface. _MALI_UK_API_VERSION macro defines the current version + * of the API. + * + * The compatible member must be checked to see if the version of the user-side + * interface is compatible with the kernel-side interface, since future versions + * of the interface may be backwards compatible. + */ +typedef struct { + u32 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ + int compatible; /**< [out] @c 1 when @version is compatible, @c 0 otherwise */ +} _mali_uk_get_api_version_s; + +/** @brief Arguments for _mali_uk_get_api_version_v2() + * + * The user-side interface version must be written into the version member, + * encoded using _MAKE_VERSION_ID(). It will be compared to the API version of + * the kernel-side interface. + * + * On successful return, the version member will be the API version of the + * kernel-side interface. _MALI_UK_API_VERSION macro defines the current version + * of the API. + * + * The compatible member must be checked to see if the version of the user-side + * interface is compatible with the kernel-side interface, since future versions + * of the interface may be backwards compatible. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ + int compatible; /**< [out] @c 1 when @version is compatible, @c 0 otherwise */ +} _mali_uk_get_api_version_v2_s; + +typedef struct +{ + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ +} _mali_uk_get_mali_version_in_rk30_s; + +/* rk_ext : rk_ko_ver_t. */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ +} _mali_rk_ko_version_s; +/** @} */ /* end group _mali_uk_getapiversion_s */ + +/** @defgroup _mali_uk_get_user_settings_s Get user space settings */ + +/** @brief struct to keep the matching values of the user space settings within certain context + * + * Each member of the settings array corresponds to a matching setting in the user space and its value is the value + * of that particular setting. + * + * All settings are given reference to the context pointed to by the ctx pointer. + * + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 settings[_MALI_UK_USER_SETTING_MAX]; /**< [out] The values for all settings */ +} _mali_uk_get_user_settings_s; + +/** @brief struct to hold the value of a particular setting from the user space within a given context + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_user_setting_t setting; /**< [in] setting to get */ + u32 value; /**< [out] value of setting */ +} _mali_uk_get_user_setting_s; + +/** @brief Arguments for _mali_ukk_request_high_priority() */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ +} _mali_uk_request_high_priority_s; + +/** @brief Arguments for _mali_ukk_pending_submit() */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ +} _mali_uk_pending_submit_s; + +/** @} */ /* end group _mali_uk_core */ + + +/** @defgroup _mali_uk_memory U/K Memory + * @{ */ + +#define _MALI_MEMORY_ALLOCATE_RESIZEABLE (1<<4) /* BUFFER can trim dow/grow*/ +#define _MALI_MEMORY_ALLOCATE_NO_BIND_GPU (1<<5) /*Not map to GPU when allocate, must call bind later*/ +#define _MALI_MEMORY_ALLOCATE_SWAPPABLE (1<<6) /* Allocate swappale memory. */ +#define _MALI_MEMORY_ALLOCATE_DEFER_BIND (1<<7) /*Not map to GPU when allocate, must call bind later*/ +#define _MALI_MEMORY_ALLOCATE_SECURE (1<<8) /* Allocate secure memory. */ + + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 gpu_vaddr; /**< [in] GPU virtual address */ + u32 vsize; /**< [in] vitrual size of the allocation */ + u32 psize; /**< [in] physical size of the allocation */ + u32 flags; + u64 backend_handle; /**< [out] backend handle */ + s32 secure_shared_fd; /** < [in] the mem handle for secure mem */ +} _mali_uk_alloc_mem_s; + + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 gpu_vaddr; /**< [in] use as handle to free allocation */ + u32 free_pages_nr; /** < [out] record the number of free pages */ +} _mali_uk_free_mem_s; + + +#define _MALI_MEMORY_BIND_BACKEND_UMP (1<<8) +#define _MALI_MEMORY_BIND_BACKEND_DMA_BUF (1<<9) +#define _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY (1<<10) +#define _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY (1<<11) +#define _MALI_MEMORY_BIND_BACKEND_EXT_COW (1<<12) +#define _MALI_MEMORY_BIND_BACKEND_HAVE_ALLOCATION (1<<13) + + +#define _MALI_MEMORY_BIND_BACKEND_MASK (_MALI_MEMORY_BIND_BACKEND_UMP| \ + _MALI_MEMORY_BIND_BACKEND_DMA_BUF |\ + _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY |\ + _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY |\ + _MALI_MEMORY_BIND_BACKEND_EXT_COW |\ + _MALI_MEMORY_BIND_BACKEND_HAVE_ALLOCATION) + + +#define _MALI_MEMORY_GPU_READ_ALLOCATE (1<<16) + + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 vaddr; /**< [in] mali address to map the physical memory to */ + u32 size; /**< [in] size */ + u32 flags; /**< [in] see_MALI_MEMORY_BIND_BACKEND_* */ + u32 padding; /** padding for 32/64 struct alignment */ + union { + struct { + u32 secure_id; /**< [in] secure id */ + u32 rights; /**< [in] rights necessary for accessing memory */ + u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ + } bind_ump; + struct { + u32 mem_fd; /**< [in] Memory descriptor */ + u32 rights; /**< [in] rights necessary for accessing memory */ + u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ + } bind_dma_buf; + struct { + u32 phys_addr; /**< [in] physical address */ + u32 rights; /**< [in] rights necessary for accessing memory */ + u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ + } bind_ext_memory; + } mem_union; +} _mali_uk_bind_mem_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 flags; /**< [in] see_MALI_MEMORY_BIND_BACKEND_* */ + u32 vaddr; /**< [in] identifier for mapped memory object in kernel space */ +} _mali_uk_unbind_mem_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 target_handle; /**< [in] handle of allocation need to do COW */ + u32 target_offset; /**< [in] offset in target allocation to do COW(for support COW a memory allocated from memory_bank, PAGE_SIZE align)*/ + u32 target_size; /**< [in] size of target allocation to do COW (for support memory bank, PAGE_SIZE align)(in byte) */ + u32 range_start; /**< [in] re allocate range start offset, offset from the start of allocation (PAGE_SIZE align)*/ + u32 range_size; /**< [in] re allocate size (PAGE_SIZE align)*/ + u32 vaddr; /**< [in] mali address for the new allocaiton */ + u32 backend_handle; /**< [out] backend handle */ + u32 flags; +} _mali_uk_cow_mem_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 range_start; /**< [in] re allocate range start offset, offset from the start of allocation */ + u32 size; /**< [in] re allocate size*/ + u32 vaddr; /**< [in] mali address for the new allocaiton */ + s32 change_pages_nr; /**< [out] record the page number change for cow operation */ +} _mali_uk_cow_modify_range_s; + + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 mem_fd; /**< [in] Memory descriptor */ + u32 size; /**< [out] size */ +} _mali_uk_dma_buf_get_size_s; + +/** Flag for _mali_uk_map_external_mem_s, _mali_uk_attach_ump_mem_s and _mali_uk_attach_dma_buf_s */ +#define _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE (1<<0) + + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 vaddr; /* the buffer to do resize*/ + u32 psize; /* wanted physical size of this memory */ +} _mali_uk_mem_resize_s; + +/** + * @brief Arguments for _mali_uk[uk]_mem_write_safe() + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 src; /**< [in] Pointer to source data */ + u64 dest; /**< [in] Destination Mali buffer */ + u32 size; /**< [in,out] Number of bytes to write/copy on input, number of bytes actually written/copied on output */ +} _mali_uk_mem_write_safe_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 size; /**< [out] size of MMU page table information (registers + page tables) */ +} _mali_uk_query_mmu_page_table_dump_size_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 size; /**< [in] size of buffer to receive mmu page table information */ + u64 buffer; /**< [in,out] buffer to receive mmu page table information */ + u32 register_writes_size; /**< [out] size of MMU register dump */ + u64 register_writes; /**< [out] pointer within buffer where MMU register dump is stored */ + u32 page_table_dump_size; /**< [out] size of MMU page table dump */ + u64 page_table_dump; /**< [out] pointer within buffer where MMU page table dump is stored */ +} _mali_uk_dump_mmu_page_table_s; + +/** @} */ /* end group _mali_uk_memory */ + + +/** @addtogroup _mali_uk_pp U/K Fragment Processor + * @{ */ + +/** @brief Arguments for _mali_ukk_get_pp_number_of_cores() + * + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * - Upon successful return from _mali_ukk_get_pp_number_of_cores(), @c number_of_cores + * will contain the number of Fragment Processor cores in the system. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 number_of_total_cores; /**< [out] Total number of Fragment Processor cores in the system */ + u32 number_of_enabled_cores; /**< [out] Number of enabled Fragment Processor cores */ +} _mali_uk_get_pp_number_of_cores_s; + +/** @brief Arguments for _mali_ukk_get_pp_core_version() + * + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * - Upon successful return from _mali_ukk_get_pp_core_version(), @c version contains + * the version that all Fragment Processor cores are compatible with. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_core_version version; /**< [out] version returned from core, see \ref _mali_core_version */ + u32 padding; +} _mali_uk_get_pp_core_version_s; + +/** @} */ /* end group _mali_uk_pp */ + + +/** @addtogroup _mali_uk_gp U/K Vertex Processor + * @{ */ + +/** @brief Arguments for _mali_ukk_get_gp_number_of_cores() + * + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * - Upon successful return from _mali_ukk_get_gp_number_of_cores(), @c number_of_cores + * will contain the number of Vertex Processor cores in the system. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 number_of_cores; /**< [out] number of Vertex Processor cores in the system */ +} _mali_uk_get_gp_number_of_cores_s; + +/** @brief Arguments for _mali_ukk_get_gp_core_version() + * + * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() + * - Upon successful return from _mali_ukk_get_gp_core_version(), @c version contains + * the version that all Vertex Processor cores are compatible with. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_core_version version; /**< [out] version returned from core, see \ref _mali_core_version */ +} _mali_uk_get_gp_core_version_s; + +/** @} */ /* end group _mali_uk_gp */ + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 event_id; /**< [in] event id to register (see enum mali_profiling_events for values) */ + u32 data[5]; /**< [in] event specific data */ +} _mali_uk_profiling_add_event_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 memory_usage; /**< [out] total memory usage */ + u32 vaddr; /**< [in] mali address for the cow allocaiton */ + s32 change_pages_nr; /**< [out] record the page number change for cow operation */ +} _mali_uk_profiling_memory_usage_get_s; + + +/** @addtogroup _mali_uk_memory U/K Memory + * @{ */ + +/** @brief Arguments to _mali_ukk_mem_mmap() + * + * Use of the phys_addr member depends on whether the driver is compiled for + * Mali-MMU or nonMMU: + * - in the nonMMU case, this is the physical address of the memory as seen by + * the CPU (which may be a constant offset from that used by Mali) + * - in the MMU case, this is the Mali Virtual base address of the memory to + * allocate, and the particular physical pages used to back the memory are + * entirely determined by _mali_ukk_mem_mmap(). The details of the physical pages + * are not reported to user-space for security reasons. + * + * The cookie member must be stored for use later when freeing the memory by + * calling _mali_ukk_mem_munmap(). In the Mali-MMU case, the cookie is secure. + * + * The ukk_private word must be set to zero when calling from user-space. On + * Kernel-side, the OS implementation of the U/K interface can use it to + * communicate data to the OS implementation of the OSK layer. In particular, + * _mali_ukk_get_big_block() directly calls _mali_ukk_mem_mmap directly, and + * will communicate its own ukk_private word through the ukk_private member + * here. The common code itself will not inspect or modify the ukk_private + * word, and so it may be safely used for whatever purposes necessary to + * integrate Mali Memory handling into the OS. + * + * The uku_private member is currently reserved for use by the user-side + * implementation of the U/K interface. Its value must be zero. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + void *mapping; /**< [out] Returns user-space virtual address for the mapping */ + u32 size; /**< [in] Size of the requested mapping */ + u32 phys_addr; /**< [in] Physical address - could be offset, depending on caller+callee convention */ + mali_bool writeable; +} _mali_uk_mem_mmap_s; + +/** @brief Arguments to _mali_ukk_mem_munmap() + * + * The cookie and mapping members must be that returned from the same previous + * call to _mali_ukk_mem_mmap(). The size member must correspond to cookie + * and mapping - that is, it must be the value originally supplied to a call to + * _mali_ukk_mem_mmap that returned the values of mapping and cookie. + * + * An error will be returned if an attempt is made to unmap only part of the + * originally obtained range, or to unmap more than was originally obtained. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + void *mapping; /**< [in] The mapping returned from mmap call */ + u32 size; /**< [in] The size passed to mmap call */ +} _mali_uk_mem_munmap_s; +/** @} */ /* end group _mali_uk_memory */ + +/** @defgroup _mali_uk_vsync U/K VSYNC Wait Reporting Module + * @{ */ + +/** @brief VSYNC events + * + * These events are reported when DDK starts to wait for vsync and when the + * vsync has occured and the DDK can continue on the next frame. + */ +typedef enum _mali_uk_vsync_event { + _MALI_UK_VSYNC_EVENT_BEGIN_WAIT = 0, + _MALI_UK_VSYNC_EVENT_END_WAIT +} _mali_uk_vsync_event; + +/** @brief Arguments to _mali_ukk_vsync_event() + * + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_vsync_event event; /**< [in] VSYNCH event type */ +} _mali_uk_vsync_event_report_s; + +/** @} */ /* end group _mali_uk_vsync */ + +/** @defgroup _mali_uk_sw_counters_report U/K Software Counter Reporting + * @{ */ + +/** @brief Software counter values + * + * Values recorded for each of the software counters during a single renderpass. + */ +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 counters; /**< [in] The array of u32 counter values */ + u32 num_counters; /**< [in] The number of elements in counters array */ +} _mali_uk_sw_counters_report_s; + +/** @} */ /* end group _mali_uk_sw_counters_report */ + +/** @defgroup _mali_uk_timeline U/K Mali Timeline + * @{ */ + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 timeline; /**< [in] timeline id */ + u32 point; /**< [out] latest point on timeline */ +} _mali_uk_timeline_get_latest_point_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_fence_t fence; /**< [in] fence */ + u32 timeout; /**< [in] timeout (0 for no wait, -1 for blocking) */ + u32 status; /**< [out] status of fence (1 if signaled, 0 if timeout) */ +} _mali_uk_timeline_wait_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + _mali_uk_fence_t fence; /**< [in] mali fence to create linux sync fence from */ + s32 sync_fd; /**< [out] file descriptor for new linux sync fence */ +} _mali_uk_timeline_create_sync_fence_s; + +/** @} */ /* end group _mali_uk_timeline */ + +/** @} */ /* end group u_k_api */ + +/** @} */ /* end group uddapi */ + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + s32 stream_fd; /**< [in] The profiling kernel base stream fd handle */ +} _mali_uk_profiling_stream_fd_get_s; + +typedef struct { + u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ + u64 control_packet_data; /**< [in] the control packet data for control settings */ + u32 control_packet_size; /**< [in] The control packet size */ + u64 response_packet_data; /** < [out] The response packet data */ + u32 response_packet_size; /** < [in,out] The response packet data */ +} _mali_uk_profiling_control_set_s; + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UTGARD_UK_TYPES_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h b/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h new file mode 100755 index 000000000000..6fafc6777e48 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2010, 2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_kernel_license.h + * Defines for the macro MODULE_LICENSE. + */ + +#ifndef __MALI_KERNEL_LICENSE_H__ +#define __MALI_KERNEL_LICENSE_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define MALI_KERNEL_LINUX_LICENSE "GPL" +#define MALI_LICENSE_IS_GPL 1 + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_KERNEL_LICENSE_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.c b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.c new file mode 100755 index 000000000000..260c2a8227a9 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.c @@ -0,0 +1,354 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_osk_mali.h" +#include "mali_kernel_common.h" + +#include +#include +#include +#include +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif + +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) +#include +#else /* Linux >= 3.13 */ +/* In 3.13 the OPP include header file, types, and functions were all + * renamed. Use the old filename for the include, and define the new names to + * the old, when an old kernel is detected. + */ +#include +#define dev_pm_opp opp +#define dev_pm_opp_get_voltage opp_get_voltage +#define dev_pm_opp_get_opp_count opp_get_opp_count +#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil +#endif /* Linux >= 3.13 */ + +#include "mali_pm_metrics.h" + +#include +#include + +static struct monitor_dev_profile mali_mdevp = { + .type = MONITOR_TPYE_DEV, + .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, + .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, +}; + +static struct devfreq_simple_ondemand_data ondemand_data; + +static int +mali_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) +{ + struct mali_device *mdev = dev_get_drvdata(dev); + struct dev_pm_opp *opp; + unsigned long freq = 0; + unsigned long old_freq = mdev->current_freq; + unsigned long voltage; + int err; + + freq = *target_freq; + + opp = devfreq_recommended_opp(dev, &freq, flags); + if (IS_ERR(opp)) { + MALI_PRINT_ERROR(("Failed to get opp (%ld)\n", PTR_ERR(opp))); + return PTR_ERR(opp); + } + voltage = dev_pm_opp_get_voltage(opp); + dev_pm_opp_put(opp); + + MALI_DEBUG_PRINT(2, ("mali_devfreq_target:set_freq = %lld flags = 0x%x\n", freq, flags)); + /* + * Only update if there is a change of frequency + */ + if (old_freq == freq) { + *target_freq = freq; + mali_pm_reset_dvfs_utilisation(mdev); +#ifdef CONFIG_REGULATOR + if (mdev->current_voltage == voltage) + return 0; + err = regulator_set_voltage(mdev->regulator, voltage, INT_MAX); + if (err) { + dev_err(dev, "Failed to set voltage (%d)\n", err); + return err; + } + mdev->current_voltage = voltage; +#endif + return 0; + } + +#ifdef CONFIG_REGULATOR + if (mdev->regulator && mdev->current_voltage != voltage && + old_freq < freq) { + err = regulator_set_voltage(mdev->regulator, voltage, INT_MAX); + if (err) { + MALI_PRINT_ERROR(("Failed to increase voltage (%d)\n", err)); + return err; + } + } +#endif + + err = clk_set_rate(mdev->clock, freq); + if (err) { + MALI_PRINT_ERROR(("Failed to set clock %lu (target %lu)\n", freq, *target_freq)); + return err; + } + *target_freq = freq; + mdev->current_freq = freq; + if (mdev->devfreq) + mdev->devfreq->last_status.current_frequency = freq; + +#ifdef CONFIG_REGULATOR + if (mdev->regulator && mdev->current_voltage != voltage && + old_freq > freq) { + err = regulator_set_voltage(mdev->regulator, voltage, INT_MAX); + if (err) { + MALI_PRINT_ERROR(("Failed to decrease voltage (%d)\n", err)); + return err; + } + } +#endif + + mdev->current_voltage = voltage; + + mali_pm_reset_dvfs_utilisation(mdev); + + return err; +} + +static int +mali_devfreq_cur_freq(struct device *dev, unsigned long *freq) +{ + struct mali_device *mdev = dev_get_drvdata(dev); + + *freq = mdev->current_freq; + + MALI_DEBUG_PRINT(2, ("mali_devfreq_cur_freq: freq = %d \n", *freq)); + return 0; +} + +static int +mali_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) +{ + struct mali_device *mdev = dev_get_drvdata(dev); + + stat->current_frequency = mdev->current_freq; + + mali_pm_get_dvfs_utilisation(mdev, + &stat->total_time, &stat->busy_time); + + stat->private_data = NULL; + +#ifdef CONFIG_DEVFREQ_THERMAL + memcpy(&mdev->devfreq->last_status, stat, sizeof(*stat)); +#endif + + return 0; +} + +/* setup platform specific opp in platform.c*/ +int __weak setup_opps(void) +{ + return 0; +} + +/* term platform specific opp in platform.c*/ +int __weak term_opps(struct device *dev) +{ + return 0; +} + +static int mali_devfreq_init_freq_table(struct mali_device *mdev, + struct devfreq_dev_profile *dp) +{ + int err, count; + int i = 0; + unsigned long freq = 0; + struct dev_pm_opp *opp; + + err = setup_opps(); + if (err) + return err; + + count = dev_pm_opp_get_opp_count(mdev->dev); + if (count < 0) { + return count; + } + + MALI_DEBUG_PRINT(2, ("mali devfreq table count %d\n", count)); + + dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), + GFP_KERNEL); + if (!dp->freq_table) + return -ENOMEM; + + for (i = 0; i < count; i++, freq++) { + opp = dev_pm_opp_find_freq_ceil(mdev->dev, &freq); + if (IS_ERR(opp)) + break; + dev_pm_opp_put(opp); + + dp->freq_table[i] = freq; + MALI_DEBUG_PRINT(2, ("mali devfreq table array[%d] = %d\n", i, freq)); + } + + if (count != i) + MALI_PRINT_ERROR(("Unable to enumerate all OPPs (%d!=%d)\n", + count, i)); + + dp->max_state = i; + + return 0; +} + +static void mali_devfreq_term_freq_table(struct mali_device *mdev) +{ + struct devfreq_dev_profile *dp = mdev->devfreq->profile; + + kfree(dp->freq_table); + term_opps(mdev->dev); +} + +static void mali_devfreq_exit(struct device *dev) +{ + struct mali_device *mdev = dev_get_drvdata(dev); + + mali_devfreq_term_freq_table(mdev); +} + +int mali_devfreq_init(struct mali_device *mdev) +{ + struct device_node *np = mdev->dev->of_node; +#ifdef CONFIG_DEVFREQ_THERMAL + struct devfreq_cooling_power *callbacks = NULL; + _mali_osk_device_data data; +#endif + struct devfreq_dev_profile *dp; + struct dev_pm_opp *opp; + unsigned long opp_rate; + int err; + + MALI_DEBUG_PRINT(2, ("Init Mali devfreq\n")); + + if (!mdev->clock) + return -ENODEV; + + mdev->current_freq = clk_get_rate(mdev->clock); + + dp = &mdev->devfreq_profile; + + dp->initial_freq = mdev->current_freq; + dp->polling_ms = 100; + dp->target = mali_devfreq_target; + dp->get_dev_status = mali_devfreq_status; + dp->get_cur_freq = mali_devfreq_cur_freq; + dp->exit = mali_devfreq_exit; + + if (mali_devfreq_init_freq_table(mdev, dp)) + return -EFAULT; + + of_property_read_u32(np, "upthreshold", + &ondemand_data.upthreshold); + of_property_read_u32(np, "downdifferential", + &ondemand_data.downdifferential); + + mdev->devfreq = devfreq_add_device(mdev->dev, dp, + "simple_ondemand", &ondemand_data); + if (IS_ERR(mdev->devfreq)) { + mali_devfreq_term_freq_table(mdev); + return PTR_ERR(mdev->devfreq); + } + + err = devfreq_register_opp_notifier(mdev->dev, mdev->devfreq); + if (err) { + MALI_PRINT_ERROR(("Failed to register OPP notifier (%d)\n", err)); + goto opp_notifier_failed; + } + + opp_rate = mdev->current_freq; + opp = devfreq_recommended_opp(mdev->dev, &opp_rate, 0); + if (!IS_ERR(opp)) + dev_pm_opp_put(opp); + mdev->devfreq->last_status.current_frequency = opp_rate; + + mali_mdevp.data = mdev->devfreq; + mdev->mdev_info = rockchip_system_monitor_register(mdev->dev, + &mali_mdevp); + if (IS_ERR(mdev->mdev_info)) { + dev_dbg(mdev->dev, "without system monitor\n"); + mdev->mdev_info = NULL; + } +#ifdef CONFIG_DEVFREQ_THERMAL + if (of_machine_is_compatible("rockchip,rk3036")) + return 0; + + /* Initilization last_status it will be used when first power allocate called */ + mdev->devfreq->last_status.current_frequency = mdev->current_freq; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + if (NULL != data.gpu_cooling_ops) { + callbacks = data.gpu_cooling_ops; + MALI_DEBUG_PRINT(2, ("Mali GPU Thermal: Callback handler installed \n")); + } + } + + if (callbacks) { + mdev->devfreq_cooling = of_devfreq_cooling_register_power( + mdev->dev->of_node, + mdev->devfreq, + callbacks); + if (IS_ERR_OR_NULL(mdev->devfreq_cooling)) { + err = PTR_ERR(mdev->devfreq_cooling); + MALI_PRINT_ERROR(("Failed to register cooling device (%d)\n", err)); + goto cooling_failed; + } else { + MALI_DEBUG_PRINT(2, ("Mali GPU Thermal Cooling installed \n")); + } + } +#endif + + return 0; + +#ifdef CONFIG_DEVFREQ_THERMAL +cooling_failed: + devfreq_unregister_opp_notifier(mdev->dev, mdev->devfreq); +#endif /* CONFIG_DEVFREQ_THERMAL */ +opp_notifier_failed: + err = devfreq_remove_device(mdev->devfreq); + if (err) + MALI_PRINT_ERROR(("Failed to terminate devfreq (%d)\n", err)); + else + mdev->devfreq = NULL; + + return err; +} + +void mali_devfreq_term(struct mali_device *mdev) +{ + int err; + + MALI_DEBUG_PRINT(2, ("Term Mali devfreq\n")); + + rockchip_system_monitor_unregister(mdev->mdev_info); +#ifdef CONFIG_DEVFREQ_THERMAL + devfreq_cooling_unregister(mdev->devfreq_cooling); +#endif + + devfreq_unregister_opp_notifier(mdev->dev, mdev->devfreq); + + err = devfreq_remove_device(mdev->devfreq); + if (err) + MALI_PRINT_ERROR(("Failed to terminate devfreq (%d)\n", err)); + else + mdev->devfreq = NULL; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.h b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.h new file mode 100755 index 000000000000..ba7c017d88dc --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#ifndef _MALI_DEVFREQ_H_ +#define _MALI_DEVFREQ_H_ + +int mali_devfreq_init(struct mali_device *mdev); + +void mali_devfreq_term(struct mali_device *mdev); + +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c b/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c new file mode 100755 index 000000000000..95c3ea12d645 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c @@ -0,0 +1,36 @@ +/** + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_device_pause_resume.c + * Implementation of the Mali pause/resume functionality + */ + +#include +#include +#include "mali_pm.h" + +void mali_dev_pause(void) +{ + /* + * Deactive all groups to prevent hardware being touched + * during the period of mali device pausing + */ + mali_pm_os_suspend(MALI_FALSE); +} + +EXPORT_SYMBOL(mali_dev_pause); + +void mali_dev_resume(void) +{ + mali_pm_os_resume(); +} + +EXPORT_SYMBOL(mali_dev_resume); diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c new file mode 100755 index 000000000000..e026e11e4bc5 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c @@ -0,0 +1,439 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include "mali_osk.h" +#include "mali_kernel_common.h" + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include "mali_dma_fence.h" +#include +#include +#endif + +static DEFINE_SPINLOCK(mali_dma_fence_lock); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static bool mali_dma_fence_enable_signaling(struct dma_fence *fence) +{ + MALI_IGNORE(fence); + return true; +} + +static const char *mali_dma_fence_get_driver_name(struct dma_fence *fence) +{ + MALI_IGNORE(fence); + return "mali"; +} + +static const char *mali_dma_fence_get_timeline_name(struct dma_fence *fence) +{ + MALI_IGNORE(fence); + return "mali_dma_fence"; +} + +static const struct dma_fence_ops mali_dma_fence_ops = { + .get_driver_name = mali_dma_fence_get_driver_name, + .get_timeline_name = mali_dma_fence_get_timeline_name, + .enable_signaling = mali_dma_fence_enable_signaling, + .signaled = NULL, + .wait = dma_fence_default_wait, + .release = NULL +}; +#else +static bool mali_dma_fence_enable_signaling(struct fence *fence) +{ + MALI_IGNORE(fence); + return true; +} + +static const char *mali_dma_fence_get_driver_name(struct fence *fence) +{ + MALI_IGNORE(fence); + return "mali"; +} + +static const char *mali_dma_fence_get_timeline_name(struct fence *fence) +{ + MALI_IGNORE(fence); + return "mali_dma_fence"; +} + +static const struct fence_ops mali_dma_fence_ops = { + .get_driver_name = mali_dma_fence_get_driver_name, + .get_timeline_name = mali_dma_fence_get_timeline_name, + .enable_signaling = mali_dma_fence_enable_signaling, + .signaled = NULL, + .wait = fence_default_wait, + .release = NULL +}; +#endif + +static void mali_dma_fence_context_cleanup(struct mali_dma_fence_context *dma_fence_context) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(dma_fence_context); + + for (i = 0; i < dma_fence_context->num_dma_fence_waiter; i++) { + if (dma_fence_context->mali_dma_fence_waiters[i]) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_remove_callback(dma_fence_context->mali_dma_fence_waiters[i]->fence, + &dma_fence_context->mali_dma_fence_waiters[i]->base); + dma_fence_put(dma_fence_context->mali_dma_fence_waiters[i]->fence); + +#else + fence_remove_callback(dma_fence_context->mali_dma_fence_waiters[i]->fence, + &dma_fence_context->mali_dma_fence_waiters[i]->base); + fence_put(dma_fence_context->mali_dma_fence_waiters[i]->fence); +#endif + kfree(dma_fence_context->mali_dma_fence_waiters[i]); + dma_fence_context->mali_dma_fence_waiters[i] = NULL; + } + } + + if (NULL != dma_fence_context->mali_dma_fence_waiters) + kfree(dma_fence_context->mali_dma_fence_waiters); + + dma_fence_context->mali_dma_fence_waiters = NULL; + dma_fence_context->num_dma_fence_waiter = 0; +} + +static void mali_dma_fence_context_work_func(struct work_struct *work_handle) +{ + struct mali_dma_fence_context *dma_fence_context; + + MALI_DEBUG_ASSERT_POINTER(work_handle); + + dma_fence_context = container_of(work_handle, struct mali_dma_fence_context, work_handle); + + dma_fence_context->cb_func(dma_fence_context->pp_job_ptr); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static void mali_dma_fence_callback(struct dma_fence *fence, struct dma_fence_cb *cb) +#else +static void mali_dma_fence_callback(struct fence *fence, struct fence_cb *cb) +#endif +{ + struct mali_dma_fence_waiter *dma_fence_waiter = NULL; + struct mali_dma_fence_context *dma_fence_context = NULL; + + MALI_DEBUG_ASSERT_POINTER(fence); + MALI_DEBUG_ASSERT_POINTER(cb); + + MALI_IGNORE(fence); + + dma_fence_waiter = container_of(cb, struct mali_dma_fence_waiter, base); + dma_fence_context = dma_fence_waiter->parent; + + MALI_DEBUG_ASSERT_POINTER(dma_fence_context); + + if (atomic_dec_and_test(&dma_fence_context->count)) + schedule_work(&dma_fence_context->work_handle); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static _mali_osk_errcode_t mali_dma_fence_add_callback(struct mali_dma_fence_context *dma_fence_context, struct dma_fence *fence) +#else +static _mali_osk_errcode_t mali_dma_fence_add_callback(struct mali_dma_fence_context *dma_fence_context, struct fence *fence) +#endif +{ + int ret = 0; + struct mali_dma_fence_waiter *dma_fence_waiter; + struct mali_dma_fence_waiter **dma_fence_waiters; + + MALI_DEBUG_ASSERT_POINTER(dma_fence_context); + MALI_DEBUG_ASSERT_POINTER(fence); + + dma_fence_waiters = krealloc(dma_fence_context->mali_dma_fence_waiters, + (dma_fence_context->num_dma_fence_waiter + 1) + * sizeof(struct mali_dma_fence_waiter *), + GFP_KERNEL); + + if (NULL == dma_fence_waiters) { + MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to realloc the dma fence waiters.\n")); + return _MALI_OSK_ERR_NOMEM; + } + + dma_fence_context->mali_dma_fence_waiters = dma_fence_waiters; + + dma_fence_waiter = kzalloc(sizeof(struct mali_dma_fence_waiter), GFP_KERNEL); + + if (NULL == dma_fence_waiter) { + MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to create mali dma fence waiter.\n")); + return _MALI_OSK_ERR_NOMEM; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_get(fence); +#else + fence_get(fence); +#endif + dma_fence_waiter->fence = fence; + dma_fence_waiter->parent = dma_fence_context; + atomic_inc(&dma_fence_context->count); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + ret = dma_fence_add_callback(fence, &dma_fence_waiter->base, + mali_dma_fence_callback); +#else + ret = fence_add_callback(fence, &dma_fence_waiter->base, + mali_dma_fence_callback); +#endif + if (0 > ret) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_put(fence); +#else + fence_put(fence); +#endif + kfree(dma_fence_waiter); + atomic_dec(&dma_fence_context->count); + if (-ENOENT == ret) { + /*-ENOENT if fence has already been signaled, return _MALI_OSK_ERR_OK*/ + return _MALI_OSK_ERR_OK; + } + /* Failed to add the fence callback into fence, return _MALI_OSK_ERR_FAULT*/ + MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into fence.\n")); + return _MALI_OSK_ERR_FAULT; + } + + dma_fence_context->mali_dma_fence_waiters[dma_fence_context->num_dma_fence_waiter] = dma_fence_waiter; + dma_fence_context->num_dma_fence_waiter++; + + return _MALI_OSK_ERR_OK; +} + + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +struct dma_fence *mali_dma_fence_new(u32 context, u32 seqno) +#else +struct fence *mali_dma_fence_new(u32 context, u32 seqno) +#endif +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + struct dma_fence *fence = NULL; + fence = kzalloc(sizeof(struct dma_fence), GFP_KERNEL); +#else + struct fence *fence = NULL; + fence = kzalloc(sizeof(struct fence), GFP_KERNEL); +#endif + if (NULL == fence) { + MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to create dma fence.\n")); + return fence; + } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_init(fence, + &mali_dma_fence_ops, + &mali_dma_fence_lock, + context, seqno); +#else + fence_init(fence, + &mali_dma_fence_ops, + &mali_dma_fence_lock, + context, seqno); +#endif + return fence; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +void mali_dma_fence_signal_and_put(struct dma_fence **fence) +#else +void mali_dma_fence_signal_and_put(struct fence **fence) +#endif +{ + MALI_DEBUG_ASSERT_POINTER(fence); + MALI_DEBUG_ASSERT_POINTER(*fence); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_signal(*fence); + dma_fence_put(*fence); +#else + fence_signal(*fence); + fence_put(*fence); +#endif + *fence = NULL; +} + +void mali_dma_fence_context_init(struct mali_dma_fence_context *dma_fence_context, + mali_dma_fence_context_callback_func_t cb_func, + void *pp_job_ptr) +{ + MALI_DEBUG_ASSERT_POINTER(dma_fence_context); + + INIT_WORK(&dma_fence_context->work_handle, mali_dma_fence_context_work_func); + atomic_set(&dma_fence_context->count, 1); + dma_fence_context->num_dma_fence_waiter = 0; + dma_fence_context->mali_dma_fence_waiters = NULL; + dma_fence_context->cb_func = cb_func; + dma_fence_context->pp_job_ptr = pp_job_ptr; +} + +_mali_osk_errcode_t mali_dma_fence_context_add_waiters(struct mali_dma_fence_context *dma_fence_context, + struct reservation_object *dma_reservation_object) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; + u32 shared_count = 0, i; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + struct dma_fence *exclusive_fence = NULL; + struct dma_fence **shared_fences = NULL; +#else + struct fence *exclusive_fence = NULL; + struct fence **shared_fences = NULL; +#endif + MALI_DEBUG_ASSERT_POINTER(dma_fence_context); + MALI_DEBUG_ASSERT_POINTER(dma_reservation_object); + + /* Get all the shared/exclusive fences in the reservation object of dma buf*/ + ret = reservation_object_get_fences_rcu(dma_reservation_object, &exclusive_fence, + &shared_count, &shared_fences); + if (ret < 0) { + MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to get shared or exclusive_fence dma fences from the reservation object of dma buf.\n")); + return _MALI_OSK_ERR_FAULT; + } + + if (exclusive_fence) { + ret = mali_dma_fence_add_callback(dma_fence_context, exclusive_fence); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into exclusive fence.\n")); + mali_dma_fence_context_cleanup(dma_fence_context); + goto ended; + } + } + + + for (i = 0; i < shared_count; i++) { + ret = mali_dma_fence_add_callback(dma_fence_context, shared_fences[i]); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into shared fence [%d].\n", i)); + mali_dma_fence_context_cleanup(dma_fence_context); + break; + } + } + +ended: + + if (exclusive_fence) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_put(exclusive_fence); +#else + fence_put(exclusive_fence); +#endif + + if (shared_fences) { + for (i = 0; i < shared_count; i++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_put(shared_fences[i]); +#else + fence_put(shared_fences[i]); +#endif + } + kfree(shared_fences); + } + + return ret; +} + + +void mali_dma_fence_context_term(struct mali_dma_fence_context *dma_fence_context) +{ + MALI_DEBUG_ASSERT_POINTER(dma_fence_context); + atomic_set(&dma_fence_context->count, 0); + if (dma_fence_context->work_handle.func) { + cancel_work_sync(&dma_fence_context->work_handle); + } + mali_dma_fence_context_cleanup(dma_fence_context); +} + +void mali_dma_fence_context_dec_count(struct mali_dma_fence_context *dma_fence_context) +{ + MALI_DEBUG_ASSERT_POINTER(dma_fence_context); + + if (atomic_dec_and_test(&dma_fence_context->count)) + schedule_work(&dma_fence_context->work_handle); +} + + +void mali_dma_fence_add_reservation_object_list(struct reservation_object *dma_reservation_object, + struct reservation_object **dma_reservation_object_list, + u32 *num_dma_reservation_object) +{ + u32 i; + + MALI_DEBUG_ASSERT_POINTER(dma_reservation_object); + MALI_DEBUG_ASSERT_POINTER(dma_reservation_object_list); + MALI_DEBUG_ASSERT_POINTER(num_dma_reservation_object); + + for (i = 0; i < *num_dma_reservation_object; i++) { + if (dma_reservation_object_list[i] == dma_reservation_object) + return; + } + + dma_reservation_object_list[*num_dma_reservation_object] = dma_reservation_object; + (*num_dma_reservation_object)++; +} + +int mali_dma_fence_lock_reservation_object_list(struct reservation_object **dma_reservation_object_list, + u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx) +{ + u32 i; + + struct reservation_object *reservation_object_to_slow_lock = NULL; + + MALI_DEBUG_ASSERT_POINTER(dma_reservation_object_list); + MALI_DEBUG_ASSERT_POINTER(ww_actx); + + ww_acquire_init(ww_actx, &reservation_ww_class); + +again: + for (i = 0; i < num_dma_reservation_object; i++) { + int ret; + + if (dma_reservation_object_list[i] == reservation_object_to_slow_lock) { + reservation_object_to_slow_lock = NULL; + continue; + } + + ret = ww_mutex_lock(&dma_reservation_object_list[i]->lock, ww_actx); + + if (ret < 0) { + u32 slow_lock_index = i; + + /* unlock all pre locks we have already locked.*/ + while (i > 0) { + i--; + ww_mutex_unlock(&dma_reservation_object_list[i]->lock); + } + + if (NULL != reservation_object_to_slow_lock) + ww_mutex_unlock(&reservation_object_to_slow_lock->lock); + + if (ret == -EDEADLK) { + reservation_object_to_slow_lock = dma_reservation_object_list[slow_lock_index]; + ww_mutex_lock_slow(&reservation_object_to_slow_lock->lock, ww_actx); + goto again; + } + ww_acquire_fini(ww_actx); + MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to lock all dma reservation objects.\n", i)); + return ret; + } + } + + ww_acquire_done(ww_actx); + return 0; +} + +void mali_dma_fence_unlock_reservation_object_list(struct reservation_object **dma_reservation_object_list, + u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx) +{ + u32 i; + + for (i = 0; i < num_dma_reservation_object; i++) + ww_mutex_unlock(&dma_reservation_object_list[i]->lock); + + ww_acquire_fini(ww_actx); +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.h b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.h new file mode 100755 index 000000000000..d44f6d1a8926 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.h @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_dma_fence.h + * + * Mali interface for Linux dma buf fence objects. + */ + +#ifndef _MALI_DMA_FENCE_H_ +#define _MALI_DMA_FENCE_H_ + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +#include +#else +#include +#endif +#include +#endif + +struct mali_dma_fence_context; + +/* The mali dma fence context callback function */ +typedef void (*mali_dma_fence_context_callback_func_t)(void *pp_job_ptr); + +struct mali_dma_fence_waiter { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + struct dma_fence *fence; + struct dma_fence_cb base; +#else + struct fence_cb base; + struct fence *fence; +#endif + struct mali_dma_fence_context *parent; +}; + +struct mali_dma_fence_context { + struct work_struct work_handle; + struct mali_dma_fence_waiter **mali_dma_fence_waiters; + u32 num_dma_fence_waiter; + atomic_t count; + void *pp_job_ptr; /* the mali pp job pointer */; + mali_dma_fence_context_callback_func_t cb_func; +}; + +/* Create a dma fence + * @param context The execution context this fence is run on + * @param seqno A linearly increasing sequence number for this context + * @return the new dma fence if success, or NULL on failure. + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +struct dma_fence *mali_dma_fence_new(u32 context, u32 seqno); +#else +struct fence *mali_dma_fence_new(u32 context, u32 seqno); +#endif +/* Signal and put dma fence + * @param fence The dma fence to signal and put + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +void mali_dma_fence_signal_and_put(struct dma_fence **fence); +#else +void mali_dma_fence_signal_and_put(struct fence **fence); +#endif +/** + * Initialize a mali dma fence context for pp job. + * @param dma_fence_context The mali dma fence context to initialize. + * @param cb_func The dma fence context callback function to call when all dma fence release. + * @param pp_job_ptr The pp_job to call function with. + */ +void mali_dma_fence_context_init(struct mali_dma_fence_context *dma_fence_context, + mali_dma_fence_context_callback_func_t cb_func, + void *pp_job_ptr); + +/** + * Add new mali dma fence waiter into mali dma fence context + * @param dma_fence_context The mali dma fence context + * @param dma_reservation_object the reservation object to create new mali dma fence waiters + * @return _MALI_OSK_ERR_OK if success, or not. + */ +_mali_osk_errcode_t mali_dma_fence_context_add_waiters(struct mali_dma_fence_context *dma_fence_context, + struct reservation_object *dma_reservation_object); + +/** + * Release the dma fence context + * @param dma_fence_text The mali dma fence context. + */ +void mali_dma_fence_context_term(struct mali_dma_fence_context *dma_fence_context); + +/** + * Decrease the dma fence context atomic count + * @param dma_fence_text The mali dma fence context. + */ +void mali_dma_fence_context_dec_count(struct mali_dma_fence_context *dma_fence_context); + +/** + * Get all reservation object + * @param dma_reservation_object The reservation object to add into the reservation object list + * @param dma_reservation_object_list The reservation object list to store all reservation object + * @param num_dma_reservation_object The number of all reservation object + */ +void mali_dma_fence_add_reservation_object_list(struct reservation_object *dma_reservation_object, + struct reservation_object **dma_reservation_object_list, + u32 *num_dma_reservation_object); + +/** + * Wait/wound mutex lock to lock all reservation object. + */ +int mali_dma_fence_lock_reservation_object_list(struct reservation_object **dma_reservation_object_list, + u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx); + +/** + * Wait/wound mutex lock to unlock all reservation object. + */ +void mali_dma_fence_unlock_reservation_object_list(struct reservation_object **dma_reservation_object_list, + u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx); +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.c b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.c new file mode 100755 index 000000000000..e13cbad3e513 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.c @@ -0,0 +1,783 @@ +/* + * Copyright (C) 2012-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_internal_sync.h" +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_kernel_common.h" +#if defined(DEBUG) +#include "mali_session.h" +#include "mali_timeline.h" +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static const struct dma_fence_ops fence_ops; +#else +static const struct fence_ops fence_ops; +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static struct mali_internal_sync_point *mali_internal_fence_to_sync_pt(struct dma_fence *fence) +#else +static struct mali_internal_sync_point *mali_internal_fence_to_sync_pt(struct fence *fence) +#endif +{ + MALI_DEBUG_ASSERT_POINTER(fence); + return container_of(fence, struct mali_internal_sync_point, base); +} + +static inline struct mali_internal_sync_timeline *mali_internal_sync_pt_to_sync_timeline(struct mali_internal_sync_point *sync_pt) +{ + MALI_DEBUG_ASSERT_POINTER(sync_pt); + return container_of(sync_pt->base.lock, struct mali_internal_sync_timeline, sync_pt_list_lock); +} + +static void mali_internal_sync_timeline_free(struct kref *kref_count) +{ + struct mali_internal_sync_timeline *sync_timeline; + + MALI_DEBUG_ASSERT_POINTER(kref_count); + + sync_timeline = container_of(kref_count, struct mali_internal_sync_timeline, kref_count); + + if (sync_timeline->ops->release_obj) + sync_timeline->ops->release_obj(sync_timeline); + + kfree(sync_timeline); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) +static void mali_internal_fence_check_cb_func(struct fence *fence, struct fence_cb *cb) +#else +static void mali_internal_fence_check_cb_func(struct dma_fence *fence, struct dma_fence_cb *cb) +#endif +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + struct mali_internal_sync_fence_cb *check; +#else + struct mali_internal_sync_fence_waiter *waiter; +#endif + struct mali_internal_sync_fence *sync_fence; + int ret; + MALI_DEBUG_ASSERT_POINTER(cb); + MALI_IGNORE(fence); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + check = container_of(cb, struct mali_internal_sync_fence_cb, cb); + sync_fence = check->sync_file; +#else + waiter = container_of(cb, struct mali_internal_sync_fence_waiter, cb); + sync_fence = (struct mali_internal_sync_fence *)waiter->work.private; +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + ret = atomic_dec_and_test(&sync_fence->status); + if (ret) + wake_up_all(&sync_fence->wq); +#else + ret = sync_fence->fence->ops->signaled(sync_fence->fence); + + if (0 > ret) + MALI_PRINT_ERROR(("Mali internal sync:Failed to wait fence 0x%x for sync_fence 0x%x.\n", fence, sync_fence)); + if (1 == ret) + wake_up_all(&sync_fence->wq); +#endif +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) +static void mali_internal_sync_fence_add_fence(struct mali_internal_sync_fence *sync_fence, struct fence *sync_pt) +{ + int fence_num = 0; + MALI_DEBUG_ASSERT_POINTER(sync_fence); + MALI_DEBUG_ASSERT_POINTER(sync_pt); + + fence_num = sync_fence->num_fences; + + sync_fence->cbs[fence_num].fence = sync_pt; + sync_fence->cbs[fence_num].sync_file = sync_fence; + + if (!fence_add_callback(sync_pt, &sync_fence->cbs[fence_num].cb, mali_internal_fence_check_cb_func)) { + fence_get(sync_pt); + sync_fence->num_fences++; + atomic_inc(&sync_fence->status); + } +} +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) +static int mali_internal_sync_fence_wake_up_wq(wait_queue_entry_t *curr, unsigned mode, + int wake_flags, void *key) +#else +static int mali_internal_sync_fence_wake_up_wq(wait_queue_t *curr, unsigned mode, + int wake_flags, void *key) +#endif +{ + struct mali_internal_sync_fence_waiter *wait; + MALI_IGNORE(mode); + MALI_IGNORE(wake_flags); + MALI_IGNORE(key); + + wait = container_of(curr, struct mali_internal_sync_fence_waiter, work); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) + list_del_init(&wait->work.entry); +#else + list_del_init(&wait->work.task_list); +#endif + wait->callback(wait->work.private, wait); + return 1; +} + +struct mali_internal_sync_timeline *mali_internal_sync_timeline_create(const struct mali_internal_sync_timeline_ops *ops, + int size, const char *name) +{ + struct mali_internal_sync_timeline *sync_timeline = NULL; + + MALI_DEBUG_ASSERT_POINTER(ops); + + if (size < sizeof(struct mali_internal_sync_timeline)) { + MALI_PRINT_ERROR(("Mali internal sync:Invalid size to create the mali internal sync timeline.\n")); + goto err; + } + + sync_timeline = kzalloc(size, GFP_KERNEL); + if (NULL == sync_timeline) { + MALI_PRINT_ERROR(("Mali internal sync:Failed to allocate buffer for the mali internal sync timeline.\n")); + goto err; + } + kref_init(&sync_timeline->kref_count); + sync_timeline->ops = ops; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + sync_timeline->fence_context = dma_fence_context_alloc(1); +#else + sync_timeline->fence_context = fence_context_alloc(1); +#endif + strlcpy(sync_timeline->name, name, sizeof(sync_timeline->name)); + + INIT_LIST_HEAD(&sync_timeline->sync_pt_list_head); + spin_lock_init(&sync_timeline->sync_pt_list_lock); + + return sync_timeline; +err: + if (NULL != sync_timeline) { + kfree(sync_timeline); + } + return NULL; +} + +void mali_internal_sync_timeline_destroy(struct mali_internal_sync_timeline *sync_timeline) +{ + MALI_DEBUG_ASSERT_POINTER(sync_timeline); + + sync_timeline->destroyed = MALI_TRUE; + + smp_wmb(); + + mali_internal_sync_timeline_signal(sync_timeline); + kref_put(&sync_timeline->kref_count, mali_internal_sync_timeline_free); +} + +void mali_internal_sync_timeline_signal(struct mali_internal_sync_timeline *sync_timeline) +{ + unsigned long flags; + struct mali_internal_sync_point *sync_pt, *next; + + MALI_DEBUG_ASSERT_POINTER(sync_timeline); + + spin_lock_irqsave(&sync_timeline->sync_pt_list_lock, flags); + + list_for_each_entry_safe(sync_pt, next, &sync_timeline->sync_pt_list_head, + sync_pt_list) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + if (dma_fence_is_signaled_locked(&sync_pt->base)) +#else + if (fence_is_signaled_locked(&sync_pt->base)) +#endif + list_del_init(&sync_pt->sync_pt_list); + } + + spin_unlock_irqrestore(&sync_timeline->sync_pt_list_lock, flags); +} + +struct mali_internal_sync_point *mali_internal_sync_point_create(struct mali_internal_sync_timeline *sync_timeline, int size) +{ + unsigned long flags; + struct mali_internal_sync_point *sync_pt = NULL; + + MALI_DEBUG_ASSERT_POINTER(sync_timeline); + + if (size < sizeof(struct mali_internal_sync_point)) { + MALI_PRINT_ERROR(("Mali internal sync:Invalid size to create the mali internal sync point.\n")); + goto err; + } + + sync_pt = kzalloc(size, GFP_KERNEL); + if (NULL == sync_pt) { + MALI_PRINT_ERROR(("Mali internal sync:Failed to allocate buffer for the mali internal sync point.\n")); + goto err; + } + spin_lock_irqsave(&sync_timeline->sync_pt_list_lock, flags); + kref_get(&sync_timeline->kref_count); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_init(&sync_pt->base, &fence_ops, &sync_timeline->sync_pt_list_lock, + sync_timeline->fence_context, ++sync_timeline->value); +#else + fence_init(&sync_pt->base, &fence_ops, &sync_timeline->sync_pt_list_lock, + sync_timeline->fence_context, ++sync_timeline->value); +#endif + INIT_LIST_HEAD(&sync_pt->sync_pt_list); + spin_unlock_irqrestore(&sync_timeline->sync_pt_list_lock, flags); + + return sync_pt; +err: + if (NULL != sync_pt) { + kfree(sync_pt); + } + return NULL; +} + +struct mali_internal_sync_fence *mali_internal_sync_fence_fdget(int fd) +{ + struct file *file = fget(fd); + + if (NULL == file) { + return NULL; + } + + return file->private_data; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) +struct mali_internal_sync_fence *mali_internal_sync_fence_merge( + struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2) +{ + struct mali_internal_sync_fence *new_sync_fence; + int i, j, num_fence1, num_fence2, total_fences; + struct fence *fence0 = NULL; + + MALI_DEBUG_ASSERT_POINTER(sync_fence1); + MALI_DEBUG_ASSERT_POINTER(sync_fence2); + + num_fence1 = sync_fence1->num_fences; + num_fence2 = sync_fence2->num_fences; + + total_fences = num_fence1 + num_fence2; + + i = 0; + j = 0; + + if (num_fence1 > 0) { + fence0 = sync_fence1->cbs[i].fence; + i = 1; + } else if (num_fence2 > 0) { + fence0 = sync_fence2->cbs[i].fence; + j = 1; + } + + new_sync_fence = (struct mali_internal_sync_fence *)sync_file_create(fence0); + if (NULL == new_sync_fence) { + MALI_PRINT_ERROR(("Mali internal sync:Failed to create the mali internal sync fence when merging sync fence.\n")); + return NULL; + } + + fence_remove_callback(new_sync_fence->cb[0].fence, &new_sync_fence->cb[0].cb); + new_sync_fence->num_fences = 0; + atomic_dec(&new_sync_fence->status); + + for (; i < num_fence1 && j < num_fence2;) { + struct fence *fence1 = sync_fence1->cbs[i].fence; + struct fence *fence2 = sync_fence2->cbs[j].fence; + + if (fence1->context < fence2->context) { + mali_internal_sync_fence_add_fence(new_sync_fence, fence1); + + i++; + } else if (fence1->context > fence2->context) { + mali_internal_sync_fence_add_fence(new_sync_fence, fence2); + + j++; + } else { + if (fence1->seqno - fence2->seqno <= INT_MAX) + mali_internal_sync_fence_add_fence(new_sync_fence, fence1); + else + mali_internal_sync_fence_add_fence(new_sync_fence, fence2); + i++; + j++; + } + } + + for (; i < num_fence1; i++) + mali_internal_sync_fence_add_fence(new_sync_fence, sync_fence1->cbs[i].fence); + + for (; j < num_fence2; j++) + mali_internal_sync_fence_add_fence(new_sync_fence, sync_fence2->cbs[j].fence); + + return new_sync_fence; +} +#else +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) +static struct fence **mali_internal_get_fences(struct mali_internal_sync_fence *sync_fence, int *num_fences) +#else +static struct dma_fence **mali_internal_get_fences(struct mali_internal_sync_fence *sync_fence, int *num_fences) +#endif +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + if (sync_fence->fence->ops == &fence_array_ops) { + struct fence_array *fence_array = container_of(sync_fence->fence, struct fence_array, base); + *num_fences = fence_array->num_fences; + return fence_array->fences; + } +#else + if (sync_fence->fence->ops == &dma_fence_array_ops) { + struct dma_fence_array *fence_array = container_of(sync_fence->fence, struct dma_fence_array, base); + *num_fences = fence_array->num_fences; + return fence_array->fences; + } +#endif + *num_fences = 1; + return &sync_fence->fence; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) +static void mali_internal_add_fence_array(struct fence **fences, int *num_fences, struct fence *fence) +#else +static void mali_internal_add_fence_array(struct dma_fence **fences, int *num_fences, struct dma_fence *fence) +#endif +{ + fences[*num_fences] = fence; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + if (!fence_is_signaled(fence)) { + fence_get(fence); + (*num_fences)++; + } +#else + if (!dma_fence_is_signaled(fence)) { + dma_fence_get(fence); + (*num_fences)++; + } +#endif +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) +static int mali_internal_sync_fence_set_fence_array(struct mali_internal_sync_fence *sync_fence, + struct fence **fences, int num_fences) +#else +static int mali_internal_sync_fence_set_fence_array(struct mali_internal_sync_fence *sync_fence, + struct dma_fence **fences, int num_fences) +#endif +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + struct fence_array *array; +#else + struct dma_fence_array *array; +#endif + if(num_fences == 1) { + sync_fence->fence =fences[0]; + kfree(fences); + } else { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + array = fence_array_create(num_fences, fences, + fence_context_alloc(1), 1, false); +#else + array = dma_fence_array_create(num_fences, fences, + dma_fence_context_alloc(1), 1, false); +#endif + if (!array){ + return -ENOMEM; + } + sync_fence->fence = &array->base; + } + return 0; +} + +struct mali_internal_sync_fence *mali_internal_sync_fence_merge( + struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2) +{ + struct mali_internal_sync_fence *sync_fence; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + struct fence **fences, **nfences, **fences1, **fences2; +#else + struct dma_fence **fences, **nfences, **fences1, **fences2; +#endif + int real_num_fences, i, j, num_fences, num_fences1, num_fences2; + + fences1 = mali_internal_get_fences(sync_fence1, &num_fences1); + fences2 = mali_internal_get_fences(sync_fence2, &num_fences2); + + num_fences = num_fences1 + num_fences2; + + fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL); + if (!fences) { + MALI_PRINT_ERROR(("Mali internal sync:Failed to alloc buffer for fences.\n")); + goto fences_alloc_failed; + } + + for (real_num_fences = i = j = 0; i < num_fences1 && j < num_fences2;) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + struct fence *fence1 = fences1[i]; + struct fence *fence2 = fences2[j]; +#else + struct dma_fence *fence1 = fences1[i]; + struct dma_fence *fence2 = fences2[j]; +#endif + if (fence1->context < fence2->context) { + mali_internal_add_fence_array(fences, &real_num_fences, fence1); + + i++; + } else if (fence1->context > fence2->context) { + mali_internal_add_fence_array(fences, &real_num_fences, fence2); + + j++; + } else { + if (fence1->seqno - fence2->seqno <= INT_MAX) + mali_internal_add_fence_array(fences, &real_num_fences, fence1); + else + mali_internal_add_fence_array(fences, &real_num_fences, fence2); + + i++; + j++; + } + } + + for (; i < num_fences1; i++) + mali_internal_add_fence_array(fences, &real_num_fences, fences1[i]); + + for (; j < num_fences2; j++) + mali_internal_add_fence_array(fences, &real_num_fences, fences2[j]); + + if (0 == real_num_fences) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + fences[real_num_fences++] = fence_get(fences1[0]); +#else + fences[real_num_fences++] = dma_fence_get(fences1[0]); +#endif + + if (num_fences > real_num_fences) { + nfences = krealloc(fences, real_num_fences * sizeof(*fences), + GFP_KERNEL); + if (!nfences) + goto nfences_alloc_failed; + + fences = nfences; + } + + sync_fence = (struct mali_internal_sync_fence *)sync_file_create(fences[0]); + if (NULL == sync_fence) { + MALI_PRINT_ERROR(("Mali internal sync:Failed to create the mali internal sync fence when merging sync fence.\n")); + goto sync_fence_alloc_failed; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + fence_put(fences[0]); +#else + dma_fence_put(fences[0]); +#endif + + if (mali_internal_sync_fence_set_fence_array(sync_fence, fences, real_num_fences) < 0) { + MALI_PRINT_ERROR(("Mali internal sync:Failed to set fence for sync fence.\n")); + goto sync_fence_set_failed; + } + + return sync_fence; + +sync_fence_set_failed: + fput(sync_fence->file); +sync_fence_alloc_failed: + for (i = 0; i < real_num_fences; i++) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + fence_put(fences[i]); +#else + dma_fence_put(fences[i]); +#endif +nfences_alloc_failed: + kfree(fences); +fences_alloc_failed: + return NULL; +} +#endif + +void mali_internal_sync_fence_waiter_init(struct mali_internal_sync_fence_waiter *waiter, + mali_internal_sync_callback_t callback) +{ + MALI_DEBUG_ASSERT_POINTER(waiter); + MALI_DEBUG_ASSERT_POINTER(callback); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) + INIT_LIST_HEAD(&waiter->work.entry); +#else + INIT_LIST_HEAD(&waiter->work.task_list); +#endif + waiter->callback = callback; +} + +int mali_internal_sync_fence_wait_async(struct mali_internal_sync_fence *sync_fence, + struct mali_internal_sync_fence_waiter *waiter) +{ + int err; + unsigned long flags; + + MALI_DEBUG_ASSERT_POINTER(sync_fence); + MALI_DEBUG_ASSERT_POINTER(waiter); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + err = atomic_read(&sync_fence->status); + + if (0 > err) + return err; + + if (!err) + return 1; + + init_waitqueue_func_entry(&waiter->work, mali_internal_sync_fence_wake_up_wq); + waiter->work.private = sync_fence; + + spin_lock_irqsave(&sync_fence->wq.lock, flags); + err = atomic_read(&sync_fence->status); + + if (0 < err) + __add_wait_queue_tail(&sync_fence->wq, &waiter->work); + spin_unlock_irqrestore(&sync_fence->wq.lock, flags); + + if (0 > err) + return err; + + return !err; +#else + if ((sync_fence->fence) && (sync_fence->fence->ops) && (sync_fence->fence->ops->signaled)) + err = sync_fence->fence->ops->signaled(sync_fence->fence); + else + err = -1; + + if (0 > err) + return err; + + if (1 == err) + return err; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + err = dma_fence_add_callback(sync_fence->fence, &waiter->cb, mali_internal_fence_check_cb_func); +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) + err = fence_add_callback(sync_fence->fence, &waiter->cb, mali_internal_fence_check_cb_func); +#endif + + if (0 != err) { + if (-ENOENT == err) + err = 1; + return err; + } + init_waitqueue_func_entry(&waiter->work, mali_internal_sync_fence_wake_up_wq); + waiter->work.private = sync_fence; + + spin_lock_irqsave(&sync_fence->wq.lock, flags); + err = sync_fence->fence->ops->signaled(sync_fence->fence); + + if (0 == err){ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) + __add_wait_queue_entry_tail(&sync_fence->wq, &waiter->work); +#else + __add_wait_queue_tail(&sync_fence->wq, &waiter->work); +#endif + } + spin_unlock_irqrestore(&sync_fence->wq.lock, flags); + + return err; +#endif +} + +int mali_internal_sync_fence_cancel_async(struct mali_internal_sync_fence *sync_fence, + struct mali_internal_sync_fence_waiter *waiter) +{ + unsigned long flags; + int ret = 0; + + MALI_DEBUG_ASSERT_POINTER(sync_fence); + MALI_DEBUG_ASSERT_POINTER(waiter); + + spin_lock_irqsave(&sync_fence->wq.lock, flags); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) + if (!list_empty(&waiter->work.entry)) + list_del_init(&waiter->work.entry); +#else + if (!list_empty(&waiter->work.task_list)) + list_del_init(&waiter->work.task_list); +#endif + else + ret = -ENOENT; + spin_unlock_irqrestore(&sync_fence->wq.lock, flags); + + if (0 == ret) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_remove_callback(sync_fence->fence, &waiter->cb); +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) + fence_remove_callback(sync_fence->fence, &waiter->cb); +#endif + + } + + return ret; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static const char *mali_internal_fence_get_driver_name(struct dma_fence *fence) +#else +static const char *mali_internal_fence_get_driver_name(struct fence *fence) +#endif +{ + struct mali_internal_sync_point *sync_pt; + struct mali_internal_sync_timeline *parent; + + MALI_DEBUG_ASSERT_POINTER(fence); + + sync_pt = mali_internal_fence_to_sync_pt(fence); + parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); + + return parent->ops->driver_name; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static const char *mali_internal_fence_get_timeline_name(struct dma_fence *fence) +#else +static const char *mali_internal_fence_get_timeline_name(struct fence *fence) +#endif +{ + struct mali_internal_sync_point *sync_pt; + struct mali_internal_sync_timeline *parent; + + MALI_DEBUG_ASSERT_POINTER(fence); + + sync_pt = mali_internal_fence_to_sync_pt(fence); + parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); + + return parent->name; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static void mali_internal_fence_release(struct dma_fence *fence) +#else +static void mali_internal_fence_release(struct fence *fence) +#endif +{ + unsigned long flags; + struct mali_internal_sync_point *sync_pt; + struct mali_internal_sync_timeline *parent; + + MALI_DEBUG_ASSERT_POINTER(fence); + + sync_pt = mali_internal_fence_to_sync_pt(fence); + parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); + + + spin_lock_irqsave(fence->lock, flags); + if (WARN_ON_ONCE(!list_empty(&sync_pt->sync_pt_list))) + list_del(&sync_pt->sync_pt_list); + spin_unlock_irqrestore(fence->lock, flags); + + if (parent->ops->free_pt) + parent->ops->free_pt(sync_pt); + + kref_put(&parent->kref_count, mali_internal_sync_timeline_free); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_free(&sync_pt->base); +#else + fence_free(&sync_pt->base); +#endif +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static bool mali_internal_fence_signaled(struct dma_fence *fence) +#else +static bool mali_internal_fence_signaled(struct fence *fence) +#endif +{ + int ret; + struct mali_internal_sync_point *sync_pt; + struct mali_internal_sync_timeline *parent; + + MALI_DEBUG_ASSERT_POINTER(fence); + + sync_pt = mali_internal_fence_to_sync_pt(fence); + parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); + + ret = parent->ops->has_signaled(sync_pt); + if (0 > ret) +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) \ + || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 68))) + fence->error = ret; +#else + fence->status = ret; +#endif + return ret; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static bool mali_internal_fence_enable_signaling(struct dma_fence *fence) +#else +static bool mali_internal_fence_enable_signaling(struct fence *fence) +#endif +{ + struct mali_internal_sync_point *sync_pt; + struct mali_internal_sync_timeline *parent; + + MALI_DEBUG_ASSERT_POINTER(fence); + + sync_pt = mali_internal_fence_to_sync_pt(fence); + parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); + + if (mali_internal_fence_signaled(fence)) + return false; + + list_add_tail(&sync_pt->sync_pt_list, &parent->sync_pt_list_head); + return true; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static void mali_internal_fence_value_str(struct dma_fence *fence, char *str, int size) +#else +static void mali_internal_fence_value_str(struct fence *fence, char *str, int size) +#endif +{ + struct mali_internal_sync_point *sync_pt; + struct mali_internal_sync_timeline *parent; + + MALI_DEBUG_ASSERT_POINTER(fence); + MALI_IGNORE(str); + MALI_IGNORE(size); + + sync_pt = mali_internal_fence_to_sync_pt(fence); + parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); + + parent->ops->print_sync_pt(sync_pt); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static const struct dma_fence_ops fence_ops = { +#else +static const struct fence_ops fence_ops = { +#endif + .get_driver_name = mali_internal_fence_get_driver_name, + .get_timeline_name = mali_internal_fence_get_timeline_name, + .enable_signaling = mali_internal_fence_enable_signaling, + .signaled = mali_internal_fence_signaled, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + .wait = dma_fence_default_wait, +#else + .wait = fence_default_wait, +#endif + .release = mali_internal_fence_release, + .fence_value_str = mali_internal_fence_value_str, +}; +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.h b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.h new file mode 100755 index 000000000000..dbb29222ba98 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.h @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2012-2015, 2017-2018 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_internal_sync.h + * + * Mali internal structure/interface for sync. + */ + +#ifndef _MALI_INTERNAL_SYNC_H +#define _MALI_INTERNAL_SYNC_H +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) +#include +#else +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +#include +#else +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) +#include +#else +#include +#endif +#endif + +struct mali_internal_sync_timeline; +struct mali_internal_sync_point; +struct mali_internal_sync_fence; + +struct mali_internal_sync_timeline_ops { + const char *driver_name; + int (*has_signaled)(struct mali_internal_sync_point *pt); + void (*free_pt)(struct mali_internal_sync_point *sync_pt); + void (*release_obj)(struct mali_internal_sync_timeline *sync_timeline); + void (*print_sync_pt)(struct mali_internal_sync_point *sync_pt); +}; + +struct mali_internal_sync_timeline { + struct kref kref_count; + const struct mali_internal_sync_timeline_ops *ops; + char name[32]; + bool destroyed; + int fence_context; + int value; + spinlock_t sync_pt_list_lock; + struct list_head sync_pt_list_head; +}; + +struct mali_internal_sync_point { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + struct dma_fence base; +#else + struct fence base; +#endif + struct list_head sync_pt_list; +}; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) +struct mali_internal_sync_fence_cb { + struct fence_cb cb; + struct fence *fence; + struct mali_internal_sync_fence *sync_file; +}; +#endif + +struct mali_internal_sync_fence { + struct file *file; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) + struct kref kref; +#endif + char name[32]; +#ifdef CONFIG_DEBUG_FS + struct list_head sync_file_list; +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + int num_fences; +#endif + wait_queue_head_t wq; +#if LINUX_VERSION_CODE > KERNEL_VERSION(4, 12, 0) + unsigned long flags; +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + atomic_t status; + struct mali_internal_sync_fence_cb cbs[]; +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + struct fence *fence; + struct fence_cb cb; +#else + struct dma_fence *fence; + struct dma_fence_cb cb; +#endif +}; + +struct mali_internal_sync_fence_waiter; + +typedef void (*mali_internal_sync_callback_t)(struct mali_internal_sync_fence *sync_fence, + struct mali_internal_sync_fence_waiter *waiter); + +struct mali_internal_sync_fence_waiter { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) + wait_queue_entry_t work; +#else + wait_queue_t work; +#endif + mali_internal_sync_callback_t callback; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + struct fence_cb cb; +#else + struct dma_fence_cb cb; +#endif +#endif +}; + +/** + * Create a mali internal sync timeline. + * @param ops The implementation ops for the mali internal sync timeline + * @param size The size to allocate + * @param name The sync_timeline name + * @return The new mali internal sync timeline if successful, NULL if not. + */ +struct mali_internal_sync_timeline *mali_internal_sync_timeline_create(const struct mali_internal_sync_timeline_ops *ops, + int size, const char *name); + +/** + * Destroy one mali internal sync timeline. + * @param sync_timeline The mali internal sync timeline to destroy. + */ +void mali_internal_sync_timeline_destroy(struct mali_internal_sync_timeline *sync_timeline); + +/** + * Signal one mali internal sync timeline. + * @param sync_timeline The mali internal sync timeline to signal. + */ +void mali_internal_sync_timeline_signal(struct mali_internal_sync_timeline *sync_timeline); + +/** + * Create one mali internal sync point. + * @param sync_timeline The mali internal sync timeline to add this mali internal sync point. + * @return the new mali internal sync point if successful, NULL if not. + */ +struct mali_internal_sync_point *mali_internal_sync_point_create(struct mali_internal_sync_timeline *sync_timeline, int size); + +/** + * Merge mali internal sync fences + * @param sync_fence1 The mali internal sync fence to merge + * @param sync_fence2 The mali internal sync fence to merge + * @return the new mali internal sync fence if successful, NULL if not. + */ +struct mali_internal_sync_fence *mali_internal_sync_fence_merge(struct mali_internal_sync_fence *sync_fence1, + struct mali_internal_sync_fence *sync_fence2); + +/** + * Get the mali internal sync fence from sync fd + * @param fd The sync handle to get the mali internal sync fence + * @return the mali internal sync fence if successful, NULL if not. + */ +struct mali_internal_sync_fence *mali_internal_sync_fence_fdget(int fd); + + +void mali_internal_sync_fence_waiter_init(struct mali_internal_sync_fence_waiter *waiter, + mali_internal_sync_callback_t callback); + +int mali_internal_sync_fence_wait_async(struct mali_internal_sync_fence *sync_fence, + struct mali_internal_sync_fence_waiter *waiter); + +int mali_internal_sync_fence_cancel_async(struct mali_internal_sync_fence *sync_fence, + struct mali_internal_sync_fence_waiter *waiter); + +#endif /*LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)*/ +#endif /* _MALI_INTERNAL_SYNC_H */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c new file mode 100755 index 000000000000..e45c7d2f2b1e --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c @@ -0,0 +1,1154 @@ +/** + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + + +/** + * @file mali_kernel_linux.c + * Implementation of the Linux device driver entrypoints + */ +#include "../platform/rk/custom_log.h" +#include "../platform/rk/rk_ext.h" + +#include /* kernel module definitions */ +#include /* file system operations */ +#include /* character device definitions */ +#include /* memory manager definitions */ +#include +#include +#include +#include "mali_kernel_license.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_kernel_core.h" +#include "mali_osk.h" +#include "mali_kernel_linux.h" +#include "mali_ukk.h" +#include "mali_ukk_wrappers.h" +#include "mali_kernel_sysfs.h" +#include "mali_pm.h" +#include "mali_kernel_license.h" +#include "mali_memory.h" +#include "mali_memory_dma_buf.h" +#include "mali_memory_manager.h" +#include "mali_memory_swap_alloc.h" +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) +#include "mali_profiling_internal.h" +#endif +#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) +#include "mali_osk_profiling.h" +#include "mali_dvfs_policy.h" + +static int is_first_resume = 1; +/*Store the clk and vol for boot/insmod and mali_resume*/ +static struct mali_gpu_clk_item mali_gpu_clk[2]; +#endif + +/* Streamline support for the Mali driver */ +#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_MALI400_PROFILING) +/* Ask Linux to create the tracepoints */ +#define CREATE_TRACE_POINTS +#include "mali_linux_trace.h" + +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_event); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_hw_counter); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_sw_counters); +#endif /* CONFIG_TRACEPOINTS */ + +#ifdef CONFIG_MALI_DEVFREQ +#include "mali_devfreq.h" +#include "mali_osk_mali.h" + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#include +#else +/* In 3.13 the OPP include header file, types, and functions were all + * renamed. Use the old filename for the include, and define the new names to + * the old, when an old kernel is detected. + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) +#include +#else +#include +#endif /* Linux >= 3.13*/ +#define dev_pm_opp_of_add_table of_init_opp_table +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) +#define dev_pm_opp_of_remove_table of_free_opp_table +#endif /* Linux >= 3.19 */ +#endif /* Linux >= 4.4.0 */ +#endif + +/* from the __malidrv_build_info.c file that is generated during build */ +extern const char *__malidrv_build_info(void); + +/* Module parameter to control log level */ +int mali_debug_level = 2; +module_param(mali_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */ +MODULE_PARM_DESC(mali_debug_level, "Higher number, more dmesg output"); + +extern int mali_max_job_runtime; +module_param(mali_max_job_runtime, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_max_job_runtime, "Maximum allowed job runtime in msecs.\nJobs will be killed after this no matter what"); + +extern int mali_l2_max_reads; +module_param(mali_l2_max_reads, int, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_l2_max_reads, "Maximum reads for Mali L2 cache"); + +extern unsigned int mali_dedicated_mem_start; +module_param(mali_dedicated_mem_start, uint, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_dedicated_mem_start, "Physical start address of dedicated Mali GPU memory."); + +extern unsigned int mali_dedicated_mem_size; +module_param(mali_dedicated_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_dedicated_mem_size, "Size of dedicated Mali GPU memory."); + +extern unsigned int mali_shared_mem_size; +module_param(mali_shared_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_shared_mem_size, "Size of shared Mali GPU memory."); + +#if defined(CONFIG_MALI400_PROFILING) +extern int mali_boot_profiling; +module_param(mali_boot_profiling, int, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_boot_profiling, "Start profiling as a part of Mali driver initialization"); +#endif + +extern int mali_max_pp_cores_group_1; +module_param(mali_max_pp_cores_group_1, int, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_max_pp_cores_group_1, "Limit the number of PP cores to use from first PP group."); + +extern int mali_max_pp_cores_group_2; +module_param(mali_max_pp_cores_group_2, int, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_max_pp_cores_group_2, "Limit the number of PP cores to use from second PP group (Mali-450 only)."); + +extern unsigned int mali_mem_swap_out_threshold_value; +module_param(mali_mem_swap_out_threshold_value, uint, S_IRUSR | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_mem_swap_out_threshold_value, "Threshold value used to limit how much swappable memory cached in Mali driver."); + +#if defined(CONFIG_MALI_DVFS) +/** the max fps the same as display vsync default 60, can set by module insert parameter */ +extern int mali_max_system_fps; +module_param(mali_max_system_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_max_system_fps, "Max system fps the same as display VSYNC."); + +/** a lower limit on their desired FPS default 58, can set by module insert parameter*/ +extern int mali_desired_fps; +module_param(mali_desired_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); +MODULE_PARM_DESC(mali_desired_fps, "A bit lower than max_system_fps which user desired fps"); +#endif + +#if MALI_ENABLE_CPU_CYCLES +#include +#include +#include +static struct timer_list mali_init_cpu_clock_timers[8]; +static u32 mali_cpu_clock_last_value[8] = {0,}; +#endif + +/* Export symbols from common code: mali_user_settings.c */ +#include "mali_user_settings_db.h" +EXPORT_SYMBOL(mali_set_user_setting); +EXPORT_SYMBOL(mali_get_user_setting); + +static char mali_dev_name[] = "mali"; /* should be const, but the functions we call requires non-cost */ + +/* This driver only supports one Mali device, and this variable stores this single platform device */ +struct platform_device *mali_platform_device = NULL; + +/* This driver only supports one Mali device, and this variable stores the exposed misc device (/dev/mali) */ +static struct miscdevice mali_miscdevice = { 0, }; + +static int mali_miscdevice_register(struct platform_device *pdev); +static void mali_miscdevice_unregister(void); + +static int mali_open(struct inode *inode, struct file *filp); +static int mali_release(struct inode *inode, struct file *filp); +static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); + +static int mali_probe(struct platform_device *pdev); +static int mali_remove(struct platform_device *pdev); + +static int mali_driver_suspend_scheduler(struct device *dev); +static int mali_driver_resume_scheduler(struct device *dev); + +#ifdef CONFIG_PM_RUNTIME +static int mali_driver_runtime_suspend(struct device *dev); +static int mali_driver_runtime_resume(struct device *dev); +static int mali_driver_runtime_idle(struct device *dev); +#endif + +#if defined(MALI_FAKE_PLATFORM_DEVICE) +#if defined(CONFIG_MALI_DT) +extern int mali_platform_device_init(struct platform_device *device); +extern int mali_platform_device_deinit(struct platform_device *device); +#else +extern int mali_platform_device_register(void); +extern int mali_platform_device_unregister(void); +#endif +#endif + +extern int rk_platform_init_opp_table(struct device *dev); + +/* Linux power management operations provided by the Mali device driver */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)) +struct pm_ext_ops mali_dev_ext_pm_ops = { + .base = + { + .suspend = mali_driver_suspend_scheduler, + .resume = mali_driver_resume_scheduler, + .freeze = mali_driver_suspend_scheduler, + .thaw = mali_driver_resume_scheduler, + }, +}; +#else +static const struct dev_pm_ops mali_dev_pm_ops = { +#ifdef CONFIG_PM_RUNTIME + .runtime_suspend = mali_driver_runtime_suspend, + .runtime_resume = mali_driver_runtime_resume, + .runtime_idle = mali_driver_runtime_idle, +#endif + .suspend = mali_driver_suspend_scheduler, + .resume = mali_driver_resume_scheduler, + .freeze = mali_driver_suspend_scheduler, + .thaw = mali_driver_resume_scheduler, + .poweroff = mali_driver_suspend_scheduler, +}; +#endif + +#ifdef CONFIG_MALI_DT +static struct of_device_id base_dt_ids[] = { + {.compatible = "arm,mali-300"}, + /*-------------------------------------------------------*/ + /* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */ + // {.compatible = "arm,mali-400"}, + {.compatible = "arm,mali400"}, + /*-------------------------------------------------------*/ + {.compatible = "arm,mali-450"}, + {.compatible = "arm,mali-470"}, + {}, +}; + +MODULE_DEVICE_TABLE(of, base_dt_ids); +#endif + +/* The Mali device driver struct */ +static struct platform_driver mali_platform_driver = { + .probe = mali_probe, + .remove = mali_remove, +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)) + .pm = &mali_dev_ext_pm_ops, +#endif + .driver = + { + .name = MALI_GPU_NAME_UTGARD, + .owner = THIS_MODULE, + .bus = &platform_bus_type, +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)) + .pm = &mali_dev_pm_ops, +#endif +#ifdef CONFIG_MALI_DT + .of_match_table = of_match_ptr(base_dt_ids), +#endif + }, +}; + +/* Linux misc device operations (/dev/mali) */ +struct file_operations mali_fops = { + .owner = THIS_MODULE, + .open = mali_open, + .release = mali_release, + .unlocked_ioctl = mali_ioctl, + .compat_ioctl = mali_ioctl, + .mmap = mali_mmap +}; + +#if MALI_ENABLE_CPU_CYCLES +void mali_init_cpu_time_counters(int reset, int enable_divide_by_64) +{ + /* The CPU assembly reference used is: ARM Architecture Reference Manual ARMv7-AR C.b */ + u32 write_value; + + /* See B4.1.116 PMCNTENSET, Performance Monitors Count Enable Set register, VMSA */ + /* setting p15 c9 c12 1 to 0x8000000f==CPU_CYCLE_ENABLE |EVENT_3_ENABLE|EVENT_2_ENABLE|EVENT_1_ENABLE|EVENT_0_ENABLE */ + asm volatile("mcr p15, 0, %0, c9, c12, 1" :: "r"(0x8000000f)); + + + /* See B4.1.117 PMCR, Performance Monitors Control Register. Writing to p15, c9, c12, 0 */ + write_value = 1 << 0; /* Bit 0 set. Enable counters */ + if (reset) { + write_value |= 1 << 1; /* Reset event counters */ + write_value |= 1 << 2; /* Reset cycle counter */ + } + if (enable_divide_by_64) { + write_value |= 1 << 3; /* Enable the Clock divider by 64 */ + } + write_value |= 1 << 4; /* Export enable. Not needed */ + asm volatile("MCR p15, 0, %0, c9, c12, 0\t\n" :: "r"(write_value)); + + /* PMOVSR Overflow Flag Status Register - Clear Clock and Event overflows */ + asm volatile("MCR p15, 0, %0, c9, c12, 3\t\n" :: "r"(0x8000000f)); + + + /* See B4.1.124 PMUSERENR - setting p15 c9 c14 to 1" */ + /* User mode access to the Performance Monitors enabled. */ + /* Lets User space read cpu clock cycles */ + asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(1)); +} + +/** A timer function that configures the cycle clock counter on current CPU. + * The function \a mali_init_cpu_time_counters_on_all_cpus sets up this + * function to trigger on all Cpus during module load. + */ +static void mali_init_cpu_clock_timer_func(unsigned long data) +{ + int reset_counters, enable_divide_clock_counter_by_64; + int current_cpu = raw_smp_processor_id(); + unsigned int sample0; + unsigned int sample1; + + MALI_IGNORE(data); + + reset_counters = 1; + enable_divide_clock_counter_by_64 = 0; + mali_init_cpu_time_counters(reset_counters, enable_divide_clock_counter_by_64); + + sample0 = mali_get_cpu_cyclecount(); + sample1 = mali_get_cpu_cyclecount(); + + MALI_DEBUG_PRINT(3, ("Init Cpu %d cycle counter- First two samples: %08x %08x \n", current_cpu, sample0, sample1)); +} + +/** A timer functions for storing current time on all cpus. + * Used for checking if the clocks have similar values or if they are drifting. + */ +static void mali_print_cpu_clock_timer_func(unsigned long data) +{ + int current_cpu = raw_smp_processor_id(); + unsigned int sample0; + + MALI_IGNORE(data); + sample0 = mali_get_cpu_cyclecount(); + if (current_cpu < 8) { + mali_cpu_clock_last_value[current_cpu] = sample0; + } +} + +/** Init the performance registers on all CPUs to count clock cycles. + * For init \a print_only should be 0. + * If \a print_only is 1, it will intead print the current clock value of all CPUs. + */ +void mali_init_cpu_time_counters_on_all_cpus(int print_only) +{ + int i = 0; + int cpu_number; + int jiffies_trigger; + int jiffies_wait; + + jiffies_wait = 2; + jiffies_trigger = jiffies + jiffies_wait; + + for (i = 0 ; i < 8 ; i++) { + init_timer(&mali_init_cpu_clock_timers[i]); + if (print_only) mali_init_cpu_clock_timers[i].function = mali_print_cpu_clock_timer_func; + else mali_init_cpu_clock_timers[i].function = mali_init_cpu_clock_timer_func; + mali_init_cpu_clock_timers[i].expires = jiffies_trigger ; + } + cpu_number = cpumask_first(cpu_online_mask); + for (i = 0 ; i < 8 ; i++) { + int next_cpu; + add_timer_on(&mali_init_cpu_clock_timers[i], cpu_number); + next_cpu = cpumask_next(cpu_number, cpu_online_mask); + if (next_cpu >= nr_cpu_ids) break; + cpu_number = next_cpu; + } + + while (jiffies_wait) jiffies_wait = schedule_timeout_uninterruptible(jiffies_wait); + + for (i = 0 ; i < 8 ; i++) { + del_timer_sync(&mali_init_cpu_clock_timers[i]); + } + + if (print_only) { + if ((0 == mali_cpu_clock_last_value[2]) && (0 == mali_cpu_clock_last_value[3])) { + /* Diff can be printed if we want to check if the clocks are in sync + int diff = mali_cpu_clock_last_value[0] - mali_cpu_clock_last_value[1];*/ + MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1])); + } else { + MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1], mali_cpu_clock_last_value[2], mali_cpu_clock_last_value[3])); + } + } +} +#endif + +int mali_module_init(void) +{ + int err = 0; + + MALI_DEBUG_PRINT(2, ("Inserting Mali v%d device driver. \n", _MALI_API_VERSION)); + MALI_DEBUG_PRINT(2, ("Compiled: %s, time: %s.\n", __DATE__, __TIME__)); + MALI_DEBUG_PRINT(2, ("Driver revision: %s\n", SVN_REV_STRING)); + + I("svn_rev_string_from_arm of this mali_ko is '%s', rk_ko_ver is '%d', built at '%s', on '%s'.", + SVN_REV_STRING, + RK_KO_VER, + __TIME__, + __DATE__); + +#if MALI_ENABLE_CPU_CYCLES + mali_init_cpu_time_counters_on_all_cpus(0); + MALI_DEBUG_PRINT(2, ("CPU cycle counter setup complete\n")); + /* Printing the current cpu counters */ + mali_init_cpu_time_counters_on_all_cpus(1); +#endif + + /* Initialize module wide settings */ +#ifdef MALI_FAKE_PLATFORM_DEVICE +#ifndef CONFIG_MALI_DT + MALI_DEBUG_PRINT(2, ("mali_module_init() registering device\n")); + err = mali_platform_device_register(); + if (0 != err) { + return err; + } +#endif +#endif + + MALI_DEBUG_PRINT(2, ("mali_module_init() registering driver\n")); + + err = platform_driver_register(&mali_platform_driver); + + if (0 != err) { + MALI_DEBUG_PRINT(2, ("mali_module_init() Failed to register driver (%d)\n", err)); +#ifdef MALI_FAKE_PLATFORM_DEVICE +#ifndef CONFIG_MALI_DT + mali_platform_device_unregister(); +#endif +#endif + mali_platform_device = NULL; + return err; + } + +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) + err = _mali_internal_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE); + if (0 != err) { + /* No biggie if we wheren't able to initialize the profiling */ + MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n")); + } +#endif + + /* Tracing the current frequency and voltage from boot/insmod*/ +#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) + /* Just call mali_get_current_gpu_clk_item(),to record current clk info.*/ + mali_get_current_gpu_clk_item(&mali_gpu_clk[0]); + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + mali_gpu_clk[0].clock, + mali_gpu_clk[0].vol / 1000, + 0, 0, 0); +#endif + + MALI_PRINT(("Mali device driver loaded\n")); + + return 0; /* Success */ +} + +void mali_module_exit(void) +{ + MALI_DEBUG_PRINT(2, ("Unloading Mali v%d device driver.\n", _MALI_API_VERSION)); + + MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering driver\n")); + + platform_driver_unregister(&mali_platform_driver); + +#if defined(MALI_FAKE_PLATFORM_DEVICE) +#ifndef CONFIG_MALI_DT + MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering device\n")); + mali_platform_device_unregister(); +#endif +#endif + + /* Tracing the current frequency and voltage from rmmod*/ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + 0, + 0, + 0, 0, 0); + +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) + _mali_internal_profiling_term(); +#endif + + MALI_PRINT(("Mali device driver unloaded\n")); +} + +#ifdef CONFIG_MALI_DEVFREQ +struct mali_device *mali_device_alloc(void) +{ + return kzalloc(sizeof(struct mali_device), GFP_KERNEL); +} + +void mali_device_free(struct mali_device *mdev) +{ + kfree(mdev); +} +#endif + +static int mali_probe(struct platform_device *pdev) +{ + int err; +#ifdef CONFIG_MALI_DEVFREQ + struct mali_device *mdev; +#endif + + MALI_DEBUG_PRINT(2, ("mali_probe(): Called for platform device %s\n", pdev->name)); + + if (NULL != mali_platform_device) { + /* Already connected to a device, return error */ + MALI_PRINT_ERROR(("mali_probe(): The Mali driver is already connected with a Mali device.")); + return -EEXIST; + } + + mali_platform_device = pdev; + + dev_info(&pdev->dev, "mali_platform_device->num_resources = %d\n", + mali_platform_device->num_resources); + + { + int i = 0; + + for(i = 0; i < mali_platform_device->num_resources; i++) + dev_info(&pdev->dev, + "resource[%d].start = 0x%pa\n", + i, + &mali_platform_device->resource[i].start); + } + +#ifdef CONFIG_MALI_DT + /* If we use DT to initialize our DDK, we have to prepare somethings. */ + err = mali_platform_device_init(mali_platform_device); + if (0 != err) { + MALI_PRINT_ERROR(("mali_probe(): Failed to initialize platform device.")); + mali_platform_device = NULL; + return -EFAULT; + } +#endif + +#ifdef CONFIG_MALI_DEVFREQ + mdev = mali_device_alloc(); + if (!mdev) { + MALI_PRINT_ERROR(("Can't allocate mali device private data\n")); + return -ENOMEM; + } + + mdev->dev = &pdev->dev; + dev_set_drvdata(mdev->dev, mdev); + + /*Initilization clock and regulator*/ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ + && defined(CONFIG_REGULATOR) + mdev->regulator = regulator_get_optional(mdev->dev, "mali"); + if (IS_ERR_OR_NULL(mdev->regulator)) { + MALI_DEBUG_PRINT(2, ("Continuing without Mali regulator control\n")); + mdev->regulator = NULL; + /* Allow probe to continue without regulator */ + } +#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ + + err = rk_platform_init_opp_table(mdev->dev); + if (err) + MALI_DEBUG_PRINT(3, ("Failed to init_opp_table\n")); + + /* Need to name the gpu clock "clk_mali" in the device tree */ + mdev->clock = clk_get(mdev->dev, "clk_mali"); + if (IS_ERR_OR_NULL(mdev->clock)) { + MALI_DEBUG_PRINT(2, ("Continuing without Mali clock control\n")); + mdev->clock = NULL; + /* Allow probe to continue without clock. */ + } else { + err = clk_prepare(mdev->clock); + if (err) { + MALI_PRINT_ERROR(("Failed to prepare clock (%d)\n", err)); + goto clock_prepare_failed; + } + } + + /* initilize pm metrics related */ + if (mali_pm_metrics_init(mdev) < 0) { + MALI_DEBUG_PRINT(2, ("mali pm metrics init failed\n")); + goto pm_metrics_init_failed; + } + + if (mali_devfreq_init(mdev) < 0) { + MALI_DEBUG_PRINT(2, ("mali devfreq init failed\n")); + goto devfreq_init_failed; + } +#endif + + + if (_MALI_OSK_ERR_OK == _mali_osk_wq_init()) { + /* Initialize the Mali GPU HW specified by pdev */ + if (_MALI_OSK_ERR_OK == mali_initialize_subsystems()) { + /* Register a misc device (so we are accessible from user space) */ + err = mali_miscdevice_register(pdev); + if (0 == err) { + /* Setup sysfs entries */ + err = mali_sysfs_register(mali_dev_name); + + if (0 == err) { + MALI_DEBUG_PRINT(2, ("mali_probe(): Successfully initialized driver for platform device %s\n", pdev->name)); + + return 0; + } else { + MALI_PRINT_ERROR(("mali_probe(): failed to register sysfs entries")); + } + mali_miscdevice_unregister(); + } else { + MALI_PRINT_ERROR(("mali_probe(): failed to register Mali misc device.")); + } + mali_terminate_subsystems(); + } else { + MALI_PRINT_ERROR(("mali_probe(): Failed to initialize Mali device driver.")); + } + _mali_osk_wq_term(); + } + +#ifdef CONFIG_MALI_DEVFREQ + mali_devfreq_term(mdev); +devfreq_init_failed: + mali_pm_metrics_term(mdev); +pm_metrics_init_failed: + clk_unprepare(mdev->clock); +clock_prepare_failed: + clk_put(mdev->clock); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_OF) \ + && defined(CONFIG_PM_OPP) + dev_pm_opp_of_remove_table(mdev->dev); +#endif + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ + && defined(CONFIG_REGULATOR) + regulator_put(mdev->regulator); +#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ + mali_device_free(mdev); +#endif + +#ifdef CONFIG_MALI_DT + mali_platform_device_deinit(mali_platform_device); +#endif + mali_platform_device = NULL; + return -EFAULT; +} + +static int mali_remove(struct platform_device *pdev) +{ +#ifdef CONFIG_MALI_DEVFREQ + struct mali_device *mdev = dev_get_drvdata(&pdev->dev); +#endif + + MALI_DEBUG_PRINT(2, ("mali_remove() called for platform device %s\n", pdev->name)); + mali_sysfs_unregister(); + mali_miscdevice_unregister(); + mali_terminate_subsystems(); + _mali_osk_wq_term(); + +#ifdef CONFIG_MALI_DEVFREQ + mali_devfreq_term(mdev); + + mali_pm_metrics_term(mdev); + + if (mdev->clock) { + clk_unprepare(mdev->clock); + clk_put(mdev->clock); + mdev->clock = NULL; + } +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_OF) \ + && defined(CONFIG_PM_OPP) + dev_pm_opp_of_remove_table(mdev->dev); +#endif + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ + && defined(CONFIG_REGULATOR) + regulator_put(mdev->regulator); +#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ + mali_device_free(mdev); +#endif + +#ifdef CONFIG_MALI_DT + mali_platform_device_deinit(mali_platform_device); +#endif + mali_platform_device = NULL; + return 0; +} + +static int mali_miscdevice_register(struct platform_device *pdev) +{ + int err; + + mali_miscdevice.minor = MISC_DYNAMIC_MINOR; + mali_miscdevice.name = mali_dev_name; + mali_miscdevice.fops = &mali_fops; + mali_miscdevice.parent = get_device(&pdev->dev); + + err = misc_register(&mali_miscdevice); + if (0 != err) { + MALI_PRINT_ERROR(("Failed to register misc device, misc_register() returned %d\n", err)); + } + + return err; +} + +static void mali_miscdevice_unregister(void) +{ + misc_deregister(&mali_miscdevice); +} + +static int mali_driver_suspend_scheduler(struct device *dev) +{ +#ifdef CONFIG_MALI_DEVFREQ + struct mali_device *mdev = dev_get_drvdata(dev); + if (!mdev) + return -ENODEV; +#endif + +#if defined(CONFIG_MALI_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + devfreq_suspend_device(mdev->devfreq); +#endif + + mali_pm_os_suspend(MALI_TRUE); + /* Tracing the frequency and voltage after mali is suspended */ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + 0, + 0, + 0, 0, 0); + return 0; +} + +static int mali_driver_resume_scheduler(struct device *dev) +{ +#ifdef CONFIG_MALI_DEVFREQ + struct mali_device *mdev = dev_get_drvdata(dev); + if (!mdev) + return -ENODEV; +#endif + + /* Tracing the frequency and voltage after mali is resumed */ +#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) + /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/ + if (is_first_resume == 1) { + mali_get_current_gpu_clk_item(&mali_gpu_clk[1]); + is_first_resume = 0; + } + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + mali_gpu_clk[1].clock, + mali_gpu_clk[1].vol / 1000, + 0, 0, 0); +#endif + mali_pm_os_resume(); + +#if defined(CONFIG_MALI_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + devfreq_resume_device(mdev->devfreq); +#endif + + return 0; +} + +#ifdef CONFIG_PM_RUNTIME +static int mali_driver_runtime_suspend(struct device *dev) +{ +#ifdef CONFIG_MALI_DEVFREQ + struct mali_device *mdev = dev_get_drvdata(dev); + if (!mdev) + return -ENODEV; +#endif + + if (MALI_TRUE == mali_pm_runtime_suspend()) { + /* Tracing the frequency and voltage after mali is suspended */ + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + 0, + 0, + 0, 0, 0); + +#if defined(CONFIG_MALI_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + MALI_DEBUG_PRINT(4, ("devfreq_suspend_device: stop devfreq monitor\n")); + devfreq_suspend_device(mdev->devfreq); +#endif + + return 0; + } else { + return -EBUSY; + } +} + +static int mali_driver_runtime_resume(struct device *dev) +{ +#ifdef CONFIG_MALI_DEVFREQ + struct mali_device *mdev = dev_get_drvdata(dev); + if (!mdev) + return -ENODEV; +#endif + + /* Tracing the frequency and voltage after mali is resumed */ +#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) + /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/ + if (is_first_resume == 1) { + mali_get_current_gpu_clk_item(&mali_gpu_clk[1]); + is_first_resume = 0; + } + _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | + MALI_PROFILING_EVENT_CHANNEL_GPU | + MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, + mali_gpu_clk[1].clock, + mali_gpu_clk[1].vol / 1000, + 0, 0, 0); +#endif + + mali_pm_runtime_resume(); + +#if defined(CONFIG_MALI_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + MALI_DEBUG_PRINT(4, ("devfreq_resume_device: start devfreq monitor\n")); + devfreq_resume_device(mdev->devfreq); +#endif + return 0; +} + +static int mali_driver_runtime_idle(struct device *dev) +{ + /* Nothing to do */ + return 0; +} +#endif + +static int mali_open(struct inode *inode, struct file *filp) +{ + struct mali_session_data *session_data; + _mali_osk_errcode_t err; + + /* input validation */ + if (mali_miscdevice.minor != iminor(inode)) { + MALI_PRINT_ERROR(("mali_open() Minor does not match\n")); + return -ENODEV; + } + + /* allocated struct to track this session */ + err = _mali_ukk_open((void **)&session_data); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + /* initialize file pointer */ + filp->f_pos = 0; + + /* link in our session data */ + filp->private_data = (void *)session_data; + + filp->f_mapping = mali_mem_swap_get_global_swap_file()->f_mapping; + + return 0; +} + +static int mali_release(struct inode *inode, struct file *filp) +{ + _mali_osk_errcode_t err; + + /* input validation */ + if (mali_miscdevice.minor != iminor(inode)) { + MALI_PRINT_ERROR(("mali_release() Minor does not match\n")); + return -ENODEV; + } + + err = _mali_ukk_close((void **)&filp->private_data); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} + +int map_errcode(_mali_osk_errcode_t err) +{ + switch (err) { + case _MALI_OSK_ERR_OK : + return 0; + case _MALI_OSK_ERR_FAULT: + return -EFAULT; + case _MALI_OSK_ERR_INVALID_FUNC: + return -ENOTTY; + case _MALI_OSK_ERR_INVALID_ARGS: + return -EINVAL; + case _MALI_OSK_ERR_NOMEM: + return -ENOMEM; + case _MALI_OSK_ERR_TIMEOUT: + return -ETIMEDOUT; + case _MALI_OSK_ERR_RESTARTSYSCALL: + return -ERESTARTSYS; + case _MALI_OSK_ERR_ITEM_NOT_FOUND: + return -ENOENT; + default: + return -EFAULT; + } +} + +static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + int err; + struct mali_session_data *session_data; + + MALI_DEBUG_PRINT(7, ("Ioctl received 0x%08X 0x%08lX\n", cmd, arg)); + + session_data = (struct mali_session_data *)filp->private_data; + if (NULL == session_data) { + MALI_DEBUG_PRINT(7, ("filp->private_data was NULL\n")); + return -ENOTTY; + } + + if (NULL == (void *)arg) { + MALI_DEBUG_PRINT(7, ("arg was NULL\n")); + return -ENOTTY; + } + + switch (cmd) { + case MALI_IOC_WAIT_FOR_NOTIFICATION: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_wait_for_notification_s), sizeof(u64))); + err = wait_for_notification_wrapper(session_data, (_mali_uk_wait_for_notification_s __user *)arg); + break; + + case MALI_IOC_GET_API_VERSION_V2: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_api_version_v2_s), sizeof(u64))); + err = get_api_version_v2_wrapper(session_data, (_mali_uk_get_api_version_v2_s __user *)arg); + break; + + case MALI_IOC_GET_API_VERSION: + err = get_api_version_wrapper(session_data, (_mali_uk_get_api_version_s __user *)arg); + break; + + case MALI_IOC_POST_NOTIFICATION: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_post_notification_s), sizeof(u64))); + err = post_notification_wrapper(session_data, (_mali_uk_post_notification_s __user *)arg); + break; + + /* rk_ext : 从对 r5p0-01rel0 集æˆå¼€å§‹, ä¸å†ä½¿ç”¨. */ +#if 0 + case MALI_IOC_GET_MALI_VERSION_IN_RK30: + err = get_mali_version_in_rk30_wrapper(session_data, (_mali_uk_get_mali_version_in_rk30_s __user *)arg); + break; +#else + case MALI_IOC_GET_RK_KO_VERSION: + err = get_rk_ko_version_wrapper(session_data, (_mali_rk_ko_version_s __user *)arg); + break; +#endif + + case MALI_IOC_GET_USER_SETTINGS: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_user_settings_s), sizeof(u64))); + err = get_user_settings_wrapper(session_data, (_mali_uk_get_user_settings_s __user *)arg); + break; + + case MALI_IOC_REQUEST_HIGH_PRIORITY: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_request_high_priority_s), sizeof(u64))); + err = request_high_priority_wrapper(session_data, (_mali_uk_request_high_priority_s __user *)arg); + break; + + case MALI_IOC_PENDING_SUBMIT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pending_submit_s), sizeof(u64))); + err = pending_submit_wrapper(session_data, (_mali_uk_pending_submit_s __user *)arg); + break; + +#if defined(CONFIG_MALI400_PROFILING) + case MALI_IOC_PROFILING_ADD_EVENT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_add_event_s), sizeof(u64))); + err = profiling_add_event_wrapper(session_data, (_mali_uk_profiling_add_event_s __user *)arg); + break; + + case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_sw_counters_report_s), sizeof(u64))); + err = profiling_report_sw_counters_wrapper(session_data, (_mali_uk_sw_counters_report_s __user *)arg); + break; + + case MALI_IOC_PROFILING_STREAM_FD_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_stream_fd_get_s), sizeof(u64))); + err = profiling_get_stream_fd_wrapper(session_data, (_mali_uk_profiling_stream_fd_get_s __user *)arg); + break; + + case MALI_IOC_PROILING_CONTROL_SET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_control_set_s), sizeof(u64))); + err = profiling_control_set_wrapper(session_data, (_mali_uk_profiling_control_set_s __user *)arg); + break; +#else + + case MALI_IOC_PROFILING_ADD_EVENT: /* FALL-THROUGH */ + case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: /* FALL-THROUGH */ + MALI_DEBUG_PRINT(2, ("Profiling not supported\n")); + err = -ENOTTY; + break; +#endif + + case MALI_IOC_PROFILING_MEMORY_USAGE_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64))); + err = mem_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg); + break; + + case MALI_IOC_MEM_ALLOC: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_alloc_mem_s), sizeof(u64))); + err = mem_alloc_wrapper(session_data, (_mali_uk_alloc_mem_s __user *)arg); + break; + + case MALI_IOC_MEM_FREE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_free_mem_s), sizeof(u64))); + err = mem_free_wrapper(session_data, (_mali_uk_free_mem_s __user *)arg); + break; + + case MALI_IOC_MEM_BIND: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_bind_mem_s), sizeof(u64))); + err = mem_bind_wrapper(session_data, (_mali_uk_bind_mem_s __user *)arg); + break; + + case MALI_IOC_MEM_UNBIND: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unbind_mem_s), sizeof(u64))); + err = mem_unbind_wrapper(session_data, (_mali_uk_unbind_mem_s __user *)arg); + break; + + case MALI_IOC_MEM_COW: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_mem_s), sizeof(u64))); + err = mem_cow_wrapper(session_data, (_mali_uk_cow_mem_s __user *)arg); + break; + + case MALI_IOC_MEM_COW_MODIFY_RANGE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_modify_range_s), sizeof(u64))); + err = mem_cow_modify_range_wrapper(session_data, (_mali_uk_cow_modify_range_s __user *)arg); + break; + + case MALI_IOC_MEM_RESIZE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_resize_s), sizeof(u64))); + err = mem_resize_mem_wrapper(session_data, (_mali_uk_mem_resize_s __user *)arg); + break; + + case MALI_IOC_MEM_WRITE_SAFE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64))); + err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg); + break; + + case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64))); + err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg); + break; + + case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64))); + err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg); + break; + + case MALI_IOC_MEM_DMA_BUF_GET_SIZE: +#ifdef CONFIG_DMA_SHARED_BUFFER + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dma_buf_get_size_s), sizeof(u64))); + err = mali_dma_buf_get_size(session_data, (_mali_uk_dma_buf_get_size_s __user *)arg); +#else + MALI_DEBUG_PRINT(2, ("DMA-BUF not supported\n")); + err = -ENOTTY; +#endif + break; + + case MALI_IOC_PP_START_JOB: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_start_job_s), sizeof(u64))); + err = pp_start_job_wrapper(session_data, (_mali_uk_pp_start_job_s __user *)arg); + break; + + case MALI_IOC_PP_AND_GP_START_JOB: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_and_gp_start_job_s), sizeof(u64))); + err = pp_and_gp_start_job_wrapper(session_data, (_mali_uk_pp_and_gp_start_job_s __user *)arg); + break; + + case MALI_IOC_PP_NUMBER_OF_CORES_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_number_of_cores_s), sizeof(u64))); + err = pp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_pp_number_of_cores_s __user *)arg); + break; + + case MALI_IOC_PP_CORE_VERSION_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_core_version_s), sizeof(u64))); + err = pp_get_core_version_wrapper(session_data, (_mali_uk_get_pp_core_version_s __user *)arg); + break; + + case MALI_IOC_PP_DISABLE_WB: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_disable_wb_s), sizeof(u64))); + err = pp_disable_wb_wrapper(session_data, (_mali_uk_pp_disable_wb_s __user *)arg); + break; + + case MALI_IOC_GP2_START_JOB: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_start_job_s), sizeof(u64))); + err = gp_start_job_wrapper(session_data, (_mali_uk_gp_start_job_s __user *)arg); + break; + + case MALI_IOC_GP2_NUMBER_OF_CORES_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_number_of_cores_s), sizeof(u64))); + err = gp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_gp_number_of_cores_s __user *)arg); + break; + + case MALI_IOC_GP2_CORE_VERSION_GET: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_core_version_s), sizeof(u64))); + err = gp_get_core_version_wrapper(session_data, (_mali_uk_get_gp_core_version_s __user *)arg); + break; + + case MALI_IOC_GP2_SUSPEND_RESPONSE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_suspend_response_s), sizeof(u64))); + err = gp_suspend_response_wrapper(session_data, (_mali_uk_gp_suspend_response_s __user *)arg); + break; + + case MALI_IOC_VSYNC_EVENT_REPORT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_vsync_event_report_s), sizeof(u64))); + err = vsync_event_report_wrapper(session_data, (_mali_uk_vsync_event_report_s __user *)arg); + break; + + case MALI_IOC_TIMELINE_GET_LATEST_POINT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_get_latest_point_s), sizeof(u64))); + err = timeline_get_latest_point_wrapper(session_data, (_mali_uk_timeline_get_latest_point_s __user *)arg); + break; + case MALI_IOC_TIMELINE_WAIT: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_wait_s), sizeof(u64))); + err = timeline_wait_wrapper(session_data, (_mali_uk_timeline_wait_s __user *)arg); + break; + case MALI_IOC_TIMELINE_CREATE_SYNC_FENCE: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_create_sync_fence_s), sizeof(u64))); + err = timeline_create_sync_fence_wrapper(session_data, (_mali_uk_timeline_create_sync_fence_s __user *)arg); + break; + case MALI_IOC_SOFT_JOB_START: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_start_s), sizeof(u64))); + err = soft_job_start_wrapper(session_data, (_mali_uk_soft_job_start_s __user *)arg); + break; + case MALI_IOC_SOFT_JOB_SIGNAL: + BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_signal_s), sizeof(u64))); + err = soft_job_signal_wrapper(session_data, (_mali_uk_soft_job_signal_s __user *)arg); + break; + + default: + MALI_DEBUG_PRINT(2, ("No handler for ioctl 0x%08X 0x%08lX\n", cmd, arg)); + err = -ENOTTY; + }; + + return err; +} + +late_initcall_sync(mali_module_init); +module_exit(mali_module_exit); + +MODULE_LICENSE(MALI_KERNEL_LINUX_LICENSE); +MODULE_AUTHOR("ARM Ltd."); +MODULE_VERSION(SVN_REV_STRING); diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h new file mode 100755 index 000000000000..be754cb15646 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_LINUX_H__ +#define __MALI_KERNEL_LINUX_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include /* character device definitions */ +#include +#include +#include "mali_kernel_license.h" +#include "mali_osk_types.h" +#include + +extern struct platform_device *mali_platform_device; + +/* After 3.19.0 kenrel droped CONFIG_PM_RUNTIME define,define by ourself */ +#if defined(CONFIG_PM) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) +#define CONFIG_PM_RUNTIME 1 +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_KERNEL_LINUX_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c new file mode 100755 index 000000000000..7bda438fef50 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c @@ -0,0 +1,1410 @@ +/** + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + + +/** + * @file mali_kernel_sysfs.c + * Implementation of some sysfs data exports + */ + +#include +#include +#include +#include +#include "mali_kernel_license.h" +#include "mali_kernel_common.h" +#include "mali_ukk.h" + +#if MALI_LICENSE_IS_GPL + +#include +#include +#include +#include +#include +#include "mali_kernel_sysfs.h" +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) +#include +#include "mali_osk_profiling.h" +#endif + +#include +#include "mali_pm.h" +#include "mali_pmu.h" +#include "mali_group.h" +#include "mali_gp.h" +#include "mali_pp.h" +#include "mali_l2_cache.h" +#include "mali_hw_core.h" +#include "mali_kernel_core.h" +#include "mali_user_settings_db.h" +#include "mali_profiling_internal.h" +#include "mali_gp_job.h" +#include "mali_pp_job.h" +#include "mali_executor.h" + +#define PRIVATE_DATA_COUNTER_MAKE_GP(src) (src) +#define PRIVATE_DATA_COUNTER_MAKE_PP(src) ((1 << 24) | src) +#define PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(src, sub_job) ((1 << 24) | (1 << 16) | (sub_job << 8) | src) +#define PRIVATE_DATA_COUNTER_IS_PP(a) ((((a) >> 24) & 0xFF) ? MALI_TRUE : MALI_FALSE) +#define PRIVATE_DATA_COUNTER_GET_SRC(a) (a & 0xFF) +#define PRIVATE_DATA_COUNTER_IS_SUB_JOB(a) ((((a) >> 16) & 0xFF) ? MALI_TRUE : MALI_FALSE) +#define PRIVATE_DATA_COUNTER_GET_SUB_JOB(a) (((a) >> 8) & 0xFF) + +#define POWER_BUFFER_SIZE 3 + +static struct dentry *mali_debugfs_dir = NULL; + +typedef enum { + _MALI_DEVICE_SUSPEND, + _MALI_DEVICE_RESUME, + _MALI_DEVICE_DVFS_PAUSE, + _MALI_DEVICE_DVFS_RESUME, + _MALI_MAX_EVENTS +} _mali_device_debug_power_events; + +static const char *const mali_power_events[_MALI_MAX_EVENTS] = { + [_MALI_DEVICE_SUSPEND] = "suspend", + [_MALI_DEVICE_RESUME] = "resume", + [_MALI_DEVICE_DVFS_PAUSE] = "dvfs_pause", + [_MALI_DEVICE_DVFS_RESUME] = "dvfs_resume", +}; + +static mali_bool power_always_on_enabled = MALI_FALSE; + +static int open_copy_private_data(struct inode *inode, struct file *filp) +{ + filp->private_data = inode->i_private; + return 0; +} + +static ssize_t group_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + int r; + char buffer[64]; + struct mali_group *group; + + group = (struct mali_group *)filp->private_data; + MALI_DEBUG_ASSERT_POINTER(group); + + r = snprintf(buffer, 64, "%u\n", + mali_executor_group_is_disabled(group) ? 0 : 1); + + return simple_read_from_buffer(buf, count, offp, buffer, r); +} + +static ssize_t group_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) +{ + int r; + char buffer[64]; + unsigned long val; + struct mali_group *group; + + group = (struct mali_group *)filp->private_data; + MALI_DEBUG_ASSERT_POINTER(group); + + if (count >= sizeof(buffer)) { + return -ENOMEM; + } + + if (copy_from_user(&buffer[0], buf, count)) { + return -EFAULT; + } + buffer[count] = '\0'; + + r = kstrtoul(&buffer[0], 10, &val); + if (0 != r) { + return -EINVAL; + } + + switch (val) { + case 1: + mali_executor_group_enable(group); + break; + case 0: + mali_executor_group_disable(group); + break; + default: + return -EINVAL; + break; + } + + *offp += count; + return count; +} + +static const struct file_operations group_enabled_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = group_enabled_read, + .write = group_enabled_write, +}; + +static ssize_t hw_core_base_addr_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + int r; + char buffer[64]; + struct mali_hw_core *hw_core; + + hw_core = (struct mali_hw_core *)filp->private_data; + MALI_DEBUG_ASSERT_POINTER(hw_core); + + r = snprintf(buffer, 64, "0x%lX\n", hw_core->phys_addr); + + return simple_read_from_buffer(buf, count, offp, buffer, r); +} + +static const struct file_operations hw_core_base_addr_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = hw_core_base_addr_read, +}; + +static ssize_t profiling_counter_src_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + u32 is_pp = PRIVATE_DATA_COUNTER_IS_PP((uintptr_t)filp->private_data); + u32 src_id = PRIVATE_DATA_COUNTER_GET_SRC((uintptr_t)filp->private_data); + mali_bool is_sub_job = PRIVATE_DATA_COUNTER_IS_SUB_JOB((uintptr_t)filp->private_data); + u32 sub_job = PRIVATE_DATA_COUNTER_GET_SUB_JOB((uintptr_t)filp->private_data); + char buf[64]; + int r; + u32 val; + + if (MALI_TRUE == is_pp) { + /* PP counter */ + if (MALI_TRUE == is_sub_job) { + /* Get counter for a particular sub job */ + if (0 == src_id) { + val = mali_pp_job_get_pp_counter_sub_job_src0(sub_job); + } else { + val = mali_pp_job_get_pp_counter_sub_job_src1(sub_job); + } + } else { + /* Get default counter for all PP sub jobs */ + if (0 == src_id) { + val = mali_pp_job_get_pp_counter_global_src0(); + } else { + val = mali_pp_job_get_pp_counter_global_src1(); + } + } + } else { + /* GP counter */ + if (0 == src_id) { + val = mali_gp_job_get_gp_counter_src0(); + } else { + val = mali_gp_job_get_gp_counter_src1(); + } + } + + if (MALI_HW_CORE_NO_COUNTER == val) { + r = snprintf(buf, 64, "-1\n"); + } else { + r = snprintf(buf, 64, "%u\n", val); + } + + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t profiling_counter_src_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + u32 is_pp = PRIVATE_DATA_COUNTER_IS_PP((uintptr_t)filp->private_data); + u32 src_id = PRIVATE_DATA_COUNTER_GET_SRC((uintptr_t)filp->private_data); + mali_bool is_sub_job = PRIVATE_DATA_COUNTER_IS_SUB_JOB((uintptr_t)filp->private_data); + u32 sub_job = PRIVATE_DATA_COUNTER_GET_SUB_JOB((uintptr_t)filp->private_data); + char buf[64]; + long val; + int ret; + + if (cnt >= sizeof(buf)) { + return -EINVAL; + } + + if (copy_from_user(&buf, ubuf, cnt)) { + return -EFAULT; + } + + buf[cnt] = 0; + + ret = kstrtol(buf, 10, &val); + if (ret < 0) { + return ret; + } + + if (val < 0) { + /* any negative input will disable counter */ + val = MALI_HW_CORE_NO_COUNTER; + } + + if (MALI_TRUE == is_pp) { + /* PP counter */ + if (MALI_TRUE == is_sub_job) { + /* Set counter for a particular sub job */ + if (0 == src_id) { + mali_pp_job_set_pp_counter_sub_job_src0(sub_job, (u32)val); + } else { + mali_pp_job_set_pp_counter_sub_job_src1(sub_job, (u32)val); + } + } else { + /* Set default counter for all PP sub jobs */ + if (0 == src_id) { + mali_pp_job_set_pp_counter_global_src0((u32)val); + } else { + mali_pp_job_set_pp_counter_global_src1((u32)val); + } + } + } else { + /* GP counter */ + if (0 == src_id) { + mali_gp_job_set_gp_counter_src0((u32)val); + } else { + mali_gp_job_set_gp_counter_src1((u32)val); + } + } + + *ppos += cnt; + return cnt; +} + +static const struct file_operations profiling_counter_src_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = profiling_counter_src_read, + .write = profiling_counter_src_write, +}; + +static ssize_t l2_l2x_counter_srcx_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) +{ + char buf[64]; + int r; + u32 val; + struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; + + if (0 == src_id) { + val = mali_l2_cache_core_get_counter_src0(l2_core); + } else { + val = mali_l2_cache_core_get_counter_src1(l2_core); + } + + if (MALI_HW_CORE_NO_COUNTER == val) { + r = snprintf(buf, 64, "-1\n"); + } else { + r = snprintf(buf, 64, "%u\n", val); + } + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t l2_l2x_counter_srcx_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) +{ + struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; + char buf[64]; + long val; + int ret; + + if (cnt >= sizeof(buf)) { + return -EINVAL; + } + + if (copy_from_user(&buf, ubuf, cnt)) { + return -EFAULT; + } + + buf[cnt] = 0; + + ret = kstrtol(buf, 10, &val); + if (ret < 0) { + return ret; + } + + if (val < 0) { + /* any negative input will disable counter */ + val = MALI_HW_CORE_NO_COUNTER; + } + + mali_l2_cache_core_set_counter_src(l2_core, src_id, (u32)val); + + *ppos += cnt; + return cnt; +} + +static ssize_t l2_all_counter_srcx_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) +{ + char buf[64]; + long val; + int ret; + u32 l2_id; + struct mali_l2_cache_core *l2_cache; + + if (cnt >= sizeof(buf)) { + return -EINVAL; + } + + if (copy_from_user(&buf, ubuf, cnt)) { + return -EFAULT; + } + + buf[cnt] = 0; + + ret = kstrtol(buf, 10, &val); + if (ret < 0) { + return ret; + } + + if (val < 0) { + /* any negative input will disable counter */ + val = MALI_HW_CORE_NO_COUNTER; + } + + l2_id = 0; + l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); + while (NULL != l2_cache) { + mali_l2_cache_core_set_counter_src(l2_cache, src_id, (u32)val); + + /* try next L2 */ + l2_id++; + l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); + } + + *ppos += cnt; + return cnt; +} + +static ssize_t l2_l2x_counter_src0_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_srcx_read(filp, ubuf, cnt, ppos, 0); +} + +static ssize_t l2_l2x_counter_src1_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_srcx_read(filp, ubuf, cnt, ppos, 1); +} + +static ssize_t l2_l2x_counter_src0_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_srcx_write(filp, ubuf, cnt, ppos, 0); +} + +static ssize_t l2_l2x_counter_src1_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_srcx_write(filp, ubuf, cnt, ppos, 1); +} + +static ssize_t l2_all_counter_src0_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_all_counter_srcx_write(filp, ubuf, cnt, ppos, 0); +} + +static ssize_t l2_all_counter_src1_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_all_counter_srcx_write(filp, ubuf, cnt, ppos, 1); +} + +static const struct file_operations l2_l2x_counter_src0_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = l2_l2x_counter_src0_read, + .write = l2_l2x_counter_src0_write, +}; + +static const struct file_operations l2_l2x_counter_src1_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = l2_l2x_counter_src1_read, + .write = l2_l2x_counter_src1_write, +}; + +static const struct file_operations l2_all_counter_src0_fops = { + .owner = THIS_MODULE, + .write = l2_all_counter_src0_write, +}; + +static const struct file_operations l2_all_counter_src1_fops = { + .owner = THIS_MODULE, + .write = l2_all_counter_src1_write, +}; + +static ssize_t l2_l2x_counter_valx_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) +{ + char buf[64]; + int r; + u32 src0 = 0; + u32 val0 = 0; + u32 src1 = 0; + u32 val1 = 0; + u32 val = -1; + struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; + + mali_l2_cache_core_get_counter_values(l2_core, &src0, &val0, &src1, &val1); + + if (0 == src_id) { + if (MALI_HW_CORE_NO_COUNTER != val0) { + val = val0; + } + } else { + if (MALI_HW_CORE_NO_COUNTER != val1) { + val = val1; + } + } + + r = snprintf(buf, 64, "%u\n", val); + + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t l2_l2x_counter_val0_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_valx_read(filp, ubuf, cnt, ppos, 0); +} + +static ssize_t l2_l2x_counter_val1_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + return l2_l2x_counter_valx_read(filp, ubuf, cnt, ppos, 1); +} + +static const struct file_operations l2_l2x_counter_val0_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = l2_l2x_counter_val0_read, +}; + +static const struct file_operations l2_l2x_counter_val1_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = l2_l2x_counter_val1_read, +}; + +static ssize_t power_always_on_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + unsigned long val; + int ret; + char buf[32]; + + cnt = min(cnt, sizeof(buf) - 1); + if (copy_from_user(buf, ubuf, cnt)) { + return -EFAULT; + } + buf[cnt] = '\0'; + + ret = kstrtoul(buf, 10, &val); + if (0 != ret) { + return ret; + } + + /* Update setting (not exactly thread safe) */ + if (1 == val && MALI_FALSE == power_always_on_enabled) { + power_always_on_enabled = MALI_TRUE; + _mali_osk_pm_dev_ref_get_sync(); + } else if (0 == val && MALI_TRUE == power_always_on_enabled) { + power_always_on_enabled = MALI_FALSE; + _mali_osk_pm_dev_ref_put(); + } + + *ppos += cnt; + return cnt; +} + +static ssize_t power_always_on_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + if (MALI_TRUE == power_always_on_enabled) { + return simple_read_from_buffer(ubuf, cnt, ppos, "1\n", 2); + } else { + return simple_read_from_buffer(ubuf, cnt, ppos, "0\n", 2); + } +} + +static const struct file_operations power_always_on_fops = { + .owner = THIS_MODULE, + .read = power_always_on_read, + .write = power_always_on_write, +}; + +static ssize_t power_power_events_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_SUSPEND], strlen(mali_power_events[_MALI_DEVICE_SUSPEND]) - 1)) { + mali_pm_os_suspend(MALI_TRUE); + } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_RESUME], strlen(mali_power_events[_MALI_DEVICE_RESUME]) - 1)) { + mali_pm_os_resume(); + } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_PAUSE], strlen(mali_power_events[_MALI_DEVICE_DVFS_PAUSE]) - 1)) { + mali_dev_pause(); + } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_RESUME], strlen(mali_power_events[_MALI_DEVICE_DVFS_RESUME]) - 1)) { + mali_dev_resume(); + } + *ppos += cnt; + return cnt; +} + +static loff_t power_power_events_seek(struct file *file, loff_t offset, int orig) +{ + file->f_pos = offset; + return 0; +} + +static const struct file_operations power_power_events_fops = { + .owner = THIS_MODULE, + .write = power_power_events_write, + .llseek = power_power_events_seek, +}; + +#if MALI_STATE_TRACKING +static int mali_seq_internal_state_show(struct seq_file *seq_file, void *v) +{ + u32 len = 0; + u32 size; + char *buf; + + size = seq_get_buf(seq_file, &buf); + + if (!size) { + return -ENOMEM; + } + + /* Create the internal state dump. */ + len = snprintf(buf + len, size - len, "Mali device driver %s\n", SVN_REV_STRING); + len += snprintf(buf + len, size - len, "License: %s\n\n", MALI_KERNEL_LINUX_LICENSE); + + len += _mali_kernel_core_dump_state(buf + len, size - len); + + seq_commit(seq_file, len); + + return 0; +} + +static int mali_seq_internal_state_open(struct inode *inode, struct file *file) +{ + return single_open(file, mali_seq_internal_state_show, NULL); +} + +static const struct file_operations mali_seq_internal_state_fops = { + .owner = THIS_MODULE, + .open = mali_seq_internal_state_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif /* MALI_STATE_TRACKING */ + +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) +static ssize_t profiling_record_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + int r; + + r = snprintf(buf, 64, "%u\n", _mali_internal_profiling_is_recording() ? 1 : 0); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t profiling_record_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + unsigned long val; + int ret; + + if (cnt >= sizeof(buf)) { + return -EINVAL; + } + + if (copy_from_user(&buf, ubuf, cnt)) { + return -EFAULT; + } + + buf[cnt] = 0; + + ret = kstrtoul(buf, 10, &val); + if (ret < 0) { + return ret; + } + + if (val != 0) { + u32 limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; /* This can be made configurable at a later stage if we need to */ + + /* check if we are already recording */ + if (MALI_TRUE == _mali_internal_profiling_is_recording()) { + MALI_DEBUG_PRINT(3, ("Recording of profiling events already in progress\n")); + return -EFAULT; + } + + /* check if we need to clear out an old recording first */ + if (MALI_TRUE == _mali_internal_profiling_have_recording()) { + if (_MALI_OSK_ERR_OK != _mali_internal_profiling_clear()) { + MALI_DEBUG_PRINT(3, ("Failed to clear existing recording of profiling events\n")); + return -EFAULT; + } + } + + /* start recording profiling data */ + if (_MALI_OSK_ERR_OK != _mali_internal_profiling_start(&limit)) { + MALI_DEBUG_PRINT(3, ("Failed to start recording of profiling events\n")); + return -EFAULT; + } + + MALI_DEBUG_PRINT(3, ("Profiling recording started (max %u events)\n", limit)); + } else { + /* stop recording profiling data */ + u32 count = 0; + if (_MALI_OSK_ERR_OK != _mali_internal_profiling_stop(&count)) { + MALI_DEBUG_PRINT(2, ("Failed to stop recording of profiling events\n")); + return -EFAULT; + } + + MALI_DEBUG_PRINT(2, ("Profiling recording stopped (recorded %u events)\n", count)); + } + + *ppos += cnt; + return cnt; +} + +static const struct file_operations profiling_record_fops = { + .owner = THIS_MODULE, + .read = profiling_record_read, + .write = profiling_record_write, +}; + +static void *profiling_events_start(struct seq_file *s, loff_t *pos) +{ + loff_t *spos; + + /* check if we have data avaiable */ + if (MALI_TRUE != _mali_internal_profiling_have_recording()) { + return NULL; + } + + spos = kmalloc(sizeof(loff_t), GFP_KERNEL); + if (NULL == spos) { + return NULL; + } + + *spos = *pos; + return spos; +} + +static void *profiling_events_next(struct seq_file *s, void *v, loff_t *pos) +{ + loff_t *spos = v; + + /* check if we have data avaiable */ + if (MALI_TRUE != _mali_internal_profiling_have_recording()) { + return NULL; + } + + /* check if the next entry actually is avaiable */ + if (_mali_internal_profiling_get_count() <= (u32)(*spos + 1)) { + return NULL; + } + + *pos = ++*spos; + return spos; +} + +static void profiling_events_stop(struct seq_file *s, void *v) +{ + kfree(v); +} + +static int profiling_events_show(struct seq_file *seq_file, void *v) +{ + loff_t *spos = v; + u32 index; + u64 timestamp; + u32 event_id; + u32 data[5]; + + index = (u32) * spos; + + /* Retrieve all events */ + if (_MALI_OSK_ERR_OK == _mali_internal_profiling_get_event(index, ×tamp, &event_id, data)) { + seq_printf(seq_file, "%llu %u %u %u %u %u %u\n", timestamp, event_id, data[0], data[1], data[2], data[3], data[4]); + return 0; + } + + return 0; +} + +static int profiling_events_show_human_readable(struct seq_file *seq_file, void *v) +{ +#define MALI_EVENT_ID_IS_HW(event_id) (((event_id & 0x00FF0000) >= MALI_PROFILING_EVENT_CHANNEL_GP0) && ((event_id & 0x00FF0000) <= MALI_PROFILING_EVENT_CHANNEL_PP7)) + + static u64 start_time = 0; + loff_t *spos = v; + u32 index; + u64 timestamp; + u32 event_id; + u32 data[5]; + + index = (u32) * spos; + + /* Retrieve all events */ + if (_MALI_OSK_ERR_OK == _mali_internal_profiling_get_event(index, ×tamp, &event_id, data)) { + seq_printf(seq_file, "%llu %u %u %u %u %u %u # ", timestamp, event_id, data[0], data[1], data[2], data[3], data[4]); + + if (0 == index) { + start_time = timestamp; + } + + seq_printf(seq_file, "[%06u] ", index); + + switch (event_id & 0x0F000000) { + case MALI_PROFILING_EVENT_TYPE_SINGLE: + seq_printf(seq_file, "SINGLE | "); + break; + case MALI_PROFILING_EVENT_TYPE_START: + seq_printf(seq_file, "START | "); + break; + case MALI_PROFILING_EVENT_TYPE_STOP: + seq_printf(seq_file, "STOP | "); + break; + case MALI_PROFILING_EVENT_TYPE_SUSPEND: + seq_printf(seq_file, "SUSPEND | "); + break; + case MALI_PROFILING_EVENT_TYPE_RESUME: + seq_printf(seq_file, "RESUME | "); + break; + default: + seq_printf(seq_file, "0x%01X | ", (event_id & 0x0F000000) >> 24); + break; + } + + switch (event_id & 0x00FF0000) { + case MALI_PROFILING_EVENT_CHANNEL_SOFTWARE: + seq_printf(seq_file, "SW | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_GP0: + seq_printf(seq_file, "GP0 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP0: + seq_printf(seq_file, "PP0 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP1: + seq_printf(seq_file, "PP1 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP2: + seq_printf(seq_file, "PP2 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP3: + seq_printf(seq_file, "PP3 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP4: + seq_printf(seq_file, "PP4 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP5: + seq_printf(seq_file, "PP5 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP6: + seq_printf(seq_file, "PP6 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_PP7: + seq_printf(seq_file, "PP7 | "); + break; + case MALI_PROFILING_EVENT_CHANNEL_GPU: + seq_printf(seq_file, "GPU | "); + break; + default: + seq_printf(seq_file, "0x%02X | ", (event_id & 0x00FF0000) >> 16); + break; + } + + if (MALI_EVENT_ID_IS_HW(event_id)) { + if (((event_id & 0x0F000000) == MALI_PROFILING_EVENT_TYPE_START) || ((event_id & 0x0F000000) == MALI_PROFILING_EVENT_TYPE_STOP)) { + switch (event_id & 0x0000FFFF) { + case MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL: + seq_printf(seq_file, "PHYSICAL | "); + break; + case MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL: + seq_printf(seq_file, "VIRTUAL | "); + break; + default: + seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); + break; + } + } else { + seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); + } + } else { + seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); + } + + seq_printf(seq_file, "T0 + 0x%016llX\n", timestamp - start_time); + + return 0; + } + + return 0; +} + +static const struct seq_operations profiling_events_seq_ops = { + .start = profiling_events_start, + .next = profiling_events_next, + .stop = profiling_events_stop, + .show = profiling_events_show +}; + +static int profiling_events_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &profiling_events_seq_ops); +} + +static const struct file_operations profiling_events_fops = { + .owner = THIS_MODULE, + .open = profiling_events_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static const struct seq_operations profiling_events_human_readable_seq_ops = { + .start = profiling_events_start, + .next = profiling_events_next, + .stop = profiling_events_stop, + .show = profiling_events_show_human_readable +}; + +static int profiling_events_human_readable_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &profiling_events_human_readable_seq_ops); +} + +static const struct file_operations profiling_events_human_readable_fops = { + .owner = THIS_MODULE, + .open = profiling_events_human_readable_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +#endif + +static int memory_debugfs_show(struct seq_file *s, void *private_data) +{ +#ifdef MALI_MEM_SWAP_TRACKING + seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s %-10s \n"\ + "=================================================================================================================================\n", + "Name (:bytes)", "pid", "mali_mem", "max_mali_mem", + "external_mem", "ump_mem", "dma_mem", "swap_mem"); +#else + seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s \n"\ + "========================================================================================================================\n", + "Name (:bytes)", "pid", "mali_mem", "max_mali_mem", + "external_mem", "ump_mem", "dma_mem"); +#endif + mali_session_memory_tracking(s); + return 0; +} + +static int memory_debugfs_open(struct inode *inode, struct file *file) +{ + return single_open(file, memory_debugfs_show, inode->i_private); +} + +static const struct file_operations memory_usage_fops = { + .owner = THIS_MODULE, + .open = memory_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static ssize_t utilization_gp_pp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + size_t r; + u32 uval = _mali_ukk_utilization_gp_pp(); + + r = snprintf(buf, 64, "%u\n", uval); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t utilization_gp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + size_t r; + u32 uval = _mali_ukk_utilization_gp(); + + r = snprintf(buf, 64, "%u\n", uval); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static ssize_t utilization_pp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + size_t r; + u32 uval = _mali_ukk_utilization_pp(); + + r = snprintf(buf, 64, "%u\n", uval); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + + +static const struct file_operations utilization_gp_pp_fops = { + .owner = THIS_MODULE, + .read = utilization_gp_pp_read, +}; + +static const struct file_operations utilization_gp_fops = { + .owner = THIS_MODULE, + .read = utilization_gp_read, +}; + +static const struct file_operations utilization_pp_fops = { + .owner = THIS_MODULE, + .read = utilization_pp_read, +}; + +static ssize_t user_settings_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + unsigned long val; + int ret; + _mali_uk_user_setting_t setting; + char buf[32]; + + cnt = min(cnt, sizeof(buf) - 1); + if (copy_from_user(buf, ubuf, cnt)) { + return -EFAULT; + } + buf[cnt] = '\0'; + + ret = kstrtoul(buf, 10, &val); + if (0 != ret) { + return ret; + } + + /* Update setting */ + setting = (_mali_uk_user_setting_t)(filp->private_data); + mali_set_user_setting(setting, val); + + *ppos += cnt; + return cnt; +} + +static ssize_t user_settings_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + size_t r; + u32 value; + _mali_uk_user_setting_t setting; + + setting = (_mali_uk_user_setting_t)(filp->private_data); + value = mali_get_user_setting(setting); + + r = snprintf(buf, 64, "%u\n", value); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static const struct file_operations user_settings_fops = { + .owner = THIS_MODULE, + .open = open_copy_private_data, + .read = user_settings_read, + .write = user_settings_write, +}; + +static int mali_sysfs_user_settings_register(void) +{ + struct dentry *mali_user_settings_dir = debugfs_create_dir("userspace_settings", mali_debugfs_dir); + + if (mali_user_settings_dir != NULL) { + long i; + for (i = 0; i < _MALI_UK_USER_SETTING_MAX; i++) { + debugfs_create_file(_mali_uk_user_setting_descriptions[i], + 0600, mali_user_settings_dir, (void *)i, + &user_settings_fops); + } + } + + return 0; +} + +static ssize_t pp_num_cores_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) +{ + int ret; + char buffer[32]; + unsigned long val; + + if (count >= sizeof(buffer)) { + return -ENOMEM; + } + + if (copy_from_user(&buffer[0], buf, count)) { + return -EFAULT; + } + buffer[count] = '\0'; + + ret = kstrtoul(&buffer[0], 10, &val); + if (0 != ret) { + return -EINVAL; + } + + ret = mali_executor_set_perf_level(val, MALI_TRUE); /* override even if core scaling is disabled */ + if (ret) { + return ret; + } + + *offp += count; + return count; +} + +static ssize_t pp_num_cores_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + int r; + char buffer[64]; + + r = snprintf(buffer, 64, "%u\n", mali_executor_get_num_cores_enabled()); + + return simple_read_from_buffer(buf, count, offp, buffer, r); +} + +static const struct file_operations pp_num_cores_enabled_fops = { + .owner = THIS_MODULE, + .write = pp_num_cores_enabled_write, + .read = pp_num_cores_enabled_read, + .llseek = default_llseek, +}; + +static ssize_t pp_num_cores_total_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + int r; + char buffer[64]; + + r = snprintf(buffer, 64, "%u\n", mali_executor_get_num_cores_total()); + + return simple_read_from_buffer(buf, count, offp, buffer, r); +} + +static const struct file_operations pp_num_cores_total_fops = { + .owner = THIS_MODULE, + .read = pp_num_cores_total_read, +}; + +static ssize_t pp_core_scaling_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) +{ + int ret; + char buffer[32]; + unsigned long val; + + if (count >= sizeof(buffer)) { + return -ENOMEM; + } + + if (copy_from_user(&buffer[0], buf, count)) { + return -EFAULT; + } + buffer[count] = '\0'; + + ret = kstrtoul(&buffer[0], 10, &val); + if (0 != ret) { + return -EINVAL; + } + + switch (val) { + case 1: + mali_executor_core_scaling_enable(); + break; + case 0: + mali_executor_core_scaling_disable(); + break; + default: + return -EINVAL; + break; + } + + *offp += count; + return count; +} + +static ssize_t pp_core_scaling_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + return simple_read_from_buffer(buf, count, offp, mali_executor_core_scaling_is_enabled() ? "1\n" : "0\n", 2); +} +static const struct file_operations pp_core_scaling_enabled_fops = { + .owner = THIS_MODULE, + .write = pp_core_scaling_enabled_write, + .read = pp_core_scaling_enabled_read, + .llseek = default_llseek, +}; + +static ssize_t version_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) +{ + int r = 0; + char buffer[64]; + + switch (mali_kernel_core_get_product_id()) { + case _MALI_PRODUCT_ID_MALI200: + r = snprintf(buffer, 64, "Mali-200\n"); + break; + case _MALI_PRODUCT_ID_MALI300: + r = snprintf(buffer, 64, "Mali-300\n"); + break; + case _MALI_PRODUCT_ID_MALI400: + r = snprintf(buffer, 64, "Mali-400 MP\n"); + break; + case _MALI_PRODUCT_ID_MALI450: + r = snprintf(buffer, 64, "Mali-450 MP\n"); + break; + case _MALI_PRODUCT_ID_MALI470: + r = snprintf(buffer, 64, "Mali-470 MP\n"); + break; + case _MALI_PRODUCT_ID_UNKNOWN: + return -EINVAL; + break; + }; + + return simple_read_from_buffer(buf, count, offp, buffer, r); +} + +static const struct file_operations version_fops = { + .owner = THIS_MODULE, + .read = version_read, +}; + +#if defined(DEBUG) +static int timeline_debugfs_show(struct seq_file *s, void *private_data) +{ + struct mali_session_data *session, *tmp; + u32 session_seq = 1; + + seq_printf(s, "timeline system info: \n=================\n\n"); + + mali_session_lock(); + MALI_SESSION_FOREACH(session, tmp, link) { + seq_printf(s, "session %d <%p> start:\n", session_seq, session); + mali_timeline_debug_print_system(session->timeline_system, s); + seq_printf(s, "session %d end\n\n\n", session_seq++); + } + mali_session_unlock(); + + return 0; +} + +static int timeline_debugfs_open(struct inode *inode, struct file *file) +{ + return single_open(file, timeline_debugfs_show, inode->i_private); +} + +static const struct file_operations timeline_dump_fops = { + .owner = THIS_MODULE, + .open = timeline_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release +}; +#endif + +int mali_sysfs_register(const char *mali_dev_name) +{ + mali_debugfs_dir = debugfs_create_dir(mali_dev_name, NULL); + if (ERR_PTR(-ENODEV) == mali_debugfs_dir) { + /* Debugfs not supported. */ + mali_debugfs_dir = NULL; + } else { + if (NULL != mali_debugfs_dir) { + /* Debugfs directory created successfully; create files now */ + struct dentry *mali_power_dir; + struct dentry *mali_gp_dir; + struct dentry *mali_pp_dir; + struct dentry *mali_l2_dir; + struct dentry *mali_profiling_dir; + + debugfs_create_file("version", 0400, mali_debugfs_dir, NULL, &version_fops); + + mali_power_dir = debugfs_create_dir("power", mali_debugfs_dir); + if (mali_power_dir != NULL) { + debugfs_create_file("always_on", 0600, mali_power_dir, NULL, &power_always_on_fops); + debugfs_create_file("power_events", 0200, mali_power_dir, NULL, &power_power_events_fops); + } + + mali_gp_dir = debugfs_create_dir("gp", mali_debugfs_dir); + if (mali_gp_dir != NULL) { + u32 num_groups; + long i; + + num_groups = mali_group_get_glob_num_groups(); + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + struct mali_gp_core *gp_core = mali_group_get_gp_core(group); + if (NULL != gp_core) { + struct dentry *mali_gp_gpx_dir; + mali_gp_gpx_dir = debugfs_create_dir("gp0", mali_gp_dir); + if (NULL != mali_gp_gpx_dir) { + debugfs_create_file("base_addr", 0400, mali_gp_gpx_dir, &gp_core->hw_core, &hw_core_base_addr_fops); + debugfs_create_file("enabled", 0600, mali_gp_gpx_dir, group, &group_enabled_fops); + } + break; /* no need to look for any other GP cores */ + } + + } + } + + mali_pp_dir = debugfs_create_dir("pp", mali_debugfs_dir); + if (mali_pp_dir != NULL) { + u32 num_groups; + long i; + + debugfs_create_file("num_cores_total", 0400, mali_pp_dir, NULL, &pp_num_cores_total_fops); + debugfs_create_file("num_cores_enabled", 0600, mali_pp_dir, NULL, &pp_num_cores_enabled_fops); + debugfs_create_file("core_scaling_enabled", 0600, mali_pp_dir, NULL, &pp_core_scaling_enabled_fops); + + num_groups = mali_group_get_glob_num_groups(); + for (i = 0; i < num_groups; i++) { + struct mali_group *group = mali_group_get_glob_group(i); + + struct mali_pp_core *pp_core = mali_group_get_pp_core(group); + if (NULL != pp_core) { + char buf[16]; + struct dentry *mali_pp_ppx_dir; + _mali_osk_snprintf(buf, sizeof(buf), "pp%u", mali_pp_core_get_id(pp_core)); + mali_pp_ppx_dir = debugfs_create_dir(buf, mali_pp_dir); + if (NULL != mali_pp_ppx_dir) { + debugfs_create_file("base_addr", 0400, mali_pp_ppx_dir, &pp_core->hw_core, &hw_core_base_addr_fops); + if (!mali_group_is_virtual(group)) { + debugfs_create_file("enabled", 0600, mali_pp_ppx_dir, group, &group_enabled_fops); + } + } + } + } + } + + mali_l2_dir = debugfs_create_dir("l2", mali_debugfs_dir); + if (mali_l2_dir != NULL) { + struct dentry *mali_l2_all_dir; + u32 l2_id; + struct mali_l2_cache_core *l2_cache; + + mali_l2_all_dir = debugfs_create_dir("all", mali_l2_dir); + if (mali_l2_all_dir != NULL) { + debugfs_create_file("counter_src0", 0200, mali_l2_all_dir, NULL, &l2_all_counter_src0_fops); + debugfs_create_file("counter_src1", 0200, mali_l2_all_dir, NULL, &l2_all_counter_src1_fops); + } + + l2_id = 0; + l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); + while (NULL != l2_cache) { + char buf[16]; + struct dentry *mali_l2_l2x_dir; + _mali_osk_snprintf(buf, sizeof(buf), "l2%u", l2_id); + mali_l2_l2x_dir = debugfs_create_dir(buf, mali_l2_dir); + if (NULL != mali_l2_l2x_dir) { + debugfs_create_file("counter_src0", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_src0_fops); + debugfs_create_file("counter_src1", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_src1_fops); + debugfs_create_file("counter_val0", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_val0_fops); + debugfs_create_file("counter_val1", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_val1_fops); + debugfs_create_file("base_addr", 0400, mali_l2_l2x_dir, &l2_cache->hw_core, &hw_core_base_addr_fops); + } + + /* try next L2 */ + l2_id++; + l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); + } + } + + debugfs_create_file("gpu_memory", 0444, mali_debugfs_dir, NULL, &memory_usage_fops); + + debugfs_create_file("utilization_gp_pp", 0400, mali_debugfs_dir, NULL, &utilization_gp_pp_fops); + debugfs_create_file("utilization_gp", 0400, mali_debugfs_dir, NULL, &utilization_gp_fops); + debugfs_create_file("utilization_pp", 0400, mali_debugfs_dir, NULL, &utilization_pp_fops); + + mali_profiling_dir = debugfs_create_dir("profiling", mali_debugfs_dir); + if (mali_profiling_dir != NULL) { + u32 max_sub_jobs; + long i; + struct dentry *mali_profiling_gp_dir; + struct dentry *mali_profiling_pp_dir; +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) + struct dentry *mali_profiling_proc_dir; +#endif + /* + * Create directory where we can set GP HW counters. + */ + mali_profiling_gp_dir = debugfs_create_dir("gp", mali_profiling_dir); + if (mali_profiling_gp_dir != NULL) { + debugfs_create_file("counter_src0", 0600, mali_profiling_gp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_GP(0), &profiling_counter_src_fops); + debugfs_create_file("counter_src1", 0600, mali_profiling_gp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_GP(1), &profiling_counter_src_fops); + } + + /* + * Create directory where we can set PP HW counters. + * Possible override with specific HW counters for a particular sub job + * (Disable core scaling before using the override!) + */ + mali_profiling_pp_dir = debugfs_create_dir("pp", mali_profiling_dir); + if (mali_profiling_pp_dir != NULL) { + debugfs_create_file("counter_src0", 0600, mali_profiling_pp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_PP(0), &profiling_counter_src_fops); + debugfs_create_file("counter_src1", 0600, mali_profiling_pp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_PP(1), &profiling_counter_src_fops); + } + + max_sub_jobs = mali_executor_get_num_cores_total(); + for (i = 0; i < max_sub_jobs; i++) { + char buf[16]; + struct dentry *mali_profiling_pp_x_dir; + _mali_osk_snprintf(buf, sizeof(buf), "%u", i); + mali_profiling_pp_x_dir = debugfs_create_dir(buf, mali_profiling_pp_dir); + if (NULL != mali_profiling_pp_x_dir) { + debugfs_create_file("counter_src0", + 0600, mali_profiling_pp_x_dir, + (void *)PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(0, i), + &profiling_counter_src_fops); + debugfs_create_file("counter_src1", + 0600, mali_profiling_pp_x_dir, + (void *)PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(1, i), + &profiling_counter_src_fops); + } + } + +#if defined(CONFIG_MALI400_INTERNAL_PROFILING) + mali_profiling_proc_dir = debugfs_create_dir("proc", mali_profiling_dir); + if (mali_profiling_proc_dir != NULL) { + struct dentry *mali_profiling_proc_default_dir = debugfs_create_dir("default", mali_profiling_proc_dir); + if (mali_profiling_proc_default_dir != NULL) { + debugfs_create_file("enable", 0600, mali_profiling_proc_default_dir, (void *)_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, &user_settings_fops); + } + } + debugfs_create_file("record", 0600, mali_profiling_dir, NULL, &profiling_record_fops); + debugfs_create_file("events", 0400, mali_profiling_dir, NULL, &profiling_events_fops); + debugfs_create_file("events_human_readable", 0400, mali_profiling_dir, NULL, &profiling_events_human_readable_fops); +#endif + } + +#if MALI_STATE_TRACKING + debugfs_create_file("state_dump", 0400, mali_debugfs_dir, NULL, &mali_seq_internal_state_fops); +#endif + +#if defined(DEBUG) + debugfs_create_file("timeline_dump", 0400, mali_debugfs_dir, NULL, &timeline_dump_fops); +#endif + if (mali_sysfs_user_settings_register()) { + /* Failed to create the debugfs entries for the user settings DB. */ + MALI_DEBUG_PRINT(2, ("Failed to create user setting debugfs files. Ignoring...\n")); + } + } + } + + /* Success! */ + return 0; +} + +int mali_sysfs_unregister(void) +{ + if (NULL != mali_debugfs_dir) { + debugfs_remove_recursive(mali_debugfs_dir); + } + return 0; +} + +#else /* MALI_LICENSE_IS_GPL */ + +/* Dummy implementations for non-GPL */ + +int mali_sysfs_register(struct mali_dev *device, dev_t dev, const char *mali_dev_name) +{ + return 0; +} + +int mali_sysfs_unregister(void) +{ + return 0; +} + +#endif /* MALI_LICENSE_IS_GPL */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h new file mode 100755 index 000000000000..91580a87c1e1 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2011-2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_KERNEL_SYSFS_H__ +#define __MALI_KERNEL_SYSFS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#define MALI_PROC_DIR "driver/mali" + +int mali_sysfs_register(const char *mali_dev_name); +int mali_sysfs_unregister(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_KERNEL_LINUX_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h b/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h new file mode 100755 index 000000000000..222260823c81 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h @@ -0,0 +1,161 @@ +/* + * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#if !defined (MALI_LINUX_TRACE_H) || defined (TRACE_HEADER_MULTI_READ) +#define MALI_LINUX_TRACE_H + +#include + +#include +#include + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mali + +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE mali_linux_trace + +/** + * Define the tracepoint used to communicate the status of a GPU. Called + * when a GPU turns on or turns off. + * + * @param event_id The type of the event. This parameter is a bitfield + * encoding the type of the event. + * + * @param d0 First data parameter. + * @param d1 Second data parameter. + * @param d2 Third data parameter. + * @param d3 Fourth data parameter. + * @param d4 Fifth data parameter. + */ +TRACE_EVENT(mali_timeline_event, + + TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, + unsigned int d2, unsigned int d3, unsigned int d4), + + TP_ARGS(event_id, d0, d1, d2, d3, d4), + + TP_STRUCT__entry( + __field(unsigned int, event_id) + __field(unsigned int, d0) + __field(unsigned int, d1) + __field(unsigned int, d2) + __field(unsigned int, d3) + __field(unsigned int, d4) + ), + + TP_fast_assign( + __entry->event_id = event_id; + __entry->d0 = d0; + __entry->d1 = d1; + __entry->d2 = d2; + __entry->d3 = d3; + __entry->d4 = d4; + ), + + TP_printk("event=%d", __entry->event_id) + ); + +/** + * Define a tracepoint used to regsiter the value of a hardware counter. + * Hardware counters belonging to the vertex or fragment processor are + * reported via this tracepoint each frame, whilst L2 cache hardware + * counters are reported continuously. + * + * @param counter_id The counter ID. + * @param value The value of the counter. + */ +TRACE_EVENT(mali_hw_counter, + + TP_PROTO(unsigned int counter_id, unsigned int value), + + TP_ARGS(counter_id, value), + + TP_STRUCT__entry( + __field(unsigned int, counter_id) + __field(unsigned int, value) + ), + + TP_fast_assign( + __entry->counter_id = counter_id; + ), + + TP_printk("event %d = %d", __entry->counter_id, __entry->value) + ); + +/** + * Define a tracepoint used to send a bundle of software counters. + * + * @param counters The bundle of counters. + */ +TRACE_EVENT(mali_sw_counters, + + TP_PROTO(pid_t pid, pid_t tid, void *surface_id, unsigned int *counters), + + TP_ARGS(pid, tid, surface_id, counters), + + TP_STRUCT__entry( + __field(pid_t, pid) + __field(pid_t, tid) + __field(void *, surface_id) + __field(unsigned int *, counters) + ), + + TP_fast_assign( + __entry->pid = pid; + __entry->tid = tid; + __entry->surface_id = surface_id; + __entry->counters = counters; + ), + + TP_printk("counters were %s", __entry->counters == NULL ? "NULL" : "not NULL") + ); + +/** + * Define a tracepoint used to gather core activity for systrace + * @param pid The process id for which the core activity originates from + * @param active If the core is active (1) or not (0) + * @param core_type The type of core active, either GP (1) or PP (0) + * @param core_id The core id that is active for the core_type + * @param frame_builder_id The frame builder id associated with this core activity + * @param flush_id The flush id associated with this core activity + */ +TRACE_EVENT(mali_core_active, + + TP_PROTO(pid_t pid, unsigned int active, unsigned int core_type, unsigned int core_id, unsigned int frame_builder_id, unsigned int flush_id), + + TP_ARGS(pid, active, core_type, core_id, frame_builder_id, flush_id), + + TP_STRUCT__entry( + __field(pid_t, pid) + __field(unsigned int, active) + __field(unsigned int, core_type) + __field(unsigned int, core_id) + __field(unsigned int, frame_builder_id) + __field(unsigned int, flush_id) + ), + + TP_fast_assign( + __entry->pid = pid; + __entry->active = active; + __entry->core_type = core_type; + __entry->core_id = core_id; + __entry->frame_builder_id = frame_builder_id; + __entry->flush_id = flush_id; + ), + + TP_printk("%s|%d|%s%i:%x|%d", __entry->active ? "S" : "F", __entry->pid, __entry->core_type ? "GP" : "PP", __entry->core_id, __entry->flush_id, __entry->frame_builder_id) + ); + +#endif /* MALI_LINUX_TRACE_H */ + +/* This part must exist outside the header guard. */ +#include + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory.c new file mode 100755 index 000000000000..dfc769e6cc40 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory.c @@ -0,0 +1,531 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_executor.h" + +#include "mali_memory.h" +#include "mali_memory_os_alloc.h" +#include "mali_memory_block_alloc.h" +#include "mali_memory_util.h" +#include "mali_memory_virtual.h" +#include "mali_memory_manager.h" +#include "mali_memory_cow.h" +#include "mali_memory_swap_alloc.h" +#include "mali_memory_defer_bind.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include "mali_memory_secure.h" +#endif + +extern unsigned int mali_dedicated_mem_size; +extern unsigned int mali_shared_mem_size; + +#define MALI_VM_NUM_FAULT_PREFETCH (0x8) + +static void mali_mem_vma_open(struct vm_area_struct *vma) +{ + mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; + MALI_DEBUG_PRINT(4, ("Open called on vma %p\n", vma)); + + /* If need to share the allocation, add ref_count here */ + mali_allocation_ref(alloc); + return; +} +static void mali_mem_vma_close(struct vm_area_struct *vma) +{ + /* If need to share the allocation, unref ref_count here */ + mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; + + mali_allocation_unref(&alloc); + vma->vm_private_data = NULL; +} + +static vm_fault_t mali_mem_vma_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; + mali_mem_backend *mem_bkend = NULL; + int ret; + int prefetch_num = MALI_VM_NUM_FAULT_PREFETCH; + + unsigned long address = (unsigned long)vmf->address; + MALI_DEBUG_ASSERT(alloc->backend_handle); + MALI_DEBUG_ASSERT((unsigned long)alloc->cpu_mapping.addr <= address); + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) { + MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n")); + mutex_unlock(&mali_idr_mutex); + return VM_FAULT_SIGBUS; + } + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(mem_bkend->type == alloc->type); + + if ((mem_bkend->type == MALI_MEM_COW && (MALI_MEM_BACKEND_FLAG_SWAP_COWED != + (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) && + (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE)) { + /*check if use page fault to do COW*/ + MALI_DEBUG_PRINT(4, ("mali_vma_fault: do cow allocate on demand!, address=0x%x\n", address)); + mutex_lock(&mem_bkend->mutex); + ret = mali_mem_cow_allocate_on_demand(mem_bkend, + (address - vma->vm_start) / PAGE_SIZE); + mutex_unlock(&mem_bkend->mutex); + + if (ret != _MALI_OSK_ERR_OK) { + return VM_FAULT_OOM; + } + prefetch_num = 1; + + /* handle COW modified range cpu mapping + we zap the mapping in cow_modify_range, it will trigger page fault + when CPU access it, so here we map it to CPU*/ + mutex_lock(&mem_bkend->mutex); + ret = mali_mem_cow_cpu_map_pages_locked(mem_bkend, vma, address, prefetch_num); + mutex_unlock(&mem_bkend->mutex); + + if (unlikely(ret != _MALI_OSK_ERR_OK)) { + return VM_FAULT_SIGBUS; + } + } else if ((mem_bkend->type == MALI_MEM_SWAP) || + (mem_bkend->type == MALI_MEM_COW && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + u32 offset_in_bkend = (address - vma->vm_start) / PAGE_SIZE; + int ret = _MALI_OSK_ERR_OK; + + mutex_lock(&mem_bkend->mutex); + if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE) { + ret = mali_mem_swap_cow_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page); + } else { + ret = mali_mem_swap_allocate_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page); + } + mutex_unlock(&mem_bkend->mutex); + + if (ret != _MALI_OSK_ERR_OK) { + MALI_DEBUG_PRINT(2, ("Mali swap memory page fault process failed, address=0x%x\n", address)); + return VM_FAULT_OOM; + } else { + return VM_FAULT_LOCKED; + } + } else { + MALI_PRINT_ERROR(("Mali vma fault! It never happen, indicating some logic errors in caller.\n")); + /*NOT support yet or OOM*/ + return VM_FAULT_OOM; + } + return VM_FAULT_NOPAGE; +} + +static struct vm_operations_struct mali_kernel_vm_ops = { + .open = mali_mem_vma_open, + .close = mali_mem_vma_close, + .fault = mali_mem_vma_fault, +}; + + +/** @ map mali allocation to CPU address +* +* Supported backend types: +* --MALI_MEM_OS +* -- need to add COW? + *Not supported backend types: +* -_MALI_MEMORY_BIND_BACKEND_UMP +* -_MALI_MEMORY_BIND_BACKEND_DMA_BUF +* -_MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY +* +*/ +int mali_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct mali_session_data *session; + mali_mem_allocation *mali_alloc = NULL; + u32 mali_addr = vma->vm_pgoff << PAGE_SHIFT; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_backend *mem_bkend = NULL; + int ret = -EFAULT; + + session = (struct mali_session_data *)filp->private_data; + if (NULL == session) { + MALI_PRINT_ERROR(("mmap called without any session data available\n")); + return -EFAULT; + } + + MALI_DEBUG_PRINT(4, ("MMap() handler: start=0x%08X, phys=0x%08X, size=0x%08X vma->flags 0x%08x\n", + (unsigned int)vma->vm_start, (unsigned int)(vma->vm_pgoff << PAGE_SHIFT), + (unsigned int)(vma->vm_end - vma->vm_start), vma->vm_flags)); + + /* Operations used on any memory system */ + /* do not need to anything in vm open/close now */ + + /* find mali allocation structure by vaddress*/ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + if (likely(mali_vma_node)) { + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start); + if (unlikely(mali_addr != mali_vma_node->vm_node.start)) { + /* only allow to use start address for mmap */ + MALI_DEBUG_PRINT(1, ("mali_addr != mali_vma_node->vm_node.start\n")); + return -EFAULT; + } + } else { + MALI_DEBUG_ASSERT(NULL == mali_vma_node); + return -EFAULT; + } + + mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start; + + if (mali_alloc->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) { + MALI_DEBUG_PRINT(1, ("ERROR : trying to access varying memory by CPU!\n")); + return -EFAULT; + } + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + if (!(mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle))) { + MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n")); + mutex_unlock(&mali_idr_mutex); + return -EFAULT; + } + mutex_unlock(&mali_idr_mutex); + + if (!(MALI_MEM_SWAP == mali_alloc->type || + (MALI_MEM_COW == mali_alloc->type && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) { + /* Set some bits which indicate that, the memory is IO memory, meaning + * that no paging is to be performed and the memory should not be + * included in crash dumps. And that the memory is reserved, meaning + * that it's present and can never be paged out (see also previous + * entry) + */ + vma->vm_flags |= VM_IO; + vma->vm_flags |= VM_DONTCOPY; + vma->vm_flags |= VM_PFNMAP; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) + vma->vm_flags |= VM_RESERVED; +#else + vma->vm_flags |= VM_DONTDUMP; + vma->vm_flags |= VM_DONTEXPAND; +#endif + } else if (MALI_MEM_SWAP == mali_alloc->type) { + vma->vm_pgoff = mem_bkend->start_idx; + } + + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + vma->vm_ops = &mali_kernel_vm_ops; + + mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start; + + /* If it's a copy-on-write mapping, map to read only */ + if (!(vma->vm_flags & VM_WRITE)) { + MALI_DEBUG_PRINT(4, ("mmap allocation with read only !\n")); + /* add VM_WRITE for do_page_fault will check this when a write fault */ + vma->vm_flags |= VM_WRITE | VM_READ; + vma->vm_page_prot = PAGE_READONLY; + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE; + goto out; + } + + if (mem_bkend->type == MALI_MEM_OS) { + ret = mali_mem_os_cpu_map(mem_bkend, vma); + } else if (mem_bkend->type == MALI_MEM_COW && + (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + ret = mali_mem_cow_cpu_map(mem_bkend, vma); + } else if (mem_bkend->type == MALI_MEM_BLOCK) { + ret = mali_mem_block_cpu_map(mem_bkend, vma); + } else if ((mem_bkend->type == MALI_MEM_SWAP) || (mem_bkend->type == MALI_MEM_COW && + (MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) { + /*For swappable memory, CPU page table will be created by page fault handler. */ + ret = 0; + } else if (mem_bkend->type == MALI_MEM_SECURE) { +#if defined(CONFIG_DMA_SHARED_BUFFER) + ret = mali_mem_secure_cpu_map(mem_bkend, vma); +#else + MALI_DEBUG_PRINT(1, ("DMA not supported for mali secure memory\n")); + return -EFAULT; +#endif + } else { + /* Not support yet*/ + MALI_DEBUG_PRINT_ERROR(("Invalid type of backend memory! \n")); + return -EFAULT; + } + + if (ret != 0) { + MALI_DEBUG_PRINT(1, ("ret != 0\n")); + return -EFAULT; + } +out: + MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == mali_alloc->magic); + + vma->vm_private_data = (void *)mali_alloc; + mali_alloc->cpu_mapping.vma = vma; + + mali_allocation_ref(mali_alloc); + + return 0; +} + +_mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor) +{ + u32 size = descriptor->psize; + struct mali_session_data *session = descriptor->session; + + MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); + + /* Map dma-buf into this session's page tables */ + + if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + size += MALI_MMU_PAGE_SIZE; + } + + return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start, size); +} + +_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size) +{ + u32 old_size = descriptor->psize; + struct mali_session_data *session = descriptor->session; + + MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); + + if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + new_size += MALI_MMU_PAGE_SIZE; + } + + if (new_size > old_size) { + MALI_DEBUG_ASSERT(new_size <= descriptor->mali_vma_node.vm_node.size); + return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start + old_size, new_size - old_size); + } + return _MALI_OSK_ERR_OK; +} + +void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags) +{ + if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + size += MALI_MMU_PAGE_SIZE; + } + + /* Umap and flush L2 */ + mali_mmu_pagedir_unmap(session->page_directory, vaddr, size); + mali_executor_zap_all_active(session); +} + +u32 _mali_ukk_report_memory_usage(void) +{ + u32 sum = 0; + + if (MALI_TRUE == mali_memory_have_dedicated_memory()) { + sum += mali_mem_block_allocator_stat(); + } + + sum += mali_mem_os_stat(); + + return sum; +} + +u32 _mali_ukk_report_total_memory_size(void) +{ + return mali_dedicated_mem_size + mali_shared_mem_size; +} + + +/** + * Per-session memory descriptor mapping table sizes + */ +#define MALI_MEM_DESCRIPTORS_INIT 64 +#define MALI_MEM_DESCRIPTORS_MAX 65536 + +_mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session_data) +{ + MALI_DEBUG_PRINT(5, ("Memory session begin\n")); + + session_data->memory_lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_MEM_SESSION); + + if (NULL == session_data->memory_lock) { + MALI_ERROR(_MALI_OSK_ERR_FAULT); + } + + session_data->cow_lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_UNORDERED, 0); + if (NULL == session_data->cow_lock) { + _mali_osk_mutex_term(session_data->memory_lock); + MALI_ERROR(_MALI_OSK_ERR_FAULT); + } + + mali_memory_manager_init(&session_data->allocation_mgr); + + MALI_DEBUG_PRINT(5, ("MMU session begin: success\n")); + MALI_SUCCESS; +} + +void mali_memory_session_end(struct mali_session_data *session) +{ + MALI_DEBUG_PRINT(3, ("MMU session end\n")); + + if (NULL == session) { + MALI_DEBUG_PRINT(1, ("No session data found during session end\n")); + return; + } + /* free allocation */ + mali_free_session_allocations(session); + /* do some check in unint*/ + mali_memory_manager_uninit(&session->allocation_mgr); + + /* Free the lock */ + _mali_osk_mutex_term(session->memory_lock); + _mali_osk_mutex_term(session->cow_lock); + return; +} + +_mali_osk_errcode_t mali_memory_initialize(void) +{ + _mali_osk_errcode_t err; + + idr_init(&mali_backend_idr); + mutex_init(&mali_idr_mutex); + + err = mali_mem_swap_init(); + if (err != _MALI_OSK_ERR_OK) { + return err; + } + err = mali_mem_os_init(); + if (_MALI_OSK_ERR_OK == err) { + err = mali_mem_defer_bind_manager_init(); + } + + return err; +} + +void mali_memory_terminate(void) +{ + mali_mem_swap_term(); + mali_mem_defer_bind_manager_destory(); + mali_mem_os_term(); + if (mali_memory_have_dedicated_memory()) { + mali_mem_block_allocator_destroy(); + } +} + + +struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type) +{ + mali_page_node *page_node = NULL; + + page_node = kzalloc(sizeof(mali_page_node), GFP_KERNEL); + MALI_DEBUG_ASSERT(NULL != page_node); + + if (page_node) { + page_node->type = type; + INIT_LIST_HEAD(&page_node->list); + } + + return page_node; +} + +void _mali_page_node_ref(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + /* add ref to this page */ + get_page(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + mali_mem_block_add_ref(node); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + atomic_inc(&node->swap_it->ref_count); + } else { + MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); + } +} + +void _mali_page_node_unref(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + /* unref to this page */ + put_page(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + mali_mem_block_dec_ref(node); + } else { + MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); + } +} + + +void _mali_page_node_add_page(struct mali_page_node *node, struct page *page) +{ + MALI_DEBUG_ASSERT(MALI_PAGE_NODE_OS == node->type); + node->page = page; +} + + +void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item) +{ + MALI_DEBUG_ASSERT(MALI_PAGE_NODE_SWAP == node->type); + node->swap_it = item; +} + +void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item) +{ + MALI_DEBUG_ASSERT(MALI_PAGE_NODE_BLOCK == node->type); + node->blk_it = item; +} + + +int _mali_page_node_get_ref_count(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + /* get ref count of this page */ + return page_count(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + return mali_mem_block_get_ref_count(node); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return atomic_read(&node->swap_it->ref_count); + } else { + MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); + } + return -1; +} + + +dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + return page_private(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + return _mali_blk_item_get_phy_addr(node->blk_it); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return node->swap_it->dma_addr; + } else { + MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); + } + return 0; +} + + +unsigned long _mali_page_node_get_pfn(struct mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + return page_to_pfn(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + /* get phy addr for BLOCK page*/ + return _mali_blk_item_get_pfn(node->blk_it); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return page_to_pfn(node->swap_it->page); + } else { + MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); + } + return 0; +} + + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory.h new file mode 100755 index 000000000000..efebbef235d8 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory.h @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_H__ +#define __MALI_MEMORY_H__ + +#include "mali_osk.h" +#include "mali_session.h" + +#include +#include + +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" + +_mali_osk_errcode_t mali_memory_initialize(void); +void mali_memory_terminate(void); + +/** @brief Allocate a page table page + * + * Allocate a page for use as a page directory or page table. The page is + * mapped into kernel space. + * + * @return _MALI_OSK_ERR_OK on success, otherwise an error code + * @param table_page GPU pointer to the allocated page + * @param mapping CPU pointer to the mapping of the allocated page + */ +MALI_STATIC_INLINE _mali_osk_errcode_t +mali_mmu_get_table_page(mali_dma_addr *table_page, mali_io_address *mapping) +{ + return mali_mem_os_get_table_page(table_page, mapping); +} + +/** @brief Release a page table page + * + * Release a page table page allocated through \a mali_mmu_get_table_page + * + * @param pa the GPU address of the page to release + */ +MALI_STATIC_INLINE void +mali_mmu_release_table_page(mali_dma_addr phys, void *virt) +{ + mali_mem_os_release_table_page(phys, virt); +} + +/** @brief mmap function + * + * mmap syscalls on the Mali device node will end up here. + * + * This function allocates Mali memory and maps it on CPU and Mali. + */ +int mali_mmap(struct file *filp, struct vm_area_struct *vma); + +/** @brief Start a new memory session + * + * Called when a process opens the Mali device node. + * + * @param session Pointer to session to initialize + */ +_mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session); + +/** @brief Close a memory session + * + * Called when a process closes the Mali device node. + * + * Memory allocated by the session will be freed + * + * @param session Pointer to the session to terminate + */ +void mali_memory_session_end(struct mali_session_data *session); + +/** @brief Prepare Mali page tables for mapping + * + * This function will prepare the Mali page tables for mapping the memory + * described by \a descriptor. + * + * Page tables will be reference counted and allocated, if not yet present. + * + * @param descriptor Pointer to the memory descriptor to the mapping + */ +_mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor); + +/** @brief Resize Mali page tables for mapping + * + * This function will Resize the Mali page tables for mapping the memory + * described by \a descriptor. + * + * Page tables will be reference counted and allocated, if not yet present. + * + * @param descriptor Pointer to the memory descriptor to the mapping + * @param new_size The new size of descriptor + */ +_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size); + +/** @brief Free Mali page tables for mapping + * + * This function will unmap pages from Mali memory and free the page tables + * that are now unused. + * + * The updated pages in the Mali L2 cache will be invalidated, and the MMU TLBs will be zapped if necessary. + * + * @param descriptor Pointer to the memory descriptor to unmap + */ +void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags); + +/** @brief Parse resource and prepare the OS memory allocator + * + * @param size Maximum size to allocate for Mali GPU. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size); + +/** @brief Parse resource and prepare the dedicated memory allocator + * + * @param start Physical start address of dedicated Mali GPU memory. + * @param size Size of dedicated Mali GPU memory. + * @return _MALI_OSK_ERR_OK on success, otherwise failure. + */ +_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size); + + +struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type); + +void _mali_page_node_ref(struct mali_page_node *node); +void _mali_page_node_unref(struct mali_page_node *node); +void _mali_page_node_add_page(struct mali_page_node *node, struct page *page); + +void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item); + +void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item); + +int _mali_page_node_get_ref_count(struct mali_page_node *node); +dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node); +unsigned long _mali_page_node_get_pfn(struct mali_page_node *node); + +#endif /* __MALI_MEMORY_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c new file mode 100755 index 000000000000..bccef3576914 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c @@ -0,0 +1,362 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_memory.h" +#include "mali_memory_block_alloc.h" +#include "mali_osk.h" +#include + + +static mali_block_allocator *mali_mem_block_gobal_allocator = NULL; + +unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item) +{ + return (item->phy_addr & ~(MALI_BLOCK_REF_MASK)); +} + + +unsigned long _mali_blk_item_get_pfn(mali_block_item *item) +{ + return (item->phy_addr / MALI_BLOCK_SIZE); +} + + +u32 mali_mem_block_get_ref_count(mali_page_node *node) +{ + MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); + return (node->blk_it->phy_addr & MALI_BLOCK_REF_MASK); +} + + +/* Increase the refence count +* It not atomic, so it need to get sp_lock before call this function +*/ + +u32 mali_mem_block_add_ref(mali_page_node *node) +{ + MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); + MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) < MALI_BLOCK_MAX_REF_COUNT); + return (node->blk_it->phy_addr++ & MALI_BLOCK_REF_MASK); +} + +/* Decase the refence count +* It not atomic, so it need to get sp_lock before call this function +*/ +u32 mali_mem_block_dec_ref(mali_page_node *node) +{ + MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); + MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) > 0); + return (node->blk_it->phy_addr-- & MALI_BLOCK_REF_MASK); +} + + +static mali_block_allocator *mali_mem_block_allocator_create(u32 base_address, u32 size) +{ + mali_block_allocator *info; + u32 usable_size; + u32 num_blocks; + mali_page_node *m_node; + mali_block_item *mali_blk_items = NULL; + int i = 0; + + usable_size = size & ~(MALI_BLOCK_SIZE - 1); + MALI_DEBUG_PRINT(3, ("Mali block allocator create for region starting at 0x%08X length 0x%08X\n", base_address, size)); + MALI_DEBUG_PRINT(4, ("%d usable bytes\n", usable_size)); + num_blocks = usable_size / MALI_BLOCK_SIZE; + MALI_DEBUG_PRINT(4, ("which becomes %d blocks\n", num_blocks)); + + if (usable_size == 0) { + MALI_DEBUG_PRINT(1, ("Memory block of size %d is unusable\n", size)); + return NULL; + } + + info = _mali_osk_calloc(1, sizeof(mali_block_allocator)); + if (NULL != info) { + INIT_LIST_HEAD(&info->free); + spin_lock_init(&info->sp_lock); + info->total_num = num_blocks; + mali_blk_items = _mali_osk_calloc(1, sizeof(mali_block_item) * num_blocks); + + if (mali_blk_items) { + info->items = mali_blk_items; + /* add blocks(4k size) to free list*/ + for (i = 0 ; i < num_blocks ; i++) { + /* add block information*/ + mali_blk_items[i].phy_addr = base_address + (i * MALI_BLOCK_SIZE); + /* add to free list */ + m_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK); + if (m_node == NULL) + goto fail; + _mali_page_node_add_block_item(m_node, &(mali_blk_items[i])); + list_add_tail(&m_node->list, &info->free); + atomic_add(1, &info->free_num); + } + return info; + } + } +fail: + mali_mem_block_allocator_destroy(); + return NULL; +} + +void mali_mem_block_allocator_destroy(void) +{ + struct mali_page_node *m_page, *m_tmp; + mali_block_allocator *info = mali_mem_block_gobal_allocator; + MALI_DEBUG_ASSERT_POINTER(info); + MALI_DEBUG_PRINT(4, ("Memory block destroy !\n")); + + if (NULL == info) + return; + + list_for_each_entry_safe(m_page, m_tmp , &info->free, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + list_del(&m_page->list); + kfree(m_page); + } + + _mali_osk_free(info->items); + _mali_osk_free(info); +} + +u32 mali_mem_block_release(mali_mem_backend *mem_bkend) +{ + mali_mem_allocation *alloc = mem_bkend->mali_allocation; + u32 free_pages_nr = 0; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK); + + /* Unmap the memory from the mali virtual address space. */ + mali_mem_block_mali_unmap(alloc); + mutex_lock(&mem_bkend->mutex); + free_pages_nr = mali_mem_block_free(&mem_bkend->block_mem); + mutex_unlock(&mem_bkend->mutex); + return free_pages_nr; +} + + +int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size) +{ + struct mali_page_node *m_page, *m_tmp; + size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE; + mali_block_allocator *info = mali_mem_block_gobal_allocator; + MALI_DEBUG_ASSERT_POINTER(info); + + MALI_DEBUG_PRINT(4, ("BLOCK Mem: Allocate size = 0x%x\n", size)); + /*do some init */ + INIT_LIST_HEAD(&block_mem->pfns); + + spin_lock(&info->sp_lock); + /*check if have enough space*/ + if (atomic_read(&info->free_num) > page_count) { + list_for_each_entry_safe(m_page, m_tmp , &info->free, list) { + if (page_count > 0) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(m_page) == 0); + list_move(&m_page->list, &block_mem->pfns); + block_mem->count++; + atomic_dec(&info->free_num); + _mali_page_node_ref(m_page); + } else { + break; + } + page_count--; + } + } else { + /* can't allocate from BLOCK memory*/ + spin_unlock(&info->sp_lock); + return -1; + } + + spin_unlock(&info->sp_lock); + return 0; +} + +u32 mali_mem_block_free(mali_mem_block_mem *block_mem) +{ + u32 free_pages_nr = 0; + + free_pages_nr = mali_mem_block_free_list(&block_mem->pfns); + MALI_DEBUG_PRINT(4, ("BLOCK Mem free : allocated size = 0x%x, free size = 0x%x\n", block_mem->count * _MALI_OSK_MALI_PAGE_SIZE, + free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); + block_mem->count = 0; + MALI_DEBUG_ASSERT(list_empty(&block_mem->pfns)); + + return free_pages_nr; +} + + +u32 mali_mem_block_free_list(struct list_head *list) +{ + struct mali_page_node *m_page, *m_tmp; + mali_block_allocator *info = mali_mem_block_gobal_allocator; + u32 free_pages_nr = 0; + + if (info) { + spin_lock(&info->sp_lock); + list_for_each_entry_safe(m_page, m_tmp , list, list) { + if (1 == _mali_page_node_get_ref_count(m_page)) { + free_pages_nr++; + } + mali_mem_block_free_node(m_page); + } + spin_unlock(&info->sp_lock); + } + return free_pages_nr; +} + +/* free the node,*/ +void mali_mem_block_free_node(struct mali_page_node *node) +{ + mali_block_allocator *info = mali_mem_block_gobal_allocator; + + /* only handle BLOCK node */ + if (node->type == MALI_PAGE_NODE_BLOCK && info) { + /*Need to make this atomic?*/ + if (1 == _mali_page_node_get_ref_count(node)) { + /*Move to free list*/ + _mali_page_node_unref(node); + list_move_tail(&node->list, &info->free); + atomic_add(1, &info->free_num); + } else { + _mali_page_node_unref(node); + list_del(&node->list); + kfree(node); + } + } +} + +/* unref the node, but not free it */ +_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node) +{ + mali_block_allocator *info = mali_mem_block_gobal_allocator; + mali_page_node *new_node; + + /* only handle BLOCK node */ + if (node->type == MALI_PAGE_NODE_BLOCK && info) { + /*Need to make this atomic?*/ + if (1 == _mali_page_node_get_ref_count(node)) { + /* allocate a new node, Add to free list, keep the old node*/ + _mali_page_node_unref(node); + new_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK); + if (new_node) { + memcpy(new_node, node, sizeof(mali_page_node)); + list_add(&new_node->list, &info->free); + atomic_add(1, &info->free_num); + } else + return _MALI_OSK_ERR_FAULT; + + } else { + _mali_page_node_unref(node); + } + } + return _MALI_OSK_ERR_OK; +} + + +int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props) +{ + struct mali_page_directory *pagedir = session->page_directory; + struct mali_page_node *m_page; + dma_addr_t phys; + u32 virt = vaddr; + u32 prop = props; + + list_for_each_entry(m_page, &block_mem->pfns, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + phys = _mali_page_node_get_dma_addr(m_page); +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); + virt += MALI_MMU_PAGE_SIZE; + } + + return 0; +} + +void mali_mem_block_mali_unmap(mali_mem_allocation *alloc) +{ + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + + +int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) +{ + int ret; + mali_mem_block_mem *block_mem = &mem_bkend->block_mem; + unsigned long addr = vma->vm_start; + struct mali_page_node *m_page; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK); + + list_for_each_entry(m_page, &block_mem->pfns, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); + ret = vmf_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page)); + + if (unlikely(0 != ret)) { + return -EFAULT; + } + addr += _MALI_OSK_MALI_PAGE_SIZE; + + } + + return 0; +} + + +_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size) +{ + mali_block_allocator *allocator; + + /* Do the low level linux operation first */ + + /* Request ownership of the memory */ + if (_MALI_OSK_ERR_OK != _mali_osk_mem_reqregion(start, size, "Dedicated Mali GPU memory")) { + MALI_DEBUG_PRINT(1, ("Failed to request memory region for frame buffer (0x%08X - 0x%08X)\n", start, start + size - 1)); + return _MALI_OSK_ERR_FAULT; + } + + /* Create generic block allocator object to handle it */ + allocator = mali_mem_block_allocator_create(start, size); + + if (NULL == allocator) { + MALI_DEBUG_PRINT(1, ("Memory bank registration failed\n")); + _mali_osk_mem_unreqregion(start, size); + MALI_ERROR(_MALI_OSK_ERR_FAULT); + } + + mali_mem_block_gobal_allocator = (mali_block_allocator *)allocator; + + return _MALI_OSK_ERR_OK; +} + +mali_bool mali_memory_have_dedicated_memory(void) +{ + return mali_mem_block_gobal_allocator ? MALI_TRUE : MALI_FALSE; +} + +u32 mali_mem_block_allocator_stat(void) +{ + mali_block_allocator *allocator = mali_mem_block_gobal_allocator; + MALI_DEBUG_ASSERT_POINTER(allocator); + + return (allocator->total_num - atomic_read(&allocator->free_num)) * _MALI_OSK_MALI_PAGE_SIZE; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h new file mode 100755 index 000000000000..70fd9ec25f50 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2010, 2013, 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_BLOCK_ALLOCATOR_H__ +#define __MALI_BLOCK_ALLOCATOR_H__ + +#include "mali_session.h" +#include "mali_memory.h" +#include + +#include "mali_memory_types.h" + +#define MALI_BLOCK_SIZE (PAGE_SIZE) /* 4 kB, manage BLOCK memory as page size */ +#define MALI_BLOCK_REF_MASK (0xFFF) +#define MALI_BLOCK_MAX_REF_COUNT (0xFFF) + + + +typedef struct mali_block_allocator { + /* + * In free list, each node's ref_count is 0, + * ref_count added when allocated or referenced in COW + */ + mali_block_item *items; /* information for each block item*/ + struct list_head free; /*free list of mali_memory_node*/ + spinlock_t sp_lock; /*lock for reference count & free list opertion*/ + u32 total_num; /* Number of total pages*/ + atomic_t free_num; /*number of free pages*/ +} mali_block_allocator; + +unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item); +unsigned long _mali_blk_item_get_pfn(mali_block_item *item); +u32 mali_mem_block_get_ref_count(mali_page_node *node); +u32 mali_mem_block_add_ref(mali_page_node *node); +u32 mali_mem_block_dec_ref(mali_page_node *node); +u32 mali_mem_block_release(mali_mem_backend *mem_bkend); +int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size); +int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props); +void mali_mem_block_mali_unmap(mali_mem_allocation *alloc); + +int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); +_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size); +mali_bool mali_memory_have_dedicated_memory(void); +u32 mali_mem_block_free(mali_mem_block_mem *block_mem); +u32 mali_mem_block_free_list(struct list_head *list); +void mali_mem_block_free_node(struct mali_page_node *node); +void mali_mem_block_allocator_destroy(void); +_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node); +u32 mali_mem_block_allocator_stat(void); + +#endif /* __MALI_BLOCK_ALLOCATOR_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c new file mode 100755 index 000000000000..0bdf90b167d6 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c @@ -0,0 +1,776 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_ARM +#include +#endif +#include + +#include "mali_memory.h" +#include "mali_kernel_common.h" +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_kernel_linux.h" +#include "mali_memory_cow.h" +#include "mali_memory_block_alloc.h" +#include "mali_memory_swap_alloc.h" + +/** +* allocate pages for COW backend and flush cache +*/ +static struct page *mali_mem_cow_alloc_page(void) + +{ + mali_mem_os_mem os_mem; + struct mali_page_node *node; + struct page *new_page; + + int ret = 0; + /* allocate pages from os mem */ + ret = mali_mem_os_alloc_pages(&os_mem, _MALI_OSK_MALI_PAGE_SIZE); + + if (ret) { + return NULL; + } + + MALI_DEBUG_ASSERT(1 == os_mem.count); + + node = _MALI_OSK_CONTAINER_OF(os_mem.pages.next, struct mali_page_node, list); + new_page = node->page; + node->page = NULL; + list_del(&node->list); + kfree(node); + + return new_page; +} + + +static struct list_head *_mali_memory_cow_get_node_list(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size) +{ + MALI_DEBUG_ASSERT(MALI_MEM_OS == target_bk->type || MALI_MEM_COW == target_bk->type || + MALI_MEM_BLOCK == target_bk->type || MALI_MEM_SWAP == target_bk->type); + + if (MALI_MEM_OS == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->os_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->os_mem.count); + return &target_bk->os_mem.pages; + } else if (MALI_MEM_COW == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->cow_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->cow_mem.count); + return &target_bk->cow_mem.pages; + } else if (MALI_MEM_BLOCK == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->block_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->block_mem.count); + return &target_bk->block_mem.pfns; + } else if (MALI_MEM_SWAP == target_bk->type) { + MALI_DEBUG_ASSERT(&target_bk->swap_mem); + MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->swap_mem.count); + return &target_bk->swap_mem.pages; + } + + return NULL; +} + +/** +* Do COW for os memory - support do COW for memory from bank memory +* The range_start/size can be zero, which means it will call cow_modify_range +* latter. +* This function allocate new pages for COW backend from os mem for a modified range +* It will keep the page which not in the modified range and Add ref to it +* +* @target_bk - target allocation's backend(the allocation need to do COW) +* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align) +* @target_size - size of target allocation to do COW (for support memory bank) +* @backend -COW backend +* @range_start - offset of modified range (4K align) +* @range_size - size of modified range +*/ +_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + mali_mem_cow *cow = &backend->cow_mem; + struct mali_page_node *m_page, *m_tmp, *page_node; + int target_page = 0; + struct page *new_page; + struct list_head *pages = NULL; + + pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size); + + if (NULL == pages) { + MALI_DEBUG_PRINT_ERROR(("No memory page need to cow ! \n")); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT(0 == cow->count); + + INIT_LIST_HEAD(&cow->pages); + mutex_lock(&target_bk->mutex); + list_for_each_entry_safe(m_page, m_tmp, pages, list) { + /* add page from (target_offset,target_offset+size) to cow backend */ + if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) && + (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) { + + /* allocate a new page node, alway use OS memory for COW */ + page_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS); + + if (NULL == page_node) { + mutex_unlock(&target_bk->mutex); + goto error; + } + + INIT_LIST_HEAD(&page_node->list); + + /* check if in the modified range*/ + if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && + (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { + /* need to allocate a new page */ + /* To simplify the case, All COW memory is allocated from os memory ?*/ + new_page = mali_mem_cow_alloc_page(); + + if (NULL == new_page) { + kfree(page_node); + mutex_unlock(&target_bk->mutex); + goto error; + } + + _mali_page_node_add_page(page_node, new_page); + } else { + /*Add Block memory case*/ + if (m_page->type != MALI_PAGE_NODE_BLOCK) { + _mali_page_node_add_page(page_node, m_page->page); + } else { + page_node->type = MALI_PAGE_NODE_BLOCK; + _mali_page_node_add_block_item(page_node, m_page->blk_it); + } + + /* add ref to this page */ + _mali_page_node_ref(m_page); + } + + /* add it to COW backend page list */ + list_add_tail(&page_node->list, &cow->pages); + cow->count++; + } + target_page++; + } + mutex_unlock(&target_bk->mutex); + return _MALI_OSK_ERR_OK; +error: + mali_mem_cow_release(backend, MALI_FALSE); + return _MALI_OSK_ERR_FAULT; +} + +_mali_osk_errcode_t mali_memory_cow_swap_memory(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + mali_mem_cow *cow = &backend->cow_mem; + struct mali_page_node *m_page, *m_tmp, *page_node; + int target_page = 0; + struct mali_swap_item *swap_item; + struct list_head *pages = NULL; + + pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size); + if (NULL == pages) { + MALI_DEBUG_PRINT_ERROR(("No swap memory page need to cow ! \n")); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_ASSERT(0 == cow->count); + + INIT_LIST_HEAD(&cow->pages); + mutex_lock(&target_bk->mutex); + + backend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN; + + list_for_each_entry_safe(m_page, m_tmp, pages, list) { + /* add page from (target_offset,target_offset+size) to cow backend */ + if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) && + (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) { + + /* allocate a new page node, use swap memory for COW memory swap cowed flag. */ + page_node = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP); + + if (NULL == page_node) { + mutex_unlock(&target_bk->mutex); + goto error; + } + + /* check if in the modified range*/ + if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && + (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { + /* need to allocate a new page */ + /* To simplify the case, All COW memory is allocated from os memory ?*/ + swap_item = mali_mem_swap_alloc_swap_item(); + + if (NULL == swap_item) { + kfree(page_node); + mutex_unlock(&target_bk->mutex); + goto error; + } + + swap_item->idx = mali_mem_swap_idx_alloc(); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) { + MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW.\n")); + kfree(page_node); + kfree(swap_item); + mutex_unlock(&target_bk->mutex); + goto error; + } + + _mali_page_node_add_swap_item(page_node, swap_item); + } else { + _mali_page_node_add_swap_item(page_node, m_page->swap_it); + + /* add ref to this page */ + _mali_page_node_ref(m_page); + } + + list_add_tail(&page_node->list, &cow->pages); + cow->count++; + } + target_page++; + } + mutex_unlock(&target_bk->mutex); + + return _MALI_OSK_ERR_OK; +error: + mali_mem_swap_release(backend, MALI_FALSE); + return _MALI_OSK_ERR_FAULT; + +} + + +_mali_osk_errcode_t _mali_mem_put_page_node(mali_page_node *node) +{ + if (node->type == MALI_PAGE_NODE_OS) { + return mali_mem_os_put_page(node->page); + } else if (node->type == MALI_PAGE_NODE_BLOCK) { + return mali_mem_block_unref_node(node); + } else if (node->type == MALI_PAGE_NODE_SWAP) { + return _mali_mem_swap_put_page_node(node); + } else + MALI_DEBUG_ASSERT(0); + return _MALI_OSK_ERR_FAULT; +} + + +/** +* Modify a range of a exist COW backend +* @backend -COW backend +* @range_start - offset of modified range (4K align) +* @range_size - size of modified range(in byte) +*/ +_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + mali_mem_allocation *alloc = NULL; + struct mali_session_data *session; + mali_mem_cow *cow = &backend->cow_mem; + struct mali_page_node *m_page, *m_tmp; + LIST_HEAD(pages); + struct page *new_page; + u32 count = 0; + s32 change_pages_nr = 0; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; + + if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + + alloc = backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type); + MALI_DEBUG_ASSERT(((range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE) <= cow->count); + + mutex_lock(&backend->mutex); + + /* free pages*/ + list_for_each_entry_safe(m_page, m_tmp, &cow->pages, list) { + + /* check if in the modified range*/ + if ((count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && + (count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { + if (MALI_PAGE_NODE_SWAP != m_page->type) { + new_page = mali_mem_cow_alloc_page(); + + if (NULL == new_page) { + goto error; + } + if (1 != _mali_page_node_get_ref_count(m_page)) + change_pages_nr++; + /* unref old page*/ + _mali_osk_mutex_wait(session->cow_lock); + if (_mali_mem_put_page_node(m_page)) { + __free_page(new_page); + _mali_osk_mutex_signal(session->cow_lock); + goto error; + } + _mali_osk_mutex_signal(session->cow_lock); + /* add new page*/ + /* always use OS for COW*/ + m_page->type = MALI_PAGE_NODE_OS; + _mali_page_node_add_page(m_page, new_page); + } else { + struct mali_swap_item *swap_item; + + swap_item = mali_mem_swap_alloc_swap_item(); + + if (NULL == swap_item) { + goto error; + } + + swap_item->idx = mali_mem_swap_idx_alloc(); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) { + MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW modify range.\n")); + kfree(swap_item); + goto error; + } + + if (1 != _mali_page_node_get_ref_count(m_page)) { + change_pages_nr++; + } + + if (_mali_mem_put_page_node(m_page)) { + mali_mem_swap_free_swap_item(swap_item); + goto error; + } + + _mali_page_node_add_swap_item(m_page, swap_item); + } + } + count++; + } + cow->change_pages_nr = change_pages_nr; + + MALI_DEBUG_ASSERT(MALI_MEM_COW == alloc->type); + + /* ZAP cpu mapping(modified range), and do cpu mapping here if need */ + if (NULL != alloc->cpu_mapping.vma) { + MALI_DEBUG_ASSERT(0 != alloc->backend_handle); + MALI_DEBUG_ASSERT(NULL != alloc->cpu_mapping.vma); + MALI_DEBUG_ASSERT(alloc->cpu_mapping.vma->vm_end - alloc->cpu_mapping.vma->vm_start >= range_size); + + if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { + zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size); + + ret = mali_mem_cow_cpu_map_pages_locked(backend, alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size / _MALI_OSK_MALI_PAGE_SIZE); + + if (unlikely(ret != _MALI_OSK_ERR_OK)) { + MALI_DEBUG_PRINT(2, ("mali_memory_cow_modify_range: cpu mapping failed !\n")); + ret = _MALI_OSK_ERR_FAULT; + } + } else { + /* used to trigger page fault for swappable cowed memory. */ + alloc->cpu_mapping.vma->vm_flags |= VM_PFNMAP; + alloc->cpu_mapping.vma->vm_flags |= VM_MIXEDMAP; + + zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size); + /* delete this flag to let swappble is ummapped regard to stauct page not page frame. */ + alloc->cpu_mapping.vma->vm_flags &= ~VM_PFNMAP; + alloc->cpu_mapping.vma->vm_flags &= ~VM_MIXEDMAP; + } + } + +error: + mutex_unlock(&backend->mutex); + return ret; + +} + + +/** +* Allocate pages for COW backend +* @alloc -allocation for COW allocation +* @target_bk - target allocation's backend(the allocation need to do COW) +* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align) +* @target_size - size of target allocation to do COW (for support memory bank)(in byte) +* @backend -COW backend +* @range_start - offset of modified range (4K align) +* @range_size - size of modified range(in byte) +*/ +_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size) +{ + struct mali_session_data *session = backend->mali_allocation->session; + + MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); + + /* size & offset must be a multiple of the system page size */ + if (target_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (target_offset % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + + /* check backend type */ + MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type); + + switch (target_bk->type) { + case MALI_MEM_OS: + case MALI_MEM_BLOCK: + return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + break; + case MALI_MEM_COW: + if (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) { + return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + } else { + return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + } + break; + case MALI_MEM_SWAP: + return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size); + break; + case MALI_MEM_EXTERNAL: + /*NOT support yet*/ + MALI_DEBUG_PRINT_ERROR(("External physical memory not supported ! \n")); + return _MALI_OSK_ERR_UNSUPPORTED; + break; + case MALI_MEM_DMA_BUF: + /*NOT support yet*/ + MALI_DEBUG_PRINT_ERROR(("DMA buffer not supported ! \n")); + return _MALI_OSK_ERR_UNSUPPORTED; + break; + case MALI_MEM_UMP: + /*NOT support yet*/ + MALI_DEBUG_PRINT_ERROR(("UMP buffer not supported ! \n")); + return _MALI_OSK_ERR_UNSUPPORTED; + break; + default: + /*Not support yet*/ + MALI_DEBUG_PRINT_ERROR(("Invalid memory type not supported ! \n")); + return _MALI_OSK_ERR_UNSUPPORTED; + break; + } + return _MALI_OSK_ERR_OK; +} + + +/** +* Map COW backend memory to mali +* Support OS/BLOCK for mali_page_node +*/ +int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size) +{ + mali_mem_allocation *cow_alloc; + struct mali_page_node *m_page; + struct mali_session_data *session; + struct mali_page_directory *pagedir; + u32 virt, start; + + cow_alloc = mem_bkend->mali_allocation; + virt = cow_alloc->mali_vma_node.vm_node.start; + start = virt; + + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT_POINTER(cow_alloc); + + session = cow_alloc->session; + pagedir = session->page_directory; + MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); + list_for_each_entry(m_page, &mem_bkend->cow_mem.pages, list) { + if ((virt - start >= range_start) && (virt - start < range_start + range_size)) { + dma_addr_t phys = _mali_page_node_get_dma_addr(m_page); +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, + MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + } + virt += MALI_MMU_PAGE_SIZE; + } + return 0; +} + +/** +* Map COW backend to cpu +* support OS/BLOCK memory +*/ +int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) +{ + mali_mem_cow *cow = &mem_bkend->cow_mem; + struct mali_page_node *m_page; + int ret; + unsigned long addr = vma->vm_start; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW); + + list_for_each_entry(m_page, &cow->pages, list) { + /* We should use vm_insert_page, but it does a dcache + * flush which makes it way slower than remap_pfn_range or vmf_insert_pfn. + ret = vm_insert_page(vma, addr, page); + */ + ret = vmf_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page)); + + if (unlikely(0 != ret)) { + return ret; + } + addr += _MALI_OSK_MALI_PAGE_SIZE; + } + + return 0; +} + +/** +* Map some pages(COW backend) to CPU vma@vaddr +*@ mem_bkend - COW backend +*@ vma +*@ vaddr -start CPU vaddr mapped to +*@ num - max number of pages to map to CPU vaddr +*/ +_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend, + struct vm_area_struct *vma, + unsigned long vaddr, + int num) +{ + mali_mem_cow *cow = &mem_bkend->cow_mem; + struct mali_page_node *m_page; + int ret; + int offset; + int count ; + unsigned long vstart = vma->vm_start; + count = 0; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW); + MALI_DEBUG_ASSERT(0 == vaddr % _MALI_OSK_MALI_PAGE_SIZE); + MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE); + offset = (vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE; + + list_for_each_entry(m_page, &cow->pages, list) { + if ((count >= offset) && (count < offset + num)) { + ret = vmf_insert_pfn(vma, vaddr, _mali_page_node_get_pfn(m_page)); + + if (unlikely(0 != ret)) { + if (count == offset) { + return _MALI_OSK_ERR_FAULT; + } else { + /* ret is EBUSY when page isn't in modify range, but now it's OK*/ + return _MALI_OSK_ERR_OK; + } + } + vaddr += _MALI_OSK_MALI_PAGE_SIZE; + } + count++; + } + return _MALI_OSK_ERR_OK; +} + +/** +* Release COW backend memory +* free it directly(put_page--unref page), not put into pool +*/ +u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped) +{ + mali_mem_allocation *alloc; + struct mali_session_data *session; + u32 free_pages_nr = 0; + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + alloc = mem_bkend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)) { + /* Unmap the memory from the mali virtual address space. */ + if (MALI_TRUE == is_mali_mapped) + mali_mem_os_mali_unmap(alloc); + /* free cow backend list*/ + _mali_osk_mutex_wait(session->cow_lock); + free_pages_nr = mali_mem_os_free(&mem_bkend->cow_mem.pages, mem_bkend->cow_mem.count, MALI_TRUE); + _mali_osk_mutex_signal(session->cow_lock); + + free_pages_nr += mali_mem_block_free_list(&mem_bkend->cow_mem.pages); + + MALI_DEBUG_ASSERT(list_empty(&mem_bkend->cow_mem.pages)); + } else { + free_pages_nr = mali_mem_swap_release(mem_bkend, is_mali_mapped); + } + + + MALI_DEBUG_PRINT(4, ("COW Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->cow_mem.count * _MALI_OSK_MALI_PAGE_SIZE, + free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); + + mem_bkend->cow_mem.count = 0; + return free_pages_nr; +} + + +/* Dst node could os node or swap node. */ +void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node) +{ + void *dst, *src; + struct page *dst_page; + dma_addr_t dma_addr; + + MALI_DEBUG_ASSERT(src_node != NULL); + MALI_DEBUG_ASSERT(dst_node != NULL); + MALI_DEBUG_ASSERT(dst_node->type == MALI_PAGE_NODE_OS + || dst_node->type == MALI_PAGE_NODE_SWAP); + + if (dst_node->type == MALI_PAGE_NODE_OS) { + dst_page = dst_node->page; + } else { + dst_page = dst_node->swap_it->page; + } + + dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(dst_node), + _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + /* map it , and copy the content*/ + dst = kmap_atomic(dst_page); + + if (src_node->type == MALI_PAGE_NODE_OS || + src_node->type == MALI_PAGE_NODE_SWAP) { + struct page *src_page; + + if (src_node->type == MALI_PAGE_NODE_OS) { + src_page = src_node->page; + } else { + src_page = src_node->swap_it->page; + } + + /* Clear and invaliate cache */ + /* In ARM architecture, speculative read may pull stale data into L1 cache + * for kernel linear mapping page table. DMA_BIDIRECTIONAL could + * invalidate the L1 cache so that following read get the latest data + */ + dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(src_node), + _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + src = kmap_atomic(src_page); + memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE); + kunmap_atomic(src); + dma_addr = dma_map_page(&mali_platform_device->dev, src_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + if (src_node->type == MALI_PAGE_NODE_SWAP) { + src_node->swap_it->dma_addr = dma_addr; + } + } else if (src_node->type == MALI_PAGE_NODE_BLOCK) { + /* + * use ioremap to map src for BLOCK memory + */ + src = ioremap(_mali_page_node_get_dma_addr(src_node), _MALI_OSK_MALI_PAGE_SIZE); + memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE); + iounmap(src); + } + kunmap_atomic(dst); + dma_addr = dma_map_page(&mali_platform_device->dev, dst_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + if (dst_node->type == MALI_PAGE_NODE_SWAP) { + dst_node->swap_it->dma_addr = dma_addr; + } +} + + +/* +* allocate page on demand when CPU access it, +* THis used in page fault handler +*/ +_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page) +{ + struct page *new_page = NULL; + struct mali_page_node *new_node = NULL; + int i = 0; + struct mali_page_node *m_page, *found_node = NULL; + struct mali_session_data *session = NULL; + mali_mem_cow *cow = &mem_bkend->cow_mem; + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT(offset_page < mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE); + MALI_DEBUG_PRINT(4, ("mali_mem_cow_allocate_on_demand !, offset_page =0x%x\n", offset_page)); + + /* allocate new page here */ + new_page = mali_mem_cow_alloc_page(); + if (!new_page) + return _MALI_OSK_ERR_NOMEM; + + new_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS); + if (!new_node) { + __free_page(new_page); + return _MALI_OSK_ERR_NOMEM; + } + + /* find the page in backend*/ + list_for_each_entry(m_page, &cow->pages, list) { + if (i == offset_page) { + found_node = m_page; + break; + } + i++; + } + MALI_DEBUG_ASSERT(found_node); + if (NULL == found_node) { + __free_page(new_page); + kfree(new_node); + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + _mali_page_node_add_page(new_node, new_page); + + /* Copy the src page's content to new page */ + _mali_mem_cow_copy_page(found_node, new_node); + + MALI_DEBUG_ASSERT_POINTER(mem_bkend->mali_allocation); + session = mem_bkend->mali_allocation->session; + MALI_DEBUG_ASSERT_POINTER(session); + if (1 != _mali_page_node_get_ref_count(found_node)) { + atomic_add(1, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + mem_bkend->cow_mem.change_pages_nr++; + } + + _mali_osk_mutex_wait(session->cow_lock); + if (_mali_mem_put_page_node(found_node)) { + __free_page(new_page); + kfree(new_node); + _mali_osk_mutex_signal(session->cow_lock); + return _MALI_OSK_ERR_NOMEM; + } + _mali_osk_mutex_signal(session->cow_lock); + + list_replace(&found_node->list, &new_node->list); + + kfree(found_node); + + /* map to GPU side*/ + _mali_osk_mutex_wait(session->memory_lock); + mali_mem_cow_mali_map(mem_bkend, offset_page * _MALI_OSK_MALI_PAGE_SIZE, _MALI_OSK_MALI_PAGE_SIZE); + _mali_osk_mutex_signal(session->memory_lock); + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h new file mode 100755 index 000000000000..5f83a37fc8f8 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_COW_H__ +#define __MALI_MEMORY_COW_H__ + +#include "mali_osk.h" +#include "mali_session.h" +#include "mali_memory_types.h" + +int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); +_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend, + struct vm_area_struct *vma, + unsigned long vaddr, + int num); + +_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size); + +_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend, + u32 range_start, + u32 range_size); + +_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk, + u32 target_offset, + u32 target_size, + mali_mem_backend *backend, + u32 range_start, + u32 range_size); + +void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node); + +int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size); +u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped); +_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page); +#endif + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c new file mode 100755 index 000000000000..a9db577cb851 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c @@ -0,0 +1,262 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_ARM +#include +#endif +#include + +#include "mali_memory.h" +#include "mali_kernel_common.h" +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_kernel_linux.h" +#include "mali_memory_defer_bind.h" +#include "mali_executor.h" +#include "mali_osk.h" +#include "mali_scheduler.h" +#include "mali_gp_job.h" + +mali_defer_bind_manager *mali_dmem_man = NULL; + +static u32 mali_dmem_get_gp_varying_size(struct mali_gp_job *gp_job) +{ + return gp_job->required_varying_memsize / _MALI_OSK_MALI_PAGE_SIZE; +} + +_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void) +{ + mali_dmem_man = _mali_osk_calloc(1, sizeof(struct mali_defer_bind_manager)); + if (!mali_dmem_man) + return _MALI_OSK_ERR_NOMEM; + + atomic_set(&mali_dmem_man->num_used_pages, 0); + atomic_set(&mali_dmem_man->num_dmem, 0); + + return _MALI_OSK_ERR_OK; +} + + +void mali_mem_defer_bind_manager_destory(void) +{ + if (mali_dmem_man) { + MALI_DEBUG_ASSERT(0 == atomic_read(&mali_dmem_man->num_dmem)); + kfree(mali_dmem_man); + } + mali_dmem_man = NULL; +} + + +/*allocate pages from OS memory*/ +_mali_osk_errcode_t mali_mem_defer_alloc_mem(u32 require, struct mali_session_data *session, mali_defer_mem_block *dblock) +{ + int retval = 0; + u32 num_pages = require; + mali_mem_os_mem os_mem; + + retval = mali_mem_os_alloc_pages(&os_mem, num_pages * _MALI_OSK_MALI_PAGE_SIZE); + + /* add to free pages list */ + if (0 == retval) { + MALI_DEBUG_PRINT(4, ("mali_mem_defer_alloc_mem ,,*** pages allocate = 0x%x \n", num_pages)); + list_splice(&os_mem.pages, &dblock->free_pages); + atomic_add(os_mem.count, &dblock->num_free_pages); + atomic_add(os_mem.count, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + return _MALI_OSK_ERR_OK; + } else + return _MALI_OSK_ERR_FAULT; +} + +_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock) +{ + u32 require_page; + + if (!next_gp_job) + return _MALI_OSK_ERR_FAULT; + + require_page = mali_dmem_get_gp_varying_size(next_gp_job); + + MALI_DEBUG_PRINT(4, ("mali_mem_defer_prepare_mem_work, require alloc page 0x%x\n", + require_page)); + /* allocate more pages from OS */ + if (_MALI_OSK_ERR_OK != mali_mem_defer_alloc_mem(require_page, next_gp_job->session, dblock)) { + MALI_DEBUG_PRINT(1, ("ERROR##mali_mem_defer_prepare_mem_work, allocate page failed!!")); + return _MALI_OSK_ERR_NOMEM; + } + + next_gp_job->bind_flag = MALI_DEFER_BIND_MEMORY_PREPARED; + + return _MALI_OSK_ERR_OK; +} + + +/* do preparetion for allocation before defer bind */ +_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list, u32 *required_varying_memsize) +{ + mali_mem_backend *mem_bkend = NULL; + struct mali_backend_bind_list *bk_list = _mali_osk_calloc(1, sizeof(struct mali_backend_bind_list)); + if (NULL == bk_list) + return _MALI_OSK_ERR_FAULT; + + INIT_LIST_HEAD(&bk_list->node); + /* Get backend memory */ + mutex_lock(&mali_idr_mutex); + if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) { + MALI_DEBUG_PRINT(1, ("Can't find memory backend in defer bind!\n")); + mutex_unlock(&mali_idr_mutex); + _mali_osk_free(bk_list); + return _MALI_OSK_ERR_FAULT; + } + mutex_unlock(&mali_idr_mutex); + + /* If the mem backend has already been bound, no need to bind again.*/ + if (mem_bkend->os_mem.count > 0) { + _mali_osk_free(bk_list); + return _MALI_OSK_ERR_OK; + } + + MALI_DEBUG_PRINT(4, ("bind_allocation_prepare:: allocation =%x vaddr=0x%x!\n", alloc, alloc->mali_vma_node.vm_node.start)); + + INIT_LIST_HEAD(&mem_bkend->os_mem.pages); + + bk_list->bkend = mem_bkend; + bk_list->vaddr = alloc->mali_vma_node.vm_node.start; + bk_list->session = alloc->session; + bk_list->page_num = mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE; + *required_varying_memsize += mem_bkend->size; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); + + /* add to job to do list */ + list_add(&bk_list->node, list); + + return _MALI_OSK_ERR_OK; +} + + + +/* bind phyiscal memory to allocation +This function will be called in IRQ handler*/ +static _mali_osk_errcode_t mali_mem_defer_bind_allocation(struct mali_backend_bind_list *bk_node, + struct list_head *pages) +{ + struct mali_session_data *session = bk_node->session; + mali_mem_backend *mem_bkend = bk_node->bkend; + MALI_DEBUG_PRINT(4, ("mali_mem_defer_bind_allocation, bind bkend = %x page num=0x%x vaddr=%x session=%x\n", mem_bkend, bk_node->page_num, bk_node->vaddr, session)); + + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); + list_splice(pages, &mem_bkend->os_mem.pages); + mem_bkend->os_mem.count = bk_node->page_num; + + if (mem_bkend->type == MALI_MEM_OS) { + mali_mem_os_mali_map(&mem_bkend->os_mem, session, bk_node->vaddr, 0, + mem_bkend->os_mem.count, MALI_MMU_FLAGS_DEFAULT); + } + smp_wmb(); + bk_node->flag = MALI_DEFER_BIND_MEMORY_BINDED; + mem_bkend->flags &= ~MALI_MEM_BACKEND_FLAG_NOT_BINDED; + mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_BINDED; + return _MALI_OSK_ERR_OK; +} + + +static struct list_head *mali_mem_defer_get_free_page_list(u32 count, struct list_head *pages, mali_defer_mem_block *dblock) +{ + int i = 0; + struct mali_page_node *m_page, *m_tmp; + + if (atomic_read(&dblock->num_free_pages) < count) { + return NULL; + } else { + list_for_each_entry_safe(m_page, m_tmp, &dblock->free_pages, list) { + if (i < count) { + list_move_tail(&m_page->list, pages); + } else { + break; + } + i++; + } + MALI_DEBUG_ASSERT(i == count); + atomic_sub(count, &dblock->num_free_pages); + return pages; + } +} + + +/* called in job start IOCTL to bind physical memory for each allocations +@ bk_list backend list to do defer bind +@ pages page list to do this bind +@ count number of pages +*/ +_mali_osk_errcode_t mali_mem_defer_bind(struct mali_gp_job *gp, + struct mali_defer_mem_block *dmem_block) +{ + struct mali_defer_mem *dmem = NULL; + struct mali_backend_bind_list *bkn, *bkn_tmp; + LIST_HEAD(pages); + + if (gp->required_varying_memsize != (atomic_read(&dmem_block->num_free_pages) * _MALI_OSK_MALI_PAGE_SIZE)) { + MALI_DEBUG_PRINT_ERROR(("#BIND: The memsize of varying buffer not match to the pagesize of the dmem_block!!## \n")); + return _MALI_OSK_ERR_FAULT; + } + + MALI_DEBUG_PRINT(4, ("#BIND: GP job=%x## \n", gp)); + dmem = (mali_defer_mem *)_mali_osk_calloc(1, sizeof(struct mali_defer_mem)); + if (dmem) { + INIT_LIST_HEAD(&dmem->node); + gp->dmem = dmem; + } else { + return _MALI_OSK_ERR_NOMEM; + } + + atomic_add(1, &mali_dmem_man->num_dmem); + /* for each bk_list backend, do bind */ + list_for_each_entry_safe(bkn, bkn_tmp , &gp->vary_todo, node) { + INIT_LIST_HEAD(&pages); + if (likely(mali_mem_defer_get_free_page_list(bkn->page_num, &pages, dmem_block))) { + list_del(&bkn->node); + mali_mem_defer_bind_allocation(bkn, &pages); + _mali_osk_free(bkn); + } else { + /* not enough memory will not happen */ + MALI_DEBUG_PRINT_ERROR(("#BIND: NOT enough memory when binded !!## \n")); + _mali_osk_free(gp->dmem); + return _MALI_OSK_ERR_NOMEM; + } + } + + if (!list_empty(&gp->vary_todo)) { + MALI_DEBUG_PRINT_ERROR(("#BIND: The deferbind backend list isn't empty !!## \n")); + _mali_osk_free(gp->dmem); + return _MALI_OSK_ERR_FAULT; + } + + dmem->flag = MALI_DEFER_BIND_MEMORY_BINDED; + + return _MALI_OSK_ERR_OK; +} + +void mali_mem_defer_dmem_free(struct mali_gp_job *gp) +{ + if (gp->dmem) { + atomic_dec(&mali_dmem_man->num_dmem); + _mali_osk_free(gp->dmem); + } +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h new file mode 100755 index 000000000000..defa08d52a46 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#ifndef __MALI_MEMORY_DEFER_BIND_H_ +#define __MALI_MEMORY_DEFER_BIND_H_ + + +#include "mali_osk.h" +#include "mali_session.h" + +#include +#include +#include +#include +#include + + +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" +#include "mali_uk_types.h" + +struct mali_gp_job; + +typedef struct mali_defer_mem { + struct list_head node; /*dlist node in bind manager */ + u32 flag; +} mali_defer_mem; + + +typedef struct mali_defer_mem_block { + struct list_head free_pages; /* page pool */ + atomic_t num_free_pages; +} mali_defer_mem_block; + +/* varying memory list need to bind */ +typedef struct mali_backend_bind_list { + struct list_head node; + struct mali_mem_backend *bkend; + u32 vaddr; + u32 page_num; + struct mali_session_data *session; + u32 flag; +} mali_backend_bind_lists; + + +typedef struct mali_defer_bind_manager { + atomic_t num_used_pages; + atomic_t num_dmem; +} mali_defer_bind_manager; + +_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void); +void mali_mem_defer_bind_manager_destory(void); +_mali_osk_errcode_t mali_mem_defer_bind(struct mali_gp_job *gp, struct mali_defer_mem_block *dmem_block); +_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list, u32 *required_varying_memsize); +_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock); +void mali_mem_defer_dmem_free(struct mali_gp_job *gp); + +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c new file mode 100755 index 000000000000..1f4565127a6b --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c @@ -0,0 +1,369 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include /* file system operations */ +#include /* user space access */ +#include +#include +#include +#include +#include +#include +#include + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_kernel_linux.h" + +#include "mali_memory.h" +#include "mali_memory_dma_buf.h" +#include "mali_memory_virtual.h" +#include "mali_pp_job.h" + +/* + * Map DMA buf attachment \a mem into \a session at virtual address \a virt. + */ +static int mali_dma_buf_map(mali_mem_backend *mem_backend) +{ + mali_mem_allocation *alloc; + struct mali_dma_buf_attachment *mem; + struct mali_session_data *session; + struct mali_page_directory *pagedir; + _mali_osk_errcode_t err; + struct scatterlist *sg; + u32 virt, flags; + int i; + + MALI_DEBUG_ASSERT_POINTER(mem_backend); + + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + mem = mem_backend->dma_buf.attachment; + MALI_DEBUG_ASSERT_POINTER(mem); + + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT(mem->session == session); + + virt = alloc->mali_vma_node.vm_node.start; + flags = alloc->flags; + + mali_session_memory_lock(session); + mem->map_ref++; + + MALI_DEBUG_PRINT(5, ("Mali DMA-buf: map attachment %p, new map_ref = %d\n", mem, mem->map_ref)); + + if (1 == mem->map_ref) { + + /* First reference taken, so we need to map the dma buf */ + MALI_DEBUG_ASSERT(!mem->is_mapped); + + mem->sgt = dma_buf_map_attachment(mem->attachment, DMA_BIDIRECTIONAL); + if (IS_ERR_OR_NULL(mem->sgt)) { + MALI_DEBUG_PRINT_ERROR(("Failed to map dma-buf attachment\n")); + mem->map_ref--; + mali_session_memory_unlock(session); + return -EFAULT; + } + + err = mali_mem_mali_map_prepare(alloc); + if (_MALI_OSK_ERR_OK != err) { + MALI_DEBUG_PRINT(1, ("Mapping of DMA memory failed\n")); + mem->map_ref--; + mali_session_memory_unlock(session); + return -ENOMEM; + } + + pagedir = mali_session_get_page_directory(session); + MALI_DEBUG_ASSERT_POINTER(pagedir); + + for_each_sg(mem->sgt->sgl, sg, mem->sgt->nents, i) { + u32 size = sg_dma_len(sg); + dma_addr_t phys = sg_dma_address(sg); + + /* sg must be page aligned. */ + MALI_DEBUG_ASSERT(0 == size % MALI_MMU_PAGE_SIZE); + MALI_DEBUG_ASSERT(0 == (phys & ~(uintptr_t)0xFFFFFFFF)); + + mali_mmu_pagedir_update(pagedir, virt, phys, size, MALI_MMU_FLAGS_DEFAULT); + + virt += size; + } + + if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + u32 guard_phys; + MALI_DEBUG_PRINT(7, ("Mapping in extra guard page\n")); + + guard_phys = sg_dma_address(mem->sgt->sgl); + mali_mmu_pagedir_update(pagedir, virt, guard_phys, MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + } + + mem->is_mapped = MALI_TRUE; + mali_session_memory_unlock(session); + /* Wake up any thread waiting for buffer to become mapped */ + wake_up_all(&mem->wait_queue); + } else { + MALI_DEBUG_ASSERT(mem->is_mapped); + mali_session_memory_unlock(session); + } + + return 0; +} + +static void mali_dma_buf_unmap(mali_mem_allocation *alloc, struct mali_dma_buf_attachment *mem) +{ + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT_POINTER(mem); + MALI_DEBUG_ASSERT_POINTER(mem->attachment); + MALI_DEBUG_ASSERT_POINTER(mem->buf); + MALI_DEBUG_ASSERT_POINTER(alloc->session); + + mali_session_memory_lock(alloc->session); + mem->map_ref--; + + MALI_DEBUG_PRINT(5, ("Mali DMA-buf: unmap attachment %p, new map_ref = %d\n", mem, mem->map_ref)); + + if (0 == mem->map_ref) { + dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL); + if (MALI_TRUE == mem->is_mapped) { + mali_mem_mali_map_free(alloc->session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + } + mem->is_mapped = MALI_FALSE; + } + mali_session_memory_unlock(alloc->session); + /* Wake up any thread waiting for buffer to become unmapped */ + wake_up_all(&mem->wait_queue); +} + +#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +int mali_dma_buf_map_job(struct mali_pp_job *job) +{ + struct mali_dma_buf_attachment *mem; + _mali_osk_errcode_t err; + int i; + int ret = 0; + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + + session = mali_pp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(session); + + for (i = 0; i < num_memory_cookies; i++) { + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + if (MALI_MEM_DMA_BUF != mali_alloc->type) { + continue; + } + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + mem = mem_bkend->dma_buf.attachment; + + MALI_DEBUG_ASSERT_POINTER(mem); + MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job)); + + err = mali_dma_buf_map(mem_bkend); + if (0 != err) { + MALI_DEBUG_PRINT_ERROR(("Mali DMA-buf: Failed to map dma-buf for mali address %x\n", mali_addr)); + ret = -EFAULT; + continue; + } + } + return ret; +} + +void mali_dma_buf_unmap_job(struct mali_pp_job *job) +{ + struct mali_dma_buf_attachment *mem; + int i; + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + + session = mali_pp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(session); + + for (i = 0; i < num_memory_cookies; i++) { + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + if (MALI_MEM_DMA_BUF != mali_alloc->type) { + continue; + } + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + mem = mem_bkend->dma_buf.attachment; + + MALI_DEBUG_ASSERT_POINTER(mem); + MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job)); + mali_dma_buf_unmap(mem_bkend->mali_allocation, mem); + } +} +#endif /* !CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH */ + +int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *user_arg) +{ + _mali_uk_dma_buf_get_size_s args; + int fd; + struct dma_buf *buf; + + /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ + if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_dma_buf_get_size_s))) { + return -EFAULT; + } + + /* Do DMA-BUF stuff */ + fd = args.mem_fd; + + buf = dma_buf_get(fd); + if (IS_ERR_OR_NULL(buf)) { + MALI_DEBUG_PRINT_ERROR(("Failed to get dma-buf from fd: %d\n", fd)); + return PTR_ERR_OR_ZERO(buf); + } + + if (0 != put_user(buf->size, &user_arg->size)) { + dma_buf_put(buf); + return -EFAULT; + } + + dma_buf_put(buf); + + return 0; +} + +_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + int fd, u32 flags) +{ + struct dma_buf *buf; + struct mali_dma_buf_attachment *dma_mem; + struct mali_session_data *session = alloc->session; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT_POINTER(alloc); + + /* get dma buffer */ + buf = dma_buf_get(fd); + if (IS_ERR_OR_NULL(buf)) { + return _MALI_OSK_ERR_FAULT; + } + + /* Currently, mapping of the full buffer are supported. */ + if (alloc->psize != buf->size) { + goto failed_alloc_mem; + } + + dma_mem = _mali_osk_calloc(1, sizeof(struct mali_dma_buf_attachment)); + if (NULL == dma_mem) { + goto failed_alloc_mem; + } + + dma_mem->buf = buf; + dma_mem->session = session; + dma_mem->map_ref = 0; + init_waitqueue_head(&dma_mem->wait_queue); + + dma_mem->attachment = dma_buf_attach(dma_mem->buf, &mali_platform_device->dev); + if (NULL == dma_mem->attachment) { + goto failed_dma_attach; + } + + mem_backend->dma_buf.attachment = dma_mem; + + alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; + if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; + } + + +#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) + /* Map memory into session's Mali virtual address space. */ + if (0 != mali_dma_buf_map(mem_backend)) { + goto Failed_dma_map; + } +#endif + + return _MALI_OSK_ERR_OK; + +#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +Failed_dma_map: + mali_dma_buf_unmap(alloc, dma_mem); +#endif + /* Wait for buffer to become unmapped */ + wait_event(dma_mem->wait_queue, !dma_mem->is_mapped); + MALI_DEBUG_ASSERT(!dma_mem->is_mapped); + dma_buf_detach(dma_mem->buf, dma_mem->attachment); +failed_dma_attach: + _mali_osk_free(dma_mem); +failed_alloc_mem: + dma_buf_put(buf); + return _MALI_OSK_ERR_FAULT; +} + +void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend) +{ + struct mali_dma_buf_attachment *mem; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_DMA_BUF == mem_backend->type); + + mem = mem_backend->dma_buf.attachment; + MALI_DEBUG_ASSERT_POINTER(mem); + MALI_DEBUG_ASSERT_POINTER(mem->attachment); + MALI_DEBUG_ASSERT_POINTER(mem->buf); + MALI_DEBUG_PRINT(3, ("Mali DMA-buf: release attachment %p\n", mem)); + +#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) + MALI_DEBUG_ASSERT_POINTER(mem_backend->mali_allocation); + /* We mapped implicitly on attach, so we need to unmap on release */ + mali_dma_buf_unmap(mem_backend->mali_allocation, mem); +#endif + /* Wait for buffer to become unmapped */ + wait_event(mem->wait_queue, !mem->is_mapped); + MALI_DEBUG_ASSERT(!mem->is_mapped); + + dma_buf_detach(mem->buf, mem->attachment); + dma_buf_put(mem->buf); + + _mali_osk_free(mem); +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h new file mode 100755 index 000000000000..a9b2870389ff --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_DMA_BUF_H__ +#define __MALI_MEMORY_DMA_BUF_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_memory.h" + +struct mali_pp_job; + +struct mali_dma_buf_attachment; +struct mali_dma_buf_attachment { + struct dma_buf *buf; + struct dma_buf_attachment *attachment; + struct sg_table *sgt; + struct mali_session_data *session; + int map_ref; + struct mutex map_lock; + mali_bool is_mapped; + wait_queue_head_t wait_queue; +}; + +int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *arg); + +void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend); + +_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + int fd, u32 flags); + +#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) +int mali_dma_buf_map_job(struct mali_pp_job *job); +void mali_dma_buf_unmap_job(struct mali_pp_job *job); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_MEMORY_DMA_BUF_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c new file mode 100755 index 000000000000..76018b7ab90b --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" +#include "mali_memory.h" +#include "mali_mem_validation.h" +#include "mali_uk_types.h" + +void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend) +{ + mali_mem_allocation *alloc; + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT(MALI_MEM_EXTERNAL == mem_backend->type); + + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + +_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + u32 phys_addr, + u32 flag) +{ + struct mali_session_data *session; + _mali_osk_errcode_t err; + u32 virt, phys, size; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT_POINTER(alloc); + size = alloc->psize; + session = (struct mali_session_data *)(uintptr_t)alloc->session; + MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); + + /* check arguments */ + /* NULL might be a valid Mali address */ + if (!size) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + + /* size must be a multiple of the system page size */ + if (size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); + + /* Validate the mali physical range */ + if (_MALI_OSK_ERR_OK != mali_mem_validation_check(phys_addr, size)) { + return _MALI_OSK_ERR_FAULT; + } + + if (flag & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; + } + + mali_session_memory_lock(session); + + virt = alloc->mali_vma_node.vm_node.start; + phys = phys_addr; + + err = mali_mem_mali_map_prepare(alloc); + if (_MALI_OSK_ERR_OK != err) { + mali_session_memory_unlock(session); + return _MALI_OSK_ERR_NOMEM; + } + + mali_mmu_pagedir_update(session->page_directory, virt, phys, size, MALI_MMU_FLAGS_DEFAULT); + + if (alloc->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { + mali_mmu_pagedir_update(session->page_directory, virt + size, phys, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + } + MALI_DEBUG_PRINT(3, + ("Requested to map physical memory 0x%x-0x%x into virtual memory 0x%x\n", + phys_addr, (phys_addr + size - 1), + virt)); + mali_session_memory_unlock(session); + + MALI_SUCCESS; +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h new file mode 100755 index 000000000000..2db178d96233 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h @@ -0,0 +1,29 @@ + +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_EXTERNAL_H__ +#define __MALI_MEMORY_EXTERNAL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc, + mali_mem_backend *mem_backend, + u32 phys_addr, + u32 flag); +void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c new file mode 100755 index 000000000000..27dee0f19c81 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c @@ -0,0 +1,993 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include +#endif +#include + +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_linux.h" +#include "mali_scheduler.h" +#include "mali_memory.h" +#include "mali_memory_os_alloc.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include "mali_memory_dma_buf.h" +#include "mali_memory_secure.h" +#endif +#if defined(CONFIG_MALI400_UMP) +#include "mali_memory_ump.h" +#endif +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" +#include "mali_memory_util.h" +#include "mali_memory_external.h" +#include "mali_memory_cow.h" +#include "mali_memory_block_alloc.h" +#include "mali_ukk.h" +#include "mali_memory_swap_alloc.h" + +/* +* New memory system interface +*/ + +/*inti idr for backend memory */ +struct idr mali_backend_idr; +struct mutex mali_idr_mutex; + +/* init allocation manager */ +int mali_memory_manager_init(struct mali_allocation_manager *mgr) +{ + /* init Locks */ + rwlock_init(&mgr->vm_lock); + mutex_init(&mgr->list_mutex); + + /* init link */ + INIT_LIST_HEAD(&mgr->head); + + /* init RB tree */ + mgr->allocation_mgr_rb = RB_ROOT; + mgr->mali_allocation_num = 0; + return 0; +} + +/* Deinit allocation manager +* Do some check for debug +*/ +void mali_memory_manager_uninit(struct mali_allocation_manager *mgr) +{ + /* check RB tree is empty */ + MALI_DEBUG_ASSERT(((void *)(mgr->allocation_mgr_rb.rb_node) == (void *)rb_last(&mgr->allocation_mgr_rb))); + /* check allocation List */ + MALI_DEBUG_ASSERT(list_empty(&mgr->head)); +} + +/* Prepare memory descriptor */ +static mali_mem_allocation *mali_mem_allocation_struct_create(struct mali_session_data *session) +{ + mali_mem_allocation *mali_allocation; + + /* Allocate memory */ + mali_allocation = (mali_mem_allocation *)kzalloc(sizeof(mali_mem_allocation), GFP_KERNEL); + if (NULL == mali_allocation) { + MALI_DEBUG_PRINT(1, ("mali_mem_allocation_struct_create: descriptor was NULL\n")); + return NULL; + } + + MALI_DEBUG_CODE(mali_allocation->magic = MALI_MEM_ALLOCATION_VALID_MAGIC); + + /* do init */ + mali_allocation->flags = 0; + mali_allocation->session = session; + + INIT_LIST_HEAD(&mali_allocation->list); + _mali_osk_atomic_init(&mali_allocation->mem_alloc_refcount, 1); + + /** + *add to session list + */ + mutex_lock(&session->allocation_mgr.list_mutex); + list_add_tail(&mali_allocation->list, &session->allocation_mgr.head); + session->allocation_mgr.mali_allocation_num++; + mutex_unlock(&session->allocation_mgr.list_mutex); + + return mali_allocation; +} + +void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc) +{ + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT_POINTER(alloc->session); + mutex_lock(&alloc->session->allocation_mgr.list_mutex); + list_del(&alloc->list); + alloc->session->allocation_mgr.mali_allocation_num--; + mutex_unlock(&alloc->session->allocation_mgr.list_mutex); + + kfree(alloc); +} + +int mali_mem_backend_struct_create(mali_mem_backend **backend, u32 psize) +{ + mali_mem_backend *mem_backend = NULL; + s32 ret = -ENOSPC; + s32 index = -1; + *backend = (mali_mem_backend *)kzalloc(sizeof(mali_mem_backend), GFP_KERNEL); + if (NULL == *backend) { + MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: backend descriptor was NULL\n")); + return -1; + } + mem_backend = *backend; + mem_backend->size = psize; + mutex_init(&mem_backend->mutex); + INIT_LIST_HEAD(&mem_backend->list); + mem_backend->using_count = 0; + + + /* link backend with id */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0) +again: + if (!idr_pre_get(&mali_backend_idr, GFP_KERNEL)) { + kfree(mem_backend); + return -ENOMEM; + } + mutex_lock(&mali_idr_mutex); + ret = idr_get_new_above(&mali_backend_idr, mem_backend, 1, &index); + mutex_unlock(&mali_idr_mutex); + + if (-ENOSPC == ret) { + kfree(mem_backend); + return -ENOSPC; + } + if (-EAGAIN == ret) + goto again; +#else + mutex_lock(&mali_idr_mutex); + ret = idr_alloc(&mali_backend_idr, mem_backend, 1, MALI_S32_MAX, GFP_KERNEL); + mutex_unlock(&mali_idr_mutex); + index = ret; + if (ret < 0) { + MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: Can't allocate idr for backend! \n")); + kfree(mem_backend); + return -ENOSPC; + } +#endif + return index; +} + + +static void mali_mem_backend_struct_destory(mali_mem_backend **backend, s32 backend_handle) +{ + mali_mem_backend *mem_backend = *backend; + + mutex_lock(&mali_idr_mutex); + idr_remove(&mali_backend_idr, backend_handle); + mutex_unlock(&mali_idr_mutex); + kfree(mem_backend); + *backend = NULL; +} + +mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address) +{ + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_backend *mem_bkend = NULL; + mali_mem_allocation *mali_alloc = NULL; + MALI_DEBUG_ASSERT_POINTER(session); + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_address, 0); + if (NULL == mali_vma_node) { + MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_search:vma node was NULL\n")); + return NULL; + } + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + return mem_bkend; +} + +static _mali_osk_errcode_t mali_mem_resize(struct mali_session_data *session, mali_mem_backend *mem_backend, u32 physical_size) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + int retval = 0; + mali_mem_allocation *mali_allocation = NULL; + mali_mem_os_mem tmp_os_mem; + s32 change_page_count; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n")); + MALI_DEBUG_ASSERT(0 == physical_size % MALI_MMU_PAGE_SIZE); + + mali_allocation = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(mali_allocation); + + MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE & mali_allocation->flags); + MALI_DEBUG_ASSERT(MALI_MEM_OS == mali_allocation->type); + + mutex_lock(&mem_backend->mutex); + + /* Do resize*/ + if (physical_size > mem_backend->size) { + u32 add_size = physical_size - mem_backend->size; + + MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE); + + /* Allocate new pages from os mem */ + retval = mali_mem_os_alloc_pages(&tmp_os_mem, add_size); + + if (retval) { + if (-ENOMEM == retval) { + ret = _MALI_OSK_ERR_NOMEM; + } else { + ret = _MALI_OSK_ERR_FAULT; + } + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory allocation failed !\n")); + goto failed_alloc_memory; + } + + MALI_DEBUG_ASSERT(tmp_os_mem.count == add_size / MALI_MMU_PAGE_SIZE); + + /* Resize the memory of the backend */ + ret = mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count); + + if (ret) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory resizing failed !\n")); + goto failed_resize_pages; + } + + /*Resize cpu mapping */ + if (NULL != mali_allocation->cpu_mapping.vma) { + ret = mali_mem_os_resize_cpu_map_locked(mem_backend, mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + mem_backend->size, add_size); + if (unlikely(ret != _MALI_OSK_ERR_OK)) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: cpu mapping failed !\n")); + goto failed_cpu_map; + } + } + + /* Resize mali mapping */ + _mali_osk_mutex_wait(session->memory_lock); + ret = mali_mem_mali_map_resize(mali_allocation, physical_size); + + if (ret) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_resize: mali map resize fail !\n")); + goto failed_gpu_map; + } + + ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, mali_allocation->mali_vma_node.vm_node.start, + mali_allocation->psize / MALI_MMU_PAGE_SIZE, add_size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties); + if (ret) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: mali mapping failed !\n")); + goto failed_gpu_map; + } + + _mali_osk_mutex_signal(session->memory_lock); + } else { + u32 dec_size, page_count; + u32 vaddr = 0; + INIT_LIST_HEAD(&tmp_os_mem.pages); + tmp_os_mem.count = 0; + + dec_size = mem_backend->size - physical_size; + MALI_DEBUG_ASSERT(0 == dec_size % MALI_MMU_PAGE_SIZE); + + page_count = dec_size / MALI_MMU_PAGE_SIZE; + vaddr = mali_allocation->mali_vma_node.vm_node.start + physical_size; + + /* Resize the memory of the backend */ + ret = mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, physical_size / MALI_MMU_PAGE_SIZE, page_count); + + if (ret) { + MALI_DEBUG_PRINT(4, ("_mali_ukk_mem_resize: mali map resize failed!\n")); + goto failed_resize_pages; + } + + /* Resize mali map */ + _mali_osk_mutex_wait(session->memory_lock); + mali_mem_mali_map_free(session, dec_size, vaddr, mali_allocation->flags); + _mali_osk_mutex_signal(session->memory_lock); + + /* Zap cpu mapping */ + if (0 != mali_allocation->cpu_mapping.addr) { + MALI_DEBUG_ASSERT(NULL != mali_allocation->cpu_mapping.vma); + zap_vma_ptes(mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + physical_size, dec_size); + } + + /* Free those extra pages */ + mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE); + } + + /* Resize memory allocation and memory backend */ + change_page_count = (s32)(physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE; + mali_allocation->psize = physical_size; + mem_backend->size = physical_size; + mutex_unlock(&mem_backend->mutex); + + if (change_page_count > 0) { + atomic_add(change_page_count, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + + } else { + atomic_sub((s32)(-change_page_count), &session->mali_mem_allocated_pages); + } + + return _MALI_OSK_ERR_OK; + +failed_gpu_map: + _mali_osk_mutex_signal(session->memory_lock); +failed_cpu_map: + if (physical_size > mem_backend->size) { + mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, mem_backend->size / MALI_MMU_PAGE_SIZE, + (physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE); + } else { + mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count); + } +failed_resize_pages: + if (0 != tmp_os_mem.count) + mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE); +failed_alloc_memory: + + mutex_unlock(&mem_backend->mutex); + return ret; +} + + +/* Set GPU MMU properties */ +static void _mali_memory_gpu_map_property_set(u32 *properties, u32 flags) +{ + if (_MALI_MEMORY_GPU_READ_ALLOCATE & flags) { + *properties = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE; + } else { + *properties = MALI_MMU_FLAGS_DEFAULT; + } +} + +_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size) +{ + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_allocation *mali_allocation = NULL; + u32 new_physical_size; + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE); + + /* Get the memory backend that need to be resize. */ + mem_backend = mali_mem_backend_struct_search(session, mali_addr); + + if (NULL == mem_backend) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n")); + return ret; + } + + mali_allocation = mem_backend->mali_allocation; + + MALI_DEBUG_ASSERT_POINTER(mali_allocation); + + new_physical_size = add_size + mem_backend->size; + + if (new_physical_size > (mali_allocation->mali_vma_node.vm_node.size)) + return ret; + + MALI_DEBUG_ASSERT(new_physical_size != mem_backend->size); + + ret = mali_mem_resize(session, mem_backend, new_physical_size); + + return ret; +} + +/** +* function@_mali_ukk_mem_allocate - allocate mali memory +*/ +_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + int retval = 0; + mali_mem_allocation *mali_allocation = NULL; + struct mali_vma_node *mali_vma_node = NULL; + + MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_allocate, vaddr=0x%x, size =0x%x! \n", args->gpu_vaddr, args->psize)); + + /* Check if the address is allocated + */ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->gpu_vaddr, 0); + + if (unlikely(mali_vma_node)) { + MALI_DEBUG_PRINT_ERROR(("The mali virtual address has already been used ! \n")); + return _MALI_OSK_ERR_FAULT; + } + /** + *create mali memory allocation + */ + + mali_allocation = mali_mem_allocation_struct_create(session); + + if (mali_allocation == NULL) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_allocate: Failed to create allocation struct! \n")); + return _MALI_OSK_ERR_NOMEM; + } + mali_allocation->psize = args->psize; + mali_allocation->vsize = args->vsize; + + /* MALI_MEM_OS if need to support mem resize, + * or MALI_MEM_BLOCK if have dedicated memory, + * or MALI_MEM_OS, + * or MALI_MEM_SWAP. + */ + if (args->flags & _MALI_MEMORY_ALLOCATE_SWAPPABLE) { + mali_allocation->type = MALI_MEM_SWAP; + } else if (args->flags & _MALI_MEMORY_ALLOCATE_RESIZEABLE) { + mali_allocation->type = MALI_MEM_OS; + mali_allocation->flags |= MALI_MEM_FLAG_CAN_RESIZE; + } else if (args->flags & _MALI_MEMORY_ALLOCATE_SECURE) { + mali_allocation->type = MALI_MEM_SECURE; + } else if (MALI_TRUE == mali_memory_have_dedicated_memory()) { + mali_allocation->type = MALI_MEM_BLOCK; + } else { + mali_allocation->type = MALI_MEM_OS; + } + + /** + *add allocation node to RB tree for index + */ + mali_allocation->mali_vma_node.vm_node.start = args->gpu_vaddr; + mali_allocation->mali_vma_node.vm_node.size = args->vsize; + + mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); + + mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, args->psize); + if (mali_allocation->backend_handle < 0) { + ret = _MALI_OSK_ERR_NOMEM; + MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n")); + goto failed_alloc_backend; + } + + + mem_backend->mali_allocation = mali_allocation; + mem_backend->type = mali_allocation->type; + + mali_allocation->mali_mapping.addr = args->gpu_vaddr; + + /* set gpu mmu propery */ + _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags); + /* do prepare for MALI mapping */ + if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) { + _mali_osk_mutex_wait(session->memory_lock); + + ret = mali_mem_mali_map_prepare(mali_allocation); + if (0 != ret) { + _mali_osk_mutex_signal(session->memory_lock); + goto failed_prepare_map; + } + _mali_osk_mutex_signal(session->memory_lock); + } + + if (mali_allocation->psize == 0) { + mem_backend->os_mem.count = 0; + INIT_LIST_HEAD(&mem_backend->os_mem.pages); + goto done; + } + + if (args->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) { + mali_allocation->flags |= _MALI_MEMORY_ALLOCATE_DEFER_BIND; + mem_backend->flags |= MALI_MEM_BACKEND_FLAG_NOT_BINDED; + /* init for defer bind backend*/ + mem_backend->os_mem.count = 0; + INIT_LIST_HEAD(&mem_backend->os_mem.pages); + + goto done; + } + + if (likely(mali_allocation->psize > 0)) { + + if (MALI_MEM_SECURE == mem_backend->type) { +#if defined(CONFIG_DMA_SHARED_BUFFER) + ret = mali_mem_secure_attach_dma_buf(&mem_backend->secure_mem, mem_backend->size, args->secure_shared_fd); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Failed to attach dma buf for secure memory! \n")); + goto failed_alloc_pages; + } +#else + ret = _MALI_OSK_ERR_UNSUPPORTED; + MALI_DEBUG_PRINT(1, ("DMA not supported for mali secure memory! \n")); + goto failed_alloc_pages; +#endif + } else { + + /** + *allocate physical memory + */ + if (mem_backend->type == MALI_MEM_OS) { + retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size); + } else if (mem_backend->type == MALI_MEM_BLOCK) { + /* try to allocated from BLOCK memory first, then try OS memory if failed.*/ + if (mali_mem_block_alloc(&mem_backend->block_mem, mem_backend->size)) { + retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size); + mem_backend->type = MALI_MEM_OS; + mali_allocation->type = MALI_MEM_OS; + } + } else if (MALI_MEM_SWAP == mem_backend->type) { + retval = mali_mem_swap_alloc_pages(&mem_backend->swap_mem, mali_allocation->mali_vma_node.vm_node.size, &mem_backend->start_idx); + } else { + /* ONLY support mem_os type */ + MALI_DEBUG_ASSERT(0); + } + + if (retval) { + ret = _MALI_OSK_ERR_NOMEM; + MALI_DEBUG_PRINT(1, (" can't allocate enough pages! \n")); + goto failed_alloc_pages; + } + } + } + + /** + *map to GPU side + */ + if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) { + _mali_osk_mutex_wait(session->memory_lock); + /* Map on Mali */ + + if (mem_backend->type == MALI_MEM_OS) { + ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, args->gpu_vaddr, 0, + mem_backend->size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties); + + } else if (mem_backend->type == MALI_MEM_BLOCK) { + mali_mem_block_mali_map(&mem_backend->block_mem, session, args->gpu_vaddr, + mali_allocation->mali_mapping.properties); + } else if (mem_backend->type == MALI_MEM_SWAP) { + ret = mali_mem_swap_mali_map(&mem_backend->swap_mem, session, args->gpu_vaddr, + mali_allocation->mali_mapping.properties); + } else if (mem_backend->type == MALI_MEM_SECURE) { +#if defined(CONFIG_DMA_SHARED_BUFFER) + ret = mali_mem_secure_mali_map(&mem_backend->secure_mem, session, args->gpu_vaddr, mali_allocation->mali_mapping.properties); +#endif + } else { /* unsupport type */ + MALI_DEBUG_ASSERT(0); + } + + _mali_osk_mutex_signal(session->memory_lock); + } +done: + if (MALI_MEM_OS == mem_backend->type) { + atomic_add(mem_backend->os_mem.count, &session->mali_mem_allocated_pages); + } else if (MALI_MEM_BLOCK == mem_backend->type) { + atomic_add(mem_backend->block_mem.count, &session->mali_mem_allocated_pages); + } else if (MALI_MEM_SECURE == mem_backend->type) { + atomic_add(mem_backend->secure_mem.count, &session->mali_mem_allocated_pages); + } else { + MALI_DEBUG_ASSERT(MALI_MEM_SWAP == mem_backend->type); + atomic_add(mem_backend->swap_mem.count, &session->mali_mem_allocated_pages); + atomic_add(mem_backend->swap_mem.count, &session->mali_mem_array[mem_backend->type]); + } + + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + return _MALI_OSK_ERR_OK; + +failed_alloc_pages: + mali_mem_mali_map_free(session, mali_allocation->psize, mali_allocation->mali_vma_node.vm_node.start, mali_allocation->flags); +failed_prepare_map: + mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); +failed_alloc_backend: + + mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); + mali_mem_allocation_struct_destory(mali_allocation); + + return ret; +} + + +_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + u32 vaddr = args->gpu_vaddr; + mali_mem_allocation *mali_alloc = NULL; + struct mali_vma_node *mali_vma_node = NULL; + + /* find mali allocation structure by vaddress*/ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, vaddr, 0); + if (NULL == mali_vma_node) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_free: invalid addr: 0x%x\n", vaddr)); + return _MALI_OSK_ERR_INVALID_ARGS; + } + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + + if (mali_alloc) + /* check ref_count */ + args->free_pages_nr = mali_allocation_unref(&mali_alloc); + + return _MALI_OSK_ERR_OK; +} + + +/** +* Function _mali_ukk_mem_bind -- bind a external memory to a new GPU address +* It will allocate a new mem allocation and bind external memory to it. +* Supported backend type are: +* _MALI_MEMORY_BIND_BACKEND_UMP +* _MALI_MEMORY_BIND_BACKEND_DMA_BUF +* _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY +* CPU access is not supported yet +*/ +_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_allocation *mali_allocation = NULL; + MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_bind, vaddr=0x%x, size =0x%x! \n", args->vaddr, args->size)); + + /** + * allocate mali allocation. + */ + mali_allocation = mali_mem_allocation_struct_create(session); + + if (mali_allocation == NULL) { + return _MALI_OSK_ERR_NOMEM; + } + mali_allocation->psize = args->size; + mali_allocation->vsize = args->size; + mali_allocation->mali_mapping.addr = args->vaddr; + + /* add allocation node to RB tree for index */ + mali_allocation->mali_vma_node.vm_node.start = args->vaddr; + mali_allocation->mali_vma_node.vm_node.size = args->size; + mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); + + /* allocate backend*/ + if (mali_allocation->psize > 0) { + mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize); + if (mali_allocation->backend_handle < 0) { + goto Failed_alloc_backend; + } + + } else { + goto Failed_alloc_backend; + } + + mem_backend->size = mali_allocation->psize; + mem_backend->mali_allocation = mali_allocation; + + switch (args->flags & _MALI_MEMORY_BIND_BACKEND_MASK) { + case _MALI_MEMORY_BIND_BACKEND_UMP: +#if defined(CONFIG_MALI400_UMP) + mali_allocation->type = MALI_MEM_UMP; + mem_backend->type = MALI_MEM_UMP; + ret = mali_mem_bind_ump_buf(mali_allocation, mem_backend, + args->mem_union.bind_ump.secure_id, args->mem_union.bind_ump.flags); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Bind ump buf failed\n")); + goto Failed_bind_backend; + } +#else + MALI_DEBUG_PRINT(1, ("UMP not supported\n")); + goto Failed_bind_backend; +#endif + break; + case _MALI_MEMORY_BIND_BACKEND_DMA_BUF: +#if defined(CONFIG_DMA_SHARED_BUFFER) + mali_allocation->type = MALI_MEM_DMA_BUF; + mem_backend->type = MALI_MEM_DMA_BUF; + ret = mali_mem_bind_dma_buf(mali_allocation, mem_backend, + args->mem_union.bind_dma_buf.mem_fd, args->mem_union.bind_dma_buf.flags); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Bind dma buf failed\n")); + goto Failed_bind_backend; + } +#else + MALI_DEBUG_PRINT(1, ("DMA not supported\n")); + goto Failed_bind_backend; +#endif + break; + case _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY: + /* not allowed */ + MALI_DEBUG_PRINT_ERROR(("Mali internal memory type not supported !\n")); + goto Failed_bind_backend; + break; + + case _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY: + mali_allocation->type = MALI_MEM_EXTERNAL; + mem_backend->type = MALI_MEM_EXTERNAL; + ret = mali_mem_bind_ext_buf(mali_allocation, mem_backend, args->mem_union.bind_ext_memory.phys_addr, + args->mem_union.bind_ext_memory.flags); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("Bind external buf failed\n")); + goto Failed_bind_backend; + } + break; + + case _MALI_MEMORY_BIND_BACKEND_EXT_COW: + /* not allowed */ + MALI_DEBUG_PRINT_ERROR(("External cow memory type not supported !\n")); + goto Failed_bind_backend; + break; + + default: + MALI_DEBUG_PRINT_ERROR(("Invalid memory type not supported !\n")); + goto Failed_bind_backend; + break; + } + MALI_DEBUG_ASSERT(0 == mem_backend->size % MALI_MMU_PAGE_SIZE); + atomic_add(mem_backend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_backend->type]); + return _MALI_OSK_ERR_OK; + +Failed_bind_backend: + mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); + +Failed_alloc_backend: + mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); + mali_mem_allocation_struct_destory(mali_allocation); + + MALI_DEBUG_PRINT(1, (" _mali_ukk_mem_bind, return ERROR! \n")); + return ret; +} + + +/* +* Function _mali_ukk_mem_unbind -- unbind a external memory to a new GPU address +* This function unbind the backend memory and free the allocation +* no ref_count for this type of memory +*/ +_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args) +{ + /**/ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + mali_mem_allocation *mali_allocation = NULL; + struct mali_vma_node *mali_vma_node = NULL; + u32 mali_addr = args->vaddr; + MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_unbind, vaddr=0x%x! \n", args->vaddr)); + + /* find the allocation by vaddr */ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + if (likely(mali_vma_node)) { + MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start); + mali_allocation = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + } else { + MALI_DEBUG_ASSERT(NULL != mali_vma_node); + return _MALI_OSK_ERR_INVALID_ARGS; + } + + if (NULL != mali_allocation) + /* check ref_count */ + mali_allocation_unref(&mali_allocation); + return _MALI_OSK_ERR_OK; +} + +/* +* Function _mali_ukk_mem_cow -- COW for an allocation +* This function allocate new pages for a range (range, range+size) of allocation +* And Map it(keep use the not in range pages from target allocation ) to an GPU vaddr +*/ +_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_backend *target_backend = NULL; + mali_mem_backend *mem_backend = NULL; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_allocation = NULL; + + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + /* Get the target backend for cow */ + target_backend = mali_mem_backend_struct_search(session, args->target_handle); + + if (NULL == target_backend || 0 == target_backend->size) { + MALI_DEBUG_ASSERT_POINTER(target_backend); + MALI_DEBUG_ASSERT(0 != target_backend->size); + return ret; + } + + /*Cow not support resized mem */ + MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE != (MALI_MEM_FLAG_CAN_RESIZE & target_backend->mali_allocation->flags)); + + /* Check if the new mali address is allocated */ + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->vaddr, 0); + + if (unlikely(mali_vma_node)) { + MALI_DEBUG_PRINT_ERROR(("The mali virtual address has already been used ! \n")); + return ret; + } + + /* create new alloction for COW*/ + mali_allocation = mali_mem_allocation_struct_create(session); + if (mali_allocation == NULL) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to create allocation struct!\n")); + return _MALI_OSK_ERR_NOMEM; + } + mali_allocation->psize = args->target_size; + mali_allocation->vsize = args->target_size; + mali_allocation->type = MALI_MEM_COW; + + /*add allocation node to RB tree for index*/ + mali_allocation->mali_vma_node.vm_node.start = args->vaddr; + mali_allocation->mali_vma_node.vm_node.size = mali_allocation->vsize; + mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); + + /* create new backend for COW memory */ + mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize); + if (mali_allocation->backend_handle < 0) { + ret = _MALI_OSK_ERR_NOMEM; + MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n")); + goto failed_alloc_backend; + } + mem_backend->mali_allocation = mali_allocation; + mem_backend->type = mali_allocation->type; + + if (target_backend->type == MALI_MEM_SWAP || + (MALI_MEM_COW == target_backend->type && (MALI_MEM_BACKEND_FLAG_SWAP_COWED & target_backend->flags))) { + mem_backend->flags |= MALI_MEM_BACKEND_FLAG_SWAP_COWED; + /** + * CoWed swap backends couldn't be mapped as non-linear vma, because if one + * vma is set with flag VM_NONLINEAR, the vma->vm_private_data will be used by kernel, + * while in mali driver, we use this variable to store the pointer of mali_allocation, so there + * is a conflict. + * To resolve this problem, we have to do some fake things, we reserved about 64MB + * space from index 0, there isn't really page's index will be set from 0 to (64MB>>PAGE_SHIFT_NUM), + * and all of CoWed swap memory backends' start_idx will be assigned with 0, and these + * backends will be mapped as linear and will add to priority tree of global swap file, while + * these vmas will never be found by using normal page->index, these pages in those vma + * also couldn't be swapped out. + */ + mem_backend->start_idx = 0; + } + + /* Add the target backend's cow count, also allocate new pages for COW backend from os mem + *for a modified range and keep the page which not in the modified range and Add ref to it + */ + MALI_DEBUG_PRINT(3, ("Cow mapping: target_addr: 0x%x; cow_addr: 0x%x, size: %u\n", target_backend->mali_allocation->mali_vma_node.vm_node.start, + mali_allocation->mali_vma_node.vm_node.start, mali_allocation->mali_vma_node.vm_node.size)); + + ret = mali_memory_do_cow(target_backend, args->target_offset, args->target_size, mem_backend, args->range_start, args->range_size); + if (_MALI_OSK_ERR_OK != ret) { + MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to cow!\n")); + goto failed_do_cow; + } + + /** + *map to GPU side + */ + mali_allocation->mali_mapping.addr = args->vaddr; + /* set gpu mmu propery */ + _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags); + + _mali_osk_mutex_wait(session->memory_lock); + /* Map on Mali */ + ret = mali_mem_mali_map_prepare(mali_allocation); + if (0 != ret) { + MALI_DEBUG_PRINT(1, (" prepare map fail! \n")); + goto failed_gpu_map; + } + + if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { + mali_mem_cow_mali_map(mem_backend, 0, mem_backend->size); + } + + _mali_osk_mutex_signal(session->memory_lock); + + mutex_lock(&target_backend->mutex); + target_backend->flags |= MALI_MEM_BACKEND_FLAG_COWED; + mutex_unlock(&target_backend->mutex); + + atomic_add(args->range_size / MALI_MMU_PAGE_SIZE, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + return _MALI_OSK_ERR_OK; + +failed_gpu_map: + _mali_osk_mutex_signal(session->memory_lock); + mali_mem_cow_release(mem_backend, MALI_FALSE); + mem_backend->cow_mem.count = 0; +failed_do_cow: + mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); +failed_alloc_backend: + mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); + mali_mem_allocation_struct_destory(mali_allocation); + + return ret; +} + +_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + mali_mem_backend *mem_backend = NULL; + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + + MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_cow_modify_range called! \n")); + /* Get the backend that need to be modified. */ + mem_backend = mali_mem_backend_struct_search(session, args->vaddr); + + if (NULL == mem_backend || 0 == mem_backend->size) { + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(0 != mem_backend->size); + return ret; + } + + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_backend->type); + + ret = mali_memory_cow_modify_range(mem_backend, args->range_start, args->size); + args->change_pages_nr = mem_backend->cow_mem.change_pages_nr; + if (_MALI_OSK_ERR_OK != ret) + return ret; + _mali_osk_mutex_wait(session->memory_lock); + if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { + mali_mem_cow_mali_map(mem_backend, args->range_start, args->size); + } + _mali_osk_mutex_signal(session->memory_lock); + + atomic_add(args->change_pages_nr, &session->mali_mem_allocated_pages); + if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { + session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + + return _MALI_OSK_ERR_OK; +} + + +_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args) +{ + mali_mem_backend *mem_backend = NULL; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n")); + MALI_DEBUG_ASSERT(0 == args->psize % MALI_MMU_PAGE_SIZE); + + /* Get the memory backend that need to be resize. */ + mem_backend = mali_mem_backend_struct_search(session, args->vaddr); + + if (NULL == mem_backend) { + MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n")); + return ret; + } + + MALI_DEBUG_ASSERT(args->psize != mem_backend->size); + + ret = mali_mem_resize(session, mem_backend, args->psize); + + return ret; +} + +_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args) +{ + args->memory_usage = _mali_ukk_report_memory_usage(); + if (0 != args->vaddr) { + mali_mem_backend *mem_backend = NULL; + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + /* Get the backend that need to be modified. */ + mem_backend = mali_mem_backend_struct_search(session, args->vaddr); + if (NULL == mem_backend) { + MALI_DEBUG_ASSERT_POINTER(mem_backend); + return _MALI_OSK_ERR_FAULT; + } + + if (MALI_MEM_COW == mem_backend->type) + args->change_pages_nr = mem_backend->cow_mem.change_pages_nr; + } + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h new file mode 100755 index 000000000000..23d8cde753a1 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_MANAGER_H__ +#define __MALI_MEMORY_MANAGER_H__ + +#include "mali_osk.h" +#include +#include +#include +#include +#include +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" +#include "mali_uk_types.h" + +struct mali_allocation_manager { + rwlock_t vm_lock; + struct rb_root allocation_mgr_rb; + struct list_head head; + struct mutex list_mutex; + u32 mali_allocation_num; +}; + +extern struct idr mali_backend_idr; +extern struct mutex mali_idr_mutex; + +int mali_memory_manager_init(struct mali_allocation_manager *mgr); +void mali_memory_manager_uninit(struct mali_allocation_manager *mgr); + +void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc); +_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size); +mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address); +_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args); +_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args); +_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args); +_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args); + +#endif + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c new file mode 100755 index 000000000000..1e1f5eb4a0f7 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c @@ -0,0 +1,810 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "../platform/rk/custom_log.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_memory.h" +#include "mali_memory_os_alloc.h" +#include "mali_kernel_linux.h" + +/* Minimum size of allocator page pool */ +#define MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB * 256) +#define MALI_OS_MEMORY_POOL_TRIM_JIFFIES (10 * CONFIG_HZ) /* Default to 10s */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) +static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask); +#else +static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask); +#endif +#else +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc); +#else +static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc); +static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc); +#endif +#endif +static void mali_mem_os_trim_pool(struct work_struct *work); + +struct mali_mem_os_allocator mali_mem_os_allocator = { + .pool_lock = __SPIN_LOCK_UNLOCKED(pool_lock), + .pool_pages = LIST_HEAD_INIT(mali_mem_os_allocator.pool_pages), + .pool_count = 0, + + .allocated_pages = ATOMIC_INIT(0), + .allocation_limit = 0, + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) + .shrinker.shrink = mali_mem_os_shrink, +#else + .shrinker.count_objects = mali_mem_os_shrink_count, + .shrinker.scan_objects = mali_mem_os_shrink, +#endif + .shrinker.seeks = DEFAULT_SEEKS, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) + .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool, TIMER_DEFERRABLE), +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38) + .timed_shrinker = __DEFERRED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool), +#else + .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool), +#endif +}; + +u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag) +{ + LIST_HEAD(pages); + struct mali_page_node *m_page, *m_tmp; + u32 free_pages_nr = 0; + + if (MALI_TRUE == cow_flag) { + list_for_each_entry_safe(m_page, m_tmp, os_pages, list) { + /*only handle OS node here */ + if (m_page->type == MALI_PAGE_NODE_OS) { + if (1 == _mali_page_node_get_ref_count(m_page)) { + list_move(&m_page->list, &pages); + atomic_sub(1, &mali_mem_os_allocator.allocated_pages); + free_pages_nr ++; + } else { + _mali_page_node_unref(m_page); + m_page->page = NULL; + list_del(&m_page->list); + kfree(m_page); + } + } + } + } else { + list_cut_position(&pages, os_pages, os_pages->prev); + atomic_sub(pages_count, &mali_mem_os_allocator.allocated_pages); + free_pages_nr = pages_count; + } + + /* Put pages on pool. */ + spin_lock(&mali_mem_os_allocator.pool_lock); + list_splice(&pages, &mali_mem_os_allocator.pool_pages); + mali_mem_os_allocator.pool_count += free_pages_nr; + spin_unlock(&mali_mem_os_allocator.pool_lock); + + if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { + MALI_DEBUG_PRINT(5, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count)); + queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES); + } + return free_pages_nr; +} + +/** +* put page without put it into page pool +*/ +_mali_osk_errcode_t mali_mem_os_put_page(struct page *page) +{ + MALI_DEBUG_ASSERT_POINTER(page); + if (1 == page_count(page)) { + atomic_sub(1, &mali_mem_os_allocator.allocated_pages); + dma_unmap_page(&mali_platform_device->dev, page_private(page), + _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + ClearPagePrivate(page); + } + put_page(page); + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count) +{ + struct mali_page_node *m_page, *m_tmp; + u32 i = 0; + + MALI_DEBUG_ASSERT_POINTER(mem_from); + MALI_DEBUG_ASSERT_POINTER(mem_to); + + if (mem_from->count < start_page + page_count) { + return _MALI_OSK_ERR_INVALID_ARGS; + } + + list_for_each_entry_safe(m_page, m_tmp, &mem_from->pages, list) { + if (i >= start_page && i < start_page + page_count) { + list_move_tail(&m_page->list, &mem_to->pages); + mem_from->count--; + mem_to->count++; + } + i++; + } + + return _MALI_OSK_ERR_OK; +} + + +int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size) +{ + struct page *new_page; + LIST_HEAD(pages_list); + size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE; + size_t remaining = page_count; + struct mali_page_node *m_page, *m_tmp; + u32 i; + + MALI_DEBUG_ASSERT_POINTER(os_mem); + + if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { + MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", + size, + atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, + mali_mem_os_allocator.allocation_limit)); + return -ENOMEM; + } + + INIT_LIST_HEAD(&os_mem->pages); + os_mem->count = page_count; + + /* Grab pages from pool. */ + { + size_t pool_pages; + spin_lock(&mali_mem_os_allocator.pool_lock); + pool_pages = min(remaining, mali_mem_os_allocator.pool_count); + for (i = pool_pages; i > 0; i--) { + BUG_ON(list_empty(&mali_mem_os_allocator.pool_pages)); + list_move(mali_mem_os_allocator.pool_pages.next, &pages_list); + } + mali_mem_os_allocator.pool_count -= pool_pages; + remaining -= pool_pages; + spin_unlock(&mali_mem_os_allocator.pool_lock); + } + + /* Process pages from pool. */ + i = 0; + list_for_each_entry_safe(m_page, m_tmp, &pages_list, list) { + BUG_ON(NULL == m_page); + + list_move_tail(&m_page->list, &os_mem->pages); + } + + /* Allocate new pages, if needed. */ + for (i = 0; i < remaining; i++) { + dma_addr_t dma_addr; + gfp_t flags = __GFP_ZERO | GFP_HIGHUSER; + int err; + +#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE) + flags |= GFP_HIGHUSER; +#else +#ifdef CONFIG_ZONE_DMA32 + flags |= GFP_DMA32; +#else +#ifdef CONFIG_ZONE_DMA +#else + /* arm64 utgard only work on < 4G, but the kernel + * didn't provide method to allocte memory < 4G + */ + MALI_DEBUG_ASSERT(0); +#endif +#endif +#endif + + new_page = alloc_page(flags); + + if (unlikely(NULL == new_page)) { + E("err."); + /* Calculate the number of pages actually allocated, and free them. */ + os_mem->count = (page_count - remaining) + i; + atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); + mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); + return -ENOMEM; + } + + /* Ensure page is flushed from CPU caches. */ + dma_addr = dma_map_page(&mali_platform_device->dev, new_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + dma_unmap_page(&mali_platform_device->dev, dma_addr, + _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + dma_addr = dma_map_page(&mali_platform_device->dev, new_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + + err = dma_mapping_error(&mali_platform_device->dev, dma_addr); + if (unlikely(err)) { + MALI_DEBUG_PRINT_ERROR(("OS Mem: Failed to DMA map page %p: %u", + new_page, err)); + __free_page(new_page); + os_mem->count = (page_count - remaining) + i; + atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); + mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); + return -EFAULT; + } + + /* Store page phys addr */ + SetPagePrivate(new_page); + set_page_private(new_page, dma_addr); + + m_page = _mali_page_node_allocate(MALI_PAGE_NODE_OS); + if (unlikely(NULL == m_page)) { + MALI_PRINT_ERROR(("OS Mem: Can't allocate mali_page node! \n")); + dma_unmap_page(&mali_platform_device->dev, page_private(new_page), + _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + ClearPagePrivate(new_page); + __free_page(new_page); + os_mem->count = (page_count - remaining) + i; + atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); + mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); + return -EFAULT; + } + m_page->page = new_page; + + list_add_tail(&m_page->list, &os_mem->pages); + } + + atomic_add(page_count, &mali_mem_os_allocator.allocated_pages); + + if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) { + MALI_DEBUG_PRINT(4, ("OS Mem: Stopping pool trim timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count)); + cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker); + } + + return 0; +} + + +_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props) +{ + struct mali_page_directory *pagedir = session->page_directory; + struct mali_page_node *m_page; + u32 virt; + u32 prop = props; + + MALI_DEBUG_ASSERT_POINTER(session); + MALI_DEBUG_ASSERT_POINTER(os_mem); + + MALI_DEBUG_ASSERT(start_page <= os_mem->count); + MALI_DEBUG_ASSERT((start_page + mapping_pgae_num) <= os_mem->count); + + if ((start_page + mapping_pgae_num) == os_mem->count) { + + virt = vaddr + MALI_MMU_PAGE_SIZE * (start_page + mapping_pgae_num); + + list_for_each_entry_reverse(m_page, &os_mem->pages, list) { + + virt -= MALI_MMU_PAGE_SIZE; + if (mapping_pgae_num > 0) { + dma_addr_t phys = page_private(m_page->page); +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); + } else { + break; + } + mapping_pgae_num--; + } + + } else { + u32 i = 0; + virt = vaddr; + list_for_each_entry(m_page, &os_mem->pages, list) { + + if (i >= start_page) { + dma_addr_t phys = page_private(m_page->page); + +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (phys >> 32)); +#endif + mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); + } + i++; + virt += MALI_MMU_PAGE_SIZE; + } + } + return _MALI_OSK_ERR_OK; +} + + +void mali_mem_os_mali_unmap(mali_mem_allocation *alloc) +{ + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + +int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) +{ + mali_mem_os_mem *os_mem = &mem_bkend->os_mem; + struct mali_page_node *m_page; + struct page *page; + int ret; + unsigned long addr = vma->vm_start; + MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type); + + list_for_each_entry(m_page, &os_mem->pages, list) { + /* We should use vm_insert_page, but it does a dcache + * flush which makes it way slower than remap_pfn_range or vmf_insert_pfn. + ret = vm_insert_page(vma, addr, page); + */ + page = m_page->page; + ret = vmf_insert_pfn(vma, addr, page_to_pfn(page)); + + if (unlikely(0 != ret)) { + return -EFAULT; + } + addr += _MALI_OSK_MALI_PAGE_SIZE; + } + + return 0; +} + +_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size) +{ + mali_mem_os_mem *os_mem = &mem_bkend->os_mem; + struct mali_page_node *m_page; + int ret; + int offset; + int mapping_page_num; + int count ; + + unsigned long vstart = vma->vm_start; + count = 0; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); + MALI_DEBUG_ASSERT(0 == start_vaddr % _MALI_OSK_MALI_PAGE_SIZE); + MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE); + offset = (start_vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE; + MALI_DEBUG_ASSERT(offset <= os_mem->count); + mapping_page_num = mappig_size / _MALI_OSK_MALI_PAGE_SIZE; + MALI_DEBUG_ASSERT((offset + mapping_page_num) <= os_mem->count); + + if ((offset + mapping_page_num) == os_mem->count) { + + unsigned long vm_end = start_vaddr + mappig_size; + + list_for_each_entry_reverse(m_page, &os_mem->pages, list) { + + vm_end -= _MALI_OSK_MALI_PAGE_SIZE; + if (mapping_page_num > 0) { + ret = vmf_insert_pfn(vma, vm_end, page_to_pfn(m_page->page)); + + if (unlikely(0 != ret)) { + /*will return -EBUSY If the page has already been mapped into table, but it's OK*/ + if (-EBUSY == ret) { + break; + } else { + MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, offset is %d,page_count is %d\n", + ret, offset + mapping_page_num, os_mem->count)); + } + return _MALI_OSK_ERR_FAULT; + } + } else { + break; + } + mapping_page_num--; + + } + } else { + + list_for_each_entry(m_page, &os_mem->pages, list) { + if (count >= offset) { + + ret = vmf_insert_pfn(vma, vstart, page_to_pfn(m_page->page)); + + if (unlikely(0 != ret)) { + /*will return -EBUSY If the page has already been mapped into table, but it's OK*/ + if (-EBUSY == ret) { + break; + } else { + MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, count is %d, offset is %d,page_count is %d\n", + ret, count, offset, os_mem->count)); + } + return _MALI_OSK_ERR_FAULT; + } + } + count++; + vstart += _MALI_OSK_MALI_PAGE_SIZE; + } + } + return _MALI_OSK_ERR_OK; +} + +u32 mali_mem_os_release(mali_mem_backend *mem_bkend) +{ + + mali_mem_allocation *alloc; + struct mali_session_data *session; + u32 free_pages_nr = 0; + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type); + + alloc = mem_bkend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + /* Unmap the memory from the mali virtual address space. */ + mali_mem_os_mali_unmap(alloc); + mutex_lock(&mem_bkend->mutex); + /* Free pages */ + if (MALI_MEM_BACKEND_FLAG_COWED & mem_bkend->flags) { + /* Lock to avoid the free race condition for the cow shared memory page node. */ + _mali_osk_mutex_wait(session->cow_lock); + free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_TRUE); + _mali_osk_mutex_signal(session->cow_lock); + } else { + free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_FALSE); + } + mutex_unlock(&mem_bkend->mutex); + + MALI_DEBUG_PRINT(4, ("OS Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->os_mem.count * _MALI_OSK_MALI_PAGE_SIZE, + free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); + + mem_bkend->os_mem.count = 0; + return free_pages_nr; +} + + +#define MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE 128 +static struct { + struct { + mali_dma_addr phys; + mali_io_address mapping; + } page[MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE]; + size_t count; + spinlock_t lock; +} mali_mem_page_table_page_pool = { + .count = 0, + .lock = __SPIN_LOCK_UNLOCKED(pool_lock), +}; + +_mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping) +{ + _mali_osk_errcode_t ret = _MALI_OSK_ERR_NOMEM; + dma_addr_t tmp_phys; + + spin_lock(&mali_mem_page_table_page_pool.lock); + if (0 < mali_mem_page_table_page_pool.count) { + u32 i = --mali_mem_page_table_page_pool.count; + *phys = mali_mem_page_table_page_pool.page[i].phys; + *mapping = mali_mem_page_table_page_pool.page[i].mapping; + + ret = _MALI_OSK_ERR_OK; + } + spin_unlock(&mali_mem_page_table_page_pool.lock); + + if (_MALI_OSK_ERR_OK != ret) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + *mapping = dma_alloc_attrs(&mali_platform_device->dev, + _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys, + GFP_KERNEL, DMA_ATTR_WRITE_COMBINE); +#else + *mapping = dma_alloc_writecombine(&mali_platform_device->dev, + _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys, GFP_KERNEL); +#endif + if (NULL != *mapping) { + ret = _MALI_OSK_ERR_OK; + +#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) + /* Verify that the "physical" address is 32-bit and + * usable for Mali, when on a system with bus addresses + * wider than 32-bit. */ + MALI_DEBUG_ASSERT(0 == (tmp_phys >> 32)); +#endif + + *phys = (mali_dma_addr)tmp_phys; + } + } + + return ret; +} + +void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt) +{ + spin_lock(&mali_mem_page_table_page_pool.lock); + if (MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE > mali_mem_page_table_page_pool.count) { + u32 i = mali_mem_page_table_page_pool.count; + mali_mem_page_table_page_pool.page[i].phys = phys; + mali_mem_page_table_page_pool.page[i].mapping = virt; + + ++mali_mem_page_table_page_pool.count; + + spin_unlock(&mali_mem_page_table_page_pool.lock); + } else { + spin_unlock(&mali_mem_page_table_page_pool.lock); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + dma_free_attrs(&mali_platform_device->dev, + _MALI_OSK_MALI_PAGE_SIZE, virt, phys, + DMA_ATTR_WRITE_COMBINE); +#else + dma_free_writecombine(&mali_platform_device->dev, + _MALI_OSK_MALI_PAGE_SIZE, virt, phys); +#endif + } +} + +void mali_mem_os_free_page_node(struct mali_page_node *m_page) +{ + struct page *page = m_page->page; + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_OS); + + if (1 == page_count(page)) { + dma_unmap_page(&mali_platform_device->dev, page_private(page), + _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); + ClearPagePrivate(page); + } + __free_page(page); + m_page->page = NULL; + list_del(&m_page->list); + kfree(m_page); +} + +/* The maximum number of page table pool pages to free in one go. */ +#define MALI_MEM_OS_CHUNK_TO_FREE 64UL + +/* Free a certain number of pages from the page table page pool. + * The pool lock must be held when calling the function, and the lock will be + * released before returning. + */ +static void mali_mem_os_page_table_pool_free(size_t nr_to_free) +{ + mali_dma_addr phys_arr[MALI_MEM_OS_CHUNK_TO_FREE]; + void *virt_arr[MALI_MEM_OS_CHUNK_TO_FREE]; + u32 i; + + MALI_DEBUG_ASSERT(nr_to_free <= MALI_MEM_OS_CHUNK_TO_FREE); + + /* Remove nr_to_free pages from the pool and store them locally on stack. */ + for (i = 0; i < nr_to_free; i++) { + u32 pool_index = mali_mem_page_table_page_pool.count - i - 1; + + phys_arr[i] = mali_mem_page_table_page_pool.page[pool_index].phys; + virt_arr[i] = mali_mem_page_table_page_pool.page[pool_index].mapping; + } + + mali_mem_page_table_page_pool.count -= nr_to_free; + + spin_unlock(&mali_mem_page_table_page_pool.lock); + + /* After releasing the spinlock: free the pages we removed from the pool. */ + for (i = 0; i < nr_to_free; i++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) + dma_free_attrs(&mali_platform_device->dev, _MALI_OSK_MALI_PAGE_SIZE, + virt_arr[i], (dma_addr_t)phys_arr[i], + DMA_ATTR_WRITE_COMBINE); +#else + dma_free_writecombine(&mali_platform_device->dev, + _MALI_OSK_MALI_PAGE_SIZE, + virt_arr[i], (dma_addr_t)phys_arr[i]); +#endif + } +} + +static void mali_mem_os_trim_page_table_page_pool(void) +{ + size_t nr_to_free = 0; + size_t nr_to_keep; + + /* Keep 2 page table pages for each 1024 pages in the page cache. */ + nr_to_keep = mali_mem_os_allocator.pool_count / 512; + /* And a minimum of eight pages, to accomodate new sessions. */ + nr_to_keep += 8; + + if (0 == spin_trylock(&mali_mem_page_table_page_pool.lock)) return; + + if (nr_to_keep < mali_mem_page_table_page_pool.count) { + nr_to_free = mali_mem_page_table_page_pool.count - nr_to_keep; + nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, nr_to_free); + } + + /* Pool lock will be released by the callee. */ + mali_mem_os_page_table_pool_free(nr_to_free); +} + +static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc) +{ + return mali_mem_os_allocator.pool_count; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) +static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask) +#else +static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask) +#endif /* Linux < 2.6.35 */ +#else +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc) +#else +static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc) +#endif /* Linux < 3.12.0 */ +#endif /* Linux < 3.0.0 */ +{ + struct mali_page_node *m_page, *m_tmp; + unsigned long flags; + struct list_head *le, pages; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) + int nr = nr_to_scan; +#else + int nr = sc->nr_to_scan; +#endif + + if (0 == nr) { + return mali_mem_os_shrink_count(shrinker, sc); + } + + if (0 == spin_trylock_irqsave(&mali_mem_os_allocator.pool_lock, flags)) { + /* Not able to lock. */ + return -1; + } + + if (0 == mali_mem_os_allocator.pool_count) { + /* No pages availble */ + spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags); + return 0; + } + + /* Release from general page pool */ + nr = min((size_t)nr, mali_mem_os_allocator.pool_count); + mali_mem_os_allocator.pool_count -= nr; + list_for_each(le, &mali_mem_os_allocator.pool_pages) { + --nr; + if (0 == nr) break; + } + list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le); + spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags); + + list_for_each_entry_safe(m_page, m_tmp, &pages, list) { + mali_mem_os_free_page_node(m_page); + } + + if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) { + /* Pools are empty, stop timer */ + MALI_DEBUG_PRINT(5, ("Stopping timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count)); + cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker); + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) + return mali_mem_os_shrink_count(shrinker, sc); +#else + return nr; +#endif +} + +static void mali_mem_os_trim_pool(struct work_struct *data) +{ + struct mali_page_node *m_page, *m_tmp; + struct list_head *le; + LIST_HEAD(pages); + size_t nr_to_free; + + MALI_IGNORE(data); + + MALI_DEBUG_PRINT(3, ("OS Mem: Trimming pool %u\n", mali_mem_os_allocator.pool_count)); + + /* Release from general page pool */ + spin_lock(&mali_mem_os_allocator.pool_lock); + if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { + size_t count = mali_mem_os_allocator.pool_count - MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES; + const size_t min_to_free = min(64, MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES); + + /* Free half the pages on the pool above the static limit. Or 64 pages, 256KB. */ + nr_to_free = max(count / 2, min_to_free); + + mali_mem_os_allocator.pool_count -= nr_to_free; + list_for_each(le, &mali_mem_os_allocator.pool_pages) { + --nr_to_free; + if (0 == nr_to_free) break; + } + list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le); + } + spin_unlock(&mali_mem_os_allocator.pool_lock); + + list_for_each_entry_safe(m_page, m_tmp, &pages, list) { + mali_mem_os_free_page_node(m_page); + } + + /* Release some pages from page table page pool */ + mali_mem_os_trim_page_table_page_pool(); + + if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { + MALI_DEBUG_PRINT(4, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count)); + queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES); + } +} + +_mali_osk_errcode_t mali_mem_os_init(void) +{ + mali_mem_os_allocator.wq = alloc_workqueue("mali-mem", WQ_UNBOUND, 1); + if (NULL == mali_mem_os_allocator.wq) { + return _MALI_OSK_ERR_NOMEM; + } + + register_shrinker(&mali_mem_os_allocator.shrinker); + + return _MALI_OSK_ERR_OK; +} + +void mali_mem_os_term(void) +{ + struct mali_page_node *m_page, *m_tmp; + unregister_shrinker(&mali_mem_os_allocator.shrinker); + cancel_delayed_work_sync(&mali_mem_os_allocator.timed_shrinker); + + if (NULL != mali_mem_os_allocator.wq) { + destroy_workqueue(mali_mem_os_allocator.wq); + mali_mem_os_allocator.wq = NULL; + } + + spin_lock(&mali_mem_os_allocator.pool_lock); + list_for_each_entry_safe(m_page, m_tmp, &mali_mem_os_allocator.pool_pages, list) { + mali_mem_os_free_page_node(m_page); + + --mali_mem_os_allocator.pool_count; + } + BUG_ON(mali_mem_os_allocator.pool_count); + spin_unlock(&mali_mem_os_allocator.pool_lock); + + /* Release from page table page pool */ + do { + u32 nr_to_free; + + spin_lock(&mali_mem_page_table_page_pool.lock); + + nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, mali_mem_page_table_page_pool.count); + + /* Pool lock will be released by the callee. */ + mali_mem_os_page_table_pool_free(nr_to_free); + } while (0 != mali_mem_page_table_page_pool.count); +} + +_mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size) +{ + mali_mem_os_allocator.allocation_limit = size; + + MALI_SUCCESS; +} + +u32 mali_mem_os_stat(void) +{ + return atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h new file mode 100755 index 000000000000..8c9b35d0b230 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_OS_ALLOC_H__ +#define __MALI_MEMORY_OS_ALLOC_H__ + +#include "mali_osk.h" +#include "mali_memory_types.h" + + +/** @brief Release Mali OS memory + * + * The session memory_lock must be held when calling this function. + * + * @param mem_bkend Pointer to the mali_mem_backend to release + */ +u32 mali_mem_os_release(mali_mem_backend *mem_bkend); + +_mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping); + +void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt); + +_mali_osk_errcode_t mali_mem_os_init(void); + +void mali_mem_os_term(void); + +u32 mali_mem_os_stat(void); + +void mali_mem_os_free_page_node(struct mali_page_node *m_page); + +int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size); + +u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag); + +_mali_osk_errcode_t mali_mem_os_put_page(struct page *page); + +_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count); + +_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props); + +void mali_mem_os_mali_unmap(mali_mem_allocation *alloc); + +int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); + +_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size); + +#endif /* __MALI_MEMORY_OS_ALLOC_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.c new file mode 100755 index 000000000000..0b4f828680d0 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.c @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_memory.h" +#include "mali_memory_secure.h" +#include "mali_osk.h" +#include +#include +#include +#include + +_mali_osk_errcode_t mali_mem_secure_attach_dma_buf(mali_mem_secure *secure_mem, u32 size, int mem_fd) +{ + struct dma_buf *buf; + MALI_DEBUG_ASSERT_POINTER(secure_mem); + + /* get dma buffer */ + buf = dma_buf_get(mem_fd); + if (IS_ERR_OR_NULL(buf)) { + MALI_DEBUG_PRINT_ERROR(("Failed to get dma buf!\n")); + return _MALI_OSK_ERR_FAULT; + } + + if (size != buf->size) { + MALI_DEBUG_PRINT_ERROR(("The secure mem size not match to the dma buf size!\n")); + goto failed_alloc_mem; + } + + secure_mem->buf = buf; + secure_mem->attachment = dma_buf_attach(secure_mem->buf, &mali_platform_device->dev); + if (NULL == secure_mem->attachment) { + MALI_DEBUG_PRINT_ERROR(("Failed to get dma buf attachment!\n")); + goto failed_dma_attach; + } + + secure_mem->sgt = dma_buf_map_attachment(secure_mem->attachment, DMA_BIDIRECTIONAL); + if (IS_ERR_OR_NULL(secure_mem->sgt)) { + MALI_DEBUG_PRINT_ERROR(("Failed to map dma buf attachment\n")); + goto failed_dma_map; + } + + secure_mem->count = size / MALI_MMU_PAGE_SIZE; + + return _MALI_OSK_ERR_OK; + +failed_dma_map: + dma_buf_detach(secure_mem->buf, secure_mem->attachment); +failed_dma_attach: +failed_alloc_mem: + dma_buf_put(buf); + return _MALI_OSK_ERR_FAULT; +} + +_mali_osk_errcode_t mali_mem_secure_mali_map(mali_mem_secure *secure_mem, struct mali_session_data *session, u32 vaddr, u32 props) +{ + struct mali_page_directory *pagedir; + struct scatterlist *sg; + u32 virt = vaddr; + u32 prop = props; + int i; + + MALI_DEBUG_ASSERT_POINTER(secure_mem); + MALI_DEBUG_ASSERT_POINTER(secure_mem->sgt); + MALI_DEBUG_ASSERT_POINTER(session); + + pagedir = session->page_directory; + + for_each_sg(secure_mem->sgt->sgl, sg, secure_mem->sgt->nents, i) { + u32 size = sg_dma_len(sg); + dma_addr_t phys = sg_dma_address(sg); + + /* sg must be page aligned. */ + MALI_DEBUG_ASSERT(0 == size % MALI_MMU_PAGE_SIZE); + MALI_DEBUG_ASSERT(0 == (phys & ~(uintptr_t)0xFFFFFFFF)); + + mali_mmu_pagedir_update(pagedir, virt, phys, size, prop); + + MALI_DEBUG_PRINT(3, ("The secure mem physical address: 0x%x gpu virtual address: 0x%x! \n", phys, virt)); + virt += size; + } + + return _MALI_OSK_ERR_OK; +} + +void mali_mem_secure_mali_unmap(mali_mem_allocation *alloc) +{ + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + + +int mali_mem_secure_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) +{ + + int ret = 0; + struct scatterlist *sg; + mali_mem_secure *secure_mem = &mem_bkend->secure_mem; + unsigned long addr = vma->vm_start; + int i; + + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_SECURE); + + for_each_sg(secure_mem->sgt->sgl, sg, secure_mem->sgt->nents, i) { + phys_addr_t phys; + dma_addr_t dev_addr; + u32 size, j; + dev_addr = sg_dma_address(sg); +#if defined(CONFIG_ARM64) ||LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) + phys = dma_to_phys(&mali_platform_device->dev, dev_addr); +#else + phys = page_to_phys(pfn_to_page(dma_to_pfn(&mali_platform_device->dev, dev_addr))); +#endif + size = sg_dma_len(sg); + MALI_DEBUG_ASSERT(0 == size % _MALI_OSK_MALI_PAGE_SIZE); + + for (j = 0; j < size / _MALI_OSK_MALI_PAGE_SIZE; j++) { + ret = vmf_insert_pfn(vma, addr, PFN_DOWN(phys)); + + if (unlikely(0 != ret)) { + return -EFAULT; + } + addr += _MALI_OSK_MALI_PAGE_SIZE; + phys += _MALI_OSK_MALI_PAGE_SIZE; + + MALI_DEBUG_PRINT(3, ("The secure mem physical address: 0x%x , cpu virtual address: 0x%x! \n", phys, addr)); + } + } + return ret; +} + +u32 mali_mem_secure_release(mali_mem_backend *mem_bkend) +{ + struct mali_mem_secure *mem; + mali_mem_allocation *alloc = mem_bkend->mali_allocation; + u32 free_pages_nr = 0; + MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_SECURE); + + mem = &mem_bkend->secure_mem; + MALI_DEBUG_ASSERT_POINTER(mem->attachment); + MALI_DEBUG_ASSERT_POINTER(mem->buf); + MALI_DEBUG_ASSERT_POINTER(mem->sgt); + /* Unmap the memory from the mali virtual address space. */ + mali_mem_secure_mali_unmap(alloc); + mutex_lock(&mem_bkend->mutex); + dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL); + dma_buf_detach(mem->buf, mem->attachment); + dma_buf_put(mem->buf); + mutex_unlock(&mem_bkend->mutex); + + free_pages_nr = mem->count; + + return free_pages_nr; +} + + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.h new file mode 100755 index 000000000000..48691d4790fe --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2010, 2013, 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_SECURE_H__ +#define __MALI_MEMORY_SECURE_H__ + +#include "mali_session.h" +#include "mali_memory.h" +#include + +#include "mali_memory_types.h" + +_mali_osk_errcode_t mali_mem_secure_attach_dma_buf(mali_mem_secure *secure_mem, u32 size, int mem_fd); + +_mali_osk_errcode_t mali_mem_secure_mali_map(mali_mem_secure *secure_mem, struct mali_session_data *session, u32 vaddr, u32 props); + +void mali_mem_secure_mali_unmap(mali_mem_allocation *alloc); + +int mali_mem_secure_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); + +u32 mali_mem_secure_release(mali_mem_backend *mem_bkend); + +#endif /* __MALI_MEMORY_SECURE_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c new file mode 100755 index 000000000000..d682785b9673 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c @@ -0,0 +1,943 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_memory.h" +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" +#include "mali_memory_cow.h" +#include "mali_ukk.h" +#include "mali_kernel_utilization.h" +#include "mali_memory_swap_alloc.h" + + +static struct _mali_osk_bitmap idx_mgr; +static struct file *global_swap_file; +static struct address_space *global_swap_space; +static _mali_osk_wq_work_t *mali_mem_swap_out_workq = NULL; +static u32 mem_backend_swapped_pool_size; +#ifdef MALI_MEM_SWAP_TRACKING +static u32 mem_backend_swapped_unlock_size; +#endif +/* Lock order: mem_backend_swapped_pool_lock > each memory backend's mutex lock. + * This lock used to protect mem_backend_swapped_pool_size and mem_backend_swapped_pool. */ +static struct mutex mem_backend_swapped_pool_lock; +static struct list_head mem_backend_swapped_pool; + +extern struct mali_mem_os_allocator mali_mem_os_allocator; + +#define MALI_SWAP_LOW_MEM_DEFAULT_VALUE (60*1024*1024) +#define MALI_SWAP_INVALIDATE_MALI_ADDRESS (0) /* Used to mark the given memory cookie is invalidate. */ +#define MALI_SWAP_GLOBAL_SWAP_FILE_SIZE (0xFFFFFFFF) +#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX \ + ((MALI_SWAP_GLOBAL_SWAP_FILE_SIZE) >> PAGE_SHIFT) +#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE (1 << 15) /* Reserved for CoW nonlinear swap backend memory, the space size is 128MB. */ + +unsigned int mali_mem_swap_out_threshold_value = MALI_SWAP_LOW_MEM_DEFAULT_VALUE; + +/** + * We have two situations to do shrinking things, one is we met low GPU utilization which shows GPU needn't touch too + * swappable backends in short time, and the other one is we add new swappable backends, the total pool size exceed + * the threshold value of the swapped pool size. + */ +typedef enum { + MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION = 100, + MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS = 257, +} _mali_mem_swap_pool_shrink_type_t; + +static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg); + +_mali_osk_errcode_t mali_mem_swap_init(void) +{ + gfp_t flags = __GFP_NORETRY | __GFP_NOWARN; + + if (_MALI_OSK_ERR_OK != _mali_osk_bitmap_init(&idx_mgr, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE)) { + return _MALI_OSK_ERR_NOMEM; + } + + global_swap_file = shmem_file_setup("mali_swap", MALI_SWAP_GLOBAL_SWAP_FILE_SIZE, VM_NORESERVE); + if (IS_ERR(global_swap_file)) { + _mali_osk_bitmap_term(&idx_mgr); + return _MALI_OSK_ERR_NOMEM; + } + + global_swap_space = global_swap_file->f_path.dentry->d_inode->i_mapping; + + mali_mem_swap_out_workq = _mali_osk_wq_create_work(mali_mem_swap_swapped_bkend_pool_check_for_low_utilization, NULL); + if (NULL == mali_mem_swap_out_workq) { + _mali_osk_bitmap_term(&idx_mgr); + fput(global_swap_file); + return _MALI_OSK_ERR_NOMEM; + } + +#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE) + flags |= GFP_HIGHUSER; +#else +#ifdef CONFIG_ZONE_DMA32 + flags |= GFP_DMA32; +#else +#ifdef CONFIG_ZONE_DMA + flags |= GFP_DMA; +#else + /* arm64 utgard only work on < 4G, but the kernel + * didn't provide method to allocte memory < 4G + */ + MALI_DEBUG_ASSERT(0); +#endif +#endif +#endif + + /* When we use shmem_read_mapping_page to allocate/swap-in, it will + * use these flags to allocate new page if need.*/ + mapping_set_gfp_mask(global_swap_space, flags); + + mem_backend_swapped_pool_size = 0; +#ifdef MALI_MEM_SWAP_TRACKING + mem_backend_swapped_unlock_size = 0; +#endif + mutex_init(&mem_backend_swapped_pool_lock); + INIT_LIST_HEAD(&mem_backend_swapped_pool); + + MALI_DEBUG_PRINT(2, ("Mali SWAP: Swap out threshold vaule is %uM\n", mali_mem_swap_out_threshold_value >> 20)); + + return _MALI_OSK_ERR_OK; +} + +void mali_mem_swap_term(void) +{ + _mali_osk_bitmap_term(&idx_mgr); + + fput(global_swap_file); + + _mali_osk_wq_delete_work(mali_mem_swap_out_workq); + + MALI_DEBUG_ASSERT(list_empty(&mem_backend_swapped_pool)); + MALI_DEBUG_ASSERT(0 == mem_backend_swapped_pool_size); + + return; +} + +struct file *mali_mem_swap_get_global_swap_file(void) +{ + return global_swap_file; +} + +/* Judge if swappable backend in swapped pool. */ +static mali_bool mali_memory_swap_backend_in_swapped_pool(mali_mem_backend *mem_bkend) +{ + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + + return !list_empty(&mem_bkend->list); +} + +void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend) +{ + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + + mutex_lock(&mem_backend_swapped_pool_lock); + mutex_lock(&mem_bkend->mutex); + + if (MALI_FALSE == mali_memory_swap_backend_in_swapped_pool(mem_bkend)) { + mutex_unlock(&mem_bkend->mutex); + mutex_unlock(&mem_backend_swapped_pool_lock); + return; + } + + MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list)); + + list_del_init(&mem_bkend->list); + + mutex_unlock(&mem_bkend->mutex); + + mem_backend_swapped_pool_size -= mem_bkend->size; + + mutex_unlock(&mem_backend_swapped_pool_lock); +} + +static void mali_mem_swap_out_page_node(mali_page_node *page_node) +{ + MALI_DEBUG_ASSERT(page_node); + + dma_unmap_page(&mali_platform_device->dev, page_node->swap_it->dma_addr, + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + set_page_dirty(page_node->swap_it->page); + put_page(page_node->swap_it->page); +} + +void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend) +{ + mali_page_node *m_page; + + MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex)); + + if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN)) { + return; + } + + mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN; + + list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { + mali_mem_swap_out_page_node(m_page); + } + + return; +} + +static void mali_mem_swap_unlock_partial_locked_mem_backend(mali_mem_backend *mem_bkend, mali_page_node *page_node) +{ + mali_page_node *m_page; + + MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex)); + + list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { + if (m_page == page_node) { + break; + } + mali_mem_swap_out_page_node(m_page); + } +} + +static void mali_mem_swap_swapped_bkend_pool_shrink(_mali_mem_swap_pool_shrink_type_t shrink_type) +{ + mali_mem_backend *bkend, *tmp_bkend; + long system_free_size; + u32 last_gpu_utilization, gpu_utilization_threshold_value, temp_swap_out_threshold_value; + + MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_backend_swapped_pool_lock)); + + if (MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION == shrink_type) { + /** + * When we met that system memory is very low and Mali locked swappable memory size is less than + * threshold value, and at the same time, GPU load is very low and don't need high performance, + * at this condition, we can unlock more swap memory backend from swapped backends pool. + */ + gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION; + temp_swap_out_threshold_value = (mali_mem_swap_out_threshold_value >> 2); + } else { + /* When we add swappable memory backends to swapped pool, we need to think that we couldn't + * hold too much swappable backends in Mali driver, and also we need considering performance. + * So there is a balance for swapping out memory backend, we should follow the following conditions: + * 1. Total memory size in global mem backend swapped pool is more than the defined threshold value. + * 2. System level free memory size is less than the defined threshold value. + * 3. Please note that GPU utilization problem isn't considered in this condition. + */ + gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS; + temp_swap_out_threshold_value = mali_mem_swap_out_threshold_value; + } + + /* Get system free pages number. */ + system_free_size = global_zone_page_state(NR_FREE_PAGES) * PAGE_SIZE; + last_gpu_utilization = _mali_ukk_utilization_gp_pp(); + + if ((last_gpu_utilization < gpu_utilization_threshold_value) + && (system_free_size < mali_mem_swap_out_threshold_value) + && (mem_backend_swapped_pool_size > temp_swap_out_threshold_value)) { + list_for_each_entry_safe(bkend, tmp_bkend, &mem_backend_swapped_pool, list) { + if (mem_backend_swapped_pool_size <= temp_swap_out_threshold_value) { + break; + } + + mutex_lock(&bkend->mutex); + + /* check if backend is in use. */ + if (0 < bkend->using_count) { + mutex_unlock(&bkend->mutex); + continue; + } + + mali_mem_swap_unlock_single_mem_backend(bkend); + list_del_init(&bkend->list); + mem_backend_swapped_pool_size -= bkend->size; +#ifdef MALI_MEM_SWAP_TRACKING + mem_backend_swapped_unlock_size += bkend->size; +#endif + mutex_unlock(&bkend->mutex); + } + } + + return; +} + +static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg) +{ + MALI_IGNORE(arg); + + mutex_lock(&mem_backend_swapped_pool_lock); + + mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION); + + mutex_unlock(&mem_backend_swapped_pool_lock); +} + +/** + * After PP job finished, we add all of swappable memory backend used by this PP + * job to the tail of the global swapped pool, and if the total size of swappable memory is more than threshold + * value, we also need to shrink the swapped pool start from the head of the list. + */ +void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend) +{ + mutex_lock(&mem_backend_swapped_pool_lock); + mutex_lock(&mem_bkend->mutex); + + if (mali_memory_swap_backend_in_swapped_pool(mem_bkend)) { + MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list)); + + list_del_init(&mem_bkend->list); + list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool); + mutex_unlock(&mem_bkend->mutex); + mutex_unlock(&mem_backend_swapped_pool_lock); + return; + } + + list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool); + + mutex_unlock(&mem_bkend->mutex); + mem_backend_swapped_pool_size += mem_bkend->size; + + mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS); + + mutex_unlock(&mem_backend_swapped_pool_lock); + return; +} + + +u32 mali_mem_swap_idx_alloc(void) +{ + return _mali_osk_bitmap_alloc(&idx_mgr); +} + +void mali_mem_swap_idx_free(u32 idx) +{ + _mali_osk_bitmap_free(&idx_mgr, idx); +} + +static u32 mali_mem_swap_idx_range_alloc(u32 count) +{ + u32 index; + + index = _mali_osk_bitmap_alloc_range(&idx_mgr, count); + + return index; +} + +static void mali_mem_swap_idx_range_free(u32 idx, int num) +{ + _mali_osk_bitmap_free_range(&idx_mgr, idx, num); +} + +struct mali_swap_item *mali_mem_swap_alloc_swap_item(void) +{ + mali_swap_item *swap_item; + + swap_item = kzalloc(sizeof(mali_swap_item), GFP_KERNEL); + + if (NULL == swap_item) { + return NULL; + } + + atomic_set(&swap_item->ref_count, 1); + swap_item->page = NULL; + atomic_add(1, &mali_mem_os_allocator.allocated_pages); + + return swap_item; +} + +void mali_mem_swap_free_swap_item(mali_swap_item *swap_item) +{ + struct inode *file_node; + long long start, end; + + /* If this swap item is shared, we just reduce the reference counter. */ + if (0 == atomic_dec_return(&swap_item->ref_count)) { + file_node = global_swap_file->f_path.dentry->d_inode; + start = swap_item->idx; + start = start << 12; + end = start + PAGE_SIZE; + + shmem_truncate_range(file_node, start, (end - 1)); + + mali_mem_swap_idx_free(swap_item->idx); + + atomic_sub(1, &mali_mem_os_allocator.allocated_pages); + + kfree(swap_item); + } +} + +/* Used to allocate new swap item for new memory allocation and cow page for write. */ +struct mali_page_node *_mali_mem_swap_page_node_allocate(void) +{ + struct mali_page_node *m_page; + + m_page = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP); + + if (NULL == m_page) { + return NULL; + } + + m_page->swap_it = mali_mem_swap_alloc_swap_item(); + + if (NULL == m_page->swap_it) { + kfree(m_page); + return NULL; + } + + return m_page; +} + +_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page) +{ + + mali_mem_swap_free_swap_item(m_page->swap_it); + + return _MALI_OSK_ERR_OK; +} + +void _mali_mem_swap_page_node_free(struct mali_page_node *m_page) +{ + _mali_mem_swap_put_page_node(m_page); + + kfree(m_page); + + return; +} + +u32 mali_mem_swap_free(mali_mem_swap *swap_mem) +{ + struct mali_page_node *m_page, *m_tmp; + u32 free_pages_nr = 0; + + MALI_DEBUG_ASSERT_POINTER(swap_mem); + + list_for_each_entry_safe(m_page, m_tmp, &swap_mem->pages, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP); + + /* free the page node and release the swap item, if the ref count is 1, + * then need also free the swap item. */ + list_del(&m_page->list); + if (1 == _mali_page_node_get_ref_count(m_page)) { + free_pages_nr++; + } + + _mali_mem_swap_page_node_free(m_page); + } + + return free_pages_nr; +} + +static u32 mali_mem_swap_cow_free(mali_mem_cow *cow_mem) +{ + struct mali_page_node *m_page, *m_tmp; + u32 free_pages_nr = 0; + + MALI_DEBUG_ASSERT_POINTER(cow_mem); + + list_for_each_entry_safe(m_page, m_tmp, &cow_mem->pages, list) { + MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP); + + /* free the page node and release the swap item, if the ref count is 1, + * then need also free the swap item. */ + list_del(&m_page->list); + if (1 == _mali_page_node_get_ref_count(m_page)) { + free_pages_nr++; + } + + _mali_mem_swap_page_node_free(m_page); + } + + return free_pages_nr; +} + +u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped) +{ + mali_mem_allocation *alloc; + u32 free_pages_nr = 0; + + MALI_DEBUG_ASSERT_POINTER(mem_bkend); + alloc = mem_bkend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + if (is_mali_mapped) { + mali_mem_swap_mali_unmap(alloc); + } + + mali_memory_swap_list_backend_delete(mem_bkend); + + mutex_lock(&mem_bkend->mutex); + /* To make sure the given memory backend was unlocked from Mali side, + * and then free this memory block. */ + mali_mem_swap_unlock_single_mem_backend(mem_bkend); + mutex_unlock(&mem_bkend->mutex); + + if (MALI_MEM_SWAP == mem_bkend->type) { + free_pages_nr = mali_mem_swap_free(&mem_bkend->swap_mem); + } else { + free_pages_nr = mali_mem_swap_cow_free(&mem_bkend->cow_mem); + } + + return free_pages_nr; +} + +mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node) +{ + MALI_DEBUG_ASSERT(NULL != page_node); + + page_node->swap_it->page = shmem_read_mapping_page(global_swap_space, page_node->swap_it->idx); + + if (IS_ERR(page_node->swap_it->page)) { + MALI_DEBUG_PRINT_ERROR(("SWAP Mem: failed to swap in page with index: %d.\n", page_node->swap_it->idx)); + return MALI_FALSE; + } + + /* Ensure page is flushed from CPU caches. */ + page_node->swap_it->dma_addr = dma_map_page(&mali_platform_device->dev, page_node->swap_it->page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + return MALI_TRUE; +} + +int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx) +{ + size_t page_count = PAGE_ALIGN(size) / PAGE_SIZE; + struct mali_page_node *m_page; + long system_free_size; + u32 i, index; + mali_bool ret; + + MALI_DEBUG_ASSERT(NULL != swap_mem); + MALI_DEBUG_ASSERT(NULL != bkend_idx); + MALI_DEBUG_ASSERT(page_count <= MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE); + + if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { + MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", + size, + atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, + mali_mem_os_allocator.allocation_limit)); + return _MALI_OSK_ERR_NOMEM; + } + + INIT_LIST_HEAD(&swap_mem->pages); + swap_mem->count = page_count; + index = mali_mem_swap_idx_range_alloc(page_count); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == index) { + MALI_PRINT_ERROR(("Mali Swap: Failed to allocate continuous index for swappable Mali memory.")); + return _MALI_OSK_ERR_FAULT; + } + + for (i = 0; i < page_count; i++) { + m_page = _mali_mem_swap_page_node_allocate(); + + if (NULL == m_page) { + MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Failed to allocate mali page node.")); + swap_mem->count = i; + + mali_mem_swap_free(swap_mem); + mali_mem_swap_idx_range_free(index + i, page_count - i); + return _MALI_OSK_ERR_FAULT; + } + + m_page->swap_it->idx = index + i; + + ret = mali_mem_swap_in_page_node(m_page); + + if (MALI_FALSE == ret) { + MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Allocate new page from SHMEM file failed.")); + _mali_mem_swap_page_node_free(m_page); + mali_mem_swap_idx_range_free(index + i + 1, page_count - i - 1); + + swap_mem->count = i; + mali_mem_swap_free(swap_mem); + return _MALI_OSK_ERR_NOMEM; + } + + list_add_tail(&m_page->list, &swap_mem->pages); + } + + system_free_size = global_zone_page_state(NR_FREE_PAGES) * PAGE_SIZE; + + if ((system_free_size < mali_mem_swap_out_threshold_value) + && (mem_backend_swapped_pool_size > (mali_mem_swap_out_threshold_value >> 2)) + && mali_utilization_enabled()) { + _mali_osk_wq_schedule_work(mali_mem_swap_out_workq); + } + + *bkend_idx = index; + return 0; +} + +void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc) +{ + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + + +/* Insert these pages from shmem to mali page table*/ +_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props) +{ + struct mali_page_directory *pagedir = session->page_directory; + struct mali_page_node *m_page; + dma_addr_t phys; + u32 virt = vaddr; + u32 prop = props; + + list_for_each_entry(m_page, &swap_mem->pages, list) { + MALI_DEBUG_ASSERT(NULL != m_page->swap_it->page); + phys = m_page->swap_it->dma_addr; + + mali_mmu_pagedir_update(pagedir, virt, phys, MALI_MMU_PAGE_SIZE, prop); + virt += MALI_MMU_PAGE_SIZE; + } + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_in_pages(struct mali_pp_job *job) +{ + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + struct mali_page_node *m_page; + mali_bool swap_in_success = MALI_TRUE; + int i; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + session = mali_pp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(session); + + for (i = 0; i < num_memory_cookies; i++) { + + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + if (NULL == mali_vma_node) { + job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; + swap_in_success = MALI_FALSE; + MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr)); + continue; + } + + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + + if (MALI_MEM_SWAP != mali_alloc->type && + MALI_MEM_COW != mali_alloc->type) { + continue; + } + + /* Get backend memory & Map on GPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + /* We neednot hold backend's lock here, race safe.*/ + if ((MALI_MEM_COW == mem_bkend->type) && + (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + continue; + } + + mutex_lock(&mem_bkend->mutex); + + /* When swap_in_success is MALI_FALSE, it means this job has memory backend that could not be swapped in, + * and it will be aborted in mali scheduler, so here, we just mark those memory cookies which + * should not be swapped out when delete job to invalide */ + if (MALI_FALSE == swap_in_success) { + job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; + mutex_unlock(&mem_bkend->mutex); + continue; + } + + /* Before swap in, checking if this memory backend has been swapped in by the latest flushed jobs. */ + ++mem_bkend->using_count; + + if (1 < mem_bkend->using_count) { + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)); + mutex_unlock(&mem_bkend->mutex); + continue; + } + + if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)) { + mutex_unlock(&mem_bkend->mutex); + continue; + } + + + list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { + if (MALI_FALSE == mali_mem_swap_in_page_node(m_page)) { + /* Don't have enough memory to swap in page, so release pages have already been swapped + * in and then mark this pp job to be fail. */ + mali_mem_swap_unlock_partial_locked_mem_backend(mem_bkend, m_page); + swap_in_success = MALI_FALSE; + break; + } + } + + if (swap_in_success) { +#ifdef MALI_MEM_SWAP_TRACKING + mem_backend_swapped_unlock_size -= mem_bkend->size; +#endif + _mali_osk_mutex_wait(session->memory_lock); + mali_mem_swap_mali_map(&mem_bkend->swap_mem, session, mali_alloc->mali_mapping.addr, mali_alloc->mali_mapping.properties); + _mali_osk_mutex_signal(session->memory_lock); + + /* Remove the unlock flag from mem backend flags, mark this backend has been swapped in. */ + mem_bkend->flags &= ~(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN); + mutex_unlock(&mem_bkend->mutex); + } else { + --mem_bkend->using_count; + /* Marking that this backend is not swapped in, need not to be processed anymore. */ + job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; + mutex_unlock(&mem_bkend->mutex); + } + } + + job->swap_status = swap_in_success ? MALI_SWAP_IN_SUCC : MALI_SWAP_IN_FAIL; + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_out_pages(struct mali_pp_job *job) +{ + u32 num_memory_cookies; + struct mali_session_data *session; + struct mali_vma_node *mali_vma_node = NULL; + mali_mem_allocation *mali_alloc = NULL; + mali_mem_backend *mem_bkend = NULL; + int i; + + MALI_DEBUG_ASSERT_POINTER(job); + + num_memory_cookies = mali_pp_job_num_memory_cookies(job); + session = mali_pp_job_get_session(job); + + MALI_DEBUG_ASSERT_POINTER(session); + + + for (i = 0; i < num_memory_cookies; i++) { + u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); + + if (MALI_SWAP_INVALIDATE_MALI_ADDRESS == mali_addr) { + continue; + } + + mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); + + if (NULL == mali_vma_node) { + MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr)); + continue; + } + + mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); + MALI_DEBUG_ASSERT(NULL != mali_alloc); + + if (MALI_MEM_SWAP != mali_alloc->type && + MALI_MEM_COW != mali_alloc->type) { + continue; + } + + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + /* We neednot hold backend's lock here, race safe.*/ + if ((MALI_MEM_COW == mem_bkend->type) && + (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { + continue; + } + + mutex_lock(&mem_bkend->mutex); + + MALI_DEBUG_ASSERT(0 < mem_bkend->using_count); + + /* Reducing the using_count of mem backend means less pp job are using this memory backend, + * if this count get to zero, it means no pp job is using it now, could put it to swap out list. */ + --mem_bkend->using_count; + + if (0 < mem_bkend->using_count) { + mutex_unlock(&mem_bkend->mutex); + continue; + } + mutex_unlock(&mem_bkend->mutex); + + mali_memory_swap_list_backend_add(mem_bkend); + } + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) +{ + struct mali_page_node *m_page, *found_node = NULL; + struct page *found_page; + mali_mem_swap *swap = NULL; + mali_mem_cow *cow = NULL; + dma_addr_t dma_addr; + u32 i = 0; + + if (MALI_MEM_SWAP == mem_bkend->type) { + swap = &mem_bkend->swap_mem; + list_for_each_entry(m_page, &swap->pages, list) { + if (i == offset) { + found_node = m_page; + break; + } + i++; + } + } else { + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)); + + cow = &mem_bkend->cow_mem; + list_for_each_entry(m_page, &cow->pages, list) { + if (i == offset) { + found_node = m_page; + break; + } + i++; + } + } + + if (NULL == found_node) { + return _MALI_OSK_ERR_FAULT; + } + + found_page = shmem_read_mapping_page(global_swap_space, found_node->swap_it->idx); + + if (!IS_ERR(found_page)) { + lock_page(found_page); + dma_addr = dma_map_page(&mali_platform_device->dev, found_page, + 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + dma_unmap_page(&mali_platform_device->dev, dma_addr, + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + *pagep = found_page; + } else { + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) +{ + struct mali_page_node *m_page, *found_node = NULL, *new_node = NULL; + mali_mem_cow *cow = NULL; + u32 i = 0; + + MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)); + MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)); + MALI_DEBUG_ASSERT(!mali_memory_swap_backend_in_swapped_pool(mem_bkend)); + + cow = &mem_bkend->cow_mem; + list_for_each_entry(m_page, &cow->pages, list) { + if (i == offset) { + found_node = m_page; + break; + } + i++; + } + + if (NULL == found_node) { + return _MALI_OSK_ERR_FAULT; + } + + new_node = _mali_mem_swap_page_node_allocate(); + + if (NULL == new_node) { + return _MALI_OSK_ERR_FAULT; + } + + new_node->swap_it->idx = mali_mem_swap_idx_alloc(); + + if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == new_node->swap_it->idx) { + MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW on demand.\n")); + kfree(new_node->swap_it); + kfree(new_node); + return _MALI_OSK_ERR_FAULT; + } + + if (MALI_FALSE == mali_mem_swap_in_page_node(new_node)) { + _mali_mem_swap_page_node_free(new_node); + return _MALI_OSK_ERR_FAULT; + } + + /* swap in found node for copy in kernel. */ + if (MALI_FALSE == mali_mem_swap_in_page_node(found_node)) { + mali_mem_swap_out_page_node(new_node); + _mali_mem_swap_page_node_free(new_node); + return _MALI_OSK_ERR_FAULT; + } + + _mali_mem_cow_copy_page(found_node, new_node); + + list_replace(&found_node->list, &new_node->list); + + if (1 != _mali_page_node_get_ref_count(found_node)) { + atomic_add(1, &mem_bkend->mali_allocation->session->mali_mem_allocated_pages); + if (atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > mem_bkend->mali_allocation->session->max_mali_mem_allocated_size) { + mem_bkend->mali_allocation->session->max_mali_mem_allocated_size = atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; + } + mem_bkend->cow_mem.change_pages_nr++; + } + + mali_mem_swap_out_page_node(found_node); + _mali_mem_swap_page_node_free(found_node); + + /* When swap in the new page node, we have called dma_map_page for this page.\n */ + dma_unmap_page(&mali_platform_device->dev, new_node->swap_it->dma_addr, + _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); + + lock_page(new_node->swap_it->page); + + *pagep = new_node->swap_it->page; + + return _MALI_OSK_ERR_OK; +} + +#ifdef MALI_MEM_SWAP_TRACKING +void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size) +{ + *swap_pool_size = mem_backend_swapped_pool_size; + *unlock_size = mem_backend_swapped_unlock_size; +} +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h new file mode 100755 index 000000000000..5810960e204a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_SWAP_ALLOC_H__ +#define __MALI_MEMORY_SWAP_ALLOC_H__ + +#include "mali_osk.h" +#include "mali_session.h" + +#include "mali_memory_types.h" +#include "mali_pp_job.h" + +/** + * Initialize memory swapping module. + */ +_mali_osk_errcode_t mali_mem_swap_init(void); + +void mali_mem_swap_term(void); + +/** + * Return global share memory file to other modules. + */ +struct file *mali_mem_swap_get_global_swap_file(void); + +/** + * Unlock the given memory backend and pages in it could be swapped out by kernel. + */ +void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend); + +/** + * Remove the given memory backend from global swap list. + */ +void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend); + +/** + * Add the given memory backend to global swap list. + */ +void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend); + +/** + * Allocate 1 index from bitmap used as page index in global swap file. + */ +u32 mali_mem_swap_idx_alloc(void); + +void mali_mem_swap_idx_free(u32 idx); + +/** + * Allocate a new swap item without page index. + */ +struct mali_swap_item *mali_mem_swap_alloc_swap_item(void); + +/** + * Free a swap item, truncate the corresponding space in page cache and free index of page. + */ +void mali_mem_swap_free_swap_item(mali_swap_item *swap_item); + +/** + * Allocate a page node with swap item. + */ +struct mali_page_node *_mali_mem_swap_page_node_allocate(void); + +/** + * Reduce the reference count of given page node and if return 0, just free this page node. + */ +_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page); + +void _mali_mem_swap_page_node_free(struct mali_page_node *m_page); + +/** + * Free a swappable memory backend. + */ +u32 mali_mem_swap_free(mali_mem_swap *swap_mem); + +/** + * Ummap and free. + */ +u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped); + +/** + * Read in a page from global swap file with the pre-allcated page index. + */ +mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node); + +int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx); + +_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props); + +void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc); + +/** + * When pp job created, we need swap in all of memory backend needed by this pp job. + */ +int mali_mem_swap_in_pages(struct mali_pp_job *job); + +/** + * Put all of memory backends used this pp job to the global swap list. + */ +int mali_mem_swap_out_pages(struct mali_pp_job *job); + +/** + * This will be called in page fault to process CPU read&write. + */ +int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) ; + +/** + * Used to process cow on demand for swappable memory backend. + */ +int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep); + +#ifdef MALI_MEM_SWAP_TRACKING +void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size); +#endif +#endif /* __MALI_MEMORY_SWAP_ALLOC_H__ */ + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h new file mode 100755 index 000000000000..33db40929642 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h @@ -0,0 +1,219 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_TYPES_H__ +#define __MALI_MEMORY_TYPES_H__ + +#include + +#if defined(CONFIG_MALI400_UMP) +#include "ump_kernel_interface.h" +#endif + +typedef u32 mali_address_t; + +typedef enum mali_mem_type { + MALI_MEM_OS, + MALI_MEM_EXTERNAL, + MALI_MEM_SWAP, + MALI_MEM_DMA_BUF, + MALI_MEM_UMP, + MALI_MEM_BLOCK, + MALI_MEM_COW, + MALI_MEM_SECURE, + MALI_MEM_TYPE_MAX, +} mali_mem_type; + +typedef struct mali_block_item { + /* for block type, the block_phy is alway page size align + * so use low 12bit used for ref_cout. + */ + unsigned long phy_addr; +} mali_block_item; + +/** + * idx is used to locate the given page in the address space of swap file. + * ref_count is used to mark how many memory backends are using this item. + */ +typedef struct mali_swap_item { + u32 idx; + atomic_t ref_count; + struct page *page; + dma_addr_t dma_addr; +} mali_swap_item; + +typedef enum mali_page_node_type { + MALI_PAGE_NODE_OS, + MALI_PAGE_NODE_BLOCK, + MALI_PAGE_NODE_SWAP, +} mali_page_node_type; + +typedef struct mali_page_node { + struct list_head list; + union { + struct page *page; + mali_block_item *blk_it; /*pointer to block item*/ + mali_swap_item *swap_it; + }; + + u32 type; +} mali_page_node; + +typedef struct mali_mem_os_mem { + struct list_head pages; + u32 count; +} mali_mem_os_mem; + +typedef struct mali_mem_dma_buf { +#if defined(CONFIG_DMA_SHARED_BUFFER) + struct mali_dma_buf_attachment *attachment; +#endif +} mali_mem_dma_buf; + +typedef struct mali_mem_external { + dma_addr_t phys; + u32 size; +} mali_mem_external; + +typedef struct mali_mem_ump { +#if defined(CONFIG_MALI400_UMP) + ump_dd_handle handle; +#endif +} mali_mem_ump; + +typedef struct block_allocator_allocation { + /* The list will be released in reverse order */ + struct block_info *last_allocated; + u32 mapping_length; + struct block_allocator *info; +} block_allocator_allocation; + +typedef struct mali_mem_block_mem { + struct list_head pfns; + u32 count; +} mali_mem_block_mem; + +typedef struct mali_mem_virt_mali_mapping { + mali_address_t addr; /* Virtual Mali address */ + u32 properties; /* MMU Permissions + cache, must match MMU HW */ +} mali_mem_virt_mali_mapping; + +typedef struct mali_mem_virt_cpu_mapping { + void __user *addr; + struct vm_area_struct *vma; +} mali_mem_virt_cpu_mapping; + +#define MALI_MEM_ALLOCATION_VALID_MAGIC 0xdeda110c +#define MALI_MEM_ALLOCATION_FREED_MAGIC 0x10101010 + +typedef struct mali_mm_node { + /* MALI GPU vaddr start, use u32 for mmu only support 32bit address*/ + uint32_t start; /* GPU vaddr */ + uint32_t size; /* GPU allocation virtual size */ + unsigned allocated : 1; +} mali_mm_node; + +typedef struct mali_vma_node { + struct mali_mm_node vm_node; + struct rb_node vm_rb; +} mali_vma_node; + + +typedef struct mali_mem_allocation { + MALI_DEBUG_CODE(u32 magic); + mali_mem_type type; /**< Type of memory */ + u32 flags; /**< Flags for this allocation */ + + struct mali_session_data *session; /**< Pointer to session that owns the allocation */ + + mali_mem_virt_cpu_mapping cpu_mapping; /**< CPU mapping */ + mali_mem_virt_mali_mapping mali_mapping; /**< Mali mapping */ + + /* add for new memory system */ + struct mali_vma_node mali_vma_node; + u32 vsize; /* virtual size*/ + u32 psize; /* physical backend memory size*/ + struct list_head list; + s32 backend_handle; /* idr for mem_backend */ + _mali_osk_atomic_t mem_alloc_refcount; +} mali_mem_allocation; + +struct mali_mem_os_allocator { + spinlock_t pool_lock; + struct list_head pool_pages; + size_t pool_count; + + atomic_t allocated_pages; + size_t allocation_limit; + + struct shrinker shrinker; + struct delayed_work timed_shrinker; + struct workqueue_struct *wq; +}; + +/* COW backend memory type */ +typedef struct mali_mem_cow { + struct list_head pages; /**< all pages for this cow backend allocation, + including new allocated pages for modified range*/ + u32 count; /**< number of pages */ + s32 change_pages_nr; +} mali_mem_cow; + +typedef struct mali_mem_swap { + struct list_head pages; + u32 count; +} mali_mem_swap; + +typedef struct mali_mem_secure { +#if defined(CONFIG_DMA_SHARED_BUFFER) + struct dma_buf *buf; + struct dma_buf_attachment *attachment; + struct sg_table *sgt; +#endif + u32 count; +} mali_mem_secure; + +#define MALI_MEM_BACKEND_FLAG_COWED (0x1) /* COW has happen on this backend */ +#define MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE (0x2) /* This is an COW backend, mapped as not allowed cpu to write */ +#define MALI_MEM_BACKEND_FLAG_SWAP_COWED (0x4) /* Mark the given backend is cowed from swappable memory. */ +/* Mark this backend is not swapped_in in MALI driver, and before using it, + * we should swap it in and set up corresponding page table. */ +#define MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN (0x8) +#define MALI_MEM_BACKEND_FLAG_NOT_BINDED (0x1 << 5) /* this backend it not back with physical memory, used for defer bind */ +#define MALI_MEM_BACKEND_FLAG_BINDED (0x1 << 6) /* this backend it back with physical memory, used for defer bind */ + +typedef struct mali_mem_backend { + mali_mem_type type; /**< Type of backend memory */ + u32 flags; /**< Flags for this allocation */ + u32 size; + /* Union selected by type. */ + union { + mali_mem_os_mem os_mem; /**< MALI_MEM_OS */ + mali_mem_external ext_mem; /**< MALI_MEM_EXTERNAL */ + mali_mem_dma_buf dma_buf; /**< MALI_MEM_DMA_BUF */ + mali_mem_ump ump_mem; /**< MALI_MEM_UMP */ + mali_mem_block_mem block_mem; /**< MALI_MEM_BLOCK */ + mali_mem_cow cow_mem; + mali_mem_swap swap_mem; + mali_mem_secure secure_mem; + }; + mali_mem_allocation *mali_allocation; + struct mutex mutex; + mali_mem_type cow_type; + + struct list_head list; /**< Used to link swappable memory backend to the global swappable list */ + int using_count; /**< Mark how many PP jobs are using this memory backend */ + u32 start_idx; /**< If the correspondign vma of this backend is linear, this value will be used to set vma->vm_pgoff */ +} mali_mem_backend; + +#define MALI_MEM_FLAG_MALI_GUARD_PAGE (_MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) +#define MALI_MEM_FLAG_DONT_CPU_MAP (1 << 1) +#define MALI_MEM_FLAG_CAN_RESIZE (_MALI_MEMORY_ALLOCATE_RESIZEABLE) +#endif /* __MALI_MEMORY_TYPES__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c new file mode 100755 index 000000000000..666d4b0fb1cd --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_kernel_linux.h" +#include "mali_memory.h" +#include "ump_kernel_interface.h" + +static int mali_mem_ump_map(mali_mem_backend *mem_backend) +{ + ump_dd_handle ump_mem; + mali_mem_allocation *alloc; + struct mali_session_data *session; + u32 nr_blocks; + u32 i; + ump_dd_physical_block *ump_blocks; + struct mali_page_directory *pagedir; + u32 offset = 0; + _mali_osk_errcode_t err; + + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); + + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + + ump_mem = mem_backend->ump_mem.handle; + MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); + + nr_blocks = ump_dd_phys_block_count_get(ump_mem); + if (nr_blocks == 0) { + MALI_DEBUG_PRINT(1, ("No block count\n")); + return -EINVAL; + } + + ump_blocks = _mali_osk_malloc(sizeof(*ump_blocks) * nr_blocks); + if (NULL == ump_blocks) { + return -ENOMEM; + } + + if (UMP_DD_INVALID == ump_dd_phys_blocks_get(ump_mem, ump_blocks, nr_blocks)) { + _mali_osk_free(ump_blocks); + return -EFAULT; + } + + pagedir = session->page_directory; + + mali_session_memory_lock(session); + + err = mali_mem_mali_map_prepare(alloc); + if (_MALI_OSK_ERR_OK != err) { + MALI_DEBUG_PRINT(1, ("Mapping of UMP memory failed\n")); + + _mali_osk_free(ump_blocks); + mali_session_memory_unlock(session); + return -ENOMEM; + } + + for (i = 0; i < nr_blocks; ++i) { + u32 virt = alloc->mali_vma_node.vm_node.start + offset; + + MALI_DEBUG_PRINT(7, ("Mapping in 0x%08x size %d\n", ump_blocks[i].addr , ump_blocks[i].size)); + + mali_mmu_pagedir_update(pagedir, virt, ump_blocks[i].addr, + ump_blocks[i].size, MALI_MMU_FLAGS_DEFAULT); + + offset += ump_blocks[i].size; + } + + if (alloc->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + u32 virt = alloc->mali_vma_node.vm_node.start + offset; + + /* Map in an extra virtual guard page at the end of the VMA */ + MALI_DEBUG_PRINT(6, ("Mapping in extra guard page\n")); + + mali_mmu_pagedir_update(pagedir, virt, ump_blocks[0].addr, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); + + offset += _MALI_OSK_MALI_PAGE_SIZE; + } + mali_session_memory_unlock(session); + _mali_osk_free(ump_blocks); + return 0; +} + +static void mali_mem_ump_unmap(mali_mem_allocation *alloc) +{ + struct mali_session_data *session; + MALI_DEBUG_ASSERT_POINTER(alloc); + session = alloc->session; + MALI_DEBUG_ASSERT_POINTER(session); + mali_session_memory_lock(session); + mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, + alloc->flags); + mali_session_memory_unlock(session); +} + +int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags) +{ + ump_dd_handle ump_mem; + int ret; + MALI_DEBUG_ASSERT_POINTER(alloc); + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); + + MALI_DEBUG_PRINT(3, + ("Requested to map ump memory with secure id %d into virtual memory 0x%08X, size 0x%08X\n", + secure_id, alloc->mali_vma_node.vm_node.start, alloc->mali_vma_node.vm_node.size)); + + ump_mem = ump_dd_handle_create_from_secure_id(secure_id); + if (UMP_DD_HANDLE_INVALID == ump_mem) MALI_ERROR(_MALI_OSK_ERR_FAULT); + alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; + if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { + alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; + } + + mem_backend->ump_mem.handle = ump_mem; + + ret = mali_mem_ump_map(mem_backend); + if (0 != ret) { + ump_dd_reference_release(ump_mem); + return _MALI_OSK_ERR_FAULT; + } + MALI_DEBUG_PRINT(3, ("Returning from UMP bind\n")); + return _MALI_OSK_ERR_OK; +} + +void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend) +{ + ump_dd_handle ump_mem; + mali_mem_allocation *alloc; + MALI_DEBUG_ASSERT_POINTER(mem_backend); + MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); + ump_mem = mem_backend->ump_mem.handle; + MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); + + alloc = mem_backend->mali_allocation; + MALI_DEBUG_ASSERT_POINTER(alloc); + mali_mem_ump_unmap(alloc); + ump_dd_reference_release(ump_mem); +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h new file mode 100755 index 000000000000..c314c8dcbf1c --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_UMP_BUF_H__ +#define __MALI_MEMORY_UMP_BUF_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "mali_uk_types.h" +#include "mali_osk.h" +#include "mali_memory.h" + +int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags); +void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_MEMORY_DMA_BUF_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c new file mode 100755 index 000000000000..8e13e923c3fb --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_linux.h" +#include "mali_scheduler.h" + +#include "mali_memory.h" +#include "mali_memory_os_alloc.h" +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include "mali_memory_dma_buf.h" +#include "mali_memory_secure.h" +#endif +#if defined(CONFIG_MALI400_UMP) +#include "mali_memory_ump.h" +#endif +#include "mali_memory_external.h" +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" +#include "mali_memory_cow.h" +#include "mali_memory_block_alloc.h" +#include "mali_memory_swap_alloc.h" + + + +/** +*function @_mali_free_allocation_mem - free a memory allocation +*/ +static u32 _mali_free_allocation_mem(mali_mem_allocation *mali_alloc) +{ + mali_mem_backend *mem_bkend = NULL; + u32 free_pages_nr = 0; + + struct mali_session_data *session = mali_alloc->session; + MALI_DEBUG_PRINT(4, (" _mali_free_allocation_mem, psize =0x%x! \n", mali_alloc->psize)); + if (0 == mali_alloc->psize) + goto out; + + /* Get backend memory & Map on CPU */ + mutex_lock(&mali_idr_mutex); + mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + MALI_DEBUG_ASSERT(NULL != mem_bkend); + + switch (mem_bkend->type) { + case MALI_MEM_OS: + free_pages_nr = mali_mem_os_release(mem_bkend); + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + break; + case MALI_MEM_UMP: +#if defined(CONFIG_MALI400_UMP) + mali_mem_unbind_ump_buf(mem_bkend); + atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); +#else + MALI_DEBUG_PRINT(1, ("UMP not supported\n")); +#endif + break; + case MALI_MEM_DMA_BUF: +#if defined(CONFIG_DMA_SHARED_BUFFER) + mali_mem_unbind_dma_buf(mem_bkend); + atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); +#else + MALI_DEBUG_PRINT(1, ("DMA not supported\n")); +#endif + break; + case MALI_MEM_EXTERNAL: + mali_mem_unbind_ext_buf(mem_bkend); + atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); + break; + + case MALI_MEM_BLOCK: + free_pages_nr = mali_mem_block_release(mem_bkend); + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + break; + + case MALI_MEM_COW: + if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) { + free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE); + } else { + free_pages_nr = mali_mem_cow_release(mem_bkend, MALI_TRUE); + } + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + break; + case MALI_MEM_SWAP: + free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE); + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); + atomic_sub(free_pages_nr, &session->mali_mem_array[mem_bkend->type]); + break; + case MALI_MEM_SECURE: +#if defined(CONFIG_DMA_SHARED_BUFFER) + free_pages_nr = mali_mem_secure_release(mem_bkend); + atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); +#else + MALI_DEBUG_PRINT(1, ("DMA not supported for mali secure memory\n")); +#endif + break; + default: + MALI_DEBUG_PRINT(1, ("mem type %d is not in the mali_mem_type enum.\n", mem_bkend->type)); + break; + } + + /*Remove backend memory idex */ + mutex_lock(&mali_idr_mutex); + idr_remove(&mali_backend_idr, mali_alloc->backend_handle); + mutex_unlock(&mali_idr_mutex); + kfree(mem_bkend); +out: + /* remove memory allocation */ + mali_vma_offset_remove(&session->allocation_mgr, &mali_alloc->mali_vma_node); + mali_mem_allocation_struct_destory(mali_alloc); + return free_pages_nr; +} + +/** +* ref_count for allocation +*/ +u32 mali_allocation_unref(struct mali_mem_allocation **alloc) +{ + u32 free_pages_nr = 0; + mali_mem_allocation *mali_alloc = *alloc; + *alloc = NULL; + if (0 == _mali_osk_atomic_dec_return(&mali_alloc->mem_alloc_refcount)) { + free_pages_nr = _mali_free_allocation_mem(mali_alloc); + } + return free_pages_nr; +} + +void mali_allocation_ref(struct mali_mem_allocation *alloc) +{ + _mali_osk_atomic_inc(&alloc->mem_alloc_refcount); +} + +void mali_free_session_allocations(struct mali_session_data *session) +{ + struct mali_mem_allocation *entry, *next; + + MALI_DEBUG_PRINT(4, (" mali_free_session_allocations! \n")); + + list_for_each_entry_safe(entry, next, &session->allocation_mgr.head, list) { + mali_allocation_unref(&entry); + } +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h new file mode 100755 index 000000000000..33ac99509740 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_MEMORY_UTIL_H__ +#define __MALI_MEMORY_UTIL_H__ + +u32 mali_allocation_unref(struct mali_mem_allocation **alloc); + +void mali_allocation_ref(struct mali_mem_allocation *alloc); + +void mali_free_session_allocations(struct mali_session_data *session); + +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c new file mode 100755 index 000000000000..0b31e3a23432 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_kernel_linux.h" +#include "mali_scheduler.h" +#include "mali_memory_os_alloc.h" +#include "mali_memory_manager.h" +#include "mali_memory_virtual.h" + + +/** +*internal helper to link node into the rb-tree +*/ +static inline void _mali_vma_offset_add_rb(struct mali_allocation_manager *mgr, + struct mali_vma_node *node) +{ + struct rb_node **iter = &mgr->allocation_mgr_rb.rb_node; + struct rb_node *parent = NULL; + struct mali_vma_node *iter_node; + + while (likely(*iter)) { + parent = *iter; + iter_node = rb_entry(*iter, struct mali_vma_node, vm_rb); + + if (node->vm_node.start < iter_node->vm_node.start) + iter = &(*iter)->rb_left; + else if (node->vm_node.start > iter_node->vm_node.start) + iter = &(*iter)->rb_right; + else + MALI_DEBUG_ASSERT(0); + } + + rb_link_node(&node->vm_rb, parent, iter); + rb_insert_color(&node->vm_rb, &mgr->allocation_mgr_rb); +} + +/** + * mali_vma_offset_add() - Add offset node to RB Tree + */ +int mali_vma_offset_add(struct mali_allocation_manager *mgr, + struct mali_vma_node *node) +{ + int ret = 0; + write_lock(&mgr->vm_lock); + + if (node->vm_node.allocated) { + goto out; + } + + _mali_vma_offset_add_rb(mgr, node); + /* set to allocated */ + node->vm_node.allocated = 1; + +out: + write_unlock(&mgr->vm_lock); + return ret; +} + +/** + * mali_vma_offset_remove() - Remove offset node from RB tree + */ +void mali_vma_offset_remove(struct mali_allocation_manager *mgr, + struct mali_vma_node *node) +{ + write_lock(&mgr->vm_lock); + + if (node->vm_node.allocated) { + rb_erase(&node->vm_rb, &mgr->allocation_mgr_rb); + memset(&node->vm_node, 0, sizeof(node->vm_node)); + } + write_unlock(&mgr->vm_lock); +} + +/** +* mali_vma_offset_search - Search the node in RB tree +*/ +struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr, + unsigned long start, unsigned long pages) +{ + struct mali_vma_node *node, *best; + struct rb_node *iter; + unsigned long offset; + read_lock(&mgr->vm_lock); + + iter = mgr->allocation_mgr_rb.rb_node; + best = NULL; + + while (likely(iter)) { + node = rb_entry(iter, struct mali_vma_node, vm_rb); + offset = node->vm_node.start; + if (start >= offset) { + iter = iter->rb_right; + best = node; + if (start == offset) + break; + } else { + iter = iter->rb_left; + } + } + + if (best) { + offset = best->vm_node.start + best->vm_node.size; + if (offset <= start + pages) + best = NULL; + } + read_unlock(&mgr->vm_lock); + + return best; +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h new file mode 100755 index 000000000000..fd03ed9f2bbb --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#ifndef __MALI_GPU_VMEM_H__ +#define __MALI_GPU_VMEM_H__ + +#include "mali_osk.h" +#include "mali_session.h" +#include +#include +#include +#include +#include +#include "mali_memory_types.h" +#include "mali_memory_os_alloc.h" +#include "mali_memory_manager.h" + + + +int mali_vma_offset_add(struct mali_allocation_manager *mgr, + struct mali_vma_node *node); + +void mali_vma_offset_remove(struct mali_allocation_manager *mgr, + struct mali_vma_node *node); + +struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr, + unsigned long start, unsigned long pages); + +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c new file mode 100755 index 000000000000..5bc0e52ebe23 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_atomics.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk.h" +#include +#include "mali_kernel_common.h" + +void _mali_osk_atomic_dec(_mali_osk_atomic_t *atom) +{ + atomic_dec((atomic_t *)&atom->u.val); +} + +u32 _mali_osk_atomic_dec_return(_mali_osk_atomic_t *atom) +{ + return atomic_dec_return((atomic_t *)&atom->u.val); +} + +void _mali_osk_atomic_inc(_mali_osk_atomic_t *atom) +{ + atomic_inc((atomic_t *)&atom->u.val); +} + +u32 _mali_osk_atomic_inc_return(_mali_osk_atomic_t *atom) +{ + return atomic_inc_return((atomic_t *)&atom->u.val); +} + +void _mali_osk_atomic_init(_mali_osk_atomic_t *atom, u32 val) +{ + MALI_DEBUG_ASSERT_POINTER(atom); + atomic_set((atomic_t *)&atom->u.val, val); +} + +u32 _mali_osk_atomic_read(_mali_osk_atomic_t *atom) +{ + return atomic_read((atomic_t *)&atom->u.val); +} + +void _mali_osk_atomic_term(_mali_osk_atomic_t *atom) +{ + MALI_IGNORE(atom); +} + +u32 _mali_osk_atomic_xchg(_mali_osk_atomic_t *atom, u32 val) +{ + return atomic_xchg((atomic_t *)&atom->u.val, val); +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c new file mode 100755 index 000000000000..fb9ccd2ad1e2 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2010, 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_bitmap.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include +#include +#include +#include +#include +#include "common/mali_kernel_common.h" +#include "mali_osk_types.h" +#include "mali_osk.h" + +u32 _mali_osk_bitmap_alloc(struct _mali_osk_bitmap *bitmap) +{ + u32 obj; + + MALI_DEBUG_ASSERT_POINTER(bitmap); + + _mali_osk_spinlock_lock(bitmap->lock); + + obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->reserve); + + if (obj < bitmap->max) { + set_bit(obj, bitmap->table); + } else { + obj = -1; + } + + if (obj != -1) + --bitmap->avail; + _mali_osk_spinlock_unlock(bitmap->lock); + + return obj; +} + +void _mali_osk_bitmap_free(struct _mali_osk_bitmap *bitmap, u32 obj) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + _mali_osk_bitmap_free_range(bitmap, obj, 1); +} + +u32 _mali_osk_bitmap_alloc_range(struct _mali_osk_bitmap *bitmap, int cnt) +{ + u32 obj; + + MALI_DEBUG_ASSERT_POINTER(bitmap); + + if (0 >= cnt) { + return -1; + } + + if (1 == cnt) { + return _mali_osk_bitmap_alloc(bitmap); + } + + _mali_osk_spinlock_lock(bitmap->lock); + obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, + bitmap->last, cnt, 0); + + if (obj >= bitmap->max) { + obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, + bitmap->reserve, cnt, 0); + } + + if (obj < bitmap->max) { + bitmap_set(bitmap->table, obj, cnt); + + bitmap->last = (obj + cnt); + if (bitmap->last >= bitmap->max) { + bitmap->last = bitmap->reserve; + } + } else { + obj = -1; + } + + if (obj != -1) { + bitmap->avail -= cnt; + } + + _mali_osk_spinlock_unlock(bitmap->lock); + + return obj; +} + +u32 _mali_osk_bitmap_avail(struct _mali_osk_bitmap *bitmap) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + return bitmap->avail; +} + +void _mali_osk_bitmap_free_range(struct _mali_osk_bitmap *bitmap, u32 obj, int cnt) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + _mali_osk_spinlock_lock(bitmap->lock); + bitmap_clear(bitmap->table, obj, cnt); + bitmap->last = min(bitmap->last, obj); + + bitmap->avail += cnt; + _mali_osk_spinlock_unlock(bitmap->lock); +} + +int _mali_osk_bitmap_init(struct _mali_osk_bitmap *bitmap, u32 num, u32 reserve) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + MALI_DEBUG_ASSERT(reserve <= num); + + bitmap->reserve = reserve; + bitmap->last = reserve; + bitmap->max = num; + bitmap->avail = num - reserve; + bitmap->lock = _mali_osk_spinlock_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST); + if (!bitmap->lock) { + return _MALI_OSK_ERR_NOMEM; + } + bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * + sizeof(long), GFP_KERNEL); + if (!bitmap->table) { + _mali_osk_spinlock_term(bitmap->lock); + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void _mali_osk_bitmap_term(struct _mali_osk_bitmap *bitmap) +{ + MALI_DEBUG_ASSERT_POINTER(bitmap); + + if (NULL != bitmap->lock) { + _mali_osk_spinlock_term(bitmap->lock); + } + + if (NULL != bitmap->table) { + kfree(bitmap->table); + } +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c new file mode 100755 index 000000000000..5c8b9ceab9ab --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_irq.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include /* For memory allocation */ +#include +#include +#include + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +typedef struct _mali_osk_irq_t_struct { + u32 irqnum; + void *data; + _mali_osk_irq_uhandler_t uhandler; +} mali_osk_irq_object_t; + +typedef irqreturn_t (*irq_handler_func_t)(int, void *, struct pt_regs *); +static irqreturn_t irq_handler_upper_half(int port_name, void *dev_id); /* , struct pt_regs *regs*/ + +#if defined(DEBUG) + +struct test_interrupt_data { + _mali_osk_irq_ack_t ack_func; + void *probe_data; + mali_bool interrupt_received; + wait_queue_head_t wq; +}; + +static irqreturn_t test_interrupt_upper_half(int port_name, void *dev_id) +{ + irqreturn_t ret = IRQ_NONE; + struct test_interrupt_data *data = (struct test_interrupt_data *)dev_id; + + if (_MALI_OSK_ERR_OK == data->ack_func(data->probe_data)) { + data->interrupt_received = MALI_TRUE; + wake_up(&data->wq); + ret = IRQ_HANDLED; + } + + return ret; +} + +static _mali_osk_errcode_t test_interrupt(u32 irqnum, + _mali_osk_irq_trigger_t trigger_func, + _mali_osk_irq_ack_t ack_func, + void *probe_data, + const char *description) +{ + unsigned long irq_flags = 0; + struct test_interrupt_data data = { + .ack_func = ack_func, + .probe_data = probe_data, + .interrupt_received = MALI_FALSE, + }; + +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + irq_flags |= IRQF_SHARED; +#endif /* defined(CONFIG_MALI_SHARED_INTERRUPTS) */ + + if (0 != request_irq(irqnum, test_interrupt_upper_half, irq_flags, description, &data)) { + MALI_DEBUG_PRINT(2, ("Unable to install test IRQ handler for core '%s'\n", description)); + return _MALI_OSK_ERR_FAULT; + } + + init_waitqueue_head(&data.wq); + + trigger_func(probe_data); + wait_event_timeout(data.wq, data.interrupt_received, 100); + + free_irq(irqnum, &data); + + if (data.interrupt_received) { + MALI_DEBUG_PRINT(3, ("%s: Interrupt test OK\n", description)); + return _MALI_OSK_ERR_OK; + } else { + MALI_PRINT_ERROR(("%s: Failed interrupt test on %u\n", description, irqnum)); + return _MALI_OSK_ERR_FAULT; + } +} + +#endif /* defined(DEBUG) */ + +_mali_osk_irq_t *_mali_osk_irq_init(u32 irqnum, _mali_osk_irq_uhandler_t uhandler, void *int_data, _mali_osk_irq_trigger_t trigger_func, _mali_osk_irq_ack_t ack_func, void *probe_data, const char *description) +{ + mali_osk_irq_object_t *irq_object; + unsigned long irq_flags = 0; + +#if defined(CONFIG_MALI_SHARED_INTERRUPTS) + irq_flags |= IRQF_SHARED; +#endif /* defined(CONFIG_MALI_SHARED_INTERRUPTS) */ + + irq_object = kmalloc(sizeof(mali_osk_irq_object_t), GFP_KERNEL); + if (NULL == irq_object) { + return NULL; + } + + if (-1 == irqnum) { + /* Probe for IRQ */ + if ((NULL != trigger_func) && (NULL != ack_func)) { + unsigned long probe_count = 3; + _mali_osk_errcode_t err; + int irq; + + MALI_DEBUG_PRINT(2, ("Probing for irq\n")); + + do { + unsigned long mask; + + mask = probe_irq_on(); + trigger_func(probe_data); + + _mali_osk_time_ubusydelay(5); + + irq = probe_irq_off(mask); + err = ack_func(probe_data); + } while (irq < 0 && (err == _MALI_OSK_ERR_OK) && probe_count--); + + if (irq < 0 || (_MALI_OSK_ERR_OK != err)) irqnum = -1; + else irqnum = irq; + } else irqnum = -1; /* no probe functions, fault */ + + if (-1 != irqnum) { + /* found an irq */ + MALI_DEBUG_PRINT(2, ("Found irq %d\n", irqnum)); + } else { + MALI_DEBUG_PRINT(2, ("Probe for irq failed\n")); + } + } + + irq_object->irqnum = irqnum; + irq_object->uhandler = uhandler; + irq_object->data = int_data; + + if (-1 == irqnum) { + MALI_DEBUG_PRINT(2, ("No IRQ for core '%s' found during probe\n", description)); + kfree(irq_object); + return NULL; + } + +#if defined(DEBUG) + /* Verify that the configured interrupt settings are working */ + if (_MALI_OSK_ERR_OK != test_interrupt(irqnum, trigger_func, ack_func, probe_data, description)) { + MALI_DEBUG_PRINT(2, ("Test of IRQ(%d) handler for core '%s' failed\n", irqnum, description)); + kfree(irq_object); + return NULL; + } +#endif + + if (0 != request_irq(irqnum, irq_handler_upper_half, irq_flags, description, irq_object)) { + MALI_DEBUG_PRINT(2, ("Unable to install IRQ handler for core '%s'\n", description)); + kfree(irq_object); + return NULL; + } + + return irq_object; +} + +void _mali_osk_irq_term(_mali_osk_irq_t *irq) +{ + mali_osk_irq_object_t *irq_object = (mali_osk_irq_object_t *)irq; + free_irq(irq_object->irqnum, irq_object); + kfree(irq_object); +} + + +/** This function is called directly in interrupt context from the OS just after + * the CPU get the hw-irq from mali, or other devices on the same IRQ-channel. + * It is registered one of these function for each mali core. When an interrupt + * arrives this function will be called equal times as registered mali cores. + * That means that we only check one mali core in one function call, and the + * core we check for each turn is given by the \a dev_id variable. + * If we detect an pending interrupt on the given core, we mask the interrupt + * out by settging the core's IRQ_MASK register to zero. + * Then we schedule the mali_core_irq_handler_bottom_half to run as high priority + * work queue job. + */ +static irqreturn_t irq_handler_upper_half(int port_name, void *dev_id) /* , struct pt_regs *regs*/ +{ + irqreturn_t ret = IRQ_NONE; + mali_osk_irq_object_t *irq_object = (mali_osk_irq_object_t *)dev_id; + + if (_MALI_OSK_ERR_OK == irq_object->uhandler(irq_object->data)) { + ret = IRQ_HANDLED; + } + + return ret; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c new file mode 100755 index 000000000000..ed5f0b0da7cb --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c @@ -0,0 +1,287 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_locks.c + * Implemenation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk_locks.h" +#include "mali_kernel_common.h" +#include "mali_osk.h" + + +#ifdef DEBUG +#ifdef LOCK_ORDER_CHECKING +static DEFINE_SPINLOCK(lock_tracking_lock); +static mali_bool add_lock_to_log_and_check(struct _mali_osk_lock_debug_s *lock, uint32_t tid); +static void remove_lock_from_log(struct _mali_osk_lock_debug_s *lock, uint32_t tid); +static const char *const lock_order_to_string(_mali_osk_lock_order_t order); +#endif /* LOCK_ORDER_CHECKING */ + +void _mali_osk_locks_debug_init(struct _mali_osk_lock_debug_s *checker, _mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) +{ + checker->orig_flags = flags; + checker->owner = 0; + +#ifdef LOCK_ORDER_CHECKING + checker->order = order; + checker->next = NULL; +#endif +} + +void _mali_osk_locks_debug_add(struct _mali_osk_lock_debug_s *checker) +{ + checker->owner = _mali_osk_get_tid(); + +#ifdef LOCK_ORDER_CHECKING + if (!(checker->orig_flags & _MALI_OSK_LOCKFLAG_UNORDERED)) { + if (!add_lock_to_log_and_check(checker, _mali_osk_get_tid())) { + printk(KERN_ERR "%d: ERROR lock %p taken while holding a lock of a higher order.\n", + _mali_osk_get_tid(), checker); + dump_stack(); + } + } +#endif +} + +void _mali_osk_locks_debug_remove(struct _mali_osk_lock_debug_s *checker) +{ + +#ifdef LOCK_ORDER_CHECKING + if (!(checker->orig_flags & _MALI_OSK_LOCKFLAG_UNORDERED)) { + remove_lock_from_log(checker, _mali_osk_get_tid()); + } +#endif + checker->owner = 0; +} + + +#ifdef LOCK_ORDER_CHECKING +/* Lock order checking + * ------------------- + * + * To assure that lock ordering scheme defined by _mali_osk_lock_order_t is strictly adhered to, the + * following function will, together with a linked list and some extra members in _mali_osk_lock_debug_s, + * make sure that a lock that is taken has a higher order than the current highest-order lock a + * thread holds. + * + * This is done in the following manner: + * - A linked list keeps track of locks held by a thread. + * - A `next' pointer is added to each lock. This is used to chain the locks together. + * - When taking a lock, the `add_lock_to_log_and_check' makes sure that taking + * the given lock is legal. It will follow the linked list to find the last + * lock taken by this thread. If the last lock's order was lower than the + * lock that is to be taken, it appends the new lock to the list and returns + * true, if not, it return false. This return value is assert()'ed on in + * _mali_osk_lock_wait(). + */ + +static struct _mali_osk_lock_debug_s *lock_lookup_list; + +static void dump_lock_tracking_list(void) +{ + struct _mali_osk_lock_debug_s *l; + u32 n = 1; + + /* print list for debugging purposes */ + l = lock_lookup_list; + + while (NULL != l) { + printk(" [lock: %p, tid_owner: %d, order: %d] ->", l, l->owner, l->order); + l = l->next; + MALI_DEBUG_ASSERT(n++ < 100); + } + printk(" NULL\n"); +} + +static int tracking_list_length(void) +{ + struct _mali_osk_lock_debug_s *l; + u32 n = 0; + l = lock_lookup_list; + + while (NULL != l) { + l = l->next; + n++; + MALI_DEBUG_ASSERT(n < 100); + } + return n; +} + +static mali_bool add_lock_to_log_and_check(struct _mali_osk_lock_debug_s *lock, uint32_t tid) +{ + mali_bool ret = MALI_FALSE; + _mali_osk_lock_order_t highest_order_for_tid = _MALI_OSK_LOCK_ORDER_FIRST; + struct _mali_osk_lock_debug_s *highest_order_lock = (struct _mali_osk_lock_debug_s *)0xbeefbabe; + struct _mali_osk_lock_debug_s *l; + unsigned long local_lock_flag; + u32 len; + + spin_lock_irqsave(&lock_tracking_lock, local_lock_flag); + len = tracking_list_length(); + + l = lock_lookup_list; + if (NULL == l) { /* This is the first lock taken by this thread -- record and return true */ + lock_lookup_list = lock; + spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); + return MALI_TRUE; + } else { + /* Traverse the locks taken and find the lock of the highest order. + * Since several threads may hold locks, each lock's owner must be + * checked so that locks not owned by this thread can be ignored. */ + for (;;) { + MALI_DEBUG_ASSERT_POINTER(l); + if (tid == l->owner && l->order >= highest_order_for_tid) { + highest_order_for_tid = l->order; + highest_order_lock = l; + } + + if (NULL != l->next) { + l = l->next; + } else { + break; + } + } + + l->next = lock; + l->next = NULL; + } + + /* We have now found the highest order lock currently held by this thread and can see if it is + * legal to take the requested lock. */ + ret = highest_order_for_tid < lock->order; + + if (!ret) { + printk(KERN_ERR "Took lock of order %d (%s) while holding lock of order %d (%s)\n", + lock->order, lock_order_to_string(lock->order), + highest_order_for_tid, lock_order_to_string(highest_order_for_tid)); + dump_lock_tracking_list(); + } + + if (len + 1 != tracking_list_length()) { + printk(KERN_ERR "************ lock: %p\n", lock); + printk(KERN_ERR "************ before: %d *** after: %d ****\n", len, tracking_list_length()); + dump_lock_tracking_list(); + MALI_DEBUG_ASSERT_POINTER(NULL); + } + + spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); + return ret; +} + +static void remove_lock_from_log(struct _mali_osk_lock_debug_s *lock, uint32_t tid) +{ + struct _mali_osk_lock_debug_s *curr; + struct _mali_osk_lock_debug_s *prev = NULL; + unsigned long local_lock_flag; + u32 len; + u32 n = 0; + + spin_lock_irqsave(&lock_tracking_lock, local_lock_flag); + len = tracking_list_length(); + curr = lock_lookup_list; + + if (NULL == curr) { + printk(KERN_ERR "Error: Lock tracking list was empty on call to remove_lock_from_log\n"); + dump_lock_tracking_list(); + } + + MALI_DEBUG_ASSERT_POINTER(curr); + + + while (lock != curr) { + prev = curr; + + MALI_DEBUG_ASSERT_POINTER(curr); + curr = curr->next; + MALI_DEBUG_ASSERT(n++ < 100); + } + + if (NULL == prev) { + lock_lookup_list = curr->next; + } else { + MALI_DEBUG_ASSERT_POINTER(curr); + MALI_DEBUG_ASSERT_POINTER(prev); + prev->next = curr->next; + } + + lock->next = NULL; + + if (len - 1 != tracking_list_length()) { + printk(KERN_ERR "************ lock: %p\n", lock); + printk(KERN_ERR "************ before: %d *** after: %d ****\n", len, tracking_list_length()); + dump_lock_tracking_list(); + MALI_DEBUG_ASSERT_POINTER(NULL); + } + + spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); +} + +static const char *const lock_order_to_string(_mali_osk_lock_order_t order) +{ + switch (order) { + case _MALI_OSK_LOCK_ORDER_SESSIONS: + return "_MALI_OSK_LOCK_ORDER_SESSIONS"; + break; + case _MALI_OSK_LOCK_ORDER_MEM_SESSION: + return "_MALI_OSK_LOCK_ORDER_MEM_SESSION"; + break; + case _MALI_OSK_LOCK_ORDER_MEM_INFO: + return "_MALI_OSK_LOCK_ORDER_MEM_INFO"; + break; + case _MALI_OSK_LOCK_ORDER_MEM_PT_CACHE: + return "_MALI_OSK_LOCK_ORDER_MEM_PT_CACHE"; + break; + case _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP: + return "_MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP"; + break; + case _MALI_OSK_LOCK_ORDER_PM_EXECUTION: + return "_MALI_OSK_LOCK_ORDER_PM_EXECUTION"; + break; + case _MALI_OSK_LOCK_ORDER_EXECUTOR: + return "_MALI_OSK_LOCK_ORDER_EXECUTOR"; + break; + case _MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM: + return "_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM"; + break; + case _MALI_OSK_LOCK_ORDER_SCHEDULER: + return "_MALI_OSK_LOCK_ORDER_SCHEDULER"; + break; + case _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED: + return "_MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED"; + break; + case _MALI_OSK_LOCK_ORDER_DMA_COMMAND: + return "_MALI_OSK_LOCK_ORDER_DMA_COMMAND"; + break; + case _MALI_OSK_LOCK_ORDER_PROFILING: + return "_MALI_OSK_LOCK_ORDER_PROFILING"; + break; + case _MALI_OSK_LOCK_ORDER_L2: + return "_MALI_OSK_LOCK_ORDER_L2"; + break; + case _MALI_OSK_LOCK_ORDER_L2_COMMAND: + return "_MALI_OSK_LOCK_ORDER_L2_COMMAND"; + break; + case _MALI_OSK_LOCK_ORDER_UTILIZATION: + return "_MALI_OSK_LOCK_ORDER_UTILIZATION"; + break; + case _MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS: + return "_MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS"; + break; + case _MALI_OSK_LOCK_ORDER_PM_STATE: + return "_MALI_OSK_LOCK_ORDER_PM_STATE"; + break; + default: + return ""; + } +} +#endif /* LOCK_ORDER_CHECKING */ +#endif /* DEBUG */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h new file mode 100755 index 000000000000..6fd5af95285b --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h @@ -0,0 +1,326 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_locks.h + * Defines OS abstraction of lock and mutex + */ +#ifndef _MALI_OSK_LOCKS_H +#define _MALI_OSK_LOCKS_H + +#include +#include +#include + +#include + +#include "mali_osk_types.h" + +#ifdef _cplusplus +extern "C" { +#endif + + /* When DEBUG is enabled, this struct will be used to track owner, mode and order checking */ +#ifdef DEBUG + struct _mali_osk_lock_debug_s { + u32 owner; + _mali_osk_lock_flags_t orig_flags; + _mali_osk_lock_order_t order; + struct _mali_osk_lock_debug_s *next; + }; +#endif + + /* Anstraction of spinlock_t */ + struct _mali_osk_spinlock_s { +#ifdef DEBUG + struct _mali_osk_lock_debug_s checker; +#endif + spinlock_t spinlock; + }; + + /* Abstration of spinlock_t and lock flag which is used to store register's state before locking */ + struct _mali_osk_spinlock_irq_s { +#ifdef DEBUG + struct _mali_osk_lock_debug_s checker; +#endif + + spinlock_t spinlock; + unsigned long flags; + }; + + /* Abstraction of rw_semaphore in OS */ + struct _mali_osk_mutex_rw_s { +#ifdef DEBUG + struct _mali_osk_lock_debug_s checker; + _mali_osk_lock_mode_t mode; +#endif + + struct rw_semaphore rw_sema; + }; + + /* Mutex and mutex_interruptible functions share the same osk mutex struct */ + struct _mali_osk_mutex_s { +#ifdef DEBUG + struct _mali_osk_lock_debug_s checker; +#endif + struct mutex mutex; + }; + +#ifdef DEBUG + /** @brief _mali_osk_locks_debug_init/add/remove() functions are declared when DEBUG is enabled and + * defined in file mali_osk_locks.c. When LOCK_ORDER_CHECKING is enabled, calling these functions when we + * init/lock/unlock a lock/mutex, we could track lock order of a given tid. */ + void _mali_osk_locks_debug_init(struct _mali_osk_lock_debug_s *checker, _mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order); + void _mali_osk_locks_debug_add(struct _mali_osk_lock_debug_s *checker); + void _mali_osk_locks_debug_remove(struct _mali_osk_lock_debug_s *checker); + + /** @brief This function can return a given lock's owner when DEBUG is enabled. */ + static inline u32 _mali_osk_lock_get_owner(struct _mali_osk_lock_debug_s *lock) + { + return lock->owner; + } +#else +#define _mali_osk_locks_debug_init(x, y, z) do {} while (0) +#define _mali_osk_locks_debug_add(x) do {} while (0) +#define _mali_osk_locks_debug_remove(x) do {} while (0) +#endif + + /** @brief Before use _mali_osk_spin_lock, init function should be used to allocate memory and initial spinlock*/ + static inline _mali_osk_spinlock_t *_mali_osk_spinlock_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) + { + _mali_osk_spinlock_t *lock = NULL; + + lock = kmalloc(sizeof(_mali_osk_spinlock_t), GFP_KERNEL); + if (NULL == lock) { + return NULL; + } + spin_lock_init(&lock->spinlock); + _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); + return lock; + } + + /** @brief Lock a spinlock */ + static inline void _mali_osk_spinlock_lock(_mali_osk_spinlock_t *lock) + { + BUG_ON(NULL == lock); + spin_lock(&lock->spinlock); + _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); + } + + /** @brief Unlock a spinlock */ + static inline void _mali_osk_spinlock_unlock(_mali_osk_spinlock_t *lock) + { + BUG_ON(NULL == lock); + _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); + spin_unlock(&lock->spinlock); + } + + /** @brief Free a memory block which the argument lock pointed to and its type must be + * _mali_osk_spinlock_t *. */ + static inline void _mali_osk_spinlock_term(_mali_osk_spinlock_t *lock) + { + /* Parameter validation */ + BUG_ON(NULL == lock); + + /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ + kfree(lock); + } + + /** @brief Before _mali_osk_spinlock_irq_lock/unlock/term() is called, init function should be + * called to initial spinlock and flags in struct _mali_osk_spinlock_irq_t. */ + static inline _mali_osk_spinlock_irq_t *_mali_osk_spinlock_irq_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) + { + _mali_osk_spinlock_irq_t *lock = NULL; + lock = kmalloc(sizeof(_mali_osk_spinlock_irq_t), GFP_KERNEL); + + if (NULL == lock) { + return NULL; + } + + lock->flags = 0; + spin_lock_init(&lock->spinlock); + _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); + return lock; + } + + /** @brief Lock spinlock and save the register's state */ + static inline void _mali_osk_spinlock_irq_lock(_mali_osk_spinlock_irq_t *lock) + { + unsigned long tmp_flags; + + BUG_ON(NULL == lock); + spin_lock_irqsave(&lock->spinlock, tmp_flags); + lock->flags = tmp_flags; + _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); + } + + /** @brief Unlock spinlock with saved register's state */ + static inline void _mali_osk_spinlock_irq_unlock(_mali_osk_spinlock_irq_t *lock) + { + BUG_ON(NULL == lock); + _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); + spin_unlock_irqrestore(&lock->spinlock, lock->flags); + } + + /** @brief Destroy a given memory block which lock pointed to, and the lock type must be + * _mali_osk_spinlock_irq_t *. */ + static inline void _mali_osk_spinlock_irq_term(_mali_osk_spinlock_irq_t *lock) + { + /* Parameter validation */ + BUG_ON(NULL == lock); + + /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ + kfree(lock); + } + + /** @brief Before _mali_osk_mutex_rw_wait/signal/term() is called, we should call + * _mali_osk_mutex_rw_init() to kmalloc a memory block and initial part of elements in it. */ + static inline _mali_osk_mutex_rw_t *_mali_osk_mutex_rw_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) + { + _mali_osk_mutex_rw_t *lock = NULL; + + lock = kmalloc(sizeof(_mali_osk_mutex_rw_t), GFP_KERNEL); + + if (NULL == lock) { + return NULL; + } + + init_rwsem(&lock->rw_sema); + _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); + return lock; + } + + /** @brief When call _mali_osk_mutex_rw_wait/signal() functions, the second argument mode + * should be assigned with value _MALI_OSK_LOCKMODE_RO or _MALI_OSK_LOCKMODE_RW */ + static inline void _mali_osk_mutex_rw_wait(_mali_osk_mutex_rw_t *lock, _mali_osk_lock_mode_t mode) + { + BUG_ON(NULL == lock); + BUG_ON(!(_MALI_OSK_LOCKMODE_RO == mode || _MALI_OSK_LOCKMODE_RW == mode)); + + if (mode == _MALI_OSK_LOCKMODE_RO) { + down_read(&lock->rw_sema); + } else { + down_write(&lock->rw_sema); + } + +#ifdef DEBUG + if (mode == _MALI_OSK_LOCKMODE_RW) { + lock->mode = mode; + } else { /* mode == _MALI_OSK_LOCKMODE_RO */ + lock->mode = mode; + } + _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); +#endif + } + + /** @brief Up lock->rw_sema with up_read/write() accordinf argument mode's value. */ + static inline void _mali_osk_mutex_rw_signal(_mali_osk_mutex_rw_t *lock, _mali_osk_lock_mode_t mode) + { + BUG_ON(NULL == lock); + BUG_ON(!(_MALI_OSK_LOCKMODE_RO == mode || _MALI_OSK_LOCKMODE_RW == mode)); +#ifdef DEBUG + /* make sure the thread releasing the lock actually was the owner */ + if (mode == _MALI_OSK_LOCKMODE_RW) { + _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); + /* This lock now has no owner */ + lock->checker.owner = 0; + } +#endif + + if (mode == _MALI_OSK_LOCKMODE_RO) { + up_read(&lock->rw_sema); + } else { + up_write(&lock->rw_sema); + } + } + + /** @brief Free a given memory block which lock pointed to and its type must be + * _mali_sok_mutex_rw_t *. */ + static inline void _mali_osk_mutex_rw_term(_mali_osk_mutex_rw_t *lock) + { + /* Parameter validation */ + BUG_ON(NULL == lock); + + /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ + kfree(lock); + } + + /** @brief Mutex & mutex_interruptible share the same init and term function, because they have the + * same osk mutex struct, and the difference between them is which locking function they use */ + static inline _mali_osk_mutex_t *_mali_osk_mutex_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) + { + _mali_osk_mutex_t *lock = NULL; + + lock = kmalloc(sizeof(_mali_osk_mutex_t), GFP_KERNEL); + + if (NULL == lock) { + return NULL; + } + mutex_init(&lock->mutex); + + _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); + return lock; + } + + /** @brief Lock the lock->mutex with mutex_lock_interruptible function */ + static inline _mali_osk_errcode_t _mali_osk_mutex_wait_interruptible(_mali_osk_mutex_t *lock) + { + _mali_osk_errcode_t err = _MALI_OSK_ERR_OK; + + BUG_ON(NULL == lock); + + if (mutex_lock_interruptible(&lock->mutex)) { + printk(KERN_WARNING "Mali: Can not lock mutex\n"); + err = _MALI_OSK_ERR_RESTARTSYSCALL; + } + + _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); + return err; + } + + /** @brief Unlock the lock->mutex which is locked with mutex_lock_interruptible() function. */ + static inline void _mali_osk_mutex_signal_interruptible(_mali_osk_mutex_t *lock) + { + BUG_ON(NULL == lock); + _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); + mutex_unlock(&lock->mutex); + } + + /** @brief Lock the lock->mutex just with mutex_lock() function which could not be interruptted. */ + static inline void _mali_osk_mutex_wait(_mali_osk_mutex_t *lock) + { + BUG_ON(NULL == lock); + mutex_lock(&lock->mutex); + _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); + } + + /** @brief Unlock the lock->mutex which is locked with mutex_lock() function. */ + static inline void _mali_osk_mutex_signal(_mali_osk_mutex_t *lock) + { + BUG_ON(NULL == lock); + _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); + mutex_unlock(&lock->mutex); + } + + /** @brief Free a given memory block which lock point. */ + static inline void _mali_osk_mutex_term(_mali_osk_mutex_t *lock) + { + /* Parameter validation */ + BUG_ON(NULL == lock); + + /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ + kfree(lock); + } + +#ifdef _cplusplus +} +#endif + +#endif diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c new file mode 100755 index 000000000000..994b04dad745 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_low_level_mem.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include +#include +#include + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" + +void _mali_osk_mem_barrier(void) +{ + mb(); +} + +void _mali_osk_write_mem_barrier(void) +{ + wmb(); +} + +mali_io_address _mali_osk_mem_mapioregion(uintptr_t phys, u32 size, const char *description) +{ + return (mali_io_address)ioremap(phys, size); +} + +void _mali_osk_mem_unmapioregion(uintptr_t phys, u32 size, mali_io_address virt) +{ + iounmap((void *)virt); +} + +_mali_osk_errcode_t inline _mali_osk_mem_reqregion(uintptr_t phys, u32 size, const char *description) +{ +#if MALI_LICENSE_IS_GPL + return _MALI_OSK_ERR_OK; /* GPL driver gets the mem region for the resources registered automatically */ +#else + return ((NULL == request_mem_region(phys, size, description)) ? _MALI_OSK_ERR_NOMEM : _MALI_OSK_ERR_OK); +#endif +} + +void inline _mali_osk_mem_unreqregion(uintptr_t phys, u32 size) +{ +#if !MALI_LICENSE_IS_GPL + release_mem_region(phys, size); +#endif +} + +void inline _mali_osk_mem_iowrite32_relaxed(volatile mali_io_address addr, u32 offset, u32 val) +{ + __raw_writel(cpu_to_le32(val), ((u8 *)addr) + offset); +} + +u32 inline _mali_osk_mem_ioread32(volatile mali_io_address addr, u32 offset) +{ + return ioread32(((u8 *)addr) + offset); +} + +void inline _mali_osk_mem_iowrite32(volatile mali_io_address addr, u32 offset, u32 val) +{ + iowrite32(val, ((u8 *)addr) + offset); +} + +void _mali_osk_cache_flushall(void) +{ + /** @note Cached memory is not currently supported in this implementation */ +} + +void _mali_osk_cache_ensure_uncached_range_flushed(void *uncached_mapping, u32 offset, u32 size) +{ + _mali_osk_write_mem_barrier(); +} + +u32 _mali_osk_mem_write_safe(void __user *dest, const void __user *src, u32 size) +{ +#define MALI_MEM_SAFE_COPY_BLOCK_SIZE 4096 + u32 retval = 0; + void *temp_buf; + + temp_buf = kmalloc(MALI_MEM_SAFE_COPY_BLOCK_SIZE, GFP_KERNEL); + if (NULL != temp_buf) { + u32 bytes_left_to_copy = size; + u32 i; + for (i = 0; i < size; i += MALI_MEM_SAFE_COPY_BLOCK_SIZE) { + u32 size_to_copy; + u32 size_copied; + u32 bytes_left; + + if (bytes_left_to_copy > MALI_MEM_SAFE_COPY_BLOCK_SIZE) { + size_to_copy = MALI_MEM_SAFE_COPY_BLOCK_SIZE; + } else { + size_to_copy = bytes_left_to_copy; + } + + bytes_left = copy_from_user(temp_buf, ((char *)src) + i, size_to_copy); + size_copied = size_to_copy - bytes_left; + + bytes_left = copy_to_user(((char *)dest) + i, temp_buf, size_copied); + size_copied -= bytes_left; + + bytes_left_to_copy -= size_copied; + retval += size_copied; + + if (size_copied != size_to_copy) { + break; /* Early out, we was not able to copy this entire block */ + } + } + + kfree(temp_buf); + } + + return retval; +} + +_mali_osk_errcode_t _mali_ukk_mem_write_safe(_mali_uk_mem_write_safe_s *args) +{ + void __user *src; + void __user *dst; + struct mali_session_data *session; + + MALI_DEBUG_ASSERT_POINTER(args); + + session = (struct mali_session_data *)(uintptr_t)args->ctx; + + if (NULL == session) { + return _MALI_OSK_ERR_INVALID_ARGS; + } + + src = (void __user *)(uintptr_t)args->src; + dst = (void __user *)(uintptr_t)args->dest; + + /* Return number of bytes actually copied */ + args->size = _mali_osk_mem_write_safe(dst, src, args->size); + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c new file mode 100755 index 000000000000..a729d0499869 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c @@ -0,0 +1,505 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + + +/** + * @file mali_osk_mali.c + * Implementation of the OS abstraction layer which is specific for the Mali kernel device driver + */ +#include "../platform/rk/custom_log.h" + +#include +#include +#include +#include +#include +#include + +#include "mali_osk_mali.h" +#include "mali_kernel_common.h" /* MALI_xxx macros */ +#include "mali_osk.h" /* kernel side OS functions */ +#include "mali_kernel_linux.h" + +static mali_bool mali_secure_mode_enabled = MALI_FALSE; +static mali_bool mali_secure_mode_supported = MALI_FALSE; + +/* Function that init the mali gpu secure mode */ +void (*mali_secure_mode_deinit)(void) = NULL; +/* Function that reset GPU and enable the mali gpu secure mode */ +int (*mali_gpu_reset_and_secure_mode_enable)(void) = NULL; +/* Function that reset GPU and disable the mali gpu secure mode */ +int (*mali_gpu_reset_and_secure_mode_disable)(void) = NULL; + + +#ifdef CONFIG_MALI_DT + +#define MALI_OSK_INVALID_RESOURCE_ADDRESS 0xFFFFFFFF + +/** + * Define the max number of resource we could have. + */ +#define MALI_OSK_MAX_RESOURCE_NUMBER 27 + +/** + * Define the max number of resource with interrupts, and they are + * the first 20 elements in array mali_osk_resource_bank. + */ +#define MALI_OSK_RESOURCE_WITH_IRQ_NUMBER 20 + +/** + * pp core start and end location in mali_osk_resource_bank array. + */ +#define MALI_OSK_RESOURCE_PP_LOCATION_START 2 +#define MALI_OSK_RESOURCE_PP_LOCATION_END 17 + +/** + * L2 cache start and end location in mali_osk_resource_bank array. + */ +#define MALI_OSK_RESOURCE_L2_LOCATION_START 20 +#define MALI_OSK_RESOURCE_l2_LOCATION_END 22 + +/** + * DMA unit location. + */ +#define MALI_OSK_RESOURCE_DMA_LOCATION 26 + +static _mali_osk_resource_t mali_osk_resource_bank[MALI_OSK_MAX_RESOURCE_NUMBER] = { + /*-------------------------------------------------------*/ + /* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */ + /* {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "IRQGP",}, */ + {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "Mali_GP_IRQ",}, + /* {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "IRQGPMMU",}, */ + {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "Mali_GP_MMU_IRQ",}, + /* {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "IRQPP0",}, */ + {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "Mali_PP0_IRQ",}, + /* {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "IRQPPMMU0",}, */ + {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "Mali_PP0_MMU_IRQ",}, + /* {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "IRQPP1",}, */ + {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "Mali_PP1_IRQ",}, + /* {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "IRQPPMMU1",}, */ + {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "Mali_PP1_MMU_IRQ",}, + + {.description = "Mali_PP2", .base = MALI_OFFSET_PP2, .irq_name = "Mali_PP2_IRQ",}, + {.description = "Mali_PP2_MMU", .base = MALI_OFFSET_PP2_MMU, .irq_name = "Mali_PP2_MMU_IRQ",}, + {.description = "Mali_PP3", .base = MALI_OFFSET_PP3, .irq_name = "Mali_PP3_IRQ",}, + {.description = "Mali_PP3_MMU", .base = MALI_OFFSET_PP3_MMU, .irq_name = "Mali_PP3_MMU_IRQ",}, + /*-------------------------------------------------------*/ + {.description = "Mali_PP4", .base = MALI_OFFSET_PP4, .irq_name = "IRQPP4",}, + {.description = "Mali_PP4_MMU", .base = MALI_OFFSET_PP4_MMU, .irq_name = "IRQPPMMU4",}, + {.description = "Mali_PP5", .base = MALI_OFFSET_PP5, .irq_name = "IRQPP5",}, + {.description = "Mali_PP5_MMU", .base = MALI_OFFSET_PP5_MMU, .irq_name = "IRQPPMMU5",}, + {.description = "Mali_PP6", .base = MALI_OFFSET_PP6, .irq_name = "IRQPP6",}, + {.description = "Mali_PP6_MMU", .base = MALI_OFFSET_PP6_MMU, .irq_name = "IRQPPMMU6",}, + {.description = "Mali_PP7", .base = MALI_OFFSET_PP7, .irq_name = "IRQPP7",}, + {.description = "Mali_PP7_MMU", .base = MALI_OFFSET_PP7_MMU, .irq_name = "IRQPPMMU",}, + {.description = "Mali_PP_Broadcast", .base = MALI_OFFSET_PP_BCAST, .irq_name = "IRQPP",}, + {.description = "Mali_PMU", .base = MALI_OFFSET_PMU, .irq_name = "IRQPMU",}, + {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE0,}, + {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE1,}, + {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE2,}, + {.description = "Mali_PP_MMU_Broadcast", .base = MALI_OFFSET_PP_BCAST_MMU,}, + {.description = "Mali_Broadcast", .base = MALI_OFFSET_BCAST,}, + {.description = "Mali_DLBU", .base = MALI_OFFSET_DLBU,}, + {.description = "Mali_DMA", .base = MALI_OFFSET_DMA,}, +}; + +static int _mali_osk_get_compatible_name(const char **out_string) +{ + struct device_node *node = mali_platform_device->dev.of_node; + + MALI_DEBUG_ASSERT(NULL != node); + + return of_property_read_string(node, "compatible", out_string); +} + +_mali_osk_errcode_t _mali_osk_resource_initialize(void) +{ + mali_bool mali_is_450 = MALI_FALSE, mali_is_470 = MALI_FALSE; + int i, pp_core_num = 0, l2_core_num = 0; + struct resource *res; + const char *compatible_name = NULL; + + if (0 == _mali_osk_get_compatible_name(&compatible_name)) { + if (0 == strncmp(compatible_name, "arm,mali-450", strlen("arm,mali-450"))) { + mali_is_450 = MALI_TRUE; + MALI_DEBUG_PRINT(2, ("mali-450 device tree detected.")); + } else if (0 == strncmp(compatible_name, "arm,mali-470", strlen("arm,mali-470"))) { + mali_is_470 = MALI_TRUE; + MALI_DEBUG_PRINT(2, ("mali-470 device tree detected.")); + } + } + + for (i = 0; i < MALI_OSK_RESOURCE_WITH_IRQ_NUMBER; i++) { + res = platform_get_resource_byname(mali_platform_device, IORESOURCE_IRQ, mali_osk_resource_bank[i].irq_name); + if (res) { + mali_osk_resource_bank[i].irq = res->start; + } else { + mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; + } + } + + for (i = MALI_OSK_RESOURCE_PP_LOCATION_START; i <= MALI_OSK_RESOURCE_PP_LOCATION_END; i++) { + if (MALI_OSK_INVALID_RESOURCE_ADDRESS != mali_osk_resource_bank[i].base) { + pp_core_num++; + } + } + + /* We have to divide by 2, because we caculate twice for only one pp(pp_core and pp_mmu_core). */ + if (0 != pp_core_num % 2) { + MALI_DEBUG_PRINT(2, ("The value of pp core number isn't normal.")); + return _MALI_OSK_ERR_FAULT; + } + + pp_core_num /= 2; + + /** + * we can caculate the number of l2 cache core according the number of pp core number + * and device type(mali400/mali450/mali470). + */ + l2_core_num = 1; + if (mali_is_450) { + if (pp_core_num > 4) { + l2_core_num = 3; + } else if (pp_core_num <= 4) { + l2_core_num = 2; + } + } + + for (i = MALI_OSK_RESOURCE_l2_LOCATION_END; i > MALI_OSK_RESOURCE_L2_LOCATION_START + l2_core_num - 1; i--) { + mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; + } + + /* If device is not mali-450 type, we have to remove related resource from resource bank. */ + if (!(mali_is_450 || mali_is_470)) { + for (i = MALI_OSK_RESOURCE_l2_LOCATION_END + 1; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) { + mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; + } + } + + if (mali_is_470) + mali_osk_resource_bank[MALI_OSK_RESOURCE_DMA_LOCATION].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res) +{ + int i; + + if (NULL == mali_platform_device) { + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + /* Traverse all of resources in resources bank to find the matching one. */ + for (i = 0; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) { + if (mali_osk_resource_bank[i].base == addr) { + if (NULL != res) { + res->base = addr + _mali_osk_resource_base_address(); + res->description = mali_osk_resource_bank[i].description; + res->irq = mali_osk_resource_bank[i].irq; + } + return _MALI_OSK_ERR_OK; + } + } + + return _MALI_OSK_ERR_ITEM_NOT_FOUND; +} + +uintptr_t _mali_osk_resource_base_address(void) +{ + struct resource *reg_res = NULL; + uintptr_t ret = 0; + + reg_res = platform_get_resource(mali_platform_device, IORESOURCE_MEM, 0); + + if (NULL != reg_res) { + ret = reg_res->start; + } + + return ret; +} + +void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size) +{ + struct device_node *node = mali_platform_device->dev.of_node; + struct property *prop; + const __be32 *p; + int length = 0, i = 0; + u32 u; + + MALI_DEBUG_PRINT(2, ("Get pmu config from device tree configuration.\n")); + + MALI_DEBUG_ASSERT(NULL != node); + + if (!of_get_property(node, "pmu_domain_config", &length)) { + return; + } + + if (array_size != length / sizeof(u32)) { + MALI_PRINT_ERROR(("Wrong pmu domain config in device tree.")); + return; + } + + of_property_for_each_u32(node, "pmu_domain_config", prop, p, u) { + domain_config_array[i] = (u16)u; + i++; + } + + return; +} + +u32 _mali_osk_get_pmu_switch_delay(void) +{ + struct device_node *node = mali_platform_device->dev.of_node; + u32 switch_delay; + + MALI_DEBUG_ASSERT(NULL != node); + + if (0 == of_property_read_u32(node, "pmu_switch_delay", &switch_delay)) { + return switch_delay; + } else { + MALI_DEBUG_PRINT(2, ("Couldn't find pmu_switch_delay in device tree configuration.\n")); + } + + return 0; +} + +#else /* CONFIG_MALI_DT */ /* 若未 定义 CONFIG_MALI_DT. */ + +_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res) +{ + int i; + uintptr_t phys_addr; + + if (NULL == mali_platform_device) { + /* Not connected to a device */ + return _MALI_OSK_ERR_ITEM_NOT_FOUND; + } + + phys_addr = addr + _mali_osk_resource_base_address(); + for (i = 0; i < mali_platform_device->num_resources; i++) { + if (IORESOURCE_MEM == resource_type(&(mali_platform_device->resource[i])) && + mali_platform_device->resource[i].start == phys_addr) { + if (NULL != res) { + res->base = phys_addr; + res->description = mali_platform_device->resource[i].name; + + /* Any (optional) IRQ resource belonging to this resource will follow */ + if ((i + 1) < mali_platform_device->num_resources && + IORESOURCE_IRQ == resource_type(&(mali_platform_device->resource[i + 1]))) { + res->irq = mali_platform_device->resource[i + 1].start; + } else { + res->irq = -1; + } + } + return _MALI_OSK_ERR_OK; + } + } + + return _MALI_OSK_ERR_ITEM_NOT_FOUND; +} + +uintptr_t _mali_osk_resource_base_address(void) +{ + uintptr_t lowest_addr = (uintptr_t)(0 - 1); + uintptr_t ret = 0; + + if (NULL != mali_platform_device) { + int i; + for (i = 0; i < mali_platform_device->num_resources; i++) { + if (mali_platform_device->resource[i].flags & IORESOURCE_MEM && + mali_platform_device->resource[i].start < lowest_addr) { + lowest_addr = mali_platform_device->resource[i].start; + ret = lowest_addr; + } + } + } + + return ret; +} + +void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size) +{ + _mali_osk_device_data data = { 0, }; + + MALI_DEBUG_PRINT(2, ("Get pmu config from platform device data.\n")); + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + /* Copy the custom customer power domain config */ + _mali_osk_memcpy(domain_config_array, data.pmu_domain_config, sizeof(data.pmu_domain_config)); + } + + return; +} + +u32 _mali_osk_get_pmu_switch_delay(void) +{ + _mali_osk_errcode_t err; + _mali_osk_device_data data = { 0, }; + + err = _mali_osk_device_data_get(&data); + + if (_MALI_OSK_ERR_OK == err) { + return data.pmu_switch_delay; + } + + return 0; +} +#endif /* CONFIG_MALI_DT */ + +_mali_osk_errcode_t _mali_osk_device_data_get(_mali_osk_device_data *data) +{ + MALI_DEBUG_ASSERT_POINTER(data); + + if (NULL != mali_platform_device) { + struct mali_gpu_device_data *os_data = NULL; + + os_data = (struct mali_gpu_device_data *)mali_platform_device->dev.platform_data; + if (NULL != os_data) { + /* Copy data from OS dependant struct to Mali neutral struct (identical!) */ + BUILD_BUG_ON(sizeof(*os_data) != sizeof(*data)); + _mali_osk_memcpy(data, os_data, sizeof(*os_data)); + + return _MALI_OSK_ERR_OK; + } + } + + return _MALI_OSK_ERR_ITEM_NOT_FOUND; +} + +u32 _mali_osk_identify_gpu_resource(void) +{ + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_L2_RESOURCE1, NULL)) + /* Mali 450 */ + return 0x450; + + if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_DLBU, NULL)) + /* Mali 470 */ + return 0x470; + + /* Mali 400 */ + return 0x400; +} + +mali_bool _mali_osk_shared_interrupts(void) +{ + u32 irqs[128]; + u32 i, j, irq, num_irqs_found = 0; + + MALI_DEBUG_ASSERT_POINTER(mali_platform_device); + MALI_DEBUG_ASSERT(128 >= mali_platform_device->num_resources); + + for (i = 0; i < mali_platform_device->num_resources; i++) { + if (IORESOURCE_IRQ & mali_platform_device->resource[i].flags) { + irq = mali_platform_device->resource[i].start; + + for (j = 0; j < num_irqs_found; ++j) { + if (irq == irqs[j]) { + return MALI_TRUE; + } + } + + irqs[num_irqs_found++] = irq; + } + } + + return MALI_FALSE; +} + +_mali_osk_errcode_t _mali_osk_gpu_secure_mode_init(void) +{ + _mali_osk_device_data data = { 0, }; + + if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { + if ((NULL != data.secure_mode_init) && (NULL != data.secure_mode_deinit) + && (NULL != data.gpu_reset_and_secure_mode_enable) && (NULL != data.gpu_reset_and_secure_mode_disable)) { + int err = data.secure_mode_init(); + if (err) { + MALI_DEBUG_PRINT(1, ("Failed to init gpu secure mode.\n")); + return _MALI_OSK_ERR_FAULT; + } + + mali_secure_mode_deinit = data.secure_mode_deinit; + mali_gpu_reset_and_secure_mode_enable = data.gpu_reset_and_secure_mode_enable; + mali_gpu_reset_and_secure_mode_disable = data.gpu_reset_and_secure_mode_disable; + + mali_secure_mode_supported = MALI_TRUE; + mali_secure_mode_enabled = MALI_FALSE; + return _MALI_OSK_ERR_OK; + } + } + MALI_DEBUG_PRINT(3, ("GPU secure mode not supported.\n")); + return _MALI_OSK_ERR_UNSUPPORTED; + +} + +_mali_osk_errcode_t _mali_osk_gpu_secure_mode_deinit(void) +{ + if (NULL != mali_secure_mode_deinit) { + mali_secure_mode_deinit(); + mali_secure_mode_enabled = MALI_FALSE; + mali_secure_mode_supported = MALI_FALSE; + return _MALI_OSK_ERR_OK; + } + MALI_DEBUG_PRINT(3, ("GPU secure mode not supported.\n")); + return _MALI_OSK_ERR_UNSUPPORTED; + +} + + +_mali_osk_errcode_t _mali_osk_gpu_reset_and_secure_mode_enable(void) +{ + /* the mali executor lock must be held before enter this function. */ + + MALI_DEBUG_ASSERT(MALI_FALSE == mali_secure_mode_enabled); + + if (NULL != mali_gpu_reset_and_secure_mode_enable) { + if (mali_gpu_reset_and_secure_mode_enable()) { + MALI_DEBUG_PRINT(1, ("Failed to reset GPU or enable gpu secure mode.\n")); + return _MALI_OSK_ERR_FAULT; + } + mali_secure_mode_enabled = MALI_TRUE; + return _MALI_OSK_ERR_OK; + } + MALI_DEBUG_PRINT(1, ("GPU secure mode not supported.\n")); + return _MALI_OSK_ERR_UNSUPPORTED; +} + +_mali_osk_errcode_t _mali_osk_gpu_reset_and_secure_mode_disable(void) +{ + /* the mali executor lock must be held before enter this function. */ + + MALI_DEBUG_ASSERT(MALI_TRUE == mali_secure_mode_enabled); + + if (NULL != mali_gpu_reset_and_secure_mode_disable) { + if (mali_gpu_reset_and_secure_mode_disable()) { + MALI_DEBUG_PRINT(1, ("Failed to reset GPU or disable gpu secure mode.\n")); + return _MALI_OSK_ERR_FAULT; + } + mali_secure_mode_enabled = MALI_FALSE; + + return _MALI_OSK_ERR_OK; + + } + MALI_DEBUG_PRINT(1, ("GPU secure mode not supported.\n")); + return _MALI_OSK_ERR_UNSUPPORTED; + +} + +mali_bool _mali_osk_gpu_secure_mode_is_enabled(void) +{ + return mali_secure_mode_enabled; +} + +mali_bool _mali_osk_gpu_secure_mode_is_supported(void) +{ + return mali_secure_mode_supported; +} + + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c new file mode 100755 index 000000000000..0b2d00762771 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_math.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk.h" +#include + +u32 _mali_osk_clz(u32 input) +{ + return 32 - fls(input); +} + +u32 _mali_osk_fls(u32 input) +{ + return fls(input); +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c new file mode 100755 index 000000000000..174616b566c4 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_memory.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk.h" +#include +#include + +void inline *_mali_osk_calloc(u32 n, u32 size) +{ + return kcalloc(n, size, GFP_KERNEL); +} + +void inline *_mali_osk_malloc(u32 size) +{ + return kmalloc(size, GFP_KERNEL); +} + +void inline _mali_osk_free(void *ptr) +{ + kfree(ptr); +} + +void inline *_mali_osk_valloc(u32 size) +{ + return vmalloc(size); +} + +void inline _mali_osk_vfree(void *ptr) +{ + vfree(ptr); +} + +void inline *_mali_osk_memcpy(void *dst, const void *src, u32 len) +{ + return memcpy(dst, src, len); +} + +void inline *_mali_osk_memset(void *s, u32 c, u32 n) +{ + return memset(s, c, n); +} + +mali_bool _mali_osk_mem_check_allocated(u32 max_allocated) +{ + /* No need to prevent an out-of-memory dialogue appearing on Linux, + * so we always return MALI_TRUE. + */ + return MALI_TRUE; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c new file mode 100755 index 000000000000..9845187f8122 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_misc.c + * Implementation of the OS abstraction layer for the kernel device driver + */ +#include +#include +#include +#include +#include +#include +#include "mali_osk.h" + +#if !defined(CONFIG_MALI_QUIET) +void _mali_osk_dbgmsg(const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + vprintk(fmt, args); + va_end(args); +} +#endif /* !defined(CONFIG_MALI_QUIET) */ + +u32 _mali_osk_snprintf(char *buf, u32 size, const char *fmt, ...) +{ + int res; + va_list args; + va_start(args, fmt); + + res = vscnprintf(buf, (size_t)size, fmt, args); + + va_end(args); + return res; +} + +void _mali_osk_abort(void) +{ + /* make a simple fault by dereferencing a NULL pointer */ + dump_stack(); + *(volatile int *)0 = 0; +} + +void _mali_osk_break(void) +{ + _mali_osk_abort(); +} + +u32 _mali_osk_get_pid(void) +{ + /* Thread group ID is the process ID on Linux */ + return (u32)current->tgid; +} + +char *_mali_osk_get_comm(void) +{ + return (char *)current->comm; +} + + +u32 _mali_osk_get_tid(void) +{ + /* pid is actually identifying the thread on Linux */ + u32 tid = current->pid; + + /* If the pid is 0 the core was idle. Instead of returning 0 we return a special number + * identifying which core we are on. */ + if (0 == tid) { + tid = -(1 + raw_smp_processor_id()); + } + + return tid; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c new file mode 100755 index 000000000000..a05f8f066964 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_notification.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +#include +#include +#include + +/** + * Declaration of the notification queue object type + * Contains a linked list of notification pending delivery to user space. + * It also contains a wait queue of exclusive waiters blocked in the ioctl + * When a new notification is posted a single thread is resumed. + */ +struct _mali_osk_notification_queue_t_struct { + spinlock_t mutex; /**< Mutex protecting the list */ + wait_queue_head_t receive_queue; /**< Threads waiting for new entries to the queue */ + struct list_head head; /**< List of notifications waiting to be picked up */ +}; + +typedef struct _mali_osk_notification_wrapper_t_struct { + struct list_head list; /**< Internal linked list variable */ + _mali_osk_notification_t data; /**< Notification data */ +} _mali_osk_notification_wrapper_t; + +_mali_osk_notification_queue_t *_mali_osk_notification_queue_init(void) +{ + _mali_osk_notification_queue_t *result; + + result = (_mali_osk_notification_queue_t *)kmalloc(sizeof(_mali_osk_notification_queue_t), GFP_KERNEL); + if (NULL == result) return NULL; + + spin_lock_init(&result->mutex); + init_waitqueue_head(&result->receive_queue); + INIT_LIST_HEAD(&result->head); + + return result; +} + +_mali_osk_notification_t *_mali_osk_notification_create(u32 type, u32 size) +{ + /* OPT Recycling of notification objects */ + _mali_osk_notification_wrapper_t *notification; + + notification = (_mali_osk_notification_wrapper_t *)kmalloc(sizeof(_mali_osk_notification_wrapper_t) + size, + GFP_KERNEL | __GFP_HIGH | __GFP_RETRY_MAYFAIL); + if (NULL == notification) { + MALI_DEBUG_PRINT(1, ("Failed to create a notification object\n")); + return NULL; + } + + /* Init the list */ + INIT_LIST_HEAD(¬ification->list); + + if (0 != size) { + notification->data.result_buffer = ((u8 *)notification) + sizeof(_mali_osk_notification_wrapper_t); + } else { + notification->data.result_buffer = NULL; + } + + /* set up the non-allocating fields */ + notification->data.notification_type = type; + notification->data.result_buffer_size = size; + + /* all ok */ + return &(notification->data); +} + +void _mali_osk_notification_delete(_mali_osk_notification_t *object) +{ + _mali_osk_notification_wrapper_t *notification; + MALI_DEBUG_ASSERT_POINTER(object); + + notification = container_of(object, _mali_osk_notification_wrapper_t, data); + + /* Free the container */ + kfree(notification); +} + +void _mali_osk_notification_queue_term(_mali_osk_notification_queue_t *queue) +{ + _mali_osk_notification_t *result; + MALI_DEBUG_ASSERT_POINTER(queue); + + while (_MALI_OSK_ERR_OK == _mali_osk_notification_queue_dequeue(queue, &result)) { + _mali_osk_notification_delete(result); + } + + /* not much to do, just free the memory */ + kfree(queue); +} +void _mali_osk_notification_queue_send(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t *object) +{ +#if defined(MALI_UPPER_HALF_SCHEDULING) + unsigned long irq_flags; +#endif + + _mali_osk_notification_wrapper_t *notification; + MALI_DEBUG_ASSERT_POINTER(queue); + MALI_DEBUG_ASSERT_POINTER(object); + + notification = container_of(object, _mali_osk_notification_wrapper_t, data); + +#if defined(MALI_UPPER_HALF_SCHEDULING) + spin_lock_irqsave(&queue->mutex, irq_flags); +#else + spin_lock(&queue->mutex); +#endif + + list_add_tail(¬ification->list, &queue->head); + +#if defined(MALI_UPPER_HALF_SCHEDULING) + spin_unlock_irqrestore(&queue->mutex, irq_flags); +#else + spin_unlock(&queue->mutex); +#endif + + /* and wake up one possible exclusive waiter */ + wake_up(&queue->receive_queue); +} + +_mali_osk_errcode_t _mali_osk_notification_queue_dequeue(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result) +{ +#if defined(MALI_UPPER_HALF_SCHEDULING) + unsigned long irq_flags; +#endif + + _mali_osk_errcode_t ret = _MALI_OSK_ERR_ITEM_NOT_FOUND; + _mali_osk_notification_wrapper_t *wrapper_object; + +#if defined(MALI_UPPER_HALF_SCHEDULING) + spin_lock_irqsave(&queue->mutex, irq_flags); +#else + spin_lock(&queue->mutex); +#endif + + if (!list_empty(&queue->head)) { + wrapper_object = list_entry(queue->head.next, _mali_osk_notification_wrapper_t, list); + *result = &(wrapper_object->data); + list_del_init(&wrapper_object->list); + ret = _MALI_OSK_ERR_OK; + } + +#if defined(MALI_UPPER_HALF_SCHEDULING) + spin_unlock_irqrestore(&queue->mutex, irq_flags); +#else + spin_unlock(&queue->mutex); +#endif + + return ret; +} + +_mali_osk_errcode_t _mali_osk_notification_queue_receive(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result) +{ + /* check input */ + MALI_DEBUG_ASSERT_POINTER(queue); + MALI_DEBUG_ASSERT_POINTER(result); + + /* default result */ + *result = NULL; + + if (wait_event_interruptible(queue->receive_queue, + _MALI_OSK_ERR_OK == _mali_osk_notification_queue_dequeue(queue, result))) { + return _MALI_OSK_ERR_RESTARTSYSCALL; + } + + return _MALI_OSK_ERR_OK; /* all ok */ +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c new file mode 100755 index 000000000000..e28e2eb21fe2 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c @@ -0,0 +1,83 @@ +/** + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_pm.c + * Implementation of the callback functions from common power management + */ + +#include + +#include "mali_kernel_linux.h" +#ifdef CONFIG_PM_RUNTIME +#include +#endif /* CONFIG_PM_RUNTIME */ +#include +#include +#include "mali_osk.h" +#include "mali_kernel_common.h" + +/* Can NOT run in atomic context */ +_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_sync(void) +{ +#ifdef CONFIG_PM_RUNTIME + int err; + MALI_DEBUG_ASSERT_POINTER(mali_platform_device); + err = pm_runtime_get_sync(&(mali_platform_device->dev)); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) + pm_runtime_mark_last_busy(&(mali_platform_device->dev)); +#endif + if (0 > err) { + MALI_PRINT_ERROR(("Mali OSK PM: pm_runtime_get_sync() returned error code %d\n", err)); + return _MALI_OSK_ERR_FAULT; + } +#endif + return _MALI_OSK_ERR_OK; +} + +/* Can run in atomic context */ +_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_async(void) +{ +#ifdef CONFIG_PM_RUNTIME + int err; + MALI_DEBUG_ASSERT_POINTER(mali_platform_device); + err = pm_runtime_get(&(mali_platform_device->dev)); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) + pm_runtime_mark_last_busy(&(mali_platform_device->dev)); +#endif + if (0 > err && -EINPROGRESS != err) { + MALI_PRINT_ERROR(("Mali OSK PM: pm_runtime_get() returned error code %d\n", err)); + return _MALI_OSK_ERR_FAULT; + } +#endif + return _MALI_OSK_ERR_OK; +} + + +/* Can run in atomic context */ +void _mali_osk_pm_dev_ref_put(void) +{ +#ifdef CONFIG_PM_RUNTIME + MALI_DEBUG_ASSERT_POINTER(mali_platform_device); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) + pm_runtime_mark_last_busy(&(mali_platform_device->dev)); + pm_runtime_put_autosuspend(&(mali_platform_device->dev)); +#else + pm_runtime_put(&(mali_platform_device->dev)); +#endif +#endif +} + +void _mali_osk_pm_dev_barrier(void) +{ +#ifdef CONFIG_PM_RUNTIME + pm_runtime_barrier(&(mali_platform_device->dev)); +#endif +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c new file mode 100755 index 000000000000..9e977ea4d0ff --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c @@ -0,0 +1,1282 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include +#include +#include +#include +#include +#include + +#include +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_ukk.h" +#include "mali_uk_types.h" +#include "mali_osk_profiling.h" +#include "mali_linux_trace.h" +#include "mali_gp.h" +#include "mali_pp.h" +#include "mali_l2_cache.h" +#include "mali_user_settings_db.h" +#include "mali_executor.h" +#include "mali_memory_manager.h" + +#define MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE 100 +#define MALI_PROFILING_STREAM_HOLD_TIME 1000000 /*1 ms */ + +#define MALI_PROFILING_STREAM_BUFFER_SIZE (1 << 12) +#define MALI_PROFILING_STREAM_BUFFER_NUM 100 + +/** + * Define the mali profiling stream struct. + */ +typedef struct mali_profiling_stream { + u8 data[MALI_PROFILING_STREAM_BUFFER_SIZE]; + u32 used_size; + struct list_head list; +} mali_profiling_stream; + +typedef struct mali_profiling_stream_list { + spinlock_t spin_lock; + struct list_head free_list; + struct list_head queue_list; +} mali_profiling_stream_list; + +static const char mali_name[] = "4xx"; +static const char utgard_setup_version[] = "ANNOTATE_SETUP 1\n"; + +static u32 profiling_sample_rate = 0; +static u32 first_sw_counter_index = 0; + +static mali_bool l2_cache_counter_if_enabled = MALI_FALSE; +static u32 num_counters_enabled = 0; +static u32 mem_counters_enabled = 0; + +static _mali_osk_atomic_t stream_fd_if_used; + +static wait_queue_head_t stream_fd_wait_queue; +static mali_profiling_counter *global_mali_profiling_counters = NULL; +static u32 num_global_mali_profiling_counters = 0; + +static mali_profiling_stream_list *global_mali_stream_list = NULL; +static mali_profiling_stream *mali_counter_stream = NULL; +static mali_profiling_stream *mali_core_activity_stream = NULL; +static u64 mali_core_activity_stream_dequeue_time = 0; +static spinlock_t mali_activity_lock; +static u32 mali_activity_cores_num = 0; +static struct hrtimer profiling_sampling_timer; + +const char *_mali_mem_counter_descriptions[] = _MALI_MEM_COUTNER_DESCRIPTIONS; +const char *_mali_special_counter_descriptions[] = _MALI_SPCIAL_COUNTER_DESCRIPTIONS; + +static u32 current_profiling_pid = 0; + +static void _mali_profiling_stream_list_destory(mali_profiling_stream_list *profiling_stream_list) +{ + mali_profiling_stream *profiling_stream, *tmp_profiling_stream; + MALI_DEBUG_ASSERT_POINTER(profiling_stream_list); + + list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->free_list, list) { + list_del(&profiling_stream->list); + kfree(profiling_stream); + } + + list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->queue_list, list) { + list_del(&profiling_stream->list); + kfree(profiling_stream); + } + + kfree(profiling_stream_list); +} + +static void _mali_profiling_global_stream_list_free(void) +{ + mali_profiling_stream *profiling_stream, *tmp_profiling_stream; + unsigned long irq_flags; + + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &global_mali_stream_list->queue_list, list) { + profiling_stream->used_size = 0; + list_move(&profiling_stream->list, &global_mali_stream_list->free_list); + } + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); +} + +static _mali_osk_errcode_t _mali_profiling_global_stream_list_dequeue(struct list_head *stream_list, mali_profiling_stream **new_mali_profiling_stream) +{ + unsigned long irq_flags; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + MALI_DEBUG_ASSERT_POINTER(stream_list); + + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + + if (!list_empty(stream_list)) { + *new_mali_profiling_stream = list_entry(stream_list->next, mali_profiling_stream, list); + list_del_init(&(*new_mali_profiling_stream)->list); + } else { + ret = _MALI_OSK_ERR_NOMEM; + } + + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); + + return ret; +} + +static void _mali_profiling_global_stream_list_queue(struct list_head *stream_list, mali_profiling_stream *current_mali_profiling_stream) +{ + unsigned long irq_flags; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + MALI_DEBUG_ASSERT_POINTER(stream_list); + + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + list_add_tail(¤t_mali_profiling_stream->list, stream_list); + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); +} + +static mali_bool _mali_profiling_global_stream_queue_list_if_empty(void) +{ + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + return list_empty(&global_mali_stream_list->queue_list); +} + +static u32 _mali_profiling_global_stream_queue_list_next_size(void) +{ + unsigned long irq_flags; + u32 size = 0; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + + spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); + if (!list_empty(&global_mali_stream_list->queue_list)) { + mali_profiling_stream *next_mali_profiling_stream = + list_entry(global_mali_stream_list->queue_list.next, mali_profiling_stream, list); + size = next_mali_profiling_stream->used_size; + } + spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); + return size; +} + +/* The mali profiling stream file operations functions. */ +static ssize_t _mali_profiling_stream_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos); + +static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait); + +static int _mali_profiling_stream_release(struct inode *inode, struct file *filp); + +/* The timeline stream file operations structure. */ +static const struct file_operations mali_profiling_stream_fops = { + .release = _mali_profiling_stream_release, + .read = _mali_profiling_stream_read, + .poll = _mali_profiling_stream_poll, +}; + +static ssize_t _mali_profiling_stream_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos) +{ + u32 copy_len = 0; + mali_profiling_stream *current_mali_profiling_stream; + u32 used_size; + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + + while (!_mali_profiling_global_stream_queue_list_if_empty()) { + used_size = _mali_profiling_global_stream_queue_list_next_size(); + if (used_size <= ((u32)size - copy_len)) { + current_mali_profiling_stream = NULL; + _mali_profiling_global_stream_list_dequeue(&global_mali_stream_list->queue_list, + ¤t_mali_profiling_stream); + MALI_DEBUG_ASSERT_POINTER(current_mali_profiling_stream); + if (copy_to_user(&buffer[copy_len], current_mali_profiling_stream->data, current_mali_profiling_stream->used_size)) { + current_mali_profiling_stream->used_size = 0; + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream); + return -EFAULT; + } + copy_len += current_mali_profiling_stream->used_size; + current_mali_profiling_stream->used_size = 0; + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream); + } else { + break; + } + } + return (ssize_t)copy_len; +} + +static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait) +{ + poll_wait(filp, &stream_fd_wait_queue, wait); + if (!_mali_profiling_global_stream_queue_list_if_empty()) + return POLLIN; + return 0; +} + +static int _mali_profiling_stream_release(struct inode *inode, struct file *filp) +{ + _mali_osk_atomic_init(&stream_fd_if_used, 0); + return 0; +} + +/* The funs for control packet and stream data.*/ +static void _mali_profiling_set_packet_size(unsigned char *const buf, const u32 size) +{ + u32 i; + + for (i = 0; i < sizeof(size); ++i) + buf[i] = (size >> 8 * i) & 0xFF; +} + +static u32 _mali_profiling_get_packet_size(unsigned char *const buf) +{ + u32 i; + u32 size = 0; + for (i = 0; i < sizeof(size); ++i) + size |= (u32)buf[i] << 8 * i; + return size; +} + +static u32 _mali_profiling_read_packet_int(unsigned char *const buf, u32 *const pos, u32 const packet_size) +{ + u64 int_value = 0; + u8 shift = 0; + u8 byte_value = ~0; + + while ((byte_value & 0x80) != 0) { + if ((*pos) >= packet_size) + return -1; + byte_value = buf[*pos]; + *pos += 1; + int_value |= (u32)(byte_value & 0x7f) << shift; + shift += 7; + } + + if (shift < 8 * sizeof(int_value) && (byte_value & 0x40) != 0) { + int_value |= -(1 << shift); + } + + return int_value; +} + +static u32 _mali_profiling_pack_int(u8 *const buf, u32 const buf_size, u32 const pos, s32 value) +{ + u32 add_bytes = 0; + int more = 1; + while (more) { + /* low order 7 bits of val */ + char byte_value = value & 0x7f; + value >>= 7; + + if ((value == 0 && (byte_value & 0x40) == 0) || (value == -1 && (byte_value & 0x40) != 0)) { + more = 0; + } else { + byte_value |= 0x80; + } + + if ((pos + add_bytes) >= buf_size) + return 0; + buf[pos + add_bytes] = byte_value; + add_bytes++; + } + + return add_bytes; +} + +static int _mali_profiling_pack_long(uint8_t *const buf, u32 const buf_size, u32 const pos, s64 val) +{ + int add_bytes = 0; + int more = 1; + while (more) { + /* low order 7 bits of x */ + char byte_value = val & 0x7f; + val >>= 7; + + if ((val == 0 && (byte_value & 0x40) == 0) || (val == -1 && (byte_value & 0x40) != 0)) { + more = 0; + } else { + byte_value |= 0x80; + } + + MALI_DEBUG_ASSERT((pos + add_bytes) < buf_size); + buf[pos + add_bytes] = byte_value; + add_bytes++; + } + + return add_bytes; +} + +static void _mali_profiling_stream_add_counter(mali_profiling_stream *profiling_stream, s64 current_time, u32 key, u32 counter_value) +{ + u32 add_size = STREAM_HEADER_SIZE; + MALI_DEBUG_ASSERT_POINTER(profiling_stream); + MALI_DEBUG_ASSERT((profiling_stream->used_size) < MALI_PROFILING_STREAM_BUFFER_SIZE); + + profiling_stream->data[profiling_stream->used_size] = STREAM_HEADER_COUNTER_VALUE; + + add_size += _mali_profiling_pack_long(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, current_time); + add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, (s32)0); + add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, (s32)key); + add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, + profiling_stream->used_size + add_size, (s32)counter_value); + + _mali_profiling_set_packet_size(profiling_stream->data + profiling_stream->used_size + 1, + add_size - STREAM_HEADER_SIZE); + + profiling_stream->used_size += add_size; +} + +/* The callback function for sampling timer.*/ +static enum hrtimer_restart _mali_profiling_sampling_counters(struct hrtimer *timer) +{ + u32 counter_index; + s64 current_time; + MALI_DEBUG_ASSERT_POINTER(global_mali_profiling_counters); + MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); + + MALI_DEBUG_ASSERT(NULL == mali_counter_stream); + if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue( + &global_mali_stream_list->free_list, &mali_counter_stream)) { + + MALI_DEBUG_ASSERT_POINTER(mali_counter_stream); + MALI_DEBUG_ASSERT(0 == mali_counter_stream->used_size); + + /* Capture l2 cache counter values if enabled */ + if (MALI_TRUE == l2_cache_counter_if_enabled) { + int i, j = 0; + _mali_profiling_l2_counter_values l2_counters_values; + _mali_profiling_get_l2_counters(&l2_counters_values); + + for (i = COUNTER_L2_0_C0; i <= COUNTER_L2_2_C1; i++) { + if (0 == (j % 2)) + _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value0); + else + _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value1); + j++; + } + } + + current_time = (s64)_mali_osk_boot_time_get_ns(); + + /* Add all enabled counter values into stream */ + for (counter_index = 0; counter_index < num_global_mali_profiling_counters; counter_index++) { + /* No need to sample these couners here. */ + if (global_mali_profiling_counters[counter_index].enabled) { + if ((global_mali_profiling_counters[counter_index].counter_id >= FIRST_MEM_COUNTER && + global_mali_profiling_counters[counter_index].counter_id <= LAST_MEM_COUNTER) + || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_VP_ACTIVITY) + || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FP_ACTIVITY) + || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FILMSTRIP)) { + + continue; + } + + if (global_mali_profiling_counters[counter_index].counter_id >= COUNTER_L2_0_C0 && + global_mali_profiling_counters[counter_index].counter_id <= COUNTER_L2_2_C1) { + + u32 prev_val = global_mali_profiling_counters[counter_index].prev_counter_value; + + _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key, + global_mali_profiling_counters[counter_index].current_counter_value - prev_val); + + prev_val = global_mali_profiling_counters[counter_index].current_counter_value; + + global_mali_profiling_counters[counter_index].prev_counter_value = prev_val; + } else { + + if (global_mali_profiling_counters[counter_index].counter_id == COUNTER_TOTAL_ALLOC_PAGES) { + u32 total_alloc_mem = _mali_ukk_report_memory_usage(); + global_mali_profiling_counters[counter_index].current_counter_value = total_alloc_mem / _MALI_OSK_MALI_PAGE_SIZE; + } + _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key, + global_mali_profiling_counters[counter_index].current_counter_value); + if (global_mali_profiling_counters[counter_index].counter_id < FIRST_SPECIAL_COUNTER) + global_mali_profiling_counters[counter_index].current_counter_value = 0; + } + } + } + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_counter_stream); + mali_counter_stream = NULL; + } else { + MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n")); + } + + wake_up_interruptible(&stream_fd_wait_queue); + + /*Enable the sampling timer again*/ + if (0 != num_counters_enabled && 0 != profiling_sample_rate) { + hrtimer_forward_now(&profiling_sampling_timer, ns_to_ktime(profiling_sample_rate)); + return HRTIMER_RESTART; + } + return HRTIMER_NORESTART; +} + +static void _mali_profiling_sampling_core_activity_switch(int counter_id, int core, u32 activity, u32 pid) +{ + unsigned long irq_flags; + + spin_lock_irqsave(&mali_activity_lock, irq_flags); + if (activity == 0) + mali_activity_cores_num--; + else + mali_activity_cores_num++; + spin_unlock_irqrestore(&mali_activity_lock, irq_flags); + + if (NULL != global_mali_profiling_counters) { + int i ; + for (i = 0; i < num_global_mali_profiling_counters; i++) { + if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) { + u64 current_time = _mali_osk_boot_time_get_ns(); + u32 add_size = STREAM_HEADER_SIZE; + + if (NULL != mali_core_activity_stream) { + if ((mali_core_activity_stream_dequeue_time + MALI_PROFILING_STREAM_HOLD_TIME < current_time) || + (MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE > MALI_PROFILING_STREAM_BUFFER_SIZE + - mali_core_activity_stream->used_size)) { + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream); + mali_core_activity_stream = NULL; + wake_up_interruptible(&stream_fd_wait_queue); + } + } + + if (NULL == mali_core_activity_stream) { + if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue( + &global_mali_stream_list->free_list, &mali_core_activity_stream)) { + mali_core_activity_stream_dequeue_time = current_time; + } else { + MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n")); + wake_up_interruptible(&stream_fd_wait_queue); + break; + } + + } + + mali_core_activity_stream->data[mali_core_activity_stream->used_size] = STREAM_HEADER_CORE_ACTIVITY; + + add_size += _mali_profiling_pack_long(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s64)current_time); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, core); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s32)global_mali_profiling_counters[i].key); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, activity); + add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, + MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, pid); + + _mali_profiling_set_packet_size(mali_core_activity_stream->data + mali_core_activity_stream->used_size + 1, + add_size - STREAM_HEADER_SIZE); + + mali_core_activity_stream->used_size += add_size; + + if (0 == mali_activity_cores_num) { + _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream); + mali_core_activity_stream = NULL; + wake_up_interruptible(&stream_fd_wait_queue); + } + + break; + } + } + } +} + +static mali_bool _mali_profiling_global_counters_init(void) +{ + int core_id, counter_index, counter_number, counter_id; + u32 num_l2_cache_cores; + u32 num_pp_cores; + u32 num_gp_cores = 1; + + MALI_DEBUG_ASSERT(NULL == global_mali_profiling_counters); + num_pp_cores = mali_pp_get_glob_num_pp_cores(); + num_l2_cache_cores = mali_l2_cache_core_get_glob_num_l2_cores(); + + num_global_mali_profiling_counters = 3 * (num_gp_cores + num_pp_cores) + 2 * num_l2_cache_cores + + MALI_PROFILING_SW_COUNTERS_NUM + + MALI_PROFILING_SPECIAL_COUNTERS_NUM + + MALI_PROFILING_MEM_COUNTERS_NUM; + global_mali_profiling_counters = _mali_osk_calloc(num_global_mali_profiling_counters, sizeof(mali_profiling_counter)); + + if (NULL == global_mali_profiling_counters) + return MALI_FALSE; + + counter_index = 0; + /*Vertex processor counters */ + for (core_id = 0; core_id < num_gp_cores; core_id ++) { + global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_VP_0 + core_id; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_active", mali_name, core_id); + + for (counter_number = 0; counter_number < 2; counter_number++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = COUNTER_VP_0_C0 + (2 * core_id) + counter_number; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_cnt%d", mali_name, core_id, counter_number); + } + } + + /* Fragment processors' counters */ + for (core_id = 0; core_id < num_pp_cores; core_id++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_FP_0 + core_id; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_active", mali_name, core_id); + + for (counter_number = 0; counter_number < 2; counter_number++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = COUNTER_FP_0_C0 + (2 * core_id) + counter_number; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_cnt%d", mali_name, core_id, counter_number); + } + } + + /* L2 Cache counters */ + for (core_id = 0; core_id < num_l2_cache_cores; core_id++) { + for (counter_number = 0; counter_number < 2; counter_number++) { + counter_index++; + global_mali_profiling_counters[counter_index].counter_id = COUNTER_L2_0_C0 + (2 * core_id) + counter_number; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_L2_%d_cnt%d", mali_name, core_id, counter_number); + } + } + + /* Now set up the software counter entries */ + for (counter_id = FIRST_SW_COUNTER; counter_id <= LAST_SW_COUNTER; counter_id++) { + counter_index++; + + if (0 == first_sw_counter_index) + first_sw_counter_index = counter_index; + + global_mali_profiling_counters[counter_index].counter_id = counter_id; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_SW_%d", mali_name, counter_id - FIRST_SW_COUNTER); + } + + /* Now set up the special counter entries */ + for (counter_id = FIRST_SPECIAL_COUNTER; counter_id <= LAST_SPECIAL_COUNTER; counter_id++) { + + counter_index++; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s", + mali_name, _mali_special_counter_descriptions[counter_id - FIRST_SPECIAL_COUNTER]); + + global_mali_profiling_counters[counter_index].counter_id = counter_id; + } + + /* Now set up the mem counter entries*/ + for (counter_id = FIRST_MEM_COUNTER; counter_id <= LAST_MEM_COUNTER; counter_id++) { + + counter_index++; + _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, + sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s", + mali_name, _mali_mem_counter_descriptions[counter_id - FIRST_MEM_COUNTER]); + + global_mali_profiling_counters[counter_index].counter_id = counter_id; + } + + MALI_DEBUG_ASSERT((counter_index + 1) == num_global_mali_profiling_counters); + + return MALI_TRUE; +} + +void _mali_profiling_notification_mem_counter(struct mali_session_data *session, u32 counter_id, u32 key, int enable) +{ + + MALI_DEBUG_ASSERT_POINTER(session); + + if (NULL != session) { + _mali_osk_notification_t *notification; + _mali_osk_notification_queue_t *queue; + + queue = session->ioctl_queue; + MALI_DEBUG_ASSERT(NULL != queue); + + notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER, + sizeof(_mali_uk_annotate_profiling_mem_counter_s)); + + if (NULL != notification) { + _mali_uk_annotate_profiling_mem_counter_s *data = notification->result_buffer; + data->counter_id = counter_id; + data->key = key; + data->enable = enable; + + _mali_osk_notification_queue_send(queue, notification); + } else { + MALI_PRINT_ERROR(("Failed to create notification object!\n")); + } + } else { + MALI_PRINT_ERROR(("Failed to find the right session!\n")); + } +} + +void _mali_profiling_notification_enable(struct mali_session_data *session, u32 sampling_rate, int enable) +{ + MALI_DEBUG_ASSERT_POINTER(session); + + if (NULL != session) { + _mali_osk_notification_t *notification; + _mali_osk_notification_queue_t *queue; + + queue = session->ioctl_queue; + MALI_DEBUG_ASSERT(NULL != queue); + + notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE, + sizeof(_mali_uk_annotate_profiling_enable_s)); + + if (NULL != notification) { + _mali_uk_annotate_profiling_enable_s *data = notification->result_buffer; + data->sampling_rate = sampling_rate; + data->enable = enable; + + _mali_osk_notification_queue_send(queue, notification); + } else { + MALI_PRINT_ERROR(("Failed to create notification object!\n")); + } + } else { + MALI_PRINT_ERROR(("Failed to find the right session!\n")); + } +} + + +_mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start) +{ + int i; + mali_profiling_stream *new_mali_profiling_stream = NULL; + mali_profiling_stream_list *new_mali_profiling_stream_list = NULL; + if (MALI_TRUE == auto_start) { + mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE); + } + + /*Init the global_mali_stream_list*/ + MALI_DEBUG_ASSERT(NULL == global_mali_stream_list); + new_mali_profiling_stream_list = (mali_profiling_stream_list *)kmalloc(sizeof(mali_profiling_stream_list), GFP_KERNEL); + + if (NULL == new_mali_profiling_stream_list) { + return _MALI_OSK_ERR_NOMEM; + } + + spin_lock_init(&new_mali_profiling_stream_list->spin_lock); + INIT_LIST_HEAD(&new_mali_profiling_stream_list->free_list); + INIT_LIST_HEAD(&new_mali_profiling_stream_list->queue_list); + + spin_lock_init(&mali_activity_lock); + mali_activity_cores_num = 0; + + for (i = 0; i < MALI_PROFILING_STREAM_BUFFER_NUM; i++) { + new_mali_profiling_stream = (mali_profiling_stream *)kmalloc(sizeof(mali_profiling_stream), GFP_KERNEL); + if (NULL == new_mali_profiling_stream) { + _mali_profiling_stream_list_destory(new_mali_profiling_stream_list); + return _MALI_OSK_ERR_NOMEM; + } + + INIT_LIST_HEAD(&new_mali_profiling_stream->list); + new_mali_profiling_stream->used_size = 0; + list_add_tail(&new_mali_profiling_stream->list, &new_mali_profiling_stream_list->free_list); + + } + + _mali_osk_atomic_init(&stream_fd_if_used, 0); + init_waitqueue_head(&stream_fd_wait_queue); + + hrtimer_init(&profiling_sampling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + + profiling_sampling_timer.function = _mali_profiling_sampling_counters; + + global_mali_stream_list = new_mali_profiling_stream_list; + + return _MALI_OSK_ERR_OK; +} + +void _mali_osk_profiling_term(void) +{ + if (0 != profiling_sample_rate) { + hrtimer_cancel(&profiling_sampling_timer); + profiling_sample_rate = 0; + } + _mali_osk_atomic_term(&stream_fd_if_used); + + if (NULL != global_mali_profiling_counters) { + _mali_osk_free(global_mali_profiling_counters); + global_mali_profiling_counters = NULL; + num_global_mali_profiling_counters = 0; + } + + if (NULL != global_mali_stream_list) { + _mali_profiling_stream_list_destory(global_mali_stream_list); + global_mali_stream_list = NULL; + } + +} + +void _mali_osk_profiling_stop_sampling(u32 pid) +{ + if (pid == current_profiling_pid) { + + int i; + /* Reset all counter states when closing connection.*/ + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER); + global_mali_profiling_counters[i].enabled = 0; + global_mali_profiling_counters[i].prev_counter_value = 0; + global_mali_profiling_counters[i].current_counter_value = 0; + } + l2_cache_counter_if_enabled = MALI_FALSE; + num_counters_enabled = 0; + mem_counters_enabled = 0; + _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0); + _mali_profiling_control(SW_COUNTER_ENABLE, 0); + /* Delete sampling timer when closing connection. */ + if (0 != profiling_sample_rate) { + hrtimer_cancel(&profiling_sampling_timer); + profiling_sample_rate = 0; + } + current_profiling_pid = 0; + } +} + +void _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4) +{ + /*Record the freq & volt to global_mali_profiling_counters here. */ + if (0 != profiling_sample_rate) { + u32 channel; + u32 state; + channel = (event_id >> 16) & 0xFF; + state = ((event_id >> 24) & 0xF) << 24; + + switch (state) { + case MALI_PROFILING_EVENT_TYPE_SINGLE: + if ((MALI_PROFILING_EVENT_CHANNEL_GPU >> 16) == channel) { + u32 reason = (event_id & 0xFFFF); + if (MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE == reason) { + _mali_osk_profiling_record_global_counters(COUNTER_FREQUENCY, data0); + _mali_osk_profiling_record_global_counters(COUNTER_VOLTAGE, data1); + } + } + break; + case MALI_PROFILING_EVENT_TYPE_START: + if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) { + _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 1, data1); + } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) && + (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) { + u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16); + _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 1, data1); + } + break; + case MALI_PROFILING_EVENT_TYPE_STOP: + if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) { + _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 0, 0); + } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) && + (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) { + u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16); + _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 0, 0); + } + break; + default: + break; + } + } + trace_mali_timeline_event(event_id, data0, data1, data2, data3, data4); +} + +void _mali_osk_profiling_report_sw_counters(u32 *counters) +{ + trace_mali_sw_counters(_mali_osk_get_pid(), _mali_osk_get_tid(), NULL, counters); +} + +void _mali_osk_profiling_record_global_counters(int counter_id, u32 value) +{ + if (NULL != global_mali_profiling_counters) { + int i ; + for (i = 0; i < num_global_mali_profiling_counters; i++) { + if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) { + global_mali_profiling_counters[i].current_counter_value = value; + break; + } + } + } +} + +_mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args) +{ + /* Always add process and thread identificator in the first two data elements for events from user space */ + _mali_osk_profiling_add_event(args->event_id, _mali_osk_get_pid(), _mali_osk_get_tid(), args->data[2], args->data[3], args->data[4]); + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s *args) +{ + u32 *counters = (u32 *)(uintptr_t)args->counters; + + _mali_osk_profiling_report_sw_counters(counters); + + if (NULL != global_mali_profiling_counters) { + int i; + for (i = 0; i < MALI_PROFILING_SW_COUNTERS_NUM; i ++) { + if (global_mali_profiling_counters[first_sw_counter_index + i].enabled) { + global_mali_profiling_counters[first_sw_counter_index + i].current_counter_value = *(counters + i); + } + } + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args) +{ + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + MALI_DEBUG_ASSERT_POINTER(session); + + if (1 == _mali_osk_atomic_inc_return(&stream_fd_if_used)) { + + s32 fd = anon_inode_getfd("[mali_profiling_stream]", &mali_profiling_stream_fops, + session, + O_RDONLY | O_CLOEXEC); + + args->stream_fd = fd; + if (0 > fd) { + _mali_osk_atomic_dec(&stream_fd_if_used); + return _MALI_OSK_ERR_FAULT; + } + args->stream_fd = fd; + } else { + _mali_osk_atomic_dec(&stream_fd_if_used); + args->stream_fd = -1; + return _MALI_OSK_ERR_BUSY; + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args) +{ + u32 control_packet_size; + u32 output_buffer_size; + + struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; + MALI_DEBUG_ASSERT_POINTER(session); + + if (NULL == global_mali_profiling_counters && MALI_FALSE == _mali_profiling_global_counters_init()) { + MALI_PRINT_ERROR(("Failed to create global_mali_profiling_counters.\n")); + return _MALI_OSK_ERR_FAULT; + } + + control_packet_size = args->control_packet_size; + output_buffer_size = args->response_packet_size; + + if (0 != control_packet_size) { + u8 control_type; + u8 *control_packet_data; + u8 *response_packet_data; + u32 version_length = sizeof(utgard_setup_version) - 1; + + control_packet_data = (u8 *)(uintptr_t)args->control_packet_data; + MALI_DEBUG_ASSERT_POINTER(control_packet_data); + response_packet_data = (u8 *)(uintptr_t)args->response_packet_data; + MALI_DEBUG_ASSERT_POINTER(response_packet_data); + + /*Decide if need to ignore Utgard setup version.*/ + if (control_packet_size >= version_length) { + if (0 == memcmp(control_packet_data, utgard_setup_version, version_length)) { + if (control_packet_size == version_length) { + args->response_packet_size = 0; + return _MALI_OSK_ERR_OK; + } else { + control_packet_data += version_length; + control_packet_size -= version_length; + } + } + } + + current_profiling_pid = _mali_osk_get_pid(); + + control_type = control_packet_data[0]; + switch (control_type) { + case PACKET_HEADER_COUNTERS_REQUEST: { + int i; + + if (PACKET_HEADER_SIZE > control_packet_size || + control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { + MALI_PRINT_ERROR(("Wrong control packet size, type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); + return _MALI_OSK_ERR_FAULT; + } + + /* Send supported counters */ + if (PACKET_HEADER_SIZE > output_buffer_size) + return _MALI_OSK_ERR_FAULT; + + *response_packet_data = PACKET_HEADER_COUNTERS_ACK; + args->response_packet_size = PACKET_HEADER_SIZE; + + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + u32 name_size = strlen(global_mali_profiling_counters[i].counter_name); + + if ((args->response_packet_size + name_size + 1) > output_buffer_size) { + MALI_PRINT_ERROR(("Response packet data is too large..\n")); + return _MALI_OSK_ERR_FAULT; + } + + memcpy(response_packet_data + args->response_packet_size, + global_mali_profiling_counters[i].counter_name, name_size + 1); + + args->response_packet_size += (name_size + 1); + + if (global_mali_profiling_counters[i].counter_id == COUNTER_VP_ACTIVITY) { + args->response_packet_size += _mali_profiling_pack_int(response_packet_data, + output_buffer_size, args->response_packet_size, (s32)1); + } else if (global_mali_profiling_counters[i].counter_id == COUNTER_FP_ACTIVITY) { + args->response_packet_size += _mali_profiling_pack_int(response_packet_data, + output_buffer_size, args->response_packet_size, (s32)mali_pp_get_glob_num_pp_cores()); + } else { + args->response_packet_size += _mali_profiling_pack_int(response_packet_data, + output_buffer_size, args->response_packet_size, (s32) - 1); + } + } + + _mali_profiling_set_packet_size(response_packet_data + 1, args->response_packet_size); + break; + } + + case PACKET_HEADER_COUNTERS_ENABLE: { + int i; + u32 request_pos = PACKET_HEADER_SIZE; + mali_bool sw_counter_if_enabled = MALI_FALSE; + + if (PACKET_HEADER_SIZE > control_packet_size || + control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { + MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); + return _MALI_OSK_ERR_FAULT; + } + + /* Init all counter states before enable requested counters.*/ + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER); + global_mali_profiling_counters[i].enabled = 0; + global_mali_profiling_counters[i].prev_counter_value = 0; + global_mali_profiling_counters[i].current_counter_value = 0; + + if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER && + global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) { + _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, 0, 0); + } + } + + l2_cache_counter_if_enabled = MALI_FALSE; + num_counters_enabled = 0; + mem_counters_enabled = 0; + _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0); + _mali_profiling_control(SW_COUNTER_ENABLE, 0); + _mali_profiling_notification_enable(session, 0, 0); + + /* Enable requested counters */ + while (request_pos < control_packet_size) { + u32 begin = request_pos; + u32 event; + u32 key; + + /* Check the counter name which should be ended with null */ + while (request_pos < control_packet_size && control_packet_data[request_pos] != '\0') { + ++request_pos; + } + + if (request_pos >= control_packet_size) + return _MALI_OSK_ERR_FAULT; + + ++request_pos; + event = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); + key = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); + + for (i = 0; i < num_global_mali_profiling_counters; ++i) { + u32 name_size = strlen((char *)(control_packet_data + begin)); + + if (strncmp(global_mali_profiling_counters[i].counter_name, (char *)(control_packet_data + begin), name_size) == 0) { + if (!sw_counter_if_enabled && (FIRST_SW_COUNTER <= global_mali_profiling_counters[i].counter_id + && global_mali_profiling_counters[i].counter_id <= LAST_SW_COUNTER)) { + sw_counter_if_enabled = MALI_TRUE; + _mali_profiling_control(SW_COUNTER_ENABLE, 1); + } + + if (COUNTER_FILMSTRIP == global_mali_profiling_counters[i].counter_id) { + _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 1); + _mali_profiling_control(FBDUMP_CONTROL_RATE, event & 0xff); + _mali_profiling_control(FBDUMP_CONTROL_RESIZE_FACTOR, (event >> 8) & 0xff); + } + + if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER && + global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) { + _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, + key, 1); + mem_counters_enabled++; + } + + global_mali_profiling_counters[i].counter_event = event; + global_mali_profiling_counters[i].key = key; + global_mali_profiling_counters[i].enabled = 1; + + _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, + global_mali_profiling_counters[i].counter_event); + num_counters_enabled++; + break; + } + } + + if (i == num_global_mali_profiling_counters) { + MALI_PRINT_ERROR(("Counter name does not match for type %u.\n", control_type)); + return _MALI_OSK_ERR_FAULT; + } + } + + if (PACKET_HEADER_SIZE <= output_buffer_size) { + *response_packet_data = PACKET_HEADER_ACK; + _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE); + args->response_packet_size = PACKET_HEADER_SIZE; + } else { + return _MALI_OSK_ERR_FAULT; + } + + break; + } + + case PACKET_HEADER_START_CAPTURE_VALUE: { + u32 live_rate; + u32 request_pos = PACKET_HEADER_SIZE; + + if (PACKET_HEADER_SIZE > control_packet_size || + control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { + MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); + return _MALI_OSK_ERR_FAULT; + } + + /* Read samping rate in nanoseconds and live rate, start capture.*/ + profiling_sample_rate = _mali_profiling_read_packet_int(control_packet_data, + &request_pos, control_packet_size); + + live_rate = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); + + if (PACKET_HEADER_SIZE <= output_buffer_size) { + *response_packet_data = PACKET_HEADER_ACK; + _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE); + args->response_packet_size = PACKET_HEADER_SIZE; + } else { + return _MALI_OSK_ERR_FAULT; + } + + if (0 != num_counters_enabled && 0 != profiling_sample_rate) { + _mali_profiling_global_stream_list_free(); + if (mem_counters_enabled > 0) { + _mali_profiling_notification_enable(session, profiling_sample_rate, 1); + } + hrtimer_start(&profiling_sampling_timer, + ktime_set(profiling_sample_rate / 1000000000, profiling_sample_rate % 1000000000), + HRTIMER_MODE_REL_PINNED); + } + + break; + } + default: + MALI_PRINT_ERROR(("Unsupported profiling packet header type %u.\n", control_type)); + args->response_packet_size = 0; + return _MALI_OSK_ERR_FAULT; + } + } else { + _mali_osk_profiling_stop_sampling(current_profiling_pid); + _mali_profiling_notification_enable(session, 0, 0); + } + + return _MALI_OSK_ERR_OK; +} + +/** + * Called by gator.ko to set HW counters + * + * @param counter_id The counter ID. + * @param event_id Event ID that the counter should count (HW counter value from TRM). + * + * @return 1 on success, 0 on failure. + */ +int _mali_profiling_set_event(u32 counter_id, s32 event_id) +{ + if (COUNTER_VP_0_C0 == counter_id) { + mali_gp_job_set_gp_counter_src0(event_id); + } else if (COUNTER_VP_0_C1 == counter_id) { + mali_gp_job_set_gp_counter_src1(event_id); + } else if (COUNTER_FP_0_C0 <= counter_id && COUNTER_FP_7_C1 >= counter_id) { + /* + * Two compatibility notes for this function: + * + * 1) Previously the DDK allowed per core counters. + * + * This did not make much sense on Mali-450 with the "virtual PP core" concept, + * so this option was removed, and only the same pair of HW counters was allowed on all cores, + * beginning with r3p2 release. + * + * Starting with r4p0, it is now possible to set different HW counters for the different sub jobs. + * This should be almost the same, since sub job 0 is designed to run on core 0, + * sub job 1 on core 1, and so on. + * + * The scheduling of PP sub jobs is not predictable, and this often led to situations where core 0 ran 2 + * sub jobs, while for instance core 1 ran zero. Having the counters set per sub job would thus increase + * the predictability of the returned data (as you would be guaranteed data for all the selected HW counters). + * + * PS: Core scaling needs to be disabled in order to use this reliably (goes for both solutions). + * + * The framework/#defines with Gator still indicates that the counter is for a particular core, + * but this is internally used as a sub job ID instead (no translation needed). + * + * 2) Global/default vs per sub job counters + * + * Releases before r3p2 had only per PP core counters. + * r3p2 releases had only one set of default/global counters which applied to all PP cores + * Starting with r4p0, we have both a set of default/global counters, + * and individual counters per sub job (equal to per core). + * + * To keep compatibility with Gator/DS-5/streamline, the following scheme is used: + * + * r3p2 release; only counters set for core 0 is handled, + * this is applied as the default/global set of counters, and will thus affect all cores. + * + * r4p0 release; counters set for core 0 is applied as both the global/default set of counters, + * and counters for sub job 0. + * Counters set for core 1-7 is only applied for the corresponding sub job. + * + * This should allow the DS-5/Streamline GUI to have a simple mode where it only allows setting the + * values for core 0, and thus this will be applied to all PP sub jobs/cores. + * Advanced mode will also be supported, where individual pairs of HW counters can be selected. + * + * The GUI will (until it is updated) still refer to cores instead of sub jobs, but this is probably + * something we can live with! + * + * Mali-450 note: Each job is not divided into a deterministic number of sub jobs, as the HW DLBU + * automatically distributes the load between whatever number of cores is available at this particular time. + * A normal PP job on Mali-450 is thus considered a single (virtual) job, and it will thus only be possible + * to use a single pair of HW counters (even if the job ran on multiple PP cores). + * In other words, only the global/default pair of PP HW counters will be used for normal Mali-450 jobs. + */ + u32 sub_job = (counter_id - COUNTER_FP_0_C0) >> 1; + u32 counter_src = (counter_id - COUNTER_FP_0_C0) & 1; + if (0 == counter_src) { + mali_pp_job_set_pp_counter_sub_job_src0(sub_job, event_id); + if (0 == sub_job) { + mali_pp_job_set_pp_counter_global_src0(event_id); + } + } else { + mali_pp_job_set_pp_counter_sub_job_src1(sub_job, event_id); + if (0 == sub_job) { + mali_pp_job_set_pp_counter_global_src1(event_id); + } + } + } else if (COUNTER_L2_0_C0 <= counter_id && COUNTER_L2_2_C1 >= counter_id) { + u32 core_id = (counter_id - COUNTER_L2_0_C0) >> 1; + struct mali_l2_cache_core *l2_cache_core = mali_l2_cache_core_get_glob_l2_core(core_id); + + if (NULL != l2_cache_core) { + u32 counter_src = (counter_id - COUNTER_L2_0_C0) & 1; + mali_l2_cache_core_set_counter_src(l2_cache_core, + counter_src, event_id); + l2_cache_counter_if_enabled = MALI_TRUE; + } + } else { + return 0; /* Failure, unknown event */ + } + + return 1; /* success */ +} + +/** + * Called by gator.ko to retrieve the L2 cache counter values for all L2 cache cores. + * The L2 cache counters are unique in that they are polled by gator, rather than being + * transmitted via the tracepoint mechanism. + * + * @param values Pointer to a _mali_profiling_l2_counter_values structure where + * the counter sources and values will be output + * @return 0 if all went well; otherwise, return the mask with the bits set for the powered off cores + */ +u32 _mali_profiling_get_l2_counters(_mali_profiling_l2_counter_values *values) +{ + u32 l2_cores_num = mali_l2_cache_core_get_glob_num_l2_cores(); + u32 i; + + MALI_DEBUG_ASSERT(l2_cores_num <= 3); + + for (i = 0; i < l2_cores_num; i++) { + struct mali_l2_cache_core *l2_cache = mali_l2_cache_core_get_glob_l2_core(i); + + if (NULL == l2_cache) { + continue; + } + + mali_l2_cache_core_get_counter_values(l2_cache, + &values->cores[i].source0, + &values->cores[i].value0, + &values->cores[i].source1, + &values->cores[i].value1); + } + + return 0; +} + +/** + * Called by gator to control the production of profiling information at runtime. + */ +void _mali_profiling_control(u32 action, u32 value) +{ + switch (action) { + case FBDUMP_CONTROL_ENABLE: + mali_set_user_setting(_MALI_UK_USER_SETTING_COLORBUFFER_CAPTURE_ENABLED, (value == 0 ? MALI_FALSE : MALI_TRUE)); + break; + case FBDUMP_CONTROL_RATE: + mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_N_FRAMES, value); + break; + case SW_COUNTER_ENABLE: + mali_set_user_setting(_MALI_UK_USER_SETTING_SW_COUNTER_ENABLED, value); + break; + case FBDUMP_CONTROL_RESIZE_FACTOR: + mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_RESIZE_FACTOR, value); + break; + default: + break; /* Ignore unimplemented actions */ + } +} + +/** + * Called by gator to get mali api version. + */ +u32 _mali_profiling_get_api_version(void) +{ + return MALI_PROFILING_API_VERSION; +} + +/** +* Called by gator to get the data about Mali instance in use: +* product id, version, number of cores +*/ +void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values) +{ + values->mali_product_id = (u32)mali_kernel_core_get_product_id(); + values->mali_version_major = mali_kernel_core_get_gpu_major_version(); + values->mali_version_minor = mali_kernel_core_get_gpu_minor_version(); + values->num_of_l2_cores = mali_l2_cache_core_get_glob_num_l2_cores(); + values->num_of_fp_cores = mali_executor_get_num_cores_total(); + values->num_of_vp_cores = 1; +} + + +EXPORT_SYMBOL(_mali_profiling_set_event); +EXPORT_SYMBOL(_mali_profiling_get_l2_counters); +EXPORT_SYMBOL(_mali_profiling_control); +EXPORT_SYMBOL(_mali_profiling_get_api_version); +EXPORT_SYMBOL(_mali_profiling_get_mali_version); diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h b/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h new file mode 100755 index 000000000000..af51161f9da1 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_specific.h + * Defines per-OS Kernel level specifics, such as unusual workarounds for + * certain OSs. + */ + +#ifndef __MALI_OSK_SPECIFIC_H__ +#define __MALI_OSK_SPECIFIC_H__ + +#include +#include +#include +#include +#include +#include + + +#include "mali_osk_types.h" +#include "mali_kernel_linux.h" + +#define MALI_STATIC_INLINE static inline +#define MALI_NON_STATIC_INLINE inline + +typedef struct dma_pool *mali_dma_pool; + +typedef u32 mali_dma_addr; + +#if MALI_ENABLE_CPU_CYCLES +/* Reads out the clock cycle performance counter of the current cpu. + It is useful for cost-free (2 cycle) measuring of the time spent + in a code path. Sample before and after, the diff number of cycles. + When the CPU is idle it will not increase this clock counter. + It means that the counter is accurate if only spin-locks are used, + but mutexes may lead to too low values since the cpu might "idle" + waiting for the mutex to become available. + The clock source is configured on the CPU during mali module load, + but will not give useful output after a CPU has been power cycled. + It is therefore important to configure the system to not turn of + the cpu cores when using this functionallity.*/ +static inline unsigned int mali_get_cpu_cyclecount(void) +{ + unsigned int value; + /* Reading the CCNT Register - CPU clock counter */ + asm volatile("MRC p15, 0, %0, c9, c13, 0\t\n": "=r"(value)); + return value; +} + +void mali_init_cpu_time_counters(int reset, int enable_divide_by_64); +#endif + + +MALI_STATIC_INLINE u32 _mali_osk_copy_from_user(void *to, void *from, u32 n) +{ + return (u32)copy_from_user(to, from, (unsigned long)n); +} + +MALI_STATIC_INLINE mali_bool _mali_osk_in_atomic(void) +{ + return in_atomic(); +} + +#define _mali_osk_put_user(x, ptr) put_user(x, ptr) + +#endif /* __MALI_OSK_SPECIFIC_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c new file mode 100755 index 000000000000..d295e712ac7a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_time.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include "mali_osk.h" +#include +#include +#include + +mali_bool _mali_osk_time_after_eq(unsigned long ticka, unsigned long tickb) +{ + return time_after_eq(ticka, tickb) ? + MALI_TRUE : MALI_FALSE; +} + +unsigned long _mali_osk_time_mstoticks(u32 ms) +{ + return msecs_to_jiffies(ms); +} + +u32 _mali_osk_time_tickstoms(unsigned long ticks) +{ + return jiffies_to_msecs(ticks); +} + +unsigned long _mali_osk_time_tickcount(void) +{ + return jiffies; +} + +void _mali_osk_time_ubusydelay(u32 usecs) +{ + udelay(usecs); +} + +u64 _mali_osk_time_get_ns(void) +{ + struct timespec64 tsval; + ktime_get_real_ts64(&tsval); + return (u64)timespec64_to_ns(&tsval); +} + +u64 _mali_osk_boot_time_get_ns(void) +{ + struct timespec64 tsval; + ktime_get_boottime_ts64(&tsval); + return (u64)timespec64_to_ns(&tsval); +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c new file mode 100755 index 000000000000..d01c1148272a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_timers.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include +#include +#include "mali_osk.h" +#include "mali_kernel_common.h" + +struct _mali_osk_timer_t_struct { + struct timer_list timer; +}; + +typedef void (*timer_timeout_function_t)(unsigned long); + +_mali_osk_timer_t *_mali_osk_timer_init(_mali_osk_timer_callback_t callback) +{ + _mali_osk_timer_t *t = (_mali_osk_timer_t *)kmalloc(sizeof(_mali_osk_timer_t), GFP_KERNEL); + if (NULL != t) + timer_setup(&t->timer, + (void (*)(struct timer_list *))callback, 0); + return t; +} + +void _mali_osk_timer_add(_mali_osk_timer_t *tim, unsigned long ticks_to_expire) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + tim->timer.expires = jiffies + ticks_to_expire; + add_timer(&(tim->timer)); +} + +void _mali_osk_timer_mod(_mali_osk_timer_t *tim, unsigned long ticks_to_expire) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + mod_timer(&(tim->timer), jiffies + ticks_to_expire); +} + +void _mali_osk_timer_del(_mali_osk_timer_t *tim) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + del_timer_sync(&(tim->timer)); +} + +void _mali_osk_timer_del_async(_mali_osk_timer_t *tim) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + del_timer(&(tim->timer)); +} + +mali_bool _mali_osk_timer_pending(_mali_osk_timer_t *tim) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + return 1 == timer_pending(&(tim->timer)); +} + +void _mali_osk_timer_setcallback(_mali_osk_timer_t *tim, _mali_osk_timer_callback_t callback, void *data) +{ + MALI_DEBUG_ASSERT_POINTER(tim); +} + +void _mali_osk_timer_term(_mali_osk_timer_t *tim) +{ + MALI_DEBUG_ASSERT_POINTER(tim); + kfree(tim); +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c new file mode 100755 index 000000000000..fa12abd3f5dc --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_wait_queue.c + * Implemenation of the OS abstraction layer for the kernel device driver + */ + +#include +#include +#include + +#include "mali_osk.h" +#include "mali_kernel_common.h" + +struct _mali_osk_wait_queue_t_struct { + wait_queue_head_t wait_queue; +}; + +_mali_osk_wait_queue_t *_mali_osk_wait_queue_init(void) +{ + _mali_osk_wait_queue_t *ret = NULL; + + ret = kmalloc(sizeof(_mali_osk_wait_queue_t), GFP_KERNEL); + + if (NULL == ret) { + return ret; + } + + init_waitqueue_head(&ret->wait_queue); + MALI_DEBUG_ASSERT(!waitqueue_active(&ret->wait_queue)); + + return ret; +} + +void _mali_osk_wait_queue_wait_event(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data) +{ + MALI_DEBUG_ASSERT_POINTER(queue); + MALI_DEBUG_PRINT(6, ("Adding to wait queue %p\n", queue)); + wait_event(queue->wait_queue, condition(data)); +} + +void _mali_osk_wait_queue_wait_event_timeout(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data, u32 timeout) +{ + MALI_DEBUG_ASSERT_POINTER(queue); + MALI_DEBUG_PRINT(6, ("Adding to wait queue %p\n", queue)); + wait_event_timeout(queue->wait_queue, condition(data), _mali_osk_time_mstoticks(timeout)); +} + +void _mali_osk_wait_queue_wake_up(_mali_osk_wait_queue_t *queue) +{ + MALI_DEBUG_ASSERT_POINTER(queue); + + /* if queue is empty, don't attempt to wake up its elements */ + if (!waitqueue_active(&queue->wait_queue)) return; + + MALI_DEBUG_PRINT(6, ("Waking up elements in wait queue %p ....\n", queue)); + + wake_up_all(&queue->wait_queue); + + MALI_DEBUG_PRINT(6, ("... elements in wait queue %p woken up\n", queue)); +} + +void _mali_osk_wait_queue_term(_mali_osk_wait_queue_t *queue) +{ + /* Parameter validation */ + MALI_DEBUG_ASSERT_POINTER(queue); + + /* Linux requires no explicit termination of wait queues */ + kfree(queue); +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c new file mode 100755 index 000000000000..d5e258a83a29 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c @@ -0,0 +1,240 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_osk_wq.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +#include /* For memory allocation */ +#include +#include +#include + +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_kernel_license.h" +#include "mali_kernel_linux.h" + +typedef struct _mali_osk_wq_work_s { + _mali_osk_wq_work_handler_t handler; + void *data; + mali_bool high_pri; + struct work_struct work_handle; +} mali_osk_wq_work_object_t; + +typedef struct _mali_osk_wq_delayed_work_s { + _mali_osk_wq_work_handler_t handler; + void *data; + struct delayed_work work; +} mali_osk_wq_delayed_work_object_t; + +#if MALI_LICENSE_IS_GPL +static struct workqueue_struct *mali_wq_normal = NULL; +static struct workqueue_struct *mali_wq_high = NULL; +#endif + +static void _mali_osk_wq_work_func(struct work_struct *work); + +_mali_osk_errcode_t _mali_osk_wq_init(void) +{ +#if MALI_LICENSE_IS_GPL + MALI_DEBUG_ASSERT(NULL == mali_wq_normal); + MALI_DEBUG_ASSERT(NULL == mali_wq_high); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36) + mali_wq_normal = alloc_workqueue("mali", WQ_UNBOUND, 0); + mali_wq_high = alloc_workqueue("mali_high_pri", WQ_HIGHPRI | WQ_UNBOUND, 0); +#else + mali_wq_normal = create_workqueue("mali"); + mali_wq_high = create_workqueue("mali_high_pri"); +#endif + if (NULL == mali_wq_normal || NULL == mali_wq_high) { + MALI_PRINT_ERROR(("Unable to create Mali workqueues\n")); + + if (mali_wq_normal) destroy_workqueue(mali_wq_normal); + if (mali_wq_high) destroy_workqueue(mali_wq_high); + + mali_wq_normal = NULL; + mali_wq_high = NULL; + + return _MALI_OSK_ERR_FAULT; + } +#endif /* MALI_LICENSE_IS_GPL */ + + return _MALI_OSK_ERR_OK; +} + +void _mali_osk_wq_flush(void) +{ +#if MALI_LICENSE_IS_GPL + flush_workqueue(mali_wq_high); + flush_workqueue(mali_wq_normal); +#else + flush_scheduled_work(); +#endif +} + +void _mali_osk_wq_term(void) +{ +#if MALI_LICENSE_IS_GPL + MALI_DEBUG_ASSERT(NULL != mali_wq_normal); + MALI_DEBUG_ASSERT(NULL != mali_wq_high); + + flush_workqueue(mali_wq_normal); + destroy_workqueue(mali_wq_normal); + + flush_workqueue(mali_wq_high); + destroy_workqueue(mali_wq_high); + + mali_wq_normal = NULL; + mali_wq_high = NULL; +#else + flush_scheduled_work(); +#endif +} + +_mali_osk_wq_work_t *_mali_osk_wq_create_work(_mali_osk_wq_work_handler_t handler, void *data) +{ + mali_osk_wq_work_object_t *work = kmalloc(sizeof(mali_osk_wq_work_object_t), GFP_KERNEL); + + if (NULL == work) return NULL; + + work->handler = handler; + work->data = data; + work->high_pri = MALI_FALSE; + + INIT_WORK(&work->work_handle, _mali_osk_wq_work_func); + + return work; +} + +_mali_osk_wq_work_t *_mali_osk_wq_create_work_high_pri(_mali_osk_wq_work_handler_t handler, void *data) +{ + mali_osk_wq_work_object_t *work = kmalloc(sizeof(mali_osk_wq_work_object_t), GFP_KERNEL); + + if (NULL == work) return NULL; + + work->handler = handler; + work->data = data; + work->high_pri = MALI_TRUE; + + INIT_WORK(&work->work_handle, _mali_osk_wq_work_func); + + return work; +} + +void _mali_osk_wq_delete_work(_mali_osk_wq_work_t *work) +{ + mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; + _mali_osk_wq_flush(); + kfree(work_object); +} + +void _mali_osk_wq_delete_work_nonflush(_mali_osk_wq_work_t *work) +{ + mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; + kfree(work_object); +} + +void _mali_osk_wq_schedule_work(_mali_osk_wq_work_t *work) +{ + mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; +#if MALI_LICENSE_IS_GPL + queue_work(mali_wq_normal, &work_object->work_handle); +#else + schedule_work(&work_object->work_handle); +#endif +} + +void _mali_osk_wq_schedule_work_high_pri(_mali_osk_wq_work_t *work) +{ + mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; +#if MALI_LICENSE_IS_GPL + queue_work(mali_wq_high, &work_object->work_handle); +#else + schedule_work(&work_object->work_handle); +#endif +} + +static void _mali_osk_wq_work_func(struct work_struct *work) +{ + mali_osk_wq_work_object_t *work_object; + + work_object = _MALI_OSK_CONTAINER_OF(work, mali_osk_wq_work_object_t, work_handle); + +#if MALI_LICENSE_IS_GPL +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) + /* We want highest Dynamic priority of the thread so that the Jobs depending + ** on this thread could be scheduled in time. Without this, this thread might + ** sometimes need to wait for some threads in user mode to finish its round-robin + ** time, causing *bubble* in the Mali pipeline. Thanks to the new implementation + ** of high-priority workqueue in new kernel, this only happens in older kernel. + */ + if (MALI_TRUE == work_object->high_pri) { + set_user_nice(current, -19); + } +#endif +#endif /* MALI_LICENSE_IS_GPL */ + + work_object->handler(work_object->data); +} + +static void _mali_osk_wq_delayed_work_func(struct work_struct *work) +{ + mali_osk_wq_delayed_work_object_t *work_object; + + work_object = _MALI_OSK_CONTAINER_OF(work, mali_osk_wq_delayed_work_object_t, work.work); + work_object->handler(work_object->data); +} + +mali_osk_wq_delayed_work_object_t *_mali_osk_wq_delayed_create_work(_mali_osk_wq_work_handler_t handler, void *data) +{ + mali_osk_wq_delayed_work_object_t *work = kmalloc(sizeof(mali_osk_wq_delayed_work_object_t), GFP_KERNEL); + + if (NULL == work) return NULL; + + work->handler = handler; + work->data = data; + + INIT_DELAYED_WORK(&work->work, _mali_osk_wq_delayed_work_func); + + return work; +} + +void _mali_osk_wq_delayed_delete_work_nonflush(_mali_osk_wq_delayed_work_t *work) +{ + mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; + kfree(work_object); +} + +void _mali_osk_wq_delayed_cancel_work_async(_mali_osk_wq_delayed_work_t *work) +{ + mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; + cancel_delayed_work(&work_object->work); +} + +void _mali_osk_wq_delayed_cancel_work_sync(_mali_osk_wq_delayed_work_t *work) +{ + mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; + cancel_delayed_work_sync(&work_object->work); +} + +void _mali_osk_wq_delayed_schedule_work(_mali_osk_wq_delayed_work_t *work, u32 delay) +{ + mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; + +#if MALI_LICENSE_IS_GPL + queue_delayed_work(mali_wq_normal, &work_object->work, delay); +#else + schedule_delayed_work(&work_object->work, delay); +#endif + +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c b/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c new file mode 100755 index 000000000000..931d7f07a1d2 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c @@ -0,0 +1,23 @@ +/** + * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_pmu_power_up_down.c + */ + +#include +#include "mali_executor.h" + +int mali_perf_set_num_pp_cores(unsigned int num_cores) +{ + return mali_executor_set_perf_level(num_cores, MALI_FALSE); +} + +EXPORT_SYMBOL(mali_perf_set_num_pp_cores); diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h new file mode 100755 index 000000000000..4661cac42b3f --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2012, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PROFILING_EVENTS_H__ +#define __MALI_PROFILING_EVENTS_H__ + +/* Simple wrapper in order to find the OS specific location of this file */ +#include + +#endif /* __MALI_PROFILING_EVENTS_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h new file mode 100755 index 000000000000..6fdaa427c4cf --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2012-2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PROFILING_GATOR_API_H__ +#define __MALI_PROFILING_GATOR_API_H__ + +/* Simple wrapper in order to find the OS specific location of this file */ +#include + +#endif /* __MALI_PROFILING_GATOR_API_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c new file mode 100755 index 000000000000..c3a526f0ad90 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c @@ -0,0 +1,275 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_osk_mali.h" +#include "mali_ukk.h" +#include "mali_timestamp.h" +#include "mali_osk_profiling.h" +#include "mali_user_settings_db.h" +#include "mali_profiling_internal.h" + +typedef struct mali_profiling_entry { + u64 timestamp; + u32 event_id; + u32 data[5]; +} mali_profiling_entry; + +typedef enum mali_profiling_state { + MALI_PROFILING_STATE_UNINITIALIZED, + MALI_PROFILING_STATE_IDLE, + MALI_PROFILING_STATE_RUNNING, + MALI_PROFILING_STATE_RETURN, +} mali_profiling_state; + +static _mali_osk_mutex_t *lock = NULL; +static mali_profiling_state prof_state = MALI_PROFILING_STATE_UNINITIALIZED; +static mali_profiling_entry *profile_entries = NULL; +static _mali_osk_atomic_t profile_insert_index; +static u32 profile_mask = 0; + +static inline void add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4); + +void probe_mali_timeline_event(void *data, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned + int d2, unsigned int d3, unsigned int d4)) +{ + add_event(event_id, d0, d1, d2, d3, d4); +} + +_mali_osk_errcode_t _mali_internal_profiling_init(mali_bool auto_start) +{ + profile_entries = NULL; + profile_mask = 0; + _mali_osk_atomic_init(&profile_insert_index, 0); + + lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_PROFILING); + if (NULL == lock) { + return _MALI_OSK_ERR_FAULT; + } + + prof_state = MALI_PROFILING_STATE_IDLE; + + if (MALI_TRUE == auto_start) { + u32 limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; /* Use maximum buffer size */ + + mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE); + if (_MALI_OSK_ERR_OK != _mali_internal_profiling_start(&limit)) { + return _MALI_OSK_ERR_FAULT; + } + } + + return _MALI_OSK_ERR_OK; +} + +void _mali_internal_profiling_term(void) +{ + u32 count; + + /* Ensure profiling is stopped */ + _mali_internal_profiling_stop(&count); + + prof_state = MALI_PROFILING_STATE_UNINITIALIZED; + + if (NULL != profile_entries) { + _mali_osk_vfree(profile_entries); + profile_entries = NULL; + } + + if (NULL != lock) { + _mali_osk_mutex_term(lock); + lock = NULL; + } +} + +_mali_osk_errcode_t _mali_internal_profiling_start(u32 *limit) +{ + _mali_osk_errcode_t ret; + mali_profiling_entry *new_profile_entries; + + _mali_osk_mutex_wait(lock); + + if (MALI_PROFILING_STATE_RUNNING == prof_state) { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_BUSY; + } + + new_profile_entries = _mali_osk_valloc(*limit * sizeof(mali_profiling_entry)); + + if (NULL == new_profile_entries) { + _mali_osk_mutex_signal(lock); + _mali_osk_vfree(new_profile_entries); + return _MALI_OSK_ERR_NOMEM; + } + + if (MALI_PROFILING_MAX_BUFFER_ENTRIES < *limit) { + *limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; + } + + profile_mask = 1; + while (profile_mask <= *limit) { + profile_mask <<= 1; + } + profile_mask >>= 1; + + *limit = profile_mask; + + profile_mask--; /* turns the power of two into a mask of one less */ + + if (MALI_PROFILING_STATE_IDLE != prof_state) { + _mali_osk_mutex_signal(lock); + _mali_osk_vfree(new_profile_entries); + return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ + } + + profile_entries = new_profile_entries; + + ret = _mali_timestamp_reset(); + + if (_MALI_OSK_ERR_OK == ret) { + prof_state = MALI_PROFILING_STATE_RUNNING; + } else { + _mali_osk_vfree(profile_entries); + profile_entries = NULL; + } + + register_trace_mali_timeline_event(probe_mali_timeline_event, NULL); + + _mali_osk_mutex_signal(lock); + return ret; +} + +static inline void add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4) +{ + u32 cur_index = (_mali_osk_atomic_inc_return(&profile_insert_index) - 1) & profile_mask; + + profile_entries[cur_index].timestamp = _mali_timestamp_get(); + profile_entries[cur_index].event_id = event_id; + profile_entries[cur_index].data[0] = data0; + profile_entries[cur_index].data[1] = data1; + profile_entries[cur_index].data[2] = data2; + profile_entries[cur_index].data[3] = data3; + profile_entries[cur_index].data[4] = data4; + + /* If event is "leave API function", add current memory usage to the event + * as data point 4. This is used in timeline profiling to indicate how + * much memory was used when leaving a function. */ + if (event_id == (MALI_PROFILING_EVENT_TYPE_SINGLE | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | MALI_PROFILING_EVENT_REASON_SINGLE_SW_LEAVE_API_FUNC)) { + profile_entries[cur_index].data[4] = _mali_ukk_report_memory_usage(); + } +} + +_mali_osk_errcode_t _mali_internal_profiling_stop(u32 *count) +{ + _mali_osk_mutex_wait(lock); + + if (MALI_PROFILING_STATE_RUNNING != prof_state) { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ + } + + /* go into return state (user to retreive events), no more events will be added after this */ + prof_state = MALI_PROFILING_STATE_RETURN; + + unregister_trace_mali_timeline_event(probe_mali_timeline_event, NULL); + + _mali_osk_mutex_signal(lock); + + tracepoint_synchronize_unregister(); + + *count = _mali_osk_atomic_read(&profile_insert_index); + if (*count > profile_mask) *count = profile_mask; + + return _MALI_OSK_ERR_OK; +} + +u32 _mali_internal_profiling_get_count(void) +{ + u32 retval = 0; + + _mali_osk_mutex_wait(lock); + if (MALI_PROFILING_STATE_RETURN == prof_state) { + retval = _mali_osk_atomic_read(&profile_insert_index); + if (retval > profile_mask) retval = profile_mask; + } + _mali_osk_mutex_signal(lock); + + return retval; +} + +_mali_osk_errcode_t _mali_internal_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]) +{ + u32 raw_index = _mali_osk_atomic_read(&profile_insert_index); + + _mali_osk_mutex_wait(lock); + + if (index < profile_mask) { + if ((raw_index & ~profile_mask) != 0) { + index += raw_index; + index &= profile_mask; + } + + if (prof_state != MALI_PROFILING_STATE_RETURN) { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ + } + + if (index >= raw_index) { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_FAULT; + } + + *timestamp = profile_entries[index].timestamp; + *event_id = profile_entries[index].event_id; + data[0] = profile_entries[index].data[0]; + data[1] = profile_entries[index].data[1]; + data[2] = profile_entries[index].data[2]; + data[3] = profile_entries[index].data[3]; + data[4] = profile_entries[index].data[4]; + } else { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_FAULT; + } + + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _mali_internal_profiling_clear(void) +{ + _mali_osk_mutex_wait(lock); + + if (MALI_PROFILING_STATE_RETURN != prof_state) { + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ + } + + prof_state = MALI_PROFILING_STATE_IDLE; + profile_mask = 0; + _mali_osk_atomic_init(&profile_insert_index, 0); + + if (NULL != profile_entries) { + _mali_osk_vfree(profile_entries); + profile_entries = NULL; + } + + _mali_osk_mutex_signal(lock); + return _MALI_OSK_ERR_OK; +} + +mali_bool _mali_internal_profiling_is_recording(void) +{ + return prof_state == MALI_PROFILING_STATE_RUNNING ? MALI_TRUE : MALI_FALSE; +} + +mali_bool _mali_internal_profiling_have_recording(void) +{ + return prof_state == MALI_PROFILING_STATE_RETURN ? MALI_TRUE : MALI_FALSE; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h new file mode 100755 index 000000000000..f17b4583307a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_PROFILING_INTERNAL_H__ +#define __MALI_PROFILING_INTERNAL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "mali_osk.h" + +int _mali_internal_profiling_init(mali_bool auto_start); +void _mali_internal_profiling_term(void); + +mali_bool _mali_internal_profiling_is_recording(void); +mali_bool _mali_internal_profiling_have_recording(void); +_mali_osk_errcode_t _mali_internal_profiling_clear(void); +_mali_osk_errcode_t _mali_internal_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]); +u32 _mali_internal_profiling_get_count(void); +int _mali_internal_profiling_stop(u32 *count); +int _mali_internal_profiling_start(u32 *limit); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_PROFILING_INTERNAL_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_sync.c b/drivers/gpu/arm/mali400/mali/linux/mali_sync.c new file mode 100755 index 000000000000..0d98b518f1ac --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_sync.c @@ -0,0 +1,665 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_sync.h" + +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_timeline.h" +#include "mali_executor.h" + +#include +#include +#include + +struct mali_sync_pt { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_pt sync_pt; +#else + struct mali_internal_sync_point sync_pt; +#endif + struct mali_sync_flag *flag; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_timeline *sync_tl; /**< Sync timeline this pt is connected to. */ +#else + struct mali_internal_sync_timeline *sync_tl; /**< Sync timeline this pt is connected to. */ +#endif +}; + +/** + * The sync flag is used to connect sync fences to the Mali Timeline system. Sync fences can be + * created from a sync flag, and when the flag is signaled, the sync fences will also be signaled. + */ +struct mali_sync_flag { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_timeline *sync_tl; /**< Sync timeline this flag is connected to. */ +#else + struct mali_internal_sync_timeline *sync_tl; /**< Sync timeline this flag is connected to. */ +#endif + u32 point; /**< Point on timeline. */ + int status; /**< 0 if unsignaled, 1 if signaled without error or negative if signaled with error. */ + struct kref refcount; /**< Reference count. */ +}; + +/** + * Mali sync timeline is used to connect mali timeline to sync_timeline. + * When fence timeout can print more detailed mali timeline system info. + */ +struct mali_sync_timeline_container { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + struct sync_timeline sync_timeline; +#else + struct mali_internal_sync_timeline sync_timeline; +#endif + struct mali_timeline *timeline; +}; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +MALI_STATIC_INLINE struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) +#else +MALI_STATIC_INLINE struct mali_sync_pt *to_mali_sync_pt(struct mali_internal_sync_point *pt) +#endif +{ + return container_of(pt, struct mali_sync_pt, sync_pt); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +MALI_STATIC_INLINE struct mali_sync_timeline_container *to_mali_sync_tl_container(struct sync_timeline *sync_tl) +#else +MALI_STATIC_INLINE struct mali_sync_timeline_container *to_mali_sync_tl_container(struct mali_internal_sync_timeline *sync_tl) +#endif +{ + return container_of(sync_tl, struct mali_sync_timeline_container, sync_timeline); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +static int timeline_has_signaled(struct sync_pt *pt) +#else +static int timeline_has_signaled(struct mali_internal_sync_point *pt) +#endif +{ + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(pt); + mpt = to_mali_sync_pt(pt); + + MALI_DEBUG_ASSERT_POINTER(mpt->flag); + + return mpt->flag->status; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +static void timeline_free_pt(struct sync_pt *pt) +#else +static void timeline_free_pt(struct mali_internal_sync_point *pt) +#endif +{ + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(pt); + mpt = to_mali_sync_pt(pt); + + mali_sync_flag_put(mpt->flag); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +static void timeline_release(struct sync_timeline *sync_timeline) +#else +static void timeline_release(struct mali_internal_sync_timeline *sync_timeline) +#endif +{ + struct mali_sync_timeline_container *mali_sync_tl = NULL; + struct mali_timeline *mali_tl = NULL; + + MALI_DEBUG_ASSERT_POINTER(sync_timeline); + + mali_sync_tl = to_mali_sync_tl_container(sync_timeline); + MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); + + mali_tl = mali_sync_tl->timeline; + + /* always signaled timeline didn't have mali container */ + if (mali_tl) { + if (NULL != mali_tl->spinlock) { + mali_spinlock_reentrant_term(mali_tl->spinlock); + } + _mali_osk_free(mali_tl); + } + + module_put(THIS_MODULE); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +static struct sync_pt *timeline_dup(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt, *new_mpt; + struct sync_pt *new_pt; + MALI_DEBUG_ASSERT_POINTER(pt); + + mpt = to_mali_sync_pt(pt); + + new_pt = sync_pt_create(mpt->sync_tl, sizeof(struct mali_sync_pt)); + if (NULL == new_pt) return NULL; + + new_mpt = to_mali_sync_pt(new_pt); + + mali_sync_flag_get(mpt->flag); + new_mpt->flag = mpt->flag; + new_mpt->sync_tl = mpt->sync_tl; + + return new_pt; +} + +static int timeline_compare(struct sync_pt *pta, struct sync_pt *ptb) +{ + struct mali_sync_pt *mpta; + struct mali_sync_pt *mptb; + u32 a, b; + + MALI_DEBUG_ASSERT_POINTER(pta); + MALI_DEBUG_ASSERT_POINTER(ptb); + mpta = to_mali_sync_pt(pta); + mptb = to_mali_sync_pt(ptb); + + MALI_DEBUG_ASSERT_POINTER(mpta->flag); + MALI_DEBUG_ASSERT_POINTER(mptb->flag); + + a = mpta->flag->point; + b = mptb->flag->point; + + if (a == b) return 0; + + return ((b - a) < (a - b) ? -1 : 1); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +static void timeline_print_pt(struct seq_file *s, struct sync_pt *sync_pt) +{ + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(s); + MALI_DEBUG_ASSERT_POINTER(sync_pt); + + mpt = to_mali_sync_pt(sync_pt); + + /* It is possible this sync point is just under construct, + * make sure the flag is valid before accessing it + */ + if (mpt->flag) { + seq_printf(s, "%u", mpt->flag->point); + } else { + seq_printf(s, "uninitialized"); + } +} + +static void timeline_print_obj(struct seq_file *s, struct sync_timeline *sync_tl) +{ + struct mali_sync_timeline_container *mali_sync_tl = NULL; + struct mali_timeline *mali_tl = NULL; + + MALI_DEBUG_ASSERT_POINTER(sync_tl); + + mali_sync_tl = to_mali_sync_tl_container(sync_tl); + MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); + + mali_tl = mali_sync_tl->timeline; + + if (NULL != mali_tl) { + seq_printf(s, "oldest (%u) ", mali_tl->point_oldest); + seq_printf(s, "next (%u)", mali_tl->point_next); + seq_printf(s, "\n"); + +#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) + { + u32 tid = _mali_osk_get_tid(); + struct mali_timeline_system *system = mali_tl->system; + + mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); + if (!mali_tl->destroyed) { + mali_spinlock_reentrant_wait(system->spinlock, tid); + mali_timeline_debug_print_timeline(mali_tl, s); + mali_spinlock_reentrant_signal(system->spinlock, tid); + } + mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); + + /* dump job queue status and group running status */ + mali_executor_status_dump(); + } +#endif + } +} +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +static void timeline_pt_value_str(struct sync_pt *pt, char *str, int size) +{ + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(str); + MALI_DEBUG_ASSERT_POINTER(pt); + + mpt = to_mali_sync_pt(pt); + + /* It is possible this sync point is just under construct, + * make sure the flag is valid before accessing it + */ + if (mpt->flag) { + _mali_osk_snprintf(str, size, "%u", mpt->flag->point); + } else { + _mali_osk_snprintf(str, size, "uninitialized"); + } +} + +static void timeline_value_str(struct sync_timeline *timeline, char *str, int size) +{ + struct mali_sync_timeline_container *mali_sync_tl = NULL; + struct mali_timeline *mali_tl = NULL; + + MALI_DEBUG_ASSERT_POINTER(timeline); + + mali_sync_tl = to_mali_sync_tl_container(timeline); + MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); + + mali_tl = mali_sync_tl->timeline; + + if (NULL != mali_tl) { + _mali_osk_snprintf(str, size, "oldest (%u) ", mali_tl->point_oldest); + _mali_osk_snprintf(str, size, "next (%u)", mali_tl->point_next); + _mali_osk_snprintf(str, size, "\n"); + +#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) + { + u32 tid = _mali_osk_get_tid(); + struct mali_timeline_system *system = mali_tl->system; + + mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); + if (!mali_tl->destroyed) { + mali_spinlock_reentrant_wait(system->spinlock, tid); + mali_timeline_debug_direct_print_timeline(mali_tl); + mali_spinlock_reentrant_signal(system->spinlock, tid); + } + mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); + + /* dump job queue status and group running status */ + mali_executor_status_dump(); + } +#endif + } +} +#else +static void timeline_print_sync_pt(struct mali_internal_sync_point *sync_pt) +{ + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(sync_pt); + + mpt = to_mali_sync_pt(sync_pt); + + if (mpt->flag) { + MALI_DEBUG_PRINT(2, ("mali_internal_sync_pt: %u\n", mpt->flag->point)); + } else { + MALI_DEBUG_PRINT(2, ("uninitialized\n", mpt->flag->point)); + } +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +static struct sync_timeline_ops mali_timeline_ops = { + .driver_name = "Mali", + .dup = timeline_dup, + .has_signaled = timeline_has_signaled, + .compare = timeline_compare, + .free_pt = timeline_free_pt, + .release_obj = timeline_release, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + .print_pt = timeline_print_pt, + .print_obj = timeline_print_obj, +#else + .pt_value_str = timeline_pt_value_str, + .timeline_value_str = timeline_value_str, +#endif +}; + +struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name) +{ + struct sync_timeline *sync_tl; + struct mali_sync_timeline_container *mali_sync_tl; + + sync_tl = sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline_container), name); + if (NULL == sync_tl) return NULL; + + mali_sync_tl = to_mali_sync_tl_container(sync_tl); + mali_sync_tl->timeline = timeline; + + /* Grab a reference on the module to ensure the callbacks are present + * as long some timeline exists. The reference is released when the + * timeline is freed. + * Since this function is called from a ioctl on an open file we know + * we already have a reference, so using __module_get is safe. */ + __module_get(THIS_MODULE); + + return sync_tl; +} + +s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence) +{ + s32 fd = -1; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) + fd = get_unused_fd(); +#else + fd = get_unused_fd_flags(0); +#endif + + if (fd < 0) { + sync_fence_put(sync_fence); + return -1; + } + sync_fence_install(sync_fence, fd); + + return fd; +} + +struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2) +{ + struct sync_fence *sync_fence; + + MALI_DEBUG_ASSERT_POINTER(sync_fence1); + MALI_DEBUG_ASSERT_POINTER(sync_fence1); + + sync_fence = sync_fence_merge("mali_merge_fence", sync_fence1, sync_fence2); + sync_fence_put(sync_fence1); + sync_fence_put(sync_fence2); + + return sync_fence; +} + +struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl) +{ + struct mali_sync_flag *flag; + struct sync_fence *sync_fence; + + MALI_DEBUG_ASSERT_POINTER(sync_tl); + + flag = mali_sync_flag_create(sync_tl, 0); + if (NULL == flag) return NULL; + + sync_fence = mali_sync_flag_create_fence(flag); + + mali_sync_flag_signal(flag, 0); + mali_sync_flag_put(flag); + + return sync_fence; +} + +struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, mali_timeline_point point) +{ + struct mali_sync_flag *flag; + + if (NULL == sync_tl) return NULL; + + flag = _mali_osk_calloc(1, sizeof(*flag)); + if (NULL == flag) return NULL; + + flag->sync_tl = sync_tl; + flag->point = point; + + flag->status = 0; + kref_init(&flag->refcount); + + return flag; +} + +/** + * Create a sync point attached to given sync flag. + * + * @note Sync points must be triggered in *exactly* the same order as they are created. + * + * @param flag Sync flag. + * @return New sync point if successful, NULL if not. + */ +static struct sync_pt *mali_sync_flag_create_pt(struct mali_sync_flag *flag) +{ + struct sync_pt *pt; + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(flag); + MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); + + pt = sync_pt_create(flag->sync_tl, sizeof(struct mali_sync_pt)); + if (NULL == pt) return NULL; + + mali_sync_flag_get(flag); + + mpt = to_mali_sync_pt(pt); + mpt->flag = flag; + mpt->sync_tl = flag->sync_tl; + + return pt; +} + +struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag) +{ + struct sync_pt *sync_pt; + struct sync_fence *sync_fence; + + MALI_DEBUG_ASSERT_POINTER(flag); + MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); + + sync_pt = mali_sync_flag_create_pt(flag); + if (NULL == sync_pt) return NULL; + + sync_fence = sync_fence_create("mali_flag_fence", sync_pt); + if (NULL == sync_fence) { + sync_pt_free(sync_pt); + return NULL; + } + + return sync_fence; +} +#else +static struct mali_internal_sync_timeline_ops mali_timeline_ops = { + .driver_name = "Mali", + .has_signaled = timeline_has_signaled, + .free_pt = timeline_free_pt, + .release_obj = timeline_release, + .print_sync_pt = timeline_print_sync_pt, +}; + +struct mali_internal_sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name) +{ + struct mali_internal_sync_timeline *sync_tl; + struct mali_sync_timeline_container *mali_sync_tl; + + sync_tl = mali_internal_sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline_container), name); + if (NULL == sync_tl) return NULL; + + mali_sync_tl = to_mali_sync_tl_container(sync_tl); + mali_sync_tl->timeline = timeline; + + /* Grab a reference on the module to ensure the callbacks are present + * as long some timeline exists. The reference is released when the + * timeline is freed. + * Since this function is called from a ioctl on an open file we know + * we already have a reference, so using __module_get is safe. */ + __module_get(THIS_MODULE); + + return sync_tl; +} + +s32 mali_sync_fence_fd_alloc(struct mali_internal_sync_fence *sync_fence) +{ + s32 fd = -1; + + fd = get_unused_fd_flags(0); + + if (fd < 0) { + fput(sync_fence->file); + return -1; + } + fd_install(fd, sync_fence->file); + return fd; +} + +struct mali_internal_sync_fence *mali_sync_fence_merge(struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2) +{ + struct mali_internal_sync_fence *sync_fence; + + MALI_DEBUG_ASSERT_POINTER(sync_fence1); + MALI_DEBUG_ASSERT_POINTER(sync_fence1); + + sync_fence = mali_internal_sync_fence_merge(sync_fence1, sync_fence2); + fput(sync_fence1->file); + fput(sync_fence2->file); + + return sync_fence; +} + +struct mali_internal_sync_fence *mali_sync_timeline_create_signaled_fence(struct mali_internal_sync_timeline *sync_tl) +{ + struct mali_sync_flag *flag; + struct mali_internal_sync_fence *sync_fence; + + MALI_DEBUG_ASSERT_POINTER(sync_tl); + + flag = mali_sync_flag_create(sync_tl, 0); + if (NULL == flag) return NULL; + + sync_fence = mali_sync_flag_create_fence(flag); + + mali_sync_flag_signal(flag, 0); + mali_sync_flag_put(flag); + + return sync_fence; +} + +struct mali_sync_flag *mali_sync_flag_create(struct mali_internal_sync_timeline *sync_tl, mali_timeline_point point) +{ + struct mali_sync_flag *flag; + + if (NULL == sync_tl) return NULL; + + flag = _mali_osk_calloc(1, sizeof(*flag)); + if (NULL == flag) return NULL; + + flag->sync_tl = sync_tl; + flag->point = point; + + flag->status = 0; + kref_init(&flag->refcount); + + return flag; +} + +/** + * Create a sync point attached to given sync flag. + * + * @note Sync points must be triggered in *exactly* the same order as they are created. + * + * @param flag Sync flag. + * @return New sync point if successful, NULL if not. + */ +static struct mali_internal_sync_point *mali_sync_flag_create_pt(struct mali_sync_flag *flag) +{ + struct mali_internal_sync_point *pt; + struct mali_sync_pt *mpt; + + MALI_DEBUG_ASSERT_POINTER(flag); + MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); + + pt = mali_internal_sync_point_create(flag->sync_tl, sizeof(struct mali_sync_pt)); + + if (pt == NULL) { + MALI_PRINT_ERROR(("Mali sync: sync_pt creation failed\n")); + return NULL; + } + mali_sync_flag_get(flag); + + mpt = to_mali_sync_pt(pt); + mpt->flag = flag; + mpt->sync_tl = flag->sync_tl; + + return pt; +} + +struct mali_internal_sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag) +{ + struct mali_internal_sync_point *sync_pt; + struct mali_internal_sync_fence *sync_fence; + + MALI_DEBUG_ASSERT_POINTER(flag); + MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); + + sync_pt = mali_sync_flag_create_pt(flag); + if (NULL == sync_pt) { + MALI_PRINT_ERROR(("Mali sync: sync_pt creation failed\n")); + return NULL; + } + sync_fence = (struct mali_internal_sync_fence *)sync_file_create(&sync_pt->base); + if (NULL == sync_fence) { + MALI_PRINT_ERROR(("Mali sync: sync_fence creation failed\n")); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + dma_fence_put(&sync_pt->base); +#else + fence_put(&sync_pt->base); +#endif + return NULL; + } + + /* 'sync_pt' no longer needs to hold a refcount of '*sync_pt', to put it off. */ + dma_fence_put(&sync_pt->base); + sync_pt = NULL; + + return sync_fence; +} +#endif + +void mali_sync_flag_get(struct mali_sync_flag *flag) +{ + MALI_DEBUG_ASSERT_POINTER(flag); + kref_get(&flag->refcount); +} + +/** + * Free sync flag. + * + * @param ref kref object embedded in sync flag that should be freed. + */ +static void mali_sync_flag_free(struct kref *ref) +{ + struct mali_sync_flag *flag; + + MALI_DEBUG_ASSERT_POINTER(ref); + flag = container_of(ref, struct mali_sync_flag, refcount); + + _mali_osk_free(flag); +} + +void mali_sync_flag_put(struct mali_sync_flag *flag) +{ + MALI_DEBUG_ASSERT_POINTER(flag); + kref_put(&flag->refcount, mali_sync_flag_free); +} + +void mali_sync_flag_signal(struct mali_sync_flag *flag, int error) +{ + MALI_DEBUG_ASSERT_POINTER(flag); + + MALI_DEBUG_ASSERT(0 == flag->status); + flag->status = (0 > error) ? error : 1; + + _mali_osk_write_mem_barrier(); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + sync_timeline_signal(flag->sync_tl); +#else + mali_internal_sync_timeline_signal(flag->sync_tl); +#endif +} + + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_sync.h b/drivers/gpu/arm/mali400/mali/linux/mali_sync.h new file mode 100755 index 000000000000..91be8b9cf314 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_sync.h @@ -0,0 +1,169 @@ +/* + * Copyright (C) 2012-2015, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_sync.h + * + * Mali interface for Linux sync objects. + */ + +#ifndef _MALI_SYNC_H_ +#define _MALI_SYNC_H_ + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + +#include +#include + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) +#include +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +#include +#else +#include "mali_internal_sync.h" +#endif + + +#include "mali_osk.h" + +struct mali_sync_flag; +struct mali_timeline; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +/** + * Create a sync timeline. + * + * @param name Name of the sync timeline. + * @return The new sync timeline if successful, NULL if not. + */ +struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name); + +/** + * Creates a file descriptor representing the sync fence. Will release sync fence if allocation of + * file descriptor fails. + * + * @param sync_fence Sync fence. + * @return File descriptor representing sync fence if successful, or -1 if not. + */ +s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence); + +/** + * Merges two sync fences. Both input sync fences will be released. + * + * @param sync_fence1 First sync fence. + * @param sync_fence2 Second sync fence. + * @return New sync fence that is the result of the merger if successful, or NULL if not. + */ +struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2); + +/** + * Create a sync fence that is already signaled. + * + * @param tl Sync timeline. + * @return New signaled sync fence if successful, NULL if not. + */ +struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl); + + +/** + * Create a sync flag. + * + * @param sync_tl Sync timeline. + * @param point Point on Mali timeline. + * @return New sync flag if successful, NULL if not. + */ +struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, u32 point); + +/** + * Create a sync fence attached to given sync flag. + * + * @param flag Sync flag. + * @return New sync fence if successful, NULL if not. + */ +struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag); +#else +/** + * Create a sync timeline. + * + * @param name Name of the sync timeline. + * @return The new sync timeline if successful, NULL if not. + */ +struct mali_internal_sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name); + +/** + * Creates a file descriptor representing the sync fence. Will release sync fence if allocation of + * file descriptor fails. + * + * @param sync_fence Sync fence. + * @return File descriptor representing sync fence if successful, or -1 if not. + */ +s32 mali_sync_fence_fd_alloc(struct mali_internal_sync_fence *sync_fence); + +/** + * Merges two sync fences. Both input sync fences will be released. + * + * @param sync_fence1 First sync fence. + * @param sync_fence2 Second sync fence. + * @return New sync fence that is the result of the merger if successful, or NULL if not. + */ +struct mali_internal_sync_fence *mali_sync_fence_merge(struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2); + +/** + * Create a sync fence that is already signaled. + * + * @param tl Sync timeline. + * @return New signaled sync fence if successful, NULL if not. + */ +struct mali_internal_sync_fence *mali_sync_timeline_create_signaled_fence(struct mali_internal_sync_timeline *sync_tl); + + +/** + * Create a sync flag. + * + * @param sync_tl Sync timeline. + * @param point Point on Mali timeline. + * @return New sync flag if successful, NULL if not. + */ +struct mali_sync_flag *mali_sync_flag_create(struct mali_internal_sync_timeline *sync_tl, u32 point); + +/** + * Create a sync fence attached to given sync flag. + * + * @param flag Sync flag. + * @return New sync fence if successful, NULL if not. + */ +struct mali_internal_sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag); + +#endif +/** + * Grab sync flag reference. + * + * @param flag Sync flag. + */ +void mali_sync_flag_get(struct mali_sync_flag *flag); + +/** + * Release sync flag reference. If this was the last reference, the sync flag will be freed. + * + * @param flag Sync flag. + */ +void mali_sync_flag_put(struct mali_sync_flag *flag); + +/** + * Signal sync flag. All sync fences created from this flag will be signaled. + * + * @param flag Sync flag to signal. + * @param error Negative error code, or 0 if no error. + */ +void mali_sync_flag_signal(struct mali_sync_flag *flag, int error); + +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + +#endif /* _MALI_SYNC_H_ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h b/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h new file mode 100755 index 000000000000..68b27b8be067 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2012, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_UK_TYPES_H__ +#define __MALI_UK_TYPES_H__ + +/* Simple wrapper in order to find the OS specific location of this file */ +#include + +#endif /* __MALI_UK_TYPES_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c new file mode 100755 index 000000000000..0bd1cddb10c6 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* memort allocation functions */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +int get_api_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_s __user *uargs) +{ + _mali_uk_get_api_version_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != get_user(kargs.version, &uargs->version)) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_api_version(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; + if (0 != put_user(kargs.compatible, &uargs->compatible)) return -EFAULT; + + return 0; +} + +int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_v2_s __user *uargs) +{ + _mali_uk_get_api_version_v2_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != get_user(kargs.version, &uargs->version)) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_api_version_v2(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; + if (0 != put_user(kargs.compatible, &uargs->compatible)) return -EFAULT; + + return 0; +} + +/* rk_ext : 从对 r5p0-01rel0 集æˆå¼€å§‹, ä¸å†ä½¿ç”¨. */ +#if 0 +#define mali400_in_rk30_version 0x01 +int get_mali_version_in_rk30_wrapper(struct mali_session_data *session_data, _mali_uk_get_mali_version_in_rk30_s __user *uargs) +{ + _mali_uk_get_mali_version_in_rk30_s kargs; + MALI_CHECK_NON_NULL(uargs, -EINVAL); + kargs.ctx = (uintptr_t)session_data; + kargs.version = mali400_in_rk30_version; + if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; + return 0; +} +#else +#include "../platform/rk/rk_ext.h" +int get_rk_ko_version_wrapper(struct mali_session_data *session_data, _mali_rk_ko_version_s __user *uargs) +{ + _mali_rk_ko_version_s kargs; + MALI_CHECK_NON_NULL(uargs, -EINVAL); + kargs.ctx = (uintptr_t)session_data; + kargs.version = RK_KO_VER; + if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; + return 0; +} +#endif + +int wait_for_notification_wrapper(struct mali_session_data *session_data, _mali_uk_wait_for_notification_s __user *uargs) +{ + _mali_uk_wait_for_notification_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_wait_for_notification(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (_MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS != kargs.type) { + kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_wait_for_notification_s))) return -EFAULT; + } else { + if (0 != put_user(kargs.type, &uargs->type)) return -EFAULT; + } + + return 0; +} + +int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs) +{ + _mali_uk_post_notification_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + + if (0 != get_user(kargs.type, &uargs->type)) { + return -EFAULT; + } + + err = _mali_ukk_post_notification(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + +int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs) +{ + _mali_uk_get_user_settings_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_user_settings(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + kargs.ctx = 0; /* prevent kernel address to be returned to user space */ + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_get_user_settings_s))) return -EFAULT; + + return 0; +} + +int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs) +{ + _mali_uk_request_high_priority_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_request_high_priority(&kargs); + + kargs.ctx = 0; + + return map_errcode(err); +} + +int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs) +{ + _mali_uk_pending_submit_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_pending_submit(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c new file mode 100755 index 000000000000..68fcd971938a --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +int gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_gp_start_job_s __user *uargs) +{ + _mali_osk_errcode_t err; + + /* If the job was started successfully, 0 is returned. If there was an error, but the job + * was started, we return -ENOENT. For anything else returned, the job was not started. */ + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + err = _mali_ukk_gp_start_job(session_data, uargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} + +int gp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_core_version_s __user *uargs) +{ + _mali_uk_get_gp_core_version_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_gp_core_version(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + /* no known transactions to roll-back */ + + if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; + + return 0; +} + +int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk_gp_suspend_response_s __user *uargs) +{ + _mali_uk_gp_suspend_response_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_gp_suspend_response_s))) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_gp_suspend_response(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (0 != put_user(kargs.cookie, &uargs->cookie)) return -EFAULT; + + /* no known transactions to roll-back */ + return 0; +} + +int gp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_number_of_cores_s __user *uargs) +{ + _mali_uk_get_gp_number_of_cores_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_gp_number_of_cores(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + /* no known transactions to roll-back */ + + if (0 != put_user(kargs.number_of_cores, &uargs->number_of_cores)) return -EFAULT; + + return 0; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c new file mode 100755 index 000000000000..baea4c688db9 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c @@ -0,0 +1,333 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs) +{ + _mali_uk_alloc_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_alloc_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_allocate(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) { + return -EFAULT; + } + + return 0; +} + +int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs) +{ + _mali_uk_free_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_free_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_free(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.free_pages_nr, &uargs->free_pages_nr)) { + return -EFAULT; + } + + return 0; +} + +int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs) +{ + _mali_uk_bind_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_bind_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_bind(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + +int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs) +{ + _mali_uk_unbind_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_unbind_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_unbind(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + + +int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs) +{ + _mali_uk_cow_mem_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_mem_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_cow(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) { + return -EFAULT; + } + + return 0; +} + +int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs) +{ + _mali_uk_cow_modify_range_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_modify_range_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_cow_modify_range(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.change_pages_nr, &uargs->change_pages_nr)) { + return -EFAULT; + } + return 0; +} + + +int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs) +{ + _mali_uk_mem_resize_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_resize_s))) { + return -EFAULT; + } + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_mem_resize(&kargs); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + +int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs) +{ + _mali_uk_mem_write_safe_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_write_safe_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + + /* Check if we can access the buffers */ + if (!access_ok((const void *)(uintptr_t)kargs.dest, kargs.size) || + !access_ok((const void *)(uintptr_t)kargs.src, kargs.size)) { + return -EINVAL; + } + + /* Check if size wraps */ + if ((kargs.size + kargs.dest) <= kargs.dest + || (kargs.size + kargs.src) <= kargs.src) { + return -EINVAL; + } + + err = _mali_ukk_mem_write_safe(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != put_user(kargs.size, &uargs->size)) { + return -EFAULT; + } + + return 0; +} + + + +int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs) +{ + _mali_uk_query_mmu_page_table_dump_size_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_query_mmu_page_table_dump_size(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (0 != put_user(kargs.size, &uargs->size)) return -EFAULT; + + return 0; +} + +int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs) +{ + _mali_uk_dump_mmu_page_table_s kargs; + _mali_osk_errcode_t err; + void __user *user_buffer; + void *buffer = NULL; + int rc = -EFAULT; + + /* validate input */ + MALI_CHECK_NON_NULL(uargs, -EINVAL); + /* the session_data pointer was validated by caller */ + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_dump_mmu_page_table_s))) + goto err_exit; + + user_buffer = (void __user *)(uintptr_t)kargs.buffer; + if (!access_ok(user_buffer, kargs.size)) + goto err_exit; + + /* allocate temporary buffer (kernel side) to store mmu page table info */ + if (kargs.size <= 0) + return -EINVAL; + /* Allow at most 8MiB buffers, this is more than enough to dump a fully + * populated page table. */ + if (kargs.size > SZ_8M) + return -EINVAL; + + buffer = (void *)(uintptr_t)_mali_osk_valloc(kargs.size); + if (NULL == buffer) { + rc = -ENOMEM; + goto err_exit; + } + + kargs.ctx = (uintptr_t)session_data; + kargs.buffer = (uintptr_t)buffer; + err = _mali_ukk_dump_mmu_page_table(&kargs); + if (_MALI_OSK_ERR_OK != err) { + rc = map_errcode(err); + goto err_exit; + } + + /* copy mmu page table info back to user space and update pointers */ + if (0 != copy_to_user(user_buffer, buffer, kargs.size)) + goto err_exit; + + kargs.register_writes = kargs.register_writes - + (uintptr_t)buffer + (uintptr_t)user_buffer; + kargs.page_table_dump = kargs.page_table_dump - + (uintptr_t)buffer + (uintptr_t)user_buffer; + + if (0 != copy_to_user(uargs, &kargs, sizeof(kargs))) + goto err_exit; + + rc = 0; + +err_exit: + if (buffer) _mali_osk_vfree(buffer); + return rc; +} + +int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs) +{ + _mali_osk_errcode_t err; + _mali_uk_profiling_memory_usage_get_s kargs; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_mem_usage_get(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { + return -EFAULT; + } + + return 0; +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c new file mode 100755 index 000000000000..a9b0958c06aa --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs) +{ + _mali_osk_errcode_t err; + + /* If the job was started successfully, 0 is returned. If there was an error, but the job + * was started, we return -ENOENT. For anything else returned, the job was not started. */ + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + err = _mali_ukk_pp_start_job(session_data, uargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} + +int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs) +{ + _mali_osk_errcode_t err; + + /* If the jobs were started successfully, 0 is returned. If there was an error, but the + * jobs were started, we return -ENOENT. For anything else returned, the jobs were not + * started. */ + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + err = _mali_ukk_pp_and_gp_start_job(session_data, uargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + return 0; +} + +int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs) +{ + _mali_uk_get_pp_number_of_cores_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + + err = _mali_ukk_get_pp_number_of_cores(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_get_pp_number_of_cores_s))) { + return -EFAULT; + } + + return 0; +} + +int pp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_core_version_s __user *uargs) +{ + _mali_uk_get_pp_core_version_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_get_pp_core_version(&kargs); + if (_MALI_OSK_ERR_OK != err) return map_errcode(err); + + if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; + + return 0; +} + +int pp_disable_wb_wrapper(struct mali_session_data *session_data, _mali_uk_pp_disable_wb_s __user *uargs) +{ + _mali_uk_pp_disable_wb_s kargs; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session_data, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_pp_disable_wb_s))) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + _mali_ukk_pp_job_disable_wb(&kargs); + + return 0; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c new file mode 100755 index 000000000000..8b49ebc50b95 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c @@ -0,0 +1,183 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ +#include + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs) +{ + _mali_uk_profiling_add_event_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_add_event_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_profiling_add_event(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + +int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs) +{ + _mali_uk_sw_counters_report_s kargs; + _mali_osk_errcode_t err; + u32 *counter_buffer; + u32 __user *counters; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_sw_counters_report_s))) { + return -EFAULT; + } + + /* make sure that kargs.num_counters is [at least somewhat] sane */ + if (kargs.num_counters > 10000) { + MALI_DEBUG_PRINT(1, ("User space attempted to allocate too many counters.\n")); + return -EINVAL; + } + + counter_buffer = (u32 *)kmalloc(sizeof(u32) * kargs.num_counters, GFP_KERNEL); + if (NULL == counter_buffer) { + return -ENOMEM; + } + + counters = (u32 *)(uintptr_t)kargs.counters; + + if (0 != copy_from_user(counter_buffer, counters, sizeof(u32) * kargs.num_counters)) { + kfree(counter_buffer); + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + kargs.counters = (uintptr_t)counter_buffer; + + err = _mali_ukk_sw_counters_report(&kargs); + + kfree(counter_buffer); + + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + +int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs) +{ + _mali_uk_profiling_stream_fd_get_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_profiling_stream_fd_get(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) { + return -EFAULT; + } + + return 0; +} + +int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs) +{ + _mali_uk_profiling_control_set_s kargs; + _mali_osk_errcode_t err; + u8 *kernel_control_data = NULL; + u8 *kernel_response_data = NULL; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != get_user(kargs.control_packet_size, &uargs->control_packet_size)) return -EFAULT; + if (0 != get_user(kargs.response_packet_size, &uargs->response_packet_size)) return -EFAULT; + + kargs.ctx = (uintptr_t)session_data; + + + /* Sanity check about the size */ + if (kargs.control_packet_size > PAGE_SIZE || kargs.response_packet_size > PAGE_SIZE) + return -EINVAL; + + if (0 != kargs.control_packet_size) { + + if (0 == kargs.response_packet_size) + return -EINVAL; + + kernel_control_data = _mali_osk_calloc(1, kargs.control_packet_size); + if (NULL == kernel_control_data) { + return -ENOMEM; + } + + kernel_response_data = _mali_osk_calloc(1, kargs.response_packet_size); + if (NULL == kernel_response_data) { + _mali_osk_free(kernel_control_data); + return -ENOMEM; + } + + kargs.control_packet_data = (uintptr_t)kernel_control_data; + kargs.response_packet_data = (uintptr_t)kernel_response_data; + + if (0 != copy_from_user((void *)(uintptr_t)kernel_control_data, (void *)(uintptr_t)uargs->control_packet_data, kargs.control_packet_size)) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return -EFAULT; + } + + err = _mali_ukk_profiling_control_set(&kargs); + if (_MALI_OSK_ERR_OK != err) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return map_errcode(err); + } + + if (0 != kargs.response_packet_size && 0 != copy_to_user(((void *)(uintptr_t)uargs->response_packet_data), ((void *)(uintptr_t)kargs.response_packet_data), kargs.response_packet_size)) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return -EFAULT; + } + + if (0 != put_user(kargs.response_packet_size, &uargs->response_packet_size)) { + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + return -EFAULT; + } + + _mali_osk_free(kernel_control_data); + _mali_osk_free(kernel_response_data); + } else { + + err = _mali_ukk_profiling_control_set(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + } + return 0; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c new file mode 100755 index 000000000000..1dd4a7c6fb1e --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +#include "mali_soft_job.h" +#include "mali_timeline.h" + +int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs) +{ + _mali_uk_soft_job_start_s kargs; + u32 type, point; + u64 user_job; + struct mali_timeline_fence fence; + struct mali_soft_job *job = NULL; + u32 __user *job_id_ptr = NULL; + + /* If the job was started successfully, 0 is returned. If there was an error, but the job + * was started, we return -ENOENT. For anything else returned, the job was not started. */ + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + MALI_CHECK_NON_NULL(session, -EINVAL); + + MALI_DEBUG_ASSERT_POINTER(session->soft_job_system); + + if (0 != copy_from_user(&kargs, uargs, sizeof(kargs))) { + return -EFAULT; + } + + type = kargs.type; + user_job = kargs.user_job; + job_id_ptr = (u32 __user *)(uintptr_t)kargs.job_id_ptr; + + mali_timeline_fence_copy_uk_fence(&fence, &kargs.fence); + + if ((MALI_SOFT_JOB_TYPE_USER_SIGNALED != type) && (MALI_SOFT_JOB_TYPE_SELF_SIGNALED != type)) { + MALI_DEBUG_PRINT_ERROR(("Invalid soft job type specified\n")); + return -EINVAL; + } + + /* Create soft job. */ + job = mali_soft_job_create(session->soft_job_system, (enum mali_soft_job_type)type, user_job); + if (unlikely(NULL == job)) { + return map_errcode(_MALI_OSK_ERR_NOMEM); + } + + /* Write job id back to user space. */ + if (0 != put_user(job->id, job_id_ptr)) { + MALI_PRINT_ERROR(("Mali Soft Job: failed to put job id")); + mali_soft_job_destroy(job); + return map_errcode(_MALI_OSK_ERR_NOMEM); + } + + /* Start soft job. */ + point = mali_soft_job_start(job, &fence); + + if (0 != put_user(point, &uargs->point)) { + /* Let user space know that something failed after the job was started. */ + return -ENOENT; + } + + return 0; +} + +int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs) +{ + u32 job_id; + _mali_osk_errcode_t err; + + MALI_DEBUG_ASSERT_POINTER(session); + + if (0 != get_user(job_id, &uargs->job_id)) return -EFAULT; + + err = mali_soft_job_system_signal_job(session->soft_job_system, job_id); + + return map_errcode(err); +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c new file mode 100755 index 000000000000..ff0c909393a4 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + +#include "mali_timeline.h" +#include "mali_timeline_fence_wait.h" +#include "mali_timeline_sync_fence.h" + +int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs) +{ + u32 val; + mali_timeline_id timeline; + mali_timeline_point point; + + MALI_DEBUG_ASSERT_POINTER(session); + + if (0 != get_user(val, &uargs->timeline)) return -EFAULT; + + if (MALI_UK_TIMELINE_MAX <= val) { + return -EINVAL; + } + + timeline = (mali_timeline_id)val; + + point = mali_timeline_system_get_latest_point(session->timeline_system, timeline); + + if (0 != put_user(point, &uargs->point)) return -EFAULT; + + return 0; +} + +int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs) +{ + u32 timeout, status; + mali_bool ret; + _mali_uk_fence_t uk_fence; + struct mali_timeline_fence fence; + + MALI_DEBUG_ASSERT_POINTER(session); + + if (0 != copy_from_user(&uk_fence, &uargs->fence, sizeof(_mali_uk_fence_t))) return -EFAULT; + if (0 != get_user(timeout, &uargs->timeout)) return -EFAULT; + + mali_timeline_fence_copy_uk_fence(&fence, &uk_fence); + + ret = mali_timeline_fence_wait(session->timeline_system, &fence, timeout); + status = (MALI_TRUE == ret ? 1 : 0); + + if (0 != put_user(status, &uargs->status)) return -EFAULT; + + return 0; +} + +int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs) +{ + s32 sync_fd = -1; + _mali_uk_fence_t uk_fence; + struct mali_timeline_fence fence; + + MALI_DEBUG_ASSERT_POINTER(session); + + if (0 != copy_from_user(&uk_fence, &uargs->fence, sizeof(_mali_uk_fence_t))) return -EFAULT; + mali_timeline_fence_copy_uk_fence(&fence, &uk_fence); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + sync_fd = mali_timeline_sync_fence_create(session->timeline_system, &fence); +#else + sync_fd = -1; +#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ + + if (0 != put_user(sync_fd, &uargs->sync_fd)) return -EFAULT; + + return 0; +} diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c new file mode 100755 index 000000000000..52519d1f96e0 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +#include /* file system operations */ +#include /* user space access */ + +#include "mali_ukk.h" +#include "mali_osk.h" +#include "mali_kernel_common.h" +#include "mali_session.h" +#include "mali_ukk_wrappers.h" + + +int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs) +{ + _mali_uk_vsync_event_report_s kargs; + _mali_osk_errcode_t err; + + MALI_CHECK_NON_NULL(uargs, -EINVAL); + + if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_vsync_event_report_s))) { + return -EFAULT; + } + + kargs.ctx = (uintptr_t)session_data; + err = _mali_ukk_vsync_event_report(&kargs); + if (_MALI_OSK_ERR_OK != err) { + return map_errcode(err); + } + + return 0; +} + diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h new file mode 100755 index 000000000000..1add628fe323 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_ukk_wrappers.h + * Defines the wrapper functions for each user-kernel function + */ + +#ifndef __MALI_UKK_WRAPPERS_H__ +#define __MALI_UKK_WRAPPERS_H__ + +#include "mali_uk_types.h" +#include "mali_osk.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int wait_for_notification_wrapper(struct mali_session_data *session_data, _mali_uk_wait_for_notification_s __user *uargs); +int get_api_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_s __user *uargs); +int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_v2_s __user *uargs); +int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs); +int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs); +int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs); +int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs); + +/* rk_ext : 从对 r5p0-01rel0 集æˆå¼€å§‹, ä¸å†ä½¿ç”¨. */ +#if 0 +int get_mali_version_in_rk30_wrapper(struct mali_session_data *session_data, _mali_uk_get_mali_version_in_rk30_s __user *uargs); +#else +int get_rk_ko_version_wrapper(struct mali_session_data *session_data, _mali_rk_ko_version_s __user *uargs); +#endif + +int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs); +int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs); +int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs); +int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs); +int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs); +int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs); +int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs); +int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs); +int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs); +int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs); +int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs); + +int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs); +int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs); +int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs); +int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs); +int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs); +int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs); +int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs); +int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs); +int pp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_core_version_s __user *uargs); +int pp_disable_wb_wrapper(struct mali_session_data *session_data, _mali_uk_pp_disable_wb_s __user *uargs); +int gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_gp_start_job_s __user *uargs); +int gp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_number_of_cores_s __user *uargs); +int gp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_core_version_s __user *uargs); +int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk_gp_suspend_response_s __user *uargs); + +int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs); +int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs); +int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs); +int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs); + +int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs); + + +int map_errcode(_mali_osk_errcode_t err); + +#ifdef __cplusplus +} +#endif + +#endif /* __MALI_UKK_WRAPPERS_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm.c b/drivers/gpu/arm/mali400/mali/platform/arm/arm.c new file mode 100755 index 000000000000..fc7017bbfe08 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm.c @@ -0,0 +1,629 @@ +/* + * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file mali_platform.c + * Platform specific Mali driver functions for: + * - Realview Versatile platforms with ARM11 Mpcore and virtex 5. + * - Versatile Express platforms with ARM Cortex-A9 and virtex 6. + */ +#include +#include +#include +#include "mali_kernel_linux.h" +#ifdef CONFIG_PM_RUNTIME +#include +#endif +#include +#include +#include "mali_kernel_common.h" +#include +#include + +#include "arm_core_scaling.h" +#include "mali_executor.h" + +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) +#include +#include +#endif + +static int mali_core_scaling_enable = 0; + +void mali_gpu_utilization_callback(struct mali_gpu_utilization_data *data); +static u32 mali_read_phys(u32 phys_addr); +#if defined(CONFIG_ARCH_REALVIEW) +static void mali_write_phys(u32 phys_addr, u32 value); +#endif + +#if defined(CONFIG_ARCH_VEXPRESS) && defined(CONFIG_ARM64) + +#define SECURE_MODE_CONTROL_HANDLER 0x6F02006C +void *secure_mode_mapped_addr = NULL; +/** + * Reset GPU and enable/disable Mali secure mode. + * @Return value: + * 0: success + * non-0: failure. + */ + +static int mali_gpu_reset_and_secure_mode_enable_juno(void) +{ + u32 phys_offset = SECURE_MODE_CONTROL_HANDLER & 0x00001FFF; + MALI_DEBUG_ASSERT(NULL != secure_mode_mapped_addr); + + iowrite32(1, ((u8 *)secure_mode_mapped_addr) + phys_offset); + + if (1 == (u32)ioread32(((u8 *)secure_mode_mapped_addr) + phys_offset)) { + MALI_DEBUG_PRINT(3, ("Mali reset GPU and enable secured mode successfully! \n")); + return 0; + } + + MALI_PRINT_ERROR(("Failed to reset GPU and enable Mali secured mode !!! \n")); + + return -1; + +} + +static int mali_gpu_reset_and_secure_mode_disable_juno(void) +{ + u32 phys_offset = SECURE_MODE_CONTROL_HANDLER & 0x00001FFF; + MALI_DEBUG_ASSERT(NULL != secure_mode_mapped_addr); + + iowrite32(0, ((u8 *)secure_mode_mapped_addr) + phys_offset); + + if (0 == (u32)ioread32(((u8 *)secure_mode_mapped_addr) + phys_offset)) { + MALI_DEBUG_PRINT(3, ("Mali reset GPU and disable secured mode successfully! \n")); + return 0; + } + + MALI_PRINT_ERROR(("Failed to reset GPU and disable mali secured mode !!! \n")); + return -1; +} + +static int mali_secure_mode_init_juno(void) +{ + u32 phys_addr_page = SECURE_MODE_CONTROL_HANDLER & 0xFFFFE000; + u32 phys_offset = SECURE_MODE_CONTROL_HANDLER & 0x00001FFF; + u32 map_size = phys_offset + sizeof(u32); + + MALI_DEBUG_ASSERT(NULL == secure_mode_mapped_addr); + + secure_mode_mapped_addr = ioremap(phys_addr_page, map_size); + if (NULL != secure_mode_mapped_addr) { + return mali_gpu_reset_and_secure_mode_disable_juno(); + } + MALI_DEBUG_PRINT(2, ("Failed to ioremap for Mali secured mode! \n")); + return -1; +} + +static void mali_secure_mode_deinit_juno(void) +{ + if (NULL != secure_mode_mapped_addr) { + mali_gpu_reset_and_secure_mode_disable_juno(); + iounmap(secure_mode_mapped_addr); + secure_mode_mapped_addr = NULL; + } +} +#endif + +#ifndef CONFIG_MALI_DT +static void mali_platform_device_release(struct device *device); + +#if defined(CONFIG_ARCH_VEXPRESS) + +#if defined(CONFIG_ARM64) +/* Juno + Mali-450 MP6 in V7 FPGA */ +static struct resource mali_gpu_resources_m450_mp6[] = { + MALI_GPU_RESOURCES_MALI450_MP6_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp4[] = { + MALI_GPU_RESOURCES_MALI470_MP4_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp3[] = { + MALI_GPU_RESOURCES_MALI470_MP3_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp2[] = { + MALI_GPU_RESOURCES_MALI470_MP2_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200) +}; + +static struct resource mali_gpu_resources_m470_mp1[] = { + MALI_GPU_RESOURCES_MALI470_MP1_PMU(0x6F040000, 200, 200, 200, 200, 200) +}; + +#else +static struct resource mali_gpu_resources_m450_mp8[] = { + MALI_GPU_RESOURCES_MALI450_MP8_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) +}; + +static struct resource mali_gpu_resources_m450_mp6[] = { + MALI_GPU_RESOURCES_MALI450_MP6_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) +}; + +static struct resource mali_gpu_resources_m450_mp4[] = { + MALI_GPU_RESOURCES_MALI450_MP4_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) +}; + +static struct resource mali_gpu_resources_m470_mp4[] = { + MALI_GPU_RESOURCES_MALI470_MP4_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) +}; +#endif /* CONFIG_ARM64 */ + +#elif defined(CONFIG_ARCH_REALVIEW) + +static struct resource mali_gpu_resources_m300[] = { + MALI_GPU_RESOURCES_MALI300_PMU(0xC0000000, -1, -1, -1, -1) +}; + +static struct resource mali_gpu_resources_m400_mp1[] = { + MALI_GPU_RESOURCES_MALI400_MP1_PMU(0xC0000000, -1, -1, -1, -1) +}; + +static struct resource mali_gpu_resources_m400_mp2[] = { + MALI_GPU_RESOURCES_MALI400_MP2_PMU(0xC0000000, -1, -1, -1, -1, -1, -1) +}; + +#endif +#endif + +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + +#define FALLBACK_STATIC_TEMPERATURE 55000 + +static struct thermal_zone_device *gpu_tz; + +/* Calculate gpu static power example for reference */ +static unsigned long arm_model_static_power(struct devfreq *devfreq, + unsigned long voltage) +{ + int temperature, temp; + int temp_squared, temp_cubed, temp_scaling_factor; + const unsigned long coefficient = (410UL << 20) / (729000000UL >> 10); + const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 10; + unsigned long static_power; + + if (gpu_tz) { + int ret; + + ret = gpu_tz->ops->get_temp(gpu_tz, &temperature); + if (ret) { + MALI_DEBUG_PRINT(2, ("Error reading temperature for gpu thermal zone: %d\n", ret)); + temperature = FALLBACK_STATIC_TEMPERATURE; + } + } else { + temperature = FALLBACK_STATIC_TEMPERATURE; + } + + /* Calculate the temperature scaling factor. To be applied to the + * voltage scaled power. + */ + temp = temperature / 1000; + temp_squared = temp * temp; + temp_cubed = temp_squared * temp; + temp_scaling_factor = + (2 * temp_cubed) + - (80 * temp_squared) + + (4700 * temp) + + 32000; + + static_power = (((coefficient * voltage_cubed) >> 20) + * temp_scaling_factor) + / 1000000; + + return static_power; +} + +/* Calculate gpu dynamic power example for reference */ +static unsigned long arm_model_dynamic_power(struct devfreq *devfreq, + unsigned long freq, + unsigned long voltage) +{ + /* The inputs: freq (f) is in Hz, and voltage (v) in mV. + * The coefficient (c) is in mW/(MHz mV mV). + * + * This function calculates the dynamic power after this formula: + * Pdyn (mW) = c (mW/(MHz*mV*mV)) * v (mV) * v (mV) * f (MHz) + */ + const unsigned long v2 = (voltage * voltage) / 1000; /* m*(V*V) */ + const unsigned long f_mhz = freq / 1000000; /* MHz */ + const unsigned long coefficient = 3600; /* mW/(MHz*mV*mV) */ + unsigned long dynamic_power; + + dynamic_power = (coefficient * v2 * f_mhz) / 1000000; /* mW */ + + return dynamic_power; +} + +struct devfreq_cooling_power arm_cooling_ops = { + .get_static_power = arm_model_static_power, + .get_dynamic_power = arm_model_dynamic_power, +}; +#endif + +static struct mali_gpu_device_data mali_gpu_data = { +#ifndef CONFIG_MALI_DT + .pmu_switch_delay = 0xFF, /* do not have to be this high on FPGA, but it is good for testing to have a delay */ +#if defined(CONFIG_ARCH_VEXPRESS) + .shared_mem_size = 256 * 1024 * 1024, /* 256MB */ +#endif +#endif + .max_job_runtime = 60000, /* 60 seconds */ + +#if defined(CONFIG_ARCH_REALVIEW) + .dedicated_mem_start = 0x80000000, /* Physical start address (use 0xD0000000 for old indirect setup) */ + .dedicated_mem_size = 0x10000000, /* 256MB */ +#endif +#if defined(CONFIG_ARM64) + /* Some framebuffer drivers get the framebuffer dynamically, such as through GEM, + * in which the memory resource can't be predicted in advance. + */ + .fb_start = 0x0, + .fb_size = 0xFFFFF000, +#else + .fb_start = 0xe0000000, + .fb_size = 0x01000000, +#endif + .control_interval = 1000, /* 1000ms */ + .utilization_callback = mali_gpu_utilization_callback, + .get_clock_info = NULL, + .get_freq = NULL, + .set_freq = NULL, +#if defined(CONFIG_ARCH_VEXPRESS) && defined(CONFIG_ARM64) + .secure_mode_init = mali_secure_mode_init_juno, + .secure_mode_deinit = mali_secure_mode_deinit_juno, + .gpu_reset_and_secure_mode_enable = mali_gpu_reset_and_secure_mode_enable_juno, + .gpu_reset_and_secure_mode_disable = mali_gpu_reset_and_secure_mode_disable_juno, +#else + .secure_mode_init = NULL, + .secure_mode_deinit = NULL, + .gpu_reset_and_secure_mode_enable = NULL, + .gpu_reset_and_secure_mode_disable = NULL, +#endif +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + .gpu_cooling_ops = &arm_cooling_ops, +#endif +}; + +#ifndef CONFIG_MALI_DT +static struct platform_device mali_gpu_device = { + .name = MALI_GPU_NAME_UTGARD, + .id = 0, + .dev.release = mali_platform_device_release, + .dev.dma_mask = &mali_gpu_device.dev.coherent_dma_mask, + .dev.coherent_dma_mask = DMA_BIT_MASK(32), + + .dev.platform_data = &mali_gpu_data, +}; + +int mali_platform_device_register(void) +{ + int err = -1; + int num_pp_cores = 0; +#if defined(CONFIG_ARCH_REALVIEW) + u32 m400_gp_version; +#endif + + MALI_DEBUG_PRINT(4, ("mali_platform_device_register() called\n")); + + /* Detect present Mali GPU and connect the correct resources to the device */ +#if defined(CONFIG_ARCH_VEXPRESS) + +#if defined(CONFIG_ARM64) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) + mali_gpu_device.dev.archdata.dma_ops = &dummy_dma_ops; +#else + mali_gpu_device.dev.archdata.dma_ops = dma_ops; +#endif + if ((mali_read_phys(0x6F000000) & 0x00600450) == 0x00600450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); + num_pp_cores = 6; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp6); + mali_gpu_device.resource = mali_gpu_resources_m450_mp6; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00400430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp4); + mali_gpu_device.resource = mali_gpu_resources_m470_mp4; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00300430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP3 device\n")); + num_pp_cores = 3; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp3); + mali_gpu_device.resource = mali_gpu_resources_m470_mp3; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00200430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP2 device\n")); + num_pp_cores = 2; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp2); + mali_gpu_device.resource = mali_gpu_resources_m470_mp2; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00100430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP1 device\n")); + num_pp_cores = 1; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp1); + mali_gpu_device.resource = mali_gpu_resources_m470_mp1; + } +#else + if (mali_read_phys(0xFC000000) == 0x00000450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP8 device\n")); + num_pp_cores = 8; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp8); + mali_gpu_device.resource = mali_gpu_resources_m450_mp8; + } else if (mali_read_phys(0xFC000000) == 0x40600450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); + num_pp_cores = 6; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp6); + mali_gpu_device.resource = mali_gpu_resources_m450_mp6; + } else if (mali_read_phys(0xFC000000) == 0x40400450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP4 device\n")); + num_pp_cores = 4; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp4); + mali_gpu_device.resource = mali_gpu_resources_m450_mp4; + } else if (mali_read_phys(0xFC000000) == 0xFFFFFFFF) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp4); + mali_gpu_device.resource = mali_gpu_resources_m470_mp4; + } +#endif /* CONFIG_ARM64 */ + +#elif defined(CONFIG_ARCH_REALVIEW) + + m400_gp_version = mali_read_phys(0xC000006C); + if ((m400_gp_version & 0xFFFF0000) == 0x0C070000) { + MALI_DEBUG_PRINT(4, ("Registering Mali-300 device\n")); + num_pp_cores = 1; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m300); + mali_gpu_device.resource = mali_gpu_resources_m300; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } else if ((m400_gp_version & 0xFFFF0000) == 0x0B070000) { + u32 fpga_fw_version = mali_read_phys(0xC0010000); + if (fpga_fw_version == 0x130C008F || fpga_fw_version == 0x110C008F) { + /* Mali-400 MP1 r1p0 or r1p1 */ + MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP1 device\n")); + num_pp_cores = 1; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m400_mp1); + mali_gpu_device.resource = mali_gpu_resources_m400_mp1; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } else if (fpga_fw_version == 0x130C000F) { + /* Mali-400 MP2 r1p1 */ + MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP2 device\n")); + num_pp_cores = 2; + mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m400_mp2); + mali_gpu_device.resource = mali_gpu_resources_m400_mp2; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } + } + +#endif + /* Register the platform device */ + err = platform_device_register(&mali_gpu_device); + if (0 == err) { +#ifdef CONFIG_PM_RUNTIME +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) + pm_runtime_set_autosuspend_delay(&(mali_gpu_device.dev), 1000); + pm_runtime_use_autosuspend(&(mali_gpu_device.dev)); +#endif + pm_runtime_enable(&(mali_gpu_device.dev)); +#endif + MALI_DEBUG_ASSERT(0 < num_pp_cores); + mali_core_scaling_init(num_pp_cores); + + return 0; + } + + return err; +} + +void mali_platform_device_unregister(void) +{ + MALI_DEBUG_PRINT(4, ("mali_platform_device_unregister() called\n")); + + mali_core_scaling_term(); +#ifdef CONFIG_PM_RUNTIME + pm_runtime_disable(&(mali_gpu_device.dev)); +#endif + platform_device_unregister(&mali_gpu_device); + + platform_device_put(&mali_gpu_device); + +#if defined(CONFIG_ARCH_REALVIEW) + mali_write_phys(0xC0010020, 0x9); /* Restore default (legacy) memory mapping */ +#endif +} + +static void mali_platform_device_release(struct device *device) +{ + MALI_DEBUG_PRINT(4, ("mali_platform_device_release() called\n")); +} + +#else /* CONFIG_MALI_DT */ +int mali_platform_device_init(struct platform_device *device) +{ + int num_pp_cores = 0; + int err = -1; +#if defined(CONFIG_ARCH_REALVIEW) + u32 m400_gp_version; +#endif + + /* Detect present Mali GPU and connect the correct resources to the device */ +#if defined(CONFIG_ARCH_VEXPRESS) + +#if defined(CONFIG_ARM64) + if ((mali_read_phys(0x6F000000) & 0x00600450) == 0x00600450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); + num_pp_cores = 6; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00400430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00300430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP3 device\n")); + num_pp_cores = 3; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00200430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP2 device\n")); + num_pp_cores = 2; + } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00100430) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP1 device\n")); + num_pp_cores = 1; + } +#else + if (mali_read_phys(0xFC000000) == 0x00000450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP8 device\n")); + num_pp_cores = 8; + } else if (mali_read_phys(0xFC000000) == 0x40400450) { + MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP4 device\n")); + num_pp_cores = 4; + } else if (mali_read_phys(0xFC000000) == 0xFFFFFFFF) { + MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); + num_pp_cores = 4; + } +#endif + +#elif defined(CONFIG_ARCH_REALVIEW) + + m400_gp_version = mali_read_phys(0xC000006C); + if ((m400_gp_version & 0xFFFF0000) == 0x0C070000) { + MALI_DEBUG_PRINT(4, ("Registering Mali-300 device\n")); + num_pp_cores = 1; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } else if ((m400_gp_version & 0xFFFF0000) == 0x0B070000) { + u32 fpga_fw_version = mali_read_phys(0xC0010000); + if (fpga_fw_version == 0x130C008F || fpga_fw_version == 0x110C008F) { + /* Mali-400 MP1 r1p0 or r1p1 */ + MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP1 device\n")); + num_pp_cores = 1; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } else if (fpga_fw_version == 0x130C000F) { + /* Mali-400 MP2 r1p1 */ + MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP2 device\n")); + num_pp_cores = 2; + mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ + } + } +#endif + + /* After kernel 3.15 device tree will default set dev + * related parameters in of_platform_device_create_pdata. + * But kernel changes from version to version, + * For example 3.10 didn't include device->dev.dma_mask parameter setting, + * if we didn't include here will cause dma_mapping error, + * but in kernel 3.15 it include device->dev.dma_mask parameter setting, + * so it's better to set must need paramter by DDK itself. + */ + if (!device->dev.dma_mask) + device->dev.dma_mask = &device->dev.coherent_dma_mask; + device->dev.archdata.dma_ops = dma_ops; + + err = platform_device_add_data(device, &mali_gpu_data, sizeof(mali_gpu_data)); + + if (0 == err) { +#ifdef CONFIG_PM_RUNTIME +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) + pm_runtime_set_autosuspend_delay(&(device->dev), 1000); + pm_runtime_use_autosuspend(&(device->dev)); +#endif + pm_runtime_enable(&(device->dev)); +#endif + MALI_DEBUG_ASSERT(0 < num_pp_cores); + mali_core_scaling_init(num_pp_cores); + } + +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + /* Some Socs didn't support the devfreq thermal for mali */ + if (of_machine_is_compatible("rockchip,rk3036")) + return 0; + + /* Get thermal zone */ + gpu_tz = thermal_zone_get_zone_by_name("soc_thermal"); + if (IS_ERR(gpu_tz)) { + MALI_DEBUG_PRINT(2, ("Error getting gpu thermal zone (%ld), not yet ready?\n", + PTR_ERR(gpu_tz))); + gpu_tz = NULL; + + err = -EPROBE_DEFER; + } +#endif + + return err; +} + +int mali_platform_device_deinit(struct platform_device *device) +{ + MALI_IGNORE(device); + + MALI_DEBUG_PRINT(4, ("mali_platform_device_deinit() called\n")); + + mali_core_scaling_term(); +#ifdef CONFIG_PM_RUNTIME + pm_runtime_disable(&(device->dev)); +#endif + +#if defined(CONFIG_ARCH_REALVIEW) + mali_write_phys(0xC0010020, 0x9); /* Restore default (legacy) memory mapping */ +#endif + + return 0; +} + +#endif /* CONFIG_MALI_DT */ + +static u32 mali_read_phys(u32 phys_addr) +{ + u32 phys_addr_page = phys_addr & 0xFFFFE000; + u32 phys_offset = phys_addr & 0x00001FFF; + u32 map_size = phys_offset + sizeof(u32); + u32 ret = 0xDEADBEEF; + void *mem_mapped = ioremap(phys_addr_page, map_size); + if (NULL != mem_mapped) { + ret = (u32)ioread32(((u8 *)mem_mapped) + phys_offset); + iounmap(mem_mapped); + } + + return ret; +} + +#if defined(CONFIG_ARCH_REALVIEW) +static void mali_write_phys(u32 phys_addr, u32 value) +{ + u32 phys_addr_page = phys_addr & 0xFFFFE000; + u32 phys_offset = phys_addr & 0x00001FFF; + u32 map_size = phys_offset + sizeof(u32); + void *mem_mapped = ioremap(phys_addr_page, map_size); + if (NULL != mem_mapped) { + iowrite32(value, ((u8 *)mem_mapped) + phys_offset); + iounmap(mem_mapped); + } +} +#endif + +static int param_set_core_scaling(const char *val, const struct kernel_param *kp) +{ + int ret = param_set_int(val, kp); + + if (1 == mali_core_scaling_enable) { + mali_core_scaling_sync(mali_executor_get_num_cores_enabled()); + } + return ret; +} + +static struct kernel_param_ops param_ops_core_scaling = { + .set = param_set_core_scaling, + .get = param_get_int, +}; + +module_param_cb(mali_core_scaling_enable, ¶m_ops_core_scaling, &mali_core_scaling_enable, 0644); +MODULE_PARM_DESC(mali_core_scaling_enable, "1 means to enable core scaling policy, 0 means to disable core scaling policy"); + +void mali_gpu_utilization_callback(struct mali_gpu_utilization_data *data) +{ + if (1 == mali_core_scaling_enable) { + mali_core_scaling_update(data); + } +} diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c new file mode 100755 index 000000000000..7a2fc8107b4f --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file arm_core_scaling.c + * Example core scaling policy. + */ + +#include "arm_core_scaling.h" + +#include +#include "mali_kernel_common.h" + +#include + +static int num_cores_total; +static int num_cores_enabled; + +static struct work_struct wq_work; + +static void set_num_cores(struct work_struct *work) +{ + int err = mali_perf_set_num_pp_cores(num_cores_enabled); + MALI_DEBUG_ASSERT(0 == err); + MALI_IGNORE(err); +} + +static void enable_one_core(void) +{ + if (num_cores_enabled < num_cores_total) { + ++num_cores_enabled; + schedule_work(&wq_work); + MALI_DEBUG_PRINT(3, ("Core scaling: Enabling one more core\n")); + } + + MALI_DEBUG_ASSERT(1 <= num_cores_enabled); + MALI_DEBUG_ASSERT(num_cores_total >= num_cores_enabled); +} + +static void disable_one_core(void) +{ + if (1 < num_cores_enabled) { + --num_cores_enabled; + schedule_work(&wq_work); + MALI_DEBUG_PRINT(3, ("Core scaling: Disabling one core\n")); + } + + MALI_DEBUG_ASSERT(1 <= num_cores_enabled); + MALI_DEBUG_ASSERT(num_cores_total >= num_cores_enabled); +} + +static void enable_max_num_cores(void) +{ + if (num_cores_enabled < num_cores_total) { + num_cores_enabled = num_cores_total; + schedule_work(&wq_work); + MALI_DEBUG_PRINT(3, ("Core scaling: Enabling maximum number of cores\n")); + } + + MALI_DEBUG_ASSERT(num_cores_total == num_cores_enabled); +} + +void mali_core_scaling_init(int num_pp_cores) +{ + INIT_WORK(&wq_work, set_num_cores); + + num_cores_total = num_pp_cores; + num_cores_enabled = num_pp_cores; + + /* NOTE: Mali is not fully initialized at this point. */ +} + +void mali_core_scaling_sync(int num_cores) +{ + num_cores_enabled = num_cores; +} + +void mali_core_scaling_term(void) +{ + flush_scheduled_work(); +} + +#define PERCENT_OF(percent, max) ((int) ((percent)*(max)/100.0 + 0.5)) + +void mali_core_scaling_update(struct mali_gpu_utilization_data *data) +{ + /* + * This function implements a very trivial PP core scaling algorithm. + * + * It is _NOT_ of production quality. + * The only intention behind this algorithm is to exercise and test the + * core scaling functionality of the driver. + * It is _NOT_ tuned for neither power saving nor performance! + * + * Other metrics than PP utilization need to be considered as well + * in order to make a good core scaling algorithm. + */ + + MALI_DEBUG_PRINT(3, ("Utilization: (%3d, %3d, %3d), cores enabled: %d/%d\n", data->utilization_gpu, data->utilization_gp, data->utilization_pp, num_cores_enabled, num_cores_total)); + + /* NOTE: this function is normally called directly from the utilization callback which is in + * timer context. */ + + if (PERCENT_OF(90, 256) < data->utilization_pp) { + enable_max_num_cores(); + } else if (PERCENT_OF(50, 256) < data->utilization_pp) { + enable_one_core(); + } else if (PERCENT_OF(40, 256) < data->utilization_pp) { + /* do nothing */ + } else if (PERCENT_OF(0, 256) < data->utilization_pp) { + disable_one_core(); + } else { + /* do nothing */ + } +} diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h new file mode 100755 index 000000000000..8e0101830749 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file arm_core_scaling.h + * Example core scaling policy. + */ + +#ifndef __ARM_CORE_SCALING_H__ +#define __ARM_CORE_SCALING_H__ + +struct mali_gpu_utilization_data; + +/** + * Initialize core scaling policy. + * + * @note The core scaling policy will assume that all PP cores are on initially. + * + * @param num_pp_cores Total number of PP cores. + */ +void mali_core_scaling_init(int num_pp_cores); + +/** + * Terminate core scaling policy. + */ +void mali_core_scaling_term(void); + +/** + * Update core scaling policy with new utilization data. + * + * @param data Utilization data. + */ +void mali_core_scaling_update(struct mali_gpu_utilization_data *data); + +void mali_core_scaling_sync(int num_cores); + +#endif /* __ARM_CORE_SCALING_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/juno_opp.c b/drivers/gpu/arm/mali400/mali/platform/arm/juno_opp.c new file mode 100755 index 000000000000..e4e7ab8b2c2e --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/platform/arm/juno_opp.c @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file juno_opp.c + * Example: Set up opp table + * Using ARM64 juno specific SCPI_PROTOCOL get frequence inform + * Customer need implement your own platform releated logic + */ +#ifdef CONFIG_ARCH_VEXPRESS +#ifdef CONFIG_MALI_DEVFREQ +#ifdef CONFIG_ARM64 +#ifdef CONFIG_ARM_SCPI_PROTOCOL +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) +#include +#else /* Linux >= 3.13 */ +/* In 3.13 the OPP include header file, types, and functions were all + * renamed. Use the old filename for the include, and define the new names to + * the old, when an old kernel is detected. + */ +#include +#define dev_pm_opp_add opp_add +#define dev_pm_opp_remove opp_remove +#endif /* Linux >= 3.13 */ + +#include "mali_kernel_common.h" + +static int init_juno_opps_from_scpi(struct device *dev) +{ + struct scpi_dvfs_info *sinfo; + struct scpi_ops *sops; + + int i; + + sops = get_scpi_ops(); + if (NULL == sops) { + MALI_DEBUG_PRINT(2, ("Mali didn't get any scpi ops \n")); + return -1; + } + + /* Hard coded for Juno. 2 is GPU domain */ + sinfo = sops->dvfs_get_info(2); + if (IS_ERR_OR_NULL(sinfo)) + return PTR_ERR(sinfo); + + for (i = 0; i < sinfo->count; i++) { + struct scpi_opp *e = &sinfo->opps[i]; + + MALI_DEBUG_PRINT(2, ("Mali OPP from SCPI: %u Hz @ %u mV\n", e->freq, e->m_volt)); + + dev_pm_opp_add(dev, e->freq, e->m_volt * 1000); + } + + return 0; +} + +int setup_opps(void) +{ + struct device_node *np; + struct platform_device *pdev; + int err; + + np = of_find_node_by_name(NULL, "gpu"); + if (!np) { + pr_err("Failed to find DT entry for Mali\n"); + return -EFAULT; + } + + pdev = of_find_device_by_node(np); + if (!pdev) { + pr_err("Failed to find device for Mali\n"); + of_node_put(np); + return -EFAULT; + } + + err = init_juno_opps_from_scpi(&pdev->dev); + + of_node_put(np); + + return err; +} + +int term_opps(struct device *dev) +{ + struct scpi_dvfs_info *sinfo; + struct scpi_ops *sops; + + int i; + + sops = get_scpi_ops(); + if (NULL == sops) { + MALI_DEBUG_PRINT(2, ("Mali didn't get any scpi ops \n")); + return -1; + } + + /* Hard coded for Juno. 2 is GPU domain */ + sinfo = sops->dvfs_get_info(2); + if (IS_ERR_OR_NULL(sinfo)) + return PTR_ERR(sinfo); + + for (i = 0; i < sinfo->count; i++) { + struct scpi_opp *e = &sinfo->opps[i]; + + MALI_DEBUG_PRINT(2, ("Mali Remove OPP: %u Hz \n", e->freq)); + + dev_pm_opp_remove(dev, e->freq); + } + + return 0; + +} +#endif +#endif +#endif +#endif diff --git a/drivers/gpu/arm/mali400/mali/platform/rk/custom_log.h b/drivers/gpu/arm/mali400/mali/platform/rk/custom_log.h new file mode 100755 index 000000000000..fe5e1224149e --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/platform/rk/custom_log.h @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* ---------------------------------------------------------------------------- + * File: custom_log.h + * + * Desc: ChenZhen å好的 log 输出的定制实现. + * + * -------------------------------------------------------------------- + * < 习语 å’Œ 缩略语 > : + * + * -------------------------------------------------------------------- + * Usage: + * + * Note: + * + * Author: ChenZhen + * + * ---------------------------------------------------------------------------- + * Version: + * v1.0 + * ---------------------------------------------------------------------------- + * Log: + ----Fri Nov 19 15:20:28 2010 v1.0 + * + * ---------------------------------------------------------------------------- + */ + +#ifndef __CUSTOM_LOG_H__ +#define __CUSTOM_LOG_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/* ----------------------------------------------------------------------------- + * Include Files + * ----------------------------------------------------------------------------- + */ +#include +#include + +/* ----------------------------------------------------------------------------- + * Macros Definition + * ----------------------------------------------------------------------------- + */ + +/** 若下列 macro 有被定义, æ‰ ä½¿èƒ½ log 输出. */ +/* #define ENABLE_DEBUG_LOG */ + +/*----------------------------------------------------------------------------*/ + +#ifdef ENABLE_VERBOSE_LOG +/** Verbose log. */ +#define V(fmt, args...) \ + pr_debug("V : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) +#else +#define V(...) ((void)0) +#endif + +#ifdef ENABLE_DEBUG_LOG +/** Debug log. */ +#define D(fmt, args...) \ + pr_info("D : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) +#else +#define D(...) ((void)0) +#endif + +#define I(fmt, args...) \ + pr_info("I : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +#define W(fmt, args...) \ + pr_warn("W : [File] : %s; [Line] : %d; [Func] : %s(); " \ + fmt "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +#define E(fmt, args...) \ + pr_err("E : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +/*-------------------------------------------------------*/ + +/** 使用 D(), 以åè¿›åˆ¶çš„å½¢å¼æ‰“å°å˜é‡ 'var' çš„ value. */ +#define D_DEC(var) D(#var " = %d.", var) + +#define E_DEC(var) E(#var " = %d.", var) + +/** 使用 D(), 以åå…­è¿›åˆ¶çš„å½¢å¼æ‰“å°å˜é‡ 'var' çš„ value. */ +#define D_HEX(var) D(#var " = 0x%x.", var) + +#define E_HEX(var) E(#var " = 0x%x.", var) + +/** + * 使用 D(), 以å六进制的形å¼, + * æ‰“å°æŒ‡é’ˆç±»åž‹å˜é‡ 'ptr' çš„ value. + */ +#define D_PTR(ptr) D(#ptr " = %p.", ptr) + +#define E_PTR(ptr) E(#ptr " = %p.", ptr) + +/** 使用 D(), æ‰“å° char 字串. */ +#define D_STR(p_str) \ +do { \ + if (!p_str) { \ + D(#p_str " = NULL."); \ + else \ + D(#p_str " = '%s'.", p_str); \ +} while (0) + +#define E_STR(p_str) \ +do { \ + if (!p_str) \ + E(#p_str " = NULL."); \ + else \ + E(#p_str " = '%s'.", p_str); \ +} while (0) + +#ifdef ENABLE_DEBUG_LOG +/** + * log 从 'p_start' 地å€å¼€å§‹çš„ 'len' 个字节的数æ®. + */ +#define D_MEM(p_start, len) \ +do { \ + int i = 0; \ + char *p = (char *)(p_start); \ + D("dump memory from addr of '" #p_start "', from %p, length %d' : ", \ + (p_start), \ + (len)); \ + pr_debug("\t\t"); \ + for (i = 0; i < (len); i++) \ + pr_debug("0x%02x, ", p[i]); \ + pr_debug("\n"); \ +} while (0) +#else +#define D_MEM(...) ((void)0) +#endif + +/*-------------------------------------------------------*/ + +/** + * 在特定æ¡ä»¶ä¸‹, 判定 error å‘生, + * å°†å˜é‡ 'ret_var' 设置 'err_code', + * log 输出对应的 Error Caution, + * ç„¶åŽè·³è½¬ 'label' 指定的代ç å¤„执行. + * @param msg + * 纯字串形å¼çš„æç¤ºä¿¡æ¯. + * @param ret_var + * æ ‡è¯†å‡½æ•°æ‰§è¡ŒçŠ¶æ€æˆ–者结果的å˜é‡, + * 将被设置具体的 Error Code. + * 通常是 'ret' or 'result'. + * @param err_code + * 表å¾ç‰¹å®š error 的常数标识, + * 通常是 å®çš„å½¢æ€. + * @param label + * 程åºå°†è¦è·³è½¬åˆ°çš„错误处ç†ä»£ç çš„æ ‡å·, + * 通常就是 'EXIT'. + * @param args... + * 对应 'msg_fmt' 实å‚中, + * '%s', '%d', ... 等转æ¢è¯´æ˜Žç¬¦çš„具体å¯å˜é•¿å®žå‚. + */ +#define SET_ERROR_AND_JUMP(msg_fmt, ret_var, err_code, label, args...) \ +do { \ + E("To set '" #ret_var "' to %d('" #err_code "'), because : " msg_fmt, \ + (err_code), \ + ## args); \ + (ret_var) = (err_code); \ + goto label; \ +} while (0) + +/* ----------------------------------------------------------------------------- + * Types and Structures Definition + * ----------------------------------------------------------------------------- + */ + +/* ----------------------------------------------------------------------------- + * Global Functions' Prototype + * ----------------------------------------------------------------------------- + */ + +/* ----------------------------------------------------------------------------- + * Inline Functions Implementation + * ----------------------------------------------------------------------------- + */ + +#ifdef __cplusplus +} +#endif + +#endif /* __CUSTOM_LOG_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/platform/rk/rk.c b/drivers/gpu/arm/mali400/mali/platform/rk/rk.c new file mode 100755 index 000000000000..9a012fdf8d35 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/platform/rk/rk.c @@ -0,0 +1,676 @@ +/* + * (C) COPYRIGHT RockChip Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +/** + * @file rk.c + * implementation of platform_specific_code on rk platforms, such as rk3328h. + * + * mali_device_driver(MDD) includes 2 parts : + * .DP : platform_dependent_part : + * located in /mali/platform// + * .DP : common_part : + * common part implemented by ARM. + */ + +#define ENABLE_DEBUG_LOG +#include "custom_log.h" + +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_PM +#include +#endif +#include +#include +#include +#include +#include +#include + +#include +#include "mali_kernel_common.h" +#include "../../common/mali_osk_mali.h" + +/*---------------------------------------------------------------------------*/ + +u32 mali_group_error; + +/*---------------------------------------------------------------------------*/ + +#define DEFAULT_UTILISATION_PERIOD_IN_MS (100) + +/* + * rk_platform_context_of_mali_device. + */ +struct rk_context { + /* mali device. */ + struct device *dev; + /* is the GPU powered on? */ + bool is_powered; + /* debug only, the period in ms to count gpu_utilisation. */ + unsigned int utilisation_period; +}; + +struct rk_context *s_rk_context; + +/*---------------------------------------------------------------------------*/ + +#ifdef CONFIG_MALI_DEVFREQ +static ssize_t utilisation_period_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct rk_context *platform = s_rk_context; + ssize_t ret = 0; + + ret += snprintf(buf, PAGE_SIZE, "%u\n", platform->utilisation_period); + + return ret; +} + +static ssize_t utilisation_period_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct rk_context *platform = s_rk_context; + int ret = 0; + + ret = kstrtouint(buf, 0, &platform->utilisation_period); + if (ret) { + E("invalid input period : %s.", buf); + return ret; + } + D("set utilisation_period to '%d'.", platform->utilisation_period); + + return count; +} + +static ssize_t utilisation_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct rk_context *platform = s_rk_context; + struct mali_device *mdev = dev_get_drvdata(dev); + ssize_t ret = 0; + unsigned long period_in_us = platform->utilisation_period * 1000; + unsigned long total_time; + unsigned long busy_time; + unsigned long utilisation; + + mali_pm_reset_dvfs_utilisation(mdev); + usleep_range(period_in_us, period_in_us + 100); + mali_pm_get_dvfs_utilisation(mdev, &total_time, &busy_time); + + /* 'devfreq_dev_profile' instance registered to devfreq + * also uses mali_pm_reset_dvfs_utilisation() + * and mali_pm_get_dvfs_utilisation(). + * So, it's better to disable GPU DVFS before reading this node. + */ + D("total_time : %lu, busy_time : %lu.", total_time, busy_time); + + utilisation = busy_time / (total_time / 100); + ret += snprintf(buf, PAGE_SIZE, "%lu\n", utilisation); + + return ret; +} + +static DEVICE_ATTR_RW(utilisation_period); +static DEVICE_ATTR_RO(utilisation); +#endif + +static int rk_context_create_sysfs_files(struct device *dev) +{ +#ifdef CONFIG_MALI_DEVFREQ + int ret; + + ret = device_create_file(dev, &dev_attr_utilisation_period); + if (ret) { + E("fail to create sysfs file 'utilisation_period'."); + goto out; + } + + ret = device_create_file(dev, &dev_attr_utilisation); + if (ret) { + E("fail to create sysfs file 'utilisation'."); + goto remove_utilisation_period; + } + + return 0; + +remove_utilisation_period: + device_remove_file(dev, &dev_attr_utilisation_period); +out: + return ret; +#else + return 0; +#endif +} + +static void rk_context_remove_sysfs_files(struct device *dev) +{ +#ifdef CONFIG_MALI_DEVFREQ + device_remove_file(dev, &dev_attr_utilisation_period); + device_remove_file(dev, &dev_attr_utilisation); +#endif +} + +/*---------------------------------------------------------------------------*/ + +/* + * Init rk_platform_context of mali_device. + */ +static int rk_context_init(struct platform_device *pdev) +{ + int ret = 0; + struct device *dev = &pdev->dev; + struct rk_context *platform; /* platform_context */ + + platform = kzalloc(sizeof(*platform), GFP_KERNEL); + if (!platform) { + E("no mem."); + return _MALI_OSK_ERR_NOMEM; + } + + platform->dev = dev; + platform->is_powered = false; + + platform->utilisation_period = DEFAULT_UTILISATION_PERIOD_IN_MS; + + ret = rk_context_create_sysfs_files(dev); + if (ret) { + E("fail to create sysfs files, ret = %d", ret); + goto EXIT; + } + + s_rk_context = platform; + + pm_runtime_set_autosuspend_delay(dev, 1000); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); + +EXIT: + return ret; +} + +static void rk_context_deinit(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rk_context *platform = s_rk_context; + + pm_runtime_disable(dev); + + s_rk_context = NULL; + + rk_context_remove_sysfs_files(dev); + + if (platform) { + platform->is_powered = false; + platform->dev = NULL; + kfree(platform); + } +} + +/*---------------------------------------------------------------------------*/ +/* for devfreq cooling. */ + +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + +#define FALLBACK_STATIC_TEMPERATURE 55000 + +static u32 dynamic_coefficient; +static u32 static_coefficient; +static s32 ts[4]; +static struct thermal_zone_device *gpu_tz; + +static int power_model_simple_init(struct platform_device *pdev) +{ + struct device_node *power_model_node; + const char *tz_name; + u32 static_power, dynamic_power; + u32 voltage, voltage_squared, voltage_cubed, frequency; + + power_model_node = of_get_child_by_name(pdev->dev.of_node, + "power_model"); + if (!power_model_node) { + dev_err(&pdev->dev, "could not find power_model node\n"); + return -ENODEV; + } + if (!of_device_is_compatible(power_model_node, + "arm,mali-simple-power-model")) { + dev_err(&pdev->dev, "power_model incompatible with simple power model\n"); + return -ENODEV; + } + + if (of_property_read_string(power_model_node, "thermal-zone", + &tz_name)) { + dev_err(&pdev->dev, "ts in power_model not available\n"); + return -EINVAL; + } + + gpu_tz = thermal_zone_get_zone_by_name(tz_name); + if (IS_ERR(gpu_tz)) { + pr_warn_ratelimited("Error getting gpu thermal zone '%s'(%ld), not yet ready?\n", + tz_name, + PTR_ERR(gpu_tz)); + gpu_tz = NULL; + } + + if (of_property_read_u32(power_model_node, "static-power", + &static_power)) { + dev_err(&pdev->dev, "static-power in power_model not available\n"); + return -EINVAL; + } + if (of_property_read_u32(power_model_node, "dynamic-power", + &dynamic_power)) { + dev_err(&pdev->dev, "dynamic-power in power_model not available\n"); + return -EINVAL; + } + if (of_property_read_u32(power_model_node, "voltage", + &voltage)) { + dev_err(&pdev->dev, "voltage in power_model not available\n"); + return -EINVAL; + } + if (of_property_read_u32(power_model_node, "frequency", + &frequency)) { + dev_err(&pdev->dev, "frequency in power_model not available\n"); + return -EINVAL; + } + voltage_squared = (voltage * voltage) / 1000; + voltage_cubed = voltage * voltage * voltage; + static_coefficient = (static_power << 20) / (voltage_cubed >> 10); + dynamic_coefficient = (((dynamic_power * 1000) / voltage_squared) + * 1000) / frequency; + + if (of_property_read_u32_array(power_model_node, "ts", (u32 *)ts, 4)) { + dev_err(&pdev->dev, "ts in power_model not available\n"); + return -EINVAL; + } + + return 0; +} + +/* Calculate gpu static power example for reference */ +static unsigned long rk_model_static_power(struct devfreq *devfreq, + unsigned long voltage) +{ + int temperature, temp; + int temp_squared, temp_cubed, temp_scaling_factor; + const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 10; + unsigned long static_power; + + if (gpu_tz) { + int ret; + + ret = gpu_tz->ops->get_temp(gpu_tz, &temperature); + if (ret) { + MALI_DEBUG_PRINT(2, ("fail to read temp: %d\n", ret)); + temperature = FALLBACK_STATIC_TEMPERATURE; + } + } else { + temperature = FALLBACK_STATIC_TEMPERATURE; + } + + /* Calculate the temperature scaling factor. To be applied to the + * voltage scaled power. + */ + temp = temperature / 1000; + temp_squared = temp * temp; + temp_cubed = temp_squared * temp; + temp_scaling_factor = + (ts[3] * temp_cubed) + + (ts[2] * temp_squared) + + (ts[1] * temp) + + ts[0]; + + static_power = (((static_coefficient * voltage_cubed) >> 20) + * temp_scaling_factor) + / 1000000; + + return static_power; +} + +/* Calculate gpu dynamic power example for reference */ +static unsigned long rk_model_dynamic_power(struct devfreq *devfreq, + unsigned long freq, + unsigned long voltage) +{ + /* The inputs: freq (f) is in Hz, and voltage (v) in mV. + * The coefficient (c) is in mW/(MHz mV mV). + * + * This function calculates the dynamic power after this formula: + * Pdyn (mW) = c (mW/(MHz*mV*mV)) * v (mV) * v (mV) * f (MHz) + */ + const unsigned long v2 = (voltage * voltage) / 1000; /* m*(V*V) */ + const unsigned long f_mhz = freq / 1000000; /* MHz */ + unsigned long dynamic_power; + + dynamic_power = (dynamic_coefficient * v2 * f_mhz) / 1000000; /* mW */ + + return dynamic_power; +} + +struct devfreq_cooling_power rk_cooling_ops = { + .get_static_power = rk_model_static_power, + .get_dynamic_power = rk_model_dynamic_power, +}; +#endif + +/*---------------------------------------------------------------------------*/ + +#ifdef CONFIG_PM + +static int rk_platform_enable_clk_gpu(struct device *dev) +{ + int ret = 0; +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_HAVE_CLK) + struct mali_device *mdev = dev_get_drvdata(dev); + + if (mdev->clock) + ret = clk_enable(mdev->clock); +#endif + return ret; +} + +static void rk_platform_disable_clk_gpu(struct device *dev) +{ +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_HAVE_CLK) + struct mali_device *mdev = dev_get_drvdata(dev); + + if (mdev->clock) + clk_disable(mdev->clock); +#endif +} + +static int rk_platform_enable_gpu_regulator(struct device *dev) +{ + int ret = 0; +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_REGULATOR) + struct mali_device *mdev = dev_get_drvdata(dev); + + if (mdev->regulator) + ret = regulator_enable(mdev->regulator); +#endif + return ret; +} + +static void rk_platform_disable_gpu_regulator(struct device *dev) +{ +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_REGULATOR) + struct mali_device *mdev = dev_get_drvdata(dev); + + if (mdev->regulator) + regulator_disable(mdev->regulator); +#endif +} + +static int rk_platform_power_on_gpu(struct device *dev) +{ + struct rk_context *platform = s_rk_context; + int ret = 0; + + if (!(platform->is_powered)) { + ret = rk_platform_enable_clk_gpu(dev); + if (ret) { + E("fail to enable clk_gpu, ret : %d.", ret); + goto fail_to_enable_clk; + } + + ret = rk_platform_enable_gpu_regulator(dev); + if (ret) { + E("fail to enable vdd_gpu, ret : %d.", ret); + goto fail_to_enable_regulator; + } + + platform->is_powered = true; + } + + return 0; + +fail_to_enable_regulator: + rk_platform_disable_clk_gpu(dev); + +fail_to_enable_clk: + return ret; +} + +static void rk_platform_power_off_gpu(struct device *dev) +{ + struct rk_context *platform = s_rk_context; + + if (platform->is_powered) { + rk_platform_disable_clk_gpu(dev); + rk_platform_disable_gpu_regulator(dev); + + platform->is_powered = false; + } +} + +int rk_platform_init_opp_table(struct device *dev) +{ + return rockchip_init_opp_table(dev, NULL, "gpu_leakage", "mali"); +} + +static int mali_runtime_suspend(struct device *device) +{ + int ret = 0; + + MALI_DEBUG_PRINT(4, ("mali_runtime_suspend() called\n")); + + if (device->driver && + device->driver->pm && + device->driver->pm->runtime_suspend) { + /* Need to notify Mali driver about this event */ + ret = device->driver->pm->runtime_suspend(device); + } + + if (!ret) + rk_platform_power_off_gpu(device); + + return ret; +} + +static int mali_runtime_resume(struct device *device) +{ + int ret = 0; + + MALI_DEBUG_PRINT(4, ("mali_runtime_resume() called\n")); + + rk_platform_power_on_gpu(device); + + if (device->driver && + device->driver->pm && + device->driver->pm->runtime_resume) { + /* Need to notify Mali driver about this event */ + ret = device->driver->pm->runtime_resume(device); + } + + return ret; +} + +static int mali_runtime_idle(struct device *device) +{ + int ret = 0; + + MALI_DEBUG_PRINT(4, ("mali_runtime_idle() called\n")); + + if (device->driver && + device->driver->pm && + device->driver->pm->runtime_idle) { + /* Need to notify Mali driver about this event */ + ret = device->driver->pm->runtime_idle(device); + if (ret) + return ret; + } + + return 0; +} +#endif + +static int mali_os_suspend(struct device *device) +{ + int ret = 0; + + MALI_DEBUG_PRINT(4, ("mali_os_suspend() called\n")); + + if (device->driver && + device->driver->pm && + device->driver->pm->suspend) { + /* Need to notify Mali driver about this event */ + ret = device->driver->pm->suspend(device); + } + + if (!ret) + rk_platform_power_off_gpu(device); + + return ret; +} + +static int mali_os_resume(struct device *device) +{ + int ret = 0; + + MALI_DEBUG_PRINT(4, ("mali_os_resume() called\n")); + + rk_platform_power_on_gpu(device); + + if (device->driver && + device->driver->pm && + device->driver->pm->resume) { + /* Need to notify Mali driver about this event */ + ret = device->driver->pm->resume(device); + } + + return ret; +} + +static int mali_os_freeze(struct device *device) +{ + int ret = 0; + + MALI_DEBUG_PRINT(4, ("mali_os_freeze() called\n")); + + if (device->driver && + device->driver->pm && + device->driver->pm->freeze) { + /* Need to notify Mali driver about this event */ + ret = device->driver->pm->freeze(device); + } + + return ret; +} + +static int mali_os_thaw(struct device *device) +{ + int ret = 0; + + MALI_DEBUG_PRINT(4, ("mali_os_thaw() called\n")); + + if (device->driver && + device->driver->pm && + device->driver->pm->thaw) { + /* Need to notify Mali driver about this event */ + ret = device->driver->pm->thaw(device); + } + + return ret; +} + +static const struct dev_pm_ops mali_gpu_device_type_pm_ops = { + .suspend = mali_os_suspend, + .resume = mali_os_resume, + .freeze = mali_os_freeze, + .thaw = mali_os_thaw, +#ifdef CONFIG_PM + .runtime_suspend = mali_runtime_suspend, + .runtime_resume = mali_runtime_resume, + .runtime_idle = mali_runtime_idle, +#endif +}; + +static const struct device_type mali_gpu_device_device_type = { + .pm = &mali_gpu_device_type_pm_ops, +}; + +/* + * platform_specific_data of platform_device of mali_gpu. + */ +static const struct mali_gpu_device_data mali_gpu_data = { + .shared_mem_size = 1024 * 1024 * 1024, /* 1GB */ + .max_job_runtime = 60000, /* 60 seconds */ +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + .gpu_cooling_ops = &rk_cooling_ops, +#endif +}; + +static void mali_platform_device_add_config(struct platform_device *pdev) +{ + pdev->name = MALI_GPU_NAME_UTGARD, + pdev->id = 0; + pdev->dev.type = &mali_gpu_device_device_type; + pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask, + pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32); +} + +/*---------------------------------------------------------------------------*/ +/* platform_device_functions called by common_part. */ + +int mali_platform_device_init(struct platform_device *pdev) +{ + int err = 0; + + mali_platform_device_add_config(pdev); + + D("to add platform_specific_data to platform_device_of_mali."); + err = platform_device_add_data(pdev, + &mali_gpu_data, + sizeof(mali_gpu_data)); + if (err) { + E("fail to add platform_specific_data. err : %d.", err); + goto add_data_failed; + } + + err = rk_context_init(pdev); + if (err) { + E("fail to init rk_context. err : %d.", err); + goto init_rk_context_failed; + } + +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + if (of_machine_is_compatible("rockchip,rk3036")) + return 0; + + err = power_model_simple_init(pdev); + if (err) { + E("fail to init simple_power_model, err : %d.", err); + goto init_power_model_failed; + } +#endif + + return 0; + +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) +init_power_model_failed: + rk_context_deinit(pdev); +#endif +init_rk_context_failed: +add_data_failed: + return err; +} + +void mali_platform_device_deinit(struct platform_device *pdev) +{ + MALI_DEBUG_PRINT(4, ("mali_platform_device_unregister() called\n")); + + rk_context_deinit(pdev); +} diff --git a/drivers/gpu/arm/mali400/mali/platform/rk/rk_ext.h b/drivers/gpu/arm/mali400/mali/platform/rk/rk_ext.h new file mode 100755 index 000000000000..bd939350c425 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/platform/rk/rk_ext.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* ---------------------------------------------------------------------------- + * File: rk_ext.h + * + * Desc: rk_ext_on_mali_ko 中的 通行定义等. + * + * Usage: + * + * Note: + * + * Author: ChenZhen + * + * Log: + * + * ---------------------------------------------------------------------------- + */ + +#ifndef __RK_EXT_H__ +#define __RK_EXT_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/*---------------------------------------------------------------------------*/ + +/** version of rk_ext on mali_ko, aka. rk_ko_ver. */ +#define RK_KO_VER (5) + +/*---------------------------------------------------------------------------*/ + +#ifdef __cplusplus +} +#endif + +#endif /* __RK_EXT_H__ */ + diff --git a/drivers/gpu/arm/mali400/mali/readme.txt b/drivers/gpu/arm/mali400/mali/readme.txt new file mode 100755 index 000000000000..6785ac933b38 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/readme.txt @@ -0,0 +1,28 @@ +Building the Mali Device Driver for Linux +----------------------------------------- + +Build the Mali Device Driver for Linux by running the following make command: + +KDIR= USING_UMP= BUILD= make + +where + kdir_path: Path to your Linux Kernel directory + ump_option: 1 = Enable UMP support(*) + 0 = disable UMP support + build_option: debug = debug build of driver + release = release build of driver + +(*) For newer Linux Kernels, the Module.symvers file for the UMP device driver + must be available. The UMP_SYMVERS_FILE variable in the Makefile should + point to this file. This file is generated when the UMP driver is built. + +The result will be a mali.ko file, which can be loaded into the Linux kernel +by using the insmod command. + +Use of UMP is not recommended. The dma-buf API in the Linux kernel has +replaced UMP. The Mali Device Driver will be built with dma-buf support if the +kernel config includes enabled dma-buf. + +The kernel needs to be provided with a platform_device struct for the Mali GPU +device. See the mali_utgard.h header file for how to set up the Mali GPU +resources. diff --git a/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h b/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h new file mode 100755 index 000000000000..0345fb169a95 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef _MALI200_REGS_H_ +#define _MALI200_REGS_H_ + +/** + * Enum for management register addresses. + */ +enum mali200_mgmt_reg { + MALI200_REG_ADDR_MGMT_VERSION = 0x1000, + MALI200_REG_ADDR_MGMT_CURRENT_REND_LIST_ADDR = 0x1004, + MALI200_REG_ADDR_MGMT_STATUS = 0x1008, + MALI200_REG_ADDR_MGMT_CTRL_MGMT = 0x100c, + + MALI200_REG_ADDR_MGMT_INT_RAWSTAT = 0x1020, + MALI200_REG_ADDR_MGMT_INT_CLEAR = 0x1024, + MALI200_REG_ADDR_MGMT_INT_MASK = 0x1028, + MALI200_REG_ADDR_MGMT_INT_STATUS = 0x102c, + + MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS = 0x1050, + + MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x1080, + MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x1084, + MALI200_REG_ADDR_MGMT_PERF_CNT_0_LIMIT = 0x1088, + MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x108c, + + MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x10a0, + MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x10a4, + MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x10ac, + + MALI200_REG_ADDR_MGMT_PERFMON_CONTR = 0x10b0, + MALI200_REG_ADDR_MGMT_PERFMON_BASE = 0x10b4, + + MALI200_REG_SIZEOF_REGISTER_BANK = 0x10f0 + +}; + +#define MALI200_REG_VAL_PERF_CNT_ENABLE 1 + +enum mali200_mgmt_ctrl_mgmt { + MALI200_REG_VAL_CTRL_MGMT_STOP_BUS = (1 << 0), + MALI200_REG_VAL_CTRL_MGMT_FLUSH_CACHES = (1 << 3), + MALI200_REG_VAL_CTRL_MGMT_FORCE_RESET = (1 << 5), + MALI200_REG_VAL_CTRL_MGMT_START_RENDERING = (1 << 6), + MALI400PP_REG_VAL_CTRL_MGMT_SOFT_RESET = (1 << 7), /* Only valid for Mali-300 and later */ +}; + +enum mali200_mgmt_irq { + MALI200_REG_VAL_IRQ_END_OF_FRAME = (1 << 0), + MALI200_REG_VAL_IRQ_END_OF_TILE = (1 << 1), + MALI200_REG_VAL_IRQ_HANG = (1 << 2), + MALI200_REG_VAL_IRQ_FORCE_HANG = (1 << 3), + MALI200_REG_VAL_IRQ_BUS_ERROR = (1 << 4), + MALI200_REG_VAL_IRQ_BUS_STOP = (1 << 5), + MALI200_REG_VAL_IRQ_CNT_0_LIMIT = (1 << 6), + MALI200_REG_VAL_IRQ_CNT_1_LIMIT = (1 << 7), + MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR = (1 << 8), + MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND = (1 << 9), + MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW = (1 << 10), + MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW = (1 << 11), + MALI400PP_REG_VAL_IRQ_RESET_COMPLETED = (1 << 12), +}; + +#define MALI200_REG_VAL_IRQ_MASK_ALL ((enum mali200_mgmt_irq) (\ + MALI200_REG_VAL_IRQ_END_OF_FRAME |\ + MALI200_REG_VAL_IRQ_END_OF_TILE |\ + MALI200_REG_VAL_IRQ_HANG |\ + MALI200_REG_VAL_IRQ_FORCE_HANG |\ + MALI200_REG_VAL_IRQ_BUS_ERROR |\ + MALI200_REG_VAL_IRQ_BUS_STOP |\ + MALI200_REG_VAL_IRQ_CNT_0_LIMIT |\ + MALI200_REG_VAL_IRQ_CNT_1_LIMIT |\ + MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR |\ + MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND |\ + MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW |\ + MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW |\ + MALI400PP_REG_VAL_IRQ_RESET_COMPLETED)) + +#define MALI200_REG_VAL_IRQ_MASK_USED ((enum mali200_mgmt_irq) (\ + MALI200_REG_VAL_IRQ_END_OF_FRAME |\ + MALI200_REG_VAL_IRQ_FORCE_HANG |\ + MALI200_REG_VAL_IRQ_BUS_ERROR |\ + MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR |\ + MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND |\ + MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW |\ + MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW)) + +#define MALI200_REG_VAL_IRQ_MASK_NONE ((enum mali200_mgmt_irq)(0)) + +enum mali200_mgmt_status { + MALI200_REG_VAL_STATUS_RENDERING_ACTIVE = (1 << 0), + MALI200_REG_VAL_STATUS_BUS_STOPPED = (1 << 4), +}; + +enum mali200_render_unit { + MALI200_REG_ADDR_FRAME = 0x0000, + MALI200_REG_ADDR_RSW = 0x0004, + MALI200_REG_ADDR_STACK = 0x0030, + MALI200_REG_ADDR_STACK_SIZE = 0x0034, + MALI200_REG_ADDR_ORIGIN_OFFSET_X = 0x0040 +}; + +enum mali200_wb_unit { + MALI200_REG_ADDR_WB0 = 0x0100, + MALI200_REG_ADDR_WB1 = 0x0200, + MALI200_REG_ADDR_WB2 = 0x0300 +}; + +enum mali200_wb_unit_regs { + MALI200_REG_ADDR_WB_SOURCE_SELECT = 0x0000, + MALI200_REG_ADDR_WB_SOURCE_ADDR = 0x0004, +}; + +/* This should be in the top 16 bit of the version register of Mali PP */ +#define MALI200_PP_PRODUCT_ID 0xC807 +#define MALI300_PP_PRODUCT_ID 0xCE07 +#define MALI400_PP_PRODUCT_ID 0xCD07 +#define MALI450_PP_PRODUCT_ID 0xCF07 +#define MALI470_PP_PRODUCT_ID 0xCF08 + + + +#endif /* _MALI200_REGS_H_ */ diff --git a/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h b/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h new file mode 100755 index 000000000000..7f8b58fd6c49 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef _MALIGP2_CONROL_REGS_H_ +#define _MALIGP2_CONROL_REGS_H_ + +/** + * These are the different geometry processor control registers. + * Their usage is to control and monitor the operation of the + * Vertex Shader and the Polygon List Builder in the geometry processor. + * Addresses are in 32-bit word relative sizes. + * @see [P0081] "Geometry Processor Data Structures" for details + */ + +typedef enum { + MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR = 0x00, + MALIGP2_REG_ADDR_MGMT_VSCL_END_ADDR = 0x04, + MALIGP2_REG_ADDR_MGMT_PLBUCL_START_ADDR = 0x08, + MALIGP2_REG_ADDR_MGMT_PLBUCL_END_ADDR = 0x0c, + MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_START_ADDR = 0x10, + MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_END_ADDR = 0x14, + MALIGP2_REG_ADDR_MGMT_CMD = 0x20, + MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT = 0x24, + MALIGP2_REG_ADDR_MGMT_INT_CLEAR = 0x28, + MALIGP2_REG_ADDR_MGMT_INT_MASK = 0x2C, + MALIGP2_REG_ADDR_MGMT_INT_STAT = 0x30, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x3C, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x40, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x44, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x48, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x4C, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x50, + MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_LIMIT = 0x54, + MALIGP2_REG_ADDR_MGMT_STATUS = 0x68, + MALIGP2_REG_ADDR_MGMT_VERSION = 0x6C, + MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR_READ = 0x80, + MALIGP2_REG_ADDR_MGMT_PLBCL_START_ADDR_READ = 0x84, + MALIGP2_CONTR_AXI_BUS_ERROR_STAT = 0x94, + MALIGP2_REGISTER_ADDRESS_SPACE_SIZE = 0x98, +} maligp_reg_addr_mgmt_addr; + +#define MALIGP2_REG_VAL_PERF_CNT_ENABLE 1 + +/** + * Commands to geometry processor. + * @see MALIGP2_CTRL_REG_CMD + */ +typedef enum { + MALIGP2_REG_VAL_CMD_START_VS = (1 << 0), + MALIGP2_REG_VAL_CMD_START_PLBU = (1 << 1), + MALIGP2_REG_VAL_CMD_UPDATE_PLBU_ALLOC = (1 << 4), + MALIGP2_REG_VAL_CMD_RESET = (1 << 5), + MALIGP2_REG_VAL_CMD_FORCE_HANG = (1 << 6), + MALIGP2_REG_VAL_CMD_STOP_BUS = (1 << 9), + MALI400GP_REG_VAL_CMD_SOFT_RESET = (1 << 10), /* only valid for Mali-300 and later */ +} mgp_contr_reg_val_cmd; + + +/** @defgroup MALIGP2_IRQ + * Interrupt status of geometry processor. + * @see MALIGP2_CTRL_REG_INT_RAWSTAT, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, + * MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_ADDR_MGMT_INT_STAT + * @{ + */ +#define MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST (1 << 0) +#define MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST (1 << 1) +#define MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM (1 << 2) +#define MALIGP2_REG_VAL_IRQ_VS_SEM_IRQ (1 << 3) +#define MALIGP2_REG_VAL_IRQ_PLBU_SEM_IRQ (1 << 4) +#define MALIGP2_REG_VAL_IRQ_HANG (1 << 5) +#define MALIGP2_REG_VAL_IRQ_FORCE_HANG (1 << 6) +#define MALIGP2_REG_VAL_IRQ_PERF_CNT_0_LIMIT (1 << 7) +#define MALIGP2_REG_VAL_IRQ_PERF_CNT_1_LIMIT (1 << 8) +#define MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR (1 << 9) +#define MALIGP2_REG_VAL_IRQ_SYNC_ERROR (1 << 10) +#define MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR (1 << 11) +#define MALI400GP_REG_VAL_IRQ_AXI_BUS_STOPPED (1 << 12) +#define MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD (1 << 13) +#define MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD (1 << 14) +#define MALI400GP_REG_VAL_IRQ_RESET_COMPLETED (1 << 19) +#define MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW (1 << 20) +#define MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW (1 << 21) +#define MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS (1 << 22) + +/* Mask defining all IRQs in Mali GP */ +#define MALIGP2_REG_VAL_IRQ_MASK_ALL \ + (\ + MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | \ + MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST | \ + MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | \ + MALIGP2_REG_VAL_IRQ_VS_SEM_IRQ | \ + MALIGP2_REG_VAL_IRQ_PLBU_SEM_IRQ | \ + MALIGP2_REG_VAL_IRQ_HANG | \ + MALIGP2_REG_VAL_IRQ_FORCE_HANG | \ + MALIGP2_REG_VAL_IRQ_PERF_CNT_0_LIMIT | \ + MALIGP2_REG_VAL_IRQ_PERF_CNT_1_LIMIT | \ + MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR | \ + MALIGP2_REG_VAL_IRQ_SYNC_ERROR | \ + MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR | \ + MALI400GP_REG_VAL_IRQ_AXI_BUS_STOPPED | \ + MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD | \ + MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD | \ + MALI400GP_REG_VAL_IRQ_RESET_COMPLETED | \ + MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW | \ + MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW | \ + MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS) + +/* Mask defining the IRQs in Mali GP which we use */ +#define MALIGP2_REG_VAL_IRQ_MASK_USED \ + (\ + MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | \ + MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST | \ + MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | \ + MALIGP2_REG_VAL_IRQ_FORCE_HANG | \ + MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR | \ + MALIGP2_REG_VAL_IRQ_SYNC_ERROR | \ + MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR | \ + MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD | \ + MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD | \ + MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW | \ + MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW | \ + MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS) + +/* Mask defining non IRQs on MaliGP2*/ +#define MALIGP2_REG_VAL_IRQ_MASK_NONE 0 + +/** }@ defgroup MALIGP2_IRQ*/ + +/** @defgroup MALIGP2_STATUS + * The different Status values to the geometry processor. + * @see MALIGP2_CTRL_REG_STATUS + * @{ + */ +#define MALIGP2_REG_VAL_STATUS_VS_ACTIVE 0x0002 +#define MALIGP2_REG_VAL_STATUS_BUS_STOPPED 0x0004 +#define MALIGP2_REG_VAL_STATUS_PLBU_ACTIVE 0x0008 +#define MALIGP2_REG_VAL_STATUS_BUS_ERROR 0x0040 +#define MALIGP2_REG_VAL_STATUS_WRITE_BOUND_ERR 0x0100 +/** }@ defgroup MALIGP2_STATUS*/ + +#define MALIGP2_REG_VAL_STATUS_MASK_ACTIVE (\ + MALIGP2_REG_VAL_STATUS_VS_ACTIVE|\ + MALIGP2_REG_VAL_STATUS_PLBU_ACTIVE) + + +#define MALIGP2_REG_VAL_STATUS_MASK_ERROR (\ + MALIGP2_REG_VAL_STATUS_BUS_ERROR |\ + MALIGP2_REG_VAL_STATUS_WRITE_BOUND_ERR ) + +/* This should be in the top 16 bit of the version register of gp.*/ +#define MALI200_GP_PRODUCT_ID 0xA07 +#define MALI300_GP_PRODUCT_ID 0xC07 +#define MALI400_GP_PRODUCT_ID 0xB07 +#define MALI450_GP_PRODUCT_ID 0xD07 + +/** + * The different sources for instrumented on the geometry processor. + * @see MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC + */ + +enum MALIGP2_cont_reg_perf_cnt_src { + MALIGP2_REG_VAL_PERF_CNT1_SRC_NUMBER_OF_VERTICES_PROCESSED = 0x0a, +}; + +#endif diff --git a/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c new file mode 100755 index 000000000000..7df934c12122 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2010-2011, 2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_timestamp.h" + +/* This file is intentionally left empty, as all functions are inlined in mali_profiling_sampler.h */ diff --git a/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h new file mode 100755 index 000000000000..f52097c1901b --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_TIMESTAMP_H__ +#define __MALI_TIMESTAMP_H__ + +#include "mali_osk.h" + +MALI_STATIC_INLINE _mali_osk_errcode_t _mali_timestamp_reset(void) +{ + /* + * reset counters and overflow flags + */ + + u32 mask = (1 << 0) | /* enable all three counters */ + (0 << 1) | /* reset both Count Registers to 0x0 */ + (1 << 2) | /* reset the Cycle Counter Register to 0x0 */ + (0 << 3) | /* 1 = Cycle Counter Register counts every 64th processor clock cycle */ + (0 << 4) | /* Count Register 0 interrupt enable */ + (0 << 5) | /* Count Register 1 interrupt enable */ + (0 << 6) | /* Cycle Counter interrupt enable */ + (0 << 8) | /* Count Register 0 overflow flag (clear or write, flag on read) */ + (0 << 9) | /* Count Register 1 overflow flag (clear or write, flag on read) */ + (1 << 10); /* Cycle Counter Register overflow flag (clear or write, flag on read) */ + + __asm__ __volatile__("MCR p15, 0, %0, c15, c12, 0" : : "r"(mask)); + + return _MALI_OSK_ERR_OK; +} + +MALI_STATIC_INLINE u64 _mali_timestamp_get(void) +{ + u32 result; + + /* this is for the clock cycles */ + __asm__ __volatile__("MRC p15, 0, %0, c15, c12, 1" : "=r"(result)); + + return (u64)result; +} + +#endif /* __MALI_TIMESTAMP_H__ */ diff --git a/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c new file mode 100755 index 000000000000..7df934c12122 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2010-2011, 2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_timestamp.h" + +/* This file is intentionally left empty, as all functions are inlined in mali_profiling_sampler.h */ diff --git a/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h new file mode 100755 index 000000000000..709a16a82f31 --- /dev/null +++ b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __MALI_TIMESTAMP_H__ +#define __MALI_TIMESTAMP_H__ + +#include "mali_osk.h" + +MALI_STATIC_INLINE _mali_osk_errcode_t _mali_timestamp_reset(void) +{ + return _MALI_OSK_ERR_OK; +} + +MALI_STATIC_INLINE u64 _mali_timestamp_get(void) +{ + return _mali_osk_boot_time_get_ns(); +} + +#endif /* __MALI_TIMESTAMP_H__ */ diff --git a/drivers/gpu/arm/mali400/rk_ver_info.txt b/drivers/gpu/arm/mali400/rk_ver_info.txt new file mode 100755 index 000000000000..2a6cbbbb5a97 --- /dev/null +++ b/drivers/gpu/arm/mali400/rk_ver_info.txt @@ -0,0 +1,11 @@ + +r5p0-01rel0-1-x@0 + 对 arm_release_ver r5p0-01rel0 的定制集æˆ. + r5p0-01rel0 对 gpu çš„ dts 有大修改, 但这里出于兼容考虑, 仿—§ä½¿ç”¨ dts_for_mali_ko_befor_r5p0-01rel0. + +r5p0-01rel0-2-x@0 + æ”¯æŒ mali_so æ¥èŽ·å– rk_ko_ver. + +r5p0-01rel0-3-x@0 + 在 mali_control_timer_callback_chain 中使用 mod_timer, 而ä¸å†æ˜¯ add_timer. + diff --git a/drivers/gpu/arm/mali400/ump/Kbuild b/drivers/gpu/arm/mali400/ump/Kbuild new file mode 100755 index 000000000000..a3067ba72459 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/Kbuild @@ -0,0 +1,92 @@ +# +# Copyright (C) 2010-2012 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the GNU General Public License version 2 +# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained from Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +# Set default configuration to use, if Makefile didn't provide one. +# Change this to use a different config.h +CONFIG ?= default + +# Link arch to the selected arch-config directory +$(shell [ -L $(src)/arch ] && rm $(src)/arch) +$(shell ln -sf arch-$(CONFIG) $(src)/arch) +$(shell touch $(src)/arch/config.h) + +UDD_FILE_PREFIX = ../mali/ + +# Get subversion revision number, fall back to 0000 if no svn info is available +SVN_INFO = (cd $(src); svn info 2>/dev/null) + +ifneq ($(shell $(SVN_INFO) 2>/dev/null),) +# SVN detected +SVN_REV := $(shell $(SVN_INFO) | grep '^Revision: '| sed -e 's/^Revision: //' 2>/dev/null) +DRIVER_REV := $(MALI_RELEASE_NAME)-r$(SVN_REV) +CHANGE_DATE := $(shell $(SVN_INFO) | grep '^Last Changed Date: ' | cut -d: -f2- | cut -b2-) +CHANGED_REVISION := $(shell $(SVN_INFO) | grep '^Last Changed Rev: ' | cut -d: -f2- | cut -b2-) +REPO_URL := $(shell $(SVN_INFO) | grep '^URL: ' | cut -d: -f2- | cut -b2-) + +else # SVN +GIT_REV := $(shell cd $(src); git describe --always 2>/dev/null) +ifneq ($(GIT_REV),) +# Git detected +DRIVER_REV := $(MALI_RELEASE_NAME)-$(GIT_REV) +CHANGE_DATE := $(shell cd $(src); git log -1 --format="%ci") +CHANGED_REVISION := $(GIT_REV) +REPO_URL := $(shell cd $(src); git describe --all --always 2>/dev/null) + +else # Git +# No Git or SVN detected +DRIVER_REV := $(MALI_RELEASE_NAME) +CHANGE_DATE := $(MALI_RELEASE_NAME) +CHANGED_REVISION := $(MALI_RELEASE_NAME) +endif +endif + +ccflags-y += -DSVN_REV=$(SVN_REV) +ccflags-y += -DSVN_REV_STRING=\"$(DRIVER_REV)\" + +ccflags-y += -I$(src) -I$(src)/common -I$(src)/linux -I$(src)/../mali/common -I$(src)/../mali/linux -I$(src)/include -I$(src)/../../ump/include/ump +ccflags-y += -DMALI_STATE_TRACKING=0 +ccflags-y += -DMALI_ENABLE_CPU_CYCLES=0 +ccflags-$(CONFIG_UMP_DEBUG) += -DDEBUG + +# For customer releases the Linux Device Drivers will be provided as ARM proprietary and GPL releases: +# The ARM proprietary product will only include the license/proprietary directory +# The GPL product will only include the license/gpl directory + +ifeq ($(wildcard $(src)/linux/license/gpl/*),) +ccflags-y += -I$(src)/linux/license/proprietary -I$(src)/../mali/linux/license/proprietary +else +ccflags-y += -I$(src)/linux/license/gpl -I$(src)/../mali/linux/license/gpl +endif + +ump-y = common/ump_kernel_common.o \ + common/ump_kernel_descriptor_mapping.o \ + common/ump_kernel_api.o \ + common/ump_kernel_ref_drv.o \ + linux/ump_kernel_linux.o \ + linux/ump_kernel_memory_backend_os.o \ + linux/ump_kernel_memory_backend_dedicated.o \ + linux/ump_memory_backend.o \ + linux/ump_ukk_wrappers.o \ + linux/ump_ukk_ref_wrappers.o \ + linux/ump_osk_atomics.o \ + linux/ump_osk_low_level_mem.o \ + linux/ump_osk_misc.o \ + linux/ump_kernel_random_mapping.o + +ifneq ($(CONFIG_MALI400),y) +ump-y += $(UDD_FILE_PREFIX)linux/mali_osk_atomics.o \ + $(UDD_FILE_PREFIX)linux/mali_osk_locks.o \ + $(UDD_FILE_PREFIX)linux/mali_osk_memory.o \ + $(UDD_FILE_PREFIX)linux/mali_osk_math.o \ + $(UDD_FILE_PREFIX)linux/mali_osk_misc.o +endif + +obj-$(CONFIG_UMP) := ump.o + diff --git a/drivers/gpu/arm/mali400/ump/Kconfig b/drivers/gpu/arm/mali400/ump/Kconfig new file mode 100755 index 000000000000..ec3509057732 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0 +config UMP + tristate "UMP support" + depends on ARM + help + This enables support for the UMP memory allocation and sharing API. + + To compile this driver as a module, choose M here: the module will be + called ump. + +config UMP_DEBUG + bool "Enable extra debug in UMP" + depends on UMP + default y + help + This enabled extra debug checks and messages in UMP. + diff --git a/drivers/gpu/arm/mali400/ump/Makefile b/drivers/gpu/arm/mali400/ump/Makefile new file mode 100755 index 000000000000..88b02a22fce5 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/Makefile @@ -0,0 +1,67 @@ +# +# Copyright (C) 2010-2012, 2014, 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the GNU General Public License version 2 +# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained from Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +# For each arch check: CROSS_COMPILE , KDIR , CFLAGS += -DARCH + +export ARCH ?= arm +BUILD ?= debug + +check_cc2 = \ + $(shell if $(1) -S -o /dev/null -xc /dev/null > /dev/null 2>&1; \ + then \ + echo "$(2)"; \ + else \ + echo "$(3)"; \ + fi ;) + +# Check that required parameters are supplied. +ifeq ($(CONFIG),) +CONFIG := default +endif +ifeq ($(CPU)$(KDIR),) +$(error "KDIR or CPU must be specified.") +endif + +# Get any user defined KDIR- or maybe even a hardcoded KDIR +-include KDIR_CONFIGURATION + +# Define host system directory +KDIR-$(shell uname -m):=/lib/modules/$(shell uname -r)/build + +ifeq ($(ARCH), arm) +# when compiling for ARM we're cross compiling +export CROSS_COMPILE ?= $(call check_cc2, arm-linux-gnueabi-gcc, arm-linux-gnueabi-, arm-none-linux-gnueabi-) +endif + +# look up KDIR based om CPU selection +KDIR ?= $(KDIR-$(CPU)) + +export CONFIG + +export CONFIG_UMP := m +ifeq ($(BUILD),debug) +export CONFIG_UMP_DEBUG := y +else +export CONFIG_UMP_DEBUG := n +endif + +ifeq ($(KDIR),) +$(error No KDIR found for platform $(CPU)) +endif + +all: + $(MAKE) -C $(KDIR) M=$(CURDIR) modules + +kernelrelease: + $(MAKE) -C $(KDIR) kernelrelease + +clean: + $(MAKE) -C $(KDIR) M=$(CURDIR) clean + $(MAKE) -C $(KDIR) M=$(CURDIR)/../mali clean diff --git a/drivers/gpu/arm/mali400/ump/Makefile.common b/drivers/gpu/arm/mali400/ump/Makefile.common new file mode 100755 index 000000000000..ad2c18da98a0 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/Makefile.common @@ -0,0 +1,20 @@ +# +# Copyright (C) 2010-2011, 2013, 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the GNU General Public License version 2 +# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained from Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +SRC = $(UMP_FILE_PREFIX)common/ump_kernel_common.c \ + $(UMP_FILE_PREFIX)common/ump_kernel_descriptor_mapping.c \ + $(UMP_FILE_PREFIX)common/ump_kernel_api.c \ + $(UMP_FILE_PREFIX)common/ump_kernel_ref_drv.c + +# Get subversion revision number, fall back to 0000 if no svn info is available +SVN_REV:=$(shell ((svnversion | grep -qv exported && echo -n 'Revision: ' && svnversion) || git svn info | sed -e 's/$$$$/M/' | grep '^Revision: ' || echo ${MALI_RELEASE_NAME}) 2>/dev/null | sed -e 's/^Revision: //') + +EXTRA_CFLAGS += -DSVN_REV=$(SVN_REV) +EXTRA_CFLAGS += -DSVN_REV_STRING=\"$(SVN_REV)\" diff --git a/drivers/gpu/arm/mali400/ump/arch-default/config.h b/drivers/gpu/arm/mali400/ump/arch-default/config.h new file mode 100755 index 000000000000..d4aef9dd09c7 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/arch-default/config.h @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2010, 2012, 2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __ARCH_CONFIG_H__ +#define __ARCH_CONFIG_H__ + +/* Use OS memory. */ +#define ARCH_UMP_BACKEND_DEFAULT 1 + +/* OS memory won't need a base address. */ +#define ARCH_UMP_MEMORY_ADDRESS_DEFAULT 0x00000000 + +/* 512 MB maximum limit for UMP allocations. */ +#define ARCH_UMP_MEMORY_SIZE_DEFAULT 512UL * 1024UL * 1024UL + + +#endif /* __ARCH_CONFIG_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h b/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h new file mode 100755 index 000000000000..182e90c1d64f --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2010-2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __ARCH_CONFIG_H__ +#define __ARCH_CONFIG_H__ + +#define ARCH_UMP_BACKEND_DEFAULT 0 +#define ARCH_UMP_MEMORY_ADDRESS_DEFAULT 0xE1000000 +#define ARCH_UMP_MEMORY_SIZE_DEFAULT 16UL * 1024UL * 1024UL + +#endif /* __ARCH_CONFIG_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/arch/config.h b/drivers/gpu/arm/mali400/ump/arch/config.h new file mode 100755 index 000000000000..d4aef9dd09c7 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/arch/config.h @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2010, 2012, 2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __ARCH_CONFIG_H__ +#define __ARCH_CONFIG_H__ + +/* Use OS memory. */ +#define ARCH_UMP_BACKEND_DEFAULT 1 + +/* OS memory won't need a base address. */ +#define ARCH_UMP_MEMORY_ADDRESS_DEFAULT 0x00000000 + +/* 512 MB maximum limit for UMP allocations. */ +#define ARCH_UMP_MEMORY_SIZE_DEFAULT 512UL * 1024UL * 1024UL + + +#endif /* __ARCH_CONFIG_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c new file mode 100755 index 000000000000..36adb2f5383e --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c @@ -0,0 +1,455 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "ump_osk.h" +#include "ump_uk_types.h" +#include "ump_kernel_interface.h" +#include "ump_kernel_common.h" +#include "ump_kernel_random_mapping.h" + + + +/* ---------------- UMP kernel space API functions follows ---------------- */ + + + +UMP_KERNEL_API_EXPORT ump_secure_id ump_dd_secure_id_get(ump_dd_handle memh) +{ + ump_dd_mem *mem = (ump_dd_mem *)memh; + + DEBUG_ASSERT_POINTER(mem); + + DBG_MSG(5, ("Returning secure ID. ID: %u\n", mem->secure_id)); + + return mem->secure_id; +} + + + +UMP_KERNEL_API_EXPORT ump_dd_handle ump_dd_handle_create_from_secure_id(ump_secure_id secure_id) +{ + ump_dd_mem *mem; + + DBG_MSG(5, ("Getting handle from secure ID. ID: %u\n", secure_id)); + mem = ump_random_mapping_get(device.secure_id_map, (int)secure_id); + if (NULL == mem) { + DBG_MSG(1, ("Secure ID not found. ID: %u\n", secure_id)); + return UMP_DD_HANDLE_INVALID; + } + + /* Keep the reference taken in ump_random_mapping_get() */ + + return (ump_dd_handle)mem; +} + + + +UMP_KERNEL_API_EXPORT unsigned long ump_dd_phys_block_count_get(ump_dd_handle memh) +{ + ump_dd_mem *mem = (ump_dd_mem *) memh; + + DEBUG_ASSERT_POINTER(mem); + + return mem->nr_blocks; +} + + + +UMP_KERNEL_API_EXPORT ump_dd_status_code ump_dd_phys_blocks_get(ump_dd_handle memh, ump_dd_physical_block *blocks, unsigned long num_blocks) +{ + ump_dd_mem *mem = (ump_dd_mem *)memh; + + DEBUG_ASSERT_POINTER(mem); + + if (blocks == NULL) { + DBG_MSG(1, ("NULL parameter in ump_dd_phys_blocks_get()\n")); + return UMP_DD_INVALID; + } + + if (mem->nr_blocks != num_blocks) { + DBG_MSG(1, ("Specified number of blocks do not match actual number of blocks\n")); + return UMP_DD_INVALID; + } + + DBG_MSG(5, ("Returning physical block information. ID: %u\n", mem->secure_id)); + + _mali_osk_memcpy(blocks, mem->block_array, sizeof(ump_dd_physical_block) * mem->nr_blocks); + + return UMP_DD_SUCCESS; +} + + + +UMP_KERNEL_API_EXPORT ump_dd_status_code ump_dd_phys_block_get(ump_dd_handle memh, unsigned long index, ump_dd_physical_block *block) +{ + ump_dd_mem *mem = (ump_dd_mem *)memh; + + DEBUG_ASSERT_POINTER(mem); + + if (block == NULL) { + DBG_MSG(1, ("NULL parameter in ump_dd_phys_block_get()\n")); + return UMP_DD_INVALID; + } + + if (index >= mem->nr_blocks) { + DBG_MSG(5, ("Invalid index specified in ump_dd_phys_block_get()\n")); + return UMP_DD_INVALID; + } + + DBG_MSG(5, ("Returning physical block information. ID: %u, index: %lu\n", mem->secure_id, index)); + + *block = mem->block_array[index]; + + return UMP_DD_SUCCESS; +} + + + +UMP_KERNEL_API_EXPORT unsigned long ump_dd_size_get(ump_dd_handle memh) +{ + ump_dd_mem *mem = (ump_dd_mem *)memh; + + DEBUG_ASSERT_POINTER(mem); + + DBG_MSG(5, ("Returning size. ID: %u, size: %lu\n", mem->secure_id, mem->size_bytes)); + + return mem->size_bytes; +} + + + +UMP_KERNEL_API_EXPORT void ump_dd_reference_add(ump_dd_handle memh) +{ + ump_dd_mem *mem = (ump_dd_mem *)memh; + int new_ref; + + DEBUG_ASSERT_POINTER(mem); + + new_ref = _ump_osk_atomic_inc_and_read(&mem->ref_count); + + DBG_MSG(5, ("Memory reference incremented. ID: %u, new value: %d\n", mem->secure_id, new_ref)); +} + + + +UMP_KERNEL_API_EXPORT void ump_dd_reference_release(ump_dd_handle memh) +{ + ump_dd_mem *mem = (ump_dd_mem *)memh; + + DEBUG_ASSERT_POINTER(mem); + + ump_random_mapping_put(mem); +} + + + +/* --------------- Handling of user space requests follows --------------- */ + + +_mali_osk_errcode_t _ump_uku_get_api_version(_ump_uk_api_version_s *args) +{ + ump_session_data *session_data; + + DEBUG_ASSERT_POINTER(args); + DEBUG_ASSERT_POINTER(args->ctx); + + session_data = (ump_session_data *)args->ctx; + + /* check compatability */ + if (args->version == UMP_IOCTL_API_VERSION) { + DBG_MSG(3, ("API version set to newest %d (compatible)\n", + GET_VERSION(args->version))); + args->compatible = 1; + session_data->api_version = args->version; + } else { + DBG_MSG(2, ("API version set to %d (incompatible with client version %d)\n", + GET_VERSION(UMP_IOCTL_API_VERSION), GET_VERSION(args->version))); + args->compatible = 0; + args->version = UMP_IOCTL_API_VERSION; /* report our version */ + } + + return _MALI_OSK_ERR_OK; +} + + +_mali_osk_errcode_t _ump_ukk_release(_ump_uk_release_s *release_info) +{ + ump_session_memory_list_element *session_memory_element; + ump_session_memory_list_element *tmp; + ump_session_data *session_data; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_INVALID_FUNC; + int secure_id; + + DEBUG_ASSERT_POINTER(release_info); + DEBUG_ASSERT_POINTER(release_info->ctx); + + /* Retreive the session data */ + session_data = (ump_session_data *)release_info->ctx; + + /* If there are many items in the memory session list we + * could be de-referencing this pointer a lot so keep a local copy + */ + secure_id = release_info->secure_id; + + DBG_MSG(4, ("Releasing memory with IOCTL, ID: %u\n", secure_id)); + + /* Iterate through the memory list looking for the requested secure ID */ + _mali_osk_mutex_wait(session_data->lock); + _MALI_OSK_LIST_FOREACHENTRY(session_memory_element, tmp, &session_data->list_head_session_memory_list, ump_session_memory_list_element, list) { + if (session_memory_element->mem->secure_id == secure_id) { + ump_dd_mem *release_mem; + + release_mem = session_memory_element->mem; + _mali_osk_list_del(&session_memory_element->list); + ump_dd_reference_release(release_mem); + _mali_osk_free(session_memory_element); + + ret = _MALI_OSK_ERR_OK; + break; + } + } + + _mali_osk_mutex_signal(session_data->lock); + DBG_MSG_IF(1, _MALI_OSK_ERR_OK != ret, ("UMP memory with ID %u does not belong to this session.\n", secure_id)); + + DBG_MSG(4, ("_ump_ukk_release() returning 0x%x\n", ret)); + return ret; +} + +_mali_osk_errcode_t _ump_ukk_size_get(_ump_uk_size_get_s *user_interaction) +{ + ump_dd_mem *mem; + _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; + + DEBUG_ASSERT_POINTER(user_interaction); + + /* We lock the mappings so things don't get removed while we are looking for the memory */ + mem = ump_random_mapping_get(device.secure_id_map, user_interaction->secure_id); + if (NULL != mem) { + user_interaction->size = mem->size_bytes; + DBG_MSG(4, ("Returning size. ID: %u, size: %lu ", + (ump_secure_id)user_interaction->secure_id, + (unsigned long)user_interaction->size)); + ump_random_mapping_put(mem); + ret = _MALI_OSK_ERR_OK; + } else { + user_interaction->size = 0; + DBG_MSG(1, ("Failed to look up mapping in ump_ioctl_size_get(). ID: %u\n", + (ump_secure_id)user_interaction->secure_id)); + } + + return ret; +} + + + +void _ump_ukk_msync(_ump_uk_msync_s *args) +{ + ump_dd_mem *mem = NULL; + void *virtual = NULL; + u32 size = 0; + u32 offset = 0; + + mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); + if (NULL == mem) { + DBG_MSG(1, ("Failed to look up mapping in _ump_ukk_msync(). ID: %u\n", + (ump_secure_id)args->secure_id)); + return; + } + + /* Returns the cache settings back to Userspace */ + args->is_cached = mem->is_cached; + + /* If this flag is the only one set, we should not do the actual flush, only the readout */ + if (_UMP_UK_MSYNC_READOUT_CACHE_ENABLED == args->op) { + DBG_MSG(3, ("_ump_ukk_msync READOUT ID: %u Enabled: %d\n", (ump_secure_id)args->secure_id, mem->is_cached)); + goto msync_release_and_return; + } + + /* Nothing to do if the memory is not caches */ + if (0 == mem->is_cached) { + DBG_MSG(3, ("_ump_ukk_msync IGNORING ID: %u Enabled: %d OP: %d\n", (ump_secure_id)args->secure_id, mem->is_cached, args->op)); + goto msync_release_and_return; + } + DBG_MSG(3, ("UMP[%02u] _ump_ukk_msync Flush OP: %d Address: 0x%08x Mapping: 0x%08x\n", + (ump_secure_id)args->secure_id, args->op, args->address, args->mapping)); + + if (args->address) { + virtual = (void *)((u32)args->address); + offset = (u32)((args->address) - (args->mapping)); + } else { + /* Flush entire mapping when no address is specified. */ + virtual = args->mapping; + } + if (args->size) { + size = args->size; + } else { + /* Flush entire mapping when no size is specified. */ + size = mem->size_bytes - offset; + } + + if ((offset + size) > mem->size_bytes) { + DBG_MSG(1, ("Trying to flush more than the entire UMP allocation: offset: %u + size: %u > %u\n", offset, size, mem->size_bytes)); + goto msync_release_and_return; + } + + /* The actual cache flush - Implemented for each OS*/ + _ump_osk_msync(mem, virtual, offset, size, args->op, NULL); + +msync_release_and_return: + ump_random_mapping_put(mem); + return; +} + +void _ump_ukk_cache_operations_control(_ump_uk_cache_operations_control_s *args) +{ + ump_session_data *session_data; + ump_uk_cache_op_control op; + + DEBUG_ASSERT_POINTER(args); + DEBUG_ASSERT_POINTER(args->ctx); + + op = args->op; + session_data = (ump_session_data *)args->ctx; + + _mali_osk_mutex_wait(session_data->lock); + if (op == _UMP_UK_CACHE_OP_START) { + session_data->cache_operations_ongoing++; + DBG_MSG(4, ("Cache ops start\n")); + if (session_data->cache_operations_ongoing != 1) { + DBG_MSG(2, ("UMP: Number of simultanious cache control ops: %d\n", session_data->cache_operations_ongoing)); + } + } else if (op == _UMP_UK_CACHE_OP_FINISH) { + DBG_MSG(4, ("Cache ops finish\n")); + session_data->cache_operations_ongoing--; +#if 0 + if (session_data->has_pending_level1_cache_flush) { + /* This function will set has_pending_level1_cache_flush=0 */ + _ump_osk_msync(NULL, NULL, 0, 0, _UMP_UK_MSYNC_FLUSH_L1, session_data); + } +#endif + + /* to be on the safe side: always flush l1 cache when cache operations are done */ + _ump_osk_msync(NULL, NULL, 0, 0, _UMP_UK_MSYNC_FLUSH_L1, session_data); + DBG_MSG(4, ("Cache ops finish end\n")); + } else { + DBG_MSG(1, ("Illegal call to %s at line %d\n", __FUNCTION__, __LINE__)); + } + _mali_osk_mutex_signal(session_data->lock); + +} + +void _ump_ukk_switch_hw_usage(_ump_uk_switch_hw_usage_s *args) +{ + ump_dd_mem *mem = NULL; + ump_uk_user old_user; + ump_uk_msync_op cache_op = _UMP_UK_MSYNC_CLEAN_AND_INVALIDATE; + ump_session_data *session_data; + + DEBUG_ASSERT_POINTER(args); + DEBUG_ASSERT_POINTER(args->ctx); + + session_data = (ump_session_data *)args->ctx; + + mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); + if (NULL == mem) { + DBG_MSG(1, ("Failed to look up mapping in _ump_ukk_switch_hw_usage(). ID: %u\n", + (ump_secure_id)args->secure_id)); + return; + } + + old_user = mem->hw_device; + mem->hw_device = args->new_user; + + DBG_MSG(3, ("UMP[%02u] Switch usage Start New: %s Prev: %s.\n", + (ump_secure_id)args->secure_id, + args->new_user ? "MALI" : "CPU", + old_user ? "MALI" : "CPU")); + + if (!mem->is_cached) { + DBG_MSG(3, ("UMP[%02u] Changing owner of uncached memory. Cache flushing not needed.\n", + (ump_secure_id)args->secure_id)); + goto out; + } + + if (old_user == args->new_user) { + DBG_MSG(4, ("UMP[%02u] Setting the new_user equal to previous for. Cache flushing not needed.\n", + (ump_secure_id)args->secure_id)); + goto out; + } + if ( + /* Previous AND new is both different from CPU */ + (old_user != _UMP_UK_USED_BY_CPU) && (args->new_user != _UMP_UK_USED_BY_CPU) + ) { + DBG_MSG(4, ("UMP[%02u] Previous and new user is not CPU. Cache flushing not needed.\n", + (ump_secure_id)args->secure_id)); + goto out; + } + + if ((old_user != _UMP_UK_USED_BY_CPU) && (args->new_user == _UMP_UK_USED_BY_CPU)) { + cache_op = _UMP_UK_MSYNC_INVALIDATE; + DBG_MSG(4, ("UMP[%02u] Cache invalidation needed\n", (ump_secure_id)args->secure_id)); +#ifdef UMP_SKIP_INVALIDATION +#error + DBG_MSG(4, ("UMP[%02u] Performing Cache invalidation SKIPPED\n", (ump_secure_id)args->secure_id)); + goto out; +#endif + } + + /* Take lock to protect: session->cache_operations_ongoing and session->has_pending_level1_cache_flush */ + _mali_osk_mutex_wait(session_data->lock); + /* Actual cache flush */ + _ump_osk_msync(mem, NULL, 0, mem->size_bytes, cache_op, session_data); + _mali_osk_mutex_signal(session_data->lock); + +out: + ump_random_mapping_put(mem); + DBG_MSG(4, ("UMP[%02u] Switch usage Finish\n", (ump_secure_id)args->secure_id)); + return; +} + +void _ump_ukk_lock(_ump_uk_lock_s *args) +{ + ump_dd_mem *mem = NULL; + + mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); + if (NULL == mem) { + DBG_MSG(1, ("UMP[%02u] Failed to look up mapping in _ump_ukk_lock(). ID: %u\n", + (ump_secure_id)args->secure_id)); + return; + } + + DBG_MSG(1, ("UMP[%02u] Lock. New lock flag: %d. Old Lock flag:\n", (u32)args->secure_id, (u32)args->lock_usage, (u32) mem->lock_usage)); + + mem->lock_usage = (ump_lock_usage) args->lock_usage; + + ump_random_mapping_put(mem); +} + +void _ump_ukk_unlock(_ump_uk_unlock_s *args) +{ + ump_dd_mem *mem = NULL; + + mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); + if (NULL == mem) { + DBG_MSG(1, ("Failed to look up mapping in _ump_ukk_unlock(). ID: %u\n", + (ump_secure_id)args->secure_id)); + return; + } + + DBG_MSG(1, ("UMP[%02u] Unlocking. Old Lock flag:\n", + (u32)args->secure_id, (u32) mem->lock_usage)); + + mem->lock_usage = (ump_lock_usage) UMP_NOT_LOCKED; + + ump_random_mapping_put(mem); +} diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c new file mode 100755 index 000000000000..73aa9e4c49f9 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c @@ -0,0 +1,358 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_osk_bitops.h" +#include "mali_osk_list.h" +#include "ump_osk.h" +#include "ump_uk_types.h" +#include "ump_ukk.h" +#include "ump_kernel_common.h" +#include "ump_kernel_descriptor_mapping.h" +#include "ump_kernel_memory_backend.h" + + + +/** + * Define the initial and maximum size of number of secure_ids on the system + */ +#define UMP_SECURE_ID_TABLE_ENTRIES_INITIAL (128 ) +#define UMP_SECURE_ID_TABLE_ENTRIES_MAXIMUM (4096 ) + + +/** + * Define the initial and maximum size of the ump_session_data::cookies_map, + * which is a \ref ump_descriptor_mapping. This limits how many secure_ids + * may be mapped into a particular process using _ump_ukk_map_mem(). + */ + +#define UMP_COOKIES_PER_SESSION_INITIAL (UMP_SECURE_ID_TABLE_ENTRIES_INITIAL ) +#define UMP_COOKIES_PER_SESSION_MAXIMUM (UMP_SECURE_ID_TABLE_ENTRIES_MAXIMUM) + +struct ump_dev device; + +_mali_osk_errcode_t ump_kernel_constructor(void) +{ + _mali_osk_errcode_t err; + + /* Perform OS Specific initialization */ + err = _ump_osk_init(); + if (_MALI_OSK_ERR_OK != err) { + MSG_ERR(("Failed to initiaze the UMP Device Driver")); + return err; + } + + /* Init the global device */ + _mali_osk_memset(&device, 0, sizeof(device)); + + /* Create the descriptor map, which will be used for mapping secure ID to ump_dd_mem structs */ + device.secure_id_map = ump_random_mapping_create(); + if (NULL == device.secure_id_map) { + MSG_ERR(("Failed to create secure id lookup table\n")); + return _MALI_OSK_ERR_NOMEM; + } + + /* Init memory backend */ + device.backend = ump_memory_backend_create(); + if (NULL == device.backend) { + MSG_ERR(("Failed to create memory backend\n")); + ump_random_mapping_destroy(device.secure_id_map); + return _MALI_OSK_ERR_NOMEM; + } + + return _MALI_OSK_ERR_OK; +} + +void ump_kernel_destructor(void) +{ + DEBUG_ASSERT_POINTER(device.secure_id_map); + + ump_random_mapping_destroy(device.secure_id_map); + device.secure_id_map = NULL; + + device.backend->shutdown(device.backend); + device.backend = NULL; + + ump_memory_backend_destroy(); + + _ump_osk_term(); +} + +/** Creates a new UMP session + */ +_mali_osk_errcode_t _ump_ukk_open(void **context) +{ + struct ump_session_data *session_data; + + /* allocated struct to track this session */ + session_data = (struct ump_session_data *)_mali_osk_malloc(sizeof(struct ump_session_data)); + if (NULL == session_data) { + MSG_ERR(("Failed to allocate ump_session_data in ump_file_open()\n")); + return _MALI_OSK_ERR_NOMEM; + } + + session_data->lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_UNORDERED, 0); + if (NULL == session_data->lock) { + MSG_ERR(("Failed to initialize lock for ump_session_data in ump_file_open()\n")); + _mali_osk_free(session_data); + return _MALI_OSK_ERR_NOMEM; + } + + session_data->cookies_map = ump_descriptor_mapping_create( + UMP_COOKIES_PER_SESSION_INITIAL, + UMP_COOKIES_PER_SESSION_MAXIMUM); + + if (NULL == session_data->cookies_map) { + MSG_ERR(("Failed to create descriptor mapping for _ump_ukk_map_mem cookies\n")); + + _mali_osk_mutex_term(session_data->lock); + _mali_osk_free(session_data); + return _MALI_OSK_ERR_NOMEM; + } + + _MALI_OSK_INIT_LIST_HEAD(&session_data->list_head_session_memory_list); + + _MALI_OSK_INIT_LIST_HEAD(&session_data->list_head_session_memory_mappings_list); + + /* Since initial version of the UMP interface did not use the API_VERSION ioctl we have to assume + that it is this version, and not the "latest" one: UMP_IOCTL_API_VERSION + Current and later API versions would do an additional call to this IOCTL and update this variable + to the correct one.*/ + session_data->api_version = MAKE_VERSION_ID(1); + + *context = (void *)session_data; + + session_data->cache_operations_ongoing = 0 ; + session_data->has_pending_level1_cache_flush = 0; + + DBG_MSG(2, ("New session opened\n")); + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _ump_ukk_close(void **context) +{ + struct ump_session_data *session_data; + ump_session_memory_list_element *item; + ump_session_memory_list_element *tmp; + + session_data = (struct ump_session_data *)*context; + if (NULL == session_data) { + MSG_ERR(("Session data is NULL in _ump_ukk_close()\n")); + return _MALI_OSK_ERR_INVALID_ARGS; + } + + /* Unmap any descriptors mapped in. */ + if (0 == _mali_osk_list_empty(&session_data->list_head_session_memory_mappings_list)) { + ump_memory_allocation *descriptor; + ump_memory_allocation *temp; + + DBG_MSG(1, ("Memory mappings found on session usage list during session termination\n")); + + /* use the 'safe' list iterator, since freeing removes the active block from the list we're iterating */ + _MALI_OSK_LIST_FOREACHENTRY(descriptor, temp, &session_data->list_head_session_memory_mappings_list, ump_memory_allocation, list) { + _ump_uk_unmap_mem_s unmap_args; + DBG_MSG(4, ("Freeing block with phys address 0x%x size 0x%x mapped in user space at 0x%x\n", + descriptor->phys_addr, descriptor->size, descriptor->mapping)); + unmap_args.ctx = (void *)session_data; + unmap_args.mapping = descriptor->mapping; + unmap_args.size = descriptor->size; + unmap_args._ukk_private = NULL; /* NOTE: unused */ + unmap_args.cookie = descriptor->cookie; + + /* NOTE: This modifies the list_head_session_memory_mappings_list */ + _ump_ukk_unmap_mem(&unmap_args); + } + } + + /* ASSERT that we really did free everything, because _ump_ukk_unmap_mem() + * can fail silently. */ + DEBUG_ASSERT(_mali_osk_list_empty(&session_data->list_head_session_memory_mappings_list)); + + _MALI_OSK_LIST_FOREACHENTRY(item, tmp, &session_data->list_head_session_memory_list, ump_session_memory_list_element, list) { + _mali_osk_list_del(&item->list); + DBG_MSG(2, ("Releasing UMP memory %u as part of file close\n", item->mem->secure_id)); + ump_dd_reference_release(item->mem); + _mali_osk_free(item); + } + + ump_descriptor_mapping_destroy(session_data->cookies_map); + + _mali_osk_mutex_term(session_data->lock); + _mali_osk_free(session_data); + + DBG_MSG(2, ("Session closed\n")); + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _ump_ukk_map_mem(_ump_uk_map_mem_s *args) +{ + struct ump_session_data *session_data; + ump_memory_allocation *descriptor; /* Describes current mapping of memory */ + _mali_osk_errcode_t err; + unsigned long offset = 0; + unsigned long left; + ump_dd_handle handle; /* The real UMP handle for this memory. Its real datatype is ump_dd_mem* */ + ump_dd_mem *mem; /* The real UMP memory. It is equal to the handle, but with exposed struct */ + u32 block; + int map_id; + + session_data = (ump_session_data *)args->ctx; + if (NULL == session_data) { + MSG_ERR(("Session data is NULL in _ump_ukk_map_mem()\n")); + return _MALI_OSK_ERR_INVALID_ARGS; + } + + descriptor = (ump_memory_allocation *) _mali_osk_calloc(1, sizeof(ump_memory_allocation)); + if (NULL == descriptor) { + MSG_ERR(("ump_ukk_map_mem: descriptor allocation failed\n")); + return _MALI_OSK_ERR_NOMEM; + } + + handle = ump_dd_handle_create_from_secure_id(args->secure_id); + if (UMP_DD_HANDLE_INVALID == handle) { + _mali_osk_free(descriptor); + DBG_MSG(1, ("Trying to map unknown secure ID %u\n", args->secure_id)); + return _MALI_OSK_ERR_FAULT; + } + + mem = (ump_dd_mem *)handle; + DEBUG_ASSERT(mem); + if (mem->size_bytes != args->size) { + _mali_osk_free(descriptor); + ump_dd_reference_release(handle); + DBG_MSG(1, ("Trying to map too much or little. ID: %u, virtual size=%lu, UMP size: %lu\n", args->secure_id, args->size, mem->size_bytes)); + return _MALI_OSK_ERR_FAULT; + } + + map_id = ump_descriptor_mapping_allocate_mapping(session_data->cookies_map, (void *) descriptor); + + if (map_id < 0) { + _mali_osk_free(descriptor); + ump_dd_reference_release(handle); + DBG_MSG(1, ("ump_ukk_map_mem: unable to allocate a descriptor_mapping for return cookie\n")); + + return _MALI_OSK_ERR_NOMEM; + } + + descriptor->size = args->size; + descriptor->handle = handle; + descriptor->phys_addr = args->phys_addr; + descriptor->process_mapping_info = args->_ukk_private; + descriptor->ump_session = session_data; + descriptor->cookie = (u32)map_id; + + if (mem->is_cached) { + descriptor->is_cached = 1; + DBG_MSG(3, ("Mapping UMP secure_id: %d as cached.\n", args->secure_id)); + } else { + descriptor->is_cached = 0; + DBG_MSG(3, ("Mapping UMP secure_id: %d as Uncached.\n", args->secure_id)); + } + + _mali_osk_list_init(&descriptor->list); + + err = _ump_osk_mem_mapregion_init(descriptor); + if (_MALI_OSK_ERR_OK != err) { + DBG_MSG(1, ("Failed to initialize memory mapping in _ump_ukk_map_mem(). ID: %u\n", args->secure_id)); + ump_descriptor_mapping_free(session_data->cookies_map, map_id); + _mali_osk_free(descriptor); + ump_dd_reference_release(mem); + return err; + } + + DBG_MSG(4, ("Mapping virtual to physical memory: ID: %u, size:%lu, first physical addr: 0x%08lx, number of regions: %lu\n", + mem->secure_id, + mem->size_bytes, + ((NULL != mem->block_array) ? mem->block_array->addr : 0), + mem->nr_blocks)); + + left = descriptor->size; + /* loop over all blocks and map them in */ + for (block = 0; block < mem->nr_blocks; block++) { + unsigned long size_to_map; + + if (left > mem->block_array[block].size) { + size_to_map = mem->block_array[block].size; + } else { + size_to_map = left; + } + + if (_MALI_OSK_ERR_OK != _ump_osk_mem_mapregion_map(descriptor, offset, (u32 *) & (mem->block_array[block].addr), size_to_map)) { + DBG_MSG(1, ("WARNING: _ump_ukk_map_mem failed to map memory into userspace\n")); + ump_descriptor_mapping_free(session_data->cookies_map, map_id); + ump_dd_reference_release(mem); + _ump_osk_mem_mapregion_term(descriptor); + _mali_osk_free(descriptor); + return _MALI_OSK_ERR_FAULT; + } + left -= size_to_map; + offset += size_to_map; + } + + /* Add to the ump_memory_allocation tracking list */ + _mali_osk_mutex_wait(session_data->lock); + _mali_osk_list_add(&descriptor->list, &session_data->list_head_session_memory_mappings_list); + _mali_osk_mutex_signal(session_data->lock); + + args->mapping = descriptor->mapping; + args->cookie = descriptor->cookie; + + return _MALI_OSK_ERR_OK; +} + +void _ump_ukk_unmap_mem(_ump_uk_unmap_mem_s *args) +{ + struct ump_session_data *session_data; + ump_memory_allocation *descriptor; + ump_dd_handle handle; + + session_data = (ump_session_data *)args->ctx; + + if (NULL == session_data) { + MSG_ERR(("Session data is NULL in _ump_ukk_map_mem()\n")); + return; + } + + if (0 != ump_descriptor_mapping_get(session_data->cookies_map, (int)args->cookie, (void **)&descriptor)) { + MSG_ERR(("_ump_ukk_map_mem: cookie 0x%X not found for this session\n", args->cookie)); + return; + } + + DEBUG_ASSERT_POINTER(descriptor); + + handle = descriptor->handle; + if (UMP_DD_HANDLE_INVALID == handle) { + DBG_MSG(1, ("WARNING: Trying to unmap unknown handle: UNKNOWN\n")); + return; + } + + /* Remove the ump_memory_allocation from the list of tracked mappings */ + _mali_osk_mutex_wait(session_data->lock); + _mali_osk_list_del(&descriptor->list); + _mali_osk_mutex_signal(session_data->lock); + + ump_descriptor_mapping_free(session_data->cookies_map, (int)args->cookie); + + ump_dd_reference_release(handle); + + _ump_osk_mem_mapregion_term(descriptor); + _mali_osk_free(descriptor); +} + +u32 _ump_ukk_report_memory_usage(void) +{ + if (device.backend->stat) + return device.backend->stat(device.backend); + else + return 0; +} diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h new file mode 100755 index 000000000000..aa65f1cb6c88 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __UMP_KERNEL_COMMON_H__ +#define __UMP_KERNEL_COMMON_H__ + +#include "ump_kernel_types.h" +#include "ump_kernel_interface.h" +#include "ump_kernel_descriptor_mapping.h" +#include "ump_kernel_random_mapping.h" +#include "ump_kernel_memory_backend.h" + + +#ifdef DEBUG +extern int ump_debug_level; +#define UMP_DEBUG_PRINT(args) _mali_osk_dbgmsg args +#define UMP_DEBUG_CODE(args) args +#define DBG_MSG(level,args) do { /* args should be in brackets */ \ + ((level) <= ump_debug_level)?\ + UMP_DEBUG_PRINT(("UMP<" #level ">: ")), \ + UMP_DEBUG_PRINT(args):0; \ + } while (0) + +#define DBG_MSG_IF(level,condition,args) /* args should be in brackets */ \ + if((condition)&&((level) <= ump_debug_level)) {\ + UMP_DEBUG_PRINT(("UMP<" #level ">: ")); \ + UMP_DEBUG_PRINT(args); \ + } + +#define DBG_MSG_ELSE(level,args) /* args should be in brackets */ \ + else if((level) <= ump_debug_level) { \ + UMP_DEBUG_PRINT(("UMP<" #level ">: ")); \ + UMP_DEBUG_PRINT(args); \ + } + +#define DEBUG_ASSERT_POINTER(pointer) do {if( (pointer)== NULL) MSG_ERR(("NULL pointer " #pointer)); } while(0) +#define DEBUG_ASSERT(condition) do {if(!(condition)) MSG_ERR(("ASSERT failed: " #condition)); } while(0) +#else /* DEBUG */ +#define UMP_DEBUG_PRINT(args) do {} while(0) +#define UMP_DEBUG_CODE(args) +#define DBG_MSG(level,args) do {} while(0) +#define DBG_MSG_IF(level,condition,args) do {} while(0) +#define DBG_MSG_ELSE(level,args) do {} while(0) +#define DEBUG_ASSERT(condition) do {} while(0) +#define DEBUG_ASSERT_POINTER(pointer) do {} while(0) +#endif /* DEBUG */ + +#define MSG_ERR(args) do{ /* args should be in brackets */ \ + _mali_osk_dbgmsg("UMP: ERR: %s\n" ,__FILE__); \ + _mali_osk_dbgmsg( " %s()%4d\n", __FUNCTION__, __LINE__) ; \ + _mali_osk_dbgmsg args ; \ + _mali_osk_dbgmsg("\n"); \ + } while(0) + +#define MSG(args) do{ /* args should be in brackets */ \ + _mali_osk_dbgmsg("UMP: "); \ + _mali_osk_dbgmsg args; \ + } while (0) + + + +/* + * This struct is used to store per session data. + * A session is created when someone open() the device, and + * closed when someone close() it or the user space application terminates. + */ +typedef struct ump_session_data { + _mali_osk_list_t list_head_session_memory_list; /**< List of ump allocations made by the process (elements are ump_session_memory_list_element) */ + _mali_osk_list_t list_head_session_memory_mappings_list; /**< List of ump_memory_allocations mapped in */ + int api_version; + _mali_osk_mutex_t *lock; + ump_descriptor_mapping *cookies_map; /**< Secure mapping of cookies from _ump_ukk_map_mem() */ + int cache_operations_ongoing; + int has_pending_level1_cache_flush; +} ump_session_data; + + + +/* + * This struct is used to track the UMP memory references a session has. + * We need to track this in order to be able to clean up after user space processes + * which don't do it themself (e.g. due to a crash or premature termination). + */ +typedef struct ump_session_memory_list_element { + struct ump_dd_mem *mem; + _mali_osk_list_t list; +} ump_session_memory_list_element; + + + +/* + * Device specific data, created when device driver is loaded, and then kept as the global variable device. + */ +typedef struct ump_dev { + ump_random_mapping *secure_id_map; + ump_memory_backend *backend; +} ump_dev; + + + +extern int ump_debug_level; +extern struct ump_dev device; + +_mali_osk_errcode_t ump_kernel_constructor(void); +void ump_kernel_destructor(void); +int ump_map_errcode(_mali_osk_errcode_t err); + +/** + * variables from user space cannot be dereferenced from kernel space; tagging them + * with __user allows the GCC compiler to generate a warning. Other compilers may + * not support this so we define it here as an empty macro if the compiler doesn't + * define it. + */ +#ifndef __user +#define __user +#endif + +#endif /* __UMP_KERNEL_COMMON_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c new file mode 100755 index 000000000000..e4642f0394c2 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "mali_osk_bitops.h" +#include "ump_kernel_common.h" +#include "ump_kernel_descriptor_mapping.h" + +#define MALI_PAD_INT(x) (((x) + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1)) + +/** + * Allocate a descriptor table capable of holding 'count' mappings + * @param count Number of mappings in the table + * @return Pointer to a new table, NULL on error + */ +static ump_descriptor_table *descriptor_table_alloc(int count); + +/** + * Free a descriptor table + * @param table The table to free + */ +static void descriptor_table_free(ump_descriptor_table *table); + +ump_descriptor_mapping *ump_descriptor_mapping_create(int init_entries, int max_entries) +{ + ump_descriptor_mapping *map = _mali_osk_calloc(1, sizeof(ump_descriptor_mapping)); + + init_entries = MALI_PAD_INT(init_entries); + max_entries = MALI_PAD_INT(max_entries); + + if (NULL != map) { + map->table = descriptor_table_alloc(init_entries); + if (NULL != map->table) { + map->lock = _mali_osk_mutex_rw_init(_MALI_OSK_LOCKFLAG_UNORDERED, 0); + if (NULL != map->lock) { + _mali_osk_set_nonatomic_bit(0, map->table->usage); /* reserve bit 0 to prevent NULL/zero logic to kick in */ + map->max_nr_mappings_allowed = max_entries; + map->current_nr_mappings = init_entries; + return map; + } + descriptor_table_free(map->table); + } + _mali_osk_free(map); + } + return NULL; +} + +void ump_descriptor_mapping_destroy(ump_descriptor_mapping *map) +{ + descriptor_table_free(map->table); + _mali_osk_mutex_rw_term(map->lock); + _mali_osk_free(map); +} + +int ump_descriptor_mapping_allocate_mapping(ump_descriptor_mapping *map, void *target) +{ + int descriptor = -1;/*-EFAULT;*/ + _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); + descriptor = _mali_osk_find_first_zero_bit(map->table->usage, map->current_nr_mappings); + if (descriptor == map->current_nr_mappings) { + int nr_mappings_new; + /* no free descriptor, try to expand the table */ + ump_descriptor_table *new_table; + ump_descriptor_table *old_table = map->table; + nr_mappings_new = map->current_nr_mappings * 2; + + if (map->current_nr_mappings >= map->max_nr_mappings_allowed) { + descriptor = -1; + goto unlock_and_exit; + } + + new_table = descriptor_table_alloc(nr_mappings_new); + if (NULL == new_table) { + descriptor = -1; + goto unlock_and_exit; + } + + _mali_osk_memcpy(new_table->usage, old_table->usage, (sizeof(unsigned long)*map->current_nr_mappings) / BITS_PER_LONG); + _mali_osk_memcpy(new_table->mappings, old_table->mappings, map->current_nr_mappings * sizeof(void *)); + map->table = new_table; + map->current_nr_mappings = nr_mappings_new; + descriptor_table_free(old_table); + } + + /* we have found a valid descriptor, set the value and usage bit */ + _mali_osk_set_nonatomic_bit(descriptor, map->table->usage); + map->table->mappings[descriptor] = target; + +unlock_and_exit: + _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); + return descriptor; +} + +int ump_descriptor_mapping_get(ump_descriptor_mapping *map, int descriptor, void **target) +{ + int result = -1;/*-EFAULT;*/ + DEBUG_ASSERT(map); + _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); + if ((descriptor > 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { + *target = map->table->mappings[descriptor]; + result = 0; + } else *target = NULL; + _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); + return result; +} + +int ump_descriptor_mapping_set(ump_descriptor_mapping *map, int descriptor, void *target) +{ + int result = -1;/*-EFAULT;*/ + _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); + if ((descriptor > 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { + map->table->mappings[descriptor] = target; + result = 0; + } + _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); + return result; +} + +void ump_descriptor_mapping_free(ump_descriptor_mapping *map, int descriptor) +{ + _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); + if ((descriptor > 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { + map->table->mappings[descriptor] = NULL; + _mali_osk_clear_nonatomic_bit(descriptor, map->table->usage); + } + _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); +} + +static ump_descriptor_table *descriptor_table_alloc(int count) +{ + ump_descriptor_table *table; + + table = _mali_osk_calloc(1, sizeof(ump_descriptor_table) + ((sizeof(unsigned long) * count) / BITS_PER_LONG) + (sizeof(void *) * count)); + + if (NULL != table) { + table->usage = (u32 *)((u8 *)table + sizeof(ump_descriptor_table)); + table->mappings = (void **)((u8 *)table + sizeof(ump_descriptor_table) + ((sizeof(unsigned long) * count) / BITS_PER_LONG)); + } + + return table; +} + +static void descriptor_table_free(ump_descriptor_table *table) +{ + _mali_osk_free(table); +} + diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h new file mode 100755 index 000000000000..a888ba833fbb --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_kernel_descriptor_mapping.h + */ + +#ifndef __UMP_KERNEL_DESCRIPTOR_MAPPING_H__ +#define __UMP_KERNEL_DESCRIPTOR_MAPPING_H__ + +#include "mali_osk.h" + +/** + * The actual descriptor mapping table, never directly accessed by clients + */ +typedef struct ump_descriptor_table { + u32 *usage; /**< Pointer to bitpattern indicating if a descriptor is valid/used or not */ + void **mappings; /**< Array of the pointers the descriptors map to */ +} ump_descriptor_table; + +/** + * The descriptor mapping object + * Provides a separate namespace where we can map an integer to a pointer + */ +typedef struct ump_descriptor_mapping { + _mali_osk_mutex_rw_t *lock; /**< Lock protecting access to the mapping object */ + int max_nr_mappings_allowed; /**< Max number of mappings to support in this namespace */ + int current_nr_mappings; /**< Current number of possible mappings */ + ump_descriptor_table *table; /**< Pointer to the current mapping table */ +} ump_descriptor_mapping; + +/** + * Create a descriptor mapping object + * Create a descriptor mapping capable of holding init_entries growable to max_entries + * @param init_entries Number of entries to preallocate memory for + * @param max_entries Number of entries to max support + * @return Pointer to a descriptor mapping object, NULL on failure + */ +ump_descriptor_mapping *ump_descriptor_mapping_create(int init_entries, int max_entries); + +/** + * Destroy a descriptor mapping object + * @param map The map to free + */ +void ump_descriptor_mapping_destroy(ump_descriptor_mapping *map); + +/** + * Allocate a new mapping entry (descriptor ID) + * Allocates a new entry in the map. + * @param map The map to allocate a new entry in + * @param target The value to map to + * @return The descriptor allocated, a negative value on error + */ +int ump_descriptor_mapping_allocate_mapping(ump_descriptor_mapping *map, void *target); + +/** + * Get the value mapped to by a descriptor ID + * @param map The map to lookup the descriptor id in + * @param descriptor The descriptor ID to lookup + * @param target Pointer to a pointer which will receive the stored value + * @return 0 on successful lookup, negative on error + */ +int ump_descriptor_mapping_get(ump_descriptor_mapping *map, int descriptor, void **target); + +/** + * Set the value mapped to by a descriptor ID + * @param map The map to lookup the descriptor id in + * @param descriptor The descriptor ID to lookup + * @param target Pointer to replace the current value with + * @return 0 on successful lookup, negative on error + */ +int ump_descriptor_mapping_set(ump_descriptor_mapping *map, int descriptor, void *target); + +/** + * Free the descriptor ID + * For the descriptor to be reused it has to be freed + * @param map The map to free the descriptor from + * @param descriptor The descriptor ID to free + */ +void ump_descriptor_mapping_free(ump_descriptor_mapping *map, int descriptor); + +#endif /* __UMP_KERNEL_DESCRIPTOR_MAPPING_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h new file mode 100755 index 000000000000..2b69f68e87ac --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_kernel_memory_mapping.h + */ + +#ifndef __UMP_KERNEL_MEMORY_BACKEND_H__ +#define __UMP_KERNEL_MEMORY_BACKEND_H__ + +#include "ump_kernel_interface.h" +#include "ump_kernel_types.h" + + +typedef struct ump_memory_allocation { + void *phys_addr; + void *mapping; + unsigned long size; + ump_dd_handle handle; + void *process_mapping_info; + u32 cookie; /**< necessary on some U/K interface implementations */ + struct ump_session_data *ump_session; /**< Session that this allocation belongs to */ + _mali_osk_list_t list; /**< List for linking together memory allocations into the session's memory head */ + u32 is_cached; +} ump_memory_allocation; + +typedef struct ump_memory_backend { + int (*allocate)(void *ctx, ump_dd_mem *descriptor); + void (*release)(void *ctx, ump_dd_mem *descriptor); + void (*shutdown)(struct ump_memory_backend *backend); + u32(*stat)(struct ump_memory_backend *backend); + int (*pre_allocate_physical_check)(void *ctx, u32 size); + u32(*adjust_to_mali_phys)(void *ctx, u32 cpu_phys); + void *ctx; +} ump_memory_backend; + +ump_memory_backend *ump_memory_backend_create(void); +void ump_memory_backend_destroy(void); + +#endif /*__UMP_KERNEL_MEMORY_BACKEND_H__ */ + diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c new file mode 100755 index 000000000000..0b6434bee00f --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_osk.h" +#include "mali_osk_list.h" +#include "ump_osk.h" +#include "ump_uk_types.h" + +#include "ump_kernel_interface_ref_drv.h" +#include "ump_kernel_common.h" +#include "ump_kernel_descriptor_mapping.h" + +#define UMP_MINIMUM_SIZE 4096 +#define UMP_MINIMUM_SIZE_MASK (~(UMP_MINIMUM_SIZE-1)) +#define UMP_SIZE_ALIGN(x) (((x)+UMP_MINIMUM_SIZE-1)&UMP_MINIMUM_SIZE_MASK) +#define UMP_ADDR_ALIGN_OFFSET(x) ((x)&(UMP_MINIMUM_SIZE-1)) +static void phys_blocks_release(void *ctx, struct ump_dd_mem *descriptor); + +UMP_KERNEL_API_EXPORT ump_dd_handle ump_dd_handle_create_from_phys_blocks(ump_dd_physical_block *blocks, unsigned long num_blocks) +{ + ump_dd_mem *mem; + unsigned long size_total = 0; + int ret; + u32 i; + + /* Go through the input blocks and verify that they are sane */ + for (i = 0; i < num_blocks; i++) { + unsigned long addr = blocks[i].addr; + unsigned long size = blocks[i].size; + + DBG_MSG(5, ("Adding physical memory to new handle. Address: 0x%08lx, size: %lu\n", addr, size)); + size_total += blocks[i].size; + + if (0 != UMP_ADDR_ALIGN_OFFSET(addr)) { + MSG_ERR(("Trying to create UMP memory from unaligned physical address. Address: 0x%08lx\n", addr)); + return UMP_DD_HANDLE_INVALID; + } + + if (0 != UMP_ADDR_ALIGN_OFFSET(size)) { + MSG_ERR(("Trying to create UMP memory with unaligned size. Size: %lu\n", size)); + return UMP_DD_HANDLE_INVALID; + } + } + + /* Allocate the ump_dd_mem struct for this allocation */ + mem = _mali_osk_malloc(sizeof(*mem)); + if (NULL == mem) { + DBG_MSG(1, ("Could not allocate ump_dd_mem in ump_dd_handle_create_from_phys_blocks()\n")); + return UMP_DD_HANDLE_INVALID; + } + + /* Now, make a copy of the block information supplied by the user */ + mem->block_array = _mali_osk_malloc(sizeof(ump_dd_physical_block) * num_blocks); + if (NULL == mem->block_array) { + _mali_osk_free(mem); + DBG_MSG(1, ("Could not allocate a mem handle for function ump_dd_handle_create_from_phys_blocks().\n")); + return UMP_DD_HANDLE_INVALID; + } + + _mali_osk_memcpy(mem->block_array, blocks, sizeof(ump_dd_physical_block) * num_blocks); + + /* And setup the rest of the ump_dd_mem struct */ + _mali_osk_atomic_init(&mem->ref_count, 1); + mem->size_bytes = size_total; + mem->nr_blocks = num_blocks; + mem->backend_info = NULL; + mem->ctx = NULL; + mem->release_func = phys_blocks_release; + /* For now UMP handles created by ump_dd_handle_create_from_phys_blocks() is forced to be Uncached */ + mem->is_cached = 0; + mem->hw_device = _UMP_UK_USED_BY_CPU; + mem->lock_usage = UMP_NOT_LOCKED; + + /* Find a secure ID for this allocation */ + ret = ump_random_mapping_insert(device.secure_id_map, mem); + if (unlikely(ret)) { + _mali_osk_free(mem->block_array); + _mali_osk_free(mem); + DBG_MSG(1, ("Failed to allocate secure ID in ump_dd_handle_create_from_phys_blocks()\n")); + return UMP_DD_HANDLE_INVALID; + } + + DBG_MSG(3, ("UMP memory created. ID: %u, size: %lu\n", mem->secure_id, mem->size_bytes)); + + return (ump_dd_handle)mem; +} + +static void phys_blocks_release(void *ctx, struct ump_dd_mem *descriptor) +{ + _mali_osk_free(descriptor->block_array); + descriptor->block_array = NULL; +} + +_mali_osk_errcode_t _ump_ukk_allocate(_ump_uk_allocate_s *user_interaction) +{ + ump_session_data *session_data = NULL; + ump_dd_mem *new_allocation = NULL; + ump_session_memory_list_element *session_memory_element = NULL; + int ret; + + DEBUG_ASSERT_POINTER(user_interaction); + DEBUG_ASSERT_POINTER(user_interaction->ctx); + + session_data = (ump_session_data *) user_interaction->ctx; + + session_memory_element = _mali_osk_calloc(1, sizeof(ump_session_memory_list_element)); + if (NULL == session_memory_element) { + DBG_MSG(1, ("Failed to allocate ump_session_memory_list_element in ump_ioctl_allocate()\n")); + return _MALI_OSK_ERR_NOMEM; + } + + + new_allocation = _mali_osk_calloc(1, sizeof(ump_dd_mem)); + if (NULL == new_allocation) { + _mali_osk_free(session_memory_element); + DBG_MSG(1, ("Failed to allocate ump_dd_mem in _ump_ukk_allocate()\n")); + return _MALI_OSK_ERR_NOMEM; + } + + /* Initialize the part of the new_allocation that we know so for */ + _mali_osk_atomic_init(&new_allocation->ref_count, 1); + if (0 == (UMP_REF_DRV_UK_CONSTRAINT_USE_CACHE & user_interaction->constraints)) + new_allocation->is_cached = 0; + else new_allocation->is_cached = 1; + + /* Special case a size of 0, we should try to emulate what malloc does + * in this case, which is to return a valid pointer that must be freed, + * but can't be dereferenced */ + if (0 == user_interaction->size) { + /* Emulate by actually allocating the minimum block size */ + user_interaction->size = 1; + } + + /* Page align the size */ + new_allocation->size_bytes = UMP_SIZE_ALIGN(user_interaction->size); + new_allocation->lock_usage = UMP_NOT_LOCKED; + + /* Now, ask the active memory backend to do the actual memory allocation */ + if (!device.backend->allocate(device.backend->ctx, new_allocation)) { + DBG_MSG(3, ("OOM: No more UMP memory left. Failed to allocate memory in ump_ioctl_allocate(). Size: %lu, requested size: %lu\n", + new_allocation->size_bytes, + (unsigned long)user_interaction->size)); + _mali_osk_free(new_allocation); + _mali_osk_free(session_memory_element); + return _MALI_OSK_ERR_INVALID_FUNC; + } + new_allocation->hw_device = _UMP_UK_USED_BY_CPU; + new_allocation->ctx = device.backend->ctx; + new_allocation->release_func = device.backend->release; + + /* Initialize the session_memory_element, and add it to the session object */ + session_memory_element->mem = new_allocation; + _mali_osk_mutex_wait(session_data->lock); + _mali_osk_list_add(&(session_memory_element->list), &(session_data->list_head_session_memory_list)); + _mali_osk_mutex_signal(session_data->lock); + + /* Create a secure ID for this allocation */ + ret = ump_random_mapping_insert(device.secure_id_map, new_allocation); + if (unlikely(ret)) { + new_allocation->release_func(new_allocation->ctx, new_allocation); + _mali_osk_free(session_memory_element); + _mali_osk_free(new_allocation); + DBG_MSG(1, ("Failed to allocate secure ID in ump_ioctl_allocate()\n")); + return _MALI_OSK_ERR_INVALID_FUNC; + } + + user_interaction->secure_id = new_allocation->secure_id; + user_interaction->size = new_allocation->size_bytes; + DBG_MSG(3, ("UMP memory allocated. ID: %u, size: %lu\n", + new_allocation->secure_id, + new_allocation->size_bytes)); + + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h new file mode 100755 index 000000000000..32f32ccbe9fc --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __UMP_KERNEL_TYPES_H__ +#define __UMP_KERNEL_TYPES_H__ + +#include "ump_kernel_interface.h" +#include "mali_osk.h" + +#include +#ifdef CONFIG_DMA_SHARED_BUFFER +#include +#endif + +typedef enum { + UMP_USED_BY_CPU = 0, + UMP_USED_BY_MALI = 1, + UMP_USED_BY_UNKNOWN_DEVICE = 100, +} ump_hw_usage; + +typedef enum { + UMP_NOT_LOCKED = 0, + UMP_READ = 1, + UMP_READ_WRITE = 3, +} ump_lock_usage; + +/* + * This struct is what is "behind" a ump_dd_handle + */ +typedef struct ump_dd_mem { + struct rb_node node; + ump_secure_id secure_id; + _mali_osk_atomic_t ref_count; + unsigned long size_bytes; + unsigned long nr_blocks; + ump_dd_physical_block *block_array; + void (*release_func)(void *ctx, struct ump_dd_mem *descriptor); + void *ctx; + void *backend_info; + int is_cached; + ump_hw_usage hw_device; + ump_lock_usage lock_usage; +#ifdef CONFIG_DMA_SHARED_BUFFER + struct dma_buf_attachment *import_attach; + struct sg_table *sgt; +#endif +} ump_dd_mem; + + + +#endif /* __UMP_KERNEL_TYPES_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/common/ump_osk.h b/drivers/gpu/arm/mali400/ump/common/ump_osk.h new file mode 100755 index 000000000000..9adc4d3df3f5 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_osk.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_osk.h + * Defines the OS abstraction layer for the UMP kernel device driver (OSK) + */ + +#ifndef __UMP_OSK_H__ +#define __UMP_OSK_H__ + +#include +#include +#include "ump_uk_types.h" +#include "ump_kernel_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +_mali_osk_errcode_t _ump_osk_init(void); + +_mali_osk_errcode_t _ump_osk_term(void); + +int _ump_osk_atomic_inc_and_read(_mali_osk_atomic_t *atom); + +int _ump_osk_atomic_dec_and_read(_mali_osk_atomic_t *atom); + +_mali_osk_errcode_t _ump_osk_mem_mapregion_init(ump_memory_allocation *descriptor); + +_mali_osk_errcode_t _ump_osk_mem_mapregion_map(ump_memory_allocation *descriptor, u32 offset, u32 *phys_addr, unsigned long size); + +void _ump_osk_mem_mapregion_term(ump_memory_allocation *descriptor); + +void _ump_osk_msync(ump_dd_mem *mem, void *virt, u32 offset, u32 size, ump_uk_msync_op op, ump_session_data *session_data); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h b/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h new file mode 100755 index 000000000000..db842cdcbeff --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_uk_types.h + * Defines the types and constants used in the user-kernel interface + */ + +#ifndef __UMP_UK_TYPES_H__ +#define __UMP_UK_TYPES_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Helpers for API version handling */ +#define MAKE_VERSION_ID(x) (((x) << 16UL) | (x)) +#define IS_VERSION_ID(x) (((x) & 0xFFFF) == (((x) >> 16UL) & 0xFFFF)) +#define GET_VERSION(x) (((x) >> 16UL) & 0xFFFF) +#define IS_API_MATCH(x, y) (IS_VERSION_ID((x)) && IS_VERSION_ID((y)) && (GET_VERSION((x)) == GET_VERSION((y)))) + +/** + * API version define. + * Indicates the version of the kernel API + * The version is a 16bit integer incremented on each API change. + * The 16bit integer is stored twice in a 32bit integer + * So for version 1 the value would be 0x00010001 + */ +#define UMP_IOCTL_API_VERSION MAKE_VERSION_ID(3) + +typedef enum +{ + _UMP_IOC_QUERY_API_VERSION = 1, + _UMP_IOC_ALLOCATE, + _UMP_IOC_RELEASE, + _UMP_IOC_SIZE_GET, + _UMP_IOC_MAP_MEM, /* not used in Linux */ + _UMP_IOC_UNMAP_MEM, /* not used in Linux */ + _UMP_IOC_MSYNC, + _UMP_IOC_CACHE_OPERATIONS_CONTROL, + _UMP_IOC_SWITCH_HW_USAGE, + _UMP_IOC_LOCK, + _UMP_IOC_UNLOCK, + _UMP_IOC_DMABUF_IMPORT, +} _ump_uk_functions; + +typedef enum +{ + UMP_REF_DRV_UK_CONSTRAINT_NONE = 0, + UMP_REF_DRV_UK_CONSTRAINT_PHYSICALLY_LINEAR = 1, + UMP_REF_DRV_UK_CONSTRAINT_USE_CACHE = 4, +} ump_uk_alloc_constraints; + +typedef enum +{ + _UMP_UK_MSYNC_CLEAN = 0, + _UMP_UK_MSYNC_CLEAN_AND_INVALIDATE = 1, + _UMP_UK_MSYNC_INVALIDATE = 2, + _UMP_UK_MSYNC_FLUSH_L1 = 3, + _UMP_UK_MSYNC_READOUT_CACHE_ENABLED = 128, +} ump_uk_msync_op; + +typedef enum +{ + _UMP_UK_CACHE_OP_START = 0, + _UMP_UK_CACHE_OP_FINISH = 1, +} ump_uk_cache_op_control; + +typedef enum +{ + _UMP_UK_READ = 1, + _UMP_UK_READ_WRITE = 3, +} ump_uk_lock_usage; + +typedef enum +{ + _UMP_UK_USED_BY_CPU = 0, + _UMP_UK_USED_BY_MALI = 1, + _UMP_UK_USED_BY_UNKNOWN_DEVICE = 100, +} ump_uk_user; + +/** + * Get API version ([in,out] u32 api_version, [out] u32 compatible) + */ +typedef struct _ump_uk_api_version_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 version; /**< Set to the user space version on entry, stores the device driver version on exit */ + u32 compatible; /**< Non-null if the device is compatible with the client */ +} _ump_uk_api_version_s; + +/** + * ALLOCATE ([out] u32 secure_id, [in,out] u32 size, [in] contraints) + */ +typedef struct _ump_uk_allocate_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 secure_id; /**< Return value from DD to Userdriver */ + u32 size; /**< Input and output. Requested size; input. Returned size; output */ + ump_uk_alloc_constraints constraints; /**< Only input to Devicedriver */ +} _ump_uk_allocate_s; + +/** + * SIZE_GET ([in] u32 secure_id, [out]size ) + */ +typedef struct _ump_uk_size_get_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 secure_id; /**< Input to DD */ + u32 size; /**< Returned size; output */ +} _ump_uk_size_get_s; + +/** + * Release ([in] u32 secure_id) + */ +typedef struct _ump_uk_release_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 secure_id; /**< Input to DD */ +} _ump_uk_release_s; + +typedef struct _ump_uk_map_mem_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + void *mapping; /**< [out] Returns user-space virtual address for the mapping */ + void *phys_addr; /**< [in] physical address */ + unsigned long size; /**< [in] size */ + u32 secure_id; /**< [in] secure_id to assign to mapping */ + void *_ukk_private; /**< Only used inside linux port between kernel frontend and common part to store vma */ + u32 cookie; + u32 is_cached; /**< [in,out] caching of CPU mappings */ +} _ump_uk_map_mem_s; + +typedef struct _ump_uk_unmap_mem_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + void *mapping; + u32 size; + void *_ukk_private; + u32 cookie; +} _ump_uk_unmap_mem_s; + +typedef struct _ump_uk_msync_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + void *mapping; /**< [in] mapping addr */ + void *address; /**< [in] flush start addr */ + u32 size; /**< [in] size to flush */ + ump_uk_msync_op op; /**< [in] flush operation */ + u32 cookie; /**< [in] cookie stored with reference to the kernel mapping internals */ + u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ + u32 is_cached; /**< [out] caching of CPU mappings */ +} _ump_uk_msync_s; + +typedef struct _ump_uk_cache_operations_control_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + ump_uk_cache_op_control op; /**< [in] cache operations start/stop */ +} _ump_uk_cache_operations_control_s; + + +typedef struct _ump_uk_switch_hw_usage_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ + ump_uk_user new_user; /**< [in] cookie stored with reference to the kernel mapping internals */ + +} _ump_uk_switch_hw_usage_s; + +typedef struct _ump_uk_lock_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ + ump_uk_lock_usage lock_usage; +} _ump_uk_lock_s; + +typedef struct _ump_uk_unlock_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ +} _ump_uk_unlock_s; + +typedef struct _ump_uk_dmabuf_s +{ + void *ctx; /**< [in,out] user-kernel context (trashed on output) */ + int fd; /**< [in] dmabuf_fd that identifies the dmabuf buffer */ + size_t size; /**< [in] size of the buffer */ + u32 secure_id; /**< [out] secure_id that identifies the ump buffer */ +} _ump_uk_dmabuf_s; + +#ifdef __cplusplus +} +#endif + +#endif /* __UMP_UK_TYPES_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/common/ump_ukk.h b/drivers/gpu/arm/mali400/ump/common/ump_ukk.h new file mode 100755 index 000000000000..f2906768c37f --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/common/ump_ukk.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_ukk.h + * Defines the kernel-side interface of the user-kernel interface + */ + +#ifndef __UMP_UKK_H__ +#define __UMP_UKK_H__ + +#include "mali_osk.h" +#include "ump_uk_types.h" + + +#ifdef __cplusplus +extern "C" { +#endif + + +_mali_osk_errcode_t _ump_ukk_open(void **context); + +_mali_osk_errcode_t _ump_ukk_close(void **context); + +_mali_osk_errcode_t _ump_ukk_allocate(_ump_uk_allocate_s *user_interaction); + +_mali_osk_errcode_t _ump_ukk_release(_ump_uk_release_s *release_info); + +_mali_osk_errcode_t _ump_ukk_size_get(_ump_uk_size_get_s *user_interaction); + +_mali_osk_errcode_t _ump_ukk_map_mem(_ump_uk_map_mem_s *args); + +_mali_osk_errcode_t _ump_uku_get_api_version(_ump_uk_api_version_s *args); + +void _ump_ukk_unmap_mem(_ump_uk_unmap_mem_s *args); + +void _ump_ukk_msync(_ump_uk_msync_s *args); + +void _ump_ukk_cache_operations_control(_ump_uk_cache_operations_control_s *args); + +void _ump_ukk_switch_hw_usage(_ump_uk_switch_hw_usage_s *args); + +void _ump_ukk_lock(_ump_uk_lock_s *args); + +void _ump_ukk_unlock(_ump_uk_unlock_s *args); + +u32 _ump_ukk_report_memory_usage(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __UMP_UKK_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h b/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h new file mode 100755 index 000000000000..d0174055aa28 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2010, 2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_kernel_license.h + * Defines for the macro MODULE_LICENSE. + */ + +#ifndef __UMP_KERNEL_LICENSE_H__ +#define __UMP_KERNEL_LICENSE_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define UMP_KERNEL_LINUX_LICENSE "GPL" +#define UMP_LICENSE_IS_GPL 1 + +#ifdef __cplusplus +} +#endif + +#endif /* __UMP_KERNEL_LICENSE_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h b/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h new file mode 100755 index 000000000000..bfb4e8d64885 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2010-2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __UMP_IOCTL_H__ +#define __UMP_IOCTL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include + +#ifndef __user +#define __user +#endif + + +/** + * @file UMP_ioctl.h + * This file describes the interface needed to use the Linux device driver. + * The interface is used by the userpace UMP driver. + */ + +#define UMP_IOCTL_NR 0x90 + + +#define UMP_IOC_QUERY_API_VERSION _IOR(UMP_IOCTL_NR, _UMP_IOC_QUERY_API_VERSION, _ump_uk_api_version_s) +#define UMP_IOC_ALLOCATE _IOWR(UMP_IOCTL_NR, _UMP_IOC_ALLOCATE, _ump_uk_allocate_s) +#define UMP_IOC_RELEASE _IOR(UMP_IOCTL_NR, _UMP_IOC_RELEASE, _ump_uk_release_s) +#define UMP_IOC_SIZE_GET _IOWR(UMP_IOCTL_NR, _UMP_IOC_SIZE_GET, _ump_uk_size_get_s) +#define UMP_IOC_MSYNC _IOW(UMP_IOCTL_NR, _UMP_IOC_MSYNC, _ump_uk_msync_s) + +#define UMP_IOC_CACHE_OPERATIONS_CONTROL _IOW(UMP_IOCTL_NR, _UMP_IOC_CACHE_OPERATIONS_CONTROL, _ump_uk_cache_operations_control_s) +#define UMP_IOC_SWITCH_HW_USAGE _IOW(UMP_IOCTL_NR, _UMP_IOC_SWITCH_HW_USAGE, _ump_uk_switch_hw_usage_s) +#define UMP_IOC_LOCK _IOW(UMP_IOCTL_NR, _UMP_IOC_LOCK, _ump_uk_lock_s) +#define UMP_IOC_UNLOCK _IOW(UMP_IOCTL_NR, _UMP_IOC_UNLOCK, _ump_uk_unlock_s) + +#define UMP_IOC_DMABUF_IMPORT _IOW(UMP_IOCTL_NR, _UMP_IOC_DMABUF_IMPORT, _ump_uk_dmabuf_s) + +#ifdef __cplusplus +} +#endif + +#endif /* __UMP_IOCTL_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c new file mode 100755 index 000000000000..71b30830c308 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c @@ -0,0 +1,449 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include /* kernel module definitions */ +#include /* file system operations */ +#include /* character device definitions */ +#include /* request_mem_region */ +#include /* memory management functions and types */ +#include /* user space access */ +#include +#include +#include + +#include "arch/config.h" /* Configuration for current platform. The symlinc for arch is set by Makefile */ +#include "ump_ioctl.h" +#include "ump_kernel_common.h" +#include "ump_kernel_interface.h" +#include "ump_kernel_interface_ref_drv.h" +#include "ump_kernel_descriptor_mapping.h" +#include "ump_kernel_memory_backend.h" +#include "ump_kernel_memory_backend_os.h" +#include "ump_kernel_memory_backend_dedicated.h" +#include "ump_kernel_license.h" + +#include "ump_osk.h" +#include "ump_ukk.h" +#include "ump_uk_types.h" +#include "ump_ukk_wrappers.h" +#include "ump_ukk_ref_wrappers.h" + + +/* Module parameter to control log level */ +int ump_debug_level = 2; +module_param(ump_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */ +MODULE_PARM_DESC(ump_debug_level, "Higher number, more dmesg output"); + +/* By default the module uses any available major, but it's possible to set it at load time to a specific number */ +int ump_major = 0; +module_param(ump_major, int, S_IRUGO); /* r--r--r-- */ +MODULE_PARM_DESC(ump_major, "Device major number"); + +/* Name of the UMP device driver */ +static char ump_dev_name[] = "ump"; /* should be const, but the functions we call requires non-cost */ + + +#if UMP_LICENSE_IS_GPL +static struct dentry *ump_debugfs_dir = NULL; +#endif + +/* + * The data which we attached to each virtual memory mapping request we get. + * Each memory mapping has a reference to the UMP memory it maps. + * We release this reference when the last memory mapping is unmapped. + */ +typedef struct ump_vma_usage_tracker { + int references; + ump_dd_handle handle; +} ump_vma_usage_tracker; + +struct ump_device { + struct cdev cdev; +#if UMP_LICENSE_IS_GPL + struct class *ump_class; +#endif +}; + +/* The global variable containing the global device data */ +static struct ump_device ump_device; +struct device *ump_global_mdev = NULL; + +/* Forward declare static functions */ +static int ump_file_open(struct inode *inode, struct file *filp); +static int ump_file_release(struct inode *inode, struct file *filp); +#ifdef HAVE_UNLOCKED_IOCTL +static long ump_file_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +#else +static int ump_file_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg); +#endif +static int ump_file_mmap(struct file *filp, struct vm_area_struct *vma); + + +/* This variable defines the file operations this UMP device driver offer */ +static struct file_operations ump_fops = { + .owner = THIS_MODULE, + .open = ump_file_open, + .release = ump_file_release, +#ifdef HAVE_UNLOCKED_IOCTL + .unlocked_ioctl = ump_file_ioctl, +#else + .ioctl = ump_file_ioctl, +#endif + .mmap = ump_file_mmap +}; + + +/* This function is called by Linux to initialize this module. + * All we do is initialize the UMP device driver. + */ +static int ump_initialize_module(void) +{ + _mali_osk_errcode_t err; + + DBG_MSG(2, ("Inserting UMP device driver. Compiled: %s, time: %s\n", __DATE__, __TIME__)); + + err = ump_kernel_constructor(); + if (_MALI_OSK_ERR_OK != err) { + MSG_ERR(("UMP device driver init failed\n")); + return ump_map_errcode(err); + } + + MSG(("UMP device driver %s loaded\n", SVN_REV_STRING)); + return 0; +} + + + +/* + * This function is called by Linux to unload/terminate/exit/cleanup this module. + * All we do is terminate the UMP device driver. + */ +static void ump_cleanup_module(void) +{ + DBG_MSG(2, ("Unloading UMP device driver\n")); + ump_kernel_destructor(); + DBG_MSG(2, ("Module unloaded\n")); +} + + + +static ssize_t ump_memory_used_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + char buf[64]; + size_t r; + u32 mem = _ump_ukk_report_memory_usage(); + + r = snprintf(buf, 64, "%u\n", mem); + return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + +static const struct file_operations ump_memory_usage_fops = { + .owner = THIS_MODULE, + .read = ump_memory_used_read, +}; + +/* + * Initialize the UMP device driver. + */ +int ump_kernel_device_initialize(void) +{ + int err; + dev_t dev = 0; +#if UMP_LICENSE_IS_GPL + ump_debugfs_dir = debugfs_create_dir(ump_dev_name, NULL); + if (ERR_PTR(-ENODEV) == ump_debugfs_dir) { + ump_debugfs_dir = NULL; + } else { + debugfs_create_file("memory_usage", 0400, ump_debugfs_dir, NULL, &ump_memory_usage_fops); + } +#endif + + if (0 == ump_major) { + /* auto select a major */ + err = alloc_chrdev_region(&dev, 0, 1, ump_dev_name); + ump_major = MAJOR(dev); + } else { + /* use load time defined major number */ + dev = MKDEV(ump_major, 0); + err = register_chrdev_region(dev, 1, ump_dev_name); + } + + if (0 == err) { + memset(&ump_device, 0, sizeof(ump_device)); + + /* initialize our char dev data */ + cdev_init(&ump_device.cdev, &ump_fops); + ump_device.cdev.owner = THIS_MODULE; + ump_device.cdev.ops = &ump_fops; + + /* register char dev with the kernel */ + err = cdev_add(&ump_device.cdev, dev, 1/*count*/); + if (0 == err) { + +#if UMP_LICENSE_IS_GPL + ump_device.ump_class = class_create(THIS_MODULE, ump_dev_name); + if (IS_ERR(ump_device.ump_class)) { + err = PTR_ERR(ump_device.ump_class); + } else { + ump_global_mdev = device_create(ump_device.ump_class, NULL, dev, NULL, ump_dev_name); + if (!IS_ERR(ump_global_mdev)) { + return 0; + } + + err = PTR_ERR(ump_global_mdev); + } + cdev_del(&ump_device.cdev); +#else + return 0; +#endif + } + + unregister_chrdev_region(dev, 1); + } + + return err; +} + + + +/* + * Terminate the UMP device driver + */ +void ump_kernel_device_terminate(void) +{ + dev_t dev = MKDEV(ump_major, 0); + +#if UMP_LICENSE_IS_GPL + device_destroy(ump_device.ump_class, dev); + class_destroy(ump_device.ump_class); +#endif + + /* unregister char device */ + cdev_del(&ump_device.cdev); + + /* free major */ + unregister_chrdev_region(dev, 1); + +#if UMP_LICENSE_IS_GPL + if (ump_debugfs_dir) + debugfs_remove_recursive(ump_debugfs_dir); +#endif +} + +/* + * Open a new session. User space has called open() on us. + */ +static int ump_file_open(struct inode *inode, struct file *filp) +{ + struct ump_session_data *session_data; + _mali_osk_errcode_t err; + + /* input validation */ + if (0 != MINOR(inode->i_rdev)) { + MSG_ERR(("Minor not zero in ump_file_open()\n")); + return -ENODEV; + } + + /* Call the OS-Independent UMP Open function */ + err = _ump_ukk_open((void **) &session_data); + if (_MALI_OSK_ERR_OK != err) { + MSG_ERR(("Ump failed to open a new session\n")); + return ump_map_errcode(err); + } + + filp->private_data = (void *)session_data; + filp->f_pos = 0; + + return 0; /* success */ +} + + + +/* + * Close a session. User space has called close() or crashed/terminated. + */ +static int ump_file_release(struct inode *inode, struct file *filp) +{ + _mali_osk_errcode_t err; + + err = _ump_ukk_close((void **) &filp->private_data); + if (_MALI_OSK_ERR_OK != err) { + return ump_map_errcode(err); + } + + return 0; /* success */ +} + + + +/* + * Handle IOCTL requests. + */ +#ifdef HAVE_UNLOCKED_IOCTL +static long ump_file_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +#else +static int ump_file_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg) +#endif +{ + int err = -ENOTTY; + void __user *argument; + struct ump_session_data *session_data; + +#ifndef HAVE_UNLOCKED_IOCTL + (void)inode; /* inode not used */ +#endif + + session_data = (struct ump_session_data *)filp->private_data; + if (NULL == session_data) { + MSG_ERR(("No session data attached to file object\n")); + return -ENOTTY; + } + + /* interpret the argument as a user pointer to something */ + argument = (void __user *)arg; + + switch (cmd) { + case UMP_IOC_QUERY_API_VERSION: + err = ump_get_api_version_wrapper((u32 __user *)argument, session_data); + break; + + case UMP_IOC_ALLOCATE : + err = ump_allocate_wrapper((u32 __user *)argument, session_data); + break; + + case UMP_IOC_RELEASE: + err = ump_release_wrapper((u32 __user *)argument, session_data); + break; + + case UMP_IOC_SIZE_GET: + err = ump_size_get_wrapper((u32 __user *)argument, session_data); + break; + + case UMP_IOC_MSYNC: + err = ump_msync_wrapper((u32 __user *)argument, session_data); + break; + + case UMP_IOC_CACHE_OPERATIONS_CONTROL: + err = ump_cache_operations_control_wrapper((u32 __user *)argument, session_data); + break; + + case UMP_IOC_SWITCH_HW_USAGE: + err = ump_switch_hw_usage_wrapper((u32 __user *)argument, session_data); + break; + + case UMP_IOC_LOCK: + err = ump_lock_wrapper((u32 __user *)argument, session_data); + break; + + case UMP_IOC_UNLOCK: + err = ump_unlock_wrapper((u32 __user *)argument, session_data); + break; + + case UMP_IOC_DMABUF_IMPORT: +#ifdef CONFIG_DMA_SHARED_BUFFER + err = ump_dmabuf_import_wrapper((u32 __user *)argument, session_data); +#else + err = -EFAULT; + DBG_MSG(1, ("User space use dmabuf API, but kernel don't support DMA BUF\n")); +#endif + break; + + default: + DBG_MSG(1, ("No handler for IOCTL. cmd: 0x%08x, arg: 0x%08lx\n", cmd, arg)); + err = -EFAULT; + break; + } + + return err; +} + +int ump_map_errcode(_mali_osk_errcode_t err) +{ + switch (err) { + case _MALI_OSK_ERR_OK : + return 0; + case _MALI_OSK_ERR_FAULT: + return -EFAULT; + case _MALI_OSK_ERR_INVALID_FUNC: + return -ENOTTY; + case _MALI_OSK_ERR_INVALID_ARGS: + return -EINVAL; + case _MALI_OSK_ERR_NOMEM: + return -ENOMEM; + case _MALI_OSK_ERR_TIMEOUT: + return -ETIMEDOUT; + case _MALI_OSK_ERR_RESTARTSYSCALL: + return -ERESTARTSYS; + case _MALI_OSK_ERR_ITEM_NOT_FOUND: + return -ENOENT; + default: + return -EFAULT; + } +} + +/* + * Handle from OS to map specified virtual memory to specified UMP memory. + */ +static int ump_file_mmap(struct file *filp, struct vm_area_struct *vma) +{ + _ump_uk_map_mem_s args; + _mali_osk_errcode_t err; + struct ump_session_data *session_data; + + /* Validate the session data */ + session_data = (struct ump_session_data *)filp->private_data; + if (NULL == session_data) { + MSG_ERR(("mmap() called without any session data available\n")); + return -EFAULT; + } + + /* Re-pack the arguments that mmap() packed for us */ + args.ctx = session_data; + args.phys_addr = 0; + args.size = vma->vm_end - vma->vm_start; + args._ukk_private = vma; + args.secure_id = vma->vm_pgoff; + + /* By setting this flag, during a process fork; the child process will not have the parent UMP mappings */ + vma->vm_flags |= VM_DONTCOPY; + + DBG_MSG(4, ("UMP vma->flags: %x\n", vma->vm_flags)); + + /* Call the common mmap handler */ + err = _ump_ukk_map_mem(&args); + if (_MALI_OSK_ERR_OK != err) { + MSG_ERR(("_ump_ukk_map_mem() failed in function ump_file_mmap()")); + return ump_map_errcode(err); + } + + return 0; /* success */ +} + +/* Export UMP kernel space API functions */ +EXPORT_SYMBOL(ump_dd_secure_id_get); +EXPORT_SYMBOL(ump_dd_handle_create_from_secure_id); +EXPORT_SYMBOL(ump_dd_phys_block_count_get); +EXPORT_SYMBOL(ump_dd_phys_block_get); +EXPORT_SYMBOL(ump_dd_phys_blocks_get); +EXPORT_SYMBOL(ump_dd_size_get); +EXPORT_SYMBOL(ump_dd_reference_add); +EXPORT_SYMBOL(ump_dd_reference_release); + +/* Export our own extended kernel space allocator */ +EXPORT_SYMBOL(ump_dd_handle_create_from_phys_blocks); + +/* Setup init and exit functions for this module */ +module_init(ump_initialize_module); +module_exit(ump_cleanup_module); + +/* And some module informatio */ +MODULE_LICENSE(UMP_KERNEL_LINUX_LICENSE); +MODULE_AUTHOR("ARM Ltd."); +MODULE_VERSION(SVN_REV_STRING); diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h new file mode 100755 index 000000000000..8d32ddbb5449 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2010-2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __UMP_KERNEL_LINUX_H__ +#define __UMP_KERNEL_LINUX_H__ + +int ump_kernel_device_initialize(void); +void ump_kernel_device_terminate(void); + + +#endif /* __UMP_KERNEL_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c new file mode 100755 index 000000000000..5a1257a25b82 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/* needed to detect kernel version specific code */ +#include + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) +#include +#else /* pre 2.6.26 the file was in the arch specific location */ +#include +#endif + +#include +#include +#include +#include +#include "ump_kernel_common.h" +#include "ump_kernel_memory_backend.h" + + + +#define UMP_BLOCK_SIZE (256UL * 1024UL) /* 256kB, remember to keep the ()s */ + + + +typedef struct block_info { + struct block_info *next; +} block_info; + + + +typedef struct block_allocator { + struct semaphore mutex; + block_info *all_blocks; + block_info *first_free; + u32 base; + u32 num_blocks; + u32 num_free; +} block_allocator; + + +static void block_allocator_shutdown(ump_memory_backend *backend); +static int block_allocator_allocate(void *ctx, ump_dd_mem *mem); +static void block_allocator_release(void *ctx, ump_dd_mem *handle); +static inline u32 get_phys(block_allocator *allocator, block_info *block); +static u32 block_allocator_stat(struct ump_memory_backend *backend); + + + +/* + * Create dedicated memory backend + */ +ump_memory_backend *ump_block_allocator_create(u32 base_address, u32 size) +{ + ump_memory_backend *backend; + block_allocator *allocator; + u32 usable_size; + u32 num_blocks; + + usable_size = (size + UMP_BLOCK_SIZE - 1) & ~(UMP_BLOCK_SIZE - 1); + num_blocks = usable_size / UMP_BLOCK_SIZE; + + if (0 == usable_size) { + DBG_MSG(1, ("Memory block of size %u is unusable\n", size)); + return NULL; + } + + DBG_MSG(5, ("Creating dedicated UMP memory backend. Base address: 0x%08x, size: 0x%08x\n", base_address, size)); + DBG_MSG(6, ("%u usable bytes which becomes %u blocks\n", usable_size, num_blocks)); + + backend = kzalloc(sizeof(ump_memory_backend), GFP_KERNEL); + if (NULL != backend) { + allocator = kmalloc(sizeof(block_allocator), GFP_KERNEL); + if (NULL != allocator) { + allocator->all_blocks = kmalloc(sizeof(block_info) * num_blocks, GFP_KERNEL); + if (NULL != allocator->all_blocks) { + int i; + + allocator->first_free = NULL; + allocator->num_blocks = num_blocks; + allocator->num_free = num_blocks; + allocator->base = base_address; + sema_init(&allocator->mutex, 1); + + for (i = 0; i < num_blocks; i++) { + allocator->all_blocks[i].next = allocator->first_free; + allocator->first_free = &allocator->all_blocks[i]; + } + + backend->ctx = allocator; + backend->allocate = block_allocator_allocate; + backend->release = block_allocator_release; + backend->shutdown = block_allocator_shutdown; + backend->stat = block_allocator_stat; + backend->pre_allocate_physical_check = NULL; + backend->adjust_to_mali_phys = NULL; + + return backend; + } + kfree(allocator); + } + kfree(backend); + } + + return NULL; +} + + + +/* + * Destroy specified dedicated memory backend + */ +static void block_allocator_shutdown(ump_memory_backend *backend) +{ + block_allocator *allocator; + + BUG_ON(!backend); + BUG_ON(!backend->ctx); + + allocator = (block_allocator *)backend->ctx; + + DBG_MSG_IF(1, allocator->num_free != allocator->num_blocks, ("%u blocks still in use during shutdown\n", allocator->num_blocks - allocator->num_free)); + + kfree(allocator->all_blocks); + kfree(allocator); + kfree(backend); +} + + + +static int block_allocator_allocate(void *ctx, ump_dd_mem *mem) +{ + block_allocator *allocator; + u32 left; + block_info *last_allocated = NULL; + int i = 0; + + BUG_ON(!ctx); + BUG_ON(!mem); + + allocator = (block_allocator *)ctx; + left = mem->size_bytes; + + BUG_ON(!left); + BUG_ON(!&allocator->mutex); + + mem->nr_blocks = ((left + UMP_BLOCK_SIZE - 1) & ~(UMP_BLOCK_SIZE - 1)) / UMP_BLOCK_SIZE; + mem->block_array = (ump_dd_physical_block *)vmalloc(sizeof(ump_dd_physical_block) * mem->nr_blocks); + if (NULL == mem->block_array) { + MSG_ERR(("Failed to allocate block array\n")); + return 0; + } + + if (down_interruptible(&allocator->mutex)) { + MSG_ERR(("Could not get mutex to do block_allocate\n")); + return 0; + } + + mem->size_bytes = 0; + + while ((left > 0) && (allocator->first_free)) { + block_info *block; + + block = allocator->first_free; + allocator->first_free = allocator->first_free->next; + block->next = last_allocated; + last_allocated = block; + allocator->num_free--; + + mem->block_array[i].addr = get_phys(allocator, block); + mem->block_array[i].size = UMP_BLOCK_SIZE; + mem->size_bytes += UMP_BLOCK_SIZE; + + i++; + + if (left < UMP_BLOCK_SIZE) left = 0; + else left -= UMP_BLOCK_SIZE; + } + + if (left) { + block_info *block; + /* release all memory back to the pool */ + while (last_allocated) { + block = last_allocated->next; + last_allocated->next = allocator->first_free; + allocator->first_free = last_allocated; + last_allocated = block; + allocator->num_free++; + } + + vfree(mem->block_array); + mem->backend_info = NULL; + mem->block_array = NULL; + + DBG_MSG(4, ("Could not find a mem-block for the allocation.\n")); + up(&allocator->mutex); + + return 0; + } + + mem->backend_info = last_allocated; + + up(&allocator->mutex); + mem->is_cached = 0; + + return 1; +} + + + +static void block_allocator_release(void *ctx, ump_dd_mem *handle) +{ + block_allocator *allocator; + block_info *block, * next; + + BUG_ON(!ctx); + BUG_ON(!handle); + + allocator = (block_allocator *)ctx; + block = (block_info *)handle->backend_info; + BUG_ON(!block); + + if (down_interruptible(&allocator->mutex)) { + MSG_ERR(("Allocator release: Failed to get mutex - memory leak\n")); + return; + } + + while (block) { + next = block->next; + + BUG_ON((block < allocator->all_blocks) || (block > (allocator->all_blocks + allocator->num_blocks))); + + block->next = allocator->first_free; + allocator->first_free = block; + allocator->num_free++; + + block = next; + } + DBG_MSG(3, ("%d blocks free after release call\n", allocator->num_free)); + up(&allocator->mutex); + + vfree(handle->block_array); + handle->block_array = NULL; +} + + + +/* + * Helper function for calculating the physical base adderss of a memory block + */ +static inline u32 get_phys(block_allocator *allocator, block_info *block) +{ + return allocator->base + ((block - allocator->all_blocks) * UMP_BLOCK_SIZE); +} + +static u32 block_allocator_stat(struct ump_memory_backend *backend) +{ + block_allocator *allocator; + BUG_ON(!backend); + allocator = (block_allocator *)backend->ctx; + BUG_ON(!allocator); + + return (allocator->num_blocks - allocator->num_free) * UMP_BLOCK_SIZE; +} diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h new file mode 100755 index 000000000000..949fd245c6af --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2010, 2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_kernel_memory_backend_dedicated.h + */ + +#ifndef __UMP_KERNEL_MEMORY_BACKEND_DEDICATED_H__ +#define __UMP_KERNEL_MEMORY_BACKEND_DEDICATED_H__ + +#include "ump_kernel_memory_backend.h" + +ump_memory_backend *ump_block_allocator_create(u32 base_address, u32 size); + +#endif /* __UMP_KERNEL_MEMORY_BACKEND_DEDICATED_H__ */ + diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c new file mode 100755 index 000000000000..7cd8d5d381cf --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/* needed to detect kernel version specific code */ +#include + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) +#include +#else /* pre 2.6.26 the file was in the arch specific location */ +#include +#endif + +#include +#include +#include +#include +#include +#include +#include "ump_kernel_common.h" +#include "ump_kernel_memory_backend.h" + + + +typedef struct os_allocator { + struct semaphore mutex; + u32 num_pages_max; /**< Maximum number of pages to allocate from the OS */ + u32 num_pages_allocated; /**< Number of pages allocated from the OS */ +} os_allocator; + + + +static void os_free(void *ctx, ump_dd_mem *descriptor); +static int os_allocate(void *ctx, ump_dd_mem *descriptor); +static void os_memory_backend_destroy(ump_memory_backend *backend); +static u32 os_stat(struct ump_memory_backend *backend); + + + +/* + * Create OS memory backend + */ +ump_memory_backend *ump_os_memory_backend_create(const int max_allocation) +{ + ump_memory_backend *backend; + os_allocator *info; + + info = kmalloc(sizeof(os_allocator), GFP_KERNEL); + if (NULL == info) { + return NULL; + } + + info->num_pages_max = max_allocation >> PAGE_SHIFT; + info->num_pages_allocated = 0; + + sema_init(&info->mutex, 1); + + backend = kmalloc(sizeof(ump_memory_backend), GFP_KERNEL); + if (NULL == backend) { + kfree(info); + return NULL; + } + + backend->ctx = info; + backend->allocate = os_allocate; + backend->release = os_free; + backend->shutdown = os_memory_backend_destroy; + backend->stat = os_stat; + backend->pre_allocate_physical_check = NULL; + backend->adjust_to_mali_phys = NULL; + + return backend; +} + + + +/* + * Destroy specified OS memory backend + */ +static void os_memory_backend_destroy(ump_memory_backend *backend) +{ + os_allocator *info = (os_allocator *)backend->ctx; + + DBG_MSG_IF(1, 0 != info->num_pages_allocated, ("%d pages still in use during shutdown\n", info->num_pages_allocated)); + + kfree(info); + kfree(backend); +} + + + +/* + * Allocate UMP memory + */ +static int os_allocate(void *ctx, ump_dd_mem *descriptor) +{ + u32 left; + os_allocator *info; + int pages_allocated = 0; + int is_cached; + + BUG_ON(!descriptor); + BUG_ON(!ctx); + + info = (os_allocator *)ctx; + left = descriptor->size_bytes; + is_cached = descriptor->is_cached; + + if (down_interruptible(&info->mutex)) { + DBG_MSG(1, ("Failed to get mutex in os_free\n")); + return 0; /* failure */ + } + + descriptor->backend_info = NULL; + descriptor->nr_blocks = ((left + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) >> PAGE_SHIFT; + + DBG_MSG(5, ("Allocating page array. Size: %lu\n", descriptor->nr_blocks * sizeof(ump_dd_physical_block))); + + descriptor->block_array = (ump_dd_physical_block *)vmalloc(sizeof(ump_dd_physical_block) * descriptor->nr_blocks); + if (NULL == descriptor->block_array) { + up(&info->mutex); + DBG_MSG(1, ("Block array could not be allocated\n")); + return 0; /* failure */ + } + + while (left > 0 && ((info->num_pages_allocated + pages_allocated) < info->num_pages_max)) { + struct page *new_page; + + if (is_cached) { + new_page = alloc_page(GFP_HIGHUSER | __GFP_ZERO | __GFP_REPEAT | __GFP_NOWARN); + } else { + new_page = alloc_page(GFP_HIGHUSER | __GFP_ZERO | __GFP_REPEAT | __GFP_NOWARN | __GFP_COLD); + } + if (NULL == new_page) { + break; + } + + /* Ensure page caches are flushed. */ + if (is_cached) { + descriptor->block_array[pages_allocated].addr = page_to_phys(new_page); + descriptor->block_array[pages_allocated].size = PAGE_SIZE; + } else { + descriptor->block_array[pages_allocated].addr = dma_map_page(NULL, new_page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); + descriptor->block_array[pages_allocated].size = PAGE_SIZE; + } + + DBG_MSG(5, ("Allocated page 0x%08lx cached: %d\n", descriptor->block_array[pages_allocated].addr, is_cached)); + + if (left < PAGE_SIZE) { + left = 0; + } else { + left -= PAGE_SIZE; + } + + pages_allocated++; + } + + DBG_MSG(5, ("Alloce for ID:%2d got %d pages, cached: %d\n", descriptor->secure_id, pages_allocated)); + + if (left) { + DBG_MSG(1, ("Failed to allocate needed pages\n")); + + while (pages_allocated) { + pages_allocated--; + if (!is_cached) { + dma_unmap_page(NULL, descriptor->block_array[pages_allocated].addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + } + __free_page(pfn_to_page(descriptor->block_array[pages_allocated].addr >> PAGE_SHIFT)); + } + + up(&info->mutex); + + return 0; /* failure */ + } + + info->num_pages_allocated += pages_allocated; + + DBG_MSG(6, ("%d out of %d pages now allocated\n", info->num_pages_allocated, info->num_pages_max)); + + up(&info->mutex); + + return 1; /* success*/ +} + + +/* + * Free specified UMP memory + */ +static void os_free(void *ctx, ump_dd_mem *descriptor) +{ + os_allocator *info; + int i; + + BUG_ON(!ctx); + BUG_ON(!descriptor); + + info = (os_allocator *)ctx; + + BUG_ON(descriptor->nr_blocks > info->num_pages_allocated); + + if (down_interruptible(&info->mutex)) { + DBG_MSG(1, ("Failed to get mutex in os_free\n")); + return; + } + + DBG_MSG(5, ("Releasing %lu OS pages\n", descriptor->nr_blocks)); + + info->num_pages_allocated -= descriptor->nr_blocks; + + up(&info->mutex); + + for (i = 0; i < descriptor->nr_blocks; i++) { + DBG_MSG(6, ("Freeing physical page. Address: 0x%08lx\n", descriptor->block_array[i].addr)); + if (! descriptor->is_cached) { + dma_unmap_page(NULL, descriptor->block_array[i].addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + } + __free_page(pfn_to_page(descriptor->block_array[i].addr >> PAGE_SHIFT)); + } + + vfree(descriptor->block_array); +} + + +static u32 os_stat(struct ump_memory_backend *backend) +{ + os_allocator *info; + info = (os_allocator *)backend->ctx; + return info->num_pages_allocated * _MALI_OSK_MALI_PAGE_SIZE; +} diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h new file mode 100755 index 000000000000..d21d503512ec --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2010, 2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_kernel_memory_backend_os.h + */ + +#ifndef __UMP_KERNEL_MEMORY_BACKEND_OS_H__ +#define __UMP_KERNEL_MEMORY_BACKEND_OS_H__ + +#include "ump_kernel_memory_backend.h" + +ump_memory_backend *ump_os_memory_backend_create(const int max_allocation); + +#endif /* __UMP_KERNEL_MEMORY_BACKEND_OS_H__ */ + diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c new file mode 100755 index 000000000000..6be0f86440de --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c @@ -0,0 +1,222 @@ +/* + * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "mali_kernel_common.h" +#include "mali_osk.h" +#include "ump_osk.h" +#include "ump_kernel_common.h" +#include "ump_kernel_types.h" +#include "ump_kernel_random_mapping.h" + +#include +#include +#include +#include + + +static ump_dd_mem *search(struct rb_root *root, int id) +{ + struct rb_node *node = root->rb_node; + + while (node) { + ump_dd_mem *e = container_of(node, ump_dd_mem, node); + + if (id < e->secure_id) { + node = node->rb_left; + } else if (id > e->secure_id) { + node = node->rb_right; + } else { + return e; + } + } + + return NULL; +} + +static mali_bool insert(struct rb_root *root, int id, ump_dd_mem *mem) +{ + struct rb_node **new = &(root->rb_node); + struct rb_node *parent = NULL; + + while (*new) { + ump_dd_mem *this = container_of(*new, ump_dd_mem, node); + + parent = *new; + if (id < this->secure_id) { + new = &((*new)->rb_left); + } else if (id > this->secure_id) { + new = &((*new)->rb_right); + } else { + printk(KERN_ERR "UMP: ID already used %x\n", id); + return MALI_FALSE; + } + } + + rb_link_node(&mem->node, parent, new); + rb_insert_color(&mem->node, root); + + return MALI_TRUE; +} + + +ump_random_mapping *ump_random_mapping_create(void) +{ + ump_random_mapping *map = _mali_osk_calloc(1, sizeof(ump_random_mapping)); + + if (NULL == map) + return NULL; + + map->lock = _mali_osk_mutex_rw_init(_MALI_OSK_LOCKFLAG_ORDERED, + _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP); + if (NULL != map->lock) { + map->root = RB_ROOT; +#if UMP_RANDOM_MAP_DELAY + map->failed.count = 0; + map->failed.timestamp = jiffies; +#endif + return map; + } + return NULL; +} + +void ump_random_mapping_destroy(ump_random_mapping *map) +{ + _mali_osk_mutex_rw_term(map->lock); + _mali_osk_free(map); +} + +int ump_random_mapping_insert(ump_random_mapping *map, ump_dd_mem *mem) +{ + _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); + + while (1) { + u32 id; + + get_random_bytes(&id, sizeof(id)); + + /* Try a new random number if id happened to be the invalid + * secure ID (-1). */ + if (unlikely(id == UMP_INVALID_SECURE_ID)) + continue; + + /* Insert into the tree. If the id was already in use, get a + * new random id and try again. */ + if (insert(&map->root, id, mem)) { + mem->secure_id = id; + break; + } + } + _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); + + return 0; +} + +ump_dd_mem *ump_random_mapping_get(ump_random_mapping *map, int id) +{ + ump_dd_mem *mem = NULL; +#if UMP_RANDOM_MAP_DELAY + int do_delay = 0; +#endif + + DEBUG_ASSERT(map); + + _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); + mem = search(&map->root, id); + + if (unlikely(NULL == mem)) { +#if UMP_RANDOM_MAP_DELAY + map->failed.count++; + + if (time_is_before_jiffies(map->failed.timestamp + + UMP_FAILED_LOOKUP_DELAY * HZ)) { + /* If it is a long time since last failure, reset + * the counter and skip the delay this time. */ + map->failed.count = 0; + } else if (map->failed.count > UMP_FAILED_LOOKUPS_ALLOWED) { + do_delay = 1; + } + + map->failed.timestamp = jiffies; +#endif /* UMP_RANDOM_MAP_DELAY */ + } else { + ump_dd_reference_add(mem); + } + _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); + +#if UMP_RANDOM_MAP_DELAY + if (do_delay) { + /* Apply delay */ + schedule_timeout_killable(UMP_FAILED_LOOKUP_DELAY); + } +#endif /* UMP_RANDOM_MAP_DELAY */ + + return mem; +} + +static ump_dd_mem *ump_random_mapping_remove_internal(ump_random_mapping *map, int id) +{ + ump_dd_mem *mem = NULL; + + mem = search(&map->root, id); + + if (mem) { + rb_erase(&mem->node, &map->root); + } + + return mem; +} + +void ump_random_mapping_put(ump_dd_mem *mem) +{ + int new_ref; + + _mali_osk_mutex_rw_wait(device.secure_id_map->lock, _MALI_OSK_LOCKMODE_RW); + + new_ref = _ump_osk_atomic_dec_and_read(&mem->ref_count); + DBG_MSG(5, ("Memory reference decremented. ID: %u, new value: %d\n", + mem->secure_id, new_ref)); + + if (0 == new_ref) { + DBG_MSG(3, ("Final release of memory. ID: %u\n", mem->secure_id)); + +#ifdef CONFIG_DMA_SHARED_BUFFER + if (mem->import_attach) { + struct dma_buf_attachment *attach = mem->import_attach; + struct dma_buf *dma_buf; + + if (mem->sgt) + dma_buf_unmap_attachment(attach, mem->sgt, + DMA_BIDIRECTIONAL); + + dma_buf = attach->dmabuf; + dma_buf_detach(attach->dmabuf, attach); + dma_buf_put(dma_buf); + + } +#endif + ump_random_mapping_remove_internal(device.secure_id_map, mem->secure_id); + + mem->release_func(mem->ctx, mem); + _mali_osk_free(mem); + } + + _mali_osk_mutex_rw_signal(device.secure_id_map->lock, _MALI_OSK_LOCKMODE_RW); +} + +ump_dd_mem *ump_random_mapping_remove(ump_random_mapping *map, int descriptor) +{ + ump_dd_mem *mem; + + _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); + mem = ump_random_mapping_remove_internal(map, descriptor); + _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); + + return mem; +} diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h new file mode 100755 index 000000000000..2cea6cedc380 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_kernel_random_mapping.h + */ + +#ifndef __UMP_KERNEL_RANDOM_MAPPING_H__ +#define __UMP_KERNEL_RANDOM_MAPPING_H__ + +#include "mali_osk.h" +#include + +#define UMP_RANDOM_MAP_DELAY 1 +#define UMP_FAILED_LOOKUP_DELAY 10 /* ms */ +#define UMP_FAILED_LOOKUPS_ALLOWED 10 /* number of allowed failed lookups */ + +/** + * The random mapping object + * Provides a separate namespace where we can map an integer to a pointer + */ +typedef struct ump_random_mapping { + _mali_osk_mutex_rw_t *lock; /**< Lock protecting access to the mapping object */ + struct rb_root root; +#if UMP_RANDOM_MAP_DELAY + struct { + unsigned long count; + unsigned long timestamp; + } failed; +#endif +} ump_random_mapping; + +/** + * Create a random mapping object + * Create a random mapping capable of holding 2^20 entries + * @return Pointer to a random mapping object, NULL on failure + */ +ump_random_mapping *ump_random_mapping_create(void); + +/** + * Destroy a random mapping object + * @param map The map to free + */ +void ump_random_mapping_destroy(ump_random_mapping *map); + +/** + * Allocate a new mapping entry (random ID) + * Allocates a new entry in the map. + * @param map The map to allocate a new entry in + * @param target The value to map to + * @return The random allocated, a negative value on error + */ +int ump_random_mapping_insert(ump_random_mapping *map, ump_dd_mem *mem); + +/** + * Get the value mapped to by a random ID + * + * If the lookup fails, punish the calling thread by applying a delay. + * + * @param map The map to lookup the random id in + * @param id The ID to lookup + * @param target Pointer to a pointer which will receive the stored value + * @return ump_dd_mem pointer on successful lookup, NULL on error + */ +ump_dd_mem *ump_random_mapping_get(ump_random_mapping *map, int id); + +void ump_random_mapping_put(ump_dd_mem *mem); + +/** + * Free the random ID + * For the random to be reused it has to be freed + * @param map The map to free the random from + * @param id The ID to free + */ +ump_dd_mem *ump_random_mapping_remove(ump_random_mapping *map, int id); + +#endif /* __UMP_KERNEL_RANDOM_MAPPING_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c b/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c new file mode 100755 index 000000000000..e41931e1ea75 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include /* kernel module definitions */ +#include /* request_mem_region */ + +#include "arch/config.h" /* Configuration for current platform. The symlink for arch is set by Makefile */ + +#include "ump_osk.h" +#include "ump_kernel_common.h" +#include "ump_kernel_memory_backend_os.h" +#include "ump_kernel_memory_backend_dedicated.h" + +/* Configure which dynamic memory allocator to use */ +int ump_backend = ARCH_UMP_BACKEND_DEFAULT; +module_param(ump_backend, int, S_IRUGO); /* r--r--r-- */ +MODULE_PARM_DESC(ump_backend, "0 = dedicated memory backend (default), 1 = OS memory backend"); + +/* The base address of the memory block for the dedicated memory backend */ +unsigned int ump_memory_address = ARCH_UMP_MEMORY_ADDRESS_DEFAULT; +module_param(ump_memory_address, uint, S_IRUGO); /* r--r--r-- */ +MODULE_PARM_DESC(ump_memory_address, "The physical address to map for the dedicated memory backend"); + +/* The size of the memory block for the dedicated memory backend */ +unsigned int ump_memory_size = ARCH_UMP_MEMORY_SIZE_DEFAULT; +module_param(ump_memory_size, uint, S_IRUGO); /* r--r--r-- */ +MODULE_PARM_DESC(ump_memory_size, "The size of fixed memory to map in the dedicated memory backend"); + +ump_memory_backend *ump_memory_backend_create(void) +{ + ump_memory_backend *backend = NULL; + + /* Create the dynamic memory allocator backend */ + if (0 == ump_backend) { + DBG_MSG(2, ("Using dedicated memory backend\n")); + + DBG_MSG(2, ("Requesting dedicated memory: 0x%08x, size: %u\n", ump_memory_address, ump_memory_size)); + /* Ask the OS if we can use the specified physical memory */ + if (NULL == request_mem_region(ump_memory_address, ump_memory_size, "UMP Memory")) { + MSG_ERR(("Failed to request memory region (0x%08X - 0x%08X). Is Mali DD already loaded?\n", ump_memory_address, ump_memory_address + ump_memory_size - 1)); + return NULL; + } + backend = ump_block_allocator_create(ump_memory_address, ump_memory_size); + } else if (1 == ump_backend) { + DBG_MSG(2, ("Using OS memory backend, allocation limit: %d\n", ump_memory_size)); + backend = ump_os_memory_backend_create(ump_memory_size); + } + + return backend; +} + +void ump_memory_backend_destroy(void) +{ + if (0 == ump_backend) { + DBG_MSG(2, ("Releasing dedicated memory: 0x%08x\n", ump_memory_address)); + release_mem_region(ump_memory_address, ump_memory_size); + } +} diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c new file mode 100755 index 000000000000..2b634ba79c6e --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2010, 2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_osk_atomics.c + * Implementation of the OS abstraction layer for the UMP kernel device driver + */ + +#include "ump_osk.h" +#include + +int _ump_osk_atomic_dec_and_read(_mali_osk_atomic_t *atom) +{ + return atomic_dec_return((atomic_t *)&atom->u.val); +} + +int _ump_osk_atomic_inc_and_read(_mali_osk_atomic_t *atom) +{ + return atomic_inc_return((atomic_t *)&atom->u.val); +} diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c new file mode 100755 index 000000000000..e08bf25257b0 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c @@ -0,0 +1,314 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_osk_memory.c + * Implementation of the OS abstraction layer for the kernel device driver + */ + +/* needed to detect kernel version specific code */ +#include + +#include "ump_osk.h" +#include "ump_uk_types.h" +#include "ump_ukk.h" +#include "ump_kernel_common.h" +#include /* kernel module definitions */ +#include +#include +#include + +#include +#include /* to verify pointers from user space */ +#include +#include + +typedef struct ump_vma_usage_tracker { + atomic_t references; + ump_memory_allocation *descriptor; +} ump_vma_usage_tracker; + +static void ump_vma_open(struct vm_area_struct *vma); +static void ump_vma_close(struct vm_area_struct *vma); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) +static int ump_cpu_page_fault_handler(struct vm_area_struct *vma, struct vm_fault *vmf); +#else +static unsigned long ump_cpu_page_fault_handler(struct vm_area_struct *vma, unsigned long address); +#endif + +static struct vm_operations_struct ump_vm_ops = { + .open = ump_vma_open, + .close = ump_vma_close, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) + .fault = ump_cpu_page_fault_handler +#else + .nopfn = ump_cpu_page_fault_handler +#endif +}; + +/* + * Page fault for VMA region + * This should never happen since we always map in the entire virtual memory range. + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) +static int ump_cpu_page_fault_handler(struct vm_area_struct *vma, struct vm_fault *vmf) +#else +static unsigned long ump_cpu_page_fault_handler(struct vm_area_struct *vma, unsigned long address) +#endif +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) + void __user *address; + address = vmf->virtual_address; +#endif + MSG_ERR(("Page-fault in UMP memory region caused by the CPU\n")); + MSG_ERR(("VMA: 0x%08lx, virtual address: 0x%08lx\n", (unsigned long)vma, address)); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) + return VM_FAULT_SIGBUS; +#else + return NOPFN_SIGBUS; +#endif +} + +static void ump_vma_open(struct vm_area_struct *vma) +{ + ump_vma_usage_tracker *vma_usage_tracker; + int new_val; + + vma_usage_tracker = (ump_vma_usage_tracker *)vma->vm_private_data; + BUG_ON(NULL == vma_usage_tracker); + + new_val = atomic_inc_return(&vma_usage_tracker->references); + + DBG_MSG(4, ("VMA open, VMA reference count incremented. VMA: 0x%08lx, reference count: %d\n", (unsigned long)vma, new_val)); +} + +static void ump_vma_close(struct vm_area_struct *vma) +{ + ump_vma_usage_tracker *vma_usage_tracker; + _ump_uk_unmap_mem_s args; + int new_val; + + vma_usage_tracker = (ump_vma_usage_tracker *)vma->vm_private_data; + BUG_ON(NULL == vma_usage_tracker); + + new_val = atomic_dec_return(&vma_usage_tracker->references); + + DBG_MSG(4, ("VMA close, VMA reference count decremented. VMA: 0x%08lx, reference count: %d\n", (unsigned long)vma, new_val)); + + if (0 == new_val) { + ump_memory_allocation *descriptor; + + descriptor = vma_usage_tracker->descriptor; + + args.ctx = descriptor->ump_session; + args.cookie = descriptor->cookie; + args.mapping = descriptor->mapping; + args.size = descriptor->size; + + args._ukk_private = NULL; /** @note unused */ + + DBG_MSG(4, ("No more VMA references left, releasing UMP memory\n")); + _ump_ukk_unmap_mem(& args); + + /* vma_usage_tracker is free()d by _ump_osk_mem_mapregion_term() */ + } +} + +_mali_osk_errcode_t _ump_osk_mem_mapregion_init(ump_memory_allocation *descriptor) +{ + ump_vma_usage_tracker *vma_usage_tracker; + struct vm_area_struct *vma; + + if (NULL == descriptor) return _MALI_OSK_ERR_FAULT; + + vma_usage_tracker = kmalloc(sizeof(ump_vma_usage_tracker), GFP_KERNEL); + if (NULL == vma_usage_tracker) { + DBG_MSG(1, ("Failed to allocate memory for ump_vma_usage_tracker in _mali_osk_mem_mapregion_init\n")); + return -_MALI_OSK_ERR_FAULT; + } + + vma = (struct vm_area_struct *)descriptor->process_mapping_info; + if (NULL == vma) { + kfree(vma_usage_tracker); + return _MALI_OSK_ERR_FAULT; + } + + vma->vm_private_data = vma_usage_tracker; + vma->vm_flags |= VM_IO; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0) + vma->vm_flags |= VM_RESERVED; +#else + vma->vm_flags |= VM_DONTDUMP; + vma->vm_flags |= VM_DONTEXPAND; + vma->vm_flags |= VM_PFNMAP; +#endif + + + if (0 == descriptor->is_cached) { + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + } + DBG_MSG(3, ("Mapping with page_prot: 0x%x\n", vma->vm_page_prot)); + + /* Setup the functions which handle further VMA handling */ + vma->vm_ops = &ump_vm_ops; + + /* Do the va range allocation - in this case, it was done earlier, so we copy in that information */ + descriptor->mapping = (void __user *)vma->vm_start; + + atomic_set(&vma_usage_tracker->references, 1); /*this can later be increased if process is forked, see ump_vma_open() */ + vma_usage_tracker->descriptor = descriptor; + + return _MALI_OSK_ERR_OK; +} + +void _ump_osk_mem_mapregion_term(ump_memory_allocation *descriptor) +{ + struct vm_area_struct *vma; + ump_vma_usage_tracker *vma_usage_tracker; + + if (NULL == descriptor) return; + + /* Linux does the right thing as part of munmap to remove the mapping + * All that remains is that we remove the vma_usage_tracker setup in init() */ + vma = (struct vm_area_struct *)descriptor->process_mapping_info; + + vma_usage_tracker = vma->vm_private_data; + + /* We only get called if mem_mapregion_init succeeded */ + kfree(vma_usage_tracker); + return; +} + +_mali_osk_errcode_t _ump_osk_mem_mapregion_map(ump_memory_allocation *descriptor, u32 offset, u32 *phys_addr, unsigned long size) +{ + struct vm_area_struct *vma; + _mali_osk_errcode_t retval; + + if (NULL == descriptor) return _MALI_OSK_ERR_FAULT; + + vma = (struct vm_area_struct *)descriptor->process_mapping_info; + + if (NULL == vma) return _MALI_OSK_ERR_FAULT; + + retval = remap_pfn_range(vma, ((u32)descriptor->mapping) + offset, (*phys_addr) >> PAGE_SHIFT, size, vma->vm_page_prot) ? _MALI_OSK_ERR_FAULT : _MALI_OSK_ERR_OK;; + + DBG_MSG(4, ("Mapping virtual to physical memory. ID: %u, vma: 0x%08lx, virtual addr:0x%08lx, physical addr: 0x%08lx, size:%lu, prot:0x%x, vm_flags:0x%x RETVAL: 0x%x\n", + ump_dd_secure_id_get(descriptor->handle), + (unsigned long)vma, + (unsigned long)(vma->vm_start + offset), + (unsigned long)*phys_addr, + size, + (unsigned int)vma->vm_page_prot, vma->vm_flags, retval)); + + return retval; +} + +static void level1_cache_flush_all(void) +{ + DBG_MSG(4, ("UMP[xx] Flushing complete L1 cache\n")); + __cpuc_flush_kern_all(); +} + +void _ump_osk_msync(ump_dd_mem *mem, void *virt, u32 offset, u32 size, ump_uk_msync_op op, ump_session_data *session_data) +{ + int i; + + /* Flush L1 using virtual address, the entire range in one go. + * Only flush if user space process has a valid write mapping on given address. */ + if ((mem) && (virt != NULL) && (access_ok(virt, size))) { + __cpuc_flush_dcache_area(virt, size); + DBG_MSG(3, ("UMP[%02u] Flushing CPU L1 Cache. CPU address: %x, size: %x\n", mem->secure_id, virt, size)); + } else { + if (session_data) { + if (op == _UMP_UK_MSYNC_FLUSH_L1) { + DBG_MSG(4, ("UMP Pending L1 cache flushes: %d\n", session_data->has_pending_level1_cache_flush)); + session_data->has_pending_level1_cache_flush = 0; + level1_cache_flush_all(); + return; + } else { + if (session_data->cache_operations_ongoing) { + session_data->has_pending_level1_cache_flush++; + DBG_MSG(4, ("UMP[%02u] Defering the L1 flush. Nr pending:%d\n", mem->secure_id, session_data->has_pending_level1_cache_flush)); + } else { + /* Flushing the L1 cache for each switch_user() if ump_cache_operations_control(START) is not called */ + level1_cache_flush_all(); + } + } + } else { + DBG_MSG(4, ("Unkown state %s %d\n", __FUNCTION__, __LINE__)); + level1_cache_flush_all(); + } + } + + if (NULL == mem) return; + + if (mem->size_bytes == size) { + DBG_MSG(3, ("UMP[%02u] Flushing CPU L2 Cache\n", mem->secure_id)); + } else { + DBG_MSG(3, ("UMP[%02u] Flushing CPU L2 Cache. Blocks:%u, TotalSize:%u. FlushSize:%u Offset:0x%x FirstPaddr:0x%08x\n", + mem->secure_id, mem->nr_blocks, mem->size_bytes, size, offset, mem->block_array[0].addr)); + } + + + /* Flush L2 using physical addresses, block for block. */ + for (i = 0 ; i < mem->nr_blocks; i++) { + u32 start_p, end_p; + ump_dd_physical_block *block; + block = &mem->block_array[i]; + + if (offset >= block->size) { + offset -= block->size; + continue; + } + + if (offset) { + start_p = (u32)block->addr + offset; + /* We'll zero the offset later, after using it to calculate end_p. */ + } else { + start_p = (u32)block->addr; + } + + if (size < block->size - offset) { + end_p = start_p + size; + size = 0; + } else { + if (offset) { + end_p = start_p + (block->size - offset); + size -= block->size - offset; + offset = 0; + } else { + end_p = start_p + block->size; + size -= block->size; + } + } + + switch (op) { + case _UMP_UK_MSYNC_CLEAN: + outer_clean_range(start_p, end_p); + break; + case _UMP_UK_MSYNC_CLEAN_AND_INVALIDATE: + outer_flush_range(start_p, end_p); + break; + case _UMP_UK_MSYNC_INVALIDATE: + outer_inv_range(start_p, end_p); + break; + default: + break; + } + + if (0 == size) { + /* Nothing left to flush. */ + break; + } + } + + return; +} diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c new file mode 100755 index 000000000000..58c9f1bf27b8 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_osk_misc.c + * Implementation of the OS abstraction layer for the UMP kernel device driver + */ + + +#include "ump_osk.h" + +#include +#include "ump_kernel_linux.h" + +/* is called from ump_kernel_constructor in common code */ +_mali_osk_errcode_t _ump_osk_init(void) +{ + if (0 != ump_kernel_device_initialize()) { + return _MALI_OSK_ERR_FAULT; + } + + return _MALI_OSK_ERR_OK; +} + +_mali_osk_errcode_t _ump_osk_term(void) +{ + ump_kernel_device_terminate(); + return _MALI_OSK_ERR_OK; +} diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c new file mode 100755 index 000000000000..56a787ff64dc --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_ukk_wrappers.c + * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls for the reference implementation + */ + + +#include /* user space access */ + +#include "ump_osk.h" +#include "ump_uk_types.h" +#include "ump_ukk.h" +#include "ump_kernel_common.h" +#include +#include "ump_kernel_interface_ref_drv.h" +#include "mali_osk_list.h" + +extern struct device *ump_global_mdev; + +/* + * IOCTL operation; Allocate UMP memory + */ +int ump_allocate_wrapper(u32 __user *argument, struct ump_session_data *session_data) +{ + _ump_uk_allocate_s user_interaction; + _mali_osk_errcode_t err; + + /* Sanity check input parameters */ + if (NULL == argument || NULL == session_data) { + MSG_ERR(("NULL parameter in ump_ioctl_allocate()\n")); + return -ENOTTY; + } + + /* Copy the user space memory to kernel space (so we safely can read it) */ + if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { + MSG_ERR(("copy_from_user() in ump_ioctl_allocate()\n")); + return -EFAULT; + } + + user_interaction.ctx = (void *) session_data; + + err = _ump_ukk_allocate(&user_interaction); + if (_MALI_OSK_ERR_OK != err) { + DBG_MSG(1, ("_ump_ukk_allocate() failed in ump_ioctl_allocate()\n")); + return ump_map_errcode(err); + } + user_interaction.ctx = NULL; + + if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { + /* If the copy fails then we should release the memory. We can use the IOCTL release to accomplish this */ + _ump_uk_release_s release_args; + + MSG_ERR(("copy_to_user() failed in ump_ioctl_allocate()\n")); + + release_args.ctx = (void *) session_data; + release_args.secure_id = user_interaction.secure_id; + + err = _ump_ukk_release(&release_args); + if (_MALI_OSK_ERR_OK != err) { + MSG_ERR(("_ump_ukk_release() also failed when trying to release newly allocated memory in ump_ioctl_allocate()\n")); + } + + return -EFAULT; + } + + return 0; /* success */ +} + +#ifdef CONFIG_DMA_SHARED_BUFFER +static ump_dd_handle get_ump_handle_from_dmabuf(struct ump_session_data *session_data, + struct dma_buf *dmabuf) +{ + ump_session_memory_list_element *session_mem, *tmp; + struct dma_buf_attachment *attach; + ump_dd_handle ump_handle; + + DEBUG_ASSERT_POINTER(session_data); + + _mali_osk_mutex_wait(session_data->lock); + + _MALI_OSK_LIST_FOREACHENTRY(session_mem, tmp, + &session_data->list_head_session_memory_list, + ump_session_memory_list_element, list) { + if (session_mem->mem->import_attach) { + attach = session_mem->mem->import_attach; + if (attach->dmabuf == dmabuf) { + _mali_osk_mutex_signal(session_data->lock); + ump_handle = (ump_dd_handle)session_mem->mem; + ump_random_mapping_get(device.secure_id_map, ump_dd_secure_id_get(ump_handle)); + return ump_handle; + } + } + } + + _mali_osk_mutex_signal(session_data->lock); + + return NULL; +} + +int ump_dmabuf_import_wrapper(u32 __user *argument, + struct ump_session_data *session_data) +{ + ump_session_memory_list_element *session = NULL; + _ump_uk_dmabuf_s ump_dmabuf; + ump_dd_handle ump_handle; + ump_dd_physical_block *blocks = NULL; + struct dma_buf_attachment *attach = NULL; + struct dma_buf *dma_buf; + struct sg_table *sgt = NULL; + struct scatterlist *sgl; + unsigned int i = 0; + int ret = 0; + + /* Sanity check input parameters */ + if (!argument || !session_data) { + MSG_ERR(("NULL parameter.\n")); + return -EINVAL; + } + + if (copy_from_user(&ump_dmabuf, argument, + sizeof(_ump_uk_dmabuf_s))) { + MSG_ERR(("copy_from_user() failed.\n")); + return -EFAULT; + } + + dma_buf = dma_buf_get(ump_dmabuf.fd); + if (IS_ERR(dma_buf)) + return PTR_ERR(dma_buf); + + /* + * if already imported then increase a refcount to the ump descriptor + * and call dma_buf_put() and then go to found to return previous + * ump secure id. + */ + ump_handle = get_ump_handle_from_dmabuf(session_data, dma_buf); + if (ump_handle) { + dma_buf_put(dma_buf); + goto found; + } + + attach = dma_buf_attach(dma_buf, ump_global_mdev); + if (IS_ERR(attach)) { + ret = PTR_ERR(attach); + goto err_dma_buf_put; + } + + sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); + if (IS_ERR(sgt)) { + ret = PTR_ERR(sgt); + goto err_dma_buf_detach; + } + + blocks = (ump_dd_physical_block *)_mali_osk_malloc(sizeof(ump_dd_physical_block) * sgt->nents); + if (!blocks) { + DBG_MSG(1, ("Failed to allocate blocks.\n")); + ret = -EFAULT; + goto err_dma_buf_unmap; + } + for_each_sg(sgt->sgl, sgl, sgt->nents, i) { + blocks[i].addr = sg_phys(sgl); + blocks[i].size = sg_dma_len(sgl); + } + + /* + * Initialize the session memory list element, and add it + * to the session object + */ + session = _mali_osk_calloc(1, sizeof(*session)); + if (!session) { + DBG_MSG(1, ("Failed to allocate session.\n")); + ret = -EFAULT; + goto err_free_block; + } + + ump_handle = ump_dd_handle_create_from_phys_blocks(blocks, i); + if (UMP_DD_HANDLE_INVALID == ump_handle) { + DBG_MSG(1, ("Failed to create ump handle.\n")); + ret = -EFAULT; + goto err_free_session; + } + + session->mem = (ump_dd_mem *)ump_handle; + session->mem->import_attach = attach; + session->mem->sgt = sgt; + + _mali_osk_mutex_wait(session_data->lock); + _mali_osk_list_add(&(session->list), + &(session_data->list_head_session_memory_list)); + _mali_osk_mutex_signal(session_data->lock); + + _mali_osk_free(blocks); + +found: + ump_dmabuf.ctx = (void *)session_data; + ump_dmabuf.secure_id = ump_dd_secure_id_get(ump_handle); + ump_dmabuf.size = ump_dd_size_get(ump_handle); + + if (copy_to_user(argument, &ump_dmabuf, + sizeof(_ump_uk_dmabuf_s))) { + MSG_ERR(("copy_to_user() failed.\n")); + ret = -EFAULT; + goto err_release_ump_handle; + } + + return ret; + +err_release_ump_handle: + ump_dd_reference_release(ump_handle); +err_free_session: + _mali_osk_free(session); +err_free_block: + _mali_osk_free(blocks); +err_dma_buf_unmap: + dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL); +err_dma_buf_detach: + dma_buf_detach(dma_buf, attach); +err_dma_buf_put: + dma_buf_put(dma_buf); + return ret; +} +#endif diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h new file mode 100755 index 000000000000..61a7095a6920 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_ukk_wrappers.h + * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls for the reference implementation + */ + +#ifndef __UMP_UKK_REF_WRAPPERS_H__ +#define __UMP_UKK_REF_WRAPPERS_H__ + +#include +#include "ump_kernel_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +int ump_allocate_wrapper(u32 __user *argument, struct ump_session_data *session_data); +#ifdef CONFIG_DMA_SHARED_BUFFER +int ump_dmabuf_import_wrapper(u32 __user *argument, struct ump_session_data *session_data); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __UMP_UKK_REF_WRAPPERS_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c new file mode 100755 index 000000000000..4d6b69608fd4 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_ukk_wrappers.c + * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls + */ + +#include /* user space access */ + +#include "ump_osk.h" +#include "ump_uk_types.h" +#include "ump_ukk.h" +#include "ump_kernel_common.h" + +/* + * IOCTL operation; Negotiate version of IOCTL API + */ +int ump_get_api_version_wrapper(u32 __user *argument, struct ump_session_data *session_data) +{ + _ump_uk_api_version_s version_info; + _mali_osk_errcode_t err; + + /* Sanity check input parameters */ + if (NULL == argument || NULL == session_data) { + MSG_ERR(("NULL parameter in ump_ioctl_get_api_version()\n")); + return -ENOTTY; + } + + /* Copy the user space memory to kernel space (so we safely can read it) */ + if (0 != copy_from_user(&version_info, argument, sizeof(version_info))) { + MSG_ERR(("copy_from_user() in ump_ioctl_get_api_version()\n")); + return -EFAULT; + } + + version_info.ctx = (void *) session_data; + err = _ump_uku_get_api_version(&version_info); + if (_MALI_OSK_ERR_OK != err) { + MSG_ERR(("_ump_uku_get_api_version() failed in ump_ioctl_get_api_version()\n")); + return ump_map_errcode(err); + } + + version_info.ctx = NULL; + + /* Copy ouput data back to user space */ + if (0 != copy_to_user(argument, &version_info, sizeof(version_info))) { + MSG_ERR(("copy_to_user() failed in ump_ioctl_get_api_version()\n")); + return -EFAULT; + } + + return 0; /* success */ +} + + +/* + * IOCTL operation; Release reference to specified UMP memory. + */ +int ump_release_wrapper(u32 __user *argument, struct ump_session_data *session_data) +{ + _ump_uk_release_s release_args; + _mali_osk_errcode_t err; + + /* Sanity check input parameters */ + if (NULL == session_data) { + MSG_ERR(("NULL parameter in ump_ioctl_release()\n")); + return -ENOTTY; + } + + /* Copy the user space memory to kernel space (so we safely can read it) */ + if (0 != copy_from_user(&release_args, argument, sizeof(release_args))) { + MSG_ERR(("copy_from_user() in ump_ioctl_get_api_version()\n")); + return -EFAULT; + } + + release_args.ctx = (void *) session_data; + err = _ump_ukk_release(&release_args); + if (_MALI_OSK_ERR_OK != err) { + MSG_ERR(("_ump_ukk_release() failed in ump_ioctl_release()\n")); + return ump_map_errcode(err); + } + + + return 0; /* success */ +} + +/* + * IOCTL operation; Return size for specified UMP memory. + */ +int ump_size_get_wrapper(u32 __user *argument, struct ump_session_data *session_data) +{ + _ump_uk_size_get_s user_interaction; + _mali_osk_errcode_t err; + + /* Sanity check input parameters */ + if (NULL == argument || NULL == session_data) { + MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); + return -ENOTTY; + } + + if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { + MSG_ERR(("copy_from_user() in ump_ioctl_size_get()\n")); + return -EFAULT; + } + + user_interaction.ctx = (void *) session_data; + err = _ump_ukk_size_get(&user_interaction); + if (_MALI_OSK_ERR_OK != err) { + MSG_ERR(("_ump_ukk_size_get() failed in ump_ioctl_size_get()\n")); + return ump_map_errcode(err); + } + + user_interaction.ctx = NULL; + + if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { + MSG_ERR(("copy_to_user() failed in ump_ioctl_size_get()\n")); + return -EFAULT; + } + + return 0; /* success */ +} + +/* + * IOCTL operation; Do cache maintenance on specified UMP memory. + */ +int ump_msync_wrapper(u32 __user *argument, struct ump_session_data *session_data) +{ + _ump_uk_msync_s user_interaction; + + /* Sanity check input parameters */ + if (NULL == argument || NULL == session_data) { + MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); + return -ENOTTY; + } + + if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { + MSG_ERR(("copy_from_user() in ump_ioctl_msync()\n")); + return -EFAULT; + } + + user_interaction.ctx = (void *) session_data; + + _ump_ukk_msync(&user_interaction); + + user_interaction.ctx = NULL; + + if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { + MSG_ERR(("copy_to_user() failed in ump_ioctl_msync()\n")); + return -EFAULT; + } + + return 0; /* success */ +} +int ump_cache_operations_control_wrapper(u32 __user *argument, struct ump_session_data *session_data) +{ + _ump_uk_cache_operations_control_s user_interaction; + + /* Sanity check input parameters */ + if (NULL == argument || NULL == session_data) { + MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); + return -ENOTTY; + } + + if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { + MSG_ERR(("copy_from_user() in ump_ioctl_cache_operations_control()\n")); + return -EFAULT; + } + + user_interaction.ctx = (void *) session_data; + + _ump_ukk_cache_operations_control((_ump_uk_cache_operations_control_s *) &user_interaction); + + user_interaction.ctx = NULL; + +#if 0 /* No data to copy back */ + if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { + MSG_ERR(("copy_to_user() failed in ump_ioctl_cache_operations_control()\n")); + return -EFAULT; + } +#endif + return 0; /* success */ +} + +int ump_switch_hw_usage_wrapper(u32 __user *argument, struct ump_session_data *session_data) +{ + _ump_uk_switch_hw_usage_s user_interaction; + + /* Sanity check input parameters */ + if (NULL == argument || NULL == session_data) { + MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); + return -ENOTTY; + } + + if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { + MSG_ERR(("copy_from_user() in ump_ioctl_switch_hw_usage()\n")); + return -EFAULT; + } + + user_interaction.ctx = (void *) session_data; + + _ump_ukk_switch_hw_usage(&user_interaction); + + user_interaction.ctx = NULL; + +#if 0 /* No data to copy back */ + if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { + MSG_ERR(("copy_to_user() failed in ump_ioctl_switch_hw_usage()\n")); + return -EFAULT; + } +#endif + return 0; /* success */ +} + +int ump_lock_wrapper(u32 __user *argument, struct ump_session_data *session_data) +{ + _ump_uk_lock_s user_interaction; + + /* Sanity check input parameters */ + if (NULL == argument || NULL == session_data) { + MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); + return -ENOTTY; + } + + if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { + MSG_ERR(("copy_from_user() in ump_ioctl_switch_hw_usage()\n")); + return -EFAULT; + } + + user_interaction.ctx = (void *) session_data; + + _ump_ukk_lock(&user_interaction); + + user_interaction.ctx = NULL; + +#if 0 /* No data to copy back */ + if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { + MSG_ERR(("copy_to_user() failed in ump_ioctl_switch_hw_usage()\n")); + return -EFAULT; + } +#endif + + return 0; /* success */ +} + +int ump_unlock_wrapper(u32 __user *argument, struct ump_session_data *session_data) +{ + _ump_uk_unlock_s user_interaction; + + /* Sanity check input parameters */ + if (NULL == argument || NULL == session_data) { + MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); + return -ENOTTY; + } + + if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { + MSG_ERR(("copy_from_user() in ump_ioctl_switch_hw_usage()\n")); + return -EFAULT; + } + + user_interaction.ctx = (void *) session_data; + + _ump_ukk_unlock(&user_interaction); + + user_interaction.ctx = NULL; + +#if 0 /* No data to copy back */ + if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { + MSG_ERR(("copy_to_user() failed in ump_ioctl_switch_hw_usage()\n")); + return -EFAULT; + } +#endif + + return 0; /* success */ +} diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h new file mode 100755 index 000000000000..5f8fc683c8f5 --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file ump_ukk_wrappers.h + * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls + */ + +#ifndef __UMP_UKK_WRAPPERS_H__ +#define __UMP_UKK_WRAPPERS_H__ + +#include +#include "ump_kernel_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + + + +int ump_get_api_version_wrapper(u32 __user *argument, struct ump_session_data *session_data); +int ump_release_wrapper(u32 __user *argument, struct ump_session_data *session_data); +int ump_size_get_wrapper(u32 __user *argument, struct ump_session_data *session_data); +int ump_msync_wrapper(u32 __user *argument, struct ump_session_data *session_data); +int ump_cache_operations_control_wrapper(u32 __user *argument, struct ump_session_data *session_data); +int ump_switch_hw_usage_wrapper(u32 __user *argument, struct ump_session_data *session_data); +int ump_lock_wrapper(u32 __user *argument, struct ump_session_data *session_data); +int ump_unlock_wrapper(u32 __user *argument, struct ump_session_data *session_data); + + + + +#ifdef __cplusplus +} +#endif + + + +#endif /* __UMP_UKK_WRAPPERS_H__ */ diff --git a/drivers/gpu/arm/mali400/ump/readme.txt b/drivers/gpu/arm/mali400/ump/readme.txt new file mode 100755 index 000000000000..c238cf0f2b1f --- /dev/null +++ b/drivers/gpu/arm/mali400/ump/readme.txt @@ -0,0 +1,28 @@ +Building the UMP Device Driver for Linux +---------------------------------------- + +Build the UMP Device Driver for Linux by running the following make command: + +KDIR= CONFIG= BUILD= make + +where + kdir_path: Path to your Linux Kernel directory + your_config: Name of the sub-folder to find the required config.h file + ("arch-" will be prepended) + build_option: debug or release. Debug is default. + +The config.h contains following configuration parameters: + +ARCH_UMP_BACKEND_DEFAULT + 0 specifies the dedicated memory allocator. + 1 specifies the OS memory allocator. +ARCH_UMP_MEMORY_ADDRESS_DEFAULT + This is only required for the dedicated memory allocator, and specifies + the physical start address of the memory block reserved for UMP. +ARCH_UMP_MEMORY_SIZE_DEFAULT + This specified the size of the memory block reserved for UMP, or the + maximum limit for allocations from the OS. + +The result will be a ump.ko file, which can be loaded into the Linux kernel +by using the insmod command. The driver can also be built as a part of the +kernel itself. diff --git a/drivers/gpu/arm/mali400/umplock/Makefile b/drivers/gpu/arm/mali400/umplock/Makefile new file mode 100755 index 000000000000..e5549a33f91d --- /dev/null +++ b/drivers/gpu/arm/mali400/umplock/Makefile @@ -0,0 +1,69 @@ +# +# Copyright (C) 2012, 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the GNU General Public License version 2 +# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained from Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +# default to building for the host +ARCH ?= $(shell uname -m) + +# linux build system integration + +ifneq ($(KERNELRELEASE),) +# Inside the kernel build system + +EXTRA_CFLAGS += -I$(KBUILD_EXTMOD) + +SRC = umplock_driver.c + +MODULE:=umplock.ko + +obj-m := $(MODULE:.ko=.o) +$(MODULE:.ko=-y) := $(SRC:.c=.o) + +$(MODULE:.ko=-objs) := $(SRC:.c=.o) + +else +# Outside the kernel build system +# +# + +# Get any user defined KDIR- or maybe even a hardcoded KDIR +-include KDIR_CONFIGURATION + +# Define host system directory +KDIR-$(shell uname -m):=/lib/modules/$(shell uname -r)/build + +ifeq ($(ARCH), arm) + # when compiling for ARM we're cross compiling + export CROSS_COMPILE ?= arm-none-linux-gnueabi- + CONFIG ?= arm +else + # Compiling for the host + CONFIG ?= $(shell uname -m) +endif + +# default cpu to select +CPU ?= $(shell uname -m) + +# look up KDIR based om CPU selection +KDIR ?= $(KDIR-$(CPU)) + +ifeq ($(KDIR),) +$(error No KDIR found for platform $(CPU)) +endif + +all: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) + +kernelrelease: + $(MAKE) -C $(KDIR) kernelrelease + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean + +endif diff --git a/drivers/gpu/arm/mali400/umplock/umplock_driver.c b/drivers/gpu/arm/mali400/umplock/umplock_driver.c new file mode 100755 index 000000000000..173f4d9bb5c7 --- /dev/null +++ b/drivers/gpu/arm/mali400/umplock/umplock_driver.c @@ -0,0 +1,618 @@ +/* + * Copyright (C) 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "umplock_ioctl.h" +#include + +#define MAX_ITEMS 1024 +#define MAX_PIDS 128 + +typedef struct lock_cmd_priv { + uint32_t msg[128]; /*ioctl args*/ + u32 pid; /*process id*/ +} _lock_cmd_priv; + +typedef struct lock_ref { + int ref_count; + u32 pid; + u32 down_count; +} _lock_ref; + +typedef struct umplock_item { + u32 secure_id; + u32 id_ref_count; + u32 owner; + _lock_access_usage usage; + _lock_ref references[MAX_PIDS]; + struct semaphore item_lock; +} umplock_item; + +typedef struct umplock_device_private { + struct mutex item_list_lock; + atomic_t sessions; + umplock_item items[MAX_ITEMS]; + u32 pids[MAX_PIDS]; +} umplock_device_private; + +struct umplock_device { + struct cdev cdev; + struct class *umplock_class; +}; + +static struct umplock_device umplock_device; +static umplock_device_private device; +static dev_t umplock_dev; +static char umplock_dev_name[] = "umplock"; + +int umplock_debug_level = 0; +module_param(umplock_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */ +MODULE_PARM_DESC(umplock_debug_level, "set umplock_debug_level to print debug messages"); + +#define PDEBUG(level, fmt, args...) do { if ((level) <= umplock_debug_level) printk(KERN_DEBUG "umplock: " fmt, ##args); } while (0) +#define PERROR(fmt, args...) do { printk(KERN_ERR "umplock: " fmt, ##args); } while (0) + +int umplock_find_item(u32 secure_id) +{ + int i; + for (i = 0; i < MAX_ITEMS; i++) { + if (device.items[i].secure_id == secure_id) { + return i; + } + } + + return -1; +} + +static int umplock_find_item_by_pid(_lock_cmd_priv *lock_cmd, int *item_slot, int *ref_slot) +{ + _lock_item_s *lock_item; + int i, j; + + lock_item = (_lock_item_s *)&lock_cmd->msg; + + i = umplock_find_item(lock_item->secure_id); + + if (i < 0) { + return -1; + } + + for (j = 0; j < MAX_PIDS; j++) { + if (device.items[i].references[j].pid == lock_cmd->pid) { + *item_slot = i; + *ref_slot = j; + return 0; + } + } + return -1 ; +} + +static int umplock_find_client_valid(u32 pid) +{ + int i; + + if (pid == 0) { + return -1; + } + + for (i = 0; i < MAX_PIDS; i++) { + if (device.pids[i] == pid) { + return i; + } + } + + return -1; +} + +static int do_umplock_create_locked(_lock_cmd_priv *lock_cmd) +{ + int i_index, ref_index; + int ret; + _lock_item_s *lock_item = (_lock_item_s *)&lock_cmd->msg; + + i_index = ref_index = -1; + + ret = umplock_find_client_valid(lock_cmd->pid); + if (ret < 0) { + /*lock request from an invalid client pid, do nothing*/ + return -EINVAL; + } + + ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); + if (ret >= 0) { + } else if ((i_index = umplock_find_item(lock_item->secure_id)) >= 0) { + for (ref_index = 0; ref_index < MAX_PIDS; ref_index++) { + if (device.items[i_index].references[ref_index].pid == 0) { + break; + } + } + if (ref_index < MAX_PIDS) { + device.items[i_index].references[ref_index].pid = lock_cmd->pid; + device.items[i_index].references[ref_index].ref_count = 0; + device.items[i_index].references[ref_index].down_count = 0; + } else { + PERROR("whoops, item ran out of available reference slots\n"); + return -EINVAL; + + } + } else { + i_index = umplock_find_item(0); + + if (i_index >= 0) { + device.items[i_index].secure_id = lock_item->secure_id; + device.items[i_index].id_ref_count = 0; + device.items[i_index].usage = lock_item->usage; + device.items[i_index].references[0].pid = lock_cmd->pid; + device.items[i_index].references[0].ref_count = 0; + device.items[i_index].references[0].down_count = 0; + sema_init(&device.items[i_index].item_lock, 1); + } else { + PERROR("whoops, ran out of available slots\n"); + return -EINVAL; + } + } + + return 0; +} +/** IOCTLs **/ + +static int do_umplock_create(_lock_cmd_priv *lock_cmd) +{ + return 0; +} + +static int do_umplock_process(_lock_cmd_priv *lock_cmd) +{ + int ret, i_index, ref_index; + _lock_item_s *lock_item = (_lock_item_s *)&lock_cmd->msg; + + mutex_lock(&device.item_list_lock); + + if (0 == lock_item->secure_id) { + PERROR("IOCTL_UMPLOCK_PROCESS called with secure_id is 0, pid: %d\n", lock_cmd->pid); + mutex_unlock(&device.item_list_lock); + return -EINVAL; + } + + ret = do_umplock_create_locked(lock_cmd); + if (ret < 0) { + mutex_unlock(&device.item_list_lock); + return -EINVAL; + } + + ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); + if (ret < 0) { + /*fail to find a item*/ + PERROR("IOCTL_UMPLOCK_PROCESS called with invalid parameter, pid: %d\n", lock_cmd->pid); + mutex_unlock(&device.item_list_lock); + return -EINVAL; + } + device.items[i_index].references[ref_index].ref_count++; + device.items[i_index].id_ref_count++; + PDEBUG(1, "try to lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); + + if (lock_cmd->pid == device.items[i_index].owner) { + PDEBUG(1, "already own the lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); + mutex_unlock(&device.item_list_lock); + return 0; + } + + device.items[i_index].references[ref_index].down_count++; + mutex_unlock(&device.item_list_lock); + if (down_interruptible(&device.items[i_index].item_lock)) { + /*wait up without hold the umplock. restore previous state and return*/ + mutex_lock(&device.item_list_lock); + device.items[i_index].references[ref_index].ref_count--; + device.items[i_index].id_ref_count--; + device.items[i_index].references[ref_index].down_count--; + if (0 == device.items[i_index].references[ref_index].ref_count) { + device.items[i_index].references[ref_index].pid = 0; + if (0 == device.items[i_index].id_ref_count) { + PDEBUG(1, "release item, pid: %d, secure_id: 0x%x\n", lock_cmd->pid, lock_item->secure_id); + device.items[i_index].secure_id = 0; + } + } + + PERROR("failed lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); + + mutex_unlock(&device.item_list_lock); + return -ERESTARTSYS; + } + + mutex_lock(&device.item_list_lock); + PDEBUG(1, "got lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); + device.items[i_index].owner = lock_cmd->pid; + mutex_unlock(&device.item_list_lock); + + return 0; +} + +static int do_umplock_release(_lock_cmd_priv *lock_cmd) +{ + int ret, i_index, ref_index, call_up; + _lock_item_s *lock_item = (_lock_item_s *)&lock_cmd->msg; + + mutex_lock(&device.item_list_lock); + + if (0 == lock_item->secure_id) { + PERROR("IOCTL_UMPLOCK_RELEASE called with secure_id is 0, pid: %d\n", lock_cmd->pid); + mutex_unlock(&device.item_list_lock); + return -EINVAL; + } + + ret = umplock_find_client_valid(lock_cmd->pid); + if (ret < 0) { + /*lock request from an invalid client pid, do nothing*/ + mutex_unlock(&device.item_list_lock); + return -EPERM; + } + + i_index = ref_index = -1; + + ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); + if (ret < 0) { + /*fail to find item*/ + PERROR("IOCTL_UMPLOCK_RELEASE called with invalid parameter pid: %d, secid: 0x%x\n", lock_cmd->pid, lock_item->secure_id); + mutex_unlock(&device.item_list_lock); + return -EINVAL; + } + + /* if the lock is not owned by this process */ + if (lock_cmd->pid != device.items[i_index].owner) { + mutex_unlock(&device.item_list_lock); + return -EPERM; + } + + /* if the ref_count is 0, that means nothing to unlock, just return */ + if (0 == device.items[i_index].references[ref_index].ref_count) { + mutex_unlock(&device.item_list_lock); + return 0; + } + + device.items[i_index].references[ref_index].ref_count--; + device.items[i_index].id_ref_count--; + PDEBUG(1, "unlock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); + + call_up = 0; + if (device.items[i_index].references[ref_index].down_count > 1) { + call_up = 1; + device.items[i_index].references[ref_index].down_count--; + } + if (0 == device.items[i_index].references[ref_index].ref_count) { + device.items[i_index].references[ref_index].pid = 0; + if (0 == device.items[i_index].id_ref_count) { + PDEBUG(1, "release item, pid: %d, secure_id: 0x%x\n", lock_cmd->pid, lock_item->secure_id); + device.items[i_index].secure_id = 0; + } + device.items[i_index].owner = 0; + call_up = 1; + } + if (call_up) { + PDEBUG(1, "call up, pid: %d, secure_id: 0x%x\n", lock_cmd->pid, lock_item->secure_id); + up(&device.items[i_index].item_lock); + } + mutex_unlock(&device.item_list_lock); + + return 0; +} + +static int do_umplock_zap(void) +{ + int i; + + PDEBUG(1, "ZAP ALL ENTRIES!\n"); + + mutex_lock(&device.item_list_lock); + + for (i = 0; i < MAX_ITEMS; i++) { + device.items[i].secure_id = 0; + memset(&device.items[i].references, 0, sizeof(_lock_ref) * MAX_PIDS); + sema_init(&device.items[i].item_lock, 1); + } + + for (i = 0; i < MAX_PIDS; i++) { + device.pids[i] = 0; + } + mutex_unlock(&device.item_list_lock); + + return 0; +} + +static int do_umplock_dump(void) +{ + int i, j; + + mutex_lock(&device.item_list_lock); + PERROR("dump all the items begin\n"); + for (i = 0; i < MAX_ITEMS; i++) { + for (j = 0; j < MAX_PIDS; j++) { + if (device.items[i].secure_id != 0 && device.items[i].references[j].pid != 0) { + PERROR("item[%d]->secure_id=0x%x, owner=%d\t reference[%d].ref_count=%d.pid=%d\n", + i, + device.items[i].secure_id, + device.items[i].owner, + j, + device.items[i].references[j].ref_count, + device.items[i].references[j].pid); + } + } + } + PERROR("dump all the items end\n"); + mutex_unlock(&device.item_list_lock); + + return 0; +} + +int do_umplock_client_add(_lock_cmd_priv *lock_cmd) +{ + int i; + mutex_lock(&device.item_list_lock); + for (i = 0; i < MAX_PIDS; i++) { + if (device.pids[i] == lock_cmd->pid) { + mutex_unlock(&device.item_list_lock); + return 0; + } + } + for (i = 0; i < MAX_PIDS; i++) { + if (device.pids[i] == 0) { + device.pids[i] = lock_cmd->pid; + break; + } + } + mutex_unlock(&device.item_list_lock); + if (i == MAX_PIDS) { + PERROR("Oops, Run out of client slots\n "); + return -EINVAL; + } + return 0; +} + +int do_umplock_client_delete(_lock_cmd_priv *lock_cmd) +{ + int p_index = -1, i_index = -1, ref_index = -1; + int ret; + _lock_item_s *lock_item; + lock_item = (_lock_item_s *)&lock_cmd->msg; + + mutex_lock(&device.item_list_lock); + p_index = umplock_find_client_valid(lock_cmd->pid); + /*lock item pid is not valid.*/ + if (p_index < 0) { + mutex_unlock(&device.item_list_lock); + return 0; + } + + /*walk through umplock item list and release reference attached to this client*/ + for (i_index = 0; i_index < MAX_ITEMS; i_index++) { + lock_item->secure_id = device.items[i_index].secure_id; + + /*find the item index and reference slot for the lock_item*/ + ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); + + if (ret < 0) { + /*client has no reference on this umplock item, skip*/ + continue; + } + while (device.items[i_index].references[ref_index].ref_count) { + /*release references on this client*/ + + PDEBUG(1, "delete client, pid: %d, ref_count: %d\n", lock_cmd->pid, device.items[i_index].references[ref_index].ref_count); + + mutex_unlock(&device.item_list_lock); + do_umplock_release(lock_cmd); + mutex_lock(&device.item_list_lock); + } + } + + /*remove the pid from umplock valid pid list*/ + device.pids[p_index] = 0; + mutex_unlock(&device.item_list_lock); + + return 0; +} + +static long umplock_driver_ioctl(struct file *f, unsigned int cmd, unsigned long arg) +{ + int ret; + uint32_t size = _IOC_SIZE(cmd); + _lock_cmd_priv lock_cmd ; + + if (_IOC_TYPE(cmd) != LOCK_IOCTL_GROUP) { + return -ENOTTY; + } + + if (_IOC_NR(cmd) >= LOCK_IOCTL_MAX_CMDS) { + return -ENOTTY; + } + + switch (cmd) { + case LOCK_IOCTL_CREATE: + if (size != sizeof(_lock_item_s)) { + return -ENOTTY; + } + + if (copy_from_user(&lock_cmd.msg, (void __user *)arg, size)) { + return -EFAULT; + } + lock_cmd.pid = (u32)current->tgid; + ret = do_umplock_create(&lock_cmd); + if (ret) { + return ret; + } + return 0; + + case LOCK_IOCTL_PROCESS: + if (size != sizeof(_lock_item_s)) { + return -ENOTTY; + } + + if (copy_from_user(&lock_cmd.msg, (void __user *)arg, size)) { + return -EFAULT; + } + lock_cmd.pid = (u32)current->tgid; + return do_umplock_process(&lock_cmd); + + case LOCK_IOCTL_RELEASE: + if (size != sizeof(_lock_item_s)) { + return -ENOTTY; + } + + if (copy_from_user(&lock_cmd.msg, (void __user *)arg, size)) { + return -EFAULT; + } + lock_cmd.pid = (u32)current->tgid; + ret = do_umplock_release(&lock_cmd); + if (ret) { + return ret; + } + return 0; + + case LOCK_IOCTL_ZAP: + do_umplock_zap(); + return 0; + + case LOCK_IOCTL_DUMP: + do_umplock_dump(); + return 0; + } + + return -ENOIOCTLCMD; +} + +static int umplock_driver_open(struct inode *inode, struct file *filp) +{ + _lock_cmd_priv lock_cmd; + + atomic_inc(&device.sessions); + PDEBUG(1, "OPEN SESSION (%i references)\n", atomic_read(&device.sessions)); + + lock_cmd.pid = (u32)current->tgid; + do_umplock_client_add(&lock_cmd); + + return 0; +} + +static int umplock_driver_release(struct inode *inode, struct file *filp) +{ + int sessions = 0; + _lock_cmd_priv lock_cmd; + + lock_cmd.pid = (u32)current->tgid; + do_umplock_client_delete(&lock_cmd); + + mutex_lock(&device.item_list_lock); + atomic_dec(&device.sessions); + sessions = atomic_read(&device.sessions); + PDEBUG(1, "CLOSE SESSION (%i references)\n", sessions); + mutex_unlock(&device.item_list_lock); + if (sessions == 0) { + do_umplock_zap(); + } + + return 0; +} + +static struct file_operations umplock_fops = { + .owner = THIS_MODULE, + .open = umplock_driver_open, + .release = umplock_driver_release, + .unlocked_ioctl = umplock_driver_ioctl, +}; + +int umplock_device_initialize(void) +{ + int err; + + err = alloc_chrdev_region(&umplock_dev, 0, 1, umplock_dev_name); + + if (0 == err) { + memset(&umplock_device, 0, sizeof(umplock_device)); + cdev_init(&umplock_device.cdev, &umplock_fops); + umplock_device.cdev.owner = THIS_MODULE; + umplock_device.cdev.ops = &umplock_fops; + + err = cdev_add(&umplock_device.cdev, umplock_dev, 1); + if (0 == err) { + umplock_device.umplock_class = class_create(THIS_MODULE, umplock_dev_name); + if (IS_ERR(umplock_device.umplock_class)) { + err = PTR_ERR(umplock_device.umplock_class); + } else { + struct device *mdev; + mdev = device_create(umplock_device.umplock_class, NULL, umplock_dev, NULL, umplock_dev_name); + if (!IS_ERR(mdev)) { + return 0; /* all ok */ + } + + err = PTR_ERR(mdev); + class_destroy(umplock_device.umplock_class); + } + cdev_del(&umplock_device.cdev); + } + + unregister_chrdev_region(umplock_dev, 1); + } else { + PERROR("alloc chardev region failed\n"); + } + + return err; +} + +void umplock_device_terminate(void) +{ + device_destroy(umplock_device.umplock_class, umplock_dev); + class_destroy(umplock_device.umplock_class); + + cdev_del(&umplock_device.cdev); + unregister_chrdev_region(umplock_dev, 1); +} + +static int __init umplock_initialize_module(void) +{ + PDEBUG(1, "Inserting UMP lock device driver. Compiled: %s, time: %s\n", __DATE__, __TIME__); + + mutex_init(&device.item_list_lock); + if (umplock_device_initialize() != 0) { + PERROR("UMP lock device driver init failed\n"); + return -ENOTTY; + } + memset(&device.items, 0, sizeof(umplock_item) * MAX_ITEMS); + memset(&device.pids, 0, sizeof(u32) * MAX_PIDS); + atomic_set(&device.sessions, 0); + + PDEBUG(1, "UMP lock device driver loaded\n"); + + return 0; +} + +static void __exit umplock_cleanup_module(void) +{ + PDEBUG(1, "unloading UMP lock module\n"); + + memset(&device.items, 0, sizeof(umplock_item) * MAX_ITEMS); + memset(&device.pids, 0, sizeof(u32) * MAX_PIDS); + umplock_device_terminate(); + mutex_destroy(&device.item_list_lock); + + PDEBUG(1, "UMP lock module unloaded\n"); +} + +module_init(umplock_initialize_module); +module_exit(umplock_cleanup_module); + + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("ARM Ltd."); +MODULE_DESCRIPTION("ARM UMP locker"); diff --git a/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h b/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h new file mode 100755 index 000000000000..8afdaad7000f --- /dev/null +++ b/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2012-2013, 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained from Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __UMPLOCK_IOCTL_H__ +#define __UMPLOCK_IOCTL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#ifndef __user +#define __user +#endif + + +/** + * @file umplock_ioctl.h + * This file describes the interface needed to use the Linux device driver. + * The interface is used by the userpace Mali DDK. + */ + +typedef enum { + _LOCK_ACCESS_RENDERABLE = 1, + _LOCK_ACCESS_TEXTURE, + _LOCK_ACCESS_CPU_WRITE, + _LOCK_ACCESS_CPU_READ, +} _lock_access_usage; + +typedef struct _lock_item_s { + unsigned int secure_id; + _lock_access_usage usage; +} _lock_item_s; + + +#define LOCK_IOCTL_GROUP 0x91 + +#define _LOCK_IOCTL_CREATE_CMD 0 /* create kernel lock item */ +#define _LOCK_IOCTL_PROCESS_CMD 1 /* process kernel lock item */ +#define _LOCK_IOCTL_RELEASE_CMD 2 /* release kernel lock item */ +#define _LOCK_IOCTL_ZAP_CMD 3 /* clean up all kernel lock items */ +#define _LOCK_IOCTL_DUMP_CMD 4 /* dump all the items */ + +#define LOCK_IOCTL_MAX_CMDS 5 + +#define LOCK_IOCTL_CREATE _IOW( LOCK_IOCTL_GROUP, _LOCK_IOCTL_CREATE_CMD, _lock_item_s ) +#define LOCK_IOCTL_PROCESS _IOW( LOCK_IOCTL_GROUP, _LOCK_IOCTL_PROCESS_CMD, _lock_item_s ) +#define LOCK_IOCTL_RELEASE _IOW( LOCK_IOCTL_GROUP, _LOCK_IOCTL_RELEASE_CMD, _lock_item_s ) +#define LOCK_IOCTL_ZAP _IO ( LOCK_IOCTL_GROUP, _LOCK_IOCTL_ZAP_CMD ) +#define LOCK_IOCTL_DUMP _IO ( LOCK_IOCTL_GROUP, _LOCK_IOCTL_DUMP_CMD ) + +#ifdef __cplusplus +} +#endif + +#endif /* __UMPLOCK_IOCTL_H__ */ + diff --git a/drivers/gpu/arm/midgard/Kbuild b/drivers/gpu/arm/midgard/Kbuild new file mode 100755 index 000000000000..b2c2bbcda668 --- /dev/null +++ b/drivers/gpu/arm/midgard/Kbuild @@ -0,0 +1,221 @@ +# +# (C) COPYRIGHT 2012-2016, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + +KBUILD_CFLAGS += -include rename.h + +# Driver version string which is returned to userspace via an ioctl +MALI_RELEASE_NAME ?= "r18p0-01rel0" + +# Paths required for build + +# make $(src) as absolute path if it isn't already, by prefixing $(srctree) +src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src)) +KBASE_PATH = $(src) +KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy +UMP_PATH = $(src)/../../../base + +ifeq ($(CONFIG_MALI_ERROR_INJECTION),y) +MALI_ERROR_INJECT_ON = 1 +endif + +# Set up defaults if not defined by build system +MALI_CUSTOMER_RELEASE ?= 1 +MALI_UNIT_TEST ?= 0 +MALI_KERNEL_TEST_API ?= 0 +MALI_ERROR_INJECT_ON ?= 0 +MALI_MOCK_TEST ?= 0 +MALI_COVERAGE ?= 0 +MALI_INSTRUMENTATION_LEVEL ?= 0 +# This workaround is for what seems to be a compiler bug we observed in +# GCC 4.7 on AOSP 4.3. The bug caused an intermittent failure compiling +# the "_Pragma" syntax, where an error message is returned: +# +# "internal compiler error: unspellable token PRAGMA" +# +# This regression has thus far only been seen on the GCC 4.7 compiler bundled +# with AOSP 4.3.0. So this makefile, intended for in-tree kernel builds +# which are not known to be used with AOSP, is hardcoded to disable the +# workaround, i.e. set the define to 0. +MALI_GCC_WORKAROUND_MIDCOM_4598 ?= 0 + +# Set up our defines, which will be passed to gcc +DEFINES = \ + -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ + -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ + -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ + -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ + -DMALI_MOCK_TEST=$(MALI_MOCK_TEST) \ + -DMALI_COVERAGE=$(MALI_COVERAGE) \ + -DMALI_INSTRUMENTATION_LEVEL=$(MALI_INSTRUMENTATION_LEVEL) \ + -DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \ + -DMALI_GCC_WORKAROUND_MIDCOM_4598=$(MALI_GCC_WORKAROUND_MIDCOM_4598) + +ifeq ($(KBUILD_EXTMOD),) +# in-tree +DEFINES +=-DMALI_KBASE_THIRDPARTY_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME) +else +# out-of-tree +DEFINES +=-DMALI_KBASE_THIRDPARTY_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME) +endif + +DEFINES += -I$(srctree)/drivers/staging/android + +# Use our defines when compiling +ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux +subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(OSK_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux + +SRC := \ + mali_kbase_device.c \ + mali_kbase_cache_policy.c \ + mali_kbase_mem.c \ + mali_kbase_mmu.c \ + mali_kbase_ctx_sched.c \ + mali_kbase_jd.c \ + mali_kbase_jd_debugfs.c \ + mali_kbase_jm.c \ + mali_kbase_gpuprops.c \ + mali_kbase_js.c \ + mali_kbase_js_ctx_attr.c \ + mali_kbase_event.c \ + mali_kbase_context.c \ + mali_kbase_pm.c \ + mali_kbase_config.c \ + mali_kbase_vinstr.c \ + mali_kbase_softjobs.c \ + mali_kbase_10969_workaround.c \ + mali_kbase_hw.c \ + mali_kbase_utility.c \ + mali_kbase_debug.c \ + mali_kbase_trace_timeline.c \ + mali_kbase_gpu_memory_debugfs.c \ + mali_kbase_mem_linux.c \ + mali_kbase_core_linux.c \ + mali_kbase_replay.c \ + mali_kbase_mem_profile_debugfs.c \ + mali_kbase_mmu_mode_lpae.c \ + mali_kbase_mmu_mode_aarch64.c \ + mali_kbase_disjoint_events.c \ + mali_kbase_gator_api.c \ + mali_kbase_debug_mem_view.c \ + mali_kbase_debug_job_fault.c \ + mali_kbase_smc.c \ + mali_kbase_mem_pool.c \ + mali_kbase_mem_pool_debugfs.c \ + mali_kbase_tlstream.c \ + mali_kbase_strings.c \ + mali_kbase_as_fault_debugfs.c \ + mali_kbase_regs_history_debugfs.c + + + + +ifeq ($(MALI_UNIT_TEST),1) + SRC += mali_kbase_tlstream_test.c +endif + +ifeq ($(MALI_CUSTOMER_RELEASE),0) + SRC += mali_kbase_regs_dump_debugfs.c +endif + + +ccflags-y += -I$(KBASE_PATH) + +ifeq ($(CONFIG_MALI_PLATFORM_FAKE),y) + SRC += mali_kbase_platform_fake.c + + ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS),y) + SRC += platform/vexpress/mali_kbase_config_vexpress.c \ + platform/vexpress/mali_kbase_cpu_vexpress.c + ccflags-y += -I$(src)/platform/vexpress + endif + + ifeq ($(CONFIG_MALI_PLATFORM_RTSM_VE),y) + SRC += platform/rtsm_ve/mali_kbase_config_vexpress.c + ccflags-y += -I$(src)/platform/rtsm_ve + endif + + ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS_1XV7_A57),y) + SRC += platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c + ccflags-y += -I$(src)/platform/vexpress_1xv7_a57 + endif + + ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS_6XVIRTEX7_10MHZ),y) + SRC += platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c \ + platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c + ccflags-y += -I$(src)/platform/vexpress_6xvirtex7_10mhz + endif +endif # CONFIG_MALI_PLATFORM_FAKE=y + +# Tell the Linux build system from which .o file to create the kernel module +obj-$(CONFIG_MALI_MIDGARD) += midgard_kbase.o + +# Tell the Linux build system to enable building of our .c files +midgard_kbase-y := $(SRC:.c=.o) + +ifeq ($(CONFIG_MALI_PLATFORM_THIRDPARTY),y) + # Kconfig passes in the name with quotes for in-tree builds - remove them. + platform_name := $(shell echo $(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME)) + MALI_PLATFORM_THIRDPARTY_DIR := platform/$(platform_name) + ccflags-y += -I$(src)/$(MALI_PLATFORM_THIRDPARTY_DIR) + include $(src)/$(MALI_PLATFORM_THIRDPARTY_DIR)/Kbuild +endif + +ifeq ($(CONFIG_MALI_DEVFREQ),y) + ifeq ($(CONFIG_DEVFREQ_THERMAL),y) + include $(src)/ipa/Kbuild + endif +endif + +midgard_kbase-$(CONFIG_MALI_DMA_FENCE) += \ + mali_kbase_dma_fence.o \ + mali_kbase_fence.o +midgard_kbase-$(CONFIG_SYNC) += \ + mali_kbase_sync_android.o \ + mali_kbase_sync_common.o +midgard_kbase-$(CONFIG_SYNC_FILE) += \ + mali_kbase_sync_file.o \ + mali_kbase_sync_common.o \ + mali_kbase_fence.o + +MALI_BACKEND_PATH ?= backend +CONFIG_MALI_BACKEND ?= gpu +CONFIG_MALI_BACKEND_REAL ?= $(CONFIG_MALI_BACKEND) + +ifeq ($(MALI_MOCK_TEST),1) +ifeq ($(CONFIG_MALI_BACKEND_REAL),gpu) +# Test functionality +midgard_kbase-y += tests/internal/src/mock/mali_kbase_pm_driver_mock.o +endif +endif + +include $(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL)/Kbuild +midgard_kbase-y += $(BACKEND:.c=.o) + + +ccflags-y += -I$(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL) +subdir-ccflags-y += -I$(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL) + +# Default to devicetree platform if neither a fake platform or a thirdparty +# platform is configured. +ifeq ($(CONFIG_MALI_PLATFORM_THIRDPARTY)$(CONFIG_MALI_PLATFORM_FAKE),) +CONFIG_MALI_PLATFORM_DEVICETREE := y +endif + +midgard_kbase-$(CONFIG_MALI_PLATFORM_DEVICETREE) += \ + platform/devicetree/mali_kbase_runtime_pm.o \ + platform/devicetree/mali_kbase_config_devicetree.o +ccflags-$(CONFIG_MALI_PLATFORM_DEVICETREE) += -I$(src)/platform/devicetree + +# For kutf and mali_kutf_irq_latency_test +obj-$(CONFIG_MALI_KUTF) += tests/ diff --git a/drivers/gpu/arm/midgard/Kconfig b/drivers/gpu/arm/midgard/Kconfig new file mode 100755 index 000000000000..1b28bb73ad59 --- /dev/null +++ b/drivers/gpu/arm/midgard/Kconfig @@ -0,0 +1,248 @@ +# +# (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + +menuconfig MALI_MIDGARD + tristate "Mali Midgard series support" + select GPU_TRACEPOINTS if ANDROID + default n + help + Enable this option to build support for a ARM Mali Midgard GPU. + + To compile this driver as a module, choose M here: + this will generate a single module, called mali_kbase. + +config MALI_GATOR_SUPPORT + bool "Streamline support via Gator" + depends on MALI_MIDGARD + default n + help + Adds diagnostic support for use with the ARM Streamline Performance Analyzer. + You will need the Gator device driver already loaded before loading this driver when enabling + Streamline debug support. + This is a legacy interface required by older versions of Streamline. + +config MALI_MIDGARD_DVFS + bool "Enable legacy DVFS" + depends on MALI_MIDGARD && !MALI_DEVFREQ && !MALI_PLATFORM_DEVICETREE + default n + help + Choose this option to enable legacy DVFS in the Mali Midgard DDK. + +config MALI_MIDGARD_ENABLE_TRACE + bool "Enable kbase tracing" + depends on MALI_MIDGARD + default n + help + Enables tracing in kbase. Trace log available through + the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled + +config MALI_DEVFREQ + bool "devfreq support for Mali" + depends on MALI_MIDGARD && PM_DEVFREQ + help + Support devfreq for Mali. + + Using the devfreq framework and, by default, the simpleondemand + governor, the frequency of Mali will be dynamically selected from the + available OPPs. + +config MALI_DMA_FENCE + bool "DMA_BUF fence support for Mali" + depends on MALI_MIDGARD && !KDS + default n + help + Support DMA_BUF fences for Mali. + + This option should only be enabled if KDS is not present and + the Linux Kernel has built in support for DMA_BUF fences. + +# MALI_EXPERT configuration options + +menuconfig MALI_EXPERT + depends on MALI_MIDGARD + bool "Enable Expert Settings" + default n + help + Enabling this option and modifying the default settings may produce a driver with performance or + other limitations. + +config MALI_CORESTACK + bool "Support controlling power to the GPU core stack" + depends on MALI_MIDGARD && MALI_EXPERT + default n + help + Enabling this feature on supported GPUs will let the driver powering + on/off the GPU core stack independently without involving the Power + Domain Controller. This should only be enabled on platforms which + integration of the PDC to the Mali GPU is known to be problematic. + This feature is currently only supported on t-Six and t-HEx GPUs. + + If unsure, say N. + +config MALI_PRFCNT_SET_SECONDARY + bool "Use secondary set of performance counters" + depends on MALI_MIDGARD && MALI_EXPERT + default n + help + Select this option to use secondary set of performance counters. Kernel + features that depend on an access to the primary set of counters may + become unavailable. Enabling this option will prevent power management + from working optimally and may cause instrumentation tools to return + bogus results. + + If unsure, say N. + +config MALI_PLATFORM_FAKE + bool "Enable fake platform device support" + depends on MALI_MIDGARD && MALI_EXPERT + default n + help + When you start to work with the Mali Midgard series device driver the platform-specific code of + the Linux kernel for your platform may not be complete. In this situation the kernel device driver + supports creating the platform device outside of the Linux platform-specific code. + Enable this option if would like to use a platform device configuration from within the device driver. + +choice + prompt "Platform configuration" + depends on MALI_MIDGARD && MALI_EXPERT + default MALI_PLATFORM_DEVICETREE + help + Select the SOC platform that contains a Mali Midgard GPU + +config MALI_PLATFORM_DEVICETREE + bool "Device Tree platform" + depends on OF + help + Select this option to use Device Tree with the Mali driver. + + When using this option the Mali driver will get the details of the + GPU hardware from the Device Tree. This means that the same driver + binary can run on multiple platforms as long as all the GPU hardware + details are described in the device tree. + + Device Tree is the recommended method for the Mali driver platform + integration. + +config MALI_PLATFORM_VEXPRESS + depends on ARCH_VEXPRESS && (ARCH_VEXPRESS_CA9X4 || ARCH_VEXPRESS_CA15X4) + bool "Versatile Express" +config MALI_PLATFORM_VEXPRESS_VIRTEX7_40MHZ + depends on ARCH_VEXPRESS && (ARCH_VEXPRESS_CA9X4 || ARCH_VEXPRESS_CA15X4) + bool "Versatile Express w/Virtex7 @ 40Mhz" +config MALI_PLATFORM_GOLDFISH + depends on ARCH_GOLDFISH + bool "Android Goldfish virtual CPU" +config MALI_PLATFORM_PBX + depends on ARCH_REALVIEW && REALVIEW_EB_A9MP && MACH_REALVIEW_PBX + bool "Realview PBX-A9" +config MALI_PLATFORM_THIRDPARTY + bool "Third Party Platform" +endchoice + +config MALI_PLATFORM_THIRDPARTY_NAME + depends on MALI_MIDGARD && MALI_PLATFORM_THIRDPARTY && MALI_EXPERT + string "Third party platform name" + help + Enter the name of a third party platform that is supported. The third part configuration + file must be in midgard/config/tpip/mali_kbase_config_xxx.c where xxx is the name + specified here. + +config MALI_DEBUG + bool "Debug build" + depends on MALI_MIDGARD && MALI_EXPERT + default n + help + Select this option for increased checking and reporting of errors. + +config MALI_FENCE_DEBUG + bool "Debug sync fence usage" + depends on MALI_MIDGARD && MALI_EXPERT && (SYNC || SYNC_FILE) + default y if MALI_DEBUG + help + Select this option to enable additional checking and reporting on the + use of sync fences in the Mali driver. + + This will add a 3s timeout to all sync fence waits in the Mali + driver, so that when work for Mali has been waiting on a sync fence + for a long time a debug message will be printed, detailing what fence + is causing the block, and which dependent Mali atoms are blocked as a + result of this. + + The timeout can be changed at runtime through the js_soft_timeout + device attribute, where the timeout is specified in milliseconds. + +config MALI_NO_MALI + bool "No Mali" + depends on MALI_MIDGARD && MALI_EXPERT + default n + help + This can be used to test the driver in a simulated environment + whereby the hardware is not physically present. If the hardware is physically + present it will not be used. This can be used to test the majority of the + driver without needing actual hardware or for software benchmarking. + All calls to the simulated hardware will complete immediately as if the hardware + completed the task. + +config MALI_ERROR_INJECT + bool "Error injection" + depends on MALI_MIDGARD && MALI_EXPERT && MALI_NO_MALI + default n + help + Enables insertion of errors to test module failure and recovery mechanisms. + +config MALI_TRACE_TIMELINE + bool "Timeline tracing" + depends on MALI_MIDGARD && MALI_EXPERT + default n + help + Enables timeline tracing through the kernel tracepoint system. + +config MALI_SYSTEM_TRACE + bool "Enable system event tracing support" + depends on MALI_MIDGARD && MALI_EXPERT + default n + help + Choose this option to enable system trace events for each + kbase event. This is typically used for debugging but has + minimal overhead when not in use. Enable only if you know what + you are doing. + +config MALI_GPU_MMU_AARCH64 + bool "Use AArch64 page tables" + depends on MALI_MIDGARD && MALI_EXPERT + default n + help + Use AArch64 format page tables for the GPU instead of LPAE-style. + The two formats have the same functionality and performance but a + future GPU may deprecate or remove the legacy LPAE-style format. + + The LPAE-style format is supported on all Midgard and current Bifrost + GPUs. Enabling AArch64 format restricts the driver to only supporting + Bifrost GPUs. + + If in doubt, say N. + +config MALI_PWRSOFT_765 + bool "PWRSOFT-765 ticket" + depends on MALI_MIDGARD && MALI_EXPERT + default n + help + PWRSOFT-765 fixes devfreq cooling devices issues. However, they are + not merged in mainline kernel yet. So this define helps to guard those + parts of the code. + +source "drivers/gpu/arm/midgard/platform/Kconfig" +source "drivers/gpu/arm/midgard/tests/Kconfig" diff --git a/drivers/gpu/arm/midgard/Makefile b/drivers/gpu/arm/midgard/Makefile new file mode 100755 index 000000000000..9aa242c4f8c4 --- /dev/null +++ b/drivers/gpu/arm/midgard/Makefile @@ -0,0 +1,42 @@ +# +# (C) COPYRIGHT 2010-2016, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + +KDIR ?= /lib/modules/$(shell uname -r)/build + +BUSLOG_PATH_RELATIVE = $(CURDIR)/../../../.. +UMP_PATH_RELATIVE = $(CURDIR)/../../../base/ump +KBASE_PATH_RELATIVE = $(CURDIR) +KDS_PATH_RELATIVE = $(CURDIR)/../../../.. +EXTRA_SYMBOLS = $(UMP_PATH_RELATIVE)/src/Module.symvers + +ifeq ($(MALI_UNIT_TEST), 1) + EXTRA_SYMBOLS += $(KBASE_PATH_RELATIVE)/tests/internal/src/kernel_assert_module/linux/Module.symvers +endif + +ifeq ($(MALI_BUS_LOG), 1) +#Add bus logger symbols +EXTRA_SYMBOLS += $(BUSLOG_PATH_RELATIVE)/drivers/base/bus_logger/Module.symvers +endif + +# GPL driver supports KDS +EXTRA_SYMBOLS += $(KDS_PATH_RELATIVE)/drivers/base/kds/Module.symvers + +# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions +all: + $(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules + +clean: + $(MAKE) -C $(KDIR) M=$(CURDIR) clean diff --git a/drivers/gpu/arm/midgard/Makefile.kbase b/drivers/gpu/arm/midgard/Makefile.kbase new file mode 100755 index 000000000000..2bef9c25eaeb --- /dev/null +++ b/drivers/gpu/arm/midgard/Makefile.kbase @@ -0,0 +1,17 @@ +# +# (C) COPYRIGHT 2010 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +EXTRA_CFLAGS += -I$(ROOT) -I$(KBASE_PATH) -I$(OSK_PATH)/src/linux/include -I$(KBASE_PATH)/platform_$(PLATFORM) + diff --git a/drivers/gpu/arm/midgard/backend/gpu/Kbuild b/drivers/gpu/arm/midgard/backend/gpu/Kbuild new file mode 100755 index 000000000000..5f700e9b6b44 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/Kbuild @@ -0,0 +1,60 @@ +# +# (C) COPYRIGHT 2014,2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +BACKEND += \ + backend/gpu/mali_kbase_cache_policy_backend.c \ + backend/gpu/mali_kbase_device_hw.c \ + backend/gpu/mali_kbase_gpu.c \ + backend/gpu/mali_kbase_gpuprops_backend.c \ + backend/gpu/mali_kbase_debug_job_fault_backend.c \ + backend/gpu/mali_kbase_irq_linux.c \ + backend/gpu/mali_kbase_instr_backend.c \ + backend/gpu/mali_kbase_jm_as.c \ + backend/gpu/mali_kbase_jm_hw.c \ + backend/gpu/mali_kbase_jm_rb.c \ + backend/gpu/mali_kbase_js_affinity.c \ + backend/gpu/mali_kbase_js_backend.c \ + backend/gpu/mali_kbase_mmu_hw_direct.c \ + backend/gpu/mali_kbase_pm_backend.c \ + backend/gpu/mali_kbase_pm_driver.c \ + backend/gpu/mali_kbase_pm_metrics.c \ + backend/gpu/mali_kbase_pm_ca.c \ + backend/gpu/mali_kbase_pm_ca_fixed.c \ + backend/gpu/mali_kbase_pm_always_on.c \ + backend/gpu/mali_kbase_pm_coarse_demand.c \ + backend/gpu/mali_kbase_pm_demand.c \ + backend/gpu/mali_kbase_pm_policy.c \ + backend/gpu/mali_kbase_time.c + +ifeq ($(MALI_CUSTOMER_RELEASE),0) +BACKEND += \ + backend/gpu/mali_kbase_pm_ca_random.c \ + backend/gpu/mali_kbase_pm_demand_always_powered.c \ + backend/gpu/mali_kbase_pm_fast_start.c +endif + +ifeq ($(CONFIG_MALI_DEVFREQ),y) +BACKEND += \ + backend/gpu/mali_kbase_devfreq.c \ + backend/gpu/mali_kbase_pm_ca_devfreq.c +endif + +ifeq ($(CONFIG_MALI_NO_MALI),y) + # Dummy model + BACKEND += backend/gpu/mali_kbase_model_dummy.c + BACKEND += backend/gpu/mali_kbase_model_linux.c + # HW error simulation + BACKEND += backend/gpu/mali_kbase_model_error_generator.c +endif diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h new file mode 100755 index 000000000000..c8ae87eb84a2 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h @@ -0,0 +1,29 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Backend specific configuration + */ + +#ifndef _KBASE_BACKEND_CONFIG_H_ +#define _KBASE_BACKEND_CONFIG_H_ + +/* Enable GPU reset API */ +#define KBASE_GPU_RESET_EN 1 + +#endif /* _KBASE_BACKEND_CONFIG_H_ */ + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c new file mode 100755 index 000000000000..fef9a2cb743e --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c @@ -0,0 +1,29 @@ +/* + * + * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "backend/gpu/mali_kbase_cache_policy_backend.h" +#include + +void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, + u32 mode) +{ + kbdev->current_gpu_coherency_mode = mode; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) + kbase_reg_write(kbdev, COHERENCY_ENABLE, mode, NULL); +} + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h new file mode 100755 index 000000000000..fe9869109a82 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h @@ -0,0 +1,34 @@ +/* + * + * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +#ifndef _KBASE_CACHE_POLICY_BACKEND_H_ +#define _KBASE_CACHE_POLICY_BACKEND_H_ + +#include "mali_kbase.h" +#include "mali_base_kernel.h" + +/** + * kbase_cache_set_coherency_mode() - Sets the system coherency mode + * in the GPU. + * @kbdev: Device pointer + * @mode: Coherency mode. COHERENCY_ACE/ACE_LITE + */ +void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, + u32 mode); + +#endif /* _KBASE_CACHE_POLICY_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c new file mode 100755 index 000000000000..7851ea6466c7 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c @@ -0,0 +1,157 @@ +/* + * + * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include "mali_kbase_debug_job_fault.h" + +#ifdef CONFIG_DEBUG_FS + +/*GPU_CONTROL_REG(r)*/ +static int gpu_control_reg_snapshot[] = { + GPU_ID, + SHADER_READY_LO, + SHADER_READY_HI, + TILER_READY_LO, + TILER_READY_HI, + L2_READY_LO, + L2_READY_HI +}; + +/* JOB_CONTROL_REG(r) */ +static int job_control_reg_snapshot[] = { + JOB_IRQ_MASK, + JOB_IRQ_STATUS +}; + +/* JOB_SLOT_REG(n,r) */ +static int job_slot_reg_snapshot[] = { + JS_HEAD_LO, + JS_HEAD_HI, + JS_TAIL_LO, + JS_TAIL_HI, + JS_AFFINITY_LO, + JS_AFFINITY_HI, + JS_CONFIG, + JS_STATUS, + JS_HEAD_NEXT_LO, + JS_HEAD_NEXT_HI, + JS_AFFINITY_NEXT_LO, + JS_AFFINITY_NEXT_HI, + JS_CONFIG_NEXT +}; + +/*MMU_REG(r)*/ +static int mmu_reg_snapshot[] = { + MMU_IRQ_MASK, + MMU_IRQ_STATUS +}; + +/* MMU_AS_REG(n,r) */ +static int as_reg_snapshot[] = { + AS_TRANSTAB_LO, + AS_TRANSTAB_HI, + AS_MEMATTR_LO, + AS_MEMATTR_HI, + AS_FAULTSTATUS, + AS_FAULTADDRESS_LO, + AS_FAULTADDRESS_HI, + AS_STATUS +}; + +bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, + int reg_range) +{ + int i, j; + int offset = 0; + int slot_number; + int as_number; + + if (kctx->reg_dump == NULL) + return false; + + slot_number = kctx->kbdev->gpu_props.num_job_slots; + as_number = kctx->kbdev->gpu_props.num_address_spaces; + + /* get the GPU control registers*/ + for (i = 0; i < sizeof(gpu_control_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + GPU_CONTROL_REG(gpu_control_reg_snapshot[i]); + offset += 2; + } + + /* get the Job control registers*/ + for (i = 0; i < sizeof(job_control_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + JOB_CONTROL_REG(job_control_reg_snapshot[i]); + offset += 2; + } + + /* get the Job Slot registers*/ + for (j = 0; j < slot_number; j++) { + for (i = 0; i < sizeof(job_slot_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + JOB_SLOT_REG(j, job_slot_reg_snapshot[i]); + offset += 2; + } + } + + /* get the MMU registers*/ + for (i = 0; i < sizeof(mmu_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = MMU_REG(mmu_reg_snapshot[i]); + offset += 2; + } + + /* get the Address space registers*/ + for (j = 0; j < as_number; j++) { + for (i = 0; i < sizeof(as_reg_snapshot)/4; i++) { + kctx->reg_dump[offset] = + MMU_AS_REG(j, as_reg_snapshot[i]); + offset += 2; + } + } + + WARN_ON(offset >= (reg_range*2/4)); + + /* set the termination flag*/ + kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG; + kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG; + + dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n", + offset); + + return true; +} + +bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx) +{ + int offset = 0; + + if (kctx->reg_dump == NULL) + return false; + + while (kctx->reg_dump[offset] != REGISTER_DUMP_TERMINATION_FLAG) { + kctx->reg_dump[offset+1] = + kbase_reg_read(kctx->kbdev, + kctx->reg_dump[offset], NULL); + offset += 2; + } + return true; +} + + +#endif diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c new file mode 100755 index 000000000000..ab14bc2e2ae4 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c @@ -0,0 +1,458 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +#define ENABLE_DEBUG_LOG +#include "../../platform/rk/custom_log.h" + + +#include +#include +#include +#include + +#include +#include +#include +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif + +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) +#include +#else /* Linux >= 3.13 */ +/* In 3.13 the OPP include header file, types, and functions were all + * renamed. Use the old filename for the include, and define the new names to + * the old, when an old kernel is detected. + */ +#include +#define dev_pm_opp opp +#define dev_pm_opp_get_voltage opp_get_voltage +#define dev_pm_opp_get_opp_count opp_get_opp_count +#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil +#define dev_pm_opp_find_freq_floor opp_find_freq_floor +#endif /* Linux >= 3.13 */ +#include +#include + +static struct devfreq_simple_ondemand_data ondemand_data; + +static struct monitor_dev_profile mali_mdevp = { + .type = MONITOR_TPYE_DEV, + .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, + .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, +}; + +/** + * opp_translate - Translate nominal OPP frequency from devicetree into real + * frequency and core mask + * @kbdev: Device pointer + * @freq: Nominal frequency + * @core_mask: Pointer to u64 to store core mask to + * + * Return: Real target frequency + * + * This function will only perform translation if an operating-points-v2-mali + * table is present in devicetree. If one is not present then it will return an + * untranslated frequency and all cores enabled. + */ +static unsigned long opp_translate(struct kbase_device *kbdev, + unsigned long freq, u64 *core_mask) +{ + int i; + + for (i = 0; i < kbdev->num_opps; i++) { + if (kbdev->opp_table[i].opp_freq == freq) { + *core_mask = kbdev->opp_table[i].core_mask; + return kbdev->opp_table[i].real_freq; + } + } + + /* Failed to find OPP - return all cores enabled & nominal frequency */ + *core_mask = kbdev->gpu_props.props.raw_props.shader_present; + + return freq; +} + +static int +kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct dev_pm_opp *opp; + unsigned long nominal_freq; + unsigned long freq = 0; + unsigned long old_freq = kbdev->current_freq; + unsigned long voltage; + int err; + u64 core_mask; + + freq = *target_freq; + + opp = devfreq_recommended_opp(dev, &freq, flags); + if (IS_ERR(opp)) { + dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); + return PTR_ERR(opp); + } + voltage = dev_pm_opp_get_voltage(opp); + + nominal_freq = freq; + + /* + * Only update if there is a change of frequency + */ + if (kbdev->current_nominal_freq == nominal_freq) { + *target_freq = nominal_freq; +#ifdef CONFIG_REGULATOR + if (kbdev->current_voltage == voltage) + return 0; + err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); + if (err) { + dev_err(dev, "Failed to set voltage (%d)\n", err); + return err; + } + kbdev->current_voltage = voltage; +#endif + return 0; + } + + freq = opp_translate(kbdev, nominal_freq, &core_mask); +#ifdef CONFIG_REGULATOR + if (kbdev->regulator && kbdev->current_voltage != voltage && + old_freq < freq) { + err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); + if (err) { + dev_err(dev, "Failed to increase voltage (%d)\n", err); + return err; + } + } +#endif + + err = clk_set_rate(kbdev->clock, freq); + if (err) { + dev_err(dev, "Failed to set clock %lu (target %lu)\n", + freq, *target_freq); + return err; + } + *target_freq = freq; + kbdev->current_freq = freq; + if (kbdev->devfreq) + kbdev->devfreq->last_status.current_frequency = freq; +#ifdef CONFIG_REGULATOR + if (kbdev->regulator && kbdev->current_voltage != voltage && + old_freq > freq) { + err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); + if (err) { + dev_err(dev, "Failed to decrease voltage (%d)\n", err); + return err; + } + } +#endif + + if (kbdev->pm.backend.ca_current_policy->id == + KBASE_PM_CA_POLICY_ID_DEVFREQ) + kbase_devfreq_set_core_mask(kbdev, core_mask); + + *target_freq = nominal_freq; + kbdev->current_voltage = voltage; + kbdev->current_nominal_freq = nominal_freq; + kbdev->current_freq = freq; + kbdev->current_core_mask = core_mask; + + KBASE_TLSTREAM_AUX_DEVFREQ_TARGET((u64)nominal_freq); + + kbase_pm_reset_dvfs_utilisation(kbdev); + + return err; +} + +static int +kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + *freq = kbdev->current_nominal_freq; + + return 0; +} + +static int +kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + stat->current_frequency = kbdev->current_nominal_freq; + + kbase_pm_get_dvfs_utilisation(kbdev, + &stat->total_time, &stat->busy_time); + + stat->private_data = NULL; + + return 0; +} + +static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, + struct devfreq_dev_profile *dp) +{ + int count; + int i = 0; + unsigned long freq; + struct dev_pm_opp *opp; + + count = dev_pm_opp_get_opp_count(kbdev->dev); + if (count < 0) { + return count; + } + + dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), + GFP_KERNEL); + if (!dp->freq_table) + return -ENOMEM; + + for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) { + opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq); + if (IS_ERR(opp)) + break; + dev_pm_opp_put(opp); + + dp->freq_table[i] = freq; + } + + if (count != i) + dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n", + count, i); + + dp->max_state = i; + + return 0; +} + +static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev) +{ + struct devfreq_dev_profile *dp = kbdev->devfreq->profile; + + kfree(dp->freq_table); +} + +static void kbase_devfreq_exit(struct device *dev) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + + kbase_devfreq_term_freq_table(kbdev); +} + +static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) +{ + struct device_node *opp_node = of_parse_phandle(kbdev->dev->of_node, + "operating-points-v2", 0); + struct device_node *node; + int i = 0; + int count; + + if (!opp_node) + return 0; + if (!of_device_is_compatible(opp_node, "operating-points-v2-mali")) + return 0; + + count = dev_pm_opp_get_opp_count(kbdev->dev); + kbdev->opp_table = kmalloc_array(count, + sizeof(struct kbase_devfreq_opp), GFP_KERNEL); + if (!kbdev->opp_table) + return -ENOMEM; + + for_each_available_child_of_node(opp_node, node) { + u64 core_mask; + u64 opp_freq, real_freq; + const void *core_count_p; + + if (of_property_read_u64(node, "opp-hz", &opp_freq)) { + dev_warn(kbdev->dev, "OPP is missing required opp-hz property\n"); + continue; + } + if (of_property_read_u64(node, "opp-hz-real", &real_freq)) + real_freq = opp_freq; + if (of_property_read_u64(node, "opp-core-mask", &core_mask)) + core_mask = + kbdev->gpu_props.props.raw_props.shader_present; + core_count_p = of_get_property(node, "opp-core-count", NULL); + if (core_count_p) { + u64 remaining_core_mask = + kbdev->gpu_props.props.raw_props.shader_present; + int core_count = be32_to_cpup(core_count_p); + + core_mask = 0; + + for (; core_count > 0; core_count--) { + int core = ffs(remaining_core_mask); + + if (!core) { + dev_err(kbdev->dev, "OPP has more cores than GPU\n"); + return -ENODEV; + } + + core_mask |= (1ull << (core-1)); + remaining_core_mask &= ~(1ull << (core-1)); + } + } + + if (!core_mask) { + dev_err(kbdev->dev, "OPP has invalid core mask of 0\n"); + return -ENODEV; + } + + kbdev->opp_table[i].opp_freq = opp_freq; + kbdev->opp_table[i].real_freq = real_freq; + kbdev->opp_table[i].core_mask = core_mask; + + dev_info(kbdev->dev, "OPP %d : opp_freq=%llu real_freq=%llu core_mask=%llx\n", + i, opp_freq, real_freq, core_mask); + + i++; + } + + kbdev->num_opps = i; + + return 0; +} + +int kbase_devfreq_init(struct kbase_device *kbdev) +{ + struct device_node *np = kbdev->dev->of_node; + struct devfreq_dev_profile *dp; + struct dev_pm_opp *opp; + unsigned long opp_rate; + int err; + + if (!kbdev->clock) { + dev_err(kbdev->dev, "Clock not available for devfreq\n"); + return -ENODEV; + } + + kbdev->current_freq = clk_get_rate(kbdev->clock); + kbdev->current_nominal_freq = kbdev->current_freq; + + dp = &kbdev->devfreq_profile; + + dp->initial_freq = kbdev->current_freq; + /* .KP : set devfreq_dvfs_interval_in_ms */ + dp->polling_ms = 20; + dp->target = kbase_devfreq_target; + dp->get_dev_status = kbase_devfreq_status; + dp->get_cur_freq = kbase_devfreq_cur_freq; + dp->exit = kbase_devfreq_exit; + + if (kbase_devfreq_init_freq_table(kbdev, dp)) + return -EFAULT; + + err = kbase_devfreq_init_core_mask_table(kbdev); + if (err) + return err; + + of_property_read_u32(np, "upthreshold", + &ondemand_data.upthreshold); + of_property_read_u32(np, "downdifferential", + &ondemand_data.downdifferential); + + kbdev->devfreq = devfreq_add_device(kbdev->dev, dp, + "simple_ondemand", &ondemand_data); + if (IS_ERR(kbdev->devfreq)) { + kbase_devfreq_term_freq_table(kbdev); + return PTR_ERR(kbdev->devfreq); + } + + /* devfreq_add_device only copies a few of kbdev->dev's fields, so + * set drvdata explicitly so IPA models can access kbdev. */ + dev_set_drvdata(&kbdev->devfreq->dev, kbdev); + + err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq); + if (err) { + dev_err(kbdev->dev, + "Failed to register OPP notifier (%d)\n", err); + goto opp_notifier_failed; + } + + opp_rate = kbdev->current_freq; + opp = devfreq_recommended_opp(kbdev->dev, &opp_rate, 0); + if (!IS_ERR(opp)) + dev_pm_opp_put(opp); + kbdev->devfreq->last_status.current_frequency = opp_rate; + + mali_mdevp.data = kbdev->devfreq; + kbdev->mdev_info = rockchip_system_monitor_register(kbdev->dev, + &mali_mdevp); + if (IS_ERR(kbdev->mdev_info)) { + dev_dbg(kbdev->dev, "without system monitor\n"); + kbdev->mdev_info = NULL; + } +#ifdef CONFIG_DEVFREQ_THERMAL + err = kbase_ipa_init(kbdev); + if (err) { + dev_err(kbdev->dev, "IPA initialization failed\n"); + goto cooling_failed; + } + + kbdev->devfreq_cooling = of_devfreq_cooling_register_power( + kbdev->dev->of_node, + kbdev->devfreq, + &kbase_ipa_power_model_ops); + if (IS_ERR_OR_NULL(kbdev->devfreq_cooling)) { + err = PTR_ERR(kbdev->devfreq_cooling); + dev_err(kbdev->dev, + "Failed to register cooling device (%d)\n", + err); + goto cooling_failed; + } + I("success initing power_model_simple."); +#endif + + return 0; + +#ifdef CONFIG_DEVFREQ_THERMAL +cooling_failed: + devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); +#endif /* CONFIG_DEVFREQ_THERMAL */ +opp_notifier_failed: + if (devfreq_remove_device(kbdev->devfreq)) + dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); + else + kbdev->devfreq = NULL; + + return err; +} + +void kbase_devfreq_term(struct kbase_device *kbdev) +{ + int err; + + dev_dbg(kbdev->dev, "Term Mali devfreq\n"); + + rockchip_system_monitor_unregister(kbdev->mdev_info); +#ifdef CONFIG_DEVFREQ_THERMAL + if (kbdev->devfreq_cooling) + devfreq_cooling_unregister(kbdev->devfreq_cooling); + + kbase_ipa_term(kbdev); +#endif + + devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); + + err = devfreq_remove_device(kbdev->devfreq); + if (err) + dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); + else + kbdev->devfreq = NULL; + + kfree(kbdev->opp_table); +} diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h new file mode 100755 index 000000000000..c0bf8b15b3bc --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h @@ -0,0 +1,24 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _BASE_DEVFREQ_H_ +#define _BASE_DEVFREQ_H_ + +int kbase_devfreq_init(struct kbase_device *kbdev); +void kbase_devfreq_term(struct kbase_device *kbdev); + +#endif /* _BASE_DEVFREQ_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c new file mode 100755 index 000000000000..dcdf15cdc3e8 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c @@ -0,0 +1,255 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * + */ +#include +#include +#include + +#include + +#if !defined(CONFIG_MALI_NO_MALI) + + +#ifdef CONFIG_DEBUG_FS + + +int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size) +{ + struct kbase_io_access *old_buf; + struct kbase_io_access *new_buf; + unsigned long flags; + + if (!new_size) + goto out_err; /* The new size must not be 0 */ + + new_buf = vmalloc(new_size * sizeof(*h->buf)); + if (!new_buf) + goto out_err; + + spin_lock_irqsave(&h->lock, flags); + + old_buf = h->buf; + + /* Note: we won't bother with copying the old data over. The dumping + * logic wouldn't work properly as it relies on 'count' both as a + * counter and as an index to the buffer which would have changed with + * the new array. This is a corner case that we don't need to support. + */ + h->count = 0; + h->size = new_size; + h->buf = new_buf; + + spin_unlock_irqrestore(&h->lock, flags); + + vfree(old_buf); + + return 0; + +out_err: + return -1; +} + + +int kbase_io_history_init(struct kbase_io_history *h, u16 n) +{ + h->enabled = false; + spin_lock_init(&h->lock); + h->count = 0; + h->size = 0; + h->buf = NULL; + if (kbase_io_history_resize(h, n)) + return -1; + + return 0; +} + + +void kbase_io_history_term(struct kbase_io_history *h) +{ + vfree(h->buf); + h->buf = NULL; +} + + +/* kbase_io_history_add - add new entry to the register access history + * + * @h: Pointer to the history data structure + * @addr: Register address + * @value: The value that is either read from or written to the register + * @write: 1 if it's a register write, 0 if it's a read + */ +static void kbase_io_history_add(struct kbase_io_history *h, + void __iomem const *addr, u32 value, u8 write) +{ + struct kbase_io_access *io; + unsigned long flags; + + spin_lock_irqsave(&h->lock, flags); + + io = &h->buf[h->count % h->size]; + io->addr = (uintptr_t)addr | write; + io->value = value; + ++h->count; + /* If count overflows, move the index by the buffer size so the entire + * buffer will still be dumped later */ + if (unlikely(!h->count)) + h->count = h->size; + + spin_unlock_irqrestore(&h->lock, flags); +} + + +void kbase_io_history_dump(struct kbase_device *kbdev) +{ + struct kbase_io_history *const h = &kbdev->io_history; + u16 i; + size_t iters; + unsigned long flags; + + if (!unlikely(h->enabled)) + return; + + spin_lock_irqsave(&h->lock, flags); + + dev_err(kbdev->dev, "Register IO History:"); + iters = (h->size > h->count) ? h->count : h->size; + dev_err(kbdev->dev, "Last %zu register accesses of %zu total:\n", iters, + h->count); + for (i = 0; i < iters; ++i) { + struct kbase_io_access *io = + &h->buf[(h->count - iters + i) % h->size]; + char const access = (io->addr & 1) ? 'w' : 'r'; + + dev_err(kbdev->dev, "%6i: %c: reg 0x%p val %08x\n", i, access, + (void *)(io->addr & ~0x1), io->value); + } + + spin_unlock_irqrestore(&h->lock, flags); +} + + +#endif /* CONFIG_DEBUG_FS */ + + +void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, + struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbdev->dev != NULL); + + writel(value, kbdev->reg + offset); + +#ifdef CONFIG_DEBUG_FS + if (unlikely(kbdev->io_history.enabled)) + kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, + value, 1); +#endif /* CONFIG_DEBUG_FS */ + dev_dbg(kbdev->dev, "w: reg %04x val %08x", offset, value); + + if (kctx && kctx->jctx.tb) + kbase_device_trace_register_access(kctx, REG_WRITE, offset, + value); +} + +KBASE_EXPORT_TEST_API(kbase_reg_write); + +u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, + struct kbase_context *kctx) +{ + u32 val; + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbdev->dev != NULL); + + val = readl(kbdev->reg + offset); + +#ifdef CONFIG_DEBUG_FS + if (unlikely(kbdev->io_history.enabled)) + kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, + val, 0); +#endif /* CONFIG_DEBUG_FS */ + dev_dbg(kbdev->dev, "r: reg %04x val %08x", offset, val); + + if (kctx && kctx->jctx.tb) + kbase_device_trace_register_access(kctx, REG_READ, offset, val); + return val; +} + +KBASE_EXPORT_TEST_API(kbase_reg_read); +#endif /* !defined(CONFIG_MALI_NO_MALI) */ + +/** + * kbase_report_gpu_fault - Report a GPU fault. + * @kbdev: Kbase device pointer + * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS + * was also set + * + * This function is called from the interrupt handler when a GPU fault occurs. + * It reports the details of the fault using dev_warn(). + */ +static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) +{ + u32 status; + u64 address; + + status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL); + address = (u64) kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTADDRESS_HI), NULL) << 32; + address |= kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_FAULTADDRESS_LO), NULL); + + dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx", + status & 0xFF, + kbase_exception_name(kbdev, status), + address); + if (multiple) + dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n"); +} + +void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) +{ + KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, NULL, 0u, val); + if (val & GPU_FAULT) + kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS); + + if (val & RESET_COMPLETED) + kbase_pm_reset_done(kbdev); + + if (val & PRFCNT_SAMPLE_COMPLETED) + kbase_instr_hwcnt_sample_done(kbdev); + + if (val & CLEAN_CACHES_COMPLETED) + kbase_clean_caches_done(kbdev); + + KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, val); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, NULL); + + /* kbase_pm_check_transitions must be called after the IRQ has been + * cleared. This is because it might trigger further power transitions + * and we don't want to miss the interrupt raised to notify us that + * these further transitions have finished. + */ + if (val & POWER_CHANGED_ALL) + kbase_pm_power_changed(kbdev); + + KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, NULL, 0u, val); +} diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h new file mode 100755 index 000000000000..5b20445932fb --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h @@ -0,0 +1,67 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Backend-specific HW access device APIs + */ + +#ifndef _KBASE_DEVICE_INTERNAL_H_ +#define _KBASE_DEVICE_INTERNAL_H_ + +/** + * kbase_reg_write - write to GPU register + * @kbdev: Kbase device pointer + * @offset: Offset of register + * @value: Value to write + * @kctx: Kbase context pointer. May be NULL + * + * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If + * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr + * != KBASEP_AS_NR_INVALID). + */ +void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, + struct kbase_context *kctx); + +/** + * kbase_reg_read - read from GPU register + * @kbdev: Kbase device pointer + * @offset: Offset of register + * @kctx: Kbase context pointer. May be NULL + * + * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If + * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr + * != KBASEP_AS_NR_INVALID). + * + * Return: Value in desired register + */ +u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, + struct kbase_context *kctx); + + +/** + * kbase_gpu_interrupt - GPU interrupt handler + * @kbdev: Kbase device pointer + * @val: The value of the GPU IRQ status register which triggered the call + * + * This function is called from the interrupt handler when a GPU irq is to be + * handled. + */ +void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); + +#endif /* _KBASE_DEVICE_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpu.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpu.c new file mode 100755 index 000000000000..d578fd78e825 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpu.c @@ -0,0 +1,123 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend APIs + */ +#include +#include +#include +#include +#include +#include + +int kbase_backend_early_init(struct kbase_device *kbdev) +{ + int err; + + err = kbasep_platform_device_init(kbdev); + if (err) + return err; + + /* Ensure we can access the GPU registers */ + kbase_pm_register_access_enable(kbdev); + + /* Find out GPU properties based on the GPU feature registers */ + kbase_gpuprops_set(kbdev); + + /* We're done accessing the GPU registers for now. */ + kbase_pm_register_access_disable(kbdev); + + err = kbase_hwaccess_pm_init(kbdev); + if (err) + goto fail_pm; + + err = kbase_install_interrupts(kbdev); + if (err) + goto fail_interrupts; + + return 0; + +fail_interrupts: + kbase_hwaccess_pm_term(kbdev); +fail_pm: + kbasep_platform_device_term(kbdev); + + return err; +} + +void kbase_backend_early_term(struct kbase_device *kbdev) +{ + kbase_release_interrupts(kbdev); + kbase_hwaccess_pm_term(kbdev); + kbasep_platform_device_term(kbdev); +} + +int kbase_backend_late_init(struct kbase_device *kbdev) +{ + int err; + + err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); + if (err) + return err; + + err = kbase_backend_timer_init(kbdev); + if (err) + goto fail_timer; + +#ifdef CONFIG_MALI_DEBUG +#ifndef CONFIG_MALI_NO_MALI + if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { + dev_err(kbdev->dev, "Interrupt assigment check failed.\n"); + err = -EINVAL; + goto fail_interrupt_test; + } +#endif /* !CONFIG_MALI_NO_MALI */ +#endif /* CONFIG_MALI_DEBUG */ + + err = kbase_job_slot_init(kbdev); + if (err) + goto fail_job_slot; + + init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); + + return 0; + +fail_job_slot: + +#ifdef CONFIG_MALI_DEBUG +#ifndef CONFIG_MALI_NO_MALI +fail_interrupt_test: +#endif /* !CONFIG_MALI_NO_MALI */ +#endif /* CONFIG_MALI_DEBUG */ + + kbase_backend_timer_term(kbdev); +fail_timer: + kbase_hwaccess_pm_halt(kbdev); + + return err; +} + +void kbase_backend_late_term(struct kbase_device *kbdev) +{ + kbase_job_slot_halt(kbdev); + kbase_job_slot_term(kbdev); + kbase_backend_timer_term(kbdev); + kbase_hwaccess_pm_halt(kbdev); +} + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c new file mode 100755 index 000000000000..b395325b556b --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c @@ -0,0 +1,110 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel property query backend APIs + */ + +#include +#include +#include +#include + +void kbase_backend_gpuprops_get(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump) +{ + int i; + + /* Fill regdump with the content of the relevant registers */ + regdump->gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID), NULL); + + regdump->l2_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_FEATURES), NULL); + regdump->suspend_size = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SUSPEND_SIZE), NULL); + regdump->tiler_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_FEATURES), NULL); + regdump->mem_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(MEM_FEATURES), NULL); + regdump->mmu_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(MMU_FEATURES), NULL); + regdump->as_present = kbase_reg_read(kbdev, + GPU_CONTROL_REG(AS_PRESENT), NULL); + regdump->js_present = kbase_reg_read(kbdev, + GPU_CONTROL_REG(JS_PRESENT), NULL); + + for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) + regdump->js_features[i] = kbase_reg_read(kbdev, + GPU_CONTROL_REG(JS_FEATURES_REG(i)), NULL); + + for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) + regdump->texture_features[i] = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)), NULL); + + regdump->thread_max_threads = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_MAX_THREADS), NULL); + regdump->thread_max_workgroup_size = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE), + NULL); + regdump->thread_max_barrier_size = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE), NULL); + regdump->thread_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(THREAD_FEATURES), NULL); + + regdump->shader_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_PRESENT_LO), NULL); + regdump->shader_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_PRESENT_HI), NULL); + + regdump->tiler_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_PRESENT_LO), NULL); + regdump->tiler_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_PRESENT_HI), NULL); + + regdump->l2_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_PRESENT_LO), NULL); + regdump->l2_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_PRESENT_HI), NULL); + + regdump->stack_present_lo = kbase_reg_read(kbdev, + GPU_CONTROL_REG(STACK_PRESENT_LO), NULL); + regdump->stack_present_hi = kbase_reg_read(kbdev, + GPU_CONTROL_REG(STACK_PRESENT_HI), NULL); +} + +void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump) +{ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) { + /* Ensure we can access the GPU registers */ + kbase_pm_register_access_enable(kbdev); + + regdump->coherency_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(COHERENCY_FEATURES), NULL); + + /* We're done accessing the GPU registers for now. */ + kbase_pm_register_access_disable(kbdev); + } else { + /* Pre COHERENCY_FEATURES we only supported ACE_LITE */ + regdump->coherency_features = + COHERENCY_FEATURE_BIT(COHERENCY_NONE) | + COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); + } +} + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c new file mode 100755 index 000000000000..7ad309e8d7f4 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c @@ -0,0 +1,492 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * GPU backend instrumentation APIs. + */ + +#include +#include +#include +#include +#include +#include + +/** + * kbasep_instr_hwcnt_cacheclean - Issue Cache Clean & Invalidate command to + * hardware + * + * @kbdev: Kbase device + */ +static void kbasep_instr_hwcnt_cacheclean(struct kbase_device *kbdev) +{ + unsigned long flags; + unsigned long pm_flags; + u32 irq_mask; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_REQUEST_CLEAN); + + /* Enable interrupt */ + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + irq_mask | CLEAN_CACHES_COMPLETED, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + + /* clean&invalidate the caches so we're sure the mmu tables for the dump + * buffer is valid */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAN_INV_CACHES, NULL); + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANING; + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); +} + +int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_uk_hwcnt_setup *setup) +{ + unsigned long flags, pm_flags; + int err = -EINVAL; + u32 irq_mask; + int ret; + u64 shader_cores_needed; + u32 prfcnt_config; + + shader_cores_needed = kbase_pm_get_present_cores(kbdev, + KBASE_PM_CORE_SHADER); + + /* alignment failure */ + if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1))) + goto out_err; + + /* Override core availability policy to ensure all cores are available + */ + kbase_pm_ca_instr_enable(kbdev); + + /* Request the cores early on synchronously - we'll release them on any + * errors (e.g. instrumentation already active) */ + kbase_pm_request_cores_sync(kbdev, true, shader_cores_needed); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { + /* Instrumentation is already enabled */ + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + goto out_unrequest_cores; + } + + /* Enable interrupt */ + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | + PRFCNT_SAMPLE_COMPLETED, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + + /* In use, this context is the owner */ + kbdev->hwcnt.kctx = kctx; + /* Remember the dump address so we can reprogram it later */ + kbdev->hwcnt.addr = setup->dump_buffer; + + /* Request the clean */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; + kbdev->hwcnt.backend.triggered = 0; + /* Clean&invalidate the caches so we're sure the mmu tables for the dump + * buffer is valid */ + ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, + &kbdev->hwcnt.backend.cache_clean_work); + KBASE_DEBUG_ASSERT(ret); + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + /* Wait for cacheclean to complete */ + wait_event(kbdev->hwcnt.backend.wait, + kbdev->hwcnt.backend.triggered != 0); + + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_IDLE); + + kbase_pm_request_l2_caches(kbdev); + + /* Configure */ + prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; +#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY + { + u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + u32 product_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) + >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; + int arch_v6 = GPU_ID_IS_NEW_FORMAT(product_id); + + if (arch_v6) + prfcnt_config |= 1 << PRFCNT_CONFIG_SETSELECT_SHIFT; + } +#endif + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), + prfcnt_config | PRFCNT_CONFIG_MODE_OFF, kctx); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), + setup->dump_buffer & 0xFFFFFFFF, kctx); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), + setup->dump_buffer >> 32, kctx); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), + setup->jm_bm, kctx); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), + setup->shader_bm, kctx); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), + setup->mmu_l2_bm, kctx); + /* Due to PRLAM-8186 we need to disable the Tiler before we enable the + * HW counter dump. */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, + kctx); + else + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), + setup->tiler_bm, kctx); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), + prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL, kctx); + + /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump + */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), + setup->tiler_bm, kctx); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + err = 0; + + dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx); + return err; + out_unrequest_cores: + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_unrequest_cores(kbdev, true, shader_cores_needed); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + out_err: + return err; +} + +int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) +{ + unsigned long flags, pm_flags; + int err = -EINVAL; + u32 irq_mask; + struct kbase_device *kbdev = kctx->kbdev; + + while (1) { + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) { + /* Instrumentation is not enabled */ + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + goto out; + } + + if (kbdev->hwcnt.kctx != kctx) { + /* Instrumentation has been setup for another context */ + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + goto out; + } + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) + break; + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + /* Ongoing dump/setup - wait for its completion */ + wait_event(kbdev->hwcnt.backend.wait, + kbdev->hwcnt.backend.triggered != 0); + } + + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; + kbdev->hwcnt.backend.triggered = 0; + + /* Disable interrupt */ + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL); + + /* Disable the counters */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx); + + kbdev->hwcnt.kctx = NULL; + kbdev->hwcnt.addr = 0ULL; + + kbase_pm_ca_instr_disable(kbdev); + + kbase_pm_unrequest_cores(kbdev, true, + kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); + + kbase_pm_release_l2_caches(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", + kctx); + + err = 0; + + out: + return err; +} + +int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) +{ + unsigned long flags; + int err = -EINVAL; + struct kbase_device *kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.kctx != kctx) { + /* The instrumentation has been setup for another context */ + goto unlock; + } + + if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { + /* HW counters are disabled or another dump is ongoing, or we're + * resetting */ + goto unlock; + } + + kbdev->hwcnt.backend.triggered = 0; + + /* Mark that we're dumping - the PF handler can signal that we faulted + */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; + + /* Reconfigure the dump address */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), + kbdev->hwcnt.addr & 0xFFFFFFFF, NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), + kbdev->hwcnt.addr >> 32, NULL); + + /* Start dumping */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL, + kbdev->hwcnt.addr, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_PRFCNT_SAMPLE, kctx); + + dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx); + + err = 0; + + unlock: + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + return err; +} +KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); + +bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, + bool * const success) +{ + unsigned long flags; + bool complete = false; + struct kbase_device *kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) { + *success = true; + complete = true; + } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { + *success = false; + complete = true; + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + return complete; +} +KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); + +void kbasep_cache_clean_worker(struct work_struct *data) +{ + struct kbase_device *kbdev; + unsigned long flags; + + kbdev = container_of(data, struct kbase_device, + hwcnt.backend.cache_clean_work); + + mutex_lock(&kbdev->cacheclean_lock); + kbasep_instr_hwcnt_cacheclean(kbdev); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + /* Wait for our condition, and any reset to complete */ + while (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) { + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + wait_event(kbdev->hwcnt.backend.cache_clean_wait, + kbdev->hwcnt.backend.state != + KBASE_INSTR_STATE_CLEANING); + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + } + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_CLEANED); + + /* All finished and idle */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + mutex_unlock(&kbdev->cacheclean_lock); +} + +void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { + kbdev->hwcnt.backend.triggered = 1; + wake_up(&kbdev->hwcnt.backend.wait); + } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) { + int ret; + /* Always clean and invalidate the cache after a successful dump + */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; + ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, + &kbdev->hwcnt.backend.cache_clean_work); + KBASE_DEBUG_ASSERT(ret); + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); +} + +void kbase_clean_caches_done(struct kbase_device *kbdev) +{ + u32 irq_mask; + + if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { + unsigned long flags; + unsigned long pm_flags; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + /* Disable interrupt */ + spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); + irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), + irq_mask & ~CLEAN_CACHES_COMPLETED, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); + + /* Wakeup... */ + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) { + /* Only wake if we weren't resetting */ + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANED; + wake_up(&kbdev->hwcnt.backend.cache_clean_wait); + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + } +} + +int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + unsigned long flags; + int err; + + /* Wait for dump & cacheclean to complete */ + wait_event(kbdev->hwcnt.backend.wait, + kbdev->hwcnt.backend.triggered != 0); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { + err = -EINVAL; + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; + } else { + /* Dump done */ + KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_IDLE); + err = 0; + } + + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + return err; +} + +int kbase_instr_hwcnt_clear(struct kbase_context *kctx) +{ + unsigned long flags; + int err = -EINVAL; + struct kbase_device *kbdev = kctx->kbdev; + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + + /* Check it's the context previously set up and we're not already + * dumping */ + if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != + KBASE_INSTR_STATE_IDLE) + goto out; + + /* Clear the counters */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_PRFCNT_CLEAR, kctx); + + err = 0; + +out: + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + return err; +} +KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear); + +int kbase_instr_backend_init(struct kbase_device *kbdev) +{ + int ret = 0; + + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; + + init_waitqueue_head(&kbdev->hwcnt.backend.wait); + init_waitqueue_head(&kbdev->hwcnt.backend.cache_clean_wait); + INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work, + kbasep_cache_clean_worker); + kbdev->hwcnt.backend.triggered = 0; + + kbdev->hwcnt.backend.cache_clean_wq = + alloc_workqueue("Mali cache cleaning workqueue", 0, 1); + if (NULL == kbdev->hwcnt.backend.cache_clean_wq) + ret = -EINVAL; + + return ret; +} + +void kbase_instr_backend_term(struct kbase_device *kbdev) +{ + destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq); +} + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h new file mode 100755 index 000000000000..4794672da8f0 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h @@ -0,0 +1,58 @@ +/* + * + * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Backend-specific instrumentation definitions + */ + +#ifndef _KBASE_INSTR_DEFS_H_ +#define _KBASE_INSTR_DEFS_H_ + +/* + * Instrumentation State Machine States + */ +enum kbase_instr_state { + /* State where instrumentation is not active */ + KBASE_INSTR_STATE_DISABLED = 0, + /* State machine is active and ready for a command. */ + KBASE_INSTR_STATE_IDLE, + /* Hardware is currently dumping a frame. */ + KBASE_INSTR_STATE_DUMPING, + /* We've requested a clean to occur on a workqueue */ + KBASE_INSTR_STATE_REQUEST_CLEAN, + /* Hardware is currently cleaning and invalidating caches. */ + KBASE_INSTR_STATE_CLEANING, + /* Cache clean completed, and either a) a dump is complete, or + * b) instrumentation can now be setup. */ + KBASE_INSTR_STATE_CLEANED, + /* An error has occured during DUMPING (page fault). */ + KBASE_INSTR_STATE_FAULT +}; + +/* Structure used for instrumentation and HW counters dumping */ +struct kbase_instr_backend { + wait_queue_head_t wait; + int triggered; + + enum kbase_instr_state state; + wait_queue_head_t cache_clean_wait; + struct workqueue_struct *cache_clean_wq; + struct work_struct cache_clean_work; +}; + +#endif /* _KBASE_INSTR_DEFS_H_ */ + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h new file mode 100755 index 000000000000..e96aeae786e1 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Backend-specific HW access instrumentation APIs + */ + +#ifndef _KBASE_INSTR_INTERNAL_H_ +#define _KBASE_INSTR_INTERNAL_H_ + +/** + * kbasep_cache_clean_worker() - Workqueue for handling cache cleaning + * @data: a &struct work_struct + */ +void kbasep_cache_clean_worker(struct work_struct *data); + +/** + * kbase_clean_caches_done() - Cache clean interrupt received + * @kbdev: Kbase device + */ +void kbase_clean_caches_done(struct kbase_device *kbdev); + +/** + * kbase_instr_hwcnt_sample_done() - Dump complete interrupt received + * @kbdev: Kbase device + */ +void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev); + +#endif /* _KBASE_INSTR_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h new file mode 100755 index 000000000000..8781561e73d0 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Backend specific IRQ APIs + */ + +#ifndef _KBASE_IRQ_INTERNAL_H_ +#define _KBASE_IRQ_INTERNAL_H_ + +int kbase_install_interrupts(struct kbase_device *kbdev); + +void kbase_release_interrupts(struct kbase_device *kbdev); + +/** + * kbase_synchronize_irqs - Ensure that all IRQ handlers have completed + * execution + * @kbdev: The kbase device + */ +void kbase_synchronize_irqs(struct kbase_device *kbdev); + +int kbasep_common_test_interrupt_handlers( + struct kbase_device * const kbdev); + +#endif /* _KBASE_IRQ_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c new file mode 100755 index 000000000000..8416b80e8b77 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c @@ -0,0 +1,469 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include + +#include + +#if !defined(CONFIG_MALI_NO_MALI) + +/* GPU IRQ Tags */ +#define JOB_IRQ_TAG 0 +#define MMU_IRQ_TAG 1 +#define GPU_IRQ_TAG 2 + +static void *kbase_tag(void *ptr, u32 tag) +{ + return (void *)(((uintptr_t) ptr) | tag); +} + +static void *kbase_untag(void *ptr) +{ + return (void *)(((uintptr_t) ptr) & ~3); +} + +static irqreturn_t kbase_job_irq_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL); + +#ifdef CONFIG_MALI_DEBUG + if (!kbdev->pm.backend.driver_ready_for_irqs) + dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", + __func__, irq, val); +#endif /* CONFIG_MALI_DEBUG */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbase_job_done(kbdev, val); + + return IRQ_HANDLED; +} + +KBASE_EXPORT_TEST_API(kbase_job_irq_handler); + +static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return IRQ_NONE; + } + + atomic_inc(&kbdev->faults_pending); + + val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL); + +#ifdef CONFIG_MALI_DEBUG + if (!kbdev->pm.backend.driver_ready_for_irqs) + dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", + __func__, irq, val); +#endif /* CONFIG_MALI_DEBUG */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!val) { + atomic_dec(&kbdev->faults_pending); + return IRQ_NONE; + } + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbase_mmu_interrupt(kbdev, val); + + atomic_dec(&kbdev->faults_pending); + + return IRQ_HANDLED; +} + +static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); + +#ifdef CONFIG_MALI_DEBUG + if (!kbdev->pm.backend.driver_ready_for_irqs) + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", + __func__, irq, val); +#endif /* CONFIG_MALI_DEBUG */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbase_gpu_interrupt(kbdev, val); + + return IRQ_HANDLED; +} + +KBASE_EXPORT_TEST_API(kbase_gpu_irq_handler); + +static irq_handler_t kbase_handler_table[] = { + [JOB_IRQ_TAG] = kbase_job_irq_handler, + [MMU_IRQ_TAG] = kbase_mmu_irq_handler, + [GPU_IRQ_TAG] = kbase_gpu_irq_handler, +}; + +#ifdef CONFIG_MALI_DEBUG +#define JOB_IRQ_HANDLER JOB_IRQ_TAG +#define MMU_IRQ_HANDLER MMU_IRQ_TAG +#define GPU_IRQ_HANDLER GPU_IRQ_TAG + +/** + * kbase_set_custom_irq_handler - Set a custom IRQ handler + * @kbdev: Device for which the handler is to be registered + * @custom_handler: Handler to be registered + * @irq_type: Interrupt type + * + * Registers given interrupt handler for requested interrupt type + * In the case where irq handler is not specified, the default handler shall be + * registered + * + * Return: 0 case success, error code otherwise + */ +int kbase_set_custom_irq_handler(struct kbase_device *kbdev, + irq_handler_t custom_handler, + int irq_type) +{ + int result = 0; + irq_handler_t requested_irq_handler = NULL; + + KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) && + (GPU_IRQ_HANDLER >= irq_type)); + + /* Release previous handler */ + if (kbdev->irqs[irq_type].irq) + free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type)); + + requested_irq_handler = (NULL != custom_handler) ? custom_handler : + kbase_handler_table[irq_type]; + + if (0 != request_irq(kbdev->irqs[irq_type].irq, + requested_irq_handler, + kbdev->irqs[irq_type].flags | IRQF_SHARED, + dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) { + result = -EINVAL; + dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", + kbdev->irqs[irq_type].irq, irq_type); +#ifdef CONFIG_SPARSE_IRQ + dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); +#endif /* CONFIG_SPARSE_IRQ */ + } + + return result; +} + +KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler); + +/* test correct interrupt assigment and reception by cpu */ +struct kbasep_irq_test { + struct hrtimer timer; + wait_queue_head_t wait; + int triggered; + u32 timeout; +}; + +static struct kbasep_irq_test kbasep_irq_test_data; + +#define IRQ_TEST_TIMEOUT 500 + +static irqreturn_t kbase_job_irq_test_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL); + + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbasep_irq_test_data.triggered = 1; + wake_up(&kbasep_irq_test_data.wait); + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val, NULL); + + return IRQ_HANDLED; +} + +static irqreturn_t kbase_mmu_irq_test_handler(int irq, void *data) +{ + unsigned long flags; + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!kbdev->pm.backend.gpu_powered) { + /* GPU is turned off - IRQ is not for us */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return IRQ_NONE; + } + + val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL); + + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (!val) + return IRQ_NONE; + + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + + kbasep_irq_test_data.triggered = 1; + wake_up(&kbasep_irq_test_data.wait); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), val, NULL); + + return IRQ_HANDLED; +} + +static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) +{ + struct kbasep_irq_test *test_data = container_of(timer, + struct kbasep_irq_test, timer); + + test_data->timeout = 1; + test_data->triggered = 1; + wake_up(&test_data->wait); + return HRTIMER_NORESTART; +} + +static int kbasep_common_test_interrupt( + struct kbase_device * const kbdev, u32 tag) +{ + int err = 0; + irq_handler_t test_handler; + + u32 old_mask_val; + u16 mask_offset; + u16 rawstat_offset; + + switch (tag) { + case JOB_IRQ_TAG: + test_handler = kbase_job_irq_test_handler; + rawstat_offset = JOB_CONTROL_REG(JOB_IRQ_RAWSTAT); + mask_offset = JOB_CONTROL_REG(JOB_IRQ_MASK); + break; + case MMU_IRQ_TAG: + test_handler = kbase_mmu_irq_test_handler; + rawstat_offset = MMU_REG(MMU_IRQ_RAWSTAT); + mask_offset = MMU_REG(MMU_IRQ_MASK); + break; + case GPU_IRQ_TAG: + /* already tested by pm_driver - bail out */ + default: + return 0; + } + + /* store old mask */ + old_mask_val = kbase_reg_read(kbdev, mask_offset, NULL); + /* mask interrupts */ + kbase_reg_write(kbdev, mask_offset, 0x0, NULL); + + if (kbdev->irqs[tag].irq) { + /* release original handler and install test handler */ + if (kbase_set_custom_irq_handler(kbdev, test_handler, tag) != 0) { + err = -EINVAL; + } else { + kbasep_irq_test_data.timeout = 0; + hrtimer_init(&kbasep_irq_test_data.timer, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + kbasep_irq_test_data.timer.function = + kbasep_test_interrupt_timeout; + + /* trigger interrupt */ + kbase_reg_write(kbdev, mask_offset, 0x1, NULL); + kbase_reg_write(kbdev, rawstat_offset, 0x1, NULL); + + hrtimer_start(&kbasep_irq_test_data.timer, + HR_TIMER_DELAY_MSEC(IRQ_TEST_TIMEOUT), + HRTIMER_MODE_REL); + + wait_event(kbasep_irq_test_data.wait, + kbasep_irq_test_data.triggered != 0); + + if (kbasep_irq_test_data.timeout != 0) { + dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n", + kbdev->irqs[tag].irq, tag); + err = -EINVAL; + } else { + dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n", + kbdev->irqs[tag].irq, tag); + } + + hrtimer_cancel(&kbasep_irq_test_data.timer); + kbasep_irq_test_data.triggered = 0; + + /* mask interrupts */ + kbase_reg_write(kbdev, mask_offset, 0x0, NULL); + + /* release test handler */ + free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag)); + } + + /* restore original interrupt */ + if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag], + kbdev->irqs[tag].flags | IRQF_SHARED, + dev_name(kbdev->dev), kbase_tag(kbdev, tag))) { + dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n", + kbdev->irqs[tag].irq, tag); + err = -EINVAL; + } + } + /* restore old mask */ + kbase_reg_write(kbdev, mask_offset, old_mask_val, NULL); + + return err; +} + +int kbasep_common_test_interrupt_handlers( + struct kbase_device * const kbdev) +{ + int err; + + init_waitqueue_head(&kbasep_irq_test_data.wait); + kbasep_irq_test_data.triggered = 0; + + /* A suspend won't happen during startup/insmod */ + kbase_pm_context_active(kbdev); + + err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG); + if (err) { + dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n"); + goto out; + } + + err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG); + if (err) { + dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n"); + goto out; + } + + dev_dbg(kbdev->dev, "Interrupts are correctly assigned.\n"); + + out: + kbase_pm_context_idle(kbdev); + + return err; +} +#endif /* CONFIG_MALI_DEBUG */ + +int kbase_install_interrupts(struct kbase_device *kbdev) +{ + u32 nr = ARRAY_SIZE(kbase_handler_table); + int err; + u32 i; + + for (i = 0; i < nr; i++) { + err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i], + kbdev->irqs[i].flags | IRQF_SHARED, + dev_name(kbdev->dev), + kbase_tag(kbdev, i)); + if (err) { + dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", + kbdev->irqs[i].irq, i); +#ifdef CONFIG_SPARSE_IRQ + dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); +#endif /* CONFIG_SPARSE_IRQ */ + goto release; + } + } + + return 0; + + release: + while (i-- > 0) + free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); + + return err; +} + +void kbase_release_interrupts(struct kbase_device *kbdev) +{ + u32 nr = ARRAY_SIZE(kbase_handler_table); + u32 i; + + for (i = 0; i < nr; i++) { + if (kbdev->irqs[i].irq) + free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); + } +} + +void kbase_synchronize_irqs(struct kbase_device *kbdev) +{ + u32 nr = ARRAY_SIZE(kbase_handler_table); + u32 i; + + for (i = 0; i < nr; i++) { + if (kbdev->irqs[i].irq) + synchronize_irq(kbdev->irqs[i].irq); + } +} + +#endif /* !defined(CONFIG_MALI_NO_MALI) */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c new file mode 100755 index 000000000000..92358f2bf298 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c @@ -0,0 +1,237 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register backend context / address space management + */ + +#include +#include +#include + +/** + * assign_and_activate_kctx_addr_space - Assign an AS to a context + * @kbdev: Kbase device + * @kctx: Kbase context + * @current_as: Address Space to assign + * + * Assign an Address Space (AS) to a context, and add the context to the Policy. + * + * This includes + * setting up the global runpool_irq structure and the context on the AS, + * Activating the MMU on the AS, + * Allowing jobs to be submitted on the AS. + * + * Context: + * kbasep_js_kctx_info.jsctx_mutex held, + * kbasep_js_device_data.runpool_mutex held, + * AS transaction mutex held, + * Runpool IRQ lock held + */ +static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_as *current_as) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* Attribute handling */ + kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); + + /* Allow it to run jobs */ + kbasep_js_set_submit_allowed(js_devdata, kctx); + + kbase_js_runpool_inc_context_count(kbdev, kctx); +} + +bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + int i; + + if (kbdev->hwaccess.active_kctx == kctx) { + /* Context is already active */ + return true; + } + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + if (kbdev->as_to_kctx[i] == kctx) { + /* Context already has ASID - mark as active */ + return true; + } + } + + /* Context does not have address space assigned */ + return false; +} + +void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + int as_nr = kctx->as_nr; + + if (as_nr == KBASEP_AS_NR_INVALID) { + WARN(1, "Attempting to release context without ASID\n"); + return; + } + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (atomic_read(&kctx->refcount) != 1) { + WARN(1, "Attempting to release active ASID\n"); + return; + } + + kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx); + + kbase_ctx_sched_release_ctx(kctx); + kbase_js_runpool_dec_context_count(kbdev, kctx); +} + +void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ +} + +int kbase_backend_find_and_release_free_address_space( + struct kbase_device *kbdev, struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + unsigned long flags; + int i; + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + struct kbasep_js_kctx_info *as_js_kctx_info; + struct kbase_context *as_kctx; + + as_kctx = kbdev->as_to_kctx[i]; + as_js_kctx_info = &as_kctx->jctx.sched_info; + + /* Don't release privileged or active contexts, or contexts with + * jobs running. + * Note that a context will have at least 1 reference (which + * was previously taken by kbasep_js_schedule_ctx()) until + * descheduled. + */ + if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && + atomic_read(&as_kctx->refcount) == 1) { + if (!kbasep_js_runpool_retain_ctx_nolock(kbdev, + as_kctx)) { + WARN(1, "Failed to retain active context\n"); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return KBASEP_AS_NR_INVALID; + } + + kbasep_js_clear_submit_allowed(js_devdata, as_kctx); + + /* Drop and retake locks to take the jsctx_mutex on the + * context we're about to release without violating lock + * ordering + */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + + /* Release context from address space */ + mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx); + + if (!kbase_ctx_flag(as_kctx, KCTX_SCHEDULED)) { + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, + as_kctx, + true); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); + + return i; + } + + /* Context was retained while locks were dropped, + * continue looking for free AS */ + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + } + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return KBASEP_AS_NR_INVALID; +} + +bool kbase_backend_use_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, + int as_nr) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + struct kbase_as *new_address_space = NULL; + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + if (kbdev->hwaccess.active_kctx == kctx) { + WARN(1, "Context is already scheduled in\n"); + return false; + } + + new_address_space = &kbdev->as[as_nr]; + + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space); + + if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { + /* We need to retain it to keep the corresponding address space + */ + kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + } + + return true; +} + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h new file mode 100755 index 000000000000..08a7400e66d5 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h @@ -0,0 +1,123 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend specific definitions + */ + +#ifndef _KBASE_HWACCESS_GPU_DEFS_H_ +#define _KBASE_HWACCESS_GPU_DEFS_H_ + +/* SLOT_RB_SIZE must be < 256 */ +#define SLOT_RB_SIZE 2 +#define SLOT_RB_MASK (SLOT_RB_SIZE - 1) + +/** + * struct rb_entry - Ringbuffer entry + * @katom: Atom associated with this entry + */ +struct rb_entry { + struct kbase_jd_atom *katom; +}; + +/** + * struct slot_rb - Slot ringbuffer + * @entries: Ringbuffer entries + * @last_context: The last context to submit a job on this slot + * @read_idx: Current read index of buffer + * @write_idx: Current write index of buffer + * @job_chain_flag: Flag used to implement jobchain disambiguation + */ +struct slot_rb { + struct rb_entry entries[SLOT_RB_SIZE]; + + struct kbase_context *last_context; + + u8 read_idx; + u8 write_idx; + + u8 job_chain_flag; +}; + +/** + * struct kbase_backend_data - GPU backend specific data for HW access layer + * @slot_rb: Slot ringbuffers + * @rmu_workaround_flag: When PRLAM-8987 is present, this flag determines + * whether slots 0/1 or slot 2 are currently being + * pulled from + * @scheduling_timer: The timer tick used for rescheduling jobs + * @timer_running: Is the timer running? The runpool_mutex must be + * held whilst modifying this. + * @suspend_timer: Is the timer suspended? Set when a suspend + * occurs and cleared on resume. The runpool_mutex + * must be held whilst modifying this. + * @reset_gpu: Set to a KBASE_RESET_xxx value (see comments) + * @reset_workq: Work queue for performing the reset + * @reset_work: Work item for performing the reset + * @reset_wait: Wait event signalled when the reset is complete + * @reset_timer: Timeout for soft-stops before the reset + * @timeouts_updated: Have timeout values just been updated? + * + * The hwaccess_lock (a spinlock) must be held when accessing this structure + */ +struct kbase_backend_data { + struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS]; + + bool rmu_workaround_flag; + + struct hrtimer scheduling_timer; + + bool timer_running; + bool suspend_timer; + + atomic_t reset_gpu; + +/* The GPU reset isn't pending */ +#define KBASE_RESET_GPU_NOT_PENDING 0 +/* kbase_prepare_to_reset_gpu has been called */ +#define KBASE_RESET_GPU_PREPARED 1 +/* kbase_reset_gpu has been called - the reset will now definitely happen + * within the timeout period */ +#define KBASE_RESET_GPU_COMMITTED 2 +/* The GPU reset process is currently occuring (timeout has expired or + * kbasep_try_reset_gpu_early was called) */ +#define KBASE_RESET_GPU_HAPPENING 3 +/* Reset the GPU silently, used when resetting the GPU as part of normal + * behavior (e.g. when exiting protected mode). */ +#define KBASE_RESET_GPU_SILENT 4 + struct workqueue_struct *reset_workq; + struct work_struct reset_work; + wait_queue_head_t reset_wait; + struct hrtimer reset_timer; + + bool timeouts_updated; +}; + +/** + * struct kbase_jd_atom_backend - GPU backend specific katom data + */ +struct kbase_jd_atom_backend { +}; + +/** + * struct kbase_context_backend - GPU backend specific context data + */ +struct kbase_context_backend { +}; + +#endif /* _KBASE_HWACCESS_GPU_DEFS_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c new file mode 100755 index 000000000000..a6fb097b94f9 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c @@ -0,0 +1,1518 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Base kernel job manager APIs + */ + +#include +#include +#include +#if defined(CONFIG_MALI_GATOR_SUPPORT) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define beenthere(kctx, f, a...) \ + dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) + +#if KBASE_GPU_RESET_EN +static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev); +static void kbasep_reset_timeout_worker(struct work_struct *data); +static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer); +#endif /* KBASE_GPU_RESET_EN */ + +static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, + struct kbase_context *kctx) +{ + return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), kctx); +} + +void kbase_job_hw_submit(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + int js) +{ + struct kbase_context *kctx; + u32 cfg; + u64 jc_head = katom->jc; + + KBASE_DEBUG_ASSERT(kbdev); + KBASE_DEBUG_ASSERT(katom); + + kctx = katom->kctx; + + /* Command register must be available */ + KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx)); + /* Affinity is not violating */ + kbase_js_debug_log_current_affinities(kbdev); + KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js, + katom->affinity)); + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), + jc_head & 0xFFFFFFFF, kctx); + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), + jc_head >> 32, kctx); + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO), + katom->affinity & 0xFFFFFFFF, kctx); + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI), + katom->affinity >> 32, kctx); + + /* start MMU, medium priority, cache clean/flush on end, clean/flush on + * start */ + cfg = kctx->as_nr; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) + cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; + +#ifndef CONFIG_MALI_COH_GPU + if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) + cfg |= JS_CONFIG_START_FLUSH_NO_ACTION; + else + cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; + + if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END)) + cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; + else + cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; +#endif /* CONFIG_MALI_COH_GPU */ + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10649)) + cfg |= JS_CONFIG_START_MMU; + + cfg |= JS_CONFIG_THREAD_PRI(8); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE) && + (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED)) + cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; + + if (kbase_hw_has_feature(kbdev, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { + if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) { + cfg |= JS_CONFIG_JOB_CHAIN_FLAG; + katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN; + kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = + true; + } else { + katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN; + kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = + false; + } + } + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg, kctx); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_FLUSH_ID_NEXT), + katom->flush_id, kctx); + + /* Write an approximate start timestamp. + * It's approximate because there might be a job in the HEAD register. + */ + katom->start_timestamp = ktime_get(); + + /* GO ! */ + dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx", + katom, kctx, js, jc_head, katom->affinity); + + KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, + (u32) katom->affinity); + +#if defined(CONFIG_MALI_GATOR_SUPPORT) + kbase_trace_mali_job_slots_event( + GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js), + kctx, kbase_jd_atom_id(kctx, katom)); +#endif + KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(katom, jc_head, + katom->affinity, cfg); + KBASE_TLSTREAM_TL_RET_CTX_LPU( + kctx, + &kbdev->gpu_props.props.raw_props.js_features[ + katom->slot_nr]); + KBASE_TLSTREAM_TL_RET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); + KBASE_TLSTREAM_TL_RET_ATOM_LPU( + katom, + &kbdev->gpu_props.props.raw_props.js_features[js], + "ctx_nr,atom_nr"); +#ifdef CONFIG_GPU_TRACEPOINTS + if (!kbase_backend_nr_atoms_submitted(kbdev, js)) { + /* If this is the only job on the slot, trace it as starting */ + char js_string[16]; + + trace_gpu_sched_switch( + kbasep_make_job_slot_string(js, js_string, + sizeof(js_string)), + ktime_to_ns(katom->start_timestamp), + (u32)katom->kctx->id, 0, katom->work_id); + kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx; + } +#endif + kbase_timeline_job_slot_submit(kbdev, kctx, katom, js); + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), + JS_COMMAND_START, katom->kctx); +} + +/** + * kbasep_job_slot_update_head_start_timestamp - Update timestamp + * @kbdev: kbase device + * @js: job slot + * @end_timestamp: timestamp + * + * Update the start_timestamp of the job currently in the HEAD, based on the + * fact that we got an IRQ for the previous set of completed jobs. + * + * The estimate also takes into account the time the job was submitted, to + * work out the best estimate (which might still result in an over-estimate to + * the calculated time spent) + */ +static void kbasep_job_slot_update_head_start_timestamp( + struct kbase_device *kbdev, + int js, + ktime_t end_timestamp) +{ + if (kbase_backend_nr_atoms_on_slot(kbdev, js) > 0) { + struct kbase_jd_atom *katom; + ktime_t timestamp_diff; + /* The atom in the HEAD */ + katom = kbase_gpu_inspect(kbdev, js, 0); + + KBASE_DEBUG_ASSERT(katom != NULL); + + timestamp_diff = ktime_sub(end_timestamp, + katom->start_timestamp); + if (ktime_to_ns(timestamp_diff) >= 0) { + /* Only update the timestamp if it's a better estimate + * than what's currently stored. This is because our + * estimate that accounts for the throttle time may be + * too much of an overestimate */ + katom->start_timestamp = end_timestamp; + } + } +} + +/** + * kbasep_trace_tl_event_lpu_softstop - Call event_lpu_softstop timeline + * tracepoint + * @kbdev: kbase device + * @js: job slot + * + * Make a tracepoint call to the instrumentation module informing that + * softstop happened on given lpu (job slot). + */ +static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, + int js) +{ + KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( + &kbdev->gpu_props.props.raw_props.js_features[js]); +} + +void kbase_job_done(struct kbase_device *kbdev, u32 done) +{ + unsigned long flags; + int i; + u32 count = 0; + ktime_t end_timestamp = ktime_get(); + struct kbasep_js_device_data *js_devdata; + + KBASE_DEBUG_ASSERT(kbdev); + js_devdata = &kbdev->js_data; + + KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done); + + memset(&kbdev->slot_submit_count_irq[0], 0, + sizeof(kbdev->slot_submit_count_irq)); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + while (done) { + u32 failed = done >> 16; + + /* treat failed slots as finished slots */ + u32 finished = (done & 0xFFFF) | failed; + + /* Note: This is inherently unfair, as we always check + * for lower numbered interrupts before the higher + * numbered ones.*/ + i = ffs(finished) - 1; + KBASE_DEBUG_ASSERT(i >= 0); + + do { + int nr_done; + u32 active; + u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */ + u64 job_tail = 0; + + if (failed & (1u << i)) { + /* read out the job slot status code if the job + * slot reported failure */ + completion_code = kbase_reg_read(kbdev, + JOB_SLOT_REG(i, JS_STATUS), NULL); + + switch (completion_code) { + case BASE_JD_EVENT_STOPPED: +#if defined(CONFIG_MALI_GATOR_SUPPORT) + kbase_trace_mali_job_slots_event( + GATOR_MAKE_EVENT( + GATOR_JOB_SLOT_SOFT_STOPPED, i), + NULL, 0); +#endif + + kbasep_trace_tl_event_lpu_softstop( + kbdev, i); + + /* Soft-stopped job - read the value of + * JS_TAIL so that the job chain can + * be resumed */ + job_tail = (u64)kbase_reg_read(kbdev, + JOB_SLOT_REG(i, JS_TAIL_LO), + NULL) | + ((u64)kbase_reg_read(kbdev, + JOB_SLOT_REG(i, JS_TAIL_HI), + NULL) << 32); + break; + case BASE_JD_EVENT_NOT_STARTED: + /* PRLAM-10673 can cause a TERMINATED + * job to come back as NOT_STARTED, but + * the error interrupt helps us detect + * it */ + completion_code = + BASE_JD_EVENT_TERMINATED; + /* fall through */ + default: + dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", + i, completion_code, + kbase_exception_name + (kbdev, + completion_code)); + } + + kbase_gpu_irq_evict(kbdev, i); + } + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), + done & ((1 << i) | (1 << (i + 16))), + NULL); + active = kbase_reg_read(kbdev, + JOB_CONTROL_REG(JOB_IRQ_JS_STATE), + NULL); + + if (((active >> i) & 1) == 0 && + (((done >> (i + 16)) & 1) == 0)) { + /* There is a potential race we must work + * around: + * + * 1. A job slot has a job in both current and + * next registers + * 2. The job in current completes + * successfully, the IRQ handler reads + * RAWSTAT and calls this function with the + * relevant bit set in "done" + * 3. The job in the next registers becomes the + * current job on the GPU + * 4. Sometime before the JOB_IRQ_CLEAR line + * above the job on the GPU _fails_ + * 5. The IRQ_CLEAR clears the done bit but not + * the failed bit. This atomically sets + * JOB_IRQ_JS_STATE. However since both jobs + * have now completed the relevant bits for + * the slot are set to 0. + * + * If we now did nothing then we'd incorrectly + * assume that _both_ jobs had completed + * successfully (since we haven't yet observed + * the fail bit being set in RAWSTAT). + * + * So at this point if there are no active jobs + * left we check to see if RAWSTAT has a failure + * bit set for the job slot. If it does we know + * that there has been a new failure that we + * didn't previously know about, so we make sure + * that we record this in active (but we wait + * for the next loop to deal with it). + * + * If we were handling a job failure (i.e. done + * has the relevant high bit set) then we know + * that the value read back from + * JOB_IRQ_JS_STATE is the correct number of + * remaining jobs because the failed job will + * have prevented any futher jobs from starting + * execution. + */ + u32 rawstat = kbase_reg_read(kbdev, + JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL); + + if ((rawstat >> (i + 16)) & 1) { + /* There is a failed job that we've + * missed - add it back to active */ + active |= (1u << i); + } + } + + dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", + completion_code); + + nr_done = kbase_backend_nr_atoms_submitted(kbdev, i); + nr_done -= (active >> i) & 1; + nr_done -= (active >> (i + 16)) & 1; + + if (nr_done <= 0) { + dev_warn(kbdev->dev, "Spurious interrupt on slot %d", + i); + + goto spurious; + } + + count += nr_done; + + while (nr_done) { + if (nr_done == 1) { + kbase_gpu_complete_hw(kbdev, i, + completion_code, + job_tail, + &end_timestamp); + kbase_jm_try_kick_all(kbdev); + } else { + /* More than one job has completed. + * Since this is not the last job being + * reported this time it must have + * passed. This is because the hardware + * will not allow further jobs in a job + * slot to complete until the failed job + * is cleared from the IRQ status. + */ + kbase_gpu_complete_hw(kbdev, i, + BASE_JD_EVENT_DONE, + 0, + &end_timestamp); + } + nr_done--; + } + spurious: + done = kbase_reg_read(kbdev, + JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) { + /* Workaround for missing interrupt caused by + * PRLAM-10883 */ + if (((active >> i) & 1) && (0 == + kbase_reg_read(kbdev, + JOB_SLOT_REG(i, + JS_STATUS), NULL))) { + /* Force job slot to be processed again + */ + done |= (1u << i); + } + } + + failed = done >> 16; + finished = (done & 0xFFFF) | failed; + if (done) + end_timestamp = ktime_get(); + } while (finished & (1 << i)); + + kbasep_job_slot_update_head_start_timestamp(kbdev, i, + end_timestamp); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#if KBASE_GPU_RESET_EN + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_COMMITTED) { + /* If we're trying to reset the GPU then we might be able to do + * it early (without waiting for a timeout) because some jobs + * have completed + */ + kbasep_try_reset_gpu_early(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count); +} +KBASE_EXPORT_TEST_API(kbase_job_done); + +static bool kbasep_soft_stop_allowed(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + bool soft_stops_allowed = true; + + if (kbase_jd_katom_is_protected(katom)) { + soft_stops_allowed = false; + } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) { + if ((katom->core_req & BASE_JD_REQ_T) != 0) + soft_stops_allowed = false; + } + return soft_stops_allowed; +} + +static bool kbasep_hard_stop_allowed(struct kbase_device *kbdev, + base_jd_core_req core_reqs) +{ + bool hard_stops_allowed = true; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) { + if ((core_reqs & BASE_JD_REQ_T) != 0) + hard_stops_allowed = false; + } + return hard_stops_allowed; +} + +void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, + int js, + u32 action, + base_jd_core_req core_reqs, + struct kbase_jd_atom *target_katom) +{ + struct kbase_context *kctx = target_katom->kctx; +#if KBASE_TRACE_ENABLE + u32 status_reg_before; + u64 job_in_head_before; + u32 status_reg_after; + + KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK))); + + /* Check the head pointer */ + job_in_head_before = ((u64) kbase_reg_read(kbdev, + JOB_SLOT_REG(js, JS_HEAD_LO), NULL)) + | (((u64) kbase_reg_read(kbdev, + JOB_SLOT_REG(js, JS_HEAD_HI), NULL)) + << 32); + status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), + NULL); +#endif + + if (action == JS_COMMAND_SOFT_STOP) { + bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev, + target_katom); + + if (!soft_stop_allowed) { +#ifdef CONFIG_MALI_DEBUG + dev_dbg(kbdev->dev, + "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X", + (unsigned int)core_reqs); +#endif /* CONFIG_MALI_DEBUG */ + return; + } + + /* We are about to issue a soft stop, so mark the atom as having + * been soft stopped */ + target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED; + + /* Mark the point where we issue the soft-stop command */ + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(target_katom); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { + int i; + + for (i = 0; + i < kbase_backend_nr_atoms_submitted(kbdev, js); + i++) { + struct kbase_jd_atom *katom; + + katom = kbase_gpu_inspect(kbdev, js, i); + + KBASE_DEBUG_ASSERT(katom); + + /* For HW_ISSUE_8316, only 'bad' jobs attacking + * the system can cause this issue: normally, + * all memory should be allocated in multiples + * of 4 pages, and growable memory should be + * changed size in multiples of 4 pages. + * + * Whilst such 'bad' jobs can be cleared by a + * GPU reset, the locking up of a uTLB entry + * caused by the bad job could also stall other + * ASs, meaning that other ASs' jobs don't + * complete in the 'grace' period before the + * reset. We don't want to lose other ASs' jobs + * when they would normally complete fine, so we + * must 'poke' the MMU regularly to help other + * ASs complete */ + kbase_as_poking_timer_retain_atom( + kbdev, katom->kctx, katom); + } + } + + if (kbase_hw_has_feature( + kbdev, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { + action = (target_katom->atom_flags & + KBASE_KATOM_FLAGS_JOBCHAIN) ? + JS_COMMAND_SOFT_STOP_1 : + JS_COMMAND_SOFT_STOP_0; + } + } else if (action == JS_COMMAND_HARD_STOP) { + bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev, + core_reqs); + + if (!hard_stop_allowed) { + /* Jobs can be hard-stopped for the following reasons: + * * CFS decides the job has been running too long (and + * soft-stop has not occurred). In this case the GPU + * will be reset by CFS if the job remains on the + * GPU. + * + * * The context is destroyed, kbase_jd_zap_context + * will attempt to hard-stop the job. However it also + * has a watchdog which will cause the GPU to be + * reset if the job remains on the GPU. + * + * * An (unhandled) MMU fault occurred. As long as + * BASE_HW_ISSUE_8245 is defined then the GPU will be + * reset. + * + * All three cases result in the GPU being reset if the + * hard-stop fails, so it is safe to just return and + * ignore the hard-stop request. + */ + dev_warn(kbdev->dev, + "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X", + (unsigned int)core_reqs); + return; + } + target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED; + + if (kbase_hw_has_feature( + kbdev, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { + action = (target_katom->atom_flags & + KBASE_KATOM_FLAGS_JOBCHAIN) ? + JS_COMMAND_HARD_STOP_1 : + JS_COMMAND_HARD_STOP_0; + } + } + + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action, kctx); + +#if KBASE_TRACE_ENABLE + status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), + NULL); + if (status_reg_after == BASE_JD_EVENT_ACTIVE) { + struct kbase_jd_atom *head; + struct kbase_context *head_kctx; + + head = kbase_gpu_inspect(kbdev, js, 0); + head_kctx = head->kctx; + + if (status_reg_before == BASE_JD_EVENT_ACTIVE) + KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, + head, job_in_head_before, js); + else + KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, + 0, js); + + switch (action) { + case JS_COMMAND_SOFT_STOP: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx, + head, head->jc, js); + break; + case JS_COMMAND_SOFT_STOP_0: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, + head, head->jc, js); + break; + case JS_COMMAND_SOFT_STOP_1: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, + head, head->jc, js); + break; + case JS_COMMAND_HARD_STOP: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx, + head, head->jc, js); + break; + case JS_COMMAND_HARD_STOP_0: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, + head, head->jc, js); + break; + case JS_COMMAND_HARD_STOP_1: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, + head, head->jc, js); + break; + default: + BUG(); + break; + } + } else { + if (status_reg_before == BASE_JD_EVENT_ACTIVE) + KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, + job_in_head_before, js); + else + KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, + 0, js); + + switch (action) { + case JS_COMMAND_SOFT_STOP: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, + js); + break; + case JS_COMMAND_SOFT_STOP_0: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, + 0, js); + break; + case JS_COMMAND_SOFT_STOP_1: + KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, + 0, js); + break; + case JS_COMMAND_HARD_STOP: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, + js); + break; + case JS_COMMAND_HARD_STOP_0: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, + 0, js); + break; + case JS_COMMAND_HARD_STOP_1: + KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, + 0, js); + break; + default: + BUG(); + break; + } + } +#endif +} + +void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx) +{ + unsigned long flags; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + int i; + + KBASE_DEBUG_ASSERT(kctx != NULL); + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + js_devdata = &kbdev->js_data; + + /* Cancel any remaining running jobs for this kctx */ + mutex_lock(&kctx->jctx.lock); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Invalidate all jobs in context, to prevent re-submitting */ + for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { + if (!work_pending(&kctx->jctx.atoms[i].work)) + kctx->jctx.atoms[i].event_code = + BASE_JD_EVENT_JOB_CANCELLED; + } + + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + kbase_job_slot_hardstop(kctx, i, NULL); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kctx->jctx.lock); +} + +void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, + struct kbase_jd_atom *target_katom) +{ + struct kbase_device *kbdev; + int js = target_katom->slot_nr; + int priority = target_katom->sched_priority; + int i; + bool stop_sent = false; + + KBASE_DEBUG_ASSERT(kctx != NULL); + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { + struct kbase_jd_atom *katom; + + katom = kbase_gpu_inspect(kbdev, js, i); + if (!katom) + continue; + + if (katom->kctx != kctx) + continue; + + if (katom->sched_priority > priority) { + if (!stop_sent) + KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE( + target_katom); + + kbase_job_slot_softstop(kbdev, js, katom); + stop_sent = true; + } + } +} + +struct zap_reset_data { + /* The stages are: + * 1. The timer has never been called + * 2. The zap has timed out, all slots are soft-stopped - the GPU reset + * will happen. The GPU has been reset when + * kbdev->hwaccess.backend.reset_waitq is signalled + * + * (-1 - The timer has been cancelled) + */ + int stage; + struct kbase_device *kbdev; + struct hrtimer timer; + spinlock_t lock; /* protects updates to stage member */ +}; + +static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer) +{ + struct zap_reset_data *reset_data = container_of(timer, + struct zap_reset_data, timer); + struct kbase_device *kbdev = reset_data->kbdev; + unsigned long flags; + + spin_lock_irqsave(&reset_data->lock, flags); + + if (reset_data->stage == -1) + goto out; + +#if KBASE_GPU_RESET_EN + if (kbase_prepare_to_reset_gpu(kbdev)) { + dev_err(kbdev->dev, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", + ZAP_TIMEOUT); + kbase_reset_gpu(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + reset_data->stage = 2; + + out: + spin_unlock_irqrestore(&reset_data->lock, flags); + + return HRTIMER_NORESTART; +} + +void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + struct zap_reset_data reset_data; + unsigned long flags; + + hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + reset_data.timer.function = zap_timeout_callback; + + spin_lock_init(&reset_data.lock); + + reset_data.kbdev = kbdev; + reset_data.stage = 1; + + hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT), + HRTIMER_MODE_REL); + + /* Wait for all jobs to finish, and for the context to be not-scheduled + * (due to kbase_job_zap_context(), we also guarentee it's not in the JS + * policy queue either */ + wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0); + wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, + !kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + spin_lock_irqsave(&reset_data.lock, flags); + if (reset_data.stage == 1) { + /* The timer hasn't run yet - so cancel it */ + reset_data.stage = -1; + } + spin_unlock_irqrestore(&reset_data.lock, flags); + + hrtimer_cancel(&reset_data.timer); + + if (reset_data.stage == 2) { + /* The reset has already started. + * Wait for the reset to complete + */ + wait_event(kbdev->hwaccess.backend.reset_wait, + atomic_read(&kbdev->hwaccess.backend.reset_gpu) + == KBASE_RESET_GPU_NOT_PENDING); + } + destroy_hrtimer_on_stack(&reset_data.timer); + + dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx); + + /* Ensure that the signallers of the waitqs have finished */ + mutex_lock(&kctx->jctx.lock); + mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&kctx->jctx.lock); +} + +u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) +{ + u32 flush_id = 0; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) { + mutex_lock(&kbdev->pm.lock); + if (kbdev->pm.backend.gpu_powered) + flush_id = kbase_reg_read(kbdev, + GPU_CONTROL_REG(LATEST_FLUSH), NULL); + mutex_unlock(&kbdev->pm.lock); + } + + return flush_id; +} + +int kbase_job_slot_init(struct kbase_device *kbdev) +{ +#if KBASE_GPU_RESET_EN + kbdev->hwaccess.backend.reset_workq = alloc_workqueue( + "Mali reset workqueue", 0, 1); + if (NULL == kbdev->hwaccess.backend.reset_workq) + return -EINVAL; + + KBASE_DEBUG_ASSERT(0 == + object_is_on_stack(&kbdev->hwaccess.backend.reset_work)); + INIT_WORK(&kbdev->hwaccess.backend.reset_work, + kbasep_reset_timeout_worker); + + hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + kbdev->hwaccess.backend.reset_timer.function = + kbasep_reset_timer_callback; +#endif + + return 0; +} +KBASE_EXPORT_TEST_API(kbase_job_slot_init); + +void kbase_job_slot_halt(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +void kbase_job_slot_term(struct kbase_device *kbdev) +{ +#if KBASE_GPU_RESET_EN + destroy_workqueue(kbdev->hwaccess.backend.reset_workq); +#endif +} +KBASE_EXPORT_TEST_API(kbase_job_slot_term); + +#if KBASE_GPU_RESET_EN +/** + * kbasep_check_for_afbc_on_slot() - Check whether AFBC is in use on this slot + * @kbdev: kbase device pointer + * @kctx: context to check against + * @js: slot to check + * @target_katom: An atom to check, or NULL if all atoms from @kctx on + * slot @js should be checked + * + * This checks are based upon parameters that would normally be passed to + * kbase_job_slot_hardstop(). + * + * In the event of @target_katom being NULL, this will check the last jobs that + * are likely to be running on the slot to see if a) they belong to kctx, and + * so would be stopped, and b) whether they have AFBC + * + * In that case, It's guaranteed that a job currently executing on the HW with + * AFBC will be detected. However, this is a conservative check because it also + * detects jobs that have just completed too. + * + * Return: true when hard-stop _might_ stop an afbc atom, else false. + */ +static bool kbasep_check_for_afbc_on_slot(struct kbase_device *kbdev, + struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom) +{ + bool ret = false; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* When we have an atom the decision can be made straight away. */ + if (target_katom) + return !!(target_katom->core_req & BASE_JD_REQ_FS_AFBC); + + /* Otherwise, we must chweck the hardware to see if it has atoms from + * this context with AFBC. */ + for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { + struct kbase_jd_atom *katom; + + katom = kbase_gpu_inspect(kbdev, js, i); + if (!katom) + continue; + + /* Ignore atoms from other contexts, they won't be stopped when + * we use this for checking if we should hard-stop them */ + if (katom->kctx != kctx) + continue; + + /* An atom on this slot and this context: check for AFBC */ + if (katom->core_req & BASE_JD_REQ_FS_AFBC) { + ret = true; + break; + } + } + + return ret; +} +#endif /* KBASE_GPU_RESET_EN */ + +/** + * kbase_job_slot_softstop_swflags - Soft-stop a job with flags + * @kbdev: The kbase device + * @js: The job slot to soft-stop + * @target_katom: The job that should be soft-stopped (or NULL for any job) + * @sw_flags: Flags to pass in about the soft-stop + * + * Context: + * The job slot lock must be held when calling this function. + * The job slot must not already be in the process of being soft-stopped. + * + * Soft-stop the specified job slot, with extra information about the stop + * + * Where possible any job in the next register is evicted before the soft-stop. + */ +void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom, u32 sw_flags) +{ + KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK)); + kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom, + JS_COMMAND_SOFT_STOP | sw_flags); +} + +/** + * kbase_job_slot_softstop - Soft-stop the specified job slot + * @kbdev: The kbase device + * @js: The job slot to soft-stop + * @target_katom: The job that should be soft-stopped (or NULL for any job) + * Context: + * The job slot lock must be held when calling this function. + * The job slot must not already be in the process of being soft-stopped. + * + * Where possible any job in the next register is evicted before the soft-stop. + */ +void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom) +{ + kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u); +} + +/** + * kbase_job_slot_hardstop - Hard-stop the specified job slot + * @kctx: The kbase context that contains the job(s) that should + * be hard-stopped + * @js: The job slot to hard-stop + * @target_katom: The job that should be hard-stopped (or NULL for all + * jobs from the context) + * Context: + * The job slot lock must be held when calling this function. + */ +void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom) +{ + struct kbase_device *kbdev = kctx->kbdev; + bool stopped; +#if KBASE_GPU_RESET_EN + /* We make the check for AFBC before evicting/stopping atoms. Note + * that no other thread can modify the slots whilst we have the + * hwaccess_lock. */ + int needs_workaround_for_afbc = + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3542) + && kbasep_check_for_afbc_on_slot(kbdev, kctx, js, + target_katom); +#endif + + stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js, + target_katom, + JS_COMMAND_HARD_STOP); +#if KBASE_GPU_RESET_EN + if (stopped && (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) || + kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) || + needs_workaround_for_afbc)) { + /* MIDBASE-2916 if a fragment job with AFBC encoding is + * hardstopped, ensure to do a soft reset also in order to + * clear the GPU status. + * Workaround for HW issue 8401 has an issue,so after + * hard-stopping just reset the GPU. This will ensure that the + * jobs leave the GPU.*/ + if (kbase_prepare_to_reset_gpu_locked(kbdev)) { + dev_err(kbdev->dev, "Issueing GPU soft-reset after hard stopping due to hardware issue"); + kbase_reset_gpu_locked(kbdev); + } + } +#endif +} + +/** + * kbase_job_check_enter_disjoint - potentiall enter disjoint mode + * @kbdev: kbase device + * @action: the event which has occurred + * @core_reqs: core requirements of the atom + * @target_katom: the atom which is being affected + * + * For a certain soft/hard-stop action, work out whether to enter disjoint + * state. + * + * This does not register multiple disjoint events if the atom has already + * started a disjoint period + * + * @core_reqs can be supplied as 0 if the atom had not started on the hardware + * (and so a 'real' soft/hard-stop was not required, but it still interrupted + * flow, perhaps on another context) + * + * kbase_job_check_leave_disjoint() should be used to end the disjoint + * state when the soft/hard-stop action is complete + */ +void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, + base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom) +{ + u32 hw_action = action & JS_COMMAND_MASK; + + /* For hard-stop, don't enter if hard-stop not allowed */ + if (hw_action == JS_COMMAND_HARD_STOP && + !kbasep_hard_stop_allowed(kbdev, core_reqs)) + return; + + /* For soft-stop, don't enter if soft-stop not allowed, or isn't + * causing disjoint */ + if (hw_action == JS_COMMAND_SOFT_STOP && + !(kbasep_soft_stop_allowed(kbdev, target_katom) && + (action & JS_COMMAND_SW_CAUSES_DISJOINT))) + return; + + /* Nothing to do if already logged disjoint state on this atom */ + if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) + return; + + target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT; + kbase_disjoint_state_up(kbdev); +} + +/** + * kbase_job_check_enter_disjoint - potentially leave disjoint state + * @kbdev: kbase device + * @target_katom: atom which is finishing + * + * Work out whether to leave disjoint state when finishing an atom that was + * originated by kbase_job_check_enter_disjoint(). + */ +void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, + struct kbase_jd_atom *target_katom) +{ + if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) { + target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT; + kbase_disjoint_state_down(kbdev); + } +} + + +#if KBASE_GPU_RESET_EN +static void kbase_debug_dump_registers(struct kbase_device *kbdev) +{ + int i; + + kbase_io_history_dump(kbdev); + + dev_err(kbdev->dev, "Register state:"); + dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL)); + dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x", + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL), + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL)); + for (i = 0; i < 3; i++) { + dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x", + i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS), + NULL), + i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO), + NULL)); + } + dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", + kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL)); + dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL), + kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL), + kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL)); + dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL)); + dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL)); + dev_err(kbdev->dev, " TILER_CONFIG=0x%08x JM_CONFIG=0x%08x", + kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG), NULL)); +} + +static void kbasep_reset_timeout_worker(struct work_struct *data) +{ + unsigned long flags; + struct kbase_device *kbdev; + ktime_t end_timestamp = ktime_get(); + struct kbasep_js_device_data *js_devdata; + bool try_schedule = false; + bool silent = false; + u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; + + KBASE_DEBUG_ASSERT(data); + + kbdev = container_of(data, struct kbase_device, + hwaccess.backend.reset_work); + + KBASE_DEBUG_ASSERT(kbdev); + js_devdata = &kbdev->js_data; + + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_SILENT) + silent = true; + + KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); + + /* Suspend vinstr. + * This call will block until vinstr is suspended. */ + kbase_vinstr_suspend(kbdev->vinstr_ctx); + + /* Make sure the timer has completed - this cannot be done from + * interrupt context, so this cannot be done within + * kbasep_try_reset_gpu_early. */ + hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); + + if (kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + /* This would re-activate the GPU. Since it's already idle, + * there's no need to reset it */ + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING); + kbase_disjoint_state_down(kbdev); + wake_up(&kbdev->hwaccess.backend.reset_wait); + kbase_vinstr_resume(kbdev->vinstr_ctx); + return; + } + + KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false); + + spin_lock_irqsave(&kbdev->hwcnt.lock, flags); + spin_lock(&kbdev->hwaccess_lock); + spin_lock(&kbdev->mmu_mask_change); + /* We're about to flush out the IRQs and their bottom half's */ + kbdev->irq_reset_flush = true; + + /* Disable IRQ to avoid IRQ handlers to kick in after releasing the + * spinlock; this also clears any outstanding interrupts */ + kbase_pm_disable_interrupts_nolock(kbdev); + + spin_unlock(&kbdev->mmu_mask_change); + spin_unlock(&kbdev->hwaccess_lock); + spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); + + /* Ensure that any IRQ handlers have finished + * Must be done without any locks IRQ handlers will take */ + kbase_synchronize_irqs(kbdev); + + /* Flush out any in-flight work items */ + kbase_flush_mmu_wqs(kbdev); + + /* The flush has completed so reset the active indicator */ + kbdev->irq_reset_flush = false; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { + /* Ensure that L2 is not transitioning when we send the reset + * command */ + while (--max_loops && kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_L2)) + ; + + WARN(!max_loops, "L2 power transition timed out while trying to reset\n"); + } + + mutex_lock(&kbdev->pm.lock); + /* We hold the pm lock, so there ought to be a current policy */ + KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy); + + /* All slot have been soft-stopped and we've waited + * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we + * assume that anything that is still left on the GPU is stuck there and + * we'll kill it when we reset the GPU */ + + if (!silent) + dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", + RESET_TIMEOUT); + + /* Output the state of some interesting registers to help in the + * debugging of GPU resets */ + if (!silent) + kbase_debug_dump_registers(kbdev); + + /* Complete any jobs that were still on the GPU */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->protected_mode = false; + kbase_backend_reset(kbdev, &end_timestamp); + kbase_pm_metrics_update(kbdev, NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Reset the GPU */ + kbase_pm_init_hw(kbdev, 0); + + mutex_unlock(&kbdev->pm.lock); + + mutex_lock(&js_devdata->runpool_mutex); + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_restore_all_as(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_pm_enable_interrupts(kbdev); + + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING); + + kbase_disjoint_state_down(kbdev); + + wake_up(&kbdev->hwaccess.backend.reset_wait); + if (!silent) + dev_err(kbdev->dev, "Reset complete"); + + if (js_devdata->nr_contexts_pullable > 0 && !kbdev->poweroff_pending) + try_schedule = true; + + mutex_unlock(&js_devdata->runpool_mutex); + + mutex_lock(&kbdev->pm.lock); + + /* Find out what cores are required now */ + kbase_pm_update_cores_state(kbdev); + + /* Synchronously request and wait for those cores, because if + * instrumentation is enabled it would need them immediately. */ + kbase_pm_check_transitions_sync(kbdev); + + mutex_unlock(&kbdev->pm.lock); + + /* Try submitting some jobs to restart processing */ + if (try_schedule) { + KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, + 0); + kbase_js_sched_all(kbdev); + } + + /* Process any pending slot updates */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_backend_slot_update(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_pm_context_idle(kbdev); + + /* Release vinstr */ + kbase_vinstr_resume(kbdev->vinstr_ctx); + + KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); +} + +static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) +{ + struct kbase_device *kbdev = container_of(timer, struct kbase_device, + hwaccess.backend.reset_timer); + + KBASE_DEBUG_ASSERT(kbdev); + + /* Reset still pending? */ + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) == + KBASE_RESET_GPU_COMMITTED) + queue_work(kbdev->hwaccess.backend.reset_workq, + &kbdev->hwaccess.backend.reset_work); + + return HRTIMER_NORESTART; +} + +/* + * If all jobs are evicted from the GPU then we can reset the GPU + * immediately instead of waiting for the timeout to elapse + */ + +static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) +{ + int i; + int pending_jobs = 0; + + KBASE_DEBUG_ASSERT(kbdev); + + /* Count the number of jobs */ + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i); + + if (pending_jobs > 0) { + /* There are still jobs on the GPU - wait */ + return; + } + + /* To prevent getting incorrect registers when dumping failed job, + * skip early reset. + */ + if (kbdev->job_fault_debug != false) + return; + + /* Check that the reset has been committed to (i.e. kbase_reset_gpu has + * been called), and that no other thread beat this thread to starting + * the reset */ + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != + KBASE_RESET_GPU_COMMITTED) { + /* Reset has already occurred */ + return; + } + + queue_work(kbdev->hwaccess.backend.reset_workq, + &kbdev->hwaccess.backend.reset_work); +} + +static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) +{ + unsigned long flags; + struct kbasep_js_device_data *js_devdata; + + js_devdata = &kbdev->js_data; + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbasep_try_reset_gpu_early_locked(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +/** + * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU + * @kbdev: kbase device + * + * This function just soft-stops all the slots to ensure that as many jobs as + * possible are saved. + * + * Return: + * The function returns a boolean which should be interpreted as follows: + * true - Prepared for reset, kbase_reset_gpu_locked should be called. + * false - Another thread is performing a reset, kbase_reset_gpu should + * not be called. + */ +bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) +{ + int i; + + KBASE_DEBUG_ASSERT(kbdev); + + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING, + KBASE_RESET_GPU_PREPARED) != + KBASE_RESET_GPU_NOT_PENDING) { + /* Some other thread is already resetting the GPU */ + return false; + } + + kbase_disjoint_state_up(kbdev); + + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) + kbase_job_slot_softstop(kbdev, i, NULL); + + return true; +} + +bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) +{ + unsigned long flags; + bool ret; + struct kbasep_js_device_data *js_devdata; + + js_devdata = &kbdev->js_data; + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + ret = kbase_prepare_to_reset_gpu_locked(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return ret; +} +KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); + +/* + * This function should be called after kbase_prepare_to_reset_gpu if it + * returns true. It should never be called without a corresponding call to + * kbase_prepare_to_reset_gpu. + * + * After this function is called (or not called if kbase_prepare_to_reset_gpu + * returned false), the caller should wait for + * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset + * has completed. + */ +void kbase_reset_gpu(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + + /* Note this is an assert/atomic_set because it is a software issue for + * a race to be occuring here */ + KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_PREPARED); + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED); + + dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", + kbdev->reset_timeout_ms); + + hrtimer_start(&kbdev->hwaccess.backend.reset_timer, + HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), + HRTIMER_MODE_REL); + + /* Try resetting early */ + kbasep_try_reset_gpu_early(kbdev); +} +KBASE_EXPORT_TEST_API(kbase_reset_gpu); + +void kbase_reset_gpu_locked(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + + /* Note this is an assert/atomic_set because it is a software issue for + * a race to be occuring here */ + KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_PREPARED); + atomic_set(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_COMMITTED); + + dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", + kbdev->reset_timeout_ms); + hrtimer_start(&kbdev->hwaccess.backend.reset_timer, + HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), + HRTIMER_MODE_REL); + + /* Try resetting early */ + kbasep_try_reset_gpu_early_locked(kbdev); +} + +void kbase_reset_gpu_silent(struct kbase_device *kbdev) +{ + if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, + KBASE_RESET_GPU_NOT_PENDING, + KBASE_RESET_GPU_SILENT) != + KBASE_RESET_GPU_NOT_PENDING) { + /* Some other thread is already resetting the GPU */ + return; + } + + kbase_disjoint_state_up(kbdev); + + queue_work(kbdev->hwaccess.backend.reset_workq, + &kbdev->hwaccess.backend.reset_work); +} + +bool kbase_reset_gpu_active(struct kbase_device *kbdev) +{ + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == + KBASE_RESET_GPU_NOT_PENDING) + return false; + + return true; +} +#endif /* KBASE_GPU_RESET_EN */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h new file mode 100755 index 000000000000..1f382b3c1af4 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h @@ -0,0 +1,164 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Job Manager backend-specific low-level APIs. + */ + +#ifndef _KBASE_JM_HWACCESS_H_ +#define _KBASE_JM_HWACCESS_H_ + +#include +#include +#include + +#include + +/** + * kbase_job_submit_nolock() - Submit a job to a certain job-slot + * @kbdev: Device pointer + * @katom: Atom to submit + * @js: Job slot to submit on + * + * The caller must check kbasep_jm_is_submit_slots_free() != false before + * calling this. + * + * The following locking conditions are made on the caller: + * - it must hold the hwaccess_lock + */ +void kbase_job_submit_nolock(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, int js); + +/** + * kbase_job_done_slot() - Complete the head job on a particular job-slot + * @kbdev: Device pointer + * @s: Job slot + * @completion_code: Completion code of job reported by GPU + * @job_tail: Job tail address reported by GPU + * @end_timestamp: Timestamp of job completion + */ +void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, + u64 job_tail, ktime_t *end_timestamp); + +#ifdef CONFIG_GPU_TRACEPOINTS +static inline char *kbasep_make_job_slot_string(int js, char *js_string, + size_t js_size) +{ + snprintf(js_string, js_size, "job_slot_%i", js); + return js_string; +} +#endif + +/** + * kbase_job_hw_submit() - Submit a job to the GPU + * @kbdev: Device pointer + * @katom: Atom to submit + * @js: Job slot to submit on + * + * The caller must check kbasep_jm_is_submit_slots_free() != false before + * calling this. + * + * The following locking conditions are made on the caller: + * - it must hold the hwaccess_lock + */ +void kbase_job_hw_submit(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + int js); + +/** + * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop + * on the specified atom + * @kbdev: Device pointer + * @js: Job slot to stop on + * @action: The action to perform, either JSn_COMMAND_HARD_STOP or + * JSn_COMMAND_SOFT_STOP + * @core_reqs: Core requirements of atom to stop + * @target_katom: Atom to stop + * + * The following locking conditions are made on the caller: + * - it must hold the hwaccess_lock + */ +void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, + int js, + u32 action, + base_jd_core_req core_reqs, + struct kbase_jd_atom *target_katom); + +/** + * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job + * slot belonging to a given context. + * @kbdev: Device pointer + * @kctx: Context pointer. May be NULL + * @katom: Specific atom to stop. May be NULL + * @js: Job slot to hard stop + * @action: The action to perform, either JSn_COMMAND_HARD_STOP or + * JSn_COMMAND_SOFT_STOP + * + * If no context is provided then all jobs on the slot will be soft or hard + * stopped. + * + * If a katom is provided then only that specific atom will be stopped. In this + * case the kctx parameter is ignored. + * + * Jobs that are on the slot but are not yet on the GPU will be unpulled and + * returned to the job scheduler. + * + * Return: true if an atom was stopped, false otherwise + */ +bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js, + struct kbase_jd_atom *katom, + u32 action); + +/** + * kbase_job_slot_init - Initialise job slot framework + * @kbdev: Device pointer + * + * Called on driver initialisation + * + * Return: 0 on success + */ +int kbase_job_slot_init(struct kbase_device *kbdev); + +/** + * kbase_job_slot_halt - Halt the job slot framework + * @kbdev: Device pointer + * + * Should prevent any further job slot processing + */ +void kbase_job_slot_halt(struct kbase_device *kbdev); + +/** + * kbase_job_slot_term - Terminate job slot framework + * @kbdev: Device pointer + * + * Called on driver termination + */ +void kbase_job_slot_term(struct kbase_device *kbdev); + +/** + * kbase_gpu_cacheclean - Cause a GPU cache clean & flush + * @kbdev: Device pointer + * + * Caller must not be in IRQ context + */ +void kbase_gpu_cacheclean(struct kbase_device *kbdev); + +#endif /* _KBASE_JM_HWACCESS_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c new file mode 100755 index 000000000000..4b4541660ec4 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c @@ -0,0 +1,1952 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend specific APIs + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Return whether the specified ringbuffer is empty. HW access lock must be + * held */ +#define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) +/* Return number of atoms currently in the specified ringbuffer. HW access lock + * must be held */ +#define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) + +static void kbase_gpu_release_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + ktime_t *end_timestamp); + +/** + * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer + * @kbdev: Device pointer + * @katom: Atom to enqueue + * + * Context: Caller must hold the HW access lock + */ +static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; + + WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; + rb->write_idx++; + + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; +} + +/** + * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once + * it has been completed + * @kbdev: Device pointer + * @js: Job slot to remove atom from + * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in + * which case current time will be used. + * + * Context: Caller must hold the HW access lock + * + * Return: Atom removed from ringbuffer + */ +static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, + int js, + ktime_t *end_timestamp) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; + struct kbase_jd_atom *katom; + + if (SLOT_RB_EMPTY(rb)) { + WARN(1, "GPU ringbuffer unexpectedly empty\n"); + return NULL; + } + + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; + + kbase_gpu_release_atom(kbdev, katom, end_timestamp); + + rb->read_idx++; + + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; + + kbase_js_debug_log_current_affinities(kbdev); + + return katom; +} + +struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, + int idx) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if ((SLOT_RB_ENTRIES(rb) - 1) < idx) + return NULL; /* idx out of range */ + + return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; +} + +struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, + int js) +{ + return kbase_gpu_inspect(kbdev, js, 0); +} + +struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, + int js) +{ + struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; + + if (SLOT_RB_EMPTY(rb)) + return NULL; + + return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; +} + +/** + * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently + * on the GPU + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return: true if there are atoms on the GPU for slot js, + * false otherwise + */ +static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js) +{ + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (!katom) + return false; + if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED || + katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY) + return true; + } + + return false; +} + +/** + * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms + * currently on the GPU + * @kbdev: Device pointer + * + * Return: true if there are any atoms on the GPU, false otherwise + */ +static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) +{ + int js; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) + return true; + } + } + return false; +} + +int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js) +{ + int nr = 0; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (katom && (katom->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED)) + nr++; + } + + return nr; +} + +int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js) +{ + int nr = 0; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + if (kbase_gpu_inspect(kbdev, js, i)) + nr++; + } + + return nr; +} + +static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js, + enum kbase_atom_gpu_rb_state min_rb_state) +{ + int nr = 0; + int i; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); + + if (katom && (katom->gpu_rb_state >= min_rb_state)) + nr++; + } + + return nr; +} + +/** + * check_secure_atom - Check if the given atom is in the given secure state and + * has a ringbuffer state of at least + * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION + * @katom: Atom pointer + * @secure: Desired secure state + * + * Return: true if atom is in the given state, false otherwise + */ +static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) +{ + if (katom->gpu_rb_state >= + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && + ((kbase_jd_katom_is_protected(katom) && secure) || + (!kbase_jd_katom_is_protected(katom) && !secure))) + return true; + + return false; +} + +/** + * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given + * secure state in the ringbuffers of at least + * state + * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE + * @kbdev: Device pointer + * @secure: Desired secure state + * + * Return: true if any atoms are in the given state, false otherwise + */ +static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, + bool secure) +{ + int js, i; + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + for (i = 0; i < SLOT_RB_SIZE; i++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, + js, i); + + if (katom) { + if (check_secure_atom(katom, secure)) + return true; + } + } + } + + return false; +} + +int kbase_backend_slot_free(struct kbase_device *kbdev, int js) +{ + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != + KBASE_RESET_GPU_NOT_PENDING) { + /* The GPU is being reset - so prevent submission */ + return 0; + } + + return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); +} + + +static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev, + int js, + struct kbase_jd_atom *katom) +{ + /* The most recently checked affinity. Having this at this scope allows + * us to guarantee that we've checked the affinity in this function + * call. + */ + u64 recently_chosen_affinity = 0; + bool chosen_affinity = false; + bool retry; + + do { + retry = false; + + /* NOTE: The following uses a number of FALLTHROUGHs to optimize + * the calls to this function. Ending of the function is + * indicated by BREAK OUT */ + switch (katom->coreref_state) { + /* State when job is first attempted to be run */ + case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: + KBASE_DEBUG_ASSERT(katom->affinity == 0); + + /* Compute affinity */ + if (false == kbase_js_choose_affinity( + &recently_chosen_affinity, kbdev, katom, + js)) { + /* No cores are currently available */ + /* *** BREAK OUT: No state transition *** */ + break; + } + + chosen_affinity = true; + + /* Request the cores */ + kbase_pm_request_cores(kbdev, + katom->core_req & BASE_JD_REQ_T, + recently_chosen_affinity); + + katom->affinity = recently_chosen_affinity; + + /* Proceed to next state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: + { + enum kbase_pm_cores_ready cores_ready; + + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + + cores_ready = kbase_pm_register_inuse_cores( + kbdev, + katom->core_req & BASE_JD_REQ_T, + katom->affinity); + if (cores_ready == KBASE_NEW_AFFINITY) { + /* Affinity no longer valid - return to + * previous state */ + kbasep_js_job_check_deref_cores(kbdev, + katom); + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_REGISTER_INUSE_FAILED, + katom->kctx, katom, + katom->jc, js, + (u32) katom->affinity); + /* *** BREAK OUT: Return to previous + * state, retry *** */ + retry = true; + break; + } + if (cores_ready == KBASE_CORES_NOT_READY) { + /* Stay in this state and return, to + * retry at this state later */ + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_REGISTER_INUSE_FAILED, + katom->kctx, katom, + katom->jc, js, + (u32) katom->affinity); + /* *** BREAK OUT: No state transition + * *** */ + break; + } + /* Proceed to next state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; + } + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + + /* Optimize out choosing the affinity twice in the same + * function call */ + if (chosen_affinity == false) { + /* See if the affinity changed since a previous + * call. */ + if (false == kbase_js_choose_affinity( + &recently_chosen_affinity, + kbdev, katom, js)) { + /* No cores are currently available */ + kbasep_js_job_check_deref_cores(kbdev, + katom); + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_REQUEST_ON_RECHECK_FAILED, + katom->kctx, katom, + katom->jc, js, + (u32) recently_chosen_affinity); + /* *** BREAK OUT: Transition to lower + * state *** */ + break; + } + chosen_affinity = true; + } + + /* Now see if this requires a different set of cores */ + if (recently_chosen_affinity != katom->affinity) { + enum kbase_pm_cores_ready cores_ready; + + kbase_pm_request_cores(kbdev, + katom->core_req & BASE_JD_REQ_T, + recently_chosen_affinity); + + /* Register new cores whilst we still hold the + * old ones, to minimize power transitions */ + cores_ready = + kbase_pm_register_inuse_cores(kbdev, + katom->core_req & BASE_JD_REQ_T, + recently_chosen_affinity); + kbasep_js_job_check_deref_cores(kbdev, katom); + + /* Fixup the state that was reduced by + * deref_cores: */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; + katom->affinity = recently_chosen_affinity; + if (cores_ready == KBASE_NEW_AFFINITY) { + /* Affinity no longer valid - return to + * previous state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; + + kbasep_js_job_check_deref_cores(kbdev, + katom); + + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_REGISTER_INUSE_FAILED, + katom->kctx, katom, + katom->jc, js, + (u32) katom->affinity); + /* *** BREAK OUT: Return to previous + * state, retry *** */ + retry = true; + break; + } + /* Now might be waiting for powerup again, with + * a new affinity */ + if (cores_ready == KBASE_CORES_NOT_READY) { + /* Return to previous state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_REGISTER_ON_RECHECK_FAILED, + katom->kctx, katom, + katom->jc, js, + (u32) katom->affinity); + /* *** BREAK OUT: Transition to lower + * state *** */ + break; + } + } + /* Proceed to next state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS; + + /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS: + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + KBASE_DEBUG_ASSERT(katom->affinity == + recently_chosen_affinity); + + /* Note: this is where the caller must've taken the + * hwaccess_lock */ + + /* Check for affinity violations - if there are any, + * then we just ask the caller to requeue and try again + * later */ + if (kbase_js_affinity_would_violate(kbdev, js, + katom->affinity) != false) { + /* Return to previous state */ + katom->coreref_state = + KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; + /* *** BREAK OUT: Transition to lower state *** + */ + KBASE_TRACE_ADD_SLOT_INFO(kbdev, + JS_CORE_REF_AFFINITY_WOULD_VIOLATE, + katom->kctx, katom, katom->jc, js, + (u32) katom->affinity); + break; + } + + /* No affinity violations would result, so the cores are + * ready */ + katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY; + /* *** BREAK OUT: Cores Ready *** */ + break; + + default: + KBASE_DEBUG_ASSERT_MSG(false, + "Unhandled kbase_atom_coreref_state %d", + katom->coreref_state); + break; + } + } while (retry != false); + + return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY); +} + +static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(katom != NULL); + + switch (katom->coreref_state) { + case KBASE_ATOM_COREREF_STATE_READY: + /* State where atom was submitted to the HW - just proceed to + * power-down */ + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + + /* fallthrough */ + + case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: + /* State where cores were registered */ + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T, + katom->affinity); + + break; + + case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: + /* State where cores were requested, but not registered */ + KBASE_DEBUG_ASSERT(katom->affinity != 0 || + (katom->core_req & BASE_JD_REQ_T)); + kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T, + katom->affinity); + break; + + case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: + /* Initial state - nothing required */ + KBASE_DEBUG_ASSERT(katom->affinity == 0); + break; + + default: + KBASE_DEBUG_ASSERT_MSG(false, + "Unhandled coreref_state: %d", + katom->coreref_state); + break; + } + + katom->affinity = 0; + katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; +} + +static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev, + base_jd_core_req core_req, u64 affinity, + enum kbase_atom_coreref_state coreref_state) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + switch (coreref_state) { + case KBASE_ATOM_COREREF_STATE_READY: + /* State where atom was submitted to the HW - just proceed to + * power-down */ + KBASE_DEBUG_ASSERT(affinity != 0 || + (core_req & BASE_JD_REQ_T)); + + /* fallthrough */ + + case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: + /* State where cores were registered */ + KBASE_DEBUG_ASSERT(affinity != 0 || + (core_req & BASE_JD_REQ_T)); + kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T, + affinity); + + break; + + case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: + /* State where cores were requested, but not registered */ + KBASE_DEBUG_ASSERT(affinity != 0 || + (core_req & BASE_JD_REQ_T)); + kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T, + affinity); + break; + + case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: + /* Initial state - nothing required */ + KBASE_DEBUG_ASSERT(affinity == 0); + break; + + default: + KBASE_DEBUG_ASSERT_MSG(false, + "Unhandled coreref_state: %d", + coreref_state); + break; + } +} + +static void kbase_gpu_release_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + ktime_t *end_timestamp) +{ + struct kbase_context *kctx = katom->kctx; + + switch (katom->gpu_rb_state) { + case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: + /* Should be impossible */ + WARN(1, "Attempting to release atom not in ringbuffer\n"); + break; + + case KBASE_ATOM_GPU_RB_SUBMITTED: + /* Inform power management at start/finish of atom so it can + * update its GPU utilisation metrics. Mark atom as not + * submitted beforehand. */ + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; + kbase_pm_metrics_update(kbdev, end_timestamp); + + if (katom->core_req & BASE_JD_REQ_PERMON) + kbase_pm_release_gpu_cycle_counter_nolock(kbdev); + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + + KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, + &kbdev->gpu_props.props.raw_props.js_features + [katom->slot_nr]); + KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); + KBASE_TLSTREAM_TL_NRET_CTX_LPU(kctx, + &kbdev->gpu_props.props.raw_props.js_features + [katom->slot_nr]); + /* fallthrough */ + case KBASE_ATOM_GPU_RB_READY: + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: + kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr, + katom->affinity); + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: + break; + + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: + if (katom->protected_state.enter != + KBASE_ATOM_ENTER_PROTECTED_CHECK || + katom->protected_state.exit != + KBASE_ATOM_EXIT_PROTECTED_CHECK) + kbdev->protected_mode_transition = false; + + if (kbase_jd_katom_is_protected(katom) && + (katom->protected_state.enter == + KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) { + kbase_vinstr_resume(kbdev->vinstr_ctx); + + /* Go back to configured model for IPA */ + kbase_ipa_model_use_configured_locked(kbdev); + } + + + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_RETURN_TO_JS: + break; + } + + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; + katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; +} + +static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + kbase_gpu_release_atom(kbdev, katom, NULL); + katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; +} + +static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + bool slot_busy[3]; + + if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) + return true; + slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0, + KBASE_ATOM_GPU_RB_WAITING_AFFINITY); + slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1, + KBASE_ATOM_GPU_RB_WAITING_AFFINITY); + slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2, + KBASE_ATOM_GPU_RB_WAITING_AFFINITY); + + if ((js == 2 && !(slot_busy[0] || slot_busy[1])) || + (js != 2 && !slot_busy[2])) + return true; + + /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */ + if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) || + kbase_gpu_atoms_submitted(kbdev, 1) || + backend->rmu_workaround_flag)) + return false; + + /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */ + if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) || + !backend->rmu_workaround_flag)) + return false; + + backend->rmu_workaround_flag = !backend->rmu_workaround_flag; + + return true; +} + +/** + * other_slots_busy - Determine if any job slots other than @js are currently + * running atoms + * @kbdev: Device pointer + * @js: Job slot + * + * Return: true if any slots other than @js are busy, false otherwise + */ +static inline bool other_slots_busy(struct kbase_device *kbdev, int js) +{ + int slot; + + for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) { + if (slot == js) + continue; + + if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot, + KBASE_ATOM_GPU_RB_SUBMITTED)) + return true; + } + + return false; +} + +static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) +{ + return kbdev->protected_mode; +} + +static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) +{ + int err = -EINVAL; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ONCE(!kbdev->protected_ops, + "Cannot enter protected mode: protected callbacks not specified.\n"); + + /* + * When entering into protected mode, we must ensure that the + * GPU is not operating in coherent mode as well. This is to + * ensure that no protected memory can be leaked. + */ + if (kbdev->system_coherency == COHERENCY_ACE) + kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); + + if (kbdev->protected_ops) { + /* Switch GPU to protected mode */ + err = kbdev->protected_ops->protected_mode_enable( + kbdev->protected_dev); + + if (err) + dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", + err); + else + kbdev->protected_mode = true; + } + + return err; +} + +static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ONCE(!kbdev->protected_ops, + "Cannot exit protected mode: protected callbacks not specified.\n"); + + if (!kbdev->protected_ops) + return -EINVAL; + + /* The protected mode disable callback will be called as part of reset + */ + kbase_reset_gpu_silent(kbdev); + + return 0; +} + +static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, + struct kbase_jd_atom **katom, int idx, int js) +{ + int err = 0; + + switch (katom[idx]->protected_state.enter) { + case KBASE_ATOM_ENTER_PROTECTED_CHECK: + KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev); + /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV + * should ensure that we are not already transitiong, and that + * there are no atoms currently on the GPU. */ + WARN_ON(kbdev->protected_mode_transition); + WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); + + kbdev->protected_mode_transition = true; + katom[idx]->protected_state.enter = + KBASE_ATOM_ENTER_PROTECTED_VINSTR; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_ENTER_PROTECTED_VINSTR: + if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) { + /* + * We can't switch now because + * the vinstr core state switch + * is not done yet. + */ + return -EAGAIN; + } + + /* Use generic model for IPA in protected mode */ + kbase_ipa_model_use_fallback_locked(kbdev); + + /* Once reaching this point GPU must be + * switched to protected mode or vinstr + * re-enabled. */ + + /* + * Not in correct mode, begin protected mode switch. + * Entering protected mode requires us to power down the L2, + * and drop out of fully coherent mode. + */ + katom[idx]->protected_state.enter = + KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; + + kbase_pm_update_cores_state_nolock(kbdev); + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: + /* Avoid unnecessary waiting on non-ACE platforms. */ + if (kbdev->current_gpu_coherency_mode == COHERENCY_ACE) { + if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || + kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { + /* + * The L2 is still powered, wait for all the users to + * finish with it before doing the actual reset. + */ + return -EAGAIN; + } + } + + katom[idx]->protected_state.enter = + KBASE_ATOM_ENTER_PROTECTED_FINISHED; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_ENTER_PROTECTED_FINISHED: + + /* No jobs running, so we can switch GPU mode right now. */ + err = kbase_gpu_protected_mode_enter(kbdev); + + /* + * Regardless of result, we are no longer transitioning + * the GPU. + */ + kbdev->protected_mode_transition = false; + KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev); + if (err) { + /* + * Failed to switch into protected mode, resume + * vinstr core and fail atom. + */ + kbase_vinstr_resume(kbdev->vinstr_ctx); + katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; + kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); + /* Only return if head atom or previous atom + * already removed - as atoms must be returned + * in order. */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, katom[idx]); + } + + /* Go back to configured model for IPA */ + kbase_ipa_model_use_configured_locked(kbdev); + + return -EINVAL; + } + + /* Protected mode sanity checks. */ + KBASE_DEBUG_ASSERT_MSG( + kbase_jd_katom_is_protected(katom[idx]) == + kbase_gpu_in_protected_mode(kbdev), + "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", + kbase_jd_katom_is_protected(katom[idx]), + kbase_gpu_in_protected_mode(kbdev)); + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_READY; + } + + return 0; +} + +static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, + struct kbase_jd_atom **katom, int idx, int js) +{ + int err = 0; + + + switch (katom[idx]->protected_state.exit) { + case KBASE_ATOM_EXIT_PROTECTED_CHECK: + KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev); + /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV + * should ensure that we are not already transitiong, and that + * there are no atoms currently on the GPU. */ + WARN_ON(kbdev->protected_mode_transition); + WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); + + /* + * Exiting protected mode requires a reset, but first the L2 + * needs to be powered down to ensure it's not active when the + * reset is issued. + */ + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; + + kbdev->protected_mode_transition = true; + kbase_pm_update_cores_state_nolock(kbdev); + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: + if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || + kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { + /* + * The L2 is still powered, wait for all the users to + * finish with it before doing the actual reset. + */ + return -EAGAIN; + } + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_RESET; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_EXIT_PROTECTED_RESET: + /* Issue the reset to the GPU */ + err = kbase_gpu_protected_mode_reset(kbdev); + + if (err) { + kbdev->protected_mode_transition = false; + + /* Failed to exit protected mode, fail atom */ + katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; + kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); + /* Only return if head atom or previous atom + * already removed - as atoms must be returned + * in order */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, katom[idx]); + } + + kbase_vinstr_resume(kbdev->vinstr_ctx); + + /* Use generic model for IPA in protected mode */ + kbase_ipa_model_use_fallback_locked(kbdev); + + return -EINVAL; + } + + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: + /* A GPU reset is issued when exiting protected mode. Once the + * reset is done all atoms' state will also be reset. For this + * reason, if the atom is still in this state we can safely + * say that the reset has not completed i.e., we have not + * finished exiting protected mode yet. + */ + return -EAGAIN; + } + + return 0; +} + +void kbase_backend_slot_update(struct kbase_device *kbdev) +{ + int js; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + struct kbase_jd_atom *katom[2]; + int idx; + + katom[0] = kbase_gpu_inspect(kbdev, js, 0); + katom[1] = kbase_gpu_inspect(kbdev, js, 1); + WARN_ON(katom[1] && !katom[0]); + + for (idx = 0; idx < SLOT_RB_SIZE; idx++) { + bool cores_ready; + int ret; + + if (!katom[idx]) + continue; + + switch (katom[idx]->gpu_rb_state) { + case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: + /* Should be impossible */ + WARN(1, "Attempting to update atom not in ringbuffer\n"); + break; + + case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: + if (katom[idx]->atom_flags & + KBASE_KATOM_FLAG_X_DEP_BLOCKED) + break; + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: + if (kbase_gpu_check_secure_atoms(kbdev, + !kbase_jd_katom_is_protected( + katom[idx]))) + break; + + if ((idx == 1) && (kbase_jd_katom_is_protected( + katom[0]) != + kbase_jd_katom_is_protected( + katom[1]))) + break; + + if (kbdev->protected_mode_transition) + break; + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: + + /* + * Exiting protected mode must be done before + * the references on the cores are taken as + * a power down the L2 is required which + * can't happen after the references for this + * atom are taken. + */ + + if (!kbase_gpu_in_protected_mode(kbdev) && + kbase_jd_katom_is_protected(katom[idx])) { + /* Atom needs to transition into protected mode. */ + ret = kbase_jm_enter_protected_mode(kbdev, + katom, idx, js); + if (ret) + break; + } else if (kbase_gpu_in_protected_mode(kbdev) && + !kbase_jd_katom_is_protected(katom[idx])) { + /* Atom needs to transition out of protected mode. */ + ret = kbase_jm_exit_protected_mode(kbdev, + katom, idx, js); + if (ret) + break; + } + katom[idx]->protected_state.exit = + KBASE_ATOM_EXIT_PROTECTED_CHECK; + + /* Atom needs no protected mode transition. */ + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: + if (katom[idx]->will_fail_event_code) { + kbase_gpu_mark_atom_for_return(kbdev, + katom[idx]); + /* Set EVENT_DONE so this atom will be + completed, not unpulled. */ + katom[idx]->event_code = + BASE_JD_EVENT_DONE; + /* Only return if head atom or previous + * atom already removed - as atoms must + * be returned in order. */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, katom[idx]); + } + break; + } + + cores_ready = + kbasep_js_job_check_ref_cores(kbdev, js, + katom[idx]); + + if (katom[idx]->event_code == + BASE_JD_EVENT_PM_EVENT) { + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_RETURN_TO_JS; + break; + } + + if (!cores_ready) + break; + + kbase_js_affinity_retain_slot_cores(kbdev, js, + katom[idx]->affinity); + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_WAITING_AFFINITY; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: + if (!kbase_gpu_rmu_workaround(kbdev, js)) + break; + + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_READY; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_READY: + + if (idx == 1) { + /* Only submit if head atom or previous + * atom already submitted */ + if ((katom[0]->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED && + katom[0]->gpu_rb_state != + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) + break; + + /* If intra-slot serialization in use + * then don't submit atom to NEXT slot + */ + if (kbdev->serialize_jobs & + KBASE_SERIALIZE_INTRA_SLOT) + break; + } + + /* If inter-slot serialization in use then don't + * submit atom if any other slots are in use */ + if ((kbdev->serialize_jobs & + KBASE_SERIALIZE_INTER_SLOT) && + other_slots_busy(kbdev, js)) + break; + + if ((kbdev->serialize_jobs & + KBASE_SERIALIZE_RESET) && + kbase_reset_gpu_active(kbdev)) + break; + + /* Check if this job needs the cycle counter + * enabled before submission */ + if (katom[idx]->core_req & BASE_JD_REQ_PERMON) + kbase_pm_request_gpu_cycle_counter_l2_is_on( + kbdev); + + kbase_job_hw_submit(kbdev, katom[idx], js); + katom[idx]->gpu_rb_state = + KBASE_ATOM_GPU_RB_SUBMITTED; + + /* Inform power management at start/finish of + * atom so it can update its GPU utilisation + * metrics. */ + kbase_pm_metrics_update(kbdev, + &katom[idx]->start_timestamp); + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_SUBMITTED: + /* Atom submitted to HW, nothing else to do */ + break; + + case KBASE_ATOM_GPU_RB_RETURN_TO_JS: + /* Only return if head atom or previous atom + * already removed - as atoms must be returned + * in order */ + if (idx == 0 || katom[0]->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + kbase_jm_return_atom_to_js(kbdev, + katom[idx]); + } + break; + } + } + } + + /* Warn if PRLAM-8987 affinity restrictions are violated */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) + WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) || + kbase_gpu_atoms_submitted(kbdev, 1)) && + kbase_gpu_atoms_submitted(kbdev, 2)); +} + + +void kbase_backend_run_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + kbase_gpu_enqueue_atom(kbdev, katom); + kbase_backend_slot_update(kbdev); +} + +#define HAS_DEP(katom) (katom->pre_dep || katom->atom_flags & \ + (KBASE_KATOM_FLAG_X_DEP_BLOCKED | KBASE_KATOM_FLAG_FAIL_BLOCKER)) + +bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js) +{ + struct kbase_jd_atom *katom; + struct kbase_jd_atom *next_katom; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom = kbase_gpu_inspect(kbdev, js, 0); + next_katom = kbase_gpu_inspect(kbdev, js, 1); + + if (next_katom && katom->kctx == next_katom->kctx && + next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && + HAS_DEP(next_katom) && + (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) + != 0 || + kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) + != 0)) { + kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), + JS_COMMAND_NOP, NULL); + next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; + + KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, + &kbdev->gpu_props.props.raw_props.js_features + [katom->slot_nr]); + KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as + [katom->kctx->as_nr]); + KBASE_TLSTREAM_TL_NRET_CTX_LPU(katom->kctx, + &kbdev->gpu_props.props.raw_props.js_features + [katom->slot_nr]); + + return true; + } + + return false; +} + +void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, + u32 completion_code, + u64 job_tail, + ktime_t *end_timestamp) +{ + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); + struct kbase_context *kctx = katom->kctx; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* + * When a hard-stop is followed close after a soft-stop, the completion + * code may be set to STOPPED, even though the job is terminated + */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) { + if (completion_code == BASE_JD_EVENT_STOPPED && + (katom->atom_flags & + KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { + completion_code = BASE_JD_EVENT_TERMINATED; + } + } + + if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) || (katom->core_req & + BASE_JD_REQ_SKIP_CACHE_END)) && + completion_code != BASE_JD_EVENT_DONE && + !(completion_code & BASE_JD_SW_EVENT)) { + /* When a job chain fails, on a T60x or when + * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not + * flushed. To prevent future evictions causing possible memory + * corruption we need to flush the cache manually before any + * affected memory gets reused. */ + katom->need_cache_flush_cores_retained = katom->affinity; + kbase_pm_request_cores(kbdev, false, katom->affinity); + } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) { + if (kbdev->gpu_props.num_core_groups > 1 && + !(katom->affinity & + kbdev->gpu_props.props.coherency_info.group[0].core_mask + ) && + (katom->affinity & + kbdev->gpu_props.props.coherency_info.group[1].core_mask + )) { + dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n"); + katom->need_cache_flush_cores_retained = + katom->affinity; + kbase_pm_request_cores(kbdev, false, + katom->affinity); + } + } + + katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); + kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0); + + if (completion_code == BASE_JD_EVENT_STOPPED) { + struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, + 0); + + /* + * Dequeue next atom from ringbuffers on same slot if required. + * This atom will already have been removed from the NEXT + * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that + * the atoms on this slot are returned in the correct order. + */ + if (next_katom && katom->kctx == next_katom->kctx && + next_katom->sched_priority == + katom->sched_priority) { + kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); + kbase_jm_return_atom_to_js(kbdev, next_katom); + } + } else if (completion_code != BASE_JD_EVENT_DONE) { + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + int i; + +#if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0 + KBASE_TRACE_DUMP(kbdev); +#endif + kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); + + /* + * Remove all atoms on the same context from ringbuffers. This + * will not remove atoms that are already on the GPU, as these + * are guaranteed not to have fail dependencies on the failed + * atom. + */ + for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { + struct kbase_jd_atom *katom_idx0 = + kbase_gpu_inspect(kbdev, i, 0); + struct kbase_jd_atom *katom_idx1 = + kbase_gpu_inspect(kbdev, i, 1); + + if (katom_idx0 && katom_idx0->kctx == katom->kctx && + HAS_DEP(katom_idx0) && + katom_idx0->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Dequeue katom_idx0 from ringbuffer */ + kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); + + if (katom_idx1 && + katom_idx1->kctx == katom->kctx + && HAS_DEP(katom_idx1) && + katom_idx0->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Dequeue katom_idx1 from ringbuffer */ + kbase_gpu_dequeue_atom(kbdev, i, + end_timestamp); + + katom_idx1->event_code = + BASE_JD_EVENT_STOPPED; + kbase_jm_return_atom_to_js(kbdev, + katom_idx1); + } + katom_idx0->event_code = BASE_JD_EVENT_STOPPED; + kbase_jm_return_atom_to_js(kbdev, katom_idx0); + + } else if (katom_idx1 && + katom_idx1->kctx == katom->kctx && + HAS_DEP(katom_idx1) && + katom_idx1->gpu_rb_state != + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Can not dequeue this atom yet - will be + * dequeued when atom at idx0 completes */ + katom_idx1->event_code = BASE_JD_EVENT_STOPPED; + kbase_gpu_mark_atom_for_return(kbdev, + katom_idx1); + } + } + } + + KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, + js, completion_code); + + if (job_tail != 0 && job_tail != katom->jc) { + bool was_updated = (job_tail != katom->jc); + + /* Some of the job has been executed, so we update the job chain + * address to where we should resume from */ + katom->jc = job_tail; + if (was_updated) + KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, + katom, job_tail, js); + } + + /* Only update the event code for jobs that weren't cancelled */ + if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) + katom->event_code = (base_jd_event_code)completion_code; + + kbase_device_trace_register_access(kctx, REG_WRITE, + JOB_CONTROL_REG(JOB_IRQ_CLEAR), + 1 << js); + + /* Complete the job, and start new ones + * + * Also defer remaining work onto the workqueue: + * - Re-queue Soft-stopped jobs + * - For any other jobs, queue the job back into the dependency system + * - Schedule out the parent context if necessary, and schedule a new + * one in. + */ +#ifdef CONFIG_GPU_TRACEPOINTS + { + /* The atom in the HEAD */ + struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, + 0); + + if (next_katom && next_katom->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED) { + char js_string[16]; + + trace_gpu_sched_switch(kbasep_make_job_slot_string(js, + js_string, + sizeof(js_string)), + ktime_to_ns(*end_timestamp), + (u32)next_katom->kctx->id, 0, + next_katom->work_id); + kbdev->hwaccess.backend.slot_rb[js].last_context = + next_katom->kctx; + } else { + char js_string[16]; + + trace_gpu_sched_switch(kbasep_make_job_slot_string(js, + js_string, + sizeof(js_string)), + ktime_to_ns(ktime_get()), 0, 0, + 0); + kbdev->hwaccess.backend.slot_rb[js].last_context = 0; + } + } +#endif + + if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) + kbase_reset_gpu_silent(kbdev); + + if (completion_code == BASE_JD_EVENT_STOPPED) + katom = kbase_jm_return_atom_to_js(kbdev, katom); + else + katom = kbase_jm_complete(kbdev, katom, end_timestamp); + + if (katom) { + /* Cross-slot dependency has now become runnable. Try to submit + * it. */ + + /* Check if there are lower priority jobs to soft stop */ + kbase_job_slot_ctx_priority_check_locked(kctx, katom); + + kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); + } + + /* Job completion may have unblocked other atoms. Try to update all job + * slots */ + kbase_backend_slot_update(kbdev); +} + +void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) +{ + int js; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* Reset should always take the GPU out of protected mode */ + WARN_ON(kbase_gpu_in_protected_mode(kbdev)); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + int atom_idx = 0; + int idx; + + for (idx = 0; idx < SLOT_RB_SIZE; idx++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, + js, atom_idx); + bool keep_in_jm_rb = false; + + if (!katom) + break; + if (katom->protected_state.exit == + KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) + { + KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev); + + kbase_vinstr_resume(kbdev->vinstr_ctx); + + /* protected mode sanity checks */ + KBASE_DEBUG_ASSERT_MSG( + kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev), + "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", + kbase_jd_katom_is_protected(katom), kbase_gpu_in_protected_mode(kbdev)); + KBASE_DEBUG_ASSERT_MSG( + (kbase_jd_katom_is_protected(katom) && js == 0) || + !kbase_jd_katom_is_protected(katom), + "Protected atom on JS%d not supported", js); + } + if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) + keep_in_jm_rb = true; + + kbase_gpu_release_atom(kbdev, katom, NULL); + + /* + * If the atom wasn't on HW when the reset was issued + * then leave it in the RB and next time we're kicked + * it will be processed again from the starting state. + */ + if (keep_in_jm_rb) { + kbasep_js_job_check_deref_cores(kbdev, katom); + katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; + katom->affinity = 0; + katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; + /* As the atom was not removed, increment the + * index so that we read the correct atom in the + * next iteration. */ + atom_idx++; + continue; + } + + /* + * The atom was on the HW when the reset was issued + * all we can do is fail the atom. + */ + kbase_gpu_dequeue_atom(kbdev, js, NULL); + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + kbase_jm_complete(kbdev, katom, end_timestamp); + } + } + + kbdev->protected_mode_transition = false; +} + +static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, + int js, + struct kbase_jd_atom *katom, + u32 action) +{ + u32 hw_action = action & JS_COMMAND_MASK; + + kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); + kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, + katom->core_req, katom); + katom->kctx->blocked_js[js][katom->sched_priority] = true; +} + +static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + u32 action, + bool disjoint) +{ + katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; + kbase_gpu_mark_atom_for_return(kbdev, katom); + katom->kctx->blocked_js[katom->slot_nr][katom->sched_priority] = true; + + if (disjoint) + kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, + katom); +} + +static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) +{ + if (katom->x_post_dep) { + struct kbase_jd_atom *dep_atom = katom->x_post_dep; + + if (dep_atom->gpu_rb_state != + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && + dep_atom->gpu_rb_state != + KBASE_ATOM_GPU_RB_RETURN_TO_JS) + return dep_atom->slot_nr; + } + return -1; +} + +static void kbase_job_evicted(struct kbase_jd_atom *katom) +{ + kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom, + katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT); +} + +bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js, + struct kbase_jd_atom *katom, + u32 action) +{ + struct kbase_jd_atom *katom_idx0; + struct kbase_jd_atom *katom_idx1; + + bool katom_idx0_valid, katom_idx1_valid; + + bool ret = false; + + int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; + int prio_idx0 = 0, prio_idx1 = 0; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); + katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); + + if (katom_idx0) + prio_idx0 = katom_idx0->sched_priority; + if (katom_idx1) + prio_idx1 = katom_idx1->sched_priority; + + if (katom) { + katom_idx0_valid = (katom_idx0 == katom); + /* If idx0 is to be removed and idx1 is on the same context, + * then idx1 must also be removed otherwise the atoms might be + * returned out of order */ + if (katom_idx1) + katom_idx1_valid = (katom_idx1 == katom) || + (katom_idx0_valid && + (katom_idx0->kctx == + katom_idx1->kctx)); + else + katom_idx1_valid = false; + } else { + katom_idx0_valid = (katom_idx0 && + (!kctx || katom_idx0->kctx == kctx)); + katom_idx1_valid = (katom_idx1 && + (!kctx || katom_idx1->kctx == kctx) && + prio_idx0 == prio_idx1); + } + + if (katom_idx0_valid) + stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); + if (katom_idx1_valid) + stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); + + if (katom_idx0_valid) { + if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Simple case - just dequeue and return */ + kbase_gpu_dequeue_atom(kbdev, js, NULL); + if (katom_idx1_valid) { + kbase_gpu_dequeue_atom(kbdev, js, NULL); + katom_idx1->event_code = + BASE_JD_EVENT_REMOVED_FROM_NEXT; + kbase_jm_return_atom_to_js(kbdev, katom_idx1); + katom_idx1->kctx->blocked_js[js][prio_idx1] = + true; + } + + katom_idx0->event_code = + BASE_JD_EVENT_REMOVED_FROM_NEXT; + kbase_jm_return_atom_to_js(kbdev, katom_idx0); + katom_idx0->kctx->blocked_js[js][prio_idx0] = true; + } else { + /* katom_idx0 is on GPU */ + if (katom_idx1 && katom_idx1->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED) { + /* katom_idx0 and katom_idx1 are on GPU */ + + if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_COMMAND_NEXT), NULL) == 0) { + /* idx0 has already completed - stop + * idx1 if needed*/ + if (katom_idx1_valid) { + kbase_gpu_stop_atom(kbdev, js, + katom_idx1, + action); + ret = true; + } + } else { + /* idx1 is in NEXT registers - attempt + * to remove */ + kbase_reg_write(kbdev, + JOB_SLOT_REG(js, + JS_COMMAND_NEXT), + JS_COMMAND_NOP, NULL); + + if (kbase_reg_read(kbdev, + JOB_SLOT_REG(js, + JS_HEAD_NEXT_LO), NULL) + != 0 || + kbase_reg_read(kbdev, + JOB_SLOT_REG(js, + JS_HEAD_NEXT_HI), NULL) + != 0) { + /* idx1 removed successfully, + * will be handled in IRQ */ + kbase_job_evicted(katom_idx1); + kbase_gpu_remove_atom(kbdev, + katom_idx1, + action, true); + stop_x_dep_idx1 = + should_stop_x_dep_slot(katom_idx1); + + /* stop idx0 if still on GPU */ + kbase_gpu_stop_atom(kbdev, js, + katom_idx0, + action); + ret = true; + } else if (katom_idx1_valid) { + /* idx0 has already completed, + * stop idx1 if needed */ + kbase_gpu_stop_atom(kbdev, js, + katom_idx1, + action); + ret = true; + } + } + } else if (katom_idx1_valid) { + /* idx1 not on GPU but must be dequeued*/ + + /* idx1 will be handled in IRQ */ + kbase_gpu_remove_atom(kbdev, katom_idx1, action, + false); + /* stop idx0 */ + /* This will be repeated for anything removed + * from the next registers, since their normal + * flow was also interrupted, and this function + * might not enter disjoint state e.g. if we + * don't actually do a hard stop on the head + * atom */ + kbase_gpu_stop_atom(kbdev, js, katom_idx0, + action); + ret = true; + } else { + /* no atom in idx1 */ + /* just stop idx0 */ + kbase_gpu_stop_atom(kbdev, js, katom_idx0, + action); + ret = true; + } + } + } else if (katom_idx1_valid) { + if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { + /* Mark for return */ + /* idx1 will be returned once idx0 completes */ + kbase_gpu_remove_atom(kbdev, katom_idx1, action, + false); + } else { + /* idx1 is on GPU */ + if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_COMMAND_NEXT), NULL) == 0) { + /* idx0 has already completed - stop idx1 */ + kbase_gpu_stop_atom(kbdev, js, katom_idx1, + action); + ret = true; + } else { + /* idx1 is in NEXT registers - attempt to + * remove */ + kbase_reg_write(kbdev, JOB_SLOT_REG(js, + JS_COMMAND_NEXT), + JS_COMMAND_NOP, NULL); + + if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_HEAD_NEXT_LO), NULL) != 0 || + kbase_reg_read(kbdev, JOB_SLOT_REG(js, + JS_HEAD_NEXT_HI), NULL) != 0) { + /* idx1 removed successfully, will be + * handled in IRQ once idx0 completes */ + kbase_job_evicted(katom_idx1); + kbase_gpu_remove_atom(kbdev, katom_idx1, + action, + false); + } else { + /* idx0 has already completed - stop + * idx1 */ + kbase_gpu_stop_atom(kbdev, js, + katom_idx1, + action); + ret = true; + } + } + } + } + + + if (stop_x_dep_idx0 != -1) + kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0, + NULL, action); + + if (stop_x_dep_idx1 != -1) + kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1, + NULL, action); + + return ret; +} + +void kbase_gpu_cacheclean(struct kbase_device *kbdev) +{ + /* Limit the number of loops to avoid a hang if the interrupt is missed + */ + u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; + + mutex_lock(&kbdev->cacheclean_lock); + + /* use GPU_COMMAND completion solution */ + /* clean & invalidate the caches */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAN_INV_CACHES, NULL); + + /* wait for cache flush to complete before continuing */ + while (--max_loops && + (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & + CLEAN_CACHES_COMPLETED) == 0) + ; + + /* clear the CLEAN_CACHES_COMPLETED irq */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, + CLEAN_CACHES_COMPLETED); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), + CLEAN_CACHES_COMPLETED, NULL); + KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state != + KBASE_INSTR_STATE_CLEANING, + "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang."); + + mutex_unlock(&kbdev->cacheclean_lock); +} + +void kbase_backend_cacheclean(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + if (katom->need_cache_flush_cores_retained) { + unsigned long flags; + + kbase_gpu_cacheclean(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_unrequest_cores(kbdev, false, + katom->need_cache_flush_cores_retained); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + katom->need_cache_flush_cores_retained = 0; + } +} + +void kbase_backend_complete_wq(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + /* + * If cache flush required due to HW workaround then perform the flush + * now + */ + kbase_backend_cacheclean(kbdev, katom); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) && + (katom->core_req & BASE_JD_REQ_FS) && + katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT && + (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) && + !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) { + dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n"); + if (kbasep_10969_workaround_clamp_coordinates(katom)) { + /* The job had a TILE_RANGE_FAULT after was soft-stopped + * Due to an HW issue we try to execute the job again. + */ + dev_dbg(kbdev->dev, + "Clamping has been executed, try to rerun the job\n" + ); + katom->event_code = BASE_JD_EVENT_STOPPED; + katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN; + } + } + + /* Clear the coreref_state now - while check_deref_cores() may not have + * been called yet, the caller will have taken a copy of this field. If + * this is not done, then if the atom is re-scheduled (following a soft + * stop) then the core reference would not be retaken. */ + katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; + katom->affinity = 0; +} + +void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, + base_jd_core_req core_req, u64 affinity, + enum kbase_atom_coreref_state coreref_state) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity, + coreref_state); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (!kbdev->pm.active_count) { + mutex_lock(&kbdev->js_data.runpool_mutex); + mutex_lock(&kbdev->pm.lock); + kbase_pm_update_active(kbdev); + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&kbdev->js_data.runpool_mutex); + } +} + +void kbase_gpu_dump_slots(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata; + unsigned long flags; + int js; + + js_devdata = &kbdev->js_data; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n"); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + int idx; + + for (idx = 0; idx < SLOT_RB_SIZE; idx++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, + js, + idx); + + if (katom) + dev_info(kbdev->dev, + " js%d idx%d : katom=%p gpu_rb_state=%d\n", + js, idx, katom, katom->gpu_rb_state); + else + dev_info(kbdev->dev, " js%d idx%d : empty\n", + js, idx); + } + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + + + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h new file mode 100755 index 000000000000..1e0e05ad3ea4 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h @@ -0,0 +1,76 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend specific APIs + */ + +#ifndef _KBASE_HWACCESS_GPU_H_ +#define _KBASE_HWACCESS_GPU_H_ + +#include + +/** + * kbase_gpu_irq_evict - Evict an atom from a NEXT slot + * + * @kbdev: Device pointer + * @js: Job slot to evict from + * + * Evict the atom in the NEXT slot for the specified job slot. This function is + * called from the job complete IRQ handler when the previous job has failed. + * + * Return: true if job evicted from NEXT registers, false otherwise + */ +bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js); + +/** + * kbase_gpu_complete_hw - Complete an atom on job slot js + * + * @kbdev: Device pointer + * @js: Job slot that has completed + * @completion_code: Event code from job that has completed + * @job_tail: The tail address from the hardware if the job has partially + * completed + * @end_timestamp: Time of completion + */ +void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, + u32 completion_code, + u64 job_tail, + ktime_t *end_timestamp); + +/** + * kbase_gpu_inspect - Inspect the contents of the HW access ringbuffer + * + * @kbdev: Device pointer + * @js: Job slot to inspect + * @idx: Index into ringbuffer. 0 is the job currently running on + * the slot, 1 is the job waiting, all other values are invalid. + * Return: The atom at that position in the ringbuffer + * or NULL if no atom present + */ +struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, + int idx); + +/** + * kbase_gpu_dump_slots - Print the contents of the slot ringbuffers + * + * @kbdev: Device pointer + */ +void kbase_gpu_dump_slots(struct kbase_device *kbdev); + +#endif /* _KBASE_HWACCESS_GPU_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.c new file mode 100755 index 000000000000..54d8ddd80097 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.c @@ -0,0 +1,303 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel affinity manager APIs + */ + +#include +#include "mali_kbase_js_affinity.h" +#include "mali_kbase_hw.h" + +#include + + +bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, + int js) +{ + /* + * Here are the reasons for using job slot 2: + * - BASE_HW_ISSUE_8987 (which is entirely used for that purpose) + * - In absence of the above, then: + * - Atoms with BASE_JD_REQ_COHERENT_GROUP + * - But, only when there aren't contexts with + * KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, because the atoms that run on + * all cores on slot 1 could be blocked by those using a coherent group + * on slot 2 + * - And, only when you actually have 2 or more coregroups - if you + * only have 1 coregroup, then having jobs for slot 2 implies they'd + * also be for slot 1, meaning you'll get interference from them. Jobs + * able to run on slot 2 could also block jobs that can only run on + * slot 1 (tiler jobs) + */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) + return true; + + if (js != 2) + return true; + + /* Only deal with js==2 now: */ + if (kbdev->gpu_props.num_core_groups > 1) { + /* Only use slot 2 in the 2+ coregroup case */ + if (kbasep_js_ctx_attr_is_attr_on_runpool(kbdev, + KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) == + false) { + /* ...But only when we *don't* have atoms that run on + * all cores */ + + /* No specific check for BASE_JD_REQ_COHERENT_GROUP + * atoms - the policy will sort that out */ + return true; + } + } + + /* Above checks failed mean we shouldn't use slot 2 */ + return false; +} + +/* + * As long as it has been decided to have a deeper modification of + * what job scheduler, power manager and affinity manager will + * implement, this function is just an intermediate step that + * assumes: + * - all working cores will be powered on when this is called. + * - largest current configuration is 2 core groups. + * - It has been decided not to have hardcoded values so the low + * and high cores in a core split will be evently distributed. + * - Odd combinations of core requirements have been filtered out + * and do not get to this function (e.g. CS+T+NSS is not + * supported here). + * - This function is frequently called and can be optimized, + * (see notes in loops), but as the functionallity will likely + * be modified, optimization has not been addressed. +*/ +bool kbase_js_choose_affinity(u64 * const affinity, + struct kbase_device *kbdev, + struct kbase_jd_atom *katom, int js) +{ + base_jd_core_req core_req = katom->core_req; + unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; + u64 core_availability_mask; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + core_availability_mask = kbase_pm_ca_get_core_mask(kbdev); + + /* + * If no cores are currently available (core availability policy is + * transitioning) then fail. + */ + if (0 == core_availability_mask) { + *affinity = 0; + return false; + } + + KBASE_DEBUG_ASSERT(js >= 0); + + if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) == + BASE_JD_REQ_T) { + /* If the hardware supports XAFFINITY then we'll only enable + * the tiler (which is the default so this is a no-op), + * otherwise enable shader core 0. */ + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) + *affinity = 1; + else + *affinity = 0; + + return true; + } + + if (1 == kbdev->gpu_props.num_cores) { + /* trivial case only one core, nothing to do */ + *affinity = core_availability_mask & + kbdev->pm.debug_core_mask[js]; + } else { + if ((core_req & (BASE_JD_REQ_COHERENT_GROUP | + BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) { + if (js == 0 || num_core_groups == 1) { + /* js[0] and single-core-group systems just get + * the first core group */ + *affinity = + kbdev->gpu_props.props.coherency_info.group[0].core_mask + & core_availability_mask & + kbdev->pm.debug_core_mask[js]; + } else { + /* js[1], js[2] use core groups 0, 1 for + * dual-core-group systems */ + u32 core_group_idx = ((u32) js) - 1; + + KBASE_DEBUG_ASSERT(core_group_idx < + num_core_groups); + *affinity = + kbdev->gpu_props.props.coherency_info.group[core_group_idx].core_mask + & core_availability_mask & + kbdev->pm.debug_core_mask[js]; + + /* If the job is specifically targeting core + * group 1 and the core availability policy is + * keeping that core group off, then fail */ + if (*affinity == 0 && core_group_idx == 1 && + kbdev->pm.backend.cg1_disabled + == true) + katom->event_code = + BASE_JD_EVENT_PM_EVENT; + } + } else { + /* All cores are available when no core split is + * required */ + *affinity = core_availability_mask & + kbdev->pm.debug_core_mask[js]; + } + } + + /* + * If no cores are currently available in the desired core group(s) + * (core availability policy is transitioning) then fail. + */ + if (*affinity == 0) + return false; + + /* Enable core 0 if tiler required for hardware without XAFFINITY + * support (notes above) */ + if (core_req & BASE_JD_REQ_T) { + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) + *affinity = *affinity | 1; + } + + return true; +} + +static inline bool kbase_js_affinity_is_violating( + struct kbase_device *kbdev, + u64 *affinities) +{ + /* This implementation checks whether the two slots involved in Generic + * thread creation have intersecting affinity. This is due to micro- + * architectural issues where a job in slot A targetting cores used by + * slot B could prevent the job in slot B from making progress until the + * job in slot A has completed. + */ + u64 affinity_set_left; + u64 affinity_set_right; + u64 intersection; + + KBASE_DEBUG_ASSERT(affinities != NULL); + + affinity_set_left = affinities[1]; + + affinity_set_right = affinities[2]; + + /* A violation occurs when any bit in the left_set is also in the + * right_set */ + intersection = affinity_set_left & affinity_set_right; + + return (bool) (intersection != (u64) 0u); +} + +bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, + u64 affinity) +{ + struct kbasep_js_device_data *js_devdata; + u64 new_affinities[BASE_JM_MAX_NR_SLOTS]; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); + js_devdata = &kbdev->js_data; + + memcpy(new_affinities, js_devdata->runpool_irq.slot_affinities, + sizeof(js_devdata->runpool_irq.slot_affinities)); + + new_affinities[js] |= affinity; + + return kbase_js_affinity_is_violating(kbdev, new_affinities); +} + +void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, + u64 affinity) +{ + struct kbasep_js_device_data *js_devdata; + u64 cores; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); + js_devdata = &kbdev->js_data; + + KBASE_DEBUG_ASSERT(kbase_js_affinity_would_violate(kbdev, js, affinity) + == false); + + cores = affinity; + while (cores) { + int bitnum = fls64(cores) - 1; + u64 bit = 1ULL << bitnum; + s8 cnt; + + cnt = + ++(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]); + + if (cnt == 1) + js_devdata->runpool_irq.slot_affinities[js] |= bit; + + cores &= ~bit; + } +} + +void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, + u64 affinity) +{ + struct kbasep_js_device_data *js_devdata; + u64 cores; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); + js_devdata = &kbdev->js_data; + + cores = affinity; + while (cores) { + int bitnum = fls64(cores) - 1; + u64 bit = 1ULL << bitnum; + s8 cnt; + + KBASE_DEBUG_ASSERT( + js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum] > 0); + + cnt = + --(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]); + + if (0 == cnt) + js_devdata->runpool_irq.slot_affinities[js] &= ~bit; + + cores &= ~bit; + } +} + +#if KBASE_TRACE_ENABLE +void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata; + int slot_nr; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + js_devdata = &kbdev->js_data; + + for (slot_nr = 0; slot_nr < 3; ++slot_nr) + KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_AFFINITY_CURRENT, NULL, + NULL, 0u, slot_nr, + (u32) js_devdata->runpool_irq.slot_affinities[slot_nr]); +} +#endif /* KBASE_TRACE_ENABLE */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.h new file mode 100755 index 000000000000..35d9781ae092 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.h @@ -0,0 +1,129 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Affinity Manager internal APIs. + */ + +#ifndef _KBASE_JS_AFFINITY_H_ +#define _KBASE_JS_AFFINITY_H_ + +/** + * kbase_js_can_run_job_on_slot_no_lock - Decide whether it is possible to + * submit a job to a particular job slot in the current status + * + * @kbdev: The kbase device structure of the device + * @js: Job slot number to check for allowance + * + * Will check if submitting to the given job slot is allowed in the current + * status. For example using job slot 2 while in soft-stoppable state and only + * having 1 coregroup is not allowed by the policy. This function should be + * called prior to submitting a job to a slot to make sure policy rules are not + * violated. + * + * The following locking conditions are made on the caller + * - it must hold hwaccess_lock + */ +bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, int js); + +/** + * kbase_js_choose_affinity - Compute affinity for a given job. + * + * @affinity: Affinity bitmap computed + * @kbdev: The kbase device structure of the device + * @katom: Job chain of which affinity is going to be found + * @js: Slot the job chain is being submitted + * + * Currently assumes an all-on/all-off power management policy. + * Also assumes there is at least one core with tiler available. + * + * Returns true if a valid affinity was chosen, false if + * no cores were available. + */ +bool kbase_js_choose_affinity(u64 * const affinity, + struct kbase_device *kbdev, + struct kbase_jd_atom *katom, + int js); + +/** + * kbase_js_affinity_would_violate - Determine whether a proposed affinity on + * job slot @js would cause a violation of affinity restrictions. + * + * @kbdev: Kbase device structure + * @js: The job slot to test + * @affinity: The affinity mask to test + * + * The following locks must be held by the caller + * - hwaccess_lock + * + * Return: true if the affinity would violate the restrictions + */ +bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, + u64 affinity); + +/** + * kbase_js_affinity_retain_slot_cores - Affinity tracking: retain cores used by + * a slot + * + * @kbdev: Kbase device structure + * @js: The job slot retaining the cores + * @affinity: The cores to retain + * + * The following locks must be held by the caller + * - hwaccess_lock + */ +void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, + u64 affinity); + +/** + * kbase_js_affinity_release_slot_cores - Affinity tracking: release cores used + * by a slot + * + * @kbdev: Kbase device structure + * @js: Job slot + * @affinity: Bit mask of core to be released + * + * Cores must be released as soon as a job is dequeued from a slot's 'submit + * slots', and before another job is submitted to those slots. Otherwise, the + * refcount could exceed the maximum number submittable to a slot, + * %BASE_JM_SUBMIT_SLOTS. + * + * The following locks must be held by the caller + * - hwaccess_lock + */ +void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, + u64 affinity); + +/** + * kbase_js_debug_log_current_affinities - log the current affinities + * + * @kbdev: Kbase device structure + * + * Output to the Trace log the current tracked affinities on all slots + */ +#if KBASE_TRACE_ENABLE +void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev); +#else /* KBASE_TRACE_ENABLE */ +static inline void +kbase_js_debug_log_current_affinities(struct kbase_device *kbdev) +{ +} +#endif /* KBASE_TRACE_ENABLE */ + +#endif /* _KBASE_JS_AFFINITY_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c new file mode 100755 index 000000000000..a8c1af23a369 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c @@ -0,0 +1,356 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend specific job scheduler APIs + */ + +#include +#include +#include +#include + +/* + * Define for when dumping is enabled. + * This should not be based on the instrumentation level as whether dumping is + * enabled for a particular level is down to the integrator. However this is + * being used for now as otherwise the cinstr headers would be needed. + */ +#define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL) + +/* + * Hold the runpool_mutex for this + */ +static inline bool timer_callback_should_run(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + s8 nr_running_ctxs; + + lockdep_assert_held(&kbdev->js_data.runpool_mutex); + + /* Timer must stop if we are suspending */ + if (backend->suspend_timer) + return false; + + /* nr_contexts_pullable is updated with the runpool_mutex. However, the + * locking in the caller gives us a barrier that ensures + * nr_contexts_pullable is up-to-date for reading */ + nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); + +#ifdef CONFIG_MALI_DEBUG + if (kbdev->js_data.softstop_always) { + /* Debug support for allowing soft-stop on a single context */ + return true; + } +#endif /* CONFIG_MALI_DEBUG */ + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) { + /* Timeouts would have to be 4x longer (due to micro- + * architectural design) to support OpenCL conformance tests, so + * only run the timer when there's: + * - 2 or more CL contexts + * - 1 or more GLES contexts + * + * NOTE: We will treat a context that has both Compute and Non- + * Compute jobs will be treated as an OpenCL context (hence, we + * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). + */ + { + s8 nr_compute_ctxs = + kbasep_js_ctx_attr_count_on_runpool(kbdev, + KBASEP_JS_CTX_ATTR_COMPUTE); + s8 nr_noncompute_ctxs = nr_running_ctxs - + nr_compute_ctxs; + + return (bool) (nr_compute_ctxs >= 2 || + nr_noncompute_ctxs > 0); + } + } else { + /* Run the timer callback whenever you have at least 1 context + */ + return (bool) (nr_running_ctxs > 0); + } +} + +static enum hrtimer_restart timer_callback(struct hrtimer *timer) +{ + unsigned long flags; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + struct kbase_backend_data *backend; + int s; + bool reset_needed = false; + + KBASE_DEBUG_ASSERT(timer != NULL); + + backend = container_of(timer, struct kbase_backend_data, + scheduling_timer); + kbdev = container_of(backend, struct kbase_device, hwaccess.backend); + js_devdata = &kbdev->js_data; + + /* Loop through the slots */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) { + struct kbase_jd_atom *atom = NULL; + + if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) { + atom = kbase_gpu_inspect(kbdev, s, 0); + KBASE_DEBUG_ASSERT(atom != NULL); + } + + if (atom != NULL) { + /* The current version of the model doesn't support + * Soft-Stop */ + if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { + u32 ticks = atom->ticks++; + +#if !CINSTR_DUMPING_ENABLED + u32 soft_stop_ticks, hard_stop_ticks, + gpu_reset_ticks; + if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { + soft_stop_ticks = + js_devdata->soft_stop_ticks_cl; + hard_stop_ticks = + js_devdata->hard_stop_ticks_cl; + gpu_reset_ticks = + js_devdata->gpu_reset_ticks_cl; + } else { + soft_stop_ticks = + js_devdata->soft_stop_ticks; + hard_stop_ticks = + js_devdata->hard_stop_ticks_ss; + gpu_reset_ticks = + js_devdata->gpu_reset_ticks_ss; + } + + /* If timeouts have been changed then ensure + * that atom tick count is not greater than the + * new soft_stop timeout. This ensures that + * atoms do not miss any of the timeouts due to + * races between this worker and the thread + * changing the timeouts. */ + if (backend->timeouts_updated && + ticks > soft_stop_ticks) + ticks = atom->ticks = soft_stop_ticks; + + /* Job is Soft-Stoppable */ + if (ticks == soft_stop_ticks) { + int disjoint_threshold = + KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD; + u32 softstop_flags = 0u; + /* Job has been scheduled for at least + * js_devdata->soft_stop_ticks ticks. + * Soft stop the slot so we can run + * other jobs. + */ + dev_dbg(kbdev->dev, "Soft-stop"); +#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS + /* nr_user_contexts_running is updated + * with the runpool_mutex, but we can't + * take that here. + * + * However, if it's about to be + * increased then the new context can't + * run any jobs until they take the + * hwaccess_lock, so it's OK to observe + * the older value. + * + * Similarly, if it's about to be + * decreased, the last job from another + * context has already finished, so it's + * not too bad that we observe the older + * value and register a disjoint event + * when we try soft-stopping */ + if (js_devdata->nr_user_contexts_running + >= disjoint_threshold) + softstop_flags |= + JS_COMMAND_SW_CAUSES_DISJOINT; + + kbase_job_slot_softstop_swflags(kbdev, + s, atom, softstop_flags); +#endif + } else if (ticks == hard_stop_ticks) { + /* Job has been scheduled for at least + * js_devdata->hard_stop_ticks_ss ticks. + * It should have been soft-stopped by + * now. Hard stop the slot. + */ +#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS + int ms = + js_devdata->scheduling_period_ns + / 1000000u; + dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", + (unsigned long)ticks, + (unsigned long)ms); + kbase_job_slot_hardstop(atom->kctx, s, + atom); +#endif + } else if (ticks == gpu_reset_ticks) { + /* Job has been scheduled for at least + * js_devdata->gpu_reset_ticks_ss ticks. + * It should have left the GPU by now. + * Signal that the GPU needs to be + * reset. + */ + reset_needed = true; + } +#else /* !CINSTR_DUMPING_ENABLED */ + /* NOTE: During CINSTR_DUMPING_ENABLED, we use + * the alternate timeouts, which makes the hard- + * stop and GPU reset timeout much longer. We + * also ensure that we don't soft-stop at all. + */ + if (ticks == js_devdata->soft_stop_ticks) { + /* Job has been scheduled for at least + * js_devdata->soft_stop_ticks. We do + * not soft-stop during + * CINSTR_DUMPING_ENABLED, however. + */ + dev_dbg(kbdev->dev, "Soft-stop"); + } else if (ticks == + js_devdata->hard_stop_ticks_dumping) { + /* Job has been scheduled for at least + * js_devdata->hard_stop_ticks_dumping + * ticks. Hard stop the slot. + */ +#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS + int ms = + js_devdata->scheduling_period_ns + / 1000000u; + dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", + (unsigned long)ticks, + (unsigned long)ms); + kbase_job_slot_hardstop(atom->kctx, s, + atom); +#endif + } else if (ticks == + js_devdata->gpu_reset_ticks_dumping) { + /* Job has been scheduled for at least + * js_devdata->gpu_reset_ticks_dumping + * ticks. It should have left the GPU by + * now. Signal that the GPU needs to be + * reset. + */ + reset_needed = true; + } +#endif /* !CINSTR_DUMPING_ENABLED */ + } + } + } +#if KBASE_GPU_RESET_EN + if (reset_needed) { + dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve."); + + if (kbase_prepare_to_reset_gpu_locked(kbdev)) + kbase_reset_gpu_locked(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + /* the timer is re-issued if there is contexts in the run-pool */ + + if (backend->timer_running) + hrtimer_start(&backend->scheduling_timer, + HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), + HRTIMER_MODE_REL); + + backend->timeouts_updated = false; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return HRTIMER_NORESTART; +} + +void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + unsigned long flags; + + lockdep_assert_held(&js_devdata->runpool_mutex); + + if (!timer_callback_should_run(kbdev)) { + /* Take spinlock to force synchronisation with timer */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + backend->timer_running = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + /* From now on, return value of timer_callback_should_run() will + * also cause the timer to not requeue itself. Its return value + * cannot change, because it depends on variables updated with + * the runpool_mutex held, which the caller of this must also + * hold */ + hrtimer_cancel(&backend->scheduling_timer); + } + + if (timer_callback_should_run(kbdev) && !backend->timer_running) { + /* Take spinlock to force synchronisation with timer */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + backend->timer_running = true; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + hrtimer_start(&backend->scheduling_timer, + HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), + HRTIMER_MODE_REL); + + KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, + 0u); + } +} + +int kbase_backend_timer_init(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + backend->scheduling_timer.function = timer_callback; + + backend->timer_running = false; + + return 0; +} + +void kbase_backend_timer_term(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + hrtimer_cancel(&backend->scheduling_timer); +} + +void kbase_backend_timer_suspend(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + backend->suspend_timer = true; + + kbase_backend_ctx_count_changed(kbdev); +} + +void kbase_backend_timer_resume(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + backend->suspend_timer = false; + + kbase_backend_ctx_count_changed(kbdev); +} + +void kbase_backend_timeouts_changed(struct kbase_device *kbdev) +{ + struct kbase_backend_data *backend = &kbdev->hwaccess.backend; + + backend->timeouts_updated = true; +} + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h new file mode 100755 index 000000000000..3f53779c6747 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h @@ -0,0 +1,69 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Register-based HW access backend specific job scheduler APIs + */ + +#ifndef _KBASE_JS_BACKEND_H_ +#define _KBASE_JS_BACKEND_H_ + +/** + * kbase_backend_timer_init() - Initialise the JS scheduling timer + * @kbdev: Device pointer + * + * This function should be called at driver initialisation + * + * Return: 0 on success + */ +int kbase_backend_timer_init(struct kbase_device *kbdev); + +/** + * kbase_backend_timer_term() - Terminate the JS scheduling timer + * @kbdev: Device pointer + * + * This function should be called at driver termination + */ +void kbase_backend_timer_term(struct kbase_device *kbdev); + +/** + * kbase_backend_timer_suspend - Suspend is happening, stop the JS scheduling + * timer + * @kbdev: Device pointer + * + * This function should be called on suspend, after the active count has reached + * zero. This is required as the timer may have been started on job submission + * to the job scheduler, but before jobs are submitted to the GPU. + * + * Caller must hold runpool_mutex. + */ +void kbase_backend_timer_suspend(struct kbase_device *kbdev); + +/** + * kbase_backend_timer_resume - Resume is happening, re-evaluate the JS + * scheduling timer + * @kbdev: Device pointer + * + * This function should be called on resume. Note that is is not guaranteed to + * re-start the timer, only evalute whether it should be re-started. + * + * Caller must hold runpool_mutex. + */ +void kbase_backend_timer_resume(struct kbase_device *kbdev); + +#endif /* _KBASE_JS_BACKEND_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.c new file mode 100755 index 000000000000..ba826184dd3f --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.c @@ -0,0 +1,407 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +/* #define ENABLE_DEBUG_LOG */ +#include "../../platform/rk/custom_log.h" + +#include + +#include +#include +#include +#include +#include +#include + +static inline u64 lock_region(struct kbase_device *kbdev, u64 pfn, + u32 num_pages) +{ + u64 region; + + /* can't lock a zero sized range */ + KBASE_DEBUG_ASSERT(num_pages); + + region = pfn << PAGE_SHIFT; + /* + * fls returns (given the ASSERT above): + * 1 .. 32 + * + * 10 + fls(num_pages) + * results in the range (11 .. 42) + */ + + /* gracefully handle num_pages being zero */ + if (0 == num_pages) { + region |= 11; + } else { + u8 region_width; + + region_width = 10 + fls(num_pages); + if (num_pages != (1ul << (region_width - 11))) { + /* not pow2, so must go up to the next pow2 */ + region_width += 1; + } + KBASE_DEBUG_ASSERT(region_width <= KBASE_LOCK_REGION_MAX_SIZE); + KBASE_DEBUG_ASSERT(region_width >= KBASE_LOCK_REGION_MIN_SIZE); + region |= region_width; + } + + return region; +} + +static int wait_ready(struct kbase_device *kbdev, + unsigned int as_nr, struct kbase_context *kctx) +{ + unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; + u32 val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), kctx); + + /* Wait for the MMU status to indicate there is no active command, in + * case one is pending. Do not log remaining register accesses. */ + while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) + val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), NULL); + + if (max_loops == 0) { + dev_err(kbdev->dev, "AS_ACTIVE bit stuck\n"); + return -1; + } + + /* If waiting in loop was performed, log last read value. */ + if (KBASE_AS_INACTIVE_MAX_LOOPS - 1 > max_loops) + kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), kctx); + + return 0; +} + +static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd, + struct kbase_context *kctx) +{ + int status; + + /* write AS_COMMAND when MMU is ready to accept another command */ + status = wait_ready(kbdev, as_nr, kctx); + if (status == 0) + kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd, + kctx); + + return status; +} + +static void validate_protected_page_fault(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + /* GPUs which support (native) protected mode shall not report page + * fault addresses unless it has protected debug mode and protected + * debug mode is turned on */ + u32 protected_debug_mode = 0; + + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) + return; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { + protected_debug_mode = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_STATUS), + kctx) & GPU_DBGEN; + } + + if (!protected_debug_mode) { + /* fault_addr should never be reported in protected mode. + * However, we just continue by printing an error message */ + dev_err(kbdev->dev, "Fault address reported in protected mode\n"); + } +} + +void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) +{ + const int num_as = 16; + const int busfault_shift = MMU_PAGE_FAULT_FLAGS; + const int pf_shift = 0; + const unsigned long as_bit_mask = (1UL << num_as) - 1; + unsigned long flags; + u32 new_mask; + u32 tmp; + + /* bus faults */ + u32 bf_bits = (irq_stat >> busfault_shift) & as_bit_mask; + /* page faults (note: Ignore ASes with both pf and bf) */ + u32 pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + + /* remember current mask */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL); + /* mask interrupts for now */ + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); + + while (bf_bits | pf_bits) { + struct kbase_as *as; + int as_no; + struct kbase_context *kctx; + + /* + * the while logic ensures we have a bit set, no need to check + * for not-found here + */ + as_no = ffs(bf_bits | pf_bits) - 1; + as = &kbdev->as[as_no]; + + /* + * Refcount the kctx ASAP - it shouldn't disappear anyway, since + * Bus/Page faults _should_ only occur whilst jobs are running, + * and a job causing the Bus/Page fault shouldn't complete until + * the MMU is updated + */ + kctx = kbasep_js_runpool_lookup_ctx(kbdev, as_no); + if (!kctx) { + E("fail to lookup ctx, to break out."); + break; + } + + + /* find faulting address */ + as->fault_addr = kbase_reg_read(kbdev, + MMU_AS_REG(as_no, + AS_FAULTADDRESS_HI), + kctx); + as->fault_addr <<= 32; + as->fault_addr |= kbase_reg_read(kbdev, + MMU_AS_REG(as_no, + AS_FAULTADDRESS_LO), + kctx); + + /* Mark the fault protected or not */ + as->protected_mode = kbdev->protected_mode; + + if (kbdev->protected_mode && as->fault_addr) + { + /* check if address reporting is allowed */ + validate_protected_page_fault(kbdev, kctx); + } + + /* report the fault to debugfs */ + kbase_as_fault_debugfs_new(kbdev, as_no); + + /* record the fault status */ + as->fault_status = kbase_reg_read(kbdev, + MMU_AS_REG(as_no, + AS_FAULTSTATUS), + kctx); + + /* find the fault type */ + as->fault_type = (bf_bits & (1 << as_no)) ? + KBASE_MMU_FAULT_TYPE_BUS : + KBASE_MMU_FAULT_TYPE_PAGE; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { + as->fault_extra_addr = kbase_reg_read(kbdev, + MMU_AS_REG(as_no, AS_FAULTEXTRA_HI), + kctx); + as->fault_extra_addr <<= 32; + as->fault_extra_addr |= kbase_reg_read(kbdev, + MMU_AS_REG(as_no, AS_FAULTEXTRA_LO), + kctx); + } + + if (kbase_as_has_bus_fault(as)) { + /* Mark bus fault as handled. + * Note that a bus fault is processed first in case + * where both a bus fault and page fault occur. + */ + bf_bits &= ~(1UL << as_no); + + /* remove the queued BF (and PF) from the mask */ + new_mask &= ~(MMU_BUS_ERROR(as_no) | + MMU_PAGE_FAULT(as_no)); + } else { + /* Mark page fault as handled */ + pf_bits &= ~(1UL << as_no); + + /* remove the queued PF from the mask */ + new_mask &= ~MMU_PAGE_FAULT(as_no); + } + + /* Process the interrupt for this address space */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_interrupt_process(kbdev, kctx, as); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + /* reenable interrupts */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL); + new_mask |= tmp; + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask, NULL); + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); +} + +void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx) +{ + struct kbase_mmu_setup *current_setup = &as->current_setup; + u32 transcfg = 0; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { + transcfg = current_setup->transcfg & 0xFFFFFFFFUL; + + /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK */ + /* Clear PTW_MEMATTR bits */ + transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; + /* Enable correct PTW_MEMATTR bits */ + transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; + + if (kbdev->system_coherency == COHERENCY_ACE) { + /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) */ + /* Clear PTW_SH bits */ + transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); + /* Enable correct PTW_SH bits */ + transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); + } + + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), + transcfg, kctx); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), + (current_setup->transcfg >> 32) & 0xFFFFFFFFUL, + kctx); + } else { + if (kbdev->system_coherency == COHERENCY_ACE) + current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER; + } + + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO), + current_setup->transtab & 0xFFFFFFFFUL, kctx); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI), + (current_setup->transtab >> 32) & 0xFFFFFFFFUL, kctx); + + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO), + current_setup->memattr & 0xFFFFFFFFUL, kctx); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI), + (current_setup->memattr >> 32) & 0xFFFFFFFFUL, kctx); + + KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(as, + current_setup->transtab, + current_setup->memattr, + transcfg); + + write_cmd(kbdev, as->number, AS_COMMAND_UPDATE, kctx); +} + +int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, u64 vpfn, u32 nr, u32 op, + unsigned int handling_irq) +{ + int ret; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + + if (op == AS_COMMAND_UNLOCK) { + /* Unlock doesn't require a lock first */ + ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); + } else { + u64 lock_addr = lock_region(kbdev, vpfn, nr); + + /* Lock the region that needs to be updated */ + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_LO), + lock_addr & 0xFFFFFFFFUL, kctx); + kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_HI), + (lock_addr >> 32) & 0xFFFFFFFFUL, kctx); + write_cmd(kbdev, as->number, AS_COMMAND_LOCK, kctx); + + /* Run the MMU operation */ + write_cmd(kbdev, as->number, op, kctx); + + /* Wait for the flush to complete */ + ret = wait_ready(kbdev, as->number, kctx); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9630)) { + /* Issue an UNLOCK command to ensure that valid page + tables are re-read by the GPU after an update. + Note that, the FLUSH command should perform all the + actions necessary, however the bus logs show that if + multiple page faults occur within an 8 page region + the MMU does not always re-read the updated page + table entries for later faults or is only partially + read, it subsequently raises the page fault IRQ for + the same addresses, the unlock ensures that the MMU + cache is flushed, so updates can be re-read. As the + region is now unlocked we need to issue 2 UNLOCK + commands in order to flush the MMU/uTLB, + see PRLAM-8812. + */ + write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); + write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); + } + } + + return ret; +} + +void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, enum kbase_mmu_fault_type type) +{ + unsigned long flags; + u32 pf_bf_mask; + + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + + /* + * A reset is in-flight and we're flushing the IRQ + bottom half + * so don't update anything as it could race with the reset code. + */ + if (kbdev->irq_reset_flush) + goto unlock; + + /* Clear the page (and bus fault IRQ as well in case one occurred) */ + pf_bf_mask = MMU_PAGE_FAULT(as->number); + if (type == KBASE_MMU_FAULT_TYPE_BUS || + type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) + pf_bf_mask |= MMU_BUS_ERROR(as->number); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask, kctx); + +unlock: + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); +} + +void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, enum kbase_mmu_fault_type type) +{ + unsigned long flags; + u32 irq_mask; + + /* Enable the page fault IRQ (and bus fault IRQ as well in case one + * occurred) */ + spin_lock_irqsave(&kbdev->mmu_mask_change, flags); + + /* + * A reset is in-flight and we're flushing the IRQ + bottom half + * so don't update anything as it could race with the reset code. + */ + if (kbdev->irq_reset_flush) + goto unlock; + + irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), kctx) | + MMU_PAGE_FAULT(as->number); + + if (type == KBASE_MMU_FAULT_TYPE_BUS || + type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) + irq_mask |= MMU_BUS_ERROR(as->number); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask, kctx); + +unlock: + spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); +} diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.h new file mode 100755 index 000000000000..c02253c6acc3 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.h @@ -0,0 +1,42 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Interface file for the direct implementation for MMU hardware access + * + * Direct MMU hardware interface + * + * This module provides the interface(s) that are required by the direct + * register access implementation of the MMU hardware interface + */ + +#ifndef _MALI_KBASE_MMU_HW_DIRECT_H_ +#define _MALI_KBASE_MMU_HW_DIRECT_H_ + +#include + +/** + * kbase_mmu_interrupt - Process an MMU interrupt. + * + * Process the MMU interrupt that was reported by the &kbase_device. + * + * @kbdev: kbase context to clear the fault from. + * @irq_stat: Value of the MMU_IRQ_STATUS register + */ +void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); + +#endif /* _MALI_KBASE_MMU_HW_DIRECT_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c new file mode 100755 index 000000000000..0614348e935a --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c @@ -0,0 +1,63 @@ +/* + * + * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * "Always on" power management policy + */ + +#include +#include + +static u64 always_on_get_core_mask(struct kbase_device *kbdev) +{ + return kbdev->gpu_props.props.raw_props.shader_present; +} + +static bool always_on_get_core_active(struct kbase_device *kbdev) +{ + return true; +} + +static void always_on_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static void always_on_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +/* + * The struct kbase_pm_policy structure for the demand power policy. + * + * This is the static structure that defines the demand power policy's callback + * and name. + */ +const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { + "always_on", /* name */ + always_on_init, /* init */ + always_on_term, /* term */ + always_on_get_core_mask, /* get_core_mask */ + always_on_get_core_active, /* get_core_active */ + 0u, /* flags */ + KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ +}; + +KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h new file mode 100755 index 000000000000..f9d244b01bc2 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h @@ -0,0 +1,77 @@ + +/* + * + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * "Always on" power management policy + */ + +#ifndef MALI_KBASE_PM_ALWAYS_ON_H +#define MALI_KBASE_PM_ALWAYS_ON_H + +/** + * DOC: + * The "Always on" power management policy has the following + * characteristics: + * + * - When KBase indicates that the GPU will be powered up, but we don't yet + * know which Job Chains are to be run: + * All Shader Cores are powered up, regardless of whether or not they will + * be needed later. + * + * - When KBase indicates that a set of Shader Cores are needed to submit the + * currently queued Job Chains: + * All Shader Cores are kept powered, regardless of whether or not they will + * be needed + * + * - When KBase indicates that the GPU need not be powered: + * The Shader Cores are kept powered, regardless of whether or not they will + * be needed. The GPU itself is also kept powered, even though it is not + * needed. + * + * This policy is automatically overridden during system suspend: the desired + * core state is ignored, and the cores are forced off regardless of what the + * policy requests. After resuming from suspend, new changes to the desired + * core state made by the policy are honored. + * + * Note: + * + * - KBase indicates the GPU will be powered up when it has a User Process that + * has just started to submit Job Chains. + * + * - KBase indicates the GPU need not be powered when all the Job Chains from + * User Processes have finished, and it is waiting for a User Process to + * submit some more Job Chains. + */ + +/** + * struct kbasep_pm_policy_always_on - Private struct for policy instance data + * @dummy: unused dummy variable + * + * This contains data that is private to the particular power policy that is + * active. + */ +struct kbasep_pm_policy_always_on { + int dummy; +}; + +extern const struct kbase_pm_policy kbase_pm_always_on_policy_ops; + +#endif /* MALI_KBASE_PM_ALWAYS_ON_H */ + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c new file mode 100755 index 000000000000..146fd48bab92 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c @@ -0,0 +1,482 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * GPU backend implementation of base kernel power management APIs + */ + +#include +#include +#include +#ifdef CONFIG_MALI_PLATFORM_DEVICETREE +#include +#endif /* CONFIG_MALI_PLATFORM_DEVICETREE */ + +#include +#include +#include +#include +#include +#include + +static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data); + +void kbase_pm_register_access_enable(struct kbase_device *kbdev) +{ + struct kbase_pm_callback_conf *callbacks; + + callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; + + if (callbacks) + callbacks->power_on_callback(kbdev); + + kbdev->pm.backend.gpu_powered = true; +} + +void kbase_pm_register_access_disable(struct kbase_device *kbdev) +{ + struct kbase_pm_callback_conf *callbacks; + + callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; + + if (callbacks) + callbacks->power_off_callback(kbdev); + + kbdev->pm.backend.gpu_powered = false; +} + +int kbase_hwaccess_pm_init(struct kbase_device *kbdev) +{ + int ret = 0; + struct kbase_pm_callback_conf *callbacks; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + mutex_init(&kbdev->pm.lock); + + kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait", + WQ_HIGHPRI | WQ_UNBOUND, 1); + if (!kbdev->pm.backend.gpu_poweroff_wait_wq) + return -ENOMEM; + + INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, + kbase_pm_gpu_poweroff_wait_wq); + + kbdev->pm.backend.gpu_powered = false; + kbdev->pm.suspending = false; +#ifdef CONFIG_MALI_DEBUG + kbdev->pm.backend.driver_ready_for_irqs = false; +#endif /* CONFIG_MALI_DEBUG */ + kbdev->pm.backend.gpu_in_desired_state = true; + init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); + + callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; + if (callbacks) { + kbdev->pm.backend.callback_power_on = + callbacks->power_on_callback; + kbdev->pm.backend.callback_power_off = + callbacks->power_off_callback; + kbdev->pm.backend.callback_power_suspend = + callbacks->power_suspend_callback; + kbdev->pm.backend.callback_power_resume = + callbacks->power_resume_callback; + kbdev->pm.callback_power_runtime_init = + callbacks->power_runtime_init_callback; + kbdev->pm.callback_power_runtime_term = + callbacks->power_runtime_term_callback; + kbdev->pm.backend.callback_power_runtime_on = + callbacks->power_runtime_on_callback; + kbdev->pm.backend.callback_power_runtime_off = + callbacks->power_runtime_off_callback; + kbdev->pm.backend.callback_power_runtime_idle = + callbacks->power_runtime_idle_callback; + } else { + kbdev->pm.backend.callback_power_on = NULL; + kbdev->pm.backend.callback_power_off = NULL; + kbdev->pm.backend.callback_power_suspend = NULL; + kbdev->pm.backend.callback_power_resume = NULL; + kbdev->pm.callback_power_runtime_init = NULL; + kbdev->pm.callback_power_runtime_term = NULL; + kbdev->pm.backend.callback_power_runtime_on = NULL; + kbdev->pm.backend.callback_power_runtime_off = NULL; + kbdev->pm.backend.callback_power_runtime_idle = NULL; + } + + /* Initialise the metrics subsystem */ + ret = kbasep_pm_metrics_init(kbdev); + if (ret) + return ret; + + init_waitqueue_head(&kbdev->pm.backend.l2_powered_wait); + kbdev->pm.backend.l2_powered = 0; + + init_waitqueue_head(&kbdev->pm.backend.reset_done_wait); + kbdev->pm.backend.reset_done = false; + + init_waitqueue_head(&kbdev->pm.zero_active_count_wait); + kbdev->pm.active_count = 0; + + spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock); + spin_lock_init(&kbdev->pm.backend.gpu_powered_lock); + + init_waitqueue_head(&kbdev->pm.backend.poweroff_wait); + + if (kbase_pm_ca_init(kbdev) != 0) + goto workq_fail; + + if (kbase_pm_policy_init(kbdev) != 0) + goto pm_policy_fail; + + return 0; + +pm_policy_fail: + kbase_pm_ca_term(kbdev); +workq_fail: + kbasep_pm_metrics_term(kbdev); + return -EINVAL; +} + +void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) +{ + lockdep_assert_held(&kbdev->pm.lock); + + /* Turn clocks and interrupts on - no-op if we haven't done a previous + * kbase_pm_clock_off() */ + kbase_pm_clock_on(kbdev, is_resume); + + /* Update core status as required by the policy */ + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START); + kbase_pm_update_cores_state(kbdev); + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END); + + /* NOTE: We don't wait to reach the desired state, since running atoms + * will wait for that state to be reached anyway */ +} + +static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) +{ + struct kbase_device *kbdev = container_of(data, struct kbase_device, + pm.backend.gpu_poweroff_wait_work); + struct kbase_pm_device_data *pm = &kbdev->pm; + struct kbase_pm_backend_data *backend = &pm->backend; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + unsigned long flags; + +#if !PLATFORM_POWER_DOWN_ONLY + /* Wait for power transitions to complete. We do this with no locks held + * so that we don't deadlock with any pending workqueues */ + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START); + kbase_pm_check_transitions_sync(kbdev); + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END); +#endif /* !PLATFORM_POWER_DOWN_ONLY */ + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + +#if PLATFORM_POWER_DOWN_ONLY + if (kbdev->pm.backend.gpu_powered) { + if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2)) { + /* If L2 cache is powered then we must flush it before + * we power off the GPU. Normally this would have been + * handled when the L2 was powered off. */ + kbase_gpu_cacheclean(kbdev); + } + } +#endif /* PLATFORM_POWER_DOWN_ONLY */ + + if (!backend->poweron_required) { +#if !PLATFORM_POWER_DOWN_ONLY + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + WARN_ON(kbdev->l2_available_bitmap || + kbdev->shader_available_bitmap || + kbdev->tiler_available_bitmap); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#endif /* !PLATFORM_POWER_DOWN_ONLY */ + + /* Consume any change-state events */ + kbase_timeline_pm_check_handle_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + + /* Disable interrupts and turn the clock off */ + if (!kbase_pm_clock_off(kbdev, backend->poweroff_is_suspend)) { + /* + * Page/bus faults are pending, must drop locks to + * process. Interrupts are disabled so no more faults + * should be generated at this point. + */ + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + kbase_flush_mmu_wqs(kbdev); + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + /* Turn off clock now that fault have been handled. We + * dropped locks so poweron_required may have changed - + * power back on if this is the case.*/ + if (backend->poweron_required) + kbase_pm_clock_on(kbdev, false); + else + WARN_ON(!kbase_pm_clock_off(kbdev, + backend->poweroff_is_suspend)); + } + } + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + backend->poweroff_wait_in_progress = false; + if (backend->poweron_required) { + backend->poweron_required = false; + kbase_pm_update_cores_state_nolock(kbdev); + kbase_backend_slot_update(kbdev); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + + wake_up(&kbdev->pm.backend.poweroff_wait); +} + +void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend) +{ + unsigned long flags; + + lockdep_assert_held(&kbdev->pm.lock); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (!kbdev->pm.backend.poweroff_wait_in_progress) { + /* Force all cores off */ + kbdev->pm.backend.desired_shader_state = 0; + kbdev->pm.backend.desired_tiler_state = 0; + + /* Force all cores to be unavailable, in the situation where + * transitions are in progress for some cores but not others, + * and kbase_pm_check_transitions_nolock can not immediately + * power off the cores */ + kbdev->shader_available_bitmap = 0; + kbdev->tiler_available_bitmap = 0; + kbdev->l2_available_bitmap = 0; + + kbdev->pm.backend.poweroff_wait_in_progress = true; + kbdev->pm.backend.poweroff_is_suspend = is_suspend; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + /*Kick off wq here. Callers will have to wait*/ + queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq, + &kbdev->pm.backend.gpu_poweroff_wait_work); + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } +} + +static bool is_poweroff_in_progress(struct kbase_device *kbdev) +{ + bool ret; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + ret = (kbdev->pm.backend.poweroff_wait_in_progress == false); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return ret; +} + +void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev) +{ + wait_event_killable(kbdev->pm.backend.poweroff_wait, + is_poweroff_in_progress(kbdev)); +} + +int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, + unsigned int flags) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + unsigned long irq_flags; + int ret; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + /* A suspend won't happen during startup/insmod */ + KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); + + /* Power up the GPU, don't enable IRQs as we are not ready to receive + * them. */ + ret = kbase_pm_init_hw(kbdev, flags); + if (ret) { + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + return ret; + } + + kbasep_pm_init_core_use_bitmaps(kbdev); + + kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] = + kbdev->pm.debug_core_mask[1] = + kbdev->pm.debug_core_mask[2] = + kbdev->gpu_props.props.raw_props.shader_present; + + /* Pretend the GPU is active to prevent a power policy turning the GPU + * cores off */ + kbdev->pm.active_count = 1; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + /* Ensure cycle counter is off */ + kbdev->pm.backend.gpu_cycle_counter_requests = 0; + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + + /* We are ready to receive IRQ's now as power policy is set up, so + * enable them now. */ +#ifdef CONFIG_MALI_DEBUG + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, irq_flags); + kbdev->pm.backend.driver_ready_for_irqs = true; + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, irq_flags); +#endif + kbase_pm_enable_interrupts(kbdev); + + /* Turn on the GPU and any cores needed by the policy */ + kbase_pm_do_poweron(kbdev, false); + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + + /* Idle the GPU and/or cores, if the policy wants it to */ + kbase_pm_context_idle(kbdev); + + return 0; +} + +void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + mutex_lock(&kbdev->pm.lock); + kbase_pm_cancel_deferred_poweroff(kbdev); + kbase_pm_do_poweroff(kbdev, false); + mutex_unlock(&kbdev->pm.lock); +} + +KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); + +void kbase_hwaccess_pm_term(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0); + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0); + + /* Free any resources the policy allocated */ + kbase_pm_policy_term(kbdev); + kbase_pm_ca_term(kbdev); + + /* Shut down the metrics subsystem */ + kbasep_pm_metrics_term(kbdev); + + destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); +} + +void kbase_pm_power_changed(struct kbase_device *kbdev) +{ + bool cores_are_available; + unsigned long flags; + + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + cores_are_available = kbase_pm_check_transitions_nolock(kbdev); + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END); + + if (cores_are_available) { + /* Log timelining information that a change in state has + * completed */ + kbase_timeline_pm_handle_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + + kbase_backend_slot_update(kbdev); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, + u64 new_core_mask_js0, u64 new_core_mask_js1, + u64 new_core_mask_js2) +{ + kbdev->pm.debug_core_mask[0] = new_core_mask_js0; + kbdev->pm.debug_core_mask[1] = new_core_mask_js1; + kbdev->pm.debug_core_mask[2] = new_core_mask_js2; + kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 | + new_core_mask_js2; + + kbase_pm_update_cores_state_nolock(kbdev); +} + +void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) +{ + kbase_pm_update_active(kbdev); +} + +void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev) +{ + kbase_pm_update_active(kbdev); +} + +void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + /* Force power off the GPU and all cores (regardless of policy), only + * after the PM active count reaches zero (otherwise, we risk turning it + * off prematurely) */ + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + kbase_pm_cancel_deferred_poweroff(kbdev); + kbase_pm_do_poweroff(kbdev, true); + + kbase_backend_timer_suspend(kbdev); + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + + kbase_pm_wait_for_poweroff_complete(kbdev); +} + +void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + kbdev->pm.suspending = false; + kbase_pm_do_poweron(kbdev, true); + + kbase_backend_timer_resume(kbdev); + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); +} diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c new file mode 100755 index 000000000000..85890f1e85f5 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c @@ -0,0 +1,182 @@ +/* + * + * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Base kernel core availability APIs + */ + +#include +#include +#include + +static const struct kbase_pm_ca_policy *const policy_list[] = { + &kbase_pm_ca_fixed_policy_ops, +#ifdef CONFIG_MALI_DEVFREQ + &kbase_pm_ca_devfreq_policy_ops, +#endif +#if !MALI_CUSTOMER_RELEASE + &kbase_pm_ca_random_policy_ops +#endif +}; + +/** + * POLICY_COUNT - The number of policies available in the system. + * + * This is derived from the number of functions listed in policy_list. + */ +#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list)) + +int kbase_pm_ca_init(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + kbdev->pm.backend.ca_current_policy = policy_list[0]; + + kbdev->pm.backend.ca_current_policy->init(kbdev); + + return 0; +} + +void kbase_pm_ca_term(struct kbase_device *kbdev) +{ + kbdev->pm.backend.ca_current_policy->term(kbdev); +} + +int kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **list) +{ + if (!list) + return POLICY_COUNT; + + *list = policy_list; + + return POLICY_COUNT; +} + +KBASE_EXPORT_TEST_API(kbase_pm_ca_list_policies); + +const struct kbase_pm_ca_policy +*kbase_pm_ca_get_policy(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + return kbdev->pm.backend.ca_current_policy; +} + +KBASE_EXPORT_TEST_API(kbase_pm_ca_get_policy); + +void kbase_pm_ca_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_ca_policy *new_policy) +{ + const struct kbase_pm_ca_policy *old_policy; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(new_policy != NULL); + + KBASE_TRACE_ADD(kbdev, PM_CA_SET_POLICY, NULL, NULL, 0u, + new_policy->id); + + /* During a policy change we pretend the GPU is active */ + /* A suspend won't happen here, because we're in a syscall from a + * userspace thread */ + kbase_pm_context_active(kbdev); + + mutex_lock(&kbdev->pm.lock); + + /* Remove the policy to prevent IRQ handlers from working on it */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + old_policy = kbdev->pm.backend.ca_current_policy; + kbdev->pm.backend.ca_current_policy = NULL; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (old_policy->term) + old_policy->term(kbdev); + + if (new_policy->init) + new_policy->init(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.ca_current_policy = new_policy; + + /* If any core power state changes were previously attempted, but + * couldn't be made because the policy was changing (current_policy was + * NULL), then re-try them here. */ + kbase_pm_update_cores_state_nolock(kbdev); + + kbdev->pm.backend.ca_current_policy->update_core_status(kbdev, + kbdev->shader_ready_bitmap, + kbdev->shader_transitioning_bitmap); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->pm.lock); + + /* Now the policy change is finished, we release our fake context active + * reference */ + kbase_pm_context_idle(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_ca_set_policy); + +u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* All cores must be enabled when instrumentation is in use */ + if (kbdev->pm.backend.instr_enabled) + return kbdev->gpu_props.props.raw_props.shader_present & + kbdev->pm.debug_core_mask_all; + + if (kbdev->pm.backend.ca_current_policy == NULL) + return kbdev->gpu_props.props.raw_props.shader_present & + kbdev->pm.debug_core_mask_all; + + return kbdev->pm.backend.ca_current_policy->get_core_mask(kbdev) & + kbdev->pm.debug_core_mask_all; +} + +KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask); + +void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, + u64 cores_transitioning) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->pm.backend.ca_current_policy != NULL) + kbdev->pm.backend.ca_current_policy->update_core_status(kbdev, + cores_ready, + cores_transitioning); +} + +void kbase_pm_ca_instr_enable(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.instr_enabled = true; + + kbase_pm_update_cores_state_nolock(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +void kbase_pm_ca_instr_disable(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + kbdev->pm.backend.instr_enabled = false; + + kbase_pm_update_cores_state_nolock(kbdev); +} diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h new file mode 100755 index 000000000000..ee9e751f2d79 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h @@ -0,0 +1,92 @@ +/* + * + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Base kernel core availability APIs + */ + +#ifndef _KBASE_PM_CA_H_ +#define _KBASE_PM_CA_H_ + +/** + * kbase_pm_ca_init - Initialize core availability framework + * + * Must be called before calling any other core availability function + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 if the core availability framework was successfully initialized, + * -errno otherwise + */ +int kbase_pm_ca_init(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_term - Terminate core availability framework + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_ca_term(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_get_core_mask - Get currently available shaders core mask + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Returns a mask of the currently available shader cores. + * Calls into the core availability policy + * + * Return: The bit mask of available cores + */ +u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_update_core_status - Update core status + * + * @kbdev: The kbase device structure for the device (must be + * a valid pointer) + * @cores_ready: The bit mask of cores ready for job submission + * @cores_transitioning: The bit mask of cores that are transitioning power + * state + * + * Update core availability policy with current core power status + * + * Calls into the core availability policy + */ +void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, + u64 cores_transitioning); + +/** + * kbase_pm_ca_instr_enable - Enable override for instrumentation + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This overrides the output of the core availability policy, ensuring that all + * cores are available + */ +void kbase_pm_ca_instr_enable(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_instr_disable - Disable override for instrumentation + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This disables any previously enabled override, and resumes normal policy + * functionality + */ +void kbase_pm_ca_instr_disable(struct kbase_device *kbdev); + +#endif /* _KBASE_PM_CA_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.c new file mode 100755 index 000000000000..66bf660cffb6 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.c @@ -0,0 +1,129 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * A core availability policy implementing core mask selection from devfreq OPPs + * + */ + +#include +#include +#include +#include + +void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) +{ + struct kbasep_pm_ca_policy_devfreq *data = + &kbdev->pm.backend.ca_policy_data.devfreq; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + data->cores_desired = core_mask; + + /* Disable any cores that are now unwanted */ + data->cores_enabled &= data->cores_desired; + + kbdev->pm.backend.ca_in_transition = true; + + /* If there are no cores to be powered off then power on desired cores + */ + if (!(data->cores_used & ~data->cores_desired)) { + data->cores_enabled = data->cores_desired; + kbdev->pm.backend.ca_in_transition = false; + } + + kbase_pm_update_cores_state_nolock(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX %llX\n", + data->cores_desired, data->cores_enabled); +} + +static void devfreq_init(struct kbase_device *kbdev) +{ + struct kbasep_pm_ca_policy_devfreq *data = + &kbdev->pm.backend.ca_policy_data.devfreq; + + if (kbdev->current_core_mask) { + data->cores_enabled = kbdev->current_core_mask; + data->cores_desired = kbdev->current_core_mask; + } else { + data->cores_enabled = + kbdev->gpu_props.props.raw_props.shader_present; + data->cores_desired = + kbdev->gpu_props.props.raw_props.shader_present; + } + data->cores_used = 0; + kbdev->pm.backend.ca_in_transition = false; +} + +static void devfreq_term(struct kbase_device *kbdev) +{ +} + +static u64 devfreq_get_core_mask(struct kbase_device *kbdev) +{ + return kbdev->pm.backend.ca_policy_data.devfreq.cores_enabled; +} + +static void devfreq_update_core_status(struct kbase_device *kbdev, + u64 cores_ready, + u64 cores_transitioning) +{ + struct kbasep_pm_ca_policy_devfreq *data = + &kbdev->pm.backend.ca_policy_data.devfreq; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + data->cores_used = cores_ready | cores_transitioning; + + /* If in desired state then clear transition flag */ + if (data->cores_enabled == data->cores_desired) + kbdev->pm.backend.ca_in_transition = false; + + /* If all undesired cores are now off then power on desired cores. + * The direct comparison against cores_enabled limits potential + * recursion to one level */ + if (!(data->cores_used & ~data->cores_desired) && + data->cores_enabled != data->cores_desired) { + data->cores_enabled = data->cores_desired; + + kbase_pm_update_cores_state_nolock(kbdev); + + kbdev->pm.backend.ca_in_transition = false; + } +} + +/* + * The struct kbase_pm_ca_policy structure for the devfreq core availability + * policy. + * + * This is the static structure that defines the devfreq core availability power + * policy's callback and name. + */ +const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops = { + "devfreq", /* name */ + devfreq_init, /* init */ + devfreq_term, /* term */ + devfreq_get_core_mask, /* get_core_mask */ + devfreq_update_core_status, /* update_core_status */ + 0u, /* flags */ + KBASE_PM_CA_POLICY_ID_DEVFREQ, /* id */ +}; + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h new file mode 100755 index 000000000000..7ab3cd4d8460 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h @@ -0,0 +1,55 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * A core availability policy for use with devfreq, where core masks are + * associated with OPPs. + */ + +#ifndef MALI_KBASE_PM_CA_DEVFREQ_H +#define MALI_KBASE_PM_CA_DEVFREQ_H + +/** + * struct kbasep_pm_ca_policy_devfreq - Private structure for devfreq ca policy + * + * This contains data that is private to the devfreq core availability + * policy. + * + * @cores_desired: Cores that the policy wants to be available + * @cores_enabled: Cores that the policy is currently returning as available + * @cores_used: Cores currently powered or transitioning + */ +struct kbasep_pm_ca_policy_devfreq { + u64 cores_desired; + u64 cores_enabled; + u64 cores_used; +}; + +extern const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops; + +/** + * kbase_devfreq_set_core_mask - Set core mask for policy to use + * @kbdev: Device pointer + * @core_mask: New core mask + * + * The new core mask will have immediate effect if the GPU is powered, or will + * take effect when it is next powered on. + */ +void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask); + +#endif /* MALI_KBASE_PM_CA_DEVFREQ_H */ + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.c new file mode 100755 index 000000000000..864612d31f9b --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.c @@ -0,0 +1,65 @@ +/* + * + * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * A power policy implementing fixed core availability + */ + +#include +#include + +static void fixed_init(struct kbase_device *kbdev) +{ + kbdev->pm.backend.ca_in_transition = false; +} + +static void fixed_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static u64 fixed_get_core_mask(struct kbase_device *kbdev) +{ + return kbdev->gpu_props.props.raw_props.shader_present; +} + +static void fixed_update_core_status(struct kbase_device *kbdev, + u64 cores_ready, + u64 cores_transitioning) +{ + CSTD_UNUSED(kbdev); + CSTD_UNUSED(cores_ready); + CSTD_UNUSED(cores_transitioning); +} + +/* + * The struct kbase_pm_policy structure for the fixed power policy. + * + * This is the static structure that defines the fixed power policy's callback + * and name. + */ +const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops = { + "fixed", /* name */ + fixed_init, /* init */ + fixed_term, /* term */ + fixed_get_core_mask, /* get_core_mask */ + fixed_update_core_status, /* update_core_status */ + 0u, /* flags */ + KBASE_PM_CA_POLICY_ID_FIXED, /* id */ +}; + +KBASE_EXPORT_TEST_API(kbase_pm_ca_fixed_policy_ops); diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.h new file mode 100755 index 000000000000..a763155cb703 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.h @@ -0,0 +1,40 @@ +/* + * + * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * A power policy implementing fixed core availability + */ + +#ifndef MALI_KBASE_PM_CA_FIXED_H +#define MALI_KBASE_PM_CA_FIXED_H + +/** + * struct kbasep_pm_ca_policy_fixed - Private structure for policy instance data + * + * @dummy: Dummy member - no state is needed + * + * This contains data that is private to the particular power policy that is + * active. + */ +struct kbasep_pm_ca_policy_fixed { + int dummy; +}; + +extern const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops; + +#endif /* MALI_KBASE_PM_CA_FIXED_H */ + diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c new file mode 100755 index 000000000000..f891fa225a89 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c @@ -0,0 +1,70 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * "Coarse Demand" power management policy + */ + +#include +#include + +static u64 coarse_demand_get_core_mask(struct kbase_device *kbdev) +{ + if (kbdev->pm.active_count == 0) + return 0; + + return kbdev->gpu_props.props.raw_props.shader_present; +} + +static bool coarse_demand_get_core_active(struct kbase_device *kbdev) +{ + if (0 == kbdev->pm.active_count && !(kbdev->shader_needed_bitmap | + kbdev->shader_inuse_bitmap) && !kbdev->tiler_needed_cnt + && !kbdev->tiler_inuse_cnt) + return false; + + return true; +} + +static void coarse_demand_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static void coarse_demand_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +/* The struct kbase_pm_policy structure for the demand power policy. + * + * This is the static structure that defines the demand power policy's callback + * and name. + */ +const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { + "coarse_demand", /* name */ + coarse_demand_init, /* init */ + coarse_demand_term, /* term */ + coarse_demand_get_core_mask, /* get_core_mask */ + coarse_demand_get_core_active, /* get_core_active */ + 0u, /* flags */ + KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ +}; + +KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h new file mode 100755 index 000000000000..749d305eee9a --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h @@ -0,0 +1,64 @@ +/* + * + * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * "Coarse Demand" power management policy + */ + +#ifndef MALI_KBASE_PM_COARSE_DEMAND_H +#define MALI_KBASE_PM_COARSE_DEMAND_H + +/** + * DOC: + * The "Coarse" demand power management policy has the following + * characteristics: + * - When KBase indicates that the GPU will be powered up, but we don't yet + * know which Job Chains are to be run: + * - All Shader Cores are powered up, regardless of whether or not they will + * be needed later. + * - When KBase indicates that a set of Shader Cores are needed to submit the + * currently queued Job Chains: + * - All Shader Cores are kept powered, regardless of whether or not they will + * be needed + * - When KBase indicates that the GPU need not be powered: + * - The Shader Cores are powered off, and the GPU itself is powered off too. + * + * @note: + * - KBase indicates the GPU will be powered up when it has a User Process that + * has just started to submit Job Chains. + * - KBase indicates the GPU need not be powered when all the Job Chains from + * User Processes have finished, and it is waiting for a User Process to + * submit some more Job Chains. + */ + +/** + * struct kbasep_pm_policy_coarse_demand - Private structure for coarse demand + * policy + * + * This contains data that is private to the coarse demand power policy. + * + * @dummy: Dummy member - no state needed + */ +struct kbasep_pm_policy_coarse_demand { + int dummy; +}; + +extern const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops; + +#endif /* MALI_KBASE_PM_COARSE_DEMAND_H */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h new file mode 100755 index 000000000000..352744ee6d73 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h @@ -0,0 +1,519 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Backend-specific Power Manager definitions + */ + +#ifndef _KBASE_PM_HWACCESS_DEFS_H_ +#define _KBASE_PM_HWACCESS_DEFS_H_ + +#include "mali_kbase_pm_ca_fixed.h" +#include "mali_kbase_pm_ca_devfreq.h" +#if !MALI_CUSTOMER_RELEASE +#include "mali_kbase_pm_ca_random.h" +#endif + +#include "mali_kbase_pm_always_on.h" +#include "mali_kbase_pm_coarse_demand.h" +#include "mali_kbase_pm_demand.h" +#if !MALI_CUSTOMER_RELEASE +#include "mali_kbase_pm_demand_always_powered.h" +#include "mali_kbase_pm_fast_start.h" +#endif + +/* Forward definition - see mali_kbase.h */ +struct kbase_device; +struct kbase_jd_atom; + +/** + * enum kbase_pm_core_type - The types of core in a GPU. + * + * These enumerated values are used in calls to + * - kbase_pm_get_present_cores() + * - kbase_pm_get_active_cores() + * - kbase_pm_get_trans_cores() + * - kbase_pm_get_ready_cores(). + * + * They specify which type of core should be acted on. These values are set in + * a manner that allows core_type_to_reg() function to be simpler and more + * efficient. + * + * @KBASE_PM_CORE_L2: The L2 cache + * @KBASE_PM_CORE_SHADER: Shader cores + * @KBASE_PM_CORE_TILER: Tiler cores + * @KBASE_PM_CORE_STACK: Core stacks + */ +enum kbase_pm_core_type { + KBASE_PM_CORE_L2 = L2_PRESENT_LO, + KBASE_PM_CORE_SHADER = SHADER_PRESENT_LO, + KBASE_PM_CORE_TILER = TILER_PRESENT_LO, + KBASE_PM_CORE_STACK = STACK_PRESENT_LO +}; + +/** + * struct kbasep_pm_metrics_data - Metrics data collected for use by the power + * management framework. + * + * @time_period_start: time at which busy/idle measurements started + * @time_busy: number of ns the GPU was busy executing jobs since the + * @time_period_start timestamp. + * @time_idle: number of ns since time_period_start the GPU was not executing + * jobs since the @time_period_start timestamp. + * @prev_busy: busy time in ns of previous time period. + * Updated when metrics are reset. + * @prev_idle: idle time in ns of previous time period + * Updated when metrics are reset. + * @gpu_active: true when the GPU is executing jobs. false when + * not. Updated when the job scheduler informs us a job in submitted + * or removed from a GPU slot. + * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that + * if two CL jobs were active for 400ns, this value would be updated + * with 800. + * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that + * if two GL jobs were active for 400ns, this value would be updated + * with 800. + * @active_cl_ctx: number of CL jobs active on the GPU. Array is per-device. + * @active_gl_ctx: number of GL jobs active on the GPU. Array is per-slot. As + * GL jobs never run on slot 2 this slot is not recorded. + * @lock: spinlock protecting the kbasep_pm_metrics_data structure + * @timer: timer to regularly make DVFS decisions based on the power + * management metrics. + * @timer_active: boolean indicating @timer is running + * @platform_data: pointer to data controlled by platform specific code + * @kbdev: pointer to kbase device for which metrics are collected + * + */ +struct kbasep_pm_metrics_data { + ktime_t time_period_start; + u32 time_busy; + u32 time_idle; + u32 prev_busy; + u32 prev_idle; + bool gpu_active; + u32 busy_cl[2]; + u32 busy_gl; + u32 active_cl_ctx[2]; + u32 active_gl_ctx[2]; /* GL jobs can only run on 2 of the 3 job slots */ + spinlock_t lock; + +#ifdef CONFIG_MALI_MIDGARD_DVFS + struct hrtimer timer; + bool timer_active; +#endif + + void *platform_data; + struct kbase_device *kbdev; +}; + +union kbase_pm_policy_data { + struct kbasep_pm_policy_always_on always_on; + struct kbasep_pm_policy_coarse_demand coarse_demand; + struct kbasep_pm_policy_demand demand; +#if !MALI_CUSTOMER_RELEASE + struct kbasep_pm_policy_demand_always_powered demand_always_powered; + struct kbasep_pm_policy_fast_start fast_start; +#endif +}; + +union kbase_pm_ca_policy_data { + struct kbasep_pm_ca_policy_fixed fixed; + struct kbasep_pm_ca_policy_devfreq devfreq; +#if !MALI_CUSTOMER_RELEASE + struct kbasep_pm_ca_policy_random random; +#endif +}; + +/** + * struct kbase_pm_backend_data - Data stored per device for power management. + * + * This structure contains data for the power management framework. There is one + * instance of this structure per device in the system. + * + * @ca_current_policy: The policy that is currently actively controlling core + * availability. + * @pm_current_policy: The policy that is currently actively controlling the + * power state. + * @ca_policy_data: Private data for current CA policy + * @pm_policy_data: Private data for current PM policy + * @ca_in_transition: Flag indicating when core availability policy is + * transitioning cores. The core availability policy must + * set this when a change in core availability is occurring. + * power_change_lock must be held when accessing this. + * @reset_done: Flag when a reset is complete + * @reset_done_wait: Wait queue to wait for changes to @reset_done + * @l2_powered_wait: Wait queue for whether the l2 cache has been powered as + * requested + * @l2_powered: State indicating whether all the l2 caches are powered. + * Non-zero indicates they're *all* powered + * Zero indicates that some (or all) are not powered + * @gpu_cycle_counter_requests: The reference count of active gpu cycle counter + * users + * @gpu_cycle_counter_requests_lock: Lock to protect @gpu_cycle_counter_requests + * @desired_shader_state: A bit mask identifying the shader cores that the + * power policy would like to be on. The current state + * of the cores may be different, but there should be + * transitions in progress that will eventually achieve + * this state (assuming that the policy doesn't change + * its mind in the mean time). + * @powering_on_shader_state: A bit mask indicating which shader cores are + * currently in a power-on transition + * @desired_tiler_state: A bit mask identifying the tiler cores that the power + * policy would like to be on. See @desired_shader_state + * @powering_on_tiler_state: A bit mask indicating which tiler core are + * currently in a power-on transition + * @powering_on_l2_state: A bit mask indicating which l2-caches are currently + * in a power-on transition + * @powering_on_stack_state: A bit mask indicating which core stacks are + * currently in a power-on transition + * @gpu_in_desired_state: This flag is set if the GPU is powered as requested + * by the desired_xxx_state variables + * @gpu_in_desired_state_wait: Wait queue set when @gpu_in_desired_state != 0 + * @gpu_powered: Set to true when the GPU is powered and register + * accesses are possible, false otherwise + * @instr_enabled: Set to true when instrumentation is enabled, + * false otherwise + * @cg1_disabled: Set if the policy wants to keep the second core group + * powered off + * @driver_ready_for_irqs: Debug state indicating whether sufficient + * initialization of the driver has occurred to handle + * IRQs + * @gpu_powered_lock: Spinlock that must be held when writing @gpu_powered or + * accessing @driver_ready_for_irqs + * @metrics: Structure to hold metrics for the GPU + * @gpu_poweroff_pending: number of poweroff timer ticks until the GPU is + * powered off + * @shader_poweroff_pending_time: number of poweroff timer ticks until shaders + * and/or timers are powered off + * @gpu_poweroff_timer: Timer for powering off GPU + * @gpu_poweroff_wq: Workqueue to power off GPU on when timer fires + * @gpu_poweroff_work: Workitem used on @gpu_poweroff_wq + * @shader_poweroff_pending: Bit mask of shaders to be powered off on next + * timer callback + * @tiler_poweroff_pending: Bit mask of tilers to be powered off on next timer + * callback + * @poweroff_timer_needed: true if the poweroff timer is currently required, + * false otherwise + * @poweroff_timer_running: true if the poweroff timer is currently running, + * false otherwise + * power_change_lock should be held when accessing, + * unless there is no way the timer can be running (eg + * hrtimer_cancel() was called immediately before) + * @poweroff_wait_in_progress: true if a wait for GPU power off is in progress. + * hwaccess_lock must be held when accessing + * @poweron_required: true if a GPU power on is required. Should only be set + * when poweroff_wait_in_progress is true, and therefore the + * GPU can not immediately be powered on. pm.lock must be + * held when accessing + * @poweroff_is_suspend: true if the GPU is being powered off due to a suspend + * request. pm.lock must be held when accessing + * @gpu_poweroff_wait_wq: workqueue for waiting for GPU to power off + * @gpu_poweroff_wait_work: work item for use with @gpu_poweroff_wait_wq + * @poweroff_wait: waitqueue for waiting for @gpu_poweroff_wait_work to complete + * @callback_power_on: Callback when the GPU needs to be turned on. See + * &struct kbase_pm_callback_conf + * @callback_power_off: Callback when the GPU may be turned off. See + * &struct kbase_pm_callback_conf + * @callback_power_suspend: Callback when a suspend occurs and the GPU needs to + * be turned off. See &struct kbase_pm_callback_conf + * @callback_power_resume: Callback when a resume occurs and the GPU needs to + * be turned on. See &struct kbase_pm_callback_conf + * @callback_power_runtime_on: Callback when the GPU needs to be turned on. See + * &struct kbase_pm_callback_conf + * @callback_power_runtime_off: Callback when the GPU may be turned off. See + * &struct kbase_pm_callback_conf + * @callback_power_runtime_idle: Optional callback when the GPU may be idle. See + * &struct kbase_pm_callback_conf + * + * Note: + * During an IRQ, @ca_current_policy or @pm_current_policy can be NULL when the + * policy is being changed with kbase_pm_ca_set_policy() or + * kbase_pm_set_policy(). The change is protected under + * kbase_device.pm.power_change_lock. Direct access to this + * from IRQ context must therefore check for NULL. If NULL, then + * kbase_pm_ca_set_policy() or kbase_pm_set_policy() will re-issue the policy + * functions that would have been done under IRQ. + */ +struct kbase_pm_backend_data { + const struct kbase_pm_ca_policy *ca_current_policy; + const struct kbase_pm_policy *pm_current_policy; + union kbase_pm_ca_policy_data ca_policy_data; + union kbase_pm_policy_data pm_policy_data; + bool ca_in_transition; + bool reset_done; + wait_queue_head_t reset_done_wait; + wait_queue_head_t l2_powered_wait; + int l2_powered; + int gpu_cycle_counter_requests; + spinlock_t gpu_cycle_counter_requests_lock; + + u64 desired_shader_state; + u64 powering_on_shader_state; + u64 desired_tiler_state; + u64 powering_on_tiler_state; + u64 powering_on_l2_state; +#ifdef CONFIG_MALI_CORESTACK + u64 powering_on_stack_state; +#endif /* CONFIG_MALI_CORESTACK */ + + bool gpu_in_desired_state; + wait_queue_head_t gpu_in_desired_state_wait; + + bool gpu_powered; + + bool instr_enabled; + + bool cg1_disabled; + +#ifdef CONFIG_MALI_DEBUG + bool driver_ready_for_irqs; +#endif /* CONFIG_MALI_DEBUG */ + + spinlock_t gpu_powered_lock; + + + struct kbasep_pm_metrics_data metrics; + + int gpu_poweroff_pending; + int shader_poweroff_pending_time; + + struct hrtimer gpu_poweroff_timer; + struct workqueue_struct *gpu_poweroff_wq; + struct work_struct gpu_poweroff_work; + + u64 shader_poweroff_pending; + u64 tiler_poweroff_pending; + + bool poweroff_timer_needed; + bool poweroff_timer_running; + + bool poweroff_wait_in_progress; + bool poweron_required; + bool poweroff_is_suspend; + + struct workqueue_struct *gpu_poweroff_wait_wq; + struct work_struct gpu_poweroff_wait_work; + + wait_queue_head_t poweroff_wait; + + int (*callback_power_on)(struct kbase_device *kbdev); + void (*callback_power_off)(struct kbase_device *kbdev); + void (*callback_power_suspend)(struct kbase_device *kbdev); + void (*callback_power_resume)(struct kbase_device *kbdev); + int (*callback_power_runtime_on)(struct kbase_device *kbdev); + void (*callback_power_runtime_off)(struct kbase_device *kbdev); + int (*callback_power_runtime_idle)(struct kbase_device *kbdev); +}; + + +/* List of policy IDs */ +enum kbase_pm_policy_id { + KBASE_PM_POLICY_ID_DEMAND = 1, + KBASE_PM_POLICY_ID_ALWAYS_ON, + KBASE_PM_POLICY_ID_COARSE_DEMAND, +#if !MALI_CUSTOMER_RELEASE + KBASE_PM_POLICY_ID_DEMAND_ALWAYS_POWERED, + KBASE_PM_POLICY_ID_FAST_START +#endif +}; + +typedef u32 kbase_pm_policy_flags; + +/** + * struct kbase_pm_policy - Power policy structure. + * + * Each power policy exposes a (static) instance of this structure which + * contains function pointers to the policy's methods. + * + * @name: The name of this policy + * @init: Function called when the policy is selected + * @term: Function called when the policy is unselected + * @get_core_mask: Function called to get the current shader core mask + * @get_core_active: Function called to get the current overall GPU power + * state + * @flags: Field indicating flags for this policy + * @id: Field indicating an ID for this policy. This is not + * necessarily the same as its index in the list returned + * by kbase_pm_list_policies(). + * It is used purely for debugging. + */ +struct kbase_pm_policy { + char *name; + + /** + * Function called when the policy is selected + * + * This should initialize the kbdev->pm.pm_policy_data structure. It + * should not attempt to make any changes to hardware state. + * + * It is undefined what state the cores are in when the function is + * called. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + */ + void (*init)(struct kbase_device *kbdev); + + /** + * Function called when the policy is unselected. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + */ + void (*term)(struct kbase_device *kbdev); + + /** + * Function called to get the current shader core mask + * + * The returned mask should meet or exceed (kbdev->shader_needed_bitmap + * | kbdev->shader_inuse_bitmap). + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * + * Return: The mask of shader cores to be powered + */ + u64 (*get_core_mask)(struct kbase_device *kbdev); + + /** + * Function called to get the current overall GPU power state + * + * This function should consider the state of kbdev->pm.active_count. If + * this count is greater than 0 then there is at least one active + * context on the device and the GPU should be powered. If it is equal + * to 0 then there are no active contexts and the GPU could be powered + * off if desired. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * + * Return: true if the GPU should be powered, false otherwise + */ + bool (*get_core_active)(struct kbase_device *kbdev); + + kbase_pm_policy_flags flags; + enum kbase_pm_policy_id id; +}; + + +enum kbase_pm_ca_policy_id { + KBASE_PM_CA_POLICY_ID_FIXED = 1, + KBASE_PM_CA_POLICY_ID_DEVFREQ, + KBASE_PM_CA_POLICY_ID_RANDOM +}; + +typedef u32 kbase_pm_ca_policy_flags; + +/** + * Maximum length of a CA policy names + */ +#define KBASE_PM_CA_MAX_POLICY_NAME_LEN 15 + +/** + * struct kbase_pm_ca_policy - Core availability policy structure. + * + * Each core availability policy exposes a (static) instance of this structure + * which contains function pointers to the policy's methods. + * + * @name: The name of this policy + * @init: Function called when the policy is selected + * @term: Function called when the policy is unselected + * @get_core_mask: Function called to get the current shader core + * availability mask + * @update_core_status: Function called to update the current core status + * @flags: Field indicating flags for this policy + * @id: Field indicating an ID for this policy. This is not + * necessarily the same as its index in the list returned + * by kbase_pm_list_policies(). + * It is used purely for debugging. + */ +struct kbase_pm_ca_policy { + char name[KBASE_PM_CA_MAX_POLICY_NAME_LEN + 1]; + + /** + * Function called when the policy is selected + * + * This should initialize the kbdev->pm.ca_policy_data structure. It + * should not attempt to make any changes to hardware state. + * + * It is undefined what state the cores are in when the function is + * called. + * + * @kbdev The kbase device structure for the device (must be a + * valid pointer) + */ + void (*init)(struct kbase_device *kbdev); + + /** + * Function called when the policy is unselected. + * + * @kbdev The kbase device structure for the device (must be a + * valid pointer) + */ + void (*term)(struct kbase_device *kbdev); + + /** + * Function called to get the current shader core availability mask + * + * When a change in core availability is occurring, the policy must set + * kbdev->pm.ca_in_transition to true. This is to indicate that + * reporting changes in power state cannot be optimized out, even if + * kbdev->pm.desired_shader_state remains unchanged. This must be done + * by any functions internal to the Core Availability Policy that change + * the return value of kbase_pm_ca_policy::get_core_mask. + * + * @kbdev The kbase device structure for the device (must be a + * valid pointer) + * + * Return: The current core availability mask + */ + u64 (*get_core_mask)(struct kbase_device *kbdev); + + /** + * Function called to update the current core status + * + * If none of the cores in core group 0 are ready or transitioning, then + * the policy must ensure that the next call to get_core_mask does not + * return 0 for all cores in core group 0. It is an error to disable + * core group 0 through the core availability policy. + * + * When a change in core availability has finished, the policy must set + * kbdev->pm.ca_in_transition to false. This is to indicate that + * changes in power state can once again be optimized out when + * kbdev->pm.desired_shader_state is unchanged. + * + * @kbdev: The kbase device structure for the device + * (must be a valid pointer) + * @cores_ready: The mask of cores currently powered and + * ready to run jobs + * @cores_transitioning: The mask of cores currently transitioning + * power state + */ + void (*update_core_status)(struct kbase_device *kbdev, u64 cores_ready, + u64 cores_transitioning); + + kbase_pm_ca_policy_flags flags; + + /** + * Field indicating an ID for this policy. This is not necessarily the + * same as its index in the list returned by kbase_pm_list_policies(). + * It is used purely for debugging. + */ + enum kbase_pm_ca_policy_id id; +}; + +#endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.c new file mode 100755 index 000000000000..81322fd0dd17 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.c @@ -0,0 +1,73 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * A simple demand based power management policy + */ + +#include +#include + +static u64 demand_get_core_mask(struct kbase_device *kbdev) +{ + u64 desired = kbdev->shader_needed_bitmap | kbdev->shader_inuse_bitmap; + + if (0 == kbdev->pm.active_count) + return 0; + + return desired; +} + +static bool demand_get_core_active(struct kbase_device *kbdev) +{ + if (0 == kbdev->pm.active_count && !(kbdev->shader_needed_bitmap | + kbdev->shader_inuse_bitmap) && !kbdev->tiler_needed_cnt + && !kbdev->tiler_inuse_cnt) + return false; + + return true; +} + +static void demand_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static void demand_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +/* + * The struct kbase_pm_policy structure for the demand power policy. + * + * This is the static structure that defines the demand power policy's callback + * and name. + */ +const struct kbase_pm_policy kbase_pm_demand_policy_ops = { + "demand", /* name */ + demand_init, /* init */ + demand_term, /* term */ + demand_get_core_mask, /* get_core_mask */ + demand_get_core_active, /* get_core_active */ + 0u, /* flags */ + KBASE_PM_POLICY_ID_DEMAND, /* id */ +}; + +KBASE_EXPORT_TEST_API(kbase_pm_demand_policy_ops); diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.h new file mode 100755 index 000000000000..c0c84b6e9189 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.h @@ -0,0 +1,64 @@ +/* + * + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * A simple demand based power management policy + */ + +#ifndef MALI_KBASE_PM_DEMAND_H +#define MALI_KBASE_PM_DEMAND_H + +/** + * DOC: Demand power management policy + * + * The demand power management policy has the following characteristics: + * - When KBase indicates that the GPU will be powered up, but we don't yet + * know which Job Chains are to be run: + * - The Shader Cores are not powered up + * + * - When KBase indicates that a set of Shader Cores are needed to submit the + * currently queued Job Chains: + * - Only those Shader Cores are powered up + * + * - When KBase indicates that the GPU need not be powered: + * - The Shader Cores are powered off, and the GPU itself is powered off too. + * + * Note: + * - KBase indicates the GPU will be powered up when it has a User Process that + * has just started to submit Job Chains. + * + * - KBase indicates the GPU need not be powered when all the Job Chains from + * User Processes have finished, and it is waiting for a User Process to + * submit some more Job Chains. + */ + +/** + * struct kbasep_pm_policy_demand - Private structure for policy instance data + * + * @dummy: No state is needed, a dummy variable + * + * This contains data that is private to the demand power policy. + */ +struct kbasep_pm_policy_demand { + int dummy; +}; + +extern const struct kbase_pm_policy kbase_pm_demand_policy_ops; + +#endif /* MALI_KBASE_PM_DEMAND_H */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c new file mode 100755 index 000000000000..82727937c545 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c @@ -0,0 +1,1713 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel Power Management hardware control + */ + +// #define ENABLE_DEBUG_LOG +#include "../../platform/rk/custom_log.h" + +#include +#include +#include +#if defined(CONFIG_MALI_GATOR_SUPPORT) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if MALI_MOCK_TEST +#define MOCKABLE(function) function##_original +#else +#define MOCKABLE(function) function +#endif /* MALI_MOCK_TEST */ + +/** + * enum kbasep_pm_action - Actions that can be performed on a core. + * + * This enumeration is private to the file. Its values are set to allow + * core_type_to_reg() function, which decodes this enumeration, to be simpler + * and more efficient. + * + * @ACTION_PRESENT: The cores that are present + * @ACTION_READY: The cores that are ready + * @ACTION_PWRON: Power on the cores specified + * @ACTION_PWROFF: Power off the cores specified + * @ACTION_PWRTRANS: The cores that are transitioning + * @ACTION_PWRACTIVE: The cores that are active + */ +enum kbasep_pm_action { + ACTION_PRESENT = 0, + ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO), + ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO), + ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO), + ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO), + ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO) +}; + +/*---------------------------------------------------------------------------*/ + +static bool is_action_of_powering_off_l2(enum kbase_pm_core_type core_type, + enum kbasep_pm_action active) +{ + return (KBASE_PM_CORE_L2 == core_type) && (ACTION_PWROFF == active); +} + +static bool is_action_of_powering_off_shader(enum kbase_pm_core_type core_type, + enum kbasep_pm_action active) +{ + return (KBASE_PM_CORE_SHADER == core_type) && (ACTION_PWROFF == active); +} + +static bool is_action_of_powering_off_tiler(enum kbase_pm_core_type core_type, + enum kbasep_pm_action active) +{ + return (KBASE_PM_CORE_TILER == core_type) && (ACTION_PWROFF == active); +} + +static u64 kbase_pm_get_state( + struct kbase_device *kbdev, + enum kbase_pm_core_type core_type, + enum kbasep_pm_action action); + +/** + * core_type_to_reg - Decode a core type and action to a register. + * + * Given a core type (defined by kbase_pm_core_type) and an action (defined + * by kbasep_pm_action) this function will return the register offset that + * will perform the action on the core type. The register returned is the _LO + * register and an offset must be applied to use the _HI register. + * + * @core_type: The type of core + * @action: The type of action + * + * Return: The register offset of the _LO register that performs an action of + * type @action on a core of type @core_type. + */ +static u32 core_type_to_reg(enum kbase_pm_core_type core_type, + enum kbasep_pm_action action) +{ +#ifdef CONFIG_MALI_CORESTACK + if (core_type == KBASE_PM_CORE_STACK) { + switch (action) { + case ACTION_PRESENT: + return STACK_PRESENT_LO; + case ACTION_READY: + return STACK_READY_LO; + case ACTION_PWRON: + return STACK_PWRON_LO; + case ACTION_PWROFF: + return STACK_PWROFF_LO; + case ACTION_PWRTRANS: + return STACK_PWRTRANS_LO; + default: + BUG(); + } + } +#endif /* CONFIG_MALI_CORESTACK */ + + return (u32)core_type + (u32)action; +} + +#ifdef CONFIG_ARM64 +static void mali_cci_flush_l2(struct kbase_device *kbdev) +{ + const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; + u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS; + u32 raw; + + /* + * Note that we don't take the cache flush mutex here since + * we expect to be the last user of the L2, all other L2 users + * would have dropped their references, to initiate L2 power + * down, L2 power down being the only valid place for this + * to be called from. + */ + + kbase_reg_write(kbdev, + GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CLEAN_INV_CACHES, + NULL); + + raw = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), + NULL); + + /* Wait for cache flush to complete before continuing, exit on + * gpu resets or loop expiry. */ + while (((raw & mask) == 0) && --loops) { + raw = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), + NULL); + } +} +#endif + +/** + * kbase_pm_invoke - Invokes an action on a core set + * + * This function performs the action given by @action on a set of cores of a + * type given by @core_type. It is a static function used by + * kbase_pm_transition_core_type() + * + * @kbdev: The kbase device structure of the device + * @core_type: The type of core that the action should be performed on + * @cores: A bit mask of cores to perform the action on (low 32 bits) + * @action: The action to perform on the cores + */ +static void kbase_pm_invoke(struct kbase_device *kbdev, + enum kbase_pm_core_type core_type, + u64 cores, + enum kbasep_pm_action action) +{ + u32 reg; + u32 lo = cores & 0xFFFFFFFF; + u32 hi = (cores >> 32) & 0xFFFFFFFF; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /*-------------------------------------------------------*/ + + if ( is_action_of_powering_off_l2(core_type, action) ) { + D("not to power off l2 actually."); + return; + } + if ( is_action_of_powering_off_shader(core_type, action) ) { + D("not to power off shader actually. cores_lo : 0x%x, hi : 0x%x.", + lo, + hi); + return; + } + if ( is_action_of_powering_off_tiler(core_type, action) ) { + D("not to power off tiler actually."); + return; + } + + /*-------------------------------------------------------*/ + + reg = core_type_to_reg(core_type, action); + + KBASE_DEBUG_ASSERT(reg); +#if defined(CONFIG_MALI_GATOR_SUPPORT) + if (cores) { + if (action == ACTION_PWRON) + kbase_trace_mali_pm_power_on(core_type, cores); + else if (action == ACTION_PWROFF) + kbase_trace_mali_pm_power_off(core_type, cores); + } +#endif + + if (cores) { + u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY); + + if (action == ACTION_PWRON) + state |= cores; + else if (action == ACTION_PWROFF) + state &= ~cores; + KBASE_TLSTREAM_AUX_PM_STATE(core_type, state); + } + + /* Tracing */ + if (cores) { + if (action == ACTION_PWRON) + switch (core_type) { + case KBASE_PM_CORE_SHADER: + KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u, + lo); + break; + case KBASE_PM_CORE_TILER: + KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL, + NULL, 0u, lo); + break; + case KBASE_PM_CORE_L2: + KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL, + 0u, lo); + break; + default: + break; + } + else if (action == ACTION_PWROFF) + switch (core_type) { + case KBASE_PM_CORE_SHADER: + KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL, + 0u, lo); + break; + case KBASE_PM_CORE_TILER: + KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, + NULL, 0u, lo); + break; + case KBASE_PM_CORE_L2: + KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL, + 0u, lo); + /* disable snoops before L2 is turned off */ + kbase_pm_cache_snoop_disable(kbdev); + break; + default: + break; + } + } + + if (lo != 0) + kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL); + + if (hi != 0) + kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL); +} + +/** + * kbase_pm_get_state - Get information about a core set + * + * This function gets information (chosen by @action) about a set of cores of + * a type given by @core_type. It is a static function used by + * kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and + * kbase_pm_get_ready_cores(). + * + * @kbdev: The kbase device structure of the device + * @core_type: The type of core that the should be queried + * @action: The property of the cores to query + * + * Return: A bit mask specifying the state of the cores + */ +static u64 kbase_pm_get_state(struct kbase_device *kbdev, + enum kbase_pm_core_type core_type, + enum kbasep_pm_action action) +{ + u32 reg; + u32 lo, hi; + + reg = core_type_to_reg(core_type, action); + + KBASE_DEBUG_ASSERT(reg); + + lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL); + hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL); + + return (((u64) hi) << 32) | ((u64) lo); +} + +void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev) +{ + kbdev->shader_inuse_bitmap = 0; + kbdev->shader_needed_bitmap = 0; + kbdev->shader_available_bitmap = 0; + kbdev->tiler_available_bitmap = 0; + kbdev->l2_users_count = 0; + kbdev->l2_available_bitmap = 0; + kbdev->tiler_needed_cnt = 0; + kbdev->tiler_inuse_cnt = 0; + + memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt)); +} + +/** + * kbase_pm_get_present_cores - Get the cores that are present + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of the cores that are present + */ +u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + switch (type) { + case KBASE_PM_CORE_L2: + return kbdev->gpu_props.props.raw_props.l2_present; + case KBASE_PM_CORE_SHADER: + return kbdev->gpu_props.props.raw_props.shader_present; + case KBASE_PM_CORE_TILER: + return kbdev->gpu_props.props.raw_props.tiler_present; +#ifdef CONFIG_MALI_CORESTACK + case KBASE_PM_CORE_STACK: + return kbdev->gpu_props.props.raw_props.stack_present; +#endif /* CONFIG_MALI_CORESTACK */ + default: + break; + } + KBASE_DEBUG_ASSERT(0); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores); + +/** + * kbase_pm_get_active_cores - Get the cores that are "active" + * (busy processing work) + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of cores that are active + */ +u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE); +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores); + +/** + * kbase_pm_get_trans_cores - Get the cores that are transitioning between + * power states + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of cores that are transitioning + */ +u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS); +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores); + +/** + * kbase_pm_get_ready_cores - Get the cores that are powered on + * + * @kbdev: Kbase device + * @type: The type of cores to query + * + * Return: Bitmask of cores that are ready (powered on) + */ +u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type) +{ + u64 result; + + result = kbase_pm_get_state(kbdev, type, ACTION_READY); + + switch (type) { + case KBASE_PM_CORE_SHADER: + KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u, + (u32) result); + break; + case KBASE_PM_CORE_TILER: + KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u, + (u32) result); + break; + case KBASE_PM_CORE_L2: + KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u, + (u32) result); + break; + default: + break; + } + + return result; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores); + +/** + * kbase_pm_transition_core_type - Perform power transitions for a particular + * core type. + * + * This function will perform any available power transitions to make the actual + * hardware state closer to the desired state. If a core is currently + * transitioning then changes to the power state of that call cannot be made + * until the transition has finished. Cores which are not present in the + * hardware are ignored if they are specified in the desired_state bitmask, + * however the return value will always be 0 in this case. + * + * @kbdev: The kbase device + * @type: The core type to perform transitions for + * @desired_state: A bit mask of the desired state of the cores + * @in_use: A bit mask of the cores that are currently running + * jobs. These cores have to be kept powered up because + * there are jobs running (or about to run) on them. + * @available: Receives a bit mask of the cores that the job + * scheduler can use to submit jobs to. May be NULL if + * this is not needed. + * @powering_on: Bit mask to update with cores that are + * transitioning to a power-on state. + * + * Return: true if the desired state has been reached, false otherwise + */ +static bool kbase_pm_transition_core_type(struct kbase_device *kbdev, + enum kbase_pm_core_type type, + u64 desired_state, + u64 in_use, + u64 * const available, + u64 *powering_on) +{ + u64 present; + u64 ready; + u64 trans; + u64 powerup; + u64 powerdown; + u64 powering_on_trans; + u64 desired_state_in_use; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* Get current state */ + present = kbase_pm_get_present_cores(kbdev, type); + trans = kbase_pm_get_trans_cores(kbdev, type); + ready = kbase_pm_get_ready_cores(kbdev, type); + /* mask off ready from trans in case transitions finished between the + * register reads */ + trans &= ~ready; + + if (trans) /* Do not progress if any cores are transitioning */ + return false; + + powering_on_trans = trans & *powering_on; + *powering_on = powering_on_trans; + + if (available != NULL) + *available = (ready | powering_on_trans) & desired_state; + + /* Update desired state to include the in-use cores. These have to be + * kept powered up because there are jobs running or about to run on + * these cores + */ + desired_state_in_use = desired_state | in_use; + + /* Update state of whether l2 caches are powered */ + if (type == KBASE_PM_CORE_L2) { + if ((ready == present) && (desired_state_in_use == ready) && + (trans == 0)) { + /* All are ready, none will be turned off, and none are + * transitioning */ + kbdev->pm.backend.l2_powered = 1; + /* + * Ensure snoops are enabled after L2 is powered up, + * note that kbase keeps track of the snoop state, so + * safe to repeatedly call. + */ + kbase_pm_cache_snoop_enable(kbdev); + if (kbdev->l2_users_count > 0) { + /* Notify any registered l2 cache users + * (optimized out when no users waiting) */ + wake_up(&kbdev->pm.backend.l2_powered_wait); + } + } else + kbdev->pm.backend.l2_powered = 0; + } + + if (desired_state == ready && (trans == 0)) + return true; + + /* Restrict the cores to those that are actually present */ + powerup = desired_state_in_use & present; + powerdown = (~desired_state_in_use) & present; + + /* Restrict to cores that are not already in the desired state */ + powerup &= ~ready; + powerdown &= ready; + + /* Don't transition any cores that are already transitioning, except for + * Mali cores that support the following case: + * + * If the SHADER_PWRON or TILER_PWRON registers are written to turn on + * a core that is currently transitioning to power off, then this is + * remembered and the shader core is automatically powered up again once + * the original transition completes. Once the automatic power on is + * complete any job scheduled on the shader core should start. + */ + powerdown &= ~trans; + + if (kbase_hw_has_feature(kbdev, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS)) + if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type) + trans = powering_on_trans; /* for exception cases, only + * mask off cores in power on + * transitions */ + + powerup &= ~trans; + + /* Perform transitions if any */ + kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON); +#if !PLATFORM_POWER_DOWN_ONLY + kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF); +#endif + + /* Recalculate cores transitioning on, and re-evaluate our state */ + powering_on_trans |= powerup; + *powering_on = powering_on_trans; + if (available != NULL) + *available = (ready | powering_on_trans) & desired_state; + + return false; +} + +KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type); + +/** + * get_desired_cache_status - Determine which caches should be on for a + * particular core state + * + * This function takes a bit mask of the present caches and the cores (or + * caches) that are attached to the caches that will be powered. It then + * computes which caches should be turned on to allow the cores requested to be + * powered up. + * + * @present: The bit mask of present caches + * @cores_powered: A bit mask of cores (or L2 caches) that are desired to + * be powered + * @tilers_powered: The bit mask of tilers that are desired to be powered + * + * Return: A bit mask of the caches that should be turned on + */ +static u64 get_desired_cache_status(u64 present, u64 cores_powered, + u64 tilers_powered) +{ + u64 desired = 0; + + while (present) { + /* Find out which is the highest set bit */ + u64 bit = fls64(present) - 1; + u64 bit_mask = 1ull << bit; + /* Create a mask which has all bits from 'bit' upwards set */ + + u64 mask = ~(bit_mask - 1); + + /* If there are any cores powered at this bit or above (that + * haven't previously been processed) then we need this core on + */ + if (cores_powered & mask) + desired |= bit_mask; + + /* Remove bits from cores_powered and present */ + cores_powered &= ~mask; + present &= ~bit_mask; + } + + /* Power up the required L2(s) for the tiler */ + if (tilers_powered) + desired |= 1; + + return desired; +} + +KBASE_EXPORT_TEST_API(get_desired_cache_status); + +#ifdef CONFIG_MALI_CORESTACK +u64 kbase_pm_core_stack_mask(u64 cores) +{ + u64 stack_mask = 0; + size_t const MAX_CORE_ID = 31; + size_t const NUM_CORES_PER_STACK = 4; + size_t i; + + for (i = 0; i <= MAX_CORE_ID; ++i) { + if (test_bit(i, (unsigned long *)&cores)) { + /* Every core which ID >= 16 is filled to stacks 4-7 + * instead of 0-3 */ + size_t const stack_num = (i > 16) ? + (i % NUM_CORES_PER_STACK) + 4 : + (i % NUM_CORES_PER_STACK); + set_bit(stack_num, (unsigned long *)&stack_mask); + } + } + + return stack_mask; +} +#endif /* CONFIG_MALI_CORESTACK */ + +bool +MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev) +{ + bool cores_are_available = false; + bool in_desired_state = true; + u64 desired_l2_state; +#ifdef CONFIG_MALI_CORESTACK + u64 desired_stack_state; + u64 stacks_powered; +#endif /* CONFIG_MALI_CORESTACK */ + u64 cores_powered; + u64 tilers_powered; + u64 tiler_available_bitmap; + u64 tiler_transitioning_bitmap; + u64 shader_available_bitmap; + u64 shader_ready_bitmap; + u64 shader_transitioning_bitmap; + u64 l2_available_bitmap; + u64 prev_l2_available_bitmap; + u64 l2_inuse_bitmap; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->hwaccess_lock); + + spin_lock(&kbdev->pm.backend.gpu_powered_lock); + if (kbdev->pm.backend.gpu_powered == false) { + spin_unlock(&kbdev->pm.backend.gpu_powered_lock); + if (kbdev->pm.backend.desired_shader_state == 0 && + kbdev->pm.backend.desired_tiler_state == 0) + return true; + return false; + } + + /* Trace that a change-state is being requested, and that it took + * (effectively) no time to start it. This is useful for counting how + * many state changes occurred, in a way that's backwards-compatible + * with processing the trace data */ + kbase_timeline_pm_send_event(kbdev, + KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); + kbase_timeline_pm_handle_event(kbdev, + KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); + + /* If any cores are already powered then, we must keep the caches on */ + shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_SHADER); + cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); + cores_powered |= kbdev->pm.backend.desired_shader_state; + +#ifdef CONFIG_MALI_CORESTACK + /* Work out which core stacks want to be powered */ + desired_stack_state = kbase_pm_core_stack_mask(cores_powered); + stacks_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK) | + desired_stack_state; +#endif /* CONFIG_MALI_CORESTACK */ + + /* Work out which tilers want to be powered */ + tiler_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_TILER); + tilers_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER); + tilers_powered |= kbdev->pm.backend.desired_tiler_state; + + /* If there are l2 cache users registered, keep all l2s powered even if + * all other cores are off. */ + if (kbdev->l2_users_count > 0) + cores_powered |= kbdev->gpu_props.props.raw_props.l2_present; + + desired_l2_state = get_desired_cache_status( + kbdev->gpu_props.props.raw_props.l2_present, + cores_powered, tilers_powered); + + l2_inuse_bitmap = get_desired_cache_status( + kbdev->gpu_props.props.raw_props.l2_present, + cores_powered | shader_transitioning_bitmap, + tilers_powered | tiler_transitioning_bitmap); + +#ifdef CONFIG_MALI_CORESTACK + if (stacks_powered) + desired_l2_state |= 1; +#endif /* CONFIG_MALI_CORESTACK */ + + /* If any l2 cache is on, then enable l2 #0, for use by job manager */ + if (0 != desired_l2_state) + desired_l2_state |= 1; + + prev_l2_available_bitmap = kbdev->l2_available_bitmap; + in_desired_state &= kbase_pm_transition_core_type(kbdev, + KBASE_PM_CORE_L2, desired_l2_state, l2_inuse_bitmap, + &l2_available_bitmap, + &kbdev->pm.backend.powering_on_l2_state); + + if (kbdev->l2_available_bitmap != l2_available_bitmap) + KBASE_TIMELINE_POWER_L2(kbdev, l2_available_bitmap); + + kbdev->l2_available_bitmap = l2_available_bitmap; + + +#ifdef CONFIG_MALI_CORESTACK + if (in_desired_state) { + in_desired_state &= kbase_pm_transition_core_type(kbdev, + KBASE_PM_CORE_STACK, desired_stack_state, 0, + &kbdev->stack_available_bitmap, + &kbdev->pm.backend.powering_on_stack_state); + } +#endif /* CONFIG_MALI_CORESTACK */ + + if (in_desired_state) { + in_desired_state &= kbase_pm_transition_core_type(kbdev, + KBASE_PM_CORE_TILER, + kbdev->pm.backend.desired_tiler_state, + 0, &tiler_available_bitmap, + &kbdev->pm.backend.powering_on_tiler_state); + in_desired_state &= kbase_pm_transition_core_type(kbdev, + KBASE_PM_CORE_SHADER, + kbdev->pm.backend.desired_shader_state, + kbdev->shader_inuse_bitmap, + &shader_available_bitmap, + &kbdev->pm.backend.powering_on_shader_state); + + if (kbdev->shader_available_bitmap != shader_available_bitmap) { + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, + NULL, 0u, + (u32) shader_available_bitmap); + KBASE_TIMELINE_POWER_SHADER(kbdev, + shader_available_bitmap); + } + + kbdev->shader_available_bitmap = shader_available_bitmap; + + if (kbdev->tiler_available_bitmap != tiler_available_bitmap) { + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, + NULL, NULL, 0u, + (u32) tiler_available_bitmap); + KBASE_TIMELINE_POWER_TILER(kbdev, + tiler_available_bitmap); + } + + kbdev->tiler_available_bitmap = tiler_available_bitmap; + + } else if ((l2_available_bitmap & + kbdev->gpu_props.props.raw_props.tiler_present) != + kbdev->gpu_props.props.raw_props.tiler_present) { + tiler_available_bitmap = 0; + + if (kbdev->tiler_available_bitmap != tiler_available_bitmap) + KBASE_TIMELINE_POWER_TILER(kbdev, + tiler_available_bitmap); + + kbdev->tiler_available_bitmap = tiler_available_bitmap; + } + + /* State updated for slow-path waiters */ + kbdev->pm.backend.gpu_in_desired_state = in_desired_state; + + shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_SHADER); + shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, + KBASE_PM_CORE_SHADER); + + /* Determine whether the cores are now available (even if the set of + * available cores is empty). Note that they can be available even if + * we've not finished transitioning to the desired state */ + if ((kbdev->shader_available_bitmap & + kbdev->pm.backend.desired_shader_state) + == kbdev->pm.backend.desired_shader_state && + (kbdev->tiler_available_bitmap & + kbdev->pm.backend.desired_tiler_state) + == kbdev->pm.backend.desired_tiler_state) { + cores_are_available = true; + + KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u, + (u32)(kbdev->shader_available_bitmap & + kbdev->pm.backend.desired_shader_state)); + KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u, + (u32)(kbdev->tiler_available_bitmap & + kbdev->pm.backend.desired_tiler_state)); + + /* Log timelining information about handling events that power + * up cores, to match up either with immediate submission either + * because cores already available, or from PM IRQ */ + if (!in_desired_state) + kbase_timeline_pm_send_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + } + + if (in_desired_state) { + KBASE_DEBUG_ASSERT(cores_are_available); + +#if defined(CONFIG_MALI_GATOR_SUPPORT) + kbase_trace_mali_pm_status(KBASE_PM_CORE_L2, + kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_L2)); + kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER, + kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_SHADER)); + kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER, + kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_TILER)); +#ifdef CONFIG_MALI_CORESTACK + kbase_trace_mali_pm_status(KBASE_PM_CORE_STACK, + kbase_pm_get_ready_cores(kbdev, + KBASE_PM_CORE_STACK)); +#endif /* CONFIG_MALI_CORESTACK */ +#endif + + KBASE_TLSTREAM_AUX_PM_STATE( + KBASE_PM_CORE_L2, + kbase_pm_get_ready_cores( + kbdev, KBASE_PM_CORE_L2)); + KBASE_TLSTREAM_AUX_PM_STATE( + KBASE_PM_CORE_SHADER, + kbase_pm_get_ready_cores( + kbdev, KBASE_PM_CORE_SHADER)); + KBASE_TLSTREAM_AUX_PM_STATE( + KBASE_PM_CORE_TILER, + kbase_pm_get_ready_cores( + kbdev, + KBASE_PM_CORE_TILER)); +#ifdef CONFIG_MALI_CORESTACK + KBASE_TLSTREAM_AUX_PM_STATE( + KBASE_PM_CORE_STACK, + kbase_pm_get_ready_cores( + kbdev, + KBASE_PM_CORE_STACK)); +#endif /* CONFIG_MALI_CORESTACK */ + + KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL, + kbdev->pm.backend.gpu_in_desired_state, + (u32)kbdev->pm.backend.desired_shader_state); + KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u, + (u32)kbdev->pm.backend.desired_tiler_state); + + /* Log timelining information for synchronous waiters */ + kbase_timeline_pm_send_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + /* Wake slow-path waiters. Job scheduler does not use this. */ + KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0); + + wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); + } + + spin_unlock(&kbdev->pm.backend.gpu_powered_lock); + + /* kbase_pm_ca_update_core_status can cause one-level recursion into + * this function, so it must only be called once all changes to kbdev + * have been committed, and after the gpu_powered_lock has been + * dropped. */ + if (kbdev->shader_ready_bitmap != shader_ready_bitmap || + kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) { + kbdev->shader_ready_bitmap = shader_ready_bitmap; + kbdev->shader_transitioning_bitmap = + shader_transitioning_bitmap; + + kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap, + shader_transitioning_bitmap); + } + + /* The core availability policy is not allowed to keep core group 0 + * turned off (unless it was changing the l2 power state) */ + if (!((shader_ready_bitmap | shader_transitioning_bitmap) & + kbdev->gpu_props.props.coherency_info.group[0].core_mask) && + (prev_l2_available_bitmap == desired_l2_state) && + !(kbase_pm_ca_get_core_mask(kbdev) & + kbdev->gpu_props.props.coherency_info.group[0].core_mask)) + BUG(); + + /* The core availability policy is allowed to keep core group 1 off, + * but all jobs specifically targeting CG1 must fail */ + if (!((shader_ready_bitmap | shader_transitioning_bitmap) & + kbdev->gpu_props.props.coherency_info.group[1].core_mask) && + !(kbase_pm_ca_get_core_mask(kbdev) & + kbdev->gpu_props.props.coherency_info.group[1].core_mask)) + kbdev->pm.backend.cg1_disabled = true; + else + kbdev->pm.backend.cg1_disabled = false; + + return cores_are_available; +} +KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock); + +/* Timeout for kbase_pm_check_transitions_sync when wait_event_killable has + * aborted due to a fatal signal. If the time spent waiting has exceeded this + * threshold then there is most likely a hardware issue. */ +#define PM_TIMEOUT (5*HZ) /* 5s */ + +void kbase_pm_check_transitions_sync(struct kbase_device *kbdev) +{ + unsigned long flags; + unsigned long timeout; + bool cores_are_available; + int ret; + + /* Force the transition to be checked and reported - the cores may be + * 'available' (for job submission) but not fully powered up. */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + cores_are_available = kbase_pm_check_transitions_nolock(kbdev); + + /* Don't need 'cores_are_available', because we don't return anything */ + CSTD_UNUSED(cores_are_available); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + timeout = jiffies + PM_TIMEOUT; + + /* Wait for cores */ + ret = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait, + kbdev->pm.backend.gpu_in_desired_state); + + if (ret < 0 && time_after(jiffies, timeout)) { + dev_err(kbdev->dev, "Power transition timed out unexpectedly\n"); + dev_err(kbdev->dev, "Desired state :\n"); + dev_err(kbdev->dev, "\tShader=%016llx\n", + kbdev->pm.backend.desired_shader_state); + dev_err(kbdev->dev, "\tTiler =%016llx\n", + kbdev->pm.backend.desired_tiler_state); + dev_err(kbdev->dev, "Current state :\n"); + dev_err(kbdev->dev, "\tShader=%08x%08x\n", + kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_READY_HI), NULL), + kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_READY_LO), + NULL)); + dev_err(kbdev->dev, "\tTiler =%08x%08x\n", + kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_READY_HI), NULL), + kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_READY_LO), NULL)); + dev_err(kbdev->dev, "\tL2 =%08x%08x\n", + kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_READY_HI), NULL), + kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_READY_LO), NULL)); + dev_err(kbdev->dev, "Cores transitioning :\n"); + dev_err(kbdev->dev, "\tShader=%08x%08x\n", + kbase_reg_read(kbdev, GPU_CONTROL_REG( + SHADER_PWRTRANS_HI), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG( + SHADER_PWRTRANS_LO), NULL)); + dev_err(kbdev->dev, "\tTiler =%08x%08x\n", + kbase_reg_read(kbdev, GPU_CONTROL_REG( + TILER_PWRTRANS_HI), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG( + TILER_PWRTRANS_LO), NULL)); + dev_err(kbdev->dev, "\tL2 =%08x%08x\n", + kbase_reg_read(kbdev, GPU_CONTROL_REG( + L2_PWRTRANS_HI), NULL), + kbase_reg_read(kbdev, GPU_CONTROL_REG( + L2_PWRTRANS_LO), NULL)); +#if KBASE_GPU_RESET_EN + dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n"); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); +#endif /* KBASE_GPU_RESET_EN */ + } else { + /* Log timelining information that a change in state has + * completed */ + kbase_timeline_pm_handle_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); + } +} +KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync); + +void kbase_pm_enable_interrupts(struct kbase_device *kbdev) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + /* + * Clear all interrupts, + * and unmask them all. + */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, + NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL, + NULL); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, + NULL); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL); +} + +KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); + +void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(NULL != kbdev); + /* + * Mask all interrupts, + * and clear them all. + */ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, + NULL); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL); + kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, + NULL); + + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); + kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); +} + +void kbase_pm_disable_interrupts(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_disable_interrupts_nolock(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts); + + +/* + * pmu layout: + * 0x0000: PMU TAG (RO) (0xCAFECAFE) + * 0x0004: PMU VERSION ID (RO) (0x00000000) + * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE) + */ +void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) +{ + bool reset_required = is_resume; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->pm.lock); + + if (kbdev->pm.backend.gpu_powered) { + /* Already turned on */ + if (kbdev->poweroff_pending) + kbase_pm_enable_interrupts(kbdev); + kbdev->poweroff_pending = false; + KBASE_DEBUG_ASSERT(!is_resume); + return; + } + + kbdev->poweroff_pending = false; + + KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u); + + if (is_resume && kbdev->pm.backend.callback_power_resume) { + kbdev->pm.backend.callback_power_resume(kbdev); + return; + } else if (kbdev->pm.backend.callback_power_on) { + kbdev->pm.backend.callback_power_on(kbdev); + /* If your platform properly keeps the GPU state you may use the + * return value of the callback_power_on function to + * conditionally reset the GPU on power up. Currently we are + * conservative and always reset the GPU. */ + reset_required = true; + } + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + kbdev->pm.backend.gpu_powered = true; + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (reset_required) { + /* GPU state was lost, reset GPU to ensure it is in a + * consistent state */ + kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); + } + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_restore_all_as(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + /* Lastly, enable the interrupts */ + kbase_pm_enable_interrupts(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_clock_on); + +bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->pm.lock); + + /* ASSERT that the cores should now be unavailable. No lock needed. */ + KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u); + + kbdev->poweroff_pending = true; + + if (!kbdev->pm.backend.gpu_powered) { + /* Already turned off */ + if (is_suspend && kbdev->pm.backend.callback_power_suspend) + kbdev->pm.backend.callback_power_suspend(kbdev); + return true; + } + + KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u); + + /* Disable interrupts. This also clears any outstanding interrupts */ + kbase_pm_disable_interrupts(kbdev); + /* Ensure that any IRQ handlers have finished */ + kbase_synchronize_irqs(kbdev); + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (atomic_read(&kbdev->faults_pending)) { + /* Page/bus faults are still being processed. The GPU can not + * be powered off until they have completed */ + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + flags); + return false; + } + + kbase_pm_cache_snoop_disable(kbdev); + + /* The GPU power may be turned off from this point */ + kbdev->pm.backend.gpu_powered = false; + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); + + if (is_suspend && kbdev->pm.backend.callback_power_suspend) + kbdev->pm.backend.callback_power_suspend(kbdev); + else if (kbdev->pm.backend.callback_power_off) + kbdev->pm.backend.callback_power_off(kbdev); + return true; +} + +KBASE_EXPORT_TEST_API(kbase_pm_clock_off); + +struct kbasep_reset_timeout_data { + struct hrtimer timer; + bool timed_out; + struct kbase_device *kbdev; +}; + +void kbase_pm_reset_done(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + kbdev->pm.backend.reset_done = true; + wake_up(&kbdev->pm.backend.reset_done_wait); +} + +/** + * kbase_pm_wait_for_reset - Wait for a reset to happen + * + * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state. + * + * @kbdev: Kbase device + */ +static void kbase_pm_wait_for_reset(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->pm.lock); + + wait_event(kbdev->pm.backend.reset_done_wait, + (kbdev->pm.backend.reset_done)); + kbdev->pm.backend.reset_done = false; +} + +KBASE_EXPORT_TEST_API(kbase_pm_reset_done); + +static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) +{ + struct kbasep_reset_timeout_data *rtdata = + container_of(timer, struct kbasep_reset_timeout_data, timer); + + rtdata->timed_out = 1; + + /* Set the wait queue to wake up kbase_pm_init_hw even though the reset + * hasn't completed */ + kbase_pm_reset_done(rtdata->kbdev); + + return HRTIMER_NORESTART; +} + +static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev) +{ + struct device_node *np = kbdev->dev->of_node; + u32 jm_values[4]; + const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT; + const u32 major = (gpu_id & GPU_ID_VERSION_MAJOR) >> + GPU_ID_VERSION_MAJOR_SHIFT; + + kbdev->hw_quirks_sc = 0; + + /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443. + * and needed due to MIDGLES-3539. See PRLAM-11035 */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) || + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035)) + kbdev->hw_quirks_sc |= SC_LS_PAUSEBUFFER_DISABLE; + + /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327. + */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327)) + kbdev->hw_quirks_sc |= SC_SDC_DISABLE_OQ_DISCARD; + +#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY + /* Enable alternative hardware counter selection if configured. */ + if (!GPU_ID_IS_NEW_FORMAT(prod_id)) + kbdev->hw_quirks_sc |= SC_ALT_COUNTERS; +#endif + + /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797)) + kbdev->hw_quirks_sc |= SC_ENABLE_TEXGRD_FLAGS; + + if (!kbase_hw_has_issue(kbdev, GPUCORE_1619)) { + if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ + kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; + else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */ + kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; + } + + if (!kbdev->hw_quirks_sc) + kbdev->hw_quirks_sc = kbase_reg_read(kbdev, + GPU_CONTROL_REG(SHADER_CONFIG), NULL); + + kbdev->hw_quirks_tiler = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TILER_CONFIG), NULL); + + /* Set tiler clock gate override if required */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) + kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; + + /* Limit the GPU bus bandwidth if the platform needs this. */ + kbdev->hw_quirks_mmu = kbase_reg_read(kbdev, + GPU_CONTROL_REG(L2_MMU_CONFIG), NULL); + + /* Limit read ID width for AXI */ + kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS); + kbdev->hw_quirks_mmu |= (DEFAULT_ARID_LIMIT & 0x3) << + L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT; + + /* Limit write ID width for AXI */ + kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES); + kbdev->hw_quirks_mmu |= (DEFAULT_AWID_LIMIT & 0x3) << + L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT; + + if (kbdev->system_coherency == COHERENCY_ACE) { + /* Allow memory configuration disparity to be ignored, we + * optimize the use of shared memory and thus we expect + * some disparity in the memory configuration */ + kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY; + } + + kbdev->hw_quirks_jm = 0; + /* Only for T86x/T88x-based products after r2p0 */ + if (prod_id >= 0x860 && prod_id <= 0x880 && major >= 2) { + + if (of_property_read_u32_array(np, + "jm_config", + &jm_values[0], + ARRAY_SIZE(jm_values))) { + /* Entry not in device tree, use defaults */ + jm_values[0] = 0; + jm_values[1] = 0; + jm_values[2] = 0; + jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT; + } + + /* Limit throttle limit to 6 bits*/ + if (jm_values[3] > JM_MAX_JOB_THROTTLE_LIMIT) { + dev_dbg(kbdev->dev, "JOB_THROTTLE_LIMIT supplied in device tree is too large. Limiting to MAX (63)."); + jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT; + } + + /* Aggregate to one integer. */ + kbdev->hw_quirks_jm |= (jm_values[0] ? + JM_TIMESTAMP_OVERRIDE : 0); + kbdev->hw_quirks_jm |= (jm_values[1] ? + JM_CLOCK_GATE_OVERRIDE : 0); + kbdev->hw_quirks_jm |= (jm_values[2] ? + JM_JOB_THROTTLE_ENABLE : 0); + kbdev->hw_quirks_jm |= (jm_values[3] << + JM_JOB_THROTTLE_LIMIT_SHIFT); + + } else if (GPU_ID_IS_NEW_FORMAT(prod_id) && + (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == + GPU_ID2_PRODUCT_TMIX)) { + /* Only for tMIx */ + u32 coherency_features; + + coherency_features = kbase_reg_read(kbdev, + GPU_CONTROL_REG(COHERENCY_FEATURES), NULL); + + /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly + * documented for tMIx so force correct value here. + */ + if (coherency_features == + COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { + kbdev->hw_quirks_jm |= + (COHERENCY_ACE_LITE | COHERENCY_ACE) << + JM_FORCE_COHERENCY_FEATURES_SHIFT; + } + } + + if (!kbdev->hw_quirks_jm) + kbdev->hw_quirks_jm = kbase_reg_read(kbdev, + GPU_CONTROL_REG(JM_CONFIG), NULL); + +#ifdef CONFIG_MALI_CORESTACK +#define MANUAL_POWER_CONTROL ((u32)(1 << 8)) + kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL; +#endif /* CONFIG_MALI_CORESTACK */ +} + +static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) +{ + kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), + kbdev->hw_quirks_sc, NULL); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG), + kbdev->hw_quirks_tiler, NULL); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), + kbdev->hw_quirks_mmu, NULL); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG), + kbdev->hw_quirks_jm, NULL); + +} + +void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) +{ + if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && + !kbdev->cci_snoop_enabled) { +#ifdef CONFIG_ARM64 + if (kbdev->snoop_enable_smc != 0) + kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0); +#endif /* CONFIG_ARM64 */ + dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n"); + kbdev->cci_snoop_enabled = true; + } +} + +void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) +{ + if (kbdev->cci_snoop_enabled) { +#ifdef CONFIG_ARM64 + if (kbdev->snoop_disable_smc != 0) { + mali_cci_flush_l2(kbdev); + kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0); + } +#endif /* CONFIG_ARM64 */ + dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n"); + kbdev->cci_snoop_enabled = false; + } +} + +static int kbase_pm_do_reset(struct kbase_device *kbdev) +{ + struct kbasep_reset_timeout_data rtdata; + + KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0); + + KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_SOFT_RESET, NULL); + + /* Unmask the reset complete interrupt only */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED, + NULL); + + /* Initialize a structure for tracking the status of the reset */ + rtdata.kbdev = kbdev; + rtdata.timed_out = 0; + + /* Create a timer to use as a timeout on the reset */ + hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rtdata.timer.function = kbasep_reset_timeout; + + hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), + HRTIMER_MODE_REL); + + /* Wait for the RESET_COMPLETED interrupt to be raised */ + kbase_pm_wait_for_reset(kbdev); + + if (rtdata.timed_out == 0) { + /* GPU has been reset */ + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + return 0; + } + + /* No interrupt has been received - check if the RAWSTAT register says + * the reset has completed */ + if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & + RESET_COMPLETED) { + /* The interrupt is set in the RAWSTAT; this suggests that the + * interrupts are not getting to the CPU */ + dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); + /* If interrupts aren't working we can't continue. */ + destroy_hrtimer_on_stack(&rtdata.timer); + return -EINVAL; + } + + /* The GPU doesn't seem to be responding to the reset so try a hard + * reset */ + dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", + RESET_TIMEOUT); + KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_HARD_RESET, NULL); + + /* Restart the timer to wait for the hard reset to complete */ + rtdata.timed_out = 0; + + hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), + HRTIMER_MODE_REL); + + /* Wait for the RESET_COMPLETED interrupt to be raised */ + kbase_pm_wait_for_reset(kbdev); + + if (rtdata.timed_out == 0) { + /* GPU has been reset */ + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + return 0; + } + + destroy_hrtimer_on_stack(&rtdata.timer); + + dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", + RESET_TIMEOUT); + + return -EINVAL; +} + +static int kbasep_protected_mode_enable(struct protected_mode_device *pdev) +{ + struct kbase_device *kbdev = pdev->data; + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_SET_PROTECTED_MODE, NULL); + return 0; +} + +static int kbasep_protected_mode_disable(struct protected_mode_device *pdev) +{ + struct kbase_device *kbdev = pdev->data; + + lockdep_assert_held(&kbdev->pm.lock); + + return kbase_pm_do_reset(kbdev); +} + +struct protected_mode_ops kbase_native_protected_ops = { + .protected_mode_enable = kbasep_protected_mode_enable, + .protected_mode_disable = kbasep_protected_mode_disable +}; + +int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) +{ + unsigned long irq_flags; + int err; + bool resume_vinstr = false; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + lockdep_assert_held(&kbdev->pm.lock); + + /* Ensure the clock is on before attempting to access the hardware */ + if (!kbdev->pm.backend.gpu_powered) { + if (kbdev->pm.backend.callback_power_on) + kbdev->pm.backend.callback_power_on(kbdev); + + spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, + irq_flags); + kbdev->pm.backend.gpu_powered = true; + spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, + irq_flags); + } + + /* Ensure interrupts are off to begin with, this also clears any + * outstanding interrupts */ + kbase_pm_disable_interrupts(kbdev); + /* Ensure cache snoops are disabled before reset. */ + kbase_pm_cache_snoop_disable(kbdev); + /* Prepare for the soft-reset */ + kbdev->pm.backend.reset_done = false; + + /* The cores should be made unavailable due to the reset */ + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + if (kbdev->shader_available_bitmap != 0u) + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, + NULL, 0u, (u32)0u); + if (kbdev->tiler_available_bitmap != 0u) + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, + NULL, NULL, 0u, (u32)0u); + kbdev->shader_available_bitmap = 0u; + kbdev->tiler_available_bitmap = 0u; + kbdev->l2_available_bitmap = 0u; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); + + /* Soft reset the GPU */ + if (kbdev->protected_mode_support) + err = kbdev->protected_ops->protected_mode_disable( + kbdev->protected_dev); + else + err = kbase_pm_do_reset(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + if (kbdev->protected_mode) + resume_vinstr = true; + kbdev->protected_mode = false; + kbase_ipa_model_use_configured_locked(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); + + if (err) + goto exit; + + if (flags & PM_HW_ISSUES_DETECT) + kbase_pm_hw_issues_detect(kbdev); + + kbase_pm_hw_issues_apply(kbdev); + kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); + + /* Sanity check protected mode was left after reset */ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { + u32 gpu_status = kbase_reg_read(kbdev, + GPU_CONTROL_REG(GPU_STATUS), NULL); + + WARN_ON(gpu_status & GPU_STATUS_PROTECTED_MODE_ACTIVE); + } + + /* If cycle counter was in use re-enable it, enable_irqs will only be + * false when called from kbase_pm_powerup */ + if (kbdev->pm.backend.gpu_cycle_counter_requests && + (flags & PM_ENABLE_IRQS)) { + /* enable interrupts as the L2 may have to be powered on */ + kbase_pm_enable_interrupts(kbdev); + kbase_pm_request_l2_caches(kbdev); + + /* Re-enable the counters if we need to */ + spin_lock_irqsave( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + if (kbdev->pm.backend.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CYCLE_COUNT_START, NULL); + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + irq_flags); + + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); + kbase_pm_release_l2_caches(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); + + kbase_pm_disable_interrupts(kbdev); + } + + if (flags & PM_ENABLE_IRQS) + kbase_pm_enable_interrupts(kbdev); + +exit: + /* If GPU is leaving protected mode resume vinstr operation. */ + if (kbdev->vinstr_ctx && resume_vinstr) + kbase_vinstr_resume(kbdev->vinstr_ctx); + + return err; +} + +/** + * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters + * + * Increase the count of cycle counter users and turn the cycle counters on if + * they were previously off + * + * This function is designed to be called by + * kbase_pm_request_gpu_cycle_counter() or + * kbase_pm_request_gpu_cycle_counter_l2_is_on() only + * + * When this function is called the l2 cache must be on and the l2 cache users + * count must have been incremented by a call to ( + * kbase_pm_request_l2_caches() or kbase_pm_request_l2_caches_l2_on() ) + * + * @kbdev: The kbase device structure of the device + */ +static void +kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); + + ++kbdev->pm.backend.gpu_cycle_counter_requests; + + if (1 == kbdev->pm.backend.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CYCLE_COUNT_START, NULL); + + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); +} + +void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < + INT_MAX); + + kbase_pm_request_l2_caches(kbdev); + + kbase_pm_request_gpu_cycle_counter_do_request(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter); + +void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < + INT_MAX); + + kbase_pm_request_l2_caches_l2_is_on(kbdev); + + kbase_pm_request_gpu_cycle_counter_do_request(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on); + +void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0); + + --kbdev->pm.backend.gpu_cycle_counter_requests; + + if (0 == kbdev->pm.backend.gpu_cycle_counter_requests) + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_CYCLE_COUNT_STOP, NULL); + + spin_unlock_irqrestore( + &kbdev->pm.backend.gpu_cycle_counter_requests_lock, + flags); + + kbase_pm_release_l2_caches(kbdev); +} + +void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_pm_release_gpu_cycle_counter_nolock(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter); diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h new file mode 100755 index 000000000000..6804f45ac27b --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h @@ -0,0 +1,548 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Power management API definitions used internally by GPU backend + */ + +#ifndef _KBASE_BACKEND_PM_INTERNAL_H_ +#define _KBASE_BACKEND_PM_INTERNAL_H_ + +#include + +#include "mali_kbase_pm_ca.h" +#include "mali_kbase_pm_policy.h" + + +/** + * kbase_pm_dev_idle - The GPU is idle. + * + * The OS may choose to turn off idle devices + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_dev_idle(struct kbase_device *kbdev); + +/** + * kbase_pm_dev_activate - The GPU is active. + * + * The OS should avoid opportunistically turning off the GPU while it is active + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_dev_activate(struct kbase_device *kbdev); + +/** + * kbase_pm_get_present_cores - Get details of the cores that are present in + * the device. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) present in the GPU device and also a count of + * the number of cores. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of cores present + */ +u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_get_active_cores - Get details of the cores that are currently + * active in the device. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) that are actively processing work (i.e. + * turned on *and* busy). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of active cores + */ +u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_get_trans_cores - Get details of the cores that are currently + * transitioning between power states. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) that are currently transitioning between + * power states. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of transitioning cores + */ +u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_get_ready_cores - Get details of the cores that are currently + * powered and ready for jobs. + * + * This function can be called by the active power policy to return a bitmask of + * the cores (of a specified type) that are powered and ready for jobs (they may + * or may not be currently executing jobs). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @type: The type of core (see the enum kbase_pm_core_type enumeration) + * + * Return: The bit mask of ready cores + */ +u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, + enum kbase_pm_core_type type); + +/** + * kbase_pm_clock_on - Turn the clock for the device on, and enable device + * interrupts. + * + * This function can be used by a power policy to turn the clock for the GPU on. + * It should be modified during integration to perform the necessary actions to + * ensure that the GPU is fully powered and clocked. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @is_resume: true if clock on due to resume after suspend, false otherwise + */ +void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume); + +/** + * kbase_pm_clock_off - Disable device interrupts, and turn the clock for the + * device off. + * + * This function can be used by a power policy to turn the clock for the GPU + * off. It should be modified during integration to perform the necessary + * actions to turn the clock off (if this is possible in the integration). + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @is_suspend: true if clock off due to suspend, false otherwise + * + * Return: true if clock was turned off, or + * false if clock can not be turned off due to pending page/bus fault + * workers. Caller must flush MMU workqueues and retry + */ +bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend); + +/** + * kbase_pm_enable_interrupts - Enable interrupts on the device. + * + * Interrupts are also enabled after a call to kbase_pm_clock_on(). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_enable_interrupts(struct kbase_device *kbdev); + +/** + * kbase_pm_disable_interrupts - Disable interrupts on the device. + * + * This prevents delivery of Power Management interrupts to the CPU so that + * kbase_pm_check_transitions_nolock() will not be called from the IRQ handler + * until kbase_pm_enable_interrupts() or kbase_pm_clock_on() is called. + * + * Interrupts are also disabled after a call to kbase_pm_clock_off(). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_disable_interrupts(struct kbase_device *kbdev); + +/** + * kbase_pm_disable_interrupts_nolock - Version of kbase_pm_disable_interrupts() + * that does not take the hwaccess_lock + * + * Caller must hold the hwaccess_lock. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_init_hw - Initialize the hardware. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @flags: Flags specifying the type of PM init + * + * This function checks the GPU ID register to ensure that the GPU is supported + * by the driver and performs a reset on the device so that it is in a known + * state before the device is used. + * + * Return: 0 if the device is supported and successfully reset. + */ +int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags); + +/** + * kbase_pm_reset_done - The GPU has been reset successfully. + * + * This function must be called by the GPU interrupt handler when the + * RESET_COMPLETED bit is set. It signals to the power management initialization + * code that the GPU has been successfully reset. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_reset_done(struct kbase_device *kbdev); + + +/** + * kbase_pm_check_transitions_nolock - Check if there are any power transitions + * to make, and if so start them. + * + * This function will check the desired_xx_state members of + * struct kbase_pm_device_data and the actual status of the hardware to see if + * any power transitions can be made at this time to make the hardware state + * closer to the state desired by the power policy. + * + * The return value can be used to check whether all the desired cores are + * available, and so whether it's worth submitting a job (e.g. from a Power + * Management IRQ). + * + * Note that this still returns true when desired_xx_state has no + * cores. That is: of the no cores desired, none were *un*available. In + * this case, the caller may still need to try submitting jobs. This is because + * the Core Availability Policy might have taken us to an intermediate state + * where no cores are powered, before powering on more cores (e.g. for core + * rotation) + * + * The caller must hold kbase_device.pm.power_change_lock + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: non-zero when all desired cores are available. That is, + * it's worthwhile for the caller to submit a job. + * false otherwise + */ +bool kbase_pm_check_transitions_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_check_transitions_sync - Synchronous and locking variant of + * kbase_pm_check_transitions_nolock() + * + * On returning, the desired state at the time of the call will have been met. + * + * There is nothing to stop the core being switched off by calls to + * kbase_pm_release_cores() or kbase_pm_unrequest_cores(). Therefore, the + * caller must have already made a call to + * kbase_pm_request_cores()/kbase_pm_request_cores_sync() previously. + * + * The usual use-case for this is to ensure cores are 'READY' after performing + * a GPU Reset. + * + * Unlike kbase_pm_check_transitions_nolock(), the caller must not hold + * kbase_device.pm.power_change_lock, because this function will take that + * lock itself. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_check_transitions_sync(struct kbase_device *kbdev); + +/** + * kbase_pm_update_cores_state_nolock - Variant of kbase_pm_update_cores_state() + * where the caller must hold + * kbase_device.pm.power_change_lock + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_update_cores_state - Update the desired state of shader cores from + * the Power Policy, and begin any power + * transitions. + * + * This function will update the desired_xx_state members of + * struct kbase_pm_device_data by calling into the current Power Policy. It will + * then begin power transitions to make the hardware acheive the desired shader + * core state. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_update_cores_state(struct kbase_device *kbdev); + +/** + * kbase_pm_cancel_deferred_poweroff - Cancel any pending requests to power off + * the GPU and/or shader cores. + * + * This should be called by any functions which directly power off the GPU. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev); + +/** + * kbasep_pm_init_core_use_bitmaps - Initialise data tracking the required + * and used cores. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev); + +/** + * kbasep_pm_metrics_init - Initialize the metrics gathering framework. + * + * This must be called before other metric gathering APIs are called. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Return: 0 on success, error code on error + */ +int kbasep_pm_metrics_init(struct kbase_device *kbdev); + +/** + * kbasep_pm_metrics_term - Terminate the metrics gathering framework. + * + * This must be called when metric gathering is no longer required. It is an + * error to call any metrics gathering function (other than + * kbasep_pm_metrics_init()) after calling this function. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbasep_pm_metrics_term(struct kbase_device *kbdev); + +/** + * kbase_pm_report_vsync - Function to be called by the frame buffer driver to + * update the vsync metric. + * + * This function should be called by the frame buffer driver to update whether + * the system is hitting the vsync target or not. buffer_updated should be true + * if the vsync corresponded with a new frame being displayed, otherwise it + * should be false. This function does not need to be called every vsync, but + * only when the value of @buffer_updated differs from a previous call. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * @buffer_updated: True if the buffer has been updated on this VSync, + * false otherwise + */ +void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated); + +/** + * kbase_pm_get_dvfs_action - Determine whether the DVFS system should change + * the clock speed of the GPU. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This function should be called regularly by the DVFS system to check whether + * the clock speed of the GPU needs updating. + */ +void kbase_pm_get_dvfs_action(struct kbase_device *kbdev); + +/** + * kbase_pm_request_gpu_cycle_counter - Mark that the GPU cycle counter is + * needed + * + * If the caller is the first caller then the GPU cycle counters will be enabled + * along with the l2 cache + * + * The GPU must be powered when calling this function (i.e. + * kbase_pm_context_active() must have been called). + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev); + +/** + * kbase_pm_request_gpu_cycle_counter_l2_is_on - Mark GPU cycle counter is + * needed (l2 cache already on) + * + * This is a version of the above function + * (kbase_pm_request_gpu_cycle_counter()) suitable for being called when the + * l2 cache is known to be on and assured to be on until the subsequent call of + * kbase_pm_release_gpu_cycle_counter() such as when a job is submitted. It does + * not sleep and can be called from atomic functions. + * + * The GPU must be powered when calling this function (i.e. + * kbase_pm_context_active() must have been called) and the l2 cache must be + * powered on. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev); + +/** + * kbase_pm_release_gpu_cycle_counter - Mark that the GPU cycle counter is no + * longer in use + * + * If the caller is the last caller then the GPU cycle counters will be + * disabled. A request must have been made before a call to this. + * + * Caller must not hold the hwaccess_lock, as it will be taken in this function. + * If the caller is already holding this lock then + * kbase_pm_release_gpu_cycle_counter_nolock() must be used instead. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev); + +/** + * kbase_pm_release_gpu_cycle_counter_nolock - Version of kbase_pm_release_gpu_cycle_counter() + * that does not take hwaccess_lock + * + * Caller must hold the hwaccess_lock. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev); + +/** + * kbase_pm_wait_for_poweroff_complete - Wait for the poweroff workqueue to + * complete + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev); + +/** + * kbase_pm_register_access_enable - Enable access to GPU registers + * + * Enables access to the GPU registers before power management has powered up + * the GPU with kbase_pm_powerup(). + * + * Access to registers should be done using kbase_os_reg_read()/write() at this + * stage, not kbase_reg_read()/write(). + * + * This results in the power management callbacks provided in the driver + * configuration to get called to turn on power and/or clocks to the GPU. See + * kbase_pm_callback_conf. + * + * This should only be used before power management is powered up with + * kbase_pm_powerup() + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_register_access_enable(struct kbase_device *kbdev); + +/** + * kbase_pm_register_access_disable - Disable early register access + * + * Disables access to the GPU registers enabled earlier by a call to + * kbase_pm_register_access_enable(). + * + * This results in the power management callbacks provided in the driver + * configuration to get called to turn off power and/or clocks to the GPU. See + * kbase_pm_callback_conf + * + * This should only be used before power management is powered up with + * kbase_pm_powerup() + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_register_access_disable(struct kbase_device *kbdev); + +/* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline + * function */ + +/** + * kbase_pm_metrics_is_active - Check if the power management metrics + * collection is active. + * + * Note that this returns if the power management metrics collection was + * active at the time of calling, it is possible that after the call the metrics + * collection enable may have changed state. + * + * The caller must handle the consequence that the state may have changed. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * Return: true if metrics collection was active else false. + */ +bool kbase_pm_metrics_is_active(struct kbase_device *kbdev); + +/** + * kbase_pm_do_poweron - Power on the GPU, and any cores that are requested. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @is_resume: true if power on due to resume after suspend, + * false otherwise + */ +void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume); + +/** + * kbase_pm_do_poweroff - Power off the GPU, and any cores that have been + * requested. + * + * @kbdev: The kbase device structure for the device (must be a valid + * pointer) + * @is_suspend: true if power off due to suspend, + * false otherwise + */ +void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend); + +#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) +void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, + unsigned long *total, unsigned long *busy); +void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev); +#endif /* defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) */ + +#ifdef CONFIG_MALI_MIDGARD_DVFS + +/** + * kbase_platform_dvfs_event - Report utilisation to DVFS code + * + * Function provided by platform specific code when DVFS is enabled to allow + * the power management metrics system to report utilisation. + * + * @kbdev: The kbase device structure for the device (must be a + * valid pointer) + * @utilisation: The current calculated utilisation by the metrics system. + * @util_gl_share: The current calculated gl share of utilisation. + * @util_cl_share: The current calculated cl share of utilisation per core + * group. + * Return: Returns 0 on failure and non zero on success. + */ + +int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, + u32 util_gl_share, u32 util_cl_share[2]); +#endif + +void kbase_pm_power_changed(struct kbase_device *kbdev); + +/** + * kbase_pm_metrics_update - Inform the metrics system that an atom is either + * about to be run or has just completed. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @now: Pointer to the timestamp of the change, or NULL to use current time + * + * Caller must hold hwaccess_lock + */ +void kbase_pm_metrics_update(struct kbase_device *kbdev, + ktime_t *now); + +/** + * kbase_pm_cache_snoop_enable - Allow CPU snoops on the GPU + * If the GPU does not have coherency this is a no-op + * @kbdev: Device pointer + * + * This function should be called after L2 power up. + */ + +void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev); + +/** + * kbase_pm_cache_snoop_disable - Prevent CPU snoops on the GPU + * If the GPU does not have coherency this is a no-op + * @kbdev: Device pointer + * + * This function should be called before L2 power off. + */ +void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev); + +#endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c new file mode 100755 index 000000000000..024248ca7123 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c @@ -0,0 +1,401 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Metrics for power management + */ + +#include +#include +#include +#include + +/* When VSync is being hit aim for utilisation between 70-90% */ +#define KBASE_PM_VSYNC_MIN_UTILISATION 70 +#define KBASE_PM_VSYNC_MAX_UTILISATION 90 +/* Otherwise aim for 10-40% */ +#define KBASE_PM_NO_VSYNC_MIN_UTILISATION 10 +#define KBASE_PM_NO_VSYNC_MAX_UTILISATION 40 + +/* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns + * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly + * under 11s. Exceeding this will cause overflow */ +#define KBASE_PM_TIME_SHIFT 8 + +/* Maximum time between sampling of utilization data, without resetting the + * counters. */ +#define MALI_UTILIZATION_MAX_PERIOD 100000 /* ns = 100ms */ + +#ifdef CONFIG_MALI_MIDGARD_DVFS +static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) +{ + unsigned long flags; + struct kbasep_pm_metrics_data *metrics; + + KBASE_DEBUG_ASSERT(timer != NULL); + + metrics = container_of(timer, struct kbasep_pm_metrics_data, timer); + kbase_pm_get_dvfs_action(metrics->kbdev); + + spin_lock_irqsave(&metrics->lock, flags); + + if (metrics->timer_active) + hrtimer_start(timer, + HR_TIMER_DELAY_MSEC(metrics->kbdev->pm.dvfs_period), + HRTIMER_MODE_REL); + + spin_unlock_irqrestore(&metrics->lock, flags); + + return HRTIMER_NORESTART; +} +#endif /* CONFIG_MALI_MIDGARD_DVFS */ + +int kbasep_pm_metrics_init(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + kbdev->pm.backend.metrics.kbdev = kbdev; + + kbdev->pm.backend.metrics.time_period_start = ktime_get(); + kbdev->pm.backend.metrics.time_busy = 0; + kbdev->pm.backend.metrics.time_idle = 0; + kbdev->pm.backend.metrics.prev_busy = 0; + kbdev->pm.backend.metrics.prev_idle = 0; + kbdev->pm.backend.metrics.gpu_active = false; + kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; + kbdev->pm.backend.metrics.busy_cl[0] = 0; + kbdev->pm.backend.metrics.busy_cl[1] = 0; + kbdev->pm.backend.metrics.busy_gl = 0; + + spin_lock_init(&kbdev->pm.backend.metrics.lock); + +#ifdef CONFIG_MALI_MIDGARD_DVFS + kbdev->pm.backend.metrics.timer_active = true; + hrtimer_init(&kbdev->pm.backend.metrics.timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + kbdev->pm.backend.metrics.timer.function = dvfs_callback; + + hrtimer_start(&kbdev->pm.backend.metrics.timer, + HR_TIMER_DELAY_MSEC(kbdev->pm.dvfs_period), + HRTIMER_MODE_REL); +#endif /* CONFIG_MALI_MIDGARD_DVFS */ + + return 0; +} + +KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init); + +void kbasep_pm_metrics_term(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_MIDGARD_DVFS + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + kbdev->pm.backend.metrics.timer_active = false; + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); + + hrtimer_cancel(&kbdev->pm.backend.metrics.timer); +#endif /* CONFIG_MALI_MIDGARD_DVFS */ +} + +KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); + +/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this + * function + */ +static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, + ktime_t now) +{ + ktime_t diff; + + lockdep_assert_held(&kbdev->pm.backend.metrics.lock); + + diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start); + if (ktime_to_ns(diff) < 0) + return; + + if (kbdev->pm.backend.metrics.gpu_active) { + u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); + + kbdev->pm.backend.metrics.time_busy += ns_time; + if (kbdev->pm.backend.metrics.active_cl_ctx[0]) + kbdev->pm.backend.metrics.busy_cl[0] += ns_time; + if (kbdev->pm.backend.metrics.active_cl_ctx[1]) + kbdev->pm.backend.metrics.busy_cl[1] += ns_time; + if (kbdev->pm.backend.metrics.active_gl_ctx[0]) + kbdev->pm.backend.metrics.busy_gl += ns_time; + if (kbdev->pm.backend.metrics.active_gl_ctx[1]) + kbdev->pm.backend.metrics.busy_gl += ns_time; + } else { + kbdev->pm.backend.metrics.time_idle += (u32) (ktime_to_ns(diff) + >> KBASE_PM_TIME_SHIFT); + } + + kbdev->pm.backend.metrics.time_period_start = now; +} + +#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) +/* Caller needs to hold kbdev->pm.backend.metrics.lock before calling this + * function. + */ +static void kbase_pm_reset_dvfs_utilisation_unlocked(struct kbase_device *kbdev, + ktime_t now) +{ + /* Store previous value */ + kbdev->pm.backend.metrics.prev_idle = + kbdev->pm.backend.metrics.time_idle; + kbdev->pm.backend.metrics.prev_busy = + kbdev->pm.backend.metrics.time_busy; + + /* Reset current values */ + kbdev->pm.backend.metrics.time_period_start = now; + kbdev->pm.backend.metrics.time_idle = 0; + kbdev->pm.backend.metrics.time_busy = 0; + kbdev->pm.backend.metrics.busy_cl[0] = 0; + kbdev->pm.backend.metrics.busy_cl[1] = 0; + kbdev->pm.backend.metrics.busy_gl = 0; +} + +void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, ktime_get()); + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); +} + +void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, + unsigned long *total_out, unsigned long *busy_out) +{ + ktime_t now = ktime_get(); + unsigned long flags, busy, total; + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + kbase_pm_get_dvfs_utilisation_calc(kbdev, now); + + busy = kbdev->pm.backend.metrics.time_busy; + total = busy + kbdev->pm.backend.metrics.time_idle; + + /* Reset stats if older than MALI_UTILIZATION_MAX_PERIOD (default + * 100ms) */ + if (total >= MALI_UTILIZATION_MAX_PERIOD) { + kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); + } else if (total < (MALI_UTILIZATION_MAX_PERIOD / 2)) { + total += kbdev->pm.backend.metrics.prev_idle + + kbdev->pm.backend.metrics.prev_busy; + busy += kbdev->pm.backend.metrics.prev_busy; + } + + *total_out = total; + *busy_out = busy; + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); +} +#endif + +#ifdef CONFIG_MALI_MIDGARD_DVFS + +/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this + * function + */ +int kbase_pm_get_dvfs_utilisation_old(struct kbase_device *kbdev, + int *util_gl_share, + int util_cl_share[2], + ktime_t now) +{ + int utilisation; + int busy; + + kbase_pm_get_dvfs_utilisation_calc(kbdev, now); + + if (kbdev->pm.backend.metrics.time_idle + + kbdev->pm.backend.metrics.time_busy == 0) { + /* No data - so we return NOP */ + utilisation = -1; + if (util_gl_share) + *util_gl_share = -1; + if (util_cl_share) { + util_cl_share[0] = -1; + util_cl_share[1] = -1; + } + goto out; + } + + utilisation = (100 * kbdev->pm.backend.metrics.time_busy) / + (kbdev->pm.backend.metrics.time_idle + + kbdev->pm.backend.metrics.time_busy); + + busy = kbdev->pm.backend.metrics.busy_gl + + kbdev->pm.backend.metrics.busy_cl[0] + + kbdev->pm.backend.metrics.busy_cl[1]; + + if (busy != 0) { + if (util_gl_share) + *util_gl_share = + (100 * kbdev->pm.backend.metrics.busy_gl) / + busy; + if (util_cl_share) { + util_cl_share[0] = + (100 * kbdev->pm.backend.metrics.busy_cl[0]) / + busy; + util_cl_share[1] = + (100 * kbdev->pm.backend.metrics.busy_cl[1]) / + busy; + } + } else { + if (util_gl_share) + *util_gl_share = -1; + if (util_cl_share) { + util_cl_share[0] = -1; + util_cl_share[1] = -1; + } + } + +out: + return utilisation; +} + +void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) +{ + unsigned long flags; + int utilisation, util_gl_share; + int util_cl_share[2]; + ktime_t now; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + + now = ktime_get(); + + utilisation = kbase_pm_get_dvfs_utilisation_old(kbdev, &util_gl_share, + util_cl_share, now); + + if (utilisation < 0 || util_gl_share < 0 || util_cl_share[0] < 0 || + util_cl_share[1] < 0) { + utilisation = 0; + util_gl_share = 0; + util_cl_share[0] = 0; + util_cl_share[1] = 0; + goto out; + } + +out: +#ifdef CONFIG_MALI_MIDGARD_DVFS + kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, + util_cl_share); +#endif /*CONFIG_MALI_MIDGARD_DVFS */ + + kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); + + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); +} + +bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) +{ + bool isactive; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + isactive = kbdev->pm.backend.metrics.timer_active; + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); + + return isactive; +} +KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active); + +#endif /* CONFIG_MALI_MIDGARD_DVFS */ + +/** + * kbase_pm_metrics_active_calc - Update PM active counts based on currently + * running atoms + * @kbdev: Device pointer + * + * The caller must hold kbdev->pm.backend.metrics.lock + */ +static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) +{ + int js; + + lockdep_assert_held(&kbdev->pm.backend.metrics.lock); + + kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; + kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; + kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; + kbdev->pm.backend.metrics.gpu_active = false; + + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); + + /* Head atom may have just completed, so if it isn't running + * then try the next atom */ + if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) + katom = kbase_gpu_inspect(kbdev, js, 1); + + if (katom && katom->gpu_rb_state == + KBASE_ATOM_GPU_RB_SUBMITTED) { + if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { + int device_nr = (katom->core_req & + BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) + ? katom->device_nr : 0; + if (!WARN_ON(device_nr >= 2)) + kbdev->pm.backend.metrics. + active_cl_ctx[device_nr] = 1; + } else { + /* Slot 2 should not be running non-compute + * atoms */ + if (!WARN_ON(js >= 2)) + kbdev->pm.backend.metrics. + active_gl_ctx[js] = 1; + } + kbdev->pm.backend.metrics.gpu_active = true; + } + } +} + +/* called when job is submitted to or removed from a GPU slot */ +void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) +{ + unsigned long flags; + ktime_t now; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); + + if (!timestamp) { + now = ktime_get(); + timestamp = &now; + } + + /* Track how long CL and/or GL jobs have been busy for */ + kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); + + kbase_pm_metrics_active_calc(kbdev); + + spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); +} diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c new file mode 100755 index 000000000000..075f020c66e6 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c @@ -0,0 +1,973 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Power policy API implementations + */ + +#include +#include +#include +#include +#include + +static const struct kbase_pm_policy *const policy_list[] = { +#ifdef CONFIG_MALI_NO_MALI + &kbase_pm_always_on_policy_ops, + &kbase_pm_demand_policy_ops, + &kbase_pm_coarse_demand_policy_ops, +#if !MALI_CUSTOMER_RELEASE + &kbase_pm_demand_always_powered_policy_ops, + &kbase_pm_fast_start_policy_ops, +#endif +#else /* CONFIG_MALI_NO_MALI */ +#if !PLATFORM_POWER_DOWN_ONLY + &kbase_pm_demand_policy_ops, +#endif /* !PLATFORM_POWER_DOWN_ONLY */ + &kbase_pm_coarse_demand_policy_ops, + &kbase_pm_always_on_policy_ops, +#if !MALI_CUSTOMER_RELEASE +#if !PLATFORM_POWER_DOWN_ONLY + &kbase_pm_demand_always_powered_policy_ops, + &kbase_pm_fast_start_policy_ops, +#endif /* !PLATFORM_POWER_DOWN_ONLY */ +#endif +#endif /* CONFIG_MALI_NO_MALI */ +}; + +/* The number of policies available in the system. + * This is derived from the number of functions listed in policy_get_functions. + */ +#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list)) + + +/* Function IDs for looking up Timeline Trace codes in + * kbase_pm_change_state_trace_code */ +enum kbase_pm_func_id { + KBASE_PM_FUNC_ID_REQUEST_CORES_START, + KBASE_PM_FUNC_ID_REQUEST_CORES_END, + KBASE_PM_FUNC_ID_RELEASE_CORES_START, + KBASE_PM_FUNC_ID_RELEASE_CORES_END, + /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither + * expect to hit it nor tend to hit it very much anyway. We can detect + * whether we need more instrumentation by a difference between + * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */ + + /* Must be the last */ + KBASE_PM_FUNC_ID_COUNT +}; + + +/* State changes during request/unrequest/release-ing cores */ +enum { + KBASE_PM_CHANGE_STATE_SHADER = (1u << 0), + KBASE_PM_CHANGE_STATE_TILER = (1u << 1), + + /* These two must be last */ + KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER | + KBASE_PM_CHANGE_STATE_SHADER), + KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1 +}; +typedef u32 kbase_pm_change_state; + + +#ifdef CONFIG_MALI_TRACE_TIMELINE +/* Timeline Trace code lookups for each function */ +static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT] + [KBASE_PM_CHANGE_STATE_COUNT] = { + /* kbase_pm_request_cores */ + [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0, + [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START, + [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START, + [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER | + KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START, + + [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0, + [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END, + [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END, + [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER | + KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END, + + /* kbase_pm_release_cores */ + [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0, + [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START, + [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START, + [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER | + KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START, + + [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0, + [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END, + [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END, + [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER | + KBASE_PM_CHANGE_STATE_TILER] = + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END +}; + +static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev, + enum kbase_pm_func_id func_id, + kbase_pm_change_state state) +{ + int trace_code; + + KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT); + KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) == + state); + + trace_code = kbase_pm_change_state_trace_code[func_id][state]; + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code); +} + +#else /* CONFIG_MALI_TRACE_TIMELINE */ +static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev, + enum kbase_pm_func_id func_id, kbase_pm_change_state state) +{ +} + +#endif /* CONFIG_MALI_TRACE_TIMELINE */ + +/** + * kbasep_pm_do_poweroff_cores - Process a poweroff request and power down any + * requested shader cores + * @kbdev: Device pointer + */ +static void kbasep_pm_do_poweroff_cores(struct kbase_device *kbdev) +{ + u64 prev_shader_state = kbdev->pm.backend.desired_shader_state; + u64 prev_tiler_state = kbdev->pm.backend.desired_tiler_state; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->pm.backend.desired_shader_state &= + ~kbdev->pm.backend.shader_poweroff_pending; + kbdev->pm.backend.desired_tiler_state &= + ~kbdev->pm.backend.tiler_poweroff_pending; + + kbdev->pm.backend.shader_poweroff_pending = 0; + kbdev->pm.backend.tiler_poweroff_pending = 0; + + if (prev_shader_state != kbdev->pm.backend.desired_shader_state || + prev_tiler_state != + kbdev->pm.backend.desired_tiler_state || + kbdev->pm.backend.ca_in_transition) { + bool cores_are_available; + + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START); + cores_are_available = kbase_pm_check_transitions_nolock(kbdev); + KBASE_TIMELINE_PM_CHECKTRANS(kbdev, + SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END); + + /* Don't need 'cores_are_available', + * because we don't return anything */ + CSTD_UNUSED(cores_are_available); + } +} + +static enum hrtimer_restart +kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer) +{ + struct kbase_device *kbdev; + unsigned long flags; + + kbdev = container_of(timer, struct kbase_device, + pm.backend.gpu_poweroff_timer); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* It is safe for this call to do nothing if the work item is already + * queued. The worker function will read the must up-to-date state of + * kbdev->pm.backend.gpu_poweroff_pending under lock. + * + * If a state change occurs while the worker function is processing, + * this call will succeed as a work item can be requeued once it has + * started processing. + */ + if (kbdev->pm.backend.gpu_poweroff_pending) + queue_work(kbdev->pm.backend.gpu_poweroff_wq, + &kbdev->pm.backend.gpu_poweroff_work); + + if (kbdev->pm.backend.shader_poweroff_pending || + kbdev->pm.backend.tiler_poweroff_pending) { + kbdev->pm.backend.shader_poweroff_pending_time--; + + KBASE_DEBUG_ASSERT( + kbdev->pm.backend.shader_poweroff_pending_time + >= 0); + + if (!kbdev->pm.backend.shader_poweroff_pending_time) + kbasep_pm_do_poweroff_cores(kbdev); + } + + if (kbdev->pm.backend.poweroff_timer_needed) { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time); + + return HRTIMER_RESTART; + } + + kbdev->pm.backend.poweroff_timer_running = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return HRTIMER_NORESTART; +} + +static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data) +{ + unsigned long flags; + struct kbase_device *kbdev; + bool do_poweroff = false; + + kbdev = container_of(data, struct kbase_device, + pm.backend.gpu_poweroff_work); + + mutex_lock(&kbdev->pm.lock); + + if (kbdev->pm.backend.gpu_poweroff_pending == 0) { + mutex_unlock(&kbdev->pm.lock); + return; + } + + kbdev->pm.backend.gpu_poweroff_pending--; + + if (kbdev->pm.backend.gpu_poweroff_pending > 0) { + mutex_unlock(&kbdev->pm.lock); + return; + } + + KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Only power off the GPU if a request is still pending */ + if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev)) + do_poweroff = true; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (do_poweroff) { + kbdev->pm.backend.poweroff_timer_needed = false; + hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer); + kbdev->pm.backend.poweroff_timer_running = false; + + /* Power off the GPU */ + kbase_pm_do_poweroff(kbdev, false); + } + + mutex_unlock(&kbdev->pm.lock); +} + +int kbase_pm_policy_init(struct kbase_device *kbdev) +{ + struct workqueue_struct *wq; + + wq = alloc_workqueue("kbase_pm_do_poweroff", + WQ_HIGHPRI | WQ_UNBOUND, 1); + if (!wq) + return -ENOMEM; + + kbdev->pm.backend.gpu_poweroff_wq = wq; + INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work, + kbasep_pm_do_gpu_poweroff_wq); + hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + kbdev->pm.backend.gpu_poweroff_timer.function = + kbasep_pm_do_gpu_poweroff_callback; + kbdev->pm.backend.pm_current_policy = policy_list[0]; + kbdev->pm.backend.pm_current_policy->init(kbdev); + kbdev->pm.gpu_poweroff_time = + HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS); + kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; + kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU; + + return 0; +} + +void kbase_pm_policy_term(struct kbase_device *kbdev) +{ + kbdev->pm.backend.pm_current_policy->term(kbdev); + destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq); +} + +void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev) +{ + unsigned long flags; + + lockdep_assert_held(&kbdev->pm.lock); + + kbdev->pm.backend.poweroff_timer_needed = false; + hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.poweroff_timer_running = false; + + /* If wq is already running but is held off by pm.lock, make sure it has + * no effect */ + kbdev->pm.backend.gpu_poweroff_pending = 0; + + kbdev->pm.backend.shader_poweroff_pending = 0; + kbdev->pm.backend.tiler_poweroff_pending = 0; + kbdev->pm.backend.shader_poweroff_pending_time = 0; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +void kbase_pm_update_active(struct kbase_device *kbdev) +{ + struct kbase_pm_device_data *pm = &kbdev->pm; + struct kbase_pm_backend_data *backend = &pm->backend; + unsigned long flags; + bool active; + + lockdep_assert_held(&pm->lock); + + /* pm_current_policy will never be NULL while pm.lock is held */ + KBASE_DEBUG_ASSERT(backend->pm_current_policy); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + active = backend->pm_current_policy->get_core_active(kbdev); + + if (active) { + if (backend->gpu_poweroff_pending) { + /* Cancel any pending power off request */ + backend->gpu_poweroff_pending = 0; + + /* If a request was pending then the GPU was still + * powered, so no need to continue */ + if (!kbdev->poweroff_pending) { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + return; + } + } + + if (!backend->poweroff_timer_running && !backend->gpu_powered && + (pm->poweroff_gpu_ticks || + pm->poweroff_shader_ticks)) { + backend->poweroff_timer_needed = true; + backend->poweroff_timer_running = true; + hrtimer_start(&backend->gpu_poweroff_timer, + pm->gpu_poweroff_time, + HRTIMER_MODE_REL); + } + + /* Power on the GPU and any cores requested by the policy */ + if (pm->backend.poweroff_wait_in_progress) { + pm->backend.poweron_required = true; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_pm_do_poweron(kbdev, false); + } + } else { + /* It is an error for the power policy to power off the GPU + * when there are contexts active */ + KBASE_DEBUG_ASSERT(pm->active_count == 0); + + if (backend->shader_poweroff_pending || + backend->tiler_poweroff_pending) { + backend->shader_poweroff_pending = 0; + backend->tiler_poweroff_pending = 0; + backend->shader_poweroff_pending_time = 0; + } + + /* Request power off */ + if (pm->backend.gpu_powered) { + if (pm->poweroff_gpu_ticks) { + backend->gpu_poweroff_pending = + pm->poweroff_gpu_ticks; + backend->poweroff_timer_needed = true; + if (!backend->poweroff_timer_running) { + /* Start timer if not running (eg if + * power policy has been changed from + * always_on to something else). This + * will ensure the GPU is actually + * powered off */ + backend->poweroff_timer_running + = true; + hrtimer_start( + &backend->gpu_poweroff_timer, + pm->gpu_poweroff_time, + HRTIMER_MODE_REL); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + + /* Power off the GPU immediately */ + kbase_pm_do_poweroff(kbdev, false); + } + } else { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + } +} + +void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) +{ + u64 desired_bitmap; + u64 desired_tiler_bitmap; + bool cores_are_available; + bool do_poweroff = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->pm.backend.pm_current_policy == NULL) + return; + if (kbdev->pm.backend.poweroff_wait_in_progress) + return; + + if (kbdev->protected_mode_transition && !kbdev->shader_needed_bitmap && + !kbdev->shader_inuse_bitmap && !kbdev->tiler_needed_cnt + && !kbdev->tiler_inuse_cnt) { + /* We are trying to change in/out of protected mode - force all + * cores off so that the L2 powers down */ + desired_bitmap = 0; + desired_tiler_bitmap = 0; + } else { + desired_bitmap = + kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev); + desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev); + + if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0) + desired_tiler_bitmap = 1; + else + desired_tiler_bitmap = 0; + + if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) { + /* Unless XAFFINITY is supported, enable core 0 if tiler + * required, regardless of core availability */ + if (kbdev->tiler_needed_cnt > 0 || + kbdev->tiler_inuse_cnt > 0) + desired_bitmap |= 1; + } + } + + if (kbdev->pm.backend.desired_shader_state != desired_bitmap) + KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u, + (u32)desired_bitmap); + /* Are any cores being powered on? */ + if (~kbdev->pm.backend.desired_shader_state & desired_bitmap || + ~kbdev->pm.backend.desired_tiler_state & desired_tiler_bitmap || + kbdev->pm.backend.ca_in_transition) { + /* Check if we are powering off any cores before updating shader + * state */ + if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap || + kbdev->pm.backend.desired_tiler_state & + ~desired_tiler_bitmap) { + /* Start timer to power off cores */ + kbdev->pm.backend.shader_poweroff_pending |= + (kbdev->pm.backend.desired_shader_state & + ~desired_bitmap); + kbdev->pm.backend.tiler_poweroff_pending |= + (kbdev->pm.backend.desired_tiler_state & + ~desired_tiler_bitmap); + + if (kbdev->pm.poweroff_shader_ticks && + !kbdev->protected_mode_transition) + kbdev->pm.backend.shader_poweroff_pending_time = + kbdev->pm.poweroff_shader_ticks; + else + do_poweroff = true; + } + + kbdev->pm.backend.desired_shader_state = desired_bitmap; + kbdev->pm.backend.desired_tiler_state = desired_tiler_bitmap; + + /* If any cores are being powered on, transition immediately */ + cores_are_available = kbase_pm_check_transitions_nolock(kbdev); + } else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap || + kbdev->pm.backend.desired_tiler_state & + ~desired_tiler_bitmap) { + /* Start timer to power off cores */ + kbdev->pm.backend.shader_poweroff_pending |= + (kbdev->pm.backend.desired_shader_state & + ~desired_bitmap); + kbdev->pm.backend.tiler_poweroff_pending |= + (kbdev->pm.backend.desired_tiler_state & + ~desired_tiler_bitmap); + if (kbdev->pm.poweroff_shader_ticks && + !kbdev->protected_mode_transition) + kbdev->pm.backend.shader_poweroff_pending_time = + kbdev->pm.poweroff_shader_ticks; + else + kbasep_pm_do_poweroff_cores(kbdev); + } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 && + desired_tiler_bitmap != 0 && + kbdev->pm.backend.poweroff_timer_needed) { + /* If power policy is keeping cores on despite there being no + * active contexts then disable poweroff timer as it isn't + * required. + * Only reset poweroff_timer_needed if we're not in the middle + * of the power off callback */ + kbdev->pm.backend.poweroff_timer_needed = false; + } + + /* Ensure timer does not power off wanted cores and make sure to power + * off unwanted cores */ + if (kbdev->pm.backend.shader_poweroff_pending || + kbdev->pm.backend.tiler_poweroff_pending) { + kbdev->pm.backend.shader_poweroff_pending &= + ~(kbdev->pm.backend.desired_shader_state & + desired_bitmap); + kbdev->pm.backend.tiler_poweroff_pending &= + ~(kbdev->pm.backend.desired_tiler_state & + desired_tiler_bitmap); + + if (!kbdev->pm.backend.shader_poweroff_pending && + !kbdev->pm.backend.tiler_poweroff_pending) + kbdev->pm.backend.shader_poweroff_pending_time = 0; + } + + /* Shader poweroff is deferred to the end of the function, to eliminate + * issues caused by the core availability policy recursing into this + * function */ + if (do_poweroff) + kbasep_pm_do_poweroff_cores(kbdev); + + /* Don't need 'cores_are_available', because we don't return anything */ + CSTD_UNUSED(cores_are_available); +} + +void kbase_pm_update_cores_state(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_pm_update_cores_state_nolock(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +int kbase_pm_list_policies(const struct kbase_pm_policy * const **list) +{ + if (!list) + return POLICY_COUNT; + + *list = policy_list; + + return POLICY_COUNT; +} + +KBASE_EXPORT_TEST_API(kbase_pm_list_policies); + +const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + return kbdev->pm.backend.pm_current_policy; +} + +KBASE_EXPORT_TEST_API(kbase_pm_get_policy); + +void kbase_pm_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_policy *new_policy) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + const struct kbase_pm_policy *old_policy; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(new_policy != NULL); + + KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id); + + /* During a policy change we pretend the GPU is active */ + /* A suspend won't happen here, because we're in a syscall from a + * userspace thread */ + kbase_pm_context_active(kbdev); + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + /* Remove the policy to prevent IRQ handlers from working on it */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + old_policy = kbdev->pm.backend.pm_current_policy; + kbdev->pm.backend.pm_current_policy = NULL; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, + old_policy->id); + if (old_policy->term) + old_policy->term(kbdev); + + KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, + new_policy->id); + if (new_policy->init) + new_policy->init(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.pm_current_policy = new_policy; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* If any core power state changes were previously attempted, but + * couldn't be made because the policy was changing (current_policy was + * NULL), then re-try them here. */ + kbase_pm_update_active(kbdev); + kbase_pm_update_cores_state(kbdev); + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + + /* Now the policy change is finished, we release our fake context active + * reference */ + kbase_pm_context_idle(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_set_policy); + +/* Check whether a state change has finished, and trace it as completed */ +static void +kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev) +{ + if ((kbdev->shader_available_bitmap & + kbdev->pm.backend.desired_shader_state) + == kbdev->pm.backend.desired_shader_state && + (kbdev->tiler_available_bitmap & + kbdev->pm.backend.desired_tiler_state) + == kbdev->pm.backend.desired_tiler_state) + kbase_timeline_pm_check_handle_event(kbdev, + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); +} + +void kbase_pm_request_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores) +{ + u64 cores; + + kbase_pm_change_state change_gpu_state = 0u; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + cores = shader_cores; + while (cores) { + int bitnum = fls64(cores) - 1; + u64 bit = 1ULL << bitnum; + + /* It should be almost impossible for this to overflow. It would + * require 2^32 atoms to request a particular core, which would + * require 2^24 contexts to submit. This would require an amount + * of memory that is impossible on a 32-bit system and extremely + * unlikely on a 64-bit system. */ + int cnt = ++kbdev->shader_needed_cnt[bitnum]; + + if (1 == cnt) { + kbdev->shader_needed_bitmap |= bit; + change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; + } + + cores &= ~bit; + } + + if (tiler_required) { + int cnt = ++kbdev->tiler_needed_cnt; + + if (1 == cnt) + change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; + + KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0); + } + + if (change_gpu_state) { + KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL, + NULL, 0u, (u32) kbdev->shader_needed_bitmap); + + kbase_timeline_pm_cores_func(kbdev, + KBASE_PM_FUNC_ID_REQUEST_CORES_START, + change_gpu_state); + kbase_pm_update_cores_state_nolock(kbdev); + kbase_timeline_pm_cores_func(kbdev, + KBASE_PM_FUNC_ID_REQUEST_CORES_END, + change_gpu_state); + } +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_cores); + +void kbase_pm_unrequest_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores) +{ + kbase_pm_change_state change_gpu_state = 0u; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + while (shader_cores) { + int bitnum = fls64(shader_cores) - 1; + u64 bit = 1ULL << bitnum; + int cnt; + + KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0); + + cnt = --kbdev->shader_needed_cnt[bitnum]; + + if (0 == cnt) { + kbdev->shader_needed_bitmap &= ~bit; + + change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; + } + + shader_cores &= ~bit; + } + + if (tiler_required) { + int cnt; + + KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0); + + cnt = --kbdev->tiler_needed_cnt; + + if (0 == cnt) + change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; + } + + if (change_gpu_state) { + KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL, + NULL, 0u, (u32) kbdev->shader_needed_bitmap); + + kbase_pm_update_cores_state_nolock(kbdev); + + /* Trace that any state change effectively completes immediately + * - no-one will wait on the state change */ + kbase_pm_trace_check_and_finish_state_change(kbdev); + } +} + +KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores); + +enum kbase_pm_cores_ready +kbase_pm_register_inuse_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores) +{ + u64 prev_shader_needed; /* Just for tracing */ + u64 prev_shader_inuse; /* Just for tracing */ + + lockdep_assert_held(&kbdev->hwaccess_lock); + + prev_shader_needed = kbdev->shader_needed_bitmap; + prev_shader_inuse = kbdev->shader_inuse_bitmap; + + /* If desired_shader_state does not contain the requested cores, then + * power management is not attempting to powering those cores (most + * likely due to core availability policy) and a new job affinity must + * be chosen */ + if ((kbdev->pm.backend.desired_shader_state & shader_cores) != + shader_cores) { + return (kbdev->pm.backend.poweroff_wait_in_progress || + kbdev->pm.backend.pm_current_policy == NULL) ? + KBASE_CORES_NOT_READY : KBASE_NEW_AFFINITY; + } + + if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores || + (tiler_required && !kbdev->tiler_available_bitmap)) { + /* Trace ongoing core transition */ + kbase_timeline_pm_l2_transition_start(kbdev); + return KBASE_CORES_NOT_READY; + } + + /* If we started to trace a state change, then trace it has being + * finished by now, at the very latest */ + kbase_pm_trace_check_and_finish_state_change(kbdev); + /* Trace core transition done */ + kbase_timeline_pm_l2_transition_done(kbdev); + + while (shader_cores) { + int bitnum = fls64(shader_cores) - 1; + u64 bit = 1ULL << bitnum; + int cnt; + + KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0); + + cnt = --kbdev->shader_needed_cnt[bitnum]; + + if (0 == cnt) + kbdev->shader_needed_bitmap &= ~bit; + + /* shader_inuse_cnt should not overflow because there can only + * be a very limited number of jobs on the h/w at one time */ + + kbdev->shader_inuse_cnt[bitnum]++; + kbdev->shader_inuse_bitmap |= bit; + + shader_cores &= ~bit; + } + + if (tiler_required) { + KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0); + + --kbdev->tiler_needed_cnt; + + kbdev->tiler_inuse_cnt++; + + KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0); + } + + if (prev_shader_needed != kbdev->shader_needed_bitmap) + KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL, + NULL, 0u, (u32) kbdev->shader_needed_bitmap); + + if (prev_shader_inuse != kbdev->shader_inuse_bitmap) + KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL, + NULL, 0u, (u32) kbdev->shader_inuse_bitmap); + + return KBASE_CORES_READY; +} + +KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores); + +void kbase_pm_release_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores) +{ + kbase_pm_change_state change_gpu_state = 0u; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + while (shader_cores) { + int bitnum = fls64(shader_cores) - 1; + u64 bit = 1ULL << bitnum; + int cnt; + + KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0); + + cnt = --kbdev->shader_inuse_cnt[bitnum]; + + if (0 == cnt) { + kbdev->shader_inuse_bitmap &= ~bit; + change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; + } + + shader_cores &= ~bit; + } + + if (tiler_required) { + int cnt; + + KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0); + + cnt = --kbdev->tiler_inuse_cnt; + + if (0 == cnt) + change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; + } + + if (change_gpu_state) { + KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL, + NULL, 0u, (u32) kbdev->shader_inuse_bitmap); + + kbase_timeline_pm_cores_func(kbdev, + KBASE_PM_FUNC_ID_RELEASE_CORES_START, + change_gpu_state); + kbase_pm_update_cores_state_nolock(kbdev); + kbase_timeline_pm_cores_func(kbdev, + KBASE_PM_FUNC_ID_RELEASE_CORES_END, + change_gpu_state); + + /* Trace that any state change completed immediately */ + kbase_pm_trace_check_and_finish_state_change(kbdev); + } +} + +KBASE_EXPORT_TEST_API(kbase_pm_release_cores); + +void kbase_pm_request_cores_sync(struct kbase_device *kbdev, + bool tiler_required, + u64 shader_cores) +{ + unsigned long flags; + + kbase_pm_wait_for_poweroff_complete(kbdev); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_request_cores(kbdev, tiler_required, shader_cores); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_pm_check_transitions_sync(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync); + +void kbase_pm_request_l2_caches(struct kbase_device *kbdev) +{ + unsigned long flags; + u32 prior_l2_users_count; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + prior_l2_users_count = kbdev->l2_users_count++; + + KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0); + + /* if the GPU is reset while the l2 is on, l2 will be off but + * prior_l2_users_count will be > 0. l2_available_bitmap will have been + * set to 0 though by kbase_pm_init_hw */ + if (!prior_l2_users_count || !kbdev->l2_available_bitmap) + kbase_pm_check_transitions_nolock(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + wait_event(kbdev->pm.backend.l2_powered_wait, + kbdev->pm.backend.l2_powered == 1); + + /* Trace that any state change completed immediately */ + kbase_pm_trace_check_and_finish_state_change(kbdev); +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches); + +void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbdev->l2_users_count++; +} + +KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on); + +void kbase_pm_release_l2_caches(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0); + + --kbdev->l2_users_count; + + if (!kbdev->l2_users_count) { + kbase_pm_check_transitions_nolock(kbdev); + /* Trace that any state change completed immediately */ + kbase_pm_trace_check_and_finish_state_change(kbdev); + } +} + +KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches); diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h new file mode 100755 index 000000000000..611a90e66e65 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h @@ -0,0 +1,227 @@ +/* + * + * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Power policy API definitions + */ + +#ifndef _KBASE_PM_POLICY_H_ +#define _KBASE_PM_POLICY_H_ + +/** + * kbase_pm_policy_init - Initialize power policy framework + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Must be called before calling any other policy function + * + * Return: 0 if the power policy framework was successfully + * initialized, -errno otherwise. + */ +int kbase_pm_policy_init(struct kbase_device *kbdev); + +/** + * kbase_pm_policy_term - Terminate power policy framework + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_policy_term(struct kbase_device *kbdev); + +/** + * kbase_pm_update_active - Update the active power state of the GPU + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Calls into the current power policy + */ +void kbase_pm_update_active(struct kbase_device *kbdev); + +/** + * kbase_pm_update_cores - Update the desired core state of the GPU + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Calls into the current power policy + */ +void kbase_pm_update_cores(struct kbase_device *kbdev); + + +enum kbase_pm_cores_ready { + KBASE_CORES_NOT_READY = 0, + KBASE_NEW_AFFINITY = 1, + KBASE_CORES_READY = 2 +}; + + +/** + * kbase_pm_request_cores_sync - Synchronous variant of kbase_pm_request_cores() + * + * @kbdev: The kbase device structure for the device + * @tiler_required: true if the tiler is required, false otherwise + * @shader_cores: A bitmask of shader cores which are necessary for the job + * + * When this function returns, the @shader_cores will be in the READY state. + * + * This is safe variant of kbase_pm_check_transitions_sync(): it handles the + * work of ensuring the requested cores will remain powered until a matching + * call to kbase_pm_unrequest_cores()/kbase_pm_release_cores() (as appropriate) + * is made. + */ +void kbase_pm_request_cores_sync(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores); + +/** + * kbase_pm_request_cores - Mark one or more cores as being required + * for jobs to be submitted + * + * @kbdev: The kbase device structure for the device + * @tiler_required: true if the tiler is required, false otherwise + * @shader_cores: A bitmask of shader cores which are necessary for the job + * + * This function is called by the job scheduler to mark one or more cores as + * being required to submit jobs that are ready to run. + * + * The cores requested are reference counted and a subsequent call to + * kbase_pm_register_inuse_cores() or kbase_pm_unrequest_cores() should be + * made to dereference the cores as being 'needed'. + * + * The active power policy will meet or exceed the requirements of the + * requested cores in the system. Any core transitions needed will be begun + * immediately, but they might not complete/the cores might not be available + * until a Power Management IRQ. + * + * Return: 0 if the cores were successfully requested, or -errno otherwise. + */ +void kbase_pm_request_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores); + +/** + * kbase_pm_unrequest_cores - Unmark one or more cores as being required for + * jobs to be submitted. + * + * @kbdev: The kbase device structure for the device + * @tiler_required: true if the tiler is required, false otherwise + * @shader_cores: A bitmask of shader cores (as given to + * kbase_pm_request_cores() ) + * + * This function undoes the effect of kbase_pm_request_cores(). It should be + * used when a job is not going to be submitted to the hardware (e.g. the job is + * cancelled before it is enqueued). + * + * The active power policy will meet or exceed the requirements of the + * requested cores in the system. Any core transitions needed will be begun + * immediately, but they might not complete until a Power Management IRQ. + * + * The policy may use this as an indication that it can power down cores. + */ +void kbase_pm_unrequest_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores); + +/** + * kbase_pm_register_inuse_cores - Register a set of cores as in use by a job + * + * @kbdev: The kbase device structure for the device + * @tiler_required: true if the tiler is required, false otherwise + * @shader_cores: A bitmask of shader cores (as given to + * kbase_pm_request_cores() ) + * + * This function should be called after kbase_pm_request_cores() when the job + * is about to be submitted to the hardware. It will check that the necessary + * cores are available and if so update the 'needed' and 'inuse' bitmasks to + * reflect that the job is now committed to being run. + * + * If the necessary cores are not currently available then the function will + * return %KBASE_CORES_NOT_READY and have no effect. + * + * Return: %KBASE_CORES_NOT_READY if the cores are not immediately ready, + * + * %KBASE_NEW_AFFINITY if the affinity requested is not allowed, + * + * %KBASE_CORES_READY if the cores requested are already available + */ +enum kbase_pm_cores_ready kbase_pm_register_inuse_cores( + struct kbase_device *kbdev, + bool tiler_required, + u64 shader_cores); + +/** + * kbase_pm_release_cores - Release cores after a job has run + * + * @kbdev: The kbase device structure for the device + * @tiler_required: true if the tiler is required, false otherwise + * @shader_cores: A bitmask of shader cores (as given to + * kbase_pm_register_inuse_cores() ) + * + * This function should be called when a job has finished running on the + * hardware. A call to kbase_pm_register_inuse_cores() must have previously + * occurred. The reference counts of the specified cores will be decremented + * which may cause the bitmask of 'inuse' cores to be reduced. The power policy + * may then turn off any cores which are no longer 'inuse'. + */ +void kbase_pm_release_cores(struct kbase_device *kbdev, + bool tiler_required, u64 shader_cores); + +/** + * kbase_pm_request_l2_caches - Request l2 caches + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Request the use of l2 caches for all core groups, power up, wait and prevent + * the power manager from powering down the l2 caches. + * + * This tells the power management that the caches should be powered up, and + * they should remain powered, irrespective of the usage of shader cores. This + * does not return until the l2 caches are powered up. + * + * The caller must call kbase_pm_release_l2_caches() when they are finished + * to allow normal power management of the l2 caches to resume. + * + * This should only be used when power management is active. + */ +void kbase_pm_request_l2_caches(struct kbase_device *kbdev); + +/** + * kbase_pm_request_l2_caches_l2_is_on - Request l2 caches but don't power on + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Increment the count of l2 users but do not attempt to power on the l2 + * + * It is the callers responsibility to ensure that the l2 is already powered up + * and to eventually call kbase_pm_release_l2_caches() + */ +void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev); + +/** + * kbase_pm_request_l2_caches - Release l2 caches + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Release the use of l2 caches for all core groups and allow the power manager + * to power them down when necessary. + * + * This tells the power management that the caches can be powered down if + * necessary, with respect to the usage of shader cores. + * + * The caller must have called kbase_pm_request_l2_caches() prior to a call + * to this. + * + * This should only be used when power management is active. + */ +void kbase_pm_release_l2_caches(struct kbase_device *kbdev); + +#endif /* _KBASE_PM_POLICY_H_ */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c new file mode 100755 index 000000000000..d08c628dd433 --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c @@ -0,0 +1,103 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include + +void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, + u64 *system_time, struct timespec64 *ts) +{ + u32 hi1, hi2; + + kbase_pm_request_gpu_cycle_counter(kbdev); + + /* Read hi, lo, hi to ensure that overflow from lo to hi is handled + * correctly */ + do { + hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), + NULL); + *cycle_counter = kbase_reg_read(kbdev, + GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL); + hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), + NULL); + *cycle_counter |= (((u64) hi1) << 32); + } while (hi1 != hi2); + + /* Read hi, lo, hi to ensure that overflow from lo to hi is handled + * correctly */ + do { + hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), + NULL); + *system_time = kbase_reg_read(kbdev, + GPU_CONTROL_REG(TIMESTAMP_LO), NULL); + hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), + NULL); + *system_time |= (((u64) hi1) << 32); + } while (hi1 != hi2); + + /* Record the CPU's idea of current time */ + ktime_get_raw_ts64(ts); + + kbase_pm_release_gpu_cycle_counter(kbdev); +} + +/** + * kbase_wait_write_flush - Wait for GPU write flush + * @kctx: Context pointer + * + * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush + * its write buffer. + * + * Only in use for BASE_HW_ISSUE_6367 + * + * Note : If GPU resets occur then the counters are reset to zero, the delay may + * not be as expected. + */ +#ifndef CONFIG_MALI_NO_MALI +void kbase_wait_write_flush(struct kbase_context *kctx) +{ + u32 base_count = 0; + + /* + * The caller must be holding onto the kctx or the call is from + * userspace. + */ + kbase_pm_context_active(kctx->kbdev); + kbase_pm_request_gpu_cycle_counter(kctx->kbdev); + + while (true) { + u32 new_count; + + new_count = kbase_reg_read(kctx->kbdev, + GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL); + /* First time around, just store the count. */ + if (base_count == 0) { + base_count = new_count; + continue; + } + + /* No need to handle wrapping, unsigned maths works for this. */ + if ((new_count - base_count) > 1000) + break; + } + + kbase_pm_release_gpu_cycle_counter(kctx->kbdev); + kbase_pm_context_idle(kctx->kbdev); +} +#endif /* CONFIG_MALI_NO_MALI */ diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.h new file mode 100755 index 000000000000..433aa4b9cb5e --- /dev/null +++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.h @@ -0,0 +1,52 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_BACKEND_TIME_H_ +#define _KBASE_BACKEND_TIME_H_ + +/** + * kbase_backend_get_gpu_time() - Get current GPU time + * @kbdev: Device pointer + * @cycle_counter: Pointer to u64 to store cycle counter in + * @system_time: Pointer to u64 to store system time in + * @ts: Pointer to struct timespec64 to store current monotonic + * time in + */ +void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, + u64 *system_time, struct timespec64 *ts); + +/** + * kbase_wait_write_flush() - Wait for GPU write flush + * @kctx: Context pointer + * + * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush + * its write buffer. + * + * If GPU resets occur then the counters are reset to zero, the delay may not be + * as expected. + * + * This function is only in use for BASE_HW_ISSUE_6367 + */ +#ifdef CONFIG_MALI_NO_MALI +static inline void kbase_wait_write_flush(struct kbase_context *kctx) +{ +} +#else +void kbase_wait_write_flush(struct kbase_context *kctx); +#endif + +#endif /* _KBASE_BACKEND_TIME_H_ */ diff --git a/drivers/gpu/arm/midgard/docs/Doxyfile b/drivers/gpu/arm/midgard/docs/Doxyfile new file mode 100755 index 000000000000..35ff2f1ce4a0 --- /dev/null +++ b/drivers/gpu/arm/midgard/docs/Doxyfile @@ -0,0 +1,126 @@ +# +# (C) COPYRIGHT 2011-2013, 2015 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + +############################################################################## + +# This file contains per-module Doxygen configuration. Please do not add +# extra settings to this file without consulting all stakeholders, as they +# may cause override project-wide settings. +# +# Additionally, when defining aliases, macros, sections etc, use the module +# name as a prefix e.g. gles_my_alias. + +############################################################################## + +@INCLUDE = ../../bldsys/Doxyfile_common + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT += ../../kernel/drivers/gpu/arm/midgard/ + +############################################################################## +# Everything below here is optional, and in most cases not required +############################################################################## + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES += + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS += + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS += + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +EXCLUDE += ../../kernel/drivers/gpu/arm/midgard/platform ../../kernel/drivers/gpu/arm/midgard/platform_dummy ../../kernel/drivers/gpu/arm/midgard/scripts ../../kernel/drivers/gpu/arm/midgard/tests ../../kernel/drivers/gpu/arm/midgard/Makefile ../../kernel/drivers/gpu/arm/midgard/Makefile.kbase ../../kernel/drivers/gpu/arm/midgard/Kbuild ../../kernel/drivers/gpu/arm/midgard/Kconfig ../../kernel/drivers/gpu/arm/midgard/sconscript ../../kernel/drivers/gpu/arm/midgard/docs ../../kernel/drivers/gpu/arm/midgard/pm_test_script.sh ../../kernel/drivers/gpu/arm/midgard/mali_uk.h ../../kernel/drivers/gpu/arm/midgard/Makefile + + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS += + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS += + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH += + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH += + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH += + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED += + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED += + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS += ../../kernel/drivers/gpu/arm/midgard/docs + diff --git a/drivers/gpu/arm/midgard/docs/policy_operation_diagram.dot b/drivers/gpu/arm/midgard/docs/policy_operation_diagram.dot new file mode 100755 index 000000000000..7ae05c2f8ded --- /dev/null +++ b/drivers/gpu/arm/midgard/docs/policy_operation_diagram.dot @@ -0,0 +1,112 @@ +/* + * + * (C) COPYRIGHT 2010 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +digraph policy_objects_diagram { + rankdir=LR; + size="12,8"; + compound=true; + + node [ shape = box ]; + + subgraph cluster_policy_queues { + low_queue [ shape=record label = "LowP | {ctx_lo | ... | ctx_i | ... | ctx_hi}" ]; + queues_middle_sep [ label="" shape=plaintext width=0 height=0 ]; + + rt_queue [ shape=record label = "RT | {ctx_lo | ... | ctx_j | ... | ctx_hi}" ]; + + label = "Policy's Queue(s)"; + } + + call_enqueue [ shape=plaintext label="enqueue_ctx()" ]; + + { + rank=same; + ordering=out; + call_dequeue [ shape=plaintext label="dequeue_head_ctx()\n+ runpool_add_ctx()" ]; + call_ctxfinish [ shape=plaintext label="runpool_remove_ctx()" ]; + + call_ctxdone [ shape=plaintext label="don't requeue;\n/* ctx has no more jobs */" ]; + } + + subgraph cluster_runpool { + + as0 [ width=2 height = 0.25 label="AS0: Job_1, ..., Job_n" ]; + as1 [ width=2 height = 0.25 label="AS1: Job_1, ..., Job_m" ]; + as2 [ width=2 height = 0.25 label="AS2: Job_1, ..., Job_p" ]; + as3 [ width=2 height = 0.25 label="AS3: Job_1, ..., Job_q" ]; + + label = "Policy's Run Pool"; + } + + { + rank=same; + call_jdequeue [ shape=plaintext label="dequeue_job()" ]; + sstop_dotfixup [ shape=plaintext label="" width=0 height=0 ]; + } + + { + rank=same; + ordering=out; + sstop [ shape=ellipse label="SS-Timer expires" ] + jobslots [ shape=record label="Jobslots: | <0>js[0] | <1>js[1] | <2>js[2]" ]; + + irq [ label="IRQ" shape=ellipse ]; + + job_finish [ shape=plaintext label="don't requeue;\n/* job done */" ]; + } + + hstop [ shape=ellipse label="HS-Timer expires" ] + + /* + * Edges + */ + + call_enqueue -> queues_middle_sep [ lhead=cluster_policy_queues ]; + + low_queue:qr -> call_dequeue:w; + rt_queue:qr -> call_dequeue:w; + + call_dequeue -> as1 [lhead=cluster_runpool]; + + as1->call_jdequeue [ltail=cluster_runpool]; + call_jdequeue->jobslots:0; + call_jdequeue->sstop_dotfixup [ arrowhead=none]; + sstop_dotfixup->sstop [label="Spawn SS-Timer"]; + sstop->jobslots [label="SoftStop"]; + sstop->hstop [label="Spawn HS-Timer"]; + hstop->jobslots:ne [label="HardStop"]; + + + as3->call_ctxfinish:ne [ ltail=cluster_runpool ]; + call_ctxfinish:sw->rt_queue:qm [ lhead=cluster_policy_queues label="enqueue_ctx()\n/* ctx still has jobs */" ]; + + call_ctxfinish->call_ctxdone [constraint=false]; + + call_ctxdone->call_enqueue [weight=0.1 labeldistance=20.0 labelangle=0.0 taillabel="Job submitted to the ctx" style=dotted constraint=false]; + + + { + jobslots->irq [constraint=false]; + + irq->job_finish [constraint=false]; + } + + irq->as2 [lhead=cluster_runpool label="requeue_job()\n/* timeslice expired */" ]; + +} diff --git a/drivers/gpu/arm/midgard/docs/policy_overview.dot b/drivers/gpu/arm/midgard/docs/policy_overview.dot new file mode 100755 index 000000000000..159b993b7d61 --- /dev/null +++ b/drivers/gpu/arm/midgard/docs/policy_overview.dot @@ -0,0 +1,63 @@ +/* + * + * (C) COPYRIGHT 2010 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +digraph policy_objects_diagram { + rankdir=LR + size="6,6" + compound=true; + + node [ shape = box ]; + + call_enqueue [ shape=plaintext label="enqueue ctx" ]; + + + policy_queue [ label="Policy's Queue" ]; + + { + rank=same; + runpool [ label="Policy's Run Pool" ]; + + ctx_finish [ label="ctx finished" ]; + } + + { + rank=same; + jobslots [ shape=record label="Jobslots: | <0>js[0] | <1>js[1] | <2>js[2]" ]; + + job_finish [ label="Job finished" ]; + } + + + + /* + * Edges + */ + + call_enqueue -> policy_queue; + + policy_queue->runpool [label="dequeue ctx" weight=0.1]; + runpool->policy_queue [label="requeue ctx" weight=0.1]; + + runpool->ctx_finish [ style=dotted ]; + + runpool->jobslots [label="dequeue job" weight=0.1]; + jobslots->runpool [label="requeue job" weight=0.1]; + + jobslots->job_finish [ style=dotted ]; +} diff --git a/drivers/gpu/arm/midgard/ipa/Kbuild b/drivers/gpu/arm/midgard/ipa/Kbuild new file mode 100755 index 000000000000..602b15f5225c --- /dev/null +++ b/drivers/gpu/arm/midgard/ipa/Kbuild @@ -0,0 +1,24 @@ +# +# (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +midgard_kbase-y += \ + ipa/mali_kbase_ipa_simple.o \ + ipa/mali_kbase_ipa.o + +midgard_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o + +ifneq ($(wildcard $(src)/ipa/mali_kbase_ipa_tmix.c),) + midgard_kbase-y += ipa/mali_kbase_ipa_tmix.o +endif diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c new file mode 100755 index 000000000000..01bdbb4e8eb1 --- /dev/null +++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c @@ -0,0 +1,585 @@ +/* + * + * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +#include +#include +#include +#include "mali_kbase.h" +#include "mali_kbase_ipa.h" +#include "mali_kbase_ipa_debugfs.h" + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) +#include +#else +#include +#define dev_pm_opp_find_freq_exact opp_find_freq_exact +#define dev_pm_opp_get_voltage opp_get_voltage +#define dev_pm_opp opp +#endif +#include + +#define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model" + +static struct kbase_ipa_model_ops *kbase_ipa_all_model_ops[] = { + &kbase_simple_ipa_model_ops, +}; + +int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) +{ + int err = 0; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (model->ops->recalculate) { + err = model->ops->recalculate(model); + if (err) { + dev_err(model->kbdev->dev, + "recalculation of power model %s returned error %d\n", + model->ops->name, err); + } + } + + return err; +} + +static struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, + const char *name) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(kbase_ipa_all_model_ops); ++i) { + struct kbase_ipa_model_ops *ops = kbase_ipa_all_model_ops[i]; + + if (!strcmp(ops->name, name)) + return ops; + } + + dev_err(kbdev->dev, "power model \'%s\' not found\n", name); + + return NULL; +} + +void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) +{ + atomic_set(&kbdev->ipa_use_configured_model, false); +} + +void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) +{ + atomic_set(&kbdev->ipa_use_configured_model, true); +} + +const char *kbase_ipa_model_name_from_id(u32 gpu_id) +{ + const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + if (GPU_ID_IS_NEW_FORMAT(prod_id)) { + switch (GPU_ID2_MODEL_MATCH_VALUE(prod_id)) { + case GPU_ID2_PRODUCT_TMIX: + return KBASE_IPA_FALLBACK_MODEL_NAME; + default: + return KBASE_IPA_FALLBACK_MODEL_NAME; + } + } + + return KBASE_IPA_FALLBACK_MODEL_NAME; +} + +static struct device_node *get_model_dt_node(struct kbase_ipa_model *model) +{ + struct device_node *model_dt_node; + char compat_string[64]; + + snprintf(compat_string, sizeof(compat_string), "arm,%s", + model->ops->name); + + model_dt_node = of_find_compatible_node(model->kbdev->dev->of_node, + NULL, compat_string); + if (!model_dt_node && !model->missing_dt_node_warning) { + dev_warn(model->kbdev->dev, + "Couldn't find power_model DT node matching \'%s\'\n", + compat_string); + model->missing_dt_node_warning = true; + } + + return model_dt_node; +} + +int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, + const char *name, s32 *addr, + size_t num_elems, bool dt_required) +{ + int err, i; + struct device_node *model_dt_node = get_model_dt_node(model); + char *origin; + + err = of_property_read_u32_array(model_dt_node, name, addr, num_elems); + + if (err && dt_required) { + memset(addr, 0, sizeof(s32) * num_elems); + dev_warn(model->kbdev->dev, + "Error %d, no DT entry: %s.%s = %zu*[0]\n", + err, model->ops->name, name, num_elems); + origin = "zero"; + } else if (err && !dt_required) { + origin = "default"; + } else /* !err */ { + origin = "DT"; + } + + /* Create a unique debugfs entry for each element */ + for (i = 0; i < num_elems; ++i) { + char elem_name[32]; + + if (num_elems == 1) + snprintf(elem_name, sizeof(elem_name), "%s", name); + else + snprintf(elem_name, sizeof(elem_name), "%s.%d", + name, i); + + dev_dbg(model->kbdev->dev, "%s.%s = %d (%s)\n", + model->ops->name, elem_name, addr[i], origin); + + err = kbase_ipa_model_param_add(model, elem_name, + &addr[i], sizeof(s32), + PARAM_TYPE_S32); + if (err) + goto exit; + } +exit: + return err; +} + +int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, + const char *name, char *addr, + size_t size, bool dt_required) +{ + int err; + struct device_node *model_dt_node = get_model_dt_node(model); + const char *string_prop_value; + char *origin; + + err = of_property_read_string(model_dt_node, name, + &string_prop_value); + if (err && dt_required) { + strncpy(addr, "", size - 1); + dev_warn(model->kbdev->dev, + "Error %d, no DT entry: %s.%s = \'%s\'\n", + err, model->ops->name, name, addr); + err = 0; + origin = "zero"; + } else if (err && !dt_required) { + origin = "default"; + } else /* !err */ { + strncpy(addr, string_prop_value, size - 1); + origin = "DT"; + } + + addr[size - 1] = '\0'; + + dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (%s)\n", + model->ops->name, name, string_prop_value, origin); + + err = kbase_ipa_model_param_add(model, name, addr, size, + PARAM_TYPE_STRING); + + return err; +} + +void kbase_ipa_term_model(struct kbase_ipa_model *model) +{ + if (!model) + return; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (model->ops->term) + model->ops->term(model); + + kbase_ipa_model_param_free_all(model); + + kfree(model); +} +KBASE_EXPORT_TEST_API(kbase_ipa_term_model); + +struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, + struct kbase_ipa_model_ops *ops) +{ + struct kbase_ipa_model *model; + int err; + + lockdep_assert_held(&kbdev->ipa.lock); + + if (!ops || !ops->name) + return NULL; + + model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL); + if (!model) + return NULL; + + model->kbdev = kbdev; + model->ops = ops; + INIT_LIST_HEAD(&model->params); + + err = model->ops->init(model); + if (err) { + dev_err(kbdev->dev, + "init of power model \'%s\' returned error %d\n", + ops->name, err); + goto term_model; + } + + err = kbase_ipa_model_recalculate(model); + if (err) + goto term_model; + + return model; + +term_model: + kbase_ipa_term_model(model); + return NULL; +} +KBASE_EXPORT_TEST_API(kbase_ipa_init_model); + +static void kbase_ipa_term_locked(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->ipa.lock); + + /* Clean up the models */ + if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) + kbase_ipa_term_model(kbdev->ipa.configured_model); + kbase_ipa_term_model(kbdev->ipa.fallback_model); + + kbdev->ipa.configured_model = NULL; + kbdev->ipa.fallback_model = NULL; +} + +int kbase_ipa_init(struct kbase_device *kbdev) +{ + + const char *model_name; + struct kbase_ipa_model_ops *ops; + struct kbase_ipa_model *default_model = NULL; + int err; + + mutex_init(&kbdev->ipa.lock); + /* + * Lock during init to avoid warnings from lockdep_assert_held (there + * shouldn't be any concurrent access yet). + */ + mutex_lock(&kbdev->ipa.lock); + + /* The simple IPA model must *always* be present.*/ + ops = kbase_ipa_model_ops_find(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME); + + if (!ops->do_utilization_scaling_in_framework) { + dev_err(kbdev->dev, + "Fallback IPA model %s should not account for utilization\n", + ops->name); + err = -EINVAL; + goto end; + } + + default_model = kbase_ipa_init_model(kbdev, ops); + if (!default_model) { + err = -EINVAL; + goto end; + } + + kbdev->ipa.fallback_model = default_model; + err = of_property_read_string(kbdev->dev->of_node, + "ipa-model", + &model_name); + if (err) { + /* Attempt to load a match from GPU-ID */ + u32 gpu_id; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + model_name = kbase_ipa_model_name_from_id(gpu_id); + dev_dbg(kbdev->dev, + "Inferring model from GPU ID 0x%x: \'%s\'\n", + gpu_id, model_name); + } else { + dev_dbg(kbdev->dev, + "Using ipa-model parameter from DT: \'%s\'\n", + model_name); + } + + if (strcmp(KBASE_IPA_FALLBACK_MODEL_NAME, model_name) != 0) { + ops = kbase_ipa_model_ops_find(kbdev, model_name); + kbdev->ipa.configured_model = kbase_ipa_init_model(kbdev, ops); + if (!kbdev->ipa.configured_model) { + err = -EINVAL; + goto end; + } + } else { + kbdev->ipa.configured_model = default_model; + err = 0; + } + + kbase_ipa_model_use_configured_locked(kbdev); + +end: + if (err) + kbase_ipa_term_locked(kbdev); + else + dev_info(kbdev->dev, + "Using configured power model %s, and fallback %s\n", + kbdev->ipa.configured_model->ops->name, + kbdev->ipa.fallback_model->ops->name); + + mutex_unlock(&kbdev->ipa.lock); + return err; +} +KBASE_EXPORT_TEST_API(kbase_ipa_init); + +void kbase_ipa_term(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->ipa.lock); + kbase_ipa_term_locked(kbdev); + mutex_unlock(&kbdev->ipa.lock); +} +KBASE_EXPORT_TEST_API(kbase_ipa_term); + +/** + * kbase_scale_dynamic_power() - Scale a dynamic power coefficient to an OPP + * @c: Dynamic model coefficient, in pW/(Hz V^2). Should be in range + * 0 < c < 2^26 to prevent overflow. + * @freq: Frequency, in Hz. Range: 2^23 < freq < 2^30 (~8MHz to ~1GHz) + * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) + * + * Keep a record of the approximate range of each value at every stage of the + * calculation, to ensure we don't overflow. This makes heavy use of the + * approximations 1000 = 2^10 and 1000000 = 2^20, but does the actual + * calculations in decimal for increased accuracy. + * + * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) + */ +static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq, + const u32 voltage) +{ + /* Range: 2^8 < v2 < 2^16 m(V^2) */ + const u32 v2 = (voltage * voltage) / 1000; + + /* Range: 2^3 < f_MHz < 2^10 MHz */ + const u32 f_MHz = freq / 1000000; + + /* Range: 2^11 < v2f_big < 2^26 kHz V^2 */ + const u32 v2f_big = v2 * f_MHz; + + /* Range: 2^1 < v2f < 2^16 MHz V^2 */ + const u32 v2f = v2f_big / 1000; + + /* Range (working backwards from next line): 0 < v2fc < 2^23 uW. + * Must be < 2^42 to avoid overflowing the return value. */ + const u64 v2fc = (u64) c * (u64) v2f; + u32 remainder; + + /* Range: 0 < v2fc / 1000 < 2^13 mW */ + // static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) + return div_u64_rem(v2fc, 1000, &remainder); +} + +/** + * kbase_scale_static_power() - Scale a static power coefficient to an OPP + * @c: Static model coefficient, in uW/V^3. Should be in range + * 0 < c < 2^32 to prevent overflow. + * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) + * + * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) + */ +u32 kbase_scale_static_power(const u32 c, const u32 voltage) +{ + /* Range: 2^8 < v2 < 2^16 m(V^2) */ + const u32 v2 = (voltage * voltage) / 1000; + + /* Range: 2^17 < v3_big < 2^29 m(V^2) mV */ + const u32 v3_big = v2 * voltage; + + /* Range: 2^7 < v3 < 2^19 m(V^3) */ + const u32 v3 = v3_big / 1000; + + /* + * Range (working backwards from next line): 0 < v3c_big < 2^33 nW. + * The result should be < 2^52 to avoid overflowing the return value. + */ + const u64 v3c_big = (u64) c * (u64) v3; + u32 remainder; + + /* Range: 0 < v3c_big / 1000000 < 2^13 mW */ + // return v3c_big / 1000000; + return div_u64_rem(v3c_big, 1000000, &remainder); +} + +static struct kbase_ipa_model *get_current_model(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->ipa.lock); + + if (atomic_read(&kbdev->ipa_use_configured_model)) + return kbdev->ipa.configured_model; + else + return kbdev->ipa.fallback_model; +} + +static u32 get_static_power_locked(struct kbase_device *kbdev, + struct kbase_ipa_model *model, + unsigned long voltage) +{ + u32 power = 0; + int err; + u32 power_coeff; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + if (!model->ops->get_static_coeff) + model = kbdev->ipa.fallback_model; + + if (model->ops->get_static_coeff) { + err = model->ops->get_static_coeff(model, &power_coeff); + if (!err) + power = kbase_scale_static_power(power_coeff, + (u32) voltage); + } + + return power; +} + +#ifdef CONFIG_MALI_PWRSOFT_765 +static unsigned long kbase_get_static_power(struct devfreq *df, + unsigned long voltage) +#else +static unsigned long kbase_get_static_power(unsigned long voltage) +#endif +{ + struct kbase_ipa_model *model; + u32 power = 0; +#ifdef CONFIG_MALI_PWRSOFT_765 + struct kbase_device *kbdev = dev_get_drvdata(&df->dev); +#else + struct kbase_device *kbdev = kbase_find_device(-1); +#endif + + mutex_lock(&kbdev->ipa.lock); + + model = get_current_model(kbdev); + power = get_static_power_locked(kbdev, model, voltage); + + mutex_unlock(&kbdev->ipa.lock); + +#ifndef CONFIG_MALI_PWRSOFT_765 + kbase_release_device(kbdev); +#endif + + return power; +} + +#ifdef CONFIG_MALI_PWRSOFT_765 +static unsigned long kbase_get_dynamic_power(struct devfreq *df, + unsigned long freq, + unsigned long voltage) +#else +static unsigned long kbase_get_dynamic_power(unsigned long freq, + unsigned long voltage) +#endif +{ + struct kbase_ipa_model *model; + u32 power_coeff = 0, power = 0; + int err = 0; +#ifdef CONFIG_MALI_PWRSOFT_765 + struct kbase_device *kbdev = dev_get_drvdata(&df->dev); +#else + struct kbase_device *kbdev = kbase_find_device(-1); +#endif + + mutex_lock(&kbdev->ipa.lock); + + model = kbdev->ipa.fallback_model; + + err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); + + if (!err) + power = kbase_scale_dynamic_power(power_coeff, freq, voltage); + else + dev_err_ratelimited(kbdev->dev, + "Model %s returned error code %d\n", + model->ops->name, err); + + mutex_unlock(&kbdev->ipa.lock); + +#ifndef CONFIG_MALI_PWRSOFT_765 + kbase_release_device(kbdev); +#endif + + return power; +} + +int kbase_get_real_power(struct devfreq *df, u32 *power, + unsigned long freq, + unsigned long voltage) +{ + struct kbase_ipa_model *model; + u32 power_coeff = 0; + int err = 0; + struct kbase_device *kbdev = dev_get_drvdata(&df->dev); + + mutex_lock(&kbdev->ipa.lock); + + model = get_current_model(kbdev); + + err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); + + /* If we switch to protected model between get_current_model() and + * get_dynamic_coeff(), counter reading could fail. If that happens + * (unlikely, but possible), revert to the fallback model. */ + if (err && model != kbdev->ipa.fallback_model) { + model = kbdev->ipa.fallback_model; + err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); + } + + if (err) + goto exit_unlock; + + *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); + + if (model->ops->do_utilization_scaling_in_framework) { + struct devfreq_dev_status *status = &df->last_status; + unsigned long total_time = max(status->total_time, 1ul); + u64 busy_time = min(status->busy_time, total_time); + u32 remainder; + + // *power = ((u64) *power * (u64) busy_time) / total_time; + *power = div_u64_rem(((u64) *power * (u64) busy_time), total_time, &remainder); + } + + *power += get_static_power_locked(kbdev, model, voltage); + +exit_unlock: + mutex_unlock(&kbdev->ipa.lock); + + return err; +} +KBASE_EXPORT_TEST_API(kbase_get_real_power); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +struct devfreq_cooling_ops kbase_ipa_power_model_ops = { +#else +struct devfreq_cooling_power kbase_ipa_power_model_ops = { +#endif + .get_static_power = &kbase_get_static_power, + .get_dynamic_power = &kbase_get_dynamic_power, +}; +KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops); diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h new file mode 100755 index 000000000000..b2d3db149579 --- /dev/null +++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h @@ -0,0 +1,148 @@ +/* + * + * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_IPA_H_ +#define _KBASE_IPA_H_ + +#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) + +struct devfreq; + +struct kbase_ipa_model { + struct list_head link; + struct kbase_device *kbdev; + void *model_data; + struct kbase_ipa_model_ops *ops; + struct list_head params; + bool missing_dt_node_warning; +}; + +/** + * kbase_ipa_model_add_param_s32 - Add an integer model parameter + * @model: pointer to IPA model + * @name: name of corresponding debugfs entry + * @addr: address where the value is stored + * @num_elems: number of elements (1 if not an array) + * @dt_required: if false, a corresponding devicetree entry is not required, + * and the current value will be used. If true, a warning is + * output and the data is zeroed + * + * Return: 0 on success, or an error code + */ +int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, + const char *name, s32 *addr, + size_t num_elems, bool dt_required); + +/** + * kbase_ipa_model_add_param_string - Add a string model parameter + * @model: pointer to IPA model + * @name: name of corresponding debugfs entry + * @addr: address where the value is stored + * @size: size, in bytes, of the value storage (so the maximum string + * length is size - 1) + * @dt_required: if false, a corresponding devicetree entry is not required, + * and the current value will be used. If true, a warning is + * output and the data is zeroed + * + * Return: 0 on success, or an error code + */ +int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, + const char *name, char *addr, + size_t size, bool dt_required); + +struct kbase_ipa_model_ops { + char *name; + /* The init, recalculate and term ops on the default model are always + * called. However, all the other models are only invoked if the model + * is selected in the device tree. Otherwise they are never + * initialized. Additional resources can be acquired by models in + * init(), however they must be terminated in the term(). + */ + int (*init)(struct kbase_ipa_model *model); + /* Called immediately after init(), or when a parameter is changed, so + * that any coefficients derived from model parameters can be + * recalculated. */ + int (*recalculate)(struct kbase_ipa_model *model); + void (*term)(struct kbase_ipa_model *model); + /* + * get_dynamic_coeff() - calculate dynamic power coefficient + * @model: pointer to model + * @coeffp: pointer to return value location + * @current_freq: frequency the GPU has been running at for the + * previous sampling period. + * + * Calculate a dynamic power coefficient, with units pW/(Hz V^2), which + * is then scaled by the IPA framework according to the current OPP's + * frequency and voltage. + * + * Return: 0 on success, or an error code. + */ + int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp, + u32 current_freq); + /* + * get_static_coeff() - calculate static power coefficient + * @model: pointer to model + * @coeffp: pointer to return value location + * + * Calculate a static power coefficient, with units uW/(V^3), which is + * scaled by the IPA framework according to the current OPP's voltage. + * + * Return: 0 on success, or an error code. + */ + int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); + /* If false, the model's get_dynamic_coeff() method accounts for how + * long the GPU was active over the sample period. If true, the + * framework will scale the calculated power according to the + * utilization stats recorded by devfreq in get_real_power(). */ + bool do_utilization_scaling_in_framework; +}; + +/* Models can be registered only in the platform's platform_init_func call */ +int kbase_ipa_model_ops_register(struct kbase_device *kbdev, + struct kbase_ipa_model_ops *new_model_ops); +struct kbase_ipa_model *kbase_ipa_get_model(struct kbase_device *kbdev, + const char *name); + +int kbase_ipa_init(struct kbase_device *kbdev); +void kbase_ipa_term(struct kbase_device *kbdev); +void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev); +void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev); +int kbase_ipa_model_recalculate(struct kbase_ipa_model *model); +struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, + struct kbase_ipa_model_ops *ops); +void kbase_ipa_term_model(struct kbase_ipa_model *model); + +extern struct kbase_ipa_model_ops kbase_simple_ipa_model_ops; + + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +extern struct devfreq_cooling_ops kbase_ipa_power_model_ops; +#else +extern struct devfreq_cooling_power kbase_ipa_power_model_ops; +#endif + +#else /* !(defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ + +static inline void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) +{ } + +static inline void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) +{ } + +#endif /* (defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ + +#endif diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c new file mode 100755 index 000000000000..eafc14009ddc --- /dev/null +++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c @@ -0,0 +1,219 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include + +#include "mali_kbase.h" +#include "mali_kbase_ipa.h" +#include "mali_kbase_ipa_debugfs.h" + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)) +#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE +#endif + +struct kbase_ipa_model_param { + char *name; + union { + void *voidp; + s32 *s32p; + char *str; + } addr; + size_t size; + enum kbase_ipa_model_param_type type; + struct kbase_ipa_model *model; + struct list_head link; +}; + +static int param_int_get(void *data, u64 *val) +{ + struct kbase_ipa_model_param *param = data; + + mutex_lock(¶m->model->kbdev->ipa.lock); + *(s64 *) val = *param->addr.s32p; + mutex_unlock(¶m->model->kbdev->ipa.lock); + + return 0; +} + +static int param_int_set(void *data, u64 val) +{ + struct kbase_ipa_model_param *param = data; + struct kbase_ipa_model *model = param->model; + s64 sval = (s64) val; + int err = 0; + + if (sval < S32_MIN || sval > S32_MAX) + return -ERANGE; + + mutex_lock(¶m->model->kbdev->ipa.lock); + *param->addr.s32p = val; + err = kbase_ipa_model_recalculate(model); + mutex_unlock(¶m->model->kbdev->ipa.lock); + + return err; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_s32, param_int_get, param_int_set, "%lld\n"); + +static ssize_t param_string_get(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct kbase_ipa_model_param *param = file->private_data; + ssize_t ret; + size_t len; + + mutex_lock(¶m->model->kbdev->ipa.lock); + len = strnlen(param->addr.str, param->size - 1) + 1; + ret = simple_read_from_buffer(user_buf, count, ppos, + param->addr.str, len); + mutex_unlock(¶m->model->kbdev->ipa.lock); + + return ret; +} + +static ssize_t param_string_set(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct kbase_ipa_model_param *param = file->private_data; + struct kbase_ipa_model *model = param->model; + ssize_t ret = count; + size_t buf_size; + int err; + + mutex_lock(&model->kbdev->ipa.lock); + + if (count > param->size) { + ret = -EINVAL; + goto end; + } + + buf_size = min(param->size - 1, count); + if (copy_from_user(param->addr.str, user_buf, buf_size)) { + ret = -EFAULT; + goto end; + } + + param->addr.str[buf_size] = '\0'; + + err = kbase_ipa_model_recalculate(model); + if (err < 0) + ret = err; + +end: + mutex_unlock(&model->kbdev->ipa.lock); + + return ret; +} + +static const struct file_operations fops_string = { + .read = param_string_get, + .write = param_string_set, + .open = simple_open, + .llseek = default_llseek, +}; + +int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, + void *addr, size_t size, + enum kbase_ipa_model_param_type type) +{ + struct kbase_ipa_model_param *param; + + param = kzalloc(sizeof(*param), GFP_KERNEL); + + if (!param) + return -ENOMEM; + + /* 'name' is stack-allocated for array elements, so copy it into + * heap-allocated storage */ + param->name = kstrdup(name, GFP_KERNEL); + param->addr.voidp = addr; + param->size = size; + param->type = type; + param->model = model; + + list_add(¶m->link, &model->params); + + return 0; +} + +void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_param *param_p, *param_n; + + list_for_each_entry_safe(param_p, param_n, &model->params, link) { + list_del(¶m_p->link); + kfree(param_p->name); + kfree(param_p); + } +} + +static void kbase_ipa_model_debugfs_init(struct kbase_ipa_model *model) +{ + struct list_head *it; + struct dentry *dir; + + lockdep_assert_held(&model->kbdev->ipa.lock); + + dir = debugfs_create_dir(model->ops->name, + model->kbdev->mali_debugfs_directory); + + if (!dir) { + dev_err(model->kbdev->dev, + "Couldn't create mali debugfs %s directory", + model->ops->name); + return; + } + + list_for_each(it, &model->params) { + struct kbase_ipa_model_param *param = + list_entry(it, + struct kbase_ipa_model_param, + link); + const struct file_operations *fops = NULL; + + switch (param->type) { + case PARAM_TYPE_S32: + fops = &fops_s32; + break; + case PARAM_TYPE_STRING: + fops = &fops_string; + break; + } + + if (unlikely(!fops)) { + dev_err(model->kbdev->dev, + "Type not set for %s parameter %s\n", + model->ops->name, param->name); + } else { + debugfs_create_file(param->name, S_IRUGO | S_IWUSR, + dir, param, fops); + } + } +} + +void kbase_ipa_debugfs_init(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->ipa.lock); + + if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) + kbase_ipa_model_debugfs_init(kbdev->ipa.configured_model); + kbase_ipa_model_debugfs_init(kbdev->ipa.fallback_model); + + mutex_unlock(&kbdev->ipa.lock); +} diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h new file mode 100755 index 000000000000..ec06e2096f94 --- /dev/null +++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h @@ -0,0 +1,49 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_IPA_DEBUGFS_H_ +#define _KBASE_IPA_DEBUGFS_H_ + +enum kbase_ipa_model_param_type { + PARAM_TYPE_S32 = 1, + PARAM_TYPE_STRING, +}; + +#ifdef CONFIG_DEBUG_FS + +void kbase_ipa_debugfs_init(struct kbase_device *kbdev); +int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, + void *addr, size_t size, + enum kbase_ipa_model_param_type type); +void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model); + +#else /* CONFIG_DEBUG_FS */ + +static inline int kbase_ipa_model_param_add(struct kbase_ipa_model *model, + const char *name, void *addr, + size_t size, + enum kbase_ipa_model_param_type type) +{ + return 0; +} + +static inline void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) +{ } + +#endif /* CONFIG_DEBUG_FS */ + +#endif /* _KBASE_IPA_DEBUGFS_H_ */ diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c new file mode 100755 index 000000000000..da0a4d4a0e7e --- /dev/null +++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c @@ -0,0 +1,222 @@ +/* + * + * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif +#include +#include + +#include "mali_kbase.h" +#include "mali_kbase_defs.h" + +/* + * This model is primarily designed for the Juno platform. It may not be + * suitable for other platforms. The additional resources in this model + * should preferably be minimal, as this model is rarely used when a dynamic + * model is available. + */ + +/** + * struct kbase_ipa_model_simple_data - IPA context per device + * @dynamic_coefficient: dynamic coefficient of the model + * @static_coefficient: static coefficient of the model + * @ts: Thermal scaling coefficients of the model + * @tz_name: Thermal zone name + * @gpu_tz: thermal zone device + */ + +struct kbase_ipa_model_simple_data { + u32 dynamic_coefficient; + u32 static_coefficient; + s32 ts[4]; + char tz_name[16]; + struct thermal_zone_device *gpu_tz; +}; +#define FALLBACK_STATIC_TEMPERATURE 55000 + +/** + * calculate_temp_scaling_factor() - Calculate temperature scaling coefficient + * @ts: Signed coefficients, in order t^0 to t^3, with units Deg^-N + * @t: Temperature, in mDeg C. Range: -2^17 < t < 2^17 + * + * Scale the temperature according to a cubic polynomial whose coefficients are + * provided in the device tree. The result is used to scale the static power + * coefficient, where 1000000 means no change. + * + * Return: Temperature scaling factor. Approx range 0 < ret < 10,000,000. + */ +static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t) +{ + /* Range: -2^24 < t2 < 2^24 m(Deg^2) */ + u32 remainder; + // static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder) + const s64 t2 = div_s64_rem((t * t), 1000, &remainder); + + /* Range: -2^31 < t3 < 2^31 m(Deg^3) */ + const s64 t3 = div_s64_rem((t * t2), 1000, &remainder); + + /* + * Sum the parts. t^[1-3] are in m(Deg^N), but the coefficients are in + * Deg^-N, so we need to multiply the last coefficient by 1000. + * Range: -2^63 < res_big < 2^63 + */ + const s64 res_big = ts[3] * t3 /* +/- 2^62 */ + + ts[2] * t2 /* +/- 2^55 */ + + ts[1] * t /* +/- 2^48 */ + + ts[0] * 1000; /* +/- 2^41 */ + + /* Range: -2^60 < res_unclamped < 2^60 */ + s64 res_unclamped = div_s64_rem(res_big, 1000, &remainder); + + /* Clamp to range of 0x to 10x the static power */ + return clamp(res_unclamped, (s64) 0, (s64) 10000000); +} + +static int model_static_coeff(struct kbase_ipa_model *model, u32 *coeffp) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) + unsigned long temp; +#else + int temp; +#endif + u32 temp_scaling_factor; + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *) model->model_data; + struct thermal_zone_device *gpu_tz = model_data->gpu_tz; + u64 coeffp_big; + + if (gpu_tz) { + int ret; + + ret = gpu_tz->ops->get_temp(gpu_tz, &temp); + if (ret) { + pr_warn_ratelimited("Error reading temperature for gpu thermal zone: %d\n", + ret); + temp = FALLBACK_STATIC_TEMPERATURE; + } + } else { + temp = FALLBACK_STATIC_TEMPERATURE; + } + + temp_scaling_factor = calculate_temp_scaling_factor(model_data->ts, + temp); + coeffp_big = (u64)model_data->static_coefficient * temp_scaling_factor; + *coeffp = div_u64(coeffp_big, 1000000); + + return 0; +} + +static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, + u32 current_freq) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *) model->model_data; + + *coeffp = model_data->dynamic_coefficient; + + return 0; +} + +static int add_params(struct kbase_ipa_model *model) +{ + int err = 0; + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *)model->model_data; + + err = kbase_ipa_model_add_param_s32(model, "static-coefficient", + &model_data->static_coefficient, + 1, true); + if (err) + goto end; + + err = kbase_ipa_model_add_param_s32(model, "dynamic-coefficient", + &model_data->dynamic_coefficient, + 1, true); + if (err) + goto end; + + err = kbase_ipa_model_add_param_s32(model, "ts", + model_data->ts, 4, true); + if (err) + goto end; + + err = kbase_ipa_model_add_param_string(model, "thermal-zone", + model_data->tz_name, + sizeof(model_data->tz_name), true); + +end: + return err; +} + +static int kbase_simple_power_model_init(struct kbase_ipa_model *model) +{ + int err; + struct kbase_ipa_model_simple_data *model_data; + + model_data = kzalloc(sizeof(struct kbase_ipa_model_simple_data), + GFP_KERNEL); + if (!model_data) + return -ENOMEM; + + model->model_data = (void *) model_data; + + err = add_params(model); + + return err; +} + +static int kbase_simple_power_model_recalculate(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *)model->model_data; + + if (!strnlen(model_data->tz_name, sizeof(model_data->tz_name))) { + model_data->gpu_tz = NULL; + } else { + model_data->gpu_tz = thermal_zone_get_zone_by_name(model_data->tz_name); + + if (IS_ERR(model_data->gpu_tz)) { + pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n", + PTR_ERR(model_data->gpu_tz), + model_data->tz_name); + model_data->gpu_tz = NULL; + return -EPROBE_DEFER; + } + } + + return 0; +} + +static void kbase_simple_power_model_term(struct kbase_ipa_model *model) +{ + struct kbase_ipa_model_simple_data *model_data = + (struct kbase_ipa_model_simple_data *)model->model_data; + + kfree(model_data); +} + +struct kbase_ipa_model_ops kbase_simple_ipa_model_ops = { + .name = "mali-simple-power-model", + .init = &kbase_simple_power_model_init, + .recalculate = &kbase_simple_power_model_recalculate, + .term = &kbase_simple_power_model_term, + .get_dynamic_coeff = &model_dynamic_coeff, + .get_static_coeff = &model_static_coeff, + .do_utilization_scaling_in_framework = true, +}; diff --git a/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h b/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h new file mode 100755 index 000000000000..6be0a334f99f --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h @@ -0,0 +1,311 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, + * please update base/tools/hwconfig_generator/hwc_{issues,features}.py + * For more information see base/tools/hwconfig_generator/README + */ + +#ifndef _BASE_HWCONFIG_FEATURES_H_ +#define _BASE_HWCONFIG_FEATURES_H_ + +enum base_hw_feature { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, + BASE_HW_FEATURE_IMAGES_IN_FRAGMENT_SHADERS, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_V4, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_AARCH64_MMU, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_generic[] = { + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t60x[] = { + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_V4, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t62x[] = { + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_V4, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t72x[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_V4, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t76x[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tFxx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t83x[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_t82x[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tMIx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tHEx[] = { + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + +static const enum base_hw_feature base_hw_features_tSIx[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + + +#ifdef MALI_INCLUDE_TKAX +static const enum base_hw_feature base_hw_features_tKAx[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + +#endif /* MALI_INCLUDE_TKAX */ + +#ifdef MALI_INCLUDE_TTRX +static const enum base_hw_feature base_hw_features_tTRx[] = { + BASE_HW_FEATURE_33BIT_VA, + BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, + BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, + BASE_HW_FEATURE_XAFFINITY, + BASE_HW_FEATURE_WARPING, + BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, + BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, + BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, + BASE_HW_FEATURE_BRNDOUT_CC, + BASE_HW_FEATURE_BRNDOUT_KILL, + BASE_HW_FEATURE_LD_ST_LEA_TEX, + BASE_HW_FEATURE_LD_ST_TILEBUFFER, + BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, + BASE_HW_FEATURE_MRT, + BASE_HW_FEATURE_MSAA_16X, + BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, + BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, + BASE_HW_FEATURE_T7XX_PAIRING_RULES, + BASE_HW_FEATURE_TEST4_DATUM_MODE, + BASE_HW_FEATURE_FLUSH_REDUCTION, + BASE_HW_FEATURE_PROTECTED_MODE, + BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + BASE_HW_FEATURE_COHERENCY_REG, + BASE_HW_FEATURE_END +}; + +#endif /* MALI_INCLUDE_TTRX */ + +#endif /* _BASE_HWCONFIG_FEATURES_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h b/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h new file mode 100755 index 000000000000..6d7e5c57e6a4 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h @@ -0,0 +1,1098 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, + * please update base/tools/hwconfig_generator/hwc_{issues,features}.py + * For more information see base/tools/hwconfig_generator/README + */ + +#ifndef _BASE_HWCONFIG_ISSUES_H_ +#define _BASE_HWCONFIG_ISSUES_H_ + +enum base_hw_issue { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_6367, + BASE_HW_ISSUE_6398, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_6787, + BASE_HW_ISSUE_7027, + BASE_HW_ISSUE_7144, + BASE_HW_ISSUE_7304, + BASE_HW_ISSUE_8073, + BASE_HW_ISSUE_8186, + BASE_HW_ISSUE_8215, + BASE_HW_ISSUE_8245, + BASE_HW_ISSUE_8250, + BASE_HW_ISSUE_8260, + BASE_HW_ISSUE_8280, + BASE_HW_ISSUE_8316, + BASE_HW_ISSUE_8381, + BASE_HW_ISSUE_8394, + BASE_HW_ISSUE_8401, + BASE_HW_ISSUE_8408, + BASE_HW_ISSUE_8443, + BASE_HW_ISSUE_8456, + BASE_HW_ISSUE_8564, + BASE_HW_ISSUE_8634, + BASE_HW_ISSUE_8778, + BASE_HW_ISSUE_8791, + BASE_HW_ISSUE_8833, + BASE_HW_ISSUE_8879, + BASE_HW_ISSUE_8896, + BASE_HW_ISSUE_8975, + BASE_HW_ISSUE_8986, + BASE_HW_ISSUE_8987, + BASE_HW_ISSUE_9010, + BASE_HW_ISSUE_9418, + BASE_HW_ISSUE_9423, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_9510, + BASE_HW_ISSUE_9566, + BASE_HW_ISSUE_9630, + BASE_HW_ISSUE_10127, + BASE_HW_ISSUE_10327, + BASE_HW_ISSUE_10410, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10487, + BASE_HW_ISSUE_10607, + BASE_HW_ISSUE_10632, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10676, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10797, + BASE_HW_ISSUE_10817, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10959, + BASE_HW_ISSUE_10969, + BASE_HW_ISSUE_10984, + BASE_HW_ISSUE_10995, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11035, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + GPUCORE_1619, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_generic[] = { + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t60x_r0p0_15dev0[] = { + BASE_HW_ISSUE_6367, + BASE_HW_ISSUE_6398, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_6787, + BASE_HW_ISSUE_7027, + BASE_HW_ISSUE_7144, + BASE_HW_ISSUE_7304, + BASE_HW_ISSUE_8073, + BASE_HW_ISSUE_8186, + BASE_HW_ISSUE_8215, + BASE_HW_ISSUE_8245, + BASE_HW_ISSUE_8250, + BASE_HW_ISSUE_8260, + BASE_HW_ISSUE_8280, + BASE_HW_ISSUE_8316, + BASE_HW_ISSUE_8381, + BASE_HW_ISSUE_8394, + BASE_HW_ISSUE_8401, + BASE_HW_ISSUE_8408, + BASE_HW_ISSUE_8443, + BASE_HW_ISSUE_8456, + BASE_HW_ISSUE_8564, + BASE_HW_ISSUE_8634, + BASE_HW_ISSUE_8778, + BASE_HW_ISSUE_8791, + BASE_HW_ISSUE_8833, + BASE_HW_ISSUE_8896, + BASE_HW_ISSUE_8975, + BASE_HW_ISSUE_8986, + BASE_HW_ISSUE_8987, + BASE_HW_ISSUE_9010, + BASE_HW_ISSUE_9418, + BASE_HW_ISSUE_9423, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_9510, + BASE_HW_ISSUE_9566, + BASE_HW_ISSUE_9630, + BASE_HW_ISSUE_10410, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10487, + BASE_HW_ISSUE_10607, + BASE_HW_ISSUE_10632, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10676, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10969, + BASE_HW_ISSUE_10984, + BASE_HW_ISSUE_10995, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11035, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_3964, + GPUCORE_1619, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t60x_r0p0_eac[] = { + BASE_HW_ISSUE_6367, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_6787, + BASE_HW_ISSUE_7027, + BASE_HW_ISSUE_7304, + BASE_HW_ISSUE_8408, + BASE_HW_ISSUE_8564, + BASE_HW_ISSUE_8778, + BASE_HW_ISSUE_8975, + BASE_HW_ISSUE_9010, + BASE_HW_ISSUE_9418, + BASE_HW_ISSUE_9423, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_9510, + BASE_HW_ISSUE_10410, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10487, + BASE_HW_ISSUE_10607, + BASE_HW_ISSUE_10632, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10676, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10969, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11035, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t60x_r0p1[] = { + BASE_HW_ISSUE_6367, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_6787, + BASE_HW_ISSUE_7027, + BASE_HW_ISSUE_7304, + BASE_HW_ISSUE_8408, + BASE_HW_ISSUE_8564, + BASE_HW_ISSUE_8778, + BASE_HW_ISSUE_8975, + BASE_HW_ISSUE_9010, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_9510, + BASE_HW_ISSUE_10410, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10487, + BASE_HW_ISSUE_10607, + BASE_HW_ISSUE_10632, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10676, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11035, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t62x_r0p1[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10127, + BASE_HW_ISSUE_10327, + BASE_HW_ISSUE_10410, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10487, + BASE_HW_ISSUE_10607, + BASE_HW_ISSUE_10632, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10676, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10817, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10959, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11035, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t62x_r1p0[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10959, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t62x_r1p1[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_10959, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r0p1_50rel0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r0p2[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r0p3[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_26, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3542, + BASE_HW_ISSUE_T76X_3556, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t76x_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t72x_r0p0[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10797, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t72x_r1p0[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10797, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t72x_r1p1[] = { + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10684, + BASE_HW_ISSUE_10797, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t72x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10471, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10797, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t76x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t60x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_8778, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t62x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_6402, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10472, + BASE_HW_ISSUE_10649, + BASE_HW_ISSUE_10931, + BASE_HW_ISSUE_11012, + BASE_HW_ISSUE_11020, + BASE_HW_ISSUE_11024, + BASE_HW_ISSUE_11042, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3964, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tFRx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tFRx_r0p2[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tFRx_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tFRx_r2p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tFRx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t86x_r0p2[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t86x_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t86x_r2p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3966, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t86x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t83x_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t83x_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t83x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + GPUCORE_1619, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t82x_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3964, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t82x_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1909, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_t82x_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10821, + BASE_HW_ISSUE_10883, + BASE_HW_ISSUE_10946, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T720_1386, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_T76X_3960, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_t82x[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_11051, + BASE_HW_ISSUE_T76X_1963, + BASE_HW_ISSUE_T76X_3086, + BASE_HW_ISSUE_T76X_3700, + BASE_HW_ISSUE_T76X_3793, + BASE_HW_ISSUE_T76X_3979, + BASE_HW_ISSUE_TMIX_7891, + GPUCORE_1619, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_T76X_3953, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_11054, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8463, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tMIx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, + BASE_HW_ISSUE_TMIX_8206, + BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8456, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_10682, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tHEx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_8042, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +static const enum base_hw_issue base_hw_issues_model_tSIx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + + + +#ifdef MALI_INCLUDE_TKAX +static const enum base_hw_issue base_hw_issues_tKAx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +#endif /* MALI_INCLUDE_TKAX */ + +#ifdef MALI_INCLUDE_TKAX +static const enum base_hw_issue base_hw_issues_model_tKAx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +#endif /* MALI_INCLUDE_TKAX */ + +#ifdef MALI_INCLUDE_TTRX +static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +#endif /* MALI_INCLUDE_TTRX */ + +#ifdef MALI_INCLUDE_TTRX +static const enum base_hw_issue base_hw_issues_model_tTRx[] = { + BASE_HW_ISSUE_5736, + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_END +}; + +#endif /* MALI_INCLUDE_TTRX */ + +#endif /* _BASE_HWCONFIG_ISSUES_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_base_kernel.h b/drivers/gpu/arm/midgard/mali_base_kernel.h new file mode 100755 index 000000000000..ea5e473caef6 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_base_kernel.h @@ -0,0 +1,1858 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file + * Base structures shared with the kernel. + */ + +#ifndef _BASE_KERNEL_H_ +#define _BASE_KERNEL_H_ + +#ifndef __user +#define __user +#endif + +/* Support UK6 IOCTLS */ +#define BASE_LEGACY_UK6_SUPPORT 1 + +/* Support UK7 IOCTLS */ +/* NB: To support UK6 we also need to support UK7 */ +#define BASE_LEGACY_UK7_SUPPORT 1 + +/* Support UK8 IOCTLS */ +#define BASE_LEGACY_UK8_SUPPORT 1 + +/* Support UK9 IOCTLS */ +#define BASE_LEGACY_UK9_SUPPORT 1 + +/* Support UK10_2 IOCTLS */ +#define BASE_LEGACY_UK10_2_SUPPORT 1 + +/* Support UK10_4 IOCTLS */ +#define BASE_LEGACY_UK10_4_SUPPORT 1 + +typedef struct base_mem_handle { + struct { + u64 handle; + } basep; +} base_mem_handle; + +#include "mali_base_mem_priv.h" +#include "mali_kbase_profiling_gator_api.h" +#include "mali_midg_coherency.h" +#include "mali_kbase_gpu_id.h" + +/* + * Dependency stuff, keep it private for now. May want to expose it if + * we decide to make the number of semaphores a configurable + * option. + */ +#define BASE_JD_ATOM_COUNT 512 + +#define BASEP_JD_SEM_PER_WORD_LOG2 5 +#define BASEP_JD_SEM_PER_WORD (1 << BASEP_JD_SEM_PER_WORD_LOG2) +#define BASEP_JD_SEM_WORD_NR(x) ((x) >> BASEP_JD_SEM_PER_WORD_LOG2) +#define BASEP_JD_SEM_MASK_IN_WORD(x) (1 << ((x) & (BASEP_JD_SEM_PER_WORD - 1))) +#define BASEP_JD_SEM_ARRAY_SIZE BASEP_JD_SEM_WORD_NR(BASE_JD_ATOM_COUNT) + +/* Set/reset values for a software event */ +#define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) +#define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) + +#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3 + +#define BASE_MAX_COHERENT_GROUPS 16 + +#if defined CDBG_ASSERT +#define LOCAL_ASSERT CDBG_ASSERT +#elif defined KBASE_DEBUG_ASSERT +#define LOCAL_ASSERT KBASE_DEBUG_ASSERT +#else +#error assert macro not defined! +#endif + +#if defined PAGE_MASK +#define LOCAL_PAGE_LSB ~PAGE_MASK +#else +#include + +#if defined OSU_CONFIG_CPU_PAGE_SIZE_LOG2 +#define LOCAL_PAGE_LSB ((1ul << OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - 1) +#else +#error Failed to find page size +#endif +#endif + +/** 32/64-bit neutral way to represent pointers */ +typedef union kbase_pointer { + void __user *value; /**< client should store their pointers here */ + u32 compat_value; /**< 64-bit kernels should fetch value here when handling 32-bit clients */ + u64 sizer; /**< Force 64-bit storage for all clients regardless */ +} kbase_pointer; + +/** + * @addtogroup base_user_api User-side Base APIs + * @{ + */ + +/** + * @addtogroup base_user_api_memory User-side Base Memory APIs + * @{ + */ + +/** + * typedef base_mem_alloc_flags - Memory allocation, access/hint flags. + * + * A combination of MEM_PROT/MEM_HINT flags must be passed to each allocator + * in order to determine the best cache policy. Some combinations are + * of course invalid (e.g. MEM_PROT_CPU_WR | MEM_HINT_CPU_RD), + * which defines a write-only region on the CPU side, which is + * heavily read by the CPU... + * Other flags are only meaningful to a particular allocator. + * More flags can be added to this list, as long as they don't clash + * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). + */ +typedef u32 base_mem_alloc_flags; + +/* Memory allocation, access/hint flags. + * + * See base_mem_alloc_flags. + */ + +/* IN */ +/* Read access CPU side + */ +#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) + +/* Write access CPU side + */ +#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) + +/* Read access GPU side + */ +#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) + +/* Write access GPU side + */ +#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) + +/* Execute allowed on the GPU side + */ +#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) + + /* BASE_MEM_HINT flags have been removed, but their values are reserved + * for backwards compatibility with older user-space drivers. The values + * can be re-used once support for r5p0 user-space drivers is removed, + * presumably in r7p0. + * + * RESERVED: (1U << 5) + * RESERVED: (1U << 6) + * RESERVED: (1U << 7) + * RESERVED: (1U << 8) + */ + +/* Grow backing store on GPU Page Fault + */ +#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) + +/* Page coherence Outer shareable, if available + */ +#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) + +/* Page coherence Inner shareable + */ +#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) + +/* Should be cached on the CPU + */ +#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) + +/* IN/OUT */ +/* Must have same VA on both the GPU and the CPU + */ +#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) + +/* OUT */ +/* Must call mmap to acquire a GPU address for the alloc + */ +#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) + +/* IN */ +/* Page coherence Outer shareable, required. + */ +#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) + +/* Secure memory + */ +#define BASE_MEM_SECURE ((base_mem_alloc_flags)1 << 16) + +/* Not needed physical memory + */ +#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) + +/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the + * addresses to be the same + */ +#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) + +/* Number of bits used as flags for base memory management + * + * Must be kept in sync with the base_mem_alloc_flags flags + */ +#define BASE_MEM_FLAGS_NR_BITS 19 + +/* A mask for all output bits, excluding IN/OUT bits. + */ +#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP + +/* A mask for all input bits, including IN/OUT bits. + */ +#define BASE_MEM_FLAGS_INPUT_MASK \ + (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) + +/* A mask for all the flags which are modifiable via the base_mem_set_flags + * interface. + */ +#define BASE_MEM_FLAGS_MODIFIABLE \ + (BASE_MEM_DONT_NEED | BASE_MEM_COHERENT_SYSTEM | \ + BASE_MEM_COHERENT_LOCAL) + +/** + * enum base_mem_import_type - Memory types supported by @a base_mem_import + * + * @BASE_MEM_IMPORT_TYPE_INVALID: Invalid type + * @BASE_MEM_IMPORT_TYPE_UMP: UMP import. Handle type is ump_secure_id. + * @BASE_MEM_IMPORT_TYPE_UMM: UMM import. Handle type is a file descriptor (int) + * @BASE_MEM_IMPORT_TYPE_USER_BUFFER: User buffer import. Handle is a + * base_mem_import_user_buffer + * + * Each type defines what the supported handle type is. + * + * If any new type is added here ARM must be contacted + * to allocate a numeric value for it. + * Do not just add a new type without synchronizing with ARM + * as future releases from ARM might include other new types + * which could clash with your custom types. + */ +typedef enum base_mem_import_type { + BASE_MEM_IMPORT_TYPE_INVALID = 0, + BASE_MEM_IMPORT_TYPE_UMP = 1, + BASE_MEM_IMPORT_TYPE_UMM = 2, + BASE_MEM_IMPORT_TYPE_USER_BUFFER = 3 +} base_mem_import_type; + +/** + * struct base_mem_import_user_buffer - Handle of an imported user buffer + * + * @ptr: kbase_pointer to imported user buffer + * @length: length of imported user buffer in bytes + * + * This structure is used to represent a handle of an imported user buffer. + */ + +struct base_mem_import_user_buffer { + kbase_pointer ptr; + u64 length; +}; + +/** + * @brief Invalid memory handle. + * + * Return value from functions returning @ref base_mem_handle on error. + * + * @warning @ref base_mem_handle_new_invalid must be used instead of this macro + * in C++ code or other situations where compound literals cannot be used. + */ +#define BASE_MEM_INVALID_HANDLE ((base_mem_handle) { {BASEP_MEM_INVALID_HANDLE} }) + +/** + * @brief Special write-alloc memory handle. + * + * A special handle is used to represent a region where a special page is mapped + * with a write-alloc cache setup, typically used when the write result of the + * GPU isn't needed, but the GPU must write anyway. + * + * @warning @ref base_mem_handle_new_write_alloc must be used instead of this macro + * in C++ code or other situations where compound literals cannot be used. + */ +#define BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ((base_mem_handle) { {BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE} }) + +#define BASEP_MEM_INVALID_HANDLE (0ull << 12) +#define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12) +#define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12) +#define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12) +#define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12) +/* reserved handles ..-64< for future special handles */ +#define BASE_MEM_COOKIE_BASE (64ul << 12) +#define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ + BASE_MEM_COOKIE_BASE) + +/* Mask to detect 4GB boundary alignment */ +#define BASE_MEM_MASK_4GB 0xfffff000UL + + +/* Bit mask of cookies used for for memory allocation setup */ +#define KBASE_COOKIE_MASK ~1UL /* bit 0 is reserved */ + + +/** + * @brief Result codes of changing the size of the backing store allocated to a tmem region + */ +typedef enum base_backing_threshold_status { + BASE_BACKING_THRESHOLD_OK = 0, /**< Resize successful */ + BASE_BACKING_THRESHOLD_ERROR_OOM = -2, /**< Increase failed due to an out-of-memory condition */ + BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS = -4 /**< Invalid arguments (not tmem, illegal size request, etc.) */ +} base_backing_threshold_status; + +/** + * @addtogroup base_user_api_memory_defered User-side Base Defered Memory Coherency APIs + * @{ + */ + +/** + * @brief a basic memory operation (sync-set). + * + * The content of this structure is private, and should only be used + * by the accessors. + */ +typedef struct base_syncset { + struct basep_syncset basep_sset; +} base_syncset; + +/** @} end group base_user_api_memory_defered */ + +/** + * Handle to represent imported memory object. + * Simple opague handle to imported memory, can't be used + * with anything but base_external_resource_init to bind to an atom. + */ +typedef struct base_import_handle { + struct { + u64 handle; + } basep; +} base_import_handle; + +/** @} end group base_user_api_memory */ + +/** + * @addtogroup base_user_api_job_dispatch User-side Base Job Dispatcher APIs + * @{ + */ + +typedef int platform_fence_type; +#define INVALID_PLATFORM_FENCE ((platform_fence_type)-1) + +/** + * Base stream handle. + * + * References an underlying base stream object. + */ +typedef struct base_stream { + struct { + int fd; + } basep; +} base_stream; + +/** + * Base fence handle. + * + * References an underlying base fence object. + */ +typedef struct base_fence { + struct { + int fd; + int stream_fd; + } basep; +} base_fence; + +/** + * @brief Per-job data + * + * This structure is used to store per-job data, and is completely unused + * by the Base driver. It can be used to store things such as callback + * function pointer, data to handle job completion. It is guaranteed to be + * untouched by the Base driver. + */ +typedef struct base_jd_udata { + u64 blob[2]; /**< per-job data array */ +} base_jd_udata; + +/** + * @brief Memory aliasing info + * + * Describes a memory handle to be aliased. + * A subset of the handle can be chosen for aliasing, given an offset and a + * length. + * A special handle BASE_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a + * region where a special page is mapped with a write-alloc cache setup, + * typically used when the write result of the GPU isn't needed, but the GPU + * must write anyway. + * + * Offset and length are specified in pages. + * Offset must be within the size of the handle. + * Offset+length must not overrun the size of the handle. + * + * @handle Handle to alias, can be BASE_MEM_WRITE_ALLOC_PAGES_HANDLE + * @offset Offset within the handle to start aliasing from, in pages. + * Not used with BASE_MEM_WRITE_ALLOC_PAGES_HANDLE. + * @length Length to alias, in pages. For BASE_MEM_WRITE_ALLOC_PAGES_HANDLE + * specifies the number of times the special page is needed. + */ +struct base_mem_aliasing_info { + base_mem_handle handle; + u64 offset; + u64 length; +}; + +/** + * struct base_jit_alloc_info - Structure which describes a JIT allocation + * request. + * @gpu_alloc_addr: The GPU virtual address to write the JIT + * allocated GPU virtual address to. + * @va_pages: The minimum number of virtual pages required. + * @commit_pages: The minimum number of physical pages which + * should back the allocation. + * @extent: Granularity of physical pages to grow the + * allocation by during a fault. + * @id: Unique ID provided by the caller, this is used + * to pair allocation and free requests. + * Zero is not a valid value. + */ +struct base_jit_alloc_info { + u64 gpu_alloc_addr; + u64 va_pages; + u64 commit_pages; + u64 extent; + u8 id; +}; + +/** + * @brief Job dependency type. + * + * A flags field will be inserted into the atom structure to specify whether a dependency is a data or + * ordering dependency (by putting it before/after 'core_req' in the structure it should be possible to add without + * changing the structure size). + * When the flag is set for a particular dependency to signal that it is an ordering only dependency then + * errors will not be propagated. + */ +typedef u8 base_jd_dep_type; + + +#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */ +#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */ +#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */ + +/** + * @brief Job chain hardware requirements. + * + * A job chain must specify what GPU features it needs to allow the + * driver to schedule the job correctly. By not specifying the + * correct settings can/will cause an early job termination. Multiple + * values can be ORed together to specify multiple requirements. + * Special case is ::BASE_JD_REQ_DEP, which is used to express complex + * dependencies, and that doesn't execute anything on the hardware. + */ +typedef u32 base_jd_core_req; + +/* Requirements that come from the HW */ + +/** + * No requirement, dependency only + */ +#define BASE_JD_REQ_DEP ((base_jd_core_req)0) + +/** + * Requires fragment shaders + */ +#define BASE_JD_REQ_FS ((base_jd_core_req)1 << 0) + +/** + * Requires compute shaders + * This covers any of the following Midgard Job types: + * - Vertex Shader Job + * - Geometry Shader Job + * - An actual Compute Shader Job + * + * Compare this with @ref BASE_JD_REQ_ONLY_COMPUTE, which specifies that the + * job is specifically just the "Compute Shader" job type, and not the "Vertex + * Shader" nor the "Geometry Shader" job type. + */ +#define BASE_JD_REQ_CS ((base_jd_core_req)1 << 1) +#define BASE_JD_REQ_T ((base_jd_core_req)1 << 2) /**< Requires tiling */ +#define BASE_JD_REQ_CF ((base_jd_core_req)1 << 3) /**< Requires cache flushes */ +#define BASE_JD_REQ_V ((base_jd_core_req)1 << 4) /**< Requires value writeback */ + +/* SW-only requirements - the HW does not expose these as part of the job slot capabilities */ + +/* Requires fragment job with AFBC encoding */ +#define BASE_JD_REQ_FS_AFBC ((base_jd_core_req)1 << 13) + +/** + * SW-only requirement: coalesce completion events. + * If this bit is set then completion of this atom will not cause an event to + * be sent to userspace, whether successful or not; completion events will be + * deferred until an atom completes which does not have this bit set. + * + * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES. + */ +#define BASE_JD_REQ_EVENT_COALESCE ((base_jd_core_req)1 << 5) + +/** + * SW Only requirement: the job chain requires a coherent core group. We don't + * mind which coherent core group is used. + */ +#define BASE_JD_REQ_COHERENT_GROUP ((base_jd_core_req)1 << 6) + +/** + * SW Only requirement: The performance counters should be enabled only when + * they are needed, to reduce power consumption. + */ + +#define BASE_JD_REQ_PERMON ((base_jd_core_req)1 << 7) + +/** + * SW Only requirement: External resources are referenced by this atom. + * When external resources are referenced no syncsets can be bundled with the atom + * but should instead be part of a NULL jobs inserted into the dependency tree. + * The first pre_dep object must be configured for the external resouces to use, + * the second pre_dep object can be used to create other dependencies. + * + * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE. + */ +#define BASE_JD_REQ_EXTERNAL_RESOURCES ((base_jd_core_req)1 << 8) + +/** + * SW Only requirement: Software defined job. Jobs with this bit set will not be submitted + * to the hardware but will cause some action to happen within the driver + */ +#define BASE_JD_REQ_SOFT_JOB ((base_jd_core_req)1 << 9) + +#define BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME (BASE_JD_REQ_SOFT_JOB | 0x1) +#define BASE_JD_REQ_SOFT_FENCE_TRIGGER (BASE_JD_REQ_SOFT_JOB | 0x2) +#define BASE_JD_REQ_SOFT_FENCE_WAIT (BASE_JD_REQ_SOFT_JOB | 0x3) + +/** + * SW Only requirement : Replay job. + * + * If the preceding job fails, the replay job will cause the jobs specified in + * the list of base_jd_replay_payload pointed to by the jc pointer to be + * replayed. + * + * A replay job will only cause jobs to be replayed up to BASEP_JD_REPLAY_LIMIT + * times. If a job fails more than BASEP_JD_REPLAY_LIMIT times then the replay + * job is failed, as well as any following dependencies. + * + * The replayed jobs will require a number of atom IDs. If there are not enough + * free atom IDs then the replay job will fail. + * + * If the preceding job does not fail, then the replay job is returned as + * completed. + * + * The replayed jobs will never be returned to userspace. The preceding failed + * job will be returned to userspace as failed; the status of this job should + * be ignored. Completion should be determined by the status of the replay soft + * job. + * + * In order for the jobs to be replayed, the job headers will have to be + * modified. The Status field will be reset to NOT_STARTED. If the Job Type + * field indicates a Vertex Shader Job then it will be changed to Null Job. + * + * The replayed jobs have the following assumptions : + * + * - No external resources. Any required external resources will be held by the + * replay atom. + * - Pre-dependencies are created based on job order. + * - Atom numbers are automatically assigned. + * - device_nr is set to 0. This is not relevant as + * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set. + * - Priority is inherited from the replay job. + */ +#define BASE_JD_REQ_SOFT_REPLAY (BASE_JD_REQ_SOFT_JOB | 0x4) +/** + * SW only requirement: event wait/trigger job. + * + * - BASE_JD_REQ_SOFT_EVENT_WAIT: this job will block until the event is set. + * - BASE_JD_REQ_SOFT_EVENT_SET: this job sets the event, thus unblocks the + * other waiting jobs. It completes immediately. + * - BASE_JD_REQ_SOFT_EVENT_RESET: this job resets the event, making it + * possible for other jobs to wait upon. It completes immediately. + */ +#define BASE_JD_REQ_SOFT_EVENT_WAIT (BASE_JD_REQ_SOFT_JOB | 0x5) +#define BASE_JD_REQ_SOFT_EVENT_SET (BASE_JD_REQ_SOFT_JOB | 0x6) +#define BASE_JD_REQ_SOFT_EVENT_RESET (BASE_JD_REQ_SOFT_JOB | 0x7) + +#define BASE_JD_REQ_SOFT_DEBUG_COPY (BASE_JD_REQ_SOFT_JOB | 0x8) + +/** + * SW only requirement: Just In Time allocation + * + * This job requests a JIT allocation based on the request in the + * @base_jit_alloc_info structure which is passed via the jc element of + * the atom. + * + * It should be noted that the id entry in @base_jit_alloc_info must not + * be reused until it has been released via @BASE_JD_REQ_SOFT_JIT_FREE. + * + * Should this soft job fail it is expected that a @BASE_JD_REQ_SOFT_JIT_FREE + * soft job to free the JIT allocation is still made. + * + * The job will complete immediately. + */ +#define BASE_JD_REQ_SOFT_JIT_ALLOC (BASE_JD_REQ_SOFT_JOB | 0x9) +/** + * SW only requirement: Just In Time free + * + * This job requests a JIT allocation created by @BASE_JD_REQ_SOFT_JIT_ALLOC + * to be freed. The ID of the JIT allocation is passed via the jc element of + * the atom. + * + * The job will complete immediately. + */ +#define BASE_JD_REQ_SOFT_JIT_FREE (BASE_JD_REQ_SOFT_JOB | 0xa) + +/** + * SW only requirement: Map external resource + * + * This job requests external resource(s) are mapped once the dependencies + * of the job have been satisfied. The list of external resources are + * passed via the jc element of the atom which is a pointer to a + * @base_external_resource_list. + */ +#define BASE_JD_REQ_SOFT_EXT_RES_MAP (BASE_JD_REQ_SOFT_JOB | 0xb) +/** + * SW only requirement: Unmap external resource + * + * This job requests external resource(s) are unmapped once the dependencies + * of the job has been satisfied. The list of external resources are + * passed via the jc element of the atom which is a pointer to a + * @base_external_resource_list. + */ +#define BASE_JD_REQ_SOFT_EXT_RES_UNMAP (BASE_JD_REQ_SOFT_JOB | 0xc) + +/** + * HW Requirement: Requires Compute shaders (but not Vertex or Geometry Shaders) + * + * This indicates that the Job Chain contains Midgard Jobs of the 'Compute Shaders' type. + * + * In contrast to @ref BASE_JD_REQ_CS, this does \b not indicate that the Job + * Chain contains 'Geometry Shader' or 'Vertex Shader' jobs. + */ +#define BASE_JD_REQ_ONLY_COMPUTE ((base_jd_core_req)1 << 10) + +/** + * HW Requirement: Use the base_jd_atom::device_nr field to specify a + * particular core group + * + * If both @ref BASE_JD_REQ_COHERENT_GROUP and this flag are set, this flag takes priority + * + * This is only guaranteed to work for @ref BASE_JD_REQ_ONLY_COMPUTE atoms. + * + * If the core availability policy is keeping the required core group turned off, then + * the job will fail with a @ref BASE_JD_EVENT_PM_EVENT error code. + */ +#define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ((base_jd_core_req)1 << 11) + +/** + * SW Flag: If this bit is set then the successful completion of this atom + * will not cause an event to be sent to userspace + */ +#define BASE_JD_REQ_EVENT_ONLY_ON_FAILURE ((base_jd_core_req)1 << 12) + +/** + * SW Flag: If this bit is set then completion of this atom will not cause an + * event to be sent to userspace, whether successful or not. + */ +#define BASEP_JD_REQ_EVENT_NEVER ((base_jd_core_req)1 << 14) + +/** + * SW Flag: Skip GPU cache clean and invalidation before starting a GPU job. + * + * If this bit is set then the GPU's cache will not be cleaned and invalidated + * until a GPU job starts which does not have this bit set or a job completes + * which does not have the @ref BASE_JD_REQ_SKIP_CACHE_END bit set. Do not use if + * the CPU may have written to memory addressed by the job since the last job + * without this bit set was submitted. + */ +#define BASE_JD_REQ_SKIP_CACHE_START ((base_jd_core_req)1 << 15) + +/** + * SW Flag: Skip GPU cache clean and invalidation after a GPU job completes. + * + * If this bit is set then the GPU's cache will not be cleaned and invalidated + * until a GPU job completes which does not have this bit set or a job starts + * which does not have the @ref BASE_JD_REQ_SKIP_CACHE_START bti set. Do not use if + * the CPU may read from or partially overwrite memory addressed by the job + * before the next job without this bit set completes. + */ +#define BASE_JD_REQ_SKIP_CACHE_END ((base_jd_core_req)1 << 16) + +/** + * These requirement bits are currently unused in base_jd_core_req + */ +#define BASEP_JD_REQ_RESERVED \ + (~(BASE_JD_REQ_ATOM_TYPE | BASE_JD_REQ_EXTERNAL_RESOURCES | \ + BASE_JD_REQ_EVENT_ONLY_ON_FAILURE | BASEP_JD_REQ_EVENT_NEVER | \ + BASE_JD_REQ_EVENT_COALESCE | \ + BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | \ + BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ + BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END)) + +/** + * Mask of all bits in base_jd_core_req that control the type of the atom. + * + * This allows dependency only atoms to have flags set + */ +#define BASE_JD_REQ_ATOM_TYPE \ + (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T | BASE_JD_REQ_CF | \ + BASE_JD_REQ_V | BASE_JD_REQ_SOFT_JOB | BASE_JD_REQ_ONLY_COMPUTE) + +/** + * Mask of all bits in base_jd_core_req that control the type of a soft job. + */ +#define BASE_JD_REQ_SOFT_JOB_TYPE (BASE_JD_REQ_SOFT_JOB | 0x1f) + +/* + * Returns non-zero value if core requirements passed define a soft job or + * a dependency only job. + */ +#define BASE_JD_REQ_SOFT_JOB_OR_DEP(core_req) \ + ((core_req & BASE_JD_REQ_SOFT_JOB) || \ + (core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) + +/** + * @brief States to model state machine processed by kbasep_js_job_check_ref_cores(), which + * handles retaining cores for power management and affinity management. + * + * The state @ref KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY prevents an attack + * where lots of atoms could be submitted before powerup, and each has an + * affinity chosen that causes other atoms to have an affinity + * violation. Whilst the affinity was not causing violations at the time it + * was chosen, it could cause violations thereafter. For example, 1000 jobs + * could have had their affinity chosen during the powerup time, so any of + * those 1000 jobs could cause an affinity violation later on. + * + * The attack would otherwise occur because other atoms/contexts have to wait for: + * -# the currently running atoms (which are causing the violation) to + * finish + * -# and, the atoms that had their affinity chosen during powerup to + * finish. These are run preferentially because they don't cause a + * violation, but instead continue to cause the violation in others. + * -# or, the attacker is scheduled out (which might not happen for just 2 + * contexts) + * + * By re-choosing the affinity (which is designed to avoid violations at the + * time it's chosen), we break condition (2) of the wait, which minimizes the + * problem to just waiting for current jobs to finish (which can be bounded if + * the Job Scheduling Policy has a timer). + */ +enum kbase_atom_coreref_state { + /** Starting state: No affinity chosen, and cores must be requested. kbase_jd_atom::affinity==0 */ + KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED, + /** Cores requested, but waiting for them to be powered. Requested cores given by kbase_jd_atom::affinity */ + KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES, + /** Cores given by kbase_jd_atom::affinity are powered, but affinity might be out-of-date, so must recheck */ + KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY, + /** Cores given by kbase_jd_atom::affinity are powered, and affinity is up-to-date, but must check for violations */ + KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS, + /** Cores are powered, kbase_jd_atom::affinity up-to-date, no affinity violations: atom can be submitted to HW */ + KBASE_ATOM_COREREF_STATE_READY +}; + +/* + * Base Atom priority + * + * Only certain priority levels are actually implemented, as specified by the + * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority + * level that is not one of those defined below. + * + * Priority levels only affect scheduling between atoms of the same type within + * a base context, and only after the atoms have had dependencies resolved. + * Fragment atoms does not affect non-frament atoms with lower priorities, and + * the other way around. For example, a low priority atom that has had its + * dependencies resolved might run before a higher priority atom that has not + * had its dependencies resolved. + * + * The scheduling between base contexts/processes and between atoms from + * different base contexts/processes is unaffected by atom priority. + * + * The atoms are scheduled as follows with respect to their priorities: + * - Let atoms 'X' and 'Y' be for the same job slot who have dependencies + * resolved, and atom 'X' has a higher priority than atom 'Y' + * - If atom 'Y' is currently running on the HW, then it is interrupted to + * allow atom 'X' to run soon after + * - If instead neither atom 'Y' nor atom 'X' are running, then when choosing + * the next atom to run, atom 'X' will always be chosen instead of atom 'Y' + * - Any two atoms that have the same priority could run in any order with + * respect to each other. That is, there is no ordering constraint between + * atoms of the same priority. + */ +typedef u8 base_jd_prio; + +/* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */ +#define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0) +/* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and + * BASE_JD_PRIO_LOW */ +#define BASE_JD_PRIO_HIGH ((base_jd_prio)1) +/* Low atom priority. */ +#define BASE_JD_PRIO_LOW ((base_jd_prio)2) + +/* Count of the number of priority levels. This itself is not a valid + * base_jd_prio setting */ +#define BASE_JD_NR_PRIO_LEVELS 3 + +enum kbase_jd_atom_state { + /** Atom is not used */ + KBASE_JD_ATOM_STATE_UNUSED, + /** Atom is queued in JD */ + KBASE_JD_ATOM_STATE_QUEUED, + /** Atom has been given to JS (is runnable/running) */ + KBASE_JD_ATOM_STATE_IN_JS, + /** Atom has been completed, but not yet handed back to job dispatcher + * for dependency resolution */ + KBASE_JD_ATOM_STATE_HW_COMPLETED, + /** Atom has been completed, but not yet handed back to userspace */ + KBASE_JD_ATOM_STATE_COMPLETED +}; + +typedef u16 base_atom_id; /**< Type big enough to store an atom number in */ + +struct base_dependency { + base_atom_id atom_id; /**< An atom number */ + base_jd_dep_type dependency_type; /**< Dependency type */ +}; + +/* This structure has changed since UK 10.2 for which base_jd_core_req was a u16 value. + * In order to keep the size of the structure same, padding field has been adjusted + * accordingly and core_req field of a u32 type (to which UK 10.3 base_jd_core_req defines) + * is added at the end of the structure. Place in the structure previously occupied by u16 core_req + * is kept but renamed to compat_core_req and as such it can be used in ioctl call for job submission + * as long as UK 10.2 legacy is supported. Once when this support ends, this field can be left + * for possible future use. */ +typedef struct base_jd_atom_v2 { + u64 jc; /**< job-chain GPU address */ + struct base_jd_udata udata; /**< user data */ + kbase_pointer extres_list; /**< list of external resources */ + u16 nr_extres; /**< nr of external resources */ + u16 compat_core_req; /**< core requirements which correspond to the legacy support for UK 10.2 */ + struct base_dependency pre_dep[2]; /**< pre-dependencies, one need to use SETTER function to assign this field, + this is done in order to reduce possibility of improper assigment of a dependency field */ + base_atom_id atom_number; /**< unique number to identify the atom */ + base_jd_prio prio; /**< Atom priority. Refer to @ref base_jd_prio for more details */ + u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */ + u8 padding[1]; + base_jd_core_req core_req; /**< core requirements */ +} base_jd_atom_v2; + +#ifdef BASE_LEGACY_UK6_SUPPORT +struct base_jd_atom_v2_uk6 { + u64 jc; /**< job-chain GPU address */ + struct base_jd_udata udata; /**< user data */ + kbase_pointer extres_list; /**< list of external resources */ + u16 nr_extres; /**< nr of external resources */ + u16 core_req; /**< core requirements */ + base_atom_id pre_dep[2]; /**< pre-dependencies */ + base_atom_id atom_number; /**< unique number to identify the atom */ + base_jd_prio prio; /**< priority - smaller is higher priority */ + u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */ + u8 padding[7]; +}; +#endif /* BASE_LEGACY_UK6_SUPPORT */ + +typedef enum base_external_resource_access { + BASE_EXT_RES_ACCESS_SHARED, + BASE_EXT_RES_ACCESS_EXCLUSIVE +} base_external_resource_access; + +typedef struct base_external_resource { + u64 ext_resource; +} base_external_resource; + + +/** + * The maximum number of external resources which can be mapped/unmapped + * in a single request. + */ +#define BASE_EXT_RES_COUNT_MAX 10 + +/** + * struct base_external_resource_list - Structure which describes a list of + * external resources. + * @count: The number of resources. + * @ext_res: Array of external resources which is + * sized at allocation time. + */ +struct base_external_resource_list { + u64 count; + struct base_external_resource ext_res[1]; +}; + +struct base_jd_debug_copy_buffer { + u64 address; + u64 size; + struct base_external_resource extres; +}; + +/** + * @brief Setter for a dependency structure + * + * @param[in] dep The kbase jd atom dependency to be initialized. + * @param id The atom_id to be assigned. + * @param dep_type The dep_type to be assigned. + * + */ +static inline void base_jd_atom_dep_set(struct base_dependency *dep, + base_atom_id id, base_jd_dep_type dep_type) +{ + LOCAL_ASSERT(dep != NULL); + + /* + * make sure we don't set not allowed combinations + * of atom_id/dependency_type. + */ + LOCAL_ASSERT((id == 0 && dep_type == BASE_JD_DEP_TYPE_INVALID) || + (id > 0 && dep_type != BASE_JD_DEP_TYPE_INVALID)); + + dep->atom_id = id; + dep->dependency_type = dep_type; +} + +/** + * @brief Make a copy of a dependency structure + * + * @param[in,out] dep The kbase jd atom dependency to be written. + * @param[in] from The dependency to make a copy from. + * + */ +static inline void base_jd_atom_dep_copy(struct base_dependency *dep, + const struct base_dependency *from) +{ + LOCAL_ASSERT(dep != NULL); + + base_jd_atom_dep_set(dep, from->atom_id, from->dependency_type); +} + +/** + * @brief Soft-atom fence trigger setup. + * + * Sets up an atom to be a SW-only atom signaling a fence + * when it reaches the run state. + * + * Using the existing base dependency system the fence can + * be set to trigger when a GPU job has finished. + * + * The base fence object must not be terminated until the atom + * has been submitted to @a base_jd_submit and @a base_jd_submit has returned. + * + * @a fence must be a valid fence set up with @a base_fence_init. + * Calling this function with a uninitialized fence results in undefined behavior. + * + * @param[out] atom A pre-allocated atom to configure as a fence trigger SW atom + * @param[in] fence The base fence object to trigger. + */ +static inline void base_jd_fence_trigger_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence) +{ + LOCAL_ASSERT(atom); + LOCAL_ASSERT(fence); + LOCAL_ASSERT(fence->basep.fd == INVALID_PLATFORM_FENCE); + LOCAL_ASSERT(fence->basep.stream_fd >= 0); + atom->jc = (uintptr_t) fence; + atom->core_req = BASE_JD_REQ_SOFT_FENCE_TRIGGER; +} + +/** + * @brief Soft-atom fence wait setup. + * + * Sets up an atom to be a SW-only atom waiting on a fence. + * When the fence becomes triggered the atom becomes runnable + * and completes immediately. + * + * Using the existing base dependency system the fence can + * be set to block a GPU job until it has been triggered. + * + * The base fence object must not be terminated until the atom + * has been submitted to @a base_jd_submit and @a base_jd_submit has returned. + * + * @a fence must be a valid fence set up with @a base_fence_init or @a base_fence_import. + * Calling this function with a uninitialized fence results in undefined behavior. + * + * @param[out] atom A pre-allocated atom to configure as a fence wait SW atom + * @param[in] fence The base fence object to wait on + */ +static inline void base_jd_fence_wait_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence) +{ + LOCAL_ASSERT(atom); + LOCAL_ASSERT(fence); + LOCAL_ASSERT(fence->basep.fd >= 0); + atom->jc = (uintptr_t) fence; + atom->core_req = BASE_JD_REQ_SOFT_FENCE_WAIT; +} + +/** + * @brief External resource info initialization. + * + * Sets up an external resource object to reference + * a memory allocation and the type of access requested. + * + * @param[in] res The resource object to initialize + * @param handle The handle to the imported memory object, must be + * obtained by calling @ref base_mem_as_import_handle(). + * @param access The type of access requested + */ +static inline void base_external_resource_init(struct base_external_resource *res, struct base_import_handle handle, base_external_resource_access access) +{ + u64 address; + + address = handle.basep.handle; + + LOCAL_ASSERT(res != NULL); + LOCAL_ASSERT(0 == (address & LOCAL_PAGE_LSB)); + LOCAL_ASSERT(access == BASE_EXT_RES_ACCESS_SHARED || access == BASE_EXT_RES_ACCESS_EXCLUSIVE); + + res->ext_resource = address | (access & LOCAL_PAGE_LSB); +} + +/** + * @brief Job chain event code bits + * Defines the bits used to create ::base_jd_event_code + */ +enum { + BASE_JD_SW_EVENT_KERNEL = (1u << 15), /**< Kernel side event */ + BASE_JD_SW_EVENT = (1u << 14), /**< SW defined event */ + BASE_JD_SW_EVENT_SUCCESS = (1u << 13), /**< Event idicates success (SW events only) */ + BASE_JD_SW_EVENT_JOB = (0u << 11), /**< Job related event */ + BASE_JD_SW_EVENT_BAG = (1u << 11), /**< Bag related event */ + BASE_JD_SW_EVENT_INFO = (2u << 11), /**< Misc/info event */ + BASE_JD_SW_EVENT_RESERVED = (3u << 11), /**< Reserved event type */ + BASE_JD_SW_EVENT_TYPE_MASK = (3u << 11) /**< Mask to extract the type from an event code */ +}; + +/** + * @brief Job chain event codes + * + * HW and low-level SW events are represented by event codes. + * The status of jobs which succeeded are also represented by + * an event code (see ::BASE_JD_EVENT_DONE). + * Events are usually reported as part of a ::base_jd_event. + * + * The event codes are encoded in the following way: + * @li 10:0 - subtype + * @li 12:11 - type + * @li 13 - SW success (only valid if the SW bit is set) + * @li 14 - SW event (HW event if not set) + * @li 15 - Kernel event (should never be seen in userspace) + * + * Events are split up into ranges as follows: + * - BASE_JD_EVENT_RANGE_\_START + * - BASE_JD_EVENT_RANGE_\_END + * + * \a code is in \'s range when: + * - BASE_JD_EVENT_RANGE_\_START <= code < BASE_JD_EVENT_RANGE_\_END + * + * Ranges can be asserted for adjacency by testing that the END of the previous + * is equal to the START of the next. This is useful for optimizing some tests + * for range. + * + * A limitation is that the last member of this enum must explicitly be handled + * (with an assert-unreachable statement) in switch statements that use + * variables of this type. Otherwise, the compiler warns that we have not + * handled that enum value. + */ +typedef enum base_jd_event_code { + /* HW defined exceptions */ + + /** Start of HW Non-fault status codes + * + * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault, + * because the job was hard-stopped + */ + BASE_JD_EVENT_RANGE_HW_NONFAULT_START = 0, + + /* non-fatal exceptions */ + BASE_JD_EVENT_NOT_STARTED = 0x00, /**< Can't be seen by userspace, treated as 'previous job done' */ + BASE_JD_EVENT_DONE = 0x01, + BASE_JD_EVENT_STOPPED = 0x03, /**< Can't be seen by userspace, becomes TERMINATED, DONE or JOB_CANCELLED */ + BASE_JD_EVENT_TERMINATED = 0x04, /**< This is actually a fault status code - the job was hard stopped */ + BASE_JD_EVENT_ACTIVE = 0x08, /**< Can't be seen by userspace, jobs only returned on complete/fail/cancel */ + + /** End of HW Non-fault status codes + * + * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault, + * because the job was hard-stopped + */ + BASE_JD_EVENT_RANGE_HW_NONFAULT_END = 0x40, + + /** Start of HW fault and SW Error status codes */ + BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START = 0x40, + + /* job exceptions */ + BASE_JD_EVENT_JOB_CONFIG_FAULT = 0x40, + BASE_JD_EVENT_JOB_POWER_FAULT = 0x41, + BASE_JD_EVENT_JOB_READ_FAULT = 0x42, + BASE_JD_EVENT_JOB_WRITE_FAULT = 0x43, + BASE_JD_EVENT_JOB_AFFINITY_FAULT = 0x44, + BASE_JD_EVENT_JOB_BUS_FAULT = 0x48, + BASE_JD_EVENT_INSTR_INVALID_PC = 0x50, + BASE_JD_EVENT_INSTR_INVALID_ENC = 0x51, + BASE_JD_EVENT_INSTR_TYPE_MISMATCH = 0x52, + BASE_JD_EVENT_INSTR_OPERAND_FAULT = 0x53, + BASE_JD_EVENT_INSTR_TLS_FAULT = 0x54, + BASE_JD_EVENT_INSTR_BARRIER_FAULT = 0x55, + BASE_JD_EVENT_INSTR_ALIGN_FAULT = 0x56, + BASE_JD_EVENT_DATA_INVALID_FAULT = 0x58, + BASE_JD_EVENT_TILE_RANGE_FAULT = 0x59, + BASE_JD_EVENT_STATE_FAULT = 0x5A, + BASE_JD_EVENT_OUT_OF_MEMORY = 0x60, + BASE_JD_EVENT_UNKNOWN = 0x7F, + + /* GPU exceptions */ + BASE_JD_EVENT_DELAYED_BUS_FAULT = 0x80, + BASE_JD_EVENT_SHAREABILITY_FAULT = 0x88, + + /* MMU exceptions */ + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1 = 0xC1, + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2 = 0xC2, + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3 = 0xC3, + BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4 = 0xC4, + BASE_JD_EVENT_PERMISSION_FAULT = 0xC8, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1 = 0xD1, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2 = 0xD2, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3 = 0xD3, + BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4 = 0xD4, + BASE_JD_EVENT_ACCESS_FLAG = 0xD8, + + /* SW defined exceptions */ + BASE_JD_EVENT_MEM_GROWTH_FAILED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x000, + BASE_JD_EVENT_TIMED_OUT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x001, + BASE_JD_EVENT_JOB_CANCELLED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x002, + BASE_JD_EVENT_JOB_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x003, + BASE_JD_EVENT_PM_EVENT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x004, + BASE_JD_EVENT_FORCE_REPLAY = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x005, + + BASE_JD_EVENT_BAG_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_BAG | 0x003, + + /** End of HW fault and SW Error status codes */ + BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_RESERVED | 0x3FF, + + /** Start of SW Success status codes */ + BASE_JD_EVENT_RANGE_SW_SUCCESS_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | 0x000, + + BASE_JD_EVENT_PROGRESS_REPORT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_JOB | 0x000, + BASE_JD_EVENT_BAG_DONE = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_BAG | 0x000, + BASE_JD_EVENT_DRV_TERMINATED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_INFO | 0x000, + + /** End of SW Success status codes */ + BASE_JD_EVENT_RANGE_SW_SUCCESS_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_RESERVED | 0x3FF, + + /** Start of Kernel-only status codes. Such codes are never returned to user-space */ + BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | 0x000, + BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x000, + + /** End of Kernel-only status codes. */ + BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_RESERVED | 0x3FF +} base_jd_event_code; + +/** + * @brief Event reporting structure + * + * This structure is used by the kernel driver to report information + * about GPU events. The can either be HW-specific events or low-level + * SW events, such as job-chain completion. + * + * The event code contains an event type field which can be extracted + * by ANDing with ::BASE_JD_SW_EVENT_TYPE_MASK. + * + * Based on the event type base_jd_event::data holds: + * @li ::BASE_JD_SW_EVENT_JOB : the offset in the ring-buffer for the completed + * job-chain + * @li ::BASE_JD_SW_EVENT_BAG : The address of the ::base_jd_bag that has + * been completed (ie all contained job-chains have been completed). + * @li ::BASE_JD_SW_EVENT_INFO : base_jd_event::data not used + */ +typedef struct base_jd_event_v2 { + base_jd_event_code event_code; /**< event code */ + base_atom_id atom_number; /**< the atom number that has completed */ + struct base_jd_udata udata; /**< user data */ +} base_jd_event_v2; + +/** + * Padding required to ensure that the @ref struct base_dump_cpu_gpu_counters structure fills + * a full cache line. + */ + +#define BASE_CPU_GPU_CACHE_LINE_PADDING (36) + + +/** + * @brief Structure for BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS jobs. + * + * This structure is stored into the memory pointed to by the @c jc field of @ref base_jd_atom. + * + * This structure must be padded to ensure that it will occupy whole cache lines. This is to avoid + * cases where access to pages containing the structure is shared between cached and un-cached + * memory regions, which would cause memory corruption. Here we set the structure size to be 64 bytes + * which is the cache line for ARM A15 processors. + */ + +typedef struct base_dump_cpu_gpu_counters { + u64 system_time; + u64 cycle_counter; + u64 sec; + u32 usec; + u8 padding[BASE_CPU_GPU_CACHE_LINE_PADDING]; +} base_dump_cpu_gpu_counters; + + + +/** @} end group base_user_api_job_dispatch */ + +#define GPU_MAX_JOB_SLOTS 16 + +/** + * @page page_base_user_api_gpuprops User-side Base GPU Property Query API + * + * The User-side Base GPU Property Query API encapsulates two + * sub-modules: + * + * - @ref base_user_api_gpuprops_dyn "Dynamic GPU Properties" + * - @ref base_plat_config_gpuprops "Base Platform Config GPU Properties" + * + * There is a related third module outside of Base, which is owned by the MIDG + * module: + * - @ref gpu_props_static "Midgard Compile-time GPU Properties" + * + * Base only deals with properties that vary between different Midgard + * implementations - the Dynamic GPU properties and the Platform Config + * properties. + * + * For properties that are constant for the Midgard Architecture, refer to the + * MIDG module. However, we will discuss their relevance here just to + * provide background information. + * + * @section sec_base_user_api_gpuprops_about About the GPU Properties in Base and MIDG modules + * + * The compile-time properties (Platform Config, Midgard Compile-time + * properties) are exposed as pre-processor macros. + * + * Complementing the compile-time properties are the Dynamic GPU + * Properties, which act as a conduit for the Midgard Configuration + * Discovery. + * + * In general, the dynamic properties are present to verify that the platform + * has been configured correctly with the right set of Platform Config + * Compile-time Properties. + * + * As a consistent guide across the entire DDK, the choice for dynamic or + * compile-time should consider the following, in order: + * -# Can the code be written so that it doesn't need to know the + * implementation limits at all? + * -# If you need the limits, get the information from the Dynamic Property + * lookup. This should be done once as you fetch the context, and then cached + * as part of the context data structure, so it's cheap to access. + * -# If there's a clear and arguable inefficiency in using Dynamic Properties, + * then use a Compile-Time Property (Platform Config, or Midgard Compile-time + * property). Examples of where this might be sensible follow: + * - Part of a critical inner-loop + * - Frequent re-use throughout the driver, causing significant extra load + * instructions or control flow that would be worthwhile optimizing out. + * + * We cannot provide an exhaustive set of examples, neither can we provide a + * rule for every possible situation. Use common sense, and think about: what + * the rest of the driver will be doing; how the compiler might represent the + * value if it is a compile-time constant; whether an OEM shipping multiple + * devices would benefit much more from a single DDK binary, instead of + * insignificant micro-optimizations. + * + * @section sec_base_user_api_gpuprops_dyn Dynamic GPU Properties + * + * Dynamic GPU properties are presented in two sets: + * -# the commonly used properties in @ref base_gpu_props, which have been + * unpacked from GPU register bitfields. + * -# The full set of raw, unprocessed properties in @ref gpu_raw_gpu_props + * (also a member of @ref base_gpu_props). All of these are presented in + * the packed form, as presented by the GPU registers themselves. + * + * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to + * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device + * behaving differently?". In this case, all information about the + * configuration is potentially useful, but it does not need to be processed + * by the driver. Instead, the raw registers can be processed by the Mali + * Tools software on the host PC. + * + * The properties returned extend the Midgard Configuration Discovery + * registers. For example, GPU clock speed is not specified in the Midgard + * Architecture, but is necessary for OpenCL's clGetDeviceInfo() function. + * + * The GPU properties are obtained by a call to + * _mali_base_get_gpu_props(). This simply returns a pointer to a const + * base_gpu_props structure. It is constant for the life of a base + * context. Multiple calls to _mali_base_get_gpu_props() to a base context + * return the same pointer to a constant structure. This avoids cache pollution + * of the common data. + * + * This pointer must not be freed, because it does not point to the start of a + * region allocated by the memory allocator; instead, just close the @ref + * base_context. + * + * + * @section sec_base_user_api_gpuprops_config Platform Config Compile-time Properties + * + * The Platform Config File sets up gpu properties that are specific to a + * certain platform. Properties that are 'Implementation Defined' in the + * Midgard Architecture spec are placed here. + * + * @note Reference configurations are provided for Midgard Implementations, such as + * the Mali-T600 family. The customer need not repeat this information, and can select one of + * these reference configurations. For example, VA_BITS, PA_BITS and the + * maximum number of samples per pixel might vary between Midgard Implementations, but + * \b not for platforms using the Mali-T604. This information is placed in + * the reference configuration files. + * + * The System Integrator creates the following structure: + * - platform_XYZ + * - platform_XYZ/plat + * - platform_XYZ/plat/plat_config.h + * + * They then edit plat_config.h, using the example plat_config.h files as a + * guide. + * + * At the very least, the customer must set @ref CONFIG_GPU_CORE_TYPE, and will + * receive a helpful \#error message if they do not do this correctly. This + * selects the Reference Configuration for the Midgard Implementation. The rationale + * behind this decision (against asking the customer to write \#include + * in their plat_config.h) is as follows: + * - This mechanism 'looks' like a regular config file (such as Linux's + * .config) + * - It is difficult to get wrong in a way that will produce strange build + * errors: + * - They need not know where the mali_t600.h, other_midg_gpu.h etc. files are stored - and + * so they won't accidentally pick another file with 'mali_t600' in its name + * - When the build doesn't work, the System Integrator may think the DDK is + * doesn't work, and attempt to fix it themselves: + * - For the @ref CONFIG_GPU_CORE_TYPE mechanism, the only way to get past the + * error is to set @ref CONFIG_GPU_CORE_TYPE, and this is what the \#error tells + * you. + * - For a \#include mechanism, checks must still be made elsewhere, which the + * System Integrator may try working around by setting \#defines (such as + * VA_BITS) themselves in their plat_config.h. In the worst case, they may + * set the prevention-mechanism \#define of + * "A_CORRECT_MIDGARD_CORE_WAS_CHOSEN". + * - In this case, they would believe they are on the right track, because + * the build progresses with their fix, but with errors elsewhere. + * + * However, there is nothing to prevent the customer using \#include to organize + * their own configurations files hierarchically. + * + * The mechanism for the header file processing is as follows: + * + * @dot + digraph plat_config_mechanism { + rankdir=BT + size="6,6" + + "mali_base.h"; + "gpu/mali_gpu.h"; + + node [ shape=box ]; + { + rank = same; ordering = out; + + "gpu/mali_gpu_props.h"; + "base/midg_gpus/mali_t600.h"; + "base/midg_gpus/other_midg_gpu.h"; + } + { rank = same; "plat/plat_config.h"; } + { + rank = same; + "gpu/mali_gpu.h" [ shape=box ]; + gpu_chooser [ label="" style="invisible" width=0 height=0 fixedsize=true ]; + select_gpu [ label="Mali-T600 | Other\n(select_gpu.h)" shape=polygon,sides=4,distortion=0.25 width=3.3 height=0.99 fixedsize=true ] ; + } + node [ shape=box ]; + { rank = same; "plat/plat_config.h"; } + { rank = same; "mali_base.h"; } + + "mali_base.h" -> "gpu/mali_gpu.h" -> "gpu/mali_gpu_props.h"; + "mali_base.h" -> "plat/plat_config.h" ; + "mali_base.h" -> select_gpu ; + + "plat/plat_config.h" -> gpu_chooser [style="dotted,bold" dir=none weight=4] ; + gpu_chooser -> select_gpu [style="dotted,bold"] ; + + select_gpu -> "base/midg_gpus/mali_t600.h" ; + select_gpu -> "base/midg_gpus/other_midg_gpu.h" ; + } + @enddot + * + * + * @section sec_base_user_api_gpuprops_kernel Kernel Operation + * + * During Base Context Create time, user-side makes a single kernel call: + * - A call to fill user memory with GPU information structures + * + * The kernel-side will fill the provided the entire processed @ref base_gpu_props + * structure, because this information is required in both + * user and kernel side; it does not make sense to decode it twice. + * + * Coherency groups must be derived from the bitmasks, but this can be done + * kernel side, and just once at kernel startup: Coherency groups must already + * be known kernel-side, to support chains that specify a 'Only Coherent Group' + * SW requirement, or 'Only Coherent Group with Tiler' SW requirement. + * + * @section sec_base_user_api_gpuprops_cocalc Coherency Group calculation + * Creation of the coherent group data is done at device-driver startup, and so + * is one-time. This will most likely involve a loop with CLZ, shifting, and + * bit clearing on the L2_PRESENT mask, depending on whether the + * system is L2 Coherent. The number of shader cores is done by a + * population count, since faulty cores may be disabled during production, + * producing a non-contiguous mask. + * + * The memory requirements for this algorithm can be determined either by a u64 + * population count on the L2_PRESENT mask (a LUT helper already is + * required for the above), or simple assumption that there can be no more than + * 16 coherent groups, since core groups are typically 4 cores. + */ + +/** + * @addtogroup base_user_api_gpuprops User-side Base GPU Property Query APIs + * @{ + */ + +/** + * @addtogroup base_user_api_gpuprops_dyn Dynamic HW Properties + * @{ + */ + +#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3 + +#define BASE_MAX_COHERENT_GROUPS 16 + +struct mali_base_gpu_core_props { + /** + * Product specific value. + */ + u32 product_id; + + /** + * Status of the GPU release. + * No defined values, but starts at 0 and increases by one for each + * release status (alpha, beta, EAC, etc.). + * 4 bit values (0-15). + */ + u16 version_status; + + /** + * Minor release number of the GPU. "P" part of an "RnPn" release number. + * 8 bit values (0-255). + */ + u16 minor_revision; + + /** + * Major release number of the GPU. "R" part of an "RnPn" release number. + * 4 bit values (0-15). + */ + u16 major_revision; + + u16 padding; + + /** + * This property is deprecated since it has not contained the real current + * value of GPU clock speed. It is kept here only for backwards compatibility. + * For the new ioctl interface, it is ignored and is treated as a padding + * to keep the structure of the same size and retain the placement of its + * members. + */ + u32 gpu_speed_mhz; + + /** + * @usecase GPU clock max/min speed is required for computing best/worst case + * in tasks as job scheduling ant irq_throttling. (It is not specified in the + * Midgard Architecture). + * Also, GPU clock max speed is used for OpenCL's clGetDeviceInfo() function. + */ + u32 gpu_freq_khz_max; + u32 gpu_freq_khz_min; + + /** + * Size of the shader program counter, in bits. + */ + u32 log2_program_counter_size; + + /** + * TEXTURE_FEATURES_x registers, as exposed by the GPU. This is a + * bitpattern where a set bit indicates that the format is supported. + * + * Before using a texture format, it is recommended that the corresponding + * bit be checked. + */ + u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; + + /** + * Theoretical maximum memory available to the GPU. It is unlikely that a + * client will be able to allocate all of this memory for their own + * purposes, but this at least provides an upper bound on the memory + * available to the GPU. + * + * This is required for OpenCL's clGetDeviceInfo() call when + * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The + * client will not be expecting to allocate anywhere near this value. + */ + u64 gpu_available_memory_size; +}; + +/** + * + * More information is possible - but associativity and bus width are not + * required by upper-level apis. + */ +struct mali_base_gpu_l2_cache_props { + u8 log2_line_size; + u8 log2_cache_size; + u8 num_l2_slices; /* Number of L2C slices. 1 or higher */ + u8 padding[5]; +}; + +struct mali_base_gpu_tiler_props { + u32 bin_size_bytes; /* Max is 4*2^15 */ + u32 max_active_levels; /* Max is 2^15 */ +}; + +/** + * GPU threading system details. + */ +struct mali_base_gpu_thread_props { + u32 max_threads; /* Max. number of threads per core */ + u32 max_workgroup_size; /* Max. number of threads per workgroup */ + u32 max_barrier_size; /* Max. number of threads that can synchronize on a simple barrier */ + u16 max_registers; /* Total size [1..65535] of the register file available per core. */ + u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */ + u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */ + u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */ + u8 padding[7]; +}; + +/** + * @brief descriptor for a coherent group + * + * \c core_mask exposes all cores in that coherent group, and \c num_cores + * provides a cached population-count for that mask. + * + * @note Whilst all cores are exposed in the mask, not all may be available to + * the application, depending on the Kernel Power policy. + * + * @note if u64s must be 8-byte aligned, then this structure has 32-bits of wastage. + */ +struct mali_base_gpu_coherent_group { + u64 core_mask; /**< Core restriction mask required for the group */ + u16 num_cores; /**< Number of cores in the group */ + u16 padding[3]; +}; + +/** + * @brief Coherency group information + * + * Note that the sizes of the members could be reduced. However, the \c group + * member might be 8-byte aligned to ensure the u64 core_mask is 8-byte + * aligned, thus leading to wastage if the other members sizes were reduced. + * + * The groups are sorted by core mask. The core masks are non-repeating and do + * not intersect. + */ +struct mali_base_gpu_coherent_group_info { + u32 num_groups; + + /** + * Number of core groups (coherent or not) in the GPU. Equivalent to the number of L2 Caches. + * + * The GPU Counter dumping writes 2048 bytes per core group, regardless of + * whether the core groups are coherent or not. Hence this member is needed + * to calculate how much memory is required for dumping. + * + * @note Do not use it to work out how many valid elements are in the + * group[] member. Use num_groups instead. + */ + u32 num_core_groups; + + /** + * Coherency features of the memory, accessed by @ref gpu_mem_features + * methods + */ + u32 coherency; + + u32 padding; + + /** + * Descriptors of coherent groups + */ + struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS]; +}; + +/** + * A complete description of the GPU's Hardware Configuration Discovery + * registers. + * + * The information is presented inefficiently for access. For frequent access, + * the values should be better expressed in an unpacked form in the + * base_gpu_props structure. + * + * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to + * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device + * behaving differently?". In this case, all information about the + * configuration is potentially useful, but it does not need to be processed + * by the driver. Instead, the raw registers can be processed by the Mali + * Tools software on the host PC. + * + */ +struct gpu_raw_gpu_props { + u64 shader_present; + u64 tiler_present; + u64 l2_present; + u64 stack_present; + + u32 l2_features; + u32 suspend_size; /* API 8.2+ */ + u32 mem_features; + u32 mmu_features; + + u32 as_present; + + u32 js_present; + u32 js_features[GPU_MAX_JOB_SLOTS]; + u32 tiler_features; + u32 texture_features[3]; + + u32 gpu_id; + + u32 thread_max_threads; + u32 thread_max_workgroup_size; + u32 thread_max_barrier_size; + u32 thread_features; + + /* + * Note: This is the _selected_ coherency mode rather than the + * available modes as exposed in the coherency_features register. + */ + u32 coherency_mode; +}; + +/** + * Return structure for _mali_base_get_gpu_props(). + * + * NOTE: the raw_props member in this data structure contains the register + * values from which the value of the other members are derived. The derived + * members exist to allow for efficient access and/or shielding the details + * of the layout of the registers. + * + */ +typedef struct mali_base_gpu_props { + struct mali_base_gpu_core_props core_props; + struct mali_base_gpu_l2_cache_props l2_props; + u64 unused_1; /* keep for backwards compatibility */ + struct mali_base_gpu_tiler_props tiler_props; + struct mali_base_gpu_thread_props thread_props; + + /** This member is large, likely to be 128 bytes */ + struct gpu_raw_gpu_props raw_props; + + /** This must be last member of the structure */ + struct mali_base_gpu_coherent_group_info coherency_info; +} base_gpu_props; + +/** @} end group base_user_api_gpuprops_dyn */ + +/** @} end group base_user_api_gpuprops */ + +/** + * @addtogroup base_user_api_core User-side Base core APIs + * @{ + */ + +/** + * \enum base_context_create_flags + * + * Flags to pass to ::base_context_init. + * Flags can be ORed together to enable multiple things. + * + * These share the same space as BASEP_CONTEXT_FLAG_*, and so must + * not collide with them. + */ +enum base_context_create_flags { + /** No flags set */ + BASE_CONTEXT_CREATE_FLAG_NONE = 0, + + /** Base context is embedded in a cctx object (flag used for CINSTR software counter macros) */ + BASE_CONTEXT_CCTX_EMBEDDED = (1u << 0), + + /** Base context is a 'System Monitor' context for Hardware counters. + * + * One important side effect of this is that job submission is disabled. */ + BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED = (1u << 1) +}; + +/** + * Bitpattern describing the ::base_context_create_flags that can be passed to base_context_init() + */ +#define BASE_CONTEXT_CREATE_ALLOWED_FLAGS \ + (((u32)BASE_CONTEXT_CCTX_EMBEDDED) | \ + ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED)) + +/** + * Bitpattern describing the ::base_context_create_flags that can be passed to the kernel + */ +#define BASE_CONTEXT_CREATE_KERNEL_FLAGS \ + ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) + +/* + * Private flags used on the base context + * + * These start at bit 31, and run down to zero. + * + * They share the same space as @ref base_context_create_flags, and so must + * not collide with them. + */ +/** Private flag tracking whether job descriptor dumping is disabled */ +#define BASEP_CONTEXT_FLAG_JOB_DUMP_DISABLED ((u32)(1 << 31)) + +/** @} end group base_user_api_core */ + +/** @} end group base_user_api */ + +/** + * @addtogroup base_plat_config_gpuprops Base Platform Config GPU Properties + * @{ + * + * C Pre-processor macros are exposed here to do with Platform + * Config. + * + * These include: + * - GPU Properties that are constant on a particular Midgard Family + * Implementation e.g. Maximum samples per pixel on Mali-T600. + * - General platform config for the GPU, such as the GPU major and minor + * revison. + */ + +/** @} end group base_plat_config_gpuprops */ + +/** + * @addtogroup base_api Base APIs + * @{ + */ + +/** + * @brief The payload for a replay job. This must be in GPU memory. + */ +typedef struct base_jd_replay_payload { + /** + * Pointer to the first entry in the base_jd_replay_jc list. These + * will be replayed in @b reverse order (so that extra ones can be added + * to the head in future soft jobs without affecting this soft job) + */ + u64 tiler_jc_list; + + /** + * Pointer to the fragment job chain. + */ + u64 fragment_jc; + + /** + * Pointer to the tiler heap free FBD field to be modified. + */ + u64 tiler_heap_free; + + /** + * Hierarchy mask for the replayed fragment jobs. May be zero. + */ + u16 fragment_hierarchy_mask; + + /** + * Hierarchy mask for the replayed tiler jobs. May be zero. + */ + u16 tiler_hierarchy_mask; + + /** + * Default weight to be used for hierarchy levels not in the original + * mask. + */ + u32 hierarchy_default_weight; + + /** + * Core requirements for the tiler job chain + */ + base_jd_core_req tiler_core_req; + + /** + * Core requirements for the fragment job chain + */ + base_jd_core_req fragment_core_req; +} base_jd_replay_payload; + +#ifdef BASE_LEGACY_UK10_2_SUPPORT +typedef struct base_jd_replay_payload_uk10_2 { + u64 tiler_jc_list; + u64 fragment_jc; + u64 tiler_heap_free; + u16 fragment_hierarchy_mask; + u16 tiler_hierarchy_mask; + u32 hierarchy_default_weight; + u16 tiler_core_req; + u16 fragment_core_req; + u8 padding[4]; +} base_jd_replay_payload_uk10_2; +#endif /* BASE_LEGACY_UK10_2_SUPPORT */ + +/** + * @brief An entry in the linked list of job chains to be replayed. This must + * be in GPU memory. + */ +typedef struct base_jd_replay_jc { + /** + * Pointer to next entry in the list. A setting of NULL indicates the + * end of the list. + */ + u64 next; + + /** + * Pointer to the job chain. + */ + u64 jc; + +} base_jd_replay_jc; + +/* Maximum number of jobs allowed in a fragment chain in the payload of a + * replay job */ +#define BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT 256 + +/** @} end group base_api */ + +typedef struct base_profiling_controls { + u32 profiling_controls[FBDUMP_CONTROL_MAX]; +} base_profiling_controls; + +/* Enable additional tracepoints for latency measurements (TL_ATOM_READY, + * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) */ +#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) + +/* Indicate that job dumping is enabled. This could affect certain timers + * to account for the performance impact. */ +#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) + +#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ + BASE_TLSTREAM_JOB_DUMPING_ENABLED) + +#endif /* _BASE_KERNEL_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_base_mem_priv.h b/drivers/gpu/arm/midgard/mali_base_mem_priv.h new file mode 100755 index 000000000000..4a98a72cc37a --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_base_mem_priv.h @@ -0,0 +1,52 @@ +/* + * + * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _BASE_MEM_PRIV_H_ +#define _BASE_MEM_PRIV_H_ + +#define BASE_SYNCSET_OP_MSYNC (1U << 0) +#define BASE_SYNCSET_OP_CSYNC (1U << 1) + +/* + * This structure describe a basic memory coherency operation. + * It can either be: + * @li a sync from CPU to Memory: + * - type = ::BASE_SYNCSET_OP_MSYNC + * - mem_handle = a handle to the memory object on which the operation + * is taking place + * - user_addr = the address of the range to be synced + * - size = the amount of data to be synced, in bytes + * - offset is ignored. + * @li a sync from Memory to CPU: + * - type = ::BASE_SYNCSET_OP_CSYNC + * - mem_handle = a handle to the memory object on which the operation + * is taking place + * - user_addr = the address of the range to be synced + * - size = the amount of data to be synced, in bytes. + * - offset is ignored. + */ +struct basep_syncset { + base_mem_handle mem_handle; + u64 user_addr; + u64 size; + u8 type; + u8 padding[7]; +}; + +#endif diff --git a/drivers/gpu/arm/midgard/mali_base_vendor_specific_func.h b/drivers/gpu/arm/midgard/mali_base_vendor_specific_func.h new file mode 100755 index 000000000000..be454a216a39 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_base_vendor_specific_func.h @@ -0,0 +1,24 @@ +/* + * + * (C) COPYRIGHT 2010, 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +#ifndef _BASE_VENDOR_SPEC_FUNC_H_ +#define _BASE_VENDOR_SPEC_FUNC_H_ + +int kbase_get_vendor_specific_cpu_clock_speed(u32 * const); + +#endif /*_BASE_VENDOR_SPEC_FUNC_H_*/ diff --git a/drivers/gpu/arm/midgard/mali_kbase.h b/drivers/gpu/arm/midgard/mali_kbase.h new file mode 100755 index 000000000000..0d9bf23dc685 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase.h @@ -0,0 +1,612 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_H_ +#define _KBASE_H_ + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mali_base_kernel.h" +#include +#include + +/* + * Include mali_kbase_defs.h first as this provides types needed by other local + * header files. + */ +#include "mali_kbase_defs.h" + +#include "mali_kbase_context.h" +#include "mali_kbase_strings.h" +#include "mali_kbase_mem_lowlevel.h" +#include "mali_kbase_trace_timeline.h" +#include "mali_kbase_js.h" +#include "mali_kbase_mem.h" +#include "mali_kbase_utility.h" +#include "mali_kbase_gpu_memory_debugfs.h" +#include "mali_kbase_mem_profile_debugfs.h" +#include "mali_kbase_debug_job_fault.h" +#include "mali_kbase_jd_debugfs.h" +#include "mali_kbase_gpuprops.h" +#include "mali_kbase_jm.h" +#include "mali_kbase_vinstr.h" + +#include "ipa/mali_kbase_ipa.h" + +#ifdef CONFIG_GPU_TRACEPOINTS +#include +#endif +/** + * @page page_base_kernel_main Kernel-side Base (KBase) APIs + */ + +/** + * @defgroup base_kbase_api Kernel-side Base (KBase) APIs + */ + +struct kbase_device *kbase_device_alloc(void); +/* +* note: configuration attributes member of kbdev needs to have +* been setup before calling kbase_device_init +*/ + +/* +* API to acquire device list semaphore and return pointer +* to the device list head +*/ +const struct list_head *kbase_dev_list_get(void); +/* API to release the device list semaphore */ +void kbase_dev_list_put(const struct list_head *dev_list); + +int kbase_device_init(struct kbase_device * const kbdev); +void kbase_device_term(struct kbase_device *kbdev); +void kbase_device_free(struct kbase_device *kbdev); +int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature); + +/* Needed for gator integration and for reporting vsync information */ +struct kbase_device *kbase_find_device(int minor); +void kbase_release_device(struct kbase_device *kbdev); + +void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value); + +struct kbase_context * +kbase_create_context(struct kbase_device *kbdev, bool is_compat); +void kbase_destroy_context(struct kbase_context *kctx); + +int kbase_jd_init(struct kbase_context *kctx); +void kbase_jd_exit(struct kbase_context *kctx); + +/** + * kbase_jd_submit - Submit atoms to the job dispatcher + * + * @kctx: The kbase context to submit to + * @user_addr: The address in user space of the struct base_jd_atom_v2 array + * @nr_atoms: The number of atoms in the array + * @stride: sizeof(struct base_jd_atom_v2) + * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) + * + * Return: 0 on success or error code + */ +int kbase_jd_submit(struct kbase_context *kctx, + void __user *user_addr, u32 nr_atoms, u32 stride, + bool uk6_atom); + +/** + * kbase_jd_done_worker - Handle a job completion + * @data: a &struct work_struct + * + * This function requeues the job from the runpool (if it was soft-stopped or + * removed from NEXT registers). + * + * Removes it from the system if it finished/failed/was cancelled. + * + * Resolves dependencies to add dependent jobs to the context, potentially + * starting them if necessary (which may add more references to the context) + * + * Releases the reference to the context from the no-longer-running job. + * + * Handles retrying submission outside of IRQ context if it failed from within + * IRQ context. + */ +void kbase_jd_done_worker(struct work_struct *data); + +void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp, + kbasep_js_atom_done_code done_code); +void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom); +void kbase_jd_zap_context(struct kbase_context *kctx); +bool jd_done_nolock(struct kbase_jd_atom *katom, + struct list_head *completed_jobs_ctx); +void kbase_jd_free_external_resources(struct kbase_jd_atom *katom); +bool jd_submit_atom(struct kbase_context *kctx, + const struct base_jd_atom_v2 *user_atom, + struct kbase_jd_atom *katom); +void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom); + +void kbase_job_done(struct kbase_device *kbdev, u32 done); + +/** + * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms + * and soft stop them + * @kctx: Pointer to context to check. + * @katom: Pointer to priority atom. + * + * Atoms from @kctx on the same job slot as @katom, which have lower priority + * than @katom will be soft stopped and put back in the queue, so that atoms + * with higher priority can run. + * + * The hwaccess_lock must be held when calling this function. + */ +void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom); +void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, + struct kbase_jd_atom *target_katom, u32 sw_flags); +void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom); +void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, + base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom); +void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, + struct kbase_jd_atom *target_katom); + +void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event); +int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent); +int kbase_event_pending(struct kbase_context *ctx); +int kbase_event_init(struct kbase_context *kctx); +void kbase_event_close(struct kbase_context *kctx); +void kbase_event_cleanup(struct kbase_context *kctx); +void kbase_event_wakeup(struct kbase_context *kctx); + +int kbase_process_soft_job(struct kbase_jd_atom *katom); +int kbase_prepare_soft_job(struct kbase_jd_atom *katom); +void kbase_finish_soft_job(struct kbase_jd_atom *katom); +void kbase_cancel_soft_job(struct kbase_jd_atom *katom); +void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev); +void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom); +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom); +#endif +int kbase_soft_event_update(struct kbase_context *kctx, + u64 event, + unsigned char new_status); + +bool kbase_replay_process(struct kbase_jd_atom *katom); + +void kbasep_soft_job_timeout_worker(struct timer_list *t); +void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt); + +/* api used internally for register access. Contains validation and tracing */ +void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value); +int kbase_device_trace_buffer_install( + struct kbase_context *kctx, u32 *tb, size_t size); +void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx); + +/* api to be ported per OS, only need to do the raw register access */ +void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value); +u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset); + +void kbasep_as_do_poke(struct work_struct *work); + +/** Returns the name associated with a Mali exception code + * + * This function is called from the interrupt handler when a GPU fault occurs. + * It reports the details of the fault using KBASE_DEBUG_PRINT_WARN. + * + * @param[in] kbdev The kbase device that the GPU fault occurred from. + * @param[in] exception_code exception code + * @return name associated with the exception code + */ +const char *kbase_exception_name(struct kbase_device *kbdev, + u32 exception_code); + +/** + * Check whether a system suspend is in progress, or has already been suspended + * + * The caller should ensure that either kbdev->pm.active_count_lock is held, or + * a dmb was executed recently (to ensure the value is most + * up-to-date). However, without a lock the value could change afterwards. + * + * @return false if a suspend is not in progress + * @return !=false otherwise + */ +static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) +{ + return kbdev->pm.suspending; +} + +/** + * Return the atom's ID, as was originally supplied by userspace in + * base_jd_atom_v2::atom_number + */ +static inline int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + int result; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(katom->kctx == kctx); + + result = katom - &kctx->jctx.atoms[0]; + KBASE_DEBUG_ASSERT(result >= 0 && result <= BASE_JD_ATOM_COUNT); + return result; +} + +/** + * kbase_jd_atom_from_id - Return the atom structure for the given atom ID + * @kctx: Context pointer + * @id: ID of atom to retrieve + * + * Return: Pointer to struct kbase_jd_atom associated with the supplied ID + */ +static inline struct kbase_jd_atom *kbase_jd_atom_from_id( + struct kbase_context *kctx, int id) +{ + return &kctx->jctx.atoms[id]; +} + +/** + * Initialize the disjoint state + * + * The disjoint event count and state are both set to zero. + * + * Disjoint functions usage: + * + * The disjoint event count should be incremented whenever a disjoint event occurs. + * + * There are several cases which are regarded as disjoint behavior. Rather than just increment + * the counter during disjoint events we also increment the counter when jobs may be affected + * by what the GPU is currently doing. To facilitate this we have the concept of disjoint state. + * + * Disjoint state is entered during GPU reset and for the entire time that an atom is replaying + * (as part of the replay workaround). Increasing the disjoint state also increases the count of + * disjoint events. + * + * The disjoint state is then used to increase the count of disjoint events during job submission + * and job completion. Any atom submitted or completed while the disjoint state is greater than + * zero is regarded as a disjoint event. + * + * The disjoint event counter is also incremented immediately whenever a job is soft stopped + * and during context creation. + * + * @param kbdev The kbase device + */ +void kbase_disjoint_init(struct kbase_device *kbdev); + +/** + * Increase the count of disjoint events + * called when a disjoint event has happened + * + * @param kbdev The kbase device + */ +void kbase_disjoint_event(struct kbase_device *kbdev); + +/** + * Increase the count of disjoint events only if the GPU is in a disjoint state + * + * This should be called when something happens which could be disjoint if the GPU + * is in a disjoint state. The state refcount keeps track of this. + * + * @param kbdev The kbase device + */ +void kbase_disjoint_event_potential(struct kbase_device *kbdev); + +/** + * Returns the count of disjoint events + * + * @param kbdev The kbase device + * @return the count of disjoint events + */ +u32 kbase_disjoint_event_get(struct kbase_device *kbdev); + +/** + * Increment the refcount state indicating that the GPU is in a disjoint state. + * + * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) + * eventually after the disjoint state has completed @ref kbase_disjoint_state_down + * should be called + * + * @param kbdev The kbase device + */ +void kbase_disjoint_state_up(struct kbase_device *kbdev); + +/** + * Decrement the refcount state + * + * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) + * + * Called after @ref kbase_disjoint_state_up once the disjoint state is over + * + * @param kbdev The kbase device + */ +void kbase_disjoint_state_down(struct kbase_device *kbdev); + +/** + * If a job is soft stopped and the number of contexts is >= this value + * it is reported as a disjoint event + */ +#define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2 + +#if !defined(UINT64_MAX) + #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL) +#endif + +#if KBASE_TRACE_ENABLE +void kbasep_trace_debugfs_init(struct kbase_device *kbdev); + +#ifndef CONFIG_MALI_SYSTEM_TRACE +/** Add trace values about a job-slot + * + * @note Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any + * functions called to get the parameters supplied to this macro must: + * - be static or static inline + * - must just return 0 and have no other statements present in the body. + */ +#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot) \ + kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ + KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, 0) + +/** Add trace values about a job-slot, with info + * + * @note Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any + * functions called to get the parameters supplied to this macro must: + * - be static or static inline + * - must just return 0 and have no other statements present in the body. + */ +#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val) \ + kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ + KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, info_val) + +/** Add trace values about a ctx refcount + * + * @note Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any + * functions called to get the parameters supplied to this macro must: + * - be static or static inline + * - must just return 0 and have no other statements present in the body. + */ +#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount) \ + kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ + KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, 0) +/** Add trace values about a ctx refcount, and info + * + * @note Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any + * functions called to get the parameters supplied to this macro must: + * - be static or static inline + * - must just return 0 and have no other statements present in the body. + */ +#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val) \ + kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ + KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, info_val) + +/** Add trace values (no slot or refcount) + * + * @note Any functions called through this macro will still be evaluated in + * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any + * functions called to get the parameters supplied to this macro must: + * - be static or static inline + * - must just return 0 and have no other statements present in the body. + */ +#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val) \ + kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ + 0, 0, 0, info_val) + +/** Clear the trace */ +#define KBASE_TRACE_CLEAR(kbdev) \ + kbasep_trace_clear(kbdev) + +/** Dump the slot trace */ +#define KBASE_TRACE_DUMP(kbdev) \ + kbasep_trace_dump(kbdev) + +/** PRIVATE - do not use directly. Use KBASE_TRACE_ADD() instead */ +void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val); +/** PRIVATE - do not use directly. Use KBASE_TRACE_CLEAR() instead */ +void kbasep_trace_clear(struct kbase_device *kbdev); +#else /* #ifndef CONFIG_MALI_SYSTEM_TRACE */ +/* Dispatch kbase trace events as system trace events */ +#include +#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ + trace_mali_##code(jobslot, 0) + +#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ + trace_mali_##code(jobslot, info_val) + +#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ + trace_mali_##code(refcount, 0) + +#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ + trace_mali_##code(refcount, info_val) + +#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val)\ + trace_mali_##code(gpu_addr, info_val) + +#define KBASE_TRACE_CLEAR(kbdev)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(0);\ + } while (0) +#define KBASE_TRACE_DUMP(kbdev)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(0);\ + } while (0) + +#endif /* #ifndef CONFIG_MALI_SYSTEM_TRACE */ +#else +#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(ctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(jobslot);\ + } while (0) + +#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(ctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(jobslot);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(ctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(refcount);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(ctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(gpu_addr);\ + CSTD_UNUSED(info_val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_TRACE_ADD(kbdev, code, subcode, ctx, katom, val)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(code);\ + CSTD_UNUSED(subcode);\ + CSTD_UNUSED(ctx);\ + CSTD_UNUSED(katom);\ + CSTD_UNUSED(val);\ + CSTD_NOP(0);\ + } while (0) + +#define KBASE_TRACE_CLEAR(kbdev)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(0);\ + } while (0) +#define KBASE_TRACE_DUMP(kbdev)\ + do {\ + CSTD_UNUSED(kbdev);\ + CSTD_NOP(0);\ + } while (0) +#endif /* KBASE_TRACE_ENABLE */ +/** PRIVATE - do not use directly. Use KBASE_TRACE_DUMP() instead */ +void kbasep_trace_dump(struct kbase_device *kbdev); + +#ifdef CONFIG_MALI_DEBUG +/** + * kbase_set_driver_inactive - Force driver to go inactive + * @kbdev: Device pointer + * @inactive: true if driver should go inactive, false otherwise + * + * Forcing the driver inactive will cause all future IOCTLs to wait until the + * driver is made active again. This is intended solely for the use of tests + * which require that no jobs are running while the test executes. + */ +void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive); +#endif /* CONFIG_MALI_DEBUG */ + + +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) + +/* kbase_io_history_init - initialize data struct for register access history + * + * @kbdev The register history to initialize + * @n The number of register accesses that the buffer could hold + * + * @return 0 if successfully initialized, failure otherwise + */ +int kbase_io_history_init(struct kbase_io_history *h, u16 n); + +/* kbase_io_history_term - uninit all resources for the register access history + * + * @h The register history to terminate + */ +void kbase_io_history_term(struct kbase_io_history *h); + +/* kbase_io_history_dump - print the register history to the kernel ring buffer + * + * @kbdev Pointer to kbase_device containing the register history to dump + */ +void kbase_io_history_dump(struct kbase_device *kbdev); + +/** + * kbase_io_history_resize - resize the register access history buffer. + * + * @h: Pointer to a valid register history to resize + * @new_size: Number of accesses the buffer could hold + * + * A successful resize will clear all recent register accesses. + * If resizing fails for any reason (e.g., could not allocate memory, invalid + * buffer size) then the original buffer will be kept intact. + * + * @return 0 if the buffer was resized, failure otherwise + */ +int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size); + +#else /* CONFIG_DEBUG_FS */ + +#define kbase_io_history_init(...) ((int)0) + +#define kbase_io_history_term CSTD_NOP + +#define kbase_io_history_dump CSTD_NOP + +#define kbase_io_history_resize CSTD_NOP + +#endif /* CONFIG_DEBUG_FS */ + + +#endif + + + diff --git a/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.c b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.c new file mode 100755 index 000000000000..fde0f8ff8582 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.c @@ -0,0 +1,209 @@ +/* + * + * (C) COPYRIGHT 2013-2015, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +#include +#include +#include + +/* This function is used to solve an HW issue with single iterator GPUs. + * If a fragment job is soft-stopped on the edge of its bounding box, can happen that the + * restart index is out of bounds and the rerun causes a tile range fault. If this happens + * we try to clamp the restart index to a correct value and rerun the job. + */ +/* Mask of X and Y coordinates for the coordinates words in the descriptors*/ +#define X_COORDINATE_MASK 0x00000FFF +#define Y_COORDINATE_MASK 0x0FFF0000 +/* Max number of words needed from the fragment shader job descriptor */ +#define JOB_HEADER_SIZE_IN_WORDS 10 +#define JOB_HEADER_SIZE (JOB_HEADER_SIZE_IN_WORDS*sizeof(u32)) + +/* Word 0: Status Word */ +#define JOB_DESC_STATUS_WORD 0 +/* Word 1: Restart Index */ +#define JOB_DESC_RESTART_INDEX_WORD 1 +/* Word 2: Fault address low word */ +#define JOB_DESC_FAULT_ADDR_LOW_WORD 2 +/* Word 8: Minimum Tile Coordinates */ +#define FRAG_JOB_DESC_MIN_TILE_COORD_WORD 8 +/* Word 9: Maximum Tile Coordinates */ +#define FRAG_JOB_DESC_MAX_TILE_COORD_WORD 9 + +int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom) +{ + struct device *dev = katom->kctx->kbdev->dev; + u32 clamped = 0; + struct kbase_va_region *region; + phys_addr_t *page_array; + u64 page_index; + u32 offset = katom->jc & (~PAGE_MASK); + u32 *page_1 = NULL; + u32 *page_2 = NULL; + u32 job_header[JOB_HEADER_SIZE_IN_WORDS]; + void *dst = job_header; + u32 minX, minY, maxX, maxY; + u32 restartX, restartY; + struct page *p; + u32 copy_size; + + dev_warn(dev, "Called TILE_RANGE_FAULT workaround clamping function.\n"); + if (!(katom->core_req & BASE_JD_REQ_FS)) + return 0; + + kbase_gpu_vm_lock(katom->kctx); + region = kbase_region_tracker_find_region_enclosing_address(katom->kctx, + katom->jc); + if (!region || (region->flags & KBASE_REG_FREE)) + goto out_unlock; + + page_array = kbase_get_cpu_phy_pages(region); + if (!page_array) + goto out_unlock; + + page_index = (katom->jc >> PAGE_SHIFT) - region->start_pfn; + + p = pfn_to_page(PFN_DOWN(page_array[page_index])); + + /* we need the first 10 words of the fragment shader job descriptor. + * We need to check that the offset + 10 words is less that the page + * size otherwise we need to load the next page. + * page_size_overflow will be equal to 0 in case the whole descriptor + * is within the page > 0 otherwise. + */ + copy_size = MIN(PAGE_SIZE - offset, JOB_HEADER_SIZE); + + page_1 = kmap_atomic(p); + + /* page_1 is a u32 pointer, offset is expressed in bytes */ + page_1 += offset>>2; + + kbase_sync_single_for_cpu(katom->kctx->kbdev, + kbase_dma_addr(p) + offset, + copy_size, DMA_BIDIRECTIONAL); + + memcpy(dst, page_1, copy_size); + + /* The data needed overflows page the dimension, + * need to map the subsequent page */ + if (copy_size < JOB_HEADER_SIZE) { + p = pfn_to_page(PFN_DOWN(page_array[page_index + 1])); + page_2 = kmap_atomic(p); + + kbase_sync_single_for_cpu(katom->kctx->kbdev, + kbase_dma_addr(p), + JOB_HEADER_SIZE - copy_size, DMA_BIDIRECTIONAL); + + memcpy(dst + copy_size, page_2, JOB_HEADER_SIZE - copy_size); + } + + /* We managed to correctly map one or two pages (in case of overflow) */ + /* Get Bounding Box data and restart index from fault address low word */ + minX = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & X_COORDINATE_MASK; + minY = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & Y_COORDINATE_MASK; + maxX = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & X_COORDINATE_MASK; + maxY = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & Y_COORDINATE_MASK; + restartX = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & X_COORDINATE_MASK; + restartY = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & Y_COORDINATE_MASK; + + dev_warn(dev, "Before Clamping:\n" + "Jobstatus: %08x\n" + "restartIdx: %08x\n" + "Fault_addr_low: %08x\n" + "minCoordsX: %08x minCoordsY: %08x\n" + "maxCoordsX: %08x maxCoordsY: %08x\n", + job_header[JOB_DESC_STATUS_WORD], + job_header[JOB_DESC_RESTART_INDEX_WORD], + job_header[JOB_DESC_FAULT_ADDR_LOW_WORD], + minX, minY, + maxX, maxY); + + /* Set the restart index to the one which generated the fault*/ + job_header[JOB_DESC_RESTART_INDEX_WORD] = + job_header[JOB_DESC_FAULT_ADDR_LOW_WORD]; + + if (restartX < minX) { + job_header[JOB_DESC_RESTART_INDEX_WORD] = (minX) | restartY; + dev_warn(dev, + "Clamping restart X index to minimum. %08x clamped to %08x\n", + restartX, minX); + clamped = 1; + } + if (restartY < minY) { + job_header[JOB_DESC_RESTART_INDEX_WORD] = (minY) | restartX; + dev_warn(dev, + "Clamping restart Y index to minimum. %08x clamped to %08x\n", + restartY, minY); + clamped = 1; + } + if (restartX > maxX) { + job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxX) | restartY; + dev_warn(dev, + "Clamping restart X index to maximum. %08x clamped to %08x\n", + restartX, maxX); + clamped = 1; + } + if (restartY > maxY) { + job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxY) | restartX; + dev_warn(dev, + "Clamping restart Y index to maximum. %08x clamped to %08x\n", + restartY, maxY); + clamped = 1; + } + + if (clamped) { + /* Reset the fault address low word + * and set the job status to STOPPED */ + job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] = 0x0; + job_header[JOB_DESC_STATUS_WORD] = BASE_JD_EVENT_STOPPED; + dev_warn(dev, "After Clamping:\n" + "Jobstatus: %08x\n" + "restartIdx: %08x\n" + "Fault_addr_low: %08x\n" + "minCoordsX: %08x minCoordsY: %08x\n" + "maxCoordsX: %08x maxCoordsY: %08x\n", + job_header[JOB_DESC_STATUS_WORD], + job_header[JOB_DESC_RESTART_INDEX_WORD], + job_header[JOB_DESC_FAULT_ADDR_LOW_WORD], + minX, minY, + maxX, maxY); + + /* Flush CPU cache to update memory for future GPU reads*/ + memcpy(page_1, dst, copy_size); + p = pfn_to_page(PFN_DOWN(page_array[page_index])); + + kbase_sync_single_for_device(katom->kctx->kbdev, + kbase_dma_addr(p) + offset, + copy_size, DMA_TO_DEVICE); + + if (copy_size < JOB_HEADER_SIZE) { + memcpy(page_2, dst + copy_size, + JOB_HEADER_SIZE - copy_size); + p = pfn_to_page(PFN_DOWN(page_array[page_index + 1])); + + kbase_sync_single_for_device(katom->kctx->kbdev, + kbase_dma_addr(p), + JOB_HEADER_SIZE - copy_size, + DMA_TO_DEVICE); + } + } + if (copy_size < JOB_HEADER_SIZE) + kunmap_atomic(page_2); + + kunmap_atomic(page_1); + +out_unlock: + kbase_gpu_vm_unlock(katom->kctx); + return clamped; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.h b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.h new file mode 100755 index 000000000000..099a29861672 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.h @@ -0,0 +1,23 @@ +/* + * + * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_10969_WORKAROUND_ +#define _KBASE_10969_WORKAROUND_ + +int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom); + +#endif /* _KBASE_10969_WORKAROUND_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c new file mode 100755 index 000000000000..f910fe970feb --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c @@ -0,0 +1,102 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include + +#include +#include + +#ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_MALI_DEBUG + +static int kbase_as_fault_read(struct seq_file *sfile, void *data) +{ + uintptr_t as_no = (uintptr_t) sfile->private; + + struct list_head *entry; + const struct list_head *kbdev_list; + struct kbase_device *kbdev = NULL; + + kbdev_list = kbase_dev_list_get(); + + list_for_each(entry, kbdev_list) { + kbdev = list_entry(entry, struct kbase_device, entry); + + if(kbdev->debugfs_as_read_bitmap & (1ULL << as_no)) { + + /* don't show this one again until another fault occors */ + kbdev->debugfs_as_read_bitmap &= ~(1ULL << as_no); + + /* output the last page fault addr */ + seq_printf(sfile, "%llu\n", (u64) kbdev->as[as_no].fault_addr); + } + + } + + kbase_dev_list_put(kbdev_list); + + return 0; +} + +static int kbase_as_fault_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbase_as_fault_read , in->i_private); +} + +static const struct file_operations as_fault_fops = { + .open = kbase_as_fault_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +#endif /* CONFIG_MALI_DEBUG */ +#endif /* CONFIG_DEBUG_FS */ + +/* + * Initialize debugfs entry for each address space + */ +void kbase_as_fault_debugfs_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_MALI_DEBUG + uint i; + char as_name[64]; + struct dentry *debugfs_directory; + + kbdev->debugfs_as_read_bitmap = 0ULL; + + KBASE_DEBUG_ASSERT(kbdev->nr_hw_address_spaces); + KBASE_DEBUG_ASSERT(sizeof(kbdev->as[0].fault_addr) == sizeof(u64)); + + debugfs_directory = debugfs_create_dir("address_spaces", + kbdev->mali_debugfs_directory); + + if(debugfs_directory) { + for(i = 0; i < kbdev->nr_hw_address_spaces; i++) { + snprintf(as_name, ARRAY_SIZE(as_name), "as%u", i); + debugfs_create_file(as_name, S_IRUGO, + debugfs_directory, (void*) ((uintptr_t) i), &as_fault_fops); + } + } + else + dev_warn(kbdev->dev, "unable to create address_spaces debugfs directory"); + +#endif /* CONFIG_MALI_DEBUG */ +#endif /* CONFIG_DEBUG_FS */ + return; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h new file mode 100755 index 000000000000..3ed2248897fc --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_AS_FAULT_DEBUG_FS_H +#define _KBASE_AS_FAULT_DEBUG_FS_H + +/** + * kbase_as_fault_debugfs_init() - Add debugfs files for reporting page faults + * + * @kbdev: Pointer to kbase_device + */ +void kbase_as_fault_debugfs_init(struct kbase_device *kbdev); + +/** + * kbase_as_fault_debugfs_new() - make the last fault available on debugfs + * + * @kbdev: Pointer to kbase_device + * @as_no: The address space the fault occurred on + */ +static inline void +kbase_as_fault_debugfs_new(struct kbase_device *kbdev, int as_no) +{ +#ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_MALI_DEBUG + kbdev->debugfs_as_read_bitmap |= (1ULL << as_no); +#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_MALI_DEBUG */ + return; +} + +#endif /*_KBASE_AS_FAULT_DEBUG_FS_H*/ diff --git a/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c new file mode 100755 index 000000000000..c67b3e97f1af --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c @@ -0,0 +1,64 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Cache Policy API. + */ + +#include "mali_kbase_cache_policy.h" + +/* + * The output flags should be a combination of the following values: + * KBASE_REG_CPU_CACHED: CPU cache should be enabled. + */ +u32 kbase_cache_enabled(u32 flags, u32 nr_pages) +{ + u32 cache_flags = 0; + + CSTD_UNUSED(nr_pages); + + if (flags & BASE_MEM_CACHED_CPU) + cache_flags |= KBASE_REG_CPU_CACHED; + + return cache_flags; +} + + +void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir) +{ +/* Check if kernel is using coherency with GPU */ +#ifdef CONFIG_MALI_COH_KERN + if (kbdev->system_coherency == COHERENCY_ACE) + return; +#endif /* CONFIG_MALI_COH_KERN */ + dma_sync_single_for_device(kbdev->dev, handle, size, dir); +} + + +void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir) +{ +/* Check if kernel is using coherency with GPU */ +#ifdef CONFIG_MALI_COH_KERN + if (kbdev->system_coherency == COHERENCY_ACE) + return; +#endif /* CONFIG_MALI_COH_KERN */ + dma_sync_single_for_cpu(kbdev->dev, handle, size, dir); +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h new file mode 100755 index 000000000000..0c18bdb357b0 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Cache Policy API. + */ + +#ifndef _KBASE_CACHE_POLICY_H_ +#define _KBASE_CACHE_POLICY_H_ + +#include "mali_kbase.h" +#include "mali_base_kernel.h" + +/** + * kbase_cache_enabled - Choose the cache policy for a specific region + * @flags: flags describing attributes of the region + * @nr_pages: total number of pages (backed or not) for the region + * + * Tells whether the CPU and GPU caches should be enabled or not for a specific + * region. + * This function can be modified to customize the cache policy depending on the + * flags and size of the region. + * + * Return: a combination of %KBASE_REG_CPU_CACHED and %KBASE_REG_GPU_CACHED + * depending on the cache policy + */ +u32 kbase_cache_enabled(u32 flags, u32 nr_pages); + +#endif /* _KBASE_CACHE_POLICY_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_config.c b/drivers/gpu/arm/midgard/mali_kbase_config.c new file mode 100755 index 000000000000..fb615ae02ead --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_config.c @@ -0,0 +1,51 @@ +/* + * + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include + +int kbasep_platform_device_init(struct kbase_device *kbdev) +{ + struct kbase_platform_funcs_conf *platform_funcs_p; + + platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; + if (platform_funcs_p && platform_funcs_p->platform_init_func) + return platform_funcs_p->platform_init_func(kbdev); + + return 0; +} + +void kbasep_platform_device_term(struct kbase_device *kbdev) +{ + struct kbase_platform_funcs_conf *platform_funcs_p; + + platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; + if (platform_funcs_p && platform_funcs_p->platform_term_func) + platform_funcs_p->platform_term_func(kbdev); +} + +int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed) +{ + KBASE_DEBUG_ASSERT(NULL != clock_speed); + + *clock_speed = 100; + return 0; +} + diff --git a/drivers/gpu/arm/midgard/mali_kbase_config.h b/drivers/gpu/arm/midgard/mali_kbase_config.h new file mode 100755 index 000000000000..356d52bcd774 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_config.h @@ -0,0 +1,345 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_config.h + * Configuration API and Attributes for KBase + */ + +#ifndef _KBASE_CONFIG_H_ +#define _KBASE_CONFIG_H_ + +#include + +#include +#include + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_config Configuration API and Attributes + * @{ + */ + +#include + +/* Forward declaration of struct kbase_device */ +struct kbase_device; + +/** + * kbase_platform_funcs_conf - Specifies platform init/term function pointers + * + * Specifies the functions pointers for platform specific initialization and + * termination. By default no functions are required. No additional platform + * specific control is necessary. + */ +struct kbase_platform_funcs_conf { + /** + * platform_init_func - platform specific init function pointer + * @kbdev - kbase_device pointer + * + * Returns 0 on success, negative error code otherwise. + * + * Function pointer for platform specific initialization or NULL if no + * initialization function is required. At the point this the GPU is + * not active and its power and clocks are in unknown (platform specific + * state) as kbase doesn't yet have control of power and clocks. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed (and possibly initialized) in here. + */ + int (*platform_init_func)(struct kbase_device *kbdev); + /** + * platform_term_func - platform specific termination function pointer + * @kbdev - kbase_device pointer + * + * Function pointer for platform specific termination or NULL if no + * termination function is required. At the point this the GPU will be + * idle but still powered and clocked. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed (and possibly terminated) in here. + */ + void (*platform_term_func)(struct kbase_device *kbdev); +}; + +/* + * @brief Specifies the callbacks for power management + * + * By default no callbacks will be made and the GPU must not be powered off. + */ +struct kbase_pm_callback_conf { + /** Callback for when the GPU is idle and the power to it can be switched off. + * + * The system integrator can decide whether to either do nothing, just switch off + * the clocks to the GPU, or to completely power down the GPU. + * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the + * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). + */ + void (*power_off_callback)(struct kbase_device *kbdev); + + /** Callback for when the GPU is about to become active and power must be supplied. + * + * This function must not return until the GPU is powered and clocked sufficiently for register access to + * succeed. The return value specifies whether the GPU was powered down since the call to power_off_callback. + * If the GPU state has been lost then this function must return 1, otherwise it should return 0. + * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the + * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). + * + * The return value of the first call to this function is ignored. + * + * @return 1 if the GPU state may have been lost, 0 otherwise. + */ + int (*power_on_callback)(struct kbase_device *kbdev); + + /** Callback for when the system is requesting a suspend and GPU power + * must be switched off. + * + * Note that if this callback is present, then this may be called + * without a preceding call to power_off_callback. Therefore this + * callback must be able to take any action that might otherwise happen + * in power_off_callback. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed and modified in here. It is the platform \em + * callbacks responsibility to initialize and terminate this pointer if + * used (see @ref kbase_platform_funcs_conf). + */ + void (*power_suspend_callback)(struct kbase_device *kbdev); + + /** Callback for when the system is resuming from a suspend and GPU + * power must be switched on. + * + * Note that if this callback is present, then this may be called + * without a following call to power_on_callback. Therefore this + * callback must be able to take any action that might otherwise happen + * in power_on_callback. + * + * The platform specific private pointer kbase_device::platform_context + * can be accessed and modified in here. It is the platform \em + * callbacks responsibility to initialize and terminate this pointer if + * used (see @ref kbase_platform_funcs_conf). + */ + void (*power_resume_callback)(struct kbase_device *kbdev); + + /** Callback for handling runtime power management initialization. + * + * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback + * will become active from calls made to the OS from within this function. + * The runtime calls can be triggered by calls from @ref power_off_callback and @ref power_on_callback. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * + * @return 0 on success, else int error code. + */ + int (*power_runtime_init_callback)(struct kbase_device *kbdev); + + /** Callback for handling runtime power management termination. + * + * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback + * should no longer be called by the OS on completion of this function. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + */ + void (*power_runtime_term_callback)(struct kbase_device *kbdev); + + /** Callback for runtime power-off power management callback + * + * For linux this callback will be called by the kernel runtime_suspend callback. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * + * @return 0 on success, else OS error code. + */ + void (*power_runtime_off_callback)(struct kbase_device *kbdev); + + /** Callback for runtime power-on power management callback + * + * For linux this callback will be called by the kernel runtime_resume callback. + * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + */ + int (*power_runtime_on_callback)(struct kbase_device *kbdev); + + /* + * Optional callback for checking if GPU can be suspended when idle + * + * This callback will be called by the runtime power management core + * when the reference count goes to 0 to provide notification that the + * GPU now seems idle. + * + * If this callback finds that the GPU can't be powered off, or handles + * suspend by powering off directly or queueing up a power off, a + * non-zero value must be returned to prevent the runtime PM core from + * also triggering a suspend. + * + * Returning 0 will cause the runtime PM core to conduct a regular + * autosuspend. + * + * This callback is optional and if not provided regular autosuspend + * will be triggered. + * + * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use + * this feature. + * + * Return 0 if GPU can be suspended, positive value if it can not be + * suspeneded by runtime PM, else OS error code + */ + int (*power_runtime_idle_callback)(struct kbase_device *kbdev); +}; + +/** + * kbase_cpuprops_get_default_clock_speed - default for CPU_SPEED_FUNC + * @clock_speed - see kbase_cpu_clk_speed_func for details on the parameters + * + * Returns 0 on success, negative error code otherwise. + * + * Default implementation of CPU_SPEED_FUNC. This function sets clock_speed + * to 100, so will be an underestimate for any real system. + */ +int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed); + +/** + * kbase_cpu_clk_speed_func - Type of the function pointer for CPU_SPEED_FUNC + * @param clock_speed - pointer to store the current CPU clock speed in MHz + * + * Returns 0 on success, otherwise negative error code. + * + * This is mainly used to implement OpenCL's clGetDeviceInfo(). + */ +typedef int (*kbase_cpu_clk_speed_func) (u32 *clock_speed); + +/** + * kbase_gpu_clk_speed_func - Type of the function pointer for GPU_SPEED_FUNC + * @param clock_speed - pointer to store the current GPU clock speed in MHz + * + * Returns 0 on success, otherwise negative error code. + * When an error is returned the caller assumes maximum GPU speed stored in + * gpu_freq_khz_max. + * + * If the system timer is not available then this function is required + * for the OpenCL queue profiling to return correct timing information. + * + */ +typedef int (*kbase_gpu_clk_speed_func) (u32 *clock_speed); + +#ifdef CONFIG_OF +struct kbase_platform_config { +}; +#else + +/* + * @brief Specifies start and end of I/O memory region. + */ +struct kbase_io_memory_region { + u64 start; + u64 end; +}; + +/* + * @brief Specifies I/O related resources like IRQs and memory region for I/O operations. + */ +struct kbase_io_resources { + u32 job_irq_number; + u32 mmu_irq_number; + u32 gpu_irq_number; + struct kbase_io_memory_region io_memory_region; +}; + +struct kbase_platform_config { + const struct kbase_io_resources *io_resources; +}; + +#endif /* CONFIG_OF */ + +/** + * @brief Gets the pointer to platform config. + * + * @return Pointer to the platform config + */ +struct kbase_platform_config *kbase_get_platform_config(void); + +/** + * kbasep_platform_device_init: - Platform specific call to initialize hardware + * @kbdev: kbase device pointer + * + * Function calls a platform defined routine if specified in the configuration + * attributes. The routine can initialize any hardware and context state that + * is required for the GPU block to function. + * + * Return: 0 if no errors have been found in the config. + * Negative error code otherwise. + */ +int kbasep_platform_device_init(struct kbase_device *kbdev); + +/** + * kbasep_platform_device_term - Platform specific call to terminate hardware + * @kbdev: Kbase device pointer + * + * Function calls a platform defined routine if specified in the configuration + * attributes. The routine can destroy any platform specific context state and + * shut down any hardware functionality that are outside of the Power Management + * callbacks. + * + */ +void kbasep_platform_device_term(struct kbase_device *kbdev); + + +/** + * kbase_platform_early_init - Early initialisation of the platform code + * + * This function will be called when the module is loaded to perform any + * early initialisation required by the platform code. Such as reading + * platform specific device tree entries for the GPU. + * + * Return: 0 for success, any other fail causes module initialisation to fail + */ +int kbase_platform_early_init(void); + +#ifndef CONFIG_OF +#ifdef CONFIG_MALI_PLATFORM_FAKE +/** + * kbase_platform_fake_register - Register a platform device for the GPU + * + * This can be used to register a platform device on systems where device tree + * is not enabled and the platform initialisation code in the kernel doesn't + * create the GPU device. Where possible device tree should be used instead. + * + * Return: 0 for success, any other fail causes module initialisation to fail + */ +int kbase_platform_fake_register(void); + +/** + * kbase_platform_fake_unregister - Unregister a fake platform device + * + * Unregister the platform device created with kbase_platform_fake_register() + */ +void kbase_platform_fake_unregister(void); +#endif +#endif + + /** @} *//* end group kbase_config */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_CONFIG_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h b/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h new file mode 100755 index 000000000000..1cf44b3500cf --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h @@ -0,0 +1,227 @@ +/* + * + * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_config_defaults.h + * + * Default values for configuration settings + * + */ + +#ifndef _KBASE_CONFIG_DEFAULTS_H_ +#define _KBASE_CONFIG_DEFAULTS_H_ + +/* Include mandatory definitions per platform */ +#include + +/** +* Boolean indicating whether the driver is configured to be secure at +* a potential loss of performance. +* +* This currently affects only r0p0-15dev0 HW and earlier. +* +* On r0p0-15dev0 HW and earlier, there are tradeoffs between security and +* performance: +* +* - When this is set to true, the driver remains fully secure, +* but potentially loses performance compared with setting this to +* false. +* - When set to false, the driver is open to certain security +* attacks. +* +* From r0p0-00rel0 and onwards, there is no security loss by setting +* this to false, and no performance loss by setting it to +* true. +*/ +#define DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE false + +enum { + /** + * Use unrestricted Address ID width on the AXI bus. + */ + KBASE_AID_32 = 0x0, + + /** + * Restrict GPU to a half of maximum Address ID count. + * This will reduce performance, but reduce bus load due to GPU. + */ + KBASE_AID_16 = 0x3, + + /** + * Restrict GPU to a quarter of maximum Address ID count. + * This will reduce performance, but reduce bus load due to GPU. + */ + KBASE_AID_8 = 0x2, + + /** + * Restrict GPU to an eighth of maximum Address ID count. + * This will reduce performance, but reduce bus load due to GPU. + */ + KBASE_AID_4 = 0x1 +}; + +/** + * Default setting for read Address ID limiting on AXI bus. + * + * Attached value: u32 register value + * KBASE_AID_32 - use the full 32 IDs (5 ID bits) + * KBASE_AID_16 - use 16 IDs (4 ID bits) + * KBASE_AID_8 - use 8 IDs (3 ID bits) + * KBASE_AID_4 - use 4 IDs (2 ID bits) + * Default value: KBASE_AID_32 (no limit). Note hardware implementation + * may limit to a lower value. + */ +#define DEFAULT_ARID_LIMIT KBASE_AID_32 + +/** + * Default setting for write Address ID limiting on AXI. + * + * Attached value: u32 register value + * KBASE_AID_32 - use the full 32 IDs (5 ID bits) + * KBASE_AID_16 - use 16 IDs (4 ID bits) + * KBASE_AID_8 - use 8 IDs (3 ID bits) + * KBASE_AID_4 - use 4 IDs (2 ID bits) + * Default value: KBASE_AID_32 (no limit). Note hardware implementation + * may limit to a lower value. + */ +#define DEFAULT_AWID_LIMIT KBASE_AID_32 + +/** + * Default UMP device mapping. A UMP_DEVICE__SHIFT value which + * defines which UMP device this GPU should be mapped to. + */ +#define DEFAULT_UMP_GPU_DEVICE_SHIFT UMP_DEVICE_Z_SHIFT + +/* + * Default period for DVFS sampling + */ +// #define DEFAULT_PM_DVFS_PERIOD 100 /* 100ms */ +#define DEFAULT_PM_DVFS_PERIOD 20 /* 20 ms */ + +/* + * Power Management poweroff tick granuality. This is in nanoseconds to + * allow HR timer support. + * + * On each scheduling tick, the power manager core may decide to: + * -# Power off one or more shader cores + * -# Power off the entire GPU + */ +#define DEFAULT_PM_GPU_POWEROFF_TICK_NS (400000) /* 400us */ + +/* + * Power Manager number of ticks before shader cores are powered off + */ +#define DEFAULT_PM_POWEROFF_TICK_SHADER (2) /* 400-800us */ + +/* + * Power Manager number of ticks before GPU is powered off + */ +#define DEFAULT_PM_POWEROFF_TICK_GPU (2) /* 400-800us */ + +/* + * Default scheduling tick granuality + */ +#define DEFAULT_JS_SCHEDULING_PERIOD_NS (100000000u) /* 100ms */ + +/* + * Default minimum number of scheduling ticks before jobs are soft-stopped. + * + * This defines the time-slice for a job (which may be different from that of a + * context) + */ +#define DEFAULT_JS_SOFT_STOP_TICKS (1) /* 100ms-200ms */ + +/* + * Default minimum number of scheduling ticks before CL jobs are soft-stopped. + */ +#define DEFAULT_JS_SOFT_STOP_TICKS_CL (1) /* 100ms-200ms */ + +/* + * Default minimum number of scheduling ticks before jobs are hard-stopped + */ +#define DEFAULT_JS_HARD_STOP_TICKS_SS (50) /* 5s */ +#define DEFAULT_JS_HARD_STOP_TICKS_SS_8408 (300) /* 30s */ + +/* + * Default minimum number of scheduling ticks before CL jobs are hard-stopped. + */ +#define DEFAULT_JS_HARD_STOP_TICKS_CL (50) /* 5s */ + +/* + * Default minimum number of scheduling ticks before jobs are hard-stopped + * during dumping + */ +#define DEFAULT_JS_HARD_STOP_TICKS_DUMPING (15000) /* 1500s */ + +/* + * Default timeout for some software jobs, after which the software event wait + * jobs will be cancelled. + */ +#define DEFAULT_JS_SOFT_JOB_TIMEOUT (3000) /* 3s */ + +/* + * Default minimum number of scheduling ticks before the GPU is reset to clear a + * "stuck" job + */ +#define DEFAULT_JS_RESET_TICKS_SS (55) /* 5.5s */ +#define DEFAULT_JS_RESET_TICKS_SS_8408 (450) /* 45s */ + +/* + * Default minimum number of scheduling ticks before the GPU is reset to clear a + * "stuck" CL job. + */ +#define DEFAULT_JS_RESET_TICKS_CL (55) /* 5.5s */ + +/* + * Default minimum number of scheduling ticks before the GPU is reset to clear a + * "stuck" job during dumping. + */ +#define DEFAULT_JS_RESET_TICKS_DUMPING (15020) /* 1502s */ + +/* + * Default number of milliseconds given for other jobs on the GPU to be + * soft-stopped when the GPU needs to be reset. + */ +#define DEFAULT_RESET_TIMEOUT_MS (3000) /* 3s */ + +/* + * Default timeslice that a context is scheduled in for, in nanoseconds. + * + * When a context has used up this amount of time across its jobs, it is + * scheduled out to let another run. + * + * @note the resolution is nanoseconds (ns) here, because that's the format + * often used by the OS. + */ +#define DEFAULT_JS_CTX_TIMESLICE_NS (50000000) /* 50ms */ + +/* + * Perform GPU power down using only platform specific code, skipping DDK power + * management. + * + * If this is non-zero then kbase will avoid powering down shader cores, the + * tiler, and the L2 cache, instead just powering down the entire GPU through + * platform specific code. This may be required for certain platform + * integrations. + * + * Note that as this prevents kbase from powering down shader cores, this limits + * the available power policies to coarse_demand and always_on. + */ +#define PLATFORM_POWER_DOWN_ONLY (1) + +#endif /* _KBASE_CONFIG_DEFAULTS_H_ */ + diff --git a/drivers/gpu/arm/midgard/mali_kbase_context.c b/drivers/gpu/arm/midgard/mali_kbase_context.c new file mode 100755 index 000000000000..6338a7e221ed --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_context.c @@ -0,0 +1,342 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel context APIs + */ + +#include +#include +#include +#include +#include + +/** + * kbase_create_context() - Create a kernel base context. + * @kbdev: Kbase device + * @is_compat: Force creation of a 32-bit context + * + * Allocate and init a kernel base context. + * + * Return: new kbase context + */ +struct kbase_context * +kbase_create_context(struct kbase_device *kbdev, bool is_compat) +{ + struct kbase_context *kctx; + int err; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + /* zero-inited as lot of code assume it's zero'ed out on create */ + kctx = vzalloc(sizeof(*kctx)); + + if (!kctx) + goto out; + + /* creating a context is considered a disjoint event */ + kbase_disjoint_event(kbdev); + + kctx->kbdev = kbdev; + kctx->as_nr = KBASEP_AS_NR_INVALID; + atomic_set(&kctx->refcount, 0); + if (is_compat) + kbase_ctx_flag_set(kctx, KCTX_COMPAT); +#ifdef CONFIG_MALI_TRACE_TIMELINE + kctx->timeline.owner_tgid = task_tgid_nr(current); +#endif + atomic_set(&kctx->setup_complete, 0); + atomic_set(&kctx->setup_in_progress, 0); + spin_lock_init(&kctx->mm_update_lock); + kctx->process_mm = NULL; + atomic_set(&kctx->nonmapped_pages, 0); + kctx->slots_pullable = 0; + kctx->tgid = current->tgid; + kctx->pid = current->pid; + + err = kbase_mem_pool_init(&kctx->mem_pool, + kbdev->mem_pool_max_size_default, + kctx->kbdev, &kbdev->mem_pool); + if (err) + goto free_kctx; + + err = kbase_mem_evictable_init(kctx); + if (err) + goto free_pool; + + atomic_set(&kctx->used_pages, 0); + + err = kbase_jd_init(kctx); + if (err) + goto deinit_evictable; + + err = kbasep_js_kctx_init(kctx); + if (err) + goto free_jd; /* safe to call kbasep_js_kctx_term in this case */ + + err = kbase_event_init(kctx); + if (err) + goto free_jd; + + atomic_set(&kctx->drain_pending, 0); + + mutex_init(&kctx->reg_lock); + + INIT_LIST_HEAD(&kctx->waiting_soft_jobs); + spin_lock_init(&kctx->waiting_soft_jobs_lock); +#ifdef CONFIG_KDS + INIT_LIST_HEAD(&kctx->waiting_kds_resource); +#endif + err = kbase_dma_fence_init(kctx); + if (err) + goto free_event; + + err = kbase_mmu_init(kctx); + if (err) + goto term_dma_fence; + + do { + err = kbase_mem_pool_grow(&kctx->mem_pool, + MIDGARD_MMU_BOTTOMLEVEL); + if (err) + goto pgd_no_mem; + + mutex_lock(&kctx->mmu_lock); + kctx->pgd = kbase_mmu_alloc_pgd(kctx); + mutex_unlock(&kctx->mmu_lock); + } while (!kctx->pgd); + + kctx->aliasing_sink_page = kbase_mem_alloc_page(kctx->kbdev); + if (!kctx->aliasing_sink_page) + goto no_sink_page; + + init_waitqueue_head(&kctx->event_queue); + + kctx->cookies = KBASE_COOKIE_MASK; + + /* Make sure page 0 is not used... */ + err = kbase_region_tracker_init(kctx); + if (err) + goto no_region_tracker; + + err = kbase_sticky_resource_init(kctx); + if (err) + goto no_sticky; + + err = kbase_jit_init(kctx); + if (err) + goto no_jit; +#ifdef CONFIG_GPU_TRACEPOINTS + atomic_set(&kctx->jctx.work_id, 0); +#endif +#ifdef CONFIG_MALI_TRACE_TIMELINE + atomic_set(&kctx->timeline.jd_atoms_in_flight, 0); +#endif + + kctx->id = atomic_add_return(1, &(kbdev->ctx_num)) - 1; + + mutex_init(&kctx->vinstr_cli_lock); + + timer_setup(&kctx->soft_job_timeout, + kbasep_soft_job_timeout_worker, + 0); + + return kctx; + +no_jit: + kbase_gpu_vm_lock(kctx); + kbase_sticky_resource_term(kctx); + kbase_gpu_vm_unlock(kctx); +no_sticky: + kbase_region_tracker_term(kctx); +no_region_tracker: + kbase_mem_pool_free(&kctx->mem_pool, kctx->aliasing_sink_page, false); +no_sink_page: + /* VM lock needed for the call to kbase_mmu_free_pgd */ + kbase_gpu_vm_lock(kctx); + kbase_mmu_free_pgd(kctx); + kbase_gpu_vm_unlock(kctx); +pgd_no_mem: + kbase_mmu_term(kctx); +term_dma_fence: + kbase_dma_fence_term(kctx); +free_event: + kbase_event_cleanup(kctx); +free_jd: + /* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */ + kbasep_js_kctx_term(kctx); + kbase_jd_exit(kctx); +deinit_evictable: + kbase_mem_evictable_deinit(kctx); +free_pool: + kbase_mem_pool_term(&kctx->mem_pool); +free_kctx: + vfree(kctx); +out: + return NULL; +} +KBASE_EXPORT_SYMBOL(kbase_create_context); + +static void kbase_reg_pending_dtor(struct kbase_va_region *reg) +{ + dev_dbg(reg->kctx->kbdev->dev, "Freeing pending unmapped region\n"); + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); + kfree(reg); +} + +/** + * kbase_destroy_context - Destroy a kernel base context. + * @kctx: Context to destroy + * + * Calls kbase_destroy_os_context() to free OS specific structures. + * Will release all outstanding regions. + */ +void kbase_destroy_context(struct kbase_context *kctx) +{ + struct kbase_device *kbdev; + int pages; + unsigned long pending_regions_to_clean; + unsigned long flags; + + KBASE_DEBUG_ASSERT(NULL != kctx); + + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(NULL != kbdev); + + KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u); + + /* Ensure the core is powered up for the destroy process */ + /* A suspend won't happen here, because we're in a syscall from a userspace + * thread. */ + kbase_pm_context_active(kbdev); + + kbase_jd_zap_context(kctx); + +#ifdef CONFIG_DEBUG_FS + /* Removing the rest of the debugfs entries here as we want to keep the + * atom debugfs interface alive until all atoms have completed. This + * is useful for debugging hung contexts. */ + debugfs_remove_recursive(kctx->kctx_dentry); +#endif + + kbase_event_cleanup(kctx); + + /* + * JIT must be terminated before the code below as it must be called + * without the region lock being held. + * The code above ensures no new JIT allocations can be made by + * by the time we get to this point of context tear down. + */ + kbase_jit_term(kctx); + + kbase_gpu_vm_lock(kctx); + + kbase_sticky_resource_term(kctx); + + /* MMU is disabled as part of scheduling out the context */ + kbase_mmu_free_pgd(kctx); + + /* drop the aliasing sink page now that it can't be mapped anymore */ + kbase_mem_pool_free(&kctx->mem_pool, kctx->aliasing_sink_page, false); + + /* free pending region setups */ + pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK; + while (pending_regions_to_clean) { + unsigned int cookie = __ffs(pending_regions_to_clean); + + BUG_ON(!kctx->pending_regions[cookie]); + + kbase_reg_pending_dtor(kctx->pending_regions[cookie]); + + kctx->pending_regions[cookie] = NULL; + pending_regions_to_clean &= ~(1UL << cookie); + } + + kbase_region_tracker_term(kctx); + kbase_gpu_vm_unlock(kctx); + + /* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */ + kbasep_js_kctx_term(kctx); + + kbase_jd_exit(kctx); + + kbase_pm_context_idle(kbdev); + + kbase_dma_fence_term(kctx); + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); + kbase_ctx_sched_remove_ctx(kctx); + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_mmu_term(kctx); + + pages = atomic_read(&kctx->used_pages); + if (pages != 0) + dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); + + kbase_mem_evictable_deinit(kctx); + kbase_mem_pool_term(&kctx->mem_pool); + WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0); + + vfree(kctx); +} +KBASE_EXPORT_SYMBOL(kbase_destroy_context); + +/** + * kbase_context_set_create_flags - Set creation flags on a context + * @kctx: Kbase context + * @flags: Flags to set + * + * Return: 0 on success + */ +int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags) +{ + int err = 0; + struct kbasep_js_kctx_info *js_kctx_info; + unsigned long irq_flags; + + KBASE_DEBUG_ASSERT(NULL != kctx); + + js_kctx_info = &kctx->jctx.sched_info; + + /* Validate flags */ + if (flags != (flags & BASE_CONTEXT_CREATE_KERNEL_FLAGS)) { + err = -EINVAL; + goto out; + } + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); + + /* Translate the flags */ + if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) + kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); + + /* Latch the initial attributes into the Job Scheduler */ + kbasep_js_ctx_attr_set_initial_attrs(kctx->kbdev, kctx); + + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + out: + return err; +} +KBASE_EXPORT_SYMBOL(kbase_context_set_create_flags); diff --git a/drivers/gpu/arm/midgard/mali_kbase_context.h b/drivers/gpu/arm/midgard/mali_kbase_context.h new file mode 100755 index 000000000000..a3f5bb0ce0da --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_context.h @@ -0,0 +1,90 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_CONTEXT_H_ +#define _KBASE_CONTEXT_H_ + +#include + + +int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags); + +/** + * kbase_ctx_flag - Check if @flag is set on @kctx + * @kctx: Pointer to kbase context to check + * @flag: Flag to check + * + * Return: true if @flag is set on @kctx, false if not. + */ +static inline bool kbase_ctx_flag(struct kbase_context *kctx, + enum kbase_context_flags flag) +{ + return atomic_read(&kctx->flags) & flag; +} + +/** + * kbase_ctx_flag_clear - Clear @flag on @kctx + * @kctx: Pointer to kbase context + * @flag: Flag to clear + * + * Clear the @flag on @kctx. This is done atomically, so other flags being + * cleared or set at the same time will be safe. + * + * Some flags have locking requirements, check the documentation for the + * respective flags. + */ +static inline void kbase_ctx_flag_clear(struct kbase_context *kctx, + enum kbase_context_flags flag) +{ +#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE + /* + * Earlier kernel versions doesn't have atomic_andnot() or + * atomic_and(). atomic_clear_mask() was only available on some + * architectures and removed on arm in v3.13 on arm and arm64. + * + * Use a compare-exchange loop to clear the flag on pre 4.3 kernels, + * when atomic_andnot() becomes available. + */ + int old, new; + + do { + old = atomic_read(&kctx->flags); + new = old & ~flag; + + } while (atomic_cmpxchg(&kctx->flags, old, new) != old); +#else + atomic_andnot(flag, &kctx->flags); +#endif +} + +/** + * kbase_ctx_flag_set - Set @flag on @kctx + * @kctx: Pointer to kbase context + * @flag: Flag to clear + * + * Set the @flag on @kctx. This is done atomically, so other flags being + * cleared or set at the same time will be safe. + * + * Some flags have locking requirements, check the documentation for the + * respective flags. + */ +static inline void kbase_ctx_flag_set(struct kbase_context *kctx, + enum kbase_context_flags flag) +{ + atomic_or(flag, &kctx->flags); +} +#endif /* _KBASE_CONTEXT_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_core_linux.c b/drivers/gpu/arm/midgard/mali_kbase_core_linux.c new file mode 100755 index 000000000000..da55cb080d00 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_core_linux.c @@ -0,0 +1,4990 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + +#define ENABLE_DEBUG_LOG +#include "platform/rk/custom_log.h" + +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_MALI_DEVFREQ +#include +#include +#ifdef CONFIG_DEVFREQ_THERMAL +#include +#endif /* CONFIG_DEVFREQ_THERMAL */ +#endif /* CONFIG_MALI_DEVFREQ */ +#ifdef CONFIG_MALI_NO_MALI +#include "mali_kbase_model_linux.h" +#endif /* CONFIG_MALI_NO_MALI */ +#include "mali_kbase_mem_profile_debugfs_buf_size.h" +#include "mali_kbase_debug_mem_view.h" +#include "mali_kbase_mem.h" +#include "mali_kbase_mem_pool_debugfs.h" +#if !MALI_CUSTOMER_RELEASE +#include "mali_kbase_regs_dump_debugfs.h" +#endif /* !MALI_CUSTOMER_RELEASE */ +#include "mali_kbase_regs_history_debugfs.h" +#include +#include +#include +#include +#include "mali_kbase_ioctl.h" + +#ifdef CONFIG_KDS +#include +#include +#include +#endif /* CONFIG_KDS */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* is_compat_task */ +#include +#include +#ifdef CONFIG_MALI_PLATFORM_DEVICETREE +#include +#endif /* CONFIG_MALI_PLATFORM_DEVICETREE */ +#include +#include +#ifdef CONFIG_MALI_PLATFORM_FAKE +#include +#endif /*CONFIG_MALI_PLATFORM_FAKE */ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ +#include +#include + +#include + + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) +#include +#else +#include +#endif + +#include + +#include + +/* GPU IRQ Tags */ +#define JOB_IRQ_TAG 0 +#define MMU_IRQ_TAG 1 +#define GPU_IRQ_TAG 2 + +#if MALI_UNIT_TEST +static struct kbase_exported_test_data shared_kernel_test_data; +EXPORT_SYMBOL(shared_kernel_test_data); +#endif /* MALI_UNIT_TEST */ + +/** rk_ext : version of rk_ext on mali_ko, aka. rk_ko_ver. */ +#define ROCKCHIP_VERSION (13) + +static int kbase_dev_nr; + +static DEFINE_MUTEX(kbase_dev_list_lock); +static LIST_HEAD(kbase_dev_list); + +#define KERNEL_SIDE_DDK_VERSION_STRING "K:" MALI_RELEASE_NAME "(GPL)" +static inline void __compile_time_asserts(void) +{ + CSTD_COMPILE_TIME_ASSERT(sizeof(KERNEL_SIDE_DDK_VERSION_STRING) <= KBASE_GET_VERSION_BUFFER_SIZE); +} + +static int kbase_api_handshake(struct kbase_context *kctx, + struct kbase_ioctl_version_check *version) +{ + switch (version->major) { +#ifdef BASE_LEGACY_UK6_SUPPORT + case 6: + /* We are backwards compatible with version 6, + * so pretend to be the old version */ + version->major = 6; + version->minor = 1; + break; +#endif /* BASE_LEGACY_UK6_SUPPORT */ +#ifdef BASE_LEGACY_UK7_SUPPORT + case 7: + /* We are backwards compatible with version 7, + * so pretend to be the old version */ + version->major = 7; + version->minor = 1; + break; +#endif /* BASE_LEGACY_UK7_SUPPORT */ +#ifdef BASE_LEGACY_UK8_SUPPORT + case 8: + /* We are backwards compatible with version 8, + * so pretend to be the old version */ + version->major = 8; + version->minor = 4; + break; +#endif /* BASE_LEGACY_UK8_SUPPORT */ +#ifdef BASE_LEGACY_UK9_SUPPORT + case 9: + /* We are backwards compatible with version 9, + * so pretend to be the old version */ + version->major = 9; + version->minor = 0; + break; +#endif /* BASE_LEGACY_UK8_SUPPORT */ + case BASE_UK_VERSION_MAJOR: + /* set minor to be the lowest common */ + version->minor = min_t(int, BASE_UK_VERSION_MINOR, + (int)version->minor); + break; + default: + /* We return our actual version regardless if it + * matches the version returned by userspace - + * userspace can bail if it can't handle this + * version */ + version->major = BASE_UK_VERSION_MAJOR; + version->minor = BASE_UK_VERSION_MINOR; + break; + } + + /* save the proposed version number for later use */ + kctx->api_version = KBASE_API_VERSION(version->major, version->minor); + + return 0; +} + +/** + * enum mali_error - Mali error codes shared with userspace + * + * This is subset of those common Mali errors that can be returned to userspace. + * Values of matching user and kernel space enumerators MUST be the same. + * MALI_ERROR_NONE is guaranteed to be 0. + * + * @MALI_ERROR_NONE: Success + * @MALI_ERROR_OUT_OF_GPU_MEMORY: Not used in the kernel driver + * @MALI_ERROR_OUT_OF_MEMORY: Memory allocation failure + * @MALI_ERROR_FUNCTION_FAILED: Generic error code + */ +enum mali_error { + MALI_ERROR_NONE = 0, + MALI_ERROR_OUT_OF_GPU_MEMORY, + MALI_ERROR_OUT_OF_MEMORY, + MALI_ERROR_FUNCTION_FAILED, +}; + +enum { + inited_mem = (1u << 0), + inited_js = (1u << 1), + inited_pm_runtime_init = (1u << 2), +#ifdef CONFIG_MALI_DEVFREQ + inited_devfreq = (1u << 3), +#endif /* CONFIG_MALI_DEVFREQ */ + inited_tlstream = (1u << 4), + inited_backend_early = (1u << 5), + inited_backend_late = (1u << 6), + inited_device = (1u << 7), + inited_vinstr = (1u << 8), + + inited_job_fault = (1u << 10), + inited_sysfs_group = (1u << 11), + inited_misc_register = (1u << 12), + inited_get_device = (1u << 13), + inited_dev_list = (1u << 14), + inited_debugfs = (1u << 15), + inited_gpu_device = (1u << 16), + inited_registers_map = (1u << 17), + inited_io_history = (1u << 18), + inited_power_control = (1u << 19), + inited_buslogger = (1u << 20), + inited_protected = (1u << 21), + inited_ctx_sched = (1u << 22) +}; + + +#ifdef CONFIG_MALI_DEBUG +#define INACTIVE_WAIT_MS (5000) + +void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive) +{ + kbdev->driver_inactive = inactive; + wake_up(&kbdev->driver_inactive_wait); + + /* Wait for any running IOCTLs to complete */ + if (inactive) + msleep(INACTIVE_WAIT_MS); +} +KBASE_EXPORT_TEST_API(kbase_set_driver_inactive); +#endif /* CONFIG_MALI_DEBUG */ + +/** + * kbase_legacy_dispatch - UKK dispatch function + * + * This is the dispatch function for the legacy UKK ioctl interface. No new + * ioctls should be added to this function, see kbase_ioctl instead. + * + * @kctx: The kernel context structure + * @args: Pointer to the data structure passed from/to user space + * @args_size: Size of the data structure + */ +static int kbase_legacy_dispatch(struct kbase_context *kctx, + void * const args, u32 args_size) +{ + struct kbase_device *kbdev; + union uk_header *ukh = args; + u32 id; + int ret = 0; + + KBASE_DEBUG_ASSERT(ukh != NULL); + + kbdev = kctx->kbdev; + id = ukh->id; + ukh->ret = MALI_ERROR_NONE; /* Be optimistic */ + +#ifdef CONFIG_MALI_DEBUG + wait_event(kbdev->driver_inactive_wait, + kbdev->driver_inactive == false); +#endif /* CONFIG_MALI_DEBUG */ + + if (UKP_FUNC_ID_CHECK_VERSION == id) { + struct uku_version_check_args *version_check; + struct kbase_ioctl_version_check version; + + if (args_size != sizeof(struct uku_version_check_args)) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + return 0; + } + version_check = (struct uku_version_check_args *)args; + version.minor = version_check->minor; + version.major = version_check->major; + + kbase_api_handshake(kctx, &version); + + version_check->minor = version.minor; + version_check->major = version.major; + ukh->ret = MALI_ERROR_NONE; + return 0; + } + + /* block calls until version handshake */ + if (kctx->api_version == 0) + return -EINVAL; + + if (!atomic_read(&kctx->setup_complete)) { + struct kbase_uk_set_flags *kbase_set_flags; + + /* setup pending, try to signal that we'll do the setup, + * if setup was already in progress, err this call + */ + if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) + return -EINVAL; + + /* if unexpected call, will stay stuck in setup mode + * (is it the only call we accept?) + */ + if (id != KBASE_FUNC_SET_FLAGS) + return -EINVAL; + + kbase_set_flags = (struct kbase_uk_set_flags *)args; + + /* if not matching the expected call, stay in setup mode */ + if (sizeof(*kbase_set_flags) != args_size) + goto bad_size; + + /* if bad flags, will stay stuck in setup mode */ + if (kbase_context_set_create_flags(kctx, + kbase_set_flags->create_flags) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + + atomic_set(&kctx->setup_complete, 1); + return 0; + } + + /* setup complete, perform normal operation */ + switch (id) { + case KBASE_FUNC_MEM_JIT_INIT: + { + struct kbase_uk_mem_jit_init *jit_init = args; + + if (sizeof(*jit_init) != args_size) + goto bad_size; + + if (kbase_region_tracker_init_jit(kctx, + jit_init->va_pages)) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + case KBASE_FUNC_MEM_ALLOC: + { + struct kbase_uk_mem_alloc *mem = args; + struct kbase_va_region *reg; + + if (sizeof(*mem) != args_size) + goto bad_size; + +#if defined(CONFIG_64BIT) + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* force SAME_VA if a 64-bit client */ + mem->flags |= BASE_MEM_SAME_VA; + } +#endif + + reg = kbase_mem_alloc(kctx, mem->va_pages, + mem->commit_pages, mem->extent, + &mem->flags, &mem->gpu_va); + mem->va_alignment = 0; + + if (!reg) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + case KBASE_FUNC_MEM_IMPORT: { + struct kbase_uk_mem_import *mem_import = args; + void __user *phandle; + + if (sizeof(*mem_import) != args_size) + goto bad_size; +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + phandle = compat_ptr(mem_import->phandle.compat_value); + else +#endif + phandle = mem_import->phandle.value; + + if (mem_import->type == BASE_MEM_IMPORT_TYPE_INVALID) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + + if (kbase_mem_import(kctx, + (enum base_mem_import_type) + mem_import->type, + phandle, + 0, + &mem_import->gpu_va, + &mem_import->va_pages, + &mem_import->flags)) { + mem_import->type = BASE_MEM_IMPORT_TYPE_INVALID; + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + } + break; + } + case KBASE_FUNC_MEM_ALIAS: { + struct kbase_uk_mem_alias *alias = args; + struct base_mem_aliasing_info __user *user_ai; + struct base_mem_aliasing_info *ai; + + if (sizeof(*alias) != args_size) + goto bad_size; + + if (alias->nents > 2048) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + if (!alias->nents) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + user_ai = compat_ptr(alias->ai.compat_value); + else +#endif + user_ai = alias->ai.value; + + ai = vmalloc(sizeof(*ai) * alias->nents); + + if (!ai) { + ukh->ret = MALI_ERROR_OUT_OF_MEMORY; + break; + } + + if (copy_from_user(ai, user_ai, + sizeof(*ai) * alias->nents)) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + goto copy_failed; + } + + alias->gpu_va = kbase_mem_alias(kctx, &alias->flags, + alias->stride, + alias->nents, ai, + &alias->va_pages); + if (!alias->gpu_va) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + goto no_alias; + } +no_alias: +copy_failed: + vfree(ai); + break; + } + case KBASE_FUNC_MEM_COMMIT: + { + struct kbase_uk_mem_commit *commit = args; + int ret; + + if (sizeof(*commit) != args_size) + goto bad_size; + + ret = kbase_mem_commit(kctx, commit->gpu_addr, + commit->pages); + + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + commit->result_subcode = + BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS; + + if (ret == 0) { + ukh->ret = MALI_ERROR_NONE; + commit->result_subcode = + BASE_BACKING_THRESHOLD_OK; + } else if (ret == -ENOMEM) { + commit->result_subcode = + BASE_BACKING_THRESHOLD_ERROR_OOM; + } + + break; + } + + case KBASE_FUNC_MEM_QUERY: + { + struct kbase_uk_mem_query *query = args; + + if (sizeof(*query) != args_size) + goto bad_size; + + if (kbase_mem_query(kctx, query->gpu_addr, + query->query, &query->value) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + ukh->ret = MALI_ERROR_NONE; + break; + } + break; + + case KBASE_FUNC_MEM_FLAGS_CHANGE: + { + struct kbase_uk_mem_flags_change *fc = args; + + if (sizeof(*fc) != args_size) + goto bad_size; + + if (kbase_mem_flags_change(kctx, fc->gpu_va, + fc->flags, fc->mask) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + + break; + } + case KBASE_FUNC_MEM_FREE: + { + struct kbase_uk_mem_free *mem = args; + + if (sizeof(*mem) != args_size) + goto bad_size; + + if (kbase_mem_free(kctx, mem->gpu_addr) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + + case KBASE_FUNC_JOB_SUBMIT: + { + struct kbase_uk_job_submit *job = args; + void __user *user_addr = NULL; + + if (sizeof(*job) != args_size) + goto bad_size; + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + user_addr = compat_ptr(job->addr.compat_value); + else +#endif + user_addr = job->addr.value; + + if (kbase_jd_submit(kctx, user_addr, job->nr_atoms, + job->stride, false) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + +#ifdef BASE_LEGACY_UK6_SUPPORT + case KBASE_FUNC_JOB_SUBMIT_UK6: + { + struct kbase_uk_job_submit *job = args; + void __user *user_addr = NULL; + + if (sizeof(*job) != args_size) + goto bad_size; + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + user_addr = compat_ptr(job->addr.compat_value); + else +#endif + user_addr = job->addr.value; + + if (kbase_jd_submit(kctx, user_addr, job->nr_atoms, + job->stride, true) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } +#endif + + case KBASE_FUNC_SYNC: + { + struct kbase_uk_sync_now *sn = args; + + if (sizeof(*sn) != args_size) + goto bad_size; + +#ifndef CONFIG_MALI_COH_USER + if (kbase_sync_now(kctx, &sn->sset.basep_sset) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; +#endif + break; + } + + case KBASE_FUNC_DISJOINT_QUERY: + { + struct kbase_uk_disjoint_query *dquery = args; + + if (sizeof(*dquery) != args_size) + goto bad_size; + + /* Get the disjointness counter value. */ + dquery->counter = kbase_disjoint_event_get(kctx->kbdev); + break; + } + + case KBASE_FUNC_POST_TERM: + { + kbase_event_close(kctx); + break; + } + + case KBASE_FUNC_HWCNT_SETUP: + { + struct kbase_uk_hwcnt_setup *setup = args; + + if (sizeof(*setup) != args_size) + goto bad_size; + + mutex_lock(&kctx->vinstr_cli_lock); + if (kbase_vinstr_legacy_hwc_setup(kbdev->vinstr_ctx, + &kctx->vinstr_cli, setup) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + mutex_unlock(&kctx->vinstr_cli_lock); + break; + } + + case KBASE_FUNC_HWCNT_DUMP: + { + /* args ignored */ + mutex_lock(&kctx->vinstr_cli_lock); + if (kbase_vinstr_hwc_dump(kctx->vinstr_cli, + BASE_HWCNT_READER_EVENT_MANUAL) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + mutex_unlock(&kctx->vinstr_cli_lock); + break; + } + + case KBASE_FUNC_HWCNT_CLEAR: + { + /* args ignored */ + mutex_lock(&kctx->vinstr_cli_lock); + if (kbase_vinstr_hwc_clear(kctx->vinstr_cli) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + mutex_unlock(&kctx->vinstr_cli_lock); + break; + } + + case KBASE_FUNC_HWCNT_READER_SETUP: + { + struct kbase_uk_hwcnt_reader_setup *setup = args; + + if (sizeof(*setup) != args_size) + goto bad_size; + + mutex_lock(&kctx->vinstr_cli_lock); + if (kbase_vinstr_hwcnt_reader_setup(kbdev->vinstr_ctx, + setup) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + mutex_unlock(&kctx->vinstr_cli_lock); + break; + } + + case KBASE_FUNC_GPU_PROPS_REG_DUMP: + { + struct kbase_uk_gpuprops *setup = args; + + if (sizeof(*setup) != args_size) + goto bad_size; + + if (kbase_gpuprops_uk_get_props(kctx, setup) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + case KBASE_FUNC_FIND_CPU_OFFSET: + { + struct kbase_uk_find_cpu_offset *find = args; + + if (sizeof(*find) != args_size) + goto bad_size; + + if (find->gpu_addr & ~PAGE_MASK) { + dev_warn(kbdev->dev, + "kbase_legacy_dispatch case KBASE_FUNC_FIND_CPU_OFFSET: find->gpu_addr: passed parameter is invalid"); + goto out_bad; + } + + if (find->size > SIZE_MAX || find->cpu_addr > ULONG_MAX) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + } else { + int err; + + err = kbasep_find_enclosing_cpu_mapping_offset( + kctx, + find->cpu_addr, + find->size, + &find->offset); + + if (err) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + } + break; + } + case KBASE_FUNC_GET_VERSION: + { + struct kbase_uk_get_ddk_version *get_version = (struct kbase_uk_get_ddk_version *)args; + + if (sizeof(*get_version) != args_size) + goto bad_size; + + /* version buffer size check is made in compile time assert */ + memcpy(get_version->version_buffer, + KERNEL_SIDE_DDK_VERSION_STRING, + sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); + get_version->version_string_size = + sizeof(KERNEL_SIDE_DDK_VERSION_STRING); + get_version->rk_version = ROCKCHIP_VERSION; + break; + } + + case KBASE_FUNC_STREAM_CREATE: + { +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + struct kbase_uk_stream_create *screate = (struct kbase_uk_stream_create *)args; + + if (sizeof(*screate) != args_size) + goto bad_size; + + if (strnlen(screate->name, sizeof(screate->name)) >= sizeof(screate->name)) { + /* not NULL terminated */ + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } + + if (kbase_sync_fence_stream_create(screate->name, + &screate->fd) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + ukh->ret = MALI_ERROR_NONE; +#else /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + ukh->ret = MALI_ERROR_FUNCTION_FAILED; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + break; + } + case KBASE_FUNC_FENCE_VALIDATE: + { +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + struct kbase_uk_fence_validate *fence_validate = (struct kbase_uk_fence_validate *)args; + + if (sizeof(*fence_validate) != args_size) + goto bad_size; + + if (kbase_sync_fence_validate(fence_validate->fd) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + ukh->ret = MALI_ERROR_NONE; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + break; + } + + case KBASE_FUNC_SET_TEST_DATA: + { +#if MALI_UNIT_TEST + struct kbase_uk_set_test_data *set_data = args; + + shared_kernel_test_data = set_data->test_data; + shared_kernel_test_data.kctx.value = (void __user *)kctx; + shared_kernel_test_data.mm.value = (void __user *)current->mm; + ukh->ret = MALI_ERROR_NONE; +#endif /* MALI_UNIT_TEST */ + break; + } + + case KBASE_FUNC_INJECT_ERROR: + { +#ifdef CONFIG_MALI_ERROR_INJECT + unsigned long flags; + struct kbase_error_params params = ((struct kbase_uk_error_params *)args)->params; + + /*mutex lock */ + spin_lock_irqsave(&kbdev->reg_op_lock, flags); + if (job_atom_inject_error(¶ms) != 0) + ukh->ret = MALI_ERROR_OUT_OF_MEMORY; + else + ukh->ret = MALI_ERROR_NONE; + spin_unlock_irqrestore(&kbdev->reg_op_lock, flags); + /*mutex unlock */ +#endif /* CONFIG_MALI_ERROR_INJECT */ + break; + } + + case KBASE_FUNC_MODEL_CONTROL: + { +#ifdef CONFIG_MALI_NO_MALI + unsigned long flags; + struct kbase_model_control_params params = + ((struct kbase_uk_model_control_params *)args)->params; + + /*mutex lock */ + spin_lock_irqsave(&kbdev->reg_op_lock, flags); + if (gpu_model_control(kbdev->model, ¶ms) != 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + ukh->ret = MALI_ERROR_NONE; + spin_unlock_irqrestore(&kbdev->reg_op_lock, flags); + /*mutex unlock */ +#endif /* CONFIG_MALI_NO_MALI */ + break; + } + +#ifdef BASE_LEGACY_UK8_SUPPORT + case KBASE_FUNC_KEEP_GPU_POWERED: + { + dev_warn(kbdev->dev, "kbase_legacy_dispatch case KBASE_FUNC_KEEP_GPU_POWERED: function is deprecated and disabled\n"); + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + break; + } +#endif /* BASE_LEGACY_UK8_SUPPORT */ + + case KBASE_FUNC_GET_PROFILING_CONTROLS: + { + struct kbase_uk_profiling_controls *controls = + (struct kbase_uk_profiling_controls *)args; + u32 i; + + if (sizeof(*controls) != args_size) + goto bad_size; + + for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) + controls->profiling_controls[i] = + kbdev->kbase_profiling_controls[i]; + + break; + } + + /* used only for testing purposes; these controls are to be set by gator through gator API */ + case KBASE_FUNC_SET_PROFILING_CONTROLS: + { + struct kbase_uk_profiling_controls *controls = + (struct kbase_uk_profiling_controls *)args; + u32 i; + + if (sizeof(*controls) != args_size) + goto bad_size; + + for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) + _mali_profiling_control(i, controls->profiling_controls[i]); + + break; + } + + case KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD: + { + struct kbase_uk_debugfs_mem_profile_add *add_data = + (struct kbase_uk_debugfs_mem_profile_add *)args; + char *buf; + char __user *user_buf; + + if (sizeof(*add_data) != args_size) + goto bad_size; + + if (add_data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { + dev_err(kbdev->dev, "buffer too big\n"); + goto out_bad; + } + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + user_buf = + compat_ptr(add_data->buf.compat_value); + else +#endif + user_buf = add_data->buf.value; + + buf = kmalloc(add_data->len, GFP_KERNEL); + if (ZERO_OR_NULL_PTR(buf)) + goto out_bad; + + if (0 != copy_from_user(buf, user_buf, add_data->len)) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + kfree(buf); + goto out_bad; + } + + if (kbasep_mem_profile_debugfs_insert(kctx, buf, + add_data->len)) { + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + goto out_bad; + } + + break; + } + +#ifdef CONFIG_MALI_NO_MALI + case KBASE_FUNC_SET_PRFCNT_VALUES: + { + + struct kbase_uk_prfcnt_values *params = + ((struct kbase_uk_prfcnt_values *)args); + gpu_model_set_dummy_prfcnt_sample(params->data, + params->size); + + break; + } +#endif /* CONFIG_MALI_NO_MALI */ +#ifdef BASE_LEGACY_UK10_4_SUPPORT + case KBASE_FUNC_TLSTREAM_ACQUIRE_V10_4: + { + struct kbase_uk_tlstream_acquire_v10_4 *tlstream_acquire + = args; + int ret; + + if (sizeof(*tlstream_acquire) != args_size) + goto bad_size; + + ret = kbase_tlstream_acquire( + kctx, 0); + if (ret < 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + tlstream_acquire->fd = ret; + break; + } +#endif /* BASE_LEGACY_UK10_4_SUPPORT */ + case KBASE_FUNC_TLSTREAM_ACQUIRE: + { + struct kbase_uk_tlstream_acquire *tlstream_acquire = + args; + int ret; + + if (sizeof(*tlstream_acquire) != args_size) + goto bad_size; + + if (tlstream_acquire->flags & ~BASE_TLSTREAM_FLAGS_MASK) + goto out_bad; + + ret = kbase_tlstream_acquire( + kctx, tlstream_acquire->flags); + if (ret < 0) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + else + tlstream_acquire->fd = ret; + break; + } + case KBASE_FUNC_TLSTREAM_FLUSH: + { + struct kbase_uk_tlstream_flush *tlstream_flush = + args; + + if (sizeof(*tlstream_flush) != args_size) + goto bad_size; + + kbase_tlstream_flush_streams(); + break; + } +#if MALI_UNIT_TEST + case KBASE_FUNC_TLSTREAM_TEST: + { + struct kbase_uk_tlstream_test *tlstream_test = args; + + if (sizeof(*tlstream_test) != args_size) + goto bad_size; + + kbase_tlstream_test( + tlstream_test->tpw_count, + tlstream_test->msg_delay, + tlstream_test->msg_count, + tlstream_test->aux_msg); + break; + } + case KBASE_FUNC_TLSTREAM_STATS: + { + struct kbase_uk_tlstream_stats *tlstream_stats = args; + + if (sizeof(*tlstream_stats) != args_size) + goto bad_size; + + kbase_tlstream_stats( + &tlstream_stats->bytes_collected, + &tlstream_stats->bytes_generated); + break; + } +#endif /* MALI_UNIT_TEST */ + + case KBASE_FUNC_GET_CONTEXT_ID: + { + struct kbase_uk_context_id *info = args; + + info->id = kctx->id; + break; + } + + case KBASE_FUNC_SOFT_EVENT_UPDATE: + { + struct kbase_uk_soft_event_update *update = args; + + if (sizeof(*update) != args_size) + goto bad_size; + + if (((update->new_status != BASE_JD_SOFT_EVENT_SET) && + (update->new_status != BASE_JD_SOFT_EVENT_RESET)) || + (update->flags != 0)) + goto out_bad; + + if (kbase_soft_event_update(kctx, update->evt, + update->new_status)) + ukh->ret = MALI_ERROR_FUNCTION_FAILED; + + break; + } + + default: + dev_err(kbdev->dev, "unknown ioctl %u\n", id); + goto out_bad; + } + + return ret; + +bad_size: + dev_err(kbdev->dev, "Wrong syscall size (%d) for %08x\n", args_size, id); +out_bad: + return -EINVAL; +} + +static struct kbase_device *to_kbase_device(struct device *dev) +{ + return dev_get_drvdata(dev); +} + +static int assign_irqs(struct platform_device *pdev) +{ + struct kbase_device *kbdev = to_kbase_device(&pdev->dev); + int i; + + if (!kbdev) + return -ENODEV; + + /* 3 IRQ resources */ + for (i = 0; i < 3; i++) { + struct resource *irq_res; + int irqtag; + + irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, i); + if (!irq_res) { + dev_err(kbdev->dev, "No IRQ resource at index %d\n", i); + return -ENOENT; + } + +#ifdef CONFIG_OF + if (!strncasecmp(irq_res->name, "JOB", 3)) { + irqtag = JOB_IRQ_TAG; + } else if (!strncasecmp(irq_res->name, "MMU", 3)) { + irqtag = MMU_IRQ_TAG; + } else if (!strncasecmp(irq_res->name, "GPU", 3)) { + irqtag = GPU_IRQ_TAG; + } else { + dev_err(&pdev->dev, "Invalid irq res name: '%s'\n", + irq_res->name); + return -EINVAL; + } +#else + irqtag = i; +#endif /* CONFIG_OF */ + kbdev->irqs[irqtag].irq = irq_res->start; + kbdev->irqs[irqtag].flags = irq_res->flags & IRQF_TRIGGER_MASK; + } + + return 0; +} + +/* + * API to acquire device list mutex and + * return pointer to the device list head + */ +const struct list_head *kbase_dev_list_get(void) +{ + mutex_lock(&kbase_dev_list_lock); + return &kbase_dev_list; +} +KBASE_EXPORT_TEST_API(kbase_dev_list_get); + +/* API to release the device list mutex */ +void kbase_dev_list_put(const struct list_head *dev_list) +{ + mutex_unlock(&kbase_dev_list_lock); +} +KBASE_EXPORT_TEST_API(kbase_dev_list_put); + +/* Find a particular kbase device (as specified by minor number), or find the "first" device if -1 is specified */ +struct kbase_device *kbase_find_device(int minor) +{ + struct kbase_device *kbdev = NULL; + struct list_head *entry; + const struct list_head *dev_list = kbase_dev_list_get(); + + list_for_each(entry, dev_list) { + struct kbase_device *tmp; + + tmp = list_entry(entry, struct kbase_device, entry); + if (tmp->mdev.minor == minor || minor == -1) { + kbdev = tmp; + get_device(kbdev->dev); + break; + } + } + kbase_dev_list_put(dev_list); + + return kbdev; +} +EXPORT_SYMBOL(kbase_find_device); + +void kbase_release_device(struct kbase_device *kbdev) +{ + put_device(kbdev->dev); +} +EXPORT_SYMBOL(kbase_release_device); + +#if KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE +/* + * Older versions, before v4.6, of the kernel doesn't have + * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28 + */ +static int kstrtobool_from_user(const char __user *s, size_t count, bool *res) +{ + char buf[32]; + + count = min(sizeof(buf), count); + + if (copy_from_user(buf, s, count)) + return -EFAULT; + buf[count] = '\0'; + + return strtobool(buf, res); +} +#endif + +static ssize_t write_ctx_infinite_cache(struct file *f, const char __user *ubuf, size_t size, loff_t *off) +{ + struct kbase_context *kctx = f->private_data; + int err; + bool value; + + err = kstrtobool_from_user(ubuf, size, &value); + if (err) + return err; + + if (value) + kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); + else + kbase_ctx_flag_clear(kctx, KCTX_INFINITE_CACHE); + + return size; +} + +static ssize_t read_ctx_infinite_cache(struct file *f, char __user *ubuf, size_t size, loff_t *off) +{ + struct kbase_context *kctx = f->private_data; + char buf[32]; + int count; + bool value; + + value = kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE); + + count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); + + return simple_read_from_buffer(ubuf, size, off, buf, count); +} + +static const struct file_operations kbase_infinite_cache_fops = { + .open = simple_open, + .write = write_ctx_infinite_cache, + .read = read_ctx_infinite_cache, +}; + +static int kbase_open(struct inode *inode, struct file *filp) +{ + struct kbase_device *kbdev = NULL; + struct kbase_context *kctx; + int ret = 0; +#ifdef CONFIG_DEBUG_FS + char kctx_name[64]; +#endif + + kbdev = kbase_find_device(iminor(inode)); + + if (!kbdev) + return -ENODEV; + + kctx = kbase_create_context(kbdev, is_compat_task()); + if (!kctx) { + ret = -ENOMEM; + goto out; + } + + init_waitqueue_head(&kctx->event_queue); + filp->f_mode |= FMODE_UNSIGNED_OFFSET; + filp->private_data = kctx; + kctx->filp = filp; + + if (kbdev->infinite_cache_active_default) + kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); + +#ifdef CONFIG_DEBUG_FS + snprintf(kctx_name, 64, "%d_%d", kctx->tgid, kctx->id); + + kctx->kctx_dentry = debugfs_create_dir(kctx_name, + kbdev->debugfs_ctx_directory); + + if (IS_ERR_OR_NULL(kctx->kctx_dentry)) { + ret = -ENOMEM; + goto out; + } + +#ifdef CONFIG_MALI_COH_USER + /* if cache is completely coherent at hardware level, then remove the + * infinite cache control support from debugfs. + */ +#else + debugfs_create_file("infinite_cache", 0644, kctx->kctx_dentry, + kctx, &kbase_infinite_cache_fops); +#endif /* CONFIG_MALI_COH_USER */ + + mutex_init(&kctx->mem_profile_lock); + + kbasep_jd_debugfs_ctx_init(kctx); + kbase_debug_mem_view_init(filp); + + kbase_debug_job_fault_context_init(kctx); + + kbase_mem_pool_debugfs_init(kctx->kctx_dentry, &kctx->mem_pool); + + kbase_jit_debugfs_init(kctx); +#endif /* CONFIG_DEBUG_FS */ + + dev_dbg(kbdev->dev, "created base context\n"); + + { + struct kbasep_kctx_list_element *element; + + element = kzalloc(sizeof(*element), GFP_KERNEL); + if (element) { + mutex_lock(&kbdev->kctx_list_lock); + element->kctx = kctx; + list_add(&element->link, &kbdev->kctx_list); + KBASE_TLSTREAM_TL_NEW_CTX( + element->kctx, + (u32)(element->kctx->id), + (u32)(element->kctx->tgid)); + mutex_unlock(&kbdev->kctx_list_lock); + } else { + /* we don't treat this as a fail - just warn about it */ + dev_warn(kbdev->dev, "couldn't add kctx to kctx_list\n"); + } + } + return 0; + + out: + kbase_release_device(kbdev); + return ret; +} + +static int kbase_release(struct inode *inode, struct file *filp) +{ + struct kbase_context *kctx = filp->private_data; + struct kbase_device *kbdev = kctx->kbdev; + struct kbasep_kctx_list_element *element, *tmp; + bool found_element = false; + + KBASE_TLSTREAM_TL_DEL_CTX(kctx); + +#ifdef CONFIG_DEBUG_FS + kbasep_mem_profile_debugfs_remove(kctx); + kbase_debug_job_fault_context_term(kctx); +#endif + + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) { + if (element->kctx == kctx) { + list_del(&element->link); + kfree(element); + found_element = true; + } + } + mutex_unlock(&kbdev->kctx_list_lock); + if (!found_element) + dev_warn(kbdev->dev, "kctx not in kctx_list\n"); + + filp->private_data = NULL; + + mutex_lock(&kctx->vinstr_cli_lock); + /* If this client was performing hwcnt dumping and did not explicitly + * detach itself, remove it from the vinstr core now */ + if (kctx->vinstr_cli) { + struct kbase_uk_hwcnt_setup setup; + + setup.dump_buffer = 0llu; + kbase_vinstr_legacy_hwc_setup( + kbdev->vinstr_ctx, &kctx->vinstr_cli, &setup); + } + mutex_unlock(&kctx->vinstr_cli_lock); + + kbase_destroy_context(kctx); + + dev_dbg(kbdev->dev, "deleted base context\n"); + kbase_release_device(kbdev); + return 0; +} + +#define CALL_MAX_SIZE 536 + +static long kbase_legacy_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + u64 msg[(CALL_MAX_SIZE + 7) >> 3] = { 0xdeadbeefdeadbeefull }; /* alignment fixup */ + u32 size = _IOC_SIZE(cmd); + struct kbase_context *kctx = filp->private_data; + + if (size > CALL_MAX_SIZE) + return -ENOTTY; + + if (0 != copy_from_user(&msg, (void __user *)arg, size)) { + dev_err(kctx->kbdev->dev, "failed to copy ioctl argument into kernel space\n"); + return -EFAULT; + } + + if (kbase_legacy_dispatch(kctx, &msg, size) != 0) + return -EFAULT; + + if (0 != copy_to_user((void __user *)arg, &msg, size)) { + dev_err(kctx->kbdev->dev, "failed to copy results of UK call back to user space\n"); + return -EFAULT; + } + return 0; +} + +static int kbase_api_set_flags(struct kbase_context *kctx, + struct kbase_ioctl_set_flags *flags) +{ + int err; + + /* setup pending, try to signal that we'll do the setup, + * if setup was already in progress, err this call + */ + if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) + return -EINVAL; + + err = kbase_context_set_create_flags(kctx, flags->create_flags); + /* if bad flags, will stay stuck in setup mode */ + if (err) + return err; + + atomic_set(&kctx->setup_complete, 1); + return 0; +} + +static int kbase_api_job_submit(struct kbase_context *kctx, + struct kbase_ioctl_job_submit *submit) +{ + void __user *user_addr = NULL; + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + user_addr = compat_ptr(submit->addr.compat_value); + else +#endif + user_addr = submit->addr.value; + + return kbase_jd_submit(kctx, user_addr, submit->nr_atoms, + submit->stride, false); +} + +static int kbase_api_get_gpuprops(struct kbase_context *kctx, + struct kbase_ioctl_get_gpuprops *get_props) +{ + struct kbase_gpu_props *kprops = &kctx->kbdev->gpu_props; + int err; + + if (get_props->flags != 0) { + dev_err(kctx->kbdev->dev, "Unsupported flags to get_gpuprops"); + return -EINVAL; + } + + if (get_props->size == 0) + return kprops->prop_buffer_size; + if (get_props->size < kprops->prop_buffer_size) + return -EINVAL; + + err = copy_to_user(get_props->buffer.value, kprops->prop_buffer, + kprops->prop_buffer_size); + if (err) + return err; + return kprops->prop_buffer_size; +} + +static int kbase_api_post_term(struct kbase_context *kctx) +{ + kbase_event_close(kctx); + return 0; +} + +static int kbase_api_mem_alloc(struct kbase_context *kctx, + union kbase_ioctl_mem_alloc *alloc) +{ + struct kbase_va_region *reg; + u64 flags = alloc->in.flags; + u64 gpu_va; + +#if defined(CONFIG_64BIT) + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* force SAME_VA if a 64-bit client */ + flags |= BASE_MEM_SAME_VA; + } +#endif + + reg = kbase_mem_alloc(kctx, alloc->in.va_pages, + alloc->in.commit_pages, + alloc->in.extent, + &flags, &gpu_va); + + if (!reg) + return -ENOMEM; + + alloc->out.flags = flags; + alloc->out.gpu_va = gpu_va; + + return 0; +} + +static int kbase_api_mem_query(struct kbase_context *kctx, + union kbase_ioctl_mem_query *query) +{ + return kbase_mem_query(kctx, query->in.gpu_addr, + query->in.query, &query->out.value); +} + +static int kbase_api_mem_free(struct kbase_context *kctx, + struct kbase_ioctl_mem_free *free) +{ + return kbase_mem_free(kctx, free->gpu_addr); +} + +static int kbase_api_hwcnt_reader_setup(struct kbase_context *kctx, + struct kbase_ioctl_hwcnt_reader_setup *setup) +{ + int ret; + struct kbase_uk_hwcnt_reader_setup args = { + .buffer_count = setup->buffer_count, + .jm_bm = setup->jm_bm, + .shader_bm = setup->shader_bm, + .tiler_bm = setup->tiler_bm, + .mmu_l2_bm = setup->mmu_l2_bm + }; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_hwcnt_reader_setup(kctx->kbdev->vinstr_ctx, &args); + mutex_unlock(&kctx->vinstr_cli_lock); + + if (ret) + return ret; + return args.fd; +} + +static int kbase_api_hwcnt_enable(struct kbase_context *kctx, + struct kbase_ioctl_hwcnt_enable *enable) +{ + int ret; + struct kbase_uk_hwcnt_setup args = { + .dump_buffer = enable->dump_buffer, + .jm_bm = enable->jm_bm, + .shader_bm = enable->shader_bm, + .tiler_bm = enable->tiler_bm, + .mmu_l2_bm = enable->mmu_l2_bm + }; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_legacy_hwc_setup(kctx->kbdev->vinstr_ctx, + &kctx->vinstr_cli, &args); + mutex_unlock(&kctx->vinstr_cli_lock); + + return ret; +} + +static int kbase_api_hwcnt_dump(struct kbase_context *kctx) +{ + int ret; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_hwc_dump(kctx->vinstr_cli, + BASE_HWCNT_READER_EVENT_MANUAL); + mutex_unlock(&kctx->vinstr_cli_lock); + + return ret; +} + +static int kbase_api_hwcnt_clear(struct kbase_context *kctx) +{ + int ret; + + mutex_lock(&kctx->vinstr_cli_lock); + ret = kbase_vinstr_hwc_clear(kctx->vinstr_cli); + mutex_unlock(&kctx->vinstr_cli_lock); + + return ret; +} + +static int kbase_api_disjoint_query(struct kbase_context *kctx, + struct kbase_ioctl_disjoint_query *query) +{ + query->counter = kbase_disjoint_event_get(kctx->kbdev); + + return 0; +} + +static int kbase_api_get_ddk_version(struct kbase_context *kctx, + struct kbase_ioctl_get_ddk_version *version) +{ + int ret; + int len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); + + if (version->version_buffer.value == NULL) + return len; + + if (version->size < len) + return -EOVERFLOW; + + ret = copy_to_user(version->version_buffer.value, + KERNEL_SIDE_DDK_VERSION_STRING, + sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); + + if (ret) + return ret; + + return len; +} + +static int kbase_api_mem_jit_init(struct kbase_context *kctx, + struct kbase_ioctl_mem_jit_init *jit_init) +{ + return kbase_region_tracker_init_jit(kctx, jit_init->va_pages); +} + +static int kbase_api_mem_sync(struct kbase_context *kctx, + struct kbase_ioctl_mem_sync *sync) +{ +#ifdef CONFIG_MALI_COH_USER + return 0; +#endif + struct basep_syncset sset = { + .mem_handle.basep.handle = sync->handle, + .user_addr = sync->user_addr, + .size = sync->size, + .type = sync->type + }; + + return kbase_sync_now(kctx, &sset); +} + +static int kbase_api_mem_find_cpu_offset(struct kbase_context *kctx, + union kbase_ioctl_mem_find_cpu_offset *find) +{ + return kbasep_find_enclosing_cpu_mapping_offset( + kctx, + find->in.cpu_addr, + find->in.size, + &find->out.offset); +} + +static int kbase_api_get_context_id(struct kbase_context *kctx, + struct kbase_ioctl_get_context_id *info) +{ + info->id = kctx->id; + + return 0; +} + +static int kbase_api_tlstream_acquire(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_acquire *acquire) +{ + return kbase_tlstream_acquire(kctx, acquire->flags); +} + +static int kbase_api_tlstream_flush(struct kbase_context *kctx) +{ + kbase_tlstream_flush_streams(); + + return 0; +} + +static int kbase_api_mem_commit(struct kbase_context *kctx, + struct kbase_ioctl_mem_commit *commit) +{ + return kbase_mem_commit(kctx, commit->gpu_addr, commit->pages); +} + +static int kbase_api_mem_alias(struct kbase_context *kctx, + union kbase_ioctl_mem_alias *alias) +{ + struct base_mem_aliasing_info *ai; + void __user *user_addr = NULL; + u64 flags; + int err; + + if (alias->in.nents == 0 || alias->in.nents > 2048) + return -EINVAL; + + ai = vmalloc(sizeof(*ai) * alias->in.nents); + if (!ai) + return -ENOMEM; + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + user_addr = + compat_ptr(alias->in.aliasing_info.compat_value); + else +#endif + user_addr = alias->in.aliasing_info.value; + + err = copy_from_user(ai, user_addr, sizeof(*ai) * alias->in.nents); + if (err) { + vfree(ai); + return err; + } + + flags = alias->in.flags; + + alias->out.gpu_va = kbase_mem_alias(kctx, &flags, + alias->in.stride, alias->in.nents, + ai, &alias->out.va_pages); + + alias->out.flags = flags; + + vfree(ai); + + if (alias->out.gpu_va == 0) + return -ENOMEM; + + return 0; +} + +static int kbase_api_mem_import(struct kbase_context *kctx, + union kbase_ioctl_mem_import *import) +{ + int ret; + u64 flags = import->in.flags; + void __user *phandle; + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + phandle = compat_ptr(import->in.phandle.compat_value); + else +#endif + phandle = import->in.phandle.value; + + ret = kbase_mem_import(kctx, + import->in.type, + phandle, + import->in.padding, + &import->out.gpu_va, + &import->out.va_pages, + &flags); + + import->out.flags = flags; + + return ret; +} + +static int kbase_api_mem_flags_change(struct kbase_context *kctx, + struct kbase_ioctl_mem_flags_change *change) +{ + return kbase_mem_flags_change(kctx, change->gpu_va, + change->flags, change->mask); +} + +static int kbase_api_stream_create(struct kbase_context *kctx, + struct kbase_ioctl_stream_create *stream) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + int fd, ret; + + /* Name must be NULL-terminated and padded with NULLs, so check last + * character is NULL + */ + if (stream->name[sizeof(stream->name)-1] != 0) + return -EINVAL; + + ret = kbase_sync_fence_stream_create(stream->name, &fd); + + if (ret) + return ret; + return fd; +#else + return -ENOENT; +#endif +} + +static int kbase_api_fence_validate(struct kbase_context *kctx, + struct kbase_ioctl_fence_validate *validate) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + return kbase_sync_fence_validate(validate->fd); +#else + return -ENOENT; +#endif +} + +static int kbase_api_get_profiling_controls(struct kbase_context *kctx, + struct kbase_ioctl_get_profiling_controls *controls) +{ + if (controls->count > FBDUMP_CONTROL_MAX) + return -EINVAL; + + return copy_to_user(controls->buffer.value, + &kctx->kbdev->kbase_profiling_controls[ + FBDUMP_CONTROL_MIN], + controls->count * sizeof(u32)); +} + +static int kbase_api_mem_profile_add(struct kbase_context *kctx, + struct kbase_ioctl_mem_profile_add *data) +{ + char __user *user_buf; + char *buf; + int err; + + if (data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { + dev_err(kctx->kbdev->dev, "mem_profile_add: buffer too big\n"); + return -EINVAL; + } + + buf = kmalloc(data->len, GFP_KERNEL); + if (ZERO_OR_NULL_PTR(buf)) + return -ENOMEM; + +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + user_buf = compat_ptr(data->buffer.compat_value); + else +#endif + user_buf = data->buffer.value; + + err = copy_from_user(buf, user_buf, data->len); + if (err) { + kfree(buf); + return err; + } + + return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); +} + +static int kbase_api_soft_event_update(struct kbase_context *kctx, + struct kbase_ioctl_soft_event_update *update) +{ + if (update->flags != 0) + return -EINVAL; + + return kbase_soft_event_update(kctx, update->event, update->new_status); +} + +#if MALI_UNIT_TEST +static int kbase_api_tlstream_test(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_test *test) +{ + kbase_tlstream_test( + test->tpw_count, + test->msg_delay, + test->msg_count, + test->aux_msg); + + return 0; +} + +static int kbase_api_tlstream_stats(struct kbase_context *kctx, + struct kbase_ioctl_tlstream_stats *stats) +{ + kbase_tlstream_stats( + &stats->bytes_collected, + &stats->bytes_generated); + + return 0; +} +#endif /* MALI_UNIT_TEST */ + +#define KBASE_HANDLE_IOCTL(cmd, function) \ + case cmd: \ + do { \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ + return function(kctx); \ + } while (0) + +#define KBASE_HANDLE_IOCTL_IN(cmd, function, type) \ + case cmd: \ + do { \ + type param; \ + int err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return function(kctx, ¶m); \ + } while (0) + +#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type) \ + case cmd: \ + do { \ + type param; \ + int ret, err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + ret = function(kctx, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return ret; \ + } while (0) + +#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type) \ + case cmd: \ + do { \ + type param; \ + int ret, err; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + ret = function(kctx, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (err) \ + return -EFAULT; \ + return ret; \ + } while (0) + +static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct kbase_context *kctx = filp->private_data; + struct kbase_device *kbdev = kctx->kbdev; + void __user *uarg = (void __user *)arg; + + /* The UK ioctl values overflow the cmd field causing the type to be + * incremented + */ + if (_IOC_TYPE(cmd) == LINUX_UK_BASE_MAGIC+2) + return kbase_legacy_ioctl(filp, cmd, arg); + + /* The UK version check IOCTL doesn't overflow the cmd field, so is + * handled separately here + */ + if (cmd == _IOC(_IOC_READ|_IOC_WRITE, LINUX_UK_BASE_MAGIC, + UKP_FUNC_ID_CHECK_VERSION, + sizeof(struct uku_version_check_args))) + return kbase_legacy_ioctl(filp, cmd, arg); + + /* Only these ioctls are available until setup is complete */ + switch (cmd) { + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK, + kbase_api_handshake, + struct kbase_ioctl_version_check); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_FLAGS, + kbase_api_set_flags, + struct kbase_ioctl_set_flags); + } + + /* Block call until version handshake and setup is complete */ + if (kctx->api_version == 0 || !atomic_read(&kctx->setup_complete)) + return -EINVAL; + + /* Normal ioctls */ + switch (cmd) { + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, + kbase_api_job_submit, + struct kbase_ioctl_job_submit); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_GPUPROPS, + kbase_api_get_gpuprops, + struct kbase_ioctl_get_gpuprops); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, + kbase_api_post_term); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, + kbase_api_mem_alloc, + union kbase_ioctl_mem_alloc); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY, + kbase_api_mem_query, + union kbase_ioctl_mem_query); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FREE, + kbase_api_mem_free, + struct kbase_ioctl_mem_free); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_READER_SETUP, + kbase_api_hwcnt_reader_setup, + struct kbase_ioctl_hwcnt_reader_setup); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_ENABLE, + kbase_api_hwcnt_enable, + struct kbase_ioctl_hwcnt_enable); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_DUMP, + kbase_api_hwcnt_dump); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_CLEAR, + kbase_api_hwcnt_clear); + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_DISJOINT_QUERY, + kbase_api_disjoint_query, + struct kbase_ioctl_disjoint_query); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_DDK_VERSION, + kbase_api_get_ddk_version, + struct kbase_ioctl_get_ddk_version); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT, + kbase_api_mem_jit_init, + struct kbase_ioctl_mem_jit_init); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_SYNC, + kbase_api_mem_sync, + struct kbase_ioctl_mem_sync); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_CPU_OFFSET, + kbase_api_mem_find_cpu_offset, + union kbase_ioctl_mem_find_cpu_offset); + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_GET_CONTEXT_ID, + kbase_api_get_context_id, + struct kbase_ioctl_get_context_id); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_ACQUIRE, + kbase_api_tlstream_acquire, + struct kbase_ioctl_tlstream_acquire); + KBASE_HANDLE_IOCTL(KBASE_IOCTL_TLSTREAM_FLUSH, + kbase_api_tlstream_flush); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_COMMIT, + kbase_api_mem_commit, + struct kbase_ioctl_mem_commit); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALIAS, + kbase_api_mem_alias, + union kbase_ioctl_mem_alias); + KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_IMPORT, + kbase_api_mem_import, + union kbase_ioctl_mem_import); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FLAGS_CHANGE, + kbase_api_mem_flags_change, + struct kbase_ioctl_mem_flags_change); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STREAM_CREATE, + kbase_api_stream_create, + struct kbase_ioctl_stream_create); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_FENCE_VALIDATE, + kbase_api_fence_validate, + struct kbase_ioctl_fence_validate); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_PROFILING_CONTROLS, + kbase_api_get_profiling_controls, + struct kbase_ioctl_get_profiling_controls); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_PROFILE_ADD, + kbase_api_mem_profile_add, + struct kbase_ioctl_mem_profile_add); + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, + kbase_api_soft_event_update, + struct kbase_ioctl_soft_event_update); + +#if MALI_UNIT_TEST + KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_TEST, + kbase_api_tlstream_test, + struct kbase_ioctl_tlstream_test); + KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, + kbase_api_tlstream_stats, + struct kbase_ioctl_tlstream_stats); +#endif + } + + dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); + + return -ENOIOCTLCMD; +} + +static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) +{ + struct kbase_context *kctx = filp->private_data; + struct base_jd_event_v2 uevent; + int out_count = 0; + + if (count < sizeof(uevent)) + return -ENOBUFS; + + do { + while (kbase_event_dequeue(kctx, &uevent)) { + if (out_count > 0) + goto out; + + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; + + if (wait_event_interruptible(kctx->event_queue, + kbase_event_pending(kctx)) != 0) + return -ERESTARTSYS; + } + if (uevent.event_code == BASE_JD_EVENT_DRV_TERMINATED) { + if (out_count == 0) + return -EPIPE; + goto out; + } + + if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) + return -EFAULT; + + buf += sizeof(uevent); + out_count++; + count -= sizeof(uevent); + } while (count >= sizeof(uevent)); + + out: + return out_count * sizeof(uevent); +} + +static unsigned int kbase_poll(struct file *filp, poll_table *wait) +{ + struct kbase_context *kctx = filp->private_data; + + poll_wait(filp, &kctx->event_queue, wait); + if (kbase_event_pending(kctx)) + return POLLIN | POLLRDNORM; + + return 0; +} + +void kbase_event_wakeup(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx); + + wake_up_interruptible(&kctx->event_queue); +} + +KBASE_EXPORT_TEST_API(kbase_event_wakeup); + +static int kbase_check_flags(int flags) +{ + /* Enforce that the driver keeps the O_CLOEXEC flag so that execve() always + * closes the file descriptor in a child process. + */ + if (0 == (flags & O_CLOEXEC)) + return -EINVAL; + + return 0; +} + + +/** + * align_and_check - Align the specified pointer to the provided alignment and + * check that it is still in range. + * @gap_end: Highest possible start address for allocation (end of gap in + * address space) + * @gap_start: Start address of current memory area / gap in address space + * @info: vm_unmapped_area_info structure passed to caller, containing + * alignment, length and limits for the allocation + * @is_shader_code: True if the allocation is for shader code (which has + * additional alignment requirements) + * + * Return: true if gap_end is now aligned correctly and is still in range, + * false otherwise + */ +static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, + struct vm_unmapped_area_info *info, bool is_shader_code) +{ + /* Compute highest gap address at the desired alignment */ + (*gap_end) -= info->length; + (*gap_end) -= (*gap_end - info->align_offset) & info->align_mask; + + if (is_shader_code) { + /* Check for 4GB boundary */ + if (0 == (*gap_end & BASE_MEM_MASK_4GB)) + (*gap_end) -= (info->align_offset ? info->align_offset : + info->length); + if (0 == ((*gap_end + info->length) & BASE_MEM_MASK_4GB)) + (*gap_end) -= (info->align_offset ? info->align_offset : + info->length); + + if (!(*gap_end & BASE_MEM_MASK_4GB) || !((*gap_end + + info->length) & BASE_MEM_MASK_4GB)) + return false; + } + + + if ((*gap_end < info->low_limit) || (*gap_end < gap_start)) + return false; + + + return true; +} + +/* The following function is taken from the kernel and just + * renamed. As it's not exported to modules we must copy-paste it here. + */ + +static unsigned long kbase_unmapped_area_topdown(struct vm_unmapped_area_info + *info, bool is_shader_code) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + unsigned long length, low_limit, high_limit, gap_start, gap_end; + + /* Adjust search length to account for worst case alignment overhead */ + length = info->length + info->align_mask; + if (length < info->length) + return -ENOMEM; + + /* + * Adjust search limits by the desired length. + * See implementation comment at top of unmapped_area(). + */ + gap_end = info->high_limit; + if (gap_end < length) + return -ENOMEM; + high_limit = gap_end - length; + + if (info->low_limit > high_limit) + return -ENOMEM; + low_limit = info->low_limit + length; + + /* Check highest gap, which does not precede any rbtree node */ + gap_start = mm->highest_vm_end; + if (gap_start <= high_limit) { + if (align_and_check(&gap_end, gap_start, info, is_shader_code)) + return gap_end; + } + + /* Check if rbtree root looks promising */ + if (RB_EMPTY_ROOT(&mm->mm_rb)) + return -ENOMEM; + vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); + if (vma->rb_subtree_gap < length) + return -ENOMEM; + + while (true) { + /* Visit right subtree if it looks promising */ + gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0; + if (gap_start <= high_limit && vma->vm_rb.rb_right) { + struct vm_area_struct *right = + rb_entry(vma->vm_rb.rb_right, + struct vm_area_struct, vm_rb); + if (right->rb_subtree_gap >= length) { + vma = right; + continue; + } + } + +check_current: + /* Check if current node has a suitable gap */ + gap_end = vma->vm_start; + if (gap_end < low_limit) + return -ENOMEM; + if (gap_start <= high_limit && gap_end - gap_start >= length) { + /* We found a suitable gap. Clip it with the original + * high_limit. */ + if (gap_end > info->high_limit) + gap_end = info->high_limit; + + if (align_and_check(&gap_end, gap_start, info, + is_shader_code)) + return gap_end; + } + + /* Visit left subtree if it looks promising */ + if (vma->vm_rb.rb_left) { + struct vm_area_struct *left = + rb_entry(vma->vm_rb.rb_left, + struct vm_area_struct, vm_rb); + if (left->rb_subtree_gap >= length) { + vma = left; + continue; + } + } + + /* Go back up the rbtree to find next candidate node */ + while (true) { + struct rb_node *prev = &vma->vm_rb; + if (!rb_parent(prev)) + return -ENOMEM; + vma = rb_entry(rb_parent(prev), + struct vm_area_struct, vm_rb); + if (prev == vma->vm_rb.rb_right) { + gap_start = vma->vm_prev ? + vma->vm_prev->vm_end : 0; + goto check_current; + } + } + } + + return -ENOMEM; +} + +static unsigned long kbase_get_unmapped_area(struct file *filp, + const unsigned long addr, const unsigned long len, + const unsigned long pgoff, const unsigned long flags) +{ + /* based on get_unmapped_area, but simplified slightly due to that some + * values are known in advance */ + struct kbase_context *kctx = filp->private_data; + struct mm_struct *mm = current->mm; + struct vm_unmapped_area_info info; + unsigned long align_offset = 0; + unsigned long align_mask = 0; + unsigned long high_limit = mm->mmap_base; + unsigned long low_limit = PAGE_SIZE; + int cpu_va_bits = BITS_PER_LONG; + int gpu_pc_bits = + kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; + bool is_shader_code = false; + unsigned long ret; + + /* err on fixed address */ + if ((flags & MAP_FIXED) || addr) + return -EINVAL; + +#ifdef CONFIG_64BIT + /* too big? */ + if (len > TASK_SIZE - SZ_2M) + return -ENOMEM; + + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + + if (kbase_hw_has_feature(kctx->kbdev, + BASE_HW_FEATURE_33BIT_VA)) { + high_limit = kctx->same_va_end << PAGE_SHIFT; + } else { + high_limit = min_t(unsigned long, mm->mmap_base, + (kctx->same_va_end << PAGE_SHIFT)); + if (len >= SZ_2M) { + align_offset = SZ_2M; + align_mask = SZ_2M - 1; + } + } + + low_limit = SZ_2M; + } else { + cpu_va_bits = 32; + } +#endif /* CONFIG_64BIT */ + if ((PFN_DOWN(BASE_MEM_COOKIE_BASE) <= pgoff) && + (PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) > pgoff)) { + int cookie = pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); + + if (!kctx->pending_regions[cookie]) + return -EINVAL; + + if (!(kctx->pending_regions[cookie]->flags & + KBASE_REG_GPU_NX)) { + if (cpu_va_bits > gpu_pc_bits) { + align_offset = 1ULL << gpu_pc_bits; + align_mask = align_offset - 1; + is_shader_code = true; + } + } +#ifndef CONFIG_64BIT + } else { + return current->mm->get_unmapped_area(filp, addr, len, pgoff, + flags); +#endif + } + + info.flags = 0; + info.length = len; + info.low_limit = low_limit; + info.high_limit = high_limit; + info.align_offset = align_offset; + info.align_mask = align_mask; + + ret = kbase_unmapped_area_topdown(&info, is_shader_code); + + if (IS_ERR_VALUE(ret) && high_limit == mm->mmap_base && + high_limit < (kctx->same_va_end << PAGE_SHIFT)) { + /* Retry above mmap_base */ + info.low_limit = mm->mmap_base; + info.high_limit = min_t(u64, TASK_SIZE, + (kctx->same_va_end << PAGE_SHIFT)); + + ret = kbase_unmapped_area_topdown(&info, is_shader_code); + } + + return ret; +} + +static const struct file_operations kbase_fops = { + .owner = THIS_MODULE, + .open = kbase_open, + .release = kbase_release, + .read = kbase_read, + .poll = kbase_poll, + .unlocked_ioctl = kbase_ioctl, + .compat_ioctl = kbase_ioctl, + .mmap = kbase_mmap, + .check_flags = kbase_check_flags, + .get_unmapped_area = kbase_get_unmapped_area, +}; + +#ifndef CONFIG_MALI_NO_MALI +void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value) +{ + writel(value, kbdev->reg + offset); +} + +u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset) +{ + return readl(kbdev->reg + offset); +} +#endif /* !CONFIG_MALI_NO_MALI */ + +/** + * show_policy - Show callback for the power_policy sysfs file. + * + * This function is called to get the contents of the power_policy sysfs + * file. This is a list of the available policies with the currently active one + * surrounded by square brackets. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf) +{ + struct kbase_device *kbdev; + const struct kbase_pm_policy *current_policy; + const struct kbase_pm_policy *const *policy_list; + int policy_count; + int i; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + current_policy = kbase_pm_get_policy(kbdev); + + policy_count = kbase_pm_list_policies(&policy_list); + + for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { + if (policy_list[i] == current_policy) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); + else + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); + } + + if (ret < PAGE_SIZE - 1) { + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); + } else { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/** + * set_policy - Store callback for the power_policy sysfs file. + * + * This function is called when the power_policy sysfs file is written to. + * It matches the requested policy against the available policies and if a + * matching policy is found calls kbase_pm_set_policy() to change the + * policy. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + const struct kbase_pm_policy *new_policy = NULL; + const struct kbase_pm_policy *const *policy_list; + int policy_count; + int i; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + policy_count = kbase_pm_list_policies(&policy_list); + + for (i = 0; i < policy_count; i++) { + if (sysfs_streq(policy_list[i]->name, buf)) { + new_policy = policy_list[i]; + break; + } + } + + if (!new_policy) { + dev_err(dev, "power_policy: policy not found\n"); + return -EINVAL; + } + + kbase_pm_set_policy(kbdev, new_policy); + + return count; +} + +/* + * The sysfs file power_policy. + * + * This is used for obtaining information about the available policies, + * determining which policy is currently active, and changing the active + * policy. + */ +static DEVICE_ATTR(power_policy, S_IRUGO | S_IWUSR, show_policy, set_policy); + +/** + * show_ca_policy - Show callback for the core_availability_policy sysfs file. + * + * This function is called to get the contents of the core_availability_policy + * sysfs file. This is a list of the available policies with the currently + * active one surrounded by square brackets. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_ca_policy(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + const struct kbase_pm_ca_policy *current_policy; + const struct kbase_pm_ca_policy *const *policy_list; + int policy_count; + int i; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + current_policy = kbase_pm_ca_get_policy(kbdev); + + policy_count = kbase_pm_ca_list_policies(&policy_list); + + for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { + if (policy_list[i] == current_policy) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); + else + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); + } + + if (ret < PAGE_SIZE - 1) { + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); + } else { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/** + * set_ca_policy - Store callback for the core_availability_policy sysfs file. + * + * This function is called when the core_availability_policy sysfs file is + * written to. It matches the requested policy against the available policies + * and if a matching policy is found calls kbase_pm_set_policy() to change + * the policy. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_ca_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + const struct kbase_pm_ca_policy *new_policy = NULL; + const struct kbase_pm_ca_policy *const *policy_list; + int policy_count; + int i; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + policy_count = kbase_pm_ca_list_policies(&policy_list); + + for (i = 0; i < policy_count; i++) { + if (sysfs_streq(policy_list[i]->name, buf)) { + new_policy = policy_list[i]; + break; + } + } + + if (!new_policy) { + dev_err(dev, "core_availability_policy: policy not found\n"); + return -EINVAL; + } + + kbase_pm_ca_set_policy(kbdev, new_policy); + + return count; +} + +/* + * The sysfs file core_availability_policy + * + * This is used for obtaining information about the available policies, + * determining which policy is currently active, and changing the active + * policy. + */ +static DEVICE_ATTR(core_availability_policy, S_IRUGO | S_IWUSR, show_ca_policy, set_ca_policy); + +/* + * show_core_mask - Show callback for the core_mask sysfs file. + * + * This function is called to get the contents of the core_mask sysfs file. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Current core mask (JS0) : 0x%llX\n", + kbdev->pm.debug_core_mask[0]); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Current core mask (JS1) : 0x%llX\n", + kbdev->pm.debug_core_mask[1]); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Current core mask (JS2) : 0x%llX\n", + kbdev->pm.debug_core_mask[2]); + ret += scnprintf(buf + ret, PAGE_SIZE - ret, + "Available core mask : 0x%llX\n", + kbdev->gpu_props.props.raw_props.shader_present); + + return ret; +} + +/** + * set_core_mask - Store callback for the core_mask sysfs file. + * + * This function is called when the core_mask sysfs file is written to. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + u64 new_core_mask[3]; + int items; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + items = sscanf(buf, "%llx %llx %llx", + &new_core_mask[0], &new_core_mask[1], + &new_core_mask[2]); + + if (items == 1) + new_core_mask[1] = new_core_mask[2] = new_core_mask[0]; + + if (items == 1 || items == 3) { + u64 shader_present = + kbdev->gpu_props.props.raw_props.shader_present; + u64 group0_core_mask = + kbdev->gpu_props.props.coherency_info.group[0]. + core_mask; + + if ((new_core_mask[0] & shader_present) != new_core_mask[0] || + !(new_core_mask[0] & group0_core_mask) || + (new_core_mask[1] & shader_present) != + new_core_mask[1] || + !(new_core_mask[1] & group0_core_mask) || + (new_core_mask[2] & shader_present) != + new_core_mask[2] || + !(new_core_mask[2] & group0_core_mask)) { + dev_err(dev, "power_policy: invalid core specification\n"); + return -EINVAL; + } + + if (kbdev->pm.debug_core_mask[0] != new_core_mask[0] || + kbdev->pm.debug_core_mask[1] != + new_core_mask[1] || + kbdev->pm.debug_core_mask[2] != + new_core_mask[2]) { + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_pm_set_debug_core_mask(kbdev, new_core_mask[0], + new_core_mask[1], new_core_mask[2]); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } + + return count; + } + + dev_err(kbdev->dev, "Couldn't process set_core_mask write operation.\n" + "Use format \n" + "or \n"); + return -EINVAL; +} + +/* + * The sysfs file core_mask. + * + * This is used to restrict shader core availability for debugging purposes. + * Reading it will show the current core mask and the mask of cores available. + * Writing to it will set the current core mask. + */ +static DEVICE_ATTR(core_mask, S_IRUGO | S_IWUSR, show_core_mask, set_core_mask); + +/** + * set_soft_job_timeout - Store callback for the soft_job_timeout sysfs + * file. + * + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The value written to the sysfs file. + * @count: The number of bytes written to the sysfs file. + * + * This allows setting the timeout for software jobs. Waiting soft event wait + * jobs will be cancelled after this period expires, while soft fence wait jobs + * will print debug information if the fence debug feature is enabled. + * + * This is expressed in milliseconds. + * + * Return: count if the function succeeded. An error code on failure. + */ +static ssize_t set_soft_job_timeout(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int soft_job_timeout_ms; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + if ((kstrtoint(buf, 0, &soft_job_timeout_ms) != 0) || + (soft_job_timeout_ms <= 0)) + return -EINVAL; + + atomic_set(&kbdev->js_data.soft_job_timeout_ms, + soft_job_timeout_ms); + + return count; +} + +/** + * show_soft_job_timeout - Show callback for the soft_job_timeout sysfs + * file. + * + * This will return the timeout for the software jobs. + * + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer for the sysfs file contents. + * + * Return: The number of bytes output to buf. + */ +static ssize_t show_soft_job_timeout(struct device *dev, + struct device_attribute *attr, + char * const buf) +{ + struct kbase_device *kbdev; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + return scnprintf(buf, PAGE_SIZE, "%i\n", + atomic_read(&kbdev->js_data.soft_job_timeout_ms)); +} + +static DEVICE_ATTR(soft_job_timeout, S_IRUGO | S_IWUSR, + show_soft_job_timeout, set_soft_job_timeout); + +static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, + int default_ticks, u32 old_ticks) +{ + if (timeout_ms > 0) { + u64 ticks = timeout_ms * 1000000ULL; + do_div(ticks, kbdev->js_data.scheduling_period_ns); + if (!ticks) + return 1; + return ticks; + } else if (timeout_ms < 0) { + return default_ticks; + } else { + return old_ticks; + } +} + +/** + * set_js_timeouts - Store callback for the js_timeouts sysfs file. + * + * This function is called to get the contents of the js_timeouts sysfs + * file. This file contains five values separated by whitespace. The values + * are basically the same as %JS_SOFT_STOP_TICKS, %JS_HARD_STOP_TICKS_SS, + * %JS_HARD_STOP_TICKS_DUMPING, %JS_RESET_TICKS_SS, %JS_RESET_TICKS_DUMPING + * configuration values (in that order), with the difference that the js_timeout + * values are expressed in MILLISECONDS. + * + * The js_timeouts sysfile file allows the current values in + * use by the job scheduler to get override. Note that a value needs to + * be other than 0 for it to override the current job scheduler value. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_js_timeouts(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int items; + long js_soft_stop_ms; + long js_soft_stop_ms_cl; + long js_hard_stop_ms_ss; + long js_hard_stop_ms_cl; + long js_hard_stop_ms_dumping; + long js_reset_ms_ss; + long js_reset_ms_cl; + long js_reset_ms_dumping; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + items = sscanf(buf, "%ld %ld %ld %ld %ld %ld %ld %ld", + &js_soft_stop_ms, &js_soft_stop_ms_cl, + &js_hard_stop_ms_ss, &js_hard_stop_ms_cl, + &js_hard_stop_ms_dumping, &js_reset_ms_ss, + &js_reset_ms_cl, &js_reset_ms_dumping); + + if (items == 8) { + struct kbasep_js_device_data *js_data = &kbdev->js_data; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + +#define UPDATE_TIMEOUT(ticks_name, ms_name, default) do {\ + js_data->ticks_name = timeout_ms_to_ticks(kbdev, ms_name, \ + default, js_data->ticks_name); \ + dev_dbg(kbdev->dev, "Overriding " #ticks_name \ + " with %lu ticks (%lu ms)\n", \ + (unsigned long)js_data->ticks_name, \ + ms_name); \ + } while (0) + + UPDATE_TIMEOUT(soft_stop_ticks, js_soft_stop_ms, + DEFAULT_JS_SOFT_STOP_TICKS); + UPDATE_TIMEOUT(soft_stop_ticks_cl, js_soft_stop_ms_cl, + DEFAULT_JS_SOFT_STOP_TICKS_CL); + UPDATE_TIMEOUT(hard_stop_ticks_ss, js_hard_stop_ms_ss, + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? + DEFAULT_JS_HARD_STOP_TICKS_SS_8408 : + DEFAULT_JS_HARD_STOP_TICKS_SS); + UPDATE_TIMEOUT(hard_stop_ticks_cl, js_hard_stop_ms_cl, + DEFAULT_JS_HARD_STOP_TICKS_CL); + UPDATE_TIMEOUT(hard_stop_ticks_dumping, + js_hard_stop_ms_dumping, + DEFAULT_JS_HARD_STOP_TICKS_DUMPING); + UPDATE_TIMEOUT(gpu_reset_ticks_ss, js_reset_ms_ss, + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? + DEFAULT_JS_RESET_TICKS_SS_8408 : + DEFAULT_JS_RESET_TICKS_SS); + UPDATE_TIMEOUT(gpu_reset_ticks_cl, js_reset_ms_cl, + DEFAULT_JS_RESET_TICKS_CL); + UPDATE_TIMEOUT(gpu_reset_ticks_dumping, js_reset_ms_dumping, + DEFAULT_JS_RESET_TICKS_DUMPING); + + kbase_js_set_timeouts(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return count; + } + + dev_err(kbdev->dev, "Couldn't process js_timeouts write operation.\n" + "Use format \n" + "Write 0 for no change, -1 to restore default timeout\n"); + return -EINVAL; +} + +static unsigned long get_js_timeout_in_ms( + u32 scheduling_period_ns, + u32 ticks) +{ + u64 ms = (u64)ticks * scheduling_period_ns; + + do_div(ms, 1000000UL); + return ms; +} + +/** + * show_js_timeouts - Show callback for the js_timeouts sysfs file. + * + * This function is called to get the contents of the js_timeouts sysfs + * file. It returns the last set values written to the js_timeouts sysfs file. + * If the file didn't get written yet, the values will be current setting in + * use. + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_js_timeouts(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + unsigned long js_soft_stop_ms; + unsigned long js_soft_stop_ms_cl; + unsigned long js_hard_stop_ms_ss; + unsigned long js_hard_stop_ms_cl; + unsigned long js_hard_stop_ms_dumping; + unsigned long js_reset_ms_ss; + unsigned long js_reset_ms_cl; + unsigned long js_reset_ms_dumping; + u32 scheduling_period_ns; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + scheduling_period_ns = kbdev->js_data.scheduling_period_ns; + +#define GET_TIMEOUT(name) get_js_timeout_in_ms(\ + scheduling_period_ns, \ + kbdev->js_data.name) + + js_soft_stop_ms = GET_TIMEOUT(soft_stop_ticks); + js_soft_stop_ms_cl = GET_TIMEOUT(soft_stop_ticks_cl); + js_hard_stop_ms_ss = GET_TIMEOUT(hard_stop_ticks_ss); + js_hard_stop_ms_cl = GET_TIMEOUT(hard_stop_ticks_cl); + js_hard_stop_ms_dumping = GET_TIMEOUT(hard_stop_ticks_dumping); + js_reset_ms_ss = GET_TIMEOUT(gpu_reset_ticks_ss); + js_reset_ms_cl = GET_TIMEOUT(gpu_reset_ticks_cl); + js_reset_ms_dumping = GET_TIMEOUT(gpu_reset_ticks_dumping); + +#undef GET_TIMEOUT + + ret = scnprintf(buf, PAGE_SIZE, "%lu %lu %lu %lu %lu %lu %lu %lu\n", + js_soft_stop_ms, js_soft_stop_ms_cl, + js_hard_stop_ms_ss, js_hard_stop_ms_cl, + js_hard_stop_ms_dumping, js_reset_ms_ss, + js_reset_ms_cl, js_reset_ms_dumping); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/* + * The sysfs file js_timeouts. + * + * This is used to override the current job scheduler values for + * JS_STOP_STOP_TICKS_SS + * JS_STOP_STOP_TICKS_CL + * JS_HARD_STOP_TICKS_SS + * JS_HARD_STOP_TICKS_CL + * JS_HARD_STOP_TICKS_DUMPING + * JS_RESET_TICKS_SS + * JS_RESET_TICKS_CL + * JS_RESET_TICKS_DUMPING. + */ +static DEVICE_ATTR(js_timeouts, S_IRUGO | S_IWUSR, show_js_timeouts, set_js_timeouts); + +static u32 get_new_js_timeout( + u32 old_period, + u32 old_ticks, + u32 new_scheduling_period_ns) +{ + u64 ticks = (u64)old_period * (u64)old_ticks; + do_div(ticks, new_scheduling_period_ns); + return ticks?ticks:1; +} + +/** + * set_js_scheduling_period - Store callback for the js_scheduling_period sysfs + * file + * @dev: The device the sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the js_scheduling_period sysfs file is written + * to. It checks the data written, and if valid updates the js_scheduling_period + * value + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_js_scheduling_period(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + unsigned int js_scheduling_period; + u32 new_scheduling_period_ns; + u32 old_period; + struct kbasep_js_device_data *js_data; + unsigned long flags; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + js_data = &kbdev->js_data; + + ret = kstrtouint(buf, 0, &js_scheduling_period); + if (ret || !js_scheduling_period) { + dev_err(kbdev->dev, "Couldn't process js_scheduling_period write operation.\n" + "Use format \n"); + return -EINVAL; + } + + new_scheduling_period_ns = js_scheduling_period * 1000000; + + /* Update scheduling timeouts */ + mutex_lock(&js_data->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* If no contexts have been scheduled since js_timeouts was last written + * to, the new timeouts might not have been latched yet. So check if an + * update is pending and use the new values if necessary. */ + + /* Use previous 'new' scheduling period as a base if present. */ + old_period = js_data->scheduling_period_ns; + +#define SET_TIMEOUT(name) \ + (js_data->name = get_new_js_timeout(\ + old_period, \ + kbdev->js_data.name, \ + new_scheduling_period_ns)) + + SET_TIMEOUT(soft_stop_ticks); + SET_TIMEOUT(soft_stop_ticks_cl); + SET_TIMEOUT(hard_stop_ticks_ss); + SET_TIMEOUT(hard_stop_ticks_cl); + SET_TIMEOUT(hard_stop_ticks_dumping); + SET_TIMEOUT(gpu_reset_ticks_ss); + SET_TIMEOUT(gpu_reset_ticks_cl); + SET_TIMEOUT(gpu_reset_ticks_dumping); + +#undef SET_TIMEOUT + + js_data->scheduling_period_ns = new_scheduling_period_ns; + + kbase_js_set_timeouts(kbdev); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_data->runpool_mutex); + + dev_dbg(kbdev->dev, "JS scheduling period: %dms\n", + js_scheduling_period); + + return count; +} + +/** + * show_js_scheduling_period - Show callback for the js_scheduling_period sysfs + * entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current period used for the JS scheduling + * period. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_js_scheduling_period(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + u32 period; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + period = kbdev->js_data.scheduling_period_ns; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", + period / 1000000); + + return ret; +} + +static DEVICE_ATTR(js_scheduling_period, S_IRUGO | S_IWUSR, + show_js_scheduling_period, set_js_scheduling_period); + +#if !MALI_CUSTOMER_RELEASE +/** + * set_force_replay - Store callback for the force_replay sysfs file. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_force_replay(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + if (!strncmp("limit=", buf, MIN(6, count))) { + int force_replay_limit; + int items = sscanf(buf, "limit=%u", &force_replay_limit); + + if (items == 1) { + kbdev->force_replay_random = false; + kbdev->force_replay_limit = force_replay_limit; + kbdev->force_replay_count = 0; + + return count; + } + } else if (!strncmp("random_limit", buf, MIN(12, count))) { + kbdev->force_replay_random = true; + kbdev->force_replay_count = 0; + + return count; + } else if (!strncmp("norandom_limit", buf, MIN(14, count))) { + kbdev->force_replay_random = false; + kbdev->force_replay_limit = KBASEP_FORCE_REPLAY_DISABLED; + kbdev->force_replay_count = 0; + + return count; + } else if (!strncmp("core_req=", buf, MIN(9, count))) { + unsigned int core_req; + int items = sscanf(buf, "core_req=%x", &core_req); + + if (items == 1) { + kbdev->force_replay_core_req = (base_jd_core_req)core_req; + + return count; + } + } + dev_err(kbdev->dev, "Couldn't process force_replay write operation.\nPossible settings: limit=, random_limit, norandom_limit, core_req=\n"); + return -EINVAL; +} + +/** + * show_force_replay - Show callback for the force_replay sysfs file. + * + * This function is called to get the contents of the force_replay sysfs + * file. It returns the last set value written to the force_replay sysfs file. + * If the file didn't get written yet, the values will be 0. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_force_replay(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + if (kbdev->force_replay_random) + ret = scnprintf(buf, PAGE_SIZE, + "limit=0\nrandom_limit\ncore_req=%x\n", + kbdev->force_replay_core_req); + else + ret = scnprintf(buf, PAGE_SIZE, + "limit=%u\nnorandom_limit\ncore_req=%x\n", + kbdev->force_replay_limit, + kbdev->force_replay_core_req); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/* + * The sysfs file force_replay. + */ +static DEVICE_ATTR(force_replay, S_IRUGO | S_IWUSR, show_force_replay, + set_force_replay); +#endif /* !MALI_CUSTOMER_RELEASE */ + +#ifdef CONFIG_MALI_DEBUG +static ssize_t set_js_softstop_always(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + int softstop_always; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtoint(buf, 0, &softstop_always); + if (ret || ((softstop_always != 0) && (softstop_always != 1))) { + dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->js_data.softstop_always = (bool) softstop_always; + dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", + (kbdev->js_data.softstop_always) ? + "Enabled" : "Disabled"); + return count; +} + +static ssize_t show_js_softstop_always(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->js_data.softstop_always); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/* + * By default, soft-stops are disabled when only a single context is present. + * The ability to enable soft-stop when only a single context is present can be + * used for debug and unit-testing purposes. + * (see CL t6xx_stress_1 unit-test as an example whereby this feature is used.) + */ +static DEVICE_ATTR(js_softstop_always, S_IRUGO | S_IWUSR, show_js_softstop_always, set_js_softstop_always); +#endif /* CONFIG_MALI_DEBUG */ + +#ifdef CONFIG_MALI_DEBUG +typedef void (kbasep_debug_command_func) (struct kbase_device *); + +enum kbasep_debug_command_code { + KBASEP_DEBUG_COMMAND_DUMPTRACE, + + /* This must be the last enum */ + KBASEP_DEBUG_COMMAND_COUNT +}; + +struct kbasep_debug_command { + char *str; + kbasep_debug_command_func *func; +}; + +/* Debug commands supported by the driver */ +static const struct kbasep_debug_command debug_commands[] = { + { + .str = "dumptrace", + .func = &kbasep_trace_dump, + } +}; + +/** + * show_debug - Show callback for the debug_command sysfs file. + * + * This function is called to get the contents of the debug_command sysfs + * file. This is a list of the available debug commands, separated by newlines. + * + * @dev: The device this sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The output buffer for the sysfs file contents + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_debug(struct device *dev, struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + int i; + ssize_t ret = 0; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT && ret < PAGE_SIZE; i++) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s\n", debug_commands[i].str); + + if (ret >= PAGE_SIZE) { + buf[PAGE_SIZE - 2] = '\n'; + buf[PAGE_SIZE - 1] = '\0'; + ret = PAGE_SIZE - 1; + } + + return ret; +} + +/** + * issue_debug - Store callback for the debug_command sysfs file. + * + * This function is called when the debug_command sysfs file is written to. + * It matches the requested command against the available commands, and if + * a matching command is found calls the associated function from + * @debug_commands to issue the command. + * + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t issue_debug(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int i; + + kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT; i++) { + if (sysfs_streq(debug_commands[i].str, buf)) { + debug_commands[i].func(kbdev); + return count; + } + } + + /* Debug Command not found */ + dev_err(dev, "debug_command: command not known\n"); + return -EINVAL; +} + +/* The sysfs file debug_command. + * + * This is used to issue general debug commands to the device driver. + * Reading it will produce a list of debug commands, separated by newlines. + * Writing to it with one of those commands will issue said command. + */ +static DEVICE_ATTR(debug_command, S_IRUGO | S_IWUSR, show_debug, issue_debug); +#endif /* CONFIG_MALI_DEBUG */ + +/** + * kbase_show_gpuinfo - Show callback for the gpuinfo sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get a description of the present Mali + * GPU via the gpuinfo sysfs entry. This includes the GPU family, the + * number of cores, the hardware version and the raw product id. For + * example + * + * Mali-T60x MP4 r0p0 0x6956 + * + * Return: The number of bytes output to @buf. + */ +static ssize_t kbase_show_gpuinfo(struct device *dev, + struct device_attribute *attr, char *buf) +{ + static const struct gpu_product_id_name { + unsigned id; + char *name; + } gpu_product_id_names[] = { + { .id = GPU_ID_PI_T60X, .name = "Mali-T60x" }, + { .id = GPU_ID_PI_T62X, .name = "Mali-T62x" }, + { .id = GPU_ID_PI_T72X, .name = "Mali-T72x" }, + { .id = GPU_ID_PI_T76X, .name = "Mali-T76x" }, + { .id = GPU_ID_PI_T82X, .name = "Mali-T82x" }, + { .id = GPU_ID_PI_T83X, .name = "Mali-T83x" }, + { .id = GPU_ID_PI_T86X, .name = "Mali-T86x" }, + { .id = GPU_ID_PI_TFRX, .name = "Mali-T88x" }, + { .id = GPU_ID2_PRODUCT_TMIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G71" }, + { .id = GPU_ID2_PRODUCT_THEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-THEx" }, + { .id = GPU_ID2_PRODUCT_TSIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G51" }, + }; + const char *product_name = "(Unknown Mali GPU)"; + struct kbase_device *kbdev; + u32 gpu_id; + unsigned product_id, product_id_mask; + unsigned i; + bool is_new_format; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + product_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; + is_new_format = GPU_ID_IS_NEW_FORMAT(product_id); + product_id_mask = + (is_new_format ? + GPU_ID2_PRODUCT_MODEL : + GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + for (i = 0; i < ARRAY_SIZE(gpu_product_id_names); ++i) { + const struct gpu_product_id_name *p = &gpu_product_id_names[i]; + + if ((GPU_ID_IS_NEW_FORMAT(p->id) == is_new_format) && + (p->id & product_id_mask) == + (product_id & product_id_mask)) { + product_name = p->name; + break; + } + } + + return scnprintf(buf, PAGE_SIZE, "%s %d cores r%dp%d 0x%04X\n", + product_name, kbdev->gpu_props.num_cores, + (gpu_id & GPU_ID_VERSION_MAJOR) >> GPU_ID_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID_VERSION_MINOR) >> GPU_ID_VERSION_MINOR_SHIFT, + product_id); +} +static DEVICE_ATTR(gpuinfo, S_IRUGO, kbase_show_gpuinfo, NULL); + +/** + * set_dvfs_period - Store callback for the dvfs_period sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the dvfs_period sysfs file is written to. It + * checks the data written, and if valid updates the DVFS period variable, + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_dvfs_period(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + int dvfs_period; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtoint(buf, 0, &dvfs_period); + if (ret || dvfs_period <= 0) { + dev_err(kbdev->dev, "Couldn't process dvfs_period write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->pm.dvfs_period = dvfs_period; + dev_dbg(kbdev->dev, "DVFS period: %dms\n", dvfs_period); + + return count; +} + +/** + * show_dvfs_period - Show callback for the dvfs_period sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current period used for the DVFS sample + * timer. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_dvfs_period(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->pm.dvfs_period); + + return ret; +} + +static DEVICE_ATTR(dvfs_period, S_IRUGO | S_IWUSR, show_dvfs_period, + set_dvfs_period); + +/** + * set_pm_poweroff - Store callback for the pm_poweroff sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the pm_poweroff sysfs file is written to. + * + * This file contains three values separated by whitespace. The values + * are gpu_poweroff_time (the period of the poweroff timer, in ns), + * poweroff_shader_ticks (the number of poweroff timer ticks before an idle + * shader is powered off), and poweroff_gpu_ticks (the number of poweroff timer + * ticks before the GPU is powered off), in that order. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_pm_poweroff(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int items; + s64 gpu_poweroff_time; + int poweroff_shader_ticks, poweroff_gpu_ticks; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + items = sscanf(buf, "%llu %u %u", &gpu_poweroff_time, + &poweroff_shader_ticks, + &poweroff_gpu_ticks); + if (items != 3) { + dev_err(kbdev->dev, "Couldn't process pm_poweroff write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->pm.gpu_poweroff_time = HR_TIMER_DELAY_NSEC(gpu_poweroff_time); + kbdev->pm.poweroff_shader_ticks = poweroff_shader_ticks; + kbdev->pm.poweroff_gpu_ticks = poweroff_gpu_ticks; + + return count; +} + +/** + * show_pm_poweroff - Show callback for the pm_poweroff sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current period used for the DVFS sample + * timer. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_pm_poweroff(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%llu %u %u\n", + ktime_to_ns(kbdev->pm.gpu_poweroff_time), + kbdev->pm.poweroff_shader_ticks, + kbdev->pm.poweroff_gpu_ticks); + + return ret; +} + +static DEVICE_ATTR(pm_poweroff, S_IRUGO | S_IWUSR, show_pm_poweroff, + set_pm_poweroff); + +/** + * set_reset_timeout - Store callback for the reset_timeout sysfs file. + * @dev: The device with sysfs file is for + * @attr: The attributes of the sysfs file + * @buf: The value written to the sysfs file + * @count: The number of bytes written to the sysfs file + * + * This function is called when the reset_timeout sysfs file is written to. It + * checks the data written, and if valid updates the reset timeout. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t set_reset_timeout(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + int reset_timeout; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtoint(buf, 0, &reset_timeout); + if (ret || reset_timeout <= 0) { + dev_err(kbdev->dev, "Couldn't process reset_timeout write operation.\n" + "Use format \n"); + return -EINVAL; + } + + kbdev->reset_timeout_ms = reset_timeout; + dev_dbg(kbdev->dev, "Reset timeout: %dms\n", reset_timeout); + + return count; +} + +/** + * show_reset_timeout - Show callback for the reset_timeout sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU information. + * + * This function is called to get the current reset timeout. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t show_reset_timeout(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->reset_timeout_ms); + + return ret; +} + +static DEVICE_ATTR(reset_timeout, S_IRUGO | S_IWUSR, show_reset_timeout, + set_reset_timeout); + + + +static ssize_t show_mem_pool_size(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%zu\n", + kbase_mem_pool_size(&kbdev->mem_pool)); + + return ret; +} + +static ssize_t set_mem_pool_size(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + size_t new_size; + int err; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + err = kstrtoul(buf, 0, (unsigned long *)&new_size); + if (err) + return err; + + kbase_mem_pool_trim(&kbdev->mem_pool, new_size); + + return count; +} + +static DEVICE_ATTR(mem_pool_size, S_IRUGO | S_IWUSR, show_mem_pool_size, + set_mem_pool_size); + +static ssize_t show_mem_pool_max_size(struct device *dev, + struct device_attribute *attr, char * const buf) +{ + struct kbase_device *kbdev; + ssize_t ret; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = scnprintf(buf, PAGE_SIZE, "%zu\n", + kbase_mem_pool_max_size(&kbdev->mem_pool)); + + return ret; +} + +static ssize_t set_mem_pool_max_size(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct kbase_device *kbdev; + size_t new_max_size; + int err; + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + err = kstrtoul(buf, 0, (unsigned long *)&new_max_size); + if (err) + return -EINVAL; + + kbase_mem_pool_set_max_size(&kbdev->mem_pool, new_max_size); + + return count; +} + +static DEVICE_ATTR(mem_pool_max_size, S_IRUGO | S_IWUSR, show_mem_pool_max_size, + set_mem_pool_max_size); + +#ifdef CONFIG_DEBUG_FS + +/* Number of entries in serialize_jobs_settings[] */ +#define NR_SERIALIZE_JOBS_SETTINGS 5 +/* Maximum string length in serialize_jobs_settings[].name */ +#define MAX_SERIALIZE_JOBS_NAME_LEN 16 + +static struct +{ + char *name; + u8 setting; +} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { + {"none", 0}, + {"intra-slot", KBASE_SERIALIZE_INTRA_SLOT}, + {"inter-slot", KBASE_SERIALIZE_INTER_SLOT}, + {"full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT}, + {"full-reset", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | + KBASE_SERIALIZE_RESET} +}; + +/** + * kbasep_serialize_jobs_seq_show - Show callback for the serialize_jobs debugfs + * file + * @sfile: seq_file pointer + * @data: Private callback data + * + * This function is called to get the contents of the serialize_jobs debugfs + * file. This is a list of the available settings with the currently active one + * surrounded by square brackets. + * + * Return: 0 on success, or an error code on error + */ +static int kbasep_serialize_jobs_seq_show(struct seq_file *sfile, void *data) +{ + struct kbase_device *kbdev = sfile->private; + int i; + + CSTD_UNUSED(data); + + for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { + if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) + seq_printf(sfile, "[%s] ", + serialize_jobs_settings[i].name); + else + seq_printf(sfile, "%s ", + serialize_jobs_settings[i].name); + } + + seq_puts(sfile, "\n"); + + return 0; +} + +/** + * kbasep_serialize_jobs_debugfs_write - Store callback for the serialize_jobs + * debugfs file. + * @file: File pointer + * @ubuf: User buffer containing data to store + * @count: Number of bytes in user buffer + * @ppos: File position + * + * This function is called when the serialize_jobs debugfs file is written to. + * It matches the requested setting against the available settings and if a + * matching setting is found updates kbdev->serialize_jobs. + * + * Return: @count if the function succeeded. An error code on failure. + */ +static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct kbase_device *kbdev = s->private; + char buf[MAX_SERIALIZE_JOBS_NAME_LEN]; + int i; + bool valid = false; + + CSTD_UNUSED(ppos); + + count = min_t(size_t, sizeof(buf) - 1, count); + if (copy_from_user(buf, ubuf, count)) + return -EFAULT; + + buf[count] = 0; + + for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { + if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { + kbdev->serialize_jobs = + serialize_jobs_settings[i].setting; + valid = true; + break; + } + } + + if (!valid) { + dev_err(kbdev->dev, "serialize_jobs: invalid setting\n"); + return -EINVAL; + } + + return count; +} + +/** + * kbasep_serialize_jobs_debugfs_open - Open callback for the serialize_jobs + * debugfs file + * @in: inode pointer + * @file: file pointer + * + * Return: Zero on success, error code on failure + */ +static int kbasep_serialize_jobs_debugfs_open(struct inode *in, + struct file *file) +{ + return single_open(file, kbasep_serialize_jobs_seq_show, in->i_private); +} + +static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { + .open = kbasep_serialize_jobs_debugfs_open, + .read = seq_read, + .write = kbasep_serialize_jobs_debugfs_write, + .llseek = seq_lseek, + .release = single_release, +}; + +#endif /* CONFIG_DEBUG_FS */ + +static int kbasep_protected_mode_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_OF + struct device_node *protected_node; + struct platform_device *pdev; + struct protected_mode_device *protected_dev; +#endif + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { + /* Use native protected ops */ + kbdev->protected_dev = kzalloc(sizeof(*kbdev->protected_dev), + GFP_KERNEL); + if (!kbdev->protected_dev) + return -ENOMEM; + kbdev->protected_dev->data = kbdev; + kbdev->protected_ops = &kbase_native_protected_ops; + kbdev->protected_mode_support = true; + return 0; + } + + kbdev->protected_mode_support = false; + +#ifdef CONFIG_OF + protected_node = of_parse_phandle(kbdev->dev->of_node, + "protected-mode-switcher", 0); + + if (!protected_node) + protected_node = of_parse_phandle(kbdev->dev->of_node, + "secure-mode-switcher", 0); + + if (!protected_node) { + /* If protected_node cannot be looked up then we assume + * protected mode is not supported on this platform. */ + dev_info(kbdev->dev, "Protected mode not available\n"); + return 0; + } + + pdev = of_find_device_by_node(protected_node); + if (!pdev) + return -EINVAL; + + protected_dev = platform_get_drvdata(pdev); + if (!protected_dev) + return -EPROBE_DEFER; + + kbdev->protected_ops = &protected_dev->ops; + kbdev->protected_dev = protected_dev; + + if (kbdev->protected_ops) { + int err; + + /* Make sure protected mode is disabled on startup */ + mutex_lock(&kbdev->pm.lock); + err = kbdev->protected_ops->protected_mode_disable( + kbdev->protected_dev); + mutex_unlock(&kbdev->pm.lock); + + /* protected_mode_disable() returns -EINVAL if not supported */ + kbdev->protected_mode_support = (err != -EINVAL); + } +#endif + return 0; +} + +static void kbasep_protected_mode_term(struct kbase_device *kbdev) +{ + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) + kfree(kbdev->protected_dev); +} + +#ifdef CONFIG_MALI_NO_MALI +static int kbase_common_reg_map(struct kbase_device *kbdev) +{ + return 0; +} +static void kbase_common_reg_unmap(struct kbase_device * const kbdev) +{ +} +#else /* CONFIG_MALI_NO_MALI */ +static int kbase_common_reg_map(struct kbase_device *kbdev) +{ + int err = -ENOMEM; + + if (!request_mem_region(kbdev->reg_start, kbdev->reg_size, dev_name(kbdev->dev))) { + dev_err(kbdev->dev, "Register window unavailable\n"); + err = -EIO; + goto out_region; + } + + kbdev->reg = ioremap(kbdev->reg_start, kbdev->reg_size); + if (!kbdev->reg) { + dev_err(kbdev->dev, "Can't remap register window\n"); + err = -EINVAL; + goto out_ioremap; + } + + return 0; + + out_ioremap: + release_mem_region(kbdev->reg_start, kbdev->reg_size); + out_region: + return err; +} + +static void kbase_common_reg_unmap(struct kbase_device * const kbdev) +{ + if (kbdev->reg) { + iounmap(kbdev->reg); + release_mem_region(kbdev->reg_start, kbdev->reg_size); + kbdev->reg = NULL; + kbdev->reg_start = 0; + kbdev->reg_size = 0; + } +} +#endif /* CONFIG_MALI_NO_MALI */ + +static int registers_map(struct kbase_device * const kbdev) +{ + + /* the first memory resource is the physical address of the GPU + * registers */ + struct platform_device *pdev = to_platform_device(kbdev->dev); + struct resource *reg_res; + int err; + + reg_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!reg_res) { + dev_err(kbdev->dev, "Invalid register resource\n"); + return -ENOENT; + } + + kbdev->reg_start = reg_res->start; + kbdev->reg_size = resource_size(reg_res); + + err = kbase_common_reg_map(kbdev); + if (err) { + dev_err(kbdev->dev, "Failed to map registers\n"); + return err; + } + + return 0; +} + +static void registers_unmap(struct kbase_device *kbdev) +{ + kbase_common_reg_unmap(kbdev); +} + +static int power_control_init(struct platform_device *pdev) +{ + struct kbase_device *kbdev = to_kbase_device(&pdev->dev); + int err = 0; + + if (!kbdev) + return -ENODEV; + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ + && defined(CONFIG_REGULATOR) + kbdev->regulator = regulator_get_optional(kbdev->dev, "mali"); + if (IS_ERR_OR_NULL(kbdev->regulator)) { + err = PTR_ERR(kbdev->regulator); + kbdev->regulator = NULL; + if (err == -EPROBE_DEFER) { + dev_err(&pdev->dev, "Failed to get regulator\n"); + return err; + } + dev_info(kbdev->dev, + "Continuing without Mali regulator control\n"); + /* Allow probe to continue without regulator */ + } +#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ + + kbdev->clock = clk_get(kbdev->dev, "clk_mali"); + if (IS_ERR_OR_NULL(kbdev->clock)) { + err = PTR_ERR(kbdev->clock); + kbdev->clock = NULL; + if (err == -EPROBE_DEFER) { + dev_err(&pdev->dev, "Failed to get clock\n"); + goto fail; + } + dev_info(kbdev->dev, "Continuing without Mali clock control\n"); + /* Allow probe to continue without clock. */ + } else { + err = clk_prepare(kbdev->clock); + if (err) { + dev_err(kbdev->dev, + "Failed to prepare and enable clock (%d)\n", + err); + goto fail; + } + } + + err = kbase_platform_rk_init_opp_table(kbdev); + if (err) + dev_err(kbdev->dev, "Failed to init_opp_table (%d)\n", err); + + return 0; + +fail: + +if (kbdev->clock != NULL) { + clk_put(kbdev->clock); + kbdev->clock = NULL; +} + +#ifdef CONFIG_REGULATOR + if (NULL != kbdev->regulator) { + regulator_put(kbdev->regulator); + kbdev->regulator = NULL; + } +#endif + + return err; +} + +static void power_control_term(struct kbase_device *kbdev) +{ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) || \ + defined(LSK_OPPV2_BACKPORT) + dev_pm_opp_of_remove_table(kbdev->dev); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) + of_free_opp_table(kbdev->dev); +#endif + + if (kbdev->clock) { + clk_unprepare(kbdev->clock); + clk_put(kbdev->clock); + kbdev->clock = NULL; + } + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ + && defined(CONFIG_REGULATOR) + if (kbdev->regulator) { + regulator_put(kbdev->regulator); + kbdev->regulator = NULL; + } +#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ +} + +#ifdef CONFIG_DEBUG_FS + +#if KBASE_GPU_RESET_EN +#include + +static void trigger_quirks_reload(struct kbase_device *kbdev) +{ + kbase_pm_context_active(kbdev); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + kbase_pm_context_idle(kbdev); +} + +#define MAKE_QUIRK_ACCESSORS(type) \ +static int type##_quirks_set(void *data, u64 val) \ +{ \ + struct kbase_device *kbdev; \ + kbdev = (struct kbase_device *)data; \ + kbdev->hw_quirks_##type = (u32)val; \ + trigger_quirks_reload(kbdev); \ + return 0;\ +} \ +\ +static int type##_quirks_get(void *data, u64 *val) \ +{ \ + struct kbase_device *kbdev;\ + kbdev = (struct kbase_device *)data;\ + *val = kbdev->hw_quirks_##type;\ + return 0;\ +} \ +DEFINE_SIMPLE_ATTRIBUTE(fops_##type##_quirks, type##_quirks_get,\ + type##_quirks_set, "%llu\n") + +MAKE_QUIRK_ACCESSORS(sc); +MAKE_QUIRK_ACCESSORS(tiler); +MAKE_QUIRK_ACCESSORS(mmu); +MAKE_QUIRK_ACCESSORS(jm); + +#endif /* KBASE_GPU_RESET_EN */ + +/** + * debugfs_protected_debug_mode_read - "protected_debug_mode" debugfs read + * @file: File object to read is for + * @buf: User buffer to populate with data + * @len: Length of user buffer + * @ppos: Offset within file object + * + * Retrieves the current status of protected debug mode + * (0 = disabled, 1 = enabled) + * + * Return: Number of bytes added to user buffer + */ +static ssize_t debugfs_protected_debug_mode_read(struct file *file, + char __user *buf, size_t len, loff_t *ppos) +{ + struct kbase_device *kbdev = (struct kbase_device *)file->private_data; + u32 gpu_status; + ssize_t ret_val; + + kbase_pm_context_active(kbdev); + gpu_status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL); + kbase_pm_context_idle(kbdev); + + if (gpu_status & GPU_DBGEN) + ret_val = simple_read_from_buffer(buf, len, ppos, "1\n", 2); + else + ret_val = simple_read_from_buffer(buf, len, ppos, "0\n", 2); + + return ret_val; +} + +/* + * struct fops_protected_debug_mode - "protected_debug_mode" debugfs fops + * + * Contains the file operations for the "protected_debug_mode" debugfs file + */ +static const struct file_operations fops_protected_debug_mode = { + .open = simple_open, + .read = debugfs_protected_debug_mode_read, + .llseek = default_llseek, +}; + +static int kbase_device_debugfs_init(struct kbase_device *kbdev) +{ + struct dentry *debugfs_ctx_defaults_directory; + int err; + + kbdev->mali_debugfs_directory = debugfs_create_dir(kbdev->devname, + NULL); + if (!kbdev->mali_debugfs_directory) { + dev_err(kbdev->dev, "Couldn't create mali debugfs directory\n"); + err = -ENOMEM; + goto out; + } + + kbdev->debugfs_ctx_directory = debugfs_create_dir("ctx", + kbdev->mali_debugfs_directory); + if (!kbdev->debugfs_ctx_directory) { + dev_err(kbdev->dev, "Couldn't create mali debugfs ctx directory\n"); + err = -ENOMEM; + goto out; + } + + debugfs_ctx_defaults_directory = debugfs_create_dir("defaults", + kbdev->debugfs_ctx_directory); + if (!debugfs_ctx_defaults_directory) { + dev_err(kbdev->dev, "Couldn't create mali debugfs ctx defaults directory\n"); + err = -ENOMEM; + goto out; + } + +#if !MALI_CUSTOMER_RELEASE + kbasep_regs_dump_debugfs_init(kbdev); +#endif /* !MALI_CUSTOMER_RELEASE */ + kbasep_regs_history_debugfs_init(kbdev); + + kbase_debug_job_fault_debugfs_init(kbdev); + kbasep_gpu_memory_debugfs_init(kbdev); + kbase_as_fault_debugfs_init(kbdev); +#if KBASE_GPU_RESET_EN + /* fops_* variables created by invocations of macro + * MAKE_QUIRK_ACCESSORS() above. */ + debugfs_create_file("quirks_sc", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_sc_quirks); + debugfs_create_file("quirks_tiler", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_tiler_quirks); + debugfs_create_file("quirks_mmu", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_mmu_quirks); + debugfs_create_file("quirks_jm", 0644, + kbdev->mali_debugfs_directory, kbdev, + &fops_jm_quirks); +#endif /* KBASE_GPU_RESET_EN */ + +#ifndef CONFIG_MALI_COH_USER + debugfs_create_bool("infinite_cache", 0644, + debugfs_ctx_defaults_directory, + (bool*)&(kbdev->infinite_cache_active_default)); +#endif /* CONFIG_MALI_COH_USER */ + + debugfs_create_size_t("mem_pool_max_size", 0644, + debugfs_ctx_defaults_directory, + &kbdev->mem_pool_max_size_default); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { + debugfs_create_file("protected_debug_mode", S_IRUGO, + kbdev->mali_debugfs_directory, kbdev, + &fops_protected_debug_mode); + } + +#if KBASE_TRACE_ENABLE + kbasep_trace_debugfs_init(kbdev); +#endif /* KBASE_TRACE_ENABLE */ + +#ifdef CONFIG_MALI_TRACE_TIMELINE + kbasep_trace_timeline_debugfs_init(kbdev); +#endif /* CONFIG_MALI_TRACE_TIMELINE */ + +#ifdef CONFIG_MALI_DEVFREQ +#ifdef CONFIG_DEVFREQ_THERMAL + if (kbdev->inited_subsys & inited_devfreq) + kbase_ipa_debugfs_init(kbdev); +#endif /* CONFIG_DEVFREQ_THERMAL */ +#endif /* CONFIG_MALI_DEVFREQ */ + +#ifdef CONFIG_DEBUG_FS + debugfs_create_file("serialize_jobs", S_IRUGO | S_IWUSR, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_serialize_jobs_debugfs_fops); +#endif /* CONFIG_DEBUG_FS */ + + return 0; + +out: + debugfs_remove_recursive(kbdev->mali_debugfs_directory); + return err; +} + +static void kbase_device_debugfs_term(struct kbase_device *kbdev) +{ + debugfs_remove_recursive(kbdev->mali_debugfs_directory); +} + +#else /* CONFIG_DEBUG_FS */ +static inline int kbase_device_debugfs_init(struct kbase_device *kbdev) +{ + return 0; +} + +static inline void kbase_device_debugfs_term(struct kbase_device *kbdev) { } +#endif /* CONFIG_DEBUG_FS */ + +static void kbase_device_coherency_init(struct kbase_device *kbdev, + unsigned prod_id) +{ +#ifdef CONFIG_OF + u32 supported_coherency_bitmap = + kbdev->gpu_props.props.raw_props.coherency_mode; + const void *coherency_override_dts; + u32 override_coherency; + + /* Only for tMIx : + * (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly + * documented for tMIx so force correct value here. + */ + if (GPU_ID_IS_NEW_FORMAT(prod_id) && + (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == + GPU_ID2_PRODUCT_TMIX)) + if (supported_coherency_bitmap == + COHERENCY_FEATURE_BIT(COHERENCY_ACE)) + supported_coherency_bitmap |= + COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); + +#endif /* CONFIG_OF */ + + kbdev->system_coherency = COHERENCY_NONE; + + /* device tree may override the coherency */ +#ifdef CONFIG_OF + coherency_override_dts = of_get_property(kbdev->dev->of_node, + "system-coherency", + NULL); + if (coherency_override_dts) { + + override_coherency = be32_to_cpup(coherency_override_dts); + + if ((override_coherency <= COHERENCY_NONE) && + (supported_coherency_bitmap & + COHERENCY_FEATURE_BIT(override_coherency))) { + + kbdev->system_coherency = override_coherency; + + dev_info(kbdev->dev, + "Using coherency mode %u set from dtb", + override_coherency); + } else + dev_warn(kbdev->dev, + "Ignoring unsupported coherency mode %u set from dtb", + override_coherency); + } + +#endif /* CONFIG_OF */ + + kbdev->gpu_props.props.raw_props.coherency_mode = + kbdev->system_coherency; +} + +#ifdef CONFIG_MALI_FPGA_BUS_LOGGER + +/* Callback used by the kbase bus logger client, to initiate a GPU reset + * when the bus log is restarted. GPU reset is used as reference point + * in HW bus log analyses. + */ +static void kbase_logging_started_cb(void *data) +{ + struct kbase_device *kbdev = (struct kbase_device *)data; + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + dev_info(kbdev->dev, "KBASE - Bus logger restarted\n"); +} +#endif + +static struct attribute *kbase_attrs[] = { +#ifdef CONFIG_MALI_DEBUG + &dev_attr_debug_command.attr, + &dev_attr_js_softstop_always.attr, +#endif +#if !MALI_CUSTOMER_RELEASE + &dev_attr_force_replay.attr, +#endif + &dev_attr_js_timeouts.attr, + &dev_attr_soft_job_timeout.attr, + &dev_attr_gpuinfo.attr, + &dev_attr_dvfs_period.attr, + &dev_attr_pm_poweroff.attr, + &dev_attr_reset_timeout.attr, + &dev_attr_js_scheduling_period.attr, + &dev_attr_power_policy.attr, + &dev_attr_core_availability_policy.attr, + &dev_attr_core_mask.attr, + &dev_attr_mem_pool_size.attr, + &dev_attr_mem_pool_max_size.attr, + NULL +}; + +static const struct attribute_group kbase_attr_group = { + .attrs = kbase_attrs, +}; + +static int kbase_platform_device_remove(struct platform_device *pdev) +{ + struct kbase_device *kbdev = to_kbase_device(&pdev->dev); + const struct list_head *dev_list; + + if (!kbdev) + return -ENODEV; + + kfree(kbdev->gpu_props.prop_buffer); + +#ifdef CONFIG_MALI_FPGA_BUS_LOGGER + if (kbdev->inited_subsys & inited_buslogger) { + bl_core_client_unregister(kbdev->buslogger); + kbdev->inited_subsys &= ~inited_buslogger; + } +#endif + + + if (kbdev->inited_subsys & inited_dev_list) { + dev_list = kbase_dev_list_get(); + list_del(&kbdev->entry); + kbase_dev_list_put(dev_list); + kbdev->inited_subsys &= ~inited_dev_list; + } + + if (kbdev->inited_subsys & inited_misc_register) { + misc_deregister(&kbdev->mdev); + kbdev->inited_subsys &= ~inited_misc_register; + } + + if (kbdev->inited_subsys & inited_sysfs_group) { + sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); + kbdev->inited_subsys &= ~inited_sysfs_group; + } + + if (kbdev->inited_subsys & inited_get_device) { + put_device(kbdev->dev); + kbdev->inited_subsys &= ~inited_get_device; + } + + if (kbdev->inited_subsys & inited_debugfs) { + kbase_device_debugfs_term(kbdev); + kbdev->inited_subsys &= ~inited_debugfs; + } + + if (kbdev->inited_subsys & inited_job_fault) { + kbase_debug_job_fault_dev_term(kbdev); + kbdev->inited_subsys &= ~inited_job_fault; + } + if (kbdev->inited_subsys & inited_vinstr) { + kbase_vinstr_term(kbdev->vinstr_ctx); + kbdev->inited_subsys &= ~inited_vinstr; + } + +#ifdef CONFIG_MALI_DEVFREQ + if (kbdev->inited_subsys & inited_devfreq) { + kbase_devfreq_term(kbdev); + kbdev->inited_subsys &= ~inited_devfreq; + } +#endif + + if (kbdev->inited_subsys & inited_backend_late) { + kbase_backend_late_term(kbdev); + kbdev->inited_subsys &= ~inited_backend_late; + } + + if (kbdev->inited_subsys & inited_tlstream) { + kbase_tlstream_term(); + kbdev->inited_subsys &= ~inited_tlstream; + } + + /* Bring job and mem sys to a halt before we continue termination */ + + if (kbdev->inited_subsys & inited_js) + kbasep_js_devdata_halt(kbdev); + + if (kbdev->inited_subsys & inited_mem) + kbase_mem_halt(kbdev); + + if (kbdev->inited_subsys & inited_protected) { + kbasep_protected_mode_term(kbdev); + kbdev->inited_subsys &= ~inited_protected; + } + + if (kbdev->inited_subsys & inited_js) { + kbasep_js_devdata_term(kbdev); + kbdev->inited_subsys &= ~inited_js; + } + + if (kbdev->inited_subsys & inited_mem) { + kbase_mem_term(kbdev); + kbdev->inited_subsys &= ~inited_mem; + } + + if (kbdev->inited_subsys & inited_pm_runtime_init) { + kbdev->pm.callback_power_runtime_term(kbdev); + kbdev->inited_subsys &= ~inited_pm_runtime_init; + } + + if (kbdev->inited_subsys & inited_ctx_sched) { + kbase_ctx_sched_term(kbdev); + kbdev->inited_subsys &= ~inited_ctx_sched; + } + + if (kbdev->inited_subsys & inited_device) { + kbase_device_term(kbdev); + kbdev->inited_subsys &= ~inited_device; + } + + if (kbdev->inited_subsys & inited_backend_early) { + kbase_backend_early_term(kbdev); + kbdev->inited_subsys &= ~inited_backend_early; + } + + if (kbdev->inited_subsys & inited_io_history) { + kbase_io_history_term(&kbdev->io_history); + kbdev->inited_subsys &= ~inited_io_history; + } + + if (kbdev->inited_subsys & inited_power_control) { + power_control_term(kbdev); + kbdev->inited_subsys &= ~inited_power_control; + } + + if (kbdev->inited_subsys & inited_registers_map) { + registers_unmap(kbdev); + kbdev->inited_subsys &= ~inited_registers_map; + } + +#ifdef CONFIG_MALI_NO_MALI + if (kbdev->inited_subsys & inited_gpu_device) { + gpu_device_destroy(kbdev); + kbdev->inited_subsys &= ~inited_gpu_device; + } +#endif /* CONFIG_MALI_NO_MALI */ + + if (kbdev->inited_subsys != 0) + dev_err(kbdev->dev, "Missing sub system termination\n"); + + kbase_device_free(kbdev); + + return 0; +} + +extern void kbase_platform_rk_shutdown(struct kbase_device *kbdev); +static void kbase_platform_device_shutdown(struct platform_device *pdev) +{ + struct kbase_device *kbdev = to_kbase_device(&pdev->dev); + + kbase_platform_rk_shutdown(kbdev); +} + +/* Number of register accesses for the buffer that we allocate during + * initialization time. The buffer size can be changed later via debugfs. */ +#define KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ((u16)512) + +static int kbase_platform_device_probe(struct platform_device *pdev) +{ + struct kbase_device *kbdev; + struct mali_base_gpu_core_props *core_props; + u32 gpu_id; + unsigned prod_id; + const struct list_head *dev_list; + int err = 0; + +#ifdef CONFIG_OF + err = kbase_platform_early_init(); + if (err) { + dev_err(&pdev->dev, "Early platform initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } +#endif + kbdev = kbase_device_alloc(); + if (!kbdev) { + dev_err(&pdev->dev, "Allocate device failed\n"); + kbase_platform_device_remove(pdev); + return -ENOMEM; + } + + kbdev->dev = &pdev->dev; + dev_set_drvdata(kbdev->dev, kbdev); + +#ifdef CONFIG_MALI_NO_MALI + err = gpu_device_create(kbdev); + if (err) { + dev_err(&pdev->dev, "Dummy model initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_gpu_device; +#endif /* CONFIG_MALI_NO_MALI */ + + err = assign_irqs(pdev); + if (err) { + dev_err(&pdev->dev, "IRQ search failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + + err = registers_map(kbdev); + if (err) { + dev_err(&pdev->dev, "Register map failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_registers_map; + + err = power_control_init(pdev); + if (err) { + dev_err(&pdev->dev, "Power control initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_power_control; + + err = kbase_io_history_init(&kbdev->io_history, + KBASEP_DEFAULT_REGISTER_HISTORY_SIZE); + if (err) { + dev_err(&pdev->dev, "Register access history initialization failed\n"); + kbase_platform_device_remove(pdev); + return -ENOMEM; + } + kbdev->inited_subsys |= inited_io_history; + + err = kbase_backend_early_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Early backend initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_backend_early; + + scnprintf(kbdev->devname, DEVNAME_SIZE, "%s%d", kbase_drv_name, + kbase_dev_nr); + + kbase_disjoint_init(kbdev); + + /* obtain min/max configured gpu frequencies */ + core_props = &(kbdev->gpu_props.props.core_props); + core_props->gpu_freq_khz_min = GPU_FREQ_KHZ_MIN; + core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX; + + err = kbase_device_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Device initialization failed (%d)\n", err); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_device; + + err = kbase_ctx_sched_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Context scheduler initialization failed (%d)\n", + err); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_ctx_sched; + + if (kbdev->pm.callback_power_runtime_init) { + err = kbdev->pm.callback_power_runtime_init(kbdev); + if (err) { + dev_err(kbdev->dev, + "Runtime PM initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_pm_runtime_init; + } + + err = kbase_mem_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Memory subsystem initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_mem; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + gpu_id &= GPU_ID_VERSION_PRODUCT_ID; + prod_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + kbase_device_coherency_init(kbdev, prod_id); + + err = kbasep_protected_mode_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Protected mode subsystem initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_protected; + + dev_list = kbase_dev_list_get(); + list_add(&kbdev->entry, &kbase_dev_list); + kbase_dev_list_put(dev_list); + kbdev->inited_subsys |= inited_dev_list; + + err = kbasep_js_devdata_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Job JS devdata initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_js; + + err = kbase_tlstream_init(); + if (err) { + dev_err(kbdev->dev, "Timeline stream initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_tlstream; + + err = kbase_backend_late_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Late backend initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_backend_late; + +#ifdef CONFIG_MALI_DEVFREQ + err = kbase_devfreq_init(kbdev); + if (!err) + kbdev->inited_subsys |= inited_devfreq; + else + dev_err(kbdev->dev, "Continuing without devfreq\n"); +#endif /* CONFIG_MALI_DEVFREQ */ + + kbdev->vinstr_ctx = kbase_vinstr_init(kbdev); + if (!kbdev->vinstr_ctx) { + dev_err(kbdev->dev, + "Virtual instrumentation initialization failed\n"); + kbase_platform_device_remove(pdev); + return -EINVAL; + } + kbdev->inited_subsys |= inited_vinstr; + + err = kbase_debug_job_fault_dev_init(kbdev); + if (err) { + dev_err(kbdev->dev, "Job fault debug initialization failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_job_fault; + + err = kbase_device_debugfs_init(kbdev); + if (err) { + dev_err(kbdev->dev, "DebugFS initialization failed"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_debugfs; + + /* initialize the kctx list */ + mutex_init(&kbdev->kctx_list_lock); + INIT_LIST_HEAD(&kbdev->kctx_list); + + kbdev->mdev.minor = MISC_DYNAMIC_MINOR; + kbdev->mdev.name = kbdev->devname; + kbdev->mdev.fops = &kbase_fops; + kbdev->mdev.parent = get_device(kbdev->dev); + kbdev->inited_subsys |= inited_get_device; + + /* This needs to happen before registering the device with misc_register(), + * otherwise it causes a race condition between registering the device and a + * uevent event being generated for userspace, causing udev rules to run + * which might expect certain sysfs attributes present. As a result of the + * race condition we avoid, some Mali sysfs entries may have appeared to + * udev to not exist. + + * For more information, see + * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the + * paragraph that starts with "Word of warning", currently the second-last + * paragraph. + */ + err = sysfs_create_group(&kbdev->dev->kobj, &kbase_attr_group); + if (err) { + dev_err(&pdev->dev, "SysFS group creation failed\n"); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_sysfs_group; + + err = misc_register(&kbdev->mdev); + if (err) { + dev_err(kbdev->dev, "Misc device registration failed for %s\n", + kbdev->devname); + kbase_platform_device_remove(pdev); + return err; + } + kbdev->inited_subsys |= inited_misc_register; + + +#ifdef CONFIG_MALI_FPGA_BUS_LOGGER + err = bl_core_client_register(kbdev->devname, + kbase_logging_started_cb, + kbdev, &kbdev->buslogger, + THIS_MODULE, NULL); + if (err == 0) { + kbdev->inited_subsys |= inited_buslogger; + bl_core_set_threshold(kbdev->buslogger, 1024*1024*1024); + } else { + dev_warn(kbdev->dev, "Bus log client registration failed\n"); + err = 0; + } +#endif + + err = kbase_gpuprops_populate_user_buffer(kbdev); + if (err) { + dev_err(&pdev->dev, "GPU property population failed"); + kbase_platform_device_remove(pdev); + return err; + } + + dev_info(kbdev->dev, + "Probed as %s\n", dev_name(kbdev->mdev.this_device)); + + kbase_dev_nr++; + + return err; +} + +#undef KBASEP_DEFAULT_REGISTER_HISTORY_SIZE + +/** + * kbase_device_suspend - Suspend callback from the OS. + * + * This is called by Linux when the device should suspend. + * + * @dev: The device to suspend + * + * Return: A standard Linux error code + */ +static int kbase_device_suspend(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + +#if defined(CONFIG_MALI_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + if (kbdev->inited_subsys & inited_devfreq) + devfreq_suspend_device(kbdev->devfreq); +#endif + + kbase_pm_suspend(kbdev); + return 0; +} + +/** + * kbase_device_resume - Resume callback from the OS. + * + * This is called by Linux when the device should resume from suspension. + * + * @dev: The device to resume + * + * Return: A standard Linux error code + */ +static int kbase_device_resume(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + kbase_pm_resume(kbdev); + +#if defined(CONFIG_MALI_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + if (kbdev->inited_subsys & inited_devfreq) + devfreq_resume_device(kbdev->devfreq); +#endif + return 0; +} + +/** + * kbase_device_runtime_suspend - Runtime suspend callback from the OS. + * + * This is called by Linux when the device should prepare for a condition in + * which it will not be able to communicate with the CPU(s) and RAM due to + * power management. + * + * @dev: The device to suspend + * + * Return: A standard Linux error code + */ +#ifdef KBASE_PM_RUNTIME +static int kbase_device_runtime_suspend(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + +#if defined(CONFIG_MALI_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + if (kbdev->inited_subsys & inited_devfreq) + devfreq_suspend_device(kbdev->devfreq); +#endif + + if (kbdev->pm.backend.callback_power_runtime_off) { + kbdev->pm.backend.callback_power_runtime_off(kbdev); + dev_dbg(dev, "runtime suspend\n"); + } + return 0; +} +#endif /* KBASE_PM_RUNTIME */ + +/** + * kbase_device_runtime_resume - Runtime resume callback from the OS. + * + * This is called by Linux when the device should go into a fully active state. + * + * @dev: The device to suspend + * + * Return: A standard Linux error code + */ + +#ifdef KBASE_PM_RUNTIME +static int kbase_device_runtime_resume(struct device *dev) +{ + int ret = 0; + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + if (kbdev->pm.backend.callback_power_runtime_on) { + ret = kbdev->pm.backend.callback_power_runtime_on(kbdev); + dev_dbg(dev, "runtime resume\n"); + } + +#if defined(CONFIG_MALI_DEVFREQ) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) + if (kbdev->inited_subsys & inited_devfreq) + devfreq_resume_device(kbdev->devfreq); +#endif + + return ret; +} +#endif /* KBASE_PM_RUNTIME */ + + +#ifdef KBASE_PM_RUNTIME +/** + * kbase_device_runtime_idle - Runtime idle callback from the OS. + * @dev: The device to suspend + * + * This is called by Linux when the device appears to be inactive and it might + * be placed into a low power state. + * + * Return: 0 if device can be suspended, non-zero to avoid runtime autosuspend, + * otherwise a standard Linux error code + */ +static int kbase_device_runtime_idle(struct device *dev) +{ + struct kbase_device *kbdev = to_kbase_device(dev); + + if (!kbdev) + return -ENODEV; + + /* Use platform specific implementation if it exists. */ + if (kbdev->pm.backend.callback_power_runtime_idle) + return kbdev->pm.backend.callback_power_runtime_idle(kbdev); + + return 0; +} +#endif /* KBASE_PM_RUNTIME */ + +/* The power management operations for the platform driver. + */ +static const struct dev_pm_ops kbase_pm_ops = { + .suspend = kbase_device_suspend, + .resume = kbase_device_resume, +#ifdef KBASE_PM_RUNTIME + .runtime_suspend = kbase_device_runtime_suspend, + .runtime_resume = kbase_device_runtime_resume, + .runtime_idle = kbase_device_runtime_idle, +#endif /* KBASE_PM_RUNTIME */ +}; + +#ifdef CONFIG_OF +static const struct of_device_id kbase_dt_ids[] = { + { .compatible = "arm,malit7xx" }, + { .compatible = "arm,mali-midgard" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, kbase_dt_ids); +#endif + +static struct platform_driver kbase_platform_driver = { + .probe = kbase_platform_device_probe, + .remove = kbase_platform_device_remove, + .shutdown = kbase_platform_device_shutdown, + .driver = { + .name = "midgard", + .owner = THIS_MODULE, + .pm = &kbase_pm_ops, + .of_match_table = of_match_ptr(kbase_dt_ids), + }, +}; + +/* + * The driver will not provide a shortcut to create the Mali platform device + * anymore when using Device Tree. + */ +#ifdef CONFIG_OF +module_platform_driver(kbase_platform_driver); +#else + +static int __init rockchip_gpu_init_driver(void) +{ + return platform_driver_register(&kbase_platform_driver); +} +late_initcall(rockchip_gpu_init_driver); + +static int __init kbase_driver_init(void) +{ + int ret; + + ret = kbase_platform_early_init(); + if (ret) + return ret; + +#ifdef CONFIG_MALI_PLATFORM_FAKE + ret = kbase_platform_fake_register(); + if (ret) + return ret; +#endif + ret = platform_driver_register(&kbase_platform_driver); +#ifdef CONFIG_MALI_PLATFORM_FAKE + if (ret) + kbase_platform_fake_unregister(); +#endif + return ret; +} + +static void __exit kbase_driver_exit(void) +{ + platform_driver_unregister(&kbase_platform_driver); +#ifdef CONFIG_MALI_PLATFORM_FAKE + kbase_platform_fake_unregister(); +#endif +} + +module_init(kbase_driver_init); +module_exit(kbase_driver_exit); + +#endif /* CONFIG_OF */ + +MODULE_LICENSE("GPL"); +MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \ + __stringify(BASE_UK_VERSION_MAJOR) "." \ + __stringify(BASE_UK_VERSION_MINOR) ")"); + +#if defined(CONFIG_MALI_GATOR_SUPPORT) || defined(CONFIG_MALI_SYSTEM_TRACE) +#define CREATE_TRACE_POINTS +#endif + +#ifdef CONFIG_MALI_GATOR_SUPPORT +/* Create the trace points (otherwise we just get code to call a tracepoint) */ +#include "mali_linux_trace.h" + +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_job_slots_event); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_status); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_on); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_off); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_page_fault_insert_pages); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_in_use); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_released); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_total_alloc_pages_change); + +void kbase_trace_mali_pm_status(u32 event, u64 value) +{ + trace_mali_pm_status(event, value); +} + +void kbase_trace_mali_pm_power_off(u32 event, u64 value) +{ + trace_mali_pm_power_off(event, value); +} + +void kbase_trace_mali_pm_power_on(u32 event, u64 value) +{ + trace_mali_pm_power_on(event, value); +} + +void kbase_trace_mali_job_slots_event(u32 event, const struct kbase_context *kctx, u8 atom_id) +{ + trace_mali_job_slots_event(event, (kctx != NULL ? kctx->tgid : 0), (kctx != NULL ? kctx->pid : 0), atom_id); +} + +void kbase_trace_mali_page_fault_insert_pages(int event, u32 value) +{ + trace_mali_page_fault_insert_pages(event, value); +} + +void kbase_trace_mali_mmu_as_in_use(int event) +{ + trace_mali_mmu_as_in_use(event); +} + +void kbase_trace_mali_mmu_as_released(int event) +{ + trace_mali_mmu_as_released(event); +} + +void kbase_trace_mali_total_alloc_pages_change(long long int event) +{ + trace_mali_total_alloc_pages_change(event); +} +#endif /* CONFIG_MALI_GATOR_SUPPORT */ +#ifdef CONFIG_MALI_SYSTEM_TRACE +#include "mali_linux_kbase_trace.h" +#endif diff --git a/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c new file mode 100755 index 000000000000..ce004841403f --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c @@ -0,0 +1,208 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include + +#include "mali_kbase_ctx_sched.h" + +int kbase_ctx_sched_init(struct kbase_device *kbdev) +{ + int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; + + /* These two must be recalculated if nr_hw_address_spaces changes + * (e.g. for HW workarounds) */ + kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) { + bool use_workaround; + + use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE; + if (use_workaround) { + dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only"); + kbdev->nr_user_address_spaces = 1; + } + } + + kbdev->as_free = as_present; /* All ASs initially free */ + + memset(kbdev->as_to_kctx, 0, sizeof(kbdev->as_to_kctx)); + + return 0; +} + +void kbase_ctx_sched_term(struct kbase_device *kbdev) +{ + s8 i; + + /* Sanity checks */ + for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { + WARN_ON(kbdev->as_to_kctx[i] != NULL); + WARN_ON(!(kbdev->as_free & (1u << i))); + } +} + +/* kbasep_ctx_sched_find_as_for_ctx - Find a free address space + * + * @kbdev: The context for which to find a free address space + * + * Return: A valid AS if successful, otherwise KBASEP_AS_NR_INVALID + * + * This function returns an address space available for use. It would prefer + * returning an AS that has been previously assigned to the context to + * avoid having to reprogram the MMU. + */ +static int kbasep_ctx_sched_find_as_for_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + int free_as; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /* First check if the previously assigned AS is available */ + if ((kctx->as_nr != KBASEP_AS_NR_INVALID) && + (kbdev->as_free & (1u << kctx->as_nr))) + return kctx->as_nr; + + /* The previously assigned AS was taken, we'll be returning any free + * AS at this point. + */ + free_as = ffs(kbdev->as_free) - 1; + if (free_as >= 0 && free_as < kbdev->nr_hw_address_spaces) + return free_as; + + return KBASEP_AS_NR_INVALID; +} + +int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(!kbdev->pm.backend.gpu_powered); + + if (atomic_inc_return(&kctx->refcount) == 1) { + int const free_as = kbasep_ctx_sched_find_as_for_ctx(kctx); + + if (free_as != KBASEP_AS_NR_INVALID) { + kbdev->as_free &= ~(1u << free_as); + /* Only program the MMU if the context has not been + * assigned the same address space before. + */ + if (free_as != kctx->as_nr) { + struct kbase_context *const prev_kctx = + kbdev->as_to_kctx[free_as]; + + if (prev_kctx) { + WARN_ON(atomic_read(&prev_kctx->refcount) != 0); + kbase_mmu_disable(prev_kctx); + prev_kctx->as_nr = KBASEP_AS_NR_INVALID; + } + + kctx->as_nr = free_as; + kbdev->as_to_kctx[free_as] = kctx; + kbase_mmu_update(kctx); + } + } else { + atomic_dec(&kctx->refcount); + + /* Failed to find an available address space, we must + * be returning an error at this point. + */ + WARN_ON(kctx->as_nr != KBASEP_AS_NR_INVALID); + } + } + + return kctx->as_nr; +} + +int kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->hwaccess_lock); + WARN_ON(atomic_read(&kctx->refcount) == 0); + if (atomic_read(&kctx->refcount) == 0) + return -1; + + WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID); + WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx); + + atomic_inc(&kctx->refcount); + + return 0; +} + +void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (atomic_dec_return(&kctx->refcount) == 0) + kbdev->as_free |= (1u << kctx->as_nr); +} + +void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(atomic_read(&kctx->refcount) != 0); + + if (kctx->as_nr != KBASEP_AS_NR_INVALID) { + if (kbdev->pm.backend.gpu_powered) + kbase_mmu_disable(kctx); + + kbdev->as_to_kctx[kctx->as_nr] = NULL; + kctx->as_nr = KBASEP_AS_NR_INVALID; + } +} + +void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) +{ + s8 i; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(!kbdev->pm.backend.gpu_powered); + + for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { + struct kbase_context *kctx; + + kctx = kbdev->as_to_kctx[i]; + if (kctx) { + if (atomic_read(&kctx->refcount)) { + WARN_ON(kctx->as_nr != i); + + kbase_mmu_update(kctx); + } else { + /* This context might have been assigned an + * AS before, clear it. + */ + kbdev->as_to_kctx[kctx->as_nr] = NULL; + kctx->as_nr = KBASEP_AS_NR_INVALID; + } + } else { + kbase_mmu_disable_as(kbdev, i); + } + } +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h new file mode 100755 index 000000000000..47474fecc2a9 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h @@ -0,0 +1,134 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_CTX_SCHED_H_ +#define _KBASE_CTX_SCHED_H_ + +#include + +/* The Context Scheduler manages address space assignment and reference + * counting to kbase_context. The interface has been designed to minimise + * interactions between the Job Scheduler and Power Management/MMU to support + * both the existing Job Scheduler and Command Stream Frontend interface. + * + * The initial implementation of the Context Scheduler does not schedule + * contexts. Instead it relies on the Job Scheduler/CSF to make decisions of + * when to schedule/evict contexts if address spaces are starved. In the + * future, once an interface between the CS and JS/CSF have been devised to + * provide enough information about how each context is consuming GPU resources, + * those decisions can be made in the CS itself, thereby reducing duplicated + * code. + */ + +/* base_ctx_sched_init - Initialise the context scheduler + * + * @kbdev: The device for which the context scheduler needs to be + * initialised + * + * Return: 0 for success, otherwise failure + * + * This must be called during device initilisation. The number of hardware + * address spaces must already be established before calling this function. + */ +int kbase_ctx_sched_init(struct kbase_device *kbdev); + +/* base_ctx_sched_term - Terminate the context scheduler + * + * @kbdev: The device for which the context scheduler needs to be + * terminated + * + * This must be called during device termination after all contexts have been + * destroyed. + */ +void kbase_ctx_sched_term(struct kbase_device *kbdev); + +/* kbase_ctx_sched_retain_ctx - Retain a reference to the @ref kbase_context + * + * @kctx: The context to which to retain a reference + * + * Return: The address space that the context has been assigned to or + * KBASEP_AS_NR_INVALID if no address space was available. + * + * This function should be called whenever an address space should be assigned + * to a context and programmed onto the MMU. It should typically be called + * when jobs are ready to be submitted to the GPU. + * + * It can be called as many times as necessary. The address space will be + * assigned to the context for as long as there is a reference to said context. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx); + +/* kbase_ctx_sched_retain_ctx_refcount + * + * @kctx: The context to which to retain a reference + * + * This function only retains a reference to the context. It must be called + * only when the context already has a reference. + * + * This is typically called inside an atomic session where we know the context + * is already scheduled in but want to take an extra reference to ensure that + * it doesn't get descheduled. + * + * The kbase_device::hwaccess_lock must be held whilst calling this function + * @return + * è‹¥æˆåŠŸ, 返回 0; + * è‹¥ *kctx 状æ€å¼‚常, 返回 -1. + */ +int kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx); + +/* kbase_ctx_sched_release_ctx - Release a reference to the @ref kbase_context + * + * @kctx: The context from which to release a reference + * + * This function should be called whenever an address space could be unassigned + * from a context. When there are no more references to said context, the + * address space previously assigned to this context shall be reassigned to + * other contexts as needed. + * + * The kbase_device::hwaccess_lock must be held whilst calling this function + */ +void kbase_ctx_sched_release_ctx(struct kbase_context *kctx); + +/* kbase_ctx_sched_remove_ctx - Unassign previously assigned address space + * + * @kctx: The context to be removed + * + * This function should be called when a context is being destroyed. The + * context must no longer have any reference. If it has been assigned an + * address space before then the AS will be unprogrammed. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); + +/* kbase_ctx_sched_restore_all_as - Reprogram all address spaces + * + * @kbdev: The device for which address spaces to be reprogrammed + * + * This function shall reprogram all address spaces previously assigned to + * contexts. It can be used after the GPU is reset. + * + * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be + * held whilst calling this function. + */ +void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev); + +#endif /* _KBASE_CTX_SCHED_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug.c b/drivers/gpu/arm/midgard/mali_kbase_debug.c new file mode 100755 index 000000000000..fb57ac2e31ad --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_debug.c @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include + +static struct kbasep_debug_assert_cb kbasep_debug_assert_registered_cb = { + NULL, + NULL +}; + +void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param) +{ + kbasep_debug_assert_registered_cb.func = func; + kbasep_debug_assert_registered_cb.param = param; +} + +void kbasep_debug_assert_call_hook(void) +{ + if (kbasep_debug_assert_registered_cb.func != NULL) + kbasep_debug_assert_registered_cb.func(kbasep_debug_assert_registered_cb.param); +} +KBASE_EXPORT_SYMBOL(kbasep_debug_assert_call_hook); + diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug.h b/drivers/gpu/arm/midgard/mali_kbase_debug.h new file mode 100755 index 000000000000..5fff2892bb55 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_debug.h @@ -0,0 +1,164 @@ +/* + * + * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_DEBUG_H +#define _KBASE_DEBUG_H + +#include + +/** @brief If equals to 0, a trace containing the file, line, and function will be displayed before each message. */ +#define KBASE_DEBUG_SKIP_TRACE 0 + +/** @brief If different from 0, the trace will only contain the file and line. */ +#define KBASE_DEBUG_SKIP_FUNCTION_NAME 0 + +/** @brief Disable the asserts tests if set to 1. Default is to disable the asserts in release. */ +#ifndef KBASE_DEBUG_DISABLE_ASSERTS +#ifdef CONFIG_MALI_DEBUG +#define KBASE_DEBUG_DISABLE_ASSERTS 0 +#else +#define KBASE_DEBUG_DISABLE_ASSERTS 1 +#endif +#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ + +/** Function type that is called on an KBASE_DEBUG_ASSERT() or KBASE_DEBUG_ASSERT_MSG() */ +typedef void (kbase_debug_assert_hook) (void *); + +struct kbasep_debug_assert_cb { + kbase_debug_assert_hook *func; + void *param; +}; + +/** + * @def KBASEP_DEBUG_PRINT_TRACE + * @brief Private macro containing the format of the trace to display before every message + * @sa KBASE_DEBUG_SKIP_TRACE, KBASE_DEBUG_SKIP_FUNCTION_NAME + */ +#if !KBASE_DEBUG_SKIP_TRACE +#define KBASEP_DEBUG_PRINT_TRACE \ + "In file: " __FILE__ " line: " CSTD_STR2(__LINE__) +#if !KBASE_DEBUG_SKIP_FUNCTION_NAME +#define KBASEP_DEBUG_PRINT_FUNCTION __func__ +#else +#define KBASEP_DEBUG_PRINT_FUNCTION "" +#endif +#else +#define KBASEP_DEBUG_PRINT_TRACE "" +#endif + +/** + * @def KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) + * @brief (Private) system printing function associated to the @see KBASE_DEBUG_ASSERT_MSG event. + * @param trace location in the code from where the message is printed + * @param function function from where the message is printed + * @param ... Format string followed by format arguments. + * @note function parameter cannot be concatenated with other strings + */ +/* Select the correct system output function*/ +#ifdef CONFIG_MALI_DEBUG +#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...)\ + do { \ + pr_err("Mali: %s function:%s ", trace, function);\ + pr_err(__VA_ARGS__);\ + pr_err("\n");\ + } while (false) +#else +#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) CSTD_NOP() +#endif + +#ifdef CONFIG_MALI_DEBUG +#define KBASE_CALL_ASSERT_HOOK() kbasep_debug_assert_call_hook() +#else +#define KBASE_CALL_ASSERT_HOOK() CSTD_NOP() +#endif + +/** + * @def KBASE_DEBUG_ASSERT(expr) + * @brief Calls @see KBASE_PRINT_ASSERT and prints the expression @a expr if @a expr is false + * + * @note This macro does nothing if the flag @see KBASE_DEBUG_DISABLE_ASSERTS is set to 1 + * + * @param expr Boolean expression + */ +#define KBASE_DEBUG_ASSERT(expr) \ + KBASE_DEBUG_ASSERT_MSG(expr, #expr) + +#if KBASE_DEBUG_DISABLE_ASSERTS +#define KBASE_DEBUG_ASSERT_MSG(expr, ...) CSTD_NOP() +#else + /** + * @def KBASE_DEBUG_ASSERT_MSG(expr, ...) + * @brief Calls @see KBASEP_DEBUG_ASSERT_OUT and prints the given message if @a expr is false + * + * @note This macro does nothing if the flag @see KBASE_DEBUG_DISABLE_ASSERTS is set to 1 + * + * @param expr Boolean expression + * @param ... Message to display when @a expr is false, as a format string followed by format arguments. + */ +#define KBASE_DEBUG_ASSERT_MSG(expr, ...) \ + do { \ + if (!(expr)) { \ + KBASEP_DEBUG_ASSERT_OUT(KBASEP_DEBUG_PRINT_TRACE, KBASEP_DEBUG_PRINT_FUNCTION, __VA_ARGS__);\ + KBASE_CALL_ASSERT_HOOK();\ + BUG();\ + } \ + } while (false) +#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ + +/** + * @def KBASE_DEBUG_CODE( X ) + * @brief Executes the code inside the macro only in debug mode + * + * @param X Code to compile only in debug mode. + */ +#ifdef CONFIG_MALI_DEBUG +#define KBASE_DEBUG_CODE(X) X +#else +#define KBASE_DEBUG_CODE(X) CSTD_NOP() +#endif /* CONFIG_MALI_DEBUG */ + +/** @} */ + +/** + * @brief Register a function to call on ASSERT + * + * Such functions will \b only be called during Debug mode, and for debugging + * features \b only. Do not rely on them to be called in general use. + * + * To disable the hook, supply NULL to \a func. + * + * @note This function is not thread-safe, and should only be used to + * register/deregister once in the module's lifetime. + * + * @param[in] func the function to call when an assert is triggered. + * @param[in] param the parameter to pass to \a func when calling it + */ +void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param); + +/** + * @brief Call a debug assert hook previously registered with kbase_debug_assert_register_hook() + * + * @note This function is not thread-safe with respect to multiple threads + * registering functions and parameters with + * kbase_debug_assert_register_hook(). Otherwise, thread safety is the + * responsibility of the registered hook. + */ +void kbasep_debug_assert_call_hook(void); + +#endif /* _KBASE_DEBUG_H */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c new file mode 100755 index 000000000000..f29430ddf8f9 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c @@ -0,0 +1,499 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include + +#ifdef CONFIG_DEBUG_FS + +static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev) +{ + struct list_head *event_list = &kbdev->job_fault_event_list; + unsigned long flags; + bool ret; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + ret = !list_empty(event_list); + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + return ret; +} + +static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + struct list_head *event_list = &kctx->kbdev->job_fault_event_list; + struct base_job_fault_event *event; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + if (list_empty(event_list)) { + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + return true; + } + list_for_each_entry(event, event_list, head) { + if (event->katom->kctx == kctx) { + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, + flags); + return false; + } + } + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + return true; +} + +/* wait until the fault happen and copy the event */ +static int kbase_job_fault_event_wait(struct kbase_device *kbdev, + struct base_job_fault_event *event) +{ + struct list_head *event_list = &kbdev->job_fault_event_list; + struct base_job_fault_event *event_in; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + if (list_empty(event_list)) { + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + if (wait_event_interruptible(kbdev->job_fault_wq, + kbase_is_job_fault_event_pending(kbdev))) + return -ERESTARTSYS; + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + } + + event_in = list_entry(event_list->next, + struct base_job_fault_event, head); + event->event_code = event_in->event_code; + event->katom = event_in->katom; + + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + return 0; + +} + +/* remove the event from the queue */ +static struct base_job_fault_event *kbase_job_fault_event_dequeue( + struct kbase_device *kbdev, struct list_head *event_list) +{ + struct base_job_fault_event *event; + + event = list_entry(event_list->next, + struct base_job_fault_event, head); + list_del(event_list->next); + + return event; + +} + +/* Remove all the following atoms after the failed atom in the same context + * Call the postponed bottom half of job done. + * Then, this context could be rescheduled. + */ +static void kbase_job_fault_resume_event_cleanup(struct kbase_context *kctx) +{ + struct list_head *event_list = &kctx->job_fault_resume_event_list; + + while (!list_empty(event_list)) { + struct base_job_fault_event *event; + + event = kbase_job_fault_event_dequeue(kctx->kbdev, + &kctx->job_fault_resume_event_list); + kbase_jd_done_worker(&event->katom->work); + } + +} + +/* Remove all the failed atoms that belong to different contexts + * Resume all the contexts that were suspend due to failed job + */ +static void kbase_job_fault_event_cleanup(struct kbase_device *kbdev) +{ + struct list_head *event_list = &kbdev->job_fault_event_list; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + while (!list_empty(event_list)) { + kbase_job_fault_event_dequeue(kbdev, event_list); + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + wake_up(&kbdev->job_fault_resume_wq); + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + } + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); +} + +static void kbase_job_fault_resume_worker(struct work_struct *data) +{ + struct base_job_fault_event *event = container_of(data, + struct base_job_fault_event, job_fault_work); + struct kbase_context *kctx; + struct kbase_jd_atom *katom; + + katom = event->katom; + kctx = katom->kctx; + + dev_info(kctx->kbdev->dev, "Job dumping wait\n"); + + /* When it was waked up, it need to check if queue is empty or the + * failed atom belongs to different context. If yes, wake up. Both + * of them mean the failed job has been dumped. Please note, it + * should never happen that the job_fault_event_list has the two + * atoms belong to the same context. + */ + wait_event(kctx->kbdev->job_fault_resume_wq, + kbase_ctx_has_no_event_pending(kctx)); + + atomic_set(&kctx->job_fault_count, 0); + kbase_jd_done_worker(&katom->work); + + /* In case the following atoms were scheduled during failed job dump + * the job_done_worker was held. We need to rerun it after the dump + * was finished + */ + kbase_job_fault_resume_event_cleanup(kctx); + + dev_info(kctx->kbdev->dev, "Job dumping finish, resume scheduler\n"); +} + +static struct base_job_fault_event *kbase_job_fault_event_queue( + struct list_head *event_list, + struct kbase_jd_atom *atom, + u32 completion_code) +{ + struct base_job_fault_event *event; + + event = &atom->fault_event; + + event->katom = atom; + event->event_code = completion_code; + + list_add_tail(&event->head, event_list); + + return event; + +} + +static void kbase_job_fault_event_post(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, u32 completion_code) +{ + struct base_job_fault_event *event; + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list, + katom, completion_code); + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + + wake_up_interruptible(&kbdev->job_fault_wq); + + INIT_WORK(&event->job_fault_work, kbase_job_fault_resume_worker); + queue_work(kbdev->job_fault_resume_workq, &event->job_fault_work); + + dev_info(katom->kctx->kbdev->dev, "Job fault happen, start dump: %d_%d", + katom->kctx->tgid, katom->kctx->id); + +} + +/* + * This function will process the job fault + * Get the register copy + * Send the failed job dump event + * Create a Wait queue to wait until the job dump finish + */ + +bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, + u32 completion_code) +{ + struct kbase_context *kctx = katom->kctx; + + /* Check if dumping is in the process + * only one atom of each context can be dumped at the same time + * If the atom belongs to different context, it can be dumped + */ + if (atomic_read(&kctx->job_fault_count) > 0) { + kbase_job_fault_event_queue( + &kctx->job_fault_resume_event_list, + katom, completion_code); + dev_info(kctx->kbdev->dev, "queue:%d\n", + kbase_jd_atom_id(kctx, katom)); + return true; + } + + if (kctx->kbdev->job_fault_debug == true) { + + if (completion_code != BASE_JD_EVENT_DONE) { + + if (kbase_job_fault_get_reg_snapshot(kctx) == false) { + dev_warn(kctx->kbdev->dev, "get reg dump failed\n"); + return false; + } + + kbase_job_fault_event_post(kctx->kbdev, katom, + completion_code); + atomic_inc(&kctx->job_fault_count); + dev_info(kctx->kbdev->dev, "post:%d\n", + kbase_jd_atom_id(kctx, katom)); + return true; + + } + } + return false; + +} + +static int debug_job_fault_show(struct seq_file *m, void *v) +{ + struct kbase_device *kbdev = m->private; + struct base_job_fault_event *event = (struct base_job_fault_event *)v; + struct kbase_context *kctx = event->katom->kctx; + int i; + + dev_info(kbdev->dev, "debug job fault seq show:%d_%d, %d", + kctx->tgid, kctx->id, event->reg_offset); + + if (kctx->reg_dump == NULL) { + dev_warn(kbdev->dev, "reg dump is NULL"); + return -1; + } + + if (kctx->reg_dump[event->reg_offset] == + REGISTER_DUMP_TERMINATION_FLAG) { + /* Return the error here to stop the read. And the + * following next() will not be called. The stop can + * get the real event resource and release it + */ + return -1; + } + + if (event->reg_offset == 0) + seq_printf(m, "%d_%d\n", kctx->tgid, kctx->id); + + for (i = 0; i < 50; i++) { + if (kctx->reg_dump[event->reg_offset] == + REGISTER_DUMP_TERMINATION_FLAG) { + break; + } + seq_printf(m, "%08x: %08x\n", + kctx->reg_dump[event->reg_offset], + kctx->reg_dump[1+event->reg_offset]); + event->reg_offset += 2; + + } + + + return 0; +} +static void *debug_job_fault_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct kbase_device *kbdev = m->private; + struct base_job_fault_event *event = (struct base_job_fault_event *)v; + + dev_info(kbdev->dev, "debug job fault seq next:%d, %d", + event->reg_offset, (int)*pos); + + return event; +} + +static void *debug_job_fault_start(struct seq_file *m, loff_t *pos) +{ + struct kbase_device *kbdev = m->private; + struct base_job_fault_event *event; + + dev_info(kbdev->dev, "fault job seq start:%d", (int)*pos); + + /* The condition is trick here. It needs make sure the + * fault hasn't happened and the dumping hasn't been started, + * or the dumping has finished + */ + if (*pos == 0) { + event = kmalloc(sizeof(*event), GFP_KERNEL); + if (!event) + return NULL; + event->reg_offset = 0; + if (kbase_job_fault_event_wait(kbdev, event)) { + kfree(event); + return NULL; + } + + /* The cache flush workaround is called in bottom half of + * job done but we delayed it. Now we should clean cache + * earlier. Then the GPU memory dump should be correct. + */ + kbase_backend_cacheclean(kbdev, event->katom); + } else + return NULL; + + return event; +} + +static void debug_job_fault_stop(struct seq_file *m, void *v) +{ + struct kbase_device *kbdev = m->private; + + /* here we wake up the kbase_jd_done_worker after stop, it needs + * get the memory dump before the register dump in debug daemon, + * otherwise, the memory dump may be incorrect. + */ + + if (v != NULL) { + kfree(v); + dev_info(kbdev->dev, "debug job fault seq stop stage 1"); + + } else { + unsigned long flags; + + spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); + if (!list_empty(&kbdev->job_fault_event_list)) { + kbase_job_fault_event_dequeue(kbdev, + &kbdev->job_fault_event_list); + wake_up(&kbdev->job_fault_resume_wq); + } + spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); + dev_info(kbdev->dev, "debug job fault seq stop stage 2"); + } + +} + +static const struct seq_operations ops = { + .start = debug_job_fault_start, + .next = debug_job_fault_next, + .stop = debug_job_fault_stop, + .show = debug_job_fault_show, +}; + +static int debug_job_fault_open(struct inode *in, struct file *file) +{ + struct kbase_device *kbdev = in->i_private; + + seq_open(file, &ops); + + ((struct seq_file *)file->private_data)->private = kbdev; + dev_info(kbdev->dev, "debug job fault seq open"); + + kbdev->job_fault_debug = true; + + return 0; + +} + +static int debug_job_fault_release(struct inode *in, struct file *file) +{ + struct kbase_device *kbdev = in->i_private; + + seq_release(in, file); + + kbdev->job_fault_debug = false; + + /* Clean the unprocessed job fault. After that, all the suspended + * contexts could be rescheduled. + */ + kbase_job_fault_event_cleanup(kbdev); + + dev_info(kbdev->dev, "debug job fault seq close"); + + return 0; +} + +static const struct file_operations kbasep_debug_job_fault_fops = { + .open = debug_job_fault_open, + .read = seq_read, + .llseek = seq_lseek, + .release = debug_job_fault_release, +}; + +/* + * Initialize debugfs entry for job fault dump + */ +void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("job_fault", S_IRUGO, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_debug_job_fault_fops); +} + + +int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) +{ + + INIT_LIST_HEAD(&kbdev->job_fault_event_list); + + init_waitqueue_head(&(kbdev->job_fault_wq)); + init_waitqueue_head(&(kbdev->job_fault_resume_wq)); + spin_lock_init(&kbdev->job_fault_event_lock); + + kbdev->job_fault_resume_workq = alloc_workqueue( + "kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1); + if (!kbdev->job_fault_resume_workq) + return -ENOMEM; + + kbdev->job_fault_debug = false; + + return 0; +} + +/* + * Release the relevant resource per device + */ +void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) +{ + destroy_workqueue(kbdev->job_fault_resume_workq); +} + + +/* + * Initialize the relevant data structure per context + */ +void kbase_debug_job_fault_context_init(struct kbase_context *kctx) +{ + + /* We need allocate double size register range + * Because this memory will keep the register address and value + */ + kctx->reg_dump = vmalloc(0x4000 * 2); + if (kctx->reg_dump == NULL) + return; + + if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { + vfree(kctx->reg_dump); + kctx->reg_dump = NULL; + } + INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); + atomic_set(&kctx->job_fault_count, 0); + +} + +/* + * release the relevant resource per context + */ +void kbase_debug_job_fault_context_term(struct kbase_context *kctx) +{ + vfree(kctx->reg_dump); +} + +#else /* CONFIG_DEBUG_FS */ + +int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) +{ + kbdev->job_fault_debug = false; + + return 0; +} + +void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) +{ +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h new file mode 100755 index 000000000000..a2bf8983c37c --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h @@ -0,0 +1,96 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_DEBUG_JOB_FAULT_H +#define _KBASE_DEBUG_JOB_FAULT_H + +#include +#include + +#define REGISTER_DUMP_TERMINATION_FLAG 0xFFFFFFFF + +/** + * kbase_debug_job_fault_dev_init - Create the fault event wait queue + * per device and initialize the required lists. + * @kbdev: Device pointer + * + * Return: Zero on success or a negative error code. + */ +int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev); + +/** + * kbase_debug_job_fault_debugfs_init - Initialize job fault debug sysfs + * @kbdev: Device pointer + */ +void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev); + +/** + * kbase_debug_job_fault_dev_term - Clean up resources created in + * kbase_debug_job_fault_dev_init. + * @kbdev: Device pointer + */ +void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); + +/** + * kbase_debug_job_fault_context_init - Initialize the relevant + * data structure per context + * @kctx: KBase context pointer + */ +void kbase_debug_job_fault_context_init(struct kbase_context *kctx); + +/** + * kbase_debug_job_fault_context_term - Release the relevant + * resource per context + * @kctx: KBase context pointer + */ +void kbase_debug_job_fault_context_term(struct kbase_context *kctx); + +/** + * kbase_debug_job_fault_process - Process the failed job. + * It will send a event and wake up the job fault waiting queue + * Then create a work queue to wait for job dump finish + * This function should be called in the interrupt handler and before + * jd_done that make sure the jd_done_worker will be delayed until the + * job dump finish + * @katom: The failed atom pointer + * @completion_code: the job status + * @return true if dump is going on + */ +bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, + u32 completion_code); + + +/** + * kbase_debug_job_fault_reg_snapshot_init - Set the interested registers + * address during the job fault process, the relevant registers will + * be saved when a job fault happen + * @kctx: KBase context pointer + * @reg_range: Maximum register address space + * @return true if initializing successfully + */ +bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, + int reg_range); + +/** + * kbase_job_fault_get_reg_snapshot - Read the interested registers for + * failed job dump + * @kctx: KBase context pointer + * @return true if getting registers successfully + */ +bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx); + +#endif /*_KBASE_DEBUG_JOB_FAULT_H*/ diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c new file mode 100755 index 000000000000..6f2cbdf571cb --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c @@ -0,0 +1,306 @@ +/* + * + * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Debugfs interface to dump the memory visible to the GPU + */ + +#include "mali_kbase_debug_mem_view.h" +#include "mali_kbase.h" + +#include +#include + +#ifdef CONFIG_DEBUG_FS + +struct debug_mem_mapping { + struct list_head node; + + struct kbase_mem_phy_alloc *alloc; + unsigned long flags; + + u64 start_pfn; + size_t nr_pages; +}; + +struct debug_mem_data { + struct list_head mapping_list; + struct kbase_context *kctx; +}; + +struct debug_mem_seq_off { + struct list_head *lh; + size_t offset; +}; + +static void *debug_mem_start(struct seq_file *m, loff_t *_pos) +{ + struct debug_mem_data *mem_data = m->private; + struct debug_mem_seq_off *data; + struct debug_mem_mapping *map; + loff_t pos = *_pos; + + list_for_each_entry(map, &mem_data->mapping_list, node) { + if (pos >= map->nr_pages) { + pos -= map->nr_pages; + } else { + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + data->lh = &map->node; + data->offset = pos; + return data; + } + } + + /* Beyond the end */ + return NULL; +} + +static void debug_mem_stop(struct seq_file *m, void *v) +{ + kfree(v); +} + +static void *debug_mem_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct debug_mem_data *mem_data = m->private; + struct debug_mem_seq_off *data = v; + struct debug_mem_mapping *map; + + map = list_entry(data->lh, struct debug_mem_mapping, node); + + if (data->offset < map->nr_pages - 1) { + data->offset++; + ++*pos; + return data; + } + + if (list_is_last(data->lh, &mem_data->mapping_list)) { + kfree(data); + return NULL; + } + + data->lh = data->lh->next; + data->offset = 0; + ++*pos; + + return data; +} + +static int debug_mem_show(struct seq_file *m, void *v) +{ + struct debug_mem_data *mem_data = m->private; + struct debug_mem_seq_off *data = v; + struct debug_mem_mapping *map; + int i, j; + struct page *page; + uint32_t *mapping; + pgprot_t prot = PAGE_KERNEL; + + map = list_entry(data->lh, struct debug_mem_mapping, node); + + kbase_gpu_vm_lock(mem_data->kctx); + + if (data->offset >= map->alloc->nents) { + seq_printf(m, "%016llx: Unbacked page\n\n", (map->start_pfn + + data->offset) << PAGE_SHIFT); + goto out; + } + + if (!(map->flags & KBASE_REG_CPU_CACHED)) + prot = pgprot_writecombine(prot); + + page = pfn_to_page(PFN_DOWN(map->alloc->pages[data->offset])); + mapping = vmap(&page, 1, VM_MAP, prot); + if (!mapping) + goto out; + + for (i = 0; i < PAGE_SIZE; i += 4*sizeof(*mapping)) { + seq_printf(m, "%016llx:", i + ((map->start_pfn + + data->offset) << PAGE_SHIFT)); + + for (j = 0; j < 4*sizeof(*mapping); j += sizeof(*mapping)) + seq_printf(m, " %08x", mapping[(i+j)/sizeof(*mapping)]); + seq_putc(m, '\n'); + } + + vunmap(mapping); + + seq_putc(m, '\n'); + +out: + kbase_gpu_vm_unlock(mem_data->kctx); + return 0; +} + +static const struct seq_operations ops = { + .start = debug_mem_start, + .next = debug_mem_next, + .stop = debug_mem_stop, + .show = debug_mem_show, +}; + +static int debug_mem_zone_open(struct rb_root *rbtree, + struct debug_mem_data *mem_data) +{ + int ret = 0; + struct rb_node *p; + struct kbase_va_region *reg; + struct debug_mem_mapping *mapping; + + for (p = rb_first(rbtree); p; p = rb_next(p)) { + reg = rb_entry(p, struct kbase_va_region, rblink); + + if (reg->gpu_alloc == NULL) + /* Empty region - ignore */ + continue; + + mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) { + ret = -ENOMEM; + goto out; + } + + mapping->alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + mapping->start_pfn = reg->start_pfn; + mapping->nr_pages = reg->nr_pages; + mapping->flags = reg->flags; + list_add_tail(&mapping->node, &mem_data->mapping_list); + } + +out: + return ret; +} + +static int debug_mem_open(struct inode *i, struct file *file) +{ + struct file *kctx_file = i->i_private; + struct kbase_context *kctx = kctx_file->private_data; + struct debug_mem_data *mem_data; + int ret; + + ret = seq_open(file, &ops); + if (ret) + return ret; + + mem_data = kmalloc(sizeof(*mem_data), GFP_KERNEL); + if (!mem_data) { + ret = -ENOMEM; + goto out; + } + + mem_data->kctx = kctx; + + INIT_LIST_HEAD(&mem_data->mapping_list); + + get_file(kctx_file); + + kbase_gpu_vm_lock(kctx); + + ret = debug_mem_zone_open(&kctx->reg_rbtree_same, mem_data); + if (0 != ret) { + kbase_gpu_vm_unlock(kctx); + goto out; + } + + ret = debug_mem_zone_open(&kctx->reg_rbtree_exec, mem_data); + if (0 != ret) { + kbase_gpu_vm_unlock(kctx); + goto out; + } + + ret = debug_mem_zone_open(&kctx->reg_rbtree_custom, mem_data); + if (0 != ret) { + kbase_gpu_vm_unlock(kctx); + goto out; + } + + kbase_gpu_vm_unlock(kctx); + + ((struct seq_file *)file->private_data)->private = mem_data; + + return 0; + +out: + if (mem_data) { + while (!list_empty(&mem_data->mapping_list)) { + struct debug_mem_mapping *mapping; + + mapping = list_first_entry(&mem_data->mapping_list, + struct debug_mem_mapping, node); + kbase_mem_phy_alloc_put(mapping->alloc); + list_del(&mapping->node); + kfree(mapping); + } + fput(kctx_file); + kfree(mem_data); + } + seq_release(i, file); + return ret; +} + +static int debug_mem_release(struct inode *inode, struct file *file) +{ + struct file *kctx_file = inode->i_private; + struct seq_file *sfile = file->private_data; + struct debug_mem_data *mem_data = sfile->private; + struct debug_mem_mapping *mapping; + + seq_release(inode, file); + + while (!list_empty(&mem_data->mapping_list)) { + mapping = list_first_entry(&mem_data->mapping_list, + struct debug_mem_mapping, node); + kbase_mem_phy_alloc_put(mapping->alloc); + list_del(&mapping->node); + kfree(mapping); + } + + kfree(mem_data); + + fput(kctx_file); + + return 0; +} + +static const struct file_operations kbase_debug_mem_view_fops = { + .open = debug_mem_open, + .release = debug_mem_release, + .read = seq_read, + .llseek = seq_lseek +}; + +/** + * kbase_debug_mem_view_init - Initialise the mem_view sysfs file + * @kctx_file: The /dev/mali0 file instance for the context + * + * This function creates a "mem_view" file which can be used to get a view of + * the context's memory as the GPU sees it (i.e. using the GPU's page tables). + * + * The file is cleaned up by a call to debugfs_remove_recursive() deleting the + * parent directory. + */ +void kbase_debug_mem_view_init(struct file *kctx_file) +{ + struct kbase_context *kctx = kctx_file->private_data; + + debugfs_create_file("mem_view", S_IRUGO, kctx->kctx_dentry, kctx_file, + &kbase_debug_mem_view_fops); +} + +#endif diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h new file mode 100755 index 000000000000..20ab51a776c6 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h @@ -0,0 +1,25 @@ +/* + * + * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_DEBUG_MEM_VIEW_H +#define _KBASE_DEBUG_MEM_VIEW_H + +#include + +void kbase_debug_mem_view_init(struct file *kctx_file); + +#endif diff --git a/drivers/gpu/arm/midgard/mali_kbase_defs.h b/drivers/gpu/arm/midgard/mali_kbase_defs.h new file mode 100755 index 000000000000..f8a6f33df2da --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_defs.h @@ -0,0 +1,1602 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_defs.h + * + * Defintions (types, defines, etcs) common to Kbase. They are placed here to + * allow the hierarchy of header files to work. + */ + +#ifndef _KBASE_DEFS_H_ +#define _KBASE_DEFS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef CONFIG_MALI_FPGA_BUS_LOGGER +#include +#endif + + +#ifdef CONFIG_KDS +#include +#endif /* CONFIG_KDS */ + +#if defined(CONFIG_SYNC) +#include +#else +#include "mali_kbase_fence_defs.h" +#endif + +#ifdef CONFIG_DEBUG_FS +#include +#endif /* CONFIG_DEBUG_FS */ + +#ifdef CONFIG_MALI_DEVFREQ +#include +#endif /* CONFIG_MALI_DEVFREQ */ + +#include +#include +#include + +#if defined(CONFIG_PM) +#define KBASE_PM_RUNTIME 1 +#endif + +/** Enable SW tracing when set */ +#ifdef CONFIG_MALI_MIDGARD_ENABLE_TRACE +#define KBASE_TRACE_ENABLE 1 +#endif + +#ifndef KBASE_TRACE_ENABLE +#ifdef CONFIG_MALI_DEBUG +#define KBASE_TRACE_ENABLE 1 +#else +#define KBASE_TRACE_ENABLE 0 +#endif /* CONFIG_MALI_DEBUG */ +#endif /* KBASE_TRACE_ENABLE */ + +/** Dump Job slot trace on error (only active if KBASE_TRACE_ENABLE != 0) */ +#define KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR 1 + +/** + * Number of milliseconds before resetting the GPU when a job cannot be "zapped" from the hardware. + * Note that the time is actually ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and the GPU + * actually being reset to give other contexts time for their jobs to be soft-stopped and removed from the hardware + * before resetting. + */ +#define ZAP_TIMEOUT 1000 + +/** Number of milliseconds before we time out on a GPU soft/hard reset */ +#define RESET_TIMEOUT 500 + +/** + * Prevent soft-stops from occuring in scheduling situations + * + * This is not due to HW issues, but when scheduling is desired to be more predictable. + * + * Therefore, soft stop may still be disabled due to HW issues. + * + * @note Soft stop will still be used for non-scheduling purposes e.g. when terminating a context. + * + * @note if not in use, define this value to 0 instead of \#undef'ing it + */ +#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0 + +/** + * Prevent hard-stops from occuring in scheduling situations + * + * This is not due to HW issues, but when scheduling is desired to be more predictable. + * + * @note Hard stop will still be used for non-scheduling purposes e.g. when terminating a context. + * + * @note if not in use, define this value to 0 instead of \#undef'ing it + */ +#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 + +/** + * The maximum number of Job Slots to support in the Hardware. + * + * You can optimize this down if your target devices will only ever support a + * small number of job slots. + */ +#define BASE_JM_MAX_NR_SLOTS 3 + +/** + * The maximum number of Address Spaces to support in the Hardware. + * + * You can optimize this down if your target devices will only ever support a + * small number of Address Spaces + */ +#define BASE_MAX_NR_AS 16 + +/* mmu */ +#define MIDGARD_MMU_VA_BITS 48 + +#if MIDGARD_MMU_VA_BITS > 39 +#define MIDGARD_MMU_TOPLEVEL 0 +#else +#define MIDGARD_MMU_TOPLEVEL 1 +#endif + +#define MIDGARD_MMU_BOTTOMLEVEL 3 + +#define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW | KBASE_REG_GPU_WR) + +/** setting in kbase_context::as_nr that indicates it's invalid */ +#define KBASEP_AS_NR_INVALID (-1) + +#define KBASE_LOCK_REGION_MAX_SIZE (63) +#define KBASE_LOCK_REGION_MIN_SIZE (11) + +#define KBASE_TRACE_SIZE_LOG2 8 /* 256 entries */ +#define KBASE_TRACE_SIZE (1 << KBASE_TRACE_SIZE_LOG2) +#define KBASE_TRACE_MASK ((1 << KBASE_TRACE_SIZE_LOG2)-1) + +#include "mali_kbase_js_defs.h" +#include "mali_kbase_hwaccess_defs.h" + +#define KBASEP_FORCE_REPLAY_DISABLED 0 + +/* Maximum force replay limit when randomization is enabled */ +#define KBASEP_FORCE_REPLAY_RANDOM_LIMIT 16 + +/** Atom has been previously soft-stoppped */ +#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED (1<<1) +/** Atom has been previously retried to execute */ +#define KBASE_KATOM_FLAGS_RERUN (1<<2) +#define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3) +/** Atom has been previously hard-stopped. */ +#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4) +/** Atom has caused us to enter disjoint state */ +#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5) +/* Atom blocked on cross-slot dependency */ +#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7) +/* Atom has fail dependency on cross-slot dependency */ +#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8) +/* Atom is currently in the list of atoms blocked on cross-slot dependencies */ +#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9) +/* Atom is currently holding a context reference */ +#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10) +/* Atom requires GPU to be in protected mode */ +#define KBASE_KATOM_FLAG_PROTECTED (1<<11) +/* Atom has been stored in runnable_tree */ +#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12) + +/* SW related flags about types of JS_COMMAND action + * NOTE: These must be masked off by JS_COMMAND_MASK */ + +/** This command causes a disjoint event */ +#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100 + +/** Bitmask of all SW related flags */ +#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT) + +#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK) +#error JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK. Must update JS_COMMAND_SW_<..> bitmasks +#endif + +/** Soft-stop command that causes a Disjoint event. This of course isn't + * entirely masked off by JS_COMMAND_MASK */ +#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \ + (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP) + +#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT + +/* Serialize atoms within a slot (ie only one atom per job slot) */ +#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0) +/* Serialize atoms between slots (ie only one job slot running at any time) */ +#define KBASE_SERIALIZE_INTER_SLOT (1 << 1) +/* Reset the GPU after each atom completion */ +#define KBASE_SERIALIZE_RESET (1 << 2) + +#ifdef CONFIG_DEBUG_FS +struct base_job_fault_event { + + u32 event_code; + struct kbase_jd_atom *katom; + struct work_struct job_fault_work; + struct list_head head; + int reg_offset; +}; + +#endif + +struct kbase_jd_atom_dependency { + struct kbase_jd_atom *atom; + u8 dep_type; +}; + +/** + * struct kbase_io_access - holds information about 1 register access + * + * @addr: first bit indicates r/w (r=0, w=1) + * @value: value written or read + */ +struct kbase_io_access { + uintptr_t addr; + u32 value; +}; + +/** + * struct kbase_io_history - keeps track of all recent register accesses + * + * @enabled: true if register accesses are recorded, false otherwise + * @lock: spinlock protecting kbase_io_access array + * @count: number of registers read/written + * @size: number of elements in kbase_io_access array + * @buf: array of kbase_io_access + */ +struct kbase_io_history { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) + bool enabled; +#else + u32 enabled; +#endif + + spinlock_t lock; + size_t count; + u16 size; + struct kbase_io_access *buf; +}; + +/** + * @brief The function retrieves a read-only reference to the atom field from + * the kbase_jd_atom_dependency structure + * + * @param[in] dep kbase jd atom dependency. + * + * @return readonly reference to dependent ATOM. + */ +static inline const struct kbase_jd_atom * kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep) +{ + LOCAL_ASSERT(dep != NULL); + + return (const struct kbase_jd_atom *)(dep->atom); +} + +/** + * @brief The function retrieves a read-only reference to the dependency type field from + * the kbase_jd_atom_dependency structure + * + * @param[in] dep kbase jd atom dependency. + * + * @return A dependency type value. + */ +static inline u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency *dep) +{ + LOCAL_ASSERT(dep != NULL); + + return dep->dep_type; +} + +/** + * @brief Setter macro for dep_atom array entry in kbase_jd_atom + * + * @param[in] dep The kbase jd atom dependency. + * @param[in] a The ATOM to be set as a dependency. + * @param type The ATOM dependency type to be set. + * + */ +static inline void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency *const_dep, + struct kbase_jd_atom *a, u8 type) +{ + struct kbase_jd_atom_dependency *dep; + + LOCAL_ASSERT(const_dep != NULL); + + dep = (struct kbase_jd_atom_dependency *)const_dep; + + dep->atom = a; + dep->dep_type = type; +} + +/** + * @brief Setter macro for dep_atom array entry in kbase_jd_atom + * + * @param[in] dep The kbase jd atom dependency to be cleared. + * + */ +static inline void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency *const_dep) +{ + struct kbase_jd_atom_dependency *dep; + + LOCAL_ASSERT(const_dep != NULL); + + dep = (struct kbase_jd_atom_dependency *)const_dep; + + dep->atom = NULL; + dep->dep_type = BASE_JD_DEP_TYPE_INVALID; +} + +enum kbase_atom_gpu_rb_state { + /* Atom is not currently present in slot ringbuffer */ + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB, + /* Atom is in slot ringbuffer but is blocked on a previous atom */ + KBASE_ATOM_GPU_RB_WAITING_BLOCKED, + /* Atom is in slot ringbuffer but is waiting for a previous protected + * mode transition to complete */ + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV, + /* Atom is in slot ringbuffer but is waiting for proected mode + * transition */ + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION, + /* Atom is in slot ringbuffer but is waiting for cores to become + * available */ + KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE, + /* Atom is in slot ringbuffer but is blocked on affinity */ + KBASE_ATOM_GPU_RB_WAITING_AFFINITY, + /* Atom is in slot ringbuffer and ready to run */ + KBASE_ATOM_GPU_RB_READY, + /* Atom is in slot ringbuffer and has been submitted to the GPU */ + KBASE_ATOM_GPU_RB_SUBMITTED, + /* Atom must be returned to JS as soon as it reaches the head of the + * ringbuffer due to a previous failure */ + KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1 +}; + +enum kbase_atom_enter_protected_state { + /* + * Starting state: + * Check if a transition into protected mode is required. + * + * NOTE: The integer value of this must + * match KBASE_ATOM_EXIT_PROTECTED_CHECK. + */ + KBASE_ATOM_ENTER_PROTECTED_CHECK = 0, + /* Wait for vinstr to suspend. */ + KBASE_ATOM_ENTER_PROTECTED_VINSTR, + /* Wait for the L2 to become idle in preparation for + * the coherency change. */ + KBASE_ATOM_ENTER_PROTECTED_IDLE_L2, + /* End state; + * Prepare coherency change. */ + KBASE_ATOM_ENTER_PROTECTED_FINISHED, +}; + +enum kbase_atom_exit_protected_state { + /* + * Starting state: + * Check if a transition out of protected mode is required. + * + * NOTE: The integer value of this must + * match KBASE_ATOM_ENTER_PROTECTED_CHECK. + */ + KBASE_ATOM_EXIT_PROTECTED_CHECK = 0, + /* Wait for the L2 to become idle in preparation + * for the reset. */ + KBASE_ATOM_EXIT_PROTECTED_IDLE_L2, + /* Issue the protected reset. */ + KBASE_ATOM_EXIT_PROTECTED_RESET, + /* End state; + * Wait for the reset to complete. */ + KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT, +}; + +struct kbase_ext_res { + u64 gpu_address; + struct kbase_mem_phy_alloc *alloc; +}; + +struct kbase_jd_atom { + struct work_struct work; + ktime_t start_timestamp; + + struct base_jd_udata udata; + struct kbase_context *kctx; + + struct list_head dep_head[2]; + struct list_head dep_item[2]; + const struct kbase_jd_atom_dependency dep[2]; + /* List head used during job dispatch job_done processing - as + * dependencies may not be entirely resolved at this point, we need to + * use a separate list head. */ + struct list_head jd_item; + /* true if atom's jd_item is currently on a list. Prevents atom being + * processed twice. */ + bool in_jd_list; + + u16 nr_extres; + struct kbase_ext_res *extres; + + u32 device_nr; + u64 affinity; + u64 jc; + enum kbase_atom_coreref_state coreref_state; +#ifdef CONFIG_KDS + struct list_head node; + struct kds_resource_set *kds_rset; + bool kds_dep_satisfied; +#endif /* CONFIG_KDS */ +#if defined(CONFIG_SYNC) + /* Stores either an input or output fence, depending on soft-job type */ + struct sync_fence *fence; + struct sync_fence_waiter sync_waiter; +#endif /* CONFIG_SYNC */ +#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + struct { + /* Use the functions/API defined in mali_kbase_fence.h to + * when working with this sub struct */ +#if defined(CONFIG_SYNC_FILE) + /* Input fence */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence_in; +#else + struct dma_fence *fence_in; +#endif +#endif + /* This points to the dma-buf output fence for this atom. If + * this is NULL then there is no fence for this atom and the + * following fields related to dma_fence may have invalid data. + * + * The context and seqno fields contain the details for this + * fence. + * + * This fence is signaled when the katom is completed, + * regardless of the event_code of the katom (signal also on + * failure). + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + /* The dma-buf fence context number for this atom. A unique + * context number is allocated to each katom in the context on + * context creation. + */ + unsigned int context; + /* The dma-buf fence sequence number for this atom. This is + * increased every time this katom uses dma-buf fence. + */ + atomic_t seqno; + /* This contains a list of all callbacks set up to wait on + * other fences. This atom must be held back from JS until all + * these callbacks have been called and dep_count have reached + * 0. The initial value of dep_count must be equal to the + * number of callbacks on this list. + * + * This list is protected by jctx.lock. Callbacks are added to + * this list when the atom is built and the wait are set up. + * All the callbacks then stay on the list until all callbacks + * have been called and the atom is queued, or cancelled, and + * then all callbacks are taken off the list and freed. + */ + struct list_head callbacks; + /* Atomic counter of number of outstandind dma-buf fence + * dependencies for this atom. When dep_count reaches 0 the + * atom may be queued. + * + * The special value "-1" may only be set after the count + * reaches 0, while holding jctx.lock. This indicates that the + * atom has been handled, either queued in JS or cancelled. + * + * If anyone but the dma-fence worker sets this to -1 they must + * ensure that any potentially queued worker must have + * completed before allowing the atom to be marked as unused. + * This can be done by flushing the fence work queue: + * kctx->dma_fence.wq. + */ + atomic_t dep_count; + } dma_fence; +#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE*/ + + /* Note: refer to kbasep_js_atom_retained_state, which will take a copy of some of the following members */ + enum base_jd_event_code event_code; + base_jd_core_req core_req; /**< core requirements */ + /** Job Slot to retry submitting to if submission from IRQ handler failed + * + * NOTE: see if this can be unified into the another member e.g. the event */ + int retry_submit_on_slot; + + u32 ticks; + /* JS atom priority with respect to other atoms on its kctx. */ + int sched_priority; + + int poking; /* BASE_HW_ISSUE_8316 */ + + wait_queue_head_t completed; + enum kbase_jd_atom_state status; +#ifdef CONFIG_GPU_TRACEPOINTS + int work_id; +#endif + /* Assigned after atom is completed. Used to check whether PRLAM-10676 workaround should be applied */ + int slot_nr; + + u32 atom_flags; + + /* Number of times this atom has been retried. Used by replay soft job. + */ + int retry_count; + + enum kbase_atom_gpu_rb_state gpu_rb_state; + + u64 need_cache_flush_cores_retained; + + atomic_t blocked; + + /* Pointer to atom that this atom has same-slot dependency on */ + struct kbase_jd_atom *pre_dep; + /* Pointer to atom that has same-slot dependency on this atom */ + struct kbase_jd_atom *post_dep; + + /* Pointer to atom that this atom has cross-slot dependency on */ + struct kbase_jd_atom *x_pre_dep; + /* Pointer to atom that has cross-slot dependency on this atom */ + struct kbase_jd_atom *x_post_dep; + + /* The GPU's flush count recorded at the time of submission, used for + * the cache flush optimisation */ + u32 flush_id; + + struct kbase_jd_atom_backend backend; +#ifdef CONFIG_DEBUG_FS + struct base_job_fault_event fault_event; +#endif + + /* List head used for three different purposes: + * 1. Overflow list for JS ring buffers. If an atom is ready to run, + * but there is no room in the JS ring buffer, then the atom is put + * on the ring buffer's overflow list using this list node. + * 2. List of waiting soft jobs. + */ + struct list_head queue; + + /* Used to keep track of all JIT free/alloc jobs in submission order + */ + struct list_head jit_node; + bool jit_blocked; + + /* If non-zero, this indicates that the atom will fail with the set + * event_code when the atom is processed. */ + enum base_jd_event_code will_fail_event_code; + + /* Atoms will only ever be transitioning into, or out of + * protected mode so we do not need two separate fields. + */ + union { + enum kbase_atom_enter_protected_state enter; + enum kbase_atom_exit_protected_state exit; + } protected_state; + + struct rb_node runnable_tree_node; + + /* 'Age' of atom relative to other atoms in the context. */ + u32 age; +}; + +static inline bool kbase_jd_katom_is_protected(const struct kbase_jd_atom *katom) +{ + return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED); +} + +/* + * Theory of operations: + * + * Atom objects are statically allocated within the context structure. + * + * Each atom is the head of two lists, one for the "left" set of dependencies, one for the "right" set. + */ + +#define KBASE_JD_DEP_QUEUE_SIZE 256 + +struct kbase_jd_context { + struct mutex lock; + struct kbasep_js_kctx_info sched_info; + struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; + + /** Tracks all job-dispatch jobs. This includes those not tracked by + * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ + u32 job_nr; + + /** Waitq that reflects whether there are no jobs (including SW-only + * dependency jobs). This is set when no jobs are present on the ctx, + * and clear when there are jobs. + * + * @note: Job Dispatcher knows about more jobs than the Job Scheduler: + * the Job Scheduler is unaware of jobs that are blocked on dependencies, + * and SW-only dependency jobs. + * + * This waitq can be waited upon to find out when the context jobs are all + * done/cancelled (including those that might've been blocked on + * dependencies) - and so, whether it can be terminated. However, it should + * only be terminated once it is not present in the run-pool (see + * kbasep_js_kctx_info::ctx::is_scheduled). + * + * Since the waitq is only set under kbase_jd_context::lock, + * the waiter should also briefly obtain and drop kbase_jd_context::lock to + * guarentee that the setter has completed its work on the kbase_context + * + * This must be updated atomically with: + * - kbase_jd_context::job_nr */ + wait_queue_head_t zero_jobs_wait; + + /** Job Done workqueue. */ + struct workqueue_struct *job_done_wq; + + spinlock_t tb_lock; + u32 *tb; + size_t tb_wrap_offset; + +#ifdef CONFIG_KDS + struct kds_callback kds_cb; +#endif /* CONFIG_KDS */ +#ifdef CONFIG_GPU_TRACEPOINTS + atomic_t work_id; +#endif +}; + +struct kbase_device_info { + u32 features; +}; + +/** Poking state for BASE_HW_ISSUE_8316 */ +enum { + KBASE_AS_POKE_STATE_IN_FLIGHT = 1<<0, + KBASE_AS_POKE_STATE_KILLING_POKE = 1<<1 +}; + +/** Poking state for BASE_HW_ISSUE_8316 */ +typedef u32 kbase_as_poke_state; + +struct kbase_mmu_setup { + u64 transtab; + u64 memattr; + u64 transcfg; +}; + +/** + * Important: Our code makes assumptions that a struct kbase_as structure is always at + * kbase_device->as[number]. This is used to recover the containing + * struct kbase_device from a struct kbase_as structure. + * + * Therefore, struct kbase_as structures must not be allocated anywhere else. + */ +struct kbase_as { + int number; + + struct workqueue_struct *pf_wq; + struct work_struct work_pagefault; + struct work_struct work_busfault; + enum kbase_mmu_fault_type fault_type; + bool protected_mode; + u32 fault_status; + u64 fault_addr; + u64 fault_extra_addr; + + struct kbase_mmu_setup current_setup; + + /* BASE_HW_ISSUE_8316 */ + struct workqueue_struct *poke_wq; + struct work_struct poke_work; + /** Protected by hwaccess_lock */ + int poke_refcount; + /** Protected by hwaccess_lock */ + kbase_as_poke_state poke_state; + struct hrtimer poke_timer; +}; + +static inline int kbase_as_has_bus_fault(struct kbase_as *as) +{ + return as->fault_type == KBASE_MMU_FAULT_TYPE_BUS; +} + +static inline int kbase_as_has_page_fault(struct kbase_as *as) +{ + return as->fault_type == KBASE_MMU_FAULT_TYPE_PAGE; +} + +struct kbasep_mem_device { + atomic_t used_pages; /* Tracks usage of OS shared memory. Updated + when OS memory is allocated/freed. */ + +}; + +#define KBASE_TRACE_CODE(X) KBASE_TRACE_CODE_ ## X + +enum kbase_trace_code { + /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE + * THIS MUST BE USED AT THE START OF THE ENUM */ +#define KBASE_TRACE_CODE_MAKE_CODE(X) KBASE_TRACE_CODE(X) +#include "mali_kbase_trace_defs.h" +#undef KBASE_TRACE_CODE_MAKE_CODE + /* Comma on its own, to extend the list */ + , + /* Must be the last in the enum */ + KBASE_TRACE_CODE_COUNT +}; + +#define KBASE_TRACE_FLAG_REFCOUNT (((u8)1) << 0) +#define KBASE_TRACE_FLAG_JOBSLOT (((u8)1) << 1) + +struct kbase_trace { + struct timespec64 timestamp; + u32 thread_id; + u32 cpu; + void *ctx; + bool katom; + int atom_number; + u64 atom_udata[2]; + u64 gpu_addr; + unsigned long info_val; + u8 code; + u8 jobslot; + u8 refcount; + u8 flags; +}; + +/** Event IDs for the power management framework. + * + * Any of these events might be missed, so they should not be relied upon to + * find the precise state of the GPU at a particular time in the + * trace. Overall, we should get a high percentage of these events for + * statisical purposes, and so a few missing should not be a problem */ +enum kbase_timeline_pm_event { + /* helper for tests */ + KBASEP_TIMELINE_PM_EVENT_FIRST, + + /** Event reserved for backwards compatibility with 'init' events */ + KBASE_TIMELINE_PM_EVENT_RESERVED_0 = KBASEP_TIMELINE_PM_EVENT_FIRST, + + /** The power state of the device has changed. + * + * Specifically, the device has reached a desired or available state. + */ + KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED, + + /** The GPU is becoming active. + * + * This event is sent when the first context is about to use the GPU. + */ + KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE, + + /** The GPU is becoming idle. + * + * This event is sent when the last context has finished using the GPU. + */ + KBASE_TIMELINE_PM_EVENT_GPU_IDLE, + + /** Event reserved for backwards compatibility with 'policy_change' + * events */ + KBASE_TIMELINE_PM_EVENT_RESERVED_4, + + /** Event reserved for backwards compatibility with 'system_suspend' + * events */ + KBASE_TIMELINE_PM_EVENT_RESERVED_5, + + /** Event reserved for backwards compatibility with 'system_resume' + * events */ + KBASE_TIMELINE_PM_EVENT_RESERVED_6, + + /** The job scheduler is requesting to power up/down cores. + * + * This event is sent when: + * - powered down cores are needed to complete a job + * - powered up cores are not needed anymore + */ + KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE, + + KBASEP_TIMELINE_PM_EVENT_LAST = KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE, +}; + +#ifdef CONFIG_MALI_TRACE_TIMELINE +struct kbase_trace_kctx_timeline { + atomic_t jd_atoms_in_flight; + u32 owner_tgid; +}; + +struct kbase_trace_kbdev_timeline { + /* Note: strictly speaking, not needed, because it's in sync with + * kbase_device::jm_slots[]::submitted_nr + * + * But it's kept as an example of how to add global timeline tracking + * information + * + * The caller must hold hwaccess_lock when accessing this */ + u8 slot_atoms_submitted[BASE_JM_MAX_NR_SLOTS]; + + /* Last UID for each PM event */ + atomic_t pm_event_uid[KBASEP_TIMELINE_PM_EVENT_LAST+1]; + /* Counter for generating PM event UIDs */ + atomic_t pm_event_uid_counter; + /* + * L2 transition state - true indicates that the transition is ongoing + * Expected to be protected by hwaccess_lock */ + bool l2_transitioning; +}; +#endif /* CONFIG_MALI_TRACE_TIMELINE */ + + +struct kbasep_kctx_list_element { + struct list_head link; + struct kbase_context *kctx; +}; + +/** + * Data stored per device for power management. + * + * This structure contains data for the power management framework. There is one + * instance of this structure per device in the system. + */ +struct kbase_pm_device_data { + /** + * The lock protecting Power Management structures accessed outside of + * IRQ. + * + * This lock must also be held whenever the GPU is being powered on or + * off. + */ + struct mutex lock; + + /** The reference count of active contexts on this device. */ + int active_count; + /** Flag indicating suspending/suspended */ + bool suspending; + /* Wait queue set when active_count == 0 */ + wait_queue_head_t zero_active_count_wait; + + /** + * Bit masks identifying the available shader cores that are specified + * via sysfs. One mask per job slot. + */ + u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS]; + u64 debug_core_mask_all; + + /** + * Callback for initializing the runtime power management. + * + * @param kbdev The kbase device + * + * @return 0 on success, else error code + */ + int (*callback_power_runtime_init)(struct kbase_device *kbdev); + + /** + * Callback for terminating the runtime power management. + * + * @param kbdev The kbase device + */ + void (*callback_power_runtime_term)(struct kbase_device *kbdev); + + /* Time in milliseconds between each dvfs sample */ + u32 dvfs_period; + + /* Period of GPU poweroff timer */ + ktime_t gpu_poweroff_time; + + /* Number of ticks of GPU poweroff timer before shader is powered off */ + int poweroff_shader_ticks; + + /* Number of ticks of GPU poweroff timer before GPU is powered off */ + int poweroff_gpu_ticks; + + struct kbase_pm_backend_data backend; +}; + +/** + * struct kbase_mem_pool - Page based memory pool for kctx/kbdev + * @kbdev: Kbase device where memory is used + * @cur_size: Number of free pages currently in the pool (may exceed @max_size + * in some corner cases) + * @max_size: Maximum number of free pages in the pool + * @pool_lock: Lock protecting the pool - must be held when modifying @cur_size + * and @page_list + * @page_list: List of free pages in the pool + * @reclaim: Shrinker for kernel reclaim of free pages + * @next_pool: Pointer to next pool where pages can be allocated when this pool + * is empty. Pages will spill over to the next pool when this pool + * is full. Can be NULL if there is no next pool. + */ +struct kbase_mem_pool { + struct kbase_device *kbdev; + size_t cur_size; + size_t max_size; + spinlock_t pool_lock; + struct list_head page_list; + struct shrinker reclaim; + + struct kbase_mem_pool *next_pool; +}; + +/** + * struct kbase_devfreq_opp - Lookup table for converting between nominal OPP + * frequency, and real frequency and core mask + * @opp_freq: Nominal OPP frequency + * @real_freq: Real GPU frequency + * @core_mask: Shader core mask + */ +struct kbase_devfreq_opp { + u64 opp_freq; + u64 real_freq; + u64 core_mask; +}; + +#define DEVNAME_SIZE 16 + +struct kbase_device { + s8 slot_submit_count_irq[BASE_JM_MAX_NR_SLOTS]; + + u32 hw_quirks_sc; + u32 hw_quirks_tiler; + u32 hw_quirks_mmu; + u32 hw_quirks_jm; + + struct list_head entry; + struct device *dev; + unsigned int kbase_group_error; + struct miscdevice mdev; + u64 reg_start; + size_t reg_size; + void __iomem *reg; + + struct { + int irq; + int flags; + } irqs[3]; + + struct clk *clock; +#ifdef CONFIG_REGULATOR + struct regulator *regulator; +#endif + char devname[DEVNAME_SIZE]; + +#ifdef CONFIG_MALI_NO_MALI + void *model; + struct kmem_cache *irq_slab; + struct workqueue_struct *irq_workq; + atomic_t serving_job_irq; + atomic_t serving_gpu_irq; + atomic_t serving_mmu_irq; + spinlock_t reg_op_lock; +#endif /* CONFIG_MALI_NO_MALI */ + + struct kbase_pm_device_data pm; + struct kbasep_js_device_data js_data; + struct kbase_mem_pool mem_pool; + struct kbasep_mem_device memdev; + struct kbase_mmu_mode const *mmu_mode; + + struct kbase_as as[BASE_MAX_NR_AS]; + /* The below variables (as_free and as_to_kctx) are managed by the + * Context Scheduler. The kbasep_js_device_data::runpool_irq::lock must + * be held whilst accessing these. + */ + u16 as_free; /* Bitpattern of free Address Spaces */ + /* Mapping from active Address Spaces to kbase_context */ + struct kbase_context *as_to_kctx[BASE_MAX_NR_AS]; + + + spinlock_t mmu_mask_change; + + struct kbase_gpu_props gpu_props; + + /** List of SW workarounds for HW issues */ + unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; + /** List of features available */ + unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; + + /* Bitmaps of cores that are currently in use (running jobs). + * These should be kept up to date by the job scheduler. + * + * pm.power_change_lock should be held when accessing these members. + * + * kbase_pm_check_transitions_nolock() should be called when bits are + * cleared to update the power management system and allow transitions to + * occur. */ + u64 shader_inuse_bitmap; + + /* Refcount for cores in use */ + u32 shader_inuse_cnt[64]; + + /* Bitmaps of cores the JS needs for jobs ready to run */ + u64 shader_needed_bitmap; + + /* Refcount for cores needed */ + u32 shader_needed_cnt[64]; + + u32 tiler_inuse_cnt; + + u32 tiler_needed_cnt; + + /* struct for keeping track of the disjoint information + * + * The state is > 0 if the GPU is in a disjoint state. Otherwise 0 + * The count is the number of disjoint events that have occurred on the GPU + */ + struct { + atomic_t count; + atomic_t state; + } disjoint_event; + + /* Refcount for tracking users of the l2 cache, e.g. when using hardware counter instrumentation. */ + u32 l2_users_count; + + /* Bitmaps of cores that are currently available (powered up and the power policy is happy for jobs to be + * submitted to these cores. These are updated by the power management code. The job scheduler should avoid + * submitting new jobs to any cores that are not marked as available. + * + * pm.power_change_lock should be held when accessing these members. + */ + u64 shader_available_bitmap; + u64 tiler_available_bitmap; + u64 l2_available_bitmap; + u64 stack_available_bitmap; + + u64 shader_ready_bitmap; + u64 shader_transitioning_bitmap; + + s8 nr_hw_address_spaces; /**< Number of address spaces in the GPU (constant after driver initialisation) */ + s8 nr_user_address_spaces; /**< Number of address spaces available to user contexts */ + + /* Structure used for instrumentation and HW counters dumping */ + struct kbase_hwcnt { + /* The lock should be used when accessing any of the following members */ + spinlock_t lock; + + struct kbase_context *kctx; + u64 addr; + + struct kbase_instr_backend backend; + } hwcnt; + + struct kbase_vinstr_context *vinstr_ctx; + +#if KBASE_TRACE_ENABLE + spinlock_t trace_lock; + u16 trace_first_out; + u16 trace_next_in; + struct kbase_trace *trace_rbuf; +#endif + + u32 reset_timeout_ms; + + struct mutex cacheclean_lock; + + /* Platform specific private data to be accessed by mali_kbase_config_xxx.c only */ + void *platform_context; + + /* List of kbase_contexts created */ + struct list_head kctx_list; + struct mutex kctx_list_lock; + + struct rockchip_opp_info opp_info; +#ifdef CONFIG_MALI_DEVFREQ + struct devfreq_dev_profile devfreq_profile; + struct devfreq *devfreq; + unsigned long current_freq; + unsigned long current_nominal_freq; + unsigned long current_voltage; + u64 current_core_mask; + struct kbase_devfreq_opp *opp_table; + int num_opps; + struct monitor_dev_info *mdev_info; +#ifdef CONFIG_DEVFREQ_THERMAL +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + struct devfreq_cooling_device *devfreq_cooling; +#else + struct thermal_cooling_device *devfreq_cooling; +#endif + /* Current IPA model - true for configured model, false for fallback */ + atomic_t ipa_use_configured_model; + struct { + /* Access to this struct must be with ipa.lock held */ + struct mutex lock; + struct kbase_ipa_model *configured_model; + struct kbase_ipa_model *fallback_model; + } ipa; +#endif /* CONFIG_DEVFREQ_THERMAL */ +#endif /* CONFIG_MALI_DEVFREQ */ + + +#ifdef CONFIG_MALI_TRACE_TIMELINE + struct kbase_trace_kbdev_timeline timeline; +#endif + + /* + * Control for enabling job dump on failure, set when control debugfs + * is opened. + */ + bool job_fault_debug; + +#ifdef CONFIG_DEBUG_FS + /* directory for debugfs entries */ + struct dentry *mali_debugfs_directory; + /* Root directory for per context entry */ + struct dentry *debugfs_ctx_directory; + +#ifdef CONFIG_MALI_DEBUG + /* bit for each as, set if there is new data to report */ + u64 debugfs_as_read_bitmap; +#endif /* CONFIG_MALI_DEBUG */ + + /* failed job dump, used for separate debug process */ + wait_queue_head_t job_fault_wq; + wait_queue_head_t job_fault_resume_wq; + struct workqueue_struct *job_fault_resume_workq; + struct list_head job_fault_event_list; + spinlock_t job_fault_event_lock; + struct kbase_context *kctx_fault; + +#if !MALI_CUSTOMER_RELEASE + /* Per-device data for register dumping interface */ + struct { + u16 reg_offset; /* Offset of a GPU_CONTROL register to be + dumped upon request */ + } regs_dump_debugfs_data; +#endif /* !MALI_CUSTOMER_RELEASE */ +#endif /* CONFIG_DEBUG_FS */ + + /* fbdump profiling controls set by gator */ + u32 kbase_profiling_controls[FBDUMP_CONTROL_MAX]; + + +#if MALI_CUSTOMER_RELEASE == 0 + /* Number of jobs that are run before a job is forced to fail and + * replay. May be KBASEP_FORCE_REPLAY_DISABLED, to disable forced + * failures. */ + int force_replay_limit; + /* Count of jobs between forced failures. Incremented on each job. A + * job is forced to fail once this is greater than or equal to + * force_replay_limit. */ + int force_replay_count; + /* Core requirement for jobs to be failed and replayed. May be zero. */ + base_jd_core_req force_replay_core_req; + /* true if force_replay_limit should be randomized. The random + * value will be in the range of 1 - KBASEP_FORCE_REPLAY_RANDOM_LIMIT. + */ + bool force_replay_random; +#endif + + /* Total number of created contexts */ + atomic_t ctx_num; + +#ifdef CONFIG_DEBUG_FS + /* Holds the most recent register accesses */ + struct kbase_io_history io_history; +#endif /* CONFIG_DEBUG_FS */ + + struct kbase_hwaccess_data hwaccess; + + /* Count of page/bus faults waiting for workqueues to process */ + atomic_t faults_pending; + + /* true if GPU is powered off or power off operation is in progress */ + bool poweroff_pending; + + + /* defaults for new context created for this device */ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) + bool infinite_cache_active_default; +#else + u32 infinite_cache_active_default; +#endif + size_t mem_pool_max_size_default; + + /* current gpu coherency mode */ + u32 current_gpu_coherency_mode; + /* system coherency mode */ + u32 system_coherency; + /* Flag to track when cci snoops have been enabled on the interface */ + bool cci_snoop_enabled; + + /* SMC function IDs to call into Trusted firmware to enable/disable + * cache snooping. Value of 0 indicates that they are not used + */ + u32 snoop_enable_smc; + u32 snoop_disable_smc; + + /* Protected mode operations */ + struct protected_mode_ops *protected_ops; + + /* Protected device attached to this kbase device */ + struct protected_mode_device *protected_dev; + + /* + * true when GPU is put into protected mode + */ + bool protected_mode; + + /* + * true when GPU is transitioning into or out of protected mode + */ + bool protected_mode_transition; + + /* + * true if protected mode is supported + */ + bool protected_mode_support; + + +#ifdef CONFIG_MALI_DEBUG + wait_queue_head_t driver_inactive_wait; + bool driver_inactive; +#endif /* CONFIG_MALI_DEBUG */ + +#ifdef CONFIG_MALI_FPGA_BUS_LOGGER + /* + * Bus logger integration. + */ + struct bus_logger_client *buslogger; +#endif + /* Boolean indicating if an IRQ flush during reset is in progress. */ + bool irq_reset_flush; + + /* list of inited sub systems. Used during terminate/error recovery */ + u32 inited_subsys; + + spinlock_t hwaccess_lock; + + /* Protects access to MMU operations */ + struct mutex mmu_hw_mutex; + + /* Current serialization mode. See KBASE_SERIALIZE_* for details */ + u8 serialize_jobs; +}; + +/** + * struct jsctx_queue - JS context atom queue + * @runnable_tree: Root of RB-tree containing currently runnable atoms on this + * job slot. + * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot + * dependencies. Atoms on this list will be moved to the + * runnable_tree when the blocking atom completes. + * + * hwaccess_lock must be held when accessing this structure. + */ +struct jsctx_queue { + struct rb_root runnable_tree; + struct list_head x_dep_head; +}; + + +#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \ + (((minor) & 0xFFF) << 8) | \ + ((0 & 0xFF) << 0)) + +/** + * enum kbase_context_flags - Flags for kbase contexts + * + * @KCTX_COMPAT: Set when the context process is a compat process, 32-bit + * process on a 64-bit kernel. + * + * @KCTX_RUNNABLE_REF: Set when context is counted in + * kbdev->js_data.nr_contexts_runnable. Must hold queue_mutex when accessing. + * + * @KCTX_ACTIVE: Set when the context is active. + * + * @KCTX_PULLED: Set when last kick() caused atoms to be pulled from this + * context. + * + * @KCTX_MEM_PROFILE_INITIALIZED: Set when the context's memory profile has been + * initialized. + * + * @KCTX_INFINITE_CACHE: Set when infinite cache is to be enabled for new + * allocations. Existing allocations will not change. + * + * @KCTX_SUBMIT_DISABLED: Set to prevent context from submitting any jobs. + * + * @KCTX_PRIVILEGED:Set if the context uses an address space and should be kept + * scheduled in. + * + * @KCTX_SCHEDULED: Set when the context is scheduled on the Run Pool. + * This is only ever updated whilst the jsctx_mutex is held. + * + * @KCTX_DYING: Set when the context process is in the process of being evicted. + * + * @KCTX_NO_IMPLICIT_SYNC: Set when explicit Android fences are in use on this + * context, to disable use of implicit dma-buf fences. This is used to avoid + * potential synchronization deadlocks. + * + * All members need to be separate bits. This enum is intended for use in a + * bitmask where multiple values get OR-ed together. + */ +enum kbase_context_flags { + KCTX_COMPAT = 1U << 0, + KCTX_RUNNABLE_REF = 1U << 1, + KCTX_ACTIVE = 1U << 2, + KCTX_PULLED = 1U << 3, + KCTX_MEM_PROFILE_INITIALIZED = 1U << 4, + KCTX_INFINITE_CACHE = 1U << 5, + KCTX_SUBMIT_DISABLED = 1U << 6, + KCTX_PRIVILEGED = 1U << 7, + KCTX_SCHEDULED = 1U << 8, + KCTX_DYING = 1U << 9, + KCTX_NO_IMPLICIT_SYNC = 1U << 10, +}; + +struct kbase_context { + struct file *filp; + struct kbase_device *kbdev; + int id; /* System wide unique id */ + unsigned long api_version; + phys_addr_t pgd; + struct list_head event_list; + struct list_head event_coalesce_list; + struct mutex event_mutex; + atomic_t event_closed; + struct workqueue_struct *event_workq; + atomic_t event_count; + int event_coalesce_count; + + atomic_t flags; + + atomic_t setup_complete; + atomic_t setup_in_progress; + + u64 *mmu_teardown_pages; + + struct page *aliasing_sink_page; + + struct mutex mmu_lock; + struct mutex reg_lock; /* To be converted to a rwlock? */ + struct rb_root reg_rbtree_same; /* RB tree of GPU (live) regions, + * SAME_VA zone */ + struct rb_root reg_rbtree_exec; /* RB tree of GPU (live) regions, + * EXEC zone */ + struct rb_root reg_rbtree_custom; /* RB tree of GPU (live) regions, + * CUSTOM_VA zone */ + + unsigned long cookies; + struct kbase_va_region *pending_regions[BITS_PER_LONG]; + + wait_queue_head_t event_queue; + pid_t tgid; + pid_t pid; + + struct kbase_jd_context jctx; + atomic_t used_pages; + atomic_t nonmapped_pages; + + struct kbase_mem_pool mem_pool; + + struct shrinker reclaim; + struct list_head evict_list; + + struct list_head waiting_soft_jobs; + spinlock_t waiting_soft_jobs_lock; +#ifdef CONFIG_KDS + struct list_head waiting_kds_resource; +#endif +#ifdef CONFIG_MALI_DMA_FENCE + struct { + struct list_head waiting_resource; + struct workqueue_struct *wq; + } dma_fence; +#endif /* CONFIG_MALI_DMA_FENCE */ + /** This is effectively part of the Run Pool, because it only has a valid + * setting (!=KBASEP_AS_NR_INVALID) whilst the context is scheduled in + * + * The hwaccess_lock must be held whilst accessing this. + * + * If the context relating to this as_nr is required, you must use + * kbasep_js_runpool_retain_ctx() to ensure that the context doesn't disappear + * whilst you're using it. Alternatively, just hold the hwaccess_lock + * to ensure the context doesn't disappear (but this has restrictions on what other locks + * you can take whilst doing this) */ + int as_nr; + + /* Keeps track of the number of users of this context. A user can be a + * job that is available for execution, instrumentation needing to 'pin' + * a context for counter collection, etc. If the refcount reaches 0 then + * this context is considered inactive and the previously programmed + * AS might be cleared at any point. + */ + atomic_t refcount; + + /* NOTE: + * + * Flags are in jctx.sched_info.ctx.flags + * Mutable flags *must* be accessed under jctx.sched_info.ctx.jsctx_mutex + * + * All other flags must be added there */ + spinlock_t mm_update_lock; + struct mm_struct *process_mm; + /* End of the SAME_VA zone */ + u64 same_va_end; + +#ifdef CONFIG_MALI_TRACE_TIMELINE + struct kbase_trace_kctx_timeline timeline; +#endif +#ifdef CONFIG_DEBUG_FS + /* Content of mem_profile file */ + char *mem_profile_data; + /* Size of @c mem_profile_data */ + size_t mem_profile_size; + /* Mutex guarding memory profile state */ + struct mutex mem_profile_lock; + /* Memory profile directory under debugfs */ + struct dentry *kctx_dentry; + + /* for job fault debug */ + unsigned int *reg_dump; + atomic_t job_fault_count; + /* This list will keep the following atoms during the dump + * in the same context + */ + struct list_head job_fault_resume_event_list; + +#endif /* CONFIG_DEBUG_FS */ + + struct jsctx_queue jsctx_queue + [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS]; + + /* Number of atoms currently pulled from this context */ + atomic_t atoms_pulled; + /* Number of atoms currently pulled from this context, per slot */ + atomic_t atoms_pulled_slot[BASE_JM_MAX_NR_SLOTS]; + /* Number of atoms currently pulled from this context, per slot and + * priority. Hold hwaccess_lock when accessing */ + int atoms_pulled_slot_pri[BASE_JM_MAX_NR_SLOTS][ + KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + + /* true if slot is blocked on the given priority. This will be set on a + * soft-stop */ + bool blocked_js[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + + /* Bitmask of slots that can be pulled from */ + u32 slots_pullable; + + /* Backend specific data */ + struct kbase_context_backend backend; + + /* Work structure used for deferred ASID assignment */ + struct work_struct work; + + /* Only one userspace vinstr client per kbase context */ + struct kbase_vinstr_client *vinstr_cli; + struct mutex vinstr_cli_lock; + + /* List of completed jobs waiting for events to be posted */ + struct list_head completed_jobs; + /* Number of work items currently pending on job_done_wq */ + atomic_t work_count; + + /* Waiting soft-jobs will fail when this timer expires */ + struct timer_list soft_job_timeout; + + /* JIT allocation management */ + struct kbase_va_region *jit_alloc[256]; + struct list_head jit_active_head; + struct list_head jit_pool_head; + struct list_head jit_destroy_head; + struct mutex jit_evict_lock; + struct work_struct jit_work; + + /* A list of the JIT soft-jobs in submission order + * (protected by kbase_jd_context.lock) + */ + struct list_head jit_atoms_head; + /* A list of pending JIT alloc soft-jobs (using the 'queue' list_head) + * (protected by kbase_jd_context.lock) + */ + struct list_head jit_pending_alloc; + + /* External sticky resource management */ + struct list_head ext_res_meta_head; + + /* Used to record that a drain was requested from atomic context */ + atomic_t drain_pending; + + /* Current age count, used to determine age for newly submitted atoms */ + u32 age_count; +}; + +/** + * struct kbase_ctx_ext_res_meta - Structure which binds an external resource + * to a @kbase_context. + * @ext_res_node: List head for adding the metadata to a + * @kbase_context. + * @alloc: The physical memory allocation structure + * which is mapped. + * @gpu_addr: The GPU virtual address the resource is + * mapped to. + * + * External resources can be mapped into multiple contexts as well as the same + * context multiple times. + * As kbase_va_region itself isn't refcounted we can't attach our extra + * information to it as it could be removed under our feet leaving external + * resources pinned. + * This metadata structure binds a single external resource to a single + * context, ensuring that per context mapping is tracked separately so it can + * be overridden when needed and abuses by the application (freeing the resource + * multiple times) don't effect the refcount of the physical allocation. + */ +struct kbase_ctx_ext_res_meta { + struct list_head ext_res_node; + struct kbase_mem_phy_alloc *alloc; + u64 gpu_addr; +}; + +enum kbase_reg_access_type { + REG_READ, + REG_WRITE +}; + +enum kbase_share_attr_bits { + /* (1ULL << 8) bit is reserved */ + SHARE_BOTH_BITS = (2ULL << 8), /* inner and outer shareable coherency */ + SHARE_INNER_BITS = (3ULL << 8) /* inner shareable coherency */ +}; + +/** + * kbase_device_is_cpu_coherent - Returns if the device is CPU coherent. + * @kbdev: kbase device + * + * Return: true if the device access are coherent, false if not. + */ +static inline bool kbase_device_is_cpu_coherent(struct kbase_device *kbdev) +{ + if ((kbdev->system_coherency == COHERENCY_ACE_LITE) || + (kbdev->system_coherency == COHERENCY_ACE)) + return true; + + return false; +} + +/* Conversion helpers for setting up high resolution timers */ +#define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x))*1000000U)) +#define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x)) + +/* Maximum number of loops polling the GPU for a cache flush before we assume it must have completed */ +#define KBASE_CLEAN_CACHE_MAX_LOOPS 100000 +/* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */ +#define KBASE_AS_INACTIVE_MAX_LOOPS 100000 + +/* Maximum number of times a job can be replayed */ +#define BASEP_JD_REPLAY_LIMIT 15 + +/* JobDescriptorHeader - taken from the architecture specifications, the layout + * is currently identical for all GPU archs. */ +struct job_descriptor_header { + u32 exception_status; + u32 first_incomplete_task; + u64 fault_pointer; + u8 job_descriptor_size : 1; + u8 job_type : 7; + u8 job_barrier : 1; + u8 _reserved_01 : 1; + u8 _reserved_1 : 1; + u8 _reserved_02 : 1; + u8 _reserved_03 : 1; + u8 _reserved_2 : 1; + u8 _reserved_04 : 1; + u8 _reserved_05 : 1; + u16 job_index; + u16 job_dependency_index_1; + u16 job_dependency_index_2; + union { + u64 _64; + u32 _32; + } next_job; +}; + +#endif /* _KBASE_DEFS_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_device.c b/drivers/gpu/arm/midgard/mali_kbase_device.c new file mode 100755 index 000000000000..b0eb67da8644 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_device.c @@ -0,0 +1,674 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel device APIs + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +/* NOTE: Magic - 0x45435254 (TRCE in ASCII). + * Supports tracing feature provided in the base module. + * Please keep it in sync with the value of base module. + */ +#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254 + +#if KBASE_TRACE_ENABLE +static const char *kbasep_trace_code_string[] = { + /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE + * THIS MUST BE USED AT THE START OF THE ARRAY */ +#define KBASE_TRACE_CODE_MAKE_CODE(X) # X +#include "mali_kbase_trace_defs.h" +#undef KBASE_TRACE_CODE_MAKE_CODE +}; +#endif + +#define DEBUG_MESSAGE_SIZE 256 + +static int kbasep_trace_init(struct kbase_device *kbdev); +static void kbasep_trace_term(struct kbase_device *kbdev); +static void kbasep_trace_hook_wrapper(void *param); + +struct kbase_device *kbase_device_alloc(void) +{ + return kzalloc(sizeof(struct kbase_device), GFP_KERNEL); +} + +static int kbase_device_as_init(struct kbase_device *kbdev, int i) +{ + const char format[] = "mali_mmu%d"; + char name[sizeof(format)]; + const char poke_format[] = "mali_mmu%d_poker"; + char poke_name[sizeof(poke_format)]; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) + snprintf(poke_name, sizeof(poke_name), poke_format, i); + + snprintf(name, sizeof(name), format, i); + + kbdev->as[i].number = i; + kbdev->as[i].fault_addr = 0ULL; + + kbdev->as[i].pf_wq = alloc_workqueue(name, 0, 1); + if (!kbdev->as[i].pf_wq) + return -EINVAL; + + INIT_WORK(&kbdev->as[i].work_pagefault, page_fault_worker); + INIT_WORK(&kbdev->as[i].work_busfault, bus_fault_worker); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { + struct hrtimer *poke_timer = &kbdev->as[i].poke_timer; + struct work_struct *poke_work = &kbdev->as[i].poke_work; + + kbdev->as[i].poke_wq = alloc_workqueue(poke_name, 0, 1); + if (!kbdev->as[i].poke_wq) { + destroy_workqueue(kbdev->as[i].pf_wq); + return -EINVAL; + } + KBASE_DEBUG_ASSERT(!object_is_on_stack(poke_work)); + INIT_WORK(poke_work, kbasep_as_do_poke); + + hrtimer_init(poke_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + + poke_timer->function = kbasep_as_poke_timer_callback; + + kbdev->as[i].poke_refcount = 0; + kbdev->as[i].poke_state = 0u; + } + + return 0; +} + +static void kbase_device_as_term(struct kbase_device *kbdev, int i) +{ + destroy_workqueue(kbdev->as[i].pf_wq); + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) + destroy_workqueue(kbdev->as[i].poke_wq); +} + +static int kbase_device_all_as_init(struct kbase_device *kbdev) +{ + int i, err; + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + err = kbase_device_as_init(kbdev, i); + if (err) + goto free_workqs; + } + + return 0; + +free_workqs: + for (; i > 0; i--) + kbase_device_as_term(kbdev, i); + + return err; +} + +static void kbase_device_all_as_term(struct kbase_device *kbdev) +{ + int i; + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) + kbase_device_as_term(kbdev, i); +} + +int kbase_device_init(struct kbase_device * const kbdev) +{ + int i, err; +#ifdef CONFIG_ARM64 + struct device_node *np = NULL; +#endif /* CONFIG_ARM64 */ + + spin_lock_init(&kbdev->mmu_mask_change); + mutex_init(&kbdev->mmu_hw_mutex); +#ifdef CONFIG_ARM64 + kbdev->cci_snoop_enabled = false; + np = kbdev->dev->of_node; + if (np != NULL) { + if (of_property_read_u32(np, "snoop_enable_smc", + &kbdev->snoop_enable_smc)) + kbdev->snoop_enable_smc = 0; + if (of_property_read_u32(np, "snoop_disable_smc", + &kbdev->snoop_disable_smc)) + kbdev->snoop_disable_smc = 0; + /* Either both or none of the calls should be provided. */ + if (!((kbdev->snoop_disable_smc == 0 + && kbdev->snoop_enable_smc == 0) + || (kbdev->snoop_disable_smc != 0 + && kbdev->snoop_enable_smc != 0))) { + WARN_ON(1); + err = -EINVAL; + goto fail; + } + } +#endif /* CONFIG_ARM64 */ + /* Get the list of workarounds for issues on the current HW + * (identified by the GPU_ID register) + */ + err = kbase_hw_set_issues_mask(kbdev); + if (err) + goto fail; + + /* Set the list of features available on the current HW + * (identified by the GPU_ID register) + */ + kbase_hw_set_features_mask(kbdev); + + kbase_gpuprops_set_features(kbdev); + + /* On Linux 4.0+, dma coherency is determined from device tree */ +#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) + set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops); +#endif + + /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our + * device structure was created by device-tree + */ + if (!kbdev->dev->dma_mask) + kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask; + + err = dma_set_mask(kbdev->dev, + DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); + if (err) + goto dma_set_mask_failed; + + err = dma_set_coherent_mask(kbdev->dev, + DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); + if (err) + goto dma_set_mask_failed; + + kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces; + + err = kbase_device_all_as_init(kbdev); + if (err) + goto as_init_failed; + + spin_lock_init(&kbdev->hwcnt.lock); + + err = kbasep_trace_init(kbdev); + if (err) + goto term_as; + + mutex_init(&kbdev->cacheclean_lock); + +#ifdef CONFIG_MALI_TRACE_TIMELINE + for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) + kbdev->timeline.slot_atoms_submitted[i] = 0; + + for (i = 0; i <= KBASEP_TIMELINE_PM_EVENT_LAST; ++i) + atomic_set(&kbdev->timeline.pm_event_uid[i], 0); +#endif /* CONFIG_MALI_TRACE_TIMELINE */ + + /* fbdump profiling controls set to 0 - fbdump not enabled until changed by gator */ + for (i = 0; i < FBDUMP_CONTROL_MAX; i++) + kbdev->kbase_profiling_controls[i] = 0; + + kbase_debug_assert_register_hook(&kbasep_trace_hook_wrapper, kbdev); + + atomic_set(&kbdev->ctx_num, 0); + + err = kbase_instr_backend_init(kbdev); + if (err) + goto term_trace; + + kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; + + kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS; + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); + else + kbdev->mmu_mode = kbase_mmu_mode_get_lpae(); + +#ifdef CONFIG_MALI_DEBUG + init_waitqueue_head(&kbdev->driver_inactive_wait); +#endif /* CONFIG_MALI_DEBUG */ + + return 0; +term_trace: + kbasep_trace_term(kbdev); +term_as: + kbase_device_all_as_term(kbdev); +as_init_failed: +dma_set_mask_failed: +fail: + return err; +} + +void kbase_device_term(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + +#if KBASE_TRACE_ENABLE + kbase_debug_assert_register_hook(NULL, NULL); +#endif + + kbase_instr_backend_term(kbdev); + + kbasep_trace_term(kbdev); + + kbase_device_all_as_term(kbdev); +} + +void kbase_device_free(struct kbase_device *kbdev) +{ + kfree(kbdev); +} + +int kbase_device_trace_buffer_install( + struct kbase_context *kctx, u32 *tb, size_t size) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(tb); + + /* Interface uses 16-bit value to track last accessed entry. Each entry + * is composed of two 32-bit words. + * This limits the size that can be handled without an overflow. */ + if (0xFFFF * (2 * sizeof(u32)) < size) + return -EINVAL; + + /* set up the header */ + /* magic number in the first 4 bytes */ + tb[0] = TRACE_BUFFER_HEADER_SPECIAL; + /* Store (write offset = 0, wrap counter = 0, transaction active = no) + * write offset 0 means never written. + * Offsets 1 to (wrap_offset - 1) used to store values when trace started + */ + tb[1] = 0; + + /* install trace buffer */ + spin_lock_irqsave(&kctx->jctx.tb_lock, flags); + kctx->jctx.tb_wrap_offset = size / 8; + kctx->jctx.tb = tb; + spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); + + return 0; +} + +void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(kctx); + spin_lock_irqsave(&kctx->jctx.tb_lock, flags); + kctx->jctx.tb = NULL; + kctx->jctx.tb_wrap_offset = 0; + spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); +} + +void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value) +{ + unsigned long flags; + + spin_lock_irqsave(&kctx->jctx.tb_lock, flags); + if (kctx->jctx.tb) { + u16 wrap_count; + u16 write_offset; + u32 *tb = kctx->jctx.tb; + u32 header_word; + + header_word = tb[1]; + KBASE_DEBUG_ASSERT(0 == (header_word & 0x1)); + + wrap_count = (header_word >> 1) & 0x7FFF; + write_offset = (header_word >> 16) & 0xFFFF; + + /* mark as transaction in progress */ + tb[1] |= 0x1; + mb(); + + /* calculate new offset */ + write_offset++; + if (write_offset == kctx->jctx.tb_wrap_offset) { + /* wrap */ + write_offset = 1; + wrap_count++; + wrap_count &= 0x7FFF; /* 15bit wrap counter */ + } + + /* store the trace entry at the selected offset */ + tb[write_offset * 2 + 0] = (reg_offset & ~0x3) | ((type == REG_WRITE) ? 0x1 : 0x0); + tb[write_offset * 2 + 1] = reg_value; + mb(); + + /* new header word */ + header_word = (write_offset << 16) | (wrap_count << 1) | 0x0; /* transaction complete */ + tb[1] = header_word; + } + spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); +} + +/* + * Device trace functions + */ +#if KBASE_TRACE_ENABLE + +static int kbasep_trace_init(struct kbase_device *kbdev) +{ + struct kbase_trace *rbuf; + + rbuf = kmalloc_array(KBASE_TRACE_SIZE, sizeof(*rbuf), GFP_KERNEL); + + if (!rbuf) + return -EINVAL; + + kbdev->trace_rbuf = rbuf; + spin_lock_init(&kbdev->trace_lock); + return 0; +} + +static void kbasep_trace_term(struct kbase_device *kbdev) +{ + kfree(kbdev->trace_rbuf); +} + +static void kbasep_trace_format_msg(struct kbase_trace *trace_msg, char *buffer, int len) +{ + s32 written = 0; + + /* Initial part of message */ + written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d.%.6d,%d,%d,%s,%p,", (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, kbasep_trace_code_string[trace_msg->code], trace_msg->ctx), 0); + + if (trace_msg->katom) + written += MAX(snprintf(buffer + written, MAX(len - written, 0), "atom %d (ud: 0x%llx 0x%llx)", trace_msg->atom_number, trace_msg->atom_udata[0], trace_msg->atom_udata[1]), 0); + + written += MAX(snprintf(buffer + written, MAX(len - written, 0), ",%.8llx,", trace_msg->gpu_addr), 0); + + /* NOTE: Could add function callbacks to handle different message types */ + /* Jobslot present */ + if (trace_msg->flags & KBASE_TRACE_FLAG_JOBSLOT) + written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->jobslot), 0); + + written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); + + /* Refcount present */ + if (trace_msg->flags & KBASE_TRACE_FLAG_REFCOUNT) + written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->refcount), 0); + + written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); + + /* Rest of message */ + written += MAX(snprintf(buffer + written, MAX(len - written, 0), "0x%.8lx", trace_msg->info_val), 0); +} + +static void kbasep_trace_dump_msg(struct kbase_device *kbdev, struct kbase_trace *trace_msg) +{ + char buffer[DEBUG_MESSAGE_SIZE]; + + kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); + dev_dbg(kbdev->dev, "%s", buffer); +} + +void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val) +{ + unsigned long irqflags; + struct kbase_trace *trace_msg; + + spin_lock_irqsave(&kbdev->trace_lock, irqflags); + + trace_msg = &kbdev->trace_rbuf[kbdev->trace_next_in]; + + /* Fill the message */ + trace_msg->thread_id = task_pid_nr(current); + trace_msg->cpu = task_cpu(current); + + ktime_get_real_ts64(&trace_msg->timestamp); + + trace_msg->code = code; + trace_msg->ctx = ctx; + + if (NULL == katom) { + trace_msg->katom = false; + } else { + trace_msg->katom = true; + trace_msg->atom_number = kbase_jd_atom_id(katom->kctx, katom); + trace_msg->atom_udata[0] = katom->udata.blob[0]; + trace_msg->atom_udata[1] = katom->udata.blob[1]; + } + + trace_msg->gpu_addr = gpu_addr; + trace_msg->jobslot = jobslot; + trace_msg->refcount = MIN((unsigned int)refcount, 0xFF); + trace_msg->info_val = info_val; + trace_msg->flags = flags; + + /* Update the ringbuffer indices */ + kbdev->trace_next_in = (kbdev->trace_next_in + 1) & KBASE_TRACE_MASK; + if (kbdev->trace_next_in == kbdev->trace_first_out) + kbdev->trace_first_out = (kbdev->trace_first_out + 1) & KBASE_TRACE_MASK; + + /* Done */ + + spin_unlock_irqrestore(&kbdev->trace_lock, irqflags); +} + +void kbasep_trace_clear(struct kbase_device *kbdev) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->trace_lock, flags); + kbdev->trace_first_out = kbdev->trace_next_in; + spin_unlock_irqrestore(&kbdev->trace_lock, flags); +} + +void kbasep_trace_dump(struct kbase_device *kbdev) +{ + unsigned long flags; + u32 start; + u32 end; + + dev_dbg(kbdev->dev, "Dumping trace:\nsecs,nthread,cpu,code,ctx,katom,gpu_addr,jobslot,refcount,info_val"); + spin_lock_irqsave(&kbdev->trace_lock, flags); + start = kbdev->trace_first_out; + end = kbdev->trace_next_in; + + while (start != end) { + struct kbase_trace *trace_msg = &kbdev->trace_rbuf[start]; + + kbasep_trace_dump_msg(kbdev, trace_msg); + + start = (start + 1) & KBASE_TRACE_MASK; + } + dev_dbg(kbdev->dev, "TRACE_END"); + + spin_unlock_irqrestore(&kbdev->trace_lock, flags); + + KBASE_TRACE_CLEAR(kbdev); +} + +static void kbasep_trace_hook_wrapper(void *param) +{ + struct kbase_device *kbdev = (struct kbase_device *)param; + + kbasep_trace_dump(kbdev); +} + +#ifdef CONFIG_DEBUG_FS +struct trace_seq_state { + struct kbase_trace trace_buf[KBASE_TRACE_SIZE]; + u32 start; + u32 end; +}; + +static void *kbasep_trace_seq_start(struct seq_file *s, loff_t *pos) +{ + struct trace_seq_state *state = s->private; + int i; + + if (*pos > KBASE_TRACE_SIZE) + return NULL; + i = state->start + *pos; + if ((state->end >= state->start && i >= state->end) || + i >= state->end + KBASE_TRACE_SIZE) + return NULL; + + i &= KBASE_TRACE_MASK; + + return &state->trace_buf[i]; +} + +static void kbasep_trace_seq_stop(struct seq_file *s, void *data) +{ +} + +static void *kbasep_trace_seq_next(struct seq_file *s, void *data, loff_t *pos) +{ + struct trace_seq_state *state = s->private; + int i; + + (*pos)++; + + i = (state->start + *pos) & KBASE_TRACE_MASK; + if (i == state->end) + return NULL; + + return &state->trace_buf[i]; +} + +static int kbasep_trace_seq_show(struct seq_file *s, void *data) +{ + struct kbase_trace *trace_msg = data; + char buffer[DEBUG_MESSAGE_SIZE]; + + kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); + seq_printf(s, "%s\n", buffer); + return 0; +} + +static const struct seq_operations kbasep_trace_seq_ops = { + .start = kbasep_trace_seq_start, + .next = kbasep_trace_seq_next, + .stop = kbasep_trace_seq_stop, + .show = kbasep_trace_seq_show, +}; + +static int kbasep_trace_debugfs_open(struct inode *inode, struct file *file) +{ + struct kbase_device *kbdev = inode->i_private; + unsigned long flags; + + struct trace_seq_state *state; + + state = __seq_open_private(file, &kbasep_trace_seq_ops, sizeof(*state)); + if (!state) + return -ENOMEM; + + spin_lock_irqsave(&kbdev->trace_lock, flags); + state->start = kbdev->trace_first_out; + state->end = kbdev->trace_next_in; + memcpy(state->trace_buf, kbdev->trace_rbuf, sizeof(state->trace_buf)); + spin_unlock_irqrestore(&kbdev->trace_lock, flags); + + return 0; +} + +static const struct file_operations kbasep_trace_debugfs_fops = { + .open = kbasep_trace_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + +void kbasep_trace_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("mali_trace", S_IRUGO, + kbdev->mali_debugfs_directory, kbdev, + &kbasep_trace_debugfs_fops); +} + +#else +void kbasep_trace_debugfs_init(struct kbase_device *kbdev) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +#else /* KBASE_TRACE_ENABLE */ +static int kbasep_trace_init(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); + return 0; +} + +static void kbasep_trace_term(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +static void kbasep_trace_hook_wrapper(void *param) +{ + CSTD_UNUSED(param); +} + +void kbasep_trace_dump(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} +#endif /* KBASE_TRACE_ENABLE */ + +void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value) +{ + switch (control) { + case FBDUMP_CONTROL_ENABLE: + /* fall through */ + case FBDUMP_CONTROL_RATE: + /* fall through */ + case SW_COUNTER_ENABLE: + /* fall through */ + case FBDUMP_CONTROL_RESIZE_FACTOR: + kbdev->kbase_profiling_controls[control] = value; + break; + default: + dev_err(kbdev->dev, "Profiling control %d not found\n", control); + break; + } +} + +/* + * Called by gator to control the production of + * profiling information at runtime + * */ + +void _mali_profiling_control(u32 action, u32 value) +{ + struct kbase_device *kbdev = NULL; + + /* find the first i.e. call with -1 */ + kbdev = kbase_find_device(-1); + + if (NULL != kbdev) + kbase_set_profiling_control(kbdev, action, value); +} +KBASE_EXPORT_SYMBOL(_mali_profiling_control); + diff --git a/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c b/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c new file mode 100755 index 000000000000..f70bcccf4050 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c @@ -0,0 +1,76 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Base kernel disjoint events helper functions + */ + +#include + +void kbase_disjoint_init(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + atomic_set(&kbdev->disjoint_event.count, 0); + atomic_set(&kbdev->disjoint_event.state, 0); +} + +/* increment the disjoint event count */ +void kbase_disjoint_event(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + atomic_inc(&kbdev->disjoint_event.count); +} + +/* increment the state and the event counter */ +void kbase_disjoint_state_up(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + atomic_inc(&kbdev->disjoint_event.state); + + kbase_disjoint_event(kbdev); +} + +/* decrement the state */ +void kbase_disjoint_state_down(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(atomic_read(&kbdev->disjoint_event.state) > 0); + + kbase_disjoint_event(kbdev); + + atomic_dec(&kbdev->disjoint_event.state); +} + +/* increments the count only if the state is > 0 */ +void kbase_disjoint_event_potential(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + if (atomic_read(&kbdev->disjoint_event.state)) + kbase_disjoint_event(kbdev); +} + +u32 kbase_disjoint_event_get(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev != NULL); + + return atomic_read(&kbdev->disjoint_event.count); +} +KBASE_EXPORT_TEST_API(kbase_disjoint_event_get); diff --git a/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c new file mode 100755 index 000000000000..9197743c81d4 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c @@ -0,0 +1,449 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* Include mali_kbase_dma_fence.h before checking for CONFIG_MALI_DMA_FENCE as + * it will be set there. + */ +#include "mali_kbase_dma_fence.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static void +kbase_dma_fence_work(struct work_struct *pwork); + +static void +kbase_dma_fence_waiters_add(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + list_add_tail(&katom->queue, &kctx->dma_fence.waiting_resource); +} + +static void +kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom) +{ + list_del(&katom->queue); +} + +static int +kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info, + struct ww_acquire_ctx *ctx) +{ + struct reservation_object *content_res = NULL; + unsigned int content_res_idx = 0; + unsigned int r; + int err = 0; + + ww_acquire_init(ctx, &reservation_ww_class); + +retry: + for (r = 0; r < info->dma_fence_resv_count; r++) { + if (info->resv_objs[r] == content_res) { + content_res = NULL; + continue; + } + + err = ww_mutex_lock(&info->resv_objs[r]->lock, ctx); + if (err) + goto error; + } + + ww_acquire_done(ctx); + return err; + +error: + content_res_idx = r; + + /* Unlock the locked one ones */ + while (r--) + ww_mutex_unlock(&info->resv_objs[r]->lock); + + if (content_res) + ww_mutex_unlock(&content_res->lock); + + /* If we deadlock try with lock_slow and retry */ + if (err == -EDEADLK) { + content_res = info->resv_objs[content_res_idx]; + ww_mutex_lock_slow(&content_res->lock, ctx); + goto retry; + } + + /* If we are here the function failed */ + ww_acquire_fini(ctx); + return err; +} + +static void +kbase_dma_fence_unlock_reservations(struct kbase_dma_fence_resv_info *info, + struct ww_acquire_ctx *ctx) +{ + unsigned int r; + + for (r = 0; r < info->dma_fence_resv_count; r++) + ww_mutex_unlock(&info->resv_objs[r]->lock); + ww_acquire_fini(ctx); +} + +/** + * kbase_dma_fence_queue_work() - Queue work to handle @katom + * @katom: Pointer to atom for which to queue work + * + * Queue kbase_dma_fence_work() for @katom to clean up the fence callbacks and + * submit the atom. + */ +static void +kbase_dma_fence_queue_work(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + bool ret; + + INIT_WORK(&katom->work, kbase_dma_fence_work); + ret = queue_work(kctx->dma_fence.wq, &katom->work); + /* Warn if work was already queued, that should not happen. */ + WARN_ON(!ret); +} + +/** + * kbase_dma_fence_cancel_atom() - Cancels waiting on an atom + * @katom: Katom to cancel + * + * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. + */ +static void +kbase_dma_fence_cancel_atom(struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&katom->kctx->jctx.lock); + + /* Cancel callbacks and clean up. */ + kbase_fence_free_callbacks(katom); + + /* Mark the atom as handled in case all fences signaled just before + * canceling the callbacks and the worker was queued. + */ + kbase_fence_dep_count_set(katom, -1); + + /* Prevent job_done_nolock from being called twice on an atom when + * there is a race between job completion and cancellation. + */ + + if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { + /* Wait was cancelled - zap the atom */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); + } +} + +/** + * kbase_dma_fence_work() - Worker thread called when a fence is signaled + * @pwork: work_struct containing a pointer to a katom + * + * This function will clean and mark all dependencies as satisfied + */ +static void +kbase_dma_fence_work(struct work_struct *pwork) +{ + struct kbase_jd_atom *katom; + struct kbase_jd_context *ctx; + + katom = container_of(pwork, struct kbase_jd_atom, work); + ctx = &katom->kctx->jctx; + + mutex_lock(&ctx->lock); + if (kbase_fence_dep_count_read(katom) != 0) + goto out; + + kbase_fence_dep_count_set(katom, -1); + + /* Remove atom from list of dma-fence waiting atoms. */ + kbase_dma_fence_waiters_remove(katom); + /* Cleanup callbacks. */ + kbase_fence_free_callbacks(katom); + /* + * Queue atom on GPU, unless it has already completed due to a failing + * dependency. Run jd_done_nolock() on the katom if it is completed. + */ + if (unlikely(katom->status == KBASE_JD_ATOM_STATE_COMPLETED)) + jd_done_nolock(katom, NULL); + else + kbase_jd_dep_clear_locked(katom); + +out: + mutex_unlock(&ctx->lock); +} + +static void +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) +#else +kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) +#endif +{ + struct kbase_fence_cb *kcb = container_of(cb, + struct kbase_fence_cb, + fence_cb); + struct kbase_jd_atom *katom = kcb->katom; + + /* If the atom is zapped dep_count will be forced to a negative number + * preventing this callback from ever scheduling work. Which in turn + * would reschedule the atom. + */ + + if (kbase_fence_dep_count_dec_and_test(katom)) + kbase_dma_fence_queue_work(katom); +} + +static int +kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, + struct reservation_object *resv, + bool exclusive) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *excl_fence = NULL; + struct fence **shared_fences = NULL; +#else + struct dma_fence *excl_fence = NULL; + struct dma_fence **shared_fences = NULL; +#endif + unsigned int shared_count = 0; + int err, i; + + err = reservation_object_get_fences_rcu(resv, + &excl_fence, + &shared_count, + &shared_fences); + if (err) + return err; + + if (excl_fence) { + err = kbase_fence_add_callback(katom, + excl_fence, + kbase_dma_fence_cb); + + /* Release our reference, taken by reservation_object_get_fences_rcu(), + * to the fence. We have set up our callback (if that was possible), + * and it's the fence's owner is responsible for singling the fence + * before allowing it to disappear. + */ + dma_fence_put(excl_fence); + + if (err) + goto out; + } + + if (exclusive) { + for (i = 0; i < shared_count; i++) { + err = kbase_fence_add_callback(katom, + shared_fences[i], + kbase_dma_fence_cb); + if (err) + goto out; + } + } + + /* Release all our references to the shared fences, taken by + * reservation_object_get_fences_rcu(). We have set up our callback (if + * that was possible), and it's the fence's owner is responsible for + * signaling the fence before allowing it to disappear. + */ +out: + for (i = 0; i < shared_count; i++) + dma_fence_put(shared_fences[i]); + kfree(shared_fences); + + if (err) { + /* + * On error, cancel and clean up all callbacks that was set up + * before the error. + */ + kbase_fence_free_callbacks(katom); + } + + return err; +} + +void kbase_dma_fence_add_reservation(struct reservation_object *resv, + struct kbase_dma_fence_resv_info *info, + bool exclusive) +{ + unsigned int i; + + for (i = 0; i < info->dma_fence_resv_count; i++) { + /* Duplicate resource, ignore */ + if (info->resv_objs[i] == resv) + return; + } + + info->resv_objs[info->dma_fence_resv_count] = resv; + if (exclusive) + set_bit(info->dma_fence_resv_count, + info->dma_fence_excl_bitmap); + (info->dma_fence_resv_count)++; +} + +int kbase_dma_fence_wait(struct kbase_jd_atom *katom, + struct kbase_dma_fence_resv_info *info) +{ + int err, i; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + struct ww_acquire_ctx ww_ctx; + + lockdep_assert_held(&katom->kctx->jctx.lock); + + fence = kbase_fence_out_new(katom); + if (!fence) { + err = -ENOMEM; + dev_err(katom->kctx->kbdev->dev, + "Error %d creating fence.\n", err); + return err; + } + + kbase_fence_dep_count_set(katom, 1); + + err = kbase_dma_fence_lock_reservations(info, &ww_ctx); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d locking reservations.\n", err); + kbase_fence_dep_count_set(katom, -1); + kbase_fence_out_remove(katom); + return err; + } + + for (i = 0; i < info->dma_fence_resv_count; i++) { + struct reservation_object *obj = info->resv_objs[i]; + + if (!test_bit(i, info->dma_fence_excl_bitmap)) { + err = reservation_object_reserve_shared(obj); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d reserving space for shared fence.\n", err); + goto end; + } + + err = kbase_dma_fence_add_reservation_callback(katom, obj, false); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d adding reservation to callback.\n", err); + goto end; + } + + reservation_object_add_shared_fence(obj, fence); + } else { + err = kbase_dma_fence_add_reservation_callback(katom, obj, true); + if (err) { + dev_err(katom->kctx->kbdev->dev, + "Error %d adding reservation to callback.\n", err); + goto end; + } + + reservation_object_add_excl_fence(obj, fence); + } + } + +end: + kbase_dma_fence_unlock_reservations(info, &ww_ctx); + + if (likely(!err)) { + /* Test if the callbacks are already triggered */ + if (kbase_fence_dep_count_dec_and_test(katom)) { + kbase_fence_dep_count_set(katom, -1); + kbase_fence_free_callbacks(katom); + } else { + /* Add katom to the list of dma-buf fence waiting atoms + * only if it is still waiting. + */ + kbase_dma_fence_waiters_add(katom); + } + } else { + /* There was an error, cancel callbacks, set dep_count to -1 to + * indicate that the atom has been handled (the caller will + * kill it for us), signal the fence, free callbacks and the + * fence. + */ + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + kbase_dma_fence_signal(katom); + } + + return err; +} + +void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx) +{ + struct list_head *list = &kctx->dma_fence.waiting_resource; + + while (!list_empty(list)) { + struct kbase_jd_atom *katom; + + katom = list_first_entry(list, struct kbase_jd_atom, queue); + kbase_dma_fence_waiters_remove(katom); + kbase_dma_fence_cancel_atom(katom); + } +} + +void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom) +{ + /* Cancel callbacks and clean up. */ + if (kbase_fence_free_callbacks(katom)) + kbase_dma_fence_queue_work(katom); +} + +void kbase_dma_fence_signal(struct kbase_jd_atom *katom) +{ + if (!katom->dma_fence.fence) + return; + + /* Signal the atom's fence. */ + dma_fence_signal(katom->dma_fence.fence); + + kbase_fence_out_remove(katom); + + kbase_fence_free_callbacks(katom); +} + +void kbase_dma_fence_term(struct kbase_context *kctx) +{ + destroy_workqueue(kctx->dma_fence.wq); + kctx->dma_fence.wq = NULL; +} + +int kbase_dma_fence_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->dma_fence.waiting_resource); + + kctx->dma_fence.wq = alloc_workqueue("mali-fence-%d", + WQ_UNBOUND, 1, kctx->pid); + if (!kctx->dma_fence.wq) + return -ENOMEM; + + return 0; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h new file mode 100755 index 000000000000..c9ab40350422 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h @@ -0,0 +1,131 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_DMA_FENCE_H_ +#define _KBASE_DMA_FENCE_H_ + +#ifdef CONFIG_MALI_DMA_FENCE + +#include +#include +#include + + +/* Forward declaration from mali_kbase_defs.h */ +struct kbase_jd_atom; +struct kbase_context; + +/** + * struct kbase_dma_fence_resv_info - Structure with list of reservation objects + * @resv_objs: Array of reservation objects to attach the + * new fence to. + * @dma_fence_resv_count: Number of reservation objects in the array. + * @dma_fence_excl_bitmap: Specifies which resv_obj are exclusive. + * + * This is used by some functions to pass around a collection of data about + * reservation objects. + */ +struct kbase_dma_fence_resv_info { + struct reservation_object **resv_objs; + unsigned int dma_fence_resv_count; + unsigned long *dma_fence_excl_bitmap; +}; + +/** + * kbase_dma_fence_add_reservation() - Adds a resv to the array of resv_objs + * @resv: Reservation object to add to the array. + * @info: Pointer to struct with current reservation info + * @exclusive: Boolean indicating if exclusive access is needed + * + * The function adds a new reservation_object to an existing array of + * reservation_objects. At the same time keeps track of which objects require + * exclusive access in dma_fence_excl_bitmap. + */ +void kbase_dma_fence_add_reservation(struct reservation_object *resv, + struct kbase_dma_fence_resv_info *info, + bool exclusive); + +/** + * kbase_dma_fence_wait() - Creates a new fence and attaches it to the resv_objs + * @katom: Katom with the external dependency. + * @info: Pointer to struct with current reservation info + * + * Return: An error code or 0 if succeeds + */ +int kbase_dma_fence_wait(struct kbase_jd_atom *katom, + struct kbase_dma_fence_resv_info *info); + +/** + * kbase_dma_fence_cancel_ctx() - Cancel all dma-fences blocked atoms on kctx + * @kctx: Pointer to kbase context + * + * This function will cancel and clean up all katoms on @kctx that is waiting + * on dma-buf fences. + * + * Locking: jctx.lock needs to be held when calling this function. + */ +void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx); + +/** + * kbase_dma_fence_cancel_callbacks() - Cancel only callbacks on katom + * @katom: Pointer to katom whose callbacks are to be canceled + * + * This function cancels all dma-buf fence callbacks on @katom, but does not + * cancel the katom itself. + * + * The caller is responsible for ensuring that jd_done_nolock is called on + * @katom. + * + * Locking: jctx.lock must be held when calling this function. + */ +void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom); + +/** + * kbase_dma_fence_signal() - Signal katom's fence and clean up after wait + * @katom: Pointer to katom to signal and clean up + * + * This function will signal the @katom's fence, if it has one, and clean up + * the callback data from the katom's wait on earlier fences. + * + * Locking: jctx.lock must be held while calling this function. + */ +void kbase_dma_fence_signal(struct kbase_jd_atom *katom); + +/** + * kbase_dma_fence_term() - Terminate Mali dma-fence context + * @kctx: kbase context to terminate + */ +void kbase_dma_fence_term(struct kbase_context *kctx); + +/** + * kbase_dma_fence_init() - Initialize Mali dma-fence context + * @kctx: kbase context to initialize + */ +int kbase_dma_fence_init(struct kbase_context *kctx); + + +#else /* CONFIG_MALI_DMA_FENCE */ +/* Dummy functions for when dma-buf fence isn't enabled. */ + +static inline int kbase_dma_fence_init(struct kbase_context *kctx) +{ + return 0; +} + +static inline void kbase_dma_fence_term(struct kbase_context *kctx) {} +#endif /* CONFIG_MALI_DMA_FENCE */ +#endif diff --git a/drivers/gpu/arm/midgard/mali_kbase_event.c b/drivers/gpu/arm/midgard/mali_kbase_event.c new file mode 100755 index 000000000000..188148645f37 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_event.c @@ -0,0 +1,259 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include + +static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + struct base_jd_udata data; + + lockdep_assert_held(&kctx->jctx.lock); + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(katom != NULL); + KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); + + data = katom->udata; + + KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(1, &kctx->timeline.jd_atoms_in_flight)); + + KBASE_TLSTREAM_TL_NRET_ATOM_CTX(katom, kctx); + KBASE_TLSTREAM_TL_DEL_ATOM(katom); + + katom->status = KBASE_JD_ATOM_STATE_UNUSED; + + wake_up(&katom->completed); + + return data; +} + +int kbase_event_pending(struct kbase_context *ctx) +{ + KBASE_DEBUG_ASSERT(ctx); + + return (atomic_read(&ctx->event_count) != 0) || + (atomic_read(&ctx->event_closed) != 0); +} + +KBASE_EXPORT_TEST_API(kbase_event_pending); + +int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent) +{ + struct kbase_jd_atom *atom; + + KBASE_DEBUG_ASSERT(ctx); + + mutex_lock(&ctx->event_mutex); + + if (list_empty(&ctx->event_list)) { + if (!atomic_read(&ctx->event_closed)) { + mutex_unlock(&ctx->event_mutex); + return -1; + } + + /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */ + mutex_unlock(&ctx->event_mutex); + uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED; + memset(&uevent->udata, 0, sizeof(uevent->udata)); + dev_dbg(ctx->kbdev->dev, + "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n", + BASE_JD_EVENT_DRV_TERMINATED); + return 0; + } + + /* normal event processing */ + atomic_dec(&ctx->event_count); + atom = list_entry(ctx->event_list.next, struct kbase_jd_atom, dep_item[0]); + list_del(ctx->event_list.next); + + mutex_unlock(&ctx->event_mutex); + + dev_dbg(ctx->kbdev->dev, "event dequeuing %p\n", (void *)atom); + uevent->event_code = atom->event_code; + uevent->atom_number = (atom - ctx->jctx.atoms); + + if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) + kbase_jd_free_external_resources(atom); + + mutex_lock(&ctx->jctx.lock); + uevent->udata = kbase_event_process(ctx, atom); + mutex_unlock(&ctx->jctx.lock); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_event_dequeue); + +/** + * kbase_event_process_noreport_worker - Worker for processing atoms that do not + * return an event but do have external + * resources + * @data: Work structure + */ +static void kbase_event_process_noreport_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) + kbase_jd_free_external_resources(katom); + + mutex_lock(&kctx->jctx.lock); + kbase_event_process(kctx, katom); + mutex_unlock(&kctx->jctx.lock); +} + +/** + * kbase_event_process_noreport - Process atoms that do not return an event + * @kctx: Context pointer + * @katom: Atom to be processed + * + * Atoms that do not have external resources will be processed immediately. + * Atoms that do have external resources will be processed on a workqueue, in + * order to avoid locking issues. + */ +static void kbase_event_process_noreport(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + INIT_WORK(&katom->work, kbase_event_process_noreport_worker); + queue_work(kctx->event_workq, &katom->work); + } else { + kbase_event_process(kctx, katom); + } +} + +/** + * kbase_event_coalesce - Move pending events to the main event list + * @kctx: Context pointer + * + * kctx->event_list and kctx->event_coalesce_count must be protected + * by a lock unless this is the last thread using them + * (and we're about to terminate the lock). + * + * Return: The number of pending events moved to the main event list + */ +static int kbase_event_coalesce(struct kbase_context *kctx) +{ + const int event_count = kctx->event_coalesce_count; + + /* Join the list of pending events onto the tail of the main list + and reset it */ + list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); + kctx->event_coalesce_count = 0; + + /* Return the number of events moved */ + return event_count; +} + +void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom) +{ + if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) { + if (atom->event_code == BASE_JD_EVENT_DONE) { + /* Don't report the event */ + kbase_event_process_noreport(ctx, atom); + return; + } + } + + if (atom->core_req & BASEP_JD_REQ_EVENT_NEVER) { + /* Don't report the event */ + kbase_event_process_noreport(ctx, atom); + return; + } + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, TL_ATOM_STATE_POSTED); + if (atom->core_req & BASE_JD_REQ_EVENT_COALESCE) { + /* Don't report the event until other event(s) have completed */ + mutex_lock(&ctx->event_mutex); + list_add_tail(&atom->dep_item[0], &ctx->event_coalesce_list); + ++ctx->event_coalesce_count; + mutex_unlock(&ctx->event_mutex); + } else { + /* Report the event and any pending events now */ + int event_count = 1; + + mutex_lock(&ctx->event_mutex); + event_count += kbase_event_coalesce(ctx); + list_add_tail(&atom->dep_item[0], &ctx->event_list); + atomic_add(event_count, &ctx->event_count); + mutex_unlock(&ctx->event_mutex); + + kbase_event_wakeup(ctx); + } +} +KBASE_EXPORT_TEST_API(kbase_event_post); + +void kbase_event_close(struct kbase_context *kctx) +{ + mutex_lock(&kctx->event_mutex); + atomic_set(&kctx->event_closed, true); + mutex_unlock(&kctx->event_mutex); + kbase_event_wakeup(kctx); +} + +int kbase_event_init(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx); + + INIT_LIST_HEAD(&kctx->event_list); + INIT_LIST_HEAD(&kctx->event_coalesce_list); + mutex_init(&kctx->event_mutex); + atomic_set(&kctx->event_count, 0); + kctx->event_coalesce_count = 0; + atomic_set(&kctx->event_closed, false); + kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); + + if (NULL == kctx->event_workq) + return -EINVAL; + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_event_init); + +void kbase_event_cleanup(struct kbase_context *kctx) +{ + int event_count; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(kctx->event_workq); + + flush_workqueue(kctx->event_workq); + destroy_workqueue(kctx->event_workq); + + /* We use kbase_event_dequeue to remove the remaining events as that + * deals with all the cleanup needed for the atoms. + * + * Note: use of kctx->event_list without a lock is safe because this must be the last + * thread using it (because we're about to terminate the lock) + */ + event_count = kbase_event_coalesce(kctx); + atomic_add(event_count, &kctx->event_count); + + while (!list_empty(&kctx->event_list)) { + struct base_jd_event_v2 event; + + kbase_event_dequeue(kctx, &event); + } +} + +KBASE_EXPORT_TEST_API(kbase_event_cleanup); diff --git a/drivers/gpu/arm/midgard/mali_kbase_fence.c b/drivers/gpu/arm/midgard/mali_kbase_fence.c new file mode 100755 index 000000000000..3bcfb38c31c2 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_fence.c @@ -0,0 +1,200 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include +#include +#include + +/* Spin lock protecting all Mali fences as fence->lock. */ +static DEFINE_SPINLOCK(kbase_fence_lock); + +static const char * +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_get_driver_name(struct fence *fence) +#else +kbase_fence_get_driver_name(struct dma_fence *fence) +#endif +{ + return kbase_drv_name; +} + +static const char * +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_get_timeline_name(struct fence *fence) +#else +kbase_fence_get_timeline_name(struct dma_fence *fence) +#endif +{ + return kbase_timeline_name; +} + +static bool +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_enable_signaling(struct fence *fence) +#else +kbase_fence_enable_signaling(struct dma_fence *fence) +#endif +{ + return true; +} + +static void +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +kbase_fence_fence_value_str(struct fence *fence, char *str, int size) +#else +kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) +#endif +{ +#if (KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE) + snprintf(str, size, "%u", fence->seqno); +#else + snprintf(str, size, "%llu", fence->seqno); +#endif +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +const struct fence_ops kbase_fence_ops = { + .wait = fence_default_wait, +#else +const struct dma_fence_ops kbase_fence_ops = { + .wait = dma_fence_default_wait, +#endif + .get_driver_name = kbase_fence_get_driver_name, + .get_timeline_name = kbase_fence_get_timeline_name, + .enable_signaling = kbase_fence_enable_signaling, + .fence_value_str = kbase_fence_fence_value_str +}; + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +struct fence * +kbase_fence_out_new(struct kbase_jd_atom *katom) +#else +struct dma_fence * +kbase_fence_out_new(struct kbase_jd_atom *katom) +#endif +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + WARN_ON(katom->dma_fence.fence); + + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return NULL; + + dma_fence_init(fence, + &kbase_fence_ops, + &kbase_fence_lock, + katom->dma_fence.context, + atomic_inc_return(&katom->dma_fence.seqno)); + + katom->dma_fence.fence = fence; + + return fence; +} + +bool +kbase_fence_free_callbacks(struct kbase_jd_atom *katom) +{ + struct kbase_fence_cb *cb, *tmp; + bool res = false; + + lockdep_assert_held(&katom->kctx->jctx.lock); + + /* Clean up and free callbacks. */ + list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) { + bool ret; + + /* Cancel callbacks that hasn't been called yet. */ + ret = dma_fence_remove_callback(cb->fence, &cb->fence_cb); + if (ret) { + int ret; + + /* Fence had not signaled, clean up after + * canceling. + */ + ret = atomic_dec_return(&katom->dma_fence.dep_count); + + if (unlikely(ret == 0)) + res = true; + } + + /* + * Release the reference taken in + * kbase_fence_add_callback(). + */ + dma_fence_put(cb->fence); + list_del(&cb->node); + kfree(cb); + } + + return res; +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +int +kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct fence *fence, + fence_func_t callback) +#else +int +kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct dma_fence *fence, + dma_fence_func_t callback) +#endif +{ + int err = 0; + struct kbase_fence_cb *kbase_fence_cb; + + if (!fence) + return -EINVAL; + + kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL); + if (!kbase_fence_cb) + return -ENOMEM; + + kbase_fence_cb->fence = fence; + kbase_fence_cb->katom = katom; + INIT_LIST_HEAD(&kbase_fence_cb->node); + + err = dma_fence_add_callback(fence, &kbase_fence_cb->fence_cb, + callback); + if (err == -ENOENT) { + /* Fence signaled, clear the error and return */ + err = 0; + kfree(kbase_fence_cb); + } else if (err) { + kfree(kbase_fence_cb); + } else { + /* + * Get reference to fence that will be kept until callback gets + * cleaned up in kbase_fence_free_callbacks(). + */ + dma_fence_get(fence); + atomic_inc(&katom->dma_fence.dep_count); + /* Add callback to katom's list of callbacks */ + list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks); + } + + return err; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_fence.h b/drivers/gpu/arm/midgard/mali_kbase_fence.h new file mode 100755 index 000000000000..639cc2ef4348 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_fence.h @@ -0,0 +1,275 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_FENCE_H_ +#define _KBASE_FENCE_H_ + +/* + * mali_kbase_fence.[hc] has common fence code used by both + * - CONFIG_MALI_DMA_FENCE - implicit DMA fences + * - CONFIG_SYNC_FILE - explicit fences beginning with 4.9 kernel + */ + +#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + +#include +#include "mali_kbase_fence_defs.h" +#include "mali_kbase.h" + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +extern const struct fence_ops kbase_fence_ops; +#else +extern const struct dma_fence_ops kbase_fence_ops; +#endif + +/** +* struct kbase_fence_cb - Mali dma-fence callback data struct +* @fence_cb: Callback function +* @katom: Pointer to katom that is waiting on this callback +* @fence: Pointer to the fence object on which this callback is waiting +* @node: List head for linking this callback to the katom +*/ +struct kbase_fence_cb { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence_cb fence_cb; + struct fence *fence; +#else + struct dma_fence_cb fence_cb; + struct dma_fence *fence; +#endif + struct kbase_jd_atom *katom; + struct list_head node; +}; + +/** + * kbase_fence_out_new() - Creates a new output fence and puts it on the atom + * @katom: Atom to create an output fence for + * + * return: A new fence object on success, NULL on failure. + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); +#else +struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); +#endif + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_fence_in_set() - Assign input fence to atom + * @katom: Atom to assign input fence to + * @fence: Input fence to assign to atom + * + * This function will take ownership of one fence reference! + */ +#define kbase_fence_fence_in_set(katom, fence) \ + do { \ + WARN_ON((katom)->dma_fence.fence_in); \ + (katom)->dma_fence.fence_in = fence; \ + } while (0) +#endif + +/** + * kbase_fence_out_remove() - Removes the output fence from atom + * @katom: Atom to remove output fence for + * + * This will also release the reference to this fence which the atom keeps + */ +static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom) +{ + if (katom->dma_fence.fence) { + dma_fence_put(katom->dma_fence.fence); + katom->dma_fence.fence = NULL; + } +} + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_out_remove() - Removes the input fence from atom + * @katom: Atom to remove input fence for + * + * This will also release the reference to this fence which the atom keeps + */ +static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom) +{ + if (katom->dma_fence.fence_in) { + dma_fence_put(katom->dma_fence.fence_in); + katom->dma_fence.fence_in = NULL; + } +} +#endif + +/** + * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us + * @katom: Atom to check output fence for + * + * Return: true if fence exists and is valid, otherwise false + */ +static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom) +{ + return katom->dma_fence.fence && + katom->dma_fence.fence->ops == &kbase_fence_ops; +} + +/** + * kbase_fence_out_signal() - Signal output fence of atom + * @katom: Atom to signal output fence for + * @status: Status to signal with (0 for success, < 0 for error) + * + * Return: 0 on success, < 0 on error + */ +static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, + int status) +{ + if (status) { +#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ + KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE) + fence_set_error(katom->dma_fence.fence, status); +#elif (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) + dma_fence_set_error(katom->dma_fence.fence, status); +#else + katom->dma_fence.fence->status = status; +#endif + } + return dma_fence_signal(katom->dma_fence.fence); +} + +/** + * kbase_fence_add_callback() - Add callback on @fence to block @katom + * @katom: Pointer to katom that will be blocked by @fence + * @fence: Pointer to fence on which to set up the callback + * @callback: Pointer to function to be called when fence is signaled + * + * Caller needs to hold a reference to @fence when calling this function, and + * the caller is responsible for releasing that reference. An additional + * reference to @fence will be taken when the callback was successfully set up + * and @fence needs to be kept valid until the callback has been called and + * cleanup have been done. + * + * Return: 0 on success: fence was either already signaled, or callback was + * set up. Negative error code is returned on error. + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +int kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct fence *fence, + fence_func_t callback); +#else +int kbase_fence_add_callback(struct kbase_jd_atom *katom, + struct dma_fence *fence, + dma_fence_func_t callback); +#endif + +/** + * kbase_fence_dep_count_set() - Set dep_count value on atom to specified value + * @katom: Atom to set dep_count for + * @val: value to set dep_count to + * + * The dep_count is available to the users of this module so that they can + * synchronize completion of the wait with cancellation and adding of more + * callbacks. For instance, a user could do the following: + * + * dep_count set to 1 + * callback #1 added, dep_count is increased to 2 + * callback #1 happens, dep_count decremented to 1 + * since dep_count > 0, no completion is done + * callback #2 is added, dep_count is increased to 2 + * dep_count decremented to 1 + * callback #2 happens, dep_count decremented to 0 + * since dep_count now is zero, completion executes + * + * The dep_count can also be used to make sure that the completion only + * executes once. This is typically done by setting dep_count to -1 for the + * thread that takes on this responsibility. + */ +static inline void +kbase_fence_dep_count_set(struct kbase_jd_atom *katom, int val) +{ + atomic_set(&katom->dma_fence.dep_count, val); +} + +/** + * kbase_fence_dep_count_dec_and_test() - Decrements dep_count + * @katom: Atom to decrement dep_count for + * + * See @kbase_fence_dep_count_set for general description about dep_count + * + * Return: true if value was decremented to zero, otherwise false + */ +static inline bool +kbase_fence_dep_count_dec_and_test(struct kbase_jd_atom *katom) +{ + return atomic_dec_and_test(&katom->dma_fence.dep_count); +} + +/** + * kbase_fence_dep_count_read() - Returns the current dep_count value + * @katom: Pointer to katom + * + * See @kbase_fence_dep_count_set for general description about dep_count + * + * Return: The current dep_count value + */ +static inline int kbase_fence_dep_count_read(struct kbase_jd_atom *katom) +{ + return atomic_read(&katom->dma_fence.dep_count); +} + +/** + * kbase_fence_free_callbacks() - Free dma-fence callbacks on a katom + * @katom: Pointer to katom + * + * This function will free all fence callbacks on the katom's list of + * callbacks. Callbacks that have not yet been called, because their fence + * hasn't yet signaled, will first be removed from the fence. + * + * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. + * + * Return: true if dep_count reached 0, otherwise false. + */ +bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom); + +#if defined(CONFIG_SYNC_FILE) +/** + * kbase_fence_in_get() - Retrieve input fence for atom. + * @katom: Atom to get input fence from + * + * A ref will be taken for the fence, so use @kbase_fence_put() to release it + * + * Return: The fence, or NULL if there is no input fence for atom + */ +#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in) +#endif + +/** + * kbase_fence_out_get() - Retrieve output fence for atom. + * @katom: Atom to get output fence from + * + * A ref will be taken for the fence, so use @kbase_fence_put() to release it + * + * Return: The fence, or NULL if there is no output fence for atom + */ +#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) + +/** + * kbase_fence_put() - Releases a reference to a fence + * @fence: Fence to release reference for. + */ +#define kbase_fence_put(fence) dma_fence_put(fence) + + +#endif /* CONFIG_MALI_DMA_FENCE || defined(CONFIG_SYNC_FILE */ + +#endif /* _KBASE_FENCE_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h b/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h new file mode 100755 index 000000000000..fa2c6dfe999e --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h @@ -0,0 +1,51 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_FENCE_DEFS_H_ +#define _KBASE_FENCE_DEFS_H_ + +/* + * There was a big rename in the 4.10 kernel (fence* -> dma_fence*) + * This file hides the compatibility issues with this for the rest the driver + */ + +#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + +#include + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + +#include + +#define dma_fence_context_alloc(a) fence_context_alloc(a) +#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e) +#define dma_fence_get(a) fence_get(a) +#define dma_fence_put(a) fence_put(a) +#define dma_fence_signal(a) fence_signal(a) +#define dma_fence_is_signaled(a) fence_is_signaled(a) +#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c) +#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b) + +#else + +#include + +#endif /* < 4.10.0 */ + +#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE */ + +#endif /* _KBASE_FENCE_DEFS_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator.h b/drivers/gpu/arm/midgard/mali_kbase_gator.h new file mode 100755 index 000000000000..ce65b5562a2b --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gator.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* NB taken from gator */ +/* + * List of possible actions to be controlled by DS-5 Streamline. + * The following numbers are used by gator to control the frame buffer dumping + * and s/w counter reporting. We cannot use the enums in mali_uk_types.h because + * they are unknown inside gator. + */ +#ifndef _KBASE_GATOR_H_ +#define _KBASE_GATOR_H_ + +#ifdef CONFIG_MALI_GATOR_SUPPORT +#define GATOR_MAKE_EVENT(type, number) (((type) << 24) | ((number) << 16)) +#define GATOR_JOB_SLOT_START 1 +#define GATOR_JOB_SLOT_STOP 2 +#define GATOR_JOB_SLOT_SOFT_STOPPED 3 + +void kbase_trace_mali_job_slots_event(u32 event, const struct kbase_context *kctx, u8 atom_id); +void kbase_trace_mali_pm_status(u32 event, u64 value); +void kbase_trace_mali_pm_power_off(u32 event, u64 value); +void kbase_trace_mali_pm_power_on(u32 event, u64 value); +void kbase_trace_mali_page_fault_insert_pages(int event, u32 value); +void kbase_trace_mali_mmu_as_in_use(int event); +void kbase_trace_mali_mmu_as_released(int event); +void kbase_trace_mali_total_alloc_pages_change(long long int event); + +#endif /* CONFIG_MALI_GATOR_SUPPORT */ + +#endif /* _KBASE_GATOR_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_api.c b/drivers/gpu/arm/midgard/mali_kbase_gator_api.c new file mode 100755 index 000000000000..860e10159fb3 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gator_api.c @@ -0,0 +1,334 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "mali_kbase.h" +#include "mali_kbase_hw.h" +#include "mali_kbase_mem_linux.h" +#include "mali_kbase_gator_api.h" +#include "mali_kbase_gator_hwcnt_names.h" + +#define MALI_MAX_CORES_PER_GROUP 4 +#define MALI_MAX_NUM_BLOCKS_PER_GROUP 8 +#define MALI_COUNTERS_PER_BLOCK 64 +#define MALI_BYTES_PER_COUNTER 4 + +struct kbase_gator_hwcnt_handles { + struct kbase_device *kbdev; + struct kbase_vinstr_client *vinstr_cli; + void *vinstr_buffer; + struct work_struct dump_work; + int dump_complete; + spinlock_t dump_lock; +}; + +static void dump_worker(struct work_struct *work); + +const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters) +{ + const char * const *hardware_counters; + struct kbase_device *kbdev; + uint32_t product_id; + uint32_t count; + + if (!total_counters) + return NULL; + + /* Get the first device - it doesn't matter in this case */ + kbdev = kbase_find_device(-1); + if (!kbdev) + return NULL; + + product_id = kbdev->gpu_props.props.core_props.product_id; + + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + switch (GPU_ID2_MODEL_MATCH_VALUE(product_id)) { + case GPU_ID2_PRODUCT_TMIX: + hardware_counters = hardware_counters_mali_tMIx; + count = ARRAY_SIZE(hardware_counters_mali_tMIx); + break; + case GPU_ID2_PRODUCT_THEX: + hardware_counters = hardware_counters_mali_tHEx; + count = ARRAY_SIZE(hardware_counters_mali_tHEx); + break; + case GPU_ID2_PRODUCT_TSIX: + hardware_counters = hardware_counters_mali_tSIx; + count = ARRAY_SIZE(hardware_counters_mali_tSIx); + break; + default: + hardware_counters = NULL; + count = 0; + dev_err(kbdev->dev, "Unrecognized product ID: %u\n", + product_id); + break; + } + } else { + switch (product_id) { + /* If we are using a Mali-T60x device */ + case GPU_ID_PI_T60X: + hardware_counters = hardware_counters_mali_t60x; + count = ARRAY_SIZE(hardware_counters_mali_t60x); + break; + /* If we are using a Mali-T62x device */ + case GPU_ID_PI_T62X: + hardware_counters = hardware_counters_mali_t62x; + count = ARRAY_SIZE(hardware_counters_mali_t62x); + break; + /* If we are using a Mali-T72x device */ + case GPU_ID_PI_T72X: + hardware_counters = hardware_counters_mali_t72x; + count = ARRAY_SIZE(hardware_counters_mali_t72x); + break; + /* If we are using a Mali-T76x device */ + case GPU_ID_PI_T76X: + hardware_counters = hardware_counters_mali_t76x; + count = ARRAY_SIZE(hardware_counters_mali_t76x); + break; + /* If we are using a Mali-T82x device */ + case GPU_ID_PI_T82X: + hardware_counters = hardware_counters_mali_t82x; + count = ARRAY_SIZE(hardware_counters_mali_t82x); + break; + /* If we are using a Mali-T83x device */ + case GPU_ID_PI_T83X: + hardware_counters = hardware_counters_mali_t83x; + count = ARRAY_SIZE(hardware_counters_mali_t83x); + break; + /* If we are using a Mali-T86x device */ + case GPU_ID_PI_T86X: + hardware_counters = hardware_counters_mali_t86x; + count = ARRAY_SIZE(hardware_counters_mali_t86x); + break; + /* If we are using a Mali-T88x device */ + case GPU_ID_PI_TFRX: + hardware_counters = hardware_counters_mali_t88x; + count = ARRAY_SIZE(hardware_counters_mali_t88x); + break; + default: + hardware_counters = NULL; + count = 0; + dev_err(kbdev->dev, "Unrecognized product ID: %u\n", + product_id); + break; + } + } + + /* Release the kbdev reference. */ + kbase_release_device(kbdev); + + *total_counters = count; + + /* If we return a string array take a reference on the module (or fail). */ + if (hardware_counters && !try_module_get(THIS_MODULE)) + return NULL; + + return hardware_counters; +} +KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init_names); + +void kbase_gator_hwcnt_term_names(void) +{ + /* Release the module reference. */ + module_put(THIS_MODULE); +} +KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term_names); + +struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info) +{ + struct kbase_gator_hwcnt_handles *hand; + struct kbase_uk_hwcnt_reader_setup setup; + uint32_t dump_size = 0, i = 0; + + if (!in_out_info) + return NULL; + + hand = kzalloc(sizeof(*hand), GFP_KERNEL); + if (!hand) + return NULL; + + INIT_WORK(&hand->dump_work, dump_worker); + spin_lock_init(&hand->dump_lock); + + /* Get the first device */ + hand->kbdev = kbase_find_device(-1); + if (!hand->kbdev) + goto free_hand; + + dump_size = kbase_vinstr_dump_size(hand->kbdev); + hand->vinstr_buffer = kzalloc(dump_size, GFP_KERNEL); + if (!hand->vinstr_buffer) + goto release_device; + in_out_info->kernel_dump_buffer = hand->vinstr_buffer; + + in_out_info->nr_cores = hand->kbdev->gpu_props.num_cores; + in_out_info->nr_core_groups = hand->kbdev->gpu_props.num_core_groups; + in_out_info->gpu_id = hand->kbdev->gpu_props.props.core_props.product_id; + + /* If we are using a v4 device (Mali-T6xx or Mali-T72x) */ + if (kbase_hw_has_feature(hand->kbdev, BASE_HW_FEATURE_V4)) { + uint32_t cg, j; + uint64_t core_mask; + + /* There are 8 hardware counters blocks per core group */ + in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * + MALI_MAX_NUM_BLOCKS_PER_GROUP * + in_out_info->nr_core_groups, GFP_KERNEL); + + if (!in_out_info->hwc_layout) + goto free_vinstr_buffer; + + dump_size = in_out_info->nr_core_groups * + MALI_MAX_NUM_BLOCKS_PER_GROUP * + MALI_COUNTERS_PER_BLOCK * + MALI_BYTES_PER_COUNTER; + + for (cg = 0; cg < in_out_info->nr_core_groups; cg++) { + core_mask = hand->kbdev->gpu_props.props.coherency_info.group[cg].core_mask; + + for (j = 0; j < MALI_MAX_CORES_PER_GROUP; j++) { + if (core_mask & (1u << j)) + in_out_info->hwc_layout[i++] = SHADER_BLOCK; + else + in_out_info->hwc_layout[i++] = RESERVED_BLOCK; + } + + in_out_info->hwc_layout[i++] = TILER_BLOCK; + in_out_info->hwc_layout[i++] = MMU_L2_BLOCK; + + in_out_info->hwc_layout[i++] = RESERVED_BLOCK; + + if (0 == cg) + in_out_info->hwc_layout[i++] = JM_BLOCK; + else + in_out_info->hwc_layout[i++] = RESERVED_BLOCK; + } + /* If we are using any other device */ + } else { + uint32_t nr_l2, nr_sc_bits, j; + uint64_t core_mask; + + nr_l2 = hand->kbdev->gpu_props.props.l2_props.num_l2_slices; + + core_mask = hand->kbdev->gpu_props.props.coherency_info.group[0].core_mask; + + nr_sc_bits = fls64(core_mask); + + /* The job manager and tiler sets of counters + * are always present */ + in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * (2 + nr_sc_bits + nr_l2), GFP_KERNEL); + + if (!in_out_info->hwc_layout) + goto free_vinstr_buffer; + + dump_size = (2 + nr_sc_bits + nr_l2) * MALI_COUNTERS_PER_BLOCK * MALI_BYTES_PER_COUNTER; + + in_out_info->hwc_layout[i++] = JM_BLOCK; + in_out_info->hwc_layout[i++] = TILER_BLOCK; + + for (j = 0; j < nr_l2; j++) + in_out_info->hwc_layout[i++] = MMU_L2_BLOCK; + + while (core_mask != 0ull) { + if ((core_mask & 1ull) != 0ull) + in_out_info->hwc_layout[i++] = SHADER_BLOCK; + else + in_out_info->hwc_layout[i++] = RESERVED_BLOCK; + core_mask >>= 1; + } + } + + in_out_info->nr_hwc_blocks = i; + in_out_info->size = dump_size; + + setup.jm_bm = in_out_info->bitmask[0]; + setup.tiler_bm = in_out_info->bitmask[1]; + setup.shader_bm = in_out_info->bitmask[2]; + setup.mmu_l2_bm = in_out_info->bitmask[3]; + hand->vinstr_cli = kbase_vinstr_hwcnt_kernel_setup(hand->kbdev->vinstr_ctx, + &setup, hand->vinstr_buffer); + if (!hand->vinstr_cli) { + dev_err(hand->kbdev->dev, "Failed to register gator with vinstr core"); + goto free_layout; + } + + return hand; + +free_layout: + kfree(in_out_info->hwc_layout); + +free_vinstr_buffer: + kfree(hand->vinstr_buffer); + +release_device: + kbase_release_device(hand->kbdev); + +free_hand: + kfree(hand); + return NULL; +} +KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init); + +void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles) +{ + if (in_out_info) + kfree(in_out_info->hwc_layout); + + if (opaque_handles) { + cancel_work_sync(&opaque_handles->dump_work); + kbase_vinstr_detach_client(opaque_handles->vinstr_cli); + kfree(opaque_handles->vinstr_buffer); + kbase_release_device(opaque_handles->kbdev); + kfree(opaque_handles); + } +} +KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term); + +static void dump_worker(struct work_struct *work) +{ + struct kbase_gator_hwcnt_handles *hand; + + hand = container_of(work, struct kbase_gator_hwcnt_handles, dump_work); + if (!kbase_vinstr_hwc_dump(hand->vinstr_cli, + BASE_HWCNT_READER_EVENT_MANUAL)) { + spin_lock_bh(&hand->dump_lock); + hand->dump_complete = 1; + spin_unlock_bh(&hand->dump_lock); + } else { + schedule_work(&hand->dump_work); + } +} + +uint32_t kbase_gator_instr_hwcnt_dump_complete( + struct kbase_gator_hwcnt_handles *opaque_handles, + uint32_t * const success) +{ + + if (opaque_handles && success) { + *success = opaque_handles->dump_complete; + opaque_handles->dump_complete = 0; + return *success; + } + return 0; +} +KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_complete); + +uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles) +{ + if (opaque_handles) + schedule_work(&opaque_handles->dump_work); + return 0; +} +KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_irq); diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_api.h b/drivers/gpu/arm/midgard/mali_kbase_gator_api.h new file mode 100755 index 000000000000..ef9ac0f7b633 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gator_api.h @@ -0,0 +1,219 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_GATOR_API_H_ +#define _KBASE_GATOR_API_H_ + +/** + * @brief This file describes the API used by Gator to fetch hardware counters. + */ + +/* This define is used by the gator kernel module compile to select which DDK + * API calling convention to use. If not defined (legacy DDK) gator assumes + * version 1. The version to DDK release mapping is: + * Version 1 API: DDK versions r1px, r2px + * Version 2 API: DDK versions r3px, r4px + * Version 3 API: DDK version r5p0 and newer + * + * API Usage + * ========= + * + * 1] Call kbase_gator_hwcnt_init_names() to return the list of short counter + * names for the GPU present in this device. + * + * 2] Create a kbase_gator_hwcnt_info structure and set the counter enables for + * the counters you want enabled. The enables can all be set for simplicity in + * most use cases, but disabling some will let you minimize bandwidth impact. + * + * 3] Call kbase_gator_hwcnt_init() using the above structure, to create a + * counter context. On successful return the DDK will have populated the + * structure with a variety of useful information. + * + * 4] Call kbase_gator_hwcnt_dump_irq() to queue a non-blocking request for a + * counter dump. If this returns a non-zero value the request has been queued, + * otherwise the driver has been unable to do so (typically because of another + * user of the instrumentation exists concurrently). + * + * 5] Call kbase_gator_hwcnt_dump_complete() to test whether the previously + * requested dump has been succesful. If this returns non-zero the counter dump + * has resolved, but the value of *success must also be tested as the dump + * may have not been successful. If it returns zero the counter dump was + * abandoned due to the device being busy (typically because of another + * user of the instrumentation exists concurrently). + * + * 6] Process the counters stored in the buffer pointed to by ... + * + * kbase_gator_hwcnt_info->kernel_dump_buffer + * + * In pseudo code you can find all of the counters via this approach: + * + * + * hwcnt_info # pointer to kbase_gator_hwcnt_info structure + * hwcnt_name # pointer to name list + * + * u32 * hwcnt_data = (u32*)hwcnt_info->kernel_dump_buffer + * + * # Iterate over each 64-counter block in this GPU configuration + * for( i = 0; i < hwcnt_info->nr_hwc_blocks; i++) { + * hwc_type type = hwcnt_info->hwc_layout[i]; + * + * # Skip reserved type blocks - they contain no counters at all + * if( type == RESERVED_BLOCK ) { + * continue; + * } + * + * size_t name_offset = type * 64; + * size_t data_offset = i * 64; + * + * # Iterate over the names of the counters in this block type + * for( j = 0; j < 64; j++) { + * const char * name = hwcnt_name[name_offset+j]; + * + * # Skip empty name strings - there is no counter here + * if( name[0] == '\0' ) { + * continue; + * } + * + * u32 data = hwcnt_data[data_offset+j]; + * + * printk( "COUNTER: %s DATA: %u\n", name, data ); + * } + * } + * + * + * Note that in most implementations you typically want to either SUM or + * AVERAGE multiple instances of the same counter if, for example, you have + * multiple shader cores or multiple L2 caches. The most sensible view for + * analysis is to AVERAGE shader core counters, but SUM L2 cache and MMU + * counters. + * + * 7] Goto 4, repeating until you want to stop collecting counters. + * + * 8] Release the dump resources by calling kbase_gator_hwcnt_term(). + * + * 9] Release the name table resources by calling + * kbase_gator_hwcnt_term_names(). This function must only be called if + * init_names() returned a non-NULL value. + **/ + +#define MALI_DDK_GATOR_API_VERSION 3 + +enum hwc_type { + JM_BLOCK = 0, + TILER_BLOCK, + SHADER_BLOCK, + MMU_L2_BLOCK, + RESERVED_BLOCK +}; + +struct kbase_gator_hwcnt_info { + /* Passed from Gator to kbase */ + + /* the bitmask of enabled hardware counters for each counter block */ + uint16_t bitmask[4]; + + /* Passed from kbase to Gator */ + + /* ptr to counter dump memory */ + void *kernel_dump_buffer; + + /* size of counter dump memory */ + uint32_t size; + + /* the ID of the Mali device */ + uint32_t gpu_id; + + /* the number of shader cores in the GPU */ + uint32_t nr_cores; + + /* the number of core groups */ + uint32_t nr_core_groups; + + /* the memory layout of the performance counters */ + enum hwc_type *hwc_layout; + + /* the total number of hardware couter blocks */ + uint32_t nr_hwc_blocks; +}; + +/** + * @brief Opaque block of Mali data which Gator needs to return to the API later. + */ +struct kbase_gator_hwcnt_handles; + +/** + * @brief Initialize the resources Gator needs for performance profiling. + * + * @param in_out_info A pointer to a structure containing the enabled counters passed from Gator and all the Mali + * specific information that will be returned to Gator. On entry Gator must have populated the + * 'bitmask' field with the counters it wishes to enable for each class of counter block. + * Each entry in the array corresponds to a single counter class based on the "hwc_type" + * enumeration, and each bit corresponds to an enable for 4 sequential counters (LSB enables + * the first 4 counters in the block, and so on). See the GPU counter array as returned by + * kbase_gator_hwcnt_get_names() for the index values of each counter for the curernt GPU. + * + * @return Pointer to an opaque handle block on success, NULL on error. + */ +extern struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info); + +/** + * @brief Free all resources once Gator has finished using performance counters. + * + * @param in_out_info A pointer to a structure containing the enabled counters passed from Gator and all the + * Mali specific information that will be returned to Gator. + * @param opaque_handles A wrapper structure for kbase structures. + */ +extern void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles); + +/** + * @brief Poll whether a counter dump is successful. + * + * @param opaque_handles A wrapper structure for kbase structures. + * @param[out] success Non-zero on success, zero on failure. + * + * @return Zero if the dump is still pending, non-zero if the dump has completed. Note that a + * completed dump may not have dumped succesfully, so the caller must test for both + * a completed and successful dump before processing counters. + */ +extern uint32_t kbase_gator_instr_hwcnt_dump_complete(struct kbase_gator_hwcnt_handles *opaque_handles, uint32_t * const success); + +/** + * @brief Request the generation of a new counter dump. + * + * @param opaque_handles A wrapper structure for kbase structures. + * + * @return Zero if the hardware device is busy and cannot handle the request, non-zero otherwise. + */ +extern uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles); + +/** + * @brief This function is used to fetch the names table based on the Mali device in use. + * + * @param[out] total_counters The total number of counters short names in the Mali devices' list. + * + * @return Pointer to an array of strings of length *total_counters. + */ +extern const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters); + +/** + * @brief This function is used to terminate the use of the names table. + * + * This function must only be called if the initial call to kbase_gator_hwcnt_init_names returned a non-NULL value. + */ +extern void kbase_gator_hwcnt_term_names(void); + +#endif diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names.h new file mode 100755 index 000000000000..cad19b66200d --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names.h @@ -0,0 +1,2170 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_GATOR_HWCNT_NAMES_H_ +#define _KBASE_GATOR_HWCNT_NAMES_H_ + +/* + * "Short names" for hardware counters used by Streamline. Counters names are + * stored in accordance with their memory layout in the binary counter block + * emitted by the Mali GPU. Each "master" in the GPU emits a fixed-size block + * of 64 counters, and each GPU implements the same set of "masters" although + * the counters each master exposes within its block of 64 may vary. + * + * Counters which are an empty string are simply "holes" in the counter memory + * where no counter exists. + */ + +static const char * const hardware_counters_mali_t60x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T60x_MESSAGES_SENT", + "T60x_MESSAGES_RECEIVED", + "T60x_GPU_ACTIVE", + "T60x_IRQ_ACTIVE", + "T60x_JS0_JOBS", + "T60x_JS0_TASKS", + "T60x_JS0_ACTIVE", + "", + "T60x_JS0_WAIT_READ", + "T60x_JS0_WAIT_ISSUE", + "T60x_JS0_WAIT_DEPEND", + "T60x_JS0_WAIT_FINISH", + "T60x_JS1_JOBS", + "T60x_JS1_TASKS", + "T60x_JS1_ACTIVE", + "", + "T60x_JS1_WAIT_READ", + "T60x_JS1_WAIT_ISSUE", + "T60x_JS1_WAIT_DEPEND", + "T60x_JS1_WAIT_FINISH", + "T60x_JS2_JOBS", + "T60x_JS2_TASKS", + "T60x_JS2_ACTIVE", + "", + "T60x_JS2_WAIT_READ", + "T60x_JS2_WAIT_ISSUE", + "T60x_JS2_WAIT_DEPEND", + "T60x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T60x_TI_JOBS_PROCESSED", + "T60x_TI_TRIANGLES", + "T60x_TI_QUADS", + "T60x_TI_POLYGONS", + "T60x_TI_POINTS", + "T60x_TI_LINES", + "T60x_TI_VCACHE_HIT", + "T60x_TI_VCACHE_MISS", + "T60x_TI_FRONT_FACING", + "T60x_TI_BACK_FACING", + "T60x_TI_PRIM_VISIBLE", + "T60x_TI_PRIM_CULLED", + "T60x_TI_PRIM_CLIPPED", + "T60x_TI_LEVEL0", + "T60x_TI_LEVEL1", + "T60x_TI_LEVEL2", + "T60x_TI_LEVEL3", + "T60x_TI_LEVEL4", + "T60x_TI_LEVEL5", + "T60x_TI_LEVEL6", + "T60x_TI_LEVEL7", + "T60x_TI_COMMAND_1", + "T60x_TI_COMMAND_2", + "T60x_TI_COMMAND_3", + "T60x_TI_COMMAND_4", + "T60x_TI_COMMAND_4_7", + "T60x_TI_COMMAND_8_15", + "T60x_TI_COMMAND_16_63", + "T60x_TI_COMMAND_64", + "T60x_TI_COMPRESS_IN", + "T60x_TI_COMPRESS_OUT", + "T60x_TI_COMPRESS_FLUSH", + "T60x_TI_TIMESTAMPS", + "T60x_TI_PCACHE_HIT", + "T60x_TI_PCACHE_MISS", + "T60x_TI_PCACHE_LINE", + "T60x_TI_PCACHE_STALL", + "T60x_TI_WRBUF_HIT", + "T60x_TI_WRBUF_MISS", + "T60x_TI_WRBUF_LINE", + "T60x_TI_WRBUF_PARTIAL", + "T60x_TI_WRBUF_STALL", + "T60x_TI_ACTIVE", + "T60x_TI_LOADING_DESC", + "T60x_TI_INDEX_WAIT", + "T60x_TI_INDEX_RANGE_WAIT", + "T60x_TI_VERTEX_WAIT", + "T60x_TI_PCACHE_WAIT", + "T60x_TI_WRBUF_WAIT", + "T60x_TI_BUS_READ", + "T60x_TI_BUS_WRITE", + "", + "", + "", + "", + "", + "T60x_TI_UTLB_STALL", + "T60x_TI_UTLB_REPLAY_MISS", + "T60x_TI_UTLB_REPLAY_FULL", + "T60x_TI_UTLB_NEW_MISS", + "T60x_TI_UTLB_HIT", + + /* Shader Core */ + "", + "", + "", + "", + "T60x_FRAG_ACTIVE", + "T60x_FRAG_PRIMITIVES", + "T60x_FRAG_PRIMITIVES_DROPPED", + "T60x_FRAG_CYCLES_DESC", + "T60x_FRAG_CYCLES_PLR", + "T60x_FRAG_CYCLES_VERT", + "T60x_FRAG_CYCLES_TRISETUP", + "T60x_FRAG_CYCLES_RAST", + "T60x_FRAG_THREADS", + "T60x_FRAG_DUMMY_THREADS", + "T60x_FRAG_QUADS_RAST", + "T60x_FRAG_QUADS_EZS_TEST", + "T60x_FRAG_QUADS_EZS_KILLED", + "T60x_FRAG_THREADS_LZS_TEST", + "T60x_FRAG_THREADS_LZS_KILLED", + "T60x_FRAG_CYCLES_NO_TILE", + "T60x_FRAG_NUM_TILES", + "T60x_FRAG_TRANS_ELIM", + "T60x_COMPUTE_ACTIVE", + "T60x_COMPUTE_TASKS", + "T60x_COMPUTE_THREADS", + "T60x_COMPUTE_CYCLES_DESC", + "T60x_TRIPIPE_ACTIVE", + "T60x_ARITH_WORDS", + "T60x_ARITH_CYCLES_REG", + "T60x_ARITH_CYCLES_L0", + "T60x_ARITH_FRAG_DEPEND", + "T60x_LS_WORDS", + "T60x_LS_ISSUES", + "T60x_LS_RESTARTS", + "T60x_LS_REISSUES_MISS", + "T60x_LS_REISSUES_VD", + "T60x_LS_REISSUE_ATTRIB_MISS", + "T60x_LS_NO_WB", + "T60x_TEX_WORDS", + "T60x_TEX_BUBBLES", + "T60x_TEX_WORDS_L0", + "T60x_TEX_WORDS_DESC", + "T60x_TEX_ISSUES", + "T60x_TEX_RECIRC_FMISS", + "T60x_TEX_RECIRC_DESC", + "T60x_TEX_RECIRC_MULTI", + "T60x_TEX_RECIRC_PMISS", + "T60x_TEX_RECIRC_CONF", + "T60x_LSC_READ_HITS", + "T60x_LSC_READ_MISSES", + "T60x_LSC_WRITE_HITS", + "T60x_LSC_WRITE_MISSES", + "T60x_LSC_ATOMIC_HITS", + "T60x_LSC_ATOMIC_MISSES", + "T60x_LSC_LINE_FETCHES", + "T60x_LSC_DIRTY_LINE", + "T60x_LSC_SNOOPS", + "T60x_AXI_TLB_STALL", + "T60x_AXI_TLB_MISS", + "T60x_AXI_TLB_TRANSACTION", + "T60x_LS_TLB_MISS", + "T60x_LS_TLB_HIT", + "T60x_AXI_BEATS_READ", + "T60x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T60x_MMU_HIT", + "T60x_MMU_NEW_MISS", + "T60x_MMU_REPLAY_FULL", + "T60x_MMU_REPLAY_MISS", + "T60x_MMU_TABLE_WALK", + "", + "", + "", + "", + "", + "", + "", + "T60x_UTLB_HIT", + "T60x_UTLB_NEW_MISS", + "T60x_UTLB_REPLAY_FULL", + "T60x_UTLB_REPLAY_MISS", + "T60x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T60x_L2_EXT_WRITE_BEATS", + "T60x_L2_EXT_READ_BEATS", + "T60x_L2_ANY_LOOKUP", + "T60x_L2_READ_LOOKUP", + "T60x_L2_SREAD_LOOKUP", + "T60x_L2_READ_REPLAY", + "T60x_L2_READ_SNOOP", + "T60x_L2_READ_HIT", + "T60x_L2_CLEAN_MISS", + "T60x_L2_WRITE_LOOKUP", + "T60x_L2_SWRITE_LOOKUP", + "T60x_L2_WRITE_REPLAY", + "T60x_L2_WRITE_SNOOP", + "T60x_L2_WRITE_HIT", + "T60x_L2_EXT_READ_FULL", + "T60x_L2_EXT_READ_HALF", + "T60x_L2_EXT_WRITE_FULL", + "T60x_L2_EXT_WRITE_HALF", + "T60x_L2_EXT_READ", + "T60x_L2_EXT_READ_LINE", + "T60x_L2_EXT_WRITE", + "T60x_L2_EXT_WRITE_LINE", + "T60x_L2_EXT_WRITE_SMALL", + "T60x_L2_EXT_BARRIER", + "T60x_L2_EXT_AR_STALL", + "T60x_L2_EXT_R_BUF_FULL", + "T60x_L2_EXT_RD_BUF_FULL", + "T60x_L2_EXT_R_RAW", + "T60x_L2_EXT_W_STALL", + "T60x_L2_EXT_W_BUF_FULL", + "T60x_L2_EXT_R_W_HAZARD", + "T60x_L2_TAG_HAZARD", + "T60x_L2_SNOOP_FULL", + "T60x_L2_REPLAY_FULL" +}; +static const char * const hardware_counters_mali_t62x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T62x_MESSAGES_SENT", + "T62x_MESSAGES_RECEIVED", + "T62x_GPU_ACTIVE", + "T62x_IRQ_ACTIVE", + "T62x_JS0_JOBS", + "T62x_JS0_TASKS", + "T62x_JS0_ACTIVE", + "", + "T62x_JS0_WAIT_READ", + "T62x_JS0_WAIT_ISSUE", + "T62x_JS0_WAIT_DEPEND", + "T62x_JS0_WAIT_FINISH", + "T62x_JS1_JOBS", + "T62x_JS1_TASKS", + "T62x_JS1_ACTIVE", + "", + "T62x_JS1_WAIT_READ", + "T62x_JS1_WAIT_ISSUE", + "T62x_JS1_WAIT_DEPEND", + "T62x_JS1_WAIT_FINISH", + "T62x_JS2_JOBS", + "T62x_JS2_TASKS", + "T62x_JS2_ACTIVE", + "", + "T62x_JS2_WAIT_READ", + "T62x_JS2_WAIT_ISSUE", + "T62x_JS2_WAIT_DEPEND", + "T62x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T62x_TI_JOBS_PROCESSED", + "T62x_TI_TRIANGLES", + "T62x_TI_QUADS", + "T62x_TI_POLYGONS", + "T62x_TI_POINTS", + "T62x_TI_LINES", + "T62x_TI_VCACHE_HIT", + "T62x_TI_VCACHE_MISS", + "T62x_TI_FRONT_FACING", + "T62x_TI_BACK_FACING", + "T62x_TI_PRIM_VISIBLE", + "T62x_TI_PRIM_CULLED", + "T62x_TI_PRIM_CLIPPED", + "T62x_TI_LEVEL0", + "T62x_TI_LEVEL1", + "T62x_TI_LEVEL2", + "T62x_TI_LEVEL3", + "T62x_TI_LEVEL4", + "T62x_TI_LEVEL5", + "T62x_TI_LEVEL6", + "T62x_TI_LEVEL7", + "T62x_TI_COMMAND_1", + "T62x_TI_COMMAND_2", + "T62x_TI_COMMAND_3", + "T62x_TI_COMMAND_4", + "T62x_TI_COMMAND_5_7", + "T62x_TI_COMMAND_8_15", + "T62x_TI_COMMAND_16_63", + "T62x_TI_COMMAND_64", + "T62x_TI_COMPRESS_IN", + "T62x_TI_COMPRESS_OUT", + "T62x_TI_COMPRESS_FLUSH", + "T62x_TI_TIMESTAMPS", + "T62x_TI_PCACHE_HIT", + "T62x_TI_PCACHE_MISS", + "T62x_TI_PCACHE_LINE", + "T62x_TI_PCACHE_STALL", + "T62x_TI_WRBUF_HIT", + "T62x_TI_WRBUF_MISS", + "T62x_TI_WRBUF_LINE", + "T62x_TI_WRBUF_PARTIAL", + "T62x_TI_WRBUF_STALL", + "T62x_TI_ACTIVE", + "T62x_TI_LOADING_DESC", + "T62x_TI_INDEX_WAIT", + "T62x_TI_INDEX_RANGE_WAIT", + "T62x_TI_VERTEX_WAIT", + "T62x_TI_PCACHE_WAIT", + "T62x_TI_WRBUF_WAIT", + "T62x_TI_BUS_READ", + "T62x_TI_BUS_WRITE", + "", + "", + "", + "", + "", + "T62x_TI_UTLB_STALL", + "T62x_TI_UTLB_REPLAY_MISS", + "T62x_TI_UTLB_REPLAY_FULL", + "T62x_TI_UTLB_NEW_MISS", + "T62x_TI_UTLB_HIT", + + /* Shader Core */ + "", + "", + "", + "T62x_SHADER_CORE_ACTIVE", + "T62x_FRAG_ACTIVE", + "T62x_FRAG_PRIMITIVES", + "T62x_FRAG_PRIMITIVES_DROPPED", + "T62x_FRAG_CYCLES_DESC", + "T62x_FRAG_CYCLES_FPKQ_ACTIVE", + "T62x_FRAG_CYCLES_VERT", + "T62x_FRAG_CYCLES_TRISETUP", + "T62x_FRAG_CYCLES_EZS_ACTIVE", + "T62x_FRAG_THREADS", + "T62x_FRAG_DUMMY_THREADS", + "T62x_FRAG_QUADS_RAST", + "T62x_FRAG_QUADS_EZS_TEST", + "T62x_FRAG_QUADS_EZS_KILLED", + "T62x_FRAG_THREADS_LZS_TEST", + "T62x_FRAG_THREADS_LZS_KILLED", + "T62x_FRAG_CYCLES_NO_TILE", + "T62x_FRAG_NUM_TILES", + "T62x_FRAG_TRANS_ELIM", + "T62x_COMPUTE_ACTIVE", + "T62x_COMPUTE_TASKS", + "T62x_COMPUTE_THREADS", + "T62x_COMPUTE_CYCLES_DESC", + "T62x_TRIPIPE_ACTIVE", + "T62x_ARITH_WORDS", + "T62x_ARITH_CYCLES_REG", + "T62x_ARITH_CYCLES_L0", + "T62x_ARITH_FRAG_DEPEND", + "T62x_LS_WORDS", + "T62x_LS_ISSUES", + "T62x_LS_RESTARTS", + "T62x_LS_REISSUES_MISS", + "T62x_LS_REISSUES_VD", + "T62x_LS_REISSUE_ATTRIB_MISS", + "T62x_LS_NO_WB", + "T62x_TEX_WORDS", + "T62x_TEX_BUBBLES", + "T62x_TEX_WORDS_L0", + "T62x_TEX_WORDS_DESC", + "T62x_TEX_ISSUES", + "T62x_TEX_RECIRC_FMISS", + "T62x_TEX_RECIRC_DESC", + "T62x_TEX_RECIRC_MULTI", + "T62x_TEX_RECIRC_PMISS", + "T62x_TEX_RECIRC_CONF", + "T62x_LSC_READ_HITS", + "T62x_LSC_READ_MISSES", + "T62x_LSC_WRITE_HITS", + "T62x_LSC_WRITE_MISSES", + "T62x_LSC_ATOMIC_HITS", + "T62x_LSC_ATOMIC_MISSES", + "T62x_LSC_LINE_FETCHES", + "T62x_LSC_DIRTY_LINE", + "T62x_LSC_SNOOPS", + "T62x_AXI_TLB_STALL", + "T62x_AXI_TLB_MISS", + "T62x_AXI_TLB_TRANSACTION", + "T62x_LS_TLB_MISS", + "T62x_LS_TLB_HIT", + "T62x_AXI_BEATS_READ", + "T62x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T62x_MMU_HIT", + "T62x_MMU_NEW_MISS", + "T62x_MMU_REPLAY_FULL", + "T62x_MMU_REPLAY_MISS", + "T62x_MMU_TABLE_WALK", + "", + "", + "", + "", + "", + "", + "", + "T62x_UTLB_HIT", + "T62x_UTLB_NEW_MISS", + "T62x_UTLB_REPLAY_FULL", + "T62x_UTLB_REPLAY_MISS", + "T62x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T62x_L2_EXT_WRITE_BEATS", + "T62x_L2_EXT_READ_BEATS", + "T62x_L2_ANY_LOOKUP", + "T62x_L2_READ_LOOKUP", + "T62x_L2_SREAD_LOOKUP", + "T62x_L2_READ_REPLAY", + "T62x_L2_READ_SNOOP", + "T62x_L2_READ_HIT", + "T62x_L2_CLEAN_MISS", + "T62x_L2_WRITE_LOOKUP", + "T62x_L2_SWRITE_LOOKUP", + "T62x_L2_WRITE_REPLAY", + "T62x_L2_WRITE_SNOOP", + "T62x_L2_WRITE_HIT", + "T62x_L2_EXT_READ_FULL", + "T62x_L2_EXT_READ_HALF", + "T62x_L2_EXT_WRITE_FULL", + "T62x_L2_EXT_WRITE_HALF", + "T62x_L2_EXT_READ", + "T62x_L2_EXT_READ_LINE", + "T62x_L2_EXT_WRITE", + "T62x_L2_EXT_WRITE_LINE", + "T62x_L2_EXT_WRITE_SMALL", + "T62x_L2_EXT_BARRIER", + "T62x_L2_EXT_AR_STALL", + "T62x_L2_EXT_R_BUF_FULL", + "T62x_L2_EXT_RD_BUF_FULL", + "T62x_L2_EXT_R_RAW", + "T62x_L2_EXT_W_STALL", + "T62x_L2_EXT_W_BUF_FULL", + "T62x_L2_EXT_R_W_HAZARD", + "T62x_L2_TAG_HAZARD", + "T62x_L2_SNOOP_FULL", + "T62x_L2_REPLAY_FULL" +}; + +static const char * const hardware_counters_mali_t72x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T72x_GPU_ACTIVE", + "T72x_IRQ_ACTIVE", + "T72x_JS0_JOBS", + "T72x_JS0_TASKS", + "T72x_JS0_ACTIVE", + "T72x_JS1_JOBS", + "T72x_JS1_TASKS", + "T72x_JS1_ACTIVE", + "T72x_JS2_JOBS", + "T72x_JS2_TASKS", + "T72x_JS2_ACTIVE", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T72x_TI_JOBS_PROCESSED", + "T72x_TI_TRIANGLES", + "T72x_TI_QUADS", + "T72x_TI_POLYGONS", + "T72x_TI_POINTS", + "T72x_TI_LINES", + "T72x_TI_FRONT_FACING", + "T72x_TI_BACK_FACING", + "T72x_TI_PRIM_VISIBLE", + "T72x_TI_PRIM_CULLED", + "T72x_TI_PRIM_CLIPPED", + "", + "", + "", + "", + "", + "", + "", + "", + "T72x_TI_ACTIVE", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Shader Core */ + "", + "", + "", + "", + "T72x_FRAG_ACTIVE", + "T72x_FRAG_PRIMITIVES", + "T72x_FRAG_PRIMITIVES_DROPPED", + "T72x_FRAG_THREADS", + "T72x_FRAG_DUMMY_THREADS", + "T72x_FRAG_QUADS_RAST", + "T72x_FRAG_QUADS_EZS_TEST", + "T72x_FRAG_QUADS_EZS_KILLED", + "T72x_FRAG_THREADS_LZS_TEST", + "T72x_FRAG_THREADS_LZS_KILLED", + "T72x_FRAG_CYCLES_NO_TILE", + "T72x_FRAG_NUM_TILES", + "T72x_FRAG_TRANS_ELIM", + "T72x_COMPUTE_ACTIVE", + "T72x_COMPUTE_TASKS", + "T72x_COMPUTE_THREADS", + "T72x_TRIPIPE_ACTIVE", + "T72x_ARITH_WORDS", + "T72x_ARITH_CYCLES_REG", + "T72x_LS_WORDS", + "T72x_LS_ISSUES", + "T72x_LS_RESTARTS", + "T72x_LS_REISSUES_MISS", + "T72x_TEX_WORDS", + "T72x_TEX_BUBBLES", + "T72x_TEX_ISSUES", + "T72x_LSC_READ_HITS", + "T72x_LSC_READ_MISSES", + "T72x_LSC_WRITE_HITS", + "T72x_LSC_WRITE_MISSES", + "T72x_LSC_ATOMIC_HITS", + "T72x_LSC_ATOMIC_MISSES", + "T72x_LSC_LINE_FETCHES", + "T72x_LSC_DIRTY_LINE", + "T72x_LSC_SNOOPS", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*L2 and MMU */ + "", + "", + "", + "", + "T72x_L2_EXT_WRITE_BEAT", + "T72x_L2_EXT_READ_BEAT", + "T72x_L2_READ_SNOOP", + "T72x_L2_READ_HIT", + "T72x_L2_WRITE_SNOOP", + "T72x_L2_WRITE_HIT", + "T72x_L2_EXT_WRITE_SMALL", + "T72x_L2_EXT_BARRIER", + "T72x_L2_EXT_AR_STALL", + "T72x_L2_EXT_W_STALL", + "T72x_L2_SNOOP_FULL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" +}; + +static const char * const hardware_counters_mali_t76x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T76x_MESSAGES_SENT", + "T76x_MESSAGES_RECEIVED", + "T76x_GPU_ACTIVE", + "T76x_IRQ_ACTIVE", + "T76x_JS0_JOBS", + "T76x_JS0_TASKS", + "T76x_JS0_ACTIVE", + "", + "T76x_JS0_WAIT_READ", + "T76x_JS0_WAIT_ISSUE", + "T76x_JS0_WAIT_DEPEND", + "T76x_JS0_WAIT_FINISH", + "T76x_JS1_JOBS", + "T76x_JS1_TASKS", + "T76x_JS1_ACTIVE", + "", + "T76x_JS1_WAIT_READ", + "T76x_JS1_WAIT_ISSUE", + "T76x_JS1_WAIT_DEPEND", + "T76x_JS1_WAIT_FINISH", + "T76x_JS2_JOBS", + "T76x_JS2_TASKS", + "T76x_JS2_ACTIVE", + "", + "T76x_JS2_WAIT_READ", + "T76x_JS2_WAIT_ISSUE", + "T76x_JS2_WAIT_DEPEND", + "T76x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T76x_TI_JOBS_PROCESSED", + "T76x_TI_TRIANGLES", + "T76x_TI_QUADS", + "T76x_TI_POLYGONS", + "T76x_TI_POINTS", + "T76x_TI_LINES", + "T76x_TI_VCACHE_HIT", + "T76x_TI_VCACHE_MISS", + "T76x_TI_FRONT_FACING", + "T76x_TI_BACK_FACING", + "T76x_TI_PRIM_VISIBLE", + "T76x_TI_PRIM_CULLED", + "T76x_TI_PRIM_CLIPPED", + "T76x_TI_LEVEL0", + "T76x_TI_LEVEL1", + "T76x_TI_LEVEL2", + "T76x_TI_LEVEL3", + "T76x_TI_LEVEL4", + "T76x_TI_LEVEL5", + "T76x_TI_LEVEL6", + "T76x_TI_LEVEL7", + "T76x_TI_COMMAND_1", + "T76x_TI_COMMAND_2", + "T76x_TI_COMMAND_3", + "T76x_TI_COMMAND_4", + "T76x_TI_COMMAND_5_7", + "T76x_TI_COMMAND_8_15", + "T76x_TI_COMMAND_16_63", + "T76x_TI_COMMAND_64", + "T76x_TI_COMPRESS_IN", + "T76x_TI_COMPRESS_OUT", + "T76x_TI_COMPRESS_FLUSH", + "T76x_TI_TIMESTAMPS", + "T76x_TI_PCACHE_HIT", + "T76x_TI_PCACHE_MISS", + "T76x_TI_PCACHE_LINE", + "T76x_TI_PCACHE_STALL", + "T76x_TI_WRBUF_HIT", + "T76x_TI_WRBUF_MISS", + "T76x_TI_WRBUF_LINE", + "T76x_TI_WRBUF_PARTIAL", + "T76x_TI_WRBUF_STALL", + "T76x_TI_ACTIVE", + "T76x_TI_LOADING_DESC", + "T76x_TI_INDEX_WAIT", + "T76x_TI_INDEX_RANGE_WAIT", + "T76x_TI_VERTEX_WAIT", + "T76x_TI_PCACHE_WAIT", + "T76x_TI_WRBUF_WAIT", + "T76x_TI_BUS_READ", + "T76x_TI_BUS_WRITE", + "", + "", + "", + "", + "", + "T76x_TI_UTLB_HIT", + "T76x_TI_UTLB_NEW_MISS", + "T76x_TI_UTLB_REPLAY_FULL", + "T76x_TI_UTLB_REPLAY_MISS", + "T76x_TI_UTLB_STALL", + + /* Shader Core */ + "", + "", + "", + "", + "T76x_FRAG_ACTIVE", + "T76x_FRAG_PRIMITIVES", + "T76x_FRAG_PRIMITIVES_DROPPED", + "T76x_FRAG_CYCLES_DESC", + "T76x_FRAG_CYCLES_FPKQ_ACTIVE", + "T76x_FRAG_CYCLES_VERT", + "T76x_FRAG_CYCLES_TRISETUP", + "T76x_FRAG_CYCLES_EZS_ACTIVE", + "T76x_FRAG_THREADS", + "T76x_FRAG_DUMMY_THREADS", + "T76x_FRAG_QUADS_RAST", + "T76x_FRAG_QUADS_EZS_TEST", + "T76x_FRAG_QUADS_EZS_KILLED", + "T76x_FRAG_THREADS_LZS_TEST", + "T76x_FRAG_THREADS_LZS_KILLED", + "T76x_FRAG_CYCLES_NO_TILE", + "T76x_FRAG_NUM_TILES", + "T76x_FRAG_TRANS_ELIM", + "T76x_COMPUTE_ACTIVE", + "T76x_COMPUTE_TASKS", + "T76x_COMPUTE_THREADS", + "T76x_COMPUTE_CYCLES_DESC", + "T76x_TRIPIPE_ACTIVE", + "T76x_ARITH_WORDS", + "T76x_ARITH_CYCLES_REG", + "T76x_ARITH_CYCLES_L0", + "T76x_ARITH_FRAG_DEPEND", + "T76x_LS_WORDS", + "T76x_LS_ISSUES", + "T76x_LS_REISSUE_ATTR", + "T76x_LS_REISSUES_VARY", + "T76x_LS_VARY_RV_MISS", + "T76x_LS_VARY_RV_HIT", + "T76x_LS_NO_UNPARK", + "T76x_TEX_WORDS", + "T76x_TEX_BUBBLES", + "T76x_TEX_WORDS_L0", + "T76x_TEX_WORDS_DESC", + "T76x_TEX_ISSUES", + "T76x_TEX_RECIRC_FMISS", + "T76x_TEX_RECIRC_DESC", + "T76x_TEX_RECIRC_MULTI", + "T76x_TEX_RECIRC_PMISS", + "T76x_TEX_RECIRC_CONF", + "T76x_LSC_READ_HITS", + "T76x_LSC_READ_OP", + "T76x_LSC_WRITE_HITS", + "T76x_LSC_WRITE_OP", + "T76x_LSC_ATOMIC_HITS", + "T76x_LSC_ATOMIC_OP", + "T76x_LSC_LINE_FETCHES", + "T76x_LSC_DIRTY_LINE", + "T76x_LSC_SNOOPS", + "T76x_AXI_TLB_STALL", + "T76x_AXI_TLB_MISS", + "T76x_AXI_TLB_TRANSACTION", + "T76x_LS_TLB_MISS", + "T76x_LS_TLB_HIT", + "T76x_AXI_BEATS_READ", + "T76x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T76x_MMU_HIT", + "T76x_MMU_NEW_MISS", + "T76x_MMU_REPLAY_FULL", + "T76x_MMU_REPLAY_MISS", + "T76x_MMU_TABLE_WALK", + "T76x_MMU_REQUESTS", + "", + "", + "T76x_UTLB_HIT", + "T76x_UTLB_NEW_MISS", + "T76x_UTLB_REPLAY_FULL", + "T76x_UTLB_REPLAY_MISS", + "T76x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T76x_L2_EXT_WRITE_BEATS", + "T76x_L2_EXT_READ_BEATS", + "T76x_L2_ANY_LOOKUP", + "T76x_L2_READ_LOOKUP", + "T76x_L2_SREAD_LOOKUP", + "T76x_L2_READ_REPLAY", + "T76x_L2_READ_SNOOP", + "T76x_L2_READ_HIT", + "T76x_L2_CLEAN_MISS", + "T76x_L2_WRITE_LOOKUP", + "T76x_L2_SWRITE_LOOKUP", + "T76x_L2_WRITE_REPLAY", + "T76x_L2_WRITE_SNOOP", + "T76x_L2_WRITE_HIT", + "T76x_L2_EXT_READ_FULL", + "", + "T76x_L2_EXT_WRITE_FULL", + "T76x_L2_EXT_R_W_HAZARD", + "T76x_L2_EXT_READ", + "T76x_L2_EXT_READ_LINE", + "T76x_L2_EXT_WRITE", + "T76x_L2_EXT_WRITE_LINE", + "T76x_L2_EXT_WRITE_SMALL", + "T76x_L2_EXT_BARRIER", + "T76x_L2_EXT_AR_STALL", + "T76x_L2_EXT_R_BUF_FULL", + "T76x_L2_EXT_RD_BUF_FULL", + "T76x_L2_EXT_R_RAW", + "T76x_L2_EXT_W_STALL", + "T76x_L2_EXT_W_BUF_FULL", + "T76x_L2_EXT_R_BUF_FULL", + "T76x_L2_TAG_HAZARD", + "T76x_L2_SNOOP_FULL", + "T76x_L2_REPLAY_FULL" +}; + +static const char * const hardware_counters_mali_t82x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T82x_MESSAGES_SENT", + "T82x_MESSAGES_RECEIVED", + "T82x_GPU_ACTIVE", + "T82x_IRQ_ACTIVE", + "T82x_JS0_JOBS", + "T82x_JS0_TASKS", + "T82x_JS0_ACTIVE", + "", + "T82x_JS0_WAIT_READ", + "T82x_JS0_WAIT_ISSUE", + "T82x_JS0_WAIT_DEPEND", + "T82x_JS0_WAIT_FINISH", + "T82x_JS1_JOBS", + "T82x_JS1_TASKS", + "T82x_JS1_ACTIVE", + "", + "T82x_JS1_WAIT_READ", + "T82x_JS1_WAIT_ISSUE", + "T82x_JS1_WAIT_DEPEND", + "T82x_JS1_WAIT_FINISH", + "T82x_JS2_JOBS", + "T82x_JS2_TASKS", + "T82x_JS2_ACTIVE", + "", + "T82x_JS2_WAIT_READ", + "T82x_JS2_WAIT_ISSUE", + "T82x_JS2_WAIT_DEPEND", + "T82x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T82x_TI_JOBS_PROCESSED", + "T82x_TI_TRIANGLES", + "T82x_TI_QUADS", + "T82x_TI_POLYGONS", + "T82x_TI_POINTS", + "T82x_TI_LINES", + "T82x_TI_FRONT_FACING", + "T82x_TI_BACK_FACING", + "T82x_TI_PRIM_VISIBLE", + "T82x_TI_PRIM_CULLED", + "T82x_TI_PRIM_CLIPPED", + "", + "", + "", + "", + "", + "", + "", + "", + "T82x_TI_ACTIVE", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Shader Core */ + "", + "", + "", + "", + "T82x_FRAG_ACTIVE", + "T82x_FRAG_PRIMITIVES", + "T82x_FRAG_PRIMITIVES_DROPPED", + "T82x_FRAG_CYCLES_DESC", + "T82x_FRAG_CYCLES_FPKQ_ACTIVE", + "T82x_FRAG_CYCLES_VERT", + "T82x_FRAG_CYCLES_TRISETUP", + "T82x_FRAG_CYCLES_EZS_ACTIVE", + "T82x_FRAG_THREADS", + "T82x_FRAG_DUMMY_THREADS", + "T82x_FRAG_QUADS_RAST", + "T82x_FRAG_QUADS_EZS_TEST", + "T82x_FRAG_QUADS_EZS_KILLED", + "T82x_FRAG_THREADS_LZS_TEST", + "T82x_FRAG_THREADS_LZS_KILLED", + "T82x_FRAG_CYCLES_NO_TILE", + "T82x_FRAG_NUM_TILES", + "T82x_FRAG_TRANS_ELIM", + "T82x_COMPUTE_ACTIVE", + "T82x_COMPUTE_TASKS", + "T82x_COMPUTE_THREADS", + "T82x_COMPUTE_CYCLES_DESC", + "T82x_TRIPIPE_ACTIVE", + "T82x_ARITH_WORDS", + "T82x_ARITH_CYCLES_REG", + "T82x_ARITH_CYCLES_L0", + "T82x_ARITH_FRAG_DEPEND", + "T82x_LS_WORDS", + "T82x_LS_ISSUES", + "T82x_LS_REISSUE_ATTR", + "T82x_LS_REISSUES_VARY", + "T82x_LS_VARY_RV_MISS", + "T82x_LS_VARY_RV_HIT", + "T82x_LS_NO_UNPARK", + "T82x_TEX_WORDS", + "T82x_TEX_BUBBLES", + "T82x_TEX_WORDS_L0", + "T82x_TEX_WORDS_DESC", + "T82x_TEX_ISSUES", + "T82x_TEX_RECIRC_FMISS", + "T82x_TEX_RECIRC_DESC", + "T82x_TEX_RECIRC_MULTI", + "T82x_TEX_RECIRC_PMISS", + "T82x_TEX_RECIRC_CONF", + "T82x_LSC_READ_HITS", + "T82x_LSC_READ_OP", + "T82x_LSC_WRITE_HITS", + "T82x_LSC_WRITE_OP", + "T82x_LSC_ATOMIC_HITS", + "T82x_LSC_ATOMIC_OP", + "T82x_LSC_LINE_FETCHES", + "T82x_LSC_DIRTY_LINE", + "T82x_LSC_SNOOPS", + "T82x_AXI_TLB_STALL", + "T82x_AXI_TLB_MISS", + "T82x_AXI_TLB_TRANSACTION", + "T82x_LS_TLB_MISS", + "T82x_LS_TLB_HIT", + "T82x_AXI_BEATS_READ", + "T82x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T82x_MMU_HIT", + "T82x_MMU_NEW_MISS", + "T82x_MMU_REPLAY_FULL", + "T82x_MMU_REPLAY_MISS", + "T82x_MMU_TABLE_WALK", + "T82x_MMU_REQUESTS", + "", + "", + "T82x_UTLB_HIT", + "T82x_UTLB_NEW_MISS", + "T82x_UTLB_REPLAY_FULL", + "T82x_UTLB_REPLAY_MISS", + "T82x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T82x_L2_EXT_WRITE_BEATS", + "T82x_L2_EXT_READ_BEATS", + "T82x_L2_ANY_LOOKUP", + "T82x_L2_READ_LOOKUP", + "T82x_L2_SREAD_LOOKUP", + "T82x_L2_READ_REPLAY", + "T82x_L2_READ_SNOOP", + "T82x_L2_READ_HIT", + "T82x_L2_CLEAN_MISS", + "T82x_L2_WRITE_LOOKUP", + "T82x_L2_SWRITE_LOOKUP", + "T82x_L2_WRITE_REPLAY", + "T82x_L2_WRITE_SNOOP", + "T82x_L2_WRITE_HIT", + "T82x_L2_EXT_READ_FULL", + "", + "T82x_L2_EXT_WRITE_FULL", + "T82x_L2_EXT_R_W_HAZARD", + "T82x_L2_EXT_READ", + "T82x_L2_EXT_READ_LINE", + "T82x_L2_EXT_WRITE", + "T82x_L2_EXT_WRITE_LINE", + "T82x_L2_EXT_WRITE_SMALL", + "T82x_L2_EXT_BARRIER", + "T82x_L2_EXT_AR_STALL", + "T82x_L2_EXT_R_BUF_FULL", + "T82x_L2_EXT_RD_BUF_FULL", + "T82x_L2_EXT_R_RAW", + "T82x_L2_EXT_W_STALL", + "T82x_L2_EXT_W_BUF_FULL", + "T82x_L2_EXT_R_BUF_FULL", + "T82x_L2_TAG_HAZARD", + "T82x_L2_SNOOP_FULL", + "T82x_L2_REPLAY_FULL" +}; + +static const char * const hardware_counters_mali_t83x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T83x_MESSAGES_SENT", + "T83x_MESSAGES_RECEIVED", + "T83x_GPU_ACTIVE", + "T83x_IRQ_ACTIVE", + "T83x_JS0_JOBS", + "T83x_JS0_TASKS", + "T83x_JS0_ACTIVE", + "", + "T83x_JS0_WAIT_READ", + "T83x_JS0_WAIT_ISSUE", + "T83x_JS0_WAIT_DEPEND", + "T83x_JS0_WAIT_FINISH", + "T83x_JS1_JOBS", + "T83x_JS1_TASKS", + "T83x_JS1_ACTIVE", + "", + "T83x_JS1_WAIT_READ", + "T83x_JS1_WAIT_ISSUE", + "T83x_JS1_WAIT_DEPEND", + "T83x_JS1_WAIT_FINISH", + "T83x_JS2_JOBS", + "T83x_JS2_TASKS", + "T83x_JS2_ACTIVE", + "", + "T83x_JS2_WAIT_READ", + "T83x_JS2_WAIT_ISSUE", + "T83x_JS2_WAIT_DEPEND", + "T83x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T83x_TI_JOBS_PROCESSED", + "T83x_TI_TRIANGLES", + "T83x_TI_QUADS", + "T83x_TI_POLYGONS", + "T83x_TI_POINTS", + "T83x_TI_LINES", + "T83x_TI_FRONT_FACING", + "T83x_TI_BACK_FACING", + "T83x_TI_PRIM_VISIBLE", + "T83x_TI_PRIM_CULLED", + "T83x_TI_PRIM_CLIPPED", + "", + "", + "", + "", + "", + "", + "", + "", + "T83x_TI_ACTIVE", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Shader Core */ + "", + "", + "", + "", + "T83x_FRAG_ACTIVE", + "T83x_FRAG_PRIMITIVES", + "T83x_FRAG_PRIMITIVES_DROPPED", + "T83x_FRAG_CYCLES_DESC", + "T83x_FRAG_CYCLES_FPKQ_ACTIVE", + "T83x_FRAG_CYCLES_VERT", + "T83x_FRAG_CYCLES_TRISETUP", + "T83x_FRAG_CYCLES_EZS_ACTIVE", + "T83x_FRAG_THREADS", + "T83x_FRAG_DUMMY_THREADS", + "T83x_FRAG_QUADS_RAST", + "T83x_FRAG_QUADS_EZS_TEST", + "T83x_FRAG_QUADS_EZS_KILLED", + "T83x_FRAG_THREADS_LZS_TEST", + "T83x_FRAG_THREADS_LZS_KILLED", + "T83x_FRAG_CYCLES_NO_TILE", + "T83x_FRAG_NUM_TILES", + "T83x_FRAG_TRANS_ELIM", + "T83x_COMPUTE_ACTIVE", + "T83x_COMPUTE_TASKS", + "T83x_COMPUTE_THREADS", + "T83x_COMPUTE_CYCLES_DESC", + "T83x_TRIPIPE_ACTIVE", + "T83x_ARITH_WORDS", + "T83x_ARITH_CYCLES_REG", + "T83x_ARITH_CYCLES_L0", + "T83x_ARITH_FRAG_DEPEND", + "T83x_LS_WORDS", + "T83x_LS_ISSUES", + "T83x_LS_REISSUE_ATTR", + "T83x_LS_REISSUES_VARY", + "T83x_LS_VARY_RV_MISS", + "T83x_LS_VARY_RV_HIT", + "T83x_LS_NO_UNPARK", + "T83x_TEX_WORDS", + "T83x_TEX_BUBBLES", + "T83x_TEX_WORDS_L0", + "T83x_TEX_WORDS_DESC", + "T83x_TEX_ISSUES", + "T83x_TEX_RECIRC_FMISS", + "T83x_TEX_RECIRC_DESC", + "T83x_TEX_RECIRC_MULTI", + "T83x_TEX_RECIRC_PMISS", + "T83x_TEX_RECIRC_CONF", + "T83x_LSC_READ_HITS", + "T83x_LSC_READ_OP", + "T83x_LSC_WRITE_HITS", + "T83x_LSC_WRITE_OP", + "T83x_LSC_ATOMIC_HITS", + "T83x_LSC_ATOMIC_OP", + "T83x_LSC_LINE_FETCHES", + "T83x_LSC_DIRTY_LINE", + "T83x_LSC_SNOOPS", + "T83x_AXI_TLB_STALL", + "T83x_AXI_TLB_MISS", + "T83x_AXI_TLB_TRANSACTION", + "T83x_LS_TLB_MISS", + "T83x_LS_TLB_HIT", + "T83x_AXI_BEATS_READ", + "T83x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T83x_MMU_HIT", + "T83x_MMU_NEW_MISS", + "T83x_MMU_REPLAY_FULL", + "T83x_MMU_REPLAY_MISS", + "T83x_MMU_TABLE_WALK", + "T83x_MMU_REQUESTS", + "", + "", + "T83x_UTLB_HIT", + "T83x_UTLB_NEW_MISS", + "T83x_UTLB_REPLAY_FULL", + "T83x_UTLB_REPLAY_MISS", + "T83x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T83x_L2_EXT_WRITE_BEATS", + "T83x_L2_EXT_READ_BEATS", + "T83x_L2_ANY_LOOKUP", + "T83x_L2_READ_LOOKUP", + "T83x_L2_SREAD_LOOKUP", + "T83x_L2_READ_REPLAY", + "T83x_L2_READ_SNOOP", + "T83x_L2_READ_HIT", + "T83x_L2_CLEAN_MISS", + "T83x_L2_WRITE_LOOKUP", + "T83x_L2_SWRITE_LOOKUP", + "T83x_L2_WRITE_REPLAY", + "T83x_L2_WRITE_SNOOP", + "T83x_L2_WRITE_HIT", + "T83x_L2_EXT_READ_FULL", + "", + "T83x_L2_EXT_WRITE_FULL", + "T83x_L2_EXT_R_W_HAZARD", + "T83x_L2_EXT_READ", + "T83x_L2_EXT_READ_LINE", + "T83x_L2_EXT_WRITE", + "T83x_L2_EXT_WRITE_LINE", + "T83x_L2_EXT_WRITE_SMALL", + "T83x_L2_EXT_BARRIER", + "T83x_L2_EXT_AR_STALL", + "T83x_L2_EXT_R_BUF_FULL", + "T83x_L2_EXT_RD_BUF_FULL", + "T83x_L2_EXT_R_RAW", + "T83x_L2_EXT_W_STALL", + "T83x_L2_EXT_W_BUF_FULL", + "T83x_L2_EXT_R_BUF_FULL", + "T83x_L2_TAG_HAZARD", + "T83x_L2_SNOOP_FULL", + "T83x_L2_REPLAY_FULL" +}; + +static const char * const hardware_counters_mali_t86x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T86x_MESSAGES_SENT", + "T86x_MESSAGES_RECEIVED", + "T86x_GPU_ACTIVE", + "T86x_IRQ_ACTIVE", + "T86x_JS0_JOBS", + "T86x_JS0_TASKS", + "T86x_JS0_ACTIVE", + "", + "T86x_JS0_WAIT_READ", + "T86x_JS0_WAIT_ISSUE", + "T86x_JS0_WAIT_DEPEND", + "T86x_JS0_WAIT_FINISH", + "T86x_JS1_JOBS", + "T86x_JS1_TASKS", + "T86x_JS1_ACTIVE", + "", + "T86x_JS1_WAIT_READ", + "T86x_JS1_WAIT_ISSUE", + "T86x_JS1_WAIT_DEPEND", + "T86x_JS1_WAIT_FINISH", + "T86x_JS2_JOBS", + "T86x_JS2_TASKS", + "T86x_JS2_ACTIVE", + "", + "T86x_JS2_WAIT_READ", + "T86x_JS2_WAIT_ISSUE", + "T86x_JS2_WAIT_DEPEND", + "T86x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T86x_TI_JOBS_PROCESSED", + "T86x_TI_TRIANGLES", + "T86x_TI_QUADS", + "T86x_TI_POLYGONS", + "T86x_TI_POINTS", + "T86x_TI_LINES", + "T86x_TI_VCACHE_HIT", + "T86x_TI_VCACHE_MISS", + "T86x_TI_FRONT_FACING", + "T86x_TI_BACK_FACING", + "T86x_TI_PRIM_VISIBLE", + "T86x_TI_PRIM_CULLED", + "T86x_TI_PRIM_CLIPPED", + "T86x_TI_LEVEL0", + "T86x_TI_LEVEL1", + "T86x_TI_LEVEL2", + "T86x_TI_LEVEL3", + "T86x_TI_LEVEL4", + "T86x_TI_LEVEL5", + "T86x_TI_LEVEL6", + "T86x_TI_LEVEL7", + "T86x_TI_COMMAND_1", + "T86x_TI_COMMAND_2", + "T86x_TI_COMMAND_3", + "T86x_TI_COMMAND_4", + "T86x_TI_COMMAND_5_7", + "T86x_TI_COMMAND_8_15", + "T86x_TI_COMMAND_16_63", + "T86x_TI_COMMAND_64", + "T86x_TI_COMPRESS_IN", + "T86x_TI_COMPRESS_OUT", + "T86x_TI_COMPRESS_FLUSH", + "T86x_TI_TIMESTAMPS", + "T86x_TI_PCACHE_HIT", + "T86x_TI_PCACHE_MISS", + "T86x_TI_PCACHE_LINE", + "T86x_TI_PCACHE_STALL", + "T86x_TI_WRBUF_HIT", + "T86x_TI_WRBUF_MISS", + "T86x_TI_WRBUF_LINE", + "T86x_TI_WRBUF_PARTIAL", + "T86x_TI_WRBUF_STALL", + "T86x_TI_ACTIVE", + "T86x_TI_LOADING_DESC", + "T86x_TI_INDEX_WAIT", + "T86x_TI_INDEX_RANGE_WAIT", + "T86x_TI_VERTEX_WAIT", + "T86x_TI_PCACHE_WAIT", + "T86x_TI_WRBUF_WAIT", + "T86x_TI_BUS_READ", + "T86x_TI_BUS_WRITE", + "", + "", + "", + "", + "", + "T86x_TI_UTLB_HIT", + "T86x_TI_UTLB_NEW_MISS", + "T86x_TI_UTLB_REPLAY_FULL", + "T86x_TI_UTLB_REPLAY_MISS", + "T86x_TI_UTLB_STALL", + + /* Shader Core */ + "", + "", + "", + "", + "T86x_FRAG_ACTIVE", + "T86x_FRAG_PRIMITIVES", + "T86x_FRAG_PRIMITIVES_DROPPED", + "T86x_FRAG_CYCLES_DESC", + "T86x_FRAG_CYCLES_FPKQ_ACTIVE", + "T86x_FRAG_CYCLES_VERT", + "T86x_FRAG_CYCLES_TRISETUP", + "T86x_FRAG_CYCLES_EZS_ACTIVE", + "T86x_FRAG_THREADS", + "T86x_FRAG_DUMMY_THREADS", + "T86x_FRAG_QUADS_RAST", + "T86x_FRAG_QUADS_EZS_TEST", + "T86x_FRAG_QUADS_EZS_KILLED", + "T86x_FRAG_THREADS_LZS_TEST", + "T86x_FRAG_THREADS_LZS_KILLED", + "T86x_FRAG_CYCLES_NO_TILE", + "T86x_FRAG_NUM_TILES", + "T86x_FRAG_TRANS_ELIM", + "T86x_COMPUTE_ACTIVE", + "T86x_COMPUTE_TASKS", + "T86x_COMPUTE_THREADS", + "T86x_COMPUTE_CYCLES_DESC", + "T86x_TRIPIPE_ACTIVE", + "T86x_ARITH_WORDS", + "T86x_ARITH_CYCLES_REG", + "T86x_ARITH_CYCLES_L0", + "T86x_ARITH_FRAG_DEPEND", + "T86x_LS_WORDS", + "T86x_LS_ISSUES", + "T86x_LS_REISSUE_ATTR", + "T86x_LS_REISSUES_VARY", + "T86x_LS_VARY_RV_MISS", + "T86x_LS_VARY_RV_HIT", + "T86x_LS_NO_UNPARK", + "T86x_TEX_WORDS", + "T86x_TEX_BUBBLES", + "T86x_TEX_WORDS_L0", + "T86x_TEX_WORDS_DESC", + "T86x_TEX_ISSUES", + "T86x_TEX_RECIRC_FMISS", + "T86x_TEX_RECIRC_DESC", + "T86x_TEX_RECIRC_MULTI", + "T86x_TEX_RECIRC_PMISS", + "T86x_TEX_RECIRC_CONF", + "T86x_LSC_READ_HITS", + "T86x_LSC_READ_OP", + "T86x_LSC_WRITE_HITS", + "T86x_LSC_WRITE_OP", + "T86x_LSC_ATOMIC_HITS", + "T86x_LSC_ATOMIC_OP", + "T86x_LSC_LINE_FETCHES", + "T86x_LSC_DIRTY_LINE", + "T86x_LSC_SNOOPS", + "T86x_AXI_TLB_STALL", + "T86x_AXI_TLB_MISS", + "T86x_AXI_TLB_TRANSACTION", + "T86x_LS_TLB_MISS", + "T86x_LS_TLB_HIT", + "T86x_AXI_BEATS_READ", + "T86x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T86x_MMU_HIT", + "T86x_MMU_NEW_MISS", + "T86x_MMU_REPLAY_FULL", + "T86x_MMU_REPLAY_MISS", + "T86x_MMU_TABLE_WALK", + "T86x_MMU_REQUESTS", + "", + "", + "T86x_UTLB_HIT", + "T86x_UTLB_NEW_MISS", + "T86x_UTLB_REPLAY_FULL", + "T86x_UTLB_REPLAY_MISS", + "T86x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T86x_L2_EXT_WRITE_BEATS", + "T86x_L2_EXT_READ_BEATS", + "T86x_L2_ANY_LOOKUP", + "T86x_L2_READ_LOOKUP", + "T86x_L2_SREAD_LOOKUP", + "T86x_L2_READ_REPLAY", + "T86x_L2_READ_SNOOP", + "T86x_L2_READ_HIT", + "T86x_L2_CLEAN_MISS", + "T86x_L2_WRITE_LOOKUP", + "T86x_L2_SWRITE_LOOKUP", + "T86x_L2_WRITE_REPLAY", + "T86x_L2_WRITE_SNOOP", + "T86x_L2_WRITE_HIT", + "T86x_L2_EXT_READ_FULL", + "", + "T86x_L2_EXT_WRITE_FULL", + "T86x_L2_EXT_R_W_HAZARD", + "T86x_L2_EXT_READ", + "T86x_L2_EXT_READ_LINE", + "T86x_L2_EXT_WRITE", + "T86x_L2_EXT_WRITE_LINE", + "T86x_L2_EXT_WRITE_SMALL", + "T86x_L2_EXT_BARRIER", + "T86x_L2_EXT_AR_STALL", + "T86x_L2_EXT_R_BUF_FULL", + "T86x_L2_EXT_RD_BUF_FULL", + "T86x_L2_EXT_R_RAW", + "T86x_L2_EXT_W_STALL", + "T86x_L2_EXT_W_BUF_FULL", + "T86x_L2_EXT_R_BUF_FULL", + "T86x_L2_TAG_HAZARD", + "T86x_L2_SNOOP_FULL", + "T86x_L2_REPLAY_FULL" +}; + +static const char * const hardware_counters_mali_t88x[] = { + /* Job Manager */ + "", + "", + "", + "", + "T88x_MESSAGES_SENT", + "T88x_MESSAGES_RECEIVED", + "T88x_GPU_ACTIVE", + "T88x_IRQ_ACTIVE", + "T88x_JS0_JOBS", + "T88x_JS0_TASKS", + "T88x_JS0_ACTIVE", + "", + "T88x_JS0_WAIT_READ", + "T88x_JS0_WAIT_ISSUE", + "T88x_JS0_WAIT_DEPEND", + "T88x_JS0_WAIT_FINISH", + "T88x_JS1_JOBS", + "T88x_JS1_TASKS", + "T88x_JS1_ACTIVE", + "", + "T88x_JS1_WAIT_READ", + "T88x_JS1_WAIT_ISSUE", + "T88x_JS1_WAIT_DEPEND", + "T88x_JS1_WAIT_FINISH", + "T88x_JS2_JOBS", + "T88x_JS2_TASKS", + "T88x_JS2_ACTIVE", + "", + "T88x_JS2_WAIT_READ", + "T88x_JS2_WAIT_ISSUE", + "T88x_JS2_WAIT_DEPEND", + "T88x_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /*Tiler */ + "", + "", + "", + "T88x_TI_JOBS_PROCESSED", + "T88x_TI_TRIANGLES", + "T88x_TI_QUADS", + "T88x_TI_POLYGONS", + "T88x_TI_POINTS", + "T88x_TI_LINES", + "T88x_TI_VCACHE_HIT", + "T88x_TI_VCACHE_MISS", + "T88x_TI_FRONT_FACING", + "T88x_TI_BACK_FACING", + "T88x_TI_PRIM_VISIBLE", + "T88x_TI_PRIM_CULLED", + "T88x_TI_PRIM_CLIPPED", + "T88x_TI_LEVEL0", + "T88x_TI_LEVEL1", + "T88x_TI_LEVEL2", + "T88x_TI_LEVEL3", + "T88x_TI_LEVEL4", + "T88x_TI_LEVEL5", + "T88x_TI_LEVEL6", + "T88x_TI_LEVEL7", + "T88x_TI_COMMAND_1", + "T88x_TI_COMMAND_2", + "T88x_TI_COMMAND_3", + "T88x_TI_COMMAND_4", + "T88x_TI_COMMAND_5_7", + "T88x_TI_COMMAND_8_15", + "T88x_TI_COMMAND_16_63", + "T88x_TI_COMMAND_64", + "T88x_TI_COMPRESS_IN", + "T88x_TI_COMPRESS_OUT", + "T88x_TI_COMPRESS_FLUSH", + "T88x_TI_TIMESTAMPS", + "T88x_TI_PCACHE_HIT", + "T88x_TI_PCACHE_MISS", + "T88x_TI_PCACHE_LINE", + "T88x_TI_PCACHE_STALL", + "T88x_TI_WRBUF_HIT", + "T88x_TI_WRBUF_MISS", + "T88x_TI_WRBUF_LINE", + "T88x_TI_WRBUF_PARTIAL", + "T88x_TI_WRBUF_STALL", + "T88x_TI_ACTIVE", + "T88x_TI_LOADING_DESC", + "T88x_TI_INDEX_WAIT", + "T88x_TI_INDEX_RANGE_WAIT", + "T88x_TI_VERTEX_WAIT", + "T88x_TI_PCACHE_WAIT", + "T88x_TI_WRBUF_WAIT", + "T88x_TI_BUS_READ", + "T88x_TI_BUS_WRITE", + "", + "", + "", + "", + "", + "T88x_TI_UTLB_HIT", + "T88x_TI_UTLB_NEW_MISS", + "T88x_TI_UTLB_REPLAY_FULL", + "T88x_TI_UTLB_REPLAY_MISS", + "T88x_TI_UTLB_STALL", + + /* Shader Core */ + "", + "", + "", + "", + "T88x_FRAG_ACTIVE", + "T88x_FRAG_PRIMITIVES", + "T88x_FRAG_PRIMITIVES_DROPPED", + "T88x_FRAG_CYCLES_DESC", + "T88x_FRAG_CYCLES_FPKQ_ACTIVE", + "T88x_FRAG_CYCLES_VERT", + "T88x_FRAG_CYCLES_TRISETUP", + "T88x_FRAG_CYCLES_EZS_ACTIVE", + "T88x_FRAG_THREADS", + "T88x_FRAG_DUMMY_THREADS", + "T88x_FRAG_QUADS_RAST", + "T88x_FRAG_QUADS_EZS_TEST", + "T88x_FRAG_QUADS_EZS_KILLED", + "T88x_FRAG_THREADS_LZS_TEST", + "T88x_FRAG_THREADS_LZS_KILLED", + "T88x_FRAG_CYCLES_NO_TILE", + "T88x_FRAG_NUM_TILES", + "T88x_FRAG_TRANS_ELIM", + "T88x_COMPUTE_ACTIVE", + "T88x_COMPUTE_TASKS", + "T88x_COMPUTE_THREADS", + "T88x_COMPUTE_CYCLES_DESC", + "T88x_TRIPIPE_ACTIVE", + "T88x_ARITH_WORDS", + "T88x_ARITH_CYCLES_REG", + "T88x_ARITH_CYCLES_L0", + "T88x_ARITH_FRAG_DEPEND", + "T88x_LS_WORDS", + "T88x_LS_ISSUES", + "T88x_LS_REISSUE_ATTR", + "T88x_LS_REISSUES_VARY", + "T88x_LS_VARY_RV_MISS", + "T88x_LS_VARY_RV_HIT", + "T88x_LS_NO_UNPARK", + "T88x_TEX_WORDS", + "T88x_TEX_BUBBLES", + "T88x_TEX_WORDS_L0", + "T88x_TEX_WORDS_DESC", + "T88x_TEX_ISSUES", + "T88x_TEX_RECIRC_FMISS", + "T88x_TEX_RECIRC_DESC", + "T88x_TEX_RECIRC_MULTI", + "T88x_TEX_RECIRC_PMISS", + "T88x_TEX_RECIRC_CONF", + "T88x_LSC_READ_HITS", + "T88x_LSC_READ_OP", + "T88x_LSC_WRITE_HITS", + "T88x_LSC_WRITE_OP", + "T88x_LSC_ATOMIC_HITS", + "T88x_LSC_ATOMIC_OP", + "T88x_LSC_LINE_FETCHES", + "T88x_LSC_DIRTY_LINE", + "T88x_LSC_SNOOPS", + "T88x_AXI_TLB_STALL", + "T88x_AXI_TLB_MISS", + "T88x_AXI_TLB_TRANSACTION", + "T88x_LS_TLB_MISS", + "T88x_LS_TLB_HIT", + "T88x_AXI_BEATS_READ", + "T88x_AXI_BEATS_WRITTEN", + + /*L2 and MMU */ + "", + "", + "", + "", + "T88x_MMU_HIT", + "T88x_MMU_NEW_MISS", + "T88x_MMU_REPLAY_FULL", + "T88x_MMU_REPLAY_MISS", + "T88x_MMU_TABLE_WALK", + "T88x_MMU_REQUESTS", + "", + "", + "T88x_UTLB_HIT", + "T88x_UTLB_NEW_MISS", + "T88x_UTLB_REPLAY_FULL", + "T88x_UTLB_REPLAY_MISS", + "T88x_UTLB_STALL", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "T88x_L2_EXT_WRITE_BEATS", + "T88x_L2_EXT_READ_BEATS", + "T88x_L2_ANY_LOOKUP", + "T88x_L2_READ_LOOKUP", + "T88x_L2_SREAD_LOOKUP", + "T88x_L2_READ_REPLAY", + "T88x_L2_READ_SNOOP", + "T88x_L2_READ_HIT", + "T88x_L2_CLEAN_MISS", + "T88x_L2_WRITE_LOOKUP", + "T88x_L2_SWRITE_LOOKUP", + "T88x_L2_WRITE_REPLAY", + "T88x_L2_WRITE_SNOOP", + "T88x_L2_WRITE_HIT", + "T88x_L2_EXT_READ_FULL", + "", + "T88x_L2_EXT_WRITE_FULL", + "T88x_L2_EXT_R_W_HAZARD", + "T88x_L2_EXT_READ", + "T88x_L2_EXT_READ_LINE", + "T88x_L2_EXT_WRITE", + "T88x_L2_EXT_WRITE_LINE", + "T88x_L2_EXT_WRITE_SMALL", + "T88x_L2_EXT_BARRIER", + "T88x_L2_EXT_AR_STALL", + "T88x_L2_EXT_R_BUF_FULL", + "T88x_L2_EXT_RD_BUF_FULL", + "T88x_L2_EXT_R_RAW", + "T88x_L2_EXT_W_STALL", + "T88x_L2_EXT_W_BUF_FULL", + "T88x_L2_EXT_R_BUF_FULL", + "T88x_L2_TAG_HAZARD", + "T88x_L2_SNOOP_FULL", + "T88x_L2_REPLAY_FULL" +}; + +#include "mali_kbase_gator_hwcnt_names_tmix.h" + +#include "mali_kbase_gator_hwcnt_names_thex.h" + +#include "mali_kbase_gator_hwcnt_names_tsix.h" + + +#ifdef MALI_INCLUDE_TKAX +#include "mali_kbase_gator_hwcnt_names_tkax.h" +#endif /* MALI_INCLUDE_TKAX */ + +#endif diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_thex.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_thex.h new file mode 100755 index 000000000000..bcceef4fc9bc --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_thex.h @@ -0,0 +1,291 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * This header was autogenerated, it should not be edited. + */ + +#ifndef _KBASE_GATOR_HWCNT_NAMES_THEX_H_ +#define _KBASE_GATOR_HWCNT_NAMES_THEX_H_ + +static const char * const hardware_counters_mali_tHEx[] = { + /* Performance counters for the Job Manager */ + "", + "", + "", + "", + "THEx_MESSAGES_SENT", + "THEx_MESSAGES_RECEIVED", + "THEx_GPU_ACTIVE", + "THEx_IRQ_ACTIVE", + "THEx_JS0_JOBS", + "THEx_JS0_TASKS", + "THEx_JS0_ACTIVE", + "", + "THEx_JS0_WAIT_READ", + "THEx_JS0_WAIT_ISSUE", + "THEx_JS0_WAIT_DEPEND", + "THEx_JS0_WAIT_FINISH", + "THEx_JS1_JOBS", + "THEx_JS1_TASKS", + "THEx_JS1_ACTIVE", + "", + "THEx_JS1_WAIT_READ", + "THEx_JS1_WAIT_ISSUE", + "THEx_JS1_WAIT_DEPEND", + "THEx_JS1_WAIT_FINISH", + "THEx_JS2_JOBS", + "THEx_JS2_TASKS", + "THEx_JS2_ACTIVE", + "", + "THEx_JS2_WAIT_READ", + "THEx_JS2_WAIT_ISSUE", + "THEx_JS2_WAIT_DEPEND", + "THEx_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Performance counters for the Tiler */ + "", + "", + "", + "", + "THEx_TILER_ACTIVE", + "THEx_JOBS_PROCESSED", + "THEx_TRIANGLES", + "THEx_LINES", + "THEx_POINTS", + "THEx_FRONT_FACING", + "THEx_BACK_FACING", + "THEx_PRIM_VISIBLE", + "THEx_PRIM_CULLED", + "THEx_PRIM_CLIPPED", + "THEx_PRIM_SAT_CULLED", + "", + "", + "THEx_BUS_READ", + "", + "THEx_BUS_WRITE", + "THEx_LOADING_DESC", + "THEx_IDVS_POS_SHAD_REQ", + "THEx_IDVS_POS_SHAD_WAIT", + "THEx_IDVS_POS_SHAD_STALL", + "THEx_IDVS_POS_FIFO_FULL", + "THEx_PREFETCH_STALL", + "THEx_VCACHE_HIT", + "THEx_VCACHE_MISS", + "THEx_VCACHE_LINE_WAIT", + "THEx_VFETCH_POS_READ_WAIT", + "THEx_VFETCH_VERTEX_WAIT", + "THEx_VFETCH_STALL", + "THEx_PRIMASSY_STALL", + "THEx_BBOX_GEN_STALL", + "THEx_IDVS_VBU_HIT", + "THEx_IDVS_VBU_MISS", + "THEx_IDVS_VBU_LINE_DEALLOCATE", + "THEx_IDVS_VAR_SHAD_REQ", + "THEx_IDVS_VAR_SHAD_STALL", + "THEx_BINNER_STALL", + "THEx_ITER_STALL", + "THEx_COMPRESS_MISS", + "THEx_COMPRESS_STALL", + "THEx_PCACHE_HIT", + "THEx_PCACHE_MISS", + "THEx_PCACHE_MISS_STALL", + "THEx_PCACHE_EVICT_STALL", + "THEx_PMGR_PTR_WR_STALL", + "THEx_PMGR_PTR_RD_STALL", + "THEx_PMGR_CMD_WR_STALL", + "THEx_WRBUF_ACTIVE", + "THEx_WRBUF_HIT", + "THEx_WRBUF_MISS", + "THEx_WRBUF_NO_FREE_LINE_STALL", + "THEx_WRBUF_NO_AXI_ID_STALL", + "THEx_WRBUF_AXI_STALL", + "", + "", + "", + "THEx_UTLB_TRANS", + "THEx_UTLB_TRANS_HIT", + "THEx_UTLB_TRANS_STALL", + "THEx_UTLB_TRANS_MISS_DELAY", + "THEx_UTLB_MMU_REQ", + + /* Performance counters for the Shader Core */ + "", + "", + "", + "", + "THEx_FRAG_ACTIVE", + "THEx_FRAG_PRIMITIVES", + "THEx_FRAG_PRIM_RAST", + "THEx_FRAG_FPK_ACTIVE", + "THEx_FRAG_STARVING", + "THEx_FRAG_WARPS", + "THEx_FRAG_PARTIAL_WARPS", + "THEx_FRAG_QUADS_RAST", + "THEx_FRAG_QUADS_EZS_TEST", + "THEx_FRAG_QUADS_EZS_UPDATE", + "THEx_FRAG_QUADS_EZS_KILL", + "THEx_FRAG_LZS_TEST", + "THEx_FRAG_LZS_KILL", + "", + "THEx_FRAG_PTILES", + "THEx_FRAG_TRANS_ELIM", + "THEx_QUAD_FPK_KILLER", + "", + "THEx_COMPUTE_ACTIVE", + "THEx_COMPUTE_TASKS", + "THEx_COMPUTE_WARPS", + "THEx_COMPUTE_STARVING", + "THEx_EXEC_CORE_ACTIVE", + "THEx_EXEC_ACTIVE", + "THEx_EXEC_INSTR_COUNT", + "THEx_EXEC_INSTR_DIVERGED", + "THEx_EXEC_INSTR_STARVING", + "THEx_ARITH_INSTR_SINGLE_FMA", + "THEx_ARITH_INSTR_DOUBLE", + "THEx_ARITH_INSTR_MSG", + "THEx_ARITH_INSTR_MSG_ONLY", + "THEx_TEX_INSTR", + "THEx_TEX_INSTR_MIPMAP", + "THEx_TEX_INSTR_COMPRESSED", + "THEx_TEX_INSTR_3D", + "THEx_TEX_INSTR_TRILINEAR", + "THEx_TEX_COORD_ISSUE", + "THEx_TEX_COORD_STALL", + "THEx_TEX_STARVE_CACHE", + "THEx_TEX_STARVE_FILTER", + "THEx_LS_MEM_READ_FULL", + "THEx_LS_MEM_READ_SHORT", + "THEx_LS_MEM_WRITE_FULL", + "THEx_LS_MEM_WRITE_SHORT", + "THEx_LS_MEM_ATOMIC", + "THEx_VARY_INSTR", + "THEx_VARY_SLOT_32", + "THEx_VARY_SLOT_16", + "THEx_ATTR_INSTR", + "THEx_ARITH_INSTR_FP_MUL", + "THEx_BEATS_RD_FTC", + "THEx_BEATS_RD_FTC_EXT", + "THEx_BEATS_RD_LSC", + "THEx_BEATS_RD_LSC_EXT", + "THEx_BEATS_RD_TEX", + "THEx_BEATS_RD_TEX_EXT", + "THEx_BEATS_RD_OTHER", + "THEx_BEATS_WR_LSC", + "THEx_BEATS_WR_TIB", + "", + + /* Performance counters for the Memory System */ + "", + "", + "", + "", + "THEx_MMU_REQUESTS", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "THEx_L2_RD_MSG_IN", + "THEx_L2_RD_MSG_IN_STALL", + "THEx_L2_WR_MSG_IN", + "THEx_L2_WR_MSG_IN_STALL", + "THEx_L2_SNP_MSG_IN", + "THEx_L2_SNP_MSG_IN_STALL", + "THEx_L2_RD_MSG_OUT", + "THEx_L2_RD_MSG_OUT_STALL", + "THEx_L2_WR_MSG_OUT", + "THEx_L2_ANY_LOOKUP", + "THEx_L2_READ_LOOKUP", + "THEx_L2_WRITE_LOOKUP", + "THEx_L2_EXT_SNOOP_LOOKUP", + "THEx_L2_EXT_READ", + "THEx_L2_EXT_READ_NOSNP", + "THEx_L2_EXT_READ_UNIQUE", + "THEx_L2_EXT_READ_BEATS", + "THEx_L2_EXT_AR_STALL", + "THEx_L2_EXT_AR_CNT_Q1", + "THEx_L2_EXT_AR_CNT_Q2", + "THEx_L2_EXT_AR_CNT_Q3", + "THEx_L2_EXT_RRESP_0_127", + "THEx_L2_EXT_RRESP_128_191", + "THEx_L2_EXT_RRESP_192_255", + "THEx_L2_EXT_RRESP_256_319", + "THEx_L2_EXT_RRESP_320_383", + "THEx_L2_EXT_WRITE", + "THEx_L2_EXT_WRITE_NOSNP_FULL", + "THEx_L2_EXT_WRITE_NOSNP_PTL", + "THEx_L2_EXT_WRITE_SNP_FULL", + "THEx_L2_EXT_WRITE_SNP_PTL", + "THEx_L2_EXT_WRITE_BEATS", + "THEx_L2_EXT_W_STALL", + "THEx_L2_EXT_AW_CNT_Q1", + "THEx_L2_EXT_AW_CNT_Q2", + "THEx_L2_EXT_AW_CNT_Q3", + "THEx_L2_EXT_SNOOP", + "THEx_L2_EXT_SNOOP_STALL", + "THEx_L2_EXT_SNOOP_RESP_CLEAN", + "THEx_L2_EXT_SNOOP_RESP_DATA", + "THEx_L2_EXT_SNOOP_INTERNAL", + "", + "", + "", + "", + "", + "", + "", +}; + +#endif /* _KBASE_GATOR_HWCNT_NAMES_THEX_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tmix.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tmix.h new file mode 100755 index 000000000000..5ea06770fdb2 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tmix.h @@ -0,0 +1,291 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * This header was autogenerated, it should not be edited. + */ + +#ifndef _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ +#define _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ + +static const char * const hardware_counters_mali_tMIx[] = { + /* Performance counters for the Job Manager */ + "", + "", + "", + "", + "TMIx_MESSAGES_SENT", + "TMIx_MESSAGES_RECEIVED", + "TMIx_GPU_ACTIVE", + "TMIx_IRQ_ACTIVE", + "TMIx_JS0_JOBS", + "TMIx_JS0_TASKS", + "TMIx_JS0_ACTIVE", + "", + "TMIx_JS0_WAIT_READ", + "TMIx_JS0_WAIT_ISSUE", + "TMIx_JS0_WAIT_DEPEND", + "TMIx_JS0_WAIT_FINISH", + "TMIx_JS1_JOBS", + "TMIx_JS1_TASKS", + "TMIx_JS1_ACTIVE", + "", + "TMIx_JS1_WAIT_READ", + "TMIx_JS1_WAIT_ISSUE", + "TMIx_JS1_WAIT_DEPEND", + "TMIx_JS1_WAIT_FINISH", + "TMIx_JS2_JOBS", + "TMIx_JS2_TASKS", + "TMIx_JS2_ACTIVE", + "", + "TMIx_JS2_WAIT_READ", + "TMIx_JS2_WAIT_ISSUE", + "TMIx_JS2_WAIT_DEPEND", + "TMIx_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Performance counters for the Tiler */ + "", + "", + "", + "", + "TMIx_TILER_ACTIVE", + "TMIx_JOBS_PROCESSED", + "TMIx_TRIANGLES", + "TMIx_LINES", + "TMIx_POINTS", + "TMIx_FRONT_FACING", + "TMIx_BACK_FACING", + "TMIx_PRIM_VISIBLE", + "TMIx_PRIM_CULLED", + "TMIx_PRIM_CLIPPED", + "TMIx_PRIM_SAT_CULLED", + "", + "", + "TMIx_BUS_READ", + "", + "TMIx_BUS_WRITE", + "TMIx_LOADING_DESC", + "TMIx_IDVS_POS_SHAD_REQ", + "TMIx_IDVS_POS_SHAD_WAIT", + "TMIx_IDVS_POS_SHAD_STALL", + "TMIx_IDVS_POS_FIFO_FULL", + "TMIx_PREFETCH_STALL", + "TMIx_VCACHE_HIT", + "TMIx_VCACHE_MISS", + "TMIx_VCACHE_LINE_WAIT", + "TMIx_VFETCH_POS_READ_WAIT", + "TMIx_VFETCH_VERTEX_WAIT", + "TMIx_VFETCH_STALL", + "TMIx_PRIMASSY_STALL", + "TMIx_BBOX_GEN_STALL", + "TMIx_IDVS_VBU_HIT", + "TMIx_IDVS_VBU_MISS", + "TMIx_IDVS_VBU_LINE_DEALLOCATE", + "TMIx_IDVS_VAR_SHAD_REQ", + "TMIx_IDVS_VAR_SHAD_STALL", + "TMIx_BINNER_STALL", + "TMIx_ITER_STALL", + "TMIx_COMPRESS_MISS", + "TMIx_COMPRESS_STALL", + "TMIx_PCACHE_HIT", + "TMIx_PCACHE_MISS", + "TMIx_PCACHE_MISS_STALL", + "TMIx_PCACHE_EVICT_STALL", + "TMIx_PMGR_PTR_WR_STALL", + "TMIx_PMGR_PTR_RD_STALL", + "TMIx_PMGR_CMD_WR_STALL", + "TMIx_WRBUF_ACTIVE", + "TMIx_WRBUF_HIT", + "TMIx_WRBUF_MISS", + "TMIx_WRBUF_NO_FREE_LINE_STALL", + "TMIx_WRBUF_NO_AXI_ID_STALL", + "TMIx_WRBUF_AXI_STALL", + "", + "", + "", + "TMIx_UTLB_TRANS", + "TMIx_UTLB_TRANS_HIT", + "TMIx_UTLB_TRANS_STALL", + "TMIx_UTLB_TRANS_MISS_DELAY", + "TMIx_UTLB_MMU_REQ", + + /* Performance counters for the Shader Core */ + "", + "", + "", + "", + "TMIx_FRAG_ACTIVE", + "TMIx_FRAG_PRIMITIVES", + "TMIx_FRAG_PRIM_RAST", + "TMIx_FRAG_FPK_ACTIVE", + "TMIx_FRAG_STARVING", + "TMIx_FRAG_WARPS", + "TMIx_FRAG_PARTIAL_WARPS", + "TMIx_FRAG_QUADS_RAST", + "TMIx_FRAG_QUADS_EZS_TEST", + "TMIx_FRAG_QUADS_EZS_UPDATE", + "TMIx_FRAG_QUADS_EZS_KILL", + "TMIx_FRAG_LZS_TEST", + "TMIx_FRAG_LZS_KILL", + "", + "TMIx_FRAG_PTILES", + "TMIx_FRAG_TRANS_ELIM", + "TMIx_QUAD_FPK_KILLER", + "", + "TMIx_COMPUTE_ACTIVE", + "TMIx_COMPUTE_TASKS", + "TMIx_COMPUTE_WARPS", + "TMIx_COMPUTE_STARVING", + "TMIx_EXEC_CORE_ACTIVE", + "TMIx_EXEC_ACTIVE", + "TMIx_EXEC_INSTR_COUNT", + "TMIx_EXEC_INSTR_DIVERGED", + "TMIx_EXEC_INSTR_STARVING", + "TMIx_ARITH_INSTR_SINGLE_FMA", + "TMIx_ARITH_INSTR_DOUBLE", + "TMIx_ARITH_INSTR_MSG", + "TMIx_ARITH_INSTR_MSG_ONLY", + "TMIx_TEX_INSTR", + "TMIx_TEX_INSTR_MIPMAP", + "TMIx_TEX_INSTR_COMPRESSED", + "TMIx_TEX_INSTR_3D", + "TMIx_TEX_INSTR_TRILINEAR", + "TMIx_TEX_COORD_ISSUE", + "TMIx_TEX_COORD_STALL", + "TMIx_TEX_STARVE_CACHE", + "TMIx_TEX_STARVE_FILTER", + "TMIx_LS_MEM_READ_FULL", + "TMIx_LS_MEM_READ_SHORT", + "TMIx_LS_MEM_WRITE_FULL", + "TMIx_LS_MEM_WRITE_SHORT", + "TMIx_LS_MEM_ATOMIC", + "TMIx_VARY_INSTR", + "TMIx_VARY_SLOT_32", + "TMIx_VARY_SLOT_16", + "TMIx_ATTR_INSTR", + "TMIx_ARITH_INSTR_FP_MUL", + "TMIx_BEATS_RD_FTC", + "TMIx_BEATS_RD_FTC_EXT", + "TMIx_BEATS_RD_LSC", + "TMIx_BEATS_RD_LSC_EXT", + "TMIx_BEATS_RD_TEX", + "TMIx_BEATS_RD_TEX_EXT", + "TMIx_BEATS_RD_OTHER", + "TMIx_BEATS_WR_LSC", + "TMIx_BEATS_WR_TIB", + "", + + /* Performance counters for the Memory System */ + "", + "", + "", + "", + "TMIx_MMU_REQUESTS", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "TMIx_L2_RD_MSG_IN", + "TMIx_L2_RD_MSG_IN_STALL", + "TMIx_L2_WR_MSG_IN", + "TMIx_L2_WR_MSG_IN_STALL", + "TMIx_L2_SNP_MSG_IN", + "TMIx_L2_SNP_MSG_IN_STALL", + "TMIx_L2_RD_MSG_OUT", + "TMIx_L2_RD_MSG_OUT_STALL", + "TMIx_L2_WR_MSG_OUT", + "TMIx_L2_ANY_LOOKUP", + "TMIx_L2_READ_LOOKUP", + "TMIx_L2_WRITE_LOOKUP", + "TMIx_L2_EXT_SNOOP_LOOKUP", + "TMIx_L2_EXT_READ", + "TMIx_L2_EXT_READ_NOSNP", + "TMIx_L2_EXT_READ_UNIQUE", + "TMIx_L2_EXT_READ_BEATS", + "TMIx_L2_EXT_AR_STALL", + "TMIx_L2_EXT_AR_CNT_Q1", + "TMIx_L2_EXT_AR_CNT_Q2", + "TMIx_L2_EXT_AR_CNT_Q3", + "TMIx_L2_EXT_RRESP_0_127", + "TMIx_L2_EXT_RRESP_128_191", + "TMIx_L2_EXT_RRESP_192_255", + "TMIx_L2_EXT_RRESP_256_319", + "TMIx_L2_EXT_RRESP_320_383", + "TMIx_L2_EXT_WRITE", + "TMIx_L2_EXT_WRITE_NOSNP_FULL", + "TMIx_L2_EXT_WRITE_NOSNP_PTL", + "TMIx_L2_EXT_WRITE_SNP_FULL", + "TMIx_L2_EXT_WRITE_SNP_PTL", + "TMIx_L2_EXT_WRITE_BEATS", + "TMIx_L2_EXT_W_STALL", + "TMIx_L2_EXT_AW_CNT_Q1", + "TMIx_L2_EXT_AW_CNT_Q2", + "TMIx_L2_EXT_AW_CNT_Q3", + "TMIx_L2_EXT_SNOOP", + "TMIx_L2_EXT_SNOOP_STALL", + "TMIx_L2_EXT_SNOOP_RESP_CLEAN", + "TMIx_L2_EXT_SNOOP_RESP_DATA", + "TMIx_L2_EXT_SNOOP_INTERNAL", + "", + "", + "", + "", + "", + "", + "", +}; + +#endif /* _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tsix.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tsix.h new file mode 100755 index 000000000000..be09c4556735 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tsix.h @@ -0,0 +1,291 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * This header was autogenerated, it should not be edited. + */ + +#ifndef _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ +#define _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ + +static const char * const hardware_counters_mali_tSIx[] = { + /* Performance counters for the Job Manager */ + "", + "", + "", + "", + "TSIx_MESSAGES_SENT", + "TSIx_MESSAGES_RECEIVED", + "TSIx_GPU_ACTIVE", + "TSIx_IRQ_ACTIVE", + "TSIx_JS0_JOBS", + "TSIx_JS0_TASKS", + "TSIx_JS0_ACTIVE", + "", + "TSIx_JS0_WAIT_READ", + "TSIx_JS0_WAIT_ISSUE", + "TSIx_JS0_WAIT_DEPEND", + "TSIx_JS0_WAIT_FINISH", + "TSIx_JS1_JOBS", + "TSIx_JS1_TASKS", + "TSIx_JS1_ACTIVE", + "", + "TSIx_JS1_WAIT_READ", + "TSIx_JS1_WAIT_ISSUE", + "TSIx_JS1_WAIT_DEPEND", + "TSIx_JS1_WAIT_FINISH", + "TSIx_JS2_JOBS", + "TSIx_JS2_TASKS", + "TSIx_JS2_ACTIVE", + "", + "TSIx_JS2_WAIT_READ", + "TSIx_JS2_WAIT_ISSUE", + "TSIx_JS2_WAIT_DEPEND", + "TSIx_JS2_WAIT_FINISH", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + + /* Performance counters for the Tiler */ + "", + "", + "", + "", + "TSIx_TILER_ACTIVE", + "TSIx_JOBS_PROCESSED", + "TSIx_TRIANGLES", + "TSIx_LINES", + "TSIx_POINTS", + "TSIx_FRONT_FACING", + "TSIx_BACK_FACING", + "TSIx_PRIM_VISIBLE", + "TSIx_PRIM_CULLED", + "TSIx_PRIM_CLIPPED", + "TSIx_PRIM_SAT_CULLED", + "", + "", + "TSIx_BUS_READ", + "", + "TSIx_BUS_WRITE", + "TSIx_LOADING_DESC", + "TSIx_IDVS_POS_SHAD_REQ", + "TSIx_IDVS_POS_SHAD_WAIT", + "TSIx_IDVS_POS_SHAD_STALL", + "TSIx_IDVS_POS_FIFO_FULL", + "TSIx_PREFETCH_STALL", + "TSIx_VCACHE_HIT", + "TSIx_VCACHE_MISS", + "TSIx_VCACHE_LINE_WAIT", + "TSIx_VFETCH_POS_READ_WAIT", + "TSIx_VFETCH_VERTEX_WAIT", + "TSIx_VFETCH_STALL", + "TSIx_PRIMASSY_STALL", + "TSIx_BBOX_GEN_STALL", + "TSIx_IDVS_VBU_HIT", + "TSIx_IDVS_VBU_MISS", + "TSIx_IDVS_VBU_LINE_DEALLOCATE", + "TSIx_IDVS_VAR_SHAD_REQ", + "TSIx_IDVS_VAR_SHAD_STALL", + "TSIx_BINNER_STALL", + "TSIx_ITER_STALL", + "TSIx_COMPRESS_MISS", + "TSIx_COMPRESS_STALL", + "TSIx_PCACHE_HIT", + "TSIx_PCACHE_MISS", + "TSIx_PCACHE_MISS_STALL", + "TSIx_PCACHE_EVICT_STALL", + "TSIx_PMGR_PTR_WR_STALL", + "TSIx_PMGR_PTR_RD_STALL", + "TSIx_PMGR_CMD_WR_STALL", + "TSIx_WRBUF_ACTIVE", + "TSIx_WRBUF_HIT", + "TSIx_WRBUF_MISS", + "TSIx_WRBUF_NO_FREE_LINE_STALL", + "TSIx_WRBUF_NO_AXI_ID_STALL", + "TSIx_WRBUF_AXI_STALL", + "", + "", + "", + "TSIx_UTLB_TRANS", + "TSIx_UTLB_TRANS_HIT", + "TSIx_UTLB_TRANS_STALL", + "TSIx_UTLB_TRANS_MISS_DELAY", + "TSIx_UTLB_MMU_REQ", + + /* Performance counters for the Shader Core */ + "", + "", + "", + "", + "TSIx_FRAG_ACTIVE", + "TSIx_FRAG_PRIMITIVES", + "TSIx_FRAG_PRIM_RAST", + "TSIx_FRAG_FPK_ACTIVE", + "TSIx_FRAG_STARVING", + "TSIx_FRAG_WARPS", + "TSIx_FRAG_PARTIAL_WARPS", + "TSIx_FRAG_QUADS_RAST", + "TSIx_FRAG_QUADS_EZS_TEST", + "TSIx_FRAG_QUADS_EZS_UPDATE", + "TSIx_FRAG_QUADS_EZS_KILL", + "TSIx_FRAG_LZS_TEST", + "TSIx_FRAG_LZS_KILL", + "", + "TSIx_FRAG_PTILES", + "TSIx_FRAG_TRANS_ELIM", + "TSIx_QUAD_FPK_KILLER", + "", + "TSIx_COMPUTE_ACTIVE", + "TSIx_COMPUTE_TASKS", + "TSIx_COMPUTE_WARPS", + "TSIx_COMPUTE_STARVING", + "TSIx_EXEC_CORE_ACTIVE", + "TSIx_EXEC_ACTIVE", + "TSIx_EXEC_INSTR_COUNT", + "TSIx_EXEC_INSTR_DIVERGED", + "TSIx_EXEC_INSTR_STARVING", + "TSIx_ARITH_INSTR_SINGLE_FMA", + "TSIx_ARITH_INSTR_DOUBLE", + "TSIx_ARITH_INSTR_MSG", + "TSIx_ARITH_INSTR_MSG_ONLY", + "TSIx_TEX_MSGI_NUM_QUADS", + "TSIx_TEX_DFCH_NUM_PASSES", + "TSIx_TEX_DFCH_NUM_PASSES_MISS", + "TSIx_TEX_DFCH_NUM_PASSES_MIP_MAP", + "TSIx_TEX_TIDX_NUM_SPLIT_MIP_MAP", + "TSIx_TEX_TFCH_NUM_LINES_FETCHED", + "TSIx_TEX_TFCH_NUM_LINES_FETCHED_BLOCK_COMPRESSED", + "TSIx_TEX_TFCH_NUM_OPERATIONS", + "TSIx_TEX_FILT_NUM_OPERATIONS", + "TSIx_LS_MEM_READ_FULL", + "TSIx_LS_MEM_READ_SHORT", + "TSIx_LS_MEM_WRITE_FULL", + "TSIx_LS_MEM_WRITE_SHORT", + "TSIx_LS_MEM_ATOMIC", + "TSIx_VARY_INSTR", + "TSIx_VARY_SLOT_32", + "TSIx_VARY_SLOT_16", + "TSIx_ATTR_INSTR", + "TSIx_ARITH_INSTR_FP_MUL", + "TSIx_BEATS_RD_FTC", + "TSIx_BEATS_RD_FTC_EXT", + "TSIx_BEATS_RD_LSC", + "TSIx_BEATS_RD_LSC_EXT", + "TSIx_BEATS_RD_TEX", + "TSIx_BEATS_RD_TEX_EXT", + "TSIx_BEATS_RD_OTHER", + "TSIx_BEATS_WR_LSC", + "TSIx_BEATS_WR_TIB", + "", + + /* Performance counters for the Memory System */ + "", + "", + "", + "", + "TSIx_MMU_REQUESTS", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "TSIx_L2_RD_MSG_IN", + "TSIx_L2_RD_MSG_IN_STALL", + "TSIx_L2_WR_MSG_IN", + "TSIx_L2_WR_MSG_IN_STALL", + "TSIx_L2_SNP_MSG_IN", + "TSIx_L2_SNP_MSG_IN_STALL", + "TSIx_L2_RD_MSG_OUT", + "TSIx_L2_RD_MSG_OUT_STALL", + "TSIx_L2_WR_MSG_OUT", + "TSIx_L2_ANY_LOOKUP", + "TSIx_L2_READ_LOOKUP", + "TSIx_L2_WRITE_LOOKUP", + "TSIx_L2_EXT_SNOOP_LOOKUP", + "TSIx_L2_EXT_READ", + "TSIx_L2_EXT_READ_NOSNP", + "TSIx_L2_EXT_READ_UNIQUE", + "TSIx_L2_EXT_READ_BEATS", + "TSIx_L2_EXT_AR_STALL", + "TSIx_L2_EXT_AR_CNT_Q1", + "TSIx_L2_EXT_AR_CNT_Q2", + "TSIx_L2_EXT_AR_CNT_Q3", + "TSIx_L2_EXT_RRESP_0_127", + "TSIx_L2_EXT_RRESP_128_191", + "TSIx_L2_EXT_RRESP_192_255", + "TSIx_L2_EXT_RRESP_256_319", + "TSIx_L2_EXT_RRESP_320_383", + "TSIx_L2_EXT_WRITE", + "TSIx_L2_EXT_WRITE_NOSNP_FULL", + "TSIx_L2_EXT_WRITE_NOSNP_PTL", + "TSIx_L2_EXT_WRITE_SNP_FULL", + "TSIx_L2_EXT_WRITE_SNP_PTL", + "TSIx_L2_EXT_WRITE_BEATS", + "TSIx_L2_EXT_W_STALL", + "TSIx_L2_EXT_AW_CNT_Q1", + "TSIx_L2_EXT_AW_CNT_Q2", + "TSIx_L2_EXT_AW_CNT_Q3", + "TSIx_L2_EXT_SNOOP", + "TSIx_L2_EXT_SNOOP_STALL", + "TSIx_L2_EXT_SNOOP_RESP_CLEAN", + "TSIx_L2_EXT_SNOOP_RESP_DATA", + "TSIx_L2_EXT_SNOOP_INTERNAL", + "", + "", + "", + "", + "", + "", + "", +}; + +#endif /* _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpu_id.h b/drivers/gpu/arm/midgard/mali_kbase_gpu_id.h new file mode 100755 index 000000000000..42f0111c474f --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gpu_id.h @@ -0,0 +1,123 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +#ifndef _KBASE_GPU_ID_H_ +#define _KBASE_GPU_ID_H_ + +/* GPU_ID register */ +#define GPU_ID_VERSION_STATUS_SHIFT 0 +#define GPU_ID_VERSION_MINOR_SHIFT 4 +#define GPU_ID_VERSION_MAJOR_SHIFT 12 +#define GPU_ID_VERSION_PRODUCT_ID_SHIFT 16 +#define GPU_ID_VERSION_STATUS (0xF << GPU_ID_VERSION_STATUS_SHIFT) +#define GPU_ID_VERSION_MINOR (0xFF << GPU_ID_VERSION_MINOR_SHIFT) +#define GPU_ID_VERSION_MAJOR (0xF << GPU_ID_VERSION_MAJOR_SHIFT) +#define GPU_ID_VERSION_PRODUCT_ID (0xFFFF << GPU_ID_VERSION_PRODUCT_ID_SHIFT) + +/* Values for GPU_ID_VERSION_PRODUCT_ID bitfield */ +#define GPU_ID_PI_T60X 0x6956 +#define GPU_ID_PI_T62X 0x0620 +#define GPU_ID_PI_T76X 0x0750 +#define GPU_ID_PI_T72X 0x0720 +#define GPU_ID_PI_TFRX 0x0880 +#define GPU_ID_PI_T86X 0x0860 +#define GPU_ID_PI_T82X 0x0820 +#define GPU_ID_PI_T83X 0x0830 + +/* New GPU ID format when PRODUCT_ID is >= 0x1000 (and not 0x6956) */ +#define GPU_ID_PI_NEW_FORMAT_START 0x1000 +#define GPU_ID_IS_NEW_FORMAT(product_id) ((product_id) != GPU_ID_PI_T60X && \ + (product_id) >= \ + GPU_ID_PI_NEW_FORMAT_START) + +#define GPU_ID2_VERSION_STATUS_SHIFT 0 +#define GPU_ID2_VERSION_MINOR_SHIFT 4 +#define GPU_ID2_VERSION_MAJOR_SHIFT 12 +#define GPU_ID2_PRODUCT_MAJOR_SHIFT 16 +#define GPU_ID2_ARCH_REV_SHIFT 20 +#define GPU_ID2_ARCH_MINOR_SHIFT 24 +#define GPU_ID2_ARCH_MAJOR_SHIFT 28 +#define GPU_ID2_VERSION_STATUS (0xF << GPU_ID2_VERSION_STATUS_SHIFT) +#define GPU_ID2_VERSION_MINOR (0xFF << GPU_ID2_VERSION_MINOR_SHIFT) +#define GPU_ID2_VERSION_MAJOR (0xF << GPU_ID2_VERSION_MAJOR_SHIFT) +#define GPU_ID2_PRODUCT_MAJOR (0xF << GPU_ID2_PRODUCT_MAJOR_SHIFT) +#define GPU_ID2_ARCH_REV (0xF << GPU_ID2_ARCH_REV_SHIFT) +#define GPU_ID2_ARCH_MINOR (0xF << GPU_ID2_ARCH_MINOR_SHIFT) +#define GPU_ID2_ARCH_MAJOR (0xF << GPU_ID2_ARCH_MAJOR_SHIFT) +#define GPU_ID2_PRODUCT_MODEL (GPU_ID2_ARCH_MAJOR | GPU_ID2_PRODUCT_MAJOR) +#define GPU_ID2_VERSION (GPU_ID2_VERSION_MAJOR | \ + GPU_ID2_VERSION_MINOR | \ + GPU_ID2_VERSION_STATUS) + +/* Helper macro to create a partial GPU_ID (new format) that defines + a product ignoring its version. */ +#define GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, product_major) \ + (((arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ + ((arch_minor) << GPU_ID2_ARCH_MINOR_SHIFT) | \ + ((arch_rev) << GPU_ID2_ARCH_REV_SHIFT) | \ + ((product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) + +/* Helper macro to create a partial GPU_ID (new format) that specifies the + revision (major, minor, status) of a product */ +#define GPU_ID2_VERSION_MAKE(version_major, version_minor, version_status) \ + (((version_major) << GPU_ID2_VERSION_MAJOR_SHIFT) | \ + ((version_minor) << GPU_ID2_VERSION_MINOR_SHIFT) | \ + ((version_status) << GPU_ID2_VERSION_STATUS_SHIFT)) + +/* Helper macro to create a complete GPU_ID (new format) */ +#define GPU_ID2_MAKE(arch_major, arch_minor, arch_rev, product_major, \ + version_major, version_minor, version_status) \ + (GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, \ + product_major) | \ + GPU_ID2_VERSION_MAKE(version_major, version_minor, \ + version_status)) + +/* Helper macro to create a partial GPU_ID (new format) that identifies + a particular GPU model by its arch_major and product_major. */ +#define GPU_ID2_MODEL_MAKE(arch_major, product_major) \ + (((arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ + ((product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) + +/* Strip off the non-relevant bits from a product_id value and make it suitable + for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU + model. */ +#define GPU_ID2_MODEL_MATCH_VALUE(product_id) \ + (((product_id) << GPU_ID2_PRODUCT_MAJOR_SHIFT) & \ + GPU_ID2_PRODUCT_MODEL) + +#define GPU_ID2_PRODUCT_TMIX GPU_ID2_MODEL_MAKE(6u, 0) +#define GPU_ID2_PRODUCT_THEX GPU_ID2_MODEL_MAKE(6u, 1) +#define GPU_ID2_PRODUCT_TSIX GPU_ID2_MODEL_MAKE(7u, 0) +#ifdef MALI_INCLUDE_TKAX +#define GPU_ID2_PRODUCT_TKAX GPU_ID2_MODEL_MAKE(9u, 0) +#endif /* MALI_INCLUDE_TKAX */ +#ifdef MALI_INCLUDE_TTRX +#define GPU_ID2_PRODUCT_TTRX GPU_ID2_MODEL_MAKE(10u, 0) +#endif /* MALI_INCLUDE_TTRX */ + +/* Values for GPU_ID_VERSION_STATUS field for PRODUCT_ID GPU_ID_PI_T60X */ +#define GPU_ID_S_15DEV0 0x1 +#define GPU_ID_S_EAC 0x2 + +/* Helper macro to create a GPU_ID assuming valid values for id, major, + minor, status */ +#define GPU_ID_MAKE(id, major, minor, status) \ + (((id) << GPU_ID_VERSION_PRODUCT_ID_SHIFT) | \ + ((major) << GPU_ID_VERSION_MAJOR_SHIFT) | \ + ((minor) << GPU_ID_VERSION_MINOR_SHIFT) | \ + ((status) << GPU_ID_VERSION_STATUS_SHIFT)) + +#endif /* _KBASE_GPU_ID_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c new file mode 100755 index 000000000000..6df0a1cb1264 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c @@ -0,0 +1,97 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include + +#ifdef CONFIG_DEBUG_FS +/** Show callback for the @c gpu_memory debugfs file. + * + * This function is called to get the contents of the @c gpu_memory debugfs + * file. This is a report of current gpu memory usage. + * + * @param sfile The debugfs entry + * @param data Data associated with the entry + * + * @return 0 if successfully prints data in debugfs entry file + * -1 if it encountered an error + */ + +static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data) +{ + struct list_head *entry; + const struct list_head *kbdev_list; + + kbdev_list = kbase_dev_list_get(); + list_for_each(entry, kbdev_list) { + struct kbase_device *kbdev = NULL; + struct kbasep_kctx_list_element *element; + + kbdev = list_entry(entry, struct kbase_device, entry); + /* output the total memory usage and cap for this device */ + seq_printf(sfile, "%-16s %10u\n", + kbdev->devname, + atomic_read(&(kbdev->memdev.used_pages))); + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry(element, &kbdev->kctx_list, link) { + /* output the memory usage and cap for each kctx + * opened on this device */ + seq_printf(sfile, " %s-0x%p %10u\n", + "kctx", + element->kctx, + atomic_read(&(element->kctx->used_pages))); + } + mutex_unlock(&kbdev->kctx_list_lock); + } + kbase_dev_list_put(kbdev_list); + return 0; +} + +/* + * File operations related to debugfs entry for gpu_memory + */ +static int kbasep_gpu_memory_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_gpu_memory_seq_show , NULL); +} + +static const struct file_operations kbasep_gpu_memory_debugfs_fops = { + .open = kbasep_gpu_memory_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* + * Initialize debugfs entry for gpu_memory + */ +void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("gpu_memory", S_IRUGO, + kbdev->mali_debugfs_directory, NULL, + &kbasep_gpu_memory_debugfs_fops); + return; +} + +#else +/* + * Stub functions for when debugfs is disabled + */ +void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) +{ + return; +} +#endif diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h new file mode 100755 index 000000000000..7045693eb910 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h @@ -0,0 +1,37 @@ +/* + * + * (C) COPYRIGHT 2012-2014, 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_gpu_memory_debugfs.h + * Header file for gpu_memory entry in debugfs + * + */ + +#ifndef _KBASE_GPU_MEMORY_DEBUGFS_H +#define _KBASE_GPU_MEMORY_DEBUGFS_H + +#include +#include + +/** + * @brief Initialize gpu_memory debugfs entry + */ +void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev); + +#endif /*_KBASE_GPU_MEMORY_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c new file mode 100755 index 000000000000..a947a2e03a2f --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c @@ -0,0 +1,510 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Base kernel property query APIs + */ + +#include +#include +#include +#include +#include +#include "mali_kbase_ioctl.h" +#include + +/** + * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield. + * @value: The value from which to extract bits. + * @offset: The first bit to extract (0 being the LSB). + * @size: The number of bits to extract. + * + * Context: @offset + @size <= 32. + * + * Return: Bits [@offset, @offset + @size) from @value. + */ +/* from mali_cdsb.h */ +#define KBASE_UBFX32(value, offset, size) \ + (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1)) + +int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props) +{ + kbase_gpu_clk_speed_func get_gpu_speed_mhz; + u32 gpu_speed_mhz; + int rc = 1; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != kbase_props); + + /* Current GPU speed is requested from the system integrator via the GPU_SPEED_FUNC function. + * If that function fails, or the function is not provided by the system integrator, we report the maximum + * GPU speed as specified by GPU_FREQ_KHZ_MAX. + */ + get_gpu_speed_mhz = (kbase_gpu_clk_speed_func) GPU_SPEED_FUNC; + if (get_gpu_speed_mhz != NULL) { + rc = get_gpu_speed_mhz(&gpu_speed_mhz); +#ifdef CONFIG_MALI_DEBUG + /* Issue a warning message when the reported GPU speed falls outside the min/max range */ + if (rc == 0) { + u32 gpu_speed_khz = gpu_speed_mhz * 1000; + + if (gpu_speed_khz < kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min || + gpu_speed_khz > kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max) + dev_warn(kctx->kbdev->dev, "GPU Speed is outside of min/max range (got %lu Khz, min %lu Khz, max %lu Khz)\n", + (unsigned long)gpu_speed_khz, + (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min, + (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max); + } +#endif /* CONFIG_MALI_DEBUG */ + } + if (kctx->kbdev->clock) { + gpu_speed_mhz = clk_get_rate(kctx->kbdev->clock) / 1000000; + rc = 0; + } + if (rc != 0) + gpu_speed_mhz = kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max / 1000; + + kctx->kbdev->gpu_props.props.core_props.gpu_speed_mhz = gpu_speed_mhz; + + memcpy(&kbase_props->props, &kctx->kbdev->gpu_props.props, sizeof(kbase_props->props)); + + /* Before API 8.2 they expect L3 cache info here, which was always 0 */ + if (kctx->api_version < KBASE_API_VERSION(8, 2)) + kbase_props->props.raw_props.suspend_size = 0; + + return 0; +} + +static void kbase_gpuprops_construct_coherent_groups(base_gpu_props * const props) +{ + struct mali_base_gpu_coherent_group *current_group; + u64 group_present; + u64 group_mask; + u64 first_set, first_set_prev; + u32 num_groups = 0; + + KBASE_DEBUG_ASSERT(NULL != props); + + props->coherency_info.coherency = props->raw_props.mem_features; + props->coherency_info.num_core_groups = hweight64(props->raw_props.l2_present); + + if (props->coherency_info.coherency & GROUPS_L2_COHERENT) { + /* Group is l2 coherent */ + group_present = props->raw_props.l2_present; + } else { + /* Group is l1 coherent */ + group_present = props->raw_props.shader_present; + } + + /* + * The coherent group mask can be computed from the l2 present + * register. + * + * For the coherent group n: + * group_mask[n] = (first_set[n] - 1) & ~(first_set[n-1] - 1) + * where first_set is group_present with only its nth set-bit kept + * (i.e. the position from where a new group starts). + * + * For instance if the groups are l2 coherent and l2_present=0x0..01111: + * The first mask is: + * group_mask[1] = (first_set[1] - 1) & ~(first_set[0] - 1) + * = (0x0..010 - 1) & ~(0x0..01 - 1) + * = 0x0..00f + * The second mask is: + * group_mask[2] = (first_set[2] - 1) & ~(first_set[1] - 1) + * = (0x0..100 - 1) & ~(0x0..010 - 1) + * = 0x0..0f0 + * And so on until all the bits from group_present have been cleared + * (i.e. there is no group left). + */ + + current_group = props->coherency_info.group; + first_set = group_present & ~(group_present - 1); + + while (group_present != 0 && num_groups < BASE_MAX_COHERENT_GROUPS) { + group_present -= first_set; /* Clear the current group bit */ + first_set_prev = first_set; + + first_set = group_present & ~(group_present - 1); + group_mask = (first_set - 1) & ~(first_set_prev - 1); + + /* Populate the coherent_group structure for each group */ + current_group->core_mask = group_mask & props->raw_props.shader_present; + current_group->num_cores = hweight64(current_group->core_mask); + + num_groups++; + current_group++; + } + + if (group_present != 0) + pr_warn("Too many coherent groups (keeping only %d groups).\n", BASE_MAX_COHERENT_GROUPS); + + props->coherency_info.num_groups = num_groups; +} + +/** + * kbase_gpuprops_get_props - Get the GPU configuration + * @gpu_props: The &base_gpu_props structure + * @kbdev: The &struct kbase_device structure for the device + * + * Fill the &base_gpu_props structure with values from the GPU configuration + * registers. Only the raw properties are filled in this function + */ +static void kbase_gpuprops_get_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev) +{ + struct kbase_gpuprops_regdump regdump; + int i; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(NULL != gpu_props); + + /* Dump relevant registers */ + kbase_backend_gpuprops_get(kbdev, ®dump); + + gpu_props->raw_props.gpu_id = regdump.gpu_id; + gpu_props->raw_props.tiler_features = regdump.tiler_features; + gpu_props->raw_props.mem_features = regdump.mem_features; + gpu_props->raw_props.mmu_features = regdump.mmu_features; + gpu_props->raw_props.l2_features = regdump.l2_features; + gpu_props->raw_props.suspend_size = regdump.suspend_size; + + gpu_props->raw_props.as_present = regdump.as_present; + gpu_props->raw_props.js_present = regdump.js_present; + gpu_props->raw_props.shader_present = + ((u64) regdump.shader_present_hi << 32) + + regdump.shader_present_lo; + gpu_props->raw_props.tiler_present = + ((u64) regdump.tiler_present_hi << 32) + + regdump.tiler_present_lo; + gpu_props->raw_props.l2_present = + ((u64) regdump.l2_present_hi << 32) + + regdump.l2_present_lo; +#ifdef CONFIG_MALI_CORESTACK + gpu_props->raw_props.stack_present = + ((u64) regdump.stack_present_hi << 32) + + regdump.stack_present_lo; +#else /* CONFIG_MALI_CORESTACK */ + gpu_props->raw_props.stack_present = 0; +#endif /* CONFIG_MALI_CORESTACK */ + + for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) + gpu_props->raw_props.js_features[i] = regdump.js_features[i]; + + for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) + gpu_props->raw_props.texture_features[i] = regdump.texture_features[i]; + + gpu_props->raw_props.thread_max_barrier_size = regdump.thread_max_barrier_size; + gpu_props->raw_props.thread_max_threads = regdump.thread_max_threads; + gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size; + gpu_props->raw_props.thread_features = regdump.thread_features; +} + +void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props) +{ + gpu_props->core_props.version_status = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4); + gpu_props->core_props.minor_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8); + gpu_props->core_props.major_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4); + gpu_props->core_props.product_id = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16); +} + +/** + * kbase_gpuprops_calculate_props - Calculate the derived properties + * @gpu_props: The &base_gpu_props structure + * @kbdev: The &struct kbase_device structure for the device + * + * Fill the &base_gpu_props structure with values derived from the GPU + * configuration registers + */ +static void kbase_gpuprops_calculate_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev) +{ + int i; + + /* Populate the base_gpu_props structure */ + kbase_gpuprops_update_core_props_gpu_id(gpu_props); + gpu_props->core_props.log2_program_counter_size = KBASE_GPU_PC_SIZE_LOG2; + gpu_props->core_props.gpu_available_memory_size = totalram_pages() << PAGE_SHIFT; + + for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) + gpu_props->core_props.texture_features[i] = gpu_props->raw_props.texture_features[i]; + + gpu_props->l2_props.log2_line_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 0U, 8); + gpu_props->l2_props.log2_cache_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); + + /* Field with number of l2 slices is added to MEM_FEATURES register + * since t76x. Below code assumes that for older GPU reserved bits will + * be read as zero. */ + gpu_props->l2_props.num_l2_slices = + KBASE_UBFX32(gpu_props->raw_props.mem_features, 8U, 4) + 1; + + gpu_props->tiler_props.bin_size_bytes = 1 << KBASE_UBFX32(gpu_props->raw_props.tiler_features, 0U, 6); + gpu_props->tiler_props.max_active_levels = KBASE_UBFX32(gpu_props->raw_props.tiler_features, 8U, 4); + + if (gpu_props->raw_props.thread_max_threads == 0) + gpu_props->thread_props.max_threads = THREAD_MT_DEFAULT; + else + gpu_props->thread_props.max_threads = gpu_props->raw_props.thread_max_threads; + + if (gpu_props->raw_props.thread_max_workgroup_size == 0) + gpu_props->thread_props.max_workgroup_size = THREAD_MWS_DEFAULT; + else + gpu_props->thread_props.max_workgroup_size = gpu_props->raw_props.thread_max_workgroup_size; + + if (gpu_props->raw_props.thread_max_barrier_size == 0) + gpu_props->thread_props.max_barrier_size = THREAD_MBS_DEFAULT; + else + gpu_props->thread_props.max_barrier_size = gpu_props->raw_props.thread_max_barrier_size; + + gpu_props->thread_props.max_registers = KBASE_UBFX32(gpu_props->raw_props.thread_features, 0U, 16); + gpu_props->thread_props.max_task_queue = KBASE_UBFX32(gpu_props->raw_props.thread_features, 16U, 8); + gpu_props->thread_props.max_thread_group_split = KBASE_UBFX32(gpu_props->raw_props.thread_features, 24U, 6); + gpu_props->thread_props.impl_tech = KBASE_UBFX32(gpu_props->raw_props.thread_features, 30U, 2); + + /* If values are not specified, then use defaults */ + if (gpu_props->thread_props.max_registers == 0) { + gpu_props->thread_props.max_registers = THREAD_MR_DEFAULT; + gpu_props->thread_props.max_task_queue = THREAD_MTQ_DEFAULT; + gpu_props->thread_props.max_thread_group_split = THREAD_MTGS_DEFAULT; + } + /* Initialize the coherent_group structure for each group */ + kbase_gpuprops_construct_coherent_groups(gpu_props); +} + +void kbase_gpuprops_set(struct kbase_device *kbdev) +{ + struct kbase_gpu_props *gpu_props; + struct gpu_raw_gpu_props *raw; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + gpu_props = &kbdev->gpu_props; + raw = &gpu_props->props.raw_props; + + /* Initialize the base_gpu_props structure from the hardware */ + kbase_gpuprops_get_props(&gpu_props->props, kbdev); + + /* Populate the derived properties */ + kbase_gpuprops_calculate_props(&gpu_props->props, kbdev); + + /* Populate kbase-only fields */ + gpu_props->l2_props.associativity = KBASE_UBFX32(raw->l2_features, 8U, 8); + gpu_props->l2_props.external_bus_width = KBASE_UBFX32(raw->l2_features, 24U, 8); + + gpu_props->mem.core_group = KBASE_UBFX32(raw->mem_features, 0U, 1); + + gpu_props->mmu.va_bits = KBASE_UBFX32(raw->mmu_features, 0U, 8); + gpu_props->mmu.pa_bits = KBASE_UBFX32(raw->mmu_features, 8U, 8); + + gpu_props->num_cores = hweight64(raw->shader_present); + gpu_props->num_core_groups = hweight64(raw->l2_present); + gpu_props->num_address_spaces = hweight32(raw->as_present); + gpu_props->num_job_slots = hweight32(raw->js_present); +} + +void kbase_gpuprops_set_features(struct kbase_device *kbdev) +{ + base_gpu_props *gpu_props; + struct kbase_gpuprops_regdump regdump; + + gpu_props = &kbdev->gpu_props.props; + + /* Dump relevant registers */ + kbase_backend_gpuprops_get_features(kbdev, ®dump); + + /* + * Copy the raw value from the register, later this will get turned + * into the selected coherency mode. + * Additionally, add non-coherent mode, as this is always supported. + */ + gpu_props->raw_props.coherency_mode = regdump.coherency_features | + COHERENCY_FEATURE_BIT(COHERENCY_NONE); +} + +static struct { + u32 type; + size_t offset; + int size; +} gpu_property_mapping[] = { +#define PROP(name, member) \ + {KBASE_GPUPROP_ ## name, offsetof(struct mali_base_gpu_props, member), \ + sizeof(((struct mali_base_gpu_props *)0)->member)} + PROP(PRODUCT_ID, core_props.product_id), + PROP(VERSION_STATUS, core_props.version_status), + PROP(MINOR_REVISION, core_props.minor_revision), + PROP(MAJOR_REVISION, core_props.major_revision), + PROP(GPU_SPEED_MHZ, core_props.gpu_speed_mhz), + PROP(GPU_FREQ_KHZ_MAX, core_props.gpu_freq_khz_max), + PROP(GPU_FREQ_KHZ_MIN, core_props.gpu_freq_khz_min), + PROP(LOG2_PROGRAM_COUNTER_SIZE, core_props.log2_program_counter_size), + PROP(TEXTURE_FEATURES_0, core_props.texture_features[0]), + PROP(TEXTURE_FEATURES_1, core_props.texture_features[1]), + PROP(TEXTURE_FEATURES_2, core_props.texture_features[2]), + PROP(GPU_AVAILABLE_MEMORY_SIZE, core_props.gpu_available_memory_size), + + PROP(L2_LOG2_LINE_SIZE, l2_props.log2_line_size), + PROP(L2_LOG2_CACHE_SIZE, l2_props.log2_cache_size), + PROP(L2_NUM_L2_SLICES, l2_props.num_l2_slices), + + PROP(TILER_BIN_SIZE_BYTES, tiler_props.bin_size_bytes), + PROP(TILER_MAX_ACTIVE_LEVELS, tiler_props.max_active_levels), + + PROP(MAX_THREADS, thread_props.max_threads), + PROP(MAX_WORKGROUP_SIZE, thread_props.max_workgroup_size), + PROP(MAX_BARRIER_SIZE, thread_props.max_barrier_size), + PROP(MAX_REGISTERS, thread_props.max_registers), + PROP(MAX_TASK_QUEUE, thread_props.max_task_queue), + PROP(MAX_THREAD_GROUP_SPLIT, thread_props.max_thread_group_split), + PROP(IMPL_TECH, thread_props.impl_tech), + + PROP(RAW_SHADER_PRESENT, raw_props.shader_present), + PROP(RAW_TILER_PRESENT, raw_props.tiler_present), + PROP(RAW_L2_PRESENT, raw_props.l2_present), + PROP(RAW_STACK_PRESENT, raw_props.stack_present), + PROP(RAW_L2_FEATURES, raw_props.l2_features), + PROP(RAW_SUSPEND_SIZE, raw_props.suspend_size), + PROP(RAW_MEM_FEATURES, raw_props.mem_features), + PROP(RAW_MMU_FEATURES, raw_props.mmu_features), + PROP(RAW_AS_PRESENT, raw_props.as_present), + PROP(RAW_JS_PRESENT, raw_props.js_present), + PROP(RAW_JS_FEATURES_0, raw_props.js_features[0]), + PROP(RAW_JS_FEATURES_1, raw_props.js_features[1]), + PROP(RAW_JS_FEATURES_2, raw_props.js_features[2]), + PROP(RAW_JS_FEATURES_3, raw_props.js_features[3]), + PROP(RAW_JS_FEATURES_4, raw_props.js_features[4]), + PROP(RAW_JS_FEATURES_5, raw_props.js_features[5]), + PROP(RAW_JS_FEATURES_6, raw_props.js_features[6]), + PROP(RAW_JS_FEATURES_7, raw_props.js_features[7]), + PROP(RAW_JS_FEATURES_8, raw_props.js_features[8]), + PROP(RAW_JS_FEATURES_9, raw_props.js_features[9]), + PROP(RAW_JS_FEATURES_10, raw_props.js_features[10]), + PROP(RAW_JS_FEATURES_11, raw_props.js_features[11]), + PROP(RAW_JS_FEATURES_12, raw_props.js_features[12]), + PROP(RAW_JS_FEATURES_13, raw_props.js_features[13]), + PROP(RAW_JS_FEATURES_14, raw_props.js_features[14]), + PROP(RAW_JS_FEATURES_15, raw_props.js_features[15]), + PROP(RAW_TILER_FEATURES, raw_props.tiler_features), + PROP(RAW_TEXTURE_FEATURES_0, raw_props.texture_features[0]), + PROP(RAW_TEXTURE_FEATURES_1, raw_props.texture_features[1]), + PROP(RAW_TEXTURE_FEATURES_2, raw_props.texture_features[2]), + PROP(RAW_GPU_ID, raw_props.gpu_id), + PROP(RAW_THREAD_MAX_THREADS, raw_props.thread_max_threads), + PROP(RAW_THREAD_MAX_WORKGROUP_SIZE, + raw_props.thread_max_workgroup_size), + PROP(RAW_THREAD_MAX_BARRIER_SIZE, raw_props.thread_max_barrier_size), + PROP(RAW_THREAD_FEATURES, raw_props.thread_features), + PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode), + + PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups), + PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups), + PROP(COHERENCY_COHERENCY, coherency_info.coherency), + PROP(COHERENCY_GROUP_0, coherency_info.group[0].core_mask), + PROP(COHERENCY_GROUP_1, coherency_info.group[1].core_mask), + PROP(COHERENCY_GROUP_2, coherency_info.group[2].core_mask), + PROP(COHERENCY_GROUP_3, coherency_info.group[3].core_mask), + PROP(COHERENCY_GROUP_4, coherency_info.group[4].core_mask), + PROP(COHERENCY_GROUP_5, coherency_info.group[5].core_mask), + PROP(COHERENCY_GROUP_6, coherency_info.group[6].core_mask), + PROP(COHERENCY_GROUP_7, coherency_info.group[7].core_mask), + PROP(COHERENCY_GROUP_8, coherency_info.group[8].core_mask), + PROP(COHERENCY_GROUP_9, coherency_info.group[9].core_mask), + PROP(COHERENCY_GROUP_10, coherency_info.group[10].core_mask), + PROP(COHERENCY_GROUP_11, coherency_info.group[11].core_mask), + PROP(COHERENCY_GROUP_12, coherency_info.group[12].core_mask), + PROP(COHERENCY_GROUP_13, coherency_info.group[13].core_mask), + PROP(COHERENCY_GROUP_14, coherency_info.group[14].core_mask), + PROP(COHERENCY_GROUP_15, coherency_info.group[15].core_mask), + +#undef PROP +}; + +int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev) +{ + struct kbase_gpu_props *kprops = &kbdev->gpu_props; + struct mali_base_gpu_props *props = &kprops->props; + u32 count = ARRAY_SIZE(gpu_property_mapping); + u32 i; + u32 size = 0; + u8 *p; + + for (i = 0; i < count; i++) { + /* 4 bytes for the ID, and the size of the property */ + size += 4 + gpu_property_mapping[i].size; + } + + kprops->prop_buffer_size = size; + kprops->prop_buffer = kmalloc(size, GFP_KERNEL); + + if (!kprops->prop_buffer) { + kprops->prop_buffer_size = 0; + return -ENOMEM; + } + + p = kprops->prop_buffer; + +#define WRITE_U8(v) (*p++ = (v) & 0xFF) +#define WRITE_U16(v) do { WRITE_U8(v); WRITE_U8((v) >> 8); } while (0) +#define WRITE_U32(v) do { WRITE_U16(v); WRITE_U16((v) >> 16); } while (0) +#define WRITE_U64(v) do { WRITE_U32(v); WRITE_U32((v) >> 32); } while (0) + + for (i = 0; i < count; i++) { + u32 type = gpu_property_mapping[i].type; + u8 type_size; + void *field = ((u8 *)props) + gpu_property_mapping[i].offset; + + switch (gpu_property_mapping[i].size) { + case 1: + type_size = KBASE_GPUPROP_VALUE_SIZE_U8; + break; + case 2: + type_size = KBASE_GPUPROP_VALUE_SIZE_U16; + break; + case 4: + type_size = KBASE_GPUPROP_VALUE_SIZE_U32; + break; + case 8: + type_size = KBASE_GPUPROP_VALUE_SIZE_U64; + break; + default: + dev_err(kbdev->dev, + "Invalid gpu_property_mapping type=%d size=%d", + type, gpu_property_mapping[i].size); + return -EINVAL; + } + + WRITE_U32((type<<2) | type_size); + + switch (type_size) { + case KBASE_GPUPROP_VALUE_SIZE_U8: + WRITE_U8(*((u8 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U16: + WRITE_U16(*((u16 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U32: + WRITE_U32(*((u32 *)field)); + break; + case KBASE_GPUPROP_VALUE_SIZE_U64: + WRITE_U64(*((u64 *)field)); + break; + default: /* Cannot be reached */ + WARN_ON(1); + return -EINVAL; + } + } + + return 0; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h new file mode 100755 index 000000000000..57b3eaf9cd53 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h @@ -0,0 +1,84 @@ +/* + * + * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_gpuprops.h + * Base kernel property query APIs + */ + +#ifndef _KBASE_GPUPROPS_H_ +#define _KBASE_GPUPROPS_H_ + +#include "mali_kbase_gpuprops_types.h" + +/* Forward definition - see mali_kbase.h */ +struct kbase_device; + +/** + * @brief Set up Kbase GPU properties. + * + * Set up Kbase GPU properties with information from the GPU registers + * + * @param kbdev The struct kbase_device structure for the device + */ +void kbase_gpuprops_set(struct kbase_device *kbdev); + +/** + * kbase_gpuprops_set_features - Set up Kbase GPU properties + * @kbdev: Device pointer + * + * This function sets up GPU properties that are dependent on the hardware + * features bitmask. This function must be preceeded by a call to + * kbase_hw_set_features_mask(). + */ +void kbase_gpuprops_set_features(struct kbase_device *kbdev); + +/** + * @brief Provide GPU properties to userside through UKU call. + * + * Fill the struct kbase_uk_gpuprops with values from GPU configuration registers. + * + * @param kctx The struct kbase_context structure + * @param kbase_props A copy of the struct kbase_uk_gpuprops structure from userspace + * + * @return 0 on success. Any other value indicates failure. + */ +int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props); + +/** + * kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer + * @kbdev: The kbase device + * + * Fills kbdev->gpu_props->prop_buffer with the GPU properties for user + * space to read. + */ +int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev); + +/** + * kbase_gpuprops_update_core_props_gpu_id - break down gpu id value + * @gpu_props: the &base_gpu_props structure + * + * Break down gpu_id value stored in base_gpu_props::raw_props.gpu_id into + * separate fields (version_status, minor_revision, major_revision, product_id) + * stored in base_gpu_props::core_props. + */ +void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props); + + +#endif /* _KBASE_GPUPROPS_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h b/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h new file mode 100755 index 000000000000..10794fc27318 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h @@ -0,0 +1,92 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_gpuprops_types.h + * Base kernel property query APIs + */ + +#ifndef _KBASE_GPUPROPS_TYPES_H_ +#define _KBASE_GPUPROPS_TYPES_H_ + +#include "mali_base_kernel.h" + +#define KBASE_GPU_SPEED_MHZ 123 +#define KBASE_GPU_PC_SIZE_LOG2 24U + +struct kbase_gpuprops_regdump { + u32 gpu_id; + u32 l2_features; + u32 suspend_size; /* API 8.2+ */ + u32 tiler_features; + u32 mem_features; + u32 mmu_features; + u32 as_present; + u32 js_present; + u32 thread_max_threads; + u32 thread_max_workgroup_size; + u32 thread_max_barrier_size; + u32 thread_features; + u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; + u32 js_features[GPU_MAX_JOB_SLOTS]; + u32 shader_present_lo; + u32 shader_present_hi; + u32 tiler_present_lo; + u32 tiler_present_hi; + u32 l2_present_lo; + u32 l2_present_hi; + u32 stack_present_lo; + u32 stack_present_hi; + u32 coherency_features; +}; + +struct kbase_gpu_cache_props { + u8 associativity; + u8 external_bus_width; +}; + +struct kbase_gpu_mem_props { + u8 core_group; +}; + +struct kbase_gpu_mmu_props { + u8 va_bits; + u8 pa_bits; +}; + +struct kbase_gpu_props { + /* kernel-only properties */ + u8 num_cores; + u8 num_core_groups; + u8 num_address_spaces; + u8 num_job_slots; + + struct kbase_gpu_cache_props l2_props; + + struct kbase_gpu_mem_props mem; + struct kbase_gpu_mmu_props mmu; + + /* Properties shared with userspace */ + base_gpu_props props; + + u32 prop_buffer_size; + void *prop_buffer; +}; + +#endif /* _KBASE_GPUPROPS_TYPES_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_hw.c b/drivers/gpu/arm/midgard/mali_kbase_hw.c new file mode 100755 index 000000000000..9a390d233939 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_hw.c @@ -0,0 +1,453 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * Run-time work-arounds helpers + */ + +#include +#include +#include +#include "mali_kbase.h" +#include "mali_kbase_hw.h" + +void kbase_hw_set_features_mask(struct kbase_device *kbdev) +{ + const enum base_hw_feature *features; + u32 gpu_id; + u32 product_id; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; + product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; + + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { + case GPU_ID2_PRODUCT_TMIX: + features = base_hw_features_tMIx; + break; + case GPU_ID2_PRODUCT_THEX: + features = base_hw_features_tHEx; + break; + case GPU_ID2_PRODUCT_TSIX: + features = base_hw_features_tSIx; + break; +#ifdef MALI_INCLUDE_TKAX + case GPU_ID2_PRODUCT_TKAX: + features = base_hw_features_tKAx; + break; +#endif /* MALI_INCLUDE_TKAX */ +#ifdef MALI_INCLUDE_TTRX + case GPU_ID2_PRODUCT_TTRX: + features = base_hw_features_tTRx; + break; +#endif /* MALI_INCLUDE_TTRX */ + default: + features = base_hw_features_generic; + break; + } + } else { + switch (product_id) { + case GPU_ID_PI_TFRX: + /* FALLTHROUGH */ + case GPU_ID_PI_T86X: + features = base_hw_features_tFxx; + break; + case GPU_ID_PI_T83X: + features = base_hw_features_t83x; + break; + case GPU_ID_PI_T82X: + features = base_hw_features_t82x; + break; + case GPU_ID_PI_T76X: + features = base_hw_features_t76x; + break; + case GPU_ID_PI_T72X: + features = base_hw_features_t72x; + break; + case GPU_ID_PI_T62X: + features = base_hw_features_t62x; + break; + case GPU_ID_PI_T60X: + features = base_hw_features_t60x; + break; + default: + features = base_hw_features_generic; + break; + } + } + + for (; *features != BASE_HW_FEATURE_END; features++) + set_bit(*features, &kbdev->hw_features_mask[0]); +} + +/** + * kbase_hw_get_issues_for_new_id - Get the hardware issues for a new GPU ID + * @kbdev: Device pointer + * + * Return: pointer to an array of hardware issues, terminated by + * BASE_HW_ISSUE_END. + * + * This function can only be used on new-format GPU IDs, i.e. those for which + * GPU_ID_IS_NEW_FORMAT evaluates as true. The GPU ID is read from the @kbdev. + * + * In debugging versions of the driver, unknown versions of a known GPU will + * be treated as the most recent known version not later than the actual + * version. In such circumstances, the GPU ID in @kbdev will also be replaced + * with the most recent known version. + * + * Note: The GPU configuration must have been read by kbase_gpuprops_get_props() + * before calling this function. + */ +static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( + struct kbase_device *kbdev) +{ + const enum base_hw_issue *issues = NULL; + + struct base_hw_product { + u32 product_model; + struct { + u32 version; + const enum base_hw_issue *issues; + } map[7]; + }; + + static const struct base_hw_product base_hw_products[] = { + {GPU_ID2_PRODUCT_TMIX, + {{GPU_ID2_VERSION_MAKE(0, 0, 1), + base_hw_issues_tMIx_r0p0_05dev0}, + {GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0}, + {U32_MAX /* sentinel value */, NULL} } }, + + {GPU_ID2_PRODUCT_THEX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1}, + {U32_MAX, NULL} } }, + + {GPU_ID2_PRODUCT_TSIX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0}, + {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1}, + {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tSIx_r0p1}, + {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0}, + {GPU_ID2_VERSION_MAKE(1, 0, 1), base_hw_issues_tSIx_r1p0}, + {U32_MAX, NULL} } }, + + +#ifdef MALI_INCLUDE_TKAX + {GPU_ID2_PRODUCT_TKAX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tKAx_r0p0}, + {U32_MAX, NULL} } }, +#endif /* MALI_INCLUDE_TKAX */ + +#ifdef MALI_INCLUDE_TTRX + {GPU_ID2_PRODUCT_TTRX, + {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTRx_r0p0}, + {U32_MAX, NULL} } }, +#endif /* MALI_INCLUDE_TTRX */ + }; + + u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + const u32 product_model = gpu_id & GPU_ID2_PRODUCT_MODEL; + const struct base_hw_product *product = NULL; + size_t p; + + /* Stop when we reach the end of the products array. */ + for (p = 0; p < ARRAY_SIZE(base_hw_products); ++p) { + if (product_model == base_hw_products[p].product_model) { + product = &base_hw_products[p]; + break; + } + } + + if (product != NULL) { + /* Found a matching product. */ + const u32 version = gpu_id & GPU_ID2_VERSION; + u32 fallback_version = 0; + const enum base_hw_issue *fallback_issues = NULL; + size_t v; + + /* Stop when we reach the end of the map. */ + for (v = 0; product->map[v].version != U32_MAX; ++v) { + + if (version == product->map[v].version) { + /* Exact match so stop. */ + issues = product->map[v].issues; + break; + } + + /* Check whether this is a candidate for most recent + known version not later than the actual + version. */ + if ((version > product->map[v].version) && + (product->map[v].version >= fallback_version)) { + fallback_version = product->map[v].version; + fallback_issues = product->map[v].issues; + } + } + + if ((issues == NULL) && (fallback_issues != NULL)) { + /* Fall back to the issue set of the most recent known + version not later than the actual version. */ + issues = fallback_issues; + + dev_info(kbdev->dev, + "r%dp%d status %d is unknown; treating as r%dp%d status %d", + (gpu_id & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT, + (fallback_version & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (fallback_version & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (fallback_version & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT); + + gpu_id &= ~GPU_ID2_VERSION; + gpu_id |= fallback_version; + kbdev->gpu_props.props.raw_props.gpu_id = gpu_id; + + kbase_gpuprops_update_core_props_gpu_id(&kbdev->gpu_props.props); + } + } + return issues; +} + +int kbase_hw_set_issues_mask(struct kbase_device *kbdev) +{ + const enum base_hw_issue *issues; + u32 gpu_id; + u32 product_id; + u32 impl_tech; + + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; + product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; + impl_tech = kbdev->gpu_props.props.thread_props.impl_tech; + + if (impl_tech != IMPLEMENTATION_MODEL) { + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + issues = kbase_hw_get_issues_for_new_id(kbdev); + if (issues == NULL) { + dev_err(kbdev->dev, + "Unknown GPU ID %x", gpu_id); + return -EINVAL; + } + + /* The GPU ID might have been replaced with the last + known version of the same GPU. */ + gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; + + } else { + switch (gpu_id) { + case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_15DEV0): + issues = base_hw_issues_t60x_r0p0_15dev0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_EAC): + issues = base_hw_issues_t60x_r0p0_eac; + break; + case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 1, 0): + issues = base_hw_issues_t60x_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T62X, 0, 1, 0): + issues = base_hw_issues_t62x_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 1): + issues = base_hw_issues_t62x_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 1, 0): + issues = base_hw_issues_t62x_r1p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 0, 1): + issues = base_hw_issues_t76x_r0p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 1, 1): + issues = base_hw_issues_t76x_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 1, 9): + issues = base_hw_issues_t76x_r0p1_50rel0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 2, 1): + issues = base_hw_issues_t76x_r0p2; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 3, 1): + issues = base_hw_issues_t76x_r0p3; + break; + case GPU_ID_MAKE(GPU_ID_PI_T76X, 1, 0, 0): + issues = base_hw_issues_t76x_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 1): + case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 2): + issues = base_hw_issues_t72x_r0p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 0, 0): + issues = base_hw_issues_t72x_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 1, 0): + issues = base_hw_issues_t72x_r1p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 1, 2): + issues = base_hw_issues_tFRx_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 2, 0): + issues = base_hw_issues_tFRx_r0p2; + break; + case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 8): + issues = base_hw_issues_tFRx_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_TFRX, 2, 0, 0): + issues = base_hw_issues_tFRx_r2p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T86X, 0, 2, 0): + issues = base_hw_issues_t86x_r0p2; + break; + case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 8): + issues = base_hw_issues_t86x_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T86X, 2, 0, 0): + issues = base_hw_issues_t86x_r2p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T83X, 0, 1, 0): + issues = base_hw_issues_t83x_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 8): + issues = base_hw_issues_t83x_r1p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 0, 0): + issues = base_hw_issues_t82x_r0p0; + break; + case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 1, 0): + issues = base_hw_issues_t82x_r0p1; + break; + case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 0): + case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 8): + issues = base_hw_issues_t82x_r1p0; + break; + default: + dev_err(kbdev->dev, + "Unknown GPU ID %x", gpu_id); + return -EINVAL; + } + } + } else { + /* Software model */ + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { + case GPU_ID2_PRODUCT_TMIX: + issues = base_hw_issues_model_tMIx; + break; + case GPU_ID2_PRODUCT_THEX: + issues = base_hw_issues_model_tHEx; + break; + case GPU_ID2_PRODUCT_TSIX: + issues = base_hw_issues_model_tSIx; + break; +#ifdef MALI_INCLUDE_TKAX + case GPU_ID2_PRODUCT_TKAX: + issues = base_hw_issues_model_tKAx; + break; +#endif /* MALI_INCLUDE_TKAX */ +#ifdef MALI_INCLUDE_TTRX + case GPU_ID2_PRODUCT_TTRX: + issues = base_hw_issues_model_tTRx; + break; +#endif /* MALI_INCLUDE_TTRX */ + default: + dev_err(kbdev->dev, + "Unknown GPU ID %x", gpu_id); + return -EINVAL; + } + } else { + switch (product_id) { + case GPU_ID_PI_T60X: + issues = base_hw_issues_model_t60x; + break; + case GPU_ID_PI_T62X: + issues = base_hw_issues_model_t62x; + break; + case GPU_ID_PI_T72X: + issues = base_hw_issues_model_t72x; + break; + case GPU_ID_PI_T76X: + issues = base_hw_issues_model_t76x; + break; + case GPU_ID_PI_TFRX: + issues = base_hw_issues_model_tFRx; + break; + case GPU_ID_PI_T86X: + issues = base_hw_issues_model_t86x; + break; + case GPU_ID_PI_T83X: + issues = base_hw_issues_model_t83x; + break; + case GPU_ID_PI_T82X: + issues = base_hw_issues_model_t82x; + break; + default: + dev_err(kbdev->dev, "Unknown GPU ID %x", + gpu_id); + return -EINVAL; + } + } + } + + if (GPU_ID_IS_NEW_FORMAT(product_id)) { + dev_info(kbdev->dev, + "GPU identified as 0x%x arch %d.%d.%d r%dp%d status %d", + (gpu_id & GPU_ID2_PRODUCT_MAJOR) >> + GPU_ID2_PRODUCT_MAJOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_MAJOR) >> + GPU_ID2_ARCH_MAJOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_MINOR) >> + GPU_ID2_ARCH_MINOR_SHIFT, + (gpu_id & GPU_ID2_ARCH_REV) >> + GPU_ID2_ARCH_REV_SHIFT, + (gpu_id & GPU_ID2_VERSION_MAJOR) >> + GPU_ID2_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_MINOR) >> + GPU_ID2_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID2_VERSION_STATUS) >> + GPU_ID2_VERSION_STATUS_SHIFT); + } else { + dev_info(kbdev->dev, + "GPU identified as 0x%04x r%dp%d status %d", + (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> + GPU_ID_VERSION_PRODUCT_ID_SHIFT, + (gpu_id & GPU_ID_VERSION_MAJOR) >> + GPU_ID_VERSION_MAJOR_SHIFT, + (gpu_id & GPU_ID_VERSION_MINOR) >> + GPU_ID_VERSION_MINOR_SHIFT, + (gpu_id & GPU_ID_VERSION_STATUS) >> + GPU_ID_VERSION_STATUS_SHIFT); + } + + for (; *issues != BASE_HW_ISSUE_END; issues++) + set_bit(*issues, &kbdev->hw_issues_mask[0]); + + return 0; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_hw.h b/drivers/gpu/arm/midgard/mali_kbase_hw.h new file mode 100755 index 000000000000..754250ce968d --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_hw.h @@ -0,0 +1,65 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file + * Run-time work-arounds helpers + */ + +#ifndef _KBASE_HW_H_ +#define _KBASE_HW_H_ + +#include "mali_kbase_defs.h" + +/** + * @brief Tell whether a work-around should be enabled + */ +#define kbase_hw_has_issue(kbdev, issue)\ + test_bit(issue, &(kbdev)->hw_issues_mask[0]) + +/** + * @brief Tell whether a feature is supported + */ +#define kbase_hw_has_feature(kbdev, feature)\ + test_bit(feature, &(kbdev)->hw_features_mask[0]) + +/** + * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID + * @kbdev: Device pointer + * + * Return: 0 if the GPU ID was recognized, otherwise -EINVAL. + * + * The GPU ID is read from the @kbdev. + * + * In debugging versions of the driver, unknown versions of a known GPU with a + * new-format ID will be treated as the most recent known version not later + * than the actual version. In such circumstances, the GPU ID in @kbdev will + * also be replaced with the most recent known version. + * + * Note: The GPU configuration must have been read by + * kbase_gpuprops_get_props() before calling this function. + */ +int kbase_hw_set_issues_mask(struct kbase_device *kbdev); + +/** + * @brief Set the features mask depending on the GPU ID + */ +void kbase_hw_set_features_mask(struct kbase_device *kbdev); + +#endif /* _KBASE_HW_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h new file mode 100755 index 000000000000..b09be99e6b4e --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h @@ -0,0 +1,54 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * HW access backend common APIs + */ + +#ifndef _KBASE_HWACCESS_BACKEND_H_ +#define _KBASE_HWACCESS_BACKEND_H_ + +/** + * kbase_backend_early_init - Perform any backend-specific initialization. + * @kbdev: Device pointer + * + * Return: 0 on success, or an error code on failure. + */ +int kbase_backend_early_init(struct kbase_device *kbdev); + +/** + * kbase_backend_late_init - Perform any backend-specific initialization. + * @kbdev: Device pointer + * + * Return: 0 on success, or an error code on failure. + */ +int kbase_backend_late_init(struct kbase_device *kbdev); + +/** + * kbase_backend_early_term - Perform any backend-specific termination. + * @kbdev: Device pointer + */ +void kbase_backend_early_term(struct kbase_device *kbdev); + +/** + * kbase_backend_late_term - Perform any backend-specific termination. + * @kbdev: Device pointer + */ +void kbase_backend_late_term(struct kbase_device *kbdev); + +#endif /* _KBASE_HWACCESS_BACKEND_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h new file mode 100755 index 000000000000..0acf297192fd --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h @@ -0,0 +1,36 @@ +/* + * + * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/** + * @file mali_kbase_hwaccess_gpu_defs.h + * HW access common definitions + */ + +#ifndef _KBASE_HWACCESS_DEFS_H_ +#define _KBASE_HWACCESS_DEFS_H_ + +#include + +/* The hwaccess_lock (a spinlock) must be held when accessing this structure */ +struct kbase_hwaccess_data { + struct kbase_context *active_kctx; + + struct kbase_backend_data backend; +}; + +#endif /* _KBASE_HWACCESS_DEFS_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h new file mode 100755 index 000000000000..cf8a8131c22e --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h @@ -0,0 +1,47 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/** + * Base kernel property query backend APIs + */ + +#ifndef _KBASE_HWACCESS_GPUPROPS_H_ +#define _KBASE_HWACCESS_GPUPROPS_H_ + +/** + * kbase_backend_gpuprops_get() - Fill @regdump with GPU properties read from + * GPU + * @kbdev: Device pointer + * @regdump: Pointer to struct kbase_gpuprops_regdump structure + */ +void kbase_backend_gpuprops_get(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump); + +/** + * kbase_backend_gpuprops_get - Fill @regdump with GPU properties read from GPU + * @kbdev: Device pointer + * @regdump: Pointer to struct kbase_gpuprops_regdump structure + * + * This function reads GPU properties that are dependent on the hardware + * features bitmask + */ +void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, + struct kbase_gpuprops_regdump *regdump); + + +#endif /* _KBASE_HWACCESS_GPUPROPS_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h new file mode 100755 index 000000000000..5de2b7535bb4 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h @@ -0,0 +1,116 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* + * HW Access instrumentation common APIs + */ + +#ifndef _KBASE_HWACCESS_INSTR_H_ +#define _KBASE_HWACCESS_INSTR_H_ + +#include + +/** + * kbase_instr_hwcnt_enable_internal - Enable HW counters collection + * @kbdev: Kbase device + * @kctx: Kbase context + * @setup: HW counter setup parameters + * + * Context: might sleep, waiting for reset to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_uk_hwcnt_setup *setup); + +/** + * kbase_instr_hwcnt_disable_internal - Disable HW counters collection + * @kctx: Kbase context + * + * Context: might sleep, waiting for an ongoing dump to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx); + +/** + * kbase_instr_hwcnt_request_dump() - Request HW counter dump from GPU + * @kctx: Kbase context + * + * Caller must either wait for kbase_instr_hwcnt_dump_complete() to return true, + * of call kbase_instr_hwcnt_wait_for_dump(). + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx); + +/** + * kbase_instr_hwcnt_wait_for_dump() - Wait until pending HW counter dump has + * completed. + * @kctx: Kbase context + * + * Context: will sleep, waiting for dump to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx); + +/** + * kbase_instr_hwcnt_dump_complete - Tell whether the HW counters dump has + * completed + * @kctx: Kbase context + * @success: Set to true if successful + * + * Context: does not sleep. + * + * Return: true if the dump is complete + */ +bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, + bool * const success); + +/** + * kbase_instr_hwcnt_clear() - Clear HW counters + * @kctx: Kbase context + * + * Context: might sleep, waiting for reset to complete + * + * Return: 0 on success + */ +int kbase_instr_hwcnt_clear(struct kbase_context *kctx); + +/** + * kbase_instr_backend_init() - Initialise the instrumentation backend + * @kbdev: Kbase device + * + * This function should be called during driver initialization. + * + * Return: 0 on success + */ +int kbase_instr_backend_init(struct kbase_device *kbdev); + +/** + * kbase_instr_backend_init() - Terminate the instrumentation backend + * @kbdev: Kbase device + * + * This function should be called during driver termination. + */ +void kbase_instr_backend_term(struct kbase_device *kbdev); + +#endif /* _KBASE_HWACCESS_INSTR_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h new file mode 100755 index 000000000000..750fda2cd81d --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h @@ -0,0 +1,381 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * HW access job manager common APIs + */ + +#ifndef _KBASE_HWACCESS_JM_H_ +#define _KBASE_HWACCESS_JM_H_ + +/** + * kbase_backend_run_atom() - Run an atom on the GPU + * @kbdev: Device pointer + * @atom: Atom to run + * + * Caller must hold the HW access lock + */ +void kbase_backend_run_atom(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_backend_slot_update - Update state based on slot ringbuffers + * + * @kbdev: Device pointer + * + * Inspect the jobs in the slot ringbuffers and update state. + * + * This will cause jobs to be submitted to hardware if they are unblocked + */ +void kbase_backend_slot_update(struct kbase_device *kbdev); + +/** + * kbase_backend_find_and_release_free_address_space() - Release a free AS + * @kbdev: Device pointer + * @kctx: Context pointer + * + * This function can evict an idle context from the runpool, freeing up the + * address space it was using. + * + * The address space is marked as in use. The caller must either assign a + * context using kbase_gpu_use_ctx(), or release it using + * kbase_ctx_sched_release() + * + * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none + * available + */ +int kbase_backend_find_and_release_free_address_space( + struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the + * provided address space. + * @kbdev: Device pointer + * @kctx: Context pointer. May be NULL + * @as_nr: Free address space to use + * + * kbase_gpu_next_job() will pull atoms from the active context. + * + * Return: true if successful, false if ASID not assigned. + */ +bool kbase_backend_use_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, + int as_nr); + +/** + * kbase_backend_use_ctx_sched() - Activate a context. + * @kbdev: Device pointer + * @kctx: Context pointer + * + * kbase_gpu_next_job() will pull atoms from the active context. + * + * The context must already be scheduled and assigned to an address space. If + * the context is not scheduled, then kbase_gpu_use_ctx() should be used + * instead. + * + * Caller must hold hwaccess_lock + * + * Return: true if context is now active, false otherwise (ie if context does + * not have an address space assigned) + */ +bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbase_backend_release_ctx_irq - Release a context from the GPU. This will + * de-assign the assigned address space. + * @kbdev: Device pointer + * @kctx: Context pointer + * + * Caller must hold kbase_device->mmu_hw_mutex and hwaccess_lock + */ +void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbase_backend_release_ctx_noirq - Release a context from the GPU. This will + * de-assign the assigned address space. + * @kbdev: Device pointer + * @kctx: Context pointer + * + * Caller must hold kbase_device->mmu_hw_mutex + * + * This function must perform any operations that could not be performed in IRQ + * context by kbase_backend_release_ctx_irq(). + */ +void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * kbase_backend_cacheclean - Perform a cache clean if the given atom requires + * one + * @kbdev: Device pointer + * @katom: Pointer to the failed atom + * + * On some GPUs, the GPU cache must be cleaned following a failed atom. This + * function performs a clean if it is required by @katom. + */ +void kbase_backend_cacheclean(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + + +/** + * kbase_backend_complete_wq() - Perform backend-specific actions required on + * completing an atom. + * @kbdev: Device pointer + * @katom: Pointer to the atom to complete + * + * This function should only be called from kbase_jd_done_worker() or + * js_return_worker(). + * + * Return: true if atom has completed, false if atom should be re-submitted + */ +void kbase_backend_complete_wq(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_backend_complete_wq_post_sched - Perform backend-specific actions + * required on completing an atom, after + * any scheduling has taken place. + * @kbdev: Device pointer + * @core_req: Core requirements of atom + * @affinity: Affinity of atom + * @coreref_state: Coreref state of atom + * + * This function should only be called from kbase_jd_done_worker() or + * js_return_worker(). + */ +void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, + base_jd_core_req core_req, u64 affinity, + enum kbase_atom_coreref_state coreref_state); + +/** + * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU + * and remove any others from the ringbuffers. + * @kbdev: Device pointer + * @end_timestamp: Timestamp of reset + */ +void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp); + +/** + * kbase_backend_inspect_head() - Return the atom currently at the head of slot + * @js + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Atom currently at the head of slot @js, or NULL + */ +struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, + int js); + +/** + * kbase_backend_inspect_tail - Return the atom currently at the tail of slot + * @js + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Atom currently at the head of slot @js, or NULL + */ +struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, + int js); + +/** + * kbase_backend_nr_atoms_on_slot() - Return the number of atoms currently on a + * slot. + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Number of atoms currently on slot + */ +int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js); + +/** + * kbase_backend_nr_atoms_submitted() - Return the number of atoms on a slot + * that are currently on the GPU. + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Number of atoms currently on slot @js that are currently on the GPU. + */ +int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js); + +/** + * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs + * has changed. + * @kbdev: Device pointer + * + * Perform any required backend-specific actions (eg starting/stopping + * scheduling timers). + */ +void kbase_backend_ctx_count_changed(struct kbase_device *kbdev); + +/** + * kbase_backend_timeouts_changed() - Job Scheduler timeouts have changed. + * @kbdev: Device pointer + * + * Perform any required backend-specific actions (eg updating timeouts of + * currently running atoms). + */ +void kbase_backend_timeouts_changed(struct kbase_device *kbdev); + +/** + * kbase_backend_slot_free() - Return the number of jobs that can be currently + * submitted to slot @js. + * @kbdev: Device pointer + * @js: Job slot to inspect + * + * Return : Number of jobs that can be submitted. + */ +int kbase_backend_slot_free(struct kbase_device *kbdev, int js); + +/** + * kbase_job_check_enter_disjoint - potentially leave disjoint state + * @kbdev: kbase device + * @target_katom: atom which is finishing + * + * Work out whether to leave disjoint state when finishing an atom that was + * originated by kbase_job_check_enter_disjoint(). + */ +void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, + struct kbase_jd_atom *target_katom); + +/** + * kbase_backend_jm_kill_jobs_from_kctx - Kill all jobs that are currently + * running from a context + * @kctx: Context pointer + * + * This is used in response to a page fault to remove all jobs from the faulting + * context from the hardware. + */ +void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx); + +/** + * kbase_jm_wait_for_zero_jobs - Wait for context to have zero jobs running, and + * to be descheduled. + * @kctx: Context pointer + * + * This should be called following kbase_js_zap_context(), to ensure the context + * can be safely destroyed. + */ +void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx); + +/** + * kbase_backend_get_current_flush_id - Return the current flush ID + * + * @kbdev: Device pointer + * + * Return: the current flush ID to be recorded for each job chain + */ +u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev); + +#if KBASE_GPU_RESET_EN +/** + * kbase_prepare_to_reset_gpu - Prepare for resetting the GPU. + * @kbdev: Device pointer + * + * This function just soft-stops all the slots to ensure that as many jobs as + * possible are saved. + * + * Return: a boolean which should be interpreted as follows: + * - true - Prepared for reset, kbase_reset_gpu should be called. + * - false - Another thread is performing a reset, kbase_reset_gpu should + * not be called. + */ +bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu - Reset the GPU + * @kbdev: Device pointer + * + * This function should be called after kbase_prepare_to_reset_gpu if it returns + * true. It should never be called without a corresponding call to + * kbase_prepare_to_reset_gpu. + * + * After this function is called (or not called if kbase_prepare_to_reset_gpu + * returned false), the caller should wait for kbdev->reset_waitq to be + * signalled to know when the reset has completed. + */ +void kbase_reset_gpu(struct kbase_device *kbdev); + +/** + * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU. + * @kbdev: Device pointer + * + * This function just soft-stops all the slots to ensure that as many jobs as + * possible are saved. + * + * Return: a boolean which should be interpreted as follows: + * - true - Prepared for reset, kbase_reset_gpu should be called. + * - false - Another thread is performing a reset, kbase_reset_gpu should + * not be called. + */ +bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_locked - Reset the GPU + * @kbdev: Device pointer + * + * This function should be called after kbase_prepare_to_reset_gpu if it + * returns true. It should never be called without a corresponding call to + * kbase_prepare_to_reset_gpu. + * + * After this function is called (or not called if kbase_prepare_to_reset_gpu + * returned false), the caller should wait for kbdev->reset_waitq to be + * signalled to know when the reset has completed. + */ +void kbase_reset_gpu_locked(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_silent - Reset the GPU silently + * @kbdev: Device pointer + * + * Reset the GPU without trying to cancel jobs and don't emit messages into + * the kernel log while doing the reset. + * + * This function should be used in cases where we are doing a controlled reset + * of the GPU as part of normal processing (e.g. exiting protected mode) where + * the driver will have ensured the scheduler has been idled and all other + * users of the GPU (e.g. instrumentation) have been suspended. + */ +void kbase_reset_gpu_silent(struct kbase_device *kbdev); + +/** + * kbase_reset_gpu_active - Reports if the GPU is being reset + * @kbdev: Device pointer + * + * Return: True if the GPU is in the process of being reset. + */ +bool kbase_reset_gpu_active(struct kbase_device *kbdev); +#endif + +/** + * kbase_job_slot_hardstop - Hard-stop the specified job slot + * @kctx: The kbase context that contains the job(s) that should + * be hard-stopped + * @js: The job slot to hard-stop + * @target_katom: The job that should be hard-stopped (or NULL for all + * jobs from the context) + * Context: + * The job slot lock must be held when calling this function. + */ +void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, + struct kbase_jd_atom *target_katom); + +extern struct protected_mode_ops kbase_native_protected_ops; + +#endif /* _KBASE_HWACCESS_JM_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h new file mode 100755 index 000000000000..71c7d495c40a --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h @@ -0,0 +1,209 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/** + * @file mali_kbase_hwaccess_pm.h + * HW access power manager common APIs + */ + +#ifndef _KBASE_HWACCESS_PM_H_ +#define _KBASE_HWACCESS_PM_H_ + +#include +#include + +#include + +/* Forward definition - see mali_kbase.h */ +struct kbase_device; + +/* Functions common to all HW access backends */ + +/** + * Initialize the power management framework. + * + * Must be called before any other power management function + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * + * @return 0 if the power management framework was successfully + * initialized. + */ +int kbase_hwaccess_pm_init(struct kbase_device *kbdev); + +/** + * Terminate the power management framework. + * + * No power management functions may be called after this (except + * @ref kbase_pm_init) + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_term(struct kbase_device *kbdev); + +/** + * kbase_hwaccess_pm_powerup - Power up the GPU. + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * @flags: Flags to pass on to kbase_pm_init_hw + * + * Power up GPU after all modules have been initialized and interrupt handlers + * installed. + * + * Return: 0 if powerup was successful. + */ +int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, + unsigned int flags); + +/** + * Halt the power management framework. + * + * Should ensure that no new interrupts are generated, but allow any currently + * running interrupt handlers to complete successfully. The GPU is forced off by + * the time this function returns, regardless of whether or not the active power + * policy asks for the GPU to be powered off. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_halt(struct kbase_device *kbdev); + +/** + * Perform any backend-specific actions to suspend the GPU + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev); + +/** + * Perform any backend-specific actions to resume the GPU from a suspend + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_resume(struct kbase_device *kbdev); + +/** + * Perform any required actions for activating the GPU. Called when the first + * context goes active. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); + +/** + * Perform any required actions for idling the GPU. Called when the last + * context goes idle. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + */ +void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); + + +/** + * Set the debug core mask. + * + * This determines which cores the power manager is allowed to use. + * + * @param kbdev The kbase device structure for the device (must be a + * valid pointer) + * @param new_core_mask_js0 The core mask to use for job slot 0 + * @param new_core_mask_js0 The core mask to use for job slot 1 + * @param new_core_mask_js0 The core mask to use for job slot 2 + */ +void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, + u64 new_core_mask_js0, u64 new_core_mask_js1, + u64 new_core_mask_js2); + + +/** + * Get the current policy. + * + * Returns the policy that is currently active. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * + * @return The current policy + */ +const struct kbase_pm_ca_policy +*kbase_pm_ca_get_policy(struct kbase_device *kbdev); + +/** + * Change the policy to the one specified. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * @param policy The policy to change to (valid pointer returned from + * @ref kbase_pm_ca_list_policies) + */ +void kbase_pm_ca_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_ca_policy *policy); + +/** + * Retrieve a static list of the available policies. + * + * @param[out] policies An array pointer to take the list of policies. This may + * be NULL. The contents of this array must not be + * modified. + * + * @return The number of policies + */ +int +kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **policies); + + +/** + * Get the current policy. + * + * Returns the policy that is currently active. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * + * @return The current policy + */ +const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev); + +/** + * Change the policy to the one specified. + * + * @param kbdev The kbase device structure for the device (must be a valid + * pointer) + * @param policy The policy to change to (valid pointer returned from + * @ref kbase_pm_list_policies) + */ +void kbase_pm_set_policy(struct kbase_device *kbdev, + const struct kbase_pm_policy *policy); + +/** + * Retrieve a static list of the available policies. + * + * @param[out] policies An array pointer to take the list of policies. This may + * be NULL. The contents of this array must not be + * modified. + * + * @return The number of policies + */ +int kbase_pm_list_policies(const struct kbase_pm_policy * const **policies); + +#endif /* _KBASE_HWACCESS_PM_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h new file mode 100755 index 000000000000..10b65798e6cf --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h @@ -0,0 +1,53 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/** + * + */ + +#ifndef _KBASE_BACKEND_TIME_H_ +#define _KBASE_BACKEND_TIME_H_ + +/** + * kbase_backend_get_gpu_time() - Get current GPU time + * @kbdev: Device pointer + * @cycle_counter: Pointer to u64 to store cycle counter in + * @system_time: Pointer to u64 to store system time in + * @ts: Pointer to struct timespec64 to store current monotonic + * time in + */ +void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, + u64 *system_time, struct timespec64 *ts); + +/** + * kbase_wait_write_flush() - Wait for GPU write flush + * @kctx: Context pointer + * + * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush + * its write buffer. + * + * If GPU resets occur then the counters are reset to zero, the delay may not be + * as expected. + * + * This function is only in use for BASE_HW_ISSUE_6367 + */ +#ifndef CONFIG_MALI_NO_MALI +void kbase_wait_write_flush(struct kbase_context *kctx); +#endif + +#endif /* _KBASE_BACKEND_TIME_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h b/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h new file mode 100755 index 000000000000..cf7bf1b35dc5 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h @@ -0,0 +1,66 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_HWCNT_READER_H_ +#define _KBASE_HWCNT_READER_H_ + +/* The ids of ioctl commands. */ +#define KBASE_HWCNT_READER 0xBE +#define KBASE_HWCNT_READER_GET_HWVER _IOR(KBASE_HWCNT_READER, 0x00, u32) +#define KBASE_HWCNT_READER_GET_BUFFER_SIZE _IOR(KBASE_HWCNT_READER, 0x01, u32) +#define KBASE_HWCNT_READER_DUMP _IOW(KBASE_HWCNT_READER, 0x10, u32) +#define KBASE_HWCNT_READER_CLEAR _IOW(KBASE_HWCNT_READER, 0x11, u32) +#define KBASE_HWCNT_READER_GET_BUFFER _IOR(KBASE_HWCNT_READER, 0x20,\ + struct kbase_hwcnt_reader_metadata) +#define KBASE_HWCNT_READER_PUT_BUFFER _IOW(KBASE_HWCNT_READER, 0x21,\ + struct kbase_hwcnt_reader_metadata) +#define KBASE_HWCNT_READER_SET_INTERVAL _IOW(KBASE_HWCNT_READER, 0x30, u32) +#define KBASE_HWCNT_READER_ENABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x40, u32) +#define KBASE_HWCNT_READER_DISABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x41, u32) +#define KBASE_HWCNT_READER_GET_API_VERSION _IOW(KBASE_HWCNT_READER, 0xFF, u32) + +/** + * struct kbase_hwcnt_reader_metadata - hwcnt reader sample buffer metadata + * @timestamp: time when sample was collected + * @event_id: id of an event that triggered sample collection + * @buffer_idx: position in sampling area where sample buffer was stored + */ +struct kbase_hwcnt_reader_metadata { + u64 timestamp; + u32 event_id; + u32 buffer_idx; +}; + +/** + * enum base_hwcnt_reader_event - hwcnt dumping events + * @BASE_HWCNT_READER_EVENT_MANUAL: manual request for dump + * @BASE_HWCNT_READER_EVENT_PERIODIC: periodic dump + * @BASE_HWCNT_READER_EVENT_PREJOB: prejob dump request + * @BASE_HWCNT_READER_EVENT_POSTJOB: postjob dump request + * @BASE_HWCNT_READER_EVENT_COUNT: number of supported events + */ +enum base_hwcnt_reader_event { + BASE_HWCNT_READER_EVENT_MANUAL, + BASE_HWCNT_READER_EVENT_PERIODIC, + BASE_HWCNT_READER_EVENT_PREJOB, + BASE_HWCNT_READER_EVENT_POSTJOB, + + BASE_HWCNT_READER_EVENT_COUNT +}; + +#endif /* _KBASE_HWCNT_READER_H_ */ + diff --git a/drivers/gpu/arm/midgard/mali_kbase_ioctl.h b/drivers/gpu/arm/midgard/mali_kbase_ioctl.h new file mode 100755 index 000000000000..dcbed9c774d6 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_ioctl.h @@ -0,0 +1,656 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_IOCTL_H_ +#define _KBASE_IOCTL_H_ + +#ifdef __cpluscplus +extern "C" { +#endif + +#include + +#define KBASE_IOCTL_TYPE 0x80 + +#ifdef ANDROID +/* Android's definition of ioctl is incorrect, specifying the type argument as + * 'int'. This creates a warning when using _IOWR (as the top bit is set). Work + * round this by redefining _IOC to include a case to 'int'. + */ +#undef _IOC +#define _IOC(dir, type, nr, size) \ + ((int)(((dir) << _IOC_DIRSHIFT) | ((type) << _IOC_TYPESHIFT) | \ + ((nr) << _IOC_NRSHIFT) | ((size) << _IOC_SIZESHIFT))) +#endif + +/** + * struct kbase_ioctl_version_check - Check version compatibility with kernel + * + * @major: Major version number + * @minor: Minor version number + */ +struct kbase_ioctl_version_check { + __u16 major; + __u16 minor; +}; + +#define KBASE_IOCTL_VERSION_CHECK \ + _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) + +/** + * struct kbase_ioctl_set_flags - Set kernel context creation flags + * + * @create_flags: Flags - see base_context_create_flags + */ +struct kbase_ioctl_set_flags { + __u32 create_flags; +}; + +#define KBASE_IOCTL_SET_FLAGS \ + _IOW(KBASE_IOCTL_TYPE, 1, struct kbase_ioctl_set_flags) + +/** + * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel + * + * @addr: Memory address of an array of struct base_jd_atom_v2 + * @nr_atoms: Number of entries in the array + * @stride: sizeof(struct base_jd_atom_v2) + */ +struct kbase_ioctl_job_submit { + union kbase_pointer addr; + __u32 nr_atoms; + __u32 stride; +}; + +#define KBASE_IOCTL_JOB_SUBMIT \ + _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit) + +/** + * struct kbase_ioctl_get_gpuprops - Read GPU properties from the kernel + * + * @buffer: Pointer to the buffer to store properties into + * @size: Size of the buffer + * @flags: Flags - must be zero for now + * + * The ioctl will return the number of bytes stored into @buffer or an error + * on failure (e.g. @size is too small). If @size is specified as 0 then no + * data will be written but the return value will be the number of bytes needed + * for all the properties. + * + * @flags may be used in the future to request a different format for the + * buffer. With @flags == 0 the following format is used. + * + * The buffer will be filled with pairs of values, a u32 key identifying the + * property followed by the value. The size of the value is identified using + * the bottom bits of the key. The value then immediately followed the key and + * is tightly packed (there is no padding). All keys and values are + * little-endian. + * + * 00 = u8 + * 01 = u16 + * 10 = u32 + * 11 = u64 + */ +struct kbase_ioctl_get_gpuprops { + union kbase_pointer buffer; + __u32 size; + __u32 flags; +}; + +#define KBASE_IOCTL_GET_GPUPROPS \ + _IOW(KBASE_IOCTL_TYPE, 3, struct kbase_ioctl_get_gpuprops) + +#define KBASE_IOCTL_POST_TERM \ + _IO(KBASE_IOCTL_TYPE, 4) + +/** + * union kbase_ioctl_mem_alloc - Allocate memory on the GPU + * + * @va_pages: The number of pages of virtual address space to reserve + * @commit_pages: The number of physical pages to allocate + * @extent: The number of extra pages to allocate on each GPU fault which grows + * the region + * @flags: Flags + * @gpu_va: The GPU virtual address which is allocated + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_alloc { + struct { + __u64 va_pages; + __u64 commit_pages; + __u64 extent; + __u64 flags; + } in; + struct { + __u64 flags; + __u64 gpu_va; + } out; +}; + +#define KBASE_IOCTL_MEM_ALLOC \ + _IOWR(KBASE_IOCTL_TYPE, 5, union kbase_ioctl_mem_alloc) + +/** + * struct kbase_ioctl_mem_query - Query properties of a GPU memory region + * @gpu_addr: A GPU address contained within the region + * @query: The type of query + * @value: The result of the query + * + * Use a %KBASE_MEM_QUERY_xxx flag as input for @query. + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_query { + struct { + __u64 gpu_addr; + __u64 query; + } in; + struct { + __u64 value; + } out; +}; + +#define KBASE_IOCTL_MEM_QUERY \ + _IOWR(KBASE_IOCTL_TYPE, 6, union kbase_ioctl_mem_query) + +#define KBASE_MEM_QUERY_COMMIT_SIZE 1 +#define KBASE_MEM_QUERY_VA_SIZE 2 +#define KBASE_MEM_QUERY_FLAGS 3 + +/** + * struct kbase_ioctl_mem_free - Free a memory region + * @gpu_addr: Handle to the region to free + */ +struct kbase_ioctl_mem_free { + __u64 gpu_addr; +}; + +#define KBASE_IOCTL_MEM_FREE \ + _IOW(KBASE_IOCTL_TYPE, 7, struct kbase_ioctl_mem_free) + +/** + * struct kbase_ioctl_hwcnt_reader_setup - Setup HWC dumper/reader + * @buffer_count: requested number of dumping buffers + * @jm_bm: counters selection bitmask (JM) + * @shader_bm: counters selection bitmask (Shader) + * @tiler_bm: counters selection bitmask (Tiler) + * @mmu_l2_bm: counters selection bitmask (MMU_L2) + * + * A fd is returned from the ioctl if successful, or a negative value on error + */ +struct kbase_ioctl_hwcnt_reader_setup { + __u32 buffer_count; + __u32 jm_bm; + __u32 shader_bm; + __u32 tiler_bm; + __u32 mmu_l2_bm; +}; + +#define KBASE_IOCTL_HWCNT_READER_SETUP \ + _IOW(KBASE_IOCTL_TYPE, 8, struct kbase_ioctl_hwcnt_reader_setup) + +/** + * struct kbase_ioctl_hwcnt_enable - Enable hardware counter collection + * @dump_buffer: GPU address to write counters to + * @jm_bm: counters selection bitmask (JM) + * @shader_bm: counters selection bitmask (Shader) + * @tiler_bm: counters selection bitmask (Tiler) + * @mmu_l2_bm: counters selection bitmask (MMU_L2) + */ +struct kbase_ioctl_hwcnt_enable { + __u64 dump_buffer; + __u32 jm_bm; + __u32 shader_bm; + __u32 tiler_bm; + __u32 mmu_l2_bm; +}; + +#define KBASE_IOCTL_HWCNT_ENABLE \ + _IOW(KBASE_IOCTL_TYPE, 9, struct kbase_ioctl_hwcnt_enable) + +#define KBASE_IOCTL_HWCNT_DUMP \ + _IO(KBASE_IOCTL_TYPE, 10) + +#define KBASE_IOCTL_HWCNT_CLEAR \ + _IO(KBASE_IOCTL_TYPE, 11) + +/** + * struct kbase_ioctl_disjoint_query - Query the disjoint counter + * @counter: A counter of disjoint events in the kernel + */ +struct kbase_ioctl_disjoint_query { + __u32 counter; +}; + +#define KBASE_IOCTL_DISJOINT_QUERY \ + _IOR(KBASE_IOCTL_TYPE, 12, struct kbase_ioctl_disjoint_query) + +/** + * struct kbase_ioctl_get_ddk_version - Query the kernel version + * @version_buffer: Buffer to receive the kernel version string + * @size: Size of the buffer + * + * The ioctl will return the number of bytes written into version_buffer + * (which includes a NULL byte) or a negative error code + */ +struct kbase_ioctl_get_ddk_version { + union kbase_pointer version_buffer; + __u32 size; +}; + +#define KBASE_IOCTL_GET_DDK_VERSION \ + _IOW(KBASE_IOCTL_TYPE, 13, struct kbase_ioctl_get_ddk_version) + +/** + * struct kbase_ioctl_mem_jit_init - Initialise the JIT memory allocator + * + * @va_pages: Number of VA pages to reserve for JIT + * + * Note that depending on the VA size of the application and GPU, the value + * specified in @va_pages may be ignored. + */ +struct kbase_ioctl_mem_jit_init { + __u64 va_pages; +}; + +#define KBASE_IOCTL_MEM_JIT_INIT \ + _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init) + +/** + * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory + * + * @handle: GPU memory handle (GPU VA) + * @user_addr: The address where it is mapped in user space + * @size: The number of bytes to synchronise + * @type: The direction to synchronise: 0 is sync to memory (clean), + * 1 is sync from memory (invalidate). Use the BASE_SYNCSET_OP_xxx constants. + * @padding: Padding to round up to a multiple of 8 bytes, must be zero + */ +struct kbase_ioctl_mem_sync { + __u64 handle; + __u64 user_addr; + __u64 size; + __u8 type; + __u8 padding[7]; +}; + +#define KBASE_IOCTL_MEM_SYNC \ + _IOW(KBASE_IOCTL_TYPE, 15, struct kbase_ioctl_mem_sync) + +/** + * union kbase_ioctl_mem_find_cpu_offset - Find the offset of a CPU pointer + * + * @gpu_addr: The GPU address of the memory region + * @cpu_addr: The CPU address to locate + * @size: A size in bytes to validate is contained within the region + * @offset: The offset from the start of the memory region to @cpu_addr + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_find_cpu_offset { + struct { + __u64 gpu_addr; + __u64 cpu_addr; + __u64 size; + } in; + struct { + __u64 offset; + } out; +}; + +#define KBASE_IOCTL_MEM_FIND_CPU_OFFSET \ + _IOWR(KBASE_IOCTL_TYPE, 16, union kbase_ioctl_mem_find_cpu_offset) + +/** + * struct kbase_ioctl_get_context_id - Get the kernel context ID + * + * @id: The kernel context ID + */ +struct kbase_ioctl_get_context_id { + int id; /* This should really be __u32, but see GPUCORE-10048 */ +}; + +#define KBASE_IOCTL_GET_CONTEXT_ID \ + _IOR(KBASE_IOCTL_TYPE, 17, struct kbase_ioctl_get_context_id) + +/** + * struct kbase_ioctl_tlstream_acquire - Acquire a tlstream fd + * + * @flags: Flags + * + * The ioctl returns a file descriptor when successful + */ +struct kbase_ioctl_tlstream_acquire { + __u32 flags; +}; + +#define KBASE_IOCTL_TLSTREAM_ACQUIRE \ + _IOW(KBASE_IOCTL_TYPE, 18, struct kbase_ioctl_tlstream_acquire) + +#define KBASE_IOCTL_TLSTREAM_FLUSH \ + _IO(KBASE_IOCTL_TYPE, 19) + +/** + * struct kbase_ioctl_mem_commit - Change the amount of memory backing a region + * + * @gpu_addr: The memory region to modify + * @pages: The number of physical pages that should be present + * + * The ioctl may return on the following error codes or 0 for success: + * -ENOMEM: Out of memory + * -EINVAL: Invalid arguments + */ +struct kbase_ioctl_mem_commit { + __u64 gpu_addr; + __u64 pages; +}; + +#define KBASE_IOCTL_MEM_COMMIT \ + _IOW(KBASE_IOCTL_TYPE, 20, struct kbase_ioctl_mem_commit) + +/** + * union kbase_ioctl_mem_alias - Create an alias of memory regions + * @flags: Flags, see BASE_MEM_xxx + * @stride: Bytes between start of each memory region + * @nents: The number of regions to pack together into the alias + * @aliasing_info: Pointer to an array of struct base_mem_aliasing_info + * @gpu_va: Address of the new alias + * @va_pages: Size of the new alias + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_alias { + struct { + __u64 flags; + __u64 stride; + __u64 nents; + union kbase_pointer aliasing_info; + } in; + struct { + __u64 flags; + __u64 gpu_va; + __u64 va_pages; + } out; +}; + +#define KBASE_IOCTL_MEM_ALIAS \ + _IOWR(KBASE_IOCTL_TYPE, 21, union kbase_ioctl_mem_alias) + +/** + * union kbase_ioctl_mem_import - Import memory for use by the GPU + * @flags: Flags, see BASE_MEM_xxx + * @phandle: Handle to the external memory + * @type: Type of external memory, see base_mem_import_type + * @padding: Amount of extra VA pages to append to the imported buffer + * @gpu_va: Address of the new alias + * @va_pages: Size of the new alias + * + * @in: Input parameters + * @out: Output parameters + */ +union kbase_ioctl_mem_import { + struct { + __u64 flags; + union kbase_pointer phandle; + __u32 type; + __u32 padding; + } in; + struct { + __u64 flags; + __u64 gpu_va; + __u64 va_pages; + } out; +}; + +#define KBASE_IOCTL_MEM_IMPORT \ + _IOWR(KBASE_IOCTL_TYPE, 22, union kbase_ioctl_mem_import) + +/** + * struct kbase_ioctl_mem_flags_change - Change the flags for a memory region + * @gpu_va: The GPU region to modify + * @flags: The new flags to set + * @mask: Mask of the flags to modify + */ +struct kbase_ioctl_mem_flags_change { + __u64 gpu_va; + __u64 flags; + __u64 mask; +}; + +#define KBASE_IOCTL_MEM_FLAGS_CHANGE \ + _IOW(KBASE_IOCTL_TYPE, 23, struct kbase_ioctl_mem_flags_change) + +/** + * struct kbase_ioctl_stream_create - Create a synchronisation stream + * @name: A name to identify this stream. Must be NULL-terminated. + * + * Note that this is also called a "timeline", but is named stream to avoid + * confusion with other uses of the word. + * + * Unused bytes in @name (after the first NULL byte) must be also be NULL bytes. + * + * The ioctl returns a file descriptor. + */ +struct kbase_ioctl_stream_create { + char name[32]; +}; + +#define KBASE_IOCTL_STREAM_CREATE \ + _IOW(KBASE_IOCTL_TYPE, 24, struct kbase_ioctl_stream_create) + +/** + * struct kbase_ioctl_fence_validate - Validate a fd refers to a fence + * @fd: The file descriptor to validate + */ +struct kbase_ioctl_fence_validate { + int fd; +}; + +#define KBASE_IOCTL_FENCE_VALIDATE \ + _IOW(KBASE_IOCTL_TYPE, 25, struct kbase_ioctl_fence_validate) + +/** + * struct kbase_ioctl_get_profiling_controls - Get the profiling controls + * @count: The size of @buffer in u32 words + * @buffer: The buffer to receive the profiling controls + */ +struct kbase_ioctl_get_profiling_controls { + union kbase_pointer buffer; + __u32 count; +}; + +#define KBASE_IOCTL_GET_PROFILING_CONTROLS \ + _IOW(KBASE_IOCTL_TYPE, 26, struct kbase_ioctl_get_profiling_controls) + +/** + * struct kbase_ioctl_mem_profile_add - Provide profiling information to kernel + * @buffer: Pointer to the information + * @len: Length + * @padding: Padding + * + * The data provided is accessible through a debugfs file + */ +struct kbase_ioctl_mem_profile_add { + union kbase_pointer buffer; + __u32 len; + __u32 padding; +}; + +#define KBASE_IOCTL_MEM_PROFILE_ADD \ + _IOW(KBASE_IOCTL_TYPE, 27, struct kbase_ioctl_mem_profile_add) + +/** + * struct kbase_ioctl_soft_event_update - Update the status of a soft-event + * @event: GPU address of the event which has been updated + * @new_status: The new status to set + * @flags: Flags for future expansion + */ +struct kbase_ioctl_soft_event_update { + __u64 event; + __u32 new_status; + __u32 flags; +}; + +#define KBASE_IOCTL_SOFT_EVENT_UPDATE \ + _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) + +/*************** + * test ioctls * + ***************/ +#if MALI_UNIT_TEST +/* These ioctls are purely for test purposes and are not used in the production + * driver, they therefore may change without notice + */ + +#define KBASE_IOCTL_TEST_TYPE (KBASE_IOCTL_TYPE + 1) + +/** + * struct kbase_ioctl_tlstream_test - Start a timeline stream test + * + * @tpw_count: number of trace point writers in each context + * @msg_delay: time delay between tracepoints from one writer in milliseconds + * @msg_count: number of trace points written by one writer + * @aux_msg: if non-zero aux messages will be included + */ +struct kbase_ioctl_tlstream_test { + __u32 tpw_count; + __u32 msg_delay; + __u32 msg_count; + __u32 aux_msg; +}; + +#define KBASE_IOCTL_TLSTREAM_TEST \ + _IOW(KBASE_IOCTL_TEST_TYPE, 1, struct kbase_ioctl_tlstream_test) + +/** + * struct kbase_ioctl_tlstream_stats - Read tlstream stats for test purposes + * @bytes_collected: number of bytes read by user + * @bytes_generated: number of bytes generated by tracepoints + */ +struct kbase_ioctl_tlstream_stats { + __u32 bytes_collected; + __u32 bytes_generated; +}; + +#define KBASE_IOCTL_TLSTREAM_STATS \ + _IOR(KBASE_IOCTL_TEST_TYPE, 2, struct kbase_ioctl_tlstream_stats) + +#endif + +/********************************** + * Definitions for GPU properties * + **********************************/ +#define KBASE_GPUPROP_VALUE_SIZE_U8 (0x0) +#define KBASE_GPUPROP_VALUE_SIZE_U16 (0x1) +#define KBASE_GPUPROP_VALUE_SIZE_U32 (0x2) +#define KBASE_GPUPROP_VALUE_SIZE_U64 (0x3) + +#define KBASE_GPUPROP_PRODUCT_ID 1 +#define KBASE_GPUPROP_VERSION_STATUS 2 +#define KBASE_GPUPROP_MINOR_REVISION 3 +#define KBASE_GPUPROP_MAJOR_REVISION 4 +#define KBASE_GPUPROP_GPU_SPEED_MHZ 5 +#define KBASE_GPUPROP_GPU_FREQ_KHZ_MAX 6 +#define KBASE_GPUPROP_GPU_FREQ_KHZ_MIN 7 +#define KBASE_GPUPROP_LOG2_PROGRAM_COUNTER_SIZE 8 +#define KBASE_GPUPROP_TEXTURE_FEATURES_0 9 +#define KBASE_GPUPROP_TEXTURE_FEATURES_1 10 +#define KBASE_GPUPROP_TEXTURE_FEATURES_2 11 +#define KBASE_GPUPROP_GPU_AVAILABLE_MEMORY_SIZE 12 + +#define KBASE_GPUPROP_L2_LOG2_LINE_SIZE 13 +#define KBASE_GPUPROP_L2_LOG2_CACHE_SIZE 14 +#define KBASE_GPUPROP_L2_NUM_L2_SLICES 15 + +#define KBASE_GPUPROP_TILER_BIN_SIZE_BYTES 16 +#define KBASE_GPUPROP_TILER_MAX_ACTIVE_LEVELS 17 + +#define KBASE_GPUPROP_MAX_THREADS 18 +#define KBASE_GPUPROP_MAX_WORKGROUP_SIZE 19 +#define KBASE_GPUPROP_MAX_BARRIER_SIZE 20 +#define KBASE_GPUPROP_MAX_REGISTERS 21 +#define KBASE_GPUPROP_MAX_TASK_QUEUE 22 +#define KBASE_GPUPROP_MAX_THREAD_GROUP_SPLIT 23 +#define KBASE_GPUPROP_IMPL_TECH 24 + +#define KBASE_GPUPROP_RAW_SHADER_PRESENT 25 +#define KBASE_GPUPROP_RAW_TILER_PRESENT 26 +#define KBASE_GPUPROP_RAW_L2_PRESENT 27 +#define KBASE_GPUPROP_RAW_STACK_PRESENT 28 +#define KBASE_GPUPROP_RAW_L2_FEATURES 29 +#define KBASE_GPUPROP_RAW_SUSPEND_SIZE 30 +#define KBASE_GPUPROP_RAW_MEM_FEATURES 31 +#define KBASE_GPUPROP_RAW_MMU_FEATURES 32 +#define KBASE_GPUPROP_RAW_AS_PRESENT 33 +#define KBASE_GPUPROP_RAW_JS_PRESENT 34 +#define KBASE_GPUPROP_RAW_JS_FEATURES_0 35 +#define KBASE_GPUPROP_RAW_JS_FEATURES_1 36 +#define KBASE_GPUPROP_RAW_JS_FEATURES_2 37 +#define KBASE_GPUPROP_RAW_JS_FEATURES_3 38 +#define KBASE_GPUPROP_RAW_JS_FEATURES_4 39 +#define KBASE_GPUPROP_RAW_JS_FEATURES_5 40 +#define KBASE_GPUPROP_RAW_JS_FEATURES_6 41 +#define KBASE_GPUPROP_RAW_JS_FEATURES_7 42 +#define KBASE_GPUPROP_RAW_JS_FEATURES_8 43 +#define KBASE_GPUPROP_RAW_JS_FEATURES_9 44 +#define KBASE_GPUPROP_RAW_JS_FEATURES_10 45 +#define KBASE_GPUPROP_RAW_JS_FEATURES_11 46 +#define KBASE_GPUPROP_RAW_JS_FEATURES_12 47 +#define KBASE_GPUPROP_RAW_JS_FEATURES_13 48 +#define KBASE_GPUPROP_RAW_JS_FEATURES_14 49 +#define KBASE_GPUPROP_RAW_JS_FEATURES_15 50 +#define KBASE_GPUPROP_RAW_TILER_FEATURES 51 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_0 52 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_1 53 +#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_2 54 +#define KBASE_GPUPROP_RAW_GPU_ID 55 +#define KBASE_GPUPROP_RAW_THREAD_MAX_THREADS 56 +#define KBASE_GPUPROP_RAW_THREAD_MAX_WORKGROUP_SIZE 57 +#define KBASE_GPUPROP_RAW_THREAD_MAX_BARRIER_SIZE 58 +#define KBASE_GPUPROP_RAW_THREAD_FEATURES 59 +#define KBASE_GPUPROP_RAW_COHERENCY_MODE 60 + +#define KBASE_GPUPROP_COHERENCY_NUM_GROUPS 61 +#define KBASE_GPUPROP_COHERENCY_NUM_CORE_GROUPS 62 +#define KBASE_GPUPROP_COHERENCY_COHERENCY 63 +#define KBASE_GPUPROP_COHERENCY_GROUP_0 64 +#define KBASE_GPUPROP_COHERENCY_GROUP_1 65 +#define KBASE_GPUPROP_COHERENCY_GROUP_2 66 +#define KBASE_GPUPROP_COHERENCY_GROUP_3 67 +#define KBASE_GPUPROP_COHERENCY_GROUP_4 68 +#define KBASE_GPUPROP_COHERENCY_GROUP_5 69 +#define KBASE_GPUPROP_COHERENCY_GROUP_6 70 +#define KBASE_GPUPROP_COHERENCY_GROUP_7 71 +#define KBASE_GPUPROP_COHERENCY_GROUP_8 72 +#define KBASE_GPUPROP_COHERENCY_GROUP_9 73 +#define KBASE_GPUPROP_COHERENCY_GROUP_10 74 +#define KBASE_GPUPROP_COHERENCY_GROUP_11 75 +#define KBASE_GPUPROP_COHERENCY_GROUP_12 76 +#define KBASE_GPUPROP_COHERENCY_GROUP_13 77 +#define KBASE_GPUPROP_COHERENCY_GROUP_14 78 +#define KBASE_GPUPROP_COHERENCY_GROUP_15 79 + +#ifdef __cpluscplus +} +#endif + +#endif diff --git a/drivers/gpu/arm/midgard/mali_kbase_jd.c b/drivers/gpu/arm/midgard/mali_kbase_jd.c new file mode 100755 index 000000000000..d9d8658d31dc --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_jd.c @@ -0,0 +1,1903 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include +#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ +#ifdef CONFIG_COMPAT +#include +#endif +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "mali_kbase_dma_fence.h" + +#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) +/* random32 was renamed to prandom_u32 in 3.8 */ +#define prandom_u32 random32 +#endif + +/* Return whether katom will run on the GPU or not. Currently only soft jobs and + * dependency-only atoms do not run on the GPU */ +#define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ + ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \ + BASE_JD_REQ_DEP))) +/* + * This is the kernel side of the API. Only entry points are: + * - kbase_jd_submit(): Called from userspace to submit a single bag + * - kbase_jd_done(): Called from interrupt context to track the + * completion of a job. + * Callouts: + * - to the job manager (enqueue a job) + * - to the event subsystem (signals the completion/failure of bag/job-chains). + */ + +static void __user * +get_compat_pointer(struct kbase_context *kctx, const union kbase_pointer *p) +{ +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + return compat_ptr(p->compat_value); +#endif + return p->value; +} + +/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs + * + * Returns whether the JS needs a reschedule. + * + * Note that the caller must also check the atom status and + * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock + */ +static int jd_run_atom(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); + + if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { + /* Dependency only atom */ + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + return 0; + } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { + /* Soft-job */ + if (katom->will_fail_event_code) { + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + return 0; + } + if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) + == BASE_JD_REQ_SOFT_REPLAY) { + if (!kbase_replay_process(katom)) + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + } else if (kbase_process_soft_job(katom) == 0) { + kbase_finish_soft_job(katom); + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + } + return 0; + } + + katom->status = KBASE_JD_ATOM_STATE_IN_JS; + /* Queue an action about whether we should try scheduling a context */ + return kbasep_js_add_job(kctx, katom); +} + +#if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE) +void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom) +{ + struct kbase_device *kbdev; + + KBASE_DEBUG_ASSERT(katom); + kbdev = katom->kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev); + + /* Check whether the atom's other dependencies were already met. If + * katom is a GPU atom then the job scheduler may be able to represent + * the dependencies, hence we may attempt to submit it before they are + * met. Other atoms must have had both dependencies resolved. + */ + if (IS_GPU_ATOM(katom) || + (!kbase_jd_katom_dep_atom(&katom->dep[0]) && + !kbase_jd_katom_dep_atom(&katom->dep[1]))) { + /* katom dep complete, attempt to run it */ + bool resched = false; + + resched = jd_run_atom(katom); + + if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) { + /* The atom has already finished */ + resched |= jd_done_nolock(katom, NULL); + } + + if (resched) + kbase_js_sched_all(kbdev); + } +} +#endif + +#ifdef CONFIG_KDS + +/* Add the katom to the kds waiting list. + * Atoms must be added to the waiting list after a successful call to kds_async_waitall. + * The caller must hold the kbase_jd_context.lock */ + +static void kbase_jd_kds_waiters_add(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx; + + KBASE_DEBUG_ASSERT(katom); + + kctx = katom->kctx; + + list_add_tail(&katom->node, &kctx->waiting_kds_resource); +} + +/* Remove the katom from the kds waiting list. + * Atoms must be removed from the waiting list before a call to kds_resource_set_release_sync. + * The supplied katom must first have been added to the list with a call to kbase_jd_kds_waiters_add. + * The caller must hold the kbase_jd_context.lock */ + +static void kbase_jd_kds_waiters_remove(struct kbase_jd_atom *katom) +{ + KBASE_DEBUG_ASSERT(katom); + list_del(&katom->node); +} + +static void kds_dep_clear(void *callback_parameter, void *callback_extra_parameter) +{ + struct kbase_jd_atom *katom; + struct kbase_jd_context *ctx; + + katom = (struct kbase_jd_atom *)callback_parameter; + KBASE_DEBUG_ASSERT(katom); + + ctx = &katom->kctx->jctx; + + /* If KDS resource has already been satisfied (e.g. due to zapping) + * do nothing. + */ + mutex_lock(&ctx->lock); + if (!katom->kds_dep_satisfied) { + katom->kds_dep_satisfied = true; + kbase_jd_dep_clear_locked(katom); + } + mutex_unlock(&ctx->lock); +} + +static void kbase_cancel_kds_wait_job(struct kbase_jd_atom *katom) +{ + KBASE_DEBUG_ASSERT(katom); + + /* Prevent job_done_nolock from being called twice on an atom when + * there is a race between job completion and cancellation */ + + if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { + /* Wait was cancelled - zap the atom */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); + } +} +#endif /* CONFIG_KDS */ + +void kbase_jd_free_external_resources(struct kbase_jd_atom *katom) +{ +#ifdef CONFIG_KDS + if (katom->kds_rset) { + struct kbase_jd_context *jctx = &katom->kctx->jctx; + + /* + * As the atom is no longer waiting, remove it from + * the waiting list. + */ + + mutex_lock(&jctx->lock); + kbase_jd_kds_waiters_remove(katom); + mutex_unlock(&jctx->lock); + + /* Release the kds resource or cancel if zapping */ + kds_resource_set_release_sync(&katom->kds_rset); + } +#endif /* CONFIG_KDS */ + +#ifdef CONFIG_MALI_DMA_FENCE + /* Flush dma-fence workqueue to ensure that any callbacks that may have + * been queued are done before continuing. + * Any successfully completed atom would have had all it's callbacks + * completed before the atom was run, so only flush for failed atoms. + */ + if (katom->event_code != BASE_JD_EVENT_DONE) + flush_workqueue(katom->kctx->dma_fence.wq); +#endif /* CONFIG_MALI_DMA_FENCE */ +} + +static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) +{ + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); + +#ifdef CONFIG_KDS + /* Prevent the KDS resource from triggering the atom in case of zapping */ + if (katom->kds_rset) + katom->kds_dep_satisfied = true; +#endif /* CONFIG_KDS */ + +#ifdef CONFIG_MALI_DMA_FENCE + kbase_dma_fence_signal(katom); +#endif /* CONFIG_MALI_DMA_FENCE */ + + kbase_gpu_vm_lock(katom->kctx); + /* only roll back if extres is non-NULL */ + if (katom->extres) { + u32 res_no; + + res_no = katom->nr_extres; + while (res_no-- > 0) { + struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; + struct kbase_va_region *reg; + + reg = kbase_region_tracker_find_region_base_address( + katom->kctx, + katom->extres[res_no].gpu_address); + kbase_unmap_external_resource(katom->kctx, reg, alloc); + } + kfree(katom->extres); + katom->extres = NULL; + } + kbase_gpu_vm_unlock(katom->kctx); +} + +/* + * Set up external resources needed by this job. + * + * jctx.lock must be held when this is called. + */ + +static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom) +{ + int err_ret_val = -EINVAL; + u32 res_no; +#ifdef CONFIG_KDS + u32 kds_res_count = 0; + struct kds_resource **kds_resources = NULL; + unsigned long *kds_access_bitmap = NULL; +#endif /* CONFIG_KDS */ +#ifdef CONFIG_MALI_DMA_FENCE + struct kbase_dma_fence_resv_info info = { + .dma_fence_resv_count = 0, + }; +#ifdef CONFIG_SYNC + /* + * When both dma-buf fence and Android native sync is enabled, we + * disable dma-buf fence for contexts that are using Android native + * fences. + */ + const bool implicit_sync = !kbase_ctx_flag(katom->kctx, + KCTX_NO_IMPLICIT_SYNC); +#else /* CONFIG_SYNC */ + const bool implicit_sync = true; +#endif /* CONFIG_SYNC */ +#endif /* CONFIG_MALI_DMA_FENCE */ + struct base_external_resource *input_extres; + + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); + + /* no resources encoded, early out */ + if (!katom->nr_extres) + return -EINVAL; + + katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL); + if (NULL == katom->extres) { + err_ret_val = -ENOMEM; + goto early_err_out; + } + + /* copy user buffer to the end of our real buffer. + * Make sure the struct sizes haven't changed in a way + * we don't support */ + BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres)); + input_extres = (struct base_external_resource *) + (((unsigned char *)katom->extres) + + (sizeof(*katom->extres) - sizeof(*input_extres)) * + katom->nr_extres); + + if (copy_from_user(input_extres, + get_compat_pointer(katom->kctx, &user_atom->extres_list), + sizeof(*input_extres) * katom->nr_extres) != 0) { + err_ret_val = -EINVAL; + goto early_err_out; + } +#ifdef CONFIG_KDS + /* assume we have to wait for all */ + KBASE_DEBUG_ASSERT(0 != katom->nr_extres); + kds_resources = kmalloc_array(katom->nr_extres, sizeof(struct kds_resource *), GFP_KERNEL); + + if (!kds_resources) { + err_ret_val = -ENOMEM; + goto early_err_out; + } + + KBASE_DEBUG_ASSERT(0 != katom->nr_extres); + kds_access_bitmap = kcalloc(BITS_TO_LONGS(katom->nr_extres), + sizeof(unsigned long), + GFP_KERNEL); + if (!kds_access_bitmap) { + err_ret_val = -ENOMEM; + goto early_err_out; + } +#endif /* CONFIG_KDS */ + +#ifdef CONFIG_MALI_DMA_FENCE + if (implicit_sync) { + info.resv_objs = kmalloc_array(katom->nr_extres, + sizeof(struct reservation_object *), + GFP_KERNEL); + if (!info.resv_objs) { + err_ret_val = -ENOMEM; + goto early_err_out; + } + + info.dma_fence_excl_bitmap = + kcalloc(BITS_TO_LONGS(katom->nr_extres), + sizeof(unsigned long), GFP_KERNEL); + if (!info.dma_fence_excl_bitmap) { + err_ret_val = -ENOMEM; + goto early_err_out; + } + } +#endif /* CONFIG_MALI_DMA_FENCE */ + + /* Take the processes mmap lock */ + down_read(¤t->mm->mmap_lock); + + /* need to keep the GPU VM locked while we set up UMM buffers */ + kbase_gpu_vm_lock(katom->kctx); + for (res_no = 0; res_no < katom->nr_extres; res_no++) { + struct base_external_resource *res; + struct kbase_va_region *reg; + struct kbase_mem_phy_alloc *alloc; + bool exclusive; + + res = &input_extres[res_no]; + exclusive = (res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE) + ? true : false; + reg = kbase_region_tracker_find_region_enclosing_address( + katom->kctx, + res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE); + /* did we find a matching region object? */ + if (NULL == reg || (reg->flags & KBASE_REG_FREE)) { + /* roll back */ + goto failed_loop; + } + + if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) && + (reg->flags & KBASE_REG_SECURE)) { + katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED; + } + + alloc = kbase_map_external_resource(katom->kctx, reg, + current->mm +#ifdef CONFIG_KDS + , &kds_res_count, kds_resources, + kds_access_bitmap, exclusive +#endif + ); + if (!alloc) { + err_ret_val = -EINVAL; + goto failed_loop; + } + +#ifdef CONFIG_MALI_DMA_FENCE + if (implicit_sync && + reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { + struct reservation_object *resv; + + resv = reg->gpu_alloc->imported.umm.dma_buf->resv; + if (resv) + kbase_dma_fence_add_reservation(resv, &info, + exclusive); + } +#endif /* CONFIG_MALI_DMA_FENCE */ + + /* finish with updating out array with the data we found */ + /* NOTE: It is important that this is the last thing we do (or + * at least not before the first write) as we overwrite elements + * as we loop and could be overwriting ourself, so no writes + * until the last read for an element. + * */ + katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */ + katom->extres[res_no].alloc = alloc; + } + /* successfully parsed the extres array */ + /* drop the vm lock before we call into kds */ + kbase_gpu_vm_unlock(katom->kctx); + + /* Release the processes mmap lock */ + up_read(¤t->mm->mmap_lock); + +#ifdef CONFIG_KDS + if (kds_res_count) { + int wait_failed; + + /* We have resources to wait for with kds */ + katom->kds_dep_satisfied = false; + + wait_failed = kds_async_waitall(&katom->kds_rset, + &katom->kctx->jctx.kds_cb, katom, NULL, + kds_res_count, kds_access_bitmap, + kds_resources); + + if (wait_failed) + goto failed_kds_setup; + else + kbase_jd_kds_waiters_add(katom); + } else { + /* Nothing to wait for, so kds dep met */ + katom->kds_dep_satisfied = true; + } + kfree(kds_resources); + kfree(kds_access_bitmap); +#endif /* CONFIG_KDS */ + +#ifdef CONFIG_MALI_DMA_FENCE + if (implicit_sync) { + if (info.dma_fence_resv_count) { + int ret; + + ret = kbase_dma_fence_wait(katom, &info); + if (ret < 0) + goto failed_dma_fence_setup; + } + + kfree(info.resv_objs); + kfree(info.dma_fence_excl_bitmap); + } +#endif /* CONFIG_MALI_DMA_FENCE */ + + /* all done OK */ + return 0; + +/* error handling section */ + +#ifdef CONFIG_MALI_DMA_FENCE +failed_dma_fence_setup: +#ifdef CONFIG_KDS + /* If we are here, dma_fence setup failed but KDS didn't. + * Revert KDS setup if any. + */ + if (kds_res_count) { + mutex_unlock(&katom->kctx->jctx.lock); + kds_resource_set_release_sync(&katom->kds_rset); + mutex_lock(&katom->kctx->jctx.lock); + + kbase_jd_kds_waiters_remove(katom); + katom->kds_dep_satisfied = true; + } +#endif /* CONFIG_KDS */ +#endif /* CONFIG_MALI_DMA_FENCE */ +#ifdef CONFIG_KDS +failed_kds_setup: +#endif +#if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE) + /* Lock the processes mmap lock */ + down_read(¤t->mm->mmap_lock); + + /* lock before we unmap */ + kbase_gpu_vm_lock(katom->kctx); +#endif + + failed_loop: + /* undo the loop work */ + while (res_no-- > 0) { + struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; + + kbase_unmap_external_resource(katom->kctx, NULL, alloc); + } + kbase_gpu_vm_unlock(katom->kctx); + + /* Release the processes mmap lock */ + up_read(¤t->mm->mmap_lock); + + early_err_out: + kfree(katom->extres); + katom->extres = NULL; +#ifdef CONFIG_KDS + kfree(kds_resources); + kfree(kds_access_bitmap); +#endif /* CONFIG_KDS */ +#ifdef CONFIG_MALI_DMA_FENCE + if (implicit_sync) { + kfree(info.resv_objs); + kfree(info.dma_fence_excl_bitmap); + } +#endif + return err_ret_val; +} + +static inline void jd_resolve_dep(struct list_head *out_list, + struct kbase_jd_atom *katom, + u8 d, bool ctx_is_dying) +{ + u8 other_d = !d; + + while (!list_empty(&katom->dep_head[d])) { + struct kbase_jd_atom *dep_atom; + struct kbase_jd_atom *other_dep_atom; + u8 dep_type; + + dep_atom = list_entry(katom->dep_head[d].next, + struct kbase_jd_atom, dep_item[d]); + list_del(katom->dep_head[d].next); + + dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]); + kbase_jd_katom_dep_clear(&dep_atom->dep[d]); + + if (katom->event_code != BASE_JD_EVENT_DONE && + (dep_type != BASE_JD_DEP_TYPE_ORDER)) { +#ifdef CONFIG_KDS + if (!dep_atom->kds_dep_satisfied) { + /* Just set kds_dep_satisfied to true. If the callback happens after this then it will early out and + * do nothing. If the callback doesn't happen then kbase_jd_post_external_resources will clean up + */ + dep_atom->kds_dep_satisfied = true; + } +#endif + +#ifdef CONFIG_MALI_DMA_FENCE + kbase_dma_fence_cancel_callbacks(dep_atom); +#endif + + dep_atom->event_code = katom->event_code; + KBASE_DEBUG_ASSERT(dep_atom->status != + KBASE_JD_ATOM_STATE_UNUSED); + + if ((dep_atom->core_req & BASE_JD_REQ_SOFT_REPLAY) + != BASE_JD_REQ_SOFT_REPLAY) { + dep_atom->will_fail_event_code = + dep_atom->event_code; + } else { + dep_atom->status = + KBASE_JD_ATOM_STATE_COMPLETED; + } + } + other_dep_atom = (struct kbase_jd_atom *) + kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]); + + if (!dep_atom->in_jd_list && (!other_dep_atom || + (IS_GPU_ATOM(dep_atom) && !ctx_is_dying && + !dep_atom->will_fail_event_code && + !other_dep_atom->will_fail_event_code))) { + bool dep_satisfied = true; +#ifdef CONFIG_MALI_DMA_FENCE + int dep_count; + + dep_count = kbase_fence_dep_count_read(dep_atom); + if (likely(dep_count == -1)) { + dep_satisfied = true; + } else { + /* + * There are either still active callbacks, or + * all fences for this @dep_atom has signaled, + * but the worker that will queue the atom has + * not yet run. + * + * Wait for the fences to signal and the fence + * worker to run and handle @dep_atom. If + * @dep_atom was completed due to error on + * @katom, then the fence worker will pick up + * the complete status and error code set on + * @dep_atom above. + */ + dep_satisfied = false; + } +#endif /* CONFIG_MALI_DMA_FENCE */ + +#ifdef CONFIG_KDS + dep_satisfied = dep_satisfied && dep_atom->kds_dep_satisfied; +#endif + + if (dep_satisfied) { + dep_atom->in_jd_list = true; + list_add_tail(&dep_atom->jd_item, out_list); + } + } + } +} + +KBASE_EXPORT_TEST_API(jd_resolve_dep); + +#if MALI_CUSTOMER_RELEASE == 0 +static void jd_force_failure(struct kbase_device *kbdev, struct kbase_jd_atom *katom) +{ + kbdev->force_replay_count++; + + if (kbdev->force_replay_count >= kbdev->force_replay_limit) { + kbdev->force_replay_count = 0; + katom->event_code = BASE_JD_EVENT_FORCE_REPLAY; + + if (kbdev->force_replay_random) + kbdev->force_replay_limit = + (prandom_u32() % KBASEP_FORCE_REPLAY_RANDOM_LIMIT) + 1; + + dev_info(kbdev->dev, "force_replay : promoting to error\n"); + } +} + +/** Test to see if atom should be forced to fail. + * + * This function will check if an atom has a replay job as a dependent. If so + * then it will be considered for forced failure. */ +static void jd_check_force_failure(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + int i; + + if ((kbdev->force_replay_limit == KBASEP_FORCE_REPLAY_DISABLED) || + (katom->core_req & BASEP_JD_REQ_EVENT_NEVER)) + return; + + for (i = 1; i < BASE_JD_ATOM_COUNT; i++) { + if (kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[0]) == katom || + kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[1]) == katom) { + struct kbase_jd_atom *dep_atom = &kctx->jctx.atoms[i]; + + if ((dep_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == + BASE_JD_REQ_SOFT_REPLAY && + (dep_atom->core_req & kbdev->force_replay_core_req) + == kbdev->force_replay_core_req) { + jd_force_failure(kbdev, katom); + return; + } + } + } +} +#endif + +/** + * is_dep_valid - Validate that a dependency is valid for early dependency + * submission + * @katom: Dependency atom to validate + * + * A dependency is valid if any of the following are true : + * - It does not exist (a non-existent dependency does not block submission) + * - It is in the job scheduler + * - It has completed, does not have a failure event code, and has not been + * marked to fail in the future + * + * Return: true if valid, false otherwise + */ +static bool is_dep_valid(struct kbase_jd_atom *katom) +{ + /* If there's no dependency then this is 'valid' from the perspective of + * early dependency submission */ + if (!katom) + return true; + + /* Dependency must have reached the job scheduler */ + if (katom->status < KBASE_JD_ATOM_STATE_IN_JS) + return false; + + /* If dependency has completed and has failed or will fail then it is + * not valid */ + if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && + (katom->event_code != BASE_JD_EVENT_DONE || + katom->will_fail_event_code)) + return false; + + return true; +} + +static void jd_try_submitting_deps(struct list_head *out_list, + struct kbase_jd_atom *node) +{ + int i; + + for (i = 0; i < 2; i++) { + struct list_head *pos; + + list_for_each(pos, &node->dep_head[i]) { + struct kbase_jd_atom *dep_atom = list_entry(pos, + struct kbase_jd_atom, dep_item[i]); + + if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) { + /*Check if atom deps look sane*/ + bool dep0_valid = is_dep_valid( + dep_atom->dep[0].atom); + bool dep1_valid = is_dep_valid( + dep_atom->dep[1].atom); + bool dep_satisfied = true; +#ifdef CONFIG_MALI_DMA_FENCE + int dep_count; + + dep_count = kbase_fence_dep_count_read( + dep_atom); + if (likely(dep_count == -1)) { + dep_satisfied = true; + } else { + /* + * There are either still active callbacks, or + * all fences for this @dep_atom has signaled, + * but the worker that will queue the atom has + * not yet run. + * + * Wait for the fences to signal and the fence + * worker to run and handle @dep_atom. If + * @dep_atom was completed due to error on + * @katom, then the fence worker will pick up + * the complete status and error code set on + * @dep_atom above. + */ + dep_satisfied = false; + } +#endif /* CONFIG_MALI_DMA_FENCE */ +#ifdef CONFIG_KDS + dep_satisfied = dep_satisfied && + dep_atom->kds_dep_satisfied; +#endif + + if (dep0_valid && dep1_valid && dep_satisfied) { + dep_atom->in_jd_list = true; + list_add(&dep_atom->jd_item, out_list); + } + } + } + } +} + +/* + * Perform the necessary handling of an atom that has finished running + * on the GPU. + * + * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller + * is responsible for calling kbase_finish_soft_job *before* calling this function. + * + * The caller must hold the kbase_jd_context.lock. + */ +bool jd_done_nolock(struct kbase_jd_atom *katom, + struct list_head *completed_jobs_ctx) +{ + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct list_head completed_jobs; + struct list_head runnable_jobs; + bool need_to_try_schedule_context = false; + int i; + + INIT_LIST_HEAD(&completed_jobs); + INIT_LIST_HEAD(&runnable_jobs); + + KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); + +#if MALI_CUSTOMER_RELEASE == 0 + jd_check_force_failure(katom); +#endif + + /* This is needed in case an atom is failed due to being invalid, this + * can happen *before* the jobs that the atom depends on have completed */ + for (i = 0; i < 2; i++) { + if (kbase_jd_katom_dep_atom(&katom->dep[i])) { + list_del(&katom->dep_item[i]); + kbase_jd_katom_dep_clear(&katom->dep[i]); + } + } + + /* With PRLAM-10817 or PRLAM-10959 the last tile of a fragment job being soft-stopped can fail with + * BASE_JD_EVENT_TILE_RANGE_FAULT. + * + * So here if the fragment job failed with TILE_RANGE_FAULT and it has been soft-stopped, then we promote the + * error code to BASE_JD_EVENT_DONE + */ + + if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10817) || kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10959)) && + katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT) { + if ((katom->core_req & BASE_JD_REQ_FS) && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED)) { + /* Promote the failure to job done */ + katom->event_code = BASE_JD_EVENT_DONE; + katom->atom_flags = katom->atom_flags & (~KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED); + } + } + + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + list_add_tail(&katom->jd_item, &completed_jobs); + + while (!list_empty(&completed_jobs)) { + katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item); + list_del(completed_jobs.prev); + KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); + + for (i = 0; i < 2; i++) + jd_resolve_dep(&runnable_jobs, katom, i, + kbase_ctx_flag(kctx, KCTX_DYING)); + + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) + kbase_jd_post_external_resources(katom); + + while (!list_empty(&runnable_jobs)) { + struct kbase_jd_atom *node; + + node = list_entry(runnable_jobs.next, + struct kbase_jd_atom, jd_item); + list_del(runnable_jobs.next); + node->in_jd_list = false; + + KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); + + if (node->status != KBASE_JD_ATOM_STATE_COMPLETED && + !kbase_ctx_flag(kctx, KCTX_DYING)) { + need_to_try_schedule_context |= jd_run_atom(node); + } else { + node->event_code = katom->event_code; + + if ((node->core_req & + BASE_JD_REQ_SOFT_JOB_TYPE) == + BASE_JD_REQ_SOFT_REPLAY) { + if (kbase_replay_process(node)) + /* Don't complete this atom */ + continue; + } else if (node->core_req & + BASE_JD_REQ_SOFT_JOB) { + /* If this is a fence wait soft job + * then remove it from the list of sync + * waiters. + */ + if (BASE_JD_REQ_SOFT_FENCE_WAIT == node->core_req) + kbasep_remove_waiting_soft_job(node); + + kbase_finish_soft_job(node); + } + node->status = KBASE_JD_ATOM_STATE_COMPLETED; + } + + if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) { + list_add_tail(&node->jd_item, &completed_jobs); + } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS && + !node->will_fail_event_code) { + /* Node successfully submitted, try submitting + * dependencies as they may now be representable + * in JS */ + jd_try_submitting_deps(&runnable_jobs, node); + } + } + + /* Register a completed job as a disjoint event when the GPU + * is in a disjoint state (ie. being reset or replaying jobs). + */ + kbase_disjoint_event_potential(kctx->kbdev); + if (completed_jobs_ctx) + list_add_tail(&katom->jd_item, completed_jobs_ctx); + else + kbase_event_post(kctx, katom); + + /* Decrement and check the TOTAL number of jobs. This includes + * those not tracked by the scheduler: 'not ready to run' and + * 'dependency-only' jobs. */ + if (--kctx->jctx.job_nr == 0) + wake_up(&kctx->jctx.zero_jobs_wait); /* All events are safely queued now, and we can signal any waiter + * that we've got no more jobs (so we can be safely terminated) */ + } + + return need_to_try_schedule_context; +} + +KBASE_EXPORT_TEST_API(jd_done_nolock); + +#ifdef CONFIG_GPU_TRACEPOINTS +enum { + CORE_REQ_DEP_ONLY, + CORE_REQ_SOFT, + CORE_REQ_COMPUTE, + CORE_REQ_FRAGMENT, + CORE_REQ_VERTEX, + CORE_REQ_TILER, + CORE_REQ_FRAGMENT_VERTEX, + CORE_REQ_FRAGMENT_VERTEX_TILER, + CORE_REQ_FRAGMENT_TILER, + CORE_REQ_VERTEX_TILER, + CORE_REQ_UNKNOWN +}; +static const char * const core_req_strings[] = { + "Dependency Only Job", + "Soft Job", + "Compute Shader Job", + "Fragment Shader Job", + "Vertex/Geometry Shader Job", + "Tiler Job", + "Fragment Shader + Vertex/Geometry Shader Job", + "Fragment Shader + Vertex/Geometry Shader Job + Tiler Job", + "Fragment Shader + Tiler Job", + "Vertex/Geometry Shader Job + Tiler Job", + "Unknown Job" +}; +static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req) +{ + if (core_req & BASE_JD_REQ_SOFT_JOB) + return core_req_strings[CORE_REQ_SOFT]; + if (core_req & BASE_JD_REQ_ONLY_COMPUTE) + return core_req_strings[CORE_REQ_COMPUTE]; + switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) { + case BASE_JD_REQ_DEP: + return core_req_strings[CORE_REQ_DEP_ONLY]; + case BASE_JD_REQ_FS: + return core_req_strings[CORE_REQ_FRAGMENT]; + case BASE_JD_REQ_CS: + return core_req_strings[CORE_REQ_VERTEX]; + case BASE_JD_REQ_T: + return core_req_strings[CORE_REQ_TILER]; + case (BASE_JD_REQ_FS | BASE_JD_REQ_CS): + return core_req_strings[CORE_REQ_FRAGMENT_VERTEX]; + case (BASE_JD_REQ_FS | BASE_JD_REQ_T): + return core_req_strings[CORE_REQ_FRAGMENT_TILER]; + case (BASE_JD_REQ_CS | BASE_JD_REQ_T): + return core_req_strings[CORE_REQ_VERTEX_TILER]; + case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T): + return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER]; + } + return core_req_strings[CORE_REQ_UNKNOWN]; +} +#endif + +bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *user_atom, struct kbase_jd_atom *katom) +{ + struct kbase_jd_context *jctx = &kctx->jctx; + int queued = 0; + int i; + int sched_prio; + bool ret; + bool will_fail = false; + + /* Update the TOTAL number of jobs. This includes those not tracked by + * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ + jctx->job_nr++; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + katom->start_timestamp.tv64 = 0; +#else + katom->start_timestamp = 0; +#endif + katom->udata = user_atom->udata; + katom->kctx = kctx; + katom->nr_extres = user_atom->nr_extres; + katom->extres = NULL; + katom->device_nr = user_atom->device_nr; + katom->affinity = 0; + katom->jc = user_atom->jc; + katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; + katom->core_req = user_atom->core_req; + katom->atom_flags = 0; + katom->retry_count = 0; + katom->need_cache_flush_cores_retained = 0; + katom->pre_dep = NULL; + katom->post_dep = NULL; + katom->x_pre_dep = NULL; + katom->x_post_dep = NULL; + katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; + + /* Implicitly sets katom->protected_state.enter as well. */ + katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; + + katom->age = kctx->age_count++; + + INIT_LIST_HEAD(&katom->jd_item); +#ifdef CONFIG_KDS + /* Start by assuming that the KDS dependencies are satisfied, + * kbase_jd_pre_external_resources will correct this if there are dependencies */ + katom->kds_dep_satisfied = true; + katom->kds_rset = NULL; +#endif /* CONFIG_KDS */ +#ifdef CONFIG_MALI_DMA_FENCE + kbase_fence_dep_count_set(katom, -1); +#endif + + /* Don't do anything if there is a mess up with dependencies. + This is done in a separate cycle to check both the dependencies at ones, otherwise + it will be extra complexity to deal with 1st dependency ( just added to the list ) + if only the 2nd one has invalid config. + */ + for (i = 0; i < 2; i++) { + int dep_atom_number = user_atom->pre_dep[i].atom_id; + base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type; + + if (dep_atom_number) { + if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER && + dep_atom_type != BASE_JD_DEP_TYPE_DATA) { + katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; + katom->status = KBASE_JD_ATOM_STATE_COMPLETED; + + /* Wrong dependency setup. Atom will be sent + * back to user space. Do not record any + * dependencies. */ + KBASE_TLSTREAM_TL_NEW_ATOM( + katom, + kbase_jd_atom_id(kctx, katom)); + KBASE_TLSTREAM_TL_RET_ATOM_CTX( + katom, kctx); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, + TL_ATOM_STATE_IDLE); + + ret = jd_done_nolock(katom, NULL); + goto out; + } + } + } + + /* Add dependencies */ + for (i = 0; i < 2; i++) { + int dep_atom_number = user_atom->pre_dep[i].atom_id; + base_jd_dep_type dep_atom_type; + struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; + + dep_atom_type = user_atom->pre_dep[i].dependency_type; + kbase_jd_katom_dep_clear(&katom->dep[i]); + + if (!dep_atom_number) + continue; + + if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || + dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) { + + if (dep_atom->event_code == BASE_JD_EVENT_DONE) + continue; + /* don't stop this atom if it has an order dependency + * only to the failed one, try to submit it through + * the normal path + */ + if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER && + dep_atom->event_code > BASE_JD_EVENT_ACTIVE) { + continue; + } + + /* Atom has completed, propagate the error code if any */ + katom->event_code = dep_atom->event_code; + katom->status = KBASE_JD_ATOM_STATE_QUEUED; + + /* This atom is going through soft replay or + * will be sent back to user space. Do not record any + * dependencies. */ + KBASE_TLSTREAM_TL_NEW_ATOM( + katom, + kbase_jd_atom_id(kctx, katom)); + KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, + TL_ATOM_STATE_IDLE); + + if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) + == BASE_JD_REQ_SOFT_REPLAY) { + if (kbase_replay_process(katom)) { + ret = false; + goto out; + } + } + will_fail = true; + + } else { + /* Atom is in progress, add this atom to the list */ + list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]); + kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type); + queued = 1; + } + } + + if (will_fail) { + if (!queued) { + ret = jd_done_nolock(katom, NULL); + + goto out; + } else { + katom->will_fail_event_code = katom->event_code; + ret = false; + + goto out; + } + } else { + /* These must occur after the above loop to ensure that an atom + * that depends on a previous atom with the same number behaves + * as expected */ + katom->event_code = BASE_JD_EVENT_DONE; + katom->status = KBASE_JD_ATOM_STATE_QUEUED; + } + + /* For invalid priority, be most lenient and choose the default */ + sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); + if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) + sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; + katom->sched_priority = sched_prio; + + /* Create a new atom recording all dependencies it was set up with. */ + KBASE_TLSTREAM_TL_NEW_ATOM( + katom, + kbase_jd_atom_id(kctx, katom)); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_IDLE); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(katom, katom->sched_priority); + KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); + for (i = 0; i < 2; i++) + if (BASE_JD_DEP_TYPE_INVALID != kbase_jd_katom_dep_type( + &katom->dep[i])) { + KBASE_TLSTREAM_TL_DEP_ATOM_ATOM( + (void *)kbase_jd_katom_dep_atom( + &katom->dep[i]), + (void *)katom); + } else if (BASE_JD_DEP_TYPE_INVALID != + user_atom->pre_dep[i].dependency_type) { + /* Resolved dependency. */ + int dep_atom_number = + user_atom->pre_dep[i].atom_id; + struct kbase_jd_atom *dep_atom = + &jctx->atoms[dep_atom_number]; + + KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM( + (void *)dep_atom, + (void *)katom); + } + + /* Reject atoms with job chain = NULL, as these cause issues with soft-stop */ + if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { + dev_warn(kctx->kbdev->dev, "Rejecting atom with jc = NULL"); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + ret = jd_done_nolock(katom, NULL); + goto out; + } + + /* Reject atoms with an invalid device_nr */ + if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) && + (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) { + dev_warn(kctx->kbdev->dev, + "Rejecting atom with invalid device_nr %d", + katom->device_nr); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + ret = jd_done_nolock(katom, NULL); + goto out; + } + + /* Reject atoms with invalid core requirements */ + if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && + (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) { + dev_warn(kctx->kbdev->dev, + "Rejecting atom with invalid core requirements"); + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE; + ret = jd_done_nolock(katom, NULL); + goto out; + } + + if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + /* handle what we need to do to access the external resources */ + if (kbase_jd_pre_external_resources(katom, user_atom) != 0) { + /* setup failed (no access, bad resource, unknown resource types, etc.) */ + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + ret = jd_done_nolock(katom, NULL); + goto out; + } + } + + /* Validate the atom. Function will return error if the atom is + * malformed. + * + * Soft-jobs never enter the job scheduler but have their own initialize method. + * + * If either fail then we immediately complete the atom with an error. + */ + if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) { + if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + ret = jd_done_nolock(katom, NULL); + goto out; + } + } else { + /* Soft-job */ + if (kbase_prepare_soft_job(katom) != 0) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + ret = jd_done_nolock(katom, NULL); + goto out; + } + } + +#ifdef CONFIG_GPU_TRACEPOINTS + katom->work_id = atomic_inc_return(&jctx->work_id); + trace_gpu_job_enqueue((u32)kctx->id, katom->work_id, + kbasep_map_core_reqs_to_string(katom->core_req)); +#endif + + if (queued && !IS_GPU_ATOM(katom)) { + ret = false; + goto out; + } +#ifdef CONFIG_KDS + if (!katom->kds_dep_satisfied) { + /* Queue atom due to KDS dependency */ + ret = false; + goto out; + } +#endif /* CONFIG_KDS */ + + +#ifdef CONFIG_MALI_DMA_FENCE + if (kbase_fence_dep_count_read(katom) != -1) { + ret = false; + goto out; + } +#endif /* CONFIG_MALI_DMA_FENCE */ + + if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) + == BASE_JD_REQ_SOFT_REPLAY) { + if (kbase_replay_process(katom)) + ret = false; + else + ret = jd_done_nolock(katom, NULL); + + goto out; + } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { + if (kbase_process_soft_job(katom) == 0) { + kbase_finish_soft_job(katom); + ret = jd_done_nolock(katom, NULL); + goto out; + } + + ret = false; + } else if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { + katom->status = KBASE_JD_ATOM_STATE_IN_JS; + ret = kbasep_js_add_job(kctx, katom); + /* If job was cancelled then resolve immediately */ + if (katom->event_code == BASE_JD_EVENT_JOB_CANCELLED) + ret = jd_done_nolock(katom, NULL); + } else { + /* This is a pure dependency. Resolve it immediately */ + ret = jd_done_nolock(katom, NULL); + } + + out: + return ret; +} + +int kbase_jd_submit(struct kbase_context *kctx, + void __user *user_addr, u32 nr_atoms, u32 stride, + bool uk6_atom) +{ + struct kbase_jd_context *jctx = &kctx->jctx; + int err = 0; + int i; + bool need_to_try_schedule_context = false; + struct kbase_device *kbdev; + u32 latest_flush; + + /* + * kbase_jd_submit isn't expected to fail and so all errors with the + * jobs are reported by immediately failing them (through event system) + */ + kbdev = kctx->kbdev; + + beenthere(kctx, "%s", "Enter"); + + if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it"); + return -EINVAL; + } + + if (stride != sizeof(base_jd_atom_v2)) { + dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel"); + return -EINVAL; + } + + KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(nr_atoms, + &kctx->timeline.jd_atoms_in_flight)); + + /* All atoms submitted in this call have the same flush ID */ + latest_flush = kbase_backend_get_current_flush_id(kbdev); + + for (i = 0; i < nr_atoms; i++) { + struct base_jd_atom_v2 user_atom; + struct kbase_jd_atom *katom; + +#ifdef BASE_LEGACY_UK6_SUPPORT + BUILD_BUG_ON(sizeof(struct base_jd_atom_v2_uk6) != + sizeof(base_jd_atom_v2)); + + if (uk6_atom) { + struct base_jd_atom_v2_uk6 user_atom_v6; + base_jd_dep_type dep_types[2] = {BASE_JD_DEP_TYPE_DATA, BASE_JD_DEP_TYPE_DATA}; + + if (copy_from_user(&user_atom_v6, user_addr, + sizeof(user_atom_v6))) { + err = -EINVAL; + KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, + atomic_sub_return( + nr_atoms - i, + &kctx->timeline.jd_atoms_in_flight)); + break; + } + /* Convert from UK6 atom format to UK7 format */ + user_atom.jc = user_atom_v6.jc; + user_atom.udata = user_atom_v6.udata; + user_atom.extres_list = user_atom_v6.extres_list; + user_atom.nr_extres = user_atom_v6.nr_extres; + user_atom.core_req = (u32)(user_atom_v6.core_req & 0x7fff); + + /* atom number 0 is used for no dependency atoms */ + if (!user_atom_v6.pre_dep[0]) + dep_types[0] = BASE_JD_DEP_TYPE_INVALID; + + base_jd_atom_dep_set(&user_atom.pre_dep[0], + user_atom_v6.pre_dep[0], + dep_types[0]); + + /* atom number 0 is used for no dependency atoms */ + if (!user_atom_v6.pre_dep[1]) + dep_types[1] = BASE_JD_DEP_TYPE_INVALID; + + base_jd_atom_dep_set(&user_atom.pre_dep[1], + user_atom_v6.pre_dep[1], + dep_types[1]); + + user_atom.atom_number = user_atom_v6.atom_number; + user_atom.prio = user_atom_v6.prio; + user_atom.device_nr = user_atom_v6.device_nr; + } else { +#endif /* BASE_LEGACY_UK6_SUPPORT */ + if (copy_from_user(&user_atom, user_addr, + sizeof(user_atom)) != 0) { + err = -EINVAL; + KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, + atomic_sub_return(nr_atoms - i, + &kctx->timeline.jd_atoms_in_flight)); + break; + } +#ifdef BASE_LEGACY_UK6_SUPPORT + } +#endif + +#ifdef BASE_LEGACY_UK10_2_SUPPORT + if (KBASE_API_VERSION(10, 3) > kctx->api_version) + user_atom.core_req = (u32)(user_atom.compat_core_req + & 0x7fff); +#endif /* BASE_LEGACY_UK10_2_SUPPORT */ + + user_addr = (void __user *)((uintptr_t) user_addr + stride); + + mutex_lock(&jctx->lock); +#ifndef compiletime_assert +#define compiletime_assert_defined +#define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \ +while (false) +#endif + compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) >= + BASE_JD_ATOM_COUNT, + "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); + compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) == + sizeof(user_atom.atom_number), + "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); +#ifdef compiletime_assert_defined +#undef compiletime_assert +#undef compiletime_assert_defined +#endif + if (user_atom.atom_number >= BASE_JD_ATOM_COUNT) { + err = -EINVAL; + break; + } + user_atom.atom_number = + array_index_nospec(user_atom.atom_number, + BASE_JD_ATOM_COUNT); + katom = &jctx->atoms[user_atom.atom_number]; + + /* Record the flush ID for the cache flush optimisation */ + katom->flush_id = latest_flush; + + while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) { + /* Atom number is already in use, wait for the atom to + * complete + */ + mutex_unlock(&jctx->lock); + + /* This thread will wait for the atom to complete. Due + * to thread scheduling we are not sure that the other + * thread that owns the atom will also schedule the + * context, so we force the scheduler to be active and + * hence eventually schedule this context at some point + * later. + */ + kbase_js_sched_all(kbdev); + + if (wait_event_killable(katom->completed, + katom->status == + KBASE_JD_ATOM_STATE_UNUSED) != 0) { + /* We're being killed so the result code + * doesn't really matter + */ + return 0; + } + mutex_lock(&jctx->lock); + } + + need_to_try_schedule_context |= + jd_submit_atom(kctx, &user_atom, katom); + + /* Register a completed job as a disjoint event when the GPU is in a disjoint state + * (ie. being reset or replaying jobs). + */ + kbase_disjoint_event_potential(kbdev); + + mutex_unlock(&jctx->lock); + } + + if (need_to_try_schedule_context) + kbase_js_sched_all(kbdev); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_jd_submit); + +void kbase_jd_done_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); + struct kbase_jd_context *jctx; + struct kbase_context *kctx; + struct kbasep_js_kctx_info *js_kctx_info; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + u64 cache_jc = katom->jc; + struct kbasep_js_atom_retained_state katom_retained_state; + bool context_idle; + base_jd_core_req core_req = katom->core_req; + u64 affinity = katom->affinity; + enum kbase_atom_coreref_state coreref_state = katom->coreref_state; + + /* Soft jobs should never reach this function */ + KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); + + kctx = katom->kctx; + jctx = &kctx->jctx; + kbdev = kctx->kbdev; + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); + + kbase_backend_complete_wq(kbdev, katom); + + /* + * Begin transaction on JD context and JS context + */ + mutex_lock(&jctx->lock); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_DONE); + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + /* This worker only gets called on contexts that are scheduled *in*. This is + * because it only happens in response to an IRQ from a job that was + * running. + */ + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (katom->event_code == BASE_JD_EVENT_STOPPED) { + /* Atom has been promoted to stopped */ + unsigned long flags; + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + katom->status = KBASE_JD_ATOM_STATE_IN_JS; + kbase_js_unpull(kctx, katom); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&jctx->lock); + + return; + } + + if (katom->event_code != BASE_JD_EVENT_DONE) + dev_err(kbdev->dev, + "t6xx: GPU fault 0x%02lx from job slot %d\n", + (unsigned long)katom->event_code, + katom->slot_nr); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) + kbase_as_poking_timer_release_atom(kbdev, kctx, katom); + + /* Retain state before the katom disappears */ + kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); + + context_idle = kbase_js_complete_atom_wq(kctx, katom); + + KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state)); + + kbasep_js_remove_job(kbdev, kctx, katom); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; + /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */ + jd_done_nolock(katom, &kctx->completed_jobs); + + /* katom may have been freed now, do not use! */ + + if (context_idle) { + unsigned long flags; + + context_idle = false; + mutex_lock(&js_devdata->queue_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* If kbase_sched() has scheduled this context back in then + * KCTX_ACTIVE will have been set after we marked it as + * inactive, and another pm reference will have been taken, so + * drop our reference. But do not call kbase_jm_idle_ctx(), as + * the context is active and fast-starting is allowed. + * + * If an atom has been fast-started then kctx->atoms_pulled will + * be non-zero but KCTX_ACTIVE will still be false (as the + * previous pm reference has been inherited). Do NOT drop our + * reference, as it has been re-used, and leave the context as + * active. + * + * If no new atoms have been started then KCTX_ACTIVE will still + * be false and atoms_pulled will be zero, so drop the reference + * and call kbase_jm_idle_ctx(). + * + * As the checks are done under both the queue_mutex and + * hwaccess_lock is should be impossible for this to race + * with the scheduler code. + */ + if (kbase_ctx_flag(kctx, KCTX_ACTIVE) || + !atomic_read(&kctx->atoms_pulled)) { + /* Calling kbase_jm_idle_ctx() here will ensure that + * atoms are not fast-started when we drop the + * hwaccess_lock. This is not performed if + * KCTX_ACTIVE is set as in that case another pm + * reference has been taken and a fast-start would be + * valid. + */ + if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) + kbase_jm_idle_ctx(kbdev, kctx); + context_idle = true; + } else { + kbase_ctx_flag_set(kctx, KCTX_ACTIVE); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->queue_mutex); + } + + /* + * Transaction complete + */ + mutex_unlock(&jctx->lock); + + /* Job is now no longer running, so can now safely release the context + * reference, and handle any actions that were logged against the atom's retained state */ + + kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); + + kbase_js_sched_all(kbdev); + + if (!atomic_dec_return(&kctx->work_count)) { + /* If worker now idle then post all events that jd_done_nolock() + * has queued */ + mutex_lock(&jctx->lock); + while (!list_empty(&kctx->completed_jobs)) { + struct kbase_jd_atom *atom = list_entry( + kctx->completed_jobs.next, + struct kbase_jd_atom, jd_item); + list_del(kctx->completed_jobs.next); + + kbase_event_post(kctx, atom); + } + mutex_unlock(&jctx->lock); + } + + kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, + coreref_state); + + if (context_idle) + kbase_pm_context_idle(kbdev); + + KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); +} + +/** + * jd_cancel_worker - Work queue job cancel function. + * @data: a &struct work_struct + * + * Only called as part of 'Zapping' a context (which occurs on termination). + * Operates serially with the kbase_jd_done_worker() on the work queue. + * + * This can only be called on contexts that aren't scheduled. + * + * We don't need to release most of the resources that would occur on + * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be + * running (by virtue of only being called on contexts that aren't + * scheduled). + */ +static void jd_cancel_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); + struct kbase_jd_context *jctx; + struct kbase_context *kctx; + struct kbasep_js_kctx_info *js_kctx_info; + bool need_to_try_schedule_context; + bool attr_state_changed; + struct kbase_device *kbdev; + + /* Soft jobs should never reach this function */ + KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); + + kctx = katom->kctx; + kbdev = kctx->kbdev; + jctx = &kctx->jctx; + js_kctx_info = &kctx->jctx.sched_info; + + KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); + + /* This only gets called on contexts that are scheduled out. Hence, we must + * make sure we don't de-ref the number of running jobs (there aren't + * any), nor must we try to schedule out the context (it's already + * scheduled out). + */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + /* Scheduler: Remove the job from the system */ + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + mutex_lock(&jctx->lock); + + need_to_try_schedule_context = jd_done_nolock(katom, NULL); + /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to + * schedule the context. There's also no need for the jsctx_mutex to have been taken + * around this too. */ + KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); + + /* katom may have been freed now, do not use! */ + mutex_unlock(&jctx->lock); + + if (attr_state_changed) + kbase_js_sched_all(kbdev); +} + +/** + * kbase_jd_done - Complete a job that has been removed from the Hardware + * @katom: atom which has been completed + * @slot_nr: slot the atom was on + * @end_timestamp: completion time + * @done_code: completion code + * + * This must be used whenever a job has been removed from the Hardware, e.g.: + * An IRQ indicates that the job finished (for both error and 'done' codes), or + * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop. + * + * Some work is carried out immediately, and the rest is deferred onto a + * workqueue + * + * Context: + * This can be called safely from atomic context. + * The caller must hold kbdev->hwaccess_lock + */ +void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, + ktime_t *end_timestamp, kbasep_js_atom_done_code done_code) +{ + struct kbase_context *kctx; + struct kbase_device *kbdev; + + KBASE_DEBUG_ASSERT(katom); + kctx = katom->kctx; + KBASE_DEBUG_ASSERT(kctx); + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev); + + if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) + katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; + + KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0); + + kbase_job_check_leave_disjoint(kbdev, katom); + + katom->slot_nr = slot_nr; + + atomic_inc(&kctx->work_count); + +#ifdef CONFIG_DEBUG_FS + /* a failed job happened and is waiting for dumping*/ + if (!katom->will_fail_event_code && + kbase_debug_job_fault_process(katom, katom->event_code)) + return; +#endif + + WARN_ON(work_pending(&katom->work)); + KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); + INIT_WORK(&katom->work, kbase_jd_done_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + +KBASE_EXPORT_TEST_API(kbase_jd_done); + +void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx; + + KBASE_DEBUG_ASSERT(NULL != kbdev); + KBASE_DEBUG_ASSERT(NULL != katom); + kctx = katom->kctx; + KBASE_DEBUG_ASSERT(NULL != kctx); + + KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); + + /* This should only be done from a context that is not scheduled */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + WARN_ON(work_pending(&katom->work)); + + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); + INIT_WORK(&katom->work, jd_cancel_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + + +void kbase_jd_zap_context(struct kbase_context *kctx) +{ + struct kbase_jd_atom *katom; + struct list_head *entry, *tmp; + struct kbase_device *kbdev; + + KBASE_DEBUG_ASSERT(kctx); + + kbdev = kctx->kbdev; + + KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); + + kbase_js_zap_context(kctx); + + mutex_lock(&kctx->jctx.lock); + + /* + * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are + * queued outside the job scheduler. + */ + + del_timer_sync(&kctx->soft_job_timeout); + list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { + katom = list_entry(entry, struct kbase_jd_atom, queue); + kbase_cancel_soft_job(katom); + } + + +#ifdef CONFIG_KDS + + /* For each job waiting on a kds resource, cancel the wait and force the job to + * complete early, this is done so that we don't leave jobs outstanding waiting + * on kds resources which may never be released when contexts are zapped, resulting + * in a hang. + * + * Note that we can safely iterate over the list as the struct kbase_jd_context lock is held, + * this prevents items being removed when calling job_done_nolock in kbase_cancel_kds_wait_job. + */ + + list_for_each(entry, &kctx->waiting_kds_resource) { + katom = list_entry(entry, struct kbase_jd_atom, node); + + kbase_cancel_kds_wait_job(katom); + } +#endif + +#ifdef CONFIG_MALI_DMA_FENCE + kbase_dma_fence_cancel_all_atoms(kctx); +#endif + + mutex_unlock(&kctx->jctx.lock); + +#ifdef CONFIG_MALI_DMA_FENCE + /* Flush dma-fence workqueue to ensure that any callbacks that may have + * been queued are done before continuing. + */ + flush_workqueue(kctx->dma_fence.wq); +#endif + + kbase_jm_wait_for_zero_jobs(kctx); +} + +KBASE_EXPORT_TEST_API(kbase_jd_zap_context); + +int kbase_jd_init(struct kbase_context *kctx) +{ + int i; + int mali_err = 0; +#ifdef CONFIG_KDS + int err; +#endif /* CONFIG_KDS */ + + KBASE_DEBUG_ASSERT(kctx); + + kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", + WQ_HIGHPRI | WQ_UNBOUND, 1); + if (NULL == kctx->jctx.job_done_wq) { + mali_err = -ENOMEM; + goto out1; + } + + for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { + init_waitqueue_head(&kctx->jctx.atoms[i].completed); + + INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]); + INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]); + + /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */ + kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; + kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; + +#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) + kctx->jctx.atoms[i].dma_fence.context = + dma_fence_context_alloc(1); + atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0); + INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks); +#endif + } + + mutex_init(&kctx->jctx.lock); + + init_waitqueue_head(&kctx->jctx.zero_jobs_wait); + + spin_lock_init(&kctx->jctx.tb_lock); + +#ifdef CONFIG_KDS + err = kds_callback_init(&kctx->jctx.kds_cb, 0, kds_dep_clear); + if (0 != err) { + mali_err = -EINVAL; + goto out2; + } +#endif /* CONFIG_KDS */ + + kctx->jctx.job_nr = 0; + INIT_LIST_HEAD(&kctx->completed_jobs); + atomic_set(&kctx->work_count, 0); + + return 0; + +#ifdef CONFIG_KDS + out2: + destroy_workqueue(kctx->jctx.job_done_wq); +#endif /* CONFIG_KDS */ + out1: + return mali_err; +} + +KBASE_EXPORT_TEST_API(kbase_jd_init); + +void kbase_jd_exit(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx); + +#ifdef CONFIG_KDS + kds_callback_term(&kctx->jctx.kds_cb); +#endif /* CONFIG_KDS */ + /* Work queue is emptied by this */ + destroy_workqueue(kctx->jctx.job_done_wq); +} + +KBASE_EXPORT_TEST_API(kbase_jd_exit); diff --git a/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c new file mode 100755 index 000000000000..44643abf85aa --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c @@ -0,0 +1,233 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifdef CONFIG_DEBUG_FS + +#include +#include +#include +#include +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include +#endif + +struct kbase_jd_debugfs_depinfo { + u8 id; + char type; +}; + +static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, + struct seq_file *sfile) +{ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + struct kbase_sync_fence_info info; + int res; + + switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + res = kbase_sync_fence_out_info_get(atom, &info); + if (res == 0) + seq_printf(sfile, "Sa([%p]%d) ", + info.fence, info.status); + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + res = kbase_sync_fence_in_info_get(atom, &info); + if (res == 0) + seq_printf(sfile, "Wa([%p]%d) ", + info.fence, info.status); + break; + default: + break; + } +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + +#ifdef CONFIG_MALI_DMA_FENCE + if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + struct kbase_fence_cb *cb; + + if (atom->dma_fence.fence) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = atom->dma_fence.fence; +#else + struct dma_fence *fence = atom->dma_fence.fence; +#endif + + seq_printf(sfile, +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + "Sd(%u#%u: %s) ", +#else + "Sd(%llu#%u: %s) ", +#endif + fence->context, + fence->seqno, + dma_fence_is_signaled(fence) ? + "signaled" : "active"); + } + + list_for_each_entry(cb, &atom->dma_fence.callbacks, + node) { +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = cb->fence; +#else + struct dma_fence *fence = cb->fence; +#endif + + seq_printf(sfile, +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + "Wd(%u#%u: %s) ", +#else + "Wd(%llu#%u: %s) ", +#endif + fence->context, + fence->seqno, + dma_fence_is_signaled(fence) ? + "signaled" : "active"); + } + } +#endif /* CONFIG_MALI_DMA_FENCE */ + +} + +static void kbasep_jd_debugfs_atom_deps( + struct kbase_jd_debugfs_depinfo *deps, + struct kbase_jd_atom *atom) +{ + struct kbase_context *kctx = atom->kctx; + int i; + + for (i = 0; i < 2; i++) { + deps[i].id = (unsigned)(atom->dep[i].atom ? + kbase_jd_atom_id(kctx, atom->dep[i].atom) : 0); + + switch (atom->dep[i].dep_type) { + case BASE_JD_DEP_TYPE_INVALID: + deps[i].type = ' '; + break; + case BASE_JD_DEP_TYPE_DATA: + deps[i].type = 'D'; + break; + case BASE_JD_DEP_TYPE_ORDER: + deps[i].type = '>'; + break; + default: + deps[i].type = '?'; + break; + } + } +} +/** + * kbasep_jd_debugfs_atoms_show - Show callback for the JD atoms debugfs file. + * @sfile: The debugfs entry + * @data: Data associated with the entry + * + * This function is called to get the contents of the JD atoms debugfs file. + * This is a report of all atoms managed by kbase_jd_context.atoms + * + * Return: 0 if successfully prints data in debugfs entry file, failure + * otherwise + */ +static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) +{ + struct kbase_context *kctx = sfile->private; + struct kbase_jd_atom *atoms; + unsigned long irq_flags; + int i; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + /* Print version */ + seq_printf(sfile, "v%u\n", MALI_JD_DEBUGFS_VERSION); + + /* Print U/K API version */ + seq_printf(sfile, "ukv%u.%u\n", BASE_UK_VERSION_MAJOR, + BASE_UK_VERSION_MINOR); + + /* Print table heading */ + seq_puts(sfile, " ID, Core req, St, CR, Predeps, Start time, Additional info...\n"); + + atoms = kctx->jctx.atoms; + /* General atom states */ + mutex_lock(&kctx->jctx.lock); + /* JS-related states */ + spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); + for (i = 0; i != BASE_JD_ATOM_COUNT; ++i) { + struct kbase_jd_atom *atom = &atoms[i]; + s64 start_timestamp = 0; + struct kbase_jd_debugfs_depinfo deps[2]; + + if (atom->status == KBASE_JD_ATOM_STATE_UNUSED) + continue; + + /* start_timestamp is cleared as soon as the atom leaves UNUSED state + * and set before a job is submitted to the h/w, a non-zero value means + * it is valid */ + if (ktime_to_ns(atom->start_timestamp)) + start_timestamp = ktime_to_ns( + ktime_sub(ktime_get(), atom->start_timestamp)); + + kbasep_jd_debugfs_atom_deps(deps, atom); + + seq_printf(sfile, + "%3u, %8x, %2u, %2u, %c%3u %c%3u, %20lld, ", + i, atom->core_req, atom->status, + atom->coreref_state, + deps[0].type, deps[0].id, + deps[1].type, deps[1].id, + start_timestamp); + + + kbase_jd_debugfs_fence_info(atom, sfile); + + seq_puts(sfile, "\n"); + } + spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); + mutex_unlock(&kctx->jctx.lock); + + return 0; +} + + +/** + * kbasep_jd_debugfs_atoms_open - open operation for atom debugfs file + * @in: &struct inode pointer + * @file: &struct file pointer + * + * Return: file descriptor + */ +static int kbasep_jd_debugfs_atoms_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_jd_debugfs_atoms_show, in->i_private); +} + +static const struct file_operations kbasep_jd_debugfs_atoms_fops = { + .open = kbasep_jd_debugfs_atoms_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx != NULL); + + /* Expose all atoms */ + debugfs_create_file("atoms", S_IRUGO, kctx->kctx_dentry, kctx, + &kbasep_jd_debugfs_atoms_fops); + +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h new file mode 100755 index 000000000000..0935f1db7296 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h @@ -0,0 +1,39 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_jd_debugfs.h + * Header file for job dispatcher-related entries in debugfs + */ + +#ifndef _KBASE_JD_DEBUGFS_H +#define _KBASE_JD_DEBUGFS_H + +#include + +#include + +#define MALI_JD_DEBUGFS_VERSION 2 + +/** + * kbasep_jd_debugfs_ctx_init() - Add debugfs entries for JD system + * + * @kctx Pointer to kbase_context + */ +void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx); + +#endif /*_KBASE_JD_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/midgard/mali_kbase_jm.c b/drivers/gpu/arm/midgard/mali_kbase_jm.c new file mode 100755 index 000000000000..0c5c6a6f78cb --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_jm.c @@ -0,0 +1,131 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * HW access job manager common APIs + */ + +#include +#include "mali_kbase_hwaccess_jm.h" +#include "mali_kbase_jm.h" + +/** + * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot + * @js on the active context. + * @kbdev: Device pointer + * @js: Job slot to run on + * @nr_jobs_to_submit: Number of jobs to attempt to submit + * + * Return: true if slot can still be submitted on, false if slot is now full. + */ +static bool kbase_jm_next_job(struct kbase_device *kbdev, int js, + int nr_jobs_to_submit) +{ + struct kbase_context *kctx; + int i; + + kctx = kbdev->hwaccess.active_kctx; + + if (!kctx) + return true; + + for (i = 0; i < nr_jobs_to_submit; i++) { + struct kbase_jd_atom *katom = kbase_js_pull(kctx, js); + + if (!katom) + return true; /* Context has no jobs on this slot */ + + kbase_backend_run_atom(kbdev, katom); + } + + return false; /* Slot ringbuffer should now be full */ +} + +u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask) +{ + u32 ret_mask = 0; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + while (js_mask) { + int js = ffs(js_mask) - 1; + int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js); + + if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit)) + ret_mask |= (1 << js); + + js_mask &= ~(1 << js); + } + + return ret_mask; +} + +void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!down_trylock(&js_devdata->schedule_sem)) { + kbase_jm_kick(kbdev, js_mask); + up(&js_devdata->schedule_sem); + } +} + +void kbase_jm_try_kick_all(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!down_trylock(&js_devdata->schedule_sem)) { + kbase_jm_kick_all(kbdev); + up(&js_devdata->schedule_sem); + } +} + +void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->hwaccess.active_kctx == kctx) + kbdev->hwaccess.active_kctx = NULL; +} + +struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (katom->event_code != BASE_JD_EVENT_STOPPED && + katom->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT) { + return kbase_js_complete_atom(katom, NULL); + } else { + kbase_js_unpull(katom->kctx, katom); + return NULL; + } +} + +struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, ktime_t *end_timestamp) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + return kbase_js_complete_atom(katom, end_timestamp); +} + diff --git a/drivers/gpu/arm/midgard/mali_kbase_jm.h b/drivers/gpu/arm/midgard/mali_kbase_jm.h new file mode 100755 index 000000000000..a74ee24c8058 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_jm.h @@ -0,0 +1,110 @@ +/* + * + * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +/* + * Job manager common APIs + */ + +#ifndef _KBASE_JM_H_ +#define _KBASE_JM_H_ + +/** + * kbase_jm_kick() - Indicate that there are jobs ready to run. + * @kbdev: Device pointer + * @js_mask: Mask of the job slots that can be pulled from. + * + * Caller must hold the hwaccess_lock and schedule_sem semaphore + * + * Return: Mask of the job slots that can still be submitted to. + */ +u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask); + +/** + * kbase_jm_kick_all() - Indicate that there are jobs ready to run on all job + * slots. + * @kbdev: Device pointer + * + * Caller must hold the hwaccess_lock and schedule_sem semaphore + * + * Return: Mask of the job slots that can still be submitted to. + */ +static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev) +{ + return kbase_jm_kick(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); +} + +/** + * kbase_jm_try_kick - Attempt to call kbase_jm_kick + * @kbdev: Device pointer + * @js_mask: Mask of the job slots that can be pulled from + * Context: Caller must hold hwaccess_lock + * + * If schedule_sem can be immediately obtained then this function will call + * kbase_jm_kick() otherwise it will do nothing. + */ +void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask); + +/** + * kbase_jm_try_kick_all() - Attempt to call kbase_jm_kick_all + * @kbdev: Device pointer + * Context: Caller must hold hwaccess_lock + * + * If schedule_sem can be immediately obtained then this function will call + * kbase_jm_kick_all() otherwise it will do nothing. + */ +void kbase_jm_try_kick_all(struct kbase_device *kbdev); + +/** + * kbase_jm_idle_ctx() - Mark a context as idle. + * @kbdev: Device pointer + * @kctx: Context to mark as idle + * + * No more atoms will be pulled from this context until it is marked as active + * by kbase_js_use_ctx(). + * + * The context should have no atoms currently pulled from it + * (kctx->atoms_pulled == 0). + * + * Caller must hold the hwaccess_lock + */ +void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * kbase_jm_return_atom_to_js() - Return an atom to the job scheduler that has + * been soft-stopped or will fail due to a + * dependency + * @kbdev: Device pointer + * @katom: Atom that has been stopped or will be failed + * + * Return: Atom that has now been unblocked and can now be run, or NULL if none + */ +struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_jm_complete() - Complete an atom + * @kbdev: Device pointer + * @katom: Atom that has completed + * @end_timestamp: Timestamp of atom completion + * + * Return: Atom that has now been unblocked and can now be run, or NULL if none + */ +struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, + struct kbase_jd_atom *katom, ktime_t *end_timestamp); + +#endif /* _KBASE_JM_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_js.c b/drivers/gpu/arm/midgard/mali_kbase_js.c new file mode 100755 index 000000000000..10a1d5909bbe --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_js.c @@ -0,0 +1,2834 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + +/* #define ENABLE_DEBUG_LOG */ +#include "./platform/rk/custom_log.h" + +/* + * Job Scheduler Implementation + */ +#include +#include +#if defined(CONFIG_MALI_GATOR_SUPPORT) +#include +#endif +#include +#include +#include + +#include +#include + +#include "mali_kbase_jm.h" +#include "mali_kbase_hwaccess_jm.h" + +/* + * Private types + */ + +/* Bitpattern indicating the result of releasing a context */ +enum { + /* The context was descheduled - caller should try scheduling in a new + * one to keep the runpool full */ + KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), + /* Ctx attributes were changed - caller should try scheduling all + * contexts */ + KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) +}; + +typedef u32 kbasep_js_release_result; + +const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { + KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ + KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ + KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */ +}; + +const base_jd_prio +kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { + BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ + BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ + BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ +}; + + +/* + * Private function prototypes + */ +static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( + struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state); + +static int kbase_js_get_slot(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, + kbasep_js_ctx_job_cb callback); + +/* Helper for trace subcodes */ +#if KBASE_TRACE_ENABLE +static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + return atomic_read(&kctx->refcount); +} +#else /* KBASE_TRACE_ENABLE */ +static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + CSTD_UNUSED(kbdev); + CSTD_UNUSED(kctx); + return 0; +} +#endif /* KBASE_TRACE_ENABLE */ + +/* + * Private functions + */ + +/** + * core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements + * @features: JSn_FEATURE register value + * + * Given a JSn_FEATURE register value returns the core requirements that match + * + * Return: Core requirement bit mask + */ +static base_jd_core_req core_reqs_from_jsn_features(u16 features) +{ + base_jd_core_req core_req = 0u; + + if ((features & JS_FEATURE_SET_VALUE_JOB) != 0) + core_req |= BASE_JD_REQ_V; + + if ((features & JS_FEATURE_CACHE_FLUSH_JOB) != 0) + core_req |= BASE_JD_REQ_CF; + + if ((features & JS_FEATURE_COMPUTE_JOB) != 0) + core_req |= BASE_JD_REQ_CS; + + if ((features & JS_FEATURE_TILER_JOB) != 0) + core_req |= BASE_JD_REQ_T; + + if ((features & JS_FEATURE_FRAGMENT_JOB) != 0) + core_req |= BASE_JD_REQ_FS; + + return core_req; +} + +static void kbase_js_sync_timers(struct kbase_device *kbdev) +{ + mutex_lock(&kbdev->js_data.runpool_mutex); + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&kbdev->js_data.runpool_mutex); +} + +/* Hold the mmu_hw_mutex and hwaccess_lock for this */ +bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + bool result = false; + int as_nr; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + as_nr = kctx->as_nr; + if (atomic_read(&kctx->refcount) > 0) { + KBASE_DEBUG_ASSERT(as_nr >= 0); + + kbase_ctx_sched_retain_ctx_refcount(kctx); + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx, + NULL, 0u, atomic_read(&kctx->refcount)); + result = true; + } + + return result; +} + +/** + * jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * @prio: Priority to check. + * + * Return true if there are no atoms to pull. There may be running atoms in the + * ring buffer even if there are no atoms to pull. It is also possible for the + * ring buffer to be full (with running atoms) when this functions returns + * true. + * + * Return: true if there are no atoms to pull, false otherwise. + */ +static inline bool +jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio) +{ + struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + return RB_EMPTY_ROOT(&rb->runnable_tree); +} + +/** + * jsctx_rb_none_to_pull(): - Check if all priority ring buffers have no + * pullable atoms + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * + * Caller must hold hwaccess_lock + * + * Return: true if the ring buffers for all priorities have no pullable atoms, + * false otherwise. + */ +static inline bool +jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) +{ + int prio; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { + if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) + return false; + } + + return true; +} + +/** + * jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue. + * @kctx: Pointer to kbase context with the queue. + * @js: Job slot id to iterate. + * @prio: Priority id to iterate. + * @callback: Function pointer to callback. + * + * Iterate over a queue and invoke @callback for each entry in the queue, and + * remove the entry from the queue. + * + * If entries are added to the queue while this is running those entries may, or + * may not be covered. To ensure that all entries in the buffer have been + * enumerated when this function returns jsctx->lock must be held when calling + * this function. + * + * The HW access lock must always be held when calling this function. + */ +static void +jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, + kbasep_js_ctx_job_cb callback) +{ + struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + while (!RB_EMPTY_ROOT(&queue->runnable_tree)) { + struct rb_node *node = rb_first(&queue->runnable_tree); + struct kbase_jd_atom *entry = rb_entry(node, + struct kbase_jd_atom, runnable_tree_node); + + rb_erase(node, &queue->runnable_tree); + callback(kctx->kbdev, entry); + } + + while (!list_empty(&queue->x_dep_head)) { + struct kbase_jd_atom *entry = list_entry(queue->x_dep_head.next, + struct kbase_jd_atom, queue); + + list_del(queue->x_dep_head.next); + + callback(kctx->kbdev, entry); + } +} + +/** + * jsctx_queue_foreach(): - Execute callback for each entry in every queue + * @kctx: Pointer to kbase context with queue. + * @js: Job slot id to iterate. + * @callback: Function pointer to callback. + * + * Iterate over all the different priorities, and for each call + * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback + * for each entry, and remove the entry from the queue. + */ +static inline void +jsctx_queue_foreach(struct kbase_context *kctx, int js, + kbasep_js_ctx_job_cb callback) +{ + int prio; + + for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) + jsctx_queue_foreach_prio(kctx, js, prio, callback); +} + +/** + * jsctx_rb_peek_prio(): - Check buffer and get next atom + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * @prio: Priority id to check. + * + * Check the ring buffer for the specified @js and @prio and return a pointer to + * the next atom, unless the ring buffer is empty. + * + * Return: Pointer to next atom in buffer, or NULL if there is no atom. + */ +static inline struct kbase_jd_atom * +jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) +{ + struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + struct rb_node *node; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + node = rb_first(&rb->runnable_tree); + if (!node) + return NULL; + + return rb_entry(node, struct kbase_jd_atom, runnable_tree_node); +} + +/** + * jsctx_rb_peek(): - Check all priority buffers and get next atom + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * + * Check the ring buffers for all priorities, starting from + * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a + * pointer to the next atom, unless all the priority's ring buffers are empty. + * + * Caller must hold the hwaccess_lock. + * + * Return: Pointer to next atom in buffer, or NULL if there is no atom. + */ +static inline struct kbase_jd_atom * +jsctx_rb_peek(struct kbase_context *kctx, int js) +{ + int prio; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { + struct kbase_jd_atom *katom; + + katom = jsctx_rb_peek_prio(kctx, js, prio); + if (katom) + return katom; + } + + return NULL; +} + +/** + * jsctx_rb_pull(): - Mark atom in list as running + * @kctx: Pointer to kbase context with ring buffer. + * @katom: Pointer to katom to pull. + * + * Mark an atom previously obtained from jsctx_rb_peek() as running. + * + * @katom must currently be at the head of the ring buffer. + */ +static inline void +jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + /* Atoms must be pulled in the correct order. */ + WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio)); + + rb_erase(&katom->runnable_tree_node, &rb->runnable_tree); +} + +#define LESS_THAN_WRAP(a, b) ((s32)(a - b) < 0) + +static void +jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; + struct rb_node **new = &(queue->runnable_tree.rb_node), *parent = NULL; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + while (*new) { + struct kbase_jd_atom *entry = container_of(*new, + struct kbase_jd_atom, runnable_tree_node); + + parent = *new; + if (LESS_THAN_WRAP(katom->age, entry->age)) + new = &((*new)->rb_left); + else + new = &((*new)->rb_right); + } + + /* Add new node and rebalance tree. */ + rb_link_node(&katom->runnable_tree_node, parent, new); + rb_insert_color(&katom->runnable_tree_node, &queue->runnable_tree); +} + +/** + * jsctx_rb_unpull(): - Undo marking of atom in list as running + * @kctx: Pointer to kbase context with ring buffer. + * @katom: Pointer to katom to unpull. + * + * Undo jsctx_rb_pull() and put @katom back in the queue. + * + * jsctx_rb_unpull() must be called on atoms in the same order the atoms were + * pulled. + */ +static inline void +jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + jsctx_tree_add(kctx, katom); +} + +static bool kbase_js_ctx_pullable(struct kbase_context *kctx, + int js, + bool is_scheduled); +static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js); +static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js); + +/* + * Functions private to KBase ('Protected' functions) + */ +int kbasep_js_devdata_init(struct kbase_device * const kbdev) +{ + struct kbasep_js_device_data *jsdd; + int i; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + jsdd = &kbdev->js_data; + +#ifdef CONFIG_MALI_DEBUG + /* Soft-stop will be disabled on a single context by default unless + * softstop_always is set */ + jsdd->softstop_always = false; +#endif /* CONFIG_MALI_DEBUG */ + jsdd->nr_all_contexts_running = 0; + jsdd->nr_user_contexts_running = 0; + jsdd->nr_contexts_pullable = 0; + atomic_set(&jsdd->nr_contexts_runnable, 0); + /* No ctx allowed to submit */ + jsdd->runpool_irq.submit_allowed = 0u; + memset(jsdd->runpool_irq.ctx_attr_ref_count, 0, + sizeof(jsdd->runpool_irq.ctx_attr_ref_count)); + memset(jsdd->runpool_irq.slot_affinities, 0, + sizeof(jsdd->runpool_irq.slot_affinities)); + memset(jsdd->runpool_irq.slot_affinity_refcount, 0, + sizeof(jsdd->runpool_irq.slot_affinity_refcount)); + INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list); + + /* Config attributes */ + jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; + jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; + jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) + jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS_8408; + else + jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; + jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; + jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) + jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS_8408; + else + jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; + jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; + jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING; + jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS; + atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT); + + dev_dbg(kbdev->dev, "JS Config Attribs: "); + dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u", + jsdd->scheduling_period_ns); + dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u", + jsdd->soft_stop_ticks); + dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u", + jsdd->soft_stop_ticks_cl); + dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u", + jsdd->hard_stop_ticks_ss); + dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u", + jsdd->hard_stop_ticks_cl); + dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u", + jsdd->hard_stop_ticks_dumping); + dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u", + jsdd->gpu_reset_ticks_ss); + dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u", + jsdd->gpu_reset_ticks_cl); + dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u", + jsdd->gpu_reset_ticks_dumping); + dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u", + jsdd->ctx_timeslice_ns); + dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i", + atomic_read(&jsdd->soft_job_timeout_ms)); + + if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss && + jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss && + jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping && + jsdd->hard_stop_ticks_dumping < + jsdd->gpu_reset_ticks_dumping)) { + dev_err(kbdev->dev, "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n"); + return -EINVAL; + } + +#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS + dev_dbg(kbdev->dev, "Job Scheduling Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.", + jsdd->soft_stop_ticks, + jsdd->scheduling_period_ns); +#endif +#if KBASE_DISABLE_SCHEDULING_HARD_STOPS + dev_dbg(kbdev->dev, "Job Scheduling Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.", + jsdd->hard_stop_ticks_ss, + jsdd->hard_stop_ticks_dumping, + jsdd->scheduling_period_ns); +#endif +#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS + dev_dbg(kbdev->dev, "Note: The JS tick timer (if coded) will still be run, but do nothing."); +#endif + + for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) + jsdd->js_reqs[i] = core_reqs_from_jsn_features( + kbdev->gpu_props.props.raw_props.js_features[i]); + + /* On error, we could continue on: providing none of the below resources + * rely on the ones above */ + + mutex_init(&jsdd->runpool_mutex); + mutex_init(&jsdd->queue_mutex); + spin_lock_init(&kbdev->hwaccess_lock); + sema_init(&jsdd->schedule_sem, 1); + + for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { + INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i]); + INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i]); + } + + return 0; +} + +void kbasep_js_devdata_halt(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +void kbasep_js_devdata_term(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata; + s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { 0, }; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + js_devdata = &kbdev->js_data; + + /* The caller must de-register all contexts before calling this + */ + KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0); + KBASE_DEBUG_ASSERT(memcmp( + js_devdata->runpool_irq.ctx_attr_ref_count, + zero_ctx_attr_ref_count, + sizeof(zero_ctx_attr_ref_count)) == 0); + CSTD_UNUSED(zero_ctx_attr_ref_count); +} + +int kbasep_js_kctx_init(struct kbase_context * const kctx) +{ + struct kbase_device *kbdev; + struct kbasep_js_kctx_info *js_kctx_info; + int i, j; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) + INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]); + + js_kctx_info = &kctx->jctx.sched_info; + + js_kctx_info->ctx.nr_jobs = 0; + kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); + kbase_ctx_flag_clear(kctx, KCTX_DYING); + memset(js_kctx_info->ctx.ctx_attr_ref_count, 0, + sizeof(js_kctx_info->ctx.ctx_attr_ref_count)); + + /* Initially, the context is disabled from submission until the create + * flags are set */ + kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); + + /* On error, we could continue on: providing none of the below resources + * rely on the ones above */ + mutex_init(&js_kctx_info->ctx.jsctx_mutex); + + init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); + + for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { + for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { + INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); + kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; + } + } + + return 0; +} + +void kbasep_js_kctx_term(struct kbase_context *kctx) +{ + struct kbase_device *kbdev; + struct kbasep_js_kctx_info *js_kctx_info; + int js; + bool update_ctx_count = false; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + kbdev = kctx->kbdev; + KBASE_DEBUG_ASSERT(kbdev != NULL); + + js_kctx_info = &kctx->jctx.sched_info; + + /* The caller must de-register all jobs before calling this */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0); + + mutex_lock(&kbdev->js_data.queue_mutex); + mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + if (kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)) { + WARN_ON(atomic_read(&kbdev->js_data.nr_contexts_runnable) <= 0); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + update_ctx_count = true; + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + } + + mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&kbdev->js_data.queue_mutex); + + if (update_ctx_count) { + mutex_lock(&kbdev->js_data.runpool_mutex); + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&kbdev->js_data.runpool_mutex); + } +} + +/** + * kbase_js_ctx_list_add_pullable_nolock - Variant of + * kbase_jd_ctx_list_add_pullable() + * where the caller must hold + * hwaccess_lock + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_pullable[js]); + + if (!kctx->slots_pullable) { + kbdev->js_data.nr_contexts_pullable++; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); + atomic_inc(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable |= (1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_add_pullable_head_nolock - Variant of + * kbase_js_ctx_list_add_pullable_head() + * where the caller must hold + * hwaccess_lock + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_pullable_head_nolock( + struct kbase_device *kbdev, struct kbase_context *kctx, int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_pullable[js]); + + if (!kctx->slots_pullable) { + kbdev->js_data.nr_contexts_pullable++; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); + atomic_inc(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable |= (1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_add_pullable_head - Add context to the head of the + * per-slot pullable context queue + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * If the context is on either the pullable or unpullable queues, then it is + * removed before being added to the head. + * + * This function should be used when a context has been scheduled, but no jobs + * can currently be pulled from it. + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + ret = kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return ret; +} + +/** + * kbase_js_ctx_list_add_unpullable_nolock - Add context to the tail of the + * per-slot unpullable context queue + * @kbdev: Device pointer + * @kctx: Context to add to queue + * @js: Job slot to use + * + * The context must already be on the per-slot pullable queue. It will be + * removed from the pullable queue before being added to the unpullable queue. + * + * This function should be used when a context has been pulled from, and there + * are no jobs remaining on the specified slot. + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], + &kbdev->js_data.ctx_list_unpullable[js]); + + if (kctx->slots_pullable == (1 << js)) { + kbdev->js_data.nr_contexts_pullable--; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable &= ~(1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_remove_nolock - Remove context from the per-slot pullable + * or unpullable context queues + * @kbdev: Device pointer + * @kctx: Context to remove from queue + * @js: Job slot to use + * + * The context must already be on one of the queues. + * + * This function should be used when a context has no jobs on the GPU, and no + * jobs remaining for the specified slot. + * + * Caller must hold hwaccess_lock + * + * Return: true if caller should call kbase_backend_ctx_count_changed() + */ +static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) +{ + bool ret = false; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])); + + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + if (kctx->slots_pullable == (1 << js)) { + kbdev->js_data.nr_contexts_pullable--; + ret = true; + if (!atomic_read(&kctx->atoms_pulled)) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + } + } + kctx->slots_pullable &= ~(1 << js); + + return ret; +} + +/** + * kbase_js_ctx_list_pop_head_nolock - Variant of kbase_js_ctx_list_pop_head() + * where the caller must hold + * hwaccess_lock + * @kbdev: Device pointer + * @js: Job slot to use + * + * Caller must hold hwaccess_lock + * + * Return: Context to use for specified slot. + * NULL if no contexts present for specified slot + */ +static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( + struct kbase_device *kbdev, + int js) +{ + struct kbase_context *kctx; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (list_empty(&kbdev->js_data.ctx_list_pullable[js])) + return NULL; + + kctx = list_entry(kbdev->js_data.ctx_list_pullable[js].next, + struct kbase_context, + jctx.sched_info.ctx.ctx_list_entry[js]); + + list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + + return kctx; +} + +/** + * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable + * queue. + * @kbdev: Device pointer + * @js: Job slot to use + * + * Return: Context to use for specified slot. + * NULL if no contexts present for specified slot + */ +static struct kbase_context *kbase_js_ctx_list_pop_head( + struct kbase_device *kbdev, int js) +{ + struct kbase_context *kctx; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kctx = kbase_js_ctx_list_pop_head_nolock(kbdev, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return kctx; +} + +/** + * kbase_js_ctx_pullable - Return if a context can be pulled from on the + * specified slot + * @kctx: Context pointer + * @js: Job slot to use + * @is_scheduled: true if the context is currently scheduled + * + * Caller must hold hwaccess_lock + * + * Return: true if context can be pulled from on specified slot + * false otherwise + */ +static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js, + bool is_scheduled) +{ + struct kbasep_js_device_data *js_devdata; + struct kbase_jd_atom *katom; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + js_devdata = &kctx->kbdev->js_data; + + if (is_scheduled) { + if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) + return false; + } + katom = jsctx_rb_peek(kctx, js); + if (!katom) + return false; /* No pullable atoms */ + if (kctx->blocked_js[js][katom->sched_priority]) + return false; + if (atomic_read(&katom->blocked)) + return false; /* next atom blocked */ + if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { + if (katom->x_pre_dep->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || + katom->x_pre_dep->will_fail_event_code) + return false; + if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && + kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) + return false; + } + + return true; +} + +static bool kbase_js_dep_validate(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + struct kbase_device *kbdev = kctx->kbdev; + bool ret = true; + bool has_dep = false, has_x_dep = false; + int js = kbase_js_get_slot(kbdev, katom); + int prio = katom->sched_priority; + int i; + + for (i = 0; i < 2; i++) { + struct kbase_jd_atom *dep_atom = katom->dep[i].atom; + + if (dep_atom) { + int dep_js = kbase_js_get_slot(kbdev, dep_atom); + int dep_prio = dep_atom->sched_priority; + + /* Dependent atom must already have been submitted */ + if (!(dep_atom->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { + ret = false; + break; + } + + /* Dependencies with different priorities can't + be represented in the ringbuffer */ + if (prio != dep_prio) { + ret = false; + break; + } + + if (js == dep_js) { + /* Only one same-slot dependency can be + * represented in the ringbuffer */ + if (has_dep) { + ret = false; + break; + } + /* Each dependee atom can only have one + * same-slot dependency */ + if (dep_atom->post_dep) { + ret = false; + break; + } + has_dep = true; + } else { + /* Only one cross-slot dependency can be + * represented in the ringbuffer */ + if (has_x_dep) { + ret = false; + break; + } + /* Each dependee atom can only have one + * cross-slot dependency */ + if (dep_atom->x_post_dep) { + ret = false; + break; + } + /* The dependee atom can not already be in the + * HW access ringbuffer */ + if (dep_atom->gpu_rb_state != + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + ret = false; + break; + } + /* The dependee atom can not already have + * completed */ + if (dep_atom->status != + KBASE_JD_ATOM_STATE_IN_JS) { + ret = false; + break; + } + /* Cross-slot dependencies must not violate + * PRLAM-8987 affinity restrictions */ + if (kbase_hw_has_issue(kbdev, + BASE_HW_ISSUE_8987) && + (js == 2 || dep_js == 2)) { + ret = false; + break; + } + has_x_dep = true; + } + + /* Dependency can be represented in ringbuffers */ + } + } + + /* If dependencies can be represented by ringbuffer then clear them from + * atom structure */ + if (ret) { + for (i = 0; i < 2; i++) { + struct kbase_jd_atom *dep_atom = katom->dep[i].atom; + + if (dep_atom) { + int dep_js = kbase_js_get_slot(kbdev, dep_atom); + + if ((js != dep_js) && + (dep_atom->status != + KBASE_JD_ATOM_STATE_COMPLETED) + && (dep_atom->status != + KBASE_JD_ATOM_STATE_HW_COMPLETED) + && (dep_atom->status != + KBASE_JD_ATOM_STATE_UNUSED)) { + + katom->atom_flags |= + KBASE_KATOM_FLAG_X_DEP_BLOCKED; + katom->x_pre_dep = dep_atom; + dep_atom->x_post_dep = katom; + if (kbase_jd_katom_dep_type( + &katom->dep[i]) == + BASE_JD_DEP_TYPE_DATA) + katom->atom_flags |= + KBASE_KATOM_FLAG_FAIL_BLOCKER; + } + if ((kbase_jd_katom_dep_type(&katom->dep[i]) + == BASE_JD_DEP_TYPE_DATA) && + (js == dep_js)) { + katom->pre_dep = dep_atom; + dep_atom->post_dep = katom; + } + + list_del(&katom->dep_item[i]); + kbase_jd_katom_dep_clear(&katom->dep[i]); + } + } + } + + return ret; +} + +bool kbasep_js_add_job(struct kbase_context *kctx, + struct kbase_jd_atom *atom) +{ + unsigned long flags; + struct kbasep_js_kctx_info *js_kctx_info; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + + bool enqueue_required = false; + bool timer_sync = false; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(atom != NULL); + lockdep_assert_held(&kctx->jctx.lock); + + kbdev = kctx->kbdev; + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + /* + * Begin Runpool transaction + */ + mutex_lock(&js_devdata->runpool_mutex); + + /* Refcount ctx.nr_jobs */ + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX); + ++(js_kctx_info->ctx.nr_jobs); + + /* Setup any scheduling information */ + kbasep_js_clear_job_retry_submit(atom); + + /* Lock for state available during IRQ */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbase_js_dep_validate(kctx, atom)) { + /* Dependencies could not be represented */ + --(js_kctx_info->ctx.nr_jobs); + + /* Setting atom status back to queued as it still has unresolved + * dependencies */ + atom->status = KBASE_JD_ATOM_STATE_QUEUED; + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->runpool_mutex); + + goto out_unlock; + } + + KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, TL_ATOM_STATE_READY); + KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, atom)); + + enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); + + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, + kbasep_js_trace_get_refcnt(kbdev, kctx)); + + /* Context Attribute Refcounting */ + kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); + + if (enqueue_required) { + if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false)) + timer_sync = kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, atom->slot_nr); + else + timer_sync = kbase_js_ctx_list_add_unpullable_nolock( + kbdev, kctx, atom->slot_nr); + } + /* If this context is active and the atom is the first on its slot, + * kick the job manager to attempt to fast-start the atom */ + if (enqueue_required && kctx == kbdev->hwaccess.active_kctx) + kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); + /* End runpool transaction */ + + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + /* A job got added while/after kbase_job_zap_context() + * was called on a non-scheduled context (e.g. KDS + * dependency resolved). Kill that job by killing the + * context. */ + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, + false); + } else if (js_kctx_info->ctx.nr_jobs == 1) { + /* Handle Refcount going from 0 to 1: schedule the + * context on the Queue */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx); + + /* Queue was updated - caller must try to + * schedule the head context */ + WARN_ON(!enqueue_required); + } + } +out_unlock: + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + mutex_unlock(&js_devdata->queue_mutex); + + return enqueue_required; +} + +void kbasep_js_remove_job(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_jd_atom *atom) +{ + struct kbasep_js_kctx_info *js_kctx_info; + struct kbasep_js_device_data *js_devdata; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(atom != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, + kbasep_js_trace_get_refcnt(kbdev, kctx)); + + /* De-refcount ctx.nr_jobs */ + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); + --(js_kctx_info->ctx.nr_jobs); +} + +bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + unsigned long flags; + struct kbasep_js_atom_retained_state katom_retained_state; + struct kbasep_js_device_data *js_devdata; + bool attr_state_changed; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(katom != NULL); + + js_devdata = &kbdev->js_data; + + kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); + kbasep_js_remove_job(kbdev, kctx, katom); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* The atom has 'finished' (will not be re-run), so no need to call + * kbasep_js_has_atom_finished(). + * + * This is because it returns false for soft-stopped atoms, but we + * want to override that, because we're cancelling an atom regardless of + * whether it was soft-stopped or not */ + attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, + &katom_retained_state); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return attr_state_changed; +} + +bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + unsigned long flags; + struct kbasep_js_device_data *js_devdata; + bool result; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + js_devdata = &kbdev->js_data; + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + return result; +} + +struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, + int as_nr) +{ + int ret = 0; + unsigned long flags; + struct kbasep_js_device_data *js_devdata; + struct kbase_context *found_kctx = NULL; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); + js_devdata = &kbdev->js_data; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + found_kctx = kbdev->as_to_kctx[as_nr]; + + if (found_kctx != NULL) { + ret = kbase_ctx_sched_retain_ctx_refcount(found_kctx); + if (ret != 0) { + E("fail to retain ctx_refcount, ret : %d.", ret); + found_kctx = NULL; + } + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return found_kctx; +} + +/** + * kbasep_js_release_result - Try running more jobs after releasing a context + * and/or atom + * + * @kbdev: The kbase_device to operate on + * @kctx: The kbase_context to operate on + * @katom_retained_state: Retained state from the atom + * @runpool_ctx_attr_change: True if the runpool context attributes have changed + * + * This collates a set of actions that must happen whilst hwaccess_lock is held. + * + * This includes running more jobs when: + * - The previously released kctx caused a ctx attribute change, + * - The released atom caused a ctx attribute change, + * - Slots were previously blocked due to affinity restrictions, + * - Submission during IRQ handling failed. + * + * Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were + * changed. The caller should try scheduling all contexts + */ +static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( + struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state, + bool runpool_ctx_attr_change) +{ + struct kbasep_js_device_data *js_devdata; + kbasep_js_release_result result = 0; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(katom_retained_state != NULL); + js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (js_devdata->nr_user_contexts_running != 0) { + bool retry_submit = false; + int retry_jobslot = 0; + + if (katom_retained_state) + retry_submit = kbasep_js_get_atom_retry_submit_slot( + katom_retained_state, &retry_jobslot); + + if (runpool_ctx_attr_change || retry_submit) { + /* A change in runpool ctx attributes might mean we can + * run more jobs than before */ + result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; + + KBASE_TRACE_ADD_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, + kctx, NULL, 0u, retry_jobslot); + } + } + return result; +} + +/* + * Internal function to release the reference on a ctx and an atom's "retained + * state", only taking the runpool and as transaction mutexes + * + * This also starts more jobs running in the case of an ctx-attribute state + * change + * + * This does none of the followup actions for scheduling: + * - It does not schedule in a new context + * - It does not requeue or handle dying contexts + * + * For those tasks, just call kbasep_js_runpool_release_ctx() instead + * + * Requires: + * - Context is scheduled in, and kctx->as_nr matches kctx_as_nr + * - Context has a non-zero refcount + * - Caller holds js_kctx_info->ctx.jsctx_mutex + * - Caller holds js_devdata->runpool_mutex + */ +static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( + struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state) +{ + unsigned long flags; + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + + kbasep_js_release_result release_result = 0u; + bool runpool_ctx_attr_change = false; + int kctx_as_nr; + struct kbase_as *current_as; + int new_ref_count; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + kctx_as_nr = kctx->as_nr; + KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); + + /* + * Transaction begins on AS and runpool_irq + * + * Assert about out calling contract + */ + current_as = &kbdev->as[kctx_as_nr]; + mutex_lock(&kbdev->pm.lock); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr); + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); + + /* Update refcount */ + kbase_ctx_sched_release_ctx(kctx); + new_ref_count = atomic_read(&kctx->refcount); + + /* Release the atom if it finished (i.e. wasn't soft-stopped) */ + if (kbasep_js_has_atom_finished(katom_retained_state)) + runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom( + kbdev, kctx, katom_retained_state); + + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u, + new_ref_count); + + if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && + !kbase_pm_is_suspending(kbdev)) { + /* Context is kept scheduled into an address space even when + * there are no jobs, in this case we have to handle the + * situation where all jobs have been evicted from the GPU and + * submission is disabled. + * + * At this point we re-enable submission to allow further jobs + * to be executed + */ + kbasep_js_set_submit_allowed(js_devdata, kctx); + } + + /* Make a set of checks to see if the context should be scheduled out. + * Note that there'll always be at least 1 reference to the context + * which was previously acquired by kbasep_js_schedule_ctx(). */ + if (new_ref_count == 1 && + (!kbasep_js_is_submit_allowed(js_devdata, kctx) || + kbdev->pm.suspending)) { + int num_slots = kbdev->gpu_props.num_job_slots; + int slot; + + /* Last reference, and we've been told to remove this context + * from the Run Pool */ + dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d", + kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, + kbasep_js_is_submit_allowed(js_devdata, kctx)); + +#if defined(CONFIG_MALI_GATOR_SUPPORT) + kbase_trace_mali_mmu_as_released(kctx->as_nr); +#endif + KBASE_TLSTREAM_TL_NRET_AS_CTX(&kbdev->as[kctx->as_nr], kctx); + + kbase_backend_release_ctx_irq(kbdev, kctx); + + if (kbdev->hwaccess.active_kctx == kctx) + kbdev->hwaccess.active_kctx = NULL; + + /* Ctx Attribute handling + * + * Releasing atoms attributes must either happen before this, or + * after the KCTX_SHEDULED flag is changed, otherwise we + * double-decount the attributes + */ + runpool_ctx_attr_change |= + kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); + + /* Releasing the context and katom retained state can allow + * more jobs to run */ + release_result |= + kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, + kctx, katom_retained_state, + runpool_ctx_attr_change); + + /* + * Transaction ends on AS and runpool_irq: + * + * By this point, the AS-related data is now clear and ready + * for re-use. + * + * Since releases only occur once for each previous successful + * retain, and no more retains are allowed on this context, no + * other thread will be operating in this + * code whilst we are + */ + + /* Recalculate pullable status for all slots */ + for (slot = 0; slot < num_slots; slot++) { + if (kbase_js_ctx_pullable(kctx, slot, false)) + kbase_js_ctx_list_add_pullable_nolock(kbdev, + kctx, slot); + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + kbase_backend_release_ctx_noirq(kbdev, kctx); + + mutex_unlock(&kbdev->pm.lock); + + /* Note: Don't reuse kctx_as_nr now */ + + /* Synchronize with any timers */ + kbase_backend_ctx_count_changed(kbdev); + + /* update book-keeping info */ + kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); + /* Signal any waiter that the context is not scheduled, so is + * safe for termination - once the jsctx_mutex is also dropped, + * and jobs have finished. */ + wake_up(&js_kctx_info->ctx.is_scheduled_wait); + + /* Queue an action to occur after we've dropped the lock */ + release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED | + KBASEP_JS_RELEASE_RESULT_SCHED_ALL; + } else { + kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, + katom_retained_state, runpool_ctx_attr_change); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->pm.lock); + } + + return release_result; +} + +void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_atom_retained_state katom_retained_state; + + /* Setup a dummy katom_retained_state */ + kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); + + kbasep_js_runpool_release_ctx_internal(kbdev, kctx, + &katom_retained_state); +} + +void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx, bool has_pm_ref) +{ + struct kbasep_js_device_data *js_devdata; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_devdata = &kbdev->js_data; + + /* This is called if and only if you've you've detached the context from + * the Runpool Queue, and not added it back to the Runpool + */ + KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + /* Dying: don't requeue, but kill all jobs on the context. This + * happens asynchronously */ + dev_dbg(kbdev->dev, + "JS: ** Killing Context %p on RunPool Remove **", kctx); + kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); + } +} + +void kbasep_js_runpool_release_ctx_and_katom_retained_state( + struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbasep_js_atom_retained_state *katom_retained_state) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + kbasep_js_release_result release_result; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, + katom_retained_state); + + /* Drop the runpool mutex to allow requeing kctx */ + mutex_unlock(&js_devdata->runpool_mutex); + + if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); + + /* Drop the jsctx_mutex to allow scheduling in a new context */ + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL) + kbase_js_sched_all(kbdev); +} + +void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_atom_retained_state katom_retained_state; + + kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); + + kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, + &katom_retained_state); +} + +/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into + * kbase_js_sched_all() */ +static void kbasep_js_runpool_release_ctx_no_schedule( + struct kbase_device *kbdev, struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + kbasep_js_release_result release_result; + struct kbasep_js_atom_retained_state katom_retained_state_struct; + struct kbasep_js_atom_retained_state *katom_retained_state = + &katom_retained_state_struct; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + kbasep_js_atom_retained_state_init_invalid(katom_retained_state); + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + + release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, + katom_retained_state); + + /* Drop the runpool mutex to allow requeing kctx */ + mutex_unlock(&js_devdata->runpool_mutex); + if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); + + /* Drop the jsctx_mutex to allow scheduling in a new context */ + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + /* NOTE: could return release_result if the caller would like to know + * whether it should schedule a new context, but currently no callers do + */ +} + +void kbase_js_set_timeouts(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + kbase_backend_timeouts_changed(kbdev); +} + +static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + struct kbase_as *new_address_space = NULL; + unsigned long flags; + bool kctx_suspended = false; + int as_nr; + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + /* Pick available address space for this context */ + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + as_nr = kbase_ctx_sched_retain_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + if (as_nr == KBASEP_AS_NR_INVALID) { + as_nr = kbase_backend_find_and_release_free_address_space( + kbdev, kctx); + if (as_nr != KBASEP_AS_NR_INVALID) { + /* Attempt to retain the context again, this should + * succeed */ + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + as_nr = kbase_ctx_sched_retain_ctx(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + WARN_ON(as_nr == KBASEP_AS_NR_INVALID); + } + } + if (as_nr == KBASEP_AS_NR_INVALID) + return false; /* No address spaces currently available */ + + new_address_space = &kbdev->as[as_nr]; + + /* + * Atomic transaction on the Context and Run Pool begins + */ + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Check to see if context is dying due to kbase_job_zap_context() */ + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + /* Roll back the transaction so far and return */ + kbase_ctx_sched_release_ctx(kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return false; + } + + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, + 0u, + kbasep_js_trace_get_refcnt(kbdev, kctx)); + + kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); + + /* Assign context to previously chosen address space */ + if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { + /* Roll back the transaction so far and return */ + kbase_ctx_sched_release_ctx(kctx); + kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + return false; + } + + kbdev->hwaccess.active_kctx = kctx; + +#if defined(CONFIG_MALI_GATOR_SUPPORT) + kbase_trace_mali_mmu_as_in_use(kctx->as_nr); +#endif + KBASE_TLSTREAM_TL_RET_AS_CTX(&kbdev->as[kctx->as_nr], kctx); + + /* Cause any future waiter-on-termination to wait until the context is + * descheduled */ + wake_up(&js_kctx_info->ctx.is_scheduled_wait); + + /* Re-check for suspending: a suspend could've occurred, and all the + * contexts could've been removed from the runpool before we took this + * lock. In this case, we don't want to allow this context to run jobs, + * we just want it out immediately. + * + * The DMB required to read the suspend flag was issued recently as part + * of the hwaccess_lock locking. If a suspend occurs *after* that lock + * was taken (i.e. this condition doesn't execute), then the + * kbasep_js_suspend() code will cleanup this context instead (by virtue + * of it being called strictly after the suspend flag is set, and will + * wait for this lock to drop) */ + if (kbase_pm_is_suspending(kbdev)) { + /* Cause it to leave at some later point */ + bool retained; + + retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + KBASE_DEBUG_ASSERT(retained); + + kbasep_js_clear_submit_allowed(js_devdata, kctx); + kctx_suspended = true; + } + + /* Transaction complete */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + + /* Synchronize with any timers */ + kbase_backend_ctx_count_changed(kbdev); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + /* Note: after this point, the context could potentially get scheduled + * out immediately */ + + if (kctx_suspended) { + /* Finishing forcing out the context due to a suspend. Use a + * variant of kbasep_js_runpool_release_ctx() that doesn't + * schedule a new context, to prevent a risk of recursion back + * into this function */ + kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); + return false; + } + return true; +} + +static bool kbase_js_use_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_backend_use_ctx_sched(kbdev, kctx)) { + /* Context already has ASID - mark as active */ + kbdev->hwaccess.active_kctx = kctx; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return true; /* Context already scheduled */ + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + return kbasep_js_schedule_ctx(kbdev, kctx); +} + +void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_kctx_info *js_kctx_info; + struct kbasep_js_device_data *js_devdata; + bool is_scheduled; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + /* This must never be attempted whilst suspending - i.e. it should only + * happen in response to a syscall from a user-space thread */ + BUG_ON(kbase_pm_is_suspending(kbdev)); + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + /* Mark the context as privileged */ + kbase_ctx_flag_set(kctx, KCTX_PRIVILEGED); + + is_scheduled = kbase_ctx_flag(kctx, KCTX_SCHEDULED); + if (!is_scheduled) { + /* Add the context to the pullable list */ + if (kbase_js_ctx_list_add_pullable_head(kbdev, kctx, 0)) + kbase_js_sync_timers(kbdev); + + /* Fast-starting requires the jsctx_mutex to be dropped, + * because it works on multiple ctxs */ + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + /* Try to schedule the context in */ + kbase_js_sched_all(kbdev); + + /* Wait for the context to be scheduled in */ + wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, + kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + } else { + /* Already scheduled in - We need to retain it to keep the + * corresponding address space */ + kbasep_js_runpool_retain_ctx(kbdev, kctx); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + } +} +KBASE_EXPORT_TEST_API(kbasep_js_schedule_privileged_ctx); + +void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kctx != NULL); + js_kctx_info = &kctx->jctx.sched_info; + + /* We don't need to use the address space anymore */ + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + kbase_ctx_flag_clear(kctx, KCTX_PRIVILEGED); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + /* Release the context - it will be scheduled out */ + kbasep_js_runpool_release_ctx(kbdev, kctx); + + kbase_js_sched_all(kbdev); +} +KBASE_EXPORT_TEST_API(kbasep_js_release_privileged_ctx); + +void kbasep_js_suspend(struct kbase_device *kbdev) +{ + unsigned long flags; + struct kbasep_js_device_data *js_devdata; + int i; + u16 retained = 0u; + int nr_privileged_ctx = 0; + + KBASE_DEBUG_ASSERT(kbdev); + KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev)); + js_devdata = &kbdev->js_data; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Prevent all contexts from submitting */ + js_devdata->runpool_irq.submit_allowed = 0; + + /* Retain each of the contexts, so we can cause it to leave even if it + * had no refcount to begin with */ + for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { + struct kbase_context *kctx = kbdev->as_to_kctx[i]; + + retained = retained << 1; + + if (kctx) { + kbase_ctx_sched_retain_ctx_refcount(kctx); + retained |= 1u; + /* We can only cope with up to 1 privileged context - + * the instrumented context. It'll be suspended by + * disabling instrumentation */ + if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { + ++nr_privileged_ctx; + WARN_ON(nr_privileged_ctx != 1); + } + } + } + CSTD_UNUSED(nr_privileged_ctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* De-ref the previous retain to ensure each context gets pulled out + * sometime later. */ + for (i = 0; + i < BASE_MAX_NR_AS; + ++i, retained = retained >> 1) { + struct kbase_context *kctx = kbdev->as_to_kctx[i]; + + if (retained & 1u) + kbasep_js_runpool_release_ctx(kbdev, kctx); + } + + /* Caller must wait for all Power Manager active references to be + * dropped */ +} + +void kbasep_js_resume(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata; + int js; + + KBASE_DEBUG_ASSERT(kbdev); + js_devdata = &kbdev->js_data; + KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); + + mutex_lock(&js_devdata->queue_mutex); + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + struct kbase_context *kctx, *n; + + list_for_each_entry_safe(kctx, n, + &kbdev->js_data.ctx_list_unpullable[js], + jctx.sched_info.ctx.ctx_list_entry[js]) { + struct kbasep_js_kctx_info *js_kctx_info; + unsigned long flags; + bool timer_sync = false; + + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_js_ctx_pullable(kctx, js, false)) + timer_sync = + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + } + } + mutex_unlock(&js_devdata->queue_mutex); + + /* Restart atom processing */ + kbase_js_sched_all(kbdev); + + /* JS Resume complete */ +} + +bool kbase_js_is_atom_valid(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + if ((katom->core_req & BASE_JD_REQ_FS) && + (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | + BASE_JD_REQ_T))) + return false; + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987) && + (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) && + (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_T))) + return false; + + return true; +} + +static int kbase_js_get_slot(struct kbase_device *kbdev, + struct kbase_jd_atom *katom) +{ + if (katom->core_req & BASE_JD_REQ_FS) + return 0; + + if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { + if (katom->device_nr == 1 && + kbdev->gpu_props.num_core_groups == 2) + return 2; + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) + return 2; + } + + return 1; +} + +bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + bool enqueue_required; + + katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + lockdep_assert_held(&kctx->jctx.lock); + + /* If slot will transition from unpullable to pullable then add to + * pullable list */ + if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) { + enqueue_required = true; + } else { + enqueue_required = false; + } + if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || + (katom->pre_dep && (katom->pre_dep->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; + + list_add_tail(&katom->queue, &queue->x_dep_head); + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; + enqueue_required = false; + } else { + /* Check if there are lower priority jobs to soft stop */ + kbase_job_slot_ctx_priority_check_locked(kctx, katom); + + /* Add atom to ring buffer. */ + jsctx_tree_add(kctx, katom); + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; + } + + return enqueue_required; +} + +/** + * kbase_js_move_to_tree - Move atom (and any dependent atoms) to the + * runnable_tree, ready for execution + * @katom: Atom to submit + * + * It is assumed that @katom does not have KBASE_KATOM_FLAG_X_DEP_BLOCKED set, + * but is still present in the x_dep list. If @katom has a same-slot dependent + * atom then that atom (and any dependents) will also be moved. + */ +static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&katom->kctx->kbdev->hwaccess_lock); + + while (katom) { + WARN_ON(!(katom->atom_flags & + KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); + + if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { + list_del(&katom->queue); + katom->atom_flags &= + ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; + jsctx_tree_add(katom->kctx, katom); + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; + } else { + break; + } + + katom = katom->post_dep; + } +} + + +/** + * kbase_js_evict_deps - Evict dependencies of a failed atom. + * @kctx: Context pointer + * @katom: Pointer to the atom that has failed. + * @js: The job slot the katom was run on. + * @prio: Priority of the katom. + * + * Remove all post dependencies of an atom from the context ringbuffers. + * + * The original atom's event_code will be propogated to all dependent atoms. + * + * Context: Caller must hold the HW access lock + */ +static void kbase_js_evict_deps(struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js, int prio) +{ + struct kbase_jd_atom *x_dep = katom->x_post_dep; + struct kbase_jd_atom *next_katom = katom->post_dep; + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + if (next_katom) { + KBASE_DEBUG_ASSERT(next_katom->status != + KBASE_JD_ATOM_STATE_HW_COMPLETED); + next_katom->will_fail_event_code = katom->event_code; + + } + + /* Has cross slot depenency. */ + if (x_dep && (x_dep->atom_flags & (KBASE_KATOM_FLAG_JSCTX_IN_TREE | + KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { + /* Remove dependency.*/ + x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; + + /* Fail if it had a data dependency. */ + if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) { + x_dep->will_fail_event_code = katom->event_code; + } + if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) + kbase_js_move_to_tree(x_dep); + } +} + +struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) +{ + struct kbase_jd_atom *katom; + struct kbasep_js_device_data *js_devdata; + struct kbase_device *kbdev; + int pulled; + + KBASE_DEBUG_ASSERT(kctx); + + kbdev = kctx->kbdev; + + js_devdata = &kbdev->js_data; + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) + return NULL; + if (kbase_pm_is_suspending(kbdev)) + return NULL; + + katom = jsctx_rb_peek(kctx, js); + if (!katom) + return NULL; + if (kctx->blocked_js[js][katom->sched_priority]) + return NULL; + if (atomic_read(&katom->blocked)) + return NULL; + + /* Due to ordering restrictions when unpulling atoms on failure, we do + * not allow multiple runs of fail-dep atoms from the same context to be + * present on the same slot */ + if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { + struct kbase_jd_atom *prev_atom = + kbase_backend_inspect_tail(kbdev, js); + + if (prev_atom && prev_atom->kctx != kctx) + return NULL; + } + + if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { + if (katom->x_pre_dep->gpu_rb_state == + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || + katom->x_pre_dep->will_fail_event_code) + return NULL; + if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && + kbase_backend_nr_atoms_on_slot(kbdev, js)) + return NULL; + } + + kbase_ctx_flag_set(kctx, KCTX_PULLED); + + pulled = atomic_inc_return(&kctx->atoms_pulled); + if (pulled == 1 && !kctx->slots_pullable) { + WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); + atomic_inc(&kbdev->js_data.nr_contexts_runnable); + } + atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]); + kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++; + jsctx_rb_pull(kctx, katom); + + kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + + katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF; + + katom->ticks = 0; + + return katom; +} + + +static void js_return_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; + struct kbasep_js_atom_retained_state retained_state; + int js = katom->slot_nr; + int prio = katom->sched_priority; + bool timer_sync = false; + bool context_idle = false; + unsigned long flags; + base_jd_core_req core_req = katom->core_req; + u64 affinity = katom->affinity; + enum kbase_atom_coreref_state coreref_state = katom->coreref_state; + + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(katom); + + kbase_backend_complete_wq(kbdev, katom); + + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) + kbase_as_poking_timer_release_atom(kbdev, kctx, katom); + + kbasep_js_atom_retained_state_copy(&retained_state, katom); + + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + + atomic_dec(&kctx->atoms_pulled); + atomic_dec(&kctx->atoms_pulled_slot[js]); + + atomic_dec(&katom->blocked); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kctx->atoms_pulled_slot_pri[js][katom->sched_priority]--; + + if (!atomic_read(&kctx->atoms_pulled_slot[js]) && + jsctx_rb_none_to_pull(kctx, js)) + timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); + + /* If this slot has been blocked due to soft-stopped atoms, and all + * atoms have now been processed, then unblock the slot */ + if (!kctx->atoms_pulled_slot_pri[js][prio] && + kctx->blocked_js[js][prio]) { + kctx->blocked_js[js][prio] = false; + + /* Only mark the slot as pullable if the context is not idle - + * that case is handled below */ + if (atomic_read(&kctx->atoms_pulled) && + kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + + if (!atomic_read(&kctx->atoms_pulled)) { + if (!kctx->slots_pullable) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + timer_sync = true; + } + + if (kctx->as_nr != KBASEP_AS_NR_INVALID && + !kbase_ctx_flag(kctx, KCTX_DYING)) { + int num_slots = kbdev->gpu_props.num_job_slots; + int slot; + + if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) + kbasep_js_set_submit_allowed(js_devdata, kctx); + + for (slot = 0; slot < num_slots; slot++) { + if (kbase_js_ctx_pullable(kctx, slot, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, slot); + } + } + + kbase_jm_idle_ctx(kbdev, kctx); + + context_idle = true; + } + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (context_idle) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + kbase_pm_context_idle(kbdev); + } + + if (timer_sync) + kbase_js_sync_timers(kbdev); + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + + katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; + kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, + &retained_state); + + kbase_js_sched_all(kbdev); + + kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, + coreref_state); +} + +void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + jsctx_rb_unpull(kctx, katom); + + WARN_ON(work_pending(&katom->work)); + + /* Block re-submission until workqueue has run */ + atomic_inc(&katom->blocked); + + kbase_job_check_leave_disjoint(kctx->kbdev, katom); + + KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); + INIT_WORK(&katom->work, js_return_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + +bool kbase_js_complete_atom_wq(struct kbase_context *kctx, + struct kbase_jd_atom *katom) +{ + struct kbasep_js_kctx_info *js_kctx_info; + struct kbasep_js_device_data *js_devdata; + struct kbase_device *kbdev; + unsigned long flags; + bool timer_sync = false; + int atom_slot; + bool context_idle = false; + int prio = katom->sched_priority; + + kbdev = kctx->kbdev; + atom_slot = katom->slot_nr; + + js_kctx_info = &kctx->jctx.sched_info; + js_devdata = &kbdev->js_data; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { + context_idle = !atomic_dec_return(&kctx->atoms_pulled); + atomic_dec(&kctx->atoms_pulled_slot[atom_slot]); + kctx->atoms_pulled_slot_pri[atom_slot][prio]--; + + if (!atomic_read(&kctx->atoms_pulled) && + !kctx->slots_pullable) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); + kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); + atomic_dec(&kbdev->js_data.nr_contexts_runnable); + timer_sync = true; + } + + /* If this slot has been blocked due to soft-stopped atoms, and + * all atoms have now been processed, then unblock the slot */ + if (!kctx->atoms_pulled_slot_pri[atom_slot][prio] + && kctx->blocked_js[atom_slot][prio]) { + kctx->blocked_js[atom_slot][prio] = false; + if (kbase_js_ctx_pullable(kctx, atom_slot, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, atom_slot); + } + } + WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); + + if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) && + jsctx_rb_none_to_pull(kctx, atom_slot)) { + if (!list_empty( + &kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot])) + timer_sync |= kbase_js_ctx_list_remove_nolock( + kctx->kbdev, kctx, atom_slot); + } + + /* + * If submission is disabled on this context (most likely due to an + * atom failure) and there are now no atoms left in the system then + * re-enable submission so that context can be scheduled again. + */ + if (!kbasep_js_is_submit_allowed(js_devdata, kctx) && + !atomic_read(&kctx->atoms_pulled) && + !kbase_ctx_flag(kctx, KCTX_DYING)) { + int js; + + kbasep_js_set_submit_allowed(js_devdata, kctx); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + if (kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + } else if (katom->x_post_dep && + kbasep_js_is_submit_allowed(js_devdata, kctx)) { + int js; + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + if (kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } + } + + /* Mark context as inactive. The pm reference will be dropped later in + * jd_done_worker(). + */ + if (context_idle) + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + mutex_unlock(&js_devdata->runpool_mutex); + + return context_idle; +} + +struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, + ktime_t *end_timestamp) +{ + u64 microseconds_spent = 0; + struct kbase_device *kbdev; + struct kbase_context *kctx = katom->kctx; + struct kbase_jd_atom *x_dep = katom->x_post_dep; + + kbdev = kctx->kbdev; + + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + if (katom->will_fail_event_code) + katom->event_code = katom->will_fail_event_code; + + katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; + + if (katom->event_code != BASE_JD_EVENT_DONE) { + kbase_js_evict_deps(kctx, katom, katom->slot_nr, + katom->sched_priority); + } + +#if defined(CONFIG_MALI_GATOR_SUPPORT) + kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_STOP, + katom->slot_nr), NULL, 0); +#endif + + /* Calculate the job's time used */ + if (end_timestamp != NULL) { + /* Only calculating it for jobs that really run on the HW (e.g. + * removed from next jobs never actually ran, so really did take + * zero time) */ + ktime_t tick_diff = ktime_sub(*end_timestamp, + katom->start_timestamp); + + microseconds_spent = ktime_to_ns(tick_diff); + + do_div(microseconds_spent, 1000); + + /* Round up time spent to the minimum timer resolution */ + if (microseconds_spent < KBASEP_JS_TICK_RESOLUTION_US) + microseconds_spent = KBASEP_JS_TICK_RESOLUTION_US; + } + + + kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); + + /* Unblock cross dependency if present */ + if (x_dep && (katom->event_code == BASE_JD_EVENT_DONE || + !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) && + (x_dep->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { + bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, + false); + x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; + kbase_js_move_to_tree(x_dep); + if (!was_pullable && kbase_js_ctx_pullable(kctx, x_dep->slot_nr, + false)) + kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, + x_dep->slot_nr); + + if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) + return x_dep; + } + + return NULL; +} + +void kbase_js_sched(struct kbase_device *kbdev, int js_mask) +{ + struct kbasep_js_device_data *js_devdata; + struct kbase_context *last_active; + bool timer_sync = false; + bool ctx_waiting = false; + + js_devdata = &kbdev->js_data; + + down(&js_devdata->schedule_sem); + mutex_lock(&js_devdata->queue_mutex); + + last_active = kbdev->hwaccess.active_kctx; + + while (js_mask) { + int js; + + js = ffs(js_mask) - 1; + + while (1) { + struct kbase_context *kctx; + unsigned long flags; + bool context_idle = false; + + kctx = kbase_js_ctx_list_pop_head(kbdev, js); + + if (!kctx) { + js_mask &= ~(1 << js); + break; /* No contexts on pullable list */ + } + + if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) { + context_idle = true; + + if (kbase_pm_context_active_handle_suspend( + kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { + /* Suspend pending - return context to + * queue and stop scheduling */ + mutex_lock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + if (kbase_js_ctx_list_add_pullable_head( + kctx->kbdev, kctx, js)) + kbase_js_sync_timers(kbdev); + mutex_unlock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + up(&js_devdata->schedule_sem); + return; + } + kbase_ctx_flag_set(kctx, KCTX_ACTIVE); + } + + if (!kbase_js_use_ctx(kbdev, kctx)) { + mutex_lock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + /* Context can not be used at this time */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (kbase_js_ctx_pullable(kctx, js, false) + || kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) + timer_sync |= + kbase_js_ctx_list_add_pullable_head_nolock( + kctx->kbdev, kctx, js); + else + timer_sync |= + kbase_js_ctx_list_add_unpullable_nolock( + kctx->kbdev, kctx, js); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, + flags); + mutex_unlock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + if (context_idle) { + WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + kbase_pm_context_idle(kbdev); + } + + /* No more jobs can be submitted on this slot */ + js_mask &= ~(1 << js); + break; + } + mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_ctx_flag_clear(kctx, KCTX_PULLED); + + if (!kbase_jm_kick(kbdev, 1 << js)) + /* No more jobs can be submitted on this slot */ + js_mask &= ~(1 << js); + + if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { + bool pullable = kbase_js_ctx_pullable(kctx, js, + true); + + /* Failed to pull jobs - push to head of list. + * Unless this context is already 'active', in + * which case it's effectively already scheduled + * so push it to the back of the list. */ + if (pullable && kctx == last_active) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kctx->kbdev, + kctx, js); + else if (pullable) + timer_sync |= + kbase_js_ctx_list_add_pullable_head_nolock( + kctx->kbdev, + kctx, js); + else + timer_sync |= + kbase_js_ctx_list_add_unpullable_nolock( + kctx->kbdev, + kctx, js); + + /* If this context is not the active context, + * but the active context is pullable on this + * slot, then we need to remove the active + * marker to prevent it from submitting atoms in + * the IRQ handler, which would prevent this + * context from making progress. */ + if (last_active && kctx != last_active && + kbase_js_ctx_pullable( + last_active, js, true)) + ctx_waiting = true; + + if (context_idle) { + kbase_jm_idle_ctx(kbdev, kctx); + spin_unlock_irqrestore( + &kbdev->hwaccess_lock, + flags); + WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); + kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); + kbase_pm_context_idle(kbdev); + } else { + spin_unlock_irqrestore( + &kbdev->hwaccess_lock, + flags); + } + mutex_unlock( + &kctx->jctx.sched_info.ctx.jsctx_mutex); + + js_mask &= ~(1 << js); + break; /* Could not run atoms on this slot */ + } + + /* Push to back of list */ + if (kbase_js_ctx_pullable(kctx, js, true)) + timer_sync |= + kbase_js_ctx_list_add_pullable_nolock( + kctx->kbdev, kctx, js); + else + timer_sync |= + kbase_js_ctx_list_add_unpullable_nolock( + kctx->kbdev, kctx, js); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); + } + } + + if (timer_sync) + kbase_js_sync_timers(kbdev); + + if (kbdev->hwaccess.active_kctx == last_active && ctx_waiting) + kbdev->hwaccess.active_kctx = NULL; + + mutex_unlock(&js_devdata->queue_mutex); + up(&js_devdata->schedule_sem); +} + +void kbase_js_zap_context(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; + int js; + + /* + * Critical assumption: No more submission is possible outside of the + * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs) + * whilst the struct kbase_context is terminating. + */ + + /* First, atomically do the following: + * - mark the context as dying + * - try to evict it from the queue */ + mutex_lock(&kctx->jctx.lock); + mutex_lock(&js_devdata->queue_mutex); + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + kbase_ctx_flag_set(kctx, KCTX_DYING); + + dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx); + + /* + * At this point we know: + * - If eviction succeeded, it was in the queue, but now no + * longer is + * - We must cancel the jobs here. No Power Manager active reference to + * release. + * - This happens asynchronously - kbase_jd_zap_context() will wait for + * those jobs to be killed. + * - If eviction failed, then it wasn't in the queue. It is one + * of the following: + * - a. it didn't have any jobs, and so is not in the Queue or + * the Run Pool (not scheduled) + * - Hence, no more work required to cancel jobs. No Power Manager + * active reference to release. + * - b. it was in the middle of a scheduling transaction (and thus must + * have at least 1 job). This can happen from a syscall or a + * kernel thread. We still hold the jsctx_mutex, and so the thread + * must be waiting inside kbasep_js_try_schedule_head_ctx(), + * before checking whether the runpool is full. That thread will + * continue after we drop the mutex, and will notice the context + * is dying. It will rollback the transaction, killing all jobs at + * the same time. kbase_jd_zap_context() will wait for those jobs + * to be killed. + * - Hence, no more work required to cancel jobs, or to release the + * Power Manager active reference. + * - c. it is scheduled, and may or may not be running jobs + * - We must cause it to leave the runpool by stopping it from + * submitting any more jobs. When it finally does leave, + * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs + * (because it is dying), release the Power Manager active reference, + * and will not requeue the context in the queue. + * kbase_jd_zap_context() will wait for those jobs to be killed. + * - Hence, work required just to make it leave the runpool. Cancelling + * jobs and releasing the Power manager active reference will be + * handled when it leaves the runpool. + */ + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { + if (!list_empty( + &kctx->jctx.sched_info.ctx.ctx_list_entry[js])) + list_del_init( + &kctx->jctx.sched_info.ctx.ctx_list_entry[js]); + } + + /* The following events require us to kill off remaining jobs + * and update PM book-keeping: + * - we evicted it correctly (it must have jobs to be in the + * Queue) + * + * These events need no action, but take this path anyway: + * - Case a: it didn't have any jobs, and was never in the Queue + * - Case b: scheduling transaction will be partially rolled- + * back (this already cancels the jobs) + */ + + KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, + kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx); + + /* Only cancel jobs when we evicted from the + * queue. No Power Manager active reference was held. + * + * Having is_dying set ensures that this kills, and + * doesn't requeue */ + kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); + + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + mutex_unlock(&kctx->jctx.lock); + } else { + unsigned long flags; + bool was_retained; + + /* Case c: didn't evict, but it is scheduled - it's in the Run + * Pool */ + KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, + kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx); + + /* Disable the ctx from submitting any more jobs */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbasep_js_clear_submit_allowed(js_devdata, kctx); + + /* Retain and (later) release the context whilst it is is now + * disallowed from submitting jobs - ensures that someone + * somewhere will be removing the context later on */ + was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); + + /* Since it's scheduled and we have the jsctx_mutex, it must be + * retained successfully */ + KBASE_DEBUG_ASSERT(was_retained); + + dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx); + + /* Cancel any remaining running jobs for this kctx - if any. + * Submit is disallowed which takes effect immediately, so no + * more new jobs will appear after we do this. */ + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) + kbase_job_slot_hardstop(kctx, js, NULL); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + mutex_unlock(&js_devdata->queue_mutex); + mutex_unlock(&kctx->jctx.lock); + + dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", + kctx); + + kbasep_js_runpool_release_ctx(kbdev, kctx); + } + + KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); + + /* After this, you must wait on both the + * kbase_jd_context::zero_jobs_wait and the + * kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs + * to be destroyed, and the context to be de-scheduled (if it was on the + * runpool). + * + * kbase_jd_zap_context() will do this. */ +} + +static inline int trace_get_refcnt(struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + return atomic_read(&kctx->refcount); +} + +/** + * kbase_js_foreach_ctx_job(): - Call a function on all jobs in context + * @kctx: Pointer to context. + * @callback: Pointer to function to call for each job. + * + * Call a function on all jobs belonging to a non-queued, non-running + * context, and detach the jobs from the context as it goes. + * + * Due to the locks that might be held at the time of the call, the callback + * may need to defer work on a workqueue to complete its actions (e.g. when + * cancelling jobs) + * + * Atoms will be removed from the queue, so this must only be called when + * cancelling jobs (which occurs as part of context destruction). + * + * The locking conditions on the caller are as follows: + * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex. + */ +static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, + kbasep_js_ctx_job_cb callback) +{ + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + unsigned long flags; + u32 js; + + kbdev = kctx->kbdev; + js_devdata = &kbdev->js_data; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, + 0u, trace_get_refcnt(kbdev, kctx)); + + /* Invoke callback on jobs on each slot in turn */ + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) + jsctx_queue_foreach(kctx, js, callback); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_js.h b/drivers/gpu/arm/midgard/mali_kbase_js.h new file mode 100755 index 000000000000..ddada8e468a1 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_js.h @@ -0,0 +1,925 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_js.h + * Job Scheduler APIs. + */ + +#ifndef _KBASE_JS_H_ +#define _KBASE_JS_H_ + +#include "mali_kbase_js_defs.h" +#include "mali_kbase_context.h" +#include "mali_kbase_defs.h" +#include "mali_kbase_debug.h" + +#include "mali_kbase_js_ctx_attr.h" + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_js Job Scheduler Internal APIs + * @{ + * + * These APIs are Internal to KBase. + */ + +/** + * @brief Initialize the Job Scheduler + * + * The struct kbasep_js_device_data sub-structure of \a kbdev must be zero + * initialized before passing to the kbasep_js_devdata_init() function. This is + * to give efficient error path code. + */ +int kbasep_js_devdata_init(struct kbase_device * const kbdev); + +/** + * @brief Halt the Job Scheduler. + * + * It is safe to call this on \a kbdev even if it the kbasep_js_device_data + * sub-structure was never initialized/failed initialization, to give efficient + * error-path code. + * + * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must + * be zero initialized before passing to the kbasep_js_devdata_init() + * function. This is to give efficient error path code. + * + * It is a Programming Error to call this whilst there are still kbase_context + * structures registered with this scheduler. + * + */ +void kbasep_js_devdata_halt(struct kbase_device *kbdev); + +/** + * @brief Terminate the Job Scheduler + * + * It is safe to call this on \a kbdev even if it the kbasep_js_device_data + * sub-structure was never initialized/failed initialization, to give efficient + * error-path code. + * + * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must + * be zero initialized before passing to the kbasep_js_devdata_init() + * function. This is to give efficient error path code. + * + * It is a Programming Error to call this whilst there are still kbase_context + * structures registered with this scheduler. + */ +void kbasep_js_devdata_term(struct kbase_device *kbdev); + +/** + * @brief Initialize the Scheduling Component of a struct kbase_context on the Job Scheduler. + * + * This effectively registers a struct kbase_context with a Job Scheduler. + * + * It does not register any jobs owned by the struct kbase_context with the scheduler. + * Those must be separately registered by kbasep_js_add_job(). + * + * The struct kbase_context must be zero intitialized before passing to the + * kbase_js_init() function. This is to give efficient error path code. + */ +int kbasep_js_kctx_init(struct kbase_context * const kctx); + +/** + * @brief Terminate the Scheduling Component of a struct kbase_context on the Job Scheduler + * + * This effectively de-registers a struct kbase_context from its Job Scheduler + * + * It is safe to call this on a struct kbase_context that has never had or failed + * initialization of its jctx.sched_info member, to give efficient error-path + * code. + * + * For this to work, the struct kbase_context must be zero intitialized before passing + * to the kbase_js_init() function. + * + * It is a Programming Error to call this whilst there are still jobs + * registered with this context. + */ +void kbasep_js_kctx_term(struct kbase_context *kctx); + +/** + * @brief Add a job chain to the Job Scheduler, and take necessary actions to + * schedule the context/run the job. + * + * This atomically does the following: + * - Update the numbers of jobs information + * - Add the job to the run pool if necessary (part of init_job) + * + * Once this is done, then an appropriate action is taken: + * - If the ctx is scheduled, it attempts to start the next job (which might be + * this added job) + * - Otherwise, and if this is the first job on the context, it enqueues it on + * the Policy Queue + * + * The Policy's Queue can be updated by this in the following ways: + * - In the above case that this is the first job on the context + * - If the context is high priority and the context is not scheduled, then it + * could cause the Policy to schedule out a low-priority context, allowing + * this context to be scheduled in. + * + * If the context is already scheduled on the RunPool, then adding a job to it + * is guarenteed not to update the Policy Queue. And so, the caller is + * guarenteed to not need to try scheduling a context from the Run Pool - it + * can safely assert that the result is false. + * + * It is a programming error to have more than U32_MAX jobs in flight at a time. + * + * The following locking conditions are made on the caller: + * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * - it must \em not hold hwaccess_lock (as this will be obtained internally) + * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally). + * + * @return true indicates that the Policy Queue was updated, and so the + * caller will need to try scheduling a context onto the Run Pool. + * @return false indicates that no updates were made to the Policy Queue, + * so no further action is required from the caller. This is \b always returned + * when the context is currently scheduled. + */ +bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom); + +/** + * @brief Remove a job chain from the Job Scheduler, except for its 'retained state'. + * + * Completely removing a job requires several calls: + * - kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of + * the atom + * - kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler + * - kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the + * remaining state held as part of the job having been run. + * + * In the common case of atoms completing normally, this set of actions is more optimal for spinlock purposes than having kbasep_js_remove_job() handle all of the actions. + * + * In the case of cancelling atoms, it is easier to call kbasep_js_remove_cancelled_job(), which handles all the necessary actions. + * + * It is a programming error to call this when: + * - \a atom is not a job belonging to kctx. + * - \a atom has already been removed from the Job Scheduler. + * - \a atom is still in the runpool + * + * Do not use this for removing jobs being killed by kbase_jd_cancel() - use + * kbasep_js_remove_cancelled_job() instead. + * + * The following locking conditions are made on the caller: + * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * + */ +void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *atom); + +/** + * @brief Completely remove a job chain from the Job Scheduler, in the case + * where the job chain was cancelled. + * + * This is a variant of kbasep_js_remove_job() that takes care of removing all + * of the retained state too. This is generally useful for cancelled atoms, + * which need not be handled in an optimal way. + * + * It is a programming error to call this when: + * - \a atom is not a job belonging to kctx. + * - \a atom has already been removed from the Job Scheduler. + * - \a atom is still in the runpool: + * - it is not being killed with kbasep_jd_cancel() + * + * The following locking conditions are made on the caller: + * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * - it must \em not hold the hwaccess_lock, (as this will be obtained + * internally) + * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this could be + * obtained internally) + * + * @return true indicates that ctx attributes have changed and the caller + * should call kbase_js_sched_all() to try to run more jobs + * @return false otherwise + */ +bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, + struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +/** + * @brief Refcount a context as being busy, preventing it from being scheduled + * out. + * + * @note This function can safely be called from IRQ context. + * + * The following locking conditions are made on the caller: + * - it must \em not hold mmu_hw_mutex and hwaccess_lock, because they will be + * used internally. + * + * @return value != false if the retain succeeded, and the context will not be scheduled out. + * @return false if the retain failed (because the context is being/has been scheduled out). + */ +bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * @brief Refcount a context as being busy, preventing it from being scheduled + * out. + * + * @note This function can safely be called from IRQ context. + * + * The following locks must be held by the caller: + * - mmu_hw_mutex, hwaccess_lock + * + * @return value != false if the retain succeeded, and the context will not be scheduled out. + * @return false if the retain failed (because the context is being/has been scheduled out). + */ +bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * @brief Lookup a context in the Run Pool based upon its current address space + * and ensure that is stays scheduled in. + * + * The context is refcounted as being busy to prevent it from scheduling + * out. It must be released with kbasep_js_runpool_release_ctx() when it is no + * longer required to stay scheduled in. + * + * @note This function can safely be called from IRQ context. + * + * The following locking conditions are made on the caller: + * - it must \em not hold the hwaccess_lock, because it will be used internally. + * If the hwaccess_lock is already held, then the caller should use + * kbasep_js_runpool_lookup_ctx_nolock() instead. + * + * @return a valid struct kbase_context on success, which has been refcounted as being busy. + * @return NULL on failure, indicating that no context was found in \a as_nr + */ +struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, int as_nr); + +/** + * @brief Handling the requeuing/killing of a context that was evicted from the + * policy queue or runpool. + * + * This should be used whenever handing off a context that has been evicted + * from the policy queue or the runpool: + * - If the context is not dying and has jobs, it gets re-added to the policy + * queue + * - Otherwise, it is not added + * + * In addition, if the context is dying the jobs are killed asynchronously. + * + * In all cases, the Power Manager active reference is released + * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. \a + * has_pm_ref must be set to false whenever the context was not previously in + * the runpool and does not hold a Power Manager active refcount. Note that + * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an + * active refcount even though they weren't in the runpool. + * + * The following locking conditions are made on the caller: + * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be + * obtained internally) + */ +void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, bool has_pm_ref); + +/** + * @brief Release a refcount of a context being busy, allowing it to be + * scheduled out. + * + * When the refcount reaches zero and the context \em might be scheduled out + * (depending on whether the Scheudling Policy has deemed it so, or if it has run + * out of jobs). + * + * If the context does get scheduled out, then The following actions will be + * taken as part of deschduling a context: + * - For the context being descheduled: + * - If the context is in the processing of dying (all the jobs are being + * removed from it), then descheduling also kills off any jobs remaining in the + * context. + * - If the context is not dying, and any jobs remain after descheduling the + * context then it is re-enqueued to the Policy's Queue. + * - Otherwise, the context is still known to the scheduler, but remains absent + * from the Policy Queue until a job is next added to it. + * - In all descheduling cases, the Power Manager active reference (obtained + * during kbasep_js_try_schedule_head_ctx()) is released (kbase_pm_context_idle()). + * + * Whilst the context is being descheduled, this also handles actions that + * cause more atoms to be run: + * - Attempt submitting atoms when the Context Attributes on the Runpool have + * changed. This is because the context being scheduled out could mean that + * there are more opportunities to run atoms. + * - Attempt submitting to a slot that was previously blocked due to affinity + * restrictions. This is usually only necessary when releasing a context + * happens as part of completing a previous job, but is harmless nonetheless. + * - Attempt scheduling in a new context (if one is available), and if necessary, + * running a job from that new context. + * + * Unlike retaining a context in the runpool, this function \b cannot be called + * from IRQ context. + * + * It is a programming error to call this on a \a kctx that is not currently + * scheduled, or that already has a zero refcount. + * + * The following locking conditions are made on the caller: + * - it must \em not hold the hwaccess_lock, because it will be used internally. + * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be + * obtained internally) + * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be + * obtained internally) + * + */ +void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * @brief Variant of kbasep_js_runpool_release_ctx() that handles additional + * actions from completing an atom. + * + * This is usually called as part of completing an atom and releasing the + * refcount on the context held by the atom. + * + * Therefore, the extra actions carried out are part of handling actions queued + * on a completed atom, namely: + * - Releasing the atom's context attributes + * - Retrying the submission on a particular slot, because we couldn't submit + * on that slot from an IRQ handler. + * + * The locking conditions of this function are the same as those for + * kbasep_js_runpool_release_ctx() + */ +void kbasep_js_runpool_release_ctx_and_katom_retained_state(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); + +/** + * @brief Variant of kbase_js_runpool_release_ctx() that assumes that + * kbasep_js_device_data::runpool_mutex and + * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not + * attempt to schedule new contexts. + */ +void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, + struct kbase_context *kctx); + +/** + * @brief Schedule in a privileged context + * + * This schedules a context in regardless of the context priority. + * If the runpool is full, a context will be forced out of the runpool and the function will wait + * for the new context to be scheduled in. + * The context will be kept scheduled in (and the corresponding address space reserved) until + * kbasep_js_release_privileged_ctx is called). + * + * The following locking conditions are made on the caller: + * - it must \em not hold the hwaccess_lock, because it will be used internally. + * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be + * obtained internally) + * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally). + * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will + * be used internally. + * + */ +void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * @brief Release a privileged context, allowing it to be scheduled out. + * + * See kbasep_js_runpool_release_ctx for potential side effects. + * + * The following locking conditions are made on the caller: + * - it must \em not hold the hwaccess_lock, because it will be used internally. + * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. + * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be + * obtained internally) + * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be + * obtained internally) + * + */ +void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * @brief Try to submit the next job on each slot + * + * The following locks may be used: + * - kbasep_js_device_data::runpool_mutex + * - hwaccess_lock + */ +void kbase_js_try_run_jobs(struct kbase_device *kbdev); + +/** + * @brief Suspend the job scheduler during a Power Management Suspend event. + * + * Causes all contexts to be removed from the runpool, and prevents any + * contexts from (re)entering the runpool. + * + * This does not handle suspending the one privileged context: the caller must + * instead do this by by suspending the GPU HW Counter Instrumentation. + * + * This will eventually cause all Power Management active references held by + * contexts on the runpool to be released, without running any more atoms. + * + * The caller must then wait for all Power Mangement active refcount to become + * zero before completing the suspend. + * + * The emptying mechanism may take some time to complete, since it can wait for + * jobs to complete naturally instead of forcing them to end quickly. However, + * this is bounded by the Job Scheduler's Job Timeouts. Hence, this + * function is guaranteed to complete in a finite time. + */ +void kbasep_js_suspend(struct kbase_device *kbdev); + +/** + * @brief Resume the Job Scheduler after a Power Management Resume event. + * + * This restores the actions from kbasep_js_suspend(): + * - Schedules contexts back into the runpool + * - Resumes running atoms on the GPU + */ +void kbasep_js_resume(struct kbase_device *kbdev); + +/** + * @brief Submit an atom to the job scheduler. + * + * The atom is enqueued on the context's ringbuffer. The caller must have + * ensured that all dependencies can be represented in the ringbuffer. + * + * Caller must hold jctx->lock + * + * @param[in] kctx Context pointer + * @param[in] atom Pointer to the atom to submit + * + * @return Whether the context requires to be enqueued. */ +bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +/** + * jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer. + * @kctx: Context Pointer + * @prio: Priority (specifies the queue together with js). + * @js: Job slot (specifies the queue together with prio). + * + * Pushes all possible atoms from the linked list to the ringbuffer. + * Number of atoms are limited to free space in the ringbuffer and + * number of available atoms in the linked list. + * + */ +void jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js); +/** + * @brief Pull an atom from a context in the job scheduler for execution. + * + * The atom will not be removed from the ringbuffer at this stage. + * + * The HW access lock must be held when calling this function. + * + * @param[in] kctx Context to pull from + * @param[in] js Job slot to pull from + * @return Pointer to an atom, or NULL if there are no atoms for this + * slot that can be currently run. + */ +struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js); + +/** + * @brief Return an atom to the job scheduler ringbuffer. + * + * An atom is 'unpulled' if execution is stopped but intended to be returned to + * later. The most common reason for this is that the atom has been + * soft-stopped. + * + * Note that if multiple atoms are to be 'unpulled', they must be returned in + * the reverse order to which they were originally pulled. It is a programming + * error to return atoms in any other order. + * + * The HW access lock must be held when calling this function. + * + * @param[in] kctx Context pointer + * @param[in] atom Pointer to the atom to unpull + */ +void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom); + +/** + * @brief Complete an atom from jd_done_worker(), removing it from the job + * scheduler ringbuffer. + * + * If the atom failed then all dependee atoms marked for failure propagation + * will also fail. + * + * @param[in] kctx Context pointer + * @param[in] katom Pointer to the atom to complete + * @return true if the context is now idle (no jobs pulled) + * false otherwise + */ +bool kbase_js_complete_atom_wq(struct kbase_context *kctx, + struct kbase_jd_atom *katom); + +/** + * @brief Complete an atom. + * + * Most of the work required to complete an atom will be performed by + * jd_done_worker(). + * + * The HW access lock must be held when calling this function. + * + * @param[in] katom Pointer to the atom to complete + * @param[in] end_timestamp The time that the atom completed (may be NULL) + * + * Return: Atom that has now been unblocked and can now be run, or NULL if none + */ +struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, + ktime_t *end_timestamp); + +/** + * @brief Submit atoms from all available contexts. + * + * This will attempt to submit as many jobs as possible to the provided job + * slots. It will exit when either all job slots are full, or all contexts have + * been used. + * + * @param[in] kbdev Device pointer + * @param[in] js_mask Mask of job slots to submit to + */ +void kbase_js_sched(struct kbase_device *kbdev, int js_mask); + +/** + * kbase_jd_zap_context - Attempt to deschedule a context that is being + * destroyed + * @kctx: Context pointer + * + * This will attempt to remove a context from any internal job scheduler queues + * and perform any other actions to ensure a context will not be submitted + * from. + * + * If the context is currently scheduled, then the caller must wait for all + * pending jobs to complete before taking any further action. + */ +void kbase_js_zap_context(struct kbase_context *kctx); + +/** + * @brief Validate an atom + * + * This will determine whether the atom can be scheduled onto the GPU. Atoms + * with invalid combinations of core requirements will be rejected. + * + * @param[in] kbdev Device pointer + * @param[in] katom Atom to validate + * @return true if atom is valid + * false otherwise + */ +bool kbase_js_is_atom_valid(struct kbase_device *kbdev, + struct kbase_jd_atom *katom); + +/** + * kbase_js_set_timeouts - update all JS timeouts with user specified data + * @kbdev: Device pointer + * + * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is + * set to a positive number then that becomes the new value used, if a timeout + * is negative then the default is set. + */ +void kbase_js_set_timeouts(struct kbase_device *kbdev); + +/* + * Helpers follow + */ + +/** + * @brief Check that a context is allowed to submit jobs on this policy + * + * The purpose of this abstraction is to hide the underlying data size, and wrap up + * the long repeated line of code. + * + * As with any bool, never test the return value with true. + * + * The caller must hold hwaccess_lock. + */ +static inline bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) +{ + u16 test_bit; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + test_bit = (u16) (1u << kctx->as_nr); + + return (bool) (js_devdata->runpool_irq.submit_allowed & test_bit); +} + +/** + * @brief Allow a context to submit jobs on this policy + * + * The purpose of this abstraction is to hide the underlying data size, and wrap up + * the long repeated line of code. + * + * The caller must hold hwaccess_lock. + */ +static inline void kbasep_js_set_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) +{ + u16 set_bit; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + set_bit = (u16) (1u << kctx->as_nr); + + dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)", kctx, kctx->as_nr); + + js_devdata->runpool_irq.submit_allowed |= set_bit; +} + +/** + * @brief Prevent a context from submitting more jobs on this policy + * + * The purpose of this abstraction is to hide the underlying data size, and wrap up + * the long repeated line of code. + * + * The caller must hold hwaccess_lock. + */ +static inline void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) +{ + u16 clear_bit; + u16 clear_mask; + + /* Ensure context really is scheduled in */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + clear_bit = (u16) (1u << kctx->as_nr); + clear_mask = ~clear_bit; + + dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)", kctx, kctx->as_nr); + + js_devdata->runpool_irq.submit_allowed &= clear_mask; +} + +/** + * @brief Manage the 'retry_submit_on_slot' part of a kbase_jd_atom + */ +static inline void kbasep_js_clear_job_retry_submit(struct kbase_jd_atom *atom) +{ + atom->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID; +} + +/** + * Mark a slot as requiring resubmission by carrying that information on a + * completing atom. + * + * @note This can ASSERT in debug builds if the submit slot has been set to + * something other than the current value for @a js. This is because you might + * be unintentionally stopping more jobs being submitted on the old submit + * slot, and that might cause a scheduling-hang. + * + * @note If you can guarantee that the atoms for the original slot will be + * submitted on some other slot, then call kbasep_js_clear_job_retry_submit() + * first to silence the ASSERT. + */ +static inline void kbasep_js_set_job_retry_submit_slot(struct kbase_jd_atom *atom, int js) +{ + KBASE_DEBUG_ASSERT(0 <= js && js <= BASE_JM_MAX_NR_SLOTS); + KBASE_DEBUG_ASSERT((atom->retry_submit_on_slot == + KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID) + || (atom->retry_submit_on_slot == js)); + + atom->retry_submit_on_slot = js; +} + +/** + * Create an initial 'invalid' atom retained state, that requires no + * atom-related work to be done on releasing with + * kbasep_js_runpool_release_ctx_and_katom_retained_state() + */ +static inline void kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state *retained_state) +{ + retained_state->event_code = BASE_JD_EVENT_NOT_STARTED; + retained_state->core_req = KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID; + retained_state->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID; +} + +/** + * Copy atom state that can be made available after jd_done_nolock() is called + * on that atom. + */ +static inline void kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state *retained_state, const struct kbase_jd_atom *katom) +{ + retained_state->event_code = katom->event_code; + retained_state->core_req = katom->core_req; + retained_state->retry_submit_on_slot = katom->retry_submit_on_slot; + retained_state->sched_priority = katom->sched_priority; + retained_state->device_nr = katom->device_nr; +} + +/** + * @brief Determine whether an atom has finished (given its retained state), + * and so should be given back to userspace/removed from the system. + * + * Reasons for an atom not finishing include: + * - Being soft-stopped (and so, the atom should be resubmitted sometime later) + * + * @param[in] katom_retained_state the retained state of the atom to check + * @return false if the atom has not finished + * @return !=false if the atom has finished + */ +static inline bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state) +{ + return (bool) (katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT); +} + +/** + * @brief Determine whether a struct kbasep_js_atom_retained_state is valid + * + * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates that the + * code should just ignore it. + * + * @param[in] katom_retained_state the atom's retained state to check + * @return false if the retained state is invalid, and can be ignored + * @return !=false if the retained state is valid + */ +static inline bool kbasep_js_atom_retained_state_is_valid(const struct kbasep_js_atom_retained_state *katom_retained_state) +{ + return (bool) (katom_retained_state->core_req != KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID); +} + +static inline bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_atom_retained_state *katom_retained_state, int *res) +{ + int js = katom_retained_state->retry_submit_on_slot; + + *res = js; + return (bool) (js >= 0); +} + +/** + * @brief Variant of kbasep_js_runpool_lookup_ctx() that can be used when the + * context is guaranteed to be already previously retained. + * + * It is a programming error to supply the \a as_nr of a context that has not + * been previously retained/has a busy refcount of zero. The only exception is + * when there is no ctx in \a as_nr (NULL returned). + * + * The following locking conditions are made on the caller: + * - it must \em not hold the hwaccess_lock, because it will be used internally. + * + * @return a valid struct kbase_context on success, with a refcount that is guaranteed + * to be non-zero and unmodified by this function. + * @return NULL on failure, indicating that no context was found in \a as_nr + */ +static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device *kbdev, int as_nr) +{ + struct kbase_context *found_kctx; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); + + found_kctx = kbdev->as_to_kctx[as_nr]; + KBASE_DEBUG_ASSERT(found_kctx == NULL || + atomic_read(&found_kctx->refcount) > 0); + + return found_kctx; +} + +/* + * The following locking conditions are made on the caller: + * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. + * - The caller must hold the kbasep_js_device_data::runpool_mutex + */ +static inline void kbase_js_runpool_inc_context_count( + struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + + /* Track total contexts */ + KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX); + ++(js_devdata->nr_all_contexts_running); + + if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + /* Track contexts that can submit jobs */ + KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running < + S8_MAX); + ++(js_devdata->nr_user_contexts_running); + } +} + +/* + * The following locking conditions are made on the caller: + * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. + * - The caller must hold the kbasep_js_device_data::runpool_mutex + */ +static inline void kbase_js_runpool_dec_context_count( + struct kbase_device *kbdev, + struct kbase_context *kctx) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&js_devdata->runpool_mutex); + + /* Track total contexts */ + --(js_devdata->nr_all_contexts_running); + KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0); + + if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + /* Track contexts that can submit jobs */ + --(js_devdata->nr_user_contexts_running); + KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0); + } +} + + +/** + * @brief Submit atoms from all available contexts to all job slots. + * + * This will attempt to submit as many jobs as possible. It will exit when + * either all job slots are full, or all contexts have been used. + * + * @param[in] kbdev Device pointer + */ +static inline void kbase_js_sched_all(struct kbase_device *kbdev) +{ + kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); +} + +extern const int +kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS]; + +extern const base_jd_prio +kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; + +/** + * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio) + * to relative ordering + * @atom_prio: Priority ID to translate. + * + * Atom priority values for @ref base_jd_prio cannot be compared directly to + * find out which are higher or lower. + * + * This function will convert base_jd_prio values for successively lower + * priorities into a monotonically increasing sequence. That is, the lower the + * base_jd_prio priority, the higher the value produced by this function. This + * is in accordance with how the rest of the kernel treates priority. + * + * The mapping is 1:1 and the size of the valid input range is the same as the + * size of the valid output range, i.e. + * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS + * + * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions + * + * Return: On success: a value in the inclusive range + * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure: + * KBASE_JS_ATOM_SCHED_PRIO_INVALID + */ +static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio) +{ + if (atom_prio >= BASE_JD_NR_PRIO_LEVELS) + return KBASE_JS_ATOM_SCHED_PRIO_INVALID; + + return kbasep_js_atom_priority_to_relative[atom_prio]; +} + +static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio) +{ + unsigned int prio_idx; + + KBASE_DEBUG_ASSERT(0 <= sched_prio + && sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT); + + prio_idx = (unsigned int)sched_prio; + + return kbasep_js_relative_priority_to_atom[prio_idx]; +} + + /** @} *//* end group kbase_js */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_JS_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c new file mode 100755 index 000000000000..321506ada835 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c @@ -0,0 +1,301 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +#include +#include + +/* + * Private functions follow + */ + +/** + * @brief Check whether a ctx has a certain attribute, and if so, retain that + * attribute on the runpool. + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx is scheduled on the runpool + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { + KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX); + ++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); + + if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) { + /* First refcount indicates a state change */ + runpool_state_changed = true; + KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute); + } + } + + return runpool_state_changed; +} + +/** + * @brief Check whether a ctx has a certain attribute, and if so, release that + * attribute on the runpool. + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx is scheduled on the runpool + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_device_data *js_devdata; + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_devdata = &kbdev->js_data; + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); + + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { + KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0); + --(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); + + if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) { + /* Last de-refcount indicates a state change */ + runpool_state_changed = true; + KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute); + } + } + + return runpool_state_changed; +} + +/** + * @brief Retain a certain attribute on a ctx, also retaining it on the runpool + * if the context is scheduled. + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + * + * @return true indicates a change in ctx attributes state of the runpool. + * This may allow the scheduler to submit more jobs than previously. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX); + + ++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); + + if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { + /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */ + KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); + runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute); + } + + return runpool_state_changed; +} + +/* + * @brief Release a certain attribute on a ctx, also releasing it from the runpool + * if the context is scheduled. + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + * + * @return true indicates a change in ctx attributes state of the runpool. + * This may allow the scheduler to submit more jobs than previously. + * @return false indicates no change in ctx attributes state of the runpool. + */ +static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_kctx_info *js_kctx_info; + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_kctx_info = &kctx->jctx.sched_info; + + lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); + KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0); + + if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { + lockdep_assert_held(&kbdev->hwaccess_lock); + /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */ + runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute); + KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); + } + + /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */ + --(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); + + return runpool_state_changed; +} + +/* + * More commonly used public functions + */ + +void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + bool runpool_state_changed = false; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(kctx != NULL); + + if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { + /* This context never submits, so don't track any scheduling attributes */ + return; + } + + /* Transfer attributes held in the context flags for contexts that have submit enabled */ + + /* ... More attributes can be added here ... */ + + /* The context should not have been scheduled yet, so ASSERT if this caused + * runpool state changes (note that other threads *can't* affect the value + * of runpool_state_changed, due to how it's calculated) */ + KBASE_DEBUG_ASSERT(runpool_state_changed == false); + CSTD_UNUSED(runpool_state_changed); +} + +void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + bool runpool_state_changed; + int i; + + /* Retain any existing attributes */ + for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { + /* The context is being scheduled in, so update the runpool with the new attributes */ + runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); + + /* We don't need to know about state changed, because retaining a + * context occurs on scheduling it, and that itself will also try + * to run new atoms */ + CSTD_UNUSED(runpool_state_changed); + } + } +} + +bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) +{ + bool runpool_state_changed = false; + int i; + + /* Release any existing attributes */ + for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { + if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { + /* The context is being scheduled out, so update the runpool on the removed attributes */ + runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); + } + } + + return runpool_state_changed; +} + +void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + bool runpool_state_changed = false; + base_jd_core_req core_req; + + KBASE_DEBUG_ASSERT(katom); + core_req = katom->core_req; + + if (core_req & BASE_JD_REQ_ONLY_COMPUTE) + runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); + else + runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); + + if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { + /* Atom that can run on slot1 or slot2, and can use all cores */ + runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); + } + + /* We don't need to know about state changed, because retaining an + * atom occurs on adding it, and that itself will also try to run + * new atoms */ + CSTD_UNUSED(runpool_state_changed); +} + +bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state) +{ + bool runpool_state_changed = false; + base_jd_core_req core_req; + + KBASE_DEBUG_ASSERT(katom_retained_state); + core_req = katom_retained_state->core_req; + + /* No-op for invalid atoms */ + if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false) + return false; + + if (core_req & BASE_JD_REQ_ONLY_COMPUTE) + runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); + else + runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); + + if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { + /* Atom that can run on slot1 or slot2, and can use all cores */ + runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); + } + + return runpool_state_changed; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h new file mode 100755 index 000000000000..ce9183326a57 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h @@ -0,0 +1,158 @@ +/* + * + * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_js_ctx_attr.h + * Job Scheduler Context Attribute APIs + */ + +#ifndef _KBASE_JS_CTX_ATTR_H_ +#define _KBASE_JS_CTX_ATTR_H_ + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_js + * @{ + */ + +/** + * Set the initial attributes of a context (when context create flags are set) + * + * Requires: + * - Hold the jsctx_mutex + */ +void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * Retain all attributes of a context + * + * This occurs on scheduling in the context on the runpool (but after + * is_scheduled is set) + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx->is_scheduled is true + */ +void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * Release all attributes of a context + * + * This occurs on scheduling out the context from the runpool (but before + * is_scheduled is cleared) + * + * Requires: + * - jsctx mutex + * - runpool_irq spinlock + * - ctx->is_scheduled is true + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); + +/** + * Retain all attributes of an atom + * + * This occurs on adding an atom to a context + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + */ +void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); + +/** + * Release all attributes of an atom, given its retained state. + * + * This occurs after (permanently) removing an atom from a context + * + * Requires: + * - jsctx mutex + * - If the context is scheduled, then runpool_irq spinlock must also be held + * + * This is a no-op when \a katom_retained_state is invalid. + * + * @return true indicates a change in ctx attributes state of the runpool. + * In this state, the scheduler might be able to submit more jobs than + * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() + * or similar is called sometime later. + * @return false indicates no change in ctx attributes state of the runpool. + */ +bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); + +/** + * Requires: + * - runpool_irq spinlock + */ +static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_device_data *js_devdata; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_devdata = &kbdev->js_data; + + return js_devdata->runpool_irq.ctx_attr_ref_count[attribute]; +} + +/** + * Requires: + * - runpool_irq spinlock + */ +static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) +{ + /* In general, attributes are 'on' when they have a non-zero refcount (note: the refcount will never be < 0) */ + return (bool) kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute); +} + +/** + * Requires: + * - jsctx mutex + */ +static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) +{ + struct kbasep_js_kctx_info *js_kctx_info; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); + js_kctx_info = &kctx->jctx.sched_info; + + /* In general, attributes are 'on' when they have a refcount (which should never be < 0) */ + return (bool) (js_kctx_info->ctx.ctx_attr_ref_count[attribute]); +} + + /** @} *//* end group kbase_js */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_JS_DEFS_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_js_defs.h b/drivers/gpu/arm/midgard/mali_kbase_js_defs.h new file mode 100755 index 000000000000..ba8b6441549b --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_js_defs.h @@ -0,0 +1,386 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_js.h + * Job Scheduler Type Definitions + */ + +#ifndef _KBASE_JS_DEFS_H_ +#define _KBASE_JS_DEFS_H_ + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup kbase_js + * @{ + */ +/* Forward decls */ +struct kbase_device; +struct kbase_jd_atom; + + +typedef u32 kbase_context_flags; + +struct kbasep_atom_req { + base_jd_core_req core_req; + kbase_context_flags ctx_req; + u32 device_nr; +}; + +/** Callback function run on all of a context's jobs registered with the Job + * Scheduler */ +typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd_atom *katom); + +/** + * @brief Maximum number of jobs that can be submitted to a job slot whilst + * inside the IRQ handler. + * + * This is important because GPU NULL jobs can complete whilst the IRQ handler + * is running. Otherwise, it potentially allows an unlimited number of GPU NULL + * jobs to be submitted inside the IRQ handler, which increases IRQ latency. + */ +#define KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ 2 + +/** + * @brief Context attributes + * + * Each context attribute can be thought of as a boolean value that caches some + * state information about either the runpool, or the context: + * - In the case of the runpool, it is a cache of "Do any contexts owned by + * the runpool have attribute X?" + * - In the case of a context, it is a cache of "Do any atoms owned by the + * context have attribute X?" + * + * The boolean value of the context attributes often affect scheduling + * decisions, such as affinities to use and job slots to use. + * + * To accomodate changes of state in the context, each attribute is refcounted + * in the context, and in the runpool for all running contexts. Specifically: + * - The runpool holds a refcount of how many contexts in the runpool have this + * attribute. + * - The context holds a refcount of how many atoms have this attribute. + */ +enum kbasep_js_ctx_attr { + /** Attribute indicating a context that contains Compute jobs. That is, + * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE + * + * @note A context can be both 'Compute' and 'Non Compute' if it contains + * both types of jobs. + */ + KBASEP_JS_CTX_ATTR_COMPUTE, + + /** Attribute indicating a context that contains Non-Compute jobs. That is, + * the context has some jobs that are \b not of type @ref + * BASE_JD_REQ_ONLY_COMPUTE. + * + * @note A context can be both 'Compute' and 'Non Compute' if it contains + * both types of jobs. + */ + KBASEP_JS_CTX_ATTR_NON_COMPUTE, + + /** Attribute indicating that a context contains compute-job atoms that + * aren't restricted to a coherent group, and can run on all cores. + * + * Specifically, this is when the atom's \a core_req satisfy: + * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 + * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups + * + * Such atoms could be blocked from running if one of the coherent groups + * is being used by another job slot, so tracking this context attribute + * allows us to prevent such situations. + * + * @note This doesn't take into account the 1-coregroup case, where all + * compute atoms would effectively be able to run on 'all cores', but + * contexts will still not always get marked with this attribute. Instead, + * it is the caller's responsibility to take into account the number of + * coregroups when interpreting this attribute. + * + * @note Whilst Tiler atoms are normally combined with + * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without + * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy + * enough to handle anyway. + */ + KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, + + /** Must be the last in the enum */ + KBASEP_JS_CTX_ATTR_COUNT +}; + +enum { + /** Bit indicating that new atom should be started because this atom completed */ + KBASE_JS_ATOM_DONE_START_NEW_ATOMS = (1u << 0), + /** Bit indicating that the atom was evicted from the JS_NEXT registers */ + KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT = (1u << 1) +}; + +/** Combination of KBASE_JS_ATOM_DONE_<...> bits */ +typedef u32 kbasep_js_atom_done_code; + +/** + * @brief KBase Device Data Job Scheduler sub-structure + * + * This encapsulates the current context of the Job Scheduler on a particular + * device. This context is global to the device, and is not tied to any + * particular struct kbase_context running on the device. + * + * nr_contexts_running and as_free are optimized for packing together (by making + * them smaller types than u32). The operations on them should rarely involve + * masking. The use of signed types for arithmetic indicates to the compiler that + * the value will not rollover (which would be undefined behavior), and so under + * the Total License model, it is free to make optimizations based on that (i.e. + * to remove masking). + */ +struct kbasep_js_device_data { + /* Sub-structure to collect together Job Scheduling data used in IRQ + * context. The hwaccess_lock must be held when accessing. */ + struct runpool_irq { + /** Bitvector indicating whether a currently scheduled context is allowed to submit jobs. + * When bit 'N' is set in this, it indicates whether the context bound to address space + * 'N' is allowed to submit jobs. + */ + u16 submit_allowed; + + /** Context Attributes: + * Each is large enough to hold a refcount of the number of contexts + * that can fit into the runpool. This is currently BASE_MAX_NR_AS + * + * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store + * the refcount. Hence, it's not worthwhile reducing this to + * bit-manipulation on u32s to save space (where in contrast, 4 bit + * sub-fields would be easy to do and would save space). + * + * Whilst this must not become negative, the sign bit is used for: + * - error detection in debug builds + * - Optimization: it is undefined for a signed int to overflow, and so + * the compiler can optimize for that never happening (thus, no masking + * is required on updating the variable) */ + s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; + + /* + * Affinity management and tracking + */ + /** Bitvector to aid affinity checking. Element 'n' bit 'i' indicates + * that slot 'n' is using core i (i.e. slot_affinity_refcount[n][i] > 0) */ + u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; + /** Refcount for each core owned by each slot. Used to generate the + * slot_affinities array of bitvectors + * + * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, + * because it is refcounted only when a job is definitely about to be + * submitted to a slot, and is de-refcounted immediately after a job + * finishes */ + s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; + } runpool_irq; + + /** + * Run Pool mutex, for managing contexts within the runpool. + * Unless otherwise specified, you must hold this lock whilst accessing any + * members that follow + * + * In addition, this is used to access: + * - the kbasep_js_kctx_info::runpool substructure + */ + struct mutex runpool_mutex; + + /** + * Queue Lock, used to access the Policy's queue of contexts independently + * of the Run Pool. + * + * Of course, you don't need the Run Pool lock to access this. + */ + struct mutex queue_mutex; + + /** + * Scheduling semaphore. This must be held when calling + * kbase_jm_kick() + */ + struct semaphore schedule_sem; + + /** + * List of contexts that can currently be pulled from + */ + struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS]; + /** + * List of contexts that can not currently be pulled from, but have + * jobs currently running. + */ + struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS]; + + /** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */ + s8 nr_user_contexts_running; + /** Number of currently scheduled contexts (including ones that are not submitting jobs) */ + s8 nr_all_contexts_running; + + /** Core Requirements to match up with base_js_atom's core_req memeber + * @note This is a write-once member, and so no locking is required to read */ + base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; + + u32 scheduling_period_ns; /*< Value for JS_SCHEDULING_PERIOD_NS */ + u32 soft_stop_ticks; /*< Value for JS_SOFT_STOP_TICKS */ + u32 soft_stop_ticks_cl; /*< Value for JS_SOFT_STOP_TICKS_CL */ + u32 hard_stop_ticks_ss; /*< Value for JS_HARD_STOP_TICKS_SS */ + u32 hard_stop_ticks_cl; /*< Value for JS_HARD_STOP_TICKS_CL */ + u32 hard_stop_ticks_dumping; /*< Value for JS_HARD_STOP_TICKS_DUMPING */ + u32 gpu_reset_ticks_ss; /*< Value for JS_RESET_TICKS_SS */ + u32 gpu_reset_ticks_cl; /*< Value for JS_RESET_TICKS_CL */ + u32 gpu_reset_ticks_dumping; /*< Value for JS_RESET_TICKS_DUMPING */ + u32 ctx_timeslice_ns; /**< Value for JS_CTX_TIMESLICE_NS */ + + /**< Value for JS_SOFT_JOB_TIMEOUT */ + atomic_t soft_job_timeout_ms; + + /** List of suspended soft jobs */ + struct list_head suspended_soft_jobs_list; + +#ifdef CONFIG_MALI_DEBUG + /* Support soft-stop on a single context */ + bool softstop_always; +#endif /* CONFIG_MALI_DEBUG */ + + /** The initalized-flag is placed at the end, to avoid cache-pollution (we should + * only be using this during init/term paths). + * @note This is a write-once member, and so no locking is required to read */ + int init_status; + + /* Number of contexts that can currently be pulled from */ + u32 nr_contexts_pullable; + + /* Number of contexts that can either be pulled from or are currently + * running */ + atomic_t nr_contexts_runnable; +}; + +/** + * @brief KBase Context Job Scheduling information structure + * + * This is a substructure in the struct kbase_context that encapsulates all the + * scheduling information. + */ +struct kbasep_js_kctx_info { + + /** + * Job Scheduler Context information sub-structure. These members are + * accessed regardless of whether the context is: + * - In the Policy's Run Pool + * - In the Policy's Queue + * - Not queued nor in the Run Pool. + * + * You must obtain the jsctx_mutex before accessing any other members of + * this substructure. + * + * You may not access any of these members from IRQ context. + */ + struct kbase_jsctx { + struct mutex jsctx_mutex; /**< Job Scheduler Context lock */ + + /** Number of jobs ready to run - does \em not include the jobs waiting in + * the dispatcher, and dependency-only jobs. See kbase_jd_context::job_nr + * for such jobs*/ + u32 nr_jobs; + + /** Context Attributes: + * Each is large enough to hold a refcount of the number of atoms on + * the context. **/ + u32 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; + + /** + * Wait queue to wait for KCTX_SHEDULED flag state changes. + * */ + wait_queue_head_t is_scheduled_wait; + + /** Link implementing JS queues. Context can be present on one + * list per job slot + */ + struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS]; + } ctx; + + /* The initalized-flag is placed at the end, to avoid cache-pollution (we should + * only be using this during init/term paths) */ + int init_status; +}; + +/** Subset of atom state that can be available after jd_done_nolock() is called + * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), + * because the original atom could disappear. */ +struct kbasep_js_atom_retained_state { + /** Event code - to determine whether the atom has finished */ + enum base_jd_event_code event_code; + /** core requirements */ + base_jd_core_req core_req; + /* priority */ + int sched_priority; + /** Job Slot to retry submitting to if submission from IRQ handler failed */ + int retry_submit_on_slot; + /* Core group atom was executed on */ + u32 device_nr; + +}; + +/** + * Value signifying 'no retry on a slot required' for: + * - kbase_js_atom_retained_state::retry_submit_on_slot + * - kbase_jd_atom::retry_submit_on_slot + */ +#define KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID (-1) + +/** + * base_jd_core_req value signifying 'invalid' for a kbase_jd_atom_retained_state. + * + * @see kbase_atom_retained_state_is_valid() + */ +#define KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID BASE_JD_REQ_DEP + +/** + * @brief The JS timer resolution, in microseconds + * + * Any non-zero difference in time will be at least this size. + */ +#define KBASEP_JS_TICK_RESOLUTION_US 1 + +/* + * Internal atom priority defines for kbase_jd_atom::sched_prio + */ +enum { + KBASE_JS_ATOM_SCHED_PRIO_HIGH = 0, + KBASE_JS_ATOM_SCHED_PRIO_MED, + KBASE_JS_ATOM_SCHED_PRIO_LOW, + KBASE_JS_ATOM_SCHED_PRIO_COUNT, +}; + +/* Invalid priority for kbase_jd_atom::sched_prio */ +#define KBASE_JS_ATOM_SCHED_PRIO_INVALID -1 + +/* Default priority in the case of contexts with no atoms, or being lenient + * about invalid priorities from userspace */ +#define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED + + /** @} *//* end group kbase_js */ + /** @} *//* end group base_kbase_api */ + /** @} *//* end group base_api */ + +#endif /* _KBASE_JS_DEFS_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_linux.h b/drivers/gpu/arm/midgard/mali_kbase_linux.h new file mode 100755 index 000000000000..6d1e61fd41e0 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_linux.h @@ -0,0 +1,43 @@ +/* + * + * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_linux.h + * Base kernel APIs, Linux implementation. + */ + +#ifndef _KBASE_LINUX_H_ +#define _KBASE_LINUX_H_ + +/* All things that are needed for the Linux port. */ +#include +#include +#include +#include +#include + +#if (defined(MALI_KERNEL_TEST_API) && (1 == MALI_KERNEL_TEST_API)) + #define KBASE_EXPORT_TEST_API(func) EXPORT_SYMBOL(func) +#else + #define KBASE_EXPORT_TEST_API(func) +#endif + +#define KBASE_EXPORT_SYMBOL(func) EXPORT_SYMBOL(func) + +#endif /* _KBASE_LINUX_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem.c b/drivers/gpu/arm/midgard/mali_kbase_mem.c new file mode 100755 index 000000000000..8cbb5f6765b8 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem.c @@ -0,0 +1,2657 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mem.c + * Base kernel memory APIs + */ +#ifdef CONFIG_DMA_SHARED_BUFFER +#include +#endif /* CONFIG_DMA_SHARED_BUFFER */ +#ifdef CONFIG_UMP +#include +#endif /* CONFIG_UMP */ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/* This function finds out which RB tree the given GPU VA region belongs to + * based on the region zone */ +static struct rb_root *kbase_reg_flags_to_rbtree(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + struct rb_root *rbtree = NULL; + + switch (reg->flags & KBASE_REG_ZONE_MASK) { + case KBASE_REG_ZONE_CUSTOM_VA: + rbtree = &kctx->reg_rbtree_custom; + break; + case KBASE_REG_ZONE_EXEC: + rbtree = &kctx->reg_rbtree_exec; + break; + case KBASE_REG_ZONE_SAME_VA: + rbtree = &kctx->reg_rbtree_same; + /* fall through */ + default: + rbtree = &kctx->reg_rbtree_same; + break; + } + + return rbtree; +} + +/* This function finds out which RB tree the given pfn from the GPU VA belongs + * to based on the memory zone the pfn refers to */ +static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, + u64 gpu_pfn) +{ + struct rb_root *rbtree = NULL; + +#ifdef CONFIG_64BIT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { +#endif /* CONFIG_64BIT */ + if (gpu_pfn >= KBASE_REG_ZONE_CUSTOM_VA_BASE) + rbtree = &kctx->reg_rbtree_custom; + else if (gpu_pfn >= KBASE_REG_ZONE_EXEC_BASE) + rbtree = &kctx->reg_rbtree_exec; + else + rbtree = &kctx->reg_rbtree_same; +#ifdef CONFIG_64BIT + } else { + if (gpu_pfn >= kctx->same_va_end) + rbtree = &kctx->reg_rbtree_custom; + else + rbtree = &kctx->reg_rbtree_same; + } +#endif /* CONFIG_64BIT */ + + return rbtree; +} + +/* This function inserts a region into the tree. */ +static void kbase_region_tracker_insert(struct kbase_context *kctx, + struct kbase_va_region *new_reg) +{ + u64 start_pfn = new_reg->start_pfn; + struct rb_node **link = NULL; + struct rb_node *parent = NULL; + struct rb_root *rbtree = NULL; + + rbtree = kbase_reg_flags_to_rbtree(kctx, new_reg); + + link = &(rbtree->rb_node); + /* Find the right place in the tree using tree search */ + while (*link) { + struct kbase_va_region *old_reg; + + parent = *link; + old_reg = rb_entry(parent, struct kbase_va_region, rblink); + + /* RBTree requires no duplicate entries. */ + KBASE_DEBUG_ASSERT(old_reg->start_pfn != start_pfn); + + if (old_reg->start_pfn > start_pfn) + link = &(*link)->rb_left; + else + link = &(*link)->rb_right; + } + + /* Put the new node there, and rebalance tree */ + rb_link_node(&(new_reg->rblink), parent, link); + + rb_insert_color(&(new_reg->rblink), rbtree); +} + +/* Find allocated region enclosing free range. */ +static struct kbase_va_region *kbase_region_tracker_find_region_enclosing_range_free( + struct kbase_context *kctx, u64 start_pfn, size_t nr_pages) +{ + struct rb_node *rbnode = NULL; + struct kbase_va_region *reg = NULL; + struct rb_root *rbtree = NULL; + + u64 end_pfn = start_pfn + nr_pages; + + rbtree = kbase_gpu_va_to_rbtree(kctx, start_pfn); + + rbnode = rbtree->rb_node; + + while (rbnode) { + u64 tmp_start_pfn, tmp_end_pfn; + + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + tmp_start_pfn = reg->start_pfn; + tmp_end_pfn = reg->start_pfn + reg->nr_pages; + + /* If start is lower than this, go left. */ + if (start_pfn < tmp_start_pfn) + rbnode = rbnode->rb_left; + /* If end is higher than this, then go right. */ + else if (end_pfn > tmp_end_pfn) + rbnode = rbnode->rb_right; + else /* Enclosing */ + return reg; + } + + return NULL; +} + +/* Find region enclosing given address. */ +struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr) +{ + struct rb_node *rbnode; + struct kbase_va_region *reg; + u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; + struct rb_root *rbtree = NULL; + + KBASE_DEBUG_ASSERT(NULL != kctx); + + lockdep_assert_held(&kctx->reg_lock); + + rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); + + rbnode = rbtree->rb_node; + + while (rbnode) { + u64 tmp_start_pfn, tmp_end_pfn; + + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + tmp_start_pfn = reg->start_pfn; + tmp_end_pfn = reg->start_pfn + reg->nr_pages; + + /* If start is lower than this, go left. */ + if (gpu_pfn < tmp_start_pfn) + rbnode = rbnode->rb_left; + /* If end is higher than this, then go right. */ + else if (gpu_pfn >= tmp_end_pfn) + rbnode = rbnode->rb_right; + else /* Enclosing */ + return reg; + } + + return NULL; +} + +KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address); + +/* Find region with given base address */ +struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr) +{ + u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; + struct rb_node *rbnode = NULL; + struct kbase_va_region *reg = NULL; + struct rb_root *rbtree = NULL; + + KBASE_DEBUG_ASSERT(NULL != kctx); + + lockdep_assert_held(&kctx->reg_lock); + + rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); + + rbnode = rbtree->rb_node; + + while (rbnode) { + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + if (reg->start_pfn > gpu_pfn) + rbnode = rbnode->rb_left; + else if (reg->start_pfn < gpu_pfn) + rbnode = rbnode->rb_right; + else + return reg; + + } + + return NULL; +} + +KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address); + +/* Find region meeting given requirements */ +static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs(struct kbase_context *kctx, struct kbase_va_region *reg_reqs, size_t nr_pages, size_t align) +{ + struct rb_node *rbnode = NULL; + struct kbase_va_region *reg = NULL; + struct rb_root *rbtree = NULL; + + /* Note that this search is a linear search, as we do not have a target + address in mind, so does not benefit from the rbtree search */ + + rbtree = kbase_reg_flags_to_rbtree(kctx, reg_reqs); + + rbnode = rb_first(rbtree); + + while (rbnode) { + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + if ((reg->nr_pages >= nr_pages) && + (reg->flags & KBASE_REG_FREE)) { + /* Check alignment */ + u64 start_pfn = (reg->start_pfn + align - 1) & ~(align - 1); + + if ((start_pfn >= reg->start_pfn) && + (start_pfn <= (reg->start_pfn + reg->nr_pages - 1)) && + ((start_pfn + nr_pages - 1) <= (reg->start_pfn + reg->nr_pages - 1))) + return reg; + } + rbnode = rb_next(rbnode); + } + + return NULL; +} + +/** + * @brief Remove a region object from the global list. + * + * The region reg is removed, possibly by merging with other free and + * compatible adjacent regions. It must be called with the context + * region lock held. The associated memory is not released (see + * kbase_free_alloced_region). Internal use only. + */ +static int kbase_remove_va_region(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + struct rb_node *rbprev; + struct kbase_va_region *prev = NULL; + struct rb_node *rbnext; + struct kbase_va_region *next = NULL; + struct rb_root *reg_rbtree = NULL; + + int merged_front = 0; + int merged_back = 0; + int err = 0; + + reg_rbtree = kbase_reg_flags_to_rbtree(kctx, reg); + + /* Try to merge with the previous block first */ + rbprev = rb_prev(&(reg->rblink)); + if (rbprev) { + prev = rb_entry(rbprev, struct kbase_va_region, rblink); + if (prev->flags & KBASE_REG_FREE) { + /* We're compatible with the previous VMA, + * merge with it */ + WARN_ON((prev->flags & KBASE_REG_ZONE_MASK) != + (reg->flags & KBASE_REG_ZONE_MASK)); + prev->nr_pages += reg->nr_pages; + rb_erase(&(reg->rblink), reg_rbtree); + reg = prev; + merged_front = 1; + } + } + + /* Try to merge with the next block second */ + /* Note we do the lookup here as the tree may have been rebalanced. */ + rbnext = rb_next(&(reg->rblink)); + if (rbnext) { + /* We're compatible with the next VMA, merge with it */ + next = rb_entry(rbnext, struct kbase_va_region, rblink); + if (next->flags & KBASE_REG_FREE) { + WARN_ON((next->flags & KBASE_REG_ZONE_MASK) != + (reg->flags & KBASE_REG_ZONE_MASK)); + next->start_pfn = reg->start_pfn; + next->nr_pages += reg->nr_pages; + rb_erase(&(reg->rblink), reg_rbtree); + merged_back = 1; + if (merged_front) { + /* We already merged with prev, free it */ + kbase_free_alloced_region(reg); + } + } + } + + /* If we failed to merge then we need to add a new block */ + if (!(merged_front || merged_back)) { + /* + * We didn't merge anything. Add a new free + * placeholder and remove the original one. + */ + struct kbase_va_region *free_reg; + + free_reg = kbase_alloc_free_region(kctx, reg->start_pfn, reg->nr_pages, reg->flags & KBASE_REG_ZONE_MASK); + if (!free_reg) { + err = -ENOMEM; + goto out; + } + rb_replace_node(&(reg->rblink), &(free_reg->rblink), reg_rbtree); + } + + out: + return err; +} + +KBASE_EXPORT_TEST_API(kbase_remove_va_region); + +/** + * @brief Insert a VA region to the list, replacing the current at_reg. + */ +static int kbase_insert_va_region_nolock(struct kbase_context *kctx, struct kbase_va_region *new_reg, struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages) +{ + struct rb_root *reg_rbtree = NULL; + int err = 0; + + reg_rbtree = kbase_reg_flags_to_rbtree(kctx, at_reg); + + /* Must be a free region */ + KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0); + /* start_pfn should be contained within at_reg */ + KBASE_DEBUG_ASSERT((start_pfn >= at_reg->start_pfn) && (start_pfn < at_reg->start_pfn + at_reg->nr_pages)); + /* at least nr_pages from start_pfn should be contained within at_reg */ + KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= at_reg->start_pfn + at_reg->nr_pages); + + new_reg->start_pfn = start_pfn; + new_reg->nr_pages = nr_pages; + + /* Regions are a whole use, so swap and delete old one. */ + if (at_reg->start_pfn == start_pfn && at_reg->nr_pages == nr_pages) { + rb_replace_node(&(at_reg->rblink), &(new_reg->rblink), + reg_rbtree); + kbase_free_alloced_region(at_reg); + } + /* New region replaces the start of the old one, so insert before. */ + else if (at_reg->start_pfn == start_pfn) { + at_reg->start_pfn += nr_pages; + KBASE_DEBUG_ASSERT(at_reg->nr_pages >= nr_pages); + at_reg->nr_pages -= nr_pages; + + kbase_region_tracker_insert(kctx, new_reg); + } + /* New region replaces the end of the old one, so insert after. */ + else if ((at_reg->start_pfn + at_reg->nr_pages) == (start_pfn + nr_pages)) { + at_reg->nr_pages -= nr_pages; + + kbase_region_tracker_insert(kctx, new_reg); + } + /* New region splits the old one, so insert and create new */ + else { + struct kbase_va_region *new_front_reg; + + new_front_reg = kbase_alloc_free_region(kctx, + at_reg->start_pfn, + start_pfn - at_reg->start_pfn, + at_reg->flags & KBASE_REG_ZONE_MASK); + + if (new_front_reg) { + at_reg->nr_pages -= nr_pages + new_front_reg->nr_pages; + at_reg->start_pfn = start_pfn + nr_pages; + + kbase_region_tracker_insert(kctx, new_front_reg); + kbase_region_tracker_insert(kctx, new_reg); + } else { + err = -ENOMEM; + } + } + + return err; +} + +/** + * @brief Add a VA region to the list. + */ +int kbase_add_va_region(struct kbase_context *kctx, + struct kbase_va_region *reg, u64 addr, + size_t nr_pages, size_t align) +{ + struct kbase_va_region *tmp; + u64 gpu_pfn = addr >> PAGE_SHIFT; + int err = 0; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != reg); + + lockdep_assert_held(&kctx->reg_lock); + + if (!align) + align = 1; + + /* must be a power of 2 */ + KBASE_DEBUG_ASSERT((align & (align - 1)) == 0); + KBASE_DEBUG_ASSERT(nr_pages > 0); + + /* Path 1: Map a specific address. Find the enclosing region, which *must* be free. */ + if (gpu_pfn) { + struct device *dev = kctx->kbdev->dev; + + KBASE_DEBUG_ASSERT(!(gpu_pfn & (align - 1))); + + tmp = kbase_region_tracker_find_region_enclosing_range_free(kctx, gpu_pfn, nr_pages); + if (!tmp) { + dev_warn(dev, "Enclosing region not found: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages); + err = -ENOMEM; + goto exit; + } + if (!(tmp->flags & KBASE_REG_FREE)) { + dev_warn(dev, "Zone mismatch: %lu != %lu", tmp->flags & KBASE_REG_ZONE_MASK, reg->flags & KBASE_REG_ZONE_MASK); + dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", tmp->start_pfn, tmp->flags, tmp->nr_pages, gpu_pfn, nr_pages); + dev_warn(dev, "in function %s (%p, %p, 0x%llx, 0x%zx, 0x%zx)\n", __func__, kctx, reg, addr, nr_pages, align); + err = -ENOMEM; + goto exit; + } + + err = kbase_insert_va_region_nolock(kctx, reg, tmp, gpu_pfn, nr_pages); + if (err) { + dev_warn(dev, "Failed to insert va region"); + err = -ENOMEM; + goto exit; + } + + goto exit; + } + + /* Path 2: Map any free address which meets the requirements. */ + { + u64 start_pfn; + + /* + * Depending on the zone the allocation request is for + * we might need to retry it. + */ + do { + tmp = kbase_region_tracker_find_region_meeting_reqs( + kctx, reg, nr_pages, align); + if (tmp) { + start_pfn = (tmp->start_pfn + align - 1) & + ~(align - 1); + err = kbase_insert_va_region_nolock(kctx, reg, + tmp, start_pfn, nr_pages); + break; + } + + /* + * If the allocation is not from the same zone as JIT + * then don't retry, we're out of VA and there is + * nothing which can be done about it. + */ + if ((reg->flags & KBASE_REG_ZONE_MASK) != + KBASE_REG_ZONE_CUSTOM_VA) + break; + } while (kbase_jit_evict(kctx)); + + if (!tmp) + err = -ENOMEM; + } + + exit: + return err; +} + +KBASE_EXPORT_TEST_API(kbase_add_va_region); + +/** + * @brief Initialize the internal region tracker data structure. + */ +static void kbase_region_tracker_ds_init(struct kbase_context *kctx, + struct kbase_va_region *same_va_reg, + struct kbase_va_region *exec_reg, + struct kbase_va_region *custom_va_reg) +{ + kctx->reg_rbtree_same = RB_ROOT; + kbase_region_tracker_insert(kctx, same_va_reg); + + /* Although exec and custom_va_reg don't always exist, + * initialize unconditionally because of the mem_view debugfs + * implementation which relies on these being empty */ + kctx->reg_rbtree_exec = RB_ROOT; + kctx->reg_rbtree_custom = RB_ROOT; + + if (exec_reg) + kbase_region_tracker_insert(kctx, exec_reg); + if (custom_va_reg) + kbase_region_tracker_insert(kctx, custom_va_reg); +} + +static void kbase_region_tracker_erase_rbtree(struct rb_root *rbtree) +{ + struct rb_node *rbnode; + struct kbase_va_region *reg; + + do { + rbnode = rb_first(rbtree); + if (rbnode) { + rb_erase(rbnode, rbtree); + reg = rb_entry(rbnode, struct kbase_va_region, rblink); + kbase_free_alloced_region(reg); + } + } while (rbnode); +} + +void kbase_region_tracker_term(struct kbase_context *kctx) +{ + kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_same); + kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_exec); + kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_custom); +} + +/** + * Initialize the region tracker data structure. + */ +int kbase_region_tracker_init(struct kbase_context *kctx) +{ + struct kbase_va_region *same_va_reg; + struct kbase_va_region *exec_reg = NULL; + struct kbase_va_region *custom_va_reg = NULL; + size_t same_va_bits = sizeof(void *) * BITS_PER_BYTE; + u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE; + u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT; + u64 same_va_pages; + int err; + + /* Take the lock as kbase_free_alloced_region requires it */ + kbase_gpu_vm_lock(kctx); + +#if defined(CONFIG_ARM64) + same_va_bits = VA_BITS; +#elif defined(CONFIG_X86_64) + same_va_bits = 47; +#elif defined(CONFIG_64BIT) +#error Unsupported 64-bit architecture +#endif + +#ifdef CONFIG_64BIT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + same_va_bits = 32; + else if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA)) + same_va_bits = 33; +#endif + + if (kctx->kbdev->gpu_props.mmu.va_bits < same_va_bits) { + err = -EINVAL; + goto fail_unlock; + } + + same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; + /* all have SAME_VA */ + same_va_reg = kbase_alloc_free_region(kctx, 1, + same_va_pages, + KBASE_REG_ZONE_SAME_VA); + + if (!same_va_reg) { + err = -ENOMEM; + goto fail_unlock; + } + +#ifdef CONFIG_64BIT + /* 32-bit clients have exec and custom VA zones */ + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { +#endif + if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) { + err = -EINVAL; + goto fail_free_same_va; + } + /* If the current size of TMEM is out of range of the + * virtual address space addressable by the MMU then + * we should shrink it to fit + */ + if ((KBASE_REG_ZONE_CUSTOM_VA_BASE + KBASE_REG_ZONE_CUSTOM_VA_SIZE) >= gpu_va_limit) + custom_va_size = gpu_va_limit - KBASE_REG_ZONE_CUSTOM_VA_BASE; + + exec_reg = kbase_alloc_free_region(kctx, + KBASE_REG_ZONE_EXEC_BASE, + KBASE_REG_ZONE_EXEC_SIZE, + KBASE_REG_ZONE_EXEC); + + if (!exec_reg) { + err = -ENOMEM; + goto fail_free_same_va; + } + + custom_va_reg = kbase_alloc_free_region(kctx, + KBASE_REG_ZONE_CUSTOM_VA_BASE, + custom_va_size, KBASE_REG_ZONE_CUSTOM_VA); + + if (!custom_va_reg) { + err = -ENOMEM; + goto fail_free_exec; + } +#ifdef CONFIG_64BIT + } +#endif + + kbase_region_tracker_ds_init(kctx, same_va_reg, exec_reg, custom_va_reg); + + kctx->same_va_end = same_va_pages + 1; + + kbase_gpu_vm_unlock(kctx); + return 0; + +fail_free_exec: + kbase_free_alloced_region(exec_reg); +fail_free_same_va: + kbase_free_alloced_region(same_va_reg); +fail_unlock: + kbase_gpu_vm_unlock(kctx); + return err; +} + +int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages) +{ +#ifdef CONFIG_64BIT + struct kbase_va_region *same_va; + struct kbase_va_region *custom_va_reg; + u64 same_va_bits; + u64 total_va_size; + int err; + + /* + * Nothing to do for 32-bit clients, JIT uses the existing + * custom VA zone. + */ + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + return 0; + +#if defined(CONFIG_ARM64) + same_va_bits = VA_BITS; +#elif defined(CONFIG_X86_64) + same_va_bits = 47; +#elif defined(CONFIG_64BIT) +#error Unsupported 64-bit architecture +#endif + + if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA)) + same_va_bits = 33; + + total_va_size = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; + + kbase_gpu_vm_lock(kctx); + + /* + * Modify the same VA free region after creation. Be careful to ensure + * that allocations haven't been made as they could cause an overlap + * to happen with existing same VA allocations and the custom VA zone. + */ + same_va = kbase_region_tracker_find_region_base_address(kctx, + PAGE_SIZE); + if (!same_va) { + err = -ENOMEM; + goto fail_unlock; + } + + /* The region flag or region size has changed since creation so bail. */ + if ((!(same_va->flags & KBASE_REG_FREE)) || + (same_va->nr_pages != total_va_size)) { + err = -ENOMEM; + goto fail_unlock; + } + + if (same_va->nr_pages < jit_va_pages || + kctx->same_va_end < jit_va_pages) { + err = -ENOMEM; + goto fail_unlock; + } + + /* It's safe to adjust the same VA zone now */ + same_va->nr_pages -= jit_va_pages; + kctx->same_va_end -= jit_va_pages; + + /* + * Create a custom VA zone at the end of the VA for allocations which + * JIT can use so it doesn't have to allocate VA from the kernel. + */ + custom_va_reg = kbase_alloc_free_region(kctx, + kctx->same_va_end, + jit_va_pages, + KBASE_REG_ZONE_CUSTOM_VA); + + if (!custom_va_reg) { + /* + * The context will be destroyed if we fail here so no point + * reverting the change we made to same_va. + */ + err = -ENOMEM; + goto fail_unlock; + } + + kbase_region_tracker_insert(kctx, custom_va_reg); + + kbase_gpu_vm_unlock(kctx); + return 0; + +fail_unlock: + kbase_gpu_vm_unlock(kctx); + return err; +#else + return 0; +#endif +} + +int kbase_mem_init(struct kbase_device *kbdev) +{ + struct kbasep_mem_device *memdev; + + KBASE_DEBUG_ASSERT(kbdev); + + memdev = &kbdev->memdev; + kbdev->mem_pool_max_size_default = KBASE_MEM_POOL_MAX_SIZE_KCTX; + + /* Initialize memory usage */ + atomic_set(&memdev->used_pages, 0); + + return kbase_mem_pool_init(&kbdev->mem_pool, + KBASE_MEM_POOL_MAX_SIZE_KBDEV, kbdev, NULL); +} + +void kbase_mem_halt(struct kbase_device *kbdev) +{ + CSTD_UNUSED(kbdev); +} + +void kbase_mem_term(struct kbase_device *kbdev) +{ + struct kbasep_mem_device *memdev; + int pages; + + KBASE_DEBUG_ASSERT(kbdev); + + memdev = &kbdev->memdev; + + pages = atomic_read(&memdev->used_pages); + if (pages != 0) + dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); + + kbase_mem_pool_term(&kbdev->mem_pool); +} + +KBASE_EXPORT_TEST_API(kbase_mem_term); + + + + +/** + * @brief Allocate a free region object. + * + * The allocated object is not part of any list yet, and is flagged as + * KBASE_REG_FREE. No mapping is allocated yet. + * + * zone is KBASE_REG_ZONE_CUSTOM_VA, KBASE_REG_ZONE_SAME_VA, or KBASE_REG_ZONE_EXEC + * + */ +struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone) +{ + struct kbase_va_region *new_reg; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + /* zone argument should only contain zone related region flags */ + KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0); + KBASE_DEBUG_ASSERT(nr_pages > 0); + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE)); + + new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL); + + if (!new_reg) + return NULL; + + new_reg->cpu_alloc = NULL; /* no alloc bound yet */ + new_reg->gpu_alloc = NULL; /* no alloc bound yet */ + new_reg->kctx = kctx; + new_reg->flags = zone | KBASE_REG_FREE; + + new_reg->flags |= KBASE_REG_GROWABLE; + + new_reg->start_pfn = start_pfn; + new_reg->nr_pages = nr_pages; + + return new_reg; +} + +KBASE_EXPORT_TEST_API(kbase_alloc_free_region); + +/** + * @brief Free a region object. + * + * The described region must be freed of any mapping. + * + * If the region is not flagged as KBASE_REG_FREE, the region's + * alloc object will be released. + * It is a bug if no alloc object exists for non-free regions. + * + */ +void kbase_free_alloced_region(struct kbase_va_region *reg) +{ + if (!(reg->flags & KBASE_REG_FREE)) { + /* + * The physical allocation should have been removed from the + * eviction list before this function is called. However, in the + * case of abnormal process termination or the app leaking the + * memory kbase_mem_free_region is not called so it can still be + * on the list at termination time of the region tracker. + */ + if (!list_empty(®->gpu_alloc->evict_node)) { + /* + * Unlink the physical allocation before unmaking it + * evictable so that the allocation isn't grown back to + * its last backed size as we're going to unmap it + * anyway. + */ + reg->cpu_alloc->reg = NULL; + if (reg->cpu_alloc != reg->gpu_alloc) + reg->gpu_alloc->reg = NULL; + + /* + * If a region has been made evictable then we must + * unmake it before trying to free it. + * If the memory hasn't been reclaimed it will be + * unmapped and freed below, if it has been reclaimed + * then the operations below are no-ops. + */ + if (reg->flags & KBASE_REG_DONT_NEED) { + KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == + KBASE_MEM_TYPE_NATIVE); + kbase_mem_evictable_unmake(reg->gpu_alloc); + } + } + + /* + * Remove the region from the sticky resource metadata + * list should it be there. + */ + kbase_sticky_resource_release(reg->kctx, NULL, + reg->start_pfn << PAGE_SHIFT); + + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); + /* To detect use-after-free in debug builds */ + KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE); + } + kfree(reg); +} + +KBASE_EXPORT_TEST_API(kbase_free_alloced_region); + +int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align) +{ + int err; + size_t i = 0; + unsigned long attr; + unsigned long mask = ~KBASE_REG_MEMATTR_MASK; + + if ((kctx->kbdev->system_coherency == COHERENCY_ACE) && + (reg->flags & KBASE_REG_SHARE_BOTH)) + attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA); + else + attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC); + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != reg); + + err = kbase_add_va_region(kctx, reg, addr, nr_pages, align); + if (err) + return err; + + if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { + u64 stride; + struct kbase_mem_phy_alloc *alloc; + + alloc = reg->gpu_alloc; + stride = alloc->imported.alias.stride; + KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); + for (i = 0; i < alloc->imported.alias.nents; i++) { + if (alloc->imported.alias.aliased[i].alloc) { + err = kbase_mmu_insert_pages(kctx, + reg->start_pfn + (i * stride), + alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset, + alloc->imported.alias.aliased[i].length, + reg->flags); + if (err) + goto bad_insert; + + kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc); + } else { + err = kbase_mmu_insert_single_page(kctx, + reg->start_pfn + i * stride, + page_to_phys(kctx->aliasing_sink_page), + alloc->imported.alias.aliased[i].length, + (reg->flags & mask) | attr); + + if (err) + goto bad_insert; + } + } + } else { + err = kbase_mmu_insert_pages(kctx, reg->start_pfn, + kbase_get_gpu_phy_pages(reg), + kbase_reg_current_backed_size(reg), + reg->flags); + if (err) + goto bad_insert; + kbase_mem_phy_alloc_gpu_mapped(reg->gpu_alloc); + } + + return err; + +bad_insert: + if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { + u64 stride; + + stride = reg->gpu_alloc->imported.alias.stride; + KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); + while (i--) + if (reg->gpu_alloc->imported.alias.aliased[i].alloc) { + kbase_mmu_teardown_pages(kctx, reg->start_pfn + (i * stride), reg->gpu_alloc->imported.alias.aliased[i].length); + kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); + } + } + + kbase_remove_va_region(kctx, reg); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_gpu_mmap); + +static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc, bool writeable); + +int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + int err; + + if (reg->start_pfn == 0) + return 0; + + if (reg->gpu_alloc && reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { + size_t i; + + err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, reg->nr_pages); + KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); + for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++) + if (reg->gpu_alloc->imported.alias.aliased[i].alloc) + kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); + } else { + err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, kbase_reg_current_backed_size(reg)); + kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc); + } + + if (reg->gpu_alloc && reg->gpu_alloc->type == + KBASE_MEM_TYPE_IMPORTED_USER_BUF) { + struct kbase_alloc_import_user_buf *user_buf = + ®->gpu_alloc->imported.user_buf; + + if (user_buf->current_mapping_usage_count & PINNED_ON_IMPORT) { + user_buf->current_mapping_usage_count &= + ~PINNED_ON_IMPORT; + + kbase_jd_user_buf_unmap(kctx, reg->gpu_alloc, + (reg->flags & KBASE_REG_GPU_WR)); + } + } + + if (err) + return err; + + err = kbase_remove_va_region(kctx, reg); + return err; +} + +static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( + struct kbase_context *kctx, + unsigned long uaddr, size_t size, u64 *offset) +{ + struct vm_area_struct *vma; + struct kbase_cpu_mapping *map; + unsigned long vm_pgoff_in_region; + unsigned long vm_off_in_region; + unsigned long map_start; + size_t map_size; + + lockdep_assert_held(¤t->mm->mmap_lock); + + if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */ + return NULL; + + vma = find_vma_intersection(current->mm, uaddr, uaddr+size); + + if (!vma || vma->vm_start > uaddr) + return NULL; + if (vma->vm_ops != &kbase_vm_ops) + /* Not ours! */ + return NULL; + + map = vma->vm_private_data; + + if (map->kctx != kctx) + /* Not from this context! */ + return NULL; + + vm_pgoff_in_region = vma->vm_pgoff - map->region->start_pfn; + vm_off_in_region = vm_pgoff_in_region << PAGE_SHIFT; + map_start = vma->vm_start - vm_off_in_region; + map_size = map->region->nr_pages << PAGE_SHIFT; + + if ((uaddr + size) > (map_start + map_size)) + /* Not within the CPU mapping */ + return NULL; + + *offset = (uaddr - vma->vm_start) + vm_off_in_region; + + return map; +} + +int kbasep_find_enclosing_cpu_mapping_offset( + struct kbase_context *kctx, + unsigned long uaddr, size_t size, u64 *offset) +{ + struct kbase_cpu_mapping *map; + + kbase_os_mem_map_lock(kctx); + + map = kbasep_find_enclosing_cpu_mapping(kctx, uaddr, size, offset); + + kbase_os_mem_map_unlock(kctx); + + if (!map) + return -EINVAL; + + return 0; +} + +KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset); + +void kbase_sync_single(struct kbase_context *kctx, + phys_addr_t cpu_pa, phys_addr_t gpu_pa, + off_t offset, size_t size, enum kbase_sync_type sync_fn) +{ + struct page *cpu_page; + + cpu_page = pfn_to_page(PFN_DOWN(cpu_pa)); + + if (likely(cpu_pa == gpu_pa)) { + dma_addr_t dma_addr; + + BUG_ON(!cpu_page); + BUG_ON(offset + size > PAGE_SIZE); + + dma_addr = kbase_dma_addr(cpu_page) + offset; + if (sync_fn == KBASE_SYNC_TO_CPU) + dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr, + size, DMA_BIDIRECTIONAL); + else if (sync_fn == KBASE_SYNC_TO_DEVICE) + dma_sync_single_for_device(kctx->kbdev->dev, dma_addr, + size, DMA_BIDIRECTIONAL); + } else { + void *src = NULL; + void *dst = NULL; + struct page *gpu_page; + + if (WARN(!gpu_pa, "No GPU PA found for infinite cache op")) + return; + + gpu_page = pfn_to_page(PFN_DOWN(gpu_pa)); + + if (sync_fn == KBASE_SYNC_TO_DEVICE) { + src = ((unsigned char *)kmap(cpu_page)) + offset; + dst = ((unsigned char *)kmap(gpu_page)) + offset; + } else if (sync_fn == KBASE_SYNC_TO_CPU) { + dma_sync_single_for_cpu(kctx->kbdev->dev, + kbase_dma_addr(gpu_page) + offset, + size, DMA_BIDIRECTIONAL); + src = ((unsigned char *)kmap(gpu_page)) + offset; + dst = ((unsigned char *)kmap(cpu_page)) + offset; + } + memcpy(dst, src, size); + kunmap(gpu_page); + kunmap(cpu_page); + if (sync_fn == KBASE_SYNC_TO_DEVICE) + dma_sync_single_for_device(kctx->kbdev->dev, + kbase_dma_addr(gpu_page) + offset, + size, DMA_BIDIRECTIONAL); + } +} + +static int kbase_do_syncset(struct kbase_context *kctx, + struct basep_syncset *sset, enum kbase_sync_type sync_fn) +{ + int err = 0; + struct kbase_va_region *reg; + struct kbase_cpu_mapping *map; + unsigned long start; + size_t size; + phys_addr_t *cpu_pa; + phys_addr_t *gpu_pa; + u64 page_off, page_count; + u64 i; + u64 offset; + + kbase_os_mem_map_lock(kctx); + kbase_gpu_vm_lock(kctx); + + /* find the region where the virtual address is contained */ + reg = kbase_region_tracker_find_region_enclosing_address(kctx, + sset->mem_handle.basep.handle); + if (!reg) { + dev_warn(kctx->kbdev->dev, "Can't find region at VA 0x%016llX", + sset->mem_handle.basep.handle); + err = -EINVAL; + goto out_unlock; + } + + if (!(reg->flags & KBASE_REG_CPU_CACHED)) + goto out_unlock; + + start = (uintptr_t)sset->user_addr; + size = (size_t)sset->size; + + map = kbasep_find_enclosing_cpu_mapping(kctx, start, size, &offset); + if (!map) { + dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX", + start, sset->mem_handle.basep.handle); + err = -EINVAL; + goto out_unlock; + } + + page_off = offset >> PAGE_SHIFT; + offset &= ~PAGE_MASK; + page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + cpu_pa = kbase_get_cpu_phy_pages(reg); + gpu_pa = kbase_get_gpu_phy_pages(reg); + + if (page_off > reg->nr_pages || + page_off + page_count > reg->nr_pages) { + /* Sync overflows the region */ + err = -EINVAL; + goto out_unlock; + } + + /* Sync first page */ + if (cpu_pa[page_off]) { + size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); + + kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off], + offset, sz, sync_fn); + } + + /* Sync middle pages (if any) */ + for (i = 1; page_count > 2 && i < page_count - 1; i++) { + /* we grow upwards, so bail on first non-present page */ + if (!cpu_pa[page_off + i]) + break; + + kbase_sync_single(kctx, cpu_pa[page_off + i], + gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn); + } + + /* Sync last page (if any) */ + if (page_count > 1 && cpu_pa[page_off + page_count - 1]) { + size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1; + + kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1], + gpu_pa[page_off + page_count - 1], 0, sz, + sync_fn); + } + +out_unlock: + kbase_gpu_vm_unlock(kctx); + kbase_os_mem_map_unlock(kctx); + return err; +} + +int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) +{ + int err = -EINVAL; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(sset != NULL); + + if (sset->mem_handle.basep.handle & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, + "mem_handle: passed parameter is invalid"); + return -EINVAL; + } + + switch (sset->type) { + case BASE_SYNCSET_OP_MSYNC: + err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_DEVICE); + break; + + case BASE_SYNCSET_OP_CSYNC: + err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_CPU); + break; + + default: + dev_warn(kctx->kbdev->dev, "Unknown msync op %d\n", sset->type); + break; + } + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_sync_now); + +/* vm lock must be held */ +int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + int err; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != reg); + lockdep_assert_held(&kctx->reg_lock); + + /* + * Unlink the physical allocation before unmaking it evictable so + * that the allocation isn't grown back to its last backed size + * as we're going to unmap it anyway. + */ + reg->cpu_alloc->reg = NULL; + if (reg->cpu_alloc != reg->gpu_alloc) + reg->gpu_alloc->reg = NULL; + + /* + * If a region has been made evictable then we must unmake it + * before trying to free it. + * If the memory hasn't been reclaimed it will be unmapped and freed + * below, if it has been reclaimed then the operations below are no-ops. + */ + if (reg->flags & KBASE_REG_DONT_NEED) { + KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == + KBASE_MEM_TYPE_NATIVE); + kbase_mem_evictable_unmake(reg->gpu_alloc); + } + + err = kbase_gpu_munmap(kctx, reg); + if (err) { + dev_warn(reg->kctx->kbdev->dev, "Could not unmap from the GPU...\n"); + goto out; + } + + /* This will also free the physical pages */ + kbase_free_alloced_region(reg); + + out: + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mem_free_region); + +/** + * @brief Free the region from the GPU and unregister it. + * + * This function implements the free operation on a memory segment. + * It will loudly fail if called with outstanding mappings. + */ +int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) +{ + int err = 0; + struct kbase_va_region *reg; + + KBASE_DEBUG_ASSERT(kctx != NULL); + + if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) { + dev_warn(kctx->kbdev->dev, "kbase_mem_free: gpu_addr parameter is invalid"); + return -EINVAL; + } + + if (0 == gpu_addr) { + dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); + return -EINVAL; + } + kbase_gpu_vm_lock(kctx); + + if (gpu_addr >= BASE_MEM_COOKIE_BASE && + gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) { + int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); + + reg = kctx->pending_regions[cookie]; + if (!reg) { + err = -EINVAL; + goto out_unlock; + } + + /* ask to unlink the cookie as we'll free it */ + + kctx->pending_regions[cookie] = NULL; + kctx->cookies |= (1UL << cookie); + + kbase_free_alloced_region(reg); + } else { + /* A real GPU va */ + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (!reg || (reg->flags & KBASE_REG_FREE)) { + dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX", + gpu_addr); + err = -EINVAL; + goto out_unlock; + } + + if ((reg->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA) { + /* SAME_VA must be freed through munmap */ + dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__, + gpu_addr); + err = -EINVAL; + goto out_unlock; + } + err = kbase_mem_free_region(kctx, reg); + } + + out_unlock: + kbase_gpu_vm_unlock(kctx); + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mem_free); + +int kbase_update_region_flags(struct kbase_context *kctx, + struct kbase_va_region *reg, unsigned long flags) +{ + KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0); + + reg->flags |= kbase_cache_enabled(flags, reg->nr_pages); + /* all memory is now growable */ + reg->flags |= KBASE_REG_GROWABLE; + + if (flags & BASE_MEM_GROW_ON_GPF) + reg->flags |= KBASE_REG_PF_GROW; + + if (flags & BASE_MEM_PROT_CPU_WR) + reg->flags |= KBASE_REG_CPU_WR; + + if (flags & BASE_MEM_PROT_CPU_RD) + reg->flags |= KBASE_REG_CPU_RD; + + if (flags & BASE_MEM_PROT_GPU_WR) + reg->flags |= KBASE_REG_GPU_WR; + + if (flags & BASE_MEM_PROT_GPU_RD) + reg->flags |= KBASE_REG_GPU_RD; + + if (0 == (flags & BASE_MEM_PROT_GPU_EX)) + reg->flags |= KBASE_REG_GPU_NX; + + if (!kbase_device_is_cpu_coherent(kctx->kbdev)) { + if (flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) + return -EINVAL; + } else if (flags & (BASE_MEM_COHERENT_SYSTEM | + BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { + reg->flags |= KBASE_REG_SHARE_BOTH; + } + + if (!(reg->flags & KBASE_REG_SHARE_BOTH) && + flags & BASE_MEM_COHERENT_LOCAL) { + reg->flags |= KBASE_REG_SHARE_IN; + } + + /* Set up default MEMATTR usage */ + if (kctx->kbdev->system_coherency == COHERENCY_ACE && + (reg->flags & KBASE_REG_SHARE_BOTH)) { + reg->flags |= + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); + } else { + reg->flags |= + KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT); + } + + return 0; +} + +int kbase_alloc_phy_pages_helper( + struct kbase_mem_phy_alloc *alloc, + size_t nr_pages_requested) +{ + int new_page_count __maybe_unused; + size_t old_page_count = alloc->nents; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); + KBASE_DEBUG_ASSERT(alloc->imported.kctx); + + if (nr_pages_requested == 0) + goto done; /*nothing to do*/ + + new_page_count = kbase_atomic_add_pages( + nr_pages_requested, &alloc->imported.kctx->used_pages); + kbase_atomic_add_pages(nr_pages_requested, &alloc->imported.kctx->kbdev->memdev.used_pages); + + /* Increase mm counters before we allocate pages so that this + * allocation is visible to the OOM killer */ + kbase_process_page_usage_inc(alloc->imported.kctx, nr_pages_requested); + + if (kbase_mem_pool_alloc_pages(&alloc->imported.kctx->mem_pool, + nr_pages_requested, alloc->pages + old_page_count) != 0) + goto no_alloc; + + KBASE_TLSTREAM_AUX_PAGESALLOC( + (u32)alloc->imported.kctx->id, + (u64)new_page_count); + + alloc->nents += nr_pages_requested; +done: + return 0; + +no_alloc: + kbase_process_page_usage_dec(alloc->imported.kctx, nr_pages_requested); + kbase_atomic_sub_pages(nr_pages_requested, &alloc->imported.kctx->used_pages); + kbase_atomic_sub_pages(nr_pages_requested, &alloc->imported.kctx->kbdev->memdev.used_pages); + + return -ENOMEM; +} + +int kbase_free_phy_pages_helper( + struct kbase_mem_phy_alloc *alloc, + size_t nr_pages_to_free) +{ + struct kbase_context *kctx = alloc->imported.kctx; + bool syncback; + bool reclaimed = (alloc->evicted != 0); + phys_addr_t *start_free; + int new_page_count __maybe_unused; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); + KBASE_DEBUG_ASSERT(alloc->imported.kctx); + KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free); + + /* early out if nothing to do */ + if (0 == nr_pages_to_free) + return 0; + + start_free = alloc->pages + alloc->nents - nr_pages_to_free; + + syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; + + kbase_mem_pool_free_pages(&kctx->mem_pool, + nr_pages_to_free, + start_free, + syncback, + reclaimed); + + alloc->nents -= nr_pages_to_free; + + /* + * If the allocation was not evicted (i.e. evicted == 0) then + * the page accounting needs to be done. + */ + if (!reclaimed) { + kbase_process_page_usage_dec(kctx, nr_pages_to_free); + new_page_count = kbase_atomic_sub_pages(nr_pages_to_free, + &kctx->used_pages); + kbase_atomic_sub_pages(nr_pages_to_free, + &kctx->kbdev->memdev.used_pages); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + (u32)kctx->id, + (u64)new_page_count); + } + + return 0; +} + +void kbase_mem_kref_free(struct kref *kref) +{ + struct kbase_mem_phy_alloc *alloc; + + alloc = container_of(kref, struct kbase_mem_phy_alloc, kref); + + switch (alloc->type) { + case KBASE_MEM_TYPE_NATIVE: { + WARN_ON(!alloc->imported.kctx); + /* + * The physical allocation must have been removed from the + * eviction list before trying to free it. + */ + WARN_ON(!list_empty(&alloc->evict_node)); + kbase_free_phy_pages_helper(alloc, alloc->nents); + break; + } + case KBASE_MEM_TYPE_ALIAS: { + /* just call put on the underlying phy allocs */ + size_t i; + struct kbase_aliased *aliased; + + aliased = alloc->imported.alias.aliased; + if (aliased) { + for (i = 0; i < alloc->imported.alias.nents; i++) + if (aliased[i].alloc) + kbase_mem_phy_alloc_put(aliased[i].alloc); + vfree(aliased); + } + break; + } + case KBASE_MEM_TYPE_RAW: + /* raw pages, external cleanup */ + break; + #ifdef CONFIG_UMP + case KBASE_MEM_TYPE_IMPORTED_UMP: + ump_dd_release(alloc->imported.ump_handle); + break; +#endif +#ifdef CONFIG_DMA_SHARED_BUFFER + case KBASE_MEM_TYPE_IMPORTED_UMM: + dma_buf_detach(alloc->imported.umm.dma_buf, + alloc->imported.umm.dma_attachment); + dma_buf_put(alloc->imported.umm.dma_buf); + break; +#endif + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + if (alloc->imported.user_buf.mm) + mmdrop(alloc->imported.user_buf.mm); + kfree(alloc->imported.user_buf.pages); + break; + case KBASE_MEM_TYPE_TB:{ + void *tb; + + tb = alloc->imported.kctx->jctx.tb; + kbase_device_trace_buffer_uninstall(alloc->imported.kctx); + vfree(tb); + break; + } + default: + WARN(1, "Unexecpted free of type %d\n", alloc->type); + break; + } + + /* Free based on allocation type */ + if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) + vfree(alloc); + else + kfree(alloc); +} + +KBASE_EXPORT_TEST_API(kbase_mem_kref_free); + +int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size) +{ + KBASE_DEBUG_ASSERT(NULL != reg); + KBASE_DEBUG_ASSERT(vsize > 0); + + /* validate user provided arguments */ + if (size > vsize || vsize > reg->nr_pages) + goto out_term; + + /* Prevent vsize*sizeof from wrapping around. + * For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail. + */ + if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages))) + goto out_term; + + KBASE_DEBUG_ASSERT(0 != vsize); + + if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0) + goto out_term; + + reg->cpu_alloc->reg = reg; + if (reg->cpu_alloc != reg->gpu_alloc) { + if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0) + goto out_rollback; + reg->gpu_alloc->reg = reg; + } + + return 0; + +out_rollback: + kbase_free_phy_pages_helper(reg->cpu_alloc, size); +out_term: + return -1; +} + +KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages); + +bool kbase_check_alloc_flags(unsigned long flags) +{ + /* Only known input flags should be set. */ + if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) + return false; + + /* At least one flag should be set */ + if (flags == 0) + return false; + + /* Either the GPU or CPU must be reading from the allocated memory */ + if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0) + return false; + + /* Either the GPU or CPU must be writing to the allocated memory */ + if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0) + return false; + + /* GPU cannot be writing to GPU executable memory and cannot grow the memory on page fault. */ + if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) + return false; + + /* GPU should have at least read or write access otherwise there is no + reason for allocating. */ + if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) + return false; + + /* BASE_MEM_IMPORT_SHARED is only valid for imported memory */ + if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED) + return false; + + return true; +} + +bool kbase_check_import_flags(unsigned long flags) +{ + /* Only known input flags should be set. */ + if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) + return false; + + /* At least one flag should be set */ + if (flags == 0) + return false; + + /* Imported memory cannot be GPU executable */ + if (flags & BASE_MEM_PROT_GPU_EX) + return false; + + /* Imported memory cannot grow on page fault */ + if (flags & BASE_MEM_GROW_ON_GPF) + return false; + + /* GPU should have at least read or write access otherwise there is no + reason for importing. */ + if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) + return false; + + /* Secure memory cannot be read by the CPU */ + if ((flags & BASE_MEM_SECURE) && (flags & BASE_MEM_PROT_CPU_RD)) + return false; + + return true; +} + +/** + * @brief Acquire the per-context region list lock + */ +void kbase_gpu_vm_lock(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx != NULL); + mutex_lock(&kctx->reg_lock); +} + +KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); + +/** + * @brief Release the per-context region list lock + */ +void kbase_gpu_vm_unlock(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(kctx != NULL); + mutex_unlock(&kctx->reg_lock); +} + +KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); + +#ifdef CONFIG_DEBUG_FS +struct kbase_jit_debugfs_data { + int (*func)(struct kbase_jit_debugfs_data *); + struct mutex lock; + struct kbase_context *kctx; + u64 active_value; + u64 pool_value; + u64 destroy_value; + char buffer[50]; +}; + +static int kbase_jit_debugfs_common_open(struct inode *inode, + struct file *file, int (*func)(struct kbase_jit_debugfs_data *)) +{ + struct kbase_jit_debugfs_data *data; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->func = func; + mutex_init(&data->lock); + data->kctx = (struct kbase_context *) inode->i_private; + + file->private_data = data; + + return nonseekable_open(inode, file); +} + +static ssize_t kbase_jit_debugfs_common_read(struct file *file, + char __user *buf, size_t len, loff_t *ppos) +{ + struct kbase_jit_debugfs_data *data; + size_t size; + int ret; + + data = (struct kbase_jit_debugfs_data *) file->private_data; + mutex_lock(&data->lock); + + if (*ppos) { + size = strnlen(data->buffer, sizeof(data->buffer)); + } else { + if (!data->func) { + ret = -EACCES; + goto out_unlock; + } + + if (data->func(data)) { + ret = -EACCES; + goto out_unlock; + } + + size = scnprintf(data->buffer, sizeof(data->buffer), + "%llu,%llu,%llu", data->active_value, + data->pool_value, data->destroy_value); + } + + ret = simple_read_from_buffer(buf, len, ppos, data->buffer, size); + +out_unlock: + mutex_unlock(&data->lock); + return ret; +} + +static int kbase_jit_debugfs_common_release(struct inode *inode, + struct file *file) +{ + kfree(file->private_data); + return 0; +} + +#define KBASE_JIT_DEBUGFS_DECLARE(__fops, __func) \ +static int __fops ## _open(struct inode *inode, struct file *file) \ +{ \ + return kbase_jit_debugfs_common_open(inode, file, __func); \ +} \ +static const struct file_operations __fops = { \ + .owner = THIS_MODULE, \ + .open = __fops ## _open, \ + .release = kbase_jit_debugfs_common_release, \ + .read = kbase_jit_debugfs_common_read, \ + .write = NULL, \ + .llseek = generic_file_llseek, \ +} + +static int kbase_jit_debugfs_count_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct list_head *tmp; + + mutex_lock(&kctx->jit_evict_lock); + list_for_each(tmp, &kctx->jit_active_head) { + data->active_value++; + } + + list_for_each(tmp, &kctx->jit_pool_head) { + data->pool_value++; + } + + list_for_each(tmp, &kctx->jit_destroy_head) { + data->destroy_value++; + } + mutex_unlock(&kctx->jit_evict_lock); + + return 0; +} +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_count_fops, + kbase_jit_debugfs_count_get); + +static int kbase_jit_debugfs_vm_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct kbase_va_region *reg; + + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { + data->active_value += reg->nr_pages; + } + + list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { + data->pool_value += reg->nr_pages; + } + + list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { + data->destroy_value += reg->nr_pages; + } + mutex_unlock(&kctx->jit_evict_lock); + + return 0; +} +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_vm_fops, + kbase_jit_debugfs_vm_get); + +static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data) +{ + struct kbase_context *kctx = data->kctx; + struct kbase_va_region *reg; + + mutex_lock(&kctx->jit_evict_lock); + list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { + data->active_value += reg->gpu_alloc->nents; + } + + list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { + data->pool_value += reg->gpu_alloc->nents; + } + + list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { + data->destroy_value += reg->gpu_alloc->nents; + } + mutex_unlock(&kctx->jit_evict_lock); + + return 0; +} +KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops, + kbase_jit_debugfs_phys_get); + +void kbase_jit_debugfs_init(struct kbase_context *kctx) +{ + /* Debugfs entry for getting the number of JIT allocations. */ + debugfs_create_file("mem_jit_count", S_IRUGO, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_count_fops); + + /* + * Debugfs entry for getting the total number of virtual pages + * used by JIT allocations. + */ + debugfs_create_file("mem_jit_vm", S_IRUGO, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_vm_fops); + + /* + * Debugfs entry for getting the number of physical pages used + * by JIT allocations. + */ + debugfs_create_file("mem_jit_phys", S_IRUGO, kctx->kctx_dentry, + kctx, &kbase_jit_debugfs_phys_fops); +} +#endif /* CONFIG_DEBUG_FS */ + +/** + * kbase_jit_destroy_worker - Deferred worker which frees JIT allocations + * @work: Work item + * + * This function does the work of freeing JIT allocations whose physical + * backing has been released. + */ +static void kbase_jit_destroy_worker(struct work_struct *work) +{ + struct kbase_context *kctx; + struct kbase_va_region *reg; + + kctx = container_of(work, struct kbase_context, jit_work); + do { + mutex_lock(&kctx->jit_evict_lock); + if (list_empty(&kctx->jit_destroy_head)) { + mutex_unlock(&kctx->jit_evict_lock); + break; + } + + reg = list_first_entry(&kctx->jit_destroy_head, + struct kbase_va_region, jit_node); + + list_del(®->jit_node); + mutex_unlock(&kctx->jit_evict_lock); + + kbase_gpu_vm_lock(kctx); + kbase_mem_free_region(kctx, reg); + kbase_gpu_vm_unlock(kctx); + } while (1); +} + +int kbase_jit_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->jit_active_head); + INIT_LIST_HEAD(&kctx->jit_pool_head); + INIT_LIST_HEAD(&kctx->jit_destroy_head); + INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); + + INIT_LIST_HEAD(&kctx->jit_pending_alloc); + INIT_LIST_HEAD(&kctx->jit_atoms_head); + + return 0; +} + +struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, + struct base_jit_alloc_info *info) +{ + struct kbase_va_region *reg = NULL; + struct kbase_va_region *walker; + struct kbase_va_region *temp; + size_t current_diff = SIZE_MAX; + + int ret; + + mutex_lock(&kctx->jit_evict_lock); + /* + * Scan the pool for an existing allocation which meets our + * requirements and remove it. + */ + list_for_each_entry_safe(walker, temp, &kctx->jit_pool_head, jit_node) { + + if (walker->nr_pages >= info->va_pages) { + size_t min_size, max_size, diff; + + /* + * The JIT allocations VA requirements have been + * meet, it's suitable but other allocations + * might be a better fit. + */ + min_size = min_t(size_t, walker->gpu_alloc->nents, + info->commit_pages); + max_size = max_t(size_t, walker->gpu_alloc->nents, + info->commit_pages); + diff = max_size - min_size; + + if (current_diff > diff) { + current_diff = diff; + reg = walker; + } + + /* The allocation is an exact match, stop looking */ + if (current_diff == 0) + break; + } + } + + if (reg) { + /* + * Remove the found region from the pool and add it to the + * active list. + */ + list_move(®->jit_node, &kctx->jit_active_head); + + /* + * Remove the allocation from the eviction list as it's no + * longer eligible for eviction. This must be done before + * dropping the jit_evict_lock + */ + list_del_init(®->gpu_alloc->evict_node); + mutex_unlock(&kctx->jit_evict_lock); + + kbase_gpu_vm_lock(kctx); + + /* Make the physical backing no longer reclaimable */ + if (!kbase_mem_evictable_unmake(reg->gpu_alloc)) + goto update_failed; + + /* Grow the backing if required */ + if (reg->gpu_alloc->nents < info->commit_pages) { + size_t delta; + size_t old_size = reg->gpu_alloc->nents; + + /* Allocate some more pages */ + delta = info->commit_pages - reg->gpu_alloc->nents; + if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, delta) + != 0) + goto update_failed; + + if (reg->cpu_alloc != reg->gpu_alloc) { + if (kbase_alloc_phy_pages_helper( + reg->cpu_alloc, delta) != 0) { + kbase_free_phy_pages_helper( + reg->gpu_alloc, delta); + goto update_failed; + } + } + + ret = kbase_mem_grow_gpu_mapping(kctx, reg, + info->commit_pages, old_size); + /* + * The grow failed so put the allocation back in the + * pool and return failure. + */ + if (ret) + goto update_failed; + } + kbase_gpu_vm_unlock(kctx); + } else { + /* No suitable JIT allocation was found so create a new one */ + u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | + BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF | + BASE_MEM_COHERENT_LOCAL; + u64 gpu_addr; + + mutex_unlock(&kctx->jit_evict_lock); + + reg = kbase_mem_alloc(kctx, info->va_pages, info->commit_pages, + info->extent, &flags, &gpu_addr); + if (!reg) + goto out_unlocked; + + mutex_lock(&kctx->jit_evict_lock); + list_add(®->jit_node, &kctx->jit_active_head); + mutex_unlock(&kctx->jit_evict_lock); + } + + return reg; + +update_failed: + /* + * An update to an allocation from the pool failed, chances + * are slim a new allocation would fair any better so return + * the allocation to the pool and return the function with failure. + */ + kbase_gpu_vm_unlock(kctx); + mutex_lock(&kctx->jit_evict_lock); + list_move(®->jit_node, &kctx->jit_pool_head); + mutex_unlock(&kctx->jit_evict_lock); +out_unlocked: + return NULL; +} + +void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) +{ + /* The physical backing of memory in the pool is always reclaimable */ + kbase_gpu_vm_lock(kctx); + kbase_mem_evictable_make(reg->gpu_alloc); + kbase_gpu_vm_unlock(kctx); + + mutex_lock(&kctx->jit_evict_lock); + list_move(®->jit_node, &kctx->jit_pool_head); + mutex_unlock(&kctx->jit_evict_lock); +} + +void kbase_jit_backing_lost(struct kbase_va_region *reg) +{ + struct kbase_context *kctx = reg->kctx; + + lockdep_assert_held(&kctx->jit_evict_lock); + + /* + * JIT allocations will always be on a list, if the region + * is not on a list then it's not a JIT allocation. + */ + if (list_empty(®->jit_node)) + return; + + /* + * Freeing the allocation requires locks we might not be able + * to take now, so move the allocation to the free list and kick + * the worker which will do the freeing. + */ + list_move(®->jit_node, &kctx->jit_destroy_head); + + schedule_work(&kctx->jit_work); +} + +bool kbase_jit_evict(struct kbase_context *kctx) +{ + struct kbase_va_region *reg = NULL; + + lockdep_assert_held(&kctx->reg_lock); + + /* Free the oldest allocation from the pool */ + mutex_lock(&kctx->jit_evict_lock); + if (!list_empty(&kctx->jit_pool_head)) { + reg = list_entry(kctx->jit_pool_head.prev, + struct kbase_va_region, jit_node); + list_del(®->jit_node); + } + mutex_unlock(&kctx->jit_evict_lock); + + if (reg) + kbase_mem_free_region(kctx, reg); + + return (reg != NULL); +} + +void kbase_jit_term(struct kbase_context *kctx) +{ + struct kbase_va_region *walker; + + /* Free all allocations for this context */ + + /* + * Flush the freeing of allocations whose backing has been freed + * (i.e. everything in jit_destroy_head). + */ + cancel_work_sync(&kctx->jit_work); + + kbase_gpu_vm_lock(kctx); + mutex_lock(&kctx->jit_evict_lock); + /* Free all allocations from the pool */ + while (!list_empty(&kctx->jit_pool_head)) { + walker = list_first_entry(&kctx->jit_pool_head, + struct kbase_va_region, jit_node); + list_del(&walker->jit_node); + mutex_unlock(&kctx->jit_evict_lock); + kbase_mem_free_region(kctx, walker); + mutex_lock(&kctx->jit_evict_lock); + } + + /* Free all allocations from active list */ + while (!list_empty(&kctx->jit_active_head)) { + walker = list_first_entry(&kctx->jit_active_head, + struct kbase_va_region, jit_node); + list_del(&walker->jit_node); + mutex_unlock(&kctx->jit_evict_lock); + kbase_mem_free_region(kctx, walker); + mutex_lock(&kctx->jit_evict_lock); + } + mutex_unlock(&kctx->jit_evict_lock); + kbase_gpu_vm_unlock(kctx); +} + +static int kbase_jd_user_buf_map(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + long pinned_pages; + struct kbase_mem_phy_alloc *alloc; + struct page **pages; + phys_addr_t *pa; + long i; + int err = -ENOMEM; + unsigned long address; + struct mm_struct *mm; + struct device *dev; + unsigned long offset; + unsigned long local_size; + + alloc = reg->gpu_alloc; + pa = kbase_get_gpu_phy_pages(reg); + address = alloc->imported.user_buf.address; + mm = alloc->imported.user_buf.mm; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); + + pages = alloc->imported.user_buf.pages; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + pinned_pages = get_user_pages(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR, + 0, pages, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + pinned_pages = get_user_pages_remote(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR, + 0, pages, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) + pinned_pages = get_user_pages_remote(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) + pinned_pages = get_user_pages_remote(NULL, mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL, NULL); +#else + pinned_pages = get_user_pages_remote(mm, + address, + alloc->imported.user_buf.nr_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL, NULL); +#endif + + if (pinned_pages <= 0) + return pinned_pages; + + if (pinned_pages != alloc->imported.user_buf.nr_pages) { + for (i = 0; i < pinned_pages; i++) + put_page(pages[i]); + return -ENOMEM; + } + + dev = kctx->kbdev->dev; + offset = address & ~PAGE_MASK; + local_size = alloc->imported.user_buf.size; + + for (i = 0; i < pinned_pages; i++) { + dma_addr_t dma_addr; + unsigned long min; + + min = MIN(PAGE_SIZE - offset, local_size); + dma_addr = dma_map_page(dev, pages[i], + offset, min, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) + goto unwind; + + alloc->imported.user_buf.dma_addrs[i] = dma_addr; + pa[i] = page_to_phys(pages[i]); + + local_size -= min; + offset = 0; + } + + alloc->nents = pinned_pages; + + err = kbase_mmu_insert_pages(kctx, reg->start_pfn, pa, + kbase_reg_current_backed_size(reg), + reg->flags); + if (err == 0) + return 0; + + alloc->nents = 0; + /* fall down */ +unwind: + while (i--) { + dma_unmap_page(kctx->kbdev->dev, + alloc->imported.user_buf.dma_addrs[i], + PAGE_SIZE, DMA_BIDIRECTIONAL); + put_page(pages[i]); + pages[i] = NULL; + } + + return err; +} + +static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc, bool writeable) +{ + long i; + struct page **pages; + unsigned long size = alloc->imported.user_buf.size; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); + pages = alloc->imported.user_buf.pages; + for (i = 0; i < alloc->imported.user_buf.nr_pages; i++) { + unsigned long local_size; + dma_addr_t dma_addr = alloc->imported.user_buf.dma_addrs[i]; + + local_size = MIN(size, PAGE_SIZE - (dma_addr & ~PAGE_MASK)); + dma_unmap_page(kctx->kbdev->dev, dma_addr, local_size, + DMA_BIDIRECTIONAL); + if (writeable) + set_page_dirty_lock(pages[i]); + put_page(pages[i]); + pages[i] = NULL; + + size -= local_size; + } + alloc->nents = 0; +} + + +/* to replace sg_dma_len. */ +#define MALI_SG_DMA_LEN(sg) ((sg)->length) + +#ifdef CONFIG_DMA_SHARED_BUFFER +static int kbase_jd_umm_map(struct kbase_context *kctx, + struct kbase_va_region *reg) +{ + struct sg_table *sgt; + struct scatterlist *s; + int i; + phys_addr_t *pa; + int err; + size_t count = 0; + struct kbase_mem_phy_alloc *alloc; + + alloc = reg->gpu_alloc; + + KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM); + KBASE_DEBUG_ASSERT(NULL == alloc->imported.umm.sgt); + sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, + DMA_BIDIRECTIONAL); + + if (IS_ERR_OR_NULL(sgt)) + return -EINVAL; + + /* save for later */ + alloc->imported.umm.sgt = sgt; + + pa = kbase_get_gpu_phy_pages(reg); + KBASE_DEBUG_ASSERT(pa); + + for_each_sg(sgt->sgl, s, sgt->nents, i) { + int j; + size_t pages = PFN_UP(MALI_SG_DMA_LEN(s)); + + WARN_ONCE(MALI_SG_DMA_LEN(s) & (PAGE_SIZE-1), + "MALI_SG_DMA_LEN(s)=%u is not a multiple of PAGE_SIZE\n", + MALI_SG_DMA_LEN(s)); + + WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE-1), + "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n", + (unsigned long long) sg_dma_address(s)); + + for (j = 0; (j < pages) && (count < reg->nr_pages); j++, + count++) + *pa++ = sg_dma_address(s) + (j << PAGE_SHIFT); + WARN_ONCE(j < pages, + "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n", + alloc->imported.umm.dma_buf->size); + } + + if (!(reg->flags & KBASE_REG_IMPORT_PAD) && + WARN_ONCE(count < reg->nr_pages, + "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n", + alloc->imported.umm.dma_buf->size)) { + err = -EINVAL; + goto err_unmap_attachment; + } + + /* Update nents as we now have pages to map */ + alloc->nents = reg->nr_pages; + + err = kbase_mmu_insert_pages(kctx, reg->start_pfn, + kbase_get_gpu_phy_pages(reg), + count, + reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD); + if (err) + goto err_unmap_attachment; + + if (reg->flags & KBASE_REG_IMPORT_PAD) { + err = kbase_mmu_insert_single_page(kctx, + reg->start_pfn + count, + page_to_phys(kctx->aliasing_sink_page), + reg->nr_pages - count, + (reg->flags | KBASE_REG_GPU_RD) & + ~KBASE_REG_GPU_WR); + if (err) + goto err_teardown_orig_pages; + } + + return 0; + +err_teardown_orig_pages: + kbase_mmu_teardown_pages(kctx, reg->start_pfn, count); +err_unmap_attachment: + dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, + alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); + alloc->imported.umm.sgt = NULL; + + return err; +} + +static void kbase_jd_umm_unmap(struct kbase_context *kctx, + struct kbase_mem_phy_alloc *alloc) +{ + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(alloc); + KBASE_DEBUG_ASSERT(alloc->imported.umm.dma_attachment); + KBASE_DEBUG_ASSERT(alloc->imported.umm.sgt); + dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, + alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); + alloc->imported.umm.sgt = NULL; + alloc->nents = 0; +} +#endif /* CONFIG_DMA_SHARED_BUFFER */ + +#if (defined(CONFIG_KDS) && defined(CONFIG_UMP)) \ + || defined(CONFIG_DMA_SHARED_BUFFER_USES_KDS) +static void add_kds_resource(struct kds_resource *kds_res, + struct kds_resource **kds_resources, u32 *kds_res_count, + unsigned long *kds_access_bitmap, bool exclusive) +{ + u32 i; + + for (i = 0; i < *kds_res_count; i++) { + /* Duplicate resource, ignore */ + if (kds_resources[i] == kds_res) + return; + } + + kds_resources[*kds_res_count] = kds_res; + if (exclusive) + set_bit(*kds_res_count, kds_access_bitmap); + (*kds_res_count)++; +} +#endif + +struct kbase_mem_phy_alloc *kbase_map_external_resource( + struct kbase_context *kctx, struct kbase_va_region *reg, + struct mm_struct *locked_mm +#ifdef CONFIG_KDS + , u32 *kds_res_count, struct kds_resource **kds_resources, + unsigned long *kds_access_bitmap, bool exclusive +#endif + ) +{ + int err; + + /* decide what needs to happen for this resource */ + switch (reg->gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { + if (reg->gpu_alloc->imported.user_buf.mm != locked_mm) + goto exit; + + reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++; + if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) { + err = kbase_jd_user_buf_map(kctx, reg); + if (err) { + reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--; + goto exit; + } + } + } + break; + case KBASE_MEM_TYPE_IMPORTED_UMP: { +#if defined(CONFIG_KDS) && defined(CONFIG_UMP) + if (kds_res_count) { + struct kds_resource *kds_res; + + kds_res = ump_dd_kds_resource_get( + reg->gpu_alloc->imported.ump_handle); + if (kds_res) + add_kds_resource(kds_res, kds_resources, + kds_res_count, + kds_access_bitmap, exclusive); + } +#endif /*defined(CONFIG_KDS) && defined(CONFIG_UMP) */ + break; + } +#ifdef CONFIG_DMA_SHARED_BUFFER + case KBASE_MEM_TYPE_IMPORTED_UMM: { +#ifdef CONFIG_DMA_SHARED_BUFFER_USES_KDS + if (kds_res_count) { + struct kds_resource *kds_res; + + kds_res = get_dma_buf_kds_resource( + reg->gpu_alloc->imported.umm.dma_buf); + if (kds_res) + add_kds_resource(kds_res, kds_resources, + kds_res_count, + kds_access_bitmap, exclusive); + } +#endif + reg->gpu_alloc->imported.umm.current_mapping_usage_count++; + if (1 == reg->gpu_alloc->imported.umm.current_mapping_usage_count) { + err = kbase_jd_umm_map(kctx, reg); + if (err) { + reg->gpu_alloc->imported.umm.current_mapping_usage_count--; + goto exit; + } + } + break; + } +#endif + default: + goto exit; + } + + return kbase_mem_phy_alloc_get(reg->gpu_alloc); +exit: + return NULL; +} + +void kbase_unmap_external_resource(struct kbase_context *kctx, + struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc) +{ + switch (alloc->type) { +#ifdef CONFIG_DMA_SHARED_BUFFER + case KBASE_MEM_TYPE_IMPORTED_UMM: { + alloc->imported.umm.current_mapping_usage_count--; + + if (0 == alloc->imported.umm.current_mapping_usage_count) { + if (reg && reg->gpu_alloc == alloc) { + int err; + + err = kbase_mmu_teardown_pages( + kctx, + reg->start_pfn, + alloc->nents); + WARN_ON(err); + } + + kbase_jd_umm_unmap(kctx, alloc); + } + } + break; +#endif /* CONFIG_DMA_SHARED_BUFFER */ + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { + alloc->imported.user_buf.current_mapping_usage_count--; + + if (0 == alloc->imported.user_buf.current_mapping_usage_count) { + bool writeable = true; + + if (reg && reg->gpu_alloc == alloc) + kbase_mmu_teardown_pages( + kctx, + reg->start_pfn, + kbase_reg_current_backed_size(reg)); + + if (reg && ((reg->flags & KBASE_REG_GPU_WR) == 0)) + writeable = false; + + kbase_jd_user_buf_unmap(kctx, alloc, writeable); + } + } + break; + default: + break; + } + kbase_mem_phy_alloc_put(alloc); +} + +struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( + struct kbase_context *kctx, u64 gpu_addr) +{ + struct kbase_ctx_ext_res_meta *meta = NULL; + struct kbase_ctx_ext_res_meta *walker; + + lockdep_assert_held(&kctx->reg_lock); + + /* + * Walk the per context external resource metadata list for the + * metadata which matches the region which is being acquired. + */ + list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) { + if (walker->gpu_addr == gpu_addr) { + meta = walker; + break; + } + } + + /* No metadata exists so create one. */ + if (!meta) { + struct kbase_va_region *reg; + + /* Find the region */ + reg = kbase_region_tracker_find_region_enclosing_address( + kctx, gpu_addr); + if (NULL == reg || (reg->flags & KBASE_REG_FREE)) + goto failed; + + /* Allocate the metadata object */ + meta = kzalloc(sizeof(*meta), GFP_KERNEL); + if (!meta) + goto failed; + + /* + * Fill in the metadata object and acquire a reference + * for the physical resource. + */ + meta->alloc = kbase_map_external_resource(kctx, reg, NULL +#ifdef CONFIG_KDS + , NULL, NULL, + NULL, false +#endif + ); + + if (!meta->alloc) + goto fail_map; + + meta->gpu_addr = reg->start_pfn << PAGE_SHIFT; + + list_add(&meta->ext_res_node, &kctx->ext_res_meta_head); + } + + return meta; + +fail_map: + kfree(meta); +failed: + return NULL; +} + +bool kbase_sticky_resource_release(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) +{ + struct kbase_ctx_ext_res_meta *walker; + struct kbase_va_region *reg; + + lockdep_assert_held(&kctx->reg_lock); + + /* Search of the metadata if one isn't provided. */ + if (!meta) { + /* + * Walk the per context external resource metadata list for the + * metadata which matches the region which is being released. + */ + list_for_each_entry(walker, &kctx->ext_res_meta_head, + ext_res_node) { + if (walker->gpu_addr == gpu_addr) { + meta = walker; + break; + } + } + } + + /* No metadata so just return. */ + if (!meta) + return false; + + /* Drop the physical memory reference and free the metadata. */ + reg = kbase_region_tracker_find_region_enclosing_address( + kctx, + meta->gpu_addr); + + kbase_unmap_external_resource(kctx, reg, meta->alloc); + list_del(&meta->ext_res_node); + kfree(meta); + + return true; +} + +int kbase_sticky_resource_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->ext_res_meta_head); + + return 0; +} + +void kbase_sticky_resource_term(struct kbase_context *kctx) +{ + struct kbase_ctx_ext_res_meta *walker; + + lockdep_assert_held(&kctx->reg_lock); + + /* + * Free any sticky resources which haven't been unmapped. + * + * Note: + * We don't care about refcounts at this point as no future + * references to the meta data will be made. + * Region termination would find these if we didn't free them + * here, but it's more efficient if we do the clean up here. + */ + while (!list_empty(&kctx->ext_res_meta_head)) { + walker = list_first_entry(&kctx->ext_res_meta_head, + struct kbase_ctx_ext_res_meta, ext_res_node); + + kbase_sticky_resource_release(kctx, walker, 0); + } +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem.h b/drivers/gpu/arm/midgard/mali_kbase_mem.h new file mode 100755 index 000000000000..3f3eaa3fda98 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem.h @@ -0,0 +1,1068 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mem.h + * Base kernel memory APIs + */ + +#ifndef _KBASE_MEM_H_ +#define _KBASE_MEM_H_ + +#ifndef _KBASE_H_ +#error "Don't include this file directly, use mali_kbase.h instead" +#endif + +#include +#ifdef CONFIG_KDS +#include +#endif /* CONFIG_KDS */ +#ifdef CONFIG_UMP +#include +#endif /* CONFIG_UMP */ +#include "mali_base_kernel.h" +#include +#include "mali_kbase_pm.h" +#include "mali_kbase_defs.h" +#if defined(CONFIG_MALI_GATOR_SUPPORT) +#include "mali_kbase_gator.h" +#endif +/* Required for kbase_mem_evictable_unmake */ +#include "mali_kbase_mem_linux.h" + +/* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */ +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */ + +/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages. +The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and +page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table +updates and generates duplicate page faults as the page table information used by the MMU is not valid. */ +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */ + +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */ + +/* This must always be a power of 2 */ +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2) +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316) +#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630) +/** + * A CPU mapping + */ +struct kbase_cpu_mapping { + struct list_head mappings_list; + struct kbase_mem_phy_alloc *alloc; + struct kbase_context *kctx; + struct kbase_va_region *region; + int count; + int free_on_close; +}; + +enum kbase_memory_type { + KBASE_MEM_TYPE_NATIVE, + KBASE_MEM_TYPE_IMPORTED_UMP, + KBASE_MEM_TYPE_IMPORTED_UMM, + KBASE_MEM_TYPE_IMPORTED_USER_BUF, + KBASE_MEM_TYPE_ALIAS, + KBASE_MEM_TYPE_TB, + KBASE_MEM_TYPE_RAW +}; + +/* internal structure, mirroring base_mem_aliasing_info, + * but with alloc instead of a gpu va (handle) */ +struct kbase_aliased { + struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */ + u64 offset; /* in pages */ + u64 length; /* in pages */ +}; + +/** + * @brief Physical pages tracking object properties + */ +#define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED (1ul << 0) +#define KBASE_MEM_PHY_ALLOC_LARGE (1ul << 1) + +/* physical pages tracking object. + * Set up to track N pages. + * N not stored here, the creator holds that info. + * This object only tracks how many elements are actually valid (present). + * Changing of nents or *pages should only happen if the kbase_mem_phy_alloc is not + * shared with another region or client. CPU mappings are OK to exist when changing, as + * long as the tracked mappings objects are updated as part of the change. + */ +struct kbase_mem_phy_alloc { + struct kref kref; /* number of users of this alloc */ + atomic_t gpu_mappings; + size_t nents; /* 0..N */ + phys_addr_t *pages; /* N elements, only 0..nents are valid */ + + /* kbase_cpu_mappings */ + struct list_head mappings; + + /* Node used to store this allocation on the eviction list */ + struct list_head evict_node; + /* Physical backing size when the pages where evicted */ + size_t evicted; + /* + * Back reference to the region structure which created this + * allocation, or NULL if it has been freed. + */ + struct kbase_va_region *reg; + + /* type of buffer */ + enum kbase_memory_type type; + + unsigned long properties; + + /* member in union valid based on @a type */ + union { +#ifdef CONFIG_UMP + ump_dd_handle ump_handle; +#endif /* CONFIG_UMP */ +#if defined(CONFIG_DMA_SHARED_BUFFER) + struct { + struct dma_buf *dma_buf; + struct dma_buf_attachment *dma_attachment; + unsigned int current_mapping_usage_count; + struct sg_table *sgt; + } umm; +#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ + struct { + u64 stride; + size_t nents; + struct kbase_aliased *aliased; + } alias; + /* Used by type = (KBASE_MEM_TYPE_NATIVE, KBASE_MEM_TYPE_TB) */ + struct kbase_context *kctx; + struct kbase_alloc_import_user_buf { + unsigned long address; + unsigned long size; + unsigned long nr_pages; + struct page **pages; + /* top bit (1<<31) of current_mapping_usage_count + * specifies that this import was pinned on import + * See PINNED_ON_IMPORT + */ + u32 current_mapping_usage_count; + struct mm_struct *mm; + dma_addr_t *dma_addrs; + } user_buf; + } imported; +}; + +/* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is + * used to signify that a buffer was pinned when it was imported. Since the + * reference count is limited by the number of atoms that can be submitted at + * once there should be no danger of overflowing into this bit. + * Stealing the top bit also has the benefit that + * current_mapping_usage_count != 0 if and only if the buffer is mapped. + */ +#define PINNED_ON_IMPORT (1<<31) + +static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) +{ + KBASE_DEBUG_ASSERT(alloc); + /* we only track mappings of NATIVE buffers */ + if (alloc->type == KBASE_MEM_TYPE_NATIVE) + atomic_inc(&alloc->gpu_mappings); +} + +static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc *alloc) +{ + KBASE_DEBUG_ASSERT(alloc); + /* we only track mappings of NATIVE buffers */ + if (alloc->type == KBASE_MEM_TYPE_NATIVE) + if (0 > atomic_dec_return(&alloc->gpu_mappings)) { + pr_err("Mismatched %s:\n", __func__); + dump_stack(); + } +} + +void kbase_mem_kref_free(struct kref *kref); + +int kbase_mem_init(struct kbase_device *kbdev); +void kbase_mem_halt(struct kbase_device *kbdev); +void kbase_mem_term(struct kbase_device *kbdev); + +static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc) +{ + kref_get(&alloc->kref); + return alloc; +} + +static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc) +{ + kref_put(&alloc->kref, kbase_mem_kref_free); + return NULL; +} + +/** + * A GPU memory region, and attributes for CPU mappings. + */ +struct kbase_va_region { + struct rb_node rblink; + struct list_head link; + + struct kbase_context *kctx; /* Backlink to base context */ + + u64 start_pfn; /* The PFN in GPU space */ + size_t nr_pages; + +/* Free region */ +#define KBASE_REG_FREE (1ul << 0) +/* CPU write access */ +#define KBASE_REG_CPU_WR (1ul << 1) +/* GPU write access */ +#define KBASE_REG_GPU_WR (1ul << 2) +/* No eXecute flag */ +#define KBASE_REG_GPU_NX (1ul << 3) +/* Is CPU cached? */ +#define KBASE_REG_CPU_CACHED (1ul << 4) +/* Is GPU cached? */ +#define KBASE_REG_GPU_CACHED (1ul << 5) + +#define KBASE_REG_GROWABLE (1ul << 6) +/* Can grow on pf? */ +#define KBASE_REG_PF_GROW (1ul << 7) + +/* VA managed by us */ +#define KBASE_REG_CUSTOM_VA (1ul << 8) + +/* inner shareable coherency */ +#define KBASE_REG_SHARE_IN (1ul << 9) +/* inner & outer shareable coherency */ +#define KBASE_REG_SHARE_BOTH (1ul << 10) + +/* Space for 4 different zones */ +#define KBASE_REG_ZONE_MASK (3ul << 11) +#define KBASE_REG_ZONE(x) (((x) & 3) << 11) + +/* GPU read access */ +#define KBASE_REG_GPU_RD (1ul<<13) +/* CPU read access */ +#define KBASE_REG_CPU_RD (1ul<<14) + +/* Index of chosen MEMATTR for this region (0..7) */ +#define KBASE_REG_MEMATTR_MASK (7ul << 16) +#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16) +#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16) + +#define KBASE_REG_SECURE (1ul << 19) + +#define KBASE_REG_DONT_NEED (1ul << 20) + +/* Imported buffer is padded? */ +#define KBASE_REG_IMPORT_PAD (1ul << 21) + +#define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0) + +/* only used with 32-bit clients */ +/* + * On a 32bit platform, custom VA should be wired from (4GB + shader region) + * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface + * limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference). + * So we put the default limit to the maximum possible on Linux and shrink + * it down, if required by the GPU, during initialization. + */ + +/* + * Dedicated 16MB region for shader code: + * VA range 0x101000000-0x102000000 + */ +#define KBASE_REG_ZONE_EXEC KBASE_REG_ZONE(1) +#define KBASE_REG_ZONE_EXEC_BASE (0x101000000ULL >> PAGE_SHIFT) +#define KBASE_REG_ZONE_EXEC_SIZE ((16ULL * 1024 * 1024) >> PAGE_SHIFT) + +#define KBASE_REG_ZONE_CUSTOM_VA KBASE_REG_ZONE(2) +#define KBASE_REG_ZONE_CUSTOM_VA_BASE (KBASE_REG_ZONE_EXEC_BASE + KBASE_REG_ZONE_EXEC_SIZE) /* Starting after KBASE_REG_ZONE_EXEC */ +#define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) +/* end 32-bit clients only */ + + unsigned long flags; + + size_t extent; /* nr of pages alloc'd on PF */ + + struct kbase_mem_phy_alloc *cpu_alloc; /* the one alloc object we mmap to the CPU when mapping this region */ + struct kbase_mem_phy_alloc *gpu_alloc; /* the one alloc object we mmap to the GPU when mapping this region */ + + /* non-NULL if this memory object is a kds_resource */ + struct kds_resource *kds_res; + + /* List head used to store the region in the JIT allocation pool */ + struct list_head jit_node; +}; + +/* Common functions */ +static inline phys_addr_t *kbase_get_cpu_phy_pages(struct kbase_va_region *reg) +{ + KBASE_DEBUG_ASSERT(reg); + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); + + return reg->cpu_alloc->pages; +} + +static inline phys_addr_t *kbase_get_gpu_phy_pages(struct kbase_va_region *reg) +{ + KBASE_DEBUG_ASSERT(reg); + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); + + return reg->gpu_alloc->pages; +} + +static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg) +{ + KBASE_DEBUG_ASSERT(reg); + /* if no alloc object the backed size naturally is 0 */ + if (!reg->cpu_alloc) + return 0; + + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); + + return reg->cpu_alloc->nents; +} + +#define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */ + +static inline struct kbase_mem_phy_alloc *kbase_alloc_create(size_t nr_pages, enum kbase_memory_type type) +{ + struct kbase_mem_phy_alloc *alloc; + size_t alloc_size = sizeof(*alloc) + sizeof(*alloc->pages) * nr_pages; + size_t per_page_size = sizeof(*alloc->pages); + + /* Imported pages may have page private data already in use */ + if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { + alloc_size += nr_pages * + sizeof(*alloc->imported.user_buf.dma_addrs); + per_page_size += sizeof(*alloc->imported.user_buf.dma_addrs); + } + + /* + * Prevent nr_pages*per_page_size + sizeof(*alloc) from + * wrapping around. + */ + if (nr_pages > ((((size_t) -1) - sizeof(*alloc)) + / per_page_size)) + return ERR_PTR(-ENOMEM); + + /* Allocate based on the size to reduce internal fragmentation of vmem */ + if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) + alloc = vzalloc(alloc_size); + else + alloc = kzalloc(alloc_size, GFP_KERNEL); + + if (!alloc) + return ERR_PTR(-ENOMEM); + + /* Store allocation method */ + if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) + alloc->properties |= KBASE_MEM_PHY_ALLOC_LARGE; + + kref_init(&alloc->kref); + atomic_set(&alloc->gpu_mappings, 0); + alloc->nents = 0; + alloc->pages = (void *)(alloc + 1); + INIT_LIST_HEAD(&alloc->mappings); + alloc->type = type; + + if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) + alloc->imported.user_buf.dma_addrs = + (void *) (alloc->pages + nr_pages); + + return alloc; +} + +static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, + struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(reg); + KBASE_DEBUG_ASSERT(!reg->cpu_alloc); + KBASE_DEBUG_ASSERT(!reg->gpu_alloc); + KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE); + + reg->cpu_alloc = kbase_alloc_create(reg->nr_pages, + KBASE_MEM_TYPE_NATIVE); + if (IS_ERR(reg->cpu_alloc)) + return PTR_ERR(reg->cpu_alloc); + else if (!reg->cpu_alloc) + return -ENOMEM; + reg->cpu_alloc->imported.kctx = kctx; + INIT_LIST_HEAD(®->cpu_alloc->evict_node); + if (kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE) + && (reg->flags & KBASE_REG_CPU_CACHED)) { + reg->gpu_alloc = kbase_alloc_create(reg->nr_pages, + KBASE_MEM_TYPE_NATIVE); + reg->gpu_alloc->imported.kctx = kctx; + INIT_LIST_HEAD(®->gpu_alloc->evict_node); + } else { + reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + } + + INIT_LIST_HEAD(®->jit_node); + reg->flags &= ~KBASE_REG_FREE; + return 0; +} + +static inline int kbase_atomic_add_pages(int num_pages, atomic_t *used_pages) +{ + int new_val = atomic_add_return(num_pages, used_pages); +#if defined(CONFIG_MALI_GATOR_SUPPORT) + kbase_trace_mali_total_alloc_pages_change((long long int)new_val); +#endif + return new_val; +} + +static inline int kbase_atomic_sub_pages(int num_pages, atomic_t *used_pages) +{ + int new_val = atomic_sub_return(num_pages, used_pages); +#if defined(CONFIG_MALI_GATOR_SUPPORT) + kbase_trace_mali_total_alloc_pages_change((long long int)new_val); +#endif + return new_val; +} + +/* + * Max size for kbdev memory pool (in pages) + */ +#define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT) + +/* + * Max size for kctx memory pool (in pages) + */ +#define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) + +/** + * kbase_mem_pool_init - Create a memory pool for a kbase device + * @pool: Memory pool to initialize + * @max_size: Maximum number of free pages the pool can hold + * @kbdev: Kbase device where memory is used + * @next_pool: Pointer to the next pool or NULL. + * + * Allocations from @pool are in whole pages. Each @pool has a free list where + * pages can be quickly allocated from. The free list is initially empty and + * filled whenever pages are freed back to the pool. The number of free pages + * in the pool will in general not exceed @max_size, but the pool may in + * certain corner cases grow above @max_size. + * + * If @next_pool is not NULL, we will allocate from @next_pool before going to + * the kernel allocator. Similarily pages can spill over to @next_pool when + * @pool is full. Pages are zeroed before they spill over to another pool, to + * prevent leaking information between applications. + * + * A shrinker is registered so that Linux mm can reclaim pages from the pool as + * needed. + * + * Return: 0 on success, negative -errno on error + */ +int kbase_mem_pool_init(struct kbase_mem_pool *pool, + size_t max_size, + struct kbase_device *kbdev, + struct kbase_mem_pool *next_pool); + +/** + * kbase_mem_pool_term - Destroy a memory pool + * @pool: Memory pool to destroy + * + * Pages in the pool will spill over to @next_pool (if available) or freed to + * the kernel. + */ +void kbase_mem_pool_term(struct kbase_mem_pool *pool); + +/** + * kbase_mem_pool_alloc - Allocate a page from memory pool + * @pool: Memory pool to allocate from + * + * Allocations from the pool are made as follows: + * 1. If there are free pages in the pool, allocate a page from @pool. + * 2. Otherwise, if @next_pool is not NULL and has free pages, allocate a page + * from @next_pool. + * 3. Return NULL if no memory in the pool + * + * Return: Pointer to allocated page, or NULL if allocation failed. + */ +struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool); + +/** + * kbase_mem_pool_free - Free a page to memory pool + * @pool: Memory pool where page should be freed + * @page: Page to free to the pool + * @dirty: Whether some of the page may be dirty in the cache. + * + * Pages are freed to the pool as follows: + * 1. If @pool is not full, add @page to @pool. + * 2. Otherwise, if @next_pool is not NULL and not full, add @page to + * @next_pool. + * 3. Finally, free @page to the kernel. + */ +void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page, + bool dirty); + +/** + * kbase_mem_pool_alloc_pages - Allocate pages from memory pool + * @pool: Memory pool to allocate from + * @nr_pages: Number of pages to allocate + * @pages: Pointer to array where the physical address of the allocated + * pages will be stored. + * + * Like kbase_mem_pool_alloc() but optimized for allocating many pages. + * + * Return: 0 on success, negative -errno on error + */ +int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages, + phys_addr_t *pages); + +/** + * kbase_mem_pool_free_pages - Free pages to memory pool + * @pool: Memory pool where pages should be freed + * @nr_pages: Number of pages to free + * @pages: Pointer to array holding the physical addresses of the pages to + * free. + * @dirty: Whether any pages may be dirty in the cache. + * @reclaimed: Whether the pages where reclaimable and thus should bypass + * the pool and go straight to the kernel. + * + * Like kbase_mem_pool_free() but optimized for freeing many pages. + */ +void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, + phys_addr_t *pages, bool dirty, bool reclaimed); + +/** + * kbase_mem_pool_size - Get number of free pages in memory pool + * @pool: Memory pool to inspect + * + * Note: the size of the pool may in certain corner cases exceed @max_size! + * + * Return: Number of free pages in the pool + */ +static inline size_t kbase_mem_pool_size(struct kbase_mem_pool *pool) +{ + return READ_ONCE(pool->cur_size); +} + +/** + * kbase_mem_pool_max_size - Get maximum number of free pages in memory pool + * @pool: Memory pool to inspect + * + * Return: Maximum number of free pages in the pool + */ +static inline size_t kbase_mem_pool_max_size(struct kbase_mem_pool *pool) +{ + return pool->max_size; +} + + +/** + * kbase_mem_pool_set_max_size - Set maximum number of free pages in memory pool + * @pool: Memory pool to inspect + * @max_size: Maximum number of free pages the pool can hold + * + * If @max_size is reduced, the pool will be shrunk to adhere to the new limit. + * For details see kbase_mem_pool_shrink(). + */ +void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size); + +/** + * kbase_mem_pool_grow - Grow the pool + * @pool: Memory pool to grow + * @nr_to_grow: Number of pages to add to the pool + * + * Adds @nr_to_grow pages to the pool. Note that this may cause the pool to + * become larger than the maximum size specified. + * + * Returns: 0 on success, -ENOMEM if unable to allocate sufficent pages + */ +int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow); + +/** + * kbase_mem_pool_trim - Grow or shrink the pool to a new size + * @pool: Memory pool to trim + * @new_size: New number of pages in the pool + * + * If @new_size > @cur_size, fill the pool with new pages from the kernel, but + * not above the max_size for the pool. + * If @new_size < @cur_size, shrink the pool by freeing pages to the kernel. + */ +void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size); + +/* + * kbase_mem_alloc_page - Allocate a new page for a device + * @kbdev: The kbase device + * + * Most uses should use kbase_mem_pool_alloc to allocate a page. However that + * function can fail in the event the pool is empty. + * + * Return: A new page or NULL if no memory + */ +struct page *kbase_mem_alloc_page(struct kbase_device *kbdev); + +int kbase_region_tracker_init(struct kbase_context *kctx); +int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages); +void kbase_region_tracker_term(struct kbase_context *kctx); + +struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr); + +/** + * @brief Check that a pointer is actually a valid region. + * + * Must be called with context lock held. + */ +struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr); + +struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone); +void kbase_free_alloced_region(struct kbase_va_region *reg); +int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); + +bool kbase_check_alloc_flags(unsigned long flags); +bool kbase_check_import_flags(unsigned long flags); + +/** + * kbase_update_region_flags - Convert user space flags to kernel region flags + * + * @kctx: kbase context + * @reg: The region to update the flags on + * @flags: The flags passed from user space + * + * The user space flag BASE_MEM_COHERENT_SYSTEM_REQUIRED will be rejected and + * this function will fail if the system does not support system coherency. + * + * Return: 0 if successful, -EINVAL if the flags are not supported + */ +int kbase_update_region_flags(struct kbase_context *kctx, + struct kbase_va_region *reg, unsigned long flags); + +void kbase_gpu_vm_lock(struct kbase_context *kctx); +void kbase_gpu_vm_unlock(struct kbase_context *kctx); + +int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size); + +int kbase_mmu_init(struct kbase_context *kctx); +void kbase_mmu_term(struct kbase_context *kctx); + +phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx); +void kbase_mmu_free_pgd(struct kbase_context *kctx); +int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, + phys_addr_t *phys, size_t nr, + unsigned long flags); +int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, + phys_addr_t *phys, size_t nr, + unsigned long flags); +int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, + phys_addr_t phys, size_t nr, + unsigned long flags); + +int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr); +int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags); + +/** + * @brief Register region and map it on the GPU. + * + * Call kbase_add_va_region() and map the region on the GPU. + */ +int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); + +/** + * @brief Remove the region from the GPU and unregister it. + * + * Must be called with context lock held. + */ +int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg); + +/** + * The caller has the following locking conditions: + * - It must hold kbase_device->mmu_hw_mutex + * - It must hold the hwaccess_lock + */ +void kbase_mmu_update(struct kbase_context *kctx); + +/** + * kbase_mmu_disable() - Disable the MMU for a previously active kbase context. + * @kctx: Kbase context + * + * Disable and perform the required cache maintenance to remove the all + * data from provided kbase context from the GPU caches. + * + * The caller has the following locking conditions: + * - It must hold kbase_device->mmu_hw_mutex + * - It must hold the hwaccess_lock + */ +void kbase_mmu_disable(struct kbase_context *kctx); + +/** + * kbase_mmu_disable_as() - Set the MMU to unmapped mode for the specified + * address space. + * @kbdev: Kbase device + * @as_nr: The address space number to set to unmapped. + * + * This function must only be called during reset/power-up and it used to + * ensure the registers are in a known state. + * + * The caller must hold kbdev->mmu_hw_mutex. + */ +void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr); + +void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); + +/** Dump the MMU tables to a buffer + * + * This function allocates a buffer (of @c nr_pages pages) to hold a dump of the MMU tables and fills it. If the + * buffer is too small then the return value will be NULL. + * + * The GPU vm lock must be held when calling this function. + * + * The buffer returned should be freed with @ref vfree when it is no longer required. + * + * @param[in] kctx The kbase context to dump + * @param[in] nr_pages The number of pages to allocate for the buffer. + * + * @return The address of the buffer containing the MMU dump or NULL on error (including if the @c nr_pages is too + * small) + */ +void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); + +/** + * kbase_sync_now - Perform cache maintenance on a memory region + * + * @kctx: The kbase context of the region + * @sset: A syncset structure describing the region and direction of the + * synchronisation required + * + * Return: 0 on success or error code + */ +int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset); +void kbase_sync_single(struct kbase_context *kctx, phys_addr_t cpu_pa, + phys_addr_t gpu_pa, off_t offset, size_t size, + enum kbase_sync_type sync_fn); +void kbase_pre_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); +void kbase_post_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); + +/* OS specific functions */ +int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr); +int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg); +void kbase_os_mem_map_lock(struct kbase_context *kctx); +void kbase_os_mem_map_unlock(struct kbase_context *kctx); + +/** + * @brief Update the memory allocation counters for the current process + * + * OS specific call to updates the current memory allocation counters for the current process with + * the supplied delta. + * + * @param[in] kctx The kbase context + * @param[in] pages The desired delta to apply to the memory usage counters. + */ + +void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages); + +/** + * @brief Add to the memory allocation counters for the current process + * + * OS specific call to add to the current memory allocation counters for the current process by + * the supplied amount. + * + * @param[in] kctx The kernel base context used for the allocation. + * @param[in] pages The desired delta to apply to the memory usage counters. + */ + +static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages) +{ + kbasep_os_process_page_usage_update(kctx, pages); +} + +/** + * @brief Subtract from the memory allocation counters for the current process + * + * OS specific call to subtract from the current memory allocation counters for the current process by + * the supplied amount. + * + * @param[in] kctx The kernel base context used for the allocation. + * @param[in] pages The desired delta to apply to the memory usage counters. + */ + +static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages) +{ + kbasep_os_process_page_usage_update(kctx, 0 - pages); +} + +/** + * kbasep_find_enclosing_cpu_mapping_offset() - Find the offset of the CPU + * mapping of a memory allocation containing a given address range + * + * Searches for a CPU mapping of any part of any region that fully encloses the + * CPU virtual address range specified by @uaddr and @size. Returns a failure + * indication if only part of the address range lies within a CPU mapping. + * + * @kctx: The kernel base context used for the allocation. + * @uaddr: Start of the CPU virtual address range. + * @size: Size of the CPU virtual address range (in bytes). + * @offset: The offset from the start of the allocation to the specified CPU + * virtual address. + * + * Return: 0 if offset was obtained successfully. Error code otherwise. + */ +int kbasep_find_enclosing_cpu_mapping_offset( + struct kbase_context *kctx, + unsigned long uaddr, size_t size, u64 *offset); + +enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer); +void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); +void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); + +/** +* @brief Allocates physical pages. +* +* Allocates \a nr_pages_requested and updates the alloc object. +* +* @param[in] alloc allocation object to add pages to +* @param[in] nr_pages_requested number of physical pages to allocate +* +* @return 0 if all pages have been successfully allocated. Error code otherwise +*/ +int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_requested); + +/** +* @brief Free physical pages. +* +* Frees \a nr_pages and updates the alloc object. +* +* @param[in] alloc allocation object to free pages from +* @param[in] nr_pages_to_free number of physical pages to free +*/ +int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free); + +static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) +{ + SetPagePrivate(p); + if (sizeof(dma_addr_t) > sizeof(p->private)) { + /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the + * private field stays the same. So we have to be clever and + * use the fact that we only store DMA addresses of whole pages, + * so the low bits should be zero */ + KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1))); + set_page_private(p, dma_addr >> PAGE_SHIFT); + } else { + set_page_private(p, dma_addr); + } +} + +static inline dma_addr_t kbase_dma_addr(struct page *p) +{ + if (sizeof(dma_addr_t) > sizeof(p->private)) + return ((dma_addr_t)page_private(p)) << PAGE_SHIFT; + + return (dma_addr_t)page_private(p); +} + +static inline void kbase_clear_dma_addr(struct page *p) +{ + ClearPagePrivate(p); +} + +/** +* @brief Process a bus or page fault. +* +* This function will process a fault on a specific address space +* +* @param[in] kbdev The @ref kbase_device the fault happened on +* @param[in] kctx The @ref kbase_context for the faulting address space if +* one was found. +* @param[in] as The address space that has the fault +*/ +void kbase_mmu_interrupt_process(struct kbase_device *kbdev, + struct kbase_context *kctx, struct kbase_as *as); + +/** + * @brief Process a page fault. + * + * @param[in] data work_struct passed by queue_work() + */ +void page_fault_worker(struct work_struct *data); + +/** + * @brief Process a bus fault. + * + * @param[in] data work_struct passed by queue_work() + */ +void bus_fault_worker(struct work_struct *data); + +/** + * @brief Flush MMU workqueues. + * + * This function will cause any outstanding page or bus faults to be processed. + * It should be called prior to powering off the GPU. + * + * @param[in] kbdev Device pointer + */ +void kbase_flush_mmu_wqs(struct kbase_device *kbdev); + +/** + * kbase_sync_single_for_device - update physical memory and give GPU ownership + * @kbdev: Device pointer + * @handle: DMA address of region + * @size: Size of region to sync + * @dir: DMA data direction + */ + +void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir); + +/** + * kbase_sync_single_for_cpu - update physical memory and give CPU ownership + * @kbdev: Device pointer + * @handle: DMA address of region + * @size: Size of region to sync + * @dir: DMA data direction + */ + +void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, + size_t size, enum dma_data_direction dir); + +#ifdef CONFIG_DEBUG_FS +/** + * kbase_jit_debugfs_init - Add per context debugfs entry for JIT. + * @kctx: kbase context + */ +void kbase_jit_debugfs_init(struct kbase_context *kctx); +#endif /* CONFIG_DEBUG_FS */ + +/** + * kbase_jit_init - Initialize the JIT memory pool management + * @kctx: kbase context + * + * Returns zero on success or negative error number on failure. + */ +int kbase_jit_init(struct kbase_context *kctx); + +/** + * kbase_jit_allocate - Allocate JIT memory + * @kctx: kbase context + * @info: JIT allocation information + * + * Return: JIT allocation on success or NULL on failure. + */ +struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, + struct base_jit_alloc_info *info); + +/** + * kbase_jit_free - Free a JIT allocation + * @kctx: kbase context + * @reg: JIT allocation + * + * Frees a JIT allocation and places it into the free pool for later reuse. + */ +void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg); + +/** + * kbase_jit_backing_lost - Inform JIT that an allocation has lost backing + * @reg: JIT allocation + */ +void kbase_jit_backing_lost(struct kbase_va_region *reg); + +/** + * kbase_jit_evict - Evict a JIT allocation from the pool + * @kctx: kbase context + * + * Evict the least recently used JIT allocation from the pool. This can be + * required if normal VA allocations are failing due to VA exhaustion. + * + * Return: True if a JIT allocation was freed, false otherwise. + */ +bool kbase_jit_evict(struct kbase_context *kctx); + +/** + * kbase_jit_term - Terminate the JIT memory pool management + * @kctx: kbase context + */ +void kbase_jit_term(struct kbase_context *kctx); + +/** + * kbase_map_external_resource - Map an external resource to the GPU. + * @kctx: kbase context. + * @reg: The region to map. + * @locked_mm: The mm_struct which has been locked for this operation. + * @kds_res_count: The number of KDS resources. + * @kds_resources: Array of KDS resources. + * @kds_access_bitmap: Access bitmap for KDS. + * @exclusive: If the KDS resource requires exclusive access. + * + * Return: The physical allocation which backs the region on success or NULL + * on failure. + */ +struct kbase_mem_phy_alloc *kbase_map_external_resource( + struct kbase_context *kctx, struct kbase_va_region *reg, + struct mm_struct *locked_mm +#ifdef CONFIG_KDS + , u32 *kds_res_count, struct kds_resource **kds_resources, + unsigned long *kds_access_bitmap, bool exclusive +#endif + ); + +/** + * kbase_unmap_external_resource - Unmap an external resource from the GPU. + * @kctx: kbase context. + * @reg: The region to unmap or NULL if it has already been released. + * @alloc: The physical allocation being unmapped. + */ +void kbase_unmap_external_resource(struct kbase_context *kctx, + struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); + +/** + * kbase_sticky_resource_init - Initialize sticky resource management. + * @kctx: kbase context + * + * Returns zero on success or negative error number on failure. + */ +int kbase_sticky_resource_init(struct kbase_context *kctx); + +/** + * kbase_sticky_resource_acquire - Acquire a reference on a sticky resource. + * @kctx: kbase context. + * @gpu_addr: The GPU address of the external resource. + * + * Return: The metadata object which represents the binding between the + * external resource and the kbase context on success or NULL on failure. + */ +struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( + struct kbase_context *kctx, u64 gpu_addr); + +/** + * kbase_sticky_resource_release - Release a reference on a sticky resource. + * @kctx: kbase context. + * @meta: Binding metadata. + * @gpu_addr: GPU address of the external resource. + * + * If meta is NULL then gpu_addr will be used to scan the metadata list and + * find the matching metadata (if any), otherwise the provided meta will be + * used and gpu_addr will be ignored. + * + * Return: True if the release found the metadata and the reference was dropped. + */ +bool kbase_sticky_resource_release(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); + +/** + * kbase_sticky_resource_term - Terminate sticky resource management. + * @kctx: kbase context + */ +void kbase_sticky_resource_term(struct kbase_context *kctx); + +#endif /* _KBASE_MEM_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c new file mode 100755 index 000000000000..e20315e67242 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c @@ -0,0 +1,2578 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mem_linux.c + * Base kernel memory APIs, Linux implementation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) +#include +#endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */ +#ifdef CONFIG_DMA_SHARED_BUFFER +#include +#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ +#include +#include + +#include +#include +#include +#include +#include + +static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma); + +/** + * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation + * @kctx: Context the region belongs to + * @reg: The GPU region + * @new_pages: The number of pages after the shrink + * @old_pages: The number of pages before the shrink + * + * Shrink (or completely remove) all CPU mappings which reference the shrunk + * part of the allocation. + * + * Note: Caller must be holding the processes mmap_lock lock. + */ +static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages); + +/** + * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation + * @kctx: Context the region belongs to + * @reg: The GPU region or NULL if there isn't one + * @new_pages: The number of pages after the shrink + * @old_pages: The number of pages before the shrink + * + * Return: 0 on success, negative -errno on error + * + * Unmap the shrunk pages from the GPU mapping. Note that the size of the region + * itself is unmodified as we still need to reserve the VA, only the page tables + * will be modified by this function. + */ +static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages); + +struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, + u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, + u64 *gpu_va) +{ + int zone; + int gpu_pc_bits; + int cpu_va_bits; + struct kbase_va_region *reg; + struct device *dev; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(flags); + KBASE_DEBUG_ASSERT(gpu_va); + + dev = kctx->kbdev->dev; + *gpu_va = 0; /* return 0 on failure */ + + gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; + cpu_va_bits = BITS_PER_LONG; + + if (0 == va_pages) { + dev_warn(dev, "kbase_mem_alloc called with 0 va_pages!"); + goto bad_size; + } + + if (va_pages > (U64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + +#if defined(CONFIG_64BIT) + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + cpu_va_bits = 32; +#endif + + if (!kbase_check_alloc_flags(*flags)) { + dev_warn(dev, + "kbase_mem_alloc called with bad flags (%llx)", + (unsigned long long)*flags); + goto bad_flags; + } + + if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable"); + goto bad_flags; + } + if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ + *flags &= ~BASE_MEM_COHERENT_SYSTEM; + } + + /* Limit GPU executable allocs to GPU PC size */ + if ((*flags & BASE_MEM_PROT_GPU_EX) && + (va_pages > (1ULL << gpu_pc_bits >> PAGE_SHIFT))) + goto bad_ex_size; + + /* find out which VA zone to use */ + if (*flags & BASE_MEM_SAME_VA) + zone = KBASE_REG_ZONE_SAME_VA; + else if (*flags & BASE_MEM_PROT_GPU_EX) + zone = KBASE_REG_ZONE_EXEC; + else + zone = KBASE_REG_ZONE_CUSTOM_VA; + + reg = kbase_alloc_free_region(kctx, 0, va_pages, zone); + if (!reg) { + dev_err(dev, "Failed to allocate free region"); + goto no_region; + } + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + if (kbase_reg_prepare_native(reg, kctx) != 0) { + dev_err(dev, "Failed to prepare region"); + goto prepare_failed; + } + + if (*flags & BASE_MEM_GROW_ON_GPF) + reg->extent = extent; + else + reg->extent = 0; + + if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) { + dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)", + (unsigned long long)commit_pages, + (unsigned long long)va_pages); + goto no_mem; + } + + kbase_gpu_vm_lock(kctx); + + /* mmap needed to setup VA? */ + if (*flags & BASE_MEM_SAME_VA) { + unsigned long prot = PROT_NONE; + unsigned long va_size = va_pages << PAGE_SHIFT; + unsigned long va_map = va_size; + unsigned long cookie, cookie_nr; + unsigned long cpu_addr; + + /* Bind to a cookie */ + if (!kctx->cookies) { + dev_err(dev, "No cookies available for allocation!"); + kbase_gpu_vm_unlock(kctx); + goto no_cookie; + } + /* return a cookie */ + cookie_nr = __ffs(kctx->cookies); + kctx->cookies &= ~(1UL << cookie_nr); + BUG_ON(kctx->pending_regions[cookie_nr]); + kctx->pending_regions[cookie_nr] = reg; + + kbase_gpu_vm_unlock(kctx); + + /* relocate to correct base */ + cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE); + cookie <<= PAGE_SHIFT; + + /* + * 10.1-10.4 UKU userland relies on the kernel to call mmap. + * For all other versions we can just return the cookie + */ + if (kctx->api_version < KBASE_API_VERSION(10, 1) || + kctx->api_version > KBASE_API_VERSION(10, 4)) { + *gpu_va = (u64) cookie; + return reg; + } + if (*flags & BASE_MEM_PROT_CPU_RD) + prot |= PROT_READ; + if (*flags & BASE_MEM_PROT_CPU_WR) + prot |= PROT_WRITE; + + cpu_addr = vm_mmap(kctx->filp, 0, va_map, prot, + MAP_SHARED, cookie); + + if (IS_ERR_VALUE(cpu_addr)) { + kbase_gpu_vm_lock(kctx); + kctx->pending_regions[cookie_nr] = NULL; + kctx->cookies |= (1UL << cookie_nr); + kbase_gpu_vm_unlock(kctx); + goto no_mmap; + } + + *gpu_va = (u64) cpu_addr; + } else /* we control the VA */ { + if (kbase_gpu_mmap(kctx, reg, 0, va_pages, 1) != 0) { + dev_warn(dev, "Failed to map memory on GPU"); + kbase_gpu_vm_unlock(kctx); + goto no_mmap; + } + /* return real GPU VA */ + *gpu_va = reg->start_pfn << PAGE_SHIFT; + + kbase_gpu_vm_unlock(kctx); + } + + return reg; + +no_mmap: +no_cookie: +no_mem: + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +invalid_flags: +prepare_failed: + kfree(reg); +no_region: +bad_ex_size: +bad_flags: +bad_size: + return NULL; +} +KBASE_EXPORT_TEST_API(kbase_mem_alloc); + +int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * const out) +{ + struct kbase_va_region *reg; + int ret = -EINVAL; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(out); + + if (gpu_addr & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, "mem_query: gpu_addr: passed parameter is invalid"); + return -EINVAL; + } + + kbase_gpu_vm_lock(kctx); + + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (!reg || (reg->flags & KBASE_REG_FREE)) + goto out_unlock; + + switch (query) { + case KBASE_MEM_QUERY_COMMIT_SIZE: + if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) { + *out = kbase_reg_current_backed_size(reg); + } else { + size_t i; + struct kbase_aliased *aliased; + *out = 0; + aliased = reg->cpu_alloc->imported.alias.aliased; + for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++) + *out += aliased[i].length; + } + break; + case KBASE_MEM_QUERY_VA_SIZE: + *out = reg->nr_pages; + break; + case KBASE_MEM_QUERY_FLAGS: + { + *out = 0; + if (KBASE_REG_CPU_WR & reg->flags) + *out |= BASE_MEM_PROT_CPU_WR; + if (KBASE_REG_CPU_RD & reg->flags) + *out |= BASE_MEM_PROT_CPU_RD; + if (KBASE_REG_CPU_CACHED & reg->flags) + *out |= BASE_MEM_CACHED_CPU; + if (KBASE_REG_GPU_WR & reg->flags) + *out |= BASE_MEM_PROT_GPU_WR; + if (KBASE_REG_GPU_RD & reg->flags) + *out |= BASE_MEM_PROT_GPU_RD; + if (!(KBASE_REG_GPU_NX & reg->flags)) + *out |= BASE_MEM_PROT_GPU_EX; + if (KBASE_REG_SHARE_BOTH & reg->flags) + *out |= BASE_MEM_COHERENT_SYSTEM; + if (KBASE_REG_SHARE_IN & reg->flags) + *out |= BASE_MEM_COHERENT_LOCAL; + break; + } + default: + *out = 0; + goto out_unlock; + } + + ret = 0; + +out_unlock: + kbase_gpu_vm_unlock(kctx); + return ret; +} + +/** + * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the + * Ephemeral memory eviction list. + * @s: Shrinker + * @sc: Shrinker control + * + * Return: Number of pages which can be freed. + */ +static +unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_context *kctx; + struct kbase_mem_phy_alloc *alloc; + unsigned long pages = 0; + + kctx = container_of(s, struct kbase_context, reclaim); + + mutex_lock(&kctx->jit_evict_lock); + + list_for_each_entry(alloc, &kctx->evict_list, evict_node) + pages += alloc->nents; + + mutex_unlock(&kctx->jit_evict_lock); + return pages; +} + +/** + * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction + * list for pages and try to reclaim them. + * @s: Shrinker + * @sc: Shrinker control + * + * Return: Number of pages freed (can be less then requested) or -1 if the + * shrinker failed to free pages in its pool. + * + * Note: + * This function accesses region structures without taking the region lock, + * this is required as the OOM killer can call the shrinker after the region + * lock has already been held. + * This is safe as we can guarantee that a region on the eviction list will + * not be freed (kbase_mem_free_region removes the allocation from the list + * before destroying it), or modified by other parts of the driver. + * The eviction list itself is guarded by the eviction lock and the MMU updates + * are protected by their own lock. + */ +static +unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_context *kctx; + struct kbase_mem_phy_alloc *alloc; + struct kbase_mem_phy_alloc *tmp; + unsigned long freed = 0; + + kctx = container_of(s, struct kbase_context, reclaim); + mutex_lock(&kctx->jit_evict_lock); + + list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { + int err; + + err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg, + 0, alloc->nents); + if (err != 0) { + /* + * Failed to remove GPU mapping, tell the shrinker + * to stop trying to shrink our slab even though we + * have pages in it. + */ + freed = -1; + goto out_unlock; + } + + /* + * Update alloc->evicted before freeing the backing so the + * helper can determine that it needs to bypass the accounting + * and memory pool. + */ + alloc->evicted = alloc->nents; + + kbase_free_phy_pages_helper(alloc, alloc->evicted); + freed += alloc->evicted; + list_del_init(&alloc->evict_node); + + /* + * Inform the JIT allocator this region has lost backing + * as it might need to free the allocation. + */ + kbase_jit_backing_lost(alloc->reg); + + /* Enough pages have been freed so stop now */ + if (freed > sc->nr_to_scan) + break; + } +out_unlock: + mutex_unlock(&kctx->jit_evict_lock); + + return freed; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s, + struct shrink_control *sc) +{ + if (sc->nr_to_scan == 0) + return kbase_mem_evictable_reclaim_count_objects(s, sc); + + return kbase_mem_evictable_reclaim_scan_objects(s, sc); +} +#endif + +int kbase_mem_evictable_init(struct kbase_context *kctx) +{ + INIT_LIST_HEAD(&kctx->evict_list); + mutex_init(&kctx->jit_evict_lock); + + /* Register shrinker */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) + kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink; +#else + kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; + kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; +#endif + kctx->reclaim.seeks = DEFAULT_SEEKS; + /* Kernel versions prior to 3.1 : + * struct shrinker does not define batch */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) + kctx->reclaim.batch = 0; +#endif + register_shrinker(&kctx->reclaim); + return 0; +} + +void kbase_mem_evictable_deinit(struct kbase_context *kctx) +{ + unregister_shrinker(&kctx->reclaim); +} + +/** + * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable. + * @alloc: The physical allocation + */ +static void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc) +{ + struct kbase_context *kctx = alloc->imported.kctx; + int __maybe_unused new_page_count; + + kbase_process_page_usage_dec(kctx, alloc->nents); + new_page_count = kbase_atomic_sub_pages(alloc->nents, + &kctx->used_pages); + kbase_atomic_sub_pages(alloc->nents, &kctx->kbdev->memdev.used_pages); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + (u32)kctx->id, + (u64)new_page_count); +} + +/** + * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable. + * @alloc: The physical allocation + */ +static +void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) +{ + struct kbase_context *kctx = alloc->imported.kctx; + int __maybe_unused new_page_count; + + new_page_count = kbase_atomic_add_pages(alloc->nents, + &kctx->used_pages); + kbase_atomic_add_pages(alloc->nents, &kctx->kbdev->memdev.used_pages); + + /* Increase mm counters so that the allocation is accounted for + * against the process and thus is visible to the OOM killer. + */ + kbase_process_page_usage_inc(kctx, alloc->nents); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + (u32)kctx->id, + (u64)new_page_count); +} + +int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) +{ + struct kbase_context *kctx = gpu_alloc->imported.kctx; + + lockdep_assert_held(&kctx->reg_lock); + + /* This alloction can't already be on a list. */ + WARN_ON(!list_empty(&gpu_alloc->evict_node)); + + kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg, + 0, gpu_alloc->nents); + + /* + * Add the allocation to the eviction list, after this point the shrink + * can reclaim it. + */ + mutex_lock(&kctx->jit_evict_lock); + list_add(&gpu_alloc->evict_node, &kctx->evict_list); + mutex_unlock(&kctx->jit_evict_lock); + kbase_mem_evictable_mark_reclaim(gpu_alloc); + + gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED; + return 0; +} + +bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) +{ + struct kbase_context *kctx = gpu_alloc->imported.kctx; + int err = 0; + + lockdep_assert_held(&kctx->reg_lock); + + /* + * First remove the allocation from the eviction list as it's no + * longer eligible for eviction. + */ + list_del_init(&gpu_alloc->evict_node); + + if (gpu_alloc->evicted == 0) { + /* + * The backing is still present, update the VM stats as it's + * in use again. + */ + kbase_mem_evictable_unmark_reclaim(gpu_alloc); + } else { + /* If the region is still alive ... */ + if (gpu_alloc->reg) { + /* ... allocate replacement backing ... */ + err = kbase_alloc_phy_pages_helper(gpu_alloc, + gpu_alloc->evicted); + + /* + * ... and grow the mapping back to its + * pre-eviction size. + */ + if (!err) + err = kbase_mem_grow_gpu_mapping(kctx, + gpu_alloc->reg, + gpu_alloc->evicted, 0); + + gpu_alloc->evicted = 0; + } + } + + /* If the region is still alive remove the DONT_NEED attribute. */ + if (gpu_alloc->reg) + gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED; + + return (err == 0); +} + +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask) +{ + struct kbase_va_region *reg; + int ret = -EINVAL; + unsigned int real_flags = 0; + unsigned int prev_flags = 0; + bool prev_needed, new_needed; + + KBASE_DEBUG_ASSERT(kctx); + + if (!gpu_addr) + return -EINVAL; + + if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) + return -EINVAL; + + /* nuke other bits */ + flags &= mask; + + /* check for only supported flags */ + if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE)) + goto out; + + /* mask covers bits we don't support? */ + if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE)) + goto out; + + /* convert flags */ + if (BASE_MEM_COHERENT_SYSTEM & flags) + real_flags |= KBASE_REG_SHARE_BOTH; + else if (BASE_MEM_COHERENT_LOCAL & flags) + real_flags |= KBASE_REG_SHARE_IN; + + /* now we can lock down the context, and find the region */ + down_write(¤t->mm->mmap_lock); + kbase_gpu_vm_lock(kctx); + + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (!reg || (reg->flags & KBASE_REG_FREE)) + goto out_unlock; + + /* Is the region being transitioning between not needed and needed? */ + prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED; + new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED; + if (prev_needed != new_needed) { + /* Aliased allocations can't be made ephemeral */ + if (atomic_read(®->cpu_alloc->gpu_mappings) > 1) + goto out_unlock; + + if (new_needed) { + /* Only native allocations can be marked not needed */ + if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) { + ret = -EINVAL; + goto out_unlock; + } + ret = kbase_mem_evictable_make(reg->gpu_alloc); + if (ret) + goto out_unlock; + } else { + kbase_mem_evictable_unmake(reg->gpu_alloc); + } + } + + /* limit to imported memory */ + if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) && + (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) + goto out_unlock; + + /* no change? */ + if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) { + ret = 0; + goto out_unlock; + } + + /* save for roll back */ + prev_flags = reg->flags; + reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); + reg->flags |= real_flags; + + /* Currently supporting only imported memory */ + switch (reg->gpu_alloc->type) { +#ifdef CONFIG_UMP + case KBASE_MEM_TYPE_IMPORTED_UMP: + ret = kbase_mmu_update_pages(kctx, reg->start_pfn, kbase_get_cpu_phy_pages(reg), reg->gpu_alloc->nents, reg->flags); + break; +#endif +#ifdef CONFIG_DMA_SHARED_BUFFER + case KBASE_MEM_TYPE_IMPORTED_UMM: + /* Future use will use the new flags, existing mapping will NOT be updated + * as memory should not be in use by the GPU when updating the flags. + */ + ret = 0; + WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count); + break; +#endif + default: + break; + } + + /* roll back on error, i.e. not UMP */ + if (ret) + reg->flags = prev_flags; + +out_unlock: + kbase_gpu_vm_unlock(kctx); + up_write(¤t->mm->mmap_lock); +out: + return ret; +} + +#define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS) + +#ifdef CONFIG_UMP +static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags) +{ + struct kbase_va_region *reg; + ump_dd_handle umph; + u64 block_count; + const ump_dd_physical_block_64 *block_array; + u64 i, j; + int page = 0; + ump_alloc_flags ump_flags; + ump_alloc_flags cpu_flags; + ump_alloc_flags gpu_flags; + + if (*flags & BASE_MEM_SECURE) + goto bad_flags; + + umph = ump_dd_from_secure_id(id); + if (UMP_DD_INVALID_MEMORY_HANDLE == umph) + goto bad_id; + + ump_flags = ump_dd_allocation_flags_get(umph); + cpu_flags = (ump_flags >> UMP_DEVICE_CPU_SHIFT) & UMP_DEVICE_MASK; + gpu_flags = (ump_flags >> DEFAULT_UMP_GPU_DEVICE_SHIFT) & + UMP_DEVICE_MASK; + + *va_pages = ump_dd_size_get_64(umph); + *va_pages >>= PAGE_SHIFT; + + if (!*va_pages) + goto bad_size; + + if (*va_pages > (U64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + if (*flags & BASE_MEM_SAME_VA) + reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); + else + reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); + + if (!reg) + goto no_region; + + /* we've got pages to map now, and support SAME_VA */ + *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; + + reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc_obj; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + reg->gpu_alloc->imported.ump_handle = umph; + + reg->flags &= ~KBASE_REG_FREE; + reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */ + reg->flags &= ~KBASE_REG_GROWABLE; /* UMP cannot be grown */ + + /* Override import flags based on UMP flags */ + *flags &= ~(BASE_MEM_CACHED_CPU); + *flags &= ~(BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR); + *flags &= ~(BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR); + + if ((cpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == + (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) { + reg->flags |= KBASE_REG_CPU_CACHED; + *flags |= BASE_MEM_CACHED_CPU; + } + + if (cpu_flags & UMP_PROT_CPU_WR) { + reg->flags |= KBASE_REG_CPU_WR; + *flags |= BASE_MEM_PROT_CPU_WR; + } + + if (cpu_flags & UMP_PROT_CPU_RD) { + reg->flags |= KBASE_REG_CPU_RD; + *flags |= BASE_MEM_PROT_CPU_RD; + } + + if ((gpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == + (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) + reg->flags |= KBASE_REG_GPU_CACHED; + + if (gpu_flags & UMP_PROT_DEVICE_WR) { + reg->flags |= KBASE_REG_GPU_WR; + *flags |= BASE_MEM_PROT_GPU_WR; + } + + if (gpu_flags & UMP_PROT_DEVICE_RD) { + reg->flags |= KBASE_REG_GPU_RD; + *flags |= BASE_MEM_PROT_GPU_RD; + } + + /* ump phys block query */ + ump_dd_phys_blocks_get_64(umph, &block_count, &block_array); + + for (i = 0; i < block_count; i++) { + for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) { + reg->gpu_alloc->pages[page] = block_array[i].addr + (j << PAGE_SHIFT); + page++; + } + } + reg->gpu_alloc->nents = *va_pages; + reg->extent = 0; + + return reg; + +no_alloc_obj: + kfree(reg); +no_region: +bad_size: + ump_dd_release(umph); +bad_id: +bad_flags: + return NULL; +} +#endif /* CONFIG_UMP */ + +#ifdef CONFIG_DMA_SHARED_BUFFER +static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, + int fd, u64 *va_pages, u64 *flags, u32 padding) +{ + struct kbase_va_region *reg; + struct dma_buf *dma_buf; + struct dma_buf_attachment *dma_attachment; + bool shared_zone = false; + + dma_buf = dma_buf_get(fd); + if (IS_ERR_OR_NULL(dma_buf)) + goto no_buf; + + dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev); + if (!dma_attachment) + goto no_attachment; + + *va_pages = (PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT) + padding; + if (!*va_pages) + goto bad_size; + + if (*va_pages > (U64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + /* ignore SAME_VA */ + *flags &= ~BASE_MEM_SAME_VA; + + if (*flags & BASE_MEM_IMPORT_SHARED) + shared_zone = true; + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* + * 64-bit tasks require us to reserve VA on the CPU that we use + * on the GPU. + */ + shared_zone = true; + } +#endif + + if (shared_zone) { + *flags |= BASE_MEM_NEED_MMAP; + reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); + } else { + reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); + } + + if (!reg) + goto no_region; + + reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc_obj; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + /* No pages to map yet */ + reg->gpu_alloc->nents = 0; + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + reg->flags &= ~KBASE_REG_FREE; + reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */ + reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */ + reg->flags |= KBASE_REG_GPU_CACHED; + + if (*flags & BASE_MEM_SECURE) + reg->flags |= KBASE_REG_SECURE; + + if (padding) + reg->flags |= KBASE_REG_IMPORT_PAD; + + reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM; + reg->gpu_alloc->imported.umm.sgt = NULL; + reg->gpu_alloc->imported.umm.dma_buf = dma_buf; + reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment; + reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0; + reg->extent = 0; + + return reg; + +invalid_flags: + kbase_mem_phy_alloc_put(reg->gpu_alloc); +no_alloc_obj: + kfree(reg); +no_region: +bad_size: + dma_buf_detach(dma_buf, dma_attachment); +no_attachment: + dma_buf_put(dma_buf); +no_buf: + return NULL; +} +#endif /* CONFIG_DMA_SHARED_BUFFER */ + +static u32 kbase_get_cache_line_alignment(struct kbase_context *kctx) +{ + u32 cpu_cache_line_size = cache_line_size(); + u32 gpu_cache_line_size = + (1UL << kctx->kbdev->gpu_props.props.l2_props.log2_line_size); + + return ((cpu_cache_line_size > gpu_cache_line_size) ? + cpu_cache_line_size : + gpu_cache_line_size); +} + +static struct kbase_va_region *kbase_mem_from_user_buffer( + struct kbase_context *kctx, unsigned long address, + unsigned long size, u64 *va_pages, u64 *flags) +{ + long i; + struct kbase_va_region *reg; + long faulted_pages; + int zone = KBASE_REG_ZONE_CUSTOM_VA; + bool shared_zone = false; + u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx); + struct kbase_alloc_import_user_buf *user_buf; + struct page **pages = NULL; + + if ((address & (cache_line_alignment - 1)) != 0 || + (size & (cache_line_alignment - 1)) != 0) { + /* Coherency must be enabled to handle partial cache lines */ + if (*flags & (BASE_MEM_COHERENT_SYSTEM | + BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { + /* Force coherent system required flag, import will + * then fail if coherency isn't available + */ + *flags |= BASE_MEM_COHERENT_SYSTEM_REQUIRED; + } else { + dev_warn(kctx->kbdev->dev, + "User buffer is not cache line aligned and no coherency enabled\n"); + goto bad_size; + } + } + + *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) - + PFN_DOWN(address); + if (!*va_pages) + goto bad_size; + + if (*va_pages > (UINT64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + /* SAME_VA generally not supported with imported memory (no known use cases) */ + *flags &= ~BASE_MEM_SAME_VA; + + if (*flags & BASE_MEM_IMPORT_SHARED) + shared_zone = true; + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* + * 64-bit tasks require us to reserve VA on the CPU that we use + * on the GPU. + */ + shared_zone = true; + } +#endif + + if (shared_zone) { + *flags |= BASE_MEM_NEED_MMAP; + zone = KBASE_REG_ZONE_SAME_VA; + } + + reg = kbase_alloc_free_region(kctx, 0, *va_pages, zone); + + if (!reg) + goto no_region; + + reg->gpu_alloc = kbase_alloc_create(*va_pages, + KBASE_MEM_TYPE_IMPORTED_USER_BUF); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc_obj; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + reg->flags &= ~KBASE_REG_FREE; + reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */ + reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */ + reg->flags &= ~KBASE_REG_CPU_CACHED; + + user_buf = ®->gpu_alloc->imported.user_buf; + + user_buf->size = size; + user_buf->address = address; + user_buf->nr_pages = *va_pages; + user_buf->mm = current->mm; + user_buf->pages = kmalloc_array(*va_pages, sizeof(struct page *), + GFP_KERNEL); + + if (!user_buf->pages) + goto no_page_array; + + /* If the region is coherent with the CPU then the memory is imported + * and mapped onto the GPU immediately. + * Otherwise get_user_pages is called as a sanity check, but with + * NULL as the pages argument which will fault the pages, but not + * pin them. The memory will then be pinned only around the jobs that + * specify the region as an external resource. + */ + if (reg->flags & KBASE_REG_SHARE_BOTH) { + pages = user_buf->pages; + *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; + } + + down_read(¤t->mm->mmap_lock); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) + faulted_pages = get_user_pages(current, current->mm, address, *va_pages, + reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) + faulted_pages = get_user_pages(address, *va_pages, + reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); +#else + faulted_pages = get_user_pages(address, *va_pages, + reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, + pages, NULL); +#endif + + up_read(¤t->mm->mmap_lock); + + if (faulted_pages != *va_pages) + goto fault_mismatch; + + atomic_inc(¤t->mm->mm_count); + + reg->gpu_alloc->nents = 0; + reg->extent = 0; + + if (pages) { + struct device *dev = kctx->kbdev->dev; + unsigned long local_size = user_buf->size; + unsigned long offset = user_buf->address & ~PAGE_MASK; + phys_addr_t *pa = kbase_get_gpu_phy_pages(reg); + + /* Top bit signifies that this was pinned on import */ + user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT; + + for (i = 0; i < faulted_pages; i++) { + dma_addr_t dma_addr; + unsigned long min; + + min = MIN(PAGE_SIZE - offset, local_size); + dma_addr = dma_map_page(dev, pages[i], + offset, min, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) + goto unwind_dma_map; + + user_buf->dma_addrs[i] = dma_addr; + pa[i] = page_to_phys(pages[i]); + + local_size -= min; + offset = 0; + } + + reg->gpu_alloc->nents = faulted_pages; + } + + return reg; + +unwind_dma_map: + while (i--) { + dma_unmap_page(kctx->kbdev->dev, + user_buf->dma_addrs[i], + PAGE_SIZE, DMA_BIDIRECTIONAL); + } +fault_mismatch: + if (pages) { + for (i = 0; i < faulted_pages; i++) + put_page(pages[i]); + } + kfree(user_buf->pages); +no_page_array: +invalid_flags: + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +no_alloc_obj: + kfree(reg); +no_region: +bad_size: + return NULL; + +} + + +u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, + u64 nents, struct base_mem_aliasing_info *ai, + u64 *num_pages) +{ + struct kbase_va_region *reg; + u64 gpu_va; + size_t i; + bool coherent; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(flags); + KBASE_DEBUG_ASSERT(ai); + KBASE_DEBUG_ASSERT(num_pages); + + /* mask to only allowed flags */ + *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | + BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL | + BASE_MEM_COHERENT_SYSTEM_REQUIRED); + + if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) { + dev_warn(kctx->kbdev->dev, + "kbase_mem_alias called with bad flags (%llx)", + (unsigned long long)*flags); + goto bad_flags; + } + coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 || + (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0; + + if (!stride) + goto bad_stride; + + if (!nents) + goto bad_nents; + + if ((nents * stride) > (U64_MAX / PAGE_SIZE)) + /* 64-bit address range is the max */ + goto bad_size; + + /* calculate the number of pages this alias will cover */ + *num_pages = nents * stride; + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* 64-bit tasks must MMAP anyway, but not expose this address to + * clients */ + *flags |= BASE_MEM_NEED_MMAP; + reg = kbase_alloc_free_region(kctx, 0, *num_pages, + KBASE_REG_ZONE_SAME_VA); + } else { +#else + if (1) { +#endif + reg = kbase_alloc_free_region(kctx, 0, *num_pages, + KBASE_REG_ZONE_CUSTOM_VA); + } + + if (!reg) + goto no_reg; + + /* zero-sized page array, as we don't need one/can support one */ + reg->gpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS); + if (IS_ERR_OR_NULL(reg->gpu_alloc)) + goto no_alloc_obj; + + reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + + if (kbase_update_region_flags(kctx, reg, *flags) != 0) + goto invalid_flags; + + reg->gpu_alloc->imported.alias.nents = nents; + reg->gpu_alloc->imported.alias.stride = stride; + reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents); + if (!reg->gpu_alloc->imported.alias.aliased) + goto no_aliased_array; + + kbase_gpu_vm_lock(kctx); + + /* validate and add src handles */ + for (i = 0; i < nents; i++) { + if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) { + if (ai[i].handle.basep.handle != + BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) + goto bad_handle; /* unsupported magic handle */ + if (!ai[i].length) + goto bad_handle; /* must be > 0 */ + if (ai[i].length > stride) + goto bad_handle; /* can't be larger than the + stride */ + reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; + } else { + struct kbase_va_region *aliasing_reg; + struct kbase_mem_phy_alloc *alloc; + + aliasing_reg = kbase_region_tracker_find_region_base_address( + kctx, + (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); + + /* validate found region */ + if (!aliasing_reg) + goto bad_handle; /* Not found */ + if (aliasing_reg->flags & KBASE_REG_FREE) + goto bad_handle; /* Free region */ + if (aliasing_reg->flags & KBASE_REG_DONT_NEED) + goto bad_handle; /* Ephemeral region */ + if (!aliasing_reg->gpu_alloc) + goto bad_handle; /* No alloc */ + if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) + goto bad_handle; /* Not a native alloc */ + if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0)) + goto bad_handle; + /* Non-coherent memory cannot alias + coherent memory, and vice versa.*/ + + /* check size against stride */ + if (!ai[i].length) + goto bad_handle; /* must be > 0 */ + if (ai[i].length > stride) + goto bad_handle; /* can't be larger than the + stride */ + + alloc = aliasing_reg->gpu_alloc; + + /* check against the alloc's size */ + if (ai[i].offset > alloc->nents) + goto bad_handle; /* beyond end */ + if (ai[i].offset + ai[i].length > alloc->nents) + goto bad_handle; /* beyond end */ + + reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc); + reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; + reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset; + } + } + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { + /* Bind to a cookie */ + if (!kctx->cookies) { + dev_err(kctx->kbdev->dev, "No cookies available for allocation!"); + goto no_cookie; + } + /* return a cookie */ + gpu_va = __ffs(kctx->cookies); + kctx->cookies &= ~(1UL << gpu_va); + BUG_ON(kctx->pending_regions[gpu_va]); + kctx->pending_regions[gpu_va] = reg; + + /* relocate to correct base */ + gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); + gpu_va <<= PAGE_SHIFT; + } else /* we control the VA */ { +#else + if (1) { +#endif + if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) { + dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU"); + goto no_mmap; + } + /* return real GPU VA */ + gpu_va = reg->start_pfn << PAGE_SHIFT; + } + + reg->flags &= ~KBASE_REG_FREE; + reg->flags &= ~KBASE_REG_GROWABLE; + + kbase_gpu_vm_unlock(kctx); + + return gpu_va; + +#ifdef CONFIG_64BIT +no_cookie: +#endif +no_mmap: +bad_handle: + kbase_gpu_vm_unlock(kctx); +no_aliased_array: +invalid_flags: + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +no_alloc_obj: + kfree(reg); +no_reg: +bad_size: +bad_nents: +bad_stride: +bad_flags: + return 0; +} + +int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, + u64 *flags) +{ + struct kbase_va_region *reg; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(gpu_va); + KBASE_DEBUG_ASSERT(va_pages); + KBASE_DEBUG_ASSERT(flags); + +#ifdef CONFIG_64BIT + if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) + *flags |= BASE_MEM_SAME_VA; +#endif + + if (!kbase_check_import_flags(*flags)) { + dev_warn(kctx->kbdev->dev, + "kbase_mem_import called with bad flags (%llx)", + (unsigned long long)*flags); + goto bad_flags; + } + + if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + dev_warn(kctx->kbdev->dev, + "kbase_mem_import call required coherent mem when unavailable"); + goto bad_flags; + } + if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ + *flags &= ~BASE_MEM_COHERENT_SYSTEM; + } + + if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) { + dev_warn(kctx->kbdev->dev, + "padding is only supported for UMM"); + goto bad_flags; + } + + switch (type) { +#ifdef CONFIG_UMP + case BASE_MEM_IMPORT_TYPE_UMP: { + ump_secure_id id; + + if (get_user(id, (ump_secure_id __user *)phandle)) + reg = NULL; + else + reg = kbase_mem_from_ump(kctx, id, va_pages, flags); + } + break; +#endif /* CONFIG_UMP */ +#ifdef CONFIG_DMA_SHARED_BUFFER + case BASE_MEM_IMPORT_TYPE_UMM: { + int fd; + + if (get_user(fd, (int __user *)phandle)) + reg = NULL; + else + reg = kbase_mem_from_umm(kctx, fd, va_pages, flags, + padding); + } + break; +#endif /* CONFIG_DMA_SHARED_BUFFER */ + case BASE_MEM_IMPORT_TYPE_USER_BUFFER: { + struct base_mem_import_user_buffer user_buffer; + void __user *uptr; + + if (copy_from_user(&user_buffer, phandle, + sizeof(user_buffer))) { + reg = NULL; + } else { +#ifdef CONFIG_COMPAT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + uptr = compat_ptr(user_buffer.ptr.compat_value); + else +#endif + uptr = user_buffer.ptr.value; + + reg = kbase_mem_from_user_buffer(kctx, + (unsigned long)uptr, user_buffer.length, + va_pages, flags); + } + break; + } + default: { + reg = NULL; + break; + } + } + + if (!reg) + goto no_reg; + + kbase_gpu_vm_lock(kctx); + + /* mmap needed to setup VA? */ + if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) { + /* Bind to a cookie */ + if (!kctx->cookies) + goto no_cookie; + /* return a cookie */ + *gpu_va = __ffs(kctx->cookies); + kctx->cookies &= ~(1UL << *gpu_va); + BUG_ON(kctx->pending_regions[*gpu_va]); + kctx->pending_regions[*gpu_va] = reg; + + /* relocate to correct base */ + *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); + *gpu_va <<= PAGE_SHIFT; + + } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) { + /* we control the VA, mmap now to the GPU */ + if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0) + goto no_gpu_va; + /* return real GPU VA */ + *gpu_va = reg->start_pfn << PAGE_SHIFT; + } else { + /* we control the VA, but nothing to mmap yet */ + if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0) + goto no_gpu_va; + /* return real GPU VA */ + *gpu_va = reg->start_pfn << PAGE_SHIFT; + } + + /* clear out private flags */ + *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1); + + kbase_gpu_vm_unlock(kctx); + + return 0; + +no_gpu_va: +no_cookie: + kbase_gpu_vm_unlock(kctx); + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); + kfree(reg); +no_reg: +bad_flags: + *gpu_va = 0; + *va_pages = 0; + *flags = 0; + return -ENOMEM; +} + +int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages) +{ + phys_addr_t *phy_pages; + u64 delta = new_pages - old_pages; + int ret = 0; + + lockdep_assert_held(&kctx->reg_lock); + + /* Map the new pages into the GPU */ + phy_pages = kbase_get_gpu_phy_pages(reg); + ret = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages, + phy_pages + old_pages, delta, reg->flags); + + return ret; +} + +static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages) +{ + u64 gpu_va_start = reg->start_pfn; + + if (new_pages == old_pages) + /* Nothing to do */ + return; + + unmap_mapping_range(kctx->filp->f_inode->i_mapping, + (gpu_va_start + new_pages)<start_pfn + new_pages, delta); + + return ret; +} + +int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) +{ + u64 old_pages; + u64 delta; + int res = -EINVAL; + struct kbase_va_region *reg; + bool read_locked = false; + + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(gpu_addr != 0); + + if (gpu_addr & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid"); + return -EINVAL; + } + + down_write(¤t->mm->mmap_lock); + kbase_gpu_vm_lock(kctx); + + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (!reg || (reg->flags & KBASE_REG_FREE)) + goto out_unlock; + + KBASE_DEBUG_ASSERT(reg->cpu_alloc); + KBASE_DEBUG_ASSERT(reg->gpu_alloc); + + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) + goto out_unlock; + + if (0 == (reg->flags & KBASE_REG_GROWABLE)) + goto out_unlock; + + /* Would overflow the VA region */ + if (new_pages > reg->nr_pages) + goto out_unlock; + + /* can't be mapped more than once on the GPU */ + if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) + goto out_unlock; + /* can't grow regions which are ephemeral */ + if (reg->flags & KBASE_REG_DONT_NEED) + goto out_unlock; + + if (new_pages == reg->gpu_alloc->nents) { + /* no change */ + res = 0; + goto out_unlock; + } + + old_pages = kbase_reg_current_backed_size(reg); + if (new_pages > old_pages) { + delta = new_pages - old_pages; + + /* + * No update to the mm so downgrade the writer lock to a read + * lock so other readers aren't blocked after this point. + */ + downgrade_write(¤t->mm->mmap_lock); + read_locked = true; + + /* Allocate some more pages */ + if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) { + res = -ENOMEM; + goto out_unlock; + } + if (reg->cpu_alloc != reg->gpu_alloc) { + if (kbase_alloc_phy_pages_helper( + reg->gpu_alloc, delta) != 0) { + res = -ENOMEM; + kbase_free_phy_pages_helper(reg->cpu_alloc, + delta); + goto out_unlock; + } + } + + /* No update required for CPU mappings, that's done on fault. */ + + /* Update GPU mapping. */ + res = kbase_mem_grow_gpu_mapping(kctx, reg, + new_pages, old_pages); + + /* On error free the new pages */ + if (res) { + kbase_free_phy_pages_helper(reg->cpu_alloc, delta); + if (reg->cpu_alloc != reg->gpu_alloc) + kbase_free_phy_pages_helper(reg->gpu_alloc, + delta); + res = -ENOMEM; + goto out_unlock; + } + } else { + delta = old_pages - new_pages; + + /* Update all CPU mapping(s) */ + kbase_mem_shrink_cpu_mapping(kctx, reg, + new_pages, old_pages); + + /* Update the GPU mapping */ + res = kbase_mem_shrink_gpu_mapping(kctx, reg, + new_pages, old_pages); + if (res) { + res = -ENOMEM; + goto out_unlock; + } + + kbase_free_phy_pages_helper(reg->cpu_alloc, delta); + if (reg->cpu_alloc != reg->gpu_alloc) + kbase_free_phy_pages_helper(reg->gpu_alloc, delta); + } + +out_unlock: + kbase_gpu_vm_unlock(kctx); + if (read_locked) + up_read(¤t->mm->mmap_lock); + else + up_write(¤t->mm->mmap_lock); + + return res; +} + +static void kbase_cpu_vm_open(struct vm_area_struct *vma) +{ + struct kbase_cpu_mapping *map = vma->vm_private_data; + + KBASE_DEBUG_ASSERT(map); + KBASE_DEBUG_ASSERT(map->count > 0); + /* non-atomic as we're under Linux' mm lock */ + map->count++; +} + +static void kbase_cpu_vm_close(struct vm_area_struct *vma) +{ + struct kbase_cpu_mapping *map = vma->vm_private_data; + + KBASE_DEBUG_ASSERT(map); + KBASE_DEBUG_ASSERT(map->count > 0); + + /* non-atomic as we're under Linux' mm lock */ + if (--map->count) + return; + + KBASE_DEBUG_ASSERT(map->kctx); + KBASE_DEBUG_ASSERT(map->alloc); + + kbase_gpu_vm_lock(map->kctx); + + if (map->free_on_close) { + KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) == + KBASE_REG_ZONE_SAME_VA); + /* Avoid freeing memory on the process death which results in + * GPU Page Fault. Memory will be freed in kbase_destroy_context + */ + if (!(current->flags & PF_EXITING)) + kbase_mem_free_region(map->kctx, map->region); + } + + list_del(&map->mappings_list); + + kbase_gpu_vm_unlock(map->kctx); + + kbase_mem_phy_alloc_put(map->alloc); + kfree(map); +} + +KBASE_EXPORT_TEST_API(kbase_cpu_vm_close); + + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) +static vm_fault_t kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ +#else +static vm_fault_t kbase_cpu_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; +#endif + struct kbase_cpu_mapping *map = vma->vm_private_data; + pgoff_t rel_pgoff; + size_t i; + pgoff_t addr; + vm_fault_t ret = VM_FAULT_SIGBUS; + + KBASE_DEBUG_ASSERT(map); + KBASE_DEBUG_ASSERT(map->count > 0); + KBASE_DEBUG_ASSERT(map->kctx); + KBASE_DEBUG_ASSERT(map->alloc); + + rel_pgoff = vmf->pgoff - map->region->start_pfn; + + kbase_gpu_vm_lock(map->kctx); + if (rel_pgoff >= map->alloc->nents) + goto locked_bad_fault; + + /* Fault on access to DONT_NEED regions */ + if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED)) + goto locked_bad_fault; + + /* insert all valid pages from the fault location */ + i = rel_pgoff; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + addr = (pgoff_t)((uintptr_t)vmf->virtual_address >> PAGE_SHIFT); +#else + addr = (pgoff_t)(vmf->address >> PAGE_SHIFT); +#endif + while (i < map->alloc->nents && (addr < vma->vm_end >> PAGE_SHIFT)) { + ret = vmf_insert_pfn(vma, addr << PAGE_SHIFT, + PFN_DOWN(map->alloc->pages[i])); + if (ret != VM_FAULT_NOPAGE) + goto locked_bad_fault; + + i++; addr++; + } + + kbase_gpu_vm_unlock(map->kctx); + /* we resolved it, nothing for VM to do */ + return VM_FAULT_NOPAGE; + +locked_bad_fault: + kbase_gpu_vm_unlock(map->kctx); + return ret; +} + +const struct vm_operations_struct kbase_vm_ops = { + .open = kbase_cpu_vm_open, + .close = kbase_cpu_vm_close, + .fault = kbase_cpu_vm_fault +}; + +static int kbase_cpu_mmap(struct kbase_va_region *reg, struct vm_area_struct *vma, void *kaddr, size_t nr_pages, unsigned long aligned_offset, int free_on_close) +{ + struct kbase_cpu_mapping *map; + phys_addr_t *page_array; + int err = 0; + int i; + + map = kzalloc(sizeof(*map), GFP_KERNEL); + + if (!map) { + WARN_ON(1); + err = -ENOMEM; + goto out; + } + + /* + * VM_DONTCOPY - don't make this mapping available in fork'ed processes + * VM_DONTEXPAND - disable mremap on this region + * VM_IO - disables paging + * VM_DONTDUMP - Don't include in core dumps (3.7 only) + * VM_MIXEDMAP - Support mixing struct page*s and raw pfns. + * This is needed to support using the dedicated and + * the OS based memory backends together. + */ + /* + * This will need updating to propagate coherency flags + * See MIDBASE-1057 + */ + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) + vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; +#else + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; +#endif + vma->vm_ops = &kbase_vm_ops; + vma->vm_private_data = map; + + page_array = kbase_get_cpu_phy_pages(reg); + + if (!(reg->flags & KBASE_REG_CPU_CACHED) && + (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) { + /* We can't map vmalloc'd memory uncached. + * Other memory will have been returned from + * kbase_mem_pool which would be + * suitable for mapping uncached. + */ + BUG_ON(kaddr); + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + } + + if (!kaddr) { + unsigned long addr = vma->vm_start + aligned_offset; + u64 start_off = vma->vm_pgoff - reg->start_pfn + + (aligned_offset>>PAGE_SHIFT); + + vma->vm_flags |= VM_PFNMAP; + for (i = 0; i < nr_pages; i++) { + unsigned long pfn = PFN_DOWN(page_array[i + start_off]); + vm_fault_t ret; + + ret = vmf_insert_pfn(vma, addr, pfn); + if (WARN_ON(ret != VM_FAULT_NOPAGE)) { + if (ret == VM_FAULT_OOM) + err = -ENOMEM; + else + err = -EFAULT; + break; + } + + addr += PAGE_SIZE; + } + } else { + WARN_ON(aligned_offset); + /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */ + vma->vm_flags |= VM_MIXEDMAP; + /* vmalloc remaping is easy... */ + err = remap_vmalloc_range(vma, kaddr, 0); + WARN_ON(err); + } + + if (err) { + kfree(map); + goto out; + } + + map->region = reg; + map->free_on_close = free_on_close; + map->kctx = reg->kctx; + map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + map->count = 1; /* start with one ref */ + + if (reg->flags & KBASE_REG_CPU_CACHED) + map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; + + list_add(&map->mappings_list, &map->alloc->mappings); + + out: + return err; +} + +static int kbase_trace_buffer_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kaddr) +{ + struct kbase_va_region *new_reg; + u32 nr_pages; + size_t size; + int err = 0; + u32 *tb; + int owns_tb = 1; + + dev_dbg(kctx->kbdev->dev, "in %s\n", __func__); + size = (vma->vm_end - vma->vm_start); + nr_pages = size >> PAGE_SHIFT; + + if (!kctx->jctx.tb) { + KBASE_DEBUG_ASSERT(0 != size); + tb = vmalloc_user(size); + + if (NULL == tb) { + err = -ENOMEM; + goto out; + } + + err = kbase_device_trace_buffer_install(kctx, tb, size); + if (err) { + vfree(tb); + goto out; + } + } else { + err = -EINVAL; + goto out; + } + + *kaddr = kctx->jctx.tb; + + new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA); + if (!new_reg) { + err = -ENOMEM; + WARN_ON(1); + goto out_no_region; + } + + new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB); + if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { + err = -ENOMEM; + new_reg->cpu_alloc = NULL; + WARN_ON(1); + goto out_no_alloc; + } + + new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); + + new_reg->cpu_alloc->imported.kctx = kctx; + new_reg->flags &= ~KBASE_REG_FREE; + new_reg->flags |= KBASE_REG_CPU_CACHED; + + /* alloc now owns the tb */ + owns_tb = 0; + + if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { + err = -ENOMEM; + WARN_ON(1); + goto out_no_va_region; + } + + *reg = new_reg; + + /* map read only, noexec */ + vma->vm_flags &= ~(VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); + /* the rest of the flags is added by the cpu_mmap handler */ + + dev_dbg(kctx->kbdev->dev, "%s done\n", __func__); + return 0; + +out_no_va_region: +out_no_alloc: + kbase_free_alloced_region(new_reg); +out_no_region: + if (owns_tb) { + kbase_device_trace_buffer_uninstall(kctx); + vfree(tb); + } +out: + return err; +} + +static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kmap_addr) +{ + struct kbase_va_region *new_reg; + void *kaddr; + u32 nr_pages; + size_t size; + int err = 0; + + dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n"); + size = (vma->vm_end - vma->vm_start); + nr_pages = size >> PAGE_SHIFT; + + kaddr = kbase_mmu_dump(kctx, nr_pages); + + if (!kaddr) { + err = -ENOMEM; + goto out; + } + + new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA); + if (!new_reg) { + err = -ENOMEM; + WARN_ON(1); + goto out; + } + + new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW); + if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { + err = -ENOMEM; + new_reg->cpu_alloc = NULL; + WARN_ON(1); + goto out_no_alloc; + } + + new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); + + new_reg->flags &= ~KBASE_REG_FREE; + new_reg->flags |= KBASE_REG_CPU_CACHED; + if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { + err = -ENOMEM; + WARN_ON(1); + goto out_va_region; + } + + *kmap_addr = kaddr; + *reg = new_reg; + + dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n"); + return 0; + +out_no_alloc: +out_va_region: + kbase_free_alloced_region(new_reg); +out: + return err; +} + + +void kbase_os_mem_map_lock(struct kbase_context *kctx) +{ + struct mm_struct *mm = current->mm; + (void)kctx; + down_read(&mm->mmap_lock); +} + +void kbase_os_mem_map_unlock(struct kbase_context *kctx) +{ + struct mm_struct *mm = current->mm; + (void)kctx; + up_read(&mm->mmap_lock); +} + +static int kbasep_reg_mmap(struct kbase_context *kctx, + struct vm_area_struct *vma, + struct kbase_va_region **regm, + size_t *nr_pages, size_t *aligned_offset) + +{ + int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); + struct kbase_va_region *reg; + int err = 0; + + *aligned_offset = 0; + + dev_dbg(kctx->kbdev->dev, "in kbasep_reg_mmap\n"); + + /* SAME_VA stuff, fetch the right region */ + reg = kctx->pending_regions[cookie]; + if (!reg) { + err = -ENOMEM; + goto out; + } + + if ((reg->flags & KBASE_REG_GPU_NX) && (reg->nr_pages != *nr_pages)) { + /* incorrect mmap size */ + /* leave the cookie for a potential later + * mapping, or to be reclaimed later when the + * context is freed */ + err = -ENOMEM; + goto out; + } + + if ((vma->vm_flags & VM_READ && !(reg->flags & KBASE_REG_CPU_RD)) || + (vma->vm_flags & VM_WRITE && !(reg->flags & KBASE_REG_CPU_WR))) { + /* VM flags inconsistent with region flags */ + err = -EPERM; + dev_err(kctx->kbdev->dev, "%s:%d inconsistent VM flags\n", + __FILE__, __LINE__); + goto out; + } + + /* adjust down nr_pages to what we have physically */ + *nr_pages = kbase_reg_current_backed_size(reg); + + if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset, + reg->nr_pages, 1) != 0) { + dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__); + /* Unable to map in GPU space. */ + WARN_ON(1); + err = -ENOMEM; + goto out; + } + /* no need for the cookie anymore */ + kctx->pending_regions[cookie] = NULL; + kctx->cookies |= (1UL << cookie); + + /* + * Overwrite the offset with the region start_pfn, so we effectively + * map from offset 0 in the region. However subtract the aligned + * offset so that when user space trims the mapping the beginning of + * the trimmed VMA has the correct vm_pgoff; + */ + vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT); +out: + *regm = reg; + dev_dbg(kctx->kbdev->dev, "kbasep_reg_mmap done\n"); + + return err; +} + +int kbase_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct kbase_context *kctx = file->private_data; + struct kbase_va_region *reg = NULL; + void *kaddr = NULL; + size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + int err = 0; + int free_on_close = 0; + struct device *dev = kctx->kbdev->dev; + size_t aligned_offset = 0; + + dev_dbg(dev, "kbase_mmap\n"); + + /* strip away corresponding VM_MAY% flags to the VM_% flags requested */ + vma->vm_flags &= ~((vma->vm_flags & (VM_READ | VM_WRITE)) << 4); + + if (0 == nr_pages) { + err = -EINVAL; + goto out; + } + + if (!(vma->vm_flags & VM_SHARED)) { + err = -EINVAL; + goto out; + } + + kbase_gpu_vm_lock(kctx); + + if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) { + /* The non-mapped tracking helper page */ + err = kbase_tracking_page_setup(kctx, vma); + goto out_unlock; + } + + /* if not the MTP, verify that the MTP has been mapped */ + rcu_read_lock(); + /* catches both when the special page isn't present or + * when we've forked */ + if (rcu_dereference(kctx->process_mm) != current->mm) { + err = -EINVAL; + rcu_read_unlock(); + goto out_unlock; + } + rcu_read_unlock(); + + switch (vma->vm_pgoff) { + case PFN_DOWN(BASEP_MEM_INVALID_HANDLE): + case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE): + /* Illegal handle for direct map */ + err = -EINVAL; + goto out_unlock; + case PFN_DOWN(BASE_MEM_TRACE_BUFFER_HANDLE): + err = kbase_trace_buffer_mmap(kctx, vma, ®, &kaddr); + if (0 != err) + goto out_unlock; + dev_dbg(dev, "kbase_trace_buffer_mmap ok\n"); + /* free the region on munmap */ + free_on_close = 1; + break; + case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE): + /* MMU dump */ + err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr); + if (0 != err) + goto out_unlock; + /* free the region on munmap */ + free_on_close = 1; + break; + case PFN_DOWN(BASE_MEM_COOKIE_BASE) ... + PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { + err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, + &aligned_offset); + if (0 != err) + goto out_unlock; + /* free the region on munmap */ + free_on_close = 1; + break; + } + default: { + reg = kbase_region_tracker_find_region_enclosing_address(kctx, + (u64)vma->vm_pgoff << PAGE_SHIFT); + + if (reg && !(reg->flags & KBASE_REG_FREE)) { + /* will this mapping overflow the size of the region? */ + if (nr_pages > (reg->nr_pages - + (vma->vm_pgoff - reg->start_pfn))) { + err = -ENOMEM; + goto out_unlock; + } + + if ((vma->vm_flags & VM_READ && + !(reg->flags & KBASE_REG_CPU_RD)) || + (vma->vm_flags & VM_WRITE && + !(reg->flags & KBASE_REG_CPU_WR))) { + /* VM flags inconsistent with region flags */ + err = -EPERM; + dev_err(dev, "%s:%d inconsistent VM flags\n", + __FILE__, __LINE__); + goto out_unlock; + } + +#ifdef CONFIG_DMA_SHARED_BUFFER + if (KBASE_MEM_TYPE_IMPORTED_UMM == + reg->cpu_alloc->type) { + err = dma_buf_mmap( + reg->cpu_alloc->imported.umm.dma_buf, + vma, vma->vm_pgoff - reg->start_pfn); + goto out_unlock; + } +#endif /* CONFIG_DMA_SHARED_BUFFER */ + + /* limit what we map to the amount currently backed */ + if (reg->cpu_alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) { + if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents) + nr_pages = 0; + else + nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn); + } + } else { + err = -ENOMEM; + goto out_unlock; + } + } /* default */ + } /* switch */ + + err = kbase_cpu_mmap(reg, vma, kaddr, nr_pages, aligned_offset, free_on_close); + + if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) { + /* MMU dump - userspace should now have a reference on + * the pages, so we can now free the kernel mapping */ + vfree(kaddr); + } + +out_unlock: + kbase_gpu_vm_unlock(kctx); +out: + if (err) + dev_err(dev, "mmap failed %d\n", err); + + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mmap); + +void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, + unsigned long prot_request, struct kbase_vmap_struct *map) +{ + struct kbase_va_region *reg; + unsigned long page_index; + unsigned int offset = gpu_addr & ~PAGE_MASK; + size_t page_count = PFN_UP(offset + size); + phys_addr_t *page_array; + struct page **pages; + void *cpu_addr = NULL; + pgprot_t prot; + size_t i; + bool sync_needed; + + if (!size || !map) + return NULL; + + /* check if page_count calculation will wrap */ + if (size > ((size_t)-1 / PAGE_SIZE)) + return NULL; + + kbase_gpu_vm_lock(kctx); + + reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr); + if (!reg || (reg->flags & KBASE_REG_FREE)) + goto out_unlock; + + page_index = (gpu_addr >> PAGE_SHIFT) - reg->start_pfn; + + /* check if page_index + page_count will wrap */ + if (-1UL - page_count < page_index) + goto out_unlock; + + if (page_index + page_count > kbase_reg_current_backed_size(reg)) + goto out_unlock; + + if (reg->flags & KBASE_REG_DONT_NEED) + goto out_unlock; + + /* check access permissions can be satisfied + * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} */ + if ((reg->flags & prot_request) != prot_request) + goto out_unlock; + + page_array = kbase_get_cpu_phy_pages(reg); + if (!page_array) + goto out_unlock; + + pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); + if (!pages) + goto out_unlock; + + for (i = 0; i < page_count; i++) + pages[i] = pfn_to_page(PFN_DOWN(page_array[page_index + i])); + + prot = PAGE_KERNEL; + if (!(reg->flags & KBASE_REG_CPU_CACHED)) { + /* Map uncached */ + prot = pgprot_writecombine(prot); + } + /* Note: enforcing a RO prot_request onto prot is not done, since: + * - CPU-arch-specific integration required + * - kbase_vmap() requires no access checks to be made/enforced */ + + cpu_addr = vmap(pages, page_count, VM_MAP, prot); + + kfree(pages); + + if (!cpu_addr) + goto out_unlock; + + map->gpu_addr = gpu_addr; + map->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index]; + map->gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index]; + map->addr = (void *)((uintptr_t)cpu_addr + offset); + map->size = size; + map->is_cached = (reg->flags & KBASE_REG_CPU_CACHED) != 0; + sync_needed = map->is_cached; + +#ifdef CONFIG_MALI_COH_KERN + /* kernel can use coherent memory if supported */ + if (kctx->kbdev->system_coherency == COHERENCY_ACE) + sync_needed = false; +#endif + + if (sync_needed) { + /* Sync first page */ + size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); + phys_addr_t cpu_pa = map->cpu_pages[0]; + phys_addr_t gpu_pa = map->gpu_pages[0]; + + kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz, + KBASE_SYNC_TO_CPU); + + /* Sync middle pages (if any) */ + for (i = 1; page_count > 2 && i < page_count - 1; i++) { + cpu_pa = map->cpu_pages[i]; + gpu_pa = map->gpu_pages[i]; + kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE, + KBASE_SYNC_TO_CPU); + } + + /* Sync last page (if any) */ + if (page_count > 1) { + cpu_pa = map->cpu_pages[page_count - 1]; + gpu_pa = map->gpu_pages[page_count - 1]; + sz = ((offset + size - 1) & ~PAGE_MASK) + 1; + kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, + KBASE_SYNC_TO_CPU); + } + } + kbase_gpu_vm_unlock(kctx); + + return map->addr; + +out_unlock: + kbase_gpu_vm_unlock(kctx); + return NULL; +} + +void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, + struct kbase_vmap_struct *map) +{ + /* 0 is specified for prot_request to indicate no access checks should + * be made. + * + * As mentioned in kbase_vmap_prot() this means that a kernel-side + * CPU-RO mapping is not enforced to allow this to work */ + return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map); +} +KBASE_EXPORT_TEST_API(kbase_vmap); + +void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map) +{ + void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK); + bool sync_needed = map->is_cached; + vunmap(addr); +#ifdef CONFIG_MALI_COH_KERN + /* kernel can use coherent memory if supported */ + if (kctx->kbdev->system_coherency == COHERENCY_ACE) + sync_needed = false; +#endif + if (sync_needed) { + off_t offset = (uintptr_t)map->addr & ~PAGE_MASK; + size_t size = map->size; + size_t page_count = PFN_UP(offset + size); + size_t i; + + /* Sync first page */ + size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); + phys_addr_t cpu_pa = map->cpu_pages[0]; + phys_addr_t gpu_pa = map->gpu_pages[0]; + + kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz, + KBASE_SYNC_TO_DEVICE); + + /* Sync middle pages (if any) */ + for (i = 1; page_count > 2 && i < page_count - 1; i++) { + cpu_pa = map->cpu_pages[i]; + gpu_pa = map->gpu_pages[i]; + kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE, + KBASE_SYNC_TO_DEVICE); + } + + /* Sync last page (if any) */ + if (page_count > 1) { + cpu_pa = map->cpu_pages[page_count - 1]; + gpu_pa = map->gpu_pages[page_count - 1]; + sz = ((offset + size - 1) & ~PAGE_MASK) + 1; + kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, + KBASE_SYNC_TO_DEVICE); + } + } + map->gpu_addr = 0; + map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc); + map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc); + map->cpu_pages = NULL; + map->gpu_pages = NULL; + map->addr = NULL; + map->size = 0; + map->is_cached = false; +} +KBASE_EXPORT_TEST_API(kbase_vunmap); + +void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages) +{ + struct mm_struct *mm; + + rcu_read_lock(); + mm = rcu_dereference(kctx->process_mm); + if (mm) { + atomic_add(pages, &kctx->nonmapped_pages); +#ifdef SPLIT_RSS_COUNTING + add_mm_counter(mm, MM_FILEPAGES, pages); +#else + spin_lock(&mm->page_table_lock); + add_mm_counter(mm, MM_FILEPAGES, pages); + spin_unlock(&mm->page_table_lock); +#endif + } + rcu_read_unlock(); +} + +static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx) +{ + int pages; + struct mm_struct *mm; + + spin_lock(&kctx->mm_update_lock); + mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock)); + if (!mm) { + spin_unlock(&kctx->mm_update_lock); + return; + } + + rcu_assign_pointer(kctx->process_mm, NULL); + spin_unlock(&kctx->mm_update_lock); + synchronize_rcu(); + + pages = atomic_xchg(&kctx->nonmapped_pages, 0); +#ifdef SPLIT_RSS_COUNTING + add_mm_counter(mm, MM_FILEPAGES, -pages); +#else + spin_lock(&mm->page_table_lock); + add_mm_counter(mm, MM_FILEPAGES, -pages); + spin_unlock(&mm->page_table_lock); +#endif +} + +static void kbase_special_vm_close(struct vm_area_struct *vma) +{ + struct kbase_context *kctx; + + kctx = vma->vm_private_data; + kbasep_os_process_page_usage_drain(kctx); +} + +static const struct vm_operations_struct kbase_vm_special_ops = { + .close = kbase_special_vm_close, +}; + +static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma) +{ + /* check that this is the only tracking page */ + spin_lock(&kctx->mm_update_lock); + if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) { + spin_unlock(&kctx->mm_update_lock); + return -EFAULT; + } + + rcu_assign_pointer(kctx->process_mm, current->mm); + + spin_unlock(&kctx->mm_update_lock); + + /* no real access */ + vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; +#else + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; +#endif + vma->vm_ops = &kbase_vm_special_ops; + vma->vm_private_data = kctx; + + return 0; +} +void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle) +{ + int i; + int res; + void *va; + dma_addr_t dma_pa; + struct kbase_va_region *reg; + phys_addr_t *page_array; +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) + unsigned long attrs = DMA_ATTR_WRITE_COMBINE; +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) + DEFINE_DMA_ATTRS(attrs); +#endif + + u32 pages = ((size - 1) >> PAGE_SHIFT) + 1; + u32 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR | + BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR; + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(0 != size); + KBASE_DEBUG_ASSERT(0 != pages); + + if (size == 0) + goto err; + + /* All the alloc calls return zeroed memory */ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) + va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, + attrs); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) + dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); + va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, + &attrs); +#else + va = dma_alloc_writecombine(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL); +#endif + if (!va) + goto err; + + /* Store the state so we can free it later. */ + handle->cpu_va = va; + handle->dma_pa = dma_pa; + handle->size = size; + + + reg = kbase_alloc_free_region(kctx, 0, pages, KBASE_REG_ZONE_SAME_VA); + if (!reg) + goto no_reg; + + reg->flags &= ~KBASE_REG_FREE; + if (kbase_update_region_flags(kctx, reg, flags) != 0) + goto invalid_flags; + + reg->cpu_alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW); + if (IS_ERR_OR_NULL(reg->cpu_alloc)) + goto no_alloc; + + reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); + + page_array = kbase_get_cpu_phy_pages(reg); + + for (i = 0; i < pages; i++) + page_array[i] = dma_pa + (i << PAGE_SHIFT); + + reg->cpu_alloc->nents = pages; + + kbase_gpu_vm_lock(kctx); + res = kbase_gpu_mmap(kctx, reg, (uintptr_t) va, pages, 1); + kbase_gpu_vm_unlock(kctx); + if (res) + goto no_mmap; + + return va; + +no_mmap: + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); +no_alloc: +invalid_flags: + kfree(reg); +no_reg: +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) + dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, attrs); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) + dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, &attrs); +#else + dma_free_writecombine(kctx->kbdev->dev, size, va, dma_pa); +#endif +err: + return NULL; +} +KBASE_EXPORT_SYMBOL(kbase_va_alloc); + +void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle) +{ + struct kbase_va_region *reg; + int err; +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + DEFINE_DMA_ATTRS(attrs); +#endif + + KBASE_DEBUG_ASSERT(kctx != NULL); + KBASE_DEBUG_ASSERT(handle->cpu_va != NULL); + + kbase_gpu_vm_lock(kctx); + reg = kbase_region_tracker_find_region_base_address(kctx, (uintptr_t)handle->cpu_va); + KBASE_DEBUG_ASSERT(reg); + err = kbase_gpu_munmap(kctx, reg); + kbase_gpu_vm_unlock(kctx); + KBASE_DEBUG_ASSERT(!err); + + kbase_mem_phy_alloc_put(reg->cpu_alloc); + kbase_mem_phy_alloc_put(reg->gpu_alloc); + kfree(reg); + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) + dma_free_attrs(kctx->kbdev->dev, handle->size, + handle->cpu_va, handle->dma_pa, DMA_ATTR_WRITE_COMBINE); +#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) + dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); + dma_free_attrs(kctx->kbdev->dev, handle->size, + handle->cpu_va, handle->dma_pa, &attrs); +#else + dma_free_writecombine(kctx->kbdev->dev, handle->size, + handle->cpu_va, handle->dma_pa); +#endif +} +KBASE_EXPORT_SYMBOL(kbase_va_free); + diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h new file mode 100755 index 000000000000..33b3554f9d82 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h @@ -0,0 +1,231 @@ +/* + * + * (C) COPYRIGHT 2010, 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mem_linux.h + * Base kernel memory APIs, Linux implementation. + */ + +#ifndef _KBASE_MEM_LINUX_H_ +#define _KBASE_MEM_LINUX_H_ + +/** A HWC dump mapping */ +struct kbase_hwc_dma_mapping { + void *cpu_va; + dma_addr_t dma_pa; + size_t size; +}; + +struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, + u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, + u64 *gpu_va); +int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 *const pages); +int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, + u64 *flags); +u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages); +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask); + +/** + * kbase_mem_commit - Change the physical backing size of a region + * + * @kctx: The kernel context + * @gpu_addr: Handle to the memory region + * @new_pages: Number of physical pages to back the region with + * + * Return: 0 on success or error code + */ +int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages); + +int kbase_mmap(struct file *file, struct vm_area_struct *vma); + +/** + * kbase_mem_evictable_init - Initialize the Ephemeral memory the eviction + * mechanism. + * @kctx: The kbase context to initialize. + * + * Return: Zero on success or -errno on failure. + */ +int kbase_mem_evictable_init(struct kbase_context *kctx); + +/** + * kbase_mem_evictable_deinit - De-initialize the Ephemeral memory eviction + * mechanism. + * @kctx: The kbase context to de-initialize. + */ +void kbase_mem_evictable_deinit(struct kbase_context *kctx); + +/** + * kbase_mem_grow_gpu_mapping - Grow the GPU mapping of an allocation + * @kctx: Context the region belongs to + * @reg: The GPU region + * @new_pages: The number of pages after the grow + * @old_pages: The number of pages before the grow + * + * Return: 0 on success, -errno on error. + * + * Expand the GPU mapping to encompass the new psychical pages which have + * been added to the allocation. + * + * Note: Caller must be holding the region lock. + */ +int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, + struct kbase_va_region *reg, + u64 new_pages, u64 old_pages); + +/** + * kbase_mem_evictable_make - Make a physical allocation eligible for eviction + * @gpu_alloc: The physical allocation to make evictable + * + * Return: 0 on success, -errno on error. + * + * Take the provided region and make all the physical pages within it + * reclaimable by the kernel, updating the per-process VM stats as well. + * Remove any CPU mappings (as these can't be removed in the shrinker callback + * as mmap_lock might already be taken) but leave the GPU mapping intact as + * and until the shrinker reclaims the allocation. + * + * Note: Must be called with the region lock of the containing context. + */ +int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc); + +/** + * kbase_mem_evictable_unmake - Remove a physical allocations eligibility for + * eviction. + * @alloc: The physical allocation to remove eviction eligibility from. + * + * Return: True if the allocation had its backing restored and false if + * it hasn't. + * + * Make the physical pages in the region no longer reclaimable and update the + * per-process stats, if the shrinker has already evicted the memory then + * re-allocate it if the region is still alive. + * + * Note: Must be called with the region lock of the containing context. + */ +bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *alloc); + +struct kbase_vmap_struct { + u64 gpu_addr; + struct kbase_mem_phy_alloc *cpu_alloc; + struct kbase_mem_phy_alloc *gpu_alloc; + phys_addr_t *cpu_pages; + phys_addr_t *gpu_pages; + void *addr; + size_t size; + bool is_cached; +}; + + +/** + * kbase_vmap_prot - Map a GPU VA range into the kernel safely, only if the + * requested access permissions are supported + * @kctx: Context the VA range belongs to + * @gpu_addr: Start address of VA range + * @size: Size of VA range + * @prot_request: Flags indicating how the caller will then access the memory + * @map: Structure to be given to kbase_vunmap() on freeing + * + * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error + * + * Map a GPU VA Range into the kernel. The VA range must be contained within a + * GPU memory region. Appropriate CPU cache-flushing operations are made as + * required, dependent on the CPU mapping for the memory region. + * + * This is safer than using kmap() on the pages directly, + * because the pages here are refcounted to prevent freeing (and hence reuse + * elsewhere in the system) until an kbase_vunmap() + * + * The flags in @prot_request should use KBASE_REG_{CPU,GPU}_{RD,WR}, to check + * whether the region should allow the intended access, and return an error if + * disallowed. This is essential for security of imported memory, particularly + * a user buf from SHM mapped into the process as RO. In that case, write + * access must be checked if the intention is for kernel to write to the + * memory. + * + * The checks are also there to help catch access errors on memory where + * security is not a concern: imported memory that is always RW, and memory + * that was allocated and owned by the process attached to @kctx. In this case, + * it helps to identify memory that was was mapped with the wrong access type. + * + * Note: KBASE_REG_GPU_{RD,WR} flags are currently supported for legacy cases + * where either the security of memory is solely dependent on those flags, or + * when userspace code was expecting only the GPU to access the memory (e.g. HW + * workarounds). + * + */ +void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, + unsigned long prot_request, struct kbase_vmap_struct *map); + +/** + * kbase_vmap - Map a GPU VA range into the kernel safely + * @kctx: Context the VA range belongs to + * @gpu_addr: Start address of VA range + * @size: Size of VA range + * @map: Structure to be given to kbase_vunmap() on freeing + * + * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error + * + * Map a GPU VA Range into the kernel. The VA range must be contained within a + * GPU memory region. Appropriate CPU cache-flushing operations are made as + * required, dependent on the CPU mapping for the memory region. + * + * This is safer than using kmap() on the pages directly, + * because the pages here are refcounted to prevent freeing (and hence reuse + * elsewhere in the system) until an kbase_vunmap() + * + * kbase_vmap_prot() should be used in preference, since kbase_vmap() makes no + * checks to ensure the security of e.g. imported user bufs from RO SHM. + */ +void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, + struct kbase_vmap_struct *map); + +/** + * kbase_vunmap - Unmap a GPU VA range from the kernel + * @kctx: Context the VA range belongs to + * @map: Structure describing the mapping from the corresponding kbase_vmap() + * call + * + * Unmaps a GPU VA range from the kernel, given its @map structure obtained + * from kbase_vmap(). Appropriate CPU cache-flushing operations are made as + * required, dependent on the CPU mapping for the memory region. + * + * The reference taken on pages during kbase_vmap() is released. + */ +void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map); + +/** @brief Allocate memory from kernel space and map it onto the GPU + * + * @param kctx The context used for the allocation/mapping + * @param size The size of the allocation in bytes + * @param handle An opaque structure used to contain the state needed to free the memory + * @return the VA for kernel space and GPU MMU + */ +void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle); + +/** @brief Free/unmap memory allocated by kbase_va_alloc + * + * @param kctx The context used for the allocation/mapping + * @param handle An opaque structure returned by the kbase_va_alloc function. + */ +void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle); + +extern const struct vm_operations_struct kbase_vm_ops; + +#endif /* _KBASE_MEM_LINUX_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h b/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h new file mode 100755 index 000000000000..9725fd3f05df --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h @@ -0,0 +1,45 @@ +/* + * + * (C) COPYRIGHT 2012-2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_MEM_LOWLEVEL_H +#define _KBASE_MEM_LOWLEVEL_H + +#ifndef _KBASE_H_ +#error "Don't include this file directly, use mali_kbase.h instead" +#endif + +#include + +/** + * @brief Flags for kbase_phy_allocator_pages_alloc + */ +#define KBASE_PHY_PAGES_FLAG_DEFAULT (0) /** Default allocation flag */ +#define KBASE_PHY_PAGES_FLAG_CLEAR (1 << 0) /** Clear the pages after allocation */ +#define KBASE_PHY_PAGES_FLAG_POISON (1 << 1) /** Fill the memory with a poison value */ + +#define KBASE_PHY_PAGES_SUPPORTED_FLAGS (KBASE_PHY_PAGES_FLAG_DEFAULT|KBASE_PHY_PAGES_FLAG_CLEAR|KBASE_PHY_PAGES_FLAG_POISON) + +#define KBASE_PHY_PAGES_POISON_VALUE 0xFD /** Value to fill the memory with when KBASE_PHY_PAGES_FLAG_POISON is set */ + +enum kbase_sync_type { + KBASE_SYNC_TO_CPU, + KBASE_SYNC_TO_DEVICE +}; + +#endif /* _KBASE_LOWLEVEL_H */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c b/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c new file mode 100755 index 000000000000..a8269940a037 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c @@ -0,0 +1,569 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include +#include +#include +#include +#include + +#define pool_dbg(pool, format, ...) \ + dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \ + (pool->next_pool) ? "kctx" : "kbdev", \ + kbase_mem_pool_size(pool), \ + kbase_mem_pool_max_size(pool), \ + ##__VA_ARGS__) + +#define NOT_DIRTY false +#define NOT_RECLAIMED false + +static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool) +{ + spin_lock(&pool->pool_lock); +} + +static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) +{ + spin_unlock(&pool->pool_lock); +} + +static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool) +{ + ssize_t max_size = kbase_mem_pool_max_size(pool); + ssize_t cur_size = kbase_mem_pool_size(pool); + + return max(max_size - cur_size, (ssize_t)0); +} + +static bool kbase_mem_pool_is_full(struct kbase_mem_pool *pool) +{ + return kbase_mem_pool_size(pool) >= kbase_mem_pool_max_size(pool); +} + +static bool kbase_mem_pool_is_empty(struct kbase_mem_pool *pool) +{ + return kbase_mem_pool_size(pool) == 0; +} + +static void kbase_mem_pool_add_locked(struct kbase_mem_pool *pool, + struct page *p) +{ + lockdep_assert_held(&pool->pool_lock); + + list_add(&p->lru, &pool->page_list); + pool->cur_size++; + + pool_dbg(pool, "added page\n"); +} + +static void kbase_mem_pool_add(struct kbase_mem_pool *pool, struct page *p) +{ + kbase_mem_pool_lock(pool); + kbase_mem_pool_add_locked(pool, p); + kbase_mem_pool_unlock(pool); +} + +static void kbase_mem_pool_add_list_locked(struct kbase_mem_pool *pool, + struct list_head *page_list, size_t nr_pages) +{ + lockdep_assert_held(&pool->pool_lock); + + list_splice(page_list, &pool->page_list); + pool->cur_size += nr_pages; + + pool_dbg(pool, "added %zu pages\n", nr_pages); +} + +static void kbase_mem_pool_add_list(struct kbase_mem_pool *pool, + struct list_head *page_list, size_t nr_pages) +{ + kbase_mem_pool_lock(pool); + kbase_mem_pool_add_list_locked(pool, page_list, nr_pages); + kbase_mem_pool_unlock(pool); +} + +static struct page *kbase_mem_pool_remove_locked(struct kbase_mem_pool *pool) +{ + struct page *p; + + lockdep_assert_held(&pool->pool_lock); + + if (kbase_mem_pool_is_empty(pool)) + return NULL; + + p = list_first_entry(&pool->page_list, struct page, lru); + list_del_init(&p->lru); + pool->cur_size--; + + pool_dbg(pool, "removed page\n"); + + return p; +} + +static struct page *kbase_mem_pool_remove(struct kbase_mem_pool *pool) +{ + struct page *p; + + kbase_mem_pool_lock(pool); + p = kbase_mem_pool_remove_locked(pool); + kbase_mem_pool_unlock(pool); + + return p; +} + +static void kbase_mem_pool_sync_page(struct kbase_mem_pool *pool, + struct page *p) +{ + struct device *dev = pool->kbdev->dev; + + dma_sync_single_for_device(dev, kbase_dma_addr(p), + PAGE_SIZE, DMA_BIDIRECTIONAL); +} + +static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool, + struct page *p) +{ + clear_highpage(p); + kbase_mem_pool_sync_page(pool, p); +} + +static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, + struct page *p) +{ + /* Zero page before spilling */ + kbase_mem_pool_zero_page(next_pool, p); + + kbase_mem_pool_add(next_pool, p); +} + +struct page *kbase_mem_alloc_page(struct kbase_device *kbdev) +{ + struct page *p; + gfp_t gfp; + struct device *dev = kbdev->dev; + dma_addr_t dma_addr; + +#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) + /* DMA cache sync fails for HIGHMEM before 3.5 on ARM */ + gfp = GFP_USER | __GFP_ZERO; +#else + gfp = GFP_HIGHUSER | __GFP_ZERO; +#endif + + if (current->flags & PF_KTHREAD) { + /* Don't trigger OOM killer from kernel threads, e.g. when + * growing memory on GPU page fault */ + gfp |= __GFP_NORETRY; + } + + p = alloc_page(gfp); + if (!p) + return NULL; + + dma_addr = dma_map_page(dev, p, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) { + __free_page(p); + return NULL; + } + + WARN_ON(dma_addr != page_to_phys(p)); + + kbase_set_dma_addr(p, dma_addr); + + return p; +} + +static void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, + struct page *p) +{ + struct device *dev = pool->kbdev->dev; + dma_addr_t dma_addr = kbase_dma_addr(p); + + dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + kbase_clear_dma_addr(p); + __free_page(p); + + pool_dbg(pool, "freed page to kernel\n"); +} + +static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool, + size_t nr_to_shrink) +{ + struct page *p; + size_t i; + + lockdep_assert_held(&pool->pool_lock); + + for (i = 0; i < nr_to_shrink && !kbase_mem_pool_is_empty(pool); i++) { + p = kbase_mem_pool_remove_locked(pool); + kbase_mem_pool_free_page(pool, p); + } + + return i; +} + +static size_t kbase_mem_pool_shrink(struct kbase_mem_pool *pool, + size_t nr_to_shrink) +{ + size_t nr_freed; + + kbase_mem_pool_lock(pool); + nr_freed = kbase_mem_pool_shrink_locked(pool, nr_to_shrink); + kbase_mem_pool_unlock(pool); + + return nr_freed; +} + +int kbase_mem_pool_grow(struct kbase_mem_pool *pool, + size_t nr_to_grow) +{ + struct page *p; + size_t i; + + for (i = 0; i < nr_to_grow; i++) { + p = kbase_mem_alloc_page(pool->kbdev); + if (!p) + return -ENOMEM; + kbase_mem_pool_add(pool, p); + } + + return 0; +} + +void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size) +{ + size_t cur_size; + + cur_size = kbase_mem_pool_size(pool); + + if (new_size > pool->max_size) + new_size = pool->max_size; + + if (new_size < cur_size) + kbase_mem_pool_shrink(pool, cur_size - new_size); + else if (new_size > cur_size) + kbase_mem_pool_grow(pool, new_size - cur_size); +} + +void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size) +{ + size_t cur_size; + size_t nr_to_shrink; + + kbase_mem_pool_lock(pool); + + pool->max_size = max_size; + + cur_size = kbase_mem_pool_size(pool); + if (max_size < cur_size) { + nr_to_shrink = cur_size - max_size; + kbase_mem_pool_shrink_locked(pool, nr_to_shrink); + } + + kbase_mem_pool_unlock(pool); +} + + +static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_mem_pool *pool; + + pool = container_of(s, struct kbase_mem_pool, reclaim); + pool_dbg(pool, "reclaim count: %zu\n", kbase_mem_pool_size(pool)); + return kbase_mem_pool_size(pool); +} + +static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, + struct shrink_control *sc) +{ + struct kbase_mem_pool *pool; + unsigned long freed; + + pool = container_of(s, struct kbase_mem_pool, reclaim); + + pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan); + + freed = kbase_mem_pool_shrink(pool, sc->nr_to_scan); + + pool_dbg(pool, "reclaim freed %ld pages\n", freed); + + return freed; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static int kbase_mem_pool_reclaim_shrink(struct shrinker *s, + struct shrink_control *sc) +{ + if (sc->nr_to_scan == 0) + return kbase_mem_pool_reclaim_count_objects(s, sc); + + return kbase_mem_pool_reclaim_scan_objects(s, sc); +} +#endif + +int kbase_mem_pool_init(struct kbase_mem_pool *pool, + size_t max_size, + struct kbase_device *kbdev, + struct kbase_mem_pool *next_pool) +{ + pool->cur_size = 0; + pool->max_size = max_size; + pool->kbdev = kbdev; + pool->next_pool = next_pool; + + spin_lock_init(&pool->pool_lock); + INIT_LIST_HEAD(&pool->page_list); + + /* Register shrinker */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) + pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink; +#else + pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects; + pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects; +#endif + pool->reclaim.seeks = DEFAULT_SEEKS; + /* Kernel versions prior to 3.1 : + * struct shrinker does not define batch */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) + pool->reclaim.batch = 0; +#endif + register_shrinker(&pool->reclaim); + + pool_dbg(pool, "initialized\n"); + + return 0; +} + +void kbase_mem_pool_term(struct kbase_mem_pool *pool) +{ + struct kbase_mem_pool *next_pool = pool->next_pool; + struct page *p; + size_t nr_to_spill = 0; + LIST_HEAD(spill_list); + int i; + + pool_dbg(pool, "terminate()\n"); + + unregister_shrinker(&pool->reclaim); + + kbase_mem_pool_lock(pool); + pool->max_size = 0; + + if (next_pool && !kbase_mem_pool_is_full(next_pool)) { + /* Spill to next pool (may overspill) */ + nr_to_spill = kbase_mem_pool_capacity(next_pool); + nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill); + + /* Zero pages first without holding the next_pool lock */ + for (i = 0; i < nr_to_spill; i++) { + p = kbase_mem_pool_remove_locked(pool); + kbase_mem_pool_zero_page(pool, p); + list_add(&p->lru, &spill_list); + } + } + + while (!kbase_mem_pool_is_empty(pool)) { + /* Free remaining pages to kernel */ + p = kbase_mem_pool_remove_locked(pool); + kbase_mem_pool_free_page(pool, p); + } + + kbase_mem_pool_unlock(pool); + + if (next_pool && nr_to_spill) { + /* Add new page list to next_pool */ + kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill); + + pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill); + } + + pool_dbg(pool, "terminated\n"); +} + +struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool) +{ + struct page *p; + + do { + pool_dbg(pool, "alloc()\n"); + p = kbase_mem_pool_remove(pool); + + if (p) + return p; + + pool = pool->next_pool; + } while (pool); + + return NULL; +} + +void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, + bool dirty) +{ + struct kbase_mem_pool *next_pool = pool->next_pool; + + pool_dbg(pool, "free()\n"); + + if (!kbase_mem_pool_is_full(pool)) { + /* Add to our own pool */ + if (dirty) + kbase_mem_pool_sync_page(pool, p); + + kbase_mem_pool_add(pool, p); + } else if (next_pool && !kbase_mem_pool_is_full(next_pool)) { + /* Spill to next pool */ + kbase_mem_pool_spill(next_pool, p); + } else { + /* Free page */ + kbase_mem_pool_free_page(pool, p); + } +} + +int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages, + phys_addr_t *pages) +{ + struct page *p; + size_t nr_from_pool; + size_t i; + int err = -ENOMEM; + + pool_dbg(pool, "alloc_pages(%zu):\n", nr_pages); + + /* Get pages from this pool */ + kbase_mem_pool_lock(pool); + nr_from_pool = min(nr_pages, kbase_mem_pool_size(pool)); + for (i = 0; i < nr_from_pool; i++) { + p = kbase_mem_pool_remove_locked(pool); + pages[i] = page_to_phys(p); + } + kbase_mem_pool_unlock(pool); + + if (i != nr_pages && pool->next_pool) { + /* Allocate via next pool */ + err = kbase_mem_pool_alloc_pages(pool->next_pool, + nr_pages - i, pages + i); + + if (err) + goto err_rollback; + + i += nr_pages - i; + } + + /* Get any remaining pages from kernel */ + for (; i < nr_pages; i++) { + p = kbase_mem_alloc_page(pool->kbdev); + if (!p) + goto err_rollback; + pages[i] = page_to_phys(p); + } + + pool_dbg(pool, "alloc_pages(%zu) done\n", nr_pages); + + return 0; + +err_rollback: + kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED); + return err; +} + +static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, + size_t nr_pages, phys_addr_t *pages, bool zero, bool sync) +{ + struct page *p; + size_t nr_to_pool = 0; + LIST_HEAD(new_page_list); + size_t i; + + if (!nr_pages) + return; + + pool_dbg(pool, "add_array(%zu, zero=%d, sync=%d):\n", + nr_pages, zero, sync); + + /* Zero/sync pages first without holding the pool lock */ + for (i = 0; i < nr_pages; i++) { + if (unlikely(!pages[i])) + continue; + + p = phys_to_page(pages[i]); + + if (zero) + kbase_mem_pool_zero_page(pool, p); + else if (sync) + kbase_mem_pool_sync_page(pool, p); + + list_add(&p->lru, &new_page_list); + nr_to_pool++; + pages[i] = 0; + } + + /* Add new page list to pool */ + kbase_mem_pool_add_list(pool, &new_page_list, nr_to_pool); + + pool_dbg(pool, "add_array(%zu) added %zu pages\n", + nr_pages, nr_to_pool); +} + +void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, + phys_addr_t *pages, bool dirty, bool reclaimed) +{ + struct kbase_mem_pool *next_pool = pool->next_pool; + struct page *p; + size_t nr_to_pool; + LIST_HEAD(to_pool_list); + size_t i = 0; + + pool_dbg(pool, "free_pages(%zu):\n", nr_pages); + + if (!reclaimed) { + /* Add to this pool */ + nr_to_pool = kbase_mem_pool_capacity(pool); + nr_to_pool = min(nr_pages, nr_to_pool); + + kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty); + + i += nr_to_pool; + + if (i != nr_pages && next_pool) { + /* Spill to next pool (may overspill) */ + nr_to_pool = kbase_mem_pool_capacity(next_pool); + nr_to_pool = min(nr_pages - i, nr_to_pool); + + kbase_mem_pool_add_array(next_pool, nr_to_pool, + pages + i, true, dirty); + i += nr_to_pool; + } + } + + /* Free any remaining pages to kernel */ + for (; i < nr_pages; i++) { + if (unlikely(!pages[i])) + continue; + + p = phys_to_page(pages[i]); + + kbase_mem_pool_free_page(pool, p); + pages[i] = 0; + } + + pool_dbg(pool, "free_pages(%zu) done\n", nr_pages); +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c new file mode 100755 index 000000000000..585fba036c9e --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c @@ -0,0 +1,81 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include + +#include + +#ifdef CONFIG_DEBUG_FS + +static int kbase_mem_pool_debugfs_size_get(void *data, u64 *val) +{ + struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; + + *val = kbase_mem_pool_size(pool); + + return 0; +} + +static int kbase_mem_pool_debugfs_size_set(void *data, u64 val) +{ + struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; + + kbase_mem_pool_trim(pool, val); + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(kbase_mem_pool_debugfs_size_fops, + kbase_mem_pool_debugfs_size_get, + kbase_mem_pool_debugfs_size_set, + "%llu\n"); + +static int kbase_mem_pool_debugfs_max_size_get(void *data, u64 *val) +{ + struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; + + *val = kbase_mem_pool_max_size(pool); + + return 0; +} + +static int kbase_mem_pool_debugfs_max_size_set(void *data, u64 val) +{ + struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; + + kbase_mem_pool_set_max_size(pool, val); + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(kbase_mem_pool_debugfs_max_size_fops, + kbase_mem_pool_debugfs_max_size_get, + kbase_mem_pool_debugfs_max_size_set, + "%llu\n"); + +void kbase_mem_pool_debugfs_init(struct dentry *parent, + struct kbase_mem_pool *pool) +{ + debugfs_create_file("mem_pool_size", S_IRUGO | S_IWUSR, parent, + pool, &kbase_mem_pool_debugfs_size_fops); + + debugfs_create_file("mem_pool_max_size", S_IRUGO | S_IWUSR, parent, + pool, &kbase_mem_pool_debugfs_max_size_fops); +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h new file mode 100755 index 000000000000..1442854e8956 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h @@ -0,0 +1,36 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_MEM_POOL_DEBUGFS_H +#define _KBASE_MEM_POOL_DEBUGFS_H + +#include + +/** + * kbase_mem_pool_debugfs_init - add debugfs knobs for @pool + * @parent: Parent debugfs dentry + * @pool: Memory pool to control + * + * Adds two debugfs files under @parent: + * - mem_pool_size: get/set the current size of @pool + * - mem_pool_max_size: get/set the max size of @pool + */ +void kbase_mem_pool_debugfs_init(struct dentry *parent, + struct kbase_mem_pool *pool); + +#endif /*_KBASE_MEM_POOL_DEBUGFS_H*/ + diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c new file mode 100755 index 000000000000..d58fd8d62fde --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c @@ -0,0 +1,121 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include + +#ifdef CONFIG_DEBUG_FS + +/** Show callback for the @c mem_profile debugfs file. + * + * This function is called to get the contents of the @c mem_profile debugfs + * file. This is a report of current memory usage and distribution in userspace. + * + * @param sfile The debugfs entry + * @param data Data associated with the entry + * + * @return 0 if it successfully prints data in debugfs entry file, non-zero otherwise + */ +static int kbasep_mem_profile_seq_show(struct seq_file *sfile, void *data) +{ + struct kbase_context *kctx = sfile->private; + + mutex_lock(&kctx->mem_profile_lock); + + seq_write(sfile, kctx->mem_profile_data, kctx->mem_profile_size); + + seq_putc(sfile, '\n'); + + mutex_unlock(&kctx->mem_profile_lock); + + return 0; +} + +/* + * File operations related to debugfs entry for mem_profile + */ +static int kbasep_mem_profile_debugfs_open(struct inode *in, struct file *file) +{ + return single_open(file, kbasep_mem_profile_seq_show, in->i_private); +} + +static const struct file_operations kbasep_mem_profile_debugfs_fops = { + .open = kbasep_mem_profile_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, + size_t size) +{ + int err = 0; + + mutex_lock(&kctx->mem_profile_lock); + + dev_dbg(kctx->kbdev->dev, "initialised: %d", + kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); + + if (!kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { + if (!debugfs_create_file("mem_profile", S_IRUGO, + kctx->kctx_dentry, kctx, + &kbasep_mem_profile_debugfs_fops)) { + err = -EAGAIN; + } else { + kbase_ctx_flag_set(kctx, + KCTX_MEM_PROFILE_INITIALIZED); + } + } + + if (kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { + kfree(kctx->mem_profile_data); + kctx->mem_profile_data = data; + kctx->mem_profile_size = size; + } else { + kfree(data); + } + + dev_dbg(kctx->kbdev->dev, "returning: %d, initialised: %d", + err, kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); + + mutex_unlock(&kctx->mem_profile_lock); + + return err; +} + +void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx) +{ + mutex_lock(&kctx->mem_profile_lock); + + dev_dbg(kctx->kbdev->dev, "initialised: %d", + kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); + + kfree(kctx->mem_profile_data); + kctx->mem_profile_data = NULL; + kctx->mem_profile_size = 0; + + mutex_unlock(&kctx->mem_profile_lock); +} + +#else /* CONFIG_DEBUG_FS */ + +int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, + size_t size) +{ + kfree(data); + return 0; +} +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h new file mode 100755 index 000000000000..a1dc2e0b165b --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h @@ -0,0 +1,59 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mem_profile_debugfs.h + * Header file for mem profiles entries in debugfs + * + */ + +#ifndef _KBASE_MEM_PROFILE_DEBUGFS_H +#define _KBASE_MEM_PROFILE_DEBUGFS_H + +#include +#include + +/** + * @brief Remove entry from Mali memory profile debugfs + */ +void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx); + +/** + * @brief Insert @p data to the debugfs file so it can be read by userspace + * + * The function takes ownership of @p data and frees it later when new data + * is inserted. + * + * If the debugfs entry corresponding to the @p kctx doesn't exist, + * an attempt will be made to create it. + * + * @param kctx The context whose debugfs file @p data should be inserted to + * @param data A NULL-terminated string to be inserted to the debugfs file, + * without the trailing new line character + * @param size The length of the @p data string + * @return 0 if @p data inserted correctly + * -EAGAIN in case of error + * @post @ref mem_profile_initialized will be set to @c true + * the first time this function succeeds. + */ +int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, + size_t size); + +#endif /*_KBASE_MEM_PROFILE_DEBUGFS_H*/ + diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h new file mode 100755 index 000000000000..82f0702974c2 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h @@ -0,0 +1,33 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_mem_profile_debugfs_buf_size.h + * Header file for the size of the buffer to accumulate the histogram report text in + */ + +#ifndef _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ +#define _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ + +/** + * The size of the buffer to accumulate the histogram report text in + * @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT + */ +#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t) (64 + ((80 + (56 * 64)) * 15) + 56)) + +#endif /*_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/ + diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu.c b/drivers/gpu/arm/midgard/mali_kbase_mmu.c new file mode 100755 index 000000000000..26144850a588 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mmu.c @@ -0,0 +1,2088 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_mmu.c + * Base kernel MMU management. + */ + +/* #define DEBUG 1 */ +#include +#include +#include +#include +#if defined(CONFIG_MALI_GATOR_SUPPORT) +#include +#endif +#include +#include +#include + +#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) + +#include +#include +#include +#include +#include +#include + +#define KBASE_MMU_PAGE_ENTRIES 512 + +/** + * kbase_mmu_flush_invalidate() - Flush and invalidate the GPU caches. + * @kctx: The KBase context. + * @vpfn: The virtual page frame number to start the flush on. + * @nr: The number of pages to flush. + * @sync: Set if the operation should be synchronous or not. + * + * Issue a cache flush + invalidate to the GPU caches and invalidate the TLBs. + * + * If sync is not set then transactions still in flight when the flush is issued + * may use the old page tables and the data they write will not be written out + * to memory, this function returns after the flush has been issued but + * before all accesses which might effect the flushed region have completed. + * + * If sync is set then accesses in the flushed region will be drained + * before data is flush and invalidated through L1, L2 and into memory, + * after which point this function will return. + */ +static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, + u64 vpfn, size_t nr, bool sync); + +/** + * kbase_mmu_sync_pgd - sync page directory to memory + * @kbdev: Device pointer. + * @handle: Address of DMA region. + * @size: Size of the region to sync. + * + * This should be called after each page directory update. + */ + +static void kbase_mmu_sync_pgd(struct kbase_device *kbdev, + dma_addr_t handle, size_t size) +{ + /* If page table is not coherent then ensure the gpu can read + * the pages from memory + */ + if (kbdev->system_coherency != COHERENCY_ACE) + dma_sync_single_for_device(kbdev->dev, handle, size, + DMA_TO_DEVICE); +} + +/* + * Definitions: + * - PGD: Page Directory. + * - PTE: Page Table Entry. A 64bit value pointing to the next + * level of translation + * - ATE: Address Transation Entry. A 64bit value pointing to + * a 4kB physical page. + */ + +static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, + struct kbase_as *as, const char *reason_str); + + +static size_t make_multiple(size_t minimum, size_t multiple) +{ + size_t remainder = minimum % multiple; + + if (remainder == 0) + return minimum; + + return minimum + multiple - remainder; +} + +void page_fault_worker(struct work_struct *data) +{ + u64 fault_pfn; + u32 fault_status; + size_t new_pages; + size_t fault_rel_pfn; + struct kbase_as *faulting_as; + int as_no; + struct kbase_context *kctx; + struct kbase_device *kbdev; + struct kbase_va_region *region; + int err; + bool grown = false; + + faulting_as = container_of(data, struct kbase_as, work_pagefault); + fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT; + as_no = faulting_as->number; + + kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); + + /* Grab the context that was already refcounted in kbase_mmu_interrupt(). + * Therefore, it cannot be scheduled out of this AS until we explicitly release it + */ + kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); + if (WARN_ON(!kctx)) { + atomic_dec(&kbdev->faults_pending); + return; + } + + KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev); + + if (unlikely(faulting_as->protected_mode)) + { + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Protected mode fault"); + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + + goto fault_done; + } + + fault_status = faulting_as->fault_status; + switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) { + + case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT: + /* need to check against the region to handle this one */ + break; + + case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Permission failure"); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Translation table bus fault"); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG: + /* nothing to do, but we don't expect this fault currently */ + dev_warn(kbdev->dev, "Access flag unexpectedly set"); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Address size fault"); + else + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code"); + goto fault_done; + + case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Memory attributes fault"); + else + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code"); + goto fault_done; + + default: + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unknown fault code"); + goto fault_done; + } + + /* so we have a translation fault, let's see if it is for growable + * memory */ + kbase_gpu_vm_lock(kctx); + + region = kbase_region_tracker_find_region_enclosing_address(kctx, + faulting_as->fault_addr); + if (!region || region->flags & KBASE_REG_FREE) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Memory is not mapped on the GPU"); + goto fault_done; + } + + if (region->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "DMA-BUF is not mapped on the GPU"); + goto fault_done; + } + + if ((region->flags & GROWABLE_FLAGS_REQUIRED) + != GROWABLE_FLAGS_REQUIRED) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Memory is not growable"); + goto fault_done; + } + + if ((region->flags & KBASE_REG_DONT_NEED)) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Don't need memory can't be grown"); + goto fault_done; + } + + /* find the size we need to grow it by */ + /* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address + * validating the fault_adress to be within a size_t from the start_pfn */ + fault_rel_pfn = fault_pfn - region->start_pfn; + + if (fault_rel_pfn < kbase_reg_current_backed_size(region)) { + dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring", + faulting_as->fault_addr, region->start_pfn, + region->start_pfn + + kbase_reg_current_backed_size(region)); + + mutex_lock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + /* [1] in case another page fault occurred while we were + * handling the (duplicate) page fault we need to ensure we + * don't loose the other page fault as result of us clearing + * the MMU IRQ. Therefore, after we clear the MMU IRQ we send + * an UNLOCK command that will retry any stalled memory + * transaction (which should cause the other page fault to be + * raised again). + */ + kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0, + AS_COMMAND_UNLOCK, 1); + + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + kbase_gpu_vm_unlock(kctx); + + goto fault_done; + } + + new_pages = make_multiple(fault_rel_pfn - + kbase_reg_current_backed_size(region) + 1, + region->extent); + + /* cap to max vsize */ + if (new_pages + kbase_reg_current_backed_size(region) > + region->nr_pages) + new_pages = region->nr_pages - + kbase_reg_current_backed_size(region); + + if (0 == new_pages) { + mutex_lock(&kbdev->mmu_hw_mutex); + + /* Duplicate of a fault we've already handled, nothing to do */ + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + /* See comment [1] about UNLOCK usage */ + kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0, + AS_COMMAND_UNLOCK, 1); + + mutex_unlock(&kbdev->mmu_hw_mutex); + + kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + kbase_gpu_vm_unlock(kctx); + goto fault_done; + } + + if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) { + if (region->gpu_alloc != region->cpu_alloc) { + if (kbase_alloc_phy_pages_helper( + region->cpu_alloc, new_pages) == 0) { + grown = true; + } else { + kbase_free_phy_pages_helper(region->gpu_alloc, + new_pages); + } + } else { + grown = true; + } + } + + + if (grown) { + u64 pfn_offset; + u32 op; + + /* alloc success */ + KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages); + + /* set up the new pages */ + pfn_offset = kbase_reg_current_backed_size(region) - new_pages; + /* + * Note: + * Issuing an MMU operation will unlock the MMU and cause the + * translation to be replayed. If the page insertion fails then + * rather then trying to continue the context should be killed + * so the no_flush version of insert_pages is used which allows + * us to unlock the MMU as we see fit. + */ + err = kbase_mmu_insert_pages_no_flush(kctx, + region->start_pfn + pfn_offset, + &kbase_get_gpu_phy_pages(region)[pfn_offset], + new_pages, region->flags); + if (err) { + kbase_free_phy_pages_helper(region->gpu_alloc, new_pages); + if (region->gpu_alloc != region->cpu_alloc) + kbase_free_phy_pages_helper(region->cpu_alloc, + new_pages); + kbase_gpu_vm_unlock(kctx); + /* The locked VA region will be unlocked and the cache invalidated in here */ + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Page table update failure"); + goto fault_done; + } +#if defined(CONFIG_MALI_GATOR_SUPPORT) + kbase_trace_mali_page_fault_insert_pages(as_no, new_pages); +#endif + KBASE_TLSTREAM_AUX_PAGEFAULT(kctx->id, (u64)new_pages); + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + + /* flush L2 and unlock the VA (resumes the MMU) */ + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367)) + op = AS_COMMAND_FLUSH; + else + op = AS_COMMAND_FLUSH_PT; + + /* clear MMU interrupt - this needs to be done after updating + * the page tables but before issuing a FLUSH command. The + * FLUSH cmd has a side effect that it restarts stalled memory + * transactions in other address spaces which may cause + * another fault to occur. If we didn't clear the interrupt at + * this stage a new IRQ might not be raised when the GPU finds + * a MMU IRQ is already pending. + */ + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + + kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx, + faulting_as->fault_addr >> PAGE_SHIFT, + new_pages, + op, 1); + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + + /* reenable this in the mask */ + kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE); + kbase_gpu_vm_unlock(kctx); + } else { + /* failed to extend, handle as a normal PF */ + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Page allocation failure"); + } + +fault_done: + /* + * By this point, the fault was handled in some way, + * so release the ctx refcount + */ + kbasep_js_runpool_release_ctx(kbdev, kctx); + + atomic_dec(&kbdev->faults_pending); +} + +phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx) +{ + u64 *page; + int i; + struct page *p; + int new_page_count __maybe_unused; + + KBASE_DEBUG_ASSERT(NULL != kctx); + new_page_count = kbase_atomic_add_pages(1, &kctx->used_pages); + kbase_atomic_add_pages(1, &kctx->kbdev->memdev.used_pages); + + p = kbase_mem_pool_alloc(&kctx->mem_pool); + if (!p) + goto sub_pages; + + KBASE_TLSTREAM_AUX_PAGESALLOC( + (u32)kctx->id, + (u64)new_page_count); + + page = kmap(p); + if (NULL == page) + goto alloc_free; + + kbase_process_page_usage_inc(kctx, 1); + + for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) + kctx->kbdev->mmu_mode->entry_invalidate(&page[i]); + + kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); + + kunmap(p); + return page_to_phys(p); + +alloc_free: + kbase_mem_pool_free(&kctx->mem_pool, p, false); +sub_pages: + kbase_atomic_sub_pages(1, &kctx->used_pages); + kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd); + +/* Given PGD PFN for level N, return PGD PFN for level N+1, allocating the + * new table from the pool if needed and possible + */ +static int mmu_get_next_pgd(struct kbase_context *kctx, + phys_addr_t *pgd, u64 vpfn, int level) +{ + u64 *page; + phys_addr_t target_pgd; + struct page *p; + + KBASE_DEBUG_ASSERT(*pgd); + KBASE_DEBUG_ASSERT(NULL != kctx); + + lockdep_assert_held(&kctx->mmu_lock); + + /* + * Architecture spec defines level-0 as being the top-most. + * This is a bit unfortunate here, but we keep the same convention. + */ + vpfn >>= (3 - level) * 9; + vpfn &= 0x1FF; + + p = pfn_to_page(PFN_DOWN(*pgd)); + page = kmap(p); + if (NULL == page) { + dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kmap failure\n"); + return -EINVAL; + } + + target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]); + + if (!target_pgd) { + target_pgd = kbase_mmu_alloc_pgd(kctx); + if (!target_pgd) { + dev_dbg(kctx->kbdev->dev, "mmu_get_next_pgd: kbase_mmu_alloc_pgd failure\n"); + kunmap(p); + return -ENOMEM; + } + + kctx->kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd); + + kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); + /* Rely on the caller to update the address space flags. */ + } + + kunmap(p); + *pgd = target_pgd; + + return 0; +} + +static int mmu_get_bottom_pgd(struct kbase_context *kctx, + u64 vpfn, phys_addr_t *out_pgd) +{ + phys_addr_t pgd; + int l; + + lockdep_assert_held(&kctx->mmu_lock); + + pgd = kctx->pgd; + for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) { + int err = mmu_get_next_pgd(kctx, &pgd, vpfn, l); + /* Handle failure condition */ + if (err) { + dev_dbg(kctx->kbdev->dev, "mmu_get_bottom_pgd: mmu_get_next_pgd failure\n"); + return err; + } + } + + *out_pgd = pgd; + + return 0; +} + +static phys_addr_t mmu_insert_pages_recover_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level) +{ + u64 *page; + phys_addr_t target_pgd; + + KBASE_DEBUG_ASSERT(pgd); + KBASE_DEBUG_ASSERT(NULL != kctx); + + lockdep_assert_held(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); + + /* + * Architecture spec defines level-0 as being the top-most. + * This is a bit unfortunate here, but we keep the same convention. + */ + vpfn >>= (3 - level) * 9; + vpfn &= 0x1FF; + + page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); + /* kmap_atomic should NEVER fail */ + KBASE_DEBUG_ASSERT(NULL != page); + + target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]); + /* As we are recovering from what has already been set up, we should have a target_pgd */ + KBASE_DEBUG_ASSERT(0 != target_pgd); + kunmap_atomic(page); + return target_pgd; +} + +static phys_addr_t mmu_insert_pages_recover_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn) +{ + phys_addr_t pgd; + int l; + + lockdep_assert_held(&kctx->mmu_lock); + + pgd = kctx->pgd; + + for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) { + pgd = mmu_insert_pages_recover_get_next_pgd(kctx, pgd, vpfn, l); + /* Should never fail */ + KBASE_DEBUG_ASSERT(0 != pgd); + } + + return pgd; +} + +static void mmu_insert_pages_failure_recovery(struct kbase_context *kctx, u64 vpfn, + size_t nr) +{ + phys_addr_t pgd; + u64 *pgd_page; + struct kbase_mmu_mode const *mmu_mode; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(0 != vpfn); + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + + lockdep_assert_held(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); + + mmu_mode = kctx->kbdev->mmu_mode; + + while (nr) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; + struct page *p; + + if (count > nr) + count = nr; + + pgd = mmu_insert_pages_recover_get_bottom_pgd(kctx, vpfn); + KBASE_DEBUG_ASSERT(0 != pgd); + + p = pfn_to_page(PFN_DOWN(pgd)); + + pgd_page = kmap_atomic(p); + KBASE_DEBUG_ASSERT(NULL != pgd_page); + + /* Invalidate the entries we added */ + for (i = 0; i < count; i++) + mmu_mode->entry_invalidate(&pgd_page[index + i]); + + vpfn += count; + nr -= count; + + kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); + + kunmap_atomic(pgd_page); + } +} + +/* + * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn' + */ +int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, + phys_addr_t phys, size_t nr, + unsigned long flags) +{ + phys_addr_t pgd; + u64 *pgd_page; + /* In case the insert_single_page only partially completes we need to be + * able to recover */ + bool recover_required = false; + u64 recover_vpfn = vpfn; + size_t recover_count = 0; + size_t remain = nr; + int err; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(0 != vpfn); + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + + /* Early out if there is nothing to do */ + if (nr == 0) + return 0; + + mutex_lock(&kctx->mmu_lock); + + while (remain) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; + struct page *p; + + if (count > remain) + count = remain; + + /* + * Repeatedly calling mmu_get_bottom_pte() is clearly + * suboptimal. We don't have to re-parse the whole tree + * each time (just cache the l0-l2 sequence). + * On the other hand, it's only a gain when we map more than + * 256 pages at once (on average). Do we really care? + */ + do { + err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); + if (err != -ENOMEM) + break; + /* Fill the memory pool with enough pages for + * the page walk to succeed + */ + mutex_unlock(&kctx->mmu_lock); + err = kbase_mem_pool_grow(&kctx->mem_pool, + MIDGARD_MMU_BOTTOMLEVEL); + mutex_lock(&kctx->mmu_lock); + } while (!err); + if (err) { + dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n"); + if (recover_required) { + /* Invalidate the pages we have partially + * completed */ + mmu_insert_pages_failure_recovery(kctx, + recover_vpfn, + recover_count); + } + goto fail_unlock; + } + + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap(p); + if (!pgd_page) { + dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n"); + if (recover_required) { + /* Invalidate the pages we have partially + * completed */ + mmu_insert_pages_failure_recovery(kctx, + recover_vpfn, + recover_count); + } + err = -ENOMEM; + goto fail_unlock; + } + + for (i = 0; i < count; i++) { + unsigned int ofs = index + i; + + KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); + kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs], + phys, flags); + } + + vpfn += count; + remain -= count; + + kbase_mmu_sync_pgd(kctx->kbdev, + kbase_dma_addr(p) + (index * sizeof(u64)), + count * sizeof(u64)); + + kunmap(p); + /* We have started modifying the page table. + * If further pages need inserting and fail we need to undo what + * has already taken place */ + recover_required = true; + recover_count += count; + } + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); + return 0; + +fail_unlock: + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); + return err; +} + +int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, + phys_addr_t *phys, size_t nr, + unsigned long flags) +{ + phys_addr_t pgd; + u64 *pgd_page; + /* In case the insert_pages only partially completes we need to be able + * to recover */ + bool recover_required = false; + u64 recover_vpfn = vpfn; + size_t recover_count = 0; + size_t remain = nr; + int err; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(0 != vpfn); + /* 64-bit address range is the max */ + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + + /* Early out if there is nothing to do */ + if (nr == 0) + return 0; + + mutex_lock(&kctx->mmu_lock); + + while (remain) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; + struct page *p; + + if (count > remain) + count = remain; + + /* + * Repeatedly calling mmu_get_bottom_pte() is clearly + * suboptimal. We don't have to re-parse the whole tree + * each time (just cache the l0-l2 sequence). + * On the other hand, it's only a gain when we map more than + * 256 pages at once (on average). Do we really care? + */ + do { + err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); + if (err != -ENOMEM) + break; + /* Fill the memory pool with enough pages for + * the page walk to succeed + */ + mutex_unlock(&kctx->mmu_lock); + err = kbase_mem_pool_grow(&kctx->mem_pool, + MIDGARD_MMU_BOTTOMLEVEL); + mutex_lock(&kctx->mmu_lock); + } while (!err); + if (err) { + dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n"); + if (recover_required) { + /* Invalidate the pages we have partially + * completed */ + mmu_insert_pages_failure_recovery(kctx, + recover_vpfn, + recover_count); + } + goto fail_unlock; + } + + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap(p); + if (!pgd_page) { + dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n"); + if (recover_required) { + /* Invalidate the pages we have partially + * completed */ + mmu_insert_pages_failure_recovery(kctx, + recover_vpfn, + recover_count); + } + err = -ENOMEM; + goto fail_unlock; + } + + for (i = 0; i < count; i++) { + unsigned int ofs = index + i; + + KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); + kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs], + phys[i], flags); + } + + phys += count; + vpfn += count; + remain -= count; + + kbase_mmu_sync_pgd(kctx->kbdev, + kbase_dma_addr(p) + (index * sizeof(u64)), + count * sizeof(u64)); + + kunmap(p); + /* We have started modifying the page table. If further pages + * need inserting and fail we need to undo what has already + * taken place */ + recover_required = true; + recover_count += count; + } + + mutex_unlock(&kctx->mmu_lock); + return 0; + +fail_unlock: + mutex_unlock(&kctx->mmu_lock); + return err; +} + +/* + * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn' + */ +int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, + phys_addr_t *phys, size_t nr, + unsigned long flags) +{ + int err; + + err = kbase_mmu_insert_pages_no_flush(kctx, vpfn, phys, nr, flags); + kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages); + +/** + * kbase_mmu_flush_invalidate_noretain() - Flush and invalidate the GPU caches + * without retaining the kbase context. + * @kctx: The KBase context. + * @vpfn: The virtual page frame number to start the flush on. + * @nr: The number of pages to flush. + * @sync: Set if the operation should be synchronous or not. + * + * As per kbase_mmu_flush_invalidate but doesn't retain the kctx or do any + * other locking. + */ +static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx, + u64 vpfn, size_t nr, bool sync) +{ + struct kbase_device *kbdev = kctx->kbdev; + int err; + u32 op; + + /* Early out if there is nothing to do */ + if (nr == 0) + return; + + if (sync) + op = AS_COMMAND_FLUSH_MEM; + else + op = AS_COMMAND_FLUSH_PT; + + err = kbase_mmu_hw_do_operation(kbdev, + &kbdev->as[kctx->as_nr], + kctx, vpfn, nr, op, 0); +#if KBASE_GPU_RESET_EN + if (err) { + /* Flush failed to complete, assume the + * GPU has hung and perform a reset to + * recover */ + dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n"); + + if (kbase_prepare_to_reset_gpu_locked(kbdev)) + kbase_reset_gpu_locked(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + +#ifndef CONFIG_MALI_NO_MALI + /* + * As this function could be called in interrupt context the sync + * request can't block. Instead log the request and the next flush + * request will pick it up. + */ + if ((!err) && sync && + kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367)) + atomic_set(&kctx->drain_pending, 1); +#endif /* !CONFIG_MALI_NO_MALI */ +} + +static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, + u64 vpfn, size_t nr, bool sync) +{ + struct kbase_device *kbdev; + bool ctx_is_in_runpool; +#ifndef CONFIG_MALI_NO_MALI + bool drain_pending = false; + + if (atomic_xchg(&kctx->drain_pending, 0)) + drain_pending = true; +#endif /* !CONFIG_MALI_NO_MALI */ + + /* Early out if there is nothing to do */ + if (nr == 0) + return; + + kbdev = kctx->kbdev; + mutex_lock(&kbdev->js_data.queue_mutex); + ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx); + mutex_unlock(&kbdev->js_data.queue_mutex); + + if (ctx_is_in_runpool) { + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + + if (!kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + int err; + u32 op; + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + + if (sync) + op = AS_COMMAND_FLUSH_MEM; + else + op = AS_COMMAND_FLUSH_PT; + + err = kbase_mmu_hw_do_operation(kbdev, + &kbdev->as[kctx->as_nr], + kctx, vpfn, nr, op, 0); + +#if KBASE_GPU_RESET_EN + if (err) { + /* Flush failed to complete, assume the + * GPU has hung and perform a reset to + * recover */ + dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); + + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + +#ifndef CONFIG_MALI_NO_MALI + /* + * The transaction lock must be dropped before here + * as kbase_wait_write_flush could take it if + * the GPU was powered down (static analysis doesn't + * know this can't happen). + */ + drain_pending |= (!err) && sync && + kbase_hw_has_issue(kctx->kbdev, + BASE_HW_ISSUE_6367); + if (drain_pending) { + /* Wait for GPU to flush write buffer */ + kbase_wait_write_flush(kctx); + } +#endif /* !CONFIG_MALI_NO_MALI */ + + kbase_pm_context_idle(kbdev); + } + kbasep_js_runpool_release_ctx(kbdev, kctx); + } +} + +void kbase_mmu_update(struct kbase_context *kctx) +{ + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); + /* ASSERT that the context has a valid as_nr, which is only the case + * when it's scheduled in. + * + * as_nr won't change because the caller has the hwaccess_lock */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + + kctx->kbdev->mmu_mode->update(kctx); +} +KBASE_EXPORT_TEST_API(kbase_mmu_update); + +void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&kbdev->mmu_hw_mutex); + + kbdev->mmu_mode->disable_as(kbdev, as_nr); +} + +void kbase_mmu_disable(struct kbase_context *kctx) +{ + /* ASSERT that the context has a valid as_nr, which is only the case + * when it's scheduled in. + * + * as_nr won't change because the caller has the hwaccess_lock */ + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + + lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + + /* + * The address space is being disabled, drain all knowledge of it out + * from the caches as pages and page tables might be freed after this. + * + * The job scheduler code will already be holding the locks and context + * so just do the flush. + */ + kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true); + + kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr); +} +KBASE_EXPORT_TEST_API(kbase_mmu_disable); + +/* + * We actually only discard the ATE, and not the page table + * pages. There is a potential DoS here, as we'll leak memory by + * having PTEs that are potentially unused. Will require physical + * page accounting, so MMU pages are part of the process allocation. + * + * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is + * currently scheduled into the runpool, and so potentially uses a lot of locks. + * These locks must be taken in the correct order with respect to others + * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more + * information. + */ +int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr) +{ + phys_addr_t pgd; + u64 *pgd_page; + struct kbase_device *kbdev; + size_t requested_nr = nr; + struct kbase_mmu_mode const *mmu_mode; + int err; + + KBASE_DEBUG_ASSERT(NULL != kctx); + beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr); + + if (0 == nr) { + /* early out if nothing to do */ + return 0; + } + + mutex_lock(&kctx->mmu_lock); + + kbdev = kctx->kbdev; + mmu_mode = kbdev->mmu_mode; + + while (nr) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; + struct page *p; + + if (count > nr) + count = nr; + + err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); + if (err) { + dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: mmu_get_bottom_pgd failure\n"); + err = -EINVAL; + goto fail_unlock; + } + + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap(p); + if (!pgd_page) { + dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: kmap failure\n"); + err = -ENOMEM; + goto fail_unlock; + } + + for (i = 0; i < count; i++) + mmu_mode->entry_invalidate(&pgd_page[index + i]); + + vpfn += count; + nr -= count; + + kbase_mmu_sync_pgd(kctx->kbdev, + kbase_dma_addr(p) + (index * sizeof(u64)), + count * sizeof(u64)); + + kunmap(p); + } + + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); + return 0; + +fail_unlock: + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); + return err; +} + +KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages); + +/** + * Update the entries for specified number of pages pointed to by 'phys' at GPU PFN 'vpfn'. + * This call is being triggered as a response to the changes of the mem attributes + * + * @pre : The caller is responsible for validating the memory attributes + * + * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is + * currently scheduled into the runpool, and so potentially uses a lot of locks. + * These locks must be taken in the correct order with respect to others + * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more + * information. + */ +int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags) +{ + phys_addr_t pgd; + u64 *pgd_page; + size_t requested_nr = nr; + struct kbase_mmu_mode const *mmu_mode; + int err; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(0 != vpfn); + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + + /* Early out if there is nothing to do */ + if (nr == 0) + return 0; + + mutex_lock(&kctx->mmu_lock); + + mmu_mode = kctx->kbdev->mmu_mode; + + dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags on GPU PFN 0x%llx from phys %p, %zu pages", + vpfn, phys, nr); + + while (nr) { + unsigned int i; + unsigned int index = vpfn & 0x1FF; + size_t count = KBASE_MMU_PAGE_ENTRIES - index; + struct page *p; + + if (count > nr) + count = nr; + + do { + err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); + if (err != -ENOMEM) + break; + /* Fill the memory pool with enough pages for + * the page walk to succeed + */ + mutex_unlock(&kctx->mmu_lock); + err = kbase_mem_pool_grow(&kctx->mem_pool, + MIDGARD_MMU_BOTTOMLEVEL); + mutex_lock(&kctx->mmu_lock); + } while (!err); + if (err) { + dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd failure\n"); + goto fail_unlock; + } + + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap(p); + if (!pgd_page) { + dev_warn(kctx->kbdev->dev, "kmap failure\n"); + err = -ENOMEM; + goto fail_unlock; + } + + for (i = 0; i < count; i++) + mmu_mode->entry_set_ate(&pgd_page[index + i], phys[i], + flags); + + phys += count; + vpfn += count; + nr -= count; + + kbase_mmu_sync_pgd(kctx->kbdev, + kbase_dma_addr(p) + (index * sizeof(u64)), + count * sizeof(u64)); + + kunmap(pfn_to_page(PFN_DOWN(pgd))); + } + + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); + return 0; + +fail_unlock: + mutex_unlock(&kctx->mmu_lock); + kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); + return err; +} + +/* This is a debug feature only */ +static void mmu_check_unused(struct kbase_context *kctx, phys_addr_t pgd) +{ + u64 *page; + int i; + + lockdep_assert_held(&kctx->reg_lock); + + page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); + /* kmap_atomic should NEVER fail. */ + KBASE_DEBUG_ASSERT(NULL != page); + + for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { + if (kctx->kbdev->mmu_mode->ate_is_valid(page[i])) + beenthere(kctx, "live pte %016lx", (unsigned long)page[i]); + } + kunmap_atomic(page); +} + +static void mmu_teardown_level(struct kbase_context *kctx, phys_addr_t pgd, int level, int zap, u64 *pgd_page_buffer) +{ + phys_addr_t target_pgd; + u64 *pgd_page; + int i; + struct kbase_mmu_mode const *mmu_mode; + + KBASE_DEBUG_ASSERT(NULL != kctx); + lockdep_assert_held(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); + + pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); + /* kmap_atomic should NEVER fail. */ + KBASE_DEBUG_ASSERT(NULL != pgd_page); + /* Copy the page to our preallocated buffer so that we can minimize kmap_atomic usage */ + memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE); + kunmap_atomic(pgd_page); + pgd_page = pgd_page_buffer; + + mmu_mode = kctx->kbdev->mmu_mode; + + for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { + target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]); + + if (target_pgd) { + if (level < (MIDGARD_MMU_BOTTOMLEVEL - 1)) { + mmu_teardown_level(kctx, target_pgd, level + 1, zap, pgd_page_buffer + (PAGE_SIZE / sizeof(u64))); + } else { + /* + * So target_pte is a level-3 page. + * As a leaf, it is safe to free it. + * Unless we have live pages attached to it! + */ + mmu_check_unused(kctx, target_pgd); + } + + beenthere(kctx, "pte %lx level %d", (unsigned long)target_pgd, level + 1); + if (zap) { + struct page *p = phys_to_page(target_pgd); + + kbase_mem_pool_free(&kctx->mem_pool, p, true); + kbase_process_page_usage_dec(kctx, 1); + kbase_atomic_sub_pages(1, &kctx->used_pages); + kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); + } + } + } +} + +int kbase_mmu_init(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages); + + mutex_init(&kctx->mmu_lock); + + /* Preallocate MMU depth of four pages for mmu_teardown_level to use */ + kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); + + if (NULL == kctx->mmu_teardown_pages) + return -ENOMEM; + + return 0; +} + +void kbase_mmu_term(struct kbase_context *kctx) +{ + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); + + kfree(kctx->mmu_teardown_pages); + kctx->mmu_teardown_pages = NULL; +} + +void kbase_mmu_free_pgd(struct kbase_context *kctx) +{ + int new_page_count __maybe_unused; + + KBASE_DEBUG_ASSERT(NULL != kctx); + KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); + + mutex_lock(&kctx->mmu_lock); + mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, 1, kctx->mmu_teardown_pages); + mutex_unlock(&kctx->mmu_lock); + + beenthere(kctx, "pgd %lx", (unsigned long)kctx->pgd); + kbase_mem_pool_free(&kctx->mem_pool, phys_to_page(kctx->pgd), true); + kbase_process_page_usage_dec(kctx, 1); + new_page_count = kbase_atomic_sub_pages(1, &kctx->used_pages); + kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); + + KBASE_TLSTREAM_AUX_PAGESALLOC( + (u32)kctx->id, + (u64)new_page_count); +} + +KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd); + +static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, int level, char ** const buffer, size_t *size_left) +{ + phys_addr_t target_pgd; + u64 *pgd_page; + int i; + size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64); + size_t dump_size; + struct kbase_mmu_mode const *mmu_mode; + + KBASE_DEBUG_ASSERT(NULL != kctx); + lockdep_assert_held(&kctx->mmu_lock); + + mmu_mode = kctx->kbdev->mmu_mode; + + pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd))); + if (!pgd_page) { + dev_warn(kctx->kbdev->dev, "kbasep_mmu_dump_level: kmap failure\n"); + return 0; + } + + if (*size_left >= size) { + /* A modified physical address that contains the page table level */ + u64 m_pgd = pgd | level; + + /* Put the modified physical address in the output buffer */ + memcpy(*buffer, &m_pgd, sizeof(m_pgd)); + *buffer += sizeof(m_pgd); + + /* Followed by the page table itself */ + memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES); + *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES; + + *size_left -= size; + } + + if (level < MIDGARD_MMU_BOTTOMLEVEL) { + for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { + if (mmu_mode->pte_is_valid(pgd_page[i])) { + target_pgd = mmu_mode->pte_to_phy_addr( + pgd_page[i]); + + dump_size = kbasep_mmu_dump_level(kctx, + target_pgd, level + 1, + buffer, size_left); + if (!dump_size) { + kunmap(pfn_to_page(PFN_DOWN(pgd))); + return 0; + } + size += dump_size; + } + } + } + + kunmap(pfn_to_page(PFN_DOWN(pgd))); + + return size; +} + +void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) +{ + void *kaddr; + size_t size_left; + + KBASE_DEBUG_ASSERT(kctx); + + if (0 == nr_pages) { + /* can't dump in a 0 sized buffer, early out */ + return NULL; + } + + size_left = nr_pages * PAGE_SIZE; + + KBASE_DEBUG_ASSERT(0 != size_left); + kaddr = vmalloc_user(size_left); + + mutex_lock(&kctx->mmu_lock); + + if (kaddr) { + u64 end_marker = 0xFFULL; + char *buffer; + char *mmu_dump_buffer; + u64 config[3]; + size_t size; + + buffer = (char *)kaddr; + mmu_dump_buffer = buffer; + + if (kctx->api_version >= KBASE_API_VERSION(8, 4)) { + struct kbase_mmu_setup as_setup; + + kctx->kbdev->mmu_mode->get_as_setup(kctx, &as_setup); + config[0] = as_setup.transtab; + config[1] = as_setup.memattr; + config[2] = as_setup.transcfg; + memcpy(buffer, &config, sizeof(config)); + mmu_dump_buffer += sizeof(config); + size_left -= sizeof(config); + } + + + + size = kbasep_mmu_dump_level(kctx, + kctx->pgd, + MIDGARD_MMU_TOPLEVEL, + &mmu_dump_buffer, + &size_left); + + if (!size) + goto fail_free; + + /* Add on the size for the end marker */ + size += sizeof(u64); + /* Add on the size for the config */ + if (kctx->api_version >= KBASE_API_VERSION(8, 4)) + size += sizeof(config); + + + if (size > nr_pages * PAGE_SIZE || size_left < sizeof(u64)) { + /* The buffer isn't big enough - free the memory and return failure */ + goto fail_free; + } + + /* Add the end marker */ + memcpy(mmu_dump_buffer, &end_marker, sizeof(u64)); + } + + mutex_unlock(&kctx->mmu_lock); + return kaddr; + +fail_free: + vfree(kaddr); + mutex_unlock(&kctx->mmu_lock); + return NULL; +} +KBASE_EXPORT_TEST_API(kbase_mmu_dump); + +void bus_fault_worker(struct work_struct *data) +{ + struct kbase_as *faulting_as; + int as_no; + struct kbase_context *kctx; + struct kbase_device *kbdev; +#if KBASE_GPU_RESET_EN + bool reset_status = false; +#endif /* KBASE_GPU_RESET_EN */ + + faulting_as = container_of(data, struct kbase_as, work_busfault); + + as_no = faulting_as->number; + + kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); + + /* Grab the context that was already refcounted in kbase_mmu_interrupt(). + * Therefore, it cannot be scheduled out of this AS until we explicitly release it + */ + kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); + if (WARN_ON(!kctx)) { + atomic_dec(&kbdev->faults_pending); + return; + } + + if (unlikely(faulting_as->protected_mode)) + { + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Permission failure"); + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + kbasep_js_runpool_release_ctx(kbdev, kctx); + atomic_dec(&kbdev->faults_pending); + return; + + } + +#if KBASE_GPU_RESET_EN + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { + /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. + * We start the reset before switching to UNMAPPED to ensure that unrelated jobs + * are evicted from the GPU before the switch. + */ + dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); + reset_status = kbase_prepare_to_reset_gpu(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */ + if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + unsigned long flags; + + /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + + /* Set the MMU into unmapped mode */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_disable(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + + kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + + kbase_pm_context_idle(kbdev); + } + +#if KBASE_GPU_RESET_EN + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) + kbase_reset_gpu(kbdev); +#endif /* KBASE_GPU_RESET_EN */ + + kbasep_js_runpool_release_ctx(kbdev, kctx); + + atomic_dec(&kbdev->faults_pending); +} + +const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) +{ + const char *e; + + switch (exception_code) { + /* Non-Fault Status code */ + case 0x00: + e = "NOT_STARTED/IDLE/OK"; + break; + case 0x01: + e = "DONE"; + break; + case 0x02: + e = "INTERRUPTED"; + break; + case 0x03: + e = "STOPPED"; + break; + case 0x04: + e = "TERMINATED"; + break; + case 0x08: + e = "ACTIVE"; + break; + /* Job exceptions */ + case 0x40: + e = "JOB_CONFIG_FAULT"; + break; + case 0x41: + e = "JOB_POWER_FAULT"; + break; + case 0x42: + e = "JOB_READ_FAULT"; + break; + case 0x43: + e = "JOB_WRITE_FAULT"; + break; + case 0x44: + e = "JOB_AFFINITY_FAULT"; + break; + case 0x48: + e = "JOB_BUS_FAULT"; + break; + case 0x50: + e = "INSTR_INVALID_PC"; + break; + case 0x51: + e = "INSTR_INVALID_ENC"; + break; + case 0x52: + e = "INSTR_TYPE_MISMATCH"; + break; + case 0x53: + e = "INSTR_OPERAND_FAULT"; + break; + case 0x54: + e = "INSTR_TLS_FAULT"; + break; + case 0x55: + e = "INSTR_BARRIER_FAULT"; + break; + case 0x56: + e = "INSTR_ALIGN_FAULT"; + break; + case 0x58: + e = "DATA_INVALID_FAULT"; + break; + case 0x59: + e = "TILE_RANGE_FAULT"; + break; + case 0x5A: + e = "ADDR_RANGE_FAULT"; + break; + case 0x60: + e = "OUT_OF_MEMORY"; + break; + /* GPU exceptions */ + case 0x80: + e = "DELAYED_BUS_FAULT"; + break; + case 0x88: + e = "SHAREABILITY_FAULT"; + break; + /* MMU exceptions */ + case 0xC0: + case 0xC1: + case 0xC2: + case 0xC3: + case 0xC4: + case 0xC5: + case 0xC6: + case 0xC7: + e = "TRANSLATION_FAULT"; + break; + case 0xC8: + e = "PERMISSION_FAULT"; + break; + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "PERMISSION_FAULT"; + else + e = "UNKNOWN"; + break; + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD5: + case 0xD6: + case 0xD7: + e = "TRANSTAB_BUS_FAULT"; + break; + case 0xD8: + e = "ACCESS_FLAG"; + break; + case 0xD9: + case 0xDA: + case 0xDB: + case 0xDC: + case 0xDD: + case 0xDE: + case 0xDF: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "ACCESS_FLAG"; + else + e = "UNKNOWN"; + break; + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "ADDRESS_SIZE_FAULT"; + else + e = "UNKNOWN"; + break; + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + e = "MEMORY_ATTRIBUTES_FAULT"; + else + e = "UNKNOWN"; + break; + default: + e = "UNKNOWN"; + break; + }; + + return e; +} + +static const char *access_type_name(struct kbase_device *kbdev, + u32 fault_status) +{ + switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) { + case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + return "ATOMIC"; + else + return "UNKNOWN"; + case AS_FAULTSTATUS_ACCESS_TYPE_READ: + return "READ"; + case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: + return "WRITE"; + case AS_FAULTSTATUS_ACCESS_TYPE_EX: + return "EXECUTE"; + default: + WARN_ON(1); + return NULL; + } +} + +/** + * The caller must ensure it's retained the ctx to prevent it from being scheduled out whilst it's being worked on. + */ +static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, + struct kbase_as *as, const char *reason_str) +{ + unsigned long flags; + int exception_type; + int access_type; + int source_id; + int as_no; + struct kbase_device *kbdev; + struct kbasep_js_device_data *js_devdata; + +#if KBASE_GPU_RESET_EN + bool reset_status = false; +#endif + + as_no = as->number; + kbdev = kctx->kbdev; + js_devdata = &kbdev->js_data; + + /* ASSERT that the context won't leave the runpool */ + KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); + + /* decode the fault status */ + exception_type = as->fault_status & 0xFF; + access_type = (as->fault_status >> 8) & 0x3; + source_id = (as->fault_status >> 16); + + /* terminal fault, print info about the fault */ + dev_err(kbdev->dev, + "Unhandled Page fault in AS%d at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "decoded fault status: %s\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X\n" + "pid: %d\n", + as_no, as->fault_addr, + reason_str, + as->fault_status, + (as->fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"), + exception_type, kbase_exception_name(kbdev, exception_type), + access_type, access_type_name(kbdev, as->fault_status), + source_id, + kctx->pid); + + /* hardware counters dump fault handling */ + if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && + (kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_DUMPING)) { + unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; + + if ((as->fault_addr >= kbdev->hwcnt.addr) && + (as->fault_addr < (kbdev->hwcnt.addr + + (num_core_groups * 2048)))) + kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; + } + + /* Stop the kctx from submitting more jobs and cause it to be scheduled + * out/rescheduled - this will occur on releasing the context's refcount */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbasep_js_clear_submit_allowed(js_devdata, kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + /* Kill any running jobs from the context. Submit is disallowed, so no more jobs from this + * context can appear in the job slots from this point on */ + kbase_backend_jm_kill_jobs_from_kctx(kctx); + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); +#if KBASE_GPU_RESET_EN + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { + /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. + * We start the reset before switching to UNMAPPED to ensure that unrelated jobs + * are evicted from the GPU before the switch. + */ + dev_err(kbdev->dev, "Unhandled page fault. For this GPU version we now soft-reset the GPU as part of page fault recovery."); + reset_status = kbase_prepare_to_reset_gpu(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_mmu_disable(kctx); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + /* Clear down the fault */ + kbase_mmu_hw_clear_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + +#if KBASE_GPU_RESET_EN + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) + kbase_reset_gpu(kbdev); +#endif /* KBASE_GPU_RESET_EN */ +} + +void kbasep_as_do_poke(struct work_struct *work) +{ + struct kbase_as *as; + struct kbase_device *kbdev; + struct kbase_context *kctx; + unsigned long flags; + + KBASE_DEBUG_ASSERT(work); + as = container_of(work, struct kbase_as, poke_work); + kbdev = container_of(as, struct kbase_device, as[as->number]); + KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); + + /* GPU power will already be active by virtue of the caller holding a JS + * reference on the address space, and will not release it until this worker + * has finished */ + + /* Further to the comment above, we know that while this function is running + * the AS will not be released as before the atom is released this workqueue + * is flushed (in kbase_as_poking_timer_release_atom) + */ + kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as->number); + + /* AS transaction begin */ + mutex_lock(&kbdev->mmu_hw_mutex); + /* Force a uTLB invalidate */ + kbase_mmu_hw_do_operation(kbdev, as, kctx, 0, 0, + AS_COMMAND_UNLOCK, 0); + mutex_unlock(&kbdev->mmu_hw_mutex); + /* AS transaction end */ + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (as->poke_refcount && + !(as->poke_state & KBASE_AS_POKE_STATE_KILLING_POKE)) { + /* Only queue up the timer if we need it, and we're not trying to kill it */ + hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer) +{ + struct kbase_as *as; + int queue_work_ret; + + KBASE_DEBUG_ASSERT(NULL != timer); + as = container_of(timer, struct kbase_as, poke_timer); + KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); + + queue_work_ret = queue_work(as->poke_wq, &as->poke_work); + KBASE_DEBUG_ASSERT(queue_work_ret); + return HRTIMER_NORESTART; +} + +/** + * Retain the poking timer on an atom's context (if the atom hasn't already + * done so), and start the timer (if it's not already started). + * + * This must only be called on a context that's scheduled in, and an atom + * that's running on the GPU. + * + * The caller must hold hwaccess_lock + * + * This can be called safely from atomic context + */ +void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + struct kbase_as *as; + + KBASE_DEBUG_ASSERT(kbdev); + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (katom->poking) + return; + + katom->poking = 1; + + /* It's safe to work on the as/as_nr without an explicit reference, + * because the caller holds the hwaccess_lock, and the atom itself + * was also running and had already taken a reference */ + as = &kbdev->as[kctx->as_nr]; + + if (++(as->poke_refcount) == 1) { + /* First refcount for poke needed: check if not already in flight */ + if (!as->poke_state) { + /* need to start poking */ + as->poke_state |= KBASE_AS_POKE_STATE_IN_FLIGHT; + queue_work(as->poke_wq, &as->poke_work); + } + } +} + +/** + * If an atom holds a poking timer, release it and wait for it to finish + * + * This must only be called on a context that's scheduled in, and an atom + * that still has a JS reference on the context + * + * This must \b not be called from atomic context, since it can sleep. + */ +void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + struct kbase_as *as; + unsigned long flags; + + KBASE_DEBUG_ASSERT(kbdev); + KBASE_DEBUG_ASSERT(kctx); + KBASE_DEBUG_ASSERT(katom); + KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); + + if (!katom->poking) + return; + + as = &kbdev->as[kctx->as_nr]; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + KBASE_DEBUG_ASSERT(as->poke_refcount > 0); + KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); + + if (--(as->poke_refcount) == 0) { + as->poke_state |= KBASE_AS_POKE_STATE_KILLING_POKE; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + hrtimer_cancel(&as->poke_timer); + flush_workqueue(as->poke_wq); + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + /* Re-check whether it's still needed */ + if (as->poke_refcount) { + int queue_work_ret; + /* Poking still needed: + * - Another retain will not be starting the timer or queueing work, + * because it's still marked as in-flight + * - The hrtimer has finished, and has not started a new timer or + * queued work because it's been marked as killing + * + * So whatever happens now, just queue the work again */ + as->poke_state &= ~((kbase_as_poke_state)KBASE_AS_POKE_STATE_KILLING_POKE); + queue_work_ret = queue_work(as->poke_wq, &as->poke_work); + KBASE_DEBUG_ASSERT(queue_work_ret); + } else { + /* It isn't - so mark it as not in flight, and not killing */ + as->poke_state = 0u; + + /* The poke associated with the atom has now finished. If this is + * also the last atom on the context, then we can guarentee no more + * pokes (and thus no more poking register accesses) will occur on + * the context until new atoms are run */ + } + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + katom->poking = 0; +} + +void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (!kctx) { + dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Suprious IRQ or SW Design Error?\n", + kbase_as_has_bus_fault(as) ? "Bus error" : "Page fault", + as->number, as->fault_addr); + + /* Since no ctx was found, the MMU must be disabled. */ + WARN_ON(as->current_setup.transtab); + + if (kbase_as_has_bus_fault(as)) { + kbase_mmu_hw_clear_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); + } else if (kbase_as_has_page_fault(as)) { + kbase_mmu_hw_clear_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + kbase_mmu_hw_enable_fault(kbdev, as, kctx, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); + } + +#if KBASE_GPU_RESET_EN + if (kbase_as_has_bus_fault(as) && + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { + bool reset_status; + /* + * Reset the GPU, like in bus_fault_worker, in case an + * earlier error hasn't been properly cleared by this + * point. + */ + dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); + reset_status = kbase_prepare_to_reset_gpu_locked(kbdev); + if (reset_status) + kbase_reset_gpu_locked(kbdev); + } +#endif /* KBASE_GPU_RESET_EN */ + + return; + } + + if (kbase_as_has_bus_fault(as)) { + /* + * hw counters dumping in progress, signal the + * other thread that it failed + */ + if ((kbdev->hwcnt.kctx == kctx) && + (kbdev->hwcnt.backend.state == + KBASE_INSTR_STATE_DUMPING)) + kbdev->hwcnt.backend.state = + KBASE_INSTR_STATE_FAULT; + + /* + * Stop the kctx from submitting more jobs and cause it + * to be scheduled out/rescheduled when all references + * to it are released + */ + kbasep_js_clear_submit_allowed(js_devdata, kctx); + + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) + dev_warn(kbdev->dev, + "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", + as->number, as->fault_addr, + as->fault_extra_addr); + else + dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", + as->number, as->fault_addr); + + /* + * We need to switch to UNMAPPED mode - but we do this in a + * worker so that we can sleep + */ + kbdev->kbase_group_error++; + KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_busfault)); + WARN_ON(work_pending(&as->work_busfault)); + queue_work(as->pf_wq, &as->work_busfault); + atomic_inc(&kbdev->faults_pending); + } else { + kbdev->kbase_group_error++; + KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_pagefault)); + WARN_ON(work_pending(&as->work_pagefault)); + queue_work(as->pf_wq, &as->work_pagefault); + atomic_inc(&kbdev->faults_pending); + } +} + +void kbase_flush_mmu_wqs(struct kbase_device *kbdev) +{ + int i; + + for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + struct kbase_as *as = &kbdev->as[i]; + + flush_workqueue(as->pf_wq); + } +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h b/drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h new file mode 100755 index 000000000000..986e959e9a0c --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h @@ -0,0 +1,123 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file + * Interface file for accessing MMU hardware functionality + */ + +/** + * @page mali_kbase_mmu_hw_page MMU hardware interface + * + * @section mali_kbase_mmu_hw_intro_sec Introduction + * This module provides an abstraction for accessing the functionality provided + * by the midgard MMU and thus allows all MMU HW access to be contained within + * one common place and allows for different backends (implementations) to + * be provided. + */ + +#ifndef _MALI_KBASE_MMU_HW_H_ +#define _MALI_KBASE_MMU_HW_H_ + +/* Forward declarations */ +struct kbase_device; +struct kbase_as; +struct kbase_context; + +/** + * @addtogroup base_kbase_api + * @{ + */ + +/** + * @addtogroup mali_kbase_mmu_hw MMU access APIs + * @{ + */ + +/** @brief MMU fault type descriptor. + */ +enum kbase_mmu_fault_type { + KBASE_MMU_FAULT_TYPE_UNKNOWN = 0, + KBASE_MMU_FAULT_TYPE_PAGE, + KBASE_MMU_FAULT_TYPE_BUS, + KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED, + KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED +}; + +/** @brief Configure an address space for use. + * + * Configure the MMU using the address space details setup in the + * @ref kbase_context structure. + * + * @param[in] kbdev kbase device to configure. + * @param[in] as address space to configure. + * @param[in] kctx kbase context to configure. + */ +void kbase_mmu_hw_configure(struct kbase_device *kbdev, + struct kbase_as *as, struct kbase_context *kctx); + +/** @brief Issue an operation to the MMU. + * + * Issue an operation (MMU invalidate, MMU flush, etc) on the address space that + * is associated with the provided @ref kbase_context over the specified range + * + * @param[in] kbdev kbase device to issue the MMU operation on. + * @param[in] as address space to issue the MMU operation on. + * @param[in] kctx kbase context to issue the MMU operation on. + * @param[in] vpfn MMU Virtual Page Frame Number to start the + * operation on. + * @param[in] nr Number of pages to work on. + * @param[in] type Operation type (written to ASn_COMMAND). + * @param[in] handling_irq Is this operation being called during the handling + * of an interrupt? + * + * @return Zero if the operation was successful, non-zero otherwise. + */ +int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, u64 vpfn, u32 nr, u32 type, + unsigned int handling_irq); + +/** @brief Clear a fault that has been previously reported by the MMU. + * + * Clear a bus error or page fault that has been reported by the MMU. + * + * @param[in] kbdev kbase device to clear the fault from. + * @param[in] as address space to clear the fault from. + * @param[in] kctx kbase context to clear the fault from or NULL. + * @param[in] type The type of fault that needs to be cleared. + */ +void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, enum kbase_mmu_fault_type type); + +/** @brief Enable fault that has been previously reported by the MMU. + * + * After a page fault or bus error has been reported by the MMU these + * will be disabled. After these are handled this function needs to be + * called to enable the page fault or bus error fault again. + * + * @param[in] kbdev kbase device to again enable the fault from. + * @param[in] as address space to again enable the fault from. + * @param[in] kctx kbase context to again enable the fault from. + * @param[in] type The type of fault that needs to be enabled again. + */ +void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, + struct kbase_context *kctx, enum kbase_mmu_fault_type type); + +/** @} *//* end group mali_kbase_mmu_hw */ +/** @} *//* end group base_kbase_api */ + +#endif /* _MALI_KBASE_MMU_HW_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h new file mode 100755 index 000000000000..b487c00426ae --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h @@ -0,0 +1,47 @@ +/* + * + * (C) COPYRIGHT 2014-2015, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _MALI_KBASE_MMU_MODE_ +#define _MALI_KBASE_MMU_MODE_ + +#include + +/* Forward declarations */ +struct kbase_context; +struct kbase_device; +struct kbase_as; +struct kbase_mmu_setup; + +struct kbase_mmu_mode { + void (*update)(struct kbase_context *kctx); + void (*get_as_setup)(struct kbase_context *kctx, + struct kbase_mmu_setup * const setup); + void (*disable_as)(struct kbase_device *kbdev, int as_nr); + phys_addr_t (*pte_to_phy_addr)(u64 entry); + int (*ate_is_valid)(u64 ate); + int (*pte_is_valid)(u64 pte); + void (*entry_set_ate)(u64 *entry, phys_addr_t phy, unsigned long flags); + void (*entry_set_pte)(u64 *entry, phys_addr_t phy); + void (*entry_invalidate)(u64 *entry); +}; + +struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void); +struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void); + +#endif /* _MALI_KBASE_MMU_MODE_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_aarch64.c new file mode 100755 index 000000000000..60df171164ff --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_aarch64.c @@ -0,0 +1,200 @@ +/* + * + * (C) COPYRIGHT 2010-2014, 2016, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include "mali_kbase_mmu_mode.h" + +#include "mali_kbase.h" +#include "mali_midg_regmap.h" + +#define ENTRY_TYPE_MASK 3ULL +/* For valid ATEs bit 1 = (level == 3) ? 1 : 0. + * The MMU is only ever configured by the driver so that ATEs + * are at level 3, so bit 1 should always be set + */ +#define ENTRY_IS_ATE 3ULL +#define ENTRY_IS_INVAL 2ULL +#define ENTRY_IS_PTE 3ULL + +#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ +#define ENTRY_ACCESS_RW (1ULL << 6) /* bits 6:7 */ +#define ENTRY_ACCESS_RO (3ULL << 6) +#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ +#define ENTRY_ACCESS_BIT (1ULL << 10) +#define ENTRY_NX_BIT (1ULL << 54) + +/* Helper Function to perform assignment of page table entries, to + * ensure the use of strd, which is required on LPAE systems. + */ +static inline void page_table_entry_set(u64 *pte, u64 phy) +{ +#ifdef CONFIG_64BIT + *pte = phy; +#elif defined(CONFIG_ARM) + /* + * In order to prevent the compiler keeping cached copies of + * memory, we have to explicitly say that we have updated memory. + * + * Note: We could manually move the data ourselves into R0 and + * R1 by specifying register variables that are explicitly + * given registers assignments, the down side of this is that + * we have to assume cpu endianness. To avoid this we can use + * the ldrd to read the data from memory into R0 and R1 which + * will respect the cpu endianness, we then use strd to make + * the 64 bit assignment to the page table entry. + */ + asm volatile("ldrd r0, r1, [%[ptemp]]\n\t" + "strd r0, r1, [%[pte]]\n\t" + : "=m" (*pte) + : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy) + : "r0", "r1"); +#else +#error "64-bit atomic write must be implemented for your architecture" +#endif +} + +static void mmu_get_as_setup(struct kbase_context *kctx, + struct kbase_mmu_setup * const setup) +{ + /* Set up the required caching policies at the correct indices + * in the memattr register. + */ + setup->memattr = + (AS_MEMATTR_IMPL_DEF_CACHE_POLICY << + (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | + (AS_MEMATTR_FORCE_TO_CACHE_ALL << + (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | + (AS_MEMATTR_WRITE_ALLOC << + (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | + (AS_MEMATTR_AARCH64_OUTER_IMPL_DEF << + (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | + (AS_MEMATTR_AARCH64_OUTER_WA << + (AS_MEMATTR_INDEX_OUTER_WA * 8)); + + setup->transtab = (u64)kctx->pgd & AS_TRANSTAB_BASE_MASK; + setup->transcfg = AS_TRANSCFG_ADRMODE_AARCH64_4K; +} + +static void mmu_update(struct kbase_context *kctx) +{ + struct kbase_device * const kbdev = kctx->kbdev; + struct kbase_as * const as = &kbdev->as[kctx->as_nr]; + struct kbase_mmu_setup * const current_setup = &as->current_setup; + + mmu_get_as_setup(kctx, current_setup); + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as, kctx); +} + +static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) +{ + struct kbase_as * const as = &kbdev->as[as_nr]; + struct kbase_mmu_setup * const current_setup = &as->current_setup; + + current_setup->transtab = 0ULL; + current_setup->transcfg = AS_TRANSCFG_ADRMODE_UNMAPPED; + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as, NULL); +} + +static phys_addr_t pte_to_phy_addr(u64 entry) +{ + if (!(entry & 1)) + return 0; + + return entry & ~0xFFF; +} + +static int ate_is_valid(u64 ate) +{ + return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); +} + +static int pte_is_valid(u64 pte) +{ + return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); +} + +/* + * Map KBASE_REG flags to MMU flags + */ +static u64 get_mmu_flags(unsigned long flags) +{ + u64 mmu_flags; + + /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ + mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; + + /* Set access flags - note that AArch64 stage 1 does not support + * write-only access, so we use read/write instead + */ + if (flags & KBASE_REG_GPU_WR) + mmu_flags |= ENTRY_ACCESS_RW; + else if (flags & KBASE_REG_GPU_RD) + mmu_flags |= ENTRY_ACCESS_RO; + + /* nx if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; + + if (flags & KBASE_REG_SHARE_BOTH) { + /* inner and outer shareable */ + mmu_flags |= SHARE_BOTH_BITS; + } else if (flags & KBASE_REG_SHARE_IN) { + /* inner shareable coherency */ + mmu_flags |= SHARE_INNER_BITS; + } + + return mmu_flags; +} + +static void entry_set_ate(u64 *entry, phys_addr_t phy, unsigned long flags) +{ + page_table_entry_set(entry, (phy & ~0xFFF) | + get_mmu_flags(flags) | + ENTRY_ACCESS_BIT | ENTRY_IS_ATE); +} + +static void entry_set_pte(u64 *entry, phys_addr_t phy) +{ + page_table_entry_set(entry, (phy & ~0xFFF) | + ENTRY_ACCESS_BIT | ENTRY_IS_PTE); +} + +static void entry_invalidate(u64 *entry) +{ + page_table_entry_set(entry, ENTRY_IS_INVAL); +} + +static struct kbase_mmu_mode const aarch64_mode = { + .update = mmu_update, + .get_as_setup = mmu_get_as_setup, + .disable_as = mmu_disable_as, + .pte_to_phy_addr = pte_to_phy_addr, + .ate_is_valid = ate_is_valid, + .pte_is_valid = pte_is_valid, + .entry_set_ate = entry_set_ate, + .entry_set_pte = entry_set_pte, + .entry_invalidate = entry_invalidate +}; + +struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void) +{ + return &aarch64_mode; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_lpae.c b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_lpae.c new file mode 100755 index 000000000000..53fbbc73af91 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_lpae.c @@ -0,0 +1,198 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include "mali_kbase_mmu_mode.h" + +#include "mali_kbase.h" +#include "mali_midg_regmap.h" + +#define ENTRY_TYPE_MASK 3ULL +#define ENTRY_IS_ATE 1ULL +#define ENTRY_IS_INVAL 2ULL +#define ENTRY_IS_PTE 3ULL + +#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ +#define ENTRY_RD_BIT (1ULL << 6) +#define ENTRY_WR_BIT (1ULL << 7) +#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ +#define ENTRY_ACCESS_BIT (1ULL << 10) +#define ENTRY_NX_BIT (1ULL << 54) + +#define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | \ + ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT) + +/* Helper Function to perform assignment of page table entries, to + * ensure the use of strd, which is required on LPAE systems. + */ +static inline void page_table_entry_set(u64 *pte, u64 phy) +{ +#ifdef CONFIG_64BIT + *pte = phy; +#elif defined(CONFIG_ARM) + /* + * In order to prevent the compiler keeping cached copies of + * memory, we have to explicitly say that we have updated + * memory. + * + * Note: We could manually move the data ourselves into R0 and + * R1 by specifying register variables that are explicitly + * given registers assignments, the down side of this is that + * we have to assume cpu endianness. To avoid this we can use + * the ldrd to read the data from memory into R0 and R1 which + * will respect the cpu endianness, we then use strd to make + * the 64 bit assignment to the page table entry. + */ + asm volatile("ldrd r0, r1, [%[ptemp]]\n\t" + "strd r0, r1, [%[pte]]\n\t" + : "=m" (*pte) + : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy) + : "r0", "r1"); +#else +#error "64-bit atomic write must be implemented for your architecture" +#endif +} + +static void mmu_get_as_setup(struct kbase_context *kctx, + struct kbase_mmu_setup * const setup) +{ + /* Set up the required caching policies at the correct indices + * in the memattr register. */ + setup->memattr = + (AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY << + (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | + (AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL << + (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | + (AS_MEMATTR_LPAE_WRITE_ALLOC << + (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | + (AS_MEMATTR_LPAE_OUTER_IMPL_DEF << + (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | + (AS_MEMATTR_LPAE_OUTER_WA << + (AS_MEMATTR_INDEX_OUTER_WA * 8)) | + 0; /* The other indices are unused for now */ + + setup->transtab = ((u64)kctx->pgd & + ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_LPAE_ADDR_SPACE_MASK)) | + AS_TRANSTAB_LPAE_ADRMODE_TABLE | + AS_TRANSTAB_LPAE_READ_INNER; + + setup->transcfg = 0; +} + +static void mmu_update(struct kbase_context *kctx) +{ + struct kbase_device * const kbdev = kctx->kbdev; + struct kbase_as * const as = &kbdev->as[kctx->as_nr]; + struct kbase_mmu_setup * const current_setup = &as->current_setup; + + mmu_get_as_setup(kctx, current_setup); + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as, kctx); +} + +static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) +{ + struct kbase_as * const as = &kbdev->as[as_nr]; + struct kbase_mmu_setup * const current_setup = &as->current_setup; + + current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED; + + /* Apply the address space setting */ + kbase_mmu_hw_configure(kbdev, as, NULL); +} + +static phys_addr_t pte_to_phy_addr(u64 entry) +{ + if (!(entry & 1)) + return 0; + + return entry & ~0xFFF; +} + +static int ate_is_valid(u64 ate) +{ + return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); +} + +static int pte_is_valid(u64 pte) +{ + return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); +} + +/* + * Map KBASE_REG flags to MMU flags + */ +static u64 get_mmu_flags(unsigned long flags) +{ + u64 mmu_flags; + + /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ + mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; + + /* write perm if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0; + /* read perm if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0; + /* nx if requested */ + mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; + + if (flags & KBASE_REG_SHARE_BOTH) { + /* inner and outer shareable */ + mmu_flags |= SHARE_BOTH_BITS; + } else if (flags & KBASE_REG_SHARE_IN) { + /* inner shareable coherency */ + mmu_flags |= SHARE_INNER_BITS; + } + + return mmu_flags; +} + +static void entry_set_ate(u64 *entry, phys_addr_t phy, unsigned long flags) +{ + page_table_entry_set(entry, (phy & ~0xFFF) | + get_mmu_flags(flags) | + ENTRY_IS_ATE); +} + +static void entry_set_pte(u64 *entry, phys_addr_t phy) +{ + page_table_entry_set(entry, (phy & ~0xFFF) | ENTRY_IS_PTE); +} + +static void entry_invalidate(u64 *entry) +{ + page_table_entry_set(entry, ENTRY_IS_INVAL); +} + +static struct kbase_mmu_mode const lpae_mode = { + .update = mmu_update, + .get_as_setup = mmu_get_as_setup, + .disable_as = mmu_disable_as, + .pte_to_phy_addr = pte_to_phy_addr, + .ate_is_valid = ate_is_valid, + .pte_is_valid = pte_is_valid, + .entry_set_ate = entry_set_ate, + .entry_set_pte = entry_set_pte, + .entry_invalidate = entry_invalidate +}; + +struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void) +{ + return &lpae_mode; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c b/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c new file mode 100755 index 000000000000..1a44957fe44a --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c @@ -0,0 +1,124 @@ +/* + * + * (C) COPYRIGHT 2011-2014, 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifdef CONFIG_MALI_PLATFORM_FAKE + +#include +#include +#include +#include +#include + + +/* + * This file is included only for type definitions and functions belonging to + * specific platform folders. Do not add dependencies with symbols that are + * defined somewhere else. + */ +#include + +#define PLATFORM_CONFIG_RESOURCE_COUNT 4 +#define PLATFORM_CONFIG_IRQ_RES_COUNT 3 + +static struct platform_device *mali_device; + +#ifndef CONFIG_OF +/** + * @brief Convert data in struct kbase_io_resources struct to Linux-specific resources + * + * Function converts data in struct kbase_io_resources struct to an array of Linux resource structures. Note that function + * assumes that size of linux_resource array is at least PLATFORM_CONFIG_RESOURCE_COUNT. + * Resources are put in fixed order: I/O memory region, job IRQ, MMU IRQ, GPU IRQ. + * + * @param[in] io_resource Input IO resource data + * @param[out] linux_resources Pointer to output array of Linux resource structures + */ +static void kbasep_config_parse_io_resources(const struct kbase_io_resources *io_resources, struct resource *const linux_resources) +{ + if (!io_resources || !linux_resources) { + pr_err("%s: couldn't find proper resources\n", __func__); + return; + } + + memset(linux_resources, 0, PLATFORM_CONFIG_RESOURCE_COUNT * sizeof(struct resource)); + + linux_resources[0].start = io_resources->io_memory_region.start; + linux_resources[0].end = io_resources->io_memory_region.end; + linux_resources[0].flags = IORESOURCE_MEM; + + linux_resources[1].start = io_resources->job_irq_number; + linux_resources[1].end = io_resources->job_irq_number; + linux_resources[1].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; + + linux_resources[2].start = io_resources->mmu_irq_number; + linux_resources[2].end = io_resources->mmu_irq_number; + linux_resources[2].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; + + linux_resources[3].start = io_resources->gpu_irq_number; + linux_resources[3].end = io_resources->gpu_irq_number; + linux_resources[3].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; +} +#endif /* CONFIG_OF */ + +int kbase_platform_fake_register(void) +{ + struct kbase_platform_config *config; +#ifndef CONFIG_OF + struct resource resources[PLATFORM_CONFIG_RESOURCE_COUNT]; +#endif + int err; + + config = kbase_get_platform_config(); /* declared in midgard/mali_kbase_config.h but defined in platform folder */ + if (config == NULL) { + pr_err("%s: couldn't get platform config\n", __func__); + return -ENODEV; + } + + mali_device = platform_device_alloc("mali", 0); + if (mali_device == NULL) + return -ENOMEM; + +#ifndef CONFIG_OF + kbasep_config_parse_io_resources(config->io_resources, resources); + err = platform_device_add_resources(mali_device, resources, PLATFORM_CONFIG_RESOURCE_COUNT); + if (err) { + platform_device_put(mali_device); + mali_device = NULL; + return err; + } +#endif /* CONFIG_OF */ + + err = platform_device_add(mali_device); + if (err) { + platform_device_unregister(mali_device); + mali_device = NULL; + return err; + } + + return 0; +} +EXPORT_SYMBOL(kbase_platform_fake_register); + +void kbase_platform_fake_unregister(void) +{ + if (mali_device) + platform_device_unregister(mali_device); +} +EXPORT_SYMBOL(kbase_platform_fake_unregister); + +#endif /* CONFIG_MALI_PLATFORM_FAKE */ + diff --git a/drivers/gpu/arm/midgard/mali_kbase_pm.c b/drivers/gpu/arm/midgard/mali_kbase_pm.c new file mode 100755 index 000000000000..97d543464c28 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_pm.c @@ -0,0 +1,205 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_pm.c + * Base kernel power management APIs + */ + +#include +#include +#include + +#include + +int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) +{ + return kbase_hwaccess_pm_powerup(kbdev, flags); +} + +void kbase_pm_halt(struct kbase_device *kbdev) +{ + kbase_hwaccess_pm_halt(kbdev); +} + +void kbase_pm_context_active(struct kbase_device *kbdev) +{ + (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); +} + +int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + int c; + int old_count; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + /* Trace timeline information about how long it took to handle the decision + * to powerup. Sometimes the event might be missed due to reading the count + * outside of mutex, but this is necessary to get the trace timing + * correct. */ + old_count = kbdev->pm.active_count; + if (old_count == 0) + kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + if (kbase_pm_is_suspending(kbdev)) { + switch (suspend_handler) { + case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: + if (kbdev->pm.active_count != 0) + break; + /* FALLTHROUGH */ + case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + if (old_count == 0) + kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); + return 1; + + case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE: + /* FALLTHROUGH */ + default: + KBASE_DEBUG_ASSERT_MSG(false, "unreachable"); + break; + } + } + c = ++kbdev->pm.active_count; + KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c); + KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_ACTIVE, NULL, NULL, 0u, c); + + /* Trace the event being handled */ + if (old_count == 0) + kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); + + if (c == 1) + /* First context active: Power on the GPU and any cores requested by + * the policy */ + kbase_hwaccess_pm_gpu_active(kbdev); + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); + + return 0; +} + +KBASE_EXPORT_TEST_API(kbase_pm_context_active); + +void kbase_pm_context_idle(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + int c; + int old_count; + + KBASE_DEBUG_ASSERT(kbdev != NULL); + + /* Trace timeline information about how long it took to handle the decision + * to powerdown. Sometimes the event might be missed due to reading the + * count outside of mutex, but this is necessary to get the trace timing + * correct. */ + old_count = kbdev->pm.active_count; + if (old_count == 0) + kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE); + + mutex_lock(&js_devdata->runpool_mutex); + mutex_lock(&kbdev->pm.lock); + + c = --kbdev->pm.active_count; + KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c); + KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_IDLE, NULL, NULL, 0u, c); + + KBASE_DEBUG_ASSERT(c >= 0); + + /* Trace the event being handled */ + if (old_count == 0) + kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE); + + if (c == 0) { + /* Last context has gone idle */ + kbase_hwaccess_pm_gpu_idle(kbdev); + + /* Wake up anyone waiting for this to become 0 (e.g. suspend). The + * waiters must synchronize with us by locking the pm.lock after + * waiting */ + wake_up(&kbdev->pm.zero_active_count_wait); + } + + mutex_unlock(&kbdev->pm.lock); + mutex_unlock(&js_devdata->runpool_mutex); +} + +KBASE_EXPORT_TEST_API(kbase_pm_context_idle); + +void kbase_pm_suspend(struct kbase_device *kbdev) +{ + KBASE_DEBUG_ASSERT(kbdev); + + /* Suspend vinstr. + * This call will block until vinstr is suspended. */ + kbase_vinstr_suspend(kbdev->vinstr_ctx); + + mutex_lock(&kbdev->pm.lock); + KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); + kbdev->pm.suspending = true; + mutex_unlock(&kbdev->pm.lock); + + /* From now on, the active count will drop towards zero. Sometimes, it'll + * go up briefly before going down again. However, once it reaches zero it + * will stay there - guaranteeing that we've idled all pm references */ + + /* Suspend job scheduler and associated components, so that it releases all + * the PM active count references */ + kbasep_js_suspend(kbdev); + + /* Wait for the active count to reach zero. This is not the same as + * waiting for a power down, since not all policies power down when this + * reaches zero. */ + wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0); + + /* NOTE: We synchronize with anything that was just finishing a + * kbase_pm_context_idle() call by locking the pm.lock below */ + + kbase_hwaccess_pm_suspend(kbdev); +} + +void kbase_pm_resume(struct kbase_device *kbdev) +{ + /* MUST happen before any pm_context_active calls occur */ + kbase_hwaccess_pm_resume(kbdev); + + /* Initial active call, to power on the GPU/cores if needed */ + kbase_pm_context_active(kbdev); + + /* Resume any blocked atoms (which may cause contexts to be scheduled in + * and dependent atoms to run) */ + kbase_resume_suspended_soft_jobs(kbdev); + + /* Resume the Job Scheduler and associated components, and start running + * atoms */ + kbasep_js_resume(kbdev); + + /* Matching idle call, to power off the GPU/cores if we didn't actually + * need it and the policy doesn't want it on */ + kbase_pm_context_idle(kbdev); + + /* Resume vinstr operation */ + kbase_vinstr_resume(kbdev->vinstr_ctx); +} + diff --git a/drivers/gpu/arm/midgard/mali_kbase_pm.h b/drivers/gpu/arm/midgard/mali_kbase_pm.h new file mode 100755 index 000000000000..37fa2479df74 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_pm.h @@ -0,0 +1,171 @@ +/* + * + * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_kbase_pm.h + * Power management API definitions + */ + +#ifndef _KBASE_PM_H_ +#define _KBASE_PM_H_ + +#include "mali_kbase_hwaccess_pm.h" + +#define PM_ENABLE_IRQS 0x01 +#define PM_HW_ISSUES_DETECT 0x02 + + +/** Initialize the power management framework. + * + * Must be called before any other power management function + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + * + * @return 0 if the power management framework was successfully initialized. + */ +int kbase_pm_init(struct kbase_device *kbdev); + +/** Power up GPU after all modules have been initialized and interrupt handlers installed. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + * + * @param flags Flags to pass on to kbase_pm_init_hw + * + * @return 0 if powerup was successful. + */ +int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags); + +/** + * Halt the power management framework. + * Should ensure that no new interrupts are generated, + * but allow any currently running interrupt handlers to complete successfully. + * The GPU is forced off by the time this function returns, regardless of + * whether or not the active power policy asks for the GPU to be powered off. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_halt(struct kbase_device *kbdev); + +/** Terminate the power management framework. + * + * No power management functions may be called after this + * (except @ref kbase_pm_init) + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_term(struct kbase_device *kbdev); + +/** Increment the count of active contexts. + * + * This function should be called when a context is about to submit a job. It informs the active power policy that the + * GPU is going to be in use shortly and the policy is expected to start turning on the GPU. + * + * This function will block until the GPU is available. + * + * This function ASSERTS if a suspend is occuring/has occurred whilst this is + * in use. Use kbase_pm_contect_active_unless_suspending() instead. + * + * @note a Suspend is only visible to Kernel threads; user-space threads in a + * syscall cannot witness a suspend, because they are frozen before the suspend + * begins. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_context_active(struct kbase_device *kbdev); + + +/** Handler codes for doing kbase_pm_context_active_handle_suspend() */ +enum kbase_pm_suspend_handler { + /** A suspend is not expected/not possible - this is the same as + * kbase_pm_context_active() */ + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, + /** If we're suspending, fail and don't increase the active count */ + KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE, + /** If we're suspending, succeed and allow the active count to increase iff + * it didn't go from 0->1 (i.e., we didn't re-activate the GPU). + * + * This should only be used when there is a bounded time on the activation + * (e.g. guarantee it's going to be idled very soon after) */ + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE +}; + +/** Suspend 'safe' variant of kbase_pm_context_active() + * + * If a suspend is in progress, this allows for various different ways of + * handling the suspend. Refer to @ref enum kbase_pm_suspend_handler for details. + * + * We returns a status code indicating whether we're allowed to keep the GPU + * active during the suspend, depending on the handler code. If the status code + * indicates a failure, the caller must abort whatever operation it was + * attempting, and potentially queue it up for after the OS has resumed. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + * @param suspend_handler The handler code for how to handle a suspend that might occur + * @return zero Indicates success + * @return non-zero Indicates failure due to the system being suspending/suspended. + */ +int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler); + +/** Decrement the reference count of active contexts. + * + * This function should be called when a context becomes idle. After this call the GPU may be turned off by the power + * policy so the calling code should ensure that it does not access the GPU's registers. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_context_idle(struct kbase_device *kbdev); + +/** + * Suspend the GPU and prevent any further register accesses to it from Kernel + * threads. + * + * This is called in response to an OS suspend event, and calls into the various + * kbase components to complete the suspend. + * + * @note the mechanisms used here rely on all user-space threads being frozen + * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up + * the GPU e.g. via atom submission. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_suspend(struct kbase_device *kbdev); + +/** + * Resume the GPU, allow register accesses to it, and resume running atoms on + * the GPU. + * + * This is called in response to an OS resume event, and calls into the various + * kbase components to complete the resume. + * + * @param kbdev The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_resume(struct kbase_device *kbdev); + +/** + * kbase_pm_vsync_callback - vsync callback + * + * @buffer_updated: 1 if a new frame was displayed, 0 otherwise + * @data: Pointer to the kbase device as returned by kbase_find_device() + * + * Callback function used to notify the power management code that a vsync has + * occurred on the display. + */ +void kbase_pm_vsync_callback(int buffer_updated, void *data); + +#endif /* _KBASE_PM_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_profiling_gator_api.h b/drivers/gpu/arm/midgard/mali_kbase_profiling_gator_api.h new file mode 100755 index 000000000000..7fb674eded37 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_profiling_gator_api.h @@ -0,0 +1,40 @@ +/* + * + * (C) COPYRIGHT 2010, 2013 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_profiling_gator_api.h + * Model interface + */ + +#ifndef _KBASE_PROFILING_GATOR_API_H_ +#define _KBASE_PROFILING_GATOR_API_H_ + +/* + * List of possible actions to be controlled by Streamline. + * The following numbers are used by gator to control + * the frame buffer dumping and s/w counter reporting. + */ +#define FBDUMP_CONTROL_ENABLE (1) +#define FBDUMP_CONTROL_RATE (2) +#define SW_COUNTER_ENABLE (3) +#define FBDUMP_CONTROL_RESIZE_FACTOR (4) +#define FBDUMP_CONTROL_MAX (5) +#define FBDUMP_CONTROL_MIN FBDUMP_CONTROL_ENABLE + +void _mali_profiling_control(u32 action, u32 value); + +#endif /* _KBASE_PROFILING_GATOR_API */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c new file mode 100755 index 000000000000..c970650069cd --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c @@ -0,0 +1,130 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "mali_kbase.h" + +#include "mali_kbase_regs_history_debugfs.h" + +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) + +#include + + +static int regs_history_size_get(void *data, u64 *val) +{ + struct kbase_io_history *const h = data; + + *val = h->size; + + return 0; +} + +static int regs_history_size_set(void *data, u64 val) +{ + struct kbase_io_history *const h = data; + + return kbase_io_history_resize(h, (u16)val); +} + + +DEFINE_SIMPLE_ATTRIBUTE(regs_history_size_fops, + regs_history_size_get, + regs_history_size_set, + "%llu\n"); + + +/** + * regs_history_show - show callback for the register access history file. + * + * @sfile: The debugfs entry + * @data: Data associated with the entry + * + * This function is called to dump all recent accesses to the GPU registers. + * + * @return 0 if successfully prints data in debugfs entry file, failure + * otherwise + */ +static int regs_history_show(struct seq_file *sfile, void *data) +{ + struct kbase_io_history *const h = sfile->private; + u16 i; + size_t iters; + unsigned long flags; + + if (!h->enabled) { + seq_puts(sfile, "The register access history is disabled\n"); + goto out; + } + + spin_lock_irqsave(&h->lock, flags); + + iters = (h->size > h->count) ? h->count : h->size; + seq_printf(sfile, "Last %zu register accesses of %zu total:\n", iters, + h->count); + for (i = 0; i < iters; ++i) { + struct kbase_io_access *io = + &h->buf[(h->count - iters + i) % h->size]; + char const access = (io->addr & 1) ? 'w' : 'r'; + + seq_printf(sfile, "%6i: %c: reg 0x%p val %08x\n", i, access, + (void *)(io->addr & ~0x1), io->value); + } + + spin_unlock_irqrestore(&h->lock, flags); + +out: + return 0; +} + + +/** + * regs_history_open - open operation for regs_history debugfs file + * + * @in: &struct inode pointer + * @file: &struct file pointer + * + * @return file descriptor + */ +static int regs_history_open(struct inode *in, struct file *file) +{ + return single_open(file, ®s_history_show, in->i_private); +} + + +static const struct file_operations regs_history_fops = { + .open = ®s_history_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + + +void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_bool("regs_history_enabled", S_IRUGO | S_IWUSR, + kbdev->mali_debugfs_directory, + &kbdev->io_history.enabled); + debugfs_create_file("regs_history_size", S_IRUGO | S_IWUSR, + kbdev->mali_debugfs_directory, + &kbdev->io_history, ®s_history_size_fops); + debugfs_create_file("regs_history", S_IRUGO, + kbdev->mali_debugfs_directory, &kbdev->io_history, + ®s_history_fops); +} + + +#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h new file mode 100755 index 000000000000..f10837002330 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h @@ -0,0 +1,50 @@ +/* + * + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * Header file for register access history support via debugfs + * + * This interface is made available via /sys/kernel/debug/mali#/regs_history*. + * + * Usage: + * - regs_history_enabled: whether recording of register accesses is enabled. + * Write 'y' to enable, 'n' to disable. + * - regs_history_size: size of the register history buffer, must be > 0 + * - regs_history: return the information about last accesses to the registers. + */ + +#ifndef _KBASE_REGS_HISTORY_DEBUGFS_H +#define _KBASE_REGS_HISTORY_DEBUGFS_H + +struct kbase_device; + +#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) + +/** + * kbasep_regs_history_debugfs_init - add debugfs entries for register history + * + * @kbdev: Pointer to kbase_device containing the register history + */ +void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev); + +#else /* CONFIG_DEBUG_FS */ + +#define kbasep_regs_history_debugfs_init CSTD_NOP + +#endif /* CONFIG_DEBUG_FS */ + +#endif /*_KBASE_REGS_HISTORY_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/midgard/mali_kbase_replay.c b/drivers/gpu/arm/midgard/mali_kbase_replay.c new file mode 100755 index 000000000000..84aa3316e435 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_replay.c @@ -0,0 +1,1166 @@ +/* + * + * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_replay.c + * Replay soft job handlers + */ + +#include +#include +#include +#include +#include + +#define JOB_NOT_STARTED 0 +#define JOB_TYPE_NULL (1) +#define JOB_TYPE_VERTEX (5) +#define JOB_TYPE_TILER (7) +#define JOB_TYPE_FUSED (8) +#define JOB_TYPE_FRAGMENT (9) + +#define JOB_HEADER_32_FBD_OFFSET (31*4) +#define JOB_HEADER_64_FBD_OFFSET (44*4) + +#define FBD_POINTER_MASK (~0x3f) + +#define SFBD_TILER_OFFSET (48*4) + +#define MFBD_TILER_OFFSET (14*4) + +#define FBD_HIERARCHY_WEIGHTS 8 +#define FBD_HIERARCHY_MASK_MASK 0x1fff + +#define FBD_TYPE 1 + +#define HIERARCHY_WEIGHTS 13 + +#define JOB_HEADER_ID_MAX 0xffff + +#define JOB_SOURCE_ID(status) (((status) >> 16) & 0xFFFF) +#define JOB_POLYGON_LIST (0x03) + +struct fragment_job { + struct job_descriptor_header header; + + u32 x[2]; + union { + u64 _64; + u32 _32; + } fragment_fbd; +}; + +static void dump_job_head(struct kbase_context *kctx, char *head_str, + struct job_descriptor_header *job) +{ +#ifdef CONFIG_MALI_DEBUG + dev_dbg(kctx->kbdev->dev, "%s\n", head_str); + dev_dbg(kctx->kbdev->dev, + "addr = %p\n" + "exception_status = %x (Source ID: 0x%x Access: 0x%x Exception: 0x%x)\n" + "first_incomplete_task = %x\n" + "fault_pointer = %llx\n" + "job_descriptor_size = %x\n" + "job_type = %x\n" + "job_barrier = %x\n" + "_reserved_01 = %x\n" + "_reserved_02 = %x\n" + "_reserved_03 = %x\n" + "_reserved_04/05 = %x,%x\n" + "job_index = %x\n" + "dependencies = %x,%x\n", + job, job->exception_status, + JOB_SOURCE_ID(job->exception_status), + (job->exception_status >> 8) & 0x3, + job->exception_status & 0xFF, + job->first_incomplete_task, + job->fault_pointer, job->job_descriptor_size, + job->job_type, job->job_barrier, job->_reserved_01, + job->_reserved_02, job->_reserved_03, + job->_reserved_04, job->_reserved_05, + job->job_index, + job->job_dependency_index_1, + job->job_dependency_index_2); + + if (job->job_descriptor_size) + dev_dbg(kctx->kbdev->dev, "next = %llx\n", + job->next_job._64); + else + dev_dbg(kctx->kbdev->dev, "next = %x\n", + job->next_job._32); +#endif +} + +static int kbasep_replay_reset_sfbd(struct kbase_context *kctx, + u64 fbd_address, u64 tiler_heap_free, + u16 hierarchy_mask, u32 default_weight) +{ + struct { + u32 padding_1[1]; + u32 flags; + u64 padding_2[2]; + u64 heap_free_address; + u32 padding[8]; + u32 weights[FBD_HIERARCHY_WEIGHTS]; + } *fbd_tiler; + struct kbase_vmap_struct map; + + dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address); + + fbd_tiler = kbase_vmap(kctx, fbd_address + SFBD_TILER_OFFSET, + sizeof(*fbd_tiler), &map); + if (!fbd_tiler) { + dev_err(kctx->kbdev->dev, "kbasep_replay_reset_fbd: failed to map fbd\n"); + return -EINVAL; + } + +#ifdef CONFIG_MALI_DEBUG + dev_dbg(kctx->kbdev->dev, + "FBD tiler:\n" + "flags = %x\n" + "heap_free_address = %llx\n", + fbd_tiler->flags, fbd_tiler->heap_free_address); +#endif + if (hierarchy_mask) { + u32 weights[HIERARCHY_WEIGHTS]; + u16 old_hierarchy_mask = fbd_tiler->flags & + FBD_HIERARCHY_MASK_MASK; + int i, j = 0; + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) { + if (old_hierarchy_mask & (1 << i)) { + KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); + weights[i] = fbd_tiler->weights[j++]; + } else { + weights[i] = default_weight; + } + } + + + dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x New hierarchy mask=%x\n", + old_hierarchy_mask, hierarchy_mask); + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) + dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n", + i, weights[i]); + + j = 0; + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) { + if (hierarchy_mask & (1 << i)) { + KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); + + dev_dbg(kctx->kbdev->dev, " Writing hierarchy level %02d (%08x) to %d\n", + i, weights[i], j); + + fbd_tiler->weights[j++] = weights[i]; + } + } + + for (; j < FBD_HIERARCHY_WEIGHTS; j++) + fbd_tiler->weights[j] = 0; + + fbd_tiler->flags = hierarchy_mask | (1 << 16); + } + + fbd_tiler->heap_free_address = tiler_heap_free; + + dev_dbg(kctx->kbdev->dev, "heap_free_address=%llx flags=%x\n", + fbd_tiler->heap_free_address, fbd_tiler->flags); + + kbase_vunmap(kctx, &map); + + return 0; +} + +static int kbasep_replay_reset_mfbd(struct kbase_context *kctx, + u64 fbd_address, u64 tiler_heap_free, + u16 hierarchy_mask, u32 default_weight) +{ + struct kbase_vmap_struct map; + struct { + u32 padding_0; + u32 flags; + u64 padding_1[2]; + u64 heap_free_address; + u64 padding_2; + u32 weights[FBD_HIERARCHY_WEIGHTS]; + } *fbd_tiler; + + dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address); + + fbd_tiler = kbase_vmap(kctx, fbd_address + MFBD_TILER_OFFSET, + sizeof(*fbd_tiler), &map); + if (!fbd_tiler) { + dev_err(kctx->kbdev->dev, + "kbasep_replay_reset_fbd: failed to map fbd\n"); + return -EINVAL; + } + +#ifdef CONFIG_MALI_DEBUG + dev_dbg(kctx->kbdev->dev, "FBD tiler:\n" + "flags = %x\n" + "heap_free_address = %llx\n", + fbd_tiler->flags, + fbd_tiler->heap_free_address); +#endif + if (hierarchy_mask) { + u32 weights[HIERARCHY_WEIGHTS]; + u16 old_hierarchy_mask = (fbd_tiler->flags) & + FBD_HIERARCHY_MASK_MASK; + int i, j = 0; + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) { + if (old_hierarchy_mask & (1 << i)) { + KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); + weights[i] = fbd_tiler->weights[j++]; + } else { + weights[i] = default_weight; + } + } + + + dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x New hierarchy mask=%x\n", + old_hierarchy_mask, hierarchy_mask); + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) + dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n", + i, weights[i]); + + j = 0; + + for (i = 0; i < HIERARCHY_WEIGHTS; i++) { + if (hierarchy_mask & (1 << i)) { + KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); + + dev_dbg(kctx->kbdev->dev, + " Writing hierarchy level %02d (%08x) to %d\n", + i, weights[i], j); + + fbd_tiler->weights[j++] = weights[i]; + } + } + + for (; j < FBD_HIERARCHY_WEIGHTS; j++) + fbd_tiler->weights[j] = 0; + + fbd_tiler->flags = hierarchy_mask | (1 << 16); + } + + fbd_tiler->heap_free_address = tiler_heap_free; + + kbase_vunmap(kctx, &map); + + return 0; +} + +/** + * @brief Reset the status of an FBD pointed to by a tiler job + * + * This performs two functions : + * - Set the hierarchy mask + * - Reset the tiler free heap address + * + * @param[in] kctx Context pointer + * @param[in] job_header Address of job header to reset. + * @param[in] tiler_heap_free The value to reset Tiler Heap Free to + * @param[in] hierarchy_mask The hierarchy mask to use + * @param[in] default_weight Default hierarchy weight to write when no other + * weight is given in the FBD + * @param[in] job_64 true if this job is using 64-bit + * descriptors + * + * @return 0 on success, error code on failure + */ +static int kbasep_replay_reset_tiler_job(struct kbase_context *kctx, + u64 job_header, u64 tiler_heap_free, + u16 hierarchy_mask, u32 default_weight, bool job_64) +{ + struct kbase_vmap_struct map; + u64 fbd_address; + + if (job_64) { + u64 *job_ext; + + job_ext = kbase_vmap(kctx, + job_header + JOB_HEADER_64_FBD_OFFSET, + sizeof(*job_ext), &map); + + if (!job_ext) { + dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n"); + return -EINVAL; + } + + fbd_address = *job_ext; + + kbase_vunmap(kctx, &map); + } else { + u32 *job_ext; + + job_ext = kbase_vmap(kctx, + job_header + JOB_HEADER_32_FBD_OFFSET, + sizeof(*job_ext), &map); + + if (!job_ext) { + dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n"); + return -EINVAL; + } + + fbd_address = *job_ext; + + kbase_vunmap(kctx, &map); + } + + if (fbd_address & FBD_TYPE) { + return kbasep_replay_reset_mfbd(kctx, + fbd_address & FBD_POINTER_MASK, + tiler_heap_free, + hierarchy_mask, + default_weight); + } else { + return kbasep_replay_reset_sfbd(kctx, + fbd_address & FBD_POINTER_MASK, + tiler_heap_free, + hierarchy_mask, + default_weight); + } +} + +/** + * @brief Reset the status of a job + * + * This performs the following functions : + * + * - Reset the Job Status field of each job to NOT_STARTED. + * - Set the Job Type field of any Vertex Jobs to Null Job. + * - For any jobs using an FBD, set the Tiler Heap Free field to the value of + * the tiler_heap_free parameter, and set the hierarchy level mask to the + * hier_mask parameter. + * - Offset HW dependencies by the hw_job_id_offset parameter + * - Set the Perform Job Barrier flag if this job is the first in the chain + * - Read the address of the next job header + * + * @param[in] kctx Context pointer + * @param[in,out] job_header Address of job header to reset. Set to address + * of next job header on exit. + * @param[in] prev_jc Previous job chain to link to, if this job is + * the last in the chain. + * @param[in] hw_job_id_offset Offset for HW job IDs + * @param[in] tiler_heap_free The value to reset Tiler Heap Free to + * @param[in] hierarchy_mask The hierarchy mask to use + * @param[in] default_weight Default hierarchy weight to write when no other + * weight is given in the FBD + * @param[in] first_in_chain true if this job is the first in the chain + * @param[in] fragment_chain true if this job is in the fragment chain + * + * @return 0 on success, error code on failure + */ +static int kbasep_replay_reset_job(struct kbase_context *kctx, + u64 *job_header, u64 prev_jc, + u64 tiler_heap_free, u16 hierarchy_mask, + u32 default_weight, u16 hw_job_id_offset, + bool first_in_chain, bool fragment_chain) +{ + struct fragment_job *frag_job; + struct job_descriptor_header *job; + u64 new_job_header; + struct kbase_vmap_struct map; + + frag_job = kbase_vmap(kctx, *job_header, sizeof(*frag_job), &map); + if (!frag_job) { + dev_err(kctx->kbdev->dev, + "kbasep_replay_parse_jc: failed to map jc\n"); + return -EINVAL; + } + job = &frag_job->header; + + dump_job_head(kctx, "Job header:", job); + + if (job->exception_status == JOB_NOT_STARTED && !fragment_chain) { + dev_err(kctx->kbdev->dev, "Job already not started\n"); + goto out_unmap; + } + job->exception_status = JOB_NOT_STARTED; + + if (job->job_type == JOB_TYPE_VERTEX) + job->job_type = JOB_TYPE_NULL; + + if (job->job_type == JOB_TYPE_FUSED) { + dev_err(kctx->kbdev->dev, "Fused jobs can not be replayed\n"); + goto out_unmap; + } + + if (first_in_chain) + job->job_barrier = 1; + + if ((job->job_dependency_index_1 + hw_job_id_offset) > + JOB_HEADER_ID_MAX || + (job->job_dependency_index_2 + hw_job_id_offset) > + JOB_HEADER_ID_MAX || + (job->job_index + hw_job_id_offset) > JOB_HEADER_ID_MAX) { + dev_err(kctx->kbdev->dev, + "Job indicies/dependencies out of valid range\n"); + goto out_unmap; + } + + if (job->job_dependency_index_1) + job->job_dependency_index_1 += hw_job_id_offset; + if (job->job_dependency_index_2) + job->job_dependency_index_2 += hw_job_id_offset; + + job->job_index += hw_job_id_offset; + + if (job->job_descriptor_size) { + new_job_header = job->next_job._64; + if (!job->next_job._64) + job->next_job._64 = prev_jc; + } else { + new_job_header = job->next_job._32; + if (!job->next_job._32) + job->next_job._32 = prev_jc; + } + dump_job_head(kctx, "Updated to:", job); + + if (job->job_type == JOB_TYPE_TILER) { + bool job_64 = job->job_descriptor_size != 0; + + if (kbasep_replay_reset_tiler_job(kctx, *job_header, + tiler_heap_free, hierarchy_mask, + default_weight, job_64) != 0) + goto out_unmap; + + } else if (job->job_type == JOB_TYPE_FRAGMENT) { + u64 fbd_address; + + if (job->job_descriptor_size) + fbd_address = frag_job->fragment_fbd._64; + else + fbd_address = (u64)frag_job->fragment_fbd._32; + + if (fbd_address & FBD_TYPE) { + if (kbasep_replay_reset_mfbd(kctx, + fbd_address & FBD_POINTER_MASK, + tiler_heap_free, + hierarchy_mask, + default_weight) != 0) + goto out_unmap; + } else { + if (kbasep_replay_reset_sfbd(kctx, + fbd_address & FBD_POINTER_MASK, + tiler_heap_free, + hierarchy_mask, + default_weight) != 0) + goto out_unmap; + } + } + + kbase_vunmap(kctx, &map); + + *job_header = new_job_header; + + return 0; + +out_unmap: + kbase_vunmap(kctx, &map); + return -EINVAL; +} + +/** + * @brief Find the highest job ID in a job chain + * + * @param[in] kctx Context pointer + * @param[in] jc Job chain start address + * @param[out] hw_job_id Highest job ID in chain + * + * @return 0 on success, error code on failure + */ +static int kbasep_replay_find_hw_job_id(struct kbase_context *kctx, + u64 jc, u16 *hw_job_id) +{ + while (jc) { + struct job_descriptor_header *job; + struct kbase_vmap_struct map; + + dev_dbg(kctx->kbdev->dev, + "kbasep_replay_find_hw_job_id: parsing jc=%llx\n", jc); + + job = kbase_vmap(kctx, jc, sizeof(*job), &map); + if (!job) { + dev_err(kctx->kbdev->dev, "failed to map jc\n"); + + return -EINVAL; + } + + if (job->job_index > *hw_job_id) + *hw_job_id = job->job_index; + + if (job->job_descriptor_size) + jc = job->next_job._64; + else + jc = job->next_job._32; + + kbase_vunmap(kctx, &map); + } + + return 0; +} + +/** + * @brief Reset the status of a number of jobs + * + * This function walks the provided job chain, and calls + * kbasep_replay_reset_job for each job. It also links the job chain to the + * provided previous job chain. + * + * The function will fail if any of the jobs passed already have status of + * NOT_STARTED. + * + * @param[in] kctx Context pointer + * @param[in] jc Job chain to be processed + * @param[in] prev_jc Job chain to be added to. May be NULL + * @param[in] tiler_heap_free The value to reset Tiler Heap Free to + * @param[in] hierarchy_mask The hierarchy mask to use + * @param[in] default_weight Default hierarchy weight to write when no other + * weight is given in the FBD + * @param[in] hw_job_id_offset Offset for HW job IDs + * @param[in] fragment_chain true if this chain is the fragment chain + * + * @return 0 on success, error code otherwise + */ +static int kbasep_replay_parse_jc(struct kbase_context *kctx, + u64 jc, u64 prev_jc, + u64 tiler_heap_free, u16 hierarchy_mask, + u32 default_weight, u16 hw_job_id_offset, + bool fragment_chain) +{ + bool first_in_chain = true; + int nr_jobs = 0; + + dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: jc=%llx hw_job_id=%x\n", + jc, hw_job_id_offset); + + while (jc) { + dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: parsing jc=%llx\n", jc); + + if (kbasep_replay_reset_job(kctx, &jc, prev_jc, + tiler_heap_free, hierarchy_mask, + default_weight, hw_job_id_offset, + first_in_chain, fragment_chain) != 0) + return -EINVAL; + + first_in_chain = false; + + nr_jobs++; + if (fragment_chain && + nr_jobs >= BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT) { + dev_err(kctx->kbdev->dev, + "Exceeded maximum number of jobs in fragment chain\n"); + return -EINVAL; + } + } + + return 0; +} + +/** + * @brief Reset the status of a replay job, and set up dependencies + * + * This performs the actions to allow the replay job to be re-run following + * completion of the passed dependency. + * + * @param[in] katom The atom to be reset + * @param[in] dep_atom The dependency to be attached to the atom + */ +static void kbasep_replay_reset_softjob(struct kbase_jd_atom *katom, + struct kbase_jd_atom *dep_atom) +{ + katom->status = KBASE_JD_ATOM_STATE_QUEUED; + kbase_jd_katom_dep_set(&katom->dep[0], dep_atom, BASE_JD_DEP_TYPE_DATA); + list_add_tail(&katom->dep_item[0], &dep_atom->dep_head[0]); +} + +/** + * @brief Allocate an unused katom + * + * This will search the provided context for an unused katom, and will mark it + * as KBASE_JD_ATOM_STATE_QUEUED. + * + * If no atoms are available then the function will fail. + * + * @param[in] kctx Context pointer + * @return An atom ID, or -1 on failure + */ +static int kbasep_allocate_katom(struct kbase_context *kctx) +{ + struct kbase_jd_context *jctx = &kctx->jctx; + int i; + + for (i = BASE_JD_ATOM_COUNT-1; i > 0; i--) { + if (jctx->atoms[i].status == KBASE_JD_ATOM_STATE_UNUSED) { + jctx->atoms[i].status = KBASE_JD_ATOM_STATE_QUEUED; + dev_dbg(kctx->kbdev->dev, + "kbasep_allocate_katom: Allocated atom %d\n", + i); + return i; + } + } + + return -1; +} + +/** + * @brief Release a katom + * + * This will mark the provided atom as available, and remove any dependencies. + * + * For use on error path. + * + * @param[in] kctx Context pointer + * @param[in] atom_id ID of atom to release + */ +static void kbasep_release_katom(struct kbase_context *kctx, int atom_id) +{ + struct kbase_jd_context *jctx = &kctx->jctx; + + dev_dbg(kctx->kbdev->dev, "kbasep_release_katom: Released atom %d\n", + atom_id); + + while (!list_empty(&jctx->atoms[atom_id].dep_head[0])) + list_del(jctx->atoms[atom_id].dep_head[0].next); + + while (!list_empty(&jctx->atoms[atom_id].dep_head[1])) + list_del(jctx->atoms[atom_id].dep_head[1].next); + + jctx->atoms[atom_id].status = KBASE_JD_ATOM_STATE_UNUSED; +} + +static void kbasep_replay_create_atom(struct kbase_context *kctx, + struct base_jd_atom_v2 *atom, + int atom_nr, + base_jd_prio prio) +{ + atom->nr_extres = 0; + atom->extres_list.value = NULL; + atom->device_nr = 0; + atom->prio = prio; + atom->atom_number = atom_nr; + + base_jd_atom_dep_set(&atom->pre_dep[0], 0 , BASE_JD_DEP_TYPE_INVALID); + base_jd_atom_dep_set(&atom->pre_dep[1], 0 , BASE_JD_DEP_TYPE_INVALID); + + atom->udata.blob[0] = 0; + atom->udata.blob[1] = 0; +} + +/** + * @brief Create two atoms for the purpose of replaying jobs + * + * Two atoms are allocated and created. The jc pointer is not set at this + * stage. The second atom has a dependency on the first. The remaining fields + * are set up as follows : + * + * - No external resources. Any required external resources will be held by the + * replay atom. + * - device_nr is set to 0. This is not relevant as + * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set. + * - Priority is inherited from the replay job. + * + * @param[out] t_atom Atom to use for tiler jobs + * @param[out] f_atom Atom to use for fragment jobs + * @param[in] prio Priority of new atom (inherited from replay soft + * job) + * @return 0 on success, error code on failure + */ +static int kbasep_replay_create_atoms(struct kbase_context *kctx, + struct base_jd_atom_v2 *t_atom, + struct base_jd_atom_v2 *f_atom, + base_jd_prio prio) +{ + int t_atom_nr, f_atom_nr; + + t_atom_nr = kbasep_allocate_katom(kctx); + if (t_atom_nr < 0) { + dev_err(kctx->kbdev->dev, "Failed to allocate katom\n"); + return -EINVAL; + } + + f_atom_nr = kbasep_allocate_katom(kctx); + if (f_atom_nr < 0) { + dev_err(kctx->kbdev->dev, "Failed to allocate katom\n"); + kbasep_release_katom(kctx, t_atom_nr); + return -EINVAL; + } + + kbasep_replay_create_atom(kctx, t_atom, t_atom_nr, prio); + kbasep_replay_create_atom(kctx, f_atom, f_atom_nr, prio); + + base_jd_atom_dep_set(&f_atom->pre_dep[0], t_atom_nr , BASE_JD_DEP_TYPE_DATA); + + return 0; +} + +#ifdef CONFIG_MALI_DEBUG +static void payload_dump(struct kbase_context *kctx, base_jd_replay_payload *payload) +{ + u64 next; + + dev_dbg(kctx->kbdev->dev, "Tiler jc list :\n"); + next = payload->tiler_jc_list; + + while (next) { + struct kbase_vmap_struct map; + base_jd_replay_jc *jc_struct; + + jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &map); + + if (!jc_struct) + return; + + dev_dbg(kctx->kbdev->dev, "* jc_struct=%p jc=%llx next=%llx\n", + jc_struct, jc_struct->jc, jc_struct->next); + + next = jc_struct->next; + + kbase_vunmap(kctx, &map); + } +} +#endif + +/** + * @brief Parse a base_jd_replay_payload provided by userspace + * + * This will read the payload from userspace, and parse the job chains. + * + * @param[in] kctx Context pointer + * @param[in] replay_atom Replay soft job atom + * @param[in] t_atom Atom to use for tiler jobs + * @param[in] f_atom Atom to use for fragment jobs + * @return 0 on success, error code on failure + */ +static int kbasep_replay_parse_payload(struct kbase_context *kctx, + struct kbase_jd_atom *replay_atom, + struct base_jd_atom_v2 *t_atom, + struct base_jd_atom_v2 *f_atom) +{ + base_jd_replay_payload *payload = NULL; + u64 next; + u64 prev_jc = 0; + u16 hw_job_id_offset = 0; + int ret = -EINVAL; + struct kbase_vmap_struct map; + + dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: replay_atom->jc = %llx sizeof(payload) = %zu\n", + replay_atom->jc, sizeof(payload)); + + payload = kbase_vmap(kctx, replay_atom->jc, sizeof(*payload), &map); + if (!payload) { + dev_err(kctx->kbdev->dev, "kbasep_replay_parse_payload: failed to map payload into kernel space\n"); + return -EINVAL; + } + +#ifdef BASE_LEGACY_UK10_2_SUPPORT + if (KBASE_API_VERSION(10, 3) > replay_atom->kctx->api_version) { + base_jd_replay_payload_uk10_2 *payload_uk10_2; + u16 tiler_core_req; + u16 fragment_core_req; + + payload_uk10_2 = (base_jd_replay_payload_uk10_2 *) payload; + memcpy(&tiler_core_req, &payload_uk10_2->tiler_core_req, + sizeof(tiler_core_req)); + memcpy(&fragment_core_req, &payload_uk10_2->fragment_core_req, + sizeof(fragment_core_req)); + payload->tiler_core_req = (u32)(tiler_core_req & 0x7fff); + payload->fragment_core_req = (u32)(fragment_core_req & 0x7fff); + } +#endif /* BASE_LEGACY_UK10_2_SUPPORT */ + +#ifdef CONFIG_MALI_DEBUG + dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: payload=%p\n", payload); + dev_dbg(kctx->kbdev->dev, "Payload structure:\n" + "tiler_jc_list = %llx\n" + "fragment_jc = %llx\n" + "tiler_heap_free = %llx\n" + "fragment_hierarchy_mask = %x\n" + "tiler_hierarchy_mask = %x\n" + "hierarchy_default_weight = %x\n" + "tiler_core_req = %x\n" + "fragment_core_req = %x\n", + payload->tiler_jc_list, + payload->fragment_jc, + payload->tiler_heap_free, + payload->fragment_hierarchy_mask, + payload->tiler_hierarchy_mask, + payload->hierarchy_default_weight, + payload->tiler_core_req, + payload->fragment_core_req); + payload_dump(kctx, payload); +#endif + t_atom->core_req = payload->tiler_core_req | BASEP_JD_REQ_EVENT_NEVER; + f_atom->core_req = payload->fragment_core_req | BASEP_JD_REQ_EVENT_NEVER; + + /* Sanity check core requirements*/ + if ((t_atom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_T || + (f_atom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_FS || + t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES || + f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { + + int t_atom_type = t_atom->core_req & BASE_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP; + int f_atom_type = f_atom->core_req & BASE_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP & ~BASE_JD_REQ_FS_AFBC; + int t_has_ex_res = t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES; + int f_has_ex_res = f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES; + + if (t_atom_type != BASE_JD_REQ_T) { + dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom not a tiler job. Was: 0x%x\n Expected: 0x%x", + t_atom_type, BASE_JD_REQ_T); + } + if (f_atom_type != BASE_JD_REQ_FS) { + dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom not a fragment shader. Was 0x%x Expected: 0x%x\n", + f_atom_type, BASE_JD_REQ_FS); + } + if (t_has_ex_res) { + dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom has external resources.\n"); + } + if (f_has_ex_res) { + dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom has external resources.\n"); + } + + goto out; + } + + /* Process tiler job chains */ + next = payload->tiler_jc_list; + if (!next) { + dev_err(kctx->kbdev->dev, "Invalid tiler JC list\n"); + goto out; + } + + while (next) { + base_jd_replay_jc *jc_struct; + struct kbase_vmap_struct jc_map; + u64 jc; + + jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &jc_map); + + if (!jc_struct) { + dev_err(kctx->kbdev->dev, "Failed to map jc struct\n"); + goto out; + } + + jc = jc_struct->jc; + next = jc_struct->next; + if (next) + jc_struct->jc = 0; + + kbase_vunmap(kctx, &jc_map); + + if (jc) { + u16 max_hw_job_id = 0; + + if (kbasep_replay_find_hw_job_id(kctx, jc, + &max_hw_job_id) != 0) + goto out; + + if (kbasep_replay_parse_jc(kctx, jc, prev_jc, + payload->tiler_heap_free, + payload->tiler_hierarchy_mask, + payload->hierarchy_default_weight, + hw_job_id_offset, false) != 0) { + goto out; + } + + hw_job_id_offset += max_hw_job_id; + + prev_jc = jc; + } + } + t_atom->jc = prev_jc; + + /* Process fragment job chain */ + f_atom->jc = payload->fragment_jc; + if (kbasep_replay_parse_jc(kctx, payload->fragment_jc, 0, + payload->tiler_heap_free, + payload->fragment_hierarchy_mask, + payload->hierarchy_default_weight, 0, + true) != 0) { + goto out; + } + + if (!t_atom->jc || !f_atom->jc) { + dev_err(kctx->kbdev->dev, "Invalid payload\n"); + goto out; + } + + dev_dbg(kctx->kbdev->dev, "t_atom->jc=%llx f_atom->jc=%llx\n", + t_atom->jc, f_atom->jc); + ret = 0; + +out: + kbase_vunmap(kctx, &map); + + return ret; +} + +static void kbase_replay_process_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom; + struct kbase_context *kctx; + struct kbase_jd_context *jctx; + bool need_to_try_schedule_context = false; + + struct base_jd_atom_v2 t_atom, f_atom; + struct kbase_jd_atom *t_katom, *f_katom; + base_jd_prio atom_prio; + + katom = container_of(data, struct kbase_jd_atom, work); + kctx = katom->kctx; + jctx = &kctx->jctx; + + mutex_lock(&jctx->lock); + + atom_prio = kbasep_js_sched_prio_to_atom_prio(katom->sched_priority); + + if (kbasep_replay_create_atoms( + kctx, &t_atom, &f_atom, atom_prio) != 0) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + goto out; + } + + t_katom = &jctx->atoms[t_atom.atom_number]; + f_katom = &jctx->atoms[f_atom.atom_number]; + + if (kbasep_replay_parse_payload(kctx, katom, &t_atom, &f_atom) != 0) { + kbasep_release_katom(kctx, t_atom.atom_number); + kbasep_release_katom(kctx, f_atom.atom_number); + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + goto out; + } + + kbasep_replay_reset_softjob(katom, f_katom); + + need_to_try_schedule_context |= jd_submit_atom(kctx, &t_atom, t_katom); + if (t_katom->event_code == BASE_JD_EVENT_JOB_INVALID) { + dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n"); + kbasep_release_katom(kctx, f_atom.atom_number); + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + goto out; + } + need_to_try_schedule_context |= jd_submit_atom(kctx, &f_atom, f_katom); + if (f_katom->event_code == BASE_JD_EVENT_JOB_INVALID) { + dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n"); + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + goto out; + } + + katom->event_code = BASE_JD_EVENT_DONE; + +out: + if (katom->event_code != BASE_JD_EVENT_DONE) { + kbase_disjoint_state_down(kctx->kbdev); + + need_to_try_schedule_context |= jd_done_nolock(katom, NULL); + } + + if (need_to_try_schedule_context) + kbase_js_sched_all(kctx->kbdev); + + mutex_unlock(&jctx->lock); +} + +/** + * @brief Check job replay fault + * + * This will read the job payload, checks fault type and source, then decides + * whether replay is required. + * + * @param[in] katom The atom to be processed + * @return true (success) if replay required or false on failure. + */ +static bool kbase_replay_fault_check(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct device *dev = kctx->kbdev->dev; + base_jd_replay_payload *payload; + u64 job_header; + u64 job_loop_detect; + struct job_descriptor_header *job; + struct kbase_vmap_struct job_map; + struct kbase_vmap_struct map; + bool err = false; + + /* Replay job if fault is of type BASE_JD_EVENT_JOB_WRITE_FAULT or + * if force_replay is enabled. + */ + if (BASE_JD_EVENT_TERMINATED == katom->event_code) { + return false; + } else if (BASE_JD_EVENT_JOB_WRITE_FAULT == katom->event_code) { + return true; + } else if (BASE_JD_EVENT_FORCE_REPLAY == katom->event_code) { + katom->event_code = BASE_JD_EVENT_DATA_INVALID_FAULT; + return true; + } else if (BASE_JD_EVENT_DATA_INVALID_FAULT != katom->event_code) { + /* No replay for faults of type other than + * BASE_JD_EVENT_DATA_INVALID_FAULT. + */ + return false; + } + + /* Job fault is BASE_JD_EVENT_DATA_INVALID_FAULT, now scan fragment jc + * to find out whether the source of exception is POLYGON_LIST. Replay + * is required if the source of fault is POLYGON_LIST. + */ + payload = kbase_vmap(kctx, katom->jc, sizeof(*payload), &map); + if (!payload) { + dev_err(dev, "kbase_replay_fault_check: failed to map payload.\n"); + return false; + } + +#ifdef CONFIG_MALI_DEBUG + dev_dbg(dev, "kbase_replay_fault_check: payload=%p\n", payload); + dev_dbg(dev, "\nPayload structure:\n" + "fragment_jc = 0x%llx\n" + "fragment_hierarchy_mask = 0x%x\n" + "fragment_core_req = 0x%x\n", + payload->fragment_jc, + payload->fragment_hierarchy_mask, + payload->fragment_core_req); +#endif + /* Process fragment job chain */ + job_header = (u64) payload->fragment_jc; + job_loop_detect = job_header; + while (job_header) { + job = kbase_vmap(kctx, job_header, sizeof(*job), &job_map); + if (!job) { + dev_err(dev, "failed to map jc\n"); + /* unmap payload*/ + kbase_vunmap(kctx, &map); + return false; + } + + + dump_job_head(kctx, "\njob_head structure:\n", job); + + /* Replay only when the polygon list reader caused the + * DATA_INVALID_FAULT */ + if ((BASE_JD_EVENT_DATA_INVALID_FAULT == katom->event_code) && + (JOB_POLYGON_LIST == JOB_SOURCE_ID(job->exception_status))) { + err = true; + kbase_vunmap(kctx, &job_map); + break; + } + + /* Move on to next fragment job in the list */ + if (job->job_descriptor_size) + job_header = job->next_job._64; + else + job_header = job->next_job._32; + + kbase_vunmap(kctx, &job_map); + + /* Job chain loop detected */ + if (job_header == job_loop_detect) + break; + } + + /* unmap payload*/ + kbase_vunmap(kctx, &map); + + return err; +} + + +/** + * @brief Process a replay job + * + * Called from kbase_process_soft_job. + * + * On exit, if the job has completed, katom->event_code will have been updated. + * If the job has not completed, and is replaying jobs, then the atom status + * will have been reset to KBASE_JD_ATOM_STATE_QUEUED. + * + * @param[in] katom The atom to be processed + * @return false if the atom has completed + * true if the atom is replaying jobs + */ +bool kbase_replay_process(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct kbase_device *kbdev = kctx->kbdev; + + /* Don't replay this atom if these issues are not present in the + * hardware */ + if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11020) && + !kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11024)) { + dev_dbg(kbdev->dev, "Hardware does not need replay workaround"); + + /* Signal failure to userspace */ + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + + return false; + } + + if (katom->event_code == BASE_JD_EVENT_DONE) { + dev_dbg(kbdev->dev, "Previous job succeeded - not replaying\n"); + + if (katom->retry_count) + kbase_disjoint_state_down(kbdev); + + return false; + } + + if (kbase_ctx_flag(kctx, KCTX_DYING)) { + dev_dbg(kbdev->dev, "Not replaying; context is dying\n"); + + if (katom->retry_count) + kbase_disjoint_state_down(kbdev); + + return false; + } + + /* Check job exception type and source before replaying. */ + if (!kbase_replay_fault_check(katom)) { + dev_dbg(kbdev->dev, + "Replay cancelled on event %x\n", katom->event_code); + /* katom->event_code is already set to the failure code of the + * previous job. + */ + return false; + } + + dev_warn(kbdev->dev, "Replaying jobs retry=%d\n", + katom->retry_count); + + katom->retry_count++; + + if (katom->retry_count > BASEP_JD_REPLAY_LIMIT) { + dev_err(kbdev->dev, "Replay exceeded limit - failing jobs\n"); + + kbase_disjoint_state_down(kbdev); + + /* katom->event_code is already set to the failure code of the + previous job */ + return false; + } + + /* only enter the disjoint state once for the whole time while the replay is ongoing */ + if (katom->retry_count == 1) + kbase_disjoint_state_up(kbdev); + + INIT_WORK(&katom->work, kbase_replay_process_worker); + queue_work(kctx->event_workq, &katom->work); + + return true; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_smc.c b/drivers/gpu/arm/midgard/mali_kbase_smc.c new file mode 100755 index 000000000000..6c8cf73ae58c --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_smc.c @@ -0,0 +1,86 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifdef CONFIG_ARM64 + +#include +#include + +#include + +/* __asmeq is not available on Kernel versions >= 4.20 */ +#ifndef __asmeq +/* + * This is used to ensure the compiler did actually allocate the register we + * asked it for some inline assembly sequences. Apparently we can't trust the + * compiler from one version to another so a bit of paranoia won't hurt. This + * string is meant to be concatenated with the inline asm string and will + * cause compilation to stop on mismatch. (for details, see gcc PR 15089) + */ +#define __asmeq(x, y) ".ifnc " x "," y " ; .err ; .endif\n\t" +#endif + +static noinline u64 invoke_smc_fid(u64 function_id, + u64 arg0, u64 arg1, u64 arg2) +{ + register u64 x0 asm("x0") = function_id; + register u64 x1 asm("x1") = arg0; + register u64 x2 asm("x2") = arg1; + register u64 x3 asm("x3") = arg2; + + asm volatile( + __asmeq("%0", "x0") + __asmeq("%1", "x1") + __asmeq("%2", "x2") + __asmeq("%3", "x3") + "smc #0\n" + : "+r" (x0) + : "r" (x1), "r" (x2), "r" (x3)); + + return x0; +} + +u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2) +{ + /* Is fast call (bit 31 set) */ + KBASE_DEBUG_ASSERT(fid & ~SMC_FAST_CALL); + /* bits 16-23 must be zero for fast calls */ + KBASE_DEBUG_ASSERT((fid & (0xFF << 16)) == 0); + + return invoke_smc_fid(fid, arg0, arg1, arg2); +} + +u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, + u64 arg0, u64 arg1, u64 arg2) +{ + u32 fid = 0; + + /* Only the six bits allowed should be used. */ + KBASE_DEBUG_ASSERT((oen & ~SMC_OEN_MASK) == 0); + + fid |= SMC_FAST_CALL; /* Bit 31: Fast call */ + if (smc64) + fid |= SMC_64; /* Bit 30: 1=SMC64, 0=SMC32 */ + fid |= oen; /* Bit 29:24: OEN */ + /* Bit 23:16: Must be zero for fast calls */ + fid |= (function_number); /* Bit 15:0: function number */ + + return kbase_invoke_smc_fid(fid, arg0, arg1, arg2); +} + +#endif /* CONFIG_ARM64 */ + diff --git a/drivers/gpu/arm/midgard/mali_kbase_smc.h b/drivers/gpu/arm/midgard/mali_kbase_smc.h new file mode 100755 index 000000000000..9bff3d2e8b4d --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_smc.h @@ -0,0 +1,67 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_SMC_H_ +#define _KBASE_SMC_H_ + +#ifdef CONFIG_ARM64 + +#include + +#define SMC_FAST_CALL (1 << 31) +#define SMC_64 (1 << 30) + +#define SMC_OEN_OFFSET 24 +#define SMC_OEN_MASK (0x3F << SMC_OEN_OFFSET) /* 6 bits */ +#define SMC_OEN_SIP (2 << SMC_OEN_OFFSET) +#define SMC_OEN_STD (4 << SMC_OEN_OFFSET) + + +/** + * kbase_invoke_smc_fid - Perform a secure monitor call + * @fid: The SMC function to call, see SMC Calling convention. + * @arg0: First argument to the SMC. + * @arg1: Second argument to the SMC. + * @arg2: Third argument to the SMC. + * + * See SMC Calling Convention for details. + * + * Return: the return value from the SMC. + */ +u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2); + +/** + * kbase_invoke_smc_fid - Perform a secure monitor call + * @oen: Owning Entity number (SIP, STD etc). + * @function_number: The function number within the OEN. + * @smc64: use SMC64 calling convention instead of SMC32. + * @arg0: First argument to the SMC. + * @arg1: Second argument to the SMC. + * @arg2: Third argument to the SMC. + * + * See SMC Calling Convention for details. + * + * Return: the return value from the SMC call. + */ +u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, + u64 arg0, u64 arg1, u64 arg2); + +#endif /* CONFIG_ARM64 */ + +#endif /* _KBASE_SMC_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_softjobs.c b/drivers/gpu/arm/midgard/mali_kbase_softjobs.c new file mode 100755 index 000000000000..396953e780a8 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_softjobs.c @@ -0,0 +1,1549 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include + +#if defined(CONFIG_DMA_SHARED_BUFFER) +#include +#include +#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Mask to check cache alignment of data structures */ +#define KBASE_CACHE_ALIGNMENT_MASK ((1<kctx; + unsigned long lflags; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_add_tail(&katom->queue, &kctx->waiting_soft_jobs); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + unsigned long lflags; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_del(&katom->queue); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +static void kbasep_add_waiting_with_timeout(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + /* Record the start time of this atom so we could cancel it at + * the right time. + */ + katom->start_timestamp = ktime_get(); + + /* Add the atom to the waiting list before the timer is + * (re)started to make sure that it gets processed. + */ + kbasep_add_waiting_soft_job(katom); + + /* Schedule timeout of this atom after a period if it is not active */ + if (!timer_pending(&kctx->soft_job_timeout)) { + int timeout_ms = atomic_read( + &kctx->kbdev->js_data.soft_job_timeout_ms); + mod_timer(&kctx->soft_job_timeout, + jiffies + msecs_to_jiffies(timeout_ms)); + } +} + +static int kbasep_read_soft_event_status( + struct kbase_context *kctx, u64 evt, unsigned char *status) +{ + unsigned char *mapped_evt; + struct kbase_vmap_struct map; + + mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); + if (!mapped_evt) + return -EFAULT; + + *status = *mapped_evt; + + kbase_vunmap(kctx, &map); + + return 0; +} + +static int kbasep_write_soft_event_status( + struct kbase_context *kctx, u64 evt, unsigned char new_status) +{ + unsigned char *mapped_evt; + struct kbase_vmap_struct map; + + if ((new_status != BASE_JD_SOFT_EVENT_SET) && + (new_status != BASE_JD_SOFT_EVENT_RESET)) + return -EINVAL; + + mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); + if (!mapped_evt) + return -EFAULT; + + *mapped_evt = new_status; + + kbase_vunmap(kctx, &map); + + return 0; +} + +static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) +{ + struct kbase_vmap_struct map; + void *user_result; + struct timespec64 ts; + struct base_dump_cpu_gpu_counters data; + u64 system_time; + u64 cycle_counter; + u64 jc = katom->jc; + struct kbase_context *kctx = katom->kctx; + int pm_active_err; + + memset(&data, 0, sizeof(data)); + + /* Take the PM active reference as late as possible - otherwise, it could + * delay suspend until we process the atom (which may be at the end of a + * long chain of dependencies */ + pm_active_err = kbase_pm_context_active_handle_suspend(kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); + if (pm_active_err) { + struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; + + /* We're suspended - queue this on the list of suspended jobs + * Use dep_item[1], because dep_item[0] was previously in use + * for 'waiting_soft_jobs'. + */ + mutex_lock(&js_devdata->runpool_mutex); + list_add_tail(&katom->dep_item[1], &js_devdata->suspended_soft_jobs_list); + mutex_unlock(&js_devdata->runpool_mutex); + + /* Also adding this to the list of waiting soft job */ + kbasep_add_waiting_soft_job(katom); + + return pm_active_err; + } + + kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, + &ts); + + kbase_pm_context_idle(kctx->kbdev); + + data.sec = ts.tv_sec; + data.usec = ts.tv_nsec / 1000; + data.system_time = system_time; + data.cycle_counter = cycle_counter; + + /* Assume this atom will be cancelled until we know otherwise */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* GPU_WR access is checked on the range for returning the result to + * userspace for the following reasons: + * - security, this is currently how imported user bufs are checked. + * - userspace ddk guaranteed to assume region was mapped as GPU_WR */ + user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); + if (!user_result) + return 0; + + memcpy(user_result, &data, sizeof(data)); + + kbase_vunmap(kctx, &map); + + /* Atom was fine - mark it as done */ + katom->event_code = BASE_JD_EVENT_DONE; + + return 0; +} + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) +/* Called by the explicit fence mechanism when a fence wait has completed */ +void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + mutex_lock(&kctx->jctx.lock); + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(kctx->kbdev); + mutex_unlock(&kctx->jctx.lock); +} +#endif + +static void kbasep_soft_event_complete_job(struct work_struct *work) +{ + struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + int resched; + + mutex_lock(&kctx->jctx.lock); + resched = jd_done_nolock(katom, NULL); + mutex_unlock(&kctx->jctx.lock); + + if (resched) + kbase_js_sched_all(kctx->kbdev); +} + +void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt) +{ + int cancel_timer = 1; + struct list_head *entry, *tmp; + unsigned long lflags; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { + struct kbase_jd_atom *katom = list_entry( + entry, struct kbase_jd_atom, queue); + + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_EVENT_WAIT: + if (katom->jc == evt) { + list_del(&katom->queue); + + katom->event_code = BASE_JD_EVENT_DONE; + INIT_WORK(&katom->work, + kbasep_soft_event_complete_job); + queue_work(kctx->jctx.job_done_wq, + &katom->work); + } else { + /* There are still other waiting jobs, we cannot + * cancel the timer yet. + */ + cancel_timer = 0; + } + break; +#ifdef CONFIG_MALI_FENCE_DEBUG + case BASE_JD_REQ_SOFT_FENCE_WAIT: + /* Keep the timer running if fence debug is enabled and + * there are waiting fence jobs. + */ + cancel_timer = 0; + break; +#endif + } + } + + if (cancel_timer) + del_timer(&kctx->soft_job_timeout); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +#ifdef CONFIG_MALI_FENCE_DEBUG +static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct device *dev = kctx->kbdev->dev; + int i; + + for (i = 0; i < 2; i++) { + struct kbase_jd_atom *dep; + + list_for_each_entry(dep, &katom->dep_head[i], dep_item[i]) { + if (dep->status == KBASE_JD_ATOM_STATE_UNUSED || + dep->status == KBASE_JD_ATOM_STATE_COMPLETED) + continue; + + if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) + == BASE_JD_REQ_SOFT_FENCE_TRIGGER) { + /* Found blocked trigger fence. */ + struct kbase_sync_fence_info info; + + if (!kbase_sync_fence_in_info_get(dep, &info)) { + dev_warn(dev, + "\tVictim trigger atom %d fence [%p] %s: %s\n", + kbase_jd_atom_id(kctx, dep), + info.fence, + info.name, + kbase_sync_status_string(info.status)); + } + } + + kbase_fence_debug_check_atom(dep); + } + } +} + +static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct device *dev = katom->kctx->kbdev->dev; + int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); + unsigned long lflags; + struct kbase_sync_fence_info info; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + + if (kbase_sync_fence_in_info_get(katom, &info)) { + /* Fence must have signaled just after timeout. */ + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); + return; + } + + dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%p] after %dms\n", + kctx->tgid, kctx->id, + kbase_jd_atom_id(kctx, katom), + info.fence, timeout_ms); + dev_warn(dev, "\tGuilty fence [%p] %s: %s\n", + info.fence, info.name, + kbase_sync_status_string(info.status)); + + /* Search for blocked trigger atoms */ + kbase_fence_debug_check_atom(katom); + + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); + + kbase_sync_fence_in_dump(katom); +} + +struct kbase_fence_debug_work { + struct kbase_jd_atom *katom; + struct work_struct work; +}; + +static void kbase_fence_debug_wait_timeout_worker(struct work_struct *work) +{ + struct kbase_fence_debug_work *w = container_of(work, + struct kbase_fence_debug_work, work); + struct kbase_jd_atom *katom = w->katom; + struct kbase_context *kctx = katom->kctx; + + mutex_lock(&kctx->jctx.lock); + kbase_fence_debug_wait_timeout(katom); + mutex_unlock(&kctx->jctx.lock); + + kfree(w); +} + +static void kbase_fence_debug_timeout(struct kbase_jd_atom *katom) +{ + struct kbase_fence_debug_work *work; + struct kbase_context *kctx = katom->kctx; + + /* Enqueue fence debug worker. Use job_done_wq to get + * debug print ordered with job completion. + */ + work = kzalloc(sizeof(struct kbase_fence_debug_work), GFP_ATOMIC); + /* Ignore allocation failure. */ + if (work) { + work->katom = katom; + INIT_WORK(&work->work, kbase_fence_debug_wait_timeout_worker); + queue_work(kctx->jctx.job_done_wq, &work->work); + } +} +#endif /* CONFIG_MALI_FENCE_DEBUG */ + +void kbasep_soft_job_timeout_worker(struct timer_list *t) +{ + struct kbase_context *kctx = from_timer(kctx, t, soft_job_timeout); + u32 timeout_ms = (u32)atomic_read( + &kctx->kbdev->js_data.soft_job_timeout_ms); + struct timer_list *timer = &kctx->soft_job_timeout; + ktime_t cur_time = ktime_get(); + bool restarting = false; + unsigned long lflags; + struct list_head *entry, *tmp; + + spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); + list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { + struct kbase_jd_atom *katom = list_entry(entry, + struct kbase_jd_atom, queue); + s64 elapsed_time = ktime_to_ms(ktime_sub(cur_time, + katom->start_timestamp)); + + if (elapsed_time < (s64)timeout_ms) { + restarting = true; + continue; + } + + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_EVENT_WAIT: + /* Take it out of the list to ensure that it + * will be cancelled in all cases + */ + list_del(&katom->queue); + + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + INIT_WORK(&katom->work, kbasep_soft_event_complete_job); + queue_work(kctx->jctx.job_done_wq, &katom->work); + break; +#ifdef CONFIG_MALI_FENCE_DEBUG + case BASE_JD_REQ_SOFT_FENCE_WAIT: + kbase_fence_debug_timeout(katom); + break; +#endif + } + } + + if (restarting) + mod_timer(timer, jiffies + msecs_to_jiffies(timeout_ms)); + spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); +} + +static int kbasep_soft_event_wait(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + unsigned char status; + + /* The status of this soft-job is stored in jc */ + if (kbasep_read_soft_event_status(kctx, katom->jc, &status)) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + return 0; + } + + if (status == BASE_JD_SOFT_EVENT_SET) + return 0; /* Event already set, nothing to do */ + + kbasep_add_waiting_with_timeout(katom); + + return 1; +} + +static void kbasep_soft_event_update_locked(struct kbase_jd_atom *katom, + unsigned char new_status) +{ + /* Complete jobs waiting on the same event */ + struct kbase_context *kctx = katom->kctx; + + if (kbasep_write_soft_event_status(kctx, katom->jc, new_status) != 0) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + return; + } + + if (new_status == BASE_JD_SOFT_EVENT_SET) + kbasep_complete_triggered_soft_events(kctx, katom->jc); +} + +/** + * kbase_soft_event_update() - Update soft event state + * @kctx: Pointer to context + * @event: Event to update + * @new_status: New status value of event + * + * Update the event, and wake up any atoms waiting for the event. + * + * Return: 0 on success, a negative error code on failure. + */ +int kbase_soft_event_update(struct kbase_context *kctx, + u64 event, + unsigned char new_status) +{ + int err = 0; + + mutex_lock(&kctx->jctx.lock); + + if (kbasep_write_soft_event_status(kctx, event, new_status)) { + err = -ENOENT; + goto out; + } + + if (new_status == BASE_JD_SOFT_EVENT_SET) + kbasep_complete_triggered_soft_events(kctx, event); + +out: + mutex_unlock(&kctx->jctx.lock); + + return err; +} + +static void kbasep_soft_event_cancel_job(struct kbase_jd_atom *katom) +{ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +struct kbase_debug_copy_buffer { + size_t size; + struct page **pages; + int nr_pages; + size_t offset; + struct kbase_mem_phy_alloc *gpu_alloc; + + struct page **extres_pages; + int nr_extres_pages; +}; + +static inline void free_user_buffer(struct kbase_debug_copy_buffer *buffer) +{ + struct page **pages = buffer->extres_pages; + int nr_pages = buffer->nr_extres_pages; + + if (pages) { + int i; + + for (i = 0; i < nr_pages; i++) { + struct page *pg = pages[i]; + + if (pg) + put_page(pg); + } + kfree(pages); + } +} + +static void kbase_debug_copy_finish(struct kbase_jd_atom *katom) +{ + struct kbase_debug_copy_buffer *buffers = + (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc; + unsigned int i; + unsigned int nr = katom->nr_extres; + + if (!buffers) + return; + + kbase_gpu_vm_lock(katom->kctx); + for (i = 0; i < nr; i++) { + int p; + struct kbase_mem_phy_alloc *gpu_alloc = buffers[i].gpu_alloc; + + if (!buffers[i].pages) + break; + for (p = 0; p < buffers[i].nr_pages; p++) { + struct page *pg = buffers[i].pages[p]; + + if (pg) + put_page(pg); + } + kfree(buffers[i].pages); + if (gpu_alloc) { + switch (gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + { + free_user_buffer(&buffers[i]); + break; + } + default: + /* Nothing to be done. */ + break; + } + kbase_mem_phy_alloc_put(gpu_alloc); + } + } + kbase_gpu_vm_unlock(katom->kctx); + kfree(buffers); + + katom->jc = 0; +} + +static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) +{ + struct kbase_debug_copy_buffer *buffers; + struct base_jd_debug_copy_buffer *user_buffers = NULL; + unsigned int i; + unsigned int nr = katom->nr_extres; + int ret = 0; + void __user *user_structs = (void __user *)(uintptr_t)katom->jc; + + if (!user_structs) + return -EINVAL; + + buffers = kcalloc(nr, sizeof(*buffers), GFP_KERNEL); + if (!buffers) { + ret = -ENOMEM; + katom->jc = 0; + goto out_cleanup; + } + katom->jc = (u64)(uintptr_t)buffers; + + user_buffers = kmalloc_array(nr, sizeof(*user_buffers), GFP_KERNEL); + + if (!user_buffers) { + ret = -ENOMEM; + goto out_cleanup; + } + + ret = copy_from_user(user_buffers, user_structs, + sizeof(*user_buffers)*nr); + if (ret) + goto out_cleanup; + + for (i = 0; i < nr; i++) { + u64 addr = user_buffers[i].address; + u64 page_addr = addr & PAGE_MASK; + u64 end_page_addr = addr + user_buffers[i].size - 1; + u64 last_page_addr = end_page_addr & PAGE_MASK; + int nr_pages = (last_page_addr-page_addr)/PAGE_SIZE+1; + int pinned_pages; + struct kbase_va_region *reg; + struct base_external_resource user_extres; + + if (!addr) + continue; + + buffers[i].nr_pages = nr_pages; + buffers[i].offset = addr & ~PAGE_MASK; + if (buffers[i].offset >= PAGE_SIZE) { + ret = -EINVAL; + goto out_cleanup; + } + buffers[i].size = user_buffers[i].size; + + buffers[i].pages = kcalloc(nr_pages, sizeof(struct page *), + GFP_KERNEL); + if (!buffers[i].pages) { + ret = -ENOMEM; + goto out_cleanup; + } + + pinned_pages = get_user_pages_fast(page_addr, + nr_pages, + 1, /* Write */ + buffers[i].pages); + if (pinned_pages < 0) { + ret = pinned_pages; + goto out_cleanup; + } + if (pinned_pages != nr_pages) { + ret = -EINVAL; + goto out_cleanup; + } + + user_extres = user_buffers[i].extres; + if (user_extres.ext_resource == 0ULL) { + ret = -EINVAL; + goto out_cleanup; + } + + kbase_gpu_vm_lock(katom->kctx); + reg = kbase_region_tracker_find_region_enclosing_address( + katom->kctx, user_extres.ext_resource & + ~BASE_EXT_RES_ACCESS_EXCLUSIVE); + + if (NULL == reg || NULL == reg->gpu_alloc || + (reg->flags & KBASE_REG_FREE)) { + ret = -EINVAL; + goto out_unlock; + } + + buffers[i].gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); + buffers[i].nr_extres_pages = reg->nr_pages; + + if (reg->nr_pages*PAGE_SIZE != buffers[i].size) + dev_warn(katom->kctx->kbdev->dev, "Copy buffer is not of same size as the external resource to copy.\n"); + + switch (reg->gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + { + struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; + unsigned long nr_pages = + alloc->imported.user_buf.nr_pages; + + if (alloc->imported.user_buf.mm != current->mm) { + ret = -EINVAL; + goto out_unlock; + } + buffers[i].extres_pages = kcalloc(nr_pages, + sizeof(struct page *), GFP_KERNEL); + if (!buffers[i].extres_pages) { + ret = -ENOMEM; + goto out_unlock; + } + + ret = get_user_pages_fast( + alloc->imported.user_buf.address, + nr_pages, 0, + buffers[i].extres_pages); + if (ret != nr_pages) + goto out_unlock; + ret = 0; + break; + } + case KBASE_MEM_TYPE_IMPORTED_UMP: + { + dev_warn(katom->kctx->kbdev->dev, + "UMP is not supported for debug_copy jobs\n"); + ret = -EINVAL; + goto out_unlock; + } + default: + /* Nothing to be done. */ + break; + } + kbase_gpu_vm_unlock(katom->kctx); + } + kfree(user_buffers); + + return ret; + +out_unlock: + kbase_gpu_vm_unlock(katom->kctx); + +out_cleanup: + kfree(buffers); + kfree(user_buffers); + + /* Frees allocated memory for kbase_debug_copy_job struct, including + * members, and sets jc to 0 */ + kbase_debug_copy_finish(katom); + return ret; +} + +static void kbase_mem_copy_from_extres_page(struct kbase_context *kctx, + void *extres_page, struct page **pages, unsigned int nr_pages, + unsigned int *target_page_nr, size_t offset, size_t *to_copy) +{ + void *target_page = kmap(pages[*target_page_nr]); + size_t chunk = PAGE_SIZE-offset; + + lockdep_assert_held(&kctx->reg_lock); + + if (!target_page) { + *target_page_nr += 1; + dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job."); + return; + } + + chunk = min(chunk, *to_copy); + + memcpy(target_page + offset, extres_page, chunk); + *to_copy -= chunk; + + kunmap(pages[*target_page_nr]); + + *target_page_nr += 1; + if (*target_page_nr >= nr_pages) + return; + + target_page = kmap(pages[*target_page_nr]); + if (!target_page) { + *target_page_nr += 1; + dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job."); + return; + } + + KBASE_DEBUG_ASSERT(target_page); + + chunk = min(offset, *to_copy); + memcpy(target_page, extres_page + PAGE_SIZE-offset, chunk); + *to_copy -= chunk; + + kunmap(pages[*target_page_nr]); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) +static void *dma_buf_kmap_page(struct kbase_mem_phy_alloc *gpu_alloc, + unsigned long page_num, struct page **page) +{ + struct sg_table *sgt = gpu_alloc->imported.umm.sgt; + struct sg_page_iter sg_iter; + unsigned long page_index = 0; + + if (WARN_ON(gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) + return NULL; + + if (!sgt) + return NULL; + + if (WARN_ON(page_num >= gpu_alloc->nents)) + return NULL; + + for_each_sg_page(sgt->sgl, &sg_iter, sgt->nents, 0) { + if (page_index == page_num) { + *page = sg_page_iter_page(&sg_iter); + + return kmap(*page); + } + page_index++; + } + + return NULL; +} +#endif + +static int kbase_mem_copy_from_extres(struct kbase_context *kctx, + struct kbase_debug_copy_buffer *buf_data) +{ + unsigned int i; + unsigned int target_page_nr = 0; + struct page **pages = buf_data->pages; + u64 offset = buf_data->offset; + size_t extres_size = buf_data->nr_extres_pages*PAGE_SIZE; + size_t to_copy = min(extres_size, buf_data->size); + struct kbase_mem_phy_alloc *gpu_alloc = buf_data->gpu_alloc; + int ret = 0; + + KBASE_DEBUG_ASSERT(pages != NULL); + + kbase_gpu_vm_lock(kctx); + if (!gpu_alloc) { + ret = -EINVAL; + goto out_unlock; + } + + switch (gpu_alloc->type) { + case KBASE_MEM_TYPE_IMPORTED_USER_BUF: + { + for (i = 0; i < buf_data->nr_extres_pages; i++) { + struct page *pg = buf_data->extres_pages[i]; + void *extres_page = kmap(pg); + + if (extres_page) + kbase_mem_copy_from_extres_page(kctx, + extres_page, pages, + buf_data->nr_pages, + &target_page_nr, + offset, &to_copy); + + kunmap(pg); + if (target_page_nr >= buf_data->nr_pages) + break; + } + break; + } + break; +#ifdef CONFIG_DMA_SHARED_BUFFER + case KBASE_MEM_TYPE_IMPORTED_UMM: { + struct dma_buf *dma_buf = gpu_alloc->imported.umm.dma_buf; + + KBASE_DEBUG_ASSERT(dma_buf != NULL); + KBASE_DEBUG_ASSERT(dma_buf->size == + buf_data->nr_extres_pages * PAGE_SIZE); + + ret = dma_buf_begin_cpu_access(dma_buf, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) + 0, buf_data->nr_extres_pages*PAGE_SIZE, +#endif + DMA_FROM_DEVICE); + if (ret) + goto out_unlock; + + for (i = 0; i < buf_data->nr_extres_pages; i++) { + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) + struct page *pg; + void *extres_page = dma_buf_kmap_page(gpu_alloc, i, &pg); +#else + void *extres_page = dma_buf_kmap(dma_buf, i); +#endif + + if (extres_page) + kbase_mem_copy_from_extres_page(kctx, + extres_page, pages, + buf_data->nr_pages, + &target_page_nr, + offset, &to_copy); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) + kunmap(pg); +#else + dma_buf_kunmap(dma_buf, i, extres_page); +#endif + if (target_page_nr >= buf_data->nr_pages) + break; + } + dma_buf_end_cpu_access(dma_buf, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) + 0, buf_data->nr_extres_pages*PAGE_SIZE, +#endif + DMA_FROM_DEVICE); + break; + } +#endif + default: + ret = -EINVAL; + } +out_unlock: + kbase_gpu_vm_unlock(kctx); + return ret; + +} + +static int kbase_debug_copy(struct kbase_jd_atom *katom) +{ + struct kbase_debug_copy_buffer *buffers = + (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc; + unsigned int i; + + for (i = 0; i < katom->nr_extres; i++) { + int res = kbase_mem_copy_from_extres(katom->kctx, &buffers[i]); + + if (res) + return res; + } + + return 0; +} + +static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) +{ + __user void *data = (__user void *)(uintptr_t) katom->jc; + struct base_jit_alloc_info *info; + struct kbase_context *kctx = katom->kctx; + int ret; + + /* Fail the job if there is no info structure */ + if (!data) { + ret = -EINVAL; + goto fail; + } + + /* Copy the information for safe access and future storage */ + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) { + ret = -ENOMEM; + goto fail; + } + + if (copy_from_user(info, data, sizeof(*info)) != 0) { + ret = -EINVAL; + goto free_info; + } + + /* If the ID is zero then fail the job */ + if (info->id == 0) { + ret = -EINVAL; + goto free_info; + } + + /* Sanity check that the PA fits within the VA */ + if (info->va_pages < info->commit_pages) { + ret = -EINVAL; + goto free_info; + } + + /* Ensure the GPU address is correctly aligned */ + if ((info->gpu_alloc_addr & 0x7) != 0) { + ret = -EINVAL; + goto free_info; + } + + /* Replace the user pointer with our kernel allocated info structure */ + katom->jc = (u64)(uintptr_t) info; + katom->jit_blocked = false; + + lockdep_assert_held(&kctx->jctx.lock); + list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); + + /* + * Note: + * The provided info->gpu_alloc_addr isn't validated here as + * userland can cache allocations which means that even + * though the region is valid it doesn't represent the + * same thing it used to. + * + * Complete validation of va_pages, commit_pages and extent + * isn't done here as it will be done during the call to + * kbase_mem_alloc. + */ + return 0; + +free_info: + kfree(info); +fail: + katom->jc = 0; + return ret; +} + +static u8 kbase_jit_free_get_id(struct kbase_jd_atom *katom) +{ + if (WARN_ON(katom->core_req != BASE_JD_REQ_SOFT_JIT_FREE)) + return 0; + + return (u8) katom->jc; +} + +static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + struct base_jit_alloc_info *info; + struct kbase_va_region *reg; + struct kbase_vmap_struct mapping; + u64 *ptr, new_addr; + + if (katom->jit_blocked) { + list_del(&katom->queue); + katom->jit_blocked = false; + } + + info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc; + + /* The JIT ID is still in use so fail the allocation */ + if (kctx->jit_alloc[info->id]) { + katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; + return 0; + } + + /* Create a JIT allocation */ + reg = kbase_jit_allocate(kctx, info); + if (!reg) { + struct kbase_jd_atom *jit_atom; + bool can_block = false; + + lockdep_assert_held(&kctx->jctx.lock); + + jit_atom = list_first_entry(&kctx->jit_atoms_head, + struct kbase_jd_atom, jit_node); + + list_for_each_entry(jit_atom, &kctx->jit_atoms_head, jit_node) { + if (jit_atom == katom) + break; + if (jit_atom->core_req == BASE_JD_REQ_SOFT_JIT_FREE) { + u8 free_id = kbase_jit_free_get_id(jit_atom); + + if (free_id && kctx->jit_alloc[free_id]) { + /* A JIT free which is active and + * submitted before this atom + */ + can_block = true; + break; + } + } + } + + if (!can_block) { + /* Mark the allocation so we know it's in use even if + * the allocation itself fails. + */ + kctx->jit_alloc[info->id] = + (struct kbase_va_region *) -1; + + katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; + return 0; + } + + /* There are pending frees for an active allocation + * so we should wait to see whether they free the memory. + * Add to the beginning of the list to ensure that the atom is + * processed only once in kbase_jit_free_finish + */ + list_add(&katom->queue, &kctx->jit_pending_alloc); + katom->jit_blocked = true; + + return 1; + } + + /* + * Write the address of the JIT allocation to the user provided + * GPU allocation. + */ + ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr), + &mapping); + if (!ptr) { + /* + * Leave the allocation "live" as the JIT free jit will be + * submitted anyway. + */ + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return 0; + } + + new_addr = reg->start_pfn << PAGE_SHIFT; + *ptr = new_addr; + KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( + katom, info->gpu_alloc_addr, new_addr); + kbase_vunmap(kctx, &mapping); + + katom->event_code = BASE_JD_EVENT_DONE; + + /* + * Bind it to the user provided ID. Do this last so we can check for + * the JIT free racing this JIT alloc job. + */ + kctx->jit_alloc[info->id] = reg; + + return 0; +} + +static void kbase_jit_allocate_finish(struct kbase_jd_atom *katom) +{ + struct base_jit_alloc_info *info; + + lockdep_assert_held(&katom->kctx->jctx.lock); + + /* Remove atom from jit_atoms_head list */ + list_del(&katom->jit_node); + + if (katom->jit_blocked) { + list_del(&katom->queue); + katom->jit_blocked = false; + } + + info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc; + /* Free the info structure */ + kfree(info); +} + +static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + + lockdep_assert_held(&kctx->jctx.lock); + list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); + + return 0; +} + +static void kbase_jit_free_process(struct kbase_jd_atom *katom) +{ + struct kbase_context *kctx = katom->kctx; + u8 id = kbase_jit_free_get_id(katom); + + /* + * If the ID is zero or it is not in use yet then fail the job. + */ + if ((id == 0) || (kctx->jit_alloc[id] == NULL)) { + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + return; + } + + /* + * If the ID is valid but the allocation request failed still succeed + * this soft job but don't try and free the allocation. + */ + if (kctx->jit_alloc[id] != (struct kbase_va_region *) -1) + kbase_jit_free(kctx, kctx->jit_alloc[id]); + + kctx->jit_alloc[id] = NULL; +} + +static void kbasep_jit_free_finish_worker(struct work_struct *work) +{ + struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, + work); + struct kbase_context *kctx = katom->kctx; + int resched; + + mutex_lock(&kctx->jctx.lock); + kbase_finish_soft_job(katom); + resched = jd_done_nolock(katom, NULL); + mutex_unlock(&kctx->jctx.lock); + + if (resched) + kbase_js_sched_all(kctx->kbdev); +} + +static void kbase_jit_free_finish(struct kbase_jd_atom *katom) +{ + struct list_head *i, *tmp; + struct kbase_context *kctx = katom->kctx; + + lockdep_assert_held(&kctx->jctx.lock); + /* Remove this atom from the kctx->jit_atoms_head list */ + list_del(&katom->jit_node); + + list_for_each_safe(i, tmp, &kctx->jit_pending_alloc) { + struct kbase_jd_atom *pending_atom = list_entry(i, + struct kbase_jd_atom, queue); + if (kbase_jit_allocate_process(pending_atom) == 0) { + /* Atom has completed */ + INIT_WORK(&pending_atom->work, + kbasep_jit_free_finish_worker); + queue_work(kctx->jctx.job_done_wq, &pending_atom->work); + } + } +} + +static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) +{ + __user struct base_external_resource_list *user_ext_res; + struct base_external_resource_list *ext_res; + u64 count = 0; + size_t copy_size; + int ret; + + user_ext_res = (__user struct base_external_resource_list *) + (uintptr_t) katom->jc; + + /* Fail the job if there is no info structure */ + if (!user_ext_res) { + ret = -EINVAL; + goto fail; + } + + if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) { + ret = -EINVAL; + goto fail; + } + + /* Is the number of external resources in range? */ + if (!count || count > BASE_EXT_RES_COUNT_MAX) { + ret = -EINVAL; + goto fail; + } + + /* Copy the information for safe access and future storage */ + copy_size = sizeof(*ext_res); + copy_size += sizeof(struct base_external_resource) * (count - 1); + ext_res = kzalloc(copy_size, GFP_KERNEL); + if (!ext_res) { + ret = -ENOMEM; + goto fail; + } + + if (copy_from_user(ext_res, user_ext_res, copy_size) != 0) { + ret = -EINVAL; + goto free_info; + } + + /* + * Overwrite the count with the first value incase it was changed + * after the fact. + */ + ext_res->count = count; + + /* + * Replace the user pointer with our kernel allocated + * ext_res structure. + */ + katom->jc = (u64)(uintptr_t) ext_res; + + return 0; + +free_info: + kfree(ext_res); +fail: + return ret; +} + +static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) +{ + struct base_external_resource_list *ext_res; + int i; + bool failed = false; + + ext_res = (struct base_external_resource_list *) (uintptr_t) katom->jc; + if (!ext_res) + goto failed_jc; + + kbase_gpu_vm_lock(katom->kctx); + + for (i = 0; i < ext_res->count; i++) { + u64 gpu_addr; + + gpu_addr = ext_res->ext_res[i].ext_resource & + ~BASE_EXT_RES_ACCESS_EXCLUSIVE; + if (map) { + if (!kbase_sticky_resource_acquire(katom->kctx, + gpu_addr)) + goto failed_loop; + } else + if (!kbase_sticky_resource_release(katom->kctx, NULL, + gpu_addr)) + failed = true; + } + + /* + * In the case of unmap we continue unmapping other resources in the + * case of failure but will always report failure if _any_ unmap + * request fails. + */ + if (failed) + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + else + katom->event_code = BASE_JD_EVENT_DONE; + + kbase_gpu_vm_unlock(katom->kctx); + + return; + +failed_loop: + while (--i > 0) { + u64 gpu_addr; + + gpu_addr = ext_res->ext_res[i].ext_resource & + ~BASE_EXT_RES_ACCESS_EXCLUSIVE; + + kbase_sticky_resource_release(katom->kctx, NULL, gpu_addr); + } + + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + kbase_gpu_vm_unlock(katom->kctx); + +failed_jc: + return; +} + +static void kbase_ext_res_finish(struct kbase_jd_atom *katom) +{ + struct base_external_resource_list *ext_res; + + ext_res = (struct base_external_resource_list *) (uintptr_t) katom->jc; + /* Free the info structure */ + kfree(ext_res); +} + +int kbase_process_soft_job(struct kbase_jd_atom *katom) +{ + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: + return kbase_dump_cpu_gpu_time(katom); + +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + katom->event_code = kbase_sync_fence_out_trigger(katom, + katom->event_code == BASE_JD_EVENT_DONE ? + 0 : -EFAULT); + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + { + int ret = kbase_sync_fence_in_wait(katom); + + if (ret == 1) { +#ifdef CONFIG_MALI_FENCE_DEBUG + kbasep_add_waiting_with_timeout(katom); +#else + kbasep_add_waiting_soft_job(katom); +#endif + } + return ret; + } +#endif + + case BASE_JD_REQ_SOFT_REPLAY: + return kbase_replay_process(katom); + case BASE_JD_REQ_SOFT_EVENT_WAIT: + return kbasep_soft_event_wait(katom); + case BASE_JD_REQ_SOFT_EVENT_SET: + kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_SET); + break; + case BASE_JD_REQ_SOFT_EVENT_RESET: + kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_RESET); + break; + case BASE_JD_REQ_SOFT_DEBUG_COPY: + { + int res = kbase_debug_copy(katom); + + if (res) + katom->event_code = BASE_JD_EVENT_JOB_INVALID; + break; + } + case BASE_JD_REQ_SOFT_JIT_ALLOC: + return kbase_jit_allocate_process(katom); + case BASE_JD_REQ_SOFT_JIT_FREE: + kbase_jit_free_process(katom); + break; + case BASE_JD_REQ_SOFT_EXT_RES_MAP: + kbase_ext_res_process(katom, true); + break; + case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: + kbase_ext_res_process(katom, false); + break; + } + + /* Atom is complete */ + return 0; +} + +void kbase_cancel_soft_job(struct kbase_jd_atom *katom) +{ + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_WAIT: + kbase_sync_fence_in_cancel_wait(katom); + break; +#endif + case BASE_JD_REQ_SOFT_EVENT_WAIT: + kbasep_soft_event_cancel_job(katom); + break; + default: + /* This soft-job doesn't support cancellation! */ + KBASE_DEBUG_ASSERT(0); + } +} + +int kbase_prepare_soft_job(struct kbase_jd_atom *katom) +{ + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: + { + if (0 != (katom->jc & KBASE_CACHE_ALIGNMENT_MASK)) + return -EINVAL; + } + break; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + { + struct base_fence fence; + int fd; + + if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) + return -EINVAL; + + fd = kbase_sync_fence_out_create(katom, + fence.basep.stream_fd); + if (fd < 0) + return -EINVAL; + + fence.basep.fd = fd; + if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) { + kbase_sync_fence_out_remove(katom); + kbase_sync_fence_close_fd(fd); + fence.basep.fd = -EINVAL; + return -EINVAL; + } + } + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + { + struct base_fence fence; + int ret; + + if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) + return -EINVAL; + + /* Get a reference to the fence object */ + ret = kbase_sync_fence_in_from_fd(katom, + fence.basep.fd); + if (ret < 0) + return ret; + +#ifdef CONFIG_MALI_DMA_FENCE + /* + * Set KCTX_NO_IMPLICIT_FENCE in the context the first + * time a soft fence wait job is observed. This will + * prevent the implicit dma-buf fence to conflict with + * the Android native sync fences. + */ + if (!kbase_ctx_flag(katom->kctx, KCTX_NO_IMPLICIT_SYNC)) + kbase_ctx_flag_set(katom->kctx, KCTX_NO_IMPLICIT_SYNC); +#endif /* CONFIG_MALI_DMA_FENCE */ + } + break; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + case BASE_JD_REQ_SOFT_JIT_ALLOC: + return kbase_jit_allocate_prepare(katom); + case BASE_JD_REQ_SOFT_REPLAY: + break; + case BASE_JD_REQ_SOFT_JIT_FREE: + return kbase_jit_free_prepare(katom); + case BASE_JD_REQ_SOFT_EVENT_WAIT: + case BASE_JD_REQ_SOFT_EVENT_SET: + case BASE_JD_REQ_SOFT_EVENT_RESET: + if (katom->jc == 0) + return -EINVAL; + break; + case BASE_JD_REQ_SOFT_DEBUG_COPY: + return kbase_debug_copy_prepare(katom); + case BASE_JD_REQ_SOFT_EXT_RES_MAP: + return kbase_ext_res_prepare(katom); + case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: + return kbase_ext_res_prepare(katom); + default: + /* Unsupported soft-job */ + return -EINVAL; + } + return 0; +} + +void kbase_finish_soft_job(struct kbase_jd_atom *katom) +{ + switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { + case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: + /* Nothing to do */ + break; +#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) + case BASE_JD_REQ_SOFT_FENCE_TRIGGER: + /* If fence has not yet been signaled, do it now */ + kbase_sync_fence_out_trigger(katom, katom->event_code == + BASE_JD_EVENT_DONE ? 0 : -EFAULT); + break; + case BASE_JD_REQ_SOFT_FENCE_WAIT: + /* Release katom's reference to fence object */ + kbase_sync_fence_in_remove(katom); + break; +#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ + case BASE_JD_REQ_SOFT_DEBUG_COPY: + kbase_debug_copy_finish(katom); + break; + case BASE_JD_REQ_SOFT_JIT_ALLOC: + kbase_jit_allocate_finish(katom); + break; + case BASE_JD_REQ_SOFT_EXT_RES_MAP: + kbase_ext_res_finish(katom); + break; + case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: + kbase_ext_res_finish(katom); + break; + case BASE_JD_REQ_SOFT_JIT_FREE: + kbase_jit_free_finish(katom); + break; + } +} + +void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) +{ + LIST_HEAD(local_suspended_soft_jobs); + struct kbase_jd_atom *tmp_iter; + struct kbase_jd_atom *katom_iter; + struct kbasep_js_device_data *js_devdata; + bool resched = false; + + KBASE_DEBUG_ASSERT(kbdev); + + js_devdata = &kbdev->js_data; + + /* Move out the entire list */ + mutex_lock(&js_devdata->runpool_mutex); + list_splice_init(&js_devdata->suspended_soft_jobs_list, + &local_suspended_soft_jobs); + mutex_unlock(&js_devdata->runpool_mutex); + + /* + * Each atom must be detached from the list and ran separately - + * it could be re-added to the old list, but this is unlikely + */ + list_for_each_entry_safe(katom_iter, tmp_iter, + &local_suspended_soft_jobs, dep_item[1]) { + struct kbase_context *kctx = katom_iter->kctx; + + mutex_lock(&kctx->jctx.lock); + + /* Remove from the global list */ + list_del(&katom_iter->dep_item[1]); + /* Remove from the context's list of waiting soft jobs */ + kbasep_remove_waiting_soft_job(katom_iter); + + if (kbase_process_soft_job(katom_iter) == 0) { + kbase_finish_soft_job(katom_iter); + resched |= jd_done_nolock(katom_iter, NULL); + } else { + KBASE_DEBUG_ASSERT((katom_iter->core_req & + BASE_JD_REQ_SOFT_JOB_TYPE) + != BASE_JD_REQ_SOFT_REPLAY); + } + + mutex_unlock(&kctx->jctx.lock); + } + + if (resched) + kbase_js_sched_all(kbdev); +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_strings.c b/drivers/gpu/arm/midgard/mali_kbase_strings.c new file mode 100755 index 000000000000..c98762cec244 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_strings.c @@ -0,0 +1,23 @@ + /* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + +#include "mali_kbase_strings.h" + +#define KBASE_DRV_NAME "mali" +#define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline" + +const char kbase_drv_name[] = KBASE_DRV_NAME; +const char kbase_timeline_name[] = KBASE_TIMELINE_NAME; diff --git a/drivers/gpu/arm/midgard/mali_kbase_strings.h b/drivers/gpu/arm/midgard/mali_kbase_strings.h new file mode 100755 index 000000000000..41b8fdbec6a4 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_strings.h @@ -0,0 +1,19 @@ +/* + * + * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +extern const char kbase_drv_name[]; +extern const char kbase_timeline_name[]; diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync.h b/drivers/gpu/arm/midgard/mali_kbase_sync.h new file mode 100755 index 000000000000..33b580595563 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_sync.h @@ -0,0 +1,203 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * @file mali_kbase_sync.h + * + * This file contains our internal "API" for explicit fences. + * It hides the implementation details of the actual explicit fence mechanism + * used (Android fences or sync file with DMA fences). + */ + +#ifndef MALI_KBASE_SYNC_H +#define MALI_KBASE_SYNC_H + +#include +#ifdef CONFIG_SYNC +#include +#endif +#ifdef CONFIG_SYNC_FILE +#include "mali_kbase_fence_defs.h" +#include +#endif + +#include "mali_kbase.h" + +/** + * struct kbase_sync_fence_info - Information about a fence + * @fence: Pointer to fence (type is void*, as underlaying struct can differ) + * @name: The name given to this fence when it was created + * @status: < 0 means error, 0 means active, 1 means signaled + * + * Use kbase_sync_fence_in_info_get() or kbase_sync_fence_out_info_get() + * to get the information. + */ +struct kbase_sync_fence_info { + void *fence; + char name[32]; + int status; +}; + +/** + * kbase_sync_fence_stream_create() - Create a stream object + * @name: Name of stream (only used to ease debugging/visualization) + * @out_fd: A file descriptor representing the created stream object + * + * Can map down to a timeline implementation in some implementations. + * Exposed as a file descriptor. + * Life-time controlled via the file descriptor: + * - dup to add a ref + * - close to remove a ref + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_stream_create(const char *name, int *const out_fd); + +/** + * kbase_sync_fence_out_create Create an explicit output fence to specified atom + * @katom: Atom to assign the new explicit fence to + * @stream_fd: File descriptor for stream object to create fence on + * + * return: Valid file descriptor to fence or < 0 on error + */ +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); + +/** + * kbase_sync_fence_in_from_fd() Assigns an existing fence to specified atom + * @katom: Atom to assign the existing explicit fence to + * @fd: File descriptor to an existing fence + * + * Assigns an explicit input fence to atom. + * This can later be waited for by calling @kbase_sync_fence_in_wait + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); + +/** + * kbase_sync_fence_validate() - Validate a fd to be a valid fence + * @fd: File descriptor to check + * + * This function is only usable to catch unintentional user errors early, + * it does not stop malicious code changing the fd after this function returns. + * + * return 0: if fd is for a valid fence, < 0 if invalid + */ +int kbase_sync_fence_validate(int fd); + +/** + * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom + * @katom: Atom with an explicit fence to signal + * @result: < 0 means signal with error, 0 >= indicates success + * + * Signal output fence attached on katom and remove the fence from the atom. + * + * return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE + */ +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result); + +/** + * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled + * @katom: Atom with explicit fence to wait for + * + * If the fence is already signaled, then 0 is returned, and the caller must + * continue processing of the katom. + * + * If the fence isn't already signaled, then this kbase_sync framework will + * take responsibility to continue the processing once the fence is signaled. + * + * return: 0 if already signaled, otherwise 1 + */ +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits + * @katom: Atom to cancel wait for + * + * This function is fully responsible for continuing processing of this atom + * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all) + */ +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_in_remove() - Remove the input fence from the katom + * @katom: Atom to remove explicit input fence for + * + * This will also release the corresponding reference. + */ +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_out_remove() - Remove the output fence from the katom + * @katom: Atom to remove explicit output fence for + * + * This will also release the corresponding reference. + */ +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); + +/** + * kbase_sync_fence_close_fd() - Close a file descriptor representing a fence + * @fd: File descriptor to close + */ +static inline void kbase_sync_fence_close_fd(int fd) +{ + ksys_close(fd); +} + +/** + * kbase_sync_fence_in_info_get() - Retrieves information about input fence + * @katom: Atom to get fence information from + * @info: Struct to be filled with fence information + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info); + +/** + * kbase_sync_fence_out_info_get() - Retrieves information about output fence + * @katom: Atom to get fence information from + * @info: Struct to be filled with fence information + * + * return: 0 on success, < 0 on error + */ +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info); + +/** + * kbase_sync_status_string() - Get string matching @status + * @status: Value of fence status. + * + * return: Pointer to string describing @status. + */ +const char *kbase_sync_status_string(int status); + +/* + * Internal worker used to continue processing of atom. + */ +void kbase_sync_fence_wait_worker(struct work_struct *data); + +#ifdef CONFIG_MALI_FENCE_DEBUG +/** + * kbase_sync_fence_in_dump() Trigger a debug dump of atoms input fence state + * @katom: Atom to trigger fence debug dump for + */ +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); +#endif + +#endif /* MALI_KBASE_SYNC_H */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync_android.c b/drivers/gpu/arm/midgard/mali_kbase_sync_android.c new file mode 100755 index 000000000000..d7349dcae69a --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_sync_android.c @@ -0,0 +1,537 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Code for supporting explicit Android fences (CONFIG_SYNC) + * Known to be good for kernels 4.5 and earlier. + * Replaced with CONFIG_SYNC_FILE for 4.9 and later kernels + * (see mali_kbase_sync_file.c) + */ + +#include +#include +#include +#include +#include +#include +#include +#include "sync.h" +#include +#include + +struct mali_sync_timeline { + struct sync_timeline timeline; + atomic_t counter; + atomic_t signaled; +}; + +struct mali_sync_pt { + struct sync_pt pt; + int order; + int result; +}; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +/* For backwards compatibility with kernels before 3.17. After 3.17 + * sync_pt_parent is included in the kernel. */ +static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) +{ + return pt->parent; +} +#endif + +static struct mali_sync_timeline *to_mali_sync_timeline( + struct sync_timeline *timeline) +{ + return container_of(timeline, struct mali_sync_timeline, timeline); +} + +static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) +{ + return container_of(pt, struct mali_sync_pt, pt); +} + +static struct sync_pt *timeline_dup(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_pt *new_mpt; + struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), + sizeof(struct mali_sync_pt)); + + if (!new_pt) + return NULL; + + new_mpt = to_mali_sync_pt(new_pt); + new_mpt->order = mpt->order; + new_mpt->result = mpt->result; + + return new_pt; +} + +static int timeline_has_signaled(struct sync_pt *pt) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_timeline *mtl = to_mali_sync_timeline( + sync_pt_parent(pt)); + int result = mpt->result; + + int diff = atomic_read(&mtl->signaled) - mpt->order; + + if (diff >= 0) + return (result < 0) ? result : 1; + + return 0; +} + +static int timeline_compare(struct sync_pt *a, struct sync_pt *b) +{ + struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt); + struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt); + + int diff = ma->order - mb->order; + + if (diff == 0) + return 0; + + return (diff < 0) ? -1 : 1; +} + +static void timeline_value_str(struct sync_timeline *timeline, char *str, + int size) +{ + struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline); + + snprintf(str, size, "%d", atomic_read(&mtl->signaled)); +} + +static void pt_value_str(struct sync_pt *pt, char *str, int size) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + + snprintf(str, size, "%d(%d)", mpt->order, mpt->result); +} + +static struct sync_timeline_ops mali_timeline_ops = { + .driver_name = "Mali", + .dup = timeline_dup, + .has_signaled = timeline_has_signaled, + .compare = timeline_compare, + .timeline_value_str = timeline_value_str, + .pt_value_str = pt_value_str, +}; + +/* Allocates a timeline for Mali + * + * One timeline should be allocated per API context. + */ +static struct sync_timeline *mali_sync_timeline_alloc(const char *name) +{ + struct sync_timeline *tl; + struct mali_sync_timeline *mtl; + + tl = sync_timeline_create(&mali_timeline_ops, + sizeof(struct mali_sync_timeline), name); + if (!tl) + return NULL; + + /* Set the counter in our private struct */ + mtl = to_mali_sync_timeline(tl); + atomic_set(&mtl->counter, 0); + atomic_set(&mtl->signaled, 0); + + return tl; +} + +static int kbase_stream_close(struct inode *inode, struct file *file) +{ + struct sync_timeline *tl; + + tl = (struct sync_timeline *)file->private_data; + sync_timeline_destroy(tl); + return 0; +} + +static const struct file_operations stream_fops = { + .owner = THIS_MODULE, + .release = kbase_stream_close, +}; + +int kbase_sync_fence_stream_create(const char *name, int *const out_fd) +{ + struct sync_timeline *tl; + + if (!out_fd) + return -EINVAL; + + tl = mali_sync_timeline_alloc(name); + if (!tl) + return -EINVAL; + + *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY|O_CLOEXEC); + + if (*out_fd < 0) { + sync_timeline_destroy(tl); + return -EINVAL; + } + + return 0; +} + +/* Allocates a sync point within the timeline. + * + * The timeline must be the one allocated by kbase_sync_timeline_alloc + * + * Sync points must be triggered in *exactly* the same order as they are + * allocated. + */ +static struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent) +{ + struct sync_pt *pt = sync_pt_create(parent, + sizeof(struct mali_sync_pt)); + struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent); + struct mali_sync_pt *mpt; + + if (!pt) + return NULL; + + mpt = to_mali_sync_pt(pt); + mpt->order = atomic_inc_return(&mtl->counter); + mpt->result = 0; + + return pt; +} + +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) +{ + struct sync_timeline *tl; + struct sync_pt *pt; + struct sync_fence *fence; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) + struct files_struct *files; + struct fdtable *fdt; +#endif + int fd; + struct file *tl_file; + + tl_file = fget(tl_fd); + if (tl_file == NULL) + return -EBADF; + + if (tl_file->f_op != &stream_fops) { + fd = -EBADF; + goto out; + } + + tl = tl_file->private_data; + + pt = kbase_sync_pt_alloc(tl); + if (!pt) { + fd = -EFAULT; + goto out; + } + + fence = sync_fence_create("mali_fence", pt); + if (!fence) { + sync_pt_free(pt); + fd = -EFAULT; + goto out; + } + + /* from here the fence owns the sync_pt */ + + /* create a fd representing the fence */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) + fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); + if (fd < 0) { + sync_fence_put(fence); + goto out; + } +#else + fd = get_unused_fd(); + if (fd < 0) { + sync_fence_put(fence); + goto out; + } + + files = current->files; + spin_lock(&files->file_lock); + fdt = files_fdtable(files); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) + __set_close_on_exec(fd, fdt); +#else + FD_SET(fd, fdt->close_on_exec); +#endif + spin_unlock(&files->file_lock); +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ + + /* bind fence to the new fd */ + sync_fence_install(fence, fd); + + katom->fence = sync_fence_fdget(fd); + if (katom->fence == NULL) { + /* The only way the fence can be NULL is if userspace closed it + * for us, so we don't need to clear it up */ + fd = -EINVAL; + goto out; + } + +out: + fput(tl_file); + + return fd; +} + +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) +{ + katom->fence = sync_fence_fdget(fd); + return katom->fence ? 0 : -ENOENT; +} + +int kbase_sync_fence_validate(int fd) +{ + struct sync_fence *fence; + + fence = sync_fence_fdget(fd); + if (!fence) + return -EINVAL; + + sync_fence_put(fence); + return 0; +} + +/* Returns true if the specified timeline is allocated by Mali */ +static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) +{ + return timeline->ops == &mali_timeline_ops; +} + +/* Signals a particular sync point + * + * Sync points must be triggered in *exactly* the same order as they are + * allocated. + * + * If they are signaled in the wrong order then a message will be printed in + * debug builds and otherwise attempts to signal order sync_pts will be ignored. + * + * result can be negative to indicate error, any other value is interpreted as + * success. + */ +static void kbase_sync_signal_pt(struct sync_pt *pt, int result) +{ + struct mali_sync_pt *mpt = to_mali_sync_pt(pt); + struct mali_sync_timeline *mtl = to_mali_sync_timeline( + sync_pt_parent(pt)); + int signaled; + int diff; + + mpt->result = result; + + do { + signaled = atomic_read(&mtl->signaled); + + diff = signaled - mpt->order; + + if (diff > 0) { + /* The timeline is already at or ahead of this point. + * This should not happen unless userspace has been + * signaling fences out of order, so warn but don't + * violate the sync_pt API. + * The warning is only in debug builds to prevent + * a malicious user being able to spam dmesg. + */ +#ifdef CONFIG_MALI_DEBUG + pr_err("Fences were triggered in a different order to allocation!"); +#endif /* CONFIG_MALI_DEBUG */ + return; + } + } while (atomic_cmpxchg(&mtl->signaled, + signaled, mpt->order) != signaled); +} + +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) +{ + struct sync_pt *pt; + struct sync_timeline *timeline; + + if (!katom->fence) + return BASE_JD_EVENT_JOB_CANCELLED; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + if (!list_is_singular(&katom->fence->pt_list_head)) { +#else + if (katom->fence->num_fences != 1) { +#endif + /* Not exactly one item in the list - so it didn't (directly) + * come from us */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + pt = list_first_entry(&katom->fence->pt_list_head, + struct sync_pt, pt_list); +#else + pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); +#endif + timeline = sync_pt_parent(pt); + + if (!kbase_sync_timeline_is_ours(timeline)) { + /* Fence has a sync_pt which isn't ours! */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + + kbase_sync_signal_pt(pt, result); + + sync_timeline_signal(timeline); + + kbase_sync_fence_out_remove(katom); + + return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; +} + +static inline int kbase_fence_get_status(struct sync_fence *fence) +{ + if (!fence) + return -ENOENT; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + return fence->status; +#else + return atomic_read(&fence->status); +#endif +} + +static void kbase_fence_wait_callback(struct sync_fence *fence, + struct sync_fence_waiter *waiter) +{ + struct kbase_jd_atom *katom = container_of(waiter, + struct kbase_jd_atom, sync_waiter); + struct kbase_context *kctx = katom->kctx; + + /* Propagate the fence status to the atom. + * If negative then cancel this atom and its dependencies. + */ + if (kbase_fence_get_status(fence) < 0) + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* To prevent a potential deadlock we schedule the work onto the + * job_done_wq workqueue + * + * The issue is that we may signal the timeline while holding + * kctx->jctx.lock and the callbacks are run synchronously from + * sync_timeline_signal. So we simply defer the work. + */ + + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); +} + +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) +{ + int ret; + + sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback); + + ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter); + + if (ret == 1) { + /* Already signaled */ + return 0; + } + + if (ret < 0) { + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + /* We should cause the dependent jobs in the bag to be failed, + * to do this we schedule the work queue to complete this job */ + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(katom->kctx->jctx.job_done_wq, &katom->work); + } + + return 1; +} + +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) +{ + if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { + /* The wait wasn't cancelled - leave the cleanup for + * kbase_fence_wait_callback */ + return; + } + + /* Wait was cancelled - zap the atoms */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) +{ + if (katom->fence) { + sync_fence_put(katom->fence); + katom->fence = NULL; + } +} + +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) +{ + if (katom->fence) { + sync_fence_put(katom->fence); + katom->fence = NULL; + } +} + +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ + if (!katom->fence) + return -ENOENT; + + info->fence = katom->fence; + info->status = kbase_fence_get_status(katom->fence); + strlcpy(info->name, katom->fence->name, sizeof(info->name)); + + return 0; +} + +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ + if (!katom->fence) + return -ENOENT; + + info->fence = katom->fence; + info->status = kbase_fence_get_status(katom->fence); + strlcpy(info->name, katom->fence->name, sizeof(info->name)); + + return 0; +} + +#ifdef CONFIG_MALI_FENCE_DEBUG +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) +{ + /* Dump out the full state of all the Android sync fences. + * The function sync_dump() isn't exported to modules, so force + * sync_fence_wait() to time out to trigger sync_dump(). + */ + if (katom->fence) + sync_fence_wait(katom->fence, 1); +} +#endif diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync_common.c b/drivers/gpu/arm/midgard/mali_kbase_sync_common.c new file mode 100755 index 000000000000..457def296684 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_sync_common.c @@ -0,0 +1,43 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * @file mali_kbase_sync_common.c + * + * Common code for our explicit fence functionality + */ + +#include +#include "mali_kbase.h" + +void kbase_sync_fence_wait_worker(struct work_struct *data) +{ + struct kbase_jd_atom *katom; + + katom = container_of(data, struct kbase_jd_atom, work); + kbase_soft_event_wait_callback(katom); +} + +const char *kbase_sync_status_string(int status) +{ + if (status == 0) + return "signaled"; + else if (status > 0) + return "active"; + else + return "error"; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync_file.c b/drivers/gpu/arm/midgard/mali_kbase_sync_file.c new file mode 100755 index 000000000000..60b5d74db33e --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_sync_file.c @@ -0,0 +1,359 @@ +/* + * + * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* + * Code for supporting explicit Linux fences (CONFIG_SYNC_FILE) + * Introduced in kernel 4.9. + * Android explicit fences (CONFIG_SYNC) can be used for older kernels + * (see mali_kbase_sync_android.c) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mali_kbase_fence_defs.h" +#include "mali_kbase_sync.h" +#include "mali_kbase_fence.h" +#include "mali_kbase.h" + +static const struct file_operations stream_fops = { + .owner = THIS_MODULE +}; + +int kbase_sync_fence_stream_create(const char *name, int *const out_fd) +{ + if (!out_fd) + return -EINVAL; + + *out_fd = anon_inode_getfd(name, &stream_fops, NULL, + O_RDONLY | O_CLOEXEC); + if (*out_fd < 0) + return -EINVAL; + + return 0; +} + +int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + struct sync_file *sync_file; + int fd; + + fence = kbase_fence_out_new(katom); + if (!fence) + return -ENOMEM; + +#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) + /* Take an extra reference to the fence on behalf of the sync_file. + * This is only needed on older kernels where sync_file_create() + * does not take its own reference. This was changed in v4.9.68, + * where sync_file_create() now takes its own reference. + */ + dma_fence_get(fence); +#endif + + /* create a sync_file fd representing the fence */ + sync_file = sync_file_create(fence); + if (!sync_file) { + dma_fence_put(fence); + kbase_fence_out_remove(katom); + return -ENOMEM; + } + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + fput(sync_file->file); + kbase_fence_out_remove(katom); + return fd; + } + + fd_install(fd, sync_file->file); + + return fd; +} + +int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = sync_file_get_fence(fd); +#else + struct dma_fence *fence = sync_file_get_fence(fd); +#endif + + if (!fence) + return -ENOENT; + + kbase_fence_fence_in_set(katom, fence); + + return 0; +} + +int kbase_sync_fence_validate(int fd) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence = sync_file_get_fence(fd); +#else + struct dma_fence *fence = sync_file_get_fence(fd); +#endif + + if (!fence) + return -EINVAL; + + dma_fence_put(fence); + + return 0; /* valid */ +} + +enum base_jd_event_code +kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) +{ + int res; + + if (!kbase_fence_out_is_ours(katom)) { + /* Not our fence */ + return BASE_JD_EVENT_JOB_CANCELLED; + } + + res = kbase_fence_out_signal(katom, result); + if (unlikely(res < 0)) { + dev_warn(katom->kctx->kbdev->dev, + "fence_signal() failed with %d\n", res); + } + + kbase_sync_fence_out_remove(katom); + + return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +static void kbase_fence_wait_callback(struct fence *fence, + struct fence_cb *cb) +#else +static void kbase_fence_wait_callback(struct dma_fence *fence, + struct dma_fence_cb *cb) +#endif +{ + struct kbase_fence_cb *kcb = container_of(cb, + struct kbase_fence_cb, + fence_cb); + struct kbase_jd_atom *katom = kcb->katom; + struct kbase_context *kctx = katom->kctx; + + /* Cancel atom if fence is erroneous */ +#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ + (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ + KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) + if (dma_fence_is_signaled(kcb->fence) && kcb->fence->error) +#else + if (dma_fence_is_signaled(kcb->fence) && kcb->fence->status < 0) +#endif + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + if (kbase_fence_dep_count_dec_and_test(katom)) { + /* We take responsibility of handling this */ + kbase_fence_dep_count_set(katom, -1); + + /* To prevent a potential deadlock we schedule the work onto the + * job_done_wq workqueue + * + * The issue is that we may signal the timeline while holding + * kctx->jctx.lock and the callbacks are run synchronously from + * sync_timeline_signal. So we simply defer the work. + */ + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(kctx->jctx.job_done_wq, &katom->work); + } +} + +int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) +{ + int err; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_in_get(katom); + if (!fence) + return 0; /* no input fence to wait for, good to go! */ + + kbase_fence_dep_count_set(katom, 1); + + err = kbase_fence_add_callback(katom, fence, kbase_fence_wait_callback); + + kbase_fence_put(fence); + + if (likely(!err)) { + /* Test if the callbacks are already triggered */ + if (kbase_fence_dep_count_dec_and_test(katom)) { + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + return 0; /* Already signaled, good to go right now */ + } + + /* Callback installed, so we just need to wait for it... */ + } else { + /* Failure */ + kbase_fence_free_callbacks(katom); + kbase_fence_dep_count_set(katom, -1); + + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + /* We should cause the dependent jobs in the bag to be failed, + * to do this we schedule the work queue to complete this job */ + + INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); + queue_work(katom->kctx->jctx.job_done_wq, &katom->work); + } + + return 1; /* completion to be done later by callback/worker */ +} + +void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) +{ + if (!kbase_fence_free_callbacks(katom)) { + /* The wait wasn't cancelled - + * leave the cleanup for kbase_fence_wait_callback */ + return; + } + + /* Take responsibility of completion */ + kbase_fence_dep_count_set(katom, -1); + + /* Wait was cancelled - zap the atoms */ + katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + kbasep_remove_waiting_soft_job(katom); + kbase_finish_soft_job(katom); + + if (jd_done_nolock(katom, NULL)) + kbase_js_sched_all(katom->kctx->kbdev); +} + +void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) +{ + kbase_fence_out_remove(katom); +} + +void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) +{ + kbase_fence_free_callbacks(katom); + kbase_fence_in_remove(katom); +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) +static void kbase_sync_fence_info_get(struct fence *fence, + struct kbase_sync_fence_info *info) +#else +static void kbase_sync_fence_info_get(struct dma_fence *fence, + struct kbase_sync_fence_info *info) +#endif +{ + info->fence = fence; + + /* translate into CONFIG_SYNC status: + * < 0 : error + * 0 : active + * 1 : signaled + */ + if (dma_fence_is_signaled(fence)) { +#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ + (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ + KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) + int status = fence->error; +#else + int status = fence->status; +#endif + if (status < 0) + info->status = status; /* signaled with error */ + else + info->status = 1; /* signaled with success */ + } else { + info->status = 0; /* still active (unsignaled) */ + } + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) + scnprintf(info->name, sizeof(info->name), "%u#%u", + fence->context, fence->seqno); +#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)) + scnprintf(info->name, sizeof(info->name), "%llu#%u", + fence->context, fence->seqno); +#else + scnprintf(info->name, sizeof(info->name), "%llu#%llu", + fence->context, fence->seqno); +#endif +} + +int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_in_get(katom); + if (!fence) + return -ENOENT; + + kbase_sync_fence_info_get(fence, info); + + kbase_fence_put(fence); + + return 0; +} + +int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, + struct kbase_sync_fence_info *info) +{ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) + struct fence *fence; +#else + struct dma_fence *fence; +#endif + + fence = kbase_fence_out_get(katom); + if (!fence) + return -ENOENT; + + kbase_sync_fence_info_get(fence, info); + + kbase_fence_put(fence); + + return 0; +} + + +#ifdef CONFIG_MALI_FENCE_DEBUG +void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) +{ + /* Not implemented */ +} +#endif diff --git a/drivers/gpu/arm/midgard/mali_kbase_tlstream.c b/drivers/gpu/arm/midgard/mali_kbase_tlstream.c new file mode 100755 index 000000000000..c8310c45f143 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_tlstream.c @@ -0,0 +1,2572 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/*****************************************************************************/ + +/* The version of swtrace protocol used in timeline stream. */ +#define SWTRACE_VERSION 3 + +/* The maximum expected length of string in tracepoint descriptor. */ +#define STRLEN_MAX 64 /* bytes */ + +/* The number of nanoseconds in a second. */ +#define NSECS_IN_SEC 1000000000ull /* ns */ + +/* The period of autoflush checker execution in milliseconds. */ +#define AUTOFLUSH_INTERVAL 1000 /* ms */ + +/* The maximum size of a single packet used by timeline. */ +#define PACKET_SIZE 4096 /* bytes */ + +/* The number of packets used by one timeline stream. */ +#define PACKET_COUNT 16 + +/* The number of bytes reserved for packet header. + * These value must be defined according to MIPE documentation. */ +#define PACKET_HEADER_SIZE 8 /* bytes */ + +/* The number of bytes reserved for packet sequence number. + * These value must be defined according to MIPE documentation. */ +#define PACKET_NUMBER_SIZE 4 /* bytes */ + +/* Packet header - first word. + * These values must be defined according to MIPE documentation. */ +#define PACKET_STREAMID_POS 0 +#define PACKET_STREAMID_LEN 8 +#define PACKET_RSVD1_POS (PACKET_STREAMID_POS + PACKET_STREAMID_LEN) +#define PACKET_RSVD1_LEN 8 +#define PACKET_TYPE_POS (PACKET_RSVD1_POS + PACKET_RSVD1_LEN) +#define PACKET_TYPE_LEN 3 +#define PACKET_CLASS_POS (PACKET_TYPE_POS + PACKET_TYPE_LEN) +#define PACKET_CLASS_LEN 7 +#define PACKET_FAMILY_POS (PACKET_CLASS_POS + PACKET_CLASS_LEN) +#define PACKET_FAMILY_LEN 6 + +/* Packet header - second word + * These values must be defined according to MIPE documentation. */ +#define PACKET_LENGTH_POS 0 +#define PACKET_LENGTH_LEN 24 +#define PACKET_SEQBIT_POS (PACKET_LENGTH_POS + PACKET_LENGTH_LEN) +#define PACKET_SEQBIT_LEN 1 +#define PACKET_RSVD2_POS (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN) +#define PACKET_RSVD2_LEN 7 + +/* Types of streams generated by timeline. + * Order is significant! Header streams must precede respective body streams. */ +enum tl_stream_type { + TL_STREAM_TYPE_OBJ_HEADER, + TL_STREAM_TYPE_OBJ_SUMMARY, + TL_STREAM_TYPE_OBJ, + TL_STREAM_TYPE_AUX_HEADER, + TL_STREAM_TYPE_AUX, + + TL_STREAM_TYPE_COUNT +}; + +/* Timeline packet family ids. + * Values are significant! Check MIPE documentation. */ +enum tl_packet_family { + TL_PACKET_FAMILY_CTRL = 0, /* control packets */ + TL_PACKET_FAMILY_TL = 1, /* timeline packets */ + + TL_PACKET_FAMILY_COUNT +}; + +/* Packet classes used in timeline streams. + * Values are significant! Check MIPE documentation. */ +enum tl_packet_class { + TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */ + TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */ +}; + +/* Packet types used in timeline streams. + * Values are significant! Check MIPE documentation. */ +enum tl_packet_type { + TL_PACKET_TYPE_HEADER = 0, /* stream's header/directory */ + TL_PACKET_TYPE_BODY = 1, /* stream's body */ + TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */ +}; + +/* Message ids of trace events that are recorded in the timeline stream. */ +enum tl_msg_id_obj { + /* Timeline object events. */ + KBASE_TL_NEW_CTX, + KBASE_TL_NEW_GPU, + KBASE_TL_NEW_LPU, + KBASE_TL_NEW_ATOM, + KBASE_TL_NEW_AS, + KBASE_TL_DEL_CTX, + KBASE_TL_DEL_ATOM, + KBASE_TL_LIFELINK_LPU_GPU, + KBASE_TL_LIFELINK_AS_GPU, + KBASE_TL_RET_CTX_LPU, + KBASE_TL_RET_ATOM_CTX, + KBASE_TL_RET_ATOM_LPU, + KBASE_TL_NRET_CTX_LPU, + KBASE_TL_NRET_ATOM_CTX, + KBASE_TL_NRET_ATOM_LPU, + KBASE_TL_RET_AS_CTX, + KBASE_TL_NRET_AS_CTX, + KBASE_TL_RET_ATOM_AS, + KBASE_TL_NRET_ATOM_AS, + KBASE_TL_DEP_ATOM_ATOM, + KBASE_TL_NDEP_ATOM_ATOM, + KBASE_TL_RDEP_ATOM_ATOM, + KBASE_TL_ATTRIB_ATOM_CONFIG, + KBASE_TL_ATTRIB_ATOM_PRIORITY, + KBASE_TL_ATTRIB_ATOM_STATE, + KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, + KBASE_TL_ATTRIB_ATOM_JIT, + KBASE_TL_ATTRIB_AS_CONFIG, + KBASE_TL_EVENT_LPU_SOFTSTOP, + KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, + KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, + + /* Job dump specific events. */ + KBASE_JD_GPU_SOFT_RESET +}; + +/* Message ids of trace events that are recorded in the auxiliary stream. */ +enum tl_msg_id_aux { + KBASE_AUX_PM_STATE, + KBASE_AUX_PAGEFAULT, + KBASE_AUX_PAGESALLOC, + KBASE_AUX_DEVFREQ_TARGET, + KBASE_AUX_PROTECTED_ENTER_START, + KBASE_AUX_PROTECTED_ENTER_END, + KBASE_AUX_PROTECTED_LEAVE_START, + KBASE_AUX_PROTECTED_LEAVE_END +}; + +/*****************************************************************************/ + +/** + * struct tl_stream - timeline stream structure + * @lock: message order lock + * @buffer: array of buffers + * @wbi: write buffer index + * @rbi: read buffer index + * @numbered: if non-zero stream's packets are sequentially numbered + * @autoflush_counter: counter tracking stream's autoflush state + * + * This structure holds information needed to construct proper packets in the + * timeline stream. Each message in sequence must bear timestamp that is greater + * to one in previous message in the same stream. For this reason lock is held + * throughout the process of message creation. Each stream contains set of + * buffers. Each buffer will hold one MIPE packet. In case there is no free + * space required to store incoming message the oldest buffer is discarded. + * Each packet in timeline body stream has sequence number embedded (this value + * must increment monotonically and is used by packets receiver to discover + * buffer overflows. + * Autoflush counter is set to negative number when there is no data pending + * for flush and it is set to zero on every update of the buffer. Autoflush + * timer will increment the counter by one on every expiry. In case there will + * be no activity on the buffer during two consecutive timer expiries, stream + * buffer will be flushed. + */ +struct tl_stream { + spinlock_t lock; + + struct { + atomic_t size; /* number of bytes in buffer */ + char data[PACKET_SIZE]; /* buffer's data */ + } buffer[PACKET_COUNT]; + + atomic_t wbi; + atomic_t rbi; + + int numbered; + atomic_t autoflush_counter; +}; + +/** + * struct tp_desc - tracepoint message descriptor structure + * @id: tracepoint ID identifying message in stream + * @id_str: human readable version of tracepoint ID + * @name: tracepoint description + * @arg_types: tracepoint's arguments types declaration + * @arg_names: comma separated list of tracepoint's arguments names + */ +struct tp_desc { + u32 id; + const char *id_str; + const char *name; + const char *arg_types; + const char *arg_names; +}; + +/*****************************************************************************/ + +/* Configuration of timeline streams generated by kernel. + * Kernel emit only streams containing either timeline object events or + * auxiliary events. All streams have stream id value of 1 (as opposed to user + * space streams that have value of 0). */ +static const struct { + enum tl_packet_family pkt_family; + enum tl_packet_class pkt_class; + enum tl_packet_type pkt_type; + unsigned int stream_id; +} tl_stream_cfg[TL_STREAM_TYPE_COUNT] = { + {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_HEADER, 1}, + {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_SUMMARY, 1}, + {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY, 1}, + {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_HEADER, 1}, + {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_BODY, 1} +}; + +/* The timeline streams generated by kernel. */ +static struct tl_stream *tl_stream[TL_STREAM_TYPE_COUNT]; + +/* Autoflush timer. */ +static struct timer_list autoflush_timer; + +/* If non-zero autoflush timer is active. */ +static atomic_t autoflush_timer_active; + +/* Reader lock. Only one reader is allowed to have access to the timeline + * streams at any given time. */ +static DEFINE_MUTEX(tl_reader_lock); + +/* Timeline stream event queue. */ +static DECLARE_WAIT_QUEUE_HEAD(tl_event_queue); + +/* The timeline stream file operations functions. */ +static ssize_t kbasep_tlstream_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos); +static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait); +static int kbasep_tlstream_release(struct inode *inode, struct file *filp); + +/* The timeline stream file operations structure. */ +static const struct file_operations kbasep_tlstream_fops = { + .release = kbasep_tlstream_release, + .read = kbasep_tlstream_read, + .poll = kbasep_tlstream_poll, +}; + +/* Descriptors of timeline messages transmitted in object events stream. */ +static const struct tp_desc tp_desc_obj[] = { + { + KBASE_TL_NEW_CTX, + __stringify(KBASE_TL_NEW_CTX), + "object ctx is created", + "@pII", + "ctx,ctx_nr,tgid" + }, + { + KBASE_TL_NEW_GPU, + __stringify(KBASE_TL_NEW_GPU), + "object gpu is created", + "@pII", + "gpu,gpu_id,core_count" + }, + { + KBASE_TL_NEW_LPU, + __stringify(KBASE_TL_NEW_LPU), + "object lpu is created", + "@pII", + "lpu,lpu_nr,lpu_fn" + }, + { + KBASE_TL_NEW_ATOM, + __stringify(KBASE_TL_NEW_ATOM), + "object atom is created", + "@pI", + "atom,atom_nr" + }, + { + KBASE_TL_NEW_AS, + __stringify(KBASE_TL_NEW_AS), + "address space object is created", + "@pI", + "address_space,as_nr" + }, + { + KBASE_TL_DEL_CTX, + __stringify(KBASE_TL_DEL_CTX), + "context is destroyed", + "@p", + "ctx" + }, + { + KBASE_TL_DEL_ATOM, + __stringify(KBASE_TL_DEL_ATOM), + "atom is destroyed", + "@p", + "atom" + }, + { + KBASE_TL_LIFELINK_LPU_GPU, + __stringify(KBASE_TL_LIFELINK_LPU_GPU), + "lpu is deleted with gpu", + "@pp", + "lpu,gpu" + }, + { + KBASE_TL_LIFELINK_AS_GPU, + __stringify(KBASE_TL_LIFELINK_AS_GPU), + "address space is deleted with gpu", + "@pp", + "address_space,gpu" + }, + { + KBASE_TL_RET_CTX_LPU, + __stringify(KBASE_TL_RET_CTX_LPU), + "context is retained by lpu", + "@pp", + "ctx,lpu" + }, + { + KBASE_TL_RET_ATOM_CTX, + __stringify(KBASE_TL_RET_ATOM_CTX), + "atom is retained by context", + "@pp", + "atom,ctx" + }, + { + KBASE_TL_RET_ATOM_LPU, + __stringify(KBASE_TL_RET_ATOM_LPU), + "atom is retained by lpu", + "@pps", + "atom,lpu,attrib_match_list" + }, + { + KBASE_TL_NRET_CTX_LPU, + __stringify(KBASE_TL_NRET_CTX_LPU), + "context is released by lpu", + "@pp", + "ctx,lpu" + }, + { + KBASE_TL_NRET_ATOM_CTX, + __stringify(KBASE_TL_NRET_ATOM_CTX), + "atom is released by context", + "@pp", + "atom,ctx" + }, + { + KBASE_TL_NRET_ATOM_LPU, + __stringify(KBASE_TL_NRET_ATOM_LPU), + "atom is released by lpu", + "@pp", + "atom,lpu" + }, + { + KBASE_TL_RET_AS_CTX, + __stringify(KBASE_TL_RET_AS_CTX), + "address space is retained by context", + "@pp", + "address_space,ctx" + }, + { + KBASE_TL_NRET_AS_CTX, + __stringify(KBASE_TL_NRET_AS_CTX), + "address space is released by context", + "@pp", + "address_space,ctx" + }, + { + KBASE_TL_RET_ATOM_AS, + __stringify(KBASE_TL_RET_ATOM_AS), + "atom is retained by address space", + "@pp", + "atom,address_space" + }, + { + KBASE_TL_NRET_ATOM_AS, + __stringify(KBASE_TL_NRET_ATOM_AS), + "atom is released by address space", + "@pp", + "atom,address_space" + }, + { + KBASE_TL_DEP_ATOM_ATOM, + __stringify(KBASE_TL_DEP_ATOM_ATOM), + "atom2 depends on atom1", + "@pp", + "atom1,atom2" + }, + { + KBASE_TL_NDEP_ATOM_ATOM, + __stringify(KBASE_TL_NDEP_ATOM_ATOM), + "atom2 no longer depends on atom1", + "@pp", + "atom1,atom2" + }, + { + KBASE_TL_RDEP_ATOM_ATOM, + __stringify(KBASE_TL_RDEP_ATOM_ATOM), + "resolved dependecy of atom2 depending on atom1", + "@pp", + "atom1,atom2" + }, + { + KBASE_TL_ATTRIB_ATOM_CONFIG, + __stringify(KBASE_TL_ATTRIB_ATOM_CONFIG), + "atom job slot attributes", + "@pLLI", + "atom,descriptor,affinity,config" + }, + { + KBASE_TL_ATTRIB_ATOM_PRIORITY, + __stringify(KBASE_TL_ATTRIB_ATOM_PRIORITY), + "atom priority", + "@pI", + "atom,prio" + }, + { + KBASE_TL_ATTRIB_ATOM_STATE, + __stringify(KBASE_TL_ATTRIB_ATOM_STATE), + "atom state", + "@pI", + "atom,state" + }, + { + KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, + __stringify(KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE), + "atom caused priority change", + "@p", + "atom" + }, + { + KBASE_TL_ATTRIB_ATOM_JIT, + __stringify(KBASE_TL_ATTRIB_ATOM_JIT), + "jit done for atom", + "@pLL", + "atom,edit_addr,new_addr" + }, + { + KBASE_TL_ATTRIB_AS_CONFIG, + __stringify(KBASE_TL_ATTRIB_AS_CONFIG), + "address space attributes", + "@pLLL", + "address_space,transtab,memattr,transcfg" + }, + { + KBASE_TL_EVENT_LPU_SOFTSTOP, + __stringify(KBASE_TL_EVENT_LPU_SOFTSTOP), + "softstop event on given lpu", + "@p", + "lpu" + }, + { + KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, + __stringify(KBASE_TL_EVENT_ATOM_SOFTSTOP_EX), + "atom softstopped", + "@p", + "atom" + }, + { + KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, + __stringify(KBASE_TL_EVENT_SOFTSTOP_ISSUE), + "atom softstop issued", + "@p", + "atom" + }, + { + KBASE_JD_GPU_SOFT_RESET, + __stringify(KBASE_JD_GPU_SOFT_RESET), + "gpu soft reset", + "@p", + "gpu" + }, +}; + +/* Descriptors of timeline messages transmitted in auxiliary events stream. */ +static const struct tp_desc tp_desc_aux[] = { + { + KBASE_AUX_PM_STATE, + __stringify(KBASE_AUX_PM_STATE), + "PM state", + "@IL", + "core_type,core_state_bitset" + }, + { + KBASE_AUX_PAGEFAULT, + __stringify(KBASE_AUX_PAGEFAULT), + "Page fault", + "@IL", + "ctx_nr,page_cnt_change" + }, + { + KBASE_AUX_PAGESALLOC, + __stringify(KBASE_AUX_PAGESALLOC), + "Total alloc pages change", + "@IL", + "ctx_nr,page_cnt" + }, + { + KBASE_AUX_DEVFREQ_TARGET, + __stringify(KBASE_AUX_DEVFREQ_TARGET), + "New device frequency target", + "@L", + "target_freq" + }, + { + KBASE_AUX_PROTECTED_ENTER_START, + __stringify(KBASE_AUX_PROTECTED_ENTER_START), + "enter protected mode start", + "@p", + "gpu" + }, + { + KBASE_AUX_PROTECTED_ENTER_END, + __stringify(KBASE_AUX_PROTECTED_ENTER_END), + "enter protected mode end", + "@p", + "gpu" + }, + { + KBASE_AUX_PROTECTED_LEAVE_START, + __stringify(KBASE_AUX_PROTECTED_LEAVE_START), + "leave protected mode start", + "@p", + "gpu" + }, + { + KBASE_AUX_PROTECTED_LEAVE_END, + __stringify(KBASE_AUX_PROTECTED_LEAVE_END), + "leave protected mode end", + "@p", + "gpu" + } +}; + +#if MALI_UNIT_TEST +/* Number of bytes read by user. */ +static atomic_t tlstream_bytes_collected = {0}; + +/* Number of bytes generated by tracepoint messages. */ +static atomic_t tlstream_bytes_generated = {0}; +#endif /* MALI_UNIT_TEST */ + +/*****************************************************************************/ + +/* Indicator of whether the timeline stream file descriptor is used. */ +atomic_t kbase_tlstream_enabled = {0}; + +/*****************************************************************************/ + +/** + * kbasep_tlstream_get_timestamp - return timestamp + * + * Function returns timestamp value based on raw monotonic timer. Value will + * wrap around zero in case of overflow. + * Return: timestamp value + */ +static u64 kbasep_tlstream_get_timestamp(void) +{ + struct timespec64 ts; + u64 timestamp; + + ktime_get_raw_ts64(&ts); + timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; + return timestamp; +} + +/** + * kbasep_tlstream_write_bytes - write data to message buffer + * @buffer: buffer where data will be written + * @pos: position in the buffer where to place data + * @bytes: pointer to buffer holding data + * @len: length of data to be written + * + * Return: updated position in the buffer + */ +static size_t kbasep_tlstream_write_bytes( + char *buffer, + size_t pos, + const void *bytes, + size_t len) +{ + KBASE_DEBUG_ASSERT(buffer); + KBASE_DEBUG_ASSERT(bytes); + + memcpy(&buffer[pos], bytes, len); + + return pos + len; +} + +/** + * kbasep_tlstream_write_string - write string to message buffer + * @buffer: buffer where data will be written + * @pos: position in the buffer where to place data + * @string: pointer to buffer holding the source string + * @max_write_size: number of bytes that can be stored in buffer + * + * Return: updated position in the buffer + */ +static size_t kbasep_tlstream_write_string( + char *buffer, + size_t pos, + const char *string, + size_t max_write_size) +{ + u32 string_len; + + KBASE_DEBUG_ASSERT(buffer); + KBASE_DEBUG_ASSERT(string); + /* Timeline string consists of at least string length and nul + * terminator. */ + KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char)); + max_write_size -= sizeof(string_len); + + string_len = strlcpy( + &buffer[pos + sizeof(string_len)], + string, + max_write_size); + string_len += sizeof(char); + + /* Make sure that the source string fit into the buffer. */ + KBASE_DEBUG_ASSERT(string_len <= max_write_size); + + /* Update string length. */ + memcpy(&buffer[pos], &string_len, sizeof(string_len)); + + return pos + sizeof(string_len) + string_len; +} + +/** + * kbasep_tlstream_write_timestamp - write timestamp to message buffer + * @buffer: buffer where data will be written + * @pos: position in the buffer where to place data + * + * Return: updated position in the buffer + */ +static size_t kbasep_tlstream_write_timestamp(void *buffer, size_t pos) +{ + u64 timestamp = kbasep_tlstream_get_timestamp(); + + return kbasep_tlstream_write_bytes( + buffer, pos, + ×tamp, sizeof(timestamp)); +} + +/** + * kbasep_tlstream_put_bits - put bits in a word + * @word: pointer to the words being modified + * @value: value that shall be written to given position + * @bitpos: position where value shall be written (in bits) + * @bitlen: length of value (in bits) + */ +static void kbasep_tlstream_put_bits( + u32 *word, + u32 value, + unsigned int bitpos, + unsigned int bitlen) +{ + const u32 mask = ((1 << bitlen) - 1) << bitpos; + + KBASE_DEBUG_ASSERT(word); + KBASE_DEBUG_ASSERT((0 != bitlen) && (32 >= bitlen)); + KBASE_DEBUG_ASSERT((bitpos + bitlen) <= 32); + + *word &= ~mask; + *word |= ((value << bitpos) & mask); +} + +/** + * kbasep_tlstream_packet_header_setup - setup the packet header + * @buffer: pointer to the buffer + * @pkt_family: packet's family + * @pkt_type: packet's type + * @pkt_class: packet's class + * @stream_id: stream id + * @numbered: non-zero if this stream is numbered + * + * Function sets up immutable part of packet header in the given buffer. + */ +static void kbasep_tlstream_packet_header_setup( + char *buffer, + enum tl_packet_family pkt_family, + enum tl_packet_class pkt_class, + enum tl_packet_type pkt_type, + unsigned int stream_id, + int numbered) +{ + u32 word0 = 0; + u32 word1 = 0; + + KBASE_DEBUG_ASSERT(buffer); + KBASE_DEBUG_ASSERT(pkt_family == TL_PACKET_FAMILY_TL); + KBASE_DEBUG_ASSERT( + (pkt_type == TL_PACKET_TYPE_HEADER) || + (pkt_type == TL_PACKET_TYPE_SUMMARY) || + (pkt_type == TL_PACKET_TYPE_BODY)); + KBASE_DEBUG_ASSERT( + (pkt_class == TL_PACKET_CLASS_OBJ) || + (pkt_class == TL_PACKET_CLASS_AUX)); + + kbasep_tlstream_put_bits( + &word0, pkt_family, + PACKET_FAMILY_POS, PACKET_FAMILY_LEN); + kbasep_tlstream_put_bits( + &word0, pkt_class, + PACKET_CLASS_POS, PACKET_CLASS_LEN); + kbasep_tlstream_put_bits( + &word0, pkt_type, + PACKET_TYPE_POS, PACKET_TYPE_LEN); + kbasep_tlstream_put_bits( + &word0, stream_id, + PACKET_STREAMID_POS, PACKET_STREAMID_LEN); + + if (numbered) + kbasep_tlstream_put_bits( + &word1, 1, + PACKET_SEQBIT_POS, PACKET_SEQBIT_LEN); + + memcpy(&buffer[0], &word0, sizeof(word0)); + memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); +} + +/** + * kbasep_tlstream_packet_header_update - update the packet header + * @buffer: pointer to the buffer + * @data_size: amount of data carried in this packet + * + * Function updates mutable part of packet header in the given buffer. + * Note that value of data_size must not including size of the header. + */ +static void kbasep_tlstream_packet_header_update( + char *buffer, + size_t data_size) +{ + u32 word0; + u32 word1; + + KBASE_DEBUG_ASSERT(buffer); + CSTD_UNUSED(word0); + + memcpy(&word1, &buffer[sizeof(word0)], sizeof(word1)); + + kbasep_tlstream_put_bits( + &word1, data_size, + PACKET_LENGTH_POS, PACKET_LENGTH_LEN); + + memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); +} + +/** + * kbasep_tlstream_packet_number_update - update the packet number + * @buffer: pointer to the buffer + * @counter: value of packet counter for this packet's stream + * + * Function updates packet number embedded within the packet placed in the + * given buffer. + */ +static void kbasep_tlstream_packet_number_update(char *buffer, u32 counter) +{ + KBASE_DEBUG_ASSERT(buffer); + + memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter)); +} + +/** + * kbasep_timeline_stream_reset - reset stream + * @stream: pointer to the stream structure + * + * Function discards all pending messages and resets packet counters. + */ +static void kbasep_timeline_stream_reset(struct tl_stream *stream) +{ + unsigned int i; + + for (i = 0; i < PACKET_COUNT; i++) { + if (stream->numbered) + atomic_set( + &stream->buffer[i].size, + PACKET_HEADER_SIZE + + PACKET_NUMBER_SIZE); + else + atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE); + } + + atomic_set(&stream->wbi, 0); + atomic_set(&stream->rbi, 0); +} + +/** + * kbasep_timeline_stream_init - initialize timeline stream + * @stream: pointer to the stream structure + * @stream_type: stream type + */ +static void kbasep_timeline_stream_init( + struct tl_stream *stream, + enum tl_stream_type stream_type) +{ + unsigned int i; + + KBASE_DEBUG_ASSERT(stream); + KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); + + spin_lock_init(&stream->lock); + + /* All packets carrying tracepoints shall be numbered. */ + if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type) + stream->numbered = 1; + else + stream->numbered = 0; + + for (i = 0; i < PACKET_COUNT; i++) + kbasep_tlstream_packet_header_setup( + stream->buffer[i].data, + tl_stream_cfg[stream_type].pkt_family, + tl_stream_cfg[stream_type].pkt_class, + tl_stream_cfg[stream_type].pkt_type, + tl_stream_cfg[stream_type].stream_id, + stream->numbered); + + kbasep_timeline_stream_reset(tl_stream[stream_type]); +} + +/** + * kbasep_timeline_stream_term - terminate timeline stream + * @stream: pointer to the stream structure + */ +static void kbasep_timeline_stream_term(struct tl_stream *stream) +{ + KBASE_DEBUG_ASSERT(stream); +} + +/** + * kbasep_tlstream_msgbuf_submit - submit packet to the user space + * @stream: pointer to the stream structure + * @wb_idx_raw: write buffer index + * @wb_size: length of data stored in current buffer + * + * Function updates currently written buffer with packet header. Then write + * index is incremented and buffer is handled to user space. Parameters + * of new buffer are returned using provided arguments. + * + * Return: length of data in new buffer + * + * Warning: User must update the stream structure with returned value. + */ +static size_t kbasep_tlstream_msgbuf_submit( + struct tl_stream *stream, + unsigned int wb_idx_raw, + unsigned int wb_size) +{ + unsigned int rb_idx_raw = atomic_read(&stream->rbi); + unsigned int wb_idx = wb_idx_raw % PACKET_COUNT; + + /* Set stream as flushed. */ + atomic_set(&stream->autoflush_counter, -1); + + kbasep_tlstream_packet_header_update( + stream->buffer[wb_idx].data, + wb_size - PACKET_HEADER_SIZE); + + if (stream->numbered) + kbasep_tlstream_packet_number_update( + stream->buffer[wb_idx].data, + wb_idx_raw); + + /* Increasing write buffer index will expose this packet to the reader. + * As stream->lock is not taken on reader side we must make sure memory + * is updated correctly before this will happen. */ + smp_wmb(); + wb_idx_raw++; + atomic_set(&stream->wbi, wb_idx_raw); + + /* Inform user that packets are ready for reading. */ + wake_up_interruptible(&tl_event_queue); + + /* Detect and mark overflow in this stream. */ + if (PACKET_COUNT == wb_idx_raw - rb_idx_raw) { + /* Reader side depends on this increment to correctly handle + * overflows. The value shall be updated only if it was not + * modified by the reader. The data holding buffer will not be + * updated before stream->lock is released, however size of the + * buffer will. Make sure this increment is globally visible + * before information about selected write buffer size. */ + atomic_cmpxchg(&stream->rbi, rb_idx_raw, rb_idx_raw + 1); + } + + wb_size = PACKET_HEADER_SIZE; + if (stream->numbered) + wb_size += PACKET_NUMBER_SIZE; + + return wb_size; +} + +/** + * kbasep_tlstream_msgbuf_acquire - lock selected stream and reserves buffer + * @stream_type: type of the stream that shall be locked + * @msg_size: message size + * @flags: pointer to store flags passed back on stream release + * + * Function will lock the stream and reserve the number of bytes requested + * in msg_size for the user. + * + * Return: pointer to the buffer where message can be stored + * + * Warning: Stream must be released with kbasep_tlstream_msgbuf_release(). + * Only atomic operations are allowed while stream is locked + * (i.e. do not use any operation that may sleep). + */ +static char *kbasep_tlstream_msgbuf_acquire( + enum tl_stream_type stream_type, + size_t msg_size, + unsigned long *flags) __acquires(&stream->lock) +{ + struct tl_stream *stream; + unsigned int wb_idx_raw; + unsigned int wb_idx; + size_t wb_size; + + KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); + KBASE_DEBUG_ASSERT( + PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >= + msg_size); + + stream = tl_stream[stream_type]; + + spin_lock_irqsave(&stream->lock, *flags); + + wb_idx_raw = atomic_read(&stream->wbi); + wb_idx = wb_idx_raw % PACKET_COUNT; + wb_size = atomic_read(&stream->buffer[wb_idx].size); + + /* Select next buffer if data will not fit into current one. */ + if (PACKET_SIZE < wb_size + msg_size) { + wb_size = kbasep_tlstream_msgbuf_submit( + stream, wb_idx_raw, wb_size); + wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; + } + + /* Reserve space in selected buffer. */ + atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size); + +#if MALI_UNIT_TEST + atomic_add(msg_size, &tlstream_bytes_generated); +#endif /* MALI_UNIT_TEST */ + + return &stream->buffer[wb_idx].data[wb_size]; +} + +/** + * kbasep_tlstream_msgbuf_release - unlock selected stream + * @stream_type: type of the stream that shall be locked + * @flags: value obtained during stream acquire + * + * Function releases stream that has been previously locked with a call to + * kbasep_tlstream_msgbuf_acquire(). + */ +static void kbasep_tlstream_msgbuf_release( + enum tl_stream_type stream_type, + unsigned long flags) __releases(&stream->lock) +{ + struct tl_stream *stream; + + KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); + + stream = tl_stream[stream_type]; + + /* Mark stream as containing unflushed data. */ + atomic_set(&stream->autoflush_counter, 0); + + spin_unlock_irqrestore(&stream->lock, flags); +} + +/*****************************************************************************/ + +/** + * kbasep_tlstream_flush_stream - flush stream + * @stype: type of stream to be flushed + * + * Flush pending data in timeline stream. + */ +static void kbasep_tlstream_flush_stream(enum tl_stream_type stype) +{ + struct tl_stream *stream = tl_stream[stype]; + unsigned long flags; + unsigned int wb_idx_raw; + unsigned int wb_idx; + size_t wb_size; + size_t min_size = PACKET_HEADER_SIZE; + + if (stream->numbered) + min_size += PACKET_NUMBER_SIZE; + + spin_lock_irqsave(&stream->lock, flags); + + wb_idx_raw = atomic_read(&stream->wbi); + wb_idx = wb_idx_raw % PACKET_COUNT; + wb_size = atomic_read(&stream->buffer[wb_idx].size); + + if (wb_size > min_size) { + wb_size = kbasep_tlstream_msgbuf_submit( + stream, wb_idx_raw, wb_size); + wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; + atomic_set(&stream->buffer[wb_idx].size, wb_size); + } + spin_unlock_irqrestore(&stream->lock, flags); +} + +/** + * kbasep_tlstream_autoflush_timer_callback - autoflush timer callback + * @data: unused + * + * Timer is executed periodically to check if any of the stream contains + * buffer ready to be submitted to user space. + */ +static void kbasep_tlstream_autoflush_timer_callback(struct timer_list *t) +{ + enum tl_stream_type stype; + int rcode; + + CSTD_UNUSED(t); + + for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) { + struct tl_stream *stream = tl_stream[stype]; + unsigned long flags; + unsigned int wb_idx_raw; + unsigned int wb_idx; + size_t wb_size; + size_t min_size = PACKET_HEADER_SIZE; + + int af_cnt = atomic_read(&stream->autoflush_counter); + + /* Check if stream contain unflushed data. */ + if (0 > af_cnt) + continue; + + /* Check if stream should be flushed now. */ + if (af_cnt != atomic_cmpxchg( + &stream->autoflush_counter, + af_cnt, + af_cnt + 1)) + continue; + if (!af_cnt) + continue; + + /* Autoflush this stream. */ + if (stream->numbered) + min_size += PACKET_NUMBER_SIZE; + + spin_lock_irqsave(&stream->lock, flags); + + wb_idx_raw = atomic_read(&stream->wbi); + wb_idx = wb_idx_raw % PACKET_COUNT; + wb_size = atomic_read(&stream->buffer[wb_idx].size); + + if (wb_size > min_size) { + wb_size = kbasep_tlstream_msgbuf_submit( + stream, wb_idx_raw, wb_size); + wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; + atomic_set(&stream->buffer[wb_idx].size, + wb_size); + } + spin_unlock_irqrestore(&stream->lock, flags); + } + + if (atomic_read(&autoflush_timer_active)) + rcode = mod_timer( + &autoflush_timer, + jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); + CSTD_UNUSED(rcode); +} + +/** + * kbasep_tlstream_packet_pending - check timeline streams for pending packets + * @stype: pointer to variable where stream type will be placed + * @rb_idx_raw: pointer to variable where read buffer index will be placed + * + * Function checks all streams for pending packets. It will stop as soon as + * packet ready to be submitted to user space is detected. Variables under + * pointers, passed as the parameters to this function will be updated with + * values pointing to right stream and buffer. + * + * Return: non-zero if any of timeline streams has at last one packet ready + */ +static int kbasep_tlstream_packet_pending( + enum tl_stream_type *stype, + unsigned int *rb_idx_raw) +{ + int pending = 0; + + KBASE_DEBUG_ASSERT(stype); + KBASE_DEBUG_ASSERT(rb_idx_raw); + + for ( + *stype = 0; + (*stype < TL_STREAM_TYPE_COUNT) && !pending; + (*stype)++) { + if (NULL != tl_stream[*stype]) { + *rb_idx_raw = atomic_read(&tl_stream[*stype]->rbi); + /* Read buffer index may be updated by writer in case of + * overflow. Read and write buffer indexes must be + * loaded in correct order. */ + smp_rmb(); + if (atomic_read(&tl_stream[*stype]->wbi) != *rb_idx_raw) + pending = 1; + } + } + (*stype)--; + + return pending; +} + +/** + * kbasep_tlstream_read - copy data from streams to buffer provided by user + * @filp: pointer to file structure (unused) + * @buffer: pointer to the buffer provided by user + * @size: maximum amount of data that can be stored in the buffer + * @f_pos: pointer to file offset (unused) + * + * Return: number of bytes stored in the buffer + */ +static ssize_t kbasep_tlstream_read( + struct file *filp, + char __user *buffer, + size_t size, + loff_t *f_pos) +{ + ssize_t copy_len = 0; + + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(f_pos); + + if (!buffer) + return -EINVAL; + + if ((0 > *f_pos) || (PACKET_SIZE > size)) + return -EINVAL; + + mutex_lock(&tl_reader_lock); + + while (copy_len < size) { + enum tl_stream_type stype; + unsigned int rb_idx_raw = 0; + unsigned int rb_idx; + size_t rb_size; + + /* If we don't have any data yet, wait for packet to be + * submitted. If we already read some packets and there is no + * packet pending return back to user. */ + if (0 < copy_len) { + if (!kbasep_tlstream_packet_pending( + &stype, + &rb_idx_raw)) + break; + } else { + if (wait_event_interruptible( + tl_event_queue, + kbasep_tlstream_packet_pending( + &stype, + &rb_idx_raw))) { + copy_len = -ERESTARTSYS; + break; + } + } + + /* Check if this packet fits into the user buffer. + * If so copy its content. */ + rb_idx = rb_idx_raw % PACKET_COUNT; + rb_size = atomic_read(&tl_stream[stype]->buffer[rb_idx].size); + if (rb_size > size - copy_len) + break; + if (copy_to_user( + &buffer[copy_len], + tl_stream[stype]->buffer[rb_idx].data, + rb_size)) { + copy_len = -EFAULT; + break; + } + + /* If the rbi still points to the packet we just processed + * then there was no overflow so we add the copied size to + * copy_len and move rbi on to the next packet + */ + smp_rmb(); + if (atomic_read(&tl_stream[stype]->rbi) == rb_idx_raw) { + copy_len += rb_size; + atomic_inc(&tl_stream[stype]->rbi); + +#if MALI_UNIT_TEST + atomic_add(rb_size, &tlstream_bytes_collected); +#endif /* MALI_UNIT_TEST */ + } + } + + mutex_unlock(&tl_reader_lock); + + return copy_len; +} + +/** + * kbasep_tlstream_poll - poll timeline stream for packets + * @filp: pointer to file structure + * @wait: pointer to poll table + * Return: POLLIN if data can be read without blocking, otherwise zero + */ +static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait) +{ + enum tl_stream_type stream_type; + unsigned int rb_idx; + + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(wait); + + poll_wait(filp, &tl_event_queue, wait); + if (kbasep_tlstream_packet_pending(&stream_type, &rb_idx)) + return POLLIN; + return 0; +} + +/** + * kbasep_tlstream_release - release timeline stream descriptor + * @inode: pointer to inode structure + * @filp: pointer to file structure + * + * Return always return zero + */ +static int kbasep_tlstream_release(struct inode *inode, struct file *filp) +{ + KBASE_DEBUG_ASSERT(inode); + KBASE_DEBUG_ASSERT(filp); + CSTD_UNUSED(inode); + CSTD_UNUSED(filp); + + /* Stop autoflush timer before releasing access to streams. */ + atomic_set(&autoflush_timer_active, 0); + del_timer_sync(&autoflush_timer); + + atomic_set(&kbase_tlstream_enabled, 0); + return 0; +} + +/** + * kbasep_tlstream_timeline_header - prepare timeline header stream packet + * @stream_type: type of the stream that will carry header data + * @tp_desc: pointer to array with tracepoint descriptors + * @tp_count: number of descriptors in the given array + * + * Functions fills in information about tracepoints stored in body stream + * associated with this header stream. + */ +static void kbasep_tlstream_timeline_header( + enum tl_stream_type stream_type, + const struct tp_desc *tp_desc, + u32 tp_count) +{ + const u8 tv = SWTRACE_VERSION; /* protocol version */ + const u8 ps = sizeof(void *); /* pointer size */ + size_t msg_size = sizeof(tv) + sizeof(ps) + sizeof(tp_count); + char *buffer; + size_t pos = 0; + unsigned long flags; + unsigned int i; + + KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); + KBASE_DEBUG_ASSERT(tp_desc); + + /* Calculate the size of the timeline message. */ + for (i = 0; i < tp_count; i++) { + msg_size += sizeof(tp_desc[i].id); + msg_size += + strnlen(tp_desc[i].id_str, STRLEN_MAX) + + sizeof(char) + sizeof(u32); + msg_size += + strnlen(tp_desc[i].name, STRLEN_MAX) + + sizeof(char) + sizeof(u32); + msg_size += + strnlen(tp_desc[i].arg_types, STRLEN_MAX) + + sizeof(char) + sizeof(u32); + msg_size += + strnlen(tp_desc[i].arg_names, STRLEN_MAX) + + sizeof(char) + sizeof(u32); + } + + KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE >= msg_size); + + buffer = kbasep_tlstream_msgbuf_acquire(stream_type, msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &tv, sizeof(tv)); + pos = kbasep_tlstream_write_bytes(buffer, pos, &ps, sizeof(ps)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &tp_count, sizeof(tp_count)); + + for (i = 0; i < tp_count; i++) { + pos = kbasep_tlstream_write_bytes( + buffer, pos, + &tp_desc[i].id, sizeof(tp_desc[i].id)); + pos = kbasep_tlstream_write_string( + buffer, pos, + tp_desc[i].id_str, msg_size - pos); + pos = kbasep_tlstream_write_string( + buffer, pos, + tp_desc[i].name, msg_size - pos); + pos = kbasep_tlstream_write_string( + buffer, pos, + tp_desc[i].arg_types, msg_size - pos); + pos = kbasep_tlstream_write_string( + buffer, pos, + tp_desc[i].arg_names, msg_size - pos); + } + + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(stream_type, flags); + + /* We don't expect any more data to be read in this stream. + * As header stream must be read before its associated body stream, + * make this packet visible to the user straightaway. */ + kbasep_tlstream_flush_stream(stream_type); +} + +/*****************************************************************************/ + +int kbase_tlstream_init(void) +{ + enum tl_stream_type i; + + /* Prepare stream structures. */ + for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) { + tl_stream[i] = kmalloc(sizeof(**tl_stream), GFP_KERNEL); + if (!tl_stream[i]) + break; + kbasep_timeline_stream_init(tl_stream[i], i); + } + if (TL_STREAM_TYPE_COUNT > i) { + for (; i > 0; i--) { + kbasep_timeline_stream_term(tl_stream[i - 1]); + kfree(tl_stream[i - 1]); + } + return -ENOMEM; + } + + /* Initialize autoflush timer. */ + atomic_set(&autoflush_timer_active, 0); + timer_setup(&autoflush_timer, + kbasep_tlstream_autoflush_timer_callback, + 0); + + return 0; +} + +void kbase_tlstream_term(void) +{ + enum tl_stream_type i; + + for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) { + kbasep_timeline_stream_term(tl_stream[i]); + kfree(tl_stream[i]); + } +} + +static void kbase_create_timeline_objects(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + unsigned int lpu_id; + unsigned int as_nr; + struct kbasep_kctx_list_element *element; + + /* Create LPU objects. */ + for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { + u32 *lpu = + &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; + KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, lpu_id, *lpu); + } + + /* Create Address Space objects. */ + for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) + KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(&kbdev->as[as_nr], as_nr); + + /* Create GPU object and make it retain all LPUs and address spaces. */ + KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU( + kbdev, + kbdev->gpu_props.props.raw_props.gpu_id, + kbdev->gpu_props.num_cores); + + for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { + void *lpu = + &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; + KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, kbdev); + } + for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) + KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU( + &kbdev->as[as_nr], + kbdev); + + /* Create object for each known context. */ + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry(element, &kbdev->kctx_list, link) { + KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX( + element->kctx, + (u32)(element->kctx->id), + (u32)(element->kctx->tgid)); + } + /* Before releasing the lock, reset body stream buffers. + * This will prevent context creation message to be directed to both + * summary and body stream. + */ + kbase_tlstream_reset_body_streams(); + mutex_unlock(&kbdev->kctx_list_lock); + /* Static object are placed into summary packet that needs to be + * transmitted first. Flush all streams to make it available to + * user space. + */ + kbase_tlstream_flush_streams(); +} + +int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags) +{ + int ret; + u32 tlstream_enabled = TLSTREAM_ENABLED | flags; + + if (0 == atomic_cmpxchg(&kbase_tlstream_enabled, 0, tlstream_enabled)) { + int rcode; + + ret = anon_inode_getfd( + "[mali_tlstream]", + &kbasep_tlstream_fops, + kctx, + O_RDONLY | O_CLOEXEC); + if (ret < 0) { + atomic_set(&kbase_tlstream_enabled, 0); + return ret; + } + + /* Reset and initialize header streams. */ + kbasep_timeline_stream_reset( + tl_stream[TL_STREAM_TYPE_OBJ_HEADER]); + kbasep_timeline_stream_reset( + tl_stream[TL_STREAM_TYPE_OBJ_SUMMARY]); + kbasep_timeline_stream_reset( + tl_stream[TL_STREAM_TYPE_AUX_HEADER]); + kbasep_tlstream_timeline_header( + TL_STREAM_TYPE_OBJ_HEADER, + tp_desc_obj, + ARRAY_SIZE(tp_desc_obj)); + kbasep_tlstream_timeline_header( + TL_STREAM_TYPE_AUX_HEADER, + tp_desc_aux, + ARRAY_SIZE(tp_desc_aux)); + + /* Start autoflush timer. */ + atomic_set(&autoflush_timer_active, 1); + rcode = mod_timer( + &autoflush_timer, + jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); + CSTD_UNUSED(rcode); + + /* If job dumping is enabled, readjust the software event's + * timeout as the default value of 3 seconds is often + * insufficient. */ + if (flags & BASE_TLSTREAM_JOB_DUMPING_ENABLED) { + dev_info(kctx->kbdev->dev, + "Job dumping is enabled, readjusting the software event's timeout\n"); + atomic_set(&kctx->kbdev->js_data.soft_job_timeout_ms, + 1800000); + } + + /* Summary stream was cleared during acquire. + * Create static timeline objects that will be + * read by client. + */ + kbase_create_timeline_objects(kctx); + + } else { + ret = -EBUSY; + } + + return ret; +} + +void kbase_tlstream_flush_streams(void) +{ + enum tl_stream_type stype; + + for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) + kbasep_tlstream_flush_stream(stype); +} + +void kbase_tlstream_reset_body_streams(void) +{ + kbasep_timeline_stream_reset( + tl_stream[TL_STREAM_TYPE_OBJ]); + kbasep_timeline_stream_reset( + tl_stream[TL_STREAM_TYPE_AUX]); +} + +#if MALI_UNIT_TEST +void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated) +{ + KBASE_DEBUG_ASSERT(bytes_collected); + KBASE_DEBUG_ASSERT(bytes_generated); + *bytes_collected = atomic_read(&tlstream_bytes_collected); + *bytes_generated = atomic_read(&tlstream_bytes_generated); +} +#endif /* MALI_UNIT_TEST */ + +/*****************************************************************************/ + +void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid) +{ + const u32 msg_id = KBASE_TL_NEW_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) + + sizeof(tgid); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &nr, sizeof(nr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &tgid, sizeof(tgid)); + + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count) +{ + const u32 msg_id = KBASE_TL_NEW_GPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(id) + + sizeof(core_count); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &id, sizeof(id)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &core_count, sizeof(core_count)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn) +{ + const u32 msg_id = KBASE_TL_NEW_LPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(nr) + + sizeof(fn); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &nr, sizeof(nr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &fn, sizeof(fn)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu) +{ + const u32 msg_id = KBASE_TL_LIFELINK_LPU_GPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr) +{ + const u32 msg_id = KBASE_TL_NEW_AS; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(nr); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &nr, sizeof(nr)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu) +{ + const u32 msg_id = KBASE_TL_LIFELINK_AS_GPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ_SUMMARY, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); +} + +/*****************************************************************************/ + +void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid) +{ + const u32 msg_id = KBASE_TL_NEW_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) + + sizeof(tgid); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &nr, sizeof(nr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &tgid, sizeof(tgid)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_new_atom(void *atom, u32 nr) +{ + const u32 msg_id = KBASE_TL_NEW_ATOM; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + sizeof(atom) + + sizeof(nr); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &nr, sizeof(nr)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_del_ctx(void *context) +{ + const u32 msg_id = KBASE_TL_DEL_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(context); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_del_atom(void *atom) +{ + const u32 msg_id = KBASE_TL_DEL_ATOM; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu) +{ + const u32 msg_id = KBASE_TL_RET_CTX_LPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context) +{ + const u32 msg_id = KBASE_TL_RET_ATOM_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ret_atom_lpu( + void *atom, void *lpu, const char *attrib_match_list) +{ + const u32 msg_id = KBASE_TL_RET_ATOM_LPU; + const size_t msg_s0 = sizeof(u32) + sizeof(char) + + strnlen(attrib_match_list, STRLEN_MAX); + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + + sizeof(atom) + sizeof(lpu) + msg_s0; + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + pos = kbasep_tlstream_write_string( + buffer, pos, attrib_match_list, msg_s0); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu) +{ + const u32 msg_id = KBASE_TL_NRET_CTX_LPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context) +{ + const u32 msg_id = KBASE_TL_NRET_ATOM_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &context, sizeof(context)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2) +{ + const u32 msg_id = KBASE_TL_DEP_ATOM_ATOM; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom1, sizeof(atom1)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom2, sizeof(atom2)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2) +{ + const u32 msg_id = KBASE_TL_NDEP_ATOM_ATOM; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom1, sizeof(atom1)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom2, sizeof(atom2)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2) +{ + const u32 msg_id = KBASE_TL_RDEP_ATOM_ATOM; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom1, sizeof(atom1)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom2, sizeof(atom2)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu) +{ + const u32 msg_id = KBASE_TL_NRET_ATOM_LPU; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx) +{ + const u32 msg_id = KBASE_TL_RET_AS_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &ctx, sizeof(ctx)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx) +{ + const u32 msg_id = KBASE_TL_NRET_AS_CTX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &ctx, sizeof(ctx)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as) +{ + const u32 msg_id = KBASE_TL_RET_ATOM_AS; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as) +{ + const u32 msg_id = KBASE_TL_NRET_ATOM_AS; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_atom_config( + void *atom, u64 jd, u64 affinity, u32 config) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_CONFIG; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + + sizeof(jd) + sizeof(affinity) + sizeof(config); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &jd, sizeof(jd)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &affinity, sizeof(affinity)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &config, sizeof(config)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(prio); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &prio, sizeof(prio)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_STATE; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(state); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &state, sizeof(state)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_atom_jit( + void *atom, u64 edit_addr, u64 new_addr) +{ + const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom) + + sizeof(edit_addr) + sizeof(new_addr); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &edit_addr, sizeof(edit_addr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &new_addr, sizeof(new_addr)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_attrib_as_config( + void *as, u64 transtab, u64 memattr, u64 transcfg) +{ + const u32 msg_id = KBASE_TL_ATTRIB_AS_CONFIG; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(as) + + sizeof(transtab) + sizeof(memattr) + sizeof(transcfg); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &as, sizeof(as)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &transtab, sizeof(transtab)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &memattr, sizeof(memattr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &transcfg, sizeof(transcfg)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_event_lpu_softstop(void *lpu) +{ + const u32 msg_id = KBASE_TL_EVENT_LPU_SOFTSTOP; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(lpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &lpu, sizeof(lpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom) +{ + const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_EX; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_tl_event_atom_softstop_issue(void *atom) +{ + const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(atom); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &atom, sizeof(atom)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +void __kbase_tlstream_jd_gpu_soft_reset(void *gpu) +{ + const u32 msg_id = KBASE_JD_GPU_SOFT_RESET; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_OBJ, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); +} + +/*****************************************************************************/ + +void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state) +{ + const u32 msg_id = KBASE_AUX_PM_STATE; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(core_type) + + sizeof(state); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &core_type, sizeof(core_type)); + pos = kbasep_tlstream_write_bytes(buffer, pos, &state, sizeof(state)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} + +void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change) +{ + const u32 msg_id = KBASE_AUX_PAGEFAULT; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) + + sizeof(page_count_change); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, + &page_count_change, sizeof(page_count_change)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} + +void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count) +{ + const u32 msg_id = KBASE_AUX_PAGESALLOC; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) + + sizeof(page_count); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr)); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &page_count, sizeof(page_count)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} + +void __kbase_tlstream_aux_devfreq_target(u64 target_freq) +{ + const u32 msg_id = KBASE_AUX_DEVFREQ_TARGET; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(target_freq); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &target_freq, sizeof(target_freq)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} + +void __kbase_tlstream_aux_protected_enter_start(void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_START; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} +void __kbase_tlstream_aux_protected_enter_end(void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_END; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} + +void __kbase_tlstream_aux_protected_leave_start(void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_START; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} +void __kbase_tlstream_aux_protected_leave_end(void *gpu) +{ + const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_END; + const size_t msg_size = + sizeof(msg_id) + sizeof(u64) + sizeof(gpu); + unsigned long flags; + char *buffer; + size_t pos = 0; + + buffer = kbasep_tlstream_msgbuf_acquire( + TL_STREAM_TYPE_AUX, + msg_size, &flags); + KBASE_DEBUG_ASSERT(buffer); + + pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_tlstream_write_timestamp(buffer, pos); + pos = kbasep_tlstream_write_bytes( + buffer, pos, &gpu, sizeof(gpu)); + KBASE_DEBUG_ASSERT(msg_size == pos); + + kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_tlstream.h b/drivers/gpu/arm/midgard/mali_kbase_tlstream.h new file mode 100755 index 000000000000..c0a1117d5f25 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_tlstream.h @@ -0,0 +1,623 @@ +/* + * + * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#if !defined(_KBASE_TLSTREAM_H) +#define _KBASE_TLSTREAM_H + +#include + +/*****************************************************************************/ + +/** + * kbase_tlstream_init - initialize timeline infrastructure in kernel + * Return: zero on success, negative number on error + */ +int kbase_tlstream_init(void); + +/** + * kbase_tlstream_term - terminate timeline infrastructure in kernel + * + * Timeline need have to been previously enabled with kbase_tlstream_init(). + */ +void kbase_tlstream_term(void); + +/** + * kbase_tlstream_acquire - acquire timeline stream file descriptor + * @kctx: kernel common context + * @flags: timeline stream flags + * + * This descriptor is meant to be used by userspace timeline to gain access to + * kernel timeline stream. This stream is later broadcasted by user space to the + * timeline client. + * Only one entity can own the descriptor at any given time. Descriptor shall be + * closed if unused. If descriptor cannot be obtained (i.e. when it is already + * being used) return will be a negative value. + * + * Return: file descriptor on success, negative number on error + */ +int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags); + +/** + * kbase_tlstream_flush_streams - flush timeline streams. + * + * Function will flush pending data in all timeline streams. + */ +void kbase_tlstream_flush_streams(void); + +/** + * kbase_tlstream_reset_body_streams - reset timeline body streams. + * + * Function will discard pending data in all timeline body streams. + */ +void kbase_tlstream_reset_body_streams(void); + +#if MALI_UNIT_TEST +/** + * kbase_tlstream_test - start timeline stream data generator + * @tpw_count: number of trace point writers in each context + * @msg_delay: time delay in milliseconds between trace points written by one + * writer + * @msg_count: number of trace points written by one writer + * @aux_msg: if non-zero aux messages will be included + * + * This test starts a requested number of asynchronous writers in both IRQ and + * thread context. Each writer will generate required number of test + * tracepoints (tracepoints with embedded information about writer that + * should be verified by user space reader). Tracepoints will be emitted in + * all timeline body streams. If aux_msg is non-zero writer will also + * generate not testable tracepoints (tracepoints without information about + * writer). These tracepoints are used to check correctness of remaining + * timeline message generating functions. Writer will wait requested time + * between generating another set of messages. This call blocks until all + * writers finish. + */ +void kbase_tlstream_test( + unsigned int tpw_count, + unsigned int msg_delay, + unsigned int msg_count, + int aux_msg); + +/** + * kbase_tlstream_stats - read timeline stream statistics + * @bytes_collected: will hold number of bytes read by the user + * @bytes_generated: will hold number of bytes generated by trace points + */ +void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated); +#endif /* MALI_UNIT_TEST */ + +/*****************************************************************************/ + +#define TL_ATOM_STATE_IDLE 0 +#define TL_ATOM_STATE_READY 1 +#define TL_ATOM_STATE_DONE 2 +#define TL_ATOM_STATE_POSTED 3 + +void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid); +void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count); +void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn); +void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu); +void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr); +void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu); +void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid); +void __kbase_tlstream_tl_new_atom(void *atom, u32 nr); +void __kbase_tlstream_tl_del_ctx(void *context); +void __kbase_tlstream_tl_del_atom(void *atom); +void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu); +void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context); +void __kbase_tlstream_tl_ret_atom_lpu( + void *atom, void *lpu, const char *attrib_match_list); +void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu); +void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context); +void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu); +void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx); +void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx); +void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as); +void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as); +void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2); +void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2); +void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2); +void __kbase_tlstream_tl_attrib_atom_config( + void *atom, u64 jd, u64 affinity, u32 config); +void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio); +void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state); +void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom); +void __kbase_tlstream_tl_attrib_atom_jit( + void *atom, u64 edit_addr, u64 new_addr); +void __kbase_tlstream_tl_attrib_as_config( + void *as, u64 transtab, u64 memattr, u64 transcfg); +void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom); +void __kbase_tlstream_tl_event_lpu_softstop(void *lpu); +void __kbase_tlstream_tl_event_atom_softstop_issue(void *atom); +void __kbase_tlstream_jd_gpu_soft_reset(void *gpu); +void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state); +void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change); +void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count); +void __kbase_tlstream_aux_devfreq_target(u64 target_freq); +void __kbase_tlstream_aux_protected_enter_start(void *gpu); +void __kbase_tlstream_aux_protected_enter_end(void *gpu); +void __kbase_tlstream_aux_protected_leave_start(void *gpu); +void __kbase_tlstream_aux_protected_leave_end(void *gpu); + +#define TLSTREAM_ENABLED (1 << 31) + +extern atomic_t kbase_tlstream_enabled; + +#define __TRACE_IF_ENABLED(trace_name, ...) \ + do { \ + int enabled = atomic_read(&kbase_tlstream_enabled); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_##trace_name(__VA_ARGS__); \ + } while (0) + +#define __TRACE_IF_ENABLED_LATENCY(trace_name, ...) \ + do { \ + int enabled = atomic_read(&kbase_tlstream_enabled); \ + if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ + __kbase_tlstream_##trace_name(__VA_ARGS__); \ + } while (0) + +#define __TRACE_IF_ENABLED_JD(trace_name, ...) \ + do { \ + int enabled = atomic_read(&kbase_tlstream_enabled); \ + if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ + __kbase_tlstream_##trace_name(__VA_ARGS__); \ + } while (0) + +/*****************************************************************************/ + +/** + * KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX - create context object in timeline + * summary + * @context: name of the context object + * @nr: context number + * @tgid: thread Group Id + * + * Function emits a timeline message informing about context creation. Context + * is created with context number (its attribute), that can be used to link + * kbase context with userspace context. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX(context, nr, tgid) \ + __TRACE_IF_ENABLED(tl_summary_new_ctx, context, nr, tgid) + +/** + * KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU - create GPU object in timeline summary + * @gpu: name of the GPU object + * @id: id value of this GPU + * @core_count: number of cores this GPU hosts + * + * Function emits a timeline message informing about GPU creation. GPU is + * created with two attributes: id and core count. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU(gpu, id, core_count) \ + __TRACE_IF_ENABLED(tl_summary_new_gpu, gpu, id, core_count) + +/** + * KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU - create LPU object in timeline summary + * @lpu: name of the Logical Processing Unit object + * @nr: sequential number assigned to this LPU + * @fn: property describing this LPU's functional abilities + * + * Function emits a timeline message informing about LPU creation. LPU is + * created with two attributes: number linking this LPU with GPU's job slot + * and function bearing information about this LPU abilities. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, nr, fn) \ + __TRACE_IF_ENABLED(tl_summary_new_lpu, lpu, nr, fn) + +/** + * KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU - lifelink LPU object to GPU + * @lpu: name of the Logical Processing Unit object + * @gpu: name of the GPU object + * + * Function emits a timeline message informing that LPU object shall be deleted + * along with GPU object. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, gpu) \ + __TRACE_IF_ENABLED(tl_summary_lifelink_lpu_gpu, lpu, gpu) + +/** + * KBASE_TLSTREAM_TL_SUMMARY_NEW_AS - create address space object in timeline summary + * @as: name of the address space object + * @nr: sequential number assigned to this address space + * + * Function emits a timeline message informing about address space creation. + * Address space is created with one attribute: number identifying this + * address space. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(as, nr) \ + __TRACE_IF_ENABLED(tl_summary_new_as, as, nr) + +/** + * KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU - lifelink address space object to GPU + * @as: name of the address space object + * @gpu: name of the GPU object + * + * Function emits a timeline message informing that address space object + * shall be deleted along with GPU object. + * This message is directed to timeline summary stream. + */ +#define KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU(as, gpu) \ + __TRACE_IF_ENABLED(tl_summary_lifelink_as_gpu, as, gpu) + +/** + * KBASE_TLSTREAM_TL_NEW_CTX - create context object in timeline + * @context: name of the context object + * @nr: context number + * @tgid: thread Group Id + * + * Function emits a timeline message informing about context creation. Context + * is created with context number (its attribute), that can be used to link + * kbase context with userspace context. + */ +#define KBASE_TLSTREAM_TL_NEW_CTX(context, nr, tgid) \ + __TRACE_IF_ENABLED(tl_new_ctx, context, nr, tgid) + +/** + * KBASE_TLSTREAM_TL_NEW_ATOM - create atom object in timeline + * @atom: name of the atom object + * @nr: sequential number assigned to this atom + * + * Function emits a timeline message informing about atom creation. Atom is + * created with atom number (its attribute) that links it with actual work + * bucket id understood by hardware. + */ +#define KBASE_TLSTREAM_TL_NEW_ATOM(atom, nr) \ + __TRACE_IF_ENABLED(tl_new_atom, atom, nr) + +/** + * KBASE_TLSTREAM_TL_DEL_CTX - destroy context object in timeline + * @context: name of the context object + * + * Function emits a timeline message informing that context object ceased to + * exist. + */ +#define KBASE_TLSTREAM_TL_DEL_CTX(context) \ + __TRACE_IF_ENABLED(tl_del_ctx, context) + +/** + * KBASE_TLSTREAM_TL_DEL_ATOM - destroy atom object in timeline + * @atom: name of the atom object + * + * Function emits a timeline message informing that atom object ceased to + * exist. + */ +#define KBASE_TLSTREAM_TL_DEL_ATOM(atom) \ + __TRACE_IF_ENABLED(tl_del_atom, atom) + +/** + * KBASE_TLSTREAM_TL_RET_CTX_LPU - retain context by LPU + * @context: name of the context object + * @lpu: name of the Logical Processing Unit object + * + * Function emits a timeline message informing that context is being held + * by LPU and must not be deleted unless it is released. + */ +#define KBASE_TLSTREAM_TL_RET_CTX_LPU(context, lpu) \ + __TRACE_IF_ENABLED(tl_ret_ctx_lpu, context, lpu) + +/** + * KBASE_TLSTREAM_TL_RET_ATOM_CTX - retain atom by context + * @atom: name of the atom object + * @context: name of the context object + * + * Function emits a timeline message informing that atom object is being held + * by context and must not be deleted unless it is released. + */ +#define KBASE_TLSTREAM_TL_RET_ATOM_CTX(atom, context) \ + __TRACE_IF_ENABLED(tl_ret_atom_ctx, atom, context) + +/** + * KBASE_TLSTREAM_TL_RET_ATOM_LPU - retain atom by LPU + * @atom: name of the atom object + * @lpu: name of the Logical Processing Unit object + * @attrib_match_list: list containing match operator attributes + * + * Function emits a timeline message informing that atom object is being held + * by LPU and must not be deleted unless it is released. + */ +#define KBASE_TLSTREAM_TL_RET_ATOM_LPU(atom, lpu, attrib_match_list) \ + __TRACE_IF_ENABLED(tl_ret_atom_lpu, atom, lpu, attrib_match_list) + +/** + * KBASE_TLSTREAM_TL_NRET_CTX_LPU - release context by LPU + * @context: name of the context object + * @lpu: name of the Logical Processing Unit object + * + * Function emits a timeline message informing that context is being released + * by LPU object. + */ +#define KBASE_TLSTREAM_TL_NRET_CTX_LPU(context, lpu) \ + __TRACE_IF_ENABLED(tl_nret_ctx_lpu, context, lpu) + +/** + * KBASE_TLSTREAM_TL_NRET_ATOM_CTX - release atom by context + * @atom: name of the atom object + * @context: name of the context object + * + * Function emits a timeline message informing that atom object is being + * released by context. + */ +#define KBASE_TLSTREAM_TL_NRET_ATOM_CTX(atom, context) \ + __TRACE_IF_ENABLED(tl_nret_atom_ctx, atom, context) + +/** + * KBASE_TLSTREAM_TL_NRET_ATOM_LPU - release atom by LPU + * @atom: name of the atom object + * @lpu: name of the Logical Processing Unit object + * + * Function emits a timeline message informing that atom object is being + * released by LPU. + */ +#define KBASE_TLSTREAM_TL_NRET_ATOM_LPU(atom, lpu) \ + __TRACE_IF_ENABLED(tl_nret_atom_lpu, atom, lpu) + +/** + * KBASE_TLSTREAM_TL_RET_AS_CTX - lifelink address space object to context + * @as: name of the address space object + * @ctx: name of the context object + * + * Function emits a timeline message informing that address space object + * is being held by the context object. + */ +#define KBASE_TLSTREAM_TL_RET_AS_CTX(as, ctx) \ + __TRACE_IF_ENABLED(tl_ret_as_ctx, as, ctx) + +/** + * KBASE_TLSTREAM_TL_NRET_AS_CTX - release address space by context + * @as: name of the address space object + * @ctx: name of the context object + * + * Function emits a timeline message informing that address space object + * is being released by atom. + */ +#define KBASE_TLSTREAM_TL_NRET_AS_CTX(as, ctx) \ + __TRACE_IF_ENABLED(tl_nret_as_ctx, as, ctx) + +/** + * KBASE_TLSTREAM_TL_RET_ATOM_AS - retain atom by address space + * @atom: name of the atom object + * @as: name of the address space object + * + * Function emits a timeline message informing that atom object is being held + * by address space and must not be deleted unless it is released. + */ +#define KBASE_TLSTREAM_TL_RET_ATOM_AS(atom, as) \ + __TRACE_IF_ENABLED(tl_ret_atom_as, atom, as) + +/** + * KBASE_TLSTREAM_TL_NRET_ATOM_AS - release atom by address space + * @atom: name of the atom object + * @as: name of the address space object + * + * Function emits a timeline message informing that atom object is being + * released by address space. + */ +#define KBASE_TLSTREAM_TL_NRET_ATOM_AS(atom, as) \ + __TRACE_IF_ENABLED(tl_nret_atom_as, atom, as) + +/** + * KBASE_TLSTREAM_TL_DEP_ATOM_ATOM - parent atom depends on child atom + * @atom1: name of the child atom object + * @atom2: name of the parent atom object that depends on child atom + * + * Function emits a timeline message informing that parent atom waits for + * child atom object to be completed before start its execution. + */ +#define KBASE_TLSTREAM_TL_DEP_ATOM_ATOM(atom1, atom2) \ + __TRACE_IF_ENABLED(tl_dep_atom_atom, atom1, atom2) + +/** + * KBASE_TLSTREAM_TL_NDEP_ATOM_ATOM - dependency between atoms resolved + * @atom1: name of the child atom object + * @atom2: name of the parent atom object that depended on child atom + * + * Function emits a timeline message informing that parent atom execution + * dependency on child atom has been resolved. + */ +#define KBASE_TLSTREAM_TL_NDEP_ATOM_ATOM(atom1, atom2) \ + __TRACE_IF_ENABLED(tl_ndep_atom_atom, atom1, atom2) + +/** + * KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM - information about already resolved dependency between atoms + * @atom1: name of the child atom object + * @atom2: name of the parent atom object that depended on child atom + * + * Function emits a timeline message informing that parent atom execution + * dependency on child atom has been resolved. + */ +#define KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM(atom1, atom2) \ + __TRACE_IF_ENABLED(tl_rdep_atom_atom, atom1, atom2) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG - atom job slot attributes + * @atom: name of the atom object + * @jd: job descriptor address + * @affinity: job affinity + * @config: job config + * + * Function emits a timeline message containing atom attributes. + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(atom, jd, affinity, config) \ + __TRACE_IF_ENABLED(tl_attrib_atom_config, atom, jd, affinity, config) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY - atom priority + * @atom: name of the atom object + * @prio: atom priority + * + * Function emits a timeline message containing atom priority. + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(atom, prio) \ + __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority, atom, prio) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE - atom state + * @atom: name of the atom object + * @state: atom state + * + * Function emits a timeline message containing atom state. + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, state) \ + __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_state, atom, state) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE - atom caused priority change + * @atom: name of the atom object + * + * Function emits a timeline message signalling priority change + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE(atom) \ + __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority_change, atom) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - jit happened on atom + * @atom: atom identifier + * @edit_addr: address edited by jit + * @new_addr: address placed into the edited location + */ +#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(atom, edit_addr, new_addr) \ + __TRACE_IF_ENABLED_JD(tl_attrib_atom_jit, atom, edit_addr, new_addr) + +/** + * KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG - address space attributes + * @as: assigned address space + * @transtab: configuration of the TRANSTAB register + * @memattr: configuration of the MEMATTR register + * @transcfg: configuration of the TRANSCFG register (or zero if not present) + * + * Function emits a timeline message containing address space attributes. + */ +#define KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(as, transtab, memattr, transcfg) \ + __TRACE_IF_ENABLED(tl_attrib_as_config, as, transtab, memattr, transcfg) + +/** + * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ex + * @atom: atom identifier + */ +#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(atom) \ + __TRACE_IF_ENABLED(tl_event_atom_softstop_ex, atom) + +/** + * KBASE_TLSTREAM_TL_EVENT_LPU_softstop + * @lpu: name of the LPU object + */ +#define KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP(lpu) \ + __TRACE_IF_ENABLED(tl_event_lpu_softstop, lpu) + +/** + * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_issue + * @atom: atom identifier + */ +#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(atom) \ + __TRACE_IF_ENABLED(tl_event_atom_softstop_issue, atom) + +/** + * KBASE_TLSTREAM_JD_GPU_SOFT_RESET - The GPU is being soft reset + * @gpu: name of the GPU object + * + * This imperative tracepoint is specific to job dumping. + * Function emits a timeline message indicating GPU soft reset. + */ +#define KBASE_TLSTREAM_JD_GPU_SOFT_RESET(gpu) \ + __TRACE_IF_ENABLED(jd_gpu_soft_reset, gpu) + + +/** + * KBASE_TLSTREAM_AUX_PM_STATE - timeline message: power management state + * @core_type: core type (shader, tiler, l2 cache, l3 cache) + * @state: 64bits bitmask reporting power state of the cores (1-ON, 0-OFF) + */ +#define KBASE_TLSTREAM_AUX_PM_STATE(core_type, state) \ + __TRACE_IF_ENABLED(aux_pm_state, core_type, state) + +/** + * KBASE_TLSTREAM_AUX_PAGEFAULT - timeline message: MMU page fault event + * resulting in new pages being mapped + * @ctx_nr: kernel context number + * @page_count_change: number of pages to be added + */ +#define KBASE_TLSTREAM_AUX_PAGEFAULT(ctx_nr, page_count_change) \ + __TRACE_IF_ENABLED(aux_pagefault, ctx_nr, page_count_change) + +/** + * KBASE_TLSTREAM_AUX_PAGESALLOC - timeline message: total number of allocated + * pages is changed + * @ctx_nr: kernel context number + * @page_count: number of pages used by the context + */ +#define KBASE_TLSTREAM_AUX_PAGESALLOC(ctx_nr, page_count) \ + __TRACE_IF_ENABLED(aux_pagesalloc, ctx_nr, page_count) + +/** + * KBASE_TLSTREAM_AUX_DEVFREQ_TARGET - timeline message: new target DVFS + * frequency + * @target_freq: new target frequency + */ +#define KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(target_freq) \ + __TRACE_IF_ENABLED(aux_devfreq_target, target_freq) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START - The GPU has started transitioning + * to protected mode + * @gpu: name of the GPU object + * + * Function emits a timeline message indicating the GPU is starting to + * transition to protected mode. + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(gpu) \ + __TRACE_IF_ENABLED_LATENCY(aux_protected_enter_start, gpu) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END - The GPU has finished transitioning + * to protected mode + * @gpu: name of the GPU object + * + * Function emits a timeline message indicating the GPU has finished + * transitioning to protected mode. + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(gpu) \ + __TRACE_IF_ENABLED_LATENCY(aux_protected_enter_end, gpu) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START - The GPU has started transitioning + * to non-protected mode + * @gpu: name of the GPU object + * + * Function emits a timeline message indicating the GPU is starting to + * transition to non-protected mode. + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(gpu) \ + __TRACE_IF_ENABLED_LATENCY(aux_protected_leave_start, gpu) + +/** + * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END - The GPU has finished transitioning + * to non-protected mode + * @gpu: name of the GPU object + * + * Function emits a timeline message indicating the GPU has finished + * transitioning to non-protected mode. + */ +#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(gpu) \ + __TRACE_IF_ENABLED_LATENCY(aux_protected_leave_end, gpu) + +#endif /* _KBASE_TLSTREAM_H */ + diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_defs.h b/drivers/gpu/arm/midgard/mali_kbase_trace_defs.h new file mode 100755 index 000000000000..e2e0544208ce --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_trace_defs.h @@ -0,0 +1,264 @@ +/* + * + * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** + * ***** DO NOT INCLUDE DIRECTLY ***** + * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ + +/* + * The purpose of this header file is just to contain a list of trace code idenitifers + * + * Each identifier is wrapped in a macro, so that its string form and enum form can be created + * + * Each macro is separated with a comma, to allow insertion into an array initializer or enum definition block. + * + * This allows automatic creation of an enum and a corresponding array of strings + * + * Before #including, the includer MUST #define KBASE_TRACE_CODE_MAKE_CODE. + * After #including, the includer MUST #under KBASE_TRACE_CODE_MAKE_CODE. + * + * e.g.: + * #define KBASE_TRACE_CODE( X ) KBASE_TRACE_CODE_ ## X + * typedef enum + * { + * #define KBASE_TRACE_CODE_MAKE_CODE( X ) KBASE_TRACE_CODE( X ) + * #include "mali_kbase_trace_defs.h" + * #undef KBASE_TRACE_CODE_MAKE_CODE + * } kbase_trace_code; + * + * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE + * + * + * The use of the macro here is: + * - KBASE_TRACE_CODE_MAKE_CODE( X ) + * + * Which produces: + * - For an enum, KBASE_TRACE_CODE_X + * - For a string, "X" + * + * + * For example: + * - KBASE_TRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to: + * - KBASE_TRACE_CODE_JM_JOB_COMPLETE for the enum + * - "JM_JOB_COMPLETE" for the string + * - To use it to trace an event, do: + * - KBASE_TRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val ); + */ + +#if 0 /* Dummy section to avoid breaking formatting */ +int dummy_array[] = { +#endif + +/* + * Core events + */ + /* no info_val, no gpu_addr, no atom */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), + /* no info_val, no gpu_addr, no atom */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM), + /* info_val == GPU_IRQ_STATUS register */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ), + /* info_val == bits cleared */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR), + /* info_val == GPU_IRQ_STATUS register */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE), + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET), + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET), + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR), + /* GPU addr==dump address */ + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE), + KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES), +/* + * Job Slot management events + */ + /* info_val==irq rawstat at start */ + KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ), + /* info_val==jobs processed */ + KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ_END), +/* In the following: + * + * - ctx is set if a corresponding job found (NULL otherwise, e.g. some soft-stop cases) + * - uatom==kernel-side mapped uatom address (for correlation with user-side) + */ + /* info_val==exit code; gpu_addr==chain gpuaddr */ + KBASE_TRACE_CODE_MAKE_CODE(JM_JOB_DONE), + /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT), + /* gpu_addr is as follows: + * - If JS_STATUS active after soft-stop, val==gpu addr written to + * JS_HEAD on submit + * - otherwise gpu_addr==0 */ + KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP), + KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), + KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), + /* gpu_addr==JS_HEAD read */ + KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP), + /* gpu_addr==JS_HEAD read */ + KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), + /* gpu_addr==JS_HEAD read */ + KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), + /* gpu_addr==JS_TAIL read */ + KBASE_TRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), +/* gpu_addr is as follows: + * - If JS_STATUS active before soft-stop, val==JS_HEAD + * - otherwise gpu_addr==0 + */ + /* gpu_addr==JS_HEAD read */ + KBASE_TRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), + KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), + KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), + /* info_val == is_scheduled */ + KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), + /* info_val == is_scheduled */ + KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), + KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_DONE), + /* info_val == nr jobs submitted */ + KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), + /* gpu_addr==JS_HEAD_NEXT last written */ + KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), + KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), + KBASE_TRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), + KBASE_TRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), +/* + * Job dispatch events + */ + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_DONE), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), + /* gpu_addr==0, info_val==0, uatom==0 */ + KBASE_TRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), +/* + * Scheduler Core events + */ + KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX_NOLOCK), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JS_ADD_JOB), + /* gpu_addr==last value written/would be written to JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), + KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_RELEASE_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), + /* gpu_addr==value to write into JS_HEAD */ + KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), + /* kctx is the one being evicted, info_val == kctx to put in */ + KBASE_TRACE_CODE_MAKE_CODE(JS_FAST_START_EVICTS_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), + /* info_val == lower 32 bits of affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), + /* info_val == lower 32 bits of affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), + /* info_val == lower 32 bits of affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), + /* info_val == lower 32 bits of rechecked affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), + /* info_val == lower 32 bits of rechecked affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), + /* info_val == lower 32 bits of affinity */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), + /* info_val == the ctx attribute now on ctx */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), + /* info_val == the ctx attribute now on runpool */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), + /* info_val == the ctx attribute now off ctx */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), + /* info_val == the ctx attribute now off runpool */ + KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), +/* + * Scheduler Policy events + */ + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), + /* info_val == whether it was evicted */ + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), + /* gpu_addr==JS_HEAD to write if the job were run */ + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), + KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), +/* + * Power Management Events + */ + KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP), + KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_L2), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_L2), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE), + KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER), + /* PM_DESIRED_REACHED: gpu_addr == pm.gpu_in_desired_state */ + KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED), + KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER), + KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_SHADER_INUSE), + KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_TILER_INUSE), + KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_SHADER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_TILER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_INUSE), + KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_INUSE), + KBASE_TRACE_CODE_MAKE_CODE(PM_UNREQUEST_CHANGE_SHADER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_UNREQUEST_CHANGE_TILER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED), + KBASE_TRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS), + KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE), + KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE), + KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_ON), + KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_OFF), + /* info_val == policy number, or -1 for "Already changing" */ + KBASE_TRACE_CODE_MAKE_CODE(PM_SET_POLICY), + KBASE_TRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY), + /* info_val == policy number */ + KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT), + /* info_val == policy number */ + KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM), +/* Unused code just to make it easier to not have a comma at the end. + * All other codes MUST come before this */ + KBASE_TRACE_CODE_MAKE_CODE(DUMMY) + +#if 0 /* Dummy section to avoid breaking formatting */ +}; +#endif + +/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.c b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.c new file mode 100755 index 000000000000..5830e87f0818 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.c @@ -0,0 +1,236 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include + +#define CREATE_TRACE_POINTS + +#ifdef CONFIG_MALI_TRACE_TIMELINE +#include "mali_timeline.h" + +#include +#include + +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atoms_in_flight); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atom); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_active); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_action); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_power_active); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_l2_power_active); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_event); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_slot_atom); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_checktrans); +EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_context_active); + +struct kbase_trace_timeline_desc { + char *enum_str; + char *desc; + char *format; + char *format_desc; +}; + +static struct kbase_trace_timeline_desc kbase_trace_timeline_desc_table[] = { + #define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) { #enum_val, desc, format, format_desc } + #include "mali_kbase_trace_timeline_defs.h" + #undef KBASE_TIMELINE_TRACE_CODE +}; + +#define KBASE_NR_TRACE_CODES ARRAY_SIZE(kbase_trace_timeline_desc_table) + +static void *kbasep_trace_timeline_seq_start(struct seq_file *s, loff_t *pos) +{ + if (*pos >= KBASE_NR_TRACE_CODES) + return NULL; + + return &kbase_trace_timeline_desc_table[*pos]; +} + +static void kbasep_trace_timeline_seq_stop(struct seq_file *s, void *data) +{ +} + +static void *kbasep_trace_timeline_seq_next(struct seq_file *s, void *data, loff_t *pos) +{ + (*pos)++; + + if (*pos == KBASE_NR_TRACE_CODES) + return NULL; + + return &kbase_trace_timeline_desc_table[*pos]; +} + +static int kbasep_trace_timeline_seq_show(struct seq_file *s, void *data) +{ + struct kbase_trace_timeline_desc *trace_desc = data; + + seq_printf(s, "%s#%s#%s#%s\n", trace_desc->enum_str, trace_desc->desc, trace_desc->format, trace_desc->format_desc); + return 0; +} + + +static const struct seq_operations kbasep_trace_timeline_seq_ops = { + .start = kbasep_trace_timeline_seq_start, + .next = kbasep_trace_timeline_seq_next, + .stop = kbasep_trace_timeline_seq_stop, + .show = kbasep_trace_timeline_seq_show, +}; + +static int kbasep_trace_timeline_debugfs_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &kbasep_trace_timeline_seq_ops); +} + +static const struct file_operations kbasep_trace_timeline_debugfs_fops = { + .open = kbasep_trace_timeline_debugfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +#ifdef CONFIG_DEBUG_FS + +void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev) +{ + debugfs_create_file("mali_timeline_defs", + S_IRUGO, kbdev->mali_debugfs_directory, NULL, + &kbasep_trace_timeline_debugfs_fops); +} + +#endif /* CONFIG_DEBUG_FS */ + +void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (kbdev->timeline.slot_atoms_submitted[js] > 0) { + KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 1); + } else { + base_atom_id atom_number = kbase_jd_atom_id(kctx, katom); + + KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 1); + KBASE_TIMELINE_JOB_START(kctx, js, atom_number); + } + ++kbdev->timeline.slot_atoms_submitted[js]; + + KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, kbdev->timeline.slot_atoms_submitted[js]); +} + +void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js, + kbasep_js_atom_done_code done_code) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) { + KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 0); + } else { + /* Job finished in JS_HEAD */ + base_atom_id atom_number = kbase_jd_atom_id(kctx, katom); + + KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 0); + KBASE_TIMELINE_JOB_STOP(kctx, js, atom_number); + + /* see if we need to trace the job in JS_NEXT moving to JS_HEAD */ + if (kbase_backend_nr_atoms_submitted(kbdev, js)) { + struct kbase_jd_atom *next_katom; + struct kbase_context *next_kctx; + + /* Peek the next atom - note that the atom in JS_HEAD will already + * have been dequeued */ + next_katom = kbase_backend_inspect_head(kbdev, js); + WARN_ON(!next_katom); + next_kctx = next_katom->kctx; + KBASE_TIMELINE_JOB_START_NEXT(next_kctx, js, 0); + KBASE_TIMELINE_JOB_START_HEAD(next_kctx, js, 1); + KBASE_TIMELINE_JOB_START(next_kctx, js, kbase_jd_atom_id(next_kctx, next_katom)); + } + } + + --kbdev->timeline.slot_atoms_submitted[js]; + + KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, kbdev->timeline.slot_atoms_submitted[js]); +} + +void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent) +{ + int uid = 0; + int old_uid; + + /* If a producer already exists for the event, try to use their UID (multiple-producers) */ + uid = atomic_read(&kbdev->timeline.pm_event_uid[event_sent]); + old_uid = uid; + + /* Get a new non-zero UID if we don't have one yet */ + while (!uid) + uid = atomic_inc_return(&kbdev->timeline.pm_event_uid_counter); + + /* Try to use this UID */ + if (old_uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event_sent], old_uid, uid)) + /* If it changed, raced with another producer: we've lost this UID */ + uid = 0; + + KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_sent, uid); +} + +void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) +{ + int uid = atomic_read(&kbdev->timeline.pm_event_uid[event]); + + if (uid != 0) { + if (uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event], uid, 0)) + /* If it changed, raced with another consumer: we've lost this UID */ + uid = 0; + + KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event, uid); + } +} + +void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) +{ + int uid = atomic_read(&kbdev->timeline.pm_event_uid[event]); + + if (uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event], uid, 0)) + /* If it changed, raced with another consumer: we've lost this UID */ + uid = 0; + + KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event, uid); +} + +void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + /* Simply log the start of the transition */ + kbdev->timeline.l2_transitioning = true; + KBASE_TIMELINE_POWERING_L2(kbdev); +} + +void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + /* Simply log the end of the transition */ + if (kbdev->timeline.l2_transitioning) { + kbdev->timeline.l2_transitioning = false; + KBASE_TIMELINE_POWERED_L2(kbdev); + } +} + +#endif /* CONFIG_MALI_TRACE_TIMELINE */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.h b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.h new file mode 100755 index 000000000000..a04f7c1420e0 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.h @@ -0,0 +1,363 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#if !defined(_KBASE_TRACE_TIMELINE_H) +#define _KBASE_TRACE_TIMELINE_H + +#ifdef CONFIG_MALI_TRACE_TIMELINE + +enum kbase_trace_timeline_code { + #define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) enum_val + #include "mali_kbase_trace_timeline_defs.h" + #undef KBASE_TIMELINE_TRACE_CODE +}; + +#ifdef CONFIG_DEBUG_FS + +/** Initialize Timeline DebugFS entries */ +void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev); + +#else /* CONFIG_DEBUG_FS */ + +#define kbasep_trace_timeline_debugfs_init CSTD_NOP + +#endif /* CONFIG_DEBUG_FS */ + +/* mali_timeline.h defines kernel tracepoints used by the KBASE_TIMELINE + * functions. + * Output is timestamped by either sched_clock() (default), local_clock(), or + * cpu_clock(), depending on /sys/kernel/debug/tracing/trace_clock */ +#include "mali_timeline.h" + +/* Trace number of atoms in flight for kctx (atoms either not completed, or in + process of being returned to user */ +#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_atoms_in_flight(ts.tv_sec, ts.tv_nsec, \ + (int)kctx->timeline.owner_tgid, \ + count); \ + } while (0) + +/* Trace atom_id being Ready to Run */ +#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_atom(ts.tv_sec, ts.tv_nsec, \ + CTX_FLOW_ATOM_READY, \ + (int)kctx->timeline.owner_tgid, \ + atom_id); \ + } while (0) + +/* Trace number of atoms submitted to job slot js + * + * NOTE: This uses a different tracepoint to the head/next/soft-stop actions, + * so that those actions can be filtered out separately from this + * + * This is because this is more useful, as we can use it to calculate general + * utilization easily and accurately */ +#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_gpu_slot_active(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_SLOT_ACTIVE, \ + (int)kctx->timeline.owner_tgid, \ + js, count); \ + } while (0) + + +/* Trace atoms present in JS_NEXT */ +#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_SLOT_NEXT, \ + (int)kctx->timeline.owner_tgid, \ + js, count); \ + } while (0) + +/* Trace atoms present in JS_HEAD */ +#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_SLOT_HEAD, \ + (int)kctx->timeline.owner_tgid, \ + js, count); \ + } while (0) + +/* Trace that a soft stop/evict from next is being attempted on a slot */ +#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_SLOT_STOPPING, \ + (kctx) ? (int)kctx->timeline.owner_tgid : 0, \ + js, count); \ + } while (0) + + + +/* Trace state of overall GPU power */ +#define KBASE_TIMELINE_GPU_POWER(kbdev, active) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_POWER_ACTIVE, active); \ + } while (0) + +/* Trace state of tiler power */ +#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_POWER_TILER_ACTIVE, \ + hweight64(bitmap)); \ + } while (0) + +/* Trace number of shaders currently powered */ +#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_POWER_SHADER_ACTIVE, \ + hweight64(bitmap)); \ + } while (0) + +/* Trace state of L2 power */ +#define KBASE_TIMELINE_POWER_L2(kbdev, bitmap) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_SET_GPU_POWER_L2_ACTIVE, \ + hweight64(bitmap)); \ + } while (0) + +/* Trace state of L2 cache*/ +#define KBASE_TIMELINE_POWERING_L2(kbdev) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_FLOW_GPU_POWER_L2_POWERING, \ + 1); \ + } while (0) + +#define KBASE_TIMELINE_POWERED_L2(kbdev) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \ + SW_FLOW_GPU_POWER_L2_ACTIVE, \ + 1); \ + } while (0) + +/* Trace kbase_pm_send_event message send */ +#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \ + SW_FLOW_PM_SEND_EVENT, \ + event_type, pm_event_id); \ + } while (0) + +/* Trace kbase_pm_worker message receive */ +#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \ + SW_FLOW_PM_HANDLE_EVENT, \ + event_type, pm_event_id); \ + } while (0) + + +/* Trace atom_id starting in JS_HEAD */ +#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \ + HW_START_GPU_JOB_CHAIN_SW_APPROX, \ + (int)kctx->timeline.owner_tgid, \ + js, _consumerof_atom_number); \ + } while (0) + +/* Trace atom_id stopping on JS_HEAD */ +#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \ + HW_STOP_GPU_JOB_CHAIN_SW_APPROX, \ + (int)kctx->timeline.owner_tgid, \ + js, _producerof_atom_number_completed); \ + } while (0) + +/** Trace beginning/end of a call to kbase_pm_check_transitions_nolock from a + * certin caller */ +#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_pm_checktrans(ts.tv_sec, ts.tv_nsec, \ + trace_code, 1); \ + } while (0) + +/* Trace number of contexts active */ +#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) \ + do { \ + struct timespec64 ts; \ + ktime_get_raw_ts64(&ts); \ + trace_mali_timeline_context_active(ts.tv_sec, ts.tv_nsec, \ + count); \ + } while (0) + +/* NOTE: kbase_timeline_pm_cores_func() is in mali_kbase_pm_policy.c */ + +/** + * Trace that an atom is starting on a job slot + * + * The caller must be holding hwaccess_lock + */ +void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js); + +/** + * Trace that an atom has done on a job slot + * + * 'Done' in this sense can occur either because: + * - the atom in JS_HEAD finished + * - the atom in JS_NEXT was evicted + * + * Whether the atom finished or was evicted is passed in @a done_code + * + * It is assumed that the atom has already been removed from the submit slot, + * with either: + * - kbasep_jm_dequeue_submit_slot() + * - kbasep_jm_dequeue_tail_submit_slot() + * + * The caller must be holding hwaccess_lock + */ +void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js, + kbasep_js_atom_done_code done_code); + + +/** Trace a pm event starting */ +void kbase_timeline_pm_send_event(struct kbase_device *kbdev, + enum kbase_timeline_pm_event event_sent); + +/** Trace a pm event finishing */ +void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event); + +/** Check whether a pm event was present, and if so trace finishing it */ +void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event); + +/** Trace L2 power-up start */ +void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev); + +/** Trace L2 power-up done */ +void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev); + +#else + +#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) CSTD_NOP() + +#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) CSTD_NOP() + +#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) CSTD_NOP() + +#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) CSTD_NOP() + +#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) CSTD_NOP() + +#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) CSTD_NOP() + +#define KBASE_TIMELINE_GPU_POWER(kbdev, active) CSTD_NOP() + +#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) CSTD_NOP() + +#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) CSTD_NOP() + +#define KBASE_TIMELINE_POWER_L2(kbdev, active) CSTD_NOP() + +#define KBASE_TIMELINE_POWERING_L2(kbdev) CSTD_NOP() + +#define KBASE_TIMELINE_POWERED_L2(kbdev) CSTD_NOP() + +#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) CSTD_NOP() + +#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) CSTD_NOP() + +#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) CSTD_NOP() + +#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) CSTD_NOP() + +#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) CSTD_NOP() + +#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) CSTD_NOP() + +static inline void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); +} + +static inline void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, + struct kbase_jd_atom *katom, int js, + kbasep_js_atom_done_code done_code) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); +} + +static inline void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent) +{ +} + +static inline void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) +{ +} + +static inline void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) +{ +} + +static inline void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev) +{ +} + +static inline void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev) +{ +} +#endif /* CONFIG_MALI_TRACE_TIMELINE */ + +#endif /* _KBASE_TRACE_TIMELINE_H */ + diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_timeline_defs.h b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline_defs.h new file mode 100755 index 000000000000..156a95a67f4a --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline_defs.h @@ -0,0 +1,140 @@ +/* + * + * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** + * ***** DO NOT INCLUDE DIRECTLY ***** + * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ + +/* + * Conventions on Event Names: + * + * - The prefix determines something about how the timeline should be + * displayed, and is split up into various parts, separated by underscores: + * - 'SW' and 'HW' as the first part will be used to determine whether a + * timeline is to do with Software or Hardware - effectively, separate + * 'channels' for Software and Hardware + * - 'START', 'STOP', 'ENTER', 'LEAVE' can be used in the second part, and + * signify related pairs of events - these are optional. + * - 'FLOW' indicates a generic event, which can use dependencies + * - This gives events such as: + * - 'SW_ENTER_FOO' + * - 'SW_LEAVE_FOO' + * - 'SW_FLOW_BAR_1' + * - 'SW_FLOW_BAR_2' + * - 'HW_START_BAZ' + * - 'HW_STOP_BAZ' + * - And an unadorned HW event: + * - 'HW_BAZ_FROZBOZ' + */ + +/* + * Conventions on parameter names: + * - anything with 'instance' in the name will have a separate timeline based + * on that instances. + * - underscored-prefixed parameters will by hidden by default on timelines + * + * Hence: + * - Different job slots have their own 'instance', based on the instance value + * - Per-context info (e.g. atoms on a context) have their own 'instance' + * (i.e. each context should be on a different timeline) + * + * Note that globally-shared resources can be tagged with a tgid, but we don't + * want an instance per context: + * - There's no point having separate Job Slot timelines for each context, that + * would be confusing - there's only really 3 job slots! + * - There's no point having separate Shader-powered timelines for each + * context, that would be confusing - all shader cores (whether it be 4, 8, + * etc) are shared in the system. + */ + + /* + * CTX events + */ + /* Separate timelines for each context 'instance'*/ + KBASE_TIMELINE_TRACE_CODE(CTX_SET_NR_ATOMS_IN_FLIGHT, "CTX: Atoms in flight", "%d,%d", "_instance_tgid,_value_number_of_atoms"), + KBASE_TIMELINE_TRACE_CODE(CTX_FLOW_ATOM_READY, "CTX: Atoms Ready to Run", "%d,%d,%d", "_instance_tgid,_consumerof_atom_number,_producerof_atom_number_ready"), + + /* + * SW Events + */ + /* Separate timelines for each slot 'instance' */ + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_ACTIVE, "SW: GPU slot active", "%d,%d,%d", "_tgid,_instance_slot,_value_number_of_atoms"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_NEXT, "SW: GPU atom in NEXT", "%d,%d,%d", "_tgid,_instance_slot,_value_is_an_atom_in_next"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_HEAD, "SW: GPU atom in HEAD", "%d,%d,%d", "_tgid,_instance_slot,_value_is_an_atom_in_head"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_STOPPING, "SW: Try Soft-Stop on GPU slot", "%d,%d,%d", "_tgid,_instance_slot,_value_is_slot_stopping"), + /* Shader and overall power is shared - can't have separate instances of + * it, just tagging with the context */ + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_ACTIVE, "SW: GPU power active", "%d,%d", "_tgid,_value_is_power_active"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_TILER_ACTIVE, "SW: GPU tiler powered", "%d,%d", "_tgid,_value_number_of_tilers"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_SHADER_ACTIVE, "SW: GPU shaders powered", "%d,%d", "_tgid,_value_number_of_shaders"), + KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_L2_ACTIVE, "SW: GPU L2 powered", "%d,%d", "_tgid,_value_number_of_l2"), + + /* SW Power event messaging. _event_type is one from the kbase_pm_event enum */ + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_SEND_EVENT, "SW: PM Send Event", "%d,%d,%d", "_tgid,_event_type,_writerof_pm_event_id"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_HANDLE_EVENT, "SW: PM Handle Event", "%d,%d,%d", "_tgid,_event_type,_finalconsumerof_pm_event_id"), + /* SW L2 power events */ + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_GPU_POWER_L2_POWERING, "SW: GPU L2 powering", "%d,%d", "_tgid,_writerof_l2_transitioning"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_GPU_POWER_L2_ACTIVE, "SW: GPU L2 powering done", "%d,%d", "_tgid,_finalconsumerof_l2_transitioning"), + + KBASE_TIMELINE_TRACE_CODE(SW_SET_CONTEXT_ACTIVE, "SW: Context Active", "%d,%d", "_tgid,_value_active"), + + /* + * BEGIN: Significant SW Functions that call kbase_pm_check_transitions_nolock() + */ + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START, "SW: PM CheckTrans from kbase_pm_do_poweroff", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_poweroff"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END, "SW: PM CheckTrans from kbase_pm_do_poweroff", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_poweroff"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START, "SW: PM CheckTrans from kbase_pm_do_poweron", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_poweron"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END, "SW: PM CheckTrans from kbase_pm_do_poweron", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_poweron"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START, "SW: PM CheckTrans from kbase_gpu_interrupt", "%d,%d", "_tgid,_writerof_pm_checktrans_gpu_interrupt"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END, "SW: PM CheckTrans from kbase_gpu_interrupt", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_gpu_interrupt"), + + /* + * Significant Indirect callers of kbase_pm_check_transitions_nolock() + */ + /* kbase_pm_request_cores */ + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START, "SW: PM CheckTrans from kbase_pm_request_cores(shader)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_shader"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END, "SW: PM CheckTrans from kbase_pm_request_cores(shader)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_shader"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START, "SW: PM CheckTrans from kbase_pm_request_cores(tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END, "SW: PM CheckTrans from kbase_pm_request_cores(tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START, "SW: PM CheckTrans from kbase_pm_request_cores(shader+tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_shader_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END, "SW: PM CheckTrans from kbase_pm_request_cores(shader+tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_shader_tiler"), + /* kbase_pm_release_cores */ + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START, "SW: PM CheckTrans from kbase_pm_release_cores(shader)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_shader"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END, "SW: PM CheckTrans from kbase_pm_release_cores(shader)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_shader"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START, "SW: PM CheckTrans from kbase_pm_release_cores(tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END, "SW: PM CheckTrans from kbase_pm_release_cores(tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START, "SW: PM CheckTrans from kbase_pm_release_cores(shader+tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_shader_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END, "SW: PM CheckTrans from kbase_pm_release_cores(shader+tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_shader_tiler"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START, "SW: PM CheckTrans from kbasep_pm_do_shader_poweroff_callback", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_shader_poweroff_callback"), + KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END, "SW: PM CheckTrans from kbasep_pm_do_shader_poweroff_callback", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_shader_poweroff_callback"), + /* + * END: SW Functions that call kbase_pm_check_transitions_nolock() + */ + + /* + * HW Events + */ + KBASE_TIMELINE_TRACE_CODE(HW_MMU_FAULT, +"HW: MMU Fault", "%d,%d,%d", "_tgid,fault_type,fault_stage,asid"), + KBASE_TIMELINE_TRACE_CODE(HW_START_GPU_JOB_CHAIN_SW_APPROX, +"HW: Job Chain start (SW approximated)", "%d,%d,%d", +"_tgid,job_slot,_consumerof_atom_number_ready"), + KBASE_TIMELINE_TRACE_CODE(HW_STOP_GPU_JOB_CHAIN_SW_APPROX, +"HW: Job Chain stop (SW approximated)", "%d,%d,%d", +"_tgid,job_slot,_producerof_atom_number_completed") diff --git a/drivers/gpu/arm/midgard/mali_kbase_uku.h b/drivers/gpu/arm/midgard/mali_kbase_uku.h new file mode 100755 index 000000000000..c22a59324248 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_uku.h @@ -0,0 +1,545 @@ +/* + * + * (C) COPYRIGHT 2008-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_UKU_H_ +#define _KBASE_UKU_H_ + +#include "mali_uk.h" +#include "mali_base_kernel.h" + +/* This file needs to support being included from kernel and userside (which use different defines) */ +#if defined(CONFIG_MALI_ERROR_INJECT) || MALI_ERROR_INJECT_ON +#define SUPPORT_MALI_ERROR_INJECT +#endif /* defined(CONFIG_MALI_ERROR_INJECT) || MALI_ERROR_INJECT_ON */ +#if defined(CONFIG_MALI_NO_MALI) +#define SUPPORT_MALI_NO_MALI +#elif defined(MALI_NO_MALI) +#if MALI_NO_MALI +#define SUPPORT_MALI_NO_MALI +#endif +#endif + +#if defined(SUPPORT_MALI_NO_MALI) || defined(SUPPORT_MALI_ERROR_INJECT) +#include "backend/gpu/mali_kbase_model_dummy.h" +#endif + +#include "mali_kbase_gpuprops_types.h" + +/* + * 10.1: + * - Do mmap in kernel for SAME_VA memory allocations rather then + * calling back into the kernel as a 2nd stage of the allocation request. + * + * 10.2: + * - Add KBASE_FUNC_MEM_JIT_INIT which allows clients to request a custom VA + * region for use with JIT (ignored on 32-bit platforms) + * + * 10.3: + * - base_jd_core_req typedef-ed to u32 (instead of to u16) + * - two flags added: BASE_JD_REQ_SKIP_CACHE_STAT / _END + * + * 10.4: + * - Removed KBASE_FUNC_EXT_BUFFER_LOCK used only in internal tests + * + * 10.5: + * - Reverted to performing mmap in user space so that tools like valgrind work. + * + * 10.6: + * - Add flags input variable to KBASE_FUNC_TLSTREAM_ACQUIRE + */ +#define BASE_UK_VERSION_MAJOR 10 +#define BASE_UK_VERSION_MINOR 6 + +#define LINUX_UK_BASE_MAGIC 0x80 + +struct kbase_uk_mem_alloc { + union uk_header header; + /* IN */ + u64 va_pages; + u64 commit_pages; + u64 extent; + /* IN/OUT */ + u64 flags; + /* OUT */ + u64 gpu_va; + u16 va_alignment; + u8 padding[6]; +}; + +struct kbase_uk_mem_free { + union uk_header header; + /* IN */ + u64 gpu_addr; + /* OUT */ +}; + +struct kbase_uk_mem_alias { + union uk_header header; + /* IN/OUT */ + u64 flags; + /* IN */ + u64 stride; + u64 nents; + union kbase_pointer ai; + /* OUT */ + u64 gpu_va; + u64 va_pages; +}; + +struct kbase_uk_mem_import { + union uk_header header; + /* IN */ + union kbase_pointer phandle; + u32 type; + u32 padding; + /* IN/OUT */ + u64 flags; + /* OUT */ + u64 gpu_va; + u64 va_pages; +}; + +struct kbase_uk_mem_flags_change { + union uk_header header; + /* IN */ + u64 gpu_va; + u64 flags; + u64 mask; +}; + +struct kbase_uk_job_submit { + union uk_header header; + /* IN */ + union kbase_pointer addr; + u32 nr_atoms; + u32 stride; /* bytes between atoms, i.e. sizeof(base_jd_atom_v2) */ + /* OUT */ +}; + +struct kbase_uk_post_term { + union uk_header header; +}; + +struct kbase_uk_sync_now { + union uk_header header; + + /* IN */ + struct base_syncset sset; + + /* OUT */ +}; + +struct kbase_uk_hwcnt_setup { + union uk_header header; + + /* IN */ + u64 dump_buffer; + u32 jm_bm; + u32 shader_bm; + u32 tiler_bm; + u32 unused_1; /* keep for backwards compatibility */ + u32 mmu_l2_bm; + u32 padding; + /* OUT */ +}; + +/** + * struct kbase_uk_hwcnt_reader_setup - User/Kernel space data exchange structure + * @header: UK structure header + * @buffer_count: requested number of dumping buffers + * @jm_bm: counters selection bitmask (JM) + * @shader_bm: counters selection bitmask (Shader) + * @tiler_bm: counters selection bitmask (Tiler) + * @mmu_l2_bm: counters selection bitmask (MMU_L2) + * @fd: dumping notification file descriptor + * + * This structure sets up HWC dumper/reader for this context. + * Multiple instances can be created for single context. + */ +struct kbase_uk_hwcnt_reader_setup { + union uk_header header; + + /* IN */ + u32 buffer_count; + u32 jm_bm; + u32 shader_bm; + u32 tiler_bm; + u32 mmu_l2_bm; + + /* OUT */ + s32 fd; +}; + +struct kbase_uk_hwcnt_dump { + union uk_header header; +}; + +struct kbase_uk_hwcnt_clear { + union uk_header header; +}; + +struct kbase_uk_fence_validate { + union uk_header header; + /* IN */ + s32 fd; + u32 padding; + /* OUT */ +}; + +struct kbase_uk_stream_create { + union uk_header header; + /* IN */ + char name[32]; + /* OUT */ + s32 fd; + u32 padding; +}; + +struct kbase_uk_gpuprops { + union uk_header header; + + /* IN */ + struct mali_base_gpu_props props; + /* OUT */ +}; + +struct kbase_uk_mem_query { + union uk_header header; + /* IN */ + u64 gpu_addr; +#define KBASE_MEM_QUERY_COMMIT_SIZE 1 +#define KBASE_MEM_QUERY_VA_SIZE 2 +#define KBASE_MEM_QUERY_FLAGS 3 + u64 query; + /* OUT */ + u64 value; +}; + +struct kbase_uk_mem_commit { + union uk_header header; + /* IN */ + u64 gpu_addr; + u64 pages; + /* OUT */ + u32 result_subcode; + u32 padding; +}; + +struct kbase_uk_find_cpu_offset { + union uk_header header; + /* IN */ + u64 gpu_addr; + u64 cpu_addr; + u64 size; + /* OUT */ + u64 offset; +}; + +#define KBASE_GET_VERSION_BUFFER_SIZE 64 +struct kbase_uk_get_ddk_version { + union uk_header header; + /* OUT */ + char version_buffer[KBASE_GET_VERSION_BUFFER_SIZE]; + u32 version_string_size; + u32 padding; + u32 rk_version; +}; + +struct kbase_uk_disjoint_query { + union uk_header header; + /* OUT */ + u32 counter; + u32 padding; +}; + +struct kbase_uk_set_flags { + union uk_header header; + /* IN */ + u32 create_flags; + u32 padding; +}; + +#if MALI_UNIT_TEST +#define TEST_ADDR_COUNT 4 +#define KBASE_TEST_BUFFER_SIZE 128 +struct kbase_exported_test_data { + u64 test_addr[TEST_ADDR_COUNT]; /**< memory address */ + u32 test_addr_pages[TEST_ADDR_COUNT]; /**< memory size in pages */ + union kbase_pointer kctx; /**< base context created by process */ + union kbase_pointer mm; /**< pointer to process address space */ + u8 buffer1[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */ + u8 buffer2[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */ +}; + +struct kbase_uk_set_test_data { + union uk_header header; + /* IN */ + struct kbase_exported_test_data test_data; +}; + +#endif /* MALI_UNIT_TEST */ + +#ifdef SUPPORT_MALI_ERROR_INJECT +struct kbase_uk_error_params { + union uk_header header; + /* IN */ + struct kbase_error_params params; +}; +#endif /* SUPPORT_MALI_ERROR_INJECT */ + +#ifdef SUPPORT_MALI_NO_MALI +struct kbase_uk_model_control_params { + union uk_header header; + /* IN */ + struct kbase_model_control_params params; +}; +#endif /* SUPPORT_MALI_NO_MALI */ + +#ifdef BASE_LEGACY_UK8_SUPPORT +struct kbase_uk_keep_gpu_powered { + union uk_header header; + u32 enabled; + u32 padding; +}; +#endif /* BASE_LEGACY_UK8_SUPPORT */ + +struct kbase_uk_profiling_controls { + union uk_header header; + u32 profiling_controls[FBDUMP_CONTROL_MAX]; +}; + +struct kbase_uk_debugfs_mem_profile_add { + union uk_header header; + u32 len; + u32 padding; + union kbase_pointer buf; +}; + +struct kbase_uk_context_id { + union uk_header header; + /* OUT */ + int id; +}; + +/** + * struct kbase_uk_tlstream_acquire - User/Kernel space data exchange structure + * @header: UK structure header + * @flags: timeline stream flags + * @fd: timeline stream file descriptor + * + * This structure is used when performing a call to acquire kernel side timeline + * stream file descriptor. + */ +struct kbase_uk_tlstream_acquire { + union uk_header header; + /* IN */ + u32 flags; + /* OUT */ + s32 fd; +}; + +/** + * struct kbase_uk_tlstream_acquire_v10_4 - User/Kernel space data exchange + * structure + * @header: UK structure header + * @fd: timeline stream file descriptor + * + * This structure is used when performing a call to acquire kernel side timeline + * stream file descriptor. + */ +struct kbase_uk_tlstream_acquire_v10_4 { + union uk_header header; + /* IN */ + /* OUT */ + s32 fd; +}; + +/** + * struct kbase_uk_tlstream_flush - User/Kernel space data exchange structure + * @header: UK structure header + * + * This structure is used when performing a call to flush kernel side + * timeline streams. + */ +struct kbase_uk_tlstream_flush { + union uk_header header; + /* IN */ + /* OUT */ +}; + +#if MALI_UNIT_TEST +/** + * struct kbase_uk_tlstream_test - User/Kernel space data exchange structure + * @header: UK structure header + * @tpw_count: number of trace point writers in each context + * @msg_delay: time delay between tracepoints from one writer in milliseconds + * @msg_count: number of trace points written by one writer + * @aux_msg: if non-zero aux messages will be included + * + * This structure is used when performing a call to start timeline stream test + * embedded in kernel. + */ +struct kbase_uk_tlstream_test { + union uk_header header; + /* IN */ + u32 tpw_count; + u32 msg_delay; + u32 msg_count; + u32 aux_msg; + /* OUT */ +}; + +/** + * struct kbase_uk_tlstream_stats - User/Kernel space data exchange structure + * @header: UK structure header + * @bytes_collected: number of bytes read by user + * @bytes_generated: number of bytes generated by tracepoints + * + * This structure is used when performing a call to obtain timeline stream + * statistics. + */ +struct kbase_uk_tlstream_stats { + union uk_header header; /**< UK structure header. */ + /* IN */ + /* OUT */ + u32 bytes_collected; + u32 bytes_generated; +}; +#endif /* MALI_UNIT_TEST */ + +/** + * struct struct kbase_uk_prfcnt_value for the KBASE_FUNC_SET_PRFCNT_VALUES ioctl + * @header: UK structure header + * @data: Counter samples for the dummy model + * @size:............Size of the counter sample data + */ +struct kbase_uk_prfcnt_values { + union uk_header header; + /* IN */ + u32 *data; + u32 size; +}; + +/** + * struct kbase_uk_soft_event_update - User/Kernel space data exchange structure + * @header: UK structure header + * @evt: the GPU address containing the event + * @new_status: the new event status, must be either BASE_JD_SOFT_EVENT_SET or + * BASE_JD_SOFT_EVENT_RESET + * @flags: reserved for future uses, must be set to 0 + * + * This structure is used to update the status of a software event. If the + * event's status is set to BASE_JD_SOFT_EVENT_SET, any job currently waiting + * on this event will complete. + */ +struct kbase_uk_soft_event_update { + union uk_header header; + /* IN */ + u64 evt; + u32 new_status; + u32 flags; +}; + +/** + * struct kbase_uk_mem_jit_init - User/Kernel space data exchange structure + * @header: UK structure header + * @va_pages: Number of virtual pages required for JIT + * + * This structure is used when requesting initialization of JIT. + */ +struct kbase_uk_mem_jit_init { + union uk_header header; + /* IN */ + u64 va_pages; +}; + +enum kbase_uk_function_id { + KBASE_FUNC_MEM_ALLOC = (UK_FUNC_ID + 0), + KBASE_FUNC_MEM_IMPORT = (UK_FUNC_ID + 1), + KBASE_FUNC_MEM_COMMIT = (UK_FUNC_ID + 2), + KBASE_FUNC_MEM_QUERY = (UK_FUNC_ID + 3), + KBASE_FUNC_MEM_FREE = (UK_FUNC_ID + 4), + KBASE_FUNC_MEM_FLAGS_CHANGE = (UK_FUNC_ID + 5), + KBASE_FUNC_MEM_ALIAS = (UK_FUNC_ID + 6), + +#ifdef BASE_LEGACY_UK6_SUPPORT + KBASE_FUNC_JOB_SUBMIT_UK6 = (UK_FUNC_ID + 7), +#endif /* BASE_LEGACY_UK6_SUPPORT */ + + KBASE_FUNC_SYNC = (UK_FUNC_ID + 8), + + KBASE_FUNC_POST_TERM = (UK_FUNC_ID + 9), + + KBASE_FUNC_HWCNT_SETUP = (UK_FUNC_ID + 10), + KBASE_FUNC_HWCNT_DUMP = (UK_FUNC_ID + 11), + KBASE_FUNC_HWCNT_CLEAR = (UK_FUNC_ID + 12), + + KBASE_FUNC_GPU_PROPS_REG_DUMP = (UK_FUNC_ID + 14), + + KBASE_FUNC_FIND_CPU_OFFSET = (UK_FUNC_ID + 15), + + KBASE_FUNC_GET_VERSION = (UK_FUNC_ID + 16), + KBASE_FUNC_SET_FLAGS = (UK_FUNC_ID + 18), + + KBASE_FUNC_SET_TEST_DATA = (UK_FUNC_ID + 19), + KBASE_FUNC_INJECT_ERROR = (UK_FUNC_ID + 20), + KBASE_FUNC_MODEL_CONTROL = (UK_FUNC_ID + 21), + +#ifdef BASE_LEGACY_UK8_SUPPORT + KBASE_FUNC_KEEP_GPU_POWERED = (UK_FUNC_ID + 22), +#endif /* BASE_LEGACY_UK8_SUPPORT */ + + KBASE_FUNC_FENCE_VALIDATE = (UK_FUNC_ID + 23), + KBASE_FUNC_STREAM_CREATE = (UK_FUNC_ID + 24), + KBASE_FUNC_GET_PROFILING_CONTROLS = (UK_FUNC_ID + 25), + KBASE_FUNC_SET_PROFILING_CONTROLS = (UK_FUNC_ID + 26), + /* to be used only for testing + * purposes, otherwise these controls + * are set through gator API */ + + KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD = (UK_FUNC_ID + 27), + KBASE_FUNC_JOB_SUBMIT = (UK_FUNC_ID + 28), + KBASE_FUNC_DISJOINT_QUERY = (UK_FUNC_ID + 29), + + KBASE_FUNC_GET_CONTEXT_ID = (UK_FUNC_ID + 31), + + KBASE_FUNC_TLSTREAM_ACQUIRE_V10_4 = (UK_FUNC_ID + 32), +#if MALI_UNIT_TEST + KBASE_FUNC_TLSTREAM_TEST = (UK_FUNC_ID + 33), + KBASE_FUNC_TLSTREAM_STATS = (UK_FUNC_ID + 34), +#endif /* MALI_UNIT_TEST */ + KBASE_FUNC_TLSTREAM_FLUSH = (UK_FUNC_ID + 35), + + KBASE_FUNC_HWCNT_READER_SETUP = (UK_FUNC_ID + 36), + +#ifdef SUPPORT_MALI_NO_MALI + KBASE_FUNC_SET_PRFCNT_VALUES = (UK_FUNC_ID + 37), +#endif + + KBASE_FUNC_SOFT_EVENT_UPDATE = (UK_FUNC_ID + 38), + + KBASE_FUNC_MEM_JIT_INIT = (UK_FUNC_ID + 39), + + KBASE_FUNC_TLSTREAM_ACQUIRE = (UK_FUNC_ID + 40), + + KBASE_FUNC_MAX +}; + +#endif /* _KBASE_UKU_H_ */ + diff --git a/drivers/gpu/arm/midgard/mali_kbase_utility.c b/drivers/gpu/arm/midgard/mali_kbase_utility.c new file mode 100755 index 000000000000..be474ff87401 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_utility.c @@ -0,0 +1,33 @@ +/* + * + * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include + +bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry) +{ + struct list_head *pos = base->next; + + while (pos != base) { + if (pos == entry) + return true; + + pos = pos->next; + } + return false; +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_utility.h b/drivers/gpu/arm/midgard/mali_kbase_utility.h new file mode 100755 index 000000000000..fd7252dab0de --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_utility.h @@ -0,0 +1,37 @@ +/* + * + * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_UTILITY_H +#define _KBASE_UTILITY_H + +#ifndef _KBASE_H_ +#error "Don't include this file directly, use mali_kbase.h instead" +#endif + +/** Test whether the given list entry is a member of the given list. + * + * @param base The head of the list to be tested + * @param entry The list entry to be tested + * + * @return true if entry is a member of base + * false otherwise + */ +bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry); + +#endif /* _KBASE_UTILITY_H */ diff --git a/drivers/gpu/arm/midgard/mali_kbase_vinstr.c b/drivers/gpu/arm/midgard/mali_kbase_vinstr.c new file mode 100755 index 000000000000..8395568d0efa --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_vinstr.c @@ -0,0 +1,2070 @@ +/* + * + * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/*****************************************************************************/ + +/* Hwcnt reader API version */ +#define HWCNT_READER_API 1 + +/* The number of nanoseconds in a second. */ +#define NSECS_IN_SEC 1000000000ull /* ns */ + +/* The time resolution of dumping service. */ +#define DUMPING_RESOLUTION 500000ull /* ns */ + +/* The maximal supported number of dumping buffers. */ +#define MAX_BUFFER_COUNT 32 + +/* Size and number of hw counters blocks. */ +#define NR_CNT_BLOCKS_PER_GROUP 8 +#define NR_CNT_PER_BLOCK 64 +#define NR_BYTES_PER_CNT 4 +#define NR_BYTES_PER_HDR 16 +#define PRFCNT_EN_MASK_OFFSET 0x8 + +/*****************************************************************************/ + +enum { + SHADER_HWCNT_BM, + TILER_HWCNT_BM, + MMU_L2_HWCNT_BM, + JM_HWCNT_BM +}; + +enum vinstr_state { + VINSTR_IDLE, + VINSTR_DUMPING, + VINSTR_SUSPENDING, + VINSTR_SUSPENDED, + VINSTR_RESUMING +}; + +/** + * struct kbase_vinstr_context - vinstr context per device + * @lock: protects the entire vinstr context + * @kbdev: pointer to kbase device + * @kctx: pointer to kbase context + * @vmap: vinstr vmap for mapping hwcnt dump buffer + * @gpu_va: GPU hwcnt dump buffer address + * @cpu_va: the CPU side mapping of the hwcnt dump buffer + * @dump_size: size of the dump buffer in bytes + * @bitmap: current set of counters monitored, not always in sync + * with hardware + * @reprogram: when true, reprogram hwcnt block with the new set of + * counters + * @state: vinstr state + * @state_lock: protects information about vinstr state + * @suspend_waitq: notification queue to trigger state re-validation + * @suspend_cnt: reference counter of vinstr's suspend state + * @suspend_work: worker to execute on entering suspended state + * @resume_work: worker to execute on leaving suspended state + * @nclients: number of attached clients, pending or otherwise + * @waiting_clients: head of list of clients being periodically sampled + * @idle_clients: head of list of clients being idle + * @suspended_clients: head of list of clients being suspended + * @thread: periodic sampling thread + * @waitq: notification queue of sampling thread + * @request_pending: request for action for sampling thread + */ +struct kbase_vinstr_context { + struct mutex lock; + struct kbase_device *kbdev; + struct kbase_context *kctx; + + struct kbase_vmap_struct vmap; + u64 gpu_va; + void *cpu_va; + size_t dump_size; + u32 bitmap[4]; + bool reprogram; + + enum vinstr_state state; + struct spinlock state_lock; + wait_queue_head_t suspend_waitq; + unsigned int suspend_cnt; + struct work_struct suspend_work; + struct work_struct resume_work; + + u32 nclients; + struct list_head waiting_clients; + struct list_head idle_clients; + struct list_head suspended_clients; + + struct task_struct *thread; + wait_queue_head_t waitq; + atomic_t request_pending; +}; + +/** + * struct kbase_vinstr_client - a vinstr client attached to a vinstr context + * @vinstr_ctx: vinstr context client is attached to + * @list: node used to attach this client to list in vinstr context + * @buffer_count: number of buffers this client is using + * @event_mask: events this client reacts to + * @dump_size: size of one dump buffer in bytes + * @bitmap: bitmap request for JM, TILER, SHADER and MMU counters + * @legacy_buffer: userspace hwcnt dump buffer (legacy interface) + * @kernel_buffer: kernel hwcnt dump buffer (kernel client interface) + * @accum_buffer: temporary accumulation buffer for preserving counters + * @dump_time: next time this clients shall request hwcnt dump + * @dump_interval: interval between periodic hwcnt dumps + * @dump_buffers: kernel hwcnt dump buffers allocated by this client + * @dump_buffers_meta: metadata of dump buffers + * @meta_idx: index of metadata being accessed by userspace + * @read_idx: index of buffer read by userspace + * @write_idx: index of buffer being written by dumping service + * @waitq: client's notification queue + * @pending: when true, client has attached but hwcnt not yet updated + */ +struct kbase_vinstr_client { + struct kbase_vinstr_context *vinstr_ctx; + struct list_head list; + unsigned int buffer_count; + u32 event_mask; + size_t dump_size; + u32 bitmap[4]; + void __user *legacy_buffer; + void *kernel_buffer; + void *accum_buffer; + u64 dump_time; + u32 dump_interval; + char *dump_buffers; + struct kbase_hwcnt_reader_metadata *dump_buffers_meta; + atomic_t meta_idx; + atomic_t read_idx; + atomic_t write_idx; + wait_queue_head_t waitq; + bool pending; +}; + +/** + * struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer + * @hrtimer: high resolution timer + * @vinstr_ctx: vinstr context + */ +struct kbasep_vinstr_wake_up_timer { + struct hrtimer hrtimer; + struct kbase_vinstr_context *vinstr_ctx; +}; + +/*****************************************************************************/ + +static int kbasep_vinstr_service_task(void *data); + +static unsigned int kbasep_vinstr_hwcnt_reader_poll( + struct file *filp, + poll_table *wait); +static long kbasep_vinstr_hwcnt_reader_ioctl( + struct file *filp, + unsigned int cmd, + unsigned long arg); +static int kbasep_vinstr_hwcnt_reader_mmap( + struct file *filp, + struct vm_area_struct *vma); +static int kbasep_vinstr_hwcnt_reader_release( + struct inode *inode, + struct file *filp); + +/* The timeline stream file operations structure. */ +static const struct file_operations vinstr_client_fops = { + .poll = kbasep_vinstr_hwcnt_reader_poll, + .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, + .compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, + .mmap = kbasep_vinstr_hwcnt_reader_mmap, + .release = kbasep_vinstr_hwcnt_reader_release, +}; + +/*****************************************************************************/ + +static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_context *kctx = vinstr_ctx->kctx; + struct kbase_device *kbdev = kctx->kbdev; + struct kbase_uk_hwcnt_setup setup; + int err; + + setup.dump_buffer = vinstr_ctx->gpu_va; + setup.jm_bm = vinstr_ctx->bitmap[JM_HWCNT_BM]; + setup.tiler_bm = vinstr_ctx->bitmap[TILER_HWCNT_BM]; + setup.shader_bm = vinstr_ctx->bitmap[SHADER_HWCNT_BM]; + setup.mmu_l2_bm = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM]; + + /* Mark the context as active so the GPU is kept turned on */ + /* A suspend won't happen here, because we're in a syscall from a + * userspace thread. */ + kbase_pm_context_active(kbdev); + + /* Schedule the context in */ + kbasep_js_schedule_privileged_ctx(kbdev, kctx); + err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &setup); + if (err) { + /* Release the context. This had its own Power Manager Active + * reference */ + kbasep_js_release_privileged_ctx(kbdev, kctx); + + /* Also release our Power Manager Active reference */ + kbase_pm_context_idle(kbdev); + } + + return err; +} + +static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_context *kctx = vinstr_ctx->kctx; + struct kbase_device *kbdev = kctx->kbdev; + int err; + + err = kbase_instr_hwcnt_disable_internal(kctx); + if (err) { + dev_warn(kbdev->dev, "Failed to disable HW counters (ctx:%p)", + kctx); + return; + } + + /* Release the context. This had its own Power Manager Active reference. */ + kbasep_js_release_privileged_ctx(kbdev, kctx); + + /* Also release our Power Manager Active reference. */ + kbase_pm_context_idle(kbdev); + + dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx); +} + +static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx) +{ + disable_hwcnt(vinstr_ctx); + return enable_hwcnt(vinstr_ctx); +} + +static void hwcnt_bitmap_set(u32 dst[4], u32 src[4]) +{ + dst[JM_HWCNT_BM] = src[JM_HWCNT_BM]; + dst[TILER_HWCNT_BM] = src[TILER_HWCNT_BM]; + dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM]; + dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM]; +} + +static void hwcnt_bitmap_union(u32 dst[4], u32 src[4]) +{ + dst[JM_HWCNT_BM] |= src[JM_HWCNT_BM]; + dst[TILER_HWCNT_BM] |= src[TILER_HWCNT_BM]; + dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM]; + dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM]; +} + +size_t kbase_vinstr_dump_size(struct kbase_device *kbdev) +{ + size_t dump_size; + +#ifndef CONFIG_MALI_NO_MALI + if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) { + u32 nr_cg; + + nr_cg = kbdev->gpu_props.num_core_groups; + dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP * + NR_CNT_PER_BLOCK * + NR_BYTES_PER_CNT; + } else +#endif /* CONFIG_MALI_NO_MALI */ + { + /* assume v5 for now */ + base_gpu_props *props = &kbdev->gpu_props.props; + u32 nr_l2 = props->l2_props.num_l2_slices; + u64 core_mask = props->coherency_info.group[0].core_mask; + u32 nr_blocks = fls64(core_mask); + + /* JM and tiler counter blocks are always present */ + dump_size = (2 + nr_l2 + nr_blocks) * + NR_CNT_PER_BLOCK * + NR_BYTES_PER_CNT; + } + return dump_size; +} +KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size); + +static size_t kbasep_vinstr_dump_size_ctx( + struct kbase_vinstr_context *vinstr_ctx) +{ + return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev); +} + +static int kbasep_vinstr_map_kernel_dump_buffer( + struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_va_region *reg; + struct kbase_context *kctx = vinstr_ctx->kctx; + u64 flags, nr_pages; + + flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR; + vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx); + nr_pages = PFN_UP(vinstr_ctx->dump_size); + + reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, + &vinstr_ctx->gpu_va); + if (!reg) + return -ENOMEM; + + vinstr_ctx->cpu_va = kbase_vmap( + kctx, + vinstr_ctx->gpu_va, + vinstr_ctx->dump_size, + &vinstr_ctx->vmap); + if (!vinstr_ctx->cpu_va) { + kbase_mem_free(kctx, vinstr_ctx->gpu_va); + return -ENOMEM; + } + + return 0; +} + +static void kbasep_vinstr_unmap_kernel_dump_buffer( + struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_context *kctx = vinstr_ctx->kctx; + + kbase_vunmap(kctx, &vinstr_ctx->vmap); + kbase_mem_free(kctx, vinstr_ctx->gpu_va); +} + +/** + * kbasep_vinstr_create_kctx - create kernel context for vinstr + * @vinstr_ctx: vinstr context + * Return: zero on success + */ +static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_device *kbdev = vinstr_ctx->kbdev; + struct kbasep_kctx_list_element *element; + unsigned long flags; + bool enable_backend = false; + int err; + + vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true); + if (!vinstr_ctx->kctx) + return -ENOMEM; + + /* Map the master kernel dump buffer. The HW dumps the counters + * into this memory region. */ + err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx); + if (err) { + kbase_destroy_context(vinstr_ctx->kctx); + vinstr_ctx->kctx = NULL; + return err; + } + + /* Add kernel context to list of contexts associated with device. */ + element = kzalloc(sizeof(*element), GFP_KERNEL); + if (element) { + element->kctx = vinstr_ctx->kctx; + mutex_lock(&kbdev->kctx_list_lock); + list_add(&element->link, &kbdev->kctx_list); + + /* Inform timeline client about new context. + * Do this while holding the lock to avoid tracepoint + * being created in both body and summary stream. */ + KBASE_TLSTREAM_TL_NEW_CTX( + vinstr_ctx->kctx, + (u32)(vinstr_ctx->kctx->id), + (u32)(vinstr_ctx->kctx->tgid)); + + mutex_unlock(&kbdev->kctx_list_lock); + } else { + /* Don't treat this as a fail - just warn about it. */ + dev_warn(kbdev->dev, + "couldn't add kctx to kctx_list\n"); + } + + /* Don't enable hardware counters if vinstr is suspended. + * Note that vinstr resume code is run under vinstr context lock, + * lower layer will be enabled as needed on resume. */ + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + if (VINSTR_IDLE == vinstr_ctx->state) + enable_backend = true; + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + if (enable_backend) + err = enable_hwcnt(vinstr_ctx); + + if (err) { + kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); + kbase_destroy_context(vinstr_ctx->kctx); + if (element) { + mutex_lock(&kbdev->kctx_list_lock); + list_del(&element->link); + kfree(element); + mutex_unlock(&kbdev->kctx_list_lock); + } + KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); + vinstr_ctx->kctx = NULL; + return err; + } + + vinstr_ctx->thread = kthread_run( + kbasep_vinstr_service_task, + vinstr_ctx, + "mali_vinstr_service"); + if (!vinstr_ctx->thread) { + disable_hwcnt(vinstr_ctx); + kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); + kbase_destroy_context(vinstr_ctx->kctx); + if (element) { + mutex_lock(&kbdev->kctx_list_lock); + list_del(&element->link); + kfree(element); + mutex_unlock(&kbdev->kctx_list_lock); + } + KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); + vinstr_ctx->kctx = NULL; + return -EFAULT; + } + + return 0; +} + +/** + * kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context + * @vinstr_ctx: vinstr context + */ +static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_device *kbdev = vinstr_ctx->kbdev; + struct kbasep_kctx_list_element *element; + struct kbasep_kctx_list_element *tmp; + bool found = false; + + /* Release hw counters dumping resources. */ + vinstr_ctx->thread = NULL; + disable_hwcnt(vinstr_ctx); + kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); + kbase_destroy_context(vinstr_ctx->kctx); + + /* Remove kernel context from the device's contexts list. */ + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) { + if (element->kctx == vinstr_ctx->kctx) { + list_del(&element->link); + kfree(element); + found = true; + } + } + mutex_unlock(&kbdev->kctx_list_lock); + + if (!found) + dev_warn(kbdev->dev, "kctx not in kctx_list\n"); + + /* Inform timeline client about context destruction. */ + KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); + + vinstr_ctx->kctx = NULL; +} + +/** + * kbasep_vinstr_attach_client - Attach a client to the vinstr core + * @vinstr_ctx: vinstr context + * @buffer_count: requested number of dump buffers + * @bitmap: bitmaps describing which counters should be enabled + * @argp: pointer where notification descriptor shall be stored + * @kernel_buffer: pointer to kernel side buffer + * + * Return: vinstr opaque client handle or NULL on failure + */ +static struct kbase_vinstr_client *kbasep_vinstr_attach_client( + struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count, + u32 bitmap[4], void *argp, void *kernel_buffer) +{ + struct task_struct *thread = NULL; + struct kbase_vinstr_client *cli; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + if (buffer_count > MAX_BUFFER_COUNT + || (buffer_count & (buffer_count - 1))) + return NULL; + + cli = kzalloc(sizeof(*cli), GFP_KERNEL); + if (!cli) + return NULL; + + cli->vinstr_ctx = vinstr_ctx; + cli->buffer_count = buffer_count; + cli->event_mask = + (1 << BASE_HWCNT_READER_EVENT_MANUAL) | + (1 << BASE_HWCNT_READER_EVENT_PERIODIC); + cli->pending = true; + + hwcnt_bitmap_set(cli->bitmap, bitmap); + + mutex_lock(&vinstr_ctx->lock); + + hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap); + vinstr_ctx->reprogram = true; + + /* If this is the first client, create the vinstr kbase + * context. This context is permanently resident until the + * last client exits. */ + if (!vinstr_ctx->nclients) { + hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap); + if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0) + goto error; + + vinstr_ctx->reprogram = false; + cli->pending = false; + } + + /* The GPU resets the counter block every time there is a request + * to dump it. We need a per client kernel buffer for accumulating + * the counters. */ + cli->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx); + cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL); + if (!cli->accum_buffer) + goto error; + + /* Prepare buffers. */ + if (cli->buffer_count) { + int *fd = (int *)argp; + size_t tmp; + + /* Allocate area for buffers metadata storage. */ + tmp = sizeof(struct kbase_hwcnt_reader_metadata) * + cli->buffer_count; + cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL); + if (!cli->dump_buffers_meta) + goto error; + + /* Allocate required number of dumping buffers. */ + cli->dump_buffers = (char *)__get_free_pages( + GFP_KERNEL | __GFP_ZERO, + get_order(cli->dump_size * cli->buffer_count)); + if (!cli->dump_buffers) + goto error; + + /* Create descriptor for user-kernel data exchange. */ + *fd = anon_inode_getfd( + "[mali_vinstr_desc]", + &vinstr_client_fops, + cli, + O_RDONLY | O_CLOEXEC); + if (0 > *fd) + goto error; + } else if (kernel_buffer) { + cli->kernel_buffer = kernel_buffer; + } else { + cli->legacy_buffer = (void __user *)argp; + } + + atomic_set(&cli->read_idx, 0); + atomic_set(&cli->meta_idx, 0); + atomic_set(&cli->write_idx, 0); + init_waitqueue_head(&cli->waitq); + + vinstr_ctx->nclients++; + list_add(&cli->list, &vinstr_ctx->idle_clients); + + mutex_unlock(&vinstr_ctx->lock); + + return cli; + +error: + kfree(cli->dump_buffers_meta); + if (cli->dump_buffers) + free_pages( + (unsigned long)cli->dump_buffers, + get_order(cli->dump_size * cli->buffer_count)); + kfree(cli->accum_buffer); + if (!vinstr_ctx->nclients && vinstr_ctx->kctx) { + thread = vinstr_ctx->thread; + kbasep_vinstr_destroy_kctx(vinstr_ctx); + } + kfree(cli); + + mutex_unlock(&vinstr_ctx->lock); + + /* Thread must be stopped after lock is released. */ + if (thread) + kthread_stop(thread); + + return NULL; +} + +void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli) +{ + struct kbase_vinstr_context *vinstr_ctx; + struct kbase_vinstr_client *iter, *tmp; + struct task_struct *thread = NULL; + u32 zerobitmap[4] = { 0 }; + int cli_found = 0; + + KBASE_DEBUG_ASSERT(cli); + vinstr_ctx = cli->vinstr_ctx; + KBASE_DEBUG_ASSERT(vinstr_ctx); + + mutex_lock(&vinstr_ctx->lock); + + list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) { + if (iter == cli) { + vinstr_ctx->reprogram = true; + cli_found = 1; + list_del(&iter->list); + break; + } + } + if (!cli_found) { + list_for_each_entry_safe( + iter, tmp, &vinstr_ctx->waiting_clients, list) { + if (iter == cli) { + vinstr_ctx->reprogram = true; + cli_found = 1; + list_del(&iter->list); + break; + } + } + } + KBASE_DEBUG_ASSERT(cli_found); + + kfree(cli->dump_buffers_meta); + free_pages( + (unsigned long)cli->dump_buffers, + get_order(cli->dump_size * cli->buffer_count)); + kfree(cli->accum_buffer); + kfree(cli); + + vinstr_ctx->nclients--; + if (!vinstr_ctx->nclients) { + thread = vinstr_ctx->thread; + kbasep_vinstr_destroy_kctx(vinstr_ctx); + } + + /* Rebuild context bitmap now that the client has detached */ + hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap); + list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) + hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap); + list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) + hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap); + + mutex_unlock(&vinstr_ctx->lock); + + /* Thread must be stopped after lock is released. */ + if (thread) + kthread_stop(thread); +} +KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client); + +/* Accumulate counters in the dump buffer */ +static void accum_dump_buffer(void *dst, void *src, size_t dump_size) +{ + size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT; + u32 *d = dst; + u32 *s = src; + size_t i, j; + + for (i = 0; i < dump_size; i += block_size) { + /* skip over the header block */ + d += NR_BYTES_PER_HDR / sizeof(u32); + s += NR_BYTES_PER_HDR / sizeof(u32); + for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) { + /* saturate result if addition would result in wraparound */ + if (U32_MAX - *d < *s) + *d = U32_MAX; + else + *d += *s; + d++; + s++; + } + } +} + +/* This is the Midgard v4 patch function. It copies the headers for each + * of the defined blocks from the master kernel buffer and then patches up + * the performance counter enable mask for each of the blocks to exclude + * counters that were not requested by the client. */ +static void patch_dump_buffer_hdr_v4( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_vinstr_client *cli) +{ + u32 *mask; + u8 *dst = cli->accum_buffer; + u8 *src = vinstr_ctx->cpu_va; + u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups; + size_t i, group_size, group; + enum { + SC0_BASE = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + SC1_BASE = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + SC2_BASE = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + SC3_BASE = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + TILER_BASE = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, + JM_BASE = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT + }; + + group_size = NR_CNT_BLOCKS_PER_GROUP * + NR_CNT_PER_BLOCK * + NR_BYTES_PER_CNT; + for (i = 0; i < nr_cg; i++) { + group = i * group_size; + /* copy shader core headers */ + memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE], + NR_BYTES_PER_HDR); + memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE], + NR_BYTES_PER_HDR); + memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE], + NR_BYTES_PER_HDR); + memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE], + NR_BYTES_PER_HDR); + + /* copy tiler header */ + memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE], + NR_BYTES_PER_HDR); + + /* copy mmu header */ + memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE], + NR_BYTES_PER_HDR); + + /* copy job manager header */ + memcpy(&dst[group + JM_BASE], &src[group + JM_BASE], + NR_BYTES_PER_HDR); + + /* patch the shader core enable mask */ + mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[SHADER_HWCNT_BM]; + mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[SHADER_HWCNT_BM]; + mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[SHADER_HWCNT_BM]; + mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[SHADER_HWCNT_BM]; + + /* patch the tiler core enable mask */ + mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[TILER_HWCNT_BM]; + + /* patch the mmu core enable mask */ + mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[MMU_L2_HWCNT_BM]; + + /* patch the job manager enable mask */ + mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[JM_HWCNT_BM]; + } +} + +/* This is the Midgard v5 patch function. It copies the headers for each + * of the defined blocks from the master kernel buffer and then patches up + * the performance counter enable mask for each of the blocks to exclude + * counters that were not requested by the client. */ +static void patch_dump_buffer_hdr_v5( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_vinstr_client *cli) +{ + struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev; + u32 i, nr_l2; + u64 core_mask; + u32 *mask; + u8 *dst = cli->accum_buffer; + u8 *src = vinstr_ctx->cpu_va; + size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT; + + /* copy and patch job manager header */ + memcpy(dst, src, NR_BYTES_PER_HDR); + mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[JM_HWCNT_BM]; + dst += block_size; + src += block_size; + + /* copy and patch tiler header */ + memcpy(dst, src, NR_BYTES_PER_HDR); + mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[TILER_HWCNT_BM]; + dst += block_size; + src += block_size; + + /* copy and patch MMU/L2C headers */ + nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices; + for (i = 0; i < nr_l2; i++) { + memcpy(dst, src, NR_BYTES_PER_HDR); + mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[MMU_L2_HWCNT_BM]; + dst += block_size; + src += block_size; + } + + /* copy and patch shader core headers */ + core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; + while (0ull != core_mask) { + memcpy(dst, src, NR_BYTES_PER_HDR); + if (0ull != (core_mask & 1ull)) { + /* if block is not reserved update header */ + mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; + *mask &= cli->bitmap[SHADER_HWCNT_BM]; + } + dst += block_size; + src += block_size; + + core_mask >>= 1; + } +} + +/** + * accum_clients - accumulate dumped hw counters for all known clients + * @vinstr_ctx: vinstr context + */ +static void accum_clients(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_vinstr_client *iter; + int v4 = 0; + +#ifndef CONFIG_MALI_NO_MALI + v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4); +#endif + + list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) { + /* Don't bother accumulating clients whose hwcnt requests + * have not yet been honoured. */ + if (iter->pending) + continue; + if (v4) + patch_dump_buffer_hdr_v4(vinstr_ctx, iter); + else + patch_dump_buffer_hdr_v5(vinstr_ctx, iter); + accum_dump_buffer( + iter->accum_buffer, + vinstr_ctx->cpu_va, + iter->dump_size); + } + list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) { + /* Don't bother accumulating clients whose hwcnt requests + * have not yet been honoured. */ + if (iter->pending) + continue; + if (v4) + patch_dump_buffer_hdr_v4(vinstr_ctx, iter); + else + patch_dump_buffer_hdr_v5(vinstr_ctx, iter); + accum_dump_buffer( + iter->accum_buffer, + vinstr_ctx->cpu_va, + iter->dump_size); + } +} + +/*****************************************************************************/ + +/** + * kbasep_vinstr_get_timestamp - return timestamp + * + * Function returns timestamp value based on raw monotonic timer. Value will + * wrap around zero in case of overflow. + * + * Return: timestamp value + */ +static u64 kbasep_vinstr_get_timestamp(void) +{ + struct timespec64 ts; + + ktime_get_raw_ts64(&ts); + return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; +} + +/** + * kbasep_vinstr_add_dump_request - register client's dumping request + * @cli: requesting client + * @waiting_clients: list of pending dumping requests + */ +static void kbasep_vinstr_add_dump_request( + struct kbase_vinstr_client *cli, + struct list_head *waiting_clients) +{ + struct kbase_vinstr_client *tmp; + + if (list_empty(waiting_clients)) { + list_add(&cli->list, waiting_clients); + return; + } + list_for_each_entry(tmp, waiting_clients, list) { + if (tmp->dump_time > cli->dump_time) { + list_add_tail(&cli->list, &tmp->list); + return; + } + } + list_add_tail(&cli->list, waiting_clients); +} + +/** + * kbasep_vinstr_collect_and_accumulate - collect hw counters via low level + * dump and accumulate them for known + * clients + * @vinstr_ctx: vinstr context + * @timestamp: pointer where collection timestamp will be recorded + * + * Return: zero on success + */ +static int kbasep_vinstr_collect_and_accumulate( + struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp) +{ + unsigned long flags; + int rcode; + +#ifdef CONFIG_MALI_NO_MALI + /* The dummy model needs the CPU mapping. */ + gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va); +#endif + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + if (VINSTR_IDLE != vinstr_ctx->state) { + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + return -EAGAIN; + } else { + vinstr_ctx->state = VINSTR_DUMPING; + } + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + /* Request HW counters dump. + * Disable preemption to make dump timestamp more accurate. */ + preempt_disable(); + *timestamp = kbasep_vinstr_get_timestamp(); + rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx); + preempt_enable(); + + if (!rcode) + rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx); + WARN_ON(rcode); + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + switch (vinstr_ctx->state) + { + case VINSTR_SUSPENDING: + schedule_work(&vinstr_ctx->suspend_work); + break; + case VINSTR_DUMPING: + vinstr_ctx->state = VINSTR_IDLE; + wake_up_all(&vinstr_ctx->suspend_waitq); + break; + default: + break; + } + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + /* Accumulate values of collected counters. */ + if (!rcode) + accum_clients(vinstr_ctx); + + return rcode; +} + +/** + * kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel + * buffer + * @cli: requesting client + * @timestamp: timestamp when counters were collected + * @event_id: id of event that caused triggered counters collection + * + * Return: zero on success + */ +static int kbasep_vinstr_fill_dump_buffer( + struct kbase_vinstr_client *cli, u64 timestamp, + enum base_hwcnt_reader_event event_id) +{ + unsigned int write_idx = atomic_read(&cli->write_idx); + unsigned int read_idx = atomic_read(&cli->read_idx); + + struct kbase_hwcnt_reader_metadata *meta; + void *buffer; + + /* Check if there is a place to copy HWC block into. */ + if (write_idx - read_idx == cli->buffer_count) + return -1; + write_idx %= cli->buffer_count; + + /* Fill in dump buffer and its metadata. */ + buffer = &cli->dump_buffers[write_idx * cli->dump_size]; + meta = &cli->dump_buffers_meta[write_idx]; + meta->timestamp = timestamp; + meta->event_id = event_id; + meta->buffer_idx = write_idx; + memcpy(buffer, cli->accum_buffer, cli->dump_size); + return 0; +} + +/** + * kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer + * allocated in userspace + * @cli: requesting client + * + * Return: zero on success + * + * This is part of legacy ioctl interface. + */ +static int kbasep_vinstr_fill_dump_buffer_legacy( + struct kbase_vinstr_client *cli) +{ + void __user *buffer = cli->legacy_buffer; + int rcode; + + /* Copy data to user buffer. */ + rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size); + if (rcode) + pr_warn("error while copying buffer to user\n"); + return rcode; +} + +/** + * kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer + * allocated in kernel space + * @cli: requesting client + * + * Return: zero on success + * + * This is part of the kernel client interface. + */ +static int kbasep_vinstr_fill_dump_buffer_kernel( + struct kbase_vinstr_client *cli) +{ + memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size); + + return 0; +} + +/** + * kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst + * @vinstr_ctx: vinstr context + */ +static void kbasep_vinstr_reprogram( + struct kbase_vinstr_context *vinstr_ctx) +{ + unsigned long flags; + bool suspended = false; + + /* Don't enable hardware counters if vinstr is suspended. */ + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + if (VINSTR_IDLE != vinstr_ctx->state) + suspended = true; + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + if (suspended) + return; + + /* Change to suspended state is done while holding vinstr context + * lock. Below code will then no re-enable the instrumentation. */ + + if (vinstr_ctx->reprogram) { + struct kbase_vinstr_client *iter; + + if (!reprogram_hwcnt(vinstr_ctx)) { + vinstr_ctx->reprogram = false; + list_for_each_entry( + iter, + &vinstr_ctx->idle_clients, + list) + iter->pending = false; + list_for_each_entry( + iter, + &vinstr_ctx->waiting_clients, + list) + iter->pending = false; + } + } +} + +/** + * kbasep_vinstr_update_client - copy accumulated counters to user readable + * buffer and notify the user + * @cli: requesting client + * @timestamp: timestamp when counters were collected + * @event_id: id of event that caused triggered counters collection + * + * Return: zero on success + */ +static int kbasep_vinstr_update_client( + struct kbase_vinstr_client *cli, u64 timestamp, + enum base_hwcnt_reader_event event_id) +{ + int rcode = 0; + + /* Copy collected counters to user readable buffer. */ + if (cli->buffer_count) + rcode = kbasep_vinstr_fill_dump_buffer( + cli, timestamp, event_id); + else if (cli->kernel_buffer) + rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli); + else + rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli); + + if (rcode) + goto exit; + + + /* Notify client. Make sure all changes to memory are visible. */ + wmb(); + atomic_inc(&cli->write_idx); + wake_up_interruptible(&cli->waitq); + + /* Prepare for next request. */ + memset(cli->accum_buffer, 0, cli->dump_size); + +exit: + return rcode; +} + +/** + * kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function + * + * @hrtimer: high resolution timer + * + * Return: High resolution timer restart enum. + */ +static enum hrtimer_restart kbasep_vinstr_wake_up_callback( + struct hrtimer *hrtimer) +{ + struct kbasep_vinstr_wake_up_timer *timer = + container_of( + hrtimer, + struct kbasep_vinstr_wake_up_timer, + hrtimer); + + KBASE_DEBUG_ASSERT(timer); + + atomic_set(&timer->vinstr_ctx->request_pending, 1); + wake_up_all(&timer->vinstr_ctx->waitq); + + return HRTIMER_NORESTART; +} + +#ifdef CONFIG_DEBUG_OBJECT_TIMERS +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) +/** + * kbase_destroy_hrtimer_on_stack - kernel's destroy_hrtimer_on_stack(), + * rewritten + * + * @timer: high resolution timer + * + * destroy_hrtimer_on_stack() was exported only for 4.7.0 kernel so for + * earlier kernel versions it is not possible to call it explicitly. + * Since this function must accompany hrtimer_init_on_stack(), which + * has to be used for hrtimer initialization if CONFIG_DEBUG_OBJECT_TIMERS + * is defined in order to avoid the warning about object on stack not being + * annotated, we rewrite it here to be used for earlier kernel versions. + */ +static void kbase_destroy_hrtimer_on_stack(struct hrtimer *timer) +{ + debug_object_free(timer, &hrtimer_debug_descr); +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) */ +#endif /* CONFIG_DEBUG_OBJECT_TIMERS */ + +/** + * kbasep_vinstr_service_task - HWC dumping service thread + * + * @data: Pointer to vinstr context structure. + * + * Return: Always returns zero. + */ +static int kbasep_vinstr_service_task(void *data) +{ + struct kbase_vinstr_context *vinstr_ctx = data; + struct kbasep_vinstr_wake_up_timer timer; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + hrtimer_init_on_stack(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + + timer.hrtimer.function = kbasep_vinstr_wake_up_callback; + timer.vinstr_ctx = vinstr_ctx; + + while (!kthread_should_stop()) { + struct kbase_vinstr_client *cli = NULL; + struct kbase_vinstr_client *tmp; + int rcode; + + u64 timestamp = kbasep_vinstr_get_timestamp(); + u64 dump_time = 0; + struct list_head expired_requests; + + /* Hold lock while performing operations on lists of clients. */ + mutex_lock(&vinstr_ctx->lock); + + /* Closing thread must not interact with client requests. */ + if (current == vinstr_ctx->thread) { + atomic_set(&vinstr_ctx->request_pending, 0); + + if (!list_empty(&vinstr_ctx->waiting_clients)) { + cli = list_first_entry( + &vinstr_ctx->waiting_clients, + struct kbase_vinstr_client, + list); + dump_time = cli->dump_time; + } + } + + if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) { + mutex_unlock(&vinstr_ctx->lock); + + /* Sleep until next dumping event or service request. */ + if (cli) { + u64 diff = dump_time - timestamp; + + hrtimer_start( + &timer.hrtimer, + ns_to_ktime(diff), + HRTIMER_MODE_REL); + } + wait_event( + vinstr_ctx->waitq, + atomic_read( + &vinstr_ctx->request_pending) || + kthread_should_stop()); + hrtimer_cancel(&timer.hrtimer); + continue; + } + + rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, + ×tamp); + + INIT_LIST_HEAD(&expired_requests); + + /* Find all expired requests. */ + list_for_each_entry_safe( + cli, + tmp, + &vinstr_ctx->waiting_clients, + list) { + s64 tdiff = + (s64)(timestamp + DUMPING_RESOLUTION) - + (s64)cli->dump_time; + if (tdiff >= 0ll) { + list_del(&cli->list); + list_add(&cli->list, &expired_requests); + } else { + break; + } + } + + /* Fill data for each request found. */ + list_for_each_entry_safe(cli, tmp, &expired_requests, list) { + /* Ensure that legacy buffer will not be used from + * this kthread context. */ + BUG_ON(0 == cli->buffer_count); + /* Expect only periodically sampled clients. */ + BUG_ON(0 == cli->dump_interval); + + if (!rcode) + kbasep_vinstr_update_client( + cli, + timestamp, + BASE_HWCNT_READER_EVENT_PERIODIC); + + /* Set new dumping time. Drop missed probing times. */ + do { + cli->dump_time += cli->dump_interval; + } while (cli->dump_time < timestamp); + + list_del(&cli->list); + kbasep_vinstr_add_dump_request( + cli, + &vinstr_ctx->waiting_clients); + } + + /* Reprogram counters set if required. */ + kbasep_vinstr_reprogram(vinstr_ctx); + + mutex_unlock(&vinstr_ctx->lock); + } + +#ifdef CONFIG_DEBUG_OBJECTS_TIMERS +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) + kbase_destroy_hrtimer_on_stack(&timer.hrtimer); +#else + destroy_hrtimer_on_stack(&timer.hrtimer); +#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) */ +#endif /* CONFIG_DEBUG_OBJECTS_TIMERS */ + + return 0; +} + +/*****************************************************************************/ + +/** + * kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers + * @cli: pointer to vinstr client structure + * + * Return: non-zero if client has at least one dumping buffer filled that was + * not notified to user yet + */ +static int kbasep_vinstr_hwcnt_reader_buffer_ready( + struct kbase_vinstr_client *cli) +{ + KBASE_DEBUG_ASSERT(cli); + return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx); +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @buffer: pointer to userspace buffer + * @size: size of buffer + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( + struct kbase_vinstr_client *cli, void __user *buffer, + size_t size) +{ + unsigned int meta_idx = atomic_read(&cli->meta_idx); + unsigned int idx = meta_idx % cli->buffer_count; + + struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx]; + + /* Metadata sanity check. */ + KBASE_DEBUG_ASSERT(idx == meta->buffer_idx); + + if (sizeof(struct kbase_hwcnt_reader_metadata) != size) + return -EINVAL; + + /* Check if there is any buffer available. */ + if (atomic_read(&cli->write_idx) == meta_idx) + return -EAGAIN; + + /* Check if previously taken buffer was put back. */ + if (atomic_read(&cli->read_idx) != meta_idx) + return -EBUSY; + + /* Copy next available buffer's metadata to user. */ + if (copy_to_user(buffer, meta, size)) + return -EFAULT; + + atomic_inc(&cli->meta_idx); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @buffer: pointer to userspace buffer + * @size: size of buffer + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( + struct kbase_vinstr_client *cli, void __user *buffer, + size_t size) +{ + unsigned int read_idx = atomic_read(&cli->read_idx); + unsigned int idx = read_idx % cli->buffer_count; + + struct kbase_hwcnt_reader_metadata meta; + + if (sizeof(struct kbase_hwcnt_reader_metadata) != size) + return -EINVAL; + + /* Check if any buffer was taken. */ + if (atomic_read(&cli->meta_idx) == read_idx) + return -EPERM; + + /* Check if correct buffer is put back. */ + if (copy_from_user(&meta, buffer, size)) + return -EFAULT; + if (idx != meta.buffer_idx) + return -EINVAL; + + atomic_inc(&cli->read_idx); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @interval: periodic dumping interval (disable periodic dumping if zero) + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval( + struct kbase_vinstr_client *cli, u32 interval) +{ + struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + mutex_lock(&vinstr_ctx->lock); + + list_del(&cli->list); + + cli->dump_interval = interval; + + /* If interval is non-zero, enable periodic dumping for this client. */ + if (cli->dump_interval) { + if (DUMPING_RESOLUTION > cli->dump_interval) + cli->dump_interval = DUMPING_RESOLUTION; + cli->dump_time = + kbasep_vinstr_get_timestamp() + cli->dump_interval; + + kbasep_vinstr_add_dump_request( + cli, &vinstr_ctx->waiting_clients); + + atomic_set(&vinstr_ctx->request_pending, 1); + wake_up_all(&vinstr_ctx->waitq); + } else { + list_add(&cli->list, &vinstr_ctx->idle_clients); + } + + mutex_unlock(&vinstr_ctx->lock); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id + * @event_id: id of event + * Return: event_mask or zero if event is not supported or maskable + */ +static u32 kbasep_vinstr_hwcnt_reader_event_mask( + enum base_hwcnt_reader_event event_id) +{ + u32 event_mask = 0; + + switch (event_id) { + case BASE_HWCNT_READER_EVENT_PREJOB: + case BASE_HWCNT_READER_EVENT_POSTJOB: + /* These event are maskable. */ + event_mask = (1 << event_id); + break; + + case BASE_HWCNT_READER_EVENT_MANUAL: + case BASE_HWCNT_READER_EVENT_PERIODIC: + /* These event are non-maskable. */ + default: + /* These event are not supported. */ + break; + } + + return event_mask; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @event_id: id of event to enable + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event( + struct kbase_vinstr_client *cli, + enum base_hwcnt_reader_event event_id) +{ + struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; + u32 event_mask; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id); + if (!event_mask) + return -EINVAL; + + mutex_lock(&vinstr_ctx->lock); + cli->event_mask |= event_mask; + mutex_unlock(&vinstr_ctx->lock); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @event_id: id of event to disable + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event( + struct kbase_vinstr_client *cli, + enum base_hwcnt_reader_event event_id) +{ + struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; + u32 event_mask; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id); + if (!event_mask) + return -EINVAL; + + mutex_lock(&vinstr_ctx->lock); + cli->event_mask &= ~event_mask; + mutex_unlock(&vinstr_ctx->lock); + + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command + * @cli: pointer to vinstr client structure + * @hwver: pointer to user buffer where hw version will be stored + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( + struct kbase_vinstr_client *cli, u32 __user *hwver) +{ +#ifndef CONFIG_MALI_NO_MALI + struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; +#endif + + u32 ver = 5; + +#ifndef CONFIG_MALI_NO_MALI + KBASE_DEBUG_ASSERT(vinstr_ctx); + if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4)) + ver = 4; +#endif + + return put_user(ver, hwver); +} + +/** + * kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl + * @filp: pointer to file structure + * @cmd: user command + * @arg: command's argument + * + * Return: zero on success + */ +static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg) +{ + long rcode = 0; + struct kbase_vinstr_client *cli; + + KBASE_DEBUG_ASSERT(filp); + + cli = filp->private_data; + KBASE_DEBUG_ASSERT(cli); + + if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd))) + return -EINVAL; + + switch (cmd) { + case KBASE_HWCNT_READER_GET_API_VERSION: + rcode = put_user(HWCNT_READER_API, (u32 __user *)arg); + break; + case KBASE_HWCNT_READER_GET_HWVER: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( + cli, (u32 __user *)arg); + break; + case KBASE_HWCNT_READER_GET_BUFFER_SIZE: + KBASE_DEBUG_ASSERT(cli->vinstr_ctx); + rcode = put_user( + (u32)cli->vinstr_ctx->dump_size, + (u32 __user *)arg); + break; + case KBASE_HWCNT_READER_DUMP: + rcode = kbase_vinstr_hwc_dump( + cli, BASE_HWCNT_READER_EVENT_MANUAL); + break; + case KBASE_HWCNT_READER_CLEAR: + rcode = kbase_vinstr_hwc_clear(cli); + break; + case KBASE_HWCNT_READER_GET_BUFFER: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( + cli, (void __user *)arg, _IOC_SIZE(cmd)); + break; + case KBASE_HWCNT_READER_PUT_BUFFER: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( + cli, (void __user *)arg, _IOC_SIZE(cmd)); + break; + case KBASE_HWCNT_READER_SET_INTERVAL: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval( + cli, (u32)arg); + break; + case KBASE_HWCNT_READER_ENABLE_EVENT: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event( + cli, (enum base_hwcnt_reader_event)arg); + break; + case KBASE_HWCNT_READER_DISABLE_EVENT: + rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event( + cli, (enum base_hwcnt_reader_event)arg); + break; + default: + rcode = -EINVAL; + break; + } + + return rcode; +} + +/** + * kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll + * @filp: pointer to file structure + * @wait: pointer to poll table + * Return: POLLIN if data can be read without blocking, otherwise zero + */ +static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp, + poll_table *wait) +{ + struct kbase_vinstr_client *cli; + + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(wait); + + cli = filp->private_data; + KBASE_DEBUG_ASSERT(cli); + + poll_wait(filp, &cli->waitq, wait); + if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli)) + return POLLIN; + return 0; +} + +/** + * kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap + * @filp: pointer to file structure + * @vma: pointer to vma structure + * Return: zero on success + */ +static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp, + struct vm_area_struct *vma) +{ + struct kbase_vinstr_client *cli; + unsigned long size, addr, pfn, offset; + unsigned long vm_size = vma->vm_end - vma->vm_start; + + KBASE_DEBUG_ASSERT(filp); + KBASE_DEBUG_ASSERT(vma); + + cli = filp->private_data; + KBASE_DEBUG_ASSERT(cli); + + size = cli->buffer_count * cli->dump_size; + + if (vma->vm_pgoff > (size >> PAGE_SHIFT)) + return -EINVAL; + + offset = vma->vm_pgoff << PAGE_SHIFT; + if (vm_size > size - offset) + return -EINVAL; + + addr = __pa((unsigned long)cli->dump_buffers + offset); + pfn = addr >> PAGE_SHIFT; + + return remap_pfn_range( + vma, + vma->vm_start, + pfn, + vm_size, + vma->vm_page_prot); +} + +/** + * kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release + * @inode: pointer to inode structure + * @filp: pointer to file structure + * Return always return zero + */ +static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode, + struct file *filp) +{ + struct kbase_vinstr_client *cli; + + KBASE_DEBUG_ASSERT(inode); + KBASE_DEBUG_ASSERT(filp); + + cli = filp->private_data; + KBASE_DEBUG_ASSERT(cli); + + kbase_vinstr_detach_client(cli); + return 0; +} + +/*****************************************************************************/ + +/** + * kbasep_vinstr_kick_scheduler - trigger scheduler cycle + * @kbdev: pointer to kbase device structure + */ +static void kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev) +{ + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + unsigned long flags; + + down(&js_devdata->schedule_sem); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_backend_slot_update(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + up(&js_devdata->schedule_sem); +} + +/** + * kbasep_vinstr_suspend_worker - worker suspending vinstr module + * @data: pointer to work structure + */ +static void kbasep_vinstr_suspend_worker(struct work_struct *data) +{ + struct kbase_vinstr_context *vinstr_ctx; + unsigned long flags; + + vinstr_ctx = container_of(data, struct kbase_vinstr_context, + suspend_work); + + mutex_lock(&vinstr_ctx->lock); + + if (vinstr_ctx->kctx) + disable_hwcnt(vinstr_ctx); + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + vinstr_ctx->state = VINSTR_SUSPENDED; + wake_up_all(&vinstr_ctx->suspend_waitq); + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + mutex_unlock(&vinstr_ctx->lock); + + /* Kick GPU scheduler to allow entering protected mode. + * This must happen after vinstr was suspended. */ + kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev); +} + +/** + * kbasep_vinstr_suspend_worker - worker resuming vinstr module + * @data: pointer to work structure + */ +static void kbasep_vinstr_resume_worker(struct work_struct *data) +{ + struct kbase_vinstr_context *vinstr_ctx; + unsigned long flags; + + vinstr_ctx = container_of(data, struct kbase_vinstr_context, + resume_work); + + mutex_lock(&vinstr_ctx->lock); + + if (vinstr_ctx->kctx) + enable_hwcnt(vinstr_ctx); + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + vinstr_ctx->state = VINSTR_IDLE; + wake_up_all(&vinstr_ctx->suspend_waitq); + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + mutex_unlock(&vinstr_ctx->lock); + + /* Kick GPU scheduler to allow entering protected mode. + * Note that scheduler state machine might requested re-entry to + * protected mode before vinstr was resumed. + * This must happen after vinstr was release. */ + kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev); +} + +/*****************************************************************************/ + +struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev) +{ + struct kbase_vinstr_context *vinstr_ctx; + + vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL); + if (!vinstr_ctx) + return NULL; + + INIT_LIST_HEAD(&vinstr_ctx->idle_clients); + INIT_LIST_HEAD(&vinstr_ctx->waiting_clients); + mutex_init(&vinstr_ctx->lock); + spin_lock_init(&vinstr_ctx->state_lock); + vinstr_ctx->kbdev = kbdev; + vinstr_ctx->thread = NULL; + vinstr_ctx->state = VINSTR_IDLE; + vinstr_ctx->suspend_cnt = 0; + INIT_WORK(&vinstr_ctx->suspend_work, kbasep_vinstr_suspend_worker); + INIT_WORK(&vinstr_ctx->resume_work, kbasep_vinstr_resume_worker); + init_waitqueue_head(&vinstr_ctx->suspend_waitq); + + atomic_set(&vinstr_ctx->request_pending, 0); + init_waitqueue_head(&vinstr_ctx->waitq); + + return vinstr_ctx; +} + +void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx) +{ + struct kbase_vinstr_client *cli; + + /* Stop service thread first. */ + if (vinstr_ctx->thread) + kthread_stop(vinstr_ctx->thread); + + /* Wait for workers. */ + flush_work(&vinstr_ctx->suspend_work); + flush_work(&vinstr_ctx->resume_work); + + while (1) { + struct list_head *list = &vinstr_ctx->idle_clients; + + if (list_empty(list)) { + list = &vinstr_ctx->waiting_clients; + if (list_empty(list)) + break; + } + + cli = list_first_entry(list, struct kbase_vinstr_client, list); + list_del(&cli->list); + kfree(cli->accum_buffer); + kfree(cli); + vinstr_ctx->nclients--; + } + KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients); + if (vinstr_ctx->kctx) + kbasep_vinstr_destroy_kctx(vinstr_ctx); + kfree(vinstr_ctx); +} + +int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx, + struct kbase_uk_hwcnt_reader_setup *setup) +{ + struct kbase_vinstr_client *cli; + u32 bitmap[4]; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + KBASE_DEBUG_ASSERT(setup); + KBASE_DEBUG_ASSERT(setup->buffer_count); + + bitmap[SHADER_HWCNT_BM] = setup->shader_bm; + bitmap[TILER_HWCNT_BM] = setup->tiler_bm; + bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; + bitmap[JM_HWCNT_BM] = setup->jm_bm; + + cli = kbasep_vinstr_attach_client( + vinstr_ctx, + setup->buffer_count, + bitmap, + &setup->fd, + NULL); + + if (!cli) + return -ENOMEM; + + return 0; +} + +int kbase_vinstr_legacy_hwc_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_vinstr_client **cli, + struct kbase_uk_hwcnt_setup *setup) +{ + KBASE_DEBUG_ASSERT(vinstr_ctx); + KBASE_DEBUG_ASSERT(setup); + KBASE_DEBUG_ASSERT(cli); + + if (setup->dump_buffer) { + u32 bitmap[4]; + + bitmap[SHADER_HWCNT_BM] = setup->shader_bm; + bitmap[TILER_HWCNT_BM] = setup->tiler_bm; + bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; + bitmap[JM_HWCNT_BM] = setup->jm_bm; + + if (*cli) + return -EBUSY; + + *cli = kbasep_vinstr_attach_client( + vinstr_ctx, + 0, + bitmap, + (void *)(long)setup->dump_buffer, + NULL); + + if (!(*cli)) + return -ENOMEM; + } else { + if (!*cli) + return -EINVAL; + + kbase_vinstr_detach_client(*cli); + *cli = NULL; + } + + return 0; +} + +struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_uk_hwcnt_reader_setup *setup, + void *kernel_buffer) +{ + u32 bitmap[4]; + + if (!vinstr_ctx || !setup || !kernel_buffer) + return NULL; + + bitmap[SHADER_HWCNT_BM] = setup->shader_bm; + bitmap[TILER_HWCNT_BM] = setup->tiler_bm; + bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; + bitmap[JM_HWCNT_BM] = setup->jm_bm; + + return kbasep_vinstr_attach_client( + vinstr_ctx, + 0, + bitmap, + NULL, + kernel_buffer); +} +KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup); + +int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli, + enum base_hwcnt_reader_event event_id) +{ + int rcode = 0; + struct kbase_vinstr_context *vinstr_ctx; + u64 timestamp; + u32 event_mask; + + if (!cli) + return -EINVAL; + + vinstr_ctx = cli->vinstr_ctx; + KBASE_DEBUG_ASSERT(vinstr_ctx); + + KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT); + event_mask = 1 << event_id; + + mutex_lock(&vinstr_ctx->lock); + + if (event_mask & cli->event_mask) { + rcode = kbasep_vinstr_collect_and_accumulate( + vinstr_ctx, + ×tamp); + if (rcode) + goto exit; + + rcode = kbasep_vinstr_update_client(cli, timestamp, event_id); + if (rcode) + goto exit; + + kbasep_vinstr_reprogram(vinstr_ctx); + } + +exit: + mutex_unlock(&vinstr_ctx->lock); + + return rcode; +} +KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump); + +int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli) +{ + struct kbase_vinstr_context *vinstr_ctx; + int rcode; + u64 unused; + + if (!cli) + return -EINVAL; + + vinstr_ctx = cli->vinstr_ctx; + KBASE_DEBUG_ASSERT(vinstr_ctx); + + mutex_lock(&vinstr_ctx->lock); + + rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused); + if (rcode) + goto exit; + rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx); + if (rcode) + goto exit; + memset(cli->accum_buffer, 0, cli->dump_size); + + kbasep_vinstr_reprogram(vinstr_ctx); + +exit: + mutex_unlock(&vinstr_ctx->lock); + + return rcode; +} + +int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx) +{ + unsigned long flags; + int ret = -EAGAIN; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + switch (vinstr_ctx->state) { + case VINSTR_SUSPENDED: + vinstr_ctx->suspend_cnt++; + /* overflow shall not happen */ + BUG_ON(0 == vinstr_ctx->suspend_cnt); + ret = 0; + break; + + case VINSTR_IDLE: + vinstr_ctx->state = VINSTR_SUSPENDING; + schedule_work(&vinstr_ctx->suspend_work); + break; + + case VINSTR_DUMPING: + vinstr_ctx->state = VINSTR_SUSPENDING; + break; + + case VINSTR_SUSPENDING: + /* fall through */ + case VINSTR_RESUMING: + break; + + default: + BUG(); + break; + } + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); + + return ret; +} + +void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx) +{ + wait_event(vinstr_ctx->suspend_waitq, + (0 == kbase_vinstr_try_suspend(vinstr_ctx))); +} + +void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx) +{ + unsigned long flags; + + KBASE_DEBUG_ASSERT(vinstr_ctx); + + spin_lock_irqsave(&vinstr_ctx->state_lock, flags); + BUG_ON(VINSTR_SUSPENDING == vinstr_ctx->state); + if (VINSTR_SUSPENDED == vinstr_ctx->state) { + BUG_ON(0 == vinstr_ctx->suspend_cnt); + vinstr_ctx->suspend_cnt--; + if (0 == vinstr_ctx->suspend_cnt) { + vinstr_ctx->state = VINSTR_RESUMING; + schedule_work(&vinstr_ctx->resume_work); + } + } + spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); +} diff --git a/drivers/gpu/arm/midgard/mali_kbase_vinstr.h b/drivers/gpu/arm/midgard/mali_kbase_vinstr.h new file mode 100755 index 000000000000..6207d25aef06 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_kbase_vinstr.h @@ -0,0 +1,155 @@ +/* + * + * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KBASE_VINSTR_H_ +#define _KBASE_VINSTR_H_ + +#include +#include + +/*****************************************************************************/ + +struct kbase_vinstr_context; +struct kbase_vinstr_client; + +/*****************************************************************************/ + +/** + * kbase_vinstr_init() - initialize the vinstr core + * @kbdev: kbase device + * + * Return: pointer to the vinstr context on success or NULL on failure + */ +struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev); + +/** + * kbase_vinstr_term() - terminate the vinstr core + * @vinstr_ctx: vinstr context + */ +void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx); + +/** + * kbase_vinstr_hwcnt_reader_setup - configure hw counters reader + * @vinstr_ctx: vinstr context + * @setup: reader's configuration + * + * Return: zero on success + */ +int kbase_vinstr_hwcnt_reader_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_uk_hwcnt_reader_setup *setup); + +/** + * kbase_vinstr_legacy_hwc_setup - configure hw counters for dumping + * @vinstr_ctx: vinstr context + * @cli: pointer where to store pointer to new vinstr client structure + * @setup: hwc configuration + * + * Return: zero on success + */ +int kbase_vinstr_legacy_hwc_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_vinstr_client **cli, + struct kbase_uk_hwcnt_setup *setup); + +/** + * kbase_vinstr_hwcnt_kernel_setup - configure hw counters for kernel side + * client + * @vinstr_ctx: vinstr context + * @setup: reader's configuration + * @kernel_buffer: pointer to dump buffer + * + * setup->buffer_count and setup->fd are not used for kernel side clients. + * + * Return: pointer to client structure, or NULL on failure + */ +struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup( + struct kbase_vinstr_context *vinstr_ctx, + struct kbase_uk_hwcnt_reader_setup *setup, + void *kernel_buffer); + +/** + * kbase_vinstr_hwc_dump - issue counter dump for vinstr client + * @cli: pointer to vinstr client + * @event_id: id of event that triggered hwcnt dump + * + * Return: zero on success + */ +int kbase_vinstr_hwc_dump( + struct kbase_vinstr_client *cli, + enum base_hwcnt_reader_event event_id); + +/** + * kbase_vinstr_hwc_clear - performs a reset of the hardware counters for + * a given kbase context + * @cli: pointer to vinstr client + * + * Return: zero on success + */ +int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli); + +/** + * kbase_vinstr_try_suspend - try suspending operation of a given vinstr context + * @vinstr_ctx: vinstr context + * + * Return: 0 on success, or negative if state change is in progress + * + * Warning: This API call is non-generic. It is meant to be used only by + * job scheduler state machine. + * + * Function initiates vinstr switch to suspended state. Once it was called + * vinstr enters suspending state. If function return non-zero value, it + * indicates that state switch is not complete and function must be called + * again. On state switch vinstr will trigger job scheduler state machine + * cycle. + */ +int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx); + +/** + * kbase_vinstr_suspend - suspends operation of a given vinstr context + * @vinstr_ctx: vinstr context + * + * Function initiates vinstr switch to suspended state. Then it blocks until + * operation is completed. + */ +void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx); + +/** + * kbase_vinstr_resume - resumes operation of a given vinstr context + * @vinstr_ctx: vinstr context + * + * Function can be called only if it was preceded by a successful call + * to kbase_vinstr_suspend. + */ +void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx); + +/** + * kbase_vinstr_dump_size - Return required size of dump buffer + * @kbdev: device pointer + * + * Return : buffer size in bytes + */ +size_t kbase_vinstr_dump_size(struct kbase_device *kbdev); + +/** + * kbase_vinstr_detach_client - Detach a client from the vinstr core + * @cli: pointer to vinstr client + */ +void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli); + +#endif /* _KBASE_VINSTR_H_ */ + diff --git a/drivers/gpu/arm/midgard/mali_linux_kbase_trace.h b/drivers/gpu/arm/midgard/mali_linux_kbase_trace.h new file mode 100755 index 000000000000..5d6b4021d626 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_linux_kbase_trace.h @@ -0,0 +1,201 @@ +/* + * + * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + +#if !defined(_TRACE_MALI_KBASE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_MALI_KBASE_H + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mali + +#include + +DECLARE_EVENT_CLASS(mali_slot_template, + TP_PROTO(int jobslot, unsigned int info_val), + TP_ARGS(jobslot, info_val), + TP_STRUCT__entry( + __field(unsigned int, jobslot) + __field(unsigned int, info_val) + ), + TP_fast_assign( + __entry->jobslot = jobslot; + __entry->info_val = info_val; + ), + TP_printk("jobslot=%u info=%u", __entry->jobslot, __entry->info_val) +); + +#define DEFINE_MALI_SLOT_EVENT(name) \ +DEFINE_EVENT(mali_slot_template, mali_##name, \ + TP_PROTO(int jobslot, unsigned int info_val), \ + TP_ARGS(jobslot, info_val)) +DEFINE_MALI_SLOT_EVENT(JM_SUBMIT); +DEFINE_MALI_SLOT_EVENT(JM_JOB_DONE); +DEFINE_MALI_SLOT_EVENT(JM_UPDATE_HEAD); +DEFINE_MALI_SLOT_EVENT(JM_CHECK_HEAD); +DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP); +DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_0); +DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_1); +DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP); +DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_0); +DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_1); +DEFINE_MALI_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); +DEFINE_MALI_SLOT_EVENT(JM_SLOT_EVICT); +DEFINE_MALI_SLOT_EVENT(JM_BEGIN_RESET_WORKER); +DEFINE_MALI_SLOT_EVENT(JM_END_RESET_WORKER); +DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); +DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); +DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_CURRENT); +DEFINE_MALI_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); +DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); +DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); +DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); +DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); +DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); +DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); +DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); +DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); +#undef DEFINE_MALI_SLOT_EVENT + +DECLARE_EVENT_CLASS(mali_refcount_template, + TP_PROTO(int refcount, unsigned int info_val), + TP_ARGS(refcount, info_val), + TP_STRUCT__entry( + __field(unsigned int, refcount) + __field(unsigned int, info_val) + ), + TP_fast_assign( + __entry->refcount = refcount; + __entry->info_val = info_val; + ), + TP_printk("refcount=%u info=%u", __entry->refcount, __entry->info_val) +); + +#define DEFINE_MALI_REFCOUNT_EVENT(name) \ +DEFINE_EVENT(mali_refcount_template, mali_##name, \ + TP_PROTO(int refcount, unsigned int info_val), \ + TP_ARGS(refcount, info_val)) +DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX_NOLOCK); +DEFINE_MALI_REFCOUNT_EVENT(JS_ADD_JOB); +DEFINE_MALI_REFCOUNT_EVENT(JS_REMOVE_JOB); +DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_RELEASE_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); +DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); +DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_ACTIVE); +DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_IDLE); +#undef DEFINE_MALI_REFCOUNT_EVENT + +DECLARE_EVENT_CLASS(mali_add_template, + TP_PROTO(int gpu_addr, unsigned int info_val), + TP_ARGS(gpu_addr, info_val), + TP_STRUCT__entry( + __field(unsigned int, gpu_addr) + __field(unsigned int, info_val) + ), + TP_fast_assign( + __entry->gpu_addr = gpu_addr; + __entry->info_val = info_val; + ), + TP_printk("gpu_addr=%u info=%u", __entry->gpu_addr, __entry->info_val) +); + +#define DEFINE_MALI_ADD_EVENT(name) \ +DEFINE_EVENT(mali_add_template, mali_##name, \ + TP_PROTO(int gpu_addr, unsigned int info_val), \ + TP_ARGS(gpu_addr, info_val)) +DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY); +DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR); +DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE); +DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET); +DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET); +DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE); +DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR); +DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES); +DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER); +DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER_END); +DEFINE_MALI_ADD_EVENT(JD_CANCEL_WORKER); +DEFINE_MALI_ADD_EVENT(JD_DONE); +DEFINE_MALI_ADD_EVENT(JD_CANCEL); +DEFINE_MALI_ADD_EVENT(JD_ZAP_CONTEXT); +DEFINE_MALI_ADD_EVENT(JM_IRQ); +DEFINE_MALI_ADD_EVENT(JM_IRQ_END); +DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS); +DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS_DONE); +DEFINE_MALI_ADD_EVENT(JM_ZAP_NON_SCHEDULED); +DEFINE_MALI_ADD_EVENT(JM_ZAP_SCHEDULED); +DEFINE_MALI_ADD_EVENT(JM_ZAP_DONE); +DEFINE_MALI_ADD_EVENT(JM_SUBMIT_AFTER_RESET); +DEFINE_MALI_ADD_EVENT(JM_JOB_COMPLETE); +DEFINE_MALI_ADD_EVENT(JS_FAST_START_EVICTS_CTX); +DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); +DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); +DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); +DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); +DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_END); +DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_START); +DEFINE_MALI_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED); +DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP); +DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP); +DEFINE_MALI_ADD_EVENT(PM_PWRON); +DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER); +DEFINE_MALI_ADD_EVENT(PM_PWRON_L2); +DEFINE_MALI_ADD_EVENT(PM_PWROFF); +DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER); +DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER); +DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2); +DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED); +DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER); +DEFINE_MALI_ADD_EVENT(PM_UNREQUEST_CHANGE_SHADER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_REGISTER_CHANGE_SHADER_NEEDED); +DEFINE_MALI_ADD_EVENT(PM_REGISTER_CHANGE_SHADER_INUSE); +DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_INUSE); +DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE); +DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE); +DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER); +DEFINE_MALI_ADD_EVENT(PM_GPU_ON); +DEFINE_MALI_ADD_EVENT(PM_GPU_OFF); +DEFINE_MALI_ADD_EVENT(PM_SET_POLICY); +DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT); +DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM); +DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY); +DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); +#undef DEFINE_MALI_ADD_EVENT + +#endif /* _TRACE_MALI_KBASE_H */ + +#undef TRACE_INCLUDE_PATH +#undef linux +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE mali_linux_kbase_trace + +/* This part must be outside protection */ +#include diff --git a/drivers/gpu/arm/midgard/mali_linux_trace.h b/drivers/gpu/arm/midgard/mali_linux_trace.h new file mode 100755 index 000000000000..2be06a552768 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_linux_trace.h @@ -0,0 +1,189 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#if !defined(_TRACE_MALI_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_MALI_H + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mali +#define TRACE_INCLUDE_FILE mali_linux_trace + +#include + +#define MALI_JOB_SLOTS_EVENT_CHANGED + +/** + * mali_job_slots_event - called from mali_kbase_core_linux.c + * @event_id: ORed together bitfields representing a type of event, made with the GATOR_MAKE_EVENT() macro. + */ +TRACE_EVENT(mali_job_slots_event, + TP_PROTO(unsigned int event_id, unsigned int tgid, unsigned int pid, + unsigned char job_id), + TP_ARGS(event_id, tgid, pid, job_id), + TP_STRUCT__entry( + __field(unsigned int, event_id) + __field(unsigned int, tgid) + __field(unsigned int, pid) + __field(unsigned char, job_id) + ), + TP_fast_assign( + __entry->event_id = event_id; + __entry->tgid = tgid; + __entry->pid = pid; + __entry->job_id = job_id; + ), + TP_printk("event=%u tgid=%u pid=%u job_id=%u", + __entry->event_id, __entry->tgid, __entry->pid, __entry->job_id) +); + +/** + * mali_pm_status - Called by mali_kbase_pm_driver.c + * @event_id: core type (shader, tiler, l2 cache) + * @value: 64bits bitmask reporting either power status of the cores (1-ON, 0-OFF) + */ +TRACE_EVENT(mali_pm_status, + TP_PROTO(unsigned int event_id, unsigned long long value), + TP_ARGS(event_id, value), + TP_STRUCT__entry( + __field(unsigned int, event_id) + __field(unsigned long long, value) + ), + TP_fast_assign( + __entry->event_id = event_id; + __entry->value = value; + ), + TP_printk("event %u = %llu", __entry->event_id, __entry->value) +); + +/** + * mali_pm_power_on - Called by mali_kbase_pm_driver.c + * @event_id: core type (shader, tiler, l2 cache) + * @value: 64bits bitmask reporting the cores to power up + */ +TRACE_EVENT(mali_pm_power_on, + TP_PROTO(unsigned int event_id, unsigned long long value), + TP_ARGS(event_id, value), + TP_STRUCT__entry( + __field(unsigned int, event_id) + __field(unsigned long long, value) + ), + TP_fast_assign( + __entry->event_id = event_id; + __entry->value = value; + ), + TP_printk("event %u = %llu", __entry->event_id, __entry->value) +); + +/** + * mali_pm_power_off - Called by mali_kbase_pm_driver.c + * @event_id: core type (shader, tiler, l2 cache) + * @value: 64bits bitmask reporting the cores to power down + */ +TRACE_EVENT(mali_pm_power_off, + TP_PROTO(unsigned int event_id, unsigned long long value), + TP_ARGS(event_id, value), + TP_STRUCT__entry( + __field(unsigned int, event_id) + __field(unsigned long long, value) + ), + TP_fast_assign( + __entry->event_id = event_id; + __entry->value = value; + ), + TP_printk("event %u = %llu", __entry->event_id, __entry->value) +); + +/** + * mali_page_fault_insert_pages - Called by page_fault_worker() + * it reports an MMU page fault resulting in new pages being mapped. + * @event_id: MMU address space number. + * @value: number of newly allocated pages + */ +TRACE_EVENT(mali_page_fault_insert_pages, + TP_PROTO(int event_id, unsigned long value), + TP_ARGS(event_id, value), + TP_STRUCT__entry( + __field(int, event_id) + __field(unsigned long, value) + ), + TP_fast_assign( + __entry->event_id = event_id; + __entry->value = value; + ), + TP_printk("event %d = %lu", __entry->event_id, __entry->value) +); + +/** + * mali_mmu_as_in_use - Called by assign_and_activate_kctx_addr_space() + * it reports that a certain MMU address space is in use now. + * @event_id: MMU address space number. + */ +TRACE_EVENT(mali_mmu_as_in_use, + TP_PROTO(int event_id), + TP_ARGS(event_id), + TP_STRUCT__entry( + __field(int, event_id) + ), + TP_fast_assign( + __entry->event_id = event_id; + ), + TP_printk("event=%d", __entry->event_id) +); + +/** + * mali_mmu_as_released - Called by kbasep_js_runpool_release_ctx_internal() + * it reports that a certain MMU address space has been released now. + * @event_id: MMU address space number. + */ +TRACE_EVENT(mali_mmu_as_released, + TP_PROTO(int event_id), + TP_ARGS(event_id), + TP_STRUCT__entry( + __field(int, event_id) + ), + TP_fast_assign( + __entry->event_id = event_id; + ), + TP_printk("event=%d", __entry->event_id) +); + +/** + * mali_total_alloc_pages_change - Called by kbase_atomic_add_pages() + * and by kbase_atomic_sub_pages() + * it reports that the total number of allocated pages is changed. + * @event_id: number of pages to be added or subtracted (according to the sign). + */ +TRACE_EVENT(mali_total_alloc_pages_change, + TP_PROTO(long long int event_id), + TP_ARGS(event_id), + TP_STRUCT__entry( + __field(long long int, event_id) + ), + TP_fast_assign( + __entry->event_id = event_id; + ), + TP_printk("event=%lld", __entry->event_id) +); + +#endif /* _TRACE_MALI_H */ + +#undef TRACE_INCLUDE_PATH +#undef linux +#define TRACE_INCLUDE_PATH . + +/* This part must be outside protection */ +#include diff --git a/drivers/gpu/arm/midgard/mali_malisw.h b/drivers/gpu/arm/midgard/mali_malisw.h new file mode 100755 index 000000000000..99452933eab4 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_malisw.h @@ -0,0 +1,131 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * Kernel-wide include for common macros and types. + */ + +#ifndef _MALISW_H_ +#define _MALISW_H_ + +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#define U8_MAX ((u8)~0U) +#define S8_MAX ((s8)(U8_MAX>>1)) +#define S8_MIN ((s8)(-S8_MAX - 1)) +#define U16_MAX ((u16)~0U) +#define S16_MAX ((s16)(U16_MAX>>1)) +#define S16_MIN ((s16)(-S16_MAX - 1)) +#define U32_MAX ((u32)~0U) +#define S32_MAX ((s32)(U32_MAX>>1)) +#define S32_MIN ((s32)(-S32_MAX - 1)) +#define U64_MAX ((u64)~0ULL) +#define S64_MAX ((s64)(U64_MAX>>1)) +#define S64_MIN ((s64)(-S64_MAX - 1)) +#endif /* LINUX_VERSION_CODE */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) +#define SIZE_MAX (~(size_t)0) +#endif /* LINUX_VERSION_CODE */ + +/** + * MIN - Return the lesser of two values. + * + * As a macro it may evaluate its arguments more than once. + * Refer to MAX macro for more details + */ +#define MIN(x, y) ((x) < (y) ? (x) : (y)) + +/** + * MAX - Return the greater of two values. + * + * As a macro it may evaluate its arguments more than once. + * If called on the same two arguments as MIN it is guaranteed to return + * the one that MIN didn't return. This is significant for types where not + * all values are comparable e.g. NaNs in floating-point types. But if you want + * to retrieve the min and max of two values, consider using a conditional swap + * instead. + */ +#define MAX(x, y) ((x) < (y) ? (y) : (x)) + +/** + * @hideinitializer + * Function-like macro for suppressing unused variable warnings. Where possible + * such variables should be removed; this macro is present for cases where we + * much support API backwards compatibility. + */ +#define CSTD_UNUSED(x) ((void)(x)) + +/** + * @hideinitializer + * Function-like macro for use where "no behavior" is desired. This is useful + * when compile time macros turn a function-like macro in to a no-op, but + * where having no statement is otherwise invalid. + */ +#define CSTD_NOP(...) ((void)#__VA_ARGS__) + +/** + * Function-like macro for converting a pointer in to a u64 for storing into + * an external data structure. This is commonly used when pairing a 32-bit + * CPU with a 64-bit peripheral, such as a Midgard GPU. C's type promotion + * is complex and a straight cast does not work reliably as pointers are + * often considered as signed. + */ +#define PTR_TO_U64(x) ((uint64_t)((uintptr_t)(x))) + +/** + * @hideinitializer + * Function-like macro for stringizing a single level macro. + * @code + * #define MY_MACRO 32 + * CSTD_STR1( MY_MACRO ) + * > "MY_MACRO" + * @endcode + */ +#define CSTD_STR1(x) #x + +/** + * @hideinitializer + * Function-like macro for stringizing a macro's value. This should not be used + * if the macro is defined in a way which may have no value; use the + * alternative @c CSTD_STR2N macro should be used instead. + * @code + * #define MY_MACRO 32 + * CSTD_STR2( MY_MACRO ) + * > "32" + * @endcode + */ +#define CSTD_STR2(x) CSTD_STR1(x) + +/** + * Specify an assertion value which is evaluated at compile time. Recommended + * usage is specification of a @c static @c INLINE function containing all of + * the assertions thus: + * + * @code + * static INLINE [module]_compile_time_assertions( void ) + * { + * COMPILE_TIME_ASSERT( sizeof(uintptr_t) == sizeof(intptr_t) ); + * } + * @endcode + * + * @note Use @c static not @c STATIC. We never want to turn off this @c static + * specification for testing purposes. + */ +#define CSTD_COMPILE_TIME_ASSERT(expr) \ + do { switch (0) { case 0: case (expr):; } } while (false) + +#endif /* _MALISW_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_midg_coherency.h b/drivers/gpu/arm/midgard/mali_midg_coherency.h new file mode 100755 index 000000000000..a509cbd5f175 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_midg_coherency.h @@ -0,0 +1,26 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _MIDG_COHERENCY_H_ +#define _MIDG_COHERENCY_H_ + +#define COHERENCY_ACE_LITE 0 +#define COHERENCY_ACE 1 +#define COHERENCY_NONE 31 +#define COHERENCY_FEATURE_BIT(x) (1 << (x)) + +#endif /* _MIDG_COHERENCY_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_midg_regmap.h b/drivers/gpu/arm/midgard/mali_midg_regmap.h new file mode 100755 index 000000000000..7d7b7bcd3cc3 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_midg_regmap.h @@ -0,0 +1,611 @@ +/* + * + * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _MIDGARD_REGMAP_H_ +#define _MIDGARD_REGMAP_H_ + +#include "mali_midg_coherency.h" +#include "mali_kbase_gpu_id.h" + +/* + * Begin Register Offsets + */ + +#define GPU_CONTROL_BASE 0x0000 +#define GPU_CONTROL_REG(r) (GPU_CONTROL_BASE + (r)) +#define GPU_ID 0x000 /* (RO) GPU and revision identifier */ +#define L2_FEATURES 0x004 /* (RO) Level 2 cache features */ +#define SUSPEND_SIZE 0x008 /* (RO) Fixed-function suspend buffer + size */ +#define TILER_FEATURES 0x00C /* (RO) Tiler Features */ +#define MEM_FEATURES 0x010 /* (RO) Memory system features */ +#define MMU_FEATURES 0x014 /* (RO) MMU features */ +#define AS_PRESENT 0x018 /* (RO) Address space slots present */ +#define JS_PRESENT 0x01C /* (RO) Job slots present */ +#define GPU_IRQ_RAWSTAT 0x020 /* (RW) */ +#define GPU_IRQ_CLEAR 0x024 /* (WO) */ +#define GPU_IRQ_MASK 0x028 /* (RW) */ +#define GPU_IRQ_STATUS 0x02C /* (RO) */ + +/* IRQ flags */ +#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ +#define MULTIPLE_GPU_FAULTS (1 << 7) /* More than one GPU Fault occurred. */ +#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. Intended to use with SOFT_RESET + commands which may take time. */ +#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ +#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down + and the power manager is idle. */ + +#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ +#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ + +#define GPU_IRQ_REG_ALL (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ + | POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED) + +#define GPU_COMMAND 0x030 /* (WO) */ +#define GPU_STATUS 0x034 /* (RO) */ +#define LATEST_FLUSH 0x038 /* (RO) */ + +#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */ +#define GPU_DBGEN (1 << 8) /* DBGEN wire status */ + +#define GPU_FAULTSTATUS 0x03C /* (RO) GPU exception type and fault status */ +#define GPU_FAULTADDRESS_LO 0x040 /* (RO) GPU exception fault address, low word */ +#define GPU_FAULTADDRESS_HI 0x044 /* (RO) GPU exception fault address, high word */ + +#define PWR_KEY 0x050 /* (WO) Power manager key register */ +#define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */ +#define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */ + +#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory region base address, low word */ +#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory region base address, high word */ +#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter configuration */ +#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable flags for Job Manager */ +#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable flags for shader cores */ +#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable flags for tiler */ +#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable flags for MMU/L2 cache */ + +#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */ +#define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */ +#define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */ +#define TIMESTAMP_HI 0x09C /* (RO) Global time stamp counter, high word */ + +#define THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */ +#define THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */ +#define THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */ +#define THREAD_FEATURES 0x0AC /* (RO) Thread features */ + +#define TEXTURE_FEATURES_0 0x0B0 /* (RO) Support flags for indexed texture formats 0..31 */ +#define TEXTURE_FEATURES_1 0x0B4 /* (RO) Support flags for indexed texture formats 32..63 */ +#define TEXTURE_FEATURES_2 0x0B8 /* (RO) Support flags for indexed texture formats 64..95 */ + +#define TEXTURE_FEATURES_REG(n) GPU_CONTROL_REG(TEXTURE_FEATURES_0 + ((n) << 2)) + +#define JS0_FEATURES 0x0C0 /* (RO) Features of job slot 0 */ +#define JS1_FEATURES 0x0C4 /* (RO) Features of job slot 1 */ +#define JS2_FEATURES 0x0C8 /* (RO) Features of job slot 2 */ +#define JS3_FEATURES 0x0CC /* (RO) Features of job slot 3 */ +#define JS4_FEATURES 0x0D0 /* (RO) Features of job slot 4 */ +#define JS5_FEATURES 0x0D4 /* (RO) Features of job slot 5 */ +#define JS6_FEATURES 0x0D8 /* (RO) Features of job slot 6 */ +#define JS7_FEATURES 0x0DC /* (RO) Features of job slot 7 */ +#define JS8_FEATURES 0x0E0 /* (RO) Features of job slot 8 */ +#define JS9_FEATURES 0x0E4 /* (RO) Features of job slot 9 */ +#define JS10_FEATURES 0x0E8 /* (RO) Features of job slot 10 */ +#define JS11_FEATURES 0x0EC /* (RO) Features of job slot 11 */ +#define JS12_FEATURES 0x0F0 /* (RO) Features of job slot 12 */ +#define JS13_FEATURES 0x0F4 /* (RO) Features of job slot 13 */ +#define JS14_FEATURES 0x0F8 /* (RO) Features of job slot 14 */ +#define JS15_FEATURES 0x0FC /* (RO) Features of job slot 15 */ + +#define JS_FEATURES_REG(n) GPU_CONTROL_REG(JS0_FEATURES + ((n) << 2)) + +#define SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */ +#define SHADER_PRESENT_HI 0x104 /* (RO) Shader core present bitmap, high word */ + +#define TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */ +#define TILER_PRESENT_HI 0x114 /* (RO) Tiler core present bitmap, high word */ + +#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */ +#define L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */ + +#define STACK_PRESENT_LO 0xE00 /* (RO) Core stack present bitmap, low word */ +#define STACK_PRESENT_HI 0xE04 /* (RO) Core stack present bitmap, high word */ + + +#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */ +#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */ + +#define TILER_READY_LO 0x150 /* (RO) Tiler core ready bitmap, low word */ +#define TILER_READY_HI 0x154 /* (RO) Tiler core ready bitmap, high word */ + +#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */ +#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */ + +#define STACK_READY_LO 0xE10 /* (RO) Core stack ready bitmap, low word */ +#define STACK_READY_HI 0xE14 /* (RO) Core stack ready bitmap, high word */ + + +#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */ +#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */ + +#define TILER_PWRON_LO 0x190 /* (WO) Tiler core power on bitmap, low word */ +#define TILER_PWRON_HI 0x194 /* (WO) Tiler core power on bitmap, high word */ + +#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */ +#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */ + +#define STACK_PWRON_LO 0xE20 /* (RO) Core stack power on bitmap, low word */ +#define STACK_PWRON_HI 0xE24 /* (RO) Core stack power on bitmap, high word */ + + +#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */ +#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */ + +#define TILER_PWROFF_LO 0x1D0 /* (WO) Tiler core power off bitmap, low word */ +#define TILER_PWROFF_HI 0x1D4 /* (WO) Tiler core power off bitmap, high word */ + +#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */ +#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */ + +#define STACK_PWROFF_LO 0xE30 /* (RO) Core stack power off bitmap, low word */ +#define STACK_PRWOFF_HI 0xE34 /* (RO) Core stack power off bitmap, high word */ + + +#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */ +#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */ + +#define TILER_PWRTRANS_LO 0x210 /* (RO) Tiler core power transition bitmap, low word */ +#define TILER_PWRTRANS_HI 0x214 /* (RO) Tiler core power transition bitmap, high word */ + +#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */ +#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */ + +#define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */ +#define STACK_PRWTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */ + + +#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */ +#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */ + +#define TILER_PWRACTIVE_LO 0x250 /* (RO) Tiler core active bitmap, low word */ +#define TILER_PWRACTIVE_HI 0x254 /* (RO) Tiler core active bitmap, high word */ + +#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */ +#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */ + +#define COHERENCY_FEATURES 0x300 /* (RO) Coherency features present */ +#define COHERENCY_ENABLE 0x304 /* (RW) Coherency enable */ + +#define JM_CONFIG 0xF00 /* (RW) Job Manager configuration register (Implementation specific register) */ +#define SHADER_CONFIG 0xF04 /* (RW) Shader core configuration settings (Implementation specific register) */ +#define TILER_CONFIG 0xF08 /* (RW) Tiler core configuration settings (Implementation specific register) */ +#define L2_MMU_CONFIG 0xF0C /* (RW) Configuration of the L2 cache and MMU (Implementation specific register) */ + +#define JOB_CONTROL_BASE 0x1000 + +#define JOB_CONTROL_REG(r) (JOB_CONTROL_BASE + (r)) + +#define JOB_IRQ_RAWSTAT 0x000 /* Raw interrupt status register */ +#define JOB_IRQ_CLEAR 0x004 /* Interrupt clear register */ +#define JOB_IRQ_MASK 0x008 /* Interrupt mask register */ +#define JOB_IRQ_STATUS 0x00C /* Interrupt status register */ +#define JOB_IRQ_JS_STATE 0x010 /* status==active and _next == busy snapshot from last JOB_IRQ_CLEAR */ +#define JOB_IRQ_THROTTLE 0x014 /* cycles to delay delivering an interrupt externally. The JOB_IRQ_STATUS is NOT affected by this, just the delivery of the interrupt. */ + +#define JOB_SLOT0 0x800 /* Configuration registers for job slot 0 */ +#define JOB_SLOT1 0x880 /* Configuration registers for job slot 1 */ +#define JOB_SLOT2 0x900 /* Configuration registers for job slot 2 */ +#define JOB_SLOT3 0x980 /* Configuration registers for job slot 3 */ +#define JOB_SLOT4 0xA00 /* Configuration registers for job slot 4 */ +#define JOB_SLOT5 0xA80 /* Configuration registers for job slot 5 */ +#define JOB_SLOT6 0xB00 /* Configuration registers for job slot 6 */ +#define JOB_SLOT7 0xB80 /* Configuration registers for job slot 7 */ +#define JOB_SLOT8 0xC00 /* Configuration registers for job slot 8 */ +#define JOB_SLOT9 0xC80 /* Configuration registers for job slot 9 */ +#define JOB_SLOT10 0xD00 /* Configuration registers for job slot 10 */ +#define JOB_SLOT11 0xD80 /* Configuration registers for job slot 11 */ +#define JOB_SLOT12 0xE00 /* Configuration registers for job slot 12 */ +#define JOB_SLOT13 0xE80 /* Configuration registers for job slot 13 */ +#define JOB_SLOT14 0xF00 /* Configuration registers for job slot 14 */ +#define JOB_SLOT15 0xF80 /* Configuration registers for job slot 15 */ + +#define JOB_SLOT_REG(n, r) (JOB_CONTROL_REG(JOB_SLOT0 + ((n) << 7)) + (r)) + +#define JS_HEAD_LO 0x00 /* (RO) Job queue head pointer for job slot n, low word */ +#define JS_HEAD_HI 0x04 /* (RO) Job queue head pointer for job slot n, high word */ +#define JS_TAIL_LO 0x08 /* (RO) Job queue tail pointer for job slot n, low word */ +#define JS_TAIL_HI 0x0C /* (RO) Job queue tail pointer for job slot n, high word */ +#define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ +#define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ +#define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ +#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job + slot n */ + +#define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ +#define JS_STATUS 0x24 /* (RO) Status register for job slot n */ + +#define JS_HEAD_NEXT_LO 0x40 /* (RW) Next job queue head pointer for job slot n, low word */ +#define JS_HEAD_NEXT_HI 0x44 /* (RW) Next job queue head pointer for job slot n, high word */ + +#define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ +#define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ +#define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ +#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for + job slot n */ + +#define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ + +#define JS_FLUSH_ID_NEXT 0x70 /* (RW) Next job slot n cache flush ID */ + +#define MEMORY_MANAGEMENT_BASE 0x2000 +#define MMU_REG(r) (MEMORY_MANAGEMENT_BASE + (r)) + +#define MMU_IRQ_RAWSTAT 0x000 /* (RW) Raw interrupt status register */ +#define MMU_IRQ_CLEAR 0x004 /* (WO) Interrupt clear register */ +#define MMU_IRQ_MASK 0x008 /* (RW) Interrupt mask register */ +#define MMU_IRQ_STATUS 0x00C /* (RO) Interrupt status register */ + +#define MMU_AS0 0x400 /* Configuration registers for address space 0 */ +#define MMU_AS1 0x440 /* Configuration registers for address space 1 */ +#define MMU_AS2 0x480 /* Configuration registers for address space 2 */ +#define MMU_AS3 0x4C0 /* Configuration registers for address space 3 */ +#define MMU_AS4 0x500 /* Configuration registers for address space 4 */ +#define MMU_AS5 0x540 /* Configuration registers for address space 5 */ +#define MMU_AS6 0x580 /* Configuration registers for address space 6 */ +#define MMU_AS7 0x5C0 /* Configuration registers for address space 7 */ +#define MMU_AS8 0x600 /* Configuration registers for address space 8 */ +#define MMU_AS9 0x640 /* Configuration registers for address space 9 */ +#define MMU_AS10 0x680 /* Configuration registers for address space 10 */ +#define MMU_AS11 0x6C0 /* Configuration registers for address space 11 */ +#define MMU_AS12 0x700 /* Configuration registers for address space 12 */ +#define MMU_AS13 0x740 /* Configuration registers for address space 13 */ +#define MMU_AS14 0x780 /* Configuration registers for address space 14 */ +#define MMU_AS15 0x7C0 /* Configuration registers for address space 15 */ + +#define MMU_AS_REG(n, r) (MMU_REG(MMU_AS0 + ((n) << 6)) + (r)) + +#define AS_TRANSTAB_LO 0x00 /* (RW) Translation Table Base Address for address space n, low word */ +#define AS_TRANSTAB_HI 0x04 /* (RW) Translation Table Base Address for address space n, high word */ +#define AS_MEMATTR_LO 0x08 /* (RW) Memory attributes for address space n, low word. */ +#define AS_MEMATTR_HI 0x0C /* (RW) Memory attributes for address space n, high word. */ +#define AS_LOCKADDR_LO 0x10 /* (RW) Lock region address for address space n, low word */ +#define AS_LOCKADDR_HI 0x14 /* (RW) Lock region address for address space n, high word */ +#define AS_COMMAND 0x18 /* (WO) MMU command register for address space n */ +#define AS_FAULTSTATUS 0x1C /* (RO) MMU fault status register for address space n */ +#define AS_FAULTADDRESS_LO 0x20 /* (RO) Fault Address for address space n, low word */ +#define AS_FAULTADDRESS_HI 0x24 /* (RO) Fault Address for address space n, high word */ +#define AS_STATUS 0x28 /* (RO) Status flags for address space n */ + + +/* (RW) Translation table configuration for address space n, low word */ +#define AS_TRANSCFG_LO 0x30 +/* (RW) Translation table configuration for address space n, high word */ +#define AS_TRANSCFG_HI 0x34 +/* (RO) Secondary fault address for address space n, low word */ +#define AS_FAULTEXTRA_LO 0x38 +/* (RO) Secondary fault address for address space n, high word */ +#define AS_FAULTEXTRA_HI 0x3C + +/* End Register Offsets */ + +/* + * MMU_IRQ_RAWSTAT register values. Values are valid also for + MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers. + */ + +#define MMU_PAGE_FAULT_FLAGS 16 + +/* Macros returning a bitmask to retrieve page fault or bus error flags from + * MMU registers */ +#define MMU_PAGE_FAULT(n) (1UL << (n)) +#define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS)) + +/* + * Begin LPAE MMU TRANSTAB register values + */ +#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK 0xfffff000 +#define AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED (0u << 0) +#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY (1u << 1) +#define AS_TRANSTAB_LPAE_ADRMODE_TABLE (3u << 0) +#define AS_TRANSTAB_LPAE_READ_INNER (1u << 2) +#define AS_TRANSTAB_LPAE_SHARE_OUTER (1u << 4) + +#define AS_TRANSTAB_LPAE_ADRMODE_MASK 0x00000003 + +/* + * Begin AARCH64 MMU TRANSTAB register values + */ +#define MMU_HW_OUTA_BITS 40 +#define AS_TRANSTAB_BASE_MASK ((1ULL << MMU_HW_OUTA_BITS) - (1ULL << 4)) + +/* + * Begin MMU STATUS register values + */ +#define AS_STATUS_AS_ACTIVE 0x01 + +#define AS_FAULTSTATUS_EXCEPTION_CODE_MASK (0x7<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT (0x0<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT (0x1<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT (0x2<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG (0x3<<3) + +#define AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT (0x4<<3) +#define AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT (0x5<<3) + +#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3<<8) +#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0<<8) +#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1<<8) +#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2<<8) +#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3<<8) + +/* + * Begin MMU TRANSCFG register values + */ + +#define AS_TRANSCFG_ADRMODE_LEGACY 0 +#define AS_TRANSCFG_ADRMODE_UNMAPPED 1 +#define AS_TRANSCFG_ADRMODE_IDENTITY 2 +#define AS_TRANSCFG_ADRMODE_AARCH64_4K 6 +#define AS_TRANSCFG_ADRMODE_AARCH64_64K 8 + +#define AS_TRANSCFG_ADRMODE_MASK 0xF + + +/* + * Begin TRANSCFG register values + */ +#define AS_TRANSCFG_PTW_MEMATTR_MASK (3 << 24) +#define AS_TRANSCFG_PTW_MEMATTR_NON_CACHEABLE (1 << 24) +#define AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK (2 << 24) + +#define AS_TRANSCFG_PTW_SH_MASK ((3 << 28)) +#define AS_TRANSCFG_PTW_SH_OS (2 << 28) +#define AS_TRANSCFG_PTW_SH_IS (3 << 28) + +/* + * Begin Command Values + */ + +/* JS_COMMAND register commands */ +#define JS_COMMAND_NOP 0x00 /* NOP Operation. Writing this value is ignored */ +#define JS_COMMAND_START 0x01 /* Start processing a job chain. Writing this value is ignored */ +#define JS_COMMAND_SOFT_STOP 0x02 /* Gently stop processing a job chain */ +#define JS_COMMAND_HARD_STOP 0x03 /* Rudely stop processing a job chain */ +#define JS_COMMAND_SOFT_STOP_0 0x04 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */ +#define JS_COMMAND_HARD_STOP_0 0x05 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */ +#define JS_COMMAND_SOFT_STOP_1 0x06 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */ +#define JS_COMMAND_HARD_STOP_1 0x07 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */ + +#define JS_COMMAND_MASK 0x07 /* Mask of bits currently in use by the HW */ + +/* AS_COMMAND register commands */ +#define AS_COMMAND_NOP 0x00 /* NOP Operation */ +#define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */ +#define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */ +#define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */ +#define AS_COMMAND_FLUSH 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs + (deprecated - only for use with T60x) */ +#define AS_COMMAND_FLUSH_PT 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs */ +#define AS_COMMAND_FLUSH_MEM 0x05 /* Wait for memory accesses to complete, flush all the L1s cache then + flush all L2 caches then issue a flush region command to all MMUs */ + +/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */ +#define JS_CONFIG_START_FLUSH_NO_ACTION (0u << 0) +#define JS_CONFIG_START_FLUSH_CLEAN (1u << 8) +#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE (3u << 8) +#define JS_CONFIG_START_MMU (1u << 10) +#define JS_CONFIG_JOB_CHAIN_FLAG (1u << 11) +#define JS_CONFIG_END_FLUSH_NO_ACTION JS_CONFIG_START_FLUSH_NO_ACTION +#define JS_CONFIG_END_FLUSH_CLEAN (1u << 12) +#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE (3u << 12) +#define JS_CONFIG_ENABLE_FLUSH_REDUCTION (1u << 14) +#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK (1u << 15) +#define JS_CONFIG_THREAD_PRI(n) ((n) << 16) + +/* JS_XAFFINITY register values */ +#define JS_XAFFINITY_XAFFINITY_ENABLE (1u << 0) +#define JS_XAFFINITY_TILER_ENABLE (1u << 8) +#define JS_XAFFINITY_CACHE_ENABLE (1u << 16) + +/* JS_STATUS register values */ + +/* NOTE: Please keep this values in sync with enum base_jd_event_code in mali_base_kernel.h. + * The values are separated to avoid dependency of userspace and kernel code. + */ + +/* Group of values representing the job status insead a particular fault */ +#define JS_STATUS_NO_EXCEPTION_BASE 0x00 +#define JS_STATUS_INTERRUPTED (JS_STATUS_NO_EXCEPTION_BASE + 0x02) /* 0x02 means INTERRUPTED */ +#define JS_STATUS_STOPPED (JS_STATUS_NO_EXCEPTION_BASE + 0x03) /* 0x03 means STOPPED */ +#define JS_STATUS_TERMINATED (JS_STATUS_NO_EXCEPTION_BASE + 0x04) /* 0x04 means TERMINATED */ + +/* General fault values */ +#define JS_STATUS_FAULT_BASE 0x40 +#define JS_STATUS_CONFIG_FAULT (JS_STATUS_FAULT_BASE) /* 0x40 means CONFIG FAULT */ +#define JS_STATUS_POWER_FAULT (JS_STATUS_FAULT_BASE + 0x01) /* 0x41 means POWER FAULT */ +#define JS_STATUS_READ_FAULT (JS_STATUS_FAULT_BASE + 0x02) /* 0x42 means READ FAULT */ +#define JS_STATUS_WRITE_FAULT (JS_STATUS_FAULT_BASE + 0x03) /* 0x43 means WRITE FAULT */ +#define JS_STATUS_AFFINITY_FAULT (JS_STATUS_FAULT_BASE + 0x04) /* 0x44 means AFFINITY FAULT */ +#define JS_STATUS_BUS_FAULT (JS_STATUS_FAULT_BASE + 0x08) /* 0x48 means BUS FAULT */ + +/* Instruction or data faults */ +#define JS_STATUS_INSTRUCTION_FAULT_BASE 0x50 +#define JS_STATUS_INSTR_INVALID_PC (JS_STATUS_INSTRUCTION_FAULT_BASE) /* 0x50 means INSTR INVALID PC */ +#define JS_STATUS_INSTR_INVALID_ENC (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x01) /* 0x51 means INSTR INVALID ENC */ +#define JS_STATUS_INSTR_TYPE_MISMATCH (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x02) /* 0x52 means INSTR TYPE MISMATCH */ +#define JS_STATUS_INSTR_OPERAND_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x03) /* 0x53 means INSTR OPERAND FAULT */ +#define JS_STATUS_INSTR_TLS_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x04) /* 0x54 means INSTR TLS FAULT */ +#define JS_STATUS_INSTR_BARRIER_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x05) /* 0x55 means INSTR BARRIER FAULT */ +#define JS_STATUS_INSTR_ALIGN_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x06) /* 0x56 means INSTR ALIGN FAULT */ +/* NOTE: No fault with 0x57 code defined in spec. */ +#define JS_STATUS_DATA_INVALID_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x08) /* 0x58 means DATA INVALID FAULT */ +#define JS_STATUS_TILE_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x09) /* 0x59 means TILE RANGE FAULT */ +#define JS_STATUS_ADDRESS_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x0A) /* 0x5A means ADDRESS RANGE FAULT */ + +/* Other faults */ +#define JS_STATUS_MEMORY_FAULT_BASE 0x60 +#define JS_STATUS_OUT_OF_MEMORY (JS_STATUS_MEMORY_FAULT_BASE) /* 0x60 means OUT OF MEMORY */ +#define JS_STATUS_UNKNOWN 0x7F /* 0x7F means UNKNOWN */ + +/* GPU_COMMAND values */ +#define GPU_COMMAND_NOP 0x00 /* No operation, nothing happens */ +#define GPU_COMMAND_SOFT_RESET 0x01 /* Stop all external bus interfaces, and then reset the entire GPU. */ +#define GPU_COMMAND_HARD_RESET 0x02 /* Immediately reset the entire GPU. */ +#define GPU_COMMAND_PRFCNT_CLEAR 0x03 /* Clear all performance counters, setting them all to zero. */ +#define GPU_COMMAND_PRFCNT_SAMPLE 0x04 /* Sample all performance counters, writing them out to memory */ +#define GPU_COMMAND_CYCLE_COUNT_START 0x05 /* Starts the cycle counter, and system timestamp propagation */ +#define GPU_COMMAND_CYCLE_COUNT_STOP 0x06 /* Stops the cycle counter, and system timestamp propagation */ +#define GPU_COMMAND_CLEAN_CACHES 0x07 /* Clean all caches */ +#define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */ +#define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */ + +/* End Command Values */ + +/* GPU_STATUS values */ +#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ +#define GPU_STATUS_PROTECTED_MODE_ACTIVE (1 << 7) /* Set if protected mode is active */ + +/* PRFCNT_CONFIG register values */ +#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ +#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ +#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ + +#define PRFCNT_CONFIG_MODE_OFF 0 /* The performance counters are disabled. */ +#define PRFCNT_CONFIG_MODE_MANUAL 1 /* The performance counters are enabled, but are only written out when a PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. */ +#define PRFCNT_CONFIG_MODE_TILE 2 /* The performance counters are enabled, and are written out each time a tile finishes rendering. */ + +/* AS_MEMATTR values: */ +/* Use GPU implementation-defined caching policy. */ +#define AS_MEMATTR_IMPL_DEF_CACHE_POLICY 0x88ull +/* The attribute set to force all resources to be cached. */ +#define AS_MEMATTR_FORCE_TO_CACHE_ALL 0x8Full +/* Inner write-alloc cache setup, no outer caching */ +#define AS_MEMATTR_WRITE_ALLOC 0x8Dull + +/* Set to implementation defined, outer caching */ +#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull +/* Set to write back memory, outer caching */ +#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull + +/* Use GPU implementation-defined caching policy. */ +#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull +/* The attribute set to force all resources to be cached. */ +#define AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL 0x4Full +/* Inner write-alloc cache setup, no outer caching */ +#define AS_MEMATTR_LPAE_WRITE_ALLOC 0x4Dull +/* Set to implementation defined, outer caching */ +#define AS_MEMATTR_LPAE_OUTER_IMPL_DEF 0x88ull +/* Set to write back memory, outer caching */ +#define AS_MEMATTR_LPAE_OUTER_WA 0x8Dull + +/* Symbol for default MEMATTR to use */ + +/* Default is - HW implementation defined caching */ +#define AS_MEMATTR_INDEX_DEFAULT 0 +#define AS_MEMATTR_INDEX_DEFAULT_ACE 3 + +/* HW implementation defined caching */ +#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0 +/* Force cache on */ +#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1 +/* Write-alloc */ +#define AS_MEMATTR_INDEX_WRITE_ALLOC 2 +/* Outer coherent, inner implementation defined policy */ +#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3 +/* Outer coherent, write alloc inner */ +#define AS_MEMATTR_INDEX_OUTER_WA 4 + +/* JS_FEATURES register */ + +#define JS_FEATURE_NULL_JOB (1u << 1) +#define JS_FEATURE_SET_VALUE_JOB (1u << 2) +#define JS_FEATURE_CACHE_FLUSH_JOB (1u << 3) +#define JS_FEATURE_COMPUTE_JOB (1u << 4) +#define JS_FEATURE_VERTEX_JOB (1u << 5) +#define JS_FEATURE_GEOMETRY_JOB (1u << 6) +#define JS_FEATURE_TILER_JOB (1u << 7) +#define JS_FEATURE_FUSED_JOB (1u << 8) +#define JS_FEATURE_FRAGMENT_JOB (1u << 9) + +/* End JS_FEATURES register */ + +/* L2_MMU_CONFIG register */ +#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT (23) +#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT (24) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) + +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT (26) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) +#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) +/* End L2_MMU_CONFIG register */ + +/* THREAD_* registers */ + +/* THREAD_FEATURES IMPLEMENTATION_TECHNOLOGY values */ +#define IMPLEMENTATION_UNSPECIFIED 0 +#define IMPLEMENTATION_SILICON 1 +#define IMPLEMENTATION_FPGA 2 +#define IMPLEMENTATION_MODEL 3 + +/* Default values when registers are not supported by the implemented hardware */ +#define THREAD_MT_DEFAULT 256 +#define THREAD_MWS_DEFAULT 256 +#define THREAD_MBS_DEFAULT 256 +#define THREAD_MR_DEFAULT 1024 +#define THREAD_MTQ_DEFAULT 4 +#define THREAD_MTGS_DEFAULT 10 + +/* End THREAD_* registers */ + +/* SHADER_CONFIG register */ + +#define SC_ALT_COUNTERS (1ul << 3) +#define SC_OVERRIDE_FWD_PIXEL_KILL (1ul << 4) +#define SC_SDC_DISABLE_OQ_DISCARD (1ul << 6) +#define SC_LS_ALLOW_ATTR_TYPES (1ul << 16) +#define SC_LS_PAUSEBUFFER_DISABLE (1ul << 16) +#define SC_LS_ATTR_CHECK_DISABLE (1ul << 18) +#define SC_ENABLE_TEXGRD_FLAGS (1ul << 25) +/* End SHADER_CONFIG register */ + +/* TILER_CONFIG register */ + +#define TC_CLOCK_GATE_OVERRIDE (1ul << 0) + +/* End TILER_CONFIG register */ + +/* JM_CONFIG register */ + +#define JM_TIMESTAMP_OVERRIDE (1ul << 0) +#define JM_CLOCK_GATE_OVERRIDE (1ul << 1) +#define JM_JOB_THROTTLE_ENABLE (1ul << 2) +#define JM_JOB_THROTTLE_LIMIT_SHIFT (3) +#define JM_MAX_JOB_THROTTLE_LIMIT (0x3F) +#define JM_FORCE_COHERENCY_FEATURES_SHIFT (2) +#define JM_IDVS_GROUP_SIZE_SHIFT (16) +#define JM_MAX_IDVS_GROUP_SIZE (0x3F) +/* End JM_CONFIG register */ + + +#endif /* _MIDGARD_REGMAP_H_ */ diff --git a/drivers/gpu/arm/midgard/mali_timeline.h b/drivers/gpu/arm/midgard/mali_timeline.h new file mode 100755 index 000000000000..bd5f6614b6bb --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_timeline.h @@ -0,0 +1,396 @@ +/* + * + * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mali_timeline + +#if !defined(_MALI_TIMELINE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _MALI_TIMELINE_H + +#include + +TRACE_EVENT(mali_timeline_atoms_in_flight, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int tgid, + int count), + + TP_ARGS(ts_sec, + ts_nsec, + tgid, + count), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, tgid) + __field(int, count) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->tgid = tgid; + __entry->count = count; + ), + + TP_printk("%i,%i.%.9i,%i,%i", CTX_SET_NR_ATOMS_IN_FLIGHT, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->tgid, + __entry->count) +); + + +TRACE_EVENT(mali_timeline_atom, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int tgid, + int atom_id), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + tgid, + atom_id), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, tgid) + __field(int, atom_id) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->tgid = tgid; + __entry->atom_id = atom_id; + ), + + TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->tgid, + __entry->atom_id, + __entry->atom_id) +); + +TRACE_EVENT(mali_timeline_gpu_slot_active, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int tgid, + int js, + int count), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + tgid, + js, + count), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, tgid) + __field(int, js) + __field(int, count) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->tgid = tgid; + __entry->js = js; + __entry->count = count; + ), + + TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->tgid, + __entry->js, + __entry->count) +); + +TRACE_EVENT(mali_timeline_gpu_slot_action, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int tgid, + int js, + int count), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + tgid, + js, + count), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, tgid) + __field(int, js) + __field(int, count) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->tgid = tgid; + __entry->js = js; + __entry->count = count; + ), + + TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->tgid, + __entry->js, + __entry->count) +); + +TRACE_EVENT(mali_timeline_gpu_power_active, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int active), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + active), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, active) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->active = active; + ), + + TP_printk("%i,%i.%.9i,0,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->active) + +); + +TRACE_EVENT(mali_timeline_l2_power_active, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int state), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + state), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, state) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->state = state; + ), + + TP_printk("%i,%i.%.9i,0,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->state) + +); +TRACE_EVENT(mali_timeline_pm_event, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int pm_event_type, + unsigned int pm_event_id), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + pm_event_type, + pm_event_id), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, pm_event_type) + __field(unsigned int, pm_event_id) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->pm_event_type = pm_event_type; + __entry->pm_event_id = pm_event_id; + ), + + TP_printk("%i,%i.%.9i,0,%i,%u", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->pm_event_type, __entry->pm_event_id) + +); + +TRACE_EVENT(mali_timeline_slot_atom, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int event_type, + int tgid, + int js, + int atom_id), + + TP_ARGS(ts_sec, + ts_nsec, + event_type, + tgid, + js, + atom_id), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, event_type) + __field(int, tgid) + __field(int, js) + __field(int, atom_id) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->event_type = event_type; + __entry->tgid = tgid; + __entry->js = js; + __entry->atom_id = atom_id; + ), + + TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->tgid, + __entry->js, + __entry->atom_id) +); + +TRACE_EVENT(mali_timeline_pm_checktrans, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int trans_code, + int trans_id), + + TP_ARGS(ts_sec, + ts_nsec, + trans_code, + trans_id), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, trans_code) + __field(int, trans_id) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->trans_code = trans_code; + __entry->trans_id = trans_id; + ), + + TP_printk("%i,%i.%.9i,0,%i", __entry->trans_code, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->trans_id) + +); + +TRACE_EVENT(mali_timeline_context_active, + + TP_PROTO(u64 ts_sec, + u32 ts_nsec, + int count), + + TP_ARGS(ts_sec, + ts_nsec, + count), + + TP_STRUCT__entry( + __field(u64, ts_sec) + __field(u32, ts_nsec) + __field(int, count) + ), + + TP_fast_assign( + __entry->ts_sec = ts_sec; + __entry->ts_nsec = ts_nsec; + __entry->count = count; + ), + + TP_printk("%i,%i.%.9i,0,%i", SW_SET_CONTEXT_ACTIVE, + (int)__entry->ts_sec, + (int)__entry->ts_nsec, + __entry->count) +); + +#endif /* _MALI_TIMELINE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +/* This part must be outside protection */ +#include + diff --git a/drivers/gpu/arm/midgard/mali_uk.h b/drivers/gpu/arm/midgard/mali_uk.h new file mode 100755 index 000000000000..841d03fb5873 --- /dev/null +++ b/drivers/gpu/arm/midgard/mali_uk.h @@ -0,0 +1,141 @@ +/* + * + * (C) COPYRIGHT 2010, 2012-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +/** + * @file mali_uk.h + * Types and definitions that are common across OSs for both the user + * and kernel side of the User-Kernel interface. + */ + +#ifndef _UK_H_ +#define _UK_H_ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @addtogroup base_api + * @{ + */ + +/** + * @defgroup uk_api User-Kernel Interface API + * + * The User-Kernel Interface abstracts the communication mechanism between the user and kernel-side code of device + * drivers developed as part of the Midgard DDK. Currently that includes the Base driver and the UMP driver. + * + * It exposes an OS independent API to user-side code (UKU) which routes functions calls to an OS-independent + * kernel-side API (UKK) via an OS-specific communication mechanism. + * + * This API is internal to the Midgard DDK and is not exposed to any applications. + * + * @{ + */ + +/** + * These are identifiers for kernel-side drivers implementing a UK interface, aka UKK clients. The + * UK module maps this to an OS specific device name, e.g. "gpu_base" -> "GPU0:". Specify this + * identifier to select a UKK client to the uku_open() function. + * + * When a new UKK client driver is created a new identifier needs to be added to the uk_client_id + * enumeration and the uku_open() implemenation for the various OS ports need to be updated to + * provide a mapping of the identifier to the OS specific device name. + * + */ +enum uk_client_id { + /** + * Value used to identify the Base driver UK client. + */ + UK_CLIENT_MALI_T600_BASE, + + /** The number of uk clients supported. This must be the last member of the enum */ + UK_CLIENT_COUNT +}; + +/** + * Each function callable through the UK interface has a unique number. + * Functions provided by UK clients start from number UK_FUNC_ID. + * Numbers below UK_FUNC_ID are used for internal UK functions. + */ +enum uk_func { + UKP_FUNC_ID_CHECK_VERSION, /**< UKK Core internal function */ + /** + * Each UK client numbers the functions they provide starting from + * number UK_FUNC_ID. This number is then eventually assigned to the + * id field of the union uk_header structure when preparing to make a + * UK call. See your UK client for a list of their function numbers. + */ + UK_FUNC_ID = 512 +}; + +/** + * Arguments for a UK call are stored in a structure. This structure consists + * of a fixed size header and a payload. The header carries a 32-bit number + * identifying the UK function to be called (see uk_func). When the UKK client + * receives this header and executed the requested UK function, it will use + * the same header to store the result of the function in the form of a + * int return code. The size of this structure is such that the + * first member of the payload following the header can be accessed efficiently + * on a 32 and 64-bit kernel and the structure has the same size regardless + * of a 32 or 64-bit kernel. The uk_kernel_size_type type should be defined + * accordingly in the OS specific mali_uk_os.h header file. + */ +union uk_header { + /** + * 32-bit number identifying the UK function to be called. + * Also see uk_func. + */ + u32 id; + /** + * The int return code returned by the called UK function. + * See the specification of the particular UK function you are + * calling for the meaning of the error codes returned. All + * UK functions return 0 on success. + */ + u32 ret; + /* + * Used to ensure 64-bit alignment of this union. Do not remove. + * This field is used for padding and does not need to be initialized. + */ + u64 sizer; +}; + +/** + * This structure carries a 16-bit major and minor number and is sent along with an internal UK call + * used during uku_open to identify the versions of the UK module in use by the user-side and kernel-side. + */ +struct uku_version_check_args { + union uk_header header; + /**< UK call header */ + u16 major; + /**< This field carries the user-side major version on input and the kernel-side major version on output */ + u16 minor; + /**< This field carries the user-side minor version on input and the kernel-side minor version on output. */ + u8 padding[4]; +}; + +/** @} end group uk_api */ + +/** @} *//* end group base_api */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif /* _UK_H_ */ diff --git a/drivers/gpu/arm/midgard/platform/Kconfig b/drivers/gpu/arm/midgard/platform/Kconfig new file mode 100755 index 000000000000..8fb4e917c4fa --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/Kconfig @@ -0,0 +1,24 @@ +# +# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + + +# Add your platform specific Kconfig file here +# +# "drivers/gpu/arm/midgard/platform/xxx/Kconfig" +# +# Where xxx is the platform name is the name set in MALI_PLATFORM_THIRDPARTY_NAME +# + diff --git a/drivers/gpu/arm/midgard/platform/devicetree/Kbuild b/drivers/gpu/arm/midgard/platform/devicetree/Kbuild new file mode 100755 index 000000000000..e888a42fc69a --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/devicetree/Kbuild @@ -0,0 +1,18 @@ +# +# (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +mali_kbase-y += \ + $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_devicetree.o \ + $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_runtime_pm.o diff --git a/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c new file mode 100755 index 000000000000..b2a7c93f12a9 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c @@ -0,0 +1,31 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} + +static struct kbase_platform_config dummy_platform_config; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &dummy_platform_config; +} diff --git a/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h new file mode 100755 index 000000000000..49e107f98000 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h @@ -0,0 +1,73 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * Maximum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX (5000) +/** + * Minimum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN (5000) + +/** + * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (NULL) + +/** + * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c new file mode 100755 index 000000000000..aa4376afd3ba --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c @@ -0,0 +1,100 @@ +/* + * + * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + int ret; + + dev_dbg(kbdev->dev, "pm_callback_power_on %p\n", + (void *)kbdev->dev->pm_domain); + + ret = pm_runtime_get_sync(kbdev->dev); + + dev_dbg(kbdev->dev, "pm_runtime_get returned %d\n", ret); + + return 1; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "pm_callback_power_off\n"); + + pm_runtime_put_autosuspend(kbdev->dev); +} + +int kbase_device_runtime_init(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "kbase_device_runtime_init\n"); + pm_runtime_enable(kbdev->dev); + + return 0; +} + +void kbase_device_runtime_disable(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "kbase_device_runtime_disable\n"); + pm_runtime_disable(kbdev->dev); +} + +static int pm_callback_runtime_on(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "pm_callback_runtime_on\n"); + + return 0; +} + +static void pm_callback_runtime_off(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "pm_callback_runtime_off\n"); +} + +static void pm_callback_resume(struct kbase_device *kbdev) +{ + int ret = pm_callback_runtime_on(kbdev); + + WARN_ON(ret); +} + +static void pm_callback_suspend(struct kbase_device *kbdev) +{ + pm_callback_runtime_off(kbdev); +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = pm_callback_suspend, + .power_resume_callback = pm_callback_resume, +#ifdef KBASE_PM_RUNTIME + .power_runtime_init_callback = kbase_device_runtime_init, + .power_runtime_term_callback = kbase_device_runtime_disable, + .power_runtime_on_callback = pm_callback_runtime_on, + .power_runtime_off_callback = pm_callback_runtime_off, +#else /* KBASE_PM_RUNTIME */ + .power_runtime_init_callback = NULL, + .power_runtime_term_callback = NULL, + .power_runtime_on_callback = NULL, + .power_runtime_off_callback = NULL, +#endif /* KBASE_PM_RUNTIME */ +}; + + diff --git a/drivers/gpu/arm/midgard/platform/mali_kbase_platform_common.h b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_common.h new file mode 100755 index 000000000000..c11085af5f24 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_common.h @@ -0,0 +1,28 @@ +/* + * + * (C) COPYRIGHT 2010-2013 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + +#include + + +/** + * @brief Entry point to transfer control to a platform for early initialization + * + * This function is called early on in the initialization during execution of + * @ref kbase_driver_init. + * + * @return Zero to indicate success non-zero for failure. + */ +int kbase_platform_early_init(void); +int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev); diff --git a/drivers/gpu/arm/midgard/platform/mali_kbase_platform_fake.h b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_fake.h new file mode 100755 index 000000000000..01f9dfce93cc --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_fake.h @@ -0,0 +1,38 @@ +/* + * + * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifdef CONFIG_MALI_PLATFORM_FAKE + +/** + * kbase_platform_fake_register - Entry point for fake platform registration + * + * This function is called early on in the initialization during execution of + * kbase_driver_init. + * + * Return: 0 to indicate success, non-zero for failure. + */ +int kbase_platform_fake_register(void); + +/** + * kbase_platform_fake_unregister - Entry point for fake platform unregistration + * + * This function is called in the termination during execution of + * kbase_driver_exit. + */ +void kbase_platform_fake_unregister(void); + +#endif /* CONFIG_MALI_PLATFORM_FAKE */ diff --git a/drivers/gpu/arm/midgard/platform/rk/Kbuild b/drivers/gpu/arm/midgard/platform/rk/Kbuild new file mode 100755 index 000000000000..db993487e3be --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/rk/Kbuild @@ -0,0 +1,17 @@ +# +# (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + +midgard_kbase-y += \ + $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_rk.o + diff --git a/drivers/gpu/arm/midgard/platform/rk/custom_log.h b/drivers/gpu/arm/midgard/platform/rk/custom_log.h new file mode 100755 index 000000000000..fe5e1224149e --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/rk/custom_log.h @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* ---------------------------------------------------------------------------- + * File: custom_log.h + * + * Desc: ChenZhen å好的 log 输出的定制实现. + * + * -------------------------------------------------------------------- + * < 习语 å’Œ 缩略语 > : + * + * -------------------------------------------------------------------- + * Usage: + * + * Note: + * + * Author: ChenZhen + * + * ---------------------------------------------------------------------------- + * Version: + * v1.0 + * ---------------------------------------------------------------------------- + * Log: + ----Fri Nov 19 15:20:28 2010 v1.0 + * + * ---------------------------------------------------------------------------- + */ + +#ifndef __CUSTOM_LOG_H__ +#define __CUSTOM_LOG_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/* ----------------------------------------------------------------------------- + * Include Files + * ----------------------------------------------------------------------------- + */ +#include +#include + +/* ----------------------------------------------------------------------------- + * Macros Definition + * ----------------------------------------------------------------------------- + */ + +/** 若下列 macro 有被定义, æ‰ ä½¿èƒ½ log 输出. */ +/* #define ENABLE_DEBUG_LOG */ + +/*----------------------------------------------------------------------------*/ + +#ifdef ENABLE_VERBOSE_LOG +/** Verbose log. */ +#define V(fmt, args...) \ + pr_debug("V : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) +#else +#define V(...) ((void)0) +#endif + +#ifdef ENABLE_DEBUG_LOG +/** Debug log. */ +#define D(fmt, args...) \ + pr_info("D : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) +#else +#define D(...) ((void)0) +#endif + +#define I(fmt, args...) \ + pr_info("I : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +#define W(fmt, args...) \ + pr_warn("W : [File] : %s; [Line] : %d; [Func] : %s(); " \ + fmt "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +#define E(fmt, args...) \ + pr_err("E : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ + "\n", \ + __FILE__, \ + __LINE__, \ + __func__, \ + ## args) + +/*-------------------------------------------------------*/ + +/** 使用 D(), 以åè¿›åˆ¶çš„å½¢å¼æ‰“å°å˜é‡ 'var' çš„ value. */ +#define D_DEC(var) D(#var " = %d.", var) + +#define E_DEC(var) E(#var " = %d.", var) + +/** 使用 D(), 以åå…­è¿›åˆ¶çš„å½¢å¼æ‰“å°å˜é‡ 'var' çš„ value. */ +#define D_HEX(var) D(#var " = 0x%x.", var) + +#define E_HEX(var) E(#var " = 0x%x.", var) + +/** + * 使用 D(), 以å六进制的形å¼, + * æ‰“å°æŒ‡é’ˆç±»åž‹å˜é‡ 'ptr' çš„ value. + */ +#define D_PTR(ptr) D(#ptr " = %p.", ptr) + +#define E_PTR(ptr) E(#ptr " = %p.", ptr) + +/** 使用 D(), æ‰“å° char 字串. */ +#define D_STR(p_str) \ +do { \ + if (!p_str) { \ + D(#p_str " = NULL."); \ + else \ + D(#p_str " = '%s'.", p_str); \ +} while (0) + +#define E_STR(p_str) \ +do { \ + if (!p_str) \ + E(#p_str " = NULL."); \ + else \ + E(#p_str " = '%s'.", p_str); \ +} while (0) + +#ifdef ENABLE_DEBUG_LOG +/** + * log 从 'p_start' 地å€å¼€å§‹çš„ 'len' 个字节的数æ®. + */ +#define D_MEM(p_start, len) \ +do { \ + int i = 0; \ + char *p = (char *)(p_start); \ + D("dump memory from addr of '" #p_start "', from %p, length %d' : ", \ + (p_start), \ + (len)); \ + pr_debug("\t\t"); \ + for (i = 0; i < (len); i++) \ + pr_debug("0x%02x, ", p[i]); \ + pr_debug("\n"); \ +} while (0) +#else +#define D_MEM(...) ((void)0) +#endif + +/*-------------------------------------------------------*/ + +/** + * 在特定æ¡ä»¶ä¸‹, 判定 error å‘生, + * å°†å˜é‡ 'ret_var' 设置 'err_code', + * log 输出对应的 Error Caution, + * ç„¶åŽè·³è½¬ 'label' 指定的代ç å¤„执行. + * @param msg + * 纯字串形å¼çš„æç¤ºä¿¡æ¯. + * @param ret_var + * æ ‡è¯†å‡½æ•°æ‰§è¡ŒçŠ¶æ€æˆ–者结果的å˜é‡, + * 将被设置具体的 Error Code. + * 通常是 'ret' or 'result'. + * @param err_code + * 表å¾ç‰¹å®š error 的常数标识, + * 通常是 å®çš„å½¢æ€. + * @param label + * 程åºå°†è¦è·³è½¬åˆ°çš„错误处ç†ä»£ç çš„æ ‡å·, + * 通常就是 'EXIT'. + * @param args... + * 对应 'msg_fmt' 实å‚中, + * '%s', '%d', ... 等转æ¢è¯´æ˜Žç¬¦çš„具体å¯å˜é•¿å®žå‚. + */ +#define SET_ERROR_AND_JUMP(msg_fmt, ret_var, err_code, label, args...) \ +do { \ + E("To set '" #ret_var "' to %d('" #err_code "'), because : " msg_fmt, \ + (err_code), \ + ## args); \ + (ret_var) = (err_code); \ + goto label; \ +} while (0) + +/* ----------------------------------------------------------------------------- + * Types and Structures Definition + * ----------------------------------------------------------------------------- + */ + +/* ----------------------------------------------------------------------------- + * Global Functions' Prototype + * ----------------------------------------------------------------------------- + */ + +/* ----------------------------------------------------------------------------- + * Inline Functions Implementation + * ----------------------------------------------------------------------------- + */ + +#ifdef __cplusplus +} +#endif + +#endif /* __CUSTOM_LOG_H__ */ diff --git a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_platform.h new file mode 100755 index 000000000000..07c5b6f8a760 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_platform.h @@ -0,0 +1,88 @@ +/* + * + * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +/** + * @file mali_kbase_config_platform.h + * 声明 platform_config_of_rk (platform_rk çš„ platform_config). + */ + +/** + * Maximum frequency GPU will be clocked at. + * Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX (5000) + +/** + * Minimum frequency GPU will be clocked at. + * Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN (5000) + +/** + * CPU_SPEED_FUNC + * - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz + * - see kbase_cpu_clk_speed_func for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (NULL) + +/** + * GPU_SPEED_FUNC + * - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz + * - see kbase_gpu_clk_speed_func for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: + * pointer to @ref kbase_pm_callback_conf + * Default value: + * See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) +extern struct kbase_pm_callback_conf pm_callbacks; + +/** + * Platform specific configuration functions + * + * Attached value: + * pointer to @ref kbase_platform_funcs_conf + * Default value: + * See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (&platform_funcs) +extern struct kbase_platform_funcs_conf platform_funcs; + +/** + * Secure mode switch + * + * Attached value: pointer to @ref kbase_secure_ops + */ +#define SECURE_CALLBACKS (NULL) + diff --git a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c new file mode 100755 index 000000000000..8ad910c1256f --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c @@ -0,0 +1,492 @@ +/* + * + * (C) COPYRIGHT ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + */ + +/* #define ENABLE_DEBUG_LOG */ +#include "custom_log.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mali_kbase_rk.h" + +/** + * @file mali_kbase_config_rk.c + * 对 platform_config_of_rk 的具体实现. + * + * mali_device_driver 包å«ä¸¤éƒ¨åˆ† : + * .DP : platform_dependent_part_in_mdd : + * ä¾èµ– platform 部分, + * æºç åœ¨ /platform// + * 在 mali_device_driver 内部, + * 记为 platform_dependent_part, + * 也被记为 platform_specific_code. + * .DP : common_parts_in_mdd : + * arm 实现的通用的部分, + * æºç åœ¨ / 下. + * 在 mali_device_driver 内部, 记为 common_parts. + */ + +/*---------------------------------------------------------------------------*/ + +#ifdef CONFIG_REGULATOR +static int rk_pm_enable_regulator(struct kbase_device *kbdev); +static void rk_pm_disable_regulator(struct kbase_device *kbdev); +#else +static inline int rk_pm_enable_regulator(struct kbase_device *kbdev) +{ + return 0; +} + +static inline void rk_pm_disable_regulator(struct kbase_device *kbdev) +{ +} +#endif + +static int rk_pm_enable_clk(struct kbase_device *kbdev); + +static void rk_pm_disable_clk(struct kbase_device *kbdev); + +static int kbase_platform_rk_create_sysfs_files(struct device *dev); + +static void kbase_platform_rk_remove_sysfs_files(struct device *dev); + +/*---------------------------------------------------------------------------*/ + +static void rk_pm_power_off_delay_work(struct work_struct *work) +{ + struct rk_context *platform = + container_of(to_delayed_work(work), struct rk_context, work); + struct kbase_device *kbdev = platform->kbdev; + + if (!platform->is_powered) { + D("mali_dev is already powered off."); + return; + } + + if (pm_runtime_enabled(kbdev->dev)) { + D("to put_sync_suspend mali_dev."); + pm_runtime_put_sync_suspend(kbdev->dev); + } + + rk_pm_disable_regulator(kbdev); + + platform->is_powered = false; + KBASE_TIMELINE_GPU_POWER(kbdev, 0); + wake_unlock(&platform->wake_lock); +} + +static int kbase_platform_rk_init(struct kbase_device *kbdev) +{ + int ret = 0; + struct rk_context *platform; + + platform = kzalloc(sizeof(*platform), GFP_KERNEL); + if (!platform) { + E("err."); + return -ENOMEM; + } + + platform->is_powered = false; + platform->kbdev = kbdev; + + platform->delay_ms = 200; + if (of_property_read_u32(kbdev->dev->of_node, "power-off-delay-ms", + &platform->delay_ms)) + W("power-off-delay-ms not available."); + + platform->power_off_wq = create_freezable_workqueue("gpu_power_off_wq"); + if (!platform->power_off_wq) { + E("couldn't create workqueue"); + ret = -ENOMEM; + goto err_wq; + } + INIT_DEFERRABLE_WORK(&platform->work, rk_pm_power_off_delay_work); + + wake_lock_init(&platform->wake_lock, WAKE_LOCK_SUSPEND, "gpu"); + + platform->utilisation_period = DEFAULT_UTILISATION_PERIOD_IN_MS; + + ret = kbase_platform_rk_create_sysfs_files(kbdev->dev); + if (ret) { + E("fail to create sysfs_files. ret = %d.", ret); + goto err_sysfs_files; + } + + kbdev->platform_context = (void *)platform; + pm_runtime_enable(kbdev->dev); + + return 0; + +err_sysfs_files: + wake_lock_destroy(&platform->wake_lock); + destroy_workqueue(platform->power_off_wq); +err_wq: + return ret; +} + +static void kbase_platform_rk_term(struct kbase_device *kbdev) +{ + struct rk_context *platform = + (struct rk_context *)kbdev->platform_context; + + pm_runtime_disable(kbdev->dev); + kbdev->platform_context = NULL; + + if (platform) { + cancel_delayed_work_sync(&platform->work); + wake_lock_destroy(&platform->wake_lock); + destroy_workqueue(platform->power_off_wq); + platform->is_powered = false; + platform->kbdev = NULL; + kfree(platform); + } + kbase_platform_rk_remove_sysfs_files(kbdev->dev); +} + +struct kbase_platform_funcs_conf platform_funcs = { + .platform_init_func = &kbase_platform_rk_init, + .platform_term_func = &kbase_platform_rk_term, +}; + +/*---------------------------------------------------------------------------*/ + +static int rk_pm_callback_runtime_on(struct kbase_device *kbdev) +{ + return 0; +} + +static void rk_pm_callback_runtime_off(struct kbase_device *kbdev) +{ +} + +static int rk_pm_callback_power_on(struct kbase_device *kbdev) +{ + int ret = 1; /* Assume GPU has been powered off */ + int err = 0; + struct rk_context *platform = get_rk_context(kbdev); + + cancel_delayed_work_sync(&platform->work); + + err = rk_pm_enable_clk(kbdev); + if (err) { + E("failed to enable clk: %d", err); + return err; + } + + if (platform->is_powered) { + D("mali_device is already powered."); + return 0; + } + + /* we must enable vdd_gpu before pd_gpu_in_chip. */ + err = rk_pm_enable_regulator(kbdev); + if (err) { + E("fail to enable regulator, err : %d.", err); + return err; + } + + /* è‹¥ mali_dev çš„ runtime_pm 是 enabled çš„, 则... */ + if (pm_runtime_enabled(kbdev->dev)) { + D("to resume mali_dev syncly."); + /* 对 pd_in_chip çš„ on æ“作, + * 将在 pm_domain çš„ runtime_pm_callbacks 中完æˆ. + */ + err = pm_runtime_get_sync(kbdev->dev); + if (err < 0) { + E("failed to runtime resume device: %d.", err); + return err; + } else if (err == 1) { /* runtime_pm_status is still active */ + D("chip has NOT been powered off, no need to re-init."); + ret = 0; + } + } + + platform->is_powered = true; + KBASE_TIMELINE_GPU_POWER(kbdev, 1); + wake_lock(&platform->wake_lock); + + return ret; +} + +static void rk_pm_callback_power_off(struct kbase_device *kbdev) +{ + struct rk_context *platform = get_rk_context(kbdev); + + rk_pm_disable_clk(kbdev); + queue_delayed_work(platform->power_off_wq, &platform->work, + msecs_to_jiffies(platform->delay_ms)); +} + +int rk_kbase_device_runtime_init(struct kbase_device *kbdev) +{ + return 0; +} + +void rk_kbase_device_runtime_disable(struct kbase_device *kbdev) +{ +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = rk_pm_callback_power_on, + .power_off_callback = rk_pm_callback_power_off, +#ifdef CONFIG_PM + .power_runtime_init_callback = rk_kbase_device_runtime_init, + .power_runtime_term_callback = rk_kbase_device_runtime_disable, + .power_runtime_on_callback = rk_pm_callback_runtime_on, + .power_runtime_off_callback = rk_pm_callback_runtime_off, +#else /* CONFIG_PM */ + .power_runtime_init_callback = NULL, + .power_runtime_term_callback = NULL, + .power_runtime_on_callback = NULL, + .power_runtime_off_callback = NULL, +#endif /* CONFIG_PM */ +}; + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} + +/*---------------------------------------------------------------------------*/ + +void kbase_platform_rk_shutdown(struct kbase_device *kbdev) +{ + I("to make vdd_gpu enabled for turning off pd_gpu in pm_framework."); + rk_pm_enable_regulator(kbdev); +} + +/*---------------------------------------------------------------------------*/ + +#ifdef CONFIG_REGULATOR +static int rk_pm_enable_regulator(struct kbase_device *kbdev) +{ + int ret = 0; + + if (!kbdev->regulator) { + W("no mali regulator control, no need to enable."); + goto EXIT; + } + + D("to enable regulator."); + ret = regulator_enable(kbdev->regulator); + if (ret) { + E("fail to enable regulator, ret : %d.", ret); + goto EXIT; + } + +EXIT: + return ret; +} + +static void rk_pm_disable_regulator(struct kbase_device *kbdev) +{ + if (!(kbdev->regulator)) { + W("no mali regulator control, no need to disable."); + return; + } + + D("to disable regulator."); + regulator_disable(kbdev->regulator); +} +#endif + +static int rk_pm_enable_clk(struct kbase_device *kbdev) +{ + int err = 0; + + if (!(kbdev->clock)) { + W("no mali clock control, no need to enable."); + } else { + D("to enable clk."); + err = clk_enable(kbdev->clock); + if (err) + E("failed to enable clk: %d.", err); + } + + return err; +} + +static void rk_pm_disable_clk(struct kbase_device *kbdev) +{ + if (!(kbdev->clock)) { + W("no mali clock control, no need to disable."); + } else { + D("to disable clk."); + clk_disable(kbdev->clock); + } +} + +/*---------------------------------------------------------------------------*/ + +static ssize_t utilisation_period_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct rk_context *platform = get_rk_context(kbdev); + ssize_t ret = 0; + + ret += snprintf(buf, PAGE_SIZE, "%u\n", platform->utilisation_period); + + return ret; +} + +static ssize_t utilisation_period_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct rk_context *platform = get_rk_context(kbdev); + int ret = 0; + + ret = kstrtouint(buf, 0, &platform->utilisation_period); + if (ret) { + E("invalid input period : %s.", buf); + return ret; + } + D("set utilisation_period to '%d'.", platform->utilisation_period); + + return count; +} + +static ssize_t utilisation_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct kbase_device *kbdev = dev_get_drvdata(dev); + struct rk_context *platform = get_rk_context(kbdev); + ssize_t ret = 0; + unsigned long period_in_us = platform->utilisation_period * 1000; + unsigned long total_time; + unsigned long busy_time; + unsigned long utilisation; + + kbase_pm_reset_dvfs_utilisation(kbdev); + usleep_range(period_in_us, period_in_us + 100); + kbase_pm_get_dvfs_utilisation(kbdev, &total_time, &busy_time); + /* 'devfreq_dev_profile' instance registered to devfreq + * also uses kbase_pm_reset_dvfs_utilisation + * and kbase_pm_get_dvfs_utilisation. + * it's better to cat this file when DVFS is disabled. + */ + D("total_time : %lu, busy_time : %lu.", total_time, busy_time); + + utilisation = busy_time * 100 / total_time; + ret += snprintf(buf, PAGE_SIZE, "%ld\n", utilisation); + + return ret; +} + +static DEVICE_ATTR_RW(utilisation_period); +static DEVICE_ATTR_RO(utilisation); + +static int kbase_platform_rk_create_sysfs_files(struct device *dev) +{ + int ret = 0; + + ret = device_create_file(dev, &dev_attr_utilisation_period); + if (ret) { + E("fail to create sysfs file 'utilisation_period'."); + goto out; + } + + ret = device_create_file(dev, &dev_attr_utilisation); + if (ret) { + E("fail to create sysfs file 'utilisation'."); + goto remove_utilisation_period; + } + + return 0; + +remove_utilisation_period: + device_remove_file(dev, &dev_attr_utilisation_period); +out: + return ret; +} + +static void kbase_platform_rk_remove_sysfs_files(struct device *dev) +{ + device_remove_file(dev, &dev_attr_utilisation_period); + device_remove_file(dev, &dev_attr_utilisation); +} + +static int rk3288_get_soc_info(struct device *dev, struct device_node *np, + int *bin, int *process) +{ + int ret = -EINVAL; + u8 value = 0; + char *name; + + if (!bin) + goto out; + + if (soc_is_rk3288w()) + name = "performance-w"; + else + name = "performance"; + if (of_property_match_string(np, "nvmem-cell-names", name) >= 0) { + ret = rockchip_nvmem_cell_read_u8(np, name, &value); + if (ret) { + dev_err(dev, "Failed to get soc performance value\n"); + goto out; + } + if (value & 0x2) + *bin = 3; + else if (value & 0x01) + *bin = 2; + else + *bin = 0; + } else { + dev_err(dev, "Failed to get bin config\n"); + } + if (*bin >= 0) + dev_info(dev, "bin=%d\n", *bin); + +out: + return ret; +} + +static const struct rockchip_opp_data rk3288_gpu_opp_data = { + .get_soc_info = rk3288_get_soc_info, +}; + +static const struct of_device_id rockchip_mali_of_match[] = { + { + .compatible = "rockchip,rk3288", + .data = (void *)&rk3288_gpu_opp_data, + }, + { + .compatible = "rockchip,rk3288w", + .data = (void *)&rk3288_gpu_opp_data, + }, + {}, +}; + +int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev) +{ + rockchip_get_opp_data(rockchip_mali_of_match, &kbdev->opp_info); + + return rockchip_init_opp_table(kbdev->dev, &kbdev->opp_info, + "gpu_leakage", "mali"); +} diff --git a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_rk.h b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_rk.h new file mode 100755 index 000000000000..6eab25014d21 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_rk.h @@ -0,0 +1,62 @@ +/* drivers/gpu/t6xx/kbase/src/platform/rk/mali_kbase_platform.h + * Rockchip SoC Mali-Midgard platform-dependent codes + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software FoundatIon. + */ + +/** + * @file mali_kbase_rk.h + * + * defines work_context type of platform_dependent_part. + */ + +#ifndef _MALI_KBASE_RK_H_ +#define _MALI_KBASE_RK_H_ + +#include + +/*---------------------------------------------------------------------------*/ + +#define DEFAULT_UTILISATION_PERIOD_IN_MS (100) + +/*---------------------------------------------------------------------------*/ + +/* + * struct rk_context - work_context of platform_dependent_part_of_rk. + */ +struct rk_context { + /* + * record the status of common_parts calling 'power_on_callback' + * and 'power_off_callback'. + */ + bool is_powered; + + struct kbase_device *kbdev; + + struct workqueue_struct *power_off_wq; + /* delayed_work_to_power_off_gpu. */ + struct delayed_work work; + unsigned int delay_ms; + + /* + * WAKE_LOCK_SUSPEND for ensuring to run + * delayed_work_to_power_off_gpu before suspend. + */ + struct wake_lock wake_lock; + + /* debug only, the period in ms to count gpu_utilisation. */ + unsigned int utilisation_period; +}; + +/*---------------------------------------------------------------------------*/ + +static inline struct rk_context *get_rk_context( + const struct kbase_device *kbdev) +{ + return (struct rk_context *)(kbdev->platform_context); +} + +#endif /* _MALI_KBASE_RK_H_ */ + diff --git a/drivers/gpu/arm/midgard/platform/vexpress/Kbuild b/drivers/gpu/arm/midgard/platform/vexpress/Kbuild new file mode 100755 index 000000000000..1caa293666d3 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress/Kbuild @@ -0,0 +1,18 @@ +# +# (C) COPYRIGHT 2012-2013, 2016 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +mali_kbase-y += \ + $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_vexpress.o \ + $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_cpu_vexpress.o diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h new file mode 100755 index 000000000000..02835f129aa3 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h @@ -0,0 +1,75 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "mali_kbase_cpu_vexpress.h" + +/** + * Maximum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX kbase_get_platform_max_freq() +/** + * Minimum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN kbase_get_platform_min_freq() + +/** + * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed) + +/** + * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c new file mode 100755 index 000000000000..15ce2bc5eea5 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c @@ -0,0 +1,85 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include +#include +#include "mali_kbase_cpu_vexpress.h" +#include "mali_kbase_config_platform.h" + +#define HARD_RESET_AT_POWER_OFF 0 + +#ifndef CONFIG_OF +static struct kbase_io_resources io_resources = { + .job_irq_number = 68, + .mmu_irq_number = 69, + .gpu_irq_number = 70, + .io_memory_region = { + .start = 0xFC010000, + .end = 0xFC010000 + (4096 * 4) - 1 + } +}; +#endif /* CONFIG_OF */ + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ + return 1; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ +#if HARD_RESET_AT_POWER_OFF + /* Cause a GPU hard reset to test whether we have actually idled the GPU + * and that we properly reconfigure the GPU on power up. + * Usually this would be dangerous, but if the GPU is working correctly it should + * be completely safe as the GPU should not be active at this point. + * However this is disabled normally because it will most likely interfere with + * bus logging etc. + */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); + kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); +#endif +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = NULL, + .power_resume_callback = NULL +}; + +static struct kbase_platform_config versatile_platform_config = { +#ifndef CONFIG_OF + .io_resources = &io_resources +#endif +}; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &versatile_platform_config; +} + + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.c new file mode 100755 index 000000000000..4665f98cbbe4 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.c @@ -0,0 +1,279 @@ +/* + * + * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include "mali_kbase_cpu_vexpress.h" + +#define HZ_IN_MHZ (1000000) + +#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000) +#define MOTHERBOARD_SYS_CFG_START (0x10000000) +#define SYS_CFGDATA_OFFSET (0x000000A0) +#define SYS_CFGCTRL_OFFSET (0x000000A4) +#define SYS_CFGSTAT_OFFSET (0x000000A8) + +#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31) +#define READ_REG_BIT_VALUE (0 << 30) +#define DCC_DEFAULT_BIT_VALUE (0 << 26) +#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20) +#define SITE_DEFAULT_BIT_VALUE (1 << 16) +#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12) +#define DEVICE_DEFAULT_BIT_VALUE (2 << 0) +#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0) +#define SYS_CFG_ERROR_BIT_VALUE (1 << 1) + +#define FEED_REG_BIT_MASK (0x0F) +#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03) +#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07) +#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B) +#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F) +#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13) + +/* the following three values used for reading + * HBI value of the LogicTile daughterboard */ +#define VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 (0x10000000) +#define VE_SYS_PROC_ID1_OFFSET (0x00000088) +#define VE_LOGIC_TILE_HBI_MASK (0x00000FFF) + +#define IS_SINGLE_BIT_SET(val, pos) (val&(1<> + FCLK_PA_DIVIDE_BIT_SHIFT); + /* CFGRW0[10:7] */ + pb_divide = ((reg_val & (FEED_REG_BIT_MASK << + FCLK_PB_DIVIDE_BIT_SHIFT)) >> + FCLK_PB_DIVIDE_BIT_SHIFT); + *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1); + } else if (IS_SINGLE_BIT_SET(reg_val, 1)) { + /* CFGRW0[1] - CLKOC */ + /* CFGRW0[6:3] */ + pa_divide = ((reg_val & (FEED_REG_BIT_MASK << + FCLK_PA_DIVIDE_BIT_SHIFT)) >> + FCLK_PA_DIVIDE_BIT_SHIFT); + /* CFGRW0[14:11] */ + pc_divide = ((reg_val & (FEED_REG_BIT_MASK << + FCLK_PC_DIVIDE_BIT_SHIFT)) >> + FCLK_PC_DIVIDE_BIT_SHIFT); + *cpu_clock = osc2_value * (pa_divide + 1) / (pc_divide + 1); + } else if (IS_SINGLE_BIT_SET(reg_val, 2)) { + /* CFGRW0[2] - FACLK */ + /* CFGRW0[18:15] */ + pa_divide = ((reg_val & (FEED_REG_BIT_MASK << + AXICLK_PA_DIVIDE_BIT_SHIFT)) >> + AXICLK_PA_DIVIDE_BIT_SHIFT); + /* CFGRW0[22:19] */ + pb_divide = ((reg_val & (FEED_REG_BIT_MASK << + AXICLK_PB_DIVIDE_BIT_SHIFT)) >> + AXICLK_PB_DIVIDE_BIT_SHIFT); + *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1); + } else { + err = -EIO; + } + +set_reg_error: +ongoing_request: + raw_spin_unlock(&syscfg_lock); + *cpu_clock /= HZ_IN_MHZ; + + if (!err) + cpu_clock_speed = *cpu_clock; + + iounmap(scc_reg); + +scc_reg_map_failed: + iounmap(syscfg_reg); + +syscfg_reg_map_failed: + + return err; +} + +/** + * kbase_get_platform_logic_tile_type - determines which LogicTile type + * is used by Versatile Express + * + * When platform_config build parameter is specified as vexpress, i.e., + * platform_config=vexpress, GPU frequency may vary dependent on the + * particular platform. The GPU frequency depends on the LogicTile type. + * + * This function determines which LogicTile type is used by the platform by + * reading the HBI value of the daughterboard which holds the LogicTile: + * + * 0x217 HBI0217 Virtex-6 + * 0x192 HBI0192 Virtex-5 + * 0x247 HBI0247 Virtex-7 + * + * Return: HBI value of the logic tile daughterboard, zero if not accessible + */ +static u32 kbase_get_platform_logic_tile_type(void) +{ + void __iomem *syscfg_reg = NULL; + u32 sys_procid1 = 0; + + syscfg_reg = ioremap(VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 + VE_SYS_PROC_ID1_OFFSET, 4); + if (NULL != syscfg_reg) { + sys_procid1 = readl(syscfg_reg); + iounmap(syscfg_reg); + } + + return sys_procid1 & VE_LOGIC_TILE_HBI_MASK; +} + +u32 kbase_get_platform_min_freq(void) +{ + u32 ve_logic_tile = kbase_get_platform_logic_tile_type(); + + switch (ve_logic_tile) { + case 0x217: + /* Virtex 6, HBI0217 */ + return VE_VIRTEX6_GPU_FREQ_MIN; + case 0x247: + /* Virtex 7, HBI0247 */ + return VE_VIRTEX7_GPU_FREQ_MIN; + default: + /* all other logic tiles, i.e., Virtex 5 HBI0192 + * or unsuccessful reading from the platform - + * fall back to some default value */ + return VE_DEFAULT_GPU_FREQ_MIN; + } +} + +u32 kbase_get_platform_max_freq(void) +{ + u32 ve_logic_tile = kbase_get_platform_logic_tile_type(); + + switch (ve_logic_tile) { + case 0x217: + /* Virtex 6, HBI0217 */ + return VE_VIRTEX6_GPU_FREQ_MAX; + case 0x247: + /* Virtex 7, HBI0247 */ + return VE_VIRTEX7_GPU_FREQ_MAX; + default: + /* all other logic tiles, i.e., Virtex 5 HBI0192 + * or unsuccessful reading from the platform - + * fall back to some default value */ + return VE_DEFAULT_GPU_FREQ_MAX; + } +} diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.h b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.h new file mode 100755 index 000000000000..da865698133a --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.h @@ -0,0 +1,38 @@ +/* + * + * (C) COPYRIGHT 2012-2013, 2015-2016 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#ifndef _KBASE_CPU_VEXPRESS_H_ +#define _KBASE_CPU_VEXPRESS_H_ + +/** + * Versatile Express implementation of @ref kbase_cpu_clk_speed_func. + */ +int kbase_get_vexpress_cpu_clock_speed(u32 *cpu_clock); + +/** + * Get the minimum GPU frequency for the attached logic tile + */ +u32 kbase_get_platform_min_freq(void); + +/** + * Get the maximum GPU frequency for the attached logic tile + */ +u32 kbase_get_platform_max_freq(void); + +#endif /* _KBASE_CPU_VEXPRESS_H_ */ diff --git a/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild new file mode 100755 index 000000000000..7efe8fa4263b --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild @@ -0,0 +1,16 @@ +# +# (C) COPYRIGHT 2013-2014, 2016 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +mali_kbase-y += $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_vexpress.o diff --git a/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h new file mode 100755 index 000000000000..0efbf3962f98 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h @@ -0,0 +1,73 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/** + * Maximum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX 5000 +/** + * Minimum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN 5000 + +/** + * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (&kbase_cpuprops_get_default_clock_speed) + +/** + * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c new file mode 100755 index 000000000000..3ff0930fb4a3 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c @@ -0,0 +1,79 @@ +/* + * + * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include +#include + +#define HARD_RESET_AT_POWER_OFF 0 + +#ifndef CONFIG_OF +static struct kbase_io_resources io_resources = { + .job_irq_number = 68, + .mmu_irq_number = 69, + .gpu_irq_number = 70, + .io_memory_region = { + .start = 0x2f010000, + .end = 0x2f010000 + (4096 * 4) - 1} +}; +#endif + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ + return 1; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ +#if HARD_RESET_AT_POWER_OFF + /* Cause a GPU hard reset to test whether we have actually idled the GPU + * and that we properly reconfigure the GPU on power up. + * Usually this would be dangerous, but if the GPU is working correctly it should + * be completely safe as the GPU should not be active at this point. + * However this is disabled normally because it will most likely interfere with + * bus logging etc. + */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); + kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); +#endif +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = NULL, + .power_resume_callback = NULL +}; + +static struct kbase_platform_config versatile_platform_config = { +#ifndef CONFIG_OF + .io_resources = &io_resources +#endif +}; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &versatile_platform_config; +} + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild new file mode 100755 index 000000000000..1caa293666d3 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild @@ -0,0 +1,18 @@ +# +# (C) COPYRIGHT 2012-2013, 2016 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +mali_kbase-y += \ + $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_vexpress.o \ + $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_cpu_vexpress.o diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h new file mode 100755 index 000000000000..dbdf21e009f9 --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h @@ -0,0 +1,75 @@ +/* + * + * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include "mali_kbase_cpu_vexpress.h" + +/** + * Maximum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MAX 10000 +/** + * Minimum frequency GPU will be clocked at. Given in kHz. + * This must be specified as there is no default value. + * + * Attached value: number in kHz + * Default value: NA + */ +#define GPU_FREQ_KHZ_MIN 10000 + +/** + * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock + * + * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_cpu_clk_speed_func. + * Default Value: NA + */ +#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed) + +/** + * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock + * + * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func + * for the function prototype. + * + * Attached value: A kbase_gpu_clk_speed_func. + * Default Value: NA + */ +#define GPU_SPEED_FUNC (NULL) + +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) + +/** + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf + */ +#define PLATFORM_FUNCS (NULL) + +extern struct kbase_pm_callback_conf pm_callbacks; diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c new file mode 100755 index 000000000000..76ffe4a1e59e --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c @@ -0,0 +1,83 @@ +/* + * + * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include +#include +#include "mali_kbase_cpu_vexpress.h" + +#define HARD_RESET_AT_POWER_OFF 0 + +#ifndef CONFIG_OF +static struct kbase_io_resources io_resources = { + .job_irq_number = 75, + .mmu_irq_number = 76, + .gpu_irq_number = 77, + .io_memory_region = { + .start = 0x2F000000, + .end = 0x2F000000 + (4096 * 4) - 1} +}; +#endif + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ + return 1; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ +#if HARD_RESET_AT_POWER_OFF + /* Cause a GPU hard reset to test whether we have actually idled the GPU + * and that we properly reconfigure the GPU on power up. + * Usually this would be dangerous, but if the GPU is working correctly it should + * be completely safe as the GPU should not be active at this point. + * However this is disabled normally because it will most likely interfere with + * bus logging etc. + */ + KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); + kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); +#endif +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = NULL, + .power_resume_callback = NULL +}; + +static struct kbase_platform_config versatile_platform_config = { +#ifndef CONFIG_OF + .io_resources = &io_resources +#endif +}; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &versatile_platform_config; +} + +int kbase_platform_early_init(void) +{ + /* Nothing needed at this stage */ + return 0; +} + diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c new file mode 100755 index 000000000000..816dff49835f --- /dev/null +++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c @@ -0,0 +1,71 @@ +/* + * + * (C) COPYRIGHT 2011-2013 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + + + +#include +#include +#include "mali_kbase_cpu_vexpress.h" + +#define HZ_IN_MHZ (1000000) + +#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000) +#define MOTHERBOARD_SYS_CFG_START (0x10000000) +#define SYS_CFGDATA_OFFSET (0x000000A0) +#define SYS_CFGCTRL_OFFSET (0x000000A4) +#define SYS_CFGSTAT_OFFSET (0x000000A8) + +#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31) +#define READ_REG_BIT_VALUE (0 << 30) +#define DCC_DEFAULT_BIT_VALUE (0 << 26) +#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20) +#define SITE_DEFAULT_BIT_VALUE (1 << 16) +#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12) +#define DEVICE_DEFAULT_BIT_VALUE (2 << 0) +#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0) +#define SYS_CFG_ERROR_BIT_VALUE (1 << 1) + +#define FEED_REG_BIT_MASK (0x0F) +#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03) +#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07) +#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B) +#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F) +#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13) + +#define IS_SINGLE_BIT_SET(val, pos) (val&(1< + +/** + * @addtogroup uk_api User-Kernel Interface API + * @{ + */ + +/** + * @addtogroup uk_api_kernel UKK (Kernel side) + * @{ + */ + +/** + * Internal OS specific data structure associated with each UKK session. Part + * of a ukk_session object. + */ +typedef struct ukkp_session { + int dummy; /**< No internal OS specific data at this time */ +} ukkp_session; + +/** @} end group uk_api_kernel */ + +/** @} end group uk_api */ + +#endif /* _UKK_OS_H__ */ diff --git a/drivers/gpu/arm/midgard/protected_mode_switcher.h b/drivers/gpu/arm/midgard/protected_mode_switcher.h new file mode 100755 index 000000000000..5dc2f3ba8cf6 --- /dev/null +++ b/drivers/gpu/arm/midgard/protected_mode_switcher.h @@ -0,0 +1,64 @@ +/* + * + * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _PROTECTED_MODE_SWITCH_H_ +#define _PROTECTED_MODE_SWITCH_H_ + +struct protected_mode_device; + +/** + * struct protected_mode_ops - Callbacks for protected mode switch operations + * + * @protected_mode_enable: Callback to enable protected mode for device + * @protected_mode_disable: Callback to disable protected mode for device + */ +struct protected_mode_ops { + /** + * protected_mode_enable() - Enable protected mode on device + * @dev: The struct device + * + * Return: 0 on success, non-zero on error + */ + int (*protected_mode_enable)( + struct protected_mode_device *protected_dev); + + /** + * protected_mode_disable() - Disable protected mode on device, and + * reset device + * @dev: The struct device + * + * Return: 0 on success, non-zero on error + */ + int (*protected_mode_disable)( + struct protected_mode_device *protected_dev); +}; + +/** + * struct protected_mode_device - Device structure for protected mode devices + * + * @ops - Callbacks associated with this device + * @data - Pointer to device private data + * + * This structure should be registered with the platform device using + * platform_set_drvdata(). + */ +struct protected_mode_device { + struct protected_mode_ops ops; + void *data; +}; + +#endif /* _PROTECTED_MODE_SWITCH_H_ */ diff --git a/drivers/gpu/arm/midgard/rename.h b/drivers/gpu/arm/midgard/rename.h new file mode 100755 index 000000000000..c94b67ff2c27 --- /dev/null +++ b/drivers/gpu/arm/midgard/rename.h @@ -0,0 +1,422 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _RENAME_H +#define _RENAME_H +#define __crc_kbase_create_context midgard___crc_kbase_create_context +#define __crc_kbase_destroy_context midgard___crc_kbase_destroy_context +#define __crc_kbase_find_device midgard___crc_kbase_find_device +#define __crc_kbase_instr_hwcnt_clear midgard___crc_kbase_instr_hwcnt_clear +#define __crc_kbase_instr_hwcnt_dump_complete midgard___crc_kbase_instr_hwcnt_dump_complete +#define __crc_kbase_instr_hwcnt_request_dump midgard___crc_kbase_instr_hwcnt_request_dump +#define __crc_kbase_release_device midgard___crc_kbase_release_device +#define jd_done_nolock midgard_jd_done_nolock +#define kbase_add_va_region midgard_kbase_add_va_region +#define kbase_alloc_free_region midgard_kbase_alloc_free_region +#define kbase_alloc_phy_pages_helper midgard_kbase_alloc_phy_pages_helper +#define kbase_alloc_phy_pages midgard_kbase_alloc_phy_pages +#define kbase_as_fault_debugfs_init midgard_kbase_as_fault_debugfs_init +#define kbase_backend_complete_wq midgard_kbase_backend_complete_wq +#define kbase_backend_complete_wq_post_sched midgard_kbase_backend_complete_wq_post_sched +#define kbase_backend_ctx_count_changed midgard_kbase_backend_ctx_count_changed +#define kbase_backend_find_and_release_free_address_space midgard_kbase_backend_find_and_release_free_address_space +#define kbase_backend_get_current_flush_id midgard_kbase_backend_get_current_flush_id +#define kbase_backend_get_gpu_time midgard_kbase_backend_get_gpu_time +#define kbase_backend_gpuprops_get_features midgard_kbase_backend_gpuprops_get_features +#define kbase_backend_gpuprops_get midgard_kbase_backend_gpuprops_get +#define kbase_backend_inspect_tail midgard_kbase_backend_inspect_tail +#define kbase_backend_nr_atoms_on_slot midgard_kbase_backend_nr_atoms_on_slot +#define kbase_backend_nr_atoms_submitted midgard_kbase_backend_nr_atoms_submitted +#define kbase_backend_release_ctx_irq midgard_kbase_backend_release_ctx_irq +#define kbase_backend_release_ctx_noirq midgard_kbase_backend_release_ctx_noirq +#define kbase_backend_reset midgard_kbase_backend_reset +#define kbase_backend_run_atom midgard_kbase_backend_run_atom +#define kbase_backend_slot_free midgard_kbase_backend_slot_free +#define kbase_backend_slot_update midgard_kbase_backend_slot_update +#define kbase_backend_soft_hard_stop_slot midgard_kbase_backend_soft_hard_stop_slot +#define kbase_backend_timeouts_changed midgard_kbase_backend_timeouts_changed +#define kbase_backend_timer_init midgard_kbase_backend_timer_init +#define kbase_backend_timer_resume midgard_kbase_backend_timer_resume +#define kbase_backend_timer_suspend midgard_kbase_backend_timer_suspend +#define kbase_backend_timer_term midgard_kbase_backend_timer_term +#define kbase_backend_use_ctx midgard_kbase_backend_use_ctx +#define kbase_backend_use_ctx_sched midgard_kbase_backend_use_ctx_sched +#define kbase_cache_enabled midgard_kbase_cache_enabled +#define kbase_cache_set_coherency_mode midgard_kbase_cache_set_coherency_mode +#define kbase_cancel_soft_job midgard_kbase_cancel_soft_job +#define kbase_check_alloc_flags midgard_kbase_check_alloc_flags +#define kbase_check_import_flags midgard_kbase_check_import_flags +#define kbase_clean_caches_done midgard_kbase_clean_caches_done +#define kbase_create_context midgard_kbase_create_context +#define kbase_ctx_sched_init midgard_kbase_ctx_sched_init +#define kbase_ctx_sched_release_ctx midgard_kbase_ctx_sched_release_ctx +#define kbase_ctx_sched_remove_ctx midgard_kbase_ctx_sched_remove_ctx +#define kbase_ctx_sched_restore_all_as midgard_kbase_ctx_sched_restore_all_as +#define kbase_ctx_sched_retain_ctx midgard_kbase_ctx_sched_retain_ctx +#define kbase_ctx_sched_retain_ctx_refcount midgard_kbase_ctx_sched_retain_ctx_refcount +#define kbase_ctx_sched_term midgard_kbase_ctx_sched_term +#define kbase_debug_assert_register_hook midgard_kbase_debug_assert_register_hook +#define kbase_debug_job_fault_context_init midgard_kbase_debug_job_fault_context_init +#define kbase_debug_job_fault_context_term midgard_kbase_debug_job_fault_context_term +#define kbase_debug_job_fault_debugfs_init midgard_kbase_debug_job_fault_debugfs_init +#define kbase_debug_job_fault_dev_init midgard_kbase_debug_job_fault_dev_init +#define kbase_debug_job_fault_dev_term midgard_kbase_debug_job_fault_dev_term +#define kbase_debug_job_fault_process midgard_kbase_debug_job_fault_process +#define kbase_debug_job_fault_reg_snapshot_init midgard_kbase_debug_job_fault_reg_snapshot_init +#define kbase_debug_mem_view_init midgard_kbase_debug_mem_view_init +#define kbase_destroy_context midgard_kbase_destroy_context +#define kbase_devfreq_init midgard_kbase_devfreq_init +#define kbase_devfreq_set_core_mask midgard_kbase_devfreq_set_core_mask +#define kbase_devfreq_term midgard_kbase_devfreq_term +#define kbase_device_alloc midgard_kbase_device_alloc +#define kbase_device_free midgard_kbase_device_free +#define kbase_device_init midgard_kbase_device_init +#define kbase_device_term midgard_kbase_device_term +#define kbase_disjoint_event_get midgard_kbase_disjoint_event_get +#define kbase_disjoint_event midgard_kbase_disjoint_event +#define kbase_disjoint_event_potential midgard_kbase_disjoint_event_potential +#define kbase_disjoint_init midgard_kbase_disjoint_init +#define kbase_disjoint_state_down midgard_kbase_disjoint_state_down +#define kbase_disjoint_state_up midgard_kbase_disjoint_state_up +#define kbase_drv_name midgard_kbase_drv_name +#define kbase_event_cleanup midgard_kbase_event_cleanup +#define kbase_event_close midgard_kbase_event_close +#define kbase_event_dequeue midgard_kbase_event_dequeue +#define kbase_event_init midgard_kbase_event_init +#define kbase_event_pending midgard_kbase_event_pending +#define kbase_event_post midgard_kbase_event_post +#define kbase_event_wakeup midgard_kbase_event_wakeup +#define kbase_fence_add_callback midgard_kbase_fence_add_callback +#define kbase_fence_free_callbacks midgard_kbase_fence_free_callbacks +#define kbase_fence_ops midgard_kbase_fence_ops +#define kbase_fence_out_new midgard_kbase_fence_out_new +#define kbase_find_device midgard_kbase_find_device +#define kbase_finish_soft_job midgard_kbase_finish_soft_job +#define kbase_flush_mmu_wqs midgard_kbase_flush_mmu_wqs +#define kbase_free_alloced_region midgard_kbase_free_alloced_region +#define kbase_free_phy_pages_helper midgard_kbase_free_phy_pages_helper +#define kbase_get_real_power midgard_kbase_get_real_power +#define kbase_gpu_complete_hw midgard_kbase_gpu_complete_hw +#define kbase_gpu_dump_slots midgard_kbase_gpu_dump_slots +#define kbase_gpu_inspect midgard_kbase_gpu_inspect +#define kbase_gpu_interrupt midgard_kbase_gpu_interrupt +#define kbase_gpu_irq_evict midgard_kbase_gpu_irq_evict +#define kbase_gpu_mmap midgard_kbase_gpu_mmap +#define kbase_gpu_munmap midgard_kbase_gpu_munmap +#define kbase_gpuprops_populate_user_buffer midgard_kbase_gpuprops_populate_user_buffer +#define kbase_gpuprops_set_features midgard_kbase_gpuprops_set_features +#define kbase_gpuprops_set midgard_kbase_gpuprops_set +#define kbase_gpuprops_update_core_props_gpu_id midgard_kbase_gpuprops_update_core_props_gpu_id +#define kbase_gpu_vm_lock midgard_kbase_gpu_vm_lock +#define kbase_gpu_vm_unlock midgard_kbase_gpu_vm_unlock +#define kbase_hwaccess_pm_gpu_active midgard_kbase_hwaccess_pm_gpu_active +#define kbase_hwaccess_pm_gpu_idle midgard_kbase_hwaccess_pm_gpu_idle +#define kbase_hwaccess_pm_halt midgard_kbase_hwaccess_pm_halt +#define kbase_hwaccess_pm_init midgard_kbase_hwaccess_pm_init +#define kbase_hwaccess_pm_powerup midgard_kbase_hwaccess_pm_powerup +#define kbase_hwaccess_pm_resume midgard_kbase_hwaccess_pm_resume +#define kbase_hwaccess_pm_suspend midgard_kbase_hwaccess_pm_suspend +#define kbase_hwaccess_pm_term midgard_kbase_hwaccess_pm_term +#define kbase_hw_set_features_mask midgard_kbase_hw_set_features_mask +#define kbase_hw_set_issues_mask midgard_kbase_hw_set_issues_mask +#define kbase_install_interrupts midgard_kbase_install_interrupts +#define kbase_instr_backend_init midgard_kbase_instr_backend_init +#define kbase_instr_backend_term midgard_kbase_instr_backend_term +#define kbase_instr_hwcnt_clear midgard_kbase_instr_hwcnt_clear +#define kbase_instr_hwcnt_disable_internal midgard_kbase_instr_hwcnt_disable_internal +#define kbase_instr_hwcnt_dump_complete midgard_kbase_instr_hwcnt_dump_complete +#define kbase_instr_hwcnt_enable_internal midgard_kbase_instr_hwcnt_enable_internal +#define kbase_instr_hwcnt_request_dump midgard_kbase_instr_hwcnt_request_dump +#define kbase_instr_hwcnt_sample_done midgard_kbase_instr_hwcnt_sample_done +#define kbase_instr_hwcnt_wait_for_dump midgard_kbase_instr_hwcnt_wait_for_dump +#define kbase_invoke_smc_fid midgard_kbase_invoke_smc_fid +#define kbase_invoke_smc midgard_kbase_invoke_smc +#define kbase_io_history_dump midgard_kbase_io_history_dump +#define kbase_io_history_init midgard_kbase_io_history_init +#define kbase_io_history_term midgard_kbase_io_history_term +#define kbase_ipa_debugfs_init midgard_kbase_ipa_debugfs_init +#define kbase_ipa_init midgard_kbase_ipa_init +#define kbase_ipa_init_model midgard_kbase_ipa_init_model +#define kbase_ipa_model_add_param_s32 midgard_kbase_ipa_model_add_param_s32 +#define kbase_ipa_model_add_param_string midgard_kbase_ipa_model_add_param_string +#define kbase_ipa_model_name_from_id midgard_kbase_ipa_model_name_from_id +#define kbase_ipa_model_param_add midgard_kbase_ipa_model_param_add +#define kbase_ipa_model_param_free_all midgard_kbase_ipa_model_param_free_all +#define kbase_ipa_model_recalculate midgard_kbase_ipa_model_recalculate +#define kbase_ipa_power_model_ops midgard_kbase_ipa_power_model_ops +#define kbase_ipa_term midgard_kbase_ipa_term +#define kbase_ipa_term_model midgard_kbase_ipa_term_model +#define kbase_jd_cancel midgard_kbase_jd_cancel +#define kbase_jd_done midgard_kbase_jd_done +#define kbase_jd_done_worker midgard_kbase_jd_done_worker +#define kbase_jd_exit midgard_kbase_jd_exit +#define kbase_jd_free_external_resources midgard_kbase_jd_free_external_resources +#define kbase_jd_init midgard_kbase_jd_init +#define kbase_jd_submit midgard_kbase_jd_submit +#define kbase_jd_zap_context midgard_kbase_jd_zap_context +#define kbase_jit_allocate midgard_kbase_jit_allocate +#define kbase_jit_backing_lost midgard_kbase_jit_backing_lost +#define kbase_jit_debugfs_init midgard_kbase_jit_debugfs_init +#define kbase_jit_evict midgard_kbase_jit_evict +#define kbase_jit_free midgard_kbase_jit_free +#define kbase_jit_init midgard_kbase_jit_init +#define kbase_jit_term midgard_kbase_jit_term +#define kbase_jm_complete midgard_kbase_jm_complete +#define kbase_jm_idle_ctx midgard_kbase_jm_idle_ctx +#define kbase_jm_kick midgard_kbase_jm_kick +#define kbase_jm_return_atom_to_js midgard_kbase_jm_return_atom_to_js +#define kbase_jm_try_kick_all midgard_kbase_jm_try_kick_all +#define kbase_jm_try_kick midgard_kbase_jm_try_kick +#define kbase_jm_wait_for_zero_jobs midgard_kbase_jm_wait_for_zero_jobs +#define kbase_job_check_enter_disjoint midgard_kbase_job_check_enter_disjoint +#define kbase_job_check_leave_disjoint midgard_kbase_job_check_leave_disjoint +#define kbase_job_done midgard_kbase_job_done +#define kbase_job_fault_get_reg_snapshot midgard_kbase_job_fault_get_reg_snapshot +#define kbase_job_hw_submit midgard_kbase_job_hw_submit +#define kbase_job_slot_ctx_priority_check_locked midgard_kbase_job_slot_ctx_priority_check_locked +#define kbase_job_slot_halt midgard_kbase_job_slot_halt +#define kbase_job_slot_hardstop midgard_kbase_job_slot_hardstop +#define kbase_job_slot_init midgard_kbase_job_slot_init +#define kbase_job_slot_softstop midgard_kbase_job_slot_softstop +#define kbase_job_slot_softstop_swflags midgard_kbase_job_slot_softstop_swflags +#define kbase_job_slot_term midgard_kbase_job_slot_term +#define kbase_js_complete_atom midgard_kbase_js_complete_atom +#define kbase_js_complete_atom_wq midgard_kbase_js_complete_atom_wq +#define kbase_js_dep_resolved_submit midgard_kbase_js_dep_resolved_submit +#define kbase_js_is_atom_valid midgard_kbase_js_is_atom_valid +#define kbase_js_pull midgard_kbase_js_pull +#define kbase_js_sched midgard_kbase_js_sched +#define kbase_js_set_timeouts midgard_kbase_js_set_timeouts +#define kbase_js_unpull midgard_kbase_js_unpull +#define kbase_js_zap_context midgard_kbase_js_zap_context +#define kbase_map_external_resource midgard_kbase_map_external_resource +#define kbase_mem_alias midgard_kbase_mem_alias +#define kbase_mem_alloc midgard_kbase_mem_alloc +#define kbase_mem_alloc_page midgard_kbase_mem_alloc_page +#define kbase_mem_commit midgard_kbase_mem_commit +#define kbase_mem_evictable_deinit midgard_kbase_mem_evictable_deinit +#define kbase_mem_evictable_init midgard_kbase_mem_evictable_init +#define kbase_mem_evictable_make midgard_kbase_mem_evictable_make +#define kbase_mem_evictable_unmake midgard_kbase_mem_evictable_unmake +#define kbase_mem_flags_change midgard_kbase_mem_flags_change +#define kbase_mem_free midgard_kbase_mem_free +#define kbase_mem_free_region midgard_kbase_mem_free_region +#define kbase_mem_grow_gpu_mapping midgard_kbase_mem_grow_gpu_mapping +#define kbase_mem_halt midgard_kbase_mem_halt +#define kbase_mem_import midgard_kbase_mem_import +#define kbase_mem_init midgard_kbase_mem_init +#define kbase_mem_kref_free midgard_kbase_mem_kref_free +#define kbase_mem_pool_alloc midgard_kbase_mem_pool_alloc +#define kbase_mem_pool_alloc_pages midgard_kbase_mem_pool_alloc_pages +#define kbase_mem_pool_debugfs_init midgard_kbase_mem_pool_debugfs_init +#define kbase_mem_pool_free midgard_kbase_mem_pool_free +#define kbase_mem_pool_free_pages midgard_kbase_mem_pool_free_pages +#define kbase_mem_pool_grow midgard_kbase_mem_pool_grow +#define kbase_mem_pool_init midgard_kbase_mem_pool_init +#define kbase_mem_pool_set_max_size midgard_kbase_mem_pool_set_max_size +#define kbase_mem_pool_term midgard_kbase_mem_pool_term +#define kbase_mem_pool_trim midgard_kbase_mem_pool_trim +#define kbase_mem_query midgard_kbase_mem_query +#define kbase_mem_term midgard_kbase_mem_term +#define kbase_mmu_disable_as midgard_kbase_mmu_disable_as +#define kbase_mmu_disable midgard_kbase_mmu_disable +#define kbase_mmu_dump midgard_kbase_mmu_dump +#define kbase_mmu_hw_clear_fault midgard_kbase_mmu_hw_clear_fault +#define kbase_mmu_hw_configure midgard_kbase_mmu_hw_configure +#define kbase_mmu_hw_do_operation midgard_kbase_mmu_hw_do_operation +#define kbase_mmu_hw_enable_fault midgard_kbase_mmu_hw_enable_fault +#define kbase_mmu_init midgard_kbase_mmu_init +#define kbase_mmu_insert_pages midgard_kbase_mmu_insert_pages +#define kbase_mmu_insert_pages_no_flush midgard_kbase_mmu_insert_pages_no_flush +#define kbase_mmu_insert_single_page midgard_kbase_mmu_insert_single_page +#define kbase_mmu_interrupt midgard_kbase_mmu_interrupt +#define kbase_mmu_mode_get_aarch64 midgard_kbase_mmu_mode_get_aarch64 +#define kbase_mmu_mode_get_lpae midgard_kbase_mmu_mode_get_lpae +#define kbase_mmu_teardown_pages midgard_kbase_mmu_teardown_pages +#define kbase_mmu_term midgard_kbase_mmu_term +#define kbase_mmu_update midgard_kbase_mmu_update +#define kbase_mmu_update_pages midgard_kbase_mmu_update_pages +#define kbase_os_mem_map_lock midgard_kbase_os_mem_map_lock +#define kbase_os_mem_map_unlock midgard_kbase_os_mem_map_unlock +#define kbasep_cache_clean_worker midgard_kbasep_cache_clean_worker +#define kbasep_common_test_interrupt_handlers midgard_kbasep_common_test_interrupt_handlers +#define kbasep_complete_triggered_soft_events midgard_kbasep_complete_triggered_soft_events +#define kbasep_debug_assert_call_hook midgard_kbasep_debug_assert_call_hook +#define kbasep_find_enclosing_cpu_mapping_offset midgard_kbasep_find_enclosing_cpu_mapping_offset +#define kbasep_gpu_memory_debugfs_init midgard_kbasep_gpu_memory_debugfs_init +#define kbasep_jd_debugfs_ctx_init midgard_kbasep_jd_debugfs_ctx_init +#define kbasep_job_slot_soft_or_hard_stop_do_action midgard_kbasep_job_slot_soft_or_hard_stop_do_action +#define kbasep_js_add_job midgard_kbasep_js_add_job +#define kbasep_js_atom_priority_to_relative midgard_kbasep_js_atom_priority_to_relative +#define kbasep_js_ctx_attr_ctx_release_atom midgard_kbasep_js_ctx_attr_ctx_release_atom +#define kbasep_js_ctx_attr_ctx_retain_atom midgard_kbasep_js_ctx_attr_ctx_retain_atom +#define kbasep_js_ctx_attr_runpool_release_ctx midgard_kbasep_js_ctx_attr_runpool_release_ctx +#define kbasep_js_ctx_attr_runpool_retain_ctx midgard_kbasep_js_ctx_attr_runpool_retain_ctx +#define kbasep_js_devdata_halt midgard_kbasep_js_devdata_halt +#define kbasep_js_devdata_init midgard_kbasep_js_devdata_init +#define kbasep_js_devdata_term midgard_kbasep_js_devdata_term +#define kbasep_js_kctx_init midgard_kbasep_js_kctx_init +#define kbasep_js_kctx_term midgard_kbasep_js_kctx_term +#define kbasep_js_relative_priority_to_atom midgard_kbasep_js_relative_priority_to_atom +#define kbasep_js_release_privileged_ctx midgard_kbasep_js_release_privileged_ctx +#define kbasep_js_remove_cancelled_job midgard_kbasep_js_remove_cancelled_job +#define kbasep_js_remove_job midgard_kbasep_js_remove_job +#define kbasep_js_resume midgard_kbasep_js_resume +#define kbasep_js_runpool_release_ctx_and_katom_retained_state midgard_kbasep_js_runpool_release_ctx_and_katom_retained_state +#define kbasep_js_runpool_release_ctx midgard_kbasep_js_runpool_release_ctx +#define kbasep_js_runpool_release_ctx_nolock midgard_kbasep_js_runpool_release_ctx_nolock +#define kbasep_js_runpool_requeue_or_kill_ctx midgard_kbasep_js_runpool_requeue_or_kill_ctx +#define kbasep_js_schedule_privileged_ctx midgard_kbasep_js_schedule_privileged_ctx +#define kbasep_js_suspend midgard_kbasep_js_suspend +#define kbase_platform_early_init midgard_kbase_platform_early_init +#define kbase_platform_rk_init_opp_table midgard_kbase_platform_rk_init_opp_table +#define kbase_platform_rk_shutdown midgard_kbase_platform_rk_shutdown +#define kbase_pm_always_on_policy_ops midgard_kbase_pm_always_on_policy_ops +#define kbase_pm_cache_snoop_disable midgard_kbase_pm_cache_snoop_disable +#define kbase_pm_cache_snoop_enable midgard_kbase_pm_cache_snoop_enable +#define kbase_pm_ca_get_core_mask midgard_kbase_pm_ca_get_core_mask +#define kbase_pm_ca_init midgard_kbase_pm_ca_init +#define kbase_pm_ca_term midgard_kbase_pm_ca_term +#define kbase_pm_clock_off midgard_kbase_pm_clock_off +#define kbase_pm_clock_on midgard_kbase_pm_clock_on +#define kbase_pm_coarse_demand_policy_ops midgard_kbase_pm_coarse_demand_policy_ops +#define kbase_pm_context_active_handle_suspend midgard_kbase_pm_context_active_handle_suspend +#define kbase_pm_context_active midgard_kbase_pm_context_active +#define kbase_pm_context_idle midgard_kbase_pm_context_idle +#define kbase_pm_disable_interrupts midgard_kbase_pm_disable_interrupts +#define kbase_pm_disable_interrupts_nolock midgard_kbase_pm_disable_interrupts_nolock +#define kbase_pm_do_poweroff midgard_kbase_pm_do_poweroff +#define kbase_pm_do_poweron midgard_kbase_pm_do_poweron +#define kbasep_mem_profile_debugfs_insert midgard_kbasep_mem_profile_debugfs_insert +#define kbasep_mem_profile_debugfs_remove midgard_kbasep_mem_profile_debugfs_remove +#define kbase_pm_enable_interrupts midgard_kbase_pm_enable_interrupts +#define kbase_pm_get_active_cores midgard_kbase_pm_get_active_cores +#define kbase_pm_get_policy midgard_kbase_pm_get_policy +#define kbase_pm_get_present_cores midgard_kbase_pm_get_present_cores +#define kbase_pm_get_ready_cores midgard_kbase_pm_get_ready_cores +#define kbase_pm_get_trans_cores midgard_kbase_pm_get_trans_cores +#define kbase_pm_halt midgard_kbase_pm_halt +#define kbase_pm_init_hw midgard_kbase_pm_init_hw +#define kbase_pm_list_policies midgard_kbase_pm_list_policies +#define kbase_pm_metrics_update midgard_kbase_pm_metrics_update +#define kbase_pm_policy_init midgard_kbase_pm_policy_init +#define kbase_pm_policy_term midgard_kbase_pm_policy_term +#define kbase_pm_power_changed midgard_kbase_pm_power_changed +#define kbase_pm_powerup midgard_kbase_pm_powerup +#define kbase_pm_register_access_disable midgard_kbase_pm_register_access_disable +#define kbase_pm_register_access_enable midgard_kbase_pm_register_access_enable +#define kbase_pm_release_gpu_cycle_counter midgard_kbase_pm_release_gpu_cycle_counter +#define kbase_pm_release_gpu_cycle_counter_nolock midgard_kbase_pm_release_gpu_cycle_counter_nolock +#define kbase_pm_request_gpu_cycle_counter_l2_is_on midgard_kbase_pm_request_gpu_cycle_counter_l2_is_on +#define kbase_pm_request_gpu_cycle_counter midgard_kbase_pm_request_gpu_cycle_counter +#define kbase_pm_reset_done midgard_kbase_pm_reset_done +#define kbase_pm_resume midgard_kbase_pm_resume +#define kbase_pm_set_debug_core_mask midgard_kbase_pm_set_debug_core_mask +#define kbase_pm_set_policy midgard_kbase_pm_set_policy +#define kbase_pm_suspend midgard_kbase_pm_suspend +#define kbase_pm_update_active midgard_kbase_pm_update_active +#define kbase_pm_update_cores_state midgard_kbase_pm_update_cores_state +#define kbase_pm_update_cores_state_nolock midgard_kbase_pm_update_cores_state_nolock +#define kbase_pm_wait_for_poweroff_complete midgard_kbase_pm_wait_for_poweroff_complete +#define kbasep_os_process_page_usage_update midgard_kbasep_os_process_page_usage_update +#define kbasep_platform_device_init midgard_kbasep_platform_device_init +#define kbasep_platform_device_term midgard_kbasep_platform_device_term +#define kbasep_pm_metrics_init midgard_kbasep_pm_metrics_init +#define kbasep_pm_metrics_term midgard_kbasep_pm_metrics_term +#define kbasep_regs_history_debugfs_init midgard_kbasep_regs_history_debugfs_init +#define kbasep_remove_waiting_soft_job midgard_kbasep_remove_waiting_soft_job +#define kbase_prepare_soft_job midgard_kbase_prepare_soft_job +#define kbase_prepare_to_reset_gpu_locked midgard_kbase_prepare_to_reset_gpu_locked +#define kbase_prepare_to_reset_gpu midgard_kbase_prepare_to_reset_gpu +#define kbase_process_soft_job midgard_kbase_process_soft_job +#define kbasep_soft_job_timeout_worker midgard_kbasep_soft_job_timeout_worker +#define kbase_region_tracker_find_region_base_address midgard_kbase_region_tracker_find_region_base_address +#define kbase_region_tracker_find_region_enclosing_address midgard_kbase_region_tracker_find_region_enclosing_address +#define kbase_region_tracker_init_jit midgard_kbase_region_tracker_init_jit +#define kbase_region_tracker_init midgard_kbase_region_tracker_init +#define kbase_region_tracker_term midgard_kbase_region_tracker_term +#define kbase_reg_read midgard_kbase_reg_read +#define kbase_reg_write midgard_kbase_reg_write +#define kbase_release_device midgard_kbase_release_device +#define kbase_release_interrupts midgard_kbase_release_interrupts +#define kbase_reset_gpu_locked midgard_kbase_reset_gpu_locked +#define kbase_reset_gpu midgard_kbase_reset_gpu +#define kbase_reset_gpu_silent midgard_kbase_reset_gpu_silent +#define kbase_resume_suspended_soft_jobs midgard_kbase_resume_suspended_soft_jobs +#define kbase_scale_static_power midgard_kbase_scale_static_power +#define kbase_set_custom_irq_handler midgard_kbase_set_custom_irq_handler +#define kbase_simple_ipa_model_ops midgard_kbase_simple_ipa_model_ops +#define kbase_soft_event_update midgard_kbase_soft_event_update +#define kbase_soft_event_wait_callback midgard_kbase_soft_event_wait_callback +#define kbase_sticky_resource_acquire midgard_kbase_sticky_resource_acquire +#define kbase_sticky_resource_init midgard_kbase_sticky_resource_init +#define kbase_sticky_resource_release midgard_kbase_sticky_resource_release +#define kbase_sticky_resource_term midgard_kbase_sticky_resource_term +#define kbase_sync_fence_in_cancel_wait midgard_kbase_sync_fence_in_cancel_wait +#define kbase_sync_fence_in_dump midgard_kbase_sync_fence_in_dump +#define kbase_sync_fence_in_from_fd midgard_kbase_sync_fence_in_from_fd +#define kbase_sync_fence_in_info_get midgard_kbase_sync_fence_in_info_get +#define kbase_sync_fence_in_remove midgard_kbase_sync_fence_in_remove +#define kbase_sync_fence_in_wait midgard_kbase_sync_fence_in_wait +#define kbase_sync_fence_out_create midgard_kbase_sync_fence_out_create +#define kbase_sync_fence_out_info_get midgard_kbase_sync_fence_out_info_get +#define kbase_sync_fence_out_remove midgard_kbase_sync_fence_out_remove +#define kbase_sync_fence_out_trigger midgard_kbase_sync_fence_out_trigger +#define kbase_sync_fence_stream_create midgard_kbase_sync_fence_stream_create +#define kbase_sync_fence_validate midgard_kbase_sync_fence_validate +#define kbase_sync_fence_wait_worker midgard_kbase_sync_fence_wait_worker +#define kbase_synchronize_irqs midgard_kbase_synchronize_irqs +#define kbase_sync_now midgard_kbase_sync_now +#define kbase_sync_single_for_cpu midgard_kbase_sync_single_for_cpu +#define kbase_sync_single_for_device midgard_kbase_sync_single_for_device +#define kbase_sync_single midgard_kbase_sync_single +#define kbase_sync_status_string midgard_kbase_sync_status_string +#define kbase_timeline_name midgard_kbase_timeline_name +#define __kbase_tlstream_aux_devfreq_target midgard___kbase_tlstream_aux_devfreq_target +#define __kbase_tlstream_aux_pagefault midgard___kbase_tlstream_aux_pagefault +#define __kbase_tlstream_aux_pagesalloc midgard___kbase_tlstream_aux_pagesalloc +#define __kbase_tlstream_aux_pm_state midgard___kbase_tlstream_aux_pm_state +#define __kbase_tlstream_aux_protected_enter_end midgard___kbase_tlstream_aux_protected_enter_end +#define __kbase_tlstream_aux_protected_enter_start midgard___kbase_tlstream_aux_protected_enter_start +#define __kbase_tlstream_aux_protected_leave_end midgard___kbase_tlstream_aux_protected_leave_end +#define __kbase_tlstream_aux_protected_leave_start midgard___kbase_tlstream_aux_protected_leave_start +#define kbase_tlstream_init midgard_kbase_tlstream_init +#define __kbase_tlstream_jd_gpu_soft_reset midgard___kbase_tlstream_jd_gpu_soft_reset +#define kbase_tlstream_term midgard_kbase_tlstream_term +#define __kbase_tlstream_tl_attrib_as_config midgard___kbase_tlstream_tl_attrib_as_config +#define __kbase_tlstream_tl_attrib_atom_config midgard___kbase_tlstream_tl_attrib_atom_config +#define __kbase_tlstream_tl_attrib_atom_jit midgard___kbase_tlstream_tl_attrib_atom_jit +#define __kbase_tlstream_tl_attrib_atom_priority midgard___kbase_tlstream_tl_attrib_atom_priority +#define __kbase_tlstream_tl_attrib_atom_state midgard___kbase_tlstream_tl_attrib_atom_state +#define __kbase_tlstream_tl_del_atom midgard___kbase_tlstream_tl_del_atom +#define __kbase_tlstream_tl_del_ctx midgard___kbase_tlstream_tl_del_ctx +#define __kbase_tlstream_tl_event_atom_softstop_ex midgard___kbase_tlstream_tl_event_atom_softstop_ex +#define __kbase_tlstream_tl_event_atom_softstop_issue midgard___kbase_tlstream_tl_event_atom_softstop_issue +#define __kbase_tlstream_tl_event_lpu_softstop midgard___kbase_tlstream_tl_event_lpu_softstop +#define __kbase_tlstream_tl_new_atom midgard___kbase_tlstream_tl_new_atom +#define __kbase_tlstream_tl_new_ctx midgard___kbase_tlstream_tl_new_ctx +#define __kbase_tlstream_tl_nret_as_ctx midgard___kbase_tlstream_tl_nret_as_ctx +#define __kbase_tlstream_tl_nret_atom_as midgard___kbase_tlstream_tl_nret_atom_as +#define __kbase_tlstream_tl_nret_atom_ctx midgard___kbase_tlstream_tl_nret_atom_ctx +#define __kbase_tlstream_tl_nret_atom_lpu midgard___kbase_tlstream_tl_nret_atom_lpu +#define __kbase_tlstream_tl_nret_ctx_lpu midgard___kbase_tlstream_tl_nret_ctx_lpu +#define __kbase_tlstream_tl_ret_as_ctx midgard___kbase_tlstream_tl_ret_as_ctx +#define __kbase_tlstream_tl_ret_atom_as midgard___kbase_tlstream_tl_ret_atom_as +#define __kbase_tlstream_tl_ret_atom_ctx midgard___kbase_tlstream_tl_ret_atom_ctx +#define __kbase_tlstream_tl_ret_atom_lpu midgard___kbase_tlstream_tl_ret_atom_lpu +#define __kbase_tlstream_tl_ret_ctx_lpu midgard___kbase_tlstream_tl_ret_ctx_lpu +#define kbase_unmap_external_resource midgard_kbase_unmap_external_resource +#define kbase_update_region_flags midgard_kbase_update_region_flags +#define kbase_vinstr_hwcnt_reader_setup midgard_kbase_vinstr_hwcnt_reader_setup +#define kbase_vinstr_init midgard_kbase_vinstr_init +#define kbase_vinstr_resume midgard_kbase_vinstr_resume +#define kbase_vinstr_suspend midgard_kbase_vinstr_suspend +#define kbase_vinstr_term midgard_kbase_vinstr_term +#define kbase_vmap midgard_kbase_vmap +#define kbase_vmap_prot midgard_kbase_vmap_prot +#define kbase_vm_ops midgard_kbase_vm_ops +#define kbase_vunmap midgard_kbase_vunmap +#define _mali_profiling_control midgard__mali_profiling_control +#define platform_funcs midgard_platform_funcs +#define pm_callbacks midgard_pm_callbacks +#define rk_kbase_device_runtime_disable midgard_rk_kbase_device_runtime_disable +#define rk_kbase_device_runtime_init midgard_rk_kbase_device_runtime_init +#endif diff --git a/drivers/gpu/arm/midgard/sconscript b/drivers/gpu/arm/midgard/sconscript new file mode 100755 index 000000000000..ff23d7aebe6e --- /dev/null +++ b/drivers/gpu/arm/midgard/sconscript @@ -0,0 +1,92 @@ +# +# (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +import sys +Import('env') + +SConscript( 'tests/sconscript' ) + +mock_test = 0 + +# Fake platform is a transient solution for GPL drivers running in kernel that does not provide configuration via platform data. +# For such kernels fake_platform_device should be set to 1. For kernels providing platform data fake_platform_device should be set to 0. +if env['platform_config']=='devicetree' or env['platform_config']=='juno_soc': + fake_platform_device = 0 +else: + fake_platform_device = 1 + +# Source files required for kbase. +kbase_src = [ + Glob('*.c'), + Glob('backend/*/*.c'), + Glob('internal/*/*.c'), + Glob('ipa/*.c') +] + +if env['platform_config']=='juno_soc': + kbase_src += [Glob('platform/devicetree/*.c')] +else: + kbase_src += [Glob('platform/%s/*.c' % env['platform_config'])] + +if Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock') and env['unit'] == '1': + kbase_src += [Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock/*.c')] + mock_test = 1 + +# we need platform config for GPL version using fake platform +if fake_platform_device==1: + # Check if we are compiling for PBX + if env.KernelConfigEnabled("CONFIG_MACH_REALVIEW_PBX") and \ + env["platform_config"] in {"vexpress", "vexpress_6xvirtex7_10mhz"}: + sys.stderr.write("WARNING: Building for a PBX kernel but with platform_config=vexpress*\n") + # if the file platform config file is in the tpip directory then use that, otherwise use the default config directory + if Glob('#kernel/drivers/gpu/arm/midgard/config/tpip/*%s.c' % (env['platform_config'])): + kbase_src += Glob('#kernel/drivers/gpu/arm/midgard/config/tpip/*%s.c' % (env['platform_config'])) + else: + kbase_src += Glob('#kernel/drivers/gpu/arm/midgard/config/*%s.c' % (env['platform_config'])) + +make_args = env.kernel_get_config_defines(ret_list = True, + fake = fake_platform_device) + [ + 'PLATFORM=%s' % env['platform'], + 'MALI_ERROR_INJECT_ON=%s' % env['error_inject'], + 'MALI_KERNEL_TEST_API=%s' % env['debug'], + 'MALI_UNIT_TEST=%s' % env['unit'], + 'MALI_RELEASE_NAME=%s' % env['mali_release_name'], + 'MALI_MOCK_TEST=%s' % mock_test, + 'MALI_CUSTOMER_RELEASE=%s' % env['release'], + 'MALI_INSTRUMENTATION_LEVEL=%s' % env['instr'], + 'MALI_COVERAGE=%s' % env['coverage'], + 'MALI_BUS_LOG=%s' % env['buslog'] +] + +kbase = env.BuildKernelModule('$STATIC_LIB_PATH/mali_kbase.ko', kbase_src, + make_args = make_args) + +# Add a dependency on kds.ko. +# Only necessary when KDS is not built into the kernel. +# +if env['os'] != 'android': + if not env.KernelConfigEnabled("CONFIG_KDS"): + env.Depends(kbase, '$STATIC_LIB_PATH/kds.ko') + +# need Module.symvers from ump.ko build +if int(env['ump']) == 1: + env.Depends(kbase, '$STATIC_LIB_PATH/ump.ko') + +if 'smc_protected_mode_switcher' in env: + env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/smc_protected_mode_switcher.ko') + +env.KernelObjTarget('kbase', kbase) + +env.AppendUnique(BASE=['cutils_linked_list']) diff --git a/drivers/gpu/arm/midgard/tests/Kbuild b/drivers/gpu/arm/midgard/tests/Kbuild new file mode 100755 index 000000000000..b4bed0473439 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/Kbuild @@ -0,0 +1,17 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +obj-$(CONFIG_MALI_KUTF) += kutf/ +obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test/ diff --git a/drivers/gpu/arm/midgard/tests/Kconfig b/drivers/gpu/arm/midgard/tests/Kconfig new file mode 100755 index 000000000000..da0515c065de --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/Kconfig @@ -0,0 +1,17 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +source "drivers/gpu/arm/midgard/tests/kutf/Kconfig" +source "drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig" diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h new file mode 100755 index 000000000000..0d145e42a0ca --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h @@ -0,0 +1,65 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_MEM_H_ +#define _KERNEL_UTF_MEM_H_ + +/* kutf_mem.h + * Functions for management of memory pools in the kernel. + * + * This module implements a memory pool allocator, allowing a test + * implementation to allocate linked allocations which can then be freed by a + * single free which releases all of the resources held by the entire pool. + * + * Note that it is not possible to free single resources within the pool once + * allocated. + */ + +#include + +/** + * struct kutf_mempool - the memory pool context management structure + * @head: list head on which the allocations in this context are added to + * + */ +struct kutf_mempool { + struct list_head head; +}; + +/** + * kutf_mempool_init() - Initialize a memory pool. + * @pool: Memory pool structure to initialize, provided by the user + * + * Return: zero on success + */ +int kutf_mempool_init(struct kutf_mempool *pool); + +/** + * kutf_mempool_alloc() - Allocate memory from a pool + * @pool: Memory pool to allocate from + * @size: Size of memory wanted in number of bytes + * + * Return: Pointer to memory on success, NULL on failure. + */ +void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size); + +/** + * kutf_mempool_destroy() - Destroy a memory pool, freeing all memory within it. + * @pool: The memory pool to free + */ +void kutf_mempool_destroy(struct kutf_mempool *pool); +#endif /* _KERNEL_UTF_MEM_H_ */ diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h new file mode 100755 index 000000000000..1cc85f1b7a46 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h @@ -0,0 +1,121 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_RESULTSET_H_ +#define _KERNEL_UTF_RESULTSET_H_ + +/* kutf_resultset.h + * Functions and structures for handling test results and result sets. + * + * This section of the kernel UTF contains structures and functions used for the + * management of Results and Result Sets. + */ + +/** + * enum kutf_result_status - Status values for a single Test error. + * @KUTF_RESULT_BENCHMARK: Result is a meta-result containing benchmark + * results. + * @KUTF_RESULT_SKIP: The test was skipped. + * @KUTF_RESULT_UNKNOWN: The test has an unknown result. + * @KUTF_RESULT_PASS: The test result passed. + * @KUTF_RESULT_DEBUG: The test result passed, but raised a debug + * message. + * @KUTF_RESULT_INFO: The test result passed, but raised + * an informative message. + * @KUTF_RESULT_WARN: The test result passed, but raised a warning + * message. + * @KUTF_RESULT_FAIL: The test result failed with a non-fatal error. + * @KUTF_RESULT_FATAL: The test result failed with a fatal error. + * @KUTF_RESULT_ABORT: The test result failed due to a non-UTF + * assertion failure. + * @KUTF_RESULT_COUNT: The current number of possible status messages. + */ +enum kutf_result_status { + KUTF_RESULT_BENCHMARK = -3, + KUTF_RESULT_SKIP = -2, + KUTF_RESULT_UNKNOWN = -1, + + KUTF_RESULT_PASS = 0, + KUTF_RESULT_DEBUG = 1, + KUTF_RESULT_INFO = 2, + KUTF_RESULT_WARN = 3, + KUTF_RESULT_FAIL = 4, + KUTF_RESULT_FATAL = 5, + KUTF_RESULT_ABORT = 6, + + KUTF_RESULT_COUNT +}; + +/* The maximum size of a kutf_result_status result when + * converted to a string + */ +#define KUTF_ERROR_MAX_NAME_SIZE 21 + +#ifdef __KERNEL__ + +#include + +/** + * struct kutf_result - Represents a single test result. + * @node: Next result in the list of results. + * @status: The status summary (pass / warn / fail / etc). + * @message: A more verbose status message. + */ +struct kutf_result { + struct list_head node; + enum kutf_result_status status; + const char *message; +}; + +/** + * kutf_create_result_set() - Create a new result set + * to which results can be added. + * + * Return: The created resultset. + */ +struct kutf_result_set *kutf_create_result_set(void); + +/** + * kutf_add_result() - Add a result to the end of an existing resultset. + * + * @mempool: The memory pool to allocate the result storage from. + * @set: The resultset to add the result to. + * @status: The result status to add. + * @message: The result message to add. + */ +void kutf_add_result(struct kutf_mempool *mempool, struct kutf_result_set *set, + enum kutf_result_status status, const char *message); + +/** + * kutf_remove_result() - Remove a result from the head of a resultset. + * @set: The resultset. + * + * Return: result or NULL if there are no further results in the resultset. + */ +struct kutf_result *kutf_remove_result( + struct kutf_result_set *set); + +/** + * kutf_destroy_result_set() - Free a previously created resultset. + * + * @results: The result set whose resources to free. + */ +void kutf_destroy_result_set(struct kutf_result_set *results); + +#endif /* __KERNEL__ */ + +#endif /* _KERNEL_UTF_RESULTSET_H_ */ diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h new file mode 100755 index 000000000000..754c3adb1cca --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h @@ -0,0 +1,508 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_SUITE_H_ +#define _KERNEL_UTF_SUITE_H_ + +/* kutf_suite.h + * Functions for management of test suites. + * + * This collection of data structures, macros, and functions are used to + * create Test Suites, Tests within those Test Suites, and Fixture variants + * of each test. + */ + +#include +#include + +/** + * Pseudo-flag indicating an absence of any specified test class. Note that + * tests should not be annotated with this constant as it is simply a zero + * value; tests without a more specific class must be marked with the flag + * KUTF_F_TEST_GENERIC. + */ +#define KUTF_F_TEST_NONE ((unsigned int)(0)) + +/** + * Class indicating this test is a smoke test. + * A given set of smoke tests should be quick to run, enabling rapid turn-around + * of "regress-on-commit" test runs. + */ +#define KUTF_F_TEST_SMOKETEST ((unsigned int)(1 << 1)) + +/** + * Class indicating this test is a performance test. + * These tests typically produce a performance metric, such as "time to run" or + * "frames per second", + */ +#define KUTF_F_TEST_PERFORMANCE ((unsigned int)(1 << 2)) + +/** + * Class indicating that this test is a deprecated test. + * These tests have typically been replaced by an alternative test which is + * more efficient, or has better coverage. + */ +#define KUTF_F_TEST_DEPRECATED ((unsigned int)(1 << 3)) + +/** + * Class indicating that this test is a known failure. + * These tests have typically been run and failed, but marking them as a known + * failure means it is easier to triage results. + * + * It is typically more convenient to triage known failures using the + * results database and web UI, as this means there is no need to modify the + * test code. + */ +#define KUTF_F_TEST_EXPECTED_FAILURE ((unsigned int)(1 << 4)) + +/** + * Class indicating that this test is a generic test, which is not a member of + * a more specific test class. Tests which are not created with a specific set + * of filter flags by the user are assigned this test class by default. + */ +#define KUTF_F_TEST_GENERIC ((unsigned int)(1 << 5)) + +/** + * Class indicating this test is a resource allocation failure test. + * A resource allocation failure test will test that an error code is + * correctly propagated when an allocation fails. + */ +#define KUTF_F_TEST_RESFAIL ((unsigned int)(1 << 6)) + +/** + * Additional flag indicating that this test is an expected failure when + * run in resource failure mode. These tests are never run when running + * the low resource mode. + */ +#define KUTF_F_TEST_EXPECTED_FAILURE_RF ((unsigned int)(1 << 7)) + +/** + * Flag reserved for user-defined filter zero. + */ +#define KUTF_F_TEST_USER_0 ((unsigned int)(1 << 24)) + +/** + * Flag reserved for user-defined filter one. + */ +#define KUTF_F_TEST_USER_1 ((unsigned int)(1 << 25)) + +/** + * Flag reserved for user-defined filter two. + */ +#define KUTF_F_TEST_USER_2 ((unsigned int)(1 << 26)) + +/** + * Flag reserved for user-defined filter three. + */ +#define KUTF_F_TEST_USER_3 ((unsigned int)(1 << 27)) + +/** + * Flag reserved for user-defined filter four. + */ +#define KUTF_F_TEST_USER_4 ((unsigned int)(1 << 28)) + +/** + * Flag reserved for user-defined filter five. + */ +#define KUTF_F_TEST_USER_5 ((unsigned int)(1 << 29)) + +/** + * Flag reserved for user-defined filter six. + */ +#define KUTF_F_TEST_USER_6 ((unsigned int)(1 << 30)) + +/** + * Flag reserved for user-defined filter seven. + */ +#define KUTF_F_TEST_USER_7 ((unsigned int)(1 << 31)) + +/** + * Pseudo-flag indicating that all test classes should be executed. + */ +#define KUTF_F_TEST_ALL ((unsigned int)(0xFFFFFFFFU)) + +/** + * union kutf_callback_data - Union used to store test callback data + * @ptr_value: pointer to the location where test callback data + * are stored + * @u32_value: a number which represents test callback data + */ +union kutf_callback_data { + void *ptr_value; + u32 u32_value; +}; + +/** + * struct kutf_context - Structure representing a kernel test context + * @suite: Convenience pointer to the suite this context + * is running + * @test_fix: The fixture that is being run in this context + * @fixture_pool: The memory pool used for the duration of + * the fixture/text context. + * @fixture: The user provided fixture structure. + * @fixture_index: The index (id) of the current fixture. + * @fixture_name: The name of the current fixture (or NULL if unnamed). + * @test_data: Any user private data associated with this test + * @result_set: All the results logged by this test context + * @status: The status of the currently running fixture. + * @expected_status: The expected status on exist of the currently + * running fixture. + */ +struct kutf_context { + struct kutf_suite *suite; + struct kutf_test_fixture *test_fix; + struct kutf_mempool fixture_pool; + void *fixture; + unsigned int fixture_index; + const char *fixture_name; + union kutf_callback_data test_data; + struct kutf_result_set *result_set; + enum kutf_result_status status; + enum kutf_result_status expected_status; +}; + +/** + * struct kutf_suite - Structure representing a kernel test suite + * @app: The application this suite belongs to. + * @name: The name of this suite. + * @suite_data: Any user private data associated with this + * suite. + * @create_fixture: Function used to create a new fixture instance + * @remove_fixture: Function used to destroy a new fixture instance + * @fixture_variants: The number of variants (must be at least 1). + * @suite_default_flags: Suite global filter flags which are set on + * all tests. + * @node: List node for suite_list + * @dir: The debugfs directory for this suite + * @test_list: List head to store all the tests which are + * part of this suite + */ +struct kutf_suite { + struct kutf_application *app; + const char *name; + union kutf_callback_data suite_data; + void *(*create_fixture)(struct kutf_context *context); + void (*remove_fixture)(struct kutf_context *context); + unsigned int fixture_variants; + unsigned int suite_default_flags; + struct list_head node; + struct dentry *dir; + struct list_head test_list; +}; + +/* ============================================================================ + Application functions +============================================================================ */ + +/** + * kutf_create_application() - Create an in kernel test application. + * @name: The name of the test application. + * + * Return: pointer to the kutf_application on success or NULL + * on failure + */ +struct kutf_application *kutf_create_application(const char *name); + +/** + * kutf_destroy_application() - Destroy an in kernel test application. + * + * @app: The test application to destroy. + */ +void kutf_destroy_application(struct kutf_application *app); + +/* ============================================================================ + Suite functions +============================================================================ */ + +/** + * kutf_create_suite() - Create a kernel test suite. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * + * Suite names must be unique. Should two suites with the same name be + * registered with the same application then this function will fail, if they + * are registered with different applications then the function will not detect + * this and the call will succeed. + * + * Return: pointer to the created kutf_suite on success or NULL + * on failure + */ +struct kutf_suite *kutf_create_suite( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context)); + +/** + * kutf_create_suite_with_filters() - Create a kernel test suite with user + * defined default filters. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * @filters: Filters to apply to a test if it doesn't provide its own + * + * Suite names must be unique. Should two suites with the same name be + * registered with the same application then this function will fail, if they + * are registered with different applications then the function will not detect + * this and the call will succeed. + * + * Return: pointer to the created kutf_suite on success or NULL on failure + */ +struct kutf_suite *kutf_create_suite_with_filters( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters); + +/** + * kutf_create_suite_with_filters_and_data() - Create a kernel test suite with + * user defined default filters. + * @app: The test application to create the suite in. + * @name: The name of the suite. + * @fixture_count: The number of fixtures to run over the test + * functions in this suite + * @create_fixture: Callback used to create a fixture. The returned value + * is stored in the fixture pointer in the context for + * use in the test functions. + * @remove_fixture: Callback used to remove a previously created fixture. + * @filters: Filters to apply to a test if it doesn't provide its own + * @suite_data: Suite specific callback data, provided during the + * running of the test in the kutf_context + * + * Return: pointer to the created kutf_suite on success or NULL + * on failure + */ +struct kutf_suite *kutf_create_suite_with_filters_and_data( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data suite_data); + +/** + * kutf_add_test() - Add a test to a kernel test suite. + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * + * Note: As no filters are provided the test will use the suite filters instead + */ +void kutf_add_test(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context)); + +/** + * kutf_add_test_with_filters() - Add a test to a kernel test suite with filters + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * @filters: A set of filtering flags, assigning test categories. + */ +void kutf_add_test_with_filters(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters); + +/** + * kutf_add_test_with_filters_and_data() - Add a test to a kernel test suite + * with filters. + * @suite: The suite to add the test to. + * @id: The ID of the test. + * @name: The name of the test. + * @execute: Callback to the test function to run. + * @filters: A set of filtering flags, assigning test categories. + * @test_data: Test specific callback data, provoided during the + * running of the test in the kutf_context + */ +void kutf_add_test_with_filters_and_data( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data test_data); + +/* ============================================================================ + Test functions +============================================================================ */ +/** + * kutf_test_log_result_external() - Log a result which has been created + * externally into a in a standard form + * recognized by the log parser. + * @context: The test context the test is running in + * @message: The message for this result + * @new_status: The result status of this log message + */ +void kutf_test_log_result_external( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status); + +/** + * kutf_test_expect_abort() - Tell the kernel that you expect the current + * fixture to produce an abort. + * @context: The test context this test is running in. + */ +void kutf_test_expect_abort(struct kutf_context *context); + +/** + * kutf_test_expect_fatal() - Tell the kernel that you expect the current + * fixture to produce a fatal error. + * @context: The test context this test is running in. + */ +void kutf_test_expect_fatal(struct kutf_context *context); + +/** + * kutf_test_expect_fail() - Tell the kernel that you expect the current + * fixture to fail. + * @context: The test context this test is running in. + */ +void kutf_test_expect_fail(struct kutf_context *context); + +/** + * kutf_test_expect_warn() - Tell the kernel that you expect the current + * fixture to produce a warning. + * @context: The test context this test is running in. + */ +void kutf_test_expect_warn(struct kutf_context *context); + +/** + * kutf_test_expect_pass() - Tell the kernel that you expect the current + * fixture to pass. + * @context: The test context this test is running in. + */ +void kutf_test_expect_pass(struct kutf_context *context); + +/** + * kutf_test_skip() - Tell the kernel that the test should be skipped. + * @context: The test context this test is running in. + */ +void kutf_test_skip(struct kutf_context *context); + +/** + * kutf_test_skip_msg() - Tell the kernel that this test has been skipped, + * supplying a reason string. + * @context: The test context this test is running in. + * @message: A message string containing the reason for the skip. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a prebaked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_skip_msg(struct kutf_context *context, const char *message); + +/** + * kutf_test_pass() - Tell the kernel that this test has passed. + * @context: The test context this test is running in. + * @message: A message string containing the reason for the pass. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_pass(struct kutf_context *context, char const *message); + +/** + * kutf_test_debug() - Send a debug message + * @context: The test context this test is running in. + * @message: A message string containing the debug information. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_debug(struct kutf_context *context, char const *message); + +/** + * kutf_test_info() - Send an information message + * @context: The test context this test is running in. + * @message: A message string containing the information message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_info(struct kutf_context *context, char const *message); + +/** + * kutf_test_warn() - Send a warning message + * @context: The test context this test is running in. + * @message: A message string containing the warning message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_warn(struct kutf_context *context, char const *message); + +/** + * kutf_test_fail() - Tell the kernel that a test has failed + * @context: The test context this test is running in. + * @message: A message string containing the failure message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_fail(struct kutf_context *context, char const *message); + +/** + * kutf_test_fatal() - Tell the kernel that a test has triggered a fatal error + * @context: The test context this test is running in. + * @message: A message string containing the fatal error message. + * + * Note: The message must not be freed during the lifetime of the test run. + * This means it should either be a pre-baked string, or if a dynamic string + * is required it must be created with kutf_dsprintf which will store + * the resultant string in a buffer who's lifetime is the same as the test run. + */ +void kutf_test_fatal(struct kutf_context *context, char const *message); + +/** + * kutf_test_abort() - Tell the kernel that a test triggered an abort in the test + * + * @context: The test context this test is running in. + */ +void kutf_test_abort(struct kutf_context *context); + +#endif /* _KERNEL_UTF_SUITE_H_ */ diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h new file mode 100755 index 000000000000..c458c1f73802 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h @@ -0,0 +1,55 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#ifndef _KERNEL_UTF_UTILS_H_ +#define _KERNEL_UTF_UTILS_H_ + +/* kutf_utils.h + * Utilities for the kernel UTF test infrastructure. + * + * This collection of library functions are provided for use by kernel UTF + * and users of kernel UTF which don't directly fit within the other + * code modules. + */ + +#include + +/** + * Maximum size of the message strings within kernel UTF, messages longer then + * this will be truncated. + */ +#define KUTF_MAX_DSPRINTF_LEN 1024 + +/** + * kutf_dsprintf() - dynamic sprintf + * @pool: memory pool to allocate from + * @fmt: The format string describing the string to document. + * @... The parameters to feed in to the format string. + * + * This function implements sprintf which dynamically allocates memory to store + * the string. The library will free the memory containing the string when the + * result set is cleared or destroyed. + * + * Note The returned string may be truncated to fit an internal temporary + * buffer, which is KUTF_MAX_DSPRINTF_LEN bytes in length. + * + * Return: Returns pointer to allocated string, or NULL on error. + */ +const char *kutf_dsprintf(struct kutf_mempool *pool, + const char *fmt, ...); + +#endif /* _KERNEL_UTF_UTILS_H_ */ diff --git a/drivers/gpu/arm/midgard/tests/kutf/Kbuild b/drivers/gpu/arm/midgard/tests/kutf/Kbuild new file mode 100755 index 000000000000..6b840c2ef7b7 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/kutf/Kbuild @@ -0,0 +1,20 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +ccflags-y += -I$(src)/../include + +obj-$(CONFIG_MALI_KUTF) += kutf.o + +kutf-y := kutf_mem.o kutf_resultset.o kutf_suite.o kutf_utils.o diff --git a/drivers/gpu/arm/midgard/tests/kutf/Kconfig b/drivers/gpu/arm/midgard/tests/kutf/Kconfig new file mode 100755 index 000000000000..84364716afe3 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/kutf/Kconfig @@ -0,0 +1,22 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + + +config MALI_KUTF + tristate "Mali Kernel Unit Test Framework" + default n + help + Enables MALI testing framework. To compile it as a module, + choose M here - this will generate a single module called kutf. diff --git a/drivers/gpu/arm/midgard/tests/kutf/Makefile b/drivers/gpu/arm/midgard/tests/kutf/Makefile new file mode 100755 index 000000000000..010c92ca39b9 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/kutf/Makefile @@ -0,0 +1,29 @@ +# +# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +# linux build system bootstrap for out-of-tree module + +# default to building for the host +ARCH ?= $(shell uname -m) + +ifeq ($(KDIR),) +$(error Must specify KDIR to point to the kernel to target)) +endif + +all: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS=-I$(CURDIR)/../include modules + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c new file mode 100755 index 000000000000..5408e57d469a --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c @@ -0,0 +1,94 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF memory management functions */ + +#include +#include +#include + +#include + + +/** + * struct kutf_alloc_entry - Structure representing an allocation. + * @node: List node for use with kutf_mempool. + * @data: Data area of the allocation + */ +struct kutf_alloc_entry { + struct list_head node; + u8 data[0]; +}; + +int kutf_mempool_init(struct kutf_mempool *pool) +{ + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + return -1; + } + + INIT_LIST_HEAD(&pool->head); + + return 0; +} +EXPORT_SYMBOL(kutf_mempool_init); + +void kutf_mempool_destroy(struct kutf_mempool *pool) +{ + struct list_head *remove; + struct list_head *tmp; + + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + return; + } + + list_for_each_safe(remove, tmp, &pool->head) { + struct kutf_alloc_entry *remove_alloc; + + remove_alloc = list_entry(remove, struct kutf_alloc_entry, node); + list_del(&remove_alloc->node); + kfree(remove_alloc); + } +} +EXPORT_SYMBOL(kutf_mempool_destroy); + +void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size) +{ + struct kutf_alloc_entry *ret; + + if (!pool) { + pr_err("NULL pointer passed to %s\n", __func__); + goto fail_pool; + } + + ret = kmalloc(sizeof(*ret) + size, GFP_KERNEL); + if (!ret) { + pr_err("Failed to allocate memory\n"); + goto fail_alloc; + } + + INIT_LIST_HEAD(&ret->node); + list_add(&ret->node, &pool->head); + + return &ret->data[0]; + +fail_alloc: +fail_pool: + return NULL; +} +EXPORT_SYMBOL(kutf_mempool_alloc); diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c new file mode 100755 index 000000000000..5bd04969fd55 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c @@ -0,0 +1,95 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF result management functions */ + +#include +#include +#include + +#include + +/** + * struct kutf_result_set - Represents a set of results. + * @results: Pointer to the linked list where the results are stored. + */ +struct kutf_result_set { + struct list_head results; +}; + +struct kutf_result_set *kutf_create_result_set(void) +{ + struct kutf_result_set *set; + + set = kmalloc(sizeof(*set), GFP_KERNEL); + if (!set) { + pr_err("Failed to allocate resultset"); + goto fail_alloc; + } + + INIT_LIST_HEAD(&set->results); + + return set; + +fail_alloc: + return NULL; +} + +void kutf_add_result(struct kutf_mempool *mempool, + struct kutf_result_set *set, + enum kutf_result_status status, + const char *message) +{ + /* Create the new result */ + struct kutf_result *new_result; + + BUG_ON(set == NULL); + + new_result = kutf_mempool_alloc(mempool, sizeof(*new_result)); + if (!new_result) { + pr_err("Result allocation failed\n"); + return; + } + + INIT_LIST_HEAD(&new_result->node); + new_result->status = status; + new_result->message = message; + + list_add_tail(&new_result->node, &set->results); +} + +void kutf_destroy_result_set(struct kutf_result_set *set) +{ + if (!list_empty(&set->results)) + pr_err("kutf_destroy_result_set: Unread results from test\n"); + + kfree(set); +} + +struct kutf_result *kutf_remove_result(struct kutf_result_set *set) +{ + if (!list_empty(&set->results)) { + struct kutf_result *ret; + + ret = list_first_entry(&set->results, struct kutf_result, node); + list_del(&ret->node); + return ret; + } + + return NULL; +} + diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c new file mode 100755 index 000000000000..a7cfd3be9c46 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c @@ -0,0 +1,1041 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF suite, test and fixture management including user to kernel + * interaction */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#if defined(CONFIG_DEBUG_FS) + +/** + * struct kutf_application - Structure which represents kutf application + * @name: The name of this test application. + * @dir: The debugfs directory for this test + * @suite_list: List head to store all the suites which are part of this + * application + */ +struct kutf_application { + const char *name; + struct dentry *dir; + struct list_head suite_list; +}; + +/** + * struct kutf_test_function - Structure which represents kutf test function + * @suite: Back reference to the suite this test function + * belongs to + * @filters: Filters that apply to this test function + * @test_id: Test ID + * @execute: Function to run for this test + * @test_data: Static data for this test + * @node: List node for test_list + * @variant_list: List head to store all the variants which can run on + * this function + * @dir: debugfs directory for this test function + */ +struct kutf_test_function { + struct kutf_suite *suite; + unsigned int filters; + unsigned int test_id; + void (*execute)(struct kutf_context *context); + union kutf_callback_data test_data; + struct list_head node; + struct list_head variant_list; + struct dentry *dir; +}; + +/** + * struct kutf_test_fixture - Structure which holds information on the kutf + * test fixture + * @test_func: Test function this fixture belongs to + * @fixture_index: Index of this fixture + * @node: List node for variant_list + * @dir: debugfs directory for this test fixture + */ +struct kutf_test_fixture { + struct kutf_test_function *test_func; + unsigned int fixture_index; + struct list_head node; + struct dentry *dir; +}; + +struct dentry *base_dir; + +/** + * struct kutf_convert_table - Structure which keeps test results + * @result_name: Status of the test result + * @result: Status value for a single test + */ +struct kutf_convert_table { + char result_name[50]; + enum kutf_result_status result; +}; + +struct kutf_convert_table kutf_convert[] = { +#define ADD_UTF_RESULT(_name) \ +{ \ + #_name, \ + _name, \ +}, +ADD_UTF_RESULT(KUTF_RESULT_BENCHMARK) +ADD_UTF_RESULT(KUTF_RESULT_SKIP) +ADD_UTF_RESULT(KUTF_RESULT_UNKNOWN) +ADD_UTF_RESULT(KUTF_RESULT_PASS) +ADD_UTF_RESULT(KUTF_RESULT_DEBUG) +ADD_UTF_RESULT(KUTF_RESULT_INFO) +ADD_UTF_RESULT(KUTF_RESULT_WARN) +ADD_UTF_RESULT(KUTF_RESULT_FAIL) +ADD_UTF_RESULT(KUTF_RESULT_FATAL) +ADD_UTF_RESULT(KUTF_RESULT_ABORT) +}; + +#define UTF_CONVERT_SIZE (ARRAY_SIZE(kutf_convert)) + +/** + * kutf_create_context() - Create a test context in which a specific fixture + * of an application will be run and its results + * reported back to the user + * @test_fix: Test fixture to be run. + * + * Return: Returns the created test context on success or NULL on failure + */ +static struct kutf_context *kutf_create_context( + struct kutf_test_fixture *test_fix); + +/** + * kutf_destroy_context() - Destroy a previously created test context + * @context: Test context to destroy + */ +static void kutf_destroy_context(struct kutf_context *context); + +/** + * kutf_set_result() - Set the test result against the specified test context + * @context: Test context + * @status: Result status + */ +static void kutf_set_result(struct kutf_context *context, + enum kutf_result_status status); + +/** + * kutf_set_expected_result() - Set the expected test result for the specified + * test context + * @context: Test context + * @expected_status: Expected result status + */ +static void kutf_set_expected_result(struct kutf_context *context, + enum kutf_result_status expected_status); + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)) +/* Pre 3.4.0 kernels don't have the simple_open helper */ + +/** + * simple_open() - Helper for file opening which stores the inode private data + * into the file private data + * @inode: File entry representation + * @file: A specific opening of the file + * + * Return: always 0; if inode private data do not exist, the file will not + * be assigned private data + */ +static int simple_open(struct inode *inode, struct file *file) +{ + if (inode->i_private) + file->private_data = inode->i_private; + return 0; +} +#endif + +/** + * kutf_result_to_string() - Converts a KUTF result into a string + * @result_str: Output result string + * @result: Result status to convert + * + * Return: 1 if test result was successfully converted to string, 0 otherwise + */ +static int kutf_result_to_string(char **result_str, + enum kutf_result_status result) +{ + int i; + int ret = 0; + + for (i = 0; i < UTF_CONVERT_SIZE; i++) { + if (result == kutf_convert[i].result) { + *result_str = kutf_convert[i].result_name; + ret = 1; + } + } + return ret; +} + +/** + * kutf_debugfs_const_string_read() - Simple debugfs read callback which + * returns a constant string + * @file: Opened file to read from + * @buf: User buffer to write the data into + * @len: Amount of data to read + * @ppos: Offset into file to read from + * + * Return: On success, the number of bytes read and offset @ppos advanced by + * this number; on error, negative value + */ +static ssize_t kutf_debugfs_const_string_read(struct file *file, + char __user *buf, size_t len, loff_t *ppos) +{ + char *str = file->private_data; + + return simple_read_from_buffer(buf, len, ppos, str, strlen(str)); +} + +static const struct file_operations kutf_debugfs_const_string_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = kutf_debugfs_const_string_read, + .llseek = default_llseek, +}; + +/** + * kutf_add_explicit_result() - Check if an explicit result needs to be added + * @context: KUTF test context + */ +static void kutf_add_explicit_result(struct kutf_context *context) +{ + switch (context->expected_status) { + case KUTF_RESULT_UNKNOWN: + if (context->status == KUTF_RESULT_UNKNOWN) + kutf_test_pass(context, "(implicit pass)"); + break; + + case KUTF_RESULT_WARN: + if (context->status == KUTF_RESULT_WARN) + kutf_test_pass(context, + "Pass (expected warn occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected warn missing)"); + break; + + case KUTF_RESULT_FAIL: + if (context->status == KUTF_RESULT_FAIL) + kutf_test_pass(context, + "Pass (expected fail occurred)"); + else if (context->status != KUTF_RESULT_SKIP) { + /* Force the expected status so the fail gets logged */ + context->expected_status = KUTF_RESULT_PASS; + kutf_test_fail(context, + "Fail (expected fail missing)"); + } + break; + + case KUTF_RESULT_FATAL: + if (context->status == KUTF_RESULT_FATAL) + kutf_test_pass(context, + "Pass (expected fatal occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected fatal missing)"); + break; + + case KUTF_RESULT_ABORT: + if (context->status == KUTF_RESULT_ABORT) + kutf_test_pass(context, + "Pass (expected abort occurred)"); + else if (context->status != KUTF_RESULT_SKIP) + kutf_test_fail(context, + "Fail (expected abort missing)"); + break; + default: + break; + } +} + +/** + * kutf_debugfs_run_open() Debugfs open callback for the "run" entry. + * @inode: inode of the opened file + * @file: Opened file to read from + * + * This function retrieves the test fixture data that is associated with the + * opened file and works back to get the test, suite and application so + * it can then run the test that is associated with the file entry. + * + * Return: 0 on success + */ +static int kutf_debugfs_run_open(struct inode *inode, struct file *file) +{ + struct kutf_test_fixture *test_fix = inode->i_private; + struct kutf_test_function *test_func = test_fix->test_func; + struct kutf_suite *suite = test_func->suite; + struct kutf_context *test_context; + + test_context = kutf_create_context(test_fix); + if (!test_context) + return -ENODEV; + + file->private_data = test_context; + + /* + * Call the create fixture function if required before the + * fixture is run + */ + if (suite->create_fixture) + test_context->fixture = suite->create_fixture(test_context); + + /* Only run the test if the fixture was created (if required) */ + if ((suite->create_fixture && test_context->fixture) || + (!suite->create_fixture)) { + /* Run this fixture */ + test_func->execute(test_context); + + if (suite->remove_fixture) + suite->remove_fixture(test_context); + + kutf_add_explicit_result(test_context); + } + return 0; +} + +/** + * kutf_debugfs_run_read() - Debugfs read callback for the "run" entry. + * @file: Opened file to read from + * @buf: User buffer to write the data into + * @len: Amount of data to read + * @ppos: Offset into file to read from + * + * This function emits the results which where logged during the opening of + * the file kutf_debugfs_run_open. + * Results will be emitted one at a time, once all the results have been read + * 0 will be returned to indicate there is no more data. + * + * Return: Number of bytes read. + */ +static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, + size_t len, loff_t *ppos) +{ + struct kutf_context *test_context = file->private_data; + struct kutf_result *res; + unsigned long bytes_not_copied; + ssize_t bytes_copied = 0; + + /* Note: This code assumes a result is read completely */ + res = kutf_remove_result(test_context->result_set); + if (res) { + char *kutf_str_ptr = NULL; + unsigned int kutf_str_len = 0; + unsigned int message_len = 0; + char separator = ':'; + char terminator = '\n'; + + kutf_result_to_string(&kutf_str_ptr, res->status); + if (kutf_str_ptr) + kutf_str_len = strlen(kutf_str_ptr); + + if (res->message) + message_len = strlen(res->message); + + if ((kutf_str_len + 1 + message_len + 1) > len) { + pr_err("Not enough space in user buffer for a single result"); + return 0; + } + + /* First copy the result string */ + if (kutf_str_ptr) { + bytes_not_copied = copy_to_user(&buf[0], kutf_str_ptr, + kutf_str_len); + bytes_copied += kutf_str_len - bytes_not_copied; + if (bytes_not_copied) + goto exit; + } + + /* Then the separator */ + bytes_not_copied = copy_to_user(&buf[bytes_copied], + &separator, 1); + bytes_copied += 1 - bytes_not_copied; + if (bytes_not_copied) + goto exit; + + /* Finally Next copy the result string */ + if (res->message) { + bytes_not_copied = copy_to_user(&buf[bytes_copied], + res->message, message_len); + bytes_copied += message_len - bytes_not_copied; + if (bytes_not_copied) + goto exit; + } + + /* Finally the terminator */ + bytes_not_copied = copy_to_user(&buf[bytes_copied], + &terminator, 1); + bytes_copied += 1 - bytes_not_copied; + } +exit: + return bytes_copied; +} + +/** + * kutf_debugfs_run_release() - Debugfs release callback for the "run" entry. + * @inode: File entry representation + * @file: A specific opening of the file + * + * Release any resources that where created during the opening of the file + * + * Return: 0 on success + */ +static int kutf_debugfs_run_release(struct inode *inode, struct file *file) +{ + struct kutf_context *test_context = file->private_data; + + kutf_destroy_context(test_context); + return 0; +} + +static const struct file_operations kutf_debugfs_run_ops = { + .owner = THIS_MODULE, + .open = kutf_debugfs_run_open, + .read = kutf_debugfs_run_read, + .release = kutf_debugfs_run_release, + .llseek = default_llseek, +}; + +/** + * create_fixture_variant() - Creates a fixture variant for the specified + * test function and index and the debugfs entries + * that represent it. + * @test_func: Test function + * @fixture_index: Fixture index + * + * Return: 0 on success, negative value corresponding to error code in failure + */ +static int create_fixture_variant(struct kutf_test_function *test_func, + unsigned int fixture_index) +{ + struct kutf_test_fixture *test_fix; + char name[11]; /* Enough to print the MAX_UINT32 + the null terminator */ + struct dentry *tmp; + int err; + + test_fix = kmalloc(sizeof(*test_fix), GFP_KERNEL); + if (!test_fix) { + pr_err("Failed to create debugfs directory when adding fixture\n"); + err = -ENOMEM; + goto fail_alloc; + } + + test_fix->test_func = test_func; + test_fix->fixture_index = fixture_index; + + snprintf(name, sizeof(name), "%d", fixture_index); + test_fix->dir = debugfs_create_dir(name, test_func->dir); + if (!test_func->dir) { + pr_err("Failed to create debugfs directory when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_dir; + } + + tmp = debugfs_create_file("type", S_IROTH, test_fix->dir, "fixture\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_file; + } + + tmp = debugfs_create_file("run", S_IROTH, test_fix->dir, test_fix, + &kutf_debugfs_run_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"run\" when adding fixture\n"); + /* Might not be the right error, we don't get it passed back to us */ + err = -EEXIST; + goto fail_file; + } + + list_add(&test_fix->node, &test_func->variant_list); + return 0; + +fail_file: + debugfs_remove_recursive(test_fix->dir); +fail_dir: + kfree(test_fix); +fail_alloc: + return err; +} + +/** + * kutf_remove_test_variant() - Destroy a previously created fixture variant. + * @test_fix: Test fixture + */ +static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix) +{ + debugfs_remove_recursive(test_fix->dir); + kfree(test_fix); +} + +void kutf_add_test_with_filters_and_data( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data test_data) +{ + struct kutf_test_function *test_func; + struct dentry *tmp; + unsigned int i; + + test_func = kmalloc(sizeof(*test_func), GFP_KERNEL); + if (!test_func) { + pr_err("Failed to allocate memory when adding test %s\n", name); + goto fail_alloc; + } + + INIT_LIST_HEAD(&test_func->variant_list); + + test_func->dir = debugfs_create_dir(name, suite->dir); + if (!test_func->dir) { + pr_err("Failed to create debugfs directory when adding test %s\n", name); + goto fail_dir; + } + + tmp = debugfs_create_file("type", S_IROTH, test_func->dir, "test\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); + goto fail_file; + } + + test_func->filters = filters; + tmp = debugfs_create_x32("filters", S_IROTH, test_func->dir, + &test_func->filters); + if (!tmp) { + pr_err("Failed to create debugfs file \"filters\" when adding test %s\n", name); + goto fail_file; + } + + test_func->test_id = id; + tmp = debugfs_create_u32("test_id", S_IROTH, test_func->dir, + &test_func->test_id); + if (!tmp) { + pr_err("Failed to create debugfs file \"test_id\" when adding test %s\n", name); + goto fail_file; + } + + for (i = 0; i < suite->fixture_variants; i++) { + if (create_fixture_variant(test_func, i)) { + pr_err("Failed to create fixture %d when adding test %s\n", i, name); + goto fail_file; + } + } + + test_func->suite = suite; + test_func->execute = execute; + test_func->test_data = test_data; + + list_add(&test_func->node, &suite->test_list); + return; + +fail_file: + debugfs_remove_recursive(test_func->dir); +fail_dir: + kfree(test_func); +fail_alloc: + return; +} +EXPORT_SYMBOL(kutf_add_test_with_filters_and_data); + +void kutf_add_test_with_filters( + struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context), + unsigned int filters) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + + kutf_add_test_with_filters_and_data(suite, + id, + name, + execute, + suite->suite_default_flags, + data); +} +EXPORT_SYMBOL(kutf_add_test_with_filters); + +void kutf_add_test(struct kutf_suite *suite, + unsigned int id, + const char *name, + void (*execute)(struct kutf_context *context)) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + + kutf_add_test_with_filters_and_data(suite, + id, + name, + execute, + suite->suite_default_flags, + data); +} +EXPORT_SYMBOL(kutf_add_test); + +/** + * kutf_remove_test(): Remove a previously added test function. + * @test_func: Test function + */ +static void kutf_remove_test(struct kutf_test_function *test_func) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &test_func->variant_list) { + struct kutf_test_fixture *test_fix; + + test_fix = list_entry(pos, struct kutf_test_fixture, node); + kutf_remove_test_variant(test_fix); + } + + list_del(&test_func->node); + debugfs_remove_recursive(test_func->dir); + kfree(test_func); +} + +struct kutf_suite *kutf_create_suite_with_filters_and_data( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters, + union kutf_callback_data suite_data) +{ + struct kutf_suite *suite; + struct dentry *tmp; + + suite = kmalloc(sizeof(*suite), GFP_KERNEL); + if (!suite) { + pr_err("Failed to allocate memory when creating suite %s\n", name); + goto fail_kmalloc; + } + + suite->dir = debugfs_create_dir(name, app->dir); + if (!suite->dir) { + pr_err("Failed to create debugfs directory when adding test %s\n", name); + goto fail_debugfs; + } + + tmp = debugfs_create_file("type", S_IROTH, suite->dir, "suite\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); + goto fail_file; + } + + INIT_LIST_HEAD(&suite->test_list); + suite->app = app; + suite->name = name; + suite->fixture_variants = fixture_count; + suite->create_fixture = create_fixture; + suite->remove_fixture = remove_fixture; + suite->suite_default_flags = filters; + suite->suite_data = suite_data; + + list_add(&suite->node, &app->suite_list); + + return suite; + +fail_file: + debugfs_remove_recursive(suite->dir); +fail_debugfs: + kfree(suite); +fail_kmalloc: + return NULL; +} +EXPORT_SYMBOL(kutf_create_suite_with_filters_and_data); + +struct kutf_suite *kutf_create_suite_with_filters( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context), + unsigned int filters) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + return kutf_create_suite_with_filters_and_data(app, + name, + fixture_count, + create_fixture, + remove_fixture, + filters, + data); +} +EXPORT_SYMBOL(kutf_create_suite_with_filters); + +struct kutf_suite *kutf_create_suite( + struct kutf_application *app, + const char *name, + unsigned int fixture_count, + void *(*create_fixture)(struct kutf_context *context), + void (*remove_fixture)(struct kutf_context *context)) +{ + union kutf_callback_data data; + + data.ptr_value = NULL; + return kutf_create_suite_with_filters_and_data(app, + name, + fixture_count, + create_fixture, + remove_fixture, + KUTF_F_TEST_GENERIC, + data); +} +EXPORT_SYMBOL(kutf_create_suite); + +/** + * kutf_destroy_suite() - Destroy a previously added test suite. + * @suite: Test suite + */ +static void kutf_destroy_suite(struct kutf_suite *suite) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &suite->test_list) { + struct kutf_test_function *test_func; + + test_func = list_entry(pos, struct kutf_test_function, node); + kutf_remove_test(test_func); + } + + list_del(&suite->node); + debugfs_remove_recursive(suite->dir); + kfree(suite); +} + +struct kutf_application *kutf_create_application(const char *name) +{ + struct kutf_application *app; + struct dentry *tmp; + + app = kmalloc(sizeof(*app), GFP_KERNEL); + if (!app) { + pr_err("Failed to create allocate memory when creating application %s\n", name); + goto fail_kmalloc; + } + + app->dir = debugfs_create_dir(name, base_dir); + if (!app->dir) { + pr_err("Failed to create debugfs direcotry when creating application %s\n", name); + goto fail_debugfs; + } + + tmp = debugfs_create_file("type", S_IROTH, app->dir, "application\n", + &kutf_debugfs_const_string_ops); + if (!tmp) { + pr_err("Failed to create debugfs file \"type\" when creating application %s\n", name); + goto fail_file; + } + + INIT_LIST_HEAD(&app->suite_list); + app->name = name; + + return app; + +fail_file: + debugfs_remove_recursive(app->dir); +fail_debugfs: + kfree(app); +fail_kmalloc: + return NULL; +} +EXPORT_SYMBOL(kutf_create_application); + +void kutf_destroy_application(struct kutf_application *app) +{ + struct list_head *pos; + struct list_head *tmp; + + list_for_each_safe(pos, tmp, &app->suite_list) { + struct kutf_suite *suite; + + suite = list_entry(pos, struct kutf_suite, node); + kutf_destroy_suite(suite); + } + + debugfs_remove_recursive(app->dir); + kfree(app); +} +EXPORT_SYMBOL(kutf_destroy_application); + +static struct kutf_context *kutf_create_context( + struct kutf_test_fixture *test_fix) +{ + struct kutf_context *new_context; + + new_context = kmalloc(sizeof(*new_context), GFP_KERNEL); + if (!new_context) { + pr_err("Failed to allocate test context"); + goto fail_alloc; + } + + new_context->result_set = kutf_create_result_set(); + if (!new_context->result_set) { + pr_err("Failed to create resultset"); + goto fail_result_set; + } + + new_context->test_fix = test_fix; + /* Save the pointer to the suite as the callbacks will require it */ + new_context->suite = test_fix->test_func->suite; + new_context->status = KUTF_RESULT_UNKNOWN; + new_context->expected_status = KUTF_RESULT_UNKNOWN; + + kutf_mempool_init(&new_context->fixture_pool); + new_context->fixture = NULL; + new_context->fixture_index = test_fix->fixture_index; + new_context->fixture_name = NULL; + new_context->test_data = test_fix->test_func->test_data; + + return new_context; + +fail_result_set: + kfree(new_context); +fail_alloc: + return NULL; +} + +static void kutf_destroy_context(struct kutf_context *context) +{ + kutf_destroy_result_set(context->result_set); + kutf_mempool_destroy(&context->fixture_pool); + kfree(context); +} + +static void kutf_set_result(struct kutf_context *context, + enum kutf_result_status status) +{ + context->status = status; +} + +static void kutf_set_expected_result(struct kutf_context *context, + enum kutf_result_status expected_status) +{ + context->expected_status = expected_status; +} + +/** + * kutf_test_log_result() - Log a result for the specified test context + * @context: Test context + * @message: Result string + * @new_status: Result status + */ +static void kutf_test_log_result( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status) +{ + if (context->status < new_status) + context->status = new_status; + + if (context->expected_status != new_status) + kutf_add_result(&context->fixture_pool, context->result_set, + new_status, message); +} + +void kutf_test_log_result_external( + struct kutf_context *context, + const char *message, + enum kutf_result_status new_status) +{ + kutf_test_log_result(context, message, new_status); +} +EXPORT_SYMBOL(kutf_test_log_result_external); + +void kutf_test_expect_abort(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_ABORT); +} +EXPORT_SYMBOL(kutf_test_expect_abort); + +void kutf_test_expect_fatal(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_FATAL); +} +EXPORT_SYMBOL(kutf_test_expect_fatal); + +void kutf_test_expect_fail(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_FAIL); +} +EXPORT_SYMBOL(kutf_test_expect_fail); + +void kutf_test_expect_warn(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_WARN); +} +EXPORT_SYMBOL(kutf_test_expect_warn); + +void kutf_test_expect_pass(struct kutf_context *context) +{ + kutf_set_expected_result(context, KUTF_RESULT_PASS); +} +EXPORT_SYMBOL(kutf_test_expect_pass); + +void kutf_test_skip(struct kutf_context *context) +{ + kutf_set_result(context, KUTF_RESULT_SKIP); + kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); + + kutf_test_log_result(context, "Test skipped", KUTF_RESULT_SKIP); +} +EXPORT_SYMBOL(kutf_test_skip); + +void kutf_test_skip_msg(struct kutf_context *context, const char *message) +{ + kutf_set_result(context, KUTF_RESULT_SKIP); + kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); + + kutf_test_log_result(context, kutf_dsprintf(&context->fixture_pool, + "Test skipped: %s", message), KUTF_RESULT_SKIP); + kutf_test_log_result(context, "!!!Test skipped!!!", KUTF_RESULT_SKIP); +} +EXPORT_SYMBOL(kutf_test_skip_msg); + +void kutf_test_debug(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_DEBUG); +} +EXPORT_SYMBOL(kutf_test_debug); + +void kutf_test_pass(struct kutf_context *context, char const *message) +{ + static const char explicit_message[] = "(explicit pass)"; + + if (!message) + message = explicit_message; + + kutf_test_log_result(context, message, KUTF_RESULT_PASS); +} +EXPORT_SYMBOL(kutf_test_pass); + +void kutf_test_info(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_INFO); +} +EXPORT_SYMBOL(kutf_test_info); + +void kutf_test_warn(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_WARN); +} +EXPORT_SYMBOL(kutf_test_warn); + +void kutf_test_fail(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_FAIL); +} +EXPORT_SYMBOL(kutf_test_fail); + +void kutf_test_fatal(struct kutf_context *context, char const *message) +{ + kutf_test_log_result(context, message, KUTF_RESULT_FATAL); +} +EXPORT_SYMBOL(kutf_test_fatal); + +void kutf_test_abort(struct kutf_context *context) +{ + kutf_test_log_result(context, "", KUTF_RESULT_ABORT); +} +EXPORT_SYMBOL(kutf_test_abort); + +/** + * init_kutf_core() - Module entry point. + * + * Create the base entry point in debugfs. + */ +static int __init init_kutf_core(void) +{ + int ret; + + base_dir = debugfs_create_dir("kutf_tests", NULL); + if (!base_dir) { + ret = -ENODEV; + goto exit_dir; + } + + return 0; + +exit_dir: + return ret; +} + +/** + * exit_kutf_core() - Module exit point. + * + * Remove the base entry point in debugfs. + */ +static void __exit exit_kutf_core(void) +{ + debugfs_remove_recursive(base_dir); +} + +#else /* defined(CONFIG_DEBUG_FS) */ + +/** + * init_kutf_core() - Module entry point. + * + * Stub for when build against a kernel without debugfs support + */ +static int __init init_kutf_core(void) +{ + pr_debug("KUTF requires a kernel with debug fs support"); + + return -ENODEV; +} + +/** + * exit_kutf_core() - Module exit point. + * + * Stub for when build against a kernel without debugfs support + */ +static void __exit exit_kutf_core(void) +{ +} +#endif /* defined(CONFIG_DEBUG_FS) */ + +MODULE_LICENSE("GPL"); + +module_init(init_kutf_core); +module_exit(exit_kutf_core); diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c new file mode 100755 index 000000000000..a429a2dbf788 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c @@ -0,0 +1,71 @@ +/* + * + * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +/* Kernel UTF utility functions */ + +#include +#include +#include +#include + +#include +#include + +static char tmp_buffer[KUTF_MAX_DSPRINTF_LEN]; + +DEFINE_MUTEX(buffer_lock); + +const char *kutf_dsprintf(struct kutf_mempool *pool, + const char *fmt, ...) +{ + va_list args; + int len; + int size; + void *buffer; + + mutex_lock(&buffer_lock); + va_start(args, fmt); + len = vsnprintf(tmp_buffer, sizeof(tmp_buffer), fmt, args); + va_end(args); + + if (len < 0) { + pr_err("kutf_dsprintf: Bad format dsprintf format %s\n", fmt); + goto fail_format; + } + + if (len >= sizeof(tmp_buffer)) { + pr_warn("kutf_dsprintf: Truncated dsprintf message %s\n", fmt); + size = sizeof(tmp_buffer); + } else { + size = len + 1; + } + + buffer = kutf_mempool_alloc(pool, size); + if (!buffer) + goto fail_alloc; + + memcpy(buffer, tmp_buffer, size); + mutex_unlock(&buffer_lock); + + return buffer; + +fail_alloc: +fail_format: + mutex_unlock(&buffer_lock); + return NULL; +} +EXPORT_SYMBOL(kutf_dsprintf); diff --git a/drivers/gpu/arm/midgard/tests/kutf/sconscript b/drivers/gpu/arm/midgard/tests/kutf/sconscript new file mode 100755 index 000000000000..d7f112448e42 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/kutf/sconscript @@ -0,0 +1,21 @@ +# +# (C) COPYRIGHT 2014-2016, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +Import('kutf_env') + +make_args = kutf_env.kernel_get_config_defines(ret_list = True) + +mod = kutf_env.BuildKernelModule('$STATIC_LIB_PATH/kutf.ko', Glob('*.c'), make_args = make_args) +kutf_env.KernelObjTarget('kutf', mod) diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild new file mode 100755 index 000000000000..0cd9cebe9d8b --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild @@ -0,0 +1,20 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android + +obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test.o + +mali_kutf_irq_test-y := mali_kutf_irq_test_main.o diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig new file mode 100755 index 000000000000..16f68d15c46e --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig @@ -0,0 +1,23 @@ +# +# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +config MALI_IRQ_LATENCY + tristate "Mali GPU IRQ latency measurement" + depends on MALI_MIDGARD && MALI_DEBUG && MALI_KUTF + default n + help + This option will build a test module mali_kutf_irq_test that + can determine the latency of the Mali GPU IRQ on your system. + Choosing M here will generate a single module called mali_kutf_irq_test. diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile new file mode 100755 index 000000000000..4e948767a4ac --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile @@ -0,0 +1,51 @@ +# +# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +# linux build system bootstrap for out-of-tree module + +# default to building for the host +ARCH ?= $(shell uname -m) + +ifeq ($(KDIR),) +$(error Must specify KDIR to point to the kernel to target)) +endif + +TEST_CCFLAGS := \ + -DMALI_DEBUG=$(MALI_DEBUG) \ + -DMALI_BACKEND_KERNEL=$(MALI_BACKEND_KERNEL) \ + -DMALI_MODEL=$(MALI_MODEL) \ + -DMALI_NO_MALI=$(MALI_NO_MALI) \ + -DMALI_BASE_QA_LEAK=$(MALI_BASE_QA_LEAK) \ + -DMALI_BASE_QA_RESFAIL=$(MALI_BASE_QA_RESFAIL) \ + -DMALI_BASE_QA_USE_AFTER_FREE=$(MALI_BASE_QA_USE_AFTER_FREE) \ + -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ + -DMALI_USE_UMP=$(MALI_USE_UMP) \ + -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ + -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ + $(SCONS_CFLAGS) \ + -I$(CURDIR)/../include \ + -I$(CURDIR)/../../../../../../include \ + -I$(CURDIR)/../../../ \ + -I$(CURDIR)/../../ \ + -I$(CURDIR)/../../backend/gpu \ + -I$(CURDIR)/ \ + -I$(srctree)/drivers/staging/android \ + -I$(srctree)/include/linux + +all: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS="$(TEST_CCFLAGS)" KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../kutf/Module.symvers $(CURDIR)/../../Module.symvers" modules + +clean: + $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c new file mode 100755 index 000000000000..e2ff4432bf80 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c @@ -0,0 +1,257 @@ +/* + * + * (C) COPYRIGHT 2016, 2017 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * A copy of the licence is included with the program, and can also be obtained + * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + */ + + + +#include +#include +#include + +#include "mali_kbase.h" +#include + +#include +#include + +/* + * This file contains the code which is used for measuring interrupt latency + * of the Mali GPU IRQ. In particular, function mali_kutf_irq_latency() is + * used with this purpose and it is called within KUTF framework - a kernel + * unit test framework. The measured latency provided by this test should + * be representative for the latency of the Mali JOB/MMU IRQs as well. + */ + +/* KUTF test application pointer for this test */ +struct kutf_application *irq_app; + +/** + * struct kutf_irq_fixture data - test fixture used by the test functions. + * @kbdev: kbase device for the GPU. + * + */ +struct kutf_irq_fixture_data { + struct kbase_device *kbdev; +}; + +#define SEC_TO_NANO(s) ((s)*1000000000LL) + +/* ID for the GPU IRQ */ +#define GPU_IRQ_HANDLER 2 + +#define NR_TEST_IRQS 1000000 + +/* IRQ for the test to trigger. Currently MULTIPLE_GPU_FAULTS as we would not + * expect to see this in normal use (e.g., when Android is running). */ +#define TEST_IRQ MULTIPLE_GPU_FAULTS + +#define IRQ_TIMEOUT HZ + +/* Kernel API for setting irq throttle hook callback and irq time in us*/ +extern int kbase_set_custom_irq_handler(struct kbase_device *kbdev, + irq_handler_t custom_handler, + int irq_type); +extern irqreturn_t kbase_gpu_irq_handler(int irq, void *data); + +static DECLARE_WAIT_QUEUE_HEAD(wait); +static bool triggered; +static u64 irq_time; + +static void *kbase_untag(void *ptr) +{ + return (void *)(((uintptr_t) ptr) & ~3); +} + +/** + * kbase_gpu_irq_custom_handler - Custom IRQ throttle handler + * @irq: IRQ number + * @data: Data associated with this IRQ + * + * Return: state of the IRQ + */ +static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) +{ + struct kbase_device *kbdev = kbase_untag(data); + u32 val; + + val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); + if (val & TEST_IRQ) { + struct timespec64 tval; + + ktime_get_real_ts64(&tval); + irq_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); + + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, + NULL); + + triggered = true; + wake_up(&wait); + + return IRQ_HANDLED; + } + + /* Trigger main irq handler */ + return kbase_gpu_irq_handler(irq, data); +} + +/** + * mali_kutf_irq_default_create_fixture() - Creates the fixture data required + * for all the tests in the irq suite. + * @context: KUTF context. + * + * Return: Fixture data created on success or NULL on failure + */ +static void *mali_kutf_irq_default_create_fixture( + struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data; + + data = kutf_mempool_alloc(&context->fixture_pool, + sizeof(struct kutf_irq_fixture_data)); + + if (!data) + goto fail; + + /* Acquire the kbase device */ + data->kbdev = kbase_find_device(-1); + if (data->kbdev == NULL) { + kutf_test_fail(context, "Failed to find kbase device"); + goto fail; + } + + return data; + +fail: + return NULL; +} + +/** + * mali_kutf_irq_default_remove_fixture() - Destroy fixture data previously + * created by mali_kutf_irq_default_create_fixture. + * + * @context: KUTF context. + */ +static void mali_kutf_irq_default_remove_fixture( + struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + + kbase_release_device(kbdev); +} + +/** + * mali_kutf_irq_latency() - measure GPU IRQ latency + * @context: kutf context within which to perform the test + * + * The test triggers IRQs manually, and measures the + * time between triggering the IRQ and the IRQ handler being executed. + * + * This is not a traditional test, in that the pass/fail status has little + * meaning (other than indicating that the IRQ handler executed at all). Instead + * the results are in the latencies provided with the test result. There is no + * meaningful pass/fail result that can be obtained here, instead the latencies + * are provided for manual analysis only. + */ +static void mali_kutf_irq_latency(struct kutf_context *context) +{ + struct kutf_irq_fixture_data *data = context->fixture; + struct kbase_device *kbdev = data->kbdev; + u64 min_time = U64_MAX, max_time = 0, average_time = 0; + int i; + bool test_failed = false; + + /* Force GPU to be powered */ + kbase_pm_context_active(kbdev); + + kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler, + GPU_IRQ_HANDLER); + + for (i = 0; i < NR_TEST_IRQS; i++) { + struct timespec64 tval; + u64 start_time; + int ret; + + triggered = false; + ktime_get_real_ts64(&tval); + start_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); + + /* Trigger fake IRQ */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), + TEST_IRQ, NULL); + + ret = wait_event_timeout(wait, triggered != false, IRQ_TIMEOUT); + + if (ret == 0) { + kutf_test_fail(context, "Timed out waiting for IRQ\n"); + test_failed = true; + break; + } + + if ((irq_time - start_time) < min_time) + min_time = irq_time - start_time; + if ((irq_time - start_time) > max_time) + max_time = irq_time - start_time; + average_time += irq_time - start_time; + + udelay(10); + } + + /* Go back to default handler */ + kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_HANDLER); + + kbase_pm_context_idle(kbdev); + + if (!test_failed) { + const char *results; + + do_div(average_time, NR_TEST_IRQS); + results = kutf_dsprintf(&context->fixture_pool, + "Min latency = %lldns, Max latency = %lldns, Average latency = %lldns\n", + min_time, max_time, average_time); + kutf_test_pass(context, results); + } +} + +/** + * Module entry point for this test. + */ +int mali_kutf_irq_test_main_init(void) +{ + struct kutf_suite *suite; + + irq_app = kutf_create_application("irq"); + suite = kutf_create_suite(irq_app, "irq_default", + 1, mali_kutf_irq_default_create_fixture, + mali_kutf_irq_default_remove_fixture); + + kutf_add_test(suite, 0x0, "irq_latency", + mali_kutf_irq_latency); + return 0; +} + +/** + * Module exit point for this test. + */ +void mali_kutf_irq_test_main_exit(void) +{ + kutf_destroy_application(irq_app); +} + +module_init(mali_kutf_irq_test_main_init); +module_exit(mali_kutf_irq_test_main_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("ARM Ltd."); +MODULE_VERSION("1.0"); diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/sconscript b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/sconscript new file mode 100755 index 000000000000..ec837f16448d --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/sconscript @@ -0,0 +1,30 @@ +# +# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +import os +Import('env') + +src = [Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/*.c'), Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile')] + +if env.GetOption('clean') : + env.Execute(Action("make clean", '[CLEAN] mali_kutf_irq_test')) + cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, []) + env.KernelObjTarget('mali_kutf_irq_test', cmd) +else: + makeAction=Action("cd ${SOURCE.dir} && make MALI_DEBUG=${debug} MALI_BACKEND_KERNEL=1 MALI_ERROR_INJECT_ON=${error_inject} MALI_MODEL=${mali_model} MALI_NO_MALI=${no_mali} MALI_HW_VERSION=${hwver} MALI_UNIT_TEST=${unit} MALI_USE_UMP=${ump} MALI_CUSTOMER_RELEASE=${release} %s %s && ( ( [ -f mali_kutf_irq_test.ko ] && cp mali_kutf_irq_test.ko $STATIC_LIB_PATH/ ) || touch $STATIC_LIB_PATH/mali_kutf_irq_test.ko)" % (env.base_get_qa_settings(), env.kernel_get_config_defines()), '$MAKECOMSTR') + cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, [makeAction]) + env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/kutf.ko') + env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/mali_kbase.ko') + env.KernelObjTarget('mali_kutf_irq_test', cmd) diff --git a/drivers/gpu/arm/midgard/tests/sconscript b/drivers/gpu/arm/midgard/tests/sconscript new file mode 100755 index 000000000000..5337e1078e20 --- /dev/null +++ b/drivers/gpu/arm/midgard/tests/sconscript @@ -0,0 +1,37 @@ +# +# (C) COPYRIGHT 2010-2011, 2013, 2017 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# +# + + +Import ('env') + +kutf_env = env.Clone() +kutf_env.Append(CPPPATH = '#kernel/drivers/gpu/arm/midgard/tests/include') +Export('kutf_env') + +if Glob('internal/sconscript'): + SConscript('internal/sconscript') + +if kutf_env['debug'] == '1': + SConscript('kutf/sconscript') + SConscript('mali_kutf_irq_test/sconscript') + + if Glob('kutf_test/sconscript'): + SConscript('kutf_test/sconscript') + + if Glob('kutf_test_runner/sconscript'): + SConscript('kutf_test_runner/sconscript') + +if env['unit'] == '1': + SConscript('mali_kutf_ipa_test/sconscript') + SConscript('mali_kutf_vinstr_test/sconscript') diff --git a/drivers/gpu/arm/sconscript b/drivers/gpu/arm/sconscript new file mode 100755 index 000000000000..a06092bd5bf0 --- /dev/null +++ b/drivers/gpu/arm/sconscript @@ -0,0 +1,25 @@ +# +# (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +import glob + + +SConscript('midgard/sconscript') diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 4e9b3a95fa7c..95994151a360 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -32,6 +32,10 @@ config DRM_MIPI_DBI depends on DRM select DRM_KMS_HELPER +config DRM_IGNORE_IOTCL_PERMIT + bool "Ignore drm ioctl permission" + depends on DRM && ANDROID && NO_GKI + config DRM_MIPI_DSI bool depends on DRM diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c index e8baa07450b7..25fdee32ec69 100644 --- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c +++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +41,20 @@ struct bridge_init { struct device_node *node; }; +static bool analogix_dp_bandwidth_ok(struct analogix_dp_device *dp, + const struct drm_display_mode *mode, + unsigned int rate, unsigned int lanes) +{ + u32 max_bw, req_bw, bpp = 24; + + req_bw = mode->clock * bpp / 8; + max_bw = lanes * rate; + if (req_bw > max_bw) + return false; + + return true; +} + static int analogix_dp_init_dp(struct analogix_dp_device *dp) { int ret; @@ -64,6 +79,46 @@ static int analogix_dp_init_dp(struct analogix_dp_device *dp) return 0; } +static int analogix_dp_panel_prepare(struct analogix_dp_device *dp) +{ + int ret; + + mutex_lock(&dp->panel_lock); + + if (dp->panel_is_prepared) + goto out; + + ret = drm_panel_prepare(dp->plat_data->panel); + if (ret) + goto out; + + dp->panel_is_prepared = true; + +out: + mutex_unlock(&dp->panel_lock); + return 0; +} + +static int analogix_dp_panel_unprepare(struct analogix_dp_device *dp) +{ + int ret; + + mutex_lock(&dp->panel_lock); + + if (!dp->panel_is_prepared) + goto out; + + ret = drm_panel_unprepare(dp->plat_data->panel); + if (ret) + goto out; + + dp->panel_is_prepared = false; + +out: + mutex_unlock(&dp->panel_lock); + return 0; +} + static int analogix_dp_detect_hpd(struct analogix_dp_device *dp) { int timeout_loop = 0; @@ -108,6 +163,9 @@ static bool analogix_dp_detect_sink_psr(struct analogix_dp_device *dp) unsigned char psr_version; int ret; + if (!device_property_read_bool(dp->dev, "support-psr")) + return 0; + ret = drm_dp_dpcd_readb(&dp->aux, DP_PSR_SUPPORT, &psr_version); if (ret != 1) { dev_err(dp->dev, "failed to get PSR version, disable it\n"); @@ -216,8 +274,24 @@ static int analogix_dp_set_enhanced_mode(struct analogix_dp_device *dp) if (ret < 0) return ret; + if (!data) { + /* + * A setting of 1 indicates that this is an eDP device that + * uses only Enhanced Framing, independently of the setting by + * the source of ENHANCED_FRAME_EN + */ + ret = drm_dp_dpcd_readb(&dp->aux, DP_EDP_CONFIGURATION_CAP, + &data); + if (ret < 0) + return ret; + + data = !!(data & DP_FRAMING_CHANGE_CAP); + } + analogix_dp_enable_enhanced_mode(dp, data); + dp->link_train.enhanced_framing = data; + return 0; } @@ -233,32 +307,10 @@ static int analogix_dp_training_pattern_dis(struct analogix_dp_device *dp) return ret < 0 ? ret : 0; } -static void -analogix_dp_set_lane_lane_pre_emphasis(struct analogix_dp_device *dp, - int pre_emphasis, int lane) -{ - switch (lane) { - case 0: - analogix_dp_set_lane0_pre_emphasis(dp, pre_emphasis); - break; - case 1: - analogix_dp_set_lane1_pre_emphasis(dp, pre_emphasis); - break; - - case 2: - analogix_dp_set_lane2_pre_emphasis(dp, pre_emphasis); - break; - - case 3: - analogix_dp_set_lane3_pre_emphasis(dp, pre_emphasis); - break; - } -} - static int analogix_dp_link_start(struct analogix_dp_device *dp) { u8 buf[4]; - int lane, lane_count, pll_tries, retval; + int lane, lane_count, retval; lane_count = dp->link_train.lane_count; @@ -278,6 +330,14 @@ static int analogix_dp_link_start(struct analogix_dp_device *dp) retval = drm_dp_dpcd_write(&dp->aux, DP_LINK_BW_SET, buf, 2); if (retval < 0) return retval; + + /* Spread AMP if required, enable 8b/10b coding */ + buf[0] = analogix_dp_ssc_supported(dp) ? DP_SPREAD_AMP_0_5 : 0; + buf[1] = DP_SET_ANSI_8B10B; + retval = drm_dp_dpcd_write(&dp->aux, DP_DOWNSPREAD_CTRL, buf, 2); + if (retval < 0) + return retval; + /* set enhanced mode if available */ retval = analogix_dp_set_enhanced_mode(dp); if (retval < 0) { @@ -285,22 +345,12 @@ static int analogix_dp_link_start(struct analogix_dp_device *dp) return retval; } - /* Set TX pre-emphasis to minimum */ + /* Set TX voltage-swing and pre-emphasis to minimum */ for (lane = 0; lane < lane_count; lane++) - analogix_dp_set_lane_lane_pre_emphasis(dp, - PRE_EMPHASIS_LEVEL_0, lane); - - /* Wait for PLL lock */ - pll_tries = 0; - while (analogix_dp_get_pll_lock_status(dp) == PLL_UNLOCKED) { - if (pll_tries == DP_TIMEOUT_LOOP_COUNT) { - dev_err(dp->dev, "Wait for PLL lock timed out\n"); - return -ETIMEDOUT; - } - - pll_tries++; - usleep_range(90, 120); - } + dp->link_train.training_lane[lane] = + DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | + DP_TRAIN_PRE_EMPH_LEVEL_0; + analogix_dp_set_lane_link_training(dp); /* Set training pattern 1 */ analogix_dp_set_training_pattern(dp, TRAINING_PTN1); @@ -383,54 +433,6 @@ static unsigned char analogix_dp_get_adjust_request_pre_emphasis( return ((link_value >> shift) & 0xc) >> 2; } -static void analogix_dp_set_lane_link_training(struct analogix_dp_device *dp, - u8 training_lane_set, int lane) -{ - switch (lane) { - case 0: - analogix_dp_set_lane0_link_training(dp, training_lane_set); - break; - case 1: - analogix_dp_set_lane1_link_training(dp, training_lane_set); - break; - - case 2: - analogix_dp_set_lane2_link_training(dp, training_lane_set); - break; - - case 3: - analogix_dp_set_lane3_link_training(dp, training_lane_set); - break; - } -} - -static unsigned int -analogix_dp_get_lane_link_training(struct analogix_dp_device *dp, - int lane) -{ - u32 reg; - - switch (lane) { - case 0: - reg = analogix_dp_get_lane0_link_training(dp); - break; - case 1: - reg = analogix_dp_get_lane1_link_training(dp); - break; - case 2: - reg = analogix_dp_get_lane2_link_training(dp); - break; - case 3: - reg = analogix_dp_get_lane3_link_training(dp); - break; - default: - WARN_ON(1); - return 0; - } - - return reg; -} - static void analogix_dp_reduce_link_rate(struct analogix_dp_device *dp) { analogix_dp_training_pattern_dis(dp); @@ -463,11 +465,25 @@ static void analogix_dp_get_adjust_training_lane(struct analogix_dp_device *dp, } } +static bool analogix_dp_tps3_supported(struct analogix_dp_device *dp) +{ + bool source_tps3_supported, sink_tps3_supported; + u8 dpcd = 0; + + source_tps3_supported = + dp->video_info.max_link_rate == DP_LINK_BW_5_4; + drm_dp_dpcd_readb(&dp->aux, DP_MAX_LANE_COUNT, &dpcd); + sink_tps3_supported = dpcd & DP_TPS3_SUPPORTED; + + return source_tps3_supported && sink_tps3_supported; +} + static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp) { int lane, lane_count, retval; u8 voltage_swing, pre_emphasis, training_lane; u8 link_status[2], adjust_request[2]; + u8 training_pattern = TRAINING_PTN2; usleep_range(100, 101); @@ -483,12 +499,16 @@ static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp) return retval; if (analogix_dp_clock_recovery_ok(link_status, lane_count) == 0) { - /* set training pattern 2 for EQ */ - analogix_dp_set_training_pattern(dp, TRAINING_PTN2); + if (analogix_dp_tps3_supported(dp)) + training_pattern = TRAINING_PTN3; + + /* set training pattern for EQ */ + analogix_dp_set_training_pattern(dp, training_pattern); retval = drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET, DP_LINK_SCRAMBLING_DISABLE | - DP_TRAINING_PATTERN_2); + (training_pattern == TRAINING_PTN3 ? + DP_TRAINING_PATTERN_3 : DP_TRAINING_PATTERN_2)); if (retval < 0) return retval; @@ -519,25 +539,23 @@ static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp) return -EIO; } } - } - - analogix_dp_get_adjust_training_lane(dp, adjust_request); - for (lane = 0; lane < lane_count; lane++) - analogix_dp_set_lane_link_training(dp, - dp->link_train.training_lane[lane], lane); + analogix_dp_get_adjust_training_lane(dp, adjust_request); + analogix_dp_set_lane_link_training(dp); - retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET, - dp->link_train.training_lane, lane_count); - if (retval < 0) - return retval; + retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET, + dp->link_train.training_lane, + lane_count); + if (retval < 0) + return retval; + } return 0; } static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp) { - int lane, lane_count, retval; + int lane_count, retval; u32 reg; u8 link_align, link_status[2], adjust_request[2]; @@ -597,9 +615,7 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp) return -EIO; } - for (lane = 0; lane < lane_count; lane++) - analogix_dp_set_lane_link_training(dp, - dp->link_train.training_lane[lane], lane); + analogix_dp_set_lane_link_training(dp); retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET, dp->link_train.training_lane, lane_count); @@ -609,10 +625,11 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp) return 0; } -static void analogix_dp_get_max_rx_bandwidth(struct analogix_dp_device *dp, - u8 *bandwidth) +static int analogix_dp_get_max_rx_bandwidth(struct analogix_dp_device *dp, + u8 *bandwidth) { u8 data; + int ret; /* * For DP rev.1.1, Maximum link rate of Main Link lanes @@ -620,28 +637,41 @@ static void analogix_dp_get_max_rx_bandwidth(struct analogix_dp_device *dp, * For DP rev.1.2, Maximum link rate of Main Link lanes * 0x06 = 1.62 Gbps, 0x0a = 2.7 Gbps, 0x14 = 5.4Gbps */ - drm_dp_dpcd_readb(&dp->aux, DP_MAX_LINK_RATE, &data); + ret = drm_dp_dpcd_readb(&dp->aux, DP_MAX_LINK_RATE, &data); + if (ret < 0) + return ret; + *bandwidth = data; + + return 0; } -static void analogix_dp_get_max_rx_lane_count(struct analogix_dp_device *dp, - u8 *lane_count) +static int analogix_dp_get_max_rx_lane_count(struct analogix_dp_device *dp, + u8 *lane_count) { u8 data; + int ret; /* * For DP rev.1.1, Maximum number of Main Link lanes * 0x01 = 1 lane, 0x02 = 2 lanes, 0x04 = 4 lanes */ - drm_dp_dpcd_readb(&dp->aux, DP_MAX_LANE_COUNT, &data); + ret = drm_dp_dpcd_readb(&dp->aux, DP_MAX_LANE_COUNT, &data); + if (ret < 0) + return ret; + *lane_count = DPCD_MAX_LANE_COUNT(data); + + return 0; } static int analogix_dp_full_link_train(struct analogix_dp_device *dp, u32 max_lanes, u32 max_rate) { + struct video_info *video = &dp->video_info; int retval = 0; bool training_finished = false; + u8 dpcd; /* * MACRO_RST must be applied after the PLL_LOCK to avoid @@ -667,6 +697,16 @@ static int analogix_dp_full_link_train(struct analogix_dp_device *dp, dp->link_train.lane_count = (u8)LANE_COUNT1; } + if (!analogix_dp_bandwidth_ok(dp, &video->mode, + drm_dp_bw_code_to_link_rate(dp->link_train.link_rate), + dp->link_train.lane_count)) { + dev_err(dp->dev, "bandwidth overflow\n"); + return -EINVAL; + } + + drm_dp_dpcd_readb(&dp->aux, DP_MAX_DOWNSPREAD, &dpcd); + dp->link_train.ssc = !!(dpcd & DP_MAX_DOWNSPREAD_0_5); + /* Setup TX lane count & rate */ if (dp->link_train.lane_count > max_lanes) dp->link_train.lane_count = max_lanes; @@ -711,27 +751,15 @@ static int analogix_dp_full_link_train(struct analogix_dp_device *dp, static int analogix_dp_fast_link_train(struct analogix_dp_device *dp) { - int i, ret; + int ret; u8 link_align, link_status[2]; - enum pll_status status; analogix_dp_reset_macro(dp); analogix_dp_set_link_bandwidth(dp, dp->link_train.link_rate); analogix_dp_set_lane_count(dp, dp->link_train.lane_count); - - for (i = 0; i < dp->link_train.lane_count; i++) { - analogix_dp_set_lane_link_training(dp, - dp->link_train.training_lane[i], i); - } - - ret = readx_poll_timeout(analogix_dp_get_pll_lock_status, dp, status, - status != PLL_UNLOCKED, 120, - 120 * DP_TIMEOUT_LOOP_COUNT); - if (ret) { - DRM_DEV_ERROR(dp->dev, "Wait for pll lock failed %d\n", ret); - return ret; - } + analogix_dp_set_lane_link_training(dp); + analogix_dp_enable_enhanced_mode(dp, dp->link_train.enhanced_framing); /* source Set training pattern 1 */ analogix_dp_set_training_pattern(dp, TRAINING_PTN1); @@ -742,7 +770,6 @@ static int analogix_dp_fast_link_train(struct analogix_dp_device *dp) /* From DP spec, pattern must be on-screen for a minimum 500us */ usleep_range(500, 600); - /* TODO: enhanced_mode?*/ analogix_dp_set_training_pattern(dp, DP_NONE); /* @@ -884,25 +911,44 @@ static int analogix_dp_enable_scramble(struct analogix_dp_device *dp, return ret < 0 ? ret : 0; } +static irqreturn_t analogix_dp_hpd_irq_handler(int irq, void *arg) +{ + struct analogix_dp_device *dp = arg; + + if (dp->drm_dev) + drm_helper_hpd_irq_event(dp->drm_dev); + + return IRQ_HANDLED; +} + static irqreturn_t analogix_dp_hardirq(int irq, void *arg) { struct analogix_dp_device *dp = arg; - irqreturn_t ret = IRQ_NONE; enum dp_irq_type irq_type; + int ret; + + ret = pm_runtime_get_sync(dp->dev); + if (ret < 0) + return IRQ_NONE; irq_type = analogix_dp_get_irq_type(dp); - if (irq_type != DP_IRQ_TYPE_UNKNOWN) { + if (irq_type != DP_IRQ_TYPE_UNKNOWN) analogix_dp_mute_hpd_interrupt(dp); - ret = IRQ_WAKE_THREAD; - } - return ret; + pm_runtime_put_sync(dp->dev); + + return IRQ_WAKE_THREAD; } static irqreturn_t analogix_dp_irq_thread(int irq, void *arg) { struct analogix_dp_device *dp = arg; enum dp_irq_type irq_type; + int ret; + + ret = pm_runtime_get_sync(dp->dev); + if (ret < 0) + return IRQ_NONE; irq_type = analogix_dp_get_irq_type(dp); if (irq_type & DP_IRQ_TYPE_HP_CABLE_IN || @@ -917,6 +963,8 @@ static irqreturn_t analogix_dp_irq_thread(int irq, void *arg) analogix_dp_unmute_hpd_interrupt(dp); } + pm_runtime_put_sync(dp->dev); + return IRQ_HANDLED; } @@ -938,13 +986,12 @@ static int analogix_dp_fast_link_train_detection(struct analogix_dp_device *dp) static int analogix_dp_commit(struct analogix_dp_device *dp) { + struct video_info *video = &dp->video_info; int ret; - /* Keep the panel disabled while we configure video */ - if (dp->plat_data->panel) { - if (drm_panel_disable(dp->plat_data->panel)) - DRM_ERROR("failed to disable the panel\n"); - } + if (device_property_read_bool(dp->dev, "panel-self-test")) + return drm_dp_dpcd_writeb(&dp->aux, DP_EDP_CONFIGURATION_SET, + DP_PANEL_SELF_TEST_ENABLE); ret = analogix_dp_train_link(dp); if (ret) { @@ -959,21 +1006,17 @@ static int analogix_dp_commit(struct analogix_dp_device *dp) } analogix_dp_init_video(dp); + analogix_dp_set_video_format(dp); + + if (video->video_bist_enable) + analogix_dp_video_bist_enable(dp); + ret = analogix_dp_config_video(dp); if (ret) { dev_err(dp->dev, "unable to config video\n"); return ret; } - /* Safe to enable the panel now */ - if (dp->plat_data->panel) { - ret = drm_panel_enable(dp->plat_data->panel); - if (ret) { - DRM_ERROR("failed to enable the panel\n"); - return ret; - } - } - /* Check whether panel supports fast training */ ret = analogix_dp_fast_link_train_detection(dp); if (ret) @@ -1058,66 +1101,18 @@ static int analogix_dp_disable_psr(struct analogix_dp_device *dp) return analogix_dp_send_psr_spd(dp, &psr_vsc, true); } -/* - * This function is a bit of a catch-all for panel preparation, hopefully - * simplifying the logic of functions that need to prepare/unprepare the panel - * below. - * - * If @prepare is true, this function will prepare the panel. Conversely, if it - * is false, the panel will be unprepared. - * - * If @is_modeset_prepare is true, the function will disregard the current state - * of the panel and either prepare/unprepare the panel based on @prepare. Once - * it finishes, it will update dp->panel_is_modeset to reflect the current state - * of the panel. - */ -static int analogix_dp_prepare_panel(struct analogix_dp_device *dp, - bool prepare, bool is_modeset_prepare) -{ - int ret = 0; - - if (!dp->plat_data->panel) - return 0; - - mutex_lock(&dp->panel_lock); - - /* - * Exit early if this is a temporary prepare/unprepare and we're already - * modeset (since we neither want to prepare twice or unprepare early). - */ - if (dp->panel_is_modeset && !is_modeset_prepare) - goto out; - - if (prepare) - ret = drm_panel_prepare(dp->plat_data->panel); - else - ret = drm_panel_unprepare(dp->plat_data->panel); - - if (ret) - goto out; - - if (is_modeset_prepare) - dp->panel_is_modeset = prepare; - -out: - mutex_unlock(&dp->panel_lock); - return ret; -} - static int analogix_dp_get_modes(struct drm_connector *connector) { struct analogix_dp_device *dp = to_dp(connector); struct edid *edid; - int ret, num_modes = 0; + int num_modes = 0; - if (dp->plat_data->panel) { + if (dp->plat_data->panel) num_modes += drm_panel_get_modes(dp->plat_data->panel, connector); - } else { - ret = analogix_dp_prepare_panel(dp, true, false); - if (ret) { - DRM_ERROR("Failed to prepare panel (%d)\n", ret); - return 0; - } + + if (!num_modes) { + if (dp->plat_data->panel) + analogix_dp_panel_prepare(dp); pm_runtime_get_sync(dp->dev); edid = drm_get_edid(connector, &dp->aux.ddc); @@ -1128,15 +1123,18 @@ static int analogix_dp_get_modes(struct drm_connector *connector) num_modes += drm_add_edid_modes(&dp->connector, edid); kfree(edid); } - - ret = analogix_dp_prepare_panel(dp, false, false); - if (ret) - DRM_ERROR("Failed to unprepare panel (%d)\n", ret); } if (dp->plat_data->get_modes) num_modes += dp->plat_data->get_modes(dp->plat_data, connector); + if (num_modes > 0 && dp->plat_data->split_mode) { + struct drm_display_mode *mode; + + list_for_each_entry(mode, &connector->probed_modes, head) + dp->plat_data->convert_to_split_mode(mode); + } + return num_modes; } @@ -1182,34 +1180,52 @@ static const struct drm_connector_helper_funcs analogix_dp_connector_helper_func }; static enum drm_connector_status -analogix_dp_detect(struct drm_connector *connector, bool force) +analogix_dp_detect(struct analogix_dp_device *dp) { - struct analogix_dp_device *dp = to_dp(connector); enum drm_connector_status status = connector_status_disconnected; int ret; if (dp->plat_data->panel) - return connector_status_connected; + analogix_dp_panel_prepare(dp); - ret = analogix_dp_prepare_panel(dp, true, false); - if (ret) { - DRM_ERROR("Failed to prepare panel (%d)\n", ret); - return connector_status_disconnected; - } + pm_runtime_get_sync(dp->dev); + + if (!analogix_dp_detect_hpd(dp)) { + ret = analogix_dp_get_max_rx_bandwidth(dp, &dp->link_train.link_rate); + if (ret) { + dev_err(dp->dev, "failed to read max link rate\n"); + goto out; + } + + ret = analogix_dp_get_max_rx_lane_count(dp, &dp->link_train.lane_count); + if (ret) { + dev_err(dp->dev, "failed to read max lane count\n"); + goto out; + } - if (!analogix_dp_detect_hpd(dp)) status = connector_status_connected; + } - ret = analogix_dp_prepare_panel(dp, false, false); - if (ret) - DRM_ERROR("Failed to unprepare panel (%d)\n", ret); +out: + pm_runtime_put(dp->dev); return status; } +static enum drm_connector_status +analogix_dp_connector_detect(struct drm_connector *connector, bool force) +{ + struct analogix_dp_device *dp = to_dp(connector); + + if (dp->plat_data->right && analogix_dp_detect(dp->plat_data->right) != connector_status_connected) + return connector_status_disconnected; + + return analogix_dp_detect(dp); +} + static const struct drm_connector_funcs analogix_dp_connector_funcs = { .fill_modes = drm_helper_probe_single_connector_modes, - .detect = analogix_dp_detect, + .detect = analogix_dp_connector_detect, .destroy = drm_connector_cleanup, .reset = drm_atomic_helper_connector_reset, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, @@ -1224,10 +1240,8 @@ static int analogix_dp_bridge_attach(struct drm_bridge *bridge, struct drm_connector *connector = NULL; int ret = 0; - if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) { - DRM_ERROR("Fix bridge driver to make connector optional!"); - return -EINVAL; - } + if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) + return 0; if (!bridge->encoder) { DRM_ERROR("Parent encoder object not found"); @@ -1287,6 +1301,14 @@ struct drm_crtc *analogix_dp_get_old_crtc(struct analogix_dp_device *dp, return conn_state->crtc; } +static void analogix_dp_bridge_detach(struct drm_bridge *bridge) +{ + struct analogix_dp_device *dp = bridge->driver_private; + + if (dp->plat_data->detach) + dp->plat_data->detach(dp->plat_data, bridge); +} + static struct drm_crtc *analogix_dp_get_new_crtc(struct analogix_dp_device *dp, struct drm_atomic_state *state) @@ -1314,20 +1336,20 @@ analogix_dp_bridge_atomic_pre_enable(struct drm_bridge *bridge, struct analogix_dp_device *dp = bridge->driver_private; struct drm_crtc *crtc; struct drm_crtc_state *old_crtc_state; - int ret; - crtc = analogix_dp_get_new_crtc(dp, old_state); - if (!crtc) - return; + if (dp->psr_supported) { + crtc = analogix_dp_get_new_crtc(dp, old_state); + if (!crtc) + return; - old_crtc_state = drm_atomic_get_old_crtc_state(old_state, crtc); - /* Don't touch the panel if we're coming back from PSR */ - if (old_crtc_state && old_crtc_state->self_refresh_active) - return; + old_crtc_state = drm_atomic_get_old_crtc_state(old_state, crtc); + /* Don't touch the panel if we're coming back from PSR */ + if (old_crtc_state && old_crtc_state->self_refresh_active) + return; + } - ret = analogix_dp_prepare_panel(dp, true, true); - if (ret) - DRM_ERROR("failed to setup the panel ret = %d\n", ret); + if (dp->plat_data->panel) + analogix_dp_panel_prepare(dp); } static int analogix_dp_set_bridge(struct analogix_dp_device *dp) @@ -1336,16 +1358,10 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp) pm_runtime_get_sync(dp->dev); - ret = clk_prepare_enable(dp->clock); - if (ret < 0) { - DRM_ERROR("Failed to prepare_enable the clock clk [%d]\n", ret); - goto out_dp_clk_pre; - } - if (dp->plat_data->power_on_start) dp->plat_data->power_on_start(dp->plat_data); - phy_power_on(dp->phy); + analogix_dp_phy_power_on(dp); ret = analogix_dp_init_dp(dp); if (ret) @@ -1363,11 +1379,14 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp) } ret = analogix_dp_commit(dp); - if (ret) { + if (ret < 0) { DRM_ERROR("dp commit error, ret = %d\n", ret); goto out_dp_init; } + if (dp->plat_data->panel) + drm_panel_enable(dp->plat_data->panel); + if (dp->plat_data->power_on_end) dp->plat_data->power_on_end(dp->plat_data); @@ -1375,11 +1394,9 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp) return 0; out_dp_init: - phy_power_off(dp->phy); + analogix_dp_phy_power_off(dp); if (dp->plat_data->power_off) dp->plat_data->power_off(dp->plat_data); - clk_disable_unprepare(dp->clock); -out_dp_clk_pre: pm_runtime_put_sync(dp->dev); return ret; @@ -1396,17 +1413,19 @@ analogix_dp_bridge_atomic_enable(struct drm_bridge *bridge, int timeout_loop = 0; int ret; - crtc = analogix_dp_get_new_crtc(dp, old_state); - if (!crtc) - return; + if (dp->psr_supported) { + crtc = analogix_dp_get_new_crtc(dp, old_state); + if (!crtc) + return; - old_crtc_state = drm_atomic_get_old_crtc_state(old_state, crtc); - /* Not a full enable, just disable PSR and continue */ - if (old_crtc_state && old_crtc_state->self_refresh_active) { - ret = analogix_dp_disable_psr(dp); - if (ret) - DRM_ERROR("Failed to disable psr %d\n", ret); - return; + old_crtc_state = drm_atomic_get_old_crtc_state(old_state, crtc); + /* Not a full enable, just disable PSR and continue */ + if (old_crtc_state && old_crtc_state->self_refresh_active) { + ret = analogix_dp_disable_psr(dp); + if (ret) + DRM_ERROR("Failed to disable psr %d\n", ret); + return; + } } if (dp->dpms_mode == DRM_MODE_DPMS_ON) @@ -1428,7 +1447,6 @@ analogix_dp_bridge_atomic_enable(struct drm_bridge *bridge, static void analogix_dp_bridge_disable(struct drm_bridge *bridge) { struct analogix_dp_device *dp = bridge->driver_private; - int ret; if (dp->dpms_mode != DRM_MODE_DPMS_ON) return; @@ -1445,16 +1463,14 @@ static void analogix_dp_bridge_disable(struct drm_bridge *bridge) if (dp->plat_data->power_off) dp->plat_data->power_off(dp->plat_data); + analogix_dp_reset_aux(dp); analogix_dp_set_analog_power_down(dp, POWER_ALL, 1); - phy_power_off(dp->phy); - - clk_disable_unprepare(dp->clock); + analogix_dp_phy_power_off(dp); pm_runtime_put_sync(dp->dev); - ret = analogix_dp_prepare_panel(dp, false, true); - if (ret) - DRM_ERROR("failed to setup the panel ret = %d\n", ret); + if (dp->plat_data->panel) + analogix_dp_panel_unprepare(dp); dp->fast_train_enable = false; dp->psr_supported = false; @@ -1526,14 +1542,19 @@ analogix_dp_bridge_atomic_post_disable(struct drm_bridge *bridge, static void analogix_dp_bridge_mode_set(struct drm_bridge *bridge, const struct drm_display_mode *orig_mode, - const struct drm_display_mode *mode) + const struct drm_display_mode *adj_mode) { struct analogix_dp_device *dp = bridge->driver_private; struct drm_display_info *display_info = &dp->connector.display_info; struct video_info *video = &dp->video_info; + struct drm_display_mode *mode = &video->mode; struct device_node *dp_node = dp->dev->of_node; int vic; + drm_mode_copy(mode, adj_mode); + if (dp->plat_data->split_mode) + dp->plat_data->convert_to_origin_mode(mode); + /* Input video interlaces & hsync pol & vsync pol */ video->interlaced = !!(mode->flags & DRM_MODE_FLAG_INTERLACE); video->v_sync_polarity = !!(mode->flags & DRM_MODE_FLAG_NVSYNC); @@ -1601,6 +1622,27 @@ static void analogix_dp_bridge_mode_set(struct drm_bridge *bridge, video->interlaced = true; } +static enum drm_mode_status +analogix_dp_bridge_mode_valid(struct drm_bridge *bridge, + const struct drm_display_info *info, + const struct drm_display_mode *mode) +{ + struct analogix_dp_device *dp = bridge->driver_private; + struct drm_display_mode m; + + drm_mode_copy(&m, mode); + + if (dp->plat_data->split_mode) + dp->plat_data->convert_to_origin_mode(&m); + + if (!analogix_dp_bandwidth_ok(dp, &m, + drm_dp_bw_code_to_link_rate(dp->link_train.link_rate), + dp->link_train.lane_count)) + return MODE_BAD; + + return MODE_OK; +} + static const struct drm_bridge_funcs analogix_dp_bridge_funcs = { .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, @@ -1611,29 +1653,30 @@ static const struct drm_bridge_funcs analogix_dp_bridge_funcs = { .atomic_post_disable = analogix_dp_bridge_atomic_post_disable, .mode_set = analogix_dp_bridge_mode_set, .attach = analogix_dp_bridge_attach, + .detach = analogix_dp_bridge_detach, + .mode_valid = analogix_dp_bridge_mode_valid, }; -static int analogix_dp_create_bridge(struct drm_device *drm_dev, - struct analogix_dp_device *dp) +static int analogix_dp_bridge_init(struct analogix_dp_device *dp) { - struct drm_bridge *bridge; + struct drm_bridge *bridge = &dp->bridge; int ret; - bridge = devm_kzalloc(drm_dev->dev, sizeof(*bridge), GFP_KERNEL); - if (!bridge) { - DRM_ERROR("failed to allocate for drm bridge\n"); - return -ENOMEM; + if (!dp->plat_data->left) { + ret = drm_bridge_attach(dp->encoder, bridge, NULL, 0); + if (ret) { + DRM_ERROR("failed to attach drm bridge\n"); + return ret; + } } - dp->bridge = bridge; + if (dp->plat_data->right) { + struct analogix_dp_device *secondary = dp->plat_data->right; - bridge->driver_private = dp; - bridge->funcs = &analogix_dp_bridge_funcs; - - ret = drm_bridge_attach(dp->encoder, bridge, NULL, 0); - if (ret) { - DRM_ERROR("failed to attach drm bridge\n"); - return -EINVAL; + ret = drm_bridge_attach(dp->encoder, &secondary->bridge, bridge, + DRM_BRIDGE_ATTACH_NO_CONNECTOR); + if (ret) + return ret; } return 0; @@ -1646,7 +1689,7 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp) switch (dp->plat_data->dev_type) { case RK3288_DP: - case RK3399_EDP: + case RK3568_EDP: /* * Like Rk3288 DisplayPort TRM indicate that "Main link * containing 4 physical lanes of 2.7/1.62 Gbps/lane". @@ -1654,6 +1697,11 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp) video_info->max_link_rate = 0x0A; video_info->max_lane_count = 0x04; break; + case RK3399_EDP: + case RK3588_EDP: + video_info->max_link_rate = 0x14; + video_info->max_lane_count = 0x04; + break; case EXYNOS_DP: /* * NOTE: those property parseing code is used for @@ -1666,6 +1714,9 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp) break; } + video_info->video_bist_enable = + of_property_read_bool(dp_node, "analogix,video-bist-enable"); + return 0; } @@ -1688,13 +1739,72 @@ static ssize_t analogix_dpaux_transfer(struct drm_dp_aux *aux, return ret; } +int analogix_dp_audio_hw_params(struct analogix_dp_device *dp, + struct hdmi_codec_daifmt *daifmt, + struct hdmi_codec_params *params) +{ + switch (daifmt->fmt) { + case HDMI_SPDIF: + analogix_dp_audio_config_spdif(dp); + break; + case HDMI_I2S: + analogix_dp_audio_config_i2s(dp); + break; + default: + DRM_DEV_ERROR(dp->dev, "invalid daifmt %d\n", daifmt->fmt); + return -EINVAL; + } + + return 0; +} +EXPORT_SYMBOL_GPL(analogix_dp_audio_hw_params); + +void analogix_dp_audio_shutdown(struct analogix_dp_device *dp) +{ + analogix_dp_audio_disable(dp); +} +EXPORT_SYMBOL_GPL(analogix_dp_audio_shutdown); + +int analogix_dp_audio_startup(struct analogix_dp_device *dp) +{ + analogix_dp_audio_enable(dp); + + return 0; +} +EXPORT_SYMBOL_GPL(analogix_dp_audio_startup); + +int analogix_dp_audio_get_eld(struct analogix_dp_device *dp, u8 *buf, size_t len) +{ + memcpy(buf, dp->connector.eld, min(sizeof(dp->connector.eld), len)); + + return 0; +} +EXPORT_SYMBOL_GPL(analogix_dp_audio_get_eld); + +int analogix_dp_loader_protect(struct analogix_dp_device *dp) +{ + int ret; + + ret = pm_runtime_resume_and_get(dp->dev); + if (ret) { + dev_err(dp->dev, "failed to get runtime PM: %d\n", ret); + return ret; + } + + analogix_dp_phy_power_on(dp); + + dp->dpms_mode = DRM_MODE_DPMS_ON; + + return 0; +} +EXPORT_SYMBOL_GPL(analogix_dp_loader_protect); + struct analogix_dp_device * analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) { struct platform_device *pdev = to_platform_device(dev); struct analogix_dp_device *dp; struct resource *res; - unsigned int irq_flags; int ret; if (!plat_data) { @@ -1710,7 +1820,7 @@ analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) dp->dpms_mode = DRM_MODE_DPMS_OFF; mutex_init(&dp->panel_lock); - dp->panel_is_modeset = false; + dp->panel_is_prepared = false; /* * platform dp driver need containor_of the plat_data to get @@ -1739,13 +1849,13 @@ analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) } } - dp->clock = devm_clk_get(&pdev->dev, "dp"); - if (IS_ERR(dp->clock)) { - dev_err(&pdev->dev, "failed to get clock\n"); - return ERR_CAST(dp->clock); + ret = devm_clk_bulk_get_all(dev, &dp->clks); + if (ret < 0) { + dev_err(dev, "failed to get clocks %d\n", ret); + return ERR_PTR(ret); } - clk_prepare_enable(dp->clock); + dp->nr_clks = ret; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -1770,40 +1880,44 @@ analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) } if (dp->hpd_gpiod) { - /* - * Set up the hotplug GPIO from the device tree as an interrupt. - * Simply specifying a different interrupt in the device tree - * doesn't work since we handle hotplug rather differently when - * using a GPIO. We also need the actual GPIO specifier so - * that we can get the current state of the GPIO. - */ - dp->irq = gpiod_to_irq(dp->hpd_gpiod); - irq_flags = IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING; - } else { - dp->irq = platform_get_irq(pdev, 0); - irq_flags = 0; + ret = devm_request_threaded_irq(dev, + gpiod_to_irq(dp->hpd_gpiod), + NULL, + analogix_dp_hpd_irq_handler, + IRQF_TRIGGER_RISING | + IRQF_TRIGGER_FALLING | + IRQF_ONESHOT, + "analogix-hpd", dp); + if (ret) { + dev_err(dev, "failed to request hpd IRQ: %d\n", ret); + return ERR_PTR(ret); + } } + dp->irq = platform_get_irq(pdev, 0); if (dp->irq == -ENXIO) { dev_err(&pdev->dev, "failed to get irq\n"); ret = -ENODEV; goto err_disable_clk; } + irq_set_status_flags(dp->irq, IRQ_NOAUTOEN); ret = devm_request_threaded_irq(&pdev->dev, dp->irq, analogix_dp_hardirq, analogix_dp_irq_thread, - irq_flags, "analogix-dp", dp); + 0, "analogix-dp", dp); if (ret) { dev_err(&pdev->dev, "failed to request irq\n"); goto err_disable_clk; } - disable_irq(dp->irq); + + dp->bridge.driver_private = dp; + dp->bridge.funcs = &analogix_dp_bridge_funcs; return dp; err_disable_clk: - clk_disable_unprepare(dp->clock); + clk_bulk_disable_unprepare(dp->nr_clks, dp->clks); return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(analogix_dp_probe); @@ -1825,9 +1939,9 @@ int analogix_dp_bind(struct analogix_dp_device *dp, struct drm_device *drm_dev) pm_runtime_enable(dp->dev); - ret = analogix_dp_create_bridge(drm_dev, dp); + ret = analogix_dp_bridge_init(dp); if (ret) { - DRM_ERROR("failed to create bridge (%d)\n", ret); + DRM_ERROR("failed to init bridge (%d)\n", ret); goto err_disable_pm_runtime; } @@ -1842,14 +1956,7 @@ EXPORT_SYMBOL_GPL(analogix_dp_bind); void analogix_dp_unbind(struct analogix_dp_device *dp) { - analogix_dp_bridge_disable(dp->bridge); dp->connector.funcs->destroy(&dp->connector); - - if (dp->plat_data->panel) { - if (drm_panel_unprepare(dp->plat_data->panel)) - DRM_ERROR("failed to turnoff the panel\n"); - } - drm_dp_aux_unregister(&dp->aux); pm_runtime_disable(dp->dev); } @@ -1857,32 +1964,21 @@ EXPORT_SYMBOL_GPL(analogix_dp_unbind); void analogix_dp_remove(struct analogix_dp_device *dp) { - clk_disable_unprepare(dp->clock); } EXPORT_SYMBOL_GPL(analogix_dp_remove); -#ifdef CONFIG_PM -int analogix_dp_suspend(struct analogix_dp_device *dp) +int analogix_dp_runtime_suspend(struct analogix_dp_device *dp) { - clk_disable_unprepare(dp->clock); + clk_bulk_disable_unprepare(dp->nr_clks, dp->clks); return 0; } -EXPORT_SYMBOL_GPL(analogix_dp_suspend); +EXPORT_SYMBOL_GPL(analogix_dp_runtime_suspend); -int analogix_dp_resume(struct analogix_dp_device *dp) +int analogix_dp_runtime_resume(struct analogix_dp_device *dp) { - int ret; - - ret = clk_prepare_enable(dp->clock); - if (ret < 0) { - DRM_ERROR("Failed to prepare_enable the clock clk [%d]\n", ret); - return ret; - } - - return 0; + return clk_bulk_prepare_enable(dp->nr_clks, dp->clks); } -EXPORT_SYMBOL_GPL(analogix_dp_resume); -#endif +EXPORT_SYMBOL_GPL(analogix_dp_runtime_resume); int analogix_dp_start_crc(struct drm_connector *connector) { diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h index c051502d7fbf..28f2aaa144ef 100644 --- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h +++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h @@ -10,6 +10,7 @@ #define _ANALOGIX_DP_CORE_H #include +#include #include #define DP_TIMEOUT_LOOP_COUNT 100 @@ -69,6 +70,7 @@ enum pattern_set { D10_2, TRAINING_PTN1, TRAINING_PTN2, + TRAINING_PTN3, DP_NONE }; @@ -129,6 +131,7 @@ enum dp_irq_type { struct video_info { char *name; + struct drm_display_mode mode; bool h_sync_polarity; bool v_sync_polarity; @@ -141,6 +144,8 @@ struct video_info { int max_link_rate; enum link_lane_count_type max_lane_count; + + bool video_bist_enable; }; struct link_train { @@ -150,6 +155,8 @@ struct link_train { u8 link_rate; u8 lane_count; u8 training_lane[4]; + bool ssc; + bool enhanced_framing; enum link_training_state lt_state; }; @@ -159,15 +166,17 @@ struct analogix_dp_device { struct device *dev; struct drm_device *drm_dev; struct drm_connector connector; - struct drm_bridge *bridge; + struct drm_bridge bridge; struct drm_dp_aux aux; - struct clk *clock; + struct clk_bulk_data *clks; + int nr_clks; unsigned int irq; void __iomem *reg_base; struct video_info video_info; struct link_train link_train; struct phy *phy; + bool phy_enabled; int dpms_mode; struct gpio_desc *hpd_gpiod; bool force_hpd; @@ -175,7 +184,7 @@ struct analogix_dp_device { bool psr_supported; struct mutex panel_lock; - bool panel_is_modeset; + bool panel_is_prepared; struct analogix_dp_plat_data *plat_data; }; @@ -213,26 +222,8 @@ void analogix_dp_enable_enhanced_mode(struct analogix_dp_device *dp, bool enable); void analogix_dp_set_training_pattern(struct analogix_dp_device *dp, enum pattern_set pattern); -void analogix_dp_set_lane0_pre_emphasis(struct analogix_dp_device *dp, - u32 level); -void analogix_dp_set_lane1_pre_emphasis(struct analogix_dp_device *dp, - u32 level); -void analogix_dp_set_lane2_pre_emphasis(struct analogix_dp_device *dp, - u32 level); -void analogix_dp_set_lane3_pre_emphasis(struct analogix_dp_device *dp, - u32 level); -void analogix_dp_set_lane0_link_training(struct analogix_dp_device *dp, - u32 training_lane); -void analogix_dp_set_lane1_link_training(struct analogix_dp_device *dp, - u32 training_lane); -void analogix_dp_set_lane2_link_training(struct analogix_dp_device *dp, - u32 training_lane); -void analogix_dp_set_lane3_link_training(struct analogix_dp_device *dp, - u32 training_lane); -u32 analogix_dp_get_lane0_link_training(struct analogix_dp_device *dp); -u32 analogix_dp_get_lane1_link_training(struct analogix_dp_device *dp); -u32 analogix_dp_get_lane2_link_training(struct analogix_dp_device *dp); -u32 analogix_dp_get_lane3_link_training(struct analogix_dp_device *dp); +void analogix_dp_set_lane_link_training(struct analogix_dp_device *dp); +u32 analogix_dp_get_lane_link_training(struct analogix_dp_device *dp, u8 lane); void analogix_dp_reset_macro(struct analogix_dp_device *dp); void analogix_dp_init_video(struct analogix_dp_device *dp); @@ -255,5 +246,14 @@ int analogix_dp_send_psr_spd(struct analogix_dp_device *dp, struct dp_sdp *vsc, bool blocking); ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, struct drm_dp_aux_msg *msg); +void analogix_dp_set_video_format(struct analogix_dp_device *dp); +void analogix_dp_video_bist_enable(struct analogix_dp_device *dp); +bool analogix_dp_ssc_supported(struct analogix_dp_device *dp); +void analogix_dp_phy_power_on(struct analogix_dp_device *dp); +void analogix_dp_phy_power_off(struct analogix_dp_device *dp); +void analogix_dp_audio_config_spdif(struct analogix_dp_device *dp); +void analogix_dp_audio_config_i2s(struct analogix_dp_device *dp); +void analogix_dp_audio_enable(struct analogix_dp_device *dp); +void analogix_dp_audio_disable(struct analogix_dp_device *dp); #endif /* _ANALOGIX_DP_CORE_H */ diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c index cab3f5c4e2fc..dfbb5d519412 100644 --- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c +++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -21,20 +22,37 @@ #define COMMON_INT_MASK_2 0 #define COMMON_INT_MASK_3 0 #define COMMON_INT_MASK_4 (HOTPLUG_CHG | HPD_LOST | PLUG) -#define INT_STA_MASK INT_HPD + +static void analogix_dp_write(struct analogix_dp_device *dp, u32 reg, u32 val) +{ + if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) { + readl(dp->reg_base); + writel(val, dp->reg_base + reg); + } + + writel(val, dp->reg_base + reg); +} + +static u32 analogix_dp_read(struct analogix_dp_device *dp, u32 reg) +{ + if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) + readl(dp->reg_base + reg); + + return readl(dp->reg_base + reg); +} void analogix_dp_enable_video_mute(struct analogix_dp_device *dp, bool enable) { u32 reg; if (enable) { - reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); reg |= HDCP_VIDEO_MUTE; - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); } else { - reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); reg &= ~HDCP_VIDEO_MUTE; - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); } } @@ -42,9 +60,9 @@ void analogix_dp_stop_video(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); reg &= ~VIDEO_EN; - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); } void analogix_dp_lane_swap(struct analogix_dp_device *dp, bool enable) @@ -58,7 +76,7 @@ void analogix_dp_lane_swap(struct analogix_dp_device *dp, bool enable) reg = LANE3_MAP_LOGIC_LANE_3 | LANE2_MAP_LOGIC_LANE_2 | LANE1_MAP_LOGIC_LANE_1 | LANE0_MAP_LOGIC_LANE_0; - writel(reg, dp->reg_base + ANALOGIX_DP_LANE_MAP); + analogix_dp_write(dp, ANALOGIX_DP_LANE_MAP, reg); } void analogix_dp_init_analog_param(struct analogix_dp_device *dp) @@ -66,53 +84,54 @@ void analogix_dp_init_analog_param(struct analogix_dp_device *dp) u32 reg; reg = TX_TERMINAL_CTRL_50_OHM; - writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_1); + analogix_dp_write(dp, ANALOGIX_DP_ANALOG_CTL_1, reg); reg = SEL_24M | TX_DVDD_BIT_1_0625V; - writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_2); + analogix_dp_write(dp, ANALOGIX_DP_ANALOG_CTL_2, reg); if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) { reg = REF_CLK_24M; if (dp->plat_data->dev_type == RK3288_DP) reg ^= REF_CLK_MASK; - writel(reg, dp->reg_base + ANALOGIX_DP_PLL_REG_1); - writel(0x95, dp->reg_base + ANALOGIX_DP_PLL_REG_2); - writel(0x40, dp->reg_base + ANALOGIX_DP_PLL_REG_3); - writel(0x58, dp->reg_base + ANALOGIX_DP_PLL_REG_4); - writel(0x22, dp->reg_base + ANALOGIX_DP_PLL_REG_5); + analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_1, reg); + analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_2, 0x99); + analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_3, 0x40); + analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_4, 0x58); + analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_5, 0x22); + analogix_dp_write(dp, ANALOGIX_DP_BIAS, 0x44); } reg = DRIVE_DVDD_BIT_1_0625V | VCO_BIT_600_MICRO; - writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_3); + analogix_dp_write(dp, ANALOGIX_DP_ANALOG_CTL_3, reg); reg = PD_RING_OSC | AUX_TERMINAL_CTRL_50_OHM | TX_CUR1_2X | TX_CUR_16_MA; - writel(reg, dp->reg_base + ANALOGIX_DP_PLL_FILTER_CTL_1); + analogix_dp_write(dp, ANALOGIX_DP_PLL_FILTER_CTL_1, reg); reg = CH3_AMP_400_MV | CH2_AMP_400_MV | CH1_AMP_400_MV | CH0_AMP_400_MV; - writel(reg, dp->reg_base + ANALOGIX_DP_TX_AMP_TUNING_CTL); + analogix_dp_write(dp, ANALOGIX_DP_TX_AMP_TUNING_CTL, reg); } void analogix_dp_init_interrupt(struct analogix_dp_device *dp) { /* Set interrupt pin assertion polarity as high */ - writel(INT_POL1 | INT_POL0, dp->reg_base + ANALOGIX_DP_INT_CTL); + analogix_dp_write(dp, ANALOGIX_DP_INT_CTL, INT_POL1 | INT_POL0); /* Clear pending regisers */ - writel(0xff, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_1); - writel(0x4f, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_2); - writel(0xe0, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_3); - writel(0xe7, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_4); - writel(0x63, dp->reg_base + ANALOGIX_DP_INT_STA); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_1, 0xff); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_2, 0x4f); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_3, 0xe0); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_4, 0xe7); + analogix_dp_write(dp, ANALOGIX_DP_INT_STA, 0x63); /* 0:mask,1: unmask */ - writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_1); - writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_2); - writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_3); - writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); - writel(0x00, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_1, 0x00); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_2, 0x00); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_3, 0x00); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_4, 0x00); + analogix_dp_write(dp, ANALOGIX_DP_INT_STA_MASK, 0x00); } void analogix_dp_reset(struct analogix_dp_device *dp) @@ -130,44 +149,44 @@ void analogix_dp_reset(struct analogix_dp_device *dp) AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N | HDCP_FUNC_EN_N | SW_FUNC_EN_N; - writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1); + analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); reg = SSC_FUNC_EN_N | AUX_FUNC_EN_N | SERDES_FIFO_FUNC_EN_N | LS_CLK_DOMAIN_FUNC_EN_N; - writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); + analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); usleep_range(20, 30); analogix_dp_lane_swap(dp, 0); - writel(0x0, dp->reg_base + ANALOGIX_DP_SYS_CTL_1); - writel(0x40, dp->reg_base + ANALOGIX_DP_SYS_CTL_2); - writel(0x0, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); - writel(0x0, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_1, 0x0); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_2, 0x40); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, 0x0); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, 0x0); - writel(0x0, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); - writel(0x0, dp->reg_base + ANALOGIX_DP_HDCP_CTL); + analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, 0x0); + analogix_dp_write(dp, ANALOGIX_DP_HDCP_CTL, 0x0); - writel(0x5e, dp->reg_base + ANALOGIX_DP_HPD_DEGLITCH_L); - writel(0x1a, dp->reg_base + ANALOGIX_DP_HPD_DEGLITCH_H); + analogix_dp_write(dp, ANALOGIX_DP_HPD_DEGLITCH_L, 0x5e); + analogix_dp_write(dp, ANALOGIX_DP_HPD_DEGLITCH_H, 0x1a); - writel(0x10, dp->reg_base + ANALOGIX_DP_LINK_DEBUG_CTL); + analogix_dp_write(dp, ANALOGIX_DP_LINK_DEBUG_CTL, 0x10); - writel(0x0, dp->reg_base + ANALOGIX_DP_PHY_TEST); + analogix_dp_write(dp, ANALOGIX_DP_PHY_TEST, 0x0); - writel(0x0, dp->reg_base + ANALOGIX_DP_VIDEO_FIFO_THRD); - writel(0x20, dp->reg_base + ANALOGIX_DP_AUDIO_MARGIN); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_FIFO_THRD, 0x0); + analogix_dp_write(dp, ANALOGIX_DP_AUDIO_MARGIN, 0x20); - writel(0x4, dp->reg_base + ANALOGIX_DP_M_VID_GEN_FILTER_TH); - writel(0x2, dp->reg_base + ANALOGIX_DP_M_AUD_GEN_FILTER_TH); + analogix_dp_write(dp, ANALOGIX_DP_M_VID_GEN_FILTER_TH, 0x4); + analogix_dp_write(dp, ANALOGIX_DP_M_AUD_GEN_FILTER_TH, 0x2); - writel(0x00000101, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); + analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, 0x00000101); } void analogix_dp_swreset(struct analogix_dp_device *dp) { - writel(RESET_DP_TX, dp->reg_base + ANALOGIX_DP_TX_SW_RESET); + analogix_dp_write(dp, ANALOGIX_DP_TX_SW_RESET, RESET_DP_TX); } void analogix_dp_config_interrupt(struct analogix_dp_device *dp) @@ -176,19 +195,18 @@ void analogix_dp_config_interrupt(struct analogix_dp_device *dp) /* 0: mask, 1: unmask */ reg = COMMON_INT_MASK_1; - writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_1); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_1, reg); reg = COMMON_INT_MASK_2; - writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_2); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_2, reg); reg = COMMON_INT_MASK_3; - writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_3); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_3, reg); - reg = COMMON_INT_MASK_4; - writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); - - reg = INT_STA_MASK; - writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); + if (dp->force_hpd || dp->hpd_gpiod) + analogix_dp_mute_hpd_interrupt(dp); + else + analogix_dp_unmute_hpd_interrupt(dp); } void analogix_dp_mute_hpd_interrupt(struct analogix_dp_device *dp) @@ -196,13 +214,13 @@ void analogix_dp_mute_hpd_interrupt(struct analogix_dp_device *dp) u32 reg; /* 0: mask, 1: unmask */ - reg = readl(dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); + reg = analogix_dp_read(dp, ANALOGIX_DP_COMMON_INT_MASK_4); reg &= ~COMMON_INT_MASK_4; - writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_4, reg); - reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA_MASK); - reg &= ~INT_STA_MASK; - writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); + reg = analogix_dp_read(dp, ANALOGIX_DP_INT_STA_MASK); + reg &= ~INT_HPD; + analogix_dp_write(dp, ANALOGIX_DP_INT_STA_MASK, reg); } void analogix_dp_unmute_hpd_interrupt(struct analogix_dp_device *dp) @@ -211,17 +229,18 @@ void analogix_dp_unmute_hpd_interrupt(struct analogix_dp_device *dp) /* 0: mask, 1: unmask */ reg = COMMON_INT_MASK_4; - writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_4, reg); - reg = INT_STA_MASK; - writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); + reg = analogix_dp_read(dp, ANALOGIX_DP_INT_STA_MASK); + reg |= INT_HPD; + analogix_dp_write(dp, ANALOGIX_DP_INT_STA_MASK, reg); } enum pll_status analogix_dp_get_pll_lock_status(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_DEBUG_CTL); + reg = analogix_dp_read(dp, ANALOGIX_DP_DEBUG_CTL); if (reg & PLL_LOCK) return PLL_LOCKED; else @@ -239,12 +258,12 @@ void analogix_dp_set_pll_power_down(struct analogix_dp_device *dp, bool enable) mask = RK_PLL_PD; } - reg = readl(dp->reg_base + pd_addr); + reg = analogix_dp_read(dp, pd_addr); if (enable) reg |= mask; else reg &= ~mask; - writel(reg, dp->reg_base + pd_addr); + analogix_dp_write(dp, pd_addr, reg); } void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, @@ -265,52 +284,54 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, else mask = AUX_PD; - reg = readl(dp->reg_base + phy_pd_addr); - if (enable) + reg = analogix_dp_read(dp, phy_pd_addr); + if (enable) { + reg &= ~(DP_INC_BG | DP_EXP_BG); reg |= mask; - else + } else { reg &= ~mask; - writel(reg, dp->reg_base + phy_pd_addr); + } + analogix_dp_write(dp, phy_pd_addr, reg); break; case CH0_BLOCK: mask = CH0_PD; - reg = readl(dp->reg_base + phy_pd_addr); + reg = analogix_dp_read(dp, phy_pd_addr); if (enable) reg |= mask; else reg &= ~mask; - writel(reg, dp->reg_base + phy_pd_addr); + analogix_dp_write(dp, phy_pd_addr, reg); break; case CH1_BLOCK: mask = CH1_PD; - reg = readl(dp->reg_base + phy_pd_addr); + reg = analogix_dp_read(dp, phy_pd_addr); if (enable) reg |= mask; else reg &= ~mask; - writel(reg, dp->reg_base + phy_pd_addr); + analogix_dp_write(dp, phy_pd_addr, reg); break; case CH2_BLOCK: mask = CH2_PD; - reg = readl(dp->reg_base + phy_pd_addr); + reg = analogix_dp_read(dp, phy_pd_addr); if (enable) reg |= mask; else reg &= ~mask; - writel(reg, dp->reg_base + phy_pd_addr); + analogix_dp_write(dp, phy_pd_addr, reg); break; case CH3_BLOCK: mask = CH3_PD; - reg = readl(dp->reg_base + phy_pd_addr); + reg = analogix_dp_read(dp, phy_pd_addr); if (enable) reg |= mask; else reg &= ~mask; - writel(reg, dp->reg_base + phy_pd_addr); + analogix_dp_write(dp, phy_pd_addr, reg); break; case ANALOG_TOTAL: /* @@ -323,29 +344,29 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, else mask = DP_PHY_PD; - reg = readl(dp->reg_base + phy_pd_addr); + reg = analogix_dp_read(dp, phy_pd_addr); if (enable) reg |= mask; else reg &= ~mask; - writel(reg, dp->reg_base + phy_pd_addr); + analogix_dp_write(dp, phy_pd_addr, reg); if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) usleep_range(10, 15); break; case POWER_ALL: if (enable) { reg = DP_ALL_PD; - writel(reg, dp->reg_base + phy_pd_addr); + analogix_dp_write(dp, phy_pd_addr, reg); } else { reg = DP_ALL_PD; - writel(reg, dp->reg_base + phy_pd_addr); + analogix_dp_write(dp, phy_pd_addr, reg); usleep_range(10, 15); reg &= ~DP_INC_BG; - writel(reg, dp->reg_base + phy_pd_addr); + analogix_dp_write(dp, phy_pd_addr, reg); usleep_range(10, 15); - writel(0x00, dp->reg_base + phy_pd_addr); + analogix_dp_write(dp, phy_pd_addr, 0x00); } break; default: @@ -356,36 +377,24 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, int analogix_dp_init_analog_func(struct analogix_dp_device *dp) { u32 reg; - int timeout_loop = 0; analogix_dp_set_analog_power_down(dp, POWER_ALL, 0); reg = PLL_LOCK_CHG; - writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_1); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_1, reg); - reg = readl(dp->reg_base + ANALOGIX_DP_DEBUG_CTL); + reg = analogix_dp_read(dp, ANALOGIX_DP_DEBUG_CTL); reg &= ~(F_PLL_LOCK | PLL_LOCK_CTRL); - writel(reg, dp->reg_base + ANALOGIX_DP_DEBUG_CTL); + analogix_dp_write(dp, ANALOGIX_DP_DEBUG_CTL, reg); /* Power up PLL */ - if (analogix_dp_get_pll_lock_status(dp) == PLL_UNLOCKED) { - analogix_dp_set_pll_power_down(dp, 0); - - while (analogix_dp_get_pll_lock_status(dp) == PLL_UNLOCKED) { - timeout_loop++; - if (DP_TIMEOUT_LOOP_COUNT < timeout_loop) { - dev_err(dp->dev, "failed to get pll lock status\n"); - return -ETIMEDOUT; - } - usleep_range(10, 20); - } - } + analogix_dp_set_pll_power_down(dp, 0); /* Enable Serdes FIFO function and Link symbol clock domain module */ - reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); + reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); reg &= ~(SERDES_FIFO_FUNC_EN_N | LS_CLK_DOMAIN_FUNC_EN_N | AUX_FUNC_EN_N); - writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); + analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); return 0; } @@ -397,10 +406,10 @@ void analogix_dp_clear_hotplug_interrupts(struct analogix_dp_device *dp) return; reg = HOTPLUG_CHG | HPD_LOST | PLUG; - writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_4); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_4, reg); reg = INT_HPD; - writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA); + analogix_dp_write(dp, ANALOGIX_DP_INT_STA, reg); } void analogix_dp_init_hpd(struct analogix_dp_device *dp) @@ -412,45 +421,37 @@ void analogix_dp_init_hpd(struct analogix_dp_device *dp) analogix_dp_clear_hotplug_interrupts(dp); - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); reg &= ~(F_HPD | HPD_CTRL); - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); } void analogix_dp_force_hpd(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); - reg = (F_HPD | HPD_CTRL); - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); + reg |= (F_HPD | HPD_CTRL); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); } enum dp_irq_type analogix_dp_get_irq_type(struct analogix_dp_device *dp) { u32 reg; - if (dp->hpd_gpiod) { - reg = gpiod_get_value(dp->hpd_gpiod); - if (reg) - return DP_IRQ_TYPE_HP_CABLE_IN; - else - return DP_IRQ_TYPE_HP_CABLE_OUT; - } else { - /* Parse hotplug interrupt status register */ - reg = readl(dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_4); + /* Parse hotplug interrupt status register */ + reg = analogix_dp_read(dp, ANALOGIX_DP_COMMON_INT_STA_4); - if (reg & PLUG) - return DP_IRQ_TYPE_HP_CABLE_IN; + if (reg & PLUG) + return DP_IRQ_TYPE_HP_CABLE_IN; - if (reg & HPD_LOST) - return DP_IRQ_TYPE_HP_CABLE_OUT; + if (reg & HPD_LOST) + return DP_IRQ_TYPE_HP_CABLE_OUT; - if (reg & HOTPLUG_CHG) - return DP_IRQ_TYPE_HP_CHANGE; + if (reg & HOTPLUG_CHG) + return DP_IRQ_TYPE_HP_CHANGE; - return DP_IRQ_TYPE_UNKNOWN; - } + return DP_IRQ_TYPE_UNKNOWN; } void analogix_dp_reset_aux(struct analogix_dp_device *dp) @@ -458,9 +459,9 @@ void analogix_dp_reset_aux(struct analogix_dp_device *dp) u32 reg; /* Disable AUX channel module */ - reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); + reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); reg |= AUX_FUNC_EN_N; - writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); + analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); } void analogix_dp_init_aux(struct analogix_dp_device *dp) @@ -469,7 +470,7 @@ void analogix_dp_init_aux(struct analogix_dp_device *dp) /* Clear inerrupts related to AUX channel */ reg = RPLY_RECEIV | AUX_ERR; - writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA); + analogix_dp_write(dp, ANALOGIX_DP_INT_STA, reg); analogix_dp_set_analog_power_down(dp, AUX_BLOCK, true); usleep_range(10, 11); @@ -487,16 +488,17 @@ void analogix_dp_init_aux(struct analogix_dp_device *dp) reg |= AUX_HW_RETRY_COUNT_SEL(0) | AUX_HW_RETRY_INTERVAL_600_MICROSECONDS; - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_HW_RETRY_CTL); + analogix_dp_write(dp, ANALOGIX_DP_AUX_HW_RETRY_CTL, reg); /* Receive AUX Channel DEFER commands equal to DEFFER_COUNT*64 */ reg = DEFER_CTRL_EN | DEFER_COUNT(1); - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_DEFER_CTL); + analogix_dp_write(dp, ANALOGIX_DP_AUX_CH_DEFER_CTL, reg); /* Enable AUX channel module */ - reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); + analogix_dp_enable_sw_function(dp); + reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); reg &= ~AUX_FUNC_EN_N; - writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); + analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); } int analogix_dp_get_plug_in_status(struct analogix_dp_device *dp) @@ -507,7 +509,7 @@ int analogix_dp_get_plug_in_status(struct analogix_dp_device *dp) if (gpiod_get_value(dp->hpd_gpiod)) return 0; } else { - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); if (reg & HPD_STATUS) return 0; } @@ -519,145 +521,181 @@ void analogix_dp_enable_sw_function(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_1); + reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); reg &= ~SW_FUNC_EN_N; - writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1); + analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); } -int analogix_dp_start_aux_transaction(struct analogix_dp_device *dp) +static void analogix_dp_ssc_enable(struct analogix_dp_device *dp) { - int reg; - int retval = 0; - int timeout_loop = 0; - - /* Enable AUX CH operation */ - reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2); - reg |= AUX_EN; - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2); - - /* Is AUX CH command reply received? */ - reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); - while (!(reg & RPLY_RECEIV)) { - timeout_loop++; - if (DP_TIMEOUT_LOOP_COUNT < timeout_loop) { - dev_err(dp->dev, "AUX CH command reply failed!\n"); - return -ETIMEDOUT; - } - reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); - usleep_range(10, 11); - } - - /* Clear interrupt source for AUX CH command reply */ - writel(RPLY_RECEIV, dp->reg_base + ANALOGIX_DP_INT_STA); - - /* Clear interrupt source for AUX CH access error */ - reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); - if (reg & AUX_ERR) { - writel(AUX_ERR, dp->reg_base + ANALOGIX_DP_INT_STA); - return -EREMOTEIO; - } - - /* Check AUX CH error access status */ - reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_STA); - if ((reg & AUX_STATUS_MASK) != 0) { - dev_err(dp->dev, "AUX CH error happens: %d\n\n", - reg & AUX_STATUS_MASK); - return -EREMOTEIO; - } + u32 reg; - return retval; + /* 4500ppm */ + writel(0x19, dp->reg_base + ANALOIGX_DP_SSC_REG); + /* + * To apply updated SSC parameters into SSC operation, + * firmware must disable and enable this bit. + */ + reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); + reg |= SSC_FUNC_EN_N; + writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); + reg &= ~SSC_FUNC_EN_N; + writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); } -int analogix_dp_write_byte_to_dpcd(struct analogix_dp_device *dp, - unsigned int reg_addr, - unsigned char data) +static void analogix_dp_ssc_disable(struct analogix_dp_device *dp) { u32 reg; - int i; - int retval; - - for (i = 0; i < 3; i++) { - /* Clear AUX CH data buffer */ - reg = BUF_CLR; - writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL); - - /* Select DPCD device address */ - reg = AUX_ADDR_7_0(reg_addr); - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0); - reg = AUX_ADDR_15_8(reg_addr); - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8); - reg = AUX_ADDR_19_16(reg_addr); - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16); - - /* Write data buffer */ - reg = (unsigned int)data; - writel(reg, dp->reg_base + ANALOGIX_DP_BUF_DATA_0); - - /* - * Set DisplayPort transaction and write 1 byte - * If bit 3 is 1, DisplayPort transaction. - * If Bit 3 is 0, I2C transaction. - */ - reg = AUX_TX_COMM_DP_TRANSACTION | AUX_TX_COMM_WRITE; - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1); - /* Start AUX transaction */ - retval = analogix_dp_start_aux_transaction(dp); - if (retval == 0) - break; - - dev_dbg(dp->dev, "%s: Aux Transaction fail!\n", __func__); - } + reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); + reg |= SSC_FUNC_EN_N; + writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); +} - return retval; +bool analogix_dp_ssc_supported(struct analogix_dp_device *dp) +{ + /* Check if SSC is supported by both sides */ + return dp->plat_data->ssc && dp->link_train.ssc; } void analogix_dp_set_link_bandwidth(struct analogix_dp_device *dp, u32 bwtype) { - u32 reg; + u32 status; + int ret; - reg = bwtype; - if ((bwtype == DP_LINK_BW_2_7) || (bwtype == DP_LINK_BW_1_62)) - writel(reg, dp->reg_base + ANALOGIX_DP_LINK_BW_SET); + analogix_dp_write(dp, ANALOGIX_DP_LINK_BW_SET, bwtype); + + if (dp->phy) { + union phy_configure_opts phy_cfg = {0}; + + phy_cfg.dp.lanes = dp->link_train.lane_count; + phy_cfg.dp.link_rate = + drm_dp_bw_code_to_link_rate(dp->link_train.link_rate) / 100; + phy_cfg.dp.ssc = analogix_dp_ssc_supported(dp); + phy_cfg.dp.set_lanes = false; + phy_cfg.dp.set_rate = true; + phy_cfg.dp.set_voltages = false; + ret = phy_configure(dp->phy, &phy_cfg); + if (ret && ret != -EOPNOTSUPP) { + dev_err(dp->dev, "%s: phy_configure failed: %d\n", + __func__, ret); + return; + } + } else { + if (analogix_dp_ssc_supported(dp)) + analogix_dp_ssc_enable(dp); + else + analogix_dp_ssc_disable(dp); + } + + ret = readx_poll_timeout(analogix_dp_get_pll_lock_status, dp, status, + status != PLL_UNLOCKED, 120, + 120 * DP_TIMEOUT_LOOP_COUNT); + if (ret) { + dev_err(dp->dev, "Wait for pll lock failed %d\n", ret); + return; + } } void analogix_dp_get_link_bandwidth(struct analogix_dp_device *dp, u32 *bwtype) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_LINK_BW_SET); + reg = analogix_dp_read(dp, ANALOGIX_DP_LINK_BW_SET); *bwtype = reg; } void analogix_dp_set_lane_count(struct analogix_dp_device *dp, u32 count) { u32 reg; + int ret; reg = count; - writel(reg, dp->reg_base + ANALOGIX_DP_LANE_COUNT_SET); + analogix_dp_write(dp, ANALOGIX_DP_LANE_COUNT_SET, reg); + + if (dp->phy) { + union phy_configure_opts phy_cfg = {0}; + + phy_cfg.dp.lanes = dp->link_train.lane_count; + phy_cfg.dp.set_lanes = true; + phy_cfg.dp.set_rate = false; + phy_cfg.dp.set_voltages = false; + ret = phy_configure(dp->phy, &phy_cfg); + if (ret && ret != -EOPNOTSUPP) { + dev_err(dp->dev, "%s: phy_configure() failed: %d\n", + __func__, ret); + return; + } + } } void analogix_dp_get_lane_count(struct analogix_dp_device *dp, u32 *count) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_LANE_COUNT_SET); + reg = analogix_dp_read(dp, ANALOGIX_DP_LANE_COUNT_SET); *count = reg; } +void analogix_dp_set_lane_link_training(struct analogix_dp_device *dp) +{ + u8 lane; + int ret; + + for (lane = 0; lane < dp->link_train.lane_count; lane++) + analogix_dp_write(dp, + ANALOGIX_DP_LN0_LINK_TRAINING_CTL + 4 * lane, + dp->link_train.training_lane[lane]); + + if (dp->phy) { + union phy_configure_opts phy_cfg = {0}; + + for (lane = 0; lane < dp->link_train.lane_count; lane++) { + u8 training_lane = dp->link_train.training_lane[lane]; + u8 vs, pe; + + vs = (training_lane & DP_TRAIN_VOLTAGE_SWING_MASK) >> + DP_TRAIN_VOLTAGE_SWING_SHIFT; + pe = (training_lane & DP_TRAIN_PRE_EMPHASIS_MASK) >> + DP_TRAIN_PRE_EMPHASIS_SHIFT; + phy_cfg.dp.voltage[lane] = vs; + phy_cfg.dp.pre[lane] = pe; + } + + phy_cfg.dp.lanes = dp->link_train.lane_count; + phy_cfg.dp.link_rate = + drm_dp_bw_code_to_link_rate(dp->link_train.link_rate) / 100; + phy_cfg.dp.set_lanes = false; + phy_cfg.dp.set_rate = false; + phy_cfg.dp.set_voltages = true; + ret = phy_configure(dp->phy, &phy_cfg); + if (ret && ret != -EOPNOTSUPP) { + dev_err(dp->dev, "%s: phy_configure() failed: %d\n", + __func__, ret); + return; + } + } +} + +u32 analogix_dp_get_lane_link_training(struct analogix_dp_device *dp, u8 lane) +{ + return analogix_dp_read(dp, + ANALOGIX_DP_LN0_LINK_TRAINING_CTL + 4 * lane); +} + void analogix_dp_enable_enhanced_mode(struct analogix_dp_device *dp, bool enable) { u32 reg; if (enable) { - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); reg |= ENHANCED; - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); } else { - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); reg &= ~ENHANCED; - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); } } @@ -669,144 +707,48 @@ void analogix_dp_set_training_pattern(struct analogix_dp_device *dp, switch (pattern) { case PRBS7: reg = SCRAMBLING_ENABLE | LINK_QUAL_PATTERN_SET_PRBS7; - writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); + analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); break; case D10_2: reg = SCRAMBLING_ENABLE | LINK_QUAL_PATTERN_SET_D10_2; - writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); + analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); break; case TRAINING_PTN1: reg = SCRAMBLING_DISABLE | SW_TRAINING_PATTERN_SET_PTN1; - writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); + analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); break; case TRAINING_PTN2: reg = SCRAMBLING_DISABLE | SW_TRAINING_PATTERN_SET_PTN2; - writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); + analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); + break; + case TRAINING_PTN3: + reg = SCRAMBLING_DISABLE | SW_TRAINING_PATTERN_SET_PTN3; + analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); break; case DP_NONE: reg = SCRAMBLING_ENABLE | LINK_QUAL_PATTERN_SET_DISABLE | SW_TRAINING_PATTERN_SET_NORMAL; - writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); + analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); break; default: break; } } -void analogix_dp_set_lane0_pre_emphasis(struct analogix_dp_device *dp, - u32 level) -{ - u32 reg; - - reg = readl(dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); - reg &= ~PRE_EMPHASIS_SET_MASK; - reg |= level << PRE_EMPHASIS_SET_SHIFT; - writel(reg, dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); -} - -void analogix_dp_set_lane1_pre_emphasis(struct analogix_dp_device *dp, - u32 level) -{ - u32 reg; - - reg = readl(dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); - reg &= ~PRE_EMPHASIS_SET_MASK; - reg |= level << PRE_EMPHASIS_SET_SHIFT; - writel(reg, dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); -} - -void analogix_dp_set_lane2_pre_emphasis(struct analogix_dp_device *dp, - u32 level) -{ - u32 reg; - - reg = readl(dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); - reg &= ~PRE_EMPHASIS_SET_MASK; - reg |= level << PRE_EMPHASIS_SET_SHIFT; - writel(reg, dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); -} - -void analogix_dp_set_lane3_pre_emphasis(struct analogix_dp_device *dp, - u32 level) -{ - u32 reg; - - reg = readl(dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); - reg &= ~PRE_EMPHASIS_SET_MASK; - reg |= level << PRE_EMPHASIS_SET_SHIFT; - writel(reg, dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); -} - -void analogix_dp_set_lane0_link_training(struct analogix_dp_device *dp, - u32 training_lane) -{ - u32 reg; - - reg = training_lane; - writel(reg, dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); -} - -void analogix_dp_set_lane1_link_training(struct analogix_dp_device *dp, - u32 training_lane) -{ - u32 reg; - - reg = training_lane; - writel(reg, dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); -} - -void analogix_dp_set_lane2_link_training(struct analogix_dp_device *dp, - u32 training_lane) -{ - u32 reg; - - reg = training_lane; - writel(reg, dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); -} - -void analogix_dp_set_lane3_link_training(struct analogix_dp_device *dp, - u32 training_lane) -{ - u32 reg; - - reg = training_lane; - writel(reg, dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); -} - -u32 analogix_dp_get_lane0_link_training(struct analogix_dp_device *dp) -{ - return readl(dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); -} - -u32 analogix_dp_get_lane1_link_training(struct analogix_dp_device *dp) -{ - return readl(dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); -} - -u32 analogix_dp_get_lane2_link_training(struct analogix_dp_device *dp) -{ - return readl(dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); -} - -u32 analogix_dp_get_lane3_link_training(struct analogix_dp_device *dp) -{ - return readl(dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); -} - void analogix_dp_reset_macro(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_PHY_TEST); + reg = analogix_dp_read(dp, ANALOGIX_DP_PHY_TEST); reg |= MACRO_RST; - writel(reg, dp->reg_base + ANALOGIX_DP_PHY_TEST); + analogix_dp_write(dp, ANALOGIX_DP_PHY_TEST, reg); /* 10 us is the minimum reset time. */ usleep_range(10, 20); reg &= ~MACRO_RST; - writel(reg, dp->reg_base + ANALOGIX_DP_PHY_TEST); + analogix_dp_write(dp, ANALOGIX_DP_PHY_TEST, reg); } void analogix_dp_init_video(struct analogix_dp_device *dp) @@ -814,19 +756,19 @@ void analogix_dp_init_video(struct analogix_dp_device *dp) u32 reg; reg = VSYNC_DET | VID_FORMAT_CHG | VID_CLK_CHG; - writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_1); + analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_1, reg); reg = 0x0; - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_1); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_1, reg); reg = CHA_CRI(4) | CHA_CTRL; - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_2); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_2, reg); reg = 0x0; - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); reg = VID_HRES_TH(2) | VID_VRES_TH(0); - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_8); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_8, reg); } void analogix_dp_set_video_color_format(struct analogix_dp_device *dp) @@ -837,36 +779,36 @@ void analogix_dp_set_video_color_format(struct analogix_dp_device *dp) reg = (dp->video_info.dynamic_range << IN_D_RANGE_SHIFT) | (dp->video_info.color_depth << IN_BPC_SHIFT) | (dp->video_info.color_space << IN_COLOR_F_SHIFT); - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_2); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_2, reg); /* Set Input Color YCbCr Coefficients to ITU601 or ITU709 */ - reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_3); reg &= ~IN_YC_COEFFI_MASK; if (dp->video_info.ycbcr_coeff) reg |= IN_YC_COEFFI_ITU709; else reg |= IN_YC_COEFFI_ITU601; - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_3, reg); } int analogix_dp_is_slave_video_stream_clock_on(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_1); - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_1); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_1); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_1, reg); - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_1); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_1); if (!(reg & DET_STA)) { dev_dbg(dp->dev, "Input stream clock not detected.\n"); return -EINVAL; } - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_2); - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_2); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_2); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_2, reg); - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_2); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_2); dev_dbg(dp->dev, "wait SYS_CTL_2.\n"); if (reg & CHA_STA) { @@ -884,30 +826,30 @@ void analogix_dp_set_video_cr_mn(struct analogix_dp_device *dp, u32 reg; if (type == REGISTER_M) { - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); reg |= FIX_M_VID; - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); reg = m_value & 0xff; - writel(reg, dp->reg_base + ANALOGIX_DP_M_VID_0); + analogix_dp_write(dp, ANALOGIX_DP_M_VID_0, reg); reg = (m_value >> 8) & 0xff; - writel(reg, dp->reg_base + ANALOGIX_DP_M_VID_1); + analogix_dp_write(dp, ANALOGIX_DP_M_VID_1, reg); reg = (m_value >> 16) & 0xff; - writel(reg, dp->reg_base + ANALOGIX_DP_M_VID_2); + analogix_dp_write(dp, ANALOGIX_DP_M_VID_2, reg); reg = n_value & 0xff; - writel(reg, dp->reg_base + ANALOGIX_DP_N_VID_0); + analogix_dp_write(dp, ANALOGIX_DP_N_VID_0, reg); reg = (n_value >> 8) & 0xff; - writel(reg, dp->reg_base + ANALOGIX_DP_N_VID_1); + analogix_dp_write(dp, ANALOGIX_DP_N_VID_1, reg); reg = (n_value >> 16) & 0xff; - writel(reg, dp->reg_base + ANALOGIX_DP_N_VID_2); + analogix_dp_write(dp, ANALOGIX_DP_N_VID_2, reg); } else { - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); reg &= ~FIX_M_VID; - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); - writel(0x00, dp->reg_base + ANALOGIX_DP_N_VID_0); - writel(0x80, dp->reg_base + ANALOGIX_DP_N_VID_1); - writel(0x00, dp->reg_base + ANALOGIX_DP_N_VID_2); + analogix_dp_write(dp, ANALOGIX_DP_N_VID_0, 0x00); + analogix_dp_write(dp, ANALOGIX_DP_N_VID_1, 0x80); + analogix_dp_write(dp, ANALOGIX_DP_N_VID_2, 0x00); } } @@ -916,13 +858,13 @@ void analogix_dp_set_video_timing_mode(struct analogix_dp_device *dp, u32 type) u32 reg; if (type == VIDEO_TIMING_FROM_CAPTURE) { - reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); reg &= ~FORMAT_SEL; - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); } else { - reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); reg |= FORMAT_SEL; - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); } } @@ -931,15 +873,15 @@ void analogix_dp_enable_video_master(struct analogix_dp_device *dp, bool enable) u32 reg; if (enable) { - reg = readl(dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); + reg = analogix_dp_read(dp, ANALOGIX_DP_SOC_GENERAL_CTL); reg &= ~VIDEO_MODE_MASK; reg |= VIDEO_MASTER_MODE_EN | VIDEO_MODE_MASTER_MODE; - writel(reg, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); + analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, reg); } else { - reg = readl(dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); + reg = analogix_dp_read(dp, ANALOGIX_DP_SOC_GENERAL_CTL); reg &= ~VIDEO_MODE_MASK; reg |= VIDEO_MODE_SLAVE_MODE; - writel(reg, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); + analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, reg); } } @@ -947,19 +889,19 @@ void analogix_dp_start_video(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); reg |= VIDEO_EN; - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); } int analogix_dp_is_video_stream_on(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); - writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); - reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); if (!(reg & STRM_VALID)) { dev_dbg(dp->dev, "Input video stream is not detected.\n"); return -EINVAL; @@ -972,55 +914,55 @@ void analogix_dp_config_video_slave_mode(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_1); + reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) { reg &= ~(RK_VID_CAP_FUNC_EN_N | RK_VID_FIFO_FUNC_EN_N); } else { reg &= ~(MASTER_VID_FUNC_EN_N | SLAVE_VID_FUNC_EN_N); reg |= MASTER_VID_FUNC_EN_N; } - writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1); + analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); - reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); reg &= ~INTERACE_SCAN_CFG; reg |= (dp->video_info.interlaced << 2); - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); - reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); reg &= ~VSYNC_POLARITY_CFG; reg |= (dp->video_info.v_sync_polarity << 1); - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); - reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); reg &= ~HSYNC_POLARITY_CFG; reg |= (dp->video_info.h_sync_polarity << 0); - writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); reg = AUDIO_MODE_SPDIF_MODE | VIDEO_MODE_SLAVE_MODE; - writel(reg, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); + analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, reg); } void analogix_dp_enable_scrambling(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); + reg = analogix_dp_read(dp, ANALOGIX_DP_TRAINING_PTN_SET); reg &= ~SCRAMBLING_DISABLE; - writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); + analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); } void analogix_dp_disable_scrambling(struct analogix_dp_device *dp) { u32 reg; - reg = readl(dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); + reg = analogix_dp_read(dp, ANALOGIX_DP_TRAINING_PTN_SET); reg |= SCRAMBLING_DISABLE; - writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); + analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); } void analogix_dp_enable_psr_crc(struct analogix_dp_device *dp) { - writel(PSR_VID_CRC_ENABLE, dp->reg_base + ANALOGIX_DP_CRC_CON); + analogix_dp_write(dp, ANALOGIX_DP_CRC_CON, PSR_VID_CRC_ENABLE); } static ssize_t analogix_dp_get_psr_status(struct analogix_dp_device *dp) @@ -1044,44 +986,44 @@ int analogix_dp_send_psr_spd(struct analogix_dp_device *dp, ssize_t psr_status; /* don't send info frame */ - val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); + val = analogix_dp_read(dp, ANALOGIX_DP_PKT_SEND_CTL); val &= ~IF_EN; - writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); + analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, val); /* configure single frame update mode */ - writel(PSR_FRAME_UP_TYPE_BURST | PSR_CRC_SEL_HARDWARE, - dp->reg_base + ANALOGIX_DP_PSR_FRAME_UPDATE_CTRL); + analogix_dp_write(dp, ANALOGIX_DP_PSR_FRAME_UPDATE_CTRL, + PSR_FRAME_UP_TYPE_BURST | PSR_CRC_SEL_HARDWARE); /* configure VSC HB0~HB3 */ - writel(vsc->sdp_header.HB0, dp->reg_base + ANALOGIX_DP_SPD_HB0); - writel(vsc->sdp_header.HB1, dp->reg_base + ANALOGIX_DP_SPD_HB1); - writel(vsc->sdp_header.HB2, dp->reg_base + ANALOGIX_DP_SPD_HB2); - writel(vsc->sdp_header.HB3, dp->reg_base + ANALOGIX_DP_SPD_HB3); + analogix_dp_write(dp, ANALOGIX_DP_SPD_HB0, vsc->sdp_header.HB0); + analogix_dp_write(dp, ANALOGIX_DP_SPD_HB1, vsc->sdp_header.HB1); + analogix_dp_write(dp, ANALOGIX_DP_SPD_HB2, vsc->sdp_header.HB2); + analogix_dp_write(dp, ANALOGIX_DP_SPD_HB3, vsc->sdp_header.HB3); /* configure reused VSC PB0~PB3, magic number from vendor */ - writel(0x00, dp->reg_base + ANALOGIX_DP_SPD_PB0); - writel(0x16, dp->reg_base + ANALOGIX_DP_SPD_PB1); - writel(0xCE, dp->reg_base + ANALOGIX_DP_SPD_PB2); - writel(0x5D, dp->reg_base + ANALOGIX_DP_SPD_PB3); + analogix_dp_write(dp, ANALOGIX_DP_SPD_PB0, 0x00); + analogix_dp_write(dp, ANALOGIX_DP_SPD_PB1, 0x16); + analogix_dp_write(dp, ANALOGIX_DP_SPD_PB2, 0xCE); + analogix_dp_write(dp, ANALOGIX_DP_SPD_PB3, 0x5D); /* configure DB0 / DB1 values */ - writel(vsc->db[0], dp->reg_base + ANALOGIX_DP_VSC_SHADOW_DB0); - writel(vsc->db[1], dp->reg_base + ANALOGIX_DP_VSC_SHADOW_DB1); + analogix_dp_write(dp, ANALOGIX_DP_VSC_SHADOW_DB0, vsc->db[0]); + analogix_dp_write(dp, ANALOGIX_DP_VSC_SHADOW_DB1, vsc->db[1]); /* set reuse spd inforframe */ - val = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); + val = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_3); val |= REUSE_SPD_EN; - writel(val, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_3, val); /* mark info frame update */ - val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); + val = analogix_dp_read(dp, ANALOGIX_DP_PKT_SEND_CTL); val = (val | IF_UP) & ~IF_EN; - writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); + analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, val); /* send info frame */ - val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); + val = analogix_dp_read(dp, ANALOGIX_DP_PKT_SEND_CTL); val |= IF_EN; - writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); + analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, val); if (!blocking) return 0; @@ -1108,11 +1050,43 @@ int analogix_dp_send_psr_spd(struct analogix_dp_device *dp, return 0; } +void analogix_dp_phy_power_on(struct analogix_dp_device *dp) +{ + if (dp->phy_enabled) + return; + + phy_set_mode(dp->phy, PHY_MODE_DP); + phy_power_on(dp->phy); + + dp->phy_enabled = true; +} + +void analogix_dp_phy_power_off(struct analogix_dp_device *dp) +{ + if (!dp->phy_enabled) + return; + + phy_power_off(dp->phy); + + dp->phy_enabled = false; +} + +enum { + AUX_STATUS_OK, + AUX_STATUS_NACK_ERROR, + AUX_STATUS_TIMEOUT_ERROR, + AUX_STATUS_UNKNOWN_ERROR, + AUX_STATUS_MUCH_DEFER_ERROR, + AUX_STATUS_TX_SHORT_ERROR, + AUX_STATUS_RX_SHORT_ERROR, + AUX_STATUS_NACK_WITHOUT_M_ERROR, + AUX_STATUS_I2C_NACK_ERROR +}; + ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, struct drm_dp_aux_msg *msg) { u32 reg; - u32 status_reg; u8 *buffer = msg->buffer; unsigned int i; int num_transferred = 0; @@ -1122,9 +1096,15 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, if (WARN_ON(msg->size > 16)) return -E2BIG; + reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); + if (reg & AUX_FUNC_EN_N) { + analogix_dp_phy_power_on(dp); + analogix_dp_init_aux(dp); + } + /* Clear AUX CH data buffer */ reg = BUF_CLR; - writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL); + analogix_dp_write(dp, ANALOGIX_DP_BUFFER_DATA_CTL, reg); switch (msg->request & ~DP_AUX_I2C_MOT) { case DP_AUX_I2C_WRITE: @@ -1152,21 +1132,21 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, } reg |= AUX_LENGTH(msg->size); - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1); + analogix_dp_write(dp, ANALOGIX_DP_AUX_CH_CTL_1, reg); /* Select DPCD device address */ reg = AUX_ADDR_7_0(msg->address); - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0); + analogix_dp_write(dp, ANALOGIX_DP_AUX_ADDR_7_0, reg); reg = AUX_ADDR_15_8(msg->address); - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8); + analogix_dp_write(dp, ANALOGIX_DP_AUX_ADDR_15_8, reg); reg = AUX_ADDR_19_16(msg->address); - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16); + analogix_dp_write(dp, ANALOGIX_DP_AUX_ADDR_19_16, reg); if (!(msg->request & DP_AUX_I2C_READ)) { for (i = 0; i < msg->size; i++) { reg = buffer[i]; - writel(reg, dp->reg_base + ANALOGIX_DP_BUF_DATA_0 + - 4 * i); + analogix_dp_write(dp, ANALOGIX_DP_BUF_DATA_0 + 4 * i, + reg); num_transferred++; } } @@ -1178,7 +1158,7 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, if (msg->size < 1) reg |= ADDR_ONLY; - writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2); + analogix_dp_write(dp, ANALOGIX_DP_AUX_CH_CTL_2, reg); ret = readx_poll_timeout(readl, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2, reg, !(reg & AUX_EN), 25, 500 * 1000); @@ -1197,30 +1177,31 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, } /* Clear interrupt source for AUX CH command reply */ - writel(RPLY_RECEIV, dp->reg_base + ANALOGIX_DP_INT_STA); - - /* Clear interrupt source for AUX CH access error */ - reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); - status_reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_STA); - if ((reg & AUX_ERR) || (status_reg & AUX_STATUS_MASK)) { - writel(AUX_ERR, dp->reg_base + ANALOGIX_DP_INT_STA); + analogix_dp_write(dp, ANALOGIX_DP_INT_STA, RPLY_RECEIV); - dev_warn(dp->dev, "AUX CH error happened: %#x (%d)\n", - status_reg & AUX_STATUS_MASK, !!(reg & AUX_ERR)); - goto aux_error; - } + reg = analogix_dp_read(dp, ANALOGIX_DP_AUX_CH_STA); + if ((reg & AUX_STATUS_MASK) == AUX_STATUS_TIMEOUT_ERROR) + return -ETIMEDOUT; if (msg->request & DP_AUX_I2C_READ) { + size_t buf_data_count; + + reg = analogix_dp_read(dp, ANALOGIX_DP_BUFFER_DATA_CTL); + buf_data_count = BUF_DATA_COUNT(reg); + + if (buf_data_count != msg->size) + return -EBUSY; + for (i = 0; i < msg->size; i++) { - reg = readl(dp->reg_base + ANALOGIX_DP_BUF_DATA_0 + - 4 * i); + reg = analogix_dp_read(dp, ANALOGIX_DP_BUF_DATA_0 + + 4 * i); buffer[i] = (unsigned char)reg; num_transferred++; } } /* Check if Rx sends defer */ - reg = readl(dp->reg_base + ANALOGIX_DP_AUX_RX_COMM); + reg = analogix_dp_read(dp, ANALOGIX_DP_AUX_RX_COMM); if (reg == AUX_RX_COMM_AUX_DEFER) msg->reply = DP_AUX_NATIVE_REPLY_DEFER; else if (reg == AUX_RX_COMM_I2C_DEFER) @@ -1232,7 +1213,7 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, (msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_NATIVE_READ) msg->reply = DP_AUX_NATIVE_REPLY_ACK; - return num_transferred > 0 ? num_transferred : -EBUSY; + return (num_transferred == msg->size) ? num_transferred : -EBUSY; aux_error: /* if aux err happen, reset aux */ @@ -1240,3 +1221,119 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, return -EREMOTEIO; } + +void analogix_dp_set_video_format(struct analogix_dp_device *dp) +{ + struct video_info *video = &dp->video_info; + const struct drm_display_mode *mode = &video->mode; + unsigned int hsw, hfp, hbp, vsw, vfp, vbp; + + hsw = mode->hsync_end - mode->hsync_start; + hfp = mode->hsync_start - mode->hdisplay; + hbp = mode->htotal - mode->hsync_end; + vsw = mode->vsync_end - mode->vsync_start; + vfp = mode->vsync_start - mode->vdisplay; + vbp = mode->vtotal - mode->vsync_end; + + /* Set Video Format Parameters */ + analogix_dp_write(dp, ANALOGIX_DP_TOTAL_LINE_CFG_L, + TOTAL_LINE_CFG_L(mode->vtotal)); + analogix_dp_write(dp, ANALOGIX_DP_TOTAL_LINE_CFG_H, + TOTAL_LINE_CFG_H(mode->vtotal >> 8)); + analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_LINE_CFG_L, + ACTIVE_LINE_CFG_L(mode->vdisplay)); + analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_LINE_CFG_H, + ACTIVE_LINE_CFG_H(mode->vdisplay >> 8)); + analogix_dp_write(dp, ANALOGIX_DP_V_F_PORCH_CFG, + V_F_PORCH_CFG(vfp)); + analogix_dp_write(dp, ANALOGIX_DP_V_SYNC_WIDTH_CFG, + V_SYNC_WIDTH_CFG(vsw)); + analogix_dp_write(dp, ANALOGIX_DP_V_B_PORCH_CFG, + V_B_PORCH_CFG(vbp)); + analogix_dp_write(dp, ANALOGIX_DP_TOTAL_PIXEL_CFG_L, + TOTAL_PIXEL_CFG_L(mode->htotal)); + analogix_dp_write(dp, ANALOGIX_DP_TOTAL_PIXEL_CFG_H, + TOTAL_PIXEL_CFG_H(mode->htotal >> 8)); + analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_PIXEL_CFG_L, + ACTIVE_PIXEL_CFG_L(mode->hdisplay)); + analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_PIXEL_CFG_H, + ACTIVE_PIXEL_CFG_H(mode->hdisplay >> 8)); + analogix_dp_write(dp, ANALOGIX_DP_H_F_PORCH_CFG_L, + H_F_PORCH_CFG_L(hfp)); + analogix_dp_write(dp, ANALOGIX_DP_H_F_PORCH_CFG_H, + H_F_PORCH_CFG_H(hfp >> 8)); + analogix_dp_write(dp, ANALOGIX_DP_H_SYNC_CFG_L, + H_SYNC_CFG_L(hsw)); + analogix_dp_write(dp, ANALOGIX_DP_H_SYNC_CFG_H, + H_SYNC_CFG_H(hsw >> 8)); + analogix_dp_write(dp, ANALOGIX_DP_H_B_PORCH_CFG_L, + H_B_PORCH_CFG_L(hbp)); + analogix_dp_write(dp, ANALOGIX_DP_H_B_PORCH_CFG_H, + H_B_PORCH_CFG_H(hbp >> 8)); +} + +void analogix_dp_video_bist_enable(struct analogix_dp_device *dp) +{ + u32 reg; + + /* Enable Video BIST */ + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_4, BIST_EN); + + /* + * Note that if BIST_EN is set to 1, F_SEL must be cleared to 0 + * although video format information comes from registers set by user. + */ + reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); + reg &= ~FORMAT_SEL; + analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); +} + +void analogix_dp_audio_config_i2s(struct analogix_dp_device *dp) +{ + u32 reg; + + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); + reg &= ~FIX_M_AUD; + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); + + reg = analogix_dp_read(dp, ANALOGIX_DP_I2S_CTRL); + reg |= I2S_EN; + analogix_dp_write(dp, ANALOGIX_DP_I2S_CTRL, reg); +} + +void analogix_dp_audio_config_spdif(struct analogix_dp_device *dp) +{ + u32 reg; + + reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); + reg &= ~FIX_M_AUD; + analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); + + reg = analogix_dp_read(dp, ANALOGIX_DP_SPDIF_AUDIO_CTL_0); + reg |= AUD_SPDIF_EN; + analogix_dp_write(dp, ANALOGIX_DP_SPDIF_AUDIO_CTL_0, reg); +} + +void analogix_dp_audio_enable(struct analogix_dp_device *dp) +{ + u32 reg; + + reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); + reg &= ~(AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N); + analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); + + reg = analogix_dp_read(dp, ANALOGIX_DP_AUD_CTL); + reg |= MISC_CTRL_RESET | DP_AUDIO_EN; + analogix_dp_write(dp, ANALOGIX_DP_AUD_CTL, reg); +} + +void analogix_dp_audio_disable(struct analogix_dp_device *dp) +{ + u32 reg; + + analogix_dp_write(dp, ANALOGIX_DP_AUD_CTL, 0); + + reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); + reg |= AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N; + analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); +} diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h index e284ee8da58b..df88f1ad0883 100644 --- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h +++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h @@ -15,9 +15,27 @@ #define ANALOGIX_DP_VIDEO_CTL_1 0x20 #define ANALOGIX_DP_VIDEO_CTL_2 0x24 #define ANALOGIX_DP_VIDEO_CTL_3 0x28 +#define ANALOGIX_DP_VIDEO_CTL_4 0x2C #define ANALOGIX_DP_VIDEO_CTL_8 0x3C #define ANALOGIX_DP_VIDEO_CTL_10 0x44 +#define ANALOGIX_DP_TOTAL_LINE_CFG_L 0x48 +#define ANALOGIX_DP_TOTAL_LINE_CFG_H 0x4C +#define ANALOGIX_DP_ACTIVE_LINE_CFG_L 0x50 +#define ANALOGIX_DP_ACTIVE_LINE_CFG_H 0x54 +#define ANALOGIX_DP_V_F_PORCH_CFG 0x58 +#define ANALOGIX_DP_V_SYNC_WIDTH_CFG 0x5C +#define ANALOGIX_DP_V_B_PORCH_CFG 0x60 +#define ANALOGIX_DP_TOTAL_PIXEL_CFG_L 0x64 +#define ANALOGIX_DP_TOTAL_PIXEL_CFG_H 0x68 +#define ANALOGIX_DP_ACTIVE_PIXEL_CFG_L 0x6C +#define ANALOGIX_DP_ACTIVE_PIXEL_CFG_H 0x70 +#define ANALOGIX_DP_H_F_PORCH_CFG_L 0x74 +#define ANALOGIX_DP_H_F_PORCH_CFG_H 0x78 +#define ANALOGIX_DP_H_SYNC_CFG_L 0x7C +#define ANALOGIX_DP_H_SYNC_CFG_H 0x80 +#define ANALOGIX_DP_H_B_PORCH_CFG_L 0x84 +#define ANALOGIX_DP_H_B_PORCH_CFG_H 0x88 #define ANALOGIX_DP_SPDIF_AUDIO_CTL_0 0xD8 @@ -27,6 +45,8 @@ #define ANALOGIX_DP_PLL_REG_4 0x9ec #define ANALOGIX_DP_PLL_REG_5 0xa00 +#define ANALOIGX_DP_SSC_REG 0x104 +#define ANALOGIX_DP_BIAS 0x124 #define ANALOGIX_DP_PD 0x12c #define ANALOGIX_DP_IF_TYPE 0x244 @@ -70,7 +90,7 @@ #define ANALOGIX_DP_SYS_CTL_2 0x604 #define ANALOGIX_DP_SYS_CTL_3 0x608 #define ANALOGIX_DP_SYS_CTL_4 0x60C - +#define ANALOGIX_DP_AUD_CTL 0x618 #define ANALOGIX_DP_PKT_SEND_CTL 0x640 #define ANALOGIX_DP_HDCP_CTL 0x648 @@ -116,8 +136,9 @@ #define ANALOGIX_DP_BUF_DATA_0 0x7C0 #define ANALOGIX_DP_SOC_GENERAL_CTL 0x800 - +#define ANALOGIX_DP_AUD_CHANNEL_CTL 0x834 #define ANALOGIX_DP_CRC_CON 0x890 +#define ANALOGIX_DP_I2S_CTRL 0x9C8 /* ANALOGIX_DP_TX_SW_RESET */ #define RESET_DP_TX (0x1 << 0) @@ -171,6 +192,11 @@ #define VID_CHK_UPDATE_TYPE_0 (0x0 << 4) #define REUSE_SPD_EN (0x1 << 3) +/* ANALOGIX_DP_VIDEO_CTL_4 */ +#define BIST_EN (0x1 << 3) +#define BIST_WIDTH(x) (((x) & 0x1) << 2) +#define BIST_TYPE(x) (((x) & 0x3) << 0) + /* ANALOGIX_DP_VIDEO_CTL_8 */ #define VID_HRES_TH(x) (((x) & 0xf) << 4) #define VID_VRES_TH(x) (((x) & 0xf) << 0) @@ -181,6 +207,60 @@ #define VSYNC_POLARITY_CFG (0x1 << 1) #define HSYNC_POLARITY_CFG (0x1 << 0) +/* ANALOGIX_DP_TOTAL_LINE_CFG_L */ +#define TOTAL_LINE_CFG_L(x) (((x) & 0xff) << 0) + +/* ANALOGIX_DP_TOTAL_LINE_CFG_H */ +#define TOTAL_LINE_CFG_H(x) (((x) & 0xf) << 0) + +/* ANALOGIX_DP_ACTIVE_LINE_CFG_L */ +#define ACTIVE_LINE_CFG_L(x) (((x) & 0xff) << 0) + +/* ANALOGIX_DP_ACTIVE_LINE_CFG_H */ +#define ACTIVE_LINE_CFG_H(x) (((x) & 0xf) << 0) + +/* ANALOGIX_DP_V_F_PORCH_CFG */ +#define V_F_PORCH_CFG(x) (((x) & 0xff) << 0) + +/* ANALOGIX_DP_V_SYNC_WIDTH_CFG */ +#define V_SYNC_WIDTH_CFG(x) (((x) & 0xff) << 0) + +/* ANALOGIX_DP_V_B_PORCH_CFG */ +#define V_B_PORCH_CFG(x) (((x) & 0xff) << 0) + +/* ANALOGIX_DP_TOTAL_PIXEL_CFG_L */ +#define TOTAL_PIXEL_CFG_L(x) (((x) & 0xff) << 0) + +/* ANALOGIX_DP_TOTAL_PIXEL_CFG_H */ +#define TOTAL_PIXEL_CFG_H(x) (((x) & 0x3f) << 0) + +/* ANALOGIX_DP_ACTIVE_PIXEL_CFG_L */ +#define ACTIVE_PIXEL_CFG_L(x) (((x) & 0xff) << 0) + +/* ANALOGIX_DP_ACTIVE_PIXEL_CFG_H */ +#define ACTIVE_PIXEL_CFG_H(x) (((x) & 0x3f) << 0) + +/* ANALOGIX_DP_H_F_PORCH_CFG_L */ +#define H_F_PORCH_CFG_L(x) (((x) & 0xff) << 0) + +/* ANALOGIX_DP_H_F_PORCH_CFG_H */ +#define H_F_PORCH_CFG_H(x) (((x) & 0xf) << 0) + +/* ANALOGIX_DP_H_SYNC_CFG_L */ +#define H_SYNC_CFG_L(x) (((x) & 0xff) << 0) + +/* ANALOGIX_DP_H_SYNC_CFG_H */ +#define H_SYNC_CFG_H(x) (((x) & 0xf) << 0) + +/* ANALOGIX_DP_H_B_PORCH_CFG_L */ +#define H_B_PORCH_CFG_L(x) (((x) & 0xff) << 0) + +/* ANALOGIX_DP_H_B_PORCH_CFG_H */ +#define H_B_PORCH_CFG_H(x) (((x) & 0xf) << 0) + +/* ANALOGIX_DP_SPDIF_AUDIO_CTL_0 */ +#define AUD_SPDIF_EN (0x1 << 7) + /* ANALOGIX_DP_PLL_REG_1 */ #define REF_CLK_24M (0x1 << 0) #define REF_CLK_27M (0x0 << 0) @@ -309,6 +389,10 @@ #define FIX_M_VID (0x1 << 2) #define M_VID_UPDATE_CTRL (0x3 << 0) +/* ANALOGIX_DP_AUD_CTL */ +#define MISC_CTRL_RESET (0x1 << 4) +#define DP_AUDIO_EN (0x1 << 0) + /* ANALOGIX_DP_TRAINING_PTN_SET */ #define SCRAMBLER_TYPE (0x1 << 9) #define HW_LINK_TRAINING_PATTERN (0x1 << 8) @@ -319,6 +403,7 @@ #define LINK_QUAL_PATTERN_SET_D10_2 (0x1 << 2) #define LINK_QUAL_PATTERN_SET_DISABLE (0x0 << 2) #define SW_TRAINING_PATTERN_SET_MASK (0x3 << 0) +#define SW_TRAINING_PATTERN_SET_PTN3 (0x3 << 0) #define SW_TRAINING_PATTERN_SET_PTN2 (0x2 << 0) #define SW_TRAINING_PATTERN_SET_PTN1 (0x1 << 0) #define SW_TRAINING_PATTERN_SET_NORMAL (0x0 << 0) @@ -406,6 +491,11 @@ #define VIDEO_MODE_SLAVE_MODE (0x1 << 0) #define VIDEO_MODE_MASTER_MODE (0x0 << 0) +/* ANALOGIX_DP_AUD_CHANNEL_CTL */ +#define AUD_CHANNEL_COUNT_6 (0x5 << 0) +#define AUD_CHANNEL_COUNT_4 (0x3 << 0) +#define AUD_CHANNEL_COUNT_2 (0x1 << 0) + /* ANALOGIX_DP_PKT_SEND_CTL */ #define IF_UP (0x1 << 4) #define IF_EN (0x1 << 0) @@ -414,4 +504,7 @@ #define PSR_VID_CRC_FLUSH (0x1 << 2) #define PSR_VID_CRC_ENABLE (0x1 << 0) +/* ANALOGIX_DP_I2S_CTRL */ +#define I2S_EN (0x1 << 4) + #endif /* _ANALOGIX_DP_REG_H */ diff --git a/drivers/gpu/drm/bridge/synopsys/Makefile b/drivers/gpu/drm/bridge/synopsys/Makefile index 91d746ad5de1..ea11fa1d4800 100644 --- a/drivers/gpu/drm/bridge/synopsys/Makefile +++ b/drivers/gpu/drm/bridge/synopsys/Makefile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_DRM_DW_HDMI) += dw-hdmi.o +obj-$(CONFIG_DRM_DW_HDMI) += dw-hdmi.o dw-hdmi-hdcp.o \ + dw-hdmi-qp.o obj-$(CONFIG_DRM_DW_HDMI_AHB_AUDIO) += dw-hdmi-ahb-audio.o -obj-$(CONFIG_DRM_DW_HDMI_I2S_AUDIO) += dw-hdmi-i2s-audio.o +obj-$(CONFIG_DRM_DW_HDMI_I2S_AUDIO) += dw-hdmi-i2s-audio.o dw-hdmi-qp-i2s-audio.o obj-$(CONFIG_DRM_DW_HDMI_CEC) += dw-hdmi-cec.o obj-$(CONFIG_DRM_DW_MIPI_DSI) += dw-mipi-dsi.o diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c index 70ab4fbdc23e..48fc36d56bc2 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -262,6 +263,8 @@ static int dw_hdmi_cec_probe(struct platform_device *pdev) if (IS_ERR(cec->adap)) return PTR_ERR(cec->adap); + dw_hdmi_set_cec_adap(cec->hdmi, cec->adap); + /* override the module pointer */ cec->adap->owner = THIS_MODULE; diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.c new file mode 100755 index 000000000000..24aab5043e61 --- /dev/null +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.c @@ -0,0 +1,748 @@ +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author Huicong Xu + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dw-hdmi.h" +#include "dw-hdmi-hdcp.h" + +#define HDCP_KEY_SIZE 308 +#define HDCP_KEY_SEED_SIZE 2 + +#define KSV_LEN 5 +#define HEADER 10 +#define SHAMAX 20 + +#define MAX_DOWNSTREAM_DEVICE_NUM 5 +#define DPK_WR_OK_TIMEOUT_US 30000 +#define HDMI_HDCP1X_ID 5 + +/* HDCP Registers */ +#define HDMI_HDCPREG_RMCTL 0x780e +#define HDMI_HDCPREG_RMSTS 0x780f +#define HDMI_HDCPREG_SEED0 0x7810 +#define HDMI_HDCPREG_SEED1 0x7811 +#define HDMI_HDCPREG_DPK0 0x7812 +#define HDMI_HDCPREG_DPK1 0x7813 +#define HDMI_HDCPREG_DPK2 0x7814 +#define HDMI_HDCPREG_DPK3 0x7815 +#define HDMI_HDCPREG_DPK4 0x7816 +#define HDMI_HDCPREG_DPK5 0x7817 +#define HDMI_HDCPREG_DPK6 0x7818 +#define HDMI_HDCP2REG_CTRL 0x7904 +#define HDMI_HDCP2REG_MASK 0x790c +#define HDMI_HDCP2REG_MUTE 0x790e + +enum dw_hdmi_hdcp_state { + DW_HDCP_DISABLED, + DW_HDCP_AUTH_START, + DW_HDCP_AUTH_SUCCESS, + DW_HDCP_AUTH_FAIL, +}; + +enum { + DW_HDMI_HDCP_KSV_LEN = 8, + DW_HDMI_HDCP_SHA_LEN = 20, + DW_HDMI_HDCP_DPK_LEN = 280, + DW_HDMI_HDCP_KEY_LEN = 308, + DW_HDMI_HDCP_SEED_LEN = 2, +}; + +enum { + HDMI_MC_CLKDIS_HDCPCLK_MASK = 0x40, + HDMI_MC_CLKDIS_HDCPCLK_ENABLE = 0x00, + + HDMI_A_SRMCTRL_SHA1_FAIL_MASK = 0X08, + HDMI_A_SRMCTRL_SHA1_FAIL_DISABLE = 0X00, + HDMI_A_SRMCTRL_SHA1_FAIL_ENABLE = 0X08, + + HDMI_A_SRMCTRL_KSV_UPDATE_MASK = 0X04, + HDMI_A_SRMCTRL_KSV_UPDATE_DISABLE = 0X00, + HDMI_A_SRMCTRL_KSV_UPDATE_ENABLE = 0X04, + + HDMI_A_SRMCTRL_KSV_MEM_REQ_MASK = 0X01, + HDMI_A_SRMCTRL_KSV_MEM_REQ_DISABLE = 0X00, + HDMI_A_SRMCTRL_KSV_MEM_REQ_ENABLE = 0X01, + + HDMI_A_SRMCTRL_KSV_MEM_ACCESS_MASK = 0X02, + HDMI_A_SRMCTRL_KSV_MEM_ACCESS_DISABLE = 0X00, + HDMI_A_SRMCTRL_KSV_MEM_ACCESS_ENABLE = 0X02, + + HDMI_A_SRM_BASE_MAX_DEVS_EXCEEDED = 0x80, + HDMI_A_SRM_BASE_DEVICE_COUNT = 0x7f, + + HDMI_A_SRM_BASE_MAX_CASCADE_EXCEEDED = 0x08, + + HDMI_A_APIINTSTAT_KSVSHA1_CALC_INT = 0x02, + + /* HDCPREG_RMSTS field values */ + DPK_WR_OK_STS = 0x40, + + HDMI_A_HDCP22_MASK = 0x40, + + HDMI_HDCP2_OVR_EN_MASK = 0x02, + HDMI_HDCP2_OVR_ENABLE = 0x02, + HDMI_HDCP2_OVR_DISABLE = 0x00, + + HDMI_HDCP2_FORCE_MASK = 0x04, + HDMI_HDCP2_FORCE_ENABLE = 0x04, + HDMI_HDCP2_FORCE_DISABLE = 0x00, +}; + +struct sha_t { + u8 mlength[8]; + u8 mblock[64]; + int mindex; + int mcomputed; + int mcorrupted; + unsigned int mdigest[5]; +}; + +static struct dw_hdcp *g_hdcp; + +static inline unsigned int shacircularshift(unsigned int bits, + unsigned int word) +{ + return (((word << bits) & 0xFFFFFFFF) | (word >> (32 - bits))); +} + +static void hdcp_modb(struct dw_hdcp *hdcp, u8 data, u8 mask, unsigned int reg) +{ + struct dw_hdmi *hdmi = hdcp->hdmi; + u8 val = hdcp->read(hdmi, reg) & ~mask; + + val |= data & mask; + hdcp->write(hdmi, val, reg); +} + +static void sha_reset(struct sha_t *sha) +{ + u32 i = 0; + + sha->mindex = 0; + sha->mcomputed = false; + sha->mcorrupted = false; + for (i = 0; i < sizeof(sha->mlength); i++) + sha->mlength[i] = 0; + + sha1_init(sha->mdigest); +} + +static void sha_processblock(struct sha_t *sha) +{ + u32 array[SHA1_WORKSPACE_WORDS]; + + sha1_transform(sha->mdigest, sha->mblock, array); + sha->mindex = 0; +} + +static void sha_padmessage(struct sha_t *sha) +{ + /* + * Check to see if the current message block is too small to hold + * the initial padding bits and length. If so, we will pad the + * block, process it, and then continue padding into a second + * block. + */ + if (sha->mindex > 55) { + sha->mblock[sha->mindex++] = 0x80; + while (sha->mindex < 64) + sha->mblock[sha->mindex++] = 0; + + sha_processblock(sha); + while (sha->mindex < 56) + sha->mblock[sha->mindex++] = 0; + } else { + sha->mblock[sha->mindex++] = 0x80; + while (sha->mindex < 56) + sha->mblock[sha->mindex++] = 0; + } + + /* Store the message length as the last 8 octets */ + sha->mblock[56] = sha->mlength[7]; + sha->mblock[57] = sha->mlength[6]; + sha->mblock[58] = sha->mlength[5]; + sha->mblock[59] = sha->mlength[4]; + sha->mblock[60] = sha->mlength[3]; + sha->mblock[61] = sha->mlength[2]; + sha->mblock[62] = sha->mlength[1]; + sha->mblock[63] = sha->mlength[0]; + + sha_processblock(sha); +} + +static int sha_result(struct sha_t *sha) +{ + if (sha->mcorrupted) + return false; + + if (sha->mcomputed == 0) { + sha_padmessage(sha); + sha->mcomputed = true; + } + return true; +} + +static void sha_input(struct sha_t *sha, const u8 *data, u32 size) +{ + int i = 0; + unsigned int j = 0; + int rc = true; + + if (data == 0 || size == 0) + return; + + if (sha->mcomputed || sha->mcorrupted) { + sha->mcorrupted = true; + return; + } + while (size-- && !sha->mcorrupted) { + sha->mblock[sha->mindex++] = *data; + + for (i = 0; i < 8; i++) { + rc = true; + for (j = 0; j < sizeof(sha->mlength); j++) { + sha->mlength[j]++; + if (sha->mlength[j] != 0) { + rc = false; + break; + } + } + sha->mcorrupted = (sha->mcorrupted || + rc) ? true : false; + } + /* if corrupted then message is too long */ + if (sha->mindex == 64) + sha_processblock(sha); + data++; + } +} + +static int hdcp_verify_ksv(const u8 *data, u32 size) +{ + u32 i = 0; + struct sha_t sha; + + if ((!data) || (size < (HEADER + SHAMAX))) + return false; + + sha_reset(&sha); + sha_input(&sha, data, size - SHAMAX); + if (sha_result(&sha) == false) + return false; + + for (i = 0; i < SHAMAX; i++) { + if (data[size - SHAMAX + i] != (u8)(sha.mdigest[i / 4] + >> ((i % 4) * 8))) + return false; + } + return true; +} + +static int hdcp_load_keys_cb(struct dw_hdcp *hdcp) +{ + u32 size; + u8 hdcp_vendor_data[320]; + + hdcp->keys = kmalloc(HDCP_KEY_SIZE, GFP_KERNEL); + if (!hdcp->keys) + return -ENOMEM; + + hdcp->seeds = kmalloc(HDCP_KEY_SEED_SIZE, GFP_KERNEL); + if (!hdcp->seeds) { + kfree(hdcp->keys); + return -ENOMEM; + } + + size = rk_vendor_read(HDMI_HDCP1X_ID, hdcp_vendor_data, 314); + if (size < (HDCP_KEY_SIZE + HDCP_KEY_SEED_SIZE)) { + dev_dbg(hdcp->dev, "HDCP: read size %d\n", size); + memset(hdcp->keys, 0, HDCP_KEY_SIZE); + memset(hdcp->seeds, 0, HDCP_KEY_SEED_SIZE); + } else { + memcpy(hdcp->keys, hdcp_vendor_data, HDCP_KEY_SIZE); + memcpy(hdcp->seeds, hdcp_vendor_data + HDCP_KEY_SIZE, + HDCP_KEY_SEED_SIZE); + } + return 0; +} + +static int dw_hdmi_hdcp_load_key(struct dw_hdcp *hdcp) +{ + int i, j; + int ret, val; + void __iomem *reg_rmsts_addr; + struct hdcp_keys *hdcp_keys; + struct dw_hdmi *hdmi = hdcp->hdmi; + + if (!hdcp->keys) { + ret = hdcp_load_keys_cb(hdcp); + if (ret) + return ret; + } + hdcp_keys = hdcp->keys; + + if (hdcp->reg_io_width == 4) + reg_rmsts_addr = hdcp->regs + (HDMI_HDCPREG_RMSTS << 2); + else if (hdcp->reg_io_width == 1) + reg_rmsts_addr = hdcp->regs + HDMI_HDCPREG_RMSTS; + else + return -EPERM; + + /* Disable decryption logic */ + hdcp->write(hdmi, 0, HDMI_HDCPREG_RMCTL); + ret = readx_poll_timeout(readl, reg_rmsts_addr, val, + val & DPK_WR_OK_STS, 1000, + DPK_WR_OK_TIMEOUT_US); + if (ret) + return ret; + hdcp->write(hdmi, 0, HDMI_HDCPREG_DPK6); + hdcp->write(hdmi, 0, HDMI_HDCPREG_DPK5); + + /* The useful data in ksv should be 5 byte */ + for (i = 4; i >= 0; i--) + hdcp->write(hdmi, hdcp_keys->KSV[i], HDMI_HDCPREG_DPK0 + i); + ret = readx_poll_timeout(readl, reg_rmsts_addr, val, + val & DPK_WR_OK_STS, 1000, + DPK_WR_OK_TIMEOUT_US); + + if (ret) + return ret; + + /* Enable decryption logic */ + if (hdcp->seeds) { + hdcp->write(hdmi, 1, HDMI_HDCPREG_RMCTL); + hdcp->write(hdmi, hdcp->seeds[0], HDMI_HDCPREG_SEED1); + hdcp->write(hdmi, hdcp->seeds[1], HDMI_HDCPREG_SEED0); + } else { + hdcp->write(hdmi, 0, HDMI_HDCPREG_RMCTL); + } + + /* Write encrypt device private key */ + for (i = 0; i < DW_HDMI_HDCP_DPK_LEN - 6; i += 7) { + for (j = 6; j >= 0; j--) + hdcp->write(hdmi, hdcp_keys->devicekey[i + j], + HDMI_HDCPREG_DPK0 + j); + ret = readx_poll_timeout(readl, reg_rmsts_addr, val, + val & DPK_WR_OK_STS, 1000, + DPK_WR_OK_TIMEOUT_US); + + if (ret) + return ret; + } + return 0; +} + +static int dw_hdmi_hdcp_start(struct dw_hdcp *hdcp) +{ + struct dw_hdmi *hdmi = hdcp->hdmi; + + if (!hdcp->enable) + return -EPERM; + + if (!(hdcp->read(hdmi, HDMI_HDCPREG_RMSTS) & 0x3f)) + dw_hdmi_hdcp_load_key(hdcp); + + hdcp_modb(hdcp, HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE, + HDMI_FC_INVIDCONF_HDCP_KEEPOUT_MASK, + HDMI_FC_INVIDCONF); + + hdcp->remaining_times = hdcp->retry_times; + if (hdcp->read(hdmi, HDMI_CONFIG1_ID) & HDMI_A_HDCP22_MASK) { + if (hdcp->hdcp2_enable == 0) { + hdcp_modb(hdcp, HDMI_HDCP2_OVR_ENABLE | + HDMI_HDCP2_FORCE_DISABLE, + HDMI_HDCP2_OVR_EN_MASK | + HDMI_HDCP2_FORCE_MASK, + HDMI_HDCP2REG_CTRL); + hdcp->write(hdmi, 0xff, HDMI_HDCP2REG_MASK); + hdcp->write(hdmi, 0xff, HDMI_HDCP2REG_MUTE); + } else { + hdcp_modb(hdcp, HDMI_HDCP2_OVR_DISABLE | + HDMI_HDCP2_FORCE_DISABLE, + HDMI_HDCP2_OVR_EN_MASK | + HDMI_HDCP2_FORCE_MASK, + HDMI_HDCP2REG_CTRL); + hdcp->write(hdmi, 0x00, HDMI_HDCP2REG_MASK); + hdcp->write(hdmi, 0x00, HDMI_HDCP2REG_MUTE); + } + } + + hdcp->write(hdmi, 0x40, HDMI_A_OESSWCFG); + hdcp_modb(hdcp, HDMI_A_HDCPCFG0_BYPENCRYPTION_DISABLE | + HDMI_A_HDCPCFG0_EN11FEATURE_DISABLE | + HDMI_A_HDCPCFG0_SYNCRICHECK_ENABLE, + HDMI_A_HDCPCFG0_BYPENCRYPTION_MASK | + HDMI_A_HDCPCFG0_EN11FEATURE_MASK | + HDMI_A_HDCPCFG0_SYNCRICHECK_MASK, HDMI_A_HDCPCFG0); + + hdcp_modb(hdcp, HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_ENABLE | + HDMI_A_HDCPCFG1_PH2UPSHFTENC_ENABLE, + HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK | + HDMI_A_HDCPCFG1_PH2UPSHFTENC_MASK, HDMI_A_HDCPCFG1); + + /* Reset HDCP Engine */ + if (hdcp->read(hdmi, HDMI_MC_CLKDIS) & HDMI_MC_CLKDIS_HDCPCLK_MASK) { + hdcp_modb(hdcp, HDMI_A_HDCPCFG1_SWRESET_ASSERT, + HDMI_A_HDCPCFG1_SWRESET_MASK, HDMI_A_HDCPCFG1); + } + + hdcp->write(hdmi, 0x00, HDMI_A_APIINTMSK); + hdcp_modb(hdcp, HDMI_A_HDCPCFG0_RXDETECT_ENABLE, + HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0); + + /* + * XXX: to sleep 100ms here between output hdmi and enable hdcpclk, + * otherwise hdcp auth fail when Connect to repeater + */ + msleep(100); + hdcp_modb(hdcp, HDMI_MC_CLKDIS_HDCPCLK_ENABLE, + HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS); + + hdcp->status = DW_HDCP_AUTH_START; + dev_dbg(hdcp->dev, "%s success\n", __func__); + return 0; +} + +static int dw_hdmi_hdcp_stop(struct dw_hdcp *hdcp) +{ + struct dw_hdmi *hdmi = hdcp->hdmi; + + if (!hdcp->enable) + return -EPERM; + + hdcp_modb(hdcp, HDMI_MC_CLKDIS_HDCPCLK_DISABLE, + HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS); + hdcp->write(hdmi, 0xff, HDMI_A_APIINTMSK); + + hdcp_modb(hdcp, HDMI_A_HDCPCFG0_RXDETECT_DISABLE, + HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0); + + hdcp_modb(hdcp, HDMI_A_SRMCTRL_SHA1_FAIL_DISABLE | + HDMI_A_SRMCTRL_KSV_UPDATE_DISABLE, + HDMI_A_SRMCTRL_SHA1_FAIL_MASK | + HDMI_A_SRMCTRL_KSV_UPDATE_MASK, HDMI_A_SRMCTRL); + + hdcp->status = DW_HDCP_DISABLED; + return 0; +} + +static int dw_hdmi_hdcp_ksvsha1(struct dw_hdcp *hdcp) +{ + int rc = 0, value, list, i; + char bstaus0, bstaus1; + char *ksvlistbuf; + struct dw_hdmi *hdmi = hdcp->hdmi; + + hdcp_modb(hdcp, HDMI_A_SRMCTRL_KSV_MEM_REQ_ENABLE, + HDMI_A_SRMCTRL_KSV_MEM_REQ_MASK, HDMI_A_SRMCTRL); + + list = 20; + do { + value = hdcp->read(hdmi, HDMI_A_SRMCTRL); + usleep_range(500, 1000); + } while ((value & HDMI_A_SRMCTRL_KSV_MEM_ACCESS_MASK) == 0 && --list); + + if ((value & HDMI_A_SRMCTRL_KSV_MEM_ACCESS_MASK) == 0) { + dev_err(hdcp->dev, "KSV memory can not access\n"); + rc = -EPERM; + goto out; + } + + hdcp->read(hdmi, HDMI_A_SRM_BASE); + bstaus0 = hdcp->read(hdmi, HDMI_A_SRM_BASE + 1); + bstaus1 = hdcp->read(hdmi, HDMI_A_SRM_BASE + 2); + + if (bstaus0 & HDMI_A_SRM_BASE_MAX_DEVS_EXCEEDED) { + dev_err(hdcp->dev, "MAX_DEVS_EXCEEDED\n"); + rc = -EPERM; + goto out; + } + + list = bstaus0 & HDMI_A_SRM_BASE_DEVICE_COUNT; + if (list > MAX_DOWNSTREAM_DEVICE_NUM) { + dev_err(hdcp->dev, "MAX_DOWNSTREAM_DEVICE_NUM\n"); + rc = -EPERM; + goto out; + } + if (bstaus1 & HDMI_A_SRM_BASE_MAX_CASCADE_EXCEEDED) { + dev_err(hdcp->dev, "MAX_CASCADE_EXCEEDED\n"); + rc = -EPERM; + goto out; + } + + value = (list * KSV_LEN) + HEADER + SHAMAX; + ksvlistbuf = kmalloc(value, GFP_KERNEL); + if (!ksvlistbuf) { + rc = -ENOMEM; + goto out; + } + + ksvlistbuf[(list * KSV_LEN)] = bstaus0; + ksvlistbuf[(list * KSV_LEN) + 1] = bstaus1; + for (i = 2; i < value; i++) { + if (i < HEADER) /* BSTATUS & M0 */ + ksvlistbuf[(list * KSV_LEN) + i] = + hdcp->read(hdmi, HDMI_A_SRM_BASE + i + 1); + else if (i < (HEADER + (list * KSV_LEN))) /* KSV list */ + ksvlistbuf[i - HEADER] = + hdcp->read(hdmi, HDMI_A_SRM_BASE + i + 1); + else /* SHA */ + ksvlistbuf[i] = + hdcp->read(hdmi, HDMI_A_SRM_BASE + i + 1); + } + if (hdcp_verify_ksv(ksvlistbuf, value) == true) { + rc = 0; + dev_dbg(hdcp->dev, "ksv check valid\n"); + } else { + dev_err(hdcp->dev, "ksv check invalid\n"); + rc = -1; + } + kfree(ksvlistbuf); +out: + hdcp_modb(hdcp, HDMI_A_SRMCTRL_KSV_MEM_REQ_DISABLE, + HDMI_A_SRMCTRL_KSV_MEM_REQ_MASK, HDMI_A_SRMCTRL); + return rc; +} + +static void dw_hdmi_hdcp_2nd_auth(struct dw_hdcp *hdcp) +{ + if (dw_hdmi_hdcp_ksvsha1(hdcp)) + hdcp_modb(hdcp, HDMI_A_SRMCTRL_SHA1_FAIL_ENABLE | + HDMI_A_SRMCTRL_KSV_UPDATE_ENABLE, + HDMI_A_SRMCTRL_SHA1_FAIL_MASK | + HDMI_A_SRMCTRL_KSV_UPDATE_MASK, HDMI_A_SRMCTRL); + else + hdcp_modb(hdcp, HDMI_A_SRMCTRL_SHA1_FAIL_DISABLE | + HDMI_A_SRMCTRL_KSV_UPDATE_ENABLE, + HDMI_A_SRMCTRL_SHA1_FAIL_MASK | + HDMI_A_SRMCTRL_KSV_UPDATE_MASK, HDMI_A_SRMCTRL); +} + +static void dw_hdmi_hdcp_isr(struct dw_hdcp *hdcp, int hdcp_int) +{ + dev_dbg(hdcp->dev, "hdcp_int is 0x%02x\n", hdcp_int); + if (hdcp_int & HDMI_A_APIINTSTAT_KSVSHA1_CALC_INT) { + dev_dbg(hdcp->dev, "hdcp sink is a repeater\n"); + dw_hdmi_hdcp_2nd_auth(hdcp); + } + if (hdcp_int & 0x40) { + hdcp->status = DW_HDCP_AUTH_FAIL; + if (hdcp->remaining_times > 1) + hdcp->remaining_times--; + else if (hdcp->remaining_times == 1) + hdcp_modb(hdcp, + HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_DISABLE, + HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK, + HDMI_A_HDCPCFG1); + } + if (hdcp_int & 0x80) { + dev_dbg(hdcp->dev, "hdcp auth success\n"); + hdcp->status = DW_HDCP_AUTH_SUCCESS; + } +} + +static ssize_t hdcp_enable_read(struct device *device, + struct device_attribute *attr, char *buf) +{ + bool enable = 0; + struct dw_hdcp *hdcp = g_hdcp; + + if (hdcp) + enable = hdcp->enable; + + return snprintf(buf, PAGE_SIZE, "%d\n", enable); +} + +static ssize_t hdcp_enable_write(struct device *device, + struct device_attribute *attr, + const char *buf, size_t count) +{ + bool enable; + struct dw_hdcp *hdcp = g_hdcp; + + if (!hdcp) + return -EINVAL; + + if (kstrtobool(buf, &enable)) + return -EINVAL; + + if (hdcp->enable != enable) { + if (enable) { + hdcp->enable = enable; + if (hdcp->read(hdcp->hdmi, HDMI_PHY_STAT0) & + HDMI_PHY_HPD) + dw_hdmi_hdcp_start(hdcp); + } else { + dw_hdmi_hdcp_stop(hdcp); + hdcp->enable = enable; + } + } + + return count; +} + +static DEVICE_ATTR(enable, 0644, hdcp_enable_read, hdcp_enable_write); + +static ssize_t hdcp_trytimes_read(struct device *device, + struct device_attribute *attr, char *buf) +{ + int trytimes = 0; + struct dw_hdcp *hdcp = g_hdcp; + + if (hdcp) + trytimes = hdcp->retry_times; + + return snprintf(buf, PAGE_SIZE, "%d\n", trytimes); +} + +static ssize_t hdcp_trytimes_write(struct device *device, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int trytimes; + struct dw_hdcp *hdcp = g_hdcp; + + if (!hdcp) + return -EINVAL; + + if (kstrtoint(buf, 0, &trytimes)) + return -EINVAL; + + if (hdcp->retry_times != trytimes) { + hdcp->retry_times = trytimes; + hdcp->remaining_times = hdcp->retry_times; + } + + return count; +} + +static DEVICE_ATTR(trytimes, 0644, hdcp_trytimes_read, hdcp_trytimes_write); + +static ssize_t hdcp_status_read(struct device *device, + struct device_attribute *attr, char *buf) +{ + int status = DW_HDCP_DISABLED; + struct dw_hdcp *hdcp = g_hdcp; + + if (hdcp) + status = hdcp->status; + + if (status == DW_HDCP_DISABLED) + return snprintf(buf, PAGE_SIZE, "hdcp disable\n"); + else if (status == DW_HDCP_AUTH_START) + return snprintf(buf, PAGE_SIZE, "hdcp_auth_start\n"); + else if (status == DW_HDCP_AUTH_SUCCESS) + return snprintf(buf, PAGE_SIZE, "hdcp_auth_success\n"); + else if (status == DW_HDCP_AUTH_FAIL) + return snprintf(buf, PAGE_SIZE, "hdcp_auth_fail\n"); + else + return snprintf(buf, PAGE_SIZE, "unknown status\n"); +} + +static DEVICE_ATTR(status, 0444, hdcp_status_read, NULL); + +static int dw_hdmi_hdcp_probe(struct platform_device *pdev) +{ + int ret = 0; + struct dw_hdcp *hdcp = pdev->dev.platform_data; + + g_hdcp = hdcp; + hdcp->mdev.minor = MISC_DYNAMIC_MINOR; + hdcp->mdev.name = "hdmi_hdcp1x"; + hdcp->mdev.mode = 0666; + + if (misc_register(&hdcp->mdev)) { + dev_err(&pdev->dev, "HDCP: Could not add character driver\n"); + return -EINVAL; + } + + ret = device_create_file(hdcp->mdev.this_device, &dev_attr_enable); + if (ret) { + dev_err(&pdev->dev, "HDCP: Could not add sys file enable\n"); + ret = -EINVAL; + goto error0; + } + + ret = device_create_file(hdcp->mdev.this_device, &dev_attr_trytimes); + if (ret) { + dev_err(&pdev->dev, "HDCP: Could not add sys file trytimes\n"); + ret = -EINVAL; + goto error1; + } + + ret = device_create_file(hdcp->mdev.this_device, &dev_attr_status); + if (ret) { + dev_err(&pdev->dev, "HDCP: Could not add sys file status\n"); + ret = -EINVAL; + goto error2; + } + + /* retry time if hdcp auth fail. unlimited time if set 0 */ + hdcp->retry_times = 0; + hdcp->dev = &pdev->dev; + hdcp->hdcp_start = dw_hdmi_hdcp_start; + hdcp->hdcp_stop = dw_hdmi_hdcp_stop; + hdcp->hdcp_isr = dw_hdmi_hdcp_isr; + dev_dbg(hdcp->dev, "%s success\n", __func__); + return 0; + +error2: + device_remove_file(hdcp->mdev.this_device, &dev_attr_trytimes); +error1: + device_remove_file(hdcp->mdev.this_device, &dev_attr_enable); +error0: + misc_deregister(&hdcp->mdev); + return ret; +} + +static int dw_hdmi_hdcp_remove(struct platform_device *pdev) +{ + struct dw_hdcp *hdcp = pdev->dev.platform_data; + + device_remove_file(hdcp->mdev.this_device, &dev_attr_trytimes); + device_remove_file(hdcp->mdev.this_device, &dev_attr_enable); + device_remove_file(hdcp->mdev.this_device, &dev_attr_status); + misc_deregister(&hdcp->mdev); + + kfree(hdcp->keys); + kfree(hdcp->seeds); + + return 0; +} + +static struct platform_driver dw_hdmi_hdcp_driver = { + .probe = dw_hdmi_hdcp_probe, + .remove = dw_hdmi_hdcp_remove, + .driver = { + .name = DW_HDCP_DRIVER_NAME, + }, +}; + +module_platform_driver(dw_hdmi_hdcp_driver); +MODULE_DESCRIPTION("DW HDMI transmitter HDCP driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.h new file mode 100755 index 000000000000..d138f91f3422 --- /dev/null +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author Huicong Xu + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef DW_HDMI_HDCP_H +#define DW_HDMI_HDCP_H + +#include + +#define DW_HDCP_DRIVER_NAME "dw-hdmi-hdcp" +#define HDCP_PRIVATE_KEY_SIZE 280 +#define HDCP_KEY_SHA_SIZE 20 + +struct hdcp_keys { + u8 KSV[8]; + u8 devicekey[HDCP_PRIVATE_KEY_SIZE]; + u8 sha1[HDCP_KEY_SHA_SIZE]; +}; + +struct dw_hdcp { + bool enable; + int retry_times; + int remaining_times; + char *seeds; + int invalidkey; + char *invalidkeys; + int hdcp2_enable; + int status; + u32 reg_io_width; + + struct miscdevice mdev; + struct hdcp_keys *keys; + struct device *dev; + struct dw_hdmi *hdmi; + void __iomem *regs; + + void (*write)(struct dw_hdmi *hdmi, u8 val, int offset); + u8 (*read)(struct dw_hdmi *hdmi, int offset); + int (*hdcp_start)(struct dw_hdcp *hdcp); + int (*hdcp_stop)(struct dw_hdcp *hdcp); + void (*hdcp_isr)(struct dw_hdcp *hdcp, int hdcp_int); +}; + +#endif diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-audio.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-audio.h new file mode 100755 index 000000000000..93f1a42954e7 --- /dev/null +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-audio.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021 Rockchip Electronics Co. Ltd. + * Author: Sugar Zhang + */ + +#ifndef DW_HDMI_QP_AUDIO_H +#define DW_HDMI_QP_AUDIO_H + +struct dw_hdmi_qp; + +struct dw_hdmi_qp_audio_data { + phys_addr_t phys; + void __iomem *base; + int irq; + struct dw_hdmi_qp *hdmi; + u8 *eld; +}; + +struct dw_hdmi_qp_i2s_audio_data { + struct dw_hdmi_qp *hdmi; + u8 *eld; + + void (*write)(struct dw_hdmi_qp *hdmi, u32 val, int offset); + u32 (*read)(struct dw_hdmi_qp *hdmi, int offset); + void (*mod)(struct dw_hdmi_qp *hdmi, u32 val, u32 mask, u32 reg); +}; + +#endif diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-i2s-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-i2s-audio.c new file mode 100755 index 000000000000..650ba2ce0dc3 --- /dev/null +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-i2s-audio.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dw-hdmi-qp-i2s-audio.c + * + * Copyright (c) 2021 Rockchip Electronics Co. Ltd. + * Author: Sugar Zhang + */ + +#include +#include + +#include +#include + +#include + +#include "dw-hdmi-qp.h" +#include "dw-hdmi-qp-audio.h" + +#define DRIVER_NAME "dw-hdmi-qp-i2s-audio" + +static inline void hdmi_write(struct dw_hdmi_qp_i2s_audio_data *audio, + u32 val, int offset) +{ + struct dw_hdmi_qp *hdmi = audio->hdmi; + + audio->write(hdmi, val, offset); +} + +static inline u32 hdmi_read(struct dw_hdmi_qp_i2s_audio_data *audio, int offset) +{ + struct dw_hdmi_qp *hdmi = audio->hdmi; + + return audio->read(hdmi, offset); +} + +static inline void hdmi_mod(struct dw_hdmi_qp_i2s_audio_data *audio, + u32 data, u32 mask, u32 reg) +{ + struct dw_hdmi_qp *hdmi = audio->hdmi; + + return audio->mod(hdmi, data, mask, reg); +} + +static inline bool is_dw_hdmi_qp_clk_off(struct dw_hdmi_qp_i2s_audio_data *audio) +{ + u32 sta = hdmi_read(audio, CMU_STATUS); + + return (sta & (AUDCLK_OFF | LINKQPCLK_OFF | VIDQPCLK_OFF)); +} + +static int dw_hdmi_qp_i2s_hw_params(struct device *dev, void *data, + struct hdmi_codec_daifmt *fmt, + struct hdmi_codec_params *hparms) +{ + struct dw_hdmi_qp_i2s_audio_data *audio = data; + struct dw_hdmi_qp *hdmi = audio->hdmi; + u32 conf0 = 0; + + if (is_dw_hdmi_qp_clk_off(audio)) + return 0; + + if (fmt->bit_clk_master | fmt->frame_clk_master) { + dev_err(dev, "unsupported clock settings\n"); + return -EINVAL; + } + + /* Reset the audio data path of the AVP */ + hdmi_write(audio, AVP_DATAPATH_PACKET_AUDIO_SWINIT_P, GLOBAL_SWRESET_REQUEST); + + /* Clear the audio FIFO */ + hdmi_write(audio, AUDIO_FIFO_CLR_P, AUDIO_INTERFACE_CONTROL0); + + /* Disable AUDS, ACR, AUDI, AMD */ + hdmi_mod(audio, 0, + PKTSCHED_ACR_TX_EN | PKTSCHED_AUDS_TX_EN | + PKTSCHED_AUDI_TX_EN | PKTSCHED_AMD_TX_EN, + PKTSCHED_PKT_EN); + + /* Select I2S interface as the audio source */ + hdmi_mod(audio, AUD_IF_I2S, AUD_IF_SEL_MSK, AUDIO_INTERFACE_CONFIG0); + + /* Enable the active i2s lanes */ + switch (hparms->channels) { + case 7 ... 8: + conf0 |= I2S_LINES_EN(3); + fallthrough; + case 5 ... 6: + conf0 |= I2S_LINES_EN(2); + fallthrough; + case 3 ... 4: + conf0 |= I2S_LINES_EN(1); + fallthrough; + default: + conf0 |= I2S_LINES_EN(0); + break; + } + + hdmi_mod(audio, conf0, I2S_LINES_EN_MSK, AUDIO_INTERFACE_CONFIG0); + + /* + * Enable bpcuv generated internally for L-PCM, or received + * from stream for NLPCM/HBR. + */ + switch (fmt->fmt) { + case SNDRV_PCM_FORMAT_IEC958_SUBFRAME_LE: + conf0 = (hparms->channels == 8) ? AUD_HBR : AUD_ASP; + conf0 |= I2S_BPCUV_RCV_EN; + break; + default: + conf0 = AUD_ASP | I2S_BPCUV_RCV_DIS; + break; + } + + hdmi_mod(audio, conf0, I2S_BPCUV_RCV_MSK | AUD_FORMAT_MSK, + AUDIO_INTERFACE_CONFIG0); + + /* Enable audio FIFO auto clear when overflow */ + hdmi_mod(audio, AUD_FIFO_INIT_ON_OVF_EN, AUD_FIFO_INIT_ON_OVF_MSK, + AUDIO_INTERFACE_CONFIG0); + + dw_hdmi_qp_set_sample_rate(hdmi, hparms->sample_rate); + dw_hdmi_qp_set_channel_status(hdmi, hparms->iec.status); + dw_hdmi_qp_set_channel_count(hdmi, hparms->channels); + dw_hdmi_qp_set_channel_allocation(hdmi, hparms->cea.channel_allocation); + + /* Enable ACR, AUDI, AMD */ + hdmi_mod(audio, + PKTSCHED_ACR_TX_EN | PKTSCHED_AUDI_TX_EN | PKTSCHED_AMD_TX_EN, + PKTSCHED_ACR_TX_EN | PKTSCHED_AUDI_TX_EN | PKTSCHED_AMD_TX_EN, + PKTSCHED_PKT_EN); + + /* Enable AUDS */ + hdmi_mod(audio, PKTSCHED_AUDS_TX_EN, PKTSCHED_AUDS_TX_EN, PKTSCHED_PKT_EN); + + return 0; +} + +static int dw_hdmi_qp_i2s_audio_startup(struct device *dev, void *data) +{ + struct dw_hdmi_qp_i2s_audio_data *audio = data; + struct dw_hdmi_qp *hdmi = audio->hdmi; + + if (is_dw_hdmi_qp_clk_off(audio)) + return 0; + + dw_hdmi_qp_audio_enable(hdmi); + + return 0; +} + +static void dw_hdmi_qp_i2s_audio_shutdown(struct device *dev, void *data) +{ + struct dw_hdmi_qp_i2s_audio_data *audio = data; + struct dw_hdmi_qp *hdmi = audio->hdmi; + + if (is_dw_hdmi_qp_clk_off(audio)) + return; + + dw_hdmi_qp_audio_disable(hdmi); +} + +static int dw_hdmi_qp_i2s_get_eld(struct device *dev, void *data, uint8_t *buf, + size_t len) +{ + struct dw_hdmi_qp_i2s_audio_data *audio = data; + + memcpy(buf, audio->eld, min_t(size_t, MAX_ELD_BYTES, len)); + + return 0; +} + +static int dw_hdmi_qp_i2s_get_dai_id(struct snd_soc_component *component, + struct device_node *endpoint) +{ + struct of_endpoint of_ep; + int ret; + + ret = of_graph_parse_endpoint(endpoint, &of_ep); + if (ret < 0) + return ret; + + /* + * HDMI sound should be located as reg = <2> + * Then, it is sound port 0 + */ + if (of_ep.port == 2) + return 0; + + return -EINVAL; +} + +static int dw_hdmi_qp_i2s_hook_plugged_cb(struct device *dev, void *data, + hdmi_codec_plugged_cb fn, + struct device *codec_dev) +{ + struct dw_hdmi_qp_i2s_audio_data *audio = data; + struct dw_hdmi_qp *hdmi = audio->hdmi; + + return dw_hdmi_qp_set_plugged_cb(hdmi, fn, codec_dev); +} + +static struct hdmi_codec_ops dw_hdmi_qp_i2s_ops = { + .hw_params = dw_hdmi_qp_i2s_hw_params, + .audio_startup = dw_hdmi_qp_i2s_audio_startup, + .audio_shutdown = dw_hdmi_qp_i2s_audio_shutdown, + .get_eld = dw_hdmi_qp_i2s_get_eld, + .get_dai_id = dw_hdmi_qp_i2s_get_dai_id, + .hook_plugged_cb = dw_hdmi_qp_i2s_hook_plugged_cb, +}; + +static int snd_dw_hdmi_qp_probe(struct platform_device *pdev) +{ + struct dw_hdmi_qp_i2s_audio_data *audio = pdev->dev.platform_data; + struct platform_device_info pdevinfo; + struct hdmi_codec_pdata pdata; + struct platform_device *platform; + + pdata.ops = &dw_hdmi_qp_i2s_ops; + pdata.i2s = 1; + pdata.max_i2s_channels = 8; + pdata.data = audio; + + memset(&pdevinfo, 0, sizeof(pdevinfo)); + pdevinfo.parent = pdev->dev.parent; + pdevinfo.id = PLATFORM_DEVID_AUTO; + pdevinfo.name = HDMI_CODEC_DRV_NAME; + pdevinfo.data = &pdata; + pdevinfo.size_data = sizeof(pdata); + pdevinfo.dma_mask = DMA_BIT_MASK(32); + + platform = platform_device_register_full(&pdevinfo); + if (IS_ERR(platform)) + return PTR_ERR(platform); + + dev_set_drvdata(&pdev->dev, platform); + + return 0; +} + +static int snd_dw_hdmi_qp_remove(struct platform_device *pdev) +{ + struct platform_device *platform = dev_get_drvdata(&pdev->dev); + + platform_device_unregister(platform); + + return 0; +} + +static struct platform_driver snd_dw_hdmi_qp_driver = { + .probe = snd_dw_hdmi_qp_probe, + .remove = snd_dw_hdmi_qp_remove, + .driver = { + .name = DRIVER_NAME, + }, +}; +module_platform_driver(snd_dw_hdmi_qp_driver); + +MODULE_AUTHOR("Sugar Zhang "); +MODULE_DESCRIPTION("Synopsis Designware HDMI QP I2S ALSA SoC interface"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:" DRIVER_NAME); diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c new file mode 100755 index 000000000000..cd8150b33bbb --- /dev/null +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c @@ -0,0 +1,2455 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) Rockchip Electronics Co.Ltd + * Author: + * Algea Cao + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "dw-hdmi-qp-audio.h" +#include "dw-hdmi-qp.h" + +#include + +#define DDC_CI_ADDR 0x37 +#define DDC_SEGMENT_ADDR 0x30 + +#define HDMI_EDID_LEN 512 + +/* DW-HDMI Controller >= 0x200a are at least compliant with SCDC version 1 */ +#define SCDC_MIN_SOURCE_VERSION 0x1 + +#define HDMI14_MAX_TMDSCLK 340000000 + +static const unsigned int dw_hdmi_cable[] = { + EXTCON_DISP_HDMI, + EXTCON_NONE, +}; + +/* + * Unless otherwise noted, entries in this table are 100% optimization. + * Values can be obtained from hdmi_compute_n() but that function is + * slow so we pre-compute values we expect to see. + * + * All 32k and 48k values are expected to be the same (due to the way + * the math works) for any rate that's an exact kHz. + */ +static const struct dw_hdmi_audio_tmds_n common_tmds_n_table[] = { + { .tmds = 25175000, .n_32k = 4096, .n_44k1 = 12854, .n_48k = 6144, }, + { .tmds = 25200000, .n_32k = 4096, .n_44k1 = 5656, .n_48k = 6144, }, + { .tmds = 27000000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, + { .tmds = 28320000, .n_32k = 4096, .n_44k1 = 5586, .n_48k = 6144, }, + { .tmds = 30240000, .n_32k = 4096, .n_44k1 = 5642, .n_48k = 6144, }, + { .tmds = 31500000, .n_32k = 4096, .n_44k1 = 5600, .n_48k = 6144, }, + { .tmds = 32000000, .n_32k = 4096, .n_44k1 = 5733, .n_48k = 6144, }, + { .tmds = 33750000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, + { .tmds = 36000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, + { .tmds = 40000000, .n_32k = 4096, .n_44k1 = 5733, .n_48k = 6144, }, + { .tmds = 49500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, + { .tmds = 50000000, .n_32k = 4096, .n_44k1 = 5292, .n_48k = 6144, }, + { .tmds = 54000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, + { .tmds = 65000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, + { .tmds = 68250000, .n_32k = 4096, .n_44k1 = 5376, .n_48k = 6144, }, + { .tmds = 71000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, + { .tmds = 72000000, .n_32k = 4096, .n_44k1 = 5635, .n_48k = 6144, }, + { .tmds = 73250000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, + { .tmds = 74250000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, + { .tmds = 75000000, .n_32k = 4096, .n_44k1 = 5880, .n_48k = 6144, }, + { .tmds = 78750000, .n_32k = 4096, .n_44k1 = 5600, .n_48k = 6144, }, + { .tmds = 78800000, .n_32k = 4096, .n_44k1 = 5292, .n_48k = 6144, }, + { .tmds = 79500000, .n_32k = 4096, .n_44k1 = 4704, .n_48k = 6144, }, + { .tmds = 83500000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, + { .tmds = 85500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, + { .tmds = 88750000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, + { .tmds = 97750000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, + { .tmds = 101000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, + { .tmds = 106500000, .n_32k = 4096, .n_44k1 = 4704, .n_48k = 6144, }, + { .tmds = 108000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, + { .tmds = 115500000, .n_32k = 4096, .n_44k1 = 5712, .n_48k = 6144, }, + { .tmds = 119000000, .n_32k = 4096, .n_44k1 = 5544, .n_48k = 6144, }, + { .tmds = 135000000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, + { .tmds = 146250000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, + { .tmds = 148500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, + { .tmds = 154000000, .n_32k = 4096, .n_44k1 = 5544, .n_48k = 6144, }, + { .tmds = 162000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, + + /* For 297 MHz+ HDMI spec have some other rule for setting N */ + { .tmds = 297000000, .n_32k = 3073, .n_44k1 = 4704, .n_48k = 5120, }, + { .tmds = 594000000, .n_32k = 3073, .n_44k1 = 9408, .n_48k = 10240, }, + + /* End of table */ + { .tmds = 0, .n_32k = 0, .n_44k1 = 0, .n_48k = 0, }, +}; + +static const struct drm_display_mode dw_hdmi_default_modes[] = { + /* 16 - 1920x1080@60Hz 16:9 */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008, + 2052, 2200, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 2 - 720x480@60Hz 4:3 */ + { DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736, + 798, 858, 0, 480, 489, 495, 525, 0, + DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, }, + /* 4 - 1280x720@60Hz 16:9 */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390, + 1430, 1650, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 31 - 1920x1080@50Hz 16:9 */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448, + 2492, 2640, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 19 - 1280x720@50Hz 16:9 */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720, + 1760, 1980, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 17 - 720x576@50Hz 4:3 */ + { DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, + 796, 864, 0, 576, 581, 586, 625, 0, + DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, }, + /* 2 - 720x480@60Hz 4:3 */ + { DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736, + 798, 858, 0, 480, 489, 495, 525, 0, + DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, }, +}; + +enum frl_mask { + FRL_3GBPS_3LANE = 1, + FRL_6GBPS_3LANE, + FRL_6GBPS_4LANE, + FRL_8GBPS_4LANE, + FRL_10GBPS_4LANE, + FRL_12GBPS_4LANE, +}; + +struct hdmi_vmode_qp { + bool mdataenablepolarity; + + unsigned int previous_pixelclock; + unsigned long mpixelclock; + unsigned int mpixelrepetitioninput; + unsigned int mpixelrepetitionoutput; + unsigned long previous_tmdsclock; + unsigned int mtmdsclock; +}; + +struct hdmi_qp_data_info { + unsigned int enc_in_bus_format; + unsigned int enc_out_bus_format; + unsigned int enc_in_encoding; + unsigned int enc_out_encoding; + unsigned int quant_range; + unsigned int pix_repet_factor; + struct hdmi_vmode_qp video_mode; + bool update; +}; + +struct dw_hdmi_qp_i2c { + struct i2c_adapter adap; + + struct mutex lock; /* used to serialize data transfers */ + struct completion cmp; + u32 stat; + + u8 slave_reg; + bool is_regaddr; + bool is_segment; + + unsigned int scl_high_ns; + unsigned int scl_low_ns; +}; + +struct dw_hdmi_phy_data { + enum dw_hdmi_phy_type type; + const char *name; + unsigned int gen; + bool has_svsret; + int (*configure)(struct dw_hdmi_qp *hdmi, + const struct dw_hdmi_plat_data *pdata, + unsigned long mpixelclock); +}; + +struct dw_hdmi_qp { + struct drm_connector connector; + struct drm_bridge bridge; + struct platform_device *hdcp_dev; + struct platform_device *audio; + + struct device *dev; + struct dw_hdmi_qp_i2c *i2c; + + struct hdmi_qp_data_info hdmi_data; + const struct dw_hdmi_plat_data *plat_data; + + int vic; + int main_irq; + int avp_irq; + int earc_irq; + + u8 edid[HDMI_EDID_LEN]; + + struct { + const struct dw_hdmi_qp_phy_ops *ops; + const char *name; + void *data; + bool enabled; + } phy; + + struct drm_display_mode previous_mode; + + struct i2c_adapter *ddc; + void __iomem *regs; + bool sink_is_hdmi; + bool sink_has_audio; + bool hpd_state; + + struct mutex mutex; /* for state below and previous_mode */ + struct drm_connector *curr_conn;/* current connector (only valid when !disabled) */ + enum drm_connector_force force; /* mutex-protected force state */ + bool disabled; /* DRM has disabled our bridge */ + bool bridge_is_on; /* indicates the bridge is on */ + bool rxsense; /* rxsense state */ + u8 phy_mask; /* desired phy int mask settings */ + u8 mc_clkdis; /* clock disable register */ + + u32 scdc_intr; + u32 flt_intr; + u32 earc_intr; + + spinlock_t audio_lock; + struct mutex audio_mutex; + unsigned int sample_rate; + unsigned int audio_cts; + unsigned int audio_n; + bool audio_enable; + void (*enable_audio)(struct dw_hdmi_qp *hdmi); + void (*disable_audio)(struct dw_hdmi_qp *hdmi); + + struct dentry *debugfs_dir; + bool scramble_low_rates; + + struct extcon_dev *extcon; + + struct regmap *regm; + + bool initialized; /* hdmi is enabled before bind */ + struct completion flt_cmp; + struct completion earc_cmp; + + struct cec_notifier *cec_notifier; + struct cec_adapter *cec_adap; + struct mutex cec_notifier_mutex; + + hdmi_codec_plugged_cb plugged_cb; + struct device *codec_dev; + enum drm_connector_status last_connector_result; +}; + +static inline void hdmi_writel(struct dw_hdmi_qp *hdmi, u32 val, int offset) +{ + regmap_write(hdmi->regm, offset, val); +} + +static inline u32 hdmi_readl(struct dw_hdmi_qp *hdmi, int offset) +{ + unsigned int val = 0; + + regmap_read(hdmi->regm, offset, &val); + + return val; +} + +static void handle_plugged_change(struct dw_hdmi_qp *hdmi, bool plugged) +{ + if (hdmi->plugged_cb && hdmi->codec_dev) + hdmi->plugged_cb(hdmi->codec_dev, plugged); +} + +int dw_hdmi_qp_set_plugged_cb(struct dw_hdmi_qp *hdmi, hdmi_codec_plugged_cb fn, + struct device *codec_dev) +{ + bool plugged; + + mutex_lock(&hdmi->mutex); + hdmi->plugged_cb = fn; + hdmi->codec_dev = codec_dev; + plugged = hdmi->last_connector_result == connector_status_connected; + handle_plugged_change(hdmi, plugged); + mutex_unlock(&hdmi->mutex); + + return 0; +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_plugged_cb); + +static void hdmi_modb(struct dw_hdmi_qp *hdmi, u32 data, u32 mask, u32 reg) +{ + regmap_update_bits(hdmi->regm, reg, mask, data); +} + +static void hdmi_set_cts_n(struct dw_hdmi_qp *hdmi, unsigned int cts, + unsigned int n) +{ + /* Set N */ + hdmi_modb(hdmi, n, AUDPKT_ACR_N_VALUE, AUDPKT_ACR_CONTROL0); + + /* Set CTS */ + if (cts) + hdmi_modb(hdmi, AUDPKT_ACR_CTS_OVR_EN, AUDPKT_ACR_CTS_OVR_EN_MSK, + AUDPKT_ACR_CONTROL1); + else + hdmi_modb(hdmi, 0, AUDPKT_ACR_CTS_OVR_EN_MSK, + AUDPKT_ACR_CONTROL1); + + hdmi_modb(hdmi, AUDPKT_ACR_CTS_OVR_VAL(cts), AUDPKT_ACR_CTS_OVR_VAL_MSK, + AUDPKT_ACR_CONTROL1); +} + +static int hdmi_match_tmds_n_table(struct dw_hdmi_qp *hdmi, + unsigned long pixel_clk, + unsigned long freq) +{ + const struct dw_hdmi_plat_data *plat_data = hdmi->plat_data; + const struct dw_hdmi_audio_tmds_n *tmds_n = NULL; + int i; + + if (plat_data->tmds_n_table) { + for (i = 0; plat_data->tmds_n_table[i].tmds != 0; i++) { + if (pixel_clk == plat_data->tmds_n_table[i].tmds) { + tmds_n = &plat_data->tmds_n_table[i]; + break; + } + } + } + + if (tmds_n == NULL) { + for (i = 0; common_tmds_n_table[i].tmds != 0; i++) { + if (pixel_clk == common_tmds_n_table[i].tmds) { + tmds_n = &common_tmds_n_table[i]; + break; + } + } + } + + if (tmds_n == NULL) + return -ENOENT; + + switch (freq) { + case 32000: + return tmds_n->n_32k; + case 44100: + case 88200: + case 176400: + return (freq / 44100) * tmds_n->n_44k1; + case 48000: + case 96000: + case 192000: + return (freq / 48000) * tmds_n->n_48k; + default: + return -ENOENT; + } +} + +static u64 hdmi_audio_math_diff(unsigned int freq, unsigned int n, + unsigned int pixel_clk) +{ + u64 final, diff; + u64 cts; + + final = (u64)pixel_clk * n; + + cts = final; + do_div(cts, 128 * freq); + + diff = final - (u64)cts * (128 * freq); + + return diff; +} + +static unsigned int hdmi_compute_n(struct dw_hdmi_qp *hdmi, + unsigned long pixel_clk, + unsigned long freq) +{ + unsigned int min_n = DIV_ROUND_UP((128 * freq), 1500); + unsigned int max_n = (128 * freq) / 300; + unsigned int ideal_n = (128 * freq) / 1000; + unsigned int best_n_distance = ideal_n; + unsigned int best_n = 0; + u64 best_diff = U64_MAX; + int n; + + /* If the ideal N could satisfy the audio math, then just take it */ + if (hdmi_audio_math_diff(freq, ideal_n, pixel_clk) == 0) + return ideal_n; + + for (n = min_n; n <= max_n; n++) { + u64 diff = hdmi_audio_math_diff(freq, n, pixel_clk); + + if (diff < best_diff || (diff == best_diff && + abs(n - ideal_n) < best_n_distance)) { + best_n = n; + best_diff = diff; + best_n_distance = abs(best_n - ideal_n); + } + + /* + * The best N already satisfy the audio math, and also be + * the closest value to ideal N, so just cut the loop. + */ + if ((best_diff == 0) && (abs(n - ideal_n) > best_n_distance)) + break; + } + + return best_n; +} + +static unsigned int hdmi_find_n(struct dw_hdmi_qp *hdmi, unsigned long pixel_clk, + unsigned long sample_rate) +{ + int n; + + n = hdmi_match_tmds_n_table(hdmi, pixel_clk, sample_rate); + if (n > 0) + return n; + + dev_warn(hdmi->dev, "Rate %lu missing; compute N dynamically\n", + pixel_clk); + + return hdmi_compute_n(hdmi, pixel_clk, sample_rate); +} + +/* + * When transmitting IEC60958 linear PCM audio, these registers allow to + * configure the channel status information of all the channel status + * bits in the IEC60958 frame. For the moment this configuration is only + * used when the I2S audio interface, General Purpose Audio (GPA), + * or AHB audio DMA (AHBAUDDMA) interface is active + * (for S/PDIF interface this information comes from the stream). + */ +void dw_hdmi_qp_set_channel_status(struct dw_hdmi_qp *hdmi, + u8 *channel_status) +{ + /* Set channel status */ + hdmi_writel(hdmi, channel_status[3] | (channel_status[4] << 8), + AUDPKT_CHSTATUS_OVR1); + hdmi_modb(hdmi, AUDPKT_CHSTATUS_OVR_EN, + AUDPKT_CHSTATUS_OVR_EN_MASK, AUDPKT_CONTROL0); +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_channel_status); + +static void hdmi_set_clk_regenerator(struct dw_hdmi_qp *hdmi, + unsigned long pixel_clk, unsigned int sample_rate) +{ + unsigned int n = 0, cts = 0; + + n = hdmi_find_n(hdmi, pixel_clk, sample_rate); + + spin_lock_irq(&hdmi->audio_lock); + hdmi->audio_n = n; + hdmi->audio_cts = cts; + hdmi_set_cts_n(hdmi, cts, hdmi->audio_enable ? n : 0); + spin_unlock_irq(&hdmi->audio_lock); +} + +static void hdmi_init_clk_regenerator(struct dw_hdmi_qp *hdmi) +{ + mutex_lock(&hdmi->audio_mutex); + hdmi_set_clk_regenerator(hdmi, 74250000, hdmi->sample_rate); + mutex_unlock(&hdmi->audio_mutex); +} + +static void hdmi_clk_regenerator_update_pixel_clock(struct dw_hdmi_qp *hdmi) +{ + mutex_lock(&hdmi->audio_mutex); + hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mtmdsclock, + hdmi->sample_rate); + mutex_unlock(&hdmi->audio_mutex); +} + +void dw_hdmi_qp_set_sample_rate(struct dw_hdmi_qp *hdmi, unsigned int rate) +{ + mutex_lock(&hdmi->audio_mutex); + hdmi->sample_rate = rate; + hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mtmdsclock, + hdmi->sample_rate); + mutex_unlock(&hdmi->audio_mutex); +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_sample_rate); + +void dw_hdmi_qp_set_channel_count(struct dw_hdmi_qp *hdmi, unsigned int cnt) +{ +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_channel_count); + +void dw_hdmi_qp_set_channel_allocation(struct dw_hdmi_qp *hdmi, unsigned int ca) +{ +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_channel_allocation); + +static void hdmi_enable_audio_clk(struct dw_hdmi_qp *hdmi, bool enable) +{ + if (enable) + hdmi_modb(hdmi, 0, + AVP_DATAPATH_PACKET_AUDIO_SWDISABLE, GLOBAL_SWDISABLE); + else + hdmi_modb(hdmi, AVP_DATAPATH_PACKET_AUDIO_SWDISABLE, + AVP_DATAPATH_PACKET_AUDIO_SWDISABLE, GLOBAL_SWDISABLE); +} + +static void dw_hdmi_i2s_audio_enable(struct dw_hdmi_qp *hdmi) +{ + hdmi_set_cts_n(hdmi, hdmi->audio_cts, hdmi->audio_n); + hdmi_enable_audio_clk(hdmi, true); +} + +static void dw_hdmi_i2s_audio_disable(struct dw_hdmi_qp *hdmi) +{ + hdmi_enable_audio_clk(hdmi, false); +} + +void dw_hdmi_qp_audio_enable(struct dw_hdmi_qp *hdmi) +{ + unsigned long flags; + + spin_lock_irqsave(&hdmi->audio_lock, flags); + hdmi->audio_enable = true; + if (hdmi->enable_audio) + hdmi->enable_audio(hdmi); + spin_unlock_irqrestore(&hdmi->audio_lock, flags); +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_audio_enable); + +void dw_hdmi_qp_audio_disable(struct dw_hdmi_qp *hdmi) +{ + unsigned long flags; + + spin_lock_irqsave(&hdmi->audio_lock, flags); + hdmi->audio_enable = false; + if (hdmi->disable_audio) + hdmi->disable_audio(hdmi); + spin_unlock_irqrestore(&hdmi->audio_lock, flags); +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_audio_disable); + +static bool hdmi_bus_fmt_is_rgb(unsigned int bus_format) +{ + switch (bus_format) { + case MEDIA_BUS_FMT_RGB888_1X24: + case MEDIA_BUS_FMT_RGB101010_1X30: + case MEDIA_BUS_FMT_RGB121212_1X36: + case MEDIA_BUS_FMT_RGB161616_1X48: + return true; + + default: + return false; + } +} + +static bool hdmi_bus_fmt_is_yuv444(unsigned int bus_format) +{ + switch (bus_format) { + case MEDIA_BUS_FMT_YUV8_1X24: + case MEDIA_BUS_FMT_YUV10_1X30: + case MEDIA_BUS_FMT_YUV12_1X36: + case MEDIA_BUS_FMT_YUV16_1X48: + return true; + + default: + return false; + } +} + +static bool hdmi_bus_fmt_is_yuv422(unsigned int bus_format) +{ + switch (bus_format) { + case MEDIA_BUS_FMT_UYVY8_1X16: + case MEDIA_BUS_FMT_UYVY10_1X20: + case MEDIA_BUS_FMT_UYVY12_1X24: + return true; + + default: + return false; + } +} + +static bool hdmi_bus_fmt_is_yuv420(unsigned int bus_format) +{ + switch (bus_format) { + case MEDIA_BUS_FMT_UYYVYY8_0_5X24: + case MEDIA_BUS_FMT_UYYVYY10_0_5X30: + case MEDIA_BUS_FMT_UYYVYY12_0_5X36: + case MEDIA_BUS_FMT_UYYVYY16_0_5X48: + return true; + + default: + return false; + } +} + +static int hdmi_bus_fmt_color_depth(unsigned int bus_format) +{ + switch (bus_format) { + case MEDIA_BUS_FMT_RGB888_1X24: + case MEDIA_BUS_FMT_YUV8_1X24: + case MEDIA_BUS_FMT_UYVY8_1X16: + case MEDIA_BUS_FMT_UYYVYY8_0_5X24: + return 8; + + case MEDIA_BUS_FMT_RGB101010_1X30: + case MEDIA_BUS_FMT_YUV10_1X30: + case MEDIA_BUS_FMT_UYVY10_1X20: + case MEDIA_BUS_FMT_UYYVYY10_0_5X30: + return 10; + + case MEDIA_BUS_FMT_RGB121212_1X36: + case MEDIA_BUS_FMT_YUV12_1X36: + case MEDIA_BUS_FMT_UYVY12_1X24: + case MEDIA_BUS_FMT_UYYVYY12_0_5X36: + return 12; + + case MEDIA_BUS_FMT_RGB161616_1X48: + case MEDIA_BUS_FMT_YUV16_1X48: + case MEDIA_BUS_FMT_UYYVYY16_0_5X48: + return 16; + + default: + return 0; + } +} + +static void dw_hdmi_i2c_init(struct dw_hdmi_qp *hdmi) +{ + /* Software reset */ + hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); + hdmi_writel(hdmi, 0x00, I2CM_CONTROL0); + + hdmi_writel(hdmi, 0x085c085c, I2CM_FM_SCL_CONFIG0); + + hdmi_modb(hdmi, 0, I2CM_FM_EN, I2CM_INTERFACE_CONTROL0); + + /* Clear DONE and ERROR interrupts */ + hdmi_writel(hdmi, I2CM_OP_DONE_CLEAR | I2CM_NACK_RCVD_CLEAR, + MAINUNIT_1_INT_CLEAR); +} + +static int dw_hdmi_i2c_read(struct dw_hdmi_qp *hdmi, + unsigned char *buf, unsigned int length) +{ + struct dw_hdmi_qp_i2c *i2c = hdmi->i2c; + int stat; + + if (!i2c->is_regaddr) { + dev_dbg(hdmi->dev, "set read register address to 0\n"); + i2c->slave_reg = 0x00; + i2c->is_regaddr = true; + } + + while (length--) { + reinit_completion(&i2c->cmp); + + hdmi_modb(hdmi, i2c->slave_reg++ << 12, I2CM_ADDR, + I2CM_INTERFACE_CONTROL0); + + hdmi_modb(hdmi, I2CM_FM_READ, I2CM_WR_MASK, + I2CM_INTERFACE_CONTROL0); + + stat = wait_for_completion_timeout(&i2c->cmp, HZ / 10); + if (!stat) { + dev_err(hdmi->dev, "i2c read time out!\n"); + hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); + return -EAGAIN; + } + + /* Check for error condition on the bus */ + if (i2c->stat & I2CM_NACK_RCVD_IRQ) { + dev_err(hdmi->dev, "i2c read err!\n"); + hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); + return -EIO; + } + + *buf++ = hdmi_readl(hdmi, I2CM_INTERFACE_RDDATA_0_3) & 0xff; + dev_dbg(hdmi->dev, "i2c read done! i2c->stat:%02x 0x%02x\n", + i2c->stat, hdmi_readl(hdmi, I2CM_INTERFACE_RDDATA_0_3)); + hdmi_modb(hdmi, 0, I2CM_WR_MASK, I2CM_INTERFACE_CONTROL0); + } + i2c->is_segment = false; + + return 0; +} + +static int dw_hdmi_i2c_write(struct dw_hdmi_qp *hdmi, + unsigned char *buf, unsigned int length) +{ + struct dw_hdmi_qp_i2c *i2c = hdmi->i2c; + int stat; + + if (!i2c->is_regaddr) { + /* Use the first write byte as register address */ + i2c->slave_reg = buf[0]; + length--; + buf++; + i2c->is_regaddr = true; + } + + while (length--) { + reinit_completion(&i2c->cmp); + + hdmi_writel(hdmi, *buf++, I2CM_INTERFACE_WRDATA_0_3); + hdmi_modb(hdmi, i2c->slave_reg++ << 12, I2CM_ADDR, + I2CM_INTERFACE_CONTROL0); + hdmi_modb(hdmi, I2CM_FM_WRITE, I2CM_WR_MASK, + I2CM_INTERFACE_CONTROL0); + + stat = wait_for_completion_timeout(&i2c->cmp, HZ / 10); + if (!stat) { + dev_err(hdmi->dev, "i2c write time out!\n"); + hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); + return -EAGAIN; + } + + /* Check for error condition on the bus */ + if (i2c->stat & I2CM_NACK_RCVD_IRQ) { + dev_err(hdmi->dev, "i2c write nack!\n"); + hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); + return -EIO; + } + hdmi_modb(hdmi, 0, I2CM_WR_MASK, I2CM_INTERFACE_CONTROL0); + } + dev_dbg(hdmi->dev, "i2c write done!\n"); + return 0; +} + +static int dw_hdmi_i2c_xfer(struct i2c_adapter *adap, + struct i2c_msg *msgs, int num) +{ + struct dw_hdmi_qp *hdmi = i2c_get_adapdata(adap); + struct dw_hdmi_qp_i2c *i2c = hdmi->i2c; + u8 addr = msgs[0].addr; + int i, ret = 0; + + if (addr == DDC_CI_ADDR) + /* + * The internal I2C controller does not support the multi-byte + * read and write operations needed for DDC/CI. + * TOFIX: Blacklist the DDC/CI address until we filter out + * unsupported I2C operations. + */ + return -EOPNOTSUPP; + + dev_dbg(hdmi->dev, "i2c xfer: num: %d, addr: %#x\n", num, addr); + + for (i = 0; i < num; i++) { + if (msgs[i].len == 0) { + dev_err(hdmi->dev, + "unsupported transfer %d/%d, no data\n", + i + 1, num); + return -EOPNOTSUPP; + } + } + + mutex_lock(&i2c->lock); + + /* Unmute DONE and ERROR interrupts */ + hdmi_modb(hdmi, I2CM_NACK_RCVD_MASK_N | I2CM_OP_DONE_MASK_N, + I2CM_NACK_RCVD_MASK_N | I2CM_OP_DONE_MASK_N, + MAINUNIT_1_INT_MASK_N); + + /* Set slave device address taken from the first I2C message */ + if (addr == DDC_SEGMENT_ADDR && msgs[0].len == 1) + addr = DDC_ADDR; + + hdmi_modb(hdmi, addr << 5, I2CM_SLVADDR, I2CM_INTERFACE_CONTROL0); + + /* Set slave device register address on transfer */ + i2c->is_regaddr = false; + + /* Set segment pointer for I2C extended read mode operation */ + i2c->is_segment = false; + + for (i = 0; i < num; i++) { + dev_dbg(hdmi->dev, "xfer: num: %d/%d, len: %d, flags: %#x\n", + i + 1, num, msgs[i].len, msgs[i].flags); + + if (msgs[i].addr == DDC_SEGMENT_ADDR && msgs[i].len == 1) { + i2c->is_segment = true; + hdmi_modb(hdmi, DDC_SEGMENT_ADDR, I2CM_SEG_ADDR, + I2CM_INTERFACE_CONTROL1); + hdmi_modb(hdmi, *msgs[i].buf, I2CM_SEG_PTR, + I2CM_INTERFACE_CONTROL1); + } else { + if (msgs[i].flags & I2C_M_RD) + ret = dw_hdmi_i2c_read(hdmi, msgs[i].buf, + msgs[i].len); + else + ret = dw_hdmi_i2c_write(hdmi, msgs[i].buf, + msgs[i].len); + } + if (ret < 0) + break; + } + + if (!ret) + ret = num; + + /* Mute DONE and ERROR interrupts */ + hdmi_modb(hdmi, 0, I2CM_OP_DONE_MASK_N | I2CM_NACK_RCVD_MASK_N, + MAINUNIT_1_INT_MASK_N); + + mutex_unlock(&i2c->lock); + + return ret; +} + +static u32 dw_hdmi_i2c_func(struct i2c_adapter *adapter) +{ + return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; +} + +static const struct i2c_algorithm dw_hdmi_algorithm = { + .master_xfer = dw_hdmi_i2c_xfer, + .functionality = dw_hdmi_i2c_func, +}; + +static struct i2c_adapter *dw_hdmi_i2c_adapter(struct dw_hdmi_qp *hdmi) +{ + struct i2c_adapter *adap; + struct dw_hdmi_qp_i2c *i2c; + int ret; + + i2c = devm_kzalloc(hdmi->dev, sizeof(*i2c), GFP_KERNEL); + if (!i2c) + return ERR_PTR(-ENOMEM); + + mutex_init(&i2c->lock); + init_completion(&i2c->cmp); + + adap = &i2c->adap; + adap->class = I2C_CLASS_DDC; + adap->owner = THIS_MODULE; + adap->dev.parent = hdmi->dev; + adap->algo = &dw_hdmi_algorithm; + strscpy(adap->name, "ddc", sizeof(adap->name)); + i2c_set_adapdata(adap, hdmi); + + ret = i2c_add_adapter(adap); + if (ret) { + dev_warn(hdmi->dev, "cannot add %s I2C adapter\n", adap->name); + devm_kfree(hdmi->dev, i2c); + return ERR_PTR(ret); + } + + hdmi->i2c = i2c; + + dev_info(hdmi->dev, "registered %s I2C bus driver\n", adap->name); + + return adap; +} + +#define HDMI_PHY_EARC_MASK BIT(29) + +int dw_hdmi_qp_set_earc(struct dw_hdmi_qp *hdmi) +{ + u32 stat, ret; + + /* set hdmi phy earc mode */ + hdmi->phy.ops->set_mode(hdmi, hdmi->phy.data, HDMI_PHY_EARC_MASK, + true); + + ret = hdmi->phy.ops->init(hdmi, hdmi->phy.data, + &hdmi->previous_mode); + if (ret) + return ret; + + reinit_completion(&hdmi->earc_cmp); + + hdmi_modb(hdmi, EARCRX_CMDC_DISCOVERY_TIMEOUT_IRQ | + EARCRX_CMDC_DISCOVERY_DONE_IRQ, + EARCRX_CMDC_DISCOVERY_TIMEOUT_IRQ | + EARCRX_CMDC_DISCOVERY_DONE_IRQ, EARCRX_0_INT_MASK_N); + + /* start discovery */ + hdmi_modb(hdmi, EARCRX_CMDC_DISCOVERY_EN, EARCRX_CMDC_DISCOVERY_EN, + EARCRX_CMDC_CONTROL); + + /* + * The eARC TX device drives a logic-high-voltage-level + * pulse on the physical HPD connector pin, after + * at least 100 ms of low voltage level to start the + * eARC Discovery process. + */ + hdmi_modb(hdmi, EARCRX_CONNECTOR_HPD, EARCRX_CONNECTOR_HPD, + EARCRX_CMDC_CONTROL); + + stat = wait_for_completion_timeout(&hdmi->earc_cmp, HZ / 10); + if (!stat) + return -EAGAIN; + + if (hdmi->earc_intr & EARCRX_CMDC_DISCOVERY_TIMEOUT_IRQ) { + dev_err(hdmi->dev, "discovery timeout\n"); + return -ETIMEDOUT; + } else if (hdmi->earc_intr & EARCRX_CMDC_DISCOVERY_DONE_IRQ) { + dev_info(hdmi->dev, "discovery done\n"); + } else { + dev_err(hdmi->dev, "discovery failed\n"); + return -EINVAL; + } + + hdmi_writel(hdmi, 1, EARCRX_DMAC_PHY_CONTROL); + hdmi_modb(hdmi, EARCRX_CMDC_SWINIT_P, EARCRX_CMDC_SWINIT_P, + EARCRX_CMDC_CONFIG0); + + hdmi_writel(hdmi, 0xf3, EARCRX_DMAC_CONFIG); + hdmi_writel(hdmi, 0x63, EARCRX_DMAC_CONTROL0); + hdmi_writel(hdmi, 0xff, EARCRX_DMAC_CONTROL1); + + hdmi_modb(hdmi, EARCRX_XACTREAD_STOP_CFG | EARCRX_XACTREAD_RETRY_CFG | + EARCRX_CMDC_DSCVR_EARCVALID0_TO_DISC1 | EARCRX_CMDC_XACT_RESTART_EN, + EARCRX_XACTREAD_STOP_CFG | EARCRX_XACTREAD_RETRY_CFG | + EARCRX_CMDC_DSCVR_EARCVALID0_TO_DISC1 | EARCRX_CMDC_XACT_RESTART_EN, + EARCRX_CMDC_CONFIG0); + + hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER0); + hdmi_writel(hdmi, 0x1b0e, EARCRX_DMAC_CHSTATUS_STREAMER1); + hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER2); + hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER3); + hdmi_writel(hdmi, 0xf2000000, EARCRX_DMAC_CHSTATUS_STREAMER4); + hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER5); + hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER6); + hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER7); + hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER8); + + return 0; +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_earc); + +/* ----------------------------------------------------------------------------- + * HDMI TX Setup + */ + +static void hdmi_config_AVI(struct dw_hdmi_qp *hdmi, + const struct drm_connector *connector, + const struct drm_display_mode *mode) +{ + struct hdmi_avi_infoframe frame; + u32 val, i, j; + u8 buff[17]; + enum hdmi_quantization_range rgb_quant_range = + hdmi->hdmi_data.quant_range; + + /* Initialise info frame from DRM mode */ + drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode); + + /* + * Ignore monitor selectable quantization, use quantization set + * by the user + */ + drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, rgb_quant_range); + if (hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format)) + frame.colorspace = HDMI_COLORSPACE_YUV444; + else if (hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) + frame.colorspace = HDMI_COLORSPACE_YUV422; + else if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) + frame.colorspace = HDMI_COLORSPACE_YUV420; + else + frame.colorspace = HDMI_COLORSPACE_RGB; + + /* Set up colorimetry */ + if (!hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) { + switch (hdmi->hdmi_data.enc_out_encoding) { + case V4L2_YCBCR_ENC_601: + if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_XV601) + frame.colorimetry = HDMI_COLORIMETRY_EXTENDED; + else + frame.colorimetry = HDMI_COLORIMETRY_ITU_601; + frame.extended_colorimetry = + HDMI_EXTENDED_COLORIMETRY_XV_YCC_601; + break; + case V4L2_YCBCR_ENC_709: + if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_XV709) + frame.colorimetry = HDMI_COLORIMETRY_EXTENDED; + else + frame.colorimetry = HDMI_COLORIMETRY_ITU_709; + frame.extended_colorimetry = + HDMI_EXTENDED_COLORIMETRY_XV_YCC_709; + break; + case V4L2_YCBCR_ENC_BT2020: + if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_BT2020) + frame.colorimetry = HDMI_COLORIMETRY_EXTENDED; + else + frame.colorimetry = HDMI_COLORIMETRY_ITU_709; + frame.extended_colorimetry = + HDMI_EXTENDED_COLORIMETRY_BT2020; + break; + default: /* Carries no data */ + frame.colorimetry = HDMI_COLORIMETRY_ITU_601; + frame.extended_colorimetry = + HDMI_EXTENDED_COLORIMETRY_XV_YCC_601; + break; + } + } else { + frame.colorimetry = HDMI_COLORIMETRY_NONE; + frame.extended_colorimetry = + HDMI_EXTENDED_COLORIMETRY_XV_YCC_601; + } + + frame.scan_mode = HDMI_SCAN_MODE_NONE; + + hdmi_avi_infoframe_pack_only(&frame, buff, 17); + + /* + * The Designware IP uses a different byte format from standard + * AVI info frames, though generally the bits are in the correct + * bytes. + */ + + val = (frame.version << 8) | (frame.length << 16); + hdmi_writel(hdmi, val, PKT_AVI_CONTENTS0); + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + if (i * 4 + j >= 14) + break; + if (!j) + val = buff[i * 4 + j + 3]; + val |= buff[i * 4 + j + 3] << (8 * j); + } + + hdmi_writel(hdmi, val, PKT_AVI_CONTENTS1 + i * 4); + } + + hdmi_modb(hdmi, PKTSCHED_AVI_TX_EN | PKTSCHED_GCP_TX_EN, + PKTSCHED_AVI_TX_EN | PKTSCHED_GCP_TX_EN, + PKTSCHED_PKT_EN); +} + +static void hdmi_config_CVTEM(struct dw_hdmi_qp *hdmi) +{ + u8 ds_type = 0; + u8 sync = 1; + u8 vfr = 1; + u8 afr = 0; + u8 new = 1; + u8 end = 0; + u8 data_set_length = 136; + u8 hb1[6] = { 0x80, 0, 0, 0, 0, 0x40 }; + u8 *pps_body; + u32 val, i, reg; + struct drm_display_mode *mode = &hdmi->previous_mode; + int hsync, hfront, hback; + struct dw_hdmi_link_config *link_cfg; + void *data = hdmi->plat_data->phy_data; + + hdmi_modb(hdmi, 0, PKTSCHED_EMP_CVTEM_TX_EN, PKTSCHED_PKT_EN); + + if (hdmi->plat_data->get_link_cfg) { + link_cfg = hdmi->plat_data->get_link_cfg(data); + } else { + dev_err(hdmi->dev, "can't get frl link cfg\n"); + return; + } + + if (!link_cfg->dsc_mode) { + dev_info(hdmi->dev, "don't use dsc mode\n"); + return; + } + + pps_body = link_cfg->pps_payload; + + hsync = mode->hsync_end - mode->hsync_start; + hback = mode->htotal - mode->hsync_end; + hfront = mode->hsync_start - mode->hdisplay; + + for (i = 0; i < 6; i++) { + val = i << 16 | hb1[i] << 8; + hdmi_writel(hdmi, val, PKT0_EMP_CVTEM_CONTENTS0 + i * 0x20); + } + + val = new << 7 | end << 6 | ds_type << 4 | afr << 3 | + vfr << 2 | sync << 1; + hdmi_writel(hdmi, val, PKT0_EMP_CVTEM_CONTENTS1); + + val = data_set_length << 16 | pps_body[0] << 24; + hdmi_writel(hdmi, val, PKT0_EMP_CVTEM_CONTENTS2); + + reg = PKT0_EMP_CVTEM_CONTENTS3; + for (i = 1; i < 125; i++) { + if (reg == PKT1_EMP_CVTEM_CONTENTS0 || + reg == PKT2_EMP_CVTEM_CONTENTS0 || + reg == PKT3_EMP_CVTEM_CONTENTS0 || + reg == PKT4_EMP_CVTEM_CONTENTS0 || + reg == PKT5_EMP_CVTEM_CONTENTS0) { + reg += 4; + i--; + continue; + } + if (i % 4 == 1) + val = pps_body[i]; + if (i % 4 == 2) + val |= pps_body[i] << 8; + if (i % 4 == 3) + val |= pps_body[i] << 16; + if (!(i % 4)) { + val |= pps_body[i] << 24; + hdmi_writel(hdmi, val, reg); + reg += 4; + } + } + + val = (hfront & 0xff) << 24 | pps_body[127] << 16 | + pps_body[126] << 8 | pps_body[125]; + hdmi_writel(hdmi, val, PKT4_EMP_CVTEM_CONTENTS6); + + val = (hback & 0xff) << 24 | ((hsync >> 8) & 0xff) << 16 | + (hsync & 0xff) << 8 | ((hfront >> 8) & 0xff); + hdmi_writel(hdmi, val, PKT4_EMP_CVTEM_CONTENTS7); + + val = link_cfg->hcactive << 8 | ((hback >> 8) & 0xff); + hdmi_writel(hdmi, val, PKT5_EMP_CVTEM_CONTENTS1); + + for (i = PKT5_EMP_CVTEM_CONTENTS2; i <= PKT5_EMP_CVTEM_CONTENTS7; i += 4) + hdmi_writel(hdmi, 0, i); + + hdmi_modb(hdmi, PKTSCHED_EMP_CVTEM_TX_EN, PKTSCHED_EMP_CVTEM_TX_EN, + PKTSCHED_PKT_EN); +} + +static void hdmi_config_drm_infoframe(struct dw_hdmi_qp *hdmi, + const struct drm_connector *connector) +{ + const struct drm_connector_state *conn_state = connector->state; + struct hdr_output_metadata *hdr_metadata; + struct hdmi_drm_infoframe frame; + u8 buffer[30]; + ssize_t err; + int i; + u32 val; + + if (!hdmi->plat_data->use_drm_infoframe) + return; + + hdmi_modb(hdmi, 0, PKTSCHED_DRMI_TX_EN, PKTSCHED_PKT_EN); + + if (!hdmi->connector.hdr_sink_metadata.hdmi_type1.eotf) { + DRM_DEBUG("No need to set HDR metadata in infoframe\n"); + return; + } + + if (!conn_state->hdr_output_metadata) { + DRM_DEBUG("source metadata not set yet\n"); + return; + } + + hdr_metadata = (struct hdr_output_metadata *) + conn_state->hdr_output_metadata->data; + + if (!(hdmi->connector.hdr_sink_metadata.hdmi_type1.eotf & + BIT(hdr_metadata->hdmi_metadata_type1.eotf))) { + DRM_ERROR("Not support EOTF %d\n", + hdr_metadata->hdmi_metadata_type1.eotf); + return; + } + + err = drm_hdmi_infoframe_set_hdr_metadata(&frame, conn_state); + if (err < 0) + return; + + err = hdmi_drm_infoframe_pack(&frame, buffer, sizeof(buffer)); + if (err < 0) { + dev_err(hdmi->dev, "Failed to pack drm infoframe: %zd\n", err); + return; + } + + val = (frame.version << 8) | (frame.length << 16); + hdmi_writel(hdmi, val, PKT_DRMI_CONTENTS0); + + for (i = 0; i <= frame.length; i++) { + if (i % 4 == 0) + val = buffer[3 + i]; + val |= buffer[3 + i] << ((i % 4) * 8); + + if (i % 4 == 3 || (i == (frame.length))) + hdmi_writel(hdmi, val, PKT_DRMI_CONTENTS1 + ((i / 4) * 4)); + } + + hdmi_modb(hdmi, PKTSCHED_DRMI_TX_EN, PKTSCHED_DRMI_TX_EN, PKTSCHED_PKT_EN); + + DRM_DEBUG("%s eotf %d end\n", __func__, + hdr_metadata->hdmi_metadata_type1.eotf); +} + +/* Filter out invalid setups to avoid configuring SCDC and scrambling */ +static bool dw_hdmi_support_scdc(struct dw_hdmi_qp *hdmi, + const struct drm_display_info *display) +{ + /* Disable if no DDC bus */ + if (!hdmi->ddc) + return false; + + /* Disable if SCDC is not supported, or if an HF-VSDB block is absent */ + if (!display->hdmi.scdc.supported || + !display->hdmi.scdc.scrambling.supported) + return false; + + /* + * Disable if display only support low TMDS rates and scrambling + * for low rates is not supported either + */ + if (!display->hdmi.scdc.scrambling.low_rates && + display->max_tmds_clock <= 340000) + return false; + + return true; +} + +static int hdmi_set_frl_mask(int frl_rate) +{ + switch (frl_rate) { + case 48: + return FRL_12GBPS_4LANE; + case 40: + return FRL_10GBPS_4LANE; + case 32: + return FRL_8GBPS_4LANE; + case 24: + return FRL_6GBPS_4LANE; + case 18: + return FRL_6GBPS_3LANE; + case 9: + return FRL_3GBPS_3LANE; + } + + return 0; +} + +static int hdmi_start_flt(struct dw_hdmi_qp *hdmi, u8 rate) +{ + u8 val; + u8 ffe_lv = 0; + int i = 0, stat; + + /* FLT_READY & FFE_LEVELS read */ + for (i = 0; i < 20; i++) { + drm_scdc_readb(hdmi->ddc, SCDC_STATUS_FLAGS_0, &val); + if (val & BIT(6)) + break; + msleep(20); + } + + if (i == 20) { + dev_err(hdmi->dev, "sink flt isn't ready\n"); + return -EINVAL; + } + + hdmi_modb(hdmi, SCDC_UPD_FLAGS_RD_IRQ, SCDC_UPD_FLAGS_RD_IRQ, + MAINUNIT_1_INT_MASK_N); + hdmi_modb(hdmi, SCDC_UPD_FLAGS_POLL_EN | SCDC_UPD_FLAGS_AUTO_CLR, + SCDC_UPD_FLAGS_POLL_EN | SCDC_UPD_FLAGS_AUTO_CLR, + SCDC_CONFIG0); + + /* max ffe level 3 */ + val = 3 << 4 | hdmi_set_frl_mask(rate); + drm_scdc_writeb(hdmi->ddc, 0x31, val); + + /* select FRL_RATE & FFE_LEVELS */ + hdmi_writel(hdmi, ffe_lv, FLT_CONFIG0); + + /* Start LTS_3 state in source DUT */ + reinit_completion(&hdmi->flt_cmp); + hdmi_modb(hdmi, FLT_EXIT_TO_LTSP_IRQ, FLT_EXIT_TO_LTSP_IRQ, + MAINUNIT_1_INT_MASK_N); + hdmi_writel(hdmi, 1, FLT_CONTROL0); + + /* wait for completed link training at source side */ + stat = wait_for_completion_timeout(&hdmi->flt_cmp, HZ * 2); + if (!stat) { + dev_err(hdmi->dev, "wait lts3 finish time out\n"); + hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_POLL_EN | + SCDC_UPD_FLAGS_AUTO_CLR, SCDC_CONFIG0); + hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_RD_IRQ, + MAINUNIT_1_INT_MASK_N); + return -EAGAIN; + } + + if (!(hdmi->flt_intr & FLT_EXIT_TO_LTSP_IRQ)) { + dev_err(hdmi->dev, "not to ltsp\n"); + hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_POLL_EN | + SCDC_UPD_FLAGS_AUTO_CLR, SCDC_CONFIG0); + hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_RD_IRQ, + MAINUNIT_1_INT_MASK_N); + return -EINVAL; + } + + return 0; +} + +#define HDMI_MODE_FRL_MASK BIT(30) + +static void hdmi_set_op_mode(struct dw_hdmi_qp *hdmi, + struct dw_hdmi_link_config *link_cfg, + const struct drm_connector *connector) +{ + int frl_rate; + + hdmi_writel(hdmi, 0, FLT_CONFIG0); + if (dw_hdmi_support_scdc(hdmi, &connector->display_info)) + drm_scdc_writeb(hdmi->ddc, 0x31, 0); + msleep(20); + if (!link_cfg->frl_mode) { + dev_info(hdmi->dev, "dw hdmi qp use tmds mode\n"); + hdmi_modb(hdmi, 0, OPMODE_FRL, LINK_CONFIG0); + hdmi_modb(hdmi, 0, OPMODE_FRL_4LANES, LINK_CONFIG0); + return; + } + + if (link_cfg->frl_lanes == 4) + hdmi_modb(hdmi, OPMODE_FRL_4LANES, OPMODE_FRL_4LANES, + LINK_CONFIG0); + else + hdmi_modb(hdmi, 0, OPMODE_FRL_4LANES, LINK_CONFIG0); + + hdmi_modb(hdmi, 1, OPMODE_FRL, LINK_CONFIG0); + + frl_rate = link_cfg->frl_lanes * link_cfg->rate_per_lane; + hdmi_start_flt(hdmi, frl_rate); +} + +static unsigned long +hdmi_get_tmdsclock(struct dw_hdmi_qp *hdmi, unsigned long mpixelclock) +{ + unsigned long tmdsclock = mpixelclock; + unsigned int depth = + hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format); + + if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) { + switch (depth) { + case 16: + tmdsclock = mpixelclock * 2; + break; + case 12: + tmdsclock = mpixelclock * 3 / 2; + break; + case 10: + tmdsclock = mpixelclock * 5 / 4; + break; + default: + break; + } + } + + return tmdsclock; +} + +static int dw_hdmi_qp_setup(struct dw_hdmi_qp *hdmi, + const struct drm_connector *connector, + const struct drm_display_mode *mode) +{ + int ret; + void *data = hdmi->plat_data->phy_data; + struct hdmi_vmode_qp *vmode = &hdmi->hdmi_data.video_mode; + struct dw_hdmi_link_config *link_cfg; + u8 bytes = 0; + + hdmi->vic = drm_match_cea_mode(mode); + + if (!hdmi->vic) + dev_dbg(hdmi->dev, "Non-CEA mode used in HDMI\n"); + else + dev_dbg(hdmi->dev, "CEA mode used vic=%d\n", hdmi->vic); + + if (hdmi->plat_data->get_enc_out_encoding) + hdmi->hdmi_data.enc_out_encoding = + hdmi->plat_data->get_enc_out_encoding(data); + else if ((hdmi->vic == 6) || (hdmi->vic == 7) || + (hdmi->vic == 21) || (hdmi->vic == 22) || + (hdmi->vic == 2) || (hdmi->vic == 3) || + (hdmi->vic == 17) || (hdmi->vic == 18)) + hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_601; + else + hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_709; + + if (mode->flags & DRM_MODE_FLAG_DBLCLK) { + hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 1; + hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 1; + } else { + hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0; + hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0; + } + /* Get input format from plat data or fallback to RGB888 */ + if (hdmi->plat_data->get_input_bus_format) + hdmi->hdmi_data.enc_in_bus_format = + hdmi->plat_data->get_input_bus_format(data); + else if (hdmi->plat_data->input_bus_format) + hdmi->hdmi_data.enc_in_bus_format = + hdmi->plat_data->input_bus_format; + else + hdmi->hdmi_data.enc_in_bus_format = MEDIA_BUS_FMT_RGB888_1X24; + + /* Default to RGB888 output format */ + if (hdmi->plat_data->get_output_bus_format) + hdmi->hdmi_data.enc_out_bus_format = + hdmi->plat_data->get_output_bus_format(data); + else + hdmi->hdmi_data.enc_out_bus_format = MEDIA_BUS_FMT_RGB888_1X24; + + /* Get input encoding from plat data or fallback to none */ + if (hdmi->plat_data->get_enc_in_encoding) + hdmi->hdmi_data.enc_in_encoding = + hdmi->plat_data->get_enc_in_encoding(data); + else if (hdmi->plat_data->input_bus_encoding) + hdmi->hdmi_data.enc_in_encoding = + hdmi->plat_data->input_bus_encoding; + else + hdmi->hdmi_data.enc_in_encoding = V4L2_YCBCR_ENC_DEFAULT; + + if (hdmi->plat_data->get_quant_range) + hdmi->hdmi_data.quant_range = + hdmi->plat_data->get_quant_range(data); + else + hdmi->hdmi_data.quant_range = HDMI_QUANTIZATION_RANGE_DEFAULT; + + if (hdmi->plat_data->get_link_cfg) + link_cfg = hdmi->plat_data->get_link_cfg(data); + else + return -EINVAL; + + hdmi->phy.ops->set_mode(hdmi, hdmi->phy.data, HDMI_MODE_FRL_MASK, + link_cfg->frl_mode); + + /* + * According to the dw-hdmi specification 6.4.2 + * vp_pr_cd[3:0]: + * 0000b: No pixel repetition (pixel sent only once) + * 0001b: Pixel sent two times (pixel repeated once) + */ + hdmi->hdmi_data.pix_repet_factor = + (mode->flags & DRM_MODE_FLAG_DBLCLK) ? 1 : 0; + hdmi->hdmi_data.video_mode.mdataenablepolarity = true; + + vmode->previous_pixelclock = vmode->mpixelclock; + vmode->mpixelclock = mode->crtc_clock * 1000; + if ((mode->flags & DRM_MODE_FLAG_3D_MASK) == DRM_MODE_FLAG_3D_FRAME_PACKING) + vmode->mpixelclock *= 2; + dev_dbg(hdmi->dev, "final pixclk = %ld\n", vmode->mpixelclock); + vmode->previous_tmdsclock = vmode->mtmdsclock; + vmode->mtmdsclock = hdmi_get_tmdsclock(hdmi, vmode->mpixelclock); + if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) + vmode->mtmdsclock /= 2; + dev_info(hdmi->dev, "final tmdsclk = %d\n", vmode->mtmdsclock); + + ret = hdmi->phy.ops->init(hdmi, hdmi->phy.data, &hdmi->previous_mode); + if (ret) + return ret; + + if (hdmi->sink_has_audio) { + dev_dbg(hdmi->dev, "sink has audio support\n"); + + /* HDMI Initialization Step E - Configure audio */ + hdmi_clk_regenerator_update_pixel_clock(hdmi); + hdmi_enable_audio_clk(hdmi, hdmi->audio_enable); + } + + /* not for DVI mode */ + if (hdmi->sink_is_hdmi) { + dev_dbg(hdmi->dev, "%s HDMI mode\n", __func__); + hdmi_modb(hdmi, 0, OPMODE_DVI, LINK_CONFIG0); + hdmi_modb(hdmi, HDCP2_BYPASS, HDCP2_BYPASS, HDCP2LOGIC_CONFIG0); + if (!link_cfg->frl_mode) { + if (vmode->mtmdsclock > HDMI14_MAX_TMDSCLK) { + drm_scdc_readb(hdmi->ddc, SCDC_SINK_VERSION, &bytes); + drm_scdc_writeb(hdmi->ddc, SCDC_SOURCE_VERSION, + min_t(u8, bytes, SCDC_MIN_SOURCE_VERSION)); + drm_scdc_set_high_tmds_clock_ratio(hdmi->ddc, 1); + drm_scdc_set_scrambling(hdmi->ddc, 1); + hdmi_writel(hdmi, 1, SCRAMB_CONFIG0); + } else { + if (dw_hdmi_support_scdc(hdmi, &connector->display_info)) { + drm_scdc_set_high_tmds_clock_ratio(hdmi->ddc, 0); + drm_scdc_set_scrambling(hdmi->ddc, 0); + } + hdmi_writel(hdmi, 0, SCRAMB_CONFIG0); + } + } + /* HDMI Initialization Step F - Configure AVI InfoFrame */ + hdmi_config_AVI(hdmi, connector, mode); + hdmi_config_CVTEM(hdmi); + hdmi_config_drm_infoframe(hdmi, connector); + hdmi_set_op_mode(hdmi, link_cfg, connector); + } else { + hdmi_modb(hdmi, OPMODE_DVI, OPMODE_DVI, LINK_CONFIG0); + dev_info(hdmi->dev, "%s DVI mode\n", __func__); + } + + return 0; +} + +static enum drm_connector_status +dw_hdmi_connector_detect(struct drm_connector *connector, bool force) +{ + struct dw_hdmi_qp *hdmi = + container_of(connector, struct dw_hdmi_qp, connector); + + mutex_lock(&hdmi->mutex); + hdmi->force = DRM_FORCE_UNSPECIFIED; + mutex_unlock(&hdmi->mutex); + + return hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data); +} + +static int +dw_hdmi_update_hdr_property(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct dw_hdmi_qp *hdmi = container_of(connector, struct dw_hdmi_qp, + connector); + void *data = hdmi->plat_data->phy_data; + const struct hdr_static_metadata *metadata = + &connector->hdr_sink_metadata.hdmi_type1; + size_t size = sizeof(*metadata); + struct drm_property *property; + struct drm_property_blob *blob; + int ret; + + if (hdmi->plat_data->get_hdr_property) + property = hdmi->plat_data->get_hdr_property(data); + else + return -EINVAL; + + if (hdmi->plat_data->get_hdr_blob) + blob = hdmi->plat_data->get_hdr_blob(data); + else + return -EINVAL; + + ret = drm_property_replace_global_blob(dev, &blob, size, metadata, + &connector->base, property); + return ret; +} + +static int dw_hdmi_connector_get_modes(struct drm_connector *connector) +{ + struct dw_hdmi_qp *hdmi = + container_of(connector, struct dw_hdmi_qp, connector); + struct hdr_static_metadata *metedata = + &connector->hdr_sink_metadata.hdmi_type1; + struct edid *edid; + struct drm_display_mode *mode; + struct drm_display_info *info = &connector->display_info; + void *data = hdmi->plat_data->phy_data; + int i, ret = 0; + + if (!hdmi->ddc) + return 0; + + memset(metedata, 0, sizeof(*metedata)); + edid = drm_get_edid(connector, hdmi->ddc); + if (edid) { + dev_dbg(hdmi->dev, "got edid: width[%d] x height[%d]\n", + edid->width_cm, edid->height_cm); + + hdmi->sink_is_hdmi = drm_detect_hdmi_monitor(edid); + hdmi->sink_has_audio = drm_detect_monitor_audio(edid); + drm_connector_update_edid_property(connector, edid); + cec_notifier_set_phys_addr_from_edid(hdmi->cec_notifier, edid); + if (hdmi->plat_data->get_edid_dsc_info) + hdmi->plat_data->get_edid_dsc_info(data, edid); + ret = drm_add_edid_modes(connector, edid); + dw_hdmi_update_hdr_property(connector); + kfree(edid); + } else { + hdmi->sink_is_hdmi = true; + hdmi->sink_has_audio = true; + + for (i = 0; i < ARRAY_SIZE(dw_hdmi_default_modes); i++) { + const struct drm_display_mode *ptr = + &dw_hdmi_default_modes[i]; + + mode = drm_mode_duplicate(connector->dev, ptr); + if (mode) { + if (!i) { + mode->type = DRM_MODE_TYPE_PREFERRED; + mode->picture_aspect_ratio = + HDMI_PICTURE_ASPECT_NONE; + } + drm_mode_probed_add(connector, mode); + ret++; + } + } + info->edid_hdmi_dc_modes = 0; + info->hdmi.y420_dc_modes = 0; + info->color_formats = 0; + + dev_info(hdmi->dev, "failed to get edid\n"); + } + + return ret; +} + +static int +dw_hdmi_atomic_connector_set_property(struct drm_connector *connector, + struct drm_connector_state *state, + struct drm_property *property, + uint64_t val) +{ + struct dw_hdmi_qp *hdmi = + container_of(connector, struct dw_hdmi_qp, connector); + const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops; + + if (ops && ops->set_property) + return ops->set_property(connector, state, property, + val, hdmi->plat_data->phy_data); + else + return -EINVAL; +} + +static int +dw_hdmi_atomic_connector_get_property(struct drm_connector *connector, + const struct drm_connector_state *state, + struct drm_property *property, + uint64_t *val) +{ + struct dw_hdmi_qp *hdmi = + container_of(connector, struct dw_hdmi_qp, connector); + const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops; + + if (ops && ops->get_property) + return ops->get_property(connector, state, property, + val, hdmi->plat_data->phy_data); + else + return -EINVAL; +} + +static int +dw_hdmi_connector_set_property(struct drm_connector *connector, + struct drm_property *property, uint64_t val) +{ + return dw_hdmi_atomic_connector_set_property(connector, NULL, + property, val); +} + +static void dw_hdmi_attach_properties(struct dw_hdmi_qp *hdmi) +{ + unsigned int color = MEDIA_BUS_FMT_RGB888_1X24; + const struct dw_hdmi_property_ops *ops = + hdmi->plat_data->property_ops; + + if (ops && ops->attach_properties) + return ops->attach_properties(&hdmi->connector, color, 0, + hdmi->plat_data->phy_data); +} + +static void dw_hdmi_destroy_properties(struct dw_hdmi_qp *hdmi) +{ + const struct dw_hdmi_property_ops *ops = + hdmi->plat_data->property_ops; + + if (ops && ops->destroy_properties) + return ops->destroy_properties(&hdmi->connector, + hdmi->plat_data->phy_data); +} + +static struct drm_encoder * +dw_hdmi_connector_best_encoder(struct drm_connector *connector) +{ + struct dw_hdmi_qp *hdmi = + container_of(connector, struct dw_hdmi_qp, connector); + + return hdmi->bridge.encoder; +} + +static bool dw_hdmi_color_changed(struct drm_connector *connector) +{ + struct dw_hdmi_qp *hdmi = + container_of(connector, struct dw_hdmi_qp, connector); + void *data = hdmi->plat_data->phy_data; + bool ret = false; + + if (hdmi->plat_data->get_color_changed) + ret = hdmi->plat_data->get_color_changed(data); + + return ret; +} + +static bool hdr_metadata_equal(const struct drm_connector_state *old_state, + const struct drm_connector_state *new_state) +{ + struct drm_property_blob *old_blob = old_state->hdr_output_metadata; + struct drm_property_blob *new_blob = new_state->hdr_output_metadata; + + if (!old_blob || !new_blob) + return old_blob == new_blob; + + if (old_blob->length != new_blob->length) + return false; + + return !memcmp(old_blob->data, new_blob->data, old_blob->length); +} + +static int dw_hdmi_connector_atomic_check(struct drm_connector *connector, + struct drm_atomic_state *state) +{ + struct drm_connector_state *old_state = + drm_atomic_get_old_connector_state(state, connector); + struct drm_connector_state *new_state = + drm_atomic_get_new_connector_state(state, connector); + struct drm_crtc *crtc = new_state->crtc; + struct drm_crtc_state *crtc_state; + struct dw_hdmi_qp *hdmi = + container_of(connector, struct dw_hdmi_qp, connector); + struct drm_display_mode *mode = NULL; + void *data = hdmi->plat_data->phy_data; + struct hdmi_vmode_qp *vmode = &hdmi->hdmi_data.video_mode; + unsigned int in_bus_format = hdmi->hdmi_data.enc_in_bus_format; + unsigned int out_bus_format = hdmi->hdmi_data.enc_out_bus_format; + bool color_changed = false; + + if (!crtc) + return 0; + + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + /* + * If HDMI is enabled in uboot, it's need to record + * drm_display_mode and set phy status to enabled. + */ + if (!vmode->mpixelclock) { + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (hdmi->plat_data->get_enc_in_encoding) + hdmi->hdmi_data.enc_in_encoding = + hdmi->plat_data->get_enc_in_encoding(data); + if (hdmi->plat_data->get_enc_out_encoding) + hdmi->hdmi_data.enc_out_encoding = + hdmi->plat_data->get_enc_out_encoding(data); + if (hdmi->plat_data->get_input_bus_format) + hdmi->hdmi_data.enc_in_bus_format = + hdmi->plat_data->get_input_bus_format(data); + if (hdmi->plat_data->get_output_bus_format) + hdmi->hdmi_data.enc_out_bus_format = + hdmi->plat_data->get_output_bus_format(data); + + mode = &crtc_state->mode; + memcpy(&hdmi->previous_mode, mode, sizeof(hdmi->previous_mode)); + vmode->mpixelclock = mode->crtc_clock * 1000; + vmode->previous_pixelclock = mode->clock; + vmode->previous_tmdsclock = mode->clock; + vmode->mtmdsclock = hdmi_get_tmdsclock(hdmi, + vmode->mpixelclock); + if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) + vmode->mtmdsclock /= 2; + + if (in_bus_format != hdmi->hdmi_data.enc_in_bus_format || + out_bus_format != hdmi->hdmi_data.enc_out_bus_format) + color_changed = true; + } + + if (!hdr_metadata_equal(old_state, new_state) || + dw_hdmi_color_changed(connector) || color_changed) { + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + crtc_state->mode_changed = true; + } + + return 0; +} + +static void dw_hdmi_connector_force(struct drm_connector *connector) +{ + struct dw_hdmi_qp *hdmi = + container_of(connector, struct dw_hdmi_qp, connector); + + mutex_lock(&hdmi->mutex); + + if (hdmi->force != connector->force) { + if (!hdmi->disabled && connector->force == DRM_FORCE_OFF) + extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, + false); + else if (hdmi->disabled && connector->force == DRM_FORCE_ON) + extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, + true); + } + + hdmi->force = connector->force; + mutex_unlock(&hdmi->mutex); +} + +static int dw_hdmi_qp_fill_modes(struct drm_connector *connector, u32 max_x, + u32 max_y) +{ + return drm_helper_probe_single_connector_modes(connector, 9000, 9000); +} + +static const struct drm_connector_funcs dw_hdmi_connector_funcs = { + .fill_modes = dw_hdmi_qp_fill_modes, + .detect = dw_hdmi_connector_detect, + .destroy = drm_connector_cleanup, + .force = dw_hdmi_connector_force, + .reset = drm_atomic_helper_connector_reset, + .set_property = dw_hdmi_connector_set_property, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, + .atomic_set_property = dw_hdmi_atomic_connector_set_property, + .atomic_get_property = dw_hdmi_atomic_connector_get_property, +}; + +static const struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = { + .get_modes = dw_hdmi_connector_get_modes, + .best_encoder = dw_hdmi_connector_best_encoder, + .atomic_check = dw_hdmi_connector_atomic_check, +}; + +static int dw_hdmi_qp_bridge_attach(struct drm_bridge *bridge, + enum drm_bridge_attach_flags flags) +{ + struct dw_hdmi_qp *hdmi = bridge->driver_private; + struct drm_encoder *encoder = bridge->encoder; + struct drm_connector *connector = &hdmi->connector; + struct cec_connector_info conn_info; + struct cec_notifier *notifier; + + connector->interlace_allowed = 1; + connector->polled = DRM_CONNECTOR_POLL_HPD; + + drm_connector_helper_add(connector, &dw_hdmi_connector_helper_funcs); + + drm_connector_init(bridge->dev, connector, &dw_hdmi_connector_funcs, + DRM_MODE_CONNECTOR_HDMIA); + + drm_connector_attach_encoder(connector, encoder); + dw_hdmi_attach_properties(hdmi); + + cec_fill_conn_info_from_drm(&conn_info, connector); + notifier = cec_notifier_conn_register(hdmi->dev, NULL, &conn_info); + if (!notifier) + return -ENOMEM; + + mutex_lock(&hdmi->cec_notifier_mutex); + hdmi->cec_notifier = notifier; + mutex_unlock(&hdmi->cec_notifier_mutex); + + return 0; +} + +static void dw_hdmi_qp_bridge_detach(struct drm_bridge *bridge) +{ + struct dw_hdmi_qp *hdmi = bridge->driver_private; + + mutex_lock(&hdmi->cec_notifier_mutex); + cec_notifier_conn_unregister(hdmi->cec_notifier); + hdmi->cec_notifier = NULL; + mutex_unlock(&hdmi->cec_notifier_mutex); +} + +static enum drm_mode_status +dw_hdmi_qp_bridge_mode_valid(struct drm_bridge *bridge, + const struct drm_display_info *info, + const struct drm_display_mode *mode) +{ + struct dw_hdmi_qp *hdmi = bridge->driver_private; + struct drm_connector *connector = &hdmi->connector; + const struct dw_hdmi_plat_data *pdata = hdmi->plat_data; + enum drm_mode_status mode_status = MODE_OK; + + if (pdata->mode_valid) + mode_status = pdata->mode_valid(connector, pdata->priv_data, + info, mode); + + return mode_status; +} + +static void dw_hdmi_qp_bridge_mode_set(struct drm_bridge *bridge, + const struct drm_display_mode *orig_mode, + const struct drm_display_mode *mode) +{ + struct dw_hdmi_qp *hdmi = bridge->driver_private; + + mutex_lock(&hdmi->mutex); + + /* Store the display mode for plugin/DKMS poweron events */ + memcpy(&hdmi->previous_mode, mode, sizeof(hdmi->previous_mode)); + + mutex_unlock(&hdmi->mutex); +} + +static void dw_hdmi_qp_bridge_atomic_disable(struct drm_bridge *bridge, + struct drm_bridge_state *old_state) +{ + struct dw_hdmi_qp *hdmi = bridge->driver_private; + + extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, false); + handle_plugged_change(hdmi, false); + mutex_lock(&hdmi->mutex); + hdmi->disabled = true; + hdmi->curr_conn = NULL; + hdmi_writel(hdmi, 0, PKTSCHED_PKT_EN); + if (hdmi->phy.ops->disable) + hdmi->phy.ops->disable(hdmi, hdmi->phy.data); + mutex_unlock(&hdmi->mutex); +} + +static void dw_hdmi_qp_bridge_atomic_enable(struct drm_bridge *bridge, + struct drm_bridge_state *old_state) +{ + struct dw_hdmi_qp *hdmi = bridge->driver_private; + struct drm_atomic_state *state = old_state->base.state; + struct drm_connector *connector; + + connector = drm_atomic_get_new_connector_for_encoder(state, + bridge->encoder); + + mutex_lock(&hdmi->mutex); + hdmi->disabled = false; + hdmi->curr_conn = connector; + dw_hdmi_qp_setup(hdmi, hdmi->curr_conn, &hdmi->previous_mode); + mutex_unlock(&hdmi->mutex); + extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, true); + handle_plugged_change(hdmi, true); +} + +static const struct drm_bridge_funcs dw_hdmi_bridge_funcs = { + .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, + .atomic_reset = drm_atomic_helper_bridge_reset, + .attach = dw_hdmi_qp_bridge_attach, + .detach = dw_hdmi_qp_bridge_detach, + .mode_set = dw_hdmi_qp_bridge_mode_set, + .mode_valid = dw_hdmi_qp_bridge_mode_valid, + .atomic_enable = dw_hdmi_qp_bridge_atomic_enable, + .atomic_disable = dw_hdmi_qp_bridge_atomic_disable, +}; + +void dw_hdmi_qp_set_cec_adap(struct dw_hdmi_qp *hdmi, struct cec_adapter *adap) +{ + hdmi->cec_adap = adap; +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_cec_adap); + +static irqreturn_t dw_hdmi_qp_main_hardirq(int irq, void *dev_id) +{ + struct dw_hdmi_qp *hdmi = dev_id; + struct dw_hdmi_qp_i2c *i2c = hdmi->i2c; + u32 stat; + + stat = hdmi_readl(hdmi, MAINUNIT_1_INT_STATUS); + + i2c->stat = stat & (I2CM_OP_DONE_IRQ | I2CM_READ_REQUEST_IRQ | + I2CM_NACK_RCVD_IRQ); + hdmi->scdc_intr = stat & (SCDC_UPD_FLAGS_RD_IRQ | + SCDC_UPD_FLAGS_CHG_IRQ | + SCDC_UPD_FLAGS_CLR_IRQ | + SCDC_RR_REPLY_STOP_IRQ | + SCDC_NACK_RCVD_IRQ); + hdmi->flt_intr = stat & (FLT_EXIT_TO_LTSP_IRQ | + FLT_EXIT_TO_LTS4_IRQ | + FLT_EXIT_TO_LTSL_IRQ); + + dev_dbg(hdmi->dev, "i2c main unit irq:%#x\n", stat); + if (i2c->stat) { + hdmi_writel(hdmi, i2c->stat, MAINUNIT_1_INT_CLEAR); + complete(&i2c->cmp); + } + + if (hdmi->flt_intr) { + dev_dbg(hdmi->dev, "i2c flt irq:%#x\n", hdmi->flt_intr); + hdmi_writel(hdmi, hdmi->flt_intr, MAINUNIT_1_INT_CLEAR); + complete(&hdmi->flt_cmp); + } + + if (hdmi->scdc_intr) { + u8 val; + + dev_dbg(hdmi->dev, "i2c scdc irq:%#x\n", hdmi->scdc_intr); + hdmi_writel(hdmi, hdmi->scdc_intr, MAINUNIT_1_INT_CLEAR); + val = hdmi_readl(hdmi, SCDC_STATUS0); + + /* frl start */ + if (val & BIT(4)) { + hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_POLL_EN | + SCDC_UPD_FLAGS_AUTO_CLR, SCDC_CONFIG0); + hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_RD_IRQ, + MAINUNIT_1_INT_MASK_N); + dev_info(hdmi->dev, "frl start\n"); + } + + } + + if (stat) + return IRQ_HANDLED; + + return IRQ_NONE; +} + +static irqreturn_t dw_hdmi_qp_avp_hardirq(int irq, void *dev_id) +{ + struct dw_hdmi_qp *hdmi = dev_id; + u32 stat; + + stat = hdmi_readl(hdmi, AVP_1_INT_STATUS); + if (stat) { + dev_dbg(hdmi->dev, "HDCP irq %#x\n", stat); + stat &= ~stat; + hdmi_writel(hdmi, stat, AVP_1_INT_MASK_N); + return IRQ_WAKE_THREAD; + } + + return IRQ_NONE; +} + +static irqreturn_t dw_hdmi_qp_earc_hardirq(int irq, void *dev_id) +{ + struct dw_hdmi_qp *hdmi = dev_id; + u32 stat; + + stat = hdmi_readl(hdmi, EARCRX_0_INT_STATUS); + if (stat) { + dev_dbg(hdmi->dev, "earc irq %#x\n", stat); + stat &= ~stat; + hdmi_writel(hdmi, stat, EARCRX_0_INT_MASK_N); + return IRQ_WAKE_THREAD; + } + + return IRQ_NONE; +} + +static irqreturn_t dw_hdmi_qp_avp_irq(int irq, void *dev_id) +{ + struct dw_hdmi_qp *hdmi = dev_id; + u32 stat; + + stat = hdmi_readl(hdmi, AVP_1_INT_STATUS); + + if (!stat) + return IRQ_NONE; + + hdmi_writel(hdmi, stat, AVP_1_INT_CLEAR); + + return IRQ_HANDLED; +} + +static irqreturn_t dw_hdmi_qp_earc_irq(int irq, void *dev_id) +{ + struct dw_hdmi_qp *hdmi = dev_id; + u32 stat; + + stat = hdmi_readl(hdmi, EARCRX_0_INT_STATUS); + + if (!stat) + return IRQ_NONE; + + hdmi_writel(hdmi, stat, EARCRX_0_INT_CLEAR); + + hdmi->earc_intr = stat; + complete(&hdmi->earc_cmp); + + return IRQ_HANDLED; +} + +static int dw_hdmi_detect_phy(struct dw_hdmi_qp *hdmi) +{ + u8 phy_type; + + phy_type = hdmi->plat_data->phy_force_vendor ? + DW_HDMI_PHY_VENDOR_PHY : 0; + + if (phy_type == DW_HDMI_PHY_VENDOR_PHY) { + /* Vendor PHYs require support from the glue layer. */ + if (!hdmi->plat_data->qp_phy_ops || !hdmi->plat_data->phy_name) { + dev_err(hdmi->dev, + "Vendor HDMI PHY not supported by glue layer\n"); + return -ENODEV; + } + + hdmi->phy.ops = hdmi->plat_data->qp_phy_ops; + hdmi->phy.data = hdmi->plat_data->phy_data; + hdmi->phy.name = hdmi->plat_data->phy_name; + } + + return 0; +} + +void dw_hdmi_qp_cec_set_hpd(struct dw_hdmi_qp *hdmi, bool plug_in, bool change) +{ + enum drm_connector_status status = plug_in ? + connector_status_connected : connector_status_disconnected; + + if (!plug_in) + cec_notifier_set_phys_addr(hdmi->cec_notifier, + CEC_PHYS_ADDR_INVALID); + + if (hdmi->bridge.dev) { + if (change && hdmi->cec_adap && + hdmi->cec_adap->devnode.registered) + cec_queue_pin_hpd_event(hdmi->cec_adap, + hdmi->hpd_state, + ktime_get()); + drm_bridge_hpd_notify(&hdmi->bridge, status); + } +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_cec_set_hpd); + +static const struct regmap_config hdmi_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .max_register = EARCRX_1_INT_FORCE, +}; + +static struct dw_hdmi_qp * +__dw_hdmi_probe(struct platform_device *pdev, + const struct dw_hdmi_plat_data *plat_data) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct device_node *ddc_node; + struct dw_hdmi_qp *hdmi; + struct dw_hdmi_qp_i2s_audio_data audio; + struct platform_device_info pdevinfo; + struct resource *iores = NULL; + int irq; + int ret; + + hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL); + if (!hdmi) + return ERR_PTR(-ENOMEM); + + hdmi->connector.stereo_allowed = 1; + hdmi->plat_data = plat_data; + hdmi->dev = dev; + hdmi->sample_rate = 48000; + hdmi->disabled = true; + + mutex_init(&hdmi->mutex); + mutex_init(&hdmi->audio_mutex); + mutex_init(&hdmi->cec_notifier_mutex); + spin_lock_init(&hdmi->audio_lock); + + ddc_node = of_parse_phandle(np, "ddc-i2c-bus", 0); + if (ddc_node) { + hdmi->ddc = of_get_i2c_adapter_by_node(ddc_node); + of_node_put(ddc_node); + if (!hdmi->ddc) { + dev_dbg(hdmi->dev, "failed to read ddc node\n"); + return ERR_PTR(-EPROBE_DEFER); + } + + } else { + dev_dbg(hdmi->dev, "no ddc property found\n"); + } + + if (!plat_data->regm) { + const struct regmap_config *reg_config; + + reg_config = &hdmi_regmap_config; + + iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); + hdmi->regs = devm_ioremap_resource(dev, iores); + if (IS_ERR(hdmi->regs)) { + ret = PTR_ERR(hdmi->regs); + goto err_res; + } + + hdmi->regm = devm_regmap_init_mmio(dev, hdmi->regs, reg_config); + if (IS_ERR(hdmi->regm)) { + dev_err(dev, "Failed to configure regmap\n"); + ret = PTR_ERR(hdmi->regm); + goto err_res; + } + } else { + hdmi->regm = plat_data->regm; + } + + ret = dw_hdmi_detect_phy(hdmi); + if (ret < 0) + goto err_res; + + hdmi_writel(hdmi, 0, MAINUNIT_0_INT_MASK_N); + hdmi_writel(hdmi, 0, MAINUNIT_1_INT_MASK_N); + hdmi_writel(hdmi, 428571429, TIMER_BASE_CONFIG0); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + ret = irq; + goto err_res; + } + + hdmi->avp_irq = irq; + ret = devm_request_threaded_irq(dev, hdmi->avp_irq, + dw_hdmi_qp_avp_hardirq, + dw_hdmi_qp_avp_irq, IRQF_SHARED, + dev_name(dev), hdmi); + if (ret) + goto err_res; + + irq = platform_get_irq(pdev, 2); + if (irq < 0) { + ret = irq; + goto err_res; + } + + hdmi->earc_irq = irq; + ret = devm_request_threaded_irq(dev, hdmi->earc_irq, + dw_hdmi_qp_earc_hardirq, + dw_hdmi_qp_earc_irq, IRQF_SHARED, + dev_name(dev), hdmi); + if (ret) + goto err_res; + + irq = platform_get_irq(pdev, 3); + if (irq < 0) { + ret = irq; + goto err_res; + } + + hdmi->main_irq = irq; + ret = devm_request_threaded_irq(dev, hdmi->main_irq, + dw_hdmi_qp_main_hardirq, NULL, + IRQF_SHARED, dev_name(dev), hdmi); + if (ret) + goto err_res; + + hdmi_init_clk_regenerator(hdmi); + + /* If DDC bus is not specified, try to register HDMI I2C bus */ + if (!hdmi->ddc) { + hdmi->ddc = dw_hdmi_i2c_adapter(hdmi); + if (IS_ERR(hdmi->ddc)) + hdmi->ddc = NULL; + /* + * Read high and low time from device tree. If not available use + * the default timing scl clock rate is about 99.6KHz. + */ + if (of_property_read_u32(np, "ddc-i2c-scl-high-time-ns", + &hdmi->i2c->scl_high_ns)) + hdmi->i2c->scl_high_ns = 4708; + if (of_property_read_u32(np, "ddc-i2c-scl-low-time-ns", + &hdmi->i2c->scl_low_ns)) + hdmi->i2c->scl_low_ns = 4916; + } + + hdmi->bridge.driver_private = hdmi; + hdmi->bridge.funcs = &dw_hdmi_bridge_funcs; +#ifdef CONFIG_OF + hdmi->bridge.of_node = pdev->dev.of_node; +#endif + + if (hdmi->phy.ops->setup_hpd) + hdmi->phy.ops->setup_hpd(hdmi, hdmi->phy.data); + + hdmi->connector.ycbcr_420_allowed = hdmi->plat_data->ycbcr_420_allowed; + + audio.hdmi = hdmi; + audio.eld = hdmi->connector.eld; + audio.write = hdmi_writel; + audio.read = hdmi_readl; + audio.mod = hdmi_modb; + hdmi->enable_audio = dw_hdmi_i2s_audio_enable; + hdmi->disable_audio = dw_hdmi_i2s_audio_disable; + + memset(&pdevinfo, 0, sizeof(pdevinfo)); + pdevinfo.parent = dev; + pdevinfo.id = PLATFORM_DEVID_AUTO; + pdevinfo.name = "dw-hdmi-qp-i2s-audio"; + pdevinfo.data = &audio; + pdevinfo.size_data = sizeof(audio); + pdevinfo.dma_mask = DMA_BIT_MASK(32); + hdmi->audio = platform_device_register_full(&pdevinfo); + + hdmi->extcon = devm_extcon_dev_allocate(hdmi->dev, dw_hdmi_cable); + if (IS_ERR(hdmi->extcon)) { + dev_err(hdmi->dev, "allocate extcon failed\n"); + ret = PTR_ERR(hdmi->extcon); + goto err_res; + } + + ret = devm_extcon_dev_register(hdmi->dev, hdmi->extcon); + if (ret) { + dev_err(hdmi->dev, "failed to register extcon: %d\n", ret); + goto err_res; + } + + ret = extcon_set_property_capability(hdmi->extcon, EXTCON_DISP_HDMI, + EXTCON_PROP_DISP_HPD); + if (ret) { + dev_err(hdmi->dev, + "failed to set USB property capability: %d\n", ret); + goto err_res; + } + + /* Reset HDMI DDC I2C master controller and mute I2CM interrupts */ + if (hdmi->i2c) + dw_hdmi_i2c_init(hdmi); + + init_completion(&hdmi->flt_cmp); + init_completion(&hdmi->earc_cmp); + + if (of_property_read_bool(np, "scramble-low-rates")) + hdmi->scramble_low_rates = true; + + return hdmi; + +err_res: + if (hdmi->i2c) + i2c_del_adapter(&hdmi->i2c->adap); + else + i2c_put_adapter(hdmi->ddc); + + return ERR_PTR(ret); +} + +static void __dw_hdmi_remove(struct dw_hdmi_qp *hdmi) +{ + if (hdmi->avp_irq) + disable_irq(hdmi->avp_irq); + + if (hdmi->main_irq) + disable_irq(hdmi->main_irq); + + if (hdmi->earc_irq) + disable_irq(hdmi->earc_irq); + + dw_hdmi_destroy_properties(hdmi); + hdmi->connector.funcs->destroy(&hdmi->connector); + + if (hdmi->audio && !IS_ERR(hdmi->audio)) + platform_device_unregister(hdmi->audio); + + if (hdmi->bridge.encoder) + hdmi->bridge.encoder->funcs->destroy(hdmi->bridge.encoder); + + if (hdmi->i2c) + i2c_del_adapter(&hdmi->i2c->adap); + else + i2c_put_adapter(hdmi->ddc); +} + +/* ----------------------------------------------------------------------------- + * Bind/unbind API, used from platforms based on the component framework. + */ +struct dw_hdmi_qp *dw_hdmi_qp_bind(struct platform_device *pdev, + struct drm_encoder *encoder, + struct dw_hdmi_plat_data *plat_data) +{ + struct dw_hdmi_qp *hdmi; + int ret; + + hdmi = __dw_hdmi_probe(pdev, plat_data); + if (IS_ERR(hdmi)) + return hdmi; + + ret = drm_bridge_attach(encoder, &hdmi->bridge, NULL, 0); + if (ret) { + __dw_hdmi_remove(hdmi); + dev_err(hdmi->dev, "Failed to initialize bridge with drm\n"); + return ERR_PTR(ret); + } + + plat_data->connector = &hdmi->connector; + + return hdmi; +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_bind); + +void dw_hdmi_qp_unbind(struct dw_hdmi_qp *hdmi) +{ + __dw_hdmi_remove(hdmi); +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_unbind); + +void dw_hdmi_qp_suspend(struct device *dev, struct dw_hdmi_qp *hdmi) +{ + if (!hdmi) { + dev_warn(dev, "Hdmi has not been initialized\n"); + return; + } + + mutex_lock(&hdmi->mutex); + + /* + * When system shutdown, hdmi should be disabled. + * When system suspend, dw_hdmi_qp_bridge_disable will disable hdmi first. + * To prevent duplicate operation, we should determine whether hdmi + * has been disabled. + */ + if (!hdmi->disabled) + hdmi->disabled = true; + mutex_unlock(&hdmi->mutex); + + if (hdmi->avp_irq) + disable_irq(hdmi->avp_irq); + + if (hdmi->main_irq) + disable_irq(hdmi->main_irq); + + if (hdmi->earc_irq) + disable_irq(hdmi->earc_irq); + + pinctrl_pm_select_sleep_state(dev); +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_suspend); + +void dw_hdmi_qp_resume(struct device *dev, struct dw_hdmi_qp *hdmi) +{ + if (!hdmi) { + dev_warn(dev, "Hdmi has not been initialized\n"); + return; + } + + hdmi_writel(hdmi, 0, MAINUNIT_0_INT_MASK_N); + hdmi_writel(hdmi, 0, MAINUNIT_1_INT_MASK_N); + hdmi_writel(hdmi, 428571429, TIMER_BASE_CONFIG0); + + pinctrl_pm_select_default_state(dev); + mutex_lock(&hdmi->mutex); + if (hdmi->i2c) + dw_hdmi_i2c_init(hdmi); + if (hdmi->avp_irq) + enable_irq(hdmi->avp_irq); + + if (hdmi->main_irq) + enable_irq(hdmi->main_irq); + + if (hdmi->earc_irq) + enable_irq(hdmi->earc_irq); + + mutex_unlock(&hdmi->mutex); +} +EXPORT_SYMBOL_GPL(dw_hdmi_qp_resume); + +MODULE_AUTHOR("Algea Cao "); +MODULE_DESCRIPTION("DW HDMI QP transmitter driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:dw-hdmi-qp"); diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h new file mode 100755 index 000000000000..a891c8333e9b --- /dev/null +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h @@ -0,0 +1,824 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Rockchip Electronics Co.Ltd + * Author: + * Algea Cao + */ +#ifndef __DW_HDMI_QP_H__ +#define __DW_HDMI_QP_H__ +/* Main Unit Registers */ +#define CORE_ID 0x0 +#define VER_NUMBER 0x4 +#define VER_TYPE 0x8 +#define CONFIG_REG 0xc +#define CONFIG_CEC BIT(28) +#define CONFIG_AUD_UD BIT(23) +#define CORE_TIMESTAMP_HHMM 0x14 +#define CORE_TIMESTAMP_MMDD 0x18 +#define CORE_TIMESTAMP_YYYY 0x1c +/* Reset Manager Registers */ +#define GLOBAL_SWRESET_REQUEST 0x40 +#define EARCRX_CMDC_SWINIT_P BIT(27) +#define AVP_DATAPATH_PACKET_AUDIO_SWINIT_P BIT(10) +#define GLOBAL_SWDISABLE 0x44 +#define CEC_SWDISABLE BIT(17) +#define AVP_DATAPATH_PACKET_AUDIO_SWDISABLE BIT(10) +#define AVP_DATAPATH_VIDEO_SWDISABLE BIT(6) +#define RESET_MANAGER_CONFIG0 0x48 +#define RESET_MANAGER_STATUS0 0x50 +#define RESET_MANAGER_STATUS1 0x54 +#define RESET_MANAGER_STATUS2 0x58 +/* Timer Base Registers */ +#define TIMER_BASE_CONFIG0 0x80 +#define TIMER_BASE_STATUS0 0x84 +/* CMU Registers */ +#define CMU_CONFIG0 0xa0 +#define CMU_CONFIG1 0xa4 +#define CMU_CONFIG2 0xa8 +#define CMU_CONFIG3 0xac +#define CMU_STATUS 0xb0 +#define EARC_BPCLK_OFF BIT(9) +#define AUDCLK_OFF BIT(7) +#define LINKQPCLK_OFF BIT(5) +#define VIDQPCLK_OFF BIT(3) +#define IPI_CLK_OFF BIT(1) +#define CMU_IPI_CLK_FREQ 0xb4 +#define CMU_VIDQPCLK_FREQ 0xb8 +#define CMU_LINKQPCLK_FREQ 0xbc +#define CMU_AUDQPCLK_FREQ 0xc0 +#define CMU_EARC_BPCLK_FREQ 0xc4 +/* I2CM Registers */ +#define I2CM_SM_SCL_CONFIG0 0xe0 +#define I2CM_FM_SCL_CONFIG0 0xe4 +#define I2CM_CONFIG0 0xe8 +#define I2CM_CONTROL0 0xec +#define I2CM_STATUS0 0xf0 +#define I2CM_INTERFACE_CONTROL0 0xf4 +#define I2CM_ADDR 0xff000 +#define I2CM_SLVADDR 0xfe0 +#define I2CM_WR_MASK 0x1e +#define I2CM_EXT_READ BIT(4) +#define I2CM_SHORT_READ BIT(3) +#define I2CM_FM_READ BIT(2) +#define I2CM_FM_WRITE BIT(1) +#define I2CM_FM_EN BIT(0) +#define I2CM_INTERFACE_CONTROL1 0xf8 +#define I2CM_SEG_PTR 0x7f80 +#define I2CM_SEG_ADDR 0x7f +#define I2CM_INTERFACE_WRDATA_0_3 0xfc +#define I2CM_INTERFACE_WRDATA_4_7 0x100 +#define I2CM_INTERFACE_WRDATA_8_11 0x104 +#define I2CM_INTERFACE_WRDATA_12_15 0x108 +#define I2CM_INTERFACE_RDDATA_0_3 0x10c +#define I2CM_INTERFACE_RDDATA_4_7 0x110 +#define I2CM_INTERFACE_RDDATA_8_11 0x114 +#define I2CM_INTERFACE_RDDATA_12_15 0x118 +/* SCDC Registers */ +#define SCDC_CONFIG0 0x140 +#define SCDC_I2C_FM_EN BIT(12) +#define SCDC_UPD_FLAGS_AUTO_CLR BIT(6) +#define SCDC_UPD_FLAGS_POLL_EN BIT(4) +#define SCDC_CONTROL0 0x148 +#define SCDC_STATUS0 0x150 +#define STATUS_UPDATE BIT(0) +#define FRL_START BIT(4) +#define FLT_UPDATE BIT(5) +/* FLT Registers */ +#define FLT_CONFIG0 0x160 +#define FLT_CONFIG1 0x164 +#define FLT_CONFIG2 0x168 +#define FLT_CONTROL0 0x170 +/* Main Unit 2 Registers */ +#define MAINUNIT_STATUS0 0x180 +/* Video Interface Registers */ +#define VIDEO_INTERFACE_CONFIG0 0x800 +#define VIDEO_INTERFACE_CONFIG1 0x804 +#define VIDEO_INTERFACE_CONFIG2 0x808 +#define VIDEO_INTERFACE_CONTROL0 0x80c +#define VIDEO_INTERFACE_STATUS0 0x814 +/* Video Packing Registers */ +#define VIDEO_PACKING_CONFIG0 0x81c +/* Audio Interface Registers */ +#define AUDIO_INTERFACE_CONFIG0 0x820 +#define AUD_IF_SEL_MSK 0x3 +#define AUD_IF_SPDIF 0x2 +#define AUD_IF_I2S 0x1 +#define AUD_IF_PAI 0x0 +#define AUD_FIFO_INIT_ON_OVF_MSK BIT(2) +#define AUD_FIFO_INIT_ON_OVF_EN BIT(2) +#define I2S_LINES_EN_MSK GENMASK(7, 4) +#define I2S_LINES_EN(x) BIT(x + 4) +#define I2S_BPCUV_RCV_MSK BIT(12) +#define I2S_BPCUV_RCV_EN BIT(12) +#define I2S_BPCUV_RCV_DIS 0 +#define SPDIF_LINES_EN GENMASK(19, 16) +#define AUD_FORMAT_MSK GENMASK(26, 24) +#define AUD_3DOBA (0x7 << 24) +#define AUD_3DASP (0x6 << 24) +#define AUD_MSOBA (0x5 << 24) +#define AUD_MSASP (0x4 << 24) +#define AUD_HBR (0x3 << 24) +#define AUD_DST (0x2 << 24) +#define AUD_OBA (0x1 << 24) +#define AUD_ASP (0x0 << 24) +#define AUDIO_INTERFACE_CONFIG1 0x824 +#define AUDIO_INTERFACE_CONTROL0 0x82c +#define AUDIO_FIFO_CLR_P BIT(0) +#define AUDIO_INTERFACE_STATUS0 0x834 +/* Frame Composer Registers */ +#define FRAME_COMPOSER_CONFIG0 0x840 +#define FRAME_COMPOSER_CONFIG1 0x844 +#define FRAME_COMPOSER_CONFIG2 0x848 +#define FRAME_COMPOSER_CONFIG3 0x84c +#define FRAME_COMPOSER_CONFIG4 0x850 +#define FRAME_COMPOSER_CONFIG5 0x854 +#define FRAME_COMPOSER_CONFIG6 0x858 +#define FRAME_COMPOSER_CONFIG7 0x85c +#define FRAME_COMPOSER_CONFIG8 0x860 +#define FRAME_COMPOSER_CONFIG9 0x864 +#define FRAME_COMPOSER_CONTROL0 0x86c +/* Video Monitor Registers */ +#define VIDEO_MONITOR_CONFIG0 0x880 +#define VIDEO_MONITOR_STATUS0 0x884 +#define VIDEO_MONITOR_STATUS1 0x888 +#define VIDEO_MONITOR_STATUS2 0x88c +#define VIDEO_MONITOR_STATUS3 0x890 +#define VIDEO_MONITOR_STATUS4 0x894 +#define VIDEO_MONITOR_STATUS5 0x898 +#define VIDEO_MONITOR_STATUS6 0x89c +/* HDCP2 Logic Registers */ +#define HDCP2LOGIC_CONFIG0 0x8e0 +#define HDCP2_BYPASS BIT(0) +#define HDCP2LOGIC_ESM_GPIO_IN 0x8e4 +#define HDCP2LOGIC_ESM_GPIO_OUT 0x8e8 +/* HDCP14 Registers */ +#define HDCP14_CONFIG0 0x900 +#define HDCP14_CONFIG1 0x904 +#define HDCP14_CONFIG2 0x908 +#define HDCP14_CONFIG3 0x90c +#define HDCP14_KEY_SEED 0x914 +#define HDCP14_KEY_H 0x918 +#define HDCP14_KEY_L 0x91c +#define HDCP14_KEY_STATUS 0x920 +#define HDCP14_AKSV_H 0x924 +#define HDCP14_AKSV_L 0x928 +#define HDCP14_AN_H 0x92c +#define HDCP14_AN_L 0x930 +#define HDCP14_STATUS0 0x934 +#define HDCP14_STATUS1 0x938 +/* Scrambler Registers */ +#define SCRAMB_CONFIG0 0x960 +/* Video Configuration Registers */ +#define LINK_CONFIG0 0x968 +#define OPMODE_FRL_4LANES BIT(8) +#define OPMODE_DVI BIT(4) +#define OPMODE_FRL BIT(0) +/* TMDS FIFO Registers */ +#define TMDS_FIFO_CONFIG0 0x970 +#define TMDS_FIFO_CONTROL0 0x974 +/* FRL RSFEC Registers */ +#define FRL_RSFEC_CONFIG0 0xa20 +#define FRL_RSFEC_STATUS0 0xa30 +/* FRL Packetizer Registers */ +#define FRL_PKTZ_CONFIG0 0xa40 +#define FRL_PKTZ_CONTROL0 0xa44 +#define FRL_PKTZ_CONTROL1 0xa50 +#define FRL_PKTZ_STATUS1 0xa54 +/* Packet Scheduler Registers */ +#define PKTSCHED_CONFIG0 0xa80 +#define PKTSCHED_PRQUEUE0_CONFIG0 0xa84 +#define PKTSCHED_PRQUEUE1_CONFIG0 0xa88 +#define PKTSCHED_PRQUEUE2_CONFIG0 0xa8c +#define PKTSCHED_PRQUEUE2_CONFIG1 0xa90 +#define PKTSCHED_PRQUEUE2_CONFIG2 0xa94 +#define PKTSCHED_PKT_CONFIG0 0xa98 +#define PKTSCHED_PKT_CONFIG1 0xa9c +#define PKTSCHED_PKT_CONFIG2 0xaa0 +#define PKTSCHED_PKT_CONFIG3 0xaa4 +#define PKTSCHED_PKT_EN 0xaa8 +#define PKTSCHED_DRMI_TX_EN BIT(17) +#define PKTSCHED_AUDI_TX_EN BIT(15) +#define PKTSCHED_AVI_TX_EN BIT(13) +#define PKTSCHED_EMP_CVTEM_TX_EN BIT(10) +#define PKTSCHED_AMD_TX_EN BIT(8) +#define PKTSCHED_GCP_TX_EN BIT(3) +#define PKTSCHED_AUDS_TX_EN BIT(2) +#define PKTSCHED_ACR_TX_EN BIT(1) +#define PKTSCHED_PKT_CONTROL0 0xaac +#define PKTSCHED_PKT_SEND 0xab0 +#define PKTSCHED_PKT_STATUS0 0xab4 +#define PKTSCHED_PKT_STATUS1 0xab8 +#define PKT_NULL_CONTENTS0 0xb00 +#define PKT_NULL_CONTENTS1 0xb04 +#define PKT_NULL_CONTENTS2 0xb08 +#define PKT_NULL_CONTENTS3 0xb0c +#define PKT_NULL_CONTENTS4 0xb10 +#define PKT_NULL_CONTENTS5 0xb14 +#define PKT_NULL_CONTENTS6 0xb18 +#define PKT_NULL_CONTENTS7 0xb1c +#define PKT_ACP_CONTENTS0 0xb20 +#define PKT_ACP_CONTENTS1 0xb24 +#define PKT_ACP_CONTENTS2 0xb28 +#define PKT_ACP_CONTENTS3 0xb2c +#define PKT_ACP_CONTENTS4 0xb30 +#define PKT_ACP_CONTENTS5 0xb34 +#define PKT_ACP_CONTENTS6 0xb38 +#define PKT_ACP_CONTENTS7 0xb3c +#define PKT_ISRC1_CONTENTS0 0xb40 +#define PKT_ISRC1_CONTENTS1 0xb44 +#define PKT_ISRC1_CONTENTS2 0xb48 +#define PKT_ISRC1_CONTENTS3 0xb4c +#define PKT_ISRC1_CONTENTS4 0xb50 +#define PKT_ISRC1_CONTENTS5 0xb54 +#define PKT_ISRC1_CONTENTS6 0xb58 +#define PKT_ISRC1_CONTENTS7 0xb5c +#define PKT_ISRC2_CONTENTS0 0xb60 +#define PKT_ISRC2_CONTENTS1 0xb64 +#define PKT_ISRC2_CONTENTS2 0xb68 +#define PKT_ISRC2_CONTENTS3 0xb6c +#define PKT_ISRC2_CONTENTS4 0xb70 +#define PKT_ISRC2_CONTENTS5 0xb74 +#define PKT_ISRC2_CONTENTS6 0xb78 +#define PKT_ISRC2_CONTENTS7 0xb7c +#define PKT_GMD_CONTENTS0 0xb80 +#define PKT_GMD_CONTENTS1 0xb84 +#define PKT_GMD_CONTENTS2 0xb88 +#define PKT_GMD_CONTENTS3 0xb8c +#define PKT_GMD_CONTENTS4 0xb90 +#define PKT_GMD_CONTENTS5 0xb94 +#define PKT_GMD_CONTENTS6 0xb98 +#define PKT_GMD_CONTENTS7 0xb9c +#define PKT_AMD_CONTENTS0 0xba0 +#define PKT_AMD_CONTENTS1 0xba4 +#define PKT_AMD_CONTENTS2 0xba8 +#define PKT_AMD_CONTENTS3 0xbac +#define PKT_AMD_CONTENTS4 0xbb0 +#define PKT_AMD_CONTENTS5 0xbb4 +#define PKT_AMD_CONTENTS6 0xbb8 +#define PKT_AMD_CONTENTS7 0xbbc +#define PKT_VSI_CONTENTS0 0xbc0 +#define PKT_VSI_CONTENTS1 0xbc4 +#define PKT_VSI_CONTENTS2 0xbc8 +#define PKT_VSI_CONTENTS3 0xbcc +#define PKT_VSI_CONTENTS4 0xbd0 +#define PKT_VSI_CONTENTS5 0xbd4 +#define PKT_VSI_CONTENTS6 0xbd8 +#define PKT_VSI_CONTENTS7 0xbdc +#define PKT_AVI_CONTENTS0 0xbe0 +#define HDMI_FC_AVICONF0_ACTIVE_FMT_INFO_PRESENT BIT(4) +#define HDMI_FC_AVICONF0_BAR_DATA_VERT_BAR 0x04 +#define HDMI_FC_AVICONF0_BAR_DATA_HORIZ_BAR 0x08 +#define HDMI_FC_AVICONF2_IT_CONTENT_VALID 0x80 +#define PKT_AVI_CONTENTS1 0xbe4 +#define PKT_AVI_CONTENTS2 0xbe8 +#define PKT_AVI_CONTENTS3 0xbec +#define PKT_AVI_CONTENTS4 0xbf0 +#define PKT_AVI_CONTENTS5 0xbf4 +#define PKT_AVI_CONTENTS6 0xbf8 +#define PKT_AVI_CONTENTS7 0xbfc +#define PKT_SPDI_CONTENTS0 0xc00 +#define PKT_SPDI_CONTENTS1 0xc04 +#define PKT_SPDI_CONTENTS2 0xc08 +#define PKT_SPDI_CONTENTS3 0xc0c +#define PKT_SPDI_CONTENTS4 0xc10 +#define PKT_SPDI_CONTENTS5 0xc14 +#define PKT_SPDI_CONTENTS6 0xc18 +#define PKT_SPDI_CONTENTS7 0xc1c +#define PKT_AUDI_CONTENTS0 0xc20 +#define PKT_AUDI_CONTENTS1 0xc24 +#define PKT_AUDI_CONTENTS2 0xc28 +#define PKT_AUDI_CONTENTS3 0xc2c +#define PKT_AUDI_CONTENTS4 0xc30 +#define PKT_AUDI_CONTENTS5 0xc34 +#define PKT_AUDI_CONTENTS6 0xc38 +#define PKT_AUDI_CONTENTS7 0xc3c +#define PKT_NVI_CONTENTS0 0xc40 +#define PKT_NVI_CONTENTS1 0xc44 +#define PKT_NVI_CONTENTS2 0xc48 +#define PKT_NVI_CONTENTS3 0xc4c +#define PKT_NVI_CONTENTS4 0xc50 +#define PKT_NVI_CONTENTS5 0xc54 +#define PKT_NVI_CONTENTS6 0xc58 +#define PKT_NVI_CONTENTS7 0xc5c +#define PKT_DRMI_CONTENTS0 0xc60 +#define PKT_DRMI_CONTENTS1 0xc64 +#define PKT_DRMI_CONTENTS2 0xc68 +#define PKT_DRMI_CONTENTS3 0xc6c +#define PKT_DRMI_CONTENTS4 0xc70 +#define PKT_DRMI_CONTENTS5 0xc74 +#define PKT_DRMI_CONTENTS6 0xc78 +#define PKT_DRMI_CONTENTS7 0xc7c +#define PKT_GHDMI1_CONTENTS0 0xc80 +#define PKT_GHDMI1_CONTENTS1 0xc84 +#define PKT_GHDMI1_CONTENTS2 0xc88 +#define PKT_GHDMI1_CONTENTS3 0xc8c +#define PKT_GHDMI1_CONTENTS4 0xc90 +#define PKT_GHDMI1_CONTENTS5 0xc94 +#define PKT_GHDMI1_CONTENTS6 0xc98 +#define PKT_GHDMI1_CONTENTS7 0xc9c +#define PKT_GHDMI2_CONTENTS0 0xca0 +#define PKT_GHDMI2_CONTENTS1 0xca4 +#define PKT_GHDMI2_CONTENTS2 0xca8 +#define PKT_GHDMI2_CONTENTS3 0xcac +#define PKT_GHDMI2_CONTENTS4 0xcb0 +#define PKT_GHDMI2_CONTENTS5 0xcb4 +#define PKT_GHDMI2_CONTENTS6 0xcb8 +#define PKT_GHDMI2_CONTENTS7 0xcbc +/* EMP Packetizer Registers */ +#define PKT_EMP_CONFIG0 0xce0 +#define PKT_EMP_CONTROL0 0xcec +#define PKT_EMP_CONTROL1 0xcf0 +#define PKT_EMP_CONTROL2 0xcf4 +#define PKT_EMP_VTEM_CONTENTS0 0xd00 +#define PKT_EMP_VTEM_CONTENTS1 0xd04 +#define PKT_EMP_VTEM_CONTENTS2 0xd08 +#define PKT_EMP_VTEM_CONTENTS3 0xd0c +#define PKT_EMP_VTEM_CONTENTS4 0xd10 +#define PKT_EMP_VTEM_CONTENTS5 0xd14 +#define PKT_EMP_VTEM_CONTENTS6 0xd18 +#define PKT_EMP_VTEM_CONTENTS7 0xd1c +#define PKT0_EMP_CVTEM_CONTENTS0 0xd20 +#define PKT0_EMP_CVTEM_CONTENTS1 0xd24 +#define PKT0_EMP_CVTEM_CONTENTS2 0xd28 +#define PKT0_EMP_CVTEM_CONTENTS3 0xd2c +#define PKT0_EMP_CVTEM_CONTENTS4 0xd30 +#define PKT0_EMP_CVTEM_CONTENTS5 0xd34 +#define PKT0_EMP_CVTEM_CONTENTS6 0xd38 +#define PKT0_EMP_CVTEM_CONTENTS7 0xd3c +#define PKT1_EMP_CVTEM_CONTENTS0 0xd40 +#define PKT1_EMP_CVTEM_CONTENTS1 0xd44 +#define PKT1_EMP_CVTEM_CONTENTS2 0xd48 +#define PKT1_EMP_CVTEM_CONTENTS3 0xd4c +#define PKT1_EMP_CVTEM_CONTENTS4 0xd50 +#define PKT1_EMP_CVTEM_CONTENTS5 0xd54 +#define PKT1_EMP_CVTEM_CONTENTS6 0xd58 +#define PKT1_EMP_CVTEM_CONTENTS7 0xd5c +#define PKT2_EMP_CVTEM_CONTENTS0 0xd60 +#define PKT2_EMP_CVTEM_CONTENTS1 0xd64 +#define PKT2_EMP_CVTEM_CONTENTS2 0xd68 +#define PKT2_EMP_CVTEM_CONTENTS3 0xd6c +#define PKT2_EMP_CVTEM_CONTENTS4 0xd70 +#define PKT2_EMP_CVTEM_CONTENTS5 0xd74 +#define PKT2_EMP_CVTEM_CONTENTS6 0xd78 +#define PKT2_EMP_CVTEM_CONTENTS7 0xd7c +#define PKT3_EMP_CVTEM_CONTENTS0 0xd80 +#define PKT3_EMP_CVTEM_CONTENTS1 0xd84 +#define PKT3_EMP_CVTEM_CONTENTS2 0xd88 +#define PKT3_EMP_CVTEM_CONTENTS3 0xd8c +#define PKT3_EMP_CVTEM_CONTENTS4 0xd90 +#define PKT3_EMP_CVTEM_CONTENTS5 0xd94 +#define PKT3_EMP_CVTEM_CONTENTS6 0xd98 +#define PKT3_EMP_CVTEM_CONTENTS7 0xd9c +#define PKT4_EMP_CVTEM_CONTENTS0 0xda0 +#define PKT4_EMP_CVTEM_CONTENTS1 0xda4 +#define PKT4_EMP_CVTEM_CONTENTS2 0xda8 +#define PKT4_EMP_CVTEM_CONTENTS3 0xdac +#define PKT4_EMP_CVTEM_CONTENTS4 0xdb0 +#define PKT4_EMP_CVTEM_CONTENTS5 0xdb4 +#define PKT4_EMP_CVTEM_CONTENTS6 0xdb8 +#define PKT4_EMP_CVTEM_CONTENTS7 0xdbc +#define PKT5_EMP_CVTEM_CONTENTS0 0xdc0 +#define PKT5_EMP_CVTEM_CONTENTS1 0xdc4 +#define PKT5_EMP_CVTEM_CONTENTS2 0xdc8 +#define PKT5_EMP_CVTEM_CONTENTS3 0xdcc +#define PKT5_EMP_CVTEM_CONTENTS4 0xdd0 +#define PKT5_EMP_CVTEM_CONTENTS5 0xdd4 +#define PKT5_EMP_CVTEM_CONTENTS6 0xdd8 +#define PKT5_EMP_CVTEM_CONTENTS7 0xddc +/* Audio Packetizer Registers */ +#define AUDPKT_CONTROL0 0xe20 +#define AUDPKT_CHSTATUS_OVR_EN_MASK BIT(0) +#define AUDPKT_CHSTATUS_OVR_EN BIT(0) +#define AUDPKT_CONTROL1 0xe24 +#define AUDPKT_ACR_CONTROL0 0xe40 +#define AUDPKT_ACR_N_VALUE 0xfffff +#define AUDPKT_ACR_CONTROL1 0xe44 +#define AUDPKT_ACR_CTS_OVR_VAL_MSK GENMASK(23, 4) +#define AUDPKT_ACR_CTS_OVR_VAL(x) ((x) << 4) +#define AUDPKT_ACR_CTS_OVR_EN_MSK BIT(1) +#define AUDPKT_ACR_CTS_OVR_EN BIT(1) +#define AUDPKT_ACR_STATUS0 0xe4c +#define AUDPKT_CHSTATUS_OVR0 0xe60 +#define AUDPKT_CHSTATUS_OVR1 0xe64 +/* IEC60958 Byte 3: Sampleing frenuency Bits 24 to 27 */ +#define AUDPKT_CHSTATUS_SR_MASK GENMASK(3, 0) +#define AUDPKT_CHSTATUS_SR_22050 0x4 +#define AUDPKT_CHSTATUS_SR_24000 0x6 +#define AUDPKT_CHSTATUS_SR_32000 0x3 +#define AUDPKT_CHSTATUS_SR_44100 0x0 +#define AUDPKT_CHSTATUS_SR_48000 0x2 +#define AUDPKT_CHSTATUS_SR_88200 0x8 +#define AUDPKT_CHSTATUS_SR_96000 0xa +#define AUDPKT_CHSTATUS_SR_176400 0xc +#define AUDPKT_CHSTATUS_SR_192000 0xe +#define AUDPKT_CHSTATUS_SR_768000 0x9 +#define AUDPKT_CHSTATUS_SR_NOT_INDICATED 0x1 +/* IEC60958 Byte 4: Original Sampleing frenuency Bits 36 to 39 */ +#define AUDPKT_CHSTATUS_0SR_MASK GENMASK(15, 12) +#define AUDPKT_CHSTATUS_OSR_8000 0x6 +#define AUDPKT_CHSTATUS_OSR_11025 0xa +#define AUDPKT_CHSTATUS_OSR_12000 0x2 +#define AUDPKT_CHSTATUS_OSR_16000 0x8 +#define AUDPKT_CHSTATUS_OSR_22050 0xb +#define AUDPKT_CHSTATUS_OSR_24000 0x9 +#define AUDPKT_CHSTATUS_OSR_32000 0xc +#define AUDPKT_CHSTATUS_OSR_44100 0xf +#define AUDPKT_CHSTATUS_OSR_48000 0xd +#define AUDPKT_CHSTATUS_OSR_88200 0x7 +#define AUDPKT_CHSTATUS_OSR_96000 0x5 +#define AUDPKT_CHSTATUS_OSR_176400 0x3 +#define AUDPKT_CHSTATUS_OSR_192000 0x1 +#define AUDPKT_CHSTATUS_OSR_NOT_INDICATED 0x0 +#define AUDPKT_CHSTATUS_OVR2 0xe68 +#define AUDPKT_CHSTATUS_OVR3 0xe6c +#define AUDPKT_CHSTATUS_OVR4 0xe70 +#define AUDPKT_CHSTATUS_OVR5 0xe74 +#define AUDPKT_CHSTATUS_OVR6 0xe78 +#define AUDPKT_CHSTATUS_OVR7 0xe7c +#define AUDPKT_CHSTATUS_OVR8 0xe80 +#define AUDPKT_CHSTATUS_OVR9 0xe84 +#define AUDPKT_CHSTATUS_OVR10 0xe88 +#define AUDPKT_CHSTATUS_OVR11 0xe8c +#define AUDPKT_CHSTATUS_OVR12 0xe90 +#define AUDPKT_CHSTATUS_OVR13 0xe94 +#define AUDPKT_CHSTATUS_OVR14 0xe98 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC0 0xea0 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC1 0xea4 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC2 0xea8 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC3 0xeac +#define AUDPKT_USRDATA_OVR_MSG_GENERIC4 0xeb0 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC5 0xeb4 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC6 0xeb8 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC7 0xebc +#define AUDPKT_USRDATA_OVR_MSG_GENERIC8 0xec0 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC9 0xec4 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC10 0xec8 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC11 0xecc +#define AUDPKT_USRDATA_OVR_MSG_GENERIC12 0xed0 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC13 0xed4 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC14 0xed8 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC15 0xedc +#define AUDPKT_USRDATA_OVR_MSG_GENERIC16 0xee0 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC17 0xee4 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC18 0xee8 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC19 0xeec +#define AUDPKT_USRDATA_OVR_MSG_GENERIC20 0xef0 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC21 0xef4 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC22 0xef8 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC23 0xefc +#define AUDPKT_USRDATA_OVR_MSG_GENERIC24 0xf00 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC25 0xf04 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC26 0xf08 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC27 0xf0c +#define AUDPKT_USRDATA_OVR_MSG_GENERIC28 0xf10 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC29 0xf14 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC30 0xf18 +#define AUDPKT_USRDATA_OVR_MSG_GENERIC31 0xf1c +#define AUDPKT_USRDATA_OVR_MSG_GENERIC32 0xf20 +#define AUDPKT_VBIT_OVR0 0xf24 +/* CEC Registers */ +#define CEC_TX_CONTROL 0x1000 +#define CEC_STATUS 0x1004 +#define CEC_CONFIG 0x1008 +#define CEC_ADDR 0x100c +#define CEC_TX_COUNT 0x1020 +#define CEC_TX_DATA3_0 0x1024 +#define CEC_TX_DATA7_4 0x1028 +#define CEC_TX_DATA11_8 0x102c +#define CEC_TX_DATA15_12 0x1030 +#define CEC_RX_COUNT_STATUS 0x1040 +#define CEC_RX_DATA3_0 0x1044 +#define CEC_RX_DATA7_4 0x1048 +#define CEC_RX_DATA11_8 0x104c +#define CEC_RX_DATA15_12 0x1050 +#define CEC_LOCK_CONTROL 0x1054 +#define CEC_RXQUAL_BITTIME_CONFIG 0x1060 +#define CEC_RX_BITTIME_CONFIG 0x1064 +#define CEC_TX_BITTIME_CONFIG 0x1068 +/* eARC RX CMDC Registers */ +#define EARCRX_CMDC_CONFIG0 0x1800 +#define EARCRX_XACTREAD_STOP_CFG BIT(26) +#define EARCRX_XACTREAD_RETRY_CFG BIT(25) +#define EARCRX_CMDC_DSCVR_EARCVALID0_TO_DISC1 BIT(24) +#define EARCRX_CMDC_XACT_RESTART_EN BIT(18) +#define EARCRX_CMDC_CONFIG1 0x1804 +#define EARCRX_CMDC_CONTROL 0x1808 +#define EARCRX_CMDC_HEARTBEAT_LOSS_EN BIT(4) +#define EARCRX_CMDC_DISCOVERY_EN BIT(3) +#define EARCRX_CONNECTOR_HPD BIT(1) +#define EARCRX_CMDC_WHITELIST0_CONFIG 0x180c +#define EARCRX_CMDC_WHITELIST1_CONFIG 0x1810 +#define EARCRX_CMDC_WHITELIST2_CONFIG 0x1814 +#define EARCRX_CMDC_WHITELIST3_CONFIG 0x1818 +#define EARCRX_CMDC_STATUS 0x181c +#define EARCRX_CMDC_XACT_INFO 0x1820 +#define EARCRX_CMDC_XACT_ACTION 0x1824 +#define EARCRX_CMDC_HEARTBEAT_RXSTAT_SE 0x1828 +#define EARCRX_CMDC_HEARTBEAT_STATUS 0x182c +#define EARCRX_CMDC_XACT_WR0 0x1840 +#define EARCRX_CMDC_XACT_WR1 0x1844 +#define EARCRX_CMDC_XACT_WR2 0x1848 +#define EARCRX_CMDC_XACT_WR3 0x184c +#define EARCRX_CMDC_XACT_WR4 0x1850 +#define EARCRX_CMDC_XACT_WR5 0x1854 +#define EARCRX_CMDC_XACT_WR6 0x1858 +#define EARCRX_CMDC_XACT_WR7 0x185c +#define EARCRX_CMDC_XACT_WR8 0x1860 +#define EARCRX_CMDC_XACT_WR9 0x1864 +#define EARCRX_CMDC_XACT_WR10 0x1868 +#define EARCRX_CMDC_XACT_WR11 0x186c +#define EARCRX_CMDC_XACT_WR12 0x1870 +#define EARCRX_CMDC_XACT_WR13 0x1874 +#define EARCRX_CMDC_XACT_WR14 0x1878 +#define EARCRX_CMDC_XACT_WR15 0x187c +#define EARCRX_CMDC_XACT_WR16 0x1880 +#define EARCRX_CMDC_XACT_WR17 0x1884 +#define EARCRX_CMDC_XACT_WR18 0x1888 +#define EARCRX_CMDC_XACT_WR19 0x188c +#define EARCRX_CMDC_XACT_WR20 0x1890 +#define EARCRX_CMDC_XACT_WR21 0x1894 +#define EARCRX_CMDC_XACT_WR22 0x1898 +#define EARCRX_CMDC_XACT_WR23 0x189c +#define EARCRX_CMDC_XACT_WR24 0x18a0 +#define EARCRX_CMDC_XACT_WR25 0x18a4 +#define EARCRX_CMDC_XACT_WR26 0x18a8 +#define EARCRX_CMDC_XACT_WR27 0x18ac +#define EARCRX_CMDC_XACT_WR28 0x18b0 +#define EARCRX_CMDC_XACT_WR29 0x18b4 +#define EARCRX_CMDC_XACT_WR30 0x18b8 +#define EARCRX_CMDC_XACT_WR31 0x18bc +#define EARCRX_CMDC_XACT_WR32 0x18c0 +#define EARCRX_CMDC_XACT_WR33 0x18c4 +#define EARCRX_CMDC_XACT_WR34 0x18c8 +#define EARCRX_CMDC_XACT_WR35 0x18cc +#define EARCRX_CMDC_XACT_WR36 0x18d0 +#define EARCRX_CMDC_XACT_WR37 0x18d4 +#define EARCRX_CMDC_XACT_WR38 0x18d8 +#define EARCRX_CMDC_XACT_WR39 0x18dc +#define EARCRX_CMDC_XACT_WR40 0x18e0 +#define EARCRX_CMDC_XACT_WR41 0x18e4 +#define EARCRX_CMDC_XACT_WR42 0x18e8 +#define EARCRX_CMDC_XACT_WR43 0x18ec +#define EARCRX_CMDC_XACT_WR44 0x18f0 +#define EARCRX_CMDC_XACT_WR45 0x18f4 +#define EARCRX_CMDC_XACT_WR46 0x18f8 +#define EARCRX_CMDC_XACT_WR47 0x18fc +#define EARCRX_CMDC_XACT_WR48 0x1900 +#define EARCRX_CMDC_XACT_WR49 0x1904 +#define EARCRX_CMDC_XACT_WR50 0x1908 +#define EARCRX_CMDC_XACT_WR51 0x190c +#define EARCRX_CMDC_XACT_WR52 0x1910 +#define EARCRX_CMDC_XACT_WR53 0x1914 +#define EARCRX_CMDC_XACT_WR54 0x1918 +#define EARCRX_CMDC_XACT_WR55 0x191c +#define EARCRX_CMDC_XACT_WR56 0x1920 +#define EARCRX_CMDC_XACT_WR57 0x1924 +#define EARCRX_CMDC_XACT_WR58 0x1928 +#define EARCRX_CMDC_XACT_WR59 0x192c +#define EARCRX_CMDC_XACT_WR60 0x1930 +#define EARCRX_CMDC_XACT_WR61 0x1934 +#define EARCRX_CMDC_XACT_WR62 0x1938 +#define EARCRX_CMDC_XACT_WR63 0x193c +#define EARCRX_CMDC_XACT_WR64 0x1940 +#define EARCRX_CMDC_XACT_RD0 0x1960 +#define EARCRX_CMDC_XACT_RD1 0x1964 +#define EARCRX_CMDC_XACT_RD2 0x1968 +#define EARCRX_CMDC_XACT_RD3 0x196c +#define EARCRX_CMDC_XACT_RD4 0x1970 +#define EARCRX_CMDC_XACT_RD5 0x1974 +#define EARCRX_CMDC_XACT_RD6 0x1978 +#define EARCRX_CMDC_XACT_RD7 0x197c +#define EARCRX_CMDC_XACT_RD8 0x1980 +#define EARCRX_CMDC_XACT_RD9 0x1984 +#define EARCRX_CMDC_XACT_RD10 0x1988 +#define EARCRX_CMDC_XACT_RD11 0x198c +#define EARCRX_CMDC_XACT_RD12 0x1990 +#define EARCRX_CMDC_XACT_RD13 0x1994 +#define EARCRX_CMDC_XACT_RD14 0x1998 +#define EARCRX_CMDC_XACT_RD15 0x199c +#define EARCRX_CMDC_XACT_RD16 0x19a0 +#define EARCRX_CMDC_XACT_RD17 0x19a4 +#define EARCRX_CMDC_XACT_RD18 0x19a8 +#define EARCRX_CMDC_XACT_RD19 0x19ac +#define EARCRX_CMDC_XACT_RD20 0x19b0 +#define EARCRX_CMDC_XACT_RD21 0x19b4 +#define EARCRX_CMDC_XACT_RD22 0x19b8 +#define EARCRX_CMDC_XACT_RD23 0x19bc +#define EARCRX_CMDC_XACT_RD24 0x19c0 +#define EARCRX_CMDC_XACT_RD25 0x19c4 +#define EARCRX_CMDC_XACT_RD26 0x19c8 +#define EARCRX_CMDC_XACT_RD27 0x19cc +#define EARCRX_CMDC_XACT_RD28 0x19d0 +#define EARCRX_CMDC_XACT_RD29 0x19d4 +#define EARCRX_CMDC_XACT_RD30 0x19d8 +#define EARCRX_CMDC_XACT_RD31 0x19dc +#define EARCRX_CMDC_XACT_RD32 0x19e0 +#define EARCRX_CMDC_XACT_RD33 0x19e4 +#define EARCRX_CMDC_XACT_RD34 0x19e8 +#define EARCRX_CMDC_XACT_RD35 0x19ec +#define EARCRX_CMDC_XACT_RD36 0x19f0 +#define EARCRX_CMDC_XACT_RD37 0x19f4 +#define EARCRX_CMDC_XACT_RD38 0x19f8 +#define EARCRX_CMDC_XACT_RD39 0x19fc +#define EARCRX_CMDC_XACT_RD40 0x1a00 +#define EARCRX_CMDC_XACT_RD41 0x1a04 +#define EARCRX_CMDC_XACT_RD42 0x1a08 +#define EARCRX_CMDC_XACT_RD43 0x1a0c +#define EARCRX_CMDC_XACT_RD44 0x1a10 +#define EARCRX_CMDC_XACT_RD45 0x1a14 +#define EARCRX_CMDC_XACT_RD46 0x1a18 +#define EARCRX_CMDC_XACT_RD47 0x1a1c +#define EARCRX_CMDC_XACT_RD48 0x1a20 +#define EARCRX_CMDC_XACT_RD49 0x1a24 +#define EARCRX_CMDC_XACT_RD50 0x1a28 +#define EARCRX_CMDC_XACT_RD51 0x1a2c +#define EARCRX_CMDC_XACT_RD52 0x1a30 +#define EARCRX_CMDC_XACT_RD53 0x1a34 +#define EARCRX_CMDC_XACT_RD54 0x1a38 +#define EARCRX_CMDC_XACT_RD55 0x1a3c +#define EARCRX_CMDC_XACT_RD56 0x1a40 +#define EARCRX_CMDC_XACT_RD57 0x1a44 +#define EARCRX_CMDC_XACT_RD58 0x1a48 +#define EARCRX_CMDC_XACT_RD59 0x1a4c +#define EARCRX_CMDC_XACT_RD60 0x1a50 +#define EARCRX_CMDC_XACT_RD61 0x1a54 +#define EARCRX_CMDC_XACT_RD62 0x1a58 +#define EARCRX_CMDC_XACT_RD63 0x1a5c +#define EARCRX_CMDC_XACT_RD64 0x1a60 +#define EARCRX_CMDC_SYNC_CONFIG 0x1b00 +/* eARC RX DMAC Registers */ +#define EARCRX_DMAC_PHY_CONTROL 0x1c00 +#define EARCRX_DMAC_CONFIG 0x1c08 +#define EARCRX_DMAC_CONTROL0 0x1c0c +#define EARCRX_DMAC_AUDIO_EN BIT(1) +#define EARCRX_DMAC_EN BIT(0) +#define EARCRX_DMAC_CONTROL1 0x1c10 +#define EARCRX_DMAC_STATUS 0x1c14 +#define EARCRX_DMAC_CHSTATUS0 0x1c18 +#define EARCRX_DMAC_CHSTATUS1 0x1c1c +#define EARCRX_DMAC_CHSTATUS2 0x1c20 +#define EARCRX_DMAC_CHSTATUS3 0x1c24 +#define EARCRX_DMAC_CHSTATUS4 0x1c28 +#define EARCRX_DMAC_CHSTATUS5 0x1c2c +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC0 0x1c30 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC1 0x1c34 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC2 0x1c38 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC3 0x1c3c +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC4 0x1c40 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC5 0x1c44 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC6 0x1c48 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC7 0x1c4c +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC8 0x1c50 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC9 0x1c54 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC10 0x1c58 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC11 0x1c5c +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT0 0x1c60 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT1 0x1c64 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT2 0x1c68 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT3 0x1c6c +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT4 0x1c70 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT5 0x1c74 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT6 0x1c78 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT7 0x1c7c +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT8 0x1c80 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT9 0x1c84 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT10 0x1c88 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT11 0x1c8c +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT0 0x1c90 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT1 0x1c94 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT2 0x1c98 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT3 0x1c9c +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT4 0x1ca0 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT5 0x1ca4 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT6 0x1ca8 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT7 0x1cac +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT8 0x1cb0 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT9 0x1cb4 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT10 0x1cb8 +#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT11 0x1cbc +#define EARCRX_DMAC_USRDATA_MSG_GENERIC0 0x1cc0 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC1 0x1cc4 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC2 0x1cc8 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC3 0x1ccc +#define EARCRX_DMAC_USRDATA_MSG_GENERIC4 0x1cd0 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC5 0x1cd4 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC6 0x1cd8 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC7 0x1cdc +#define EARCRX_DMAC_USRDATA_MSG_GENERIC8 0x1ce0 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC9 0x1ce4 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC10 0x1ce8 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC11 0x1cec +#define EARCRX_DMAC_USRDATA_MSG_GENERIC12 0x1cf0 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC13 0x1cf4 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC14 0x1cf8 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC15 0x1cfc +#define EARCRX_DMAC_USRDATA_MSG_GENERIC16 0x1d00 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC17 0x1d04 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC18 0x1d08 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC19 0x1d0c +#define EARCRX_DMAC_USRDATA_MSG_GENERIC20 0x1d10 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC21 0x1d14 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC22 0x1d18 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC23 0x1d1c +#define EARCRX_DMAC_USRDATA_MSG_GENERIC24 0x1d20 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC25 0x1d24 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC26 0x1d28 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC27 0x1d2c +#define EARCRX_DMAC_USRDATA_MSG_GENERIC28 0x1d30 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC29 0x1d34 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC30 0x1d38 +#define EARCRX_DMAC_USRDATA_MSG_GENERIC31 0x1d3c +#define EARCRX_DMAC_USRDATA_MSG_GENERIC32 0x1d40 +#define EARCRX_DMAC_CHSTATUS_STREAMER0 0x1d44 +#define EARCRX_DMAC_CHSTATUS_STREAMER1 0x1d48 +#define EARCRX_DMAC_CHSTATUS_STREAMER2 0x1d4c +#define EARCRX_DMAC_CHSTATUS_STREAMER3 0x1d50 +#define EARCRX_DMAC_CHSTATUS_STREAMER4 0x1d54 +#define EARCRX_DMAC_CHSTATUS_STREAMER5 0x1d58 +#define EARCRX_DMAC_CHSTATUS_STREAMER6 0x1d5c +#define EARCRX_DMAC_CHSTATUS_STREAMER7 0x1d60 +#define EARCRX_DMAC_CHSTATUS_STREAMER8 0x1d64 +#define EARCRX_DMAC_CHSTATUS_STREAMER9 0x1d68 +#define EARCRX_DMAC_CHSTATUS_STREAMER10 0x1d6c +#define EARCRX_DMAC_CHSTATUS_STREAMER11 0x1d70 +#define EARCRX_DMAC_CHSTATUS_STREAMER12 0x1d74 +#define EARCRX_DMAC_CHSTATUS_STREAMER13 0x1d78 +#define EARCRX_DMAC_CHSTATUS_STREAMER14 0x1d7c +#define EARCRX_DMAC_USRDATA_STREAMER0 0x1d80 +/* Main Unit Interrupt Registers */ +#define MAIN_INTVEC_INDEX 0x3000 +#define MAINUNIT_0_INT_STATUS 0x3010 +#define MAINUNIT_0_INT_MASK_N 0x3014 +#define MAINUNIT_0_INT_CLEAR 0x3018 +#define MAINUNIT_0_INT_FORCE 0x301c +#define MAINUNIT_1_INT_STATUS 0x3020 +#define FLT_EXIT_TO_LTSL_IRQ BIT(22) +#define FLT_EXIT_TO_LTS4_IRQ BIT(21) +#define FLT_EXIT_TO_LTSP_IRQ BIT(20) +#define SCDC_NACK_RCVD_IRQ BIT(12) +#define SCDC_RR_REPLY_STOP_IRQ BIT(11) +#define SCDC_UPD_FLAGS_CLR_IRQ BIT(10) +#define SCDC_UPD_FLAGS_CHG_IRQ BIT(9) +#define SCDC_UPD_FLAGS_RD_IRQ BIT(8) +#define I2CM_NACK_RCVD_IRQ BIT(2) +#define I2CM_READ_REQUEST_IRQ BIT(1) +#define I2CM_OP_DONE_IRQ BIT(0) +#define MAINUNIT_1_INT_MASK_N 0x3024 +#define I2CM_NACK_RCVD_MASK_N BIT(2) +#define I2CM_READ_REQUEST_MASK_N BIT(1) +#define I2CM_OP_DONE_MASK_N BIT(0) +#define MAINUNIT_1_INT_CLEAR 0x3028 +#define I2CM_NACK_RCVD_CLEAR BIT(2) +#define I2CM_READ_REQUEST_CLEAR BIT(1) +#define I2CM_OP_DONE_CLEAR BIT(0) +#define MAINUNIT_1_INT_FORCE 0x302c +/* AVPUNIT Interrupt Registers */ +#define AVP_INTVEC_INDEX 0x3800 +#define AVP_0_INT_STATUS 0x3810 +#define AVP_0_INT_MASK_N 0x3814 +#define AVP_0_INT_CLEAR 0x3818 +#define AVP_0_INT_FORCE 0x381c +#define AVP_1_INT_STATUS 0x3820 +#define AVP_1_INT_MASK_N 0x3824 +#define HDCP14_AUTH_CHG_MASK_N BIT(6) +#define AVP_1_INT_CLEAR 0x3828 +#define AVP_1_INT_FORCE 0x382c +#define AVP_2_INT_STATUS 0x3830 +#define AVP_2_INT_MASK_N 0x3834 +#define AVP_2_INT_CLEAR 0x3838 +#define AVP_2_INT_FORCE 0x383c +#define AVP_3_INT_STATUS 0x3840 +#define AVP_3_INT_MASK_N 0x3844 +#define AVP_3_INT_CLEAR 0x3848 +#define AVP_3_INT_FORCE 0x384c +#define AVP_4_INT_STATUS 0x3850 +#define AVP_4_INT_MASK_N 0x3854 +#define AVP_4_INT_CLEAR 0x3858 +#define AVP_4_INT_FORCE 0x385c +#define AVP_5_INT_STATUS 0x3860 +#define AVP_5_INT_MASK_N 0x3864 +#define AVP_5_INT_CLEAR 0x3868 +#define AVP_5_INT_FORCE 0x386c +#define AVP_6_INT_STATUS 0x3870 +#define AVP_6_INT_MASK_N 0x3874 +#define AVP_6_INT_CLEAR 0x3878 +#define AVP_6_INT_FORCE 0x387c +/* CEC Interrupt Registers */ +#define CEC_INT_STATUS 0x4000 +#define CEC_INT_MASK_N 0x4004 +#define CEC_INT_CLEAR 0x4008 +#define CEC_INT_FORCE 0x400c +/* eARC RX Interrupt Registers */ +#define EARCRX_INTVEC_INDEX 0x4800 +#define EARCRX_0_INT_STATUS 0x4810 +#define EARCRX_CMDC_DISCOVERY_TIMEOUT_IRQ BIT(9) +#define EARCRX_CMDC_DISCOVERY_DONE_IRQ BIT(8) +#define EARCRX_0_INT_MASK_N 0x4814 +#define EARCRX_0_INT_CLEAR 0x4818 +#define EARCRX_0_INT_FORCE 0x481c +#define EARCRX_1_INT_STATUS 0x4820 +#define EARCRX_1_INT_MASK_N 0x4824 +#define EARCRX_1_INT_CLEAR 0x4828 +#define EARCRX_1_INT_FORCE 0x482c + +#endif /* __DW_HDMI_QP_H__ */ diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c index 2c3c743df950..1b25fdd32727 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include #include @@ -18,6 +20,7 @@ #include #include #include +#include #include @@ -36,6 +39,7 @@ #include "dw-hdmi-audio.h" #include "dw-hdmi-cec.h" +#include "dw-hdmi-hdcp.h" #include "dw-hdmi.h" #define DDC_CI_ADDR 0x37 @@ -48,6 +52,11 @@ #define HDMI14_MAX_TMDSCLK 340000000 +static const unsigned int dw_hdmi_cable[] = { + EXTCON_DISP_HDMI, + EXTCON_NONE, +}; + enum hdmi_datamap { RGB444_8B = 0x01, RGB444_10B = 0x03, @@ -62,6 +71,61 @@ enum hdmi_datamap { YCbCr422_12B = 0x12, }; +/* + * Unless otherwise noted, entries in this table are 100% optimization. + * Values can be obtained from hdmi_compute_n() but that function is + * slow so we pre-compute values we expect to see. + * + * All 32k and 48k values are expected to be the same (due to the way + * the math works) for any rate that's an exact kHz. + */ +static const struct dw_hdmi_audio_tmds_n common_tmds_n_table[] = { + { .tmds = 25175000, .n_32k = 4096, .n_44k1 = 12854, .n_48k = 6144, }, + { .tmds = 25200000, .n_32k = 4096, .n_44k1 = 5656, .n_48k = 6144, }, + { .tmds = 27000000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, + { .tmds = 28320000, .n_32k = 4096, .n_44k1 = 5586, .n_48k = 6144, }, + { .tmds = 30240000, .n_32k = 4096, .n_44k1 = 5642, .n_48k = 6144, }, + { .tmds = 31500000, .n_32k = 4096, .n_44k1 = 5600, .n_48k = 6144, }, + { .tmds = 32000000, .n_32k = 4096, .n_44k1 = 5733, .n_48k = 6144, }, + { .tmds = 33750000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, + { .tmds = 36000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, + { .tmds = 40000000, .n_32k = 4096, .n_44k1 = 5733, .n_48k = 6144, }, + { .tmds = 49500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, + { .tmds = 50000000, .n_32k = 4096, .n_44k1 = 5292, .n_48k = 6144, }, + { .tmds = 54000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, + { .tmds = 65000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, + { .tmds = 68250000, .n_32k = 4096, .n_44k1 = 5376, .n_48k = 6144, }, + { .tmds = 71000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, + { .tmds = 72000000, .n_32k = 4096, .n_44k1 = 5635, .n_48k = 6144, }, + { .tmds = 73250000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, + { .tmds = 74250000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, + { .tmds = 75000000, .n_32k = 4096, .n_44k1 = 5880, .n_48k = 6144, }, + { .tmds = 78750000, .n_32k = 4096, .n_44k1 = 5600, .n_48k = 6144, }, + { .tmds = 78800000, .n_32k = 4096, .n_44k1 = 5292, .n_48k = 6144, }, + { .tmds = 79500000, .n_32k = 4096, .n_44k1 = 4704, .n_48k = 6144, }, + { .tmds = 83500000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, + { .tmds = 85500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, + { .tmds = 88750000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, + { .tmds = 97750000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, + { .tmds = 101000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, + { .tmds = 106500000, .n_32k = 4096, .n_44k1 = 4704, .n_48k = 6144, }, + { .tmds = 108000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, + { .tmds = 115500000, .n_32k = 4096, .n_44k1 = 5712, .n_48k = 6144, }, + { .tmds = 119000000, .n_32k = 4096, .n_44k1 = 5544, .n_48k = 6144, }, + { .tmds = 135000000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, + { .tmds = 146250000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, + { .tmds = 148500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, + { .tmds = 154000000, .n_32k = 4096, .n_44k1 = 5544, .n_48k = 6144, }, + { .tmds = 162000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, + + /* For 297 MHz+ HDMI spec have some other rule for setting N */ + { .tmds = 297000000, .n_32k = 3073, .n_44k1 = 4704, .n_48k = 5120, }, + { .tmds = 594000000, .n_32k = 3073, .n_44k1 = 9408, .n_48k = 10240, }, + + /* End of table */ + { .tmds = 0, .n_32k = 0, .n_44k1 = 0, .n_48k = 0, }, +}; + static const u16 csc_coeff_default[3][4] = { { 0x2000, 0x0000, 0x0000, 0x0000 }, { 0x0000, 0x2000, 0x0000, 0x0000 }, @@ -98,12 +162,47 @@ static const u16 csc_coeff_rgb_full_to_rgb_limited[3][4] = { { 0x0000, 0x0000, 0x1b7c, 0x0020 } }; +static const struct drm_display_mode dw_hdmi_default_modes[] = { + /* 4 - 1280x720@60Hz 16:9 */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390, + 1430, 1650, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 16 - 1920x1080@60Hz 16:9 */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008, + 2052, 2200, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 31 - 1920x1080@50Hz 16:9 */ + { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448, + 2492, 2640, 0, 1080, 1084, 1089, 1125, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 19 - 1280x720@50Hz 16:9 */ + { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720, + 1760, 1980, 0, 720, 725, 730, 750, 0, + DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, + /* 17 - 720x576@50Hz 4:3 */ + { DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, + 796, 864, 0, 576, 581, 586, 625, 0, + DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, }, + /* 2 - 720x480@60Hz 4:3 */ + { DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736, + 798, 858, 0, 480, 489, 495, 525, 0, + DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), + .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, }, +}; + struct hdmi_vmode { bool mdataenablepolarity; + unsigned int previous_pixelclock; unsigned int mpixelclock; unsigned int mpixelrepetitioninput; unsigned int mpixelrepetitionoutput; + unsigned int previous_tmdsclock; unsigned int mtmdsclock; }; @@ -112,8 +211,8 @@ struct hdmi_data_info { unsigned int enc_out_bus_format; unsigned int enc_in_encoding; unsigned int enc_out_encoding; + unsigned int quant_range; unsigned int pix_repet_factor; - unsigned int hdcp_enable; struct hdmi_vmode video_mode; bool rgb_limited_range; }; @@ -128,6 +227,9 @@ struct dw_hdmi_i2c { u8 slave_reg; bool is_regaddr; bool is_segment; + + unsigned int scl_high_ns; + unsigned int scl_low_ns; }; struct dw_hdmi_phy_data { @@ -143,6 +245,8 @@ struct dw_hdmi_phy_data { struct dw_hdmi { struct drm_connector connector; struct drm_bridge bridge; + struct drm_bridge *next_bridge; + struct platform_device *hdcp_dev; unsigned int version; @@ -156,8 +260,10 @@ struct dw_hdmi { struct hdmi_data_info hdmi_data; const struct dw_hdmi_plat_data *plat_data; + struct dw_hdcp *hdcp; int vic; + int irq; u8 edid[HDMI_EDID_LEN]; @@ -174,6 +280,13 @@ struct dw_hdmi { void __iomem *regs; bool sink_is_hdmi; bool sink_has_audio; + bool hpd_state; + bool support_hdmi; + bool force_logo; + int force_output; + + struct delayed_work work; + struct workqueue_struct *workqueue; struct pinctrl *pinctrl; struct pinctrl_state *default_state; @@ -190,10 +303,14 @@ struct dw_hdmi { spinlock_t audio_lock; struct mutex audio_mutex; + struct dentry *debugfs_dir; unsigned int sample_rate; unsigned int audio_cts; unsigned int audio_n; bool audio_enable; + bool scramble_low_rates; + + struct extcon_dev *extcon; unsigned int reg_shift; struct regmap *regm; @@ -202,10 +319,12 @@ struct dw_hdmi { struct mutex cec_notifier_mutex; struct cec_notifier *cec_notifier; + struct cec_adapter *cec_adap; hdmi_codec_plugged_cb plugged_cb; struct device *codec_dev; enum drm_connector_status last_connector_result; + bool initialized; /* hdmi is enabled before bind */ }; #define HDMI_IH_PHY_STAT0_RX_SENSE \ @@ -263,6 +382,124 @@ static void hdmi_mask_writeb(struct dw_hdmi *hdmi, u8 data, unsigned int reg, hdmi_modb(hdmi, data << shift, mask, reg); } +static bool dw_hdmi_check_output_type_changed(struct dw_hdmi *hdmi) +{ + bool sink_hdmi; + + sink_hdmi = hdmi->sink_is_hdmi; + + if (hdmi->force_output == 1) + hdmi->sink_is_hdmi = true; + else if (hdmi->force_output == 2) + hdmi->sink_is_hdmi = false; + else + hdmi->sink_is_hdmi = hdmi->support_hdmi; + + if (sink_hdmi != hdmi->sink_is_hdmi) + return true; + + return false; +} + +static void repo_hpd_event(struct work_struct *p_work) +{ + struct dw_hdmi *hdmi = container_of(p_work, struct dw_hdmi, work.work); + enum drm_connector_status status = hdmi->hpd_state ? + connector_status_connected : connector_status_disconnected; + u8 phy_stat = hdmi_readb(hdmi, HDMI_PHY_STAT0); + + mutex_lock(&hdmi->mutex); + if (!(phy_stat & HDMI_PHY_RX_SENSE)) + hdmi->rxsense = false; + if (phy_stat & HDMI_PHY_HPD) + hdmi->rxsense = true; + mutex_unlock(&hdmi->mutex); + + if (hdmi->bridge.dev) { + bool change; + + change = drm_helper_hpd_irq_event(hdmi->bridge.dev); + if (change && hdmi->cec_adap && + hdmi->cec_adap->devnode.registered) + cec_queue_pin_hpd_event(hdmi->cec_adap, + hdmi->hpd_state, + ktime_get()); + drm_bridge_hpd_notify(&hdmi->bridge, status); + } +} + +static bool check_hdmi_irq(struct dw_hdmi *hdmi, int intr_stat, + int phy_int_pol) +{ + int msecs; + + /* To determine whether interrupt type is HPD */ + if (!(intr_stat & HDMI_IH_PHY_STAT0_HPD)) + return false; + + if (phy_int_pol & HDMI_PHY_HPD) { + dev_dbg(hdmi->dev, "dw hdmi plug in\n"); + msecs = 150; + hdmi->hpd_state = true; + } else { + dev_dbg(hdmi->dev, "dw hdmi plug out\n"); + msecs = 20; + hdmi->hpd_state = false; + } + mod_delayed_work(hdmi->workqueue, &hdmi->work, msecs_to_jiffies(msecs)); + + return true; +} + +static void init_hpd_work(struct dw_hdmi *hdmi) +{ + hdmi->workqueue = create_workqueue("hpd_queue"); + INIT_DELAYED_WORK(&hdmi->work, repo_hpd_event); +} + +static void dw_hdmi_i2c_set_divs(struct dw_hdmi *hdmi) +{ + unsigned long clk_rate_khz; + unsigned long low_ns, high_ns; + unsigned long div_low, div_high; + + /* Standard-mode */ + if (hdmi->i2c->scl_high_ns < 4000) + high_ns = 4708; + else + high_ns = hdmi->i2c->scl_high_ns; + + if (hdmi->i2c->scl_low_ns < 4700) + low_ns = 4916; + else + low_ns = hdmi->i2c->scl_low_ns; + + /* Adjust to avoid overflow */ + clk_rate_khz = DIV_ROUND_UP(clk_get_rate(hdmi->isfr_clk), 1000); + + div_low = (clk_rate_khz * low_ns) / 1000000; + if ((clk_rate_khz * low_ns) % 1000000) + div_low++; + + div_high = (clk_rate_khz * high_ns) / 1000000; + if ((clk_rate_khz * high_ns) % 1000000) + div_high++; + + /* Maximum divider supported by hw is 0xffff */ + if (div_low > 0xffff) + div_low = 0xffff; + + if (div_high > 0xffff) + div_high = 0xffff; + + hdmi_writeb(hdmi, div_high & 0xff, HDMI_I2CM_SS_SCL_HCNT_0_ADDR); + hdmi_writeb(hdmi, (div_high >> 8) & 0xff, + HDMI_I2CM_SS_SCL_HCNT_1_ADDR); + hdmi_writeb(hdmi, div_low & 0xff, HDMI_I2CM_SS_SCL_LCNT_0_ADDR); + hdmi_writeb(hdmi, (div_low >> 8) & 0xff, + HDMI_I2CM_SS_SCL_LCNT_1_ADDR); +} + static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi) { hdmi_writeb(hdmi, HDMI_PHY_I2CM_INT_ADDR_DONE_POL, @@ -276,7 +513,8 @@ static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi) hdmi_writeb(hdmi, 0x00, HDMI_I2CM_SOFTRSTZ); /* Set Standard Mode speed (determined to be 100KHz on iMX6) */ - hdmi_writeb(hdmi, 0x00, HDMI_I2CM_DIV); + hdmi_modb(hdmi, HDMI_I2CM_DIV_STD_MODE, + HDMI_I2CM_DIV_FAST_STD_MODE, HDMI_I2CM_DIV); /* Set done, not acknowledged and arbitration interrupt polarities */ hdmi_writeb(hdmi, HDMI_I2CM_INT_DONE_POL, HDMI_I2CM_INT); @@ -290,6 +528,11 @@ static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi) /* Mute DONE and ERROR interrupts */ hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE, HDMI_IH_MUTE_I2CM_STAT0); + + /* set SDA high level holding time */ + hdmi_writeb(hdmi, 0x48, HDMI_I2CM_SDA_HOLD); + + dw_hdmi_i2c_set_divs(hdmi); } static bool dw_hdmi_i2c_unwedge(struct dw_hdmi *hdmi) @@ -461,6 +704,8 @@ static int dw_hdmi_i2c_xfer(struct i2c_adapter *adap, hdmi_writeb(hdmi, 0x00, HDMI_IH_MUTE_I2CM_STAT0); /* Set slave device address taken from the first I2C message */ + if (addr == DDC_SEGMENT_ADDR && msgs[0].len == 1) + addr = DDC_ADDR; hdmi_writeb(hdmi, addr, HDMI_I2CM_SLAVE); /* Set slave device register address on transfer */ @@ -570,60 +815,117 @@ static void hdmi_set_cts_n(struct dw_hdmi *hdmi, unsigned int cts, hdmi_writeb(hdmi, n & 0xff, HDMI_AUD_N1); } -static unsigned int hdmi_compute_n(unsigned int freq, unsigned long pixel_clk) +static int hdmi_match_tmds_n_table(struct dw_hdmi *hdmi, + unsigned long pixel_clk, + unsigned long freq) { - unsigned int n = (128 * freq) / 1000; - unsigned int mult = 1; + const struct dw_hdmi_plat_data *plat_data = hdmi->plat_data; + const struct dw_hdmi_audio_tmds_n *tmds_n = NULL; + int i; + + if (plat_data->tmds_n_table) { + for (i = 0; plat_data->tmds_n_table[i].tmds != 0; i++) { + if (pixel_clk == plat_data->tmds_n_table[i].tmds) { + tmds_n = &plat_data->tmds_n_table[i]; + break; + } + } + } - while (freq > 48000) { - mult *= 2; - freq /= 2; + if (tmds_n == NULL) { + for (i = 0; common_tmds_n_table[i].tmds != 0; i++) { + if (pixel_clk == common_tmds_n_table[i].tmds) { + tmds_n = &common_tmds_n_table[i]; + break; + } + } } + if (tmds_n == NULL) + return -ENOENT; + switch (freq) { case 32000: - if (pixel_clk == 25175000) - n = 4576; - else if (pixel_clk == 27027000) - n = 4096; - else if (pixel_clk == 74176000 || pixel_clk == 148352000) - n = 11648; - else - n = 4096; - n *= mult; - break; - + return tmds_n->n_32k; case 44100: - if (pixel_clk == 25175000) - n = 7007; - else if (pixel_clk == 74176000) - n = 17836; - else if (pixel_clk == 148352000) - n = 8918; - else - n = 6272; - n *= mult; - break; - + case 88200: + case 176400: + return (freq / 44100) * tmds_n->n_44k1; case 48000: - if (pixel_clk == 25175000) - n = 6864; - else if (pixel_clk == 27027000) - n = 6144; - else if (pixel_clk == 74176000) - n = 11648; - else if (pixel_clk == 148352000) - n = 5824; - else - n = 6144; - n *= mult; - break; - + case 96000: + case 192000: + return (freq / 48000) * tmds_n->n_48k; default: - break; + return -ENOENT; + } +} + +static u64 hdmi_audio_math_diff(unsigned int freq, unsigned int n, + unsigned int pixel_clk) +{ + u64 final, diff; + u64 cts; + + final = (u64)pixel_clk * n; + + cts = final; + do_div(cts, 128 * freq); + + diff = final - (u64)cts * (128 * freq); + + return diff; +} + +static unsigned int hdmi_compute_n(struct dw_hdmi *hdmi, + unsigned long pixel_clk, + unsigned long freq) +{ + unsigned int min_n = DIV_ROUND_UP((128 * freq), 1500); + unsigned int max_n = (128 * freq) / 300; + unsigned int ideal_n = (128 * freq) / 1000; + unsigned int best_n_distance = ideal_n; + unsigned int best_n = 0; + u64 best_diff = U64_MAX; + int n; + + /* If the ideal N could satisfy the audio math, then just take it */ + if (hdmi_audio_math_diff(freq, ideal_n, pixel_clk) == 0) + return ideal_n; + + for (n = min_n; n <= max_n; n++) { + u64 diff = hdmi_audio_math_diff(freq, n, pixel_clk); + + if (diff < best_diff || (diff == best_diff && + abs(n - ideal_n) < best_n_distance)) { + best_n = n; + best_diff = diff; + best_n_distance = abs(best_n - ideal_n); + } + + /* + * The best N already satisfy the audio math, and also be + * the closest value to ideal N, so just cut the loop. + */ + if ((best_diff == 0) && (abs(n - ideal_n) > best_n_distance)) + break; } - return n; + return best_n; +} + +static unsigned int hdmi_find_n(struct dw_hdmi *hdmi, unsigned long pixel_clk, + unsigned long sample_rate) +{ + int n; + + n = hdmi_match_tmds_n_table(hdmi, pixel_clk, sample_rate); + if (n > 0) + return n; + + dev_warn(hdmi->dev, "Rate %lu missing; compute N dynamically\n", + pixel_clk); + + return hdmi_compute_n(hdmi, pixel_clk, sample_rate); } /* @@ -654,7 +956,7 @@ static void hdmi_set_clk_regenerator(struct dw_hdmi *hdmi, u8 config3; u64 tmp; - n = hdmi_compute_n(sample_rate, pixel_clk); + n = hdmi_find_n(hdmi, pixel_clk, sample_rate); config3 = hdmi_readb(hdmi, HDMI_CONFIG3_ID); @@ -1013,6 +1315,15 @@ static bool is_csc_needed(struct dw_hdmi *hdmi) is_color_space_interpolation(hdmi); } +static bool is_rgb_full_to_limited_needed(struct dw_hdmi *hdmi) +{ + if (hdmi->hdmi_data.quant_range == HDMI_QUANTIZATION_RANGE_LIMITED || + (!hdmi->hdmi_data.quant_range && hdmi->hdmi_data.rgb_limited_range)) + return true; + + return false; +} + static void dw_hdmi_update_csc_coeffs(struct dw_hdmi *hdmi) { const u16 (*csc_coeff)[3][4] = &csc_coeff_default; @@ -1035,7 +1346,7 @@ static void dw_hdmi_update_csc_coeffs(struct dw_hdmi *hdmi) csc_coeff = &csc_coeff_rgb_in_eitu709; csc_scale = 0; } else if (is_input_rgb && is_output_rgb && - hdmi->hdmi_data.rgb_limited_range) { + is_rgb_full_to_limited_needed(hdmi)) { csc_coeff = &csc_coeff_rgb_full_to_rgb_limited; } @@ -1067,7 +1378,7 @@ static void hdmi_video_csc(struct dw_hdmi *hdmi) if (is_color_space_interpolation(hdmi)) interpolation = HDMI_CSC_CFG_INTMODE_CHROMA_INT_FORMULA1; else if (is_color_space_decimation(hdmi)) - decimation = HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA3; + decimation = HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA1; switch (hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format)) { case 8: @@ -1114,7 +1425,7 @@ static void hdmi_video_packetize(struct dw_hdmi *hdmi) switch (hdmi_bus_fmt_color_depth( hdmi->hdmi_data.enc_out_bus_format)) { case 8: - color_depth = 4; + color_depth = 0; output_select = HDMI_VP_CONF_OUTPUT_SELECTOR_BYPASS; break; case 10: @@ -1152,18 +1463,15 @@ static void hdmi_video_packetize(struct dw_hdmi *hdmi) } /* set the packetizer registers */ - val = ((color_depth << HDMI_VP_PR_CD_COLOR_DEPTH_OFFSET) & - HDMI_VP_PR_CD_COLOR_DEPTH_MASK) | - ((hdmi_data->pix_repet_factor << - HDMI_VP_PR_CD_DESIRED_PR_FACTOR_OFFSET) & - HDMI_VP_PR_CD_DESIRED_PR_FACTOR_MASK); + val = (color_depth << HDMI_VP_PR_CD_COLOR_DEPTH_OFFSET) & + HDMI_VP_PR_CD_COLOR_DEPTH_MASK; hdmi_writeb(hdmi, val, HDMI_VP_PR_CD); hdmi_modb(hdmi, HDMI_VP_STUFF_PR_STUFFING_STUFFING_MODE, HDMI_VP_STUFF_PR_STUFFING_MASK, HDMI_VP_STUFF); /* Data from pixel repeater block */ - if (hdmi_data->pix_repet_factor > 1) { + if (hdmi_data->pix_repet_factor > 0) { vp_conf = HDMI_VP_CONF_PR_EN_ENABLE | HDMI_VP_CONF_BYPASS_SELECT_PIX_REPEATER; } else { /* data from packetizer block */ @@ -1175,8 +1483,13 @@ static void hdmi_video_packetize(struct dw_hdmi *hdmi) HDMI_VP_CONF_PR_EN_MASK | HDMI_VP_CONF_BYPASS_SELECT_MASK, HDMI_VP_CONF); - hdmi_modb(hdmi, 1 << HDMI_VP_STUFF_IDEFAULT_PHASE_OFFSET, - HDMI_VP_STUFF_IDEFAULT_PHASE_MASK, HDMI_VP_STUFF); + if ((color_depth == 5 && hdmi->previous_mode.htotal % 4) || + (color_depth == 6 && hdmi->previous_mode.htotal % 2)) + hdmi_modb(hdmi, 0, HDMI_VP_STUFF_IDEFAULT_PHASE_MASK, + HDMI_VP_STUFF); + else + hdmi_modb(hdmi, 1 << HDMI_VP_STUFF_IDEFAULT_PHASE_OFFSET, + HDMI_VP_STUFF_IDEFAULT_PHASE_MASK, HDMI_VP_STUFF); hdmi_writeb(hdmi, remap_size, HDMI_VP_REMAP); @@ -1277,6 +1590,23 @@ static bool dw_hdmi_support_scdc(struct dw_hdmi *hdmi, return true; } +static int hdmi_phy_i2c_read(struct dw_hdmi *hdmi, unsigned char addr) +{ + int val; + + hdmi_writeb(hdmi, 0xFF, HDMI_IH_I2CMPHY_STAT0); + hdmi_writeb(hdmi, addr, HDMI_PHY_I2CM_ADDRESS_ADDR); + hdmi_writeb(hdmi, 0, HDMI_PHY_I2CM_DATAI_1_ADDR); + hdmi_writeb(hdmi, 0, HDMI_PHY_I2CM_DATAI_0_ADDR); + hdmi_writeb(hdmi, HDMI_PHY_I2CM_OPERATION_ADDR_READ, + HDMI_PHY_I2CM_OPERATION_ADDR); + hdmi_phy_wait_i2c_done(hdmi, 1000); + val = hdmi_readb(hdmi, HDMI_PHY_I2CM_DATAI_1_ADDR); + val = (val & 0xff) << 8; + val += hdmi_readb(hdmi, HDMI_PHY_I2CM_DATAI_0_ADDR) & 0xff; + return val; +} + /* * HDMI2.0 Specifies the following procedure for High TMDS Bit Rates: * - The Source shall suspend transmission of the TMDS clock and data @@ -1454,6 +1784,13 @@ static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi, const struct dw_hdmi_mpll_config *mpll_config = pdata->mpll_cfg; const struct dw_hdmi_curr_ctrl *curr_ctrl = pdata->cur_ctr; const struct dw_hdmi_phy_config *phy_config = pdata->phy_config; + unsigned int tmdsclock = hdmi->hdmi_data.video_mode.mtmdsclock; + unsigned int depth = + hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format); + + if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format) && + pdata->mpll_cfg_420) + mpll_config = pdata->mpll_cfg_420; /* TOFIX Will need 420 specific PHY configuration tables */ @@ -1463,11 +1800,11 @@ static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi, break; for (; curr_ctrl->mpixelclock != ~0UL; curr_ctrl++) - if (mpixelclock <= curr_ctrl->mpixelclock) + if (tmdsclock <= curr_ctrl->mpixelclock) break; for (; phy_config->mpixelclock != ~0UL; phy_config++) - if (mpixelclock <= phy_config->mpixelclock) + if (tmdsclock <= phy_config->mpixelclock) break; if (mpll_config->mpixelclock == ~0UL || @@ -1475,11 +1812,18 @@ static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi, phy_config->mpixelclock == ~0UL) return -EINVAL; - dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[0].cpce, + if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) + depth = fls(depth - 8); + else + depth = 0; + if (depth) + depth--; + + dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[depth].cpce, HDMI_3D_TX_PHY_CPCE_CTRL); - dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[0].gmp, + dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[depth].gmp, HDMI_3D_TX_PHY_GMPCTRL); - dw_hdmi_phy_i2c_write(hdmi, curr_ctrl->curr[0], + dw_hdmi_phy_i2c_write(hdmi, curr_ctrl->curr[depth], HDMI_3D_TX_PHY_CURRCTRL); dw_hdmi_phy_i2c_write(hdmi, 0, HDMI_3D_TX_PHY_PLLPHBYCTRL); @@ -1492,10 +1836,6 @@ static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi, dw_hdmi_phy_i2c_write(hdmi, phy_config->vlev_ctr, HDMI_3D_TX_PHY_VLEVCTRL); - /* Override and disable clock termination. */ - dw_hdmi_phy_i2c_write(hdmi, HDMI_3D_TX_PHY_CKCALCTRL_OVERRIDE, - HDMI_3D_TX_PHY_CKCALCTRL); - return 0; } @@ -1597,14 +1937,16 @@ void dw_hdmi_phy_setup_hpd(struct dw_hdmi *hdmi, void *data) hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE, HDMI_IH_PHY_STAT0); - /* Enable cable hot plug irq. */ - hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0); + if (!hdmi->next_bridge) { + /* Enable cable hot plug irq. */ + hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0); - /* Clear and unmute interrupts. */ - hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE, - HDMI_IH_PHY_STAT0); - hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE), - HDMI_IH_MUTE_PHY_STAT0); + /* Clear and unmute interrupts. */ + hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE, + HDMI_IH_PHY_STAT0); + hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE), + HDMI_IH_MUTE_PHY_STAT0); + } } EXPORT_SYMBOL_GPL(dw_hdmi_phy_setup_hpd); @@ -1620,23 +1962,36 @@ static const struct dw_hdmi_phy_ops dw_hdmi_synopsys_phy_ops = { * HDMI TX Setup */ -static void hdmi_tx_hdcp_config(struct dw_hdmi *hdmi) +static void hdmi_tx_hdcp_config(struct dw_hdmi *hdmi, + const struct drm_display_mode *mode) { - u8 de; - - if (hdmi->hdmi_data.video_mode.mdataenablepolarity) - de = HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_HIGH; - else - de = HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_LOW; - - /* disable rx detect */ - hdmi_modb(hdmi, HDMI_A_HDCPCFG0_RXDETECT_DISABLE, - HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0); - - hdmi_modb(hdmi, de, HDMI_A_VIDPOLCFG_DATAENPOL_MASK, HDMI_A_VIDPOLCFG); - - hdmi_modb(hdmi, HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_DISABLE, - HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK, HDMI_A_HDCPCFG1); + struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode; + u8 vsync_pol, hsync_pol, data_pol, hdmi_dvi; + + /* Configure the video polarity */ + vsync_pol = mode->flags & DRM_MODE_FLAG_PVSYNC ? + HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_HIGH : + HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_LOW; + hsync_pol = mode->flags & DRM_MODE_FLAG_PHSYNC ? + HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_HIGH : + HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_LOW; + data_pol = vmode->mdataenablepolarity ? + HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_HIGH : + HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_LOW; + hdmi_modb(hdmi, vsync_pol | hsync_pol | data_pol, + HDMI_A_VIDPOLCFG_VSYNCPOL_MASK | + HDMI_A_VIDPOLCFG_HSYNCPOL_MASK | + HDMI_A_VIDPOLCFG_DATAENPOL_MASK, + HDMI_A_VIDPOLCFG); + + /* Config the display mode */ + hdmi_dvi = hdmi->sink_is_hdmi ? HDMI_A_HDCPCFG0_HDMIDVI_HDMI : + HDMI_A_HDCPCFG0_HDMIDVI_DVI; + hdmi_modb(hdmi, hdmi_dvi, HDMI_A_HDCPCFG0_HDMIDVI_MASK, + HDMI_A_HDCPCFG0); + + if (hdmi->hdcp && hdmi->hdcp->hdcp_start) + hdmi->hdcp->hdcp_start(hdmi->hdcp); } static void hdmi_config_AVI(struct dw_hdmi *hdmi, @@ -1650,10 +2005,15 @@ static void hdmi_config_AVI(struct dw_hdmi *hdmi, drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode); if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) { - drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, - hdmi->hdmi_data.rgb_limited_range ? - HDMI_QUANTIZATION_RANGE_LIMITED : - HDMI_QUANTIZATION_RANGE_FULL); + /* default range */ + if (!hdmi->hdmi_data.quant_range) + drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, + hdmi->hdmi_data.rgb_limited_range ? + HDMI_QUANTIZATION_RANGE_LIMITED : + HDMI_QUANTIZATION_RANGE_FULL); + else + drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, + hdmi->hdmi_data.quant_range); } else { frame.quantization_range = HDMI_QUANTIZATION_RANGE_DEFAULT; frame.ycc_quantization_range = @@ -1688,6 +2048,14 @@ static void hdmi_config_AVI(struct dw_hdmi *hdmi, frame.extended_colorimetry = HDMI_EXTENDED_COLORIMETRY_XV_YCC_709; break; + case V4L2_YCBCR_ENC_BT2020: + if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_BT2020) + frame.colorimetry = HDMI_COLORIMETRY_EXTENDED; + else + frame.colorimetry = HDMI_COLORIMETRY_ITU_709; + frame.extended_colorimetry = + HDMI_EXTENDED_COLORIMETRY_BT2020; + break; default: /* Carries no data */ frame.colorimetry = HDMI_COLORIMETRY_ITU_601; frame.extended_colorimetry = @@ -1824,17 +2192,44 @@ static void hdmi_config_drm_infoframe(struct dw_hdmi *hdmi, const struct drm_connector *connector) { const struct drm_connector_state *conn_state = connector->state; + struct hdr_output_metadata *hdr_metadata; struct hdmi_drm_infoframe frame; u8 buffer[30]; ssize_t err; int i; + /* Dynamic Range and Mastering Infoframe is introduced in v2.11a. */ + if (hdmi->version < 0x211a) { + DRM_ERROR("Not support DRM Infoframe\n"); + return; + } + if (!hdmi->plat_data->use_drm_infoframe) return; hdmi_modb(hdmi, HDMI_FC_PACKET_TX_EN_DRM_DISABLE, HDMI_FC_PACKET_TX_EN_DRM_MASK, HDMI_FC_PACKET_TX_EN); + if (!hdmi->connector.hdr_sink_metadata.hdmi_type1.eotf) { + DRM_DEBUG("No need to set HDR metadata in infoframe\n"); + return; + } + + if (!conn_state->hdr_output_metadata) { + DRM_DEBUG("source metadata not set yet\n"); + return; + } + + hdr_metadata = (struct hdr_output_metadata *) + conn_state->hdr_output_metadata->data; + + if (!(hdmi->connector.hdr_sink_metadata.hdmi_type1.eotf & + BIT(hdr_metadata->hdmi_metadata_type1.eotf))) { + DRM_ERROR("Not support EOTF %d\n", + hdr_metadata->hdmi_metadata_type1.eotf); + return; + } + err = drm_hdmi_infoframe_set_hdr_metadata(&frame, conn_state); if (err < 0) return; @@ -1854,51 +2249,66 @@ static void hdmi_config_drm_infoframe(struct dw_hdmi *hdmi, hdmi_writeb(hdmi, 1, HDMI_FC_DRM_UP); hdmi_modb(hdmi, HDMI_FC_PACKET_TX_EN_DRM_ENABLE, HDMI_FC_PACKET_TX_EN_DRM_MASK, HDMI_FC_PACKET_TX_EN); + + DRM_DEBUG("%s eotf %d end\n", __func__, + hdr_metadata->hdmi_metadata_type1.eotf); } -static void hdmi_av_composer(struct dw_hdmi *hdmi, - const struct drm_display_info *display, - const struct drm_display_mode *mode) +static unsigned int +hdmi_get_tmdsclock(struct dw_hdmi *hdmi, unsigned long mpixelclock) { - u8 inv_val, bytes; - const struct drm_hdmi_info *hdmi_info = &display->hdmi; - struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode; - int hblank, vblank, h_de_hs, v_de_vs, hsync_len, vsync_len; - unsigned int vdisplay, hdisplay; - - vmode->mpixelclock = mode->clock * 1000; - - dev_dbg(hdmi->dev, "final pixclk = %d\n", vmode->mpixelclock); - - vmode->mtmdsclock = vmode->mpixelclock; + unsigned int tmdsclock = mpixelclock; + unsigned int depth = + hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format); if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) { - switch (hdmi_bus_fmt_color_depth( - hdmi->hdmi_data.enc_out_bus_format)) { + switch (depth) { case 16: - vmode->mtmdsclock = vmode->mpixelclock * 2; + tmdsclock = mpixelclock * 2; break; case 12: - vmode->mtmdsclock = vmode->mpixelclock * 3 / 2; + tmdsclock = mpixelclock * 3 / 2; break; case 10: - vmode->mtmdsclock = vmode->mpixelclock * 5 / 4; + tmdsclock = mpixelclock * 5 / 4; + break; + default: break; } } + return tmdsclock; +} + +static void hdmi_av_composer(struct dw_hdmi *hdmi, + const struct drm_display_info *display, + const struct drm_display_mode *mode) +{ + u8 inv_val, bytes; + const struct drm_hdmi_info *hdmi_info = &display->hdmi; + struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode; + int hblank, vblank, h_de_hs, v_de_vs, hsync_len, vsync_len; + unsigned int vdisplay, hdisplay; + + vmode->previous_pixelclock = vmode->mpixelclock; + vmode->mpixelclock = mode->crtc_clock * 1000; + if ((mode->flags & DRM_MODE_FLAG_3D_MASK) == + DRM_MODE_FLAG_3D_FRAME_PACKING) + vmode->mpixelclock *= 2; + dev_dbg(hdmi->dev, "final pixclk = %d\n", vmode->mpixelclock); + + vmode->previous_tmdsclock = vmode->mtmdsclock; + vmode->mtmdsclock = hdmi_get_tmdsclock(hdmi, vmode->mpixelclock); if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) vmode->mtmdsclock /= 2; - dev_dbg(hdmi->dev, "final tmdsclock = %d\n", vmode->mtmdsclock); - /* Set up HDMI_FC_INVIDCONF */ - inv_val = (hdmi->hdmi_data.hdcp_enable || - (dw_hdmi_support_scdc(hdmi, display) && - (vmode->mtmdsclock > HDMI14_MAX_TMDSCLK || - hdmi_info->scdc.scrambling.low_rates)) ? - HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE : - HDMI_FC_INVIDCONF_HDCP_KEEPOUT_INACTIVE); + /* Set up HDMI_FC_INVIDCONF + * Some display equipments require that the interval + * between Video Data and Data island must be at least 58 pixels, + * and fc_invidconf.HDCP_keepout set (1'b1) can meet the requirement. + */ + inv_val = HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE; inv_val |= mode->flags & DRM_MODE_FLAG_PVSYNC ? HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_HIGH : @@ -1964,7 +2374,8 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi, /* Scrambling Control */ if (dw_hdmi_support_scdc(hdmi, display)) { if (vmode->mtmdsclock > HDMI14_MAX_TMDSCLK || - hdmi_info->scdc.scrambling.low_rates) { + (hdmi_info->scdc.scrambling.low_rates && + hdmi->scramble_low_rates)) { /* * HDMI2.0 Specifies the following procedure: * After the Source Device has determined that @@ -1998,6 +2409,8 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi, HDMI_MC_SWRSTZ); drm_scdc_set_scrambling(hdmi->ddc, 0); } + } else { + hdmi_writeb(hdmi, 0, HDMI_FC_SCRAMBLER_CTRL); } /* Set up horizontal active pixel width */ @@ -2055,6 +2468,12 @@ static void dw_hdmi_enable_video_path(struct dw_hdmi *hdmi) hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_TMDSCLK_DISABLE; hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS); + /* Enable pixel repetition path */ + if (hdmi->hdmi_data.video_mode.mpixelrepetitioninput) { + hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_PREPCLK_DISABLE; + hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS); + } + /* Enable csc path */ if (is_csc_needed(hdmi)) { hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_CSCCLK_DISABLE; @@ -2130,6 +2549,7 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, const struct drm_display_mode *mode) { int ret; + void *data = hdmi->plat_data->phy_data; hdmi_disable_overflow_interrupts(hdmi); @@ -2141,48 +2561,91 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, dev_dbg(hdmi->dev, "CEA mode used vic=%d\n", hdmi->vic); } - if ((hdmi->vic == 6) || (hdmi->vic == 7) || - (hdmi->vic == 21) || (hdmi->vic == 22) || - (hdmi->vic == 2) || (hdmi->vic == 3) || - (hdmi->vic == 17) || (hdmi->vic == 18)) + if (hdmi->plat_data->get_enc_out_encoding) + hdmi->hdmi_data.enc_out_encoding = + hdmi->plat_data->get_enc_out_encoding(data); + else if ((hdmi->vic == 6) || (hdmi->vic == 7) || + (hdmi->vic == 21) || (hdmi->vic == 22) || + (hdmi->vic == 2) || (hdmi->vic == 3) || + (hdmi->vic == 17) || (hdmi->vic == 18)) hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_601; else hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_709; - hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0; - hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0; + if (mode->flags & DRM_MODE_FLAG_DBLCLK) { + hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 1; + hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 1; + } else { + hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0; + hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0; + } + /* TOFIX: Get input format from plat data or fallback to RGB888 */ + if (hdmi->plat_data->get_input_bus_format) + hdmi->hdmi_data.enc_in_bus_format = + hdmi->plat_data->get_input_bus_format(data); + else if (hdmi->plat_data->input_bus_format) + hdmi->hdmi_data.enc_in_bus_format = + hdmi->plat_data->input_bus_format; + else + hdmi->hdmi_data.enc_in_bus_format = + MEDIA_BUS_FMT_RGB888_1X24; - if (hdmi->hdmi_data.enc_in_bus_format == MEDIA_BUS_FMT_FIXED) - hdmi->hdmi_data.enc_in_bus_format = MEDIA_BUS_FMT_RGB888_1X24; + /* TOFIX: Default to RGB888 output format */ + if (hdmi->plat_data->get_output_bus_format) + hdmi->hdmi_data.enc_out_bus_format = + hdmi->plat_data->get_output_bus_format(data); + else + hdmi->hdmi_data.enc_out_bus_format = + MEDIA_BUS_FMT_RGB888_1X24; /* TOFIX: Get input encoding from plat data or fallback to none */ - if (hdmi->plat_data->input_bus_encoding) + if (hdmi->plat_data->get_enc_in_encoding) + hdmi->hdmi_data.enc_in_encoding = + hdmi->plat_data->get_enc_in_encoding(data); + else if (hdmi->plat_data->input_bus_encoding) hdmi->hdmi_data.enc_in_encoding = hdmi->plat_data->input_bus_encoding; else hdmi->hdmi_data.enc_in_encoding = V4L2_YCBCR_ENC_DEFAULT; - if (hdmi->hdmi_data.enc_out_bus_format == MEDIA_BUS_FMT_FIXED) - hdmi->hdmi_data.enc_out_bus_format = MEDIA_BUS_FMT_RGB888_1X24; + + if (hdmi->plat_data->get_quant_range) + hdmi->hdmi_data.quant_range = + hdmi->plat_data->get_quant_range(data); hdmi->hdmi_data.rgb_limited_range = hdmi->sink_is_hdmi && drm_default_rgb_quant_range(mode) == HDMI_QUANTIZATION_RANGE_LIMITED; - hdmi->hdmi_data.pix_repet_factor = 0; - hdmi->hdmi_data.hdcp_enable = 0; + if (!hdmi->sink_is_hdmi) + hdmi->hdmi_data.quant_range = HDMI_QUANTIZATION_RANGE_FULL; + + /* + * According to the dw-hdmi specification 6.4.2 + * vp_pr_cd[3:0]: + * 0000b: No pixel repetition (pixel sent only once) + * 0001b: Pixel sent two times (pixel repeated once) + */ + hdmi->hdmi_data.pix_repet_factor = + (mode->flags & DRM_MODE_FLAG_DBLCLK) ? 1 : 0; hdmi->hdmi_data.video_mode.mdataenablepolarity = true; /* HDMI Initialization Step B.1 */ hdmi_av_composer(hdmi, &connector->display_info, mode); /* HDMI Initializateion Step B.2 */ - ret = hdmi->phy.ops->init(hdmi, hdmi->phy.data, - &connector->display_info, - &hdmi->previous_mode); - if (ret) - return ret; - hdmi->phy.enabled = true; + if (!hdmi->phy.enabled || + hdmi->hdmi_data.video_mode.previous_pixelclock != + hdmi->hdmi_data.video_mode.mpixelclock || + hdmi->hdmi_data.video_mode.previous_tmdsclock != + hdmi->hdmi_data.video_mode.mtmdsclock) { + ret = hdmi->phy.ops->init(hdmi, hdmi->phy.data, + &connector->display_info, + &hdmi->previous_mode); + if (ret) + return ret; + hdmi->phy.enabled = true; + } /* HDMI Initialization Step B.3 */ dw_hdmi_enable_video_path(hdmi); @@ -2210,7 +2673,7 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, hdmi_video_packetize(hdmi); hdmi_video_csc(hdmi); hdmi_video_sample(hdmi); - hdmi_tx_hdcp_config(hdmi); + hdmi_tx_hdcp_config(hdmi, mode); dw_hdmi_clear_overflow(hdmi); @@ -2286,6 +2749,8 @@ static void dw_hdmi_poweroff(struct dw_hdmi *hdmi) hdmi->phy.enabled = false; } + if (hdmi->hdcp && hdmi->hdcp->hdcp_stop) + hdmi->hdcp->hdcp_stop(hdmi->hdcp); hdmi->bridge_is_on = false; } @@ -2303,6 +2768,10 @@ static void dw_hdmi_update_power(struct dw_hdmi *hdmi) } if (force == DRM_FORCE_OFF) { + if (hdmi->initialized) { + hdmi->initialized = false; + hdmi->disabled = true; + } if (hdmi->bridge_is_on) dw_hdmi_poweroff(hdmi); } else { @@ -2335,8 +2804,15 @@ static enum drm_connector_status dw_hdmi_detect(struct dw_hdmi *hdmi) { enum drm_connector_status result; - result = hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data); + if (!hdmi->force_logo) { + mutex_lock(&hdmi->mutex); + hdmi->force = DRM_FORCE_UNSPECIFIED; + dw_hdmi_update_power(hdmi); + dw_hdmi_update_phy_mask(hdmi); + mutex_unlock(&hdmi->mutex); + } + result = hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data); mutex_lock(&hdmi->mutex); if (result != hdmi->last_connector_result) { dev_dbg(hdmi->dev, "read_hpd result: %d", result); @@ -2346,6 +2822,11 @@ static enum drm_connector_status dw_hdmi_detect(struct dw_hdmi *hdmi) } mutex_unlock(&hdmi->mutex); + if (result == connector_status_connected) + extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, true); + else + extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, false); + return result; } @@ -2366,7 +2847,7 @@ static struct edid *dw_hdmi_get_edid(struct dw_hdmi *hdmi, dev_dbg(hdmi->dev, "got edid: width[%d] x height[%d]\n", edid->width_cm, edid->height_cm); - hdmi->sink_is_hdmi = drm_detect_hdmi_monitor(edid); + hdmi->support_hdmi = drm_detect_hdmi_monitor(edid); hdmi->sink_has_audio = drm_detect_monitor_audio(edid); return edid; @@ -2384,21 +2865,105 @@ dw_hdmi_connector_detect(struct drm_connector *connector, bool force) return dw_hdmi_detect(hdmi); } +static int +dw_hdmi_update_hdr_property(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, + connector); + void *data = hdmi->plat_data->phy_data; + const struct hdr_static_metadata *metadata = + &connector->hdr_sink_metadata.hdmi_type1; + size_t size = sizeof(*metadata); + struct drm_property *property; + struct drm_property_blob *blob; + int ret; + + if (hdmi->plat_data->get_hdr_property) + property = hdmi->plat_data->get_hdr_property(data); + else + return -EINVAL; + + if (hdmi->plat_data->get_hdr_blob) + blob = hdmi->plat_data->get_hdr_blob(data); + else + return -EINVAL; + + ret = drm_property_replace_global_blob(dev, &blob, size, metadata, + &connector->base, property); + return ret; +} + static int dw_hdmi_connector_get_modes(struct drm_connector *connector) { struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, connector); + struct hdr_static_metadata *metedata = + &connector->hdr_sink_metadata.hdmi_type1; struct edid *edid; - int ret; + struct drm_display_mode *mode; + struct drm_display_info *info = &connector->display_info; + int i, ret = 0; + memset(metedata, 0, sizeof(*metedata)); edid = dw_hdmi_get_edid(hdmi, connector); - if (!edid) - return 0; + if (edid) { + dev_dbg(hdmi->dev, "got edid: width[%d] x height[%d]\n", + edid->width_cm, edid->height_cm); + drm_connector_update_edid_property(connector, edid); + cec_notifier_set_phys_addr_from_edid(hdmi->cec_notifier, edid); + ret = drm_add_edid_modes(connector, edid); + if (hdmi->plat_data->get_color_changed) + hdmi->plat_data->get_yuv422_format(connector, edid); + dw_hdmi_update_hdr_property(connector); + kfree(edid); + } else { + hdmi->support_hdmi = true; + hdmi->sink_has_audio = true; + for (i = 0; i < ARRAY_SIZE(dw_hdmi_default_modes); i++) { + const struct drm_display_mode *ptr = + &dw_hdmi_default_modes[i]; + + mode = drm_mode_duplicate(connector->dev, ptr); + if (mode) { + if (!i) { + mode->type = DRM_MODE_TYPE_PREFERRED; + mode->picture_aspect_ratio = + HDMI_PICTURE_ASPECT_NONE; + } + drm_mode_probed_add(connector, mode); + ret++; + } + } + info->edid_hdmi_dc_modes = 0; + info->hdmi.y420_dc_modes = 0; + info->color_formats = 0; + + dev_info(hdmi->dev, "failed to get edid\n"); + } + dw_hdmi_check_output_type_changed(hdmi); + + return ret; +} + +static struct drm_encoder * +dw_hdmi_connector_best_encoder(struct drm_connector *connector) +{ + struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, + connector); + + return hdmi->bridge.encoder; +} - drm_connector_update_edid_property(connector, edid); - cec_notifier_set_phys_addr_from_edid(hdmi->cec_notifier, edid); - ret = drm_add_edid_modes(connector, edid); - kfree(edid); +static bool dw_hdmi_color_changed(struct drm_connector *connector) +{ + struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, + connector); + void *data = hdmi->plat_data->phy_data; + bool ret = false; + + if (hdmi->plat_data->get_color_changed) + ret = hdmi->plat_data->get_color_changed(data); return ret; } @@ -2427,11 +2992,54 @@ static int dw_hdmi_connector_atomic_check(struct drm_connector *connector, drm_atomic_get_new_connector_state(state, connector); struct drm_crtc *crtc = new_state->crtc; struct drm_crtc_state *crtc_state; + struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, + connector); + struct drm_display_mode *mode = NULL; + void *data = hdmi->plat_data->phy_data; + struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode; + unsigned int in_bus_format = hdmi->hdmi_data.enc_in_bus_format; + unsigned int out_bus_format = hdmi->hdmi_data.enc_out_bus_format; + bool color_changed = false; if (!crtc) return 0; - if (!hdr_metadata_equal(old_state, new_state)) { + /* + * If HDMI is enabled in uboot, it's need to record + * drm_display_mode and set phy status to enabled. + */ + if (!vmode->mpixelclock) { + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (hdmi->plat_data->get_enc_in_encoding) + hdmi->hdmi_data.enc_in_encoding = + hdmi->plat_data->get_enc_in_encoding(data); + if (hdmi->plat_data->get_enc_out_encoding) + hdmi->hdmi_data.enc_out_encoding = + hdmi->plat_data->get_enc_out_encoding(data); + if (hdmi->plat_data->get_input_bus_format) + hdmi->hdmi_data.enc_in_bus_format = + hdmi->plat_data->get_input_bus_format(data); + if (hdmi->plat_data->get_output_bus_format) + hdmi->hdmi_data.enc_out_bus_format = + hdmi->plat_data->get_output_bus_format(data); + + mode = &crtc_state->mode; + memcpy(&hdmi->previous_mode, mode, sizeof(hdmi->previous_mode)); + vmode->mpixelclock = mode->crtc_clock * 1000; + vmode->previous_pixelclock = mode->clock; + vmode->previous_tmdsclock = mode->clock; + vmode->mtmdsclock = hdmi_get_tmdsclock(hdmi, + vmode->mpixelclock); + if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) + vmode->mtmdsclock /= 2; + + if (in_bus_format != hdmi->hdmi_data.enc_in_bus_format || + out_bus_format != hdmi->hdmi_data.enc_out_bus_format) + color_changed = true; + } + + if (!hdr_metadata_equal(old_state, new_state) || + dw_hdmi_color_changed(connector) || color_changed) { crtc_state = drm_atomic_get_crtc_state(state, crtc); if (IS_ERR(crtc_state)) return PTR_ERR(crtc_state); @@ -2442,12 +3050,105 @@ static int dw_hdmi_connector_atomic_check(struct drm_connector *connector, return 0; } +static int +dw_hdmi_atomic_connector_set_property(struct drm_connector *connector, + struct drm_connector_state *state, + struct drm_property *property, + uint64_t val) +{ + struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, + connector); + const struct dw_hdmi_property_ops *ops = + hdmi->plat_data->property_ops; + + if (ops && ops->set_property) + return ops->set_property(connector, state, property, + val, hdmi->plat_data->phy_data); + else + return -EINVAL; +} + +static int +dw_hdmi_atomic_connector_get_property(struct drm_connector *connector, + const struct drm_connector_state *state, + struct drm_property *property, + uint64_t *val) +{ + struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, + connector); + const struct dw_hdmi_property_ops *ops = + hdmi->plat_data->property_ops; + + if (ops && ops->get_property) + return ops->get_property(connector, state, property, + val, hdmi->plat_data->phy_data); + else + return -EINVAL; +} + +static int +dw_hdmi_connector_set_property(struct drm_connector *connector, + struct drm_property *property, uint64_t val) +{ + return dw_hdmi_atomic_connector_set_property(connector, NULL, + property, val); +} + +void dw_hdmi_set_quant_range(struct dw_hdmi *hdmi) +{ + if (!hdmi->bridge_is_on) + return; + + hdmi_writeb(hdmi, HDMI_FC_GCP_SET_AVMUTE, HDMI_FC_GCP); + dw_hdmi_setup(hdmi, hdmi->curr_conn, &hdmi->previous_mode); + hdmi_writeb(hdmi, HDMI_FC_GCP_CLEAR_AVMUTE, HDMI_FC_GCP); +} +EXPORT_SYMBOL_GPL(dw_hdmi_set_quant_range); + +void dw_hdmi_set_output_type(struct dw_hdmi *hdmi, u64 val) +{ + hdmi->force_output = val; + + if (!dw_hdmi_check_output_type_changed(hdmi)) + return; + + if (!hdmi->bridge_is_on) + return; + + hdmi_writeb(hdmi, HDMI_FC_GCP_SET_AVMUTE, HDMI_FC_GCP); + dw_hdmi_setup(hdmi, hdmi->curr_conn, &hdmi->previous_mode); + hdmi_writeb(hdmi, HDMI_FC_GCP_CLEAR_AVMUTE, HDMI_FC_GCP); +} +EXPORT_SYMBOL_GPL(dw_hdmi_set_output_type); + +bool dw_hdmi_get_output_whether_hdmi(struct dw_hdmi *hdmi) +{ + return hdmi->sink_is_hdmi; +} +EXPORT_SYMBOL_GPL(dw_hdmi_get_output_whether_hdmi); + +int dw_hdmi_get_output_type_cap(struct dw_hdmi *hdmi) +{ + return hdmi->support_hdmi; +} +EXPORT_SYMBOL_GPL(dw_hdmi_get_output_type_cap); + static void dw_hdmi_connector_force(struct drm_connector *connector) { struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, connector); mutex_lock(&hdmi->mutex); + + if (hdmi->force != connector->force) { + if (!hdmi->disabled && connector->force == DRM_FORCE_OFF) + extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, + false); + else if (hdmi->disabled && connector->force == DRM_FORCE_ON) + extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, + true); + } + hdmi->force = connector->force; dw_hdmi_update_power(hdmi); dw_hdmi_update_phy_mask(hdmi); @@ -2460,15 +3161,98 @@ static const struct drm_connector_funcs dw_hdmi_connector_funcs = { .destroy = drm_connector_cleanup, .force = dw_hdmi_connector_force, .reset = drm_atomic_helper_connector_reset, + .set_property = dw_hdmi_connector_set_property, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, + .atomic_set_property = dw_hdmi_atomic_connector_set_property, + .atomic_get_property = dw_hdmi_atomic_connector_get_property, }; static const struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = { .get_modes = dw_hdmi_connector_get_modes, + .best_encoder = dw_hdmi_connector_best_encoder, .atomic_check = dw_hdmi_connector_atomic_check, }; +static void dw_hdmi_attach_properties(struct dw_hdmi *hdmi) +{ + unsigned int color = MEDIA_BUS_FMT_RGB888_1X24; + int video_mapping, colorspace; + enum drm_connector_status connect_status = + hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data); + const struct dw_hdmi_property_ops *ops = + hdmi->plat_data->property_ops; + + if (connect_status == connector_status_connected) { + video_mapping = (hdmi_readb(hdmi, HDMI_TX_INVID0) & + HDMI_TX_INVID0_VIDEO_MAPPING_MASK); + colorspace = (hdmi_readb(hdmi, HDMI_FC_AVICONF0) & + HDMI_FC_AVICONF0_PIX_FMT_MASK); + switch (video_mapping) { + case 0x01: + color = MEDIA_BUS_FMT_RGB888_1X24; + break; + case 0x03: + color = MEDIA_BUS_FMT_RGB101010_1X30; + break; + case 0x09: + if (colorspace == HDMI_COLORSPACE_YUV420) + color = MEDIA_BUS_FMT_UYYVYY8_0_5X24; + else if (colorspace == HDMI_COLORSPACE_YUV422) + color = MEDIA_BUS_FMT_UYVY8_1X16; + else + color = MEDIA_BUS_FMT_YUV8_1X24; + break; + case 0x0b: + if (colorspace == HDMI_COLORSPACE_YUV420) + color = MEDIA_BUS_FMT_UYYVYY10_0_5X30; + else if (colorspace == HDMI_COLORSPACE_YUV422) + color = MEDIA_BUS_FMT_UYVY10_1X20; + else + color = MEDIA_BUS_FMT_YUV10_1X30; + break; + case 0x14: + color = MEDIA_BUS_FMT_UYVY10_1X20; + break; + case 0x16: + color = MEDIA_BUS_FMT_UYVY8_1X16; + break; + default: + color = MEDIA_BUS_FMT_RGB888_1X24; + dev_err(hdmi->dev, "unexpected mapping: 0x%x\n", + video_mapping); + } + + hdmi->hdmi_data.enc_in_bus_format = color; + hdmi->hdmi_data.enc_out_bus_format = color; + /* + * input format will be set as yuv444 when output + * format is yuv420 + */ + if (color == MEDIA_BUS_FMT_UYVY10_1X20) + hdmi->hdmi_data.enc_in_bus_format = + MEDIA_BUS_FMT_YUV10_1X30; + else if (color == MEDIA_BUS_FMT_UYVY8_1X16) + hdmi->hdmi_data.enc_in_bus_format = + MEDIA_BUS_FMT_YUV8_1X24; + } + + if (ops && ops->attach_properties) + return ops->attach_properties(&hdmi->connector, + color, hdmi->version, + hdmi->plat_data->phy_data); +} + +static void dw_hdmi_destroy_properties(struct dw_hdmi *hdmi) +{ + const struct dw_hdmi_property_ops *ops = + hdmi->plat_data->property_ops; + + if (ops && ops->destroy_properties) + return ops->destroy_properties(&hdmi->connector, + hdmi->plat_data->phy_data); +} + static int dw_hdmi_connector_create(struct dw_hdmi *hdmi) { struct drm_connector *connector = &hdmi->connector; @@ -2505,6 +3289,8 @@ static int dw_hdmi_connector_create(struct dw_hdmi *hdmi) drm_connector_attach_encoder(connector, hdmi->bridge.encoder); + dw_hdmi_attach_properties(hdmi); + cec_fill_conn_info_from_drm(&conn_info, connector); notifier = cec_notifier_conn_register(hdmi->dev, NULL, &conn_info); @@ -2780,16 +3566,36 @@ static int dw_hdmi_bridge_atomic_check(struct drm_bridge *bridge, struct drm_connector_state *conn_state) { struct dw_hdmi *hdmi = bridge->driver_private; + void *data = hdmi->plat_data->phy_data; - hdmi->hdmi_data.enc_out_bus_format = - bridge_state->output_bus_cfg.format; + if (bridge_state->output_bus_cfg.format == MEDIA_BUS_FMT_FIXED) { + if (hdmi->plat_data->get_output_bus_format) + hdmi->hdmi_data.enc_out_bus_format = + hdmi->plat_data->get_output_bus_format(data); + else + hdmi->hdmi_data.enc_out_bus_format = + MEDIA_BUS_FMT_RGB888_1X24; + + if (hdmi->plat_data->get_input_bus_format) + hdmi->hdmi_data.enc_in_bus_format = + hdmi->plat_data->get_input_bus_format(data); + else if (hdmi->plat_data->input_bus_format) + hdmi->hdmi_data.enc_in_bus_format = + hdmi->plat_data->input_bus_format; + else + hdmi->hdmi_data.enc_in_bus_format = + MEDIA_BUS_FMT_RGB888_1X24; + } else { + hdmi->hdmi_data.enc_out_bus_format = + bridge_state->output_bus_cfg.format; - hdmi->hdmi_data.enc_in_bus_format = - bridge_state->input_bus_cfg.format; + hdmi->hdmi_data.enc_in_bus_format = + bridge_state->input_bus_cfg.format; - dev_dbg(hdmi->dev, "input format 0x%04x, output format 0x%04x\n", - bridge_state->input_bus_cfg.format, - bridge_state->output_bus_cfg.format); + dev_dbg(hdmi->dev, "input format 0x%04x, output format 0x%04x\n", + bridge_state->input_bus_cfg.format, + bridge_state->output_bus_cfg.format); + } return 0; } @@ -2798,10 +3604,22 @@ static int dw_hdmi_bridge_attach(struct drm_bridge *bridge, enum drm_bridge_attach_flags flags) { struct dw_hdmi *hdmi = bridge->driver_private; + int ret; if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) return 0; + if (hdmi->next_bridge) { + hdmi->next_bridge->encoder = bridge->encoder; + ret = drm_bridge_attach(bridge->encoder, hdmi->next_bridge, bridge, flags); + if (ret) { + DRM_ERROR("Failed to attach bridge with dw-hdmi\n"); + return ret; + } + + return 0; + } + return dw_hdmi_connector_create(hdmi); } @@ -2821,17 +3639,16 @@ dw_hdmi_bridge_mode_valid(struct drm_bridge *bridge, const struct drm_display_mode *mode) { struct dw_hdmi *hdmi = bridge->driver_private; + struct drm_connector *connector = &hdmi->connector; const struct dw_hdmi_plat_data *pdata = hdmi->plat_data; enum drm_mode_status mode_status = MODE_OK; - /* We don't support double-clocked modes */ - if (mode->flags & DRM_MODE_FLAG_DBLCLK) - return MODE_BAD; + if (hdmi->next_bridge) + return MODE_OK; if (pdata->mode_valid) - mode_status = pdata->mode_valid(hdmi, pdata->priv_data, info, - mode); - + mode_status = pdata->mode_valid(connector, pdata->priv_data, + info, mode); return mode_status; } @@ -2912,6 +3729,12 @@ static const struct drm_bridge_funcs dw_hdmi_bridge_funcs = { .get_edid = dw_hdmi_bridge_get_edid, }; +void dw_hdmi_set_cec_adap(struct dw_hdmi *hdmi, struct cec_adapter *adap) +{ + hdmi->cec_adap = adap; +} +EXPORT_SYMBOL_GPL(dw_hdmi_set_cec_adap); + /* ----------------------------------------------------------------------------- * IRQ Handling */ @@ -2937,7 +3760,7 @@ static irqreturn_t dw_hdmi_i2c_irq(struct dw_hdmi *hdmi) static irqreturn_t dw_hdmi_hardirq(int irq, void *dev_id) { struct dw_hdmi *hdmi = dev_id; - u8 intr_stat; + u8 intr_stat, hdcp_stat; irqreturn_t ret = IRQ_NONE; if (hdmi->i2c) @@ -2949,6 +3772,13 @@ static irqreturn_t dw_hdmi_hardirq(int irq, void *dev_id) return IRQ_WAKE_THREAD; } + hdcp_stat = hdmi_readb(hdmi, HDMI_A_APIINTSTAT); + if (hdcp_stat) { + dev_dbg(hdmi->dev, "HDCP irq %#x\n", hdcp_stat); + hdmi_writeb(hdmi, 0xff, HDMI_A_APIINTMSK); + return IRQ_WAKE_THREAD; + } + return ret; } @@ -2956,7 +3786,7 @@ void dw_hdmi_setup_rx_sense(struct dw_hdmi *hdmi, bool hpd, bool rx_sense) { mutex_lock(&hdmi->mutex); - if (!hdmi->force) { + if (!hdmi->force && !hdmi->force_logo) { /* * If the RX sense status indicates we're disconnected, * clear the software rxsense status. @@ -2983,7 +3813,7 @@ EXPORT_SYMBOL_GPL(dw_hdmi_setup_rx_sense); static irqreturn_t dw_hdmi_irq(int irq, void *dev_id) { struct dw_hdmi *hdmi = dev_id; - u8 intr_stat, phy_int_pol, phy_pol_mask, phy_stat; + u8 intr_stat, phy_int_pol, phy_pol_mask, phy_stat, hdcp_stat; enum drm_connector_status status = connector_status_unknown; intr_stat = hdmi_readb(hdmi, HDMI_IH_PHY_STAT0); @@ -3030,22 +3860,21 @@ static irqreturn_t dw_hdmi_irq(int irq, void *dev_id) if (!(phy_stat & (HDMI_PHY_HPD | HDMI_PHY_RX_SENSE))) status = connector_status_disconnected; } - - if (status != connector_status_unknown) { - dev_dbg(hdmi->dev, "EVENT=%s\n", - status == connector_status_connected ? - "plugin" : "plugout"); - - if (hdmi->bridge.dev) { - drm_helper_hpd_irq_event(hdmi->bridge.dev); - drm_bridge_hpd_notify(&hdmi->bridge, status); - } - } + check_hdmi_irq(hdmi, intr_stat, phy_int_pol); hdmi_writeb(hdmi, intr_stat, HDMI_IH_PHY_STAT0); - hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE), - HDMI_IH_MUTE_PHY_STAT0); - + if (!hdmi->next_bridge) + hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | + HDMI_IH_PHY_STAT0_RX_SENSE), + HDMI_IH_MUTE_PHY_STAT0); + + hdcp_stat = hdmi_readb(hdmi, HDMI_A_APIINTSTAT); + if (hdcp_stat) { + if (hdmi->hdcp) + hdmi->hdcp->hdcp_isr(hdmi->hdcp, hdcp_stat); + hdmi_writeb(hdmi, hdcp_stat, HDMI_A_APIINTCLR); + hdmi_writeb(hdmi, 0x00, HDMI_A_APIINTMSK); + } return IRQ_HANDLED; } @@ -3179,12 +4008,363 @@ static void dw_hdmi_init_hw(struct dw_hdmi *hdmi) * Even if we are using a separate i2c adapter doing this doesn't * hurt. */ - dw_hdmi_i2c_init(hdmi); + if (hdmi->i2c) + dw_hdmi_i2c_init(hdmi); if (hdmi->phy.ops->setup_hpd) hdmi->phy.ops->setup_hpd(hdmi, hdmi->phy.data); } +static int dw_hdmi_status_show(struct seq_file *s, void *v) +{ + struct dw_hdmi *hdmi = s->private; + u32 val; + + seq_puts(s, "PHY: "); + if (!hdmi->phy.enabled) { + seq_puts(s, "disabled\n"); + return 0; + } + seq_puts(s, "enabled\t\t\tMode: "); + if (hdmi->sink_is_hdmi) + seq_puts(s, "HDMI\n"); + else + seq_puts(s, "DVI\n"); + if (hdmi->hdmi_data.video_mode.mtmdsclock > 340000000) + val = hdmi->hdmi_data.video_mode.mtmdsclock / 4; + else + val = hdmi->hdmi_data.video_mode.mtmdsclock; + seq_printf(s, "Pixel Clk: %uHz\t\tTMDS Clk: %uHz\n", + hdmi->hdmi_data.video_mode.mpixelclock, val); + seq_puts(s, "Color Format: "); + if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) + seq_puts(s, "RGB"); + else if (hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format)) + seq_puts(s, "YUV444"); + else if (hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) + seq_puts(s, "YUV422"); + else if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) + seq_puts(s, "YUV420"); + else + seq_puts(s, "UNKNOWN"); + val = hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format); + seq_printf(s, "\t\tColor Depth: %d bit\n", val); + seq_puts(s, "Colorimetry: "); + switch (hdmi->hdmi_data.enc_out_encoding) { + case V4L2_YCBCR_ENC_601: + seq_puts(s, "ITU.BT601"); + break; + case V4L2_YCBCR_ENC_709: + seq_puts(s, "ITU.BT709"); + break; + case V4L2_YCBCR_ENC_BT2020: + seq_puts(s, "ITU.BT2020"); + break; + default: /* Carries no data */ + seq_puts(s, "ITU.BT601"); + break; + } + + seq_puts(s, "\t\tEOTF: "); + + if (hdmi->version < 0x211a) { + seq_puts(s, "Unsupported\n"); + return 0; + } + + val = hdmi_readb(hdmi, HDMI_FC_PACKET_TX_EN); + if (!(val & HDMI_FC_PACKET_TX_EN_DRM_MASK)) { + seq_puts(s, "Off\n"); + return 0; + } + + switch (hdmi_readb(hdmi, HDMI_FC_DRM_PB0)) { + case HDMI_EOTF_TRADITIONAL_GAMMA_SDR: + seq_puts(s, "SDR"); + break; + case HDMI_EOTF_TRADITIONAL_GAMMA_HDR: + seq_puts(s, "HDR"); + break; + case HDMI_EOTF_SMPTE_ST2084: + seq_puts(s, "ST2084"); + break; + case HDMI_EOTF_BT_2100_HLG: + seq_puts(s, "HLG"); + break; + default: + seq_puts(s, "Not Defined\n"); + return 0; + } + + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB3) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB2); + seq_printf(s, "\nx0: %d", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB5) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB4); + seq_printf(s, "\t\t\t\ty0: %d\n", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB7) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB6); + seq_printf(s, "x1: %d", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB9) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB8); + seq_printf(s, "\t\t\t\ty1: %d\n", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB11) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB10); + seq_printf(s, "x2: %d", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB13) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB12); + seq_printf(s, "\t\t\t\ty2: %d\n", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB15) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB14); + seq_printf(s, "white x: %d", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB17) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB16); + seq_printf(s, "\t\t\twhite y: %d\n", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB19) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB18); + seq_printf(s, "max lum: %d", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB21) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB20); + seq_printf(s, "\t\t\tmin lum: %d\n", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB23) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB22); + seq_printf(s, "max cll: %d", val); + val = hdmi_readb(hdmi, HDMI_FC_DRM_PB25) << 8; + val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB24); + seq_printf(s, "\t\t\tmax fall: %d\n", val); + return 0; +} + +static int dw_hdmi_status_open(struct inode *inode, struct file *file) +{ + return single_open(file, dw_hdmi_status_show, inode->i_private); +} + +static const struct file_operations dw_hdmi_status_fops = { + .owner = THIS_MODULE, + .open = dw_hdmi_status_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +#include +#include +#include + +struct dw_hdmi_reg_table { + int reg_base; + int reg_end; +}; + +static const struct dw_hdmi_reg_table hdmi_reg_table[] = { + {HDMI_DESIGN_ID, HDMI_CONFIG3_ID}, + {HDMI_IH_FC_STAT0, HDMI_IH_MUTE}, + {HDMI_TX_INVID0, HDMI_TX_BCBDATA1}, + {HDMI_VP_STATUS, HDMI_VP_POL}, + {HDMI_FC_INVIDCONF, HDMI_FC_DBGTMDS2}, + {HDMI_PHY_CONF0, HDMI_PHY_POL0}, + {HDMI_PHY_I2CM_SLAVE_ADDR, HDMI_PHY_I2CM_FS_SCL_LCNT_0_ADDR}, + {HDMI_AUD_CONF0, 0x3624}, + {HDMI_MC_SFRDIV, HDMI_MC_HEACPHY_RST}, + {HDMI_CSC_CFG, HDMI_CSC_COEF_C4_LSB}, + {HDMI_A_HDCPCFG0, 0x52bb}, + {0x7800, 0x7818}, + {0x7900, 0x790e}, + {HDMI_CEC_CTRL, HDMI_CEC_WKUPCTRL}, + {HDMI_I2CM_SLAVE, 0x7e31}, +}; + +static int dw_hdmi_ctrl_show(struct seq_file *s, void *v) +{ + struct dw_hdmi *hdmi = s->private; + u32 i = 0, j = 0, val = 0; + + seq_puts(s, "\n>>>hdmi_ctl reg "); + for (i = 0; i < 16; i++) + seq_printf(s, " %2x", i); + seq_puts(s, "\n---------------------------------------------------"); + + for (i = 0; i < ARRAY_SIZE(hdmi_reg_table); i++) { + for (j = hdmi_reg_table[i].reg_base; + j <= hdmi_reg_table[i].reg_end; j++) { + val = hdmi_readb(hdmi, j); + if ((j - hdmi_reg_table[i].reg_base) % 16 == 0) + seq_printf(s, "\n>>>hdmi_ctl %04x:", j); + seq_printf(s, " %02x", val); + } + } + seq_puts(s, "\n---------------------------------------------------\n"); + + return 0; +} + +static int dw_hdmi_ctrl_open(struct inode *inode, struct file *file) +{ + return single_open(file, dw_hdmi_ctrl_show, inode->i_private); +} + +static ssize_t +dw_hdmi_ctrl_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct dw_hdmi *hdmi = + ((struct seq_file *)file->private_data)->private; + u32 reg, val; + char kbuf[25]; + + if (copy_from_user(kbuf, buf, count)) + return -EFAULT; + if (sscanf(kbuf, "%x%x", ®, &val) == -1) + return -EFAULT; + if (reg > HDMI_I2CM_FS_SCL_LCNT_0_ADDR) { + dev_err(hdmi->dev, "it is no a hdmi register\n"); + return count; + } + dev_info(hdmi->dev, "/**********hdmi register config******/"); + dev_info(hdmi->dev, "\n reg=%x val=%x\n", reg, val); + hdmi_writeb(hdmi, val, reg); + return count; +} + +static const struct file_operations dw_hdmi_ctrl_fops = { + .owner = THIS_MODULE, + .open = dw_hdmi_ctrl_open, + .read = seq_read, + .write = dw_hdmi_ctrl_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static int dw_hdmi_phy_show(struct seq_file *s, void *v) +{ + struct dw_hdmi *hdmi = s->private; + u32 i; + + seq_puts(s, "\n>>>hdmi_phy reg "); + for (i = 0; i < 0x28; i++) + seq_printf(s, "regs %02x val %04x\n", + i, hdmi_phy_i2c_read(hdmi, i)); + return 0; +} + +static int dw_hdmi_phy_open(struct inode *inode, struct file *file) +{ + return single_open(file, dw_hdmi_phy_show, inode->i_private); +} + +static ssize_t +dw_hdmi_phy_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct dw_hdmi *hdmi = + ((struct seq_file *)file->private_data)->private; + u32 reg, val; + char kbuf[25]; + + if (copy_from_user(kbuf, buf, count)) + return -EFAULT; + if (sscanf(kbuf, "%x%x", ®, &val) == -1) + return -EFAULT; + if (reg > 0x28) { + dev_err(hdmi->dev, "it is not a hdmi phy register\n"); + return count; + } + dev_info(hdmi->dev, "/*******hdmi phy register config******/"); + dev_info(hdmi->dev, "\n reg=%x val=%x\n", reg, val); + dw_hdmi_phy_i2c_write(hdmi, val, reg); + return count; +} + +static const struct file_operations dw_hdmi_phy_fops = { + .owner = THIS_MODULE, + .open = dw_hdmi_phy_open, + .read = seq_read, + .write = dw_hdmi_phy_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static void dw_hdmi_register_debugfs(struct device *dev, struct dw_hdmi *hdmi) +{ + hdmi->debugfs_dir = debugfs_create_dir("dw-hdmi", NULL); + if (IS_ERR(hdmi->debugfs_dir)) { + dev_err(dev, "failed to create debugfs dir!\n"); + return; + } + debugfs_create_file("status", 0400, hdmi->debugfs_dir, + hdmi, &dw_hdmi_status_fops); + debugfs_create_file("ctrl", 0400, hdmi->debugfs_dir, + hdmi, &dw_hdmi_ctrl_fops); + debugfs_create_file("phy", 0400, hdmi->debugfs_dir, + hdmi, &dw_hdmi_phy_fops); +} + +static void dw_hdmi_register_hdcp(struct device *dev, struct dw_hdmi *hdmi, + u32 val, bool hdcp1x_enable) +{ + struct dw_hdcp hdmi_hdcp = { + .hdmi = hdmi, + .write = hdmi_writeb, + .read = hdmi_readb, + .regs = hdmi->regs, + .reg_io_width = val, + .enable = hdcp1x_enable, + }; + struct platform_device_info hdcp_device_info = { + .parent = dev, + .id = PLATFORM_DEVID_AUTO, + .res = NULL, + .num_res = 0, + .name = DW_HDCP_DRIVER_NAME, + .data = &hdmi_hdcp, + .size_data = sizeof(hdmi_hdcp), + .dma_mask = DMA_BIT_MASK(32), + }; + + hdmi->hdcp_dev = platform_device_register_full(&hdcp_device_info); + if (IS_ERR(hdmi->hdcp_dev)) + dev_err(dev, "failed to register hdcp!\n"); + else + hdmi->hdcp = hdmi->hdcp_dev->dev.platform_data; +} + +static int get_force_logo_property(struct dw_hdmi *hdmi) +{ + struct device_node *dss; + struct device_node *route; + struct device_node *route_hdmi; + + dss = of_find_node_by_name(NULL, "display-subsystem"); + if (!dss) { + dev_err(hdmi->dev, "can't find display-subsystem\n"); + return -ENODEV; + } + + route = of_find_node_by_name(dss, "route"); + if (!route) { + dev_err(hdmi->dev, "can't find route\n"); + of_node_put(dss); + return -ENODEV; + } + of_node_put(dss); + + route_hdmi = of_find_node_by_name(route, "route-hdmi"); + if (!route_hdmi) { + dev_err(hdmi->dev, "can't find route-hdmi\n"); + of_node_put(route); + return -ENODEV; + } + of_node_put(route); + + hdmi->force_logo = + of_property_read_bool(route_hdmi, "force-output"); + + of_node_put(route_hdmi); + + return 0; +} + /* ----------------------------------------------------------------------------- * Probe/remove API, used from platforms based on the DRM bridge API. */ @@ -3193,6 +4373,7 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; + struct device_node *endpoint; struct platform_device_info pdevinfo; struct device_node *ddc_node; struct dw_hdmi_cec_data cec; @@ -3205,11 +4386,13 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, u8 prod_id1; u8 config0; u8 config3; + bool hdcp1x_enable = 0; hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL); if (!hdmi) return ERR_PTR(-ENOMEM); + hdmi->connector.stereo_allowed = 1; hdmi->plat_data = plat_data; hdmi->dev = dev; hdmi->sample_rate = 48000; @@ -3340,7 +4523,24 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, prod_id1 & HDMI_PRODUCT_ID1_HDCP ? "with" : "without", hdmi->phy.name); - dw_hdmi_init_hw(hdmi); + ret = get_force_logo_property(hdmi); + if (ret) + goto err_iahb; + + hdmi->initialized = false; + ret = hdmi_readb(hdmi, HDMI_PHY_STAT0); + if (((ret & HDMI_PHY_TX_PHY_LOCK) && (ret & HDMI_PHY_HPD) && + hdmi_readb(hdmi, HDMI_FC_EXCTRLDUR)) || hdmi->force_logo) { + hdmi->mc_clkdis = hdmi_readb(hdmi, HDMI_MC_CLKDIS); + hdmi->disabled = false; + hdmi->bridge_is_on = true; + hdmi->phy.enabled = true; + hdmi->initialized = true; + } else if (ret & HDMI_PHY_TX_PHY_LOCK) { + hdmi->phy.ops->disable(hdmi, hdmi->phy.data); + } + + init_hpd_work(hdmi); irq = platform_get_irq(pdev, 0); if (irq < 0) { @@ -3348,6 +4548,7 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, goto err_iahb; } + hdmi->irq = irq; ret = devm_request_threaded_irq(dev, irq, dw_hdmi_hardirq, dw_hdmi_irq, IRQF_SHARED, dev_name(dev), hdmi); @@ -3383,8 +4584,20 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, hdmi->ddc = dw_hdmi_i2c_adapter(hdmi); if (IS_ERR(hdmi->ddc)) hdmi->ddc = NULL; + /* + * Read high and low time from device tree. If not available use + * the default timing scl clock rate is about 99.6KHz. + */ + if (of_property_read_u32(np, "ddc-i2c-scl-high-time-ns", + &hdmi->i2c->scl_high_ns)) + hdmi->i2c->scl_high_ns = 4708; + if (of_property_read_u32(np, "ddc-i2c-scl-low-time-ns", + &hdmi->i2c->scl_low_ns)) + hdmi->i2c->scl_low_ns = 4916; } + dw_hdmi_init_hw(hdmi); + hdmi->bridge.driver_private = hdmi; hdmi->bridge.funcs = &dw_hdmi_bridge_funcs; hdmi->bridge.ops = DRM_BRIDGE_OP_DETECT | DRM_BRIDGE_OP_EDID @@ -3393,6 +4606,30 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, hdmi->bridge.of_node = pdev->dev.of_node; #endif + endpoint = of_graph_get_endpoint_by_regs(hdmi->dev->of_node, 1, -1); + if (endpoint && of_device_is_available(endpoint)) { + struct device_node *remote; + + remote = of_graph_get_remote_port_parent(endpoint); + of_node_put(endpoint); + if (!remote || !of_device_is_available(remote)) { + of_node_put(remote); + ret = -ENODEV; + goto err_iahb; + } + + hdmi->next_bridge = of_drm_find_bridge(remote); + of_node_put(remote); + if (!hdmi->next_bridge) { + dev_err(hdmi->dev, "can't find next bridge\n"); + ret = -EPROBE_DEFER; + goto err_iahb; + } + + hdmi->sink_is_hdmi = true; + hdmi->sink_has_audio = true; + } + memset(&pdevinfo, 0, sizeof(pdevinfo)); pdevinfo.parent = dev; pdevinfo.id = PLATFORM_DEVID_AUTO; @@ -3446,8 +4683,40 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, hdmi->cec = platform_device_register_full(&pdevinfo); } + hdmi->extcon = devm_extcon_dev_allocate(hdmi->dev, dw_hdmi_cable); + if (IS_ERR(hdmi->extcon)) { + ret = PTR_ERR(hdmi->extcon); + dev_err(hdmi->dev, "allocate extcon failed: %d\n", ret); + goto err_iahb; + } + + ret = devm_extcon_dev_register(hdmi->dev, hdmi->extcon); + if (ret) { + dev_err(hdmi->dev, "failed to register extcon: %d\n", + ret); + goto err_iahb; + } + + ret = extcon_set_property_capability(hdmi->extcon, EXTCON_DISP_HDMI, + EXTCON_PROP_DISP_HPD); + if (ret) { + dev_err(hdmi->dev, + "failed to set USB property capability: %d\n", + ret); + goto err_iahb; + } + drm_bridge_add(&hdmi->bridge); + dw_hdmi_register_debugfs(dev, hdmi); + + if (of_property_read_bool(np, "scramble-low-rates")) + hdmi->scramble_low_rates = true; + + if (of_property_read_bool(np, "hdcp1x-enable")) + hdcp1x_enable = 1; + dw_hdmi_register_hdcp(dev, hdmi, val, hdcp1x_enable); + return hdmi; err_iahb: @@ -3457,7 +4726,10 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, err_isfr: clk_disable_unprepare(hdmi->isfr_clk); err_res: - i2c_put_adapter(hdmi->ddc); + if (hdmi->i2c) + i2c_del_adapter(&hdmi->i2c->adap); + else + i2c_put_adapter(hdmi->ddc); return ERR_PTR(ret); } @@ -3465,16 +4737,35 @@ EXPORT_SYMBOL_GPL(dw_hdmi_probe); void dw_hdmi_remove(struct dw_hdmi *hdmi) { + if (hdmi->irq) + disable_irq(hdmi->irq); + + cancel_delayed_work(&hdmi->work); + flush_workqueue(hdmi->workqueue); + destroy_workqueue(hdmi->workqueue); + + debugfs_remove_recursive(hdmi->debugfs_dir); + drm_bridge_remove(&hdmi->bridge); if (hdmi->audio && !IS_ERR(hdmi->audio)) platform_device_unregister(hdmi->audio); + if (hdmi->hdcp_dev && !IS_ERR(hdmi->hdcp_dev)) + platform_device_unregister(hdmi->hdcp_dev); if (!IS_ERR(hdmi->cec)) platform_device_unregister(hdmi->cec); /* Disable all interrupts */ hdmi_writeb(hdmi, ~0, HDMI_IH_MUTE_PHY_STAT0); + if (!hdmi->next_bridge) { + dw_hdmi_destroy_properties(hdmi); + hdmi->connector.funcs->destroy(&hdmi->connector); + } + + if (hdmi->bridge.encoder) + hdmi->bridge.encoder->funcs->destroy(hdmi->bridge.encoder); + clk_disable_unprepare(hdmi->iahb_clk); clk_disable_unprepare(hdmi->isfr_clk); if (hdmi->cec_clk) @@ -3492,7 +4783,7 @@ EXPORT_SYMBOL_GPL(dw_hdmi_remove); */ struct dw_hdmi *dw_hdmi_bind(struct platform_device *pdev, struct drm_encoder *encoder, - const struct dw_hdmi_plat_data *plat_data) + struct dw_hdmi_plat_data *plat_data) { struct dw_hdmi *hdmi; int ret; @@ -3508,6 +4799,9 @@ struct dw_hdmi *dw_hdmi_bind(struct platform_device *pdev, return ERR_PTR(ret); } + if (!hdmi->next_bridge) + plat_data->connector = &hdmi->connector; + return hdmi; } EXPORT_SYMBOL_GPL(dw_hdmi_bind); @@ -3518,9 +4812,87 @@ void dw_hdmi_unbind(struct dw_hdmi *hdmi) } EXPORT_SYMBOL_GPL(dw_hdmi_unbind); +static void dw_hdmi_reg_initial(struct dw_hdmi *hdmi) +{ + if (hdmi_readb(hdmi, HDMI_IH_MUTE)) { + initialize_hdmi_ih_mutes(hdmi); + /* unmute cec irq */ + hdmi_writeb(hdmi, 0x68, HDMI_IH_MUTE_CEC_STAT0); + + hdmi_writeb(hdmi, HDMI_PHY_I2CM_INT_ADDR_DONE_POL, + HDMI_PHY_I2CM_INT_ADDR); + + hdmi_writeb(hdmi, HDMI_PHY_I2CM_CTLINT_ADDR_NAC_POL | + HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_POL, + HDMI_PHY_I2CM_CTLINT_ADDR); + + if (!hdmi->next_bridge) { + hdmi_writeb(hdmi, HDMI_PHY_HPD | HDMI_PHY_RX_SENSE, + HDMI_PHY_POL0); + hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0); + hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | + HDMI_IH_PHY_STAT0_RX_SENSE), + HDMI_IH_MUTE_PHY_STAT0); + } + } +} + +void dw_hdmi_suspend(struct dw_hdmi *hdmi) +{ + if (!hdmi) + return; + + mutex_lock(&hdmi->mutex); + + /* + * When system shutdown, hdmi should be disabled. + * When system suspend, dw_hdmi_bridge_disable will disable hdmi first. + * To prevent duplicate operation, we should determine whether hdmi + * has been disabled. + */ + if (!hdmi->disabled) { + hdmi->disabled = true; + dw_hdmi_update_power(hdmi); + dw_hdmi_update_phy_mask(hdmi); + } + mutex_unlock(&hdmi->mutex); + + if (hdmi->irq) + disable_irq(hdmi->irq); + cancel_delayed_work(&hdmi->work); + flush_workqueue(hdmi->workqueue); + pinctrl_pm_select_sleep_state(hdmi->dev); +} +EXPORT_SYMBOL_GPL(dw_hdmi_suspend); + void dw_hdmi_resume(struct dw_hdmi *hdmi) { - dw_hdmi_init_hw(hdmi); + if (!hdmi) + return; + + pinctrl_pm_select_default_state(hdmi->dev); + mutex_lock(&hdmi->mutex); + dw_hdmi_reg_initial(hdmi); + if (hdmi->i2c) + dw_hdmi_i2c_init(hdmi); + if (hdmi->irq) + enable_irq(hdmi->irq); + /* + * HDMI status maybe incorrect in the following condition: + * HDMI plug in -> system sleep -> HDMI plug out -> system wake up. + * At this time, cat /sys/class/drm/card 0-HDMI-A-1/status is connected. + * There is no hpd interrupt, because HDMI is powerdown during suspend. + * So we need check the current HDMI status in this case. + */ + if (hdmi->connector.status == connector_status_connected) { + if (hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data) == + connector_status_disconnected) { + hdmi->hpd_state = false; + mod_delayed_work(hdmi->workqueue, &hdmi->work, + msecs_to_jiffies(20)); + } + } + mutex_unlock(&hdmi->mutex); } EXPORT_SYMBOL_GPL(dw_hdmi_resume); diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h index 1999db05bc3b..50973280048c 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h @@ -509,6 +509,51 @@ #define HDMI_A_PRESETUP 0x501A #define HDMI_A_SRM_BASE 0x5020 +/* CEC Engine Registers */ +#define HDMI_CEC_CTRL 0x7D00 +#define HDMI_CEC_STAT 0x7D01 +#define HDMI_CEC_MASK 0x7D02 +#define HDMI_CEC_POLARITY 0x7D03 +#define HDMI_CEC_INT 0x7D04 +#define HDMI_CEC_ADDR_L 0x7D05 +#define HDMI_CEC_ADDR_H 0x7D06 +#define HDMI_CEC_TX_CNT 0x7D07 +#define HDMI_CEC_RX_CNT 0x7D08 +#define HDMI_CEC_TX_DATA0 0x7D10 +#define HDMI_CEC_TX_DATA1 0x7D11 +#define HDMI_CEC_TX_DATA2 0x7D12 +#define HDMI_CEC_TX_DATA3 0x7D13 +#define HDMI_CEC_TX_DATA4 0x7D14 +#define HDMI_CEC_TX_DATA5 0x7D15 +#define HDMI_CEC_TX_DATA6 0x7D16 +#define HDMI_CEC_TX_DATA7 0x7D17 +#define HDMI_CEC_TX_DATA8 0x7D18 +#define HDMI_CEC_TX_DATA9 0x7D19 +#define HDMI_CEC_TX_DATA10 0x7D1a +#define HDMI_CEC_TX_DATA11 0x7D1b +#define HDMI_CEC_TX_DATA12 0x7D1c +#define HDMI_CEC_TX_DATA13 0x7D1d +#define HDMI_CEC_TX_DATA14 0x7D1e +#define HDMI_CEC_TX_DATA15 0x7D1f +#define HDMI_CEC_RX_DATA0 0x7D20 +#define HDMI_CEC_RX_DATA1 0x7D21 +#define HDMI_CEC_RX_DATA2 0x7D22 +#define HDMI_CEC_RX_DATA3 0x7D23 +#define HDMI_CEC_RX_DATA4 0x7D24 +#define HDMI_CEC_RX_DATA5 0x7D25 +#define HDMI_CEC_RX_DATA6 0x7D26 +#define HDMI_CEC_RX_DATA7 0x7D27 +#define HDMI_CEC_RX_DATA8 0x7D28 +#define HDMI_CEC_RX_DATA9 0x7D29 +#define HDMI_CEC_RX_DATA10 0x7D2a +#define HDMI_CEC_RX_DATA11 0x7D2b +#define HDMI_CEC_RX_DATA12 0x7D2c +#define HDMI_CEC_RX_DATA13 0x7D2d +#define HDMI_CEC_RX_DATA14 0x7D2e +#define HDMI_CEC_RX_DATA15 0x7D2f +#define HDMI_CEC_LOCK 0x7D30 +#define HDMI_CEC_WKUPCTRL 0x7D31 + /* I2C Master Registers (E-DDC) */ #define HDMI_I2CM_SLAVE 0x7E00 #define HDMI_I2CM_ADDRESS 0x7E01 @@ -529,6 +574,7 @@ #define HDMI_I2CM_FS_SCL_HCNT_0_ADDR 0x7E10 #define HDMI_I2CM_FS_SCL_LCNT_1_ADDR 0x7E11 #define HDMI_I2CM_FS_SCL_LCNT_0_ADDR 0x7E12 +#define HDMI_I2CM_SDA_HOLD 0x7E13 enum { /* PRODUCT_ID0 field values */ @@ -842,6 +888,10 @@ enum { HDMI_FC_AVICONF3_QUANT_RANGE_LIMITED = 0x00, HDMI_FC_AVICONF3_QUANT_RANGE_FULL = 0x04, +/* HDMI_FC_GCP */ + HDMI_FC_GCP_SET_AVMUTE = 0x2, + HDMI_FC_GCP_CLEAR_AVMUTE = 0x1, + /* FC_DBGFORCE field values */ HDMI_FC_DBGFORCE_FORCEAUDIO = 0x10, HDMI_FC_DBGFORCE_FORCEVIDEO = 0x1, @@ -1085,6 +1135,11 @@ enum { HDMI_I2CM_CTLINT_NAC_MASK = 0x40, HDMI_I2CM_CTLINT_ARB_POL = 0x8, HDMI_I2CM_CTLINT_ARB_MASK = 0x4, + +/* I2CM_DIV field values */ + HDMI_I2CM_DIV_FAST_STD_MODE = 0x8, + HDMI_I2CM_DIV_FAST_MODE = 0x8, + HDMI_I2CM_DIV_STD_MODE = 0, }; /* diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c index 376fa6eb46f6..163dcc03ba22 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c @@ -244,7 +244,7 @@ struct dw_mipi_dsi { struct device *dev; void __iomem *base; - struct clk *pclk; + struct reset_control *apb_rst; unsigned int lane_mbps; /* per lane */ u32 channel; @@ -316,15 +316,10 @@ static int dw_mipi_dsi_host_attach(struct mipi_dsi_host *host, const struct dw_mipi_dsi_plat_data *pdata = dsi->plat_data; struct drm_bridge *bridge; struct drm_panel *panel; + int max_data_lanes = dsi->plat_data->max_data_lanes; int ret; - if (device->lanes > dsi->plat_data->max_data_lanes) { - dev_err(dsi->dev, "the number of data lanes(%u) is too many\n", - device->lanes); - return -EINVAL; - } - - dsi->lanes = device->lanes; + dsi->lanes = (device->lanes > max_data_lanes) ? device->lanes / 2 : device->lanes; dsi->channel = device->channel; dsi->format = device->format; dsi->mode_flags = device->mode_flags; @@ -599,8 +594,14 @@ static void dw_mipi_dsi_set_mode(struct dw_mipi_dsi *dsi, static void dw_mipi_dsi_disable(struct dw_mipi_dsi *dsi) { + const struct dw_mipi_dsi_phy_ops *phy_ops = dsi->plat_data->phy_ops; + + if (phy_ops->power_off) + phy_ops->power_off(dsi->plat_data->priv_data); + dsi_write(dsi, DSI_PWR_UP, RESET); dsi_write(dsi, DSI_PHY_RSTZ, PHY_RSTZ); + pm_runtime_put(dsi->dev); } static void dw_mipi_dsi_init(struct dw_mipi_dsi *dsi) @@ -715,16 +716,16 @@ static u32 dw_mipi_dsi_get_hcomponent_lbcc(struct dw_mipi_dsi *dsi, const struct drm_display_mode *mode, u32 hcomponent) { - u32 frac, lbcc; + u32 lbcc; lbcc = hcomponent * dsi->lane_mbps * MSEC_PER_SEC / 8; - frac = lbcc % mode->clock; - lbcc = lbcc / mode->clock; - if (frac) - lbcc++; + if (mode->clock == 0) { + DRM_ERROR("dsi mode clock is 0!\n"); + return 0; + } - return lbcc; + return DIV_ROUND_CLOSEST_ULL(lbcc, mode->clock); } static void dw_mipi_dsi_line_timer_config(struct dw_mipi_dsi *dsi, @@ -837,13 +838,13 @@ static void dw_mipi_dsi_dphy_enable(struct dw_mipi_dsi *dsi) ret = readl_poll_timeout(dsi->base + DSI_PHY_STATUS, val, val & PHY_LOCK, 1000, PHY_STATUS_TIMEOUT_US); if (ret) - DRM_DEBUG_DRIVER("failed to wait phy lock state\n"); + DRM_ERROR("failed to wait phy lock state\n"); ret = readl_poll_timeout(dsi->base + DSI_PHY_STATUS, val, val & PHY_STOP_STATE_CLK_LANE, 1000, PHY_STATUS_TIMEOUT_US); if (ret) - DRM_DEBUG_DRIVER("failed to wait phy clk lane stop state\n"); + DRM_ERROR("failed to wait phy clk lane stop state\n"); } static void dw_mipi_dsi_clear_err(struct dw_mipi_dsi *dsi) @@ -857,7 +858,6 @@ static void dw_mipi_dsi_clear_err(struct dw_mipi_dsi *dsi) static void dw_mipi_dsi_bridge_post_disable(struct drm_bridge *bridge) { struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge); - const struct dw_mipi_dsi_phy_ops *phy_ops = dsi->plat_data->phy_ops; /* * Switch to command mode before panel-bridge post_disable & @@ -866,6 +866,8 @@ static void dw_mipi_dsi_bridge_post_disable(struct drm_bridge *bridge) * before by the drm framework. */ dw_mipi_dsi_set_mode(dsi, 0); + if (dsi->slave) + dw_mipi_dsi_set_mode(dsi->slave, 0); /* * TODO Only way found to call panel-bridge post_disable & @@ -876,18 +878,10 @@ static void dw_mipi_dsi_bridge_post_disable(struct drm_bridge *bridge) if (dsi->panel_bridge->funcs->post_disable) dsi->panel_bridge->funcs->post_disable(dsi->panel_bridge); - if (phy_ops->power_off) - phy_ops->power_off(dsi->plat_data->priv_data); - - if (dsi->slave) { + if (dsi->slave) dw_mipi_dsi_disable(dsi->slave); - clk_disable_unprepare(dsi->slave->pclk); - pm_runtime_put(dsi->slave->dev); - } - dw_mipi_dsi_disable(dsi); - clk_disable_unprepare(dsi->pclk); - pm_runtime_put(dsi->dev); + dw_mipi_dsi_disable(dsi); } static unsigned int dw_mipi_dsi_get_lanes(struct dw_mipi_dsi *dsi) @@ -912,7 +906,11 @@ static void dw_mipi_dsi_mode_set(struct dw_mipi_dsi *dsi, int ret; u32 lanes = dw_mipi_dsi_get_lanes(dsi); - clk_prepare_enable(dsi->pclk); + if (dsi->apb_rst) { + reset_control_assert(dsi->apb_rst); + usleep_range(10, 20); + reset_control_deassert(dsi->apb_rst); + } ret = phy_ops->get_lane_mbps(priv_data, adjusted_mode, dsi->mode_flags, lanes, dsi->format, &dsi->lane_mbps); @@ -939,15 +937,15 @@ static void dw_mipi_dsi_mode_set(struct dw_mipi_dsi *dsi, if (ret) DRM_DEBUG_DRIVER("Phy init() failed\n"); + if (phy_ops->power_on) + phy_ops->power_on(dsi->plat_data->priv_data); + dw_mipi_dsi_dphy_enable(dsi); dw_mipi_dsi_wait_for_two_frames(adjusted_mode); /* Switch to cmd mode for panel-bridge pre_enable & panel prepare */ dw_mipi_dsi_set_mode(dsi, 0); - - if (phy_ops->power_on) - phy_ops->power_on(dsi->plat_data->priv_data); } static void dw_mipi_dsi_bridge_mode_set(struct drm_bridge *bridge, @@ -959,16 +957,25 @@ static void dw_mipi_dsi_bridge_mode_set(struct drm_bridge *bridge, dw_mipi_dsi_mode_set(dsi, adjusted_mode); if (dsi->slave) dw_mipi_dsi_mode_set(dsi->slave, adjusted_mode); + + DRM_DEV_INFO(dsi->dev, "final DSI-Link bandwidth: %u x %d Mbps\n", + dsi->lane_mbps, dsi->slave ? dsi->lanes * 2 : dsi->lanes); } static void dw_mipi_dsi_bridge_enable(struct drm_bridge *bridge) { struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge); - /* Switch to video mode for panel-bridge enable & panel enable */ - dw_mipi_dsi_set_mode(dsi, MIPI_DSI_MODE_VIDEO); - if (dsi->slave) - dw_mipi_dsi_set_mode(dsi->slave, MIPI_DSI_MODE_VIDEO); + /* Switch to video/cmd mode for panel-bridge enable & panel enable */ + if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO) { + dw_mipi_dsi_set_mode(dsi, MIPI_DSI_MODE_VIDEO); + if (dsi->slave) + dw_mipi_dsi_set_mode(dsi->slave, MIPI_DSI_MODE_VIDEO); + } else { + dw_mipi_dsi_set_mode(dsi, 0); + if (dsi->slave) + dw_mipi_dsi_set_mode(dsi->slave, 0); + } } static enum drm_mode_status @@ -1103,7 +1110,6 @@ __dw_mipi_dsi_probe(struct platform_device *pdev, const struct dw_mipi_dsi_plat_data *plat_data) { struct device *dev = &pdev->dev; - struct reset_control *apb_rst; struct dw_mipi_dsi *dsi; int ret; @@ -1129,20 +1135,13 @@ __dw_mipi_dsi_probe(struct platform_device *pdev, dsi->base = plat_data->base; } - dsi->pclk = devm_clk_get(dev, "pclk"); - if (IS_ERR(dsi->pclk)) { - ret = PTR_ERR(dsi->pclk); - dev_err(dev, "Unable to get pclk: %d\n", ret); - return ERR_PTR(ret); - } - /* * Note that the reset was not defined in the initial device tree, so * we have to be prepared for it not being found. */ - apb_rst = devm_reset_control_get_optional_exclusive(dev, "apb"); - if (IS_ERR(apb_rst)) { - ret = PTR_ERR(apb_rst); + dsi->apb_rst = devm_reset_control_get_optional_exclusive(dev, "apb"); + if (IS_ERR(dsi->apb_rst)) { + ret = PTR_ERR(dsi->apb_rst); if (ret != -EPROBE_DEFER) dev_err(dev, "Unable to get reset control: %d\n", ret); @@ -1150,20 +1149,6 @@ __dw_mipi_dsi_probe(struct platform_device *pdev, return ERR_PTR(ret); } - if (apb_rst) { - ret = clk_prepare_enable(dsi->pclk); - if (ret) { - dev_err(dev, "%s: Failed to enable pclk\n", __func__); - return ERR_PTR(ret); - } - - reset_control_assert(apb_rst); - usleep_range(10, 20); - reset_control_deassert(apb_rst); - - clk_disable_unprepare(dsi->pclk); - } - dw_mipi_dsi_debugfs_init(dsi); pm_runtime_enable(dev); @@ -1247,6 +1232,12 @@ void dw_mipi_dsi_unbind(struct dw_mipi_dsi *dsi) } EXPORT_SYMBOL_GPL(dw_mipi_dsi_unbind); +struct drm_connector *dw_mipi_dsi_get_connector(struct dw_mipi_dsi *dsi) +{ + return drm_panel_bridge_connector(dsi->panel_bridge); +} +EXPORT_SYMBOL_GPL(dw_mipi_dsi_get_connector); + MODULE_AUTHOR("Chris Zhong "); MODULE_AUTHOR("Philippe Cornu "); MODULE_DESCRIPTION("DW MIPI DSI host controller driver"); diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 7fc8e7000046..4108c7265d53 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -296,12 +296,14 @@ update_connector_routing(struct drm_atomic_state *state, if (old_connector_state->crtc != new_connector_state->crtc) { if (old_connector_state->crtc) { crtc_state = drm_atomic_get_new_crtc_state(state, old_connector_state->crtc); - crtc_state->connectors_changed = true; + if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) + crtc_state->connectors_changed = true; } if (new_connector_state->crtc) { crtc_state = drm_atomic_get_new_crtc_state(state, new_connector_state->crtc); - crtc_state->connectors_changed = true; + if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) + crtc_state->connectors_changed = true; } } @@ -386,7 +388,8 @@ update_connector_routing(struct drm_atomic_state *state, set_best_encoder(state, new_connector_state, new_encoder); - crtc_state->connectors_changed = true; + if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) + crtc_state->connectors_changed = true; DRM_DEBUG_ATOMIC("[CONNECTOR:%d:%s] using [ENCODER:%d:%s] on [CRTC:%d:%s]\n", connector->base.id, @@ -3554,6 +3557,9 @@ int drm_atomic_helper_legacy_gamma_set(struct drm_crtc *crtc, replaced = drm_property_replace_blob(&crtc_state->degamma_lut, NULL); replaced |= drm_property_replace_blob(&crtc_state->ctm, NULL); replaced |= drm_property_replace_blob(&crtc_state->gamma_lut, blob); +#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) + replaced |= drm_property_replace_blob(&crtc_state->cubic_lut, NULL); +#endif crtc_state->color_mgmt_changed |= replaced; ret = drm_atomic_commit(state); diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c index 9ad74045158e..c29183d2aa4e 100644 --- a/drivers/gpu/drm/drm_atomic_state_helper.c +++ b/drivers/gpu/drm/drm_atomic_state_helper.c @@ -141,6 +141,10 @@ void __drm_atomic_helper_crtc_duplicate_state(struct drm_crtc *crtc, drm_property_blob_get(state->ctm); if (state->gamma_lut) drm_property_blob_get(state->gamma_lut); +#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) + if (state->cubic_lut) + drm_property_blob_get(state->cubic_lut); +#endif state->mode_changed = false; state->active_changed = false; state->planes_changed = false; @@ -213,6 +217,9 @@ void __drm_atomic_helper_crtc_destroy_state(struct drm_crtc_state *state) drm_property_blob_put(state->degamma_lut); drm_property_blob_put(state->ctm); drm_property_blob_put(state->gamma_lut); +#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) + drm_property_blob_put(state->cubic_lut); +#endif } EXPORT_SYMBOL(__drm_atomic_helper_crtc_destroy_state); diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index 25c269bc4681..975ece7e00a4 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -459,6 +459,16 @@ static int drm_atomic_crtc_set_property(struct drm_crtc *crtc, &replaced); state->color_mgmt_changed |= replaced; return ret; +#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) + } else if (property == config->cubic_lut_property) { + ret = drm_atomic_replace_property_blob_from_id(dev, + &state->cubic_lut, + val, + -1, sizeof(struct drm_color_lut), + &replaced); + state->color_mgmt_changed |= replaced; + return ret; +#endif } else if (property == config->prop_out_fence_ptr) { s32 __user *fence_ptr = u64_to_user_ptr(val); @@ -501,6 +511,10 @@ drm_atomic_crtc_get_property(struct drm_crtc *crtc, *val = (state->ctm) ? state->ctm->base.id : 0; else if (property == config->gamma_lut_property) *val = (state->gamma_lut) ? state->gamma_lut->base.id : 0; +#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) + else if (property == config->cubic_lut_property) + *val = (state->cubic_lut) ? state->cubic_lut->base.id : 0; +#endif else if (property == config->prop_out_fence_ptr) *val = 0; else if (crtc->funcs->atomic_get_property) diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c index c7adbeaf10b1..232abbba3686 100644 --- a/drivers/gpu/drm/drm_auth.c +++ b/drivers/gpu/drm/drm_auth.c @@ -135,18 +135,16 @@ static void drm_set_master(struct drm_device *dev, struct drm_file *fpriv, static int drm_new_set_master(struct drm_device *dev, struct drm_file *fpriv) { struct drm_master *old_master; - struct drm_master *new_master; lockdep_assert_held_once(&dev->master_mutex); WARN_ON(fpriv->is_master); old_master = fpriv->master; - new_master = drm_master_create(dev); - if (!new_master) + fpriv->master = drm_master_create(dev); + if (!fpriv->master) { + fpriv->master = old_master; return -ENOMEM; - spin_lock(&fpriv->master_lookup_lock); - fpriv->master = new_master; - spin_unlock(&fpriv->master_lookup_lock); + } fpriv->is_master = 1; fpriv->authenticated = 1; @@ -304,13 +302,10 @@ int drm_master_open(struct drm_file *file_priv) /* if there is no current master make this fd it, but do not create * any master object for render clients */ mutex_lock(&dev->master_mutex); - if (!dev->master) { + if (!dev->master) ret = drm_new_set_master(dev, file_priv); - } else { - spin_lock(&file_priv->master_lookup_lock); + else file_priv->master = drm_master_get(dev->master); - spin_unlock(&file_priv->master_lookup_lock); - } mutex_unlock(&dev->master_mutex); return ret; @@ -376,31 +371,6 @@ struct drm_master *drm_master_get(struct drm_master *master) } EXPORT_SYMBOL(drm_master_get); -/** - * drm_file_get_master - reference &drm_file.master of @file_priv - * @file_priv: DRM file private - * - * Increments the reference count of @file_priv's &drm_file.master and returns - * the &drm_file.master. If @file_priv has no &drm_file.master, returns NULL. - * - * Master pointers returned from this function should be unreferenced using - * drm_master_put(). - */ -struct drm_master *drm_file_get_master(struct drm_file *file_priv) -{ - struct drm_master *master = NULL; - - spin_lock(&file_priv->master_lookup_lock); - if (!file_priv->master) - goto unlock; - master = drm_master_get(file_priv->master); - -unlock: - spin_unlock(&file_priv->master_lookup_lock); - return master; -} -EXPORT_SYMBOL(drm_file_get_master); - static void drm_master_destroy(struct kref *kref) { struct drm_master *master = container_of(kref, struct drm_master, refcount); diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c index 138ff34b31db..7b270b68ade4 100644 --- a/drivers/gpu/drm/drm_color_mgmt.c +++ b/drivers/gpu/drm/drm_color_mgmt.c @@ -33,7 +33,7 @@ /** * DOC: overview * - * Color management or color space adjustments is supported through a set of 5 + * Color management or color space adjustments is supported through a set of 7 * properties on the &drm_crtc object. They are set up by calling * drm_crtc_enable_color_mgmt(). * @@ -60,7 +60,7 @@ * “CTMâ€: * Blob property to set the current transformation matrix (CTM) apply to * pixel data after the lookup through the degamma LUT and before the - * lookup through the gamma LUT. The data is interpreted as a struct + * lookup through the cubic LUT. The data is interpreted as a struct * &drm_color_ctm. * * Setting this to NULL (blob property value set to 0) means a @@ -68,13 +68,40 @@ * boot-up state too. Drivers can access the blob for the color conversion * matrix through &drm_crtc_state.ctm. * + * â€CUBIC_LUTâ€: + * Blob property to set the cubic (3D) lookup table performing color + * mapping after the transformation matrix and before the lookup through + * the gamma LUT. Unlike the degamma and gamma LUTs that map color + * components independently, the 3D LUT converts an input color to an + * output color by indexing into the 3D table using the color components + * as a 3D coordinate. The LUT is subsampled as 8-bit (or more) precision + * would require too much storage space in the hardware, so the precision + * of the color components is reduced before the look up, and the low + * order bits may be used to interpolate between the nearest points in 3D + * space. + * + * The data is interpreted as an array of &struct drm_color_lut elements. + * Hardware might choose not to use the full precision of the LUT + * elements. + * + * Setting this to NULL (blob property value set to 0) means the output + * color is identical to the input color. This is generally the driver + * boot-up state too. Drivers can access this blob through + * &drm_crtc_state.cubic_lut. + * + * â€CUBIC_LUT_SIZEâ€: + * Unsigned range property to give the size of the lookup table to be set + * on the CUBIC_LUT property (the size depends on the underlying hardware). + * If drivers support multiple LUT sizes then they should publish the + * largest size, and sub-sample smaller sized LUTs appropriately. + * * “GAMMA_LUTâ€: * Blob property to set the gamma lookup table (LUT) mapping pixel data - * after the transformation matrix to data sent to the connector. The - * data is interpreted as an array of &struct drm_color_lut elements. - * Hardware might choose not to use the full precision of the LUT elements - * nor use all the elements of the LUT (for example the hardware might - * choose to interpolate between LUT[0] and LUT[4]). + * after the cubic LUT to data sent to the connector. The data is + * interpreted as an array of &struct drm_color_lut elements. Hardware + * might choose not to use the full precision of the LUT elements nor use + * all the elements of the LUT (for example the hardware might choose to + * interpolate between LUT[0] and LUT[4]). * * Setting this to NULL (blob property value set to 0) means a * linear/pass-thru gamma table should be used. This is generally the diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c index b0a826489488..3d7182001004 100644 --- a/drivers/gpu/drm/drm_debugfs.c +++ b/drivers/gpu/drm/drm_debugfs.c @@ -91,7 +91,6 @@ static int drm_clients_info(struct seq_file *m, void *data) mutex_lock(&dev->filelist_mutex); list_for_each_entry_reverse(priv, &dev->filelist, lhead) { struct task_struct *task; - bool is_current_master = drm_is_current_master(priv); rcu_read_lock(); /* locks pid_task()->comm */ task = pid_task(priv->pid, PIDTYPE_PID); @@ -100,7 +99,7 @@ static int drm_clients_info(struct seq_file *m, void *data) task ? task->comm : "", pid_vnr(priv->pid), priv->minor->index, - is_current_master ? 'y' : 'n', + drm_is_current_master(priv) ? 'y' : 'n', priv->authenticated ? 'y' : 'n', from_kuid_munged(seq_user_ns(m), uid), priv->magic); diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c index 39eb39e78d7a..efc2372d32a7 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -1835,20 +1835,11 @@ static void connector_bad_edid(struct drm_connector *connector, u8 *edid, int num_blocks) { int i; - u8 last_block; - - /* - * 0x7e in the EDID is the number of extension blocks. The EDID - * is 1 (base block) + num_ext_blocks big. That means we can think - * of 0x7e in the EDID of the _index_ of the last block in the - * combined chunk of memory. - */ - last_block = edid[0x7e]; + u8 num_of_ext = edid[0x7e]; /* Calculate real checksum for the last edid extension block data */ - if (last_block < num_blocks) - connector->real_edid_checksum = - drm_edid_block_checksum(edid + last_block * EDID_LENGTH); + connector->real_edid_checksum = + drm_edid_block_checksum(edid + num_of_ext * EDID_LENGTH); if (connector->bad_edid_counter++ && !drm_debug_enabled(DRM_UT_KMS)) return; @@ -4861,6 +4852,43 @@ static void drm_parse_vcdb(struct drm_connector *connector, const u8 *db) info->rgb_quant_range_selectable = true; } +#ifdef CONFIG_NO_GKI +static +void drm_get_max_frl_rate(int max_frl_rate, u8 *max_lanes, u8 *max_rate_per_lane) +{ + switch (max_frl_rate) { + case 1: + *max_lanes = 3; + *max_rate_per_lane = 3; + break; + case 2: + *max_lanes = 3; + *max_rate_per_lane = 6; + break; + case 3: + *max_lanes = 4; + *max_rate_per_lane = 6; + break; + case 4: + *max_lanes = 4; + *max_rate_per_lane = 8; + break; + case 5: + *max_lanes = 4; + *max_rate_per_lane = 10; + break; + case 6: + *max_lanes = 4; + *max_rate_per_lane = 12; + break; + case 0: + default: + *max_lanes = 0; + *max_rate_per_lane = 0; + } +} +#endif + static void drm_parse_ycbcr420_deep_color_info(struct drm_connector *connector, const u8 *db) { @@ -4914,6 +4942,76 @@ static void drm_parse_hdmi_forum_vsdb(struct drm_connector *connector, } } +#ifdef CONFIG_NO_GKI + if (hf_vsdb[7]) { + u8 max_frl_rate; + u8 dsc_max_frl_rate; + u8 dsc_max_slices; + struct drm_hdmi_dsc_cap *hdmi_dsc = &hdmi->dsc_cap; + + DRM_DEBUG_KMS("hdmi_21 sink detected. parsing edid\n"); + max_frl_rate = (hf_vsdb[7] & DRM_EDID_MAX_FRL_RATE_MASK) >> 4; + drm_get_max_frl_rate(max_frl_rate, &hdmi->max_lanes, + &hdmi->max_frl_rate_per_lane); + hdmi_dsc->v_1p2 = hf_vsdb[11] & DRM_EDID_DSC_1P2; + + if (hdmi_dsc->v_1p2) { + hdmi_dsc->native_420 = hf_vsdb[11] & DRM_EDID_DSC_NATIVE_420; + hdmi_dsc->all_bpp = hf_vsdb[11] & DRM_EDID_DSC_ALL_BPP; + + if (hf_vsdb[11] & DRM_EDID_DSC_16BPC) + hdmi_dsc->bpc_supported = 16; + else if (hf_vsdb[11] & DRM_EDID_DSC_12BPC) + hdmi_dsc->bpc_supported = 12; + else if (hf_vsdb[11] & DRM_EDID_DSC_10BPC) + hdmi_dsc->bpc_supported = 10; + else + hdmi_dsc->bpc_supported = 0; + + dsc_max_frl_rate = (hf_vsdb[12] & DRM_EDID_DSC_MAX_FRL_RATE_MASK) >> 4; + drm_get_max_frl_rate(dsc_max_frl_rate, &hdmi_dsc->max_lanes, + &hdmi_dsc->max_frl_rate_per_lane); + hdmi_dsc->total_chunk_kbytes = hf_vsdb[13] & DRM_EDID_DSC_TOTAL_CHUNK_KBYTES; + + dsc_max_slices = hf_vsdb[12] & DRM_EDID_DSC_MAX_SLICES; + switch (dsc_max_slices) { + case 1: + hdmi_dsc->max_slices = 1; + hdmi_dsc->clk_per_slice = 340; + break; + case 2: + hdmi_dsc->max_slices = 2; + hdmi_dsc->clk_per_slice = 340; + break; + case 3: + hdmi_dsc->max_slices = 4; + hdmi_dsc->clk_per_slice = 340; + break; + case 4: + hdmi_dsc->max_slices = 8; + hdmi_dsc->clk_per_slice = 340; + break; + case 5: + hdmi_dsc->max_slices = 8; + hdmi_dsc->clk_per_slice = 400; + break; + case 6: + hdmi_dsc->max_slices = 12; + hdmi_dsc->clk_per_slice = 400; + break; + case 7: + hdmi_dsc->max_slices = 16; + hdmi_dsc->clk_per_slice = 400; + break; + case 0: + default: + hdmi_dsc->max_slices = 0; + hdmi_dsc->clk_per_slice = 0; + } + } + } +#endif + drm_parse_ycbcr420_deep_color_info(connector, hf_vsdb); } diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c index 537e7de8e9c3..01670305df3d 100644 --- a/drivers/gpu/drm/drm_file.c +++ b/drivers/gpu/drm/drm_file.c @@ -177,7 +177,6 @@ struct drm_file *drm_file_alloc(struct drm_minor *minor) init_waitqueue_head(&file->event_wait); file->event_space = 4096; /* set aside 4k for event buffer */ - spin_lock_init(&file->master_lookup_lock); mutex_init(&file->event_read_lock); if (drm_core_check_feature(dev, DRIVER_GEM)) @@ -776,20 +775,19 @@ void drm_event_cancel_free(struct drm_device *dev, EXPORT_SYMBOL(drm_event_cancel_free); /** - * drm_send_event_locked - send DRM event to file descriptor + * drm_send_event_helper - send DRM event to file descriptor * @dev: DRM device * @e: DRM event to deliver + * @timestamp: timestamp to set for the fence event in kernel's CLOCK_MONOTONIC + * time domain * - * This function sends the event @e, initialized with drm_event_reserve_init(), - * to its associated userspace DRM file. Callers must already hold - * &drm_device.event_lock, see drm_send_event() for the unlocked version. - * - * Note that the core will take care of unlinking and disarming events when the - * corresponding DRM file is closed. Drivers need not worry about whether the - * DRM file for this event still exists and can call this function upon - * completion of the asynchronous work unconditionally. + * This helper function sends the event @e, initialized with + * drm_event_reserve_init(), to its associated userspace DRM file. + * The timestamp variant of dma_fence_signal is used when the caller + * sends a valid timestamp. */ -void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) +void drm_send_event_helper(struct drm_device *dev, + struct drm_pending_event *e, ktime_t timestamp) { assert_spin_locked(&dev->event_lock); @@ -800,7 +798,10 @@ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) } if (e->fence) { - dma_fence_signal(e->fence); + if (timestamp) + dma_fence_signal_timestamp(e->fence, timestamp); + else + dma_fence_signal(e->fence); dma_fence_put(e->fence); } @@ -815,6 +816,48 @@ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) wake_up_interruptible_poll(&e->file_priv->event_wait, EPOLLIN | EPOLLRDNORM); } + +/** + * drm_send_event_timestamp_locked - send DRM event to file descriptor + * @dev: DRM device + * @e: DRM event to deliver + * @timestamp: timestamp to set for the fence event in kernel's CLOCK_MONOTONIC + * time domain + * + * This function sends the event @e, initialized with drm_event_reserve_init(), + * to its associated userspace DRM file. Callers must already hold + * &drm_device.event_lock. + * + * Note that the core will take care of unlinking and disarming events when the + * corresponding DRM file is closed. Drivers need not worry about whether the + * DRM file for this event still exists and can call this function upon + * completion of the asynchronous work unconditionally. + */ +void drm_send_event_timestamp_locked(struct drm_device *dev, + struct drm_pending_event *e, ktime_t timestamp) +{ + drm_send_event_helper(dev, e, timestamp); +} +EXPORT_SYMBOL(drm_send_event_timestamp_locked); + +/** + * drm_send_event_locked - send DRM event to file descriptor + * @dev: DRM device + * @e: DRM event to deliver + * + * This function sends the event @e, initialized with drm_event_reserve_init(), + * to its associated userspace DRM file. Callers must already hold + * &drm_device.event_lock, see drm_send_event() for the unlocked version. + * + * Note that the core will take care of unlinking and disarming events when the + * corresponding DRM file is closed. Drivers need not worry about whether the + * DRM file for this event still exists and can call this function upon + * completion of the asynchronous work unconditionally. + */ +void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) +{ + drm_send_event_helper(dev, e, 0); +} EXPORT_SYMBOL(drm_send_event_locked); /** @@ -837,7 +880,7 @@ void drm_send_event(struct drm_device *dev, struct drm_pending_event *e) unsigned long irqflags; spin_lock_irqsave(&dev->event_lock, irqflags); - drm_send_event_locked(dev, e); + drm_send_event_helper(dev, e, 0); spin_unlock_irqrestore(&dev->event_lock, irqflags); } EXPORT_SYMBOL(drm_send_event); diff --git a/drivers/gpu/drm/drm_fourcc.c b/drivers/gpu/drm/drm_fourcc.c index 8d1064061e83..e4c8aa3614a4 100644 --- a/drivers/gpu/drm/drm_fourcc.c +++ b/drivers/gpu/drm/drm_fourcc.c @@ -282,6 +282,16 @@ const struct drm_format_info *__drm_format_info(u32 format) .num_planes = 2, .char_per_block = { 5, 5, 0 }, .block_w = { 4, 2, 0 }, .block_h = { 1, 1, 0 }, .hsub = 2, .vsub = 2, .is_yuv = true }, +#ifdef CONFIG_NO_GKI + { .format = DRM_FORMAT_NV20, .depth = 0, + .num_planes = 2, .char_per_block = { 5, 5, 0 }, + .block_w = { 4, 2, 0 }, .block_h = { 1, 1, 0 }, .hsub = 2, + .vsub = 1, .is_yuv = true }, + { .format = DRM_FORMAT_NV30, .depth = 0, + .num_planes = 2, .char_per_block = { 5, 5, 0 }, + .block_w = { 4, 2, 0 }, .block_h = { 1, 1, 0 }, .hsub = 1, + .vsub = 1, .is_yuv = true }, +#endif { .format = DRM_FORMAT_Q410, .depth = 0, .num_planes = 3, .char_per_block = { 2, 2, 2 }, .block_w = { 1, 1, 1 }, .block_h = { 1, 1, 1 }, .hsub = 1, diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c index c160a45a4274..73818ffa019b 100644 --- a/drivers/gpu/drm/drm_ioctl.c +++ b/drivers/gpu/drm/drm_ioctl.c @@ -543,6 +543,7 @@ int drm_version(struct drm_device *dev, void *data, */ int drm_ioctl_permit(u32 flags, struct drm_file *file_priv) { +#ifndef CONFIG_DRM_IGNORE_IOTCL_PERMIT /* ROOT_ONLY is only for CAP_SYS_ADMIN */ if (unlikely((flags & DRM_ROOT_ONLY) && !capable(CAP_SYS_ADMIN))) return -EACCES; @@ -561,6 +562,7 @@ int drm_ioctl_permit(u32 flags, struct drm_file *file_priv) if (unlikely(!(flags & DRM_RENDER_ALLOW) && drm_is_render_client(file_priv))) return -EACCES; +#endif return 0; } @@ -684,9 +686,9 @@ static const struct drm_ioctl_desc drm_ioctls[] = { DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb_ioctl, 0), DRM_IOCTL_DEF(DRM_IOCTL_MODE_PAGE_FLIP, drm_mode_page_flip_ioctl, DRM_MASTER), DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER), - DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, 0), - DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, 0), - DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, 0), + DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_GETPROPERTIES, drm_mode_obj_get_properties_ioctl, 0), DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_SETPROPERTY, drm_mode_obj_set_property_ioctl, DRM_MASTER), DRM_IOCTL_DEF(DRM_IOCTL_MODE_CURSOR2, drm_mode_cursor2_ioctl, DRM_MASTER), diff --git a/drivers/gpu/drm/drm_lease.c b/drivers/gpu/drm/drm_lease.c index aef22634005e..da4f085fc09e 100644 --- a/drivers/gpu/drm/drm_lease.c +++ b/drivers/gpu/drm/drm_lease.c @@ -107,19 +107,10 @@ static bool _drm_has_leased(struct drm_master *master, int id) */ bool _drm_lease_held(struct drm_file *file_priv, int id) { - bool ret; - struct drm_master *master; - - if (!file_priv) + if (!file_priv || !file_priv->master) return true; - master = drm_file_get_master(file_priv); - if (!master) - return true; - ret = _drm_lease_held_master(master, id); - drm_master_put(&master); - - return ret; + return _drm_lease_held_master(file_priv->master, id); } /** @@ -138,22 +129,13 @@ bool drm_lease_held(struct drm_file *file_priv, int id) struct drm_master *master; bool ret; - if (!file_priv) + if (!file_priv || !file_priv->master || !file_priv->master->lessor) return true; - master = drm_file_get_master(file_priv); - if (!master) - return true; - if (!master->lessor) { - ret = true; - goto out; - } + master = file_priv->master; mutex_lock(&master->dev->mode_config.idr_mutex); ret = _drm_lease_held_master(master, id); mutex_unlock(&master->dev->mode_config.idr_mutex); - -out: - drm_master_put(&master); return ret; } @@ -173,16 +155,10 @@ uint32_t drm_lease_filter_crtcs(struct drm_file *file_priv, uint32_t crtcs_in) int count_in, count_out; uint32_t crtcs_out = 0; - if (!file_priv) + if (!file_priv || !file_priv->master || !file_priv->master->lessor) return crtcs_in; - master = drm_file_get_master(file_priv); - if (!master) - return crtcs_in; - if (!master->lessor) { - crtcs_out = crtcs_in; - goto out; - } + master = file_priv->master; dev = master->dev; count_in = count_out = 0; @@ -201,9 +177,6 @@ uint32_t drm_lease_filter_crtcs(struct drm_file *file_priv, uint32_t crtcs_in) count_in++; } mutex_unlock(&master->dev->mode_config.idr_mutex); - -out: - drm_master_put(&master); return crtcs_out; } @@ -517,7 +490,7 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, size_t object_count; int ret = 0; struct idr leases; - struct drm_master *lessor; + struct drm_master *lessor = lessor_priv->master; struct drm_master *lessee = NULL; struct file *lessee_file = NULL; struct file *lessor_file = lessor_priv->filp; @@ -529,6 +502,12 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, if (!drm_core_check_feature(dev, DRIVER_MODESET)) return -EOPNOTSUPP; + /* Do not allow sub-leases */ + if (lessor->lessor) { + DRM_DEBUG_LEASE("recursive leasing not allowed\n"); + return -EINVAL; + } + /* need some objects */ if (cl->object_count == 0) { DRM_DEBUG_LEASE("no objects in lease\n"); @@ -540,22 +519,12 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, return -EINVAL; } - lessor = drm_file_get_master(lessor_priv); - /* Do not allow sub-leases */ - if (lessor->lessor) { - DRM_DEBUG_LEASE("recursive leasing not allowed\n"); - ret = -EINVAL; - goto out_lessor; - } - object_count = cl->object_count; object_ids = memdup_user(u64_to_user_ptr(cl->object_ids), array_size(object_count, sizeof(__u32))); - if (IS_ERR(object_ids)) { - ret = PTR_ERR(object_ids); - goto out_lessor; - } + if (IS_ERR(object_ids)) + return PTR_ERR(object_ids); idr_init(&leases); @@ -566,15 +535,14 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, if (ret) { DRM_DEBUG_LEASE("lease object lookup failed: %i\n", ret); idr_destroy(&leases); - goto out_lessor; + return ret; } /* Allocate a file descriptor for the lease */ fd = get_unused_fd_flags(cl->flags & (O_CLOEXEC | O_NONBLOCK)); if (fd < 0) { idr_destroy(&leases); - ret = fd; - goto out_lessor; + return fd; } DRM_DEBUG_LEASE("Creating lease\n"); @@ -610,7 +578,6 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, /* Hook up the fd */ fd_install(fd, lessee_file); - drm_master_put(&lessor); DRM_DEBUG_LEASE("drm_mode_create_lease_ioctl succeeded\n"); return 0; @@ -620,8 +587,6 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, out_leases: put_unused_fd(fd); -out_lessor: - drm_master_put(&lessor); DRM_DEBUG_LEASE("drm_mode_create_lease_ioctl failed: %d\n", ret); return ret; } @@ -644,7 +609,7 @@ int drm_mode_list_lessees_ioctl(struct drm_device *dev, struct drm_mode_list_lessees *arg = data; __u32 __user *lessee_ids = (__u32 __user *) (uintptr_t) (arg->lessees_ptr); __u32 count_lessees = arg->count_lessees; - struct drm_master *lessor, *lessee; + struct drm_master *lessor = lessor_priv->master, *lessee; int count; int ret = 0; @@ -655,7 +620,6 @@ int drm_mode_list_lessees_ioctl(struct drm_device *dev, if (!drm_core_check_feature(dev, DRIVER_MODESET)) return -EOPNOTSUPP; - lessor = drm_file_get_master(lessor_priv); DRM_DEBUG_LEASE("List lessees for %d\n", lessor->lessee_id); mutex_lock(&dev->mode_config.idr_mutex); @@ -679,7 +643,6 @@ int drm_mode_list_lessees_ioctl(struct drm_device *dev, arg->count_lessees = count; mutex_unlock(&dev->mode_config.idr_mutex); - drm_master_put(&lessor); return ret; } @@ -699,7 +662,7 @@ int drm_mode_get_lease_ioctl(struct drm_device *dev, struct drm_mode_get_lease *arg = data; __u32 __user *object_ids = (__u32 __user *) (uintptr_t) (arg->objects_ptr); __u32 count_objects = arg->count_objects; - struct drm_master *lessee; + struct drm_master *lessee = lessee_priv->master; struct idr *object_idr; int count; void *entry; @@ -713,7 +676,6 @@ int drm_mode_get_lease_ioctl(struct drm_device *dev, if (!drm_core_check_feature(dev, DRIVER_MODESET)) return -EOPNOTSUPP; - lessee = drm_file_get_master(lessee_priv); DRM_DEBUG_LEASE("get lease for %d\n", lessee->lessee_id); mutex_lock(&dev->mode_config.idr_mutex); @@ -741,7 +703,6 @@ int drm_mode_get_lease_ioctl(struct drm_device *dev, arg->count_objects = count; mutex_unlock(&dev->mode_config.idr_mutex); - drm_master_put(&lessee); return ret; } @@ -760,7 +721,7 @@ int drm_mode_revoke_lease_ioctl(struct drm_device *dev, void *data, struct drm_file *lessor_priv) { struct drm_mode_revoke_lease *arg = data; - struct drm_master *lessor; + struct drm_master *lessor = lessor_priv->master; struct drm_master *lessee; int ret = 0; @@ -770,7 +731,6 @@ int drm_mode_revoke_lease_ioctl(struct drm_device *dev, if (!drm_core_check_feature(dev, DRIVER_MODESET)) return -EOPNOTSUPP; - lessor = drm_file_get_master(lessor_priv); mutex_lock(&dev->mode_config.idr_mutex); lessee = _drm_find_lessee(lessor, arg->lessee_id); @@ -791,7 +751,6 @@ int drm_mode_revoke_lease_ioctl(struct drm_device *dev, fail: mutex_unlock(&dev->mode_config.idr_mutex); - drm_master_put(&lessor); return ret; } diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c index 19fb1d93a4f0..3caf9ff34a1c 100644 --- a/drivers/gpu/drm/drm_mipi_dsi.c +++ b/drivers/gpu/drm/drm_mipi_dsi.c @@ -356,6 +356,7 @@ static ssize_t mipi_dsi_device_transfer(struct mipi_dsi_device *dsi, if (dsi->mode_flags & MIPI_DSI_MODE_LPM) msg->flags |= MIPI_DSI_MSG_USE_LPM; + msg->flags |= MIPI_DSI_MSG_LASTCOMMAND; return ops->transfer(dsi->host, msg); } diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c index fad2c1181127..58050d4aeabc 100644 --- a/drivers/gpu/drm/drm_mode_config.c +++ b/drivers/gpu/drm/drm_mode_config.c @@ -364,6 +364,22 @@ static int drm_mode_create_standard_properties(struct drm_device *dev) return -ENOMEM; dev->mode_config.gamma_lut_size_property = prop; +#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) + prop = drm_property_create(dev, + DRM_MODE_PROP_BLOB, + "CUBIC_LUT", 0); + if (!prop) + return -ENOMEM; + dev->mode_config.cubic_lut_property = prop; + + prop = drm_property_create_range(dev, + DRM_MODE_PROP_IMMUTABLE, + "CUBIC_LUT_SIZE", 0, UINT_MAX); + if (!prop) + return -ENOMEM; + dev->mode_config.cubic_lut_size_property = prop; +#endif + prop = drm_property_create(dev, DRM_MODE_PROP_IMMUTABLE | DRM_MODE_PROP_BLOB, "IN_FORMATS", 0); diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c index 0f99e5453f15..d42c7310bf09 100644 --- a/drivers/gpu/drm/drm_modes.c +++ b/drivers/gpu/drm/drm_modes.c @@ -1940,6 +1940,7 @@ void drm_mode_convert_to_umode(struct drm_mode_modeinfo *out, strncpy(out->name, in->name, DRM_DISPLAY_MODE_LEN); out->name[DRM_DISPLAY_MODE_LEN-1] = 0; } +EXPORT_SYMBOL_GPL(drm_mode_convert_to_umode); /** * drm_crtc_convert_umode - convert a modeinfo into a drm_display_mode @@ -2016,6 +2017,7 @@ int drm_mode_convert_umode(struct drm_device *dev, return 0; } +EXPORT_SYMBOL_GPL(drm_mode_convert_umode); /** * drm_mode_is_420_only - if a given videomode can be only supported in YCBCR420 diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c index 825499ea3ff5..272e5cdd61ce 100644 --- a/drivers/gpu/drm/drm_prime.c +++ b/drivers/gpu/drm/drm_prime.c @@ -784,6 +784,28 @@ int drm_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) } EXPORT_SYMBOL(drm_gem_dmabuf_mmap); +/** + * drm_gem_dmabuf_get_uuid - dma_buf get_uuid implementation for GEM + * @dma_buf: buffer to query + * @uuid: uuid outparam + * + * Queries the buffer's virtio UUID. This can be used as the + * &dma_buf_ops.get_uuid callback. Calls into &drm_driver.gem_prime_get_uuid. + * + * Returns 0 on success or a negative error code on failure. + */ +int drm_gem_dmabuf_get_uuid(struct dma_buf *dma_buf, uuid_t *uuid) +{ + struct drm_gem_object *obj = dma_buf->priv; + struct drm_device *dev = obj->dev; + + if (!dev->driver->gem_prime_get_uuid) + return -ENODEV; + + return dev->driver->gem_prime_get_uuid(obj, uuid); +} +EXPORT_SYMBOL(drm_gem_dmabuf_get_uuid); + static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { .cache_sgt_mapping = true, .attach = drm_gem_map_attach, @@ -794,6 +816,7 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { .mmap = drm_gem_dmabuf_mmap, .vmap = drm_gem_dmabuf_vmap, .vunmap = drm_gem_dmabuf_vunmap, + .get_uuid = drm_gem_dmabuf_get_uuid, }; /** diff --git a/drivers/gpu/drm/drm_sync_helper.c b/drivers/gpu/drm/drm_sync_helper.c new file mode 100755 index 000000000000..0c6aa875b4c6 --- /dev/null +++ b/drivers/gpu/drm/drm_sync_helper.c @@ -0,0 +1,314 @@ +/* + * drm_sync_helper.c: software fence and helper functions for fences and + * reservations used for dma buffer access synchronization between drivers. + * + * Copyright 2014 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include + +static DEFINE_SPINLOCK(sw_fence_lock); + +void drm_add_reservation(struct reservation_object *resv, + struct reservation_object **resvs, + unsigned long *excl_resvs_bitmap, + unsigned int *num_resvs, bool exclusive) +{ + unsigned int r; + + for (r = 0; r < *num_resvs; r++) { + if (resvs[r] == resv) + return; + } + resvs[*num_resvs] = resv; + if (exclusive) + set_bit(*num_resvs, excl_resvs_bitmap); + (*num_resvs)++; +} +EXPORT_SYMBOL(drm_add_reservation); + +int drm_lock_reservations(struct reservation_object **resvs, + unsigned int num_resvs, struct ww_acquire_ctx *ctx) +{ + unsigned int r; + struct reservation_object *slow_res = NULL; + + ww_acquire_init(ctx, &reservation_ww_class); + +retry: + for (r = 0; r < num_resvs; r++) { + int ret; + /* skip the resv we locked with slow lock */ + if (resvs[r] == slow_res) { + slow_res = NULL; + continue; + } + ret = ww_mutex_lock(&resvs[r]->lock, ctx); + if (ret < 0) { + unsigned int slow_r = r; + /* + * undo all the locks we already done, + * in reverse order + */ + while (r > 0) { + r--; + ww_mutex_unlock(&resvs[r]->lock); + } + if (slow_res) + ww_mutex_unlock(&slow_res->lock); + if (ret == -EDEADLK) { + slow_res = resvs[slow_r]; + ww_mutex_lock_slow(&slow_res->lock, ctx); + goto retry; + } + ww_acquire_fini(ctx); + return ret; + } + } + + ww_acquire_done(ctx); + return 0; +} +EXPORT_SYMBOL(drm_lock_reservations); + +void drm_unlock_reservations(struct reservation_object **resvs, + unsigned int num_resvs, + struct ww_acquire_ctx *ctx) +{ + unsigned int r; + + for (r = 0; r < num_resvs; r++) + ww_mutex_unlock(&resvs[r]->lock); + + ww_acquire_fini(ctx); +} +EXPORT_SYMBOL(drm_unlock_reservations); + +static void reservation_cb_fence_cb(struct fence *fence, struct fence_cb *cb) +{ + struct drm_reservation_fence_cb *rfcb = + container_of(cb, struct drm_reservation_fence_cb, base); + struct drm_reservation_cb *rcb = rfcb->parent; + + if (atomic_dec_and_test(&rcb->count)) + schedule_work(&rcb->work); +} + +static void +reservation_cb_cleanup(struct drm_reservation_cb *rcb) +{ + unsigned cb; + + for (cb = 0; cb < rcb->num_fence_cbs; cb++) { + if (rcb->fence_cbs[cb]) { + fence_remove_callback(rcb->fence_cbs[cb]->fence, + &rcb->fence_cbs[cb]->base); + fence_put(rcb->fence_cbs[cb]->fence); + kfree(rcb->fence_cbs[cb]); + rcb->fence_cbs[cb] = NULL; + } + } + kfree(rcb->fence_cbs); + rcb->fence_cbs = NULL; + rcb->num_fence_cbs = 0; +} + +static void reservation_cb_work(struct work_struct *pwork) +{ + struct drm_reservation_cb *rcb = + container_of(pwork, struct drm_reservation_cb, work); + /* + * clean up everything before calling the callback, because the callback + * may free structure containing rcb and work_struct + */ + reservation_cb_cleanup(rcb); + rcb->func(rcb, rcb->context); +} + +static int +reservation_cb_add_fence_cb(struct drm_reservation_cb *rcb, struct fence *fence) +{ + int ret = 0; + struct drm_reservation_fence_cb *fence_cb; + struct drm_reservation_fence_cb **new_fence_cbs; + + new_fence_cbs = krealloc(rcb->fence_cbs, + (rcb->num_fence_cbs + 1) + * sizeof(struct drm_reservation_fence_cb *), + GFP_KERNEL); + if (!new_fence_cbs) + return -ENOMEM; + rcb->fence_cbs = new_fence_cbs; + + fence_cb = kzalloc(sizeof(struct drm_reservation_fence_cb), GFP_KERNEL); + if (!fence_cb) + return -ENOMEM; + + /* + * do not want for fence to disappear on us while we are waiting for + * callback and we need it in case we want to remove callbacks + */ + fence_get(fence); + fence_cb->fence = fence; + fence_cb->parent = rcb; + rcb->fence_cbs[rcb->num_fence_cbs] = fence_cb; + atomic_inc(&rcb->count); + ret = fence_add_callback(fence, &fence_cb->base, + reservation_cb_fence_cb); + if (ret == -ENOENT) { + /* already signaled */ + atomic_dec(&rcb->count); + fence_put(fence_cb->fence); + kfree(fence_cb); + ret = 0; + } else if (ret < 0) { + atomic_dec(&rcb->count); + fence_put(fence_cb->fence); + kfree(fence_cb); + return ret; + } else { + rcb->num_fence_cbs++; + } + return ret; +} + +void +drm_reservation_cb_init(struct drm_reservation_cb *rcb, + drm_reservation_cb_func_t func, void *context) +{ + INIT_WORK(&rcb->work, reservation_cb_work); + atomic_set(&rcb->count, 1); + rcb->num_fence_cbs = 0; + rcb->fence_cbs = NULL; + rcb->func = func; + rcb->context = context; +} +EXPORT_SYMBOL(drm_reservation_cb_init); + +int +drm_reservation_cb_add(struct drm_reservation_cb *rcb, + struct reservation_object *resv, bool exclusive) +{ + int ret = 0; + struct fence *fence; + unsigned shared_count = 0, f; + struct fence **shared_fences = NULL; + + /* enum all the fences in the reservation and add callbacks */ + ret = reservation_object_get_fences_rcu(resv, &fence, + &shared_count, &shared_fences); + if (ret < 0) + return ret; + + if (fence) { + ret = reservation_cb_add_fence_cb(rcb, fence); + if (ret < 0) { + reservation_cb_cleanup(rcb); + goto error; + } + } + + if (exclusive) { + for (f = 0; f < shared_count; f++) { + ret = reservation_cb_add_fence_cb(rcb, + shared_fences[f]); + if (ret < 0) { + reservation_cb_cleanup(rcb); + goto error; + } + } + } + +error: + if (fence) + fence_put(fence); + if (shared_fences) { + for (f = 0; f < shared_count; f++) + fence_put(shared_fences[f]); + kfree(shared_fences); + } + return ret; +} +EXPORT_SYMBOL(drm_reservation_cb_add); + +void +drm_reservation_cb_done(struct drm_reservation_cb *rcb) +{ + /* + * we need to decrement from initial 1 + * and trigger the callback in case all the + * fences were already triggered + */ + if (atomic_dec_and_test(&rcb->count)) { + /* + * we could call the callback here directly but in case + * the callback function needs to lock the same mutex + * as our caller it could cause a deadlock, so it is + * safer to call it from a worker + */ + schedule_work(&rcb->work); + } +} +EXPORT_SYMBOL(drm_reservation_cb_done); + +void +drm_reservation_cb_fini(struct drm_reservation_cb *rcb) +{ + /* make sure no work will be triggered */ + atomic_set(&rcb->count, 0); + cancel_work_sync(&rcb->work); + reservation_cb_cleanup(rcb); +} +EXPORT_SYMBOL(drm_reservation_cb_fini); + +static bool sw_fence_enable_signaling(struct fence *f) +{ + return true; +} + +static const char *sw_fence_get_get_driver_name(struct fence *fence) +{ + return "drm_sync_helper"; +} + +static const char *sw_fence_get_timeline_name(struct fence *f) +{ + return "drm_sync.sw"; +} + +static const struct fence_ops sw_fence_ops = { + .get_driver_name = sw_fence_get_get_driver_name, + .get_timeline_name = sw_fence_get_timeline_name, + .enable_signaling = sw_fence_enable_signaling, + .signaled = NULL, + .wait = fence_default_wait, + .release = NULL +}; + +struct fence *drm_sw_fence_new(unsigned int context, unsigned seqno) +{ + struct fence *fence; + + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return ERR_PTR(-ENOMEM); + fence_init(fence, + &sw_fence_ops, + &sw_fence_lock, + context, seqno); + + return fence; +} +EXPORT_SYMBOL(drm_sw_fence_new); diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c index f135b79593dd..286edbe1bf7a 100644 --- a/drivers/gpu/drm/drm_vblank.c +++ b/drivers/gpu/drm/drm_vblank.c @@ -1000,7 +1000,14 @@ static void send_vblank_event(struct drm_device *dev, break; } trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq); - drm_send_event_locked(dev, &e->base); + /* + * Use the same timestamp for any associated fence signal to avoid + * mismatch in timestamps for vsync & fence events triggered by the + * same HW event. Frameworks like SurfaceFlinger in Android expects the + * retire-fence timestamp to match exactly with HW vsync as it uses it + * for its software vsync modeling. + */ + drm_send_event_timestamp_locked(dev, &e->base, now); } /** diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index c940ac3aae2f..a19537706ed1 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -305,9 +305,8 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) engine->i915 = i915; engine->gt = gt; engine->uncore = gt->uncore; + engine->hw_id = engine->guc_id = info->hw_id; engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases); - engine->hw_id = info->hw_id; - engine->guc_id = MAKE_GUC_ID(info->class, info->instance); engine->class = info->class; engine->instance = info->instance; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index 6909da1e1a73..942c7c187adb 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -213,6 +213,23 @@ static u32 guc_ctl_feature_flags(struct intel_guc *guc) return flags; } +static u32 guc_ctl_ctxinfo_flags(struct intel_guc *guc) +{ + u32 flags = 0; + + if (intel_guc_submission_is_used(guc)) { + u32 ctxnum, base; + + base = intel_guc_ggtt_offset(guc, guc->stage_desc_pool); + ctxnum = GUC_MAX_STAGE_DESCRIPTORS / 16; + + base >>= PAGE_SHIFT; + flags |= (base << GUC_CTL_BASE_ADDR_SHIFT) | + (ctxnum << GUC_CTL_CTXNUM_IN16_SHIFT); + } + return flags; +} + static u32 guc_ctl_log_params_flags(struct intel_guc *guc) { u32 offset = intel_guc_ggtt_offset(guc, guc->log.vma) >> PAGE_SHIFT; @@ -274,6 +291,7 @@ static void guc_init_params(struct intel_guc *guc) BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32)); + params[GUC_CTL_CTXINFO] = guc_ctl_ctxinfo_flags(guc); params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc); params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc); params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c index 7950d28beb8c..d44061033f23 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c @@ -10,52 +10,11 @@ /* * The Additional Data Struct (ADS) has pointers for different buffers used by - * the GuC. One single gem object contains the ADS struct itself (guc_ads) and - * all the extra buffers indirectly linked via the ADS struct's entries. - * - * Layout of the ADS blob allocated for the GuC: - * - * +---------------------------------------+ <== base - * | guc_ads | - * +---------------------------------------+ - * | guc_policies | - * +---------------------------------------+ - * | guc_gt_system_info | - * +---------------------------------------+ - * | guc_clients_info | - * +---------------------------------------+ - * | guc_ct_pool_entry[size] | - * +---------------------------------------+ - * | padding | - * +---------------------------------------+ <== 4K aligned - * | private data | - * +---------------------------------------+ - * | padding | - * +---------------------------------------+ <== 4K aligned + * the GuC. One single gem object contains the ADS struct itself (guc_ads), the + * scheduling policies (guc_policies), a structure describing a collection of + * register sets (guc_mmio_reg_state) and some extra pages for the GuC to save + * its internal state for sleep. */ -struct __guc_ads_blob { - struct guc_ads ads; - struct guc_policies policies; - struct guc_gt_system_info system_info; - struct guc_clients_info clients_info; - struct guc_ct_pool_entry ct_pool[GUC_CT_POOL_SIZE]; -} __packed; - -static u32 guc_ads_private_data_size(struct intel_guc *guc) -{ - return PAGE_ALIGN(guc->fw.private_data_size); -} - -static u32 guc_ads_private_data_offset(struct intel_guc *guc) -{ - return PAGE_ALIGN(sizeof(struct __guc_ads_blob)); -} - -static u32 guc_ads_blob_size(struct intel_guc *guc) -{ - return guc_ads_private_data_offset(guc) + - guc_ads_private_data_size(guc); -} static void guc_policy_init(struct guc_policy *policy) { @@ -89,37 +48,26 @@ static void guc_ct_pool_entries_init(struct guc_ct_pool_entry *pool, u32 num) memset(pool, 0, num * sizeof(*pool)); } -static void guc_mapping_table_init(struct intel_gt *gt, - struct guc_gt_system_info *system_info) -{ - unsigned int i, j; - struct intel_engine_cs *engine; - enum intel_engine_id id; - - /* Table must be set to invalid values for entries not used */ - for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) - for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) - system_info->mapping_table[i][j] = - GUC_MAX_INSTANCES_PER_CLASS; - - for_each_engine(engine, gt, id) { - u8 guc_class = engine->class; - - system_info->mapping_table[guc_class][engine->instance] = - engine->instance; - } -} - /* * The first 80 dwords of the register state context, containing the * execlists and ppgtt registers. */ #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) +/* The ads obj includes the struct itself and buffers passed to GuC */ +struct __guc_ads_blob { + struct guc_ads ads; + struct guc_policies policies; + struct guc_mmio_reg_state reg_state; + struct guc_gt_system_info system_info; + struct guc_clients_info clients_info; + struct guc_ct_pool_entry ct_pool[GUC_CT_POOL_SIZE]; + u8 reg_state_buffer[GUC_S3_SAVE_SPACE_PAGES * PAGE_SIZE]; +} __packed; + static void __guc_ads_init(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); - struct drm_i915_private *i915 = gt->i915; struct __guc_ads_blob *blob = guc->ads_blob; const u32 skipped_size = LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE; u32 base; @@ -151,25 +99,13 @@ static void __guc_ads_init(struct intel_guc *guc) } /* System info */ - blob->system_info.engine_enabled_masks[RENDER_CLASS] = 1; - blob->system_info.engine_enabled_masks[COPY_ENGINE_CLASS] = 1; - blob->system_info.engine_enabled_masks[VIDEO_DECODE_CLASS] = VDBOX_MASK(gt); - blob->system_info.engine_enabled_masks[VIDEO_ENHANCEMENT_CLASS] = VEBOX_MASK(gt); - - blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED] = - hweight8(gt->info.sseu.slice_mask); - blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK] = - gt->info.vdbox_sfc_access; - - if (INTEL_GEN(i915) >= 12 && !IS_DGFX(i915)) { - u32 distdbreg = intel_uncore_read(gt->uncore, - GEN12_DIST_DBS_POPULATED); - blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI] = - ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) & - GEN12_DOORBELLS_PER_SQIDI) + 1; - } + blob->system_info.slice_enabled = hweight8(gt->info.sseu.slice_mask); + blob->system_info.rcs_enabled = 1; + blob->system_info.bcs_enabled = 1; - guc_mapping_table_init(guc_to_gt(guc), &blob->system_info); + blob->system_info.vdbox_enable_mask = VDBOX_MASK(gt); + blob->system_info.vebox_enable_mask = VEBOX_MASK(gt); + blob->system_info.vdbox_sfc_support_mask = gt->info.vdbox_sfc_access; base = intel_guc_ggtt_offset(guc, guc->ads_vma); @@ -182,12 +118,11 @@ static void __guc_ads_init(struct intel_guc *guc) /* ADS */ blob->ads.scheduler_policies = base + ptr_offset(blob, policies); + blob->ads.reg_state_buffer = base + ptr_offset(blob, reg_state_buffer); + blob->ads.reg_state_addr = base + ptr_offset(blob, reg_state); blob->ads.gt_system_info = base + ptr_offset(blob, system_info); blob->ads.clients_info = base + ptr_offset(blob, clients_info); - /* Private Data */ - blob->ads.private_data = base + guc_ads_private_data_offset(guc); - i915_gem_object_flush_map(guc->ads_vma->obj); } @@ -200,15 +135,14 @@ static void __guc_ads_init(struct intel_guc *guc) */ int intel_guc_ads_create(struct intel_guc *guc) { - u32 size; + const u32 size = PAGE_ALIGN(sizeof(struct __guc_ads_blob)); int ret; GEM_BUG_ON(guc->ads_vma); - size = guc_ads_blob_size(guc); - ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, (void **)&guc->ads_blob); + if (ret) return ret; @@ -222,18 +156,6 @@ void intel_guc_ads_destroy(struct intel_guc *guc) i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); } -static void guc_ads_private_data_reset(struct intel_guc *guc) -{ - u32 size; - - size = guc_ads_private_data_size(guc); - if (!size) - return; - - memset((void *)guc->ads_blob + guc_ads_private_data_offset(guc), 0, - size); -} - /** * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse * @guc: intel_guc struct @@ -246,8 +168,5 @@ void intel_guc_ads_reset(struct intel_guc *guc) { if (!guc->ads_vma) return; - __guc_ads_init(guc); - - guc_ads_private_data_reset(guc); } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h index 79c560d9c0b6..a6b733c146c9 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h @@ -26,8 +26,8 @@ #define GUC_VIDEO_ENGINE2 4 #define GUC_MAX_ENGINES_NUM (GUC_VIDEO_ENGINE2 + 1) -#define GUC_MAX_ENGINE_CLASSES 16 -#define GUC_MAX_INSTANCES_PER_CLASS 32 +#define GUC_MAX_ENGINE_CLASSES 5 +#define GUC_MAX_INSTANCES_PER_CLASS 16 #define GUC_DOORBELL_INVALID 256 @@ -62,7 +62,12 @@ #define GUC_STAGE_DESC_ATTR_PCH BIT(6) #define GUC_STAGE_DESC_ATTR_TERMINATED BIT(7) -#define GUC_CTL_LOG_PARAMS 0 +/* New GuC control data */ +#define GUC_CTL_CTXINFO 0 +#define GUC_CTL_CTXNUM_IN16_SHIFT 0 +#define GUC_CTL_BASE_ADDR_SHIFT 12 + +#define GUC_CTL_LOG_PARAMS 1 #define GUC_LOG_VALID (1 << 0) #define GUC_LOG_NOTIFY_ON_HALF_FULL (1 << 1) #define GUC_LOG_ALLOC_IN_MEGABYTE (1 << 3) @@ -74,11 +79,11 @@ #define GUC_LOG_ISR_MASK (0x7 << GUC_LOG_ISR_SHIFT) #define GUC_LOG_BUF_ADDR_SHIFT 12 -#define GUC_CTL_WA 1 -#define GUC_CTL_FEATURE 2 +#define GUC_CTL_WA 2 +#define GUC_CTL_FEATURE 3 #define GUC_CTL_DISABLE_SCHEDULER (1 << 14) -#define GUC_CTL_DEBUG 3 +#define GUC_CTL_DEBUG 4 #define GUC_LOG_VERBOSITY_SHIFT 0 #define GUC_LOG_VERBOSITY_LOW (0 << GUC_LOG_VERBOSITY_SHIFT) #define GUC_LOG_VERBOSITY_MED (1 << GUC_LOG_VERBOSITY_SHIFT) @@ -92,37 +97,12 @@ #define GUC_LOG_DISABLED (1 << 6) #define GUC_PROFILE_ENABLED (1 << 7) -#define GUC_CTL_ADS 4 +#define GUC_CTL_ADS 5 #define GUC_ADS_ADDR_SHIFT 1 #define GUC_ADS_ADDR_MASK (0xFFFFF << GUC_ADS_ADDR_SHIFT) #define GUC_CTL_MAX_DWORDS (SOFT_SCRATCH_COUNT - 2) /* [1..14] */ -/* Generic GT SysInfo data types */ -#define GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED 0 -#define GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK 1 -#define GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI 2 -#define GUC_GENERIC_GT_SYSINFO_MAX 16 - -/* - * The class goes in bits [0..2] of the GuC ID, the instance in bits [3..6]. - * Bit 7 can be used for operations that apply to all engine classes&instances. - */ -#define GUC_ENGINE_CLASS_SHIFT 0 -#define GUC_ENGINE_CLASS_MASK (0x7 << GUC_ENGINE_CLASS_SHIFT) -#define GUC_ENGINE_INSTANCE_SHIFT 3 -#define GUC_ENGINE_INSTANCE_MASK (0xf << GUC_ENGINE_INSTANCE_SHIFT) -#define GUC_ENGINE_ALL_INSTANCES BIT(7) - -#define MAKE_GUC_ID(class, instance) \ - (((class) << GUC_ENGINE_CLASS_SHIFT) | \ - ((instance) << GUC_ENGINE_INSTANCE_SHIFT)) - -#define GUC_ID_TO_ENGINE_CLASS(guc_id) \ - (((guc_id) & GUC_ENGINE_CLASS_MASK) >> GUC_ENGINE_CLASS_SHIFT) -#define GUC_ID_TO_ENGINE_INSTANCE(guc_id) \ - (((guc_id) & GUC_ENGINE_INSTANCE_MASK) >> GUC_ENGINE_INSTANCE_SHIFT) - /* Work item for submitting workloads into work queue of GuC. */ struct guc_wq_item { u32 header; @@ -356,6 +336,11 @@ struct guc_policies { } __packed; /* GuC MMIO reg state struct */ + + +#define GUC_REGSET_MAX_REGISTERS 64 +#define GUC_S3_SAVE_SPACE_PAGES 10 + struct guc_mmio_reg { u32 offset; u32 value; @@ -363,18 +348,28 @@ struct guc_mmio_reg { #define GUC_REGSET_MASKED (1 << 0) } __packed; +struct guc_mmio_regset { + struct guc_mmio_reg registers[GUC_REGSET_MAX_REGISTERS]; + u32 values_valid; + u32 number_of_registers; +} __packed; + /* GuC register sets */ -struct guc_mmio_reg_set { - u32 address; - u16 count; - u16 reserved; +struct guc_mmio_reg_state { + struct guc_mmio_regset engine_reg[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; + u32 reserved[98]; } __packed; /* HW info */ struct guc_gt_system_info { - u8 mapping_table[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; - u32 engine_enabled_masks[GUC_MAX_ENGINE_CLASSES]; - u32 generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_MAX]; + u32 slice_enabled; + u32 rcs_enabled; + u32 reserved0; + u32 bcs_enabled; + u32 vdbox_enable_mask; + u32 vdbox_sfc_support_mask; + u32 vebox_enable_mask; + u32 reserved[9]; } __packed; /* Clients info */ @@ -395,16 +390,15 @@ struct guc_clients_info { /* GuC Additional Data Struct */ struct guc_ads { - struct guc_mmio_reg_set reg_state_list[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; - u32 reserved0; + u32 reg_state_addr; + u32 reg_state_buffer; u32 scheduler_policies; u32 gt_system_info; u32 clients_info; u32 control_data; u32 golden_context_lrca[GUC_MAX_ENGINE_CLASSES]; u32 eng_state_size[GUC_MAX_ENGINE_CLASSES]; - u32 private_data; - u32 reserved[15]; + u32 reserved[16]; } __packed; /* GuC logging structures */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h index b37fc2ffaef2..1949346e714e 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h @@ -118,11 +118,6 @@ struct guc_doorbell_info { #define GEN8_DRB_VALID (1<<0) #define GEN8_DRBREGU(x) _MMIO(0x1000 + (x) * 8 + 4) -#define GEN12_DIST_DBS_POPULATED _MMIO(0xd08) -#define GEN12_DOORBELLS_PER_SQIDI_SHIFT 16 -#define GEN12_DOORBELLS_PER_SQIDI (0xff) -#define GEN12_SQIDIS_DOORBELL_EXIST (0xffff) - #define DE_GUCRMR _MMIO(0x44054) #define GUC_BCS_RCS_IER _MMIO(0xC550) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c index ee4ac3922277..80e8b6c3bc8c 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c @@ -44,19 +44,23 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, * List of required GuC and HuC binaries per-platform. * Must be ordered based on platform + revid, from newer to older. * + * TGL 35.2 is interface-compatible with 33.0 for previous Gens. The deltas + * between 33.0 and 35.2 are only related to new additions to support new Gen12 + * features. + * * Note that RKL uses the same firmware as TGL. */ #define INTEL_UC_FIRMWARE_DEFS(fw_def, guc_def, huc_def) \ - fw_def(ROCKETLAKE, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ - fw_def(TIGERLAKE, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ - fw_def(ELKHARTLAKE, 0, guc_def(ehl, 49, 0, 1), huc_def(ehl, 9, 0, 0)) \ - fw_def(ICELAKE, 0, guc_def(icl, 49, 0, 1), huc_def(icl, 9, 0, 0)) \ - fw_def(COMETLAKE, 5, guc_def(cml, 49, 0, 1), huc_def(cml, 4, 0, 0)) \ - fw_def(COFFEELAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ - fw_def(GEMINILAKE, 0, guc_def(glk, 49, 0, 1), huc_def(glk, 4, 0, 0)) \ - fw_def(KABYLAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ - fw_def(BROXTON, 0, guc_def(bxt, 49, 0, 1), huc_def(bxt, 2, 0, 0)) \ - fw_def(SKYLAKE, 0, guc_def(skl, 49, 0, 1), huc_def(skl, 2, 0, 0)) + fw_def(ROCKETLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 5, 0)) \ + fw_def(TIGERLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 5, 0)) \ + fw_def(ELKHARTLAKE, 0, guc_def(ehl, 33, 0, 4), huc_def(ehl, 9, 0, 0)) \ + fw_def(ICELAKE, 0, guc_def(icl, 33, 0, 0), huc_def(icl, 9, 0, 0)) \ + fw_def(COMETLAKE, 5, guc_def(cml, 33, 0, 0), huc_def(cml, 4, 0, 0)) \ + fw_def(COFFEELAKE, 0, guc_def(kbl, 33, 0, 0), huc_def(kbl, 4, 0, 0)) \ + fw_def(GEMINILAKE, 0, guc_def(glk, 33, 0, 0), huc_def(glk, 4, 0, 0)) \ + fw_def(KABYLAKE, 0, guc_def(kbl, 33, 0, 0), huc_def(kbl, 4, 0, 0)) \ + fw_def(BROXTON, 0, guc_def(bxt, 33, 0, 0), huc_def(bxt, 2, 0, 0)) \ + fw_def(SKYLAKE, 0, guc_def(skl, 33, 0, 0), huc_def(skl, 2, 0, 0)) #define __MAKE_UC_FW_PATH(prefix_, name_, major_, minor_, patch_) \ "i915/" \ @@ -367,9 +371,6 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) } } - if (uc_fw->type == INTEL_UC_FW_TYPE_GUC) - uc_fw->private_data_size = css->private_data_size; - obj = i915_gem_object_create_shmem_from_data(i915, fw->data, fw->size); if (IS_ERR(obj)) { err = PTR_ERR(obj); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h index 99bb1fe1af66..23d3a423ac0f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h @@ -88,8 +88,6 @@ struct intel_uc_fw { u32 rsa_size; u32 ucode_size; - - u32 private_data_size; }; #ifdef CONFIG_DRM_I915_DEBUG_GUC diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h index e41ffc7a7fbc..029214cdedd5 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h @@ -69,11 +69,7 @@ struct uc_css_header { #define CSS_SW_VERSION_UC_MAJOR (0xFF << 16) #define CSS_SW_VERSION_UC_MINOR (0xFF << 8) #define CSS_SW_VERSION_UC_PATCH (0xFF << 0) - u32 reserved0[13]; - union { - u32 private_data_size; /* only applies to GuC */ - u32 reserved1; - }; + u32 reserved[14]; u32 header_info; } __packed; static_assert(sizeof(struct uc_css_header) == 128); diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index ac96b6ab44c0..42fc5c813a9b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -557,7 +557,6 @@ nouveau_drm_device_init(struct drm_device *dev) nvkm_dbgopt(nouveau_debug, "DRM"); INIT_LIST_HEAD(&drm->clients); - mutex_init(&drm->clients_lock); spin_lock_init(&drm->tile.lock); /* workaround an odd issue on nvc1 by disabling the device's @@ -628,7 +627,6 @@ nouveau_drm_device_init(struct drm_device *dev) static void nouveau_drm_device_fini(struct drm_device *dev) { - struct nouveau_cli *cli, *temp_cli; struct nouveau_drm *drm = nouveau_drm(dev); if (nouveau_pmops_runtime()) { @@ -653,28 +651,9 @@ nouveau_drm_device_fini(struct drm_device *dev) nouveau_ttm_fini(drm); nouveau_vga_fini(drm); - /* - * There may be existing clients from as-yet unclosed files. For now, - * clean them up here rather than deferring until the file is closed, - * but this likely not correct if we want to support hot-unplugging - * properly. - */ - mutex_lock(&drm->clients_lock); - list_for_each_entry_safe(cli, temp_cli, &drm->clients, head) { - list_del(&cli->head); - mutex_lock(&cli->mutex); - if (cli->abi16) - nouveau_abi16_fini(cli->abi16); - mutex_unlock(&cli->mutex); - nouveau_cli_fini(cli); - kfree(cli); - } - mutex_unlock(&drm->clients_lock); - nouveau_cli_fini(&drm->client); nouveau_cli_fini(&drm->master); nvif_parent_dtor(&drm->parent); - mutex_destroy(&drm->clients_lock); kfree(drm); } @@ -813,7 +792,7 @@ nouveau_drm_device_remove(struct drm_device *dev) struct nvkm_client *client; struct nvkm_device *device; - drm_dev_unplug(dev); + drm_dev_unregister(dev); dev->irq_enabled = false; client = nvxx_client(&drm->client.base); @@ -1107,9 +1086,9 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv) fpriv->driver_priv = cli; - mutex_lock(&drm->clients_lock); + mutex_lock(&drm->client.mutex); list_add(&cli->head, &drm->clients); - mutex_unlock(&drm->clients_lock); + mutex_unlock(&drm->client.mutex); done: if (ret && cli) { @@ -1127,16 +1106,6 @@ nouveau_drm_postclose(struct drm_device *dev, struct drm_file *fpriv) { struct nouveau_cli *cli = nouveau_cli(fpriv); struct nouveau_drm *drm = nouveau_drm(dev); - int dev_index; - - /* - * The device is gone, and as it currently stands all clients are - * cleaned up in the removal codepath. In the future this may change - * so that we can support hot-unplugging, but for now we immediately - * return to avoid a double-free situation. - */ - if (!drm_dev_enter(dev, &dev_index)) - return; pm_runtime_get_sync(dev->dev); @@ -1145,15 +1114,14 @@ nouveau_drm_postclose(struct drm_device *dev, struct drm_file *fpriv) nouveau_abi16_fini(cli->abi16); mutex_unlock(&cli->mutex); - mutex_lock(&drm->clients_lock); + mutex_lock(&drm->client.mutex); list_del(&cli->head); - mutex_unlock(&drm->clients_lock); + mutex_unlock(&drm->client.mutex); nouveau_cli_fini(cli); kfree(cli); pm_runtime_mark_last_busy(dev->dev); pm_runtime_put_autosuspend(dev->dev); - drm_dev_exit(dev_index); } static const struct drm_ioctl_desc diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 8b252dca0fc3..b8025507a9e4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -142,11 +142,6 @@ struct nouveau_drm { struct list_head clients; - /** - * @clients_lock: Protects access to the @clients list of &struct nouveau_cli. - */ - struct mutex clients_lock; - u8 old_pm_cap; struct { diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig index 6153972e0127..b9dbedf8f15e 100644 --- a/drivers/gpu/drm/panel/Kconfig +++ b/drivers/gpu/drm/panel/Kconfig @@ -233,7 +233,6 @@ config DRM_PANEL_OLIMEX_LCD_OLINUXINO depends on OF depends on I2C depends on BACKLIGHT_CLASS_DEVICE - select CRC32 help The panel is used with different sizes LCDs, from 480x272 to 1280x800, and 24 bit per pixel. diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c index 534dd7414d42..0145129d7c66 100644 --- a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c +++ b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c @@ -590,14 +590,14 @@ static const struct drm_display_mode k101_im2byl02_default_mode = { .clock = 69700, .hdisplay = 800, - .hsync_start = 800 + 52, - .hsync_end = 800 + 52 + 8, - .htotal = 800 + 52 + 8 + 48, + .hsync_start = 800 + 6, + .hsync_end = 800 + 6 + 15, + .htotal = 800 + 6 + 15 + 16, .vdisplay = 1280, - .vsync_start = 1280 + 16, - .vsync_end = 1280 + 16 + 6, - .vtotal = 1280 + 16 + 6 + 15, + .vsync_start = 1280 + 8, + .vsync_end = 1280 + 8 + 48, + .vtotal = 1280 + 8 + 48 + 52, .width_mm = 135, .height_mm = 217, diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 1a87cc445b5e..d6c0bbc03c6d 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -30,6 +30,7 @@ #include #include